Commit 121ed6de authored by Maude Le Jeune's avatar Maude Le Jeune
Browse files

+ doc env

parent 9808f95e
......@@ -25,7 +25,7 @@ import sys, os
# Add any Sphinx extension module names here, as strings. They can be extensions
# coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
autoclass_content = 'both'
#autoclass_content = 'both'
extensions = ['sphinx.ext.autodoc', 'sphinx.ext.doctest', 'sphinx.ext.todo', 'sphinx.ext.coverage', 'sphinx.ext.pngmath', 'sphinx.ext.viewcode']
# Add any paths that contain templates here, relative to this directory.
......
......@@ -33,11 +33,7 @@ Both engine and web interface are written in Python.
Tutorial <tutorial>
Advanced usage <userguide>
API <pipelet>
Indices and tables
==================
......
......@@ -400,6 +400,8 @@ The segment code is executed in a specific environment that provides:
``hook(hookname, globals())``: execute Python script ``segname_hookname.py`` and update the namespace.
See the section :ref:`Hooking system<hook-sec>` for more details.
See the API documentation :py:mod:`environment <environment>` to get more details on those functionalities
Writing a first pipeline
************************
......
......@@ -21,16 +21,3 @@ from task import *
from environment import *
__version__='374f31751fd8422724cd7d9b8dcaa890d7ef21e3'
__version__='374f31751fd8422724cd7d9b8dcaa890d7ef21e3'
__version__='374f31751fd8422724cd7d9b8dcaa890d7ef21e3'
__version__='374f31751fd8422724cd7d9b8dcaa890d7ef21e3'
__version__='374f31751fd8422724cd7d9b8dcaa890d7ef21e3'
__version__='374f31751fd8422724cd7d9b8dcaa890d7ef21e3'
__version__='374f31751fd8422724cd7d9b8dcaa890d7ef21e3'
......@@ -135,13 +135,8 @@ class EnvironmentBase():
class Environment(EnvironmentBase):
""" Default segment's facilities.
The default environment contains some basic utilities related to :
- filename
- parameters
- sub process
The default environment can be extended to complete the
default environment with new utilities.
The default environment contains some basic utilities related to filenames, parameters and subprocesses.
It can be extended to complete the default environment with new utilities.
"""
......@@ -183,14 +178,9 @@ class Environment(EnvironmentBase):
If no segment specified, take the first. This utility replaces
the seg_input variable which type could vary as described
below.
Parameters
----------
seg: string segment name
Returns
-------
python object or None
>>> get_input(seg='third')
the_brave
"""
inp = None
if isinstance(self.seg_input, dict):
......@@ -210,31 +200,22 @@ class Environment(EnvironmentBase):
inp= self.seg_input
return inp
def set_output(self, o):
def set_output(self, lst_o):
""" Set the segment output as a list.
If o is not a list, set a list of one element o.
Parameters
----------
o : list or arbitrary python object
"""
if not isinstance (o, list):
self.seg_output=[o]
if not isinstance (lst_o, list):
self.seg_output=[lst_o]
else:
self.seg_output=o
self.seg_output=lst_o
def get_data_fn(self, x, seg=None):
""" Complete the filename with the path to the working
directory.
Parameters
----------
x: string, filename suffix
Returns
-------
string, filename
>>> get_data_fn ("test.png")
./test/first_test/first_S3CRKVY/data/test.png
"""
return self._get_data_fn(x, seg)
......@@ -245,11 +226,6 @@ class Environment(EnvironmentBase):
Search for an extra segment code file, and update dictionnary
with the result of its execution.
Parameters
----------
hook_name: string, hook name
glo: dict, global dictionnary to update.
"""
self.logger.info ("hooking %s"%hook_name)
return self._hook(hook_name, glo)
......@@ -257,27 +233,17 @@ class Environment(EnvironmentBase):
def get_parent(self):
""" Return list of parent names
Returns
-------
Return the list of parent names.
>>> get_parent()
['second', 'third']
"""
str_parents = self._worker.task.str_parents
return str_parents.keys()
def glob_parent(self, y, segs=None):
""" Globbing limited to the fatherhood
For unlimited globbing see glob_seg.
Parameters
----------
y: string, globbing pattern
segs: List of segment to glob in
""" Globbing limited to the fatherhood. For unlimited globbing see glob_seg.
Returns
-------
Return the list of filename matching the pattern y in the
data directory of parent tasks.
>>> glob_parent ('test*.txt')
['./test/first_test/first_S3CRKVY/second_WDSDP3Q/data/7_Robin/test.txt']
"""
str_parents = self._worker.task.str_parents
if segs is None:
......@@ -299,21 +265,14 @@ class Environment(EnvironmentBase):
return res
def glob_seg(self, x, y):
""" Return the list of filename matching y in the working
directory of segment x.
""" Return the list of filename matching y in the working directory of segment x.
Usage of glob_seg should be limited:
- potentially breaks the dependancy scheme
- May hurt performances as all task directories of the segment x will be searched
+ potentially breaks the dependancy scheme
+ may hurt performances as all task directories of the segment x will be searched
Parameters
----------
x: string, segment name
y: string, regexp of file to glob.
Returns
-------
list of filenames.
>>> glob_seg ('second', 'test*.txt')
['./test/first_test/first_S3CRKVY/second_WDSDP3Q/data/8_Arthur/test.txt', './test/first_test/first_S3CRKVY/second_WDSDP3Q/data/7_Robin/test.txt']
"""
segx = self._worker.pipe.find_seg(self._worker.task.seg, x)
......@@ -326,17 +285,11 @@ class Environment(EnvironmentBase):
def logged_subprocess(self, args, shell=False, except_on_failure=True, name=None):
""" Execute a subprocess and log its output.
Create files process_name.log and process_name.err
Create files process_name.log and process_name.err. If shell is set to True, the command is run via a shell. If except_on_failure set to True, raise an exception when command execution return non zero
Parameters
----------
args: a Task argument
shell: If set to True, the command is run via a shell
except_on_failure: If set to True, raise an exception when command execution return non zero
Returns
-------
tuple (outputfile, errorfile)
>>> (o,e)=logged_subprocess (["echo", "$TMP"])
>>> open(o).readlines()
['$BASH']
"""
proc = args[0]
if name is None:
......@@ -362,10 +315,9 @@ class Environment(EnvironmentBase):
Note : has to be part of the segment execution environment
The temporary file is added to the intern list for future removal.
Returns
-------
string, temporary filename.
>>> get_tmp_fn()
./test/first_test/third_QA7CX5A/fourth_LGJSKVQ/735200
"""
tf = path.join(self._worker.work_dir, str(datetime.now().toordinal()))
self._tmpf.append(tf)
......@@ -376,11 +328,9 @@ class Environment(EnvironmentBase):
""" Update the global dictionnary with parameters of a
segment.
Parameters
----------
seg : string, segment name
glo : dict, the global dictionnary
param_name : string list, parameters name.
>>> load_param('second', globals(), ["fn"])
>>> print fn
test.txt
"""
segx = self._worker.pipe.find_seg(self._worker.task.seg, seg)
if segx is None:
......@@ -393,12 +343,9 @@ class Environment(EnvironmentBase):
def save_products(self, filename, glo, param_name='*'):
""" Use pickle to save a part of a given namespace.
Parameters
----------
filename : string, save in this filename.
glo : dict, the namespace.
param_name : string list, name of objects to save. If '*' save
everything in the dictionnary.
>>> save_products(get_data_fn("test.pkl"), globals(),['i'])
>>> open(get_data_fn("test.pkl")).readlines()
['(dp1\n', "S'i'\n", 'I0\n', 's.']
"""
new_dict = {}
if param_name == '*':
......@@ -416,12 +363,9 @@ class Environment(EnvironmentBase):
""" Update a namespace by unpickling requested object from the
file.
Parameters
----------
filename: string, the pickle filename.
glo : dict, the namespace to update.
param_name : string list, names of object to unpickle. If '*',
everything from the file is loaded.
>>> load_products(get_data_fn("test.pkl", seg="second"), globals(),['i'])
>>> print i
0
"""
try:
f = file(filename)
......@@ -452,14 +396,6 @@ class Environment(EnvironmentBase):
def _make_tag(self, seg, glo):
""" Write a tag using parameters of a segment.
Parameters
----------
seg : string, segment name
glo : dict, the global dictionnary
Returns
-------
string.
"""
strtag = ''
if len(self.lst_tag)==0:
......@@ -479,14 +415,6 @@ class Environment(EnvironmentBase):
def _save_param(self, glo,param_name='*'):
""" Save parameters of a segment.
Parameters
----------
glo : dict, the global dictionnary
param_name : string list, parameters name.
Returns
-------
dictionnary
"""
new_dict = {}
if param_name == '*':
......@@ -501,14 +429,6 @@ class Environment(EnvironmentBase):
def _save_version(self, glo):
""" Save version information for all modules
Parameters
----------
glo : dict, the global dictionnary
Returns
-------
dictionnary
"""
import types
for k,v in glo.items():
......@@ -525,10 +445,6 @@ class Environment(EnvironmentBase):
def save_param (self, lst_par):
""" Set list of parameter to save on disk
Parameters
----------
lst_par: list of string
"""
if not isinstance(lst_par, list):
......@@ -537,10 +453,6 @@ class Environment(EnvironmentBase):
def expose (self, lst_tag):
""" Set list of parameter to expose from the web interface.
Parameters
----------
lst_tag: list of string
"""
if not isinstance(lst_tag, list):
lst_tag = [lst_tag]
......@@ -551,11 +463,6 @@ class Environment(EnvironmentBase):
This routine performs post processing like saving tag,
parameters, etc.
Parameters
----------
glo : dict, the global dictionnary
seg : string, segment name
"""
seg = self._worker.task.seg
self.clean_tmp()
......@@ -598,9 +505,6 @@ class SandBoxEnv(Environment):
Note : has to be part of the segment execution environment
The temporary file is added to the intern list for future removal.
Returns
-------
string, temporary filename.
"""
tf = os.tmpnam()
self._tmpf.append(tf)
......@@ -609,9 +513,6 @@ class SandBoxEnv(Environment):
def __init__(self, w):
""" Initialize the base environment with task input.
Parameters
----------
w: a worker instance
"""
self._worker = w
self.seg_input = w.task.task_input
......
......@@ -61,8 +61,6 @@ class Repository:
If permissive is False raise an error when segment script is
not found. Otherwise default to an empty script
**Exemples:**
>>> r = Repository([], permissive=True)
"""
......@@ -80,8 +78,6 @@ class Repository:
def _match_fn(fns, seg, hook=None):
""" Find filename matching pipelet rules for a given segment.
**Exemple**
>>> print Repository._match_fn( ['first.py'], 'first111')
first.py
"""
......@@ -156,8 +152,6 @@ class Repository:
def get_directive(self, Direct, seg):
""" Initialize a directive object from segment seg.
**Exemple:**
>>> r = LocalRepository(['fourth'], "./test/first_test")
>>> print r.get_directive (Multiplex, 'fourth').__class__
pipelet.directive.Multiplex
......@@ -180,8 +174,6 @@ class Repository:
def _get_hook_list(code):
""" Return the list of hooks by parsing the code source.
**Exemple:**
>>> print Repository._get_hook_list ("a=1\nhook('thehook',globals())\n")
['thehook']
"""
......@@ -228,8 +220,6 @@ class Repository:
This is the default to read dependencies of a segment.
**Exemple:**
>>> print Repository.get_file_content ('./test/first_test/third.py')[0:17]
adj = 'the_brave'
"""
......@@ -245,17 +235,13 @@ class Repository:
class LocalRepository(Repository):
""" A local repository.
A local repository is made of a main directory where to find
segment's code file.
"""A local repository is made of a main directory where to find segment's
code file.
"""
def __init__(self, lst_seg, src_path, permissive=False):
""" Initialize a local repository.
**Exemples:**
>>> r = LocalRepository(['first','second'], "./test/first_test", permissive=True)
>>> print len(r._code)
2
......@@ -277,8 +263,6 @@ class LocalRepository(Repository):
def get_code_source(filename):
""" Read source code from file
**Exemple:**
>>> print LocalRepository.get_code_source ('./test/first_test/third.py')[0:17]
adj = 'the_brave'
"""
......@@ -292,8 +276,6 @@ class LocalRepository(Repository):
def get_fns(self):
""" Return filename candidates.
**Exemple:**
>>> r = LocalRepository(['first','second'], "./test/first_test", permissive=True)
>>> print r.get_fns()
['./test/first_test/second.py', './test/first_test/fourth.py', './test/first_test/third.py', './test/first_test/main.py', './test/first_test/first.py']
......
import os
set_output(['Lancelot', 'Galahad', 'Robin', 'Arthur'])
seg_output = ['Lancelot', 'Galahad', 'Robin', 'Arthur']
number_of_knights = len(seg_output)
expose(['number_of_knights'])
tf = get_data_fn('test.txt')
f = file(tf,'w')
f.write('This segment launch %d seg_second' % number_of_knights)
f.close
os.mkdir(os.path.join(get_data_fn(""), "elnod"))
os.mkdir(os.path.join(get_data_fn(""), "ces"))
mmm=1
#multiplex cross_prod group_by "0"
##multiplex cross_prod group_by "0"
p = glob_seg('third', 'Preambule.txt')
t = glob_seg('second','test.txt')
import subprocess
......@@ -7,3 +7,4 @@ f = file(get_data_fn('result.txt'),'w')
print (p,t)
res = subprocess.Popen(['cat']+p+t,stdout=f).communicate()[0]
f.close()
......@@ -8,6 +8,5 @@ third -> fourth;
"""
P = pipeline.Pipeline(pipedot, code_dir='./', prefix='./')
P.push(first=[("2012-05-05","3x3"), ("2012-05-06","3x3"), ("2012-05-07","3x3")])
w, t = launch_interactive(P)
w.run()
#multiplex cross_prod where 'first in ["Robin", "Arthur"]'
tf = get_data_fn('test.txt')
fn='test.txt'
tf = get_data_fn(fn)
save_param(["fn"])
# Dude makes some kind of heavy computation
i = 0
product = get_input("first")
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment