Commit aa6582e4 authored by Maude Le Jeune's avatar Maude Le Jeune
Browse files

Feature #872 finalized : readme, test and code in agrement for environment

parent 4a730795
......@@ -210,7 +210,6 @@ The segment code is executed in a specific environment that provides:
1. access to the segment input and output
- seg_input: this variable is a dictionnary containing the input of the segment
- get_input():
- seg_output: this variable has to be set to a list containing the
2. Functionnalities to use the automated hierarchical data storage system.
......@@ -219,17 +218,17 @@ The segment code is executed in a specific environment that provides:
- get_tmp_fn(): return a temporary filename.
3. Functionnalities to use the automated parameters handling
- var_key: list of parameter names of the segment
- var_tag: list of parameter names which will be made visible from the web interface
- load_param(seg, var_names)
- lst_par: list of parameter names of the segment
- lst_tag: list of parameter names which will be made visible from the web interface
- load_param(seg, lst_par)
4. Various convenient functionalities
- save_products(filename=', var_names='*'): use pickle to save a
- save_products(filename=', lst_par='*'): use pickle to save a
part of a given namespace.
- load_products(filename, var_names): update the namespace by
- load_products(filename, lst_par): update the namespace by
unpickling requested object from the file.
- logged_subprocess(lst_args): execute a subprocess and log its output.
- log is a standard logging.Logger object that can be used to log the processing
- logger is a standard logging.Logger object that can be used to log the processing
5. Hooking support
Pipelet enables you to write reusable generic
......
......@@ -53,7 +53,6 @@ class EnvironmentBase():
w: a worker instance
"""
self._worker = w
self._segment_args = w.task.task_input
def _get_data_fn(self, x):
""" Complete the filename with the path to the working
......@@ -89,7 +88,7 @@ class EnvironmentBase():
""" Return the log file name
"""
return self._get_data_fn("segment_error.log")
return self._get_data_fn("seg_"+self._worker.task.seg+".log")
def _hook(self, hook_name, glo):
""" Execute hook code.
......@@ -132,7 +131,7 @@ class Environment(EnvironmentBase):
w: a worker instance
"""
self._worker = w
self.segment_input = w.task.task_input
self.seg_input = w.task.task_input
## list of temporary files
self._tmpf = []
self.logger = init_logger (self._get_data_fn(""), self._get_log_file(), level=[])
......@@ -165,9 +164,11 @@ class Environment(EnvironmentBase):
glo: dict, global dictionnary to update.
"""
self.logger.info ("hooking %s"%hook_name)
return self._hook(hook_name, glo)
def glob_seg(self, x, y):
""" Return the list of filename matching y in the working
directory of segment x.
......@@ -181,7 +182,10 @@ class Environment(EnvironmentBase):
-------
list of filenames.
"""
segx = self._worker.pipe.find_seg(self._worker.task.seg, x)
if segx is None:
self.logger.warning("No parent segment matching %s found"%x)
return glob(path.join(self._worker.pipe.get_data_dir(segx),y))\
+glob(path.join(self._worker.pipe.get_data_dir(segx),path.join('*/',y)))
......@@ -238,6 +242,8 @@ class Environment(EnvironmentBase):
param_name : string list, parameters name.
"""
segx = self._worker.pipe.find_seg(self._worker.task.seg, seg)
if segx is None:
self.logger.warning("No parent segment matching %s found"%seg)
if not path.exists(self._worker.pipe.get_param_file(segx)):
return
self.load_products(self._worker.pipe.get_param_file(segx), glo, param_name=param_name)
......@@ -260,7 +266,7 @@ class Environment(EnvironmentBase):
try:
new_dict[k] = glo[k]
except KeyError:
logger.warning( 'Fail to save object %s in file %s'%(k,filename))
logger.warning('Fail to save object %s in file %s'%(k,filename))
f = file(filename,'w')
pickle.dump(new_dict,f)
f.close()
......@@ -281,7 +287,7 @@ class Environment(EnvironmentBase):
new_dict = pickle.load(f)
f.close()
except IOError:
logger.warning( 'No such file: %s'%filename)
logger.warning('No such file: %s'%filename)
except UnpicklingError:
logger.warning('Failed to unpickle from file: %s'%filename)
f.close()
......@@ -302,7 +308,7 @@ class Environment(EnvironmentBase):
self._tmpf = []
def make_tag(self, seg, glo):
def _make_tag(self, seg, glo):
""" Write a tag using parameters of a segment.
Parameters
......@@ -315,7 +321,7 @@ class Environment(EnvironmentBase):
string.
"""
try:
var_tag = glo['var_tag']
var_tag = glo['lst_tag']
except KeyError:
var_tag = []
strtag = ''
......@@ -334,7 +340,7 @@ class Environment(EnvironmentBase):
return var_tag
def save_param(self, seg, glo,param_name='*'):
def _save_param(self, seg, glo,param_name='*'):
""" Save parameters of a segment.
Parameters
......@@ -356,21 +362,22 @@ class Environment(EnvironmentBase):
Parameters
----------
glo :
glo : dict, the global dictionnary
seg : string, segment name
"""
self.clean_tmp()
try: # Save params
var_key = glo['var_key']
self.save_param(seg, glo, param_name=var_key)
self.logger.info ('Save param file')
var_key = glo['lst_par']
self._save_param(seg, glo, param_name=var_key)
except KeyError:
self.logger.warning( 'Nothing to save in param file for seg %s' % seg)
except Exception:
self.logger.warning( 'Fail to save the param file for seg %s' % seg)
self._worker.task.param = self.make_tag(seg, glo)
self._worker.task.param = self._make_tag(seg, glo)
try:
res = glo["res"]
res = glo["seg_output"]
except:
res = None
self.logger.info("No segment output found, setting seg_output to None")
self._worker.task.task_output = res
n = 10
var_key=['n']
var_tag=['n']
lst_par=['n']
lst_tag=['n']
tf = get_data_fn('test.txt')
f = file(tf,'w')
f.write('This segment launch %d seg_second' % n)
f.close
res=range(n)
seg_output=range(n)
tf = get_data_fn('test.txt')
# Dude makes some kind of heavy computation
i = 0
product = segment_input[0]
product = seg_input[0]
f = file(tf,'w')
f.write('This segment received %d as arg'%product)
f.close
......
pre_line = 'This is the preambule'
var_key = ['pre_line']
var_tag = var_key
lst_par = ['pre_line']
lst_tag = lst_par
l = 2010
tf = get_data_fn('Preambule.txt')
f = file(tf,'w')
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment