Docker-in-Docker (DinD) capabilities of public runners deactivated. More info

Commit 684ef0d9 authored by Maude Le Jeune's avatar Maude Le Jeune
Browse files

depend copy files, included in hash, version output

parent f28cc575
......@@ -50,8 +50,9 @@ I see at least before three projects to complete before making the first release
- [X] Accept that the segment can depend from external files specified as:
#depend $SOMEPATHFROMBASHENV/myfile.pars $SOMEPATHFROMBASHENV/myfile2.pars
This is working in my v1.1
- [ ] Those files should enter in the hash
- [ ] Those files should be stored
- [X] Those files should enter in the hash
- [X] Those files should be stored
- [X] Output module version information
* In my branch the only remaining working mode is debug
- [X] It seems that manager is not able to serialize the scheduler object anymore
When trying to pickle the s object I am answered that
......
......@@ -429,6 +429,21 @@ class Environment(EnvironmentBase):
return
self.save_products(self._worker.pipe.get_param_file(seg),glo, param_name=param_name)
def _save_version(self, glo):
""" Save version information for all modules
Parameters
----------
glo : dict, the global dictionnary
"""
import types
version = {}
for k,v in glo.items():
if isinstance(v, types.ModuleType):
if hasattr(v, '__version__'):
version[k] = v.__version__
return version
def _close(self, glo):
""" Close environment.
......@@ -451,7 +466,8 @@ class Environment(EnvironmentBase):
except Exception:
self.logger.warning( 'Fail to save the param file for seg %s' % seg)
self._worker.task.param = self._make_tag(seg, glo)
self._worker.task.param = self._make_tag(seg, glo)
self._worker.task.version = self._save_version(glo)
try:
res = glo["seg_output"]
except:
......
......@@ -113,10 +113,11 @@ class Repository:
## deps
depend = self.get_directive(Depend, s)
self._deps[s] = ""
self._deps[s] = {}
for fn in depend.deps:
source = self.get_file_content(fn)
self._deps[s] += source
self._deps[s][fn] = source
self._all_string[s] +=source
def get_all_string(self, seg):
""" Return all code as a string.
......@@ -190,7 +191,7 @@ class Repository:
"""
return self._code[seg]
def get_deps_string(self, dep):
def get_deps_string(self, seg):
""" Return external dependency code as a string.
This is done to prevent file changes after hashkey computation.
......
......@@ -188,6 +188,10 @@ class Scheduler():
d['param'] = self.products_list._list[seg][-1].param
except:
logger.info("no tag saved for seg %s"%seg)
try:
d['version'] = self.products_list._list[seg][-1].version
except:
logger.info("no version saved for seg %s"%seg)
with closing(file(fn, 'w')) as f:
r = pickle.dump(d,f)
......@@ -227,6 +231,12 @@ class Scheduler():
fid = open(dest, "w")
fid.write(f[0])
fid.close()
for dk, dv in r.get_deps_string(seg).items():
if dk:
dest = d+'/'+os.path.basename(dk)
fid = open(dest, "w")
fid.write(dv)
fid.close()
parents = self.pipe.get_parents(seg) ## parents segments
d = self.tracker.get_done(seg) ## done tasks
......
......@@ -58,8 +58,10 @@ class Task:
self.ended_on = None
## List of the task id whose output become the input of this task
self.parents = parents
## List of str_input of the parent tasks
## Dict of str_input of the parent tasks
self.str_parents = {}
## Dict of dependency module version
self.version = {}
def __str__(self):
......
#multiplex cross_prod group_by "0"
import numpy
print seg_input
#depend /home/lejeune/rien.txt
import os
f = glob_seg("a", "file*.dat")
for file in f:
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment