depend copy files, included in hash, version output

TODO.org

@@ -50,8 +50,9 @@ I see at least before three projects to complete before making the first release
   - [X] Accept that the segment can depend from external files specified as:
 	#depend $SOMEPATHFROMBASHENV/myfile.pars $SOMEPATHFROMBASHENV/myfile2.pars
 	This is working in my v1.1 
-  - [ ] Those files should enter in the hash
-  - [ ] Those files should be stored
+  - [X] Those files should enter in the hash
+  - [X] Those files should be stored
+  - [X] Output module version information
 * In my branch the only remaining working mode is debug
   - [X] It seems that manager is not able to serialize the scheduler object anymore
 	When trying to pickle the s object I am answered that

pipelet/environment.py

@@ -429,6 +429,21 @@ class Environment(EnvironmentBase):
             return
         self.save_products(self._worker.pipe.get_param_file(seg),glo, param_name=param_name)
 
+    def _save_version(self, glo):
+        """ Save version information for all modules
+        
+        Parameters
+        ----------
+        glo : dict, the global dictionnary
+        
+        """
+        import types
+        version = {}
+        for k,v in glo.items():
+            if isinstance(v, types.ModuleType):
+                if hasattr(v, '__version__'):
+                    version[k] = v.__version__
+        return version
 
     def _close(self, glo):
         """ Close environment. 
@@ -451,7 +466,8 @@ class Environment(EnvironmentBase):
         except Exception:
             self.logger.warning( 'Fail to save the param file for seg %s' % seg)
 
-        self._worker.task.param = self._make_tag(seg, glo)
+        self._worker.task.param   = self._make_tag(seg, glo)
+        self._worker.task.version = self._save_version(glo)
         try:
             res = glo["seg_output"]
         except:

pipelet/repository.py

@@ -113,10 +113,11 @@ class Repository:
 
             ## deps
             depend = self.get_directive(Depend, s)
-            self._deps[s] = ""
+            self._deps[s] = {}
             for fn in depend.deps:
                 source = self.get_file_content(fn)
-                self._deps[s] += source
+                self._deps[s][fn] = source
+                self._all_string[s] +=source
 
     def get_all_string(self, seg):
         """ Return all code as a string. 
@@ -190,7 +191,7 @@ class Repository:
         """
         return self._code[seg]
 
-    def get_deps_string(self, dep):
+    def get_deps_string(self, seg):
         """ Return external dependency code as a string. 
         
         This is done to prevent file changes after hashkey computation. 

pipelet/scheduler.py

@@ -188,6 +188,10 @@ class Scheduler():
            d['param'] = self.products_list._list[seg][-1].param
        except:
            logger.info("no tag saved for seg %s"%seg)
+       try:
+           d['version'] = self.products_list._list[seg][-1].version
+       except:
+           logger.info("no version saved for seg %s"%seg)
        
        with closing(file(fn, 'w')) as f:
            r = pickle.dump(d,f)
@@ -227,6 +231,12 @@ class Scheduler():
                     fid = open(dest, "w")
                     fid.write(f[0])
                     fid.close()
+            for dk, dv in r.get_deps_string(seg).items():
+                if dk:
+                    dest = d+'/'+os.path.basename(dk)
+                    fid = open(dest, "w")
+                    fid.write(dv)
+                    fid.close()
 
         parents = self.pipe.get_parents(seg) ## parents segments
         d       = self.tracker.get_done(seg) ## done tasks

pipelet/task.py

@@ -58,8 +58,10 @@ class Task:
         self.ended_on = None
         ## List of the task id whose output become the input of this task
         self.parents = parents
-        ## List of str_input of the parent tasks
+        ## Dict of str_input of the parent tasks
         self.str_parents = {}
+        ## Dict of dependency module version
+        self.version = {}
 
 
     def __str__(self):

test/multiplex/c.py

 #multiplex cross_prod group_by "0"
-
+import numpy
 print seg_input
-
+#depend /home/lejeune/rien.txt 
 import os
 f = glob_seg("a", "file*.dat")
 for file in f: