db reconstruction updated

TODO.org

@@ -17,8 +17,8 @@ I see at least before three projects to complete before making the first release
 	- [X] problem for parents giving same str_input outside the
           special case of groud_by -> compute as many task as parents
           even if str_input is the same.  
-  - [ ] Does the tracking on disk allow to reconstruct the database
-  - [ ] I modified the task format to allow for easy glob_seg. It may have break other things (At least the database reconstruction)
+  - [X] Does the tracking on disk allow to reconstruct the database
+  - [X] I modified the task format to allow for easy glob_seg. It may have break other things (At least the database reconstruction)
   - [X] Is the treatment of redundant tasks resulting from group_by OK
   - [ ] The code of multiplex is hard to read/lacks for comments. Variable Names, data structure may be rethought.
 * We should bring the default environment to its final state:

pipelet/utils.py

@@ -392,11 +392,9 @@ def rebuild_db_from_disk(pipedir, sqlfile=None):
     seg_depend_cache = {}          ## store parents for each curr_dir
     for curr_dir in lst_dir:
         curr_dir = path.abspath(curr_dir) 
-        R        = LocalRepository(curr_dir) 
         s        = curr_dir.split("_")[-2].split("/")[-1]
-        #s        = curr_dir.split("_")[-2] ## seg name
         print "Creating segment %s instance (%s)."%(s, curr_dir)
-
+        R        = LocalRepository([s], curr_dir) 
         ## read  curr_dir/seg_s_code.py to get docstring
         try:
             docline = R.get_docline(s)
@@ -456,19 +454,18 @@ def rebuild_db_from_disk(pipedir, sqlfile=None):
             task_depend_cache[t] = meta['parents']
 
             ## insert values in db
+            task_id_cache[t] = t.split("_")[-2].split("/")[-1]
             input = None
-            c = conn.execute('insert into tasks (seg_id, output, input, begun_on, ended_on, queued_on,status,str_input) values (?, ?, ?, ?, ?, ?, ?, ?)',(v, pickle.dumps(meta["output"]), pickle.dumps(meta["input"]), meta["begun_on"], meta["ended_on"], meta["queued_on"], meta["status"],t))
+            c = conn.execute('insert into tasks (task_id, seg_id, output, input, begun_on, ended_on, queued_on,status,str_input) values (?, ?, ?, ?, ?, ?, ?, ?, ?)',(task_id_cache[t], v, pickle.dumps(meta["output"]), pickle.dumps(meta["input"]), meta["begun_on"], meta["ended_on"], meta["queued_on"], meta["status"],t))
 
-            ## retrieve the task_id
-            task_id_cache[t]  = c.lastrowid
-    
     ## 5---: task_relations from meta + task input 
     ## need to restart the loop in order to use the right task_id
     for k,v in task_id_cache.items():
         print "Storing connectivity for task %s."%k
-        for p in task_depend_cache[k]:
-            id = conn.execute('select task_id,output from tasks where str_input = ?',(p,)).fetchone()
-            conn.execute('insert into task_relations (father_id, child_id) values (?, ?)',(id[0], v)) 
+        for sp in task_depend_cache[k].values():
+            for p in sp: 
+                id = conn.execute('select task_id,output from tasks where str_input = ?',(p,)).fetchone()
+                conn.execute('insert into task_relations (father_id, child_id) values (?, ?)',(id[0], v)) 
 
     conn.commit()