debug Bug #4855 rebuild data base

pipelet/__init__.py

@@ -21,7 +21,3 @@ from task import *
 from environment import *
 
 
-
-__version__='4206ee5d2fb7a638977cef0b4bae42029f0248ae'
-
-__version__='4206ee5d2fb7a638977cef0b4bae42029f0248ae'

pipelet/utils.py

-## Copyright (C) 2008, 2009, 2010 APC LPNHE CNRS Universite Paris Diderot <lejeune@apc.univ-paris7.fr>  <betoule@apc.univ-paris7.fr>
+# Copyright (C) 2008, 2009, 2010 APC LPNHE CNRS Universite Paris Diderot <lejeune@apc.univ-paris7.fr>  <betoule@apc.univ-paris7.fr>
 ## 
 ## This program is free software; you can redistribute it and/or modify
 ## it under the terms of the GNU General Public License as published by
@@ -418,7 +418,13 @@ def rebuild_db_from_disk(pipedir, sqlfile=None):
     seg_depend_cache = {}          ## store parents for each curr_dir
     for curr_dir in lst_dir:
         curr_dir = path.abspath(curr_dir) 
-        s        = curr_dir.split("_")[-2].split("/")[-1]
+        #s        = curr_dir.split("_")[-2].split("/")[-1] ## TODO replace with regexp -> done
+        #rec      = re.search (r"/([a-zA-Z0-9]+)_([a-zA-Z0-9]_*)([a-zA-Z0-9]+)$", curr_dir)
+        gr = re.search (r"/([a-zA-Z0-9]+)(_[a-zA-Z0-9])*_([a-zA-Z0-9]+)$", curr_dir).groups()[:-1] # remove segkey
+        s  = gr[0]
+        for g in gr[1:]:
+            if g:
+                s= s+g
         print "Creating segment %s instance (%s)."%(s, curr_dir)
         R        = LocalRepository([s], curr_dir) 
         ## read  curr_dir/seg_s_code.py to get docstring
@@ -466,10 +472,12 @@ def rebuild_db_from_disk(pipedir, sqlfile=None):
                 lst_task.append(path.abspath(path.join(root, d)))
             break
         if not lst_task:
-            lst_task.append(path.abspath(path.join(k, "data")))
+            lst_task.append(path.abspath(path.join(k, "data"))+"/")
 
         for t in lst_task:
 
+            print "Creating task %s"%t 
+
             ## read task propertie from meta file
             fn = glob.glob(path.join(t, "*.meta"))
             if fn:

test/rebuild_test/first.py

+import os
+set_output(['Lancelot', 'Galahad', 'Robin', 'Arthur'])
+number_of_knights = len(seg_output)
+expose(['number_of_knights'])
+tf = get_data_fn('test.txt')
+f = file(tf,'w')
+f.write('This segment launch %d seg_second' % number_of_knights)
+f.close
+os.mkdir(os.path.join(get_data_fn(""), "elnod"))
+os.mkdir(os.path.join(get_data_fn(""), "ces"))
+mmm=1

test/rebuild_test/fourth.py

+#multiplex cross_prod group_by "0"
+p = glob_seg('third', 'Preambule.txt')
+t = glob_seg('second','test.txt')
+import subprocess
+bla = 9
+f = file(get_data_fn('result.txt'),'w')
+print (p,t)
+res = subprocess.Popen(['cat']+p+t,stdout=f).communicate()[0]
+f.close()

test/rebuild_test/main.py

+import pipelet.pipeline as pipeline
+from pipelet.launchers import launch_process, launch_interactive
+import os
+
+pipedot = """
+first -> second_1 -> fourth;
+third -> fourth;
+"""
+
+P = pipeline.Pipeline(pipedot, code_dir='./', prefix='./')
+P.push(first=[("2012-05-05","3x3"), ("2012-05-06","3x3"), ("2012-05-07","3x3")])
+w, t = launch_interactive(P)
+w.run()

test/rebuild_test/second.py

+#multiplex cross_prod where 'first in ["Robin", "Arthur"]'
+tf = get_data_fn('test.txt')
+# Dude makes some kind of heavy computation
+hook("thehook", globals())
+i = 1
+product = get_input("first")
+f = file(tf,'w')
+f.write('This segment received %s as arg'%product)
+f.close
+set_output(product)
+

test/rebuild_test/second_1.py

+#multiplex cross_prod where 'first in ["Robin", "Arthur"]'
+tf = get_data_fn('test.txt')
+# Dude makes some kind of heavy computation
+hook("thehook", globals())
+i = 1
+product = get_input("first")
+f = file(tf,'w')
+f.write('This segment received %s as arg'%product)
+f.close
+set_output(product)
+

test/rebuild_test/second_1_thehook.py

+logger.info( "from the hook")

test/rebuild_test/second_thehook.py

+logger.info( "from the hook")

test/rebuild_test/third.py

+adj = 'the_brave'
+expose(['adj'])
+l = 2010
+tf = get_data_fn('Preambule.txt')
+f = file(tf,'w')
+f.write(adj)
+f.close
+set_output(adj)