Commit 2c9e893e authored by Betoule Marc's avatar Betoule Marc
Browse files

Merge branch 'master' of gitorious.org:pipelet/pipelet

Conflicts:
	pipelet/__init__.py
	pipelet/utils.py
parents e97ec821 1b6d56b9
......@@ -20,216 +20,3 @@ from pipeline import *
from task import *
from environment import *
__version__='4206ee5d2fb7a638977cef0b4bae42029f0248ae'
__version__='4206ee5d2fb7a638977cef0b4bae42029f0248ae'
__version__='2a761817b51d950aab5ce993735cad68eb2ec245'
__version__='2a761817b51d950aab5ce993735cad68eb2ec245'
__version__='2a761817b51d950aab5ce993735cad68eb2ec245'
__version__='2a761817b51d950aab5ce993735cad68eb2ec245'
__version__='2a761817b51d950aab5ce993735cad68eb2ec245'
__version__='3055d7cfe19886fffd26d2507d175e08f9d037ba'
__version__='3055d7cfe19886fffd26d2507d175e08f9d037ba'
__version__='3055d7cfe19886fffd26d2507d175e08f9d037ba'
__version__='3055d7cfe19886fffd26d2507d175e08f9d037ba'
__version__='d36778793e30ddec71bb22a9d3935b34ae9fffaf'
__version__='d36778793e30ddec71bb22a9d3935b34ae9fffaf'
__version__='0ee7c10afe6fdbca66624c03ca58a6172c7a47c5'
__version__='0ee7c10afe6fdbca66624c03ca58a6172c7a47c5'
__version__='0ee7c10afe6fdbca66624c03ca58a6172c7a47c5'
__version__='0ee7c10afe6fdbca66624c03ca58a6172c7a47c5'
__version__='0ee7c10afe6fdbca66624c03ca58a6172c7a47c5'
__version__='0ee7c10afe6fdbca66624c03ca58a6172c7a47c5'
__version__='0ee7c10afe6fdbca66624c03ca58a6172c7a47c5'
__version__='0ee7c10afe6fdbca66624c03ca58a6172c7a47c5'
__version__='0ee7c10afe6fdbca66624c03ca58a6172c7a47c5'
__version__='0ee7c10afe6fdbca66624c03ca58a6172c7a47c5'
__version__='0ee7c10afe6fdbca66624c03ca58a6172c7a47c5'
__version__='0ee7c10afe6fdbca66624c03ca58a6172c7a47c5'
__version__='0ee7c10afe6fdbca66624c03ca58a6172c7a47c5'
__version__='0ee7c10afe6fdbca66624c03ca58a6172c7a47c5'
__version__='0ee7c10afe6fdbca66624c03ca58a6172c7a47c5'
__version__='0ee7c10afe6fdbca66624c03ca58a6172c7a47c5'
__version__='0ee7c10afe6fdbca66624c03ca58a6172c7a47c5'
__version__='0ee7c10afe6fdbca66624c03ca58a6172c7a47c5'
__version__='0ee7c10afe6fdbca66624c03ca58a6172c7a47c5'
__version__='0ee7c10afe6fdbca66624c03ca58a6172c7a47c5'
__version__='0ee7c10afe6fdbca66624c03ca58a6172c7a47c5'
__version__='0ee7c10afe6fdbca66624c03ca58a6172c7a47c5'
__version__='0ee7c10afe6fdbca66624c03ca58a6172c7a47c5'
__version__='0ee7c10afe6fdbca66624c03ca58a6172c7a47c5'
__version__='0ee7c10afe6fdbca66624c03ca58a6172c7a47c5'
__version__='0ee7c10afe6fdbca66624c03ca58a6172c7a47c5'
__version__='0ee7c10afe6fdbca66624c03ca58a6172c7a47c5'
__version__='0ee7c10afe6fdbca66624c03ca58a6172c7a47c5'
__version__='0ee7c10afe6fdbca66624c03ca58a6172c7a47c5'
__version__='0ee7c10afe6fdbca66624c03ca58a6172c7a47c5'
__version__='0ee7c10afe6fdbca66624c03ca58a6172c7a47c5'
__version__='0ee7c10afe6fdbca66624c03ca58a6172c7a47c5'
__version__='0ee7c10afe6fdbca66624c03ca58a6172c7a47c5'
__version__='0ee7c10afe6fdbca66624c03ca58a6172c7a47c5'
__version__='0ee7c10afe6fdbca66624c03ca58a6172c7a47c5'
__version__='0ee7c10afe6fdbca66624c03ca58a6172c7a47c5'
__version__='0ee7c10afe6fdbca66624c03ca58a6172c7a47c5'
__version__='0ee7c10afe6fdbca66624c03ca58a6172c7a47c5'
__version__='0ee7c10afe6fdbca66624c03ca58a6172c7a47c5'
__version__='0ee7c10afe6fdbca66624c03ca58a6172c7a47c5'
__version__='0ee7c10afe6fdbca66624c03ca58a6172c7a47c5'
__version__='0ee7c10afe6fdbca66624c03ca58a6172c7a47c5'
__version__='0ee7c10afe6fdbca66624c03ca58a6172c7a47c5'
__version__='0ee7c10afe6fdbca66624c03ca58a6172c7a47c5'
__version__='0ee7c10afe6fdbca66624c03ca58a6172c7a47c5'
__version__='0ee7c10afe6fdbca66624c03ca58a6172c7a47c5'
__version__='0ee7c10afe6fdbca66624c03ca58a6172c7a47c5'
__version__='0ee7c10afe6fdbca66624c03ca58a6172c7a47c5'
__version__='0ee7c10afe6fdbca66624c03ca58a6172c7a47c5'
__version__='0ee7c10afe6fdbca66624c03ca58a6172c7a47c5'
__version__='0ee7c10afe6fdbca66624c03ca58a6172c7a47c5'
__version__='0ee7c10afe6fdbca66624c03ca58a6172c7a47c5'
__version__='0ee7c10afe6fdbca66624c03ca58a6172c7a47c5'
__version__='0ee7c10afe6fdbca66624c03ca58a6172c7a47c5'
__version__='0ee7c10afe6fdbca66624c03ca58a6172c7a47c5'
__version__='0ee7c10afe6fdbca66624c03ca58a6172c7a47c5'
__version__='0ee7c10afe6fdbca66624c03ca58a6172c7a47c5'
__version__='0ee7c10afe6fdbca66624c03ca58a6172c7a47c5'
__version__='0ee7c10afe6fdbca66624c03ca58a6172c7a47c5'
__version__='0ee7c10afe6fdbca66624c03ca58a6172c7a47c5'
__version__='0ee7c10afe6fdbca66624c03ca58a6172c7a47c5'
__version__='0ee7c10afe6fdbca66624c03ca58a6172c7a47c5'
__version__='0ee7c10afe6fdbca66624c03ca58a6172c7a47c5'
__version__='0ee7c10afe6fdbca66624c03ca58a6172c7a47c5'
__version__='0ee7c10afe6fdbca66624c03ca58a6172c7a47c5'
__version__='0ee7c10afe6fdbca66624c03ca58a6172c7a47c5'
__version__='0ee7c10afe6fdbca66624c03ca58a6172c7a47c5'
__version__='0ee7c10afe6fdbca66624c03ca58a6172c7a47c5'
__version__='0ee7c10afe6fdbca66624c03ca58a6172c7a47c5'
__version__='0ee7c10afe6fdbca66624c03ca58a6172c7a47c5'
__version__='0ee7c10afe6fdbca66624c03ca58a6172c7a47c5'
__version__='0ee7c10afe6fdbca66624c03ca58a6172c7a47c5'
__version__='0ee7c10afe6fdbca66624c03ca58a6172c7a47c5'
__version__='0ee7c10afe6fdbca66624c03ca58a6172c7a47c5'
__version__='0ee7c10afe6fdbca66624c03ca58a6172c7a47c5'
__version__='0ee7c10afe6fdbca66624c03ca58a6172c7a47c5'
__version__='0ee7c10afe6fdbca66624c03ca58a6172c7a47c5'
__version__='0ee7c10afe6fdbca66624c03ca58a6172c7a47c5'
__version__='0ee7c10afe6fdbca66624c03ca58a6172c7a47c5'
__version__='0ee7c10afe6fdbca66624c03ca58a6172c7a47c5'
__version__='0ee7c10afe6fdbca66624c03ca58a6172c7a47c5'
__version__='0ee7c10afe6fdbca66624c03ca58a6172c7a47c5'
__version__='0ee7c10afe6fdbca66624c03ca58a6172c7a47c5'
__version__='0ee7c10afe6fdbca66624c03ca58a6172c7a47c5'
__version__='0ee7c10afe6fdbca66624c03ca58a6172c7a47c5'
__version__='0ee7c10afe6fdbca66624c03ca58a6172c7a47c5'
__version__='0ee7c10afe6fdbca66624c03ca58a6172c7a47c5'
__version__='0ee7c10afe6fdbca66624c03ca58a6172c7a47c5'
__version__='0ee7c10afe6fdbca66624c03ca58a6172c7a47c5'
__version__='0ee7c10afe6fdbca66624c03ca58a6172c7a47c5'
__version__='0ee7c10afe6fdbca66624c03ca58a6172c7a47c5'
__version__='0ee7c10afe6fdbca66624c03ca58a6172c7a47c5'
__version__='0ee7c10afe6fdbca66624c03ca58a6172c7a47c5'
## Copyright (C) 2008, 2009, 2010 APC LPNHE CNRS Universite Paris Diderot <lejeune@apc.univ-paris7.fr> <betoule@apc.univ-paris7.fr>
# Copyright (C) 2008, 2009, 2010 APC LPNHE CNRS Universite Paris Diderot <lejeune@apc.univ-paris7.fr> <betoule@apc.univ-paris7.fr>
##
## This program is free software; you can redistribute it and/or modify
## it under the terms of the GNU General Public License as published by
......@@ -433,7 +433,13 @@ def rebuild_db_from_disk(pipedir, sqlfile=None):
seg_depend_cache = {} ## store parents for each curr_dir
for curr_dir in lst_dir:
curr_dir = path.abspath(curr_dir)
s = curr_dir.split("_")[-2].split("/")[-1]
#s = curr_dir.split("_")[-2].split("/")[-1] ## TODO replace with regexp -> done
#rec = re.search (r"/([a-zA-Z0-9]+)_([a-zA-Z0-9]_*)([a-zA-Z0-9]+)$", curr_dir)
gr = re.search (r"/([a-zA-Z0-9]+)(_[a-zA-Z0-9])*_([a-zA-Z0-9]+)$", curr_dir).groups()[:-1] # remove segkey
s = gr[0]
for g in gr[1:]:
if g:
s= s+g
print "Creating segment %s instance (%s)."%(s, curr_dir)
R = LocalRepository([s], curr_dir)
## read curr_dir/seg_s_code.py to get docstring
......@@ -481,10 +487,12 @@ def rebuild_db_from_disk(pipedir, sqlfile=None):
lst_task.append(path.abspath(path.join(root, d)))
break
if not lst_task:
lst_task.append(path.abspath(path.join(k, "data")))
lst_task.append(path.abspath(path.join(k, "data"))+"/")
for t in lst_task:
print "Creating task %s"%t
## read task propertie from meta file
fn = glob.glob(path.join(t, "*.meta"))
if fn:
......
import os
set_output(['Lancelot', 'Galahad', 'Robin', 'Arthur'])
number_of_knights = len(seg_output)
expose(['number_of_knights'])
tf = get_data_fn('test.txt')
f = file(tf,'w')
f.write('This segment launch %d seg_second' % number_of_knights)
f.close
os.mkdir(os.path.join(get_data_fn(""), "elnod"))
os.mkdir(os.path.join(get_data_fn(""), "ces"))
mmm=1
#multiplex cross_prod group_by "0"
p = glob_seg('third', 'Preambule.txt')
t = glob_seg('second','test.txt')
import subprocess
bla = 9
f = file(get_data_fn('result.txt'),'w')
print (p,t)
res = subprocess.Popen(['cat']+p+t,stdout=f).communicate()[0]
f.close()
import pipelet.pipeline as pipeline
from pipelet.launchers import launch_process, launch_interactive
import os
pipedot = """
first -> second_1 -> fourth;
third -> fourth;
"""
P = pipeline.Pipeline(pipedot, code_dir='./', prefix='./')
P.push(first=[("2012-05-05","3x3"), ("2012-05-06","3x3"), ("2012-05-07","3x3")])
w, t = launch_interactive(P)
w.run()
#multiplex cross_prod where 'first in ["Robin", "Arthur"]'
tf = get_data_fn('test.txt')
# Dude makes some kind of heavy computation
hook("thehook", globals())
i = 1
product = get_input("first")
f = file(tf,'w')
f.write('This segment received %s as arg'%product)
f.close
set_output(product)
#multiplex cross_prod where 'first in ["Robin", "Arthur"]'
tf = get_data_fn('test.txt')
# Dude makes some kind of heavy computation
hook("thehook", globals())
i = 1
product = get_input("first")
f = file(tf,'w')
f.write('This segment received %s as arg'%product)
f.close
set_output(product)
logger.info( "from the hook")
logger.info( "from the hook")
adj = 'the_brave'
expose(['adj'])
l = 2010
tf = get_data_fn('Preambule.txt')
f = file(tf,'w')
f.write(adj)
f.close
set_output(adj)
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment