Commit c5354a8b authored by Marc Betoule's avatar Marc Betoule
Browse files

Fix bugs in flatten, but compute_hash remain tbd

parent 3cf0588b
......@@ -62,7 +62,7 @@ git clone git://
sudo python install
*** Running the test pipeline
*** Running a simple test pipeline
1. Run the test pipeline
......@@ -297,6 +297,10 @@ The segment code is executed in a specific environment that provides:
hook (hookname, globals()): execute Python script ‘’ and update the namespace.
*** The exemple pipelines
**** fft
**** cmb
** Running Pipes
*** The interactive mode
This mode has been designed to ease debugging. If P is an instance of
......@@ -477,8 +481,7 @@ accordingly.
Pipeline(pipedot, codedir=, prefix=, env=MyEnvironment)
** Using custom dependency schemes
** Writing custom main files
** Launching pipeweb behind apache
Pipeweb use the cherrypy web framework server and can be run behind an
......@@ -488,6 +491,8 @@ apache webserver which brings essentially two advantages:
* The pipelet actors
This section document the code for developpers.
** The Repository object
** The Pipeline object
......@@ -363,18 +363,20 @@ class Pipeline:
S = [s for s, p in self._parents.iteritems() if not p]
# copy the graph
parents = self._parents.copy()
print parents
children = self._children.copy()
while S:
s = s.pop()
s = S.pop(0)
yield s
while children[s]:
c = children[s].pop()
c = children[s].pop(0)
if not parents[c]:
yield c
for s, p in parents.iteritems():
if p:
raise PipeException("Dependency graph has at least one cyle including segment"%s)
print parents
raise PipeException("Dependency graph has at least one cyle including segment %s"%s)
def _compute_hash(self, seg):
""" Compute the hashkey for a given segment.
......@@ -98,6 +98,7 @@ def main(print_info=print_info):
## Build pipeline instance
P = Pipeline(pipe_dot, code_dir=code_dir, prefix=prefix, matplotlib=True, matplotlib_interactive=True)
cmbin = []
for sim_id in sim_ids:
cmbin.append((nside, sim_id))
......@@ -113,9 +114,9 @@ def main(print_info=print_info):
launch_process(P, options.process,log_level=log_level)
## PBS mode
elif options.add_workers:
launch_pbs(P,options.add_workers , address=(os.environ['HOST'],50000), job_name=job_name, cpu_time=cpu_time, job_header=job_header)
launch_pbs(P,options.add_workers , address=(os.environ['HOSTNAME'],50000), job_name=job_name, cpu_time=cpu_time, job_header=job_header)
launch_pbs(P, 1, address=(os.environ['HOST'],50000), job_name=job_name, cpu_time=cpu_time, server=True, job_header=job_header, log_level=log_level)
launch_pbs(P, 1, address=(os.environ['HOSTNAME'],50000), job_name=job_name, cpu_time=cpu_time, server=True, job_header=job_header, log_level=log_level)
if print_info:
print "1- Run 'pipeweb track <shortname> %s' \n to add the pipe to the web interface.\n"%(P.sqlfile)
""" Demonstration of """
import pipelet.pipeline as pipeline
from pipelet.launchers import launch_process
import os
pipedot ="""a -> b -> c;
b-> d -> c;
e -> d;
P = pipeline.Pipeline(pipedot, code_dir='./', prefix='./')
print "the execution order will be:"
for s in P.flatten():
print s
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment