Skip to content
GitLab
Projects
Groups
Snippets
Help
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Open sidebar
pipelet
Pipelet
Commits
de216678
Commit
de216678
authored
Nov 18, 2010
by
Marc Betoule
Browse files
Fix bugs in flatten, but compute_hash remain tbd
parent
f3f86b6c
Changes
9
Hide whitespace changes
Inline
Side-by-side
Showing
9 changed files
with
49 additions
and
9 deletions
+49
-9
README.org
README.org
+8
-3
pipelet/pipeline.py
pipelet/pipeline.py
+6
-4
test/cmb/main.py
test/cmb/main.py
+11
-2
test/complex_dependencies/a.py
test/complex_dependencies/a.py
+1
-0
test/complex_dependencies/b.py
test/complex_dependencies/b.py
+1
-0
test/complex_dependencies/c.py
test/complex_dependencies/c.py
+1
-0
test/complex_dependencies/d.py
test/complex_dependencies/d.py
+1
-0
test/complex_dependencies/e.py
test/complex_dependencies/e.py
+1
-0
test/complex_dependencies/main.py
test/complex_dependencies/main.py
+19
-0
No files found.
README.org
View file @
de216678
...
...
@@ -62,7 +62,7 @@ git clone git://gitorious.org/pipelet/pipelet.git
sudo python setup.py install
*** Running
th
e test pipeline
*** Running
a simpl
e test pipeline
1. Run the test pipeline
...
...
@@ -288,6 +288,10 @@ The segment code is executed in a specific environment that provides:
hook (hookname, globals()): execute Python script ‘seg_segname_hookname.py’ and update the namespace.
*** The exemple pipelines
**** fft
**** cmb
** Running Pipes
*** The interactive mode
This mode has been designed to ease debugging. If P is an instance of
...
...
@@ -468,8 +472,7 @@ accordingly.
Pipeline(pipedot, codedir=, prefix=, env=MyEnvironment)
** Using custom dependency schemes
** Writing custom main files
** Launching pipeweb behind apache
Pipeweb use the cherrypy web framework server and can be run behind an
...
...
@@ -479,6 +482,8 @@ apache webserver which brings essentially two advantages:
* The pipelet actors
This section document the code for developpers.
** The Repository object
** The Pipeline object
...
...
pipelet/pipeline.py
View file @
de216678
...
...
@@ -354,18 +354,20 @@ class Pipeline:
S
=
[
s
for
s
,
p
in
self
.
_parents
.
iteritems
()
if
not
p
]
# copy the graph
parents
=
self
.
_parents
.
copy
()
print
parents
children
=
self
.
_children
.
copy
()
while
S
:
s
=
s
.
pop
()
s
=
S
.
pop
(
0
)
yield
s
while
children
[
s
]:
c
=
children
[
s
].
pop
()
c
=
children
[
s
].
pop
(
0
)
parents
[
c
].
remove
(
s
)
if
not
parents
[
c
]:
yield
c
S
.
append
(
c
)
for
s
,
p
in
parents
.
iteritems
():
if
p
:
raise
PipeException
(
"Dependency graph has at least one cyle including segment"
%
s
)
print
parents
raise
PipeException
(
"Dependency graph has at least one cyle including segment %s"
%
s
)
def
_compute_hash
(
self
,
seg
):
""" Compute the hashkey for a given segment.
...
...
test/cmb/main.py
View file @
de216678
...
...
@@ -97,6 +97,15 @@ def main(print_info=print_info):
## Build pipeline instance
P
=
Pipeline
(
pipe_dot
,
code_dir
=
code_dir
,
prefix
=
prefix
,
matplotlib
=
True
,
matplotlib_interactive
=
True
)
<<<<<<<
HEAD
:
test
/
cmb
/
main
.
py
=======
P
.
to_dot
(
'cmb.dot'
)
cmbin
=
[]
for
sim_id
in
sim_ids
:
cmbin
.
append
((
nside
,
sim_id
))
P
.
push
(
cmb
=
cmbin
)
P
.
push
(
noise
=
[
nside
])
>>>>>>>
c5354a8
...
Fix
bugs
in
flatten
,
but
compute_hash
remain
tbd
:
test
/
cmb
/
main
.
py
## Interactive mode
if
options
.
debug
:
...
...
@@ -107,9 +116,9 @@ def main(print_info=print_info):
launch_process
(
P
,
options
.
process
,
log_level
=
log_level
)
## PBS mode
elif
options
.
add_workers
:
launch_pbs
(
P
,
options
.
add_workers
,
address
=
(
os
.
environ
[
'HOST'
],
50000
),
job_name
=
job_name
,
cpu_time
=
cpu_time
,
job_header
=
job_header
)
launch_pbs
(
P
,
options
.
add_workers
,
address
=
(
os
.
environ
[
'HOST
NAME
'
],
50000
),
job_name
=
job_name
,
cpu_time
=
cpu_time
,
job_header
=
job_header
)
else
:
launch_pbs
(
P
,
1
,
address
=
(
os
.
environ
[
'HOST'
],
50000
),
job_name
=
job_name
,
cpu_time
=
cpu_time
,
server
=
True
,
job_header
=
job_header
,
log_level
=
log_level
)
launch_pbs
(
P
,
1
,
address
=
(
os
.
environ
[
'HOST
NAME
'
],
50000
),
job_name
=
job_name
,
cpu_time
=
cpu_time
,
server
=
True
,
job_header
=
job_header
,
log_level
=
log_level
)
if
print_info
:
print
"1- Run 'pipeweb track <shortname> %s'
\n
to add the pipe to the web interface.
\n
"
%
(
P
.
sqlfile
)
...
...
test/complex_dependencies/a.py
0 → 100644
View file @
de216678
pass
test/complex_dependencies/b.py
0 → 100644
View file @
de216678
pass
test/complex_dependencies/c.py
0 → 100644
View file @
de216678
pass
test/complex_dependencies/d.py
0 → 100644
View file @
de216678
pass
test/complex_dependencies/e.py
0 → 100644
View file @
de216678
pass
test/complex_dependencies/main.py
0 → 100644
View file @
de216678
""" Demonstration of """
import
pipelet.pipeline
as
pipeline
from
pipelet.launchers
import
launch_process
import
os
pipedot
=
"""a -> b -> c;
b-> d -> c;
e -> d;
"""
P
=
pipeline
.
Pipeline
(
pipedot
,
code_dir
=
'./'
,
prefix
=
'./'
)
P
.
to_dot
(
'complex.dot'
)
print
"the execution order will be:"
for
s
in
P
.
flatten
():
print
s
launch_process
(
P
,
1
)
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment