Skip to content
GitLab
Projects
Groups
Snippets
Help
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Open sidebar
pipelet
Pipelet
Commits
aa6582e4
Commit
aa6582e4
authored
Sep 03, 2010
by
Maude Le Jeune
Browse files
Feature #872 finalized : readme, test and code in agrement for environment
parent
4a730795
Changes
5
Hide whitespace changes
Inline
Side-by-side
Showing
5 changed files
with
33 additions
and
27 deletions
+33
-27
README.org
README.org
+6
-7
pipelet/environment.py
pipelet/environment.py
+21
-14
test/seg_first_code.py
test/seg_first_code.py
+3
-3
test/seg_second_code.py
test/seg_second_code.py
+1
-1
test/seg_third_code.py
test/seg_third_code.py
+2
-2
No files found.
README.org
View file @
aa6582e4
...
...
@@ -210,7 +210,6 @@ The segment code is executed in a specific environment that provides:
1. access to the segment input and output
- seg_input: this variable is a dictionnary containing the input of the segment
- get_input():
- seg_output: this variable has to be set to a list containing the
2. Functionnalities to use the automated hierarchical data storage system.
...
...
@@ -219,17 +218,17 @@ The segment code is executed in a specific environment that provides:
- get_tmp_fn(): return a temporary filename.
3. Functionnalities to use the automated parameters handling
-
var_key
: list of parameter names of the segment
-
var
_tag: list of parameter names which will be made visible from the web interface
- load_param(seg,
var_names
)
-
lst_par
: list of parameter names of the segment
-
lst
_tag: list of parameter names which will be made visible from the web interface
- load_param(seg,
lst_par
)
4. Various convenient functionalities
- save_products(filename=',
var_names
='*'): use pickle to save a
- save_products(filename=',
lst_par
='*'): use pickle to save a
part of a given namespace.
- load_products(filename,
var_names
): update the namespace by
- load_products(filename,
lst_par
): update the namespace by
unpickling requested object from the file.
- logged_subprocess(lst_args): execute a subprocess and log its output.
- log is a standard logging.Logger object that can be used to log the processing
- log
ger
is a standard logging.Logger object that can be used to log the processing
5. Hooking support
Pipelet enables you to write reusable generic
...
...
pipelet/environment.py
View file @
aa6582e4
...
...
@@ -53,7 +53,6 @@ class EnvironmentBase():
w: a worker instance
"""
self
.
_worker
=
w
self
.
_segment_args
=
w
.
task
.
task_input
def
_get_data_fn
(
self
,
x
):
""" Complete the filename with the path to the working
...
...
@@ -89,7 +88,7 @@ class EnvironmentBase():
""" Return the log file name
"""
return
self
.
_get_data_fn
(
"seg
ment_error
.log"
)
return
self
.
_get_data_fn
(
"seg
_"
+
self
.
_worker
.
task
.
seg
+
"
.log"
)
def
_hook
(
self
,
hook_name
,
glo
):
""" Execute hook code.
...
...
@@ -132,7 +131,7 @@ class Environment(EnvironmentBase):
w: a worker instance
"""
self
.
_worker
=
w
self
.
seg
ment
_input
=
w
.
task
.
task_input
self
.
seg_input
=
w
.
task
.
task_input
## list of temporary files
self
.
_tmpf
=
[]
self
.
logger
=
init_logger
(
self
.
_get_data_fn
(
""
),
self
.
_get_log_file
(),
level
=
[])
...
...
@@ -165,9 +164,11 @@ class Environment(EnvironmentBase):
glo: dict, global dictionnary to update.
"""
self
.
logger
.
info
(
"hooking %s"
%
hook_name
)
return
self
.
_hook
(
hook_name
,
glo
)
def
glob_seg
(
self
,
x
,
y
):
""" Return the list of filename matching y in the working
directory of segment x.
...
...
@@ -181,7 +182,10 @@ class Environment(EnvironmentBase):
-------
list of filenames.
"""
segx
=
self
.
_worker
.
pipe
.
find_seg
(
self
.
_worker
.
task
.
seg
,
x
)
if
segx
is
None
:
self
.
logger
.
warning
(
"No parent segment matching %s found"
%
x
)
return
glob
(
path
.
join
(
self
.
_worker
.
pipe
.
get_data_dir
(
segx
),
y
))
\
+
glob
(
path
.
join
(
self
.
_worker
.
pipe
.
get_data_dir
(
segx
),
path
.
join
(
'*/'
,
y
)))
...
...
@@ -238,6 +242,8 @@ class Environment(EnvironmentBase):
param_name : string list, parameters name.
"""
segx
=
self
.
_worker
.
pipe
.
find_seg
(
self
.
_worker
.
task
.
seg
,
seg
)
if
segx
is
None
:
self
.
logger
.
warning
(
"No parent segment matching %s found"
%
seg
)
if
not
path
.
exists
(
self
.
_worker
.
pipe
.
get_param_file
(
segx
)):
return
self
.
load_products
(
self
.
_worker
.
pipe
.
get_param_file
(
segx
),
glo
,
param_name
=
param_name
)
...
...
@@ -260,7 +266,7 @@ class Environment(EnvironmentBase):
try
:
new_dict
[
k
]
=
glo
[
k
]
except
KeyError
:
logger
.
warning
(
'Fail to save object %s in file %s'
%
(
k
,
filename
))
logger
.
warning
(
'Fail to save object %s in file %s'
%
(
k
,
filename
))
f
=
file
(
filename
,
'w'
)
pickle
.
dump
(
new_dict
,
f
)
f
.
close
()
...
...
@@ -281,7 +287,7 @@ class Environment(EnvironmentBase):
new_dict
=
pickle
.
load
(
f
)
f
.
close
()
except
IOError
:
logger
.
warning
(
'No such file: %s'
%
filename
)
logger
.
warning
(
'No such file: %s'
%
filename
)
except
UnpicklingError
:
logger
.
warning
(
'Failed to unpickle from file: %s'
%
filename
)
f
.
close
()
...
...
@@ -302,7 +308,7 @@ class Environment(EnvironmentBase):
self
.
_tmpf
=
[]
def
make_tag
(
self
,
seg
,
glo
):
def
_
make_tag
(
self
,
seg
,
glo
):
""" Write a tag using parameters of a segment.
Parameters
...
...
@@ -315,7 +321,7 @@ class Environment(EnvironmentBase):
string.
"""
try
:
var_tag
=
glo
[
'
var
_tag'
]
var_tag
=
glo
[
'
lst
_tag'
]
except
KeyError
:
var_tag
=
[]
strtag
=
''
...
...
@@ -334,7 +340,7 @@ class Environment(EnvironmentBase):
return
var_tag
def
save_param
(
self
,
seg
,
glo
,
param_name
=
'*'
):
def
_
save_param
(
self
,
seg
,
glo
,
param_name
=
'*'
):
""" Save parameters of a segment.
Parameters
...
...
@@ -356,21 +362,22 @@ class Environment(EnvironmentBase):
Parameters
----------
glo :
glo : dict, the global dictionnary
seg : string, segment name
"""
self
.
clean_tmp
()
try
:
# Save params
var_key
=
glo
[
'var_key'
]
self
.
save_param
(
seg
,
glo
,
param_name
=
var_key
)
self
.
logger
.
info
(
'Save param file'
)
var_key
=
glo
[
'lst_par'
]
self
.
_save_param
(
seg
,
glo
,
param_name
=
var_key
)
except
KeyError
:
self
.
logger
.
warning
(
'Nothing to save in param file for seg %s'
%
seg
)
except
Exception
:
self
.
logger
.
warning
(
'Fail to save the param file for seg %s'
%
seg
)
self
.
_worker
.
task
.
param
=
self
.
make_tag
(
seg
,
glo
)
self
.
_worker
.
task
.
param
=
self
.
_
make_tag
(
seg
,
glo
)
try
:
res
=
glo
[
"
res
"
]
res
=
glo
[
"
seg_output
"
]
except
:
res
=
None
self
.
logger
.
info
(
"No segment output found, setting seg_output to None"
)
self
.
_worker
.
task
.
task_output
=
res
test/seg_first_code.py
View file @
aa6582e4
n
=
10
var_key
=
[
'n'
]
var
_tag
=
[
'n'
]
lst_par
=
[
'n'
]
lst
_tag
=
[
'n'
]
tf
=
get_data_fn
(
'test.txt'
)
f
=
file
(
tf
,
'w'
)
f
.
write
(
'This segment launch %d seg_second'
%
n
)
f
.
close
res
=
range
(
n
)
seg_output
=
range
(
n
)
test/seg_second_code.py
View file @
aa6582e4
tf
=
get_data_fn
(
'test.txt'
)
# Dude makes some kind of heavy computation
i
=
0
product
=
seg
ment
_input
[
0
]
product
=
seg_input
[
0
]
f
=
file
(
tf
,
'w'
)
f
.
write
(
'This segment received %d as arg'
%
product
)
f
.
close
...
...
test/seg_third_code.py
View file @
aa6582e4
pre_line
=
'This is the preambule'
var_key
=
[
'pre_line'
]
var
_tag
=
var_key
lst_par
=
[
'pre_line'
]
lst
_tag
=
lst_par
l
=
2010
tf
=
get_data_fn
(
'Preambule.txt'
)
f
=
file
(
tf
,
'w'
)
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment