Commit 536fd211 authored by Maude Le Jeune's avatar Maude Le Jeune
Browse files

Merge branch 'master' of git@gitorious.org:pipelet/pipelet

parents 7e3a08cb b5f2457f
......@@ -198,7 +198,7 @@ class Environment(EnvironmentBase):
+glob(path.join(self._worker.pipe.get_data_dir(segx),path.join('*/',y)))
def logged_subprocess(self, args, shell=False):
def logged_subprocess(self, args, shell=False, except_on_failure=True):
""" Execute a subprocess and log its output.
Create files process_name.log and process_name.err
......@@ -206,6 +206,8 @@ class Environment(EnvironmentBase):
Parameters
----------
args: a Task argument
shell: If set to True, the command is run via a shell
except_on_failure: If set to True, raise an exception when command execution return non zero
Returns
-------
......@@ -218,7 +220,11 @@ class Environment(EnvironmentBase):
e = file(ef,'w')
o.write('#'+' '.join([str(a) for a in args])+'\n')
o.flush()
subprocess.Popen(args, stdout=o, stderr=e, shell=shell).communicate()[0]
p=subprocess.Popen(args, stdout=o, stderr=e, shell=shell)
p.communicate()[0]
if except_on_failure:
if p.returncode != 0:
raise subprocess.CalledProcessError(p.returncode, args[0])
o.close()
e.close()
return (of, ef)
......
......@@ -256,6 +256,7 @@ echo $PYTHONPATH
_job_file = """
#/bin/zsh
export PATH=/afs/in2p3.fr/home/b/betoule/software/photcalib/tools:$PATH
echo $SHELL
echo $PYTHONPATH
python -m pipelet.launchers -H %s -p %s -s %s
......@@ -264,10 +265,11 @@ python -m pipelet.launchers -H %s -p %s -s %s
def launch_ccali(pipe, n, address=None, authkey='secret', job_dir="/sps/snls13/users/betoule/bqs/scripts",
job_name="job_", log_dir="/sps/snls13/users/betoule/bqs/logs",
cpu_time=1000, scratch="512MB", mem="512MB",plateform='LINUX',server=False):
cpu_time=1000, scratch="512MB", mem="512MB",plateform='LINUX',server=False, log_level=logging.DEBUG):
"""submit a bunch of distant workers to the BQS system
at CCIN2P3.
"""
set_logger (pipe, log_level)
s = scheduler.Scheduler(pipe)
......
......@@ -215,7 +215,7 @@ class LocalRepository(Repository):
self._hook = {}
## string, where to find segment's source code file
self.src_path = src_path
self.src_path = path.expanduser(path.expandvars(src_path))
## string, where to find segment's library
self.lib_path = lib_path
......
......@@ -173,7 +173,7 @@ class SqliteTracker(Tracker,threading.Thread):
## string, sql filename
self.sqlfile = path.join(pipe._prefix,'.sqlstatus')
if self.pipe.sqlfile is not None:
self.sqlfile = self.pipe.sqlfile
self.sqlfile = path.expanduser(path.expandvars(self.pipe.sqlfile))
logger.info("tracking job in %s" % self.sqlfile)
## sql connexion object
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment