Docker-in-Docker (DinD) capabilities of public runners deactivated. More info

Commit a00e735c authored by Betoule Marc's avatar Betoule Marc
Browse files

minor logging changes

parent 6d2e61aa
......@@ -281,13 +281,16 @@ class Environment(EnvironmentBase):
return glob(path.join(self._worker.pipe.get_data_dir(segx),y))\
+glob(path.join(self._worker.pipe.get_data_dir(segx),path.join('*/',y)))
def logged_subprocess(self, args, shell=False, except_on_failure=True, name=None, of=None):
def logged_subprocess(self, args, shell=False,
except_on_failure=True, name=None, of=None):
""" Execute a subprocess and log its output.
Create files process_name.log and process_name.err. If shell is set to True, the command is run via a shell. If except_on_failure set to True, raise an exception when command execution return non zero
>>> (o,e)=logged_subprocess (["echo", "$TMP"])
Create files process_name.log and process_name.err. If shell
is set to True, the command is run via a shell. If
except_on_failure set to True, raise an exception when command
execution return non zero
>>> (o, e) = logged_subprocess (["echo", "$TMP"])
>>> open(o).readlines()
['$BASH']
"""
......@@ -295,28 +298,34 @@ class Environment(EnvironmentBase):
if name is None:
name = path.basename(proc)
if of is None:
of = self.get_data_fn(name+'.log')
ef = self.get_data_fn(name+'.err')
o = file(of,'w')
e = file(ef,'w')
e.write('#'+' '.join([str(a) for a in args])+'\n')
of = self.get_data_fn(name + '.log')
ef = self.get_data_fn(name + '.err')
o = file(of, 'w')
e = file(ef, 'w')
command = '#' + ' '.join([str(a) for a in args])
e.write(command + '\n')
e.flush()
p=subprocess.Popen(args, stdout=o, stderr=e, shell=shell)
self.logger.info('Executing the following command %s' % command)
self.logger.debug('Log results in the following files:')
self.logger.debug(of)
self.logger.debug(ef)
p = subprocess.Popen(args, stdout=o, stderr=e, shell=shell)
p.communicate()[0]
if except_on_failure:
if p.returncode != 0:
self.logger.error('Execution of command %s failed' % proc)
raise subprocess.CalledProcessError(p.returncode, args[0])
o.close()
e.close()
self.logger.info('Command exited successfully')
return (of, ef)
def get_tmp_fn(self):
""" Obtain a temporary filename
Note : has to be part of the segment execution environment
The temporary file is added to the intern list for future removal.
The temporary file is added to the intern list for future removal.
>>> get_tmp_fn()
./test/first_test/third_QA7CX5A/fourth_LGJSKVQ/735200
"""
......@@ -324,26 +333,25 @@ class Environment(EnvironmentBase):
self._tmpf.append(tf)
return tf
def load_param(self, seg,glo,param_name='*'):
def load_param(self, seg, glo, param_name='*'):
""" Update the global dictionnary with parameters of a
segment.
>>> load_param('second', globals(), ["fn"])
>>> print fn
test.txt
"""
segx = self._worker.pipe.find_seg(self._worker.task.seg, seg)
if segx is None:
self.logger.warning("No parent segment matching %s found"%seg)
self.logger.warning("No parent segment matching %s found" % seg)
if not path.exists(self._worker.pipe.get_param_file(segx)):
return
self.load_products(self._worker.pipe.get_param_file(segx), glo, param_name=param_name)
self.load_products(self._worker.pipe.get_param_file(segx),
glo, param_name=param_name)
def save_products(self, filename, glo, param_name='*'):
""" Use pickle to save a part of a given namespace.
>>> save_products(get_data_fn("test.pkl"), globals(),['i'])
>>> open(get_data_fn("test.pkl")).readlines()
['(dp1'\\n, "S'i'\\n", 'I0\\n', 's.']
......
......@@ -529,7 +529,7 @@ Delete products directories of a given task id. The list of task id is '-' separ
if download is not None:
return serve_download(filepath)
else:
if os.path.splitext(filepath)[1] in ['.log','.txt','.list','.py']:
if os.path.splitext(filepath)[1] in ['.log', '.txt', '.list', '.py', '.err']:
return serve_file(filepath, content_type='text/plain', disposition="inline")
elif os.path.splitext(filepath)[1] in ['.fits','.fz']:
return serve_file(filepath, content_type='application/fits', disposition="inline")
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment