Docker-in-Docker (DinD) capabilities of public runners deactivated. More info

Commit 6a7a9d91 authored by Maude Le Jeune's avatar Maude Le Jeune
Browse files

improve resource monitoring

parent 15dbd3f2
......@@ -12,24 +12,23 @@
##
## You should have received a copy of the GNU General Public License
## along with this program; if not, see http://www.gnu.org/licenses/gpl.html
import os.path as path
import subprocess
import threading
from glob import glob
from datetime import datetime
import logging
from utils import str_web
from contextlib import closing
import cPickle as pickle
from pipelet.utils import init_logger, close_logger
from pipelet.repository import get_git_revision
import traceback
import signal
from contextlib import closing
from datetime import datetime
import gc
import logging
from glob import glob
import inspect
import logging
import os.path as path
import pipelet
from pipelet.repository import get_git_revision
from pipelet.utils import init_logger, close_logger, str_web
import resource
import signal
import subprocess
import threading
from time import time as timer
import traceback
class NullHandler(logging.Handler):
""" Extension of the logging handler class.
......@@ -39,6 +38,25 @@ class NullHandler(logging.Handler):
"""
pass
ressources = (
'ru_stime',
'ru_utime',
'ru_maxrss',
'ru_ixrss',
'ru_idrss',
'ru_isrss',
'ru_minflt',
'ru_majflt',
'ru_nswap',
'ru_inblock',
'ru_oublock',
'ru_msgsnd',
'ru_msgrcv',
'ru_nsignals',
'ru_nvcsw',
'ru_nivcsw',
)
used_ressource_id = [0, 1, 2, 6, 7, 9, 10, 14, 15]
class EnvironmentBase():
......@@ -282,7 +300,7 @@ class Environment(EnvironmentBase):
+glob(path.join(self._worker.pipe.get_data_dir(segx),path.join('*/',y)))
def logged_subprocess(self, args, shell=False, except_on_failure=True, name=None, of=None):
def logged_subprocess(self, args, shell=False, except_on_failure=True, name=None, of=None, stat=False):
""" Execute a subprocess and log its output.
Create files process_name.log and process_name.err. If shell is set to True, the command is run via a shell. If except_on_failure set to True, raise an exception when command execution return non zero
......@@ -301,8 +319,16 @@ class Environment(EnvironmentBase):
e = file(ef,'w')
e.write('#'+' '.join([str(a) for a in args])+'\n')
e.flush()
if stat:
res = resource.getrusage(resource.RUSAGE_CHILDREN)
wall_time = timer()
p=subprocess.Popen(args, stdout=o, stderr=e, shell=shell)
p.communicate()[0]
if stat:
wall_time = timer() - wall_time
res2 = resource.getrusage(resource.RUSAGE_CHILDREN)
resstr = 'ressources for process %s: walltime=%f, ' % (proc, wall_time)
self.logger.info(resstr + ', '.join(['%s=%f' % (ressources[k], res2[k] - res[k]) for k in used_ressource_id]))
if except_on_failure:
if p.returncode != 0:
raise subprocess.CalledProcessError(p.returncode, args[0])
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment