Commit 8b900c58 authored by Betoule Marc's avatar Betoule Marc
Browse files

Adapt ccge launcher to launch in the multiprocessing queue

parent 2970e3a2
...@@ -460,7 +460,7 @@ EOF ...@@ -460,7 +460,7 @@ EOF
'-N', name]+reduce(lambda x,y : x+y, [['-l', '%s=1'%res] for res in ressource], [])+[jobfile] '-N', name]+reduce(lambda x,y : x+y, [['-l', '%s=1'%res] for res in ressource], [])+[jobfile]
subprocess.Popen(com).communicate()[0] subprocess.Popen(com).communicate()[0]
def launch_ccage_worker (pipe, n, address=('127.0.0.1',5000), authkey='secret', log_level=logging.WARNING, project="planck", cpu_time='01:00:00', scratch="512M", mem="512M", ressource=['sps']): def launch_ccage_worker (pipe, n, address=('127.0.0.1',5000), authkey='secret', log_level=logging.WARNING, project="planck", cpu_time='01:00:00', scratch="512M", mem="512M", ressource=['sps'], multicores=False):
""" Launch n worker using job array """ Launch n worker using job array
""" """
worker_file = """#! /bin/zsh -l worker_file = """#! /bin/zsh -l
...@@ -477,6 +477,10 @@ echo "worker returned at $(date)" ...@@ -477,6 +477,10 @@ echo "worker returned at $(date)"
logfile = jobfile.replace('job', 'log') logfile = jobfile.replace('job', 'log')
with closing(file(jobfile,'w')) as f: with closing(file(jobfile,'w')) as f:
f.write(worker_file%(authkey,logfile)) f.write(worker_file%(authkey,logfile))
if multicores:
mc = ['-pe', 'multicores' , '%d'%multicores]
else:
mc = []
com=['qsub', com=['qsub',
'-P', 'P_%s'%project, '-P', 'P_%s'%project,
'-l', 'fsize=%s'%scratch, '-l', 'fsize=%s'%scratch,
...@@ -485,7 +489,7 @@ echo "worker returned at $(date)" ...@@ -485,7 +489,7 @@ echo "worker returned at $(date)"
'-t', '1-%d'%n, '-t', '1-%d'%n,
'-o' ,logfile, '-o' ,logfile,
'-e', errfile, '-e', errfile,
'-N', name]+reduce(lambda x,y : x+y, [['-l', '%s=1'%res] for res in ressource], [])+[jobfile] '-N', name]+mc+reduce(lambda x,y : x+y, [['-l', '%s=1'%res] for res in ressource], [])+[jobfile]
subprocess.Popen(com).communicate()[0] subprocess.Popen(com).communicate()[0]
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment