Commit a7b6caa8 authored by TROUSSELLIER Laurent's avatar TROUSSELLIER Laurent
Browse files

mainchange : 1 catalog1 worker

parent ca4ddf4a
......@@ -6,11 +6,13 @@
"""
IPSL_DATA_ROOT = '/bdd'
INPUT_SOURCES = {'CMIP6': ['/bdd/CMIP6/C4MIP/MOHC/.paths.txt', # CMIP6 replicas (IDRIS)
'/bdd/CMIP6/C4MIP/IPSL/.paths.txt' # CMIP6 IPSL datasets (TGCC)
IPSL_DATA_ROOT = '/bdd/'
INPUT_SOURCES = {'CMIP6': ['/home/ltroussellier/dev_catalog/test3_gui/catalog/InputFakeFile/' # CMIP6 replicas (IDRIS)
]}
# INPUT_SOURCES = {'CMIP6': ['/bdd/CMIP6/C4MIP/MOHC/.paths.txt', # CMIP6 replicas (IDRIS)
# '/bdd/CMIP6/C4MIP/IPSL/.paths.txt' # CMIP6 IPSL datasets (TGCC)
# ]}
# INPUT_SOURCES = {'CMIP6': ['/Users/glipsl/Desktop/.paths.txt'],
# 'CMIP5': ['/bdd/CMIP5/output/MOHC/.paths.txt', # CMIP5 replicas (IDRIS)
# '/bdd/CMIP5/output/IPSL/.paths.txt' # CMIP5 IPSL datasets (TGCC)
......
......@@ -131,13 +131,14 @@ class Runner(object):
#print(source)
# Get total entires in a fast way for beautiful progress bar.
total = int(subprocess.check_output(["wc", "-l", source]).split()[0])
chunksize = int(total/10)
#total = 5000000
#print("total à traiter", total)
# Instantiate pool of processes.
if self.pool:
#print("TRUE")
# Instantiate pool iterator with progress bar.
processes = tqdm(self.pool.imap(Process(ctx), self.get_entries(source)), desc='Catalog generation',
processes = tqdm(self.pool.imap(Process(ctx), self.get_entries(source), chunksize=chunksize), desc='Catalog generation',
total=total)
# Sequential processing use basic map function.
......
......@@ -18,5 +18,5 @@ echo $PWD
# Example for a Python job :
export PATH=/opt/epd_py25-4.3.0/bin:$PATH
python esm_cat_generator.py -p CMIP6 -o OutputCatalog/ -t 2 -d experiment_id
python esm_cat_generator.py -p CMIP6 -o OutputCatalog/ -t 4 -d experiment_id
/var/spool/torque/mom_priv/jobs/25273.SC: line 16: cd: dev/test3_gui/: No such file or directory
Running Host: ciclad20.private.ipsl.fr
/var/spool/torque/mom_priv/jobs/26118.SC: line 16: cd: dev/test3_gui/: No such file or directory
/home/ltroussellier
python: can't open file 'esm_cat_generator.py': [Errno 2] No such file or directory
Epilogue Args:
Job ID: 25273
Job ID: 26118
User ID: ltroussellier
Group ID: lsce
Job Name: Fabrication
Session ID: 2363
Session ID: 37303
Resource List: mem=1gb,neednodes=1:ppn=4,nodes=1:ppn=4,vmem=4gb,walltime=00:30:00
Resources Used: cput=00:00:00,mem=0kb,vmem=0kb,walltime=00:00:01
Queue Name: short
Running Host: ciclad25.private.ipsl.fr
Running Host: ciclad20.private.ipsl.fr
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment