mainchange : 1 catalog1 worker

constants.py

@@ -6,11 +6,13 @@
 
 """
 
-IPSL_DATA_ROOT = '/bdd'
-
-INPUT_SOURCES = {'CMIP6': ['/bdd/CMIP6/C4MIP/MOHC/.paths.txt',  # CMIP6 replicas (IDRIS)
-                          '/bdd/CMIP6/C4MIP/IPSL/.paths.txt'  # CMIP6 IPSL datasets (TGCC)
+IPSL_DATA_ROOT = '/bdd/'
+INPUT_SOURCES = {'CMIP6': ['/home/ltroussellier/dev_catalog/test3_gui/catalog/InputFakeFile/'  # CMIP6 replicas (IDRIS)
                           ]}
+
+# INPUT_SOURCES = {'CMIP6': ['/bdd/CMIP6/C4MIP/MOHC/.paths.txt',  # CMIP6 replicas (IDRIS)
+#                           '/bdd/CMIP6/C4MIP/IPSL/.paths.txt'  # CMIP6 IPSL datasets (TGCC)
+#                           ]}
 # INPUT_SOURCES = {'CMIP6': ['/Users/glipsl/Desktop/.paths.txt'],
 #                  'CMIP5': ['/bdd/CMIP5/output/MOHC/.paths.txt',  # CMIP5 replicas (IDRIS)
 #                            '/bdd/CMIP5/output/IPSL/.paths.txt'  # CMIP5 IPSL datasets (TGCC)

esm_cat_generator.py

@@ -131,13 +131,14 @@ class Runner(object):
             #print(source)
             # Get total entires in a fast way for beautiful progress bar.
             total = int(subprocess.check_output(["wc", "-l", source]).split()[0])
+            chunksize =  int(total/10)
             #total = 5000000
             #print("total à traiter", total)
             # Instantiate pool of processes.
             if self.pool:
                 #print("TRUE")
                 # Instantiate pool iterator with progress bar.
-                processes = tqdm(self.pool.imap(Process(ctx), self.get_entries(source)), desc='Catalog generation',
+                processes = tqdm(self.pool.imap(Process(ctx), self.get_entries(source), chunksize=chunksize), desc='Catalog generation',
                                  total=total)
 
             # Sequential processing use basic map function.

qsub/JobCMIP6Cat

@@ -18,5 +18,5 @@ echo $PWD
 
  # Example for a Python job :
 export PATH=/opt/epd_py25-4.3.0/bin:$PATH
-python esm_cat_generator.py -p CMIP6 -o OutputCatalog/ -t 2 -d experiment_id
+python esm_cat_generator.py -p CMIP6 -o OutputCatalog/ -t 4 -d experiment_id
 

qsub/std_error.out

-/var/spool/torque/mom_priv/jobs/25273.SC: line 16: cd: dev/test3_gui/: No such file or directory
+Running Host: ciclad20.private.ipsl.fr
+/var/spool/torque/mom_priv/jobs/26118.SC: line 16: cd: dev/test3_gui/: No such file or directory
 /home/ltroussellier
 python: can't open file 'esm_cat_generator.py': [Errno 2] No such file or directory
 Epilogue Args:
-Job ID: 25273
+Job ID: 26118
 User ID: ltroussellier
 Group ID: lsce
 Job Name: Fabrication
-Session ID: 2363
+Session ID: 37303
 Resource List: mem=1gb,neednodes=1:ppn=4,nodes=1:ppn=4,vmem=4gb,walltime=00:30:00
 Resources Used: cput=00:00:00,mem=0kb,vmem=0kb,walltime=00:00:01
 Queue Name: short
-Running Host: ciclad25.private.ipsl.fr
+Running Host: ciclad20.private.ipsl.fr