Bugfixes

constants.py

@@ -18,8 +18,9 @@ INPUT_PATTERNS = {'CMIP6': '/bdd/CMIP6/*/*/*/*/.paths.txt',
 
 ALLOWED_PROJECTS = ['CMIP6', 'CMIP5', 'CORDEX']
 
-CSV_EXTENSION = ".csv"
-JSON_EXTENSION = ".json"
+CSV_EXTENSION = '.csv'
+JSON_EXTENSION = '.json'
+COMPRESS_EXTENSION = '.gz'
 
 CATALOG_DESCRIPTION_TEMPLATE = "CLIMERI-France {} data catalog."
 

esm_cat_generator.py

@@ -2,7 +2,7 @@
 # -*- coding: utf-8 -*-
 import glob
 import re
-import tarfile
+import gzip
 from argparse import ArgumentParser
 from datetime import date
 from glob import glob
@@ -242,22 +242,19 @@ def main():
     # Write JSON catalog.
     make_json(CATALOG_OUTPUT_PROD, args.project, header)
 
-    # Concatenate CSV files into one CSV catalog.
+    # Concatenate CSV files into one compressed CSV catalog.
     print(f'Concatenating CSV catalogs...')
-    with open(f'{CATALOG_OUTPUT_PROD}/{args.project}{CSV_EXTENSION}', 'w') as fout:
+    csv_header = ','.join(header) + '/n'
+    with gzip.open(f'{CATALOG_OUTPUT_PROD}/{args.project}{CSV_EXTENSION}{COMPRESS_EXTENSION}', 'wb') as f_out:
+        f_out.write(csv_header.encode('ascii'))
         for f in glob(f'{CATALOG_OUTPUT}/{args.project}/*{CSV_EXTENSION}'):
-            with open(f, 'r') as fin:
-                copyfileobj(fin, fout)
-
-    # Compress CSV catalog.
-    print(f'Compressing final CSV catalog...')
-    with tarfile.open(f'{CATALOG_OUTPUT_PROD}/{args.project}{CSV_EXTENSION}.gz', 'w:gz') as tar:
-        tar.add(f'{CATALOG_OUTPUT_PROD}/{args.project}{CSV_EXTENSION}')
+            with open(f, 'rb') as f_in:
+                copyfileobj(f_in, f_out)
 
     # Backup CSV catalog.
     print(f'Backup CSV compressed catalog...')
-    copy(src=f'{CATALOG_OUTPUT_PROD}/{args.project}{CSV_EXTENSION}.gz',
-         dst=f'{CATALOG_OUTPUT_BACKUP}/{args.project}_{date.today().strftime("%Y%m%d")}{CSV_EXTENSION}.gz')
+    copy(src=f'{CATALOG_OUTPUT_PROD}/{args.project}{CSV_EXTENSION}{COMPRESS_EXTENSION}',
+         dst=f'{CATALOG_OUTPUT_BACKUP}/{args.project}_{date.today().strftime("%Y%m%d")}{CSV_EXTENSION}{COMPRESS_EXTENSION}')
 
     print(f'{args.project} intake catalogs complete.')