Improve backup process

esm_cat_generator.py

 #!bin/python
 # -*- coding: utf-8 -*-
 import glob
+import os
 import re
 import gzip
 from argparse import ArgumentParser
@@ -237,18 +238,27 @@ def main():
             _ = [x for x in processes]
 
     # Concatenate CSV files into one compressed CSV catalog.
-    print(f'Concatenating CSV catalogs...')
+    catname = f'{args.project}{CSV_EXTENSION}{COMPRESS_EXTENSION}'
+    src = os.path.join(CATALOG_OUTPUT, catname)
+    print(f'Concatenating CSV catalogs into {src}...')
     csv_header = ','.join(header) + '\n'
-    with gzip.open(f'{CATALOG_OUTPUT_PROD}/{args.project}{CSV_EXTENSION}{COMPRESS_EXTENSION}', 'wb') as f_out:
+    with gzip.open(src, 'wb') as f_out:
         f_out.write(csv_header.encode('ascii'))
         for f in glob(f'{CATALOG_OUTPUT}/{args.project}/*{CSV_EXTENSION}'):
             with open(f, 'rb') as f_in:
                 copyfileobj(f_in, f_out)
 
+    # Copy CSV catalog into production directory.
+    dst = os.path.join(CATALOG_OUTPUT_PROD, catname)
+    print(f'Copy CSV catalog from {src} to {dst}...')
+    copy(src=src, dst=dst)
+
     # Backup CSV catalog.
-    print(f'Backup CSV compressed catalog...')
-    copy(src=f'{CATALOG_OUTPUT_PROD}/{args.project}{CSV_EXTENSION}{COMPRESS_EXTENSION}',
-         dst=f'{CATALOG_OUTPUT_BACKUP}/{args.project}_{date.today().strftime("%Y%m%d")}{CSV_EXTENSION}{COMPRESS_EXTENSION}')
+    src = os.path.join(CATALOG_OUTPUT, catname)
+    catbkp = f'{args.project}_{date.today().strftime("%Y%m%d")}{CSV_EXTENSION}{COMPRESS_EXTENSION}'
+    dst = os.path.join(CATALOG_OUTPUT_BACKUP, catbkp)
+    print(f'Backup CSV catalog from {src} to {dst}...')
+    copy(src=src, dst=dst)
 
     # Write JSON catalog.
     make_json(CATALOG_OUTPUT_PROD, args.project, header)

utils.py

@@ -33,11 +33,11 @@ def make_json(catdir, project, header):
     ESM JSON catalog templating based on Jinja2.
 
     """
-    print(f'Writing JSON catalog...')
-
     # Get catalog name.
     catpath = os.path.join(catdir, f'{project}{JSON_EXTENSION}')
 
+    print(f'Writing JSON catalog into {catpath}...')
+
     # Load Jinja2 template for ESM catalogs.
     with open('ESMCollectionTemplate.json') as file:
         esm_json_template = Template(file.read())