Commit 1dc37ce1 authored by Guillaume's avatar Guillaume
Browse files

Bugfixes

parent b1482bb2
......@@ -18,8 +18,9 @@ INPUT_PATTERNS = {'CMIP6': '/bdd/CMIP6/*/*/*/*/.paths.txt',
ALLOWED_PROJECTS = ['CMIP6', 'CMIP5', 'CORDEX']
CSV_EXTENSION = ".csv"
JSON_EXTENSION = ".json"
CSV_EXTENSION = '.csv'
JSON_EXTENSION = '.json'
COMPRESS_EXTENSION = '.gz'
CATALOG_DESCRIPTION_TEMPLATE = "CLIMERI-France {} data catalog."
......
......@@ -2,7 +2,7 @@
# -*- coding: utf-8 -*-
import glob
import re
import tarfile
import gzip
from argparse import ArgumentParser
from datetime import date
from glob import glob
......@@ -242,22 +242,19 @@ def main():
# Write JSON catalog.
make_json(CATALOG_OUTPUT_PROD, args.project, header)
# Concatenate CSV files into one CSV catalog.
# Concatenate CSV files into one compressed CSV catalog.
print(f'Concatenating CSV catalogs...')
with open(f'{CATALOG_OUTPUT_PROD}/{args.project}{CSV_EXTENSION}', 'w') as fout:
csv_header = ','.join(header) + '/n'
with gzip.open(f'{CATALOG_OUTPUT_PROD}/{args.project}{CSV_EXTENSION}{COMPRESS_EXTENSION}', 'wb') as f_out:
f_out.write(csv_header.encode('ascii'))
for f in glob(f'{CATALOG_OUTPUT}/{args.project}/*{CSV_EXTENSION}'):
with open(f, 'r') as fin:
copyfileobj(fin, fout)
# Compress CSV catalog.
print(f'Compressing final CSV catalog...')
with tarfile.open(f'{CATALOG_OUTPUT_PROD}/{args.project}{CSV_EXTENSION}.gz', 'w:gz') as tar:
tar.add(f'{CATALOG_OUTPUT_PROD}/{args.project}{CSV_EXTENSION}')
with open(f, 'rb') as f_in:
copyfileobj(f_in, f_out)
# Backup CSV catalog.
print(f'Backup CSV compressed catalog...')
copy(src=f'{CATALOG_OUTPUT_PROD}/{args.project}{CSV_EXTENSION}.gz',
dst=f'{CATALOG_OUTPUT_BACKUP}/{args.project}_{date.today().strftime("%Y%m%d")}{CSV_EXTENSION}.gz')
copy(src=f'{CATALOG_OUTPUT_PROD}/{args.project}{CSV_EXTENSION}{COMPRESS_EXTENSION}',
dst=f'{CATALOG_OUTPUT_BACKUP}/{args.project}_{date.today().strftime("%Y%m%d")}{CSV_EXTENSION}{COMPRESS_EXTENSION}')
print(f'{args.project} intake catalogs complete.')
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment