Commit 8832a690 authored by Guillaume's avatar Guillaume
Browse files

Improve backup process

parent af4137ec
#!bin/python
# -*- coding: utf-8 -*-
import glob
import os
import re
import gzip
from argparse import ArgumentParser
......@@ -237,18 +238,27 @@ def main():
_ = [x for x in processes]
# Concatenate CSV files into one compressed CSV catalog.
print(f'Concatenating CSV catalogs...')
catname = f'{args.project}{CSV_EXTENSION}{COMPRESS_EXTENSION}'
src = os.path.join(CATALOG_OUTPUT, catname)
print(f'Concatenating CSV catalogs into {src}...')
csv_header = ','.join(header) + '\n'
with gzip.open(f'{CATALOG_OUTPUT_PROD}/{args.project}{CSV_EXTENSION}{COMPRESS_EXTENSION}', 'wb') as f_out:
with gzip.open(src, 'wb') as f_out:
f_out.write(csv_header.encode('ascii'))
for f in glob(f'{CATALOG_OUTPUT}/{args.project}/*{CSV_EXTENSION}'):
with open(f, 'rb') as f_in:
copyfileobj(f_in, f_out)
# Copy CSV catalog into production directory.
dst = os.path.join(CATALOG_OUTPUT_PROD, catname)
print(f'Copy CSV catalog from {src} to {dst}...')
copy(src=src, dst=dst)
# Backup CSV catalog.
print(f'Backup CSV compressed catalog...')
copy(src=f'{CATALOG_OUTPUT_PROD}/{args.project}{CSV_EXTENSION}{COMPRESS_EXTENSION}',
dst=f'{CATALOG_OUTPUT_BACKUP}/{args.project}_{date.today().strftime("%Y%m%d")}{CSV_EXTENSION}{COMPRESS_EXTENSION}')
src = os.path.join(CATALOG_OUTPUT, catname)
catbkp = f'{args.project}_{date.today().strftime("%Y%m%d")}{CSV_EXTENSION}{COMPRESS_EXTENSION}'
dst = os.path.join(CATALOG_OUTPUT_BACKUP, catbkp)
print(f'Backup CSV catalog from {src} to {dst}...')
copy(src=src, dst=dst)
# Write JSON catalog.
make_json(CATALOG_OUTPUT_PROD, args.project, header)
......
......@@ -33,11 +33,11 @@ def make_json(catdir, project, header):
ESM JSON catalog templating based on Jinja2.
"""
print(f'Writing JSON catalog...')
# Get catalog name.
catpath = os.path.join(catdir, f'{project}{JSON_EXTENSION}')
print(f'Writing JSON catalog into {catpath}...')
# Load Jinja2 template for ESM catalogs.
with open('ESMCollectionTemplate.json') as file:
esm_json_template = Template(file.read())
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment