Commit e87e8b19 authored by Guillaume's avatar Guillaume
Browse files

Bugfixes

parent 1988e0cd
......@@ -100,7 +100,7 @@ def get_facets_from_filename(self, basename):
# Check vocabulary.
try:
terms = self.file_parser.parse(basename)
self.file_parser.parse(basename)
timerange = TimeRange(basename.split('_')[-1])
tstart, tend = timerange.start, timerange.end
......
#!bin/python
# -*- coding: utf-8 -*-
import glob
import re
import subprocess
import tarfile
......@@ -8,8 +9,8 @@ from datetime import date
from glob import iglob
from multiprocessing import Pool
from pathlib import Path
from shutil import copy
from shutil import copy, copyfileobj
from glob import glob
from DRSParser import DRSParser
from utils import *
......@@ -241,10 +242,12 @@ def main():
# Write JSON catalog.
make_json(CATALOG_OUTPUT_PROD, args.project, header)
# Concatenate CSV files into one compressed CSV catalog.
# Concatenate CSV files into one CSV catalog.
print(f'Concatenating CSV catalogs...')
subprocess.check_call(
f'cat {CATALOG_OUTPUT}/{args.project}/*{CSV_EXTENSION} > {CATALOG_OUTPUT_PROD}/{args.project}{CSV_EXTENSION}')
with open(f'{CATALOG_OUTPUT_PROD}/{args.project}{CSV_EXTENSION}', 'w') as fout:
for f in glob(f'{CATALOG_OUTPUT}/{args.project}/*{CSV_EXTENSION}'):
with open(f, 'r') as fin:
copyfileobj(fin, fout)
# Compress CSV catalog.
print(f'Compressing final CSV catalog...')
......@@ -254,7 +257,7 @@ def main():
# Backup CSV catalog.
print(f'Backup CSV compressed catalog...')
copy(src=f'{CATALOG_OUTPUT_PROD}/{args.project}{CSV_EXTENSION}.gz',
dst=f'{CATALOG_OUTPUT_BACKUP}/{args.project}_{date.today().strftime("%y%m%d")}_{CSV_EXTENSION}.gz')
dst=f'{CATALOG_OUTPUT_BACKUP}/{args.project}_{date.today().strftime("%Y%m%d")}{CSV_EXTENSION}.gz')
print(f'{args.project} intake catalogs complete.')
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment