Commit 87b70b68 authored by JOSSOUD Olivier's avatar JOSSOUD Olivier
Browse files

All. Use new logger functions.

parent 4eb916cd
Pipeline #57584 passed with stages
in 1 minute and 33 seconds
......@@ -87,7 +87,7 @@ def extract_compressed_file(compressed_filepath: str, dest_dir: str, delete_comp
return files_in_zip
def compress_file(uncompressed_filepath: str, delete_uncompressed_if_success: bool = True):
def compress_file(uncompressed_filepath: str, delete_uncompressed_if_success: bool = True) -> str:
"""Compress a single file, using LZMA algorithm.
- The output compressed file is created in the same directory as the uncompressed file.
......@@ -99,6 +99,12 @@ def compress_file(uncompressed_filepath: str, delete_uncompressed_if_success: bo
Full file path of the to-be-compressed file.
delete_uncompressed_if_success: bool
If `True` delete the source uncompressed file once the compression is successfully done.
Returns
-------
str
Full file path of the LZMA-compressed output file. Basically the same as `uncompressed_filepath`, with its
extension replaced by `.lzma`.
"""
compressed_filepath = uncompressed_filepath + ".lzma"
zipf = zipfile.ZipFile(compressed_filepath, 'w', zipfile.ZIP_LZMA)
......@@ -109,3 +115,5 @@ def compress_file(uncompressed_filepath: str, delete_uncompressed_if_success: bo
os.remove(uncompressed_filepath)
else:
raise FileNotFoundError("Failed to compress " + uncompressed_filepath)
return compressed_filepath
......@@ -21,7 +21,7 @@ class Collector(utils.LogConfig):
yyyymm: str
Year and month of the to-be-downloaded data
"""
self.logger.write(self.object_id, "Download Maido's Meteo-France meteo data.")
self.logger.info("Download Maido's Meteo-France meteo data.", yyyymm)
# Build source and destination file paths.
source_filepath = self.distant_base_dir + "/" + "PMAIDO_1h_" + yyyymm + ".csv"
......@@ -33,9 +33,9 @@ class Collector(utils.LogConfig):
# Compress
if success:
self.logger.write(self.object_id, yyyymm + ": Compressing ...")
self.logger.info("Compressing ...", yyyymm)
utils.compress_file(dest_filepath)
self.logger.write(self.object_id, yyyymm + ": Done. Archive file: " + dest_filepath)
self.logger.info("Done. Archive file: " + dest_filepath, yyyymm)
def __get_dest_filepath__(self, yyyymm: str) -> str:
# Build destination directory
......
......@@ -30,22 +30,21 @@ class Collector(utils.LogConfig):
day: datetime.date
Date of the data which should be downloaded.
"""
self.logger.write(self.object_id, "Download picarro data from " + site_id)
ftp_session = ftp.connect(self.config[self.object_id], self.logger, self.object_id)
# Build source file path
source_dir = self.distant_base_dir + "/" + site_id + "/picarro/"
source_filename = site_id + "_" + picarro_id + day.strftime("%y%m%d") + ".lzma"
source_filename = site_id + "_" + picarro_id + day.strftime("%Y%m%d") + ".lzma"
source_filepath = source_dir + source_filename
# Check that to-be-downloaded file exists
source_filepaths = ftp.list_files(ftp_session, source_dir, self.logger, self.object_id)
if source_filepath not in source_filepaths:
self.logger.write(self.object_id, "File not found: " + source_filepath)
raise FileNotFoundError(source_filepath)
self.logger.error("File not found: " + source_filepath, day.strftime("%Y-%m-%d"),
exception_type=FileNotFoundError)
# Download file
self.logger.info("Download picarro data from " + site_id, day.strftime("%Y-%m-%d"))
dest_filepath = self.__get_picarro_dest_filepath__(source_filepath)
success = ftp.download_file(source_filepath, dest_filepath, self.logger, self.object_id,
ftp_session=ftp_session)
......@@ -91,7 +90,7 @@ class Collector(utils.LogConfig):
site_id: str
Site's trigram
"""
self.logger.write(self.object_id, "Download hobo data from " + site_id)
self.logger.info("Download hobo data from " + site_id)
ftp_session = ftp.connect(self.config[self.object_id], self.logger, self.object_id)
hobo_distant_path = self.distant_base_dir + "/" + site_id + "/hobo/"
......
......@@ -24,7 +24,6 @@ class Collector(utils.LogConfig):
day: datetime.date
Date of the data which should be downloaded.
"""
self.logger.write(self.object_id, "Download Maido's FTIR meteo data.")
date_str = day.strftime("%Y%m%d")
# Build source and destination file paths
......@@ -32,14 +31,15 @@ class Collector(utils.LogConfig):
dest_filepath = self.__get_dest_filepath__(day)
# Download
self.logger.info("Download Maido's FTIR meteo data.", date_str)
success = ftp.download_file(source_filepath, dest_filepath, self.logger, self.object_id,
ftp_config=self.config[self.object_id])
# Compress
if success:
self.logger.write(self.object_id, date_str + ": Compressing ...")
utils.compress_file(dest_filepath)
self.logger.write(self.object_id, date_str + ": Done. Archive file: " + dest_filepath)
self.logger.info("Compressing " + dest_filepath, date_str)
compressed_filepath = utils.compress_file(dest_filepath)
self.logger.info(self.object_id, date_str + ": Done. Archive file: " + compressed_filepath)
def __get_dest_filepath__(self, day: datetime.date) -> str:
# Build destination directory
......
......@@ -25,9 +25,9 @@ def main():
# Logger
log = logger.Logger(config_parser)
log.write(instrument="", message="------------------------------------------------------------------")
log.write(instrument="", message=" ".join(sys.argv))
log.write(instrument='', message="------------------------------------------------------------------")
log.write(object_id="", msg="------------------------------------------------------------------")
log.write(object_id="", msg=" ".join(sys.argv))
log.write(object_id='', msg="------------------------------------------------------------------")
yesterday = datetime.datetime.now() - datetime.timedelta(1)
......
......@@ -21,7 +21,7 @@ class Visualizator(utils.LogConfig):
source_filepath = os.path.join(source_dir, source_filename)
if not os.path.exists(source_filepath):
self.logger.write(self.object_id, "Failed to open source file " + source_filepath)
self.logger.error("Failed to open source file " + source_filepath, day.strftime("%Y-%m-%d"))
return
# Decompress, and read data as Pandas' DataFrame
......
......@@ -26,7 +26,7 @@ class Collector(utils.LogConfig):
day: datetime.date
Date of the data which should be downloaded.
"""
self.logger.write(self.object_id, "Download DMC Picarro data.")
day_str = day.strftime("%Y-%m-%d")
# Build source file path
picarro_number = re.sub("[^0-9]", "", picarro_id)
......@@ -37,18 +37,23 @@ class Collector(utils.LogConfig):
dest_filepath = self.__get_dest_filepath__(day, source_filepath)
# Download file
self.logger.info("Downloading DMC Picarro file from SFTP server.", day_str)
sftp_client = sftp.connect(self.config[self.object_id], self.logger, self.object_id)
success = sftp.download_file(sftp_client, source_filepath, dest_filepath, self.logger, self.object_id)
sftp_client.close()
if not success:
self.logger.error("FAILED to download file in " + dest_filepath, day_str)
else:
self.logger.info("File downloaded in " + dest_filepath, day_str)
# Re-compress, from ZIP to LZMA
if success:
self.logger.write(self.object_id, day.strftime("%Y-%m-%d") + ": Re-compressing from ZIP to LZMA...")
self.logger.info("Re-compressing from ZIP to LZMA...", day_str)
lzma_filepath = utils.recompress_file(dest_filepath)
if lzma_filepath is None:
self.logger.write(self.object_id, day.strftime("%Y-%m-%d") + ": FAILED to create archive file")
self.logger.error("FAILED to create archive file", day_str)
else:
self.logger.write(self.object_id, day.strftime("%Y-%m-%d") + ": Done. Archive file: " + lzma_filepath)
self.logger.info("Done. Archive file: " + lzma_filepath, day_str)
def __get_dest_filepath__(self, day: datetime.date, source_filepath: str) -> str:
# Build destination directory
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment