import os import re import configobj import datetime import wimcollect.common.utils as utils import wimcollect.common.logger as logger import wimcollect.common.sftp as sftp class Collector(utils.LogConfig): """Collector for Dome C / Concordia station (trigram: DMC) data, hosted on Hermes server, managed by italian's PNRA. """ def __init__(self, config_parser: configobj.ConfigObj = None, log: logger = None): utils.LogConfig.__init__(self, "SFTPDMC", config_parser, log) self.distant_base_dir = self.config[self.object_id]["distant_base_dir"] def download_picarro(self, picarro_id: str, day: datetime.date): """Download Picarro data file from SFTP server. Parameters ---------- picarro_id: str Unique identifier of the Picarro whose data should be retrieved (e.g. HIDS2319) day: datetime.date Date of the data which should be downloaded. """ day_str = day.strftime("%Y-%m-%d") # Build source file path picarro_number = re.sub("[^0-9]", "", picarro_id) source_filepath = self.distant_base_dir + "/" + picarro_number \ + "/DMC_" + picarro_id + "_" + day.strftime("%Y%m%d") + ".zip" # Build destination file path dest_filepath = self.__get_dest_filepath__(day, source_filepath) # Download file self.logger.info("Downloading DMC Picarro file from SFTP server.", day_str) sftp_client = sftp.connect(self.config[self.object_id], self.logger, self.object_id) success = sftp.download_file(sftp_client, source_filepath, dest_filepath, self.logger, self.object_id) sftp_client.close() if not success: self.logger.error("FAILED to download file in " + dest_filepath, day_str) else: self.logger.info("File downloaded in " + dest_filepath, day_str) # Re-compress, from ZIP to LZMA if success: self.logger.info("Re-compressing from ZIP to LZMA...", day_str) lzma_filepath = utils.recompress_file(dest_filepath) if lzma_filepath is None: self.logger.error("FAILED to create archive file", day_str) else: self.logger.info("Done. Archive file: " + lzma_filepath, day_str) def __get_dest_filepath__(self, day: datetime.date, source_filepath: str) -> str: # Build destination directory dest_dir = os.path.join(self.config["LOCAL"]["base_dir"], "DMC", "picarro", self.config[self.object_id]["picarro_id"], str(day.year)) if not os.path.exists(dest_dir): os.makedirs(dest_dir) # Build destination filename dest_filename = os.path.basename(source_filepath) # Build full file path dest_filepath = os.path.join(dest_dir, dest_filename) return dest_filepath