import os import re import configobj import datetime import wimcollect.common.logger as logger import wimcollect.common.ftp as ftp import wimcollect.common.utils as utils class Collector(utils.LogConfig): def __init__(self, config_parser: configobj.ConfigObj = None, log: logger = None): self.object_id = "FTPCEA" utils.LogConfig.__init__(self, self.object_id, config_parser, log) self.distant_base_dir = self.config[self.object_id]["distant_base_dir"] #################################################################################################################### # Picarro def download_picarro(self, site_id: str, picarro_id: str, day: datetime.date): """Download Picarro file from FTP server. Parameters ---------- site_id: str Site's trigram (AMS, SRT, etc.) picarro_id: str Picarro unique identifier (HIDS2189, HIDS2108, etc.) day: datetime.date Date of the data which should be downloaded. """ self.logger.write(self.object_id, "Download picarro data from " + site_id) ftp_session = ftp.ftp_connect(self.config[self.object_id], self.logger, self.object_id) # Build source file path source_dir = self.distant_base_dir + "/" + site_id + "/picarro/" source_filename = site_id + "_" + picarro_id + day.strftime("%y%m%d") + ".lzma" source_filepath = source_dir + source_filename # Check that to-be-downloaded file exists source_filepaths = ftp.ftp_list_files(ftp_session, source_dir, self.logger, self.object_id) if source_filepath not in source_filepaths: self.logger.write(self.object_id, "File not found: " + source_filepath) raise FileNotFoundError(source_filepath) # Download file dest_filepath = self.__get_picarro_dest_filepath__(source_filepath) success = ftp.ftp_download_file(source_filepath, dest_filepath, self.logger, self.object_id, ftp_session=ftp_session) ftp_session.quit() return success def __get_picarro_dest_filepath__(self, source_filepath: str) -> str: """Get the full path where the Picarro data file should be downloaded. Parameters ---------- source_filepath: str Full path of the distant (FTP) to-be-downloaded Picarro data file. Returns ------- str Local destination file path. """ filename = os.path.basename(source_filepath) site_id, picarro_id, date_str = re.split('_|\.', filename)[0:3] date = datetime.datetime.strptime(date_str, "%Y%m%d").date() dest_dir = os.path.join(self.config["LOCAL"]["base_dir"], site_id, "picarro", picarro_id, str(date.year)) if not os.path.exists(dest_dir): os.makedirs(dest_dir) dest_filepath = os.path.join(dest_dir, filename) return dest_filepath #################################################################################################################### # Hobo def download_hobo(self, site_id: str): """Download all Hobo files from FTP server. The distant files will be deleted from the server if the transfer is successful. Parameters ---------- site_id: str Site's trigram """ self.logger.write(self.object_id, "Download hobo data from " + site_id) ftp_session = ftp.ftp_connect(self.config[self.object_id], self.logger, self.object_id) hobo_distant_path = self.distant_base_dir + "/" + site_id + "/hobo/" source_filepaths = ftp.ftp_list_files(ftp_session, hobo_distant_path, self.logger, self.object_id) for source_filepath in source_filepaths: dest_filepath = self.__get_hobo_dest_filepath__(source_filepath) ftp.ftp_download_file(source_filepath, dest_filepath, self.logger, self.object_id, ftp_session=ftp_session) ftp_session.quit() def __get_hobo_dest_filepath__(self, source_filepath: str): """Get the full path where the Hobo data file should be downloaded. Parameters ---------- source_filepath: str Full path of the distant (FTP) to-be-downloaded Hobo data file. Returns ------- str Local destination file path. """ filename = os.path.basename(source_filepath) filename_parts = re.split('_|\.|-', filename) site_id = filename_parts[0] hobo_id = filename_parts[2] dest_dir = os.path.join(self.config["LOCAL"]["base_dir"], site_id, "meteo", "hobo_" + hobo_id) if not os.path.exists(dest_dir): os.makedirs(dest_dir) dest_filepath = os.path.join(dest_dir, filename) return dest_filepath