FTP CEA.

wimcollect/ftp.py

+import configobj
+import ftplib
+import codecs
+
+import wimcollect.logger as logger
+
+
+class FtpCollector:
+
+    def __init__(self, config_parser: configobj.ConfigObj, log: logger, object_id: str):
+        # Logger
+        self.logger = log
+        self.object_id = object_id
+
+        # Config
+        self.config = config_parser
+        self.base_dir = self.config[self.object_id]["root_dir"]
+
+    def _ftp_connect_(self):
+        try:
+            self.session = ftplib.FTP(host=self.config[self.object_id]["host"],
+                                      user=self.config[self.object_id]["user"],
+                                      passwd=codecs.decode(self.config[self.object_id]["password"], "rot_13"))
+        except ftplib.error_perm as e:
+            self.logger.write(self.object_id, "Failed to connect to FTP: " + str(e))
+        except BlockingIOError as e:
+            self.logger.write(self.object_id, "Host [" + self.config[self.object_id]["host"] + "] seems to be unreachable.")
+        else:
+            self.logger.write(self.object_id,
+                              "Connected to [" + self.config[self.object_id]["host"] + "] as user: "
+                              + self.config[self.object_id]["user"])
+
+    def _ftp_list_files_(self, ftp_session: ftplib.FTP, path: str) -> list:
+        filepaths = []
+
+        try:
+            filepaths = ftp_session.nlst(path)
+        except ftplib.error_perm as resp:
+            if str(resp) == "550 No files found":
+                self.logger.write(self.object_id, "No files in directory " + path)
+            else:
+                raise
+
+        return filepaths
+
+    def _ftp_download_file(self, ftp_session: ftplib.FTP,
+                           source_filepath: str, dest_filepath: str,
+                           delete_if_success: bool = False) -> None:
+        self.logger.write(self.object_id, "Downloading " + source_filepath + " ...")
+        with open(dest_filepath, 'wb') as file:
+            response = ftp_session.retrbinary('RETR %s' % source_filepath, file.write)
+        if response == "226 Transfer complete.":
+            if delete_if_success:
+                ftp_session.delete(source_filepath)
+            self.logger.write(self.object_id, "Done. Destination file: " + dest_filepath)
+        else:
+            self.logger.write(self.object_id, "FAILED. Destination file: " + dest_filepath)

wimcollect/ftpcea.py

+import os
+import re
+import configobj
+import datetime
+
+import wimcollect.logger as logger
+import wimcollect.ftp as ftp
+
+
+class Collector(ftp.FtpCollector):
+
+    def __init__(self, config_parser: configobj.ConfigObj, log: logger):
+        self.object_id = "FTPCEA"
+        ftp.FtpCollector.__init__(self, config_parser, log, self.object_id)
+
+    ####################################################################################################################
+    # Picarro
+
+    def download_picarro(self, site_id: str):
+        """Download all Picarro files from FTP server.
+
+        The distant files will be deleted from the server if the transfer is successful.
+
+        Parameters
+        ----------
+        site_id: str
+            Site's trigram
+        """
+        self.logger.write(self.object_id, "Download picarro data from " + site_id)
+
+        self._ftp_connect_()
+        picarro_distant_path = self.base_dir + "/" + site_id + "/picarro/"
+        self.session.cwd(picarro_distant_path)
+
+        source_filepaths = self._ftp_list_files_(self.session, picarro_distant_path)
+        for source_filepath in source_filepaths:
+            dest_filepath = self.__get_picarro_dest_filepath__(source_filepath)
+            self._ftp_download_file(self.session, source_filepath, dest_filepath, delete_if_success=True)
+
+        self.session.quit()
+
+    def __get_picarro_dest_filepath__(self, source_filepath: str) -> str:
+        """Get the full path where the Picarro data file should be downloaded.
+
+        Parameters
+        ----------
+        source_filepath: str
+            Full path of the distant (FTP) to-be-downloaded Picarro data file.
+
+        Returns
+        -------
+        str
+            Local destination file path.
+        """
+        filename = os.path.basename(source_filepath)
+        site_id, picarro_id, date_str = re.split('_|\.', filename)[0:3]
+        date = datetime.datetime.strptime(date_str, "%Y%m%d").date()
+
+        dest_dir = os.path.join(self.config["LOCAL"]["base_dir"], site_id, "picarro", picarro_id, str(date.year))
+        if not os.path.exists(dest_dir):
+            os.makedirs(dest_dir)
+        dest_filepath = os.path.join(dest_dir, filename)
+
+        return dest_filepath
+
+    ####################################################################################################################
+    # Hobo
+
+    def download_hobo(self, site_id: str):
+        """Download all Hobo files from FTP server.
+
+        The distant files will be deleted from the server if the transfer is successful.
+
+        Parameters
+        ----------
+        site_id: str
+            Site's trigram
+        """
+        self.logger.write(self.object_id, "Download hobo data from " + site_id)
+
+        self._ftp_connect_()
+        hobo_distant_path = self.base_dir + "/" + site_id + "/hobo/"
+        self.session.cwd(hobo_distant_path)
+
+        source_filepaths = self._ftp_list_files_(self.session, hobo_distant_path)
+        for source_filepath in source_filepaths:
+            dest_filepath = self.__get_hobo_dest_filepath__(source_filepath)
+            self._ftp_download_file(self.session, source_filepath, dest_filepath, delete_if_success=True)
+
+        self.session.quit()
+
+    def __get_hobo_dest_filepath__(self, source_filepath: str):
+        """Get the full path where the Hobo data file should be downloaded.
+
+        Parameters
+        ----------
+        source_filepath: str
+            Full path of the distant (FTP) to-be-downloaded Hobo data file.
+
+        Returns
+        -------
+        str
+            Local destination file path.
+        """
+        filename = os.path.basename(source_filepath)
+        filename_parts = re.split('_|\.|-', filename)
+        site_id = filename_parts[0]
+        hobo_id = filename_parts[2]
+
+        dest_dir = os.path.join(self.config["LOCAL"]["base_dir"], site_id, "meteo", "hobo_" + hobo_id)
+        if not os.path.exists(dest_dir):
+            os.makedirs(dest_dir)
+        dest_filepath = os.path.join(dest_dir, filename)
+
+        return dest_filepath
+
+
+
+