Create Collector superclass to manage common log and config_parser.

wimcollect/ftp.py

@@ -3,17 +3,13 @@ import ftplib
 import codecs
 
 import wimcollect.logger as logger
+import wimcollect.utils as utils
 
 
-class FtpCollector:
+class FtpCollector(utils.Collector):
 
-    def __init__(self, config_parser: configobj.ConfigObj, log: logger, object_id: str):
-        # Logger
-        self.logger = log
-        self.object_id = object_id
-
-        # Config
-        self.config = config_parser
+    def __init__(self, object_id: str, config_parser: configobj.ConfigObj = None, log: logger = None):
+        utils.Collector.__init__(self, object_id, config_parser, log)
         self.base_dir = self.config[self.object_id]["root_dir"]
 
     def _ftp_connect_(self) -> ftplib.FTP:

wimcollect/ftpcea.py

@@ -9,9 +9,9 @@ import wimcollect.ftp as ftp
 
 class Collector(ftp.FtpCollector):
 
-    def __init__(self, config_parser: configobj.ConfigObj, log: logger):
+    def __init__(self, config_parser: configobj.ConfigObj = None, log: logger = None):
         self.object_id = "FTPCEA"
-        ftp.FtpCollector.__init__(self, config_parser, log, self.object_id)
+        ftp.FtpCollector.__init__(self, self.object_id, config_parser, log)
 
     ####################################################################################################################
     # Picarro

wimcollect/ftpopar.py

@@ -9,9 +9,9 @@ import wimcollect.ftp as ftp
 
 class Collector(ftp.FtpCollector):
 
-    def __init__(self, config_parser: configobj.ConfigObj, log: logger):
+    def __init__(self, config_parser: configobj.ConfigObj = None, log: logger = None):
         self.object_id = "FTPOPAR"
-        ftp.FtpCollector.__init__(self, config_parser, log, self.object_id)
+        ftp.FtpCollector.__init__(self, self.object_id, config_parser, log)
 
     def download_maido_ftir(self, first_day: datetime.date, last_day: datetime.date):
         """Download all Picarro files from FTP server.

wimcollect/httpddu.py

@@ -7,15 +7,10 @@ import wimcollect.utils as utils
 import wimcollect.logger as logger
 
 
-class Collector:
+class Collector(utils.Collector):
 
-    def __init__(self, config_parser: configobj.ConfigObj, log: logger):
-        # Logger
-        self.logger = log
-        self.object_id = "HTTPDDU"
-
-        # Config
-        self.config = config_parser
+    def __init__(self, config_parser: configobj.ConfigObj = None, log: logger = None):
+        utils.Collector.__init__(self, "HTTPDDU", config_parser, log)
         self.base_url = self.config[self.object_id]["base_url"]
 
     def download_picarro(self, day: datetime.date):

wimcollect/main.py

@@ -30,13 +30,13 @@ def main():
 
     yesterday = datetime.datetime.now() - datetime.timedelta(1)
 
-    httpddu_col = httpddu.Collector(config_parser, log)
+    httpddu_col = httpddu.Collector()
     httpddu_col.download_picarro(yesterday)
 
-    sshdmc_col = sshdmc.Collector(config_parser, log)
+    sshdmc_col = sshdmc.Collector()
     sshdmc_col.download_picarro(yesterday)
 
-    vizual = quickviz.Visualizator(config_parser, log)
+    vizual = quickviz.Visualizator()
     vizual.create_data_source_file("DDU", "HIDS2189", yesterday)
     vizual.create_data_source_file("DMC", "HIDS2319", yesterday)
 

wimcollect/sshdmc.py

@@ -8,15 +8,10 @@ import wimcollect.utils as utils
 import wimcollect.logger as logger
 
 
-class Collector:
+class Collector(utils.Collector):
 
-    def __init__(self, config_parser: configobj.ConfigObj, log: logger):
-        # Logger
-        self.logger = log
-        self.object_id = "SSHDMC"
-
-        # Config
-        self.config = config_parser
+    def __init__(self, config_parser: configobj.ConfigObj = None, log: logger = None):
+        utils.Collector.__init__(self, "SSHDMC", config_parser, log)
         self.base_dir = self.config[self.object_id]["root_dir"]
 
     def download_picarro(self, day: datetime.date):
@@ -79,6 +74,11 @@ class Collector:
             return client.open_sftp()
 
     def __sftp_download__(self, sftp_client: paramiko.SFTPClient, source_filepath: str, dest_filepath: str) -> bool:
-        sftp_client.get(source_filepath, dest_filepath)
+        try:
+            sftp_client.get(source_filepath, dest_filepath)
+        except FileNotFoundError:
+            msg = "File not found: " + source_filepath
+            self.logger.write(self.object_id, msg)
+            raise FileNotFoundError(msg)
         success = os.path.exists(dest_filepath)
         return success

wimcollect/utils.py

 import os
-import re
+import sys
 import configobj
-import datetime
+import pkgutil
 import zipfile
-import paramiko
 
 import wimcollect.logger as logger
 
 
+class Collector:
+    def __init__(self, object_id: str, config_parser: configobj.ConfigObj = None, log: logger = None):
+        # Config
+        if config_parser is None:
+            self.config = get_config_parser()
+        else:
+            self.config = config_parser
+
+        # Logger
+        if log is None:
+            self.logger = logger.Logger(self.config)
+        else:
+            self.logger = log
+        self.object_id = object_id
+
+
+def get_config_parser() -> configobj.ConfigObj:
+    # Configuration file
+    pkgpath = os.path.dirname(pkgutil.get_loader("wimcollect").path)
+    conf_file_path = os.path.join(pkgpath, "config", "settings.ini")
+    if not os.path.exists(conf_file_path):
+        msg = "Configuration file not found in [" + conf_file_path + "]"
+        sys.stderr.write(msg)
+        raise FileNotFoundError(msg)
+    config_parser = configobj.ConfigObj(conf_file_path)
+    return config_parser
+
 def recompress_file(zip_filepath: str):
     # Directory where the original ZIP is, where the ZIP content will temporarily be extracted and where the final
     # LZMA file will be created.