Commit 0b4040ac authored by JOSSOUD Olivier's avatar JOSSOUD Olivier
Browse files

Create Collector superclass to manage common log and config_parser.

parent 662aeb9d
Pipeline #57022 passed with stages
in 2 minutes and 33 seconds
......@@ -3,17 +3,13 @@ import ftplib
import codecs
import wimcollect.logger as logger
import wimcollect.utils as utils
class FtpCollector:
class FtpCollector(utils.Collector):
def __init__(self, config_parser: configobj.ConfigObj, log: logger, object_id: str):
# Logger
self.logger = log
self.object_id = object_id
# Config
self.config = config_parser
def __init__(self, object_id: str, config_parser: configobj.ConfigObj = None, log: logger = None):
utils.Collector.__init__(self, object_id, config_parser, log)
self.base_dir = self.config[self.object_id]["root_dir"]
def _ftp_connect_(self) -> ftplib.FTP:
......
......@@ -9,9 +9,9 @@ import wimcollect.ftp as ftp
class Collector(ftp.FtpCollector):
def __init__(self, config_parser: configobj.ConfigObj, log: logger):
def __init__(self, config_parser: configobj.ConfigObj = None, log: logger = None):
self.object_id = "FTPCEA"
ftp.FtpCollector.__init__(self, config_parser, log, self.object_id)
ftp.FtpCollector.__init__(self, self.object_id, config_parser, log)
####################################################################################################################
# Picarro
......
......@@ -9,9 +9,9 @@ import wimcollect.ftp as ftp
class Collector(ftp.FtpCollector):
def __init__(self, config_parser: configobj.ConfigObj, log: logger):
def __init__(self, config_parser: configobj.ConfigObj = None, log: logger = None):
self.object_id = "FTPOPAR"
ftp.FtpCollector.__init__(self, config_parser, log, self.object_id)
ftp.FtpCollector.__init__(self, self.object_id, config_parser, log)
def download_maido_ftir(self, first_day: datetime.date, last_day: datetime.date):
"""Download all Picarro files from FTP server.
......
......@@ -7,15 +7,10 @@ import wimcollect.utils as utils
import wimcollect.logger as logger
class Collector:
class Collector(utils.Collector):
def __init__(self, config_parser: configobj.ConfigObj, log: logger):
# Logger
self.logger = log
self.object_id = "HTTPDDU"
# Config
self.config = config_parser
def __init__(self, config_parser: configobj.ConfigObj = None, log: logger = None):
utils.Collector.__init__(self, "HTTPDDU", config_parser, log)
self.base_url = self.config[self.object_id]["base_url"]
def download_picarro(self, day: datetime.date):
......
......@@ -30,13 +30,13 @@ def main():
yesterday = datetime.datetime.now() - datetime.timedelta(1)
httpddu_col = httpddu.Collector(config_parser, log)
httpddu_col = httpddu.Collector()
httpddu_col.download_picarro(yesterday)
sshdmc_col = sshdmc.Collector(config_parser, log)
sshdmc_col = sshdmc.Collector()
sshdmc_col.download_picarro(yesterday)
vizual = quickviz.Visualizator(config_parser, log)
vizual = quickviz.Visualizator()
vizual.create_data_source_file("DDU", "HIDS2189", yesterday)
vizual.create_data_source_file("DMC", "HIDS2319", yesterday)
......
......@@ -8,15 +8,10 @@ import wimcollect.utils as utils
import wimcollect.logger as logger
class Collector:
class Collector(utils.Collector):
def __init__(self, config_parser: configobj.ConfigObj, log: logger):
# Logger
self.logger = log
self.object_id = "SSHDMC"
# Config
self.config = config_parser
def __init__(self, config_parser: configobj.ConfigObj = None, log: logger = None):
utils.Collector.__init__(self, "SSHDMC", config_parser, log)
self.base_dir = self.config[self.object_id]["root_dir"]
def download_picarro(self, day: datetime.date):
......@@ -79,6 +74,11 @@ class Collector:
return client.open_sftp()
def __sftp_download__(self, sftp_client: paramiko.SFTPClient, source_filepath: str, dest_filepath: str) -> bool:
sftp_client.get(source_filepath, dest_filepath)
try:
sftp_client.get(source_filepath, dest_filepath)
except FileNotFoundError:
msg = "File not found: " + source_filepath
self.logger.write(self.object_id, msg)
raise FileNotFoundError(msg)
success = os.path.exists(dest_filepath)
return success
import os
import re
import sys
import configobj
import datetime
import pkgutil
import zipfile
import paramiko
import wimcollect.logger as logger
class Collector:
def __init__(self, object_id: str, config_parser: configobj.ConfigObj = None, log: logger = None):
# Config
if config_parser is None:
self.config = get_config_parser()
else:
self.config = config_parser
# Logger
if log is None:
self.logger = logger.Logger(self.config)
else:
self.logger = log
self.object_id = object_id
def get_config_parser() -> configobj.ConfigObj:
# Configuration file
pkgpath = os.path.dirname(pkgutil.get_loader("wimcollect").path)
conf_file_path = os.path.join(pkgpath, "config", "settings.ini")
if not os.path.exists(conf_file_path):
msg = "Configuration file not found in [" + conf_file_path + "]"
sys.stderr.write(msg)
raise FileNotFoundError(msg)
config_parser = configobj.ConfigObj(conf_file_path)
return config_parser
def recompress_file(zip_filepath: str):
# Directory where the original ZIP is, where the ZIP content will temporarily be extracted and where the final
# LZMA file will be created.
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment