Commit 0b4040ac authored by JOSSOUD Olivier's avatar JOSSOUD Olivier
Browse files

Create Collector superclass to manage common log and config_parser.

parent 662aeb9d
Pipeline #57022 passed with stages
in 2 minutes and 33 seconds
...@@ -3,17 +3,13 @@ import ftplib ...@@ -3,17 +3,13 @@ import ftplib
import codecs import codecs
import wimcollect.logger as logger import wimcollect.logger as logger
import wimcollect.utils as utils
class FtpCollector: class FtpCollector(utils.Collector):
def __init__(self, config_parser: configobj.ConfigObj, log: logger, object_id: str): def __init__(self, object_id: str, config_parser: configobj.ConfigObj = None, log: logger = None):
# Logger utils.Collector.__init__(self, object_id, config_parser, log)
self.logger = log
self.object_id = object_id
# Config
self.config = config_parser
self.base_dir = self.config[self.object_id]["root_dir"] self.base_dir = self.config[self.object_id]["root_dir"]
def _ftp_connect_(self) -> ftplib.FTP: def _ftp_connect_(self) -> ftplib.FTP:
......
...@@ -9,9 +9,9 @@ import wimcollect.ftp as ftp ...@@ -9,9 +9,9 @@ import wimcollect.ftp as ftp
class Collector(ftp.FtpCollector): class Collector(ftp.FtpCollector):
def __init__(self, config_parser: configobj.ConfigObj, log: logger): def __init__(self, config_parser: configobj.ConfigObj = None, log: logger = None):
self.object_id = "FTPCEA" self.object_id = "FTPCEA"
ftp.FtpCollector.__init__(self, config_parser, log, self.object_id) ftp.FtpCollector.__init__(self, self.object_id, config_parser, log)
#################################################################################################################### ####################################################################################################################
# Picarro # Picarro
......
...@@ -9,9 +9,9 @@ import wimcollect.ftp as ftp ...@@ -9,9 +9,9 @@ import wimcollect.ftp as ftp
class Collector(ftp.FtpCollector): class Collector(ftp.FtpCollector):
def __init__(self, config_parser: configobj.ConfigObj, log: logger): def __init__(self, config_parser: configobj.ConfigObj = None, log: logger = None):
self.object_id = "FTPOPAR" self.object_id = "FTPOPAR"
ftp.FtpCollector.__init__(self, config_parser, log, self.object_id) ftp.FtpCollector.__init__(self, self.object_id, config_parser, log)
def download_maido_ftir(self, first_day: datetime.date, last_day: datetime.date): def download_maido_ftir(self, first_day: datetime.date, last_day: datetime.date):
"""Download all Picarro files from FTP server. """Download all Picarro files from FTP server.
......
...@@ -7,15 +7,10 @@ import wimcollect.utils as utils ...@@ -7,15 +7,10 @@ import wimcollect.utils as utils
import wimcollect.logger as logger import wimcollect.logger as logger
class Collector: class Collector(utils.Collector):
def __init__(self, config_parser: configobj.ConfigObj, log: logger): def __init__(self, config_parser: configobj.ConfigObj = None, log: logger = None):
# Logger utils.Collector.__init__(self, "HTTPDDU", config_parser, log)
self.logger = log
self.object_id = "HTTPDDU"
# Config
self.config = config_parser
self.base_url = self.config[self.object_id]["base_url"] self.base_url = self.config[self.object_id]["base_url"]
def download_picarro(self, day: datetime.date): def download_picarro(self, day: datetime.date):
......
...@@ -30,13 +30,13 @@ def main(): ...@@ -30,13 +30,13 @@ def main():
yesterday = datetime.datetime.now() - datetime.timedelta(1) yesterday = datetime.datetime.now() - datetime.timedelta(1)
httpddu_col = httpddu.Collector(config_parser, log) httpddu_col = httpddu.Collector()
httpddu_col.download_picarro(yesterday) httpddu_col.download_picarro(yesterday)
sshdmc_col = sshdmc.Collector(config_parser, log) sshdmc_col = sshdmc.Collector()
sshdmc_col.download_picarro(yesterday) sshdmc_col.download_picarro(yesterday)
vizual = quickviz.Visualizator(config_parser, log) vizual = quickviz.Visualizator()
vizual.create_data_source_file("DDU", "HIDS2189", yesterday) vizual.create_data_source_file("DDU", "HIDS2189", yesterday)
vizual.create_data_source_file("DMC", "HIDS2319", yesterday) vizual.create_data_source_file("DMC", "HIDS2319", yesterday)
......
...@@ -8,15 +8,10 @@ import wimcollect.utils as utils ...@@ -8,15 +8,10 @@ import wimcollect.utils as utils
import wimcollect.logger as logger import wimcollect.logger as logger
class Collector: class Collector(utils.Collector):
def __init__(self, config_parser: configobj.ConfigObj, log: logger): def __init__(self, config_parser: configobj.ConfigObj = None, log: logger = None):
# Logger utils.Collector.__init__(self, "SSHDMC", config_parser, log)
self.logger = log
self.object_id = "SSHDMC"
# Config
self.config = config_parser
self.base_dir = self.config[self.object_id]["root_dir"] self.base_dir = self.config[self.object_id]["root_dir"]
def download_picarro(self, day: datetime.date): def download_picarro(self, day: datetime.date):
...@@ -79,6 +74,11 @@ class Collector: ...@@ -79,6 +74,11 @@ class Collector:
return client.open_sftp() return client.open_sftp()
def __sftp_download__(self, sftp_client: paramiko.SFTPClient, source_filepath: str, dest_filepath: str) -> bool: def __sftp_download__(self, sftp_client: paramiko.SFTPClient, source_filepath: str, dest_filepath: str) -> bool:
sftp_client.get(source_filepath, dest_filepath) try:
sftp_client.get(source_filepath, dest_filepath)
except FileNotFoundError:
msg = "File not found: " + source_filepath
self.logger.write(self.object_id, msg)
raise FileNotFoundError(msg)
success = os.path.exists(dest_filepath) success = os.path.exists(dest_filepath)
return success return success
import os import os
import re import sys
import configobj import configobj
import datetime import pkgutil
import zipfile import zipfile
import paramiko
import wimcollect.logger as logger import wimcollect.logger as logger
class Collector:
def __init__(self, object_id: str, config_parser: configobj.ConfigObj = None, log: logger = None):
# Config
if config_parser is None:
self.config = get_config_parser()
else:
self.config = config_parser
# Logger
if log is None:
self.logger = logger.Logger(self.config)
else:
self.logger = log
self.object_id = object_id
def get_config_parser() -> configobj.ConfigObj:
# Configuration file
pkgpath = os.path.dirname(pkgutil.get_loader("wimcollect").path)
conf_file_path = os.path.join(pkgpath, "config", "settings.ini")
if not os.path.exists(conf_file_path):
msg = "Configuration file not found in [" + conf_file_path + "]"
sys.stderr.write(msg)
raise FileNotFoundError(msg)
config_parser = configobj.ConfigObj(conf_file_path)
return config_parser
def recompress_file(zip_filepath: str): def recompress_file(zip_filepath: str):
# Directory where the original ZIP is, where the ZIP content will temporarily be extracted and where the final # Directory where the original ZIP is, where the ZIP content will temporarily be extracted and where the final
# LZMA file will be created. # LZMA file will be created.
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment