Commit f43d142f authored by JOSSOUD Olivier's avatar JOSSOUD Olivier
Browse files

Gather common modules in `common` package. Refactor FTP.

parent 6303264b
Pipeline #57275 passed with stages
in 1 minute and 8 seconds
import ftplib
import codecs
import re
import wimcollect.common.logger as logger
def ftp_connect(ftp_config: dict, log: logger.Logger, object_id: str) -> ftplib.FTP:
"""Connect to FTP server according to parameters read from settings.ini file.
Parameters
----------
ftp_config: dict
Dict containing FTP connection information: `host`, `user` and `password`.
log: wimcollect.logger
Logger to record debug/info/error messages.
object_id: str
Object identifier, used as marker for log messages.
Returns
-------
ftplib.FTP
FTP session.
"""
host = ftp_config["host"]
user = ftp_config["user"]
passwd = codecs.decode(ftp_config["password"], "rot_13")
try:
session = ftplib.FTP(host=host, user=user, passwd=passwd)
except ftplib.error_perm as e:
log.write(object_id, "Failed to connect to " + host + " as " + user + ": " + str(e))
raise
except BlockingIOError as e:
log.write(object_id, "Host [" + host + "] seems to be unreachable.")
raise
else:
log.write(object_id, "Connected to [" + host + "] as user: " + user)
return session
def _ftp_list_files_(ftp_session: ftplib.FTP, distant_directory_path: str, log: logger.Logger, object_id: str) -> list:
"""List the files in the distant FTP directory.
Parameters
----------
ftp_session: ftplib.FTP
FTP session.
distant_directory_path: str
Full path of the directory whose content should be listed.
log: wimcollect.logger
Logger to record debug/info/error messages.
object_id: str
Object identifier, used as marker for log messages.
Returns
-------
list
List of files in the FTP directory given in parameter.
"""
filepaths = []
try:
filepaths = ftp_session.nlst(distant_directory_path)
except ftplib.error_perm as resp:
if str(resp) == "550 No files found":
log.write(object_id, "No files in directory " + distant_directory_path)
else:
raise
return filepaths
def _ftp_download_file(source_filepath: str, dest_filepath: str,
log: logger.Logger, object_id: str,
ftp_config: dict = None, ftp_session: ftplib.FTP = None,
delete_if_success: bool = False) -> bool:
"""Download a file from the FTP server.
Parameters
----------
source_filepath: str
Full path of the distant to-be_downloaded file.
dest_filepath: str
Full path where the downloaded file should be stored.
log: wimcollect.logger
Logger to record debug/info/error messages.
object_id: str
Object identifier, used as marker for log messages.
ftp_config: dict, optional
Dict containing FTP connection information: `host`, `user` and `password`. If not set, `ftp_session` must be
provided.
ftp_session: ftplib.FTP, optional
FTP session. If not set, a session will be created using `ftp_config` parameters. Specifying a
custom ftp_session is useful when multiple tasks should be performed before/after downloading a single file.
delete_if_success: bool, optional
If `True`, the distant FTP file will be deleted from the server, once the transfer is successfully done.
Returns
-------
bool
`True` if everything went well, otherwise raises Exception.
"""
# Open FTP session if missing
if ftp_session is None:
close_session = True
ftp_session = ftp_connect(ftp_config, log, object_id)
else:
close_session = False
# Try to download file
log.write(object_id, "Downloading " + source_filepath + " ...")
with open(dest_filepath, 'wb') as file:
try:
response = ftp_session.retrbinary('RETR %s' % source_filepath, file.write)
except (ftplib.error_perm, ftplib.error_temp) as e:
response = str(e)
# Analyse FTP's response and emit messages accordingly
response_code = int(re.sub("[^0-9]", "", response))
if response_code == 226:
if delete_if_success:
ftp_session.delete(source_filepath)
log.write(object_id, "Done. Destination file: " + dest_filepath)
else:
log.write(object_id, "FAILED. Destination file: " + dest_filepath)
raise Exception("Failed to download [" + source_filepath + "] in [" + dest_filepath + "]."
" FTP response: [" + response + "]")
# Close the FTP session if it has been opened at the beginning of this function.
if close_session:
ftp_session.quit()
return True
......@@ -5,7 +5,7 @@ import pkgutil
import zipfile
import tempfile
import wimcollect.logger as logger
import wimcollect.common.logger as logger
class LogConfig:
......
import configobj
import ftplib
import codecs
import re
import wimcollect.logger as logger
import wimcollect.utils as utils
class FtpCollector(utils.LogConfig):
def __init__(self, object_id: str, config_parser: configobj.ConfigObj = None, log: logger = None):
utils.LogConfig.__init__(self, object_id, config_parser, log)
self.base_dir = self.config[self.object_id]["root_dir"]
def _ftp_connect_(self) -> ftplib.FTP:
"""Connect to FTP server according to parameters read from settings.ini file.
Returns
-------
ftplib.FTP
FTP session.
"""
try:
session = ftplib.FTP(host=self.config[self.object_id]["host"],
user=self.config[self.object_id]["user"],
passwd=codecs.decode(self.config[self.object_id]["password"], "rot_13"))
except ftplib.error_perm as e:
self.logger.write(self.object_id, "Failed to connect to FTP: " + str(e))
raise
except BlockingIOError as e:
self.logger.write(self.object_id, "Host [" + self.config[self.object_id]["host"] + "] seems to be unreachable.")
raise
else:
self.logger.write(self.object_id,
"Connected to [" + self.config[self.object_id]["host"] + "] as user: "
+ self.config[self.object_id]["user"])
return session
def _ftp_list_files_(self, ftp_session: ftplib.FTP, distant_directory_path: str) -> list:
"""List the files in the distant FTP directory.
Parameters
----------
ftp_session: ftplib.FTP
FTP session.
distant_directory_path: str
Full path of the directory whose content should be listed.
Returns
-------
list
List of files in the FTP directory given in parameter.
"""
filepaths = []
try:
filepaths = ftp_session.nlst(distant_directory_path)
except ftplib.error_perm as resp:
if str(resp) == "550 No files found":
self.logger.write(self.object_id, "No files in directory " + distant_directory_path)
else:
raise
return filepaths
def _ftp_download_file(self,
source_filepath: str, dest_filepath: str,
ftp_session: ftplib.FTP = None,
delete_if_success: bool = False) -> bool:
"""Download a file from the FTP server.
Parameters
----------
source_filepath: str
Full path of the distant to-be_downloaded file.
dest_filepath: str
Full path where the downloaded file should be stored.
ftp_session: ftplib.FTP, optional
FTP session. If `None` or not set, a session will be created using settings.ini parameters. Specifying a
custom ftp_session is useful when multiple tasks should be performed before/after downloading a single file.
delete_if_success: bool, optional
If `True`, the distant FTP file will be deleted from the server, once the transfer is successfully done.
Returns
-------
bool
`True` if everything went well, otherwise raises Exception.
"""
# Open FTP session if missing
if ftp_session is None:
close_session = True
ftp_session = self._ftp_connect_()
else:
close_session = False
# Try to download file
self.logger.write(self.object_id, "Downloading " + source_filepath + " ...")
with open(dest_filepath, 'wb') as file:
try:
response = ftp_session.retrbinary('RETR %s' % source_filepath, file.write)
except (ftplib.error_perm, ftplib.error_temp) as e:
response = str(e)
# Analyse FTP's response and emit messages accordingly
response_code = int(re.sub("[^0-9]", "", response))
if response_code == 226:
if delete_if_success:
ftp_session.delete(source_filepath)
self.logger.write(self.object_id, "Done. Destination file: " + dest_filepath)
else:
self.logger.write(self.object_id, "FAILED. Destination file: " + dest_filepath)
raise Exception("Failed to download [" + source_filepath + "] in [" + dest_filepath + "]."
" FTP response: [" + response + "]")
# Close the FTP session if it has been opened at the beginning of this function.
if close_session:
ftp_session.quit()
return True
import os
import configobj
import wimcollect.logger as logger
import wimcollect.utils as utils
import wimcollect.ftp as ftp
import wimcollect.common.logger as logger
import wimcollect.common.utils as utils
import wimcollect.common.ftp as ftp
class Collector(ftp.FtpCollector):
class Collector(utils.LogConfig):
def __init__(self, config_parser: configobj.ConfigObj = None, log: logger = None):
self.object_id = "FTPAERIS"
ftp.FtpCollector.__init__(self, self.object_id, config_parser, log)
utils.LogConfig.__init__(self, self.object_id, config_parser, log)
self.distant_base_dir = self.config[self.object_id]["distant_base_dir"]
def download_maido_mf_1h(self, yyyymm: str):
"""Download hourly monthly-released meteo data of Meteo-France's Maïdo station.
......@@ -23,11 +24,12 @@ class Collector(ftp.FtpCollector):
self.logger.write(self.object_id, "Download Maido's Meteo-France meteo data.")
# Build source and destination file paths.
source_filepath = self.base_dir + "/" + "PMAIDO_1h_" + yyyymm + ".csv"
source_filepath = self.distant_base_dir + "/" + "PMAIDO_1h_" + yyyymm + ".csv"
dest_filepath = self.__get_dest_filepath__(yyyymm)
# Download
success = self._ftp_download_file(source_filepath, dest_filepath)
success = ftp._ftp_download_file(source_filepath, dest_filepath, self.logger, self.object_id,
ftp_config=self.config[self.object_id])
# Compress
if success:
......
......@@ -3,15 +3,17 @@ import re
import configobj
import datetime
import wimcollect.logger as logger
import wimcollect.ftp as ftp
import wimcollect.common.logger as logger
import wimcollect.common.ftp as ftp
import wimcollect.common.utils as utils
class Collector(ftp.FtpCollector):
class Collector(utils.LogConfig):
def __init__(self, config_parser: configobj.ConfigObj = None, log: logger = None):
self.object_id = "FTPCEA"
ftp.FtpCollector.__init__(self, self.object_id, config_parser, log)
utils.LogConfig.__init__(self, self.object_id, config_parser, log)
self.distant_base_dir = self.config[self.object_id]["distant_base_dir"]
####################################################################################################################
# Picarro
......@@ -30,22 +32,23 @@ class Collector(ftp.FtpCollector):
"""
self.logger.write(self.object_id, "Download picarro data from " + site_id)
ftp_session = self._ftp_connect_()
ftp_session = ftp.ftp_connect(self.config[self.object_id], self.logger, self.object_id)
# Build source file path
source_dir = self.base_dir + "/" + site_id + "/picarro/"
source_dir = self.distant_base_dir + "/" + site_id + "/picarro/"
source_filename = site_id + "_" + picarro_id + day.strftime("%y%m%d") + ".lzma"
source_filepath = source_dir + source_filename
# Check that to-be-downloaded file exists
source_filepaths = self._ftp_list_files_(ftp_session, source_dir)
source_filepaths = ftp._ftp_list_files_(ftp_session, source_dir, self.logger, self.object_id)
if source_filepath not in source_filepaths:
self.logger.write(self.object_id, "File not found: " + source_filepath)
raise FileNotFoundError(source_filepath)
# Download file
dest_filepath = self.__get_picarro_dest_filepath__(source_filepath)
success = self._ftp_download_file(source_filepath, dest_filepath, ftp_session)
success = ftp._ftp_download_file(source_filepath, dest_filepath, self.logger, self.object_id,
ftp_session=ftp_session)
ftp_session.quit()
......@@ -90,13 +93,14 @@ class Collector(ftp.FtpCollector):
"""
self.logger.write(self.object_id, "Download hobo data from " + site_id)
ftp_session = self._ftp_connect_()
hobo_distant_path = self.base_dir + "/" + site_id + "/hobo/"
ftp_session = ftp.ftp_connect(self.config[self.object_id], self.logger, self.object_id)
hobo_distant_path = self.distant_base_dir + "/" + site_id + "/hobo/"
source_filepaths = self._ftp_list_files_(ftp_session, hobo_distant_path)
source_filepaths = ftp._ftp_list_files_(ftp_session, hobo_distant_path, self.logger, self.object_id)
for source_filepath in source_filepaths:
dest_filepath = self.__get_hobo_dest_filepath__(source_filepath)
self._ftp_download_file(source_filepath, dest_filepath, ftp_session)
ftp._ftp_download_file(source_filepath, dest_filepath, self.logger, self.object_id,
ftp_session=ftp_session)
ftp_session.quit()
......
import os
import configobj
import datetime
import zipfile
import wimcollect.logger as logger
import wimcollect.utils as utils
import wimcollect.ftp as ftp
import wimcollect.common.logger as logger
import wimcollect.common.ftp as ftp
import wimcollect.common.utils as utils
class Collector(ftp.FtpCollector):
class Collector(utils.LogConfig):
def __init__(self, config_parser: configobj.ConfigObj = None, log: logger = None):
self.object_id = "FTPOPAR"
ftp.FtpCollector.__init__(self, self.object_id, config_parser, log)
utils.LogConfig.__init__(self, self.object_id, config_parser, log)
self.distant_base_dir = self.config[self.object_id]["distant_base_dir"]
def download_maido_ftir(self, day: datetime.date):
"""Download all Picarro files from FTP server.
......@@ -28,11 +28,12 @@ class Collector(ftp.FtpCollector):
date_str = day.strftime("%Y%m%d")
# Build source and destination file paths
source_filepath = self.base_dir + "/" + date_str + "Meteo125HR.xls"
source_filepath = self.distant_base_dir + "/" + date_str + "Meteo125HR.xls"
dest_filepath = self.__get_dest_filepath__(day)
# Download
success = self._ftp_download_file(source_filepath, dest_filepath)
success = ftp._ftp_download_file(source_filepath, dest_filepath, self.logger, self.object_id,
ftp_config=self.config[self.object_id])
# Compress
if success:
......
......@@ -3,8 +3,8 @@ import configobj
import datetime
import urllib.request
import wimcollect.utils as utils
import wimcollect.logger as logger
import wimcollect.common.utils as utils
import wimcollect.common.logger as logger
class Collector(utils.LogConfig):
......
......@@ -7,7 +7,7 @@ import argparse
import wimcollect.httpddu as httpddu
import wimcollect.sshdmc as sshdmc
import wimcollect.logger as logger
import wimcollect.common.logger as logger
import wimcollect.quickviz as quickviz
import wimcollect.ftpaeris as ftpaeris
......
......@@ -5,8 +5,8 @@ import configobj
import pandas as pd
import wimcollect.logger as logger
import wimcollect.utils as utils
import wimcollect.common.logger as logger
import wimcollect.common.utils as utils
class Visualizator(utils.LogConfig):
......
......@@ -4,15 +4,17 @@ import configobj
import datetime
import paramiko
import wimcollect.utils as utils
import wimcollect.logger as logger
import wimcollect.common.utils as utils
import wimcollect.common.logger as logger
class Collector(utils.LogConfig):
"""Collector for Dome C / Concordia station (trigram: DMC) data, hosted on Hermes server, managed by italian's PNRA.
"""
def __init__(self, config_parser: configobj.ConfigObj = None, log: logger = None):
utils.LogConfig.__init__(self, "SSHDMC", config_parser, log)
self.base_dir = self.config[self.object_id]["root_dir"]
self.distant_base_dir = self.config[self.object_id]["distant_base_dir"]
def download_picarro(self, day: datetime.date):
"""Download Picarro data file from SSH server.
......@@ -30,7 +32,7 @@ class Collector(utils.LogConfig):
# Build source file path
picarro_id = self.config[self.object_id]["picarro_id"]
picarro_number = re.sub("[^0-9]", "", picarro_id)
source_filepath = self.base_dir + "/" + picarro_number \
source_filepath = self.distant_base_dir + "/" + picarro_number \
+ "/DMC_" + picarro_id + "_" + day.strftime("%Y%m%d") + ".zip"
dest_filepath = self.__get_dest_filepath__(day, source_filepath)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment