Skip to content

GitLab

Commit 4e0270cd authored by JOSSOUD Olivier's avatar JOSSOUD Olivier
Browse files

All. Use new logger functions.

parent 36372555
Pipeline #57588 passed with stages
in 58 seconds
Hide whitespace changes
Inline Side-by-side
Showing with 29 additions and 47 deletions
wimcollect/common/ftp.py
View file @ 4e0270cd
......@@ -5,7 +5,7 @@ import re
import wimcollect.common.logger as logger
def connect(ftp_config: dict, log: logger.Logger, object_id: str) -> ftplib.FTP:
def connect(ftp_config: dict, log: logger.Logger) -> ftplib.FTP:
"""Connect to FTP server.
Parameters
......@@ -14,8 +14,6 @@ def connect(ftp_config: dict, log: logger.Logger, object_id: str) -> ftplib.FTP:
Dict containing FTP connection information: `host`, `user` and `password`.
log: wimcollect.common.logger.Logger
Logger to record debug/info/error messages.
object_id: str
Object identifier, used as marker for log messages.
Returns
-------
......@@ -29,18 +27,18 @@ def connect(ftp_config: dict, log: logger.Logger, object_id: str) -> ftplib.FTP:
try:
session = ftplib.FTP(host=host, user=user, passwd=passwd)
except ftplib.error_perm as e:
log.write(object_id, "Failed to connect to " + host + " as " + user + ": " + str(e))
log.error("Failed to connect to " + host + " as " + user + ": " + str(e))
raise
except BlockingIOError as e:
log.write(object_id, "Host [" + host + "] seems to be unreachable.")
log.error("Host [" + host + "] seems to be unreachable.")
raise
else:
log.write(object_id, "Connected to [" + host + "] as user: " + user)
log.info("Connected to [" + host + "] as user: " + user)
return session
def list_files(ftp_session: ftplib.FTP, distant_directory_path: str, log: logger.Logger, object_id: str) -> list:
def list_files(ftp_session: ftplib.FTP, distant_directory_path: str, log: logger.Logger) -> list:
"""List the files in the distant FTP directory.
Parameters
......@@ -51,8 +49,6 @@ def list_files(ftp_session: ftplib.FTP, distant_directory_path: str, log: logger
Full path of the directory whose content should be listed.
log: wimcollect.common.logger.Logger
Logger to record debug/info/error messages.
object_id: str
Object identifier, used as marker for log messages.
Returns
-------
......@@ -65,7 +61,7 @@ def list_files(ftp_session: ftplib.FTP, distant_directory_path: str, log: logger
filepaths = ftp_session.nlst(distant_directory_path)
except ftplib.error_perm as resp:
if str(resp) == "550 No files found":
log.write(object_id, "No files in directory " + distant_directory_path)
log.error("No files in directory " + distant_directory_path, exception_type=FileNotFoundError)
else:
raise
......@@ -73,7 +69,7 @@ def list_files(ftp_session: ftplib.FTP, distant_directory_path: str, log: logger
def download_file(source_filepath: str, dest_filepath: str,
log: logger.Logger, object_id: str,
log: logger.Logger,
ftp_config: dict = None, ftp_session: ftplib.FTP = None,
delete_if_success: bool = False) -> bool:
"""Download a file from the FTP server.
......@@ -86,8 +82,6 @@ def download_file(source_filepath: str, dest_filepath: str,
Full path where the downloaded file should be stored.
log: wimcollect.common.logger.Logger
Logger to record debug/info/error messages.
object_id: str
Object identifier, used as marker for log messages.
ftp_config: dict, optional
Dict containing FTP connection information: `host`, `user` and `password`. If not set, `ftp_session` must be
provided.
......@@ -105,12 +99,12 @@ def download_file(source_filepath: str, dest_filepath: str,
# Open FTP session if missing
if ftp_session is None:
close_session = True
ftp_session = connect(ftp_config, log, object_id)
ftp_session = connect(ftp_config, log)
else:
close_session = False
# Try to download file
log.write(object_id, "Downloading " + source_filepath + " ...")
log.info("Downloading " + source_filepath + " ...")
with open(dest_filepath, 'wb') as file:
try:
response = ftp_session.retrbinary('RETR %s' % source_filepath, file.write)
......@@ -122,12 +116,10 @@ def download_file(source_filepath: str, dest_filepath: str,
if response_code == 226:
if delete_if_success:
ftp_session.delete(source_filepath)
log.write(object_id, "Done. Destination file: " + dest_filepath)
log.info("Done. Destination file: " + dest_filepath)
else:
msg = "Failed to download [" + source_filepath + "] in [" + dest_filepath + "]."\
" FTP response: [" + response + "]"
log.write(object_id, msg)
raise Exception(msg)
log.error("Failed to download [" + source_filepath + "] in [" + dest_filepath + "]."\
" FTP response: [" + response + "]")
# Close the FTP session if it has been opened at the beginning of this function.
if close_session:
......
wimcollect/common/sftp.py
View file @ 4e0270cd
......@@ -4,7 +4,7 @@ import os
import wimcollect.common.logger as logger
def connect(sftp_config: dict, log: logger.Logger, object_id: str) -> paramiko.SFTPClient:
def connect(sftp_config: dict, log: logger.Logger) -> paramiko.SFTPClient:
"""Connect to SFTP server.
Notes
......@@ -17,8 +17,6 @@ def connect(sftp_config: dict, log: logger.Logger, object_id: str) -> paramiko.S
Dict containing SFTP connection information: `host`, `user` and `private_key_path`.
log: wimcollect.common.logger
Logger to record debug/info/error messages.
object_id: str
Object identifier, used as marker for log messages.
Returns
-------
......@@ -36,16 +34,15 @@ def connect(sftp_config: dict, log: logger.Logger, object_id: str) -> paramiko.S
username=user,
key_filename=sftp_config["private_key_path"])
except Exception as e:
log.write(object_id, "Failed to connect to " + host + " as " + user + ": " + str(e))
raise e
log.error("Failed to connect to " + host + " as " + user + ": " + str(e))
else:
log.write(object_id, "Connected to [" + host + "] as user: " + user)
log.info("Connected to [" + host + "] as user: " + user)
return client.open_sftp()
def download_file(sftp_client: paramiko.SFTPClient, source_filepath: str, dest_filepath: str,
log: logger.Logger, object_id: str) -> bool:
log: logger.Logger) -> bool:
"""Download a file from the FTP server.
Parameters
......@@ -58,30 +55,24 @@ def download_file(sftp_client: paramiko.SFTPClient, source_filepath: str, dest_f
Full path where the downloaded file should be stored.
log: wimcollect.logger
Logger to record debug/info/error messages.
object_id: str
Object identifier, used as marker for log messages.
Returns
-------
bool
`True` if everything went well, otherwise raises Exception.
"""
log.write(object_id, "Downloading " + source_filepath + " ...")
log.info("Downloading " + source_filepath + " ...")
try:
sftp_client.get(source_filepath, dest_filepath)
except FileNotFoundError:
msg = "File not found: " + source_filepath
log.write(object_id, msg)
raise FileNotFoundError(msg)
log.error("File not found: " + source_filepath, exception_type=FileNotFoundError)
# Check if the destination file exists
success = os.path.exists(dest_filepath)
if success:
log.write(object_id, "File [" + source_filepath + "] downloaded in [" + dest_filepath + "].")
log.info("File [" + source_filepath + "] downloaded in [" + dest_filepath + "].")
else:
msg = "Failed to download [" + source_filepath + "] in [" + dest_filepath + "]."
log.write(object_id, msg)
raise Exception(msg)
log.error("Failed to download [" + source_filepath + "] in [" + dest_filepath + "].")
return success
wimcollect/ftpaeris.py
View file @ 4e0270cd
......@@ -28,8 +28,7 @@ class Collector(utils.LogConfig):
dest_filepath = self.__get_dest_filepath__(yyyymm)
# Download
success = ftp.download_file(source_filepath, dest_filepath, self.logger, self.object_id,
ftp_config=self.config[self.object_id])
success = ftp.download_file(source_filepath, dest_filepath, self.logger, ftp_config=self.config[self.object_id])
# Compress
if success:
......
wimcollect/ftpcea.py
View file @ 4e0270cd
......@@ -30,7 +30,7 @@ class Collector(utils.LogConfig):
day: datetime.date
Date of the data which should be downloaded.
"""
ftp_session = ftp.connect(self.config[self.object_id], self.logger, self.object_id)
ftp_session = ftp.connect(self.config[self.object_id], self.logger)
# Build source file path
source_dir = self.distant_base_dir + "/" + site_id + "/picarro/"
......@@ -38,7 +38,7 @@ class Collector(utils.LogConfig):
source_filepath = source_dir + source_filename
# Check that to-be-downloaded file exists
source_filepaths = ftp.list_files(ftp_session, source_dir, self.logger, self.object_id)
source_filepaths = ftp.list_files(ftp_session, source_dir, self.logger)
if source_filepath not in source_filepaths:
self.logger.error("File not found: " + source_filepath, day.strftime("%Y-%m-%d"),
exception_type=FileNotFoundError)
......@@ -46,7 +46,7 @@ class Collector(utils.LogConfig):
# Download file
self.logger.info("Download picarro data from " + site_id, day.strftime("%Y-%m-%d"))
dest_filepath = self.__get_picarro_dest_filepath__(source_filepath)
success = ftp.download_file(source_filepath, dest_filepath, self.logger, self.object_id,
success = ftp.download_file(source_filepath, dest_filepath, self.logger,
ftp_session=ftp_session)
ftp_session.quit()
......@@ -92,13 +92,13 @@ class Collector(utils.LogConfig):
"""
self.logger.info("Download hobo data from " + site_id)
ftp_session = ftp.connect(self.config[self.object_id], self.logger, self.object_id)
ftp_session = ftp.connect(self.config[self.object_id], self.logger)
hobo_distant_path = self.distant_base_dir + "/" + site_id + "/hobo/"
source_filepaths = ftp.list_files(ftp_session, hobo_distant_path, self.logger, self.object_id)
source_filepaths = ftp.list_files(ftp_session, hobo_distant_path, self.logger)
for source_filepath in source_filepaths:
dest_filepath = self.__get_hobo_dest_filepath__(source_filepath)
ftp.download_file(source_filepath, dest_filepath, self.logger, self.object_id,
ftp.download_file(source_filepath, dest_filepath, self.logger,
ftp_session=ftp_session)
ftp_session.quit()
......
wimcollect/ftpopar.py
View file @ 4e0270cd
......@@ -32,7 +32,7 @@ class Collector(utils.LogConfig):
# Download
self.logger.info("Download Maido's FTIR meteo data.", date_str)
success = ftp.download_file(source_filepath, dest_filepath, self.logger, self.object_id,
success = ftp.download_file(source_filepath, dest_filepath, self.logger,
ftp_config=self.config[self.object_id])
# Compress
......
wimcollect/sftpdmc.py
View file @ 4e0270cd
......@@ -38,7 +38,7 @@ class Collector(utils.LogConfig):
# Download file
self.logger.info("Downloading DMC Picarro file from SFTP server.", day_str)
sftp_client = sftp.connect(self.config[self.object_id], self.logger, self.object_id)
sftp_client = sftp.connect(self.config[self.object_id], self.logger)
success = sftp.download_file(sftp_client, source_filepath, dest_filepath, self.logger, self.object_id)
sftp_client.close()
if not success:
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment