Commit a9595cdd authored by JOSSOUD Olivier's avatar JOSSOUD Olivier
Browse files

FTP. Remove useless `ftp_` for ftp's functions name.

parent 7d500ea0
...@@ -5,14 +5,14 @@ import re ...@@ -5,14 +5,14 @@ import re
import wimcollect.common.logger as logger import wimcollect.common.logger as logger
def ftp_connect(ftp_config: dict, log: logger.Logger, object_id: str) -> ftplib.FTP: def connect(ftp_config: dict, log: logger.Logger, object_id: str) -> ftplib.FTP:
"""Connect to FTP server according to parameters read from settings.ini file. """Connect to FTP server.
Parameters Parameters
---------- ----------
ftp_config: dict ftp_config: dict
Dict containing FTP connection information: `host`, `user` and `password`. Dict containing FTP connection information: `host`, `user` and `password`.
log: wimcollect.logger log: wimcollect.common.logger.Logger
Logger to record debug/info/error messages. Logger to record debug/info/error messages.
object_id: str object_id: str
Object identifier, used as marker for log messages. Object identifier, used as marker for log messages.
...@@ -40,7 +40,7 @@ def ftp_connect(ftp_config: dict, log: logger.Logger, object_id: str) -> ftplib. ...@@ -40,7 +40,7 @@ def ftp_connect(ftp_config: dict, log: logger.Logger, object_id: str) -> ftplib.
return session return session
def ftp_list_files(ftp_session: ftplib.FTP, distant_directory_path: str, log: logger.Logger, object_id: str) -> list: def list_files(ftp_session: ftplib.FTP, distant_directory_path: str, log: logger.Logger, object_id: str) -> list:
"""List the files in the distant FTP directory. """List the files in the distant FTP directory.
Parameters Parameters
...@@ -49,7 +49,7 @@ def ftp_list_files(ftp_session: ftplib.FTP, distant_directory_path: str, log: lo ...@@ -49,7 +49,7 @@ def ftp_list_files(ftp_session: ftplib.FTP, distant_directory_path: str, log: lo
FTP session. FTP session.
distant_directory_path: str distant_directory_path: str
Full path of the directory whose content should be listed. Full path of the directory whose content should be listed.
log: wimcollect.logger log: wimcollect.common.logger.Logger
Logger to record debug/info/error messages. Logger to record debug/info/error messages.
object_id: str object_id: str
Object identifier, used as marker for log messages. Object identifier, used as marker for log messages.
...@@ -72,19 +72,19 @@ def ftp_list_files(ftp_session: ftplib.FTP, distant_directory_path: str, log: lo ...@@ -72,19 +72,19 @@ def ftp_list_files(ftp_session: ftplib.FTP, distant_directory_path: str, log: lo
return filepaths return filepaths
def ftp_download_file(source_filepath: str, dest_filepath: str, def download_file(source_filepath: str, dest_filepath: str,
log: logger.Logger, object_id: str, log: logger.Logger, object_id: str,
ftp_config: dict = None, ftp_session: ftplib.FTP = None, ftp_config: dict = None, ftp_session: ftplib.FTP = None,
delete_if_success: bool = False) -> bool: delete_if_success: bool = False) -> bool:
"""Download a file from the FTP server. """Download a file from the FTP server.
Parameters Parameters
---------- ----------
source_filepath: str source_filepath: str
Full path of the distant to-be_downloaded file. Full path of the distant to-be-downloaded file.
dest_filepath: str dest_filepath: str
Full path where the downloaded file should be stored. Full path where the downloaded file should be stored.
log: wimcollect.logger log: wimcollect.common.logger.Logger
Logger to record debug/info/error messages. Logger to record debug/info/error messages.
object_id: str object_id: str
Object identifier, used as marker for log messages. Object identifier, used as marker for log messages.
...@@ -105,7 +105,7 @@ def ftp_download_file(source_filepath: str, dest_filepath: str, ...@@ -105,7 +105,7 @@ def ftp_download_file(source_filepath: str, dest_filepath: str,
# Open FTP session if missing # Open FTP session if missing
if ftp_session is None: if ftp_session is None:
close_session = True close_session = True
ftp_session = ftp_connect(ftp_config, log, object_id) ftp_session = connect(ftp_config, log, object_id)
else: else:
close_session = False close_session = False
...@@ -124,9 +124,10 @@ def ftp_download_file(source_filepath: str, dest_filepath: str, ...@@ -124,9 +124,10 @@ def ftp_download_file(source_filepath: str, dest_filepath: str,
ftp_session.delete(source_filepath) ftp_session.delete(source_filepath)
log.write(object_id, "Done. Destination file: " + dest_filepath) log.write(object_id, "Done. Destination file: " + dest_filepath)
else: else:
log.write(object_id, "FAILED. Destination file: " + dest_filepath) msg = "Failed to download [" + source_filepath + "] in [" + dest_filepath + "]."\
raise Exception("Failed to download [" + source_filepath + "] in [" + dest_filepath + "]." " FTP response: [" + response + "]"
" FTP response: [" + response + "]") log.write(object_id, msg)
raise Exception(msg)
# Close the FTP session if it has been opened at the beginning of this function. # Close the FTP session if it has been opened at the beginning of this function.
if close_session: if close_session:
......
...@@ -28,8 +28,8 @@ class Collector(utils.LogConfig): ...@@ -28,8 +28,8 @@ class Collector(utils.LogConfig):
dest_filepath = self.__get_dest_filepath__(yyyymm) dest_filepath = self.__get_dest_filepath__(yyyymm)
# Download # Download
success = ftp.ftp_download_file(source_filepath, dest_filepath, self.logger, self.object_id, success = ftp.download_file(source_filepath, dest_filepath, self.logger, self.object_id,
ftp_config=self.config[self.object_id]) ftp_config=self.config[self.object_id])
# Compress # Compress
if success: if success:
......
...@@ -32,7 +32,7 @@ class Collector(utils.LogConfig): ...@@ -32,7 +32,7 @@ class Collector(utils.LogConfig):
""" """
self.logger.write(self.object_id, "Download picarro data from " + site_id) self.logger.write(self.object_id, "Download picarro data from " + site_id)
ftp_session = ftp.ftp_connect(self.config[self.object_id], self.logger, self.object_id) ftp_session = ftp.connect(self.config[self.object_id], self.logger, self.object_id)
# Build source file path # Build source file path
source_dir = self.distant_base_dir + "/" + site_id + "/picarro/" source_dir = self.distant_base_dir + "/" + site_id + "/picarro/"
...@@ -40,15 +40,15 @@ class Collector(utils.LogConfig): ...@@ -40,15 +40,15 @@ class Collector(utils.LogConfig):
source_filepath = source_dir + source_filename source_filepath = source_dir + source_filename
# Check that to-be-downloaded file exists # Check that to-be-downloaded file exists
source_filepaths = ftp.ftp_list_files(ftp_session, source_dir, self.logger, self.object_id) source_filepaths = ftp.list_files(ftp_session, source_dir, self.logger, self.object_id)
if source_filepath not in source_filepaths: if source_filepath not in source_filepaths:
self.logger.write(self.object_id, "File not found: " + source_filepath) self.logger.write(self.object_id, "File not found: " + source_filepath)
raise FileNotFoundError(source_filepath) raise FileNotFoundError(source_filepath)
# Download file # Download file
dest_filepath = self.__get_picarro_dest_filepath__(source_filepath) dest_filepath = self.__get_picarro_dest_filepath__(source_filepath)
success = ftp.ftp_download_file(source_filepath, dest_filepath, self.logger, self.object_id, success = ftp.download_file(source_filepath, dest_filepath, self.logger, self.object_id,
ftp_session=ftp_session) ftp_session=ftp_session)
ftp_session.quit() ftp_session.quit()
...@@ -93,14 +93,14 @@ class Collector(utils.LogConfig): ...@@ -93,14 +93,14 @@ class Collector(utils.LogConfig):
""" """
self.logger.write(self.object_id, "Download hobo data from " + site_id) self.logger.write(self.object_id, "Download hobo data from " + site_id)
ftp_session = ftp.ftp_connect(self.config[self.object_id], self.logger, self.object_id) ftp_session = ftp.connect(self.config[self.object_id], self.logger, self.object_id)
hobo_distant_path = self.distant_base_dir + "/" + site_id + "/hobo/" hobo_distant_path = self.distant_base_dir + "/" + site_id + "/hobo/"
source_filepaths = ftp.ftp_list_files(ftp_session, hobo_distant_path, self.logger, self.object_id) source_filepaths = ftp.list_files(ftp_session, hobo_distant_path, self.logger, self.object_id)
for source_filepath in source_filepaths: for source_filepath in source_filepaths:
dest_filepath = self.__get_hobo_dest_filepath__(source_filepath) dest_filepath = self.__get_hobo_dest_filepath__(source_filepath)
ftp.ftp_download_file(source_filepath, dest_filepath, self.logger, self.object_id, ftp.download_file(source_filepath, dest_filepath, self.logger, self.object_id,
ftp_session=ftp_session) ftp_session=ftp_session)
ftp_session.quit() ftp_session.quit()
......
...@@ -32,8 +32,8 @@ class Collector(utils.LogConfig): ...@@ -32,8 +32,8 @@ class Collector(utils.LogConfig):
dest_filepath = self.__get_dest_filepath__(day) dest_filepath = self.__get_dest_filepath__(day)
# Download # Download
success = ftp.ftp_download_file(source_filepath, dest_filepath, self.logger, self.object_id, success = ftp.download_file(source_filepath, dest_filepath, self.logger, self.object_id,
ftp_config=self.config[self.object_id]) ftp_config=self.config[self.object_id])
# Compress # Compress
if success: if success:
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment