Commit a454dcf1 authored by JOSSOUD Olivier's avatar JOSSOUD Olivier
Browse files

FTPCEA. Specify which file to download instead of downloading everything.

parent 9927e63f
Pipeline #57019 passed with stages
in 1 minute and 45 seconds
......@@ -16,28 +16,53 @@ class FtpCollector:
self.config = config_parser
self.base_dir = self.config[self.object_id]["root_dir"]
def _ftp_connect_(self):
def _ftp_connect_(self) -> ftplib.FTP:
"""Connect to FTP server according to parameters read from settings.ini file.
Returns
-------
ftplib.FTP
FTP session.
"""
try:
self.session = ftplib.FTP(host=self.config[self.object_id]["host"],
user=self.config[self.object_id]["user"],
passwd=codecs.decode(self.config[self.object_id]["password"], "rot_13"))
session = ftplib.FTP(host=self.config[self.object_id]["host"],
user=self.config[self.object_id]["user"],
passwd=codecs.decode(self.config[self.object_id]["password"], "rot_13"))
except ftplib.error_perm as e:
self.logger.write(self.object_id, "Failed to connect to FTP: " + str(e))
raise
except BlockingIOError as e:
self.logger.write(self.object_id, "Host [" + self.config[self.object_id]["host"] + "] seems to be unreachable.")
raise
else:
self.logger.write(self.object_id,
"Connected to [" + self.config[self.object_id]["host"] + "] as user: "
+ self.config[self.object_id]["user"])
def _ftp_list_files_(self, ftp_session: ftplib.FTP, path: str) -> list:
return session
def _ftp_list_files_(self, ftp_session: ftplib.FTP, distant_directory_path: str) -> list:
"""List the files in the distant FTP directory.
Parameters
----------
ftp_session: ftplib.FTP
FTP session.
distant_directory_path: str
Full path of the directory whose content should be listed.
Returns
-------
list
List of files in the FTP directory given in parameter.
"""
filepaths = []
try:
filepaths = ftp_session.nlst(path)
filepaths = ftp_session.nlst(distant_directory_path)
except ftplib.error_perm as resp:
if str(resp) == "550 No files found":
self.logger.write(self.object_id, "No files in directory " + path)
self.logger.write(self.object_id, "No files in directory " + distant_directory_path)
else:
raise
......@@ -46,6 +71,24 @@ class FtpCollector:
def _ftp_download_file(self, ftp_session: ftplib.FTP,
source_filepath: str, dest_filepath: str,
delete_if_success: bool = False) -> bool:
"""Download a file from the FTP server.
Parameters
----------
ftp_session: ftplib.FTP
FTP session.
source_filepath: str
Full path of the distant to-be_downloaded file.
dest_filepath: str
Full path where the downloaded file should be stored.
delete_if_success: bool
If `True`, the distant FTP file will be deleted from the server, once the transfer is successfully done.
Returns
-------
bool
`True` if everything went well, otherwise raises Exception.
"""
self.logger.write(self.object_id, "Downloading " + source_filepath + " ...")
with open(dest_filepath, 'wb') as file:
response = ftp_session.retrbinary('RETR %s' % source_filepath, file.write)
......@@ -56,4 +99,5 @@ class FtpCollector:
return True
else:
self.logger.write(self.object_id, "FAILED. Destination file: " + dest_filepath)
return False
raise Exception("Failed to download [" + source_filepath + "] in [" + dest_filepath + "]."
" FTP response: " + response)
......@@ -16,28 +16,40 @@ class Collector(ftp.FtpCollector):
####################################################################################################################
# Picarro
def download_picarro(self, site_id: str):
"""Download all Picarro files from FTP server.
The distant files will be deleted from the server if the transfer is successful.
def download_picarro(self, site_id: str, picarro_id: str, day: datetime.date):
"""Download Picarro file from FTP server.
Parameters
----------
site_id: str
Site's trigram
Site's trigram (AMS, SRT, etc.)
picarro_id: str
Picarro unique identifier (HIDS2189, HIDS2108, etc.)
day: datetime.date
Date of the data which should be downloaded.
"""
self.logger.write(self.object_id, "Download picarro data from " + site_id)
self._ftp_connect_()
picarro_distant_path = self.base_dir + "/" + site_id + "/picarro/"
self.session.cwd(picarro_distant_path)
ftp_session = self._ftp_connect_()
source_filepaths = self._ftp_list_files_(self.session, picarro_distant_path)
for source_filepath in source_filepaths:
dest_filepath = self.__get_picarro_dest_filepath__(source_filepath)
self._ftp_download_file(self.session, source_filepath, dest_filepath, delete_if_success=True)
# Build source file path
source_dir = self.base_dir + "/" + site_id + "/picarro/"
source_filename = site_id + "_" + picarro_id + day.strftime("%y%m%d") + ".lzma"
source_filepath = source_dir + source_filename
# Check that to-be-downloaded file exists
source_filepaths = self._ftp_list_files_(ftp_session, source_dir)
if source_filepath not in source_filepaths:
self.logger.write(self.object_id, "File not found: " + source_filepath)
raise FileNotFoundError(source_filepath)
# Download file
dest_filepath = self.__get_picarro_dest_filepath__(source_filepath)
self._ftp_download_file(ftp_session, source_filepath, dest_filepath)
ftp_session.quit()
self.session.quit()
return True
def __get_picarro_dest_filepath__(self, source_filepath: str) -> str:
"""Get the full path where the Picarro data file should be downloaded.
......@@ -78,16 +90,15 @@ class Collector(ftp.FtpCollector):
"""
self.logger.write(self.object_id, "Download hobo data from " + site_id)
self._ftp_connect_()
ftp_session = self._ftp_connect_()
hobo_distant_path = self.base_dir + "/" + site_id + "/hobo/"
self.session.cwd(hobo_distant_path)
source_filepaths = self._ftp_list_files_(self.session, hobo_distant_path)
source_filepaths = self._ftp_list_files_(ftp_session, hobo_distant_path)
for source_filepath in source_filepaths:
dest_filepath = self.__get_hobo_dest_filepath__(source_filepath)
self._ftp_download_file(self.session, source_filepath, dest_filepath, delete_if_success=True)
self._ftp_download_file(ftp_session, source_filepath, dest_filepath, delete_if_success=True)
self.session.quit()
ftp_session.quit()
def __get_hobo_dest_filepath__(self, source_filepath: str):
"""Get the full path where the Hobo data file should be downloaded.
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment