Commit 3292a4c6 authored by JOSSOUD Olivier's avatar JOSSOUD Olivier
Browse files

FTP download. Possible to let the download function create the FTP session.

parent 5fa61050
Pipeline #57069 passed with stages
in 1 minute and 18 seconds
......@@ -65,20 +65,22 @@ class FtpCollector(utils.Collector):
return filepaths
def _ftp_download_file(self, ftp_session: ftplib.FTP,
def _ftp_download_file(self,
source_filepath: str, dest_filepath: str,
ftp_session: ftplib.FTP = None,
delete_if_success: bool = False) -> bool:
"""Download a file from the FTP server.
Parameters
----------
ftp_session: ftplib.FTP
FTP session.
source_filepath: str
Full path of the distant to-be_downloaded file.
dest_filepath: str
Full path where the downloaded file should be stored.
delete_if_success: bool
ftp_session: ftplib.FTP, optional
FTP session. If `None` or not set, a session will be created using settings.ini parameters. Specifying a
custom ftp_session is useful when multiple tasks should be performed before/after downloading a single file.
delete_if_success: bool, optional
If `True`, the distant FTP file will be deleted from the server, once the transfer is successfully done.
Returns
......@@ -86,6 +88,14 @@ class FtpCollector(utils.Collector):
bool
`True` if everything went well, otherwise raises Exception.
"""
# Open FTP session if missing
if ftp_session is None:
close_session = True
ftp_session = self._ftp_connect_()
else:
close_session = False
# Try to download file
self.logger.write(self.object_id, "Downloading " + source_filepath + " ...")
with open(dest_filepath, 'wb') as file:
try:
......@@ -93,13 +103,19 @@ class FtpCollector(utils.Collector):
except (ftplib.error_perm, ftplib.error_temp) as e:
response = str(e)
# Analyse FTP's response and emit messages accordingly
response_code = int(re.sub("[^0-9]", "", response))
if response_code == 226:
if delete_if_success:
ftp_session.delete(source_filepath)
self.logger.write(self.object_id, "Done. Destination file: " + dest_filepath)
return True
else:
self.logger.write(self.object_id, "FAILED. Destination file: " + dest_filepath)
raise Exception("Failed to download [" + source_filepath + "] in [" + dest_filepath + "]."
" FTP response: [" + response + "]")
# Close the FTP session if it has been opened at the beginning of this function.
if close_session:
ftp_session.quit()
return True
......@@ -22,14 +22,12 @@ class Collector(ftp.FtpCollector):
"""
self.logger.write(self.object_id, "Download Maido's Meteo-France meteo data.")
ftp_session = self._ftp_connect_()
# Build source and destination file paths.
source_filepath = self.base_dir + "/" + "PMAIDO_1h_" + yyyymm + ".csv"
dest_filepath = self.__get_dest_filepath__(yyyymm)
# Download
success = self._ftp_download_file(ftp_session, source_filepath, dest_filepath)
success = self._ftp_download_file(source_filepath, dest_filepath)
# Compress
if success:
......@@ -37,8 +35,6 @@ class Collector(ftp.FtpCollector):
utils.compress_file(dest_filepath)
self.logger.write(self.object_id, yyyymm + ": Done. Archive file: " + dest_filepath)
ftp_session.quit()
def __get_dest_filepath__(self, yyyymm: str) -> str:
# Build destination directory
dest_dir = os.path.join(self.config["LOCAL"]["base_dir"], "REU", "meteo", "maidomf_1h")
......
......@@ -45,11 +45,11 @@ class Collector(ftp.FtpCollector):
# Download file
dest_filepath = self.__get_picarro_dest_filepath__(source_filepath)
self._ftp_download_file(ftp_session, source_filepath, dest_filepath)
success = self._ftp_download_file(source_filepath, dest_filepath, ftp_session)
ftp_session.quit()
return True
return success
def __get_picarro_dest_filepath__(self, source_filepath: str) -> str:
"""Get the full path where the Picarro data file should be downloaded.
......@@ -96,7 +96,7 @@ class Collector(ftp.FtpCollector):
source_filepaths = self._ftp_list_files_(ftp_session, hobo_distant_path)
for source_filepath in source_filepaths:
dest_filepath = self.__get_hobo_dest_filepath__(source_filepath)
self._ftp_download_file(ftp_session, source_filepath, dest_filepath, delete_if_success=True)
self._ftp_download_file(source_filepath, dest_filepath, ftp_session)
ftp_session.quit()
......
......@@ -26,15 +26,13 @@ class Collector(ftp.FtpCollector):
"""
self.logger.write(self.object_id, "Download Maido's FTIR meteo data.")
ftp_session = self._ftp_connect_()
date_str = day.strftime("%Y%m%d")
# Build source and destination file paths
source_filepath = self.base_dir + "/" + date_str + "Meteo125HR.xls"
dest_filepath = self.__get_dest_filepath__(day)
# Download
success = self._ftp_download_file(ftp_session, source_filepath, dest_filepath)
success = self._ftp_download_file(source_filepath, dest_filepath)
# Compress
if success:
......@@ -42,8 +40,6 @@ class Collector(ftp.FtpCollector):
utils.compress_file(dest_filepath)
self.logger.write(self.object_id, date_str + ": Done. Archive file: " + dest_filepath)
ftp_session.quit()
def __get_dest_filepath__(self, day: datetime.date) -> str:
# Build destination directory
dest_dir = os.path.join(self.config["LOCAL"]["base_dir"], "REU", "meteo", "maidoftir")
......
......@@ -9,6 +9,7 @@ import wimcollect.httpddu as httpddu
import wimcollect.sshdmc as sshdmc
import wimcollect.logger as logger
import wimcollect.quickviz as quickviz
import wimcollect.ftpaeris as ftpaeris
def main():
......@@ -30,6 +31,9 @@ def main():
yesterday = datetime.datetime.now() - datetime.timedelta(1)
ftpaeris_col = ftpaeris.Collector()
ftpaeris_col.download_maido_mf_1h("201913")
httpddu_col = httpddu.Collector()
httpddu_col.download_picarro(yesterday)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment