Commit 3de5db26 authored by JOSSOUD Olivier's avatar JOSSOUD Olivier
Browse files

FTPAERIS.

parent 5481ee89
Pipeline #57061 passed with stages
in 1 minute and 10 seconds
import configobj
import ftplib
import codecs
import re
import wimcollect.logger as logger
import wimcollect.utils as utils
......@@ -87,8 +88,13 @@ class FtpCollector(utils.Collector):
"""
self.logger.write(self.object_id, "Downloading " + source_filepath + " ...")
with open(dest_filepath, 'wb') as file:
response = ftp_session.retrbinary('RETR %s' % source_filepath, file.write)
if response == "226 Transfer complete.":
try:
response = ftp_session.retrbinary('RETR %s' % source_filepath, file.write)
except (ftplib.error_perm, ftplib.error_temp) as e:
response = str(e)
response_code = int(re.sub("[^0-9]", "", response))
if response_code == 226:
if delete_if_success:
ftp_session.delete(source_filepath)
self.logger.write(self.object_id, "Done. Destination file: " + dest_filepath)
......@@ -96,4 +102,4 @@ class FtpCollector(utils.Collector):
else:
self.logger.write(self.object_id, "FAILED. Destination file: " + dest_filepath)
raise Exception("Failed to download [" + source_filepath + "] in [" + dest_filepath + "]."
" FTP response: " + response)
" FTP response: [" + response + "]")
import os
import configobj
import wimcollect.logger as logger
import wimcollect.utils as utils
import wimcollect.ftp as ftp
class Collector(ftp.FtpCollector):
def __init__(self, config_parser: configobj.ConfigObj = None, log: logger = None):
self.object_id = "FTPAERIS"
ftp.FtpCollector.__init__(self, self.object_id, config_parser, log)
def download_maido_mf_1h(self, yyyymm: str):
"""Download hourly monthly-released meteo data of Meteo-France's Maïdo station.
Parameters
----------
yyyymm: str
Year and month of the to-be-downloaded data
"""
self.logger.write(self.object_id, "Download Maido's Meteo-France meteo data.")
ftp_session = self._ftp_connect_()
# Build source and destination file paths.
source_filepath = self.base_dir + "/" + "PMAIDO_1h_" + yyyymm + ".csv"
dest_filepath = self.__get_dest_filepath__(yyyymm)
# Download
success = self._ftp_download_file(ftp_session, source_filepath, dest_filepath)
# Compress
if success:
self.logger.write(self.object_id, yyyymm + ": Compressing ...")
utils.compress_file(dest_filepath)
self.logger.write(self.object_id, yyyymm + ": Done. Archive file: " + dest_filepath)
ftp_session.quit()
def __get_dest_filepath__(self, yyyymm: str) -> str:
# Build destination directory
dest_dir = os.path.join(self.config["LOCAL"]["base_dir"], "REU", "meteo", "maidomf_1h")
if not os.path.exists(dest_dir):
os.makedirs(dest_dir)
# Build destination filename
filename = "REU_maidomf_" + yyyymm + ".csv"
# Build full file path
dest_filepath = os.path.join(dest_dir, filename)
return dest_filepath
......@@ -70,3 +70,27 @@ def extract_compressed_file(compressed_filepath: str, dest_dir: str, delete_comp
if delete_compressed:
os.remove(compressed_filepath)
return files_in_zip
def compress_file(uncompressed_filepath: str, delete_uncompressed_if_success: bool = True):
"""Compress a single file, using LZMA algorithm.
- The output compressed file is created in the same directory as the uncompressed file.
- The output compressed file name is the same as the uncompressed file, followed by `.lzma`.
Parameters
----------
uncompressed_filepath: str
Full file path of the to-be-compressed file.
delete_uncompressed_if_success: bool
If `True` delete the source uncompressed file once the compression is successfully done.
"""
compressed_filepath = uncompressed_filepath + ".lzma"
zipf = zipfile.ZipFile(compressed_filepath, 'w', zipfile.ZIP_LZMA)
zipf.write(uncompressed_filepath, arcname=os.path.basename(uncompressed_filepath))
zipf.close()
if os.path.exists(compressed_filepath) and delete_uncompressed_if_success:
os.remove(uncompressed_filepath)
else:
raise FileNotFoundError("Failed to compress " + uncompressed_filepath)
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment