Commit a1bf0a8c authored by JOSSOUD Olivier's avatar JOSSOUD Olivier
Browse files

FTP AERIS. Manage multiple stations.

parent 7d5c20f2
Pipeline #73967 passed with stages
in 2 minutes and 6 seconds
......@@ -6,7 +6,7 @@ import wimcollect.common.utils as utils
import wimcollect.common.ftp as ftp
class MaidoMf1hCollector(utils.LogConfig):
class Mf1hCollector(utils.LogConfig):
"""Class to collect Maïdo's Meteo-France data: one file per month (released every 18th of the next month), 1 data
line per hour."""
......@@ -15,11 +15,20 @@ class MaidoMf1hCollector(utils.LogConfig):
utils.LogConfig.__init__(self, self.object_id, config_parser, log)
self.distant_base_dir = self.config[self.object_id]["distant_base_dir"]
def download(self, yyyymm: str) -> bool:
"""Download hourly monthly-released meteo data of Meteo-France's Maïdo station.
def download(self, site_id: str,
mf_station_original_name: str,
instrument_id: str, yyyymm: str) -> bool:
"""Download hourly monthly-released meteo data of Meteo-France's station.
Parameters
----------
site_id: str
Site's trigram (AMS, SRT, etc.)
mf_station_original_name: str
Code name for the Meteo France's station, as defined by AERIS
instrument_id: str
Local code name for the virtual "meteo instrument", as defined for the local files. E.g. for
``mf_station_original_name=PMAIDO``, ``instrument_id`` would be ``maidomf1h``.
yyyymm: str
Year and month of the to-be-downloaded data
......@@ -29,11 +38,16 @@ class MaidoMf1hCollector(utils.LogConfig):
`True` if everything went well, otherwise raises Exception.
"""
self.logger.date_str = yyyymm
self.logger.info("Download Maido's Meteo-France '1hour' meteo data.")
self.logger.info("Download " + mf_station_original_name + "'s Meteo-France '1hour' meteo data.")
# Build source and destination file paths.
source_filepath = self.distant_base_dir + "/" + "PMAIDO_1h_" + yyyymm + ".csv"
dest_filepath = utils.get_standard_filepath(self.local_base_dir, "REU", "meteo", "maidomf1h", yyyymm, "csv")
source_filepath = self.distant_base_dir + "/" + mf_station_original_name + "_1h_" + yyyymm + ".csv"
dest_filepath = utils.get_standard_filepath(self.local_base_dir,
site_id,
"meteo",
instrument_id,
yyyymm,
"csv")
# Download
success = ftp.download_file(source_filepath, dest_filepath, self.logger, ftp_config=self.config[self.object_id])
......@@ -45,7 +59,7 @@ class MaidoMf1hCollector(utils.LogConfig):
return success
class MaidoMf1mCollector(utils.LogConfig):
class Mf1mCollector(utils.LogConfig):
"""Class to collect Maïdo's Meteo-France data: one file per day (theoretically released every day, however data can
be available few days after expected release...), 1 data line per minute."""
......@@ -54,11 +68,21 @@ class MaidoMf1mCollector(utils.LogConfig):
utils.LogConfig.__init__(self, self.object_id, config_parser, log)
self.distant_base_dir = self.config[self.object_id]["distant_base_dir"]
def download(self, day: datetime.date) -> bool:
def download(self, site_id: str,
mf_station_original_name: str,
instrument_id: str,
day: datetime.date) -> bool:
"""Download minutely daily-released meteo data of Meteo-France's Maïdo station.
Parameters
----------
site_id: str
Site's trigram (AMS, SRT, etc.)
mf_station_original_name: str
Code name for the Meteo France's station, as defined by AERIS
instrument_id: str
Local code name for the virtual "meteo instrument", as defined for the local files. E.g. for
``mf_station_original_name=PMAIDO``, ``instrument_id`` would be ``maidomf1h``.
day: datetime.date
Date of the data which should be downloaded.
......@@ -70,11 +94,13 @@ class MaidoMf1mCollector(utils.LogConfig):
date_str = day.strftime("%Y%m%d")
self.logger.date_str = date_str
self.logger.info("Download Maido's Meteo-France '1minute' meteo data.")
self.logger.info("Download " + mf_station_original_name + "'s Meteo-France '1minute' meteo data.")
# Build source and destination file paths.
source_filepath = self.distant_base_dir + "/" + "PMAIDO_1mn_" + date_str + ".csv"
dest_filepath = utils.get_standard_filepath(self.local_base_dir, "REU", "meteo", "maidomf1m", date_str, "csv")
source_filepath = self.distant_base_dir + "/" + mf_station_original_name + "_1mn_" + date_str + ".csv"
dest_filepath = utils.get_standard_filepath(self.local_base_dir, site_id, "meteo", instrument_id, date_str,
"csv")
# Download
success = ftp.download_file(source_filepath, dest_filepath, self.logger, ftp_config=self.config[self.object_id])
......
......@@ -31,7 +31,7 @@ def main():
yesterday = datetime.datetime.now() - datetime.timedelta(1)
ftpaeris_col = ftpaeris.MaidoMf1hCollector()
ftpaeris_col = ftpaeris.Mf1hCollector()
ftpaeris_col.download("201913")
httpddu_col = httpddu.PicarroCollector()
......
......@@ -6,11 +6,12 @@ import wimcollect.ftpaeris as ftpaeris
class TestMaidoMf1hCollector(TestCase):
def test_download(self):
collector = ftpaeris.MaidoMf1hCollector()
self.assertTrue(collector.download("201911"))
collector = ftpaeris.Mf1hCollector()
self.assertTrue(collector.download("REU", "PMAIDO", "maido_mf_1h", "201911"))
self.assertTrue(collector.download("GUA", "CONVENANCE", "conv_mf_1h", "202003"))
class TestMaidoMf1mCollector(TestCase):
def test_download(self):
collector = ftpaeris.MaidoMf1mCollector()
self.assertTrue(collector.download(datetime.date(2020, 1, 1)))
collector = ftpaeris.Mf1mCollector()
self.assertTrue(collector.download("GUA", "CONVENANCE", "conv_mf_1min", datetime.date(2020, 6, 1)))
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment