Skip to content
GitLab
Projects
Groups
Snippets
Help
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Open sidebar
JOSSOUD Olivier
WIM Collect
Commits
a1bf0a8c
Commit
a1bf0a8c
authored
Jun 22, 2020
by
JOSSOUD Olivier
Browse files
FTP AERIS. Manage multiple stations.
parent
7d5c20f2
Pipeline
#73967
passed with stages
in 2 minutes and 6 seconds
Changes
3
Pipelines
1
Hide whitespace changes
Inline
Side-by-side
Showing
3 changed files
with
43 additions
and
16 deletions
+43
-16
wimcollect/ftpaeris.py
wimcollect/ftpaeris.py
+37
-11
wimcollect/main.py
wimcollect/main.py
+1
-1
wimcollect/tests/test_ftpaeris.py
wimcollect/tests/test_ftpaeris.py
+5
-4
No files found.
wimcollect/ftpaeris.py
View file @
a1bf0a8c
...
...
@@ -6,7 +6,7 @@ import wimcollect.common.utils as utils
import
wimcollect.common.ftp
as
ftp
class
Maido
Mf1hCollector
(
utils
.
LogConfig
):
class
Mf1hCollector
(
utils
.
LogConfig
):
"""Class to collect Maïdo's Meteo-France data: one file per month (released every 18th of the next month), 1 data
line per hour."""
...
...
@@ -15,11 +15,20 @@ class MaidoMf1hCollector(utils.LogConfig):
utils
.
LogConfig
.
__init__
(
self
,
self
.
object_id
,
config_parser
,
log
)
self
.
distant_base_dir
=
self
.
config
[
self
.
object_id
][
"distant_base_dir"
]
def
download
(
self
,
yyyymm
:
str
)
->
bool
:
"""Download hourly monthly-released meteo data of Meteo-France's Maïdo station.
def
download
(
self
,
site_id
:
str
,
mf_station_original_name
:
str
,
instrument_id
:
str
,
yyyymm
:
str
)
->
bool
:
"""Download hourly monthly-released meteo data of Meteo-France's station.
Parameters
----------
site_id: str
Site's trigram (AMS, SRT, etc.)
mf_station_original_name: str
Code name for the Meteo France's station, as defined by AERIS
instrument_id: str
Local code name for the virtual "meteo instrument", as defined for the local files. E.g. for
``mf_station_original_name=PMAIDO``, ``instrument_id`` would be ``maidomf1h``.
yyyymm: str
Year and month of the to-be-downloaded data
...
...
@@ -29,11 +38,16 @@ class MaidoMf1hCollector(utils.LogConfig):
`True` if everything went well, otherwise raises Exception.
"""
self
.
logger
.
date_str
=
yyyymm
self
.
logger
.
info
(
"Download
Maido
's Meteo-France '1hour' meteo data."
)
self
.
logger
.
info
(
"Download
"
+
mf_station_original_name
+
"
's Meteo-France '1hour' meteo data."
)
# Build source and destination file paths.
source_filepath
=
self
.
distant_base_dir
+
"/"
+
"PMAIDO_1h_"
+
yyyymm
+
".csv"
dest_filepath
=
utils
.
get_standard_filepath
(
self
.
local_base_dir
,
"REU"
,
"meteo"
,
"maidomf1h"
,
yyyymm
,
"csv"
)
source_filepath
=
self
.
distant_base_dir
+
"/"
+
mf_station_original_name
+
"_1h_"
+
yyyymm
+
".csv"
dest_filepath
=
utils
.
get_standard_filepath
(
self
.
local_base_dir
,
site_id
,
"meteo"
,
instrument_id
,
yyyymm
,
"csv"
)
# Download
success
=
ftp
.
download_file
(
source_filepath
,
dest_filepath
,
self
.
logger
,
ftp_config
=
self
.
config
[
self
.
object_id
])
...
...
@@ -45,7 +59,7 @@ class MaidoMf1hCollector(utils.LogConfig):
return
success
class
Maido
Mf1mCollector
(
utils
.
LogConfig
):
class
Mf1mCollector
(
utils
.
LogConfig
):
"""Class to collect Maïdo's Meteo-France data: one file per day (theoretically released every day, however data can
be available few days after expected release...), 1 data line per minute."""
...
...
@@ -54,11 +68,21 @@ class MaidoMf1mCollector(utils.LogConfig):
utils
.
LogConfig
.
__init__
(
self
,
self
.
object_id
,
config_parser
,
log
)
self
.
distant_base_dir
=
self
.
config
[
self
.
object_id
][
"distant_base_dir"
]
def
download
(
self
,
day
:
datetime
.
date
)
->
bool
:
def
download
(
self
,
site_id
:
str
,
mf_station_original_name
:
str
,
instrument_id
:
str
,
day
:
datetime
.
date
)
->
bool
:
"""Download minutely daily-released meteo data of Meteo-France's Maïdo station.
Parameters
----------
site_id: str
Site's trigram (AMS, SRT, etc.)
mf_station_original_name: str
Code name for the Meteo France's station, as defined by AERIS
instrument_id: str
Local code name for the virtual "meteo instrument", as defined for the local files. E.g. for
``mf_station_original_name=PMAIDO``, ``instrument_id`` would be ``maidomf1h``.
day: datetime.date
Date of the data which should be downloaded.
...
...
@@ -70,11 +94,13 @@ class MaidoMf1mCollector(utils.LogConfig):
date_str
=
day
.
strftime
(
"%Y%m%d"
)
self
.
logger
.
date_str
=
date_str
self
.
logger
.
info
(
"Download Maido's Meteo-France '1minute' meteo data."
)
self
.
logger
.
info
(
"Download "
+
mf_station_original_name
+
"'s Meteo-France '1minute' meteo data."
)
# Build source and destination file paths.
source_filepath
=
self
.
distant_base_dir
+
"/"
+
"PMAIDO_1mn_"
+
date_str
+
".csv"
dest_filepath
=
utils
.
get_standard_filepath
(
self
.
local_base_dir
,
"REU"
,
"meteo"
,
"maidomf1m"
,
date_str
,
"csv"
)
source_filepath
=
self
.
distant_base_dir
+
"/"
+
mf_station_original_name
+
"_1mn_"
+
date_str
+
".csv"
dest_filepath
=
utils
.
get_standard_filepath
(
self
.
local_base_dir
,
site_id
,
"meteo"
,
instrument_id
,
date_str
,
"csv"
)
# Download
success
=
ftp
.
download_file
(
source_filepath
,
dest_filepath
,
self
.
logger
,
ftp_config
=
self
.
config
[
self
.
object_id
])
...
...
wimcollect/main.py
View file @
a1bf0a8c
...
...
@@ -31,7 +31,7 @@ def main():
yesterday
=
datetime
.
datetime
.
now
()
-
datetime
.
timedelta
(
1
)
ftpaeris_col
=
ftpaeris
.
Maido
Mf1hCollector
()
ftpaeris_col
=
ftpaeris
.
Mf1hCollector
()
ftpaeris_col
.
download
(
"201913"
)
httpddu_col
=
httpddu
.
PicarroCollector
()
...
...
wimcollect/tests/test_ftpaeris.py
View file @
a1bf0a8c
...
...
@@ -6,11 +6,12 @@ import wimcollect.ftpaeris as ftpaeris
class
TestMaidoMf1hCollector
(
TestCase
):
def
test_download
(
self
):
collector
=
ftpaeris
.
MaidoMf1hCollector
()
self
.
assertTrue
(
collector
.
download
(
"201911"
))
collector
=
ftpaeris
.
Mf1hCollector
()
self
.
assertTrue
(
collector
.
download
(
"REU"
,
"PMAIDO"
,
"maido_mf_1h"
,
"201911"
))
self
.
assertTrue
(
collector
.
download
(
"GUA"
,
"CONVENANCE"
,
"conv_mf_1h"
,
"202003"
))
class
TestMaidoMf1mCollector
(
TestCase
):
def
test_download
(
self
):
collector
=
ftpaeris
.
Maido
Mf1mCollector
()
self
.
assertTrue
(
collector
.
download
(
datetime
.
date
(
2020
,
1
,
1
)))
collector
=
ftpaeris
.
Mf1mCollector
()
self
.
assertTrue
(
collector
.
download
(
"GUA"
,
"CONVENANCE"
,
"conv_mf_1min"
,
datetime
.
date
(
2020
,
6
,
1
)))
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment