Skip to content
GitLab
Projects
Groups
Snippets
Help
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Open sidebar
JOSSOUD Olivier
WIM Collect
Commits
3de5db26
Commit
3de5db26
authored
Jan 22, 2020
by
JOSSOUD Olivier
Browse files
FTPAERIS.
parent
5481ee89
Pipeline
#57061
passed with stages
in 1 minute and 10 seconds
Changes
3
Pipelines
1
Hide whitespace changes
Inline
Side-by-side
Showing
3 changed files
with
87 additions
and
3 deletions
+87
-3
wimcollect/ftp.py
wimcollect/ftp.py
+9
-3
wimcollect/ftpaeris.py
wimcollect/ftpaeris.py
+54
-0
wimcollect/utils.py
wimcollect/utils.py
+24
-0
No files found.
wimcollect/ftp.py
View file @
3de5db26
import
configobj
import
ftplib
import
codecs
import
re
import
wimcollect.logger
as
logger
import
wimcollect.utils
as
utils
...
...
@@ -87,8 +88,13 @@ class FtpCollector(utils.Collector):
"""
self
.
logger
.
write
(
self
.
object_id
,
"Downloading "
+
source_filepath
+
" ..."
)
with
open
(
dest_filepath
,
'wb'
)
as
file
:
response
=
ftp_session
.
retrbinary
(
'RETR %s'
%
source_filepath
,
file
.
write
)
if
response
==
"226 Transfer complete."
:
try
:
response
=
ftp_session
.
retrbinary
(
'RETR %s'
%
source_filepath
,
file
.
write
)
except
(
ftplib
.
error_perm
,
ftplib
.
error_temp
)
as
e
:
response
=
str
(
e
)
response_code
=
int
(
re
.
sub
(
"[^0-9]"
,
""
,
response
))
if
response_code
==
226
:
if
delete_if_success
:
ftp_session
.
delete
(
source_filepath
)
self
.
logger
.
write
(
self
.
object_id
,
"Done. Destination file: "
+
dest_filepath
)
...
...
@@ -96,4 +102,4 @@ class FtpCollector(utils.Collector):
else
:
self
.
logger
.
write
(
self
.
object_id
,
"FAILED. Destination file: "
+
dest_filepath
)
raise
Exception
(
"Failed to download ["
+
source_filepath
+
"] in ["
+
dest_filepath
+
"]."
" FTP response: "
+
response
)
" FTP response:
[
"
+
response
+
"]"
)
wimcollect/ftpaeris.py
0 → 100644
View file @
3de5db26
import
os
import
configobj
import
wimcollect.logger
as
logger
import
wimcollect.utils
as
utils
import
wimcollect.ftp
as
ftp
class
Collector
(
ftp
.
FtpCollector
):
def
__init__
(
self
,
config_parser
:
configobj
.
ConfigObj
=
None
,
log
:
logger
=
None
):
self
.
object_id
=
"FTPAERIS"
ftp
.
FtpCollector
.
__init__
(
self
,
self
.
object_id
,
config_parser
,
log
)
def
download_maido_mf_1h
(
self
,
yyyymm
:
str
):
"""Download hourly monthly-released meteo data of Meteo-France's Maïdo station.
Parameters
----------
yyyymm: str
Year and month of the to-be-downloaded data
"""
self
.
logger
.
write
(
self
.
object_id
,
"Download Maido's Meteo-France meteo data."
)
ftp_session
=
self
.
_ftp_connect_
()
# Build source and destination file paths.
source_filepath
=
self
.
base_dir
+
"/"
+
"PMAIDO_1h_"
+
yyyymm
+
".csv"
dest_filepath
=
self
.
__get_dest_filepath__
(
yyyymm
)
# Download
success
=
self
.
_ftp_download_file
(
ftp_session
,
source_filepath
,
dest_filepath
)
# Compress
if
success
:
self
.
logger
.
write
(
self
.
object_id
,
yyyymm
+
": Compressing ..."
)
utils
.
compress_file
(
dest_filepath
)
self
.
logger
.
write
(
self
.
object_id
,
yyyymm
+
": Done. Archive file: "
+
dest_filepath
)
ftp_session
.
quit
()
def
__get_dest_filepath__
(
self
,
yyyymm
:
str
)
->
str
:
# Build destination directory
dest_dir
=
os
.
path
.
join
(
self
.
config
[
"LOCAL"
][
"base_dir"
],
"REU"
,
"meteo"
,
"maidomf_1h"
)
if
not
os
.
path
.
exists
(
dest_dir
):
os
.
makedirs
(
dest_dir
)
# Build destination filename
filename
=
"REU_maidomf_"
+
yyyymm
+
".csv"
# Build full file path
dest_filepath
=
os
.
path
.
join
(
dest_dir
,
filename
)
return
dest_filepath
wimcollect/utils.py
View file @
3de5db26
...
...
@@ -70,3 +70,27 @@ def extract_compressed_file(compressed_filepath: str, dest_dir: str, delete_comp
if
delete_compressed
:
os
.
remove
(
compressed_filepath
)
return
files_in_zip
def
compress_file
(
uncompressed_filepath
:
str
,
delete_uncompressed_if_success
:
bool
=
True
):
"""Compress a single file, using LZMA algorithm.
- The output compressed file is created in the same directory as the uncompressed file.
- The output compressed file name is the same as the uncompressed file, followed by `.lzma`.
Parameters
----------
uncompressed_filepath: str
Full file path of the to-be-compressed file.
delete_uncompressed_if_success: bool
If `True` delete the source uncompressed file once the compression is successfully done.
"""
compressed_filepath
=
uncompressed_filepath
+
".lzma"
zipf
=
zipfile
.
ZipFile
(
compressed_filepath
,
'w'
,
zipfile
.
ZIP_LZMA
)
zipf
.
write
(
uncompressed_filepath
,
arcname
=
os
.
path
.
basename
(
uncompressed_filepath
))
zipf
.
close
()
if
os
.
path
.
exists
(
compressed_filepath
)
and
delete_uncompressed_if_success
:
os
.
remove
(
uncompressed_filepath
)
else
:
raise
FileNotFoundError
(
"Failed to compress "
+
uncompressed_filepath
)
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment