Skip to content
GitLab
Projects
Groups
Snippets
Help
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Open sidebar
JOSSOUD Olivier
WIM Collect
Commits
87b70b68
Commit
87b70b68
authored
Jan 27, 2020
by
JOSSOUD Olivier
Browse files
All. Use new logger functions.
parent
4eb916cd
Pipeline
#57584
passed with stages
in 1 minute and 33 seconds
Changes
7
Pipelines
1
Hide whitespace changes
Inline
Side-by-side
Showing
7 changed files
with
34 additions
and
22 deletions
+34
-22
wimcollect/common/utils.py
wimcollect/common/utils.py
+9
-1
wimcollect/ftpaeris.py
wimcollect/ftpaeris.py
+3
-3
wimcollect/ftpcea.py
wimcollect/ftpcea.py
+5
-6
wimcollect/ftpopar.py
wimcollect/ftpopar.py
+4
-4
wimcollect/main.py
wimcollect/main.py
+3
-3
wimcollect/quickviz.py
wimcollect/quickviz.py
+1
-1
wimcollect/sftpdmc.py
wimcollect/sftpdmc.py
+9
-4
No files found.
wimcollect/common/utils.py
View file @
87b70b68
...
@@ -87,7 +87,7 @@ def extract_compressed_file(compressed_filepath: str, dest_dir: str, delete_comp
...
@@ -87,7 +87,7 @@ def extract_compressed_file(compressed_filepath: str, dest_dir: str, delete_comp
return
files_in_zip
return
files_in_zip
def
compress_file
(
uncompressed_filepath
:
str
,
delete_uncompressed_if_success
:
bool
=
True
):
def
compress_file
(
uncompressed_filepath
:
str
,
delete_uncompressed_if_success
:
bool
=
True
)
->
str
:
"""Compress a single file, using LZMA algorithm.
"""Compress a single file, using LZMA algorithm.
- The output compressed file is created in the same directory as the uncompressed file.
- The output compressed file is created in the same directory as the uncompressed file.
...
@@ -99,6 +99,12 @@ def compress_file(uncompressed_filepath: str, delete_uncompressed_if_success: bo
...
@@ -99,6 +99,12 @@ def compress_file(uncompressed_filepath: str, delete_uncompressed_if_success: bo
Full file path of the to-be-compressed file.
Full file path of the to-be-compressed file.
delete_uncompressed_if_success: bool
delete_uncompressed_if_success: bool
If `True` delete the source uncompressed file once the compression is successfully done.
If `True` delete the source uncompressed file once the compression is successfully done.
Returns
-------
str
Full file path of the LZMA-compressed output file. Basically the same as `uncompressed_filepath`, with its
extension replaced by `.lzma`.
"""
"""
compressed_filepath
=
uncompressed_filepath
+
".lzma"
compressed_filepath
=
uncompressed_filepath
+
".lzma"
zipf
=
zipfile
.
ZipFile
(
compressed_filepath
,
'w'
,
zipfile
.
ZIP_LZMA
)
zipf
=
zipfile
.
ZipFile
(
compressed_filepath
,
'w'
,
zipfile
.
ZIP_LZMA
)
...
@@ -109,3 +115,5 @@ def compress_file(uncompressed_filepath: str, delete_uncompressed_if_success: bo
...
@@ -109,3 +115,5 @@ def compress_file(uncompressed_filepath: str, delete_uncompressed_if_success: bo
os
.
remove
(
uncompressed_filepath
)
os
.
remove
(
uncompressed_filepath
)
else
:
else
:
raise
FileNotFoundError
(
"Failed to compress "
+
uncompressed_filepath
)
raise
FileNotFoundError
(
"Failed to compress "
+
uncompressed_filepath
)
return
compressed_filepath
wimcollect/ftpaeris.py
View file @
87b70b68
...
@@ -21,7 +21,7 @@ class Collector(utils.LogConfig):
...
@@ -21,7 +21,7 @@ class Collector(utils.LogConfig):
yyyymm: str
yyyymm: str
Year and month of the to-be-downloaded data
Year and month of the to-be-downloaded data
"""
"""
self
.
logger
.
write
(
self
.
object_id
,
"Download Maido's Meteo-France meteo data."
)
self
.
logger
.
info
(
"Download Maido's Meteo-France meteo data."
,
yyyymm
)
# Build source and destination file paths.
# Build source and destination file paths.
source_filepath
=
self
.
distant_base_dir
+
"/"
+
"PMAIDO_1h_"
+
yyyymm
+
".csv"
source_filepath
=
self
.
distant_base_dir
+
"/"
+
"PMAIDO_1h_"
+
yyyymm
+
".csv"
...
@@ -33,9 +33,9 @@ class Collector(utils.LogConfig):
...
@@ -33,9 +33,9 @@ class Collector(utils.LogConfig):
# Compress
# Compress
if
success
:
if
success
:
self
.
logger
.
write
(
self
.
object_id
,
yyyymm
+
":
Compressing ..."
)
self
.
logger
.
info
(
"
Compressing ..."
,
yyyymm
)
utils
.
compress_file
(
dest_filepath
)
utils
.
compress_file
(
dest_filepath
)
self
.
logger
.
write
(
self
.
object_id
,
yyyymm
+
":
Done. Archive file: "
+
dest_filepath
)
self
.
logger
.
info
(
"
Done. Archive file: "
+
dest_filepath
,
yyyymm
)
def
__get_dest_filepath__
(
self
,
yyyymm
:
str
)
->
str
:
def
__get_dest_filepath__
(
self
,
yyyymm
:
str
)
->
str
:
# Build destination directory
# Build destination directory
...
...
wimcollect/ftpcea.py
View file @
87b70b68
...
@@ -30,22 +30,21 @@ class Collector(utils.LogConfig):
...
@@ -30,22 +30,21 @@ class Collector(utils.LogConfig):
day: datetime.date
day: datetime.date
Date of the data which should be downloaded.
Date of the data which should be downloaded.
"""
"""
self
.
logger
.
write
(
self
.
object_id
,
"Download picarro data from "
+
site_id
)
ftp_session
=
ftp
.
connect
(
self
.
config
[
self
.
object_id
],
self
.
logger
,
self
.
object_id
)
ftp_session
=
ftp
.
connect
(
self
.
config
[
self
.
object_id
],
self
.
logger
,
self
.
object_id
)
# Build source file path
# Build source file path
source_dir
=
self
.
distant_base_dir
+
"/"
+
site_id
+
"/picarro/"
source_dir
=
self
.
distant_base_dir
+
"/"
+
site_id
+
"/picarro/"
source_filename
=
site_id
+
"_"
+
picarro_id
+
day
.
strftime
(
"%
y
%m%d"
)
+
".lzma"
source_filename
=
site_id
+
"_"
+
picarro_id
+
day
.
strftime
(
"%
Y
%m%d"
)
+
".lzma"
source_filepath
=
source_dir
+
source_filename
source_filepath
=
source_dir
+
source_filename
# Check that to-be-downloaded file exists
# Check that to-be-downloaded file exists
source_filepaths
=
ftp
.
list_files
(
ftp_session
,
source_dir
,
self
.
logger
,
self
.
object_id
)
source_filepaths
=
ftp
.
list_files
(
ftp_session
,
source_dir
,
self
.
logger
,
self
.
object_id
)
if
source_filepath
not
in
source_filepaths
:
if
source_filepath
not
in
source_filepaths
:
self
.
logger
.
write
(
self
.
object_id
,
"File not found: "
+
source_filepath
)
self
.
logger
.
error
(
"File not found: "
+
source_filepath
,
day
.
strftime
(
"%Y-%m-%d"
),
raise
FileNotFoundError
(
source_filepath
)
exception_type
=
FileNotFoundError
)
# Download file
# Download file
self
.
logger
.
info
(
"Download picarro data from "
+
site_id
,
day
.
strftime
(
"%Y-%m-%d"
))
dest_filepath
=
self
.
__get_picarro_dest_filepath__
(
source_filepath
)
dest_filepath
=
self
.
__get_picarro_dest_filepath__
(
source_filepath
)
success
=
ftp
.
download_file
(
source_filepath
,
dest_filepath
,
self
.
logger
,
self
.
object_id
,
success
=
ftp
.
download_file
(
source_filepath
,
dest_filepath
,
self
.
logger
,
self
.
object_id
,
ftp_session
=
ftp_session
)
ftp_session
=
ftp_session
)
...
@@ -91,7 +90,7 @@ class Collector(utils.LogConfig):
...
@@ -91,7 +90,7 @@ class Collector(utils.LogConfig):
site_id: str
site_id: str
Site's trigram
Site's trigram
"""
"""
self
.
logger
.
write
(
self
.
object_id
,
"Download hobo data from "
+
site_id
)
self
.
logger
.
info
(
"Download hobo data from "
+
site_id
)
ftp_session
=
ftp
.
connect
(
self
.
config
[
self
.
object_id
],
self
.
logger
,
self
.
object_id
)
ftp_session
=
ftp
.
connect
(
self
.
config
[
self
.
object_id
],
self
.
logger
,
self
.
object_id
)
hobo_distant_path
=
self
.
distant_base_dir
+
"/"
+
site_id
+
"/hobo/"
hobo_distant_path
=
self
.
distant_base_dir
+
"/"
+
site_id
+
"/hobo/"
...
...
wimcollect/ftpopar.py
View file @
87b70b68
...
@@ -24,7 +24,6 @@ class Collector(utils.LogConfig):
...
@@ -24,7 +24,6 @@ class Collector(utils.LogConfig):
day: datetime.date
day: datetime.date
Date of the data which should be downloaded.
Date of the data which should be downloaded.
"""
"""
self
.
logger
.
write
(
self
.
object_id
,
"Download Maido's FTIR meteo data."
)
date_str
=
day
.
strftime
(
"%Y%m%d"
)
date_str
=
day
.
strftime
(
"%Y%m%d"
)
# Build source and destination file paths
# Build source and destination file paths
...
@@ -32,14 +31,15 @@ class Collector(utils.LogConfig):
...
@@ -32,14 +31,15 @@ class Collector(utils.LogConfig):
dest_filepath
=
self
.
__get_dest_filepath__
(
day
)
dest_filepath
=
self
.
__get_dest_filepath__
(
day
)
# Download
# Download
self
.
logger
.
info
(
"Download Maido's FTIR meteo data."
,
date_str
)
success
=
ftp
.
download_file
(
source_filepath
,
dest_filepath
,
self
.
logger
,
self
.
object_id
,
success
=
ftp
.
download_file
(
source_filepath
,
dest_filepath
,
self
.
logger
,
self
.
object_id
,
ftp_config
=
self
.
config
[
self
.
object_id
])
ftp_config
=
self
.
config
[
self
.
object_id
])
# Compress
# Compress
if
success
:
if
success
:
self
.
logger
.
write
(
self
.
object_id
,
date_str
+
": Compressing ..."
)
self
.
logger
.
info
(
"Compressing "
+
dest_filepath
,
date_str
)
utils
.
compress_file
(
dest_filepath
)
compressed_filepath
=
utils
.
compress_file
(
dest_filepath
)
self
.
logger
.
write
(
self
.
object_id
,
date_str
+
": Done. Archive file: "
+
dest
_filepath
)
self
.
logger
.
info
(
self
.
object_id
,
date_str
+
": Done. Archive file: "
+
compressed
_filepath
)
def
__get_dest_filepath__
(
self
,
day
:
datetime
.
date
)
->
str
:
def
__get_dest_filepath__
(
self
,
day
:
datetime
.
date
)
->
str
:
# Build destination directory
# Build destination directory
...
...
wimcollect/main.py
View file @
87b70b68
...
@@ -25,9 +25,9 @@ def main():
...
@@ -25,9 +25,9 @@ def main():
# Logger
# Logger
log
=
logger
.
Logger
(
config_parser
)
log
=
logger
.
Logger
(
config_parser
)
log
.
write
(
instrument
=
""
,
message
=
"------------------------------------------------------------------"
)
log
.
write
(
object_id
=
""
,
msg
=
"------------------------------------------------------------------"
)
log
.
write
(
instrument
=
""
,
message
=
" "
.
join
(
sys
.
argv
))
log
.
write
(
object_id
=
""
,
msg
=
" "
.
join
(
sys
.
argv
))
log
.
write
(
instrument
=
''
,
message
=
"------------------------------------------------------------------"
)
log
.
write
(
object_id
=
''
,
msg
=
"------------------------------------------------------------------"
)
yesterday
=
datetime
.
datetime
.
now
()
-
datetime
.
timedelta
(
1
)
yesterday
=
datetime
.
datetime
.
now
()
-
datetime
.
timedelta
(
1
)
...
...
wimcollect/quickviz.py
View file @
87b70b68
...
@@ -21,7 +21,7 @@ class Visualizator(utils.LogConfig):
...
@@ -21,7 +21,7 @@ class Visualizator(utils.LogConfig):
source_filepath
=
os
.
path
.
join
(
source_dir
,
source_filename
)
source_filepath
=
os
.
path
.
join
(
source_dir
,
source_filename
)
if
not
os
.
path
.
exists
(
source_filepath
):
if
not
os
.
path
.
exists
(
source_filepath
):
self
.
logger
.
write
(
self
.
object_id
,
"Failed to open source file "
+
source_filepath
)
self
.
logger
.
error
(
"Failed to open source file "
+
source_filepath
,
day
.
strftime
(
"%Y-%m-%d"
)
)
return
return
# Decompress, and read data as Pandas' DataFrame
# Decompress, and read data as Pandas' DataFrame
...
...
wimcollect/sftpdmc.py
View file @
87b70b68
...
@@ -26,7 +26,7 @@ class Collector(utils.LogConfig):
...
@@ -26,7 +26,7 @@ class Collector(utils.LogConfig):
day: datetime.date
day: datetime.date
Date of the data which should be downloaded.
Date of the data which should be downloaded.
"""
"""
self
.
logger
.
write
(
self
.
object_id
,
"Download DMC Picarro data.
"
)
day_str
=
day
.
strftime
(
"%Y-%m-%d
"
)
# Build source file path
# Build source file path
picarro_number
=
re
.
sub
(
"[^0-9]"
,
""
,
picarro_id
)
picarro_number
=
re
.
sub
(
"[^0-9]"
,
""
,
picarro_id
)
...
@@ -37,18 +37,23 @@ class Collector(utils.LogConfig):
...
@@ -37,18 +37,23 @@ class Collector(utils.LogConfig):
dest_filepath
=
self
.
__get_dest_filepath__
(
day
,
source_filepath
)
dest_filepath
=
self
.
__get_dest_filepath__
(
day
,
source_filepath
)
# Download file
# Download file
self
.
logger
.
info
(
"Downloading DMC Picarro file from SFTP server."
,
day_str
)
sftp_client
=
sftp
.
connect
(
self
.
config
[
self
.
object_id
],
self
.
logger
,
self
.
object_id
)
sftp_client
=
sftp
.
connect
(
self
.
config
[
self
.
object_id
],
self
.
logger
,
self
.
object_id
)
success
=
sftp
.
download_file
(
sftp_client
,
source_filepath
,
dest_filepath
,
self
.
logger
,
self
.
object_id
)
success
=
sftp
.
download_file
(
sftp_client
,
source_filepath
,
dest_filepath
,
self
.
logger
,
self
.
object_id
)
sftp_client
.
close
()
sftp_client
.
close
()
if
not
success
:
self
.
logger
.
error
(
"FAILED to download file in "
+
dest_filepath
,
day_str
)
else
:
self
.
logger
.
info
(
"File downloaded in "
+
dest_filepath
,
day_str
)
# Re-compress, from ZIP to LZMA
# Re-compress, from ZIP to LZMA
if
success
:
if
success
:
self
.
logger
.
write
(
self
.
object_id
,
day
.
strftime
(
"%Y-%m-%d"
)
+
":
Re-compressing from ZIP to LZMA..."
)
self
.
logger
.
info
(
"
Re-compressing from ZIP to LZMA..."
,
day_str
)
lzma_filepath
=
utils
.
recompress_file
(
dest_filepath
)
lzma_filepath
=
utils
.
recompress_file
(
dest_filepath
)
if
lzma_filepath
is
None
:
if
lzma_filepath
is
None
:
self
.
logger
.
write
(
self
.
object_id
,
day
.
strftime
(
"%Y-%m-%d"
)
+
":
FAILED to create archive file"
)
self
.
logger
.
error
(
"
FAILED to create archive file"
,
day_str
)
else
:
else
:
self
.
logger
.
write
(
self
.
object_id
,
day
.
strftime
(
"%Y-%m-%d"
)
+
":
Done. Archive file: "
+
lzma_filepath
)
self
.
logger
.
info
(
"
Done. Archive file: "
+
lzma_filepath
,
day_str
)
def
__get_dest_filepath__
(
self
,
day
:
datetime
.
date
,
source_filepath
:
str
)
->
str
:
def
__get_dest_filepath__
(
self
,
day
:
datetime
.
date
,
source_filepath
:
str
)
->
str
:
# Build destination directory
# Build destination directory
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment