Skip to content
GitLab
Projects
Groups
Snippets
Help
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Open sidebar
JOSSOUD Olivier
WIM Collect
Commits
4e0270cd
Commit
4e0270cd
authored
Jan 27, 2020
by
JOSSOUD Olivier
Browse files
All. Use new logger functions.
parent
36372555
Pipeline
#57588
passed with stages
in 58 seconds
Changes
6
Pipelines
1
Hide whitespace changes
Inline
Side-by-side
Showing
6 changed files
with
29 additions
and
47 deletions
+29
-47
wimcollect/common/ftp.py
wimcollect/common/ftp.py
+12
-20
wimcollect/common/sftp.py
wimcollect/common/sftp.py
+8
-17
wimcollect/ftpaeris.py
wimcollect/ftpaeris.py
+1
-2
wimcollect/ftpcea.py
wimcollect/ftpcea.py
+6
-6
wimcollect/ftpopar.py
wimcollect/ftpopar.py
+1
-1
wimcollect/sftpdmc.py
wimcollect/sftpdmc.py
+1
-1
No files found.
wimcollect/common/ftp.py
View file @
4e0270cd
...
...
@@ -5,7 +5,7 @@ import re
import
wimcollect.common.logger
as
logger
def
connect
(
ftp_config
:
dict
,
log
:
logger
.
Logger
,
object_id
:
str
)
->
ftplib
.
FTP
:
def
connect
(
ftp_config
:
dict
,
log
:
logger
.
Logger
)
->
ftplib
.
FTP
:
"""Connect to FTP server.
Parameters
...
...
@@ -14,8 +14,6 @@ def connect(ftp_config: dict, log: logger.Logger, object_id: str) -> ftplib.FTP:
Dict containing FTP connection information: `host`, `user` and `password`.
log: wimcollect.common.logger.Logger
Logger to record debug/info/error messages.
object_id: str
Object identifier, used as marker for log messages.
Returns
-------
...
...
@@ -29,18 +27,18 @@ def connect(ftp_config: dict, log: logger.Logger, object_id: str) -> ftplib.FTP:
try
:
session
=
ftplib
.
FTP
(
host
=
host
,
user
=
user
,
passwd
=
passwd
)
except
ftplib
.
error_perm
as
e
:
log
.
write
(
object_id
,
"Failed to connect to "
+
host
+
" as "
+
user
+
": "
+
str
(
e
))
log
.
error
(
"Failed to connect to "
+
host
+
" as "
+
user
+
": "
+
str
(
e
))
raise
except
BlockingIOError
as
e
:
log
.
write
(
object_id
,
"Host ["
+
host
+
"] seems to be unreachable."
)
log
.
error
(
"Host ["
+
host
+
"] seems to be unreachable."
)
raise
else
:
log
.
write
(
object_id
,
"Connected to ["
+
host
+
"] as user: "
+
user
)
log
.
info
(
"Connected to ["
+
host
+
"] as user: "
+
user
)
return
session
def
list_files
(
ftp_session
:
ftplib
.
FTP
,
distant_directory_path
:
str
,
log
:
logger
.
Logger
,
object_id
:
str
)
->
list
:
def
list_files
(
ftp_session
:
ftplib
.
FTP
,
distant_directory_path
:
str
,
log
:
logger
.
Logger
)
->
list
:
"""List the files in the distant FTP directory.
Parameters
...
...
@@ -51,8 +49,6 @@ def list_files(ftp_session: ftplib.FTP, distant_directory_path: str, log: logger
Full path of the directory whose content should be listed.
log: wimcollect.common.logger.Logger
Logger to record debug/info/error messages.
object_id: str
Object identifier, used as marker for log messages.
Returns
-------
...
...
@@ -65,7 +61,7 @@ def list_files(ftp_session: ftplib.FTP, distant_directory_path: str, log: logger
filepaths
=
ftp_session
.
nlst
(
distant_directory_path
)
except
ftplib
.
error_perm
as
resp
:
if
str
(
resp
)
==
"550 No files found"
:
log
.
write
(
object_id
,
"No files in directory "
+
distant_directory_path
)
log
.
error
(
"No files in directory "
+
distant_directory_path
,
exception_type
=
FileNotFoundError
)
else
:
raise
...
...
@@ -73,7 +69,7 @@ def list_files(ftp_session: ftplib.FTP, distant_directory_path: str, log: logger
def
download_file
(
source_filepath
:
str
,
dest_filepath
:
str
,
log
:
logger
.
Logger
,
object_id
:
str
,
log
:
logger
.
Logger
,
ftp_config
:
dict
=
None
,
ftp_session
:
ftplib
.
FTP
=
None
,
delete_if_success
:
bool
=
False
)
->
bool
:
"""Download a file from the FTP server.
...
...
@@ -86,8 +82,6 @@ def download_file(source_filepath: str, dest_filepath: str,
Full path where the downloaded file should be stored.
log: wimcollect.common.logger.Logger
Logger to record debug/info/error messages.
object_id: str
Object identifier, used as marker for log messages.
ftp_config: dict, optional
Dict containing FTP connection information: `host`, `user` and `password`. If not set, `ftp_session` must be
provided.
...
...
@@ -105,12 +99,12 @@ def download_file(source_filepath: str, dest_filepath: str,
# Open FTP session if missing
if
ftp_session
is
None
:
close_session
=
True
ftp_session
=
connect
(
ftp_config
,
log
,
object_id
)
ftp_session
=
connect
(
ftp_config
,
log
)
else
:
close_session
=
False
# Try to download file
log
.
write
(
object_id
,
"Downloading "
+
source_filepath
+
" ..."
)
log
.
info
(
"Downloading "
+
source_filepath
+
" ..."
)
with
open
(
dest_filepath
,
'wb'
)
as
file
:
try
:
response
=
ftp_session
.
retrbinary
(
'RETR %s'
%
source_filepath
,
file
.
write
)
...
...
@@ -122,12 +116,10 @@ def download_file(source_filepath: str, dest_filepath: str,
if
response_code
==
226
:
if
delete_if_success
:
ftp_session
.
delete
(
source_filepath
)
log
.
write
(
object_id
,
"Done. Destination file: "
+
dest_filepath
)
log
.
info
(
"Done. Destination file: "
+
dest_filepath
)
else
:
msg
=
"Failed to download ["
+
source_filepath
+
"] in ["
+
dest_filepath
+
"]."
\
" FTP response: ["
+
response
+
"]"
log
.
write
(
object_id
,
msg
)
raise
Exception
(
msg
)
log
.
error
(
"Failed to download ["
+
source_filepath
+
"] in ["
+
dest_filepath
+
"]."
\
" FTP response: ["
+
response
+
"]"
)
# Close the FTP session if it has been opened at the beginning of this function.
if
close_session
:
...
...
wimcollect/common/sftp.py
View file @
4e0270cd
...
...
@@ -4,7 +4,7 @@ import os
import
wimcollect.common.logger
as
logger
def
connect
(
sftp_config
:
dict
,
log
:
logger
.
Logger
,
object_id
:
str
)
->
paramiko
.
SFTPClient
:
def
connect
(
sftp_config
:
dict
,
log
:
logger
.
Logger
)
->
paramiko
.
SFTPClient
:
"""Connect to SFTP server.
Notes
...
...
@@ -17,8 +17,6 @@ def connect(sftp_config: dict, log: logger.Logger, object_id: str) -> paramiko.S
Dict containing SFTP connection information: `host`, `user` and `private_key_path`.
log: wimcollect.common.logger
Logger to record debug/info/error messages.
object_id: str
Object identifier, used as marker for log messages.
Returns
-------
...
...
@@ -36,16 +34,15 @@ def connect(sftp_config: dict, log: logger.Logger, object_id: str) -> paramiko.S
username
=
user
,
key_filename
=
sftp_config
[
"private_key_path"
])
except
Exception
as
e
:
log
.
write
(
object_id
,
"Failed to connect to "
+
host
+
" as "
+
user
+
": "
+
str
(
e
))
raise
e
log
.
error
(
"Failed to connect to "
+
host
+
" as "
+
user
+
": "
+
str
(
e
))
else
:
log
.
write
(
object_id
,
"Connected to ["
+
host
+
"] as user: "
+
user
)
log
.
info
(
"Connected to ["
+
host
+
"] as user: "
+
user
)
return
client
.
open_sftp
()
def
download_file
(
sftp_client
:
paramiko
.
SFTPClient
,
source_filepath
:
str
,
dest_filepath
:
str
,
log
:
logger
.
Logger
,
object_id
:
str
)
->
bool
:
log
:
logger
.
Logger
)
->
bool
:
"""Download a file from the FTP server.
Parameters
...
...
@@ -58,30 +55,24 @@ def download_file(sftp_client: paramiko.SFTPClient, source_filepath: str, dest_f
Full path where the downloaded file should be stored.
log: wimcollect.logger
Logger to record debug/info/error messages.
object_id: str
Object identifier, used as marker for log messages.
Returns
-------
bool
`True` if everything went well, otherwise raises Exception.
"""
log
.
write
(
object_id
,
"Downloading "
+
source_filepath
+
" ..."
)
log
.
info
(
"Downloading "
+
source_filepath
+
" ..."
)
try
:
sftp_client
.
get
(
source_filepath
,
dest_filepath
)
except
FileNotFoundError
:
msg
=
"File not found: "
+
source_filepath
log
.
write
(
object_id
,
msg
)
raise
FileNotFoundError
(
msg
)
log
.
error
(
"File not found: "
+
source_filepath
,
exception_type
=
FileNotFoundError
)
# Check if the destination file exists
success
=
os
.
path
.
exists
(
dest_filepath
)
if
success
:
log
.
write
(
object_id
,
"File ["
+
source_filepath
+
"] downloaded in ["
+
dest_filepath
+
"]."
)
log
.
info
(
"File ["
+
source_filepath
+
"] downloaded in ["
+
dest_filepath
+
"]."
)
else
:
msg
=
"Failed to download ["
+
source_filepath
+
"] in ["
+
dest_filepath
+
"]."
log
.
write
(
object_id
,
msg
)
raise
Exception
(
msg
)
log
.
error
(
"Failed to download ["
+
source_filepath
+
"] in ["
+
dest_filepath
+
"]."
)
return
success
wimcollect/ftpaeris.py
View file @
4e0270cd
...
...
@@ -28,8 +28,7 @@ class Collector(utils.LogConfig):
dest_filepath
=
self
.
__get_dest_filepath__
(
yyyymm
)
# Download
success
=
ftp
.
download_file
(
source_filepath
,
dest_filepath
,
self
.
logger
,
self
.
object_id
,
ftp_config
=
self
.
config
[
self
.
object_id
])
success
=
ftp
.
download_file
(
source_filepath
,
dest_filepath
,
self
.
logger
,
ftp_config
=
self
.
config
[
self
.
object_id
])
# Compress
if
success
:
...
...
wimcollect/ftpcea.py
View file @
4e0270cd
...
...
@@ -30,7 +30,7 @@ class Collector(utils.LogConfig):
day: datetime.date
Date of the data which should be downloaded.
"""
ftp_session
=
ftp
.
connect
(
self
.
config
[
self
.
object_id
],
self
.
logger
,
self
.
object_id
)
ftp_session
=
ftp
.
connect
(
self
.
config
[
self
.
object_id
],
self
.
logger
)
# Build source file path
source_dir
=
self
.
distant_base_dir
+
"/"
+
site_id
+
"/picarro/"
...
...
@@ -38,7 +38,7 @@ class Collector(utils.LogConfig):
source_filepath
=
source_dir
+
source_filename
# Check that to-be-downloaded file exists
source_filepaths
=
ftp
.
list_files
(
ftp_session
,
source_dir
,
self
.
logger
,
self
.
object_id
)
source_filepaths
=
ftp
.
list_files
(
ftp_session
,
source_dir
,
self
.
logger
)
if
source_filepath
not
in
source_filepaths
:
self
.
logger
.
error
(
"File not found: "
+
source_filepath
,
day
.
strftime
(
"%Y-%m-%d"
),
exception_type
=
FileNotFoundError
)
...
...
@@ -46,7 +46,7 @@ class Collector(utils.LogConfig):
# Download file
self
.
logger
.
info
(
"Download picarro data from "
+
site_id
,
day
.
strftime
(
"%Y-%m-%d"
))
dest_filepath
=
self
.
__get_picarro_dest_filepath__
(
source_filepath
)
success
=
ftp
.
download_file
(
source_filepath
,
dest_filepath
,
self
.
logger
,
self
.
object_id
,
success
=
ftp
.
download_file
(
source_filepath
,
dest_filepath
,
self
.
logger
,
ftp_session
=
ftp_session
)
ftp_session
.
quit
()
...
...
@@ -92,13 +92,13 @@ class Collector(utils.LogConfig):
"""
self
.
logger
.
info
(
"Download hobo data from "
+
site_id
)
ftp_session
=
ftp
.
connect
(
self
.
config
[
self
.
object_id
],
self
.
logger
,
self
.
object_id
)
ftp_session
=
ftp
.
connect
(
self
.
config
[
self
.
object_id
],
self
.
logger
)
hobo_distant_path
=
self
.
distant_base_dir
+
"/"
+
site_id
+
"/hobo/"
source_filepaths
=
ftp
.
list_files
(
ftp_session
,
hobo_distant_path
,
self
.
logger
,
self
.
object_id
)
source_filepaths
=
ftp
.
list_files
(
ftp_session
,
hobo_distant_path
,
self
.
logger
)
for
source_filepath
in
source_filepaths
:
dest_filepath
=
self
.
__get_hobo_dest_filepath__
(
source_filepath
)
ftp
.
download_file
(
source_filepath
,
dest_filepath
,
self
.
logger
,
self
.
object_id
,
ftp
.
download_file
(
source_filepath
,
dest_filepath
,
self
.
logger
,
ftp_session
=
ftp_session
)
ftp_session
.
quit
()
...
...
wimcollect/ftpopar.py
View file @
4e0270cd
...
...
@@ -32,7 +32,7 @@ class Collector(utils.LogConfig):
# Download
self
.
logger
.
info
(
"Download Maido's FTIR meteo data."
,
date_str
)
success
=
ftp
.
download_file
(
source_filepath
,
dest_filepath
,
self
.
logger
,
self
.
object_id
,
success
=
ftp
.
download_file
(
source_filepath
,
dest_filepath
,
self
.
logger
,
ftp_config
=
self
.
config
[
self
.
object_id
])
# Compress
...
...
wimcollect/sftpdmc.py
View file @
4e0270cd
...
...
@@ -38,7 +38,7 @@ class Collector(utils.LogConfig):
# Download file
self
.
logger
.
info
(
"Downloading DMC Picarro file from SFTP server."
,
day_str
)
sftp_client
=
sftp
.
connect
(
self
.
config
[
self
.
object_id
],
self
.
logger
,
self
.
object_id
)
sftp_client
=
sftp
.
connect
(
self
.
config
[
self
.
object_id
],
self
.
logger
)
success
=
sftp
.
download_file
(
sftp_client
,
source_filepath
,
dest_filepath
,
self
.
logger
,
self
.
object_id
)
sftp_client
.
close
()
if
not
success
:
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment