Skip to content
GitLab
Projects
Groups
Snippets
Help
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Docker-in-Docker (DinD) capabilities of public runners deactivated.
More info
Open sidebar
JOSSOUD Olivier
WIM Collect
Commits
a9595cdd
Commit
a9595cdd
authored
Jan 24, 2020
by
JOSSOUD Olivier
Browse files
FTP. Remove useless `ftp_` for ftp's functions name.
parent
7d500ea0
Changes
4
Hide whitespace changes
Inline
Side-by-side
Showing
4 changed files
with
28 additions
and
27 deletions
+28
-27
wimcollect/common/ftp.py
wimcollect/common/ftp.py
+16
-15
wimcollect/ftpaeris.py
wimcollect/ftpaeris.py
+2
-2
wimcollect/ftpcea.py
wimcollect/ftpcea.py
+8
-8
wimcollect/ftpopar.py
wimcollect/ftpopar.py
+2
-2
No files found.
wimcollect/common/ftp.py
View file @
a9595cdd
...
...
@@ -5,14 +5,14 @@ import re
import
wimcollect.common.logger
as
logger
def
ftp_
connect
(
ftp_config
:
dict
,
log
:
logger
.
Logger
,
object_id
:
str
)
->
ftplib
.
FTP
:
"""Connect to FTP server
according to parameters read from settings.ini file
.
def
connect
(
ftp_config
:
dict
,
log
:
logger
.
Logger
,
object_id
:
str
)
->
ftplib
.
FTP
:
"""Connect to FTP server.
Parameters
----------
ftp_config: dict
Dict containing FTP connection information: `host`, `user` and `password`.
log: wimcollect.
l
ogger
log: wimcollect.
common.logger.L
ogger
Logger to record debug/info/error messages.
object_id: str
Object identifier, used as marker for log messages.
...
...
@@ -40,7 +40,7 @@ def ftp_connect(ftp_config: dict, log: logger.Logger, object_id: str) -> ftplib.
return
session
def
ftp_
list_files
(
ftp_session
:
ftplib
.
FTP
,
distant_directory_path
:
str
,
log
:
logger
.
Logger
,
object_id
:
str
)
->
list
:
def
list_files
(
ftp_session
:
ftplib
.
FTP
,
distant_directory_path
:
str
,
log
:
logger
.
Logger
,
object_id
:
str
)
->
list
:
"""List the files in the distant FTP directory.
Parameters
...
...
@@ -49,7 +49,7 @@ def ftp_list_files(ftp_session: ftplib.FTP, distant_directory_path: str, log: lo
FTP session.
distant_directory_path: str
Full path of the directory whose content should be listed.
log: wimcollect.
l
ogger
log: wimcollect.
common.logger.L
ogger
Logger to record debug/info/error messages.
object_id: str
Object identifier, used as marker for log messages.
...
...
@@ -72,19 +72,19 @@ def ftp_list_files(ftp_session: ftplib.FTP, distant_directory_path: str, log: lo
return
filepaths
def
ftp_
download_file
(
source_filepath
:
str
,
dest_filepath
:
str
,
log
:
logger
.
Logger
,
object_id
:
str
,
ftp_config
:
dict
=
None
,
ftp_session
:
ftplib
.
FTP
=
None
,
delete_if_success
:
bool
=
False
)
->
bool
:
def
download_file
(
source_filepath
:
str
,
dest_filepath
:
str
,
log
:
logger
.
Logger
,
object_id
:
str
,
ftp_config
:
dict
=
None
,
ftp_session
:
ftplib
.
FTP
=
None
,
delete_if_success
:
bool
=
False
)
->
bool
:
"""Download a file from the FTP server.
Parameters
----------
source_filepath: str
Full path of the distant to-be
_
downloaded file.
Full path of the distant to-be
-
downloaded file.
dest_filepath: str
Full path where the downloaded file should be stored.
log: wimcollect.
l
ogger
log: wimcollect.
common.logger.L
ogger
Logger to record debug/info/error messages.
object_id: str
Object identifier, used as marker for log messages.
...
...
@@ -105,7 +105,7 @@ def ftp_download_file(source_filepath: str, dest_filepath: str,
# Open FTP session if missing
if
ftp_session
is
None
:
close_session
=
True
ftp_session
=
ftp_
connect
(
ftp_config
,
log
,
object_id
)
ftp_session
=
connect
(
ftp_config
,
log
,
object_id
)
else
:
close_session
=
False
...
...
@@ -124,9 +124,10 @@ def ftp_download_file(source_filepath: str, dest_filepath: str,
ftp_session
.
delete
(
source_filepath
)
log
.
write
(
object_id
,
"Done. Destination file: "
+
dest_filepath
)
else
:
log
.
write
(
object_id
,
"FAILED. Destination file: "
+
dest_filepath
)
raise
Exception
(
"Failed to download ["
+
source_filepath
+
"] in ["
+
dest_filepath
+
"]."
" FTP response: ["
+
response
+
"]"
)
msg
=
"Failed to download ["
+
source_filepath
+
"] in ["
+
dest_filepath
+
"]."
\
" FTP response: ["
+
response
+
"]"
log
.
write
(
object_id
,
msg
)
raise
Exception
(
msg
)
# Close the FTP session if it has been opened at the beginning of this function.
if
close_session
:
...
...
wimcollect/ftpaeris.py
View file @
a9595cdd
...
...
@@ -28,8 +28,8 @@ class Collector(utils.LogConfig):
dest_filepath
=
self
.
__get_dest_filepath__
(
yyyymm
)
# Download
success
=
ftp
.
ftp_
download_file
(
source_filepath
,
dest_filepath
,
self
.
logger
,
self
.
object_id
,
ftp_config
=
self
.
config
[
self
.
object_id
])
success
=
ftp
.
download_file
(
source_filepath
,
dest_filepath
,
self
.
logger
,
self
.
object_id
,
ftp_config
=
self
.
config
[
self
.
object_id
])
# Compress
if
success
:
...
...
wimcollect/ftpcea.py
View file @
a9595cdd
...
...
@@ -32,7 +32,7 @@ class Collector(utils.LogConfig):
"""
self
.
logger
.
write
(
self
.
object_id
,
"Download picarro data from "
+
site_id
)
ftp_session
=
ftp
.
ftp_
connect
(
self
.
config
[
self
.
object_id
],
self
.
logger
,
self
.
object_id
)
ftp_session
=
ftp
.
connect
(
self
.
config
[
self
.
object_id
],
self
.
logger
,
self
.
object_id
)
# Build source file path
source_dir
=
self
.
distant_base_dir
+
"/"
+
site_id
+
"/picarro/"
...
...
@@ -40,15 +40,15 @@ class Collector(utils.LogConfig):
source_filepath
=
source_dir
+
source_filename
# Check that to-be-downloaded file exists
source_filepaths
=
ftp
.
ftp_
list_files
(
ftp_session
,
source_dir
,
self
.
logger
,
self
.
object_id
)
source_filepaths
=
ftp
.
list_files
(
ftp_session
,
source_dir
,
self
.
logger
,
self
.
object_id
)
if
source_filepath
not
in
source_filepaths
:
self
.
logger
.
write
(
self
.
object_id
,
"File not found: "
+
source_filepath
)
raise
FileNotFoundError
(
source_filepath
)
# Download file
dest_filepath
=
self
.
__get_picarro_dest_filepath__
(
source_filepath
)
success
=
ftp
.
ftp_
download_file
(
source_filepath
,
dest_filepath
,
self
.
logger
,
self
.
object_id
,
ftp_session
=
ftp_session
)
success
=
ftp
.
download_file
(
source_filepath
,
dest_filepath
,
self
.
logger
,
self
.
object_id
,
ftp_session
=
ftp_session
)
ftp_session
.
quit
()
...
...
@@ -93,14 +93,14 @@ class Collector(utils.LogConfig):
"""
self
.
logger
.
write
(
self
.
object_id
,
"Download hobo data from "
+
site_id
)
ftp_session
=
ftp
.
ftp_
connect
(
self
.
config
[
self
.
object_id
],
self
.
logger
,
self
.
object_id
)
ftp_session
=
ftp
.
connect
(
self
.
config
[
self
.
object_id
],
self
.
logger
,
self
.
object_id
)
hobo_distant_path
=
self
.
distant_base_dir
+
"/"
+
site_id
+
"/hobo/"
source_filepaths
=
ftp
.
ftp_
list_files
(
ftp_session
,
hobo_distant_path
,
self
.
logger
,
self
.
object_id
)
source_filepaths
=
ftp
.
list_files
(
ftp_session
,
hobo_distant_path
,
self
.
logger
,
self
.
object_id
)
for
source_filepath
in
source_filepaths
:
dest_filepath
=
self
.
__get_hobo_dest_filepath__
(
source_filepath
)
ftp
.
ftp_
download_file
(
source_filepath
,
dest_filepath
,
self
.
logger
,
self
.
object_id
,
ftp_session
=
ftp_session
)
ftp
.
download_file
(
source_filepath
,
dest_filepath
,
self
.
logger
,
self
.
object_id
,
ftp_session
=
ftp_session
)
ftp_session
.
quit
()
...
...
wimcollect/ftpopar.py
View file @
a9595cdd
...
...
@@ -32,8 +32,8 @@ class Collector(utils.LogConfig):
dest_filepath
=
self
.
__get_dest_filepath__
(
day
)
# Download
success
=
ftp
.
ftp_
download_file
(
source_filepath
,
dest_filepath
,
self
.
logger
,
self
.
object_id
,
ftp_config
=
self
.
config
[
self
.
object_id
])
success
=
ftp
.
download_file
(
source_filepath
,
dest_filepath
,
self
.
logger
,
self
.
object_id
,
ftp_config
=
self
.
config
[
self
.
object_id
])
# Compress
if
success
:
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment