Skip to content
GitLab
Projects
Groups
Snippets
Help
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Docker-in-Docker (DinD) capabilities of public runners deactivated.
More info
Open sidebar
JOSSOUD Olivier
WIM Collect
Commits
81beebdd
Commit
81beebdd
authored
Jan 21, 2020
by
JOSSOUD Olivier
Browse files
FTP CEA.
parent
527fa4bb
Changes
2
Hide whitespace changes
Inline
Side-by-side
Showing
2 changed files
with
176 additions
and
0 deletions
+176
-0
wimcollect/ftp.py
wimcollect/ftp.py
+57
-0
wimcollect/ftpcea.py
wimcollect/ftpcea.py
+119
-0
No files found.
wimcollect/ftp.py
0 → 100644
View file @
81beebdd
import
configobj
import
ftplib
import
codecs
import
wimcollect.logger
as
logger
class
FtpCollector
:
def
__init__
(
self
,
config_parser
:
configobj
.
ConfigObj
,
log
:
logger
,
object_id
:
str
):
# Logger
self
.
logger
=
log
self
.
object_id
=
object_id
# Config
self
.
config
=
config_parser
self
.
base_dir
=
self
.
config
[
self
.
object_id
][
"root_dir"
]
def
_ftp_connect_
(
self
):
try
:
self
.
session
=
ftplib
.
FTP
(
host
=
self
.
config
[
self
.
object_id
][
"host"
],
user
=
self
.
config
[
self
.
object_id
][
"user"
],
passwd
=
codecs
.
decode
(
self
.
config
[
self
.
object_id
][
"password"
],
"rot_13"
))
except
ftplib
.
error_perm
as
e
:
self
.
logger
.
write
(
self
.
object_id
,
"Failed to connect to FTP: "
+
str
(
e
))
except
BlockingIOError
as
e
:
self
.
logger
.
write
(
self
.
object_id
,
"Host ["
+
self
.
config
[
self
.
object_id
][
"host"
]
+
"] seems to be unreachable."
)
else
:
self
.
logger
.
write
(
self
.
object_id
,
"Connected to ["
+
self
.
config
[
self
.
object_id
][
"host"
]
+
"] as user: "
+
self
.
config
[
self
.
object_id
][
"user"
])
def
_ftp_list_files_
(
self
,
ftp_session
:
ftplib
.
FTP
,
path
:
str
)
->
list
:
filepaths
=
[]
try
:
filepaths
=
ftp_session
.
nlst
(
path
)
except
ftplib
.
error_perm
as
resp
:
if
str
(
resp
)
==
"550 No files found"
:
self
.
logger
.
write
(
self
.
object_id
,
"No files in directory "
+
path
)
else
:
raise
return
filepaths
def
_ftp_download_file
(
self
,
ftp_session
:
ftplib
.
FTP
,
source_filepath
:
str
,
dest_filepath
:
str
,
delete_if_success
:
bool
=
False
)
->
None
:
self
.
logger
.
write
(
self
.
object_id
,
"Downloading "
+
source_filepath
+
" ..."
)
with
open
(
dest_filepath
,
'wb'
)
as
file
:
response
=
ftp_session
.
retrbinary
(
'RETR %s'
%
source_filepath
,
file
.
write
)
if
response
==
"226 Transfer complete."
:
if
delete_if_success
:
ftp_session
.
delete
(
source_filepath
)
self
.
logger
.
write
(
self
.
object_id
,
"Done. Destination file: "
+
dest_filepath
)
else
:
self
.
logger
.
write
(
self
.
object_id
,
"FAILED. Destination file: "
+
dest_filepath
)
wimcollect/ftpcea.py
0 → 100644
View file @
81beebdd
import
os
import
re
import
configobj
import
datetime
import
wimcollect.logger
as
logger
import
wimcollect.ftp
as
ftp
class
Collector
(
ftp
.
FtpCollector
):
def
__init__
(
self
,
config_parser
:
configobj
.
ConfigObj
,
log
:
logger
):
self
.
object_id
=
"FTPCEA"
ftp
.
FtpCollector
.
__init__
(
self
,
config_parser
,
log
,
self
.
object_id
)
####################################################################################################################
# Picarro
def
download_picarro
(
self
,
site_id
:
str
):
"""Download all Picarro files from FTP server.
The distant files will be deleted from the server if the transfer is successful.
Parameters
----------
site_id: str
Site's trigram
"""
self
.
logger
.
write
(
self
.
object_id
,
"Download picarro data from "
+
site_id
)
self
.
_ftp_connect_
()
picarro_distant_path
=
self
.
base_dir
+
"/"
+
site_id
+
"/picarro/"
self
.
session
.
cwd
(
picarro_distant_path
)
source_filepaths
=
self
.
_ftp_list_files_
(
self
.
session
,
picarro_distant_path
)
for
source_filepath
in
source_filepaths
:
dest_filepath
=
self
.
__get_picarro_dest_filepath__
(
source_filepath
)
self
.
_ftp_download_file
(
self
.
session
,
source_filepath
,
dest_filepath
,
delete_if_success
=
True
)
self
.
session
.
quit
()
def
__get_picarro_dest_filepath__
(
self
,
source_filepath
:
str
)
->
str
:
"""Get the full path where the Picarro data file should be downloaded.
Parameters
----------
source_filepath: str
Full path of the distant (FTP) to-be-downloaded Picarro data file.
Returns
-------
str
Local destination file path.
"""
filename
=
os
.
path
.
basename
(
source_filepath
)
site_id
,
picarro_id
,
date_str
=
re
.
split
(
'_|\.'
,
filename
)[
0
:
3
]
date
=
datetime
.
datetime
.
strptime
(
date_str
,
"%Y%m%d"
).
date
()
dest_dir
=
os
.
path
.
join
(
self
.
config
[
"LOCAL"
][
"base_dir"
],
site_id
,
"picarro"
,
picarro_id
,
str
(
date
.
year
))
if
not
os
.
path
.
exists
(
dest_dir
):
os
.
makedirs
(
dest_dir
)
dest_filepath
=
os
.
path
.
join
(
dest_dir
,
filename
)
return
dest_filepath
####################################################################################################################
# Hobo
def
download_hobo
(
self
,
site_id
:
str
):
"""Download all Hobo files from FTP server.
The distant files will be deleted from the server if the transfer is successful.
Parameters
----------
site_id: str
Site's trigram
"""
self
.
logger
.
write
(
self
.
object_id
,
"Download hobo data from "
+
site_id
)
self
.
_ftp_connect_
()
hobo_distant_path
=
self
.
base_dir
+
"/"
+
site_id
+
"/hobo/"
self
.
session
.
cwd
(
hobo_distant_path
)
source_filepaths
=
self
.
_ftp_list_files_
(
self
.
session
,
hobo_distant_path
)
for
source_filepath
in
source_filepaths
:
dest_filepath
=
self
.
__get_hobo_dest_filepath__
(
source_filepath
)
self
.
_ftp_download_file
(
self
.
session
,
source_filepath
,
dest_filepath
,
delete_if_success
=
True
)
self
.
session
.
quit
()
def
__get_hobo_dest_filepath__
(
self
,
source_filepath
:
str
):
"""Get the full path where the Hobo data file should be downloaded.
Parameters
----------
source_filepath: str
Full path of the distant (FTP) to-be-downloaded Hobo data file.
Returns
-------
str
Local destination file path.
"""
filename
=
os
.
path
.
basename
(
source_filepath
)
filename_parts
=
re
.
split
(
'_|\.|-'
,
filename
)
site_id
=
filename_parts
[
0
]
hobo_id
=
filename_parts
[
2
]
dest_dir
=
os
.
path
.
join
(
self
.
config
[
"LOCAL"
][
"base_dir"
],
site_id
,
"meteo"
,
"hobo_"
+
hobo_id
)
if
not
os
.
path
.
exists
(
dest_dir
):
os
.
makedirs
(
dest_dir
)
dest_filepath
=
os
.
path
.
join
(
dest_dir
,
filename
)
return
dest_filepath
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment