Skip to content
GitLab
Projects
Groups
Snippets
Help
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Open sidebar
JOSSOUD Olivier
WIM Collect
Commits
81beebdd
Commit
81beebdd
authored
Jan 21, 2020
by
JOSSOUD Olivier
Browse files
FTP CEA.
parent
527fa4bb
Changes
2
Hide whitespace changes
Inline
Side-by-side
Showing
2 changed files
with
176 additions
and
0 deletions
+176
-0
wimcollect/ftp.py
wimcollect/ftp.py
+57
-0
wimcollect/ftpcea.py
wimcollect/ftpcea.py
+119
-0
No files found.
wimcollect/ftp.py
0 → 100644
View file @
81beebdd
import
configobj
import
ftplib
import
codecs
import
wimcollect.logger
as
logger
class
FtpCollector
:
def
__init__
(
self
,
config_parser
:
configobj
.
ConfigObj
,
log
:
logger
,
object_id
:
str
):
# Logger
self
.
logger
=
log
self
.
object_id
=
object_id
# Config
self
.
config
=
config_parser
self
.
base_dir
=
self
.
config
[
self
.
object_id
][
"root_dir"
]
def
_ftp_connect_
(
self
):
try
:
self
.
session
=
ftplib
.
FTP
(
host
=
self
.
config
[
self
.
object_id
][
"host"
],
user
=
self
.
config
[
self
.
object_id
][
"user"
],
passwd
=
codecs
.
decode
(
self
.
config
[
self
.
object_id
][
"password"
],
"rot_13"
))
except
ftplib
.
error_perm
as
e
:
self
.
logger
.
write
(
self
.
object_id
,
"Failed to connect to FTP: "
+
str
(
e
))
except
BlockingIOError
as
e
:
self
.
logger
.
write
(
self
.
object_id
,
"Host ["
+
self
.
config
[
self
.
object_id
][
"host"
]
+
"] seems to be unreachable."
)
else
:
self
.
logger
.
write
(
self
.
object_id
,
"Connected to ["
+
self
.
config
[
self
.
object_id
][
"host"
]
+
"] as user: "
+
self
.
config
[
self
.
object_id
][
"user"
])
def
_ftp_list_files_
(
self
,
ftp_session
:
ftplib
.
FTP
,
path
:
str
)
->
list
:
filepaths
=
[]
try
:
filepaths
=
ftp_session
.
nlst
(
path
)
except
ftplib
.
error_perm
as
resp
:
if
str
(
resp
)
==
"550 No files found"
:
self
.
logger
.
write
(
self
.
object_id
,
"No files in directory "
+
path
)
else
:
raise
return
filepaths
def
_ftp_download_file
(
self
,
ftp_session
:
ftplib
.
FTP
,
source_filepath
:
str
,
dest_filepath
:
str
,
delete_if_success
:
bool
=
False
)
->
None
:
self
.
logger
.
write
(
self
.
object_id
,
"Downloading "
+
source_filepath
+
" ..."
)
with
open
(
dest_filepath
,
'wb'
)
as
file
:
response
=
ftp_session
.
retrbinary
(
'RETR %s'
%
source_filepath
,
file
.
write
)
if
response
==
"226 Transfer complete."
:
if
delete_if_success
:
ftp_session
.
delete
(
source_filepath
)
self
.
logger
.
write
(
self
.
object_id
,
"Done. Destination file: "
+
dest_filepath
)
else
:
self
.
logger
.
write
(
self
.
object_id
,
"FAILED. Destination file: "
+
dest_filepath
)
wimcollect/ftpcea.py
0 → 100644
View file @
81beebdd
import
os
import
re
import
configobj
import
datetime
import
wimcollect.logger
as
logger
import
wimcollect.ftp
as
ftp
class
Collector
(
ftp
.
FtpCollector
):
def
__init__
(
self
,
config_parser
:
configobj
.
ConfigObj
,
log
:
logger
):
self
.
object_id
=
"FTPCEA"
ftp
.
FtpCollector
.
__init__
(
self
,
config_parser
,
log
,
self
.
object_id
)
####################################################################################################################
# Picarro
def
download_picarro
(
self
,
site_id
:
str
):
"""Download all Picarro files from FTP server.
The distant files will be deleted from the server if the transfer is successful.
Parameters
----------
site_id: str
Site's trigram
"""
self
.
logger
.
write
(
self
.
object_id
,
"Download picarro data from "
+
site_id
)
self
.
_ftp_connect_
()
picarro_distant_path
=
self
.
base_dir
+
"/"
+
site_id
+
"/picarro/"
self
.
session
.
cwd
(
picarro_distant_path
)
source_filepaths
=
self
.
_ftp_list_files_
(
self
.
session
,
picarro_distant_path
)
for
source_filepath
in
source_filepaths
:
dest_filepath
=
self
.
__get_picarro_dest_filepath__
(
source_filepath
)
self
.
_ftp_download_file
(
self
.
session
,
source_filepath
,
dest_filepath
,
delete_if_success
=
True
)
self
.
session
.
quit
()
def
__get_picarro_dest_filepath__
(
self
,
source_filepath
:
str
)
->
str
:
"""Get the full path where the Picarro data file should be downloaded.
Parameters
----------
source_filepath: str
Full path of the distant (FTP) to-be-downloaded Picarro data file.
Returns
-------
str
Local destination file path.
"""
filename
=
os
.
path
.
basename
(
source_filepath
)
site_id
,
picarro_id
,
date_str
=
re
.
split
(
'_|\.'
,
filename
)[
0
:
3
]
date
=
datetime
.
datetime
.
strptime
(
date_str
,
"%Y%m%d"
).
date
()
dest_dir
=
os
.
path
.
join
(
self
.
config
[
"LOCAL"
][
"base_dir"
],
site_id
,
"picarro"
,
picarro_id
,
str
(
date
.
year
))
if
not
os
.
path
.
exists
(
dest_dir
):
os
.
makedirs
(
dest_dir
)
dest_filepath
=
os
.
path
.
join
(
dest_dir
,
filename
)
return
dest_filepath
####################################################################################################################
# Hobo
def
download_hobo
(
self
,
site_id
:
str
):
"""Download all Hobo files from FTP server.
The distant files will be deleted from the server if the transfer is successful.
Parameters
----------
site_id: str
Site's trigram
"""
self
.
logger
.
write
(
self
.
object_id
,
"Download hobo data from "
+
site_id
)
self
.
_ftp_connect_
()
hobo_distant_path
=
self
.
base_dir
+
"/"
+
site_id
+
"/hobo/"
self
.
session
.
cwd
(
hobo_distant_path
)
source_filepaths
=
self
.
_ftp_list_files_
(
self
.
session
,
hobo_distant_path
)
for
source_filepath
in
source_filepaths
:
dest_filepath
=
self
.
__get_hobo_dest_filepath__
(
source_filepath
)
self
.
_ftp_download_file
(
self
.
session
,
source_filepath
,
dest_filepath
,
delete_if_success
=
True
)
self
.
session
.
quit
()
def
__get_hobo_dest_filepath__
(
self
,
source_filepath
:
str
):
"""Get the full path where the Hobo data file should be downloaded.
Parameters
----------
source_filepath: str
Full path of the distant (FTP) to-be-downloaded Hobo data file.
Returns
-------
str
Local destination file path.
"""
filename
=
os
.
path
.
basename
(
source_filepath
)
filename_parts
=
re
.
split
(
'_|\.|-'
,
filename
)
site_id
=
filename_parts
[
0
]
hobo_id
=
filename_parts
[
2
]
dest_dir
=
os
.
path
.
join
(
self
.
config
[
"LOCAL"
][
"base_dir"
],
site_id
,
"meteo"
,
"hobo_"
+
hobo_id
)
if
not
os
.
path
.
exists
(
dest_dir
):
os
.
makedirs
(
dest_dir
)
dest_filepath
=
os
.
path
.
join
(
dest_dir
,
filename
)
return
dest_filepath
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment