Skip to content
GitLab
Projects
Groups
Snippets
Help
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Open sidebar
JOSSOUD Olivier
WIM Collect
Commits
b8e1d505
Commit
b8e1d505
authored
Jan 21, 2020
by
JOSSOUD Olivier
Browse files
HTTP DDU
parent
b9bf3f98
Changes
1
Hide whitespace changes
Inline
Side-by-side
Showing
1 changed file
with
61 additions
and
0 deletions
+61
-0
wimcollect/httpddu.py
wimcollect/httpddu.py
+61
-0
No files found.
wimcollect/httpddu.py
0 → 100644
View file @
b8e1d505
import
os
import
configobj
import
datetime
import
urllib.request
import
wimcollect.utils
as
utils
import
wimcollect.logger
as
logger
class
Collector
:
def
__init__
(
self
,
config_parser
:
configobj
.
ConfigObj
,
log
:
logger
):
# Logger
self
.
logger
=
log
self
.
object_id
=
"HTTPDDU"
# Config
self
.
config
=
config_parser
self
.
base_url
=
self
.
config
[
self
.
object_id
][
"base_url"
]
def
download_picarro
(
self
,
day
:
datetime
.
date
):
"""Download Picarro data file from IPEV's HTTP server.
"""
self
.
logger
.
write
(
self
.
object_id
,
"Download DDU Picarro data."
)
# Build source file url
picarro_id
=
self
.
config
[
self
.
object_id
][
"picarro_id"
]
source_fileurl
=
self
.
base_url
+
"/DDU_"
+
picarro_id
+
"_"
+
day
.
strftime
(
"%Y%m%d"
)
+
".zip"
dest_filepath
=
self
.
__get_dest_filepath__
(
day
,
source_fileurl
)
response
=
urllib
.
request
.
urlretrieve
(
source_fileurl
,
dest_filepath
)
success
=
response
[
0
]
==
dest_filepath
and
os
.
path
.
exists
(
dest_filepath
)
if
not
success
:
self
.
logger
.
write
(
self
.
object_id
,
day
.
strftime
(
"%Y-%m-%d"
)
+
": FAILED to download file."
)
return
False
# Recompress file, from ZIP to LZMA
self
.
logger
.
write
(
self
.
object_id
,
day
.
strftime
(
"%Y-%m-%d"
)
+
": Re-compressing from ZIP to LZMA..."
)
lzma_filepath
=
utils
.
recompress_file
(
dest_filepath
)
if
lzma_filepath
is
None
:
self
.
logger
.
write
(
self
.
object_id
,
day
.
strftime
(
"%Y-%m-%d"
)
+
": FAILED to create archive file"
)
return
False
else
:
self
.
logger
.
write
(
self
.
object_id
,
day
.
strftime
(
"%Y-%m-%d"
)
+
": Done. Archive file: "
+
lzma_filepath
)
return
True
def
__get_dest_filepath__
(
self
,
day
:
datetime
.
date
,
source_filepath
:
str
)
->
str
:
# Build destination directory
dest_dir
=
os
.
path
.
join
(
self
.
config
[
"LOCAL"
][
"base_dir"
],
"DDU"
,
"picarro"
,
self
.
config
[
self
.
object_id
][
"picarro_id"
],
str
(
day
.
year
))
if
not
os
.
path
.
exists
(
dest_dir
):
os
.
makedirs
(
dest_dir
)
# Build destination filename
dest_filename
=
os
.
path
.
basename
(
source_filepath
)
# Build full file path
dest_filepath
=
os
.
path
.
join
(
dest_dir
,
dest_filename
)
return
dest_filepath
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment