Skip to content
GitLab
Projects
Groups
Snippets
Help
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Docker-in-Docker (DinD) capabilities of public runners deactivated.
More info
Open sidebar
JOSSOUD Olivier
WIM Collect
Commits
b8e1d505
Commit
b8e1d505
authored
Jan 21, 2020
by
JOSSOUD Olivier
Browse files
HTTP DDU
parent
b9bf3f98
Changes
1
Hide whitespace changes
Inline
Side-by-side
Showing
1 changed file
with
61 additions
and
0 deletions
+61
-0
wimcollect/httpddu.py
wimcollect/httpddu.py
+61
-0
No files found.
wimcollect/httpddu.py
0 → 100644
View file @
b8e1d505
import
os
import
configobj
import
datetime
import
urllib.request
import
wimcollect.utils
as
utils
import
wimcollect.logger
as
logger
class
Collector
:
def
__init__
(
self
,
config_parser
:
configobj
.
ConfigObj
,
log
:
logger
):
# Logger
self
.
logger
=
log
self
.
object_id
=
"HTTPDDU"
# Config
self
.
config
=
config_parser
self
.
base_url
=
self
.
config
[
self
.
object_id
][
"base_url"
]
def
download_picarro
(
self
,
day
:
datetime
.
date
):
"""Download Picarro data file from IPEV's HTTP server.
"""
self
.
logger
.
write
(
self
.
object_id
,
"Download DDU Picarro data."
)
# Build source file url
picarro_id
=
self
.
config
[
self
.
object_id
][
"picarro_id"
]
source_fileurl
=
self
.
base_url
+
"/DDU_"
+
picarro_id
+
"_"
+
day
.
strftime
(
"%Y%m%d"
)
+
".zip"
dest_filepath
=
self
.
__get_dest_filepath__
(
day
,
source_fileurl
)
response
=
urllib
.
request
.
urlretrieve
(
source_fileurl
,
dest_filepath
)
success
=
response
[
0
]
==
dest_filepath
and
os
.
path
.
exists
(
dest_filepath
)
if
not
success
:
self
.
logger
.
write
(
self
.
object_id
,
day
.
strftime
(
"%Y-%m-%d"
)
+
": FAILED to download file."
)
return
False
# Recompress file, from ZIP to LZMA
self
.
logger
.
write
(
self
.
object_id
,
day
.
strftime
(
"%Y-%m-%d"
)
+
": Re-compressing from ZIP to LZMA..."
)
lzma_filepath
=
utils
.
recompress_file
(
dest_filepath
)
if
lzma_filepath
is
None
:
self
.
logger
.
write
(
self
.
object_id
,
day
.
strftime
(
"%Y-%m-%d"
)
+
": FAILED to create archive file"
)
return
False
else
:
self
.
logger
.
write
(
self
.
object_id
,
day
.
strftime
(
"%Y-%m-%d"
)
+
": Done. Archive file: "
+
lzma_filepath
)
return
True
def
__get_dest_filepath__
(
self
,
day
:
datetime
.
date
,
source_filepath
:
str
)
->
str
:
# Build destination directory
dest_dir
=
os
.
path
.
join
(
self
.
config
[
"LOCAL"
][
"base_dir"
],
"DDU"
,
"picarro"
,
self
.
config
[
self
.
object_id
][
"picarro_id"
],
str
(
day
.
year
))
if
not
os
.
path
.
exists
(
dest_dir
):
os
.
makedirs
(
dest_dir
)
# Build destination filename
dest_filename
=
os
.
path
.
basename
(
source_filepath
)
# Build full file path
dest_filepath
=
os
.
path
.
join
(
dest_dir
,
dest_filename
)
return
dest_filepath
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment