Skip to content
GitLab
Projects
Groups
Snippets
Help
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Open sidebar
JOSSOUD Olivier
WIM Collect
Commits
8ed70a68
Commit
8ed70a68
authored
Jan 24, 2020
by
JOSSOUD Olivier
Browse files
Code layout and comments.
parent
ba207b0f
Pipeline
#57299
passed with stages
in 1 minute and 13 seconds
Changes
4
Pipelines
1
Hide whitespace changes
Inline
Side-by-side
Showing
4 changed files
with
81 additions
and
43 deletions
+81
-43
wimcollect/config/settings_default.ini
wimcollect/config/settings_default.ini
+64
-35
wimcollect/ftpopar.py
wimcollect/ftpopar.py
+1
-1
wimcollect/httpddu.py
wimcollect/httpddu.py
+11
-2
wimcollect/sftpdmc.py
wimcollect/sftpdmc.py
+5
-5
No files found.
wimcollect/config/settings_default.ini
View file @
8ed70a68
[SITE]
[LOCAL]
# Absolute path of the local directory where the collected data file will be stored.
base_dir
=
/XXX/DATA/raw/
# Site's trigram (e.g. REU, AMS, SRT, etc.)
site_id
=
REU
########################################################################################################################
[LOGGER]
[GLOBAL]
# Log files older than `keep_log_day` will be automatically deleted.
keep_log_days
=
5
# Absolute path o
f
the
local
directory where the
to-be-sent compressed files will temporarily b
e stored.
to_send
_dir
=
C:
\
w
imcollect
\t
oSend
\
# Absolute path
t
o the directory where the
log files ar
e stored.
log
_dir
=
/XXX/
wimcollect
/log/
[PICARRO]
########################################################################################################################
[FTPCEA]
#
Picarro's unique identifier (e.g. HIDS2179, HIDS2108, etc.
)
picarro_id
=
HIDS2179
#
Address of the FTP server (without ftp://
)
host
=
XXX
# Absolute path of the local directory containing the Picarro's `.dat` data file. This is the directory containing one
# folder per year. It is very probably named `DataLog_User`.
base_dir
=
C:
\U
serData
\D
ataLog_User
\
# FTP user
user
=
XXX
# Date (in ISO format: yyyy-mm-dd) of the last successful compression of Picarro data. This parameters is automatically
# updated by the software. The user may only modify it at initialization or for debug purposes.
last_compression
=
2020-01-06
# FTP Password
password
=
XXX
# Absolute path of the distant FTP directory where the compressed files are stored.
distant_base_dir
=
/XXX/data/
[HOBO]
# ";"-separated list of Hobo instrument's reference.
hobo_ids
=
10467007;10467009
########################################################################################################################
[FTPOPAR]
# Address of the FTP server (without ftp://)
host
=
XXX
# FTP user
user
=
XXX
# FTP Password
password
=
XXX
# Absolute path of the
local directory containing the Hobo's `.csv` data file
.
base_dir
=
C:
\U
sers
\p
icarro
\D
ocuments
\H
OBOware
\
# Absolute path of the
distant FTP directory where the raw files are stored
.
distant_
base_dir
=
/XXX/data/
# Date (in ISO format: yyyy-mm-dd) of the last successful compression of Hobo data. This parameters is automatically
# updated by the software. The user may only modify it at initialization or for debug purposes.
last_compression
=
2019-12-17
[FTP]
########################################################################################################################
[FTPAERIS]
# Address of the FTP server (without ftp://)
host
=
ftp.cea.fr
host
=
XXX
# FTP user
user
=
XXX
X
user
=
XXX
# FTP Password
password
=
YYYYY
password
=
XXX
# Absolute path of the distant FTP directory where the compressed files should be uploaded. Note that subdirectories
# related to the site and instrument will automatically added.
root_dir
=
/pub/ramces/data/
# Absolute path of the distant FTP directory where the raw files are stored.
distant_base_dir
=
XXX
[LOGGER]
########################################################################################################################
[SFTPDMC]
#
Log fil
es o
lder than `keep_log_day` will be automatically deleted.
keep_log_days
=
5
#
Addr
es
s
o
f the SFTP server (without ssh:// or sftp://)
host
=
XXX
# Absolute path to the directory where the log files are stored.
log_dir
=
C:
\w
imcollect
\l
og
\
# SSH user
user
=
XXX
# Absolute path of the private SSH RSA key. On Linux, this is probably something like /home/myusername/.ssh/id_rsa
private_key_path
=
/home/XXX/.ssh/id_rsa
# Absolute path of the distant SSH directory where the Picarro files are stored.
distant_base_dir
=
/XXX/data/
########################################################################################################################
[HTTPDDU]
# Absolute URL where the data files are accessible
base_url
=
http://files.domain.fr/
########################################################################################################################
[VIZU]
root_dir
=
/XXX/wimcollect/vizu
\ No newline at end of file
wimcollect/ftpopar.py
View file @
8ed70a68
...
...
@@ -25,8 +25,8 @@ class Collector(utils.LogConfig):
Date of the data which should be downloaded.
"""
self
.
logger
.
write
(
self
.
object_id
,
"Download Maido's FTIR meteo data."
)
date_str
=
day
.
strftime
(
"%Y%m%d"
)
# Build source and destination file paths
source_filepath
=
self
.
distant_base_dir
+
"/"
+
date_str
+
"Meteo125HR.xls"
dest_filepath
=
self
.
__get_dest_filepath__
(
day
)
...
...
wimcollect/httpddu.py
View file @
8ed70a68
...
...
@@ -13,16 +13,25 @@ class Collector(utils.LogConfig):
utils
.
LogConfig
.
__init__
(
self
,
"HTTPDDU"
,
config_parser
,
log
)
self
.
base_url
=
self
.
config
[
self
.
object_id
][
"base_url"
]
def
download_picarro
(
self
,
day
:
datetime
.
date
):
def
download_picarro
(
self
,
picarro_id
:
str
,
day
:
datetime
.
date
):
"""Download Picarro data file from IPEV's HTTP server.
Parameters
----------
picarro_id: str
Unique identifier of the Picarro whose data should be retrieved (e.g. HIDS2189)
day: datetime.date
Date of the data which should be downloaded.
"""
self
.
logger
.
write
(
self
.
object_id
,
"Download DDU Picarro data."
)
# Build source file url
picarro_id
=
self
.
config
[
self
.
object_id
][
"picarro_id"
]
source_fileurl
=
self
.
base_url
+
"/DDU_"
+
picarro_id
+
"_"
+
day
.
strftime
(
"%Y%m%d"
)
+
".zip"
# Build destination file path
dest_filepath
=
self
.
__get_dest_filepath__
(
day
,
source_fileurl
)
# Download file
response
=
urllib
.
request
.
urlretrieve
(
source_fileurl
,
dest_filepath
)
success
=
response
[
0
]
==
dest_filepath
and
os
.
path
.
exists
(
dest_filepath
)
...
...
wimcollect/sftpdmc.py
View file @
8ed70a68
...
...
@@ -16,19 +16,19 @@ class Collector(utils.LogConfig):
utils
.
LogConfig
.
__init__
(
self
,
"SFTPDMC"
,
config_parser
,
log
)
self
.
distant_base_dir
=
self
.
config
[
self
.
object_id
][
"distant_base_dir"
]
def
download_picarro
(
self
,
day
:
datetime
.
date
):
def
download_picarro
(
self
,
picarro_id
:
str
,
day
:
datetime
.
date
):
"""Download Picarro data file from SFTP server.
The distant files will be deleted from the server if the transfer is successful.
Parameters
----------
picarro_id: str
Unique identifier of the Picarro whose data should be retrieved (e.g. HIDS2319)
day: datetime.date
Date of the data which should be downloaded.
"""
self
.
logger
.
write
(
self
.
object_id
,
"Download DMC Picarro data."
)
# Build source file path
picarro_id
=
self
.
config
[
self
.
object_id
][
"picarro_id"
]
picarro_number
=
re
.
sub
(
"[^0-9]"
,
""
,
picarro_id
)
source_filepath
=
self
.
distant_base_dir
+
"/"
+
picarro_number
\
+
"/DMC_"
+
picarro_id
+
"_"
+
day
.
strftime
(
"%Y%m%d"
)
+
".zip"
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment