Commit 8ed70a68 authored by JOSSOUD Olivier's avatar JOSSOUD Olivier
Browse files

Code layout and comments.

parent ba207b0f
Pipeline #57299 passed with stages
in 1 minute and 13 seconds
[SITE]
[LOCAL]
# Absolute path of the local directory where the collected data file will be stored.
base_dir = /XXX/DATA/raw/
# Site's trigram (e.g. REU, AMS, SRT, etc.)
site_id = REU
########################################################################################################################
[LOGGER]
[GLOBAL]
# Log files older than `keep_log_day` will be automatically deleted.
keep_log_days = 5
# Absolute path of the local directory where the to-be-sent compressed files will temporarily be stored.
to_send_dir = C:\wimcollect\toSend\
# Absolute path to the directory where the log files are stored.
log_dir = /XXX/wimcollect/log/
[PICARRO]
########################################################################################################################
[FTPCEA]
# Picarro's unique identifier (e.g. HIDS2179, HIDS2108, etc.)
picarro_id = HIDS2179
# Address of the FTP server (without ftp://)
host = XXX
# Absolute path of the local directory containing the Picarro's `.dat` data file. This is the directory containing one
# folder per year. It is very probably named `DataLog_User`.
base_dir = C:\UserData\DataLog_User\
# FTP user
user = XXX
# Date (in ISO format: yyyy-mm-dd) of the last successful compression of Picarro data. This parameters is automatically
# updated by the software. The user may only modify it at initialization or for debug purposes.
last_compression = 2020-01-06
# FTP Password
password = XXX
# Absolute path of the distant FTP directory where the compressed files are stored.
distant_base_dir = /XXX/data/
[HOBO]
# ";"-separated list of Hobo instrument's reference.
hobo_ids = 10467007;10467009
########################################################################################################################
[FTPOPAR]
# Address of the FTP server (without ftp://)
host = XXX
# FTP user
user = XXX
# FTP Password
password = XXX
# Absolute path of the local directory containing the Hobo's `.csv` data file.
base_dir = C:\Users\picarro\Documents\HOBOware\
# Absolute path of the distant FTP directory where the raw files are stored.
distant_base_dir = /XXX/data/
# Date (in ISO format: yyyy-mm-dd) of the last successful compression of Hobo data. This parameters is automatically
# updated by the software. The user may only modify it at initialization or for debug purposes.
last_compression = 2019-12-17
[FTP]
########################################################################################################################
[FTPAERIS]
# Address of the FTP server (without ftp://)
host = ftp.cea.fr
host = XXX
# FTP user
user = XXXX
user = XXX
# FTP Password
password = YYYYY
password = XXX
# Absolute path of the distant FTP directory where the compressed files should be uploaded. Note that subdirectories
# related to the site and instrument will automatically added.
root_dir = /pub/ramces/data/
# Absolute path of the distant FTP directory where the raw files are stored.
distant_base_dir = XXX
[LOGGER]
########################################################################################################################
[SFTPDMC]
# Log files older than `keep_log_day` will be automatically deleted.
keep_log_days = 5
# Address of the SFTP server (without ssh:// or sftp://)
host = XXX
# Absolute path to the directory where the log files are stored.
log_dir = C:\wimcollect\log\
# SSH user
user = XXX
# Absolute path of the private SSH RSA key. On Linux, this is probably something like /home/myusername/.ssh/id_rsa
private_key_path = /home/XXX/.ssh/id_rsa
# Absolute path of the distant SSH directory where the Picarro files are stored.
distant_base_dir = /XXX/data/
########################################################################################################################
[HTTPDDU]
# Absolute URL where the data files are accessible
base_url = http://files.domain.fr/
########################################################################################################################
[VIZU]
root_dir = /XXX/wimcollect/vizu
\ No newline at end of file
......@@ -25,8 +25,8 @@ class Collector(utils.LogConfig):
Date of the data which should be downloaded.
"""
self.logger.write(self.object_id, "Download Maido's FTIR meteo data.")
date_str = day.strftime("%Y%m%d")
# Build source and destination file paths
source_filepath = self.distant_base_dir + "/" + date_str + "Meteo125HR.xls"
dest_filepath = self.__get_dest_filepath__(day)
......
......@@ -13,16 +13,25 @@ class Collector(utils.LogConfig):
utils.LogConfig.__init__(self, "HTTPDDU", config_parser, log)
self.base_url = self.config[self.object_id]["base_url"]
def download_picarro(self, day: datetime.date):
def download_picarro(self, picarro_id: str, day: datetime.date):
"""Download Picarro data file from IPEV's HTTP server.
Parameters
----------
picarro_id: str
Unique identifier of the Picarro whose data should be retrieved (e.g. HIDS2189)
day: datetime.date
Date of the data which should be downloaded.
"""
self.logger.write(self.object_id, "Download DDU Picarro data.")
# Build source file url
picarro_id = self.config[self.object_id]["picarro_id"]
source_fileurl = self.base_url + "/DDU_" + picarro_id + "_" + day.strftime("%Y%m%d") + ".zip"
# Build destination file path
dest_filepath = self.__get_dest_filepath__(day, source_fileurl)
# Download file
response = urllib.request.urlretrieve(source_fileurl, dest_filepath)
success = response[0] == dest_filepath and os.path.exists(dest_filepath)
......
......@@ -16,19 +16,19 @@ class Collector(utils.LogConfig):
utils.LogConfig.__init__(self, "SFTPDMC", config_parser, log)
self.distant_base_dir = self.config[self.object_id]["distant_base_dir"]
def download_picarro(self, day: datetime.date):
def download_picarro(self, picarro_id: str, day: datetime.date):
"""Download Picarro data file from SFTP server.
The distant files will be deleted from the server if the transfer is successful.
Parameters
----------
picarro_id: str
Unique identifier of the Picarro whose data should be retrieved (e.g. HIDS2319)
day: datetime.date
Date of the data which should be downloaded.
"""
self.logger.write(self.object_id, "Download DMC Picarro data.")
# Build source file path
picarro_id = self.config[self.object_id]["picarro_id"]
picarro_number = re.sub("[^0-9]", "", picarro_id)
source_filepath = self.distant_base_dir + "/" + picarro_number \
+ "/DMC_" + picarro_id + "_" + day.strftime("%Y%m%d") + ".zip"
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment