Commit 8ed70a68 authored by JOSSOUD Olivier's avatar JOSSOUD Olivier
Browse files

Code layout and comments.

parent ba207b0f
Pipeline #57299 passed with stages
in 1 minute and 13 seconds
[SITE] [LOCAL]
# Absolute path of the local directory where the collected data file will be stored.
base_dir = /XXX/DATA/raw/
# Site's trigram (e.g. REU, AMS, SRT, etc.)
site_id = REU
########################################################################################################################
[LOGGER]
[GLOBAL] # Log files older than `keep_log_day` will be automatically deleted.
keep_log_days = 5
# Absolute path of the local directory where the to-be-sent compressed files will temporarily be stored. # Absolute path to the directory where the log files are stored.
to_send_dir = C:\wimcollect\toSend\ log_dir = /XXX/wimcollect/log/
[PICARRO] ########################################################################################################################
[FTPCEA]
# Picarro's unique identifier (e.g. HIDS2179, HIDS2108, etc.) # Address of the FTP server (without ftp://)
picarro_id = HIDS2179 host = XXX
# Absolute path of the local directory containing the Picarro's `.dat` data file. This is the directory containing one # FTP user
# folder per year. It is very probably named `DataLog_User`. user = XXX
base_dir = C:\UserData\DataLog_User\
# Date (in ISO format: yyyy-mm-dd) of the last successful compression of Picarro data. This parameters is automatically # FTP Password
# updated by the software. The user may only modify it at initialization or for debug purposes. password = XXX
last_compression = 2020-01-06
# Absolute path of the distant FTP directory where the compressed files are stored.
distant_base_dir = /XXX/data/
[HOBO]
# ";"-separated list of Hobo instrument's reference. ########################################################################################################################
hobo_ids = 10467007;10467009 [FTPOPAR]
# Address of the FTP server (without ftp://)
host = XXX
# FTP user
user = XXX
# FTP Password
password = XXX
# Absolute path of the local directory containing the Hobo's `.csv` data file. # Absolute path of the distant FTP directory where the raw files are stored.
base_dir = C:\Users\picarro\Documents\HOBOware\ distant_base_dir = /XXX/data/
# Date (in ISO format: yyyy-mm-dd) of the last successful compression of Hobo data. This parameters is automatically
# updated by the software. The user may only modify it at initialization or for debug purposes.
last_compression = 2019-12-17
[FTP] ########################################################################################################################
[FTPAERIS]
# Address of the FTP server (without ftp://) # Address of the FTP server (without ftp://)
host = ftp.cea.fr host = XXX
# FTP user # FTP user
user = XXXX user = XXX
# FTP Password # FTP Password
password = YYYYY password = XXX
# Absolute path of the distant FTP directory where the compressed files should be uploaded. Note that subdirectories # Absolute path of the distant FTP directory where the raw files are stored.
# related to the site and instrument will automatically added. distant_base_dir = XXX
root_dir = /pub/ramces/data/
[LOGGER] ########################################################################################################################
[SFTPDMC]
# Log files older than `keep_log_day` will be automatically deleted. # Address of the SFTP server (without ssh:// or sftp://)
keep_log_days = 5 host = XXX
# Absolute path to the directory where the log files are stored. # SSH user
log_dir = C:\wimcollect\log\ user = XXX
# Absolute path of the private SSH RSA key. On Linux, this is probably something like /home/myusername/.ssh/id_rsa
private_key_path = /home/XXX/.ssh/id_rsa
# Absolute path of the distant SSH directory where the Picarro files are stored.
distant_base_dir = /XXX/data/
########################################################################################################################
[HTTPDDU]
# Absolute URL where the data files are accessible
base_url = http://files.domain.fr/
########################################################################################################################
[VIZU]
root_dir = /XXX/wimcollect/vizu
\ No newline at end of file
...@@ -25,8 +25,8 @@ class Collector(utils.LogConfig): ...@@ -25,8 +25,8 @@ class Collector(utils.LogConfig):
Date of the data which should be downloaded. Date of the data which should be downloaded.
""" """
self.logger.write(self.object_id, "Download Maido's FTIR meteo data.") self.logger.write(self.object_id, "Download Maido's FTIR meteo data.")
date_str = day.strftime("%Y%m%d") date_str = day.strftime("%Y%m%d")
# Build source and destination file paths # Build source and destination file paths
source_filepath = self.distant_base_dir + "/" + date_str + "Meteo125HR.xls" source_filepath = self.distant_base_dir + "/" + date_str + "Meteo125HR.xls"
dest_filepath = self.__get_dest_filepath__(day) dest_filepath = self.__get_dest_filepath__(day)
......
...@@ -13,16 +13,25 @@ class Collector(utils.LogConfig): ...@@ -13,16 +13,25 @@ class Collector(utils.LogConfig):
utils.LogConfig.__init__(self, "HTTPDDU", config_parser, log) utils.LogConfig.__init__(self, "HTTPDDU", config_parser, log)
self.base_url = self.config[self.object_id]["base_url"] self.base_url = self.config[self.object_id]["base_url"]
def download_picarro(self, day: datetime.date): def download_picarro(self, picarro_id: str, day: datetime.date):
"""Download Picarro data file from IPEV's HTTP server. """Download Picarro data file from IPEV's HTTP server.
Parameters
----------
picarro_id: str
Unique identifier of the Picarro whose data should be retrieved (e.g. HIDS2189)
day: datetime.date
Date of the data which should be downloaded.
""" """
self.logger.write(self.object_id, "Download DDU Picarro data.") self.logger.write(self.object_id, "Download DDU Picarro data.")
# Build source file url # Build source file url
picarro_id = self.config[self.object_id]["picarro_id"]
source_fileurl = self.base_url + "/DDU_" + picarro_id + "_" + day.strftime("%Y%m%d") + ".zip" source_fileurl = self.base_url + "/DDU_" + picarro_id + "_" + day.strftime("%Y%m%d") + ".zip"
# Build destination file path
dest_filepath = self.__get_dest_filepath__(day, source_fileurl) dest_filepath = self.__get_dest_filepath__(day, source_fileurl)
# Download file
response = urllib.request.urlretrieve(source_fileurl, dest_filepath) response = urllib.request.urlretrieve(source_fileurl, dest_filepath)
success = response[0] == dest_filepath and os.path.exists(dest_filepath) success = response[0] == dest_filepath and os.path.exists(dest_filepath)
......
...@@ -16,19 +16,19 @@ class Collector(utils.LogConfig): ...@@ -16,19 +16,19 @@ class Collector(utils.LogConfig):
utils.LogConfig.__init__(self, "SFTPDMC", config_parser, log) utils.LogConfig.__init__(self, "SFTPDMC", config_parser, log)
self.distant_base_dir = self.config[self.object_id]["distant_base_dir"] self.distant_base_dir = self.config[self.object_id]["distant_base_dir"]
def download_picarro(self, day: datetime.date): def download_picarro(self, picarro_id: str, day: datetime.date):
"""Download Picarro data file from SFTP server. """Download Picarro data file from SFTP server.
The distant files will be deleted from the server if the transfer is successful.
Parameters Parameters
---------- ----------
picarro_id: str
Unique identifier of the Picarro whose data should be retrieved (e.g. HIDS2319)
day: datetime.date
Date of the data which should be downloaded.
""" """
self.logger.write(self.object_id, "Download DMC Picarro data.") self.logger.write(self.object_id, "Download DMC Picarro data.")
# Build source file path # Build source file path
picarro_id = self.config[self.object_id]["picarro_id"]
picarro_number = re.sub("[^0-9]", "", picarro_id) picarro_number = re.sub("[^0-9]", "", picarro_id)
source_filepath = self.distant_base_dir + "/" + picarro_number \ source_filepath = self.distant_base_dir + "/" + picarro_number \
+ "/DMC_" + picarro_id + "_" + day.strftime("%Y%m%d") + ".zip" + "/DMC_" + picarro_id + "_" + day.strftime("%Y%m%d") + ".zip"
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment