Commit f91ea3fa authored by JOSSOUD Olivier's avatar JOSSOUD Olivier
Browse files

HTTPDDU. Test and debug messages improvements.

parent 8ed70a68
......@@ -23,36 +23,39 @@ class Collector(utils.LogConfig):
day: datetime.date
Date of the data which should be downloaded.
"""
self.logger.write(self.object_id, "Download DDU Picarro data.")
day_str = day.strftime("%Y-%m-%d")
# Build source file url
source_fileurl = self.base_url + "/DDU_" + picarro_id + "_" + day.strftime("%Y%m%d") + ".zip"
# Build destination file path
dest_filepath = self.__get_dest_filepath__(day, source_fileurl)
dest_filepath = self.__get_dest_filepath__(picarro_id, day, source_fileurl)
# Download file
self.logger.write(self.object_id, "[" + day_str + "] Downloading DDU Picarro file from " + source_fileurl)
response = urllib.request.urlretrieve(source_fileurl, dest_filepath)
success = response[0] == dest_filepath and os.path.exists(dest_filepath)
if not success:
self.logger.write(self.object_id, day.strftime("%Y-%m-%d") + ": FAILED to download file.")
return False
msg = "[" + day_str + "] FAILED to download file."
self.logger.write(self.object_id, msg)
raise Exception(msg)
else:
self.logger.write(self.object_id, "[" + day_str + "] File downloaded in " + dest_filepath)
# Recompress file, from ZIP to LZMA
self.logger.write(self.object_id, day.strftime("%Y-%m-%d") + ": Re-compressing from ZIP to LZMA...")
self.logger.write(self.object_id, "[" + day_str + "] Re-compressing from ZIP to LZMA...")
lzma_filepath = utils.recompress_file(dest_filepath)
if lzma_filepath is None:
self.logger.write(self.object_id, day.strftime("%Y-%m-%d") + ": FAILED to create archive file")
msg = "[" + day_str + "] FAILED to create archive file"
self.logger.write(self.object_id, msg)
return False
else:
self.logger.write(self.object_id, day.strftime("%Y-%m-%d") + ": Done. Archive file: " + lzma_filepath)
self.logger.write(self.object_id, "[" + day_str + "] Done. Archive file: " + lzma_filepath)
return True
def __get_dest_filepath__(self, day: datetime.date, source_filepath: str) -> str:
def __get_dest_filepath__(self, picarro_id: str, day: datetime.date, source_filepath: str) -> str:
# Build destination directory
dest_dir = os.path.join(self.config["LOCAL"]["base_dir"],
"DDU", "picarro", self.config[self.object_id]["picarro_id"], str(day.year))
dest_dir = os.path.join(self.config["LOCAL"]["base_dir"], "DDU", "picarro", picarro_id, str(day.year))
if not os.path.exists(dest_dir):
os.makedirs(dest_dir)
......
from unittest import TestCase
import datetime
import wimcollect.httpddu as httpddu
class TestCollector(TestCase):
def test_download_picarro(self):
collector = httpddu.Collector()
self.assertTrue(collector.download_picarro("HIDS2189", datetime.date(2020, 1, 20)))
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment