HTTP DDU

wimcollect/httpddu.py

+import os
+import configobj
+import datetime
+import urllib.request
+
+import wimcollect.utils as utils
+import wimcollect.logger as logger
+
+
+class Collector:
+
+    def __init__(self, config_parser: configobj.ConfigObj, log: logger):
+        # Logger
+        self.logger = log
+        self.object_id = "HTTPDDU"
+
+        # Config
+        self.config = config_parser
+        self.base_url = self.config[self.object_id]["base_url"]
+
+    def download_picarro(self, day: datetime.date):
+        """Download Picarro data file from IPEV's HTTP server.
+        """
+        self.logger.write(self.object_id, "Download DDU Picarro data.")
+
+        # Build source file url
+        picarro_id = self.config[self.object_id]["picarro_id"]
+        source_fileurl = self.base_url + "/DDU_" + picarro_id + "_" + day.strftime("%Y%m%d") + ".zip"
+
+        dest_filepath = self.__get_dest_filepath__(day, source_fileurl)
+        response = urllib.request.urlretrieve(source_fileurl, dest_filepath)
+
+        success = response[0] == dest_filepath and os.path.exists(dest_filepath)
+        if not success:
+            self.logger.write(self.object_id, day.strftime("%Y-%m-%d") + ": FAILED to download file.")
+            return False
+
+        # Recompress file, from ZIP to LZMA
+        self.logger.write(self.object_id, day.strftime("%Y-%m-%d") + ": Re-compressing from ZIP to LZMA...")
+        lzma_filepath = utils.recompress_file(dest_filepath)
+        if lzma_filepath is None:
+            self.logger.write(self.object_id, day.strftime("%Y-%m-%d") + ": FAILED to create archive file")
+            return False
+        else:
+            self.logger.write(self.object_id, day.strftime("%Y-%m-%d") + ": Done. Archive file: " + lzma_filepath)
+            return True
+
+    def __get_dest_filepath__(self, day: datetime.date, source_filepath: str) -> str:
+        # Build destination directory
+        dest_dir = os.path.join(self.config["LOCAL"]["base_dir"],
+                                "DDU", "picarro", self.config[self.object_id]["picarro_id"], str(day.year))
+        if not os.path.exists(dest_dir):
+            os.makedirs(dest_dir)
+
+        # Build destination filename
+        dest_filename = os.path.basename(source_filepath)
+
+        # Build full file path
+        dest_filepath = os.path.join(dest_dir, dest_filename)
+
+        return dest_filepath