SSH DMC

wimcollect/requirements_pkg.txt

-configobj==5.0.*
\ No newline at end of file
+configobj==5.0.*
+paramiko==2.7.*
\ No newline at end of file

wimcollect/sshdmc.py

+import os
+import re
+import configobj
+import datetime
+import zipfile
+import paramiko
+
+import wimcollect.logger as logger
+
+
+class Collector:
+
+    def __init__(self, config_parser: configobj.ConfigObj, log: logger):
+        # Logger
+        self.logger = log
+        self.object_id = "SSHDMC"
+
+        # Config
+        self.config = config_parser
+        self.base_dir = self.config[self.object_id]["root_dir"]
+
+    def download_picarro(self, day: datetime.date):
+        """Download Picarro data file from SSH server.
+
+        The distant files will be deleted from the server if the transfer is successful.
+
+        Parameters
+        ----------
+
+        """
+        self.logger.write(self.object_id, "Download DMC Picarro data.")
+
+        sftp_client = self.__sftp_connect__()
+
+        # Build source file path
+        picarro_id = self.config[self.object_id]["picarro_id"]
+        picarro_number = re.sub("[^0-9]", "", picarro_id)
+        source_filepath = self.base_dir + "/" + picarro_number \
+                          + "/DMC_" + picarro_id + "_" + day.strftime("%Y%m%d") + ".zip"
+
+        dest_filepath = self.__get_dest_filepath__(day, source_filepath)
+        success = self.__sftp_download__(sftp_client, source_filepath, dest_filepath)
+        if success:
+            self.__recompress_file__(day, dest_filepath)
+
+        sftp_client.close()
+
+    def __get_dest_filepath__(self, day: datetime.date, source_filepath: str) -> str:
+        # Build destination directory
+        dest_dir = os.path.join(self.config["LOCAL"]["base_dir"],
+                                "DMC", "picarro", self.config[self.object_id]["picarro_id"], str(day.year))
+        if not os.path.exists(dest_dir):
+            os.makedirs(dest_dir)
+
+        # Build destination filename
+        dest_filename = os.path.basename(source_filepath)
+
+        # Build full file path
+        dest_filepath = os.path.join(dest_dir, dest_filename)
+
+        return dest_filepath
+
+    def __recompress_file__(self, day: datetime.date, zip_filepath: str):
+        self.logger.write(self.object_id, day.strftime("%Y-%m-%d") + ": Re-compressing from ZIP to LZMA...")
+
+        # Directory where the original ZIP is, where the ZIP content will temporarily be extracted and where the final
+        # LZMA file will be created.
+        current_directory = os.path.dirname(zip_filepath)
+
+        # Extract zip file and delete it
+        source_zip = zipfile.ZipFile(zip_filepath, 'r')
+        files_in_zip = [os.path.join(current_directory, filename) for filename in source_zip.namelist()]
+        source_zip.extractall(current_directory)
+        source_zip.close()
+        os.remove(zip_filepath)
+
+        # Compress the files
+        compressed_filepath = os.path.splitext(zip_filepath)[0] + ".lzma"
+        zipf = zipfile.ZipFile(compressed_filepath, 'w', zipfile.ZIP_LZMA)
+        for filepath in files_in_zip:
+            zipf.write(filepath, arcname=os.path.basename(filepath))
+        zipf.close()
+
+        if os.path.exists(compressed_filepath):
+            self.logger.write(self.object_id, day.strftime("%Y-%m-%d") + ": Done. Archive file: " + compressed_filepath)
+            for file in files_in_zip:
+                os.remove(file)
+
+    def __sftp_connect__(self) -> paramiko.SFTPClient:
+        client = paramiko.SSHClient()
+        client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+        try:
+            client.connect(hostname=self.config[self.object_id]["host"],
+                           username=self.config[self.object_id]["user"],
+                           key_filename=self.config[self.object_id]["private_key_path"])
+        except Exception as e:
+            self.logger.write(self.object_id, "Failed to connect to server:\n" + str(e))
+            raise e
+        else:
+            return client.open_sftp()
+
+    def __sftp_download__(self, sftp_client: paramiko.SFTPClient, source_filepath: str, dest_filepath: str) -> bool:
+        sftp_client.get(source_filepath, dest_filepath)
+        success = os.path.exists(dest_filepath)
+        return success
\ No newline at end of file