Docker-in-Docker (DinD) capabilities of public runners deactivated. More info

sftpdmc.py 2.85 KB
Newer Older
JOSSOUD Olivier's avatar
SSH DMC  
JOSSOUD Olivier committed
1 2 3 4 5
import os
import re
import configobj
import datetime

6 7
import wimcollect.common.utils as utils
import wimcollect.common.logger as logger
8
import wimcollect.common.sftp as sftp
JOSSOUD Olivier's avatar
SSH DMC  
JOSSOUD Olivier committed
9 10


11
class Collector(utils.LogConfig):
12 13
    """Collector for Dome C / Concordia station (trigram: DMC) data, hosted on Hermes server, managed by italian's PNRA.
    """
JOSSOUD Olivier's avatar
SSH DMC  
JOSSOUD Olivier committed
14

15
    def __init__(self, config_parser: configobj.ConfigObj = None, log: logger = None):
JOSSOUD Olivier's avatar
JOSSOUD Olivier committed
16
        utils.LogConfig.__init__(self, "SFTPDMC", config_parser, log)
17
        self.distant_base_dir = self.config[self.object_id]["distant_base_dir"]
JOSSOUD Olivier's avatar
SSH DMC  
JOSSOUD Olivier committed
18

JOSSOUD Olivier's avatar
JOSSOUD Olivier committed
19
    def download_picarro(self, picarro_id: str, day: datetime.date):
20
        """Download Picarro data file from SFTP server.
JOSSOUD Olivier's avatar
SSH DMC  
JOSSOUD Olivier committed
21 22 23

        Parameters
        ----------
JOSSOUD Olivier's avatar
JOSSOUD Olivier committed
24 25 26 27
        picarro_id: str
            Unique identifier of the Picarro whose data should be retrieved (e.g. HIDS2319)
        day: datetime.date
            Date of the data which should be downloaded.
JOSSOUD Olivier's avatar
SSH DMC  
JOSSOUD Olivier committed
28
        """
29
        day_str = day.strftime("%Y-%m-%d")
JOSSOUD Olivier's avatar
SSH DMC  
JOSSOUD Olivier committed
30 31 32

        # Build source file path
        picarro_number = re.sub("[^0-9]", "", picarro_id)
33
        source_filepath = self.distant_base_dir + "/" + picarro_number \
JOSSOUD Olivier's avatar
SSH DMC  
JOSSOUD Olivier committed
34 35
                          + "/DMC_" + picarro_id + "_" + day.strftime("%Y%m%d") + ".zip"

36
        # Build destination file path
JOSSOUD Olivier's avatar
SSH DMC  
JOSSOUD Olivier committed
37
        dest_filepath = self.__get_dest_filepath__(day, source_filepath)
38 39

        # Download file
40
        self.logger.info("Downloading DMC Picarro file from SFTP server.", day_str)
41 42 43
        sftp_client = sftp.connect(self.config[self.object_id], self.logger, self.object_id)
        success = sftp.download_file(sftp_client, source_filepath, dest_filepath, self.logger, self.object_id)
        sftp_client.close()
44 45 46 47
        if not success:
            self.logger.error("FAILED to download file in " + dest_filepath, day_str)
        else:
            self.logger.info("File downloaded in " + dest_filepath, day_str)
48 49

        # Re-compress, from ZIP to LZMA
JOSSOUD Olivier's avatar
SSH DMC  
JOSSOUD Olivier committed
50
        if success:
51
            self.logger.info("Re-compressing from ZIP to LZMA...", day_str)
52 53
            lzma_filepath = utils.recompress_file(dest_filepath)
            if lzma_filepath is None:
54
                self.logger.error("FAILED to create archive file", day_str)
55
            else:
56
                self.logger.info("Done. Archive file: " + lzma_filepath, day_str)
JOSSOUD Olivier's avatar
SSH DMC  
JOSSOUD Olivier committed
57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72

    def __get_dest_filepath__(self, day: datetime.date, source_filepath: str) -> str:
        # Build destination directory
        dest_dir = os.path.join(self.config["LOCAL"]["base_dir"],
                                "DMC", "picarro", self.config[self.object_id]["picarro_id"], str(day.year))
        if not os.path.exists(dest_dir):
            os.makedirs(dest_dir)

        # Build destination filename
        dest_filename = os.path.basename(source_filepath)

        # Build full file path
        dest_filepath = os.path.join(dest_dir, dest_filename)

        return dest_filepath

73