ftpcea.py 4.81 KB
Newer Older
JOSSOUD Olivier's avatar
JOSSOUD Olivier committed
1 2 3 4 5
import os
import re
import configobj
import datetime

6 7 8
import wimcollect.common.logger as logger
import wimcollect.common.ftp as ftp
import wimcollect.common.utils as utils
JOSSOUD Olivier's avatar
JOSSOUD Olivier committed
9 10


11
class Collector(utils.LogConfig):
JOSSOUD Olivier's avatar
JOSSOUD Olivier committed
12

13
    def __init__(self, config_parser: configobj.ConfigObj = None, log: logger = None):
JOSSOUD Olivier's avatar
JOSSOUD Olivier committed
14
        self.object_id = "FTPCEA"
15 16
        utils.LogConfig.__init__(self, self.object_id, config_parser, log)
        self.distant_base_dir = self.config[self.object_id]["distant_base_dir"]
JOSSOUD Olivier's avatar
JOSSOUD Olivier committed
17 18 19 20

    ####################################################################################################################
    # Picarro

21 22
    def download_picarro(self, site_id: str, picarro_id: str, day: datetime.date):
        """Download Picarro file from FTP server.
JOSSOUD Olivier's avatar
JOSSOUD Olivier committed
23 24 25 26

        Parameters
        ----------
        site_id: str
27 28 29 30 31
            Site's trigram (AMS, SRT, etc.)
        picarro_id: str
            Picarro unique identifier (HIDS2189, HIDS2108, etc.)
        day: datetime.date
            Date of the data which should be downloaded.
JOSSOUD Olivier's avatar
JOSSOUD Olivier committed
32 33 34
        """
        self.logger.write(self.object_id, "Download picarro data from " + site_id)

35
        ftp_session = ftp.ftp_connect(self.config[self.object_id], self.logger, self.object_id)
JOSSOUD Olivier's avatar
JOSSOUD Olivier committed
36

37
        # Build source file path
38
        source_dir = self.distant_base_dir + "/" + site_id + "/picarro/"
39 40 41 42
        source_filename = site_id + "_" + picarro_id + day.strftime("%y%m%d") + ".lzma"
        source_filepath = source_dir + source_filename

        # Check that to-be-downloaded file exists
43
        source_filepaths = ftp.ftp_list_files(ftp_session, source_dir, self.logger, self.object_id)
44 45 46 47 48 49
        if source_filepath not in source_filepaths:
            self.logger.write(self.object_id, "File not found: " + source_filepath)
            raise FileNotFoundError(source_filepath)

        # Download file
        dest_filepath = self.__get_picarro_dest_filepath__(source_filepath)
50 51
        success = ftp.ftp_download_file(source_filepath, dest_filepath, self.logger, self.object_id,
                                        ftp_session=ftp_session)
52 53

        ftp_session.quit()
JOSSOUD Olivier's avatar
JOSSOUD Olivier committed
54

55
        return success
JOSSOUD Olivier's avatar
JOSSOUD Olivier committed
56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95

    def __get_picarro_dest_filepath__(self, source_filepath: str) -> str:
        """Get the full path where the Picarro data file should be downloaded.

        Parameters
        ----------
        source_filepath: str
            Full path of the distant (FTP) to-be-downloaded Picarro data file.

        Returns
        -------
        str
            Local destination file path.
        """
        filename = os.path.basename(source_filepath)
        site_id, picarro_id, date_str = re.split('_|\.', filename)[0:3]
        date = datetime.datetime.strptime(date_str, "%Y%m%d").date()

        dest_dir = os.path.join(self.config["LOCAL"]["base_dir"], site_id, "picarro", picarro_id, str(date.year))
        if not os.path.exists(dest_dir):
            os.makedirs(dest_dir)
        dest_filepath = os.path.join(dest_dir, filename)

        return dest_filepath

    ####################################################################################################################
    # Hobo

    def download_hobo(self, site_id: str):
        """Download all Hobo files from FTP server.

        The distant files will be deleted from the server if the transfer is successful.

        Parameters
        ----------
        site_id: str
            Site's trigram
        """
        self.logger.write(self.object_id, "Download hobo data from " + site_id)

96 97
        ftp_session = ftp.ftp_connect(self.config[self.object_id], self.logger, self.object_id)
        hobo_distant_path = self.distant_base_dir + "/" + site_id + "/hobo/"
JOSSOUD Olivier's avatar
JOSSOUD Olivier committed
98

99
        source_filepaths = ftp.ftp_list_files(ftp_session, hobo_distant_path, self.logger, self.object_id)
JOSSOUD Olivier's avatar
JOSSOUD Olivier committed
100 101
        for source_filepath in source_filepaths:
            dest_filepath = self.__get_hobo_dest_filepath__(source_filepath)
102 103
            ftp.ftp_download_file(source_filepath, dest_filepath, self.logger, self.object_id,
                                  ftp_session=ftp_session)
JOSSOUD Olivier's avatar
JOSSOUD Olivier committed
104

105
        ftp_session.quit()
JOSSOUD Olivier's avatar
JOSSOUD Olivier committed
106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134

    def __get_hobo_dest_filepath__(self, source_filepath: str):
        """Get the full path where the Hobo data file should be downloaded.

        Parameters
        ----------
        source_filepath: str
            Full path of the distant (FTP) to-be-downloaded Hobo data file.

        Returns
        -------
        str
            Local destination file path.
        """
        filename = os.path.basename(source_filepath)
        filename_parts = re.split('_|\.|-', filename)
        site_id = filename_parts[0]
        hobo_id = filename_parts[2]

        dest_dir = os.path.join(self.config["LOCAL"]["base_dir"], site_id, "meteo", "hobo_" + hobo_id)
        if not os.path.exists(dest_dir):
            os.makedirs(dest_dir)
        dest_filepath = os.path.join(dest_dir, filename)

        return dest_filepath