Commit 3e82b125 authored by JOSSOUD Olivier's avatar JOSSOUD Olivier
Browse files

SSH DMC

parent 04c5d269
Pipeline #56898 passed with stages
in 3 minutes and 17 seconds
configobj==5.0.*
\ No newline at end of file
configobj==5.0.*
paramiko==2.7.*
\ No newline at end of file
import os
import re
import configobj
import datetime
import zipfile
import paramiko
import wimcollect.logger as logger
class Collector:
def __init__(self, config_parser: configobj.ConfigObj, log: logger):
# Logger
self.logger = log
self.object_id = "SSHDMC"
# Config
self.config = config_parser
self.base_dir = self.config[self.object_id]["root_dir"]
def download_picarro(self, day: datetime.date):
"""Download Picarro data file from SSH server.
The distant files will be deleted from the server if the transfer is successful.
Parameters
----------
"""
self.logger.write(self.object_id, "Download DMC Picarro data.")
sftp_client = self.__sftp_connect__()
# Build source file path
picarro_id = self.config[self.object_id]["picarro_id"]
picarro_number = re.sub("[^0-9]", "", picarro_id)
source_filepath = self.base_dir + "/" + picarro_number \
+ "/DMC_" + picarro_id + "_" + day.strftime("%Y%m%d") + ".zip"
dest_filepath = self.__get_dest_filepath__(day, source_filepath)
success = self.__sftp_download__(sftp_client, source_filepath, dest_filepath)
if success:
self.__recompress_file__(day, dest_filepath)
sftp_client.close()
def __get_dest_filepath__(self, day: datetime.date, source_filepath: str) -> str:
# Build destination directory
dest_dir = os.path.join(self.config["LOCAL"]["base_dir"],
"DMC", "picarro", self.config[self.object_id]["picarro_id"], str(day.year))
if not os.path.exists(dest_dir):
os.makedirs(dest_dir)
# Build destination filename
dest_filename = os.path.basename(source_filepath)
# Build full file path
dest_filepath = os.path.join(dest_dir, dest_filename)
return dest_filepath
def __recompress_file__(self, day: datetime.date, zip_filepath: str):
self.logger.write(self.object_id, day.strftime("%Y-%m-%d") + ": Re-compressing from ZIP to LZMA...")
# Directory where the original ZIP is, where the ZIP content will temporarily be extracted and where the final
# LZMA file will be created.
current_directory = os.path.dirname(zip_filepath)
# Extract zip file and delete it
source_zip = zipfile.ZipFile(zip_filepath, 'r')
files_in_zip = [os.path.join(current_directory, filename) for filename in source_zip.namelist()]
source_zip.extractall(current_directory)
source_zip.close()
os.remove(zip_filepath)
# Compress the files
compressed_filepath = os.path.splitext(zip_filepath)[0] + ".lzma"
zipf = zipfile.ZipFile(compressed_filepath, 'w', zipfile.ZIP_LZMA)
for filepath in files_in_zip:
zipf.write(filepath, arcname=os.path.basename(filepath))
zipf.close()
if os.path.exists(compressed_filepath):
self.logger.write(self.object_id, day.strftime("%Y-%m-%d") + ": Done. Archive file: " + compressed_filepath)
for file in files_in_zip:
os.remove(file)
def __sftp_connect__(self) -> paramiko.SFTPClient:
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
try:
client.connect(hostname=self.config[self.object_id]["host"],
username=self.config[self.object_id]["user"],
key_filename=self.config[self.object_id]["private_key_path"])
except Exception as e:
self.logger.write(self.object_id, "Failed to connect to server:\n" + str(e))
raise e
else:
return client.open_sftp()
def __sftp_download__(self, sftp_client: paramiko.SFTPClient, source_filepath: str, dest_filepath: str) -> bool:
sftp_client.get(source_filepath, dest_filepath)
success = os.path.exists(dest_filepath)
return success
\ No newline at end of file
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment