Commit ebf3341d authored by JOSSOUD Olivier's avatar JOSSOUD Olivier
Browse files

Sharebox downloader for DVS

parent 3276986d
Pipeline #177598 failed with stage
in 52 seconds
......@@ -83,6 +83,14 @@ distant_base_dir = /XXX/data/
# Absolute URL where the data files are accessible
base_url = http://files.domain.fr/
########################################################################################################################
[SHAREBOXDVS]
# Absolute URL where the data files are accessible
base_url = https://sharebox.lsce.ipsl.fr/public.php/webdav/
# Public share token
token = 123456abcdef
########################################################################################################################
[VIZU]
......
configobj==5.0.*
paramiko==2.7.*
pandas==0.25.*
\ No newline at end of file
pandas==0.25.*
requests==2.27.*
\ No newline at end of file
import os
import configobj
import datetime
import requests
import wimcollect.common.utils as utils
import wimcollect.common.logger as logger
class PicarroCollector(utils.LogConfig):
def __init__(self, config_parser: configobj.ConfigObj = None, log: logger = None):
utils.LogConfig.__init__(self, "SHAREBOXDVS", config_parser, log)
self.base_url = self.config[self.object_id]["base_url"]
def download(self, picarro_id: str, day: datetime.date) -> bool:
"""Download Picarro data file from IPEV's HTTP server.
Parameters
----------
picarro_id: str
Unique identifier of the Picarro whose data should be retrieved (e.g. HIDS2189)
day: datetime.date
Date of the data which should be downloaded.
"""
site_id = "DVS"
date_str = day.strftime("%Y%m%d")
self.logger.date_str = date_str
# Build source file url
source_fileurl = self.base_url + "/" + site_id + "_" + picarro_id + "_" + date_str + ".lzma"
# Build destination file path
dest_filepath = utils.get_standard_filepath(self.local_base_dir, site_id, "picarro", picarro_id, date_str, "lzma")
# Download file
self.logger.info("Downloading Picarro file from " + source_fileurl)
headers = {'X-Requested-With': 'XMLHttpRequest', }
response = requests.get(source_fileurl, headers=headers, auth=(self.config[self.object_id]["token"], ''))
with open(dest_filepath, 'wb') as f:
f.write(response.content)
success = os.path.exists(dest_filepath)
if not success:
self.logger.error("FAILED to download file in " + dest_filepath)
else:
self.logger.info("File downloaded in " + dest_filepath)
return True
import datetime
from unittest import TestCase
import wimcollect.shareboxdvs as shareboxdvs
class TestPicarroCollector(TestCase):
def test_download(self):
collector = shareboxdvs.PicarroCollector()
self.assertTrue(collector.download("HIDS2485", datetime.date(2022, 5, 3)))
\ No newline at end of file
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment