Commit ac74978c authored by Clément Haëck's avatar Clément Haëck
Browse files

Up data module files

parent d253f80b
"""Time series of heat fluxes in grided boxes."""
from os import path
import lib
import lib.data
ARGS_DIR = {'region', 'days'}
pregex = '%(idx:fmt=04d)/HF_TS_%(Y).nc'
ARGS = {'region', 'days'}
def PREGEX(args):
return "%(idx:fmt=04d)/HF_TS_%(Y).nc"
def get_root(args=None, **kwargs):
args = lib.data.process_args(ARGS_DIR, args, **kwargs)
root = path.join(lib.root_data, args['region'], 'HF_TS',
lib.data.get_time_folder(args))
return root
def ROOT(args):
return lib.data.get_default_directory(args, 'HF_TS')
def add_idx_dim(ds, filename, finder):
......@@ -22,5 +20,5 @@ def add_idx_dim(ds, filename, finder):
return ds
lib.data.create_data(__name__, pregex, get_root, ARGS_DIR,
lib.data.create_data(__name__, PREGEX, ROOT, ARGS,
open_mf_kw=dict(preprocess_finder=add_idx_dim))
"""Date of Surface Heat Fluxes inversion."""
from datetime import datetime
from os import path
import lib
import lib.data
ARGS_DIR = {'region', 'season'}
defaults = {'season': 'spring'}
pregex = ("%(grid_file:rgx=.*)_inversion_times_%(Y).txt")
ARGS = {'region', 'season'}
DEFAULTS = {'season': 'spring'}
def get_root(args=None, **kwargs):
args = lib.data.process_args(ARGS_DIR, args, replace_defaults=defaults,
**kwargs,)
root = path.join(lib.root_data, args['region'], 'HF', 'Inversion',
args['season'])
return root
def PREGEX(args):
return "%(grid_file:rgx=.*)_inversion_times_%(Y).txt"
auto_attr = lib.data.create_data(__name__, pregex, get_root, ARGS_DIR,
defaults)
def ROOT(args):
return [lib.root_data, args['region'], 'HF', 'Inversion', args['season']]
auto_attr = lib.data.create_data(__name__, PREGEX, ROOT, ARGS,
defaults=DEFAULTS)
def get_data(args=None, to_datetime=False, **kwargs):
......
from os import path
import lib
import lib.data
ARGS_DIR = {'region', 'days'}
pregex = 'SN_separation_%(Y).nc'
ARGS = {'region', 'days'}
grid = '4km_EPSG4326'
LAT_IS_SEPARATION = 32.
def get_root(args=None, **kwargs):
args = lib.data.process_args(ARGS_DIR, args, **kwargs)
root = path.join(lib.root_data, args['region'], 'SN_separation',
lib.data.get_time_folder(args))
return root
def PREGEX(args):
return "SN_separation_%(Y).nc"
def ROOT(args):
return lib.data.get_default_directory(args, 'SN_separation')
lib.data.create_data(__name__, pregex, get_root, ARGS_DIR)
lib.data.create_data(__name__, PREGEX, ROOT, ARGS)
def smooth(ds, time_step=8):
......
......@@ -113,6 +113,13 @@ def create_dataset(module: str, pregex_func: Callable,
return locals()
def get_default_directory(args, name, time_folder=True):
root = [lib.root_data, args['region'], name]
if time_folder:
root.append(get_time_folder(args))
return root
def get_time_folder(args) -> str:
"""Use days and climato arguments to find time folder."""
if args.get('climato', None) is not None:
......
"""Cloud coverage data."""
from os import path
import xarray as xr
import lib
import lib.data
ARGS = {'region'}
ARGS_DIR = {'region'}
pregex = 'ds_%(data:fmt=s)_%(zone:fmt=s)_%(Y).nc'
def PREGEX(args):
return "ds_%(data:fmt=s)_%(zone:fmt=s)_%(Y).nc"
def get_root(args, **kwargs):
args = lib.data.process_args(ARGS_DIR, args, **kwargs)
dirname = path.join(lib.root_data, args['region'], 'Clouds')
return dirname
def ROOT(args):
return lib.data.get_default_directory(args, 'Clouds', time_folder=False)
def add_dims(ds, filename, finder):
......@@ -25,5 +22,5 @@ def add_dims(ds, filename, finder):
return ds
lib.data.create_data(__name__, pregex, get_root, ARGS_DIR,
lib.data.create_data(__name__, PREGEX, ROOT, ARGS,
open_mf_kw=dict(preprocess_finder=add_dims))
from os import path
import lib
import lib.data
ARGS_DIR = {'region', 'days'}
pregex = '%(Y)/cloud_%(Y)%(m).nc'
ARGS = {'region', 'days'}
grid = '4km_EPSG32662'
def get_root(args=None, **kwargs):
args = lib.data.process_args(ARGS_DIR, args, **kwargs)
root = path.join(lib.root_data, args['region'], 'Clouds_maps',
lib.data.get_time_folder(args))
return root
def PREGEX(args):
return "%(Y)/cloud_%(Y)%(m).nc"
def ROOT(args):
return lib.data.get_default_directory(args, 'Clouds_maps')
lib.data.create_data(__name__, pregex, get_root, ARGS_DIR)
lib.data.create_data(__name__, PREGEX, ROOT, ARGS)
"""Import DataBase for front presence probability."""
from os import path
import lib
import lib.data
ARGS_DIR = {'region', 'period'}
pregex = '%(Y)/%(Y)%(m)%(d).nc'
ARGS = {'region', 'period'}
grid = '4km_EPSG32662'
defaults = dict(period='monthly')
DEFAULTS = dict(period='monthly')
def PREGEX(args):
return "%(Y)/%(Y)%(m)%(d).nc"
def get_root(args=None, **kwargs):
args = lib.data.process_args(ARGS_DIR | {'climato'}, args,
defaults, **kwargs)
root = path.join(lib.root_data, args['region'], 'Frt_proba',
args['period'])
def ROOT(args):
root = lib.data.get_default_directory(args, 'Frt_proba', time_folder=False)
root.append(args['period'])
return root
lib.data.create_data(__name__, pregex, get_root, ARGS_DIR, defaults=defaults)
lib.data.create_data(__name__, PREGEX, ROOT, ARGS, defaults=DEFAULTS)
from os import path
import lib
import lib.data
ARGS_DIR = {'region', 'days'}
pregex = ("number_%(number:fmt=d:rgx=%I)/"
"scale_%(scale:fmt=.1f)/"
"coef_%(coef:fmt=d)/"
"zone_%(zone:fmt=s)/"
"TS_%(time:Y)"
"_thr_%(threshold:fmt=.2f)"
".nc")
def get_root(args=None, **kwargs):
args = lib.data.process_args(ARGS_DIR, args, **kwargs)
root = path.join(lib.root_data, args['region'], 'Hists',
lib.data.get_time_folder(args))
return root
lib.data.create_data(__name__, pregex, get_root, ARGS_DIR)
......@@ -3,26 +3,21 @@
Copernicus: OCEANCOLOUR_GLO_CHL_L4_REP_OBSERVATIONS_009_082 (L4)
"""
from os import path
import lib
import lib.data
ARGS_DIR = {'region', 'days', 'level'}
pregex = r"%(Y)/CHL_%(Y)%(m)%(d).nc"
ARGS = {'region', 'days', 'level'}
grid = '4km_EPSG32662'
def get_root(args=None, **kwargs):
args = lib.data.process_args(ARGS_DIR | {'climato'}, args, **kwargs)
time_folder = lib.data.get_time_folder(args)
root = path.join(lib.root_data,
args['region'],
'GlobColour',
args['level'],
time_folder)
return root
def PREGEX(args):
return "%(Y)/CHL_%(Y)%(m)%(d).nc"
def ROOT(args):
return [lib.root_data, args['region'], 'GlobColour', args['level'],
lib.data.get_time_folder(args)]
lib.data.create_data(__name__, pregex, get_root, ARGS_DIR)
lib.data.create_data(__name__, PREGEX, ROOT, ARGS)
#!/usr/bin/env ipython
from os import path
import lib
import lib.data
ARGS_DIR = {'region', 'days'}
pregex = '%(Y)/HF_%(Y)%(m)%(d).nc'
ARGS = {'region', 'days'}
def PREGEX(args):
return "%(Y)/HF_%(Y)%(m)%(d).nc"
def get_root(args=None, **kwargs):
args = lib.data.process_args(ARGS_DIR, args, **kwargs)
root = path.join(lib.root_data, args['region'], 'HF',
lib.data.get_time_folder(args))
return root
def ROOT(args):
return lib.data.get_default_directory(args, 'HF')
def preprocess(ds):
......@@ -23,5 +20,5 @@ def preprocess(ds):
return ds
lib.data.create_data(__name__, pregex, get_root, ARGS_DIR,
lib.data.create_data(__name__, PREGEX, ROOT, ARGS,
open_mf_kw=dict(preprocess=preprocess))
......@@ -10,8 +10,23 @@ import lib
import lib.data
ARGS_DIR = {'region', 'days', 'scale', 'number'}
pregex = '%(Y)/HI_%(Y)%(m)%(d).nc'
ARGS = {'region', 'days', 'scale', 'number'}
def PREGEX(args):
return "%(Y)/HI_%(Y)%(m)%(d).nc"
def ROOT(args):
root = [lib.root_data, args['region'], 'HI',
'HI_{:.1f}_{:d}'.format(args['scale'], args['number']),
lib.data.get_time_folder(args)]
return root
lib.data.create_data(__name__, PREGEX, ROOT, ARGS)
"""Window sizes.
......@@ -60,17 +75,6 @@ def get_window_size_side(args=None, **kwargs):
return window_size_km_to_pxl[args['grid']][int(args['scale'])]
def get_root(args=None, **kwargs):
args = lib.data.process_args(ARGS_DIR | {'climato'}, args, **kwargs)
root = path.join(lib.root_data, args['region'], 'HI',
'HI_{:.1f}_{:d}'.format(args['scale'], args['number']),
lib.data.get_time_folder(args))
return root
lib.data.create_data(__name__, pregex, get_root, ARGS_DIR)
def apply_coef(ds, coef):
HI = (coef['S'] * np.fabs(ds['S'])
+ coef['V'] * ds['V']
......@@ -133,7 +137,7 @@ class Coefs:
def get_coef(args=None, **kwargs):
args = lib.data.process_args(ARGS_DIR | set(Coefs.PARAMS), args, **kwargs)
args = lib.data.process_args(ARGS | set(Coefs.PARAMS), args, **kwargs)
return coefs[args.pop('region')][args.pop('days')].get(args)
......
from functools import wraps
from os import path
import dask_histogram as dh
import numpy as np
......@@ -11,29 +10,34 @@ from scipy import stats
import lib
import lib.data
import logging
log = logging.getLogger(__name__)
ARGS = {'region', 'days'}
defaults = dict(dim2=False)
ARGS_DIR = {'region', 'days'}
pregex = ("number_%(number:fmt=d:rgx=%I)/"
"scale_%(scale:fmt=.1f)/"
"coef_%(coef:fmt=d)/"
"hist_GS3"
"_thr_%(thr_lo:fmt=.2f)_%(thr_hi:fmt=.2f)"
"_%(time:Y)"
"%(nomask:opt=:_nomask)"
".nc")
def PREGEX(args):
pregex = ["number_%(number:fmt=d:rgx=%I)/",
"scale_%(scale:fmt=.1f)/",
"coef_%(coef:fmt=d)/"]
def get_root(args=None, **kwargs):
args = lib.data.process_args(ARGS_DIR, args, **kwargs)
root = path.join(lib.root_data, args['region'], 'Hists',
lib.data.get_time_folder(args))
return root
if args['dim2']:
pregex.append("hist_2D_GS3")
else:
pregex += ["hist_GS3",
"_thr_%(thr_lo:fmt=.2f)_%(thr_hi:fmt=.2f)"]
pregex += ["_%(time:Y)",
"%(nomask:opt=:_nomask)" if args['nomask'] else "",
".nc"]
auto_attr = lib.data.create_data(__name__, pregex, get_root, ARGS_DIR)
return pregex.join('')
def ROOT(args):
return lib.data.get_default_directory(args, 'Hists')
auto_attr = lib.data.create_data(__name__, PREGEX, ROOT, ARGS)
def get_data(args=None, remove_dims=None, **kwargs):
......
import logging
from os import path
import numpy as np
import xarray as xr
import lib
import lib.data
from lib.data.hists import var_name, get_full_bins
log = logging.getLogger(__name__)
ARGS_DIR = {'region', 'days'}
pregex = ("number_%(number:fmt=d:rgx=%I)/"
"scale_%(scale:fmt=.1f)/"
"coef_%(coef:fmt=d)/"
"hist_2D_GS3"
"_%(time:Y)"
".nc")
def get_root(args=None, **kwargs):
args = lib.data.process_args(ARGS_DIR, args, **kwargs)
root = path.join(lib.root_data, args['region'], 'Hists',
lib.data.get_time_folder(args))
return root
auto_attr = lib.data.create_data(
__name__, pregex, get_root, ARGS_DIR
)
def get_pdf(hist):
"""Return histogram normalized to a probability density function."""
bins_names = [var_name('bins', v) for v in hist.VARS]
bins = [get_full_bins(hist, v) for v in bins_names]
freq = hist / hist.sum(bins_names)
wx, wy = [xr.DataArray(np.diff(b), dims=n)
for b, n in zip(bins, bins_names)]
area = wx*wy
pdf = freq / area
pdf = pdf / (area*pdf).sum(bins_names)
return pdf.rename('pdf')
......@@ -3,16 +3,13 @@
from datetime import datetime
from os import path
import matplotlib.pyplot as plt
from filefinder import Finder
import lib
import lib.data
from lib.box import Image
ARGS_DIR = {'region', 'days'}
ARGS_FIX = {'Y', 'm', 'd'}
ARGS = {'region', 'days'}
def get_images_list(args=None, **kwargs):
......@@ -46,7 +43,7 @@ def nc2ext(file, ext='txt'):
def get_images_text_files(args=None, **kwargs):
args = lib.data.process_args(ARGS_DIR | {'fixes'}, args, **kwargs)
args = lib.data.process_args(ARGS | {'fixes'}, args, **kwargs)
pregex = 'images/A_%(Y)%(m)%(d).txt'
if args['days'] == 1:
pregex = '%(Y)/%(m)/' + pregex
......@@ -57,10 +54,8 @@ def get_images_text_files(args=None, **kwargs):
def get_root(args=None, **kwargs):
args = lib.data.process_args(ARGS_DIR, args, **kwargs)
root = path.join(lib.root_data, args['region'], 'MODIS',
lib.data.get_time_folder(args))
return root
args = lib.data.process_args(ARGS, args, **kwargs)
return lib.data.get_default_directory(args, 'MODIS')
def parse_text_files(files):
......
"""MODIS Data regridded at 1km."""
from os import path
import lib
import lib.data
ARGS_DIR = {'region', 'days'}
pregex = '%(Y)/%(m)/A_%(Y)%(m)%(d).nc'
ARGS = {'region', 'days'}
grid = '1km'
def get_root(args=None, **kwargs):
args = lib.data.process_args(ARGS_DIR, args, **kwargs)
root = path.join(lib.root_data, args['region'], 'MODIS',
lib.data.get_time_folder(args))
return root
def PREGEX(args):
return "%(Y)/%(m)/A_%(Y)%(m)%(d).nc"
def ROOT(args):
return lib.data.get_default_directory(args, 'MODIS')
def preprocess(ds):
return ds.expand_dims('time')
lib.data.create_data(__name__, pregex, get_root, ARGS_DIR,
lib.data.create_data(__name__, PREGEX, ROOT, ARGS,
open_mf_kw=dict(preprocess=preprocess))
......@@ -3,25 +3,24 @@
https://podaac.jpl.nasa.gov/dataset/MUR-JPL-L4-GLOB-v4.1
"""
from os import path
import lib
import lib.data
ARGS_DIR = {'region', 'days'}
pregex = ("%(Y)/%(Y)%(m)%(d)%(time:X)"
"-JPL-L4_GHRSST-SSTfnd-MUR-GLOB-v02.0-fv04.1_subset.nc")
ARGS = {'region', 'days'}
def PREGEX(args):
pregex = ("%(Y)/%(Y)%(m)%(d)%(time:X)"
"-JPL-L4_GHRSST-SSTfnd-MUR-GLOB-v02.0-fv04.1_subset.nc")
return pregex
def get_root(args=None, **kwargs):
args = lib.data.process_args(ARGS_DIR, args, **kwargs)
root = path.join(lib.root_data, args['region'], 'MUR',
lib.data.get_time_folder(args))
return root
def ROOT(args):
return lib.data.get_default_directory(args, 'MUR')
auto_attr = lib.data.create_data(__name__, pregex, get_root, ARGS_DIR)
auto_attr = lib.data.create_data(__name__, PREGEX, ROOT, ARGS)
def get_data(args=None, **kwargs):
......
......@@ -4,7 +4,6 @@ Processed data.
Initially downloaded from CMEMS
https://resources.marine.copernicus.eu/product-detail/SST_GLO_SST_L4_REP_OBSERVATIONS_010_024/INFORMATION
"""
import lib
import lib.data
......@@ -12,6 +11,7 @@ import lib.data
ARGS = {'region', 'days', 'processed'}
DEFAULTS = dict(processed=True)
def PREGEX(args):
pregex = "%(Y)/SST{}_%(Y)%(m)%(d).nc".format(
'_processed' if args['processed'] else '')
......
......@@ -4,25 +4,22 @@ ie pixels where probability of front presence (all data considered) is above a
threshold.
"""
from os import path
import lib
import lib.data