Docker-in-Docker (DinD) capabilities of public runners deactivated. More info

Commit 94bf7df1 authored by Clément Haëck's avatar Clément Haëck
Browse files

Adapt more scripts

parent 33daa9d6
......@@ -7,30 +7,24 @@ from lib.data.hi import get_data
from lib import compute_hi, get_args
args = get_args(['scale', 'days', 'number'])
scale = float(args['scale'])
args = get_args(['region', 'days', 'year', 'scale', 'number'])
region = str(args['region'])
days = int(args['days'])
number = int(args['number'])
year = int(args['year'])
db = get_data(scale, number, days)
scale = float(args['scale'])
number = int(args['number'])
box_lat = [20, 52]
box_lon = [-82, -45]
range_time = [[2007, 1, 1], [2007, 12, 31]]
db = get_data(region, days, year, scale, number)
extremas = {'S': [0.0000, 8.6694],
'V': [0.0000, 1.0369],
'B': [0.0340, 1.4108]}
db.select_by_value(scope='avail',
lat=slice(*box_lat), lon=slice(*box_lon),
time=slice(*range_time), by_day=True)
## Compute
for time_slice in db.selected.iter_slices('time', 3):
db.load_selected(time=time_slice)
for time_slice in db.avail.iter_slices('time', 3):
db.load(time=time_slice)
compute_hi.compute_hi(db, extremas)
db.write_add_variable('HI', 'V', kwargs={'zlib': True})
......@@ -6,4 +6,5 @@ export PYTHONPATH="/home/chaeck/Fronts/:$PYTHONPATH"
conda activate py38
python "$HOME/Fronts/Compute/add_hi.py" \
--kwargs scale:10. days:8 number:1
--kwargs region:"'GS'" days:8 year:2003 \
scale:10. number:1
......@@ -21,44 +21,37 @@ import lib.data.sst
import lib.data.chl
args = get_args(['scale', 'days', 'step', 'level',
'number', 'target'])
scale = float(args['scale'])
args = get_args(['region', 'days', 'year',
'scale', 'number', 'target',
'level', 'step'])
region = str(args['region'])
days = int(args['days'])
step = int(args['step'])
level = int(args['level'])
year = int(args['year'])
scale = float(args['scale'])
number = int(args['number'])
target = int(args['target'])
box_lat = [20, 52]
box_lon = [-82, -45]
range_time = [[2007, 1, 1], [2007, 12, 31]]
level = int(args['level'])
step = int(args['step'])
cstr_hi = lib.data.mask.get_cstr(scale=scale, number=number,
days=days, chl=False)
lib.data.sst.add_sst(cstr_hi, days=days)
cstr_hi = lib.data.mask.get_cstr(region, days, year, scale, number, chl=False)
lib.data.sst.add_sst(cstr_hi, region, days, year)
db_hi = cstr_hi.make_data()
cstr_lo = lib.data.mask.get_cstr(scale=scale, number=number,
days=days, chl=True)
lib.data.chl.add_chl(cstr_lo, days=days, level=level)
cstr_lo = lib.data.mask.get_cstr(region, days, year, scale, number, chl=True)
lib.data.chl.add_chl(cstr_lo, region, days, year, level)
cstr_lo.add_scan_filename(scanlib.get_date_from_matches, 'time',
default_date={'hour': 9})
cstr_lo.set_elements_constant('time', in_idx=0)
db_lo = cstr_lo.make_data()
for db in [db_lo, db_hi]:
db.select_by_value('avail',
lat=slice(*box_lat), lon=slice(*box_lon),
time=slice(*range_time), by_day=True)
db.select_by_value('avail')
dbz_lo = get_zones('lo')
dbz_hi = get_zones('hi')
dbz_lo = get_zones(region, 'lo')
dbz_hi = get_zones(region, 'hi')
zones = ['1', '4']
for db, dbz in zip([db_lo, db_hi], [dbz_lo, dbz_hi]):
......@@ -120,8 +113,9 @@ def get_stats_init(db):
db_stats = dt.DataDisk([var, variable, mask, zone, time],
root=path.join(root_data,
'HI/{:d}days/HI_{:.1f}_{:d}/L{:d}_step{:d}'
.format(days, scale, target, level, step)),
'{:s}/{:d}days/{:}/HI/HI_{:.1f}_{:d}/L{:d}_step{:d}'
.format(region, days, year, scale,
target, level, step)),
filegroups=[])
db_stats.create_variables()
db_stats.add_filegroup(FilegroupNetCDF, [], 'stats')
......
......@@ -6,5 +6,6 @@ export PYTHONPATH="/home/chaeck/Fronts/:$PYTHONPATH"
conda activate py38
python "$HOME/Fronts/Compute/compute_deltas_daily.py" \
--kwargs scale:10. days:8 step:1 level:3 \
number:1 target:1
--kwargs region:"'GS'" days:8 year: 2003 \
scale:10. number:1 target:1 \
level:4 step:1
......@@ -10,26 +10,24 @@ from lib.data.mask import get_data
from lib.zones import get_zones
args = get_args(['scale', 'days', 'number'])
scale = float(args['scale'])
args = get_args(['region', 'days', 'year', 'scale', 'number', 'step'])
region = str(args['region'])
days = int(args['days'])
step = int(args['step'])
year = int(args['year'])
scale = float(args['scale'])
number = int(args['number'])
db_lo = get_data(scale=scale, number=number, days=days, chl=True)
db_hi = get_data(scale=scale, number=number, days=days, chl=False)
step = int(args['step'])
box_lat = [20, 52]
box_lon = [-82, -45]
range_time = [[2007, 1, 1], [2007, 12, 31]]
db_lo = get_data(region, days, year, scale, number, chl=True)
db_hi = get_data(region, days, year, scale, number, chl=False)
db_lo.select_by_value('avail', time=slice(*range_time),
lat=slice(*box_lat), lon=slice(*box_lon), by_day=True)
db_hi.select_by_value('avail', time=slice(*range_time),
lat=slice(*box_lat), lon=slice(*box_lon), by_day=True)
db_lo.select_by_value('avail')
db_hi.select_by_value('avail')
dbz_lo = get_zones('lo')
dbz_hi = get_zones('hi')
dbz_lo = get_zones(region, 'lo')
dbz_hi = get_zones(region, 'hi')
zones = ['1', '4']
for db, dbz in zip([db_lo, db_hi], [dbz_lo, dbz_hi]):
......
......@@ -6,4 +6,5 @@ export PYTHONPATH="/home/chaeck/Fronts/:$PYTHONPATH"
conda activate py38
python "$HOME/Fronts/Compute/compute_fraction.py" \
--kwargs scale:10. days:1 number:2
--kwargs region:"'GS'" days:8 year:2003 \
scale:10. number:1 step:1
......@@ -16,10 +16,6 @@ from lib import root_data, get_args
plt.style.use('common')
box_lat = [20, 52]
box_lon = [-82, -45]
range_time = [[2007, 1, 1], [2007, 12, 31]]
args = get_args(['region', 'days', 'year',
'scale', 'number', 'var',
......@@ -37,8 +33,7 @@ var = str(args['var'])
threshold_min = float(args['threshold_min'])
threshold_max = float(args['threshold_max'])
db = get_data(region=region, days=days, year=year,
scale=scale, number=number)
db = get_data(region, days, year, scale, number)
kwargs = {'dims': ['time', 'lat', 'lon'],
'var_class': vt.VariableMasked}
......@@ -47,9 +42,7 @@ db.add_variable('front', **kwargs)
db.add_variable('background', **kwargs)
db.select_by_value(scope='avail', var=var,
lat=slice(*box_lat), lon=slice(*box_lon),
time=slice(*range_time), by_day=True)
db.select_by_value(scope='avail', var=var)
# db.selected.slice(time=slice(None, None, 2))
wd = path.join(root_data,
......
......@@ -16,18 +16,16 @@ from lib import root_data, get_args
from lib.downsample_f import downsample
box_lat = [20, 52]
box_lon = [-82, -45]
range_time = [[2007, 1, 1], [2007, 12, 31]]
args = get_args(['region', 'days', 'year', 'scale', 'number'])
region = str(args['region'])
days = int(args['days'])
year = int(args['year'])
args = get_args(['scale', 'days', 'number'])
scale = float(args['scale'])
days = int(args['days'])
number = int(args['number'])
db_chl = lib.data.chl.get_data(days=days)
db_mask = lib.data.mask.get_data(scale=scale, number=number,
days=days, chl=False)
db_chl = lib.data.chl.get_data(region, days, year)
db_mask = lib.data.mask.get_data(region, days, year, scale, number)
wd = path.normpath(path.join(root_data,
db_mask.filegroups[0].root, '../masks_chl'))
......@@ -131,9 +129,7 @@ def downsample_mask(db_mask_chl, db_hi, mask):
## Compute
db_mask.select_by_value(scope='avail',
lat=slice(*box_lat), lon=slice(*box_lon),
time=slice(*range_time), by_day=True)
db_mask.select_by_value(scope='avail')
db_mask_chl = get_db_mask_chl(db_mask, db_chl)
......
......@@ -6,4 +6,5 @@ export PYTHONPATH="/home/chaeck/Fronts/:$PYTHONPATH"
conda activate py38
python "$HOME/Fronts/Compute/regrid_mask.py" \
--kwargs scale:10. days:1 number:1
--kwargs region:"'GS'" days:1 year:2003 \
scale:10. number:1
......@@ -7,7 +7,7 @@ export PYTHONPATH="/home/chaeck/Fronts/:$PYTHONPATH"
conda activate py38
year="2014"
year="2018"
region="GS"
directory="/data/chaeck/${region}/1days/${year}/SST"
......
......@@ -21,5 +21,5 @@ Collocate submesoscale fronts to phytoplankton levels using satellite imagery
shapely
All loading of data from disk (and most of writing results to disk) is done
using [tomate (v1.\*)](https://github.com/Descanonge/tomate), a custom made python
using [tomate](https://github.com/Descanonge/tomate), a custom made python
package to load data from multiple files.
......@@ -6,8 +6,8 @@ module load nco/4.7.x
wd="/data/chaeck"
variable="CHL_L4"
year="2003"
variable="SST"
year="2005"
precision=".3"
region="GS"
......
......@@ -46,7 +46,7 @@ def rasterize_zones(db, zones):
db.select_by_value(lat=slice(15, 62), lon=slice(-90, -30))
lat = db.selected.lat.copy()
lon = db.selected.lon.copy()
cstr = Constructor(path.join(root_data, 'zones'), [lat, lon])
cstr = Constructor(path.join(root_data, 'GS/zones'), [lat, lon])
cstr.add_filegroup(FilegroupNetCDF, [CSS(lat), CSS(lon)], 'ZONES')
cstr.set_variables_names(**{name: name for name in zones})
cstr.set_elements_constant('var', dimensions=('lat', 'lon'))
......@@ -60,9 +60,10 @@ def rasterize_zones(db, zones):
return db_
def add_zones(cstr, kind='hi'):
def add_zones(cstr, region='GS', kind='hi'):
cstr.add_filegroup(FilegroupNetCDF, [CSS('lat'), CSS('lon')], 'ZONES',
root='zones', variables_shared=True)
root='{:s}/zones'.format(region),
variables_shared=True)
cstr.set_fg_regex(rf'%(var:char)_{kind}\.nc')
cstr.add_scan_in_file(scanlib.nc.scan_dims, 'lat', 'lon')
cstr.add_scan_in_file(scanlib.nc.scan_variables, 'var')
......@@ -70,13 +71,13 @@ def add_zones(cstr, kind='hi'):
cstr.set_data_types([dt.DataPlot])
def get_zones(kind='hi'):
def get_zones(region='GS', kind='hi'):
if kind not in ['lo', 'hi']:
raise KeyError("kind should be 'lo' or 'hi'")
lat = Lat()
lon = Lon()
cstr = Constructor(root_data, [lat, lon])
add_zones(cstr, kind)
add_zones(cstr, region, kind)
db = cstr.make_data()
return db
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment