Docker-in-Docker (DinD) capabilities of public runners deactivated. More info

Commit f3d29998 authored by Clément Haëck's avatar Clément Haëck
Browse files

Use environment variable to define directories

cleaner multiple system support
parent 8f3249a6
# Define data directory
export SUBMESO_COLOR_DATA_DIR="/data_directory"
# Add repository to python path
export PYTHONPATH="$SUBMESO_COLOR_CODE_DIR:$PYTHONPATH"
......@@ -23,3 +23,5 @@ lib/_build_hi
SOM-Toolbox
Test/
setup_env.sh
#!/usr/bin/env bash
# To place at $HOME/.setup-shell.sh
shell="bash"
__conda_setup="$($HOME/miniconda3/bin/conda "shell.${shell}" 'hook' 2> /dev/null)"
if [ $? -eq 0 ]; then
eval "$__conda_setup"
fi
unset __conda_setup
......@@ -4,22 +4,20 @@
set -euo pipefail
current_dir="$(dirname "${0}")"
conf_file="$(readlink -m "${current_dir}/../lib/conf.ini")"
if ! root_dir="$(grep 'root_data' "${conf_file}" | cut -d ':' -f2 | xargs)"; then
echo "No data root directory found in conf file ${conf_file}'"
exit 1
fi
# shellcheck source=../setup_env_example.sh
. "${SUBMESO_COLOR_CODE_DIR}/setup_env.sh"
data="GS/HI/HI_30.0_2"
timefolder="1days"
indir="${root_dir}/${data}"
indir="${SUBMESO_COLOR_DATA_DIR}/${data}"
cd "$indir"
if ! cd "$indir"; then
echo "Could not enter ${indir}"
fi
outdir="${indir}/climato/12"
if [ ! -d "$outdir" ]; then
if [[ ! -d "$outdir" ]]; then
mkdir -p "$outdir"
fi
......
......@@ -9,10 +9,14 @@ set -euo pipefail
# gfortran find_images.f90 -O2 -I/opt/netcdf43/gfortran49/include \
# -lnetcdff -o find_images.exe
launch_script="$HOME/Fronts/launch_python.sh"
# shellcheck source=../../setup_env_example.sh
. "${SUBMESO_COLOR_CODE_DIR}/setup_env.sh"
module load netcdf4/4.3.3.1-gfortran49
module load gnu/4.9.3
if hostname | grep -q 'ciclad'; then
module load netcdf4/4.3.3.1-gfortran49
module load gnu/4.9.3
fi
bash "${launch_script}" "$HOME/Fronts/Compute/find_images/extract_images.py" \
bash "${SUBMESO_COLOR_CODE_DATA}/launch_pytho.sh" \
"${SUBMESO_COLOR_CODE_DATA}/Compute/find_images/extract_images.py" \
-nx 200 -ny 200 -cover_threshold 0.80 -year 2007 -fix 'm' '06'
......@@ -2,21 +2,20 @@
date=${1:-200701}
export PYTHONPATH="$HOME/Fronts/:$PYTHONPATH"
. "$HOME/Fronts/.setup-shell.sh"
conda activate py38
# shellcheck source=../../setup_env_example.sh
. "${SUBMESO_COLOR_CODE_DIR}/setup_env.sh"
module load nco/4.7.x
pass="$(cat $HOME/.local/passwds/earthdata)"
pass="$(cat ${HOME}/.local/passwds/earthdata)"
for data_type in OC SST; do
python $HOME/Fronts/Download/download_modis.py \
python "${SUBMESO_COLOR_CODE_DIR}/Download/download_modis.py" \
-region GS -data_type "$data_type" -date "$date" \
-user chaeck -password "$pass"
python $HOME/Fronts/Compute/regrid/regrid.py \
python "${SUBMESO_COLOR_CODE_DIR}/regrid/regrid.py" \
-region GS -data_type "$data_type" -date "$date"
done
python $HOME/Fronts/Compute/regrid/merge_daily.py \
python "${SUBMESO_COLOR_CODE_DIR}/regrid/merge_daily.py" \
-region GS -date "$date"
......@@ -3,9 +3,7 @@
# set -euo pipefail
current_file=$0
data_file="unset"
data_file_prefix="unset"
region="GS"
date_min="unset"
date_max="unset"
......@@ -13,10 +11,10 @@ prefix="unset"
format="%Y%m%d"
user="unset"
# shellcheck source=../setup_env_example.sh
. "${SUBMESO_COLOR_CODE_DIR}/setup_env.sh"
if hostname | grep -q 'ciclad'; then
export PYTHONPATH="$HOME/Fronts/:$PYTHONPATH"
. "$HOME/Fronts/.setup-shell.sh"
conda activate py38
module load nco/4.7.x
fi
......@@ -55,20 +53,20 @@ eval set -- "$PARSED_ARGUMENTS"
while :
do
case "$1" in
-d | --data) data_file="$2" ; shift 2 ;;
-r | --region) region="$2" ; shift 2 ;;
-s | --start) date_min="$2" ; shift 2 ;;
-e | --end) date_max="$2" ; shift 2 ;;
-d | --data) data_file_prefix="$2" ; shift 2 ;;
-r | --region) region="$2" ; shift 2 ;;
-s | --start) date_min="$2" ; shift 2 ;;
-e | --end) date_max="$2" ; shift 2 ;;
-f | --format) format="$2" ; shift 2 ;;
-p | --prefix) prefix="$2" ; shift 2 ;;
-u | --user) user="$2" ; shift 2 ;;
-u | --user) user="$2" ; shift 2 ;;
--) shift; break ;;
*) echo "Unexpected option: $1"
help; exit 2 ;;
esac
done
if [[ "$data_file" == "unset" ]]; then
if [[ "$data_file_prefix" == "unset" ]]; then
echo "Need data set"
help
exit 1
......@@ -92,16 +90,11 @@ if [[ "$date_max" == "unset" ]]; then
date_max="$date_min"
fi
motu_dir="$(dirname "${current_file}")/motu"
conf_file="$(readlink -m "${motu_dir}/../../lib/conf.ini")"
motu_dir="${SUBMESO_COLOR_CODE_DIR}/Download/motu"
server_file="${motu_dir}/server.ini"
region_file="${motu_dir}/${region}.ini"
data_file="${motu_dir}/${data_file}.ini"
data_file="${motu_dir}/${data_file_prefix}.ini"
if [[ ! -f "$conf_file" ]]; then
echo "Could not find conf file '${conf_file}'"
exit 1
fi
if [[ ! -f "$server_file" ]]; then
echo "Could not find server file '${server_file}'"
exit 1
......@@ -115,17 +108,13 @@ if [[ ! -f "$data_file" ]]; then
exit 1
fi
if ! root_dir="$(grep 'root_data' "${conf_file}" | cut -d ':' -f2 | xargs)"; then
echo "No data root directory found in conf file ${conf_file}'"
exit 1
fi
if ! out_dir="$(grep 'out_dir' "$data_file" | cut -d '=' -f2)"; then
echo "No output directory found in data file '${data_file}'"
exit 1
fi
year="$(date -d "$date_min" +%Y)"
out_dir="${root_dir}/${region}/${out_dir}/${year}"
out_dir="${SUBMESO_COLOR_DATA_DIR}/${region}/${out_dir}/${year}"
echo "Output dir: ${out_dir}"
if [[ ! -d "$out_dir" ]]; then
echo "Invalid output directory."
......
......@@ -11,22 +11,31 @@ Collocate submesoscale fronts to phytoplankton levels using satellite imagery
can be used.
- Plots: Plotting scripts
The root directory containing data has to be specified in the file `lib/conf.ini` as
``` py
[main]
root_data: /...
```
## Setup
The environment variable `$SUBMESO_COLOR_CODE_DIR` must be defined and point to
this repository location.
Bash scripts will source the `setup_env.sh` file in it, which defines other necessary
variables, notably `$SUBMESO_COLOR_DATA_DIR` for the location of data and plots.
It also appends this repository to the `$PYTHONPATH` variable.
The organisation of `setup_env.sh` is left to the user.
An example is given which works on a personnal computer and on the [Ciclad](https://mesocentre.ipsl.fr/) cluster.
## Requirements
python3.8
numpy
scipy
xarray
dask
matplotlib
netcdf4
numpy
scipy
motuclient
cftime
shapely
xarray
filefinder
global-land-mask
filefinder
A conda environment file is provided ([./env.yml](./env.yml))
......@@ -4,8 +4,7 @@
script=${1}
args=("$@")
export PYTHONPATH="$HOME/Fronts/:$PYTHONPATH"
. "$HOME/Fronts/.setup-shell.sh"
conda activate py38
# shellcheck source=./setup_env_example.sh
. "${SUBMESO_COLOR_CODE_DIR}/setup_env.sh"
python "${script}" "${args[@]:1}"
......@@ -11,19 +11,31 @@ import sys
import pandas as pd
# Scripts repository
try:
root_code = os.environ['SUBMESO_COLOR_CODE_DIR']
except KeyError:
raise KeyError("SUBMESO_COLOR_CODE_DIR env variable is not defined")
# Data directory
try:
root_data = os.environ['SUBMESO_COLOR_DATA_DIR']
except KeyError:
raise KeyError("SUBMESO_COLOR_DATA_DIR env variable is not defined")
if not path.isdir(root_data):
raise NameError(f"root_data '{root_data}' is not a valid directory.")
# Plot directory
root_plot = path.join(root_data, 'Plots')
# Read config file
_config = configparser.ConfigParser()
_config_file = path.join(path.dirname(__file__), 'conf.ini')
_config_file = path.join(root_code, 'lib', 'conf.ini')
if not path.isfile(_config_file):
raise RuntimeError("Missing configuration file lib/conf.ini")
_config.read(_config_file)
root_data = _config.get('main', 'root_data')
if not path.isdir(root_data):
raise NameError(f"root_data '{root_data}' is not a valid directory.")
root_plot = path.join(root_data, 'Plots')
DEFAULTS = {k: _config.get('defaults', k)
for k in _config.options('defaults')}
......
# This is an example of the lib/conf.ini file
[main]
root_data: /home/clement/Documents/Work/Data
[defaults]
region: GS
year: 2007
......
#!/usr/bin/env sh
# Define environment variables using an .envrc file, so that
# on my personnal computer, necessary variables are automatically
# defined, and I can run the python scripts directly.
# shellcheck source=./.envrc_example
. "$SUBMESO_COLOR_DATA_DIR/.envrc_example"
# If on Ciclad cluster, activate conda environment.
if hostname | grep -q 'ciclad'; then
shell="bash"
__conda_setup="$("$HOME/miniconda3/bin/conda" "shell.${shell}" 'hook' 2> /dev/null)"
if [ $? -eq 0 ]; then
eval "$__conda_setup"
fi
unset __conda_setup
conda activate submesocolor
fi
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment