Use environment variable to define directories

cleaner multiple system support

.envrc_example

+# Define data directory
+export SUBMESO_COLOR_DATA_DIR="/data_directory"
+
+# Add repository to python path
+export PYTHONPATH="$SUBMESO_COLOR_CODE_DIR:$PYTHONPATH"

.gitignore

@@ -23,3 +23,5 @@ lib/_build_hi
 SOM-Toolbox
 
 Test/
+
+setup_env.sh

.setup-shell.sh

-#!/usr/bin/env bash
-# To place at $HOME/.setup-shell.sh
-
-shell="bash"
-
-__conda_setup="$($HOME/miniconda3/bin/conda "shell.${shell}" 'hook' 2> /dev/null)"
-if [ $? -eq 0 ]; then
-    eval "$__conda_setup"
-fi
-unset __conda_setup

Compute/climato_monthly.sh

@@ -4,22 +4,20 @@
 
 set -euo pipefail
 
-current_dir="$(dirname "${0}")"
-conf_file="$(readlink -m "${current_dir}/../lib/conf.ini")"
-if ! root_dir="$(grep 'root_data' "${conf_file}" | cut -d ':' -f2 | xargs)"; then
-    echo "No data root directory found in conf file ${conf_file}'"
-    exit 1
-fi
+# shellcheck source=../setup_env_example.sh
+. "${SUBMESO_COLOR_CODE_DIR}/setup_env.sh"
 
 data="GS/HI/HI_30.0_2"
 timefolder="1days"
 
-indir="${root_dir}/${data}"
+indir="${SUBMESO_COLOR_DATA_DIR}/${data}"
 
-cd "$indir"
+if ! cd "$indir"; then
+    echo "Could not enter ${indir}"
+fi
 
 outdir="${indir}/climato/12"
-if [ ! -d "$outdir" ]; then
+if [[ ! -d "$outdir" ]]; then
     mkdir -p "$outdir"
 fi
 

Compute/find_images/extract_images.sh

@@ -9,10 +9,14 @@ set -euo pipefail
 # gfortran find_images.f90 -O2 -I/opt/netcdf43/gfortran49/include \
 #     -lnetcdff -o find_images.exe
 
-launch_script="$HOME/Fronts/launch_python.sh"
+# shellcheck source=../../setup_env_example.sh
+. "${SUBMESO_COLOR_CODE_DIR}/setup_env.sh"
 
-module load netcdf4/4.3.3.1-gfortran49
-module load gnu/4.9.3
+if hostname | grep -q 'ciclad'; then
+     module load netcdf4/4.3.3.1-gfortran49
+     module load gnu/4.9.3
+fi
 
-bash "${launch_script}" "$HOME/Fronts/Compute/find_images/extract_images.py" \
+bash "${SUBMESO_COLOR_CODE_DATA}/launch_pytho.sh" \
+     "${SUBMESO_COLOR_CODE_DATA}/Compute/find_images/extract_images.py" \
      -nx 200 -ny 200 -cover_threshold 0.80 -year 2007 -fix 'm' '06'

Compute/regrid/download_regrid.sh

@@ -2,21 +2,20 @@
 
 date=${1:-200701}
 
-export PYTHONPATH="$HOME/Fronts/:$PYTHONPATH"
-. "$HOME/Fronts/.setup-shell.sh"
-conda activate py38
+# shellcheck source=../../setup_env_example.sh
+. "${SUBMESO_COLOR_CODE_DIR}/setup_env.sh"
 
 module load nco/4.7.x
 
-pass="$(cat $HOME/.local/passwds/earthdata)"
+pass="$(cat ${HOME}/.local/passwds/earthdata)"
 
 for data_type in OC SST; do
-    python $HOME/Fronts/Download/download_modis.py \
+    python "${SUBMESO_COLOR_CODE_DIR}/Download/download_modis.py" \
         -region GS -data_type "$data_type" -date "$date" \
         -user chaeck -password "$pass"
-    python $HOME/Fronts/Compute/regrid/regrid.py \
+    python "${SUBMESO_COLOR_CODE_DIR}/regrid/regrid.py" \
         -region GS -data_type "$data_type" -date "$date"
 done
 
-python $HOME/Fronts/Compute/regrid/merge_daily.py \
+python "${SUBMESO_COLOR_CODE_DIR}/regrid/merge_daily.py" \
     -region GS -date "$date"

Download/download_cmems.sh

@@ -3,9 +3,7 @@
 
 # set -euo pipefail
 
-current_file=$0
-
-data_file="unset"
+data_file_prefix="unset"
 region="GS"
 date_min="unset"
 date_max="unset"
@@ -13,10 +11,10 @@ prefix="unset"
 format="%Y%m%d"
 user="unset"
 
+# shellcheck source=../setup_env_example.sh
+. "${SUBMESO_COLOR_CODE_DIR}/setup_env.sh"
+
 if hostname | grep -q 'ciclad'; then
-    export PYTHONPATH="$HOME/Fronts/:$PYTHONPATH"
-    . "$HOME/Fronts/.setup-shell.sh"
-    conda activate py38
     module load nco/4.7.x
 fi
 
@@ -55,20 +53,20 @@ eval set -- "$PARSED_ARGUMENTS"
 while :
 do
     case "$1" in
-        -d | --data)  data_file="$2"           ; shift 2 ;;
-        -r | --region)  region="$2"       ; shift 2 ;;
-        -s | --start) date_min="$2"        ; shift 2 ;;
-        -e | --end) date_max="$2"            ; shift 2 ;;
+        -d | --data)  data_file_prefix="$2"   ; shift 2 ;;
+        -r | --region)  region="$2"           ; shift 2 ;;
+        -s | --start) date_min="$2"           ; shift 2 ;;
+        -e | --end) date_max="$2"             ; shift 2 ;;
         -f | --format) format="$2"            ; shift 2 ;;
         -p | --prefix) prefix="$2"            ; shift 2 ;;
-        -u | --user) user="$2"            ; shift 2 ;;
+        -u | --user) user="$2"                ; shift 2 ;;
         --) shift; break ;;
         *) echo "Unexpected option: $1"
            help; exit 2 ;;
     esac
 done
 
-if [[ "$data_file" == "unset" ]]; then
+if [[ "$data_file_prefix" == "unset" ]]; then
     echo "Need data set"
     help
     exit 1
@@ -92,16 +90,11 @@ if [[ "$date_max" == "unset" ]]; then
     date_max="$date_min"
 fi
 
-motu_dir="$(dirname "${current_file}")/motu"
-conf_file="$(readlink -m "${motu_dir}/../../lib/conf.ini")"
+motu_dir="${SUBMESO_COLOR_CODE_DIR}/Download/motu"
 server_file="${motu_dir}/server.ini"
 region_file="${motu_dir}/${region}.ini"
-data_file="${motu_dir}/${data_file}.ini"
+data_file="${motu_dir}/${data_file_prefix}.ini"
 
-if [[ ! -f "$conf_file" ]]; then
-    echo "Could not find conf file '${conf_file}'"
-    exit 1
-fi
 if [[ ! -f "$server_file" ]]; then
     echo "Could not find server file '${server_file}'"
     exit 1
@@ -115,17 +108,13 @@ if [[ ! -f "$data_file" ]]; then
     exit 1
 fi
 
-if ! root_dir="$(grep 'root_data' "${conf_file}" | cut -d ':' -f2 | xargs)"; then
-    echo "No data root directory found in conf file ${conf_file}'"
-    exit 1
-fi
 if ! out_dir="$(grep 'out_dir' "$data_file" | cut -d '=' -f2)"; then
     echo "No output directory found in data file '${data_file}'"
     exit 1
 fi
 
 year="$(date -d "$date_min" +%Y)"
-out_dir="${root_dir}/${region}/${out_dir}/${year}"
+out_dir="${SUBMESO_COLOR_DATA_DIR}/${region}/${out_dir}/${year}"
 echo "Output dir: ${out_dir}"
 if [[ ! -d "$out_dir" ]]; then
     echo "Invalid output directory."

README.md

@@ -11,22 +11,31 @@ Collocate submesoscale fronts to phytoplankton levels using satellite imagery
   can be used.
 - Plots: Plotting scripts
 
-The root directory containing data has to be specified in the file `lib/conf.ini` as
-``` py
-[main]
-root_data: /...
-```
+## Setup
+
+The environment variable `$SUBMESO_COLOR_CODE_DIR` must be defined and point to
+this repository location.
+Bash scripts will source the `setup_env.sh` file in it, which defines other necessary
+variables, notably `$SUBMESO_COLOR_DATA_DIR` for the location of data and plots.
+It also appends this repository to the `$PYTHONPATH` variable.
+The organisation of `setup_env.sh` is left to the user.
+An example is given which works on a personnal computer and on the [Ciclad](https://mesocentre.ipsl.fr/) cluster.
 
 
 ## Requirements
-    
+
+    python3.8
+    numpy
+    scipy
+    xarray
     dask
     matplotlib
     netcdf4
-    numpy
-    scipy
+    motuclient
+    cftime
     shapely
-    xarray
-    filefinder
     global-land-mask
+    filefinder
+    
+A conda environment file is provided ([./env.yml](./env.yml))
 

launch_python.sh

@@ -4,8 +4,7 @@
 script=${1}
 args=("$@")
 
-export PYTHONPATH="$HOME/Fronts/:$PYTHONPATH"
-. "$HOME/Fronts/.setup-shell.sh"
-conda activate py38
+# shellcheck source=./setup_env_example.sh
+. "${SUBMESO_COLOR_CODE_DIR}/setup_env.sh"
 
 python "${script}" "${args[@]:1}"

lib/__init__.py

@@ -11,19 +11,31 @@ import sys
 
 import pandas as pd
 
+# Scripts repository
+try:
+    root_code = os.environ['SUBMESO_COLOR_CODE_DIR']
+except KeyError:
+    raise KeyError("SUBMESO_COLOR_CODE_DIR env variable is not defined")
+
+# Data directory
+try:
+    root_data = os.environ['SUBMESO_COLOR_DATA_DIR']
+except KeyError:
+    raise KeyError("SUBMESO_COLOR_DATA_DIR env variable is not defined")
+if not path.isdir(root_data):
+    raise NameError(f"root_data '{root_data}' is not a valid directory.")
+
+# Plot directory
+root_plot = path.join(root_data, 'Plots')
+
 # Read config file
 _config = configparser.ConfigParser()
-_config_file = path.join(path.dirname(__file__), 'conf.ini')
+_config_file = path.join(root_code, 'lib', 'conf.ini')
 if not path.isfile(_config_file):
     raise RuntimeError("Missing configuration file lib/conf.ini")
 
 _config.read(_config_file)
 
-root_data = _config.get('main', 'root_data')
-if not path.isdir(root_data):
-    raise NameError(f"root_data '{root_data}' is not a valid directory.")
-root_plot = path.join(root_data, 'Plots')
-
 DEFAULTS = {k: _config.get('defaults', k)
             for k in _config.options('defaults')}
 

lib/conf_example.ini

 # This is an example of the lib/conf.ini file
 
-[main]
-root_data: /home/clement/Documents/Work/Data
-
 [defaults]
 region: GS
 year: 2007

setup_env_example.sh

+#!/usr/bin/env sh
+
+# Define environment variables using an .envrc file, so that
+# on my personnal computer, necessary variables are automatically
+# defined, and I can run the python scripts directly.
+# shellcheck source=./.envrc_example
+. "$SUBMESO_COLOR_DATA_DIR/.envrc_example"
+
+# If on Ciclad cluster, activate conda environment.
+if hostname | grep -q 'ciclad'; then
+    shell="bash"
+    __conda_setup="$("$HOME/miniconda3/bin/conda" "shell.${shell}" 'hook' 2> /dev/null)"
+    if [ $? -eq 0 ]; then
+        eval "$__conda_setup"
+    fi
+    unset __conda_setup
+    conda activate submesocolor
+fi