ASAP2 croco+agrif+pisces ok on IRENE & IRENE-AMD with xios & nc4par for restart

compile/xios/xios_launch.file_IRENE-AMD.base

@@ -6,7 +6,5 @@
 #
 #    search MPMD (Multi Program Multi Data) for documentation
 #              (with irene.info for example)
-
 <nproc_model_oce> ./<model_oce>
 <nproc_xios> ./<xios_server>
-

compile/xios/xios_launch.file_IRENE.base

@@ -6,7 +6,5 @@
 #
 #    search MPMD (Multi Program Multi Data) for documentation
 #              (with irene.info for example)
-
 <nproc_model_oce> ./<model_oce>
 <nproc_xios> ./<xios_server>
-

compile/xios/xios_launch.file_JEANZAY.base

@@ -6,7 +6,5 @@
 #
 #    search MPMD (Multi Program Multi Data) for documentation
 #              (with irene.info for example)
-
 0-<nproc_model_oce-1> ./<model_oce>
 <nproc_model_oce>-<add_nproc_xios> ./<xios_server>
-

scripts/IRENE-AMD.env

@@ -57,7 +57,7 @@ if [ "${MODEL_OCE}" = "nemo" ] ; then
    export EXEC="time -p mpirun -np 126  ./nemo.exe  :  -np 2 ./xios_server.exe"
 elif [ "${MODEL_OCE}" = "croco" ] ; then
    if ${USE_XIOS} ; then 
-      export EXEC="time -p ccc_mprun --multi-prog ./xios_launch.file croco.in"
+      export EXEC="time -p ccc_mprun  -E-K1 -f ./xios_launch.file"
    else 
       export EXEC="time -p ccc_mprun ./croco_${COMPUTER}.exe croco.in"
    fi

scripts/IRENE-AMD.header

@@ -24,7 +24,7 @@
 # #MSUB -N 4         # number of nodes to use   (better to use -c option to reserve more memory)
 #MSUB -x             # requests exclusive usage of allocated nodes.
 
-#MSUB -A gen1140
+#MSUB -A gen6895
 #MSUB -m work,scratch,store
 
 #SUB -Q test        #  Queue test with 1800 s max for time

scripts/IRENE-AMD.module

-module purge 
+#!/bin/bash
 
-module load flavor/buildmpi/openmpi/4.0
-module load intel/20.0.0
-#module load intel/19.0.5.281
-module load mpi/openmpi/4.0.3
-
-module load flavor/buildcompiler/intel/20
-#module load flavor/buildcompiler/intel/19
-module load flavor/hdf5/parallel
-module load netcdf-fortran/4.4.4
-module load netcdf-c/4.6.0
-module load hdf5/1.8.20
+echo ""
+echo "module purge..."
+module purge   >/dev/null 2>&1
 
+#  from XIOS_X64_IRENE-AMD_trunk_r2219-2021_09_06/arch/arch-X64_IRENE-AMD.env
+# module load flavor/hdf5/parallel
+# module load netcdf-fortran/4.4.4
+# module load hdf5/1.8.20
+# module load gnu
+# module load boost
+# module load blitz
+# module load feature/bridge/heterogenous_mpmd
 
+for module in "flavor/hdf5/parallel" "netcdf-fortran/4.4.4" "hdf5/1.8.20" "gnu"
+do
+   echo ""
+   echo "LOAD MODULE ${module}"
+   module load ${module}
+done
 
+echo ""
 
 

scripts/IRENE.module

+#!/bin/bash
+
+echo ""
+echo "module purge..."
 module purge   >/dev/null 2>&1
 
-# From Seb 02/08/2021      see $HOME/loadintel.sh & $HOME/loadgcc.sh
-    module load intel/19.0.5.281
-    module load mpi/openmpi/4.0.2
-    module load flavor/hdf5/parallel
-    module load flavor/buildmpi/openmpi/4.0
-    module load netcdf-c/4.6.0
-    module load netcdf-fortran/4.4.4
-    module load hdf5/1.8.20
-    module load nco/4.9.1
-    module load cdo/1.9.5
-    module load ncview/2.1.7
-    module load boost/1.69.0
-    module load blitz/0.10
-    module load fftw3/mkl/19.0.5.281  # for paraver
-    module load c++/gnu/7.3.0
-    module load c/gnu/7.3.0
+# from XIOS_X64_IRENE_trunk_r2219-2021_09_06/arch/arch-X64_IRENE.env
+# module load flavor/hdf5/parallel
+# module load netcdf-fortran/4.4.4
+# module load hdf5/1.8.20
+# module load gnu/7.3.0
+# module load boost
+# module load blitz
+# module load feature/bridge/heterogenous_mpmd
 
-#module purge 
-#module load flavor/buildcompiler/intel/19
-#module load mpi/openmpi/2.0.4
-# from XIOS 
-#module load flavor/hdf5/parallel
-#module load netcdf-fortran/4.4.4
-#module load netcdf-c/4.6.0
-#module load hdf5/1.8.20
+for module in "flavor/hdf5/parallel" "netcdf-fortran/4.4.4" "hdf5/1.8.20" "gnu/7.3.0"
+do
+   echo ""
+   echo "LOAD MODULE ${module}"
+   module load ${module}
+done
 
+echo ""

scripts/JEANZAY.module

+#!/bin/bash
+
+echo ""
+echo "module purge..."
+module purge   >/dev/null 2>&1
 
 # from XIOS_X64_JEANZAY_trunk_r2219-2021_09_06/arch/arch-X64_JEANZAY.env
-# module purge
 # module load gcc/9.1.0
 # module load intel-all
 # module load gcc/9.1.0
@@ -8,20 +12,12 @@
 # module load netcdf/4.7.2-mpi
 # module load netcdf-fortran/4.5.2-mpi
 
-# from XIOS_X64_JEANZAY_v2.5_r2152-2021_06_10/arch/arch-X64_JEANZAY.env
-# module purge
-# module load intel-compilers/19.0.4 intel-mkl/19.0.4 intel-mpi/19.0.4
-# module load hdf5/1.10.5-mpi
-# module load netcdf/4.7.2-mpi
-# module load netcdf-fortran/4.5.2-mpi
-
-module purge
-module load intel-compilers/19.0.4
-module load intel-mpi/19.0.4
-module load netcdf-fortran/4.5.2-mpi
-module load netcdf/4.7.2-mpi
-module load hdf5/1.10.5-mpi
-
-
+for module in "intel-compilers/19.0.4" "intel-mpi/19.0.4" "netcdf-fortran/4.5.2-mpi" "netcdf/4.7.2-mpi" "hdf5/1.10.5-mpi"
+do
+   echo ""
+   echo "LOAD MODULE ${module}"
+   module load ${module}
+done
 
+echo ""
 

scripts/common_printing.sh

@@ -3,8 +3,12 @@
 #===============================================================================
 #  Print
 #===============================================================================
-printf "\n Modules list:\n"
-module list -l
+echo ""
+echo "modules loaded on ${COMPUTER} :"
+module list -t  2>&1
+echo ""
+echo ""
+echo ""
 
 printf "\n Period of Experiment:\n"
 printf  "%12s%10d%22s%14d%17s\n" "BEGINING:" ${DATE_BEGIN_EXP} "(in ${CALTYPE} calendar)" ${JDAY_BEGIN_EXP} "(in Julian days)"
@@ -40,3 +44,4 @@ printf "%-15s : %-80s\n\n" "EXEDIR" "${EXEDIR}"
 
 printf "%-15s : %-80s\n\n" "LD_LIBRARY_PATH" "${LD_LIBRARY_PATH}"
 
+printf "\n\n"

scripts/compile_oce.sh

@@ -16,7 +16,10 @@ fi
 . ./namelist_exp.sh
 . ./${COMPUTER}.env
 . ./${COMPUTER}.header
-. ./${COMPUTER}.module;  module list -t > ${COMPDIR}/croco/module_list_${COMPUTER}.txt
+. ./${COMPUTER}.module
+
+echo "modules loaded on ${COMPUTER} :" > ${COMPDIR}/croco/module_list_compile.txt
+module list -t >> ${COMPDIR}/croco/module_list_compile.txt 2>&1
 
 
 if [ -d ${COMPDIR}/croco ] ; then
@@ -101,6 +104,7 @@ if ( ! [ -d ${CROCO_LAST} ] ) || ( [ -d ${CROCO_LAST} ] && [ "${test_diff}" != "
       echo "----------------------------------------------------------------------"
       echo " Differences with the last compilation :"       
       echo "----------------------------------------------------------------------"
+      echo "Diff between croco & ${CROCO_LAST} dir"
       echo "${test_diff}"
       echo ""
 #     echo "   sav old compile dir in ${CROCO_LAST_OLD}"
@@ -115,7 +119,8 @@ if ( ! [ -d ${CROCO_LAST} ] ) || ( [ -d ${CROCO_LAST} ] && [ "${test_diff}" != "
    echo "*****************************************************************"
    echo "*                         MODULES                               *"
    echo "*****************************************************************"
-   module list -l
+   echo "modules loaded on ${COMPUTER} :"
+   module list -t  2>&1
    echo ""
 
    if [ -d ${CROCO_SOURCE} ] ; then

scripts/job.base.sh

@@ -36,6 +36,8 @@ mkdir -p ${OUTDIR}
 mkdir -p ${RSTDIR_OUT}
 
 
+echo "modules loaded on ${COMPUTER} :" > ${JOBDIR}/module_list_exec.txt
+module list -t >> ${JOBDIR}/module_list_exec.txt 2>&1
 
 #===============================================================================
 #  Step0 :  jobid info at start time

scripts/namelist_exp.sh

@@ -6,7 +6,9 @@
 #-------------------------------------------------------------------------------
 #  Which Computer?
 #-------------------------------------------------------------------------------
+# IRENE / IRENE-AMD / JEANZAY
 
+# If IRENE , choose AMD_Rome or Intel_SkilakeMD or Intel Processor
 export NODE=''      # for IRENE  (core Intel_Skylake)
 #export NODE='-AMD'  # for IRENE-AMD  (core AMD_Rome)
 
@@ -19,12 +21,14 @@ elif [ `hostname  |cut -c 1-8` == "jean-zay" ]; then
    export COMPUTER="JEANZAY"
 fi
 
+
 #-------------------------------------------------------------------------------
 #  Config : Ocean - Atmosphere - Coupling
 #-------------------------------------------------------------------------------
 export CONFIG=`pwd | awk -F"/" '{print $(NF-3)}'`
 export TITLE_EXP="Config Afrique du Sud  ASAP S. Pous  ("`date "+%d %m %Y"`" on ${COMPUTER})"
-printf "\n   ${TITLE_EXP}  \n\n"
+printf "\n\n   ${TITLE_EXP}  \n\n"
+
 
 #-------------------------------------------------------------------------------
 #  Debug Modes
@@ -34,7 +38,8 @@ export MODE_TEST=true   #  for running different tests in the same exp with diff
 #  export TEST_NAME="_remake_ASAP2020_KPP_Steph_Y1985M1_003_JEANZAY_12x10x2_2tsp_optim"
 #  export TEST_NAME="_remake_ASAP2020_KPP_Steph_Y1985M1_004_JEANZAY_12x10x2_2tsp_optim_MY_SRC_v2.00"
 #  export TEST_NAME="_croco+agrif+pisces_01_rst_nc4par_print_xios_2ts_12x10x1+24_debug_JEANZAY"
-   export TEST_NAME="_croco+agrif+pisces_02_rst_nc4par_print_xios_2ts_12x10x1+24_optim_JEANZAY"
+#  export TEST_NAME="_croco+agrif+pisces_02_rst_nc4par_print_xios_2ts_20x18x1+24_optim_IRENE-AMD"
+   export TEST_NAME="_croco+agrif+pisces_03_rst_nc4par_print_xios_2ts_20x18x1+24_optim_${COMPUTER}"
 
 export TEST_SCRIPT=false       #  the job is not submited     
 
@@ -48,38 +53,20 @@ export TEST_FEW_NTIMES=true    #  replace the job timestep number by DNTIMES
 export USE_OCE=true
    export MODEL_OCE="croco"
 
-#  Time Step
+###  Time Step
 export TSP_OCE=1800
 export TSP_ICE=$(( 3 * $TSP_OCE ))
 
-#  Output frequency
-if [ "${MODEL_OCE}" == "croco" ] ; then 
-
-#  export out_list="avg his diabio_avg"
-   export out_list="avg diabio_avg"
-
-   export freq_out='day'
-   export freq_his=20000;
-   export freq_avg=5;
-   export freq_diabio_avg=5
-
-#  export freq_out='tsp'
-#  export freq_his=20000
-#  export freq_avg=15
-#  export freq_diabio_avg=15
-
-fi
-
-# AGRIF
+### AGRIF
 export AGRIFZ=1        #   0 for no AGRIF , n for n grids
    export USE_AGRIF_2WAY=true   #  1-WAY by default or Activate 2-WAY nesting (update parent solution by child solution)
 	#   WARNING  :  if AGRIFZ=0 et USE_AGRIF_2WAY=true...   compilation error?? ... wetdry_.f(282): error #6404:
 
-# BIOLOGY
+### BIOLOGY
 export USE_BIOLOGY=true
    export USE_PISCES=true
 
-# MPI
+### MPI
 export USE_MPI=true 
 if [ ${COMPUTER} == "JEANZAY" ] ; then
    export NPROC_X=12; export NPROC_Y=10; export CPU_PER_TASK=1
@@ -89,18 +76,11 @@ elif [ ${COMPUTER} == "IRENE-AMD" ] ; then
    export NPROC_X=20; export NPROC_Y=18; export CPU_PER_TASK=1
 fi
 
-# OpenMP
+### OpenMP
 export USE_OPENMP=false 
    export NPP=4
-
-# XIOS
-# Modify the xml files in the param_inputs directory to select the fields and the frequency of the outputs
-export USE_XIOS=true 
-   export XIOS_VERSION="XIOS_X64_${COMPUTER}_trunk_r2219-2021_09_06"
-   export NXIOS=24 #4
-   export NXIOSper=-1   # just for nemo?  attache/detache? 
  
-# Time CPU run
+### Time CPU run
 if [ ${COMPUTER} == "JEANZAY" ] ; then
    export MAX_CPU_TIME="00:30:00"; export QOS="qos_cpu-dev" #  2h        | 128 nodes = 5120 cores
 #  export MAX_CPU_TIME="03:00:00"; export QOS="qos_cpu-t3"  #  20h       | 512 nodes = 20480 cores     (default)
@@ -113,6 +93,30 @@ elif [ ${COMPUTER} == "IRENE-AMD" ] ; then
 #  export MAX_CPU_TIME="36000"   #   10 heures
 fi
  
+### XIOS
+# Modify the xml files in the param_inputs directory to select the fields and the frequency of the outputs
+export USE_XIOS=true 
+   export XIOS_VERSION="XIOS_X64_${COMPUTER}_trunk_r2219-2021_09_06"
+   export NXIOS=24 #4
+   export NXIOSper=-1   # just for nemo?  attache/detache? 
+
+#  if USE_XIOS=false
+# Modify the namelist.base_oce (croco.in) files in the param_inputs directory to select the fields and the frequency of the outputs
+if [ "${MODEL_OCE}" == "croco" ] ; then 
+#  export out_list="avg his diabio_avg"
+   export out_list="avg diabio_avg"
+
+#  export freq_out='day'
+#  export freq_his=20000;
+#  export freq_avg=31;
+#  export freq_diabio_avg=31
+
+   export freq_out='tsp'
+   export freq_his=20000
+   export freq_avg=15
+   export freq_diabio_avg=15
+fi
+
 # files to save in ascii jobdir after running 
 if [ "${MODEL_OCE}" == "croco" ] ; then 
    export PUT_FILES_OCE="croco.in* out_run.txt croco_cpp.txt listing*"
@@ -121,6 +125,7 @@ if [ "${MODEL_OCE}" == "croco" ] ; then
    [ ${AGRIFZ} -ge 1 ] && { export PUT_FILES_OCE="${PUT_FILES_OCE} AGRIF_FixedGrids.in"; }
 fi
 
+
 #-------------------------------------------------------------------------------
 # Atmospheric model
 #-------------------------------------------------------------------------------
@@ -135,6 +140,7 @@ if [ "${MODEL_ATM}" == "wrf" ] ; then
    export PUT_FILES_ATM="namelist.input rsl.error.0000 rsl.out.000"
 fi
 
+
 #-------------------------------------------------------------------------------
 # Coupler
 #-------------------------------------------------------------------------------
@@ -143,6 +149,7 @@ export USE_CPL=${USE_ATM} && ${USE_OCE}
 #  Time Step
 export CPL_FREQ=7200
 
+
 #-------------------------------------------------------------------------------
 #  Various
 #-------------------------------------------------------------------------------
@@ -152,6 +159,7 @@ export TRDMLD=0
 export DATE1=`date "+%Y%m%d-%H_%M_%S"`
 export OLD="old_${DATE1}"
 
+
 #-------------------------------------------------------------------------------
 #  Name and directory of the experiment (from the path of the experiment)
 #-------------------------------------------------------------------------------
@@ -159,6 +167,7 @@ export EXP_NAME=`pwd | awk -F"/" '{print $(NF-1)}'`
 PWD1=$(pwd)
 export    EXPDIR=${PWD1%/*}
 
+
 #-------------------------------------------------------------------------------
 #  Directories
 #-------------------------------------------------------------------------------
@@ -169,12 +178,18 @@ export     PARAMDIR=${EXPDIR}/param_inputs
 
 ${USE_XIOS} && export XIOS_DIR="${COMPDIR}/xios/${XIOS_VERSION}"
 
+
 #-------------------------------------------------------------------------------
 #  Calendar computation
 #-------------------------------------------------------------------------------
+#  CalendarType : julian / greg / progreg / 30d / noleap 
+#                 Select in caltools.sh
+#                 Doc in caldat.sh header
+
 cd ${SCRIPTDIR}
-   . ./caltools.sh
-cd - 
+   . ./caltools.sh 
+cd - >/dev/null 2>&1
+
 
 #------------------------------------------------------------------------------
 #  Names
@@ -189,3 +204,4 @@ export    ASCII_DIR_IMG_ROOT="init"
 export    ASCII_DIR_IMG="${ASCII_DIR_IMG_ROOT}${TEST_NAME}" 
 export    SAV_OLD_PROD="production_${OLD}"
 
+

scripts/submitjob.sh

@@ -92,11 +92,12 @@ sed -e "s/<exp>/${ROOT_NAME_1}/g" \
     -e "s/<mpi_task>/$(( ${NPROC_X} * ${NPROC_Y} * ${CPU_PER_TASK} + ${NXIOS2} ))/" \
     -e "s/<cpu_per_task>/${CPU_PER_TASK}/" \
     -e "s/<max_cpu_time>/${MAX_CPU_TIME}/" \
-    ./${COMPUTER}.header > HEADER_tmp1
+    ./${COMPUTER}.header > HEADER_tmp
 
 if [ ${COMPUTER} == "JEANZAY" ] ; then
    sed -e "s/<qos>/${QOS}/" \
-    HEADER_tmp1 > HEADER_tmp
+    HEADER_tmp > HEADER_tmp1
+    mv -f HEADER_tmp1 HEADER_tmp
 fi
 
     cat HEADER_tmp ./${COMPUTER}.module ./common_definitions.sh ./namelist_exp.sh ./namelist_period.sh ./${COMPUTER}.env job.base.sh > ${JOBDIR_ROOT}/${jobname}