Commit 4556ddf7 authored by HOURDIN Christophe's avatar HOURDIN Christophe
Browse files

ASAP2 croco+agrif+pisces ok on IRENE & IRENE-AMD with xios & nc4par for restart

parent cd5e715b
......@@ -6,7 +6,5 @@
#
# search MPMD (Multi Program Multi Data) for documentation
# (with irene.info for example)
<nproc_model_oce> ./<model_oce>
<nproc_xios> ./<xios_server>
......@@ -6,7 +6,5 @@
#
# search MPMD (Multi Program Multi Data) for documentation
# (with irene.info for example)
<nproc_model_oce> ./<model_oce>
<nproc_xios> ./<xios_server>
......@@ -6,7 +6,5 @@
#
# search MPMD (Multi Program Multi Data) for documentation
# (with irene.info for example)
0-<nproc_model_oce-1> ./<model_oce>
<nproc_model_oce>-<add_nproc_xios> ./<xios_server>
......@@ -57,7 +57,7 @@ if [ "${MODEL_OCE}" = "nemo" ] ; then
export EXEC="time -p mpirun -np 126 ./nemo.exe : -np 2 ./xios_server.exe"
elif [ "${MODEL_OCE}" = "croco" ] ; then
if ${USE_XIOS} ; then
export EXEC="time -p ccc_mprun --multi-prog ./xios_launch.file croco.in"
export EXEC="time -p ccc_mprun -E-K1 -f ./xios_launch.file"
else
export EXEC="time -p ccc_mprun ./croco_${COMPUTER}.exe croco.in"
fi
......
......@@ -24,7 +24,7 @@
# #MSUB -N 4 # number of nodes to use (better to use -c option to reserve more memory)
#MSUB -x # requests exclusive usage of allocated nodes.
#MSUB -A gen1140
#MSUB -A gen6895
#MSUB -m work,scratch,store
#SUB -Q test # Queue test with 1800 s max for time
......
module purge
#!/bin/bash
module load flavor/buildmpi/openmpi/4.0
module load intel/20.0.0
#module load intel/19.0.5.281
module load mpi/openmpi/4.0.3
module load flavor/buildcompiler/intel/20
#module load flavor/buildcompiler/intel/19
module load flavor/hdf5/parallel
module load netcdf-fortran/4.4.4
module load netcdf-c/4.6.0
module load hdf5/1.8.20
echo ""
echo "module purge..."
module purge >/dev/null 2>&1
# from XIOS_X64_IRENE-AMD_trunk_r2219-2021_09_06/arch/arch-X64_IRENE-AMD.env
# module load flavor/hdf5/parallel
# module load netcdf-fortran/4.4.4
# module load hdf5/1.8.20
# module load gnu
# module load boost
# module load blitz
# module load feature/bridge/heterogenous_mpmd
for module in "flavor/hdf5/parallel" "netcdf-fortran/4.4.4" "hdf5/1.8.20" "gnu"
do
echo ""
echo "LOAD MODULE ${module}"
module load ${module}
done
echo ""
#!/bin/bash
echo ""
echo "module purge..."
module purge >/dev/null 2>&1
# From Seb 02/08/2021 see $HOME/loadintel.sh & $HOME/loadgcc.sh
module load intel/19.0.5.281
module load mpi/openmpi/4.0.2
module load flavor/hdf5/parallel
module load flavor/buildmpi/openmpi/4.0
module load netcdf-c/4.6.0
module load netcdf-fortran/4.4.4
module load hdf5/1.8.20
module load nco/4.9.1
module load cdo/1.9.5
module load ncview/2.1.7
module load boost/1.69.0
module load blitz/0.10
module load fftw3/mkl/19.0.5.281 # for paraver
module load c++/gnu/7.3.0
module load c/gnu/7.3.0
# from XIOS_X64_IRENE_trunk_r2219-2021_09_06/arch/arch-X64_IRENE.env
# module load flavor/hdf5/parallel
# module load netcdf-fortran/4.4.4
# module load hdf5/1.8.20
# module load gnu/7.3.0
# module load boost
# module load blitz
# module load feature/bridge/heterogenous_mpmd
#module purge
#module load flavor/buildcompiler/intel/19
#module load mpi/openmpi/2.0.4
# from XIOS
#module load flavor/hdf5/parallel
#module load netcdf-fortran/4.4.4
#module load netcdf-c/4.6.0
#module load hdf5/1.8.20
for module in "flavor/hdf5/parallel" "netcdf-fortran/4.4.4" "hdf5/1.8.20" "gnu/7.3.0"
do
echo ""
echo "LOAD MODULE ${module}"
module load ${module}
done
echo ""
#!/bin/bash
echo ""
echo "module purge..."
module purge >/dev/null 2>&1
# from XIOS_X64_JEANZAY_trunk_r2219-2021_09_06/arch/arch-X64_JEANZAY.env
# module purge
# module load gcc/9.1.0
# module load intel-all
# module load gcc/9.1.0
......@@ -8,20 +12,12 @@
# module load netcdf/4.7.2-mpi
# module load netcdf-fortran/4.5.2-mpi
# from XIOS_X64_JEANZAY_v2.5_r2152-2021_06_10/arch/arch-X64_JEANZAY.env
# module purge
# module load intel-compilers/19.0.4 intel-mkl/19.0.4 intel-mpi/19.0.4
# module load hdf5/1.10.5-mpi
# module load netcdf/4.7.2-mpi
# module load netcdf-fortran/4.5.2-mpi
module purge
module load intel-compilers/19.0.4
module load intel-mpi/19.0.4
module load netcdf-fortran/4.5.2-mpi
module load netcdf/4.7.2-mpi
module load hdf5/1.10.5-mpi
for module in "intel-compilers/19.0.4" "intel-mpi/19.0.4" "netcdf-fortran/4.5.2-mpi" "netcdf/4.7.2-mpi" "hdf5/1.10.5-mpi"
do
echo ""
echo "LOAD MODULE ${module}"
module load ${module}
done
echo ""
......@@ -3,8 +3,12 @@
#===============================================================================
# Print
#===============================================================================
printf "\n Modules list:\n"
module list -l
echo ""
echo "modules loaded on ${COMPUTER} :"
module list -t 2>&1
echo ""
echo ""
echo ""
printf "\n Period of Experiment:\n"
printf "%12s%10d%22s%14d%17s\n" "BEGINING:" ${DATE_BEGIN_EXP} "(in ${CALTYPE} calendar)" ${JDAY_BEGIN_EXP} "(in Julian days)"
......@@ -40,3 +44,4 @@ printf "%-15s : %-80s\n\n" "EXEDIR" "${EXEDIR}"
printf "%-15s : %-80s\n\n" "LD_LIBRARY_PATH" "${LD_LIBRARY_PATH}"
printf "\n\n"
......@@ -16,7 +16,10 @@ fi
. ./namelist_exp.sh
. ./${COMPUTER}.env
. ./${COMPUTER}.header
. ./${COMPUTER}.module; module list -t > ${COMPDIR}/croco/module_list_${COMPUTER}.txt
. ./${COMPUTER}.module
echo "modules loaded on ${COMPUTER} :" > ${COMPDIR}/croco/module_list_compile.txt
module list -t >> ${COMPDIR}/croco/module_list_compile.txt 2>&1
if [ -d ${COMPDIR}/croco ] ; then
......@@ -101,6 +104,7 @@ if ( ! [ -d ${CROCO_LAST} ] ) || ( [ -d ${CROCO_LAST} ] && [ "${test_diff}" != "
echo "----------------------------------------------------------------------"
echo " Differences with the last compilation :"
echo "----------------------------------------------------------------------"
echo "Diff between croco & ${CROCO_LAST} dir"
echo "${test_diff}"
echo ""
# echo " sav old compile dir in ${CROCO_LAST_OLD}"
......@@ -115,7 +119,8 @@ if ( ! [ -d ${CROCO_LAST} ] ) || ( [ -d ${CROCO_LAST} ] && [ "${test_diff}" != "
echo "*****************************************************************"
echo "* MODULES *"
echo "*****************************************************************"
module list -l
echo "modules loaded on ${COMPUTER} :"
module list -t 2>&1
echo ""
if [ -d ${CROCO_SOURCE} ] ; then
......
......@@ -36,6 +36,8 @@ mkdir -p ${OUTDIR}
mkdir -p ${RSTDIR_OUT}
echo "modules loaded on ${COMPUTER} :" > ${JOBDIR}/module_list_exec.txt
module list -t >> ${JOBDIR}/module_list_exec.txt 2>&1
#===============================================================================
# Step0 : jobid info at start time
......
......@@ -6,7 +6,9 @@
#-------------------------------------------------------------------------------
# Which Computer?
#-------------------------------------------------------------------------------
# IRENE / IRENE-AMD / JEANZAY
# If IRENE , choose AMD_Rome or Intel_SkilakeMD or Intel Processor
export NODE='' # for IRENE (core Intel_Skylake)
#export NODE='-AMD' # for IRENE-AMD (core AMD_Rome)
......@@ -19,12 +21,14 @@ elif [ `hostname |cut -c 1-8` == "jean-zay" ]; then
export COMPUTER="JEANZAY"
fi
#-------------------------------------------------------------------------------
# Config : Ocean - Atmosphere - Coupling
#-------------------------------------------------------------------------------
export CONFIG=`pwd | awk -F"/" '{print $(NF-3)}'`
export TITLE_EXP="Config Afrique du Sud ASAP S. Pous ("`date "+%d %m %Y"`" on ${COMPUTER})"
printf "\n ${TITLE_EXP} \n\n"
printf "\n\n ${TITLE_EXP} \n\n"
#-------------------------------------------------------------------------------
# Debug Modes
......@@ -34,7 +38,8 @@ export MODE_TEST=true # for running different tests in the same exp with diff
# export TEST_NAME="_remake_ASAP2020_KPP_Steph_Y1985M1_003_JEANZAY_12x10x2_2tsp_optim"
# export TEST_NAME="_remake_ASAP2020_KPP_Steph_Y1985M1_004_JEANZAY_12x10x2_2tsp_optim_MY_SRC_v2.00"
# export TEST_NAME="_croco+agrif+pisces_01_rst_nc4par_print_xios_2ts_12x10x1+24_debug_JEANZAY"
export TEST_NAME="_croco+agrif+pisces_02_rst_nc4par_print_xios_2ts_12x10x1+24_optim_JEANZAY"
# export TEST_NAME="_croco+agrif+pisces_02_rst_nc4par_print_xios_2ts_20x18x1+24_optim_IRENE-AMD"
export TEST_NAME="_croco+agrif+pisces_03_rst_nc4par_print_xios_2ts_20x18x1+24_optim_${COMPUTER}"
export TEST_SCRIPT=false # the job is not submited
......@@ -48,38 +53,20 @@ export TEST_FEW_NTIMES=true # replace the job timestep number by DNTIMES
export USE_OCE=true
export MODEL_OCE="croco"
# Time Step
### Time Step
export TSP_OCE=1800
export TSP_ICE=$(( 3 * $TSP_OCE ))
# Output frequency
if [ "${MODEL_OCE}" == "croco" ] ; then
# export out_list="avg his diabio_avg"
export out_list="avg diabio_avg"
export freq_out='day'
export freq_his=20000;
export freq_avg=5;
export freq_diabio_avg=5
# export freq_out='tsp'
# export freq_his=20000
# export freq_avg=15
# export freq_diabio_avg=15
fi
# AGRIF
### AGRIF
export AGRIFZ=1 # 0 for no AGRIF , n for n grids
export USE_AGRIF_2WAY=true # 1-WAY by default or Activate 2-WAY nesting (update parent solution by child solution)
# WARNING : if AGRIFZ=0 et USE_AGRIF_2WAY=true... compilation error?? ... wetdry_.f(282): error #6404:
# BIOLOGY
### BIOLOGY
export USE_BIOLOGY=true
export USE_PISCES=true
# MPI
### MPI
export USE_MPI=true
if [ ${COMPUTER} == "JEANZAY" ] ; then
export NPROC_X=12; export NPROC_Y=10; export CPU_PER_TASK=1
......@@ -89,18 +76,11 @@ elif [ ${COMPUTER} == "IRENE-AMD" ] ; then
export NPROC_X=20; export NPROC_Y=18; export CPU_PER_TASK=1
fi
# OpenMP
### OpenMP
export USE_OPENMP=false
export NPP=4
# XIOS
# Modify the xml files in the param_inputs directory to select the fields and the frequency of the outputs
export USE_XIOS=true
export XIOS_VERSION="XIOS_X64_${COMPUTER}_trunk_r2219-2021_09_06"
export NXIOS=24 #4
export NXIOSper=-1 # just for nemo? attache/detache?
# Time CPU run
### Time CPU run
if [ ${COMPUTER} == "JEANZAY" ] ; then
export MAX_CPU_TIME="00:30:00"; export QOS="qos_cpu-dev" # 2h | 128 nodes = 5120 cores
# export MAX_CPU_TIME="03:00:00"; export QOS="qos_cpu-t3" # 20h | 512 nodes = 20480 cores (default)
......@@ -113,6 +93,30 @@ elif [ ${COMPUTER} == "IRENE-AMD" ] ; then
# export MAX_CPU_TIME="36000" # 10 heures
fi
### XIOS
# Modify the xml files in the param_inputs directory to select the fields and the frequency of the outputs
export USE_XIOS=true
export XIOS_VERSION="XIOS_X64_${COMPUTER}_trunk_r2219-2021_09_06"
export NXIOS=24 #4
export NXIOSper=-1 # just for nemo? attache/detache?
# if USE_XIOS=false
# Modify the namelist.base_oce (croco.in) files in the param_inputs directory to select the fields and the frequency of the outputs
if [ "${MODEL_OCE}" == "croco" ] ; then
# export out_list="avg his diabio_avg"
export out_list="avg diabio_avg"
# export freq_out='day'
# export freq_his=20000;
# export freq_avg=31;
# export freq_diabio_avg=31
export freq_out='tsp'
export freq_his=20000
export freq_avg=15
export freq_diabio_avg=15
fi
# files to save in ascii jobdir after running
if [ "${MODEL_OCE}" == "croco" ] ; then
export PUT_FILES_OCE="croco.in* out_run.txt croco_cpp.txt listing*"
......@@ -121,6 +125,7 @@ if [ "${MODEL_OCE}" == "croco" ] ; then
[ ${AGRIFZ} -ge 1 ] && { export PUT_FILES_OCE="${PUT_FILES_OCE} AGRIF_FixedGrids.in"; }
fi
#-------------------------------------------------------------------------------
# Atmospheric model
#-------------------------------------------------------------------------------
......@@ -135,6 +140,7 @@ if [ "${MODEL_ATM}" == "wrf" ] ; then
export PUT_FILES_ATM="namelist.input rsl.error.0000 rsl.out.000"
fi
#-------------------------------------------------------------------------------
# Coupler
#-------------------------------------------------------------------------------
......@@ -143,6 +149,7 @@ export USE_CPL=${USE_ATM} && ${USE_OCE}
# Time Step
export CPL_FREQ=7200
#-------------------------------------------------------------------------------
# Various
#-------------------------------------------------------------------------------
......@@ -152,6 +159,7 @@ export TRDMLD=0
export DATE1=`date "+%Y%m%d-%H_%M_%S"`
export OLD="old_${DATE1}"
#-------------------------------------------------------------------------------
# Name and directory of the experiment (from the path of the experiment)
#-------------------------------------------------------------------------------
......@@ -159,6 +167,7 @@ export EXP_NAME=`pwd | awk -F"/" '{print $(NF-1)}'`
PWD1=$(pwd)
export EXPDIR=${PWD1%/*}
#-------------------------------------------------------------------------------
# Directories
#-------------------------------------------------------------------------------
......@@ -169,12 +178,18 @@ export PARAMDIR=${EXPDIR}/param_inputs
${USE_XIOS} && export XIOS_DIR="${COMPDIR}/xios/${XIOS_VERSION}"
#-------------------------------------------------------------------------------
# Calendar computation
#-------------------------------------------------------------------------------
# CalendarType : julian / greg / progreg / 30d / noleap
# Select in caltools.sh
# Doc in caldat.sh header
cd ${SCRIPTDIR}
. ./caltools.sh
cd -
. ./caltools.sh
cd - >/dev/null 2>&1
#------------------------------------------------------------------------------
# Names
......@@ -189,3 +204,4 @@ export ASCII_DIR_IMG_ROOT="init"
export ASCII_DIR_IMG="${ASCII_DIR_IMG_ROOT}${TEST_NAME}"
export SAV_OLD_PROD="production_${OLD}"
......@@ -92,11 +92,12 @@ sed -e "s/<exp>/${ROOT_NAME_1}/g" \
-e "s/<mpi_task>/$(( ${NPROC_X} * ${NPROC_Y} * ${CPU_PER_TASK} + ${NXIOS2} ))/" \
-e "s/<cpu_per_task>/${CPU_PER_TASK}/" \
-e "s/<max_cpu_time>/${MAX_CPU_TIME}/" \
./${COMPUTER}.header > HEADER_tmp1
./${COMPUTER}.header > HEADER_tmp
if [ ${COMPUTER} == "JEANZAY" ] ; then
sed -e "s/<qos>/${QOS}/" \
HEADER_tmp1 > HEADER_tmp
HEADER_tmp > HEADER_tmp1
mv -f HEADER_tmp1 HEADER_tmp
fi
cat HEADER_tmp ./${COMPUTER}.module ./common_definitions.sh ./namelist_exp.sh ./namelist_period.sh ./${COMPUTER}.env job.base.sh > ${JOBDIR_ROOT}/${jobname}
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment