Commit 53b006b8 authored by Patryk Kiepas's avatar Patryk Kiepas 🐦
Browse files

Merge remote-tracking branch 'origin/devel2master'

parents f0ddb6f3 08ffdb8a
ppsrc/
inc/
flags/
done/
bin/
bld/
etc/
lib/
obj/
tmp/
DySL/*/*.k90
DySL/kernels_extracted.jin
src/base/signature.f90
fcm_env.ksh
config.fcm
arch.*
.cache/
.void_file
*.lock
*.so
*.nc
*.log
*.k90
*~
._*
\#*#
.DS_Store
rebuild
Python/test/grids*
Python/test/mesh_information
stages:
- build
- check
- data_validation
- deploy
variables:
GIT_CLONE_PATH: $CI_BUILDS_DIR/$CI_PROJECT_PATH-$CI_CONCURRENT_PROJECT_ID
XIOS_REV: 2322
EXTRA_ARCH: test/arch
SCRIPTS: test/bash
COMP: gnu
CPU: skylake
PARALLEL: mpi_omp
IO: with_xios
# Find reference job ID here: https://gitlab.in2p3.fr/ipsl/projets/dynamico/dynamico/-/jobs
DCMIP41_DATA_VALIDATION_REF_JOB: 352499
#========================================= BUILD ========================================#
.build-script: &build-script
- mkdir -p tmp
- TARGET=$CPU ./make_icosa -arch_path $EXTRA_ARCH -arch $COMP -parallel $PARALLEL -$IO -job 8 2>&1 | tee tmp/$COMP.log
- grep -q 'Build command finished' tmp/$COMP.log
.build-job: &build-job
stage: build
image: gitlab-registry.in2p3.fr/ipsl/projets/dynamico/dockerimages/$COMP/xios:$XIOS_REV
script:
- *build-script
artifacts:
paths:
- tmp/
build-gnu-mpi-omp:
<<: *build-job
build-gnu-mpi:
<<: *build-job
variables: { COMP: gnu, PARALLEL: mpi, CPU: native }
tags:
- ccosvmse0035@gitlab.in2p3.fr
artifacts:
paths:
- bin/
- lib/
- arch.*
- tmp/
build-intel:
<<: *build-job
variables: { COMP: intel }
build-nvidia:
<<: *build-job
variables: { COMP: nvidia , PARALLEL: mpi , CPU: cc70}
build-noio:
<<: *build-job
variables: { IO: no_io , PARALLEL: none}
build-xml:
<<: *build-job
only:
- trunk2master
- converge/trunk
variables: { COMP: gnu , PARALLEL: none , IO: no_io }
script:
- rm -rf tools/tracer_parser/* && cp -p $SCRIPTS/fortran2xml /opt/ipsl/bin
- mkdir -p tmp
- ./make_icosa -arch_path $EXTRA_ARCH -arch xmp -parallel $PARALLEL -$IO -job 8 2>&1 | tee tmp/$COMP.log
- grep -q 'Build command finished' tmp/$COMP.log
#========================================= CHECK ========================================#
check-dcmip41:
stage: check
image: gitlab-registry.in2p3.fr/ipsl/projets/dynamico/dockerimages/$COMP/xios:$XIOS_REV
needs: ["build-gnu-mpi"]
script:
- test/bash/run_mpi.sh 8
- apt update && apt -yqq install jupyter-notebook python3-netcdf4 python3-matplotlib jupyter-nbconvert texlive-fonts-recommended texlive-generic-recommended texlive-xetex
- cd rundir
- LD_LIBRARY_PATH="" jupyter nbconvert --to pdf --execute DCMIP41.ipynb
artifacts:
paths:
- rundir/
.check-python-common-configuration: &check-python-common-configuration
stage: check
image: gitlab-registry.in2p3.fr/ipsl/projets/dynamico/dockerimages/$COMP/xios:$XIOS_REV
needs: ["build-gnu-mpi"]
only:
- devel2master
- hotfix/devel/python
# Test any branch with `python` in its name
- /^.*python.*$/
.check-python-common-script: &check-python-common-script
- cp -pr $EXTRA_ARCH/*.dev arch/ && ./make_python dev && cd Python/test
- rm -rf fig_* && mkdir fig_RSW_2D fig_slice_GW_NH fig_DCMIP2008c5
# This test currently fails in Python 3 (caldyn_step returns NaN in the mass variable)
#- OMP_NUM_THREADS=4 ./python.sh serial py/DCMIP2008c5.py
- OMP_NUM_THREADS=1 ./python.sh serial py/slice_GW_NH.py
- OMP_NUM_THREADS=4 ./python.sh serial py/RSW_2D.py
- jobs/gen_mpi_mesh_info.sh
check-python2:
<<: *check-python-common-configuration
before_script:
- apt-get update -yqq && apt-get install -yqq --no-install-recommends python2.7 python2.7-dev python-pip python-netcdf4 python-numba
- python2.7 -m pip install setuptools wheel
- python2.7 -m pip install future numpy Cython jinja2 matplotlib scipy mpi4py
# The forced reinstallation is needed, otherwise numpy has broken links to BLAS procedures
- python2.7 -m pip install --upgrade --force-reinstall numpy
# By default, `python` points to python3, we need to reset it to `python2.7`
- update-alternatives --install /usr/bin/python python /usr/bin/python2.7 30
script:
- *check-python-common-script
- OMP_NUM_THREADS=4 ./python.sh serial py/DCMIP2008c5.py
check-python3:
<<: *check-python-common-configuration
before_script:
- apt-get update -yqq && apt-get install -yqq --no-install-recommends python3-matplotlib python3-numba
- python3 -m pip install future mpi4py
script:
- *check-python-common-script
.check-jean-zay: # remove dot to activate job
stage: check
allow_failure: true
tags:
- jean-zay
script:
- hostname
- whoami
- uname -a
- ls -l
- git branch -a
- pwd
#===================================== DATA VALIDATION ===================================#
validate-dcmip41:
stage: data_validation
image: conda/miniconda3
needs: ["check-dcmip41"]
script:
- conda install -c conda-forge nccmp pandas -y
# Move current artifacts from the check-dcmip41
- mv rundir/ rundir-current/
# Download the artifact from the reference check-dcmip41 run
- curl -O "https://gitlab.in2p3.fr/api/v4/projects/ipsl%2Fprojets%2Fdynamico%2Fdynamico/jobs/${DCMIP41_DATA_VALIDATION_REF_JOB}/artifacts/rundir/output_dcmip2016_regular.nc"
- mkdir rundir-reference/
- mv output_dcmip2016_regular.nc rundir-reference/
# nccmp takes several flags: -d compares data values, -f forces comparison up to the end, -q means quite mode,
# -S shows comparison statistics, and -N treats all NaN as equal values
- nccmp rundir-current/output_dcmip2016_regular.nc rundir-reference/output_dcmip2016_regular.nc -dfqSN > DCMIP41_data_validation.txt || echo "nccmp found some differences"
- cat DCMIP41_data_validation.txt
- python3 gitlab-ci/data-validation/analyse_nccmp_result.py DCMIP41_data_validation.txt --tolerance_def gitlab-ci/data-validation/comparison-tolerance.json
artifacts:
paths:
- DCMIP41_data_validation.txt
#========================================= DEPLOY ========================================#
pages:
stage: deploy
only:
- devel2master
variables:
REVISION: $CI_COMMIT_REF_NAME:$CI_COMMIT_SHORT_SHA
image: gitlab-registry.in2p3.fr/ipsl/projets/dynamico/dockerimages/$COMP/xios:$XIOS_REV
script:
- *build-script
- apt-get update && apt-get install -yqq doxygen graphviz
- doc/dox.sh $REVISION
artifacts:
paths:
- public/
expire_in: 1h
#!/bin/bash
function filter()
{
PASS=$1 ; shift
cpp -P -DPASS_$PASS $* -include ../macros.jin
}
function log()
{
echo $*
$*
}
function extract_dysl()
{
sed -n '/BEGIN_DYSL/,/END_DYSL/{//b;p}' compute_pvort_only.F90
}
function main()
{
VARIANT=$SRCDIR/$1
cd $VARIANT
rm -f *~ tmp/*
mkdir -p tmp
cd tmp
cp $SRCDIR/macros.jin . # jinja macros common to all variants
cpp -P -DPASS_JINJA ../macros.jin >> macros.jin # variant-specific macros
# first expand user-defined CPP and jinja macros
cat $SRCDIR/kernels_*.jin | cpp -P > caldyn_all.jin
python -u $ROOT/jinja caldyn_all.jin > caldyn_all.dysl
# then expand the code-generating CPP and jinja macros
cat caldyn_all.dysl | filter PRE1 | filter PRE2 >> macros.jin
python -u $ROOT/jinja macros.jin > caldyn_all.k90
# extract each kernel and apply post-jinja CPP macros
KERNELS=$(grep KERNEL caldyn_all.dysl | cpp -P -D'KERNEL(x)=x')
for NAME in $KERNELS ; do
cat caldyn_all.k90 | filter POST1 -DKERNEL_${NAME} | filter POST2 | sed '/^$/d' > ../${NAME}.k90
head ../${NAME}.k90
done
cd ..
grep ERROR *.k90
cd $VARIANT
wc *.k90
cd $ROOT
}
set +x
ROOT=$PWD
SRCDIR=$ROOT
grep FIXME *.jin
for x in $* ; do
echo "Generating code for variant : $x"
main $x
done
#define HEX_MASTER
#define HEX_GPU
#include "../hexagonal/macros.jin"
#define HEX_MASTER
#define HEX_SIMD
#include "../hexagonal/macros.jin"
#define HEX_OPENMP
#define HEX_SIMD
#include "../hexagonal/macros.jin"
#ifdef PASS_PRE1
#define _AND_ &&
#define _OR_ ||
#define _NOT_ !
#define IS_INNER_LAYER (_NOT_ (IS_BOTTOM_LEVEL _OR_ IS_TOP_LAYER))
#define IS_INNER_INTERFACE (_NOT_ (IS_BOTTOM_LEVEL _OR_ IS_TOP_INTERFACE))
#define CELL ij,LL
#define VERTEX1 ij+{{vertex1}},LL
#define VERTEX2 ij+{{vertex2}},LL
#define EDGE ij+{{edge}},LL
#define EDGE_TRISK ij+{{edge_trisk}},LL
#define DUAL_CELL ij+{{triangle}},LL
#define CELL1 ij,LL
#define CELL2 ij+{{neighbour}},LL
#define VERTEX {{vertex}},LL
#define SIGN {{sign}}
#define LE_DE le_de(ij+{{edge}})
#define WEE w_ee(ij+{{edge}},{{itrisk}})
#define DE de(ij+{{edge}})
#define LE le(ij+{{edge}})
#define RIV2 Riv2({{vertex}},{{vertex_dir}})
#define AI Ai(ij)
#define FV fv(ij+{{triangle}})
#define AV Av(ij+{{triangle}})
#endif
#ifdef PASS_PRE2
#ifdef HEX_OPENMP
#define BARRIER !$OMP BARRIER
#else
#define BARRIER
#endif
#define KERNEL(name) {% call define_kernel(#name) %}
#define FORALL_CELLS(...) {% call forall_cells(__VA_ARGS__) %}
#define FORALL_CELLS_EXT(...) {% call forall_cells_ext(__VA_ARGS__) %}
#define ON_PRIMAL {% call(forall_edges) on_hexagons() %}
#define ON_DUAL {% call(triangle,forall_edges,forall_vertices) on_triangles() %}
#define ON_EDGES {% call(edge,neighbour,sign,vertex1,vertex2,forall_trisk) on_edges() %}
#define FORALL_EDGES {% call(edge,sign,vertex1,vertex2) forall_edges() %}
#define FORALL_VERTICES {% call(vertex,vertex_dir) forall_vertices() %}
#define FORALL_TRISK {% call(itrisk,edge_trisk) forall_trisk() %}
#define END_BLOCK {% endcall %}
#define SEQUENCE_C0 {% call(body,at_level,ext) sequence() %}
#define SEQUENCE_C1 {% call(body,at_level,ext) sequence_ext() %}
#define SEQUENCE_E0 {% call(body,at_level,ext) sequence_edge() %}
#define PROLOGUE(level) {% call(edge,neighbour) at_level(level,ext) %}
#define BODY(range){% call(edge,neighbour) body(range,ext) %}
#define EPILOGUE(level){% call(edge,neighbour) at_level(level,ext) %}
#define CST_IF(condition, action) {{ cst_if(#condition, #action) }}
#define CST_IFTHEN(condition) {{flat}}if condition
#define CST_ELSEIF(condition) {{flat}}elif condition
#define CST_ELSE {{flat}}else
#define CST_ENDIF {{flat}}endif
#endif
#if PASS_JINJA
#ifdef HEX_OPENMP
{% set ll_begin, ll_beginp1 = 'll_begin', 'll_beginp1' %}
{% set ll_endm1, ll_end, ll_endp1 = 'll_endm1', 'll_end', 'll_endp1' %}
{% macro ij_omp(ext) -%}
ij_omp_begin{{ext}},ij_omp_end{{ext}}
{%- endmacro %}
#else
{% set ll_begin, ll_beginp1 = '1', '2' %}
{% set ll_endm1, ll_end, ll_endp1 = 'llm-1', 'llm', 'llm+1' %}
{% macro ij_omp(ext) -%}
ij_begin{{ext}},ij_end{{ext}}
{%- endmacro %}
#endif
{% macro sequence() -%}
{{ caller(body_primal,at_level_primal,'') }}
{%- endmacro %}
{% macro sequence_ext() -%}
{{ caller(body_primal,at_level_primal,'_ext') }}
{%- endmacro %}
{% macro at_level_primal(lev,ext) -%}
{{ define('LL', lev) }}
DIR_SIMD
ACC_SEQ1
DO ij={{ ij_omp(ext) }}
{{ caller() }}
END DO
{%- endmacro %}
{% macro body_primal(range,ext) -%}
{{ define('LL', 'l') }}
ACC_SEQ2
DO l = {{ range }}
DIR_SIMD
DO ij={{ ij_omp(ext) }}
{{ caller() }}
END DO
END DO
{%- endmacro %}
{% macro sequence_edge() -%}
{{ caller(body_edge,at_level_edge,'') }}
{%- endmacro %}
{% macro at_level_edge(lev,ext) -%}
{{ define('LL', lev) }}
DIR_SIMD
ACC_SEQ1
DO ij={{ ij_omp(ext) }}
{{ caller('u_right','t_right') }}
{{ caller('u_lup','t_lup') }}
{{ caller('u_ldown','t_ldown') }}
END DO
{%- endmacro %}
{% macro body_edge(range,ext) -%}
{{ define('LL', 'l') }}
ACC_SEQ2
DO l = {{ range }}
DIR_SIMD
DO ij={{ ij_omp(ext) }}
{{ caller('u_right','t_right') }}
{{ caller('u_lup','t_lup') }}
{{ caller('u_ldown','t_ldown') }}
END DO
END DO
{%- endmacro %}
{#---------------------- big macro generating nested DO loops -------------------------#}
{# specialized code is generated for l=1 and l=llm,llm+1 if necessary #}
{# start/end values for l are replaced by corresponding OpenMP values #}
{% macro define_levels(ll,kdown,kup,thecode) %}
{{ define('LL', ll) }}
{{ define_cppmacro('KDOWN','(ij,ll)', kdown) if 'KDOWN' in thecode }}
{{ define_cppmacro('KUP','(ij,ll)', kup) if 'KUP' in thecode }}
{%- endmacro %}
{% macro forall(start,end,ij_range,thecode) %}
{% set start_omp={'1':ll_begin , '2':ll_beginp1}[start] %}
{% set end_omp={'llm-1':ll_endm1 , 'llm':ll_end, 'llm+1':ll_endp1}[end] %}
{% set is_top_layer, is_top_inter = False, False %}
{{ define('IS_TOP_LAYER', '_FALSE_') }}
{{ define('IS_TOP_INTERFACE', '_FALSE_') }}
{% if 'IS_BOTTOM_LEVEL' in thecode or 'KDOWN' in thecode%}
{{ 'ERROR : using IS_BOTTOM_LEVEL in a loop starting at l=2' if start=='2' }}
{# the code in the loop checks whether l==1, for the sake of performance
we shall write special code for l=1 and start the loop at l=2 #}
IF ({{ll_begin}}==1) THEN
{{ define('IS_BOTTOM_LEVEL', '_TRUE_') }}
{{ define_levels('1', 'ij,1', 'ij,1', thecode) }}
DIR_SIMD
ACC_LOOP1
DO ij={{ij_range}}
{{ thecode }}
END DO
END IF
{% set start_omp=ll_beginp1 %}
{% endif %}
{% if 'IS_TOP_LAYER' in thecode %}
{{ 'ERROR : using _IS_TOP_LAYER_ in a loop ending at l=llm+1' if end=='llm+1' }}
{# the code checks whether l==llm, write special code for l=llm and end the loop at l=llm-1 #}
{% set end_omp, is_top_layer = ll_endm1, True %}
{% endif %}
{% if 'IS_TOP_INTERFACE' in thecode or 'KUP' in thecode %}
{# the code checks whether l==llm+1, write special code for l=llm+1 and end the loop at l=llm #}
{{ 'ERROR : using _IS_TOP_INTERFACE_ in a loop ending at l=llm' if end=='llm' }}
{% set end_omp, is_top_inter = ll_end, True %}
{% endif %}
{{ define('IS_BOTTOM_LEVEL', '_FALSE_') }}
{{ define_levels('l', 'ij,l-1', 'ij,l', thecode) }}
ACC_LOOP2
DO l = {{start_omp}}, {{end_omp}}
DIR_SIMD
DO ij={{ij_range}}
{{ thecode }}
END DO
END DO
{% if is_top_layer %}
IF({{ll_end}}==llm) THEN
{{ define('IS_TOP_LAYER', '_TRUE_') }}
{{ define_levels('llm', 'ij,llm-1', 'ij,llm', thecode) }}
DIR_SIMD
ACC_LOOP1
DO ij={{ij_range}}
{{ thecode }}
END DO
END IF
{% endif %}
{% if is_top_inter %}
IF({{ll_endp1}}==llm+1) THEN
{{ define('IS_TOP_INTERFACE', '_TRUE_') }}
{{ define_levels('llm+1', 'ij,llm', 'ij,llm', thecode) }}
DIR_SIMD
ACC_LOOP1
DO ij={{ij_range}}
{{ thecode }}
END DO
END IF
{% endif %}
{% endmacro %}
{#-------------------------------------------------------------------------------------#}
{% macro forall_cells(start='1', end='llm') -%}
{{ forall(start,end,'ij_begin, ij_end', caller()) }}
{% endmacro %}
{% macro forall_cells_ext(start='1', end='llm') -%}
{{ forall(start,end,'ij_begin_ext, ij_end_ext', caller()) }}
{% endmacro %}
{% macro forall_edges_hex() -%}
{{ caller('u_rup','ne_rup', 'z_rup','z_up') }}
{{ caller('u_lup','ne_lup', 'z_lup','z_up') }}
{{ caller('u_left','ne_left','z_lup','z_ldown') }}
{{ caller('u_ldown','ne_ldown', 'z_ldown','z_down') }}
{{ caller('u_rdown','ne_rdown', 'z_rdown','z_down') }}
{{ caller('u_right','ne_right', 'z_rup','z_rdown') }}
{%- endmacro %}
{% macro on_hexagons() -%}
{{ caller(forall_edges_hex) }}
{%- endmacro %}
{% macro forall_edges_zup() -%}
{{ caller('u_rup','ne_rup', 'TODO','TODO') }}
{{ caller('t_rup+u_left','ne_left', 'TODO','TODO') }}
{{ caller('u_lup','(-ne_lup)', 'TODO','TODO') }}
{%- endmacro %}
{% macro forall_edges_zdown() -%}
{{ caller('u_rdown','(-ne_rdown)', 'TODO', 'TODO') }}
{{ caller('t_ldown+u_right','ne_right', 'TODO', 'TODO') }}
{{ caller('u_ldown','ne_ldown', 'TODO', 'TODO') }}
{%- endmacro %}
{% macro forall_vertices_zup() -%}
{{ caller('ij','vup') }}
{{ caller('ij+t_rup','vldown') }}
{{ caller('ij+t_lup','vrdown') }}
{%- endmacro %}
{% macro forall_vertices_zdown() -%}
{{ caller('ij','vdown') }}
{{ caller('ij+t_ldown','vrup') }}
{{ caller('ij+t_rdown','vlup') }}
{%- endmacro %}
{% macro on_triangles() -%}
{{ caller('z_up',forall_edges_zup,forall_vertices_zup) }}
{{ caller('z_down',forall_edges_zdown,forall_vertices_zdown) }}
{%- endmacro %}
{% macro trisk_right() -%}
{{ caller('1', 'u_rup') }}
{{ caller('2', 'u_lup') }}
{{ caller('3', 'u_left') }}
{{ caller('4', 'u_ldown') }}
{{ caller('5', 'u_rdown') }}
{{ caller('6', 't_right+u_ldown') }}
{{ caller('7', 't_right+u_rdown') }}
{{ caller('8', 't_right+u_right') }}
{{ caller('9', 't_right+u_rup') }}
{{ caller('10', 't_right+u_lup') }}
{%- endmacro %}
{% macro trisk_lup() -%}
{{ caller('1', 'u_left') }}
{{ caller('2', 'u_ldown') }}
{{ caller('3', 'u_rdown') }}
{{ caller('4', 'u_right') }}
{{ caller('5', 'u_rup') }}
{{ caller('6', 't_lup+u_right') }}
{{ caller('7', 't_lup+u_rup') }}
{{ caller('8', 't_lup+u_lup') }}
{{ caller('9', 't_lup+u_left') }}
{{ caller('10', 't_lup+u_ldown') }}
{%- endmacro %}
{% macro trisk_ldown() -%}
{{ caller('1', 'u_rdown') }}
{{ caller('2', 'u_right') }}
{{ caller('3', 'u_rup') }}
{{ caller('4', 'u_lup') }}
{{ caller('5', 'u_left') }}
{{ caller('6', 't_ldown+u_lup') }}
{{ caller('7', 't_ldown+u_left') }}
{{ caller('8', 't_ldown+u_ldown') }}
{{ caller('9', 't_ldown+u_rdown') }}
{{ caller('10', 't_ldown+u_right') }}
{%- endmacro %}
{% macro on_edges() -%}
{{ caller('u_right','t_right','ne_right','z_rdown','z_rup',trisk_right) }}
{{ caller('u_lup','t_lup','ne_lup','z_up','z_lup', trisk_lup) }}
{{ caller('u_ldown','t_ldown','ne_ldown','z_ldown','z_down', trisk_ldown) }}
{%- endmacro %}
{% set llm = 'llm' %}
#endif
#ifdef PASS_POST1
#define _TRUE_ (0==0)
#define _FALSE_ (0==1)
#define CELL ij,LL
#ifdef HEX_SIMD
#define DIR_SIMD !DIR$ SIMD
#else
#define DIR_SIMD
#endif
#ifdef HEX_GPU
#define ACC_LOOP2 !$acc parallel loop async collapse(2)
#define ACC_LOOP1 !$acc parallel loop async
#define ACC_SEQ2 !$acc kernels async
#define ACC_SEQ1 !$acc parallel loop
#else
#define ACC_LOOP2
#define ACC_SEQ2
#define ACC_LOOP1
#define ACC_SEQ1
#endif
#endif
#ifdef PASS_POST2
#define UP(ij,l) ij,l+1
#define DOWN(ij,l) ij,l-1
#define HIDX(ij,l) ij