Commit 563a6fb6 authored by Patryk Kiepas's avatar Patryk Kiepas 🐦
Browse files

Merging with recent changes from trunk2master

parents f1b2f3fe 795ecdc2
......@@ -5,7 +5,7 @@ stages:
variables:
GIT_CLONE_PATH: $CI_BUILDS_DIR/$CI_PROJECT_PATH-$CI_CONCURRENT_PROJECT_ID
XIOS_REV: 2148
XIOS_REV: 2322
EXTRA_ARCH: test/arch
SCRIPTS: test/bash
CPU: skylake
......@@ -13,10 +13,8 @@ variables:
#============================== BUILD ============================#
.build-generic: &build-generic
tags:
- docker
stage: build
image: gitlab-registry.in2p3.fr/ipsl/lmd/intro/awaca/$COMP/xios:$XIOS_REV
image: gitlab-registry.in2p3.fr/ipsl/projets/dynamico/dockerimages/$COMP/xios:$XIOS_REV
before_script:
- cp $EXTRA_ARCH/* arch/ && mkdir -p tmp
script:
......@@ -60,6 +58,7 @@ build-noio:
build-xml:
only:
- trunk2master
- hotfix/trunk/xml
- converge/trunk
variables:
COMP: gnu
......@@ -91,26 +90,34 @@ build-python:
#============================== CHECK ============================#
.check-jean-zay: # remove dot to activate job
stage: check
check-jean-zay: # remove dot to activate job
allow_failure: true
stage: check
needs: ["build-nvidia"]
tags:
- jean-zay
variables:
XIOS_DIR: /gpfsdswork/projects/rech/wuu/rdzt903/JEANZAY/XIOS/JEANZAY_NVIDIA_ACC/XIOS.2233
script:
- hostname
- whoami
- uname -a
- ls -l
- git branch -a
- pwd
- rm -f ../XIOS && ln -s $XIOS_DIR ../XIOS && ls -l ../XIOS/bin
- ./make_icosa -arch JEANZAY_NVIDIA_ACC -parallel mpi -with_xios -job 16
- cd test/jeanzay && rm -f *.xml && ln -s ../../xml/*.xml .
- python ../../xml/file_def_dynamico.xml.py > file_def_dynamico.xml
- sbatch --wait job_JEANZAY_ACC.sh
- grep ' NaN ' rundir/gcm.log && exit -1
- grep -B 5 GLOB rundir/gcm.log
- grep 'Throughput' rundir/gcm.log | tail -n 5
- grep 'dyn ' rundir/gcm.log | tail -n 1
- mv DYNAMICO*.out rundir/ && mv rundir ../..
artifacts:
paths:
- rundir
check-dcmip41:
allow_failure: true
stage: check
needs: ["build-gnu"]
tags:
- docker
image: gitlab-registry.in2p3.fr/ipsl/lmd/intro/awaca/$COMP/xios:$XIOS_REV
image: gitlab-registry.in2p3.fr/ipsl/projets/dynamico/dockerimages/$COMP/xios:$XIOS_REV
variables:
COMP: gnu
PARALLEL: mpi
......@@ -119,7 +126,7 @@ check-dcmip41:
- cp $EXTRA_ARCH/* arch/ && mkdir -p tmp
- TARGET=$CPU ./make_icosa -arch $COMP -parallel $PARALLEL -with_xios -job 8 2>&1 | tee tmp/$COMP.log
- grep -q 'Build command finished' tmp/$COMP.log
- test/bash/run_mpi.sh 10
- test/bash/run_mpi.sh 8
- apt update && apt -yqq install jupyter-notebook python3-netcdf4 python3-matplotlib jupyter-nbconvert texlive-fonts-recommended texlive-generic-recommended texlive-xetex
- cd rundir
- LD_LIBRARY_PATH="" jupyter nbconvert --to pdf --execute DCMIP41.ipynb
......@@ -133,8 +140,6 @@ pages:
stage: deploy
only:
- devel2master
tags:
- docker
variables:
COMP: gnu
PARALLEL: mpi_omp
......@@ -144,7 +149,7 @@ pages:
paths:
- public/
expire_in: 1h
image: gitlab-registry.in2p3.fr/ipsl/lmd/intro/awaca/$COMP/xios:$XIOS_REV
image: gitlab-registry.in2p3.fr/ipsl/projets/dynamico/dockerimages/$COMP/xios:$XIOS_REV
script:
- apt install -yqq doxygen graphviz
- cp $EXTRA_ARCH/* arch
......
../../XML/context_dynamico.xml
\ No newline at end of file
../../xml/context_dynamico.xml
\ No newline at end of file
../../XML/context_input_dynamico.xml
\ No newline at end of file
../../xml/context_input_dynamico.xml
\ No newline at end of file
#planetary radius : real (default=37122E6)
radius=6.4E6 #6.37122E6
# Planetary radius (m) : real (default=6371220)
radius = 6.371e6
# gravity : real (default=9.80616)
g=9.80616
# day length (s) : real (default 86400)
daysec = 86400
#planetary rotation rate (default=7.292E-5)
omega=7.292E-5
# Gravity : real (default=9.80616)
g = 9.8
# kappa=Rd/cpp (default=0.2857143)
kappa=0.2857143
# Heat capacity at constant pressure real (default=1004.70885)
cpp=1004.
# Planetary rotation rate : real (default=7.292E-5)
omega = 7.292e-5
# Reference pressure : real (default=101325.)
preff=100000.
# kappa=Rd/cpp : real (default=0.2857143)
kappa = 0.2857143
# Heat capacity at constant pressure : real (default=1004.70885)
cpp = 1004
# Reference pressure : real (default=101325)
preff = 1e5
../../XML/field_def_dynamico.xml
\ No newline at end of file
../../xml/field_def_dynamico.xml
\ No newline at end of file
../../XML/iodef.xml
\ No newline at end of file
../../xml/iodef.xml
\ No newline at end of file
......@@ -26,7 +26,9 @@ omp_level_size=1
itau_adv = 1
# Time step in s : real (default=480)
dt = 720
# dt = 720
# Alternative to specifying "dt", specify number of steps per day : day_step
day_step = 120
# Number of tracers : integer (default=1)
nqtot = 0
......@@ -38,7 +40,9 @@ nqtot = 0
time_style = none
# Run length in s : real (default=??)
run_length = 103680000
# run_length = 103680000
# Alternative to specifying "run_length", specify number of days to run : ndays
ndays=1200
# Interval in s between two outputs : integer (default=??)
write_period = 10800
......@@ -46,24 +50,7 @@ write_period = 10800
#---------------- Planet ----------------
# Planetary radius : real (default=6371220)
radius = 6.371e6
# Gravity : real (default=9.80616)
g = 9.8
# Planetary rotation rate : real (default=7.292E-5)
omega = 7.292e-5
# kappa=Rd/cpp : real (default=0.2857143)
kappa = 0.2857143
# Heat capacity at constant pressure : real (default=1004.70885)
cpp = 1004
# Reference pressure : real (default=101325)
preff = 1e5
INCLUDEDEF=earth_const.def
#---------------- Physical parameters ----------------
......
module purge
module load gcc/9.1.0
module load nvidia-compilers/21.3 openmpi/4.0.5-cuda
module load hdf5/1.12.0-mpi-cuda netcdf-c/4.7.4-mpi-cuda netcdf-fortran/4.5.3-mpi-cuda
makelocalrc -x -d . -gcc `which gcc` -gpp `which g++` -g77 `which gfortran`
export NVLOCALRC=$PWD/localrc
%COMPILER mpif90
%LINK mpif90
%FPP mpif90 -E
%AR ar
%MAKE gmake
%FPP_FLAGS -acc
%FPP_DEF KEY_NONE CPP_NETCDF4 CPP_USING_MPI_NEW
%BASE_FFLAGS -i4 -r8 -acc -ta=tesla:cc70 -Minfo=accel -pg
%PROD_FFLAGS -gopt -traceback -fast
%DEV_FFLAGS -g -O1 -traceback
%DEBUG_FFLAGS -g -Mbounds -Kieee -O0 -traceback
%MPI_FFLAGS
%OMP_FFLAGS -mp
%BASE_LD -i4 -r8 -lblas -llapack -acc -lstdc++ -pg -L$CUDA_HOME/lib64 -lnvToolsExt
%MPI_LD
%OMP_LD -mp
NETCDF_INCDIR="$(nc-config --cflags) $(nc-config --fflags)"
NETCDF_LIBDIR="$(nc-config --libs) $(nc-config --flibs)"
NETCDF_LIB=""
XIOS_ROOTDIR="$PWD/../XIOS"
XIOS_INCDIR="-I$XIOS_ROOTDIR/inc"
XIOS_LIBDIR="-L$XIOS_ROOTDIR/lib"
XIOS_LIB="-lxios"
module purge
module load intel/19.0.0
module load intelmpi/19.0.0
module load hdf5/1.12.1
module load netcdf-c/4.8.1.p
module load netcdf-fortran/4.5.4.p
module load netcdf-cxx/4.4.3.1.p
module load gcc/6.3.0
%COMPILER mpiifort
%LINK mpiifort
%FPP mpiifort -E
%AR ar
%MAKE make
%FPP_FLAGS -P
%CPP_FLAGS -P
%FPP_DEF KEY_NONE CPP_NETCDF4
%BASE_FFLAGS -real-size 64 -mkl=parallel -fp-model precise -align common -mcmodel=large -shared-intel -heap-arrays 0
%PROD_FFLAGS -O2
%DEV_FFLAGS -O2 -g -traceback -fpe0
%DEBUG_FFLAGS -fpe0 -p -g -O0 -traceback -fp-stack-check -ftrapuv -check bounds,noarg_temp_created,pointers,stack,uninit -debug full -init=arrays -init=snan
%MPI_FFLAGS
%OMP_FFLAGS -qopenmp -qopenmp-threadprivate=compat
%BASE_LD -mkl=parallel
%MPI_LD
%OMP_LD -qopenmp -qopenmp-threadprivate=compat
ROOT=$PWD
NETCDF_INCDIR="$(nf-config --cflags) $(nc-config --cflags)"
NETCDF_LIBDIR="$(nf-config --flibs)"
NETCDF_LIB=""
IOIPSL_INCDIR="-I$ROOT/../IOIPSL/inc"
IOIPSL_LIBDIR="-L$ROOT/../IOIPSL/lib"
IOIPSL_LIB="-lioipsl"
XIOS_INCDIR="-I$ROOT/../XIOS/inc"
XIOS_LIBDIR="-L$ROOT/../XIOS/lib"
XIOS_LIB="-lxios -lstdc++"
module purge
module load feature/openmpi/mpi_compiler/intel
module load c++/gnu/7.3.0
module load intel/19.0.5.281
module load mpi/openmpi/4.0.2
module unload netcdf-c netcdf-fortran hdf5 flavor perl hdf5 boost blitz mpi gnu
#module purge
module load flavor/hdf5/parallel
module load netcdf-fortran/4.4.4
module load hdf5/1.8.20
module load gnu
module load boost
module load blitz
module load feature/bridge/heterogenous_mpmd
......@@ -2,11 +2,12 @@
# module feature/openmpi/mpi_compiler/intel is made necessary by c++/gnu
module purge
module load flavor/buildcompiler/intel/20
module load feature/openmpi/mpi_compiler/intel
module load c++/gnu/7.3.0
module load intel/17.0.6.256
module load mpi/openmpi/2.0.4
module load c++/gnu/8.3.0
module load intel/20.0.0
module load mpi/openmpi/4.0.5
module load flavor/hdf5/parallel
module load netcdf-fortran/4.4.4
module load mkl/17.0.6.256
module load hdf5/1.8.20
module load netcdf-fortran/4.4.4
module load mkl/20.0.0
......@@ -5,12 +5,12 @@
%MAKE gmake
%FPP_FLAGS -P
%FPP_DEF KEY_NONE CPP_NETCDF4
%BASE_FFLAGS -i4 -r8 -auto -align all
%BASE_FFLAGS -i4 -r8 -auto -align all -fp-model strict
%PROD_FFLAGS -g -traceback -O3 -xHost -qopt-zmm-usage=high
%DEV_FFLAGS -g -O1 -traceback
%DEBUG_FFLAGS -g -traceback -check bounds -fp-model strict
%DEBUG_FFLAGS -g -traceback -check bounds
%MPI_FFLAGS
%OMP_FFLAGS -qopenmp
%OMP_FFLAGS -qopenmp -qopenmp-threadprivate=compat
%BASE_LD -i4 -r8 -auto -lmkl_intel_lp64 -lmkl_core -lmkl_sequential -lstdc++
%MPI_LD
%OMP_LD -qopenmp
%OMP_LD -qopenmp -qopenmp-threadprivate=compat
......@@ -5,10 +5,10 @@
%MAKE gmake
%FPP_FLAGS -P
%FPP_DEF KEY_NONE CPP_NETCDF4
%BASE_FFLAGS -i4 -r8 -auto -align all -I${MKLROOT}/include
%BASE_FFLAGS -i4 -r8 -auto -align all -fp-model strict -I${MKLROOT}/include
%PROD_FFLAGS -g -traceback -O3 -xAVX
%DEV_FFLAGS -g -O1 -traceback
%DEBUG_FFLAGS -g -traceback -check bounds -fp-model strict
%DEBUG_FFLAGS -g -traceback -check bounds
%MPI_FFLAGS
%OMP_FFLAGS -qopenmp -qopenmp-threadprivate=compat
%BASE_LD -i4 -r8 -auto $MKL_LIBS
......
!=================== Original source is in jsrc/ =================
MODULE field_mod
USE genmod
USE prec, ONLY : rstd
USE layout_mod
IMPLICIT NONE
......@@ -35,27 +37,116 @@ MODULE field_mod
MODULE PROCEDURE getval_{{tn}}{{rk}}d
{%- endfor %} {%- endfor %}
END INTERFACE
! This module is PUBLIC, we do not want to propagate symbols from modules usedd at module level
PRIVATE :: allocate_field_, deallocate_field_
CONTAINS
!====================================== allocate_field ===================================
!====================================== PRIVATE : allocate_field ===================================
SUBROUTINE allocate_field_(layout, assigned_dom, field, field_type, data_type, dim3, dim4, name, ondevice)
USE layout_mod, ONLY : t_layout, order_ij_l_p, order_l_ij_p
USE domain_mod
USE omp_para
TYPE(t_layout), INTENT(IN) :: layout(:)
LOGICAL, INTENT(IN) :: assigned_dom(:)
TYPE(t_field) :: field(:)
INTEGER, INTENT(IN) :: field_type
INTEGER, INTENT(IN) :: data_type
INTEGER, OPTIONAL :: dim3, dim4
CHARACTER(*), OPTIONAL :: name
LOGICAL, INTENT(IN), OPTIONAL :: ondevice
INTEGER :: ind, ij_size
DO ind=1,SIZE(field)
IF (.NOT. assigned_dom(ind) .OR. .NOT. is_omp_level_master) CYCLE
IF(PRESENT(name)) THEN
field(ind)%name = name
ELSE
field(ind)%name = '(undefined)'
END IF
IF (PRESENT(dim4)) THEN
field(ind)%ndim=4
field(ind)%dim4=dim4
field(ind)%dim3=dim3
ELSE IF (PRESENT(dim3)) THEN
field(ind)%ndim=3
field(ind)%dim3=dim3
field(ind)%dim4=1
ELSE
field(ind)%ndim=2
field(ind)%dim3=1
field(ind)%dim4=1
ENDIF
field(ind)%data_type=data_type
field(ind)%field_type=field_type
ij_size = layout(ind)%ij_size
SELECT CASE( layout(ind)%index_order )
CASE( order_ij_l_p )
IF (data_type==type_integer) ALLOCATE(field(ind)%ival4d(ij_size, field(ind)%dim3, field(ind)%dim4))
IF (data_type==type_real) ALLOCATE(field(ind)%rval4d(ij_size, field(ind)%dim3, field(ind)%dim4))
IF (data_type==type_logical) ALLOCATE(field(ind)%lval4d(ij_size, field(ind)%dim3, field(ind)%dim4))
IF (field(ind)%ndim==2) THEN
IF (data_type==type_integer) field(ind)%ival2d => field(ind)%ival4d(:,1,1)
IF (data_type==type_real) field(ind)%rval2d => field(ind)%rval4d(:,1,1)
IF (data_type==type_logical) field(ind)%lval2d => field(ind)%lval4d(:,1,1)
ENDIF
CASE( order_l_ij_p )
IF (data_type==type_integer) ALLOCATE(field(ind)%ival4d(field(ind)%dim3, ij_size, field(ind)%dim4))
IF (data_type==type_real) ALLOCATE(field(ind)%rval4d(field(ind)%dim3, ij_size, field(ind)%dim4))
IF (data_type==type_logical) ALLOCATE(field(ind)%lval4d(field(ind)%dim3, ij_size, field(ind)%dim4))
IF (field(ind)%ndim==2) THEN
IF (data_type==type_integer) field(ind)%ival2d => field(ind)%ival4d(1,:,1)
IF (data_type==type_real) field(ind)%rval2d => field(ind)%rval4d(1,:,1)
IF (data_type==type_logical) field(ind)%lval2d => field(ind)%lval4d(1,:,1)
ENDIF
END SELECT
IF (field(ind)%ndim==3) THEN
IF (data_type==type_integer) field(ind)%ival3d => field(ind)%ival4d(:,:,1)
IF (data_type==type_real) field(ind)%rval3d => field(ind)%rval4d(:,:,1)
IF (data_type==type_logical) field(ind)%lval3d => field(ind)%lval4d(:,:,1)
END IF
field(ind)%ondevice = .FALSE.
IF (PRESENT(ondevice)) THEN
IF (ondevice) CALL create_device_field(field(ind))
END IF
END DO
END SUBROUTINE allocate_field_
!====================================== PUBLIC : allocate_field ===================================
SUBROUTINE allocate_field_glo(field,field_type,data_type,dim3,dim4,name)
SUBROUTINE allocate_field_glo(field, field_type, data_type, dim3, dim4, name)
USE layout_mod, ONLY : layout => layout_glo
TYPE(t_field),POINTER :: field(:)
INTEGER,INTENT(IN) :: field_type
INTEGER,INTENT(IN) :: data_type
INTEGER,OPTIONAL :: dim3,dim4
IMPLICIT NONE
TYPE(t_field), POINTER :: field(:)
INTEGER, INTENT(IN) :: field_type
INTEGER, INTENT(IN) :: data_type
INTEGER, OPTIONAL :: dim3, dim4
CHARACTER(*), OPTIONAL :: name
INTEGER :: ndom, ind
INTEGER :: ndom
LOGICAL, ALLOCATABLE :: assigned_dom(:)
ndom = SIZE(layout,1)
ALLOCATE(field(ndom))
DO ind=1,ndom
CALL allocate_field_(layout(ind,field_type), field(ind), field_type, data_type, dim3, dim4, name)
ENDDO
ALLOCATE(assigned_dom(ndom))
! ONLY the master thread is allowed to call this routine
ALLOCATE(field(ndom))
assigned_dom(:) = .TRUE.
CALL allocate_field_(layout(:,field_type), assigned_dom, field, field_type, data_type, dim3, dim4, name)
DEALLOCATE(assigned_dom)
END SUBROUTINE allocate_field_glo
......@@ -66,28 +157,23 @@ CONTAINS
TYPE(t_field), POINTER :: field(:)
INTEGER, INTENT(IN) :: field_type
INTEGER, INTENT(IN) :: data_type
INTEGER, OPTIONAL :: dim3,dim4
INTEGER, OPTIONAL :: dim3, dim4
CHARACTER(*), OPTIONAL :: name
LOGICAL, INTENT(IN), OPTIONAL :: ondevice
INTEGER :: ndom, ind
INTEGER :: ndom
ndom = SIZE(layout,1)
!$OMP BARRIER
!$OMP MASTER
!$OMP BARRIER
!$OMP MASTER
ALLOCATE(field(ndom))
!$OMP END MASTER
!$OMP BARRIER
DO ind=1,ndom
IF (.NOT. assigned_domain(ind) .OR. .NOT. is_omp_level_master) CYCLE
CALL allocate_field_(layout(ind,field_type), field(ind), field_type, data_type, dim3, dim4, name, ondevice)
END DO
!$OMP BARRIER
!$OMP END MASTER
!$OMP BARRIER
CALL allocate_field_(layout(:,field_type), assigned_domain, field, field_type, data_type, dim3, dim4, name, ondevice)
!$OMP BARRIER
END SUBROUTINE allocate_field
SUBROUTINE allocate_fields(nfield,field,field_type,data_type,dim3,dim4,name, ondevice)
SUBROUTINE allocate_fields(nfield, field, field_type, data_type, dim3, dim4, name, ondevice)
USE layout_mod, ONLY : layout
USE domain_mod, ONLY : assigned_domain
USE omp_para, ONLY : is_omp_level_master
......@@ -95,111 +181,71 @@ CONTAINS
TYPE(t_field), POINTER :: field(:,:)
INTEGER, INTENT(IN) :: field_type
INTEGER, INTENT(IN) :: data_type
INTEGER, OPTIONAL :: dim3,dim4
INTEGER, OPTIONAL :: dim3, dim4
CHARACTER(*), OPTIONAL :: name
LOGICAL, INTENT(IN), OPTIONAL :: ondevice
INTEGER :: ndom, i, ind
INTEGER :: i, ndom
ndom = SIZE(layout,1)
!$OMP BARRIER
!$OMP MASTER
!$OMP BARRIER
!$OMP MASTER
ALLOCATE(field(ndom,nfield))
!$OMP END MASTER
!$OMP BARRIER
DO ind=1,ndom
IF (.NOT. assigned_domain(ind) .OR. .NOT. is_omp_level_master) CYCLE
DO i=1,nfield
CALL allocate_field_(layout(ind,field_type), field(ind,i),field_type, data_type, dim3, dim4, name, ondevice)
END DO
!$OMP END MASTER
!$OMP BARRIER
DO i=1,nfield
CALL allocate_field_(layout(:,field_type), assigned_domain, field(:,i), field_type, data_type, dim3, dim4, name, ondevice)
END DO
!$OMP BARRIER
!$OMP BARRIER
END SUBROUTINE allocate_fields
SUBROUTINE allocate_field_(layout, field, field_type, data_type, dim3, dim4, name, ondevice)
USE layout_mod, ONLY : t_layout
USE omp_para
TYPE(t_layout) :: layout
TYPE(t_field) :: field
INTEGER, INTENT(IN) :: field_type
INTEGER, INTENT(IN) :: data_type
INTEGER, OPTIONAL :: dim3,dim4
CHARACTER(*), OPTIONAL :: name
LOGICAL, INTENT(IN), OPTIONAL :: ondevice
INTEGER :: ij_size
!==================================== PRIVATE : deallocate_field ===================================
IF(PRESENT(name)) THEN
field%name = name
ELSE
field%name = '(undefined)'
END IF
SUBROUTINE deallocate_field_(assigned_dom, field)
USE omp_para
IMPLICIT NONE
LOGICAL, INTENT(IN) :: assigned_dom(:)
TYPE(t_field) :: field(:)
INTEGER :: data_type
INTEGER :: ind
DO ind=1,SIZE(field)
IF (.NOT. assigned_dom(ind) .OR. .NOT. is_omp_level_master) CYCLE
IF (PRESENT(dim4)) THEN
field%ndim=4
field%dim4=dim4
field%dim3=dim3
ELSE IF (PRESENT(dim3)) THEN
field%ndim=3
field%dim3=dim3
field%dim4=1
ELSE
field%ndim=2
field%dim3=1
field%dim4=1
ENDIF
field%data_type=data_type
field%field_type=field_type
ij_size = layout%ij_size
IF (data_type==type_integer) ALLOCATE(field%ival4d(ij_size, field%dim3, field%dim4))
IF (data_type==type_real) ALLOCATE(field%rval4d(ij_size, field%dim3, field%dim4))
IF (data_type==type_logical) ALLOCATE(field%lval4d(ij_size, field%dim3, field%dim4))
IF (field%ndim==3) THEN
IF (data_type==type_integer) field%ival3d => field%ival4d(:,:,1)
IF (data_type==type_real) field%rval3d => field%rval4d(:,:,1)
IF (data_type==type_logical) field%lval3d => field%lval4d(:,:,1)