Some tests added

.gitignore

@@ -32,7 +32,8 @@ xcuserdata/
 # Specific files
 *.o*
 *.png
+*.so
+*.log
 Weights/
 __pycache__/
-*.so
-*.log
\ No newline at end of file
+DomainDecompTests/Output/
\ No newline at end of file

DomainDecompTests/DomainDecomp.py

+#
+#
+#
+import numpy as np
+import os, sys
+from netCDF4 import Dataset
+from mpi4py import MPI
+comm = MPI.COMM_WORLD
+#
+sys.path.append('../')
+#
+# Get the information from the configuration file.
+#
+import configparser
+config=configparser.ConfigParser({'SaveWeights':'true', "DiagLon":"0.0, 0.0", "DiagLat":"0.0, 0.0"})
+config.read("../run.def")
+#
+import getargs
+log_master, log_world = getargs.getLogger(__name__)
+INFO, DEBUG, ERROR = log_master.info, log_master.debug, log_world.error
+INFO_ALL, DEBUG_ALL = log_world.info, log_world.debug
+#
+import ModelGrid as MG
+import Partition as PA
+import RPPtools as RPP
+#
+import matplotlib.pylab as plt
+import savefile
+#
+gg = MG.GlobalGrid()
+#
+szhalo=1
+nbc=comm.Get_size()
+rank=comm.Get_rank()
+#
+# Open write unit
+#
+if not os.path.exists('Output'):
+    os.makedirs('Output')
+wunit = open("Output/TestDomain_out_"+str(rank)+".txt", "w")
+#
+wunit.write("START :: i = 0 :"+str(gg.ni-1)+" >> j = 0 :"+str(gg.nj-1)+" SZ="+str(gg.ni*gg.nj)+" nbland="+str(gg.nbland)+'\n')
+wunit.write("======================================"+'\n')
+#
+part = PA.partition(gg.ni, gg.nj, gg.land, comm, nbc, szhalo, rank, wunit=wunit)
+
+wunit.write("Domain on proc : "+str(part.ni)+str(part.nj)+" Offset from global domain : "+str(part.istart)+str(part.jstart)+'\n')
+wunit.write("Domain with halo on proc : "+str(part.nih)+str(part.njh)+" Offset with halo : "+str(part.ihstart)+str(part.jhstart)+'\n')
+wunit.write(" i inner = "+str(part.istart)+":"+str(part.istart+part.ni-1)+" i w.halo = "+str(part.ihstart)+":"+str(part.ihstart+part.nih-1))
+wunit.write(" j inner = "+str(part.jstart)+":"+str(part.jstart+part.nj-1)+" j w.halo = "+str(part.jhstart)+":"+str(part.jhstart+part.njh-1)+'\n')
+wunit.write("=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+======================"+'\n')
+
+x=np.zeros((part.njh,part.nih))
+ihoff=part.istart-part.ihstart
+jhoff=part.jstart-part.jhstart
+x[jhoff:jhoff+part.nj,ihoff:ihoff+part.ni]=99
+
+wunit.write("====== TEST FIELD START ======================"+'\n')
+for j in range(part.njh) :
+    wunit.write(str(x[j,:].astype(int))+'\n')
+
+#sys.exit()
+    
+part.sendtohalo(x)
+
+wunit.write("====== TEST FIELD AFTER HALO EXCHANGE ======================"+'\n')
+for j in range(part.njh) :
+    wunit.write(str(x[j,:].astype(int))+'\n')
+
+x=np.zeros((part.njh,part.nih))
+ihoff=part.istart-part.ihstart
+jhoff=part.jstart-part.jhstart
+x[jhoff:jhoff+part.nj,ihoff:ihoff+part.ni]=rank
+
+wunit.write("====== GATHER FIELD START ======================"+'\n')
+for j in range(part.njh) :
+    wunit.write(str(x[j,:].astype(int))+'\n')
+
+xgather = part.gather(x)
+
+if rank == 0 :
+    print("Type of gathered variable : ", xgather.astype(int))
+
+modelgrid=MG.ModelGrid(part.ihstart+gg.igstart, part.nih, part.jhstart+gg.jgstart, part.njh)
+INFO("Longitude interval on proc "+str(rank)+" = "+str(modelgrid.box_land[0][0])+" : "+str(modelgrid.box_land[0][1]))
+INFO("Latitude interval on proc "+str(rank)+" = "+str(modelgrid.box_land[1][0])+" : "+str(modelgrid.box_land[1][1]))
+
+rland=np.zeros((modelgrid.nbland))
+rland[:] = rank
+
+xland = part.landscatgat(modelgrid, rland)
+
+if rank == 0 :
+    xland[~np.isnan(xland)] = 1
+    xland[np.isnan(xland)] = 0
+    print("Difference between global and and the gathered version :",np.sum(xland-gg.land))
+
+#
+# Test the landsendtohalo function
+#
+x=np.zeros((part.njh,part.nih))
+x[:,:]=-1
+ihoff=part.istart-part.ihstart
+jhoff=part.jstart-part.jhstart
+x[jhoff:jhoff+part.nj,ihoff:ihoff+part.ni]=rank
+
+lx = modelgrid.landgather(x)
+part.landsendtohalo(lx)
+xland = part.landscatgat(modelgrid, lx)
+ 
+if rank == 0 :
+    xland[~np.isnan(xland)] = 1
+    xland[np.isnan(xland)] = 0
+    print("Difference between global and gathered version of land only :",np.sum(xland-gg.land))
+
+print("len areas : ", len(modelgrid.area))
+areas = part.landscatgat(modelgrid, np.array(modelgrid.area))
+if rank == 0 :
+    print(areas/1000000000.)
+wunit.close()
+
+y=np.zeros((nbc,modelgrid.nbland))
+print("Test : ", y.shape, modelgrid.nbland)
+y[rank,:]=rank
+
+savefile.dumpnetcdf("TestDomainDecomp.nc", gg, modelgrid, part, rland, x, y)

DomainDecompTests/TestDomaonDecomp.pbs

+#!/bin/bash
+#
+#PBS -N TestDomainDecomp
+#
+#PBS -j oe
+#PBS -l nodes=1:ppn=64
+#PBS -l walltime=48:00:00
+#PBS -l mem=148gb
+#PBS -l vmem=148gb
+#
+#
+cd $PBS_O_WORKDIR
+/bin/rm -f Output/* TestDomainDecomp*nc
+#
+cp ../TestConfigs/run.def.05_CS run.def
+#
+# Set the right Python 3 Anaconda environment
+#
+source ../Environment
+#
+# Run domaon decomposition test for 2 possible number of cores
+#
+# N=4
+start=`date +%s`
+time mpirun -n 4 python DomainDecomp.py
+end=`date +%s`
+runtime=$((end-start))
+echo "Runtime for n=4 ${runtime}"
+mv TestDomainDecomp.nc TestDomainDecomp_n4.nc
+#
+# N=16
+#
+start=`date +%s`
+time mpirun -n 16 python DomainDecomp.py
+end=`date +%s`
+runtime=$((end-start))
+echo "Runtime for n=16 ${runtime}"
+mv TestDomainDecomp.nc TestDomainDecomp_n16.nc
+#
+# N=32
+#
+start=`date +%s`
+time mpirun -n 32 python DomainDecomp.py
+end=`date +%s`
+runtime=$((end-start))
+echo "Runtime for n=32 ${runtime}"
+mv TestDomainDecomp.nc TestDomainDecomp_n32.nc
+#
+# With the WRF grid
+#
+cp ../TestConfigs/run.def.WRF_CS run.def
+#
+# N=32
+#
+start=`date +%s`
+time mpirun -n 32 python DomainDecomp.py
+end=`date +%s`
+runtime=$((end-start))
+echo "Runtime for n=32 ${runtime}"
+mv TestDomainDecomp.nc TestDomainDecomp_WRF_n32.nc
+#
+# N=64
+#
+rm -f Output/*txt
+start=`date +%s`
+time mpirun -n 64 python DomainDecomp.py
+end=`date +%s`
+runtime=$((end-start))
+echo "Runtime for n=64 ${runtime}"
+mv TestDomainDecomp.nc TestDomainDecomp_WRF_n64.nc
+#
+ls -lR
+#

TestConfigs/run.def.05_CS

+[OverAll]
+#
+#
+EarthRadius = 6370000.
+#
+ModelGridFile = /bdd/MEDI/workspaces/polcher/NewRouting/EM_WFDEI_CRU_2000.nc
+##WEST_EAST = -9.75, 5.25
+##SOUTH_NORTH = 36.1, 44.2
+HydroFile = /bdd/MEDI/workspaces/polcher/NewRouting/routing_MED.nc
+#
+# FORTRAN interface parameters
+#
+Documentation = true
+#
+# Configuration for the graph to be generated
+#
+nbasmax = 35
+#
+# Output
+#
+GraphFile = test_graph.nc
+#
+# Diagnostics
+# You need to provide an interval in longitude and Latitude.
+#
+DiagLon = 39.9, 39.9
+DiagLat = 40.0, 40.0

TestConfigs/run.def.05_local

+[OverAll]
+#
+#
+EarthRadius = 6370000.
+#
+ModelGridFile = /home/polcher/WORK/Data/NewRouting/EM_WFDEI_CRU_2000.nc
+##WEST_EAST = -9.75, 5.25
+##SOUTH_NORTH = 36.1, 44.2
+HydroFile = /home/polcher/WORK/Data/NewRouting/routing_MED.nc
+#
+# FORTRAN interface parameters
+#
+Documentation = true
+#
+# Configuration for the graph to be generated
+#
+nbasmax = 35
+#
+# Output
+#
+GraphFile = test_graph.nc
+#
+# Diagnostics
+# You need to provide an interval in longitude and Latitude.
+#
+DiagLon = 2.9, 3.9
+DiagLat = 38.8, 40.0

TestConfigs/run.def.WRF_CS

+[OverAll]
+#
+#
+#
+EarthRadius = 6370000.
+#
+ModelGridFile = /bdd/MEDI/workspaces/polcher/NewRouting/geo_em.d01.nc
+HydroFile = /bdd/MEDI/workspaces/polcher/NewRouting/routing_MED.nc
+#
+# FORTRAN interface parameters
+#
+Documentation = true
+#
+# Configuration for the graph to be generated
+#
+nbasmax = 35
+#
+# Output
+#
+GraphFile = test_graph.nc
+#
+# Diagnostics
+# You need to provide an interval in longitude and Latitude.
+#
+DiagLon = 2.9, 3.9
+DiagLat = 38.8, 40.0
\ No newline at end of file

TestConfigs/run.def.WRF_local

+[OverAll]
+#
+#
+EarthRadius = 6370000.
+#
+ModelGridFile = /home/polcher/WORK/Data/NewRouting/geo_em.d01.nc
+# Mallorca
+WEST_EAST = 2.3, 3.5
+SOUTH_NORTH = 39.00, 40.1
+HydroFile = /home/polcher/WORK/Data/NewRouting/routing_MED.nc
+#
+# FORTRAN interface parameters
+#
+Documentation = true
+#
+# Configuration for the graph to be generated
+#
+nbasmax = 35
+#
+# Output
+#
+GraphFile = test_graph.nc
+#
+# Diagnostics
+# You need to provide an interval in longitude and Latitude.
+#
+DiagLon = 2.9, 3.9
+DiagLat = 38.8, 40.0