diff --git a/src/LIB/tensorproductmultigrid_Source/communication.f90 b/src/LIB/tensorproductmultigrid_Source/communication.f90
index b7ac6348d477ebc9952d9ea235c82414e1856b5a..c077863815235d688f0c4eaa4cac9059898289a3 100644
--- a/src/LIB/tensorproductmultigrid_Source/communication.f90
+++ b/src/LIB/tensorproductmultigrid_Source/communication.f90
@@ -40,8 +40,13 @@ module communication
#ifndef MNH
use mpi
+ use mpi, only NMNH_COMM_WORLD => MPI_COMM_WORLD
+ use mpi, only MNH_STATUSES_IGNORE => MPI_STATUSES_IGNORE
+ use mpi, only MNHREAL_MPI => MPI_DOUBLE_PRECISION
#else
use modd_mpif
+ USE MODD_VAR_ll, ONLY: NMNH_COMM_WORLD, MNH_STATUSES_IGNORE
+ use modd_precision, only: MNHREAL_MPI
#endif
use timer
@@ -211,8 +216,8 @@ contains
subroutine comm_preinitialise()
implicit none
integer :: nproc, ierr, rank
- call mpi_comm_size(MPI_COMM_WORLD, nproc, ierr)
- call mpi_comm_rank(MPI_COMM_WORLD, rank, ierr)
+ call mpi_comm_size(NMNH_COMM_WORLD, nproc, ierr)
+ call mpi_comm_rank(NMNH_COMM_WORLD, rank, ierr)
i_am_master_mpi = (rank == master_rank)
! Check that nproc = 2^(2*p)
!pproc = floor(log(1.0d0*nproc)/log(4.0d0))
@@ -265,7 +270,7 @@ contains
halo_size = comm_param%halo_size
#ifndef MPIVIDE
- call mpi_comm_size(MPI_COMM_WORLD, nproc, ierr)
+ call mpi_comm_size(NMNH_COMM_WORLD, nproc, ierr)
!
! Manage nproc=2^pproc ( and not 2^2pproc )
!
@@ -274,7 +279,7 @@ contains
pproc = pprocx
if (i_am_master_mpi) write(STDOUT,*) "pproc=",pproc," pprocx=",pprocx," pprocy=",pprocy
! Create cartesian topology
- call mpi_cart_create(MPI_COMM_WORLD, & ! Old communicator name
+ call mpi_cart_create(NMNH_COMM_WORLD, & ! Old communicator name
dim_horiz, & ! horizontal dimension
(/2**pprocy,2**pprocx/), & ! extent in each horizontal direction
(/.false.,.false./), & ! periodic?
@@ -332,7 +337,7 @@ contains
blocklength = (nz+2)*halo_size
stride = (nlocal+2*halo_size)*(nz+2)
#ifndef MPIVIDE
- call mpi_type_vector(count,blocklength,stride,MPI_DOUBLE_PRECISION, &
+ call mpi_type_vector(count,blocklength,stride,MNHREAL_MPI, &
halo_ns(level,m),ierr)
call mpi_type_commit(halo_ns(level,m),ierr)
#endif
@@ -344,7 +349,7 @@ contains
blocklength = nlocalx*halo_size ! (nz+2)*halo_size
stride = (nlocalx+2*halo_size) * (nlocaly+2*halo_size) ! (nlocal+2*halo_size)*(nz+2)
#ifndef MPIVIDE
- call mpi_type_vector(count,blocklength,stride,MPI_DOUBLE_PRECISION, &
+ call mpi_type_vector(count,blocklength,stride,MNHREAL_MPI, &
halo_nst(level,m),ierr)
call mpi_type_commit(halo_nst(level,m),ierr)
#endif
@@ -359,7 +364,7 @@ contains
blocklength = 1*halo_size ! (nz+2)*halo_size
stride = nlocalx+2*halo_size ! (nlocal+2*halo_size)*(nz+2)
#ifndef MPIVIDE
- call mpi_type_vector(count,blocklength,stride,MPI_DOUBLE_PRECISION, &
+ call mpi_type_vector(count,blocklength,stride,MNHREAL_MPI, &
halo_wet(level,m),ierr)
call mpi_type_commit(halo_wet(level,m),ierr)
#endif
@@ -381,14 +386,14 @@ contains
blocklength = nlocal*(nz+2)
stride = (nz+2)*(nlocal+2*halo_size)
#ifndef MPIVIDE
- call mpi_type_vector(count,blocklength,stride,MPI_DOUBLE_PRECISION,interior(level,m),ierr)
+ call mpi_type_vector(count,blocklength,stride,MNHREAL_MPI,interior(level,m),ierr)
call mpi_type_commit(interior(level,m),ierr)
#endif
count = nlocal/2
blocklength = nlocal/2*(nz+2)
stride = (nlocal+2*halo_size)*(nz+2)
#ifndef MPIVIDE
- call mpi_type_vector(count,blocklength,stride,MPI_DOUBLE_PRECISION,sub_interior(level,m),ierr)
+ call mpi_type_vector(count,blocklength,stride,MNHREAL_MPI,sub_interior(level,m),ierr)
call mpi_type_commit(sub_interior(level,m),ierr)
#endif
end if
@@ -400,7 +405,7 @@ contains
coord_debut = (/ 0 , 0 , 0 /)
#ifndef MPIVIDE
call MPI_TYPE_CREATE_SUBARRAY(nb_dims,profil_tab,profil_sous_tab,coord_debut,&
- MPI_ORDER_FORTRAN,MPI_DOUBLE_PRECISION,interiorT(level,m),ierr)
+ MPI_ORDER_FORTRAN,MNHREAL_MPI,interiorT(level,m),ierr)
call mpi_type_commit(interiorT(level,m),ierr)
#endif
call mnh_allocate_mg_halo(tab_interiorT_ne(level,m)%haloTin,nlocalx,nlocaly,nz+2)
@@ -417,7 +422,7 @@ contains
coord_debut = (/ 0 , 0 , 0 /)
#ifndef MPIVIDE
call MPI_TYPE_CREATE_SUBARRAY(nb_dims,profil_tab,profil_sous_tab,coord_debut,&
- MPI_ORDER_FORTRAN,MPI_DOUBLE_PRECISION,sub_interiorT(level,m),ierr)
+ MPI_ORDER_FORTRAN,MNHREAL_MPI,sub_interiorT(level,m),ierr)
call mpi_type_commit(sub_interiorT(level,m),ierr)
#endif
call mnh_allocate_mg_halo(tab_sub_interiorT_ne(level,m)%haloTin,nlocalx/2,nlocaly/2,nz+2)
@@ -622,9 +627,9 @@ contains
if (LUseO) local_sum = 0.0_rl
if (LUseT) local_sumt = 0.0_rl
end if
- if (LUseO) call mpi_allreduce(local_sum,global_sum,1,MPI_DOUBLE_PRECISION, &
+ if (LUseO) call mpi_allreduce(local_sum,global_sum,1,MNHREAL_MPI, &
MPI_SUM,MPI_COMM_HORIZ,ierr)
- if (LUseT) call mpi_allreduce(local_sumt,global_sumt,1,MPI_DOUBLE_PRECISION, &
+ if (LUseT) call mpi_allreduce(local_sumt,global_sumt,1,MNHREAL_MPI, &
MPI_SUM,MPI_COMM_HORIZ,ierr)
else
if (LUseO) then
@@ -698,7 +703,7 @@ contains
else
local_sum = 0.0_rl
end if
- call mpi_allreduce(local_sum,global_sum,1,MPI_DOUBLE_PRECISION, &
+ call mpi_allreduce(local_sum,global_sum,1,MNHREAL_MPI, &
MPI_SUM,MPI_COMM_HORIZ,ierr)
else
global_sum = 0.0_rl
@@ -955,7 +960,7 @@ contains
! Receive from west
recvtag = 1000
if (LUseO) call mpi_irecv(a%s(0,yoffset,0-(halo_size-1)),blocklength, &
- MPI_DOUBLE_PRECISION,neighbour_w_rank,recvtag, &
+ MNHREAL_MPI,neighbour_w_rank,recvtag, &
MPI_COMM_HORIZ, recv_requests(3), ierr)
recvtag = 1010
if (LUseT) then
@@ -966,7 +971,7 @@ contains
! Receive from east
sendtag = 1001
if (LUseO) call mpi_irecv(a%s(0,yoffset,a_nx+1),blocklength, &
- MPI_DOUBLE_PRECISION,neighbour_e_rank,recvtag, &
+ MNHREAL_MPI,neighbour_e_rank,recvtag, &
MPI_COMM_HORIZ, recv_requests(4), ierr)
sendtag = 1011
if (LUseT) then
@@ -977,7 +982,7 @@ contains
! Send to east
sendtag = 1000
if (LUseO) call mpi_isend(a%s(0,yoffset,a_nx-(halo_size-1)),blocklength, &
- MPI_DOUBLE_PRECISION,neighbour_e_rank,sendtag, &
+ MNHREAL_MPI,neighbour_e_rank,sendtag, &
MPI_COMM_HORIZ, send_requests(3), ierr)
sendtag = 1010
if (LUseT) then
@@ -988,7 +993,7 @@ contains
! Send to west
recvtag = 1001
if (LUseO) call mpi_isend(a%s(0,yoffset,1),blocklength, &
- MPI_DOUBLE_PRECISION,neighbour_w_rank,sendtag, &
+ MNHREAL_MPI,neighbour_w_rank,sendtag, &
MPI_COMM_HORIZ, send_requests(4), ierr)
recvtag = 1011
if (LUseT) then
@@ -1079,22 +1084,22 @@ contains
! Receive from west
recvtag = 0
call mpi_irecv(a%s(0,yoffset,0-(halo_size-1)),blocklength, &
- MPI_DOUBLE_PRECISION,neighbour_w_rank,recvtag, &
+ MNHREAL_MPI,neighbour_w_rank,recvtag, &
MPI_COMM_HORIZ, recv_requests(3), ierr)
! Receive from east
sendtag = 1
call mpi_irecv(a%s(0,yoffset,a_n+1),blocklength, &
- MPI_DOUBLE_PRECISION,neighbour_e_rank,recvtag, &
+ MNHREAL_MPI,neighbour_e_rank,recvtag, &
MPI_COMM_HORIZ, recv_requests(4), ierr)
! Send to east
sendtag = 0
call mpi_isend(a%s(0,yoffset,a_n-(halo_size-1)),blocklength, &
- MPI_DOUBLE_PRECISION,neighbour_e_rank,sendtag, &
+ MNHREAL_MPI,neighbour_e_rank,sendtag, &
MPI_COMM_HORIZ, send_requests(3), ierr)
! Send to west
recvtag = 1
call mpi_isend(a%s(0,yoffset,1),blocklength, &
- MPI_DOUBLE_PRECISION,neighbour_w_rank,sendtag, &
+ MNHREAL_MPI,neighbour_w_rank,sendtag, &
MPI_COMM_HORIZ, send_requests(4), ierr)
end if
end if
@@ -1346,7 +1351,7 @@ contains
!!$ pztab_halo_nt_haloTout => tab_halo_nt(level,m)%haloTout
!$acc host_data use_device(pztab_halo_nt_haloTout)
call mpi_irecv(pztab_halo_nt_haloTout,size(pztab_halo_nt_haloTout), &
- MPI_DOUBLE_PRECISION,neighbour_n_rank,recvtag, &
+ MNHREAL_MPI,neighbour_n_rank,recvtag, &
MPI_COMM_HORIZ, requests_nsT(1), ierr)
!$acc end host_data
end if
@@ -1369,7 +1374,7 @@ contains
!!$ pztab_halo_st_haloTout => tab_halo_st(level,m)%haloTout
!$acc host_data use_device (pztab_halo_st_haloTout)
call mpi_irecv(pztab_halo_st_haloTout,size(pztab_halo_st_haloTout), &
- MPI_DOUBLE_PRECISION,neighbour_s_rank,recvtag, &
+ MNHREAL_MPI,neighbour_s_rank,recvtag, &
MPI_COMM_HORIZ, requests_nsT(2), ierr)
!$acc end host_data
end if
@@ -1397,7 +1402,7 @@ contains
if (Gneighbour_s) then
!$acc host_data use_device(pztab_halo_st_haloTin)
call mpi_isend(pztab_halo_st_haloTin,size(pztab_halo_st_haloTin), &
- MPI_DOUBLE_PRECISION,neighbour_s_rank,sendtag, &
+ MNHREAL_MPI,neighbour_s_rank,sendtag, &
MPI_COMM_HORIZ, requests_nsT(3), ierr)
!$acc end host_data
end if
@@ -1419,7 +1424,7 @@ contains
if (Gneighbour_n) then
!$acc host_data use_device(pztab_halo_nt_haloTin)
call mpi_isend(pztab_halo_nt_haloTin,size(pztab_halo_nt_haloTin), &
- MPI_DOUBLE_PRECISION,neighbour_n_rank,sendtag, &
+ MNHREAL_MPI,neighbour_n_rank,sendtag, &
MPI_COMM_HORIZ, requests_nsT(4), ierr)
!$acc end host_data
end if
@@ -1432,13 +1437,13 @@ contains
end if
if (halo_size > 1) then
! Wait for North <-> South communication to complete
- if (LUseO) call mpi_waitall(4,requests_ns, MPI_STATUSES_IGNORE, ierr)
- if (LUseT) call mpi_waitall(4,requests_nsT, MPI_STATUSES_IGNORE, ierr)
+ if (LUseO) call mpi_waitall(4,requests_ns, MNH_STATUSES_IGNORE, ierr)
+ if (LUseT) call mpi_waitall(4,requests_nsT, MNH_STATUSES_IGNORE, ierr)
end if
! Receive from west
recvtag = 1000
if (LUseO) call mpi_irecv(a%s(0,yoffset,0-(halo_size-1)),blocklength, &
- MPI_DOUBLE_PRECISION,neighbour_w_rank,recvtag, &
+ MNHREAL_MPI,neighbour_w_rank,recvtag, &
MPI_COMM_HORIZ, requests_ew(1), ierr)
recvtag = 1010
if (LUseT) then
@@ -1447,7 +1452,7 @@ contains
!!$ pztab_halo_wt_haloTout => tab_halo_wt(level,m)%haloTout
!$acc host_data use_device(pztab_halo_wt_haloTout)
call mpi_irecv(pztab_halo_wt_haloTout,size(pztab_halo_wt_haloTout), &
- MPI_DOUBLE_PRECISION,neighbour_w_rank,recvtag, &
+ MNHREAL_MPI,neighbour_w_rank,recvtag, &
MPI_COMM_HORIZ, requests_ewT(1), ierr)
!$acc end host_data
end if
@@ -1461,7 +1466,7 @@ contains
! Receive from east
sendtag = 1001
if (LUseO) call mpi_irecv(a%s(0,yoffset,a_nx+1),blocklength, &
- MPI_DOUBLE_PRECISION,neighbour_e_rank,recvtag, &
+ MNHREAL_MPI,neighbour_e_rank,recvtag, &
MPI_COMM_HORIZ, requests_ew(2), ierr)
sendtag = 1011
if (LUseT) then
@@ -1470,7 +1475,7 @@ contains
!!$ pztab_halo_et_haloTout => tab_halo_et(level,m)%haloTout
!$acc host_data use_device(pztab_halo_et_haloTout)
call mpi_irecv(pztab_halo_et_haloTout,size(pztab_halo_et_haloTout), &
- MPI_DOUBLE_PRECISION,neighbour_e_rank,recvtag, &
+ MNHREAL_MPI,neighbour_e_rank,recvtag, &
MPI_COMM_HORIZ, requests_ewT(2), ierr)
!$acc end host_data
end if
@@ -1485,7 +1490,7 @@ contains
! Send to east
sendtag = 1000
if (LUseO) call mpi_isend(a%s(0,yoffset,a_nx-(halo_size-1)),blocklength, &
- MPI_DOUBLE_PRECISION,neighbour_e_rank,sendtag, &
+ MNHREAL_MPI,neighbour_e_rank,sendtag, &
MPI_COMM_HORIZ, requests_ew(3), ierr)
sendtag = 1010
if (LUseT) then
@@ -1493,7 +1498,7 @@ contains
if (Gneighbour_e) then
!$acc host_data use_device(pztab_halo_et_haloTin)
call mpi_isend(pztab_halo_et_haloTin,size(pztab_halo_et_haloTin), &
- MPI_DOUBLE_PRECISION,neighbour_e_rank,sendtag, &
+ MNHREAL_MPI,neighbour_e_rank,sendtag, &
MPI_COMM_HORIZ, requests_ewT(3), ierr)
!$acc end host_data
end if
@@ -1507,7 +1512,7 @@ contains
! Send to west
recvtag = 1001
if (LUseO) call mpi_isend(a%s(0,yoffset,1),blocklength, &
- MPI_DOUBLE_PRECISION,neighbour_w_rank,sendtag, &
+ MNHREAL_MPI,neighbour_w_rank,sendtag, &
MPI_COMM_HORIZ, requests_ew(4), ierr)
recvtag = 1011
if (LUseT) then
@@ -1515,7 +1520,7 @@ contains
if (Gneighbour_w) then
!$acc host_data use_device(pztab_halo_wt_haloTin)
call mpi_isend(pztab_halo_wt_haloTin,size(pztab_halo_wt_haloTin), &
- MPI_DOUBLE_PRECISION,neighbour_w_rank,sendtag, &
+ MNHREAL_MPI,neighbour_w_rank,sendtag, &
MPI_COMM_HORIZ, requests_ewT(4), ierr)
!$acc end host_data
end if
@@ -1529,11 +1534,11 @@ contains
! Wait for East <-> West communication to complete
if (halo_size == 1) then
! Wait for North <-> South communication to complete
- if (LUseO) call mpi_waitall(4,requests_ns, MPI_STATUSES_IGNORE, ierr)
- if (LUseT) call mpi_waitall(4,requests_nsT, MPI_STATUSES_IGNORE, ierr)
+ if (LUseO) call mpi_waitall(4,requests_ns, MNH_STATUSES_IGNORE, ierr)
+ if (LUseT) call mpi_waitall(4,requests_nsT, MNH_STATUSES_IGNORE, ierr)
end if
- if (LUseO) call mpi_waitall(4,requests_ew, MPI_STATUSES_IGNORE, ierr)
- if (LUseT) call mpi_waitall(4,requests_ewT, MPI_STATUSES_IGNORE, ierr)
+ if (LUseO) call mpi_waitall(4,requests_ew, MNH_STATUSES_IGNORE, ierr)
+ if (LUseT) call mpi_waitall(4,requests_ewT, MNH_STATUSES_IGNORE, ierr)
#ifdef MNH_GPUDIRECT
if (LUseT) then
if (Gneighbour_n) then
@@ -1672,34 +1677,34 @@ contains
MPI_COMM_HORIZ, requests_ns(4), ierr)
if (halo_size > 1) then
! Wait for North <-> South communication to complete
- call mpi_waitall(4,requests_ns, MPI_STATUSES_IGNORE, ierr)
+ call mpi_waitall(4,requests_ns, MNH_STATUSES_IGNORE, ierr)
end if
! Receive from west
recvtag = 0
call mpi_irecv(a%s(0,yoffset,0-(halo_size-1)),blocklength, &
- MPI_DOUBLE_PRECISION,neighbour_w_rank,recvtag, &
+ MNHREAL_MPI,neighbour_w_rank,recvtag, &
MPI_COMM_HORIZ, requests_ew(1), ierr)
! Receive from east
sendtag = 1
call mpi_irecv(a%s(0,yoffset,a_n+1),blocklength, &
- MPI_DOUBLE_PRECISION,neighbour_e_rank,recvtag, &
+ MNHREAL_MPI,neighbour_e_rank,recvtag, &
MPI_COMM_HORIZ, requests_ew(2), ierr)
! Send to east
sendtag = 0
call mpi_isend(a%s(0,yoffset,a_n-(halo_size-1)),blocklength, &
- MPI_DOUBLE_PRECISION,neighbour_e_rank,sendtag, &
+ MNHREAL_MPI,neighbour_e_rank,sendtag, &
MPI_COMM_HORIZ, requests_ew(3), ierr)
! Send to west
recvtag = 1
call mpi_isend(a%s(0,yoffset,1),blocklength, &
- MPI_DOUBLE_PRECISION,neighbour_w_rank,sendtag, &
+ MNHREAL_MPI,neighbour_w_rank,sendtag, &
MPI_COMM_HORIZ, requests_ew(4), ierr)
! Wait for East <-> West communication to complete
if (halo_size == 1) then
! Wait for North <-> South communication to complete
- call mpi_waitall(4,requests_ns, MPI_STATUSES_IGNORE, ierr)
+ call mpi_waitall(4,requests_ns, MNH_STATUSES_IGNORE, ierr)
end if
- call mpi_waitall(4,requests_ew, MPI_STATUSES_IGNORE, ierr)
+ call mpi_waitall(4,requests_ew, MNH_STATUSES_IGNORE, ierr)
end if
if (comm_measuretime) then
call finish_timer(t_haloswap(level,m))
@@ -1803,7 +1808,7 @@ contains
ztab_sub_interiorT_ne_m_1_haloTout => tab_sub_interiorT_ne(level,m-1)%haloTout
!$acc host_data use_device(ztab_sub_interiorT_ne_m_1_haloTout)
call mpi_irecv(ztab_sub_interiorT_ne_m_1_haloTout,size(ztab_sub_interiorT_ne_m_1_haloTout), &
- MPI_DOUBLE_PRECISION,source_rank, recv_tag, MPI_COMM_HORIZ, &
+ MNHREAL_MPI,source_rank, recv_tag, MPI_COMM_HORIZ, &
recv_requestT(1),ierr)
!$acc end host_data
#else
@@ -1831,7 +1836,7 @@ contains
ztab_sub_interiorT_sw_m_1_haloTout => tab_sub_interiorT_sw(level,m-1)%haloTout
!$acc host_data use_device(ztab_sub_interiorT_sw_m_1_haloTout)
call mpi_irecv(ztab_sub_interiorT_sw_m_1_haloTout,size(ztab_sub_interiorT_sw_m_1_haloTout), &
- MPI_DOUBLE_PRECISION,source_rank, recv_tag, MPI_COMM_HORIZ, &
+ MNHREAL_MPI,source_rank, recv_tag, MPI_COMM_HORIZ, &
recv_requestT(2),ierr)
!$acc end host_data
#else
@@ -1860,7 +1865,7 @@ contains
ztab_sub_interiorT_se_m_1_haloTout => tab_sub_interiorT_se(level,m-1)%haloTout
!$acc host_data use_device(ztab_sub_interiorT_se_m_1_haloTout)
call mpi_irecv(ztab_sub_interiorT_se_m_1_haloTout,size(ztab_sub_interiorT_se_m_1_haloTout), &
- MPI_DOUBLE_PRECISION,source_rank, recv_tag, MPI_COMM_HORIZ, &
+ MNHREAL_MPI,source_rank, recv_tag, MPI_COMM_HORIZ, &
recv_requestT(3),ierr)
!$acc end host_data
#else
@@ -1888,8 +1893,8 @@ contains
#endif
end if
! Wait for receives to complete before proceeding
- if (LUseO) call mpi_waitall(3,recv_request,MPI_STATUSES_IGNORE,ierr)
- if (LUseT) call mpi_waitall(3,recv_requestT,MPI_STATUSES_IGNORE,ierr)
+ if (LUseO) call mpi_waitall(3,recv_request,MNH_STATUSES_IGNORE,ierr)
+ if (LUseT) call mpi_waitall(3,recv_requestT,MNH_STATUSES_IGNORE,ierr)
#ifdef MNH_GPUDIRECT
if (LUseT) then
zb_st => b%st
@@ -1938,7 +1943,7 @@ contains
!$acc end kernels
!$acc host_data use_device(ztab_interiorT_ne_m_haloTin)
call mpi_send(ztab_interiorT_ne_m_haloTin,size(ztab_interiorT_ne_m_haloTin), &
- MPI_DOUBLE_PRECISION,dest_rank,send_tag,MPI_COMM_HORIZ,ierr)
+ MNHREAL_MPI,dest_rank,send_tag,MPI_COMM_HORIZ,ierr)
!$acc end host_data
#else
call mpi_send(a%st(1,1,0),1,interiorT(level,m),dest_rank,send_tag,MPI_COMM_HORIZ,ierr)
@@ -1972,7 +1977,7 @@ contains
!$acc end kernels
!$acc host_data use_device(ztab_interiorT_sw_m_haloTin)
call mpi_send(ztab_interiorT_sw_m_haloTin,size(ztab_interiorT_sw_m_haloTin), &
- MPI_DOUBLE_PRECISION,dest_rank,send_tag,MPI_COMM_HORIZ,ierr)
+ MNHREAL_MPI,dest_rank,send_tag,MPI_COMM_HORIZ,ierr)
!$acc end host_data
#else
call mpi_send(a%st(1,1,0),1,interiorT(level,m),dest_rank,send_tag,MPI_COMM_HORIZ,ierr)
@@ -2007,7 +2012,7 @@ contains
!$acc end kernels
!$acc host_data use_device(ztab_interiorT_se_m_haloTin)
call mpi_send(ztab_interiorT_se_m_haloTin,size(ztab_interiorT_se_m_haloTin), &
- MPI_DOUBLE_PRECISION,dest_rank,send_tag,MPI_COMM_HORIZ,ierr)
+ MNHREAL_MPI,dest_rank,send_tag,MPI_COMM_HORIZ,ierr)
!$acc end host_data
#else
call mpi_send(a%st(1,1,0),1,interiorT(level,m),dest_rank,send_tag,MPI_COMM_HORIZ,ierr)
@@ -2125,7 +2130,7 @@ contains
!$acc end kernels
!$acc host_data use_device(ztab_sub_interiorT_ne_m_1_haloTin)
call mpi_isend(ztab_sub_interiorT_ne_m_1_haloTin,size(ztab_sub_interiorT_ne_m_1_haloTin), &
- MPI_DOUBLE_PRECISION,dest_rank, send_tag, &
+ MNHREAL_MPI,dest_rank, send_tag, &
MPI_COMM_HORIZ,send_requestT(1),ierr)
!$acc end host_data
#else
@@ -2158,7 +2163,7 @@ contains
!$acc end kernels
!$acc host_data use_device(ztab_sub_interiorT_sw_m_1_haloTin)
call mpi_isend(ztab_sub_interiorT_sw_m_1_haloTin,size(ztab_sub_interiorT_sw_m_1_haloTin), &
- MPI_DOUBLE_PRECISION, dest_rank, send_tag, &
+ MNHREAL_MPI, dest_rank, send_tag, &
MPI_COMM_HORIZ, send_requestT(2), ierr)
!$acc end host_data
#else
@@ -2192,7 +2197,7 @@ contains
!$acc end kernels
!$acc host_data use_device(ztab_sub_interiorT_se_m_1_haloTin)
call mpi_isend(ztab_sub_interiorT_se_m_1_haloTin,size(ztab_sub_interiorT_se_m_1_haloTin), &
- MPI_DOUBLE_PRECISION, dest_rank, send_tag, &
+ MNHREAL_MPI, dest_rank, send_tag, &
MPI_COMM_HORIZ, send_requestT(3), ierr)
!$acc end host_data
#else
@@ -2220,8 +2225,8 @@ contains
#endif
end if
! Only proceed when async sends to complete
- if (LUseO) call mpi_waitall(3, send_request, MPI_STATUSES_IGNORE, ierr)
- if (LUseT) call mpi_waitall(3, send_requestT, MPI_STATUSES_IGNORE, ierr)
+ if (LUseO) call mpi_waitall(3, send_request, MNH_STATUSES_IGNORE, ierr)
+ if (LUseT) call mpi_waitall(3, send_requestT, MNH_STATUSES_IGNORE, ierr)
end if
if ( corner_ne ) then
@@ -2242,7 +2247,7 @@ contains
ztab_interiorT_ne_m_haloTout => tab_interiorT_ne(level,m)%haloTout
!$acc host_data use_device(ztab_interiorT_ne_m_haloTout)
call mpi_recv(ztab_interiorT_ne_m_haloTout,size(ztab_interiorT_ne_m_haloTout), &
- MPI_DOUBLE_PRECISION,source_rank,recv_tag,MPI_COMM_HORIZ,stat,ierr)
+ MNHREAL_MPI,source_rank,recv_tag,MPI_COMM_HORIZ,stat,ierr)
!$acc end host_data
!$acc kernels present_cr(zb_st,ztab_interiorT_ne_m_haloTout)
!$mnh_do_concurrent(ii=1:b_nx,ij=1:b_ny,ik=1:nz+2)
@@ -2276,7 +2281,7 @@ contains
ztab_interiorT_sw_m_haloTout => tab_interiorT_sw(level,m)%haloTout
!$acc host_data use_device(ztab_interiorT_sw_m_haloTout)
call mpi_recv(ztab_interiorT_sw_m_haloTout,size(ztab_interiorT_sw_m_haloTout), &
- MPI_DOUBLE_PRECISION,source_rank,recv_tag,MPI_COMM_HORIZ,stat,ierr)
+ MNHREAL_MPI,source_rank,recv_tag,MPI_COMM_HORIZ,stat,ierr)
!$acc end host_data
!$acc kernels present_cr(zb_st,ztab_interiorT_sw_m_haloTout)
!$mnh_do_concurrent(ii=1:b_nx,ij=1:b_ny,ik=1:nz+2)
@@ -2310,7 +2315,7 @@ contains
ztab_interiorT_se_m_haloTout => tab_interiorT_se(level,m)%haloTout
!$acc host_data use_device(ztab_interiorT_se_m_haloTout)
call mpi_recv(ztab_interiorT_se_m_haloTout,size(ztab_interiorT_se_m_haloTout), &
- MPI_DOUBLE_PRECISION,source_rank,recv_tag,MPI_COMM_HORIZ,stat,ierr)
+ MNHREAL_MPI,source_rank,recv_tag,MPI_COMM_HORIZ,stat,ierr)
!$acc end host_data
!$acc kernels present_cr(zb_st,ztab_interiorT_se_m_haloTout)
!$mnh_do_concurrent(ii=1:b_nx,ij=1:b_ny,ik=1:nz+2)
diff --git a/src/LIB/tensorproductmultigrid_Source/conjugategradient.f90 b/src/LIB/tensorproductmultigrid_Source/conjugategradient.f90
index d6f883a98a348630a0c5cb84ea5fa811953ae43f..c259a72aa5a76479bcb132bb46857e692d3b4096 100644
--- a/src/LIB/tensorproductmultigrid_Source/conjugategradient.f90
+++ b/src/LIB/tensorproductmultigrid_Source/conjugategradient.f90
@@ -235,7 +235,7 @@ contains
call scalarprod_mnh(m,r,z,rz_old)
! res0 <- ||r||
call scalarprod_mnh(m,r,r,res0)
- res0 = dsqrt(res0)
+ res0 = sqrt(res0)
if (cg_param%verbose > 0) then
if (i_am_master_mpi) then
write(STDOUT,'(" *** CG Solver ( ",I10," dof ) ***")') n_lin
@@ -257,7 +257,7 @@ contains
if (LUseO) call daxpy(n_lin,-alpha,Ap%s,1,r%s,1)
if (LUseT) call daxpy(n_lin,-alpha,Ap%st,1,r%st,1)
call scalarprod_mnh(m,r,r,res)
- res = dsqrt(res)
+ res = sqrt(res)
if (cg_param%verbose > 1) then
if (i_am_master_mpi) then
write(STDOUT,'(" <CG> Iteration ",I6," ||r|| = ",E12.6)') &
@@ -451,7 +451,7 @@ contains
call scalarprod(m,r,z,rz_old)
! res0 <- ||r||
call scalarprod(m,r,r,res0)
- res0 = dsqrt(res0)
+ res0 = sqrt(res0)
if (cg_param%verbose > 0) then
if (i_am_master_mpi) then
write(STDOUT,'(" *** CG Solver ( ",I10," dof ) ***")') n_lin
@@ -473,7 +473,7 @@ contains
if (LUseO) call daxpy(n_lin,-alpha,Ap%s,1,r%s,1)
if (LUseT) call daxpy(n_lin,-alpha,Ap%st,1,r%st,1)
call scalarprod(m,r,r,res)
- res = dsqrt(res)
+ res = sqrt(res)
if (cg_param%verbose > 1) then
if (i_am_master_mpi) then
write(STDOUT,'(" <CG> Iteration ",I6," ||r|| = ",E12.6)') &
diff --git a/src/LIB/tensorproductmultigrid_Source/datatypes.f90 b/src/LIB/tensorproductmultigrid_Source/datatypes.f90
index fddc325865d6341b19de414d7dffc746c0ca6d53..4fe009e6a4e9f505840e9f3f31053e19d1626c3c 100644
--- a/src/LIB/tensorproductmultigrid_Source/datatypes.f90
+++ b/src/LIB/tensorproductmultigrid_Source/datatypes.f90
@@ -41,8 +41,12 @@ module datatypes
#ifndef MNH
use mpi
+ use mpi, only NMNH_COMM_WORLD => MPI_COMM_WORLD
+ use mpi, only MNHREAL_MPI => MPI_DOUBLE_PRECISION
#else
use modd_mpif
+ USE MODD_VAR_ll, ONLY: NMNH_COMM_WORLD
+ use modd_precision, only: MNHREAL_MPI
#endif
use parameters
use messages
@@ -385,13 +389,13 @@ private
if (LUseO) then
call mpi_allreduce(tmp,global_tmp, 1, &
- MPI_DOUBLE_PRECISION,MPI_SUM,MPI_COMM_WORLD,ierr)
- global_tmp = dsqrt(global_tmp)
+ MNHREAL_MPI,MPI_SUM,NMNH_COMM_WORLD,ierr)
+ global_tmp = sqrt(global_tmp)
endif
if (LUseT) then
call mpi_allreduce(tmpt,global_tmpt, 1, &
- MPI_DOUBLE_PRECISION,MPI_SUM,MPI_COMM_WORLD,ierr)
- global_tmpt = dsqrt(global_tmpt)
+ MNHREAL_MPI,MPI_SUM,NMNH_COMM_WORLD,ierr)
+ global_tmpt = sqrt(global_tmpt)
end if
if (LUseO) then
l2norm_mnh = global_tmp
@@ -451,8 +455,8 @@ private
end if
call mpi_allreduce(tmp,global_tmp, 1, &
- MPI_DOUBLE_PRECISION,MPI_SUM,MPI_COMM_WORLD,ierr)
- l2norm = dsqrt(global_tmp)
+ MNHREAL_MPI,MPI_SUM,NMNH_COMM_WORLD,ierr)
+ l2norm = sqrt(global_tmp)
end function l2norm
!==================================================================
diff --git a/src/LIB/tensorproductmultigrid_Source/discretisation.f90 b/src/LIB/tensorproductmultigrid_Source/discretisation.f90
index 4929ad8a52862008152fea1c326c35d71b5356b1..06352e12b753d5000bd8ca36016856d39b8708c2 100644
--- a/src/LIB/tensorproductmultigrid_Source/discretisation.f90
+++ b/src/LIB/tensorproductmultigrid_Source/discretisation.f90
@@ -67,8 +67,11 @@ module discretisation
use communication
#ifndef MNH
use mpi
+ use mpi, only NMNH_COMM_WORLD => MPI_COMM_WORLD
+ use mpi, only MNH_STATUSES_IGNORE => MPI_STATUSES_IGNORE
#else
use modd_mpif
+ USE MODD_VAR_ll, ONLY: NMNH_COMM_WORLD, MNH_STATUSES_IGNORE
#endif
implicit none
@@ -1184,10 +1187,10 @@ end subroutine construct_vertical_coeff
call loop_over_grid_mnh(iblock)
if (overlap_comms) then
if (m > 0) then
- if (LUseO) call mpi_waitall(4,recv_requests, MPI_STATUSES_IGNORE, ierr)
- if (LUseO) call mpi_waitall(4,send_requests, MPI_STATUSES_IGNORE, ierr)
- if (LUseT) call mpi_waitall(4,recv_requestsT, MPI_STATUSES_IGNORE, ierr)
- if (LUseT) call mpi_waitall(4,send_requestsT, MPI_STATUSES_IGNORE, ierr)
+ if (LUseO) call mpi_waitall(4,recv_requests, MNH_STATUSES_IGNORE, ierr)
+ if (LUseO) call mpi_waitall(4,send_requests, MNH_STATUSES_IGNORE, ierr)
+ if (LUseT) call mpi_waitall(4,recv_requestsT, MNH_STATUSES_IGNORE, ierr)
+ if (LUseT) call mpi_waitall(4,send_requestsT, MNH_STATUSES_IGNORE, ierr)
end if
else
call haloswap_mnh(level,m,u)
@@ -1365,7 +1368,7 @@ end subroutine construct_vertical_coeff
call loop_over_grid(iblock)
if (overlap_comms) then
if (m > 0) then
- call mpi_waitall(4,recv_requests, MPI_STATUSES_IGNORE, ierr)
+ call mpi_waitall(4,recv_requests, MNH_STATUSES_IGNORE, ierr)
end if
else
call haloswap(level,m,u)
@@ -1675,10 +1678,10 @@ end subroutine construct_vertical_coeff
call loop_over_grid_jacobi_mnh(iblock)
if (overlap_comms) then
if (m > 0) then
- if (LUseO) call mpi_waitall(4,recv_requests, MPI_STATUSES_IGNORE, ierr)
- if (LUseO) call mpi_waitall(4,send_requests, MPI_STATUSES_IGNORE, ierr)
- if (LUseT) call mpi_waitall(4,recv_requestsT, MPI_STATUSES_IGNORE, ierr)
- if (LUseT) call mpi_waitall(4,send_requestsT, MPI_STATUSES_IGNORE, ierr)
+ if (LUseO) call mpi_waitall(4,recv_requests, MNH_STATUSES_IGNORE, ierr)
+ if (LUseO) call mpi_waitall(4,send_requests, MNH_STATUSES_IGNORE, ierr)
+ if (LUseT) call mpi_waitall(4,recv_requestsT, MNH_STATUSES_IGNORE, ierr)
+ if (LUseT) call mpi_waitall(4,send_requestsT, MNH_STATUSES_IGNORE, ierr)
end if
else
call haloswap_mnh(level,m,u)
@@ -1864,7 +1867,7 @@ end subroutine line_Jacobi_mnh
call loop_over_grid(iblock)
if (overlap_comms) then
if (m > 0) then
- call mpi_waitall(4,recv_requests, MPI_STATUSES_IGNORE, ierr)
+ call mpi_waitall(4,recv_requests, MNH_STATUSES_IGNORE, ierr)
end if
else
call haloswap(level,m,u)
diff --git a/src/LIB/tensorproductmultigrid_Source/messages.f90 b/src/LIB/tensorproductmultigrid_Source/messages.f90
index 7b4345777a92dcb235e12c456284ecc66b06c980..addcc9b9d9bad61b13b9ac6f48f89af7a57f5970 100644
--- a/src/LIB/tensorproductmultigrid_Source/messages.f90
+++ b/src/LIB/tensorproductmultigrid_Source/messages.f90
@@ -38,8 +38,10 @@ module messages
use parameters
#ifndef MNH
use mpi
+ use mpi, only NMNH_COMM_WORLD => MPI_COMM_WORLD
#else
use modd_mpif
+ USE MODD_VAR_ll, ONLY: NMNH_COMM_WORLD
#endif
implicit none
@@ -54,7 +56,7 @@ contains
character(len=*), intent(in) :: message
integer :: ierr, rank
integer, parameter :: errorcode = -1
- call mpi_comm_rank(MPI_COMM_WORLD,rank,ierr)
+ call mpi_comm_rank(NMNH_COMM_WORLD,rank,ierr)
if (rank == 0) then
write(STDERR,'("FATAL ERROR: ",A)') message
end if
@@ -69,7 +71,7 @@ contains
implicit none
character(len=*), intent(in) :: message
integer :: ierr, rank
- call mpi_comm_rank(MPI_COMM_WORLD,rank,ierr)
+ call mpi_comm_rank(NMNH_COMM_WORLD,rank,ierr)
if (rank == 0) then
write(STDERR,'("ERROR: ",A)') message
end if
@@ -82,7 +84,7 @@ contains
implicit none
character(len=*), intent(in) :: message
integer :: ierr, rank
- call mpi_comm_rank(MPI_COMM_WORLD,rank,ierr)
+ call mpi_comm_rank(NMNH_COMM_WORLD,rank,ierr)
if (rank == 0) then
write(STDERR,'("WARNING: ",A)') message
end if
@@ -95,7 +97,7 @@ contains
implicit none
character(len=*), intent(in) :: message
integer :: ierr, rank
- call mpi_comm_rank(MPI_COMM_WORLD,rank,ierr)
+ call mpi_comm_rank(NMNH_COMM_WORLD,rank,ierr)
if (rank == 0) then
write(STDERR,'("INFO: ",A)') message
end if
diff --git a/src/LIB/tensorproductmultigrid_Source/mode_mg_read_param.f90 b/src/LIB/tensorproductmultigrid_Source/mode_mg_read_param.f90
index 3ec13931a243b0469159a09b0a1d30f9b465782b..d61155c7de8dde99da4cd383fadd42289d99c8c5 100644
--- a/src/LIB/tensorproductmultigrid_Source/mode_mg_read_param.f90
+++ b/src/LIB/tensorproductmultigrid_Source/mode_mg_read_param.f90
@@ -38,8 +38,12 @@ subroutine read_general_parameters(filename,savefields_out)
use messages
#ifndef MNH
use mpi
+ use mpi, only NMNH_COMM_WORLD => MPI_COMM_WORLD
+ use mpi, only MNHREAL_MPI => MPI_DOUBLE_PRECISION
#else
use modd_mpif
+ USE MODD_VAR_ll, ONLY: NMNH_COMM_WORLD
+ use modd_precision, only: MNHREAL_MPI
#endif
implicit none
character(*), intent(in) :: filename
@@ -59,7 +63,7 @@ subroutine read_general_parameters(filename,savefields_out)
write(STDOUT,'("---------------------------------------------- ")')
write(STDOUT,'("")')
end if
- call mpi_bcast(savefields,1,MPI_LOGICAL,master_rank,MPI_COMM_WORLD,ierr)
+ call mpi_bcast(savefields,1,MPI_LOGICAL,master_rank,NMNH_COMM_WORLD,ierr)
savefields_out = savefields
end subroutine read_general_parameters
@@ -73,8 +77,10 @@ subroutine read_solver_parameters(filename,solver_param_out)
use messages
#ifndef MNH
use mpi
+ use mpi, only MNHREAL_MPI => MPI_DOUBLE_PRECISION
#else
use modd_mpif
+ use modd_precision, only: MNHREAL_MPI
#endif
implicit none
character(*), intent(in) :: filename
@@ -108,12 +114,12 @@ subroutine read_solver_parameters(filename,solver_param_out)
write(STDOUT,'("---------------------------------------------- ")')
write(*,'("")')
end if
- call mpi_bcast(solvertype,1,MPI_INTEGER,master_rank,MPI_COMM_WORLD,ierr)
- call mpi_bcast(maxiter,1,MPI_INTEGER,master_rank,MPI_COMM_WORLD,ierr)
- call mpi_bcast(resreduction,1,MPI_DOUBLE_PRECISION,master_rank,MPI_COMM_WORLD,ierr)
- call mpi_bcast(LMean,1,MPI_LOGICAL,master_rank,MPI_COMM_WORLD,ierr)
- call mpi_bcast(LUseO,1,MPI_LOGICAL,master_rank,MPI_COMM_WORLD,ierr)
- call mpi_bcast(LUseT,1,MPI_LOGICAL,master_rank,MPI_COMM_WORLD,ierr)
+ call mpi_bcast(solvertype,1,MPI_INTEGER,master_rank,NMNH_COMM_WORLD,ierr)
+ call mpi_bcast(maxiter,1,MPI_INTEGER,master_rank,NMNH_COMM_WORLD,ierr)
+ call mpi_bcast(resreduction,1,MNHREAL_MPI,master_rank,NMNH_COMM_WORLD,ierr)
+ call mpi_bcast(LMean,1,MPI_LOGICAL,master_rank,NMNH_COMM_WORLD,ierr)
+ call mpi_bcast(LUseO,1,MPI_LOGICAL,master_rank,NMNH_COMM_WORLD,ierr)
+ call mpi_bcast(LUseT,1,MPI_LOGICAL,master_rank,NMNH_COMM_WORLD,ierr)
solver_param_out%solvertype = solvertype
solver_param_out%maxiter = maxiter
solver_param_out%resreduction = resreduction
@@ -129,8 +135,10 @@ subroutine read_grid_parameters_mnh(filename,grid_param,KN,KNZ,PL,PH)
use messages
#ifndef MNH
use mpi
+ use mpi, only MNHREAL_MPI => MPI_DOUBLE_PRECISION
#else
use modd_mpif
+ use modd_precision, only: MNHREAL_MPI
#endif
implicit none
character(*), intent(in) :: filename
@@ -163,12 +171,12 @@ subroutine read_grid_parameters_mnh(filename,grid_param,KN,KNZ,PL,PH)
write(STDOUT,'("---------------------------------------------- ")')
write(STDOUT,'("")')
end if
- call mpi_bcast(n,1,MPI_INTEGER,master_rank,MPI_COMM_WORLD,ierr)
- call mpi_bcast(nz,1,MPI_INTEGER,master_rank,MPI_COMM_WORLD,ierr)
- call mpi_bcast(L,1,MPI_DOUBLE_PRECISION,master_rank,MPI_COMM_WORLD,ierr)
- call mpi_bcast(H,1,MPI_DOUBLE_PRECISION,master_rank,MPI_COMM_WORLD,ierr)
- call mpi_bcast(vertbc,1,MPI_INTEGER,master_rank,MPI_COMM_WORLD,ierr)
- call mpi_bcast(graded,1,MPI_LOGICAL,master_rank,MPI_COMM_WORLD,ierr)
+ call mpi_bcast(n,1,MPI_INTEGER,master_rank,NMNH_COMM_WORLD,ierr)
+ call mpi_bcast(nz,1,MPI_INTEGER,master_rank,NMNH_COMM_WORLD,ierr)
+ call mpi_bcast(L,1,MNHREAL_MPI,master_rank,NMNH_COMM_WORLD,ierr)
+ call mpi_bcast(H,1,MNHREAL_MPI,master_rank,NMNH_COMM_WORLD,ierr)
+ call mpi_bcast(vertbc,1,MPI_INTEGER,master_rank,NMNH_COMM_WORLD,ierr)
+ call mpi_bcast(graded,1,MPI_LOGICAL,master_rank,NMNH_COMM_WORLD,ierr)
IF (PRESENT(KN)) THEN
n = KN
nz = KNZ
@@ -204,8 +212,10 @@ subroutine read_grid_parameters(filename,grid_param)
use messages
#ifndef MNH
use mpi
+ use mpi, only MNHREAL_MPI => MPI_DOUBLE_PRECISION
#else
use modd_mpif
+ use modd_precision, only: MNHREAL_MPI
#endif
implicit none
@@ -241,12 +251,12 @@ subroutine read_grid_parameters(filename,grid_param)
write(STDOUT,'("---------------------------------------------- ")')
write(STDOUT,'("")')
end if
- call mpi_bcast(n,1,MPI_INTEGER,master_rank,MPI_COMM_WORLD,ierr)
- call mpi_bcast(nz,1,MPI_INTEGER,master_rank,MPI_COMM_WORLD,ierr)
- call mpi_bcast(L,1,MPI_DOUBLE_PRECISION,master_rank,MPI_COMM_WORLD,ierr)
- call mpi_bcast(H,1,MPI_DOUBLE_PRECISION,master_rank,MPI_COMM_WORLD,ierr)
- call mpi_bcast(vertbc,1,MPI_INTEGER,master_rank,MPI_COMM_WORLD,ierr)
- call mpi_bcast(graded,1,MPI_LOGICAL,master_rank,MPI_COMM_WORLD,ierr)
+ call mpi_bcast(n,1,MPI_INTEGER,master_rank,NMNH_COMM_WORLD,ierr)
+ call mpi_bcast(nz,1,MPI_INTEGER,master_rank,NMNH_COMM_WORLD,ierr)
+ call mpi_bcast(L,1,MNHREAL_MPI,master_rank,NMNH_COMM_WORLD,ierr)
+ call mpi_bcast(H,1,MNHREAL_MPI,master_rank,NMNH_COMM_WORLD,ierr)
+ call mpi_bcast(vertbc,1,MPI_INTEGER,master_rank,NMNH_COMM_WORLD,ierr)
+ call mpi_bcast(graded,1,MPI_LOGICAL,master_rank,NMNH_COMM_WORLD,ierr)
grid_param%n = n
grid_param%nz = nz
grid_param%L = L
@@ -266,8 +276,10 @@ subroutine read_comm_parameters(filename,comm_param)
use messages
#ifndef MNH
use mpi
+ use mpi, only MNHREAL_MPI => MPI_DOUBLE_PRECISION
#else
use modd_mpif
+ use modd_precision, only: MNHREAL_MPI
#endif
implicit none
character(*), intent(in) :: filename
@@ -291,7 +303,7 @@ subroutine read_comm_parameters(filename,comm_param)
halo_size = -1
end if
end if
- call mpi_bcast(halo_size,1,MPI_INTEGER,master_rank,MPI_COMM_WORLD,ierr)
+ call mpi_bcast(halo_size,1,MPI_INTEGER,master_rank,NMNH_COMM_WORLD,ierr)
comm_param%halo_size = halo_size
if (halo_size == -1) then
call fatalerror("Halo size has to be 1.")
@@ -308,8 +320,10 @@ subroutine read_model_parameters(filename,model_param)
use messages
#ifndef MNH
use mpi
+ use mpi, only MNHREAL_MPI => MPI_DOUBLE_PRECISION
#else
use modd_mpif
+ use modd_precision, only: MNHREAL_MPI
#endif
implicit none
character(*), intent(in) :: filename
@@ -331,9 +345,9 @@ subroutine read_model_parameters(filename,model_param)
write(STDOUT,'("---------------------------------------------- ")')
write(STDOUT,'("")')
end if
- call mpi_bcast(omega2,1,MPI_DOUBLE_PRECISION,master_rank,MPI_COMM_WORLD,ierr)
- call mpi_bcast(lambda2,1,MPI_DOUBLE_PRECISION,master_rank,MPI_COMM_WORLD,ierr)
- call mpi_bcast(delta,1,MPI_DOUBLE_PRECISION,master_rank,MPI_COMM_WORLD,ierr)
+ call mpi_bcast(omega2,1,MNHREAL_MPI,master_rank,NMNH_COMM_WORLD,ierr)
+ call mpi_bcast(lambda2,1,MNHREAL_MPI,master_rank,NMNH_COMM_WORLD,ierr)
+ call mpi_bcast(delta,1,MNHREAL_MPI,master_rank,NMNH_COMM_WORLD,ierr)
model_param%omega2 = omega2
model_param%lambda2 = lambda2
model_param%delta = delta
@@ -349,8 +363,10 @@ subroutine read_smoother_parameters(filename,smoother_param)
use messages
#ifndef MNH
use mpi
+ use mpi, only MNHREAL_MPI => MPI_DOUBLE_PRECISION
#else
use modd_mpif
+ use modd_precision, only: MNHREAL_MPI
#endif
implicit none
character(*), intent(in) :: filename
@@ -392,9 +408,9 @@ subroutine read_smoother_parameters(filename,smoother_param)
write(STDOUT,'("---------------------------------------------- ")')
write(STDOUT,'("")')
end if
- call mpi_bcast(smoother,1,MPI_INTEGER,master_rank,MPI_COMM_WORLD,ierr)
- call mpi_bcast(ordering,1,MPI_INTEGER,master_rank,MPI_COMM_WORLD,ierr)
- call mpi_bcast(rho,1,MPI_DOUBLE_PRECISION,master_rank,MPI_COMM_WORLD,ierr)
+ call mpi_bcast(smoother,1,MPI_INTEGER,master_rank,NMNH_COMM_WORLD,ierr)
+ call mpi_bcast(ordering,1,MPI_INTEGER,master_rank,NMNH_COMM_WORLD,ierr)
+ call mpi_bcast(rho,1,MNHREAL_MPI,master_rank,NMNH_COMM_WORLD,ierr)
smoother_param%smoother = smoother
smoother_param%ordering = ordering
smoother_param%rho = rho
@@ -417,8 +433,10 @@ subroutine read_multigrid_parameters(filename,mg_param)
use messages
#ifndef MNH
use mpi
+ use mpi, only MNHREAL_MPI => MPI_DOUBLE_PRECISION
#else
use modd_mpif
+ use modd_precision, only: MNHREAL_MPI
#endif
use MODE_OPENACC_SET_DEVICE, only : iswitch_cpu_gpu
@@ -493,17 +511,17 @@ subroutine read_multigrid_parameters(filename,mg_param)
write(*,'("")')
end if
- call mpi_bcast(verbose,1,MPI_INTEGER,master_rank,MPI_COMM_WORLD,ierr)
- call mpi_bcast(n_lev,1,MPI_INTEGER,master_rank,MPI_COMM_WORLD,ierr)
- call mpi_bcast(lev_split,1,MPI_INTEGER,master_rank,MPI_COMM_WORLD,ierr)
- call mpi_bcast(iswitch_cpu_gpu,1,MPI_INTEGER,master_rank,MPI_COMM_WORLD,ierr)
- call mpi_bcast(n_presmooth,1,MPI_INTEGER,master_rank,MPI_COMM_WORLD,ierr)
- call mpi_bcast(n_postsmooth,1,MPI_INTEGER,master_rank,MPI_COMM_WORLD,ierr)
- call mpi_bcast(n_coarsegridsmooth,1,MPI_INTEGER,master_rank,MPI_COMM_WORLD, &
+ call mpi_bcast(verbose,1,MPI_INTEGER,master_rank,NMNH_COMM_WORLD,ierr)
+ call mpi_bcast(n_lev,1,MPI_INTEGER,master_rank,NMNH_COMM_WORLD,ierr)
+ call mpi_bcast(lev_split,1,MPI_INTEGER,master_rank,NMNH_COMM_WORLD,ierr)
+ call mpi_bcast(iswitch_cpu_gpu,1,MPI_INTEGER,master_rank,NMNH_COMM_WORLD,ierr)
+ call mpi_bcast(n_presmooth,1,MPI_INTEGER,master_rank,NMNH_COMM_WORLD,ierr)
+ call mpi_bcast(n_postsmooth,1,MPI_INTEGER,master_rank,NMNH_COMM_WORLD,ierr)
+ call mpi_bcast(n_coarsegridsmooth,1,MPI_INTEGER,master_rank,NMNH_COMM_WORLD, &
ierr)
- call mpi_bcast(prolongation,1,MPI_INTEGER,master_rank,MPI_COMM_WORLD,ierr)
- call mpi_bcast(restriction,1,MPI_INTEGER,master_rank,MPI_COMM_WORLD,ierr)
- call mpi_bcast(coarsegridsolver,1,MPI_Integer,master_rank,MPI_COMM_WORLD,ierr)
+ call mpi_bcast(prolongation,1,MPI_INTEGER,master_rank,NMNH_COMM_WORLD,ierr)
+ call mpi_bcast(restriction,1,MPI_INTEGER,master_rank,NMNH_COMM_WORLD,ierr)
+ call mpi_bcast(coarsegridsolver,1,MPI_Integer,master_rank,NMNH_COMM_WORLD,ierr)
mg_param%verbose = verbose
mg_param%n_lev = n_lev
mg_param%lev_split = lev_split
@@ -535,8 +553,10 @@ subroutine read_conjugategradient_parameters(filename,cg_param)
use messages
#ifndef MNH
use mpi
+ use mpi, only MNHREAL_MPI => MPI_DOUBLE_PRECISION
#else
use modd_mpif
+ use modd_precision, only: MNHREAL_MPI
#endif
implicit none
character(*), intent(in) :: filename
@@ -563,10 +583,10 @@ subroutine read_conjugategradient_parameters(filename,cg_param)
write(STDOUT,'("---------------------------------------------- ")')
write(STDOUT,'("")')
end if
- call mpi_bcast(verbose,1,MPI_INTEGER,master_rank,MPI_COMM_WORLD,ierr)
- call mpi_bcast(maxiter,1,MPI_INTEGER,master_rank,MPI_COMM_WORLD,ierr)
- call mpi_bcast(resreduction,1,MPI_DOUBLE_PRECISION,master_rank,MPI_COMM_WORLD,ierr)
- call mpi_bcast(n_prec,1,MPI_INTEGER,master_rank,MPI_COMM_WORLD,ierr)
+ call mpi_bcast(verbose,1,MPI_INTEGER,master_rank,NMNH_COMM_WORLD,ierr)
+ call mpi_bcast(maxiter,1,MPI_INTEGER,master_rank,NMNH_COMM_WORLD,ierr)
+ call mpi_bcast(resreduction,1,MNHREAL_MPI,master_rank,NMNH_COMM_WORLD,ierr)
+ call mpi_bcast(n_prec,1,MPI_INTEGER,master_rank,NMNH_COMM_WORLD,ierr)
cg_param%verbose = verbose
cg_param%maxiter = maxiter
cg_param%resreduction = resreduction
diff --git a/src/LIB/tensorproductmultigrid_Source/multigrid.f90 b/src/LIB/tensorproductmultigrid_Source/multigrid.f90
index 217ad17235bfe0de7b88583e7570af4fcb6c6a82..ff1a75b0ffdbedacf31c74163ee9874ed3630335 100644
--- a/src/LIB/tensorproductmultigrid_Source/multigrid.f90
+++ b/src/LIB/tensorproductmultigrid_Source/multigrid.f90
@@ -38,8 +38,12 @@ module multigrid
#ifndef MNH
use mpi
+ use mpi, only NMNH_COMM_WORLD => MPI_COMM_WORLD
+ use mpi, only MNH_STATUSES_IGNORE => MPI_STATUSES_IGNORE
#else
use modd_mpif
+ USE MODD_VAR_ll, ONLY: NMNH_COMM_WORLD
+ USE MODD_VAR_ll, ONLY: NMNH_COMM_WORLD, MNH_STATUSES_IGNORE
#endif
use parameters
use datatypes
@@ -757,10 +761,10 @@ contains
end if
if (overlap_comms) then
if (m > 0) then
- if (LUseO) call mpi_waitall(4,recv_requests, MPI_STATUSES_IGNORE, ierr)
- if (LUseO) call mpi_waitall(4,send_requests, MPI_STATUSES_IGNORE, ierr)
- if (LUseT) call mpi_waitall(4,recv_requestsT, MPI_STATUSES_IGNORE, ierr)
- if (LUseT) call mpi_waitall(4,send_requestsT, MPI_STATUSES_IGNORE, ierr)
+ if (LUseO) call mpi_waitall(4,recv_requests, MNH_STATUSES_IGNORE, ierr)
+ if (LUseO) call mpi_waitall(4,send_requests, MNH_STATUSES_IGNORE, ierr)
+ if (LUseT) call mpi_waitall(4,recv_requestsT, MNH_STATUSES_IGNORE, ierr)
+ if (LUseT) call mpi_waitall(4,send_requestsT, MNH_STATUSES_IGNORE, ierr)
end if
else
call haloswap_mnh(level,m,phifine)
@@ -993,7 +997,7 @@ contains
end if
if (overlap_comms) then
if (m > 0) then
- call mpi_waitall(4,recv_requests, MPI_STATUSES_IGNORE, ierr)
+ call mpi_waitall(4,recv_requests, MNH_STATUSES_IGNORE, ierr)
end if
else
call haloswap(level,m,phifine)
@@ -1691,7 +1695,7 @@ contains
end if
end if
- call mpi_barrier(MPI_COMM_WORLD,ierr)
+ call mpi_barrier(NMNH_COMM_WORLD,ierr)
call start_timer(t_mainloop)
if (solvertype == SOLVER_CG) then
! NB: b(level,pproc) will be used as r in the following
@@ -1907,7 +1911,7 @@ contains
end if
end if
- call mpi_barrier(MPI_COMM_WORLD,ierr)
+ call mpi_barrier(NMNH_COMM_WORLD,ierr)
call start_timer(t_mainloop)
if (solvertype == SOLVER_CG) then
! NB: b(level,pproc) will be used as r in the following
diff --git a/src/LIB/tensorproductmultigrid_Source/parameters.f90 b/src/LIB/tensorproductmultigrid_Source/parameters.f90
index 5195982fdeffc44390f2a2fa9fd116f6e0d76660..e37fb2a51dd4dff27ff74416be911a88de1cf653 100644
--- a/src/LIB/tensorproductmultigrid_Source/parameters.f90
+++ b/src/LIB/tensorproductmultigrid_Source/parameters.f90
@@ -41,8 +41,12 @@ module parameters
! floating point precision. Always use rl_kind in code
integer, parameter :: single_precision=4 ! single precision
integer, parameter :: double_precision=8 ! double precision
+#ifndef MNH
integer, parameter :: rl=double_precision ! global switch between
! single/double precision
+#else
+ integer, parameter :: rl=MNH_REAL
+#endif
! NOTE: As we still use BLAS subroutines, these need to be
! replaced as well when switching between double and
! single precision!
diff --git a/src/LIB/tensorproductmultigrid_Source/timer.f90 b/src/LIB/tensorproductmultigrid_Source/timer.f90
index da0f17907a28016c0aad1fee4b78d348cf72ecfd..451f933ee11d5a4408168cfe015d2ad3a9485851 100644
--- a/src/LIB/tensorproductmultigrid_Source/timer.f90
+++ b/src/LIB/tensorproductmultigrid_Source/timer.f90
@@ -38,8 +38,12 @@ module timer
#ifndef MNH
use mpi
+ use mpi, only NMNH_COMM_WORLD => MPI_COMM_WORLD
+ use mpi, only MNHREAL_MPI => MPI_DOUBLE_PRECISION
#else
use modd_mpif
+ USE MODD_VAR_ll, ONLY: NMNH_COMM_WORLD
+ use modd_precision, only: MNHREAL_MPI
#endif
use parameters
@@ -79,7 +83,7 @@ contains
subroutine initialise_timing(filename)
implicit none
character(len=*), intent(in) :: filename
- call mpi_comm_rank(MPI_COMM_WORLD,rank,ierr)
+ call mpi_comm_rank(NMNH_COMM_WORLD,rank,ierr)
if (rank==0) then
open(UNIT=TIMEROUT,FILE=trim(filename))
write(STDOUT,'("Writing timer information to file ",A40)') filename
@@ -165,14 +169,14 @@ contains
elapsedtime = (t%elapsed) * factor
- call mpi_reduce(elapsedtime,t_min,1,MPI_DOUBLE_PRECISION, &
- MPI_MIN, 0, MPI_COMM_WORLD,ierr)
- call mpi_reduce(elapsedtime,t_avg,1,MPI_DOUBLE_PRECISION, &
- MPI_SUM, 0, MPI_COMM_WORLD,ierr)
- call mpi_reduce(elapsedtime,t_max,1,MPI_DOUBLE_PRECISION, &
- MPI_MAX, 0, MPI_COMM_WORLD,ierr)
- call mpi_comm_size(MPI_COMM_WORLD,nprocs,ierr)
- call mpi_comm_rank(MPI_COMM_WORLD,rank,ierr)
+ call mpi_reduce(elapsedtime,t_min,1,MNHREAL_MPI, &
+ MPI_MIN, 0, NMNH_COMM_WORLD,ierr)
+ call mpi_reduce(elapsedtime,t_avg,1,MNHREAL_MPI, &
+ MPI_SUM, 0, NMNH_COMM_WORLD,ierr)
+ call mpi_reduce(elapsedtime,t_max,1,MNHREAL_MPI, &
+ MPI_MAX, 0, NMNH_COMM_WORLD,ierr)
+ call mpi_comm_size(NMNH_COMM_WORLD,nprocs,ierr)
+ call mpi_comm_rank(NMNH_COMM_WORLD,rank,ierr)
t_avg = t_avg/nprocs
nc = t%ncall
if (nc == 0) nc = 1
diff --git a/src/MNH/gradient_m.f90 b/src/MNH/gradient_m.f90
index 0a5fa93268dc88af4ecde5be0b4035e73560077d..68888988b80a4bb0b270481af916aa7c13e9f107 100644
--- a/src/MNH/gradient_m.f90
+++ b/src/MNH/gradient_m.f90
@@ -657,7 +657,7 @@ call Print_msg( NVERB_WARNING, 'GEN', 'GZ_M_M_DEVICE', 'OpenACC: not yet tested'
! --------------------
!
CALL DZM_DEVICE( PA(:,:,:), ZTMP1(:,:,:) )
-!$acc kernels
+!$acc kernels present_cr(ZTMP2)
ZTMP2(:,:,:) = ZTMP1(:,:,:) / PDZZ(:,:,:)
!$acc end kernels
CALL MZF_DEVICE( ZTMP2(:,:,:), PGZ_M_M(:,:,:) )
diff --git a/src/MNH/ice4_fast_rh.f90 b/src/MNH/ice4_fast_rh.f90
index 04c1d71eed20ff8b52631deaafe84e6a4e9d7590..2ca8082110c0a88588b7d3430284a5483db93946 100644
--- a/src/MNH/ice4_fast_rh.f90
+++ b/src/MNH/ice4_fast_rh.f90
@@ -569,7 +569,7 @@ ELSE
PRH_TEND(:, IRRWETH)=0.
!$acc end kernels
IF(IGWET>0)THEN
-!$acc kernels present_cr(ZVEC1,ZVEC2)
+!$acc kernels present_cr(ZVEC1,ZVEC2,ZVEC3)
!
!* 7.2.12 select the (PLBDAH,PLBDAR) couplet
!
diff --git a/src/MNH/ice4_nucleation_wrapper.f90 b/src/MNH/ice4_nucleation_wrapper.f90
index 74df1318aead24a5cac0de764f031c4a2bcadf42..37543ce88d9bafce5a67eccefadfdb215801edf8 100644
--- a/src/MNH/ice4_nucleation_wrapper.f90
+++ b/src/MNH/ice4_nucleation_wrapper.f90
@@ -146,7 +146,7 @@ CALL MNH_MEM_GET( gnegt, kit, kjt, kkt )
! optimization by looking for locations where
! the temperature is negative only !!!
!
-!$acc kernels
+!$acc kernels present_cr(GNEGT)
GNEGT(:,:,:)=PT(:,:,:)<XTT .AND. LDMASK
INEGT = COUNT(GNEGT(:,:,:))
!$acc end kernels
diff --git a/src/MNH/ice4_tendencies.f90 b/src/MNH/ice4_tendencies.f90
index 4ccb0613056c4844ebb85626e4eecb82088a2248..b7712efae6e53f24f444533d3feba567e5601e92 100644
--- a/src/MNH/ice4_tendencies.f90
+++ b/src/MNH/ice4_tendencies.f90
@@ -637,7 +637,7 @@ IF(ISIZE>0) THEN
PHLI_HCF, PHLI_LCF, PHLI_HRI, PHLI_LRI, ZRF)
IF(HSUBG_RC_RR_ACCR=='PRFR' .OR. HSUBG_RR_EVAP=='PRFR') THEN
!Diagnostic of precipitation fraction
-!$acc kernels
+!$acc kernels present_cr(ZRRT3D,ZRST3D,ZRGT3D,ZRHT3D)
PRAINFR(:,:,:) = 0.
ZRRT3D (:,:,:) = 0.
ZRST3D (:,:,:) = 0.
diff --git a/src/MNH/mode_openacc_set_device.f90 b/src/MNH/mode_openacc_set_device.f90
index 5a9585d7db1417d98ec3f8dea24940dd7e908d5e..d897963127e57f380625b25f59ae40aa561c3d4a 100644
--- a/src/MNH/mode_openacc_set_device.f90
+++ b/src/MNH/mode_openacc_set_device.f90
@@ -36,7 +36,7 @@ CONTAINS
if ( mnh_idevice_type_at_init .EQ. -1000 ) then
mnh_idevice_type_at_init = acc_get_device_type()
mnh_idevice_type_current = mnh_idevice_type_at_init
- print*,'mnh_idevice_type_at_init=',mnh_idevice_type_at_init
+ !print*,'mnh_idevice_type_at_init=',mnh_idevice_type_at_init
end if
#endif
@@ -56,7 +56,7 @@ CONTAINS
idevice_type = acc_get_device_type()
mnh_idevice_type_current = idevice_type
- print*,'idevice_type=',idevice_type
+ !print*,'idevice_type=',idevice_type
#endif
END SUBROUTINE MNH_OPENACC_GET_DEVICE
diff --git a/src/MNH/prep_ideal_case.f90 b/src/MNH/prep_ideal_case.f90
index c04dd33a34248af2fd9d099a7d8a861466168191..d01b3b59887d6d1dbd70dbd03df87767285346dc 100644
--- a/src/MNH/prep_ideal_case.f90
+++ b/src/MNH/prep_ideal_case.f90
@@ -446,6 +446,11 @@ USE MODD_GET_n
!
USE MODN_CONFIO, ONLY : NAM_CONFIO
!
+#ifdef MNH_OPENACC
+USE MODE_MNH_ZWORK
+USE MODD_HALO_D
+#endif
+!
IMPLICIT NONE
!
!* 0.1 Declarations of global variables not declared in the modules
@@ -1210,6 +1215,10 @@ END IF
CALL MNH_ALLOC_ZWORK( 1 )
#endif
!
+#ifdef MNH_OPENACC
+CALL MNH_ALLOC_ZWORK(1)
+CALL INIT_HALO_D()
+#endif
!
!-------------------------------------------------------------------------------
!
diff --git a/src/MNH/rain_ice_fast_rh.f90 b/src/MNH/rain_ice_fast_rh.f90
index 83e4237fd3ee0cf3518ba7331ac141b49d3fae26..89b735b2c3cdea323fd3576d27192e35f4cc5089 100644
--- a/src/MNH/rain_ice_fast_rh.f90
+++ b/src/MNH/rain_ice_fast_rh.f90
@@ -178,7 +178,7 @@ CALL MNH_MEM_GET( ZZW1, SIZE(PRHODREF), 7 )
!$acc data present( I1H, I1W, GWORK, ZZW, ZZW1 )
#endif
-!$acc kernels
+!$acc kernels present_cr(GWORK)
GWORK(:) = PRHT(:)>XRTMIN(7)
!$acc end kernels
#ifndef MNH_OPENACC
@@ -229,7 +229,7 @@ CALL MNH_MEM_GET( ZZW1, SIZE(PRHODREF), 7 )
!
!* 7.2.1 accretion of aggregates on the hailstones
!
-!$acc kernels
+!$acc kernels present_cr(GWORK)
GWORK(1:IHAIL) = PRST(I1H(1:IHAIL))>XRTMIN(5) .AND. PRSS(I1H(1:IHAIL))>0.0
!$acc end kernels
#ifndef MNH_OPENACC
@@ -327,7 +327,7 @@ CALL MNH_MEM_GET( ZZW1, SIZE(PRHODREF), 7 )
!
!* 7.2.6 accretion of graupeln on the hailstones
!
-!$acc kernels
+!$acc kernels present_cr(GWORK)
GWORK(1:IHAIL) = PRGT(I1H(1:IHAIL))>XRTMIN(6) .AND. PRGS(I1H(1:IHAIL))>0.0
!$acc end kernels
#ifndef MNH_OPENACC
diff --git a/src/MNH/turb_hor_splt.f90 b/src/MNH/turb_hor_splt.f90
index 2f920f93e55efab5dd207079d29360aacea47815..684f888911e78494c952076402630ad7c2cfc31a 100644
--- a/src/MNH/turb_hor_splt.f90
+++ b/src/MNH/turb_hor_splt.f90
@@ -556,7 +556,7 @@ IF (KSPLIT>1 .AND. CPROGRAM=='MESONH') THEN
! ---------------
!
!
- !$acc kernels
+ !$acc kernels present_cr(ZSVM,ZRM)
ZUM=PUM
ZVM=PVM
ZWM=PWM
@@ -613,7 +613,7 @@ IF (KSPLIT>1 .AND. CPROGRAM=='MESONH') THEN
#else
CALL MXM_DEVICE(PRHODJ,ZTMP1_DEVICE)
CALL MYM_DEVICE(PRHODJ,ZTMP2_DEVICE)
-!$acc kernels
+!$acc kernels present_cr(ZSVM,ZRM)
ZUM=PUM+(ZRUS/KSPLIT-PRUS)/ZTMP1_DEVICE*PTSTEP
ZVM=PVM+(ZRVS/KSPLIT-PRVS)/ZTMP2_DEVICE*PTSTEP
#endif
@@ -659,7 +659,7 @@ IF (KSPLIT>1 .AND. CPROGRAM=='MESONH') THEN
#endif
!
IF ( HLBCX(1) /= "CYCL" .AND. LWEST_ll()) THEN
-!$acc kernels
+!$acc kernels present_cr(ZSVM,ZRM)
ZUM(IIB ,:,:)=PUM(IIB ,:,:)
ZVM(IIB-1,:,:)=PVM(IIB-1,:,:)
ZWM(IIB-1,:,:)=PWM(IIB-1,:,:)
@@ -671,7 +671,7 @@ IF (KSPLIT>1 .AND. CPROGRAM=='MESONH') THEN
ENDIF
!
IF ( HLBCX(2) /= "CYCL" .AND. LEAST_ll()) THEN
-!$acc kernels
+!$acc kernels present_cr(ZSVM,ZRM)
ZUM(IIE+1,:,:)=PUM(IIE+1,:,:)
ZVM(IIE+1,:,:)=PVM(IIE+1,:,:)
ZWM(IIE+1,:,:)=PWM(IIE+1,:,:)
@@ -683,7 +683,7 @@ IF (KSPLIT>1 .AND. CPROGRAM=='MESONH') THEN
ENDIF
!
IF ( HLBCY(1) /= "CYCL" .AND. LSOUTH_ll()) THEN
-!$acc kernels
+!$acc kernels present_cr(ZSVM,ZRM)
ZUM(:,IJB-1,:)=PUM(:,IJB-1,:)
ZVM(:,IJB ,:)=PVM(:,IJB ,:)
ZWM(:,IJB-1,:)=PWM(:,IJB-1,:)
@@ -695,7 +695,7 @@ IF (KSPLIT>1 .AND. CPROGRAM=='MESONH') THEN
ENDIF
!
IF ( HLBCY(2) /= "CYCL" .AND. LNORTH_ll()) THEN
-!$acc kernels
+!$acc kernels present_cr(ZSVM,ZRM)
ZUM(:,IJE+1,:)=PUM(:,IJE+1,:)
ZVM(:,IJE+1,:)=PVM(:,IJE+1,:)
ZWM(:,IJE+1,:)=PWM(:,IJE+1,:)
@@ -706,7 +706,7 @@ IF (KSPLIT>1 .AND. CPROGRAM=='MESONH') THEN
!$acc end kernels
ENDIF
!
-!$acc kernels
+!$acc kernels present_cr(ZSVM,ZRM)
ZUM(:,:,IKB-1)=ZUM(:,:,IKB)
ZVM(:,:,IKB-1)=ZVM(:,:,IKB)
ZWM(:,:,IKB-1)=ZWM(:,:,IKB)
diff --git a/src/MNH/turb_hor_tke.f90 b/src/MNH/turb_hor_tke.f90
index f724b68e3c183261e475f1b9c9cbae2b8f0fc9e9..a5ac4acdf06bff01bcd905d29859d69c61cafe0c 100644
--- a/src/MNH/turb_hor_tke.f90
+++ b/src/MNH/turb_hor_tke.f90
@@ -319,7 +319,7 @@ IF (.NOT. LFLAT) THEN
!$acc end kernels
ELSE
CALL MXM_DEVICE(PRHODJ,ZTMP1_DEVICE)
- !$acc kernels
+ !$acc kernels present_cr(ZTMP2_DEVICE,ZTMP1_DEVICE)
ZTMP2_DEVICE = ZTMP1_DEVICE * ZFLX * PINV_PDXX
!$acc end kernels
CALL DXF_DEVICE(ZTMP2_DEVICE,ZTMP1_DEVICE)
@@ -458,7 +458,7 @@ IF (.NOT. L2D) THEN
!$acc end kernels
ELSE
CALL MYM_DEVICE(PRHODJ,ZTMP1_DEVICE)
- !$acc kernels
+ !$acc kernels present_cr(ZTMP2_DEVICE,ZTMP1_DEVICE)
ZTMP2_DEVICE = ZTMP1_DEVICE * ZFLX * PINV_PDYY
!$acc end kernels
CALL DYF_DEVICE(ZTMP2_DEVICE,ZTMP1_DEVICE)
diff --git a/src/Makefile b/src/Makefile
index ee69febd6d0ad21fe7075d4e0278e34ea5c4dfa6..da60ed11c6376865888a644934c1204621538665 100644
--- a/src/Makefile
+++ b/src/Makefile
@@ -322,9 +322,9 @@ $(ECCODES_MOD) :
cd ${DIR_ECCODES_BUILD} && \
AEC_PATH=$(CDF_PATH) cmake ${DIR_ECCODES_SRC} -DCMAKE_INSTALL_PREFIX=${DIR_ECCODES_INSTALL} -DBUILD_SHARED_LIBS=OFF \
-DENABLE_NETCDF=OFF -DENABLE_JPG=OFF -DENABLE_PYTHON=OFF -DENABLE_EXAMPLES=OFF \
- -DCMAKE_Fortran_COMPILER=$(FC) -DCMAKE_C_COMPILER=$(CC) \
- -DCMAKE_Fortran_FLAGS=$(ECCODES_FFLAGS) -DCMAKE_C_FLAGS=$(ECCODES_CFLAGS) ${EC_CONF} && \
- $(MAKE) && $(MAKE) install && $(MAKE) clean
+ -DCMAKE_Fortran_COMPILER=$(FC) -DCMAKE_C_COMPILER="$(CC)" \
+ -DCMAKE_Fortran_FLAGS="$(ECCODES_FFLAGS)" -DCMAKE_C_FLAGS="$(ECCODES_CFLAGS)" ${EC_CONF} && \
+ $(MAKE) && $(MAKE) install && $(MAKE) clean && [ -d ${DIR_ECCODES_BUILD} ] && rm -fr ${DIR_ECCODES_BUILD}
eccodes_lib_clean :
- [ -d ${DIR_ECCODES_BUILD} ] && rm -fr ${DIR_ECCODES_BUILD}
diff --git a/src/Rules.LXarm.mk b/src/Rules.LXarm.mk
index 8cc83a2459f6d0109ffe63a3e91823d8a141038c..beb8f78f1aa0d7f05c57bf4139ada0e7e216ff2b 100644
--- a/src/Rules.LXarm.mk
+++ b/src/Rules.LXarm.mk
@@ -14,13 +14,16 @@ GFV=123
#
#OBJDIR_PATH=/home/escj/azertyuiopqsdfghjklm/wxcvbn/azertyuiopqsdfghjklmwxcvbn
#
-OPT_BASE = -g -fno-backslash
+OPT_BASE = -g -fPIC -fno-backslash -gdwarf-4
#OPT_BASE = -g -fPIC -fno-backslash
#
-OPT_PERF0 = -O0 -mcpu=native
-OPT_PERF2 = -O2 -mcpu=native
+#TP= -mcpu=native
+TP ?= -mcpu=neoverse-n1
+
+OPT_PERF0 = -O0 $(TP)
+OPT_PERF2 = -O2 $(TP)
# -mcpu=thunderx2t99 neoverse-n1
-OPT_PERF3 = -O3 -mcpu=native
+OPT_PERF3 = -O3 $(TP)
#-mcpu=thunderx2t99 -ffp-contract=fast
OPT_CHECK =
OPT_I8 = -i8
@@ -50,6 +53,13 @@ OPT = $(OPT_BASE) $(OPT_PERF2)
OPT0 = $(OPT_BASE) $(OPT_PERF0)
OPT_NOCB = $(OPT_BASE) $(OPT_PERF2)
#
+# Source to compile without 'simd' or 'fma' for bit-reproductibility
+BJS_REPROD= spll_mode_sum_ll.o mode_sum_ll.mod spll_mode_repro_sum.o mode_repro_sum.mod
+#
+# BITREP flags <-> use BR_ routines
+#
+MNH_BITREP = NO
+#
ifeq "$(OPTLEVEL)" "DEBUG"
OPT = $(OPT_BASE) $(OPT_PERF0) $(OPT_CHECK)
OPT0 = $(OPT_BASE) $(OPT_PERF0) $(OPT_CHECK)
@@ -57,6 +67,14 @@ OPT_NOCB = $(OPT_BASE) $(OPT_PERF0)
CFLAGS += -g -O0
endif
#
+ifeq "$(OPTLEVEL)" "OPENACCDEFONLY"
+CPPFLAGS += -DMNH_OPENACC -D_FAKEOPENACC
+OPT = $(OPT_BASE) $(OPT_NOOPENACC) $(OPT_PERF2)
+OPT0 = $(OPT_BASE) $(OPT_NOOPENACC) $(OPT_PERF0)
+OPT_NOCB = $(OPT_BASE) $(OPT_NOOPENACC) $(OPT_PERF2)
+CXXFLAGS = -Kieee -Mnofma $(OPT_NOOPENACC)
+$(OBJS_REPROD) : OPT = $(OPT_BASE) $(OPT_NOOPENACC) $(OPT_PERF2) -ffp-contract=off
+endif
ifeq "$(OPTLEVEL)" "O3"
OPT = $(OPT_BASE) $(OPT_PERF3)
OPT0 = $(OPT_BASE) $(OPT_PERF0)
@@ -65,6 +83,7 @@ endif
#
#
CC = armclang
+CXX = armclang++
FC = armflang
ifeq "$(VER_MPI)" "MPIAUTO"
F90 = mpif90
@@ -94,7 +113,15 @@ CPPFLAGS_NEWLFI = -DSWAPIO -DLINUX -DLFI_INT=${LFI_INT}
CPPFLAGS_MNH = -DMNH -DSFX_MNH
CPPFLAGS_MNH += -imacros MNH_OPENACC_NV_CR.CPP
CPPFLAGS_MNH += -DMNH_EXPAND
-
+#
+# BITREP flags
+#
+#if MNH_BITREP exists => compile with the BITREP library
+MNH_BITREP ?= YES
+ifeq "$(MNH_BITREP)" "YES"
+CPPFLAGS_MNH += -DMNH_BITREP
+endif
+#
ifdef VER_GA
CPPFLAGS_SURCOUCHE += -DMNH_GA
INC += -I${GA_ROOT}/include
@@ -103,7 +130,9 @@ endif
#
# Netcdf/HDF5 flags
#
-HDF_CONF= CFLAGS=-std=c99
+#HDF_CONF= CFLAGS=-std=c99
+NETCDF_OPT ?= $(TP) -fPIC -gdwarf-4
+HDF_OPT ?= $(TP) -fPIC -std=c99 -gdwarf-4
#
## LIBTOOLS flags
#
@@ -139,6 +168,8 @@ MNH_MGSOLVER ?= YES
# if set to yes: use grib_api (deprecated library)
#
MNH_GRIBAPI=no
+ECCODES_FFLAGS= $(TP) -gdwarf-4
+ECCODES_CFLAGS= $(TP) -gdwarf-4
#
##########################################################
# #
@@ -167,11 +198,10 @@ OBJS_I4=spll_modd_netcdf.o
$(OBJS_I4) : OPT = $(OPT_BASE_I4)
endif
-ifeq ($(shell test $(GFV) -le 482 ; echo $$?),0)
ifneq "$(OPTLEVEL)" "DEBUG"
-OBJS_O0= spll_lima_phillips_integ.o
-$(OBJS_O0) : OPT = $(OPT_BASE) $(OPT_PERF0)
-endif
+OBJS_O1= spll_modd_pack_gr_field_n.o modd_pack_gr_field_n.mod spll_modd_les_n.o modd_les_n.mod spll_write_lfifm1_for_diag.o
+$(OBJS_O1) : OPT = $(OPT_BASE) $(OPT_PERF1)
endif
+
SPLL = spll_new
diff --git a/src/Rules.LXcray.mk b/src/Rules.LXcray.mk
index 9d23c4171761b42a1a7944cbf9b904e87bc114c0..673d668efc1148c4e9a49b35f9173ed691dd8fc5 100644
--- a/src/Rules.LXcray.mk
+++ b/src/Rules.LXcray.mk
@@ -14,7 +14,7 @@ CFV=$(shell crayftn --version | grep -E -m1 -o ' [[:digit:]\.]{2,}( |$$)' | sed
#OBJDIR_PATH=/home/escj/azertyuiopqsdfghjklm/wxcvbn/azertyuiopqsdfghjklmwxcvbn
#
OPT_BASE = -hpic -emf -h acc_model=auto_async_none:no_fast_addr:no_deep_copy -halias=none
-OPT_PERF0 = -O0 -g
+OPT_PERF0 = -G0 -O0
OPT_PERF1 = -G2 -O1 -hflex_mp=intolerant -Ofp0 -hnofma -hnoomp -K trap=fp -hipa0
OPT_PERF2 = -G2 -O2 -hflex_mp=intolerant -Ofp0 -hnofma -hnoomp -hipa0
#
@@ -81,6 +81,7 @@ OPT_NOCB = $(OPT_BASE) $(OPT_PERF2) $(OPT_OPENACC)
OBJS_CONCURRENT=spll_multigrid.o spll_turb_ver_thermo_flux.o
#spll_resolved_cloud.o
$(OBJS_CONCURRENT) : OPT = $(OPT_BASE) $(OPT_PERF2) $(OPT_OPENACC) -h concurrent -rimo
+OBJS0 += spll_ini_model_n.o
endif
ifeq "$(OPTLEVEL)" "OPENACCNOBITREP"
@@ -254,14 +255,20 @@ MNH_MGSOLVER ?= YES
#
MNH_GRIBAPI=no
#
+ifeq "$(VER_CDF)" "CDFCRAY"
+INC += $(NETCDF_DIR)/include
+CDF_PATH = $(NETCDF_DIR)
+else
NETCDF_SUPPFLAGS='-emf'
-ECCODES_FFLAGS='-hcpu=x86-64 -emf -hsystem_alloc'
-ECCODES_CFLAGS='-hcpu=x86-64 -gdwarf-4 '
-EC_CONF='-DIEEE_LE=1'
+ECCODES_FFLAGS= -hcpu=x86-64 -emf
+# -hsystem_alloc
+ECCODES_CFLAGS= -hcpu=x86-64 -gdwarf-4
+EC_CONF= -DCMAKE_BUILD_TYPE="RELEASE" -DIEEE_LE=1
CDF_CONF= CPP=cpp
HDF_CONF= CPP=cpp
HDF_OPT ?= -hcpu=x86-64
NETCDF_OPT ?= -hcpu=x86-64
+endif
#
# Bypat nvidia70 problème
ifeq "$(CRAY_ACCEL_TARGET)" "nvidia70"
diff --git a/src/Rules.LXgfortran.mk b/src/Rules.LXgfortran.mk
index c9c93db8f82097adfd5dc54bc4fc8c8f20ab4ec7..c171e1ebad660877d3dfd4341f157eaed1ed1275 100644
--- a/src/Rules.LXgfortran.mk
+++ b/src/Rules.LXgfortran.mk
@@ -74,9 +74,17 @@ CXXFLAGS = $(OPT_OPENACC)
#$(OBJS_REPROD) : OPT = $(OPT_BASE) $(OPT_OPENACC) $(OPT_PERF2) -Mvect=nosimd -Minfo=all -g
endif
#
-CC ?= gcc
-CXX ?= g++
-FC ?= gfortran
+ifneq ($(origin CC),environment)
+# default value CC=cc not correct for gnu compiler
+CC = gcc
+endif
+ifneq ($(origin CXX),environment)
+CXX = g++
+endif
+ifneq ($(origin FC),environment)
+# default value FC=f77 not correct for gnu compiler
+FC = gfortran
+endif
# Gfortran version
GFV=$(shell $(F90) --version | grep -E -m1 -o ' [[:digit:]\.]{2,}( |$$)' | sed 's/\.//g' )
diff --git a/src/Rules.LXnvhpc2202.mk b/src/Rules.LXnvhpc2202.mk
index a6d8a5c76ff6540c53115c0e596d581daf3c151e..69a4dfff1d57d6033f4f0451e5166af32bfe7859 100644
--- a/src/Rules.LXnvhpc2202.mk
+++ b/src/Rules.LXnvhpc2202.mk
@@ -50,10 +50,10 @@ OPT_PERF0 = -O0
OPT_PERF1 = -O1
OPT_PERF2 = -O2
#
-OPT_MANAGED = -Mframe -Mnostack_arrays -acc=host,gpu -gpu=nofma,$(OPT_CPTCAP),$(CUDALEVEL),managed $(OPT_INFO) $(OPT_PROF)
+OPT_MANAGED = -Mframe -Mnostack_arrays -acc=host,gpu -gpu=nofma,$(OPT_CPTCAP),$(CUDALEVEL),managed,lineinfo $(OPT_INFO) $(OPT_PROF)
OPT_MULTICORE = -acc=multicore $(OPT_INFO) $(OPT_PROF)
OPT_NOOPENACC = -acc=host $(OPT_INFO) $(OPT_PROF)
-OPT_OPENACC = -Mframe -Mnostack_arrays -acc=host,gpu -gpu=nofma,$(OPT_CPTCAP),$(CUDALEVEL) $(OPT_INFO) $(OPT_PROF)
+OPT_OPENACC = -Mframe -Mnostack_arrays -acc=host,gpu -gpu=nofma,$(OPT_CPTCAP),$(CUDALEVEL),lineinfo $(OPT_INFO) $(OPT_PROF)
#
OPT_CHECK = -C #-Mchkfpstk -Mchkptr
OPT_I8 = -i8
@@ -164,9 +164,9 @@ endif
#
ifeq "$(OPTLEVEL)" "OPENACCDEFONLY"
CPPFLAGS += -DMNH_OPENACC -D_FAKEOPENACC
-OPT = -g $(OPT_BASE) $(OPT_NOOPENACC) $(OPT_PERF0)
+OPT = -g $(OPT_BASE) $(OPT_NOOPENACC) $(OPT_PERF2)
OPT0 = -g $(OPT_BASE) $(OPT_NOOPENACC) $(OPT_PERF0)
-OPT_NOCB = -g $(OPT_BASE) $(OPT_NOOPENACC) $(OPT_PERF0)
+OPT_NOCB = -g $(OPT_BASE) $(OPT_NOOPENACC) $(OPT_PERF2)
CXXFLAGS = -Kieee -Mnofma $(OPT_NOOPENACC)
$(OBJS_REPROD) : OPT = -g $(OPT_BASE) $(OPT_NOOPENACC) $(OPT_PERF2) -Mvect=nosimd -Minfo=all
endif
@@ -202,6 +202,12 @@ FX90FLAGS = $(OPT)
#
LDFLAGS = -Wl,-warn-once $(OPT)
#
+# ArmForge/MAP/DDT: Exemple of LDFLAGS for executable directly linked with MAP,lib
+#
+#VERMPI=3.1.5
+#VERMPI ?= 4.0.5
+#LDFLAGS += -L/users/m23008/escobar/MAP/$(VERMPI) -lmap-sampler-pmpi -lmap-sampler -Wl,--eh-frame-hdr -Wl,-rpath=/users/m23008/escobar/MAP/$(VERMPI)
+#
# preprocessing flags
#
#CPP = cpp -P -traditional -Wcomment -D_OPENACC=201711 -imacros MNH_OPENACC_NV_CR.CPP
@@ -260,6 +266,8 @@ MNH_GRIBAPI=no
#
CDF_CONF="CPP=cpp"
HDF_CONF="CPP=cpp"
+HDF_OPT ?= -g
+NETCDF_OPT ?= -g
#
##########################################################
# #
diff --git a/src/configure b/src/configure
index 1f59609e9f9e08fff38fd23353c56bdf4adb1f4b..a154339d1b07cce957d3dd42b7ecd06187a30d69 100755
--- a/src/configure
+++ b/src/configure
@@ -617,15 +617,18 @@ if [ "x${MVWORK}" == "xYES" ] ; then
echo ' ---> initialisez la et relance "./configure"'
exit 1
fi
- if [ -d $WORKDIR ] && [ ${LOCAL} == ${LOCAL##/work} ] ; then
+ if [ -d ${WORKDIR} ] && [ ${LOCAL} == ${LOCAL##/work} ] ; then
# if we are not allready in the workdir --> move stuff
WORK=$WORKDIR/${RACINE}_WORKDIR
mkdir -p ${WORK}
- for dir in MY_RUN exe pub
+ for dir in MY_RUN exe pub src/LIB
do
- [ -d $LOCAL/${dir} ] && [ ! -L $LOCAL/${dir} ] \
- && mv $LOCAL/${dir} ${WORK}/. \
- && ln -s ${WORK}/${dir} $LOCAL/.
+ if [ -d ${LOCAL}/${dir} ] && [ ! -L ${LOCAL}/${dir} ] ; then
+ DIR=$( dirname ${WORK}/${dir} )
+ mkdir -p ${DIR}
+ mv ${LOCAL}/${dir} ${DIR}/.
+ ln -fs ${WORK}/${dir} ${LOCAL}/${dir}
+ fi
done
fi
fi