diff --git a/tensorproductmultigrid_Source/communication.f90 b/tensorproductmultigrid_Source/communication.f90
index e362a6f2d1b59b90967586d9a93d6b3ba6121e8b..b7b1fef019a506cd0e79221c34ab869645069cc2 100644
--- a/tensorproductmultigrid_Source/communication.f90
+++ b/tensorproductmultigrid_Source/communication.f90
@@ -37,9 +37,12 @@
module communication
use messages
use datatypes
- use parameters
- !use mpi
+
+#ifndef MNH
+ use mpi
+#else
use modd_mpif
+#endif
use timer
implicit none
diff --git a/tensorproductmultigrid_Source/conjugategradient.f90 b/tensorproductmultigrid_Source/conjugategradient.f90
index 320e272c9ffbd58d44279094ad1edcd57656c184..30277d8625ad6735444ad7335b761ab6ed369b7a 100644
--- a/tensorproductmultigrid_Source/conjugategradient.f90
+++ b/tensorproductmultigrid_Source/conjugategradient.f90
@@ -40,8 +40,11 @@ module conjugategradient
use discretisation
use messages
use communication
- !use mpi
+#ifndef MNH
+ use mpi
+#else
use modd_mpif
+#endif
implicit none
diff --git a/tensorproductmultigrid_Source/datatypes.f90 b/tensorproductmultigrid_Source/datatypes.f90
index ef71df25c8e791932abd914914712bbeadf38ce2..7f7c6eab07766a2a9978a089b07262f4e0b405f6 100644
--- a/tensorproductmultigrid_Source/datatypes.f90
+++ b/tensorproductmultigrid_Source/datatypes.f90
@@ -39,8 +39,11 @@
module datatypes
- !use mpi
+#ifndef MNH
+ use mpi
+#else
use modd_mpif
+#endif
use parameters
use messages
@@ -341,7 +344,7 @@ private
if (LUseT) then
zphi_st => phi%st
- !$acc kernels loop collapse(3)
+ !$acc kernels loop dtype(nvidia) collapse(3)
do iz=1,nz
do iy=1,nlocaly
do ix=1,nlocalx
diff --git a/tensorproductmultigrid_Source/discretisation.f90 b/tensorproductmultigrid_Source/discretisation.f90
index a42888a1d43b6c71fd93a7526f2a472bf321a569..9fa41dd2779aac88d52de2f14bc225b1351e1c0d 100644
--- a/tensorproductmultigrid_Source/discretisation.f90
+++ b/tensorproductmultigrid_Source/discretisation.f90
@@ -65,8 +65,11 @@ module discretisation
use messages
use datatypes
use communication
- !use mpi
+#ifndef MNH
+ use mpi
+#else
use modd_mpif
+#endif
implicit none
@@ -631,9 +634,10 @@ end subroutine construct_vertical_coeff
real(kind=rl) :: tmp
integer :: iib,iie,ijb,ije
- real, dimension(:,:,:) , pointer :: zv_st , zu_st
- real, dimension(:) , pointer :: za_k, zb_k, zc_k, zd_k
+ real(kind=rl), dimension(:,:,:) , pointer :: zv_st , zu_st
+ real(kind=rl), dimension(:) , pointer :: za_k, zb_k, zc_k, zd_k
integer :: ii,ij
+ integer :: ize
call boundary_mnh(u)
@@ -676,22 +680,29 @@ end subroutine construct_vertical_coeff
iie=u%icompx_max
ijb=u%icompy_min
ije=u%icompy_max
+ ize=u%grid_param%nz
!
zv_st => v%st
zu_st => u%st
zb_k => vert_coeff%b
zc_k => vert_coeff%c
zd_k => vert_coeff%d
-
- !$acc kernels
+
+ !$acc kernels
iz=1
- zv_st(iib:iie,ijb:ije,iz) = zd_k(iz)* ( (-zb_k(iz)-zc_k(iz))*Tij * zu_st(iib:iie,ijb:ije,iz ) &
- +zb_k(iz) *Tij * zu_st(iib:iie,ijb:ije,iz+1) )
+ !$acc loop independent dtype(nvidia) collapse(2)
+ do ij=ijb,ije
+ do ii=iib,iie
+ zv_st(ii,ij,iz) = zd_k(iz)* ( (-zb_k(iz)-zc_k(iz))*Tij * zu_st(ii,ij,iz ) &
+ +zb_k(iz) *Tij * zu_st(ii,ij,iz+1) )
+ end do
+ end do
!
- do iz=2,u%grid_param%nz-1
+ do iz=2,ize-1
+ !$acc loop independent dtype(nvidia) collapse(2)
do ij=ijb,ije
do ii=iib,iie
- zv_st(ii,ij,iz) = zd_k(iz)* ( ((-zb_k(iz)-zc_k(iz))*Tij - 4.0_rl ) * zu_st(ii,ij,iz) &
+ zv_st(ii,ij,iz) = zd_k(iz)* ( ((-zb_k(iz)-zc_k(iz))*Tij - 4.0_rl ) * zu_st(ii,ij,iz) &
+zb_k(iz) *Tij * zu_st(ii,ij,iz+1) &
+zc_k(iz) *Tij * zu_st(ii,ij,iz-1) &
+ zu_st(ii+1,ij,iz) &
@@ -703,9 +714,15 @@ end subroutine construct_vertical_coeff
end do
end do
!
- iz=u%grid_param%nz
- zv_st(iib:iie,ijb:ije,iz) = zd_k(iz)* ( (-zb_k(iz)-zc_k(iz))*Tij * zu_st(iib:iie,ijb:ije,iz) &
- +zc_k(iz) *Tij * zu_st(iib:iie,ijb:ije,iz-1) )
+ iz=ize
+ !$acc loop independent dtype(nvidia) collapse(2)
+ do ij=ijb,ije
+ do ii=iib,iie
+ zv_st(ii,ij,iz) = zd_k(iz)* ( (-zb_k(iz)-zc_k(iz))*Tij * zu_st(ii,ij,iz) &
+ +zc_k(iz) *Tij * zu_st(ii,ij,iz-1) )
+ end do
+ end do
+
!$acc end kernels
endif
diff --git a/tensorproductmultigrid_Source/messages.f90 b/tensorproductmultigrid_Source/messages.f90
index ce0fd76c35e8517fa73c433258e2b3f2d23fe0e2..7b4345777a92dcb235e12c456284ecc66b06c980 100644
--- a/tensorproductmultigrid_Source/messages.f90
+++ b/tensorproductmultigrid_Source/messages.f90
@@ -36,8 +36,11 @@
module messages
use parameters
- !use mpi
+#ifndef MNH
+ use mpi
+#else
use modd_mpif
+#endif
implicit none
diff --git a/tensorproductmultigrid_Source/mg_main.f90 b/tensorproductmultigrid_Source/mg_main.f90
index 2ce0607a8279cc04855c7e0d3c408acd9372d6a7..a97d51b494603178c8f34336f074d49d4f8aa8ec 100644
--- a/tensorproductmultigrid_Source/mg_main.f90
+++ b/tensorproductmultigrid_Source/mg_main.f90
@@ -51,8 +51,11 @@ program mg_main
use messages
use communication
use timer
- !use mpi
+#ifndef MNH
+ use mpi
+#else
use modd_mpif
+#endif
use mode_mg_read_param
use mode_mg
diff --git a/tensorproductmultigrid_Source/mg_main_mnh.f90 b/tensorproductmultigrid_Source/mg_main_mnh.f90
index 4263ea443b6c440e41143473a2db1149ccd9166e..cbe833f9a7d71a8dd327c5c53796ec9fc6471e06 100644
--- a/tensorproductmultigrid_Source/mg_main_mnh.f90
+++ b/tensorproductmultigrid_Source/mg_main_mnh.f90
@@ -51,8 +51,11 @@ module mode_mg_main_mnh
use messages
use communication
use timer
- !use mpi
+#ifndef MNH
+ use mpi
+#else
use modd_mpif
+#endif
use mode_mg_read_param
use mode_mg
@@ -143,7 +146,7 @@ end subroutine mg_main_get_u_mnh
call mg_main_mnh_init()
- DO it=1,10
+ DO it=1,1
call mg_main_initialise_rhs_mnh()
call mg_main_initialise_u_mnh()
diff --git a/tensorproductmultigrid_Source/mode_mg.f90 b/tensorproductmultigrid_Source/mode_mg.f90
index c388cee404dfd606e9a815d13adbd4677cce2ee1..806825f1b47d9e14c0d528fe57666e2931faeb0e 100644
--- a/tensorproductmultigrid_Source/mode_mg.f90
+++ b/tensorproductmultigrid_Source/mode_mg.f90
@@ -8,8 +8,11 @@ module mode_mg
use solver
use parameters
use timer
- !use mpi
- use modd_mpif
+#ifndef MNH
+ use mpi
+#else
+ use modd_mpif
+#endif
use profiles
use mode_mg_read_param
diff --git a/tensorproductmultigrid_Source/mode_mg_read_param.f90 b/tensorproductmultigrid_Source/mode_mg_read_param.f90
index 87b1ae7f2b2d8e4c162e386a9f6d29ddb5015a28..63c7f5ff7a00cc85146093b2641d84209fb2867d 100644
--- a/tensorproductmultigrid_Source/mode_mg_read_param.f90
+++ b/tensorproductmultigrid_Source/mode_mg_read_param.f90
@@ -36,8 +36,11 @@ subroutine read_general_parameters(filename,savefields_out)
use parameters
use communication
use messages
- !use mpi
+#ifndef MNH
+ use mpi
+#else
use modd_mpif
+#endif
implicit none
character(*), intent(in) :: filename
logical, intent(out) :: savefields_out
@@ -68,8 +71,11 @@ subroutine read_solver_parameters(filename,solver_param_out)
use parameters
use communication
use messages
- !use mpi
+#ifndef MNH
+ use mpi
+#else
use modd_mpif
+#endif
implicit none
character(*), intent(in) :: filename
type(solver_parameters), intent(out) :: solver_param_out
@@ -121,8 +127,11 @@ subroutine read_grid_parameters_mnh(filename,grid_param,KN,KNZ,PL,PH)
use datatypes
use communication
use messages
- !use mpi
+#ifndef MNH
+ use mpi
+#else
use modd_mpif
+#endif
implicit none
character(*), intent(in) :: filename
type(grid_parameters), intent(out) :: grid_param
@@ -193,8 +202,12 @@ subroutine read_grid_parameters(filename,grid_param)
use datatypes
use communication
use messages
- !use mpi
+#ifndef MNH
+ use mpi
+#else
use modd_mpif
+#endif
+
implicit none
character(*), intent(in) :: filename
type(grid_parameters), intent(out) :: grid_param
@@ -251,8 +264,11 @@ subroutine read_comm_parameters(filename,comm_param)
use parameters
use communication
use messages
- !use mpi
+#ifndef MNH
+ use mpi
+#else
use modd_mpif
+#endif
implicit none
character(*), intent(in) :: filename
type(comm_parameters), intent(out) :: comm_param
@@ -290,8 +306,11 @@ subroutine read_model_parameters(filename,model_param)
use discretisation
use communication
use messages
- !use mpi
+#ifndef MNH
+ use mpi
+#else
use modd_mpif
+#endif
implicit none
character(*), intent(in) :: filename
type(model_parameters), intent(out) :: model_param
@@ -328,8 +347,11 @@ subroutine read_smoother_parameters(filename,smoother_param)
use discretisation
use communication
use messages
- !use mpi
+#ifndef MNH
+ use mpi
+#else
use modd_mpif
+#endif
implicit none
character(*), intent(in) :: filename
type(smoother_parameters), intent(out) :: smoother_param
@@ -393,8 +415,11 @@ subroutine read_multigrid_parameters(filename,mg_param)
use multigrid
use communication
use messages
- !use mpi
+#ifndef MNH
+ use mpi
+#else
use modd_mpif
+#endif
implicit none
character(*), intent(in) :: filename
type(mg_parameters), intent(out) :: mg_param
@@ -493,8 +518,11 @@ subroutine read_conjugategradient_parameters(filename,cg_param)
use conjugategradient
use communication
use messages
- !use mpi
+#ifndef MNH
+ use mpi
+#else
use modd_mpif
+#endif
implicit none
character(*), intent(in) :: filename
type(cg_parameters), intent(out) :: cg_param
diff --git a/tensorproductmultigrid_Source/multigrid.f90 b/tensorproductmultigrid_Source/multigrid.f90
index 2a594d28ef2e0d7ab82d2fb10ff8b8e955a1ffa7..d00404df25f62bc23ed6844d84f92a913cd9ad39 100644
--- a/tensorproductmultigrid_Source/multigrid.f90
+++ b/tensorproductmultigrid_Source/multigrid.f90
@@ -36,8 +36,11 @@
!==================================================================
module multigrid
- !use mpi
+#ifndef MNH
+ use mpi
+#else
use modd_mpif
+#endif
use parameters
use datatypes
use discretisation
@@ -483,7 +486,7 @@ contains
if (LUseT) then
zphifine_st => phifine%st
zphicoarse_st => phicoarse%st
- !$acc kernels loop independent collapse(3)
+ !$acc kernels loop independent dtype(nvidia) collapse(3)
do iz=1,phicoarse%grid_param%nz
do iy=iy_min,iy_max
do ix=ix_min,ix_max
@@ -807,7 +810,7 @@ contains
zphifine_st => phifine%st
zphicoarse_st => phicoarse%st
- !$acc kernels loop independent collapse(5)
+ !$acc kernels loop independent dtype(nvidia) collapse(5)
do iz=1,phicoarse%grid_param%nz
do diy = -1,0
do dix = -1,0
diff --git a/tensorproductmultigrid_Source/timer.f90 b/tensorproductmultigrid_Source/timer.f90
index 6966c9bb850a959566fed4d06d100418b9f9b950..da0f17907a28016c0aad1fee4b78d348cf72ecfd 100644
--- a/tensorproductmultigrid_Source/timer.f90
+++ b/tensorproductmultigrid_Source/timer.f90
@@ -36,8 +36,11 @@
module timer
- !use mpi
+#ifndef MNH
+ use mpi
+#else
use modd_mpif
+#endif
use parameters
implicit none