From 51bab8e9828c0ab3f03122b26566157afbf75262 Mon Sep 17 00:00:00 2001
From: Philippe WAUTELET <philippe.wautelet@aero.obs-mip.fr>
Date: Tue, 8 Feb 2022 10:44:24 +0100
Subject: [PATCH] Philippe 08/02/2022: OpenACC: replace MEM_GET_ZT3D family
calls by MNH_MEM_ALLOCATE family calls (not yet finished)
---
src/MNH/advec_ppm_algo.f90 | 82 ++------
src/MNH/advection_metsv.f90 | 14 +-
src/MNH/advection_uvw.f90 | 77 +-------
src/MNH/advection_uvw_cen.f90 | 16 +-
src/MNH/advecuvw_4th.f90 | 112 ++++-------
src/MNH/advecuvw_rk.f90 | 146 ++++++--------
src/MNH/advecuvw_weno_k.f90 | 306 +++++++++++++++++-------------
src/MNH/contrav.f90 | 31 ++-
src/MNH/gradient_m.f90 | 128 ++++++++-----
src/ZSOLVER/advection_metsv.f90 | 14 +-
src/ZSOLVER/advection_uvw.f90 | 78 +-------
src/ZSOLVER/advection_uvw_cen.f90 | 16 +-
src/ZSOLVER/advecuvw_rk.f90 | 140 ++++++--------
src/ZSOLVER/advecuvw_weno_k.f90 | 302 ++++++++++++++++-------------
src/ZSOLVER/contrav.f90 | 39 ++--
src/ZSOLVER/gdiv.f90 | 9 +-
16 files changed, 659 insertions(+), 851 deletions(-)
diff --git a/src/MNH/advec_ppm_algo.f90 b/src/MNH/advec_ppm_algo.f90
index 8e0bcaf4a..b594d954a 100644
--- a/src/MNH/advec_ppm_algo.f90
+++ b/src/MNH/advec_ppm_algo.f90
@@ -42,70 +42,11 @@ END INTERFACE
END MODULE MODI_ADVEC_PPM_ALGO
!
!
-#ifdef MNH_OPENACC
-! ##########################################################################
- SUBROUTINE ADVEC_PPM_ALGO(HMET_ADV_SCHEME, HLBCX, HLBCY, KGRID, PFIELDT, &
- PRHODJ, PTSTEP, PTSTEP_PPM, &
- PRHOX1, PRHOX2, PRHOY1, PRHOY2, PRHOZ1,PRHOZ2,&
- PSRC, TPDTCUR, PCRU, PCRV, PCRW)
-!
-USE MODE_MNH_ZWORK, ONLY : ZT3D, MNH_GET_ZT3D , MNH_REL_ZT3D
-USE MODD_TIME, ONLY: DATE_TIME
-!
-IMPLICIT NONE
-!
-!* 0.1 Declarations of dummy arguments :
-!
-CHARACTER (LEN=4), DIMENSION(2), INTENT(IN) :: HLBCX ! X direction LBC type
-CHARACTER (LEN=4), DIMENSION(2), INTENT(IN) :: HLBCY ! Y direction LBC type
-CHARACTER (LEN=6), INTENT(IN) :: HMET_ADV_SCHEME
-!
-REAL, DIMENSION(:,:,:), INTENT(IN) :: PFIELDT ! variable at t
-INTEGER, INTENT(IN) :: KGRID ! C grid localisation
-!
-REAL, DIMENSION(:,:,:), INTENT(IN) :: PCRU, PCRV, PCRW ! Courant numbers
-!
-REAL, DIMENSION(:,:,:), INTENT(IN) :: PRHODJ ! density
-REAL, DIMENSION(:,:,:), INTENT(IN) :: PRHOX1, PRHOX2
-REAL, DIMENSION(:,:,:), INTENT(IN) :: PRHOY1, PRHOY2
-REAL, DIMENSION(:,:,:), INTENT(IN) :: PRHOZ1, PRHOZ2
-REAL, INTENT(IN) :: PTSTEP ! Time step
-REAL, INTENT(IN) :: PTSTEP_PPM ! Time Step PPM
-TYPE (DATE_TIME), INTENT(IN) :: TPDTCUR ! current date and time
-!
-REAL, DIMENSION(:,:,:), INTENT(OUT) :: PSRC ! source term after advection
-
-INTEGER :: IZPPM
-
-!$acc data present( PFIELDT, PCRU, PCRV, PCRW, PRHODJ, PRHOX1, PRHOX2, PRHOY1, PRHOY2, PRHOZ1, PRHOZ2, PSRC )
-
- CALL MNH_GET_ZT3D(IZPPM)
-
- CALL ADVEC_PPM_ALGO_D(HMET_ADV_SCHEME, HLBCX, HLBCY, KGRID, PFIELDT, &
- & PRHODJ, PTSTEP, PTSTEP_PPM, &
- & PRHOX1, PRHOX2, PRHOY1, PRHOY2, PRHOZ1,PRHOZ2,&
- & PSRC, TPDTCUR, PCRU, PCRV, PCRW, &
- & ZT3D(:,:,:,IZPPM) )
-
- CALL MNH_REL_ZT3D(IZPPM)
-
-!$acc end data
-
-CONTAINS
-#endif
! ##########################################################################
-#ifndef MNH_OPENACC
SUBROUTINE ADVEC_PPM_ALGO(HMET_ADV_SCHEME, HLBCX, HLBCY, KGRID, PFIELDT, &
PRHODJ, PTSTEP, PTSTEP_PPM, &
PRHOX1, PRHOX2, PRHOY1, PRHOY2, PRHOZ1,PRHOZ2,&
PSRC, TPDTCUR, PCRU, PCRV, PCRW)
-#else
- SUBROUTINE ADVEC_PPM_ALGO_D(HMET_ADV_SCHEME, HLBCX, HLBCY, KGRID, PFIELDT, &
- PRHODJ, PTSTEP, PTSTEP_PPM, &
- PRHOX1, PRHOX2, PRHOY1, PRHOY2, PRHOZ1,PRHOZ2,&
- PSRC, TPDTCUR, PCRU, PCRV, PCRW, &
- ZPPM)
-#endif
! ##########################################################################
!!
!!**** *ADVEC_PPM_ALGO* - interface for 3D advection with PPM type scheme
@@ -137,6 +78,7 @@ CONTAINS
USE MODD_TYPE_DATE
#ifdef MNH_OPENACC
USE MODE_DEVICE
+USE MODE_MNH_ZWORK, ONLY: MNH_MEM_GET, MNH_MEM_POSITION_PIN, MNH_MEM_RELEASE
#endif
use mode_mppdb
#ifdef MNH_OPENACC
@@ -175,12 +117,12 @@ REAL, DIMENSION(:,:,:), INTENT(OUT) :: PSRC ! source term after advection
!
LOGICAL :: GFLAG ! Logical flag
#ifdef MNH_OPENACC
-REAL, DIMENSION(:,:,:) :: ZPPM ! temp PPM output
+REAL, DIMENSION(:,:,:), POINTER, CONTIGUOUS :: ZPPM ! temp PPM output
#endif
!
!-------------------------------------------------------------------------------
!
-!$acc data present( PFIELDT, PCRU, PCRV, PCRW, PRHODJ, PRHOX1, PRHOX2, PRHOY1, PRHOY2, PRHOZ1, PRHOZ2, PSRC, ZPPM )
+!$acc data present( PFIELDT, PCRU, PCRV, PCRW, PRHODJ, PRHOX1, PRHOX2, PRHOY1, PRHOY2, PRHOZ1, PRHOZ2, PSRC )
IF (MPPDB_INITIALIZED) THEN
!Check all IN arrays
@@ -303,6 +245,13 @@ CASE('PPM_00')
CASE('PPM_01')
!
#ifdef MNH_OPENACC
+ !Pin positions in the pools of MNH memory
+ CALL MNH_MEM_POSITION_PIN()
+
+ CALL MNH_MEM_GET( ZPPM, SIZE( PSRC, 1 ), SIZE( PSRC, 2 ), SIZE( PSRC, 3 ) )
+
+ !$acc data present( ZPPM )
+
CALL INIT_ON_HOST_AND_DEVICE(ZPPM,PVALUE=-1e99,HNAME='ADVEC_PPM_ALGO::ZPPM')
#endif
IF (GFLAG ) THEN
@@ -400,6 +349,13 @@ CASE('PPM_01')
#endif
!
END IF
+
+ !$acc end data
+
+#ifdef MNH_OPENACC
+ !Release all memory allocated with MNH_MEM_GET calls since last call to MNH_MEM_POSITION_PIN
+ CALL MNH_MEM_RELEASE()
+#endif
!
! monotonic scheme (Skamarock notation)
!
@@ -507,8 +463,4 @@ END IF
!
!$acc end data
-#ifdef MNH_OPENACC
-END SUBROUTINE ADVEC_PPM_ALGO_D
-#endif
-!
END SUBROUTINE ADVEC_PPM_ALGO
diff --git a/src/MNH/advection_metsv.f90 b/src/MNH/advection_metsv.f90
index ea8140def..6ba1850b6 100644
--- a/src/MNH/advection_metsv.f90
+++ b/src/MNH/advection_metsv.f90
@@ -174,8 +174,7 @@ use mode_argslist_ll, only: ADD3DFIELD_ll, ADD4DFIELD_ll, CLEANLIST_ll, LIST_
use mode_budget, only: Budget_store_init, Budget_store_end
#ifdef MNH_OPENACC
USE MODE_DEVICE
-USE MODE_MNH_ZWORK, ONLY: MNH_MEM_GET, MNH_MEM_POSITION_PIN, MNH_MEM_RELEASE, &
- MNH_GET_ZT3D, MNH_REL_ZT3D, ZT3D
+USE MODE_MNH_ZWORK, ONLY: MNH_MEM_GET, MNH_MEM_POSITION_PIN, MNH_MEM_RELEASE
#endif
use mode_exchange_ll, only: UPDATE_HALO_ll
USE MODE_IO_FIELD_WRITE, only: IO_Field_write
@@ -299,7 +298,6 @@ INTEGER :: ISPLIT_PPM ! temporal time splitting
INTEGER :: IIB, IIE, IJB, IJE,IKB,IKE
#ifdef MNH_OPENACC
CHARACTER(LEN=3) :: YNUM
-INTEGER :: IZ1, IZ2
#endif
TYPE(TFIELDDATA) :: TZFIELD
!
@@ -481,8 +479,6 @@ CALL INIT_ON_HOST_AND_DEVICE(ZRHOY1,PVALUE=-1e95,HNAME='ADVECTION_METSV::ZRHOY1'
CALL INIT_ON_HOST_AND_DEVICE(ZRHOY2,PVALUE=-1e96,HNAME='ADVECTION_METSV::ZRHOY2')
CALL INIT_ON_HOST_AND_DEVICE(ZRHOZ1,PVALUE=-1e97,HNAME='ADVECTION_METSV::ZRHOZ1')
CALL INIT_ON_HOST_AND_DEVICE(ZRHOZ2,PVALUE=-1e98,HNAME='ADVECTION_METSV::ZRHOZ2')
-!
-CALL MNH_GET_ZT3D(IZ1, IZ2)
#endif
!
IF(LBLOWSNOW) THEN ! Put 2D Canopy blowing snow variables into a 3D array for advection
@@ -515,10 +511,10 @@ END IF
#else
IF (HUVW_ADV_SCHEME=='CEN2ND' ) THEN
CALL CONTRAV_DEVICE (HLBCX,HLBCY,PUT,PVT,PWT,PDXX,PDYY,PDZZ,PDZX,PDZY,ZRUCPPM,ZRVCPPM,ZRWCPPM,2, &
- ZT3D(:,:,:,IZ1),ZT3D(:,:,:,IZ2),ODATA_ON_DEVICE=.TRUE.)
+ ODATA_ON_DEVICE=.TRUE.)
ELSE
CALL CONTRAV_DEVICE (HLBCX,HLBCY,PUT,PVT,PWT,PDXX,PDYY,PDZZ,PDZX,PDZY,ZRUCPPM,ZRVCPPM,ZRWCPPM,4, &
- ZT3D(:,:,:,IZ1),ZT3D(:,:,:,IZ2),ODATA_ON_DEVICE=.TRUE.)
+ ODATA_ON_DEVICE=.TRUE.)
END IF
#endif
!
@@ -1072,10 +1068,6 @@ end if
! Remove non-physical negative values (unnecessary in a perfect world) + corresponding budgets
call Sources_neg_correct( hcloud, 'NEADV', krr, ptstep, ppabst, ptht, prt, prths, prrs, prsvs )
-#ifdef MNH_OPENACC
-CALL MNH_REL_ZT3D(IZ1, IZ2)
-#endif
-
IF (MPPDB_INITIALIZED) THEN
!Check all INOUT arrays
CALL MPPDB_CHECK(PRTHS,"ADVECTION_METSV end:PRTHS")
diff --git a/src/MNH/advection_uvw.f90 b/src/MNH/advection_uvw.f90
index 24047a92f..2d08a7842 100644
--- a/src/MNH/advection_uvw.f90
+++ b/src/MNH/advection_uvw.f90
@@ -107,7 +107,6 @@ use mode_budget, only: Budget_store_init, Budget_store_end
USE MODE_ll
#ifdef MNH_OPENACC
USE MODE_DEVICE
-USE MODE_MNH_ZWORK, ONLY : ZT3D, MNH_GET_ZT3D , MNH_REL_ZT3D, MNH_GET_ZT4D , MNH_REL_ZT4D
#endif
use mode_mppdb
@@ -193,11 +192,6 @@ TYPE(LIST_ll), POINTER :: TZFIELD_ll ! list of fields to exchange
TYPE(LIST_ll), POINTER :: TZFIELDS_ll ! list of fields to exchange
TYPE(LIST_ll), POINTER :: TZFIELDS0_ll ! list of fields to exchange
!
-#ifdef MNH_OPENACC
-INTEGER :: ISPL, IZUT, IZVT, IZWT, IZ1, IZ2
-INTEGER :: IZRUSB, IZRUSE, IZRVSB, IZRVSE, IZRWSB, IZRWSE, IIBMS, IIBME
-#endif
-!
INTEGER :: IIU,IJU,IKU
!
!-------------------------------------------------------------------------------
@@ -255,60 +249,6 @@ ALLOCATE( ZMZM_RHODJ( IIU,IJU,IKU ) )
!$acc & zrus_other, zrvs_other, zrws_other, zrus_adv, zrvs_adv, zrws_adv, &
!$acc & zmxm_rhodj, zmym_rhodj, zmzm_rhodj )
-#ifdef MNH_OPENACC
-#if 0
-CALL INIT_ON_HOST_AND_DEVICE(ZRUT,-1e99,'ADVECTION_UVW::ZRUT')
-CALL INIT_ON_HOST_AND_DEVICE(ZRVT,-2e99,'ADVECTION_UVW::ZRVT')
-CALL INIT_ON_HOST_AND_DEVICE(ZRWT,-3e99,'ADVECTION_UVW::ZRWT')
-CALL INIT_ON_HOST_AND_DEVICE(ZRUCT,-1e98,'ADVECTION_UVW::ZRUCT')
-CALL INIT_ON_HOST_AND_DEVICE(ZRVCT,-2e98,'ADVECTION_UVW::ZRVCT')
-CALL INIT_ON_HOST_AND_DEVICE(ZRWCT,-3e98,'ADVECTION_UVW::ZRWCT')
-CALL INIT_ON_HOST_AND_DEVICE(ZU,-1e99,'ADVECTION_UVW::ZU')
-CALL INIT_ON_HOST_AND_DEVICE(ZV,-1e99,'ADVECTION_UVW::ZV')
-CALL INIT_ON_HOST_AND_DEVICE(ZW,-1e99,'ADVECTION_UVW::ZW')
-CALL INIT_ON_HOST_AND_DEVICE(ZRUS_OTHER,-1e99,'ADVECTION_UVW::ZRUS_OTHER')
-CALL INIT_ON_HOST_AND_DEVICE(ZRVS_OTHER,-1e99,'ADVECTION_UVW::ZRVS_OTHER')
-CALL INIT_ON_HOST_AND_DEVICE(ZRWS_OTHER,-1e99,'ADVECTION_UVW::ZRWS_OTHER')
-CALL INIT_ON_HOST_AND_DEVICE(ZRUS_ADV,-1e99,'ADVECTION_UVW::ZRUS_ADV')
-CALL INIT_ON_HOST_AND_DEVICE(ZRVS_ADV,-1e99,'ADVECTION_UVW::ZRVS_ADV')
-CALL INIT_ON_HOST_AND_DEVICE(ZRWS_ADV,-1e99,'ADVECTION_UVW::ZRWS_ADV')
-CALL INIT_ON_HOST_AND_DEVICE(ZMXM_RHODJ,-1e97,'ADVECTION_UVW::ZMXM_RHODJ')
-CALL INIT_ON_HOST_AND_DEVICE(ZMYM_RHODJ,-2e97,'ADVECTION_UVW::ZMYM_RHODJ')
-CALL INIT_ON_HOST_AND_DEVICE(ZMZM_RHODJ,-3e97,'ADVECTION_UVW::ZMZM_RHODJ')
-#endif
-!
-SELECT CASE (HTEMP_SCHEME)
- CASE('RK11')
- ISPL = 1
- CASE('RK21')
- ISPL = 2
- CASE('NP32')
- ISPL = 3
- CASE('SP32')
- ISPL = 3
- CASE('RK33')
- ISPL = 3
- CASE('RKC4')
- ISPL = 4
- CASE('RK4B')
- ISPL = 4
- CASE('RK53')
- ISPL = 5
- CASE('RK62')
- ISPL = 6
- CASE('RK65')
- ISPL = 6
- CASE DEFAULT
- call Print_msg( NVERB_FATAL, 'GEN', 'ADVECTION_UVW', 'unknown htemp_scheme' )
-END SELECT
-!
-CALL MNH_GET_ZT3D(IZUT, IZVT, IZWT, IZ1, IZ2)
-CALL MNH_GET_ZT4D(ISPL, IZRUSB, IZRUSE)
-CALL MNH_GET_ZT4D(ISPL, IZRVSB, IZRVSE)
-CALL MNH_GET_ZT4D(ISPL, IZRWSB, IZRWSE)
-CALL MNH_GET_ZT4D(3, IIBMS, IIBME)
-#endif
-!
IKE = SIZE(PWT,3) - JPVEXT
!
#ifndef MNH_OPENACC
@@ -358,7 +298,7 @@ NULLIFY(TZFIELD_ll)
CALL CONTRAV (HLBCX,HLBCY,ZRUT,ZRVT,ZRWT,PDXX,PDYY,PDZZ,PDZX,PDZY,ZRUCT,ZRVCT,ZRWCT,4)
#else
CALL CONTRAV_DEVICE (HLBCX,HLBCY,ZRUT,ZRVT,ZRWT,PDXX,PDYY,PDZZ,PDZX,PDZY,ZRUCT,ZRVCT,ZRWCT,4,&
- ZT3D(:,:,:,IZ1),ZT3D(:,:,:,IZ2),ODATA_ON_DEVICE=.TRUE.)
+ ODATA_ON_DEVICE=.TRUE.)
!Not necessary: already done in contrav_device !$acc update self(ZRUCT,ZRVCT,ZRWCT)
#endif
!
@@ -463,13 +403,7 @@ DO JSPL=1,ISPLIT
ZMXM_RHODJ, ZMYM_RHODJ, ZMZM_RHODJ, &
ZRUCT, ZRVCT, ZRWCT, &
ZRUS_ADV, ZRVS_ADV, ZRWS_ADV, &
- ZRUS_OTHER, ZRVS_OTHER, ZRWS_OTHER &
-#ifndef MNH_OPENACC
- )
-#else
- ,ZT3D(:,:,:,IZUT), ZT3D(:,:,:,IZVT), ZT3D(:,:,:,IZWT), &
- ZT3D(:,:,:,IZRUSB:IZRUSE), ZT3D(:,:,:,IZRVSB:IZRVSE), ZT3D(:,:,:,IZRWSB:IZRWSE), ZT3D(:,:,:,IIBMS:IIBME) )
-#endif
+ ZRUS_OTHER, ZRVS_OTHER, ZRWS_OTHER )
!
! Tendencies on wind
!$acc update device(ZRUS_ADV,ZRVS_ADV,ZRWS_ADV)
@@ -522,13 +456,6 @@ if ( lbudget_w ) call Budget_store_end( tbudgets(NBUDGET_W), 'ADV', prws(:, :, :
!-------------------------------------------------------------------------------
!
-#ifdef MNH_OPENACC
-CALL MNH_REL_ZT4D(3, IIBMS )
-CALL MNH_REL_ZT4D(ISPL, IZRWSB)
-CALL MNH_REL_ZT4D(ISPL, IZRVSB)
-CALL MNH_REL_ZT4D(ISPL, IZRUSB)
-CALL MNH_REL_ZT3D(IZUT, IZVT, IZWT, IZ1, IZ2)
-#endif
IF (MPPDB_INITIALIZED) THEN
!Check all INOUT arrays
diff --git a/src/MNH/advection_uvw_cen.f90 b/src/MNH/advection_uvw_cen.f90
index 442c45475..b16d803f2 100644
--- a/src/MNH/advection_uvw_cen.f90
+++ b/src/MNH/advection_uvw_cen.f90
@@ -1,4 +1,4 @@
-!MNH_LIC Copyright 2013-2020 CNRS, Meteo-France and Universite Paul Sabatier
+!MNH_LIC Copyright 2013-2022 CNRS, Meteo-France and Universite Paul Sabatier
!MNH_LIC This is part of the Meso-NH software governed by the CeCILL-C licence
!MNH_LIC version 1. See LICENSE, CeCILL-C_V1-en.txt and CeCILL-C_V1-fr.txt
!MNH_LIC for details. version 1.
@@ -104,7 +104,6 @@ use mode_budget, only: Budget_store_init, Budget_store_end
USE MODE_ll
#ifdef MNH_OPENACC
USE MODE_DEVICE
-USE MODE_MNH_ZWORK, ONLY : ZT3D, MNH_GET_ZT3D , MNH_REL_ZT3D
use mode_msg
#endif
use mode_mppdb
@@ -177,9 +176,6 @@ REAL, DIMENSION(:,:,:), allocatable :: ZMZM_RHODJ
!
INTEGER :: IINFO_ll ! return code of parallel routine
TYPE(LIST_ll), POINTER :: TZFIELDS_ll ! list of fields to exchange
-#ifdef MNH_OPENACC
-INTEGER :: IZ1, IZ2
-#endif
!
!-------------------------------------------------------------------------------
!$acc data present( PUM, PVM, PWM, PDUM, PDVM, PDWM, PUT, PVT, PWT, PRHODJ, PDXX, PDYY, PDZZ, PDZX, PDZY, PRUS, PRVS, PRWS )
@@ -246,8 +242,6 @@ CALL INIT_ON_HOST_AND_DEVICE(ZRWCT,-3e98,'ADVECTION_UVW_CEN::ZRWCT')
CALL INIT_ON_HOST_AND_DEVICE(ZMXM_RHODJ,-1e97,'ADVECTION_UVW_CEN::ZMXM_RHODJ')
CALL INIT_ON_HOST_AND_DEVICE(ZMYM_RHODJ,-2e97,'ADVECTION_UVW_CEN::ZMYM_RHODJ')
CALL INIT_ON_HOST_AND_DEVICE(ZMZM_RHODJ,-3e97,'ADVECTION_UVW_CEN::ZMZM_RHODJ')
-!
-CALL MNH_GET_ZT3D(IZ1, IZ2)
#endif
#ifndef MNH_OPENACC
@@ -278,10 +272,10 @@ END IF
#else
IF (HUVW_ADV_SCHEME=='CEN2ND' ) THEN
CALL CONTRAV_DEVICE (HLBCX,HLBCY,ZRUT,ZRVT,ZRWT,PDXX,PDYY,PDZZ,PDZX,PDZY,ZRUCT,ZRVCT,ZRWCT,2, &
- ZT3D(:,:,:,IZ1),ZT3D(:,:,:,IZ2),ODATA_ON_DEVICE=.TRUE.)
+ ODATA_ON_DEVICE=.TRUE.)
ELSEIF (HUVW_ADV_SCHEME=='CEN4TH') THEN
CALL CONTRAV_DEVICE (HLBCX,HLBCY,ZRUT,ZRVT,ZRWT,PDXX,PDYY,PDZZ,PDZX,PDZY,ZRUCT,ZRVCT,ZRWCT,4, &
- ZT3D(:,:,:,IZ1),ZT3D(:,:,:,IZ2),ODATA_ON_DEVICE=.TRUE.)
+ ODATA_ON_DEVICE=.TRUE.)
END IF
!Not necessary: already done in contrav_device !$acc update self(ZRUCT,ZRVCT,ZRWCT)
#endif
@@ -349,10 +343,6 @@ if ( lbudget_u ) call Budget_store_end( tbudgets(NBUDGET_U), 'ADV', prus(:, :, :
if ( lbudget_v ) call Budget_store_end( tbudgets(NBUDGET_V), 'ADV', prvs(:, :, :) )
if ( lbudget_w ) call Budget_store_end( tbudgets(NBUDGET_W), 'ADV', prws(:, :, :) )
-#ifdef MNH_OPENACC
-CALL MNH_REL_ZT3D(IZ1, IZ2)
-#endif
-
IF (MPPDB_INITIALIZED) THEN
!Check all INOUT arrays
CALL MPPDB_CHECK(PRUS,"ADVECTION_UVW_CEN end:PRUS")
diff --git a/src/MNH/advecuvw_4th.f90 b/src/MNH/advecuvw_4th.f90
index c220818c8..7f596bd9d 100644
--- a/src/MNH/advecuvw_4th.f90
+++ b/src/MNH/advecuvw_4th.f90
@@ -34,73 +34,10 @@ END INTERFACE
END MODULE MODI_ADVECUVW_4TH
!
!
-#ifdef MNH_OPENACC
-! ######################################################################
- SUBROUTINE ADVECUVW_4TH ( HLBCX, HLBCY, PRUCT, PRVCT, PRWCT, &
- PUT, PVT, PWT, PRUS, PRVS, PRWS, TPHALO2LIST )
-! ######################################################################
-
- USE MODD_ARGSLIST_ll, ONLY : HALO2LIST_ll
- USE MODE_MNH_ZWORK , ONLY : ZT3D, MNH_GET_ZT3D , MNH_REL_ZT3D
- USE MODE_MNH_ZWORK, ONLY : IIU,IJU,IKU
-
-IMPLICIT NONE
-!
-!* 0.1 Declarations of dummy arguments :
-!
-!
-CHARACTER (LEN=4), DIMENSION(2), INTENT(IN) :: HLBCX ! X direction LBC type
-CHARACTER (LEN=4), DIMENSION(2), INTENT(IN) :: HLBCY ! Y direction LBC type
-!
-!
-REAL, DIMENSION(:,:,:), INTENT(IN) :: PRUCT ! contravariant
-REAL, DIMENSION(:,:,:), INTENT(IN) :: PRVCT ! components
-REAL, DIMENSION(:,:,:), INTENT(IN) :: PRWCT ! of momentum
-!
-REAL, DIMENSION(:,:,:), INTENT(IN) :: PUT, PVT, PWT ! Variables at t
-!
-REAL, DIMENSION(:,:,:), INTENT(INOUT) :: PRUS, PRVS, PRWS ! Source terms
-!
-TYPE(HALO2LIST_ll), POINTER :: TPHALO2LIST ! list for diffusion
-!
-!* 0.2 Declarations of local variables :
-!
-INTEGER:: IIB,IJB ! Begining useful area in x,y,z directions
-INTEGER:: IIE,IJE ! End useful area in x,y,z directions
-!
-TYPE(HALO2LIST_ll), POINTER :: TZHALO2LIST
-
-INTEGER :: IZMEANX, IZMEANY, IZTEMP1,IZTEMP2,IZTEMP3,IZTEMP4
-
-!$acc data present( PRUCT, PRVCT, PRWCT, PUT, PVT, PWT, PRUS, PRVS, PRWS )
- CALL MNH_GET_ZT3D(IZMEANX, IZMEANY,IZTEMP1,IZTEMP2,IZTEMP3,IZTEMP4 )
-
- CALL ADVECUVW_4TH_D ( IIU,IJU,IKU,HLBCX, HLBCY, &
- & PRUCT, PRVCT, PRWCT, &
- & PUT, PVT, PWT, PRUS, PRVS, PRWS, TPHALO2LIST, &
- & ZT3D(:,:,:,IZMEANX),ZT3D(:,:,:,IZMEANY), &
- & ZT3D(:,:,:,IZTEMP1),ZT3D(:,:,:,IZTEMP2), &
- & ZT3D(:,:,:,IZTEMP3),ZT3D(:,:,:,IZTEMP4) &
- & )
-
- CALL MNH_REL_ZT3D(IZMEANX, IZMEANY, IZTEMP1,IZTEMP2,IZTEMP3,IZTEMP4)
-!$acc end data
-!
-CONTAINS
-!
-! ######################################################################
- SUBROUTINE ADVECUVW_4TH_D ( IIU,IJU,IKU,HLBCX, HLBCY, &
- & PRUCT, PRVCT, PRWCT, &
- & PUT, PVT, PWT, PRUS, PRVS, PRWS, TPHALO2LIST , &
- & ZMEANX, ZMEANY, ZTEMP1,ZTEMP2,ZTEMP3,ZTEMP4 )
-
-! ######################################################################
-#else
! ######################################################################
SUBROUTINE ADVECUVW_4TH ( HLBCX, HLBCY, PRUCT, PRVCT, PRWCT, &
PUT, PVT, PWT, PRUS, PRVS, PRWS, TPHALO2LIST )
! ######################################################################
-#endif
!
!!**** *ADVECUVW_4TH * - routine to compute the 4th order centered
!! advection tendency of momentum (U,V,W)
@@ -175,6 +112,9 @@ USE MODD_GRID_n
USE MODD_PARAMETERS
USE MODE_ll
+#ifdef MNH_OPENACC
+USE MODE_MNH_ZWORK, ONLY: MNH_MEM_GET, MNH_MEM_POSITION_PIN, MNH_MEM_RELEASE
+#endif
use mode_mppdb
USE MODI_ADVEC_4TH_ORDER_AUX
@@ -189,9 +129,6 @@ IMPLICIT NONE
!* 0.1 Declarations of dummy arguments :
!
!
-#ifdef MNH_OPENACC
-INTEGER , INTENT(IN) :: IIU,IJU,IKU
-#endif
CHARACTER (LEN=4), DIMENSION(2), INTENT(IN) :: HLBCX ! X direction LBC type
CHARACTER (LEN=4), DIMENSION(2), INTENT(IN) :: HLBCY ! Y direction LBC type
!
@@ -214,11 +151,10 @@ INTEGER :: IGRID ! localisation on the model grid
#ifndef MNH_OPENACC
REAL, DIMENSION(SIZE(PUT,1),SIZE(PUT,2),SIZE(PUT,3)) :: ZMEANX, ZMEANY ! fluxes
#else
-REAL, DIMENSION(:,:,:) :: ZMEANX, ZMEANY ! fluxes
+INTEGER :: IIU, IJU, IKU
+REAL, DIMENSION(:,:,:), POINTER, CONTIGUOUS :: ZMEANX, ZMEANY ! fluxes
!
-REAL, DIMENSION(:,:,:) :: ZTEMP1,ZTEMP2,ZTEMP3,ZTEMP4
-
-INTEGER :: II
+REAL, DIMENSION(:,:,:), POINTER, CONTIGUOUS :: ZTEMP1, ZTEMP2, ZTEMP3, ZTEMP4
#endif
!
#if 0
@@ -240,10 +176,6 @@ INTEGER :: II
PMZF4(:,:,1) = 0.5*( PA(:,:,2)+PA(:,:,1) ) ; PMZF4(:,:,IKU-1) = 0.5*( PA(:,:,IKU)+PA(:,:,IKU-1) ) ; PMZF4(:,:,IKU) = -999.
#endif
!
-
-
-!$acc data present( PRUCT, PRVCT, PRWCT, PUT, PVT, PWT, PRUS, PRVS, PRWS, ZMEANX, ZMEANY, ZTEMP1, ZTEMP2, ZTEMP3, ZTEMP4 )
-
IF (MPPDB_INITIALIZED) THEN
!Check all IN arrays
CALL MPPDB_CHECK(PRUCT,"ADVECUVW_4TH beg:PRUCT")
@@ -257,6 +189,26 @@ IF (MPPDB_INITIALIZED) THEN
CALL MPPDB_CHECK(PRVS,"ADVECUVW_4TH beg:PRVS")
CALL MPPDB_CHECK(PRWS,"ADVECUVW_4TH beg:PRWS")
END IF
+
+#ifdef MNH_OPENACC
+IIU = SIZE( PUT, 1 )
+IJU = SIZE( PUT, 2 )
+IKU = SIZE( PUT, 3 )
+
+!Pin positions in the pools of MNH memory
+CALL MNH_MEM_POSITION_PIN()
+
+CALL MNH_MEM_GET( ZMEANX, IIU, IJU, IKU )
+CALL MNH_MEM_GET( ZMEANY, IIU, IJU, IKU )
+
+CALL MNH_MEM_GET( ZTEMP1, IIU, IJU, IKU )
+CALL MNH_MEM_GET( ZTEMP2, IIU, IJU, IKU )
+CALL MNH_MEM_GET( ZTEMP3, IIU, IJU, IKU )
+CALL MNH_MEM_GET( ZTEMP4, IIU, IJU, IKU )
+#endif
+
+!$acc data present( PRUCT, PRVCT, PRWCT, PUT, PVT, PWT, PRUS, PRVS, PRWS, ZMEANX, ZMEANY, ZTEMP1, ZTEMP2, ZTEMP3, ZTEMP4 )
+
!-------------------------------------------------------------------------------
!
!* 2. CALL THE ADVEC_4TH_ORDER_ALGO ROUTINE FOR MOMENTUM
@@ -409,7 +361,12 @@ call dzm_device( ZTEMP1, ZTEMP4 )
PRWS(:,:,:) = PRWS(:,:,:) - ZTEMP4
!$acc end kernels
#endif
-!
+
+!$acc end data
+
+!Release all memory allocated with MNH_MEM_GET calls since last call to MNH_MEM_POSITION_PIN
+CALL MNH_MEM_RELEASE()
+
IF (MPPDB_INITIALIZED) THEN
!Check all INOUT arrays
CALL MPPDB_CHECK(PRUS,"ADVECUVW_4TH end:PRUS")
@@ -417,12 +374,7 @@ IF (MPPDB_INITIALIZED) THEN
CALL MPPDB_CHECK(PRWS,"ADVECUVW_4TH end:PRWS")
END IF
-!$acc end data
-
!-------------------------------------------------------------------------------
!
-#ifdef MNH_OPENACC
-END SUBROUTINE ADVECUVW_4TH_D
-#endif
!
END SUBROUTINE ADVECUVW_4TH
diff --git a/src/MNH/advecuvw_rk.f90 b/src/MNH/advecuvw_rk.f90
index 403eb3277..fe1f69f13 100644
--- a/src/MNH/advecuvw_rk.f90
+++ b/src/MNH/advecuvw_rk.f90
@@ -1,4 +1,4 @@
-!MNH_LIC Copyright 1994-2021 CNRS, Meteo-France and Universite Paul Sabatier
+!MNH_LIC Copyright 1994-2022 CNRS, Meteo-France and Universite Paul Sabatier
!MNH_LIC This is part of the Meso-NH software governed by the CeCILL-C licence
!MNH_LIC version 1. See LICENSE, CeCILL-C_V1-en.txt and CeCILL-C_V1-fr.txt
!MNH_LIC for details. version 1.
@@ -16,12 +16,7 @@ INTERFACE
PMXM_RHODJ, PMYM_RHODJ, PMZM_RHODJ, &
PRUCT, PRVCT, PRWCT, &
PRUS_ADV, PRVS_ADV, PRWS_ADV, &
- PRUS_OTHER, PRVS_OTHER, PRWS_OTHER &
-#ifndef MNH_OPENACC
- )
-#else
- , ZUT, ZVT, ZWT, ZRUS, ZRVS, ZRWS, ZIBM )
-#endif
+ PRUS_OTHER, PRVS_OTHER, PRWS_OTHER )
!
CHARACTER(LEN=6), INTENT(IN) :: HUVW_ADV_SCHEME! to the selected
CHARACTER(LEN=4), INTENT(IN) :: HTEMP_SCHEME ! Temporal scheme
@@ -47,13 +42,6 @@ REAL, DIMENSION(:,:,:), INTENT(OUT) :: PRUS_ADV , PRVS_ADV, PRWS_ADV
! Tendency due to advection
REAL, DIMENSION(:,:,:), INTENT(IN) :: PRUS_OTHER , PRVS_OTHER, PRWS_OTHER
! ! tendencies from other processes
-#ifdef MNH_OPENACC
-! Work arrays
-REAL, DIMENSION(:,:,:) :: ZUT, ZVT, ZWT
-REAL, DIMENSION(:,:,:,:) :: ZRUS, ZRVS, ZRWS, ZIBM
-#endif
-!
-!
END SUBROUTINE ADVECUVW_RK
!
END INTERFACE
@@ -68,12 +56,7 @@ END MODULE MODI_ADVECUVW_RK
PMXM_RHODJ, PMYM_RHODJ, PMZM_RHODJ, &
PRUCT, PRVCT, PRWCT, &
PRUS_ADV, PRVS_ADV, PRWS_ADV, &
- PRUS_OTHER, PRVS_OTHER, PRWS_OTHER &
-#ifndef MNH_OPENACC
- )
-#else
- , ZUT, ZVT, ZWT, ZRUS, ZRVS, ZRWS, ZIBM )
-#endif
+ PRUS_OTHER, PRVS_OTHER, PRWS_OTHER )
! ##########################################################################
!
!!**** *ADVECUVW_RK * - routine to call the specialized advection routines for wind
@@ -150,7 +133,7 @@ USE MODI_SHUMAN
!
#ifdef MNH_OPENACC
USE MODE_DEVICE
-USE MODE_MNH_ZWORK, ONLY : ZT3D, MNH_GET_ZT3D , MNH_REL_ZT3D
+USE MODE_MNH_ZWORK, ONLY: MNH_MEM_GET, MNH_MEM_POSITION_PIN, MNH_MEM_RELEASE
#endif
!
!-------------------------------------------------------------------------------
@@ -183,17 +166,9 @@ REAL, DIMENSION(:,:,:), INTENT(OUT) :: PRUS_ADV , PRVS_ADV, PRWS_ADV
! Tendency due to advection
REAL, DIMENSION(:,:,:), INTENT(IN) :: PRUS_OTHER , PRVS_OTHER, PRWS_OTHER
! ! tendencies from other processes
-#ifdef MNH_OPENACC
-REAL, DIMENSION(:,:,:) :: ZUT, ZVT, ZWT
-REAL, DIMENSION(:,:,:,:) :: ZRUS, ZRVS, ZRWS, ZIBM
-#endif
-!
-!
!
!* 0.2 declarations of local variables
!
-!
-!
character(len=3) :: ynum
INTEGER :: IKE ! indice K End in z direction
!
@@ -201,13 +176,23 @@ INTEGER :: IKE ! indice K End in z direction
REAL, DIMENSION(:,:,:), ALLOCATABLE :: ZUT, ZVT, ZWT
! Intermediate Guesses inside the RK loop
!
-REAL, DIMENSION(:,:,:,:), ALLOCATABLE :: ZRUS,ZRVS,ZRWS,ZIBM
-#endif
+REAL, DIMENSION(:,:,:,:), ALLOCATABLE :: ZRUS, ZRVS, ZRWS, ZIBM
! Momentum tendencies due to advection
REAL, DIMENSION(:,:), ALLOCATABLE :: ZBUT ! Butcher array coefficients
! at the RK sub time step
REAL, DIMENSION(:), ALLOCATABLE :: ZBUTS! Butcher array coefficients
! at the end of the RK loop
+#else
+REAL, DIMENSION(:,:,:), POINTER, CONTIGUOUS :: ZUT, ZVT, ZWT
+! Intermediate Guesses inside the RK loop
+!
+REAL, DIMENSION(:,:,:,:), POINTER, CONTIGUOUS :: ZRUS, ZRVS, ZRWS, ZIBM
+! Momentum tendencies due to advection
+REAL, DIMENSION(:,:), POINTER, CONTIGUOUS :: ZBUT ! Butcher array coefficients
+ ! at the RK sub time step
+REAL, DIMENSION(:), POINTER, CONTIGUOUS :: ZBUTS! Butcher array coefficients
+ ! at the end of the RK loop
+#endif
!JUAN
TYPE(LIST_ll), POINTER :: TZFIELDMT_ll ! list of fields to exchange
TYPE(HALO2LIST_ll), POINTER :: TZHALO2MT_ll ! momentum variables
@@ -215,9 +200,6 @@ INTEGER :: INBVAR
INTEGER :: IIU, IJU, IKU ! array sizes
!JUAN
-#ifdef MNH_OPENACC
-INTEGER :: IZMEAN, IZWORK
-#endif
! Momentum tendencies due to advection
INTEGER :: ISPL ! Number of RK splitting loops
INTEGER :: JI, JS ! Loop index
@@ -232,10 +214,6 @@ LOGICAL :: GIBM !Intermediate variable used to work around a Cray compiler bug (
REAL :: ZIBM_EPSI !Intermediate variable used to work around a Cray compiler bug (CCE 13.0.0)
REAL :: ZTIME1,ZTIME2
!-------------------------------------------------------------------------------
-!$acc data present( PU, PV, PW, PUT, PVT, PWT, PMXM_RHODJ, PMYM_RHODJ, PMZM_RHODJ, &
-!$acc & PRUCT, PRVCT, PRWCT, PRUS_ADV, PRVS_ADV, PRWS_ADV, &
-!$acc & PRUS_OTHER, PRVS_OTHER, PRWS_OTHER, ZUT, ZVT, ZWT, &
-!$acc & ZRUS, ZRVS, ZRWS, ZIBM )
IF (MPPDB_INITIALIZED) THEN
!Check all IN arrays
@@ -266,20 +244,6 @@ if ( LIBM ) call Print_msg( NVERB_FATAL, 'GEN', 'ADVECUVW_RK', 'OpenACC: LIBM=T
!* 0. INITIALIZATION
! ---------------------
!
-#ifndef MNH_OPENACC
-allocate(ZUT(SIZE(PUT,1),SIZE(PUT,2),SIZE(PUT,3)))
-allocate(ZVT(SIZE(PUT,1),SIZE(PUT,2),SIZE(PUT,3)))
-allocate(ZWT(SIZE(PUT,1),SIZE(PUT,2),SIZE(PUT,3)))
-#endif
-
-#ifdef MNH_OPENACC
-CALL INIT_ON_HOST_AND_DEVICE(ZUT,4e99,'ADVECUVW_RK::ZUT')
-CALL INIT_ON_HOST_AND_DEVICE(ZVT,5e99,'ADVECUVW_RK::ZVT')
-CALL INIT_ON_HOST_AND_DEVICE(ZWT,6e99,'ADVECUVW_RK::ZWT')
-!
-CALL MNH_GET_ZT3D(IZMEAN,IZWORK)
-#endif
-!
IKE = SIZE(PWT,3) - JPVEXT
IIU=SIZE(PUT,1)
IJU=SIZE(PUT,2)
@@ -310,11 +274,47 @@ SELECT CASE (HTEMP_SCHEME)
call Print_msg(NVERB_FATAL,'GEN','ADVECUVW_RK','unknown HTEMP_SCHEME')
END SELECT
!
-!
+#ifndef MNH_OPENACC
+allocate( ZUT(IIU, IJU, IKU) )
+allocate( ZVT(IIU, IJU, IKU) )
+allocate( ZWT(IIU, IJU, IKU) )
+
+ALLOCATE( ZRUS(IIU, IJU, IKU, ISPL) )
+ALLOCATE( ZRVS(IIU, IJU, IKU, ISPL) )
+ALLOCATE( ZRWS(IIU, IJU, IKU, ISPL) )
+
+IF ( GIBM ) ALLOCATE( ZIBM(IIU, IJU, IKU, 3) )
+
ALLOCATE(ZBUT(ISPL-1,ISPL-1))
ALLOCATE(ZBUTS(ISPL))
+#else
+!Pin positions in the pools of MNH memory
+CALL MNH_MEM_POSITION_PIN()
+
+CALL MNH_MEM_GET( ZUT, IIU, IJU, IKU )
+CALL MNH_MEM_GET( ZVT, IIU, IJU, IKU )
+CALL MNH_MEM_GET( ZWT, IIU, IJU, IKU )
-!$acc data create(ZBUT,ZBUTS)
+CALL MNH_MEM_GET( ZRUS, IIU, IJU, IKU, ISPL )
+CALL MNH_MEM_GET( ZRVS, IIU, IJU, IKU, ISPL )
+CALL MNH_MEM_GET( ZRWS, IIU, IJU, IKU, ISPL )
+
+IF ( GIBM ) CALL MNH_MEM_GET( ZIBM, IIU, IJU, IKU, 3 )
+
+CALL MNH_MEM_GET( ZBUT, ISPL-1, ISPL-1 )
+CALL MNH_MEM_GET( ZBUTS, ISPL )
+#endif
+
+#ifdef MNH_OPENACC
+CALL INIT_ON_HOST_AND_DEVICE(ZUT,4e99,'ADVECUVW_RK::ZUT')
+CALL INIT_ON_HOST_AND_DEVICE(ZVT,5e99,'ADVECUVW_RK::ZVT')
+CALL INIT_ON_HOST_AND_DEVICE(ZWT,6e99,'ADVECUVW_RK::ZWT')
+#endif
+
+!$acc data present( PU, PV, PW, PUT, PVT, PWT, PMXM_RHODJ, PMYM_RHODJ, PMZM_RHODJ, &
+!$acc & PRUCT, PRVCT, PRWCT, PRUS_ADV, PRVS_ADV, PRWS_ADV, &
+!$acc & PRUS_OTHER, PRVS_OTHER, PRWS_OTHER, ZUT, ZVT, ZWT, &
+!$acc & ZRUS, ZRVS, ZRWS, ZIBM, ZBUT, ZBUTS )
SELECT CASE (HTEMP_SCHEME)
CASE('RK11')
@@ -401,16 +401,7 @@ CASE('RK65')
END SELECT
!$acc update device(ZBUTS,ZBUT)
!
-#ifndef MNH_OPENACC
-ALLOCATE(ZRUS(SIZE(PUT,1),SIZE(PUT,2),SIZE(PWT,3),ISPL))
-ALLOCATE(ZRVS(SIZE(PUT,1),SIZE(PUT,2),SIZE(PWT,3),ISPL))
-ALLOCATE(ZRWS(SIZE(PUT,1),SIZE(PUT,2),SIZE(PWT,3),ISPL))
-#endif
-
IF ( GIBM ) THEN
-#ifndef MNH_OPENACC
- ALLOCATE( ZIBM(SIZE(PUT,1), SIZE(PUT,2), SIZE(PWT,3), 3) )
-#endif
!$acc kernels
ZIBM(:,:,:,:) = 1.
!$acc end kernels
@@ -506,11 +497,7 @@ RKLOOP: DO JS = 1, ISPL
CALL ADVECUVW_WENO_K (HLBCX, HLBCY, KWENO_ORDER, ZUT, ZVT, ZWT, &
PRUCT, PRVCT, PRWCT, &
ZRUS(:,:,:,JS), ZRVS(:,:,:,JS), ZRWS(:,:,:,JS), &
-#ifndef MNH_OPENACC
TZHALO2MT_ll )
-#else
- TZHALO2MT_ll, ZT3D(:,:,:,IZMEAN), ZT3D(:,:,:,IZWORK) )
-#endif
ELSE IF ((HUVW_ADV_SCHEME=='CEN4TH') .AND. (HTEMP_SCHEME=='RKC4')) THEN
CALL ADVECUVW_4TH (HLBCX, HLBCY, PRUCT, PRVCT, PRWCT, &
ZUT, ZVT, ZWT, &
@@ -523,11 +510,7 @@ RKLOOP: DO JS = 1, ISPL
CALL ADVECUVW_WENO_K (HLBCX, HLBCY, 3, ZUT, ZVT, ZWT, &
PRUCT, PRVCT, PRWCT, &
ZIBM(:,:,:,1), ZIBM(:,:,:,2), ZIBM(:,:,:,3) ,&
-#ifndef MNH_OPENACC
TZHALO2MT_ll )
-#else
- TZHALO2MT_ll, ZT3D(:,:,:,IZMEAN), ZT3D(:,:,:,IZWORK) )
-#endif
ENDIF
IF (HUVW_ADV_SCHEME=='CEN4TH') THEN
CALL ADVECUVW_2ND (ZUT, ZVT, ZWT, PRUCT, PRVCT, PRWCT, &
@@ -587,7 +570,7 @@ RKLOOP: DO JS = 1, ISPL
! !$acc & present(ZRUS,ZRVS,ZRWS,ZIBM) present(PRUS_OTHER,PRVS_OTHER,PRWS_OTHER) &
! !$acc & present(PMXM_RHODJ,PMYM_RHODJ,PMZM_RHODJ)
!
-!$acc kernels
+!$acc kernels present( ZUT, ZVT, ZWT )
ZUT(:,:,:) = PU(:,:,:)
ZVT(:,:,:) = PV(:,:,:)
ZWT(:,:,:) = PW(:,:,:)
@@ -598,7 +581,7 @@ RKLOOP: DO JS = 1, ISPL
! Intermediate guesses inside the RK loop
!
IF ( .NOT. GIBM ) THEN
-!$acc kernels
+!$acc kernels present( ZUT, ZVT, ZWT, ZRUS, ZRVS, ZRWS )
ZUT(:,:,:) = ZUT(:,:,:) + ZBUT(JS,JI) * PTSTEP * &
( ZRUS(:,:,:,JI) + PRUS_OTHER(:,:,:) ) / PMXM_RHODJ(:,:,:)
ZVT(:,:,:) = ZVT(:,:,:) + ZBUT(JS,JI) * PTSTEP * &
@@ -607,7 +590,7 @@ RKLOOP: DO JS = 1, ISPL
( ZRWS(:,:,:,JI) + PRWS_OTHER(:,:,:) ) / PMZM_RHODJ(:,:,:)
!$acc end kernels
ELSE
-!$acc kernels
+!$acc kernels present( ZUT, ZVT, ZWT, ZRUS, ZRVS, ZRWS, ZIBM )
ZUT(:,:,:) = ZUT(:,:,:) + ZBUT(JS,JI) * PTSTEP * &
( ZRUS(:,:,:,JI) + PRUS_OTHER(:,:,:) ) / PMXM_RHODJ(:,:,:) * ZIBM(:,:,:,1)
ZVT(:,:,:) = ZVT(:,:,:) + ZBUT(JS,JI) * PTSTEP * &
@@ -626,15 +609,16 @@ RKLOOP: DO JS = 1, ISPL
END DO RKLOOP
!
!
-#ifdef MNH_OPENACC
-CALL MNH_REL_ZT3D(IZMEAN,IZWORK)
-#endif
-!
CALL CLEANLIST_ll(TZFIELDMT_ll)
CALL DEL_HALO2_ll(TZHALO2MT_ll)
!$acc update self(PRUS_ADV,PRVS_ADV,PRWS_ADV)
!-------------------------------------------------------------------------------
-!
+
+!$acc end data
+
+!Release all memory allocated with MNH_MEM_GET calls since last call to MNH_MEM_POSITION_PIN
+CALL MNH_MEM_RELEASE()
+
IF (MPPDB_INITIALIZED) THEN
!Check all OUT arrays
CALL MPPDB_CHECK(PRUS_ADV,"ADVECUVW_RK end:PRUS_ADV")
@@ -642,8 +626,4 @@ IF (MPPDB_INITIALIZED) THEN
CALL MPPDB_CHECK(PRWS_ADV,"ADVECUVW_RK end:PRWS_ADV")
END IF
-!$acc end data
-
-!$acc end data
-
END SUBROUTINE ADVECUVW_RK
diff --git a/src/MNH/advecuvw_weno_k.f90 b/src/MNH/advecuvw_weno_k.f90
index 4353eab11..3d5e27a03 100644
--- a/src/MNH/advecuvw_weno_k.f90
+++ b/src/MNH/advecuvw_weno_k.f90
@@ -11,11 +11,7 @@ INTERFACE
!
SUBROUTINE ADVECUVW_WENO_K(HLBCX, HLBCY, KWENO_ORDER, PUT, PVT, PWT, &
PRUCT, PRVCT, PRWCT, PRUS, PRVS, PRWS, TPHALO2LIST &
-#ifndef MNH_OPENACC
)
-#else
- , ZMEAN, ZWORK)
-#endif
!
USE MODD_ARGSLIST_ll, ONLY : HALO2LIST_ll
!
@@ -34,10 +30,6 @@ REAL, DIMENSION(:,:,:), INTENT(INOUT) :: PRUS, PRVS, PRWS ! Source terms
!
TYPE(HALO2LIST_ll), POINTER :: TPHALO2LIST ! list for diffusion
!
-#ifdef MNH_OPENACC
-! Work arrays
-REAL, DIMENSION(:,:,:) :: ZMEAN, ZWORK
-#endif
!
END SUBROUTINE ADVECUVW_WENO_K
!
@@ -48,11 +40,7 @@ END MODULE MODI_ADVECUVW_WENO_K
! ##########################################################################
SUBROUTINE ADVECUVW_WENO_K(HLBCX, HLBCY, KWENO_ORDER, PUT, PVT, PWT, &
PRUCT, PRVCT, PRWCT, PRUS, PRVS, PRWS, TPHALO2LIST &
-#ifndef MNH_OPENACC
)
-#else
- , ZMEAN, ZWORK)
-#endif
! ##########################################################################
!
!! AUTHOR
@@ -78,7 +66,7 @@ USE MODD_PARAMETERS
USE MODE_ll
#ifdef MNH_OPENACC
USE MODE_DEVICE
-USE MODE_MNH_ZWORK, ONLY : ZT3D, MNH_GET_ZT3D , MNH_REL_ZT3D
+USE MODE_MNH_ZWORK, ONLY: MNH_MEM_GET, MNH_MEM_POSITION_PIN, MNH_MEM_RELEASE
#endif
USE MODE_MPPDB
@@ -113,27 +101,23 @@ TYPE(HALO2LIST_ll), POINTER :: TPHALO2LIST ! list for diffusion
!
!* 0.2 Declarations of local variables :
!
+INTEGER :: IIU, IJU, IKU
TYPE(HALO2LIST_ll), POINTER :: TZHALO2_UT,TZHALO2_VT,TZHALO2_WT
-
-TYPE(LIST_ll), POINTER :: TZHALO2_ZMEAN
INTEGER :: IINFO_ll ! return code of parallel routine
!
#ifndef MNH_OPENACC
REAL, DIMENSION(:,:,:), allocatable :: ZMEAN, ZWORK
#else
-REAL, DIMENSION(:,:,:) :: ZMEAN, ZWORK
+REAL, DIMENSION(:,:,:), POINTER ,CONTIGUOUS :: ZMEAN, ZWORK
+!
+REAL, DIMENSION(:,:,:), POINTER ,CONTIGUOUS :: ZFPOS1, ZFPOS2, ZFPOS3
+REAL, DIMENSION(:,:,:), POINTER ,CONTIGUOUS :: ZFNEG1, ZFNEG2, ZFNEG3
+REAL, DIMENSION(:,:,:), POINTER ,CONTIGUOUS :: ZBPOS1, ZBPOS2, ZBPOS3
+REAL, DIMENSION(:,:,:), POINTER ,CONTIGUOUS :: ZBNEG1, ZBNEG2, ZBNEG3
+REAL, DIMENSION(:,:,:), POINTER ,CONTIGUOUS :: ZOMP1, ZOMP2, ZOMP3
+REAL, DIMENSION(:,:,:), POINTER ,CONTIGUOUS :: ZOMN1, ZOMN2, ZOMN3
#endif
!
-#ifdef MNH_OPENACC
-INTEGER :: IZFPOS1, IZFPOS2, IZFPOS3
-INTEGER :: IZFNEG1, IZFNEG2, IZFNEG3
-INTEGER :: IZBPOS1, IZBPOS2, IZBPOS3
-INTEGER :: IZBNEG1, IZBNEG2, IZBNEG3
-INTEGER :: IZOMP1, IZOMP2, IZOMP3
-INTEGER :: IZOMN1, IZOMN2, IZOMN3
-#endif
-!
-!$acc data present( PRUCT, PRVCT, PRWCT, PUT, PVT, PWT, PRUS, PRVS, PRWS, ZMEAN, ZWORK )
!
IF (MPPDB_INITIALIZED) THEN
!Check all IN arrays
@@ -149,9 +133,19 @@ IF (MPPDB_INITIALIZED) THEN
CALL MPPDB_CHECK(PRWS,"ADVECUVW_WENO_K beg:PRWS")
END IF
+IIU = SIZE( PUT, 1 )
+IJU = SIZE( PUT, 2 )
+IKU = SIZE( PUT, 3 )
+
#ifndef MNH_OPENACC
-allocate(ZMEAN(SIZE(PUT,1), SIZE(PUT,2), SIZE(PUT,3)))
-allocate(ZWORK(SIZE(PUT,1), SIZE(PUT,2), SIZE(PUT,3)))
+allocate( ZMEAN(IIU, IJU, IKU) )
+allocate( ZWORK(IIU, IJU, IKU) )
+#else
+!Pin positions in the pools of MNH memory
+CALL MNH_MEM_POSITION_PIN()
+
+CALL MNH_MEM_GET( ZMEAN, IIU, IJU, IKU )
+CALL MNH_MEM_GET( ZWORK, IIU, IJU, IKU )
#endif
#ifdef MNH_OPENACC
@@ -167,7 +161,9 @@ TZHALO2_VT => TPHALO2LIST%NEXT ! 2nd add3dfield in model_n
TZHALO2_WT => TPHALO2LIST%NEXT%NEXT ! 3rst add3dfield in model_n
!
! -------------------------------------------------------
-!
+
+!$acc data present( PRUCT, PRVCT, PRWCT, PUT, PVT, PWT, PRUS, PRVS, PRWS, ZMEAN, ZWORK )
+
SELECT CASE(KWENO_ORDER)
!
CASE(1) ! WENO 1
@@ -324,15 +320,29 @@ CASE(3) ! WENO 3
!
PRWS = PRWS - DZM(WENO_K_2_WZ(PWT,MZF(PRWCT)))
#else
- CALL MNH_GET_ZT3D(IZFPOS1,IZFPOS2,IZFNEG1,IZFNEG2,IZBPOS1,IZBPOS2,IZBNEG1,IZBNEG2,IZOMP1,IZOMP2,IZOMN1,IZOMN2)
+ !Pin positions in the pools of MNH memory
+ CALL MNH_MEM_POSITION_PIN()
+
+ CALL MNH_MEM_GET( ZFPOS1, IIU, IJU, IKU )
+ CALL MNH_MEM_GET( ZFPOS2, IIU, IJU, IKU )
+ CALL MNH_MEM_GET( ZFNEG1, IIU, IJU, IKU )
+ CALL MNH_MEM_GET( ZFNEG2, IIU, IJU, IKU )
+ CALL MNH_MEM_GET( ZBPOS1, IIU, IJU, IKU )
+ CALL MNH_MEM_GET( ZBPOS2, IIU, IJU, IKU )
+ CALL MNH_MEM_GET( ZBNEG1, IIU, IJU, IKU )
+ CALL MNH_MEM_GET( ZBNEG2, IIU, IJU, IKU )
+ CALL MNH_MEM_GET( ZOMP1, IIU, IJU, IKU )
+ CALL MNH_MEM_GET( ZOMP2, IIU, IJU, IKU )
+ CALL MNH_MEM_GET( ZOMN1, IIU, IJU, IKU )
+ CALL MNH_MEM_GET( ZOMN2, IIU, IJU, IKU )
!
! U component
!
CALL MXF_DEVICE(PRUCT,ZWORK)
- CALL ADVEC_WENO_K_2_UX(HLBCX, PUT, ZWORK, ZMEAN, TZHALO2_UT%HALO2%WEST, TZHALO2_UT%HALO2%EAST, &
- ZT3D(:,:,:,IZFPOS1), ZT3D(:,:,:,IZFPOS2), ZT3D(:,:,:,IZFNEG1), ZT3D(:,:,:,IZFNEG2), &
- ZT3D(:,:,:,IZBPOS1), ZT3D(:,:,:,IZBPOS2), ZT3D(:,:,:,IZBNEG1), ZT3D(:,:,:,IZBNEG2), &
- ZT3D(:,:,:,IZOMP1), ZT3D(:,:,:,IZOMP2), ZT3D(:,:,:,IZOMN1), ZT3D(:,:,:,IZOMN2) )
+ CALL ADVEC_WENO_K_2_UX(HLBCX, PUT, ZWORK, ZMEAN, TZHALO2_UT%HALO2%WEST, TZHALO2_UT%HALO2%EAST, &
+ ZFPOS1(:,:,:), ZFPOS2(:,:,:), ZFNEG1(:,:,:), ZFNEG2(:,:,:), &
+ ZBPOS1(:,:,:), ZBPOS2(:,:,:), ZBNEG1(:,:,:), ZBNEG2(:,:,:), &
+ ZOMP1(:,:,:), ZOMP2(:,:,:), ZOMN1(:,:,:), ZOMN2(:,:,:) )
CALL DXM_DEVICE(ZMEAN,ZWORK)
!$acc kernels
PRUS(:,:,:) = PRUS(:,:,:) - ZWORK(:,:,:)
@@ -340,10 +350,10 @@ CASE(3) ! WENO 3
!
IF (.NOT.L2D) THEN
CALL MXM_DEVICE(PRVCT,ZWORK)
- CALL ADVEC_WENO_K_2_MY(HLBCY, PUT, ZWORK, ZMEAN, TZHALO2_UT%HALO2%NORTH, TZHALO2_UT%HALO2%SOUTH, &
- ZT3D(:,:,:,IZFPOS1), ZT3D(:,:,:,IZFPOS2), ZT3D(:,:,:,IZFNEG1), ZT3D(:,:,:,IZFNEG2), &
- ZT3D(:,:,:,IZBPOS1), ZT3D(:,:,:,IZBPOS2), ZT3D(:,:,:,IZBNEG1), ZT3D(:,:,:,IZBNEG2), &
- ZT3D(:,:,:,IZOMP1), ZT3D(:,:,:,IZOMP2), ZT3D(:,:,:,IZOMN1), ZT3D(:,:,:,IZOMN2) )
+ CALL ADVEC_WENO_K_2_MY(HLBCY, PUT, ZWORK, ZMEAN, TZHALO2_UT%HALO2%NORTH, TZHALO2_UT%HALO2%SOUTH, &
+ ZFPOS1(:,:,:), ZFPOS2(:,:,:), ZFNEG1(:,:,:), ZFNEG2(:,:,:), &
+ ZBPOS1(:,:,:), ZBPOS2(:,:,:), ZBNEG1(:,:,:), ZBNEG2(:,:,:), &
+ ZOMP1(:,:,:), ZOMP2(:,:,:), ZOMN1(:,:,:), ZOMN2(:,:,:) )
CALL DYF_DEVICE(ZMEAN,ZWORK)
!$acc kernels
PRUS(:,:,:) = PRUS(:,:,:) - ZWORK(:,:,:)
@@ -352,10 +362,10 @@ CASE(3) ! WENO 3
!
! PRUS = PRUS - DZF(WENO_K_2_MZ(PUT, MXM(PRWCT)))
CALL MXM_DEVICE(PRWCT,ZWORK)
- CALL WENO_K_2_MZ(PUT, ZWORK, ZMEAN, &
- ZT3D(:,:,:,IZFPOS1), ZT3D(:,:,:,IZFPOS2), ZT3D(:,:,:,IZFNEG1), ZT3D(:,:,:,IZFNEG2), &
- ZT3D(:,:,:,IZBPOS1), ZT3D(:,:,:,IZBPOS2), ZT3D(:,:,:,IZBNEG1), ZT3D(:,:,:,IZBNEG2), &
- ZT3D(:,:,:,IZOMP1), ZT3D(:,:,:,IZOMP2), ZT3D(:,:,:,IZOMN1), ZT3D(:,:,:,IZOMN2) )
+ CALL WENO_K_2_MZ(PUT, ZWORK, ZMEAN, &
+ ZFPOS1(:,:,:), ZFPOS2(:,:,:), ZFNEG1(:,:,:), ZFNEG2(:,:,:), &
+ ZBPOS1(:,:,:), ZBPOS2(:,:,:), ZBNEG1(:,:,:), ZBNEG2(:,:,:), &
+ ZOMP1(:,:,:), ZOMP2(:,:,:), ZOMN1(:,:,:), ZOMN2(:,:,:) )
CALL DZF_DEVICE( ZMEAN, ZWORK )
!$acc kernels
PRUS(:,:,:) = PRUS(:,:,:) - ZWORK(:,:,:)
@@ -365,20 +375,20 @@ CASE(3) ! WENO 3
!
IF (.NOT.L2D) THEN
CALL MYM_DEVICE(PRUCT,ZWORK)
- CALL ADVEC_WENO_K_2_MX(HLBCX, PVT, ZWORK, ZMEAN, TZHALO2_VT%HALO2%WEST, TZHALO2_VT%HALO2%EAST, &
- ZT3D(:,:,:,IZFPOS1), ZT3D(:,:,:,IZFPOS2), ZT3D(:,:,:,IZFNEG1), ZT3D(:,:,:,IZFNEG2), &
- ZT3D(:,:,:,IZBPOS1), ZT3D(:,:,:,IZBPOS2), ZT3D(:,:,:,IZBNEG1), ZT3D(:,:,:,IZBNEG2), &
- ZT3D(:,:,:,IZOMP1), ZT3D(:,:,:,IZOMP2), ZT3D(:,:,:,IZOMN1), ZT3D(:,:,:,IZOMN2) )
+ CALL ADVEC_WENO_K_2_MX(HLBCX, PVT, ZWORK, ZMEAN, TZHALO2_VT%HALO2%WEST, TZHALO2_VT%HALO2%EAST, &
+ ZFPOS1(:,:,:), ZFPOS2(:,:,:), ZFNEG1(:,:,:), ZFNEG2(:,:,:), &
+ ZBPOS1(:,:,:), ZBPOS2(:,:,:), ZBNEG1(:,:,:), ZBNEG2(:,:,:), &
+ ZOMP1(:,:,:), ZOMP2(:,:,:), ZOMN1(:,:,:), ZOMN2(:,:,:) )
CALL DXF_DEVICE(ZMEAN,ZWORK)
!$acc kernels
PRVS(:,:,:) = PRVS(:,:,:) - ZWORK(:,:,:)
!$acc end kernels
!
CALL MYF_DEVICE(PRVCT,ZWORK)
- CALL ADVEC_WENO_K_2_VY(HLBCY, PVT, ZWORK, ZMEAN, TZHALO2_VT%HALO2%NORTH, TZHALO2_VT%HALO2%SOUTH, &
- ZT3D(:,:,:,IZFPOS1), ZT3D(:,:,:,IZFPOS2), ZT3D(:,:,:,IZFNEG1), ZT3D(:,:,:,IZFNEG2), &
- ZT3D(:,:,:,IZBPOS1), ZT3D(:,:,:,IZBPOS2), ZT3D(:,:,:,IZBNEG1), ZT3D(:,:,:,IZBNEG2), &
- ZT3D(:,:,:,IZOMP1), ZT3D(:,:,:,IZOMP2), ZT3D(:,:,:,IZOMN1), ZT3D(:,:,:,IZOMN2) )
+ CALL ADVEC_WENO_K_2_VY(HLBCY, PVT, ZWORK, ZMEAN, TZHALO2_VT%HALO2%NORTH, TZHALO2_VT%HALO2%SOUTH, &
+ ZFPOS1(:,:,:), ZFPOS2(:,:,:), ZFNEG1(:,:,:), ZFNEG2(:,:,:), &
+ ZBPOS1(:,:,:), ZBPOS2(:,:,:), ZBNEG1(:,:,:), ZBNEG2(:,:,:), &
+ ZOMP1(:,:,:), ZOMP2(:,:,:), ZOMN1(:,:,:), ZOMN2(:,:,:) )
CALL DYM_DEVICE(ZMEAN,ZWORK)
!$acc kernels
PRVS(:,:,:) = PRVS(:,:,:) - ZWORK(:,:,:)
@@ -386,10 +396,10 @@ CASE(3) ! WENO 3
!
! PRVS = PRVS - DZF(WENO_K_2_MZ(PVT, MYM(PRWCT)))
CALL MYM_DEVICE(PRWCT,ZWORK)
- CALL WENO_K_2_MZ(PVT, ZWORK, ZMEAN, &
- ZT3D(:,:,:,IZFPOS1), ZT3D(:,:,:,IZFPOS2), ZT3D(:,:,:,IZFNEG1), ZT3D(:,:,:,IZFNEG2), &
- ZT3D(:,:,:,IZBPOS1), ZT3D(:,:,:,IZBPOS2), ZT3D(:,:,:,IZBNEG1), ZT3D(:,:,:,IZBNEG2), &
- ZT3D(:,:,:,IZOMP1), ZT3D(:,:,:,IZOMP2), ZT3D(:,:,:,IZOMN1), ZT3D(:,:,:,IZOMN2) )
+ CALL WENO_K_2_MZ(PVT, ZWORK, ZMEAN, &
+ ZFPOS1(:,:,:), ZFPOS2(:,:,:), ZFNEG1(:,:,:), ZFNEG2(:,:,:), &
+ ZBPOS1(:,:,:), ZBPOS2(:,:,:), ZBNEG1(:,:,:), ZBNEG2(:,:,:), &
+ ZOMP1(:,:,:), ZOMP2(:,:,:), ZOMN1(:,:,:), ZOMN2(:,:,:) )
CALL DZF_DEVICE( ZMEAN, ZWORK )
!$acc kernels
PRVS(:,:,:) = PRVS(:,:,:) - ZWORK(:,:,:)
@@ -400,10 +410,10 @@ CASE(3) ! WENO 3
!
! ZWORK = MZM(PRUCT)
CALL MZM_DEVICE(PRUCT,ZWORK)
- CALL ADVEC_WENO_K_2_MX(HLBCX, PWT, ZWORK, ZMEAN, TZHALO2_WT%HALO2%WEST, TZHALO2_WT%HALO2%EAST, &
- ZT3D(:,:,:,IZFPOS1), ZT3D(:,:,:,IZFPOS2), ZT3D(:,:,:,IZFNEG1), ZT3D(:,:,:,IZFNEG2), &
- ZT3D(:,:,:,IZBPOS1), ZT3D(:,:,:,IZBPOS2), ZT3D(:,:,:,IZBNEG1), ZT3D(:,:,:,IZBNEG2), &
- ZT3D(:,:,:,IZOMP1), ZT3D(:,:,:,IZOMP2), ZT3D(:,:,:,IZOMN1), ZT3D(:,:,:,IZOMN2) )
+ CALL ADVEC_WENO_K_2_MX(HLBCX, PWT, ZWORK, ZMEAN, TZHALO2_WT%HALO2%WEST, TZHALO2_WT%HALO2%EAST, &
+ ZFPOS1(:,:,:), ZFPOS2(:,:,:), ZFNEG1(:,:,:), ZFNEG2(:,:,:), &
+ ZBPOS1(:,:,:), ZBPOS2(:,:,:), ZBNEG1(:,:,:), ZBNEG2(:,:,:), &
+ ZOMP1(:,:,:), ZOMP2(:,:,:), ZOMN1(:,:,:), ZOMN2(:,:,:) )
CALL DXF_DEVICE(ZMEAN,ZWORK)
!$acc kernels
PRWS(:,:,:) = PRWS(:,:,:) - ZWORK(:,:,:)
@@ -412,10 +422,10 @@ CASE(3) ! WENO 3
IF (.NOT.L2D) THEN
! ZWORK = MZM(PRVCT)
CALL MZM_DEVICE(PRVCT,ZWORK)
- CALL ADVEC_WENO_K_2_MY(HLBCY, PWT, ZWORK, ZMEAN, TZHALO2_WT%HALO2%NORTH, TZHALO2_WT%HALO2%SOUTH, &
- ZT3D(:,:,:,IZFPOS1), ZT3D(:,:,:,IZFPOS2), ZT3D(:,:,:,IZFNEG1), ZT3D(:,:,:,IZFNEG2), &
- ZT3D(:,:,:,IZBPOS1), ZT3D(:,:,:,IZBPOS2), ZT3D(:,:,:,IZBNEG1), ZT3D(:,:,:,IZBNEG2), &
- ZT3D(:,:,:,IZOMP1), ZT3D(:,:,:,IZOMP2), ZT3D(:,:,:,IZOMN1), ZT3D(:,:,:,IZOMN2) )
+ CALL ADVEC_WENO_K_2_MY(HLBCY, PWT, ZWORK, ZMEAN, TZHALO2_WT%HALO2%NORTH, TZHALO2_WT%HALO2%SOUTH, &
+ ZFPOS1(:,:,:), ZFPOS2(:,:,:), ZFNEG1(:,:,:), ZFNEG2(:,:,:), &
+ ZBPOS1(:,:,:), ZBPOS2(:,:,:), ZBNEG1(:,:,:), ZBNEG2(:,:,:), &
+ ZOMP1(:,:,:), ZOMP2(:,:,:), ZOMN1(:,:,:), ZOMN2(:,:,:) )
CALL DYF_DEVICE(ZMEAN,ZWORK)
!$acc kernels
PRWS(:,:,:) = PRWS(:,:,:) - ZWORK(:,:,:)
@@ -424,16 +434,17 @@ CASE(3) ! WENO 3
!
! PRWS = PRWS - DZM(WENO_K_2_WZ(PWT,MZF(PRWCT)))
CALL MZF_DEVICE( PRWCT, ZWORK )
- CALL WENO_K_2_WZ(PWT, ZWORK, ZMEAN, &
- ZT3D(:,:,:,IZFPOS1), ZT3D(:,:,:,IZFPOS2), ZT3D(:,:,:,IZFNEG1), ZT3D(:,:,:,IZFNEG2), &
- ZT3D(:,:,:,IZBPOS1), ZT3D(:,:,:,IZBPOS2), ZT3D(:,:,:,IZBNEG1), ZT3D(:,:,:,IZBNEG2), &
- ZT3D(:,:,:,IZOMP1), ZT3D(:,:,:,IZOMP2), ZT3D(:,:,:,IZOMN1), ZT3D(:,:,:,IZOMN2) )
+ CALL WENO_K_2_WZ(PWT, ZWORK, ZMEAN, &
+ ZFPOS1(:,:,:), ZFPOS2(:,:,:), ZFNEG1(:,:,:), ZFNEG2(:,:,:), &
+ ZBPOS1(:,:,:), ZBPOS2(:,:,:), ZBNEG1(:,:,:), ZBNEG2(:,:,:), &
+ ZOMP1(:,:,:), ZOMP2(:,:,:), ZOMN1(:,:,:), ZOMN2(:,:,:) )
CALL DZM_DEVICE( ZMEAN, ZWORK )
!$acc kernels
PRWS(:,:,:) = PRWS(:,:,:) - ZWORK(:,:,:)
!$acc end kernels
!
- CALL MNH_REL_ZT3D(IZFPOS1,IZFPOS2,IZFNEG1,IZFNEG2,IZBPOS1,IZBPOS2,IZBNEG1,IZBNEG2,IZOMP1,IZOMP2,IZOMN1,IZOMN2)
+ !Release all memory allocated with MNH_MEM_GET calls since last call to MNH_MEM_POSITION_PIN
+ CALL MNH_MEM_RELEASE()
#endif
!
!
@@ -496,19 +507,38 @@ CASE(5) ! WENO 5
CALL GET_HALO(ZMEAN)! Update HALO - maybe not necessary (T.Lunet)
PRWS = PRWS - DZM(ZMEAN)
#else
- CALL MNH_GET_ZT3D(IZFPOS1,IZFPOS2,IZFPOS3,IZFNEG1,IZFNEG2,IZFNEG3,IZBPOS1, &
- IZBPOS2,IZBPOS3,IZBNEG1,IZBNEG2,IZBNEG3,IZOMP1,IZOMP2,IZOMP3,IZOMN1,IZOMN2,IZOMN3)
+ !Pin positions in the pools of MNH memory
+ CALL MNH_MEM_POSITION_PIN()
+
+ CALL MNH_MEM_GET( ZFPOS1, IIU, IJU, IKU )
+ CALL MNH_MEM_GET( ZFPOS2, IIU, IJU, IKU )
+ CALL MNH_MEM_GET( ZFPOS3, IIU, IJU, IKU )
+ CALL MNH_MEM_GET( ZFNEG1, IIU, IJU, IKU )
+ CALL MNH_MEM_GET( ZFNEG2, IIU, IJU, IKU )
+ CALL MNH_MEM_GET( ZFNEG3, IIU, IJU, IKU )
+ CALL MNH_MEM_GET( ZBPOS1, IIU, IJU, IKU )
+ CALL MNH_MEM_GET( ZBPOS2, IIU, IJU, IKU )
+ CALL MNH_MEM_GET( ZBPOS3, IIU, IJU, IKU )
+ CALL MNH_MEM_GET( ZBNEG1, IIU, IJU, IKU )
+ CALL MNH_MEM_GET( ZBNEG2, IIU, IJU, IKU )
+ CALL MNH_MEM_GET( ZBNEG3, IIU, IJU, IKU )
+ CALL MNH_MEM_GET( ZOMP1, IIU, IJU, IKU )
+ CALL MNH_MEM_GET( ZOMP2, IIU, IJU, IKU )
+ CALL MNH_MEM_GET( ZOMP3, IIU, IJU, IKU )
+ CALL MNH_MEM_GET( ZOMN1, IIU, IJU, IKU )
+ CALL MNH_MEM_GET( ZOMN2, IIU, IJU, IKU )
+ CALL MNH_MEM_GET( ZOMN3, IIU, IJU, IKU )
!
! U component
!
CALL MXF_DEVICE(PRUCT,ZWORK)
- CALL ADVEC_WENO_K_3_UX(HLBCX, PUT, ZWORK, ZMEAN, &
- ZT3D(:,:,:,IZFPOS1), ZT3D(:,:,:,IZFPOS2), ZT3D(:,:,:,IZFPOS3), &
- ZT3D(:,:,:,IZFNEG1), ZT3D(:,:,:,IZFNEG2), ZT3D(:,:,:,IZFNEG3), &
- ZT3D(:,:,:,IZBPOS1), ZT3D(:,:,:,IZBPOS2), ZT3D(:,:,:,IZBPOS3), &
- ZT3D(:,:,:,IZBNEG1), ZT3D(:,:,:,IZBNEG2), ZT3D(:,:,:,IZBNEG3), &
- ZT3D(:,:,:,IZOMP1), ZT3D(:,:,:,IZOMP2), ZT3D(:,:,:,IZOMP3), &
- ZT3D(:,:,:,IZOMN1), ZT3D(:,:,:,IZOMN2), ZT3D(:,:,:,IZOMN3) )
+ CALL ADVEC_WENO_K_3_UX(HLBCX, PUT, ZWORK, ZMEAN, &
+ ZFPOS1(:,:,:), ZFPOS2(:,:,:), ZFPOS3(:,:,:), &
+ ZFNEG1(:,:,:), ZFNEG2(:,:,:), ZFNEG3(:,:,:), &
+ ZBPOS1(:,:,:), ZBPOS2(:,:,:), ZBPOS3(:,:,:), &
+ ZBNEG1(:,:,:), ZBNEG2(:,:,:), ZBNEG3(:,:,:), &
+ ZOMP1(:,:,:), ZOMP2(:,:,:), ZOMP3(:,:,:), &
+ ZOMN1(:,:,:), ZOMN2(:,:,:), ZOMN3(:,:,:) )
CALL GET_HALO_D(ZMEAN)! Update HALO
CALL DXM_DEVICE(ZMEAN,ZWORK)
!$acc kernels
@@ -517,13 +547,13 @@ CASE(5) ! WENO 5
!
IF (.NOT.L2D) THEN
CALL MXM_DEVICE(PRVCT,ZWORK)
- CALL ADVEC_WENO_K_3_MY(HLBCY, PUT, ZWORK, ZMEAN, &
- ZT3D(:,:,:,IZFPOS1), ZT3D(:,:,:,IZFPOS2), ZT3D(:,:,:,IZFPOS3), &
- ZT3D(:,:,:,IZFNEG1), ZT3D(:,:,:,IZFNEG2), ZT3D(:,:,:,IZFNEG3), &
- ZT3D(:,:,:,IZBPOS1), ZT3D(:,:,:,IZBPOS2), ZT3D(:,:,:,IZBPOS3), &
- ZT3D(:,:,:,IZBNEG1), ZT3D(:,:,:,IZBNEG2), ZT3D(:,:,:,IZBNEG3), &
- ZT3D(:,:,:,IZOMP1), ZT3D(:,:,:,IZOMP2), ZT3D(:,:,:,IZOMP3), &
- ZT3D(:,:,:,IZOMN1), ZT3D(:,:,:,IZOMN2), ZT3D(:,:,:,IZOMN3) )
+ CALL ADVEC_WENO_K_3_MY(HLBCY, PUT, ZWORK, ZMEAN, &
+ ZFPOS1(:,:,:), ZFPOS2(:,:,:), ZFPOS3(:,:,:), &
+ ZFNEG1(:,:,:), ZFNEG2(:,:,:), ZFNEG3(:,:,:), &
+ ZBPOS1(:,:,:), ZBPOS2(:,:,:), ZBPOS3(:,:,:), &
+ ZBNEG1(:,:,:), ZBNEG2(:,:,:), ZBNEG3(:,:,:), &
+ ZOMP1(:,:,:), ZOMP2(:,:,:), ZOMP3(:,:,:), &
+ ZOMN1(:,:,:), ZOMN2(:,:,:), ZOMN3(:,:,:) )
CALL GET_HALO_D(ZMEAN)! Update HALO
CALL DYF_DEVICE(ZMEAN,ZWORK)
!$acc kernels
@@ -532,13 +562,13 @@ CASE(5) ! WENO 5
END IF
!
CALL MXM_DEVICE(PRWCT,ZWORK)
- CALL WENO_K_3_MZ(PUT,ZWORK,ZMEAN, &
- ZT3D(:,:,:,IZFPOS1), ZT3D(:,:,:,IZFPOS2), ZT3D(:,:,:,IZFPOS3), &
- ZT3D(:,:,:,IZFNEG1), ZT3D(:,:,:,IZFNEG2), ZT3D(:,:,:,IZFNEG3), &
- ZT3D(:,:,:,IZBPOS1), ZT3D(:,:,:,IZBPOS2), ZT3D(:,:,:,IZBPOS3), &
- ZT3D(:,:,:,IZBNEG1), ZT3D(:,:,:,IZBNEG2), ZT3D(:,:,:,IZBNEG3), &
- ZT3D(:,:,:,IZOMP1), ZT3D(:,:,:,IZOMP2), ZT3D(:,:,:,IZOMP3), &
- ZT3D(:,:,:,IZOMN1), ZT3D(:,:,:,IZOMN2), ZT3D(:,:,:,IZOMN3) )
+ CALL WENO_K_3_MZ(PUT,ZWORK,ZMEAN, &
+ ZFPOS1(:,:,:), ZFPOS2(:,:,:), ZFPOS3(:,:,:), &
+ ZFNEG1(:,:,:), ZFNEG2(:,:,:), ZFNEG3(:,:,:), &
+ ZBPOS1(:,:,:), ZBPOS2(:,:,:), ZBPOS3(:,:,:), &
+ ZBNEG1(:,:,:), ZBNEG2(:,:,:), ZBNEG3(:,:,:), &
+ ZOMP1(:,:,:), ZOMP2(:,:,:), ZOMP3(:,:,:), &
+ ZOMN1(:,:,:), ZOMN2(:,:,:), ZOMN3(:,:,:) )
CALL GET_HALO_D(ZMEAN)! Update HALO - maybe not necessary (T.Lunet)
CALL DZF_DEVICE( ZMEAN, ZWORK )
!$acc kernels
@@ -549,13 +579,13 @@ CASE(5) ! WENO 5
!
IF (.NOT.L2D) THEN
CALL MYM_DEVICE(PRUCT,ZWORK)
- CALL ADVEC_WENO_K_3_MX(HLBCX, PVT, ZWORK, ZMEAN, &
- ZT3D(:,:,:,IZFPOS1), ZT3D(:,:,:,IZFPOS2), ZT3D(:,:,:,IZFPOS3), &
- ZT3D(:,:,:,IZFNEG1), ZT3D(:,:,:,IZFNEG2), ZT3D(:,:,:,IZFNEG3), &
- ZT3D(:,:,:,IZBPOS1), ZT3D(:,:,:,IZBPOS2), ZT3D(:,:,:,IZBPOS3), &
- ZT3D(:,:,:,IZBNEG1), ZT3D(:,:,:,IZBNEG2), ZT3D(:,:,:,IZBNEG3), &
- ZT3D(:,:,:,IZOMP1), ZT3D(:,:,:,IZOMP2), ZT3D(:,:,:,IZOMP3), &
- ZT3D(:,:,:,IZOMN1), ZT3D(:,:,:,IZOMN2), ZT3D(:,:,:,IZOMN3) )
+ CALL ADVEC_WENO_K_3_MX(HLBCX, PVT, ZWORK, ZMEAN, &
+ ZFPOS1(:,:,:), ZFPOS2(:,:,:), ZFPOS3(:,:,:), &
+ ZFNEG1(:,:,:), ZFNEG2(:,:,:), ZFNEG3(:,:,:), &
+ ZBPOS1(:,:,:), ZBPOS2(:,:,:), ZBPOS3(:,:,:), &
+ ZBNEG1(:,:,:), ZBNEG2(:,:,:), ZBNEG3(:,:,:), &
+ ZOMP1(:,:,:), ZOMP2(:,:,:), ZOMP3(:,:,:), &
+ ZOMN1(:,:,:), ZOMN2(:,:,:), ZOMN3(:,:,:) )
CALL GET_HALO_D(ZMEAN)! Update HALO
CALL DXF_DEVICE(ZMEAN,ZWORK)
!$acc kernels
@@ -563,13 +593,13 @@ CASE(5) ! WENO 5
!$acc end kernels
!
CALL MYF_DEVICE(PRVCT,ZWORK)
- CALL ADVEC_WENO_K_3_VY(HLBCY, PVT, ZWORK, ZMEAN, &
- ZT3D(:,:,:,IZFPOS1), ZT3D(:,:,:,IZFPOS2), ZT3D(:,:,:,IZFPOS3), &
- ZT3D(:,:,:,IZFNEG1), ZT3D(:,:,:,IZFNEG2), ZT3D(:,:,:,IZFNEG3), &
- ZT3D(:,:,:,IZBPOS1), ZT3D(:,:,:,IZBPOS2), ZT3D(:,:,:,IZBPOS3), &
- ZT3D(:,:,:,IZBNEG1), ZT3D(:,:,:,IZBNEG2), ZT3D(:,:,:,IZBNEG3), &
- ZT3D(:,:,:,IZOMP1), ZT3D(:,:,:,IZOMP2), ZT3D(:,:,:,IZOMP3), &
- ZT3D(:,:,:,IZOMN1), ZT3D(:,:,:,IZOMN2), ZT3D(:,:,:,IZOMN3) )
+ CALL ADVEC_WENO_K_3_VY(HLBCY, PVT, ZWORK, ZMEAN, &
+ ZFPOS1(:,:,:), ZFPOS2(:,:,:), ZFPOS3(:,:,:), &
+ ZFNEG1(:,:,:), ZFNEG2(:,:,:), ZFNEG3(:,:,:), &
+ ZBPOS1(:,:,:), ZBPOS2(:,:,:), ZBPOS3(:,:,:), &
+ ZBNEG1(:,:,:), ZBNEG2(:,:,:), ZBNEG3(:,:,:), &
+ ZOMP1(:,:,:), ZOMP2(:,:,:), ZOMP3(:,:,:), &
+ ZOMN1(:,:,:), ZOMN2(:,:,:), ZOMN3(:,:,:) )
CALL GET_HALO_D(ZMEAN)! Update HALO
CALL DYM_DEVICE(ZMEAN,ZWORK)
!$acc kernels
@@ -577,13 +607,13 @@ CASE(5) ! WENO 5
!$acc end kernels
!
CALL MYM_DEVICE(PRWCT,ZWORK)
- CALL WENO_K_3_MZ(PVT,ZWORK,ZMEAN, &
- ZT3D(:,:,:,IZFPOS1), ZT3D(:,:,:,IZFPOS2), ZT3D(:,:,:,IZFPOS3), &
- ZT3D(:,:,:,IZFNEG1), ZT3D(:,:,:,IZFNEG2), ZT3D(:,:,:,IZFNEG3), &
- ZT3D(:,:,:,IZBPOS1), ZT3D(:,:,:,IZBPOS2), ZT3D(:,:,:,IZBPOS3), &
- ZT3D(:,:,:,IZBNEG1), ZT3D(:,:,:,IZBNEG2), ZT3D(:,:,:,IZBNEG3), &
- ZT3D(:,:,:,IZOMP1), ZT3D(:,:,:,IZOMP2), ZT3D(:,:,:,IZOMP3), &
- ZT3D(:,:,:,IZOMN1), ZT3D(:,:,:,IZOMN2), ZT3D(:,:,:,IZOMN3) )
+ CALL WENO_K_3_MZ(PVT, ZWORK, ZMEAN, &
+ ZFPOS1(:,:,:), ZFPOS2(:,:,:), ZFPOS3(:,:,:), &
+ ZFNEG1(:,:,:), ZFNEG2(:,:,:), ZFNEG3(:,:,:), &
+ ZBPOS1(:,:,:), ZBPOS2(:,:,:), ZBPOS3(:,:,:), &
+ ZBNEG1(:,:,:), ZBNEG2(:,:,:), ZBNEG3(:,:,:), &
+ ZOMP1(:,:,:), ZOMP2(:,:,:), ZOMP3(:,:,:), &
+ ZOMN1(:,:,:), ZOMN2(:,:,:), ZOMN3(:,:,:) )
CALL GET_HALO_D(ZMEAN)! Update HALO - maybe not necessary (T.Lunet)
CALL DZF_DEVICE( ZMEAN, ZWORK )
!$acc kernels
@@ -594,13 +624,13 @@ CASE(5) ! WENO 5
! W component
!
CALL MZM_DEVICE(PRUCT,ZWORK)
- CALL ADVEC_WENO_K_3_MX(HLBCX, PWT, ZWORK, ZMEAN, &
- ZT3D(:,:,:,IZFPOS1), ZT3D(:,:,:,IZFPOS2), ZT3D(:,:,:,IZFPOS3), &
- ZT3D(:,:,:,IZFNEG1), ZT3D(:,:,:,IZFNEG2), ZT3D(:,:,:,IZFNEG3), &
- ZT3D(:,:,:,IZBPOS1), ZT3D(:,:,:,IZBPOS2), ZT3D(:,:,:,IZBPOS3), &
- ZT3D(:,:,:,IZBNEG1), ZT3D(:,:,:,IZBNEG2), ZT3D(:,:,:,IZBNEG3), &
- ZT3D(:,:,:,IZOMP1), ZT3D(:,:,:,IZOMP2), ZT3D(:,:,:,IZOMP3), &
- ZT3D(:,:,:,IZOMN1), ZT3D(:,:,:,IZOMN2), ZT3D(:,:,:,IZOMN3) )
+ CALL ADVEC_WENO_K_3_MX(HLBCX, PWT, ZWORK, ZMEAN, &
+ ZFPOS1(:,:,:), ZFPOS2(:,:,:), ZFPOS3(:,:,:), &
+ ZFNEG1(:,:,:), ZFNEG2(:,:,:), ZFNEG3(:,:,:), &
+ ZBPOS1(:,:,:), ZBPOS2(:,:,:), ZBPOS3(:,:,:), &
+ ZBNEG1(:,:,:), ZBNEG2(:,:,:), ZBNEG3(:,:,:), &
+ ZOMP1(:,:,:), ZOMP2(:,:,:), ZOMP3(:,:,:), &
+ ZOMN1(:,:,:), ZOMN2(:,:,:), ZOMN3(:,:,:) )
CALL GET_HALO_D(ZMEAN)! Update HALO
CALL DXF_DEVICE(ZMEAN,ZWORK)
!$acc kernels
@@ -609,13 +639,13 @@ CASE(5) ! WENO 5
!
IF (.NOT.L2D) THEN
CALL MZM_DEVICE(PRVCT,ZWORK)
- CALL ADVEC_WENO_K_3_MY(HLBCY, PWT, ZWORK, ZMEAN, &
- ZT3D(:,:,:,IZFPOS1), ZT3D(:,:,:,IZFPOS2), ZT3D(:,:,:,IZFPOS3), &
- ZT3D(:,:,:,IZFNEG1), ZT3D(:,:,:,IZFNEG2), ZT3D(:,:,:,IZFNEG3), &
- ZT3D(:,:,:,IZBPOS1), ZT3D(:,:,:,IZBPOS2), ZT3D(:,:,:,IZBPOS3), &
- ZT3D(:,:,:,IZBNEG1), ZT3D(:,:,:,IZBNEG2), ZT3D(:,:,:,IZBNEG3), &
- ZT3D(:,:,:,IZOMP1), ZT3D(:,:,:,IZOMP2), ZT3D(:,:,:,IZOMP3), &
- ZT3D(:,:,:,IZOMN1), ZT3D(:,:,:,IZOMN2), ZT3D(:,:,:,IZOMN3) )
+ CALL ADVEC_WENO_K_3_MY(HLBCY, PWT, ZWORK, ZMEAN, &
+ ZFPOS1(:,:,:), ZFPOS2(:,:,:), ZFPOS3(:,:,:), &
+ ZFNEG1(:,:,:), ZFNEG2(:,:,:), ZFNEG3(:,:,:), &
+ ZBPOS1(:,:,:), ZBPOS2(:,:,:), ZBPOS3(:,:,:), &
+ ZBNEG1(:,:,:), ZBNEG2(:,:,:), ZBNEG3(:,:,:), &
+ ZOMP1(:,:,:), ZOMP2(:,:,:), ZOMP3(:,:,:), &
+ ZOMN1(:,:,:), ZOMN2(:,:,:), ZOMN3(:,:,:) )
CALL GET_HALO_D(ZMEAN)! Update HALO
CALL DYF_DEVICE(ZMEAN,ZWORK)
!$acc kernels
@@ -624,34 +654,38 @@ CASE(5) ! WENO 5
END IF
!
CALL MZF_DEVICE( PRWCT, ZWORK )
- CALL WENO_K_3_WZ(PWT,ZWORK,ZMEAN, &
- ZT3D(:,:,:,IZFPOS1), ZT3D(:,:,:,IZFPOS2), ZT3D(:,:,:,IZFPOS3), &
- ZT3D(:,:,:,IZFNEG1), ZT3D(:,:,:,IZFNEG2), ZT3D(:,:,:,IZFNEG3), &
- ZT3D(:,:,:,IZBPOS1), ZT3D(:,:,:,IZBPOS2), ZT3D(:,:,:,IZBPOS3), &
- ZT3D(:,:,:,IZBNEG1), ZT3D(:,:,:,IZBNEG2), ZT3D(:,:,:,IZBNEG3), &
- ZT3D(:,:,:,IZOMP1), ZT3D(:,:,:,IZOMP2), ZT3D(:,:,:,IZOMP3), &
- ZT3D(:,:,:,IZOMN1), ZT3D(:,:,:,IZOMN2), ZT3D(:,:,:,IZOMN3) )
+ CALL WENO_K_3_WZ(PWT,ZWORK,ZMEAN, &
+ ZFPOS1(:,:,:), ZFPOS2(:,:,:), ZFPOS3(:,:,:), &
+ ZFNEG1(:,:,:), ZFNEG2(:,:,:), ZFNEG3(:,:,:), &
+ ZBPOS1(:,:,:), ZBPOS2(:,:,:), ZBPOS3(:,:,:), &
+ ZBNEG1(:,:,:), ZBNEG2(:,:,:), ZBNEG3(:,:,:), &
+ ZOMP1(:,:,:), ZOMP2(:,:,:), ZOMP3(:,:,:), &
+ ZOMN1(:,:,:), ZOMN2(:,:,:), ZOMN3(:,:,:) )
CALL GET_HALO_D(ZMEAN)! Update HALO - maybe not necessary (T.Lunet)
CALL DZM_DEVICE( ZMEAN, ZWORK )
!$acc kernels
PRWS(:,:,:) = PRWS(:,:,:) - ZWORK(:,:,:)
!$acc end kernels
!
- CALL MNH_REL_ZT3D(IZFPOS1,IZFPOS2,IZFPOS3,IZFNEG1,IZFNEG2,IZFNEG3,IZBPOS1, &
- IZBPOS2,IZBPOS3,IZBNEG1,IZBNEG2,IZBNEG3,IZOMP1,IZOMP2,IZOMP3,IZOMN1,IZOMN2,IZOMN3)
+ !Release all memory allocated with MNH_MEM_GET calls since last call to MNH_MEM_POSITION_PIN
+ CALL MNH_MEM_RELEASE()
#endif
!
!
END SELECT
! ---------------------------------
!$acc update self(PRUS,PRVS,PRWS)
-!
+
+!$acc end data
+
+!Release all memory allocated with MNH_MEM_GET calls since beginning of subroutine
+CALL MNH_MEM_RELEASE()
+
IF (MPPDB_INITIALIZED) THEN
CALL MPPDB_CHECK(PRUS,"ADVECUVW_WENO_K end:PRUS")
CALL MPPDB_CHECK(PRVS,"ADVECUVW_WENO_K end:PRVS")
CALL MPPDB_CHECK(PRWS,"ADVECUVW_WENO_K end:PRWS")
END IF
-!$acc end data
END SUBROUTINE ADVECUVW_WENO_K
diff --git a/src/MNH/contrav.f90 b/src/MNH/contrav.f90
index cf8a2c9b2..7bf731022 100644
--- a/src/MNH/contrav.f90
+++ b/src/MNH/contrav.f90
@@ -32,7 +32,7 @@ END SUBROUTINE CONTRAV
!
#ifdef MNH_OPENACC
SUBROUTINE CONTRAV_DEVICE(HLBCX,HLBCY,PRUT,PRVT,PRWT,PDXX,PDYY,PDZZ,PDZX,PDZY, &
- PRUCT,PRVCT,PRWCT,KADV_ORDER,Z1,Z2,ODATA_ON_DEVICE )
+ PRUCT,PRVCT,PRWCT,KADV_ORDER,ODATA_ON_DEVICE )
!
!
CHARACTER (LEN=4), DIMENSION(2), INTENT(IN) :: HLBCX ! X direction LBC type
@@ -49,7 +49,6 @@ REAL, DIMENSION(:,:,:), INTENT(OUT) :: PRUCT ! Contrav comp along x-bar
REAL, DIMENSION(:,:,:), INTENT(OUT) :: PRVCT ! Contrav comp along y-bar
REAL, DIMENSION(:,:,:), INTENT(OUT) :: PRWCT ! Contrav comp along z-bar
INTEGER, INTENT(IN) :: KADV_ORDER ! Order of the advection scheme
-REAL, DIMENSION(:,:,:), INTENT(OUT) :: Z1,Z2 ! Work arrays
LOGICAL, OPTIONAL, INTENT(IN) :: ODATA_ON_DEVICE ! Is some of the data on the accelerator device
!
!
@@ -450,7 +449,7 @@ END SUBROUTINE CONTRAV
#ifdef MNH_OPENACC
! ##############################################################
SUBROUTINE CONTRAV_DEVICE(HLBCX,HLBCY,PRUT,PRVT,PRWT,PDXX,PDYY,PDZZ,PDZX,PDZY, &
- PRUCT,PRVCT,PRWCT,KADV_ORDER,Z1,Z2,ODATA_ON_DEVICE )
+ PRUCT,PRVCT,PRWCT,KADV_ORDER,ODATA_ON_DEVICE )
! ##############################################################
!
!!**** *CONTRAV * - computes the contravariant components from the
@@ -524,6 +523,7 @@ USE MODD_PARAMETERS
USE MODE_ll
USE MODE_MPPDB
#ifdef MNH_OPENACC
+USE MODE_MNH_ZWORK, ONLY: MNH_MEM_GET, MNH_MEM_POSITION_PIN, MNH_MEM_RELEASE
use mode_msg
#endif
!
@@ -552,7 +552,6 @@ REAL, DIMENSION(:,:,:), INTENT(OUT) :: PRUCT ! Contrav comp along x-bar
REAL, DIMENSION(:,:,:), INTENT(OUT) :: PRVCT ! Contrav comp along y-bar
REAL, DIMENSION(:,:,:), INTENT(OUT) :: PRWCT ! Contrav comp along z-bar
INTEGER, INTENT(IN) :: KADV_ORDER ! Order of the advection scheme
-REAL, DIMENSION(:,:,:), INTENT(OUT) :: Z1,Z2 ! Work arrays
LOGICAL, OPTIONAL, INTENT(IN) :: ODATA_ON_DEVICE ! Is some of the data on the accelerator device
!
!
@@ -566,12 +565,12 @@ INTEGER :: IINFO_ll
LOGICAL :: GDATA_ON_DEVICE
real :: ZTMP1, ZTMP2 ! Intermediate work variables
REAL, DIMENSION(:,:), ALLOCATABLE :: ZU_EAST, ZV_NORTH, ZDZX_EAST, ZDZY_NORTH
+REAL, DIMENSION(:,:,:), POINTER, CONTIGUOUS :: Z1,Z2 ! Work arrays
TYPE(LIST_ll), POINTER :: TZFIELD_U, TZFIELD_V, TZFIELD_DZX, TZFIELD_DZY
TYPE(HALO2LIST_ll), POINTER :: TZHALO2_U, TZHALO2_V, TZHALO2_DZX, TZHALO2_DZY
!
LOGICAL :: GWEST,GEAST,GSOUTH,GNORTH
!
-!$acc data present( PRUT, PRVT, PRWT, PDXX, PDYY, PDZZ, PDZX, PDZY, PRUCT, PRVCT, PRWCT, Z1, Z2 )
IF ( PRESENT(ODATA_ON_DEVICE) ) THEN
GDATA_ON_DEVICE = ODATA_ON_DEVICE
@@ -598,7 +597,13 @@ END IF
IIU= SIZE(PDXX,1)
IJU= SIZE(PDXX,2)
IKU= SIZE(PDXX,3)
-!
+
+!Pin positions in the pools of MNH memory
+CALL MNH_MEM_POSITION_PIN()
+
+CALL MNH_MEM_GET( Z1, IIU, IJU, IKU )
+CALL MNH_MEM_GET( Z2, IIU, IJU, IKU )
+
GWEST = ( HLBCX(1) /= 'CYCL' .AND. LWEST_ll() )
GEAST = ( HLBCX(2) /= 'CYCL' .AND. LEAST_ll() )
GSOUTH = ( HLBCY(1) /= 'CYCL' .AND. LSOUTH_ll() )
@@ -608,7 +613,9 @@ CALL GET_INDICE_ll( IIB,IJB,IIE,IJE)
!
IKB=1+JPVEXT
IKE=IKU - JPVEXT
-!
+
+!$acc data present( PRUT, PRVT, PRWT, PDXX, PDYY, PDZZ, PDZX, PDZY, PRUCT, PRVCT, PRWCT, Z1, Z2 )
+
IF (GDATA_ON_DEVICE) THEN
!PW TODO:remplacer (ailleurs aussi...) 1/PDXX... par PINV_PDXX (fait pour la turbulence...) cfr MNH/turb_hor_splt.f90
!$acc kernels
@@ -891,7 +898,12 @@ IF (KADV_ORDER == 4 ) THEN
END IF
END IF FLAT
-!-----------------------------------------------------------------------
+
+!$acc end data
+
+!Release all memory allocated with MNH_MEM_GET calls since last call to MNH_MEM_POSITION_PIN
+CALL MNH_MEM_RELEASE()
+
IF (MPPDB_INITIALIZED) THEN
!Check all OUT arrays
CALL MPPDB_CHECK(PRUCT,"CONTRAV end:PRUCT")
@@ -899,7 +911,6 @@ IF (MPPDB_INITIALIZED) THEN
CALL MPPDB_CHECK(PRWCT,"CONTRAV end:PRWCT")
END IF
-!$acc end data
-
+!-----------------------------------------------------------------------
END SUBROUTINE CONTRAV_DEVICE
#endif
diff --git a/src/MNH/gradient_m.f90 b/src/MNH/gradient_m.f90
index f99ed00d5..4bcc5378a 100644
--- a/src/MNH/gradient_m.f90
+++ b/src/MNH/gradient_m.f90
@@ -277,9 +277,9 @@ END FUNCTION GX_M_M
!
!
USE MODI_SHUMAN_DEVICE
-USE MODD_CONF, ONLY:LFLAT
+USE MODD_CONF, ONLY: LFLAT
!
-USE MODE_MNH_ZWORK, ONLY : ZT3D, MNH_GET_ZT3D , MNH_REL_ZT3D
+USE MODE_MNH_ZWORK, ONLY: MNH_MEM_GET, MNH_MEM_POSITION_PIN, MNH_MEM_RELEASE
!
IMPLICIT NONE
!
@@ -297,44 +297,54 @@ REAL, DIMENSION(SIZE(PA,1),SIZE(PA,2),SIZE(PA,3)), INTENT(OUT) :: PGX_M_M ! res
!
!* 0.2 declaration of local variables
!
-INTEGER :: IDX1,IDX2,IDX3,IDX4
+INTEGER :: IIU, IJU, IKU
+REAL, DIMENSION(:,:,:), POINTER , CONTIGUOUS :: ZTMP1, ZTMP2, ZTMP3, ZTMP4
!
!----------------------------------------------------------------------------
+IIU = SIZE( PA, 1 )
+IJU = SIZE( PA, 2 )
+IKU = SIZE( PA, 3 )
+
+!Pin positions in the pools of MNH memory
+CALL MNH_MEM_POSITION_PIN()
-!$acc data present( PA, PDXX, PDZZ, PDZX, PGX_M_M )
+CALL MNH_MEM_GET( ZTMP1, IIU, IJU, IKU )
+CALL MNH_MEM_GET( ZTMP2, IIU, IJU, IKU )
+CALL MNH_MEM_GET( ZTMP3, IIU, IJU, IKU )
+CALL MNH_MEM_GET( ZTMP4, IIU, IJU, IKU )
-CALL MNH_GET_ZT3D(IDX1,IDX2,IDX3,IDX4)
+!$acc data present( PA, PDXX, PDZZ, PDZX, PGX_M_M, ZTMP1, ZTMP2, ZTMP3, ZTMP4)
!
!* 1. DEFINITION of GX_M_M
! --------------------
!
IF (.NOT. LFLAT) THEN
- CALL MXM_DEVICE(PA(:,:,:),ZT3D(:,:,:,IDX1))
- CALL DXF_DEVICE(ZT3D(:,:,:,IDX1),ZT3D(:,:,:,IDX2))
- CALL MXF_DEVICE(PDZX,ZT3D(:,:,:,IDX1))
- CALL DZM_DEVICE( PA(:,:,:), ZT3D(:,:,:,IDX3) )
+ CALL MXM_DEVICE( PA(:,:,:), ZTMP1(:,:,:) )
+ CALL DXF_DEVICE( ZTMP1(:,:,:), ZTMP2(:,:,:) )
+ CALL MXF_DEVICE( PDZX, ZTMP1(:,:,:) )
+ CALL DZM_DEVICE( PA(:,:,:), ZTMP3(:,:,:) )
!$acc kernels
- ZT3D(:,:,:,IDX4) = ZT3D(:,:,:,IDX1)*ZT3D(:,:,:,IDX3)/PDZZ(:,:,:)
+ ZTMP4(:,:,:) = ZTMP1(:,:,:) * ZTMP3(:,:,:) / PDZZ(:,:,:)
!$acc end kernels
- CALL MZF_DEVICE( ZT3D(:,:,:,IDX4), ZT3D(:,:,:,IDX1) )
- CALL MXF_DEVICE(PDXX(:,:,:),ZT3D(:,:,:,IDX3))
+ CALL MZF_DEVICE( ZTMP4(:,:,:), ZTMP1(:,:,:) )
+ CALL MXF_DEVICE( PDXX(:,:,:), ZTMP3(:,:,:) )
!$acc kernels
- PGX_M_M(:,:,:)= (ZT3D(:,:,:,IDX2) - ZT3D(:,:,:,IDX1) &
- ) /ZT3D(:,:,:,IDX3)
+ PGX_M_M(:,:,:)= (ZTMP2(:,:,:) - ZTMP1(:,:,:) ) / ZTMP3(:,:,:)
!$acc end kernels
ELSE
- CALL MXM_DEVICE(PA(:,:,:),ZT3D(:,:,:,IDX1))
- CALL DXF_DEVICE(ZT3D(:,:,:,IDX1),ZT3D(:,:,:,IDX2))
- CALL MXF_DEVICE(PDXX(:,:,:),ZT3D(:,:,:,IDX3))
+ CALL MXM_DEVICE( PA(:,:,:), ZTMP1(:,:,:) )
+ CALL DXF_DEVICE( ZTMP1(:,:,:), ZTMP2(:,:,:) )
+ CALL MXF_DEVICE( PDXX(:,:,:), ZTMP3(:,:,:) )
!$acc kernels
- PGX_M_M(:,:,:)= ZT3D(:,:,:,IDX2) / ZT3D(:,:,:,IDX3)
+ PGX_M_M(:,:,:)= ZTMP2(:,:,:) / ZTMP3(:,:,:)
!$acc end kernels
END IF
!
-CALL MNH_REL_ZT3D(IDX1,IDX2,IDX3,IDX4)
-
!$acc end data
+!Release all memory allocated with MNH_MEM_GET calls since last call to MNH_MEM_POSITION_PIN
+CALL MNH_MEM_RELEASE()
+
!----------------------------------------------------------------------------
!
END SUBROUTINE GX_M_M_DEVICE
@@ -444,10 +454,10 @@ END FUNCTION GY_M_M
!* 0. DECLARATIONS
!
!
-USE MODD_CONF, ONLY:LFLAT
+USE MODD_CONF, ONLY:LFLAT
USE MODI_SHUMAN_DEVICE
!
-USE MODE_MNH_ZWORK, ONLY : ZT3D, MNH_GET_ZT3D , MNH_REL_ZT3D
+USE MODE_MNH_ZWORK, ONLY: MNH_MEM_GET, MNH_MEM_POSITION_PIN, MNH_MEM_RELEASE
!
IMPLICIT NONE
!
@@ -463,45 +473,55 @@ REAL, DIMENSION(SIZE(PA,1),SIZE(PA,2),SIZE(PA,3)), INTENT(OUT) :: PGY_M_M ! res
!
!* 0.2 declaration of local variables
!
-INTEGER :: IDX1,IDX2,IDX3,IDX4
+INTEGER :: IIU, IJU, IKU
+REAL, DIMENSION(:,:,:), POINTER, CONTIGUOUS :: ZTMP1, ZTMP2, ZTMP3, ZTMP4
!
!----------------------------------------------------------------------------
+IIU = SIZE( PA, 1 )
+IJU = SIZE( PA, 2 )
+IKU = SIZE( PA, 3 )
+
+!Pin positions in the pools of MNH memory
+CALL MNH_MEM_POSITION_PIN()
-!$acc data present( PA, PDYY, PDZZ, PDZY, PGY_M_M )
+CALL MNH_MEM_GET( ZTMP1, IIU, IJU, IKU )
+CALL MNH_MEM_GET( ZTMP2, IIU, IJU, IKU )
+CALL MNH_MEM_GET( ZTMP3, IIU, IJU, IKU )
+CALL MNH_MEM_GET( ZTMP4, IIU, IJU, IKU )
-CALL MNH_GET_ZT3D(IDX1,IDX2,IDX3,IDX4)
+!$acc data present( PA, PDYY, PDZZ, PDZY, PGY_M_M, ZTMP1, ZTMP2, ZTMP3, ZTMP4)
!
!* 1. DEFINITION of GY_M_M
! --------------------
!
!
IF (.NOT. LFLAT) THEN
- CALL MYM_DEVICE(PA,ZT3D(:,:,:,IDX1))
- CALL DYF_DEVICE(ZT3D(:,:,:,IDX1),ZT3D(:,:,:,IDX2))
- CALL MYF_DEVICE(PDZY,ZT3D(:,:,:,IDX1))
- CALL DZM_DEVICE( PA, ZT3D(:,:,:,IDX3) )
+ CALL MYM_DEVICE( PA, ZTMP1(:,:,:) )
+ CALL DYF_DEVICE( ZTMP1(:,:,:), ZTMP2(:,:,:) )
+ CALL MYF_DEVICE( PDZY, ZTMP1(:,:,:) )
+ CALL DZM_DEVICE( PA, ZTMP3(:,:,:) )
!$acc kernels
- ZT3D(:,:,:,IDX4) = ZT3D(:,:,:,IDX1)*ZT3D(:,:,:,IDX3)/PDZZ
+ ZTMP4(:,:,:) = ZTMP1(:,:,:) * ZTMP3(:,:,:) / PDZZ(:,:,:)
!$acc end kernels
- CALL MZF_DEVICE( ZT3D(:,:,:,IDX4), ZT3D(:,:,:,IDX1) )
- CALL MYF_DEVICE(PDYY,ZT3D(:,:,:,IDX3))
+ CALL MZF_DEVICE( ZTMP4(:,:,:), ZTMP1(:,:,:) )
+ CALL MYF_DEVICE( PDYY, ZTMP3(:,:,:) )
!$acc kernels
- PGY_M_M(:,:,:)= (ZT3D(:,:,:,IDX2) - ZT3D(:,:,:,IDX1) &
- ) /ZT3D(:,:,:,IDX3)
+ PGY_M_M(:,:,:)= ( ZTMP2(:,:,:) - ZTMP1(:,:,:) ) / ZTMP3(:,:,:)
!$acc end kernels
ELSE
- CALL MYM_DEVICE(PA,ZT3D(:,:,:,IDX1))
- CALL DYF_DEVICE(ZT3D(:,:,:,IDX1),ZT3D(:,:,:,IDX2))
- CALL MYF_DEVICE(PDYY,ZT3D(:,:,:,IDX1))
+ CALL MYM_DEVICE( PA, ZTMP1(:,:,:) )
+ CALL DYF_DEVICE( ZTMP1(:,:,:), ZTMP2(:,:,:) )
+ CALL MYF_DEVICE( PDYY, ZTMP1(:,:,:) )
!$acc kernels
- PGY_M_M(:,:,:)= ZT3D(:,:,:,IDX2)/ZT3D(:,:,:,IDX1)
+ PGY_M_M(:,:,:)= ZTMP2(:,:,:) / ZTMP1(:,:,:)
!$acc end kernels
ENDIF
!
-CALL MNH_REL_ZT3D(IDX1,IDX2,IDX3,IDX4)
-
!$acc end data
+!Release all memory allocated with MNH_MEM_GET calls since last call to MNH_MEM_POSITION_PIN
+CALL MNH_MEM_RELEASE()
+
!----------------------------------------------------------------------------
!
END SUBROUTINE GY_M_M_DEVICE
@@ -600,7 +620,7 @@ END FUNCTION GZ_M_M
!
USE MODI_SHUMAN_DEVICE
!
-USE MODE_MNH_ZWORK, ONLY : ZT3D, MNH_GET_ZT3D , MNH_REL_ZT3D
+USE MODE_MNH_ZWORK, ONLY: MNH_MEM_GET, MNH_MEM_POSITION_PIN, MNH_MEM_RELEASE
use mode_msg
!
IMPLICIT NONE
@@ -615,28 +635,38 @@ REAL, DIMENSION(SIZE(PA,1),SIZE(PA,2),SIZE(PA,3)), INTENT(OUT) :: PGZ_M_M ! res
!
!* 0.2 declaration of local variables
!
-INTEGER :: IDX1,IDX2
+INTEGER :: IIU, IJU, IKU
+REAL, DIMENSION(:,:,:), POINTER , CONTIGUOUS :: ZTMP1, ZTMP2, ZTMP3, ZTMP4
!
!----------------------------------------------------------------------------
+IIU = SIZE( PA, 1 )
+IJU = SIZE( PA, 2 )
+IKU = SIZE( PA, 3 )
+
+!Pin positions in the pools of MNH memory
+CALL MNH_MEM_POSITION_PIN()
-!$acc data present( PA, PDZZ, PGZ_M_M )
+CALL MNH_MEM_GET( ZTMP1, IIU, IJU, IKU )
+CALL MNH_MEM_GET( ZTMP2, IIU, IJU, IKU )
+
+!$acc data present( PA, PDZZ, PGZ_M_M, ZTMP1, ZTMP2 )
call Print_msg( NVERB_WARNING, 'GEN', 'GZ_M_M_DEVICE', 'OpenACC: not yet tested' )
-CALL MNH_GET_ZT3D(IDX1,IDX2)
!
!* 1. DEFINITION of GZ_M_M
! --------------------
!
-CALL DZM_DEVICE( PA(:,:,:), ZT3D(:,:,:,IDX1) )
+CALL DZM_DEVICE( PA(:,:,:), ZTMP1(:,:,:) )
!$acc kernels
-ZT3D(:,:,:,IDX2) = ZT3D(:,:,:,IDX1)/PDZZ(:,:,:)
+ZTMP2(:,:,:) = ZTMP1(:,:,:) / PDZZ(:,:,:)
!$acc end kernels
-CALL MZF_DEVICE( ZT3D(:,:,:,IDX2), PGZ_M_M )
-!
-CALL MNH_REL_ZT3D(IDX1,IDX2)
+CALL MZF_DEVICE( ZTMP2(:,:,:), PGZ_M_M(:,:,:) )
!$acc end data
+!Release all memory allocated with MNH_MEM_GET calls since last call to MNH_MEM_POSITION_PIN
+CALL MNH_MEM_RELEASE()
+
!----------------------------------------------------------------------------
!
END SUBROUTINE GZ_M_M_DEVICE
diff --git a/src/ZSOLVER/advection_metsv.f90 b/src/ZSOLVER/advection_metsv.f90
index 71049d6b1..e0e2c003d 100644
--- a/src/ZSOLVER/advection_metsv.f90
+++ b/src/ZSOLVER/advection_metsv.f90
@@ -175,8 +175,7 @@ use mode_argslist_ll, only: ADD3DFIELD_ll, ADD4DFIELD_ll, CLEANLIST_ll, LIST_
use mode_budget, only: Budget_store_init, Budget_store_end
#ifdef MNH_OPENACC
USE MODE_DEVICE
-USE MODE_MNH_ZWORK, ONLY: MNH_MEM_GET, MNH_MEM_POSITION_PIN, MNH_MEM_RELEASE, &
- MNH_GET_ZT3D, MNH_REL_ZT3D, ZT3D
+USE MODE_MNH_ZWORK, ONLY: MNH_MEM_GET, MNH_MEM_POSITION_PIN, MNH_MEM_RELEASE
#endif
use mode_exchange_ll, only: UPDATE_HALO_ll
USE MODE_IO_FIELD_WRITE, only: IO_Field_write
@@ -301,7 +300,6 @@ INTEGER :: ISPLIT_PPM ! temporal time splitting
INTEGER :: IIB, IIE, IJB, IJE,IKB,IKE
#ifdef MNH_OPENACC
CHARACTER(LEN=3) :: YNUM
-INTEGER :: IZ1, IZ2
#endif
TYPE(TFIELDDATA) :: TZFIELD
!
@@ -483,8 +481,6 @@ CALL INIT_ON_HOST_AND_DEVICE(ZRHOY1,PVALUE=-1e95,HNAME='ADVECTION_METSV::ZRHOY1'
CALL INIT_ON_HOST_AND_DEVICE(ZRHOY2,PVALUE=-1e96,HNAME='ADVECTION_METSV::ZRHOY2')
CALL INIT_ON_HOST_AND_DEVICE(ZRHOZ1,PVALUE=-1e97,HNAME='ADVECTION_METSV::ZRHOZ1')
CALL INIT_ON_HOST_AND_DEVICE(ZRHOZ2,PVALUE=-1e98,HNAME='ADVECTION_METSV::ZRHOZ2')
-!
-CALL MNH_GET_ZT3D(IZ1, IZ2)
#endif
!
IF(LBLOWSNOW) THEN ! Put 2D Canopy blowing snow variables into a 3D array for advection
@@ -517,10 +513,10 @@ END IF
#else
IF (HUVW_ADV_SCHEME=='CEN2ND' ) THEN
CALL CONTRAV_DEVICE (HLBCX,HLBCY,PUT,PVT,PWT,PDXX,PDYY,PDZZ,PDZX,PDZY,ZRUCPPM,ZRVCPPM,ZRWCPPM,2, &
- ZT3D(:,:,:,IZ1),ZT3D(:,:,:,IZ2),ODATA_ON_DEVICE=.TRUE.)
+ ODATA_ON_DEVICE=.TRUE.)
ELSE
CALL CONTRAV_DEVICE (HLBCX,HLBCY,PUT,PVT,PWT,PDXX,PDYY,PDZZ,PDZX,PDZY,ZRUCPPM,ZRVCPPM,ZRWCPPM,4, &
- ZT3D(:,:,:,IZ1),ZT3D(:,:,:,IZ2),ODATA_ON_DEVICE=.TRUE.)
+ ODATA_ON_DEVICE=.TRUE.)
END IF
#endif
!
@@ -1074,10 +1070,6 @@ end if
! Remove non-physical negative values (unnecessary in a perfect world) + corresponding budgets
call Sources_neg_correct( hcloud, 'NEADV', krr, ptstep, ppabst, ptht, prt, prths, prrs, prsvs )
-#ifdef MNH_OPENACC
-CALL MNH_REL_ZT3D(IZ1, IZ2)
-#endif
-
IF (MPPDB_INITIALIZED) THEN
!Check all INOUT arrays
CALL MPPDB_CHECK(PRTHS,"ADVECTION_METSV end:PRTHS")
diff --git a/src/ZSOLVER/advection_uvw.f90 b/src/ZSOLVER/advection_uvw.f90
index 6f2e14a80..b731655b2 100644
--- a/src/ZSOLVER/advection_uvw.f90
+++ b/src/ZSOLVER/advection_uvw.f90
@@ -108,8 +108,6 @@ use mode_budget, only: Budget_store_init, Budget_store_end
USE MODE_ll
#ifdef MNH_OPENACC
USE MODE_DEVICE
-USE MODE_MNH_ZWORK, ONLY: MNH_MEM_GET, MNH_MEM_POSITION_PIN, MNH_MEM_RELEASE, &
- MNH_GET_ZT3D, MNH_GET_ZT4D, MNH_REL_ZT3D, MNH_REL_ZT4D, ZT3D
#endif
use mode_mppdb
@@ -196,11 +194,6 @@ TYPE(LIST_ll), POINTER :: TZFIELD_ll ! list of fields to exchange
TYPE(LIST_ll), POINTER :: TZFIELDS_ll ! list of fields to exchange
TYPE(LIST_ll), POINTER :: TZFIELDS0_ll ! list of fields to exchange
!
-#ifdef MNH_OPENACC
-INTEGER :: ISPL, IZUT, IZVT, IZWT, IZ1, IZ2
-INTEGER :: IZRUSB, IZRUSE, IZRVSB, IZRVSE, IZRWSB, IZRWSE, IIBMS, IIBME
-#endif
-!
INTEGER :: IIU,IJU,IKU
!
!-------------------------------------------------------------------------------
@@ -282,60 +275,6 @@ CALL MNH_MEM_GET( ZMZM_RHODJ, IIU, IJU, IKU )
!$acc & zrus_other, zrvs_other, zrws_other, zrus_adv, zrvs_adv, zrws_adv, &
!$acc & zmxm_rhodj, zmym_rhodj, zmzm_rhodj )
-#ifdef MNH_OPENACC
-#if 0
-CALL INIT_ON_HOST_AND_DEVICE(ZRUT,-1e99,'ADVECTION_UVW::ZRUT')
-CALL INIT_ON_HOST_AND_DEVICE(ZRVT,-2e99,'ADVECTION_UVW::ZRVT')
-CALL INIT_ON_HOST_AND_DEVICE(ZRWT,-3e99,'ADVECTION_UVW::ZRWT')
-CALL INIT_ON_HOST_AND_DEVICE(ZRUCT,-1e98,'ADVECTION_UVW::ZRUCT')
-CALL INIT_ON_HOST_AND_DEVICE(ZRVCT,-2e98,'ADVECTION_UVW::ZRVCT')
-CALL INIT_ON_HOST_AND_DEVICE(ZRWCT,-3e98,'ADVECTION_UVW::ZRWCT')
-CALL INIT_ON_HOST_AND_DEVICE(ZU,-1e99,'ADVECTION_UVW::ZU')
-CALL INIT_ON_HOST_AND_DEVICE(ZV,-1e99,'ADVECTION_UVW::ZV')
-CALL INIT_ON_HOST_AND_DEVICE(ZW,-1e99,'ADVECTION_UVW::ZW')
-CALL INIT_ON_HOST_AND_DEVICE(ZRUS_OTHER,-1e99,'ADVECTION_UVW::ZRUS_OTHER')
-CALL INIT_ON_HOST_AND_DEVICE(ZRVS_OTHER,-1e99,'ADVECTION_UVW::ZRVS_OTHER')
-CALL INIT_ON_HOST_AND_DEVICE(ZRWS_OTHER,-1e99,'ADVECTION_UVW::ZRWS_OTHER')
-CALL INIT_ON_HOST_AND_DEVICE(ZRUS_ADV,-1e99,'ADVECTION_UVW::ZRUS_ADV')
-CALL INIT_ON_HOST_AND_DEVICE(ZRVS_ADV,-1e99,'ADVECTION_UVW::ZRVS_ADV')
-CALL INIT_ON_HOST_AND_DEVICE(ZRWS_ADV,-1e99,'ADVECTION_UVW::ZRWS_ADV')
-CALL INIT_ON_HOST_AND_DEVICE(ZMXM_RHODJ,-1e97,'ADVECTION_UVW::ZMXM_RHODJ')
-CALL INIT_ON_HOST_AND_DEVICE(ZMYM_RHODJ,-2e97,'ADVECTION_UVW::ZMYM_RHODJ')
-CALL INIT_ON_HOST_AND_DEVICE(ZMZM_RHODJ,-3e97,'ADVECTION_UVW::ZMZM_RHODJ')
-#endif
-!
-SELECT CASE (HTEMP_SCHEME)
- CASE('RK11')
- ISPL = 1
- CASE('RK21')
- ISPL = 2
- CASE('NP32')
- ISPL = 3
- CASE('SP32')
- ISPL = 3
- CASE('RK33')
- ISPL = 3
- CASE('RKC4')
- ISPL = 4
- CASE('RK4B')
- ISPL = 4
- CASE('RK53')
- ISPL = 5
- CASE('RK62')
- ISPL = 6
- CASE('RK65')
- ISPL = 6
- CASE DEFAULT
- call Print_msg( NVERB_FATAL, 'GEN', 'ADVECTION_UVW', 'unknown htemp_scheme' )
-END SELECT
-!
-CALL MNH_GET_ZT3D(IZUT, IZVT, IZWT, IZ1, IZ2)
-CALL MNH_GET_ZT4D(ISPL, IZRUSB, IZRUSE)
-CALL MNH_GET_ZT4D(ISPL, IZRVSB, IZRVSE)
-CALL MNH_GET_ZT4D(ISPL, IZRWSB, IZRWSE)
-CALL MNH_GET_ZT4D(3, IIBMS, IIBME)
-#endif
-!
IKE = SIZE(PWT,3) - JPVEXT
!
#ifndef MNH_OPENACC
@@ -385,7 +324,7 @@ NULLIFY(TZFIELD_ll)
CALL CONTRAV (HLBCX,HLBCY,ZRUT,ZRVT,ZRWT,PDXX,PDYY,PDZZ,PDZX,PDZY,ZRUCT,ZRVCT,ZRWCT,4)
#else
CALL CONTRAV_DEVICE (HLBCX,HLBCY,ZRUT,ZRVT,ZRWT,PDXX,PDYY,PDZZ,PDZX,PDZY,ZRUCT,ZRVCT,ZRWCT,4,&
- ZT3D(:,:,:,IZ1),ZT3D(:,:,:,IZ2),ODATA_ON_DEVICE=.TRUE.)
+ ODATA_ON_DEVICE=.TRUE.)
!Not necessary: already done in contrav_device !$acc update self(ZRUCT,ZRVCT,ZRWCT)
#endif
!
@@ -490,13 +429,7 @@ DO JSPL=1,ISPLIT
ZMXM_RHODJ, ZMYM_RHODJ, ZMZM_RHODJ, &
ZRUCT, ZRVCT, ZRWCT, &
ZRUS_ADV, ZRVS_ADV, ZRWS_ADV, &
- ZRUS_OTHER, ZRVS_OTHER, ZRWS_OTHER &
-#ifndef MNH_OPENACC
- )
-#else
- ,ZT3D(:,:,:,IZUT), ZT3D(:,:,:,IZVT), ZT3D(:,:,:,IZWT), &
- ZT3D(:,:,:,IZRUSB:IZRUSE), ZT3D(:,:,:,IZRVSB:IZRVSE), ZT3D(:,:,:,IZRWSB:IZRWSE), ZT3D(:,:,:,IIBMS:IIBME) )
-#endif
+ ZRUS_OTHER, ZRVS_OTHER, ZRWS_OTHER )
!
! Tendencies on wind
!$acc update device(ZRUS_ADV,ZRVS_ADV,ZRWS_ADV)
@@ -551,13 +484,6 @@ if ( lbudget_w ) call Budget_store_end( tbudgets(NBUDGET_W), 'ADV', prws(:, :, :
!-------------------------------------------------------------------------------
!
-#ifdef MNH_OPENACC
-CALL MNH_REL_ZT4D(3, IIBMS )
-CALL MNH_REL_ZT4D(ISPL, IZRWSB)
-CALL MNH_REL_ZT4D(ISPL, IZRVSB)
-CALL MNH_REL_ZT4D(ISPL, IZRUSB)
-CALL MNH_REL_ZT3D(IZUT, IZVT, IZWT, IZ1, IZ2)
-#endif
IF (MPPDB_INITIALIZED) THEN
!Check all INOUT arrays
diff --git a/src/ZSOLVER/advection_uvw_cen.f90 b/src/ZSOLVER/advection_uvw_cen.f90
index fd980ac36..7a8df745e 100644
--- a/src/ZSOLVER/advection_uvw_cen.f90
+++ b/src/ZSOLVER/advection_uvw_cen.f90
@@ -1,4 +1,4 @@
-!MNH_LIC Copyright 2013-2020 CNRS, Meteo-France and Universite Paul Sabatier
+!MNH_LIC Copyright 2013-2022 CNRS, Meteo-France and Universite Paul Sabatier
!MNH_LIC This is part of the Meso-NH software governed by the CeCILL-C licence
!MNH_LIC version 1. See LICENSE, CeCILL-C_V1-en.txt and CeCILL-C_V1-fr.txt
!MNH_LIC for details. version 1.
@@ -105,7 +105,6 @@ use mode_budget, only: Budget_store_init, Budget_store_end
USE MODE_ll
#ifdef MNH_OPENACC
USE MODE_DEVICE
-USE MODE_MNH_ZWORK, ONLY : ZT3D, MNH_GET_ZT3D , MNH_REL_ZT3D
use mode_msg
#endif
use mode_mppdb
@@ -179,9 +178,6 @@ REAL, DIMENSION(:,:,:), allocatable :: ZMZM_RHODJ
!
INTEGER :: IINFO_ll ! return code of parallel routine
TYPE(LIST_ll), POINTER :: TZFIELDS_ll ! list of fields to exchange
-#ifdef MNH_OPENACC
-INTEGER :: IZ1, IZ2
-#endif
!
!-------------------------------------------------------------------------------
!$acc data present( PUM, PVM, PWM, PDUM, PDVM, PDWM, PUT, PVT, PWT, PRHODJ, PDXX, PDYY, PDZZ, PDZX, PDZY, PRUS, PRVS, PRWS )
@@ -248,8 +244,6 @@ CALL INIT_ON_HOST_AND_DEVICE(ZRWCT,-3e98,'ADVECTION_UVW_CEN::ZRWCT')
CALL INIT_ON_HOST_AND_DEVICE(ZMXM_RHODJ,-1e97,'ADVECTION_UVW_CEN::ZMXM_RHODJ')
CALL INIT_ON_HOST_AND_DEVICE(ZMYM_RHODJ,-2e97,'ADVECTION_UVW_CEN::ZMYM_RHODJ')
CALL INIT_ON_HOST_AND_DEVICE(ZMZM_RHODJ,-3e97,'ADVECTION_UVW_CEN::ZMZM_RHODJ')
-!
-CALL MNH_GET_ZT3D(IZ1, IZ2)
#endif
#ifndef MNH_OPENACC
@@ -280,10 +274,10 @@ END IF
#else
IF (HUVW_ADV_SCHEME=='CEN2ND' ) THEN
CALL CONTRAV_DEVICE (HLBCX,HLBCY,ZRUT,ZRVT,ZRWT,PDXX,PDYY,PDZZ,PDZX,PDZY,ZRUCT,ZRVCT,ZRWCT,2, &
- ZT3D(:,:,:,IZ1),ZT3D(:,:,:,IZ2),ODATA_ON_DEVICE=.TRUE.)
+ ODATA_ON_DEVICE=.TRUE.)
ELSEIF (HUVW_ADV_SCHEME=='CEN4TH') THEN
CALL CONTRAV_DEVICE (HLBCX,HLBCY,ZRUT,ZRVT,ZRWT,PDXX,PDYY,PDZZ,PDZX,PDZY,ZRUCT,ZRVCT,ZRWCT,4, &
- ZT3D(:,:,:,IZ1),ZT3D(:,:,:,IZ2),ODATA_ON_DEVICE=.TRUE.)
+ ODATA_ON_DEVICE=.TRUE.)
END IF
!Not necessary: already done in contrav_device !$acc update self(ZRUCT,ZRVCT,ZRWCT)
#endif
@@ -351,10 +345,6 @@ if ( lbudget_u ) call Budget_store_end( tbudgets(NBUDGET_U), 'ADV', prus(:, :, :
if ( lbudget_v ) call Budget_store_end( tbudgets(NBUDGET_V), 'ADV', prvs(:, :, :) )
if ( lbudget_w ) call Budget_store_end( tbudgets(NBUDGET_W), 'ADV', prws(:, :, :) )
-#ifdef MNH_OPENACC
-CALL MNH_REL_ZT3D(IZ1, IZ2)
-#endif
-
IF (MPPDB_INITIALIZED) THEN
!Check all INOUT arrays
CALL MPPDB_CHECK(PRUS,"ADVECTION_UVW_CEN end:PRUS")
diff --git a/src/ZSOLVER/advecuvw_rk.f90 b/src/ZSOLVER/advecuvw_rk.f90
index 6a4b8117c..8f62e91d0 100644
--- a/src/ZSOLVER/advecuvw_rk.f90
+++ b/src/ZSOLVER/advecuvw_rk.f90
@@ -1,4 +1,4 @@
-!MNH_LIC Copyright 1994-2021 CNRS, Meteo-France and Universite Paul Sabatier
+!MNH_LIC Copyright 1994-2022 CNRS, Meteo-France and Universite Paul Sabatier
!MNH_LIC This is part of the Meso-NH software governed by the CeCILL-C licence
!MNH_LIC version 1. See LICENSE, CeCILL-C_V1-en.txt and CeCILL-C_V1-fr.txt
!MNH_LIC for details. version 1.
@@ -16,12 +16,7 @@ INTERFACE
PMXM_RHODJ, PMYM_RHODJ, PMZM_RHODJ, &
PRUCT, PRVCT, PRWCT, &
PRUS_ADV, PRVS_ADV, PRWS_ADV, &
- PRUS_OTHER, PRVS_OTHER, PRWS_OTHER &
-#ifndef MNH_OPENACC
- )
-#else
- , ZUT, ZVT, ZWT, ZRUS, ZRVS, ZRWS, ZIBM )
-#endif
+ PRUS_OTHER, PRVS_OTHER, PRWS_OTHER )
!
CHARACTER(LEN=6), INTENT(IN) :: HUVW_ADV_SCHEME! to the selected
CHARACTER(LEN=4), INTENT(IN) :: HTEMP_SCHEME ! Temporal scheme
@@ -47,13 +42,6 @@ REAL, DIMENSION(:,:,:), INTENT(OUT) :: PRUS_ADV , PRVS_ADV, PRWS_ADV
! Tendency due to advection
REAL, DIMENSION(:,:,:), INTENT(IN) :: PRUS_OTHER , PRVS_OTHER, PRWS_OTHER
! ! tendencies from other processes
-#ifdef MNH_OPENACC
-! Work arrays
-REAL, DIMENSION(:,:,:) :: ZUT, ZVT, ZWT
-REAL, DIMENSION(:,:,:,:) :: ZRUS, ZRVS, ZRWS, ZIBM
-#endif
-!
-!
END SUBROUTINE ADVECUVW_RK
!
END INTERFACE
@@ -68,12 +56,7 @@ END MODULE MODI_ADVECUVW_RK
PMXM_RHODJ, PMYM_RHODJ, PMZM_RHODJ, &
PRUCT, PRVCT, PRWCT, &
PRUS_ADV, PRVS_ADV, PRWS_ADV, &
- PRUS_OTHER, PRVS_OTHER, PRWS_OTHER &
-#ifndef MNH_OPENACC
- )
-#else
- , ZUT, ZVT, ZWT, ZRUS, ZRVS, ZRWS, ZIBM )
-#endif
+ PRUS_OTHER, PRVS_OTHER, PRWS_OTHER )
! ##########################################################################
!
!!**** *ADVECUVW_RK * - routine to call the specialized advection routines for wind
@@ -150,7 +133,7 @@ USE MODI_SHUMAN
!
#ifdef MNH_OPENACC
USE MODE_DEVICE
-USE MODE_MNH_ZWORK, ONLY : ZT3D, MNH_GET_ZT3D , MNH_REL_ZT3D
+USE MODE_MNH_ZWORK, ONLY: MNH_MEM_GET, MNH_MEM_POSITION_PIN, MNH_MEM_RELEASE
#endif
!
!-------------------------------------------------------------------------------
@@ -183,17 +166,9 @@ REAL, DIMENSION(:,:,:), INTENT(OUT) :: PRUS_ADV , PRVS_ADV, PRWS_ADV
! Tendency due to advection
REAL, DIMENSION(:,:,:), INTENT(IN) :: PRUS_OTHER , PRVS_OTHER, PRWS_OTHER
! ! tendencies from other processes
-#ifdef MNH_OPENACC
-REAL, DIMENSION(:,:,:) :: ZUT, ZVT, ZWT
-REAL, DIMENSION(:,:,:,:) :: ZRUS, ZRVS, ZRWS, ZIBM
-#endif
-!
-!
!
!* 0.2 declarations of local variables
!
-!
-!
character(len=3) :: ynum
INTEGER :: IKE ! indice K End in z direction
!
@@ -201,13 +176,23 @@ INTEGER :: IKE ! indice K End in z direction
REAL, DIMENSION(:,:,:), ALLOCATABLE :: ZUT, ZVT, ZWT
! Intermediate Guesses inside the RK loop
!
-REAL, DIMENSION(:,:,:,:), ALLOCATABLE :: ZRUS,ZRVS,ZRWS,ZIBM
-#endif
+REAL, DIMENSION(:,:,:,:), ALLOCATABLE :: ZRUS, ZRVS, ZRWS, ZIBM
! Momentum tendencies due to advection
REAL, DIMENSION(:,:), ALLOCATABLE :: ZBUT ! Butcher array coefficients
! at the RK sub time step
REAL, DIMENSION(:), ALLOCATABLE :: ZBUTS! Butcher array coefficients
! at the end of the RK loop
+#else
+REAL, DIMENSION(:,:,:), POINTER, CONTIGUOUS :: ZUT, ZVT, ZWT
+! Intermediate Guesses inside the RK loop
+!
+REAL, DIMENSION(:,:,:,:), POINTER, CONTIGUOUS :: ZRUS, ZRVS, ZRWS, ZIBM
+! Momentum tendencies due to advection
+REAL, DIMENSION(:,:), POINTER, CONTIGUOUS :: ZBUT ! Butcher array coefficients
+ ! at the RK sub time step
+REAL, DIMENSION(:), POINTER, CONTIGUOUS :: ZBUTS! Butcher array coefficients
+ ! at the end of the RK loop
+#endif
!JUAN
TYPE(LIST_ll), POINTER :: TZFIELDMT_ll ! list of fields to exchange
TYPE(HALO2LIST_ll), POINTER :: TZHALO2MT_ll ! momentum variables
@@ -219,9 +204,6 @@ INTEGER :: INBVAR
INTEGER :: IIU, IJU, IKU ! array sizes
!JUAN
-#ifdef MNH_OPENACC
-INTEGER :: IZMEAN, IZWORK
-#endif
! Momentum tendencies due to advection
INTEGER :: ISPL ! Number of RK splitting loops
INTEGER :: JI, JS ! Loop index
@@ -236,10 +218,6 @@ LOGICAL :: GIBM !Intermediate variable used to work around a Cray compiler bug (
REAL :: ZIBM_EPSI !Intermediate variable used to work around a Cray compiler bug (CCE 13.0.0)
REAL :: ZTIME1,ZTIME2
!-------------------------------------------------------------------------------
-!$acc data present( PU, PV, PW, PUT, PVT, PWT, PMXM_RHODJ, PMYM_RHODJ, PMZM_RHODJ, &
-!$acc & PRUCT, PRVCT, PRWCT, PRUS_ADV, PRVS_ADV, PRWS_ADV, &
-!$acc & PRUS_OTHER, PRVS_OTHER, PRWS_OTHER, ZUT, ZVT, ZWT, &
-!$acc & ZRUS, ZRVS, ZRWS, ZIBM )
IF (MPPDB_INITIALIZED) THEN
!Check all IN arrays
@@ -270,20 +248,6 @@ if ( LIBM ) call Print_msg( NVERB_FATAL, 'GEN', 'ADVECUVW_RK', 'OpenACC: LIBM=T
!* 0. INITIALIZATION
! ---------------------
!
-#ifndef MNH_OPENACC
-allocate(ZUT(SIZE(PUT,1),SIZE(PUT,2),SIZE(PUT,3)))
-allocate(ZVT(SIZE(PUT,1),SIZE(PUT,2),SIZE(PUT,3)))
-allocate(ZWT(SIZE(PUT,1),SIZE(PUT,2),SIZE(PUT,3)))
-#endif
-
-#ifdef MNH_OPENACC
-CALL INIT_ON_HOST_AND_DEVICE(ZUT,4e99,'ADVECUVW_RK::ZUT')
-CALL INIT_ON_HOST_AND_DEVICE(ZVT,5e99,'ADVECUVW_RK::ZVT')
-CALL INIT_ON_HOST_AND_DEVICE(ZWT,6e99,'ADVECUVW_RK::ZWT')
-!
-CALL MNH_GET_ZT3D(IZMEAN,IZWORK)
-#endif
-!
IKE = SIZE(PWT,3) - JPVEXT
IIU=SIZE(PUT,1)
IJU=SIZE(PUT,2)
@@ -314,11 +278,47 @@ SELECT CASE (HTEMP_SCHEME)
call Print_msg(NVERB_FATAL,'GEN','ADVECUVW_RK','unknown HTEMP_SCHEME')
END SELECT
!
-!
+#ifndef MNH_OPENACC
+allocate( ZUT(IIU, IJU, IKU) )
+allocate( ZVT(IIU, IJU, IKU) )
+allocate( ZWT(IIU, IJU, IKU) )
+
+ALLOCATE( ZRUS(IIU, IJU, IKU, ISPL) )
+ALLOCATE( ZRVS(IIU, IJU, IKU, ISPL) )
+ALLOCATE( ZRWS(IIU, IJU, IKU, ISPL) )
+
+IF ( GIBM ) ALLOCATE( ZIBM(IIU, IJU, IKU, 3) )
+
ALLOCATE(ZBUT(ISPL-1,ISPL-1))
ALLOCATE(ZBUTS(ISPL))
+#else
+!Pin positions in the pools of MNH memory
+CALL MNH_MEM_POSITION_PIN()
+
+CALL MNH_MEM_GET( ZUT, IIU, IJU, IKU )
+CALL MNH_MEM_GET( ZVT, IIU, IJU, IKU )
+CALL MNH_MEM_GET( ZWT, IIU, IJU, IKU )
+
+CALL MNH_MEM_GET( ZRUS, IIU, IJU, IKU, ISPL )
+CALL MNH_MEM_GET( ZRVS, IIU, IJU, IKU, ISPL )
+CALL MNH_MEM_GET( ZRWS, IIU, IJU, IKU, ISPL )
+
+IF ( GIBM ) CALL MNH_MEM_GET( ZIBM, IIU, IJU, IKU, 3 )
-!$acc data create(ZBUT,ZBUTS)
+CALL MNH_MEM_GET( ZBUT, ISPL-1, ISPL-1 )
+CALL MNH_MEM_GET( ZBUTS, ISPL )
+#endif
+
+#ifdef MNH_OPENACC
+CALL INIT_ON_HOST_AND_DEVICE(ZUT,4e99,'ADVECUVW_RK::ZUT')
+CALL INIT_ON_HOST_AND_DEVICE(ZVT,5e99,'ADVECUVW_RK::ZVT')
+CALL INIT_ON_HOST_AND_DEVICE(ZWT,6e99,'ADVECUVW_RK::ZWT')
+#endif
+
+!$acc data present( PU, PV, PW, PUT, PVT, PWT, PMXM_RHODJ, PMYM_RHODJ, PMZM_RHODJ, &
+!$acc & PRUCT, PRVCT, PRWCT, PRUS_ADV, PRVS_ADV, PRWS_ADV, &
+!$acc & PRUS_OTHER, PRVS_OTHER, PRWS_OTHER, ZUT, ZVT, ZWT, &
+!$acc & ZRUS, ZRVS, ZRWS, ZIBM, ZBUT, ZBUTS )
SELECT CASE (HTEMP_SCHEME)
CASE('RK11')
@@ -405,16 +405,7 @@ CASE('RK65')
END SELECT
!$acc update device(ZBUTS,ZBUT)
!
-#ifndef MNH_OPENACC
-ALLOCATE(ZRUS(SIZE(PUT,1),SIZE(PUT,2),SIZE(PWT,3),ISPL))
-ALLOCATE(ZRVS(SIZE(PUT,1),SIZE(PUT,2),SIZE(PWT,3),ISPL))
-ALLOCATE(ZRWS(SIZE(PUT,1),SIZE(PUT,2),SIZE(PWT,3),ISPL))
-#endif
-
IF ( GIBM ) THEN
-#ifndef MNH_OPENACC
- ALLOCATE( ZIBM(SIZE(PUT,1), SIZE(PUT,2), SIZE(PWT,3), 3) )
-#endif
!$acc kernels
ZIBM(:,:,:,:) = 1.
!$acc end kernels
@@ -522,8 +513,7 @@ RKLOOP: DO JS = 1, ISPL
#ifndef MNH_OPENACC
TZHALO2MT_ll )
#else
- TZHALO2_UT,TZHALO2_VT,TZHALO2_WT &
- , ZT3D(:,:,:,IZMEAN), ZT3D(:,:,:,IZWORK) )
+ TZHALO2_UT,TZHALO2_VT,TZHALO2_WT )
#endif
ELSE IF ((HUVW_ADV_SCHEME=='CEN4TH') .AND. (HTEMP_SCHEME=='RKC4')) THEN
CALL ADVECUVW_4TH (HLBCX, HLBCY, PRUCT, PRVCT, PRWCT, &
@@ -540,8 +530,7 @@ RKLOOP: DO JS = 1, ISPL
#ifndef MNH_OPENACC
TZHALO2MT_ll )
#else
- TZHALO2_UT,TZHALO2_VT,TZHALO2_WT &
- , ZT3D(:,:,:,IZMEAN), ZT3D(:,:,:,IZWORK) )
+ TZHALO2_UT,TZHALO2_VT,TZHALO2_WT )
#endif
ENDIF
IF (HUVW_ADV_SCHEME=='CEN4TH') THEN
@@ -599,7 +588,7 @@ RKLOOP: DO JS = 1, ISPL
IF ( JS < ISPL ) THEN
!$acc kernels present(ZUT,ZVT,ZWT) present(ZBUT) present(PU,PV,PW) &
!$acc & present(ZRUS,ZRVS,ZRWS) present(PRUS_OTHER,PRVS_OTHER,PRWS_OTHER) &
-!$acc & present(PMXM_RHODJ,PMYM_RHODJ,PMZM_RHODJ)
+!$acc & present(PMXM_RHODJ,PMYM_RHODJ,PMZM_RHODJ, ZIBM )
!
ZUT(:,:,:) = PU(:,:,:)
ZVT(:,:,:) = PV(:,:,:)
@@ -634,15 +623,16 @@ RKLOOP: DO JS = 1, ISPL
END DO RKLOOP
!
!
-#ifdef MNH_OPENACC
-CALL MNH_REL_ZT3D(IZMEAN,IZWORK)
-#endif
-!
CALL CLEANLIST_ll(TZFIELDMT_ll)
CALL DEL_HALO2_ll(TZHALO2MT_ll)
!$acc update self(PRUS_ADV,PRVS_ADV,PRWS_ADV)
!-------------------------------------------------------------------------------
-!
+
+!$acc end data
+
+!Release all memory allocated with MNH_MEM_GET calls since last call to MNH_MEM_POSITION_PIN
+CALL MNH_MEM_RELEASE()
+
IF (MPPDB_INITIALIZED) THEN
!Check all OUT arrays
CALL MPPDB_CHECK(PRUS_ADV,"ADVECUVW_RK end:PRUS_ADV")
@@ -650,8 +640,4 @@ IF (MPPDB_INITIALIZED) THEN
CALL MPPDB_CHECK(PRWS_ADV,"ADVECUVW_RK end:PRWS_ADV")
END IF
-!$acc end data
-
-!$acc end data
-
END SUBROUTINE ADVECUVW_RK
diff --git a/src/ZSOLVER/advecuvw_weno_k.f90 b/src/ZSOLVER/advecuvw_weno_k.f90
index 8486449ef..098ab2c8c 100644
--- a/src/ZSOLVER/advecuvw_weno_k.f90
+++ b/src/ZSOLVER/advecuvw_weno_k.f90
@@ -14,8 +14,7 @@ INTERFACE
#ifndef MNH_OPENACC
TPHALO2LIST )
#else
- TPHALO2_UT,TPHALO2_VT,TPHALO2_WT &
- , ZMEAN, ZWORK)
+ TPHALO2_UT,TPHALO2_VT,TPHALO2_WT )
#endif
!
USE MODD_ARGSLIST_ll, ONLY : HALO2LIST_ll
@@ -39,10 +38,6 @@ TYPE(HALO2LIST_ll), POINTER :: TPHALO2LIST ! list for diffusion
TYPE(HALO2LIST_ll), POINTER :: TPHALO2_UT,TPHALO2_VT,TPHALO2_WT
#endif
!
-#ifdef MNH_OPENACC
-! Work arrays
-REAL, DIMENSION(:,:,:) :: ZMEAN, ZWORK
-#endif
!
END SUBROUTINE ADVECUVW_WENO_K
!
@@ -56,8 +51,7 @@ END MODULE MODI_ADVECUVW_WENO_K
#ifndef MNH_OPENACC
TPHALO2LIST )
#else
- TPHALO2_UT,TPHALO2_VT,TPHALO2_WT &
- , ZMEAN, ZWORK)
+ TPHALO2_UT,TPHALO2_VT,TPHALO2_WT )
#endif
! ##########################################################################
!
@@ -84,7 +78,7 @@ USE MODD_PARAMETERS
USE MODE_ll
#ifdef MNH_OPENACC
USE MODE_DEVICE
-USE MODE_MNH_ZWORK, ONLY : ZT3D, MNH_GET_ZT3D , MNH_REL_ZT3D
+USE MODE_MNH_ZWORK, ONLY: MNH_MEM_GET, MNH_MEM_POSITION_PIN, MNH_MEM_RELEASE
#endif
USE MODE_MPPDB
@@ -123,25 +117,23 @@ TYPE(HALO2LIST_ll), POINTER :: TPHALO2_UT,TPHALO2_VT,TPHALO2_WT
!
!* 0.2 Declarations of local variables :
!
+INTEGER :: IIU, IJU, IKU
TYPE(HALO2LIST_ll), POINTER :: TZHALO2_UT,TZHALO2_VT,TZHALO2_WT
INTEGER :: IINFO_ll ! return code of parallel routine
!
#ifndef MNH_OPENACC
REAL, DIMENSION(:,:,:), allocatable :: ZMEAN, ZWORK
#else
-REAL, DIMENSION(:,:,:) :: ZMEAN, ZWORK
-#endif
-!
-#ifdef MNH_OPENACC
-INTEGER :: IZFPOS1, IZFPOS2, IZFPOS3
-INTEGER :: IZFNEG1, IZFNEG2, IZFNEG3
-INTEGER :: IZBPOS1, IZBPOS2, IZBPOS3
-INTEGER :: IZBNEG1, IZBNEG2, IZBNEG3
-INTEGER :: IZOMP1, IZOMP2, IZOMP3
-INTEGER :: IZOMN1, IZOMN2, IZOMN3
+REAL, DIMENSION(:,:,:), POINTER ,CONTIGUOUS :: ZMEAN, ZWORK
+!
+REAL, DIMENSION(:,:,:), POINTER ,CONTIGUOUS :: ZFPOS1, ZFPOS2, ZFPOS3
+REAL, DIMENSION(:,:,:), POINTER ,CONTIGUOUS :: ZFNEG1, ZFNEG2, ZFNEG3
+REAL, DIMENSION(:,:,:), POINTER ,CONTIGUOUS :: ZBPOS1, ZBPOS2, ZBPOS3
+REAL, DIMENSION(:,:,:), POINTER ,CONTIGUOUS :: ZBNEG1, ZBNEG2, ZBNEG3
+REAL, DIMENSION(:,:,:), POINTER ,CONTIGUOUS :: ZOMP1, ZOMP2, ZOMP3
+REAL, DIMENSION(:,:,:), POINTER ,CONTIGUOUS :: ZOMN1, ZOMN2, ZOMN3
#endif
!
-!$acc data present( PRUCT, PRVCT, PRWCT, PUT, PVT, PWT, PRUS, PRVS, PRWS, ZMEAN, ZWORK )
!
IF (MPPDB_INITIALIZED) THEN
!Check all IN arrays
@@ -157,9 +149,19 @@ IF (MPPDB_INITIALIZED) THEN
CALL MPPDB_CHECK(PRWS,"ADVECUVW_WENO_K beg:PRWS")
END IF
+IIU = SIZE( PUT, 1 )
+IJU = SIZE( PUT, 2 )
+IKU = SIZE( PUT, 3 )
+
#ifndef MNH_OPENACC
-allocate(ZMEAN(SIZE(PUT,1), SIZE(PUT,2), SIZE(PUT,3)))
-allocate(ZWORK(SIZE(PUT,1), SIZE(PUT,2), SIZE(PUT,3)))
+allocate( ZMEAN(IIU, IJU, IKU) )
+allocate( ZWORK(IIU, IJU, IKU) )
+#else
+!Pin positions in the pools of MNH memory
+CALL MNH_MEM_POSITION_PIN()
+
+CALL MNH_MEM_GET( ZMEAN, IIU, IJU, IKU )
+CALL MNH_MEM_GET( ZWORK, IIU, IJU, IKU )
#endif
#ifdef MNH_OPENACC
@@ -181,7 +183,9 @@ TZHALO2_WT => TPHALO2_WT
#endif
!
! -------------------------------------------------------
-!
+
+!$acc data present( PRUCT, PRVCT, PRWCT, PUT, PVT, PWT, PRUS, PRVS, PRWS, ZMEAN, ZWORK )
+
SELECT CASE(KWENO_ORDER)
!
CASE(1) ! WENO 1
@@ -338,15 +342,29 @@ CASE(3) ! WENO 3
!
PRWS = PRWS - DZM(WENO_K_2_WZ(PWT,MZF(PRWCT)))
#else
- CALL MNH_GET_ZT3D(IZFPOS1,IZFPOS2,IZFNEG1,IZFNEG2,IZBPOS1,IZBPOS2,IZBNEG1,IZBNEG2,IZOMP1,IZOMP2,IZOMN1,IZOMN2)
+ !Pin positions in the pools of MNH memory
+ CALL MNH_MEM_POSITION_PIN()
+
+ CALL MNH_MEM_GET( ZFPOS1, IIU, IJU, IKU )
+ CALL MNH_MEM_GET( ZFPOS2, IIU, IJU, IKU )
+ CALL MNH_MEM_GET( ZFNEG1, IIU, IJU, IKU )
+ CALL MNH_MEM_GET( ZFNEG2, IIU, IJU, IKU )
+ CALL MNH_MEM_GET( ZBPOS1, IIU, IJU, IKU )
+ CALL MNH_MEM_GET( ZBPOS2, IIU, IJU, IKU )
+ CALL MNH_MEM_GET( ZBNEG1, IIU, IJU, IKU )
+ CALL MNH_MEM_GET( ZBNEG2, IIU, IJU, IKU )
+ CALL MNH_MEM_GET( ZOMP1, IIU, IJU, IKU )
+ CALL MNH_MEM_GET( ZOMP2, IIU, IJU, IKU )
+ CALL MNH_MEM_GET( ZOMN1, IIU, IJU, IKU )
+ CALL MNH_MEM_GET( ZOMN2, IIU, IJU, IKU )
!
! U component
!
CALL MXF_DEVICE(PRUCT,ZWORK)
- CALL ADVEC_WENO_K_2_UX(HLBCX, PUT, ZWORK, ZMEAN, TZHALO2_UT%HALO2%WEST, TZHALO2_UT%HALO2%EAST, &
- ZT3D(:,:,:,IZFPOS1), ZT3D(:,:,:,IZFPOS2), ZT3D(:,:,:,IZFNEG1), ZT3D(:,:,:,IZFNEG2), &
- ZT3D(:,:,:,IZBPOS1), ZT3D(:,:,:,IZBPOS2), ZT3D(:,:,:,IZBNEG1), ZT3D(:,:,:,IZBNEG2), &
- ZT3D(:,:,:,IZOMP1), ZT3D(:,:,:,IZOMP2), ZT3D(:,:,:,IZOMN1), ZT3D(:,:,:,IZOMN2) )
+ CALL ADVEC_WENO_K_2_UX(HLBCX, PUT, ZWORK, ZMEAN, TZHALO2_UT%HALO2%WEST, TZHALO2_UT%HALO2%EAST, &
+ ZFPOS1(:,:,:), ZFPOS2(:,:,:), ZFNEG1(:,:,:), ZFNEG2(:,:,:), &
+ ZBPOS1(:,:,:), ZBPOS2(:,:,:), ZBNEG1(:,:,:), ZBNEG2(:,:,:), &
+ ZOMP1(:,:,:), ZOMP2(:,:,:), ZOMN1(:,:,:), ZOMN2(:,:,:) )
CALL DXM_DEVICE(ZMEAN,ZWORK)
!$acc kernels
PRUS(:,:,:) = PRUS(:,:,:) - ZWORK(:,:,:)
@@ -354,10 +372,10 @@ CASE(3) ! WENO 3
!
IF (.NOT.L2D) THEN
CALL MXM_DEVICE(PRVCT,ZWORK)
- CALL ADVEC_WENO_K_2_MY(HLBCY, PUT, ZWORK, ZMEAN, TZHALO2_UT%HALO2%NORTH, TZHALO2_UT%HALO2%SOUTH, &
- ZT3D(:,:,:,IZFPOS1), ZT3D(:,:,:,IZFPOS2), ZT3D(:,:,:,IZFNEG1), ZT3D(:,:,:,IZFNEG2), &
- ZT3D(:,:,:,IZBPOS1), ZT3D(:,:,:,IZBPOS2), ZT3D(:,:,:,IZBNEG1), ZT3D(:,:,:,IZBNEG2), &
- ZT3D(:,:,:,IZOMP1), ZT3D(:,:,:,IZOMP2), ZT3D(:,:,:,IZOMN1), ZT3D(:,:,:,IZOMN2) )
+ CALL ADVEC_WENO_K_2_MY(HLBCY, PUT, ZWORK, ZMEAN, TZHALO2_UT%HALO2%NORTH, TZHALO2_UT%HALO2%SOUTH, &
+ ZFPOS1(:,:,:), ZFPOS2(:,:,:), ZFNEG1(:,:,:), ZFNEG2(:,:,:), &
+ ZBPOS1(:,:,:), ZBPOS2(:,:,:), ZBNEG1(:,:,:), ZBNEG2(:,:,:), &
+ ZOMP1(:,:,:), ZOMP2(:,:,:), ZOMN1(:,:,:), ZOMN2(:,:,:) )
CALL DYF_DEVICE(ZMEAN,ZWORK)
!$acc kernels
PRUS(:,:,:) = PRUS(:,:,:) - ZWORK(:,:,:)
@@ -366,10 +384,10 @@ CASE(3) ! WENO 3
!
! PRUS = PRUS - DZF(WENO_K_2_MZ(PUT, MXM(PRWCT)))
CALL MXM_DEVICE(PRWCT,ZWORK)
- CALL WENO_K_2_MZ(PUT, ZWORK, ZMEAN, &
- ZT3D(:,:,:,IZFPOS1), ZT3D(:,:,:,IZFPOS2), ZT3D(:,:,:,IZFNEG1), ZT3D(:,:,:,IZFNEG2), &
- ZT3D(:,:,:,IZBPOS1), ZT3D(:,:,:,IZBPOS2), ZT3D(:,:,:,IZBNEG1), ZT3D(:,:,:,IZBNEG2), &
- ZT3D(:,:,:,IZOMP1), ZT3D(:,:,:,IZOMP2), ZT3D(:,:,:,IZOMN1), ZT3D(:,:,:,IZOMN2) )
+ CALL WENO_K_2_MZ(PUT, ZWORK, ZMEAN, &
+ ZFPOS1(:,:,:), ZFPOS2(:,:,:), ZFNEG1(:,:,:), ZFNEG2(:,:,:), &
+ ZBPOS1(:,:,:), ZBPOS2(:,:,:), ZBNEG1(:,:,:), ZBNEG2(:,:,:), &
+ ZOMP1(:,:,:), ZOMP2(:,:,:), ZOMN1(:,:,:), ZOMN2(:,:,:) )
CALL DZF_DEVICE( ZMEAN, ZWORK )
!$acc kernels
PRUS(:,:,:) = PRUS(:,:,:) - ZWORK(:,:,:)
@@ -379,20 +397,20 @@ CASE(3) ! WENO 3
!
IF (.NOT.L2D) THEN
CALL MYM_DEVICE(PRUCT,ZWORK)
- CALL ADVEC_WENO_K_2_MX(HLBCX, PVT, ZWORK, ZMEAN, TZHALO2_VT%HALO2%WEST, TZHALO2_VT%HALO2%EAST, &
- ZT3D(:,:,:,IZFPOS1), ZT3D(:,:,:,IZFPOS2), ZT3D(:,:,:,IZFNEG1), ZT3D(:,:,:,IZFNEG2), &
- ZT3D(:,:,:,IZBPOS1), ZT3D(:,:,:,IZBPOS2), ZT3D(:,:,:,IZBNEG1), ZT3D(:,:,:,IZBNEG2), &
- ZT3D(:,:,:,IZOMP1), ZT3D(:,:,:,IZOMP2), ZT3D(:,:,:,IZOMN1), ZT3D(:,:,:,IZOMN2) )
+ CALL ADVEC_WENO_K_2_MX(HLBCX, PVT, ZWORK, ZMEAN, TZHALO2_VT%HALO2%WEST, TZHALO2_VT%HALO2%EAST, &
+ ZFPOS1(:,:,:), ZFPOS2(:,:,:), ZFNEG1(:,:,:), ZFNEG2(:,:,:), &
+ ZBPOS1(:,:,:), ZBPOS2(:,:,:), ZBNEG1(:,:,:), ZBNEG2(:,:,:), &
+ ZOMP1(:,:,:), ZOMP2(:,:,:), ZOMN1(:,:,:), ZOMN2(:,:,:) )
CALL DXF_DEVICE(ZMEAN,ZWORK)
!$acc kernels
PRVS(:,:,:) = PRVS(:,:,:) - ZWORK(:,:,:)
!$acc end kernels
!
CALL MYF_DEVICE(PRVCT,ZWORK)
- CALL ADVEC_WENO_K_2_VY(HLBCY, PVT, ZWORK, ZMEAN, TZHALO2_VT%HALO2%NORTH, TZHALO2_VT%HALO2%SOUTH, &
- ZT3D(:,:,:,IZFPOS1), ZT3D(:,:,:,IZFPOS2), ZT3D(:,:,:,IZFNEG1), ZT3D(:,:,:,IZFNEG2), &
- ZT3D(:,:,:,IZBPOS1), ZT3D(:,:,:,IZBPOS2), ZT3D(:,:,:,IZBNEG1), ZT3D(:,:,:,IZBNEG2), &
- ZT3D(:,:,:,IZOMP1), ZT3D(:,:,:,IZOMP2), ZT3D(:,:,:,IZOMN1), ZT3D(:,:,:,IZOMN2) )
+ CALL ADVEC_WENO_K_2_VY(HLBCY, PVT, ZWORK, ZMEAN, TZHALO2_VT%HALO2%NORTH, TZHALO2_VT%HALO2%SOUTH, &
+ ZFPOS1(:,:,:), ZFPOS2(:,:,:), ZFNEG1(:,:,:), ZFNEG2(:,:,:), &
+ ZBPOS1(:,:,:), ZBPOS2(:,:,:), ZBNEG1(:,:,:), ZBNEG2(:,:,:), &
+ ZOMP1(:,:,:), ZOMP2(:,:,:), ZOMN1(:,:,:), ZOMN2(:,:,:) )
CALL DYM_DEVICE(ZMEAN,ZWORK)
!$acc kernels
PRVS(:,:,:) = PRVS(:,:,:) - ZWORK(:,:,:)
@@ -400,10 +418,10 @@ CASE(3) ! WENO 3
!
! PRVS = PRVS - DZF(WENO_K_2_MZ(PVT, MYM(PRWCT)))
CALL MYM_DEVICE(PRWCT,ZWORK)
- CALL WENO_K_2_MZ(PVT, ZWORK, ZMEAN, &
- ZT3D(:,:,:,IZFPOS1), ZT3D(:,:,:,IZFPOS2), ZT3D(:,:,:,IZFNEG1), ZT3D(:,:,:,IZFNEG2), &
- ZT3D(:,:,:,IZBPOS1), ZT3D(:,:,:,IZBPOS2), ZT3D(:,:,:,IZBNEG1), ZT3D(:,:,:,IZBNEG2), &
- ZT3D(:,:,:,IZOMP1), ZT3D(:,:,:,IZOMP2), ZT3D(:,:,:,IZOMN1), ZT3D(:,:,:,IZOMN2) )
+ CALL WENO_K_2_MZ(PVT, ZWORK, ZMEAN, &
+ ZFPOS1(:,:,:), ZFPOS2(:,:,:), ZFNEG1(:,:,:), ZFNEG2(:,:,:), &
+ ZBPOS1(:,:,:), ZBPOS2(:,:,:), ZBNEG1(:,:,:), ZBNEG2(:,:,:), &
+ ZOMP1(:,:,:), ZOMP2(:,:,:), ZOMN1(:,:,:), ZOMN2(:,:,:) )
CALL DZF_DEVICE( ZMEAN, ZWORK )
!$acc kernels
PRVS(:,:,:) = PRVS(:,:,:) - ZWORK(:,:,:)
@@ -414,10 +432,10 @@ CASE(3) ! WENO 3
!
! ZWORK = MZM(PRUCT)
CALL MZM_DEVICE(PRUCT,ZWORK)
- CALL ADVEC_WENO_K_2_MX(HLBCX, PWT, ZWORK, ZMEAN, TZHALO2_WT%HALO2%WEST, TZHALO2_WT%HALO2%EAST, &
- ZT3D(:,:,:,IZFPOS1), ZT3D(:,:,:,IZFPOS2), ZT3D(:,:,:,IZFNEG1), ZT3D(:,:,:,IZFNEG2), &
- ZT3D(:,:,:,IZBPOS1), ZT3D(:,:,:,IZBPOS2), ZT3D(:,:,:,IZBNEG1), ZT3D(:,:,:,IZBNEG2), &
- ZT3D(:,:,:,IZOMP1), ZT3D(:,:,:,IZOMP2), ZT3D(:,:,:,IZOMN1), ZT3D(:,:,:,IZOMN2) )
+ CALL ADVEC_WENO_K_2_MX(HLBCX, PWT, ZWORK, ZMEAN, TZHALO2_WT%HALO2%WEST, TZHALO2_WT%HALO2%EAST, &
+ ZFPOS1(:,:,:), ZFPOS2(:,:,:), ZFNEG1(:,:,:), ZFNEG2(:,:,:), &
+ ZBPOS1(:,:,:), ZBPOS2(:,:,:), ZBNEG1(:,:,:), ZBNEG2(:,:,:), &
+ ZOMP1(:,:,:), ZOMP2(:,:,:), ZOMN1(:,:,:), ZOMN2(:,:,:) )
CALL DXF_DEVICE(ZMEAN,ZWORK)
!$acc kernels
PRWS(:,:,:) = PRWS(:,:,:) - ZWORK(:,:,:)
@@ -426,10 +444,10 @@ CASE(3) ! WENO 3
IF (.NOT.L2D) THEN
! ZWORK = MZM(PRVCT)
CALL MZM_DEVICE(PRVCT,ZWORK)
- CALL ADVEC_WENO_K_2_MY(HLBCY, PWT, ZWORK, ZMEAN, TZHALO2_WT%HALO2%NORTH, TZHALO2_WT%HALO2%SOUTH, &
- ZT3D(:,:,:,IZFPOS1), ZT3D(:,:,:,IZFPOS2), ZT3D(:,:,:,IZFNEG1), ZT3D(:,:,:,IZFNEG2), &
- ZT3D(:,:,:,IZBPOS1), ZT3D(:,:,:,IZBPOS2), ZT3D(:,:,:,IZBNEG1), ZT3D(:,:,:,IZBNEG2), &
- ZT3D(:,:,:,IZOMP1), ZT3D(:,:,:,IZOMP2), ZT3D(:,:,:,IZOMN1), ZT3D(:,:,:,IZOMN2) )
+ CALL ADVEC_WENO_K_2_MY(HLBCY, PWT, ZWORK, ZMEAN, TZHALO2_WT%HALO2%NORTH, TZHALO2_WT%HALO2%SOUTH, &
+ ZFPOS1(:,:,:), ZFPOS2(:,:,:), ZFNEG1(:,:,:), ZFNEG2(:,:,:), &
+ ZBPOS1(:,:,:), ZBPOS2(:,:,:), ZBNEG1(:,:,:), ZBNEG2(:,:,:), &
+ ZOMP1(:,:,:), ZOMP2(:,:,:), ZOMN1(:,:,:), ZOMN2(:,:,:) )
CALL DYF_DEVICE(ZMEAN,ZWORK)
!$acc kernels
PRWS(:,:,:) = PRWS(:,:,:) - ZWORK(:,:,:)
@@ -438,16 +456,17 @@ CASE(3) ! WENO 3
!
! PRWS = PRWS - DZM(WENO_K_2_WZ(PWT,MZF(PRWCT)))
CALL MZF_DEVICE( PRWCT, ZWORK )
- CALL WENO_K_2_WZ(PWT, ZWORK, ZMEAN, &
- ZT3D(:,:,:,IZFPOS1), ZT3D(:,:,:,IZFPOS2), ZT3D(:,:,:,IZFNEG1), ZT3D(:,:,:,IZFNEG2), &
- ZT3D(:,:,:,IZBPOS1), ZT3D(:,:,:,IZBPOS2), ZT3D(:,:,:,IZBNEG1), ZT3D(:,:,:,IZBNEG2), &
- ZT3D(:,:,:,IZOMP1), ZT3D(:,:,:,IZOMP2), ZT3D(:,:,:,IZOMN1), ZT3D(:,:,:,IZOMN2) )
+ CALL WENO_K_2_WZ(PWT, ZWORK, ZMEAN, &
+ ZFPOS1(:,:,:), ZFPOS2(:,:,:), ZFNEG1(:,:,:), ZFNEG2(:,:,:), &
+ ZBPOS1(:,:,:), ZBPOS2(:,:,:), ZBNEG1(:,:,:), ZBNEG2(:,:,:), &
+ ZOMP1(:,:,:), ZOMP2(:,:,:), ZOMN1(:,:,:), ZOMN2(:,:,:) )
CALL DZM_DEVICE( ZMEAN, ZWORK )
!$acc kernels
PRWS(:,:,:) = PRWS(:,:,:) - ZWORK(:,:,:)
!$acc end kernels
!
- CALL MNH_REL_ZT3D(IZFPOS1,IZFPOS2,IZFNEG1,IZFNEG2,IZBPOS1,IZBPOS2,IZBNEG1,IZBNEG2,IZOMP1,IZOMP2,IZOMN1,IZOMN2)
+ !Release all memory allocated with MNH_MEM_GET calls since last call to MNH_MEM_POSITION_PIN
+ CALL MNH_MEM_RELEASE()
#endif
!
!
@@ -510,19 +529,38 @@ CASE(5) ! WENO 5
CALL GET_HALO(ZMEAN)! Update HALO - maybe not necessary (T.Lunet)
PRWS = PRWS - DZM(ZMEAN)
#else
- CALL MNH_GET_ZT3D(IZFPOS1,IZFPOS2,IZFPOS3,IZFNEG1,IZFNEG2,IZFNEG3,IZBPOS1, &
- IZBPOS2,IZBPOS3,IZBNEG1,IZBNEG2,IZBNEG3,IZOMP1,IZOMP2,IZOMP3,IZOMN1,IZOMN2,IZOMN3)
+ !Pin positions in the pools of MNH memory
+ CALL MNH_MEM_POSITION_PIN()
+
+ CALL MNH_MEM_GET( ZFPOS1, IIU, IJU, IKU )
+ CALL MNH_MEM_GET( ZFPOS2, IIU, IJU, IKU )
+ CALL MNH_MEM_GET( ZFPOS3, IIU, IJU, IKU )
+ CALL MNH_MEM_GET( ZFNEG1, IIU, IJU, IKU )
+ CALL MNH_MEM_GET( ZFNEG2, IIU, IJU, IKU )
+ CALL MNH_MEM_GET( ZFNEG3, IIU, IJU, IKU )
+ CALL MNH_MEM_GET( ZBPOS1, IIU, IJU, IKU )
+ CALL MNH_MEM_GET( ZBPOS2, IIU, IJU, IKU )
+ CALL MNH_MEM_GET( ZBPOS3, IIU, IJU, IKU )
+ CALL MNH_MEM_GET( ZBNEG1, IIU, IJU, IKU )
+ CALL MNH_MEM_GET( ZBNEG2, IIU, IJU, IKU )
+ CALL MNH_MEM_GET( ZBNEG3, IIU, IJU, IKU )
+ CALL MNH_MEM_GET( ZOMP1, IIU, IJU, IKU )
+ CALL MNH_MEM_GET( ZOMP2, IIU, IJU, IKU )
+ CALL MNH_MEM_GET( ZOMP3, IIU, IJU, IKU )
+ CALL MNH_MEM_GET( ZOMN1, IIU, IJU, IKU )
+ CALL MNH_MEM_GET( ZOMN2, IIU, IJU, IKU )
+ CALL MNH_MEM_GET( ZOMN3, IIU, IJU, IKU )
!
! U component
!
CALL MXF_DEVICE(PRUCT,ZWORK)
- CALL ADVEC_WENO_K_3_UX(HLBCX, PUT, ZWORK, ZMEAN, &
- ZT3D(:,:,:,IZFPOS1), ZT3D(:,:,:,IZFPOS2), ZT3D(:,:,:,IZFPOS3), &
- ZT3D(:,:,:,IZFNEG1), ZT3D(:,:,:,IZFNEG2), ZT3D(:,:,:,IZFNEG3), &
- ZT3D(:,:,:,IZBPOS1), ZT3D(:,:,:,IZBPOS2), ZT3D(:,:,:,IZBPOS3), &
- ZT3D(:,:,:,IZBNEG1), ZT3D(:,:,:,IZBNEG2), ZT3D(:,:,:,IZBNEG3), &
- ZT3D(:,:,:,IZOMP1), ZT3D(:,:,:,IZOMP2), ZT3D(:,:,:,IZOMP3), &
- ZT3D(:,:,:,IZOMN1), ZT3D(:,:,:,IZOMN2), ZT3D(:,:,:,IZOMN3) )
+ CALL ADVEC_WENO_K_3_UX(HLBCX, PUT, ZWORK, ZMEAN, &
+ ZFPOS1(:,:,:), ZFPOS2(:,:,:), ZFPOS3(:,:,:), &
+ ZFNEG1(:,:,:), ZFNEG2(:,:,:), ZFNEG3(:,:,:), &
+ ZBPOS1(:,:,:), ZBPOS2(:,:,:), ZBPOS3(:,:,:), &
+ ZBNEG1(:,:,:), ZBNEG2(:,:,:), ZBNEG3(:,:,:), &
+ ZOMP1(:,:,:), ZOMP2(:,:,:), ZOMP3(:,:,:), &
+ ZOMN1(:,:,:), ZOMN2(:,:,:), ZOMN3(:,:,:) )
CALL GET_HALO_D(ZMEAN)! Update HALO
CALL DXM_DEVICE(ZMEAN,ZWORK)
!$acc kernels
@@ -531,13 +569,13 @@ CASE(5) ! WENO 5
!
IF (.NOT.L2D) THEN
CALL MXM_DEVICE(PRVCT,ZWORK)
- CALL ADVEC_WENO_K_3_MY(HLBCY, PUT, ZWORK, ZMEAN, &
- ZT3D(:,:,:,IZFPOS1), ZT3D(:,:,:,IZFPOS2), ZT3D(:,:,:,IZFPOS3), &
- ZT3D(:,:,:,IZFNEG1), ZT3D(:,:,:,IZFNEG2), ZT3D(:,:,:,IZFNEG3), &
- ZT3D(:,:,:,IZBPOS1), ZT3D(:,:,:,IZBPOS2), ZT3D(:,:,:,IZBPOS3), &
- ZT3D(:,:,:,IZBNEG1), ZT3D(:,:,:,IZBNEG2), ZT3D(:,:,:,IZBNEG3), &
- ZT3D(:,:,:,IZOMP1), ZT3D(:,:,:,IZOMP2), ZT3D(:,:,:,IZOMP3), &
- ZT3D(:,:,:,IZOMN1), ZT3D(:,:,:,IZOMN2), ZT3D(:,:,:,IZOMN3) )
+ CALL ADVEC_WENO_K_3_MY(HLBCY, PUT, ZWORK, ZMEAN, &
+ ZFPOS1(:,:,:), ZFPOS2(:,:,:), ZFPOS3(:,:,:), &
+ ZFNEG1(:,:,:), ZFNEG2(:,:,:), ZFNEG3(:,:,:), &
+ ZBPOS1(:,:,:), ZBPOS2(:,:,:), ZBPOS3(:,:,:), &
+ ZBNEG1(:,:,:), ZBNEG2(:,:,:), ZBNEG3(:,:,:), &
+ ZOMP1(:,:,:), ZOMP2(:,:,:), ZOMP3(:,:,:), &
+ ZOMN1(:,:,:), ZOMN2(:,:,:), ZOMN3(:,:,:) )
CALL GET_HALO_D(ZMEAN)! Update HALO
CALL DYF_DEVICE(ZMEAN,ZWORK)
!$acc kernels
@@ -546,13 +584,13 @@ CASE(5) ! WENO 5
END IF
!
CALL MXM_DEVICE(PRWCT,ZWORK)
- CALL WENO_K_3_MZ(PUT,ZWORK,ZMEAN, &
- ZT3D(:,:,:,IZFPOS1), ZT3D(:,:,:,IZFPOS2), ZT3D(:,:,:,IZFPOS3), &
- ZT3D(:,:,:,IZFNEG1), ZT3D(:,:,:,IZFNEG2), ZT3D(:,:,:,IZFNEG3), &
- ZT3D(:,:,:,IZBPOS1), ZT3D(:,:,:,IZBPOS2), ZT3D(:,:,:,IZBPOS3), &
- ZT3D(:,:,:,IZBNEG1), ZT3D(:,:,:,IZBNEG2), ZT3D(:,:,:,IZBNEG3), &
- ZT3D(:,:,:,IZOMP1), ZT3D(:,:,:,IZOMP2), ZT3D(:,:,:,IZOMP3), &
- ZT3D(:,:,:,IZOMN1), ZT3D(:,:,:,IZOMN2), ZT3D(:,:,:,IZOMN3) )
+ CALL WENO_K_3_MZ(PUT,ZWORK,ZMEAN, &
+ ZFPOS1(:,:,:), ZFPOS2(:,:,:), ZFPOS3(:,:,:), &
+ ZFNEG1(:,:,:), ZFNEG2(:,:,:), ZFNEG3(:,:,:), &
+ ZBPOS1(:,:,:), ZBPOS2(:,:,:), ZBPOS3(:,:,:), &
+ ZBNEG1(:,:,:), ZBNEG2(:,:,:), ZBNEG3(:,:,:), &
+ ZOMP1(:,:,:), ZOMP2(:,:,:), ZOMP3(:,:,:), &
+ ZOMN1(:,:,:), ZOMN2(:,:,:), ZOMN3(:,:,:) )
CALL GET_HALO_D(ZMEAN)! Update HALO - maybe not necessary (T.Lunet)
CALL DZF_DEVICE( ZMEAN, ZWORK )
!$acc kernels
@@ -563,13 +601,13 @@ CASE(5) ! WENO 5
!
IF (.NOT.L2D) THEN
CALL MYM_DEVICE(PRUCT,ZWORK)
- CALL ADVEC_WENO_K_3_MX(HLBCX, PVT, ZWORK, ZMEAN, &
- ZT3D(:,:,:,IZFPOS1), ZT3D(:,:,:,IZFPOS2), ZT3D(:,:,:,IZFPOS3), &
- ZT3D(:,:,:,IZFNEG1), ZT3D(:,:,:,IZFNEG2), ZT3D(:,:,:,IZFNEG3), &
- ZT3D(:,:,:,IZBPOS1), ZT3D(:,:,:,IZBPOS2), ZT3D(:,:,:,IZBPOS3), &
- ZT3D(:,:,:,IZBNEG1), ZT3D(:,:,:,IZBNEG2), ZT3D(:,:,:,IZBNEG3), &
- ZT3D(:,:,:,IZOMP1), ZT3D(:,:,:,IZOMP2), ZT3D(:,:,:,IZOMP3), &
- ZT3D(:,:,:,IZOMN1), ZT3D(:,:,:,IZOMN2), ZT3D(:,:,:,IZOMN3) )
+ CALL ADVEC_WENO_K_3_MX(HLBCX, PVT, ZWORK, ZMEAN, &
+ ZFPOS1(:,:,:), ZFPOS2(:,:,:), ZFPOS3(:,:,:), &
+ ZFNEG1(:,:,:), ZFNEG2(:,:,:), ZFNEG3(:,:,:), &
+ ZBPOS1(:,:,:), ZBPOS2(:,:,:), ZBPOS3(:,:,:), &
+ ZBNEG1(:,:,:), ZBNEG2(:,:,:), ZBNEG3(:,:,:), &
+ ZOMP1(:,:,:), ZOMP2(:,:,:), ZOMP3(:,:,:), &
+ ZOMN1(:,:,:), ZOMN2(:,:,:), ZOMN3(:,:,:) )
CALL GET_HALO_D(ZMEAN)! Update HALO
CALL DXF_DEVICE(ZMEAN,ZWORK)
!$acc kernels
@@ -577,13 +615,13 @@ CASE(5) ! WENO 5
!$acc end kernels
!
CALL MYF_DEVICE(PRVCT,ZWORK)
- CALL ADVEC_WENO_K_3_VY(HLBCY, PVT, ZWORK, ZMEAN, &
- ZT3D(:,:,:,IZFPOS1), ZT3D(:,:,:,IZFPOS2), ZT3D(:,:,:,IZFPOS3), &
- ZT3D(:,:,:,IZFNEG1), ZT3D(:,:,:,IZFNEG2), ZT3D(:,:,:,IZFNEG3), &
- ZT3D(:,:,:,IZBPOS1), ZT3D(:,:,:,IZBPOS2), ZT3D(:,:,:,IZBPOS3), &
- ZT3D(:,:,:,IZBNEG1), ZT3D(:,:,:,IZBNEG2), ZT3D(:,:,:,IZBNEG3), &
- ZT3D(:,:,:,IZOMP1), ZT3D(:,:,:,IZOMP2), ZT3D(:,:,:,IZOMP3), &
- ZT3D(:,:,:,IZOMN1), ZT3D(:,:,:,IZOMN2), ZT3D(:,:,:,IZOMN3) )
+ CALL ADVEC_WENO_K_3_VY(HLBCY, PVT, ZWORK, ZMEAN, &
+ ZFPOS1(:,:,:), ZFPOS2(:,:,:), ZFPOS3(:,:,:), &
+ ZFNEG1(:,:,:), ZFNEG2(:,:,:), ZFNEG3(:,:,:), &
+ ZBPOS1(:,:,:), ZBPOS2(:,:,:), ZBPOS3(:,:,:), &
+ ZBNEG1(:,:,:), ZBNEG2(:,:,:), ZBNEG3(:,:,:), &
+ ZOMP1(:,:,:), ZOMP2(:,:,:), ZOMP3(:,:,:), &
+ ZOMN1(:,:,:), ZOMN2(:,:,:), ZOMN3(:,:,:) )
CALL GET_HALO_D(ZMEAN)! Update HALO
CALL DYM_DEVICE(ZMEAN,ZWORK)
!$acc kernels
@@ -591,13 +629,13 @@ CASE(5) ! WENO 5
!$acc end kernels
!
CALL MYM_DEVICE(PRWCT,ZWORK)
- CALL WENO_K_3_MZ(PVT,ZWORK,ZMEAN, &
- ZT3D(:,:,:,IZFPOS1), ZT3D(:,:,:,IZFPOS2), ZT3D(:,:,:,IZFPOS3), &
- ZT3D(:,:,:,IZFNEG1), ZT3D(:,:,:,IZFNEG2), ZT3D(:,:,:,IZFNEG3), &
- ZT3D(:,:,:,IZBPOS1), ZT3D(:,:,:,IZBPOS2), ZT3D(:,:,:,IZBPOS3), &
- ZT3D(:,:,:,IZBNEG1), ZT3D(:,:,:,IZBNEG2), ZT3D(:,:,:,IZBNEG3), &
- ZT3D(:,:,:,IZOMP1), ZT3D(:,:,:,IZOMP2), ZT3D(:,:,:,IZOMP3), &
- ZT3D(:,:,:,IZOMN1), ZT3D(:,:,:,IZOMN2), ZT3D(:,:,:,IZOMN3) )
+ CALL WENO_K_3_MZ(PVT, ZWORK, ZMEAN, &
+ ZFPOS1(:,:,:), ZFPOS2(:,:,:), ZFPOS3(:,:,:), &
+ ZFNEG1(:,:,:), ZFNEG2(:,:,:), ZFNEG3(:,:,:), &
+ ZBPOS1(:,:,:), ZBPOS2(:,:,:), ZBPOS3(:,:,:), &
+ ZBNEG1(:,:,:), ZBNEG2(:,:,:), ZBNEG3(:,:,:), &
+ ZOMP1(:,:,:), ZOMP2(:,:,:), ZOMP3(:,:,:), &
+ ZOMN1(:,:,:), ZOMN2(:,:,:), ZOMN3(:,:,:) )
CALL GET_HALO_D(ZMEAN)! Update HALO - maybe not necessary (T.Lunet)
CALL DZF_DEVICE( ZMEAN, ZWORK )
!$acc kernels
@@ -608,13 +646,13 @@ CASE(5) ! WENO 5
! W component
!
CALL MZM_DEVICE(PRUCT,ZWORK)
- CALL ADVEC_WENO_K_3_MX(HLBCX, PWT, ZWORK, ZMEAN, &
- ZT3D(:,:,:,IZFPOS1), ZT3D(:,:,:,IZFPOS2), ZT3D(:,:,:,IZFPOS3), &
- ZT3D(:,:,:,IZFNEG1), ZT3D(:,:,:,IZFNEG2), ZT3D(:,:,:,IZFNEG3), &
- ZT3D(:,:,:,IZBPOS1), ZT3D(:,:,:,IZBPOS2), ZT3D(:,:,:,IZBPOS3), &
- ZT3D(:,:,:,IZBNEG1), ZT3D(:,:,:,IZBNEG2), ZT3D(:,:,:,IZBNEG3), &
- ZT3D(:,:,:,IZOMP1), ZT3D(:,:,:,IZOMP2), ZT3D(:,:,:,IZOMP3), &
- ZT3D(:,:,:,IZOMN1), ZT3D(:,:,:,IZOMN2), ZT3D(:,:,:,IZOMN3) )
+ CALL ADVEC_WENO_K_3_MX(HLBCX, PWT, ZWORK, ZMEAN, &
+ ZFPOS1(:,:,:), ZFPOS2(:,:,:), ZFPOS3(:,:,:), &
+ ZFNEG1(:,:,:), ZFNEG2(:,:,:), ZFNEG3(:,:,:), &
+ ZBPOS1(:,:,:), ZBPOS2(:,:,:), ZBPOS3(:,:,:), &
+ ZBNEG1(:,:,:), ZBNEG2(:,:,:), ZBNEG3(:,:,:), &
+ ZOMP1(:,:,:), ZOMP2(:,:,:), ZOMP3(:,:,:), &
+ ZOMN1(:,:,:), ZOMN2(:,:,:), ZOMN3(:,:,:) )
CALL GET_HALO_D(ZMEAN)! Update HALO
CALL DXF_DEVICE(ZMEAN,ZWORK)
!$acc kernels
@@ -623,13 +661,13 @@ CASE(5) ! WENO 5
!
IF (.NOT.L2D) THEN
CALL MZM_DEVICE(PRVCT,ZWORK)
- CALL ADVEC_WENO_K_3_MY(HLBCY, PWT, ZWORK, ZMEAN, &
- ZT3D(:,:,:,IZFPOS1), ZT3D(:,:,:,IZFPOS2), ZT3D(:,:,:,IZFPOS3), &
- ZT3D(:,:,:,IZFNEG1), ZT3D(:,:,:,IZFNEG2), ZT3D(:,:,:,IZFNEG3), &
- ZT3D(:,:,:,IZBPOS1), ZT3D(:,:,:,IZBPOS2), ZT3D(:,:,:,IZBPOS3), &
- ZT3D(:,:,:,IZBNEG1), ZT3D(:,:,:,IZBNEG2), ZT3D(:,:,:,IZBNEG3), &
- ZT3D(:,:,:,IZOMP1), ZT3D(:,:,:,IZOMP2), ZT3D(:,:,:,IZOMP3), &
- ZT3D(:,:,:,IZOMN1), ZT3D(:,:,:,IZOMN2), ZT3D(:,:,:,IZOMN3) )
+ CALL ADVEC_WENO_K_3_MY(HLBCY, PWT, ZWORK, ZMEAN, &
+ ZFPOS1(:,:,:), ZFPOS2(:,:,:), ZFPOS3(:,:,:), &
+ ZFNEG1(:,:,:), ZFNEG2(:,:,:), ZFNEG3(:,:,:), &
+ ZBPOS1(:,:,:), ZBPOS2(:,:,:), ZBPOS3(:,:,:), &
+ ZBNEG1(:,:,:), ZBNEG2(:,:,:), ZBNEG3(:,:,:), &
+ ZOMP1(:,:,:), ZOMP2(:,:,:), ZOMP3(:,:,:), &
+ ZOMN1(:,:,:), ZOMN2(:,:,:), ZOMN3(:,:,:) )
CALL GET_HALO_D(ZMEAN)! Update HALO
CALL DYF_DEVICE(ZMEAN,ZWORK)
!$acc kernels
@@ -638,34 +676,38 @@ CASE(5) ! WENO 5
END IF
!
CALL MZF_DEVICE( PRWCT, ZWORK )
- CALL WENO_K_3_WZ(PWT,ZWORK,ZMEAN, &
- ZT3D(:,:,:,IZFPOS1), ZT3D(:,:,:,IZFPOS2), ZT3D(:,:,:,IZFPOS3), &
- ZT3D(:,:,:,IZFNEG1), ZT3D(:,:,:,IZFNEG2), ZT3D(:,:,:,IZFNEG3), &
- ZT3D(:,:,:,IZBPOS1), ZT3D(:,:,:,IZBPOS2), ZT3D(:,:,:,IZBPOS3), &
- ZT3D(:,:,:,IZBNEG1), ZT3D(:,:,:,IZBNEG2), ZT3D(:,:,:,IZBNEG3), &
- ZT3D(:,:,:,IZOMP1), ZT3D(:,:,:,IZOMP2), ZT3D(:,:,:,IZOMP3), &
- ZT3D(:,:,:,IZOMN1), ZT3D(:,:,:,IZOMN2), ZT3D(:,:,:,IZOMN3) )
+ CALL WENO_K_3_WZ(PWT,ZWORK,ZMEAN, &
+ ZFPOS1(:,:,:), ZFPOS2(:,:,:), ZFPOS3(:,:,:), &
+ ZFNEG1(:,:,:), ZFNEG2(:,:,:), ZFNEG3(:,:,:), &
+ ZBPOS1(:,:,:), ZBPOS2(:,:,:), ZBPOS3(:,:,:), &
+ ZBNEG1(:,:,:), ZBNEG2(:,:,:), ZBNEG3(:,:,:), &
+ ZOMP1(:,:,:), ZOMP2(:,:,:), ZOMP3(:,:,:), &
+ ZOMN1(:,:,:), ZOMN2(:,:,:), ZOMN3(:,:,:) )
CALL GET_HALO_D(ZMEAN)! Update HALO - maybe not necessary (T.Lunet)
CALL DZM_DEVICE( ZMEAN, ZWORK )
!$acc kernels
PRWS(:,:,:) = PRWS(:,:,:) - ZWORK(:,:,:)
!$acc end kernels
!
- CALL MNH_REL_ZT3D(IZFPOS1,IZFPOS2,IZFPOS3,IZFNEG1,IZFNEG2,IZFNEG3,IZBPOS1, &
- IZBPOS2,IZBPOS3,IZBNEG1,IZBNEG2,IZBNEG3,IZOMP1,IZOMP2,IZOMP3,IZOMN1,IZOMN2,IZOMN3)
+ !Release all memory allocated with MNH_MEM_GET calls since last call to MNH_MEM_POSITION_PIN
+ CALL MNH_MEM_RELEASE()
#endif
!
!
END SELECT
! ---------------------------------
!$acc update self(PRUS,PRVS,PRWS)
-!
+
+!$acc end data
+
+!Release all memory allocated with MNH_MEM_GET calls since beginning of subroutine
+CALL MNH_MEM_RELEASE()
+
IF (MPPDB_INITIALIZED) THEN
CALL MPPDB_CHECK(PRUS,"ADVECUVW_WENO_K end:PRUS")
CALL MPPDB_CHECK(PRVS,"ADVECUVW_WENO_K end:PRVS")
CALL MPPDB_CHECK(PRWS,"ADVECUVW_WENO_K end:PRWS")
END IF
-!$acc end data
END SUBROUTINE ADVECUVW_WENO_K
diff --git a/src/ZSOLVER/contrav.f90 b/src/ZSOLVER/contrav.f90
index 8329a9eb2..ec2915c81 100644
--- a/src/ZSOLVER/contrav.f90
+++ b/src/ZSOLVER/contrav.f90
@@ -32,7 +32,7 @@ END SUBROUTINE CONTRAV
!
#ifdef MNH_OPENACC
SUBROUTINE CONTRAV_DEVICE(HLBCX,HLBCY,PRUT,PRVT,PRWT,PDXX,PDYY,PDZZ,PDZX,PDZY, &
- PRUCT,PRVCT,PRWCT,KADV_ORDER,Z1,Z2,ODATA_ON_DEVICE )
+ PRUCT,PRVCT,PRWCT,KADV_ORDER,ODATA_ON_DEVICE )
!
!
CHARACTER (LEN=4), DIMENSION(2), INTENT(IN) :: HLBCX ! X direction LBC type
@@ -49,7 +49,6 @@ REAL, DIMENSION(:,:,:), INTENT(OUT) :: PRUCT ! Contrav comp along x-bar
REAL, DIMENSION(:,:,:), INTENT(OUT) :: PRVCT ! Contrav comp along y-bar
REAL, DIMENSION(:,:,:), INTENT(OUT) :: PRWCT ! Contrav comp along z-bar
INTEGER, INTENT(IN) :: KADV_ORDER ! Order of the advection scheme
-REAL, DIMENSION(:,:,:), INTENT(OUT) :: Z1,Z2 ! Work arrays
LOGICAL, OPTIONAL, INTENT(IN) :: ODATA_ON_DEVICE ! Is some of the data on the accelerator device
!
!
@@ -450,7 +449,7 @@ END SUBROUTINE CONTRAV
#ifdef MNH_OPENACC
! ##############################################################
SUBROUTINE CONTRAV_DEVICE(HLBCX,HLBCY,PRUT,PRVT,PRWT,PDXX,PDYY,PDZZ,PDZX,PDZY, &
- PRUCT,PRVCT,PRWCT,KADV_ORDER,Z1,Z2,ODATA_ON_DEVICE )
+ PRUCT,PRVCT,PRWCT,KADV_ORDER,ODATA_ON_DEVICE )
! ##############################################################
!
!!**** *CONTRAV * - computes the contravariant components from the
@@ -524,6 +523,7 @@ USE MODD_PARAMETERS
USE MODE_ll
USE MODE_MPPDB
#ifdef MNH_OPENACC
+USE MODE_MNH_ZWORK, ONLY: MNH_MEM_GET, MNH_MEM_POSITION_PIN, MNH_MEM_RELEASE
use mode_msg
#endif
!
@@ -552,7 +552,6 @@ REAL, DIMENSION(:,:,:), INTENT(OUT) :: PRUCT ! Contrav comp along x-bar
REAL, DIMENSION(:,:,:), INTENT(OUT) :: PRVCT ! Contrav comp along y-bar
REAL, DIMENSION(:,:,:), INTENT(OUT) :: PRWCT ! Contrav comp along z-bar
INTEGER, INTENT(IN) :: KADV_ORDER ! Order of the advection scheme
-REAL, DIMENSION(:,:,:), INTENT(OUT) :: Z1,Z2 ! Work arrays
LOGICAL, OPTIONAL, INTENT(IN) :: ODATA_ON_DEVICE ! Is some of the data on the accelerator device
!
!
@@ -566,13 +565,13 @@ INTEGER :: IINFO_ll
LOGICAL :: GDATA_ON_DEVICE
real :: ZTMP1, ZTMP2 ! Intermediate work variables
REAL, DIMENSION(:,:), POINTER , CONTIGUOUS :: ZU_EAST, ZV_NORTH, ZDZX_EAST, ZDZY_NORTH
+REAL, DIMENSION(:,:,:), POINTER, CONTIGUOUS :: Z1,Z2 ! Work arrays
TYPE(LIST_ll), POINTER :: TZFIELD_U, TZFIELD_V, TZFIELD_DZX, TZFIELD_DZY
TYPE(HALO2LIST_ll), SAVE, POINTER :: TZHALO2_U, TZHALO2_V, TZHALO2_DZX, TZHALO2_DZY
LOGICAL , SAVE :: GFIRST_CALL_CONTRAV_DEVICE = .TRUE.
!
LOGICAL :: GWEST,GEAST,GSOUTH,GNORTH
!
-!$acc data present( PRUT, PRVT, PRWT, PDXX, PDYY, PDZZ, PDZX, PDZY, PRUCT, PRVCT, PRWCT, Z1, Z2 )
IF ( PRESENT(ODATA_ON_DEVICE) ) THEN
GDATA_ON_DEVICE = ODATA_ON_DEVICE
@@ -599,7 +598,13 @@ END IF
IIU= SIZE(PDXX,1)
IJU= SIZE(PDXX,2)
IKU= SIZE(PDXX,3)
-!
+
+!Pin positions in the pools of MNH memory
+CALL MNH_MEM_POSITION_PIN()
+
+CALL MNH_MEM_GET( Z1, IIU, IJU, IKU )
+CALL MNH_MEM_GET( Z2, IIU, IJU, IKU )
+
GWEST = ( HLBCX(1) /= 'CYCL' .AND. LWEST_ll() )
GEAST = ( HLBCX(2) /= 'CYCL' .AND. LEAST_ll() )
GSOUTH = ( HLBCY(1) /= 'CYCL' .AND. LSOUTH_ll() )
@@ -609,7 +614,9 @@ CALL GET_INDICE_ll( IIB,IJB,IIE,IJE)
!
IKB=1+JPVEXT
IKE=IKU - JPVEXT
-!
+
+!$acc data present( PRUT, PRVT, PRWT, PDXX, PDYY, PDZZ, PDZX, PDZY, PRUCT, PRVCT, PRWCT, Z1, Z2 )
+
IF (GDATA_ON_DEVICE) THEN
!PW TODO:remplacer (ailleurs aussi...) 1/PDXX... par PINV_PDXX (fait pour la turbulence...) cfr MNH/turb_hor_splt.f90
!$acc kernels
@@ -734,12 +741,12 @@ ELSE IF (KADV_ORDER == 4 ) THEN
END IF
IE=IIE-1
!!$ ELSE
-!!$ IF (LWEST_ll()) THEN
+!!$ IF ( GWEST ) THEN
!!$ IW=IIB+1
!!$ ELSE
!!$ IW=IIB
!!$ END IF
-!!$ IF (LEAST_ll() .AND. HLBCX(2)/='CYCL' ) THEN
+!!$ IF ( GEAST ) THEN
!!$ IE=IIE-1
!!$ ELSE
!!$ IE=IIE
@@ -754,12 +761,12 @@ ELSE IF (KADV_ORDER == 4 ) THEN
END IF
IN=IJE-1
!!$ ELSE
-!!$ IF (LSOUTH_ll()) THEN
+!!$ IF ( GSOUTH ) THEN
!!$ IS=IJB+1
!!$ ELSE
!!$ IS=IJB
!!$ END IF
-!!$ IF (LNORTH_ll() .AND. HLBCY(2)/='CYCL' ) THEN
+!!$ IF ( GNORTH ) THEN
!!$ IN=IJE-1
!!$ ELSE
!!$ IN=IJE
@@ -902,7 +909,12 @@ IF (KADV_ORDER == 4 ) THEN
END IF
END IF FLAT
-!-----------------------------------------------------------------------
+
+!$acc end data
+
+!Release all memory allocated with MNH_MEM_GET calls since last call to MNH_MEM_POSITION_PIN
+CALL MNH_MEM_RELEASE()
+
IF (MPPDB_INITIALIZED) THEN
!Check all OUT arrays
CALL MPPDB_CHECK(PRUCT,"CONTRAV end:PRUCT")
@@ -910,7 +922,6 @@ IF (MPPDB_INITIALIZED) THEN
CALL MPPDB_CHECK(PRWCT,"CONTRAV end:PRWCT")
END IF
-!$acc end data
-
+!-----------------------------------------------------------------------
END SUBROUTINE CONTRAV_DEVICE
#endif
diff --git a/src/ZSOLVER/gdiv.f90 b/src/ZSOLVER/gdiv.f90
index b82d37e71..d1db7f3ef 100644
--- a/src/ZSOLVER/gdiv.f90
+++ b/src/ZSOLVER/gdiv.f90
@@ -151,10 +151,6 @@ INTEGER :: JI,JJ,JK ! loop indexes
!
INTEGER :: IIU,IJU,IKU
!
-#ifdef MNH_OPENACC
-REAL, DIMENSION(:,:,:) , POINTER , CONTIGUOUS :: ZTMP1,ZTMP2
-#endif
-!
LOGICAL :: GWEST,GEAST,GSOUTH,GNORTH
!
!-------------------------------------------------------------------------------
@@ -185,9 +181,6 @@ CALL MNH_MEM_GET( ZWC, IIU, IJU, IKU )
CALL MNH_MEM_GET( Z1, IIU, IJU, IKU )
CALL MNH_MEM_GET( Z2, IIU, IJU, IKU )
CALL MNH_MEM_GET( Z3, IIU, IJU, IKU )
-
-CALL MNH_MEM_GET( ZTMP1, IIU, IJU, IKU )
-CALL MNH_MEM_GET( ZTMP2, IIU, IJU, IKU )
#endif
!
GWEST = ( HLBCX(1) /= 'CYCL' .AND. LWEST_ll() )
@@ -219,7 +212,7 @@ GNORTH = ( .NOT. L2D .AND. HLBCY(2) /= 'CYCL' .AND. LNORTH_ll() )
CALL CONTRAV(HLBCX,HLBCY,PU,PV,PW,PDXX,PDYY,PDZZ,PDZX,PDZY,ZUC,ZVC,ZWC,4)
#else
CALL CONTRAV_DEVICE(HLBCX,HLBCY,PU,PV,PW,PDXX,PDYY,PDZZ,PDZX,PDZY,ZUC,ZVC,ZWC,4, &
- ZTMP1,ZTMP2,ODATA_ON_DEVICE=.TRUE.)
+ ODATA_ON_DEVICE=.TRUE.)
#endif
!
!-------------------------------------------------------------------------------
--
GitLab