From 5bd4760dfde5a8b51f10c53f86434597caf40f3a Mon Sep 17 00:00:00 2001
From: Philippe WAUTELET <philippe.wautelet@aero.obs-mip.fr>
Date: Tue, 24 Jan 2023 13:15:41 +0100
Subject: [PATCH] Philippe 24/01/2023: OpenACC: more data always on GPU (less
data transfers)
---
src/MNH/ini_modeln.f90 | 5 ++++-
src/MNH/modeln.f90 | 30 +++++++++++++++++-------------
2 files changed, 21 insertions(+), 14 deletions(-)
diff --git a/src/MNH/ini_modeln.f90 b/src/MNH/ini_modeln.f90
index 962c404f2..45d2945f1 100644
--- a/src/MNH/ini_modeln.f90
+++ b/src/MNH/ini_modeln.f90
@@ -860,6 +860,7 @@ ALLOCATE(XRWS_PRES(IIU,IJU,IKU)); XRWS_PRES = 0.0
ALLOCATE(XRTHS(IIU,IJU,IKU)) ; XRTHS = 0.0
!$acc enter data copyin(XRTHS)
ALLOCATE(XRTHS_CLD(IIU,IJU,IKU)); XRTHS_CLD = 0.0
+!$acc enter data copyin(XRTHS_CLD)
IF ( LIBM ) THEN
ALLOCATE(ZIBM_LS(IIU,IJU,IKU)) ; ZIBM_LS = 0.0
@@ -939,6 +940,7 @@ ALLOCATE(XRT(IIU,IJU,IKU,NRR)) ; XRT = 0.0
!$acc enter data copyin( XRT )
ALLOCATE(XRRS(IIU,IJU,IKU,NRR)) ; XRRS = 0.0
ALLOCATE(XRRS_CLD(IIU,IJU,IKU,NRR)); XRRS_CLD = 0.0
+!$acc enter data copyin( XRRS, XRRS_CLD )
!
IF (CTURB /= 'NONE' .AND. NRR>1) THEN
ALLOCATE(XSRCT(IIU,IJU,IKU))
@@ -974,6 +976,7 @@ END IF
ALLOCATE(XSVT(IIU,IJU,IKU,NSV)) ; XSVT = 0.
ALLOCATE(XRSVS(IIU,IJU,IKU,NSV)); XRSVS = 0.
ALLOCATE(XRSVS_CLD(IIU,IJU,IKU,NSV)); XRSVS_CLD = 0.0
+!$acc enter data copyin( XRSVS, XRSVS_CLD )
ALLOCATE(XZWS(IIU,IJU)) ; XZWS(:,:) = XZWS_DEFAULT
!
IF (LPASPOL) THEN
@@ -1942,7 +1945,7 @@ CALL READ_FIELD(KMI,TPINIFILE,IIU,IJU,IKU, &
ZIBM_LS,XIBM_XMUT,XUMEANW,XVMEANW,XWMEANW,XUMEANN,XVMEANN, &
XWMEANN,XUMEANE,XVMEANE,XWMEANE,XUMEANS,XVMEANS,XWMEANS )
-!$acc update device( XPABST, XRT, XRUS_PRES, XRVS_PRES, XRWS_PRES )
+!$acc update device( XPABST, XRT, XRUS_PRES, XRVS_PRES, XRWS_PRES, XRRS_CLD, XRSVS_CLD )
!
!-------------------------------------------------------------------------------
!
diff --git a/src/MNH/modeln.f90 b/src/MNH/modeln.f90
index d79846939..ceda1aedb 100644
--- a/src/MNH/modeln.f90
+++ b/src/MNH/modeln.f90
@@ -1633,19 +1633,19 @@ XTIME_BU_PROCESS = 0.
XTIME_LES_BU_PROCESS = 0.
!
!
-!XRWS_PRES copy and not copyout (hidden in UPDATE_HALO)
!$acc data create ( XUT, XVT, XWT ) &
!$acc & present( XTHT, XRT, XPABST, XTHVREF, XRHODJ ) &
!$acc & present( XDXX, XDYY, XDZZ, XDZX, XDZY, XRUS, XRVS, XRWS, XRWS_PRES ) &
-!$acc & copyin ( XSVT, XRTHS_CLD, XRRS_CLD ) &
-!$acc & copy ( XRRS )
+!$acc & present ( XRRS, XRSVS, XRTHS_CLD, XRRS_CLD, XRSVS_CLD ) &
+!$acc & copyin ( XSVT )
!
-!$acc update device( XRUS, XRVS, XRWS )
+!$acc update device( XRUS, XRVS, XRWS, XRRS, XRSVS )
!$acc update device( XRUS_PRES, XRVS_PRES, XRWS_PRES )
!$acc update device( XUT, XVT, XWT, XTHT, XRT, XRHODJ, XRTHS )
+!$acc update device( XRTHS_CLD, XRRS_CLD, XRSVS_CLD )
!
-!$acc data copyin (XTKET, XRSVS_CLD) &
-!$acc & copy (XRTKES, XRSVS) &
+!$acc data copyin (XTKET) &
+!$acc & copy (XRTKES) &
!$acc & copyout(XRTKEMS)
#ifdef MNH_BITREP_OMP
CALL SBR_FZ(XRRS_CLD)
@@ -1661,7 +1661,6 @@ CALL ADVECTION_METSV ( TZBAKFILE, CUVW_ADV_SCHEME, &
XRTHS_CLD, XRRS_CLD, XRSVS_CLD, XRTKEMS )
!$acc end data
!
-!$acc update host(XRTHS)
!
CALL SECOND_MNH2(ZTIME2)
!
@@ -1825,9 +1824,11 @@ ZPABST(:,:,:) = XPABST(:,:,:)
!
IF(.NOT. L1D) THEN
!
+!$acc kernels
XRUS_PRES = XRUS
XRVS_PRES = XRVS
XRWS_PRES = XRWS
+!$acc end kernels
!
!$acc data present( XRHODJ, XDXX, XDYY, XDZZ, XDZX, XDZY ) &
!$acc & present( XRHOM, XAF, XBFY, XCF, XTRIGSX, XTRIGSY, NIFAXX, NIFAXY ) &
@@ -1950,21 +1951,22 @@ IF (CCLOUD /= 'NONE' .AND. CELEC == 'NONE') THEN
XWT_ACT_NUC(:,:,:) = 0.
END IF
!
+!$acc kernels
XRTHS_CLD(:, :, : ) = XRTHS(:, :, : )
XRRS_CLD (:, :, :, : ) = XRRS(:, :, :, : )
XRSVS_CLD(:, :, :, : ) = XRSVS(:, :, :, : )
-!$acc data present( XRHODJ, XRHODREF, XEXNREF, XRTHS, ZPABST, XTHT ) &
+!$acc end kernels
+
+!$acc data present( XRHODJ, XRHODREF, XEXNREF, XRTHS, XRRS, XRSVS, ZPABST, XTHT ) &
!$acc & copyin (XZZ, XSIGS, VSIGQSAT, XMFCONV, XTHM, XPABSM, &
!$acc & XRCM, XWT_ACT_NUC, XDTHRAD, XCF_MF, XRC_MF, XRI_MF, &
!$acc & XSOLORG, XMI) &
!$acc & copy (XSUPSAT, XNACT, XNPRO, XSSPRO, &
-!$acc & XRT, XRRS, XSVT, XRSVS, XCLDFR, XCIT, XINPRR3D, XEVAP3D, &
+!$acc & XRT, XSVT, XCLDFR, XCIT, XINPRR3D, XEVAP3D, &
!$acc & XINPRC, XINPRR, XINPRS, XINPRG, XINPRH, XINDEP, &
!$acc & XHLC_HRC, XHLC_HCF, XHLI_HRI, XHLI_HCF) &
!$acc & copyout(XSRCT, XRAINFR)
-!$acc update device ( XRTHS )
-
IF (CSURF=='EXTE') THEN
#ifndef MNH_OPENACC
ALLOCATE (ZSEA(SIZE(XRHODJ,1),SIZE(XRHODJ,2)))
@@ -2026,11 +2028,13 @@ IF (CCLOUD /= 'NONE' .AND. CELEC == 'NONE') THEN
END IF
!$acc end data
-!$acc update host(XRTHS)
-
+!$acc kernels
XRTHS_CLD(:, :, : ) = XRTHS(:, :, : ) - XRTHS_CLD(:, :, : )
XRRS_CLD (:, :, :, : ) = XRRS (:, :, :, : ) - XRRS_CLD (:, :, :, : )
XRSVS_CLD(:, :, :, : ) = XRSVS(:, :, :, : ) - XRSVS_CLD(:, :, :, : )
+!$acc end kernels
+!
+!$acc update host( XRTHS, XRRS, XRSVS, XRTHS_CLD, XRRS_CLD, XRSVS_CLD )
!
IF (CCLOUD /= 'REVE' ) THEN
XACPRR = XACPRR + XINPRR * XTSTEP
--
GitLab