Philippe 24/01/2023: OpenACC: more data always on GPU (less data transfers)

src/MNH/ini_modeln.f90

@@ -860,6 +860,7 @@ ALLOCATE(XRWS_PRES(IIU,IJU,IKU)); XRWS_PRES = 0.0
 ALLOCATE(XRTHS(IIU,IJU,IKU))    ; XRTHS = 0.0
 !$acc enter data copyin(XRTHS)
 ALLOCATE(XRTHS_CLD(IIU,IJU,IKU)); XRTHS_CLD = 0.0
+!$acc enter data copyin(XRTHS_CLD)
 
 IF ( LIBM ) THEN
   ALLOCATE(ZIBM_LS(IIU,IJU,IKU))  ; ZIBM_LS = 0.0
@@ -939,6 +940,7 @@ ALLOCATE(XRT(IIU,IJU,IKU,NRR)) ;     XRT = 0.0
 !$acc enter data copyin( XRT )
 ALLOCATE(XRRS(IIU,IJU,IKU,NRR)) ;    XRRS = 0.0
 ALLOCATE(XRRS_CLD(IIU,IJU,IKU,NRR)); XRRS_CLD = 0.0
+!$acc enter data copyin( XRRS, XRRS_CLD )
 !
 IF (CTURB /= 'NONE' .AND. NRR>1) THEN
   ALLOCATE(XSRCT(IIU,IJU,IKU))
@@ -974,6 +976,7 @@ END IF
 ALLOCATE(XSVT(IIU,IJU,IKU,NSV)) ;     XSVT  = 0.
 ALLOCATE(XRSVS(IIU,IJU,IKU,NSV));     XRSVS = 0.
 ALLOCATE(XRSVS_CLD(IIU,IJU,IKU,NSV)); XRSVS_CLD = 0.0
+!$acc enter data copyin( XRSVS, XRSVS_CLD )
 ALLOCATE(XZWS(IIU,IJU)) ;             XZWS(:,:) = XZWS_DEFAULT
 !
 IF (LPASPOL) THEN
@@ -1942,7 +1945,7 @@ CALL READ_FIELD(KMI,TPINIFILE,IIU,IJU,IKU,                                    &
                 ZIBM_LS,XIBM_XMUT,XUMEANW,XVMEANW,XWMEANW,XUMEANN,XVMEANN,    &
                 XWMEANN,XUMEANE,XVMEANE,XWMEANE,XUMEANS,XVMEANS,XWMEANS       )
 
-!$acc update device( XPABST, XRT, XRUS_PRES, XRVS_PRES, XRWS_PRES )
+!$acc update device( XPABST, XRT, XRUS_PRES, XRVS_PRES, XRWS_PRES, XRRS_CLD, XRSVS_CLD )
 !
 !-------------------------------------------------------------------------------
 !

src/MNH/modeln.f90

@@ -1633,19 +1633,19 @@ XTIME_BU_PROCESS = 0.
 XTIME_LES_BU_PROCESS = 0.
 !
 !
-!XRWS_PRES copy and not copyout (hidden in UPDATE_HALO)
 !$acc data create ( XUT, XVT, XWT )                                             &
 !$acc &    present( XTHT, XRT, XPABST, XTHVREF, XRHODJ )                        &
 !$acc &    present( XDXX, XDYY, XDZZ, XDZX, XDZY, XRUS, XRVS, XRWS, XRWS_PRES ) &
-!$acc &    copyin ( XSVT, XRTHS_CLD, XRRS_CLD )                                 &
-!$acc &    copy   ( XRRS )
+!$acc &    present ( XRRS, XRSVS, XRTHS_CLD, XRRS_CLD, XRSVS_CLD )              &
+!$acc &    copyin ( XSVT )
 !
-!$acc update device( XRUS, XRVS, XRWS )
+!$acc update device( XRUS, XRVS, XRWS, XRRS, XRSVS )
 !$acc update device( XRUS_PRES, XRVS_PRES, XRWS_PRES )
 !$acc update device( XUT, XVT, XWT, XTHT, XRT, XRHODJ, XRTHS )
+!$acc update device( XRTHS_CLD, XRRS_CLD, XRSVS_CLD )
 !
-!$acc data copyin (XTKET, XRSVS_CLD) &
-!$acc &    copy   (XRTKES, XRSVS)    &
+!$acc data copyin (XTKET)   &
+!$acc &    copy   (XRTKES)  &
 !$acc &    copyout(XRTKEMS)
 #ifdef MNH_BITREP_OMP
 CALL SBR_FZ(XRRS_CLD)
@@ -1661,7 +1661,6 @@ CALL ADVECTION_METSV ( TZBAKFILE, CUVW_ADV_SCHEME,                     &
                  XRTHS_CLD, XRRS_CLD, XRSVS_CLD, XRTKEMS               )
 !$acc end data
 !
-!$acc update host(XRTHS)
 !
 CALL SECOND_MNH2(ZTIME2)
 !
@@ -1825,9 +1824,11 @@ ZPABST(:,:,:) = XPABST(:,:,:)
 !
 IF(.NOT. L1D) THEN
 !
+!$acc kernels
   XRUS_PRES = XRUS
   XRVS_PRES = XRVS
   XRWS_PRES = XRWS
+!$acc end kernels
 !
 !$acc data present( XRHODJ, XDXX, XDYY, XDZZ, XDZX, XDZY )                    &
 !$acc &    present( XRHOM, XAF, XBFY, XCF, XTRIGSX, XTRIGSY, NIFAXX, NIFAXY ) &
@@ -1950,21 +1951,22 @@ IF (CCLOUD /= 'NONE' .AND. CELEC == 'NONE') THEN
     XWT_ACT_NUC(:,:,:) = 0.
   END IF
 !
+!$acc kernels
   XRTHS_CLD(:, :, : )    = XRTHS(:, :, : )
   XRRS_CLD (:, :, :, : ) = XRRS(:, :, :, : )
   XRSVS_CLD(:, :, :, : ) = XRSVS(:, :, :, : )
-!$acc data present( XRHODJ, XRHODREF, XEXNREF, XRTHS, ZPABST, XTHT )                             &
+!$acc end kernels
+
+!$acc data present( XRHODJ, XRHODREF, XEXNREF, XRTHS, XRRS, XRSVS, ZPABST, XTHT )                &
 !$acc &    copyin (XZZ, XSIGS, VSIGQSAT, XMFCONV, XTHM, XPABSM,                                  &
 !$acc &            XRCM, XWT_ACT_NUC, XDTHRAD, XCF_MF, XRC_MF, XRI_MF,                           &
 !$acc &            XSOLORG, XMI)                                                                 &
 !$acc &    copy   (XSUPSAT, XNACT, XNPRO, XSSPRO,                                                &
-!$acc &            XRT, XRRS, XSVT, XRSVS, XCLDFR, XCIT, XINPRR3D, XEVAP3D,                      &
+!$acc &            XRT, XSVT, XCLDFR, XCIT, XINPRR3D, XEVAP3D,                                   &
 !$acc &            XINPRC, XINPRR, XINPRS, XINPRG, XINPRH, XINDEP,                               &
 !$acc &            XHLC_HRC, XHLC_HCF, XHLI_HRI, XHLI_HCF)                                       &
 !$acc &    copyout(XSRCT, XRAINFR)
 
-!$acc update device ( XRTHS )
-  
   IF (CSURF=='EXTE') THEN
 #ifndef MNH_OPENACC
     ALLOCATE (ZSEA(SIZE(XRHODJ,1),SIZE(XRHODJ,2)))
@@ -2026,11 +2028,13 @@ IF (CCLOUD /= 'NONE' .AND. CELEC == 'NONE') THEN
   END IF
 !$acc end data
 
-!$acc update host(XRTHS)
-  
+!$acc kernels
   XRTHS_CLD(:, :, : )    = XRTHS(:, :, : )    - XRTHS_CLD(:, :, : )
   XRRS_CLD (:, :, :, : ) = XRRS (:, :, :, : ) - XRRS_CLD (:, :, :, : )
   XRSVS_CLD(:, :, :, : ) = XRSVS(:, :, :, : ) - XRSVS_CLD(:, :, :, : )
+!$acc end kernels
+!
+!$acc update host( XRTHS, XRRS, XRSVS, XRTHS_CLD, XRRS_CLD, XRSVS_CLD )
 !
   IF (CCLOUD /= 'REVE' ) THEN
     XACPRR = XACPRR + XINPRR * XTSTEP