From ce39d7dcfcceec472bfb2f3feb0cd04f9eb74726 Mon Sep 17 00:00:00 2001
From: Philippe WAUTELET <philippe.wautelet@aero.obs-mip.fr>
Date: Tue, 29 Aug 2023 13:32:16 +0200
Subject: [PATCH] Philippe 29/08/2023: OpenACC: work around for a problem with
 NVHPC 23.7 (not present in older versions)

---
 src/MNH/turb_ver_thermo_flux.f90 | 50 ++++++++++++++++++++++----------
 1 file changed, 35 insertions(+), 15 deletions(-)

diff --git a/src/MNH/turb_ver_thermo_flux.f90 b/src/MNH/turb_ver_thermo_flux.f90
index 3a7e9e213..f29f28866 100644
--- a/src/MNH/turb_ver_thermo_flux.f90
+++ b/src/MNH/turb_ver_thermo_flux.f90
@@ -1102,13 +1102,18 @@ END IF
 #else
 IF ( KRRL >= 1 ) THEN
   IF ( KRRI >= 1 ) THEN
+!Remark: to work around a problem with NVHPC 23.7 (not present in previous NVHPC versions),
+! the computation of ZTMP1_DEVICE and ZTMP2_DEVICE has been split in 2 different kernels
+!$acc kernels
+    !dir$ concurrent
+    ZTMP1_DEVICE(:,:,:) = PRHODJ(:,:,:) * PATHETA(:,:,:) * 2. * PSRCM(:,:,:)
+!$acc end kernels
 #ifdef MNH_COMPILER_CCE
 !$acc kernels present(ZTMP2_DEVICE)
 #else
-!$acc kernels 
+!$acc kernels
 #endif
-    !dir$ concurrent 
-    ZTMP1_DEVICE(:,:,:) = PRHODJ(:,:,:) * PATHETA(:,:,:) * 2. * PSRCM(:,:,:)
+    !dir$ concurrent
     ZTMP2_DEVICE(:,:,:) = ZFLXZ(:,:,:) / PDZZ(:,:,:)
 !$acc end kernels
     CALL DZF_DEVICE( ZTMP2_DEVICE, ZTMP3_DEVICE )
@@ -1117,13 +1122,18 @@ IF ( KRRL >= 1 ) THEN
     PRRS(:,:,:,4) = PRRS(:,:,:,4) - ZTMP1_DEVICE(:,:,:) * ZTMP3_DEVICE * PFRAC_ICE(:,:,:)
 !$acc end kernels
   ELSE
+!Remark: to work around a problem with NVHPC 23.7 (not present in previous NVHPC versions),
+! the computation of ZTMP1_DEVICE and ZTMP2_DEVICE has been split in 2 different kernels
+!$acc kernels
+    !dir$ concurrent
+    ZTMP1_DEVICE(:,:,:) = PRHODJ(:,:,:) * PATHETA(:,:,:) * 2. * PSRCM(:,:,:)
+!$acc end kernels
 #ifdef MNH_COMPILER_CCE
 !$acc kernels present(ZTMP2_DEVICE)
 #else
-!$acc kernels 
+!$acc kernels
 #endif
-    !dir$ concurrent 
-    ZTMP1_DEVICE(:,:,:) = PRHODJ(:,:,:) * PATHETA(:,:,:) * 2. * PSRCM(:,:,:)
+    !dir$ concurrent
     ZTMP2_DEVICE(:,:,:) = ZFLXZ(:,:,:) / PDZZ(:,:,:)
 !$acc end kernels
     CALL DZF_DEVICE( ZTMP2_DEVICE, ZTMP3_DEVICE )
@@ -1596,28 +1606,38 @@ IF (KRR /= 0) THEN
 #else
   IF ( KRRL >= 1 ) THEN
     IF ( KRRI >= 1 ) THEN
+!Remark: to work around a problem with NVHPC 23.7 (not present in previous NVHPC versions),
+! the computation of ZTMP1_DEVICE and ZTMP2_DEVICE has been split in 2 different kernels
+!$acc kernels
+      !dir$ concurrent
+      ZTMP1_DEVICE(:,:,:) = PRHODJ(:,:,:)*PAMOIST(:,:,:)*2.*PSRCM(:,:,:)
+!$acc end kernels
 #ifdef MNH_COMPILER_CCE
-      !$acc kernels present(ZTMP2_DEVICE)
+!$acc kernels present(ZTMP2_DEVICE)
 #else
-      !$acc kernels 
+!$acc kernels
 #endif
-      !dir$ concurrent 
-      ZTMP1_DEVICE(:,:,:) = PRHODJ(:,:,:)*PAMOIST(:,:,:)*2.*PSRCM(:,:,:)
+      !dir$ concurrent
       ZTMP2_DEVICE(:,:,:) = ZFLXZ(:,:,:)/PDZZ(:,:,:)
-      !$acc end kernels
+!$acc end kernels
       CALL DZF_DEVICE( ZTMP2_DEVICE, ZTMP3_DEVICE )
 !$acc kernels
       PRRS(:,:,:,2) = PRRS(:,:,:,2) - ZTMP1_DEVICE(:,:,:) * ZTMP3_DEVICE(:,:,:) * (1.0-PFRAC_ICE(:,:,:))
       PRRS(:,:,:,4) = PRRS(:,:,:,4) - ZTMP1_DEVICE(:,:,:) * ZTMP3_DEVICE(:,:,:) * PFRAC_ICE(:,:,:)
 !$acc end kernels
     ELSE
+!Remark: to work around a problem with NVHPC 23.7 (not present in previous NVHPC versions),
+! the computation of ZTMP1_DEVICE and ZTMP2_DEVICE has been split in 2 different kernels
+!$acc kernels
+      !dir$ concurrent
+      ZTMP1_DEVICE(:,:,:) = PRHODJ(:,:,:)*PAMOIST(:,:,:)*2.*PSRCM(:,:,:)
+!$acc end kernels
 #ifdef MNH_COMPILER_CCE
-      !$acc kernels present(ZTMP2_DEVICE)
+!$acc kernels present(ZTMP2_DEVICE)
 #else
-      !$acc kernels 
+!$acc kernels
 #endif
-      !dir$ concurrent 
-      ZTMP1_DEVICE(:,:,:) = PRHODJ(:,:,:)*PAMOIST(:,:,:)*2.*PSRCM(:,:,:)
+      !dir$ concurrent
       ZTMP2_DEVICE(:,:,:) = ZFLXZ(:,:,:)/PDZZ(:,:,:)
 !$acc end kernels
       CALL DZF_DEVICE( ZTMP2_DEVICE, ZTMP3_DEVICE )
-- 
GitLab