From 0693a08188c2f9b545cc53832b6f6ffbb0127e6f Mon Sep 17 00:00:00 2001
From: Philippe WAUTELET <philippe.wautelet@aero.obs-mip.fr>
Date: Mon, 24 Jan 2022 16:18:52 +0100
Subject: [PATCH] Philippe 24/01/2022: OpenACC: workarounds for problems with
 NVHPC

---
 src/MNH/turb.f90     | 43 +++++++++++++++++++++++++++++++++----------
 src/ZSOLVER/turb.f90 | 43 +++++++++++++++++++++++++++++++++----------
 2 files changed, 66 insertions(+), 20 deletions(-)

diff --git a/src/MNH/turb.f90 b/src/MNH/turb.f90
index 0c23b9dda..ce9dcaf0c 100644
--- a/src/MNH/turb.f90
+++ b/src/MNH/turb.f90
@@ -1084,11 +1084,11 @@ ZTAU33M(:,:) =ZTAU11M(:,:)
 !            ------------------------------------------------------------------
 !
 !
-ZMWTH = 0.     ! w'2th'
-ZMWR  = 0.     ! w'2r'
-ZMTH2 = 0.     ! w'th'2
-ZMR2  = 0.     ! w'r'2
-ZMTHR = 0.     ! w'th'r'
+ZMWTH(:,:,:) = 0.     ! w'2th'
+ZMWR(:,:,:)  = 0.     ! w'2r'
+ZMTH2(:,:,:) = 0.     ! w'th'2
+ZMR2(:,:,:)  = 0.     ! w'r'2
+ZMTHR(:,:,:) = 0.     ! w'th'r'
 !$acc end kernels
 
 IF (HTOM=='TM06') THEN
@@ -1133,11 +1133,11 @@ IF (HTOM=='TM06') THEN
 !$acc end kernels
 ELSE
 !$acc kernels
-  ZFWTH = 0.
-  ZFWR  = 0.
-  ZFTH2 = 0.
-  ZFR2  = 0.
-  ZFTHR = 0.
+  ZFWTH(:,:,:) = 0.
+  ZFWR(:,:,:)  = 0.
+  ZFTH2(:,:,:) = 0.
+  ZFR2(:,:,:)  = 0.
+  ZFTHR(:,:,:) = 0.
 !$acc end kernels
 ENDIF
 !
@@ -2537,10 +2537,33 @@ ELSE
 ENDIF
 !
 IF (LOCEAN) THEN
+#if 0
+!PW: bug: crash with nvhpc 21.11 (OK with 21.9)
   ZWORK2D(:,:)=XG*(XALPHAOC*ZDTHLDZ(:,:,KKB)-XBETAOC*ZDRTDZ(:,:,KKB))
+#else
+!PW: bug: nvhpc 21.11 does not parallelize this loop even with loop independent directive!
+#ifdef MNH_COMPILER_NVHPC
+!$acc loop independent collapse(2)
+#endif
+  DO CONCURRENT( JI = 1 : JIU, JJ = 1 : JJU )
+    ZWORK2D(JI,JJ)=XG*(XALPHAOC*ZDTHLDZ(JI,JJ,KKB)-XBETAOC*ZDRTDZ(JI,JJ,KKB))
+  END DO
+#endif
 ELSE
+#if 0
+!PW: bug: crash with nvhpc 21.11 (OK with 21.9)
   ZWORK2D(:,:)=XG/PTHVREF(:,:,KKB)*                                           &
               (ZETHETA(:,:,KKB)*ZDTHLDZ(:,:,KKB)+ZEMOIST(:,:,KKB)*ZDRTDZ(:,:,KKB))
+#else
+!PW: bug: nvhpc 21.11 does not parallelize this loop even with loop independent directive!
+#ifdef MNH_COMPILER_NVHPC
+!$acc loop independent collapse(2)
+#endif
+  DO CONCURRENT( JI = 1 : JIU, JJ = 1 : JJU )
+    ZWORK2D(JI,JJ)=XG/PTHVREF(JI,JJ,KKB)*                                           &
+              (ZETHETA(JI,JJ,KKB)*ZDTHLDZ(JI,JJ,KKB)+ZEMOIST(JI,JJ,KKB)*ZDRTDZ(JI,JJ,KKB))
+  END DO
+#endif
 END IF
 WHERE(ZWORK2D(:,:)>0.)
   PLM(:,:,KKB)=MAX(XMNH_EPSILON,MIN( PLM(:,:,KKB),                 &
diff --git a/src/ZSOLVER/turb.f90 b/src/ZSOLVER/turb.f90
index 7915e7860..bb9d6007e 100644
--- a/src/ZSOLVER/turb.f90
+++ b/src/ZSOLVER/turb.f90
@@ -1084,11 +1084,11 @@ ZTAU33M(:,:) =ZTAU11M(:,:)
 !            ------------------------------------------------------------------
 !
 !
-ZMWTH = 0.     ! w'2th'
-ZMWR  = 0.     ! w'2r'
-ZMTH2 = 0.     ! w'th'2
-ZMR2  = 0.     ! w'r'2
-ZMTHR = 0.     ! w'th'r'
+ZMWTH(:,:,:) = 0.     ! w'2th'
+ZMWR(:,:,:)  = 0.     ! w'2r'
+ZMTH2(:,:,:) = 0.     ! w'th'2
+ZMR2(:,:,:)  = 0.     ! w'r'2
+ZMTHR(:,:,:) = 0.     ! w'th'r'
 !$acc end kernels
 
 IF (HTOM=='TM06') THEN
@@ -1133,11 +1133,11 @@ IF (HTOM=='TM06') THEN
 !$acc end kernels
 ELSE
 !$acc kernels
-  ZFWTH = 0.
-  ZFWR  = 0.
-  ZFTH2 = 0.
-  ZFR2  = 0.
-  ZFTHR = 0.
+  ZFWTH(:,:,:) = 0.
+  ZFWR(:,:,:)  = 0.
+  ZFTH2(:,:,:) = 0.
+  ZFR2(:,:,:)  = 0.
+  ZFTHR(:,:,:) = 0.
 !$acc end kernels
 ENDIF
 !
@@ -2547,10 +2547,33 @@ ELSE
 ENDIF
 !
 IF (LOCEAN) THEN
+#if 0
+!PW: bug: crash with nvhpc 21.11 (OK with 21.9)
   ZWORK2D(:,:)=XG*(XALPHAOC*ZDTHLDZ(:,:,KKB)-XBETAOC*ZDRTDZ(:,:,KKB))
+#else
+!PW: bug: nvhpc 21.11 does not parallelize this loop even with loop independent directive!
+#ifdef MNH_COMPILER_NVHPC
+!$acc loop independent collapse(2)
+#endif
+  DO CONCURRENT( JI = 1 : JIU, JJ = 1 : JJU )
+    ZWORK2D(JI,JJ)=XG*(XALPHAOC*ZDTHLDZ(JI,JJ,KKB)-XBETAOC*ZDRTDZ(JI,JJ,KKB))
+  END DO
+#endif
 ELSE
+#if 0
+!PW: bug: crash with nvhpc 21.11 (OK with 21.9)
   ZWORK2D(:,:)=XG/PTHVREF(:,:,KKB)*                                           &
               (ZETHETA(:,:,KKB)*ZDTHLDZ(:,:,KKB)+ZEMOIST(:,:,KKB)*ZDRTDZ(:,:,KKB))
+#else
+!PW: bug: nvhpc 21.11 does not parallelize this loop even with loop independent directive!
+#ifdef MNH_COMPILER_NVHPC
+!$acc loop independent collapse(2)
+#endif
+  DO CONCURRENT( JI = 1 : JIU, JJ = 1 : JJU )
+    ZWORK2D(JI,JJ)=XG/PTHVREF(JI,JJ,KKB)*                                           &
+              (ZETHETA(JI,JJ,KKB)*ZDTHLDZ(JI,JJ,KKB)+ZEMOIST(JI,JJ,KKB)*ZDRTDZ(JI,JJ,KKB))
+  END DO
+#endif
 END IF
 WHERE(ZWORK2D(:,:)>0.)
   PLM(:,:,KKB)=MAX(XMNH_EPSILON,MIN( PLM(:,:,KKB),                 &
-- 
GitLab