Juan 21/09/2022:ZSOLVER/p_abs.f90/turb.f90 , GPU nvhpc/cray optimization , add...

Juan 21/09/2022:ZSOLVER/p_abs.f90/turb.f90 , GPU nvhpc/cray optimization , add mnh_do_concurrent for nvhpc + mnh_undef(LOOP/OPENACC) for cray

src/ZSOLVER/p_abs.f90

@@ -123,6 +123,10 @@ USE MODE_REPRO_SUM
 #if defined(MNH_BITREP) || defined(MNH_BITREP_OMP)
 USE MODI_BITREP
 #endif
+#ifdef MNH_BITREP_OMP
+!$mnh_undef(LOOP)
+!$mnh_undef(OPENACC)
+#endif
 #ifdef MNH_OPENACC
 USE MODE_MNH_ZWORK,   ONLY: MNH_MEM_GET, MNH_MEM_POSITION_PIN, MNH_MEM_RELEASE
 #endif
@@ -289,8 +293,7 @@ IF ( CEQNSYS=='DUR' .OR. CEQNSYS=='MAE' ) THEN
     !
     !$acc loop seq
     DO JK = IKB,IKE
-       !$acc_nv loop independent collapse(2)
-       DO CONCURRENT (JI = IIB:IIE , JJ = IJB:IJE )
+       !$mnh_do_concurrent(JI = IIB:IIE , JJ = IJB:IJE )
            ZMASSGUESS_2D(JI,JJ)  = ZMASSGUESS_2D(JI,JJ) +        &
 #if !defined(MNH_BITREP) && !defined(MNH_BITREP_OMP)
                 (PEXNREF(JI,JJ,JK)+PPHIT(JI,JJ,JK))**ZCVD_O_RD   &
@@ -301,7 +304,7 @@ IF ( CEQNSYS=='DUR' .OR. CEQNSYS=='MAE' ) THEN
           ZMASS_O_PI_2D(JI,JJ)  = ZMASS_O_PI_2D(JI,JJ) + ZWORK(JI,JJ,JK) / PTHETAV(JI,JJ,JK)
           ZWATERMASST_2D(JI,JJ) = ZWATERMASST_2D(JI,JJ) +       &
             ZRTOT(JI,JJ,JK) * ZWORK(JI,JJ,JK) * PRHODREF(JI,JJ,JK)
-       END DO
+       !$mnh_end_do()
     END DO
     !$acc end kernels
     !
@@ -348,8 +351,7 @@ IF ( CEQNSYS=='DUR' .OR. CEQNSYS=='MAE' ) THEN
      !$acc kernels
      !$acc loop seq 
      DO JK = IKB,IKE
-        !$acc_nv loop independent collapse(2)
-        DO CONCURRENT (JI = IIB:IIE , JJ = IJB:IJE )
+        !$mnh_do_concurrent(JI = IIB:IIE , JJ = IJB:IJE )
               ZMASSGUESS_2D(JI,JJ)  = ZMASSGUESS_2D(JI,JJ) +               &
 #if !defined(MNH_BITREP) && !defined(MNH_BITREP_OMP)                   
                    (PEXNREF(JI,JJ,JK)+PPHIT(JI,JJ,JK))**ZCVD_O_RD          &
@@ -357,7 +359,7 @@ IF ( CEQNSYS=='DUR' .OR. CEQNSYS=='MAE' ) THEN
                    BR_POW((PEXNREF(JI,JJ,JK)+PPHIT(JI,JJ,JK)),ZCVD_O_RD) &
 #endif
                    * ZWORK(JI,JJ,JK) / PTHETAV(JI,JJ,JK)
-        END DO
+        !$mnh_end_do()
      END DO
      !$acc end kernels
   ELSE

src/ZSOLVER/turb.f90

@@ -14,6 +14,10 @@ module mode_turb
 #if defined(MNH_BITREP) || defined(MNH_BITREP_OMP)
 use modi_bitrep
 #endif
+#ifdef MNH_BITREP_OMP
+!$mnh_undef(LOOP)
+!$mnh_undef(OPENACC)  
+#endif
 
 implicit none
 
@@ -710,7 +714,7 @@ GTURBLEN_BL89_TURBLEN_RM17_TURBLEN_ADAP_ORMC01 = &
      HTURBLEN=='BL89' .OR. HTURBLEN=='RM17' .OR. HTURBLEN == 'ADAP' .OR. ORMC01
 !
 !$acc update device(PTHLT,PRT)
-!$acc kernels present_cr(ZCOEF_DISS,ZTHLM,ZRM)
+!$acc kernels present_cr(ZCOEF_DISS,ZTHLM,ZRM,zcp)
 !Copy data into ZTHLM and ZRM only if needed
 IF (GTURBLEN_BL89_TURBLEN_RM17_TURBLEN_ADAP_ORMC01) THEN
    DO CONCURRENT(JI=1:JIU,JJ=1:JJU,JK=1:JKU) 
@@ -1009,11 +1013,13 @@ IF (ORMC01) THEN
 #ifdef MNH_OPENACC
   call Print_msg( NVERB_FATAL, 'GEN', 'TURB', 'OpenACC: ORMC01 not yet implemented' )
 #endif
+!$mnh_expand_array(JI=1:JIU,JJ=1:JJU)
 #if !defined(MNH_BITREP) && !defined(MNH_BITREP_OMP)
   ZUSTAR(:,:) = (PSFU(:,:)**2+PSFV(:,:)**2)**(0.25)
 #else
   ZUSTAR(:,:) = BR_POW( BR_P2( PSFU(:,:) ) + BR_P2( PSFV(:,:) ), 0.25 )
 #endif
+!$mnh_end_expand_array()
   IF (KRR>0) THEN
     ZLMO(:,:) = LMO( ZUSTAR(:,:), ZTHLM(:,:,IKB), ZRM(:,:,IKB,1), PSFTH(:,:), PSFRV(:,:) )
   ELSE
@@ -2323,6 +2329,7 @@ IF (.NOT. ORMC01) THEN
   DO JJ=1,JJU
     DO JI=1,JIU
       IF (GOCEAN) THEN
+        !$acc loop seq
         DO JK=KKTE,KKTB,-1
           ZD=ZALPHA*(PZZ(JI,JJ,KKTE+1)-PZZ(JI,JJ,JK))
           IF ( PLM(JI,JJ,JK)>ZD) THEN
@@ -2332,6 +2339,7 @@ IF (.NOT. ORMC01) THEN
           ENDIF
        END DO
       ELSE
+        !$acc loop seq
         DO JK=KKTB,KKTE
           ZD = ZALPHA * ( 0.5 * ( PZZ(JI, JJ, JK) + PZZ(JI, JJ, JK+KKL) ) - PZZ(JI, JJ, KKB) ) * PDIRCOSZW(JI, JJ)
           IF ( PLM(JI,JJ,JK) > ZD ) THEN
@@ -2678,6 +2686,7 @@ IF (.NOT. ORMC01) THEN
   DO JJ=1,SIZE(PLM,2)
     DO JI=1,SIZE(PLM,1)
       IF (GOCEAN) THEN
+        !$acc loop seq
         DO JK=KKTE,KKTB,-1
           ZD=ZALPHA*(PZZ(JI,JJ,KKTE+1)-PZZ(JI,JJ,JK))
           IF ( PLM(JI,JJ,JK)>ZD) THEN
@@ -2687,6 +2696,7 @@ IF (.NOT. ORMC01) THEN
           ENDIF
         END DO
       ELSE
+        !$acc loop seq
         DO JK=KKTB,KKTE
           ZD=ZALPHA*(0.5*(PZZ(JI,JJ,JK)+PZZ(JI,JJ,JK+KKL))-PZZ(JI,JJ,KKB)) &
             *PDIRCOSZW(JI,JJ)