diff --git a/src/MNH/advecuvw_rk.f90 b/src/MNH/advecuvw_rk.f90
index 529e14d218a17e0528f27cf04ff73696450e57fa..bcb889933f39f9d2bf19b8cf8a03bdd47c89a526 100644
--- a/src/MNH/advecuvw_rk.f90
+++ b/src/MNH/advecuvw_rk.f90
@@ -580,36 +580,43 @@ RKLOOP: DO JS = 1, ISPL
!$acc end kernels
!
IF ( JS < ISPL ) THEN
-!$acc kernels present(ZUT,ZVT,ZWT) present(ZBUT) present(PU,PV,PW) &
-!$acc & present(ZRUS,ZRVS,ZRWS,ZIBM) present(PRUS_OTHER,PRVS_OTHER,PRWS_OTHER) &
-!$acc & present(PMXM_RHODJ,PMYM_RHODJ,PMZM_RHODJ)
+!PW: note: 20211025: kernels split because performance problems if in 1 block with NVHPC 21.9
+! !$acc kernels present(ZUT,ZVT,ZWT) present(ZBUT) present(PU,PV,PW) &
+! !$acc & present(ZRUS,ZRVS,ZRWS,ZIBM) present(PRUS_OTHER,PRVS_OTHER,PRWS_OTHER) &
+! !$acc & present(PMXM_RHODJ,PMYM_RHODJ,PMZM_RHODJ)
!
+!$acc kernels
ZUT(:,:,:) = PU(:,:,:)
ZVT(:,:,:) = PV(:,:,:)
ZWT(:,:,:) = PW(:,:,:)
+!$acc end kernels
!
DO JI = 1, JS
!
! Intermediate guesses inside the RK loop
!
IF ( .NOT. LIBM ) THEN
+!$acc kernels
ZUT(:,:,:) = ZUT(:,:,:) + ZBUT(JS,JI) * PTSTEP * &
( ZRUS(:,:,:,JI) + PRUS_OTHER(:,:,:) ) / PMXM_RHODJ(:,:,:)
ZVT(:,:,:) = ZVT(:,:,:) + ZBUT(JS,JI) * PTSTEP * &
( ZRVS(:,:,:,JI) + PRVS_OTHER(:,:,:) ) / PMYM_RHODJ(:,:,:)
ZWT(:,:,:) = ZWT(:,:,:) + ZBUT(JS,JI) * PTSTEP * &
( ZRWS(:,:,:,JI) + PRWS_OTHER(:,:,:) ) / PMZM_RHODJ(:,:,:)
+!$acc end kernels
ELSE
+!$acc kernels
ZUT(:,:,:) = ZUT(:,:,:) + ZBUT(JS,JI) * PTSTEP * &
( ZRUS(:,:,:,JI) + PRUS_OTHER(:,:,:) ) / PMXM_RHODJ(:,:,:) * ZIBM(:,:,:,1)
ZVT(:,:,:) = ZVT(:,:,:) + ZBUT(JS,JI) * PTSTEP * &
( ZRVS(:,:,:,JI) + PRVS_OTHER(:,:,:) ) / PMYM_RHODJ(:,:,:) * ZIBM(:,:,:,2)
ZWT(:,:,:) = ZWT(:,:,:) + ZBUT(JS,JI) * PTSTEP * &
( ZRWS(:,:,:,JI) + PRWS_OTHER(:,:,:) ) / PMZM_RHODJ(:,:,:) * ZIBM(:,:,:,3)
+!$acc end kernels
END IF
!
END DO
-!$acc end kernels
+! !$acc end kernels
!$acc update self(ZUT,ZVT,ZWT)
END IF
!
diff --git a/src/MNH/condensation.f90 b/src/MNH/condensation.f90
index bf65c2ae4aa1b835cefdccedeeaa2b0f5394cebf..4216322b05baf248ba8b5649561c920916f93b46 100644
--- a/src/MNH/condensation.f90
+++ b/src/MNH/condensation.f90
@@ -394,8 +394,9 @@ IF (OUSERI) CALL COMPUTE_FRAC_ICE(HFRAC_ICE, ZFRAC, PT)
IF (OUSERI) CALL COMPUTE_FRAC_ICE3D_DEVICE(HFRAC_ICE, ZFRAC, PT)
#endif
!
-!$acc kernels
DO JK=IKTB,IKTE
+!PW: note: 10x faster to put the kernels zone inside the JK loop (NVHPC 21.9)
+!$acc kernels
JKP=MAX(MIN(JK+KKL,IKTE),IKTB)
JKM=MAX(MIN(JK-KKL,IKTE),IKTB)
!$acc loop independent collapse(2)
@@ -593,8 +594,8 @@ DO JK=IKTB,IKTE
END DO
END DO
-END DO
!$acc end kernels
+END DO
!
IF (MPPDB_INITIALIZED) THEN
!Check all INOUT arrays
diff --git a/src/MNH/ice4_fast_rg.f90 b/src/MNH/ice4_fast_rg.f90
index 5e04585a92ed0e0e0c480d81888de05816138ca5..c66c5f85ce474fd2a92a05a69cc30157455f2d66 100644
--- a/src/MNH/ice4_fast_rg.f90
+++ b/src/MNH/ice4_fast_rg.f90
@@ -315,6 +315,7 @@ ELSE
END WHERE
IF(LCRFLIMIT) THEN
+!$acc loop independent
DO JL=1, ISIZE
!Comparison between heat to be released (to freeze rain) and heat sink (rain and ice temperature change)
!ZZW is the proportion of process that can take place
@@ -352,17 +353,19 @@ IF(LDSOFT) THEN
PRG_TEND(JL, IRCDRYG)=ZMASK(JL)*PRG_TEND(JL, IRCDRYG)
ENDDO
ELSE
- ZZW(:)=0.
- WHERE(ZMASK(:)==1.)
+!$acc loop independent
+ DO JL = 1, ISIZE
+ IF ( ZMASK(JL) == 1. ) THEN
#ifndef MNH_BITREP
- ZZW(:)=PLBDAG(:)**(XCXG-XDG-2.) * PRHODREF(:)**(-XCEXVT)
+ ZZW(JL) = PLBDAG(JL)**(XCXG-XDG-2.) * PRHODREF(JL)**(-XCEXVT)
#else
- ZZW(:)=BR_POW(PLBDAG(:),XCXG-XDG-2.) * BR_POW(PRHODREF(:),-XCEXVT)
+ ZZW(JL) = BR_POW( PLBDAG(JL), XCXG - XDG - 2. ) * BR_POW( PRHODREF(JL), -XCEXVT )
#endif
- END WHERE
- DO JL=1, ISIZE
- PRG_TEND(JL, IRCDRYG)=ZMASK(JL)*XFCDRYG * PRCT(JL) * ZZW(JL)
- ENDDO
+ PRG_TEND(JL, IRCDRYG) = XFCDRYG * PRCT(JL) * ZZW(JL)
+ ELSE
+ PRG_TEND(JL, IRCDRYG) = 0.
+ END IF
+ END DO
ENDIF
DO JL=1, ISIZE
@@ -376,19 +379,23 @@ IF(LDSOFT) THEN
PRG_TEND(JL, IRIWETG)=ZMASK(JL) * PRG_TEND(JL, IRIWETG)
ENDDO
ELSE
- PRG_TEND(:, IRIDRYG)=0.
- PRG_TEND(:, IRIWETG)=0.
- WHERE(ZMASK(:)==1.)
+!$acc loop independent
+ DO JL = 1, ISIZE
+ IF ( ZMASK(JL) == 1. ) THEN
#ifndef MNH_BITREP
- ZZW(:)=PLBDAG(:)**(XCXG-XDG-2.) * PRHODREF(:)**(-XCEXVT)
- PRG_TEND(:, IRIDRYG)=XFIDRYG*EXP(XCOLEXIG*(PT(:)-XTT))*PRIT(:)*ZZW(:)
- PRG_TEND(:, IRIWETG)=PRG_TEND(:, IRIDRYG) / (XCOLIG*EXP(XCOLEXIG*(PT(:)-XTT)))
+ ZZW(JL) = PLBDAG(JL)**(XCXG-XDG-2.) * PRHODREF(JL)**(-XCEXVT)
+ PRG_TEND(JL, IRIDRYG) = XFIDRYG * EXP( XCOLEXIG * ( PT(JL) - XTT ) ) * PRIT(JL) * ZZW(JL)
+ PRG_TEND(JL, IRIWETG) = PRG_TEND(JL, IRIDRYG) / ( XCOLIG * EXP( XCOLEXIG * ( PT(JL) - XTT ) ) )
#else
- ZZW(:)=BR_POW(PLBDAG(:),XCXG-XDG-2.) * BR_POW(PRHODREF(:),-XCEXVT)
- PRG_TEND(:, IRIDRYG)=XFIDRYG*BR_EXP(XCOLEXIG*(PT(:)-XTT))*PRIT(:)*ZZW(:)
- PRG_TEND(:, IRIWETG)=PRG_TEND(:, IRIDRYG) / (XCOLIG*BR_EXP(XCOLEXIG*(PT(:)-XTT)))
+ ZZW(JL) = BR_POW( PLBDAG(JL), XCXG - XDG - 2. ) * BR_POW( PRHODREF(JL), -XCEXVT )
+ PRG_TEND(JL, IRIDRYG) = XFIDRYG * BR_EXP( XCOLEXIG * ( PT(JL) - XTT ) ) * PRIT(JL) * ZZW(JL)
+ PRG_TEND(JL, IRIWETG) = PRG_TEND(JL, IRIDRYG) / ( XCOLIG * BR_EXP( XCOLEXIG * ( PT(JL) - XTT ) ) )
#endif
- END WHERE
+ ELSE
+ PRG_TEND(JL, IRIDRYG) = 0.
+ PRG_TEND(JL, IRIWETG) = 0.
+ END IF
+ END DO
ENDIF
@@ -435,11 +442,13 @@ ELSE
PRG_TEND(:, IRSWETG)=0.
!$acc end kernels
+!$acc update self( IGDRY )
IF(IGDRY>0)THEN
!$acc kernels
!
!* 6.2.3 select the (PLBDAG,PLBDAS) couplet
!
+!$acc loop independent
DO JJ = 1, IGDRY
ZVEC1(JJ) = PLBDAG(I1(JJ))
ZVEC2(JJ) = PLBDAS(I1(JJ))
@@ -449,27 +458,31 @@ ELSE
! in the geometrical set of (Lbda_g,Lbda_s) couplet use to
! tabulate the SDRYG-kernel
!
- ZVEC1(1:IGDRY)=MAX(1.00001, MIN(REAL(NDRYLBDAG)-0.00001, &
+!$acc loop independent
+ DO CONCURRENT( JJ = 1 : IGDRY )
+ ZVEC1(JJ) = MAX( 1.00001, MIN( REAL( NDRYLBDAG ) - 0.00001, &
#ifndef MNH_BITREP
- XDRYINTP1G*LOG(ZVEC1(1:IGDRY))+XDRYINTP2G))
+ XDRYINTP1G * LOG( ZVEC1(JJ) ) + XDRYINTP2G ) )
#else
- XDRYINTP1G*BR_LOG(ZVEC1(1:IGDRY))+XDRYINTP2G))
+ XDRYINTP1G * BR_LOG( ZVEC1(JJ) ) + XDRYINTP2G ) )
#endif
- IVEC1(1:IGDRY)=INT(ZVEC1(1:IGDRY) )
- ZVEC1(1:IGDRY)=ZVEC1(1:IGDRY)-REAL(IVEC1(1:IGDRY))
- !
- ZVEC2(1:IGDRY)=MAX(1.00001, MIN( REAL(NDRYLBDAS)-0.00001, &
+ IVEC1(JJ) = INT( ZVEC1(JJ) )
+ ZVEC1(JJ) = ZVEC1(JJ) - REAL( IVEC1(JJ) )
+ !
+ ZVEC2(JJ) = MAX( 1.00001, MIN( REAL( NDRYLBDAS ) - 0.00001, &
#ifndef MNH_BITREP
- XDRYINTP1S*LOG(ZVEC2(1:IGDRY))+XDRYINTP2S))
+ XDRYINTP1S * LOG( ZVEC2(JJ) ) + XDRYINTP2S ) )
#else
- XDRYINTP1S*BR_LOG(ZVEC2(1:IGDRY))+XDRYINTP2S))
+ XDRYINTP1S * BR_LOG( ZVEC2(JJ) ) + XDRYINTP2S ) )
#endif
- IVEC2(1:IGDRY)=INT(ZVEC2(1:IGDRY))
- ZVEC2(1:IGDRY)=ZVEC2(1:IGDRY)-REAL(IVEC2(1:IGDRY))
+ IVEC2(JJ) = INT( ZVEC2(JJ) )
+ ZVEC2(JJ) = ZVEC2(JJ) - REAL( IVEC2(JJ) )
+ END DO
!
!* 6.2.5 perform the bilinear interpolation of the normalized
! SDRYG-kernel
!
+!$acc loop independent
DO JJ=1, IGDRY
ZVEC3(JJ) = ( XKER_SDRYG(IVEC1(JJ)+1,IVEC2(JJ)+1)* ZVEC2(JJ) &
- XKER_SDRYG(IVEC1(JJ)+1,IVEC2(JJ) )*(ZVEC2(JJ) - 1.0) ) &
@@ -479,6 +492,7 @@ ELSE
*(ZVEC1(JJ) - 1.0)
END DO
ZZW(:) = 0.
+!$acc loop independent
DO JJ = 1, IGDRY
ZZW(I1(JJ)) = ZVEC3(JJ)
END DO
@@ -540,12 +554,14 @@ ELSE
!$acc kernels
PRG_TEND(:, IRRDRYG)=0.
!$acc end kernels
+!$acc update self( IGDRY )
!
- IF(IGDRY>0) THEN
!$acc kernels
+ IF(IGDRY>0) THEN
!
!* 6.2.8 select the (PLBDAG,PLBDAR) couplet
!
+!$acc loop independent
DO JJ = 1, IGDRY
ZVEC1(JJ) = PLBDAG(I1(JJ))
ZVEC2(JJ) = PLBDAR(I1(JJ))
@@ -555,27 +571,31 @@ ELSE
! in the geometrical set of (Lbda_g,Lbda_r) couplet use to
! tabulate the RDRYG-kernel
!
- ZVEC1(1:IGDRY)=MAX(1.00001, MIN( REAL(NDRYLBDAG)-0.00001, &
+!$acc loop independent
+ DO JJ = 1, IGDRY
+ ZVEC1(JJ) = MAX( 1.00001, MIN( REAL( NDRYLBDAG ) - 0.00001, &
#ifndef MNH_BITREP
- XDRYINTP1G*LOG(ZVEC1(1:IGDRY))+XDRYINTP2G))
+ XDRYINTP1G * LOG( ZVEC1(JJ) ) + XDRYINTP2G ) )
#else
- XDRYINTP1G*BR_LOG(ZVEC1(1:IGDRY))+XDRYINTP2G))
+ XDRYINTP1G * BR_LOG( ZVEC1(JJ) ) + XDRYINTP2G ) )
#endif
- IVEC1(1:IGDRY)=INT(ZVEC1(1:IGDRY))
- ZVEC1(1:IGDRY)=ZVEC1(1:IGDRY)-REAL(IVEC1(1:IGDRY))
- !
- ZVEC2(1:IGDRY)=MAX(1.00001, MIN( REAL(NDRYLBDAR)-0.00001, &
+ IVEC1(JJ) = INT( ZVEC1(JJ) )
+ ZVEC1(JJ) = ZVEC1(JJ) - REAL( IVEC1(JJ) )
+ !
+ ZVEC2(JJ) = MAX( 1.00001, MIN( REAL( NDRYLBDAR ) - 0.00001, &
#ifndef MNH_BITREP
- XDRYINTP1R*LOG(ZVEC2(1:IGDRY))+XDRYINTP2R))
+ XDRYINTP1R * LOG( ZVEC2(JJ) ) + XDRYINTP2R ) )
#else
- XDRYINTP1R*BR_LOG(ZVEC2(1:IGDRY))+XDRYINTP2R))
+ XDRYINTP1R * BR_LOG( ZVEC2(JJ) ) + XDRYINTP2R ) )
#endif
- IVEC2(1:IGDRY)=INT(ZVEC2(1:IGDRY))
- ZVEC2(1:IGDRY)=ZVEC2(1:IGDRY)-REAL(IVEC2(1:IGDRY))
+ IVEC2(JJ) = INT( ZVEC2(JJ) )
+ ZVEC2(JJ) = ZVEC2(JJ) - REAL( IVEC2(JJ) )
+ END DO
!
!* 6.2.10 perform the bilinear interpolation of the normalized
! RDRYG-kernel
!
+!$acc loop independent
DO JJ=1, IGDRY
ZVEC3(JJ)= ( XKER_RDRYG(IVEC1(JJ)+1,IVEC2(JJ)+1)* ZVEC2(JJ) &
- XKER_RDRYG(IVEC1(JJ)+1,IVEC2(JJ) )*(ZVEC2(JJ) - 1.0) ) &
@@ -585,6 +605,7 @@ ELSE
*(ZVEC1(JJ) - 1.0)
END DO
ZZW(:) = 0.
+!$acc loop independent
DO JJ = 1, IGDRY
ZZW(I1(JJ)) = ZVEC3(JJ)
END DO
@@ -606,8 +627,8 @@ ELSE
XLBRDRYG3/ BR_P2(PLBDAR(:)) )
#endif
END WHERE
-!$acc end kernels
ENDIF
+!$acc end kernels
ENDIF
!$acc kernels
@@ -656,6 +677,7 @@ ELSE
( PRHODREF(:)*(XLMTT-XCL*(XTT-PT(:))) )
END WHERE
ENDIF
+!$acc loop independent
DO JL=1, ISIZE
!We must agregate, at least, the cold species
ZRWETG_INIT(JL)=ZMASK(JL) * MAX(PRG_TEND(JL, IRIWETG)+PRG_TEND(JL, IRSWETG), &
@@ -684,6 +706,7 @@ IF(.NOT. LWETGPOST) THEN
PWETG(JL) = PWETG(JL) * MAX(0., -SIGN(1., PT(JL)-XTT))
ENDDO
ENDIF
+!$acc loop independent
DO JL=1, ISIZE
ZDRYG(JL) = ZMASK(JL) * & !
& MAX(0., -SIGN(1., PT(JL)-XTT)) * & ! WHERE(PT(:)<XTT)
diff --git a/src/MNH/ice4_fast_rs.f90 b/src/MNH/ice4_fast_rs.f90
index 7acd05b48351b1e4f74c635489dfce5f80000efd..e1f78b374be7ddcfded7ac0727bb9f71ca890996 100644
--- a/src/MNH/ice4_fast_rs.f90
+++ b/src/MNH/ice4_fast_rs.f90
@@ -282,6 +282,7 @@ ELSE
( PRHODREF(:)*(XLMTT-XCL*(XTT-PT(:))) )
END WHERE
ENDIF
+!$acc loop independent
DO JL=1, ISIZE
!We must agregate, at least, the cold species
!And we are only interested by the freezing rate of liquid species
@@ -332,7 +333,8 @@ ELSE
! 5.1.1 select the PLBDAS
!
!$acc kernels
- DO JJ = 1, IGRIM
+!$acc loop independent
+ DO CONCURRENT( JJ = 1 : IGRIM )
ZVEC1(JJ) = PLBDAS(I1(JJ))
END DO
!
@@ -361,16 +363,18 @@ ELSE
!
! 5.1.4 riming of the small sized aggregates
!
- WHERE (GRIM(:))
- PRS_TEND(:, IRCRIMSS) = XCRIMSS * ZZW(:) * PRCT(:) & ! RCRIMSS
+ DO CONCURRENT( JJ = 1 : IGRIM )
+ IF ( GRIM(JJ) ) THEN
+ PRS_TEND(JJ, IRCRIMSS) = XCRIMSS * ZZW(JJ) * PRCT(JJ) & ! RCRIMSS
#ifndef MNH_BITREP
- * PLBDAS(:)**XEXCRIMSS &
- * PRHODREF(:)**(-XCEXVT)
+ * PLBDAS(JJ)**XEXCRIMSS &
+ * PRHODREF(JJ)**(-XCEXVT)
#else
- * BR_POW(PLBDAS(:),XEXCRIMSS) &
- * BR_POW(PRHODREF(:),-XCEXVT)
+ * BR_POW(PLBDAS(JJ),XEXCRIMSS) &
+ * BR_POW(PRHODREF(JJ),-XCEXVT)
#endif
- END WHERE
+ END IF
+ END DO
!
! 5.1.5 perform the linear interpolation of the normalized
! "XBS"-moment of the incomplete gamma function (XGAMINC_RIM2) and
@@ -430,6 +434,7 @@ ELSE
ENDIF
!
!$acc kernels copyout(igacc)
+!$acc loop independent
DO JL=1, ISIZE
! More restrictive RIM mask to be used for riming by negative temperature only
ZRIM(JL)=ZRIM(JL) * &
@@ -539,22 +544,25 @@ ELSE
!
! 5.2.4 raindrop accretion on the small sized aggregates
!
- WHERE(GACC(:))
+!$acc loop independent
+ DO JL=1, ISIZE
+ IF ( GACC(JL) ) THEN
#ifndef MNH_BITREP
- ZZW6(:) = & !! coef of RRACCS
- XFRACCSS*( PLBDAS(:)**XCXS )*( PRHODREF(:)**(-XCEXVT-1.) ) &
- *( XLBRACCS1/((PLBDAS(:)**2) ) + &
- XLBRACCS2/( PLBDAS(:) * PLBDAR(:) ) + &
- XLBRACCS3/( (PLBDAR(:)**2)) )/PLBDAR(:)**4
+ ZZW6(JL) = & !! coef of RRACCS
+ XFRACCSS*( PLBDAS(JL)**XCXS )*( PRHODREF(JL)**(-XCEXVT-1.) ) &
+ *( XLBRACCS1/((PLBDAS(JL)**2) ) + &
+ XLBRACCS2/( PLBDAS(JL) * PLBDAR(JL) ) + &
+ XLBRACCS3/( (PLBDAR(JL)**2)) )/PLBDAR(JL)**4
#else
- ZZW6(:) = & !! coef of RRACCS
- XFRACCSS* BR_POW(PLBDAS(:),XCXS) * BR_POW(PRHODREF(:),-XCEXVT-1.) &
- *( XLBRACCS1/BR_P2(PLBDAS(:)) + &
- XLBRACCS2/( PLBDAS(:) * PLBDAR(:) ) + &
- XLBRACCS3/ (BR_P2(PLBDAR(:))) )/BR_P4(PLBDAR(:))
+ ZZW6(JL) = & !! coef of RRACCS
+ XFRACCSS* BR_POW(PLBDAS(JL),XCXS) * BR_POW(PRHODREF(JL),-XCEXVT-1.) &
+ *( XLBRACCS1/BR_P2(PLBDAS(JL)) + &
+ XLBRACCS2/( PLBDAS(JL) * PLBDAR(JL) ) + &
+ XLBRACCS3/ (BR_P2(PLBDAR(JL))) )/BR_P4(PLBDAR(JL))
#endif
- PRS_TEND(:, IRRACCSS) =ZZW(:)*ZZW6(:)
- END WHERE
+ PRS_TEND(JL, IRRACCSS) =ZZW(JL)*ZZW6(JL)
+ END IF
+ END DO
!
! 5.2.4b perform the bilinear interpolation of the normalized
! RACCS-kernel
@@ -613,6 +621,7 @@ ELSE
ENDIF
!$acc kernels
!
+!$acc loop independent
DO JL=1, ISIZE
! More restrictive ACC mask to be used for accretion by negative temperature only
ZACC(JL) = ZACC(JL) * &
diff --git a/src/MNH/ice4_sedimentation_split.f90 b/src/MNH/ice4_sedimentation_split.f90
index 8eb745cb782f20e77a9ae950fba93d3c98a91a1b..25c9ab140a2570488c485080651a8401f09dbdbe 100644
--- a/src/MNH/ice4_sedimentation_split.f90
+++ b/src/MNH/ice4_sedimentation_split.f90
@@ -456,7 +456,8 @@ REAL, DIMENSION(:,:), POINTER, CONTIGUOUS :: ZREMAINT ! R
REAL, DIMENSION(:,:,:), POINTER, CONTIGUOUS :: ZWSED ! Sedimentation fluxes
!$acc data present(PRHODREF,POORHODZ,PDZZ,PPABST,PTHT,PRXT,PRXS,PINPRX,PPRXS, &
-!$acc& PRAY,PLBC,PFSEDC,PCONC3D,PFPR)
+!$acc& PRAY,PLBC,PFSEDC,PCONC3D,PFPR) &
+!$acc& copyin( XRTMIN )
IF (MPPDB_INITIALIZED) THEN
!Check all IN arrays
@@ -525,20 +526,20 @@ ZREMAINT(:,:) = PTSTEP
!$acc update self(ZREMAINT)
!
DO WHILE (ANY(ZREMAINT>0.))
-!$acc kernels
+!$acc kernels copyout( ISEDIM )
ISEDIM = 0
-!acc loop independent private(IDX,JI,JJ,JK)
+!$acc loop independent private(IDX)
DO JK = KKTB,KKTE
-!acc loop independent collapse(2)
+!$acc loop independent collapse(2)
DO JJ = KJB,KJE
DO JI = KIB,KIE
IF( (PRXT (JI,JJ,JK)>XRTMIN(KSPE) .OR. &
PPRXS(JI,JJ,JK)>ZRSMIN(KSPE)) .AND. &
ZREMAINT(JI,JJ)>0. ) THEN
-!acc atomic capture
+!$acc atomic capture
ISEDIM = ISEDIM + 1
IDX = ISEDIM
-!acc end atomic
+!$acc end atomic
I1(IDX) = JI
I2(IDX) = JJ
I3(IDX) = JK
@@ -548,8 +549,6 @@ DO WHILE (ANY(ZREMAINT>0.))
END DO
!$acc end kernels
!
-!PW:BUG/TODO: semble nécessaire!!!
-if (JK==-9999) print *,'PW: ISEDIM=',ISEDIM
!
!
!* 1. Parameters for cloud sedimentation
@@ -666,31 +665,30 @@ if (JK==-9999) print *,'PW: ISEDIM=',ISEDIM
!$acc kernels
ZMAX_TSTEP(:,:) = ZREMAINT(:,:)
!$acc loop independent private(JI,JJ,JK,JL)
- DO JL=1, ISEDIM
+ DO CONCURRENT( JL = 1 : ISEDIM )
JI=I1(JL)
JJ=I2(JL)
JK=I3(JL)
IF(PRXT(JI,JJ,JK)>XRTMIN(KSPE) .AND. ZWSED(JI, JJ, JK)>1.E-20) THEN
-!$acc atomic update
+!acc atomic update
ZMAX_TSTEP(JI, JJ) = MIN(ZMAX_TSTEP(JI, JJ), PMAXCFL * PRHODREF(JI, JJ, JK) * &
PRXT(JI, JJ, JK) * PDZZ(JI, JJ, JK) / ZWSED(JI, JJ, JK))
-!$acc end atomic
+!acc end atomic
ENDIF
ENDDO
ZREMAINT(:,:) = ZREMAINT(:,:) - ZMAX_TSTEP(:,:)
+!$acc end kernels
!PW:BUG PGI: do not set independent (wrong results with PGI 19.10)
!acc loop independent private(ZMRCHANGE)
DO JK = KKTB , KKTE
+!$acc kernels
ZMRCHANGE(:,:) = ZMAX_TSTEP(:,:) * POORHODZ(:,:,JK)*(ZWSED(:,:,JK+KKL)-ZWSED(:,:,JK))
PRXT(:,:,JK) = PRXT(:,:,JK) + ZMRCHANGE(:,:) + PPRXS(:,:,JK) * ZMAX_TSTEP(:,:)
PRXS(:,:,JK) = PRXS(:,:,JK) + ZMRCHANGE(:,:) * ZINVTSTEP
- ENDDO
!$acc end kernels
-!$acc update self(ZWSED,ZMAX_TSTEP,PINPRX)
-!PW: BUG PGI: done on CPU to work around a PGI 19.10 bug
- PINPRX(:,:) = PINPRX(:,:) + ZWSED(:,:,KKB) / XRHOLW * (ZMAX_TSTEP(:,:) * ZINVTSTEP)
-!$acc update device(PINPRX)
+ ENDDO
!$acc kernels
+ PINPRX(:,:) = PINPRX(:,:) + ZWSED(:,:,KKB) / XRHOLW * (ZMAX_TSTEP(:,:) * ZINVTSTEP)
IF (GPRESENT_PFPR) THEN
!$acc loop independent
DO JK = KKTB , KKTE
diff --git a/src/MNH/ice4_tendencies.f90 b/src/MNH/ice4_tendencies.f90
index 5f2fea0d1b29e20e841c21e4a4a75069a4869298..5be0afd9452a17d2353885f08f783aa2de71e66d 100644
--- a/src/MNH/ice4_tendencies.f90
+++ b/src/MNH/ice4_tendencies.f90
@@ -351,9 +351,9 @@ INTEGER :: IZRRT3D, IZRST3D, IZRGT3D, IZRHT3D
!$acc& PRCRIMSS,PRCRIMSG,PRSRIMCG,PRRACCSS,PRRACCSG,PRSACCRG,PRSMLTG,PRCMLTSR,PRICFRRG,PRRCFRIG, &
!$acc& PRICFRR,PRCWETG,PRIWETG,PRRWETG,PRSWETG,PRCDRYG,PRIDRYG,PRRDRYG,PRSDRYG, &
!$acc& PRWETGH,PRWETGH_MR,PRGMLTR,PRCWETH,PRIWETH,PRSWETH,PRGWETH,PRRWETH, &
-!$acc& PRCDRYH,PRIDRYH,PRSDRYH,PRRDRYH,PRGDRYH,PRDRYHG,PRHMLTR,PRCBERI,PRS_TEND,PRG_TEND,PRH_TEND, &
+!$acc& PRCDRYH,PRIDRYH,PRSDRYH,PRRDRYH,PRGDRYH,PRDRYHG,PRHMLTR,PRCBERI,PRS_TEND,PRG_TEND,PRH_TEND,PSSI, &
!$acc& PA_TH,PA_RV,PA_RC,PA_RR,PA_RI,PA_RS,PA_RG,PA_RH,PB_TH,PB_RV,PB_RC,PB_RR,PB_RI,PB_RS,PB_RG,PB_RH, &
-!$acc& PHLC_HCF,PHLC_LCF,PHLC_HRC,PHLC_LRC,PHLI_HCF,PHLI_LCF,PHLI_HRI,PHLI_LRI,PRAINFR)
+!$acc& PHLC_HCF,PHLC_LCF,PHLC_HRC,PHLC_LRC,PHLI_HCF,PHLI_LCF,PHLI_HRI,PHLI_LRI,PRAINFR) copyin(XRTMIN)
IF (MPPDB_INITIALIZED) THEN
!Check all IN arrays
@@ -487,7 +487,8 @@ PB_RS(:)=0.
PB_RG(:)=0.
PB_RH(:)=0.
!
-DO JL=1, ISIZE
+!$acc loop independent
+DO CONCURRENT ( JL = 1 : ISIZE )
ZRVT(JL)=PRVT(JL)
ZRCT(JL)=PRCT(JL)
ZRRT(JL)=PRRT(JL)
@@ -519,7 +520,8 @@ ELSE
ZRVT, &
PCIT, PRVHENI_MR, PB_TH, PB_RV, PB_RI)
!$acc kernels
- DO JL=1, ISIZE
+!$acc loop independent
+ DO CONCURRENT ( JL = 1 : ISIZE )
ZRIT(JL)=ZRIT(JL) + PRVHENI_MR(JL)
ZRVT(JL)=ZRVT(JL) - PRVHENI_MR(JL)
ZTHT(JL)=ZTHT(JL) + PRVHENI_MR(JL)*PLSFACT(JL)
@@ -535,7 +537,8 @@ ELSE
&ZTHT, &
&PRRHONG_MR, PB_TH, PB_RR, PB_RG)
!$acc kernels
- DO JL=1, ISIZE
+!$acc loop independent
+ DO CONCURRENT ( JL = 1 : ISIZE )
ZRGT(JL) = ZRGT(JL) + PRRHONG_MR(JL)
ZRRT(JL) = ZRRT(JL) - PRRHONG_MR(JL)
ZTHT(JL) = ZTHT(JL) + PRRHONG_MR(JL)*(PLSFACT(JL)-PLVFACT(JL)) ! f(L_f*(RRHONG))
@@ -551,7 +554,8 @@ ELSE
&ZTHT, ZRIT, &
&PRIMLTC_MR, PB_TH, PB_RC, PB_RI)
!$acc kernels
- DO JL=1, ISIZE
+!$acc loop independent
+ DO CONCURRENT ( JL = 1 : ISIZE )
ZRCT(JL) = ZRCT(JL) + PRIMLTC_MR(JL)
ZRIT(JL) = ZRIT(JL) - PRIMLTC_MR(JL)
ZTHT(JL) = ZTHT(JL) - PRIMLTC_MR(JL)*(PLSFACT(JL)-PLVFACT(JL)) ! f(L_f*(-RIMLTC))
@@ -563,14 +567,18 @@ ELSE
!
IF(CSNOWRIMING=='OLD ') THEN
!$acc kernels
- ZLBDAS(:)=0.
- WHERE(ZRST(:)>0.)
+!$acc loop independent
+ DO CONCURRENT ( JL = 1 : ISIZE )
+ IF ( ZRST(JL) > 0. ) THEN
#ifndef MNH_BITREP
- ZLBDAS(:) = MIN(XLBDAS_MAX, XLBS*(PRHODREF(:)*MAX(ZRST(:), XRTMIN(5)))**XLBEXS)
+ ZLBDAS(JL) = MIN(XLBDAS_MAX, XLBS*(PRHODREF(JL)*MAX(ZRST(JL), XRTMIN(5)))**XLBEXS)
#else
- ZLBDAS(:) = MIN(XLBDAS_MAX, XLBS*BR_POW(PRHODREF(:)*MAX(ZRST(:), XRTMIN(5)),XLBEXS))
+ ZLBDAS(JL) = MIN(XLBDAS_MAX, XLBS*BR_POW(PRHODREF(JL)*MAX(ZRST(JL), XRTMIN(5)),XLBEXS))
#endif
- END WHERE
+ ELSE
+ ZLBDAS(JL) = 0.
+ END IF
+ END DO
!$acc end kernels
CALL ICE4_RSRIMCG_OLD(ODSOFT, PCOMPUTE==1., &
&PRHODREF, &
@@ -578,7 +586,8 @@ ELSE
&ZT, ZRCT, ZRST, &
&PRSRIMCG_MR, PB_RS, PB_RG)
!$acc kernels
- DO JL=1, ISIZE
+!$acc loop independent
+ DO CONCURRENT ( JL = 1 : ISIZE )
ZRST(JL) = ZRST(JL) - PRSRIMCG_MR(JL)
ZRGT(JL) = ZRGT(JL) + PRSRIMCG_MR(JL)
ENDDO
@@ -595,13 +604,14 @@ ENDIF
IF(ISIZE>0) THEN
IF(.NOT. ODSOFT) THEN
!$acc kernels
+!$acc loop independent
+ DO JL = 1, ISIZE
#ifndef MNH_BITREP
- ZZW(:) = EXP(XALPI-XBETAI/ZT(:)-XGAMI*ALOG(ZT(:)))
+ ZZW(JL) = EXP(XALPI-XBETAI/ZT(JL)-XGAMI*ALOG(ZT(JL)))
#else
- ZZW(:) = BR_EXP(XALPI-XBETAI/ZT(:)-XGAMI*BR_LOG(ZT(:)))
+ ZZW(JL) = BR_EXP(XALPI-XBETAI/ZT(JL)-XGAMI*BR_LOG(ZT(JL)))
#endif
-!$acc loop independent
- DO JL = 1, ISIZE
+
PSSI(JL) = ZRVT(JL)*( PPRES(JL)-ZZW(JL) ) / ( XEPSILO * ZZW(JL) ) - 1.0
! Supersaturation over ice
ZKA(JL) = 2.38E-2 + 0.0071E-2*(ZT(JL)-XTT) ! k_a
@@ -653,13 +663,15 @@ IF(ISIZE>0) THEN
ZRST3D (K1(JL), K2(JL), K3(JL)) = ZRST(JL)
ZRGT3D (K1(JL), K2(JL), K3(JL)) = ZRGT(JL)
END DO
+!$acc end kernels
IF (KRR==7) THEN
+!$acc kernels
!$acc loop independent
DO JL=1,ISIZE
ZRHT3D (K1(JL), K2(JL), K3(JL)) = ZRHT(JL)
- ENDDO
- ENDIF
+ END DO
!$acc end kernels
+ ENDIF
CALL ICE4_RAINFR_VERT(KIB, KIE, KIT, KJB, KJE, KJT, KKB, KKE, KKT, KKL, PRAINFR(:,:,:), ZRRT3D(:,:,:), &
&ZRST3D(:,:,:), ZRGT3D(:,:,:), ZRHT3D(:,:,:))
!$acc kernels
@@ -678,53 +690,84 @@ IF(ISIZE>0) THEN
!* compute the slope parameters
!
!$acc kernels
- ZLBDAS(:)=0.
- WHERE(ZRST(:)>0.)
+!$acc loop independent
+ DO CONCURRENT ( JL = 1 : ISIZE )
+ IF ( ZRST(JL) > 0. ) THEN
#ifndef MNH_BITREP
- ZLBDAS(:) = MIN(XLBDAS_MAX, XLBS*(PRHODREF(:)*MAX(ZRST(:), XRTMIN(5)))**XLBEXS)
+ ZLBDAS(JL) = MIN(XLBDAS_MAX, XLBS*(PRHODREF(JL)*MAX(ZRST(JL), XRTMIN(5)))**XLBEXS)
#else
- ZLBDAS(:) = MIN(XLBDAS_MAX, XLBS*BR_POW(PRHODREF(:)*MAX(ZRST(:), XRTMIN(5)),XLBEXS))
+ ZLBDAS(JL) = MIN(XLBDAS_MAX, XLBS*BR_POW(PRHODREF(JL)*MAX(ZRST(JL), XRTMIN(5)),XLBEXS))
#endif
- END WHERE
- ZLBDAG(:)=0.
- WHERE(ZRGT(:)>0.)
+ ELSE
+ ZLBDAS(JL) = 0.
+ END IF
+ END DO
+!$acc loop independent
+ DO CONCURRENT ( JL = 1 : ISIZE )
+ IF ( ZRGT(JL) > 0. ) THEN
#ifndef MNH_BITREP
- ZLBDAG(:) = XLBG*(PRHODREF(:)*MAX(ZRGT(:), XRTMIN(6)))**XLBEXG
+ ZLBDAG(JL) = XLBG*(PRHODREF(JL)*MAX(ZRGT(JL), XRTMIN(6)))**XLBEXG
#else
- ZLBDAG(:) = XLBG*BR_POW(PRHODREF(:)*MAX(ZRGT(:), XRTMIN(6)),XLBEXG)
+ ZLBDAG(JL) = XLBG*BR_POW(PRHODREF(JL)*MAX(ZRGT(JL), XRTMIN(6)),XLBEXG)
#endif
- END WHERE
+ ELSE
+ ZLBDAG(JL) = 0.
+ END IF
+ END DO
!ZLBDAR will be used when we consider rain diluted over the grid box
- ZLBDAR(:)=0.
- WHERE(ZRRT(:)>0.)
+!$acc loop independent
+ DO CONCURRENT ( JL = 1 : ISIZE )
+ IF ( ZRRT(JL) > 0. ) THEN
#ifndef MNH_BITREP
- ZLBDAR(:) = XLBR*( PRHODREF(:)*MAX( ZRRT(:), XRTMIN(3)))**XLBEXR
+ ZLBDAR(JL) = XLBR*( PRHODREF(JL)*MAX( ZRRT(JL), XRTMIN(3)))**XLBEXR
#else
- ZLBDAR(:) = XLBR*BR_POW(PRHODREF(:)*MAX( ZRRT(:), XRTMIN(3)),XLBEXR)
+ ZLBDAR(JL) = XLBR*BR_POW(PRHODREF(JL)*MAX( ZRRT(JL), XRTMIN(3)),XLBEXR)
#endif
- END WHERE
+ ELSE
+ ZLBDAR(JL) = 0.
+ END IF
+ END DO
+!$acc end kernels
!ZLBDAR_RF is used when we consider rain concentrated in its fraction
IF (HSUBG_RC_RR_ACCR=='PRFR' .OR. HSUBG_RR_EVAP=='PRFR') THEN
- ZLBDAR_RF(:)=0.
- WHERE(ZRRT(:)>0. .AND. ZRF(:)>0.)
+!PW: 20211012: kernels directive inside IF to prevent crash with nvhpc 21.9
+!$acc kernels
+!$acc loop independent
+ DO CONCURRENT( JL = 1 : ISIZE )
+ IF ( ZRRT(JL) >0. .AND. ZRF(JL) > 0. ) THEN
#ifndef MNH_BITREP
- ZLBDAR_RF(:) = XLBR*( PRHODREF(:) *MAX( ZRRT(:)/ZRF(:) , XRTMIN(3)))**XLBEXR
+ ZLBDAR_RF(JL) = XLBR*( PRHODREF(JL) *MAX( ZRRT(JL)/ZRF(JL) , XRTMIN(3)))**XLBEXR
#else
- ZLBDAR_RF(:) = XLBR*BR_POW(PRHODREF(:) *MAX( ZRRT(:)/ZRF(:) , XRTMIN(3)),XLBEXR)
+ ZLBDAR_RF(JL) = XLBR*BR_POW(PRHODREF(JL) *MAX( ZRRT(JL)/ZRF(JL) , XRTMIN(3)),XLBEXR)
#endif
- END WHERE
+ ELSE
+ ZLBDAR_RF(JL) = 0.
+ END IF
+ END DO
+!$acc end kernels
ELSE
- ZLBDAR_RF(:) = ZLBDAR(:)
+!$acc kernels
+! ZLBDAR_RF(:) = ZLBDAR(:)
+!$acc loop independent
+ DO CONCURRENT( JL = 1 : ISIZE )
+ ZLBDAR_RF(JL) = ZLBDAR(JL)
+ END DO
+!$acc end kernels
ENDIF
+!$acc kernels
IF(KRR==7) THEN
- ZLBDAH(:)=0.
- WHERE(PRHT(:)>0.)
+!$acc loop independent
+ DO CONCURRENT ( JL = 1 : ISIZE )
+ IF ( PRHT(JL) > 0. ) THEN
#ifndef MNH_BITREP
- ZLBDAH(:) = XLBH*(PRHODREF(:)*MAX(PRHT(:), XRTMIN(7)))**XLBEXH
+ ZLBDAH(JL) = XLBH*(PRHODREF(JL)*MAX(PRHT(JL), XRTMIN(7)))**XLBEXH
#else
- ZLBDAH(:) = XLBH*BR_POW(PRHODREF(:)*MAX(PRHT(:), XRTMIN(7)),XLBEXH)
+ ZLBDAH(JL) = XLBH*BR_POW(PRHODREF(JL)*MAX(PRHT(JL), XRTMIN(7)),XLBEXH)
#endif
- END WHERE
+ ELSE
+ ZLBDAH(JL) = 0.
+ END IF
+ END DO
ENDIF
!$acc end kernels
ENDIF
@@ -790,7 +833,8 @@ CALL ICE4_FAST_RS(ODSOFT, PCOMPUTE, &
! ------------------------------------------------------
!
!$acc kernels
-DO JL=1, ISIZE
+!$acc loop independent
+DO CONCURRENT ( JL = 1 : ISIZE )
ZRGSI(JL) = PRVDEPG(JL) + PRSMLTG(JL) + PRRACCSG(JL) + &
& PRSACCRG(JL) + PRCRIMSG(JL) + PRSRIMCG(JL)
ZRGSI_MR(JL) = PRRHONG_MR(JL) + PRSRIMCG_MR(JL)