diff --git a/src/ZSOLVER/rain_ice_red.f90 b/src/ZSOLVER/rain_ice_red.f90
index 3aeb5a1335d50b469af71689f3e57fb2b2e8e8a8..a5f8dcd59e603326ad4f89b0cbc21e3cfffd20bd 100644
--- a/src/ZSOLVER/rain_ice_red.f90
+++ b/src/ZSOLVER/rain_ice_red.f90
@@ -690,13 +690,6 @@ REAL, DIMENSION(:,:,:), pointer, contiguous :: ZTEMP_BUD
!
LOGICAL :: GTEST ! temporary variable for OpenACC character limitation (Cray CCE)
-!$acc data present( ODMICRO, PEXN, PDZZ, PRHODJ, PRHODREF, PEXNREF, PPABST, PCIT, PCLDFR, &
-!$acc & PHLC_HRC, PTHT, PRVT, &
-!$acc & PRCT, PHLC_HCF, PHLI_HRI, PHLI_HCF, PRRT, PRIT, PRST, PRGT, PSIGS, &
-!$acc & PTHS, PRVS, PRCS, PRRS, PRIS, PRSS, PRGS, &
-!$acc & PINPRC, PINDEP, PINPRR, PEVAP3D, PINPRS, PINPRG, PRAINFR, &
-!$acc & PSEA, PTOWN, PRHT, PRHS, PINPRH, PFPR )
-
IF (MPPDB_INITIALIZED) THEN
!Check all IN arrays
CALL MPPDB_CHECK(ODMICRO,"RAIN_ICE_RED beg:ODMICRO")
@@ -735,6 +728,13 @@ IF (MPPDB_INITIALIZED) THEN
IF (PRESENT(PRHS)) CALL MPPDB_CHECK(PRHS,"RAIN_ICE_RED beg:PRHS")
END IF
+!$acc data present( ODMICRO, PEXN, PDZZ, PRHODJ, PRHODREF, PEXNREF, PPABST, PCIT, PCLDFR, &
+!$acc & PHLC_HRC, PTHT, PRVT, &
+!$acc & PRCT, PHLC_HCF, PHLI_HRI, PHLI_HCF, PRRT, PRIT, PRST, PRGT, PSIGS, &
+!$acc & PTHS, PRVS, PRCS, PRRS, PRIS, PRSS, PRGS, &
+!$acc & PINPRC, PINDEP, PINPRR, PEVAP3D, PINPRS, PINPRG, PRAINFR, &
+!$acc & PSEA, PTOWN, PRHT, PRHS, PINPRH, PFPR )
+
!$acc kernels
imicro = count(odmicro)
!$acc end kernels
@@ -1697,8 +1697,9 @@ DO WHILE(ANY(ZTIME(:)<PTSTEP)) ! Loop to *really* compute tendencies
&ZHLC_HCF, ZHLC_LCF, ZHLC_HRC, ZHLC_LRC, &
&ZHLI_HCF, ZHLI_LCF, ZHLI_HRI, ZHLI_LRI, PRAINFR)
! External tendencies
-!$acc kernels
+!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
IF(GEXT_TEND) THEN
+!$acc kernels
!$acc loop independent
DO JL=1, IMICRO
ZA_TH(JL) = ZA_TH(JL) + ZEXT_TH(JL)
@@ -1710,7 +1711,9 @@ DO WHILE(ANY(ZTIME(:)<PTSTEP)) ! Loop to *really* compute tendencies
ZA_RG(JL) = ZA_RG(JL) + ZEXT_RG(JL)
ZA_RH(JL) = ZA_RH(JL) + ZEXT_RH(JL)
ENDDO
- ENDIF
+!$acc end kernels
+ ENDIF
+!$acc kernels
!
!*** 4.2 Integration time
!
@@ -1735,6 +1738,8 @@ DO WHILE(ANY(ZTIME(:)<PTSTEP)) ! Loop to *really* compute tendencies
ZW1D(JL) * MIN(ZMAXTIME(JL), ZTIME_THRESHOLD(JL))
ENDDO
ENDIF
+!$acc end kernels
+!$acc kernels
!We need to adjust tendencies when a specy disappears
!When a species is missing, only the external tendencies can be negative (and we must keep track of it)
!$acc loop independent
@@ -1769,7 +1774,8 @@ DO WHILE(ANY(ZTIME(:)<PTSTEP)) ! Loop to *really* compute tendencies
ZMAXTIME(JL)=(1.-ZW1D(JL)) * ZMAXTIME(JL) + &
&ZW1D(JL) * MIN(ZMAXTIME(JL), -(ZB_RG(JL)+ZRGT(JL))/MIN(ZA_RG(JL), -1.E-20))
ENDDO
-
+!$acc end kernels
+!$acc kernels
IF(KRR==7) THEN
!$acc loop independent
DO JL=1, IMICRO
@@ -1778,7 +1784,9 @@ DO WHILE(ANY(ZTIME(:)<PTSTEP)) ! Loop to *really* compute tendencies
ZMAXTIME(JL)=(1.-ZW1D(JL)) * ZMAXTIME(JL) + &
&ZW1D(JL) * MIN(ZMAXTIME(JL), -(ZB_RH(JL)+ZRHT(JL))/MIN(ZA_RH(JL), -1.E-20))
ENDDO
- ENDIF
+ ENDIF
+!$acc end kernels
+!$acc kernels
!We stop when the end of the timestep is reached
ZCOMPUTE(:)=ZCOMPUTE(:) * MAX(0., -SIGN(1., ZTIME(:)+ZMAXTIME(:)-PTSTEP))
@@ -1796,7 +1804,10 @@ DO WHILE(ANY(ZTIME(:)<PTSTEP)) ! Loop to *really* compute tendencies
!We must recompute tendencies when the maximum allowed change is reached
!When a specy is missing, only the external tendencies can be active and we do not want to recompute
!the microphysical tendencies when external tendencies are negative (results won't change because specy was already missing)
+!$acc end kernels
+
IF(XMRSTEP/=0.) THEN
+!$acc kernels
!$acc loop independent
DO JL=1, IMICRO
ZW1D(JL)=MAX(0., -SIGN(1., IITER(JL)-INB_ITER_MAX+0.)) * & ! WHERE(IITER(:)<INB_ITER_MAX)
@@ -1877,7 +1888,8 @@ DO WHILE(ANY(ZTIME(:)<PTSTEP)) ! Loop to *really* compute tendencies
&ZW1D(JL)*MIN(ZMAXTIME(JL), ZTIME_THRESHOLD(JL))
ZCOMPUTE(JL)=ZCOMPUTE(JL) * (1. - ZW1D(JL))
ENDDO
-
+!$acc end kernels
+!$acc kernels
IF(KRR==7) THEN
!$acc loop independent
DO JL=1, IMICRO
@@ -1895,7 +1907,8 @@ DO WHILE(ANY(ZTIME(:)<PTSTEP)) ! Loop to *really* compute tendencies
ZCOMPUTE(JL)=ZCOMPUTE(JL) * (1. - ZW1D(JL))
ENDDO
ENDIF
-
+!$acc end kernels
+!$acc kernels
!$acc loop independent
DO JL=1, IMICRO
ZW1D(JL)=MAX(ABS(ZB_RV(JL)), ABS(ZB_RC(JL)), ABS(ZB_RR(JL)), ABS(ZB_RI(JL)), &
@@ -1905,10 +1918,12 @@ DO WHILE(ANY(ZTIME(:)<PTSTEP)) ! Loop to *really* compute tendencies
ZMAXTIME(JL)=(1.-ZW1D(JL))*ZMAXTIME(JL)
ZCOMPUTE(JL)=ZCOMPUTE(JL) * (1. - ZW1D(JL))
ENDDO
- ENDIF
+!$acc end kernels
+ ENDIF
!
!*** 4.3 New values of variables for next iteration
!
+!$acc kernels
!$acc loop independent
DO JL=1, IMICRO
ZTHT(JL)=ZTHT(JL)+ZA_TH(JL)*ZMAXTIME(JL)+ZB_TH(JL)
@@ -1994,7 +2009,7 @@ ENDDO
!* 5. UNPACKING DIAGNOSTICS
! ---------------------
!
-! !$acc kernels
+! ! acc kernels
IF(IMICRO>0) THEN
!$acc kernels present_cr(ZHLC_HCF3D,ZHLC_LCF3D,ZHLC_HRC3D,ZHLC_LRC3D,ZHLI_HCF3D,ZHLI_LCF3D,ZHLI_HRI3D,ZHLI_LRI3D)
ZHLC_HCF3D(:,:,:)=0.
@@ -2052,20 +2067,16 @@ CALL ICE4_NUCLEATION_WRAPPER(KIT, KJT, KKT, GDNOTMICRO, &
PTHT, PPABST, PRHODREF, PEXN, ZLSFACT3D, ZT, &
PRVT, &
PCIT, ZZ_RVHENI_MR)
-!$acc kernels
-!$acc loop independent collapse(3)
-DO JK = 1, KKT
- DO JJ = 1, KJT
- DO JI = 1, KIT
- ZZ_LSFACT(JI,JJ,JK)=ZZ_LSFACT(JI,JJ,JK)/PEXNREF(JI,JJ,JK)
- ZZ_LVFACT(JI,JJ,JK)=ZZ_LVFACT(JI,JJ,JK)/PEXNREF(JI,JJ,JK)
- ZZ_RVHENI(JI,JJ,JK) = MIN(PRVS(JI,JJ,JK), ZZ_RVHENI_MR(JI,JJ,JK)/PTSTEP)
- PRIS(JI,JJ,JK)=PRIS(JI,JJ,JK)+ZZ_RVHENI(JI,JJ,JK)
- PRVS(JI,JJ,JK)=PRVS(JI,JJ,JK)-ZZ_RVHENI(JI,JJ,JK)
- PTHS(JI,JJ,JK)=PTHS(JI,JJ,JK) + ZZ_RVHENI(JI,JJ,JK)*ZZ_LSFACT(JI,JJ,JK)
- ENDDO
- ENDDO
-ENDDO
+
+!$acc kernels present_cr( PEXNREF, PRIS, PRVS, PTHS, ZZ_LSFACT, ZZ_LVFACT, ZZ_RVHENI )
+!$mnh_do_concurrent( JI = 1 : KIT, JJ = 1 : KJT, JK = 1 : KKT )
+ ZZ_LSFACT(JI,JJ,JK)=ZZ_LSFACT(JI,JJ,JK)/PEXNREF(JI,JJ,JK)
+ ZZ_LVFACT(JI,JJ,JK)=ZZ_LVFACT(JI,JJ,JK)/PEXNREF(JI,JJ,JK)
+ ZZ_RVHENI(JI,JJ,JK) = MIN(PRVS(JI,JJ,JK), ZZ_RVHENI_MR(JI,JJ,JK)/PTSTEP)
+ PRIS(JI,JJ,JK)=PRIS(JI,JJ,JK)+ZZ_RVHENI(JI,JJ,JK)
+ PRVS(JI,JJ,JK)=PRVS(JI,JJ,JK)-ZZ_RVHENI(JI,JJ,JK)
+ PTHS(JI,JJ,JK)=PTHS(JI,JJ,JK) + ZZ_RVHENI(JI,JJ,JK)*ZZ_LSFACT(JI,JJ,JK)
+!$mnh_end_do()
!$acc end kernels
!$acc update self(PRIS,PRVS,PTHS)
!
@@ -2104,7 +2115,7 @@ IF(GEXT_TEND) THEN
!$acc kernels
!Z..T variables contain the exeternal tendency, we substract it
!$acc loop independent
- DO CONCURRENT ( JL = 1 : IMICRO )
+ !$mnh_do_concurrent ( JL = 1 : IMICRO )
ZRVT(JL) = ZRVT(JL) - ZEXT_RV(JL) * PTSTEP
ZRCT(JL) = ZRCT(JL) - ZEXT_RC(JL) * PTSTEP
ZRRT(JL) = ZRRT(JL) - ZEXT_RR(JL) * PTSTEP
@@ -2112,14 +2123,14 @@ IF(GEXT_TEND) THEN
ZRST(JL) = ZRST(JL) - ZEXT_RS(JL) * PTSTEP
ZRGT(JL) = ZRGT(JL) - ZEXT_RG(JL) * PTSTEP
ZTHT(JL) = ZTHT(JL) - ZEXT_TH(JL) * PTSTEP
- END DO
+ !$mnh_end_do()
!$acc end kernels
IF (KRR==7) THEN
!$acc kernels
!$acc loop independent
- DO CONCURRENT ( JL = 1 : IMICRO )
+ !$mnh_do_concurrent ( JL = 1 : IMICRO )
ZRHT(JL) = ZRHT(JL) - ZEXT_RH(JL) * PTSTEP
- END DO
+ !$mnh_end_do()
!$acc end kernels
END IF
END IF
@@ -2181,14 +2192,14 @@ IF (KRR==7) THEN
ELSE
!PW: BUG: this should work...
-! !$acc kernels
+! ! acc kernels
! ZW_RVS(JI,JJ,JK) = 0.
! ZW_RCS(JI,JJ,JK) = 0.
! ZW_RRS(JI,JJ,JK) = 0.
! ZW_RIS(JI,JJ,JK) = 0.
! ZW_RSS(JI,JJ,JK) = 0.
! ZW_RGS(JI,JJ,JK) = 0.
-! !$acc loop independent
+! ! acc loop independent
! DO JL=1,IMICRO
! ZW_RVS(I1(JL), I2(JL), I3(JL)) = ( ZRVT(JL) - PRVT(I1(JL), I2(JL), I3(JL)) ) * ZINV_TSTEP
! ZW_RCS(I1(JL), I2(JL), I3(JL)) = ( ZRCT(JL) - PRCT(I1(JL), I2(JL), I3(JL)) ) * ZINV_TSTEP
@@ -2197,7 +2208,7 @@ ELSE
! ZW_RSS(I1(JL), I2(JL), I3(JL)) = ( ZRST(JL) - PRST(I1(JL), I2(JL), I3(JL)) ) * ZINV_TSTEP
! ZW_RGS(I1(JL), I2(JL), I3(JL)) = ( ZRGT(JL) - PRGT(I1(JL), I2(JL), I3(JL)) ) * ZINV_TSTEP
! END DO
-! !$acc end kernels
+! ! acc end kernels
#if 0
!$acc kernels
@@ -2767,6 +2778,8 @@ ENDIF
!$acc end data
+!$acc end data
+
#ifdef MNH_OPENACC
!Release all memory allocated with MNH_MEM_GET calls since last call to MNH_MEM_POSITION_PIN
CALL MNH_MEM_RELEASE()
@@ -2794,8 +2807,6 @@ IF (MPPDB_INITIALIZED) THEN
IF (PRESENT(PFPR)) CALL MPPDB_CHECK(PFPR, "RAIN_ICE_RED end:PFPR")
END IF
-!$acc end data
-
CONTAINS
!
SUBROUTINE CORRECT_NEGATIVITIES(KIT, KJT, KKT, KRR, PRV, PRC, PRR, &