Juan 16/12/2022:MNH/ice4*, for atomic add "acc COPY()" clause for probleme with CCE/15.0.0

src/MNH/ice4_fast_rg.f90

@@ -404,7 +404,7 @@ ENDIF
 ! Wet and dry collection of rs on graupel (6.2.1)
 IGDRY = 0
 !$acc end kernels
-!$acc parallel loop private(idx) independent
+!$acc parallel loop private(idx) copy(IGDRY) independent
 DO CONCURRENT( JJ = 1 : SIZE( GDRY ) )
   ZDRY(JJ)=MAX(0., -SIGN(1., XRTMIN(5)-PRST(JJ))) * & ! WHERE(PRST(:)>XRTMIN(5))
           &MAX(0., -SIGN(1., XRTMIN(6)-PRGT(JJ))) * & ! WHERE(PRGT(:)>XRTMIN(6))
@@ -518,7 +518,7 @@ ENDIF
 !*       6.2.6  accretion of raindrops on the graupeln
 !
 IGDRY = 0
-!$acc parallel loop private(idx) independent
+!$acc parallel loop private(idx) copy(IGDRY) independent
 DO CONCURRENT( JJ = 1 : SIZE( GDRY ) )
   ZDRY(JJ)=MAX(0., -SIGN(1., XRTMIN(3)-PRRT(JJ))) * & ! WHERE(PRRT(:)>XRTMIN(3))
           &MAX(0., -SIGN(1., XRTMIN(6)-PRGT(JJ))) * & ! WHERE(PRGT(:)>XRTMIN(6))

src/MNH/ice4_fast_rh.f90

@@ -318,11 +318,12 @@ ELSE
 #endif
   END WHERE
 ENDIF
+!$acc end kernels
 !
 !*       7.2.1  accretion of aggregates on the hailstones
 !
 IGWET = 0
-!$acc loop private(IDX) independent
+!$acc parallel loop private(IDX) copy(IGWET) independent
 DO JJ = 1, SIZE(GWET)
   ZWET(JJ) = MAX(0., -SIGN(1., XRTMIN(7)-PRHT(JJ))) * & ! WHERE(PRHT(:)>XRTMIN(7))
             &MAX(0., -SIGN(1., XRTMIN(5)-PRST(JJ))) * & ! WHERE(PRST(:)>XRTMIN(5))
@@ -338,7 +339,7 @@ DO JJ = 1, SIZE(GWET)
     GWET(JJ) = .FALSE.
   END IF
 END DO
-!$acc end kernels
+! acc end parallel loop 
 
 IF(LDSOFT) THEN
 !$acc kernels
@@ -353,7 +354,7 @@ ELSE
   PRH_TEND(:, IRSDRYH)=0.
 !$acc end kernels
   IF(IGWET>0)THEN
-!$acc kernels
+!$acc kernels present_cr(ZVEC1,ZVEC2,ZVEC3,ZZW)
     !
     !*       7.2.3  select the (PLBDAH,PLBDAS) couplet
     !
@@ -396,6 +397,7 @@ ELSE
                                                          * (ZVEC1(JJ) - 1.0)
     END DO
     ZZW(:) = 0.
+    !$acc loop independent
     DO JJ = 1, IGWET
       ZZW(I1(JJ)) = ZVEC3(JJ)
     END DO
@@ -425,9 +427,8 @@ ENDIF
 !
 !*       7.2.6  accretion of graupeln on the hailstones
 !
-!$acc kernels
 IGWET = 0
-!$acc loop private(IDX) independent
+!$acc parallel loop private(IDX) copy(IGWET) independent
 DO JJ = 1, SIZE(GWET)
   ZWET(JJ)=MAX(0., -SIGN(1., XRTMIN(7)-PRHT(JJ))) * & ! WHERE(PRHT(:)>XRTMIN(7))
           &MAX(0., -SIGN(1., XRTMIN(6)-PRGT(JJ))) * & ! WHERE(PRGT(:)>XRTMIN(6))
@@ -443,7 +444,7 @@ DO JJ = 1, SIZE(GWET)
     GWET(JJ) = .FALSE.
   END IF
 END DO
-!$acc end kernels
+! acc end parallel loop
 
 IF(LDSOFT) THEN
 !$acc kernels
@@ -458,7 +459,7 @@ ELSE
   PRH_TEND(:, IRGDRYH)=0.
 !$acc end kernels
   IF(IGWET>0)THEN
-!$acc kernels
+!$acc kernels present_cr(ZVEC1,ZVEC2,ZVEC3,ZZW)
     !
     !*       7.2.8  select the (PLBDAH,PLBDAG) couplet
     !
@@ -501,6 +502,7 @@ ELSE
                                                         * (ZVEC1(JJ) - 1.0)
     END DO
     ZZW(:) = 0.
+    !$acc loop independent
     DO JJ = 1, IGWET
       ZZW(I1(JJ)) = ZVEC3(JJ)
     END DO
@@ -534,12 +536,11 @@ ELSE
 !$acc end kernels
   END IF
 ENDIF
-!$acc kernels
 !
 !*       7.2.11  accretion of raindrops on the hailstones
 !
 IGWET = 0
-!$acc loop private(IDX) independent
+!$acc parallel loop private(IDX) copy(IGWET) independent
 DO JJ = 1, SIZE(GWET)
   ZWET(JJ)=MAX(0., -SIGN(1., XRTMIN(7)-PRHT(JJ))) * & ! WHERE(PRHT(:)>XRTMIN(7))
           &MAX(0., -SIGN(1., XRTMIN(3)-PRRT(JJ))) * & ! WHERE(PRRT(:)>XRTMIN(3))
@@ -555,7 +556,7 @@ DO JJ = 1, SIZE(GWET)
     GWET(JJ) = .FALSE.
   END IF
 END DO
-!$acc end kernels
+! acc end parallel loop 
 
 IF(LDSOFT) THEN
 !$acc kernels
@@ -568,7 +569,7 @@ ELSE
   PRH_TEND(:, IRRWETH)=0.
 !$acc end kernels
   IF(IGWET>0)THEN
-!$acc kernels
+!$acc kernels present_cr(ZVEC1,ZVEC2)
     !
     !*       7.2.12  select the (PLBDAH,PLBDAR) couplet
     !
@@ -611,6 +612,7 @@ ELSE
                                                          *(ZVEC1(JJ) - 1.0)
     END DO
     ZZW(:) = 0.
+    !$acc loop independent
     DO JJ = 1, IGWET
       ZZW(I1(JJ)) = ZVEC3(JJ)
     END DO
@@ -683,6 +685,7 @@ ELSE
                           ( PRHODREF(:)*(XLMTT-XCL*(XTT-PT(:))) )
   END WHERE
 ENDIF
+!$acc loop independent
 DO JL=1, ISIZE
   !We must agregate, at least, the cold species
   ZRWETH_INIT(JL)=ZHAIL(JL) * MAX(PRH_TEND(JL, IRIWETH)+PRH_TEND(JL, IRSWETH)+PRH_TEND(JL, IRGWETH), &
@@ -694,16 +697,19 @@ ENDDO
 !*       7.4    Select Wet or Dry case
 !
 !Wet case
+!$acc loop independent
 DO JL=1, ISIZE
   ZWETH(JL) = ZHAIL(JL) * &
             & MAX(0., SIGN(1., MAX(0., ZRDRYH_INIT(JL)-PRH_TEND(JL, IRIDRYH)-PRH_TEND(JL, IRSDRYH)-PRH_TEND(JL, IRGDRYH)) - &
                               &MAX(0., ZRWETH_INIT(JL)-PRH_TEND(JL, IRIWETH)-PRH_TEND(JL, IRSWETH)-PRH_TEND(JL, IRGWETH))))
 ENDDO
 IF(LNULLWETH) THEN
+  !$acc loop independent
   DO JL=1, ISIZE
     ZWETH(JL) = ZWETH(JL) * MAX(0., -SIGN(1., -ZRDRYH_INIT(JL))) ! WHERE(ZRDRYH_INIT(:)>0.)
   ENDDO
 ELSE
+  !$acc loop independent
   DO JL=1, ISIZE
     ZWETH(JL) = ZWETH(JL) * MAX(0., -SIGN(1., -ZRWETH_INIT(JL))) ! WHERE(ZRWETH_INIT(:)>0.)
   ENDDO
@@ -713,6 +719,7 @@ IF(.NOT. LWETHPOST) THEN
     ZWETH(JL) = ZWETH(JL) * MAX(0., -SIGN(1., PT(JL)-XTT)) ! WHERE(PT(:)<XTT)
   ENDDO
 ENDIF
+!$acc loop independent
 DO JL=1, ISIZE
   ZDRYH(JL) = ZHAIL(JL) * &
             & MAX(0., -SIGN(1., PT(JL)-XTT)) * & ! WHERE(PT(:)<XTT)

src/MNH/ice4_fast_rs.f90

@@ -287,12 +287,11 @@ DO JL=1, ISIZE
                                               &PRS_TEND(JL, IFREEZ2) * PRIAGGS(JL)) - &
                                       PRIAGGS(JL))
 ENDDO
-!
+!$acc end kernels
 !*       5.1    cloud droplet riming of the aggregates
 !
 IGRIM = 0
-!$acc end kernels
-!$acc parallel loop private(idx) independent present_cr( I1, GRIM, PCOMPUTE, PRCT, PRST, XRTMIN, ZRIM )
+!$acc parallel loop private(idx) copy(IGRIM) independent present_cr( I1, GRIM, PCOMPUTE, PRCT, PRST, XRTMIN, ZRIM )
 DO CONCURRENT( JJ = 1 : SIZE( GRIM ) )
   ZRIM(JJ)=MAX(0., -SIGN(1., XRTMIN(2)-PRCT(JJ))) * & !WHERE(PRCT(:)>XRTMIN(2))
           &MAX(0., -SIGN(1., XRTMIN(5)-PRST(JJ))) * & !WHERE(PRST(:)>XRTMIN(5))
@@ -461,12 +460,12 @@ DO JL=1, ISIZE
   PA_RG(JL) = PA_RG(JL) + PRCRIMSG(JL)+PRSRIMCG(JL)
   PA_TH(JL) = PA_TH(JL) + PRCRIMSG(JL)*(PLSFACT(JL)-PLVFACT(JL))
 ENDDO
+!$acc end kernels
 !
 !*       5.2    rain accretion onto the aggregates
 !
 IGACC = 0
-!$acc end kernels
-!$acc parallel loop private(idx) independent present_cr( I1, GACC, PCOMPUTE, PRRT, PRST, XRTMIN, ZACC )
+!$acc parallel loop private(idx) copy(IGACC) independent present_cr( I1, GACC, PCOMPUTE, PRRT, PRST, XRTMIN, ZACC )
 DO CONCURRENT( JJ = 1 : SIZE( GACC ) )
   ZACC(JJ)=MAX(0., -SIGN(1., XRTMIN(3)-PRRT(JJ))) * & !WHERE(PRRT(:)>XRTMIN(3))
           &MAX(0., -SIGN(1., XRTMIN(5)-PRST(JJ))) * & !WHERE(PRST(:)>XRTMIN(5))

src/MNH/ice4_rsrimcg_old.f90

@@ -137,27 +137,32 @@ CALL MNH_MEM_GET( zzw,   size( prhodref ) )
 !
 !$acc kernels
 PRSRIMCG_MR(:)=0.
+!$acc end kernels
 !
 IF(.NOT. ODSOFT) THEN
-  IGRIM = 0
-  GRIM(:) = .FALSE.
-!$acc loop private(IDX) independent
+!$acc kernels
+   GRIM(:) = .FALSE.
+!$acc end kernels
+   IGRIM = 0
+!$acc parallel loop private(IDX) copy(IGRIM) independent
   DO JL = 1, SIZE(GRIM)
     IF ( PRCT(JL)>XRTMIN(2) .AND. PRST(JL)>XRTMIN(5) .AND. ODCOMPUTE(JL) .AND. PT(JL)<XTT ) THEN
 !$acc atomic capture
       IGRIM = IGRIM + 1
       IDX = IGRIM
 !$acc end atomic
-      IVEC1(IDX) = Jl
+      IVEC1(IDX) = JL
       GRIM(JL) = .TRUE.
     END IF
   END DO
   !
+! acc end parallel loop
   IF(IGRIM>0) THEN
+    !$acc kernels  
     !
     !        5.1.1  select the PLBDAS
     !
-!$acc loop independent
+    !$acc loop independent
     DO CONCURRENT( JL = 1 : IGRIM )
       ZVEC1(JL) = PLBDAS(IVEC1(JL))
       !
@@ -181,11 +186,11 @@ IF(.NOT. ODSOFT) THEN
       ZVEC1(JL) =  XGAMINC_RIM2( IVEC2(JL)+1 )* ZVEC2(JL)      &
                       - XGAMINC_RIM2( IVEC2(JL)   )*(ZVEC2(JL) - 1.0)
     END DO
-!$acc loop independent
+    !$acc loop independent
     DO CONCURRENT( JL = 1 : size( prhodref ) )
       ZZW(JL) = 0.
     END DO
-!$acc loop independent
+    !$acc loop independent
     DO CONCURRENT( JL = 1 : IGRIM )
       ZZW(IVEC1(JL)) = ZVEC1(JL)
     END DO
@@ -193,7 +198,7 @@ IF(.NOT. ODSOFT) THEN
     !        5.1.6  riming-conversion of the large sized aggregates into graupeln
     !
     !
-!$acc loop independent
+    !$acc loop independent
     DO CONCURRENT( JL = 1 : size( prhodref ) )
       IF ( GRIM(JL) ) THEN
 #ifndef MNH_BITREP
@@ -205,8 +210,10 @@ IF(.NOT. ODSOFT) THEN
         PRSRIMCG_MR(:)=MIN(PRST(:), PRSRIMCG_MR(:))
       END IF
     END DO
-  END IF
-ENDIF
+ !$acc end kernels
+ END IF ! IGRIM
+ENDIF ! ODSOFT
+!$acc kernels
 PB_RS(:) = PB_RS(:) - PRSRIMCG_MR(:)
 PB_RG(:) = PB_RG(:) + PRSRIMCG_MR(:)
 !$acc end kernels

src/MNH/ice4_sedimentation_split.f90

@@ -614,7 +614,7 @@ ZREMAINT(:,:) = PTSTEP
 !
 DO WHILE (ANY(ZREMAINT>0.))
   ISEDIM = 0
-!$acc parallel loop private(idx) independent
+!$acc parallel loop private(idx) copy(ISEDIM) independent
   DO JK = KKTB,KKTE
 !$acc loop independent collapse(2)
     DO JJ = KJB,KJE