From a70adbcaa341c1378ff3c09e7f1ca592040a6b0c Mon Sep 17 00:00:00 2001
From: Juan ESCOBAR <juan.escobar@aero.obs-mip.fr>
Date: Fri, 16 Dec 2022 14:16:04 +0100
Subject: [PATCH] Juan 16/12/2022:MNH/ice4*, for atomic add "acc COPY()" clause
for probleme with CCE/15.0.0
---
src/MNH/ice4_fast_rg.f90 | 4 ++--
src/MNH/ice4_fast_rh.f90 | 29 +++++++++++++++++-----------
src/MNH/ice4_fast_rs.f90 | 9 ++++-----
src/MNH/ice4_rsrimcg_old.f90 | 27 ++++++++++++++++----------
src/MNH/ice4_sedimentation_split.f90 | 2 +-
5 files changed, 42 insertions(+), 29 deletions(-)
diff --git a/src/MNH/ice4_fast_rg.f90 b/src/MNH/ice4_fast_rg.f90
index 8a49ae40f..0e2e92deb 100644
--- a/src/MNH/ice4_fast_rg.f90
+++ b/src/MNH/ice4_fast_rg.f90
@@ -404,7 +404,7 @@ ENDIF
! Wet and dry collection of rs on graupel (6.2.1)
IGDRY = 0
!$acc end kernels
-!$acc parallel loop private(idx) independent
+!$acc parallel loop private(idx) copy(IGDRY) independent
DO CONCURRENT( JJ = 1 : SIZE( GDRY ) )
ZDRY(JJ)=MAX(0., -SIGN(1., XRTMIN(5)-PRST(JJ))) * & ! WHERE(PRST(:)>XRTMIN(5))
&MAX(0., -SIGN(1., XRTMIN(6)-PRGT(JJ))) * & ! WHERE(PRGT(:)>XRTMIN(6))
@@ -518,7 +518,7 @@ ENDIF
!* 6.2.6 accretion of raindrops on the graupeln
!
IGDRY = 0
-!$acc parallel loop private(idx) independent
+!$acc parallel loop private(idx) copy(IGDRY) independent
DO CONCURRENT( JJ = 1 : SIZE( GDRY ) )
ZDRY(JJ)=MAX(0., -SIGN(1., XRTMIN(3)-PRRT(JJ))) * & ! WHERE(PRRT(:)>XRTMIN(3))
&MAX(0., -SIGN(1., XRTMIN(6)-PRGT(JJ))) * & ! WHERE(PRGT(:)>XRTMIN(6))
diff --git a/src/MNH/ice4_fast_rh.f90 b/src/MNH/ice4_fast_rh.f90
index 33ce69daa..04c1d71ee 100644
--- a/src/MNH/ice4_fast_rh.f90
+++ b/src/MNH/ice4_fast_rh.f90
@@ -318,11 +318,12 @@ ELSE
#endif
END WHERE
ENDIF
+!$acc end kernels
!
!* 7.2.1 accretion of aggregates on the hailstones
!
IGWET = 0
-!$acc loop private(IDX) independent
+!$acc parallel loop private(IDX) copy(IGWET) independent
DO JJ = 1, SIZE(GWET)
ZWET(JJ) = MAX(0., -SIGN(1., XRTMIN(7)-PRHT(JJ))) * & ! WHERE(PRHT(:)>XRTMIN(7))
&MAX(0., -SIGN(1., XRTMIN(5)-PRST(JJ))) * & ! WHERE(PRST(:)>XRTMIN(5))
@@ -338,7 +339,7 @@ DO JJ = 1, SIZE(GWET)
GWET(JJ) = .FALSE.
END IF
END DO
-!$acc end kernels
+! acc end parallel loop
IF(LDSOFT) THEN
!$acc kernels
@@ -353,7 +354,7 @@ ELSE
PRH_TEND(:, IRSDRYH)=0.
!$acc end kernels
IF(IGWET>0)THEN
-!$acc kernels
+!$acc kernels present_cr(ZVEC1,ZVEC2,ZVEC3,ZZW)
!
!* 7.2.3 select the (PLBDAH,PLBDAS) couplet
!
@@ -396,6 +397,7 @@ ELSE
* (ZVEC1(JJ) - 1.0)
END DO
ZZW(:) = 0.
+ !$acc loop independent
DO JJ = 1, IGWET
ZZW(I1(JJ)) = ZVEC3(JJ)
END DO
@@ -425,9 +427,8 @@ ENDIF
!
!* 7.2.6 accretion of graupeln on the hailstones
!
-!$acc kernels
IGWET = 0
-!$acc loop private(IDX) independent
+!$acc parallel loop private(IDX) copy(IGWET) independent
DO JJ = 1, SIZE(GWET)
ZWET(JJ)=MAX(0., -SIGN(1., XRTMIN(7)-PRHT(JJ))) * & ! WHERE(PRHT(:)>XRTMIN(7))
&MAX(0., -SIGN(1., XRTMIN(6)-PRGT(JJ))) * & ! WHERE(PRGT(:)>XRTMIN(6))
@@ -443,7 +444,7 @@ DO JJ = 1, SIZE(GWET)
GWET(JJ) = .FALSE.
END IF
END DO
-!$acc end kernels
+! acc end parallel loop
IF(LDSOFT) THEN
!$acc kernels
@@ -458,7 +459,7 @@ ELSE
PRH_TEND(:, IRGDRYH)=0.
!$acc end kernels
IF(IGWET>0)THEN
-!$acc kernels
+!$acc kernels present_cr(ZVEC1,ZVEC2,ZVEC3,ZZW)
!
!* 7.2.8 select the (PLBDAH,PLBDAG) couplet
!
@@ -501,6 +502,7 @@ ELSE
* (ZVEC1(JJ) - 1.0)
END DO
ZZW(:) = 0.
+ !$acc loop independent
DO JJ = 1, IGWET
ZZW(I1(JJ)) = ZVEC3(JJ)
END DO
@@ -534,12 +536,11 @@ ELSE
!$acc end kernels
END IF
ENDIF
-!$acc kernels
!
!* 7.2.11 accretion of raindrops on the hailstones
!
IGWET = 0
-!$acc loop private(IDX) independent
+!$acc parallel loop private(IDX) copy(IGWET) independent
DO JJ = 1, SIZE(GWET)
ZWET(JJ)=MAX(0., -SIGN(1., XRTMIN(7)-PRHT(JJ))) * & ! WHERE(PRHT(:)>XRTMIN(7))
&MAX(0., -SIGN(1., XRTMIN(3)-PRRT(JJ))) * & ! WHERE(PRRT(:)>XRTMIN(3))
@@ -555,7 +556,7 @@ DO JJ = 1, SIZE(GWET)
GWET(JJ) = .FALSE.
END IF
END DO
-!$acc end kernels
+! acc end parallel loop
IF(LDSOFT) THEN
!$acc kernels
@@ -568,7 +569,7 @@ ELSE
PRH_TEND(:, IRRWETH)=0.
!$acc end kernels
IF(IGWET>0)THEN
-!$acc kernels
+!$acc kernels present_cr(ZVEC1,ZVEC2)
!
!* 7.2.12 select the (PLBDAH,PLBDAR) couplet
!
@@ -611,6 +612,7 @@ ELSE
*(ZVEC1(JJ) - 1.0)
END DO
ZZW(:) = 0.
+ !$acc loop independent
DO JJ = 1, IGWET
ZZW(I1(JJ)) = ZVEC3(JJ)
END DO
@@ -683,6 +685,7 @@ ELSE
( PRHODREF(:)*(XLMTT-XCL*(XTT-PT(:))) )
END WHERE
ENDIF
+!$acc loop independent
DO JL=1, ISIZE
!We must agregate, at least, the cold species
ZRWETH_INIT(JL)=ZHAIL(JL) * MAX(PRH_TEND(JL, IRIWETH)+PRH_TEND(JL, IRSWETH)+PRH_TEND(JL, IRGWETH), &
@@ -694,16 +697,19 @@ ENDDO
!* 7.4 Select Wet or Dry case
!
!Wet case
+!$acc loop independent
DO JL=1, ISIZE
ZWETH(JL) = ZHAIL(JL) * &
& MAX(0., SIGN(1., MAX(0., ZRDRYH_INIT(JL)-PRH_TEND(JL, IRIDRYH)-PRH_TEND(JL, IRSDRYH)-PRH_TEND(JL, IRGDRYH)) - &
&MAX(0., ZRWETH_INIT(JL)-PRH_TEND(JL, IRIWETH)-PRH_TEND(JL, IRSWETH)-PRH_TEND(JL, IRGWETH))))
ENDDO
IF(LNULLWETH) THEN
+ !$acc loop independent
DO JL=1, ISIZE
ZWETH(JL) = ZWETH(JL) * MAX(0., -SIGN(1., -ZRDRYH_INIT(JL))) ! WHERE(ZRDRYH_INIT(:)>0.)
ENDDO
ELSE
+ !$acc loop independent
DO JL=1, ISIZE
ZWETH(JL) = ZWETH(JL) * MAX(0., -SIGN(1., -ZRWETH_INIT(JL))) ! WHERE(ZRWETH_INIT(:)>0.)
ENDDO
@@ -713,6 +719,7 @@ IF(.NOT. LWETHPOST) THEN
ZWETH(JL) = ZWETH(JL) * MAX(0., -SIGN(1., PT(JL)-XTT)) ! WHERE(PT(:)<XTT)
ENDDO
ENDIF
+!$acc loop independent
DO JL=1, ISIZE
ZDRYH(JL) = ZHAIL(JL) * &
& MAX(0., -SIGN(1., PT(JL)-XTT)) * & ! WHERE(PT(:)<XTT)
diff --git a/src/MNH/ice4_fast_rs.f90 b/src/MNH/ice4_fast_rs.f90
index 94a0ad92c..4fb4434d6 100644
--- a/src/MNH/ice4_fast_rs.f90
+++ b/src/MNH/ice4_fast_rs.f90
@@ -287,12 +287,11 @@ DO JL=1, ISIZE
&PRS_TEND(JL, IFREEZ2) * PRIAGGS(JL)) - &
PRIAGGS(JL))
ENDDO
-!
+!$acc end kernels
!* 5.1 cloud droplet riming of the aggregates
!
IGRIM = 0
-!$acc end kernels
-!$acc parallel loop private(idx) independent present_cr( I1, GRIM, PCOMPUTE, PRCT, PRST, XRTMIN, ZRIM )
+!$acc parallel loop private(idx) copy(IGRIM) independent present_cr( I1, GRIM, PCOMPUTE, PRCT, PRST, XRTMIN, ZRIM )
DO CONCURRENT( JJ = 1 : SIZE( GRIM ) )
ZRIM(JJ)=MAX(0., -SIGN(1., XRTMIN(2)-PRCT(JJ))) * & !WHERE(PRCT(:)>XRTMIN(2))
&MAX(0., -SIGN(1., XRTMIN(5)-PRST(JJ))) * & !WHERE(PRST(:)>XRTMIN(5))
@@ -461,12 +460,12 @@ DO JL=1, ISIZE
PA_RG(JL) = PA_RG(JL) + PRCRIMSG(JL)+PRSRIMCG(JL)
PA_TH(JL) = PA_TH(JL) + PRCRIMSG(JL)*(PLSFACT(JL)-PLVFACT(JL))
ENDDO
+!$acc end kernels
!
!* 5.2 rain accretion onto the aggregates
!
IGACC = 0
-!$acc end kernels
-!$acc parallel loop private(idx) independent present_cr( I1, GACC, PCOMPUTE, PRRT, PRST, XRTMIN, ZACC )
+!$acc parallel loop private(idx) copy(IGACC) independent present_cr( I1, GACC, PCOMPUTE, PRRT, PRST, XRTMIN, ZACC )
DO CONCURRENT( JJ = 1 : SIZE( GACC ) )
ZACC(JJ)=MAX(0., -SIGN(1., XRTMIN(3)-PRRT(JJ))) * & !WHERE(PRRT(:)>XRTMIN(3))
&MAX(0., -SIGN(1., XRTMIN(5)-PRST(JJ))) * & !WHERE(PRST(:)>XRTMIN(5))
diff --git a/src/MNH/ice4_rsrimcg_old.f90 b/src/MNH/ice4_rsrimcg_old.f90
index 6d4fb2fd3..8abfaed13 100644
--- a/src/MNH/ice4_rsrimcg_old.f90
+++ b/src/MNH/ice4_rsrimcg_old.f90
@@ -137,27 +137,32 @@ CALL MNH_MEM_GET( zzw, size( prhodref ) )
!
!$acc kernels
PRSRIMCG_MR(:)=0.
+!$acc end kernels
!
IF(.NOT. ODSOFT) THEN
- IGRIM = 0
- GRIM(:) = .FALSE.
-!$acc loop private(IDX) independent
+!$acc kernels
+ GRIM(:) = .FALSE.
+!$acc end kernels
+ IGRIM = 0
+!$acc parallel loop private(IDX) copy(IGRIM) independent
DO JL = 1, SIZE(GRIM)
IF ( PRCT(JL)>XRTMIN(2) .AND. PRST(JL)>XRTMIN(5) .AND. ODCOMPUTE(JL) .AND. PT(JL)<XTT ) THEN
!$acc atomic capture
IGRIM = IGRIM + 1
IDX = IGRIM
!$acc end atomic
- IVEC1(IDX) = Jl
+ IVEC1(IDX) = JL
GRIM(JL) = .TRUE.
END IF
END DO
!
+! acc end parallel loop
IF(IGRIM>0) THEN
+ !$acc kernels
!
! 5.1.1 select the PLBDAS
!
-!$acc loop independent
+ !$acc loop independent
DO CONCURRENT( JL = 1 : IGRIM )
ZVEC1(JL) = PLBDAS(IVEC1(JL))
!
@@ -181,11 +186,11 @@ IF(.NOT. ODSOFT) THEN
ZVEC1(JL) = XGAMINC_RIM2( IVEC2(JL)+1 )* ZVEC2(JL) &
- XGAMINC_RIM2( IVEC2(JL) )*(ZVEC2(JL) - 1.0)
END DO
-!$acc loop independent
+ !$acc loop independent
DO CONCURRENT( JL = 1 : size( prhodref ) )
ZZW(JL) = 0.
END DO
-!$acc loop independent
+ !$acc loop independent
DO CONCURRENT( JL = 1 : IGRIM )
ZZW(IVEC1(JL)) = ZVEC1(JL)
END DO
@@ -193,7 +198,7 @@ IF(.NOT. ODSOFT) THEN
! 5.1.6 riming-conversion of the large sized aggregates into graupeln
!
!
-!$acc loop independent
+ !$acc loop independent
DO CONCURRENT( JL = 1 : size( prhodref ) )
IF ( GRIM(JL) ) THEN
#ifndef MNH_BITREP
@@ -205,8 +210,10 @@ IF(.NOT. ODSOFT) THEN
PRSRIMCG_MR(:)=MIN(PRST(:), PRSRIMCG_MR(:))
END IF
END DO
- END IF
-ENDIF
+ !$acc end kernels
+ END IF ! IGRIM
+ENDIF ! ODSOFT
+!$acc kernels
PB_RS(:) = PB_RS(:) - PRSRIMCG_MR(:)
PB_RG(:) = PB_RG(:) + PRSRIMCG_MR(:)
!$acc end kernels
diff --git a/src/MNH/ice4_sedimentation_split.f90 b/src/MNH/ice4_sedimentation_split.f90
index 05c7df02d..255d6a688 100644
--- a/src/MNH/ice4_sedimentation_split.f90
+++ b/src/MNH/ice4_sedimentation_split.f90
@@ -614,7 +614,7 @@ ZREMAINT(:,:) = PTSTEP
!
DO WHILE (ANY(ZREMAINT>0.))
ISEDIM = 0
-!$acc parallel loop private(idx) independent
+!$acc parallel loop private(idx) copy(ISEDIM) independent
DO JK = KKTB,KKTE
!$acc loop independent collapse(2)
DO JJ = KJB,KJE
--
GitLab