Quentin 26/09/2024: add kernels ice4_sedimentation + _split

src/PHYEX/micro/mode_ice4_sedimentation.f90

@@ -123,6 +123,8 @@ IF (BUCONF%LBUDGET_RG)              CALL BUDGET_STORE_INIT_PHY(D, TBUDGETS(NBUDG
 IF (BUCONF%LBUDGET_RH .AND. KRR==7) CALL BUDGET_STORE_INIT_PHY(D, TBUDGETS(NBUDGET_RH), 'SEDI', PRHS(:, :) * PRHODJ(:, :))
 
 IF(PARAMI%CSEDIM=='STAT') THEN
+!$acc kernels
+!$acc loop independent collapse(2)
   DO JK = IKTB,IKTE
     DO JIJ = IIJB,IIJE
       ZRCT(JIJ,JK)=PRCS(JIJ,JK)*PTSTEP
@@ -133,6 +135,7 @@ IF(PARAMI%CSEDIM=='STAT') THEN
       IF (KRR==7) ZRHT(JIJ,JK)=PRHS(JIJ,JK)*PTSTEP
     ENDDO
   ENDDO
+!$acc end kernels
   CALL ICE4_SEDIMENTATION_STAT(D, CST, ICEP, ICED, PARAMI, &
                               &PTSTEP, KRR, PDZZ, &
                               &PRHODREF, PPABST, PTHT, PT, PRHODJ, &
@@ -141,7 +144,9 @@ IF(PARAMI%CSEDIM=='STAT') THEN
                               &PINPRC, PINPRR, ZINPRI, PINPRS, PINPRG, &
                               &PSEA=PSEA, PTOWN=PTOWN, &
                               &PINPRH=PINPRH, PRHT=ZRHT, PRHS=PRHS, PFPR=PFPR)
+!$acc kernels
   PINPRS(:) = PINPRS(:) + ZINPRI(:)
+!$acc end kernels
   !No negativity correction here as we apply sedimentation on PR.S*PTSTEP variables
 ELSEIF(PARAMI%CSEDIM=='SPLI') THEN
   CALL ICE4_SEDIMENTATION_SPLIT(D, CST, ICEP, ICED, PARAMI, &
@@ -151,7 +156,9 @@ ELSEIF(PARAMI%CSEDIM=='SPLI') THEN
                                &PINPRC, PINPRR, ZINPRI, PINPRS, PINPRG, &
                                &PSEA=PSEA, PTOWN=PTOWN, &
                                &PINPRH=PINPRH, PRHT=PRHT, PRHS=PRHS, PFPR=PFPR)
+!$acc kernels
   PINPRS(:) = PINPRS(:) + ZINPRI(:)
+!$acc end kernels
   !We correct negativities with conservation
   !SPLI algorith uses a time-splitting. Inside the loop a temporary m.r. is used.
   !   It is initialized with the m.r. at T and is modified by two tendencies:

src/PHYEX/micro/mode_ice4_sedimentation_split.f90

@@ -43,7 +43,13 @@ USE MODD_PARAM_ICE,      ONLY: PARAM_ICE_t
 !
 USE MODE_MSG, ONLY: PRINT_MSG, NVERB_FATAL
 !
-USE MODI_GAMMA, ONLY: GAMMA
+!USE MODI_GAMMA, ONLY: GAMMA
+#ifndef MNH_COMPILER_CCE
+USE MODI_GAMMA
+#endif
+#if defined(TARGET_NV70)
+USE MODI_GAMMA
+#endif
 !
 IMPLICIT NONE
 !
@@ -135,23 +141,35 @@ END IF
 !
 !        O. Initialization of for sedimentation
 !
+!$acc kernels
 ZINVTSTEP=1./PTSTEP
+!$acc end kernels
 IF (GPRESENT_PFPR) THEN
+!$acc kernels
   PFPR(:,:,:) = 0.
+!$acc end kernels
 END IF
 !
 !*       1. Parameters for cloud sedimentation
 !
 IF (GSEDIC) THEN
+!$acc kernels
   ZRAY(:,:)   = 0.
   ZLBC(:,:)   = ICED%XLBC(1)
   ZFSEDC(:,:) = ICEP%XFSEDC(1)
   ZCONC3D(:,:)= ICED%XCONC_LAND
   ZCONC_TMP(:)= ICED%XCONC_LAND
+!$acc end kernels
   IF (GPRESENT_PSEA) THEN
+!$acc kernels
+!$acc loop independent
     DO JIJ = IIJB, IIJE
       ZCONC_TMP(JIJ)=PSEA(JIJ)*ICED%XCONC_SEA+(1.-PSEA(JIJ))*ICED%XCONC_LAND
     ENDDO
+!$acc end kernels
+
+!$acc kernels
+!$acc loop independent collapse(2)
     DO JK=IKTB, IKTE
       DO JIJ = IIJB, IIJE
           ZLBC(JIJ,JK)   = PSEA(JIJ)*ICED%XLBC(2)+(1.-PSEA(JIJ))*ICED%XLBC(1)
@@ -162,16 +180,22 @@ IF (GSEDIC) THEN
                          & PSEA(JIJ)*GAMMA(ICED%XNUC2+1.0/ICED%XALPHAC2)/(GAMMA(ICED%XNUC2)))
       ENDDO
     END DO
+!$acc end kernels
   ELSE
+!$acc kernels
     ZCONC3D(:,:) = ICED%XCONC_LAND
     ZRAY(:,:)  = 0.5*(GAMMA(ICED%XNUC+1.0/ICED%XALPHAC)/(GAMMA(ICED%XNUC)))
+!$acc end kernels
   END IF
+!$acc kernels
+!$acc loop independent collapse(2)
   DO JK=IKTB, IKTE
     DO JIJ = IIJB, IIJE
       ZRAY(JIJ,JK)      = MAX(1.,ZRAY(JIJ,JK))
       ZLBC(JIJ,JK)      = MAX(MIN(ICED%XLBC(1),ICED%XLBC(2)),ZLBC(JIJ,JK))
     ENDDO
   ENDDO
+!$acc end kernels
 ENDIF
 !
 !*       2.    compute the fluxes
@@ -180,6 +204,8 @@ ENDIF
 !  the precipitating fields are larger than a minimal value only !!!
 !  For optimization we consider each variable separately
 !
+!$acc kernels
+!$acc loop independent collapse(2)
 DO JK=IKTB, IKTE
   DO JIJ = IIJB, IIJE
     ! External tendecies
@@ -207,6 +233,7 @@ DO JK=IKTB, IKTE
     ZW(JIJ,JK) =1./(PRHODREF(JIJ,JK)* PDZZ(JIJ,JK))
   ENDDO
 ENDDO
+!$acc end kernels
 !
 !
 !*       2.1   for cloud
@@ -341,11 +368,14 @@ ELSE
   GPRESENT_PFPR = .FALSE.
 END IF
 !
+!$acc kernels
 PINPRX(:) = 0.
 ZINVTSTEP=1./PTSTEP
 ZRSMIN(:) = ICED%XRTMIN(:) * ZINVTSTEP
 ZREMAINT(:) = 0.
 ZREMAINT(:) = PTSTEP
+!$acc end kernels
+!$acc update self(ZREMAINT)
 !
 DO WHILE (ANY(ZREMAINT>0.))
   !
@@ -358,7 +388,11 @@ DO WHILE (ANY(ZREMAINT>0.))
   !
   IF(KSPE==2) THEN
     !******* for cloud
+!$acc kernels
     ZWSED(:,:) = 0.
+!$acc end kernels
+!$acc kernels
+!$acc loop independent collapse(2)
     DO JK = IKTB,IKTE
       DO JIJ = IIJB,IIJE
         IF(PRXT(JIJ,JK)>ICED%XRTMIN(KSPE) .AND. ZREMAINT(JIJ)>0.) THEN
@@ -374,9 +408,14 @@ DO WHILE (ANY(ZREMAINT>0.))
         ENDIF
       ENDDO
     ENDDO
+!$acc end kernels
   ELSEIF(KSPE==4) THEN
     ! ******* for pristine ice
+!$acc kernels
     ZWSED(:,:) = 0.
+!$acc end kernels
+!$acc kernels
+!$acc loop independent collapse(2)
     DO JK = IKTB,IKTE
       DO JIJ = IIJB,IIJE
         IF(PRXT(JIJ, JK) .GT. MAX(ICED%XRTMIN(4), 1.0E-7) .AND. ZREMAINT(JIJ)>0.) THEN
@@ -387,11 +426,16 @@ DO WHILE (ANY(ZREMAINT>0.))
         ENDIF
       ENDDO
     ENDDO
+!$acc end kernels
 #if defined(REPRO48) 
 #else
   ELSEIF(KSPE==5) THEN
     ! ******* for snow
+!$acc kernels
     ZWSED(:,:) = 0.
+!$acc end kernels
+!$acc kernels
+!$acc loop independent collapse(2)
     DO JK = IKTB,IKTE
       DO JIJ = IIJB,IIJE
         IF(PRXT(JIJ,JK)> ICED%XRTMIN(KSPE) .AND. ZREMAINT(JIJ)>0.) THEN
@@ -411,6 +455,7 @@ DO WHILE (ANY(ZREMAINT>0.))
         ENDIF
       ENDDO
     ENDDO
+!$acc end kernels
 #endif
   ELSE
     ! ******* for other species
@@ -435,7 +480,11 @@ DO WHILE (ANY(ZREMAINT>0.))
         CALL PRINT_MSG(NVERB_FATAL, 'GEN', 'ICE4_SEDIMENTATION_SPLIT', 'no sedimentation parameter for KSPE='//TRIM(YSPE) )
     END SELECT
     !
+!$acc kernels
     ZWSED(:,:) = 0.
+!$acc end kernels
+!$acc kernels
+!$acc loop independent collapse(2)
     DO JK = IKTB,IKTE
       DO JIJ = IIJB,IIJE
         IF(PRXT(JIJ,JK)>ICED%XRTMIN(KSPE) .AND. ZREMAINT(JIJ)>0.) THEN
@@ -444,8 +493,15 @@ DO WHILE (ANY(ZREMAINT>0.))
         ENDIF
       ENDDO
     ENDDO
+!$acc end kernels
   ENDIF
+!$acc kernels
   ZMAX_TSTEP(:) = ZREMAINT(:)
+!QR: gives wrong result with acc loop independent.
+! Tested with smaller kernels: execution crashes
+! call to cuEventSynchronize returned error 700: Illegal address
+! acc loop independent collapse(2)
+! TODO: NEXT DO BLOCS ARE SEQUENTIAL FOR NOW
   DO JK = IKTB,IKTE
     DO JIJ = IIJB,IIJE
       IF(PRXT(JIJ,JK)>ICED%XRTMIN(KSPE) .AND. ZWSED(JIJ, JK)>1.E-20 .AND. ZREMAINT(JIJ)>0.) THEN
@@ -454,13 +510,17 @@ DO WHILE (ANY(ZREMAINT>0.))
       ENDIF
     ENDDO
   ENDDO
-
+!$acc end kernels
+!$acc kernels
+!$acc loop independent
   DO JIJ = IIJB, IIJE
       ZREMAINT(JIJ) = ZREMAINT(JIJ) - ZMAX_TSTEP(JIJ)
       PINPRX(JIJ) = PINPRX(JIJ) + ZWSED(JIJ,IKB) / CST%XRHOLW * (ZMAX_TSTEP(JIJ) * ZINVTSTEP)
   ENDDO
-
+!$acc end kernels
+!$acc kernels
   DO JK = IKTB , IKTE
+!$acc loop independent
     DO JIJ = IIJB, IIJE
       ZMRCHANGE = ZMAX_TSTEP(JIJ) * POORHODZ(JIJ,JK)*(ZWSED(JIJ,JK+IKL)-ZWSED(JIJ,JK))
       PRXT(JIJ,JK) = PRXT(JIJ,JK) + ZMRCHANGE + PPRXS(JIJ,JK) * ZMAX_TSTEP(JIJ)
@@ -470,6 +530,7 @@ DO WHILE (ANY(ZREMAINT>0.))
       ENDIF
     ENDDO
   ENDDO
+!$acc end kernels
 !
 END DO
 !