Quentin 02/10/2024: add kernels ice4_pack:

- remove IC=0 (useless in the IF condition), bit-repro CPU/GPU
- remove EXIT of the loop (impossible to use with OpenACC)
- add kernels to internal loop with atomic capture of IC (with change of variable IDX)

src/PHYEX/micro/mode_ice4_pack.f90

@@ -161,7 +161,7 @@ REAL, DIMENSION(KSIZE2,0:7) :: ZEXTPK   !To take into acount external tendencies
 INTEGER, DIMENSION(KPROMA) :: I1,I2 ! Used to replace the COUNT and PACK intrinsics on variables
 INTEGER, DIMENSION(KSIZE) :: I1TOT, I2TOT ! Used to replace the COUNT and PACK intrinsics
 !
-INTEGER :: IC, JMICRO
+INTEGER :: IC, JMICRO, IDX
 LOGICAL :: LLSIGMA_RC, LL_AUCV_ADJU
 !
 !-------------------------------------------------------------------------------
@@ -215,7 +215,6 @@ IF(PARAMI%LPACK_MICRO) THEN
     IC=0
     ISTK=IKTB
     ISTIJ=IIJB
-  
     DO JMICRO=1,KSIZE,KPROMA
   
       IMICRO=MIN(KPROMA,KSIZE-JMICRO+1)
@@ -225,70 +224,82 @@ IF(PARAMI%LPACK_MICRO) THEN
       !
   
       ! Setup packing parameters
+!$acc kernels
+!$acc loop seq
       OUTER_LOOP: DO JK = ISTK, IKTE
-        IF (ANY(ODMICRO(:,JK))) THEN
-          DO JIJ = ISTIJ, IIJE
+        IF (ANY(ODMICRO(:,JK))) THEN !This test is useless ? => tested 4 lignes further
+          !$acc loop gang vector independent
+	  DO JIJ = ISTIJ, IIJE
             IF (ODMICRO(JIJ,JK)) THEN
+	    !$acc atomic capture
               IC=IC+1
-              LLMICRO(IC)=.TRUE.
+              IDX=IC !change of variable to use acc atomic capture (IC is shared)
+             !$acc end atomic
+              LLMICRO(IDX)=.TRUE.
               ! Initialization of variables in packed format :
-              ZVART(IC, ITH)=PWR(JIJ, JK, ITH)
-              ZVART(IC, IRV)=PWR(JIJ, JK, IRV)
-              ZVART(IC, IRC)=PWR(JIJ, JK, IRC)
-              ZVART(IC, IRR)=PWR(JIJ, JK, IRR)
-              ZVART(IC, IRI)=PWR(JIJ, JK, IRI)
-              ZVART(IC, IRS)=PWR(JIJ, JK, IRS)
-              ZVART(IC, IRG)=PWR(JIJ, JK, IRG)
+              ZVART(IDX, ITH)=PWR(JIJ, JK, ITH)
+              ZVART(IDX, IRV)=PWR(JIJ, JK, IRV)
+              ZVART(IDX, IRC)=PWR(JIJ, JK, IRC)
+              ZVART(IDX, IRR)=PWR(JIJ, JK, IRR)
+              ZVART(IDX, IRI)=PWR(JIJ, JK, IRI)
+              ZVART(IDX, IRS)=PWR(JIJ, JK, IRS)
+              ZVART(IDX, IRG)=PWR(JIJ, JK, IRG)
               IF (KRR==7) THEN
-                ZVART(IC, IRH)=PWR(JIJ, JK, IRH)
+                ZVART(IDX, IRH)=PWR(JIJ, JK, IRH)
               ENDIF
               IF (GEXT_TEND) THEN
                 !The th tendency is not related to a mixing ratio change, there is no exn/exnref issue here
-                ZEXTPK(IC, ITH)=PTHS(JIJ, JK)
-                ZEXTPK(IC, IRV)=PRVS(JIJ, JK)
-                ZEXTPK(IC, IRC)=PRCS(JIJ, JK)
-                ZEXTPK(IC, IRR)=PRRS(JIJ, JK)
-                ZEXTPK(IC, IRI)=PRIS(JIJ, JK)
-                ZEXTPK(IC, IRS)=PRSS(JIJ, JK)
-                ZEXTPK(IC, IRG)=PRGS(JIJ, JK)
+                ZEXTPK(IDX, ITH)=PTHS(JIJ, JK)
+                ZEXTPK(IDX, IRV)=PRVS(JIJ, JK)
+                ZEXTPK(IDX, IRC)=PRCS(JIJ, JK)
+                ZEXTPK(IDX, IRR)=PRRS(JIJ, JK)
+                ZEXTPK(IDX, IRI)=PRIS(JIJ, JK)
+                ZEXTPK(IDX, IRS)=PRSS(JIJ, JK)
+                ZEXTPK(IDX, IRG)=PRGS(JIJ, JK)
                 IF (KRR==7) THEN
-                  ZEXTPK(IC, IRH)=PRHS(JIJ, JK)
+                  ZEXTPK(IDX, IRH)=PRHS(JIJ, JK)
                 ENDIF
               ENDIF
-              ZCIT       (IC)=PCIT    (JIJ, JK)
-              ZCF        (IC)=PCLDFR  (JIJ, JK)
-              ZRHODREF   (IC)=PRHODREF(JIJ, JK)
-              ZPRES      (IC)=PPABST  (JIJ, JK)
-              ZEXN       (IC)=PEXN    (JIJ, JK)
+              ZCIT       (IDX)=PCIT    (JIJ, JK)
+              ZCF        (IDX)=PCLDFR  (JIJ, JK)
+              ZRHODREF   (IDX)=PRHODREF(JIJ, JK)
+              ZPRES      (IDX)=PPABST  (JIJ, JK)
+              ZEXN       (IDX)=PEXN    (JIJ, JK)
               IF(LLSIGMA_RC) THEN
-                ZSIGMA_RC(IC)=PSIGS   (JIJ, JK)
+                ZSIGMA_RC(IDX)=PSIGS   (JIJ, JK)
               ENDIF
               IF (LL_AUCV_ADJU) THEN
-                ZHLC_HCF(IC) = PHLC_HCF(JIJ, JK)
-                ZHLC_HRC(IC) = PHLC_HRC(JIJ, JK)
-                ZHLI_HCF(IC) = PHLI_HCF(JIJ, JK)
-                ZHLI_HRI(IC) = PHLI_HRI(JIJ, JK)
+                ZHLC_HCF(IDX) = PHLC_HCF(JIJ, JK)
+                ZHLC_HRC(IDX) = PHLC_HRC(JIJ, JK)
+                ZHLI_HCF(IDX) = PHLI_HCF(JIJ, JK)
+                ZHLI_HRI(IDX) = PHLI_HRI(JIJ, JK)
               ENDIF
               ! Save indices for later usages:
-              I1(IC) = JIJ
-              I2(IC) = JK
+              I1(IDX) = JIJ
+              I2(IDX) = JK
               I1TOT(JMICRO+IC-1)=JIJ
               I2TOT(JMICRO+IC-1)=JK
               IF (IC==IMICRO) THEN
                 ! the end of the chunk has been reached, then reset the starting index :
                 ISTIJ=JIJ+1
                 IF (ISTIJ <= IIJE) THEN
+	    !$acc atomic write
                   ISTK=JK
+            !$acc end atomic
                 ELSE
                   ! end of line, restart from 1 and increment upper loop
+	    !$acc atomic write
                   ISTK=JK+1
+            !$acc end atomic
                   IF (ISTK > IKTE) THEN
                     ! end of line, restart from 1
+	    !$acc atomic write
                     ISTK=IKTB
+            !$acc end atomic
                   ENDIF
                 ENDIF
-                IC=0
-                EXIT OUTER_LOOP
+            !    IC=0 ! this re-init is useless, because we leave the loop
+             !  EXIT OUTER_LOOP ! This must be suppressed for openACC use
               ENDIF
             ENDIF
           ENDDO
@@ -296,7 +307,7 @@ IF(PARAMI%LPACK_MICRO) THEN
         ! restart inner loop on JIJ :
         ISTIJ=IIJB
       ENDDO OUTER_LOOP
-  
+  !$acc end kernels
       !
       !*       5.     TENDENCIES COMPUTATION
       !               ----------------------
@@ -317,6 +328,8 @@ IF(PARAMI%LPACK_MICRO) THEN
       !*       6.     UNPACKING
       !               ---------
       !
+!$acc kernels
+!$acc loop independent
       DO JL=1, IMICRO
         PCIT  (I1(JL),I2(JL))=ZCIT   (JL)
         IF(PARAMI%LWARM) THEN
@@ -332,12 +345,16 @@ IF(PARAMI%LPACK_MICRO) THEN
           PWR(I1(JL),I2(JL),IRH)=ZVART(JL, IRH)
         ENDIF
       ENDDO
+!$acc end kernels 
       IF(BUCONF%LBU_ENABLE) THEN
+!$acc kernels
+!$acc loop independent collapse(2)
         DO JV=1, IBUNUM-IBUNUM_EXTRA
           DO JL=1, IMICRO
             ZBU_PACK(JMICRO+JL-1, JV) = ZBU_SUM(JL, JV)
           ENDDO
         ENDDO
+!$acc end kernels
       ENDIF