Philippe 11/04/2023: OpenACC: merge ALL_SEND_RECV_DEVICE into ALL_SEND_RECV

src/LIB/SURCOUCHE/src/mode_splittingz_ll.f90

@@ -1122,15 +1122,9 @@ CONTAINS
     !
     !-------------------------------------------------------------------------------
     !
-#ifndef MNH_OPENACC
     CALL ALL_SEND_RECV(TCRRT_COMDATA%TSEND_BOX_SXP1_YP2_Z_SX_YP2_ZP1, &
          TCRRT_COMDATA%TRECV_BOX_SXP1_YP2_Z_SX_YP2_ZP1, &
          PFIELDIN, PFIELDOUT, KINFO)
-#else
-    CALL ALL_SEND_RECV_DEVICE(TCRRT_COMDATA%TSEND_BOX_SXP1_YP2_Z_SX_YP2_ZP1, &
-         TCRRT_COMDATA%TRECV_BOX_SXP1_YP2_Z_SX_YP2_ZP1, &
-         PFIELDIN, PFIELDOUT, KINFO)
-#endif
     ENDIF
 
 !$acc end data
@@ -1273,15 +1267,9 @@ CONTAINS
          TCRRT_COMDATA%TSEND_SXP1_YP2_Z_SX_YP2_ZP1, &
          PFIELDIN, PFIELDOUT, KINFO)
     ELSE
-#ifndef MNH_OPENACC
     CALL ALL_SEND_RECV(TCRRT_COMDATA%TRECV_BOX_SXP1_YP2_Z_SX_YP2_ZP1, &
          TCRRT_COMDATA%TSEND_BOX_SXP1_YP2_Z_SX_YP2_ZP1, &
          PFIELDIN, PFIELDOUT, KINFO)
-#else
-    CALL ALL_SEND_RECV_DEVICE(TCRRT_COMDATA%TRECV_BOX_SXP1_YP2_Z_SX_YP2_ZP1, &
-         TCRRT_COMDATA%TSEND_BOX_SXP1_YP2_Z_SX_YP2_ZP1, &
-         PFIELDIN, PFIELDOUT, KINFO)
-#endif
     ENDIF
 
 !$acc end data
@@ -1359,15 +1347,9 @@ CONTAINS
     !
     !-------------------------------------------------------------------------------
     !
-#ifndef MNH_OPENACC
     CALL ALL_SEND_RECV(TCRRT_COMDATA%TSEND_BOX_SX_YP2_ZP1_SXP2_Y_ZP1, &
          TCRRT_COMDATA%TRECV_BOX_SX_YP2_ZP1_SXP2_Y_ZP1, &
          PFIELDIN, PFIELDOUT, KINFO)
-#else
-    CALL ALL_SEND_RECV_DEVICE(TCRRT_COMDATA%TSEND_BOX_SX_YP2_ZP1_SXP2_Y_ZP1, &
-         TCRRT_COMDATA%TRECV_BOX_SX_YP2_ZP1_SXP2_Y_ZP1, &
-         PFIELDIN, PFIELDOUT, KINFO)
-#endif
     ENDIF
     !
 
@@ -1443,15 +1425,9 @@ CONTAINS
     !
     !-------------------------------------------------------------------------------
     !
-#ifndef MNH_OPENACC
     CALL ALL_SEND_RECV(TCRRT_COMDATA%TRECV_BOX_SX_YP2_ZP1_SXP2_Y_ZP1, &
          TCRRT_COMDATA%TSEND_BOX_SX_YP2_ZP1_SXP2_Y_ZP1, &
          PFIELDIN, PFIELDOUT, KINFO)
-#else
-    CALL ALL_SEND_RECV_DEVICE(TCRRT_COMDATA%TRECV_BOX_SX_YP2_ZP1_SXP2_Y_ZP1, &
-         TCRRT_COMDATA%TSEND_BOX_SX_YP2_ZP1_SXP2_Y_ZP1, &
-         PFIELDIN, PFIELDOUT, KINFO)
-#endif
     ENDIF
     !
     !-------------------------------------------------------------------------------
@@ -1525,15 +1501,9 @@ CONTAINS
          TCRRT_COMDATA%TRECV_SXP2_Y_ZP1_SXP2_YP1_Z, &
          PFIELDIN, PFIELDOUT, KINFO)
     ELSE
-#ifndef MNH_OPENACC
     CALL ALL_SEND_RECV(TCRRT_COMDATA%TSEND_BOX_SXP2_Y_ZP1_SXP2_YP1_Z, &
          TCRRT_COMDATA%TRECV_BOX_SXP2_Y_ZP1_SXP2_YP1_Z, &
          PFIELDIN, PFIELDOUT, KINFO)
-#else
-    CALL ALL_SEND_RECV_DEVICE(TCRRT_COMDATA%TSEND_BOX_SXP2_Y_ZP1_SXP2_YP1_Z, &
-         TCRRT_COMDATA%TRECV_BOX_SXP2_Y_ZP1_SXP2_YP1_Z, &
-         PFIELDIN, PFIELDOUT, KINFO)
-#endif
     ENDIF
 
 !$acc end data
@@ -1676,15 +1646,9 @@ CONTAINS
          TCRRT_COMDATA%TSEND_SXP2_Y_ZP1_SXP2_YP1_Z, &
          PFIELDIN, PFIELDOUT, KINFO)
     ELSE
-#ifndef MNH_OPENACC
     CALL ALL_SEND_RECV(TCRRT_COMDATA%TRECV_BOX_SXP2_Y_ZP1_SXP2_YP1_Z, &
          TCRRT_COMDATA%TSEND_BOX_SXP2_Y_ZP1_SXP2_YP1_Z, &
          PFIELDIN, PFIELDOUT, KINFO)
-#else
-    CALL ALL_SEND_RECV_DEVICE(TCRRT_COMDATA%TRECV_BOX_SXP2_Y_ZP1_SXP2_YP1_Z, &
-         TCRRT_COMDATA%TSEND_BOX_SXP2_Y_ZP1_SXP2_YP1_Z, &
-         PFIELDIN, PFIELDOUT, KINFO)
-#endif
     END IF
 
 !$acc end data
@@ -1939,68 +1903,6 @@ END FUNCTION LSOUTHZ_ll
       use modd_precision,    only: MNHREAL_MPI
       USE MODD_STRUCTURE_ll, ONLY: BOX_ll
       USE MODD_VAR_ll,       ONLY: NMNH_COMM_WORLD
-      !
-      IMPLICIT NONE
-      !
-      ! Argument
-      !
-      TYPE(BOX_ll)                , POINTER :: TSEND_BOX_FROM,TRECV_BOX_TO
-      REAL, DIMENSION(:,:,:), INTENT(INOUT) :: PFIELDIN ! field to be sent
-      REAL, DIMENSION(:,:,:), INTENT(INOUT) :: PFIELDOUT ! reception field
-      INTEGER                               :: KINFO ! return status
-      !
-      ! local var
-      !
-      INTEGER                               :: JB,JI,JJ,JK, JCNT ! loop
-      REAL, DIMENSION(:), ALLOCATABLE       :: ZSEND ! buffer to be sent
-      REAL, DIMENSION(:), ALLOCATABLE       :: ZRECV ! buffer to be recv      
-      !
-      ALLOCATE(ZSEND(TSEND_BOX_FROM%NSIZE))
-      ALLOCATE(ZRECV(TRECV_BOX_TO%NSIZE))
-      !
-      JCNT = 0
-      DO JB = 1, TSEND_BOX_FROM%NBOX
-         IF  ( TSEND_BOX_FROM%NCNT(JB) .NE. 0 ) THEN
-            DO JK=TSEND_BOX_FROM%NZOR(JB),TSEND_BOX_FROM%NZEND(JB)
-               DO JJ=TSEND_BOX_FROM%NYOR(JB),TSEND_BOX_FROM%NYEND(JB)
-                  DO JI=TSEND_BOX_FROM%NXOR(JB),TSEND_BOX_FROM%NXEND(JB)
-                     JCNT =  JCNT + 1
-                     ZSEND(JCNT) = PFIELDIN(JI,JJ,JK)
-                  END DO
-               END DO
-            END DO
-         END IF
-      END DO
-      !
-      CALL mpi_alltoallv(ZSEND,TSEND_BOX_FROM%NCNT,TSEND_BOX_FROM%NSTRT,MNHREAL_MPI,&
-                         ZRECV,TRECV_BOX_TO%NCNT  ,TRECV_BOX_TO%NSTRT  ,MNHREAL_MPI,&
-                         TSEND_BOX_FROM%NCOM,KINFO)
-      !
-      JCNT = 0
-      PFIELDOUT = 0.0
-      DO JB = 1, TRECV_BOX_TO%NBOX
-         IF  ( TRECV_BOX_TO%NCNT(JB) .NE. 0 ) THEN
-            DO JK=TRECV_BOX_TO%NZOR(JB),TRECV_BOX_TO%NZEND(JB)
-               DO JJ=TRECV_BOX_TO%NYOR(JB),TRECV_BOX_TO%NYEND(JB)
-                  DO JI=TRECV_BOX_TO%NXOR(JB),TRECV_BOX_TO%NXEND(JB)
-                     JCNT =  JCNT + 1
-                     PFIELDOUT(JI,JJ,JK) =  ZRECV(JCNT)
-                  END DO
-               END DO
-            END DO
-         END IF
-      END DO
-      !
-      DEALLOCATE(ZSEND,ZRECV)
-    END SUBROUTINE ALL_SEND_RECV
-
-#ifdef MNH_OPENACC
-    SUBROUTINE ALL_SEND_RECV_DEVICE(TSEND_BOX_FROM,TRECV_BOX_TO, &
-         PFIELDIN, PFIELDOUT, KINFO)
-      !
-      use modd_precision,    only: MNHREAL_MPI
-      USE MODD_STRUCTURE_ll, ONLY: BOX_ll
-      USE MODD_VAR_ll,       ONLY: NMNH_COMM_WORLD
 
 #ifdef MNH_OPENACC
       USE MODE_MNH_ZWORK,    ONLY: MNH_MEM_GET, MNH_MEM_POSITION_PIN, MNH_MEM_RELEASE
@@ -2017,13 +1919,15 @@ END FUNCTION LSOUTHZ_ll
       !
       ! local var
       !
+#ifdef MNH_OPENACC
       INTEGER                               :: IXOR, IXEND, IYOR, IYEND, IZOR, IZEND
+#endif
       INTEGER                               :: JB,JI,JJ,JK, JCNT ! loop
-      INTEGER                               :: JCNT0
 #ifndef MNH_OPENACC
       REAL, DIMENSION(:), ALLOCATABLE       :: ZSEND ! buffer to be sent
       REAL, DIMENSION(:), ALLOCATABLE       :: ZRECV ! buffer to be recv
 #else
+      INTEGER                               :: JCNT0
       REAL, DIMENSION(:), POINTER, CONTIGUOUS :: ZSEND ! buffer to be sent
       REAL, DIMENSION(:), POINTER, CONTIGUOUS :: ZRECV ! buffer to be recv
 #endif
@@ -2033,15 +1937,29 @@ END FUNCTION LSOUTHZ_ll
       ALLOCATE(ZRECV(TRECV_BOX_TO%NSIZE))
 #else
       !Pin positions in the pools of MNH memory
-      CALL MNH_MEM_POSITION_PIN()
+      CALL MNH_MEM_POSITION_PIN( 'ALL_SEND_RECV' )
 
       CALL MNH_MEM_GET( ZSEND, TSEND_BOX_FROM%NSIZE )
       CALL MNH_MEM_GET( ZRECV, TRECV_BOX_TO%NSIZE   )
 #endif
       !
+#ifndef MNH_OPENACC
+      JCNT = 0
+#else
       JCNT0 = 0
+#endif
       DO JB = 1, TSEND_BOX_FROM%NBOX
         IF  ( TSEND_BOX_FROM%NCNT(JB) /= 0 ) THEN
+#ifndef MNH_OPENACC
+          DO JK=TSEND_BOX_FROM%NZOR(JB),TSEND_BOX_FROM%NZEND(JB)
+               DO JJ=TSEND_BOX_FROM%NYOR(JB),TSEND_BOX_FROM%NYEND(JB)
+                  DO JI=TSEND_BOX_FROM%NXOR(JB),TSEND_BOX_FROM%NXEND(JB)
+                     JCNT =  JCNT + 1
+                     ZSEND(JCNT) = PFIELDIN(JI,JJ,JK)
+                  END DO
+               END DO
+            END DO
+#else
           IXOR  = TSEND_BOX_FROM%NXOR(JB)
           IXEND = TSEND_BOX_FROM%NXEND(JB)
           IYOR  = TSEND_BOX_FROM%NYOR(JB)
@@ -2055,8 +1973,8 @@ END FUNCTION LSOUTHZ_ll
             DO JJ = IYOR, IYEND
               DO JI = IXOR, IXEND
                 JCNT =   ( JK - IZOR ) * ( IYEND - IYOR + 1 ) * ( IXEND - IXOR + 1 ) &
-                      + ( JJ - IYOR ) * ( IXEND - IXOR + 1 ) &
-                      + ( JI - IXOR ) + 1 + JCNT0
+                       + ( JJ - IYOR ) * ( IXEND - IXOR + 1 ) &
+                       + ( JI - IXOR ) + 1 + JCNT0
                 ZSEND(JCNT) = PFIELDIN(JI,JJ,JK)
               END DO
             END DO
@@ -2064,6 +1982,7 @@ END FUNCTION LSOUTHZ_ll
 !$acc end kernels
 
           JCNT0 = JCNT0 + TSEND_BOX_FROM%NCNT(JB)
+#endif
         END IF
       END DO
       !
@@ -2073,13 +1992,27 @@ END FUNCTION LSOUTHZ_ll
                          TSEND_BOX_FROM%NCOM,KINFO)
 !$acc end host_data
       !
+#ifndef MNH_OPENACC
+      JCNT = 0
+#else
       JCNT0 = 0
+#endif
 !$acc kernels
       PFIELDOUT(:,:,:) = 0.0
 !$acc end kernels
 
       DO JB = 1, TRECV_BOX_TO%NBOX
         IF  ( TRECV_BOX_TO%NCNT(JB) /= 0 ) THEN
+#ifndef MNH_OPENACC
+          DO JK=TRECV_BOX_TO%NZOR(JB),TRECV_BOX_TO%NZEND(JB)
+            DO JJ=TRECV_BOX_TO%NYOR(JB),TRECV_BOX_TO%NYEND(JB)
+              DO JI=TRECV_BOX_TO%NXOR(JB),TRECV_BOX_TO%NXEND(JB)
+                JCNT =  JCNT + 1
+                PFIELDOUT(JI,JJ,JK) =  ZRECV(JCNT)
+              END DO
+            END DO
+          END DO
+#else
           IXOR  = TRECV_BOX_TO%NXOR(JB)
           IXEND = TRECV_BOX_TO%NXEND(JB)
           IYOR  = TRECV_BOX_TO%NYOR(JB)
@@ -2101,6 +2034,7 @@ END FUNCTION LSOUTHZ_ll
 !$acc end kernels
 
           JCNT0 = JCNT0 + TRECV_BOX_TO%NCNT(JB)
+#endif
         END IF
       END DO
       !
@@ -2108,11 +2042,10 @@ END FUNCTION LSOUTHZ_ll
       DEALLOCATE(ZSEND,ZRECV)
 #else
       !Release all memory allocated with MNH_MEM_GET calls since last call to MNH_MEM_POSITION_PIN
-      CALL MNH_MEM_RELEASE()
+      CALL MNH_MEM_RELEASE( 'ALL_SEND_RECV' )
 #endif
+    END SUBROUTINE ALL_SEND_RECV
 
-    END SUBROUTINE ALL_SEND_RECV_DEVICE
-#endif
 
 !     ##########################################
       SUBROUTINE GET_ORZ_ll( HSPLIT, KXOR, KYOR )