From 9c71ea02a5514bfa798e26d68f055a35c2c36426 Mon Sep 17 00:00:00 2001
From: Philippe WAUTELET <philippe.wautelet@aero.obs-mip.fr>
Date: Tue, 11 Apr 2023 13:32:57 +0200
Subject: [PATCH] Philippe 11/04/2023: OpenACC: merge ALL_SEND_RECV_DEVICE into
ALL_SEND_RECV
---
src/LIB/SURCOUCHE/src/mode_splittingz_ll.f90 | 143 +++++--------------
1 file changed, 38 insertions(+), 105 deletions(-)
diff --git a/src/LIB/SURCOUCHE/src/mode_splittingz_ll.f90 b/src/LIB/SURCOUCHE/src/mode_splittingz_ll.f90
index 496cd5d30..868ce4fca 100644
--- a/src/LIB/SURCOUCHE/src/mode_splittingz_ll.f90
+++ b/src/LIB/SURCOUCHE/src/mode_splittingz_ll.f90
@@ -1122,15 +1122,9 @@ CONTAINS
!
!-------------------------------------------------------------------------------
!
-#ifndef MNH_OPENACC
CALL ALL_SEND_RECV(TCRRT_COMDATA%TSEND_BOX_SXP1_YP2_Z_SX_YP2_ZP1, &
TCRRT_COMDATA%TRECV_BOX_SXP1_YP2_Z_SX_YP2_ZP1, &
PFIELDIN, PFIELDOUT, KINFO)
-#else
- CALL ALL_SEND_RECV_DEVICE(TCRRT_COMDATA%TSEND_BOX_SXP1_YP2_Z_SX_YP2_ZP1, &
- TCRRT_COMDATA%TRECV_BOX_SXP1_YP2_Z_SX_YP2_ZP1, &
- PFIELDIN, PFIELDOUT, KINFO)
-#endif
ENDIF
!$acc end data
@@ -1273,15 +1267,9 @@ CONTAINS
TCRRT_COMDATA%TSEND_SXP1_YP2_Z_SX_YP2_ZP1, &
PFIELDIN, PFIELDOUT, KINFO)
ELSE
-#ifndef MNH_OPENACC
CALL ALL_SEND_RECV(TCRRT_COMDATA%TRECV_BOX_SXP1_YP2_Z_SX_YP2_ZP1, &
TCRRT_COMDATA%TSEND_BOX_SXP1_YP2_Z_SX_YP2_ZP1, &
PFIELDIN, PFIELDOUT, KINFO)
-#else
- CALL ALL_SEND_RECV_DEVICE(TCRRT_COMDATA%TRECV_BOX_SXP1_YP2_Z_SX_YP2_ZP1, &
- TCRRT_COMDATA%TSEND_BOX_SXP1_YP2_Z_SX_YP2_ZP1, &
- PFIELDIN, PFIELDOUT, KINFO)
-#endif
ENDIF
!$acc end data
@@ -1359,15 +1347,9 @@ CONTAINS
!
!-------------------------------------------------------------------------------
!
-#ifndef MNH_OPENACC
CALL ALL_SEND_RECV(TCRRT_COMDATA%TSEND_BOX_SX_YP2_ZP1_SXP2_Y_ZP1, &
TCRRT_COMDATA%TRECV_BOX_SX_YP2_ZP1_SXP2_Y_ZP1, &
PFIELDIN, PFIELDOUT, KINFO)
-#else
- CALL ALL_SEND_RECV_DEVICE(TCRRT_COMDATA%TSEND_BOX_SX_YP2_ZP1_SXP2_Y_ZP1, &
- TCRRT_COMDATA%TRECV_BOX_SX_YP2_ZP1_SXP2_Y_ZP1, &
- PFIELDIN, PFIELDOUT, KINFO)
-#endif
ENDIF
!
@@ -1443,15 +1425,9 @@ CONTAINS
!
!-------------------------------------------------------------------------------
!
-#ifndef MNH_OPENACC
CALL ALL_SEND_RECV(TCRRT_COMDATA%TRECV_BOX_SX_YP2_ZP1_SXP2_Y_ZP1, &
TCRRT_COMDATA%TSEND_BOX_SX_YP2_ZP1_SXP2_Y_ZP1, &
PFIELDIN, PFIELDOUT, KINFO)
-#else
- CALL ALL_SEND_RECV_DEVICE(TCRRT_COMDATA%TRECV_BOX_SX_YP2_ZP1_SXP2_Y_ZP1, &
- TCRRT_COMDATA%TSEND_BOX_SX_YP2_ZP1_SXP2_Y_ZP1, &
- PFIELDIN, PFIELDOUT, KINFO)
-#endif
ENDIF
!
!-------------------------------------------------------------------------------
@@ -1525,15 +1501,9 @@ CONTAINS
TCRRT_COMDATA%TRECV_SXP2_Y_ZP1_SXP2_YP1_Z, &
PFIELDIN, PFIELDOUT, KINFO)
ELSE
-#ifndef MNH_OPENACC
CALL ALL_SEND_RECV(TCRRT_COMDATA%TSEND_BOX_SXP2_Y_ZP1_SXP2_YP1_Z, &
TCRRT_COMDATA%TRECV_BOX_SXP2_Y_ZP1_SXP2_YP1_Z, &
PFIELDIN, PFIELDOUT, KINFO)
-#else
- CALL ALL_SEND_RECV_DEVICE(TCRRT_COMDATA%TSEND_BOX_SXP2_Y_ZP1_SXP2_YP1_Z, &
- TCRRT_COMDATA%TRECV_BOX_SXP2_Y_ZP1_SXP2_YP1_Z, &
- PFIELDIN, PFIELDOUT, KINFO)
-#endif
ENDIF
!$acc end data
@@ -1676,15 +1646,9 @@ CONTAINS
TCRRT_COMDATA%TSEND_SXP2_Y_ZP1_SXP2_YP1_Z, &
PFIELDIN, PFIELDOUT, KINFO)
ELSE
-#ifndef MNH_OPENACC
CALL ALL_SEND_RECV(TCRRT_COMDATA%TRECV_BOX_SXP2_Y_ZP1_SXP2_YP1_Z, &
TCRRT_COMDATA%TSEND_BOX_SXP2_Y_ZP1_SXP2_YP1_Z, &
PFIELDIN, PFIELDOUT, KINFO)
-#else
- CALL ALL_SEND_RECV_DEVICE(TCRRT_COMDATA%TRECV_BOX_SXP2_Y_ZP1_SXP2_YP1_Z, &
- TCRRT_COMDATA%TSEND_BOX_SXP2_Y_ZP1_SXP2_YP1_Z, &
- PFIELDIN, PFIELDOUT, KINFO)
-#endif
END IF
!$acc end data
@@ -1939,68 +1903,6 @@ END FUNCTION LSOUTHZ_ll
use modd_precision, only: MNHREAL_MPI
USE MODD_STRUCTURE_ll, ONLY: BOX_ll
USE MODD_VAR_ll, ONLY: NMNH_COMM_WORLD
- !
- IMPLICIT NONE
- !
- ! Argument
- !
- TYPE(BOX_ll) , POINTER :: TSEND_BOX_FROM,TRECV_BOX_TO
- REAL, DIMENSION(:,:,:), INTENT(INOUT) :: PFIELDIN ! field to be sent
- REAL, DIMENSION(:,:,:), INTENT(INOUT) :: PFIELDOUT ! reception field
- INTEGER :: KINFO ! return status
- !
- ! local var
- !
- INTEGER :: JB,JI,JJ,JK, JCNT ! loop
- REAL, DIMENSION(:), ALLOCATABLE :: ZSEND ! buffer to be sent
- REAL, DIMENSION(:), ALLOCATABLE :: ZRECV ! buffer to be recv
- !
- ALLOCATE(ZSEND(TSEND_BOX_FROM%NSIZE))
- ALLOCATE(ZRECV(TRECV_BOX_TO%NSIZE))
- !
- JCNT = 0
- DO JB = 1, TSEND_BOX_FROM%NBOX
- IF ( TSEND_BOX_FROM%NCNT(JB) .NE. 0 ) THEN
- DO JK=TSEND_BOX_FROM%NZOR(JB),TSEND_BOX_FROM%NZEND(JB)
- DO JJ=TSEND_BOX_FROM%NYOR(JB),TSEND_BOX_FROM%NYEND(JB)
- DO JI=TSEND_BOX_FROM%NXOR(JB),TSEND_BOX_FROM%NXEND(JB)
- JCNT = JCNT + 1
- ZSEND(JCNT) = PFIELDIN(JI,JJ,JK)
- END DO
- END DO
- END DO
- END IF
- END DO
- !
- CALL mpi_alltoallv(ZSEND,TSEND_BOX_FROM%NCNT,TSEND_BOX_FROM%NSTRT,MNHREAL_MPI,&
- ZRECV,TRECV_BOX_TO%NCNT ,TRECV_BOX_TO%NSTRT ,MNHREAL_MPI,&
- TSEND_BOX_FROM%NCOM,KINFO)
- !
- JCNT = 0
- PFIELDOUT = 0.0
- DO JB = 1, TRECV_BOX_TO%NBOX
- IF ( TRECV_BOX_TO%NCNT(JB) .NE. 0 ) THEN
- DO JK=TRECV_BOX_TO%NZOR(JB),TRECV_BOX_TO%NZEND(JB)
- DO JJ=TRECV_BOX_TO%NYOR(JB),TRECV_BOX_TO%NYEND(JB)
- DO JI=TRECV_BOX_TO%NXOR(JB),TRECV_BOX_TO%NXEND(JB)
- JCNT = JCNT + 1
- PFIELDOUT(JI,JJ,JK) = ZRECV(JCNT)
- END DO
- END DO
- END DO
- END IF
- END DO
- !
- DEALLOCATE(ZSEND,ZRECV)
- END SUBROUTINE ALL_SEND_RECV
-
-#ifdef MNH_OPENACC
- SUBROUTINE ALL_SEND_RECV_DEVICE(TSEND_BOX_FROM,TRECV_BOX_TO, &
- PFIELDIN, PFIELDOUT, KINFO)
- !
- use modd_precision, only: MNHREAL_MPI
- USE MODD_STRUCTURE_ll, ONLY: BOX_ll
- USE MODD_VAR_ll, ONLY: NMNH_COMM_WORLD
#ifdef MNH_OPENACC
USE MODE_MNH_ZWORK, ONLY: MNH_MEM_GET, MNH_MEM_POSITION_PIN, MNH_MEM_RELEASE
@@ -2017,13 +1919,15 @@ END FUNCTION LSOUTHZ_ll
!
! local var
!
+#ifdef MNH_OPENACC
INTEGER :: IXOR, IXEND, IYOR, IYEND, IZOR, IZEND
+#endif
INTEGER :: JB,JI,JJ,JK, JCNT ! loop
- INTEGER :: JCNT0
#ifndef MNH_OPENACC
REAL, DIMENSION(:), ALLOCATABLE :: ZSEND ! buffer to be sent
REAL, DIMENSION(:), ALLOCATABLE :: ZRECV ! buffer to be recv
#else
+ INTEGER :: JCNT0
REAL, DIMENSION(:), POINTER, CONTIGUOUS :: ZSEND ! buffer to be sent
REAL, DIMENSION(:), POINTER, CONTIGUOUS :: ZRECV ! buffer to be recv
#endif
@@ -2033,15 +1937,29 @@ END FUNCTION LSOUTHZ_ll
ALLOCATE(ZRECV(TRECV_BOX_TO%NSIZE))
#else
!Pin positions in the pools of MNH memory
- CALL MNH_MEM_POSITION_PIN()
+ CALL MNH_MEM_POSITION_PIN( 'ALL_SEND_RECV' )
CALL MNH_MEM_GET( ZSEND, TSEND_BOX_FROM%NSIZE )
CALL MNH_MEM_GET( ZRECV, TRECV_BOX_TO%NSIZE )
#endif
!
+#ifndef MNH_OPENACC
+ JCNT = 0
+#else
JCNT0 = 0
+#endif
DO JB = 1, TSEND_BOX_FROM%NBOX
IF ( TSEND_BOX_FROM%NCNT(JB) /= 0 ) THEN
+#ifndef MNH_OPENACC
+ DO JK=TSEND_BOX_FROM%NZOR(JB),TSEND_BOX_FROM%NZEND(JB)
+ DO JJ=TSEND_BOX_FROM%NYOR(JB),TSEND_BOX_FROM%NYEND(JB)
+ DO JI=TSEND_BOX_FROM%NXOR(JB),TSEND_BOX_FROM%NXEND(JB)
+ JCNT = JCNT + 1
+ ZSEND(JCNT) = PFIELDIN(JI,JJ,JK)
+ END DO
+ END DO
+ END DO
+#else
IXOR = TSEND_BOX_FROM%NXOR(JB)
IXEND = TSEND_BOX_FROM%NXEND(JB)
IYOR = TSEND_BOX_FROM%NYOR(JB)
@@ -2055,8 +1973,8 @@ END FUNCTION LSOUTHZ_ll
DO JJ = IYOR, IYEND
DO JI = IXOR, IXEND
JCNT = ( JK - IZOR ) * ( IYEND - IYOR + 1 ) * ( IXEND - IXOR + 1 ) &
- + ( JJ - IYOR ) * ( IXEND - IXOR + 1 ) &
- + ( JI - IXOR ) + 1 + JCNT0
+ + ( JJ - IYOR ) * ( IXEND - IXOR + 1 ) &
+ + ( JI - IXOR ) + 1 + JCNT0
ZSEND(JCNT) = PFIELDIN(JI,JJ,JK)
END DO
END DO
@@ -2064,6 +1982,7 @@ END FUNCTION LSOUTHZ_ll
!$acc end kernels
JCNT0 = JCNT0 + TSEND_BOX_FROM%NCNT(JB)
+#endif
END IF
END DO
!
@@ -2073,13 +1992,27 @@ END FUNCTION LSOUTHZ_ll
TSEND_BOX_FROM%NCOM,KINFO)
!$acc end host_data
!
+#ifndef MNH_OPENACC
+ JCNT = 0
+#else
JCNT0 = 0
+#endif
!$acc kernels
PFIELDOUT(:,:,:) = 0.0
!$acc end kernels
DO JB = 1, TRECV_BOX_TO%NBOX
IF ( TRECV_BOX_TO%NCNT(JB) /= 0 ) THEN
+#ifndef MNH_OPENACC
+ DO JK=TRECV_BOX_TO%NZOR(JB),TRECV_BOX_TO%NZEND(JB)
+ DO JJ=TRECV_BOX_TO%NYOR(JB),TRECV_BOX_TO%NYEND(JB)
+ DO JI=TRECV_BOX_TO%NXOR(JB),TRECV_BOX_TO%NXEND(JB)
+ JCNT = JCNT + 1
+ PFIELDOUT(JI,JJ,JK) = ZRECV(JCNT)
+ END DO
+ END DO
+ END DO
+#else
IXOR = TRECV_BOX_TO%NXOR(JB)
IXEND = TRECV_BOX_TO%NXEND(JB)
IYOR = TRECV_BOX_TO%NYOR(JB)
@@ -2101,6 +2034,7 @@ END FUNCTION LSOUTHZ_ll
!$acc end kernels
JCNT0 = JCNT0 + TRECV_BOX_TO%NCNT(JB)
+#endif
END IF
END DO
!
@@ -2108,11 +2042,10 @@ END FUNCTION LSOUTHZ_ll
DEALLOCATE(ZSEND,ZRECV)
#else
!Release all memory allocated with MNH_MEM_GET calls since last call to MNH_MEM_POSITION_PIN
- CALL MNH_MEM_RELEASE()
+ CALL MNH_MEM_RELEASE( 'ALL_SEND_RECV' )
#endif
+ END SUBROUTINE ALL_SEND_RECV
- END SUBROUTINE ALL_SEND_RECV_DEVICE
-#endif
! ##########################################
SUBROUTINE GET_ORZ_ll( HSPLIT, KXOR, KYOR )
--
GitLab