diff --git a/src/MNH/get_halo.f90 b/src/MNH/get_halo.f90
index d76d70497baa04b424d8e0eaa2327935a9bf81bf..6c72fecedf7b7ce01038259defbd8bfa9c88085d 100644
--- a/src/MNH/get_halo.f90
+++ b/src/MNH/get_halo.f90
@@ -38,6 +38,20 @@ INTERFACE
END INTERFACE
!
#ifdef MNH_OPENACC
+INTERFACE
+ SUBROUTINE GET_HALO2_DF(PSRC, TP_PSRC_HALO2_ll, HNAME)
+ !
+ USE MODD_ARGSLIST_ll, ONLY : HALO2LIST_ll
+ !
+ IMPLICIT NONE
+ !
+ REAL, DIMENSION(:,:,:), INTENT(INOUT) :: PSRC ! variable at t
+ TYPE(HALO2LIST_ll), POINTER :: TP_PSRC_HALO2_ll ! halo2 for SRC
+ character(len=*), optional, intent(in) :: HNAME ! Name of the field to be added
+ !
+ END SUBROUTINE GET_HALO2_DF
+END INTERFACE
+
INTERFACE
SUBROUTINE GET_HALO_D(PSRC, HDIR, HNAME)
IMPLICIT NONE
@@ -196,11 +210,15 @@ MODULE MODD_HALO_D
REAL, SAVE , ALLOCATABLE, DIMENSION(:,:) :: ZNORTHC_2D_IN, ZSOUTHC_2D_IN, ZWESTC_2D_IN, ZEASTC_2D_IN
REAL, SAVE , ALLOCATABLE, DIMENSION(:,:) :: ZNORTHC_2D_OUT, ZSOUTHC_2D_OUT, ZWESTC_2D_OUT, ZEASTC_2D_OUT
+ REAL, SAVE , ALLOCATABLE, DIMENSION(:,:,:) :: ZNORTH2F_IN, ZSOUTH2F_IN, ZWEST2F_IN, ZEAST2F_IN
+ REAL, SAVE , ALLOCATABLE, DIMENSION(:,:,:) :: ZNORTH2F_OUT, ZSOUTH2F_OUT, ZWEST2F_OUT, ZEAST2F_OUT
+
LOGICAL, SAVE :: GFIRST_GET_HALO_D = .TRUE.
LOGICAL, SAVE :: GFIRST_INIT_HALO_D = .TRUE.
INTEGER, SAVE :: IHALO_1
INTEGER, SAVE :: NP_NORTH,NP_SOUTH,NP_WEST,NP_EAST
+ INTEGER, SAVE :: IHALO2,IHALO2_1
CONTAINS
@@ -218,8 +236,12 @@ CONTAINS
IF (GFIRST_INIT_HALO_D) THEN
!
- IHALO_1 = NHALO-1
- !
+ IHALO_1 = NHALO-1
+ IHALO2 = MAX(2,NHALO)
+ IHALO2_1 = IHALO2-1
+ !
+ ! Init HALO
+ !
ALLOCATE ( ZSOUTH_IN ( IIB:IIE , IJB:IJB+IHALO_1 , IKU ) )
ALLOCATE ( ZNORTH_IN ( IIB:IIE , IJE-IHALO_1:IJE , IKU ) )
ALLOCATE ( ZWEST_IN ( IIB:IIB+IHALO_1 , IJB:IJE , IKU ) )
@@ -259,7 +281,21 @@ CONTAINS
ALLOCATE ( ZWESTC_2D_OUT ( 1:IIB-1 , IJB:IJE ) )
ALLOCATE ( ZEASTC_2D_OUT ( IIE+1:IIU , IJB:IJE ) )
!$acc enter data create (ZNORTHC_2D_OUT, ZSOUTHC_2D_OUT, ZWESTC_2D_OUT, ZEASTC_2D_OUT)
-
+ !
+ ! Init HALO2 for Full update in 1 time <-> GET_HALO + GET_HALO2
+ !
+ ALLOCATE ( ZSOUTH2F_IN ( IIB:IIE , IJB:IJB+IHALO2_1 , IKU ) )
+ ALLOCATE ( ZNORTH2F_IN ( IIB:IIE , IJE-IHALO2_1:IJE , IKU ) )
+ ALLOCATE ( ZWEST2F_IN ( IIB:IIB+IHALO2_1 , IJB:IJE , IKU ) )
+ ALLOCATE ( ZEAST2F_IN ( IIE-IHALO2_1:IIE , IJB:IJE , IKU ) )
+ !$acc enter data create (ZNORTH2F_IN, ZSOUTH2F_IN, ZWEST2F_IN, ZEAST2F_IN)
+ !
+ ALLOCATE ( ZSOUTH2F_OUT ( IIB:IIE , IJB-IHALO2:IJB-1 , IKU ) )
+ ALLOCATE ( ZNORTH2F_OUT ( IIB:IIE , IJE+1:IJE+IHALO2 , IKU ) )
+ ALLOCATE ( ZWEST2F_OUT ( IIB-IHALO2:IIB-1 , IJB:IJE , IKU ) )
+ ALLOCATE ( ZEAST2F_OUT ( IIE+1:IIE+IHALO2 , IJB:IJE , IKU ) )
+ !$acc enter data create (ZNORTH2F_OUT, ZSOUTH2F_OUT, ZWEST2F_OUT, ZEAST2F_OUT)
+
IF (.NOT. GWEST ) THEN
NP_WEST = ( IP-1 -1 ) + 1
ELSE
@@ -314,9 +350,13 @@ character(len=*), optional, intent(in) :: HNAME ! Name of the field to be added
!
INTEGER :: INB_REQ , IREQ(8)
!
+CALL MPPDB_CHECK(PSRC,"GET_HALO_D big:PSRC")
+!
CALL GET_HALO_START_D(PSRC,INB_REQ,IREQ,HDIR)
CALL GET_HALO_STOP_D(PSRC,INB_REQ,IREQ,HDIR)
!
+CALL MPPDB_CHECK(PSRC,"GET_HALO_D end:PSRC")
+!
END SUBROUTINE GET_HALO_D
! #########################
SUBROUTINE GET_HALO_START_D(PSRC,KNB_REQ,KREQ,HDIR)
@@ -349,6 +389,11 @@ INTEGER,PARAMETER :: IS_WEST=1 , IS_EAST=2, IS_SOUTH=3, IS_NORTH=4
LOGICAL :: LX , LY
INTEGER :: NB_REQ, IERR
!
+INTEGER :: JI,JJ,JK, JIU,JJU,JKU
+
+JIU = SIZE(PSRC,1)
+JJU = SIZE(PSRC,2)
+JKU = SIZE(PSRC,3)
CALL INIT_HALO_D()
@@ -438,12 +483,16 @@ END IF
IF (LX) THEN
IF (.NOT. GWEST) THEN
!$acc kernels async(IS_WEST)
- ZWEST_IN ( IIB:IIB+IHALO_1 , IJB:IJE , : ) = PSRC( IIB:IIB+IHALO_1 , IJB:IJE , : )
+ !$mnh_expand_array(JI=IIB:IIB+IHALO_1 , JJ=IJB:IJE , JK=1:JKU )
+ ZWEST_IN ( IIB:IIB+IHALO_1 , IJB:IJE , : ) = PSRC( IIB:IIB+IHALO_1 , IJB:IJE , : )
+ !$mnh_end_expand_array()
!$acc end kernels
END IF
IF (.NOT.GEAST) THEN
!$acc kernels async(IS_EAST)
- ZEAST_IN ( IIE-IHALO_1:IIE , IJB:IJE , : ) = PSRC( IIE-IHALO_1:IIE , IJB:IJE , : )
+ !$mnh_expand_array(JI=IIE-IHALO_1:IIE , JJ=IJB:IJE , JK=1:JKU)
+ ZEAST_IN ( IIE-IHALO_1:IIE , IJB:IJE , : ) = PSRC( IIE-IHALO_1:IIE , IJB:IJE , : )
+ !$mnh_end_expand_array()
!$acc end kernels
ENDIF
END IF
@@ -451,12 +500,16 @@ END IF
IF (LY) THEN
IF (.NOT.GSOUTH) THEN
!$acc kernels async(IS_SOUTH)
- ZSOUTH_IN ( IIB:IIE , IJB:IJB+IHALO_1 , : ) = PSRC( IIB:IIE , IJB:IJB+IHALO_1 , : )
+ !$mnh_expand_array(JI=IIB:IIE , JJ=IJB:IJB+IHALO_1 , JK=1:JKU )
+ ZSOUTH_IN ( IIB:IIE , IJB:IJB+IHALO_1 , : ) = PSRC( IIB:IIE , IJB:IJB+IHALO_1 , : )
+ !$mnh_end_expand_array()
!$acc end kernels
ENDIF
IF (.NOT.GNORTH) THEN
!$acc kernels async(IS_NORTH)
- ZNORTH_IN ( IIB:IIE , IJE-IHALO_1:IJE , : ) = PSRC( IIB:IIE , IJE-IHALO_1:IJE , : )
+ !$mnh_expand_array(JI=IIB:IIE , JJ=IJE-IHALO_1:IJE , JK=1:JKU )
+ ZNORTH_IN ( IIB:IIE , IJE-IHALO_1:IJE , : ) = PSRC( IIB:IIE , IJE-IHALO_1:IJE , : )
+ !$mnh_end_expand_array()
!$acc end kernels
ENDIF
ENDIF
@@ -561,6 +614,11 @@ INTEGER,PARAMETER :: IS_WEST=1 , IS_EAST=2, IS_SOUTH=3, IS_NORTH=4
LOGICAL :: LX , LY
INTEGER :: NB_REQ, IERR
!
+INTEGER :: JI,JJ,JK, JIU,JJU,JKU
+
+JIU = SIZE(PSRC,1)
+JJU = SIZE(PSRC,2)
+JKU = SIZE(PSRC,3)
CALL INIT_HALO_D()
@@ -600,7 +658,9 @@ IF (LX) THEN
!$acc update device(ZWEST_OUT) async(IS_WEST)
#endif
!$acc kernels async(IS_WEST)
- PSRC( 1:IIB-1 , IJB:IJE , : ) = ZWEST_OUT( 1:IIB-1 , IJB:IJE , : )
+ !$mnh_expand_array(JI=1:IIB-1 , JJ=IJB:IJE , JK=1:JKU )
+ PSRC( 1:IIB-1 , IJB:IJE , : ) = ZWEST_OUT( 1:IIB-1 , IJB:IJE , : )
+ !$mnh_end_expand_array()
!$acc end kernels
ENDIF
IF (.NOT.GEAST) THEN
@@ -608,7 +668,9 @@ IF (LX) THEN
!$acc update device(ZEAST_OUT) async(IS_EAST)
#endif
!$acc kernels async(IS_EAST)
- PSRC( IIE+1:IIU , IJB:IJE , : ) = ZEAST_OUT( IIE+1:IIU , IJB:IJE , : )
+ !$mnh_expand_array(JI=IIE+1:IIU , JJ=IJB:IJE , JK=1:JKU )
+ PSRC( IIE+1:IIU , IJB:IJE , : ) = ZEAST_OUT( IIE+1:IIU , IJB:IJE , : )
+ !$mnh_end_expand_array()
!$acc end kernels
ENDIF
END IF
@@ -618,7 +680,9 @@ IF (LY) THEN
!$acc update device(ZSOUTH_OUT) async(IS_SOUTH)
#endif
!$acc kernels async(IS_SOUTH)
- PSRC( IIB:IIE , 1:IJB-1 , : ) = ZSOUTH_OUT( IIB:IIE , 1:IJB-1 , : )
+ !$mnh_expand_array(JI=IIB:IIE , JJ=1:IJB-1 , JK=1:JKU )
+ PSRC( IIB:IIE , 1:IJB-1 , : ) = ZSOUTH_OUT( IIB:IIE , 1:IJB-1 , : )
+ !$mnh_end_expand_array()
!$acc end kernels
ENDIF
IF (.NOT.GNORTH) THEN
@@ -626,7 +690,9 @@ IF (LY) THEN
!$acc update device(ZNORTH_OUT) async(IS_NORTH)
#endif
!$acc kernels async(IS_NORTH)
- PSRC( IIB:IIE , IJE+1:IJU , : ) = ZNORTH_OUT ( IIB:IIE , IJE+1:IJU , : )
+ !$mnh_expand_array(JI=IIB:IIE , JJ=IJE+1:IJU , JK=1:JKU )
+ PSRC( IIB:IIE , IJE+1:IJU , : ) = ZNORTH_OUT ( IIB:IIE , IJE+1:IJU , : )
+ !$mnh_end_expand_array()
!$acc end kernels
ENDIF
END IF
@@ -658,6 +724,9 @@ USE MODD_VAR_ll, ONLY : NMNH_COMM_WORLD
USE MODD_MPIF, ONLY : MPI_STATUSES_IGNORE
USE MODD_PRECISION, ONLY : MNHREAL_MPI
!
+USE MODE_MNH_ZWORK, ONLY : IIU,IJU,IKU
+USE MODE_MNH_ZWORK, ONLY : IIB,IJB ,IIE,IJE
+!
IMPLICIT NONE
!
REAL, DIMENSION(:,:,:), INTENT(INOUT) :: PSRC ! variable at t
@@ -667,78 +736,27 @@ character(len=*), optional, intent(in) :: HNAME ! Name of the field to be added
character(len=:), allocatable :: yname
TYPE(LIST_ll) , POINTER :: TZ_PSRC_ll ! halo
INTEGER :: IERROR ! error return code
-INTEGER, SAVE :: IIB,IJB ! Begining useful area in x,y,z directions
-INTEGER, SAVE :: IIE,IJE ! End useful area in x,y,z directions
-
-INTEGER,SAVE :: IIU,IJU,IKU
INTEGER,PARAMETER :: IS_WEST=1 , IS_EAST=2, IS_SOUTH=3, IS_NORTH=4
-INTEGER, SAVE :: IIBX,IJBX ! Extended Begining useful area in x,y,z directions
-INTEGER, SAVE :: IIEX,IJEX ! Extended End useful area in x,y,z directions
-
LOGICAL :: LX , LY
INTEGER :: INB_REQ , IREQ(8)
INTEGER :: IERR
-if ( NPROC == 1 ) RETURN
+CALL MPPDB_CHECK(PSRC,"GET_HALO_DD big:PSRC")
+
+if ( NPROC == 1 ) THEN
+ CALL MPPDB_CHECK(PSRC,"GET_HALO_DD end:PSRC")
+ RETURN
+end if
+
+CALL INIT_HALO_D()
!$acc data present ( PSRC )
NULLIFY( TZ_PSRC_ll)
!
-IF (GFIRST_GET_HALO_D ) THEN
- CALL GET_INDICE_ll(IIB,IJB,IIE,IJE)
- !
- !
- IIU=size(psrc,1)
- IJU=size(psrc,2)
- IKU=size(psrc,3)
- !
- IHALO_1 = NHALO-1
- !
-!!$ IIBX= 1 ; IIEX= IIU ; IJBX= 1 ; IJEX= IJU
- IIBX= IIB ; IIEX= IIE ; IJBX= IJB ; IJEX= IJE
-
- ALLOCATE ( ZSOUTH_IN ( IIBX:IIEX , IJB:IJB+IHALO_1 , IKU ) )
- ALLOCATE ( ZNORTH_IN ( IIBX:IIEX , IJE-IHALO_1:IJE , IKU ) )
- ALLOCATE ( ZWEST_IN ( IIB:IIB+IHALO_1 , IJBX:IJEX , IKU ) )
- ALLOCATE ( ZEAST_IN ( IIE-IHALO_1:IIE , IJBX:IJEX , IKU ) )
- !$acc enter data create (ZNORTH_IN, ZSOUTH_IN, ZWEST_IN, ZEAST_IN)
- !
- ALLOCATE ( ZSOUTH_OUT ( IIBX:IIEX , 1:IJB-1 , IKU ) )
- ALLOCATE ( ZNORTH_OUT ( IIBX:IIEX , IJE+1:IJU , IKU ) )
- ALLOCATE ( ZWEST_OUT ( 1:IIB-1 , IJBX:IJEX , IKU ) )
- ALLOCATE ( ZEAST_OUT ( IIE+1:IIU , IJBX:IJEX , IKU ) )
- !$acc enter data create (ZNORTH_OUT, ZSOUTH_OUT, ZWEST_OUT, ZEAST_OUT)
-
- IF (.NOT. GWEST ) THEN
- NP_WEST = ( IP-1 -1 ) + 1
- ELSE
- NP_WEST = 0
- ENDIF
- IF (.NOT. GEAST ) THEN
- NP_EAST = ( IP-1 +1 ) + 1
- ELSE
- NP_EAST = 0
- ENDIF
- IF (.NOT. GSOUTH ) THEN
- NP_SOUTH = ( IP-1 -NP1 ) + 1
- ELSE
- NP_SOUTH = 0
- ENDIF
- IF (.NOT. GNORTH ) THEN
- NP_NORTH = ( IP-1 +NP1 ) + 1
- ELSE
- NP_NORTH = 0
- ENDIF
-
- !print*,"PROC=",IP, GWEST,NP_WEST, GEAST,NP_EAST, GSOUTH,NP_SOUTH , GNORTH,NP_NORTH
-
- GFIRST_GET_HALO_D = .FALSE.
-
-END IF
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
@@ -827,24 +845,24 @@ END IF
IF (LX) THEN
IF (.NOT. GWEST) THEN
!$acc kernels async(IS_WEST)
- ZWEST_IN ( IIB:IIB+IHALO_1 , IJBX:IJEX , : ) = PSRC( IIB:IIB+IHALO_1 , IJBX:IJEX , : )
+ ZWEST_IN ( IIB:IIB+IHALO_1 , IJB:IJE , : ) = PSRC( IIB:IIB+IHALO_1 , IJB:IJE , : )
!$acc end kernels
END IF
IF (.NOT.GEAST) THEN
!$acc kernels async(IS_EAST)
- ZEAST_IN ( IIE-IHALO_1:IIE , IJBX:IJEX , : ) = PSRC( IIE-IHALO_1:IIE , IJBX:IJEX , : )
+ ZEAST_IN ( IIE-IHALO_1:IIE , IJB:IJE , : ) = PSRC( IIE-IHALO_1:IIE , IJB:IJE , : )
!$acc end kernels
ENDIF
END IF
IF (LY) THEN
IF (.NOT.GSOUTH) THEN
!$acc kernels async(IS_SOUTH)
- ZSOUTH_IN ( IIBX:IIEX , IJB:IJB+IHALO_1 , : ) = PSRC( IIBX:IIEX , IJB:IJB+IHALO_1 , : )
+ ZSOUTH_IN ( IIB:IIE , IJB:IJB+IHALO_1 , : ) = PSRC( IIB:IIE , IJB:IJB+IHALO_1 , : )
!$acc end kernels
ENDIF
IF (.NOT.GNORTH) THEN
!$acc kernels async(IS_NORTH)
- ZNORTH_IN ( IIBX:IIEX , IJE-IHALO_1:IJE , : ) = PSRC( IIBX:IIEX , IJE-IHALO_1:IJE , : )
+ ZNORTH_IN ( IIB:IIE , IJE-IHALO_1:IJE , : ) = PSRC( IIB:IIE , IJE-IHALO_1:IJE , : )
!$acc end kernels
ENDIF
ENDIF
@@ -931,7 +949,7 @@ IF (LX) THEN
!$acc update device(ZWEST_OUT) async(IS_WEST)
#endif
!$acc kernels async(IS_WEST)
- PSRC( 1:IIB-1 , IJBX:IJEX , : ) = ZWEST_OUT( 1:IIB-1 , IJBX:IJEX , : )
+ PSRC( 1:IIB-1 , IJB:IJE , : ) = ZWEST_OUT( 1:IIB-1 , IJB:IJE , : )
!$acc end kernels
ENDIF
IF (.NOT.GEAST) THEN
@@ -939,7 +957,7 @@ IF (LX) THEN
!$acc update device(ZEAST_OUT) async(IS_EAST)
#endif
!$acc kernels async(IS_EAST)
- PSRC( IIE+1:IIU , IJBX:IJEX , : ) = ZEAST_OUT( IIE+1:IIU , IJBX:IJEX , : )
+ PSRC( IIE+1:IIU , IJB:IJE , : ) = ZEAST_OUT( IIE+1:IIU , IJB:IJE , : )
!$acc end kernels
ENDIF
END IF
@@ -949,7 +967,7 @@ IF (LY) THEN
!$acc update device(ZSOUTH_OUT) async(IS_SOUTH)
#endif
!$acc kernels async(IS_SOUTH)
- PSRC( IIBX:IIEX , 1:IJB-1 , : ) = ZSOUTH_OUT( IIBX:IIEX , 1:IJB-1 , : )
+ PSRC( IIB:IIE , 1:IJB-1 , : ) = ZSOUTH_OUT( IIB:IIE , 1:IJB-1 , : )
!$acc end kernels
ENDIF
IF (.NOT.GNORTH) THEN
@@ -957,7 +975,7 @@ IF (LY) THEN
!$acc update device(ZNORTH_OUT) async(IS_NORTH)
#endif
!$acc kernels async(IS_NORTH)
- PSRC( IIBX:IIEX , IJE+1:IJU , : ) = ZNORTH_OUT ( IIBX:IIEX , IJE+1:IJU , : )
+ PSRC( IIB:IIE , IJE+1:IJU , : ) = ZNORTH_OUT ( IIB:IIE , IJE+1:IJU , : )
!$acc end kernels
ENDIF
END IF
@@ -965,6 +983,8 @@ END IF
!$acc end data
+CALL MPPDB_CHECK(PSRC,"GET_HALO_DD end:PSRC")
+
END SUBROUTINE GET_HALO_DD
!-------------------------------------------------------------------------------
! ########################################
@@ -1600,6 +1620,291 @@ CALL MPPDB_CHECK(PSRC,"GET_2D_HALO_DDC end:PSRC")
!CALL MPPDB_CHECK(PSRC,'UPDATE_HALO_ll::'//TRIM(HNAME))
END SUBROUTINE GET_2D_HALO_DDC
+!-------------------------------------------------------------------------------
+! ########################################
+ SUBROUTINE GET_HALO2_DF(PSRC, TP_PSRC_HALO2F_ll, HNAME)
+! ########################################
+#define MNH_GPUDIRECT
+!
+USE MODD_HALO_D
+USE MODE_ll
+USE MODD_ARGSLIST_ll, ONLY : LIST_ll
+USE MODD_PARAMETERS, ONLY : JPHEXT
+!
+USE MODD_IO, ONLY : GSMONOPROC
+USE MODE_MNH_ZWORK, ONLY : GWEST , GEAST, GSOUTH , GNORTH
+USE MODE_MNH_ZWORK, ONLY : IIU,IJU,IKU
+USE MODE_MNH_ZWORK, ONLY : IIB,IJB ,IIE,IJE
+!
+USE MODD_CONF, ONLY : NHALO
+USE MODE_DEVICE
+USE MODE_MPPDB
+!
+USE MODD_VAR_ll, ONLY : IP,NPROC,NP1,NP2
+USE MODD_VAR_ll, ONLY : NMNH_COMM_WORLD
+USE MODD_MPIF, ONLY : MPI_STATUSES_IGNORE
+USE MODD_PRECISION, ONLY : MNHREAL_MPI
+!
+IMPLICIT NONE
+!
+REAL, DIMENSION(:,:,:), INTENT(INOUT) :: PSRC ! variable at t
+TYPE(HALO2LIST_ll), POINTER :: TP_PSRC_HALO2F_ll ! halo2 for SRC
+character(len=*), optional, intent(in) :: HNAME ! Name of the field to be added
+!
+character(len=:), allocatable :: yname
+INTEGER :: IERROR ! error return code
+
+INTEGER,PARAMETER :: IS_WEST=1 , IS_EAST=2, IS_SOUTH=3, IS_NORTH=4
+
+LOGICAL :: LX , LY
+
+INTEGER :: INB_REQ , IREQ(8)
+INTEGER :: IERR
+
+REAL , DIMENSION(:,:) , POINTER , CONTIGUOUS :: ZH2F_EAST,ZH2F_WEST,ZH2F_NORTH,ZH2F_SOUTH
+
+CALL MPPDB_CHECK(PSRC,"GET_HALO2_DF beg:PSRC")
+
+if ( NPROC == 1 ) then
+ CALL MPPDB_CHECK(PSRC,"GET_HALO2_DF end:PSRC")
+ !CALL MPPDB_CHECK(PSRC,'UPDATE_HALO_ll::'//TRIM(HNAME))
+ !CALL MPPDB_CHECK(PSRC,'UPDATE_HALO2_ll::'//TRIM(HNAME))
+ RETURN
+end if
+
+CALL INIT_HALO_D()
+
+!$acc data present ( PSRC ) &
+!$acc present (ZNORTH2F_IN, ZSOUTH2F_IN, ZWEST2F_IN, ZEAST2F_IN) &
+!$acc present (ZNORTH2F_OUT, ZSOUTH2F_OUT, ZWEST2F_OUT, ZEAST2F_OUT)
+!
+!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+
+LX = .TRUE.
+LY = .TRUE.
+
+!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+
+INB_REQ = 0
+
+!
+! Post the recieve of Zxxxx_IN buffer first via MPI(Gpu_direct)
+!
+
+IF (LX) THEN
+ IF (.NOT. GWEST) THEN
+#ifdef MNH_GPUDIRECT
+ !$acc host_data use_device(ZWEST2F_OUT)
+#endif
+ INB_REQ = INB_REQ + 1
+ CALL MPI_IRECV(ZWEST2F_OUT,SIZE(ZWEST2F_OUT),MNHREAL_MPI,NP_WEST-1,1000+IS_EAST,&
+ NMNH_COMM_WORLD,IREQ(INB_REQ),IERR)
+#ifdef MNH_GPUDIRECT
+ !$acc end host_data
+#endif
+ END IF
+ IF (.NOT.GEAST) THEN
+#ifdef MNH_GPUDIRECT
+ !$acc host_data use_device(ZEAST2F_OUT)
+#endif
+ INB_REQ = INB_REQ + 1
+ CALL MPI_IRECV(ZEAST2F_OUT,SIZE(ZEAST2F_OUT),MNHREAL_MPI,NP_EAST-1,1000+IS_WEST,&
+ NMNH_COMM_WORLD,IREQ(INB_REQ),IERR)
+#ifdef MNH_GPUDIRECT
+ !$acc end host_data
+#endif
+ ENDIF
+END IF
+
+IF (LY) THEN
+ IF (.NOT.GSOUTH) THEN
+#ifdef MNH_GPUDIRECT
+ !$acc host_data use_device(ZSOUTH2F_OUT)
+#endif
+ INB_REQ = INB_REQ + 1
+ CALL MPI_IRECV(ZSOUTH2F_OUT,SIZE(ZSOUTH2F_OUT),MNHREAL_MPI,NP_SOUTH-1,1000+IS_NORTH,&
+ NMNH_COMM_WORLD,IREQ(INB_REQ),IERR)
+#ifdef MNH_GPUDIRECT
+ !$acc end host_data
+#endif
+ ENDIF
+ IF (.NOT.GNORTH) THEN
+#ifdef MNH_GPUDIRECT
+ !$acc host_data use_device(ZNORTH2F_OUT)
+#endif
+ INB_REQ = INB_REQ + 1
+ CALL MPI_IRECV(ZNORTH2F_OUT,SIZE(ZNORTH2F_OUT),MNHREAL_MPI,NP_NORTH-1,1000+IS_SOUTH,&
+ NMNH_COMM_WORLD,IREQ(INB_REQ),IERR)
+#ifdef MNH_GPUDIRECT
+ !$acc end host_data
+#endif
+ ENDIF
+END IF
+
+!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+
+!Copy the halo on the device PSRC to Zxxxx_IN
+
+IF (LX) THEN
+ IF (.NOT. GWEST) THEN
+ !$acc kernels async(IS_WEST)
+ ZWEST2F_IN ( IIB:IIB+IHALO2_1 , IJB:IJE , : ) = PSRC( IIB:IIB+IHALO2_1 , IJB:IJE , : )
+!!$ ZWEST2F_IN ( : , : ) = PSRC( IIB+1 , : , : )
+ !$acc end kernels
+ END IF
+ IF (.NOT.GEAST) THEN
+ !$acc kernels async(IS_EAST)
+ ZEAST2F_IN ( IIE-IHALO2_1:IIE , IJB:IJE , : ) = PSRC( IIE-IHALO2_1:IIE , IJB:IJE , : )
+!!$ ZEAST2F_IN ( : , : ) = PSRC( IIE-1 , : , : )
+ !$acc end kernels
+ ENDIF
+END IF
+IF (LY) THEN
+ IF (.NOT.GSOUTH) THEN
+ !$acc kernels async(IS_SOUTH)
+ ZSOUTH2F_IN ( IIB:IIE , IJB:IJB+IHALO2_1 , : ) = PSRC( IIB:IIE , IJB:IJB+IHALO2_1 , : )
+!!$ ZSOUTH2F_IN ( : , : ) = PSRC( : , IJB+1 , : )
+ !$acc end kernels
+ ENDIF
+ IF (.NOT.GNORTH) THEN
+ !$acc kernels async(IS_NORTH)
+ ZNORTH2F_IN ( IIB:IIE , IJE-IHALO2_1:IJE , : ) = PSRC( IIB:IIE , IJE-IHALO2_1:IJE , : )
+!!$ ZNORTH2F_IN ( : , : ) = PSRC( : , IJE-1 , : )
+ !$acc end kernels
+ ENDIF
+ENDIF
+!$acc wait
+
+!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+!
+! Send Zxxxx2F_IN buffer via MPI(Gpu_direct) or copy to host
+!
+IF (LX) THEN
+ IF (.NOT. GWEST) THEN
+#ifdef MNH_GPUDIRECT
+ !$acc host_data use_device(ZWEST2F_IN)
+#else
+ !$acc update host(ZWEST2F_IN)
+#endif
+ INB_REQ = INB_REQ + 1
+ CALL MPI_ISEND(ZWEST2F_IN,SIZE(ZWEST2F_IN) ,MNHREAL_MPI,NP_WEST-1,1000+IS_WEST,&
+ NMNH_COMM_WORLD,IREQ(INB_REQ),IERR)
+#ifdef MNH_GPUDIRECT
+ !$acc end host_data
+#endif
+ END IF
+ IF (.NOT.GEAST) THEN
+#ifdef MNH_GPUDIRECT
+ !$acc host_data use_device(ZEAST2F_IN)
+#else
+ !$acc update host(ZEAST2F_IN)
+#endif
+ INB_REQ = INB_REQ + 1
+ CALL MPI_ISEND(ZEAST2F_IN,SIZE(ZEAST2F_IN) ,MNHREAL_MPI,NP_EAST-1,1000+IS_EAST,&
+ NMNH_COMM_WORLD,IREQ(INB_REQ),IERR)
+#ifdef MNH_GPUDIRECT
+ !$acc end host_data
+#endif
+ ENDIF
+END IF
+
+IF (LY) THEN
+ IF (.NOT.GSOUTH) THEN
+#ifdef MNH_GPUDIRECT
+ !$acc host_data use_device(ZSOUTH2F_IN)
+#else
+ !$acc update host(ZSOUTH2F_IN)
+#endif
+ INB_REQ = INB_REQ + 1
+ CALL MPI_ISEND(ZSOUTH2F_IN,SIZE(ZSOUTH2F_IN) ,MNHREAL_MPI,NP_SOUTH-1,1000+IS_SOUTH,&
+ NMNH_COMM_WORLD,IREQ(INB_REQ),IERR)
+#ifdef MNH_GPUDIRECT
+ !$acc end host_data
+#endif
+ ENDIF
+ IF (.NOT.GNORTH) THEN
+#ifdef MNH_GPUDIRECT
+ !$acc host_data use_device(ZNORTH2F_IN)
+#else
+ !$acc update host(ZNORTH2F_IN)
+#endif
+ INB_REQ = INB_REQ + 1
+ CALL MPI_ISEND(ZNORTH2F_IN,SIZE(ZNORTH2F_IN) ,MNHREAL_MPI,NP_NORTH-1,1000+IS_NORTH,&
+ NMNH_COMM_WORLD,IREQ(INB_REQ),IERR)
+#ifdef MNH_GPUDIRECT
+ !$acc end host_data
+#endif
+ ENDIF
+ENDIF
+
+!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+
+IF ( INB_REQ > 0 ) THEN
+ CALL MPI_WAITALL(INB_REQ,IREQ,MPI_STATUSES_IGNORE,IERR)
+END IF
+
+!PW:disabled car KO si on compare avec 1 run 1 prc
+! CALL MPPDB_CHECK(PSRC,'UPDATE_HALO_ll::'//TRIM(HNAME))
+
+!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+
+! Update halo in PSRC + %HALO2
+
+!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+
+
+IF (LX) THEN
+ IF (.NOT.GWEST) THEN
+#ifndef MNH_GPUDIRECT
+ !$acc update device(ZWEST2F_OUT) async(IS_WEST)
+#endif
+ ZH2F_WEST => TP_PSRC_HALO2F_ll%HALO2%WEST
+ !$acc kernels async(IS_WEST)
+ PSRC( 1:IIB-1 , IJB:IJE , : ) = ZWEST2F_OUT( 1:IIB-1 , IJB:IJE , : )
+ ZH2F_WEST( IJB:IJE , : ) = ZWEST2F_OUT( IIB-2, IJB:IJE , : )
+ !$acc end kernels
+ ENDIF
+ IF (.NOT.GEAST) THEN
+#ifndef MNH_GPUDIRECT
+ !$acc update device(ZEAST2F_OUT) async(IS_EAST)
+#endif
+ ZH2F_EAST => TP_PSRC_HALO2F_ll%HALO2%EAST
+ !$acc kernels async(IS_EAST)
+ PSRC( IIE+1:IIU , IJB:IJE , : ) = ZEAST2F_OUT( IIE+1:IIU , IJB:IJE , : )
+ ZH2F_EAST( IJB:IJE , : ) = ZEAST2F_OUT( IIE+2 , IJB:IJE , : )
+ !$acc end kernels
+ ENDIF
+END IF
+IF (LY) THEN
+ IF (.NOT.GSOUTH) THEN
+#ifndef MNH_GPUDIRECT
+ !$acc update device(ZSOUTH2F_OUT) async(IS_SOUTH)
+#endif
+ ZH2F_SOUTH => TP_PSRC_HALO2F_ll%HALO2%SOUTH
+ !$acc kernels async(IS_SOUTH)
+ PSRC( IIB:IIE , 1:IJB-1 , : ) = ZSOUTH2F_OUT( IIB:IIE , 1:IJB-1 , : )
+ ZH2F_SOUTH( IIB:IIE , : ) = ZSOUTH2F_OUT( IIB:IIE , IJB-2 , : )
+ !$acc end kernels
+ ENDIF
+ IF (.NOT.GNORTH) THEN
+#ifndef MNH_GPUDIRECT
+ !$acc update device(ZNORTH2F_OUT) async(IS_NORTH)
+#endif
+ ZH2F_NORTH => TP_PSRC_HALO2F_ll%HALO2%NORTH
+ !$acc kernels async(IS_NORTH)
+ PSRC( IIB:IIE , IJE+1:IJU , : ) = ZNORTH2F_OUT ( IIB:IIE , IJE+1:IJU , : )
+ ZH2F_NORTH( IIB:IIE , : ) = ZNORTH2F_OUT ( IIB:IIE , IJE+2 , : )
+ !$acc end kernels
+ ENDIF
+END IF
+!$acc wait
+
+!$acc end data
+
+CALL MPPDB_CHECK(PSRC,"GET_HALO2_DF end:PSRC")
+!CALL MPPDB_CHECK(PSRC,'UPDATE_HALO2_ll::'//TRIM(HNAME))
+
+END SUBROUTINE GET_HALO2_DF
#endif
!-----------------------------------------------------------------------
!