Skip to content
Snippets Groups Projects
Commit 5c99d58e authored by ESCOBAR MUNOZ Juan's avatar ESCOBAR MUNOZ Juan
Browse files

Juan 09/03/2023:ZSOLVER/get_halo.f90, Cray OPENACC Opt, pass ZTAB by arg +... 

Juan 09/03/2023:ZSOLVER/get_halo.f90, Cray OPENACC Opt, pass ZTAB by arg + dims in GET_HALO_START/STOP_D => 30% perf up on Advec MET+UVW !!!
parent fa6b5fdc
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
src/ZSOLVER/get_halo.f90
+ 119
− 73
View file @ 5c99d58e
...@@ -88,23 +88,49 @@ INTERFACE ...@@ -88,23 +88,49 @@ INTERFACE
END INTERFACE END INTERFACE
! !
INTERFACE INTERFACE
SUBROUTINE GET_HALO_START_D(PSRC,KNB_REQ,KREQ,HDIR) SUBROUTINE GET_HALO_START_D(PSRC,KNB_REQ,KREQ,&
PZNORTH_IN , PZSOUTH_IN , PZWEST_IN , PZEAST_IN , &
PZNORTH_OUT, PZSOUTH_OUT, PZWEST_OUT, PZEAST_OUT, &
KIIB,KIIE,KIJB,KIJE,KIIU,KIJU,KIKU,KIHALO_1,&
HDIR)
IMPLICIT NONE IMPLICIT NONE
! !
REAL, DIMENSION(:,:,:), INTENT(INOUT) :: PSRC ! variable at t REAL, DIMENSION(KIIU,KIJU,KIKU), INTENT(INOUT) :: PSRC ! variable at t
! acc declare present (PSRC) ! acc declare present (PSRC)
INTEGER :: KNB_REQ , KREQ(8) INTEGER :: KNB_REQ , KREQ(8)
REAL :: PZSOUTH_IN ( KIIB:KIIE , KIJB:KIJB+KIHALO_1 , KIKU ) ,&
PZNORTH_IN ( KIIB:KIIE , KIJE-KIHALO_1:KIJE , KIKU ) ,&
PZWEST_IN ( KIIB:KIIB+KIHALO_1 , KIJB:KIJE , KIKU ) ,&
PZEAST_IN ( KIIE-KIHALO_1:KIIE , KIJB:KIJE , KIKU ) ,&
!
PZSOUTH_OUT ( KIIB:KIIE , 1:KIJB-1 , KIKU ) ,&
PZNORTH_OUT ( KIIB:KIIE , KIJE+1:KIJU , KIKU ) ,&
PZWEST_OUT ( 1:KIIB-1 , KIJB:KIJE , KIKU ) ,&
PZEAST_OUT ( KIIE+1:KIIU , KIJB:KIJE , KIKU )
INTEGER :: KIIB,KIIE,KIJB,KIJE,KIIU,KIJU,KIKU,KIHALO_1
CHARACTER(len=4), OPTIONAL :: HDIR ! to send only halo on X or Y direction CHARACTER(len=4), OPTIONAL :: HDIR ! to send only halo on X or Y direction
! !
END SUBROUTINE GET_HALO_START_D END SUBROUTINE GET_HALO_START_D
END INTERFACE END INTERFACE
INTERFACE INTERFACE
SUBROUTINE GET_HALO_STOP_D(PSRC,KNB_REQ,KREQ,HDIR) SUBROUTINE GET_HALO_STOP_D(PSRC,KNB_REQ,KREQ,&
PZNORTH_OUT, PZSOUTH_OUT, PZWEST_OUT, PZEAST_OUT, &
KIIB,KIIE,KIJB,KIJE,KIIU,KIJU,KIKU,&
HDIR)
IMPLICIT NONE IMPLICIT NONE
REAL, DIMENSION(:,:,:), INTENT(INOUT) :: PSRC ! variable at t !
REAL, DIMENSION(KIIU,KIJU,KIKU), INTENT(INOUT) :: PSRC ! variable at t
! acc declare present (PSRC) ! acc declare present (PSRC)
INTEGER :: KNB_REQ , KREQ(8) INTEGER :: KNB_REQ , KREQ(8)
REAL :: PZSOUTH_OUT ( KIIB:KIIE , 1:KIJB-1 , KIKU ) ,&
PZNORTH_OUT ( KIIB:KIIE , KIJE+1:KIJU , KIKU ) ,&
PZWEST_OUT ( 1:KIIB-1 , KIJB:KIJE , KIKU ) ,&
PZEAST_OUT ( KIIE+1:KIIU , KIJB:KIJE , KIKU )
INTEGER :: KIIB,KIIE,KIJB,KIJE,KIIU,KIJU,KIKU
CHARACTER(len=4), OPTIONAL :: HDIR ! to send only halo on X or Y direction CHARACTER(len=4), OPTIONAL :: HDIR ! to send only halo on X or Y direction
!
END SUBROUTINE GET_HALO_STOP_D END SUBROUTINE GET_HALO_STOP_D
END INTERFACE END INTERFACE
INTERFACE INTERFACE
...@@ -409,8 +435,9 @@ END MODULE MODD_HALO_D ...@@ -409,8 +435,9 @@ END MODULE MODD_HALO_D
USE MODD_HALO_D USE MODD_HALO_D
!USE MODE_MNH_ZWORK, ONLY : GWEST , GEAST, GSOUTH , GNORTH !USE MODE_MNH_ZWORK, ONLY : GWEST , GEAST, GSOUTH , GNORTH
!USE MODE_MNH_ZWORK, ONLY : IIU,IJU,IKU USE MODE_MNH_ZWORK, ONLY : IIU,IJU,IKU
!USE MODE_MNH_ZWORK, ONLY : IIB,IJB ,IIE,IJE USE MODE_MNH_ZWORK, ONLY : IIB,IJB ,IIE,IJE
!! !!
!USE MODE_DEVICE !USE MODE_DEVICE
USE MODE_MPPDB USE MODE_MPPDB
...@@ -427,22 +454,30 @@ INTEGER :: INB_REQ , IREQ(8) ...@@ -427,22 +454,30 @@ INTEGER :: INB_REQ , IREQ(8)
! !
CALL MPPDB_CHECK(PSRC,"GET_HALO_D big:PSRC") CALL MPPDB_CHECK(PSRC,"GET_HALO_D big:PSRC")
! !
CALL GET_HALO_START_D(PSRC,INB_REQ,IREQ,HDIR) CALL GET_HALO_START_D(PSRC,INB_REQ,IREQ,&
CALL GET_HALO_STOP_D(PSRC,INB_REQ,IREQ,HDIR) ZNORTH_IN , ZSOUTH_IN , ZWEST_IN , ZEAST_IN , &
ZNORTH_OUT, ZSOUTH_OUT, ZWEST_OUT, ZEAST_OUT, &
IIB,IIE,IJB,IJE,IIU,IJU,IKU,IHALO_1,HDIR)
CALL GET_HALO_STOP_D(PSRC,INB_REQ,IREQ,&
ZNORTH_OUT, ZSOUTH_OUT, ZWEST_OUT, ZEAST_OUT, &
IIB,IIE,IJB,IJE,IIU,IJU,IKU,HDIR)
! !
CALL MPPDB_CHECK(PSRC,"GET_HALO_D end:PSRC") CALL MPPDB_CHECK(PSRC,"GET_HALO_D end:PSRC")
! !
END SUBROUTINE GET_HALO_D END SUBROUTINE GET_HALO_D
! ######################### ! #########################
SUBROUTINE GET_HALO_START_D(PSRC,KNB_REQ,KREQ,HDIR) SUBROUTINE GET_HALO_START_D(PSRC,KNB_REQ,KREQ,&
PZNORTH_IN , PZSOUTH_IN , PZWEST_IN , PZEAST_IN , &
PZNORTH_OUT, PZSOUTH_OUT, PZWEST_OUT, PZEAST_OUT, &
KIIB,KIIE,KIJB,KIJE,KIIU,KIJU,KIKU,KIHALO_1,&
HDIR)
! ######################### ! #########################
#define MNH_GPUDIRECT #define MNH_GPUDIRECT
! !
USE MODD_HALO_D USE MODD_HALO_D
USE MODE_MNH_ZWORK, ONLY : GWEST , GEAST, GSOUTH , GNORTH USE MODE_MNH_ZWORK, ONLY : GWEST , GEAST, GSOUTH , GNORTH
USE MODE_MNH_ZWORK, ONLY : IIU,IJU,IKU
USE MODE_MNH_ZWORK, ONLY : IIB,IJB ,IIE,IJE
! !
USE MODD_VAR_ll, ONLY : NMNH_COMM_WORLD USE MODD_VAR_ll, ONLY : NMNH_COMM_WORLD
USE MODD_MPIF, ONLY : MPI_STATUSES_IGNORE USE MODD_MPIF, ONLY : MPI_STATUSES_IGNORE
...@@ -453,9 +488,20 @@ USE MODE_MPPDB ...@@ -453,9 +488,20 @@ USE MODE_MPPDB
! !
IMPLICIT NONE IMPLICIT NONE
! !
REAL, DIMENSION(:,:,:), INTENT(INOUT) :: PSRC ! variable at t REAL, DIMENSION(KIIU,KIJU,KIKU), INTENT(INOUT) :: PSRC ! variable at t
! acc declare present (PSRC) ! acc declare present (PSRC)
INTEGER :: KNB_REQ , KREQ(8) INTEGER :: KNB_REQ , KREQ(8)
REAL :: PZSOUTH_IN ( KIIB:KIIE , KIJB:KIJB+KIHALO_1 , KIKU ) ,&
PZNORTH_IN ( KIIB:KIIE , KIJE-KIHALO_1:KIJE , KIKU ) ,&
PZWEST_IN ( KIIB:KIIB+KIHALO_1 , KIJB:KIJE , KIKU ) ,&
PZEAST_IN ( KIIE-KIHALO_1:KIIE , KIJB:KIJE , KIKU ) ,&
!
PZSOUTH_OUT ( KIIB:KIIE , 1:KIJB-1 , KIKU ) ,&
PZNORTH_OUT ( KIIB:KIIE , KIJE+1:KIJU , KIKU ) ,&
PZWEST_OUT ( 1:KIIB-1 , KIJB:KIJE , KIKU ) ,&
PZEAST_OUT ( KIIE+1:KIIU , KIJB:KIJE , KIKU )
INTEGER :: KIIB,KIIE,KIJB,KIJE,KIIU,KIJU,KIKU,KIHALO_1
CHARACTER(len=4), OPTIONAL :: HDIR ! to send only halo on X or Y direction CHARACTER(len=4), OPTIONAL :: HDIR ! to send only halo on X or Y direction
! !
INTEGER :: IERROR ! error return code INTEGER :: IERROR ! error return code
...@@ -464,19 +510,15 @@ INTEGER,PARAMETER :: IS_WEST=1 , IS_EAST=2, IS_SOUTH=3, IS_NORTH=4 ...@@ -464,19 +510,15 @@ INTEGER,PARAMETER :: IS_WEST=1 , IS_EAST=2, IS_SOUTH=3, IS_NORTH=4
LOGICAL :: LX , LY LOGICAL :: LX , LY
INTEGER :: NB_REQ, IERR INTEGER :: NB_REQ, IERR
! !
INTEGER :: JI,JJ,JK, JIU,JJU,JKU INTEGER :: JI,JJ,JK
JIU = SIZE(PSRC,1)
JJU = SIZE(PSRC,2)
JKU = SIZE(PSRC,3)
CALL INIT_HALO_D() CALL INIT_HALO_D()
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
!$acc data present (PSRC) & !$acc data present (PSRC) &
!$acc present (ZNORTH_IN, ZSOUTH_IN, ZWEST_IN, ZEAST_IN) & !$acc present (PZNORTH_IN, PZSOUTH_IN, PZWEST_IN, PZEAST_IN) &
!$acc present (ZNORTH_OUT, ZSOUTH_OUT, ZWEST_OUT, ZEAST_OUT) !$acc present (PZNORTH_OUT, PZSOUTH_OUT, PZWEST_OUT, PZEAST_OUT)
LX = .FALSE. LX = .FALSE.
...@@ -490,7 +532,7 @@ ELSE ...@@ -490,7 +532,7 @@ ELSE
!!$LY = ( HDIR == "01_Y" .OR. HDIR == "S0_Y" ) !!$LY = ( HDIR == "01_Y" .OR. HDIR == "S0_Y" )
LX = ( HDIR == "01_X" .OR. HDIR == "S0_X" .OR. HDIR == "S0_Y" ) LX = ( HDIR == "01_X" .OR. HDIR == "S0_X" .OR. HDIR == "S0_Y" )
LY = ( HDIR == "01_Y" .OR. HDIR == "S0_Y" .OR. HDIR == "S0_X" ) LY = ( HDIR == "01_Y" .OR. HDIR == "S0_Y" .OR. HDIR == "S0_X" )
!!$print *,"IIB=",IIB," HDIR=",HDIR," LX=",LX," LY=",LY ; call flush(6) !!$print *,"KIIB=",KIIB," HDIR=",HDIR," LX=",LX," LY=",LY ; call flush(6)
END IF END IF
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
...@@ -504,10 +546,10 @@ NB_REQ = 0 ...@@ -504,10 +546,10 @@ NB_REQ = 0
IF (LX) THEN IF (LX) THEN
IF (.NOT. GWEST) THEN IF (.NOT. GWEST) THEN
#ifdef MNH_GPUDIRECT #ifdef MNH_GPUDIRECT
!$acc host_data use_device(ZWEST_OUT) !$acc host_data use_device(PZWEST_OUT)
#endif #endif
NB_REQ = NB_REQ + 1 NB_REQ = NB_REQ + 1
CALL MPI_IRECV(ZWEST_OUT,SIZE(ZWEST_OUT),MNHREAL_MPI,NP_WEST-1,1000+IS_EAST,& CALL MPI_IRECV(PZWEST_OUT,SIZE(PZWEST_OUT),MNHREAL_MPI,NP_WEST-1,1000+IS_EAST,&
NMNH_COMM_WORLD,KREQ(NB_REQ),IERR) NMNH_COMM_WORLD,KREQ(NB_REQ),IERR)
#ifdef MNH_GPUDIRECT #ifdef MNH_GPUDIRECT
!$acc end host_data !$acc end host_data
...@@ -515,10 +557,10 @@ IF (LX) THEN ...@@ -515,10 +557,10 @@ IF (LX) THEN
END IF END IF
IF (.NOT.GEAST) THEN IF (.NOT.GEAST) THEN
#ifdef MNH_GPUDIRECT #ifdef MNH_GPUDIRECT
!$acc host_data use_device(ZEAST_OUT) !$acc host_data use_device(PZEAST_OUT)
#endif #endif
NB_REQ = NB_REQ + 1 NB_REQ = NB_REQ + 1
CALL MPI_IRECV(ZEAST_OUT,SIZE(ZEAST_OUT),MNHREAL_MPI,NP_EAST-1,1000+IS_WEST,& CALL MPI_IRECV(PZEAST_OUT,SIZE(PZEAST_OUT),MNHREAL_MPI,NP_EAST-1,1000+IS_WEST,&
NMNH_COMM_WORLD,KREQ(NB_REQ),IERR) NMNH_COMM_WORLD,KREQ(NB_REQ),IERR)
#ifdef MNH_GPUDIRECT #ifdef MNH_GPUDIRECT
!$acc end host_data !$acc end host_data
...@@ -529,10 +571,10 @@ END IF ...@@ -529,10 +571,10 @@ END IF
IF (LY) THEN IF (LY) THEN
IF (.NOT.GSOUTH) THEN IF (.NOT.GSOUTH) THEN
#ifdef MNH_GPUDIRECT #ifdef MNH_GPUDIRECT
!$acc host_data use_device(ZSOUTH_OUT) !$acc host_data use_device(PZSOUTH_OUT)
#endif #endif
NB_REQ = NB_REQ + 1 NB_REQ = NB_REQ + 1
CALL MPI_IRECV(ZSOUTH_OUT,SIZE(ZSOUTH_OUT),MNHREAL_MPI,NP_SOUTH-1,1000+IS_NORTH,& CALL MPI_IRECV(PZSOUTH_OUT,SIZE(PZSOUTH_OUT),MNHREAL_MPI,NP_SOUTH-1,1000+IS_NORTH,&
NMNH_COMM_WORLD,KREQ(NB_REQ),IERR) NMNH_COMM_WORLD,KREQ(NB_REQ),IERR)
#ifdef MNH_GPUDIRECT #ifdef MNH_GPUDIRECT
!$acc end host_data !$acc end host_data
...@@ -540,10 +582,10 @@ IF (LY) THEN ...@@ -540,10 +582,10 @@ IF (LY) THEN
ENDIF ENDIF
IF (.NOT.GNORTH) THEN IF (.NOT.GNORTH) THEN
#ifdef MNH_GPUDIRECT #ifdef MNH_GPUDIRECT
!$acc host_data use_device(ZNORTH_OUT) !$acc host_data use_device(PZNORTH_OUT)
#endif #endif
NB_REQ = NB_REQ + 1 NB_REQ = NB_REQ + 1
CALL MPI_IRECV(ZNORTH_OUT,SIZE(ZNORTH_OUT),MNHREAL_MPI,NP_NORTH-1,1000+IS_SOUTH,& CALL MPI_IRECV(PZNORTH_OUT,SIZE(PZNORTH_OUT),MNHREAL_MPI,NP_NORTH-1,1000+IS_SOUTH,&
NMNH_COMM_WORLD,KREQ(NB_REQ),IERR) NMNH_COMM_WORLD,KREQ(NB_REQ),IERR)
#ifdef MNH_GPUDIRECT #ifdef MNH_GPUDIRECT
!$acc end host_data !$acc end host_data
...@@ -558,15 +600,15 @@ END IF ...@@ -558,15 +600,15 @@ END IF
IF (LX) THEN IF (LX) THEN
IF (.NOT. GWEST) THEN IF (.NOT. GWEST) THEN
!$acc kernels async(IS_WEST) !$acc kernels async(IS_WEST)
!$mnh_expand_array(JI=IIB:IIB+IHALO_1 , JJ=IJB:IJE , JK=1:JKU ) !$mnh_expand_array(JI=KIIB:KIIB+KIHALO_1 , JJ=KIJB:KIJE , JK=1:KIKU )
ZWEST_IN ( IIB:IIB+IHALO_1 , IJB:IJE , : ) = PSRC( IIB:IIB+IHALO_1 , IJB:IJE , : ) PZWEST_IN ( KIIB:KIIB+KIHALO_1 , KIJB:KIJE , : ) = PSRC( KIIB:KIIB+KIHALO_1 , KIJB:KIJE , : )
!$mnh_end_expand_array() !$mnh_end_expand_array()
!$acc end kernels !$acc end kernels
END IF END IF
IF (.NOT.GEAST) THEN IF (.NOT.GEAST) THEN
!$acc kernels async(IS_EAST) !$acc kernels async(IS_EAST)
!$mnh_expand_array(JI=IIE-IHALO_1:IIE , JJ=IJB:IJE , JK=1:JKU) !$mnh_expand_array(JI=KIIE-KIHALO_1:KIIE , JJ=KIJB:KIJE , JK=1:KIKU)
ZEAST_IN ( IIE-IHALO_1:IIE , IJB:IJE , : ) = PSRC( IIE-IHALO_1:IIE , IJB:IJE , : ) PZEAST_IN ( KIIE-KIHALO_1:KIIE , KIJB:KIJE , : ) = PSRC( KIIE-KIHALO_1:KIIE , KIJB:KIJE , : )
!$mnh_end_expand_array() !$mnh_end_expand_array()
!$acc end kernels !$acc end kernels
ENDIF ENDIF
...@@ -575,15 +617,15 @@ END IF ...@@ -575,15 +617,15 @@ END IF
IF (LY) THEN IF (LY) THEN
IF (.NOT.GSOUTH) THEN IF (.NOT.GSOUTH) THEN
!$acc kernels async(IS_SOUTH) !$acc kernels async(IS_SOUTH)
!$mnh_expand_array(JI=IIB:IIE , JJ=IJB:IJB+IHALO_1 , JK=1:JKU ) !$mnh_expand_array(JI=KIIB:KIIE , JJ=KIJB:KIJB+KIHALO_1 , JK=1:KIKU )
ZSOUTH_IN ( IIB:IIE , IJB:IJB+IHALO_1 , : ) = PSRC( IIB:IIE , IJB:IJB+IHALO_1 , : ) PZSOUTH_IN ( KIIB:KIIE , KIJB:KIJB+KIHALO_1 , : ) = PSRC( KIIB:KIIE , KIJB:KIJB+KIHALO_1 , : )
!$mnh_end_expand_array() !$mnh_end_expand_array()
!$acc end kernels !$acc end kernels
ENDIF ENDIF
IF (.NOT.GNORTH) THEN IF (.NOT.GNORTH) THEN
!$acc kernels async(IS_NORTH) !$acc kernels async(IS_NORTH)
!$mnh_expand_array(JI=IIB:IIE , JJ=IJE-IHALO_1:IJE , JK=1:JKU ) !$mnh_expand_array(JI=KIIB:KIIE , JJ=KIJE-KIHALO_1:KIJE , JK=1:KIKU )
ZNORTH_IN ( IIB:IIE , IJE-IHALO_1:IJE , : ) = PSRC( IIB:IIE , IJE-IHALO_1:IJE , : ) PZNORTH_IN ( KIIB:KIIE , KIJE-KIHALO_1:KIJE , : ) = PSRC( KIIB:KIIE , KIJE-KIHALO_1:KIJE , : )
!$mnh_end_expand_array() !$mnh_end_expand_array()
!$acc end kernels !$acc end kernels
ENDIF ENDIF
...@@ -597,12 +639,12 @@ ENDIF ...@@ -597,12 +639,12 @@ ENDIF
IF (LX) THEN IF (LX) THEN
IF (.NOT. GWEST) THEN IF (.NOT. GWEST) THEN
#ifdef MNH_GPUDIRECT #ifdef MNH_GPUDIRECT
!$acc host_data use_device(ZWEST_IN) !$acc host_data use_device(PZWEST_IN)
#else #else
!$acc update host(ZWEST_IN) !$acc update host(PZWEST_IN)
#endif #endif
NB_REQ = NB_REQ + 1 NB_REQ = NB_REQ + 1
CALL MPI_ISEND(ZWEST_IN,SIZE(ZWEST_IN) ,MNHREAL_MPI,NP_WEST-1,1000+IS_WEST,& CALL MPI_ISEND(PZWEST_IN,SIZE(PZWEST_IN) ,MNHREAL_MPI,NP_WEST-1,1000+IS_WEST,&
NMNH_COMM_WORLD,KREQ(NB_REQ),IERR) NMNH_COMM_WORLD,KREQ(NB_REQ),IERR)
#ifdef MNH_GPUDIRECT #ifdef MNH_GPUDIRECT
!$acc end host_data !$acc end host_data
...@@ -610,12 +652,12 @@ IF (LX) THEN ...@@ -610,12 +652,12 @@ IF (LX) THEN
END IF END IF
IF (.NOT.GEAST) THEN IF (.NOT.GEAST) THEN
#ifdef MNH_GPUDIRECT #ifdef MNH_GPUDIRECT
!$acc host_data use_device(ZEAST_IN) !$acc host_data use_device(PZEAST_IN)
#else #else
!$acc update host(ZEAST_IN) !$acc update host(PZEAST_IN)
#endif #endif
NB_REQ = NB_REQ + 1 NB_REQ = NB_REQ + 1
CALL MPI_ISEND(ZEAST_IN,SIZE(ZEAST_IN) ,MNHREAL_MPI,NP_EAST-1,1000+IS_EAST,& CALL MPI_ISEND(PZEAST_IN,SIZE(PZEAST_IN) ,MNHREAL_MPI,NP_EAST-1,1000+IS_EAST,&
NMNH_COMM_WORLD,KREQ(NB_REQ),IERR) NMNH_COMM_WORLD,KREQ(NB_REQ),IERR)
#ifdef MNH_GPUDIRECT #ifdef MNH_GPUDIRECT
!$acc end host_data !$acc end host_data
...@@ -626,12 +668,12 @@ END IF ...@@ -626,12 +668,12 @@ END IF
IF (LY) THEN IF (LY) THEN
IF (.NOT.GSOUTH) THEN IF (.NOT.GSOUTH) THEN
#ifdef MNH_GPUDIRECT #ifdef MNH_GPUDIRECT
!$acc host_data use_device(ZSOUTH_IN) !$acc host_data use_device(PZSOUTH_IN)
#else #else
!$acc update host(ZSOUTH_IN) !$acc update host(PZSOUTH_IN)
#endif #endif
NB_REQ = NB_REQ + 1 NB_REQ = NB_REQ + 1
CALL MPI_ISEND(ZSOUTH_IN,SIZE(ZSOUTH_IN) ,MNHREAL_MPI,NP_SOUTH-1,1000+IS_SOUTH,& CALL MPI_ISEND(PZSOUTH_IN,SIZE(PZSOUTH_IN) ,MNHREAL_MPI,NP_SOUTH-1,1000+IS_SOUTH,&
NMNH_COMM_WORLD,KREQ(NB_REQ),IERR) NMNH_COMM_WORLD,KREQ(NB_REQ),IERR)
#ifdef MNH_GPUDIRECT #ifdef MNH_GPUDIRECT
!$acc end host_data !$acc end host_data
...@@ -639,12 +681,12 @@ IF (LY) THEN ...@@ -639,12 +681,12 @@ IF (LY) THEN
ENDIF ENDIF
IF (.NOT.GNORTH) THEN IF (.NOT.GNORTH) THEN
#ifdef MNH_GPUDIRECT #ifdef MNH_GPUDIRECT
!$acc host_data use_device(ZNORTH_IN) !$acc host_data use_device(PZNORTH_IN)
#else #else
!$acc update host(ZNORTH_IN) !$acc update host(PZNORTH_IN)
#endif #endif
NB_REQ = NB_REQ + 1 NB_REQ = NB_REQ + 1
CALL MPI_ISEND(ZNORTH_IN,SIZE(ZNORTH_IN) ,MNHREAL_MPI,NP_NORTH-1,1000+IS_NORTH,& CALL MPI_ISEND(PZNORTH_IN,SIZE(PZNORTH_IN) ,MNHREAL_MPI,NP_NORTH-1,1000+IS_NORTH,&
NMNH_COMM_WORLD,KREQ(NB_REQ),IERR) NMNH_COMM_WORLD,KREQ(NB_REQ),IERR)
#ifdef MNH_GPUDIRECT #ifdef MNH_GPUDIRECT
!$acc end host_data !$acc end host_data
...@@ -659,15 +701,18 @@ KNB_REQ = NB_REQ ...@@ -659,15 +701,18 @@ KNB_REQ = NB_REQ
END SUBROUTINE GET_HALO_START_D END SUBROUTINE GET_HALO_START_D
! !
! ######################### ! #########################
SUBROUTINE GET_HALO_STOP_D(PSRC,KNB_REQ,KREQ,HDIR) SUBROUTINE GET_HALO_STOP_D(PSRC,KNB_REQ,KREQ,&
PZNORTH_OUT, PZSOUTH_OUT, PZWEST_OUT, PZEAST_OUT, &
KIIB,KIIE,KIJB,KIJE,KIIU,KIJU,KIKU,&
HDIR)
! ######################### ! #########################
#define MNH_GPUDIRECT #define MNH_GPUDIRECT
! !
USE MODD_HALO_D USE MODD_HALO_D
USE MODE_MNH_ZWORK, ONLY : GWEST , GEAST, GSOUTH , GNORTH USE MODE_MNH_ZWORK, ONLY : GWEST , GEAST, GSOUTH , GNORTH
USE MODE_MNH_ZWORK, ONLY : IIU,IJU,IKU !!$USE MODE_MNH_ZWORK, ONLY : IIU,IJU,IKU
USE MODE_MNH_ZWORK, ONLY : IIB,IJB ,IIE,IJE !!$USE MODE_MNH_ZWORK, ONLY : IIB,IJB ,IIE,IJE
! !
USE MODD_VAR_ll, ONLY : NMNH_COMM_WORLD USE MODD_VAR_ll, ONLY : NMNH_COMM_WORLD
USE MODD_MPIF, ONLY : MPI_STATUSES_IGNORE USE MODD_MPIF, ONLY : MPI_STATUSES_IGNORE
...@@ -678,9 +723,15 @@ USE MODE_MPPDB ...@@ -678,9 +723,15 @@ USE MODE_MPPDB
! !
IMPLICIT NONE IMPLICIT NONE
! !
REAL, DIMENSION(:,:,:), INTENT(INOUT) :: PSRC ! variable at t REAL, DIMENSION(KIIU,KIJU,KIKU), INTENT(INOUT) :: PSRC ! variable at t
! acc declare present (PSRC) ! acc declare present (PSRC)
INTEGER :: KNB_REQ , KREQ(8) INTEGER :: KNB_REQ , KREQ(8)
REAL :: PZSOUTH_OUT ( KIIB:KIIE , 1:KIJB-1 , KIKU ) ,&
PZNORTH_OUT ( KIIB:KIIE , KIJE+1:KIJU , KIKU ) ,&
PZWEST_OUT ( 1:KIIB-1 , KIJB:KIJE , KIKU ) ,&
PZEAST_OUT ( KIIE+1:KIIU , KIJB:KIJE , KIKU )
INTEGER :: KIIB,KIIE,KIJB,KIJE,KIIU,KIJU,KIKU
CHARACTER(len=4), OPTIONAL :: HDIR ! to send only halo on X or Y direction CHARACTER(len=4), OPTIONAL :: HDIR ! to send only halo on X or Y direction
! !
INTEGER :: IERROR ! error return code INTEGER :: IERROR ! error return code
...@@ -689,19 +740,14 @@ INTEGER,PARAMETER :: IS_WEST=1 , IS_EAST=2, IS_SOUTH=3, IS_NORTH=4 ...@@ -689,19 +740,14 @@ INTEGER,PARAMETER :: IS_WEST=1 , IS_EAST=2, IS_SOUTH=3, IS_NORTH=4
LOGICAL :: LX , LY LOGICAL :: LX , LY
INTEGER :: NB_REQ, IERR INTEGER :: NB_REQ, IERR
! !
INTEGER :: JI,JJ,JK, JIU,JJU,JKU INTEGER :: JI,JJ,JK
JIU = SIZE(PSRC,1)
JJU = SIZE(PSRC,2)
JKU = SIZE(PSRC,3)
CALL INIT_HALO_D() CALL INIT_HALO_D()
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
!$acc data present (PSRC) & !$acc data present (PSRC) &
!$acc present (ZNORTH_IN, ZSOUTH_IN, ZWEST_IN, ZEAST_IN) & !$acc & present (PZNORTH_OUT,PZSOUTH_OUT,PZWEST_OUT,PZEAST_OUT)
!$acc present (ZNORTH_OUT, ZSOUTH_OUT, ZWEST_OUT, ZEAST_OUT)
LX = .FALSE. LX = .FALSE.
LY = .FALSE. LY = .FALSE.
...@@ -714,7 +760,7 @@ ELSE ...@@ -714,7 +760,7 @@ ELSE
!!$LY = ( HDIR == "01_Y" .OR. HDIR == "S0_Y" ) !!$LY = ( HDIR == "01_Y" .OR. HDIR == "S0_Y" )
LX = ( HDIR == "01_X" .OR. HDIR == "S0_X" .OR. HDIR == "S0_Y" ) LX = ( HDIR == "01_X" .OR. HDIR == "S0_X" .OR. HDIR == "S0_Y" )
LY = ( HDIR == "01_Y" .OR. HDIR == "S0_Y" .OR. HDIR == "S0_X" ) LY = ( HDIR == "01_Y" .OR. HDIR == "S0_Y" .OR. HDIR == "S0_X" )
!!$print *,"IIB=",IIB," HDIR=",HDIR," LX=",LX," LY=",LY ; call flush(6) !!$print *,"KIIB=",KIIB," HDIR=",HDIR," LX=",LX," LY=",LY ; call flush(6)
END IF END IF
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
...@@ -730,21 +776,21 @@ CALL MPI_WAITALL(NB_REQ,KREQ,MPI_STATUSES_IGNORE,IERR) ...@@ -730,21 +776,21 @@ CALL MPI_WAITALL(NB_REQ,KREQ,MPI_STATUSES_IGNORE,IERR)
IF (LX) THEN IF (LX) THEN
IF (.NOT.GWEST) THEN IF (.NOT.GWEST) THEN
#ifndef MNH_GPUDIRECT #ifndef MNH_GPUDIRECT
!$acc update device(ZWEST_OUT) async(IS_WEST) !$acc update device(PZWEST_OUT) async(IS_WEST)
#endif #endif
!$acc kernels async(IS_WEST) !$acc kernels async(IS_WEST)
!$mnh_expand_array(JI=1:IIB-1 , JJ=IJB:IJE , JK=1:JKU ) !$mnh_expand_array(JI=1:KIIB-1 , JJ=KIJB:KIJE , JK=1:KIKU )
PSRC( 1:IIB-1 , IJB:IJE , : ) = ZWEST_OUT( 1:IIB-1 , IJB:IJE , : ) PSRC( 1:KIIB-1 , KIJB:KIJE , : ) = PZWEST_OUT( 1:KIIB-1 , KIJB:KIJE , : )
!$mnh_end_expand_array() !$mnh_end_expand_array()
!$acc end kernels !$acc end kernels
ENDIF ENDIF
IF (.NOT.GEAST) THEN IF (.NOT.GEAST) THEN
#ifndef MNH_GPUDIRECT #ifndef MNH_GPUDIRECT
!$acc update device(ZEAST_OUT) async(IS_EAST) !$acc update device(PZEAST_OUT) async(IS_EAST)
#endif #endif
!$acc kernels async(IS_EAST) !$acc kernels async(IS_EAST)
!$mnh_expand_array(JI=IIE+1:IIU , JJ=IJB:IJE , JK=1:JKU ) !$mnh_expand_array(JI=KIIE+1:KIIU , JJ=KIJB:KIJE , JK=1:KIKU )
PSRC( IIE+1:IIU , IJB:IJE , : ) = ZEAST_OUT( IIE+1:IIU , IJB:IJE , : ) PSRC( KIIE+1:KIIU , KIJB:KIJE , : ) = PZEAST_OUT( KIIE+1:KIIU , KIJB:KIJE , : )
!$mnh_end_expand_array() !$mnh_end_expand_array()
!$acc end kernels !$acc end kernels
ENDIF ENDIF
...@@ -752,21 +798,21 @@ END IF ...@@ -752,21 +798,21 @@ END IF
IF (LY) THEN IF (LY) THEN
IF (.NOT.GSOUTH) THEN IF (.NOT.GSOUTH) THEN
#ifndef MNH_GPUDIRECT #ifndef MNH_GPUDIRECT
!$acc update device(ZSOUTH_OUT) async(IS_SOUTH) !$acc update device(PZSOUTH_OUT) async(IS_SOUTH)
#endif #endif
!$acc kernels async(IS_SOUTH) !$acc kernels async(IS_SOUTH)
!$mnh_expand_array(JI=IIB:IIE , JJ=1:IJB-1 , JK=1:JKU ) !$mnh_expand_array(JI=KIIB:KIIE , JJ=1:KIJB-1 , JK=1:KIKU )
PSRC( IIB:IIE , 1:IJB-1 , : ) = ZSOUTH_OUT( IIB:IIE , 1:IJB-1 , : ) PSRC( KIIB:KIIE , 1:KIJB-1 , : ) = PZSOUTH_OUT( KIIB:KIIE , 1:KIJB-1 , : )
!$mnh_end_expand_array() !$mnh_end_expand_array()
!$acc end kernels !$acc end kernels
ENDIF ENDIF
IF (.NOT.GNORTH) THEN IF (.NOT.GNORTH) THEN
#ifndef MNH_GPUDIRECT #ifndef MNH_GPUDIRECT
!$acc update device(ZNORTH_OUT) async(IS_NORTH) !$acc update device(PZNORTH_OUT) async(IS_NORTH)
#endif #endif
!$acc kernels async(IS_NORTH) !$acc kernels async(IS_NORTH)
!$mnh_expand_array(JI=IIB:IIE , JJ=IJE+1:IJU , JK=1:JKU ) !$mnh_expand_array(JI=KIIB:KIIE , JJ=KIJE+1:KIJU , JK=1:KIKU )
PSRC( IIB:IIE , IJE+1:IJU , : ) = ZNORTH_OUT ( IIB:IIE , IJE+1:IJU , : ) PSRC( KIIB:KIIE , KIJE+1:KIJU , : ) = PZNORTH_OUT ( KIIB:KIIE , KIJE+1:KIJU , : )
!$mnh_end_expand_array() !$mnh_end_expand_array()
!$acc end kernels !$acc end kernels
ENDIF ENDIF
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment

Documentation | Mentions légales | CGU | Accessibilité : non conforme