diff --git a/src/ZSOLVER/advec_4th_order_aux.f90 b/src/ZSOLVER/advec_4th_order_aux.f90 index 041ac45177d492ee723bffc082f27c942348466b..9838c4762cf61ca7e0ba265ebd579076e5cc3352 100644 --- a/src/ZSOLVER/advec_4th_order_aux.f90 +++ b/src/ZSOLVER/advec_4th_order_aux.f90 @@ -27,24 +27,40 @@ END SUBROUTINE ADVEC_4TH_ORDER_ALGO ! !------------------------------------------------------------------------------- ! +#ifndef MNH_OPENACC FUNCTION MZF4(PA) RESULT(PMZF4) +#else + SUBROUTINE MZF4(PA,PMZF4) +#endif ! REAL, DIMENSION(:,:,:), INTENT(IN) :: PA ! variable at flux ! side REAL, DIMENSION(SIZE(PA,1),SIZE(PA,2),SIZE(PA,3)) :: PMZF4 ! result at mass ! localization ! -END FUNCTION MZF4 +#ifndef MNH_OPENACC + END FUNCTION MZF4 +#else + END SUBROUTINE MZF4 +#endif ! !------------------------------------------------------------------------------- ! +#ifndef MNH_OPENACC FUNCTION MZM4(PA) RESULT(PMZM4) +#else + SUBROUTINE MZM4(PA,PMZM4) +#endif ! REAL, DIMENSION(:,:,:), INTENT(IN) :: PA ! variable at mass ! localization REAL, DIMENSION(SIZE(PA,1),SIZE(PA,2),SIZE(PA,3)) :: PMZM4 ! result at flux ! localization -END FUNCTION MZM4 +#ifndef MNH_OPENACC + END FUNCTION MZM4 +#else + END SUBROUTINE MZM4 +#endif ! !------------------------------------------------------------------------------- ! @@ -143,13 +159,8 @@ INTEGER:: ILUOUT,IRESP ! for prints ! JUAN ACC LOGICAL :: GWEST , GEAST LOGICAL :: GSOUTH , GNORTH -#ifndef MNH_OPENACC -REAL, DIMENSION(:,:), ALLOCATABLE :: ZHALO2_WEST, ZHALO2_EAST -REAL, DIMENSION(:,:), ALLOCATABLE :: ZHALO2_SOUTH, ZHALO2_NORTH -#else REAL, DIMENSION(:,:), pointer, contiguous :: ZHALO2_WEST, ZHALO2_EAST REAL, DIMENSION(:,:), pointer, contiguous :: ZHALO2_SOUTH, ZHALO2_NORTH -#endif ! !$acc data present( PMEANX, PMEANY, PFIELDT ) @@ -159,34 +170,11 @@ IF (MPPDB_INITIALIZED) THEN CALL MPPDB_CHECK(PFIELDT,"ADVEC_4TH_ORDER_ALGO beg:PFIELDT") END IF -#ifndef MNH_OPENACC -allocate( zhalo2_west ( size( pfieldt, 2 ), size( pfieldt, 3 ) ) ) -allocate( zhalo2_east ( size( pfieldt, 2 ), size( pfieldt, 3 ) ) ) -allocate( zhalo2_south( size( pfieldt, 2 ), size( pfieldt, 3 ) ) ) -allocate( zhalo2_north( size( pfieldt, 2 ), size( pfieldt, 3 ) ) ) -#else -!Pin positions in the pools of MNH memory -CALL MNH_MEM_POSITION_PIN( 'ADVEC_4TH_ORDER_ALGO' ) - -CALL MNH_MEM_GET( zhalo2_west, size( pfieldt, 2 ), size( pfieldt, 3 ) ) -CALL MNH_MEM_GET( zhalo2_east, size( pfieldt, 2 ), size( pfieldt, 3 ) ) -CALL MNH_MEM_GET( zhalo2_south, size( pfieldt, 2 ), size( pfieldt, 3 ) ) -CALL MNH_MEM_GET( zhalo2_north, size( pfieldt, 2 ), size( pfieldt, 3 ) ) - -!$acc data present ( zhalo2_west, zhalo2_east, zhalo2_south, zhalo2_north ) -#endif - !------------------------------------------------------------------------------- ! !* 0.3. COMPUTES THE DOMAIN DIMENSIONS ! ------------------------------ ! -#ifdef MNH_OPENACC -CALL INIT_ON_HOST_AND_DEVICE(ZHALO2_WEST,-1e99,'ADVEC_4TH_ORDER_ALGO::ZHALO2_WEST') -CALL INIT_ON_HOST_AND_DEVICE(ZHALO2_EAST,-1e99,'ADVEC_4TH_ORDER_ALGO::ZHALO2_EAST') -CALL INIT_ON_HOST_AND_DEVICE(ZHALO2_SOUTH,-1e99,'ADVEC_4TH_ORDER_ALGO::ZHALO2_SOUTH') -CALL INIT_ON_HOST_AND_DEVICE(ZHALO2_NORTH,-1e99,'ADVEC_4TH_ORDER_ALGO::ZHALO2_NORTH') -#endif ! CALL GET_INDICE_ll(IIB,IJB,IIE,IJE) ! @@ -220,11 +208,10 @@ CASE ('CYCL') ! In that case one must have HLBCX(1) == HLBCX(2) #ifdef MNH_OPENACC call Print_msg( NVERB_WARNING, 'GEN', 'ADVEC_4TH_ORDER_ALGO', 'OpenACC: HLBCX(1) AND CYCL not yet tested' ) #endif -ZHALO2_WEST(:,:) = TPHALO2%WEST(:,:) -ZHALO2_EAST(:,:) = TPHALO2%EAST(:,:) -!$acc update device (ZHALO2_WEST,ZHALO2_EAST) +ZHALO2_WEST => TPHALO2%WEST +ZHALO2_EAST => TPHALO2%EAST ! -!$acc kernels present(PMEANX) +!$acc kernels present(PMEANX,ZHALO2_WEST,ZHALO2_EAST) IW=IIB+1 IE=IIE ! @@ -265,11 +252,10 @@ ZHALO2_EAST(:,:) = TPHALO2%EAST(:,:) ! CASE ('OPEN','WALL','NEST') ! -ZHALO2_WEST(:,:) = TPHALO2%WEST(:,:) -ZHALO2_EAST(:,:) = TPHALO2%EAST(:,:) -!$acc update device (ZHALO2_WEST,ZHALO2_EAST) +ZHALO2_WEST => TPHALO2%WEST +ZHALO2_EAST => TPHALO2%EAST ! -!$acc kernels present(PMEANX) +!$acc kernels present(PMEANX,ZHALO2_WEST,ZHALO2_EAST) IF (GWEST) THEN IF(KGRID == 2) THEN IW=IIB+2 ! special case of C grid @@ -348,11 +334,10 @@ IF ( .NOT. L2D ) THEN #ifdef MNH_OPENACC call Print_msg( NVERB_WARNING, 'GEN', 'ADVEC_4TH_ORDER_ALGO', 'OpenACC: HLBCX(2) AND CYCL not yet tested' ) #endif -ZHALO2_SOUTH(:,:) = TPHALO2%SOUTH(:,:) -ZHALO2_NORTH(:,:) = TPHALO2%NORTH(:,:) -!$acc update device (ZHALO2_SOUTH,ZHALO2_NORTH) +ZHALO2_SOUTH => TPHALO2%SOUTH +ZHALO2_NORTH => TPHALO2%NORTH ! -!$acc kernels present(PMEANY) +!$acc kernels present(PMEANY,ZHALO2_SOUTH,ZHALO2_NORTH) ! ! IS=IJB+1 @@ -392,12 +377,11 @@ ZHALO2_NORTH(:,:) = TPHALO2%NORTH(:,:) !* 2.2 NON CYCLIC CASE IN THE Y DIRECTION ! CASE ('OPEN','WALL','NEST') +! +ZHALO2_SOUTH => TPHALO2%SOUTH +ZHALO2_NORTH => TPHALO2%NORTH ! -ZHALO2_SOUTH(:,:) = TPHALO2%SOUTH(:,:) -ZHALO2_NORTH(:,:) = TPHALO2%NORTH(:,:) -!$acc update device (ZHALO2_SOUTH,ZHALO2_NORTH) -! -!$acc kernels present(PMEANY) +!$acc kernels present(PMEANY,ZHALO2_SOUTH,ZHALO2_NORTH) IF (GSOUTH) THEN IF(KGRID == 3) THEN IS=IJB+2 ! special case of C grid @@ -475,13 +459,6 @@ END IF !$acc end data -#ifdef MNH_OPENACC -!Release all memory allocated with MNH_MEM_GET calls since last call to MNH_MEM_POSITION_PIN -CALL MNH_MEM_RELEASE( 'ADVEC_4TH_ORDER_ALGO' ) -#endif - -!$acc end data - !------------------------------------------------------------------------------- ! END SUBROUTINE ADVEC_4TH_ORDER_ALGO @@ -489,7 +466,11 @@ END SUBROUTINE ADVEC_4TH_ORDER_ALGO !------------------------------------------------------------------------------- ! ! ################################ +#ifndef MNH_OPENACC FUNCTION MZF4(PA) RESULT(PMZF4) +#else + SUBROUTINE MZF4(PA,PMZF4) +#endif ! ################################ ! !!**** *MZF4* - 4th order Shuman operator : mean operator in z direction for a @@ -610,12 +591,20 @@ END DO !------------------------------------------------------------------------------- ! -END FUNCTION MZF4 +#ifndef MNH_OPENACC + END FUNCTION MZF4 +#else + END SUBROUTINE MZF4 +#endif ! !------------------------------------------------------------------------------- ! ! ################################ +#ifndef MNH_OPENACC FUNCTION MZM4(PA) RESULT(PMZM4) +#else + SUBROUTINE MZM4(PA,PMZM4) +#endif ! ################################ ! !!**** *MZM4* - 4th order Shuman operator : mean operator in z direction for a @@ -732,4 +721,8 @@ END DO !------------------------------------------------------------------------------- ! -END FUNCTION MZM4 +#ifndef MNH_OPENACC + END FUNCTION MZM4 +#else + END SUBROUTINE MZM4 +#endif diff --git a/src/ZSOLVER/advection_uvw_cen.f90 b/src/ZSOLVER/advection_uvw_cen.f90 index 289737f14f2e05774aa9ce3acba37a52c2e1f3d7..313a36457d62b7b2e3433bcd700663cfbd5034d9 100644 --- a/src/ZSOLVER/advection_uvw_cen.f90 +++ b/src/ZSOLVER/advection_uvw_cen.f90 @@ -16,7 +16,11 @@ INTERFACE PUT, PVT, PWT, & PRHODJ, PDXX, PDYY, PDZZ, PDZX, PDZY, & PRUS,PRVS, PRWS, & +#ifndef MNH_OPENACC TPHALO2MLIST ) +#else + TPHALO2_UT,TPHALO2_VT,TPHALO2_WT ) +#endif ! USE MODD_ARGSLIST_ll, ONLY : HALO2LIST_ll ! @@ -37,7 +41,11 @@ REAL, DIMENSION(:,:,:), INTENT(INOUT) :: PRUS , PRVS , PRWS ! Sources terms ! ! halo lists for 4th order advection +#ifndef MNH_OPENACC TYPE(HALO2LIST_ll), POINTER :: TPHALO2MLIST ! momentum variables +#else +TYPE(HALO2LIST_ll), POINTER :: TPHALO2_UT,TPHALO2_VT,TPHALO2_WT +#endif ! END SUBROUTINE ADVECTION_UVW_CEN ! @@ -53,7 +61,11 @@ END MODULE MODI_ADVECTION_UVW_CEN PUT, PVT, PWT, & PRHODJ, PDXX, PDYY, PDZZ, PDZX, PDZY, & PRUS,PRVS, PRWS, & +#ifndef MNH_OPENACC TPHALO2MLIST ) +#else + TPHALO2_UT,TPHALO2_VT,TPHALO2_WT ) +#endif ! ########################################################################## ! !!**** *ADVECTION * - routine to call the specialized advection routines @@ -141,7 +153,11 @@ REAL, DIMENSION(:,:,:), INTENT(INOUT) :: PRUS , PRVS , PRWS ! Sources terms ! ! halo lists for 4th order advection +#ifndef MNH_OPENACC TYPE(HALO2LIST_ll), POINTER :: TPHALO2MLIST ! momentum variables +#else +TYPE(HALO2LIST_ll), POINTER :: TPHALO2_UT,TPHALO2_VT,TPHALO2_WT +#endif ! ! !* 0.2 declarations of local variables @@ -378,7 +394,12 @@ IF (HUVW_ADV_SCHEME=='CEN2ND' ) THEN ELSEIF (HUVW_ADV_SCHEME=='CEN4TH') THEN ! CALL ADVECUVW_4TH ( HLBCX, HLBCY, ZRUCT, ZRVCT, ZRWCT, & - PUT, PVT, PWT, ZRUS, ZRVS, ZRWS, TPHALO2MLIST ) + PUT, PVT, PWT, ZRUS, ZRVS, ZRWS, & +#ifndef MNH_OPENACC + TPHALO2MLIST ) +#else + TPHALO2_UT,TPHALO2_VT,TPHALO2_WT ) +#endif ! END IF ! diff --git a/src/ZSOLVER/advecuvw_4th.f90 b/src/ZSOLVER/advecuvw_4th.f90 index d523b56f51fcb0cc6d9f1c9edced72c873377657..5705d338cd931d3d5e538cc72088273bcbcfbb92 100644 --- a/src/ZSOLVER/advecuvw_4th.f90 +++ b/src/ZSOLVER/advecuvw_4th.f90 @@ -10,7 +10,12 @@ INTERFACE ! SUBROUTINE ADVECUVW_4TH ( HLBCX, HLBCY, PRUCT, PRVCT, PRWCT, & - PUT, PVT, PWT, PRUS, PRVS, PRWS, TPHALO2LIST ) + PUT, PVT, PWT, PRUS, PRVS, PRWS, & +#ifndef MNH_OPENACC + TPHALO2LIST ) +#else + TPHALO2_UT,TPHALO2_VT,TPHALO2_WT ) +#endif ! USE MODD_ARGSLIST_ll, ONLY : HALO2LIST_ll ! @@ -25,7 +30,11 @@ REAL, DIMENSION(:,:,:), INTENT(IN) :: PUT, PVT, PWT ! U,V,W at t ! REAL, DIMENSION(:,:,:), INTENT(INOUT) :: PRUS, PRVS, PRWS ! Source terms ! +#ifndef MNH_OPENACC TYPE(HALO2LIST_ll), POINTER :: TPHALO2LIST ! list for diffusion +#else +TYPE(HALO2LIST_ll), POINTER :: TPHALO2_UT,TPHALO2_VT,TPHALO2_WT +#endif ! END SUBROUTINE ADVECUVW_4TH ! @@ -36,7 +45,12 @@ END MODULE MODI_ADVECUVW_4TH ! ! ###################################################################### SUBROUTINE ADVECUVW_4TH ( HLBCX, HLBCY, PRUCT, PRVCT, PRWCT, & - PUT, PVT, PWT, PRUS, PRVS, PRWS, TPHALO2LIST ) + PUT, PVT, PWT, PRUS, PRVS, PRWS, & +#ifndef MNH_OPENACC + TPHALO2LIST ) +#else + TPHALO2_UT,TPHALO2_VT,TPHALO2_WT ) +#endif ! ###################################################################### ! !!**** *ADVECUVW_4TH * - routine to compute the 4th order centered @@ -141,7 +155,11 @@ REAL, DIMENSION(:,:,:), INTENT(IN) :: PUT, PVT, PWT ! Variables at t ! REAL, DIMENSION(:,:,:), INTENT(INOUT) :: PRUS, PRVS, PRWS ! Source terms ! +#ifndef MNH_OPENACC TYPE(HALO2LIST_ll), POINTER :: TPHALO2LIST ! list for diffusion +#else +TYPE(HALO2LIST_ll), POINTER :: TPHALO2_UT,TPHALO2_VT,TPHALO2_WT +#endif ! !* 0.2 Declarations of local variables : ! @@ -216,7 +234,11 @@ CALL MNH_MEM_GET( ZTEMP4, IIU, IJU, IKU ) ! IGRID = 2 !!$IF(NHALO == 1) THEN +#ifndef MNH_OPENACC TZHALO2LIST => TPHALO2LIST +#else + TZHALO2LIST => TPHALO2_UT +#endif CALL ADVEC_4TH_ORDER_ALGO(HLBCX, HLBCY, PUT, IGRID, ZMEANX, ZMEANY, & TZHALO2LIST%HALO2 ) !!$ELSE @@ -251,7 +273,7 @@ call dyf_device(ZTEMP2,ZTEMP3) PRUS(:,:,:) = PRUS(:,:,:) - ZTEMP3 !$acc end kernels ! -ZTEMP1 = MZM4( PUT ) +call MZM4( PUT , ZTEMP1 ) call mxm_device(PRWCT,ZTEMP2) !$acc kernels ZTEMP3 = ZTEMP1 * ZTEMP2 @@ -265,7 +287,11 @@ PRUS(:,:,:) = PRUS(:,:,:) - ZTEMP4 ! IGRID = 3 !!$IF(NHALO == 1) THEN +#ifndef MNH_OPENACC TZHALO2LIST => TZHALO2LIST%NEXT +#else + TZHALO2LIST => TPHALO2_VT +#endif CALL ADVEC_4TH_ORDER_ALGO(HLBCX, HLBCY, PVT, IGRID, ZMEANX, ZMEANY, & TZHALO2LIST%HALO2 ) !!$ELSE @@ -301,7 +327,7 @@ PRVS(:,:,:) = PRVS(:,:,:) - ZTEMP3 !$acc end kernels ! call mym_device(PRWCT,ZTEMP1) -ZTEMP2 = MZM4( PVT ) +CALL MZM4( PVT , ZTEMP2) !$acc kernels ZTEMP3 = ZTEMP1 * ZTEMP2 !$acc end kernels @@ -316,7 +342,11 @@ CALL MPPDB_CHECK(PRUCT,"ADVECUVW_4TH 02: PRUCT") IGRID = 4 ! !!$IF(NHALO == 1) THEN +#ifndef MNH_OPENACC TZHALO2LIST => TZHALO2LIST%NEXT +#else + TZHALO2LIST => TPHALO2_WT +#endif CALL ADVEC_4TH_ORDER_ALGO(HLBCX, HLBCY, PWT, IGRID, ZMEANX, ZMEANY, & TZHALO2LIST%HALO2 ) !!$ELSE @@ -352,7 +382,7 @@ PRWS(:,:,:) = PRWS(:,:,:) - ZTEMP3 !$acc end kernels ! call mzf_device( PRWCT, ZTEMP1 ) -ZTEMP2 = MZF4( PWT ) +CALL MZF4( PWT , ZTEMP2 ) !$acc kernels ZTEMP1 = ZTEMP1 * ZTEMP2 !$acc end kernels diff --git a/src/ZSOLVER/advecuvw_rk.f90 b/src/ZSOLVER/advecuvw_rk.f90 index 831fb4385b589548272dcb90ca3f64f6a72a5238..4ffe3257610dd7e3cbfcea9ebbe733039379aa0b 100644 --- a/src/ZSOLVER/advecuvw_rk.f90 +++ b/src/ZSOLVER/advecuvw_rk.f90 @@ -529,12 +529,16 @@ ENDIF #endif ELSE IF ((HUVW_ADV_SCHEME=='CEN4TH') .AND. (HTEMP_SCHEME=='RKC4')) THEN #ifdef MNH_OPENACC - STOP "HUVW_ADV_SCHEME=='CEN4TH') .AND. (HTEMP_SCHEME=='RKC4') NOT TESTED WITH OPENACC" + !STOP "HUVW_ADV_SCHEME=='CEN4TH') .AND. (HTEMP_SCHEME=='RKC4') NOT TESTED WITH OPENACC" #endif CALL ADVECUVW_4TH (HLBCX, HLBCY, PRUCT, PRVCT, PRWCT, & ZUT, ZVT, ZWT, & ZRUS(:,:,:,JS), ZRVS(:,:,:,JS), ZRWS(:,:,:,JS), & - TZHALO2MT_ll ) +#ifndef MNH_OPENACC + TZHALO2MT_ll ) +#else + TZHALO2_UT,TZHALO2_VT,TZHALO2_WT ) +#endif ENDIF ! IF (GIBM_LOWORD) THEN diff --git a/src/ZSOLVER/modeln.f90 b/src/ZSOLVER/modeln.f90 index c36c99582f7efc80ed9e44bfffa215911304c774..9b7ece1a6c90222eea6f1d307f74324ef4fa6ed8 100644 --- a/src/ZSOLVER/modeln.f90 +++ b/src/ZSOLVER/modeln.f90 @@ -560,6 +560,10 @@ REAL, DIMENSION(:,:,:), ALLOCATABLE :: ZJ ! TYPE(LIST_ll), POINTER :: TZFIELDC_ll ! list of fields to exchange TYPE(HALO2LIST_ll), POINTER :: TZHALO2C_ll ! list of fields to exchange +#ifdef MNH_OPENACC +TYPE(HALO2LIST_ll), SAVE , POINTER :: TZHALO2_UT,TZHALO2_VT,TZHALO2_WT +LOGICAL , SAVE :: GFIRST_CALL_MODELN = .TRUE. +#endif LOGICAL :: GCLD ! conditionnal call for dust wet deposition LOGICAL :: GCLOUD_ONLY ! conditionnal radiation computations for ! the only cloudy columns @@ -1711,9 +1715,23 @@ IF ((CUVW_ADV_SCHEME(1:3)=='CEN') .AND. (CTEMP_SCHEME == 'LEFR')) THEN CALL ADD3DFIELD_ll( TZFIELDC_ll, XUT, 'MODEL_n::XUT' ) CALL ADD3DFIELD_ll( TZFIELDC_ll, XVT, 'MODEL_n::XVT' ) CALL ADD3DFIELD_ll( TZFIELDC_ll, XWT, 'MODEL_n::XWT' ) +#ifndef MNH_OPENACC CALL INIT_HALO2_ll(TZHALO2C_ll,3,IIU,IJU,IKU) CALL UPDATE_HALO_ll(TZFIELDC_ll,IINFO_ll) CALL UPDATE_HALO2_ll(TZFIELDC_ll, TZHALO2C_ll, IINFO_ll) +#else + IF (GFIRST_CALL_MODELN) THEN + GFIRST_CALL_MODELN = .FALSE. + NULLIFY(TZHALO2_UT,TZHALO2_VT,TZHALO2_WT) + CALL INIT_HALO2_ll(TZHALO2_UT,1,IIU,IJU,IKU) + CALL INIT_HALO2_ll(TZHALO2_VT,1,IIU,IJU,IKU) + CALL INIT_HALO2_ll(TZHALO2_WT,1,IIU,IJU,IKU) + END IF + + CALL GET_HALO2_DF(XUT,TZHALO2_UT,HNAME='XUT') + CALL GET_HALO2_DF(XVT,TZHALO2_VT,HNAME='XVT') + CALL GET_HALO2_DF(XWT,TZHALO2_WT,HNAME='XWT') +#endif !$acc update device(XUT, XVT, XWT) END IF !$acc data copyin(XUM, XVM, XWM) & @@ -1725,13 +1743,19 @@ IF ((CUVW_ADV_SCHEME(1:3)=='CEN') .AND. (CTEMP_SCHEME == 'LEFR')) THEN XUT, XVT, XWT, & XRHODJ, XDXX, XDYY, XDZZ, XDZX, XDZY, & XRUS,XRVS, XRWS, & +#ifndef MNH_OPENACC TZHALO2C_ll ) +#else + TZHALO2_UT,TZHALO2_VT,TZHALO2_WT ) +#endif !$acc end data IF (CUVW_ADV_SCHEME=='CEN4TH') THEN CALL CLEANLIST_ll(TZFIELDC_ll) NULLIFY(TZFIELDC_ll) +#ifndef MNH_OPENACC CALL DEL_HALO2_ll(TZHALO2C_ll) NULLIFY(TZHALO2C_ll) +#endif END IF ELSE