diff --git a/src/MNH/advection_uvw.f90 b/src/MNH/advection_uvw.f90 index 558f09c441d5785d214a673d39cc6e73c00b9399..3603c22ca0b935ac55e11644635fbd3502227c36 100644 --- a/src/MNH/advection_uvw.f90 +++ b/src/MNH/advection_uvw.f90 @@ -451,9 +451,7 @@ END DO ! Guesses for next time splitting loop ! ! -#ifdef MNH_COMPILER_NVHPC -!$acc loop independent collapse(3) -#endif +!$acc_nv loop independent collapse(3) DO CONCURRENT (JI=1:IIU , JJ=1:IJU , JK=1:IKU ) ZU(JI,JJ,JK) = ZU(JI,JJ,JK) + ZTSTEP / ZMXM_RHODJ(JI,JJ,JK) * & (ZRUS_OTHER(JI,JJ,JK) + ZRUS_ADV(JI,JJ,JK)) diff --git a/src/MNH/ice_adjust.f90 b/src/MNH/ice_adjust.f90 index 715f6e82af6438950234008ef54cb6d35e976c49..d19aa6b93f9c21a966943bdc63c3e4ab4343f274 100644 --- a/src/MNH/ice_adjust.f90 +++ b/src/MNH/ice_adjust.f90 @@ -305,7 +305,8 @@ REAL, DIMENSION(:,:,:), POINTER, CONTIGUOUS & ZLS, & ! guess of the Ls at t+1 ZW1,ZW2, & ! Work arrays for intermediate fields ZCRIAUT, & ! Autoconversion thresholds - ZHCF, ZHR + ZHCF, ZHR +REAL, DIMENSION(:,:,:), pointer , contiguous :: ZTEMP_BUD #endif ! LOGICAL :: GPOUT_RV,GPOUT_RC,GPOUT_RI,GPOUT_TH @@ -383,6 +384,7 @@ allocate( zw2 (IIU, IJU, IKU ) ) allocate( zcriaut(IIU, IJU, IKU ) ) allocate( zhcf (IIU, IJU, IKU ) ) allocate( zhr (IIU, IJU, IKU ) ) +allocate( ZTEMP_BUD (IIU, IJU, IKU ) ) #else !Pin positions in the pools of MNH memory CALL MNH_MEM_POSITION_PIN() @@ -402,14 +404,35 @@ CALL MNH_MEM_GET( zw2 , IIU, IJU, IKU ) CALL MNH_MEM_GET( zcriaut, IIU, IJU, IKU ) CALL MNH_MEM_GET( zhcf , IIU, IJU, IKU ) CALL MNH_MEM_GET( zhr , IIU, IJU, IKU ) +CALL MNH_MEM_GET( ZTEMP_BUD , IIU, IJU, IKU ) !$acc data present( gtemp, zsigs, zsrcs, zt, zrv, zrc, zri, zcph, zlv, zls, zw1, zw2, zcriaut, zhcf, zhr ) #endif -if ( lbudget_th ) call Budget_store_init( tbudgets(NBUDGET_TH), trim( hbuname ), pths(:, :, :) * prhodj(:, :, :) ) -if ( lbudget_rv ) call Budget_store_init( tbudgets(NBUDGET_RV), trim( hbuname ), prvs(:, :, :) * prhodj(:, :, :) ) -if ( lbudget_rc ) call Budget_store_init( tbudgets(NBUDGET_RC), trim( hbuname ), prcs(:, :, :) * prhodj(:, :, :) ) -if ( lbudget_ri ) call Budget_store_init( tbudgets(NBUDGET_RI), trim( hbuname ), pris(:, :, :) * prhodj(:, :, :) ) +if ( lbudget_th ) then + !$acc kernels + ZTEMP_BUD(:,:,:) = pths(:, :, :) * prhodj(:, :, :) + !$acc end kernels + call Budget_store_init( tbudgets(NBUDGET_TH), trim( hbuname ), ZTEMP_BUD(:,:,:) ) +end if +if ( lbudget_rv ) then + !$acc kernels + ZTEMP_BUD(:,:,:) = prvs(:, :, :) * prhodj(:, :, :) + !$acc end kernels + call Budget_store_init( tbudgets(NBUDGET_RV), trim( hbuname ), ZTEMP_BUD(:,:,:) ) +end if +if ( lbudget_rc ) then + !$acc kernels + ZTEMP_BUD(:,:,:) = prcs(:, :, :) * prhodj(:, :, :) + !$acc end kernels + call Budget_store_init( tbudgets(NBUDGET_RC), trim( hbuname ), ZTEMP_BUD(:,:,:) ) +end if +if ( lbudget_ri ) then + !$acc kernels + ZTEMP_BUD(:,:,:) = pris(:, :, :) * prhodj(:, :, :) + !$acc end kernels + call Budget_store_init( tbudgets(NBUDGET_RI), trim( hbuname ), ZTEMP_BUD(:,:,:) ) +end if ! ITERMAX=1 ! @@ -432,7 +455,7 @@ DO JITER =1,ITERMAX !* 2.3 compute the latent heat of vaporization Lv(T*) at t+1 ! and the latent heat of sublimation Ls(T*) at t+1 ! -!$acc kernels +!$acc kernels present_cr(ZLV,ZLS,ZCPH) ZLV(:,:,:) = XLVTT + ( XCPV - XCL ) * ( ZT(:,:,:) -XTT ) ZLS(:,:,:) = XLSTT + ( XCPV - XCI ) * ( ZT(:,:,:) -XTT ) ! @@ -545,13 +568,15 @@ ENDDO !* 5.2 compute the cloud fraction PCLDFR ! IF ( .NOT. OSUBG_COND ) THEN -!$acc kernels - GTEMP(:,:,:) = PRCS(:,:,:) + PRIS(:,:,:) > 1.E-12 / PTSTEP - WHERE ( GTEMP(:,:,:) ) - PCLDFR(:,:,:) = 1. - ELSEWHERE - PCLDFR(:,:,:) = 0. - ENDWHERE +!$acc kernels present_cr(GTEMP) +DO CONCURRENT (JI=1:IIU,JJ=1:IJU,JK=1:IKU) + GTEMP(JI,JJ,JK) = PRCS(JI,JJ,JK) + PRIS(JI,JJ,JK) > 1.E-12 / PTSTEP + IF ( GTEMP(JI,JJ,JK) )THEN + PCLDFR(JI,JJ,JK) = 1. + ELSE + PCLDFR(JI,JJ,JK) = 0. + ENDIF +ENDDO IF ( SIZE(PSRCS,3) /= 0 ) THEN PSRCS(:,:,:) = PCLDFR(:,:,:) END IF @@ -657,26 +682,34 @@ IF(GPOUT_TH) POUT_TH=ZT / PEXN(:,:,:) ! ---------------------- ! if ( lbudget_th ) then -!$acc update self(pths) - call Budget_store_end( tbudgets(NBUDGET_TH), trim( hbuname ), pths(:, :, :) * prhodj(:, :, :) ) + !$acc kernels + ZTEMP_BUD(:,:,:) = pths(:, :, :) * prhodj(:, :, :) + !$acc end kernels + call Budget_store_end( tbudgets(NBUDGET_TH), trim( hbuname ), ZTEMP_BUD(:,:,:) ) end if if ( lbudget_rv ) then -!$acc update self(prvs) - call Budget_store_end( tbudgets(NBUDGET_RV), trim( hbuname ), prvs(:, :, :) * prhodj(:, :, :) ) + !$acc kernels + ZTEMP_BUD(:,:,:) = prvs(:, :, :) * prhodj(:, :, :) + !$acc end kernels + call Budget_store_end( tbudgets(NBUDGET_RV), trim( hbuname ), ZTEMP_BUD(:,:,:) ) end if if ( lbudget_rc ) then -!$acc update self(prcs) - call Budget_store_end( tbudgets(NBUDGET_RC), trim( hbuname ), prcs(:, :, :) * prhodj(:, :, :) ) + !$acc kernels + ZTEMP_BUD(:,:,:) = prcs(:, :, :) * prhodj(:, :, :) + !$acc end kernels + call Budget_store_end( tbudgets(NBUDGET_RC), trim( hbuname ), ZTEMP_BUD(:,:,:) ) end if if ( lbudget_ri ) then -!$acc update self(pris) - call Budget_store_end( tbudgets(NBUDGET_RI), trim( hbuname ), pris(:, :, :) * prhodj(:, :, :) ) + !$acc kernels + ZTEMP_BUD(:,:,:) = pris(:, :, :) * prhodj(:, :, :) + !$acc end kernels + call Budget_store_end( tbudgets(NBUDGET_RI), trim( hbuname ), ZTEMP_BUD(:,:,:) ) end if !$acc end data #ifndef MNH_OPENACC -deallocate( gtemp, zsigs, zsrcs, zt, zrv, zrc, zri, zcph, zlv, zls, zw1, zw2, zcriaut, zhcf, zhr ) +deallocate( gtemp, zsigs, zsrcs, zt, zrv, zrc, zri, zcph, zlv, zls, zw1, zw2, zcriaut, zhcf, zhr , ZTEMP_BUD ) #else !Release all memory allocated with MNH_MEM_GET calls since last call to MNH_MEM_POSITION_PIN CALL MNH_MEM_RELEASE() diff --git a/src/MNH/resolved_cloud.f90 b/src/MNH/resolved_cloud.f90 index f9f937e8b3907711da77b5f5069cb33c4a0da921..0012f0876dad025ead30e4b3f7b258e85c71b554 100644 --- a/src/MNH/resolved_cloud.f90 +++ b/src/MNH/resolved_cloud.f90 @@ -710,6 +710,7 @@ ENDIF ! ! complete the lateral boundaries to avoid possible problems ! +!dir$ concurrent DO JI=1,JPHEXT PTHS(JI,:,:) = PTHS(IIB,:,:) PTHS(IIE+JI,:,:) = PTHS(IIE,:,:) diff --git a/src/MNH/sources_neg_correct.f90 b/src/MNH/sources_neg_correct.f90 index 0ac60ad45f48e5a0c0299957ee40a4497bde6db7..5602b75ccf615817e299c332cf382beb9053c7f6 100644 --- a/src/MNH/sources_neg_correct.f90 +++ b/src/MNH/sources_neg_correct.f90 @@ -63,11 +63,53 @@ integer :: jrmax integer :: jsv integer :: isv_lima_end real, dimension(:, :, :), pointer, contiguous :: zt, zexn, zlv, zls, zcph, zcor +REAL, DIMENSION(:,:,:), pointer , contiguous :: ZTEMP_BUD if ( krr == 0 ) return zcor => null() +jiu = Size(prths, 1 ) +jju = Size(prths, 2 ) +jku = Size(prths, 3 ) + +#ifndef MNH_OPENACC +allocate( zt ( jiu, jju, jku ) ) +allocate( zexn( jiu, jju, jku ) ) +allocate( zlv ( jiu, jju, jku ) ) +allocate( zcph( jiu, jju, jku ) ) +if ( hcloud == 'ICE3' .or. hcloud == 'ICE4' .or. hcloud == 'LIMA' ) then + allocate( zls( jiu, jju, jku ) ) + if ( krr > 3 ) then + allocate( zcor( jiu, jju, jku ) ) + end if +else + allocate( zls(0, 0, 0) ) +end if +if ( .not. Associated( zcor ) ) Allocate( zcor(0, 0, 0) ) +ALLOCATE(ZTEMP_BUD( jiu, jju, jku )) +#else +!Pin positions in the pools of MNH memory +call Mnh_mem_position_pin() + +call Mnh_mem_get( zt, jiu, jju, jku ) +call Mnh_mem_get( zexn, jiu, jju, jku ) +call Mnh_mem_get( zlv, jiu, jju, jku ) +call Mnh_mem_get( zcph, jiu, jju, jku ) +if ( hcloud == 'ICE3' .or. hcloud == 'ICE4' .or. hcloud == 'LIMA' ) then + call Mnh_mem_get( zls, jiu, jju, jku ) + if ( krr > 3 ) then + call Mnh_mem_get( zcor, jiu, jju, jku ) + else + call Mnh_mem_get( zcor, 0, 0, 0 ) + end if +else + call Mnh_mem_get( zls, 0, 0, 0 ) + call Mnh_mem_get( zcor, 0, 0, 0 ) +end if +call MNH_MEM_GET( ZTEMP_BUD, jiu, jju, jku ) +#endif + !$acc data present( ppabst, ptht, prt, prths, prrs, prsvs, prhodj ) if ( mppdb_initialized ) then @@ -129,70 +171,79 @@ else !NECON + NEGA if ( hcloud == 'KESS' .or. hcloud == 'ICE3' .or. hcloud == 'ICE4' .or. & hcloud == 'KHKO' .or. hcloud == 'C2R2' .or. hcloud == 'LIMA' ) then - if ( lbudget_th) call Budget_store_init( tbudgets(NBUDGET_TH), Trim( hbudname ), prths(:, :, :) * prhodj(:, :, :) ) - if ( lbudget_rv) call Budget_store_init( tbudgets(NBUDGET_RV), Trim( hbudname ), prrs (:, :, :, 1) * prhodj(:, :, :) ) - if ( lbudget_rc) call Budget_store_init( tbudgets(NBUDGET_RC), Trim( hbudname ), prrs (:, :, :, 2) * prhodj(:, :, :) ) - if ( lbudget_rr) call Budget_store_init( tbudgets(NBUDGET_RR), Trim( hbudname ), prrs (:, :, :, 3) * prhodj(:, :, :) ) - if ( lbudget_ri) call Budget_store_init( tbudgets(NBUDGET_RI), Trim( hbudname ), prrs (:, :, :, 4) * prhodj(:, :, :) ) - if ( lbudget_rs) call Budget_store_init( tbudgets(NBUDGET_RS), Trim( hbudname ), prrs (:, :, :, 5) * prhodj(:, :, :) ) - if ( lbudget_rg) call Budget_store_init( tbudgets(NBUDGET_RG), Trim( hbudname ), prrs (:, :, :, 6) * prhodj(:, :, :) ) - if ( lbudget_rh) call Budget_store_init( tbudgets(NBUDGET_RH), Trim( hbudname ), prrs (:, :, :, 7) * prhodj(:, :, :) ) + if ( lbudget_th) then + !$acc kernels present(ZTEMP_BUD) + ZTEMP_BUD(:,:,:) = prths(:, :, :) * prhodj(:, :, :) + !$acc end kernels + call Budget_store_init( tbudgets(NBUDGET_TH), Trim( hbudname ), ZTEMP_BUD(:,:,:) ) + end if + if ( lbudget_rv) then + !$acc kernels present(ZTEMP_BUD) + ZTEMP_BUD(:,:,:) = prrs (:, :, :, 1) * prhodj(:, :, :) + !$acc end kernels + call Budget_store_init( tbudgets(NBUDGET_RV), Trim( hbudname ), ZTEMP_BUD(:,:,:) ) + end if + if ( lbudget_rc) then + !$acc kernels present(ZTEMP_BUD) + ZTEMP_BUD(:,:,:) = prrs (:, :, :, 2) * prhodj(:, :, :) + !$acc end kernels + call Budget_store_init( tbudgets(NBUDGET_RC), Trim( hbudname ), ZTEMP_BUD(:,:,:) ) + end if + if ( lbudget_rr) then + !$acc kernels present(ZTEMP_BUD) + ZTEMP_BUD(:,:,:) = prrs (:, :, :, 3) * prhodj(:, :, :) + !$acc end kernels + call Budget_store_init( tbudgets(NBUDGET_RR), Trim( hbudname ), ZTEMP_BUD(:,:,:) ) + end if + if ( lbudget_ri) then + !$acc kernels present(ZTEMP_BUD) + ZTEMP_BUD(:,:,:) = prrs (:, :, :, 4) * prhodj(:, :, :) + !$acc end kernels + call Budget_store_init( tbudgets(NBUDGET_RI), Trim( hbudname ), ZTEMP_BUD(:,:,:) ) + end if + if ( lbudget_rs) then + !$acc kernels present(ZTEMP_BUD) + ZTEMP_BUD(:,:,:) = prrs (:, :, :, 5) * prhodj(:, :, :) + !$acc end kernels + call Budget_store_init( tbudgets(NBUDGET_RS), Trim( hbudname ), ZTEMP_BUD(:,:,:) ) + end if + if ( lbudget_rg) then + !$acc kernels present(ZTEMP_BUD) + ZTEMP_BUD(:,:,:) = prrs (:, :, :, 6) * prhodj(:, :, :) + !$acc end kernels + call Budget_store_init( tbudgets(NBUDGET_RG), Trim( hbudname ), ZTEMP_BUD(:,:,:) ) + end if + if ( lbudget_rh) then + !$acc kernels present(ZTEMP_BUD) + ZTEMP_BUD(:,:,:) = prrs (:, :, :, 7) * prhodj(:, :, :) + !$acc end kernels + call Budget_store_init( tbudgets(NBUDGET_RH), Trim( hbudname ), ZTEMP_BUD(:,:,:) ) + end if end if if ( lbudget_sv .and. ( hcloud == 'C2R2' .or. hcloud == 'KHKO' ) ) then do ji = nsv_c2r2beg, nsv_c2r2end - call Budget_store_init( tbudgets(NBUDGET_SV1 - 1 + ji), Trim( hbudname ), prsvs(:, :, :, ji) * prhodj(:, :, :) ) + !$acc kernels present(ZTEMP_BUD) + ZTEMP_BUD(:,:,:) = prsvs(:, :, :, ji) * prhodj(:, :, :) + !$acc end kernels + call Budget_store_init( tbudgets(NBUDGET_SV1 - 1 + ji), Trim( hbudname ), ZTEMP_BUD(:,:,:) ) end do end if if ( lbudget_sv .and. hcloud == 'LIMA' ) then do ji = nsv_lima_beg, isv_lima_end - call Budget_store_init( tbudgets(NBUDGET_SV1 - 1 + ji), Trim( hbudname ), prsvs(:, :, :, ji) * prhodj(:, :, :) ) + !$acc kernels present(ZTEMP_BUD) + ZTEMP_BUD(:,:,:) = prsvs(:, :, :, ji) * prhodj(:, :, :) + !$acc end kernels + call Budget_store_init( tbudgets(NBUDGET_SV1 - 1 + ji), Trim( hbudname ), ZTEMP_BUD(:,:,:) ) end do end if end if -jiu = Size(prths, 1 ) -jju = Size(prths, 2 ) -jku = Size(prths, 3 ) -#ifndef MNH_OPENACC -allocate( zt ( jiu, jju, jku ) ) -allocate( zexn( jiu, jju, jku ) ) -allocate( zlv ( jiu, jju, jku ) ) -allocate( zcph( jiu, jju, jku ) ) -if ( hcloud == 'ICE3' .or. hcloud == 'ICE4' .or. hcloud == 'LIMA' ) then - allocate( zls( jiu, jju, jku ) ) - if ( krr > 3 ) then - allocate( zcor( jiu, jju, jku ) ) - end if -else - allocate( zls(0, 0, 0) ) -end if -if ( .not. Associated( zcor ) ) Allocate( zcor(0, 0, 0) ) -#else -!Pin positions in the pools of MNH memory -call Mnh_mem_position_pin() -call Mnh_mem_get( zt, jiu, jju, jku ) -call Mnh_mem_get( zexn, jiu, jju, jku ) -call Mnh_mem_get( zlv, jiu, jju, jku ) -call Mnh_mem_get( zcph, jiu, jju, jku ) -if ( hcloud == 'ICE3' .or. hcloud == 'ICE4' .or. hcloud == 'LIMA' ) then - call Mnh_mem_get( zls, jiu, jju, jku ) - if ( krr > 3 ) then - call Mnh_mem_get( zcor, jiu, jju, jku ) - else - call Mnh_mem_get( zcor, 0, 0, 0 ) - end if -else - call Mnh_mem_get( zls, 0, 0, 0 ) - call Mnh_mem_get( zcor, 0, 0, 0 ) -end if -#endif +!$acc data present( zt, zexn, zlv, zcph, zls, zcor ) -!$acc data create( zt, zexn, zlv, zcph, zls, zcor ) - -!$acc kernels present_cr(zt, zexn, zlv, zcph, zls, zcor) +!$acc kernels present_cr(zexn,zt,zlv) #ifndef MNH_BITREP zexn(:, :, :) = ( ppabst(:, :, :) / xp00 ) ** (xrd / xcpd ) #else @@ -202,7 +253,7 @@ zt (:, :, :) = ptht(:, :, :) * zexn(:, :, :) zlv (:, :, :) = xlvtt + ( xcpv - xcl ) * ( zt(:, :, :) - xtt ) !$acc end kernels if ( hcloud == 'ICE3' .or. hcloud == 'ICE4' .or. hcloud == 'LIMA' ) then -!$acc kernels +!$acc kernels present_cr(zls) zls(:, :, :) = xlstt + ( xcpv - xci ) * ( zt(:, :, :) - xtt ) !$acc end kernels end if @@ -409,7 +460,7 @@ end select CLOUD !$acc end data #ifndef MNH_OPENACC -deallocate( zexn, zlv, zcph, zls, zcor ) +deallocate( zexn, zlv, zcph, zls, zcor , ZTEMP_BUD) #else !Release all memory allocated with Mnh_mem_get calls since last call to Mnh_mem_position_pin call Mnh_mem_release() @@ -447,24 +498,70 @@ if ( hbudname /= 'NECON' .and. hbudname /= 'NEGA' ) then else !NECON + NEGA if ( hcloud == 'KESS' .or. hcloud == 'ICE3' .or. hcloud == 'ICE4' .or. & hcloud == 'KHKO' .or. hcloud == 'C2R2' .or. hcloud == 'LIMA' ) then - if ( lbudget_th) call Budget_store_end( tbudgets(NBUDGET_TH), Trim( hbudname ), prths(:, :, :) * prhodj(:, :, :) ) - if ( lbudget_rv) call Budget_store_end( tbudgets(NBUDGET_RV), Trim( hbudname ), prrs (:, :, :, 1) * prhodj(:, :, :) ) - if ( lbudget_rc) call Budget_store_end( tbudgets(NBUDGET_RC), Trim( hbudname ), prrs (:, :, :, 2) * prhodj(:, :, :) ) - if ( lbudget_rr) call Budget_store_end( tbudgets(NBUDGET_RR), Trim( hbudname ), prrs (:, :, :, 3) * prhodj(:, :, :) ) - if ( lbudget_ri) call Budget_store_end( tbudgets(NBUDGET_RI), Trim( hbudname ), prrs (:, :, :, 4) * prhodj(:, :, :) ) - if ( lbudget_rs) call Budget_store_end( tbudgets(NBUDGET_RS), Trim( hbudname ), prrs (:, :, :, 5) * prhodj(:, :, :) ) - if ( lbudget_rg) call Budget_store_end( tbudgets(NBUDGET_RG), Trim( hbudname ), prrs (:, :, :, 6) * prhodj(:, :, :) ) - if ( lbudget_rh) call Budget_store_end( tbudgets(NBUDGET_RH), Trim( hbudname ), prrs (:, :, :, 7) * prhodj(:, :, :) ) + if ( lbudget_th) then + !$acc kernels present(ZTEMP_BUD) + ZTEMP_BUD(:,:,:) = prths(:, :, :) * prhodj(:, :, :) + !$acc end kernels + call Budget_store_end( tbudgets(NBUDGET_TH), Trim( hbudname ), ZTEMP_BUD(:,:,:) ) + end if + if ( lbudget_rv) then + !$acc kernels present(ZTEMP_BUD) + ZTEMP_BUD(:,:,:) = prrs (:, :, :, 1) * prhodj(:, :, :) + !$acc end kernels + call Budget_store_end( tbudgets(NBUDGET_RV), Trim( hbudname ), ZTEMP_BUD(:,:,:) ) + end if + if ( lbudget_rc) then + !$acc kernels present(ZTEMP_BUD) + ZTEMP_BUD(:,:,:) = prrs (:, :, :, 2) * prhodj(:, :, :) + !$acc end kernels + call Budget_store_end( tbudgets(NBUDGET_RC), Trim( hbudname ), ZTEMP_BUD(:,:,:) ) + end if + if ( lbudget_rr) then + !$acc kernels present(ZTEMP_BUD) + ZTEMP_BUD(:,:,:) = prrs (:, :, :, 3) * prhodj(:, :, :) + !$acc end kernels + call Budget_store_end( tbudgets(NBUDGET_RR), Trim( hbudname ), ZTEMP_BUD(:,:,:) ) + end if + if ( lbudget_ri) then + !$acc kernels present(ZTEMP_BUD) + ZTEMP_BUD(:,:,:) = prrs (:, :, :, 4) * prhodj(:, :, :) + !$acc end kernels + call Budget_store_end( tbudgets(NBUDGET_RI), Trim( hbudname ), ZTEMP_BUD(:,:,:) ) + end if + if ( lbudget_rs) then + !$acc kernels present(ZTEMP_BUD) + ZTEMP_BUD(:,:,:) = prrs (:, :, :, 5) * prhodj(:, :, :) + !$acc end kernels + call Budget_store_end( tbudgets(NBUDGET_RS), Trim( hbudname ), ZTEMP_BUD(:,:,:) ) + end if + if ( lbudget_rg) then + !$acc kernels present(ZTEMP_BUD) + ZTEMP_BUD(:,:,:) = prrs (:, :, :, 6) * prhodj(:, :, :) + !$acc end kernels + call Budget_store_end( tbudgets(NBUDGET_RG), Trim( hbudname ), ZTEMP_BUD(:,:,:) ) + end if + if ( lbudget_rh) then + !$acc kernels present(ZTEMP_BUD) + ZTEMP_BUD(:,:,:) = prrs (:, :, :, 7) * prhodj(:, :, :) + !$acc end kernels + call Budget_store_end( tbudgets(NBUDGET_RH), Trim( hbudname ), ZTEMP_BUD(:,:,:) ) + end if end if if ( lbudget_sv .and. ( hcloud == 'C2R2' .or. hcloud == 'KHKO' ) ) then - do ji = nsv_c2r2beg, nsv_c2r2end - call Budget_store_end( tbudgets(NBUDGET_SV1 - 1 + ji), Trim( hbudname ), prsvs(:, :, :, ji) * prhodj(:, :, :) ) + do ji = nsv_c2r2beg, nsv_c2r2end + !$acc kernels present(ZTEMP_BUD) + ZTEMP_BUD(:,:,:) = prsvs(:, :, :, ji) * prhodj(:, :, :) + !$acc end kernels + call Budget_store_end( tbudgets(NBUDGET_SV1 - 1 + ji), Trim( hbudname ), ZTEMP_BUD(:,:,:) ) end do end if if ( lbudget_sv .and. hcloud == 'LIMA' ) then - do ji = nsv_lima_beg, isv_lima_end - call Budget_store_end( tbudgets(NBUDGET_SV1 - 1 + ji), Trim( hbudname ), prsvs(:, :, :, ji) * prhodj(:, :, :) ) + do ji = nsv_lima_beg, isv_lima_end + !$acc kernels present(ZTEMP_BUD) + ZTEMP_BUD(:,:,:) = prsvs(:, :, :, ji) * prhodj(:, :, :) + !$acc end kernels + call Budget_store_end( tbudgets(NBUDGET_SV1 - 1 + ji), Trim( hbudname ), ZTEMP_BUD(:,:,:) ) end do end if end if diff --git a/src/MNH/tools.f90 b/src/MNH/tools.f90 index d72fe60251fe26df22fbb7fc4c6f7b69120da766..846462cbbcc4e3a6b61c4724bb643ba9874501a6 100644 --- a/src/MNH/tools.f90 +++ b/src/MNH/tools.f90 @@ -337,13 +337,10 @@ ic = 0 ! different runs of this subroutine BUT final result should be the same !Comment the following line + atomic directives to have consistent values for debugging !Warning: huge impact on performance -!#ifdef MNH_COMPILER_NVHPC !$acc parallel loop collapse(3) private(idx) independent -!#endif do jk = 1, size( ltab, 3 ) do jj = 1, size( ltab, 2 ) do ji = 1, size( ltab, 1 ) -!!$ do concurrent ( ji=1:size(ltab,1) , jj=1:size(ltab,2) , jk=1:size(ltab,3 ) ) if ( ltab(ji, jj, jk ) ) then !$acc atomic capture ic = ic +1 @@ -444,5 +441,3 @@ function Upcase(hstring) end if end do end function Upcase - -!!$end module mode_tools diff --git a/src/MNH/tridiag_thermo.f90 b/src/MNH/tridiag_thermo.f90 index 5cf3bb25b7d0468e5750d5d548b4ae063febd078..c96d99382707f1623ae9460b973256a7855ce63b 100644 --- a/src/MNH/tridiag_thermo.f90 +++ b/src/MNH/tridiag_thermo.f90 @@ -258,13 +258,11 @@ ZMZM_RHODJ = MZM(PRHODJ) #else CALL MZM_DEVICE(PRHODJ,ZMZM_RHODJ) #endif -!$acc kernels ! async +!$acc kernels present_cr(ZRHODJ_DFDDTDZ_O_DZ2) ! async #ifndef MNH_BITREP -ZRHODJ_DFDDTDZ_O_DZ2 = ZMZM_RHODJ*PDFDDTDZ/PDZZ**2 +ZRHODJ_DFDDTDZ_O_DZ2(:,:,:) = ZMZM_RHODJ(:,:,:)*PDFDDTDZ(:,:,:)/PDZZ(:,:,:)**2 #else -#ifdef MNH_COMPILER_NVHPC -!$acc loop independent collapse(3) -#endif +!$acc_nv loop independent collapse(3) DO CONCURRENT ( JI=1:JIU,JJ=1:JJU,JK=1:JKU) ZRHODJ_DFDDTDZ_O_DZ2(JI,JJ,JK) = ZMZM_RHODJ(JI,JJ,JK)*PDFDDTDZ(JI,JJ,JK)/BR_P2(PDZZ(JI,JJ,JK)) END DO !CONCURRENT @@ -394,21 +392,15 @@ END DO !CONCURRENT ! !$acc loop seq DO JK = IKB+KKL,IKE-KKL,KKL -#ifdef MNH_COMPILER_NVHPC - !$acc loop independent collapse(2) -#endif - ! acc loop gang, vector collapse(2) + ! gang+vector needed or parallisation vector only + !$acc_nv loop independent gang, vector collapse(2) DO CONCURRENT ( JI=1:JIU,JJ=1:JJU) - !DO JJ=1,JJU - ! DO JI=1,JIU ZGAM(JI,JJ,JK) = ZC(JI,JJ,JK-KKL) / ZBET(JI,JJ) ! gam(k) = c(k-1) / bet ZBET(JI,JJ) = ZB(JI,JJ,JK) - ZA(JI,JJ,JK) * ZGAM(JI,JJ,JK) ! bet = b(k) - a(k)* gam(k) PVARP(JI,JJ,JK)= ( ZY(JI,JJ,JK) - ZA(JI,JJ,JK) * PVARP(JI,JJ,JK-KKL) ) / ZBET(JI,JJ) ! res(k) = (y(k) -a(k)*res(k-1))/ bet - ! END DO - !END DO END DO !CONCURRENT END DO ! special treatment for the last level @@ -429,10 +421,8 @@ END DO !CONCURRENT ! !$acc loop seq DO JK = IKE-KKL,IKB,-1*KKL -#ifdef MNH_COMPILER_NVHPC - !$acc loop independent collapse(2) -#endif - ! acc loop gang, vector collapse(2) + ! gang+vector needed or parallisation vector only + !$acc_nv loop independent gang, vector collapse(2) DO CONCURRENT ( JI=1:JIU,JJ=1:JJU) PVARP(JI,JJ,JK) = PVARP(JI,JJ,JK) - ZGAM(JI,JJ,JK+KKL) * PVARP(JI,JJ,JK+KKL) END DO !CONCURRENT