diff --git a/src/LIB/SURCOUCHE/src/mode_sum_ll.f90 b/src/LIB/SURCOUCHE/src/mode_sum_ll.f90
index 613d6c3eee38a2cc70052f92f6da957aceaacc8f..ccc372cbd28abe31c9a2347d47d814364a6475b3 100644
--- a/src/LIB/SURCOUCHE/src/mode_sum_ll.f90
+++ b/src/LIB/SURCOUCHE/src/mode_sum_ll.f90
@@ -7,6 +7,7 @@
! J. Escobar 15/09/2015: WENO5 & JPHEXT <> 1
! P. Wautelet 26/04/2019: use modd_precision parameters for datatypes of MPI communications
! P. Wautelet 21/06/2019: mode REDUCESUM_ll subroutine to mode_reduce_sum.f90 (to remove circular dependencies between modules)
+! P. Wautelet 23/07/2019: OpenACC: add directives to MIN_ll and SUM3D_ll
!-----------------------------------------------------------------
! ##################
@@ -77,7 +78,7 @@ CONTAINS
! contained in a geographic region given by its first point
! and its last point. If these two points are omitted,
! the whole physical domain is considered.
-!
+!
!! Purpose
!! -------
!
@@ -89,7 +90,7 @@ CONTAINS
! an intermediate buffer of the dimension of the result with its elements.
! Then we make a summation of all the intermediate buffers (MPI reduction)
! and obtain the result.
-!
+!
!! External
!! --------
!
@@ -174,7 +175,7 @@ CONTAINS
INORTH = 0
!
IF (LWEST_ll()) IWEST = -1
- IF (LEAST_ll()) IEAST = 1
+ IF (LEAST_ll()) IEAST = 1
IF (LSOUTH_ll()) ISOUTH = -1
IF (LNORTH_ll()) INORTH = 1
!
@@ -321,9 +322,9 @@ CONTAINS
! by its first point and its last point.
! If these two points are omitted,
! the whole physical domain is considered.
-!
+!
! The result is a 2D array.
-!
+!
!! Purpose
!! -------
!
@@ -335,7 +336,7 @@ CONTAINS
! the summation and fills an intermediate buffer of the dimension
! of the result. Then we make a summation of all
! the intermediate buffers (MPI reduction) and obtain the result.
-!
+!
!! External
!! --------
!
@@ -354,10 +355,10 @@ CONTAINS
!
!! Implicit Arguments
!! ------------------
-!
+!
!! Reference
!! ---------
-!
+!
!! Author
!! ------
! R. Guivarch * CERFACS *
@@ -365,7 +366,7 @@ CONTAINS
!! Modifications
!! -------------
! Original 03/07/98
-!
+!
!-------------------------------------------------------------------------------
!
!* 0. DECLARATIONS
@@ -425,7 +426,7 @@ CONTAINS
INORTH = 0
!
IF (LWEST_ll()) IWEST = -1
- IF (LEAST_ll()) IEAST = 1
+ IF (LEAST_ll()) IEAST = 1
IF (LSOUTH_ll()) ISOUTH = -1
IF (LNORTH_ll()) INORTH = 1
!
@@ -437,7 +438,7 @@ CONTAINS
!
IF(PRESENT(KXOR) .AND. PRESENT(KYOR) .AND. PRESENT(KZOR) .AND. &
PRESENT(KXEND) .AND. PRESENT(KYEND) .AND. PRESENT(KZEND) ) THEN
-!
+!
IXOR = KXOR
IYOR = KYOR
IZOR = KZOR
@@ -451,7 +452,7 @@ CONTAINS
!
IXOR = 1 + JPHEXT
IYOR = 1 + JPHEXT
- IZOR = 1 + JPVEXT
+ IZOR = 1 + JPVEXT
IXEND = IIMAX + JPHEXT
IYEND = IJMAX + JPHEXT
IZEND = SIZE(PFIELD,3)-JPVEXT
@@ -564,7 +565,7 @@ CONTAINS
IIE_BUF = IIE - IXOR + 1
IJE_BUF = IJE - IYOR + 1
!
-!* 3.3.2 switch the displacements in PFIELD to local indices
+!* 3.3.2 switch the displacements in PFIELD to local indices
!
IIB = IIB - TCRRT_COMDATA%TSPLIT_B%NXORE + 1
IJB = IJB - TCRRT_COMDATA%TSPLIT_B%NYORE + 1
@@ -622,9 +623,9 @@ CONTAINS
! region given by its first point and its last point.
! If these two points are omitted,
! the whole physical domain is considered.
-!
+!
! The result is a 1D array.
-!
+!
!! Purpose
!! -------
!
@@ -636,7 +637,7 @@ CONTAINS
! the summation and fills an intermediate buffer of the dimension
! of the result. Then we make a summation of all
! the intermediate buffers (MPI reduction) and obtain the result.
-!
+!
!! External
!! --------
!
@@ -655,10 +656,10 @@ CONTAINS
!
!! Implicit Arguments
!! ------------------
-!
+!
!! Reference
!! ---------
-!
+!
!! Author
!! ------
! R. Guivarch * CERFACS *
@@ -667,7 +668,7 @@ CONTAINS
!! -------------
! Original 03/07/98
! J.ESCOBAR 9/03/2008 : BUG remove deallocate(ZBUF...) or result is NULL !!!
-!
+!
!-------------------------------------------------------------------------------
!
!* 0. DECLARATIONS
@@ -740,7 +741,7 @@ CONTAINS
!
IF(PRESENT(KXOR) .AND. PRESENT(KYOR) .AND. PRESENT(KZOR) .AND. &
PRESENT(KXEND) .AND. PRESENT(KYEND) .AND. PRESENT(KZEND) ) THEN
-!
+!
IXOR = KXOR
IYOR = KYOR
IZOR = KZOR
@@ -754,7 +755,7 @@ CONTAINS
!
IXOR = 1 + JPHEXT
IYOR = 1 + JPHEXT
- IZOR = 1 + JPVEXT
+ IZOR = 1 + JPVEXT
IXEND = IIMAX + JPHEXT
IYEND = IJMAX + JPHEXT
IZEND = SIZE(PFIELD,3)-JPVEXT
@@ -915,7 +916,11 @@ CONTAINS
!
! ##########################################################
REAL FUNCTION SUM3D_ll( PFIELD, KINFO, KXOR, KYOR, KZOR, &
+#ifndef _OPENACC
KXEND, KYEND, KZEND )
+#else
+ KXEND, KYEND, KZEND, oondevice )
+#endif
! ##########################################################
!
!!**** *SUM3D_ll* - function to perform a summation in the three directions
@@ -923,9 +928,9 @@ CONTAINS
! by its first point and its last point.
! If these two points are omitted,
! the whole physical domain is considered.
-!
+!
! The result is a scalar
-!
+!
!! Purpose
!! -------
!
@@ -956,10 +961,10 @@ CONTAINS
!
!! Implicit Arguments
!! ------------------
-!
+!
!! Reference
!! ---------
-!
+!
!! Author
!! ------
! R. Guivarch * CERFACS *
@@ -968,7 +973,7 @@ CONTAINS
!! -------------
! Original 03/07/98
! Modif : V.Masson , 08/2010 : for reproducibility
-!
+!
!-------------------------------------------------------------------------------
!
!* 0. DECLARATIONS
@@ -982,7 +987,7 @@ CONTAINS
!
!JUAN
!!$ USE MODE_GATHER_ll
- USE MODE_REPRO_SUM
+ USE MODE_REPRO_SUM
!JUAN
!
IMPLICIT NONE
@@ -995,6 +1000,9 @@ CONTAINS
!
INTEGER, OPTIONAL, INTENT(IN) :: KXOR, KYOR, KZOR, & ! Coordinates
KXEND, KYEND, KZEND ! of the region
+#ifdef _OPENACC
+ logical, optional, intent(in) :: oondevice ! True if PFIELD is on device (OpenACC)
+#endif
!
!* 0.2 declarations of local variables
!
@@ -1012,11 +1020,23 @@ CONTAINS
! the corresponding boundary
!
INTEGER :: IIMAX, IJMAX ! global dimensions of the physical model
+#ifdef _OPENACC
+ integer :: ji, jj, jk
+ logical :: gondevice
+#endif
!
REAL, DIMENSION(SIZE(PFIELD,1),SIZE(PFIELD,2)) :: ZSUM
-REAL, DIMENSION(:,:), ALLOCATABLE :: ZSUM_ll
+REAL, DIMENSION(:,:), ALLOCATABLE :: ZSUM_ll
!-------------------------------------------------------------------------------
!
+#ifdef _OPENACC
+ if ( present( oondevice ) ) then
+ gondevice = oondevice
+ else
+ gondevice = .false.
+ end if
+#endif
+
KINFO = 0
!
ZSUM3D = 0
@@ -1043,7 +1063,7 @@ REAL, DIMENSION(:,:), ALLOCATABLE :: ZSUM_ll
!
IF(PRESENT(KXOR) .AND. PRESENT(KYOR) .AND. PRESENT(KZOR) .AND. &
PRESENT(KXEND) .AND. PRESENT(KYEND) .AND. PRESENT(KZEND) ) THEN
-!
+!
IXOR = KXOR
IYOR = KYOR
IZOR = KZOR
@@ -1057,7 +1077,7 @@ REAL, DIMENSION(:,:), ALLOCATABLE :: ZSUM_ll
!
IXOR = 1 + JPHEXT
IYOR = 1 + JPHEXT
- IZOR = 1 + JPVEXT
+ IZOR = 1 + JPVEXT
IXEND = IIMAX + JPHEXT
IYEND = IJMAX + JPHEXT
IZEND = SIZE(PFIELD,3)-JPVEXT
@@ -1120,8 +1140,29 @@ REAL, DIMENSION(:,:), ALLOCATABLE :: ZSUM_ll
!
!!$ ZSUM3D = SUM(SUM(SUM(PFIELD(IIB:IIE, IJB:IJE, IZOR:IZEND), 3), 2), 1)
! vertical sum only, on each processor
- ZSUM(:,:) = 0.
- ZSUM(IIB:IIE, IJB:IJE) = SUM(PFIELD(IIB:IIE, IJB:IJE,IZOR:IZEND),3)
+#ifdef _OPENACC
+ if ( gondevice ) then
+!The present directive is useful to check if the array is really available
+!$acc kernels present(PFIELD) copyout(ZSUM)
+ ZSUM(:,:) = 0.
+!PW: bug PGI 18.10: SUM function seems bugged on GPU
+! ZSUM(IIB:IIE, IJB:IJE) = SUM(PFIELD(IIB:IIE, IJB:IJE,IZOR:IZEND),3)
+ do jk = izor, izend
+ do jj = ijb, ije
+ do ji = iib, iie
+ zsum(ji, jj ) = zsum(ji, jj ) + pfield(ji, jj, jk )
+ end do
+ end do
+ end do
+!$acc end kernels
+ else
+ ZSUM(:,:) = 0.
+ ZSUM(IIB:IIE, IJB:IJE) = SUM(PFIELD(IIB:IIE, IJB:IJE,IZOR:IZEND),3)
+ end if
+#else
+ ZSUM(:,:) = 0.
+ ZSUM(IIB:IIE, IJB:IJE) = SUM(PFIELD(IIB:IIE, IJB:IJE,IZOR:IZEND),3)
+#endif
!
ENDIF
!
@@ -1138,9 +1179,9 @@ REAL, DIMENSION(:,:), ALLOCATABLE :: ZSUM_ll
!!$!* computes the sum
!!$ SUM3D_ll= SUM(ZSUM_ll(:,:))
!!$ DEALLOCATE(ZSUM_ll)
-!JUAN
+!JUAN
SUM3D_ll= SUM_DD_R2_ll(ZSUM(:,:))
-!JUAN
+!JUAN
ENDIF
!
!-------------------------------------------------------------------------------
@@ -1151,14 +1192,14 @@ REAL, DIMENSION(:,:), ALLOCATABLE :: ZSUM_ll
FUNCTION SUM_1DFIELD_ll(PFIELD, HDIR, KOR, KEND, KERR) RESULT(ZSUM)
! ###################################################################
!
-!!**** *SUM_1DFIELD_ll* - this function calculates the sum of the PFIELD
+!!**** *SUM_1DFIELD_ll* - this function calculates the sum of the PFIELD
! one-dimensional field according to the HDIR direction
! The KOR and KEND arguments can be used to specify
! the boundaries of the sum.
! The dimension of PFIELD is supposed to be
! the dimension of the extended subdomain
! in the HDIR direction.
-!
+!
!! Purpose
!! -------
!
@@ -1167,7 +1208,7 @@ REAL, DIMENSION(:,:), ALLOCATABLE :: ZSUM_ll
! The sizes and positions of each local array are gathered on each proc.
! Then the MPI_ALLGATHERV routine is called to gather the global 1D field.
! Then the sum of the 1D global field is calculated.
-!
+!
!! External
!! --------
!
@@ -1233,7 +1274,7 @@ REAL, DIMENSION(:,:), ALLOCATABLE :: ZSUM_ll
!
REAL, DIMENSION(:), ALLOCATABLE :: ZGLOBFIELD ! global field
TYPE(MODELSPLITTING_ll), POINTER :: TZSPLIT
- INTEGER, DIMENSION(2) :: IMAX ! global dimensions of the current model
+ INTEGER, DIMENSION(2) :: IMAX ! global dimensions of the current model
INTEGER, DIMENSION(2) :: IOR, IEND, IORP, IENDP, IORE, IENDE
INTEGER :: IDIR ! direction of the 1D field
INTEGER :: IERR
@@ -1299,8 +1340,8 @@ REAL, DIMENSION(:,:), ALLOCATABLE :: ZSUM_ll
!
ENDIF
!
- IB = MAX(KOR, IOR(IDIR))
- IE = MIN(KEND, IEND(IDIR))
+ IB = MAX(KOR, IOR(IDIR))
+ IE = MIN(KEND, IEND(IDIR))
!
ELSE ! default : physical zone
!
@@ -1327,7 +1368,7 @@ REAL, DIMENSION(:,:), ALLOCATABLE :: ZSUM_ll
!
CALL MPI_ALLGATHER( (/ IB-IGB /), 1, MNHINT_MPI, IDISPL, 1, MNHINT_MPI, &
NMNH_COMM_WORLD, IERR)
-!
+!
!* 2.2 Get the global field
!
ALLOCATE(ZGLOBFIELD(IGE-IGB+1))
@@ -1364,9 +1405,9 @@ REAL, DIMENSION(:,:), ALLOCATABLE :: ZSUM_ll
! contained in a geographic region given by its first point
! and its last point. If these two points are omitted,
! the whole physical domain is considered.
-!
+!
! The result is a scalar
-!
+!
!! Purpose
!! -------
!
@@ -1377,7 +1418,7 @@ REAL, DIMENSION(:,:), ALLOCATABLE :: ZSUM_ll
! If this intersection is not empty, the processor searchs
! the maximum in this intersection.
! Then we make a MPI reduction with all the local maxima.
-!
+!
!! External
!! --------
!
@@ -1477,7 +1518,7 @@ REAL, DIMENSION(:,:), ALLOCATABLE :: ZSUM_ll
!
IF(PRESENT(KXOR) .AND. PRESENT(KYOR) .AND. PRESENT(KZOR) .AND. &
PRESENT(KXEND) .AND. PRESENT(KYEND) .AND. PRESENT(KZEND) ) THEN
-!
+!
IXOR = KXOR
IYOR = KYOR
IZOR = KZOR
@@ -1491,7 +1532,7 @@ REAL, DIMENSION(:,:), ALLOCATABLE :: ZSUM_ll
!
IXOR = 1 + JPHEXT
IYOR = 1 + JPHEXT
- IZOR = 1 + JPVEXT
+ IZOR = 1 + JPVEXT
IXEND = IIMAX + JPHEXT
IYEND = IJMAX + JPHEXT
IZEND = SIZE(PFIELD,3)-JPVEXT
@@ -1569,16 +1610,20 @@ REAL, DIMENSION(:,:), ALLOCATABLE :: ZSUM_ll
!
! ########################################################
REAL FUNCTION MIN_ll( PFIELD, KINFO, KXOR, KYOR, KZOR, &
+#ifndef _OPENACC
KXEND, KYEND, KZEND )
+#else
+ KXEND, KYEND, KZEND, oondevice )
+#endif
! ########################################################
!
!!**** *MIN_ll* - function to find the minimum value in a 3D field
! contained in a geographic region given by its first point
! and its last point. If these two points are omitted,
! the whole physical domain is considered.
-!
+!
! the result is a scalar
-!
+!
!! Purpose
!! -------
!
@@ -1589,7 +1634,7 @@ REAL, DIMENSION(:,:), ALLOCATABLE :: ZSUM_ll
! If this intersection is not empty, the processor searchs the minimum
! in this intersection.
! Then we make a MPI reduction with all the local minima.
-!
+!
!! External
!! --------
!
@@ -1628,6 +1673,9 @@ REAL, DIMENSION(:,:), ALLOCATABLE :: ZSUM_ll
!
!* 0. DECLARATIONS
!
+#ifdef _OPENACC
+ use modd_cst, only: XMNH_HUGE
+#endif
USE MODD_PARAMETERS_ll, ONLY: JPHEXT, JPVEXT
USE MODD_VAR_ll, ONLY: TCRRT_COMDATA
!
@@ -1644,6 +1692,9 @@ REAL, DIMENSION(:,:), ALLOCATABLE :: ZSUM_ll
!
INTEGER, OPTIONAL, INTENT(IN) :: KXOR, KYOR, KZOR, & ! Coordinates
KXEND, KYEND, KZEND ! of the region
+#ifdef _OPENACC
+ logical, optional, intent(in) :: oondevice ! True if PFIELD is on device (OpenACC)
+#endif
!
!* 0.2 declarations of local variables
!
@@ -1661,9 +1712,21 @@ REAL, DIMENSION(:,:), ALLOCATABLE :: ZSUM_ll
! the corresponding boundary
!
INTEGER :: IIMAX, IJMAX ! global dimensions of the physical model
+#ifdef _OPENACC
+ integer :: ji, jj, jk
+ logical :: gondevice
+#endif
!
!-------------------------------------------------------------------------------
!
+#ifdef _OPENACC
+ if ( present( oondevice ) ) then
+ gondevice = oondevice
+ else
+ gondevice = .false.
+ end if
+#endif
+
KINFO = 0
!
ZMIN = 0
@@ -1690,7 +1753,7 @@ REAL, DIMENSION(:,:), ALLOCATABLE :: ZSUM_ll
!
IF(PRESENT(KXOR) .AND. PRESENT(KYOR) .AND. PRESENT(KZOR) .AND. &
PRESENT(KXEND) .AND. PRESENT(KYEND) .AND. PRESENT(KZEND) ) THEN
-!
+!
IXOR = KXOR
IYOR = KYOR
IZOR = KZOR
@@ -1704,7 +1767,7 @@ REAL, DIMENSION(:,:), ALLOCATABLE :: ZSUM_ll
!
IXOR = 1 + JPHEXT
IYOR = 1 + JPHEXT
- IZOR = 1 + JPVEXT
+ IZOR = 1 + JPVEXT
IXEND = IIMAX + JPHEXT
IYEND = IJMAX + JPHEXT
IZEND = SIZE(PFIELD,3)-JPVEXT
@@ -1764,7 +1827,27 @@ REAL, DIMENSION(:,:), ALLOCATABLE :: ZSUM_ll
!
! 3.2.2 perform the search
!
+#ifdef _OPENACC
+ if ( gondevice ) then
+!The present directive is useful to check if the array is really available
+!$acc kernels present(PFIELD)
+!PW: bug PGI 18.10: MINVAL function seems bugged on GPU
+! ZMIN = MINVAL(PFIELD(IIB:IIE, IJB:IJE, IZOR:IZEND))
+ zmin = XMNH_HUGE
+ do jk = izor, izend
+ do jj = ijb, ije
+ do ji = iib, iie
+ if ( pfield(ji, jj, jk ) < zmin ) zmin = pfield(ji, jj, jk )
+ end do
+ end do
+ end do
+!$acc end kernels
+ else
+ ZMIN = MINVAL(PFIELD(IIB:IIE, IJB:IJE, IZOR:IZEND))
+ end if
+#else
ZMIN = MINVAL(PFIELD(IIB:IIE, IJB:IJE, IZOR:IZEND))
+#endif
!
ENDIF
!
@@ -1786,9 +1869,9 @@ REAL, DIMENSION(:,:), ALLOCATABLE :: ZSUM_ll
!!**** *SUMMASK_ll* function to calculate the sum of the 3D field PFIELD
! on the whole domain according the directions x and y
! for the points that pass the 2D-horizontal mask OMASK.
-!
+!
! The result is a 1D array of dimension SIZE(PFIELD,3).
-!
+!
!! Purpose
!! -------
!
@@ -1796,10 +1879,10 @@ REAL, DIMENSION(:,:), ALLOCATABLE :: ZSUM_ll
!! ------
! Each processor computes the summation in its subdomain
! then we make a MPI reduction with the local result.
-!
+!
!! External
!! --------
-!
+!
!! Implicit Arguments
!! ------------------
!
@@ -1841,7 +1924,7 @@ REAL, DIMENSION(:,:), ALLOCATABLE :: ZSUM_ll
!
REAL, DIMENSION(SIZE(PFIELD,3)), TARGET :: ZBUF, ZBUFD ! Intermediate buffers
!
- INTEGER :: JK ! Loop control
+ INTEGER :: JK ! Loop control
!
!-------------------------------------------------------------------------------
!
@@ -1878,9 +1961,9 @@ REAL, DIMENSION(:,:), ALLOCATABLE :: ZSUM_ll
!!**** *SUMMASKCOMP_ll* function to calculate the sum of the 3D field PFIELD
! on the whole domain according the three directions
! for the points that pass the 2D-horizontal mask OMASK.
-!
+!
! The result is a scalar
-!
+!
!! Purpose
!! -------
!
@@ -1888,10 +1971,10 @@ REAL, DIMENSION(:,:), ALLOCATABLE :: ZSUM_ll
!! ------
! Each processor computes the summation in its subdomain
! then we make a MPI reduction with the local result.
-!
+!
! External
!! --------
-!
+!
!! Implicit Arguments
!! ------------------
!
@@ -1918,7 +2001,7 @@ REAL, DIMENSION(:,:), ALLOCATABLE :: ZSUM_ll
REAL, DIMENSION(:,:,:), INTENT(IN) :: PFIELD ! 3d Field
!
LOGICAL, DIMENSION(:,:), INTENT(IN) :: OMASK ! 2d Mask
-!
+!
INTEGER, INTENT(OUT) :: KINFO ! Returned Info
!
!* 0.2 declarations of local variables
@@ -1964,12 +2047,12 @@ REAL, DIMENSION(:,:), ALLOCATABLE :: ZSUM_ll
! result of local summations in y-direction.
! The purpose of this routine is to merge all the local sum arrays in a
! global one PRES.
-!
+!
!! Method
!! ------
! Each processor fills its part of PRES array in an intermediate buffer
! ZBUF with its local PFIELD, then we reduce the buffer in the result PRES.
-!
+!
!! External
!! --------
!
@@ -2033,7 +2116,7 @@ REAL, DIMENSION(:,:), ALLOCATABLE :: ZSUM_ll
!
IF (LWEST_ll()) THEN
IIB = 1
- IIBG = TZSPLIT%NXORE
+ IIBG = TZSPLIT%NXORE
ELSE
IIB = 1+JPHALO
IIBG = TZSPLIT%NXORP
@@ -2079,12 +2162,12 @@ REAL, DIMENSION(:,:), ALLOCATABLE :: ZSUM_ll
! result of local summations in x-direction.
! The purpose of this routine is to merge all the local sum arrays in a
! global one PRES.
-!
+!
!! Method
!! ------
! Each processor fills its part of PRES array in an intermediate buffer
! ZBUF with its local PFIELD, then we reduce the buffer in the result PRES.
-!
+!
!! External
!! --------
!
@@ -2226,11 +2309,11 @@ REAL, DIMENSION(:,:), ALLOCATABLE :: ZSUM_ll
!
!* 2. Merge local sums
!
- CALL REDUCE_SUM_1DD_ll(ZBUF1D_ll, KINFO)
+ CALL REDUCE_SUM_1DD_ll(ZBUF1D_ll, KINFO)
PRES = ZBUF1D_ll%R
ENDIF
ENDIF
-
+
END SUBROUTINE SUM_DIM1_DD_ll
! #####################################################
SUBROUTINE SUM_DIM1_ll(PFIELD, PRES, KINFO)
@@ -2244,12 +2327,12 @@ REAL, DIMENSION(:,:), ALLOCATABLE :: ZSUM_ll
! result of local summations in x-direction.
! The purpose of this routine is to merge all the local sum arrays in a
! global one PRES.
-!
+!
!! Method
!! ------
! Each processor fills its part of PRES array in an intermediate buffer
! ZBUF with its local PFIELD, then we reduce the buffer in the result PRES.
-!
+!
!! External
!! --------
!
@@ -2381,7 +2464,7 @@ REAL, DIMENSION(:,:), ALLOCATABLE :: ZSUM_ll
CALL MPI_ALLREDUCE(ZBUF, PRES, SIZE(PRES, 1) * SIZE(PRES,2), MNHREAL_MPI, MPI_SUM, NMNH_COMM_WORLD,KINFO)
ENDIF
ENDIF
-
+
END SUBROUTINE SUM_DIM1_ll
!
END MODULE MODE_SUM_ll
diff --git a/src/Rules.LXpgi.mk b/src/Rules.LXpgi.mk
index 21e995fab794f1cf2d243cc41e751e8f7b69ba0d..5f582f403c8ff5c50a41ed3ec5c480556eb52b9b 100644
--- a/src/Rules.LXpgi.mk
+++ b/src/Rules.LXpgi.mk
@@ -98,7 +98,7 @@ OPT0 = $(OPT_BASE) $(OPT_OPENACC) $(OPT_PERF0)
OPT_NOCB = $(OPT_BASE) $(OPT_OPENACC) $(OPT_PERF2)
CXXFLAGS = -acc -Kieee -Mnofma $(OPT_OPENACC)
OBJS_REPROD= spll_mode_sum_ll.o
-$(OBJS_REPROD) : OPT = $(OPT_BASE) $(OPT_PERF2) -Mvect=nosimd -Minfo=all -g
+$(OBJS_REPROD) : OPT = $(OPT_BASE) $(OPT_OPENACC) $(OPT_PERF2) -Mvect=nosimd -Minfo=all -g
endif
#
ifeq "$(OPTLEVEL)" "OPENACCDEFONLY"
@@ -108,7 +108,7 @@ OPT0 = $(OPT_BASE) $(OPT_NOOPENACC) $(OPT_PERF0)
OPT_NOCB = $(OPT_BASE) $(OPT_NOOPENACC) $(OPT_PERF2)
CXXFLAGS = -Kieee -Mnofma $(OPT_NOOPENACC)
OBJS_REPROD= spll_mode_sum_ll.o
-$(OBJS_REPROD) : OPT = $(OPT_BASE) $(OPT_PERF2) -Mvect=nosimd -Minfo=all -g
+$(OBJS_REPROD) : OPT = $(OPT_BASE) $(OPT_NOOPENACC) $(OPT_PERF2) -Mvect=nosimd -Minfo=all -g
endif
#
ifeq "$(OPTLEVEL)" "NOOPENACC"
@@ -118,7 +118,7 @@ OPT0 = $(OPT_BASE) $(OPT_NOOPENACC) $(OPT_PERF0)
OPT_NOCB = $(OPT_BASE) $(OPT_NOOPENACC) $(OPT_PERF2)
CXXFLAGS = -Kieee -Mnofma $(OPT_NOOPENACC)
OBJS_REPROD= spll_mode_sum_ll.o
-$(OBJS_REPROD) : OPT = $(OPT_BASE) $(OPT_PERF2) -Mvect=nosimd -Minfo=all -g
+$(OBJS_REPROD) : OPT = $(OPT_BASE) $(OPT_NOOPENACC) $(OPT_PERF2) -Mvect=nosimd -Minfo=all -g
endif
#
CC = pgcc