From 8321b0b37a01d8bd8ee0016fb714f32b07f99957 Mon Sep 17 00:00:00 2001
From: Juan ESCOBAR <juan.escobar@aero.obs-mip.fr>
Date: Thu, 16 Mar 2023 18:24:32 +0100
Subject: [PATCH] Juan 16/03/2023:ZSOLVER/contrav.f90, Cray OPENACC Opt, pass
 ztab* by args + dim in contrav_device/_dim

---
 src/ZSOLVER/contrav.f90 | 107 ++++++++++++++++++++++++++++++----------
 1 file changed, 80 insertions(+), 27 deletions(-)

diff --git a/src/ZSOLVER/contrav.f90 b/src/ZSOLVER/contrav.f90
index ac9d43370..a1e367d35 100644
--- a/src/ZSOLVER/contrav.f90
+++ b/src/ZSOLVER/contrav.f90
@@ -615,7 +615,64 @@ CALL GET_INDICE_ll( IIB,IJB,IIE,IJE)
 IKB=1+JPVEXT
 IKE=IKU - JPVEXT
 
-!$acc data present( PRUT, PRVT, PRWT, PDXX, PDYY, PDZZ, PDZX, PDZY, PRUCT, PRVCT, PRWCT, Z1, Z2 )
+IF (KADV_ORDER == 4 ) THEN
+   IF( .NOT. LFLAT) THEN
+      IF ( GFIRST_CALL_CONTRAV_DEVICE ) THEN
+         GFIRST_CALL_CONTRAV_DEVICE = .FALSE.
+         NULLIFY(TZHALO2_U)
+         NULLIFY(TZHALO2_V)
+         NULLIFY(TZHALO2_DZX)
+         NULLIFY(TZHALO2_DZY)
+         CALL INIT_HALO2_ll(TZHALO2_U,1,IIU,IJU,IKU)
+         CALL INIT_HALO2_ll(TZHALO2_V,1,IIU,IJU,IKU)
+         CALL INIT_HALO2_ll(TZHALO2_DZX,1,IIU,IJU,IKU)
+         CALL INIT_HALO2_ll(TZHALO2_DZY,1,IIU,IJU,IKU)
+    END IF
+    ZU_EAST => TZHALO2_U%HALO2%EAST
+    ZDZX_EAST => TZHALO2_DZX%HALO2%EAST
+    ZV_NORTH => TZHALO2_V%HALO2%NORTH
+    ZDZY_NORTH => TZHALO2_DZY%HALO2%NORTH
+   END IF
+END IF
+
+CALL CONTRAV_DEVICE_DIM(&
+         PRUT,PRVT,PRWT,&
+         PDXX,PDYY,PDZZ,PDZX,PDZY,&
+         PRUCT,PRVCT,PRWCT,Z1,Z2,&
+         ZU_EAST,ZV_NORTH,ZDZX_EAST,ZDZY_NORTH)
+
+!Release all memory allocated with MNH_MEM_GET calls since last call to MNH_MEM_POSITION_PIN
+CALL MNH_MEM_RELEASE()
+
+IF (MPPDB_INITIALIZED) THEN
+  !Check all OUT arrays
+  CALL MPPDB_CHECK(PRUCT,"CONTRAV end:PRUCT")
+  CALL MPPDB_CHECK(PRVCT,"CONTRAV end:PRVCT")
+  CALL MPPDB_CHECK(PRWCT,"CONTRAV end:PRWCT")
+END IF
+
+CONTAINS
+
+  SUBROUTINE CONTRAV_DEVICE_DIM(&
+         PRUT,PRVT,PRWT,&
+         PDXX,PDYY,PDZZ,PDZX,PDZY,&
+         PRUCT,PRVCT,PRWCT,Z1,Z2,&
+         ZU_EAST,ZV_NORTH,ZDZX_EAST,ZDZY_NORTH)
+
+    IMPLICIT NONE
+
+    REAL, DIMENSION(IIU, IJU, IKU) :: &
+         PRUT,PRVT,PRWT,&
+         PDXX,PDYY,PDZZ,PDZX,PDZY,&
+         PRUCT,PRVCT,PRWCT,Z1,Z2
+    REAL :: ZU_EAST(IJU,IKU),ZV_NORTH(IIU,IKU),ZDZX_EAST(IJU,IKU),ZDZY_NORTH(IIU,IKU)
+    
+!
+!  Begin Compute
+!
+
+!$acc data present( PRUT, PRVT, PRWT, PDXX, PDYY, PDZZ, PDZX, PDZY, PRUCT, PRVCT, PRWCT, Z1, Z2 ) &
+!$acc&     present(ZU_EAST,ZV_NORTH,ZDZX_EAST,ZDZY_NORTH)    
 
 IF (GDATA_ON_DEVICE) THEN
 !PW TODO:remplacer (ailleurs aussi...) 1/PDXX... par PINV_PDXX (fait pour la turbulence...) cfr MNH/turb_hor_splt.f90
@@ -642,17 +699,17 @@ IF (KADV_ORDER == 4 ) THEN
 !!$  NULLIFY(TZFIELD_DZY)
 !!$  CALL ADD3DFIELD_ll( TZFIELD_DZX, PDZX, 'CONTRAV::PDZX' )
 !!$  CALL ADD3DFIELD_ll( TZFIELD_DZY, PDZY, 'CONTRAV::PDZY' )
-    IF ( GFIRST_CALL_CONTRAV_DEVICE ) THEN
-       GFIRST_CALL_CONTRAV_DEVICE = .FALSE.
-       NULLIFY(TZHALO2_U)
-       NULLIFY(TZHALO2_V)
-       NULLIFY(TZHALO2_DZX)
-       NULLIFY(TZHALO2_DZY)
-       CALL INIT_HALO2_ll(TZHALO2_U,1,IIU,IJU,IKU)
-       CALL INIT_HALO2_ll(TZHALO2_V,1,IIU,IJU,IKU)
-       CALL INIT_HALO2_ll(TZHALO2_DZX,1,IIU,IJU,IKU)
-       CALL INIT_HALO2_ll(TZHALO2_DZY,1,IIU,IJU,IKU)
-    END IF
+!!$    IF ( GFIRST_CALL_CONTRAV_DEVICE ) THEN
+!!$       GFIRST_CALL_CONTRAV_DEVICE = .FALSE.
+!!$       NULLIFY(TZHALO2_U)
+!!$       NULLIFY(TZHALO2_V)
+!!$       NULLIFY(TZHALO2_DZX)
+!!$       NULLIFY(TZHALO2_DZY)
+!!$       CALL INIT_HALO2_ll(TZHALO2_U,1,IIU,IJU,IKU)
+!!$       CALL INIT_HALO2_ll(TZHALO2_V,1,IIU,IJU,IKU)
+!!$       CALL INIT_HALO2_ll(TZHALO2_DZX,1,IIU,IJU,IKU)
+!!$       CALL INIT_HALO2_ll(TZHALO2_DZY,1,IIU,IJU,IKU)
+!!$    END IF
 !!$  CALL UPDATE_HALO2_ll(TZFIELD_U, TZHALO2_U, IINFO_ll)
 !!$  CALL UPDATE_HALO2_ll(TZFIELD_V, TZHALO2_V, IINFO_ll)
 !!$  CALL UPDATE_HALO2_ll(TZFIELD_DZX, TZHALO2_DZX, IINFO_ll)
@@ -670,14 +727,14 @@ IF (KADV_ORDER == 4 ) THEN
 !!$  ALLOCATE(ZU_EAST(IJU,IKU),ZV_NORTH(IIU,IKU),ZDZX_EAST(IJU,IKU),ZDZY_NORTH(IIU,IKU))
 !!$  !$acc enter data create( zu_east, zv_north, zdzx_east, zdzy_north )
 !!$  !$acc kernels
-  ZU_EAST => TZHALO2_U%HALO2%EAST
-  ZDZX_EAST => TZHALO2_DZX%HALO2%EAST
-  ZV_NORTH => TZHALO2_V%HALO2%NORTH
-  ZDZY_NORTH => TZHALO2_DZY%HALO2%NORTH
+!!$  ZU_EAST => TZHALO2_U%HALO2%EAST
+!!$  ZDZX_EAST => TZHALO2_DZX%HALO2%EAST
+!!$  ZV_NORTH => TZHALO2_V%HALO2%NORTH
+!!$  ZDZY_NORTH => TZHALO2_DZY%HALO2%NORTH
 !!$  !$acc end kernels
 !!$!$acc update device(ZU_EAST,ZV_NORTH,ZDZX_EAST,ZDZY_NORTH)
- END IF
-END IF
+ END IF ! NOT FLAT
+END IF ! KADV_ORDER == 4
 !
 !
 !*       2.    Compute the vertical contravariant components (flat case)
@@ -897,15 +954,11 @@ END IF FLAT
 
 !$acc end data
 
-!Release all memory allocated with MNH_MEM_GET calls since last call to MNH_MEM_POSITION_PIN
-CALL MNH_MEM_RELEASE()
-
-IF (MPPDB_INITIALIZED) THEN
-  !Check all OUT arrays
-  CALL MPPDB_CHECK(PRUCT,"CONTRAV end:PRUCT")
-  CALL MPPDB_CHECK(PRVCT,"CONTRAV end:PRVCT")
-  CALL MPPDB_CHECK(PRWCT,"CONTRAV end:PRWCT")
-END IF
+!
+! End Compute
+!
+   
+  END SUBROUTINE CONTRAV_DEVICE_DIM
 
 !-----------------------------------------------------------------------
 END SUBROUTINE CONTRAV_DEVICE
-- 
GitLab