Rules.LXnvhpc2202.mk

#MNH_LIC Copyright 1994-2023 CNRS, Meteo-France and Universite Paul Sabatier
#MNH_LIC This is part of the Meso-NH software governed by the CeCILL-C licence
#MNH_LIC version 1. See LICENSE, CeCILL-C_V1-en.txt and CeCILL-C_V1-fr.txt
#MNH_LIC for details. version 1.
##########################################################
#                                                        #
# Compiler Options                                       #
#                                                        #
##########################################################
#OBJDIR_PATH=${WORKDIR}
#
# Processor type
#TP= -tp=sandybridge, haswell, skylake
#all x86 TP= -tp=px
#Pentium4 64-bit: p7-64
#Nuwa n375: Xeon E5-2680 v2: Ivy Bridge => between Sandy bridge and Haswell => sandybridge
#aeropc45-3: Xeon W-2265: Cascade Lake (version amelioree de Skylake) => skylake
#TP= -tp=sandybridge
#TP= -tp=skylake
#TP= -tp=p7-64
#TP= -tp=px
TP ?= -tp=native
#
#Version of CUDA
#(8.0 at least if compute capability >= 6.0)
CUDALEVEL ?= cuda11.0
#
#Compute capability of GPU
#
#OPT_CPTCAP=cc35,cc50,cc70
#OPT_CPTCAP=cc35,cc50,cc60,cc70
OPT_CPTCAP ?= cc70
#Aeropc45: cc50
#Nuwa: cc35
#Ouessant Firestone K80: cc35
#Ouessant Minsky P100: cc60
#
#Compiler info level
#OPT_INFO = -Minfo=ftn,accel,inline,ipa,loop,lre,mp,opt,par,unified,vect,ccff
OPT_INFO = -Minfo=accel
#
#Compiler profiling options
#OPT_PROF = -Mprof=ccff
#
#PW: if -Ktrap=fp: nvprof/pgprof do not work   OPT_BASE   =  -Ktrap=fp ...
#PW: -g: big impact on performance
OPT_BASE   =  $(TP) -Mbackslash -Mextend -Kieee -Mnofma  -Mallocatable=95
#
OPT_PERF0  =  -O0
OPT_PERF1  =  -O1
OPT_PERF2  =  -O2
#
OPT_MANAGED    = -Mframe -Mnostack_arrays -acc=host,gpu -gpu=nofma,$(OPT_CPTCAP),$(CUDALEVEL),managed,lineinfo $(OPT_INFO) $(OPT_PROF)
OPT_MULTICORE  = -acc=multicore $(OPT_INFO) $(OPT_PROF)
OPT_NOOPENACC  = -acc=host $(OPT_INFO) $(OPT_PROF)
OPT_OPENACC    = -Mframe -Mnostack_arrays -acc=host,gpu -gpu=nofma,$(OPT_CPTCAP),$(CUDALEVEL),lineinfo $(OPT_INFO) $(OPT_PROF)
#
OPT_CHECK  =  -C #-Mchkfpstk -Mchkptr
OPT_I8     =  -i8
OPT_R8     =  -r8
#
ifeq "$(VER_USER)" "ZSOLVER"
CPPFLAGS  += -DCARTESIANGEOMETRY -DPIECEWISELINEAR
PROG_LIST += MG_MAIN MG_MAIN_MNH_ALL
endif
#
IGNORE_OBJS += pgprof.o
#
# Real/integer 4/8 option
#
MNH_REAL  ?=8
MNH_INT   ?=4
#
ifneq "$(MNH_REAL)" "4"
OPT_BASE           += $(OPT_R8)
endif
#
OPT_BASE_I4       := $(OPT_BASE)
ifeq "$(MNH_INT)" "8"
OPT_BASE          += $(OPT_I8)
LFI_INT           ?=8
else
LFI_INT           ?=4
endif
#
#
OPT       = $(OPT_BASE) $(OPT_PERF2)
OPT0      = $(OPT_BASE) $(OPT_PERF0)
OPT_NOCB  = $(OPT_BASE) $(OPT_PERF2)
#
#
# List of Files with compilation problem in O2
#
OBJS_REPROD= spll_mode_sum_ll.o mode_sum_ll.mod  spll_mode_repro_sum.o mode_repro_sum.mod

OBJS_O1_OPENACC= spll_ice4_tendencies.o spll_turb_ver_thermo_flux.o spll_mode_device.o mode_device.mod spll_mppdb_check3d_real_mg.o \
                 spll_mppdb_check0d_real_mg.o spll_mode_turb.o mode_turb.mod spll_modd_les_n.o modd_les_n.mod
#
#
ifeq "$(OPTLEVEL)" "DEBUG"
OPT       = -g $(OPT_BASE) $(OPT_PERF0) $(OPT_CHECK)
OPT0      = -g $(OPT_BASE) $(OPT_PERF0) $(OPT_CHECK)
OPT_NOCB  = -g $(OPT_BASE) $(OPT_PERF0)
endif
#
ifeq "$(OPTLEVEL)" "MANAGED"
CPPFLAGS    += -DMNH_OPENACC -DMNH_GPUDIRECT
OPT       = -g $(OPT_BASE) $(OPT_MANAGED) $(OPT_PERF2)
OPT0      = -g $(OPT_BASE) $(OPT_MANAGED) $(OPT_PERF0)
OPT_NOCB  = -g $(OPT_BASE) $(OPT_MANAGED) $(OPT_PERF2) 
CXXFLAGS = -g -acc -Kieee -Mnofma $(OPT_MANAGED)
$(OBJS_REPROD) : OPT = $(OPT_BASE) $(OPT_MANAGED) $(OPT_PERF2) -Mvect=nosimd -Minfo=all -g
OBJS_OPENACC = spll_modd_halo_d.o modd_halo_d.mod spll_mode_mnh_allocate_mg_halo.o mode_mnh_allocate_mg_halo.mod
$(OBJS_OPENACC) : OPT = -g $(OPT_BASE) $(OPT_OPENACC) $(OPT_PERF2)
$(OBJS_O1_OPENACC) : OPT = -g $(OPT_BASE) $(OPT_MANAGED) $(OPT_PERF1)
endif
#
ifeq "$(OPTLEVEL)" "MANAGEDO2"
CPPFLAGS    += -DMNH_OPENACC -DMNH_GPUDIRECT
OPT       = $(OPT_BASE) $(OPT_MANAGED) $(OPT_PERF2)
OPT0      = $(OPT_BASE) $(OPT_MANAGED) $(OPT_PERF0)
OPT_NOCB  = $(OPT_BASE) $(OPT_MANAGED) $(OPT_PERF2) 
CXXFLAGS = -acc -Kieee -Mnofma $(OPT_MANAGED) -gpu=nofma
$(OBJS_REPROD) : OPT = $(OPT_BASE) $(OPT_MANAGED) $(OPT_PERF2) -Mvect=nosimd -Minfo=all
OBJS_OPENACC = spll_modd_halo_d.o modd_halo_d.mod spll_mode_mnh_allocate_mg_halo.o mode_mnh_allocate_mg_halo.mod
$(OBJS_OPENACC) : OPT = $(OPT_BASE) $(OPT_OPENACC) $(OPT_PERF2)
$(OBJS_O1_OPENACC) : OPT = $(OPT_BASE) $(OPT_MANAGED) $(OPT_PERF1)
endif
#
ifeq "$(OPTLEVEL)" "MULTICORE"
CPPFLAGS    += -DMNH_OPENACC
OPT       = $(OPT_BASE) $(OPT_MULTICORE) 
OPT0      = $(OPT_BASE) $(OPT_MULTICORE) $(OPT_PERF0)
OPT_NOCB  = $(OPT_BASE) $(OPT_MULTICORE) 
CXXFLAGS = -acc -Kieee -Mnofma $(OPT_MULTICORE)
endif

ifeq "$(OPTLEVEL)" "OPENACC"
CPPFLAGS    += -DMNH_OPENACC -DMNH_GPUDIRECT
# -imacros MNH_OPENACC_SHUMAN_MACRO.CPP
OPT       = -gopt $(OPT_BASE) $(OPT_OPENACC) $(OPT_PERF2)
OPT0      = -gopt $(OPT_BASE) $(OPT_OPENACC) $(OPT_PERF0)
OPT_NOCB  = -gopt $(OPT_BASE) $(OPT_OPENACC) $(OPT_PERF2) 
CXXFLAGS =  -gopt -acc -Kieee -Mnofma $(OPT_OPENACC)
$(OBJS_REPROD) : OPT = $(OPT_BASE) $(OPT_OPENACC) $(OPT_PERF2) -Mvect=nosimd -Minfo=all -gopt
$(OBJS_O1_OPENACC) : OPT = -gopt $(OPT_BASE) $(OPT_OPENACC) $(OPT_PERF1)
endif
ifeq "$(OPTLEVEL)" "OPENACCO2"
CPPFLAGS    += -DMNH_OPENACC -DMNH_GPUDIRECT
OPT       = $(OPT_BASE) $(OPT_OPENACC) $(OPT_PERF2)
OPT0      = $(OPT_BASE) $(OPT_OPENACC) $(OPT_PERF0)
OPT_NOCB  = $(OPT_BASE) $(OPT_OPENACC) $(OPT_PERF2) 
CXXFLAGS = -acc -Kieee -Mnofma $(OPT_OPENACC)
$(OBJS_REPROD) : OPT = $(OPT_BASE) $(OPT_OPENACC) $(OPT_PERF2) -Mvect=nosimd -Minfo=all
$(OBJS_O1_OPENACC) : OPT = $(OPT_BASE) $(OPT_OPENACC) $(OPT_PERF1)
endif
ifeq "$(OPTLEVEL)" "OPENACCO2NOBITREP"
MNH_BITREP = NO
CPPFLAGS    += -DMNH_OPENACC -DMNH_GPUDIRECT
OPT       = $(OPT_BASE) $(OPT_OPENACC) $(OPT_PERF2)
OPT0      = $(OPT_BASE) $(OPT_OPENACC) $(OPT_PERF0)
OPT_NOCB  = $(OPT_BASE) $(OPT_OPENACC) $(OPT_PERF2) 
CXXFLAGS = -acc -Kieee -Mnofma $(OPT_OPENACC)
$(OBJS_REPROD) : OPT = $(OPT_BASE) $(OPT_OPENACC) $(OPT_PERF2) -Mvect=nosimd -Minfo=all
$(OBJS_O1_OPENACC) : OPT = $(OPT_BASE) $(OPT_OPENACC) $(OPT_PERF1)
endif
#
ifeq "$(OPTLEVEL)" "OPENACCDEFONLY"
CPPFLAGS    += -DMNH_OPENACC -D_FAKEOPENACC
OPT       = -g $(OPT_BASE) $(OPT_NOOPENACC) $(OPT_PERF2)
OPT0      = -g $(OPT_BASE) $(OPT_NOOPENACC) $(OPT_PERF0)
OPT_NOCB  = -g $(OPT_BASE) $(OPT_NOOPENACC) $(OPT_PERF2)
CXXFLAGS = -Kieee -Mnofma $(OPT_NOOPENACC) 
$(OBJS_REPROD) : OPT = -g $(OPT_BASE) $(OPT_NOOPENACC) $(OPT_PERF2) -Mvect=nosimd -Minfo=all 
endif
#
ifeq "$(OPTLEVEL)" "NOOPENACC"
#CPPFLAGS   += -D_OPT_LINEARIZED_LOOPS
OPT       = $(OPT_BASE) $(OPT_NOOPENACC) $(OPT_PERF2)
OPT0      = $(OPT_BASE) $(OPT_NOOPENACC) $(OPT_PERF0)
OPT_NOCB  = $(OPT_BASE) $(OPT_NOOPENACC) $(OPT_PERF2)
CXXFLAGS = -Kieee -Mnofma $(OPT_NOOPENACC)
$(OBJS_REPROD) : OPT = $(OPT_BASE) $(OPT_NOOPENACC) $(OPT_PERF2) -Mvect=nosimd -Minfo=all -g
endif
#
CC = gcc
CXXFLAGS += $(TP)
#CFLAGS   += $(TP)
FC = pgf90
ifeq "$(VER_MPI)" "MPIAUTO"
F90 = mpif90
#CC = mpicc
CXX = mpicxx
else
F90 = pgf90
CC = pgcc
CXX = pgcxx
endif
#
F77FLAGS  =  $(OPT)
F77 = $(F90)
F90FLAGS  =  $(OPT)
FX90 = $(F90)
FX90FLAGS =  $(OPT)
#
LDFLAGS    =   -Wl,-warn-once $(OPT)
#
# ArmForge/MAP/DDT: Exemple of LDFLAGS for executable directly linked with MAP,lib
#
#VERMPI=3.1.5
#VERMPI ?= 4.0.5
#LDFLAGS    += -L/users/m23008/escobar/MAP/$(VERMPI) -lmap-sampler-pmpi -lmap-sampler -Wl,--eh-frame-hdr -Wl,-rpath=/users/m23008/escobar/MAP/$(VERMPI)
#
# preprocessing flags 
#
#CPP = cpp -P -traditional -Wcomment -D_OPENACC=201711 -imacros MNH_OPENACC_NV_CR.CPP
CPP = mnh_expand -D_OPENACC=201711 -imacros MNH_OPENACC_NV_CR.CPP
#
CPPFLAGS_C         = -DLITTLE_endian
CPPFLAGS_SURFEX    =
CPPFLAGS_SURCOUCHE += -DMNH_COMPILER_NVHPC
CPPFLAGS_RAD       =
CPPFLAGS_NEWLFI    = -DSWAPIO -DLINUX -DLFI_INT=${LFI_INT}
CPPFLAGS_MNH       = -DMNH=MNH -DSFX_MNH
CPPFLAGS_MNH      += -Uvector -Upixel
CPPFLAGS_MNH      += -DMNH_EXPAND -DMNH_EXPAND_OPENACC
#CPPFLAGS_MNH      += -imacros MNH_OPENACC_NV_CR.CPP
#
# BITREP flags
#
#if MNH_BITREP exists => compile with the BITREP library
MNH_BITREP ?= YES
ifeq "$(MNH_BITREP)" "YES"
CPPFLAGS_MNH += -DMNH_BITREP
endif
#
# LIBTOOLS flags
#
#if MNH_TOOLS exists => compile the tools
MNH_TOOLS = yes
#
## IOLFI flag
#
#if MNH_IOLFI exists => compile the NEWLFI library (for LFI files)
MNH_IOLFI=yes
#
## COMPRESS flag
#
#if MNH_COMPRESS exists => compile the COMPRESS library (for LFI files)
MNH_COMPRESS=yes
#
## S4PY flag
#
#if MNH_S4PY exists => compile the libs4py library (for epygram)
#MNH_S4PY=no
#
## MGSOLVER flag (multigrid solver)
#
#if MNH_MGSOLVER exists => compile the multigrid solver
MNH_MGSOLVER ?= YES
#
## ecCodes or grib_api selection
#MNH_GRIBAPI: if set to no:  use ecCodes
#             if set to yes: use grib_api (deprecated library)
#
MNH_GRIBAPI=no
#
# cdf / hdf
#
CDF_CONF="CPP=cpp"
HDF_CONF="CPP=cpp"
HDF_OPT ?= -g
NETCDF_OPT ?= -g
#
##########################################################
#                                                        #
# Source of MESONH PACKAGE  Distribution                 #
#                                                        #
##########################################################
#DIR_SURFEX   += ARCH_SRC/surfex.MNH-462

OBJS_NOCB +=  spll_isba.o
#
include Makefile.MESONH.mk
#
##########################################################
#                                                        #
# extra VPATH, Compilation flag modification             #
#         systeme module , etc ...                       #
#         external precompiled module librairie          #
#         etc ...                                        #
#                                                        #
##########################################################
ifneq "$(OPTLEVEL)" "DEBUG"
OBJS_O1 += spll_modd_isba_n.o modd_isba_n.mod spll_mode_construct_ll.o mode_construct_ll.mod \
         spll_init_surf_atm_n.o spll_mode_scatter_ll.o mode_scatter_ll.mod spll_convert_patch_teb.o \
         spll_define_mask_n.o spll_del1dfield_ll.o spll_mode_fm.o mode_fm.mod spll_mode_gather_ll.o mode_gather_ll.mod \
         spll_convect_updraft.o spll_convect_updraft_shal.o \
         spll_mode_dustopt.o mode_dustopt.mod spll_mode_saltopt.o mode_saltopt.mod \
         spll_aeroopt_get.o spll_write_lfifm1_for_diag_supp.o spll_write_lfifm1_for_diag.o spll_write_lfifm_n.o \

#spll_unpack_1d_2d_from2d.o
#spll_pack_isba_patch_n.o 
#spll_phys_param_n.o
$(OBJS_O1) : OPT = -g $(OPT_BASE) $(OPT_PERF1)

OBJS_O0= spll_mode_mppdb.o mode_mppdb.mod \
         spll_modd_les_n.o modd_les_n.mod \
         spll_default_desfm_n.o \
         spll_modd_pack_gr_field_n.o modd_pack_gr_field_n.mod

#         spll_fft55.o spll_fft.o spll_flat_invz.o \
$(OBJS_O0) : OPT = -g $(OPT_BASE) $(OPT_PERF0)

#default_desfmn: workaround a nvhpc 21.9 bug (run stall in default_desfmn)
#         spll_fast_terms.o 
#         spll_modd_ch_solver_n.o \
#         spll_modd_dummy_gr_field_n.o spll_modd_dyn_n.o 
# spll_mode_sum_ll.o


OBJS_O2= spll_mode_device.o mode_device.mod
$(OBJS_O2) : OPT = $(OPT_BASE) $(OPT_OPENACC) $(OPT_PERF2)
endif

ifneq "$(findstring 8,$(LFI_INT))" ""
OBJS_I8=spll_NEWLFI_ALL.o
$(OBJS_I8) : OPT = $(OPT_BASE) $(OPT_PERF2) $(OPT_I8)
endif

ifeq "$(MNH_INT)" "8"
OBJS_I4=spll_modd_netcdf.o modd_netcdf.mod
$(OBJS_I4) : OPT = $(OPT_BASE_I4)
endif

SPLL = spll_new

PHYEX_OPTDEFAULT = --addMPPDB_CHECKS --addStack MESONH --stopScopes toto --mathFunctoBRFunc

PHYEX_LIST = $(notdir $(shell find PHYEX/micro PHYEX/turb -follow -type f -name "*.f*" -not -name "minpack.f90" | sed -e 's/\(.*\)\(\.\).*/\1.D/g' ))
$(PHYEX_LIST) : PYFT = pyft_tool.py $(PHYEX_OPTDEFAULT)

PHYEX_SHUMAN = mode_tke_eps_sources.D mode_turb_ver_thermo_flux.D mode_turb_ver_thermo_corr.D mode_turb_ver_dyn_flux.D mode_turb_hor_splt.D mode_turb_ver_sv_corr.D mode_turb_ver_sv_flux.D mode_turb_hor_thermo_flux.D mode_turb_hor_thermo_corr.D mode_turb_hor_uv.D
$(PHYEX_SHUMAN) : PYFT = pyft_tool.py --shumanFUNCtoCALL --expandAllArraysConcurrent $(PHYEX_OPTDEFAULT)