From 491b22818cc8669db777622f620041314e40e13e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?S=C3=A9bastien=20Riette?= <sebastien.riette@meteo.fr>
Date: Wed, 29 Nov 2023 18:34:31 +0100
Subject: [PATCH] S. Riette 29 Nov 2023: testprogs compilation on NEC
---
build/with_fcm/README.md | 37 +++++++++++--------
.../arch/arch-ECMWF_NEC440MPI225DP.AU.x.env | 31 ++++++++++++++++
.../arch/arch-ECMWF_NEC440MPI225DP.AU.x.fcm | 25 +++++++++++++
.../arch/arch-ECMWF_NEC440MPI225SP.AU.x.env | 31 ++++++++++++++++
.../arch/arch-ECMWF_NEC440MPI225SP.AU.x.fcm | 25 +++++++++++++
build/with_fcm/arch/arch-ECMWF_gnu8.5.0.env | 1 +
build/with_fcm/arch/arch-ECMWF_gnu8.5.0.fcm | 26 +++++++++++++
build/with_fcm/make_fcm.sh | 10 ++++-
tools/check_commit_testprogs.sh | 1 +
tools/compare.py | 3 +-
tools/prep_code.sh | 25 +++++++------
11 files changed, 186 insertions(+), 29 deletions(-)
create mode 100644 build/with_fcm/arch/arch-ECMWF_NEC440MPI225DP.AU.x.env
create mode 100644 build/with_fcm/arch/arch-ECMWF_NEC440MPI225DP.AU.x.fcm
create mode 100644 build/with_fcm/arch/arch-ECMWF_NEC440MPI225SP.AU.x.env
create mode 100644 build/with_fcm/arch/arch-ECMWF_NEC440MPI225SP.AU.x.fcm
create mode 100644 build/with_fcm/arch/arch-ECMWF_gnu8.5.0.env
create mode 100644 build/with_fcm/arch/arch-ECMWF_gnu8.5.0.fcm
diff --git a/build/with_fcm/README.md b/build/with_fcm/README.md
index a8f26bac8..4bde6c0f5 100644
--- a/build/with_fcm/README.md
+++ b/build/with_fcm/README.md
@@ -35,25 +35,32 @@ In addition, other architectures can be added in the ${HOME}/.phyex/fcm\_arch di
The fcm file contains the compilation options to be used by the fcm tool.
Such a file can be build using a gmkfile with the command:
make\_fcm.sh --gmkfile \<GMKFILE\> --arch \<new arch name\>
-The env file is sourced juste before the compilation step (for loading modules for instance) and also
-before populating the build directory with the source code.
-Indeed, during this second step, the source code is copied or cloned and transformed by the pyft\_tool.py script.
-The active transformations are controlled by the --noexpand/--expand options given to the
-different check\_commit\_\* scripts and by the PYFT\_OPTS that can be set in the env file (only for testprogs).
-
-If the environment variable is a multi-lines string, each line must take the form
-> FILE\_DESCRIPTOR:OPTIONS
-where FILE\_DESCRIPTOR is regular expression to test against the filename. If there
-is a match, the OPTIONS can be used for the file. The last matching FILE\_DESCRIPTOR
-is used. The regular expression is tested using 'grep -e'. If a line doesn't contain
-the FILE\_DESCRIPTOR part, it applies to all source code.
+The env file is sourced several times:
+ - before populating the build directory with the source code.
+ Indeed, during this second step, the source code is copied or cloned and transformed by the pyft\_tool.py script.
+ The active transformations are controlled by the --noexpand/--expand options given to the
+ different check\_commit\_\* scripts and by the PYFT\_OPTS that can be set in the env file (only for testprogs).
+ The syntax is given below.
+ - just before the compilation step for loading modules or to defined (LIBS variable)
+ the list of system libraries to link with (defaults to 'rt ld' to link with librt and libdl).
+ - just before execution to set environment variable specific to this architecture
+ needed during execution
+
+The PYFT\_OPTS environment variable can contain a multi-lines string.
+For each file, the PYFT\_OPTS is read line by line and the last applicable line is used.
+A line can take one of these two forms:
+ - FILE\_DESCRIPTOR:OPTIONS
+ where FILE\_DESCRIPTOR is a regular expression to test against the filename. If there
+ is a match, the OPTIONS can be used for the file. The regular expression is
+ tested using 'grep -e'.
+ - OPTIONS
+ If the line doesn't contain the FILE\_DESCRIPTOR part, it applies to all source code.
For example, to transform all source code in lower case:
-> export OPTS='--lowerCase'; $0 --pyft\_opts\_env OPTS ...
+> export OPTS='--lowerCase'; pyft\_tool.py --pyft\_opts\_env OPTS ...
To transform all source code in lower case, except routines in turb directory which must be
in upper case but keeping the turb.F90 in lower case:
> export OPTS='--lowerCase
> ^turb/:--upperCase
-> ^turb/turb\..90:--lowerCase'; $0 --pyft\_opts\_env OPTS ...
-
+> ^turb/turb\..90:--lowerCase'; pyft\_tool.py --pyft\_opts\_env OPTS ...
diff --git a/build/with_fcm/arch/arch-ECMWF_NEC440MPI225DP.AU.x.env b/build/with_fcm/arch/arch-ECMWF_NEC440MPI225DP.AU.x.env
new file mode 100644
index 000000000..a5c1f093d
--- /dev/null
+++ b/build/with_fcm/arch/arch-ECMWF_NEC440MPI225DP.AU.x.env
@@ -0,0 +1,31 @@
+#######################################################
+#Selection from /home/rme/public/bin/env_nec-4.4.0_mpi-2.25.0
+
+# MPI :
+shift $# #This line suppresses the arguments received by the calling script
+ #when the current file is sourced. It is needed because
+ #necmpivars.sh tests the presence of arguments
+source /opt/nec/ve/mpi/2.25.0/bin64/necmpivars.sh
+
+# Increased stack size needed for the compiler :
+ulimit -s unlimited
+
+#######################################################
+#Selection from /home/rme/public/scripts/O-vector-engine
+# Open-MP business :
+export OMP_STACKSIZE=4G
+# VE settings
+export VE_FORT_UFMTENDIAN=ALL
+#export VE_BLAS_DGEMM_MODE=GREEDYA #needed for reporducibility
+export VE_FORT_RECORDBUF=3072000 # make input buffer longer
+#export VE_INIT_HEAP=ZERO
+export VE_FPE_ENABLE=(DIV,FOF,FUF,INV,INE)
+export VE_LD_PRELOAD=libveaccio.so.1 #use accelerated I/O
+#verbose execution summary
+export VE_TRACEBACK=VERBOSE
+#Profile report
+export MPIPROGINF=DETAIL
+
+########################################################
+#We must not use -ldl during link edition
+LIBS='rt'
diff --git a/build/with_fcm/arch/arch-ECMWF_NEC440MPI225DP.AU.x.fcm b/build/with_fcm/arch/arch-ECMWF_NEC440MPI225DP.AU.x.fcm
new file mode 100644
index 000000000..70973947a
--- /dev/null
+++ b/build/with_fcm/arch/arch-ECMWF_NEC440MPI225DP.AU.x.fcm
@@ -0,0 +1,25 @@
+# Compilation
+$FCOMPILER = mpinfort
+$BASE_FFLAGS = -c -fopenmp -traceback=verbose -fdiag-vector=3 -fdiag-inline=2 -fpic -mstack-arrays -fdefault-real=8
+$PROD_FFLAGS = -g -O3 -mno-vector-fma -finline-functions -finline-max-depth=3 -mvector-floating-divide-instruction -mvector-sqrt-instruction -fcse-after-vectorization -mvector-advance-gather
+$DEV_FFLAGS = -g -O0
+$DEBUG_FFLAGS = -g -O0 -fbounds-check -minit-stack=snan
+$CCOMPILER = mpincc
+$BASE_CFLAGS = -c -fopenmp -traceback=verbose -fdiag-vector=3 -fdiag-inline=2 -fpic
+$PROD_CFLAGS = -g -O2 -mno-vector-fma -finline-functions
+$DEV_CFLAGS =
+$DEBUG_CFLAGS =
+$OMP_FFLAGS =
+
+# Preprocessor
+$FPP_FLAGS = LINUX LITTLE_ENDIAN LITTLE ADDRESS64
+$CPP_FLAGS = LINUX LITTLE_ENDIAN LITTLE
+
+# Linker
+$LINK = mpinfort
+$BASE_LD = -traceback=verbose -fopenmp -static -static-nec -ftrace
+$OMP_LD =
+$LD_EXE_TO_SHARED =
+
+# Other
+$AR = nar
diff --git a/build/with_fcm/arch/arch-ECMWF_NEC440MPI225SP.AU.x.env b/build/with_fcm/arch/arch-ECMWF_NEC440MPI225SP.AU.x.env
new file mode 100644
index 000000000..a5c1f093d
--- /dev/null
+++ b/build/with_fcm/arch/arch-ECMWF_NEC440MPI225SP.AU.x.env
@@ -0,0 +1,31 @@
+#######################################################
+#Selection from /home/rme/public/bin/env_nec-4.4.0_mpi-2.25.0
+
+# MPI :
+shift $# #This line suppresses the arguments received by the calling script
+ #when the current file is sourced. It is needed because
+ #necmpivars.sh tests the presence of arguments
+source /opt/nec/ve/mpi/2.25.0/bin64/necmpivars.sh
+
+# Increased stack size needed for the compiler :
+ulimit -s unlimited
+
+#######################################################
+#Selection from /home/rme/public/scripts/O-vector-engine
+# Open-MP business :
+export OMP_STACKSIZE=4G
+# VE settings
+export VE_FORT_UFMTENDIAN=ALL
+#export VE_BLAS_DGEMM_MODE=GREEDYA #needed for reporducibility
+export VE_FORT_RECORDBUF=3072000 # make input buffer longer
+#export VE_INIT_HEAP=ZERO
+export VE_FPE_ENABLE=(DIV,FOF,FUF,INV,INE)
+export VE_LD_PRELOAD=libveaccio.so.1 #use accelerated I/O
+#verbose execution summary
+export VE_TRACEBACK=VERBOSE
+#Profile report
+export MPIPROGINF=DETAIL
+
+########################################################
+#We must not use -ldl during link edition
+LIBS='rt'
diff --git a/build/with_fcm/arch/arch-ECMWF_NEC440MPI225SP.AU.x.fcm b/build/with_fcm/arch/arch-ECMWF_NEC440MPI225SP.AU.x.fcm
new file mode 100644
index 000000000..1303abd94
--- /dev/null
+++ b/build/with_fcm/arch/arch-ECMWF_NEC440MPI225SP.AU.x.fcm
@@ -0,0 +1,25 @@
+# Compilation
+$FCOMPILER = mpinfort
+$BASE_FFLAGS = -c -fopenmp -traceback=verbose -fdiag-vector=3 -fdiag-inline=2 -fpic -mstack-arrays
+$PROD_FFLAGS = -g -O3 -mno-vector-fma -finline-functions -finline-max-depth=3 -mvector-floating-divide-instruction -mvector-sqrt-instruction -fcse-after-vectorization -mvector-advance-gather
+$DEV_FFLAGS = -g -O0
+$DEBUG_FFLAGS = -g -O0 -fbounds-check -minit-stack=snanf
+$CCOMPILER = mpincc
+$BASE_CFLAGS = -c -fopenmp -traceback=verbose -fdiag-vector=3 -fdiag-inline=2 -fpic
+$PROD_CFLAGS = -g -O2 -mno-vector-fma -finline-functions
+$DEV_CFLAGS =
+$DEBUG_CFLAGS =
+$OMP_FFLAGS =
+
+# Preprocessor
+$FPP_FLAGS = LINUX LITTLE_ENDIAN LITTLE ADDRESS64 PARKIND1_SINGLE
+$CPP_FLAGS = LINUX LITTLE_ENDIAN LITTLE PARKIND1_SINGLE
+
+# Linker
+$LINK = mpinfort
+$BASE_LD = -traceback=verbose -fopenmp -static -static-nec -ftrace
+$OMP_LD =
+$LD_EXE_TO_SHARED =
+
+# Other
+$AR = nar
diff --git a/build/with_fcm/arch/arch-ECMWF_gnu8.5.0.env b/build/with_fcm/arch/arch-ECMWF_gnu8.5.0.env
new file mode 100644
index 000000000..601688a7c
--- /dev/null
+++ b/build/with_fcm/arch/arch-ECMWF_gnu8.5.0.env
@@ -0,0 +1 @@
+export OMP_NUM_THREADS=8 #to ease comparison with VE
diff --git a/build/with_fcm/arch/arch-ECMWF_gnu8.5.0.fcm b/build/with_fcm/arch/arch-ECMWF_gnu8.5.0.fcm
new file mode 100644
index 000000000..1f756eea9
--- /dev/null
+++ b/build/with_fcm/arch/arch-ECMWF_gnu8.5.0.fcm
@@ -0,0 +1,26 @@
+# Compilation
+$FCOMPILER = gfortran
+$BASE_FFLAGS = -c -fPIC -fdefault-real-8 -fdefault-double-8 -ffree-line-length-none -fopenmp -fconvert=swap
+$PROD_FFLAGS = -O3
+$DEV_FFLAGS = -O1
+$DEBUG_FFLAGS = -O0 -g -fbounds-check -finit-real=snan -ffpe-trap=invalid,zero,overflow
+$CCOMPILER = gcc
+$BASE_CFLAGS = -c -fPIC -fopenmp
+$PROD_CFLAGS = -O3
+$DEV_CFLAGS = -O1
+$DEBUG_CFLAGS = -fbounds-check
+$OMP_FFLAGS =
+
+# Preprocessor
+$FPP_FLAGS = LINUX LITTLE_ENDIAN LITTLE REPRO48
+$CPP_FLAGS = LINUX LITTLE_ENDIAN LITTLE
+
+# Linker
+$LINK = gfortran
+$BASE_LD = -fPIC -fdefault-real-8 -fdefault-double-8 -fopenmp
+$OMP_LD =
+$LD_EXE_TO_SHARED = -shared
+
+# Other
+$AR = ar
+
diff --git a/build/with_fcm/make_fcm.sh b/build/with_fcm/make_fcm.sh
index ac7d03ce6..4e95821a6 100755
--- a/build/with_fcm/make_fcm.sh
+++ b/build/with_fcm/make_fcm.sh
@@ -273,7 +273,7 @@ COMPIL_CFLAGS="\\\$\${level}_CFLAGS"
LD_FLAGS="\\\$BASE_LD"
LD_FLAGS="\$LD_FLAGS \$OMP_LD"
-LIBS="rt dl"
+LIBS="${LIBS:-rt dl}"
ENTRYPOINTS="rain_ice.o shallow_mf.o turb.o ice_adjust.o"
@@ -420,12 +420,20 @@ fi
# Build the compilation script and run it
if [ $compilation -eq 1 ]; then
cd -P $DIR/arch_$ARCH
+ . arch.env
build_compilation_script src
./compilation.sh
ln -s build/bin/libphyex.so .
# Check if python can open the resulting shared lib
+ set +e
+ #On NEC, the shared library cannot be loaded as other lib if it was compiled for the Vector Engine
python3 -c "from ctypes import cdll; cdll.LoadLibrary('./libphyex.so')"
+ if [ $? -ne 0 ]; then
+ echo "On some systems (cross-compilation) it's normal to obtain an error here"
+ echo "when python tries to open the shared lib."
+ fi
+ set -e
# ldd -r ./libphyex.so should also give interesting results
fi
diff --git a/tools/check_commit_testprogs.sh b/tools/check_commit_testprogs.sh
index 5f00b3e81..7affedec6 100755
--- a/tools/check_commit_testprogs.sh
+++ b/tools/check_commit_testprogs.sh
@@ -389,6 +389,7 @@ if [ $run -ge 1 ]; then
export DR_HOOK=1
export DR_HOOK_IGNORE_SIGNALS=-1
fi
+ . $TESTDIR/$name/build/with_fcm/arch_${archfile}/arch.env
set +e
$TESTDIR/$name/build/with_fcm/arch_${archfile}/build/bin/main_${t}.exe --check $extrapolation_opts > Output_run 2> Stderr_run
stat=$?
diff --git a/tools/compare.py b/tools/compare.py
index 215039faf..bbeebdc00 100755
--- a/tools/compare.py
+++ b/tools/compare.py
@@ -384,7 +384,8 @@ def comp_testprogs(f1, f2):
for p in ('\.\.', '~=', '!='): s = re.sub(p, '', s)
s = re.sub(r'\-0.00000E\+00([|\- ])', r' 0.00000E+00\1', s)
s = re.sub(r'\n\sTotal time:.*\n', '\n', s)
- return s[247:]
+ s = re.sub(r'IBL =[ ]*', 'IBL = ', s)
+ return s[s.index('IBL'):]
r = read(f1) == read(f2)
if not r:
print("{f1} {f2} differ".format(f1=f1, f2=f2))
diff --git a/tools/prep_code.sh b/tools/prep_code.sh
index 92702aedc..5165fffc4 100755
--- a/tools/prep_code.sh
+++ b/tools/prep_code.sh
@@ -47,23 +47,24 @@ function usage {
echo "To use the pyft tool... it must be installed"
echo ""
echo "The variable name sent with --pyft_opts_env must correspond to an exported environement"
- echo "variable. This variable can be a one-line string or a multi-line string. In case of a one-line"
- echo "string, the string is added to the call to pyft_tool.py script for each file."
- echo "If the environment variable is a multi-lines string, each line must take the form"
- echo "FILE_DESCRIPTOR:OPTIONS"
- echo "where FILE_DESCRIPTOR is regular expression to test against the filename. If there"
- echo "is a match, the OPTIONS can be used for the file. The last matching FILE_DESCRIPTOR"
- echo "is used. The regular expression is tested using 'grep -e'. If a line doesn't contain"
- echo "the FILE_DESCRIPTOR part, it applies to all source code."
+ echo "variable. The variable can contain a multi-lines string."
+ echo "The variable is read line by line and the last applicable line is used."
+ echo "A line can take one of these two forms:"
+ echo " - FILE_DESCRIPTOR:OPTIONS"
+ echo " where FILE_DESCRIPTOR is a regular expression to test against the filename. If there"
+ echo " is a match, the OPTIONS can be used for the file. The regular expression is"
+ echo " tested using 'grep -e'."
+ echo " - OPTIONS"
+ echo " If the line doesn't contain the FILE_DESCRIPTOR part, it applies to all source code."
echo ""
echo "For example, to transform all source code in lower case:"
- echo "export OPTS='--lowerCase'; $0 --pyft_opts_env OPTS ..."
+ echo "> export OPTS='--lowerCase'; $0 --pyft\_opts\_env OPTS ..."
echo ""
echo "To transform all source code in lower case, except routines in turb directory which must be"
echo "in upper case but keeping the turb.F90 in lower case:"
- echo "export OPTS='--lowerCase"
- echo "^turb/:--upperCase"
- echo "^turb/turb\..90:--lowerCase'; $0 --pyft_opts_env OPTS ..."
+ echo "> export OPTS='--lowerCase "
+ echo "> ^turb/:--upperCase "
+ echo "> ^turb/turb\..90:--lowerCase'; $0 --pyft\_opts\_env OPTS ..."
}
full_command="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/$(basename "${BASH_SOURCE[0]}") $@"
--
GitLab