From 66f2106826b88da8ba15010706f58cc3785b0dd6 Mon Sep 17 00:00:00 2001
From: Quentin Rodier <quentin.rodier@meteo.fr>
Date: Fri, 17 Jun 2022 17:46:12 +0200
Subject: [PATCH] Quentin 17/06/2022: add KTEST 007_16janvier/turb3D and
COLD_BUBBLE + adapt compare and check for 2D simulations
---
check_commit_mesonh.sh | 57 ++++++++++++++++++++++++++++++++++++------
compare.py | 44 +++++++++++++++++++++-----------
2 files changed, 79 insertions(+), 22 deletions(-)
diff --git a/check_commit_mesonh.sh b/check_commit_mesonh.sh
index 994fceeee..fc995dab3 100755
--- a/check_commit_mesonh.sh
+++ b/check_commit_mesonh.sh
@@ -8,23 +8,27 @@ set -e
# Repertoire où MNH-V5-5-0_PHYEX.tar.gz modifie pour accueillir PHYEX se trouve
#TARGZDIR=$HOME
-availTests="007_16janvier/008_run2"
+availTests="007_16janvier/008_run2, 007_16janvier/008_run2_turb3D, COLD_BUBBLE/002_mesonh"
defaultTest="007_16janvier/008_run2"
separator='_' #- be carrefull, gmkpack (at least on belenos) has multiple allergies (':', '.', '@')
#- seprator must be in sync with prep_code.sh separator
PHYEXTOOLSDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
function usage {
- echo "Usage: $0 [-h] [-c] [-r] [-C] [-s] [--expand] commit reference"
+ echo "Usage: $0 [-h] [-c] [-r] [-C] [-s] [--expand] [-t test] commit reference"
echo "commit commit hash (or a directory)"
echo "reference commit hash or a directory or nothing for ref"
echo "-s suppress compilation pack"
echo "-c performs compilation"
echo "-r runs the tests"
echo "-C checks the result against the reference"
+ echo "-t comma separated list of tests to execute"
+ echo " or ALL to execute all tests"
echo "--expand use mnh_expand (code will use do loops)"
echo ""
echo "If nothing is asked (compilation, running, check) everything is done"
+ echo
+ echo "If no test is aked for, the default on ($defaultTest) is executed"
echo
echo "The directory (for commit only, not ref) can take the form server:directory"
echo
@@ -47,6 +51,7 @@ while [ -n "$1" ]; do
'-c') compilation=1;;
'-r') run=$(($run+1));;
'-C') check=1;;
+ '-t') tests="$2"; shift;;
'--expand') useexpand=1;;
#-b) param="$2"; shift ;;
#--) shift; break ;;
@@ -267,9 +272,19 @@ if [ $check -eq 1 ]; then
for t in $(echo $tests | sed 's/,/ /g'); do
case=$(echo $t | cut -d / -f 1)
exedir=$(echo $t | cut -d / -f 2)
+ if [ $t == 007_16janvier/008_run2 ]; then
+ path_user=$path_user_beg/MY_RUN/KTEST/007_16janvier/008_run2$path_user_end
+ path_ref=$path_ref_beg/MY_RUN/KTEST/007_16janvier/008_run2$path_ref_end
+ elif [ $t == 007_16janvier/008_run2_turb3D ]; then
+ path_user=$path_user_beg/MY_RUN/KTEST/007_16janvier/008_run2_turb3D$path_user_end
+ path_ref=$path_ref_beg/MY_RUN/KTEST/007_16janvier/008_run2_turb3D$path_ref_end
+ elif [ $t == COLD_BUBBLE/002_mesonh ]; then
+ path_user=$path_user_beg/MY_RUN/KTEST/COLD_BUBBLE/002_mesonh$path_user_end
+ path_ref=$path_ref_beg/MY_RUN/KTEST/COLD_BUBBLE/002_mesonh$path_ref_end
+ else
+ echo "cas $t non reconnu"
+ fi
- path_user=$path_user_beg/MY_RUN/KTEST/007_16janvier/008_run2$path_user_end
- path_ref=$path_ref_beg/MY_RUN/KTEST/007_16janvier/008_run2$path_ref_end
if [ ! -d $path_user ]; then
echo "$path_user is missing, please run the simulation"
exit 7
@@ -282,15 +297,17 @@ if [ $check -eq 1 ]; then
if [ $case == 007_16janvier ]; then
echo "Compare with python..."
# Compare variable of both Synchronous and Diachronic files with printing difference
+ file1=$path_user/16JAN.1.12B18.001.nc
+ file2=$path_ref/16JAN.1.12B18.001.nc
+ file3=$path_user/16JAN.1.12B18.000.nc
+ file4=$path_ref/16JAN.1.12B18.000.nc
set +e
- $PHYEXTOOLSDIR/compare.py $path_user $path_ref
+ $PHYEXTOOLSDIR/compare.py --f1 $file1 --f2 $file2 --f3 $file3 --f4 $file4
t=$?
set -e
allt=$(($allt+$t))
#Check bit-repro before date of creation of Synchronous file from ncdump of all values (pb with direct .nc file checks)
- file1=$path_user/16JAN.1.12B18.001.nc
- file2=$path_ref/16JAN.1.12B18.001.nc
echo "Compare with ncdump..."
if [ -f $file1 -a -f $file2 ]; then
set +e
@@ -304,6 +321,32 @@ if [ $check -eq 1 ]; then
allt=$(($allt+1))
fi
fi
+
+ if [ $case == COLD_BUBBLE ]; then
+ echo "Compare with python..."
+ # Compare variable of both Synchronous files with printing difference
+ file1=$path_user/BUBBL.1.CEN4T.001.nc
+ file2=$path_ref/BUBBL.1.CEN4T.001.nc
+ set +e
+ $PHYEXTOOLSDIR/compare.py --f1 $file1 --f2 $file2
+ t=$?
+ set -e
+ allt=$(($allt+$t))
+
+ #Check bit-repro before date of creation of Synchronous file from ncdump of all values (pb with direct .nc file checks)
+ echo "Compare with ncdump..."
+ if [ -f $file1 -a -f $file2 ]; then
+ set +e
+ diff <(ncdump $file1 | head -c 27350) <(ncdump $file2 | head -c 27350)
+ t=$?
+ set -e
+ allt=$(($allt+$t))
+ else
+ [ ! -f $file1 ] && echo " $file1 is missing"
+ [ ! -f $file2 ] && echo " $file2 is missing"
+ allt=$(($allt+1))
+ fi
+ fi
done
if [ $allt -eq 0 ]; then
diff --git a/compare.py b/compare.py
index 4e89274b4..a35bdf8b7 100755
--- a/compare.py
+++ b/compare.py
@@ -2,12 +2,10 @@
import xarray as xr
-def compareFiles(path_user, path_ref):
+def compareBACKUPFiles(file_user, file_ref):
status = 0
-
- filen = '16JAN.1.12B18.001.nc'
- da = xr.open_dataset(path_user + '/' + filen)
- da2 = xr.open_dataset(path_ref + '/' + filen)
+ da = xr.open_dataset(file_user)
+ da2 = xr.open_dataset(file_ref)
JPHEXT=1
JPVEXT=1
ni=len(da['ni'])
@@ -20,10 +18,18 @@ def compareFiles(path_user, path_ref):
ecart_min=float(da2[var][0,JPVEXT:nk-1-JPVEXT,JPHEXT:nj-1-JPHEXT,JPHEXT:ni-1-JPHEXT].min())-float(da[var][0,JPVEXT:nk-1-JPVEXT,JPHEXT:nj-1-JPHEXT,JPHEXT:ni-1-JPHEXT].min())
ecart_moy=float(da2[var][0,JPVEXT:nk-1-JPVEXT,JPHEXT:nj-1-JPHEXT,JPHEXT:ni-1-JPHEXT].mean())-float(da[var][0,JPVEXT:nk-1-JPVEXT,JPHEXT:nj-1-JPHEXT,JPHEXT:ni-1-JPHEXT].mean())
ecart_max=float(da2[var][0,JPVEXT:nk-1-JPVEXT,JPHEXT:nj-1-JPHEXT,JPHEXT:ni-1-JPHEXT].max())-float(da[var][0,JPVEXT:nk-1-JPVEXT,JPHEXT:nj-1-JPHEXT,JPHEXT:ni-1-JPHEXT].max())
- elif da[var].ndim == 3: #Variables time, nj, ni
+ elif da[var].ndim == 3 and da['L2D'] == 0: #Variables time, nj, ni
ecart_min=float(da2[var][0,JPHEXT:nj-1-JPHEXT,JPHEXT:ni-1-JPHEXT].min())-float(da[var][0,JPHEXT:nj-1-JPHEXT,JPHEXT:ni-1-JPHEXT].min())
ecart_moy=float(da2[var][0,JPHEXT:nj-1-JPHEXT,JPHEXT:ni-1-JPHEXT].mean())-float(da[var][0,JPHEXT:nj-1-JPHEXT,JPHEXT:ni-1-JPHEXT].mean())
ecart_max=float(da2[var][0,JPHEXT:nj-1-JPHEXT,JPHEXT:ni-1-JPHEXT].max())-float(da[var][0,JPHEXT:nj-1-JPHEXT,JPHEXT:ni-1-JPHEXT].max())
+ elif da[var].ndim == 3 and da['L2D'] == 1: #Variables time, level, nj or ni (2D simulation)
+ if da['ni'] > da['nj']:
+ nij=da['ni']
+ else:
+ nij=da['nj']
+ ecart_min=float(da2[var][0,JPVEXT:nk-1-JPVEXT,JPHEXT:nij-1-JPHEXT].min())-float(da[var][0,JPVEXT:nk-1-JPVEXT,JPHEXT:nij-1-JPHEXT].min())
+ ecart_moy=float(da2[var][0,JPVEXT:nk-1-JPVEXT,JPHEXT:nij-1-JPHEXT].mean())-float(da[var][0,JVHEXT:nk-1-JPVEXT,JPHEXT:nij-1-JPHEXT].mean())
+ ecart_max=float(da2[var][0,JPVEXT:nk-1-JPVEXT,JPHEXT:nij-1-JPHEXT].max())-float(da[var][0,JPVEXT:nk-1-JPVEXT,JPHEXT:nij-1-JPHEXT].max())
else:
ecart_min=float(da2[var].min())-float(da[var].min())
ecart_moy=float(da2[var].mean())-float(da[var].mean())
@@ -31,12 +37,15 @@ def compareFiles(path_user, path_ref):
if (ecart_min !=0 or ecart_moy !=0 or ecart_max !=0):
status += 1
print(var, ecart_min, ecart_moy, ecart_max)
+ nvar_tested+=1
except:
pass
-
- filen = '16JAN.1.12B18.000.nc'
- da = xr.open_dataset(path_user + '/' + filen)
- da2 = xr.open_dataset(path_ref + '/' + filen)
+ return status
+
+def compareTSERIESFiles(file_user, file_ref):
+ status = 0
+ da = xr.open_dataset(file_user)
+ da2 = xr.open_dataset(file_ref)
variables = list(da.keys())
for var in variables:
try:
@@ -48,16 +57,21 @@ def compareFiles(path_user, path_ref):
print(var, ecart_min, ecart_moy, ecart_max)
except:
pass
-
return status
if __name__ == "__main__":
import argparse
import sys
- parser = argparse.ArgumentParser(description='Compare toutes les variables si trouvées dans les deux fichiers')
+ parser = argparse.ArgumentParser(description='Compare toutes les variables si trouvées dans les fichiers backup et time series')
value = argparse.ArgumentParser()
- parser.add_argument('file1', metavar='file1', type=str, help="file1 user ")
- parser.add_argument('file2', metavar='file2', type=str, help="file2 reference")
+ parser.add_argument('--f1', metavar='file1', type=str, help="Backup file1 user ")
+ parser.add_argument('--f2', metavar='file2', type=str, help="Backup file2 reference")
+ parser.add_argument('--f3', metavar='file3', type=str, help=".000 file1 user ")
+ parser.add_argument('--f4', metavar='file4', type=str, help=".000 file2 reference")
args = parser.parse_args()
- sys.exit(compareFiles(args.file1, args.file2))
+ status1=compareBACKUPFiles(args.f1, args.f2)
+ print('status1 = ' + str(status1))
+ if args.f3:
+ status2=compareTSERIESFiles(args.f3, args.f4)
+ print('status2 = ' + str(status2))
--
GitLab