diff --git a/INSTALL b/INSTALL index 55400ca8ab50f1f06ed3dbb078bfd698a76e1d73..3b21b97b773a70f667c9021fdd313c00347b6a88 100644 --- a/INSTALL +++ b/INSTALL @@ -1,10 +1,14 @@ Some installations are needed: -1) REFERENCE PACK - The check_commit.sh script, by default, build a pack on top of a precompiled pack. +1) IAL REFERENCE PACK + The check_commit_ial.sh script, by default, build a pack on top of a precompiled pack. This precompiled pack must be build beforehand. Instructions can be found in - ** INSTALL_pack ** + ** INSTALL_pack_ial ** -2) MNH_EXPAND +2) MESONH REFERENCE PACK + The reference pack for Meso-NH must be installed. Instructions can be found in + ** INSTALL_pack_mesonh ** + +3) MNH_EXPAND This tool is composed of two parts: the filepp software, and the MNH_Expand_Array package. Installation is described in ** INSTALL_mnh_expand ** diff --git a/INSTALL_pack b/INSTALL_pack_ial similarity index 100% rename from INSTALL_pack rename to INSTALL_pack_ial diff --git a/INSTALL_pack_mesonh b/INSTALL_pack_mesonh new file mode 100644 index 0000000000000000000000000000000000000000..a6269053f67b1b87b911b972c9e4d76debc48e84 --- /dev/null +++ b/INSTALL_pack_mesonh @@ -0,0 +1,23 @@ + +1) Compilation de la master +Il faut installer dans le répertoire pack une version de mesonh officielle (exemple pour sxphynh): +cd <git repository>/pack +cp ~rodierq/MNH-V5-5-0.tar.gz . +tar xvfz MNH-V5-5-0.tar.gz +cd MNH-V5-5-0/src +./configure +. ../conf/profile_mesonh-LXgfortran-R8I4-MNH-V5-5-0-MPIAUTO-DEBUG +make -j 8 +make installmaster + +2) Lancer les étapes de pré-processing au moins une fois en master sur le KTEST : +cd ../MY_RUN/KTEST/007_16janvier +# J'ai modifié la namelists du cas pour activer plus d'option, récupérer le nouveau dossier de run : +rm -Rf 008_run2; cp -R ~rodierq/MNH-V5-5-0/MY_RUN/KTEST/007_16janvier/008_run2 . +make clean +make #l'étape 10 ne se déroule sans erreur que si un display (X11) est disponible + #les étapes suivantes peuvent être alors lancées par "make E011_ncl E012_spectre" + +3) Récupérer le pack modifie pour accueillir PHYEX (essentiellement le makefile modifié et le répertoire PHYEX créé) +cd <git repository>/pack +cp ~rodierq/MNH-V5-5-0_PHYEX.tar.gz . diff --git a/README.md b/README.md index f992cfa00f1ed2dd652410eafa19a209115aecfc..3a92475719248be492eeb91b9e75c63380b71148 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,9 @@ This package contains tools related to the PHYEX package (https://github.com/Que Specifically, the prep_code.sh scripts prepare the source code for inclusion in the compilation machinery of the different models. -And, the check_commit.sh script compiles, executes test cases and compares results agains a reference simulation. +And, the check_commit_ial.sh script compiles, executes IAL test cases and compares the results againts a reference simulation. + +Moreover, the check_commit_mesonh.sh script compiles, executes a test case and compares the results againts a reference simulation. ## Installation @@ -12,4 +14,6 @@ Instructions can be found in INSTALL file. ## Usage -Help on check_commit.sh and prep_code.sh can be printed with the '-h' option. +Help on check_commit_ial.sh, check_commit_mesonh.sh and prep_code.sh can be printed with the '-h' option. + +For check_commit_mesonh.sh, it is mandatory to source the env.sh file diff --git a/check_commit.sh b/check_commit_ial.sh similarity index 99% rename from check_commit.sh rename to check_commit_ial.sh index c8ba46be5f7a3d74d0a7b3a400457f9f9cdfff67..6a44641839699f4b8f5f6bfd342b302f70061c52 100755 --- a/check_commit.sh +++ b/check_commit_ial.sh @@ -43,7 +43,7 @@ mainPackVersion=${mainPackVersion:-${defaultMainPackVersion}} extraCompilationCheck=1 function usage { - echo "Usage: $0 [-h] [-c] [-r ] [-C reference] [-t test] commit reference" + echo "Usage: $0 [-h] [-c] [-r] [-C] [-s] [-f] [--noexpand] [-t test] commit reference" echo "commit commit hash (or a directory, or among $specialPack) to test" echo "reference commit hash (or a directory, or among $specialPack) REF to use as a reference" echo "-s suppress compilation pack" @@ -90,7 +90,7 @@ while [ -n "$1" ]; do *) if [ -z "${commit-}" ]; then commit=$1 else - if [ -z "${reference-}" ]; then + if [ -z "${reference-}" ]; then reference=$1 else echo "Only two commit hash allowed on command line" diff --git a/check_commit_mesonh.sh b/check_commit_mesonh.sh new file mode 100755 index 0000000000000000000000000000000000000000..00cd074e3e554d95f00d82b584480509c001c2f7 --- /dev/null +++ b/check_commit_mesonh.sh @@ -0,0 +1,211 @@ +#!/bin/bash + +#set -x +set -e + +# Repertoire où Mesonh MNH-V5-5-0 officiel est installe +#REFDIR=$HOME +# Repertoire où MNH-V5-5-0_PHYEX.tar.gz modifie pour accueillir PHYEX se trouve +#TARGZDIR=$HOME + +MNHPACK=${MNHPACK:=$HOME/MesoNH/PHYEX} +PHYEXTOOLSDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +function usage { + echo "Usage: $0 [-h] [-c] [-r] [-C] commit reference" + echo "commit commit hash" + echo "reference commit hash or nothing for ref" + echo "-s suppress compilation pack" + echo "-c performs compilation" + echo "-r runs the tests, if option appears twice, run is also executed on only 2 procs (instead of 4 procs)" + echo "-C checks the result against the reference" + echo "" + echo "If nothing is asked (compilation, running, check) everything is done" +} + +compilation=0 +run=0 +check=0 +commit="" +reference="" +suppress=0 + +while [ -n "$1" ]; do + case "$1" in + '-h') usage;; + '-s') suppress=1;; + '-c') compilation=1;; + '-r') run=$(($run+1));; + '-C') check=1;; + #-b) param="$2"; shift ;; + #--) shift; break ;; + *) if [ -z "${commit-}" ]; then + commit=$1 + else + if [ -z "${reference-}" ]; then + reference=$1 + else + echo "Only two commit hash allowed on command line" + exit 1 + fi + fi;; + esac + shift +done + +if [ $compilation -eq 0 -a \ + $run -eq 0 -a \ + $check -eq 0 ]; then + compilation=1 + run=1 + check=1 +fi + +if [ -z "${commit-}" ]; then + echo "At least one commit hash must be provided on command line" + exit 2 +fi + +#if [ $check -eq 1 -a -z "${reference-}" ]; then +# echo "To perform a comparison two commit hashes are mandatory on the command line" +# exit 3 +#fi + +fromdir=$commit +if echo $commit | grep '/' > /dev/null; then + name=MNH-V5-5-0-$(echo $commit | sed 's/\//_/g') + [ $suppress -eq 1 -a -d $MNHPACK/$name ] && rm -rf $MNHPACK/$name +else + name=MNH-V5-5-0-$commit + [ $suppress -eq 1 -a -d $MNHPACK/$name ] && rm -rf $MNHPACK/$name +fi + +if [ $compilation -eq 1 ]; then + echo "### Compilation of commit $commit" + + if [ -d $MNHPACK/$name ]; then + echo "Pack already exists ($MNHPACK/$name), suppress it to be able to compile it again (or use the -s option to automatically suppress it)" + exit 5 + fi + + # Prepare the pack + cd $MNHPACK + cp $TARGZDIR/MNH-V5-5-0_PHYEX.tar.gz . + tar xvfz MNH-V5-5-0_PHYEX.tar.gz + rm MNH-V5-5-0_PHYEX.tar.gz + mv MNH-V5-5-0 $name + cd $name/src + + cd $MNHPACK + echo "Clone repository, and checkout commit $commit" + git clone https://github.com/QuentinRodier/PHYEX.git + cd PHYEX + git checkout $commit + + cd src/common/turb + # Rename all *.F90 to *.f90 + for rep in turb micro aux; do + cd ../$rep + for f in *.F90; do + mv -- "$f" "${f%.F90}.f90" + done + done + cd ../../../ + + for rep in turb micro conv ext aux; do + [ ! -d ../$rep ] && mkdir ../$rep + [ -d src/common/$rep ] && mv -f src/common/$rep/* ../$rep/ + [ -d src/mesonh/$rep ] && mv -f src/mesonh/$rep/* ../$rep/ + touch ../$rep/* + done + cd .. + # Move PHYEX files inside MNH/src/PHYEX + for rep in turb micro conv aux; do + mv $rep/* $name/src/PHYEX/$rep/. + rmdir $rep + done + + # Move manually ext/ files in src/MNH + mv -f ext/* $name/src/MNH/. + + # Clean folder + rmdir ext + rm -Rf PHYEX + + cd $name/src/PHYEX/turb + # Delete files of MNH-V5-5-0/src/MNH and MNH/src/LIB/SURCOUCHE/src with same name + for rep in turb micro conv aux ; do + cd ../$rep + for f in *.f90; do + echo $f + rm -f ../../MNH/$f + rm -f ../../LIB/SURCOUCHE/src/$f + done + done + cd .. + + # Delete old files of MNH-V5-5-0/src/MNH that is now called by mode_... NO /aux NEEDED! + find turb micro conv -name 'mode_*' > remove_non_mode.sh + sed -i 's/turb\/mode_/rm -f MNH\//g' remove_non_mode.sh + sed -i 's/micro\/mode_/rm -f MNH\//g' remove_non_mode.sh + sed -i 's/conv\/mode_/rm -f MNH\//g' remove_non_mode.sh + chmod +x remove_non_mode.sh + mv remove_non_mode.sh ../. + cd ../ + ./remove_non_mode.sh + # nettoyage, routine non appellee : + rm -f MNH/mf_turb_greyzone.f90 + rm -f MNH/compute_frac_ice.f90 + + #Configure and compilation + ./configure + set +e #file ends with a test that can return false + . ../conf/profile_mesonh-LXgfortran-R8I4-MNH-V5-5-0-MPIAUTO-DEBUG + set -e + make -j 8 + make installmaster +fi + +if [ $run -ge 1 ]; then + echo "### Running of commit $commit" + echo $commit + if [ ! -f $MNHPACK/$name/exe/MESONH* ]; then + echo "Pack does not exist ($MNHPACK/$name) or compilation has failed, please check" + exit 6 + fi + + cd $REFDIR/MNH-V5-5-0/MY_RUN/KTEST/007_16janvier/ + [ ! -d 008_run2_$commit ] && cp -R 008_run2 008_run2_$commit + cd $REFDIR/MNH-V5-5-0/MY_RUN/KTEST/007_16janvier/008_run2_$commit + + set +e #file ends with a test that can return false + [ $compilation -eq 0 ] && . $MNHPACK/$name/conf/profile_mesonh-LXgfortran-R8I4-MNH-V5-5-0-MPIAUTO-DEBUG + set -e + ./clean_mesonh_xyz + ./run_mesonh_xyz +fi + +if [ $check -eq 1 ]; then + echo "### Check commit $commit against commit $reference" + echo "Compare with python..." + # Compare variable of both Synchronous and Diachronic files with printing difference + if [ "$reference" == "" ]; then + python3 $PHYEXTOOLSDIR/compare.py $commit ref + else + python3 $PHYEXTOOLSDIR/compare.py $commit $reference + fi + + #Check bit-repro after date of creation of Synchronous file from ncdump of all values (pb with direct .nc file checks) + file1=$REFDIR/MNH-V5-5-0/MY_RUN/KTEST/007_16janvier/008_run2_$commit/16JAN.1.12B18.001.nc + if [ "$reference" == "" ]; then + file2="$REFDIR/MNH-V5-5-0/MY_RUN/KTEST/007_16janvier/008_run2/16JAN.1.12B18.001.nc" + else + file2=$REFDIR/MNH-V5-5-0/MY_RUN/KTEST/007_16janvier/008_run2_$reference/16JAN.1.12B18.001.nc + fi + echo "Compare with ncdump..." + ncdump $file1 > dump_$commit + ncdump $file2 > dump_$reference + cmp -n 62000 dump_$commit dump_$reference + rm -f dump_$commit dump_$reference + echo "...comparison done" +fi diff --git a/compare.py b/compare.py new file mode 100644 index 0000000000000000000000000000000000000000..55effd003dd99fda823d04d8f215adf02bb0cf4f --- /dev/null +++ b/compare.py @@ -0,0 +1,51 @@ +import xarray as xr +import os + +#REFDIR a renseigner +#REFDIR="/home/rodierq/" +REFDIR = os.environ['REFDIR'] + +def compareFiles(file1,file2): + path_user=REFDIR+'MNH-V5-5-0/MY_RUN/KTEST/007_16janvier/008_run2_'+file1 + if file2 == "ref": + path_ref=REFDIR+'MNH-V5-5-0/MY_RUN/KTEST/007_16janvier/008_run2' + else: + path_ref=REFDIR+'MNH-V5-5-0/MY_RUN/KTEST/007_16janvier/008_run2_'+file2 + + filen='16JAN.1.12B18.001.nc' + da = xr.open_dataset(path_user+'/'+filen) + da2 = xr.open_dataset(path_ref+'/'+filen) + variables=list(da.keys()) + for var in variables: + try: + ecart_min=float(da2[var].min())-float(da[var].min()) + ecart_moy=float(da2[var].mean())-float(da[var].mean()) + ecart_max=float(da2[var].max())-float(da[var].max()) + if (ecart_min !=0 or ecart_moy !=0 or ecart_max !=0): + print(var,ecart_min,ecart_moy,ecart_max) + except: + pass + + filen='16JAN.1.12B18.000.nc' + da = xr.open_dataset(path_user+'/'+filen) + da2 = xr.open_dataset(path_ref+'/'+filen) + variables=list(da.keys()) + for var in variables: + try: + ecart_min=float(da2[var].min())-float(da[var].min()) + ecart_moy=float(da2[var].mean())-float(da[var].mean()) + ecart_max=float(da2[var].max())-float(da[var].max()) + if (ecart_min !=0 or ecart_moy !=0 or ecart_max !=0): + print(var,ecart_min,ecart_moy,ecart_max) + except: + pass + +if __name__ == "__main__": + import argparse + parser = argparse.ArgumentParser(description='Compare toutes les variables si trouvées dans les deux fichiers') + value = argparse.ArgumentParser() + parser.add_argument('file1', metavar='file1', type=str, help="file1 user ") + parser.add_argument('file2', metavar='file2', type=str, help="file2 reference; ref for MNH-V5-5-0 reference") + args = parser.parse_args() + compareFiles(args.file1,args.file2) + diff --git a/env.sh b/env.sh index 1bec04b6d4f2486ecf5c15117ceca0a3aa584607..1d52bba34111dccc931031f646444107c74c9a4d 100644 --- a/env.sh +++ b/env.sh @@ -2,3 +2,6 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" MNH_EXPAND_DIR=$DIR/mnh_expand export PATH=$DIR:$MNH_EXPAND_DIR/filepp:$MNH_EXPAND_DIR/MNH_Expand_Array:$PATH + +export REFDIR=$DIR/pack/ +export TARGZDIR=$DIR/pack/ diff --git a/update_from_sxphynh.sh b/update_from_sxphynh.sh index 425cd86b065fbab00a427e6f2ea5c5442b0da99f..cc1d53de0c4bce39f05afcbcf2984e282db956b2 100755 --- a/update_from_sxphynh.sh +++ b/update_from_sxphynh.sh @@ -2,7 +2,7 @@ touch /scratch/work/riette/202005_externalisation_physique/update_from_sxphynh.sh -for file in check_commit.sh comp_DDH.py diffNODE.001_01 Tools prep_code.sh conf_tests update_from_sxphynh.sh env.sh; do +for file in check_commit_ial.sh comp_DDH.py diffNODE.001_01 Tools prep_code.sh conf_tests update_from_sxphynh.sh env.sh; do rsync -rltp --delete --timeout=30 \ sxphynh.cnrm.meteo.fr:/cnrm/phynh/data1/riette/DATA/202005_externalisation_physique/$file \ /scratch/work/riette/202005_externalisation_physique/