check_commit for mesonh added

INSTALL

 Some installations are needed:
 
-1) REFERENCE PACK
-   The check_commit.sh script, by default, build a pack on top of a precompiled pack.
+1) IAL REFERENCE PACK
+   The check_commit_ial.sh script, by default, build a pack on top of a precompiled pack.
    This precompiled pack must be build beforehand. Instructions can be found in
-   ** INSTALL_pack **
+   ** INSTALL_pack_ial **
 
-2) MNH_EXPAND
+2) MESONH REFERENCE PACK
+   The reference pack for Meso-NH must be installed. Instructions can be found in
+   ** INSTALL_pack_mesonh **
+
+3) MNH_EXPAND
    This tool is composed of two parts: the filepp software, and the MNH_Expand_Array package.
    Installation is described in ** INSTALL_mnh_expand **

INSTALL_pack_mesonh

+
+1) Compilation de la master
+Il faut installer dans le répertoire pack une version de mesonh officielle (exemple pour sxphynh):
+cd <git repository>/pack
+cp ~rodierq/MNH-V5-5-0.tar.gz .
+tar xvfz MNH-V5-5-0.tar.gz
+cd MNH-V5-5-0/src
+./configure
+. ../conf/profile_mesonh-LXgfortran-R8I4-MNH-V5-5-0-MPIAUTO-DEBUG
+make -j 8
+make installmaster
+
+2) Lancer les étapes de pré-processing au moins une fois en master sur le KTEST :
+cd ../MY_RUN/KTEST/007_16janvier
+# J'ai modifié la namelists du cas pour activer plus d'option, récupérer le nouveau dossier de run :
+rm -Rf 008_run2; cp -R ~rodierq/MNH-V5-5-0/MY_RUN/KTEST/007_16janvier/008_run2 .
+make clean
+make #l'étape 10 ne se déroule sans erreur que si un display (X11) est disponible
+     #les étapes suivantes peuvent être alors lancées par "make E011_ncl E012_spectre"
+
+3) Récupérer le pack modifie pour accueillir PHYEX (essentiellement le makefile modifié et le répertoire PHYEX créé)
+cd <git repository>/pack
+cp ~rodierq/MNH-V5-5-0_PHYEX.tar.gz .

README.md

@@ -4,7 +4,9 @@ This package contains tools related to the PHYEX package (https://github.com/Que
 Specifically, the prep_code.sh scripts prepare the source code for inclusion in the compilation machinery
 of the different models.
 
-And, the check_commit.sh script compiles, executes test cases and compares results agains a reference simulation.
+And, the check_commit_ial.sh script compiles, executes IAL test cases and compares the results againts a reference simulation.
+
+Moreover, the check_commit_mesonh.sh script compiles, executes a test case and compares the results againts a reference simulation.
 
 ## Installation
 
@@ -12,4 +14,6 @@ Instructions can be found in INSTALL file.
 
 ## Usage
 
-Help on check_commit.sh and prep_code.sh can be printed with the '-h' option.
+Help on check_commit_ial.sh, check_commit_mesonh.sh and prep_code.sh can be printed with the '-h' option.
+
+For check_commit_mesonh.sh, it is mandatory to source the env.sh file

check_commit.sh → check_commit_ial.sh

@@ -43,7 +43,7 @@ mainPackVersion=${mainPackVersion:-${defaultMainPackVersion}}
 extraCompilationCheck=1
 
 function usage {
-  echo "Usage: $0 [-h] [-c] [-r ] [-C reference] [-t test] commit reference"
+  echo "Usage: $0 [-h] [-c] [-r] [-C] [-s] [-f] [--noexpand] [-t test] commit reference"
   echo "commit          commit hash (or a directory, or among $specialPack) to test"
   echo "reference       commit hash (or a directory, or among $specialPack) REF to use as a reference"
   echo "-s              suppress compilation pack"
@@ -90,7 +90,7 @@ while [ -n "$1" ]; do
      *) if [ -z "${commit-}" ]; then
           commit=$1
         else
-          if [ -z "${reference-}" ]; then		
+          if [ -z "${reference-}" ]; then
             reference=$1
           else
             echo "Only two commit hash allowed on command line"

check_commit_mesonh.sh

+#!/bin/bash
+
+#set -x
+set -e
+
+# Repertoire où Mesonh MNH-V5-5-0 officiel est installe
+#REFDIR=$HOME
+# Repertoire où MNH-V5-5-0_PHYEX.tar.gz modifie pour accueillir PHYEX se trouve
+#TARGZDIR=$HOME
+
+MNHPACK=${MNHPACK:=$HOME/MesoNH/PHYEX}
+PHYEXTOOLSDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+function usage {
+  echo "Usage: $0 [-h] [-c] [-r] [-C] commit reference"
+  echo "commit          commit hash"
+  echo "reference       commit hash or nothing for ref"
+  echo "-s              suppress compilation pack"
+  echo "-c              performs compilation"
+  echo "-r              runs the tests, if option appears twice, run is also executed on only 2 procs (instead of 4 procs)"
+  echo "-C              checks the result against the reference"
+  echo ""
+  echo "If nothing is asked (compilation, running, check) everything is done"
+}
+
+compilation=0
+run=0
+check=0
+commit=""
+reference=""
+suppress=0
+
+while [ -n "$1" ]; do
+  case "$1" in
+    '-h') usage;;
+    '-s') suppress=1;;
+    '-c') compilation=1;;
+    '-r') run=$(($run+1));;
+    '-C') check=1;;
+    #-b) param="$2"; shift ;;
+    #--) shift; break ;;
+     *) if [ -z "${commit-}" ]; then
+          commit=$1
+        else
+          if [ -z "${reference-}" ]; then
+            reference=$1
+          else
+            echo "Only two commit hash allowed on command line"
+            exit 1
+          fi
+        fi;;
+  esac
+  shift
+done
+
+if [ $compilation -eq 0 -a \
+     $run -eq 0 -a \
+     $check -eq 0 ]; then
+  compilation=1
+  run=1
+  check=1
+fi
+
+if [ -z "${commit-}" ]; then
+  echo "At least one commit hash must be provided on command line"
+  exit 2
+fi
+
+#if [ $check -eq 1 -a -z "${reference-}" ]; then
+#  echo "To perform a comparison two commit hashes are mandatory on the command line"
+#  exit 3
+#fi
+
+fromdir=$commit
+if echo $commit | grep '/' > /dev/null; then
+  name=MNH-V5-5-0-$(echo $commit | sed 's/\//_/g')
+  [ $suppress -eq 1 -a -d $MNHPACK/$name ] && rm -rf $MNHPACK/$name
+else
+  name=MNH-V5-5-0-$commit
+  [ $suppress -eq 1 -a -d $MNHPACK/$name ] && rm -rf $MNHPACK/$name
+fi
+
+if [ $compilation -eq 1 ]; then
+  echo "### Compilation of commit $commit"
+
+  if [ -d $MNHPACK/$name ]; then
+    echo "Pack already exists ($MNHPACK/$name), suppress it to be able to compile it again (or use the -s option to automatically suppress it)"
+    exit 5
+  fi
+
+  # Prepare the pack
+  cd $MNHPACK
+  cp $TARGZDIR/MNH-V5-5-0_PHYEX.tar.gz .
+  tar xvfz MNH-V5-5-0_PHYEX.tar.gz 
+  rm MNH-V5-5-0_PHYEX.tar.gz
+  mv MNH-V5-5-0 $name
+  cd $name/src
+
+  cd $MNHPACK
+  echo "Clone repository, and checkout commit $commit"
+  git clone https://github.com/QuentinRodier/PHYEX.git
+  cd PHYEX
+  git checkout $commit
+  
+  cd src/common/turb
+  # Rename all *.F90 to *.f90
+  for rep in turb micro aux; do
+    cd ../$rep  
+    for f in *.F90; do 
+      mv -- "$f" "${f%.F90}.f90"
+    done
+  done
+  cd ../../../
+
+  for rep in turb micro conv ext aux; do
+    [ ! -d ../$rep ] && mkdir ../$rep
+    [ -d src/common/$rep ] && mv -f src/common/$rep/* ../$rep/
+    [ -d src/mesonh/$rep ] && mv -f src/mesonh/$rep/* ../$rep/
+    touch ../$rep/*
+  done
+  cd ..
+  # Move PHYEX files inside MNH/src/PHYEX
+  for rep in turb micro conv aux; do
+    mv $rep/* $name/src/PHYEX/$rep/.
+    rmdir $rep
+  done
+
+  # Move manually ext/ files in src/MNH
+  mv -f ext/* $name/src/MNH/. 
+
+  # Clean folder
+  rmdir ext
+  rm -Rf PHYEX
+
+  cd $name/src/PHYEX/turb
+  # Delete files of MNH-V5-5-0/src/MNH and MNH/src/LIB/SURCOUCHE/src with same name
+  for rep in turb micro conv aux ; do
+    cd ../$rep
+    for f in *.f90; do
+      echo $f
+      rm -f ../../MNH/$f
+      rm -f ../../LIB/SURCOUCHE/src/$f
+    done
+  done
+  cd ..
+  
+  # Delete old files of MNH-V5-5-0/src/MNH that is now called by mode_... NO /aux NEEDED!
+  find turb micro conv -name 'mode_*' > remove_non_mode.sh
+  sed -i 's/turb\/mode_/rm -f MNH\//g' remove_non_mode.sh
+  sed -i 's/micro\/mode_/rm -f MNH\//g' remove_non_mode.sh
+  sed -i 's/conv\/mode_/rm -f MNH\//g' remove_non_mode.sh
+  chmod +x remove_non_mode.sh
+  mv remove_non_mode.sh ../.
+  cd ../
+  ./remove_non_mode.sh
+  # nettoyage, routine non appellee : 
+  rm -f MNH/mf_turb_greyzone.f90
+  rm -f MNH/compute_frac_ice.f90
+
+  #Configure and compilation
+  ./configure
+  set +e #file ends with a test that can return false
+  . ../conf/profile_mesonh-LXgfortran-R8I4-MNH-V5-5-0-MPIAUTO-DEBUG
+  set -e
+  make -j 8
+  make installmaster
+fi
+
+if [ $run -ge 1 ]; then
+  echo "### Running of commit $commit"
+  echo $commit
+  if [ ! -f $MNHPACK/$name/exe/MESONH* ]; then
+    echo "Pack does not exist ($MNHPACK/$name) or compilation has failed, please check"
+    exit 6
+  fi
+
+  cd $REFDIR/MNH-V5-5-0/MY_RUN/KTEST/007_16janvier/
+  [ ! -d 008_run2_$commit ] && cp -R 008_run2 008_run2_$commit
+  cd $REFDIR/MNH-V5-5-0/MY_RUN/KTEST/007_16janvier/008_run2_$commit
+
+  set +e #file ends with a test that can return false
+  [ $compilation -eq 0 ] && . $MNHPACK/$name/conf/profile_mesonh-LXgfortran-R8I4-MNH-V5-5-0-MPIAUTO-DEBUG
+  set -e
+  ./clean_mesonh_xyz
+  ./run_mesonh_xyz
+fi
+  
+if [ $check -eq 1 ]; then
+  echo "### Check commit $commit against commit $reference"
+  echo "Compare with python..."
+  # Compare variable of both Synchronous and Diachronic files with printing difference
+  if [ "$reference" == "" ]; then
+    python3 $PHYEXTOOLSDIR/compare.py $commit ref
+  else
+    python3 $PHYEXTOOLSDIR/compare.py $commit $reference
+  fi
+  
+  #Check bit-repro after date of creation of Synchronous file from ncdump of all values (pb with direct .nc file checks)
+  file1=$REFDIR/MNH-V5-5-0/MY_RUN/KTEST/007_16janvier/008_run2_$commit/16JAN.1.12B18.001.nc 
+  if [ "$reference" == "" ]; then
+    file2="$REFDIR/MNH-V5-5-0/MY_RUN/KTEST/007_16janvier/008_run2/16JAN.1.12B18.001.nc"
+  else
+    file2=$REFDIR/MNH-V5-5-0/MY_RUN/KTEST/007_16janvier/008_run2_$reference/16JAN.1.12B18.001.nc
+  fi
+  echo "Compare with ncdump..."
+  ncdump $file1 > dump_$commit
+  ncdump $file2 > dump_$reference
+  cmp -n 62000 dump_$commit dump_$reference
+  rm -f dump_$commit dump_$reference
+  echo "...comparison done"
+fi

compare.py

+import xarray as xr
+import os
+
+#REFDIR a renseigner
+#REFDIR="/home/rodierq/"
+REFDIR = os.environ['REFDIR']
+
+def compareFiles(file1,file2):
+ path_user=REFDIR+'MNH-V5-5-0/MY_RUN/KTEST/007_16janvier/008_run2_'+file1
+ if file2 == "ref":
+  path_ref=REFDIR+'MNH-V5-5-0/MY_RUN/KTEST/007_16janvier/008_run2'
+ else:
+  path_ref=REFDIR+'MNH-V5-5-0/MY_RUN/KTEST/007_16janvier/008_run2_'+file2
+ 
+ filen='16JAN.1.12B18.001.nc'
+ da = xr.open_dataset(path_user+'/'+filen)
+ da2 = xr.open_dataset(path_ref+'/'+filen)
+ variables=list(da.keys())
+ for var in variables:
+  try:
+   ecart_min=float(da2[var].min())-float(da[var].min())
+   ecart_moy=float(da2[var].mean())-float(da[var].mean())
+   ecart_max=float(da2[var].max())-float(da[var].max())
+   if (ecart_min !=0 or ecart_moy !=0 or ecart_max !=0):
+    print(var,ecart_min,ecart_moy,ecart_max)
+  except:
+   pass
+ 
+ filen='16JAN.1.12B18.000.nc'
+ da = xr.open_dataset(path_user+'/'+filen)
+ da2 = xr.open_dataset(path_ref+'/'+filen)
+ variables=list(da.keys())
+ for var in variables:
+  try:
+   ecart_min=float(da2[var].min())-float(da[var].min())
+   ecart_moy=float(da2[var].mean())-float(da[var].mean())
+   ecart_max=float(da2[var].max())-float(da[var].max())
+   if (ecart_min !=0 or ecart_moy !=0 or ecart_max !=0):
+    print(var,ecart_min,ecart_moy,ecart_max)
+  except:
+   pass
+
+if __name__ == "__main__":
+ import argparse
+ parser = argparse.ArgumentParser(description='Compare toutes les variables si trouvées dans les deux fichiers')
+ value = argparse.ArgumentParser()
+ parser.add_argument('file1', metavar='file1', type=str, help="file1 user ")
+ parser.add_argument('file2', metavar='file2', type=str, help="file2 reference; ref for MNH-V5-5-0 reference")
+ args = parser.parse_args()
+ compareFiles(args.file1,args.file2)
+

env.sh

@@ -2,3 +2,6 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 
 MNH_EXPAND_DIR=$DIR/mnh_expand
 export PATH=$DIR:$MNH_EXPAND_DIR/filepp:$MNH_EXPAND_DIR/MNH_Expand_Array:$PATH
+
+export REFDIR=$DIR/pack/
+export TARGZDIR=$DIR/pack/

update_from_sxphynh.sh

@@ -2,7 +2,7 @@
 
 touch /scratch/work/riette/202005_externalisation_physique/update_from_sxphynh.sh
 
-for file in check_commit.sh comp_DDH.py diffNODE.001_01 Tools prep_code.sh conf_tests update_from_sxphynh.sh env.sh; do
+for file in check_commit_ial.sh comp_DDH.py diffNODE.001_01 Tools prep_code.sh conf_tests update_from_sxphynh.sh env.sh; do
   rsync -rltp --delete --timeout=30 \
       sxphynh.cnrm.meteo.fr:/cnrm/phynh/data1/riette/DATA/202005_externalisation_physique/$file \
       /scratch/work/riette/202005_externalisation_physique/