Philippe 14/12/2023: IO: add lossy compression for output files

src/LIB/SURCOUCHE/src/modd_io.f90

-!MNH_LIC Copyright 1994-2021 CNRS, Meteo-France and Universite Paul Sabatier
+!MNH_LIC Copyright 1994-2023 CNRS, Meteo-France and Universite Paul Sabatier
 !MNH_LIC This is part of the Meso-NH software governed by the CeCILL-C licence
 !MNH_LIC version 1. See LICENSE, CeCILL-C_V1-en.txt and CeCILL-C_V1-fr.txt
 !MNH_LIC for details. version 1.
@@ -12,12 +12,17 @@
 !  P. Wautelet 17/01/2020: add 'BUD' category for Print_msg + corresponding namelist variables
 !  P. Wautelet 22/09/2020: add ldimreduced in tfiledata
 !  P. Wautelet 10/11/2020: new data structures for netCDF dimensions
+!  P. Wautelet 14/12/2023: add lossy compression for output files
 !-----------------------------------------------------------------
 
 #define MNH_REDUCE_DIMENSIONS_IN_FILES 1
 
 MODULE MODD_IO
 !
+#ifdef MNH_IOCDF4
+USE NETCDF, ONLY: NF90_QUANTIZE_GRANULARBR
+#endif
+!
 use modd_netcdf,     only: tdimsnc
 USE MODD_PARAMETERS, ONLY: NDIRNAMELGTMAX, NFILENAMELGTMAX
 use modd_precision,  only: CDFINT, LFIINT
@@ -123,6 +128,9 @@ TYPE TFILEDATA
                                                                 ! instead of double precision
   LOGICAL                :: LNCCOMPRESS = .FALSE. ! Do compression on fields
   INTEGER(KIND=CDFINT)   :: NNCCOMPRESS_LEVEL = 0 ! Compression level
+  LOGICAL                :: LNCCOMPRESS_LOSSY      = .FALSE.                  ! Do lossy compression on float fields
+  INTEGER(KIND=CDFINT)   :: NNCCOMPRESS_LOSSY_ALGO = NF90_QUANTIZE_GRANULARBR ! Lossy compression algorithm
+  INTEGER(KIND=CDFINT)   :: NNCCOMPRESS_LOSSY_NSD  = 3                  ! Number of Significant Digits (or Bits)
   type(tdimsnc), pointer :: tncdims => Null()     ! Dimensions of netCDF file
 #endif
   !

src/LIB/SURCOUCHE/src/mode_io_manage_struct.f90

@@ -22,6 +22,7 @@
 !  P. Wautelet 18/03/2022: minor bugfix in ISTEP_MAX computation + adapt diagnostics messages
 !                          (change verbosity level and remove some unnecessary warnings)
 !  P. Wautelet 13/01/2023: set NMODEL field for backup and output files
+!  P. Wautelet 14/12/2023: add lossy compression for output files
 !-----------------------------------------------------------------
 MODULE MODE_IO_MANAGE_STRUCT
 !
@@ -523,7 +524,13 @@ END SUBROUTINE SORT_ENTRIES
 SUBROUTINE POPULATE_STRUCT(TPFILE_FIRST,TPFILE_LAST,KSTEPS,HFILETYPE,TPBAKOUTN,KMI)
 !#########################################################################
   !
-  USE MODD_CONFZ, ONLY: NB_PROCIO_W
+#ifdef MNH_IOCDF4
+  USE NETCDF, ONLY: NF90_QUANTIZE_BITGROOM, NF90_QUANTIZE_BITROUND, NF90_QUANTIZE_GRANULARBR
+#endif
+  !
+  USE MODD_CONFZ,      ONLY: NB_PROCIO_W
+  !
+  USE MODE_TOOLS,      ONLY: UPCASE
   !
   TYPE(TFILEDATA),           POINTER,INTENT(INOUT) :: TPFILE_FIRST,TPFILE_LAST
   INTEGER,DIMENSION(:),              INTENT(IN)    :: KSTEPS
@@ -559,16 +566,109 @@ SUBROUTINE POPULATE_STRUCT(TPFILE_FIRST,TPFILE_LAST,KSTEPS,HFILETYPE,TPBAKOUTN,K
         IF (TRIM(HFILETYPE)=='MNHOUTPUT') THEN
           ! Add a "OUT" suffix for output files
           TPBAKOUTN(IPOS)%TFILE%CNAME=ADJUSTL(ADJUSTR(IO_SURF_MNH_MODEL(IMI)%COUTFILE)//'.OUT.'//YNUMBER)
+
+#ifdef MNH_IOCDF4
           !Reduce the float precision if asked
           TPBAKOUTN(IPOS)%TFILE%LNCREDUCE_FLOAT_PRECISION = LOUT_REDUCE_FLOAT_PRECISION(IMI)
+
           !Set compression if asked
           TPBAKOUTN(IPOS)%TFILE%LNCCOMPRESS = LOUT_COMPRESS(IMI)
           IF ( NOUT_COMPRESS_LEVEL(IMI)<0 .OR. NOUT_COMPRESS_LEVEL(IMI)>9 ) THEN
-            CALL PRINT_MSG(NVERB_WARNING,'IO','POPULATE_STRUCT',&
-                           'NOUT_COMPRESS_LEVEL must be in the [0..9] range. Value forced to 4')
+            CALL PRINT_MSG( NVERB_ERROR, 'IO', 'POPULATE_STRUCT', &
+                            'NOUT_COMPRESS_LEVEL must be in the [0..9] range' )
             NOUT_COMPRESS_LEVEL(IMI) = 4
           END IF
           TPBAKOUTN(IPOS)%TFILE%NNCCOMPRESS_LEVEL = NOUT_COMPRESS_LEVEL(IMI)
+
+          !Set lossy compression
+          TPBAKOUTN(IPOS)%TFILE%LNCCOMPRESS_LOSSY = LOUT_COMPRESS_LOSSY(IMI)
+          IF ( LOUT_COMPRESS_LOSSY(IMI) ) THEN
+            !Force compression if lossy compression is enabled
+            TPBAKOUTN(IPOS)%TFILE%LNCCOMPRESS = .TRUE.
+
+            !Set lossy compression algorithm
+            SELECT CASE ( UPCASE( COUT_COMPRESS_LOSSY_ALGO(IMI) ) )
+              CASE ( 'BITGROOM' )
+                TPBAKOUTN(IPOS)%TFILE%NNCCOMPRESS_LOSSY_ALGO = NF90_QUANTIZE_BITGROOM
+              CASE ( 'GRANULARBR' )
+                TPBAKOUTN(IPOS)%TFILE%NNCCOMPRESS_LOSSY_ALGO = NF90_QUANTIZE_GRANULARBR
+              CASE ( 'BITROUND' )
+                TPBAKOUTN(IPOS)%TFILE%NNCCOMPRESS_LOSSY_ALGO = NF90_QUANTIZE_BITROUND
+              CASE DEFAULT
+                CMNHMSG(1) = 'invalid COUT_COMPRESS_LOSSY_ALGO'
+                CMNHMSG(2) = 'Accepted algorithms: BITGROOM, GRANULARBR (default choice), BITROUND'
+                CALL PRINT_MSG( NVERB_ERROR, 'IO', 'POPULATE_STRUCT' )
+                TPBAKOUTN(IPOS)%TFILE%NNCCOMPRESS_LOSSY_ALGO = NF90_QUANTIZE_GRANULARBR
+            END SELECT
+
+            !Set number of significant digits/bits for lossy compression algorithm
+#if (MNH_REAL == 4)
+            SELECT CASE ( TPBAKOUTN(IPOS)%TFILE%NNCCOMPRESS_LOSSY_ALGO )
+              CASE ( NF90_QUANTIZE_BITROUND )
+                ! For 32 bit reals, number of significant bits must be in the 1 to 23 range
+                IF ( NOUT_COMPRESS_LOSSY_NSD(IMI) < 1 .OR. NOUT_COMPRESS_LOSSY_NSD(IMI) > 23 ) THEN
+                  CALL PRINT_MSG( NVERB_ERROR, 'IO', 'POPULATE_STRUCT', &
+                                  'NOUT_COMPRESS_LOSSY_NSD must be in the [1..23] range' )
+                  NOUT_COMPRESS_LOSSY_NSD(IMI) = 7
+                END IF
+              CASE ( NF90_QUANTIZE_BITGROOM, NF90_QUANTIZE_GRANULARBR )
+                ! For 32 bit reals, number of significant digits must be in the 1 to 7 range
+                IF ( NOUT_COMPRESS_LOSSY_NSD(IMI) < 1 .OR. NOUT_COMPRESS_LOSSY_NSD(IMI) > 7 ) THEN
+                  CALL PRINT_MSG( NVERB_ERROR, 'IO', 'POPULATE_STRUCT', &
+                                  'NOUT_COMPRESS_LOSSY_NSD must be in the [1..7] range' )
+                  NOUT_COMPRESS_LOSSY_NSD(IMI) = 3
+                END IF
+              CASE DEFAULT
+                CALL PRINT_MSG( NVERB_FATAL, 'IO', 'POPULATE_STRUCT', 'invalid NNCCOMPRESS_LOSSY_ALGO (internal fatal error)' )
+            END SELECT
+#elif (MNH_REAL == 8)
+            IF ( TPBAKOUTN(IPOS)%TFILE%LNCREDUCE_FLOAT_PRECISION ) THEN
+              SELECT CASE ( TPBAKOUTN(IPOS)%TFILE%NNCCOMPRESS_LOSSY_ALGO )
+                CASE ( NF90_QUANTIZE_BITROUND )
+                  ! For 32 bit reals, number of significant bits must be in the 1 to 23 range
+                  IF ( NOUT_COMPRESS_LOSSY_NSD(IMI) < 1 .OR. NOUT_COMPRESS_LOSSY_NSD(IMI) > 23 ) THEN
+                    CALL PRINT_MSG( NVERB_ERROR, 'IO', 'POPULATE_STRUCT', &
+                                    'NOUT_COMPRESS_LOSSY_NSD must be in the [1..23] range' )
+                    NOUT_COMPRESS_LOSSY_NSD(IMI) = 7
+                  END IF
+                CASE ( NF90_QUANTIZE_BITGROOM, NF90_QUANTIZE_GRANULARBR )
+                  ! For 32 bit reals, number of significant digits must be in the 1 to 7 range
+                  IF ( NOUT_COMPRESS_LOSSY_NSD(IMI) < 1 .OR. NOUT_COMPRESS_LOSSY_NSD(IMI) > 7 ) THEN
+                    CALL PRINT_MSG( NVERB_ERROR, 'IO', 'POPULATE_STRUCT', &
+                                    'NOUT_COMPRESS_LOSSY_NSD must be in the [1..7] range' )
+                    NOUT_COMPRESS_LOSSY_NSD(IMI) = 3
+                  END IF
+                CASE DEFAULT
+                  CALL PRINT_MSG( NVERB_FATAL, 'IO', 'POPULATE_STRUCT', 'invalid NNCCOMPRESS_LOSSY_ALGO (internal fatal error)' )
+              END SELECT
+            ELSE
+              SELECT CASE ( TPBAKOUTN(IPOS)%TFILE%NNCCOMPRESS_LOSSY_ALGO )
+                CASE ( NF90_QUANTIZE_BITROUND )
+                  ! For 64 bit reals, number of significant bits must be in the 1 to 52 range
+                  IF ( NOUT_COMPRESS_LOSSY_NSD(IMI) < 1 .OR. NOUT_COMPRESS_LOSSY_NSD(IMI) > 52 ) THEN
+                    CALL PRINT_MSG( NVERB_ERROR, 'IO', 'POPULATE_STRUCT', &
+                                    'NOUT_COMPRESS_LOSSY_NSD must be in the [1..52] range' )
+                    NOUT_COMPRESS_LOSSY_NSD(IMI) = 7
+                  END IF
+                CASE ( NF90_QUANTIZE_BITGROOM, NF90_QUANTIZE_GRANULARBR )
+                  ! For 64 bit reals, number of significant digits must be in the 1 to 15 range
+                  IF ( NOUT_COMPRESS_LOSSY_NSD(IMI) < 1 .OR. NOUT_COMPRESS_LOSSY_NSD(IMI) > 15 ) THEN
+                    CALL PRINT_MSG( NVERB_ERROR, 'IO', 'POPULATE_STRUCT', &
+                                    'NOUT_COMPRESS_LOSSY_NSD must be in the [1..15] range')
+                    NOUT_COMPRESS_LOSSY_NSD(IMI) = 3
+                  END IF
+                CASE DEFAULT
+                  CALL PRINT_MSG( NVERB_FATAL, 'IO', 'POPULATE_STRUCT', 'invalid NNCCOMPRESS_LOSSY_ALGO (internal fatal error)' )
+              END SELECT
+            END IF
+#else
+#error "Invalid MNH_REAL"
+#endif
+            TPBAKOUTN(IPOS)%TFILE%NNCCOMPRESS_LOSSY_NSD = NOUT_COMPRESS_LOSSY_NSD(IMI)
+          END IF
+#endif
+
+          !Set output directory
           IF (LEN_TRIM(COUT_DIR)>0) THEN
             TPBAKOUTN(IPOS)%TFILE%CDIRNAME=TRIM(COUT_DIR)
           ELSE IF (LEN_TRIM(CIO_DIR)>0) THEN

src/MNH/modd_bakout.f90

-!MNH_LIC Copyright 1996-2018 CNRS, Meteo-France and Universite Paul Sabatier
+!MNH_LIC Copyright 1996-2023 CNRS, Meteo-France and Universite Paul Sabatier
 !MNH_LIC This is part of the Meso-NH software governed by the CeCILL-C licence
-!MNH_LIC version 1. See LICENSE, CeCILL-C_V1-en.txt and CeCILL-C_V1-fr.txt  
+!MNH_LIC version 1. See LICENSE, CeCILL-C_V1-en.txt and CeCILL-C_V1-fr.txt
 !MNH_LIC for details. version 1.
 !-----------------------------------------------------------------
 !     ##################
@@ -16,7 +16,7 @@
 !       (compression and precision reduction) of the outputs and backups realized by
 !       all models.
 !       Introduced to facilitate the output FM-file managment in case of nesting
-!     
+!
 !!**  IMPLICIT ARGUMENTS
 !!    ------------------
 !!
@@ -26,15 +26,16 @@
 !!    REFERENCE
 !!    ---------
 !!      Book2 of Meso-NH documentation (module MODD_BAKOUT)
-!!          
+!!
 !!    AUTHOR
 !!    ------
 !!	J.P. Lafore      *Meteo France*
 !!
 !!    MODIFICATIONS
 !!    -------------
-!!      Original    26/07/96                     
-!!  Philippe Wautelet: 05/2016-04/2018: new data structures and calls for I/O
+!!      Original    26/07/96
+!  P. Wautelet 05/2016-04/2018: new data structures and calls for I/O
+!  P. Wautelet 14/12/2023: add lossy compression for output files
 !-------------------------------------------------------------------------------
 !
 !*       0.   DECLARATIONS
@@ -44,31 +45,43 @@ USE MODD_PARAMETERS
 !
 IMPLICIT NONE
 !
-LOGICAL,SAVE :: LBAK_BEG = .FALSE. ! Force a backup/output at the first timestep
-LOGICAL,SAVE :: LOUT_BEG = .FALSE. ! of the segment for all models
-LOGICAL,SAVE :: LBAK_END = .FALSE. ! Force a backup/output at the last timestep
-LOGICAL,SAVE :: LOUT_END = .FALSE. ! of the segment for all models
-LOGICAL,SAVE,DIMENSION(JPMODELMAX) :: LOUT_REDUCE_FLOAT_PRECISION = .FALSE.
+SAVE
+!
+INTEGER, PARAMETER :: NCOMPRNAMELGTMAX  = 10 ! Maximum length of compression algorithm name
+!
+LOGICAL :: LBAK_BEG = .FALSE. ! Force a backup/output at the first timestep
+LOGICAL :: LOUT_BEG = .FALSE. ! of the segment for all models
+LOGICAL :: LBAK_END = .FALSE. ! Force a backup/output at the last timestep
+LOGICAL :: LOUT_END = .FALSE. ! of the segment for all models
+
+! Compression
+LOGICAL, DIMENSION(JPMODELMAX) :: LOUT_REDUCE_FLOAT_PRECISION = .FALSE.
 ! Reduce the precision of floats to single precision instead of double precision (for netCDF)
-LOGICAL,SAVE,DIMENSION(JPMODELMAX) :: LOUT_COMPRESS = .FALSE. ! Compress (float) arrays (for netCDF)
-INTEGER,SAVE,DIMENSION(JPMODELMAX) :: NOUT_COMPRESS_LEVEL = 4 ! Compression level (for netCDF)
-REAL,SAVE,ALLOCATABLE,DIMENSION(:,:)  ::   XBAK_TIME, XOUT_TIME
-! XBAK_TIME(m,i) / XOUT_TIME(m,i) array of 
+LOGICAL, DIMENSION(JPMODELMAX) :: LOUT_COMPRESS = .FALSE. ! Compress (float) arrays (for netCDF)
+INTEGER, DIMENSION(JPMODELMAX) :: NOUT_COMPRESS_LEVEL = 4 ! Compression level (for netCDF)
+
+! Lossy compression
+LOGICAL, DIMENSION(JPMODELMAX) :: LOUT_COMPRESS_LOSSY = .FALSE. ! Lossy compression of (float) arrays (for netCDF)
+CHARACTER(LEN=NCOMPRNAMELGTMAX), DIMENSION(JPMODELMAX) :: COUT_COMPRESS_LOSSY_ALGO = 'GRANULARBR' ! Quantization algorithm
+INTEGER, DIMENSION(JPMODELMAX) :: NOUT_COMPRESS_LOSSY_NSD  = 3  ! Number of Significant Digits (or Bits)
+
+REAL, ALLOCATABLE, DIMENSION(:,:)  ::   XBAK_TIME, XOUT_TIME
+! XBAK_TIME(m,i) / XOUT_TIME(m,i) array of
 ! the increments in seconds from the beginning of the segment to the
 ! instant where the i-th fields output on FM-files is realized by model "m"
-INTEGER,SAVE,ALLOCATABLE,DIMENSION(:,:)  ::   NBAK_STEP, NOUT_STEP
+INTEGER, ALLOCATABLE, DIMENSION(:,:)  ::   NBAK_STEP, NOUT_STEP
 ! NBAK_STEP(m,i) / NOUT_STEP(m,i) array of
 ! the increments in steps from the beginning of the segment to the
 ! step where the i-th fields output on FM-files is realized by model "m"
-INTEGER,SAVE,DIMENSION(JPMODELMAX) :: NBAK_STEP_FREQ = NNEGUNDEF, NOUT_STEP_FREQ = NNEGUNDEF
+INTEGER, DIMENSION(JPMODELMAX) :: NBAK_STEP_FREQ = NNEGUNDEF, NOUT_STEP_FREQ = NNEGUNDEF
 ! Number of timesteps between 2 backups/outputs for each model
-INTEGER,SAVE,DIMENSION(JPMODELMAX) :: NBAK_STEP_FREQ_FIRST = 1, NOUT_STEP_FREQ_FIRST = 1
+INTEGER, DIMENSION(JPMODELMAX) :: NBAK_STEP_FREQ_FIRST = 1, NOUT_STEP_FREQ_FIRST = 1
 ! First timestep numbers between 2 backups/outputs for each model (if NBAK/OUT_STEP_FREQ is set)
-REAL,SAVE,DIMENSION(JPMODELMAX) :: XBAK_TIME_FREQ = XNEGUNDEF, XOUT_TIME_FREQ = XNEGUNDEF
+REAL, DIMENSION(JPMODELMAX) :: XBAK_TIME_FREQ = XNEGUNDEF, XOUT_TIME_FREQ = XNEGUNDEF
 ! Time between 2 backups/outputs for each model
-REAL,SAVE,DIMENSION(JPMODELMAX) :: XBAK_TIME_FREQ_FIRST = 0., XOUT_TIME_FREQ_FIRST = 0.
+REAL, DIMENSION(JPMODELMAX) :: XBAK_TIME_FREQ_FIRST = 0., XOUT_TIME_FREQ_FIRST = 0.
 ! Time for first backup/output for each model (if XBAK/OUT_TIME_FREQ is set)
-CHARACTER(LEN=NMNHNAMELGTMAX),SAVE,ALLOCATABLE,DIMENSION(:,:) :: COUT_VAR ! Name of the fields to output
+CHARACTER(LEN=NMNHNAMELGTMAX), ALLOCATABLE, DIMENSION(:,:) :: COUT_VAR ! Name of the fields to output
 !
 !Directory names for backups/outputs
 CHARACTER(LEN=NDIRNAMELGTMAX) :: CBAK_DIR='', COUT_DIR=''

src/MNH/modn_output.f90

-!MNH_LIC Copyright 1996-2017 CNRS, Meteo-France and Universite Paul Sabatier
+!MNH_LIC Copyright 1996-2023 CNRS, Meteo-France and Universite Paul Sabatier
 !MNH_LIC This is part of the Meso-NH software governed by the CeCILL-C licence
-!MNH_LIC version 1. See LICENSE, CeCILL-C_V1-en.txt and CeCILL-C_V1-fr.txt  
+!MNH_LIC version 1. See LICENSE, CeCILL-C_V1-en.txt and CeCILL-C_V1-fr.txt
 !MNH_LIC for details. version 1.
 !-----------------------------------------------------------------
 !     ##################
@@ -24,16 +24,17 @@
 !!    REFERENCE
 !!    ---------
 !!      Book2 of Meso-NH documentation (module MODD_BAKOUT)
-!!          
+!!
 !!    AUTHOR
 !!    ------
 !!	J.P. Lafore      *Meteo France*
 !!
 !!    MODIFICATIONS
 !!    -------------
-!!      Original    26/07/96                      
-!!      Ph. Wautelet 2016       new structures for outputs/backups
-!!      Ph. Wautelet 02/10/2017 split NAM_OUTPUT in NAM_BACKUP and NAM_OUTPUT
+!!      Original    26/07/96
+!  P. Wautelet       2016: new structures for outputs/backups
+!  P. Wautelet 02/10/2017: split NAM_OUTPUT in NAM_BACKUP and NAM_OUTPUT
+!  P. Wautelet 14/12/2023: add lossy compression for output files
 !-------------------------------------------------------------------------------
 !
 !*       0.   DECLARATIONS
@@ -50,6 +51,7 @@ NAMELIST/NAM_OUTPUT/LOUT_BEG,LOUT_END,&
                    COUT_VAR, &
                    LOUT_REDUCE_FLOAT_PRECISION, &
                    LOUT_COMPRESS, NOUT_COMPRESS_LEVEL,&
+                   LOUT_COMPRESS_LOSSY, COUT_COMPRESS_LOSSY_ALGO, NOUT_COMPRESS_LOSSY_NSD, &
                    COUT_DIR
 !
 END MODULE MODN_OUTPUT