Philippe 13/12/2016: added possibility to compress and to reduce precision for...

Philippe 13/12/2016: added possibility to compress and to reduce precision for netCDF outputs + removed old (unused) references to netCDF compression

src/LIB/SURCOUCHE/src/modd_io.f90

@@ -33,7 +33,6 @@ LOGICAL, SAVE :: LPACK = .FALSE. ! TRUE if FM compression occurs in 1D or 2D mod
 LOGICAL, SAVE :: LIOCDF4    = .FALSE. ! TRUE will enable full NetCDF4 (HDF5) I/O support
 LOGICAL, SAVE :: LLFIOUT    = .FALSE. ! TRUE will also force LFI output when LIOCDF4 is on (debug only)  
 LOGICAL, SAVE :: LLFIREAD   = .FALSE. ! TRUE will force LFI read (instead of NetCDF) when LIOCDF4 is on (debug only)  
-LOGICAL, SAVE :: LDEFLATEX2 = .FALSE. ! TRUE to enable Zlib deflate compression on X2 fields  
 
 TYPE LFIPARAM
   INTEGER :: FITYP   ! FM File Type (used in FMCLOSE)
@@ -69,7 +68,9 @@ TYPE TFILEDATA
   ! Fields for netCDF files
   INTEGER(KIND=IDCDF_KIND) :: NNCID = -1 !File identifier
   LOGICAL                  :: LNCREDUCE_FLOAT_PRECISION = .FALSE. ! Reduce the precision of floats to single precision
-                                                                  ! instead of double precision (for netCDF)
+                                                                  ! instead of double precision
+  LOGICAL                  :: LNCCOMPRESS = .FALSE. ! Do compression on fields
+  INTEGER(KIND=IDCDF_KIND) :: NNCCOMPRESS_LEVEL     ! Compression level
   !
   TYPE(TFILEDATA),POINTER :: TFILE_PREV => NULL()
   TYPE(TFILEDATA),POINTER :: TFILE_NEXT => NULL()

src/LIB/SURCOUCHE/src/mode_io_manage_struct.f90

@@ -494,6 +494,15 @@ SUBROUTINE POPULATE_STRUCT(TPFILE_FIRST,TPFILE_LAST,KSTEPS,HFILETYPE,TPBAKOUTN)
         IF (HFILETYPE=='OUTPUT') THEN
           ! Add a "OUT" suffix for output files
           TPBAKOUTN(IPOS)%TFILE%CNAME=ADJUSTL(ADJUSTR(IO_SURF_MNH_MODEL(IMI)%COUTFILE)//'.OUT'//YNUMBER)
+          !Reduce the float precision if asked
+          TPBAKOUTN(IPOS)%TFILE%LNCREDUCE_FLOAT_PRECISION = LOUT_REDUCE_FLOAT_PRECISION(IMI)
+          !Set compression if asked
+          TPBAKOUTN(IPOS)%TFILE%LNCCOMPRESS = LOUT_COMPRESS(IMI)
+          IF ( NOUT_COMPRESS_LEVEL(IMI)<0 .OR. NOUT_COMPRESS_LEVEL(IMI)>9 ) THEN
+            PRINT *,'ERROR: NOUT_COMPRESS_LEVEL must be in the [0..9] range. Value forced to 4'
+            NOUT_COMPRESS_LEVEL(IMI) = 4
+          END IF
+          TPBAKOUTN(IPOS)%TFILE%NNCCOMPRESS_LEVEL = NOUT_COMPRESS_LEVEL(IMI)
         ELSE IF (HFILETYPE=='BACKUP') THEN
           TPBAKOUTN(IPOS)%TFILE%CNAME=ADJUSTL(ADJUSTR(IO_SURF_MNH_MODEL(IMI)%COUTFILE)//YNUMBER)
         ELSE

src/LIB/SURCOUCHE/src/mode_netcdf.f90

@@ -18,6 +18,9 @@ IMPLICIT NONE
 
 PRIVATE
 
+INTEGER(KIND=IDCDF_KIND),PARAMETER :: SHUFFLE = 1 !Set to 1 for (usually) better compression
+INTEGER(KIND=IDCDF_KIND),PARAMETER :: DEFLATE = 1
+
 INTERFACE IO_WRITE_FIELD_NC4
    MODULE PROCEDURE IO_WRITE_FIELD_NC4_X2,IO_WRITE_FIELD_NC4_X3, &
                     IO_WRITE_FIELD_NC4_N0, &
@@ -494,7 +497,6 @@ KRESP = IRESP
 END SUBROUTINE NCWRITX1
 
 SUBROUTINE NCWRITX2(PZCDF, HVARNAME, HDIR, PFIELD, TPFMH, KRESP)
-USE MODD_IO_ll, ONLY : LDEFLATEX2
 USE MODD_FM, ONLY : FMHEADER
 TYPE(IOCDF), POINTER             :: PZCDF
 CHARACTER(LEN=*),     INTENT(IN) :: HVARNAME
@@ -503,10 +505,6 @@ REAL, DIMENSION(:,:), INTENT(IN) :: PFIELD
 TYPE(FMHEADER),       INTENT(IN) :: TPFMH
 INTEGER,              INTENT(OUT):: KRESP
 
-INTEGER(KIND=IDCDF_KIND),PARAMETER :: SHUFFLE = 0
-INTEGER(KIND=IDCDF_KIND),PARAMETER :: DEFLATE = 1
-INTEGER(KIND=IDCDF_KIND),PARAMETER :: DEFLATE_LEVEL = 2
-
 INTEGER(KIND=IDCDF_KIND) :: STATUS
 INTEGER(KIND=IDCDF_KIND) :: INCID
 CHARACTER(LEN=30)     :: YVARNAME
@@ -531,11 +529,6 @@ IF (STATUS /= NF90_NOERR) THEN
    ! Define the variable 
    STATUS = NF90_DEF_VAR(INCID, YVARNAME, NF90_DOUBLE, IVDIMS, IVARID)
    IF (status /= NF90_NOERR) CALL HANDLE_ERR(status,__LINE__,'NCWRITX2[NF90_DEF_VAR]')
-   IF (LDEFLATEX2) THEN
-      ! Compress the variable with deflate level 2
-      STATUS = NF90_DEF_VAR_DEFLATE(INCID, IVARID, SHUFFLE, DEFLATE, DEFLATE_LEVEL)
-      IF (status /= NF90_NOERR) CALL HANDLE_ERR(status,__LINE__,'NCWRITX2[NF90_DEF_VAR_DEFLATE]')
-   END IF
    CALL WRITATTR(INCID, IVARID, TPFMH)
 ELSE
    PRINT *,'NCWRITX2 : ', TRIM(YVARNAME), ' already defined !'
@@ -590,8 +583,17 @@ IF (STATUS /= NF90_NOERR) THEN
    CALL FILLVDIMS(PZCDF, INT(SHAPE(PFIELD),KIND=IDCDF_KIND), TPFIELD%CDIR, IVDIMS)
 
    ! Define the variable 
-   STATUS = NF90_DEF_VAR(INCID, YVARNAME, NF90_DOUBLE, IVDIMS, IVARID)
+   IF (TPFILE%LNCREDUCE_FLOAT_PRECISION) THEN
+     STATUS = NF90_DEF_VAR(INCID, YVARNAME, NF90_FLOAT,  IVDIMS, IVARID)
+   ELSE
+     STATUS = NF90_DEF_VAR(INCID, YVARNAME, NF90_DOUBLE, IVDIMS, IVARID)
+   END IF
    IF (status /= NF90_NOERR) CALL HANDLE_ERR(status,__LINE__,'IO_WRITE_FIELD_NC4_X2[NF90_DEF_VAR]')
+   ! Add compression if asked for
+   IF (TPFILE%LNCCOMPRESS) THEN
+     STATUS = NF90_DEF_VAR_DEFLATE(INCID, IVARID, SHUFFLE, DEFLATE, TPFILE%NNCCOMPRESS_LEVEL)
+     IF (status /= NF90_NOERR) CALL HANDLE_ERR(status,__LINE__,'IO_WRITE_FIELD_NC4_X2[NF90_DEF_VAR_DEFLATE]')
+   END IF
    CALL IO_WRITE_FIELD_ATTR_NC4(TPFIELD,INCID,IVARID)
 ELSE
    PRINT *,'IO_WRITE_FIELD_NC4_X2: ', TRIM(YVARNAME), ' already defined !'
@@ -681,7 +683,16 @@ IF (STATUS /= NF90_NOERR) THEN
    CALL FILLVDIMS(PZCDF, INT(SHAPE(PFIELD),KIND=IDCDF_KIND), TPFIELD%CDIR, IVDIMS)
 
    ! Define the variable 
-   STATUS = NF90_DEF_VAR(INCID, YVARNAME, NF90_DOUBLE, IVDIMS, IVARID)
+   IF (TPFILE%LNCREDUCE_FLOAT_PRECISION) THEN
+     STATUS = NF90_DEF_VAR(INCID, YVARNAME, NF90_FLOAT,  IVDIMS, IVARID)
+   ELSE
+     STATUS = NF90_DEF_VAR(INCID, YVARNAME, NF90_DOUBLE, IVDIMS, IVARID)
+   END IF
+   ! Add compression if asked for
+   IF (TPFILE%LNCCOMPRESS) THEN
+     STATUS = NF90_DEF_VAR_DEFLATE(INCID, IVARID, SHUFFLE, DEFLATE, TPFILE%NNCCOMPRESS_LEVEL)
+     IF (status /= NF90_NOERR) CALL HANDLE_ERR(status,__LINE__,'IO_WRITE_FIELD_NC4_X3[NF90_DEF_VAR_DEFLATE]')
+   END IF
    IF (status /= NF90_NOERR) CALL HANDLE_ERR(status,__LINE__,'IO_WRITE_FIELD_NC4_X3[NF90_DEF_VAR]')
    CALL IO_WRITE_FIELD_ATTR_NC4(TPFIELD,INCID,IVARID)
 ELSE

src/MNH/modd_fmout.f90

@@ -52,6 +52,10 @@ LOGICAL,SAVE :: LBAK_BEG = .FALSE. ! Force a backup/output at the first timestep
 LOGICAL,SAVE :: LOUT_BEG = .FALSE. ! of the segment for all models
 LOGICAL,SAVE :: LBAK_END = .FALSE. ! Force a backup/output at the last timestep
 LOGICAL,SAVE :: LOUT_END = .FALSE. ! of the segment for all models
+LOGICAL,SAVE,DIMENSION(JPMODELMAX) :: LOUT_REDUCE_FLOAT_PRECISION = .FALSE.
+! Reduce the precision of floats to single precision instead of double precision (for netCDF)
+LOGICAL,SAVE,DIMENSION(JPMODELMAX) :: LOUT_COMPRESS = .FALSE. ! Compress (float) arrays (for netCDF)
+INTEGER,SAVE,DIMENSION(JPMODELMAX) :: NOUT_COMPRESS_LEVEL = 4 ! Compression level (for netCDF)
 REAL,SAVE,ALLOCATABLE,DIMENSION(:,:)  ::   XBAK_TIME, XOUT_TIME
 ! XBAK_TIME(m,i) / XOUT_TIME(m,i) array of 
 ! the increments in seconds from the beginning of the segment to the

src/MNH/modn_fmout.f90

@@ -53,6 +53,8 @@ NAMELIST/NAM_FMOUT/LBAK_BEG,LBAK_END,&
                    XOUT_TIME,NOUT_STEP,&
                    NOUT_STEP_FREQ,NOUT_STEP_FREQ_FIRST,&
                    XOUT_TIME_FREQ,XOUT_TIME_FREQ_FIRST, &
-                   COUT_VAR
+                   COUT_VAR, &
+                   LOUT_REDUCE_FLOAT_PRECISION, &
+                   LOUT_COMPRESS, NOUT_COMPRESS_LEVEL
 !
 END MODULE MODN_FMOUT

src/Makefile

@@ -342,7 +342,7 @@ NETCDF_OPT = ${OPT_BASE_I4:-$OPT_BASE}
 #
 cdf : $(CDF_INC)
 $(CDF_INC) : 
-	cd ${DIR_HDF} && ./configure --disable-shared --prefix=${CDF_PATH} --libdir=${CDF_PATH}/lib64 --with-zlib=no \
+	cd ${DIR_HDF} && ./configure --disable-shared --prefix=${CDF_PATH} --libdir=${CDF_PATH}/lib64 \
 	FC="$(FC)" FCFLAGS="$(HDF_OPT)" CPPFLAGS="-DpgiFortran" ${HDF_CONF} && make && make install && $(MAKE) -j 1 clean
 	cd ${DIR_CDF} && ./configure --disable-shared --prefix=${CDF_PATH} --libdir=${CDF_PATH}/lib64 --disable-cxx --enable-f90 --disable-dap \
 	FC="$(FC)" FCFLAGS="$(NETCDF_OPT)" CPPFLAGS="-DpgiFortran ${INC_NETCDF}" ${CDF_CONF} LDFLAGS=" -L${CDF_PATH}/lib64" && make && make install && $(MAKE) -j 1 clean

src/Makefile.MESONH.mk

@@ -433,7 +433,7 @@ CDF_PATH?=${DIR_CDF}-${ARCH}I${MNH_INT}
 CDF_INC?=${CDF_PATH}/include/netcdf.inc
 #
 INC_NETCDF     ?= -I${CDF_PATH}/include
-LIB_NETCDF     ?= -L${CDF_PATH}/lib -L${CDF_PATH}/lib64 -lnetcdff -lnetcdf  -lhdf5_hl -lhdf5
+LIB_NETCDF     ?= -L${CDF_PATH}/lib -L${CDF_PATH}/lib64 -lnetcdff -lnetcdf  -lhdf5_hl -lhdf5 -lz
 INC            += $(INC_NETCDF)
 LIBS           += $(LIB_NETCDF)
 #