From 2254ca6181b8191e930d73d972c5c663bf9d55fc Mon Sep 17 00:00:00 2001
From: Philippe WAUTELET <philippe.wautelet@aero.obs-mip.fr>
Date: Tue, 13 Dec 2016 09:47:39 +0100
Subject: [PATCH] Philippe 13/12/2016: added possibility to compress and to
reduce precision for netCDF outputs + removed old (unused) references to
netCDF compression
---
src/LIB/SURCOUCHE/src/modd_io.f90 | 5 +--
.../SURCOUCHE/src/mode_io_manage_struct.f90 | 9 +++++
src/LIB/SURCOUCHE/src/mode_netcdf.f90 | 35 ++++++++++++-------
src/MNH/modd_fmout.f90 | 4 +++
src/MNH/modn_fmout.f90 | 4 ++-
src/Makefile | 2 +-
src/Makefile.MESONH.mk | 2 +-
7 files changed, 44 insertions(+), 17 deletions(-)
diff --git a/src/LIB/SURCOUCHE/src/modd_io.f90 b/src/LIB/SURCOUCHE/src/modd_io.f90
index cd955daff..7cb024053 100644
--- a/src/LIB/SURCOUCHE/src/modd_io.f90
+++ b/src/LIB/SURCOUCHE/src/modd_io.f90
@@ -33,7 +33,6 @@ LOGICAL, SAVE :: LPACK = .FALSE. ! TRUE if FM compression occurs in 1D or 2D mod
LOGICAL, SAVE :: LIOCDF4 = .FALSE. ! TRUE will enable full NetCDF4 (HDF5) I/O support
LOGICAL, SAVE :: LLFIOUT = .FALSE. ! TRUE will also force LFI output when LIOCDF4 is on (debug only)
LOGICAL, SAVE :: LLFIREAD = .FALSE. ! TRUE will force LFI read (instead of NetCDF) when LIOCDF4 is on (debug only)
-LOGICAL, SAVE :: LDEFLATEX2 = .FALSE. ! TRUE to enable Zlib deflate compression on X2 fields
TYPE LFIPARAM
INTEGER :: FITYP ! FM File Type (used in FMCLOSE)
@@ -69,7 +68,9 @@ TYPE TFILEDATA
! Fields for netCDF files
INTEGER(KIND=IDCDF_KIND) :: NNCID = -1 !File identifier
LOGICAL :: LNCREDUCE_FLOAT_PRECISION = .FALSE. ! Reduce the precision of floats to single precision
- ! instead of double precision (for netCDF)
+ ! instead of double precision
+ LOGICAL :: LNCCOMPRESS = .FALSE. ! Do compression on fields
+ INTEGER(KIND=IDCDF_KIND) :: NNCCOMPRESS_LEVEL ! Compression level
!
TYPE(TFILEDATA),POINTER :: TFILE_PREV => NULL()
TYPE(TFILEDATA),POINTER :: TFILE_NEXT => NULL()
diff --git a/src/LIB/SURCOUCHE/src/mode_io_manage_struct.f90 b/src/LIB/SURCOUCHE/src/mode_io_manage_struct.f90
index e08d5a6af..f85038a76 100644
--- a/src/LIB/SURCOUCHE/src/mode_io_manage_struct.f90
+++ b/src/LIB/SURCOUCHE/src/mode_io_manage_struct.f90
@@ -494,6 +494,15 @@ SUBROUTINE POPULATE_STRUCT(TPFILE_FIRST,TPFILE_LAST,KSTEPS,HFILETYPE,TPBAKOUTN)
IF (HFILETYPE=='OUTPUT') THEN
! Add a "OUT" suffix for output files
TPBAKOUTN(IPOS)%TFILE%CNAME=ADJUSTL(ADJUSTR(IO_SURF_MNH_MODEL(IMI)%COUTFILE)//'.OUT'//YNUMBER)
+ !Reduce the float precision if asked
+ TPBAKOUTN(IPOS)%TFILE%LNCREDUCE_FLOAT_PRECISION = LOUT_REDUCE_FLOAT_PRECISION(IMI)
+ !Set compression if asked
+ TPBAKOUTN(IPOS)%TFILE%LNCCOMPRESS = LOUT_COMPRESS(IMI)
+ IF ( NOUT_COMPRESS_LEVEL(IMI)<0 .OR. NOUT_COMPRESS_LEVEL(IMI)>9 ) THEN
+ PRINT *,'ERROR: NOUT_COMPRESS_LEVEL must be in the [0..9] range. Value forced to 4'
+ NOUT_COMPRESS_LEVEL(IMI) = 4
+ END IF
+ TPBAKOUTN(IPOS)%TFILE%NNCCOMPRESS_LEVEL = NOUT_COMPRESS_LEVEL(IMI)
ELSE IF (HFILETYPE=='BACKUP') THEN
TPBAKOUTN(IPOS)%TFILE%CNAME=ADJUSTL(ADJUSTR(IO_SURF_MNH_MODEL(IMI)%COUTFILE)//YNUMBER)
ELSE
diff --git a/src/LIB/SURCOUCHE/src/mode_netcdf.f90 b/src/LIB/SURCOUCHE/src/mode_netcdf.f90
index 0f806e5be..da68aef61 100644
--- a/src/LIB/SURCOUCHE/src/mode_netcdf.f90
+++ b/src/LIB/SURCOUCHE/src/mode_netcdf.f90
@@ -18,6 +18,9 @@ IMPLICIT NONE
PRIVATE
+INTEGER(KIND=IDCDF_KIND),PARAMETER :: SHUFFLE = 1 !Set to 1 for (usually) better compression
+INTEGER(KIND=IDCDF_KIND),PARAMETER :: DEFLATE = 1
+
INTERFACE IO_WRITE_FIELD_NC4
MODULE PROCEDURE IO_WRITE_FIELD_NC4_X2,IO_WRITE_FIELD_NC4_X3, &
IO_WRITE_FIELD_NC4_N0, &
@@ -494,7 +497,6 @@ KRESP = IRESP
END SUBROUTINE NCWRITX1
SUBROUTINE NCWRITX2(PZCDF, HVARNAME, HDIR, PFIELD, TPFMH, KRESP)
-USE MODD_IO_ll, ONLY : LDEFLATEX2
USE MODD_FM, ONLY : FMHEADER
TYPE(IOCDF), POINTER :: PZCDF
CHARACTER(LEN=*), INTENT(IN) :: HVARNAME
@@ -503,10 +505,6 @@ REAL, DIMENSION(:,:), INTENT(IN) :: PFIELD
TYPE(FMHEADER), INTENT(IN) :: TPFMH
INTEGER, INTENT(OUT):: KRESP
-INTEGER(KIND=IDCDF_KIND),PARAMETER :: SHUFFLE = 0
-INTEGER(KIND=IDCDF_KIND),PARAMETER :: DEFLATE = 1
-INTEGER(KIND=IDCDF_KIND),PARAMETER :: DEFLATE_LEVEL = 2
-
INTEGER(KIND=IDCDF_KIND) :: STATUS
INTEGER(KIND=IDCDF_KIND) :: INCID
CHARACTER(LEN=30) :: YVARNAME
@@ -531,11 +529,6 @@ IF (STATUS /= NF90_NOERR) THEN
! Define the variable
STATUS = NF90_DEF_VAR(INCID, YVARNAME, NF90_DOUBLE, IVDIMS, IVARID)
IF (status /= NF90_NOERR) CALL HANDLE_ERR(status,__LINE__,'NCWRITX2[NF90_DEF_VAR]')
- IF (LDEFLATEX2) THEN
- ! Compress the variable with deflate level 2
- STATUS = NF90_DEF_VAR_DEFLATE(INCID, IVARID, SHUFFLE, DEFLATE, DEFLATE_LEVEL)
- IF (status /= NF90_NOERR) CALL HANDLE_ERR(status,__LINE__,'NCWRITX2[NF90_DEF_VAR_DEFLATE]')
- END IF
CALL WRITATTR(INCID, IVARID, TPFMH)
ELSE
PRINT *,'NCWRITX2 : ', TRIM(YVARNAME), ' already defined !'
@@ -590,8 +583,17 @@ IF (STATUS /= NF90_NOERR) THEN
CALL FILLVDIMS(PZCDF, INT(SHAPE(PFIELD),KIND=IDCDF_KIND), TPFIELD%CDIR, IVDIMS)
! Define the variable
- STATUS = NF90_DEF_VAR(INCID, YVARNAME, NF90_DOUBLE, IVDIMS, IVARID)
+ IF (TPFILE%LNCREDUCE_FLOAT_PRECISION) THEN
+ STATUS = NF90_DEF_VAR(INCID, YVARNAME, NF90_FLOAT, IVDIMS, IVARID)
+ ELSE
+ STATUS = NF90_DEF_VAR(INCID, YVARNAME, NF90_DOUBLE, IVDIMS, IVARID)
+ END IF
IF (status /= NF90_NOERR) CALL HANDLE_ERR(status,__LINE__,'IO_WRITE_FIELD_NC4_X2[NF90_DEF_VAR]')
+ ! Add compression if asked for
+ IF (TPFILE%LNCCOMPRESS) THEN
+ STATUS = NF90_DEF_VAR_DEFLATE(INCID, IVARID, SHUFFLE, DEFLATE, TPFILE%NNCCOMPRESS_LEVEL)
+ IF (status /= NF90_NOERR) CALL HANDLE_ERR(status,__LINE__,'IO_WRITE_FIELD_NC4_X2[NF90_DEF_VAR_DEFLATE]')
+ END IF
CALL IO_WRITE_FIELD_ATTR_NC4(TPFIELD,INCID,IVARID)
ELSE
PRINT *,'IO_WRITE_FIELD_NC4_X2: ', TRIM(YVARNAME), ' already defined !'
@@ -681,7 +683,16 @@ IF (STATUS /= NF90_NOERR) THEN
CALL FILLVDIMS(PZCDF, INT(SHAPE(PFIELD),KIND=IDCDF_KIND), TPFIELD%CDIR, IVDIMS)
! Define the variable
- STATUS = NF90_DEF_VAR(INCID, YVARNAME, NF90_DOUBLE, IVDIMS, IVARID)
+ IF (TPFILE%LNCREDUCE_FLOAT_PRECISION) THEN
+ STATUS = NF90_DEF_VAR(INCID, YVARNAME, NF90_FLOAT, IVDIMS, IVARID)
+ ELSE
+ STATUS = NF90_DEF_VAR(INCID, YVARNAME, NF90_DOUBLE, IVDIMS, IVARID)
+ END IF
+ ! Add compression if asked for
+ IF (TPFILE%LNCCOMPRESS) THEN
+ STATUS = NF90_DEF_VAR_DEFLATE(INCID, IVARID, SHUFFLE, DEFLATE, TPFILE%NNCCOMPRESS_LEVEL)
+ IF (status /= NF90_NOERR) CALL HANDLE_ERR(status,__LINE__,'IO_WRITE_FIELD_NC4_X3[NF90_DEF_VAR_DEFLATE]')
+ END IF
IF (status /= NF90_NOERR) CALL HANDLE_ERR(status,__LINE__,'IO_WRITE_FIELD_NC4_X3[NF90_DEF_VAR]')
CALL IO_WRITE_FIELD_ATTR_NC4(TPFIELD,INCID,IVARID)
ELSE
diff --git a/src/MNH/modd_fmout.f90 b/src/MNH/modd_fmout.f90
index d9811f75e..65be0daed 100644
--- a/src/MNH/modd_fmout.f90
+++ b/src/MNH/modd_fmout.f90
@@ -52,6 +52,10 @@ LOGICAL,SAVE :: LBAK_BEG = .FALSE. ! Force a backup/output at the first timestep
LOGICAL,SAVE :: LOUT_BEG = .FALSE. ! of the segment for all models
LOGICAL,SAVE :: LBAK_END = .FALSE. ! Force a backup/output at the last timestep
LOGICAL,SAVE :: LOUT_END = .FALSE. ! of the segment for all models
+LOGICAL,SAVE,DIMENSION(JPMODELMAX) :: LOUT_REDUCE_FLOAT_PRECISION = .FALSE.
+! Reduce the precision of floats to single precision instead of double precision (for netCDF)
+LOGICAL,SAVE,DIMENSION(JPMODELMAX) :: LOUT_COMPRESS = .FALSE. ! Compress (float) arrays (for netCDF)
+INTEGER,SAVE,DIMENSION(JPMODELMAX) :: NOUT_COMPRESS_LEVEL = 4 ! Compression level (for netCDF)
REAL,SAVE,ALLOCATABLE,DIMENSION(:,:) :: XBAK_TIME, XOUT_TIME
! XBAK_TIME(m,i) / XOUT_TIME(m,i) array of
! the increments in seconds from the beginning of the segment to the
diff --git a/src/MNH/modn_fmout.f90 b/src/MNH/modn_fmout.f90
index f67f995a9..388fb7057 100644
--- a/src/MNH/modn_fmout.f90
+++ b/src/MNH/modn_fmout.f90
@@ -53,6 +53,8 @@ NAMELIST/NAM_FMOUT/LBAK_BEG,LBAK_END,&
XOUT_TIME,NOUT_STEP,&
NOUT_STEP_FREQ,NOUT_STEP_FREQ_FIRST,&
XOUT_TIME_FREQ,XOUT_TIME_FREQ_FIRST, &
- COUT_VAR
+ COUT_VAR, &
+ LOUT_REDUCE_FLOAT_PRECISION, &
+ LOUT_COMPRESS, NOUT_COMPRESS_LEVEL
!
END MODULE MODN_FMOUT
diff --git a/src/Makefile b/src/Makefile
index 4cac9720e..c8103b99d 100644
--- a/src/Makefile
+++ b/src/Makefile
@@ -342,7 +342,7 @@ NETCDF_OPT = ${OPT_BASE_I4:-$OPT_BASE}
#
cdf : $(CDF_INC)
$(CDF_INC) :
- cd ${DIR_HDF} && ./configure --disable-shared --prefix=${CDF_PATH} --libdir=${CDF_PATH}/lib64 --with-zlib=no \
+ cd ${DIR_HDF} && ./configure --disable-shared --prefix=${CDF_PATH} --libdir=${CDF_PATH}/lib64 \
FC="$(FC)" FCFLAGS="$(HDF_OPT)" CPPFLAGS="-DpgiFortran" ${HDF_CONF} && make && make install && $(MAKE) -j 1 clean
cd ${DIR_CDF} && ./configure --disable-shared --prefix=${CDF_PATH} --libdir=${CDF_PATH}/lib64 --disable-cxx --enable-f90 --disable-dap \
FC="$(FC)" FCFLAGS="$(NETCDF_OPT)" CPPFLAGS="-DpgiFortran ${INC_NETCDF}" ${CDF_CONF} LDFLAGS=" -L${CDF_PATH}/lib64" && make && make install && $(MAKE) -j 1 clean
diff --git a/src/Makefile.MESONH.mk b/src/Makefile.MESONH.mk
index dbff3238b..8cbe5bb78 100644
--- a/src/Makefile.MESONH.mk
+++ b/src/Makefile.MESONH.mk
@@ -433,7 +433,7 @@ CDF_PATH?=${DIR_CDF}-${ARCH}I${MNH_INT}
CDF_INC?=${CDF_PATH}/include/netcdf.inc
#
INC_NETCDF ?= -I${CDF_PATH}/include
-LIB_NETCDF ?= -L${CDF_PATH}/lib -L${CDF_PATH}/lib64 -lnetcdff -lnetcdf -lhdf5_hl -lhdf5
+LIB_NETCDF ?= -L${CDF_PATH}/lib -L${CDF_PATH}/lib64 -lnetcdff -lnetcdf -lhdf5_hl -lhdf5 -lz
INC += $(INC_NETCDF)
LIBS += $(LIB_NETCDF)
#
--
GitLab