Newer
Older
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
RODIER Quentin
committed
MNH_LIC Copyright 1994-2021 CNRS, Meteo-France and Universite Paul Sabatier
MNH_LIC This is part of the Meso-NH software governed by the CeCILL-C licence
MNH_LIC version 1. See LICENSE, CeCILL-C_V1-en.txt and CeCILL-C_V1-fr.txt
MNH_LIC for details. version 1.
RODIER Quentin
committed
@author: 07/2021 Quentin Rodier
"""
import netCDF4 as nc
import numpy as np
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
def read_withEPY(LnameFiles,Dvar_input, Dvar_output={}, path='.'):
import epygram
epygram.init_env()
for i,keyFiles in enumerate(Dvar_input.keys()):
print('Reading file ' + keyFiles)
theFile = epygram.formats.resource(LnameFiles[i],'r')
Dvar_output[keyFiles] = {} #initialize dic for each files
for var in Dvar_input[keyFiles]: #For each files
# Read variables
if(theFile.format == 'FA'):
Dvar_output[keyFiles][var] = theFile.readfield(var)
elif(theFile.format == 'LFI'):
if(var[1]==None or var[1]==0): # 2D Field
Dvar_output[keyFiles][var[0]] = theFile.readfield(var)
else: # 3D Field
Dvar_output[keyFiles][var[0]+str(var[1])] = theFile.readfield(var).getlevel(k=var[1])
elif(theFile.format == 'netCDFMNH'):
if(var[1]==None or var[1]==0): # 2D Field
Dvar_output[keyFiles][var[0]] = theFile.readfield(var[0])
else:
Dvar_output[keyFiles][var[0]+str(var[1])] = theFile.readfield(var[0]).getlevel(k=var[1])
elif(theFile.format == 'GRIB'):
if len(var)==6: # GRIB2
Dvar_output[keyFiles][var[5]] = theFile.readfield({'discipline': var[0], 'parameterCategory': var[1], 'typeOfFirstFixedSurface': var[2],'parameterNumber': var[3], 'level': var[4]})
elif len(var)==5: # GRIB1
Dvar_output[keyFiles][var[4]] = theFile.readfield({'indicatorOfParameter': var[0], 'paramId': var[1], 'indicatorOfTypeOfLevel': var[2], 'level': var[3]})
else: epygramError("GRIB format error. GRIB1 expects 4 values : [indicatorOfParameter, paramId, indicatorOfTypeOfLevel, level, 'casual name'], GRIB2 expects 5 values [discipline, parameterCategory, typeOfFirstFixedSurface, parameterNumber, level, casual name]")
else:
raise epygramError("Unknown format file, please use FA, LFI, GRIB or MNH NetCDF")
theFile.close()
# Transform spectral data to physics space (for AROME and ARPEGE)
for f in Dvar_output:
for var in Dvar_output[f]:
try:
if(Dvar_output[f][var].spectral):
Dvar_output[f][var].sp2gp()
except:
break
return Dvar_output
RODIER Quentin
committed
def read_netcdf(LnameFiles, Dvar_input, path='.', get_data_only=True, del_empty_dim=True, removeHALO=True):
"""Read a netCDF4 Meso-NH file
For each file, call functions to read diachronic or synchronous file
Parameters
----------
LnameFiles : list of str
list of Meso-NH netCDF4 file (diachronic or synchronous)
Dvar_input : Dict{'keyFile' : 'var_name',('group_name','var_name')}
RODIER Quentin
committed
where
'keyFile' is a str corresponding to a key for the file number in LnameFiles (by order)
RODIER Quentin
committed
'var_name' is the exact str of the netCDF4 variable name
('group_name','var_name') is the exact tuple of the (sub-)groups name and the netCDF4 variable name
e.g. : {'f1':['ZS', 'WT','ni', 'level'],
'f2':[('/LES_budgets/Cartesian/Not_time_averaged/Not_normalized/cart/',MEAN_TH'),('/Budgets/RI','AVEF')]
}
path : str
unique path of the files
get_data_only : bool, default: True
if True, the function returns Dvar as masked_array (only data)
if False, the function returns Dvar as netCDF4._netCDF4.Variable
del_empty_dim : bool, default: True
if get_data_only=True and del_empty_dim=True, returns Dvar as an array without dimensions with size 1 and 0
e.g. : an array of dimensions (time_budget, cart_level, cart_nj, cart_ni) with shape (180,1,50,1) is returned (180,50)
removeHALO : bool, default: True
if True, remove first and last (NHALO=1) point [1:-1] if get_data_only=True on each
level, level_w, ni, ni_u, ni_v, nj, nj_u, nj_v dimensions
Returns
-------
Dvar : Dict
Dvar[ifile]['var_name'] if the group contains only one variable
Dvar[ifile][('group_name','var_name')] if the group contains more than one variable
"""
Dvar = {}
for i,keyFiles in enumerate(Dvar_input.keys()):
print('Reading file ' + keyFiles)
print(path + LnameFiles[i])
theFile = nc.Dataset(path + LnameFiles[i],'r')
Dvar[keyFiles] = {}
if '000' in LnameFiles[i][-6:-3]:
RODIER Quentin
committed
if theFile['MASDEV'][0] <= 54:
raise TypeError('The python lib is available for MNH >= 5.5')
else:
Dvar[keyFiles] = read_TIMESfiles_55(theFile, Dvar_input[keyFiles], Dvar[keyFiles], get_data_only, del_empty_dim, removeHALO)
RODIER Quentin
committed
else:
Dvar[keyFiles]= read_BACKUPfile(theFile, Dvar_input[keyFiles], Dvar[keyFiles], get_data_only, del_empty_dim, removeHALO)
#theFile.close()
RODIER Quentin
committed
return Dvar
def read_var(theFile, Dvar, var_name, get_data_only, del_empty_dim=True, removeHALO=True):
RODIER Quentin
committed
"""Read a netCDF4 variable
Parameters
----------
theFile : netCDF4._netCDF4.Dataset
a Meso-NH diachronic netCDF4 file
var_name : str
a Meso-NH netCDF4 variable name
get_data_only : bool, default: True
if True, the function returns Dvar as masked_array (only data)
if False, the function returns Dvar as netCDF4._netCDF4.Variable
RODIER Quentin
committed
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
del_empty_dim : bool, default: True
if get_data_only=True and del_empty_dim=True, returns Dvar as an array without dimensions with size 1 and 0
e.g. : an array of dimensions (time_budget, cart_level, cart_nj, cart_ni) with shape (180,1,50,1) is returned (180,50)
removeHALO : bool, default: True
if True, remove first and last (NHALO=1) point [1:-1] if get_data_only=True on each
level, level_w, ni, ni_u, ni_v, nj, nj_u, nj_v dimensions
Returns
-------
Dvar : Dict
Dvar['var_name'] if the group contains only one variable
Dvar[('group_name','var_name')] if the group contains more than one variable
"""
try:
var_dim = theFile.variables[var_name].ndim
var_dim_name = theFile.variables[var_name].dimensions
except:
raise KeyError("Group and variable name not found in the file, check the group/variable name with ncdump -h YourMNHFile.000.nc. You asked for variable : " + var_name)
if not get_data_only:
Dvar[var_name] = theFile.variables[var_name]
else:
if var_dim == 0:
Dvar[var_name] = theFile.variables[var_name][0].data
elif var_dim == 1:
Dvar[var_name] = theFile.variables[var_name][:]
elif var_dim == 2:
Dvar[var_name] = theFile.variables[var_name][:,:]
elif var_dim == 3:
Dvar[var_name] = theFile.variables[var_name][:,:,:]
elif var_dim == 4:
Dvar[var_name] = theFile.variables[var_name][:,:,:,:]
elif var_dim == 5:
Dvar[var_name] = theFile.variables[var_name][:,:,:,:,:]
elif var_dim == 6:
Dvar[var_name] = theFile.variables[var_name][:,:,:,:,:,:]
elif var_dim == 7:
Dvar[var_name] = theFile.variables[var_name][:,:,:,:,:,:,:]
if removeHALO:
RODIER Quentin
committed
for i in range(8):
try:
if var_dim_name[i]=='level' or var_dim_name[i]=='level_w' or \
var_dim_name[i]=='ni' or var_dim_name[i]=='ni_u' or var_dim_name[i]=='ni_v' or \
var_dim_name[i]=='nj' or var_dim_name[i]=='nj_u' or var_dim_name[i]=='nj_v':
if var_dim != 0:
Dvar[var_name] = removetheHALO(i+1, Dvar[var_name])
except:
break
if del_empty_dim:
Ldimtosqueeze=[]
var_shape = theFile.variables[var_name].shape
for i in range(8):
try:
if var_shape[i]==1: Ldimtosqueeze.append(i)
except IndexError:
break
Ldimtosqueeze=tuple(Ldimtosqueeze)
Dvar[var_name] = np.squeeze(Dvar[var_name], axis=Ldimtosqueeze)
RODIER Quentin
committed
return Dvar
def read_from_group(theFile, Dvar, group_name, var_name, get_data_only, del_empty_dim=True,removeHALO=True):
RODIER Quentin
committed
"""Read a variable from a netCDF4 group
RODIER Quentin
committed
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
Parameters
----------
theFile : netCDF4._netCDF4.Dataset
a Meso-NH diachronic netCDF4 file
group_name : str
a Meso-NH netCDF4 group name
var_name : str
a Meso-NH netCDF4 variable name
get_data_only : bool, default: True
if True, the function returns Dvar as masked_array (only data)
if False, the function returns Dvar as netCDF4._netCDF4.Variable
del_empty_dim : bool, default: True
if get_data_only=True and del_empty_dim=True, returns Dvar as an array without dimensions with size 1 and 0
e.g. : an array of dimensions (time_budget, cart_level, cart_nj, cart_ni) with shape (180,1,50,1) is returned (180,50)
removeHALO : bool, default: True
if True, remove first and last (NHALO=1) point [1:-1] if get_data_only=True on each
level, level_w, ni, ni_u, ni_v, nj, nj_u, nj_v dimensions
Returns
-------
Dvar : Dict
Dvar['var_name'] if the group contains only one variable
Dvar[('group_name','var_name')] if the group contains more than one variable
"""
try:
var_dim = theFile[group_name].variables[var_name].ndim
var_dim_name = theFile[group_name].variables[var_name].dimensions
RODIER Quentin
committed
raise KeyError("Group and variable name not found in the file, check the group/variable name with ncdump -h YourMNHFile.000.nc. You asked for group/variable : " + group_name + var_name)
RODIER Quentin
committed
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
if not get_data_only:
Dvar[(group_name,var_name)] = theFile[group_name].variables[var_name]
else:
if var_dim == 0:
Dvar[(group_name,var_name)] = theFile[group_name].variables[var_name][0].data
if var_dim == 1:
Dvar[(group_name,var_name)] = theFile[group_name].variables[var_name][:]
elif var_dim == 2:
Dvar[(group_name,var_name)] = theFile[group_name].variables[var_name][:,:]
elif var_dim == 3:
Dvar[(group_name,var_name)] = theFile[group_name].variables[var_name][:,:,:]
elif var_dim == 4:
Dvar[(group_name,var_name)] = theFile[group_name].variables[var_name][:,:,:,:]
elif var_dim == 5:
Dvar[(group_name,var_name)] = theFile[group_name].variables[var_name][:,:,:,:,:]
elif var_dim == 6:
Dvar[(group_name,var_name)] = theFile[group_name].variables[var_name][:,:,:,:,:,:]
elif var_dim == 7:
Dvar[(group_name,var_name)] = theFile[group_name].variables[var_name][:,:,:,:,:,:,:]
if removeHALO:
for i in range(8):
try:
if var_dim_name[i]=='level' or var_dim_name[i]=='level_w' or \
var_dim_name[i]=='ni' or var_dim_name[i]=='ni_u' or var_dim_name[i]=='ni_v' or \
var_dim_name[i]=='nj' or var_dim_name[i]=='nj_u' or var_dim_name[i]=='nj_v':
if var_dim != 0:
RODIER Quentin
committed
Dvar[(group_name,var_name)] = removetheHALO(i+1, Dvar[(group_name,var_name)])
RODIER Quentin
committed
except:
break
if del_empty_dim:
Ldimtosqueeze=[]
var_shape = Dvar[(group_name,var_name)].shape
for i in range(8):
try:
if var_shape[i]==1: Ldimtosqueeze.append(i)
except IndexError:
break
Ldimtosqueeze=tuple(Ldimtosqueeze)
Dvar[(group_name,var_name)] = np.squeeze(Dvar[(group_name,var_name)], axis=Ldimtosqueeze)
RODIER Quentin
committed
# LES budget, ZTSERIES needs to be transposed to use psection functions without specifying .T each time
RODIER Quentin
committed
if 'LES_budget' in group_name or 'ZTSERIES' in group_name or 'XTSERIES' in group_name:
RODIER Quentin
committed
Dvar[(group_name,var_name)] = Dvar[(group_name,var_name)].T
return Dvar
def read_BACKUPfile(theFile, Dvar_input, Dvar, get_data_only, del_empty_dim=True, removeHALO=True):
RODIER Quentin
committed
"""Read variables from Meso-NH MASDEV >= 5.5.0 synchronous file
For all variables in Dvar_input of one file, call functions to read the variable of the group+variable
Parameters
----------
theFile : netCDF4._netCDF4.Dataset
a Meso-NH diachronic netCDF4 file
RODIER Quentin
committed
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
Dvar_input : Dict{'var_name',('group_name','var_name')}
with
'var_name' is the exact str of the netCDF4 variable name
('group_name','var_name') is the exact tuple of the (sub-)groups name and the netCDF4 variable name
e.g. : {'f1':['ZS', 'WT','ni', 'level'],
'f2':[('/LES_budgets/Cartesian/Not_time_averaged/Not_normalized/cart/',MEAN_TH'),('/Budgets/RI','AVEF')]
}
get_data_only: bool, default: True
if True, the function returns Dvar as masked_array (only data)
if False, the function returns Dvar as netCDF4._netCDF4.Variable
del_empty_dim: bool, default: True
if get_data_only=True and del_empty_dim=True, returns Dvar as masked_array without dimensions with size 1 and 0
e.g. : an array of dimensions (time_budget, cart_level, cart_nj, cart_ni) with shape (180,1,50,1) is returned (180,50)
Returns
-------
Dvar : Dict
Dvar['var_name'] if the group contains only one variable
Dvar[('group_name','var_name')] if the group contains more than one variable
"""
# Reading date since beginning of the model run
Dvar['time'] = theFile.variables['time'][0]
Dvar['date'] = nc.num2date(Dvar['time'],units=theFile.variables['time'].units, calendar = theFile.variables['time'].calendar)
for var in Dvar_input:
if type(var) == tuple:
Dvar = read_from_group(theFile, Dvar, var[0], var[1], get_data_only, del_empty_dim, removeHALO)
else:
Dvar = read_var(theFile, Dvar, var, get_data_only, del_empty_dim, removeHALO)
# For all variables except scalars, change Fill_Value to NaN
if get_data_only:
Dvar[var]= np.where(Dvar[var] != -99999.0, Dvar[var], np.nan)
Dvar[var]= np.where(Dvar[var] != 999.0, Dvar[var], np.nan)
RODIER Quentin
committed
return Dvar
def read_TIMESfiles_55(theFile, Dvar_input, Dvar, get_data_only, del_empty_dim=True, removeHALO=True):
RODIER Quentin
committed
"""Read variables from Meso-NH MASDEV >= 5.5.0 diachronic file
For all variables in Dvar_input of one file, call functions to read the variable of the group+variable
RODIER Quentin
committed
RODIER Quentin
committed
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
Parameters
----------
theFile : netCDF4._netCDF4.Dataset
a Meso-NH diachronic netCDF4 file
Dvar_input : Dict{'var_name',('group_name','var_name')}
with
'var_name' is the exact str of the netCDF4 variable name
('group_name','var_name') is the exact tuple of the (sub-)groups name and the netCDF4 variable name
e.g. : {'f1':['ZS', 'WT','ni', 'level'],
'f2':[('/LES_budgets/Cartesian/Not_time_averaged/Not_normalized/cart/',MEAN_TH'),('/Budgets/RI','AVEF')]
}
get_data_only: bool, default: True
if True, the function returns Dvar as masked_array (only data)
if False, the function returns Dvar as netCDF4._netCDF4.Variable
del_empty_dim: bool, default: True
if get_data_only=True and del_empty_dim=True, returns Dvar as masked_array without dimensions with size 1 and 0
e.g. : an array of dimensions (time_budget, cart_level, cart_nj, cart_ni) with shape (180,1,50,1) is returned (180,50)
Returns
-------
Dvar : Dict
Dvar[ifile]['var_name'] if the group contains only one variable
Dvar[ifile][('group_name','var_name')] if the group contains more than one variable
"""
for var in Dvar_input:
RODIER Quentin
committed
if type(var) == tuple:
RODIER Quentin
committed
Dvar = read_from_group(theFile, Dvar, var[0], var[1], get_data_only, del_empty_dim, removeHALO)
RODIER Quentin
committed
else:
RODIER Quentin
committed
Dvar = read_var(theFile, Dvar, var, get_data_only, del_empty_dim, removeHALO)
return Dvar
RODIER Quentin
committed
def removetheHALO(idim, var):
"""Remove a NHALO=1 point [1:-1] at a given dimension idim of a variable var
TODO: with NHALO=3 with WENO5
TODO: with the use of get_data_only=False (NetCDF4 output)
RODIER Quentin
committed
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
Parameters
----------
idim: int
the dimension over which remove the first and last point
var: array
a Meso-NH netCDF4 variable name
Returns
-------
var : array
"""
if idim == 1:
var = var[1:-1]
elif idim == 2:
var = var[:,1:-1]
elif idim == 3:
var = var[:,:,1:-1]
elif idim == 4:
var = var[:,:,:,1:-1]
elif idim == 5:
var = var[:,:,:,:,1:-1]
elif idim == 6:
var = var[:,:,:,:,:,1:-1]
elif idim == 7:
var = var[:,:,:,:,:,:,1:-1]