MPAS-JEDI
config.py
Go to the documentation of this file.
1 #!/usr/bin/env python3
2 
3 import binning_utils as bu
4 import predefined_configs as pconf
5 from collections import defaultdict
6 from copy import deepcopy
7 import os
8 import var_utils as vu
9 
10 #==============================================================
11 # Sub-selections of pconf.binVarConfigs for specific DiagSpaces
12 #==============================================================
13 
14 #################################################################
15 ## Generic binVarConfigs that apply to all observation categories
16 #################################################################
17 obsBinVars = defaultdict(list)
18 obsBinVars[vu.obsVarQC] += [bu.goodQCMethod, bu.badQCMethod, bu.allQCMethod]
19 obsBinVars[vu.obsVarLat] += [bu.identityBinMethod, bu.latbandsMethod]
20 obsBinVars[vu.obsVarLT] += [bu.identityBinMethod]
21 obsBinVars[vu.obsVarNormErr] += [bu.identityBinMethod]
22 obsBinVars[vu.obsRegionBinVar] += ['CONUS']
23 obsBinVars[vu.obsRegionBinVar] += [bu.geoirlatlonboxMethod]
24 
25 
26 ################################
27 ## binVarConfigs for surface obs
28 ################################
29 surfBinVars = deepcopy(obsBinVars)
30 
31 
32 ##########################################################
33 ## binVarConfigs for profile obs w/ pressure vertical bins
34 ##########################################################
35 profPressBinVars = deepcopy(obsBinVars)
36 profPressBinVars[vu.obsVarPrs] += [bu.identityBinMethod, bu.PjetMethod]
37 profPressBinVars[vu.obsVarLat] += [bu.PjetMethod]
38 
39 # 2D pressure bins with named latitude-band methods
40 for latBand in pconf.namedLatBands['values']:
41  profPressBinVars[vu.obsVarPrs] += [latBand]
42 
43 
44 ##########################################################
45 ## binVarConfigs for profile obs w/ altitude vertical bins
46 ##########################################################
47 profAltBinVars = deepcopy(obsBinVars)
48 profAltBinVars[vu.obsVarAlt] += [bu.identityBinMethod, bu.altjetMethod]
49 profAltBinVars[vu.obsVarLat] += [bu.altjetMethod]
50 
51 # 2D altitude bins with named latitude-band methods
52 for latBand in pconf.namedLatBands['values']:
53  profAltBinVars[vu.obsVarAlt] += [latBand]
54 
55 
56 #################################
57 ## binVarConfigs for radiance obs
58 #################################
59 radianceBinVars = deepcopy(obsBinVars)
60 radianceBinVars[vu.obsVarGlint] += [bu.identityBinMethod]
61 radianceBinVars[vu.obsVarSenZen] += [bu.identityBinMethod]
62 
63 
64 # binBYLandFrac controls whether to bin by land-ocean fractions and categories
65 # NOTE: requires geovals to be written for applicable DiagSpaces
66 binBYLandFrac = False
67 if binBYLandFrac:
68  radianceBinVars[vu.obsVarLandFrac] += [bu.identityBinMethod,
69  bu.surfbandsMethod]
70 
71 
72 ############################################
73 ## binVarConfigs for polar-orbitting MW obs,
74 ## including cloud metrics
75 ## e.g., AMSUA
76 ############################################
77 polmwBinVars = deepcopy(radianceBinVars)
78 polmwBinVars[vu.obsVarACI] += [bu.identityBinMethod]
79 polmwBinVars[vu.obsVarSCI] += [bu.OkamotoMethod]
80 
81 
82 ##########################################
83 ## binVarConfigs for geostationary IR obs
84 ## including cloud metrics
85 ## e.f., GOES-ABI, Himawari-AHI
86 ##########################################
87 geoirBinVars = deepcopy(radianceBinVars)
88 geoirBinVars[vu.obsVarACI] += [bu.identityBinMethod]
89 geoirBinVars[vu.obsVarCldFrac] += [bu.identityBinMethod,
90  bu.cloudbandsMethod]
91 
92 # Binning variables with clr-/cld-sky methods
93 for var in pconf.cldfracBinVars.keys():
94  if var != vu.obsVarLandFrac or binBYLandFrac:
95  geoirBinVars[var] += [bu.clrskyMethod]
96  geoirBinVars[var] += [bu.cldskyMethod]
97 
98 # Binning variables with land category methods
99 if binBYLandFrac:
100  for var in pconf.landfracBinVars.keys():
101  for surfBand in pconf.namedLandFracBands['values']:
102  geoirBinVars[var] += [surfBand]
103 
104 # symmetric cloud impact (expensive)
105 geoirSCIMethods = [
106  bu.OkamotoMethod,
107  bu.ScaleOkamotoMethod,
108 # bu.ModHarnischMethod,
109 # bu.ScaleModHarnischMethod,
110 ]
111 
112 for method in geoirSCIMethods:
113  geoirBinVars[vu.obsVarSCI] += [method]
114 # uncomment to diagnose (HofX - ObsValue) / ObsError for SCI-parameterized ObsError
115 # note: requires SCIErrParams to be defined for all all DiagSpaces that use it
116 # geoirBinVars[vu.obsVarNormErr] += [method]
117 
118 
119 #########################################
120 # binVarConfigs for model space variables
121 #########################################
122 modelBinVars = defaultdict(list)
123 modelBinVars[vu.noBinVar] += [bu.noBinMethod]
124 modelBinVars[vu.modVarLat] += [bu.identityBinMethod]
125 modelBinVars[vu.modVarLat] += [bu.latbandsMethod]
126 modelBinVars[vu.modVarLev] += [bu.identityBinMethod]
127 # 2D model level bins with named latitude-band methods
128 for latBand in pconf.namedLatBands['values']:
129  modelBinVars[vu.modVarLev] += [latBand]
130 modelBinVars[vu.modelRegionBinVar] += [bu.geoirlatlonboxMethod]
131 
132 #=======================
133 # DiagSpace definitions
134 # e.g. IODA ObsSpace
135 # MPAS ModelSpace
136 #=======================
137 nullBinVars = {vu.miss_s: []}
138 
139 nullDiagSpaceInfo = {
140  'DiagSpaceGrp': vu.miss_s,
141  'process': False,
142  'binVarConfigs': nullBinVars,
143  'diagNames': [],
144 }
145 
146 profile_s = 'profile'
147 sfc_s = 'surface'
148 radiance_s = 'radiance'
149 model_s = 'model'
150 
151 ## analysis groups for configuring AnalyzeStats + Analyses
152 convGrp = 'conv'
153 abiGrp = 'abi'
154 ahiGrp = 'ahi'
155 amsuaGrp = 'amsua'
156 amsuacldGrp = 'amsuacld'
157 mhsGrp = 'mhs'
158 modelGrp = 'model'
159 
160 anGroupConfig = {
161  convGrp: {'npwork': 36, 'npread': 36, 'analyze_walltime': '00:10:00'},
162  abiGrp: {'npwork': 12, 'npread': 36, 'analyze_walltime': '01:00:00'},
163  ahiGrp: {'npwork': 12, 'npread': 36, 'analyze_walltime': '01:00:00'},
164  amsuaGrp: {'npwork': 36, 'npread': 36, 'analyze_walltime': '00:12:00'},
165  amsuacldGrp: {'npwork': 36, 'npread': 36, 'analyze_walltime': '00:12:00'},
166  mhsGrp: {'npwork': 36, 'npread': 36, 'analyze_walltime': '00:08:00'},
167  modelGrp: {'npwork': 10, 'npread': 36, 'analyze_walltime': '03:30:00'},
168 }
169 
170 # Each entry of DiagSpaceConfig is a key-value pair, with the following possible values:
171 #DiagSpaceName (fromYAML):{
172 # 'DiagSpaceGrp': any of profile_s, radiance_s, model_s
173 # 'process': True or False
174 # 'anGrp': used to configure statistical analysis jobs and scripts
175 # 'binVarConfigs': binning variable configurations used when calculating statistics on diagnostics
176 # 'diagNames': list of selected diagnostic names,
177 # e.g., pconf.diffDiagnostics[+pconf.absDiagnostics][+pconf.cloudyRadDiagnostics]
178 # new diagNames should be defined in diag_utils
179 # 'channels': channel selection for plot_obs_nc_loc
180 #}
181 
182 DiagSpaceConfig = {
183 #conventional
184  'aircraft': {
185  'DiagSpaceGrp': profile_s,
186  'process': True,
187  'anGrp': convGrp,
188  'binVarConfigs': profPressBinVars,
189  'diagNames': pconf.defaultDiagnostics+pconf.sigmaDiagnostics,
190  },
191  'gnssro': {
192  'DiagSpaceGrp': profile_s,
193  'process': True,
194  'anGrp': convGrp,
195  'binVarConfigs': profAltBinVars,
196  'diagNames': pconf.relativeDiagnostics+pconf.sigmaDiagnostics,
197  },
198  'gnssrobndropp1d': {
199  'DiagSpaceGrp': profile_s,
200  'process': True,
201  'anGrp': convGrp,
202  'binVarConfigs': profAltBinVars,
203  'diagNames': pconf.relativeDiagnostics+pconf.sigmaDiagnostics,
204  },
205  'gnssroref': {
206  'DiagSpaceGrp': profile_s,
207  'process': True,
208  'anGrp': convGrp,
209  'binVarConfigs': profAltBinVars,
210  'diagNames': pconf.relativeDiagnostics+pconf.sigmaDiagnostics,
211  },
212  'satwind': {
213  'DiagSpaceGrp': profile_s,
214  'process': True,
215  'anGrp': convGrp,
216  'binVarConfigs': profPressBinVars,
217  'diagNames': pconf.defaultDiagnostics+pconf.sigmaDiagnostics,
218  },
219  'sfc': {
220  'DiagSpaceGrp': sfc_s,
221  'process': True,
222  'anGrp': convGrp,
223  'binVarConfigs': surfBinVars,
224  'diagNames': pconf.defaultDiagnostics+pconf.sigmaDiagnostics,
225  },
226  'sondes': {
227  'DiagSpaceGrp': profile_s,
228  'process': True,
229  'anGrp': convGrp,
230  'binVarConfigs': profPressBinVars,
231  'diagNames': pconf.defaultDiagnostics+pconf.sigmaDiagnostics,
232  },
233 #radiances
234  'abi_g16': {
235  'DiagSpaceGrp': radiance_s,
236  'process': True,
237  'anGrp': abiGrp,
238  'binVarConfigs': geoirBinVars,
239  'diagNames': pconf.defaultDiagnostics+pconf.sigmaDiagnostics,
240  'channels': [8,9,10,11,13,14,15,16],
241  },
242  'ahi_himawari8': {
243  'DiagSpaceGrp': radiance_s,
244  'process': True,
245  'anGrp': ahiGrp,
246  'binVarConfigs': geoirBinVars,
247  'diagNames': pconf.defaultDiagnostics+pconf.sigmaDiagnostics,
248  'channels': [8,9,10,11,13,14,15,16],
249  },
250  'abi-clr_g16': {
251  'DiagSpaceGrp': radiance_s,
252  'process': True,
253  'anGrp': abiGrp,
254  'binVarConfigs': geoirBinVars,
255  'diagNames': pconf.defaultDiagnostics,
256  'channels': [8,9,10,11,13,14,15,16],
257  },
258  'ahi-clr_himawari8': {
259  'DiagSpaceGrp': radiance_s,
260  'process': True,
261  'anGrp': ahiGrp,
262  'binVarConfigs': geoirBinVars,
263  'diagNames': pconf.defaultDiagnostics,
264  'channels': [8,9,10,11,13,14,15,16],
265  },
266  'airs_aqua': {
267  'DiagSpaceGrp': radiance_s,
268  'process': False,
269  'binVarConfigs': radianceBinVars,
270  'diagNames': pconf.defaultDiagnostics,
271  'channels': [1,6,7],
272  },
273  'amsua_aqua': {
274  'DiagSpaceGrp': radiance_s,
275  'process': True,
276  'anGrp': amsuaGrp,
277  'binVarConfigs': radianceBinVars,
278  'diagNames': pconf.defaultDiagnostics+pconf.sigmaDiagnostics,
279  'channels': [8,9],
280  },
281  'amsua_metop-a': {
282  'DiagSpaceGrp': radiance_s,
283  'process': True,
284  'anGrp': amsuaGrp,
285  'binVarConfigs': radianceBinVars,
286  'diagNames': pconf.defaultDiagnostics+pconf.sigmaDiagnostics,
287  'channels': [5,6,9],
288  },
289  'amsua_metop-b': {
290  'DiagSpaceGrp': radiance_s,
291  'process': True,
292  'anGrp': amsuaGrp,
293  'binVarConfigs': radianceBinVars,
294  'diagNames': pconf.defaultDiagnostics+pconf.sigmaDiagnostics,
295  'channels': [],
296  },
297  'amsua_n15': {
298  'DiagSpaceGrp': radiance_s,
299  'process': True,
300  'anGrp': amsuaGrp,
301  'binVarConfigs': radianceBinVars,
302  'diagNames': pconf.defaultDiagnostics+pconf.sigmaDiagnostics,
303  'channels': [5,6,7,8,9],
304  },
305  'amsua_n18': {
306  'DiagSpaceGrp': radiance_s,
307  'process': True,
308  'anGrp': amsuaGrp,
309  'binVarConfigs': radianceBinVars,
310  'diagNames': pconf.defaultDiagnostics+pconf.sigmaDiagnostics,
311  'channels': [5,6,7,8,9],
312  },
313  'amsua_n19': {
314  'DiagSpaceGrp': radiance_s,
315  'process': True,
316  'anGrp': amsuaGrp,
317  'binVarConfigs': radianceBinVars,
318  'diagNames': pconf.defaultDiagnostics+pconf.sigmaDiagnostics,
319  'channels': [5,6,7,9],
320  },
321  'amsua_n19--hydro': {
322  'DiagSpaceGrp': radiance_s,
323  'process': False,
324  'binVarConfigs': radianceBinVars,
325  'diagNames': pconf.defaultDiagnostics,
326  'channels': [1,2,3,15],
327  },
328  'amsua_n19--nohydro': {
329  'DiagSpaceGrp': radiance_s,
330  'process': False,
331  'binVarConfigs': radianceBinVars,
332  'diagNames': pconf.defaultDiagnostics,
333  'channels': [4,5,6,7,9,10,11,12,13,14],
334  },
335  'amsua-cld_aqua': {
336  'DiagSpaceGrp': radiance_s,
337  'process': True,
338  'anGrp': amsuacldGrp,
339  'binVarConfigs': polmwBinVars,
340  'diagNames': pconf.defaultDiagnostics+pconf.sigmaDiagnostics,
341  'channels': [1,2,3,4,15],
342  },
343  'amsua-cld_metop-a': {
344  'DiagSpaceGrp': radiance_s,
345  'process': True,
346  'anGrp': amsuacldGrp,
347  'binVarConfigs': polmwBinVars,
348  'diagNames': pconf.defaultDiagnostics+pconf.sigmaDiagnostics,
349  'channels': [1,2,3,4,15],
350  },
351  'amsua-cld_metop-b': {
352  'DiagSpaceGrp': radiance_s,
353  'process': True,
354  'anGrp': amsuacldGrp,
355  'binVarConfigs': polmwBinVars,
356  'diagNames': pconf.defaultDiagnostics+pconf.sigmaDiagnostics,
357  'channels': [1,2,3,4,15],
358  },
359  'amsua-cld_n15': {
360  'DiagSpaceGrp': radiance_s,
361  'process': True,
362  'anGrp': amsuacldGrp,
363  'binVarConfigs': polmwBinVars,
364  'diagNames': pconf.defaultDiagnostics+pconf.sigmaDiagnostics,
365  'channels': [1,2,3,4,15],
366  },
367  'amsua-cld_n18': {
368  'DiagSpaceGrp': radiance_s,
369  'process': True,
370  'anGrp': amsuacldGrp,
371  'binVarConfigs': polmwBinVars,
372  'diagNames': pconf.defaultDiagnostics+pconf.sigmaDiagnostics,
373  'channels': [1,2,3,4,15],
374  },
375  'amsua-cld_n19': {
376  'DiagSpaceGrp': radiance_s,
377  'process': True,
378  'anGrp': amsuacldGrp,
379  'binVarConfigs': polmwBinVars,
380  'diagNames': pconf.defaultDiagnostics+pconf.sigmaDiagnostics,
381  'channels': [1,2,3,4,15],
382  },
383  'cris-fsr_npp': {
384  'DiagSpaceGrp': radiance_s,
385  'process': False,
386  'binVarConfigs': radianceBinVars,
387  'diagNames': pconf.defaultDiagnostics,
388  'channels': [24,26,28,32,37,39],
389  },
390  'hirs4_metop-a': {
391  'DiagSpaceGrp': radiance_s,
392  'process': False,
393  'binVarConfigs': radianceBinVars,
394  'diagNames': pconf.defaultDiagnostics,
395  'channels': range(1,16),
396  },
397  'iasi_metop-a': {
398  'DiagSpaceGrp': radiance_s,
399  'process': False,
400  'binVarConfigs': radianceBinVars,
401  'diagNames': pconf.defaultDiagnostics,
402  'channels': [16,29,32,35,38,41,44],
403  },
404  'mhs_metop-a': {
405  'DiagSpaceGrp': radiance_s,
406  'process': True,
407  'anGrp': mhsGrp,
408  'binVarConfigs': radianceBinVars,
409  'diagNames': pconf.defaultDiagnostics+pconf.sigmaDiagnostics,
410  'channels': range(1,6),
411  },
412  'mhs_metop-b': {
413  'DiagSpaceGrp': radiance_s,
414  'process': True,
415  'anGrp': mhsGrp,
416  'binVarConfigs': radianceBinVars,
417  'diagNames': pconf.defaultDiagnostics+pconf.sigmaDiagnostics,
418  'channels': range(1,6),
419  },
420  'mhs_n18': {
421  'DiagSpaceGrp': radiance_s,
422  'process': True,
423  'anGrp': mhsGrp,
424  'binVarConfigs': radianceBinVars,
425  'diagNames': pconf.defaultDiagnostics+pconf.sigmaDiagnostics,
426  'channels': range(1,6),
427  },
428  'mhs_n19': {
429  'DiagSpaceGrp': radiance_s,
430  'process': True,
431  'anGrp': mhsGrp,
432  'binVarConfigs': radianceBinVars,
433  'diagNames': pconf.defaultDiagnostics+pconf.sigmaDiagnostics,
434  'channels': range(1,6),
435  },
436  'seviri_m08': {
437  'DiagSpaceGrp': radiance_s,
438  'process': False,
439  'binVarConfigs': radianceBinVars,
440  'diagNames': pconf.defaultDiagnostics,
441  'channels': [5],
442  },
443  'sndrd1_g15': {
444  'DiagSpaceGrp': radiance_s,
445  'process': False,
446  'binVarConfigs': radianceBinVars,
447  'diagNames': pconf.defaultDiagnostics,
448  'channels': range(1,16),
449  },
450  'sndrd2_g15': {
451  'DiagSpaceGrp': radiance_s,
452  'process': False,
453  'binVarConfigs': radianceBinVars,
454  'diagNames': pconf.defaultDiagnostics,
455  'channels': range(1,16),
456  },
457  'sndrd3_g15': {
458  'DiagSpaceGrp': radiance_s,
459  'process': False,
460  'binVarConfigs': radianceBinVars,
461  'diagNames': pconf.defaultDiagnostics,
462  'channels': range(1,16),
463  },
464  'sndrd4_g15': {
465  'DiagSpaceGrp': radiance_s,
466  'process': False,
467  'binVarConfigs': radianceBinVars,
468  'diagNames': pconf.defaultDiagnostics,
469  'channels': range(1,16),
470  },
471 #models
472  'mpas': {
473  'DiagSpaceGrp': model_s,
474  'process': True,
475  'anGrp': modelGrp,
476  'binVarConfigs': modelBinVars,
477  'diagNames': pconf.modelDiags,
478  },
479 }
480