From 469764894ac844c6b9cb2d86978eada050a2080d Mon Sep 17 00:00:00 2001 From: Ricardo Todling Date: Thu, 1 Aug 2024 16:17:10 -0400 Subject: [PATCH 1/5] the entry here allows for missing obs file w/o a crash --- .../jedi/interfaces/geos_atmosphere/observations/aircraft.yaml | 1 + .../jedi/interfaces/geos_atmosphere/observations/airs_aqua.yaml | 1 + .../interfaces/geos_atmosphere/observations/amsr2_gcom-w1.yaml | 1 + .../jedi/interfaces/geos_atmosphere/observations/amsua_aqua.yaml | 1 + .../interfaces/geos_atmosphere/observations/amsua_metop-b.yaml | 1 + .../interfaces/geos_atmosphere/observations/amsua_metop-c.yaml | 1 + .../jedi/interfaces/geos_atmosphere/observations/amsua_n15.yaml | 1 + .../jedi/interfaces/geos_atmosphere/observations/amsua_n18.yaml | 1 + .../jedi/interfaces/geos_atmosphere/observations/amsua_n19.yaml | 1 + .../jedi/interfaces/geos_atmosphere/observations/atms_n20.yaml | 1 + .../jedi/interfaces/geos_atmosphere/observations/atms_npp.yaml | 1 + .../interfaces/geos_atmosphere/observations/avhrr3_metop-b.yaml | 1 + .../jedi/interfaces/geos_atmosphere/observations/avhrr3_n18.yaml | 1 + .../jedi/interfaces/geos_atmosphere/observations/avhrr3_n19.yaml | 1 + .../interfaces/geos_atmosphere/observations/cris-fsr_n20.yaml | 1 + .../interfaces/geos_atmosphere/observations/cris-fsr_npp.yaml | 1 + .../jedi/interfaces/geos_atmosphere/observations/gmi_gpm.yaml | 1 + .../jedi/interfaces/geos_atmosphere/observations/gps.yaml | 1 + .../interfaces/geos_atmosphere/observations/iasi_metop-b.yaml | 1 + .../interfaces/geos_atmosphere/observations/iasi_metop-c.yaml | 1 + .../interfaces/geos_atmosphere/observations/mhs_metop-b.yaml | 1 + .../interfaces/geos_atmosphere/observations/mhs_metop-c.yaml | 1 + .../jedi/interfaces/geos_atmosphere/observations/mhs_n19.yaml | 1 + .../jedi/interfaces/geos_atmosphere/observations/mls55_aura.yaml | 1 + .../jedi/interfaces/geos_atmosphere/observations/omi_aura.yaml | 1 + .../jedi/interfaces/geos_atmosphere/observations/ompsnm_npp.yaml | 1 + 26 files changed, 26 insertions(+) diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/aircraft.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/aircraft.yaml index 14eb2df19..0e1ee201f 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/aircraft.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/aircraft.yaml @@ -4,6 +4,7 @@ obs space: engine: type: H5File obsfile: '{{cycle_dir}}/aircraft.{{window_begin}}.nc4' + missing file action: warn obsgrouping: group variables: ["stationIdentification", "releaseTime"] sort variable: "pressure" diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/airs_aqua.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/airs_aqua.yaml index d12fd9e1e..2d62d7265 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/airs_aqua.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/airs_aqua.yaml @@ -4,6 +4,7 @@ obs space: engine: type: H5File obsfile: '{{cycle_dir}}/airs_aqua.{{window_begin}}.nc4' + missing file action: warn obsdataout: engine: type: H5File diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/amsr2_gcom-w1.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/amsr2_gcom-w1.yaml index fab565b80..4f7a107b0 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/amsr2_gcom-w1.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/amsr2_gcom-w1.yaml @@ -4,6 +4,7 @@ obs space: engine: type: H5File obsfile: '{{cycle_dir}}/amsr2_gcom-w1.{{window_begin}}.nc4' + missing file action: warn obsdataout: engine: type: H5File diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/amsua_aqua.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/amsua_aqua.yaml index 992829e0c..237984750 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/amsua_aqua.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/amsua_aqua.yaml @@ -4,6 +4,7 @@ obs space: engine: type: H5File obsfile: '{{cycle_dir}}/amsua_aqua.{{window_begin}}.nc4' + missing file action: warn obsdataout: engine: type: H5File diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/amsua_metop-b.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/amsua_metop-b.yaml index 2488694d8..da9d7e6b5 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/amsua_metop-b.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/amsua_metop-b.yaml @@ -4,6 +4,7 @@ obs space: engine: type: H5File obsfile: '{{cycle_dir}}/amsua_metop-b.{{window_begin}}.nc4' + missing file action: warn obsdataout: engine: type: H5File diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/amsua_metop-c.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/amsua_metop-c.yaml index 72f88db3b..14666631d 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/amsua_metop-c.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/amsua_metop-c.yaml @@ -4,6 +4,7 @@ obs space: engine: type: H5File obsfile: '{{cycle_dir}}/amsua_metop-c.{{window_begin}}.nc4' + missing file action: warn obsdataout: engine: type: H5File diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/amsua_n15.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/amsua_n15.yaml index 3f86648a3..cbb34d1ff 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/amsua_n15.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/amsua_n15.yaml @@ -4,6 +4,7 @@ obs space: engine: type: H5File obsfile: '{{cycle_dir}}/amsua_n15.{{window_begin}}.nc4' + missing file action: warn obsdataout: engine: type: H5File diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/amsua_n18.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/amsua_n18.yaml index 0c23b8241..252ba25bc 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/amsua_n18.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/amsua_n18.yaml @@ -4,6 +4,7 @@ obs space: engine: type: H5File obsfile: '{{cycle_dir}}/amsua_n18.{{window_begin}}.nc4' + missing file action: warn obsdataout: engine: type: H5File diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/amsua_n19.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/amsua_n19.yaml index 5e493c28f..f4214327a 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/amsua_n19.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/amsua_n19.yaml @@ -4,6 +4,7 @@ obs space: engine: type: H5File obsfile: '{{cycle_dir}}/amsua_n19.{{window_begin}}.nc4' + missing file action: warn obsdataout: engine: type: H5File diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/atms_n20.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/atms_n20.yaml index fcbad1538..1ac0e2a9a 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/atms_n20.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/atms_n20.yaml @@ -4,6 +4,7 @@ obs space: engine: type: H5File obsfile: '{{cycle_dir}}/atms_n20.{{window_begin}}.nc4' + missing file action: warn obsdataout: engine: type: H5File diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/atms_npp.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/atms_npp.yaml index 1da8e0843..e253acd38 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/atms_npp.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/atms_npp.yaml @@ -4,6 +4,7 @@ obs space: engine: type: H5File obsfile: '{{cycle_dir}}/atms_npp.{{window_begin}}.nc4' + missing file action: warn obsdataout: engine: type: H5File diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/avhrr3_metop-b.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/avhrr3_metop-b.yaml index 2f26187a5..cba62d1c0 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/avhrr3_metop-b.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/avhrr3_metop-b.yaml @@ -4,6 +4,7 @@ obs space: engine: type: H5File obsfile: '{{cycle_dir}}/avhrr3_metop-b.{{window_begin}}.nc4' + missing file action: warn obsdataout: engine: type: H5File diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/avhrr3_n18.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/avhrr3_n18.yaml index f1767c62b..dde919208 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/avhrr3_n18.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/avhrr3_n18.yaml @@ -4,6 +4,7 @@ obs space: engine: type: H5File obsfile: '{{cycle_dir}}/avhrr3_n18.{{window_begin}}.nc4' + missing file action: warn obsdataout: engine: type: H5File diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/avhrr3_n19.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/avhrr3_n19.yaml index a44deb6b5..59f76078a 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/avhrr3_n19.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/avhrr3_n19.yaml @@ -4,6 +4,7 @@ obs space: engine: type: H5File obsfile: '{{cycle_dir}}/avhrr3_n19.{{window_begin}}.nc4' + missing file action: warn obsdataout: engine: type: H5File diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/cris-fsr_n20.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/cris-fsr_n20.yaml index eb4705485..2ca033e39 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/cris-fsr_n20.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/cris-fsr_n20.yaml @@ -4,6 +4,7 @@ obs space: engine: type: H5File obsfile: '{{cycle_dir}}/cris-fsr_n20.{{window_begin}}.nc4' + missing file action: warn obsdataout: engine: type: H5File diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/cris-fsr_npp.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/cris-fsr_npp.yaml index 2f1937b2a..a76a60c3a 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/cris-fsr_npp.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/cris-fsr_npp.yaml @@ -4,6 +4,7 @@ obs space: engine: type: H5File obsfile: '{{cycle_dir}}/cris-fsr_npp.{{window_begin}}.nc4' + missing file action: warn obsdataout: engine: type: H5File diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/gmi_gpm.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/gmi_gpm.yaml index 928d05422..00f4168ca 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/gmi_gpm.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/gmi_gpm.yaml @@ -4,6 +4,7 @@ obs space: engine: type: H5File obsfile: '{{cycle_dir}}/gmi_gpm.{{window_begin}}.nc4' + missing file action: warn obsdataout: engine: type: H5File diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/gps.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/gps.yaml index ec721d136..69df472c2 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/gps.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/gps.yaml @@ -4,6 +4,7 @@ obs space: engine: type: H5File obsfile: '{{cycle_dir}}/gps.{{window_begin}}.nc4' + missing file action: warn obsgrouping: group variables: [ 'sequenceNumber' ] sort variable: 'impactHeightRO' diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/iasi_metop-b.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/iasi_metop-b.yaml index f837142a4..ae7035801 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/iasi_metop-b.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/iasi_metop-b.yaml @@ -4,6 +4,7 @@ obs space: engine: type: H5File obsfile: '{{cycle_dir}}/iasi_metop-b.{{window_begin}}.nc4' + missing file action: warn obsdataout: engine: type: H5File diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/iasi_metop-c.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/iasi_metop-c.yaml index 5a28a4e19..51e722085 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/iasi_metop-c.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/iasi_metop-c.yaml @@ -4,6 +4,7 @@ obs space: engine: type: H5File obsfile: '{{cycle_dir}}/iasi_metop-c.{{window_begin}}.nc4' + missing file action: warn obsdataout: engine: type: H5File diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/mhs_metop-b.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/mhs_metop-b.yaml index bda7d6de1..3482d3785 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/mhs_metop-b.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/mhs_metop-b.yaml @@ -4,6 +4,7 @@ obs space: engine: type: H5File obsfile: '{{cycle_dir}}/mhs_metop-b.{{window_begin}}.nc4' + missing file action: warn obsdataout: engine: type: H5File diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/mhs_metop-c.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/mhs_metop-c.yaml index fe148ddde..430e5e925 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/mhs_metop-c.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/mhs_metop-c.yaml @@ -4,6 +4,7 @@ obs space: engine: type: H5File obsfile: '{{cycle_dir}}/mhs_metop-c.{{window_begin}}.nc4' + missing file action: warn obsdataout: engine: type: H5File diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/mhs_n19.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/mhs_n19.yaml index d092fd180..3e62069e9 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/mhs_n19.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/mhs_n19.yaml @@ -4,6 +4,7 @@ obs space: engine: type: H5File obsfile: '{{cycle_dir}}/mhs_n19.{{window_begin}}.nc4' + missing file action: warn obsdataout: engine: type: H5File diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/mls55_aura.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/mls55_aura.yaml index 029377f03..edb90762d 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/mls55_aura.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/mls55_aura.yaml @@ -4,6 +4,7 @@ obs space: engine: type: H5File obsfile: '{{cycle_dir}}/mls55_aura.{{window_begin}}.nc4' + missing file action: warn obsdataout: engine: type: H5File diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/omi_aura.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/omi_aura.yaml index cb77f93ff..ecf4f873d 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/omi_aura.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/omi_aura.yaml @@ -4,6 +4,7 @@ obs space: engine: type: H5File obsfile: '{{cycle_dir}}/omi_aura.{{window_begin}}.nc4' + missing file action: warn obsdataout: engine: type: H5File diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/ompsnm_npp.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/ompsnm_npp.yaml index 91b001b53..3ce44036a 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/ompsnm_npp.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/ompsnm_npp.yaml @@ -4,6 +4,7 @@ obs space: engine: type: H5File obsfile: '{{cycle_dir}}/ompsnm_npp.{{window_begin}}.nc4' + missing file action: warn obsdataout: engine: type: H5File From e824e3b9424b3c940920d235e84597150821fd7e Mon Sep 17 00:00:00 2001 From: gmao-jjin3 Date: Thu, 29 Aug 2024 15:58:51 -0400 Subject: [PATCH 2/5] Allow missing but skip zero-sized observational files in jedi_variational_config.yaml. --- .../geos_atmosphere/observations/pibal.yaml | 1 + .../geos_atmosphere/observations/satwind.yaml | 1 + .../geos_atmosphere/observations/scatwind.yaml | 1 + .../geos_atmosphere/observations/sfc.yaml | 1 + .../geos_atmosphere/observations/sfcship.yaml | 1 + .../geos_atmosphere/observations/sondes.yaml | 1 + .../geos_atmosphere/observations/ssmis_f17.yaml | 1 + src/swell/utilities/run_jedi_executables.py | 14 +++++--------- 8 files changed, 12 insertions(+), 9 deletions(-) diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/pibal.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/pibal.yaml index 5ed87531f..12e5126b6 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/pibal.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/pibal.yaml @@ -4,6 +4,7 @@ obs space: engine: type: H5File obsfile: '{{cycle_dir}}/pibal.{{window_begin}}.nc4' + missing file action: warn obsgrouping: group variables: ["stationIdentification", "releaseTime"] sort variable: "pressure" diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/satwind.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/satwind.yaml index bd9def8c7..aad7291ea 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/satwind.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/satwind.yaml @@ -4,6 +4,7 @@ obs space: engine: type: H5File obsfile: '{{cycle_dir}}/satwind.{{window_begin}}.nc4' + missing file action: warn obsdataout: engine: type: H5File diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/scatwind.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/scatwind.yaml index 7e2b6f9d9..567106293 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/scatwind.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/scatwind.yaml @@ -4,6 +4,7 @@ obs space: engine: type: H5File obsfile: '{{cycle_dir}}/scatwind.{{window_begin}}.nc4' + missing file action: warn obsdataout: engine: type: H5File diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/sfc.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/sfc.yaml index f5dbd5a39..0e170a4e8 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/sfc.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/sfc.yaml @@ -4,6 +4,7 @@ obs space: engine: type: H5File obsfile: '{{cycle_dir}}/sfc.{{window_begin}}.nc4' + missing file action: warn obsdataout: engine: type: H5File diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/sfcship.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/sfcship.yaml index 040ae240c..466ff464e 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/sfcship.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/sfcship.yaml @@ -4,6 +4,7 @@ obs space: engine: type: H5File obsfile: '{{cycle_dir}}/sfcship.{{window_begin}}.nc4' + missing file action: warn obsdataout: engine: type: H5File diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/sondes.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/sondes.yaml index a65e3020c..ff36851c8 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/sondes.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/sondes.yaml @@ -4,6 +4,7 @@ obs space: engine: type: H5File obsfile: '{{cycle_dir}}/sondes.{{window_begin}}.nc4' + missing file action: warn obsgrouping: group variables: ["stationIdentification", "releaseTime"] sort variable: "pressure" diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/ssmis_f17.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/ssmis_f17.yaml index 6da3c0313..c3cc6a331 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/ssmis_f17.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/ssmis_f17.yaml @@ -14,6 +14,7 @@ engine: type: H5File obsfile: '{{cycle_dir}}/ssmis_f17.{{window_begin}}.nc4' + missing file action: warn obsdataout: engine: type: H5File diff --git a/src/swell/utilities/run_jedi_executables.py b/src/swell/utilities/run_jedi_executables.py index ea2d2d122..3462d795a 100644 --- a/src/swell/utilities/run_jedi_executables.py +++ b/src/swell/utilities/run_jedi_executables.py @@ -17,20 +17,16 @@ def check_obs(path_to_observing_sys_yamls, observation, obs_dict, cycle_time): - use_observation = False + use_observation = True # Check if file exists # -------------------- filename = obs_dict['obs space']['obsdatain']['engine']['obsfile'] if os.path.exists(filename): - - # Open file and check if number of location dimension is nonzero - # -------------------------------------------------------------- - dataset = nc.Dataset(filename, 'r') - - for dim_name, dim in dataset.dimensions.items(): - if dim_name == 'Location' and dim.size > 0: - use_observation = True + # Check if file is not empty (size > 0) + # ------------------------------------- + if os.path.getsize(filename) < 1: + use_observation = False return use_observation From 3b5450a951ec4753cfb617f87c8961d960f0b3e2 Mon Sep 17 00:00:00 2001 From: gmao-jjin3 Date: Thu, 29 Aug 2024 16:51:58 -0400 Subject: [PATCH 3/5] Skip missing convectional and ozone observational files which can cause error. --- .../interfaces/geos_atmosphere/observations/aircraft.yaml | 2 +- .../jedi/interfaces/geos_atmosphere/observations/gps.yaml | 2 +- .../geos_atmosphere/observations/mls55_aura.yaml | 2 +- .../interfaces/geos_atmosphere/observations/omi_aura.yaml | 2 +- .../geos_atmosphere/observations/ompsnm_npp.yaml | 2 +- .../interfaces/geos_atmosphere/observations/pibal.yaml | 2 +- .../interfaces/geos_atmosphere/observations/satwind.yaml | 2 +- .../interfaces/geos_atmosphere/observations/scatwind.yaml | 2 +- .../jedi/interfaces/geos_atmosphere/observations/sfc.yaml | 2 +- .../interfaces/geos_atmosphere/observations/sfcship.yaml | 2 +- .../interfaces/geos_atmosphere/observations/sondes.yaml | 2 +- src/swell/utilities/run_jedi_executables.py | 8 +++++++- 12 files changed, 18 insertions(+), 12 deletions(-) diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/aircraft.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/aircraft.yaml index 0e1ee201f..9b66e895f 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/aircraft.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/aircraft.yaml @@ -4,7 +4,7 @@ obs space: engine: type: H5File obsfile: '{{cycle_dir}}/aircraft.{{window_begin}}.nc4' - missing file action: warn + missing file action: error obsgrouping: group variables: ["stationIdentification", "releaseTime"] sort variable: "pressure" diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/gps.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/gps.yaml index 69df472c2..372027c9c 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/gps.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/gps.yaml @@ -4,7 +4,7 @@ obs space: engine: type: H5File obsfile: '{{cycle_dir}}/gps.{{window_begin}}.nc4' - missing file action: warn + missing file action: error obsgrouping: group variables: [ 'sequenceNumber' ] sort variable: 'impactHeightRO' diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/mls55_aura.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/mls55_aura.yaml index edb90762d..fdbbbef13 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/mls55_aura.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/mls55_aura.yaml @@ -4,7 +4,7 @@ obs space: engine: type: H5File obsfile: '{{cycle_dir}}/mls55_aura.{{window_begin}}.nc4' - missing file action: warn + missing file action: error obsdataout: engine: type: H5File diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/omi_aura.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/omi_aura.yaml index ecf4f873d..47f5df41d 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/omi_aura.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/omi_aura.yaml @@ -4,7 +4,7 @@ obs space: engine: type: H5File obsfile: '{{cycle_dir}}/omi_aura.{{window_begin}}.nc4' - missing file action: warn + missing file action: error obsdataout: engine: type: H5File diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/ompsnm_npp.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/ompsnm_npp.yaml index 3ce44036a..398359b17 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/ompsnm_npp.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/ompsnm_npp.yaml @@ -4,7 +4,7 @@ obs space: engine: type: H5File obsfile: '{{cycle_dir}}/ompsnm_npp.{{window_begin}}.nc4' - missing file action: warn + missing file action: error obsdataout: engine: type: H5File diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/pibal.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/pibal.yaml index 12e5126b6..5dee748e4 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/pibal.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/pibal.yaml @@ -4,7 +4,7 @@ obs space: engine: type: H5File obsfile: '{{cycle_dir}}/pibal.{{window_begin}}.nc4' - missing file action: warn + missing file action: error obsgrouping: group variables: ["stationIdentification", "releaseTime"] sort variable: "pressure" diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/satwind.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/satwind.yaml index aad7291ea..9b9fd4c85 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/satwind.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/satwind.yaml @@ -4,7 +4,7 @@ obs space: engine: type: H5File obsfile: '{{cycle_dir}}/satwind.{{window_begin}}.nc4' - missing file action: warn + missing file action: error obsdataout: engine: type: H5File diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/scatwind.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/scatwind.yaml index 567106293..6c1c75105 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/scatwind.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/scatwind.yaml @@ -4,7 +4,7 @@ obs space: engine: type: H5File obsfile: '{{cycle_dir}}/scatwind.{{window_begin}}.nc4' - missing file action: warn + missing file action: error obsdataout: engine: type: H5File diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/sfc.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/sfc.yaml index 0e170a4e8..a8e927030 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/sfc.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/sfc.yaml @@ -4,7 +4,7 @@ obs space: engine: type: H5File obsfile: '{{cycle_dir}}/sfc.{{window_begin}}.nc4' - missing file action: warn + missing file action: error obsdataout: engine: type: H5File diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/sfcship.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/sfcship.yaml index 466ff464e..56caec597 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/sfcship.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/sfcship.yaml @@ -4,7 +4,7 @@ obs space: engine: type: H5File obsfile: '{{cycle_dir}}/sfcship.{{window_begin}}.nc4' - missing file action: warn + missing file action: error obsdataout: engine: type: H5File diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/sondes.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/sondes.yaml index ff36851c8..795690ae5 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/sondes.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/sondes.yaml @@ -4,7 +4,7 @@ obs space: engine: type: H5File obsfile: '{{cycle_dir}}/sondes.{{window_begin}}.nc4' - missing file action: warn + missing file action: error obsgrouping: group variables: ["stationIdentification", "releaseTime"] sort variable: "pressure" diff --git a/src/swell/utilities/run_jedi_executables.py b/src/swell/utilities/run_jedi_executables.py index 3462d795a..bc5e3a56f 100644 --- a/src/swell/utilities/run_jedi_executables.py +++ b/src/swell/utilities/run_jedi_executables.py @@ -27,7 +27,13 @@ def check_obs(path_to_observing_sys_yamls, observation, obs_dict, cycle_time): # ------------------------------------- if os.path.getsize(filename) < 1: use_observation = False - + else: + miss_file_action = obs_dict['obs space']['obsdatain']['engine']['missing file action'] + # Check how to handle missing files + # --------------------------------- + if miss_file_action == 'error': + use_observation = False + return use_observation From 3cf8ac7d63e2f56028b171123fdd4c47337c7dee Mon Sep 17 00:00:00 2001 From: gmao-jjin3 Date: Tue, 3 Sep 2024 12:40:02 -0400 Subject: [PATCH 4/5] Code norm. --- src/swell/utilities/run_jedi_executables.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/swell/utilities/run_jedi_executables.py b/src/swell/utilities/run_jedi_executables.py index bc5e3a56f..60572ad9a 100644 --- a/src/swell/utilities/run_jedi_executables.py +++ b/src/swell/utilities/run_jedi_executables.py @@ -26,14 +26,14 @@ def check_obs(path_to_observing_sys_yamls, observation, obs_dict, cycle_time): # Check if file is not empty (size > 0) # ------------------------------------- if os.path.getsize(filename) < 1: - use_observation = False + use_observation = False else: miss_file_action = obs_dict['obs space']['obsdatain']['engine']['missing file action'] - # Check how to handle missing files + # Check how to handle missing files # --------------------------------- if miss_file_action == 'error': - use_observation = False - + use_observation = False + return use_observation From 55825adcf558abfdda344bac3c0b2d3ae8df8de4 Mon Sep 17 00:00:00 2001 From: gmao-jjin3 Date: Tue, 3 Sep 2024 12:45:17 -0400 Subject: [PATCH 5/5] Code norm. --- src/swell/utilities/run_jedi_executables.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/swell/utilities/run_jedi_executables.py b/src/swell/utilities/run_jedi_executables.py index 60572ad9a..25f2690f7 100644 --- a/src/swell/utilities/run_jedi_executables.py +++ b/src/swell/utilities/run_jedi_executables.py @@ -9,7 +9,6 @@ import os -import netCDF4 as nc from swell.utilities.shell_commands import run_track_log_subprocess # --------------------------------------------------------------------------------------------------