diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/aircraft.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/aircraft.yaml index 14eb2df19..9b66e895f 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/aircraft.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/aircraft.yaml @@ -4,6 +4,7 @@ obs space: engine: type: H5File obsfile: '{{cycle_dir}}/aircraft.{{window_begin}}.nc4' + missing file action: error obsgrouping: group variables: ["stationIdentification", "releaseTime"] sort variable: "pressure" diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/airs_aqua.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/airs_aqua.yaml index d12fd9e1e..2d62d7265 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/airs_aqua.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/airs_aqua.yaml @@ -4,6 +4,7 @@ obs space: engine: type: H5File obsfile: '{{cycle_dir}}/airs_aqua.{{window_begin}}.nc4' + missing file action: warn obsdataout: engine: type: H5File diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/amsr2_gcom-w1.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/amsr2_gcom-w1.yaml index fab565b80..4f7a107b0 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/amsr2_gcom-w1.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/amsr2_gcom-w1.yaml @@ -4,6 +4,7 @@ obs space: engine: type: H5File obsfile: '{{cycle_dir}}/amsr2_gcom-w1.{{window_begin}}.nc4' + missing file action: warn obsdataout: engine: type: H5File diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/amsua_aqua.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/amsua_aqua.yaml index 992829e0c..237984750 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/amsua_aqua.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/amsua_aqua.yaml @@ -4,6 +4,7 @@ obs space: engine: type: H5File obsfile: '{{cycle_dir}}/amsua_aqua.{{window_begin}}.nc4' + missing file action: warn obsdataout: engine: type: H5File diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/amsua_metop-b.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/amsua_metop-b.yaml index 2488694d8..da9d7e6b5 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/amsua_metop-b.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/amsua_metop-b.yaml @@ -4,6 +4,7 @@ obs space: engine: type: H5File obsfile: '{{cycle_dir}}/amsua_metop-b.{{window_begin}}.nc4' + missing file action: warn obsdataout: engine: type: H5File diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/amsua_metop-c.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/amsua_metop-c.yaml index 72f88db3b..14666631d 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/amsua_metop-c.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/amsua_metop-c.yaml @@ -4,6 +4,7 @@ obs space: engine: type: H5File obsfile: '{{cycle_dir}}/amsua_metop-c.{{window_begin}}.nc4' + missing file action: warn obsdataout: engine: type: H5File diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/amsua_n15.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/amsua_n15.yaml index 3f86648a3..cbb34d1ff 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/amsua_n15.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/amsua_n15.yaml @@ -4,6 +4,7 @@ obs space: engine: type: H5File obsfile: '{{cycle_dir}}/amsua_n15.{{window_begin}}.nc4' + missing file action: warn obsdataout: engine: type: H5File diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/amsua_n18.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/amsua_n18.yaml index 0c23b8241..252ba25bc 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/amsua_n18.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/amsua_n18.yaml @@ -4,6 +4,7 @@ obs space: engine: type: H5File obsfile: '{{cycle_dir}}/amsua_n18.{{window_begin}}.nc4' + missing file action: warn obsdataout: engine: type: H5File diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/amsua_n19.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/amsua_n19.yaml index 5e493c28f..f4214327a 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/amsua_n19.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/amsua_n19.yaml @@ -4,6 +4,7 @@ obs space: engine: type: H5File obsfile: '{{cycle_dir}}/amsua_n19.{{window_begin}}.nc4' + missing file action: warn obsdataout: engine: type: H5File diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/atms_n20.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/atms_n20.yaml index fcbad1538..1ac0e2a9a 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/atms_n20.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/atms_n20.yaml @@ -4,6 +4,7 @@ obs space: engine: type: H5File obsfile: '{{cycle_dir}}/atms_n20.{{window_begin}}.nc4' + missing file action: warn obsdataout: engine: type: H5File diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/atms_npp.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/atms_npp.yaml index 1da8e0843..e253acd38 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/atms_npp.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/atms_npp.yaml @@ -4,6 +4,7 @@ obs space: engine: type: H5File obsfile: '{{cycle_dir}}/atms_npp.{{window_begin}}.nc4' + missing file action: warn obsdataout: engine: type: H5File diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/avhrr3_metop-b.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/avhrr3_metop-b.yaml index 2f26187a5..cba62d1c0 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/avhrr3_metop-b.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/avhrr3_metop-b.yaml @@ -4,6 +4,7 @@ obs space: engine: type: H5File obsfile: '{{cycle_dir}}/avhrr3_metop-b.{{window_begin}}.nc4' + missing file action: warn obsdataout: engine: type: H5File diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/avhrr3_n18.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/avhrr3_n18.yaml index f1767c62b..dde919208 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/avhrr3_n18.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/avhrr3_n18.yaml @@ -4,6 +4,7 @@ obs space: engine: type: H5File obsfile: '{{cycle_dir}}/avhrr3_n18.{{window_begin}}.nc4' + missing file action: warn obsdataout: engine: type: H5File diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/avhrr3_n19.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/avhrr3_n19.yaml index a44deb6b5..59f76078a 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/avhrr3_n19.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/avhrr3_n19.yaml @@ -4,6 +4,7 @@ obs space: engine: type: H5File obsfile: '{{cycle_dir}}/avhrr3_n19.{{window_begin}}.nc4' + missing file action: warn obsdataout: engine: type: H5File diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/cris-fsr_n20.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/cris-fsr_n20.yaml index eb4705485..2ca033e39 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/cris-fsr_n20.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/cris-fsr_n20.yaml @@ -4,6 +4,7 @@ obs space: engine: type: H5File obsfile: '{{cycle_dir}}/cris-fsr_n20.{{window_begin}}.nc4' + missing file action: warn obsdataout: engine: type: H5File diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/cris-fsr_npp.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/cris-fsr_npp.yaml index 2f1937b2a..a76a60c3a 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/cris-fsr_npp.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/cris-fsr_npp.yaml @@ -4,6 +4,7 @@ obs space: engine: type: H5File obsfile: '{{cycle_dir}}/cris-fsr_npp.{{window_begin}}.nc4' + missing file action: warn obsdataout: engine: type: H5File diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/gmi_gpm.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/gmi_gpm.yaml index 928d05422..00f4168ca 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/gmi_gpm.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/gmi_gpm.yaml @@ -4,6 +4,7 @@ obs space: engine: type: H5File obsfile: '{{cycle_dir}}/gmi_gpm.{{window_begin}}.nc4' + missing file action: warn obsdataout: engine: type: H5File diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/gps.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/gps.yaml index ec721d136..372027c9c 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/gps.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/gps.yaml @@ -4,6 +4,7 @@ obs space: engine: type: H5File obsfile: '{{cycle_dir}}/gps.{{window_begin}}.nc4' + missing file action: error obsgrouping: group variables: [ 'sequenceNumber' ] sort variable: 'impactHeightRO' diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/iasi_metop-b.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/iasi_metop-b.yaml index f837142a4..ae7035801 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/iasi_metop-b.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/iasi_metop-b.yaml @@ -4,6 +4,7 @@ obs space: engine: type: H5File obsfile: '{{cycle_dir}}/iasi_metop-b.{{window_begin}}.nc4' + missing file action: warn obsdataout: engine: type: H5File diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/iasi_metop-c.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/iasi_metop-c.yaml index 5a28a4e19..51e722085 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/iasi_metop-c.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/iasi_metop-c.yaml @@ -4,6 +4,7 @@ obs space: engine: type: H5File obsfile: '{{cycle_dir}}/iasi_metop-c.{{window_begin}}.nc4' + missing file action: warn obsdataout: engine: type: H5File diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/mhs_metop-b.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/mhs_metop-b.yaml index bda7d6de1..3482d3785 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/mhs_metop-b.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/mhs_metop-b.yaml @@ -4,6 +4,7 @@ obs space: engine: type: H5File obsfile: '{{cycle_dir}}/mhs_metop-b.{{window_begin}}.nc4' + missing file action: warn obsdataout: engine: type: H5File diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/mhs_metop-c.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/mhs_metop-c.yaml index fe148ddde..430e5e925 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/mhs_metop-c.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/mhs_metop-c.yaml @@ -4,6 +4,7 @@ obs space: engine: type: H5File obsfile: '{{cycle_dir}}/mhs_metop-c.{{window_begin}}.nc4' + missing file action: warn obsdataout: engine: type: H5File diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/mhs_n19.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/mhs_n19.yaml index d092fd180..3e62069e9 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/mhs_n19.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/mhs_n19.yaml @@ -4,6 +4,7 @@ obs space: engine: type: H5File obsfile: '{{cycle_dir}}/mhs_n19.{{window_begin}}.nc4' + missing file action: warn obsdataout: engine: type: H5File diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/mls55_aura.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/mls55_aura.yaml index 029377f03..fdbbbef13 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/mls55_aura.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/mls55_aura.yaml @@ -4,6 +4,7 @@ obs space: engine: type: H5File obsfile: '{{cycle_dir}}/mls55_aura.{{window_begin}}.nc4' + missing file action: error obsdataout: engine: type: H5File diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/omi_aura.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/omi_aura.yaml index cb77f93ff..47f5df41d 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/omi_aura.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/omi_aura.yaml @@ -4,6 +4,7 @@ obs space: engine: type: H5File obsfile: '{{cycle_dir}}/omi_aura.{{window_begin}}.nc4' + missing file action: error obsdataout: engine: type: H5File diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/ompsnm_npp.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/ompsnm_npp.yaml index 91b001b53..398359b17 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/ompsnm_npp.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/ompsnm_npp.yaml @@ -4,6 +4,7 @@ obs space: engine: type: H5File obsfile: '{{cycle_dir}}/ompsnm_npp.{{window_begin}}.nc4' + missing file action: error obsdataout: engine: type: H5File diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/pibal.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/pibal.yaml index 5ed87531f..5dee748e4 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/pibal.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/pibal.yaml @@ -4,6 +4,7 @@ obs space: engine: type: H5File obsfile: '{{cycle_dir}}/pibal.{{window_begin}}.nc4' + missing file action: error obsgrouping: group variables: ["stationIdentification", "releaseTime"] sort variable: "pressure" diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/satwind.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/satwind.yaml index bd9def8c7..9b9fd4c85 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/satwind.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/satwind.yaml @@ -4,6 +4,7 @@ obs space: engine: type: H5File obsfile: '{{cycle_dir}}/satwind.{{window_begin}}.nc4' + missing file action: error obsdataout: engine: type: H5File diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/scatwind.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/scatwind.yaml index 7e2b6f9d9..6c1c75105 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/scatwind.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/scatwind.yaml @@ -4,6 +4,7 @@ obs space: engine: type: H5File obsfile: '{{cycle_dir}}/scatwind.{{window_begin}}.nc4' + missing file action: error obsdataout: engine: type: H5File diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/sfc.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/sfc.yaml index f5dbd5a39..a8e927030 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/sfc.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/sfc.yaml @@ -4,6 +4,7 @@ obs space: engine: type: H5File obsfile: '{{cycle_dir}}/sfc.{{window_begin}}.nc4' + missing file action: error obsdataout: engine: type: H5File diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/sfcship.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/sfcship.yaml index 040ae240c..56caec597 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/sfcship.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/sfcship.yaml @@ -4,6 +4,7 @@ obs space: engine: type: H5File obsfile: '{{cycle_dir}}/sfcship.{{window_begin}}.nc4' + missing file action: error obsdataout: engine: type: H5File diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/sondes.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/sondes.yaml index a65e3020c..795690ae5 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/sondes.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/sondes.yaml @@ -4,6 +4,7 @@ obs space: engine: type: H5File obsfile: '{{cycle_dir}}/sondes.{{window_begin}}.nc4' + missing file action: error obsgrouping: group variables: ["stationIdentification", "releaseTime"] sort variable: "pressure" diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/ssmis_f17.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/ssmis_f17.yaml index 6da3c0313..c3cc6a331 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/ssmis_f17.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/ssmis_f17.yaml @@ -14,6 +14,7 @@ engine: type: H5File obsfile: '{{cycle_dir}}/ssmis_f17.{{window_begin}}.nc4' + missing file action: warn obsdataout: engine: type: H5File diff --git a/src/swell/utilities/run_jedi_executables.py b/src/swell/utilities/run_jedi_executables.py index ea2d2d122..25f2690f7 100644 --- a/src/swell/utilities/run_jedi_executables.py +++ b/src/swell/utilities/run_jedi_executables.py @@ -9,7 +9,6 @@ import os -import netCDF4 as nc from swell.utilities.shell_commands import run_track_log_subprocess # -------------------------------------------------------------------------------------------------- @@ -17,20 +16,22 @@ def check_obs(path_to_observing_sys_yamls, observation, obs_dict, cycle_time): - use_observation = False + use_observation = True # Check if file exists # -------------------- filename = obs_dict['obs space']['obsdatain']['engine']['obsfile'] if os.path.exists(filename): - - # Open file and check if number of location dimension is nonzero - # -------------------------------------------------------------- - dataset = nc.Dataset(filename, 'r') - - for dim_name, dim in dataset.dimensions.items(): - if dim_name == 'Location' and dim.size > 0: - use_observation = True + # Check if file is not empty (size > 0) + # ------------------------------------- + if os.path.getsize(filename) < 1: + use_observation = False + else: + miss_file_action = obs_dict['obs space']['obsdatain']['engine']['missing file action'] + # Check how to handle missing files + # --------------------------------- + if miss_file_action == 'error': + use_observation = False return use_observation