Fearture/jjin3/allow for missing ob type b

src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/aircraft.yaml

@@ -4,6 +4,7 @@ obs space:
     engine:
       type: H5File
       obsfile: '{{cycle_dir}}/aircraft.{{window_begin}}.nc4'
+      missing file action: error
     obsgrouping:
       group variables: ["stationIdentification", "releaseTime"]
       sort variable: "pressure"

src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/airs_aqua.yaml

@@ -4,6 +4,7 @@ obs space:
     engine:
       type: H5File
       obsfile: '{{cycle_dir}}/airs_aqua.{{window_begin}}.nc4'
+      missing file action: warn
   obsdataout:
     engine:
       type: H5File

src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/amsr2_gcom-w1.yaml

@@ -4,6 +4,7 @@ obs space:
     engine:
       type: H5File
       obsfile: '{{cycle_dir}}/amsr2_gcom-w1.{{window_begin}}.nc4'
+      missing file action: warn
   obsdataout:
     engine:
       type: H5File

src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/amsua_aqua.yaml

@@ -4,6 +4,7 @@ obs space:
     engine:
       type: H5File
       obsfile: '{{cycle_dir}}/amsua_aqua.{{window_begin}}.nc4'
+      missing file action: warn
   obsdataout:
     engine:
       type: H5File

src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/amsua_metop-b.yaml

@@ -4,6 +4,7 @@ obs space:
     engine:
       type: H5File
       obsfile: '{{cycle_dir}}/amsua_metop-b.{{window_begin}}.nc4'
+      missing file action: warn
   obsdataout:
     engine:
       type: H5File

src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/amsua_metop-c.yaml

@@ -4,6 +4,7 @@ obs space:
     engine:
       type: H5File
       obsfile: '{{cycle_dir}}/amsua_metop-c.{{window_begin}}.nc4'
+      missing file action: warn
   obsdataout:
     engine:
       type: H5File

src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/amsua_n15.yaml

@@ -4,6 +4,7 @@ obs space:
     engine:
       type: H5File
       obsfile: '{{cycle_dir}}/amsua_n15.{{window_begin}}.nc4'
+      missing file action: warn
   obsdataout:
     engine:
       type: H5File

src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/amsua_n18.yaml

@@ -4,6 +4,7 @@ obs space:
     engine:
       type: H5File
       obsfile: '{{cycle_dir}}/amsua_n18.{{window_begin}}.nc4'
+      missing file action: warn
   obsdataout:
     engine:
       type: H5File

src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/amsua_n19.yaml

@@ -4,6 +4,7 @@ obs space:
     engine:
       type: H5File
       obsfile: '{{cycle_dir}}/amsua_n19.{{window_begin}}.nc4'
+      missing file action: warn
   obsdataout:
     engine:
       type: H5File

src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/atms_n20.yaml

@@ -4,6 +4,7 @@ obs space:
     engine:
       type: H5File
       obsfile: '{{cycle_dir}}/atms_n20.{{window_begin}}.nc4'
+      missing file action: warn
   obsdataout:
     engine:
       type: H5File

src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/atms_npp.yaml

@@ -4,6 +4,7 @@ obs space:
     engine:
       type: H5File
       obsfile: '{{cycle_dir}}/atms_npp.{{window_begin}}.nc4'
+      missing file action: warn
   obsdataout:
     engine:
       type: H5File

src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/avhrr3_metop-b.yaml

@@ -4,6 +4,7 @@ obs space:
     engine:
       type: H5File
       obsfile: '{{cycle_dir}}/avhrr3_metop-b.{{window_begin}}.nc4'
+      missing file action: warn
   obsdataout:
     engine:
       type: H5File

src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/avhrr3_n18.yaml

@@ -4,6 +4,7 @@ obs space:
     engine:
       type: H5File
       obsfile: '{{cycle_dir}}/avhrr3_n18.{{window_begin}}.nc4'
+      missing file action: warn
   obsdataout:
     engine:
       type: H5File

src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/avhrr3_n19.yaml

@@ -4,6 +4,7 @@ obs space:
     engine:
       type: H5File
       obsfile: '{{cycle_dir}}/avhrr3_n19.{{window_begin}}.nc4'
+      missing file action: warn
   obsdataout:
     engine:
       type: H5File

src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/cris-fsr_n20.yaml

@@ -4,6 +4,7 @@ obs space:
     engine:
       type: H5File
       obsfile: '{{cycle_dir}}/cris-fsr_n20.{{window_begin}}.nc4'
+      missing file action: warn
   obsdataout:
     engine:
       type: H5File

src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/cris-fsr_npp.yaml

@@ -4,6 +4,7 @@ obs space:
     engine:
       type: H5File
       obsfile: '{{cycle_dir}}/cris-fsr_npp.{{window_begin}}.nc4'
+      missing file action: warn
   obsdataout:
     engine:
       type: H5File

src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/gmi_gpm.yaml

@@ -4,6 +4,7 @@ obs space:
     engine:
       type: H5File
       obsfile: '{{cycle_dir}}/gmi_gpm.{{window_begin}}.nc4'
+      missing file action: warn
   obsdataout:
     engine:
       type: H5File

src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/gps.yaml

@@ -4,6 +4,7 @@ obs space:
     engine:
       type: H5File
       obsfile: '{{cycle_dir}}/gps.{{window_begin}}.nc4'
+      missing file action: error
     obsgrouping:
       group variables: [ 'sequenceNumber' ]
       sort variable: 'impactHeightRO'

src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/iasi_metop-b.yaml

@@ -4,6 +4,7 @@ obs space:
     engine:
       type: H5File
       obsfile: '{{cycle_dir}}/iasi_metop-b.{{window_begin}}.nc4'
+      missing file action: warn
   obsdataout:
     engine:
       type: H5File

src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/iasi_metop-c.yaml

@@ -4,6 +4,7 @@ obs space:
     engine:
       type: H5File
       obsfile: '{{cycle_dir}}/iasi_metop-c.{{window_begin}}.nc4'
+      missing file action: warn
   obsdataout:
     engine:
       type: H5File

src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/mhs_metop-b.yaml

@@ -4,6 +4,7 @@ obs space:
     engine:
       type: H5File
       obsfile: '{{cycle_dir}}/mhs_metop-b.{{window_begin}}.nc4'
+      missing file action: warn
   obsdataout:
     engine:
       type: H5File

src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/mhs_metop-c.yaml

@@ -4,6 +4,7 @@ obs space:
     engine:
       type: H5File
       obsfile: '{{cycle_dir}}/mhs_metop-c.{{window_begin}}.nc4'
+      missing file action: warn
   obsdataout:
     engine:
       type: H5File

src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/mhs_n19.yaml

@@ -4,6 +4,7 @@ obs space:
     engine:
       type: H5File
       obsfile: '{{cycle_dir}}/mhs_n19.{{window_begin}}.nc4'
+      missing file action: warn
   obsdataout:
     engine:
       type: H5File

src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/mls55_aura.yaml

@@ -4,6 +4,7 @@ obs space:
     engine:
       type: H5File
       obsfile: '{{cycle_dir}}/mls55_aura.{{window_begin}}.nc4'
+      missing file action: error
   obsdataout:
     engine:
       type: H5File

src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/omi_aura.yaml

@@ -4,6 +4,7 @@ obs space:
     engine:
       type: H5File
       obsfile: '{{cycle_dir}}/omi_aura.{{window_begin}}.nc4'
+      missing file action: error
   obsdataout:
     engine:
       type: H5File

src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/ompsnm_npp.yaml

@@ -4,6 +4,7 @@ obs space:
     engine:
       type: H5File
       obsfile: '{{cycle_dir}}/ompsnm_npp.{{window_begin}}.nc4'
+      missing file action: error
   obsdataout:
     engine:
       type: H5File

src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/pibal.yaml

@@ -4,6 +4,7 @@ obs space:
     engine:
       type: H5File
       obsfile: '{{cycle_dir}}/pibal.{{window_begin}}.nc4'
+      missing file action: error
     obsgrouping:
       group variables: ["stationIdentification", "releaseTime"]
       sort variable: "pressure"

src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/satwind.yaml

@@ -4,6 +4,7 @@ obs space:
     engine:
       type: H5File
       obsfile: '{{cycle_dir}}/satwind.{{window_begin}}.nc4'
+      missing file action: error 
   obsdataout:
     engine:
       type: H5File

src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/scatwind.yaml

@@ -4,6 +4,7 @@ obs space:
     engine:
       type: H5File
       obsfile: '{{cycle_dir}}/scatwind.{{window_begin}}.nc4'
+      missing file action: error
   obsdataout:
     engine:
       type: H5File

src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/sfc.yaml

@@ -4,6 +4,7 @@ obs space:
     engine:
       type: H5File
       obsfile: '{{cycle_dir}}/sfc.{{window_begin}}.nc4'
+      missing file action: error
   obsdataout:
     engine:
       type: H5File

src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/sfcship.yaml

@@ -4,6 +4,7 @@ obs space:
     engine:
       type: H5File
       obsfile: '{{cycle_dir}}/sfcship.{{window_begin}}.nc4'
+      missing file action: error
   obsdataout:
     engine:
       type: H5File

src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/sondes.yaml

@@ -4,6 +4,7 @@ obs space:
     engine:
       type: H5File
       obsfile: '{{cycle_dir}}/sondes.{{window_begin}}.nc4'
+      missing file action: error
     obsgrouping:
       group variables: ["stationIdentification", "releaseTime"]
       sort variable: "pressure"

src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/ssmis_f17.yaml

@@ -14,6 +14,7 @@
       engine:
         type: H5File
         obsfile: '{{cycle_dir}}/ssmis_f17.{{window_begin}}.nc4'
+        missing file action: warn
     obsdataout:
       engine:
         type: H5File

src/swell/utilities/run_jedi_executables.py

@@ -9,28 +9,29 @@
 
 
 import os
-import netCDF4 as nc
 from swell.utilities.shell_commands import run_track_log_subprocess
 
 # --------------------------------------------------------------------------------------------------
 
 
 def check_obs(path_to_observing_sys_yamls, observation, obs_dict, cycle_time):
 
-    use_observation = False
+    use_observation = True
 
     # Check if file exists
     # --------------------
     filename = obs_dict['obs space']['obsdatain']['engine']['obsfile']
     if os.path.exists(filename):
-
-        # Open file and check if number of location dimension is nonzero
-        # --------------------------------------------------------------
-        dataset = nc.Dataset(filename, 'r')
-
-        for dim_name, dim in dataset.dimensions.items():
-            if dim_name == 'Location' and dim.size > 0:
-                use_observation = True
+        # Check if file is not empty (size > 0)
+        # -------------------------------------
+        if os.path.getsize(filename) < 1:
+            use_observation = False
+    else:
+        miss_file_action = obs_dict['obs space']['obsdatain']['engine']['missing file action']
+        # Check how to handle missing files
+        # ---------------------------------
+        if miss_file_action == 'error':
+            use_observation = False
 
     return use_observation