Add 4D-Ens-Var option for GEOS-atmosphere

src/swell/configuration/jedi/interfaces/geos_atmosphere/geos_atmosphere.yaml

@@ -3,7 +3,7 @@ total_processors: 6*{{npx_proc}}*{{npy_proc}}
 executables:
   hofx3D: fv3jedi_hofx_nomodel.x
   hofx4D: fv3jedi_hofx.x
-  variational3D: fv3jedi_var.x
-  variational4D: fv3jedi_var.x
-  variational4DEnsVar: fv3jedi_var.x
+  variational3D-Var: fv3jedi_var.x
+  variational4D-Var: fv3jedi_var.x
+  variational4D-Ens-Var: fv3jedi_var.x
   localensembleda: fv3jedi_letkf.x

src/swell/configuration/jedi/interfaces/geos_atmosphere/model/background4D.yaml

@@ -0,0 +1,70 @@
+- datetime: '{{local_background_time_iso}}'
+  filetype: cube sphere history
+  provider: geos
+  datapath: ''
+  filenames: ['{{cycle_dir}}/bkg.%yyyy%mm%ddT%hh%MM%ssZ.nc4',
+              '{{experiment_root}}/{{experiment_id}}/stage/fv3-jedi/geos_atmosphere/bkg/geos.crtmsrf.{{horizontal_resolution}}.nc4']
+  state variables: [u,v,ua,va,t,delp,ps,q,qi,ql,qr,qs,o3ppmv,phis,
+                    qls,qcn,cfcn,frocean,frland,varflt,ustar,bstar,
+                    zpbl,cm,ct,cq,kcbl,tsm,khl,khu,frlake,frseaice,vtype,
+                    stype,vfrac,sheleg,ts,soilt,soilm,u10m,v10m]
+- datetime: '2021-12-11T22:00:00Z'
+  filetype: cube sphere history
+  provider: geos
+  datapath: ''
+  filenames: ['{{cycle_dir}}/bkg.%yyyy%mm%ddT%hh%MM%ssZ.nc4',
+              '{{experiment_root}}/{{experiment_id}}/stage/fv3-jedi/geos_atmosphere/bkg/geos.crtmsrf.{{horizontal_resolution}}.nc4']
+  state variables: [u,v,ua,va,t,delp,ps,q,qi,ql,qr,qs,o3ppmv,phis,
+                    qls,qcn,cfcn,frocean,frland,varflt,ustar,bstar,
+                    zpbl,cm,ct,cq,kcbl,tsm,khl,khu,frlake,frseaice,vtype,
+                    stype,vfrac,sheleg,ts,soilt,soilm,u10m,v10m]
+- datetime: '2021-12-11T23:00:00Z'
+  filetype: cube sphere history
+  provider: geos
+  datapath: ''
+  filenames: ['{{cycle_dir}}/bkg.%yyyy%mm%ddT%hh%MM%ssZ.nc4',
+              '{{experiment_root}}/{{experiment_id}}/stage/fv3-jedi/geos_atmosphere/bkg/geos.crtmsrf.{{horizontal_resolution}}.nc4']
+  state variables: [u,v,ua,va,t,delp,ps,q,qi,ql,qr,qs,o3ppmv,phis,
+                    qls,qcn,cfcn,frocean,frland,varflt,ustar,bstar,
+                    zpbl,cm,ct,cq,kcbl,tsm,khl,khu,frlake,frseaice,vtype,
+                    stype,vfrac,sheleg,ts,soilt,soilm,u10m,v10m]
+- datetime: '2021-12-12T00:00:00Z'
+  filetype: cube sphere history
+  provider: geos
+  datapath: ''
+  filenames: ['{{cycle_dir}}/bkg.%yyyy%mm%ddT%hh%MM%ssZ.nc4',
+              '{{experiment_root}}/{{experiment_id}}/stage/fv3-jedi/geos_atmosphere/bkg/geos.crtmsrf.{{horizontal_resolution}}.nc4']
+  state variables: [u,v,ua,va,t,delp,ps,q,qi,ql,qr,qs,o3ppmv,phis,
+                    qls,qcn,cfcn,frocean,frland,varflt,ustar,bstar,
+                    zpbl,cm,ct,cq,kcbl,tsm,khl,khu,frlake,frseaice,vtype,
+                    stype,vfrac,sheleg,ts,soilt,soilm,u10m,v10m]
+- datetime: '2021-12-12T01:00:00Z'
+  filetype: cube sphere history
+  provider: geos
+  datapath: ''
+  filenames: ['{{cycle_dir}}/bkg.%yyyy%mm%ddT%hh%MM%ssZ.nc4',
+              '{{experiment_root}}/{{experiment_id}}/stage/fv3-jedi/geos_atmosphere/bkg/geos.crtmsrf.{{horizontal_resolution}}.nc4']
+  state variables: [u,v,ua,va,t,delp,ps,q,qi,ql,qr,qs,o3ppmv,phis,
+                    qls,qcn,cfcn,frocean,frland,varflt,ustar,bstar,
+                    zpbl,cm,ct,cq,kcbl,tsm,khl,khu,frlake,frseaice,vtype,
+                    stype,vfrac,sheleg,ts,soilt,soilm,u10m,v10m]
+- datetime: '2021-12-12T02:00:00Z'
+  filetype: cube sphere history
+  provider: geos
+  datapath: ''
+  filenames: ['{{cycle_dir}}/bkg.%yyyy%mm%ddT%hh%MM%ssZ.nc4',
+              '{{experiment_root}}/{{experiment_id}}/stage/fv3-jedi/geos_atmosphere/bkg/geos.crtmsrf.{{horizontal_resolution}}.nc4']
+  state variables: [u,v,ua,va,t,delp,ps,q,qi,ql,qr,qs,o3ppmv,phis,
+                    qls,qcn,cfcn,frocean,frland,varflt,ustar,bstar,
+                    zpbl,cm,ct,cq,kcbl,tsm,khl,khu,frlake,frseaice,vtype,
+                    stype,vfrac,sheleg,ts,soilt,soilm,u10m,v10m]
+- datetime: '2021-12-12T03:00:00Z'
+  filetype: cube sphere history
+  provider: geos
+  datapath: ''
+  filenames: ['{{cycle_dir}}/bkg.%yyyy%mm%ddT%hh%MM%ssZ.nc4',
+              '{{experiment_root}}/{{experiment_id}}/stage/fv3-jedi/geos_atmosphere/bkg/geos.crtmsrf.{{horizontal_resolution}}.nc4']
+  state variables: [u,v,ua,va,t,delp,ps,q,qi,ql,qr,qs,o3ppmv,phis,
+                    qls,qcn,cfcn,frocean,frland,varflt,ustar,bstar,
+                    zpbl,cm,ct,cq,kcbl,tsm,khl,khu,frlake,frseaice,vtype,
+                    stype,vfrac,sheleg,ts,soilt,soilm,u10m,v10m]

src/swell/configuration/jedi/interfaces/geos_atmosphere/model/background_error.yaml

@@ -4,7 +4,7 @@ saber central block:
   read:
     gsi akbk: '{{experiment_root}}/{{experiment_id}}/stage/fv3-jedi/geos_atmosphere/fv3files/akbk{{vertical_resolution}}.nc4'
     gsi error covariance file: '{{experiment_root}}/{{experiment_id}}/stage/fv3-jedi/geos_atmosphere/gsibec/gsibec_coefficients_c{{horizontal_resolution}}.nc4'
-    gsi berror namelist file: '{{experiment_root}}/{{experiment_id}}/stage/fv3-jedi/geos_atmosphere/gsibec/gsibec_configuration_c{{horizontal_resolution}}.nml'
+    gsi berror namelist file: '{{experiment_root}}/{{experiment_id}}/stage/fv3-jedi/geos_atmosphere/gsibec/{{gsibec_configuration}}_c{{horizontal_resolution}}.nml'
     processor layout x direction: {{gsibec_npx_proc}}
     processor layout y direction: {{gsibec_npy_proc}}
     debugging mode: false
@@ -13,12 +13,10 @@ saber outer blocks:
   state variables to inverse: &bvars [eastward_wind,northward_wind,air_temperature,surface_pressure,
                                       specific_humidity,cloud_liquid_ice,cloud_liquid_water,
                                       mole_fraction_of_ozone_in_air,
-                                      skin_temperature,
-                                      fraction_of_ocean,fraction_of_lake,fraction_of_ice,
                                       sfc_geopotential_height_times_grav]
   gsi akbk: '{{experiment_root}}/{{experiment_id}}/stage/fv3-jedi/geos_atmosphere/fv3files/akbk{{vertical_resolution}}.nc4'
   gsi error covariance file: '{{experiment_root}}/{{experiment_id}}/stage/fv3-jedi/geos_atmosphere/gsibec/gsibec_coefficients_c{{horizontal_resolution}}.nc4'
-  gsi berror namelist file: '{{experiment_root}}/{{experiment_id}}/stage/fv3-jedi/geos_atmosphere/gsibec/gsibec_configuration_c{{horizontal_resolution}}.nml'
+  gsi berror namelist file: '{{experiment_root}}/{{experiment_id}}/stage/fv3-jedi/geos_atmosphere/gsibec/{{gsibec_configuration}}_c{{horizontal_resolution}}.nml'
   processor layout x direction: {{gsibec_npx_proc}}
   processor layout y direction: {{gsibec_npy_proc}}
   debugging mode: false

src/swell/configuration/jedi/interfaces/geos_atmosphere/model/cost_type.yaml

@@ -0,0 +1 @@
+'{{cost_type}}'

src/swell/configuration/jedi/interfaces/geos_atmosphere/model/stage.yaml

@@ -4,4 +4,5 @@
       - ['{{experiment_root}}/{{experiment_id}}/jedi_bundle/source/fv3-jedi/test/Data/fieldmetadata/*', '{{experiment_root}}/{{experiment_id}}/stage/fv3-jedi/geos_atmosphere/fieldmetadata/']
       - ['{{experiment_root}}/{{experiment_id}}/jedi_bundle/source/fv3-jedi/test/Data/fv3files/*', '{{experiment_root}}/{{experiment_id}}/stage/fv3-jedi/geos_atmosphere/fv3files/']
       - ['{{swell_static_files}}/jedi/interfaces/geos_atmosphere/gsibec/gsibec_coefficients_c{{horizontal_resolution}}.nc4', '{{experiment_root}}/{{experiment_id}}/stage/fv3-jedi/geos_atmosphere/gsibec/']
-      - ['{{swell_static_files}}/jedi/interfaces/geos_atmosphere/gsibec/gsibec_configuration_c{{horizontal_resolution}}.nml', '{{experiment_root}}/{{experiment_id}}/stage/fv3-jedi/geos_atmosphere/gsibec/']
+      - ['{{swell_static_files}}/jedi/interfaces/geos_atmosphere/gsibec/{{gsibec_configuration}}_c{{horizontal_resolution}}.nml', '{{experiment_root}}/{{experiment_id}}/stage/fv3-jedi/geos_atmosphere/gsibec/']
+      - ['{{swell_static_files}}/jedi/interfaces/geos_atmosphere/rcov/1.0.0/*', '{{experiment_root}}/{{experiment_id}}/stage/fv3-jedi/geos_atmosphere/rcov/']

src/swell/configuration/jedi/interfaces/geos_atmosphere/model/var4Dincrement1.yaml

@@ -0,0 +1,39 @@
+write increment: true
+increment:
+  state component:
+    states:
+    - date: 2021-12-11T21:00:00Z
+      filetype: auxgrid
+      gridtype: latlon
+      datapath: './'
+      filename: jedi.increment-iter1.
+    - date: 2021-12-11T22:00:00Z
+      filetype: auxgrid
+      gridtype: latlon
+      datapath: './'
+      filename: jedi.increment-iter1.
+    - date: 2021-12-11T23:00:00Z
+      filetype: auxgrid
+      gridtype: latlon
+      datapath: './'
+      filename: jedi.increment-iter1.
+    - date: 2021-12-12T00:00:00Z
+      filetype: auxgrid
+      gridtype: latlon
+      datapath: './'
+      filename: jedi.increment-iter1.
+    - date: 2021-12-12T01:00:00Z
+      filetype: auxgrid
+      gridtype: latlon
+      datapath: './'
+      filename: jedi.increment-iter1.
+    - date: 2021-12-12T02:00:00Z
+      filetype: auxgrid
+      gridtype: latlon
+      datapath: './'
+      filename: jedi.increment-iter1.
+    - date: 2021-12-12T03:00:00Z
+      filetype: auxgrid
+      gridtype: latlon
+      datapath: './'
+      filename: jedi.increment-iter1.

src/swell/configuration/jedi/interfaces/geos_atmosphere/model/varincrement1.yaml

@@ -0,0 +1,7 @@
+write increment: true
+increment:
+  state component:
+    filetype: auxgrid
+    gridtype: latlon
+    datapath: ./
+    filename: {{experiment_id}}.increment-iter1.

src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/aircraft.yaml

@@ -24,8 +24,8 @@ obs operator:
     - name: windNorthward
 
     # Hofx scaling
-    hofx scaling field: SurfaceWindScalingPressure
-    hofx scaling field group: DerivedVariables
+#   hofx scaling field: SurfaceWindScalingPressure
+#   hofx scaling field group: DerivedVariables
 
   - name: VertInterp
     observation alias file: '{{experiment_root}}/{{experiment_id}}/configuration/jedi/interfaces/{{model_component}}/observations/obsop_name_map.yaml'

src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/airs_aqua.yaml

@@ -21,7 +21,7 @@ obs operator:
 
   linear obs operator:
     Absorbers: [H2O,O3]
-    Surfaces: [Water_Temperature]
+    Surfaces: [Water_Temperature,Land_Temperature,Ice_Temperature,Snow_Temperature]
 
 obs bias:
   input file: '{{cycle_dir}}/airs_aqua.{{background_time}}.satbias.nc4'
@@ -53,6 +53,10 @@ obs bias:
         ratio for small dataset: 2.0
     output file: '{{cycle_dir}}/airs_aqua.{{window_begin}}.satbias.nc4'
 
+obs error:
+  covariance model: cross variable covariances
+  input file: '{{experiment_root}}/{{experiment_id}}/stage/fv3-jedi/geos_atmosphere/rcov/airs_aqua_119_jedi_rcov.nc4'
+
 obs prior filters:
 - filter: Perform Action
   filter variables:

src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/amsua_aqua.yaml

@@ -24,6 +24,8 @@ obs operator:
 
 obs bias:
   input file: '{{cycle_dir}}/amsua_aqua.{{background_time}}.satbias.nc4'
+  variables without bc: [brightnessTemperature]
+  channels: 14
   variational bc:
     predictors:
     - name: constant

src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/amsua_metop-c.yaml

@@ -52,11 +52,11 @@ obs bias:
     step size: 1.0e-4
     largest analysis variance: 10000.0
     prior:
-      input file: '{{cycle_dir}}/INSTRUMENT.{{background_time}}.satbias.nc4'
+      input file: '{{cycle_dir}}/amsua_metop-c.{{background_time}}.satbias.nc4'
       inflation:
         ratio: 1.1
         ratio for small dataset: 2.0
-    output file: '{{cycle_dir}}/INSTRUMENT.{{window_begin}}.satbias.nc4'
+    output file: '{{cycle_dir}}/amsua_metop-c.{{window_begin}}.satbias.nc4'
 
 obs prior filters:
 # Assign obs error

src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/cris-fsr_n20.yaml

@@ -20,7 +20,7 @@ obs operator:
     CoefficientPath: '{{crtm_coeff_dir}}/'
   linear obs operator:
     Absorbers: [H2O,O3]
-    Surfaces: [Water_Temperature]
+    Surfaces: [Water_Temperature,Land_Temperature,Ice_Temperature,Snow_Temperature]
 
 obs bias:
   input file: '{{cycle_dir}}/cris-fsr_n20.{{background_time}}.satbias.nc4'
@@ -52,6 +52,10 @@ obs bias:
         ratio for small dataset: 2.0
     output file: '{{cycle_dir}}/cris-fsr_n20.{{window_begin}}.satbias.nc4'
 
+obs error:
+  covariance model: cross variable covariances
+  input file: '{{experiment_root}}/{{experiment_id}}/stage/fv3-jedi/geos_atmosphere/rcov/cris-fsr_108_jedi_rcov.nc4'
+
 obs prior filters:
 - filter: Perform Action
   filter variables:

src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/cris-fsr_npp.yaml

@@ -20,7 +20,7 @@ obs operator:
     CoefficientPath: '{{crtm_coeff_dir}}/'
   linear obs operator:
     Absorbers: [H2O,O3]
-    Surfaces: [Water_Temperature]
+    Surfaces: [Water_Temperature,Land_Temperature,Ice_Temperature,Snow_Temperature]
 
 obs bias:
   input file: '{{cycle_dir}}/cris-fsr_npp.{{background_time}}.satbias.nc4'
@@ -52,6 +52,10 @@ obs bias:
         ratio for small dataset: 2.0
     output file: '{{cycle_dir}}/cris-fsr_npp.{{window_begin}}.satbias.nc4'
 
+obs error:
+  covariance model: cross variable covariances
+  input file: '{{experiment_root}}/{{experiment_id}}/stage/fv3-jedi/geos_atmosphere/rcov/cris-fsr_108_jedi_rcov.nc4'
+
 obs prior filters:
 - filter: Perform Action
   filter variables:

src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/iasi_metop-b.yaml

@@ -20,7 +20,7 @@ obs operator:
     CoefficientPath: '{{crtm_coeff_dir}}/'
   linear obs operator:
     Absorbers: [H2O,O3]
-    Surfaces: [Water_Temperature]
+    Surfaces: [Water_Temperature,Land_Temperature,Ice_Temperature,Snow_Temperature]
 
 obs bias:
   input file: '{{cycle_dir}}/iasi_metop-b.{{background_time}}.satbias.nc4'
@@ -52,6 +52,10 @@ obs bias:
         ratio for small dataset: 2.0
     output file: '{{cycle_dir}}/iasi_metop-b.{{window_begin}}.satbias.nc4'
 
+obs error:
+  covariance model: cross variable covariances
+  input file: '{{experiment_root}}/{{experiment_id}}/stage/fv3-jedi/geos_atmosphere/rcov/iasi_metop_141_jedi_rcov.nc4'
+
 obs prior filters:
 - filter: Perform Action
   filter variables:

src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/iasi_metop-c.yaml

@@ -20,7 +20,7 @@ obs operator:
     CoefficientPath: '{{crtm_coeff_dir}}/'
   linear obs operator:
     Absorbers: [H2O,O3]
-    Surfaces: [Water_Temperature]
+    Surfaces: [Water_Temperature,Land_Temperature,Ice_Temperature,Snow_Temperature]
 
 obs bias:
   input file: '{{cycle_dir}}/iasi_metop-c.{{background_time}}.satbias.nc4'
@@ -52,6 +52,10 @@ obs bias:
         ratio for small dataset: 2.0
     output file: '{{cycle_dir}}/iasi_metop-c.{{window_begin}}.satbias.nc4'
 
+obs error:
+  covariance model: cross variable covariances
+  input file: '{{experiment_root}}/{{experiment_id}}/stage/fv3-jedi/geos_atmosphere/rcov/iasi_metop_141_jedi_rcov.nc4'
+
 obs prior filters:
 - filter: Perform Action
   filter variables:

src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/mhs_metop-c.yaml

@@ -21,7 +21,7 @@ obs operator:
     EndianType: little_endian
     CoefficientPath: '{{crtm_coeff_dir}}/'
   linear obs operator:
-    Absorbers: [H2O,O3,CO2]
+    Absorbers: [H2O,O3]
     Clouds: [Water]
     Surfaces: [Water_Temperature]
 

src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/mhs_n19.yaml

@@ -21,7 +21,7 @@ obs operator:
     EndianType: little_endian
     CoefficientPath: '{{crtm_coeff_dir}}/'
   linear obs operator:
-    Absorbers: [H2O,O3,CO2]
+    Absorbers: [H2O,O3]
     Clouds: [Water]
     Surfaces: [Water_Temperature]
 

src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/omi_aura.yaml

@@ -42,20 +42,20 @@ obs filters:
   action:
     name: reject
 # Reject rows 25+ (somewhat stringent but thats what we do in GEOS)
-- filter: Bounds Check
-  filter variables:
-  - name: ozoneTotal
-  test variables:
-  - name: MetaData/sensorScanPosition
-  minvalue: 3
-  maxvalue: 24
-  action:
-    name: reject
+#- filter: Bounds Check
+#  filter variables:
+#  - name: ozoneTotal
+#  test variables:
+#  - name: MetaData/sensorScanPosition
+#  minvalue: 3
+#  maxvalue: 24
+#  action:
+#    name: reject
 
  # Gross check
-- filter: Background Check
-  filter variables:
-  - name: ozoneTotal
-  threshold: 5.0
-  action:
-    name: reject
+#- filter: Background Check
+#  filter variables:
+#  - name: ozoneTotal
+#  threshold: 5.0
+#  action:
+#    name: reject

src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/ompsnm_npp.yaml

@@ -41,9 +41,9 @@ obs filters:
   action:
     name: reject
 # Gross check
-- filter: Background Check
-  filter variables:
-  - name: ozoneTotal
-  threshold: 5.0
-  action:
-    name: reject
+#- filter: Background Check
+#  filter variables:
+#  - name: ozoneTotal
+#  threshold: 5.0
+#  action:
+#    name: reject

src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/pibal.yaml

@@ -33,8 +33,8 @@ obs operator:
       interpolation method backup: log-linear
 
       # Hofx scaling
-      hofx scaling field: SurfaceWindScalingCombined
-      hofx scaling field group: DerivedVariables
+#     hofx scaling field: SurfaceWindScalingCombined
+#     hofx scaling field group: DerivedVariables
 
 obs pre filters:
 

src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/satwind.yaml

@@ -13,8 +13,8 @@ obs space:
 obs operator:
   name: VertInterp
   observation alias file: '{{experiment_root}}/{{experiment_id}}/configuration/jedi/interfaces/{{model_component}}/observations/obsop_name_map.yaml'
-  hofx scaling field: SurfaceWindScalingPressure
-  hofx scaling field group: DerivedVariables
+# hofx scaling field: SurfaceWindScalingPressure
+# hofx scaling field group: DerivedVariables
 
 obs prior filters:
 

src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/scatwind.yaml

@@ -20,8 +20,8 @@ obs operator:
   observation vertical coordinate group: DerivedVariables
   observation vertical coordinate: adjustedHeight
   interpolation method: linear
-  hofx scaling field: SurfaceWindScalingHeight
-  hofx scaling field group: DerivedVariables
+# hofx scaling field: SurfaceWindScalingHeight
+# hofx scaling field group: DerivedVariables
 
 obs prior filters: