From d205d89b1f5e11aaeecfebd39eb3afe78d201594 Mon Sep 17 00:00:00 2001 From: Ricardo Todling Date: Thu, 11 Dec 2025 12:45:44 -0500 Subject: [PATCH 1/2] test now both bufr2ioda AMSUA and IASI and now working in Var w/ minor changes to var yamls --- .../jedi/bufr2ioda/bufr_ncep_mtiasi.yaml | 34 +++++++++++++++++-- .../jedi/observation_ioda_names.yaml | 4 +-- 2 files changed, 34 insertions(+), 4 deletions(-) diff --git a/src/swell/configuration/jedi/bufr2ioda/bufr_ncep_mtiasi.yaml b/src/swell/configuration/jedi/bufr2ioda/bufr_ncep_mtiasi.yaml index bbd605f3a..ec689ad11 100644 --- a/src/swell/configuration/jedi/bufr2ioda/bufr_ncep_mtiasi.yaml +++ b/src/swell/configuration/jedi/bufr2ioda/bufr_ncep_mtiasi.yaml @@ -62,6 +62,18 @@ observations: query: "*/SELV" type: float + sensorChannelStart1: + query: "*/STCH" +# mnemonic: STCH + + sensorChannelEnd1: + query: "*/ENCH" +# mnemonic: ENCH + + channelScaleFactor1: + query: "*/CHSF" +# mnemonic: CHSF + sensorViewAngle: sensorScanAngle: fieldOfViewNumber: "*/FOVN" @@ -233,9 +245,27 @@ observations: units: "1" range: [0, 1] + - name: "MetaData/sensorChannelStart1" + source: variables/sensorChannelStart1 + dimensions: ["ChannelBlock"] + longName: "Starting channel number" + units: "" + + - name: "MetaData/sensorChannelEnd1" + source: variables/sensorChannelEnd1 + dimensions: ["ChannelBlock"] + longName: "Ending channel number" + units: "" + + - name: "MetaData/channelScaleFactor1" + source: variables/channelScaleFactor1 + dimensions: ["ChannelBlock"] + longName: "Channel scale factor" + units: "" + # The unit from BUFR is W m-2 sr-1 m -- this is radiance per wavenumber -# - name: "ObsValue/spectralRadiance" - - name: "ObsValue/radiance" +# - name: "ObsValue/radiance" + - name: "ObsValue/spectralRadiance" source: variables/spectralRadiance longName: "IASI Spectral Radiance" units: "W m-2 sr-1" diff --git a/src/swell/configuration/jedi/observation_ioda_names.yaml b/src/swell/configuration/jedi/observation_ioda_names.yaml index 2645fdce0..c66b759fa 100644 --- a/src/swell/configuration/jedi/observation_ioda_names.yaml +++ b/src/swell/configuration/jedi/observation_ioda_names.yaml @@ -88,10 +88,10 @@ ioda instrument names: inst type: radiance - ioda name: iasi_metop-b full name: IASI (METOP-B) - inst type: radiance + inst type: spectralRadiance - ioda name: iasi_metop-c full name: IASI (METOP-C) - inst type: radiance + inst type: spectralRadiance - ioda name: mhs_metop-a full name: MHS (METOP-A) inst type: radiance From 0622ee1593eaed15885a9c660c9bf1dc55c51da7 Mon Sep 17 00:00:00 2001 From: Ricardo Todling Date: Thu, 11 Dec 2025 14:27:17 -0500 Subject: [PATCH 2/2] pre-filter added to work together with var when IODA from bufr-converter --- .../observations/pre-filters/amsua_aqua.yaml | 23 +++++ .../pre-filters/amsua_metop-b.yaml | 23 +++++ .../pre-filters/amsua_metop-c.yaml | 23 +++++ .../observations/pre-filters/amsua_n15.yaml | 23 +++++ .../observations/pre-filters/amsua_n18.yaml | 23 +++++ .../observations/pre-filters/amsua_n19.yaml | 23 +++++ .../pre-filters/iasi_metop-b.yaml | 94 +++++++++++++++++++ .../pre-filters/iasi_metop-c.yaml | 94 +++++++++++++++++++ 8 files changed, 326 insertions(+) create mode 100644 src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/pre-filters/amsua_aqua.yaml create mode 100644 src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/pre-filters/amsua_metop-b.yaml create mode 100644 src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/pre-filters/amsua_metop-c.yaml create mode 100644 src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/pre-filters/amsua_n15.yaml create mode 100644 src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/pre-filters/amsua_n18.yaml create mode 100644 src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/pre-filters/amsua_n19.yaml create mode 100644 src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/pre-filters/iasi_metop-b.yaml create mode 100644 src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/pre-filters/iasi_metop-c.yaml diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/pre-filters/amsua_aqua.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/pre-filters/amsua_aqua.yaml new file mode 100644 index 000000000..37ecb9f8e --- /dev/null +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/pre-filters/amsua_aqua.yaml @@ -0,0 +1,23 @@ + obs pre filters: + # Remove Observations from the Edge of the Scan + - filter: Domain Check + filter variables: + - name: brightnessTemperature + channels: 1-15 + where: + - variable: + name: MetaData/sensorScanPosition + is_in: 4-27 + action: + name: reduce obs space + + # Data Thinning + - filter: Gaussian Thinning + horizontal_mesh: 145 + use_reduced_horizontal_grid: true + distance_norm: geodesic + # round_horizontal_bin_count_to_nearest: true + # partition_longitude_bins_using_mesh: true + action: + name: reduce obs space + diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/pre-filters/amsua_metop-b.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/pre-filters/amsua_metop-b.yaml new file mode 100644 index 000000000..37ecb9f8e --- /dev/null +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/pre-filters/amsua_metop-b.yaml @@ -0,0 +1,23 @@ + obs pre filters: + # Remove Observations from the Edge of the Scan + - filter: Domain Check + filter variables: + - name: brightnessTemperature + channels: 1-15 + where: + - variable: + name: MetaData/sensorScanPosition + is_in: 4-27 + action: + name: reduce obs space + + # Data Thinning + - filter: Gaussian Thinning + horizontal_mesh: 145 + use_reduced_horizontal_grid: true + distance_norm: geodesic + # round_horizontal_bin_count_to_nearest: true + # partition_longitude_bins_using_mesh: true + action: + name: reduce obs space + diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/pre-filters/amsua_metop-c.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/pre-filters/amsua_metop-c.yaml new file mode 100644 index 000000000..37ecb9f8e --- /dev/null +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/pre-filters/amsua_metop-c.yaml @@ -0,0 +1,23 @@ + obs pre filters: + # Remove Observations from the Edge of the Scan + - filter: Domain Check + filter variables: + - name: brightnessTemperature + channels: 1-15 + where: + - variable: + name: MetaData/sensorScanPosition + is_in: 4-27 + action: + name: reduce obs space + + # Data Thinning + - filter: Gaussian Thinning + horizontal_mesh: 145 + use_reduced_horizontal_grid: true + distance_norm: geodesic + # round_horizontal_bin_count_to_nearest: true + # partition_longitude_bins_using_mesh: true + action: + name: reduce obs space + diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/pre-filters/amsua_n15.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/pre-filters/amsua_n15.yaml new file mode 100644 index 000000000..37ecb9f8e --- /dev/null +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/pre-filters/amsua_n15.yaml @@ -0,0 +1,23 @@ + obs pre filters: + # Remove Observations from the Edge of the Scan + - filter: Domain Check + filter variables: + - name: brightnessTemperature + channels: 1-15 + where: + - variable: + name: MetaData/sensorScanPosition + is_in: 4-27 + action: + name: reduce obs space + + # Data Thinning + - filter: Gaussian Thinning + horizontal_mesh: 145 + use_reduced_horizontal_grid: true + distance_norm: geodesic + # round_horizontal_bin_count_to_nearest: true + # partition_longitude_bins_using_mesh: true + action: + name: reduce obs space + diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/pre-filters/amsua_n18.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/pre-filters/amsua_n18.yaml new file mode 100644 index 000000000..37ecb9f8e --- /dev/null +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/pre-filters/amsua_n18.yaml @@ -0,0 +1,23 @@ + obs pre filters: + # Remove Observations from the Edge of the Scan + - filter: Domain Check + filter variables: + - name: brightnessTemperature + channels: 1-15 + where: + - variable: + name: MetaData/sensorScanPosition + is_in: 4-27 + action: + name: reduce obs space + + # Data Thinning + - filter: Gaussian Thinning + horizontal_mesh: 145 + use_reduced_horizontal_grid: true + distance_norm: geodesic + # round_horizontal_bin_count_to_nearest: true + # partition_longitude_bins_using_mesh: true + action: + name: reduce obs space + diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/pre-filters/amsua_n19.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/pre-filters/amsua_n19.yaml new file mode 100644 index 000000000..37ecb9f8e --- /dev/null +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/pre-filters/amsua_n19.yaml @@ -0,0 +1,23 @@ + obs pre filters: + # Remove Observations from the Edge of the Scan + - filter: Domain Check + filter variables: + - name: brightnessTemperature + channels: 1-15 + where: + - variable: + name: MetaData/sensorScanPosition + is_in: 4-27 + action: + name: reduce obs space + + # Data Thinning + - filter: Gaussian Thinning + horizontal_mesh: 145 + use_reduced_horizontal_grid: true + distance_norm: geodesic + # round_horizontal_bin_count_to_nearest: true + # partition_longitude_bins_using_mesh: true + action: + name: reduce obs space + diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/pre-filters/iasi_metop-b.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/pre-filters/iasi_metop-b.yaml new file mode 100644 index 000000000..bae4d4a9c --- /dev/null +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/pre-filters/iasi_metop-b.yaml @@ -0,0 +1,94 @@ + # Observation Pre Filters (QC) + # ---------------------------- + obs pre filters: + # Step 1: Create Derived Variables + # Assign channel wavenumbers in m-1 + - filter: Variable Assignment + assignments: + - name: MetaData/sensorCentralWavenumber + type: float + channels: *iasi_metop-b_channels + function: + name: ObsFunction/LinearCombination + options: + variables: + - name: ObsValue/spectralRadiance + channels: *iasi_metop-b_channels + coefs: [25.0] + intercept: 64475.0 + use channel numbers: true + + ### Do not need this in the present setting + ### Scaled radiance to spectral radiance + ### creates a spectral radiance in (W / (m^2.sr.m^-1)) +# - filter: Variable Transforms +# Transform: SatRadianceFromScaledRadiance +# transform from: +# name: ObsValue/spectralRadiance +# channels: *iasi_metop-b_channels +# number of scale factors: 1 +# scale factor variable: MetaData/channelScaleFactor +# scale factor start: MetaData/sensorChannelStart +# scale factor end: MetaData/sensorChannelEnd +# get scaling factors from multiple arrays: true + + # Step 2: Transform radiance to brightness temperature + - filter: Variable Transforms + Transform: SatBrightnessTempFromRad + transform from: + name: ObsValue/spectralRadiance + channels: *iasi_metop-b_channels + spectral variable: + name: MetaData/sensorCentralWavenumber + channels: *iasi_metop-b_channels + radiance units: wavenumber + planck1: 1.191042953e-16 + planck2: 1.4387774e-2 + + # Step 3: Assign Observation Error + - filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *iasi_metop-b_channels + action: + name: assign error + error parameter vector: *iasi_metop-b_obserr + + # Step 4: Remove Observations from the Edge of the Scan + - filter: Domain Check + filter variables: + - name: brightnessTemperature + channels: *iasi_metop-b_channels + where: + - variable: + name: MetaData/sensorScanPosition + is_in: 5-56 + action: + name: reduce obs space + + # Step 5: Create cloudFree variable (for thinning) + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/fractionOfClearPixelsInFOV + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: MetaData/fractionOfClearPixelsInFOV + coefs: [1.0] + + # Step 6: Data Thinning + - filter: Gaussian Thinning + filter variables: + - name: brightnessTemperature + channels: *iasi_metop-b_channels +# GSI value leads to consirably more obs than GSI uses! +# horizontal_mesh: 145 + horizontal_mesh: 175 + use_reduced_horizontal_grid: true + distance_norm: geodesic + priority_variable: DerivedMetaData/fractionOfClearPixelsInFOV + action: + name: reduce obs space + diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/pre-filters/iasi_metop-c.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/pre-filters/iasi_metop-c.yaml new file mode 100644 index 000000000..7f9d2a628 --- /dev/null +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/pre-filters/iasi_metop-c.yaml @@ -0,0 +1,94 @@ + # Observation Pre Filters (QC) + # ---------------------------- + obs pre filters: + # Step 1: Create Derived Variables + # Assign channel wavenumbers in m-1 + - filter: Variable Assignment + assignments: + - name: MetaData/sensorCentralWavenumber + type: float + channels: *iasi_metop-c_channels + function: + name: ObsFunction/LinearCombination + options: + variables: + - name: ObsValue/spectralRadiance + channels: *iasi_metop-c_channels + coefs: [25.0] + intercept: 64475.0 + use channel numbers: true + + ### Do not need this in the present setting + ### Scaled radiance to spectral radiance + ### creates a spectral radiance in (W / (m^2.sr.m^-1)) +# - filter: Variable Transforms +# Transform: SatRadianceFromScaledRadiance +# transform from: +# name: ObsValue/spectralRadiance +# channels: *iasi_metop-c_channels +# number of scale factors: 1 +# scale factor variable: MetaData/channelScaleFactor +# scale factor start: MetaData/sensorChannelStart +# scale factor end: MetaData/sensorChannelEnd +# get scaling factors from multiple arrays: true + + # Step 2: Transform radiance to brightness temperature + - filter: Variable Transforms + Transform: SatBrightnessTempFromRad + transform from: + name: ObsValue/spectralRadiance + channels: *iasi_metop-c_channels + spectral variable: + name: MetaData/sensorCentralWavenumber + channels: *iasi_metop-c_channels + radiance units: wavenumber + planck1: 1.191042953e-16 + planck2: 1.4387774e-2 + + # Step 3: Assign Observation Error + - filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *iasi_metop-c_channels + action: + name: assign error + error parameter vector: *iasi_metop-c_obserr + + # Step 4: Remove Observations from the Edge of the Scan + - filter: Domain Check + filter variables: + - name: brightnessTemperature + channels: *iasi_metop-c_channels + where: + - variable: + name: MetaData/sensorScanPosition + is_in: 5-56 + action: + name: reduce obs space + + # Step 5: Create cloudFree variable (for thinning) + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/fractionOfClearPixelsInFOV + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: MetaData/fractionOfClearPixelsInFOV + coefs: [1.0] + + # Step 6: Data Thinning + - filter: Gaussian Thinning + filter variables: + - name: brightnessTemperature + channels: *iasi_metop-c_channels +# GSI value leads to consirably more obs than GSI uses! +# horizontal_mesh: 145 + horizontal_mesh: 175 + use_reduced_horizontal_grid: true + distance_norm: geodesic + priority_variable: DerivedMetaData/fractionOfClearPixelsInFOV + action: + name: reduce obs space +