From 2c71a0be3c508013e5a12ea5c22977d5ee2e6771 Mon Sep 17 00:00:00 2001 From: gmao-msienkie Date: Wed, 3 Sep 2025 16:02:36 -0400 Subject: [PATCH 1/9] try changes for 'tcp' --- .../jedi/interfaces/geos_atmosphere/task_questions.yaml | 2 ++ src/swell/configuration/jedi/observation_ioda_names.yaml | 3 +++ src/swell/suites/3dfgat_atmos/suite_config.py | 3 ++- src/swell/suites/3dvar_atmos/suite_config.py | 3 ++- src/swell/suites/convert_ncdiags/suite_config.py | 3 ++- src/swell/suites/hofx/suite_config.py | 1 + src/swell/suites/localensembleda/suite_config.py | 1 + src/swell/suites/ufo_testing/suite_config.py | 3 ++- 8 files changed, 15 insertions(+), 4 deletions(-) diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/task_questions.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/task_questions.yaml index 67c657dbc..17890c547 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/task_questions.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/task_questions.yaml @@ -200,6 +200,7 @@ observations: - sfcship - sfc - sondes + - tcp - airs_aqua - amsr2_gcom-w1 - amsua_aqua @@ -234,6 +235,7 @@ observations: - sfcship - sfc - sondes + - tcp - airs_aqua - amsr2_gcom-w1 - amsua_aqua diff --git a/src/swell/configuration/jedi/observation_ioda_names.yaml b/src/swell/configuration/jedi/observation_ioda_names.yaml index 2645fdce0..d36ba2081 100644 --- a/src/swell/configuration/jedi/observation_ioda_names.yaml +++ b/src/swell/configuration/jedi/observation_ioda_names.yaml @@ -149,6 +149,9 @@ ioda instrument names: - ioda name: ssmis_f17 full name: SSMIS (DMSP-F17) inst type: radiance + - ioda name: tcp + full name: Tropical Cyclone central pressure + inst type: conventional - ioda name: vadwind full name: Velocity Azimuth Display Wind Profile inst type: conventional diff --git a/src/swell/suites/3dfgat_atmos/suite_config.py b/src/swell/suites/3dfgat_atmos/suite_config.py index f5302ba24..686d1ee01 100644 --- a/src/swell/suites/3dfgat_atmos/suite_config.py +++ b/src/swell/suites/3dfgat_atmos/suite_config.py @@ -76,7 +76,8 @@ class SuiteConfig(QuestionContainer, Enum): "sfcship", "sfc", "sondes", - "ssmis_f17" + "ssmis_f17", + "tcp" ]), qd.gradient_norm_reduction("1e-3"), qd.number_of_iterations([10]), diff --git a/src/swell/suites/3dvar_atmos/suite_config.py b/src/swell/suites/3dvar_atmos/suite_config.py index d32755f4a..b20732990 100644 --- a/src/swell/suites/3dvar_atmos/suite_config.py +++ b/src/swell/suites/3dvar_atmos/suite_config.py @@ -81,7 +81,8 @@ class SuiteConfig(QuestionContainer, Enum): "sfcship", "sfc", "sondes", - "ssmis_f17" + "ssmis_f17", + "tcp" ]), qd.clean_patterns(['*.txt', '*.csv']), ] diff --git a/src/swell/suites/convert_ncdiags/suite_config.py b/src/swell/suites/convert_ncdiags/suite_config.py index c8669546c..5b766c81c 100644 --- a/src/swell/suites/convert_ncdiags/suite_config.py +++ b/src/swell/suites/convert_ncdiags/suite_config.py @@ -76,7 +76,8 @@ class SuiteConfig(QuestionContainer, Enum): "sfcship", "sfc", "sondes", - "ssmis_f17" + "ssmis_f17", + "tcp" ]), qd.path_to_gsi_nc_diags("/discover/nobackup/projects/gmao/advda/SwellTestData/" "ufo_testing/ncdiagv2/%Y%m%d%H"), diff --git a/src/swell/suites/hofx/suite_config.py b/src/swell/suites/hofx/suite_config.py index 90723ef84..9d0cb0e72 100644 --- a/src/swell/suites/hofx/suite_config.py +++ b/src/swell/suites/hofx/suite_config.py @@ -71,6 +71,7 @@ class SuiteConfig(QuestionContainer, Enum): "sfc", "sondes", "ssmis_f17" + "tcp" ]), qd.clean_patterns([]), ] diff --git a/src/swell/suites/localensembleda/suite_config.py b/src/swell/suites/localensembleda/suite_config.py index d4a53db43..0d2176f7f 100644 --- a/src/swell/suites/localensembleda/suite_config.py +++ b/src/swell/suites/localensembleda/suite_config.py @@ -70,6 +70,7 @@ class SuiteConfig(QuestionContainer, Enum): "scatwind", "sfcship", "sfc", + "tcp", "mhs_metop-b", "mhs_metop-c", "mhs_n19", diff --git a/src/swell/suites/ufo_testing/suite_config.py b/src/swell/suites/ufo_testing/suite_config.py index ab10cc015..bd60d9629 100644 --- a/src/swell/suites/ufo_testing/suite_config.py +++ b/src/swell/suites/ufo_testing/suite_config.py @@ -61,7 +61,8 @@ class SuiteConfig(QuestionContainer, Enum): "sfc", "sfcship", "sondes", - "ssmis_f17" + "ssmis_f17", + "tcp" ]), qd.produce_geovals(False), qd.clean_patterns([ From c708bd5dc59911673ceff83b6b8136116815533b Mon Sep 17 00:00:00 2001 From: gmao-msienkie Date: Wed, 3 Sep 2025 17:04:05 -0400 Subject: [PATCH 2/9] add missing comma --- src/swell/suites/hofx/suite_config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/swell/suites/hofx/suite_config.py b/src/swell/suites/hofx/suite_config.py index 9d0cb0e72..71d43a27d 100644 --- a/src/swell/suites/hofx/suite_config.py +++ b/src/swell/suites/hofx/suite_config.py @@ -70,7 +70,7 @@ class SuiteConfig(QuestionContainer, Enum): "sfcship", "sfc", "sondes", - "ssmis_f17" + "ssmis_f17", "tcp" ]), qd.clean_patterns([]), From ea5f328c001823d683369f872fd7671ab87c3fcc Mon Sep 17 00:00:00 2001 From: gmao-msienkie Date: Wed, 10 Sep 2025 10:24:13 -0400 Subject: [PATCH 3/9] initial try for tcp configuration copied and modified from sfcship configuration --- .../geos_atmosphere/observations/tcp.yaml | 160 ++++++++++++++++++ 1 file changed, 160 insertions(+) create mode 100644 src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/tcp.yaml diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/tcp.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/tcp.yaml new file mode 100644 index 000000000..b9de1160c --- /dev/null +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/tcp.yaml @@ -0,0 +1,160 @@ +obs space: + name: TC vital storm center pressure + obsdatain: + engine: + type: H5File + obsfile: '{{cycle_dir}}/tcp.{{window_begin}}.nc4' + obsdataout: + engine: + type: H5File + obsfile: '{{cycle_dir}}/{{experiment_id}}.tcp.{{window_begin}}.nc4' + simulated variables: [stationPressure] + +obs operator: + # Surface pressure is handled using the correction scheme + name: SfcPCorrected + variables: + - name: stationPressure + da_psfc_scheme: GSI + geovar_geomz: geopotential_height + geovar_sfc_geomz: height_above_mean_sea_level_at_surface + station_altitude: height + + +obs prior filters: + +# Apply variable changes needed for rescaled height coordinate +# ------------------------------------------------------------ +- filter: Variable Transforms + Transform: AdjustedHeightCoordinate + SkipWhenNoObs: False + +obs post filters: + +# ------------------ +# stationPressure (surface pressure) +# ------------------ +# PreUseFlag is assigned in reading programs in GSI. (filled in as '1') +- filter: Perform Action + filter variables: + - name: stationPressure + where: + - variable: PreUseFlag/stationPressure + # PreUseFlag should =< ijter ( # of outloop(0,1,2) + 1) in GSI + minvalue: 1 + action: + name: reject +# PrepBUFR check to be supplemented later (filled in as '1') +- filter: RejectList + where: + - variable: + name: PreQC/stationPressure + is_in: 4-15 +# error assignments +# Original error is not from error table +# error inflation based on Ps correction +# tcp_refps=1000.0_r_kind +# tcp_width=50.0_r_kind +# tcp_ermin=0.75_r_kind +# tcp_ermax=5.0_r_kind +# psob=stormpsmin(i) ! in mb +# psdif=tcp_refps-psob ! in mb +# alpha=max(min(psdif/tcp_width,one),zero) +# oberr=tcp_ermin+(tcp_ermax-tcp_ermin)*alpha + +- filter: Perform Action + filter variables: + - name: stationPressure + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorSfcPressure + options: + geovar_geomz: geopotential_height + geovar_sfc_geomz: height_above_mean_sea_level_at_surface + station_altitude: height +# assign TempObsErrorData/stationPressure <--- ObsErrorData before changing its Min and Max +- filter: Variable Assignment + assignments: + - name: TempObsErrorData/stationPressure + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/stationPressure + defer to post: true +# tcp 112 0 1 3.0 0 0 0 75.0 5.0 1.0 75.0 0.000000 0 0. 0. 0 0. 0. +# set ObsErrorData 100 Pa if it < 100 Pa. +- filter: Perform Action + filter variables: + - name: stationPressure + action: + name: assign error + error parameter: 100 + where: + - variable: + name: TempObsErrorData/stationPressure + maxvalue: 100 + - variable: + name: TempObsErrorData/stationPressure + value: is_valid + defer to post: true +# set ObsErrorData 500 Pa if it > 500 Pa. +- filter: Perform Action + filter variables: + - name: stationPressure + action: + name: assign error + error parameter: 500 + where: + - variable: + name: TempObsErrorData/stationPressure + minvalue: 500 + - variable: + name: TempObsErrorData/stationPressure + value: is_valid + defer to post: true +# Background check +# background check for obstype 112 +# threshold 75.0 if PreQC /= 3 +- filter: Background Check + filter variables: + - name: stationPressure + threshold: 75.0 + action: + name: reject + where: + - variable: PreQC/stationPressure + is_not_in: [3] + - variable: ObsType/stationPressure + is_in: 180,183 + defer to post: true +#?? threshold * 0.7 if PreQC=3 +#- filter: Background Check +# filter variables: +# - name: stationPressure +# threshold: 2.8 +# action: +# name: reject +# where: +# - variable: PreQC/stationPressure +# is_in: [3] +# - variable: ObsType/stationPressure +# is_in: 180,183 +# defer to post: true +# Re-assign error ObsErrorData/stationPressure <--- TempObsErrorData/stationPressure +- filter: Perform Action + filter variables: + - name: stationPressure + action: + name: assign error + error function: TempObsErrorData/stationPressure + where: + - variable: + name: TempObsErrorData/stationPressure + value: is_valid + defer to post: true +# # No duplicate check + + From f206ce4bb5d01cc0fecfce5e2ff5a3dacfa95978 Mon Sep 17 00:00:00 2001 From: gmao-msienkie Date: Wed, 15 Oct 2025 18:03:31 -0400 Subject: [PATCH 4/9] TCVITALS configuration - prior to SfcPCorrected -> SfcCorrected --- .../geos_atmosphere/observations/tcp.yaml | 83 +++++-------------- 1 file changed, 23 insertions(+), 60 deletions(-) diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/tcp.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/tcp.yaml index b9de1160c..4cff38eb6 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/tcp.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/tcp.yaml @@ -11,14 +11,17 @@ obs space: simulated variables: [stationPressure] obs operator: + name: Composite + components: + # Surface pressure is handled using the correction scheme - name: SfcPCorrected - variables: - - name: stationPressure - da_psfc_scheme: GSI - geovar_geomz: geopotential_height - geovar_sfc_geomz: height_above_mean_sea_level_at_surface - station_altitude: height + - name: SfcPCorrected + variables: + - name: stationPressure + da_psfc_scheme: GSI + geovar_geomz: geopotential_height + geovar_sfc_geomz: height_above_mean_sea_level_at_surface + station_altitude: height obs prior filters: @@ -29,38 +32,18 @@ obs prior filters: Transform: AdjustedHeightCoordinate SkipWhenNoObs: False -obs post filters: - -# ------------------ -# stationPressure (surface pressure) -# ------------------ -# PreUseFlag is assigned in reading programs in GSI. (filled in as '1') -- filter: Perform Action - filter variables: - - name: stationPressure - where: - - variable: PreUseFlag/stationPressure - # PreUseFlag should =< ijter ( # of outloop(0,1,2) + 1) in GSI - minvalue: 1 - action: - name: reject -# PrepBUFR check to be supplemented later (filled in as '1') -- filter: RejectList +# limit to obs with pressures in valid range +- filter: Domain Check where: - variable: - name: PreQC/stationPressure - is_in: 4-15 -# error assignments -# Original error is not from error table -# error inflation based on Ps correction -# tcp_refps=1000.0_r_kind -# tcp_width=50.0_r_kind -# tcp_ermin=0.75_r_kind -# tcp_ermax=5.0_r_kind -# psob=stormpsmin(i) ! in mb -# psdif=tcp_refps-psob ! in mb -# alpha=max(min(psdif/tcp_width,one),zero) -# oberr=tcp_ermin+(tcp_ermax-tcp_ermin)*alpha + name: MetaData/pressure + minvalue: 85000 + maxvalue: 102500 + +obs post filters: + +#ps 180 1 1 3.0 0 0 0 4.0 3.0 1.0 4.0 0.000300 0 0. 0. 0 0. 0. +#tcp 112 0 1 3.0 0 0 0 75.0 5.0 1.0 75.0 0.000000 0 0. 0. 0 0. 0. - filter: Perform Action filter variables: @@ -84,7 +67,6 @@ obs post filters: variables: - name: ObsErrorData/stationPressure defer to post: true -# tcp 112 0 1 3.0 0 0 0 75.0 5.0 1.0 75.0 0.000000 0 0. 0. 0 0. 0. # set ObsErrorData 100 Pa if it < 100 Pa. - filter: Perform Action filter variables: @@ -115,34 +97,16 @@ obs post filters: name: TempObsErrorData/stationPressure value: is_valid defer to post: true -# Background check -# background check for obstype 112 -# threshold 75.0 if PreQC /= 3 + +# Background check - filter: Background Check filter variables: - name: stationPressure threshold: 75.0 action: name: reject - where: - - variable: PreQC/stationPressure - is_not_in: [3] - - variable: ObsType/stationPressure - is_in: 180,183 defer to post: true -#?? threshold * 0.7 if PreQC=3 -#- filter: Background Check -# filter variables: -# - name: stationPressure -# threshold: 2.8 -# action: -# name: reject -# where: -# - variable: PreQC/stationPressure -# is_in: [3] -# - variable: ObsType/stationPressure -# is_in: 180,183 -# defer to post: true + # Re-assign error ObsErrorData/stationPressure <--- TempObsErrorData/stationPressure - filter: Perform Action filter variables: @@ -155,6 +119,5 @@ obs post filters: name: TempObsErrorData/stationPressure value: is_valid defer to post: true -# # No duplicate check From dbcee3be9bf55483a4479e31d938a2a0cf494920 Mon Sep 17 00:00:00 2001 From: gmao-msienkie Date: Wed, 15 Oct 2025 18:05:16 -0400 Subject: [PATCH 5/9] Add tcvitals observations --- src/swell/suites/eva_capabilities/suite_config.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/swell/suites/eva_capabilities/suite_config.py b/src/swell/suites/eva_capabilities/suite_config.py index fc892cac4..3dc8f5824 100644 --- a/src/swell/suites/eva_capabilities/suite_config.py +++ b/src/swell/suites/eva_capabilities/suite_config.py @@ -96,7 +96,8 @@ class SuiteConfig(QuestionContainer, Enum): "sfcship", "sfc", "sondes", - "ssmis_f17" + "ssmis_f17", + "tcp" ]), qd.ncdiag_experiments(['x0050_fgat']), qd.clean_patterns(['*.txt', '*.csv']), From 35c85d6acf611bcc2fd7878d37fe10bcdaea64fd Mon Sep 17 00:00:00 2001 From: gmao-msienkie Date: Wed, 15 Oct 2025 18:29:36 -0400 Subject: [PATCH 6/9] change 'background' to 'observation' --- src/swell/tasks/get_obs_not_in_r2d2.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/swell/tasks/get_obs_not_in_r2d2.py b/src/swell/tasks/get_obs_not_in_r2d2.py index 6ad2e21e2..5c2ccd5de 100644 --- a/src/swell/tasks/get_obs_not_in_r2d2.py +++ b/src/swell/tasks/get_obs_not_in_r2d2.py @@ -26,7 +26,7 @@ def execute(self) -> None: # -------------------- cycle_date = self.__datetime__.string_directory() - # Get the path and pattern for the background files + # Get the path and pattern for the observation files # ------------------------------------------------- existing_path = self.config.ioda_locations_not_in_r2d2() @@ -50,8 +50,8 @@ def execute(self) -> None: # Assert that some files were found # --------------------------------- - self.logger.assert_abort(len(existing_path_files) > 0, f'No background ' - 'files matching cycle in background directory.') + self.logger.assert_abort(len(existing_path_files) > 0, f'No observation ' + 'files matching cycle in observation directory.') # Loop over all the files # ----------------------- From 0dd08e40064f7411cfa79c2403d202c8b4377e04 Mon Sep 17 00:00:00 2001 From: gmao-msienkie Date: Thu, 16 Oct 2025 02:16:02 -0400 Subject: [PATCH 7/9] Looks like new SfcCorrected operator is in the current build Update tcp.yaml to use SfcCorrected instead of SfcPCorrected --- .../geos_atmosphere/observations/tcp.yaml | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/tcp.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/tcp.yaml index 4cff38eb6..74308afa6 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/tcp.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/tcp.yaml @@ -10,18 +10,16 @@ obs space: obsfile: '{{cycle_dir}}/{{experiment_id}}.tcp.{{window_begin}}.nc4' simulated variables: [stationPressure] +# Surface pressure is handled using the correction scheme obs operator: - name: Composite - components: - # Surface pressure is handled using the correction scheme - - name: SfcPCorrected - variables: - - name: stationPressure - da_psfc_scheme: GSI - geovar_geomz: geopotential_height - geovar_sfc_geomz: height_above_mean_sea_level_at_surface - station_altitude: height + name: SfcCorrected + variables: + - name: stationPressure + correction scheme to use: GSL + geovar_geomz: geopotential_height + geovar_sfc_geomz: height_above_mean_sea_level_at_surface + station_altitude: height obs prior filters: From 733846c3dcb2435f1cb2b34d02e1597ff803e7ec Mon Sep 17 00:00:00 2001 From: gmao-msienkie Date: Thu, 16 Oct 2025 16:06:32 -0400 Subject: [PATCH 8/9] add linear operator for station pressure --- .../interfaces/geos_atmosphere/observations/tcp.yaml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/tcp.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/tcp.yaml index 74308afa6..cd15fbf96 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/tcp.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/tcp.yaml @@ -10,9 +10,9 @@ obs space: obsfile: '{{cycle_dir}}/{{experiment_id}}.tcp.{{window_begin}}.nc4' simulated variables: [stationPressure] -# Surface pressure is handled using the correction scheme obs operator: + # Surface pressure is handled using the correction scheme name: SfcCorrected variables: - name: stationPressure @@ -21,6 +21,13 @@ obs operator: geovar_sfc_geomz: height_above_mean_sea_level_at_surface station_altitude: height +linear obs operator: + + # station pressure is using the lowest model level (identity) + name: Identity + variables: + - name: stationPressure + obs prior filters: From 7d3b9ecdc5f099090c2a4512e2dd9f3cc7506500 Mon Sep 17 00:00:00 2001 From: gmao-msienkie Date: Wed, 17 Dec 2025 15:10:07 -0500 Subject: [PATCH 9/9] remove incorrect code in obs operator - didn't work anyway --- .../jedi/interfaces/geos_atmosphere/observations/tcp.yaml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/tcp.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/tcp.yaml index cd15fbf96..ad85ccc99 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/tcp.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/tcp.yaml @@ -21,14 +21,6 @@ obs operator: geovar_sfc_geomz: height_above_mean_sea_level_at_surface station_altitude: height -linear obs operator: - - # station pressure is using the lowest model level (identity) - name: Identity - variables: - - name: stationPressure - - obs prior filters: # Apply variable changes needed for rescaled height coordinate