From 04200ef73272669a653401ba14cdc4f5ac8436c0 Mon Sep 17 00:00:00 2001 From: Michael Levy Date: Wed, 20 Nov 2024 15:28:17 -0700 Subject: [PATCH 01/10] Bring ADF example into key_metrics This configuration does not work as expected -- there are no plots generated. I think it might be due to the fact that 092 is output on the native grid (atm/hist/regrid/ has history files on the FV grid if we can point ADF to it) --- examples/.gitignore | 2 +- examples/key_metrics/config.yml | 12 ++++++++++++ helper_scripts/.gitignore | 1 + 3 files changed, 14 insertions(+), 1 deletion(-) create mode 100644 helper_scripts/.gitignore diff --git a/examples/.gitignore b/examples/.gitignore index d4c29b6b..ad4ae93b 100644 --- a/examples/.gitignore +++ b/examples/.gitignore @@ -1 +1 @@ -externals +*/ADF_output diff --git a/examples/key_metrics/config.yml b/examples/key_metrics/config.yml index 55dabbbd..b9e903f7 100644 --- a/examples/key_metrics/config.yml +++ b/examples/key_metrics/config.yml @@ -120,6 +120,17 @@ compute_notebooks: regridded_output: False # it looks like output is already on f09 grid, didn't need to regrid time-series file base_regridded_output: True validation_path: '/glade/campaign/cesm/development/cross-wg/diagnostic_framework/nmse_validation/fv0.9x1.25' + link_to_ADF: + kernel_name: cupid-dev + parameter_groups: + none: + adf_root: ../../external_diag_packages/ADF/ + case_year_range: "1_100" + base_case_year_range: "1_100" + external_tool: + tool_name: 'ADF' + vars: ['PRECT', 'SST'] + plotting_scripts: ["global_latlon_map", "global_latlon_vect_map"] glc: Greenland_SMB_visual_compare_obs: @@ -189,6 +200,7 @@ book_toc: - caption: Atmosphere chapters: - file: atm/Global_PSL_NMSE_compare_obs_lens + - file: atm/link_to_ADF # - caption: Ocean # chapters: diff --git a/helper_scripts/.gitignore b/helper_scripts/.gitignore new file mode 100644 index 00000000..1cda54be --- /dev/null +++ b/helper_scripts/.gitignore @@ -0,0 +1 @@ +*.yml From b45ae1fdb7ce109acd59b20a74c7942bb6ea6dde Mon Sep 17 00:00:00 2001 From: Michael Levy Date: Thu, 21 Nov 2024 11:56:42 -0700 Subject: [PATCH 02/10] Add link_to_ADF.ipynb to key_metrics Future PRs will update examples/key_metrics/config.yml to change the ADF configuration --- examples/external_diag_packages/config.yml | 2 +- examples/key_metrics/config.yml | 3 +- helper_scripts/generate_adf_config_file.py | 37 ++++++++++++++++++++++ 3 files changed, 40 insertions(+), 2 deletions(-) diff --git a/examples/external_diag_packages/config.yml b/examples/external_diag_packages/config.yml index a9d7327b..85bf42ee 100644 --- a/examples/external_diag_packages/config.yml +++ b/examples/external_diag_packages/config.yml @@ -118,7 +118,7 @@ compute_notebooks: kernel_name: cupid-dev parameter_groups: none: - adf_root: ../../external_diag_packages/ADF/ + adf_root: ../../external_diag_packages/ADF_output/ case_year_range: "1_100" base_case_year_range: "1995_2006" external_tool: diff --git a/examples/key_metrics/config.yml b/examples/key_metrics/config.yml index b9e903f7..a9ce9f74 100644 --- a/examples/key_metrics/config.yml +++ b/examples/key_metrics/config.yml @@ -124,13 +124,14 @@ compute_notebooks: kernel_name: cupid-dev parameter_groups: none: - adf_root: ../../external_diag_packages/ADF/ + adf_root: ../../key_metrics/ADF_output/ case_year_range: "1_100" base_case_year_range: "1_100" external_tool: tool_name: 'ADF' vars: ['PRECT', 'SST'] plotting_scripts: ["global_latlon_map", "global_latlon_vect_map"] + base_regridded_output: True glc: Greenland_SMB_visual_compare_obs: diff --git a/helper_scripts/generate_adf_config_file.py b/helper_scripts/generate_adf_config_file.py index 810ebf97..8a00cb2d 100755 --- a/helper_scripts/generate_adf_config_file.py +++ b/helper_scripts/generate_adf_config_file.py @@ -108,6 +108,25 @@ def generate_adf_config(cesm_root, cupid_example, adf_file, out_file): "proc", "climo", ) + # UPDATE PATHS FOR REGRIDDED DATA + try: + if c_dict["compute_notebooks"]["atm"]["link_to_ADF"]["external_tool"][ + "regridded_output" + ]: + a_dict["diag_cam_climo"]["cam_hist_loc"] = os.path.join( + a_dict["diag_cam_climo"]["cam_hist_loc"], + "regrid", + ) + a_dict["diag_cam_climo"]["cam_ts_loc"] = os.path.join( + a_dict["diag_cam_climo"]["cam_ts_loc"], + "regrid", + ) + a_dict["diag_cam_climo"]["cam_climo_loc"] = os.path.join( + a_dict["diag_cam_climo"]["cam_climo_loc"], + "regrid", + ) + except: # noqa: E722 + pass # TEST CASE START / END YEARS test_case_cupid_ts_index = ( ts_case_names.index(test_case_name) if test_case_name in ts_case_names else None @@ -158,6 +177,24 @@ def generate_adf_config(cesm_root, cupid_example, adf_file, out_file): "proc", "climo", ) + try: + if c_dict["compute_notebooks"]["atm"]["link_to_ADF"]["external_tool"][ + "base_regridded_output" + ]: + a_dict["diag_cam_baseline_climo"]["cam_hist_loc"] = os.path.join( + a_dict["diag_cam_baseline_climo"]["cam_hist_loc"], + "regrid", + ) + a_dict["diag_cam_baseline_climo"]["cam_ts_loc"] = os.path.join( + a_dict["diag_cam_baseline_climo"]["cam_ts_loc"], + "regrid", + ) + a_dict["diag_cam_baseline_climo"]["cam_climo_loc"] = os.path.join( + a_dict["diag_cam_baseline_climo"]["cam_climo_loc"], + "regrid", + ) + except: # noqa: E722 + pass a_dict["diag_cam_baseline_climo"]["start_year"] = base_start_date a_dict["diag_cam_baseline_climo"]["end_year"] = base_end_date From b558fb00c1440550a5ba8c6c369be989f0d661da Mon Sep 17 00:00:00 2001 From: Michael Levy Date: Wed, 4 Dec 2024 14:11:15 -0700 Subject: [PATCH 03/10] Drop case_year_range from notebook params Can use start_date and end_date instead to be consistent with other notebooks (same for base_case_year_range using base_start_date and base_end_date) --- examples/key_metrics/config.yml | 2 - nblibrary/atm/link_to_ADF.ipynb | 68 ++++++++++++++++++++++++++++++--- 2 files changed, 62 insertions(+), 8 deletions(-) diff --git a/examples/key_metrics/config.yml b/examples/key_metrics/config.yml index b14e2dc1..c26ac7f1 100644 --- a/examples/key_metrics/config.yml +++ b/examples/key_metrics/config.yml @@ -129,8 +129,6 @@ compute_notebooks: parameter_groups: none: adf_root: ../../examples/key_metrics/ADF_output/ - case_year_range: "1_100" - base_case_year_range: "1_100" external_tool: tool_name: 'ADF' vars: ['PRECT', 'SST'] diff --git a/nblibrary/atm/link_to_ADF.ipynb b/nblibrary/atm/link_to_ADF.ipynb index 75adc481..29698f29 100644 --- a/nblibrary/atm/link_to_ADF.ipynb +++ b/nblibrary/atm/link_to_ADF.ipynb @@ -59,21 +59,77 @@ "adf_root = \".\"\n", "case_name = None\n", "base_case_name = None\n", - "case_year_range = None\n", - "base_case_year_range = None\n", + "start_date = \"\"\n", + "end_date = \"\"\n", + "base_start_date = None\n", + "base_end_date = None\n", "# adf_root will be external_diag_packages/computed_notebooks/ADF/" ] }, + { + "cell_type": "code", + "execution_count": null, + "id": "81bf4573-8806-486f-a2f6-79e244415c07", + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": [ + "hide-cell" + ] + }, + "outputs": [], + "source": [ + "# Want some base case parameter defaults to equal control case values\n", + "if base_case_name is not None:\n", + " if base_start_date is None:\n", + " base_start_date = start_date\n", + "\n", + " if base_end_date is None:\n", + " base_end_date = end_date" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4104f878-a8c3-4d0f-8c00-555e6b2b3273", + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": [ + "hide-cell" + ] + }, + "outputs": [], + "source": [ + "# convert start-date and end-date to year range\n", + "case_year_range = [int(start_date.split(\"-\")[0]), int(end_date.split(\"-\")[0]) - 1]\n", + "\n", + "base_case_year_range = [\n", + " int(base_start_date.split(\"-\")[0]),\n", + " int(base_end_date.split(\"-\")[0]) - 1,\n", + "]" + ] + }, { "cell_type": "code", "execution_count": null, "id": "ee0774cc-c68a-4385-b94c-08fa806f28c0", - "metadata": {}, + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": [ + "hide-input" + ] + }, "outputs": [], "source": [ - "adf_comparison_name = (\n", - " f\"{case_name}_{case_year_range}_vs_{base_case_name}_{base_case_year_range}\"\n", - ")\n", + "adf_comparison_name = f\"{case_name}_{case_year_range[0]}_{case_year_range[1]}_vs_{base_case_name}_{base_case_year_range[0]}_{base_case_year_range[1]}\"\n", "adf_root = os.path.join(adf_root, adf_comparison_name)\n", "display(\n", " HTML(\n", From a8bb7794d1f08d1c48f29e7af81b14ce713459b5 Mon Sep 17 00:00:00 2001 From: justin-richling <56696811+justin-richling@users.noreply.github.com> Date: Wed, 4 Dec 2024 14:15:40 -0700 Subject: [PATCH 04/10] Update config.yml Update atm key metric standard variables and plots --- examples/key_metrics/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/key_metrics/config.yml b/examples/key_metrics/config.yml index c26ac7f1..19186492 100644 --- a/examples/key_metrics/config.yml +++ b/examples/key_metrics/config.yml @@ -131,8 +131,8 @@ compute_notebooks: adf_root: ../../examples/key_metrics/ADF_output/ external_tool: tool_name: 'ADF' - vars: ['PRECT', 'SST'] - plotting_scripts: ["global_latlon_map", "global_latlon_vect_map"] + vars: ['SST', 'TS', 'SWCF', 'LWCF', 'PRECT', 'TAUX', 'TGCLDLWP'] + plotting_scripts: ["amwg_table", "global_latlon_map", "global_latlon_vect_map"] base_regridded_output: True glc: From 28d54c537abf50f72a4100a90590d4cb7fef9628 Mon Sep 17 00:00:00 2001 From: Michael Levy Date: Wed, 4 Dec 2024 14:35:55 -0700 Subject: [PATCH 05/10] Allow users to set analysis_scripts for ADF Similar to plotting_scripts, if users include it in config.yml then generate_adf_config_file.py will overwrite the default from the template --- examples/key_metrics/config.yml | 3 ++- helper_scripts/generate_adf_config_file.py | 8 ++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/examples/key_metrics/config.yml b/examples/key_metrics/config.yml index 19186492..66f11f56 100644 --- a/examples/key_metrics/config.yml +++ b/examples/key_metrics/config.yml @@ -132,7 +132,8 @@ compute_notebooks: external_tool: tool_name: 'ADF' vars: ['SST', 'TS', 'SWCF', 'LWCF', 'PRECT', 'TAUX', 'TGCLDLWP'] - plotting_scripts: ["amwg_table", "global_latlon_map", "global_latlon_vect_map"] + plotting_scripts: ["global_latlon_map", "global_latlon_vect_map"] + analysis_scripts: ["amwg_table"] base_regridded_output: True glc: diff --git a/helper_scripts/generate_adf_config_file.py b/helper_scripts/generate_adf_config_file.py index 72bedcb2..d110f181 100755 --- a/helper_scripts/generate_adf_config_file.py +++ b/helper_scripts/generate_adf_config_file.py @@ -216,6 +216,7 @@ def generate_adf_config(cesm_root, cupid_example, adf_file, out_file): a_dict["user"] = os.environ["USER"] diag_var_list = [] + analysis_scripts = [] plotting_scripts = [] for component in c_dict["compute_notebooks"]: for nb in c_dict["compute_notebooks"][component]: @@ -232,11 +233,18 @@ def generate_adf_config(cesm_root, cupid_example, adf_file, out_file): diag_var_list.append(var) for script in c_dict["compute_notebooks"][component][nb][ "external_tool" + ].get("analysis_scripts", []): + if script not in analysis_scripts: + analysis_scripts.append(script) + for script in c_dict["compute_notebooks"][component][nb][ + "external_tool" ].get("plotting_scripts", []): if script not in plotting_scripts: plotting_scripts.append(script) if diag_var_list: a_dict["diag_var_list"] = diag_var_list + if analysis_scripts: + a_dict["analysis_scripts"] = analysis_scripts if plotting_scripts: a_dict["plotting_scripts"] = plotting_scripts From 14192b2095617acc9562c2b00aa9bcc1aab9a314 Mon Sep 17 00:00:00 2001 From: Teagan King Date: Wed, 4 Dec 2024 14:37:59 -0700 Subject: [PATCH 06/10] update external diag config to match needs for link to adf notebook --- examples/external_diag_packages/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/external_diag_packages/config.yml b/examples/external_diag_packages/config.yml index 5ab4d2fe..35a5691d 100644 --- a/examples/external_diag_packages/config.yml +++ b/examples/external_diag_packages/config.yml @@ -115,12 +115,12 @@ compute_notebooks: parameter_groups: none: adf_root: ../../examples/external_diag_packages/ADF_output/ - case_year_range: "1_100" - base_case_year_range: "1995_2006" external_tool: tool_name: 'ADF' vars: ['PRECT', 'SST'] plotting_scripts: ["global_latlon_map", "global_latlon_vect_map"] + analysis_scripts: ["amwg_table"] + base_regridded_output: True # glc: # LIWG_SMB_diagnostic: From 90713d7c0ae000d9f66809e181c38028f51e9060 Mon Sep 17 00:00:00 2001 From: Michael Levy Date: Wed, 4 Dec 2024 15:06:29 -0700 Subject: [PATCH 07/10] Update to latest version of ADF Requires xesmf in output; also, ADF changes hist_str from h0a -> cam.h0a in time series generation so we do the same for consistency in file names --- environments/cupid-analysis.yml | 1 + examples/key_metrics/config.yml | 2 +- externals/ADF | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/environments/cupid-analysis.yml b/environments/cupid-analysis.yml index 3a82150d..c6f6e6f6 100644 --- a/environments/cupid-analysis.yml +++ b/environments/cupid-analysis.yml @@ -22,6 +22,7 @@ dependencies: - python==3.11.4 - uxarray - xarray + - xesmf - yaml - zarr - pip: diff --git a/examples/key_metrics/config.yml b/examples/key_metrics/config.yml index 66f11f56..6e5de2c9 100644 --- a/examples/key_metrics/config.yml +++ b/examples/key_metrics/config.yml @@ -58,7 +58,7 @@ timeseries: atm: vars: ['PSL'] derive_vars: [] - hist_str: 'h0a' + hist_str: 'cam.h0a' start_years: [1,1] end_years: [100,100] level: 'lev' diff --git a/externals/ADF b/externals/ADF index b9615225..8adaef41 160000 --- a/externals/ADF +++ b/externals/ADF @@ -1 +1 @@ -Subproject commit b96152255f7d386de34587eb8ed389f17db485b3 +Subproject commit 8adaef41dc480cc13937fdf3f712357a13ddeb84 From 02cb0ce130941cf0800474d71233d41f88d7416d Mon Sep 17 00:00:00 2001 From: Michael Levy Date: Wed, 4 Dec 2024 18:57:41 -0700 Subject: [PATCH 08/10] Add the ADF comparison table to notebook ADF stores the data for its comparison table in a CSV file, so we can import pandas and then use display(HTML(df.to_html())) to get it rendered properly in the notebook. Also cleaned up the hist_str for time series in external_diag_packages --- examples/external_diag_packages/config.yml | 2 +- nblibrary/atm/link_to_ADF.ipynb | 28 ++++++++++++++++++++-- 2 files changed, 27 insertions(+), 3 deletions(-) diff --git a/examples/external_diag_packages/config.yml b/examples/external_diag_packages/config.yml index 35a5691d..abec6580 100644 --- a/examples/external_diag_packages/config.yml +++ b/examples/external_diag_packages/config.yml @@ -58,7 +58,7 @@ timeseries: atm: vars: ['PSL'] derive_vars: [] - hist_str: 'h0a' + hist_str: 'cam.h0a' start_years: [1,1] end_years: [100,100] level: 'lev' diff --git a/nblibrary/atm/link_to_ADF.ipynb b/nblibrary/atm/link_to_ADF.ipynb index 29698f29..953e1e15 100644 --- a/nblibrary/atm/link_to_ADF.ipynb +++ b/nblibrary/atm/link_to_ADF.ipynb @@ -36,8 +36,10 @@ "outputs": [], "source": [ "import os\n", + "\n", "from IPython.core.display import HTML, Image\n", - "from IPython.display import display" + "from IPython.display import display\n", + "import pandas as pd" ] }, { @@ -144,7 +146,29 @@ "metadata": {}, "source": [ "## Key Metrics from ADF\n", - "These will eventually be put in the key_metrics example, but are a proof of concept of including individual plots from ADF directly linked in the JupyterBook." + "\n", + "Some important things to look at from ADF include a comparison table and a few maps:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a5b02451-98ad-4089-b184-75391a262db3", + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": [ + "hide-input" + ] + }, + "outputs": [], + "source": [ + "comparison_table = os.path.join(adf_root, \"amwg_table_comp.csv\")\n", + "if os.path.isfile(comparison_table):\n", + " table = pd.read_csv(comparison_table)\n", + " display(HTML(table.to_html(index=False)))" ] }, { From eff58b94d2d049d6a615711e29add202179122b4 Mon Sep 17 00:00:00 2001 From: Michael Levy Date: Wed, 4 Dec 2024 19:14:51 -0700 Subject: [PATCH 09/10] Add TAUY to ADF output This generates the wind stress vector map, which adds a map to the output in the Link To ADF Output notebook as well as producing an additional Vector Map page on the ADF website --- examples/key_metrics/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/key_metrics/config.yml b/examples/key_metrics/config.yml index 6e5de2c9..8fd88e21 100644 --- a/examples/key_metrics/config.yml +++ b/examples/key_metrics/config.yml @@ -131,7 +131,7 @@ compute_notebooks: adf_root: ../../examples/key_metrics/ADF_output/ external_tool: tool_name: 'ADF' - vars: ['SST', 'TS', 'SWCF', 'LWCF', 'PRECT', 'TAUX', 'TGCLDLWP'] + vars: ['SST', 'TS', 'SWCF', 'LWCF', 'PRECT', 'TAUX', 'TAUY', 'TGCLDLWP'] plotting_scripts: ["global_latlon_map", "global_latlon_vect_map"] analysis_scripts: ["amwg_table"] base_regridded_output: True From 5c206479b567d16a3573e4fa292fa39d5f2174d5 Mon Sep 17 00:00:00 2001 From: Michael Levy Date: Wed, 4 Dec 2024 19:28:23 -0700 Subject: [PATCH 10/10] Improve table formatting --- nblibrary/atm/link_to_ADF.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nblibrary/atm/link_to_ADF.ipynb b/nblibrary/atm/link_to_ADF.ipynb index 953e1e15..ae5b9889 100644 --- a/nblibrary/atm/link_to_ADF.ipynb +++ b/nblibrary/atm/link_to_ADF.ipynb @@ -168,7 +168,7 @@ "comparison_table = os.path.join(adf_root, \"amwg_table_comp.csv\")\n", "if os.path.isfile(comparison_table):\n", " table = pd.read_csv(comparison_table)\n", - " display(HTML(table.to_html(index=False)))" + " display(HTML(table.to_html(index=False, float_format=\"{:6g}\".format)))" ] }, {