diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs index 518de3672c..203d7b487a 100644 --- a/.git-blame-ignore-revs +++ b/.git-blame-ignore-revs @@ -69,3 +69,4 @@ cdf40d265cc82775607a1bf25f5f527bacc97405 742cfa606039ab89602fde5fef46458516f56fd4 4ad46f46de7dde753b4653c15f05326f55116b73 75db098206b064b8b7b2a0604d3f0bf8fdb950cc +84609494b54ea9732f64add43b2f1dd035632b4c diff --git a/.github/workflows/formatting_python.yml b/.github/workflows/python-tests.yml similarity index 68% rename from .github/workflows/formatting_python.yml rename to .github/workflows/python-tests.yml index 131e44a7af..a13e594acd 100644 --- a/.github/workflows/formatting_python.yml +++ b/.github/workflows/python-tests.yml @@ -1,4 +1,4 @@ -name: Check Python formatting +name: Run Python tests on: push: @@ -18,7 +18,27 @@ on: - 'cime_config/buildnml/**' jobs: - lint-and-format-check: + python-unit-tests: + runs-on: ubuntu-latest + steps: + # Checkout the code + - uses: actions/checkout@v4 + + # Set up the conda environment + - uses: conda-incubator/setup-miniconda@v3 + with: + activate-environment: ctsm_pylib + environment-file: python/conda_env_ctsm_py.yml + channels: conda-forge + auto-activate-base: false + + # Run Python unit tests check + - name: Run Python unit tests + run: | + cd python + conda run -n ctsm_pylib ./run_ctsm_py_tests -u + + python-lint-and-black: runs-on: ubuntu-latest steps: # Checkout the code diff --git a/bld/namelist_files/namelist_definition_ctsm.xml b/bld/namelist_files/namelist_definition_ctsm.xml index 820975655d..210e4bf243 100644 --- a/bld/namelist_files/namelist_definition_ctsm.xml +++ b/bld/namelist_files/namelist_definition_ctsm.xml @@ -3012,7 +3012,7 @@ snow melt of Brock et al. (2006) group="clm_initinterp_inparm" valid_values="" > If FALSE (which is the default): If an output type cannot be found in the input for initInterp, code aborts -If TRUE: If an output type cannot be found in the input, fill with closest natural veg column +If TRUE: If a non-urban output type cannot be found in the input, fill with closest natural veg column (using bare soil for patch-level variables) NOTE: Natural vegetation and crop landunits always behave as if this were true. e.g., if @@ -3021,6 +3021,14 @@ always fill with the closest natural veg patch / column, regardless of the value flag. So interpolation from non-crop to crop cases can be done without setting this flag. + +If FALSE (which is the default): If an urban output type cannot be found in the input for initInterp, +code aborts +If TRUE: If an urban output type cannot be found in the input, fill with closest urban high density +(HD) landunit + + diff --git a/cime_config/SystemTests/lreprstruct.py b/cime_config/SystemTests/lreprstruct.py index a03fb1815b..baf172fffe 100644 --- a/cime_config/SystemTests/lreprstruct.py +++ b/cime_config/SystemTests/lreprstruct.py @@ -16,6 +16,8 @@ """ +import re + from CIME.SystemTests.system_tests_compare_two import SystemTestsCompareTwo from CIME.XML.standard_module_setup import * from CIME.SystemTests.test_utils.user_nl_utils import append_to_user_nl_files @@ -53,13 +55,16 @@ def _case_one_setup(self): user_nl_clm_path = os.path.join(self._get_caseroot(), "user_nl_clm") with open(user_nl_clm_path) as f: user_nl_clm_text = f.read() - for grain_output in re.findall("GRAIN\w*", user_nl_clm_text): - user_nl_clm_text = user_nl_clm_text.replace( - grain_output, + + def replace_grain(match): + grain_output = match.group() + return ( grain_output.replace("GRAIN", "REPRODUCTIVE1") + "', '" - + grain_output.replace("GRAIN", "REPRODUCTIVE2"), + + grain_output.replace("GRAIN", "REPRODUCTIVE2") ) + + user_nl_clm_text = re.sub(r"GRAIN\w*", replace_grain, user_nl_clm_text) with open(user_nl_clm_path, "w") as f: f.write(user_nl_clm_text) diff --git a/cime_config/SystemTests/sspmatrixcn.py b/cime_config/SystemTests/sspmatrixcn.py index 17ac8abd74..87c4ab2e80 100644 --- a/cime_config/SystemTests/sspmatrixcn.py +++ b/cime_config/SystemTests/sspmatrixcn.py @@ -14,6 +14,7 @@ """ import shutil, glob, os, sys +from pathlib import Path from datetime import datetime if __name__ == "__main__": @@ -205,9 +206,9 @@ def run_indv(self, nstep, st_archive=True): restdir = os.path.join(rest_r, rundate) os.mkdir(restdir) rpoint = os.path.join(restdir, "rpointer.clm." + rundate) - os.mknod(rpoint) + Path.touch(rpoint) rpoint = os.path.join(restdir, "rpointer.cpl." + rundate) - os.mknod(rpoint) + Path.touch(rpoint) def run_phase(self): "Run phase" diff --git a/cime_config/config_pes.xml b/cime_config/config_pes.xml index d39ba06e49..bb10b8019c 100644 --- a/cime_config/config_pes.xml +++ b/cime_config/config_pes.xml @@ -1115,7 +1115,7 @@ - none + default ne120 layout for any machine -16 -16 @@ -1148,6 +1148,196 @@ + + + + + + eXtra-Large Derecho ne120 layout + + -1 + -44 + -44 + -44 + -44 + -44 + -44 + -44 + + + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + + + 0 + -1 + -1 + -1 + -1 + -1 + -1 + -1 + + + + + + + + + Large Derecho ne120 layout + + -1 + -22 + -22 + -22 + -22 + -22 + -22 + -22 + + + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + + + 0 + -1 + -1 + -1 + -1 + -1 + -1 + -1 + + + + + + + + + Medium Derecho ne120 layout + + -1 + -11 + -11 + -11 + -11 + -11 + -11 + -11 + + + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + + + 0 + -1 + -1 + -1 + -1 + -1 + -1 + -1 + + + + + + + + + Small Derecho ne120 layout + + -1 + -6 + -6 + -6 + -6 + -6 + -6 + -6 + + + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + + + 0 + -1 + -1 + -1 + -1 + -1 + -1 + -1 + + + + + + + + + eXtra-Small Derecho ne120 layout + + -1 + -3 + -3 + -3 + -3 + -3 + -3 + -3 + + + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + + + 0 + -1 + -1 + -1 + -1 + -1 + -1 + -1 + + + @@ -1751,7 +1941,7 @@ - none + Derecho mpasa15 layout -1 -36 @@ -1786,6 +1976,123 @@ + + + + + Large Derecho mpasa15 layout + + -1 + -72 + -72 + -72 + -72 + -72 + -72 + -72 + -72 + + + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + + + 0 + -1 + -1 + -1 + -1 + -1 + -1 + -1 + + + + + + + + + Small Derecho mpasa15 layout + + -1 + -18 + -18 + -18 + -18 + -18 + -18 + -18 + -18 + + + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + + + 0 + -1 + -1 + -1 + -1 + -1 + -1 + -1 + + + + + + + + + eXtra-Small Derecho mpasa15 layout + + -1 + -9 + -9 + -9 + -9 + -9 + -9 + -9 + -9 + + + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + + + 0 + -1 + -1 + -1 + -1 + -1 + -1 + -1 + + + + diff --git a/cime_config/testdefs/ExpectedTestFails.xml b/cime_config/testdefs/ExpectedTestFails.xml index bb691b62b4..ac35ad812e 100644 --- a/cime_config/testdefs/ExpectedTestFails.xml +++ b/cime_config/testdefs/ExpectedTestFails.xml @@ -361,4 +361,13 @@ + + + + + FAIL + #3316 + + + diff --git a/cime_config/testdefs/testlist_clm.xml b/cime_config/testdefs/testlist_clm.xml index 351b737c92..41f5d2a4ff 100644 --- a/cime_config/testdefs/testlist_clm.xml +++ b/cime_config/testdefs/testlist_clm.xml @@ -13,6 +13,7 @@ rxcropmaturity: Short tests to be run during development related to prescribed crop calendars matrixcn: Tests exercising the matrix-CN capability aux_clm_mpi_serial: aux_clm tests using mpi-serial. Useful for redoing tests that failed due to https://github.com/ESCOMP/CTSM/issues/2916, after having replaced libraries/mpi-serial with a fresh copy. + decomp_init: Initialization tests specifically for examining the PE layout decomposition initialization --> @@ -172,6 +173,16 @@ + + + + + + + + + + @@ -4152,6 +4163,53 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/cime_config/testdefs/testmods_dirs/clm/f09_FillMissingW_Urban/include_user_mods b/cime_config/testdefs/testmods_dirs/clm/f09_FillMissingW_Urban/include_user_mods new file mode 100644 index 0000000000..fe0e18cf88 --- /dev/null +++ b/cime_config/testdefs/testmods_dirs/clm/f09_FillMissingW_Urban/include_user_mods @@ -0,0 +1 @@ +../default diff --git a/cime_config/testdefs/testmods_dirs/clm/f09_FillMissingW_Urban/user_nl_clm b/cime_config/testdefs/testmods_dirs/clm/f09_FillMissingW_Urban/user_nl_clm new file mode 100644 index 0000000000..499b6026ea --- /dev/null +++ b/cime_config/testdefs/testmods_dirs/clm/f09_FillMissingW_Urban/user_nl_clm @@ -0,0 +1,6 @@ +! NOTE: Using an initial file that does NOT have TBD on it and 5.4 landuse timeseries dataset that has TBD on it +fsurdat = '$DIN_LOC_ROOT/lnd/clm2/surfdata_esmf/ctsm5.4.0/surfdata_0.9x1.25_hist_1850_78pfts_c250428.nc' +flanduse_timeseries = '$DIN_LOC_ROOT/lnd/clm2/surfdata_esmf/ctsm5.4.0/landuse.timeseries_0.9x1.25_hist_1850-2023_78pfts_c250428.nc' +finidat = '$DIN_LOC_ROOT/lnd/clm2/initdata_esmf/ctsm5.4/ctsm53041_54surfdata_snowTherm_100_pSASU.clm2.r.0161-01-01-00000.nc' +init_interp_fill_missing_urban_with_HD = .true. +use_init_interp = .true. diff --git a/cime_config/testdefs/testmods_dirs/clm/run_self_tests/shell_commands b/cime_config/testdefs/testmods_dirs/clm/run_self_tests/shell_commands index d426269206..7762f69e36 100755 --- a/cime_config/testdefs/testmods_dirs/clm/run_self_tests/shell_commands +++ b/cime_config/testdefs/testmods_dirs/clm/run_self_tests/shell_commands @@ -1,5 +1,15 @@ #!/bin/bash ./xmlchange CLM_FORCE_COLDSTART="on" -# We use this testmod in a _Ln1 test; this requires forcing the ROF coupling frequency to every time step -./xmlchange ROF_NCPL=48 +# We use this testmod in a _Ln1 test; this requires forcing the ROF coupling frequency to same frequency as DATM +./xmlchange ROF_NCPL='$ATM_NCPL' + +# Turn MEGAN off to run faster +./xmlchange CLM_BLDNML_OPTS='--no-megan' --append + +# Use fast structure and NWP configuration for speed +./xmlchange CLM_STRUCTURE="fast" +./xmlchange CLM_CONFIGURATION="nwp" + +# Turn cpl history off +./xmlchange HIST_OPTION="never" \ No newline at end of file diff --git a/doc/ChangeLog b/doc/ChangeLog index ca13522df8..d728cf2464 100644 --- a/doc/ChangeLog +++ b/doc/ChangeLog @@ -1,4 +1,97 @@ =============================================================== +Tag name: ctsm5.3.063 +Originator(s): samrabin (Sam Rabin, UCAR/TSS) +Date: Thu Jul 10 12:28:36 MDT 2025 +One-line Summary: Merge b4b-dev to master + +Purpose and description of changes +---------------------------------- + +Regular merge of b4b-dev branch to master. See "Bugs fixed" and "Other details" for more information. + + +Significant changes to scientifically-supported configurations +-------------------------------------------------------------- + +Does this tag change answers significantly for any of the following physics configurations? +(Details of any changes will be given in the "Answer changes" section below.) + + [Put an [X] in the box for any configuration with significant answer changes.] + +[ ] clm6_0 + +[ ] clm5_0 + +[ ] ctsm5_0-nwp + +[ ] clm4_5 + + +Bugs fixed +---------- + +List of CTSM issues fixed: +- [Issue #2985: Fix need/checks for inputdata path in subset_data and Python testing](https://github.com/ESCOMP/CTSM/issues/2985) +- [Issue #2986: Avoid use of os.mknod() in Python testing for portability](https://github.com/ESCOMP/CTSM/issues/2986) +- [Issue #2984: Python unit tests aren't portable](https://github.com/ESCOMP/CTSM/issues/2984) +- [Issue #3279: subset_data still having trouble with Longitude](https://github.com/ESCOMP/CTSM/issues/3279) +- [Issue #2911: Docs: Specify that snow/ice units are liquid water equivalent](https://github.com/ESCOMP/CTSM/issues/2911) +- [Issue #3312: Add some PE layout test sizes for some resolutions to facilitate testing decompInit time testing for different problem sizes / task counts](https://github.com/ESCOMP/CTSM/issues/3312) +- [Issue #3313: Hist fields REPRODUCTIVE1N_TO_FOOD_PERHARV and _ANN lose their suffixes in LREPR* tests](https://github.com/ESCOMP/CTSM/issues/3313) +- [Issue #3110: Initialization of historical using CTSM5.4 surface datasets fails](https://github.com/ESCOMP/CTSM/issues/3110) + + +Notes of particular relevance for users +--------------------------------------- + +Changes to CTSM's user interface (e.g., new/renamed XML or namelist variables): +- New init_interp_fill_missing_urban_with_HD option (default `.false.`). See [Pull Request #3132: Fix #3110 (Initialization of historical using CTSM5.4 surface datasets fails) by olyson](https://github.com/ESCOMP/CTSM/pull/3132). + +Changes to documentation: +- Tech Note now specifies that snow/ice units are liquid water equivalent. + + +Notes of particular relevance for developers: +--------------------------------------------- + +Changes to tests or testing: +- Adds SMS_D_Ld10.f09_f09_mt232.IHistClm60BgcCrop.derecho_intel.clm-f09_FillMissingW_Urban test to aux_clm +- Adds various tests to new decomp_init test suite ("Initialization tests specifically for examining the PE layout decomposition initialization") +- Adds various Python unit and system tests + + +Testing summary: +---------------- + + [PASS means all tests PASS; OK means tests PASS other than expected fails.] + + python testing (if python code has changed; see instructions in python/README.md; document testing done): + + derecho - PASS + + regular tests (aux_clm: https://github.com/ESCOMP/CTSM/wiki/System-Testing-Guide#pre-merge-system-testing): + + derecho ----- OK + izumi ------- OK + + fates tests: (give name of baseline if different from CTSM tagname, normally fates baselines are fates--) + * Will run fates suite after aux_clm, just to generate fates-sci.1.84.0_api.40.0.0-ctsm5.3.063 baseline. Will not compare against any baseline or even check for errors. + + +Other details +------------- + +Pull Requests that document the changes: +- [Pull Request #3238: Add GitHub workflow for Python unit tests by samsrabin](https://github.com/ESCOMP/CTSM/pull/3238) +- [Pull Request #3286: subset_data: Fix conversion of Longitude to string by samsrabin](https://github.com/ESCOMP/CTSM/pull/3286) +- [Pull Request #3247: add notes to specify snow/ice units are liquid water equivalent by sy-li](https://github.com/ESCOMP/CTSM/pull/3247) +- [Pull Request #3315: Add decomp_init testlist and some extra PE layouts for some grids by ekluzek](https://github.com/ESCOMP/CTSM/pull/3315) +- [Pull Request #3314: Fix string replacements in lreprstruct test by billsacks](https://github.com/ESCOMP/CTSM/pull/3314) +- [Pull Request #3132: Fix #3110 (Initialization of historical using CTSM5.4 surface datasets fails) by olyson](https://github.com/ESCOMP/CTSM/pull/3132) +- [Pull Request #3240: tips-for-working-with-rst.md: Add common errors, cheatsheet links. by samsrabin](https://github.com/ESCOMP/CTSM/pull/3240) + +=============================================================== +=============================================================== Tag name: ctsm5.3.062 Originator(s): slevis (Samuel Levis,UCAR/TSS,303-665-1310) Date: Wed 09 Jul 2025 09:03:55 AM MDT diff --git a/doc/ChangeSum b/doc/ChangeSum index 844b216d99..473d47e2b1 100644 --- a/doc/ChangeSum +++ b/doc/ChangeSum @@ -1,5 +1,6 @@ Tag Who Date Summary ============================================================================================================================ + ctsm5.3.063 samrabin 07/10/2025 Merge b4b-dev to master ctsm5.3.062 slevis 07/09/2025 Put inst. and non-inst. fields on separate hist files ctsm5.3.061 slevis 06/26/2025 Merge b4b-dev to master ctsm5.3.060 slevis 06/24/2025 Preliminary update of ctsm54 defaults (answer changing) diff --git a/doc/source/tech_note/Snow_Hydrology/CLM50_Tech_Note_Snow_Hydrology.rst b/doc/source/tech_note/Snow_Hydrology/CLM50_Tech_Note_Snow_Hydrology.rst index d0bff7592d..acd0ae3e22 100644 --- a/doc/source/tech_note/Snow_Hydrology/CLM50_Tech_Note_Snow_Hydrology.rst +++ b/doc/source/tech_note/Snow_Hydrology/CLM50_Tech_Note_Snow_Hydrology.rst @@ -15,6 +15,21 @@ Shown are three snow layers, :math:`i=-2`, :math:`i=-1`, and :math:`i=0`. The la The state variables for snow are the mass of water :math:`w_{liq,i}` (kg m\ :sup:`-2`), mass of ice :math:`w_{ice,i}` (kg m\ :sup:`-2`), layer thickness :math:`\Delta z_{i}` (m), and temperature :math:`T_{i}` (Chapter :numref:`rst_Soil and Snow Temperatures`). The water vapor phase is neglected. Snow can also exist in the model without being represented by explicit snow layers. This occurs when the snowpack is less than a specified minimum snow depth (:math:`z_{sno} < 0.01` m). In this case, the state variable is the mass of snow :math:`W_{sno}` (kg m\ :sup:`-2`). +.. note:: + In CLM, all water-related state variables, including snow and ice, are reported in **liquid water equivalent** units. This means that quantities such as snow water equivalent (SWE), soil ice content, and snowmelt are expressed in terms of the depth of liquid water that would result if the frozen material melted completely. + + For example: + + - ``H2OSNO`` represents the total snow water equivalent in mm. + - ``H2OSOI_ICE`` is the soil ice content in mm. + - ``QSNOMELT`` is the snow melt rate in mm/s. + + In contrast, some glaciological or cryosphere models (e.g., PISM, RACMO2, Crocus) may output variables in **ice-equivalent** units, depending on the modeling context. When necessary, conversion from ice equivalent to water equivalent should account for the density of ice versus liquid water (:numref:`Table Physical Constants`): + + .. math:: + + \text{Water equivalent} = \text{Ice equivalent} \times \frac{\rho_\text{ice}}{\rho_\text{liq}} + Section :numref:`Snow Covered Area Fraction` describes the calculation of fractional snow covered area, which is used in the surface albedo calculation (Chapter :numref:`rst_Surface Albedos`) and the surface flux calculations (Chapter :numref:`rst_Momentum, Sensible Heat, and Latent Heat Fluxes`). The following two sections (:numref:`Ice Content` and :numref:`Water Content`) describe the ice and water content of the snow pack assuming that at least one snow layer exists. Section :numref:`Black and organic carbon and mineral dust within snow` describes how black and organic carbon and mineral dust particles are represented within snow, including meltwater flushing. See Section :numref:`Initialization of snow layer` for a description of how a snow layer is initialized. .. _Snow Covered Area Fraction: diff --git a/doc/source/users_guide/working-with-documentation/tips-for-working-with-rst.md b/doc/source/users_guide/working-with-documentation/tips-for-working-with-rst.md index 2dcc7f455e..164f24115b 100644 --- a/doc/source/users_guide/working-with-documentation/tips-for-working-with-rst.md +++ b/doc/source/users_guide/working-with-documentation/tips-for-working-with-rst.md @@ -2,6 +2,22 @@ # Tips for working with reStructuredText +If you've never used reStructuredText before, you should be aware that its syntax is pretty different from anything you've ever used before. We recommend the following resources as references for the syntax: +- [Sphinx's reStructuredText Primer](https://www.sphinx-doc.org/en/master/usage/restructuredtext/basics.html) +- The [Quick reStructuredText](https://docutils.sourceforge.io/docs/user/rst/quickref.html) cheat sheet + +Some especially useful bits: +- [Section headers](https://www.sphinx-doc.org/en/master/usage/restructuredtext/basics.html#sections) +- [Hyperlinks](https://www.sphinx-doc.org/en/master/usage/restructuredtext/basics.html#hyperlinks) +- [Callout blocks (e.g., warning, tip)](https://www.sphinx-doc.org/en/master/usage/restructuredtext/directives.html#admonitions-messages-and-warnings) + +On this page, we've compiled some supplemental information that might be helpful, including a list of common errors and their causes. + +.. contents:: + :depth: 1 + :backlinks: top + :local: + .. _rst-math: ## reStructuredText: Math @@ -22,10 +38,6 @@ Note (a) the leading spaces for each line after `.. math::` and (b) the empty li reStructuredText math largely follows LaTeX syntax. -Common errors: -- 'ERROR: Error in "math" directive: invalid option block': You might have forgotten the empty line after your equation label. -- "WARNING: Explicit markup ends without a blank line; unexpected unindent": You might have forgotten the leading spaces for every line after `.. math::`. You need at least one leading space on each line. - .. _rst-cross-references: ## reStructuredText: Cross-references @@ -56,12 +68,6 @@ You can have any link (except for equations) show custom text by putting the ref Note that this is necessary for labels that aren't immediately followed by a section heading, a table with a caption, or a figure with a caption. For instance, to refer to labels in our bibliography, you could do ``:ref:`(Bonan, 1996)``` → :ref:`(Bonan, 1996)`. -Common errors: -- "WARNING: Failed to create a cross reference. A title or caption not found": This probably means you tried to `:ref:` a label that's not immediately followed by (a) a table/figure with a caption or section or (b) a section (see above). -- "WARNING: undefined label": If you're sure the label you referenced actually exists, this probably means you tried to ``:numref:`` a label that's not immediately followed by a table, figure, or section (see above). Alternatively, you might have tried to ``:ref:`` an :ref:`equation`; in that case, use ``:eq:`` instead. -- "WARNING: malformed hyperlink target": You may have forgotten the trailing `:` on a label line. -- If you forget to surround a label with blank lines, you will get errors like "Explicit markup ends without a blank line; unexpected unindent [docutils]" that often point to lines far away from the actual problem. - .. _rst-comments: ## reStructuredText: Comments @@ -80,3 +86,80 @@ Make sure to include at least one empty line after the comment text. Tables defined with the [:table: directive](https://docutils.sourceforge.io/docs/ref/rst/directives.html#table) can be annoying because they're very sensitive to the cells inside them being precisely the right widths, as defined by the first `====` strings. If you don't get the widths right, you'll see "Text in column margin" errors. Instead, define your tables using the [list-table](https://docutils.sourceforge.io/docs/ref/rst/directives.html#list-table) directive. If you already have a table in some other format, like comma-separated values (CSV), you may want to check out the R package [knitr](https://cran.r-project.org/web/packages/knitr/index.html). Its [kable](https://bookdown.org/yihui/rmarkdown-cookbook/kable.html) command allows automatic conversion of R dataframes to tables in reStructuredText and other formats. + + +## reStructuredText: Common error messages and how to handle them + +.. _error-unexpected-unindent: + +### "ERROR: Unexpected indentation" + +Like Python, reStructuredText is very particular about how lines are indented. Indentation is used, for example, to denote [code ("literal") blocks](https://www.sphinx-doc.org/en/master/usage/restructuredtext/basics.html#literal-blocks) and [quote blocks](https://www.sphinx-doc.org/en/master/usage/restructuredtext/basics.html#lists-and-quote-like-blocks). An error like +``` +/path/to/file.rst:102: ERROR: Unexpected indentation. [docutils] +``` +indicates that line 102 is indented but not in a way that reStructuredText expects. + +### "WARNING: Block quote ends without a blank line; unexpected unindent" + +This is essentially the inverse of :ref:`error-unexpected-unindent`: The above line was indented but this one isn't. reStructuredText tried to interpret the indented line as a [block quote](https://www.sphinx-doc.org/en/master/usage/restructuredtext/basics.html#lists-and-quote-like-blocks), but block quotes require a blank line after them. + +.. _inline-literal-start-without-end: + +### "WARNING: Inline literal start-string without end-string" + +An "inline literal" is when you want to mix code into a normal line of text (as opposed to in its own code block) ``like this``. This is accomplished with double-backticks: +```reStructuredText +An "inline literal" is when you want to mix code into a normal line of +text (as opposed to in its own code block) ``like this``. +``` +(A backtick is what you get if you press the key to the left of 1 on a standard US English keyboard.) + +If you have a double-backtick on a line, reStructuredText will think, "They want to start an inline literal here," then look for another double-backtick to end the literal. The "WARNING: Inline literal start-string without end-string" means it can't find one on that line. + +This might happen, for example, if you try to put a [Markdown code block](https://docs.github.com/en/get-started/writing-on-github/working-with-advanced-formatting/creating-and-highlighting-code-blocks) in a .rst file. In that case, use the [reStructuredText code block syntax](https://www.sphinx-doc.org/en/master/usage/restructuredtext/basics.html#literal-blocks) instead (optionally with [syntax highlighting](https://www.sphinx-doc.org/en/master/usage/restructuredtext/directives.html#directive-highlight)). + +### "WARNING: Inline interpreted text or phrase reference start-string without end-string" + +Like :ref:`inline-literal-start-without-end`, this is probably related to having one double-backtick without another on the same line. As with that other error, it could be the result of a Markdown code block in a .rst file. + +### "ERROR: Error in "code" directive: maximum 1 argument(s) allowed, 19 supplied" + +This error might show something other than "code," like "highlight" or "sourcecode". It also will probably show a second number that's not 19. The problem is that you tried to write a [reStructuredText code block with syntax highlighting](https://www.sphinx-doc.org/en/master/usage/restructuredtext/directives.html#directive-highlight) but didn't include a blank line after the first one: + +```reStructuredText +.. code:: shell + # How to list all the available grids + cd cime/scripts + ./query_config --grids +``` + +Fix this by adding a blank line: +```reStructuredText +.. code:: shell + + # How to list all the available grids + cd cime/scripts + ./query_config --grids +``` + +### 'ERROR: Error in "math" directive: invalid option block' + +You might have forgotten the empty line after an equation label. + +### "WARNING: Explicit markup ends without a blank line; unexpected unindent" + +You might have forgotten the leading spaces for every line after `.. math::`. As a reminder, you need at least one leading space on each line. + +You can also get this error if you forget to surround a :ref:`cross-reference label` with blank lines. In this case, the error message might point to lines far away from the actual problem. + +### "WARNING: Failed to create a cross reference: A title or caption not found" +This probably means you tried to `:ref:` a label that's not immediately followed by (a) a table/figure with a caption or (b) a section. + +### "WARNING: undefined label" + +If you're sure the label you referenced actually exists, this probably means you tried to ``:numref:`` a label that's not immediately followed by a table, figure, or section (see above). Alternatively, you might have tried to ``:ref:`` an :ref:`equation`; in that case, use ``:eq:`` instead. + +### "WARNING: malformed hyperlink target" + +You may have forgotten the trailing `:` on a label line. \ No newline at end of file diff --git a/python/ctsm/longitude.py b/python/ctsm/longitude.py index fb5998524d..96fd134082 100644 --- a/python/ctsm/longitude.py +++ b/python/ctsm/longitude.py @@ -177,6 +177,13 @@ def __ge__(self, other): self._check_lons_same_type(other) return self._lon >= other._lon + def __str__(self): + """ + We don't allow implicit string conversion because the user should always specify the + Longitude type they want + """ + raise NotImplementedError("Use Longitude.get_str() instead of implicit string conversion") + def get(self, lon_type_out): """ Get the longitude value, converting longitude type if needed @@ -189,6 +196,14 @@ def get(self, lon_type_out): return _convert_lon_type_180_to_360(self._lon) raise RuntimeError(f"Add handling for lon_type_out {lon_type_out}") + def get_str(self, lon_type_out): + """ + Get the longitude value as a string, converting longitude type if needed + """ + lon_out = self.get(lon_type_out) + # Use float() because the standard in CTSM filenames is to put .0 after whole-number values + return str(float(lon_out)) + def lon_type(self): """ Getter method for self._lon_type diff --git a/python/ctsm/site_and_regional/regional_case.py b/python/ctsm/site_and_regional/regional_case.py index ed91f3d474..1b52e72ab4 100644 --- a/python/ctsm/site_and_regional/regional_case.py +++ b/python/ctsm/site_and_regional/regional_case.py @@ -160,6 +160,20 @@ def _subset_lon_lat(self, x_dim, y_dim, f_in): f_out = f_in.isel({y_dim: yind, x_dim: xind}) return f_out + def _get_lon_strings(self): + """ + Get the string versions of the region's longitudes + """ + if isinstance(self.lon1, Longitude): + lon1_str = self.lon1.get_str(self.lon1.lon_type()) + else: + lon1_str = str(self.lon1) + if isinstance(self.lon2, Longitude): + lon2_str = self.lon2.get_str(self.lon2.lon_type()) + else: + lon2_str = str(self.lon2) + return lon1_str, lon2_str + def create_tag(self): """ Create a tag for a region which is either the region name @@ -169,9 +183,8 @@ def create_tag(self): if self.reg_name: self.tag = self.reg_name else: - self.tag = "{}-{}_{}-{}".format( - str(self.lon1), str(self.lon2), str(self.lat1), str(self.lat2) - ) + lon1_str, lon2_str = self._get_lon_strings() + self.tag = "{}-{}_{}-{}".format(lon1_str, lon2_str, str(self.lat1), str(self.lat2)) def check_region_bounds(self): """ @@ -179,10 +192,11 @@ def check_region_bounds(self): """ # If you're calling this, lat/lon bounds need to have been provided if any(x is None for x in [self.lon1, self.lon2, self.lat1, self.lat2]): + lon1_str, lon2_str = self._get_lon_strings() raise argparse.ArgumentTypeError( "Latitude and longitude bounds must be provided and not None.\n" - + f" lon1: {self.lon1}\n" - + f" lon2: {self.lon2}\n" + + f" lon1: {lon1_str}\n" + + f" lon2: {lon2_str}\n" + f" lat1: {self.lat1}\n" + f" lat2: {self.lat2}" ) diff --git a/python/ctsm/site_and_regional/single_point_case.py b/python/ctsm/site_and_regional/single_point_case.py index d71d014f36..c99a240513 100644 --- a/python/ctsm/site_and_regional/single_point_case.py +++ b/python/ctsm/site_and_regional/single_point_case.py @@ -159,9 +159,12 @@ def convert_plon_to_filetype_if_needed(self, lon_da): plon_orig = plon_in.get(plon_type) plon_out = plon_in.get(f_lon_type) if plon_orig != plon_out: - print( - f"Converted plon from type {plon_type} (value {plon_orig}) " - f"to type {f_lon_type} (value {plon_out})" + logger.info( + "Converted plon from type %s (value %f) to type %s (value %f)", + plon_type, + plon_orig, + f_lon_type, + plon_out, ) return plon_out @@ -173,7 +176,7 @@ def create_tag(self): if self.site_name: self.tag = self.site_name else: - self.tag = "{}_{}".format(str(self.plon), str(self.plat)) + self.tag = "{}_{}".format(self.plon.get_str(self.plon.lon_type()), str(self.plat)) def check_dom_pft(self): """ @@ -333,7 +336,11 @@ def create_domain_at_point(self, indir, file): Create domain file for this SinglePointCase class. """ logger.info("----------------------------------------------------------------------") - logger.info("Creating domain file at %s, %s.", str(self.plon), str(self.plat)) + logger.info( + "Creating domain file at %s, %s.", + self.plon.get_str(self.plon.lon_type()), + str(self.plat), + ) # specify files fdomain_in = os.path.join(indir, file) @@ -367,7 +374,7 @@ def create_landuse_at_point(self, indir, file, user_mods_dir): logger.info("----------------------------------------------------------------------") logger.info( "Creating land use file at %s, %s.", - str(self.plon), + self.plon.get_str(self.plon.lon_type()), str(self.plat), ) @@ -502,7 +509,7 @@ def create_surfdata_at_point(self, indir, file, user_mods_dir, specify_fsurf_out logger.info("----------------------------------------------------------------------") logger.info( "Creating surface dataset file at %s, %s", - str(self.plon), + self.plon.get_str(self.plon.lon_type()), str(self.plat), ) @@ -577,7 +584,7 @@ def create_datmdomain_at_point(self, datm_tuple: DatmFiles): logger.info("----------------------------------------------------------------------") logger.info( "Creating DATM domain file at %s, %s", - str(self.plon), + self.plon.get_str(self.plon.lon_type()), str(self.plat), ) @@ -646,7 +653,9 @@ def write_shell_commands(self, file, datm_syr, datm_eyr): with open(file, "w") as nl_file: self.write_to_file("# Change below line if you move the subset data directory", nl_file) self.write_to_file("./xmlchange {}={}".format(USRDAT_DIR, self.out_dir), nl_file) - self.write_to_file("./xmlchange PTS_LON={}".format(str(self.plon)), nl_file) + self.write_to_file( + "./xmlchange PTS_LON={}".format(self.plon.get_str(self.plon.lon_type())), nl_file + ) self.write_to_file("./xmlchange PTS_LAT={}".format(str(self.plat)), nl_file) self.write_to_file("./xmlchange MPILIB=mpi-serial", nl_file) if self.create_datm: @@ -672,7 +681,9 @@ def create_datm_at_point(self, datm_tuple: DatmFiles, datm_syr, datm_eyr, datm_s Create all of a DATM dataset at a point. """ logger.info("----------------------------------------------------------------------") - logger.info("Creating DATM files at %s, %s", str(self.plon), str(self.plat)) + logger.info( + "Creating DATM files at %s, %s", self.plon.get_str(self.plon.lon_type()), str(self.plat) + ) # -- create data files infile = [] diff --git a/python/ctsm/subset_data.py b/python/ctsm/subset_data.py index de4e51db9b..ed9282ef46 100644 --- a/python/ctsm/subset_data.py +++ b/python/ctsm/subset_data.py @@ -605,7 +605,7 @@ def determine_num_pft(crop): return num_pft -def setup_files(args, defaults, cesmroot): +def setup_files(args, defaults, cesmroot, testing=False): """ Sets up the files and folders needed for this program """ @@ -623,9 +623,9 @@ def setup_files(args, defaults, cesmroot): else: clmforcingindir = args.inputdatadir - if not os.path.isdir(clmforcingindir): + if not testing and not os.path.isdir(clmforcingindir): logger.info("clmforcingindir does not exist: %s", clmforcingindir) - abort("inputdata directory does not exist") + abort(f"inputdata directory does not exist: {clmforcingindir}") file_dict = {"main_dir": clmforcingindir} @@ -822,7 +822,7 @@ def subset_region(args, file_dict: dict): print("\nFor running this regional case with the created user_mods : ") print( "./create_newcase --case case --res CLM_USRDAT --compset I2000Clm60BgcCrop", - "--run-unsupported --user-mods-dirs ", + "--run-unsupported --user-mods-dir ", args.user_mods_dir, "\n\n", ) diff --git a/python/ctsm/test/test_sys_gen_mksurfdata_jobscript_single_derecho.py b/python/ctsm/test/test_sys_gen_mksurfdata_jobscript_single_derecho.py new file mode 100755 index 0000000000..627fb1d32b --- /dev/null +++ b/python/ctsm/test/test_sys_gen_mksurfdata_jobscript_single_derecho.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python3 + +""" +System tests for gen_mksurfdata_jobscript_single.py subroutines on Derecho +""" + +import unittest +import os + +from ctsm import unit_testing +from ctsm.test_gen_mksurfdata_jobscript_single_parent import TestFGenMkSurfJobscriptSingleParent +from ctsm.path_utils import path_to_cime +from ctsm.os_utils import run_cmd_output_on_error +from ctsm.toolchain.gen_mksurfdata_jobscript_single import get_parser +from ctsm.toolchain.gen_mksurfdata_jobscript_single import get_mpirun +from ctsm.toolchain.gen_mksurfdata_jobscript_single import check_parser_args +from ctsm.toolchain.gen_mksurfdata_jobscript_single import write_runscript_part1 + + +# Allow test names that pylint doesn't like; otherwise hard to make them +# readable +# pylint: disable=invalid-name + + +# pylint: disable=protected-access +# pylint: disable=too-many-instance-attributes +class TestFGenMkSurfJobscriptSingleDerecho(TestFGenMkSurfJobscriptSingleParent): + """Tests the gen_mksurfdata_jobscript_single subroutines on Derecho""" + + def test_derecho_mpirun(self): + """ + test derecho mpirun. This would've helped caught a problem we ran into + It will also be helpful when sumodules are updated to guide to solutions + to problems + """ + machine = "derecho" + nodes = 4 + tasks = 128 + unit_testing.add_machine_node_args(machine, nodes, tasks) + args = get_parser().parse_args() + check_parser_args(args) + self.assertEqual(machine, args.machine) + self.assertEqual(tasks, args.tasks_per_node) + self.assertEqual(nodes, args.number_of_nodes) + self.assertEqual(self._account, args.account) + # Create the env_mach_specific.xml file needed for get_mpirun + # This will catch problems with our usage of CIME objects + # Doing this here will also catch potential issues in the gen_mksurfdata_build script + configure_path = os.path.join(path_to_cime(), "CIME", "scripts", "configure") + self.assertTrue(os.path.exists(configure_path)) + options = " --macros-format CMake --silent --compiler intel --machine " + machine + cmd = configure_path + options + cmd_list = cmd.split() + run_cmd_output_on_error( + cmd=cmd_list, errmsg="Trouble running configure", cwd=self._bld_path + ) + self.assertTrue(os.path.exists(self._env_mach)) + expected_attribs = {"mpilib": "default"} + with open(self._jobscript_file, "w", encoding="utf-8") as runfile: + attribs = write_runscript_part1( + number_of_nodes=nodes, + tasks_per_node=tasks, + machine=machine, + account=self._account, + walltime=args.walltime, + runfile=runfile, + ) + self.assertEqual(attribs, expected_attribs) + (executable, mksurfdata_path, env_mach_path) = get_mpirun(args, attribs) + expected_exe = "time mpibind " + self.assertEqual(executable, expected_exe) + self.assertEqual(mksurfdata_path, self._mksurf_exe) + self.assertEqual(env_mach_path, self._env_mach) + + +if __name__ == "__main__": + unit_testing.setup_for_tests() + unittest.main() diff --git a/python/ctsm/test/test_sys_subset_data.py b/python/ctsm/test/test_sys_subset_data.py index 39d448cccd..453df7c18e 100644 --- a/python/ctsm/test/test_sys_subset_data.py +++ b/python/ctsm/test/test_sys_subset_data.py @@ -12,6 +12,7 @@ import tempfile import inspect import xarray as xr +from CIME.scripts.create_newcase import _main_func as create_newcase # pylint: disable=import-error # -- add python/ctsm to path (needed if we want to run the test stand-alone) _CTSM_PYTHON = os.path.join(os.path.dirname(os.path.realpath(__file__)), os.pardir, os.pardir) @@ -23,19 +24,52 @@ from ctsm.utils import find_one_file_matching_pattern +def _get_sitename_str_point(include_sitename, sitename, lon, lat): + """ + Given a site, return the string to use in output filenames + """ + if include_sitename: + sitename_str = sitename + else: + sitename_str = f"{float(lon)}_{float(lat)}" + return sitename_str + + class TestSubsetDataSys(unittest.TestCase): """ Basic class for testing subset_data.py. """ def setUp(self): + self.previous_dir = os.getcwd() self.temp_dir_out = tempfile.TemporaryDirectory() self.temp_dir_umd = tempfile.TemporaryDirectory() + self.temp_dir_caseparent = tempfile.TemporaryDirectory() self.inputdata_dir = os.path.join(os.path.dirname(__file__), os.pardir, os.pardir) def tearDown(self): self.temp_dir_out.cleanup() self.temp_dir_umd.cleanup() + os.chdir(self.previous_dir) + + def _check_create_newcase(self): + """ + Check that you can call create_newcase using the usermods from subset_data + """ + case_dir = os.path.join(self.temp_dir_caseparent.name, "case") + sys.argv = [ + "create_newcase", + "--case", + case_dir, + "--res", + "CLM_USRDAT", + "--compset", + "I2000Clm60Bgc", + "--run-unsupported", + "--user-mods-dir", + self.temp_dir_umd.name, + ] + create_newcase() def _check_result_file_matches_expected(self, expected_output_files, caller_n): """ @@ -46,9 +80,18 @@ def _check_result_file_matches_expected(self, expected_output_files, caller_n): caller_n = 1. If the test is calling a function that calls this function, caller_n = 2. Etc. """ all_files_present_and_match = True + result_file_found = True + expected_file_found = True for basename in expected_output_files: + + # Check whether result (output) file exists. If not, note it but continue. result_file = os.path.join(self.temp_dir_out.name, basename) - result_file = find_one_file_matching_pattern(result_file) + try: + result_file = find_one_file_matching_pattern(result_file) + except FileNotFoundError: + result_file_found = False + + # Check whether expected file exists. If not, note it but continue. expected_file = os.path.join( os.path.dirname(__file__), "testinputs", @@ -56,7 +99,27 @@ def _check_result_file_matches_expected(self, expected_output_files, caller_n): inspect.stack()[caller_n][3], # Name of calling function (i.e., test name) basename, ) - expected_file = find_one_file_matching_pattern(expected_file) + try: + expected_file = find_one_file_matching_pattern(expected_file) + except FileNotFoundError: + expected_file_found = False + + # Raise an AssertionError if either file was not found + if not (result_file_found and expected_file_found): + msg = "" + if not result_file_found: + this_dir = os.path.dirname(result_file) + msg += f"\nResult file '{result_file}' not found. " + msg += f"Contents of directory '{this_dir}':\n\t" + msg += "\n\t".join(os.listdir(this_dir)) + if not expected_file_found: + this_dir = os.path.dirname(expected_file) + msg += f"\nExpected file '{expected_file}' not found. " + msg += f"Contents of directory '{this_dir}':\n\t" + msg += "\n\t".join(os.listdir(this_dir)) + raise AssertionError(msg) + + # Compare the two files ds_result = xr.open_dataset(result_file) ds_expected = xr.open_dataset(expected_file) if not ds_result.equals(ds_expected): @@ -66,10 +129,15 @@ def _check_result_file_matches_expected(self, expected_output_files, caller_n): all_files_present_and_match = False return all_files_present_and_match - def test_subset_data_reg_amazon(self): + def _do_test_subset_data_reg_amazon(self, include_regname=True): """ - Test subset_data for Amazon region + Convenience function for multiple tests of subset_data region for the Amazon """ + regname = "TMP" + lat1 = -12 + lat2 = -7 + lon1 = 291 + lon2 = 299 cfg_file = os.path.join( self.inputdata_dir, "ctsm", @@ -82,15 +150,13 @@ def test_subset_data_reg_amazon(self): "subset_data", "region", "--lat1", - "-12", + str(lat1), "--lat2", - "-7", + str(lat2), "--lon1", - "291", + str(lon1), "--lon2", - "299", - "--reg", - "TMP", + str(lon2), "--create-mesh", "--create-domain", "--create-surface", @@ -107,16 +173,40 @@ def test_subset_data_reg_amazon(self): cfg_file, "--overwrite", ] + if include_regname: + sys.argv += ["--reg", regname] subset_data.main() # Loop through all the output files, making sure they match what we expect. daystr = "[0-9][0-9][0-9][0-9][0-9][0-9]" # 6-digit day code, yymmdd + if include_regname: + regname_str = regname + else: + regname_str = f"{float(lon1)}-{float(lon2)}_{float(lat1)}-{float(lat2)}" expected_output_files = [ - f"domain.lnd.5x5pt-amazon_navy_TMP_c{daystr}_ESMF_UNSTRUCTURED_MESH.nc", - f"domain.lnd.5x5pt-amazon_navy_TMP_c{daystr}.nc", - f"surfdata_TMP_amazon_hist_16pfts_CMIP6_2000_c{daystr}.nc", + f"domain.lnd.5x5pt-amazon_navy_{regname_str}_c{daystr}_ESMF_UNSTRUCTURED_MESH.nc", + f"domain.lnd.5x5pt-amazon_navy_{regname_str}_c{daystr}.nc", + f"surfdata_{regname_str}_amazon_hist_16pfts_CMIP6_2000_c{daystr}.nc", ] - self.assertTrue(self._check_result_file_matches_expected(expected_output_files, 1)) + self.assertTrue(self._check_result_file_matches_expected(expected_output_files, 2)) + + # Check that create_newcase works + # SHOULD WORK ONLY ON CESM-SUPPORTED MACHINES + self._check_create_newcase() + + def test_subset_data_reg_amazon(self): + """ + Test subset_data for Amazon region + SHOULD WORK ONLY ON CESM-SUPPORTED MACHINES + """ + self._do_test_subset_data_reg_amazon() + + def test_subset_data_reg_amazon_noregname(self): + """ + Test subset_data for Amazon region + SHOULD WORK ONLY ON CESM-SUPPORTED MACHINES + """ + self._do_test_subset_data_reg_amazon(include_regname=False) def test_subset_data_reg_infile_detect360(self): """ @@ -189,10 +279,11 @@ def test_subset_data_reg_infile_detect180_error(self): ): subset_data.main() - def _do_test_subset_data_pt_surface(self, lon): + def _do_test_subset_data_pt_surface(self, lon, include_sitename=True): """ Given a longitude, test subset_data point --create-surface """ + lat = -12 cfg_file = os.path.join( self.inputdata_dir, "ctsm", @@ -205,11 +296,9 @@ def _do_test_subset_data_pt_surface(self, lon): "subset_data", "point", "--lat", - "-12", + str(lat), "--lon", str(lon), - "--site", - "TMP", "--create-domain", "--create-surface", "--surf-year", @@ -225,31 +314,51 @@ def _do_test_subset_data_pt_surface(self, lon): cfg_file, "--overwrite", ] + sitename = "TMP" + if include_sitename: + sys.argv += ["--site", sitename] subset_data.main() # Loop through all the output files, making sure they match what we expect. daystr = "[0-9][0-9][0-9][0-9][0-9][0-9]" # 6-digit day code, yymmdd + sitename_str = _get_sitename_str_point(include_sitename, sitename, lon, lat) expected_output_files = [ - f"surfdata_TMP_amazon_hist_16pfts_CMIP6_2000_c{daystr}.nc", + f"surfdata_{sitename_str}_amazon_hist_16pfts_CMIP6_2000_c{daystr}.nc", ] self.assertTrue(self._check_result_file_matches_expected(expected_output_files, 2)) + # Check that create_newcase works + # SHOULD WORK ONLY ON CESM-SUPPORTED MACHINES + self._check_create_newcase() + def test_subset_data_pt_surface_amazon_type360(self): """ Test subset_data --create-surface for Amazon point with longitude type 360 + SHOULD WORK ONLY ON CESM-SUPPORTED MACHINES """ self._do_test_subset_data_pt_surface(291) def test_subset_data_pt_surface_amazon_type180(self): """ Test subset_data --create-surface for Amazon point with longitude type 180 + SHOULD WORK ONLY ON CESM-SUPPORTED MACHINES """ self._do_test_subset_data_pt_surface(-69) - def _do_test_subset_data_pt_landuse(self, lon): + def test_subset_data_pt_surface_amazon_type180_nositename(self): + """ + Test subset_data --create-surface for Amazon point with longitude type 180 + without specifying a site name + SHOULD WORK ONLY ON CESM-SUPPORTED MACHINES + """ + self._do_test_subset_data_pt_surface(-69, include_sitename=False) + + def _do_test_subset_data_pt_landuse(self, lon, include_sitename=True): """ Given a longitude, test subset_data point --create-landuse """ + lat = -12 + sitename = "TMP" cfg_file = os.path.join( self.inputdata_dir, "ctsm", @@ -262,11 +371,9 @@ def _do_test_subset_data_pt_landuse(self, lon): "subset_data", "point", "--lat", - "-12", + str(lat), "--lon", str(lon), - "--site", - "TMP", "--create-domain", "--create-surface", "--surf-year", @@ -283,45 +390,60 @@ def _do_test_subset_data_pt_landuse(self, lon): cfg_file, "--overwrite", ] + if include_sitename: + sys.argv += ["--site", sitename] subset_data.main() # Loop through all the output files, making sure they match what we expect. daystr = "[0-9][0-9][0-9][0-9][0-9][0-9]" # 6-digit day code, yymmdd + sitename_str = _get_sitename_str_point(include_sitename, sitename, lon, lat) expected_output_files = [ - f"surfdata_TMP_amazon_hist_1850_78pfts_c{daystr}.nc", - f"landuse.timeseries_TMP_amazon_hist_1850-1853_78pfts_c{daystr}.nc", + f"surfdata_{sitename_str}_amazon_hist_1850_78pfts_c{daystr}.nc", + f"landuse.timeseries_{sitename_str}_amazon_hist_1850-1853_78pfts_c{daystr}.nc", ] self.assertTrue(self._check_result_file_matches_expected(expected_output_files, 2)) + # Check that create_newcase works + # SHOULD WORK ONLY ON CESM-SUPPORTED MACHINES + self._check_create_newcase() + def test_subset_data_pt_landuse_amazon_type360(self): """ Test subset_data --create-landuse for Amazon point with longitude type 360 + SHOULD WORK ONLY ON CESM-SUPPORTED MACHINES """ self._do_test_subset_data_pt_landuse(291) + def test_subset_data_pt_landuse_amazon_type360_nositename(self): + """ + Test subset_data --create-landuse for Amazon point with longitude type 360 and no site name + SHOULD WORK ONLY ON CESM-SUPPORTED MACHINES + """ + self._do_test_subset_data_pt_landuse(291, include_sitename=False) + def test_subset_data_pt_landuse_amazon_type180(self): """ Test subset_data --create-landuse for Amazon point with longitude type 180 + SHOULD WORK ONLY ON CESM-SUPPORTED MACHINES """ self._do_test_subset_data_pt_landuse(-69) - def _do_test_subset_data_pt_datm(self, lon): + def _do_test_subset_data_pt_datm(self, lon, include_sitename=True): """ Given a longitude, test subset_data point --create-datm """ start_year = 1986 end_year = 1988 sitename = "TMP" + lat = -12 outdir = self.temp_dir_out.name sys.argv = [ "subset_data", "point", "--lat", - "-12", + str(lat), "--lon", str(lon), - "--site", - sitename, "--create-datm", "--datm-syr", str(start_year), @@ -334,21 +456,27 @@ def _do_test_subset_data_pt_datm(self, lon): self.temp_dir_umd.name, "--overwrite", ] + if include_sitename: + sys.argv += ["--site", sitename] subset_data.main() # Loop through all the output files, making sure they match what we expect. daystr = "[0-9][0-9][0-9][0-9][0-9][0-9]" # 6-digit day code, yymmdd + sitename_str = _get_sitename_str_point(include_sitename, sitename, lon, lat) expected_output_files = [ - f"domain.crujra_v2.3_0.5x0.5_{sitename}_c{daystr}.nc", + f"domain.crujra_v2.3_0.5x0.5_{sitename_str}_c{daystr}.nc", ] for year in list(range(start_year, end_year + 1)): for forcing in ["Solr", "Prec", "TPQWL"]: expected_output_files.append( - f"clmforc.CRUJRAv2.5_0.5x0.5.{forcing}.{sitename}.{year}.nc" + f"clmforc.CRUJRAv2.5_0.5x0.5.{forcing}.{sitename_str}.{year}.nc" ) expected_output_files = [os.path.join("datmdata", x) for x in expected_output_files] self.assertTrue(self._check_result_file_matches_expected(expected_output_files, 2)) + # Check that create_newcase works + self._check_create_newcase() + def test_subset_data_pt_datm_amazon_type360(self): """ Test subset_data --create-datm for Amazon point with longitude type 360 @@ -363,6 +491,14 @@ def test_subset_data_pt_datm_amazon_type180(self): """ self._do_test_subset_data_pt_datm(-69) + def test_subset_data_pt_datm_amazon_type180_nositename(self): + """ + Test subset_data --create-datm for Amazon point with longitude type 180 without providing + site name. + FOR NOW CAN ONLY BE RUN ON DERECHO/CASPER + """ + self._do_test_subset_data_pt_datm(-69, include_sitename=False) + if __name__ == "__main__": unit_testing.setup_for_tests() diff --git a/python/ctsm/test/test_unit_gen_mksurfdata_jobscript_single.py b/python/ctsm/test/test_unit_gen_mksurfdata_jobscript_single.py index bee1aac715..3980b3bd49 100755 --- a/python/ctsm/test/test_unit_gen_mksurfdata_jobscript_single.py +++ b/python/ctsm/test/test_unit_gen_mksurfdata_jobscript_single.py @@ -6,40 +6,15 @@ import unittest import os -import sys import shutil -import tempfile - from ctsm import unit_testing -from ctsm.path_utils import path_to_ctsm_root -from ctsm.path_utils import path_to_cime -from ctsm.os_utils import run_cmd_output_on_error +from ctsm.test_gen_mksurfdata_jobscript_single_parent import TestFGenMkSurfJobscriptSingleParent from ctsm.toolchain.gen_mksurfdata_jobscript_single import get_parser -from ctsm.toolchain.gen_mksurfdata_jobscript_single import get_mpirun from ctsm.toolchain.gen_mksurfdata_jobscript_single import check_parser_args from ctsm.toolchain.gen_mksurfdata_jobscript_single import write_runscript_part1 -def add_args(machine, nodes, tasks): - """add arguments to sys.argv""" - args_to_add = [ - "--machine", - machine, - "--number-of-nodes", - str(nodes), - "--tasks-per-node", - str(tasks), - ] - for item in args_to_add: - sys.argv.append(item) - - -def create_empty_file(filename): - """create an empty file""" - os.system("touch " + filename) - - # Allow test names that pylint doesn't like; otherwise hard to make them # readable # pylint: disable=invalid-name @@ -47,65 +22,9 @@ def create_empty_file(filename): # pylint: disable=protected-access # pylint: disable=too-many-instance-attributes -class TestFGenMkSurfJobscriptSingle(unittest.TestCase): +class TestFGenMkSurfJobscriptSingle(TestFGenMkSurfJobscriptSingleParent): """Tests the gen_mksurfdata_jobscript_single subroutines""" - def setUp(self): - """Setup for trying out the methods""" - testinputs_path = os.path.join(path_to_ctsm_root(), "python/ctsm/test/testinputs") - self._testinputs_path = testinputs_path - self._previous_dir = os.getcwd() - self._tempdir = tempfile.mkdtemp() - os.chdir(self._tempdir) - self._account = "ACCOUNT_NUMBER" - self._jobscript_file = "output_jobscript" - self._output_compare = """#!/bin/bash -# Edit the batch directives for your batch system -# Below are default batch directives for derecho -#PBS -N mksurfdata -#PBS -j oe -#PBS -k eod -#PBS -S /bin/bash -#PBS -l walltime=12:00:00 -#PBS -A ACCOUNT_NUMBER -#PBS -q main -#PBS -l select=1:ncpus=128:mpiprocs=64:mem=218GB - -# This is a batch script to run a set of resolutions for mksurfdata_esmf input namelist -# NOTE: THIS SCRIPT IS AUTOMATICALLY GENERATED SO IN GENERAL YOU SHOULD NOT EDIT it!! - -""" - self._bld_path = os.path.join(self._tempdir, "tools_bld") - os.makedirs(self._bld_path) - self.assertTrue(os.path.isdir(self._bld_path)) - self._nlfile = os.path.join(self._tempdir, "namelist_file") - create_empty_file(self._nlfile) - self.assertTrue(os.path.exists(self._nlfile)) - self._mksurf_exe = os.path.join(self._bld_path, "mksurfdata") - create_empty_file(self._mksurf_exe) - self.assertTrue(os.path.exists(self._mksurf_exe)) - self._env_mach = os.path.join(self._bld_path, ".env_mach_specific.sh") - create_empty_file(self._env_mach) - self.assertTrue(os.path.exists(self._env_mach)) - sys.argv = [ - "gen_mksurfdata_jobscript_single", - "--bld-path", - self._bld_path, - "--namelist-file", - self._nlfile, - "--jobscript-file", - self._jobscript_file, - "--account", - self._account, - ] - - def tearDown(self): - """ - Remove temporary directory - """ - os.chdir(self._previous_dir) - shutil.rmtree(self._tempdir, ignore_errors=True) - def assertFileContentsEqual(self, expected, filepath, msg=None): """Asserts that the contents of the file given by 'filepath' are equal to the string given by 'expected'. 'msg' gives an optional message to be @@ -123,7 +42,7 @@ def test_simple_derecho_args(self): machine = "derecho" nodes = 1 tasks = 64 - add_args(machine, nodes, tasks) + unit_testing.add_machine_node_args(machine, nodes, tasks) args = get_parser().parse_args() check_parser_args(args) with open(self._jobscript_file, "w", encoding="utf-8") as runfile: @@ -139,57 +58,12 @@ def test_simple_derecho_args(self): self.assertFileContentsEqual(self._output_compare, self._jobscript_file) - def test_derecho_mpirun(self): - """ - test derecho mpirun. This would've helped caught a problem we ran into - It will also be helpful when sumodules are updated to guide to solutions - to problems - """ - machine = "derecho" - nodes = 4 - tasks = 128 - add_args(machine, nodes, tasks) - args = get_parser().parse_args() - check_parser_args(args) - self.assertEqual(machine, args.machine) - self.assertEqual(tasks, args.tasks_per_node) - self.assertEqual(nodes, args.number_of_nodes) - self.assertEqual(self._account, args.account) - # Create the env_mach_specific.xml file needed for get_mpirun - # This will catch problems with our usage of CIME objects - # Doing this here will also catch potential issues in the gen_mksurfdata_build script - configure_path = os.path.join(path_to_cime(), "CIME", "scripts", "configure") - self.assertTrue(os.path.exists(configure_path)) - options = " --macros-format CMake --silent --compiler intel --machine " + machine - cmd = configure_path + options - cmd_list = cmd.split() - run_cmd_output_on_error( - cmd=cmd_list, errmsg="Trouble running configure", cwd=self._bld_path - ) - self.assertTrue(os.path.exists(self._env_mach)) - expected_attribs = {"mpilib": "default"} - with open(self._jobscript_file, "w", encoding="utf-8") as runfile: - attribs = write_runscript_part1( - number_of_nodes=nodes, - tasks_per_node=tasks, - machine=machine, - account=self._account, - walltime=args.walltime, - runfile=runfile, - ) - self.assertEqual(attribs, expected_attribs) - (executable, mksurfdata_path, env_mach_path) = get_mpirun(args, attribs) - expected_exe = "time mpibind " - self.assertEqual(executable, expected_exe) - self.assertEqual(mksurfdata_path, self._mksurf_exe) - self.assertEqual(env_mach_path, self._env_mach) - def test_too_many_tasks(self): """test trying to use too many tasks""" machine = "derecho" nodes = 1 tasks = 129 - add_args(machine, nodes, tasks) + unit_testing.add_machine_node_args(machine, nodes, tasks) args = get_parser().parse_args() check_parser_args(args) with open(self._jobscript_file, "w", encoding="utf-8") as runfile: @@ -212,7 +86,7 @@ def test_zero_tasks(self): machine = "derecho" nodes = 5 tasks = 0 - add_args(machine, nodes, tasks) + unit_testing.add_machine_node_args(machine, nodes, tasks) args = get_parser().parse_args() with self.assertRaisesRegex( SystemExit, @@ -225,7 +99,7 @@ def test_bld_build_path(self): machine = "derecho" nodes = 10 tasks = 64 - add_args(machine, nodes, tasks) + unit_testing.add_machine_node_args(machine, nodes, tasks) # Remove the build path directory shutil.rmtree(self._bld_path, ignore_errors=True) args = get_parser().parse_args() @@ -237,7 +111,7 @@ def test_mksurfdata_exist(self): machine = "derecho" nodes = 10 tasks = 64 - add_args(machine, nodes, tasks) + unit_testing.add_machine_node_args(machine, nodes, tasks) args = get_parser().parse_args() os.remove(self._mksurf_exe) with self.assertRaisesRegex(SystemExit, "mksurfdata_esmf executable "): @@ -248,7 +122,7 @@ def test_env_mach_specific_exist(self): machine = "derecho" nodes = 10 tasks = 64 - add_args(machine, nodes, tasks) + unit_testing.add_machine_node_args(machine, nodes, tasks) args = get_parser().parse_args() os.remove(self._env_mach) with self.assertRaisesRegex(SystemExit, "Environment machine specific file"): @@ -259,7 +133,7 @@ def test_bad_machine(self): machine = "zztop" nodes = 1 tasks = 64 - add_args(machine, nodes, tasks) + unit_testing.add_machine_node_args(machine, nodes, tasks) with self.assertRaises(SystemExit): get_parser().parse_args() diff --git a/python/ctsm/test/test_unit_longitude.py b/python/ctsm/test/test_unit_longitude.py index 6766f90764..2382b2c303 100644 --- a/python/ctsm/test/test_unit_longitude.py +++ b/python/ctsm/test/test_unit_longitude.py @@ -446,6 +446,27 @@ def test_lon_type_getter(self): lon = Longitude(55, 180) self.assertEqual(lon.lon_type(), 180) + def test_no_implicit_string_conversion(self): + """Ensure that implicit string conversion is disallowed""" + lon = Longitude(55, 180) + with self.assertRaisesRegex( + NotImplementedError, r"Use Longitude\.get_str\(\) instead of implicit string conversion" + ): + _ = f"{lon}" + with self.assertRaisesRegex( + NotImplementedError, r"Use Longitude\.get_str\(\) instead of implicit string conversion" + ): + _ = str(lon) + + def test_get_str(self): + """Ensure that explicit string conversion works as expected""" + lon = Longitude(55, 180) + self.assertEqual(lon.get_str(180), "55.0") + self.assertEqual(lon.get_str(360), "55.0") + lon = Longitude(-55, 180) + self.assertEqual(lon.get_str(180), "-55.0") + self.assertEqual(lon.get_str(360), "305.0") + if __name__ == "__main__": unit_testing.setup_for_tests() diff --git a/python/ctsm/test/test_unit_singlept_data.py b/python/ctsm/test/test_unit_singlept_data.py index bf29ced331..489763282d 100755 --- a/python/ctsm/test/test_unit_singlept_data.py +++ b/python/ctsm/test/test_unit_singlept_data.py @@ -19,6 +19,7 @@ from ctsm import unit_testing from ctsm.site_and_regional.single_point_case import SinglePointCase from ctsm.pft_utils import MAX_PFT_GENERICCROPS, MAX_PFT_MANAGEDCROPS +from ctsm.longitude import Longitude # pylint: disable=invalid-name @@ -29,7 +30,7 @@ class TestSinglePointCase(unittest.TestCase): """ plat = 20.1 - plon = 50.5 + plon = Longitude(50.5, lon_type=180) site_name = None create_domain = True create_surfdata = True diff --git a/python/ctsm/test/test_unit_singlept_data_surfdata.py b/python/ctsm/test/test_unit_singlept_data_surfdata.py index d163c29e4f..11ee416d4a 100755 --- a/python/ctsm/test/test_unit_singlept_data_surfdata.py +++ b/python/ctsm/test/test_unit_singlept_data_surfdata.py @@ -24,6 +24,7 @@ from ctsm import unit_testing from ctsm.site_and_regional.single_point_case import SinglePointCase from ctsm.pft_utils import MAX_PFT_GENERICCROPS, MAX_PFT_MANAGEDCROPS +from ctsm.longitude import Longitude # pylint: disable=invalid-name # pylint: disable=too-many-lines @@ -37,7 +38,7 @@ class TestSinglePointCaseSurfaceNoCrop(unittest.TestCase): """ plat = 20.1 - plon = 50.5 + plon = Longitude(50.5, lon_type=180) site_name = None create_domain = True create_surfdata = True @@ -658,7 +659,7 @@ class TestSinglePointCaseSurfaceCrop(unittest.TestCase): """ plat = 20.1 - plon = 50.5 + plon = Longitude(50.5, lon_type=180) site_name = None create_domain = True create_surfdata = True diff --git a/python/ctsm/test/test_unit_sspmatrix.py b/python/ctsm/test/test_unit_sspmatrix.py index 1b1bc60185..dd81a7df4f 100755 --- a/python/ctsm/test/test_unit_sspmatrix.py +++ b/python/ctsm/test/test_unit_sspmatrix.py @@ -53,7 +53,7 @@ def create_clone( Extend to handle creation of user_nl_clm file """ clone = super().create_clone(newcase, keepexe=keepexe) - os.mknod(os.path.join(newcase, "user_nl_clm")) + Path.touch(os.path.join(newcase, "user_nl_clm")) # Also make the needed case directories clone.make_case_dirs(self._tempdir) return clone @@ -165,7 +165,7 @@ def test_append_user_nl_step2(self): if os.path.exists(ufile): os.remove(ufile) - os.mknod(ufile) + Path.touch(ufile) expect = "\nhist_nhtfrq = -8760, hist_mfilt = 2\n" self.ssp.append_user_nl(caseroot=".", n=2) diff --git a/python/ctsm/test/test_unit_subset_data.py b/python/ctsm/test/test_unit_subset_data.py index c4ce21e959..a127a282e0 100755 --- a/python/ctsm/test/test_unit_subset_data.py +++ b/python/ctsm/test/test_unit_subset_data.py @@ -104,7 +104,7 @@ def test_inputdata_setup_files_basic(self): Test """ self.args = check_args(self.args) - files = setup_files(self.args, self.defaults, self.cesmroot) + files = setup_files(self.args, self.defaults, self.cesmroot, testing=True) self.assertEqual( files["fsurf_in"], "surfdata_0.9x1.25_hist_2000_16pfts_c240908.nc", @@ -184,7 +184,7 @@ def test_check_args_outsurfdat_provided(self): sys.argv = ["subset_data", "point", "--create-surface", "--out-surface", "outputsurface.nc"] self.args = self.parser.parse_args() self.args = check_args(self.args) - files = setup_files(self.args, self.defaults, self.cesmroot) + files = setup_files(self.args, self.defaults, self.cesmroot, testing=True) self.assertEqual( files["fsurf_out"], "outputsurface.nc", diff --git a/python/ctsm/test/testinputs/expected_result_files/test_subset_data_pt_datm_amazon_type180_nositename/datmdata/clmforc.CRUJRAv2.5_0.5x0.5.Prec.-69.0_-12.0.1986.nc b/python/ctsm/test/testinputs/expected_result_files/test_subset_data_pt_datm_amazon_type180_nositename/datmdata/clmforc.CRUJRAv2.5_0.5x0.5.Prec.-69.0_-12.0.1986.nc new file mode 120000 index 0000000000..0e14bd986a --- /dev/null +++ b/python/ctsm/test/testinputs/expected_result_files/test_subset_data_pt_datm_amazon_type180_nositename/datmdata/clmforc.CRUJRAv2.5_0.5x0.5.Prec.-69.0_-12.0.1986.nc @@ -0,0 +1 @@ +../../test_subset_data_pt_datm_amazon_type180/datmdata//clmforc.CRUJRAv2.5_0.5x0.5.Prec.TMP.1986.nc \ No newline at end of file diff --git a/python/ctsm/test/testinputs/expected_result_files/test_subset_data_pt_datm_amazon_type180_nositename/datmdata/clmforc.CRUJRAv2.5_0.5x0.5.Prec.-69.0_-12.0.1987.nc b/python/ctsm/test/testinputs/expected_result_files/test_subset_data_pt_datm_amazon_type180_nositename/datmdata/clmforc.CRUJRAv2.5_0.5x0.5.Prec.-69.0_-12.0.1987.nc new file mode 120000 index 0000000000..28b7abf80d --- /dev/null +++ b/python/ctsm/test/testinputs/expected_result_files/test_subset_data_pt_datm_amazon_type180_nositename/datmdata/clmforc.CRUJRAv2.5_0.5x0.5.Prec.-69.0_-12.0.1987.nc @@ -0,0 +1 @@ +../../test_subset_data_pt_datm_amazon_type180/datmdata//clmforc.CRUJRAv2.5_0.5x0.5.Prec.TMP.1987.nc \ No newline at end of file diff --git a/python/ctsm/test/testinputs/expected_result_files/test_subset_data_pt_datm_amazon_type180_nositename/datmdata/clmforc.CRUJRAv2.5_0.5x0.5.Prec.-69.0_-12.0.1988.nc b/python/ctsm/test/testinputs/expected_result_files/test_subset_data_pt_datm_amazon_type180_nositename/datmdata/clmforc.CRUJRAv2.5_0.5x0.5.Prec.-69.0_-12.0.1988.nc new file mode 120000 index 0000000000..a238ab07c2 --- /dev/null +++ b/python/ctsm/test/testinputs/expected_result_files/test_subset_data_pt_datm_amazon_type180_nositename/datmdata/clmforc.CRUJRAv2.5_0.5x0.5.Prec.-69.0_-12.0.1988.nc @@ -0,0 +1 @@ +../../test_subset_data_pt_datm_amazon_type180/datmdata//clmforc.CRUJRAv2.5_0.5x0.5.Prec.TMP.1988.nc \ No newline at end of file diff --git a/python/ctsm/test/testinputs/expected_result_files/test_subset_data_pt_datm_amazon_type180_nositename/datmdata/clmforc.CRUJRAv2.5_0.5x0.5.Solr.-69.0_-12.0.1986.nc b/python/ctsm/test/testinputs/expected_result_files/test_subset_data_pt_datm_amazon_type180_nositename/datmdata/clmforc.CRUJRAv2.5_0.5x0.5.Solr.-69.0_-12.0.1986.nc new file mode 120000 index 0000000000..a2045e914c --- /dev/null +++ b/python/ctsm/test/testinputs/expected_result_files/test_subset_data_pt_datm_amazon_type180_nositename/datmdata/clmforc.CRUJRAv2.5_0.5x0.5.Solr.-69.0_-12.0.1986.nc @@ -0,0 +1 @@ +../../test_subset_data_pt_datm_amazon_type180/datmdata//clmforc.CRUJRAv2.5_0.5x0.5.Solr.TMP.1986.nc \ No newline at end of file diff --git a/python/ctsm/test/testinputs/expected_result_files/test_subset_data_pt_datm_amazon_type180_nositename/datmdata/clmforc.CRUJRAv2.5_0.5x0.5.Solr.-69.0_-12.0.1987.nc b/python/ctsm/test/testinputs/expected_result_files/test_subset_data_pt_datm_amazon_type180_nositename/datmdata/clmforc.CRUJRAv2.5_0.5x0.5.Solr.-69.0_-12.0.1987.nc new file mode 120000 index 0000000000..24cc171353 --- /dev/null +++ b/python/ctsm/test/testinputs/expected_result_files/test_subset_data_pt_datm_amazon_type180_nositename/datmdata/clmforc.CRUJRAv2.5_0.5x0.5.Solr.-69.0_-12.0.1987.nc @@ -0,0 +1 @@ +../../test_subset_data_pt_datm_amazon_type180/datmdata//clmforc.CRUJRAv2.5_0.5x0.5.Solr.TMP.1987.nc \ No newline at end of file diff --git a/python/ctsm/test/testinputs/expected_result_files/test_subset_data_pt_datm_amazon_type180_nositename/datmdata/clmforc.CRUJRAv2.5_0.5x0.5.Solr.-69.0_-12.0.1988.nc b/python/ctsm/test/testinputs/expected_result_files/test_subset_data_pt_datm_amazon_type180_nositename/datmdata/clmforc.CRUJRAv2.5_0.5x0.5.Solr.-69.0_-12.0.1988.nc new file mode 120000 index 0000000000..00eacece43 --- /dev/null +++ b/python/ctsm/test/testinputs/expected_result_files/test_subset_data_pt_datm_amazon_type180_nositename/datmdata/clmforc.CRUJRAv2.5_0.5x0.5.Solr.-69.0_-12.0.1988.nc @@ -0,0 +1 @@ +../../test_subset_data_pt_datm_amazon_type180/datmdata//clmforc.CRUJRAv2.5_0.5x0.5.Solr.TMP.1988.nc \ No newline at end of file diff --git a/python/ctsm/test/testinputs/expected_result_files/test_subset_data_pt_datm_amazon_type180_nositename/datmdata/clmforc.CRUJRAv2.5_0.5x0.5.TPQWL.-69.0_-12.0.1986.nc b/python/ctsm/test/testinputs/expected_result_files/test_subset_data_pt_datm_amazon_type180_nositename/datmdata/clmforc.CRUJRAv2.5_0.5x0.5.TPQWL.-69.0_-12.0.1986.nc new file mode 120000 index 0000000000..3806e36151 --- /dev/null +++ b/python/ctsm/test/testinputs/expected_result_files/test_subset_data_pt_datm_amazon_type180_nositename/datmdata/clmforc.CRUJRAv2.5_0.5x0.5.TPQWL.-69.0_-12.0.1986.nc @@ -0,0 +1 @@ +../../test_subset_data_pt_datm_amazon_type180/datmdata//clmforc.CRUJRAv2.5_0.5x0.5.TPQWL.TMP.1986.nc \ No newline at end of file diff --git a/python/ctsm/test/testinputs/expected_result_files/test_subset_data_pt_datm_amazon_type180_nositename/datmdata/clmforc.CRUJRAv2.5_0.5x0.5.TPQWL.-69.0_-12.0.1987.nc b/python/ctsm/test/testinputs/expected_result_files/test_subset_data_pt_datm_amazon_type180_nositename/datmdata/clmforc.CRUJRAv2.5_0.5x0.5.TPQWL.-69.0_-12.0.1987.nc new file mode 120000 index 0000000000..44ce035a2a --- /dev/null +++ b/python/ctsm/test/testinputs/expected_result_files/test_subset_data_pt_datm_amazon_type180_nositename/datmdata/clmforc.CRUJRAv2.5_0.5x0.5.TPQWL.-69.0_-12.0.1987.nc @@ -0,0 +1 @@ +../../test_subset_data_pt_datm_amazon_type180/datmdata//clmforc.CRUJRAv2.5_0.5x0.5.TPQWL.TMP.1987.nc \ No newline at end of file diff --git a/python/ctsm/test/testinputs/expected_result_files/test_subset_data_pt_datm_amazon_type180_nositename/datmdata/clmforc.CRUJRAv2.5_0.5x0.5.TPQWL.-69.0_-12.0.1988.nc b/python/ctsm/test/testinputs/expected_result_files/test_subset_data_pt_datm_amazon_type180_nositename/datmdata/clmforc.CRUJRAv2.5_0.5x0.5.TPQWL.-69.0_-12.0.1988.nc new file mode 120000 index 0000000000..cd8cdfb7c9 --- /dev/null +++ b/python/ctsm/test/testinputs/expected_result_files/test_subset_data_pt_datm_amazon_type180_nositename/datmdata/clmforc.CRUJRAv2.5_0.5x0.5.TPQWL.-69.0_-12.0.1988.nc @@ -0,0 +1 @@ +../../test_subset_data_pt_datm_amazon_type180/datmdata//clmforc.CRUJRAv2.5_0.5x0.5.TPQWL.TMP.1988.nc \ No newline at end of file diff --git a/python/ctsm/test/testinputs/expected_result_files/test_subset_data_pt_datm_amazon_type180_nositename/datmdata/domain.crujra_v2.3_0.5x0.5_-69.0_-12.0_c250620.nc b/python/ctsm/test/testinputs/expected_result_files/test_subset_data_pt_datm_amazon_type180_nositename/datmdata/domain.crujra_v2.3_0.5x0.5_-69.0_-12.0_c250620.nc new file mode 120000 index 0000000000..1dc0d0d5f2 --- /dev/null +++ b/python/ctsm/test/testinputs/expected_result_files/test_subset_data_pt_datm_amazon_type180_nositename/datmdata/domain.crujra_v2.3_0.5x0.5_-69.0_-12.0_c250620.nc @@ -0,0 +1 @@ +../../test_subset_data_pt_datm_amazon_type180/datmdata//domain.crujra_v2.3_0.5x0.5_TMP_c250620.nc \ No newline at end of file diff --git a/python/ctsm/test/testinputs/expected_result_files/test_subset_data_pt_landuse_amazon_type360_nositename/landuse.timeseries_291.0_-12.0_amazon_hist_1850-1853_78pfts_c250618.nc b/python/ctsm/test/testinputs/expected_result_files/test_subset_data_pt_landuse_amazon_type360_nositename/landuse.timeseries_291.0_-12.0_amazon_hist_1850-1853_78pfts_c250618.nc new file mode 120000 index 0000000000..8678639d98 --- /dev/null +++ b/python/ctsm/test/testinputs/expected_result_files/test_subset_data_pt_landuse_amazon_type360_nositename/landuse.timeseries_291.0_-12.0_amazon_hist_1850-1853_78pfts_c250618.nc @@ -0,0 +1 @@ +../test_subset_data_pt_landuse_amazon_type360/landuse.timeseries_TMP_amazon_hist_1850-1853_78pfts_c250618.nc \ No newline at end of file diff --git a/python/ctsm/test/testinputs/expected_result_files/test_subset_data_pt_landuse_amazon_type360_nositename/surfdata_291.0_-12.0_amazon_hist_1850_78pfts_c250618.nc b/python/ctsm/test/testinputs/expected_result_files/test_subset_data_pt_landuse_amazon_type360_nositename/surfdata_291.0_-12.0_amazon_hist_1850_78pfts_c250618.nc new file mode 120000 index 0000000000..2efbb745f4 --- /dev/null +++ b/python/ctsm/test/testinputs/expected_result_files/test_subset_data_pt_landuse_amazon_type360_nositename/surfdata_291.0_-12.0_amazon_hist_1850_78pfts_c250618.nc @@ -0,0 +1 @@ +../test_subset_data_pt_landuse_amazon_type360/surfdata_TMP_amazon_hist_1850_78pfts_c250618.nc \ No newline at end of file diff --git a/python/ctsm/test/testinputs/expected_result_files/test_subset_data_pt_surface_amazon_type180_nositename/surfdata_-69.0_-12.0_amazon_hist_16pfts_CMIP6_2000_c250617.nc b/python/ctsm/test/testinputs/expected_result_files/test_subset_data_pt_surface_amazon_type180_nositename/surfdata_-69.0_-12.0_amazon_hist_16pfts_CMIP6_2000_c250617.nc new file mode 120000 index 0000000000..9e811ca9c3 --- /dev/null +++ b/python/ctsm/test/testinputs/expected_result_files/test_subset_data_pt_surface_amazon_type180_nositename/surfdata_-69.0_-12.0_amazon_hist_16pfts_CMIP6_2000_c250617.nc @@ -0,0 +1 @@ +../test_subset_data_pt_surface_amazon_type180/surfdata_TMP_amazon_hist_16pfts_CMIP6_2000_c250617.nc \ No newline at end of file diff --git a/python/ctsm/test/testinputs/expected_result_files/test_subset_data_reg_amazon_noregname/domain.lnd.5x5pt-amazon_navy_291.0-299.0_-12.0--7.0_c250508.nc b/python/ctsm/test/testinputs/expected_result_files/test_subset_data_reg_amazon_noregname/domain.lnd.5x5pt-amazon_navy_291.0-299.0_-12.0--7.0_c250508.nc new file mode 120000 index 0000000000..99d8401b46 --- /dev/null +++ b/python/ctsm/test/testinputs/expected_result_files/test_subset_data_reg_amazon_noregname/domain.lnd.5x5pt-amazon_navy_291.0-299.0_-12.0--7.0_c250508.nc @@ -0,0 +1 @@ +../test_subset_data_reg_amazon/domain.lnd.5x5pt-amazon_navy_TMP_c250508.nc \ No newline at end of file diff --git a/python/ctsm/test/testinputs/expected_result_files/test_subset_data_reg_amazon_noregname/domain.lnd.5x5pt-amazon_navy_291.0-299.0_-12.0--7.0_c250508_ESMF_UNSTRUCTURED_MESH.nc b/python/ctsm/test/testinputs/expected_result_files/test_subset_data_reg_amazon_noregname/domain.lnd.5x5pt-amazon_navy_291.0-299.0_-12.0--7.0_c250508_ESMF_UNSTRUCTURED_MESH.nc new file mode 120000 index 0000000000..6f8ca4e665 --- /dev/null +++ b/python/ctsm/test/testinputs/expected_result_files/test_subset_data_reg_amazon_noregname/domain.lnd.5x5pt-amazon_navy_291.0-299.0_-12.0--7.0_c250508_ESMF_UNSTRUCTURED_MESH.nc @@ -0,0 +1 @@ +../test_subset_data_reg_amazon/domain.lnd.5x5pt-amazon_navy_TMP_c250508_ESMF_UNSTRUCTURED_MESH.nc \ No newline at end of file diff --git a/python/ctsm/test/testinputs/expected_result_files/test_subset_data_reg_amazon_noregname/surfdata_291.0-299.0_-12.0--7.0_amazon_hist_16pfts_CMIP6_2000_c250508.nc b/python/ctsm/test/testinputs/expected_result_files/test_subset_data_reg_amazon_noregname/surfdata_291.0-299.0_-12.0--7.0_amazon_hist_16pfts_CMIP6_2000_c250508.nc new file mode 120000 index 0000000000..73bde404b0 --- /dev/null +++ b/python/ctsm/test/testinputs/expected_result_files/test_subset_data_reg_amazon_noregname/surfdata_291.0-299.0_-12.0--7.0_amazon_hist_16pfts_CMIP6_2000_c250508.nc @@ -0,0 +1 @@ +../test_subset_data_reg_amazon/surfdata_TMP_amazon_hist_16pfts_CMIP6_2000_c250508.nc \ No newline at end of file diff --git a/python/ctsm/test_gen_mksurfdata_jobscript_single_parent.py b/python/ctsm/test_gen_mksurfdata_jobscript_single_parent.py new file mode 100755 index 0000000000..b6a3741444 --- /dev/null +++ b/python/ctsm/test_gen_mksurfdata_jobscript_single_parent.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python3 + +""" +Parent class for some unittest modules relating to gen_mksurfdata_jobscript_single.py +""" + +import unittest +import os +import sys +import shutil +from pathlib import Path + +import tempfile + +from ctsm.path_utils import path_to_ctsm_root + + +# pylint: disable=too-many-instance-attributes +class TestFGenMkSurfJobscriptSingleParent(unittest.TestCase): + """Parent class for some unittest modules relating to gen_mksurfdata_jobscript_single.py""" + + def setUp(self): + """Setup for trying out the methods""" + testinputs_path = os.path.join(path_to_ctsm_root(), "python/ctsm/test/testinputs") + self._testinputs_path = testinputs_path + self._previous_dir = os.getcwd() + self._tempdir = tempfile.mkdtemp() + os.chdir(self._tempdir) + self._account = "ACCOUNT_NUMBER" + self._jobscript_file = "output_jobscript" + self._output_compare = """#!/bin/bash +# Edit the batch directives for your batch system +# Below are default batch directives for derecho +#PBS -N mksurfdata +#PBS -j oe +#PBS -k eod +#PBS -S /bin/bash +#PBS -l walltime=12:00:00 +#PBS -A ACCOUNT_NUMBER +#PBS -q main +#PBS -l select=1:ncpus=128:mpiprocs=64:mem=218GB + +# This is a batch script to run a set of resolutions for mksurfdata_esmf input namelist +# NOTE: THIS SCRIPT IS AUTOMATICALLY GENERATED SO IN GENERAL YOU SHOULD NOT EDIT it!! + +""" + self._bld_path = os.path.join(self._tempdir, "tools_bld") + os.makedirs(self._bld_path) + self.assertTrue(os.path.isdir(self._bld_path)) + self._nlfile = os.path.join(self._tempdir, "namelist_file") + Path.touch(self._nlfile) + self.assertTrue(os.path.exists(self._nlfile)) + self._mksurf_exe = os.path.join(self._bld_path, "mksurfdata") + Path.touch(self._mksurf_exe) + self.assertTrue(os.path.exists(self._mksurf_exe)) + self._env_mach = os.path.join(self._bld_path, ".env_mach_specific.sh") + Path.touch(self._env_mach) + self.assertTrue(os.path.exists(self._env_mach)) + sys.argv = [ + "gen_mksurfdata_jobscript_single", + "--bld-path", + self._bld_path, + "--namelist-file", + self._nlfile, + "--jobscript-file", + self._jobscript_file, + "--account", + self._account, + ] + + def tearDown(self): + """ + Remove temporary directory + """ + os.chdir(self._previous_dir) + shutil.rmtree(self._tempdir, ignore_errors=True) diff --git a/python/ctsm/unit_testing.py b/python/ctsm/unit_testing.py index d3a308c796..8370830b4d 100644 --- a/python/ctsm/unit_testing.py +++ b/python/ctsm/unit_testing.py @@ -1,8 +1,22 @@ """Functions to aid unit tests""" +import sys from ctsm.ctsm_logging import setup_logging_for_tests +def add_machine_node_args(machine, nodes, tasks): + """add arguments to sys.argv""" + args_to_add = [ + "--machine", + machine, + "--number-of-nodes", + str(nodes), + "--tasks-per-node", + str(tasks), + ] + sys.argv += args_to_add + + def setup_for_tests(enable_critical_logs=False): """Call this at the beginning of unit testing diff --git a/src/cpl/nuopc/lnd_comp_nuopc.F90 b/src/cpl/nuopc/lnd_comp_nuopc.F90 index b7ef7216d9..af1426bf9b 100644 --- a/src/cpl/nuopc/lnd_comp_nuopc.F90 +++ b/src/cpl/nuopc/lnd_comp_nuopc.F90 @@ -642,8 +642,10 @@ subroutine InitializeRealize(gcomp, importState, exportState, clock, rc) if (ChkErr(rc,__LINE__,u_FILE_u)) return call ESMF_GridCompGet(gcomp, vm=vm, rc=rc) if (ChkErr(rc,__LINE__,u_FILE_u)) return + call t_startf ('lc_lnd_set_decomp_and_domain_from_readmesh') call lnd_set_decomp_and_domain_from_readmesh(driver='cmeps', vm=vm, & meshfile_lnd=model_meshfile, meshfile_mask=meshfile_mask, mesh_ctsm=mesh, ni=ni, nj=nj, rc=rc) + call t_stopf ('lc_lnd_set_decomp_and_domain_from_readmesh') if (ChkErr(rc,__LINE__,u_FILE_u)) return end if diff --git a/src/init_interp/initInterp.F90 b/src/init_interp/initInterp.F90 index c8f80e16ae..3ccdcd9b58 100644 --- a/src/init_interp/initInterp.F90 +++ b/src/init_interp/initInterp.F90 @@ -75,6 +75,9 @@ module initInterpMod ! patch-level variables) logical :: init_interp_fill_missing_with_natveg + ! If true, fill missing urban landunit type with closest urban high density (HD) landunit + logical :: init_interp_fill_missing_urban_with_HD + character(len=*), parameter, private :: sourcefile = & __FILE__ @@ -106,11 +109,13 @@ subroutine initInterp_readnl(NLFilename) !----------------------------------------------------------------------- namelist /clm_initinterp_inparm/ & - init_interp_method, init_interp_fill_missing_with_natveg + init_interp_method, init_interp_fill_missing_with_natveg, & + init_interp_fill_missing_urban_with_HD ! Initialize options to default values, in case they are not specified in the namelist init_interp_method = ' ' init_interp_fill_missing_with_natveg = .false. + init_interp_fill_missing_urban_with_HD = .false. if (masterproc) then unitn = getavu() @@ -130,6 +135,7 @@ subroutine initInterp_readnl(NLFilename) call shr_mpi_bcast (init_interp_method, mpicom) call shr_mpi_bcast (init_interp_fill_missing_with_natveg, mpicom) + call shr_mpi_bcast (init_interp_fill_missing_urban_with_HD, mpicom) if (masterproc) then write(iulog,*) ' ' @@ -287,12 +293,36 @@ subroutine initInterp (filei, fileo, bounds, glc_behavior) status = pio_get_att(ncidi, pio_global, & 'icol_vegetated_or_bare_soil', & subgrid_special_indices%icol_vegetated_or_bare_soil) + status = pio_get_att(ncidi, pio_global, & + 'icol_urban_roof', & + subgrid_special_indices%icol_urban_roof) + status = pio_get_att(ncidi, pio_global, & + 'icol_urban_sunwall', & + subgrid_special_indices%icol_urban_sunwall) + status = pio_get_att(ncidi, pio_global, & + 'icol_urban_shadewall', & + subgrid_special_indices%icol_urban_shadewall) + status = pio_get_att(ncidi, pio_global, & + 'icol_urban_impervious_road', & + subgrid_special_indices%icol_urban_impervious_road) + status = pio_get_att(ncidi, pio_global, & + 'icol_urban_pervious_road', & + subgrid_special_indices%icol_urban_pervious_road) status = pio_get_att(ncidi, pio_global, & 'ilun_vegetated_or_bare_soil', & subgrid_special_indices%ilun_vegetated_or_bare_soil) status = pio_get_att(ncidi, pio_global, & 'ilun_crop', & subgrid_special_indices%ilun_crop) + status = pio_get_att(ncidi, pio_global, & + 'ilun_urban_tbd', & + subgrid_special_indices%ilun_urban_TBD) + status = pio_get_att(ncidi, pio_global, & + 'ilun_urban_hd', & + subgrid_special_indices%ilun_urban_HD) + status = pio_get_att(ncidi, pio_global, & + 'ilun_urban_md', & + subgrid_special_indices%ilun_urban_MD) ! BACKWARDS_COMPATIBILITY(wjs, 2021-04-16) ilun_landice_multiple_elevation_classes has ! been renamed to ilun_landice. For now we need to handle both possibilities for the @@ -321,10 +351,26 @@ subroutine initInterp (filei, fileo, bounds, glc_behavior) subgrid_special_indices%ipft_not_vegetated write(iulog,*)'icol_vegetated_or_bare_soil = ' , & subgrid_special_indices%icol_vegetated_or_bare_soil + write(iulog,*)'icol_urban_roof = ' , & + subgrid_special_indices%icol_urban_roof + write(iulog,*)'icol_urban_sunwall = ' , & + subgrid_special_indices%icol_urban_sunwall + write(iulog,*)'icol_urban_shadewall = ' , & + subgrid_special_indices%icol_urban_shadewall + write(iulog,*)'icol_urban_impervious_road = ' , & + subgrid_special_indices%icol_urban_impervious_road + write(iulog,*)'icol_urban_pervious_road = ' , & + subgrid_special_indices%icol_urban_pervious_road write(iulog,*)'ilun_vegetated_or_bare_soil = ' , & subgrid_special_indices%ilun_vegetated_or_bare_soil write(iulog,*)'ilun_crop = ' , & subgrid_special_indices%ilun_crop + write(iulog,*)'ilun_urban_tbd = ' , & + subgrid_special_indices%ilun_urban_TBD + write(iulog,*)'ilun_urban_hd = ' , & + subgrid_special_indices%ilun_urban_HD + write(iulog,*)'ilun_urban_md = ' , & + subgrid_special_indices%ilun_urban_MD write(iulog,*)'ilun_landice = ' , & subgrid_special_indices%ilun_landice write(iulog,*)'create_glacier_mec_landunits = ', & @@ -839,6 +885,7 @@ subroutine findMinDist( dimname, begi, endi, bego, endo, ncidi, ncido, & glc_behavior=glc_behavior, & glc_elevclasses_same=glc_elevclasses_same, & fill_missing_with_natveg=init_interp_fill_missing_with_natveg, & + fill_missing_urban_with_HD=init_interp_fill_missing_urban_with_HD, & mindist_index=minindx) case (interp_method_finidat_areas) if (masterproc) then diff --git a/src/init_interp/initInterpMindist.F90 b/src/init_interp/initInterpMindist.F90 index f6853b1cd3..9fdc9f81dd 100644 --- a/src/init_interp/initInterpMindist.F90 +++ b/src/init_interp/initInterpMindist.F90 @@ -32,11 +32,20 @@ module initInterpMindist type, public :: subgrid_special_indices_type integer :: ipft_not_vegetated integer :: icol_vegetated_or_bare_soil + integer :: icol_urban_roof + integer :: icol_urban_sunwall + integer :: icol_urban_shadewall + integer :: icol_urban_impervious_road + integer :: icol_urban_pervious_road integer :: ilun_vegetated_or_bare_soil integer :: ilun_crop integer :: ilun_landice + integer :: ilun_urban_TBD + integer :: ilun_urban_HD + integer :: ilun_urban_MD contains procedure :: is_vegetated_landunit ! returns true if the given landunit type is natural veg or crop + procedure :: is_urban_landunit ! returns true if the given landunit type is urban end type subgrid_special_indices_type type, public :: subgrid_type @@ -58,8 +67,10 @@ module initInterpMindist private :: set_glc_must_be_same_type private :: set_ice_adjustable_type private :: do_fill_missing_with_natveg + private :: do_fill_missing_urban_with_HD private :: is_sametype private :: is_baresoil + private :: is_urban_HD character(len=*), parameter, private :: sourcefile = & __FILE__ @@ -147,7 +158,7 @@ end subroutine destroy_subgrid_type subroutine set_mindist(begi, endi, bego, endo, activei, activeo, subgridi, subgrido, & subgrid_special_indices, glc_behavior, glc_elevclasses_same, & - fill_missing_with_natveg, mindist_index) + fill_missing_with_natveg, fill_missing_urban_with_HD, mindist_index) ! -------------------------------------------------------------------- ! arguments @@ -165,7 +176,7 @@ subroutine set_mindist(begi, endi, bego, endo, activei, activeo, subgridi, subgr logical , intent(in) :: glc_elevclasses_same ! If false: if an output type cannot be found in the input, code aborts - ! If true: if an output type cannot be found in the input, fill with closest natural + ! If true: if a non-urban output type cannot be found in the input, fill with closest natural ! veg column (using bare soil for patch-level variables) ! ! NOTE: always treated as true for natural veg and crop landunits/columns/patches in @@ -173,6 +184,11 @@ subroutine set_mindist(begi, endi, bego, endo, activei, activeo, subgridi, subgr ! use the closest natural veg column, regardless of the value of this flag. logical , intent(in) :: fill_missing_with_natveg + + ! If false: if an urban output type cannot be found in the input, code aborts + ! If true: if an urban output type cannot be found in the input, fill with closest urban HD + logical , intent(in) :: fill_missing_urban_with_HD + integer , intent(out) :: mindist_index(bego:endo) ! ! local variables @@ -187,6 +203,8 @@ subroutine set_mindist(begi, endi, bego, endo, activei, activeo, subgridi, subgr ! considered the same type. This is only valid for glc points, and is only valid ! for subgrid name = 'pft' or 'column'. logical :: glc_must_be_same_type_o(bego:endo) + + character(len=*), parameter :: subname = 'set_mindist' ! -------------------------------------------------------------------- if (associated(subgridi%topoglc) .and. associated(subgrido%topoglc)) then @@ -221,7 +239,8 @@ subroutine set_mindist(begi, endi, bego, endo, activei, activeo, subgridi, subgr subgridi = subgridi, subgrido = subgrido, & subgrid_special_indices = subgrid_special_indices, & glc_must_be_same_type = glc_must_be_same_type_o(no), & - veg_patch_just_considers_ptype = .true.)) then + veg_patch_just_considers_ptype = .true., & + do_fill_missing_urban_with_HD = .false.)) then dy = abs(subgrido%lat(no)-subgridi%lat(ni))*re dx = abs(subgrido%lon(no)-subgridi%lon(ni))*re * & 0.5_r8*(subgrido%coslat(no)+subgridi%coslat(ni)) @@ -260,7 +279,11 @@ subroutine set_mindist(begi, endi, bego, endo, activei, activeo, subgridi, subgr end if end do - ! If output type is not contained in input dataset, then use closest bare soil, + ! Note that do_fill_missing_with_natveg below will return .false. for pfts and columnns associated + ! with urban landunits so that the fill missing with bare soil will be implemented only for + ! non-urban types (pfts, columns, landunits, gridcells). + + ! If non-urban output type is not contained in input dataset, then use closest bare soil, ! if this point is one for which we fill missing with natveg. if ( distmin == spval .and. & do_fill_missing_with_natveg( & @@ -279,6 +302,50 @@ subroutine set_mindist(begi, endi, bego, endo, activei, activeo, subgridi, subgr end if end if end do + + ! If urban output type is not contained in input dataset, then use closest urban HD, + ! if this point is one for which we fill missing urban with urban HD. + else if (distmin == spval & + .and. do_fill_missing_urban_with_HD( & + fill_missing_urban_with_HD, no, subgrido, subgrid_special_indices)) then + do ni = begi, endi + if (activei(ni)) then + ! We need to call is_sametype for pfts and columns here to make sure that each + ! urban input pft and column type matches the output pft and column type. We don't + ! want to call it for landunits because they intentionally won't be the same type + ! (since we are filling missing urban landunits with HD) + if (subgrido%name .eq. 'landunit') then + if ( is_urban_HD(ni, subgridi, subgrid_special_indices)) then + dy = abs(subgrido%lat(no)-subgridi%lat(ni))*re + dx = abs(subgrido%lon(no)-subgridi%lon(ni))*re * & + 0.5_r8*(subgrido%coslat(no)+subgridi%coslat(ni)) + dist = dx*dx + dy*dy + if ( dist < distmin )then + distmin = dist + nmin = ni + end if + end if + else + if (is_sametype(ni = ni, no = no, & + subgridi = subgridi, subgrido = subgrido, & + subgrid_special_indices = subgrid_special_indices, & + glc_must_be_same_type = glc_must_be_same_type_o(no), & + veg_patch_just_considers_ptype = .false., & + do_fill_missing_urban_with_HD = .true.)) then + if ( is_urban_HD(ni, subgridi, subgrid_special_indices)) then + dy = abs(subgrido%lat(no)-subgridi%lat(ni))*re + dx = abs(subgrido%lon(no)-subgridi%lon(ni))*re * & + 0.5_r8*(subgrido%coslat(no)+subgridi%coslat(ni)) + dist = dx*dx + dy*dy + if ( dist < distmin )then + distmin = dist + nmin = ni + end if + end if + end if + end if + end if + end do end if ! Error conditions @@ -287,13 +354,29 @@ subroutine set_mindist(begi, endi, bego, endo, activei, activeo, subgridi, subgr &Cannot find any input points matching output point:' call subgrido%print_point(no, iulog) write(iulog,*) ' ' - write(iulog,*) 'Consider rerunning with the following in user_nl_clm:' + write(iulog,*) 'If this is an urban type' + write(iulog,*) '(ltype = ', subgrid_special_indices%ilun_urban_TBD, & + ',', subgrid_special_indices%ilun_urban_HD, & + ', or', subgrid_special_indices%ilun_urban_MD, ')' + write(iulog,*) 'then consider rerunning with the following in user_nl_clm:' + write(iulog,*) 'init_interp_fill_missing_urban_with_HD = .true.' + write(iulog,*) 'However, note that this will fill all urban missing types in the output' + write(iulog,*) 'with the closest urban high density (HD) type in the input' + write(iulog,*) 'So, you should consider whether that is what you want.' + write(iulog,*) ' ' + write(iulog,*) 'If this is a non-urban type' + write(iulog,*) '(ltype \= ',subgrid_special_indices%ilun_urban_TBD, & + ',', subgrid_special_indices%ilun_urban_HD, & + ', or', subgrid_special_indices%ilun_urban_MD, ')' + write(iulog,*) 'consider rerunning with the following in user_nl_clm:' write(iulog,*) 'init_interp_fill_missing_with_natveg = .true.' - write(iulog,*) 'However, note that this will fill all missing types in the output' + write(iulog,*) 'However, note that this will fill all non-urban missing types in the output' write(iulog,*) 'with the closest natural veg column in the input' write(iulog,*) '(using bare soil for patch-level variables).' write(iulog,*) 'So, you should consider whether that is what you want.' - call endrun(msg=errMsg(sourcefile, __LINE__)) + write(iulog,*) errMsg(sourcefile, __LINE__) + call endrun(msg=subname// & + ' ERROR: Cannot find any input points matching output point') end if mindist_index(no) = nmin @@ -378,7 +461,8 @@ subroutine set_single_match(begi, endi, bego, endo, activeo, subgridi, subgrido, subgridi = subgridi, subgrido = subgrido, & subgrid_special_indices = subgrid_special_indices, & glc_must_be_same_type = glc_must_be_same_type_o(no), & - veg_patch_just_considers_ptype = .false.) + veg_patch_just_considers_ptype = .false., & + do_fill_missing_urban_with_HD = .false.) if (ni_sametype) then if (found) then write(iulog,*) subname// & @@ -555,7 +639,7 @@ function do_fill_missing_with_natveg(fill_missing_with_natveg, & no, subgrido, subgrid_special_indices) ! ! !DESCRIPTION: - ! Returns true if the given output point, if missing, should be filled with the + ! Returns true if the given non-urban output point, if missing, should be filled with the ! closest natural veg point. ! ! !ARGUMENTS: @@ -576,8 +660,8 @@ function do_fill_missing_with_natveg(fill_missing_with_natveg, & if (subgrido%name == 'gridcell') then ! It makes no sense to try to fill missing with natveg for gridcell-level values do_fill_missing_with_natveg = .false. - else if (fill_missing_with_natveg) then - ! User has asked for all missing points to be filled with natveg + else if (fill_missing_with_natveg .and. .not. subgrid_special_indices%is_urban_landunit(subgrido%ltype(no))) then + ! User has asked for all non-urban missing points to be filled with natveg do_fill_missing_with_natveg = .true. else if (subgrid_special_indices%is_vegetated_landunit(subgrido%ltype(no))) then ! Even if user hasn't asked for it, we fill missing vegetated points (natural veg @@ -591,11 +675,46 @@ function do_fill_missing_with_natveg(fill_missing_with_natveg, & end function do_fill_missing_with_natveg + !----------------------------------------------------------------------- + function do_fill_missing_urban_with_HD(fill_missing_urban_with_HD, & + no, subgrido, subgrid_special_indices) + ! + ! !DESCRIPTION: + ! Returns true if the given urban output point, if missing, should be filled with the + ! closest urban HD point. + ! + ! !ARGUMENTS: + logical :: do_fill_missing_urban_with_HD ! function result + + ! whether we should fill ALL missing points with urban HD + logical, intent(in) :: fill_missing_urban_with_HD + + integer , intent(in) :: no + type(subgrid_type), intent(in) :: subgrido + type(subgrid_special_indices_type), intent(in) :: subgrid_special_indices + ! + ! !LOCAL VARIABLES: + + character(len=*), parameter :: subname = 'do_fill_missing_urban_with_HD' + !----------------------------------------------------------------------- + + if (subgrido%name == 'gridcell') then + ! It makes no sense to try to fill missing with urban HD for gridcell-level values + do_fill_missing_urban_with_HD = .false. + else if (fill_missing_urban_with_HD) then + ! User has asked for all missing urban points to be filled with urban HD + do_fill_missing_urban_with_HD = .true. + else + do_fill_missing_urban_with_HD = .false. + end if + + end function do_fill_missing_urban_with_HD !======================================================================= logical function is_sametype (ni, no, subgridi, subgrido, subgrid_special_indices, & - glc_must_be_same_type, veg_patch_just_considers_ptype) + glc_must_be_same_type, veg_patch_just_considers_ptype, & + do_fill_missing_urban_with_HD) ! -------------------------------------------------------------------- ! arguments @@ -620,6 +739,12 @@ logical function is_sametype (ni, no, subgridi, subgrido, subgrid_special_indice ! If false, then they need to have the same column and landunit types, too (as is the ! general case). logical, intent(in) :: veg_patch_just_considers_ptype + + ! If True, we allow for landunits to be different when checking if pft and column are + ! the same type, to allow for HD fill of missing urban output points. + logical, intent(in) :: do_fill_missing_urban_with_HD + + ! For urban columns/patches ! -------------------------------------------------------------------- is_sametype = .false. @@ -644,6 +769,10 @@ logical function is_sametype (ni, no, subgridi, subgrido, subgrid_special_indice subgridi%ptype(ni) == subgrido%ptype(no)) then is_sametype = .true. end if + else if (subgridi%ptype(ni) == subgrido%ptype(no) .and. & + subgridi%ctype(ni) == subgrido%ctype(no) .and. & + do_fill_missing_urban_with_HD) then + is_sametype = .true. else if (subgridi%ptype(ni) == subgrido%ptype(no) .and. & subgridi%ctype(ni) == subgrido%ctype(no) .and. & subgridi%ltype(ni) == subgrido%ltype(no)) then @@ -654,6 +783,9 @@ logical function is_sametype (ni, no, subgridi, subgrido, subgrid_special_indice subgridi%ltype(ni) == subgrid_special_indices%ilun_landice .and. & subgrido%ltype(no) == subgrid_special_indices%ilun_landice ) then is_sametype = .true. + else if (subgridi%ctype(ni) == subgrido%ctype(no) .and. & + do_fill_missing_urban_with_HD) then + is_sametype = .true. else if (subgridi%ctype(ni) == subgrido%ctype(no) .and. & subgridi%ltype(ni) == subgrido%ltype(no)) then is_sametype = .true. @@ -712,6 +844,31 @@ logical function is_baresoil (n, subgrid, subgrid_special_indices) end function is_baresoil + !----------------------------------------------------------------------- + logical function is_urban_HD (n, subgrid, subgrid_special_indices) + + ! -------------------------------------------------------------------- + ! arguments + integer , intent(in) :: n + type(subgrid_type), intent(in) :: subgrid + type(subgrid_special_indices_type), intent(in) :: subgrid_special_indices + ! -------------------------------------------------------------------- + + is_urban_HD = .false. + + if (subgrid%name == 'pft' .or. subgrid%name == 'column' .or. subgrid%name == 'landunit') then + if (subgrid%ltype(n) == subgrid_special_indices%ilun_urban_HD) then + is_urban_HD = .true. + end if + else + if (masterproc) then + write(iulog,*)'ERROR interpinic: is_urban_HD subgrid type ',subgrid%name,' not supported' + end if + call endrun(msg=errMsg(sourcefile, __LINE__)) + end if + + end function is_urban_HD + !----------------------------------------------------------------------- function is_vegetated_landunit(this, ltype) ! @@ -739,5 +896,30 @@ function is_vegetated_landunit(this, ltype) end function is_vegetated_landunit + function is_urban_landunit(this, ltype) + ! + ! !DESCRIPTION: + ! Returns true if the given landunit type is urban + ! + ! !USES: + ! + ! !ARGUMENTS: + logical :: is_urban_landunit ! function result + class(subgrid_special_indices_type), intent(in) :: this + integer, intent(in) :: ltype ! landunit type of interest + ! + ! !LOCAL VARIABLES: + + character(len=*), parameter :: subname = 'is_urban_landunit' + !----------------------------------------------------------------------- + + if (ltype == this%ilun_urban_TBD .or. ltype == this%ilun_urban_HD & + .or. ltype == this%ilun_urban_MD) then + is_urban_landunit = .true. + else + is_urban_landunit = .false. + end if + + end function is_urban_landunit end module initInterpMindist diff --git a/src/init_interp/test/initInterpMindist_test/initInterpMindistTestUtils.pf b/src/init_interp/test/initInterpMindist_test/initInterpMindistTestUtils.pf index 04f09cb55d..7d277566af 100644 --- a/src/init_interp/test/initInterpMindist_test/initInterpMindistTestUtils.pf +++ b/src/init_interp/test/initInterpMindist_test/initInterpMindistTestUtils.pf @@ -19,12 +19,20 @@ module initInterpMindistTestUtils subgrid_special_indices_type( & ipft_not_vegetated = 0, & icol_vegetated_or_bare_soil = 10, & + icol_urban_roof = 71, & + icol_urban_sunwall = 72, & + icol_urban_shadewall = 73, & + icol_urban_impervious_road = 74, & + icol_urban_pervious_road = 75, & ilun_vegetated_or_bare_soil = 3, & ilun_crop = 4, & - ilun_landice = 5) + ilun_landice = 5, & + ilun_urban_TBD = 7, & + ilun_urban_HD = 8, & + ilun_urban_MD = 9) - ! value we can use for a special landunit; note that this just needs to differ from - ! ilun_vegetated_or_bare_soil and from ilun_crop + ! value we can use for a special landunit; note that this needs to differ from + ! ilun_vegetated_or_bare_soil, ilun_crop, ilun_urban_TBD, ilun_urban_HD, ilun_urban_MD integer, parameter, public :: ilun_special = 6 contains diff --git a/src/init_interp/test/initInterpMindist_test/test_set_mindist.pf b/src/init_interp/test/initInterpMindist_test/test_set_mindist.pf index 06ce20d7de..7a2c51456d 100644 --- a/src/init_interp/test/initInterpMindist_test/test_set_mindist.pf +++ b/src/init_interp/test/initInterpMindist_test/test_set_mindist.pf @@ -10,6 +10,7 @@ module test_set_mindist use clm_varcon , only: spval use unittestSimpleSubgridSetupsMod use unittestSubgridMod + use unittestUtils, only : endrun_msg use glcBehaviorMod, only: glc_behavior_type implicit none @@ -41,7 +42,7 @@ contains end subroutine tearDown subroutine wrap_set_mindist(subgridi, subgrido, mindist_index, activei, activeo, & - glc_behavior, glc_elevclasses_same, fill_missing_with_natveg) + glc_behavior, glc_elevclasses_same, fill_missing_with_natveg, fill_missing_urban_with_HD) ! Wrap the call to set_mindist. ! ! If activei / activeo are not provided, they are assumed to be .true. for all points. @@ -52,6 +53,7 @@ contains ! If glc_elevclasses_same is not present, it is assumed to be true. ! ! If fill_missing_with_natveg is not provided, it is assumed to be false + ! If fill_missing_urban_with_HD is not provided, it is assumed to be false ! Arguments: type(subgrid_type), intent(in) :: subgridi @@ -62,6 +64,7 @@ contains type(glc_behavior_type), intent(in), optional :: glc_behavior logical, intent(in), optional :: glc_elevclasses_same logical, intent(in), optional :: fill_missing_with_natveg + logical, intent(in), optional :: fill_missing_urban_with_HD ! Local variables: integer :: npts_i, npts_o @@ -71,6 +74,7 @@ contains type(glc_behavior_type) :: l_glc_behavior logical :: l_glc_elevclasses_same logical :: l_fill_missing_with_natveg + logical :: l_fill_missing_urban_with_HD !----------------------------------------------------------------------- @@ -115,12 +119,19 @@ contains l_fill_missing_with_natveg = .false. end if + if (present(fill_missing_urban_with_HD)) then + l_fill_missing_urban_with_HD = fill_missing_urban_with_HD + else + l_fill_missing_urban_with_HD = .false. + end if + call set_mindist(begi = 1, endi = npts_i, bego = bego, endo = endo, & activei = l_activei, activeo = l_activeo, subgridi = subgridi, subgrido = subgrido, & subgrid_special_indices = subgrid_special_indices, & glc_behavior = l_glc_behavior, & glc_elevclasses_same = l_glc_elevclasses_same, & fill_missing_with_natveg = l_fill_missing_with_natveg, & + fill_missing_urban_with_HD = l_fill_missing_urban_with_HD, & mindist_index = mindist_index) end subroutine wrap_set_mindist @@ -724,6 +735,186 @@ contains end associate end subroutine newveg_usesBaresoil + @Test + subroutine TBDurban_usesHDurban(this) + ! If there's a new urban TBD type, this should take inputs from the closest + ! HD type, if fill_missing_urban_with_HD = .true and fill_missing_with_natveg = .false. + ! + class(TestSetMindist), intent(inout) :: this + type(subgrid_type) :: subgridi, subgrido + real(r8), parameter :: my_lat = 31._r8 + real(r8), parameter :: my_lon = 41._r8 + integer :: i + integer :: mindist_index(1) + + associate( & + icol_urban_roof => subgrid_special_indices%icol_urban_roof, & + icol_urban_sunwall => subgrid_special_indices%icol_urban_sunwall, & + icol_urban_shadewall => subgrid_special_indices%icol_urban_shadewall, & + icol_urban_impervious_road => subgrid_special_indices%icol_urban_impervious_road, & + icol_urban_pervious_road => subgrid_special_indices%icol_urban_pervious_road, & + ilun_urban_TBD => subgrid_special_indices%ilun_urban_TBD, & + ilun_urban_HD => subgrid_special_indices%ilun_urban_HD, & + ilun_urban_MD => subgrid_special_indices%ilun_urban_MD & + ) + + call setup_landunit_ncols(ltype=ilun_urban_TBD, & + ctypes=[icol_urban_roof,icol_urban_sunwall,icol_urban_shadewall, & + icol_urban_impervious_road,icol_urban_pervious_road], & + cweights=[0.6_r8,0.1_r8,0.1_r8,0.1_r8,0.1_r8], & + ptype=0) + + call create_subgrid_info( & + subgrid_info = subgrido, & + npts = 1, & + beg = 1, & + name = 'landunit', & + ltype = [ilun_urban_TBD], & + lat = [my_lat], & + lon = [my_lon]) + + ! Input points differ in landunit type + call create_subgrid_info( & + subgrid_info = subgridi, & + npts = 2, & + name = 'landunit', & + ltype = [ilun_urban_MD, ilun_urban_HD], & + lat = [(my_lat, i=1,2)], & + lon = [(my_lon, i=1,2)]) + + call wrap_set_mindist(subgridi, subgrido, mindist_index, & + fill_missing_urban_with_HD = .true., & + fill_missing_with_natveg = .false.) + + ! Note that the mindist_index should return the second index of the + ! ltype array (2), not the actual value of ilun_urban_HD + @assertEqual(2, mindist_index(1)) + + end associate + end subroutine TBDurban_usesHDurban + + @Test + subroutine TBDurban_usesHDurban_aborts(this) + ! If there's a new urban TBD type, this should take inputs from the closest + ! HD type. This test will abort correctly if fill_missing_urban_with_HD = .false. + ! + class(TestSetMindist), intent(inout) :: this + type(subgrid_type) :: subgridi, subgrido + real(r8), parameter :: my_lat = 31._r8 + real(r8), parameter :: my_lon = 41._r8 + integer :: i + integer :: mindist_index(1) + character(len=:), allocatable :: expected_msg + + associate( & + icol_urban_roof => subgrid_special_indices%icol_urban_roof, & + icol_urban_sunwall => subgrid_special_indices%icol_urban_sunwall, & + icol_urban_shadewall => subgrid_special_indices%icol_urban_shadewall, & + icol_urban_impervious_road => subgrid_special_indices%icol_urban_impervious_road, & + icol_urban_pervious_road => subgrid_special_indices%icol_urban_pervious_road, & + ilun_urban_TBD => subgrid_special_indices%ilun_urban_TBD, & + ilun_urban_HD => subgrid_special_indices%ilun_urban_HD, & + ilun_urban_MD => subgrid_special_indices%ilun_urban_MD & + ) + + call setup_landunit_ncols(ltype=ilun_urban_TBD, & + ctypes=[icol_urban_roof,icol_urban_sunwall,icol_urban_shadewall, & + icol_urban_impervious_road,icol_urban_pervious_road], & + cweights=[0.6_r8,0.1_r8,0.1_r8,0.1_r8,0.1_r8], & + ptype=0) + + call create_subgrid_info( & + subgrid_info = subgrido, & + npts = 1, & + beg = 1, & + name = 'landunit', & + ltype = [ilun_urban_TBD], & + lat = [my_lat], & + lon = [my_lon]) + + ! Input points differ in landunit type + call create_subgrid_info( & + subgrid_info = subgridi, & + npts = 2, & + name = 'landunit', & + ltype = [ilun_urban_MD, ilun_urban_HD], & + lat = [(my_lat, i=1,2)], & + lon = [(my_lon, i=1,2)]) + + call wrap_set_mindist(subgridi, subgrido, mindist_index, & + fill_missing_urban_with_HD = .false.) + + expected_msg = endrun_msg( & + 'set_mindist ERROR: Cannot find any input points matching output point') + @assertExceptionRaised(expected_msg) + + end associate + end subroutine TBDurban_usesHDurban_aborts + + @Test + subroutine urbanlandunits_NotFilled_with_natveg_aborts(this) + ! With fill_missing_urban_with_HD = .false. and fill_missing_with_natveg = .true., + ! urban landunit should not be filled with natveg, and an error in set_mindist will be + ! thrown, and this test should pass. + ! + class(TestSetMindist), intent(inout) :: this + type(subgrid_type) :: subgridi, subgrido + real(r8), parameter :: my_lat = 31._r8 + real(r8), parameter :: my_lon = 41._r8 + integer :: i + integer :: mindist_index(1) + character(len=:), allocatable :: expected_msg + + associate( & + ipft_bare => subgrid_special_indices%ipft_not_vegetated, & + icol_urban_roof => subgrid_special_indices%icol_urban_roof, & + icol_urban_sunwall => subgrid_special_indices%icol_urban_sunwall, & + icol_urban_shadewall => subgrid_special_indices%icol_urban_shadewall, & + icol_urban_impervious_road => subgrid_special_indices%icol_urban_impervious_road, & + icol_urban_pervious_road => subgrid_special_indices%icol_urban_pervious_road, & + icol_natveg => subgrid_special_indices%icol_vegetated_or_bare_soil, & + ilun_natveg => subgrid_special_indices%ilun_vegetated_or_bare_soil, & + ilun_urban_TBD => subgrid_special_indices%ilun_urban_TBD & + ) + + call setup_landunit_ncols(ltype=ilun_urban_TBD, & + ctypes=[icol_urban_roof,icol_urban_sunwall,icol_urban_shadewall, & + icol_urban_impervious_road,icol_urban_pervious_road], & + cweights=[0.6_r8,0.1_r8,0.1_r8,0.1_r8,0.1_r8], & + ptype=0) + + call create_subgrid_info( & + subgrid_info = subgrido, & + npts = 1, & + beg = 1, & + name = 'pft', & + ptype = [0], & + ctype = [icol_urban_roof], & + ltype = [ilun_urban_TBD], & + lat = [my_lat], & + lon = [my_lon]) + + call create_subgrid_info( & + subgrid_info = subgridi, & + npts = 1, & + name = 'pft', & + ptype = [ipft_bare], & + ctype = [icol_natveg], & + ltype = [ilun_natveg], & + lat = [my_lat], & + lon = [my_lon]) + + call wrap_set_mindist(subgridi, subgrido, mindist_index, & + fill_missing_urban_with_HD = .false., & + fill_missing_with_natveg = .true.) + + expected_msg = endrun_msg( & + 'set_mindist ERROR: Cannot find any input points matching output point') + @assertExceptionRaised(expected_msg) + + end associate + end subroutine urbanlandunits_NotFilled_with_natveg_aborts + @Test subroutine baresoil_ignoresSpecialLandunits(this) ! This test ensures that, when finding a match for a bare soil patch, we ignore diff --git a/src/main/clm_initializeMod.F90 b/src/main/clm_initializeMod.F90 index 8c0b50230b..da8185be31 100644 --- a/src/main/clm_initializeMod.F90 +++ b/src/main/clm_initializeMod.F90 @@ -279,7 +279,9 @@ subroutine initialize2(ni,nj, currtime) end if ! Determine decomposition of subgrid scale landunits, columns, patches + call t_startf('clm_decompInit_clumps') call decompInit_clumps(ni, nj, glc_behavior) + call t_stopf('clm_decompInit_clumps') ! *** Get ALL processor bounds - for gridcells, landunit, columns and patches *** call get_proc_bounds(bounds_proc) @@ -304,7 +306,9 @@ subroutine initialize2(ni,nj, currtime) !$OMP END PARALLEL DO ! Set global seg maps for gridcells, landlunits, columns and patches + call t_startf('clm_decompInit_glcp') call decompInit_glcp(ni, nj, glc_behavior) + call t_stopf('clm_decompInit_glcp') if (use_hillslope) then ! Initialize hillslope properties