diff --git a/CIME/build.py b/CIME/build.py index 64f9a528b85..ed9ca2fd3ec 100644 --- a/CIME/build.py +++ b/CIME/build.py @@ -738,44 +738,53 @@ def _build_libraries( if not os.path.exists(shared_item): os.makedirs(shared_item) - mpilib = case.get_value("MPILIB") - ufs_driver = os.environ.get("UFS_DRIVER") + libs = list(dict.fromkeys(case.get_values("CASE_SUPPORT_LIBRARIES"))) + logger.info(f"libs from case_support_libraries {libs}") + build_script = {} cpl_in_complist = False for l in complist: if "cpl" in l: cpl_in_complist = True - if ufs_driver: - logger.info("UFS_DRIVER is set to {}".format(ufs_driver)) - - # This is a bit hacky. The host model should define whatever - # shared libs it might need. - if ufs_driver and ufs_driver == "nems" and not cpl_in_complist: - libs = [] - elif case.get_value("MODEL") == "cesm": - libs = ["gptl", "pio", "csm_share"] - elif case.get_value("MODEL") == "e3sm": - libs = ["gptl", "mct", "spio", "csm_share"] - else: - libs = ["gptl", "mct", "pio", "csm_share"] + # The libs variable should include a list of required support libraries. + # The following block is provided for backward compatibility. + if len(libs) < 1: + logger.warning( + "The model is using a deprecated method of determining support " + "libraries, please migrate to 'CASE_SUPPORT_LIBRARIES' variable." + ) + mpilib = case.get_value("MPILIB") + ufs_driver = os.environ.get("UFS_DRIVER") + if ufs_driver: + logger.info("UFS_DRIVER is set to {}".format(ufs_driver)) + + # This is a bit hacky. The host model should define whatever + # shared libs it might need. + if ufs_driver and ufs_driver == "nems" and not cpl_in_complist: + libs = [] + elif case.get_value("MODEL") == "cesm": + libs = ["gptl", "pio", "csm_share"] + elif case.get_value("MODEL") == "e3sm": + libs = ["gptl", "mct", "spio", "csm_share"] + else: + libs = ["gptl", "mct", "pio", "csm_share"] - libs.append("FTorch") + libs.append("FTorch") - if mpilib == "mpi-serial": - libs.insert(0, mpilib) + if mpilib == "mpi-serial": + libs.insert(0, mpilib) - if uses_kokkos(case) and comp_interface != "nuopc": - libs.append("ekat") + if uses_kokkos(case) and comp_interface != "nuopc": + libs.append("ekat") - # Build shared code of CDEPS nuopc data models - build_script = {} - if comp_interface == "nuopc" and (not ufs_driver or ufs_driver != "nems"): - libs.append("CDEPS") + # Build shared code of CDEPS nuopc data models + if comp_interface == "nuopc" and (not ufs_driver or ufs_driver != "nems"): + libs.append("CDEPS") - ocn_model = case.get_value("COMP_OCN") + ocn_model = case.get_value("COMP_OCN") - atm_dycore = case.get_value("CAM_DYCORE") - if ocn_model == "mom" or (atm_dycore and atm_dycore == "fv3"): - libs.append("FMS") + atm_dycore = case.get_value("CAM_DYCORE") + if ocn_model == "mom" or (atm_dycore and atm_dycore == "fv3"): + libs.append("FMS") files = Files(comp_interface=comp_interface) for lib in libs: diff --git a/CIME/tests/test_unit_build.py b/CIME/tests/test_unit_build.py new file mode 100644 index 00000000000..4d915283ee5 --- /dev/null +++ b/CIME/tests/test_unit_build.py @@ -0,0 +1,550 @@ +#!/usr/bin/env python3 + +import os +import unittest +from unittest import mock +from pathlib import Path + +from CIME import build +from .utils import mock_case + + +class TestBuild(unittest.TestCase): + @mock_case() + @mock.patch("CIME.build.uses_kokkos") + @mock.patch("CIME.build.generate_makefile_macro") # no need for makefile macros + @mock.patch( + "CIME.build.Files" + ) # used to check expected behavior by checking libs variable + def test__build_libraries_case_support_libraries( + self, Files, _, uses_kokkos, case, caseroot, **kwargs + ): + uses_kokkos.return_value = False + + exeroot = os.path.join(caseroot, "bld") + cimeroot = os.getcwd() + libroot = os.path.join(caseroot, "libs") + buildlist = [] + complist = [] + + case.get_values = mock.MagicMock( + side_effect=[ + ["gptl", "mct"], # CASE_SUPPORT_LIBRARIES + ] + ) + + case.get_value = mock.MagicMock( + side_effect=[ + "", # SHAREDLIBROOT + False, # TEST + ] + ) + + build._build_libraries( + case, + exeroot, + "shared", + caseroot, + cimeroot, + libroot, + "unique", + "gnu", + buildlist, + "mct", + complist, + ) + + get_value = Files.return_value.get_value + + expected = [ + mock.call("BUILD_LIB_FILE", {"lib": "gptl"}, attribute_required=True), + mock.call("BUILD_LIB_FILE", {"lib": "mct"}, attribute_required=True), + ] + + assert get_value.call_args_list == expected, get_value.call_args_list + + @mock.patch.dict(os.environ, {"UFS_DRIVER": "nems"}) + @mock_case() + @mock.patch("CIME.build.uses_kokkos") + @mock.patch("CIME.build.generate_makefile_macro") # no need for makefile macros + @mock.patch( + "CIME.build.Files" + ) # used to check expected behavior by checking libs variable + def test__build_libraries_ufs_driver( + self, Files, _, uses_kokkos, case, caseroot, **kwargs + ): + uses_kokkos.return_value = False + + exeroot = os.path.join(caseroot, "bld") + cimeroot = os.getcwd() + libroot = os.path.join(caseroot, "libs") + buildlist = ["cpl"] + complist = [] + + case.get_values = mock.MagicMock( + side_effect=[ + [], # CASE_SUPPORT_LIBRARIES + ] + ) + + case.get_value = mock.MagicMock( + side_effect=[ + "openmpi", # MPILIB + "e3sm", # MODEL + "e3sm", # MODEL + "", # COMP_OCN + "", # CAM_DYCORE + "", # SHAREDLIBROOT + False, # TEST + ] + ) + + build._build_libraries( + case, + exeroot, + "shared", + caseroot, + cimeroot, + libroot, + "unique", + "gnu", + buildlist, + "mct", + complist, + ) + + get_value = Files.return_value.get_value + + expected = [ + mock.call("BUILD_LIB_FILE", {"lib": "FTorch"}, attribute_required=True), + ] + + assert get_value.call_args_list == expected, get_value.call_args_list + + @mock_case() + @mock.patch("CIME.build.uses_kokkos") + @mock.patch("CIME.build.generate_makefile_macro") # no need for makefile macros + @mock.patch( + "CIME.build.Files" + ) # used to check expected behavior by checking libs variable + def test__build_libraries_cesm_model( + self, Files, _, uses_kokkos, case, caseroot, **kwargs + ): + uses_kokkos.return_value = False + + exeroot = os.path.join(caseroot, "bld") + cimeroot = os.getcwd() + libroot = os.path.join(caseroot, "libs") + buildlist = [] + complist = [] + + case.get_values = mock.MagicMock( + side_effect=[ + [], # CASE_SUPPORT_LIBRARIES + ] + ) + + case.get_value = mock.MagicMock( + side_effect=[ + "openmpi", # MPILIB + "cesm", # MODEL + "cesm", # MODEL + "", # COMP_OCN + "", # CAM_DYCORE + "", # SHAREDLIBROOT + False, # TEST + ] + ) + + build._build_libraries( + case, + exeroot, + "shared", + caseroot, + cimeroot, + libroot, + "unique", + "gnu", + buildlist, + "mct", + complist, + ) + + get_value = Files.return_value.get_value + + expected = [ + mock.call("BUILD_LIB_FILE", {"lib": "gptl"}, attribute_required=True), + mock.call("BUILD_LIB_FILE", {"lib": "pio"}, attribute_required=True), + mock.call("BUILD_LIB_FILE", {"lib": "csm_share"}, attribute_required=True), + mock.call("BUILD_LIB_FILE", {"lib": "FTorch"}, attribute_required=True), + ] + + assert get_value.call_args_list == expected, get_value.call_args_list + + @mock_case() + @mock.patch("CIME.build.uses_kokkos") + @mock.patch("CIME.build.generate_makefile_macro") # no need for makefile macros + @mock.patch( + "CIME.build.Files" + ) # used to check expected behavior by checking libs variable + def test__build_libraries_other_model( + self, Files, _, uses_kokkos, case, caseroot, **kwargs + ): + uses_kokkos.return_value = False + + exeroot = os.path.join(caseroot, "bld") + cimeroot = os.getcwd() + libroot = os.path.join(caseroot, "libs") + buildlist = [] + complist = [] + + case.get_values = mock.MagicMock( + side_effect=[ + [], # CASE_SUPPORT_LIBRARIES + ] + ) + + case.get_value = mock.MagicMock( + side_effect=[ + "openmpi", # MPILIB + "new", # MODEL + "new", # MODEL + "", # COMP_OCN + "", # CAM_DYCORE + "", # SHAREDLIBROOT + False, # TEST + ] + ) + + build._build_libraries( + case, + exeroot, + "shared", + caseroot, + cimeroot, + libroot, + "unique", + "gnu", + buildlist, + "mct", + complist, + ) + + get_value = Files.return_value.get_value + + expected = [ + mock.call("BUILD_LIB_FILE", {"lib": "gptl"}, attribute_required=True), + mock.call("BUILD_LIB_FILE", {"lib": "mct"}, attribute_required=True), + mock.call("BUILD_LIB_FILE", {"lib": "pio"}, attribute_required=True), + mock.call("BUILD_LIB_FILE", {"lib": "csm_share"}, attribute_required=True), + mock.call("BUILD_LIB_FILE", {"lib": "FTorch"}, attribute_required=True), + ] + + assert get_value.call_args_list == expected, get_value.call_args_list + + @mock_case() + @mock.patch("CIME.build.uses_kokkos") + @mock.patch("CIME.build.generate_makefile_macro") # no need for makefile macros + @mock.patch( + "CIME.build.Files" + ) # used to check expected behavior by checking libs variable + def test__build_libraries_mpi_serial( + self, Files, _, uses_kokkos, case, caseroot, **kwargs + ): + uses_kokkos.return_value = False + + exeroot = os.path.join(caseroot, "bld") + cimeroot = os.getcwd() + libroot = os.path.join(caseroot, "libs") + buildlist = [] + complist = [] + + case.get_values = mock.MagicMock( + side_effect=[ + [], # CASE_SUPPORT_LIBRARIES + ] + ) + + case.get_value = mock.MagicMock( + side_effect=[ + "mpi-serial", # MPILIB + "e3sm", # MODEL + "e3sm", # MODEL + "", # COMP_OCN + "", # CAM_DYCORE + "", # SHAREDLIBROOT + False, # TEST + ] + ) + + build._build_libraries( + case, + exeroot, + "shared", + caseroot, + cimeroot, + libroot, + "unique", + "gnu", + buildlist, + "mct", + complist, + ) + + get_value = Files.return_value.get_value + + expected = [ + mock.call("BUILD_LIB_FILE", {"lib": "mpi-serial"}, attribute_required=True), + mock.call("BUILD_LIB_FILE", {"lib": "gptl"}, attribute_required=True), + mock.call("BUILD_LIB_FILE", {"lib": "mct"}, attribute_required=True), + mock.call("BUILD_LIB_FILE", {"lib": "spio"}, attribute_required=True), + mock.call("BUILD_LIB_FILE", {"lib": "csm_share"}, attribute_required=True), + mock.call("BUILD_LIB_FILE", {"lib": "FTorch"}, attribute_required=True), + ] + + assert get_value.call_args_list == expected, get_value.call_args_list + + @mock_case() + @mock.patch("CIME.build.uses_kokkos") + @mock.patch("CIME.build.generate_makefile_macro") # no need for makefile macros + @mock.patch( + "CIME.build.Files" + ) # used to check expected behavior by checking libs variable + def test__build_libraries_use_kokkos( + self, Files, _, uses_kokkos, case, caseroot, **kwargs + ): + uses_kokkos.return_value = True + + exeroot = os.path.join(caseroot, "bld") + cimeroot = os.getcwd() + libroot = os.path.join(caseroot, "libs") + buildlist = [] + complist = [] + + case.get_values = mock.MagicMock( + side_effect=[ + [], # CASE_SUPPORT_LIBRARIES + ] + ) + + case.get_value = mock.MagicMock( + side_effect=[ + "openmpi", # MPILIB + "e3sm", # MODEL + "e3sm", # MODEL + "", # COMP_OCN + "", # CAM_DYCORE + "", # SHAREDLIBROOT + False, # TEST + ] + ) + + build._build_libraries( + case, + exeroot, + "shared", + caseroot, + cimeroot, + libroot, + "unique", + "gnu", + buildlist, + "mct", + complist, + ) + + get_value = Files.return_value.get_value + + expected = [ + mock.call("BUILD_LIB_FILE", {"lib": "gptl"}, attribute_required=True), + mock.call("BUILD_LIB_FILE", {"lib": "mct"}, attribute_required=True), + mock.call("BUILD_LIB_FILE", {"lib": "spio"}, attribute_required=True), + mock.call("BUILD_LIB_FILE", {"lib": "csm_share"}, attribute_required=True), + mock.call("BUILD_LIB_FILE", {"lib": "FTorch"}, attribute_required=True), + mock.call("BUILD_LIB_FILE", {"lib": "ekat"}, attribute_required=True), + ] + + assert get_value.call_args_list == expected, get_value.call_args_list + + @mock_case() + @mock.patch("CIME.build.uses_kokkos") + @mock.patch("CIME.build.generate_makefile_macro") # no need for makefile macros + @mock.patch( + "CIME.build.Files" + ) # used to check expected behavior by checking libs variable + def test__build_libraries_nuopc( + self, Files, _, uses_kokkos, case, caseroot, **kwargs + ): + uses_kokkos.return_value = False + + exeroot = os.path.join(caseroot, "bld") + cimeroot = os.getcwd() + libroot = os.path.join(caseroot, "libs") + buildlist = [] + complist = [] + + case.get_values = mock.MagicMock( + side_effect=[ + [], # CASE_SUPPORT_LIBRARIES + ] + ) + + case.get_value = mock.MagicMock( + side_effect=[ + "openmpi", # MPILIB + "e3sm", # MODEL + "e3sm", # MODEL + "", # COMP_OCN + "", # CAM_DYCORE + "", # SHAREDLIBROOT + False, # TEST + ] + ) + + build._build_libraries( + case, + exeroot, + "shared", + caseroot, + cimeroot, + libroot, + "unique", + "gnu", + buildlist, + "nuopc", + complist, + ) + + get_value = Files.return_value.get_value + + expected = [ + mock.call("BUILD_LIB_FILE", {"lib": "gptl"}, attribute_required=True), + mock.call("BUILD_LIB_FILE", {"lib": "mct"}, attribute_required=True), + mock.call("BUILD_LIB_FILE", {"lib": "spio"}, attribute_required=True), + mock.call("BUILD_LIB_FILE", {"lib": "csm_share"}, attribute_required=True), + mock.call("BUILD_LIB_FILE", {"lib": "FTorch"}, attribute_required=True), + mock.call("BUILD_LIB_FILE", {"lib": "CDEPS"}, attribute_required=True), + ] + + assert get_value.call_args_list == expected, get_value.call_args_list + + @mock_case() + @mock.patch("CIME.build.uses_kokkos") + @mock.patch("CIME.build.generate_makefile_macro") # no need for makefile macros + @mock.patch( + "CIME.build.Files" + ) # used to check expected behavior by checking libs variable + def test__build_libraries_fms( + self, Files, _, uses_kokkos, case, caseroot, **kwargs + ): + uses_kokkos.return_value = False + + exeroot = os.path.join(caseroot, "bld") + cimeroot = os.getcwd() + libroot = os.path.join(caseroot, "libs") + buildlist = [] + complist = [] + + case.get_values = mock.MagicMock( + side_effect=[ + [], # CASE_SUPPORT_LIBRARIES + ] + ) + + case.get_value = mock.MagicMock( + side_effect=[ + "openmpi", # MPILIB + "e3sm", # MODEL + "e3sm", # MODEL + "mom", # COMP_OCN + "fv3", # CAM_DYCORE + "", # SHAREDLIBROOT + False, # TEST + ] + ) + + build._build_libraries( + case, + exeroot, + "shared", + caseroot, + cimeroot, + libroot, + "unique", + "gnu", + buildlist, + "mct", + complist, + ) + + get_value = Files.return_value.get_value + + expected = [ + mock.call("BUILD_LIB_FILE", {"lib": "gptl"}, attribute_required=True), + mock.call("BUILD_LIB_FILE", {"lib": "mct"}, attribute_required=True), + mock.call("BUILD_LIB_FILE", {"lib": "spio"}, attribute_required=True), + mock.call("BUILD_LIB_FILE", {"lib": "csm_share"}, attribute_required=True), + mock.call("BUILD_LIB_FILE", {"lib": "FTorch"}, attribute_required=True), + mock.call("BUILD_LIB_FILE", {"lib": "FMS"}, attribute_required=True), + ] + + assert get_value.call_args_list == expected, get_value.call_args_list + + @mock_case() + @mock.patch("CIME.build.uses_kokkos") + @mock.patch("CIME.build.generate_makefile_macro") # no need for makefile macros + @mock.patch( + "CIME.build.Files" + ) # used to check expected behavior by checking libs variable + def test__build_libraries(self, Files, _, uses_kokkos, case, caseroot, **kwargs): + uses_kokkos.return_value = False + + exeroot = os.path.join(caseroot, "bld") + cimeroot = os.getcwd() + libroot = os.path.join(caseroot, "libs") + buildlist = [] + complist = [] + + case.get_values = mock.MagicMock( + side_effect=[ + [], # CASE_SUPPORT_LIBRARIES + ] + ) + + case.get_value = mock.MagicMock( + side_effect=[ + "openmpi", # MPILIB + "e3sm", # MODEL + "e3sm", # MODEL + "", # COMP_OCN + "", # CAM_DYCORE + "", # SHAREDLIBROOT + False, # TEST + ] + ) + + build._build_libraries( + case, + exeroot, + "shared", + caseroot, + cimeroot, + libroot, + "unique", + "gnu", + buildlist, + "mct", + complist, + ) + + get_value = Files.return_value.get_value + + expected = [ + mock.call("BUILD_LIB_FILE", {"lib": "gptl"}, attribute_required=True), + mock.call("BUILD_LIB_FILE", {"lib": "mct"}, attribute_required=True), + mock.call("BUILD_LIB_FILE", {"lib": "spio"}, attribute_required=True), + mock.call("BUILD_LIB_FILE", {"lib": "csm_share"}, attribute_required=True), + mock.call("BUILD_LIB_FILE", {"lib": "FTorch"}, attribute_required=True), + ] + + assert get_value.call_args_list == expected, get_value.call_args_list diff --git a/CIME/tests/utils.py b/CIME/tests/utils.py index 7679080eb01..951b5a8390e 100644 --- a/CIME/tests/utils.py +++ b/CIME/tests/utils.py @@ -165,7 +165,7 @@ def __str__(self): return ET.tostring(self.root.xml_element, encoding="unicode", method="xml") -def mock_case(*args, empty_env=False, filename=None, **kwargs): +def mock_case(*args, empty_env=False, filename=None, mock_set_value=False, **kwargs): if filename is None: filename = "env_test.xml" @@ -176,6 +176,8 @@ def wrapper(self, read_xml, *args, **kwargs): with tempfile.TemporaryDirectory() as tempdir: caseroot = f"{tempdir}/case" with Case(caseroot, read_only=False) as case: + case.flush = lambda: None + env = TestEnv() env.filename = f"{os.getcwd()}/{filename}" @@ -184,6 +186,9 @@ def wrapper(self, read_xml, *args, **kwargs): case._env_entryid_files = [env] + if mock_set_value: + case.set_value = mock.MagicMock() + func( self, *args, diff --git a/doc/source/ccs/model-configuration/config-files.rst b/doc/source/ccs/model-configuration/config-files.rst index e1051ed4b4e..e8ce9c8dc60 100644 --- a/doc/source/ccs/model-configuration/config-files.rst +++ b/doc/source/ccs/model-configuration/config-files.rst @@ -59,7 +59,31 @@ schema Path to a schema file that will be used to validate the conten Variables ::::::::: -These variables will define values or reference additional files to make up a models configuration. +These variables define literal values. + +MODEL +..... +This variable defines the name of the Model. + +Entry +''''' +The following is an example entry for ``MODEL`` in ``config_files.xml``. + +Only a single value is required. + +.. code-block:: xml + + + char + e3sm + case_der + env_case.xml + model system name + + +Reference Variables +::::::::::::::::::: +These variables reference files containing additional variables. .. toctree:: :maxdepth: 1 @@ -77,7 +101,6 @@ These variables will define values or reference additional files to make up a mo variables/grids.rst variables/inputdata.rst variables/machine.rst - variables/model.rst variables/namelist-definition.rst variables/pes.rst variables/pio.rst diff --git a/doc/source/ccs/model-configuration/variables/build-lib.rst b/doc/source/ccs/model-configuration/variables/build-lib.rst index 29506ab5c66..d49797d8d93 100644 --- a/doc/source/ccs/model-configuration/variables/build-lib.rst +++ b/doc/source/ccs/model-configuration/variables/build-lib.rst @@ -36,6 +36,7 @@ Each ``value`` corresponds to a library that can be built. The ``lib`` attribute path to buildlib script for the given library + Build library -------------- Implementing a ``buildlib`` for a component is as simple as creating a python file and defining a single function; *buildlib*. @@ -78,4 +79,4 @@ Example cmake_args = get_standard_cmake_args(case, installpath) - run_bld_cmd_ensure_logging(f"cmake {cmake_args}", logger, from_dir=libdir) \ No newline at end of file + run_bld_cmd_ensure_logging(f"cmake {cmake_args}", logger, from_dir=libdir) diff --git a/doc/source/ccs/model-configuration/variables/component.rst b/doc/source/ccs/model-configuration/variables/component.rst index feef4ec9802..189e5aec5ad 100644 --- a/doc/source/ccs/model-configuration/variables/component.rst +++ b/doc/source/ccs/model-configuration/variables/component.rst @@ -32,7 +32,6 @@ There will be multiple ``entry`` elements, one for each component supported by t Schema Definition ----------------- - The configuration is stored in ``config_component.xml`` under the components ``cime_config`` directory e.g. ``mosart/cime_config/config_component.xml``. This file will store multiple variables for the component defined using :ref:`*entry*` elements. Example contents of ``config_component.xml``. @@ -71,9 +70,46 @@ help Help text for the component. +Define support libraries +------------------------ +This variable is a list of support libraries that should be built by the case. +It is defined by the driver and component buildnml scripts and is used with ``BUILD_LIB_FILE`` to +locate required buildlib scripts. It is a list of libraries which will be built in list order so +it is important that dependent libraries are identified. + +The ``CASE_SUPPORT_LIBRARIES`` variable should be defined in the drivers ``config_component.xml`` file as + +.. code-block:: xml + + + char + + + gptl,pio,csm_share,FTorch,CDEPS + + build_def + env_build.xml + Support libraries required + + +The components ``buildnml`` script can modify the variable and add a list of libraries needed by the given component. +The list should be ordered so that a library comes after all of the libraries it depends on. + +The following is a small example of a ``buildnml`` script modifying ``CASE_SUPPORT_LIBRARIES``. + +.. code-block:: python + + def buildnml(case, caseroot, component): + ... + libs = case.get_value("CASE_SUPPORT_LIBRARIES") + mpilib = case.get_value("MPILIB") + if mpilib == "mpi-serial": + libs.insert(0, mpilib) + case.set_value("CASE_SUPPORT_LIBRARIES", ",".join(libs)) + ... + Triggering a rebuild -------------------- - It's the responsibility of a component to define which settings will require a component to be rebuilt. These triggers can be defined as follows. diff --git a/doc/source/ccs/model-configuration/variables/model.rst b/doc/source/ccs/model-configuration/variables/model.rst deleted file mode 100644 index 12caf58e81d..00000000000 --- a/doc/source/ccs/model-configuration/variables/model.rst +++ /dev/null @@ -1,27 +0,0 @@ -.. _model_config_model: - -MODEL -===== - -.. contents:: - :local: - -Overview --------- -This variable defines the name of the Model. - -Entry ------ -The following is an example entry for ``MODEL`` in ``config_files.xml``. - -Only a single value is required. - -.. code-block:: xml - - - char - e3sm - case_der - env_case.xml - model system name -