diff --git a/ci/distributed.yml b/ci/distributed.yml index 50c7f85444..ebfb3ac453 100644 --- a/ci/distributed.yml +++ b/ci/distributed.yml @@ -91,7 +91,7 @@ build_distributed: - ci/scripts/ci-mpi-wrapper.sh pytest -sv -k mpi_tests --with-mpi --backend=$BACKEND model/$COMPONENT --level=$LEVEL parallel: matrix: - - COMPONENT: [atmosphere/diffusion, atmosphere/dycore, common] + - COMPONENT: [atmosphere/diffusion, atmosphere/dycore, common, standalone_driver] # TODO(msimberg): Enable dace_gpu when compilation doesn't take as long # or when we can cache across CI jobs. BACKEND: [embedded, gtfn_cpu, dace_cpu, gtfn_gpu] @@ -113,6 +113,12 @@ build_distributed: - if: $COMPONENT == 'atmosphere/dycore' variables: SLURM_TIMELIMIT: '00:15:00' + - if: $COMPONENT == 'standalone_driver' + variables: + SLURM_TIMELIMIT: '00:50:00' + - if: $COMPONENT == 'standalone_driver' + variables: + SLURM_TIMELIMIT: '00:50:00' - when: on_success variables: SLURM_TIMELIMIT: '00:45:00' diff --git a/model/atmosphere/diffusion/tests/diffusion/mpi_tests/test_parallel_diffusion.py b/model/atmosphere/diffusion/tests/diffusion/mpi_tests/test_parallel_diffusion.py index c50885d158..f1a23fff41 100644 --- a/model/atmosphere/diffusion/tests/diffusion/mpi_tests/test_parallel_diffusion.py +++ b/model/atmosphere/diffusion/tests/diffusion/mpi_tests/test_parallel_diffusion.py @@ -39,6 +39,7 @@ "2021-06-20T12:00:10.000", ), (definitions.Experiments.EXCLAIM_APE, "2000-01-01T00:00:02.000", "2000-01-01T00:00:02.000"), + (definitions.Experiments.JW, "2008-09-01T00:05:00.000", "2008-09-01T00:05:00.000"), ], ) @pytest.mark.parametrize("ndyn_substeps", [2]) @@ -142,6 +143,12 @@ def test_parallel_diffusion( prognostic_state=prognostic_state, dtime=dtime, ) + if experiment == definitions.Experiments.JW: + # TODO (jcanton,ongchia): move this exchange inside + # diffusion.run with the proper DiffusionConfig equivalent to + # the fortran + # IF ( linit .OR. (iforcing /= inwp .AND. iforcing /= iaes) ) THEN + exchange.exchange(dims.CellDim, prognostic_state.w) _log.info(f"rank={processor_props.rank}/{processor_props.comm_size}: diffusion run ") utils.verify_diffusion_fields( diff --git a/model/common/src/icon4py/model/common/decomposition/definitions.py b/model/common/src/icon4py/model/common/decomposition/definitions.py index 9ebc5f40c8..6296b26737 100644 --- a/model/common/src/icon4py/model/common/decomposition/definitions.py +++ b/model/common/src/icon4py/model/common/decomposition/definitions.py @@ -12,10 +12,11 @@ import dataclasses import functools import logging +import warnings from collections.abc import Sequence from enum import Enum from types import ModuleType -from typing import Any, Literal, Protocol, TypeAlias, overload, runtime_checkable +from typing import Any, ClassVar, Literal, Protocol, TypeAlias, overload, runtime_checkable import dace # type: ignore[import-untyped] import gt4py.next as gtx @@ -160,6 +161,8 @@ class EntryType(int, Enum): ALL = 0 OWNED = 1 HALO = 2 + HALO_LEVEL_1 = 11 + HALO_LEVEL_2 = 12 @utils.chainable def set_dimension( @@ -187,6 +190,14 @@ def local_index( index = self._to_local_index(dim) mask = self._owner_mask[dim] return index[~mask] + case DecompositionInfo.EntryType.HALO_LEVEL_1: + index = self._to_local_index(dim) + mask = self.halo_level_mask(dim=dim, level=DecompositionFlag.FIRST_HALO_LEVEL) + return index[mask] + case DecompositionInfo.EntryType.HALO_LEVEL_2: + index = self._to_local_index(dim) + mask = self.halo_level_mask(dim=dim, level=DecompositionFlag.SECOND_HALO_LEVEL) + return index[mask] case DecompositionInfo.EntryType.OWNED: index = self._to_local_index(dim) mask = self._owner_mask[dim] @@ -212,6 +223,12 @@ def global_index( return self._global_index[dim][self._owner_mask[dim]] case DecompositionInfo.EntryType.HALO: return self._global_index[dim][~self._owner_mask[dim]] + case DecompositionInfo.EntryType.HALO_LEVEL_1: + mask = self.halo_level_mask(dim=dim, level=DecompositionFlag.FIRST_HALO_LEVEL) + return self._global_index[dim][mask] + case DecompositionInfo.EntryType.HALO_LEVEL_2: + mask = self.halo_level_mask(dim=dim, level=DecompositionFlag.SECOND_HALO_LEVEL) + return self._global_index[dim][mask] case _: raise NotImplementedError() @@ -408,12 +425,25 @@ def __str__(self) -> str: @dataclasses.dataclass class SingleNodeExchange(ExchangeRuntime): + _warning_emitted: ClassVar[bool] = False + + @classmethod + def _warn_if_used(cls, *, stacklevel: int = 3) -> None: + if not cls._warning_emitted: + warnings.warn( + "***** SingleNodeExchange is in use; HALO EXCHANGE IS RUNNING IN SINGLE-NODE *****", + RuntimeWarning, + stacklevel=stacklevel, + ) + cls._warning_emitted = True + def start( self, dim: gtx.Dimension, *fields: gtx.Field | data_alloc.NDArray, stream: StreamLike = DEFAULT_STREAM, ) -> ExchangeResult: + self._warn_if_used() return SingleNodeResult() def my_rank(self) -> int: @@ -563,11 +593,19 @@ def max( ) -> state_utils.ScalarType: ... def sum( - self, buffer: data_alloc.NDArray, array_ns: ModuleType = np + self, + buffer: data_alloc.NDArray, + lower_bound: gtx.int32, + upper_bound: gtx.int32, + array_ns: ModuleType = np, ) -> state_utils.ScalarType: ... def mean( - self, buffer: data_alloc.NDArray, array_ns: ModuleType = np + self, + buffer: data_alloc.NDArray, + lower_bound: gtx.int32, + upper_bound: gtx.int32, + array_ns: ModuleType = np, ) -> state_utils.ScalarType: ... @@ -578,10 +616,22 @@ def min(self, buffer: data_alloc.NDArray, array_ns: ModuleType = np) -> state_ut def max(self, buffer: data_alloc.NDArray, array_ns: ModuleType = np) -> state_utils.ScalarType: return array_ns.max(buffer).item() - def sum(self, buffer: data_alloc.NDArray, array_ns: ModuleType = np) -> state_utils.ScalarType: + def sum( + self, + buffer: data_alloc.NDArray, + lower_bound: gtx.int32, + upper_bound: gtx.int32, + array_ns: ModuleType = np, + ) -> state_utils.ScalarType: return array_ns.sum(buffer).item() - def mean(self, buffer: data_alloc.NDArray, array_ns: ModuleType = np) -> state_utils.ScalarType: + def mean( + self, + buffer: data_alloc.NDArray, + lower_bound: gtx.int32, + upper_bound: gtx.int32, + array_ns: ModuleType = np, + ) -> state_utils.ScalarType: return array_ns.sum(buffer).item() / buffer.size diff --git a/model/common/src/icon4py/model/common/decomposition/mpi_decomposition.py b/model/common/src/icon4py/model/common/decomposition/mpi_decomposition.py index 19173bbe46..0314313513 100644 --- a/model/common/src/icon4py/model/common/decomposition/mpi_decomposition.py +++ b/model/common/src/icon4py/model/common/decomposition/mpi_decomposition.py @@ -248,9 +248,9 @@ def start( stream: definitions.StreamLike = definitions.DEFAULT_STREAM, ) -> MultiNodeResult: """Synchronize with `stream` and start the halo exchange of `*fields`.""" - assert ( - dim in dims.MAIN_HORIZONTAL_DIMENSIONS.values() - ), f"first dimension must be one of ({dims.MAIN_HORIZONTAL_DIMENSIONS.values()})" + assert dim in dims.MAIN_HORIZONTAL_DIMENSIONS.values(), ( + f"first dimension must be one of ({dims.MAIN_HORIZONTAL_DIMENSIONS.values()})" + ) applied_patterns = [self._get_applied_pattern(dim, f) for f in fields] if not ghex.__config__["gpu"]: @@ -489,7 +489,15 @@ def max(self, buffer: data_alloc.NDArray, array_ns: ModuleType = np) -> state_ut array_ns, ) - def sum(self, buffer: data_alloc.NDArray, array_ns: ModuleType = np) -> state_utils.ScalarType: + def sum( + self, + buffer: data_alloc.NDArray, + lower_bound: gtx.int32, + upper_bound: gtx.int32, + array_ns: ModuleType = np, + ) -> state_utils.ScalarType: + # TODO (nfarabullini): use owned mask instead of upper lower bound, and move as "internal argument" + buffer = buffer[lower_bound:upper_bound] if self._calc_buffer_size(buffer, array_ns) == 0: raise ValueError("global_sum requires a non-empty buffer") return self._reduce( @@ -499,7 +507,15 @@ def sum(self, buffer: data_alloc.NDArray, array_ns: ModuleType = np) -> state_ut array_ns, ) - def mean(self, buffer: data_alloc.NDArray, array_ns: ModuleType = np) -> state_utils.ScalarType: + def mean( + self, + buffer: data_alloc.NDArray, + lower_bound: gtx.int32, + upper_bound: gtx.int32, + array_ns: ModuleType = np, + ) -> state_utils.ScalarType: + # TODO (nfarabullini): use owned mask instead of upper lower bound, and move as "internal argument" + buffer = buffer[lower_bound:upper_bound] global_buffer_size = self._calc_buffer_size(buffer, array_ns) if global_buffer_size == 0: raise ValueError("global_mean requires a non-empty buffer") diff --git a/model/common/src/icon4py/model/common/grid/geometry.py b/model/common/src/icon4py/model/common/grid/geometry.py index 79bcd3847b..7d395dea57 100644 --- a/model/common/src/icon4py/model/common/grid/geometry.py +++ b/model/common/src/icon4py/model/common/grid/geometry.py @@ -110,7 +110,9 @@ def __init__( self._decomposition_info = decomposition_info self._attrs = metadata self._geometry_type: base.GeometryType = grid.global_properties.geometry_type + self._cell_domain = h_grid.domain(dims.CellDim) self._edge_domain = h_grid.domain(dims.EdgeDim) + self._vertex_domain = h_grid.domain(dims.VertexDim) self._exchange = exchange self._global_reductions = global_reductions log.info( @@ -323,6 +325,10 @@ def _register_computed_fields(self) -> None: deps={ "buffer": attrs.EDGE_LENGTH, }, + params={ + "lower_bound": self._grid.start_index(self._edge_domain(h_grid.Zone.LOCAL)), + "upper_bound": self._grid.end_index(self._edge_domain(h_grid.Zone.LOCAL)), + }, fields=(attrs.MEAN_EDGE_LENGTH,), ) self.register_provider(mean_edge_length_np) @@ -336,6 +342,10 @@ def _register_computed_fields(self) -> None: deps={ "buffer": attrs.DUAL_EDGE_LENGTH, }, + params={ + "lower_bound": self._grid.start_index(self._edge_domain(h_grid.Zone.LOCAL)), + "upper_bound": self._grid.end_index(self._edge_domain(h_grid.Zone.LOCAL)), + }, fields=(attrs.MEAN_DUAL_EDGE_LENGTH,), ) self.register_provider(mean_dual_edge_length_np) @@ -349,6 +359,10 @@ def _register_computed_fields(self) -> None: deps={ "buffer": attrs.CELL_AREA, }, + params={ + "lower_bound": self._grid.start_index(self._cell_domain(h_grid.Zone.LOCAL)), + "upper_bound": self._grid.end_index(self._cell_domain(h_grid.Zone.LOCAL)), + }, fields=(attrs.MEAN_CELL_AREA,), ) self.register_provider(mean_cell_area_np) @@ -362,6 +376,10 @@ def _register_computed_fields(self) -> None: deps={ "buffer": attrs.DUAL_AREA, }, + params={ + "lower_bound": self._grid.start_index(self._vertex_domain(h_grid.Zone.LOCAL)), + "upper_bound": self._grid.end_index(self._vertex_domain(h_grid.Zone.LOCAL)), + }, fields=(attrs.MEAN_DUAL_AREA,), ) self.register_provider(mean_dual_cell_area_np) diff --git a/model/common/src/icon4py/model/common/grid/grid_manager.py b/model/common/src/icon4py/model/common/grid/grid_manager.py index 1fde396bf5..46be5fcee6 100644 --- a/model/common/src/icon4py/model/common/grid/grid_manager.py +++ b/model/common/src/icon4py/model/common/grid/grid_manager.py @@ -106,8 +106,8 @@ def __call__( allocator: gtx_typing.Allocator | None, keep_skip_values: bool, decomposer: decomp.Decomposer = _single_node_decomposer, - run_properties=_single_process_props, - ): + run_properties: decomposition.ProcessProperties = _single_process_props, + ) -> None: if not run_properties.is_single_rank() and isinstance( decomposer, decomp.SingleNodeDecomposer ): diff --git a/model/common/src/icon4py/model/common/math/stencils/generic_math_operations_array_ns.py b/model/common/src/icon4py/model/common/math/stencils/generic_math_operations_array_ns.py new file mode 100644 index 0000000000..3aaf8d8655 --- /dev/null +++ b/model/common/src/icon4py/model/common/math/stencils/generic_math_operations_array_ns.py @@ -0,0 +1,57 @@ +# ICON4Py - ICON inspired code in Python and GT4Py +# +# Copyright (c) 2022-2024, ETH Zurich and MeteoSwiss +# All rights reserved. +# +# Please, refer to the LICENSE file in the root directory. +# SPDX-License-Identifier: BSD-3-Clause + +from types import ModuleType + +from icon4py.model.common.utils import data_allocation as data_alloc + + +def compute_directional_derivative_on_cells( + cell_field: data_alloc.NDArray, + e2c: data_alloc.NDArray, + inv_dual_edge_length: data_alloc.NDArray, + lb_e: int, + ub_e: int, + num_edges: int, + array_ns: ModuleType, +) -> data_alloc.NDArray: + """ + Compute directional derivative of a cell centered variable with respect to + direction normal to triangle edge. + """ + result = array_ns.zeros((num_edges,)) + result[lb_e:ub_e] = ( + cell_field[e2c[lb_e:ub_e, 1]] - cell_field[e2c[lb_e:ub_e, 0]] + ) * inv_dual_edge_length[lb_e:ub_e] + return result + + +def interpolate_edges_to_cell( + edge_field: data_alloc.NDArray, + c2e: data_alloc.NDArray, + e2c: data_alloc.NDArray, + edge_cell_length: data_alloc.NDArray, + primal_edge_length: data_alloc.NDArray, + cell_area: data_alloc.NDArray, + ub_c: int, + num_cells: int, + array_ns: ModuleType, +) -> data_alloc.NDArray: + """ + Compute interpolation of scalar fields from edge points to cell centers. + """ + e_inn_c = array_ns.zeros((num_cells, 3)) + jc_indices = array_ns.arange(ub_c)[:, array_ns.newaxis] + c2e_local = c2e[:ub_c] + idx_ce = (e2c[c2e_local][:, :, 0] != jc_indices).astype(int) + e_inn_c[:ub_c] = ( + edge_cell_length[c2e_local, idx_ce] + * primal_edge_length[c2e_local] + / cell_area[:ub_c, array_ns.newaxis] + ) + return array_ns.sum(edge_field[c2e] * e_inn_c, axis=1) diff --git a/model/common/src/icon4py/model/common/metrics/metrics_factory.py b/model/common/src/icon4py/model/common/metrics/metrics_factory.py index 8a6ed38dda..3d2b68d39f 100644 --- a/model/common/src/icon4py/model/common/metrics/metrics_factory.py +++ b/model/common/src/icon4py/model/common/metrics/metrics_factory.py @@ -642,7 +642,7 @@ def _register_computed_fields(self) -> None: # noqa: PLR0915 [too-many-statemen ), }, fields={"wgtfac_e": attrs.WGTFAC_E}, - do_exchange=False, + do_exchange=True, ) self.register_provider(compute_wgtfac_e) diff --git a/model/common/tests/common/decomposition/unit_tests/test_definitions.py b/model/common/tests/common/decomposition/unit_tests/test_definitions.py index d99514a3a6..2a7df1db6b 100644 --- a/model/common/tests/common/decomposition/unit_tests/test_definitions.py +++ b/model/common/tests/common/decomposition/unit_tests/test_definitions.py @@ -6,6 +6,9 @@ # Please, refer to the LICENSE file in the root directory. # SPDX-License-Identifier: BSD-3-Clause +import sys +import warnings + import gt4py.next as gtx import numpy as np import pytest @@ -13,6 +16,7 @@ import icon4py.model.common.dimension as dims import icon4py.model.common.utils.data_allocation as data_alloc +from icon4py.model.common import dimension as dims from icon4py.model.common.decomposition import definitions from icon4py.model.common.decomposition.definitions import ( DecompositionInfo, @@ -94,3 +98,57 @@ def test_decomposition_info_is_distributed(flag, expected) -> None: np.ones((mesh.num_cells,)) * flag, ) assert decomp.is_distributed() == expected + + +def test_single_node_exchange_warns_on_first_use(monkeypatch): + monkeypatch.setattr(SingleNodeExchange, "_warning_emitted", False) + + exchange = SingleNodeExchange() + + with pytest.warns(RuntimeWarning, match="SingleNodeExchange"): + exchange.start(dims.CellDim) + + +def test_single_node_exchange_does_not_warn_on_construction_or_repeat_use(monkeypatch): + monkeypatch.setattr(SingleNodeExchange, "_warning_emitted", False) + + with warnings.catch_warnings(record=True) as recorded_warnings: + warnings.simplefilter("always") + exchange = SingleNodeExchange() + + assert len(recorded_warnings) == 0 + + with pytest.warns(RuntimeWarning, match="SingleNodeExchange"): + exchange.exchange(dims.CellDim) + + with warnings.catch_warnings(record=True) as repeated_warnings: + warnings.simplefilter("always") + exchange.start(dims.CellDim) + + assert len(repeated_warnings) == 0 + + +def _assert_warning_points_to_call_site(monkeypatch, func, expected_line): + monkeypatch.setattr(SingleNodeExchange, "_warning_emitted", False) + + with warnings.catch_warnings(record=True) as caught_warnings: + warnings.simplefilter("always") + func() + + assert len(caught_warnings) == 1 + assert caught_warnings[0].filename == __file__ + assert caught_warnings[0].lineno == expected_line + + +def test_single_node_exchange_warning_points_to_call_site(monkeypatch): + exchange = SingleNodeExchange() + + exchange_line = sys._getframe().f_lineno + 1 + _assert_warning_points_to_call_site( + monkeypatch, lambda: exchange.start(dims.CellDim), exchange_line + ) + + wait_line = sys._getframe().f_lineno + 1 + _assert_warning_points_to_call_site( + monkeypatch, lambda: exchange.exchange(dims.CellDim), wait_line + ) diff --git a/model/common/tests/common/grid/mpi_tests/test_parallel_geometry.py b/model/common/tests/common/grid/mpi_tests/test_parallel_geometry.py index a063fc33be..f5fc44a970 100644 --- a/model/common/tests/common/grid/mpi_tests/test_parallel_geometry.py +++ b/model/common/tests/common/grid/mpi_tests/test_parallel_geometry.py @@ -212,57 +212,18 @@ def test_cartesian_geometry_attr_no_halos( "attr_name", ["mean_edge_length", "mean_dual_edge_length", "mean_cell_area", "mean_dual_area"] ) def test_distributed_geometry_mean_fields( - backend: gtx_typing.Backend, - grid_savepoint: sb.IconGridSavepoint, + experiment: test_defs.Experiment, processor_props: decomposition.ProcessProperties, decomposition_info: decomposition.DecompositionInfo, geometry_from_savepoint: geometry.GridGeometry, attr_name: str, ) -> None: - if processor_props.comm_size > 1: - pytest.skip("Values not serialized for multiple processors") + if experiment.grid.params.limited_area: + pytest.xfail("Limited-area grids not yet supported") parallel_helpers.check_comm_size(processor_props) parallel_helpers.log_process_properties(processor_props) parallel_helpers.log_local_field_size(decomposition_info) - assert hasattr(experiment, "name") value_ref = utils.GRID_REFERENCE_VALUES[experiment.name][attr_name] value = geometry_from_savepoint.get(attr_name) assert value == pytest.approx(value_ref) - - -@pytest.mark.datatest -@pytest.mark.mpi -@pytest.mark.parametrize("processor_props", [True], indirect=True) -def test_distributed_mean_cell_area( - backend: gtx_typing.Backend, - grid_savepoint: sb.IconGridSavepoint, - processor_props: decomposition.ProcessProperties, - decomposition_info: decomposition.DecompositionInfo, - geometry_from_savepoint: geometry.GridGeometry, -) -> None: - parallel_helpers.check_comm_size(processor_props) - parallel_helpers.log_process_properties(processor_props) - parallel_helpers.log_local_field_size(decomposition_info) - value_ref = grid_savepoint.mean_cell_area() - value = geometry_from_savepoint.get("mean_cell_area") - assert value == pytest.approx(value_ref, rel=1e-1) - - -@pytest.mark.datatest -@pytest.mark.mpi -@pytest.mark.parametrize("processor_props", [True], indirect=True) -def test_distributed_mean_dual_edge_length( - backend: gtx_typing.Backend, - grid_savepoint: sb.IconGridSavepoint, - processor_props: decomposition.ProcessProperties, - decomposition_info: decomposition.DecompositionInfo, - geometry_from_savepoint: geometry.GridGeometry, -) -> None: - parallel_helpers.check_comm_size(processor_props) - parallel_helpers.log_process_properties(processor_props) - parallel_helpers.log_local_field_size(decomposition_info) - - value_ref = np.mean(grid_savepoint.dual_edge_length().asnumpy()) - value = geometry_from_savepoint.get("mean_dual_edge_length") - assert value == pytest.approx(value_ref, rel=1e-1) diff --git a/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py b/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py index c062594487..566774f408 100644 --- a/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py +++ b/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py @@ -55,7 +55,7 @@ def test_grid_manager_validate_decomposer( processor_props: decomp_defs.ProcessProperties, experiment: test_defs.Experiment, ) -> None: - if experiment == test_defs.Experiments.MCH_CH_R04B09: + if experiment.grid.params.limited_area: pytest.xfail("Limited-area grids not yet supported") file = grid_utils.resolve_full_grid_file_name(experiment.grid) @@ -271,7 +271,7 @@ def _compare_interpolation_fields_single_multi_rank( experiment: test_defs.Experiment, attrs_name: str, ) -> None: - if experiment == test_defs.Experiments.MCH_CH_R04B09: + if experiment.grid.params.limited_area: pytest.xfail("Limited-area grids not yet supported") if attrs_name in embedded_broken_fields and test_utils.is_embedded(backend): @@ -424,7 +424,7 @@ def _compare_metrics_fields_single_multi_rank( experiment: test_defs.Experiment, attrs_name: str, ) -> None: - if experiment == test_defs.Experiments.MCH_CH_R04B09: + if experiment.grid.params.limited_area: pytest.xfail("Limited-area grids not yet supported") if attrs_name in embedded_broken_fields and test_utils.is_embedded(backend): @@ -590,7 +590,7 @@ def _compare_metrics_fields_single_multi_rank( dim=field_ref.domain.dims[0], global_reference_field=field_ref.asnumpy(), local_field=field.asnumpy(), - check_halos=(attrs_name != metrics_attributes.WGTFAC_E), + check_halos=True, atol=0.0, ) @@ -689,7 +689,7 @@ def test_metrics_mask_prog_halo_c( backend: gtx_typing.Backend | None, experiment: test_defs.Experiment, ) -> None: - if experiment == test_defs.Experiments.MCH_CH_R04B09: + if experiment.grid.params.limited_area: pytest.xfail("Limited-area grids not yet supported") file = grid_utils.resolve_full_grid_file_name(experiment.grid) @@ -822,7 +822,7 @@ def test_validate_skip_values_in_distributed_connectivities( experiment: test_defs.Experiment, backend: gtx_typing.Backend | None, ) -> None: - if experiment == test_defs.Experiments.MCH_CH_R04B09: + if experiment.grid.params.limited_area: pytest.xfail("Limited-area grids not yet supported") file = grid_utils.resolve_full_grid_file_name(experiment.grid) diff --git a/model/standalone_driver/src/icon4py/model/standalone_driver/driver_utils.py b/model/standalone_driver/src/icon4py/model/standalone_driver/driver_utils.py index 294c740c30..d534b38f02 100644 --- a/model/standalone_driver/src/icon4py/model/standalone_driver/driver_utils.py +++ b/model/standalone_driver/src/icon4py/model/standalone_driver/driver_utils.py @@ -23,12 +23,12 @@ from icon4py.model.atmosphere.dycore import dycore_states, solve_nonhydro as solve_nh from icon4py.model.common import ( constants, - dimension as dims, field_type_aliases as fa, model_backends, type_alias as ta, ) from icon4py.model.common.decomposition import ( + decomposer as decomp, definitions as decomposition_defs, mpi_decomposition as mpi_decomp, ) @@ -44,7 +44,6 @@ from icon4py.model.common.interpolation import interpolation_attributes, interpolation_factory from icon4py.model.common.metrics import metrics_attributes, metrics_factory from icon4py.model.common.states import factory as states_factory -from icon4py.model.common.utils import data_allocation as data_alloc from icon4py.model.standalone_driver import config as driver_config, driver_states @@ -64,38 +63,30 @@ def create_grid_manager( grid_file_path: pathlib.Path, vertical_grid_config: v_grid.VerticalGridConfig, allocator: gtx_typing.Allocator, + parallel_props: decomposition_defs.ProcessProperties, global_reductions: decomposition_defs.Reductions = decomposition_defs.single_node_reductions, ) -> gm.GridManager: + decomposer = ( + decomp.MetisDecomposer() + if not parallel_props.is_single_rank() + else decomp.SingleNodeDecomposer() + ) grid_manager = gm.GridManager( grid_file=grid_file_path, config=vertical_grid_config, offset_transformation=gridfile.ToZeroBasedIndexTransformation(), global_reductions=global_reductions, ) - grid_manager(allocator=allocator, keep_skip_values=True) + grid_manager( + allocator=allocator, + keep_skip_values=True, + run_properties=parallel_props, + decomposer=decomposer, + ) return grid_manager -def create_decomposition_info( - grid_manager: gm.GridManager, - allocator: gtx_typing.Allocator, -) -> decomposition_defs.DecompositionInfo: - decomposition_info = decomposition_defs.DecompositionInfo() - xp = data_alloc.import_array_ns(allocator) - - def _add_dimension(dim: gtx.Dimension) -> None: - indices = data_alloc.index_field(grid_manager.grid, dim, allocator=allocator) - owner_mask = xp.ones((grid_manager.grid.size[dim],), dtype=bool) - decomposition_info.set_dimension(dim, indices.ndarray, owner_mask, None) - - _add_dimension(dims.EdgeDim) - _add_dimension(dims.VertexDim) - _add_dimension(dims.CellDim) - - return decomposition_info - - def create_vertical_grid( vertical_grid_config: v_grid.VerticalGridConfig, allocator: gtx_typing.Allocator, @@ -118,6 +109,8 @@ def create_static_field_factories( vertical_grid: v_grid.VerticalGrid, cell_topography: fa.CellField[ta.wpfloat], backend: gtx_typing.Backend | None, + exchange: decomposition_defs.ExchangeRuntime, + global_reductions: decomposition_defs.Reductions, ) -> driver_states.StaticFieldFactories: geometry_field_source = grid_geometry.GridGeometry( grid=grid_manager.grid, @@ -126,6 +119,8 @@ def create_static_field_factories( coordinates=grid_manager.coordinates, extra_fields=grid_manager.geometry_fields, metadata=geometry_meta.attrs, + exchange=exchange, + global_reductions=global_reductions, ) interpolation_field_source = interpolation_factory.InterpolationFieldsFactory( @@ -134,6 +129,7 @@ def create_static_field_factories( geometry_source=geometry_field_source, backend=backend, metadata=interpolation_attributes.attrs, + exchange=exchange, ) metrics_field_source = metrics_factory.MetricsFieldsFactory( @@ -151,6 +147,8 @@ def create_static_field_factories( vwind_offctr=0.15, thslp_zdiffu=0.025, thhgtd_zdiffu=200.0, + exchange=exchange, + global_reductions=global_reductions, ) return driver_states.StaticFieldFactories( @@ -344,6 +342,7 @@ def initialize_granules( edge_geometry=edge_geometry, cell_geometry=cell_geometry, owner_mask=owner_mask, + exchange=exchange, ) advection_granule = advection.convert_config_to_advection( @@ -579,12 +578,15 @@ def configure_logging( display_icon4py_logo_in_log_file() -def get_backend_from_name(backend_name: str) -> model_backends.BackendLike: +def get_backend_from_name( + backend_name: str | model_backends.BackendLike | None, +) -> model_backends.BackendLike: if backend_name not in model_backends.BACKENDS: raise ValueError( f"Invalid driver backend: {backend_name}. \n" f"Available backends are {', '.join([*model_backends.BACKENDS.keys()])}" ) + assert isinstance(backend_name, str) backend = model_backends.BACKENDS[backend_name] log.info(f"Backend name used for the model: {backend_name}") log.info(f"BackendLike derived from the backend name: {backend}") diff --git a/model/standalone_driver/src/icon4py/model/standalone_driver/main.py b/model/standalone_driver/src/icon4py/model/standalone_driver/main.py index 034cfabc70..6204635149 100644 --- a/model/standalone_driver/src/icon4py/model/standalone_driver/main.py +++ b/model/standalone_driver/src/icon4py/model/standalone_driver/main.py @@ -12,6 +12,7 @@ import typer from icon4py.model.common import model_backends +from icon4py.model.common.decomposition import definitions as decomp_defs from icon4py.model.standalone_driver import driver_states, driver_utils, standalone_driver from icon4py.model.standalone_driver.testcases import initial_condition @@ -39,7 +40,13 @@ def main( help=f"Logging level of the model. Possible options are {' / '.join([*driver_utils._LOGGING_LEVELS.keys()])}", ), ] = next(iter(driver_utils._LOGGING_LEVELS.keys())), -) -> driver_states.DriverStates: + force_serial_run: Annotated[ + bool, + typer.Option( + help="Force a single-node run even if MPI is available. Useful to build serial reference output within MPI test sessions.", + ), + ] = False, +) -> tuple[driver_states.DriverStates, decomp_defs.DecompositionInfo]: """ This is a function that runs the icon4py driver from a grid file with the initial condition from the Jablonowski Williamson test case @@ -55,6 +62,7 @@ def main( grid_file_path=grid_file_path, log_level=log_level, backend_name=icon4py_backend, + force_serial_run=force_serial_run, ) log.info("Generating the initial condition") @@ -68,6 +76,7 @@ def main( model_top_height=icon4py_driver.vertical_grid_config.model_top_height, stretch_factor=icon4py_driver.vertical_grid_config.stretch_factor, damping_height=icon4py_driver.vertical_grid_config.rayleigh_damping_height, + exchange=icon4py_driver.exchange, ) log.info("driver setup: DONE") @@ -79,7 +88,7 @@ def main( ) log.info("time loop: DONE") - return ds + return ds, icon4py_driver.decomposition_info if __name__ == "__main__": diff --git a/model/standalone_driver/src/icon4py/model/standalone_driver/standalone_driver.py b/model/standalone_driver/src/icon4py/model/standalone_driver/standalone_driver.py index 8036f8cdf9..2a9c3cfe70 100644 --- a/model/standalone_driver/src/icon4py/model/standalone_driver/standalone_driver.py +++ b/model/standalone_driver/src/icon4py/model/standalone_driver/standalone_driver.py @@ -22,7 +22,10 @@ from icon4py.model.atmosphere.diffusion import diffusion, diffusion_states from icon4py.model.atmosphere.dycore import dycore_states, solve_nonhydro as solve_nh from icon4py.model.common import dimension as dims, model_backends, model_options, type_alias as ta -from icon4py.model.common.decomposition import definitions as decomposition_defs +from icon4py.model.common.decomposition import ( + definitions as decomposition_defs, + mpi_decomposition as mpi_decomp, +) from icon4py.model.common.grid import geometry_attributes as geom_attr, vertical as v_grid from icon4py.model.common.grid.icon import IconGrid from icon4py.model.common.initialization import topography @@ -46,15 +49,18 @@ def __init__( config: driver_config.DriverConfig, backend: gtx.typing.Backend | None, grid: IconGrid, + decomposition_info: decomposition_defs.DecompositionInfo, static_field_factories: driver_states.StaticFieldFactories, diffusion_granule: diffusion.Diffusion, solve_nonhydro_granule: solve_nh.SolveNonhydro, vertical_grid_config: v_grid.VerticalGridConfig, tracer_advection_granule: advection.Advection, + exchange: decomposition_defs.ExchangeRuntime, ): self.config = config self.backend = backend self.grid = grid + self.decomposition_info = decomposition_info self.static_field_factories = static_field_factories self.diffusion = diffusion_granule self.solve_nonhydro = solve_nonhydro_granule @@ -64,6 +70,7 @@ def __init__( self.timer_collection = driver_states.TimerCollection( [timer.value for timer in driver_states.DriverTimers] ) + self.exchange = exchange driver_utils.display_driver_setup_in_log_file( self.model_time_variables.n_time_steps, @@ -182,6 +189,16 @@ def _integrate_one_time_step( prognostic_states.next, self.model_time_variables.dtime_in_seconds, ) + # TODO (jcanton,ongchia): move this exchange inside + # diffusion.run with the proper DiffusionConfig equivalent to + # the fortran + # IF ( linit .OR. (iforcing /= inwp .AND. iforcing /= iaes) ) THEN + self.exchange.exchange( + dims.CellDim, + prognostic_states.next.w, + prognostic_states.next.theta_v, + prognostic_states.next.exner, + ) timer_diffusion.capture() # TODO(ricoh): [c34] optionally move the loop into the granule (for efficiency gains) @@ -553,7 +570,8 @@ def initialize_driver( output_path: pathlib.Path, grid_file_path: pathlib.Path, log_level: str, - backend_name: str, + backend_name: str | model_backends.BackendLike | None, + force_serial_run: bool = False, ) -> Icon4pyDriver: """ Initialize the driver: @@ -573,8 +591,19 @@ def initialize_driver( Driver: driver object """ + # Detect if we're running under MPI (not just if mpi4py is installed). + # - mpi4py not installed → serial + # - mpi4py installed but COMM_WORLD.Get_size() == 1 → serial + # - mpi4py installed and COMM_WORLD.Get_size() > 1 → MPI mode + # - force_serial_run=True → always serial (reserved for single vs distributed tests) + if force_serial_run or mpi_decomp.mpi4py is None: + with_mpi = False + else: + mpi_decomp.init_mpi() + with_mpi = mpi_decomp.mpi4py.MPI.COMM_WORLD.Get_size() > 1 + parallel_props = decomposition_defs.get_processor_properties( - decomposition_defs.get_runtype(with_mpi=False) + decomposition_defs.get_runtype(with_mpi=with_mpi) ) driver_utils.configure_logging( logging_level=log_level, @@ -611,15 +640,12 @@ def initialize_driver( grid_file_path=grid_file_path, vertical_grid_config=vertical_grid_config, allocator=allocator, + parallel_props=parallel_props, global_reductions=global_reductions, ) log.info("creating the decomposition info") - - decomposition_info = driver_utils.create_decomposition_info( - grid_manager=grid_manager, - allocator=allocator, - ) + decomposition_info = grid_manager.decomposition_info exchange = decomposition_defs.create_exchange(parallel_props, decomposition_info) log.info("initializing the vertical grid") @@ -642,6 +668,8 @@ def initialize_driver( vertical_grid=vertical_grid, cell_topography=gtx.as_field((dims.CellDim,), data=cell_topography, allocator=allocator), # type: ignore[arg-type] # due to array_ns opacity backend=backend, + exchange=exchange, + global_reductions=global_reductions, ) log.info("initializing granules") @@ -668,11 +696,13 @@ def initialize_driver( config=driver_config, backend=backend, grid=grid_manager.grid, + decomposition_info=decomposition_info, static_field_factories=static_field_factories, diffusion_granule=diffusion_granule, solve_nonhydro_granule=solve_nonhydro_granule, vertical_grid_config=vertical_grid_config, tracer_advection_granule=tracer_advection_granule, + exchange=exchange, ) return icon4py_driver diff --git a/model/standalone_driver/src/icon4py/model/standalone_driver/testcases/initial_condition.py b/model/standalone_driver/src/icon4py/model/standalone_driver/testcases/initial_condition.py index 02899c8b9a..eaebdc29bb 100644 --- a/model/standalone_driver/src/icon4py/model/standalone_driver/testcases/initial_condition.py +++ b/model/standalone_driver/src/icon4py/model/standalone_driver/testcases/initial_condition.py @@ -5,7 +5,7 @@ # # Please, refer to the LICENSE file in the root directory. # SPDX-License-Identifier: BSD-3-Clause -import functools + import logging import math @@ -21,6 +21,7 @@ model_backends, type_alias as ta, ) +from icon4py.model.common.decomposition import definitions as decomposition_defs from icon4py.model.common.grid import ( geometry as grid_geometry, geometry_attributes as geometry_meta, @@ -57,6 +58,7 @@ def jablonowski_williamson( # noqa: PLR0915 [too-many-statements] model_top_height: float, stretch_factor: float, damping_height: float, + exchange: decomposition_defs.ExchangeRuntime = decomposition_defs.SingleNodeExchange, ) -> driver_states.DriverStates: """ Initial condition of Jablonowski-Williamson test. Set jw_baroclinic_amplitude to values larger than 0.01 if @@ -83,11 +85,18 @@ def jablonowski_williamson( # noqa: PLR0915 [too-many-statements] exner_ref_mc = metrics_field_source.get(metrics_attributes.EXNER_REF_MC).ndarray d_exner_dz_ref_ic = metrics_field_source.get(metrics_attributes.D_EXNER_DZ_REF_IC).ndarray geopot = phy_const.GRAV * metrics_field_source.get(metrics_attributes.Z_MC).ndarray + z_ifc = metrics_field_source.get(metrics_attributes.CELL_HEIGHT_ON_HALF_LEVEL).ndarray cell_lat = geometry_field_source.get(geometry_meta.CELL_LAT).ndarray edge_lat = geometry_field_source.get(geometry_meta.EDGE_LAT).ndarray edge_lon = geometry_field_source.get(geometry_meta.EDGE_LON).ndarray primal_normal_x = geometry_field_source.get(geometry_meta.EDGE_NORMAL_U).ndarray + inv_dual_edge_length = geometry_field_source.get( + f"inverse_of_{geometry_meta.DUAL_EDGE_LENGTH}" + ).ndarray + edge_cell_distance = geometry_field_source.get(geometry_meta.EDGE_CELL_DISTANCE).ndarray + primal_edge_length = geometry_field_source.get(geometry_meta.EDGE_LENGTH).ndarray + cell_area = geometry_field_source.get(geometry_meta.CELL_AREA).ndarray cell_2_edge_coeff = interpolation_field_source.get(interpolation_attributes.C_LIN_E) rbf_vec_coeff_c1 = interpolation_field_source.get(interpolation_attributes.RBF_VEC_COEFF_C1) @@ -240,11 +249,10 @@ def jablonowski_williamson( # noqa: PLR0915 [too-many-statements] vertical_end=num_levels, offset_provider=grid.connectivities, ) + exchange(eta_v_at_edge, dim=dims.EdgeDim) log.info("Cell-to-edge eta_v computation completed.") - prognostic_state_now.vn.ndarray[:, :] = functools.partial( - testcases_utils.zonalwind_2_normalwind_ndarray, array_ns=xp - )( + prognostic_state_now.vn.ndarray[:, :] = testcases_utils.zonalwind_2_normalwind_ndarray( grid=grid, jw_u0=jw_u0, jw_baroclinic_amplitude=jw_baroclinic_amplitude, @@ -254,7 +262,10 @@ def jablonowski_williamson( # noqa: PLR0915 [too-many-statements] edge_lon=edge_lon, primal_normal_x=primal_normal_x, eta_v_at_edge=eta_v_at_edge.ndarray, + array_ns=xp, ) + log.info("U2vn computation completed.") + vertical_config = v_grid.VerticalGridConfig( grid.num_levels, lowest_layer_thickness=lowest_layer_thickness, @@ -266,24 +277,20 @@ def jablonowski_williamson( # noqa: PLR0915 [too-many-statements] _, vct_b = v_grid.get_vct_a_and_vct_b(vertical_config, model_backends.get_allocator(backend)) prognostic_state_now.w.ndarray[:, :] = testcases_utils.init_w( - grid, - c2e=grid.get_connectivity(dims.C2E).ndarray, - e2c=grid.get_connectivity(dims.E2C).ndarray, - z_ifc=metrics_field_source.get(metrics_attributes.CELL_HEIGHT_ON_HALF_LEVEL).ndarray, - inv_dual_edge_length=geometry_field_source.get( - f"inverse_of_{geometry_meta.DUAL_EDGE_LENGTH}" - ).ndarray, - edge_cell_length=geometry_field_source.get(geometry_meta.EDGE_CELL_DISTANCE).ndarray, - primal_edge_length=geometry_field_source.get(geometry_meta.EDGE_LENGTH).ndarray, - cell_area=geometry_field_source.get(geometry_meta.CELL_AREA).ndarray, + grid=grid, + z_ifc=z_ifc, + inv_dual_edge_length=inv_dual_edge_length, + edge_cell_distance=edge_cell_distance, + primal_edge_length=primal_edge_length, + cell_area=cell_area, vn=prognostic_state_now.vn.ndarray, vct_b=vct_b.ndarray, nlev=num_levels, array_ns=xp, ) - log.info("U2vn computation completed.") + exchange(prognostic_state_now.w, dim=dims.CellDim) - functools.partial(testcases_utils.apply_hydrostatic_adjustment_ndarray, array_ns=xp)( + testcases_utils.apply_hydrostatic_adjustment_ndarray( rho=rho_ndarray, exner=exner_ndarray, theta_v=theta_v_ndarray, @@ -294,6 +301,7 @@ def jablonowski_williamson( # noqa: PLR0915 [too-many-statements] wgtfac_c=wgtfac_c, ddqz_z_half=ddqz_z_half, num_levels=num_levels, + array_ns=xp, ) log.info("Hydrostatic adjustment computation completed.") prognostic_state_next = prognostics.PrognosticState( @@ -317,6 +325,8 @@ def jablonowski_williamson( # noqa: PLR0915 [too-many-statements] vertical_end=num_levels, offset_provider=grid.connectivities, ) + exchange(diagnostic_state.u, dim=dims.CellDim) + exchange(diagnostic_state.v, dim=dims.CellDim) log.info("U, V computation completed.") diff --git a/model/standalone_driver/src/icon4py/model/standalone_driver/testcases/utils.py b/model/standalone_driver/src/icon4py/model/standalone_driver/testcases/utils.py index 86bf7390f5..f7161d53f5 100644 --- a/model/standalone_driver/src/icon4py/model/standalone_driver/testcases/utils.py +++ b/model/standalone_driver/src/icon4py/model/standalone_driver/testcases/utils.py @@ -5,12 +5,14 @@ # # Please, refer to the LICENSE file in the root directory. # SPDX-License-Identifier: BSD-3-Clause + from types import ModuleType import numpy as np from icon4py.model.common import constants as phy_const, dimension as dims from icon4py.model.common.grid import horizontal as h_grid, icon as icon_grid +from icon4py.model.common.math.stencils import generic_math_operations_array_ns from icon4py.model.common.utils import data_allocation as data_alloc @@ -172,11 +174,9 @@ def zonalwind_2_normalwind_ndarray( def init_w( grid: icon_grid.IconGrid, - c2e: data_alloc.NDArray, - e2c: data_alloc.NDArray, z_ifc: data_alloc.NDArray, inv_dual_edge_length: data_alloc.NDArray, - edge_cell_length: data_alloc.NDArray, + edge_cell_distance: data_alloc.NDArray, primal_edge_length: data_alloc.NDArray, cell_area: data_alloc.NDArray, vn: data_alloc.NDArray, @@ -184,32 +184,34 @@ def init_w( nlev: int, array_ns: ModuleType, ) -> data_alloc.NDArray: + # The bounds need to include the first halo line because of the e2c -> c2e connectivity lb_e = grid.start_index(h_grid.domain(dims.EdgeDim)(h_grid.Zone.LATERAL_BOUNDARY_LEVEL_2)) - ub_e = grid.end_index(h_grid.domain(dims.EdgeDim)(h_grid.Zone.INTERIOR)) - + ub_e = grid.end_index(h_grid.domain(dims.EdgeDim)(h_grid.Zone.END)) lb_c = grid.start_index(h_grid.domain(dims.CellDim)(h_grid.Zone.LATERAL_BOUNDARY_LEVEL_2)) - ub_c = grid.end_index(h_grid.domain(dims.CellDim)(h_grid.Zone.INTERIOR)) + ub_c = grid.end_index(h_grid.domain(dims.CellDim)(h_grid.Zone.END)) - z_wsfc_e = array_ns.zeros((ub_e,)) - for je in range(lb_e, ub_e): - z_wsfc_e[je] = ( - vn[je, nlev - 1] - * ((z_ifc[e2c[:, 1]] - z_ifc[e2c[:, 0]])[je, :] * inv_dual_edge_length[je])[nlev] - ) + c2e = grid.get_connectivity(dims.C2E).ndarray + e2c = grid.get_connectivity(dims.E2C).ndarray + + z_grad_e = generic_math_operations_array_ns.compute_directional_derivative_on_cells( + z_ifc[:, nlev], e2c, inv_dual_edge_length, lb_e, ub_e, grid.num_edges, array_ns + ) + z_wsfc_e = vn[:, nlev - 1] * z_grad_e + + z_wsfc_c = generic_math_operations_array_ns.interpolate_edges_to_cell( + z_wsfc_e, + c2e, + e2c, + edge_cell_distance, + primal_edge_length, + cell_area, + ub_c, + grid.num_cells, + array_ns, + ) - e_inn_c = array_ns.zeros((ub_c, 3)) # or 1 - for jc in range(ub_c): - for je in range(3): - idx_ce = 0 if e2c[c2e][jc, je, 0] == jc else 1 - e_inn_c[jc, je] = ( - edge_cell_length[c2e[jc, je], idx_ce] - * primal_edge_length[c2e[jc, je]] - / cell_area[jc] - ) - z_wsfc_c = array_ns.sum(z_wsfc_e[c2e] * e_inn_c, axis=1) - - w = array_ns.zeros((ub_c, nlev + 1)) - w[lb_c:, nlev] = z_wsfc_c[lb_c:ub_c] - w[lb_c:, 1:] = z_wsfc_c[lb_c:ub_c, array_ns.newaxis] * vct_b[array_ns.newaxis, 1:] + w = array_ns.zeros((grid.num_cells, nlev + 1)) + w[lb_c:ub_c, nlev] = z_wsfc_c[lb_c:ub_c] + w[lb_c:ub_c, 1:] = z_wsfc_c[lb_c:ub_c, array_ns.newaxis] * vct_b[array_ns.newaxis, 1:] return w diff --git a/model/standalone_driver/tests/standalone_driver/fixtures.py b/model/standalone_driver/tests/standalone_driver/fixtures.py index adf3f9f2ff..63014736a3 100644 --- a/model/standalone_driver/tests/standalone_driver/fixtures.py +++ b/model/standalone_driver/tests/standalone_driver/fixtures.py @@ -9,6 +9,7 @@ from icon4py.model.testing import serialbox from icon4py.model.testing.fixtures import ( + backend, damping_height, data_provider, download_ser_data, diff --git a/model/standalone_driver/tests/standalone_driver/integration_tests/test_initial_condition.py b/model/standalone_driver/tests/standalone_driver/integration_tests/test_initial_condition.py index aba7b35f9a..962a1dfd67 100644 --- a/model/standalone_driver/tests/standalone_driver/integration_tests/test_initial_condition.py +++ b/model/standalone_driver/tests/standalone_driver/integration_tests/test_initial_condition.py @@ -9,11 +9,10 @@ import pytest -from icon4py.model.common import dimension as dims, model_backends, model_options -from icon4py.model.common.utils import data_allocation as data_alloc -from icon4py.model.standalone_driver import driver_states, driver_utils, standalone_driver +from icon4py.model.common import model_backends +from icon4py.model.standalone_driver import driver_utils, standalone_driver from icon4py.model.standalone_driver.testcases import initial_condition -from icon4py.model.testing import definitions, grid_utils, serialbox, serialbox as sb, test_utils +from icon4py.model.testing import definitions, grid_utils, serialbox as sb, test_utils from icon4py.model.testing.fixtures.datatest import ( backend, backend_like, @@ -31,7 +30,7 @@ def test_standalone_driver_initial_condition( backend_like: model_backends.BackendLike, tmp_path: pathlib.Path, experiment: definitions.Experiment, - data_provider: serialbox.IconSerialDataProvider, + data_provider: sb.IconSerialDataProvider, ) -> None: backend_name = next( (k for k, v in model_backends.BACKENDS.items() if backend_like == v), "embedded" @@ -53,6 +52,7 @@ def test_standalone_driver_initial_condition( model_top_height=icon4py_driver.vertical_grid_config.model_top_height, stretch_factor=icon4py_driver.vertical_grid_config.stretch_factor, damping_height=icon4py_driver.vertical_grid_config.rayleigh_damping_height, + exchange=icon4py_driver.exchange, ) jabw_exit_savepoint = data_provider.from_savepoint_jabw_exit() diff --git a/model/standalone_driver/tests/standalone_driver/integration_tests/test_standalone_driver.py b/model/standalone_driver/tests/standalone_driver/integration_tests/test_standalone_driver.py index e3617291a2..af00872b57 100644 --- a/model/standalone_driver/tests/standalone_driver/integration_tests/test_standalone_driver.py +++ b/model/standalone_driver/tests/standalone_driver/integration_tests/test_standalone_driver.py @@ -9,11 +9,10 @@ import pytest -from icon4py.model.common import model_backends, model_options -from icon4py.model.common.utils import data_allocation as data_alloc -from icon4py.model.standalone_driver import driver_utils, main -from icon4py.model.testing import definitions, grid_utils, serialbox as sb, test_utils -from icon4py.model.testing.fixtures.datatest import backend, backend_like +from icon4py.model.common import model_backends +from icon4py.model.standalone_driver import main +from icon4py.model.testing import definitions as test_defs, grid_utils, serialbox as sb, test_utils +from icon4py.model.testing.fixtures.datatest import backend_like from ..fixtures import * # noqa: F403 @@ -24,7 +23,7 @@ "experiment, istep_exit, substep_exit, timeloop_date_init, timeloop_date_exit, step_date_exit, timeloop_diffusion_linit_init, timeloop_diffusion_linit_exit", [ ( - definitions.Experiments.JW, + test_defs.Experiments.JW, 2, 5, "2008-09-01T00:00:00.000", @@ -36,7 +35,7 @@ ], ) def test_standalone_driver( - experiment: definitions.Experiment, + experiment: test_defs.Experiment, timeloop_date_init: str, timeloop_date_exit: str, timeloop_diffusion_linit_init: bool, @@ -52,7 +51,7 @@ def test_standalone_driver( ) grid_file_path = grid_utils._download_grid_file(experiment.grid) output_path = tmp_path / f"ci_driver_output_for_backend_{backend_name}" - ds = main.main( + ds, _ = main.main( grid_file_path=grid_file_path, icon4py_backend=backend_name, output_path=output_path, @@ -66,7 +65,7 @@ def test_standalone_driver( assert test_utils.dallclose( ds.prognostics.current.vn.asnumpy(), vn_sp.asnumpy(), - atol=9e-7, + atol=5e-7, ) assert test_utils.dallclose( diff --git a/model/standalone_driver/tests/standalone_driver/mpi_tests/__init__.py b/model/standalone_driver/tests/standalone_driver/mpi_tests/__init__.py new file mode 100644 index 0000000000..de9850de36 --- /dev/null +++ b/model/standalone_driver/tests/standalone_driver/mpi_tests/__init__.py @@ -0,0 +1,7 @@ +# ICON4Py - ICON inspired code in Python and GT4Py +# +# Copyright (c) 2022-2024, ETH Zurich and MeteoSwiss +# All rights reserved. +# +# Please, refer to the LICENSE file in the root directory. +# SPDX-License-Identifier: BSD-3-Clause diff --git a/model/standalone_driver/tests/standalone_driver/mpi_tests/test_parallel_initial_conditions.py b/model/standalone_driver/tests/standalone_driver/mpi_tests/test_parallel_initial_conditions.py new file mode 100644 index 0000000000..541800fc37 --- /dev/null +++ b/model/standalone_driver/tests/standalone_driver/mpi_tests/test_parallel_initial_conditions.py @@ -0,0 +1,147 @@ +# ICON4Py - ICON inspired code in Python and GT4Py +# +# Copyright (c) 2022-2024, ETH Zurich and MeteoSwiss +# All rights reserved. +# +# Please, refer to the LICENSE file in the root directory. +# SPDX-License-Identifier: BSD-3-Clause + +import logging +import pathlib + +import pytest + +from icon4py.model.common import model_backends +from icon4py.model.common.decomposition import definitions as decomp_defs, mpi_decomposition +from icon4py.model.standalone_driver import driver_states, standalone_driver +from icon4py.model.standalone_driver.testcases import initial_condition +from icon4py.model.testing import definitions as test_defs, grid_utils, parallel_helpers +from icon4py.model.testing.fixtures.datatest import backend_like, experiment, processor_props + + +if mpi_decomposition.mpi4py is None: + pytest.skip("Skipping parallel tests on single node installation", allow_module_level=True) + +_log = logging.getLogger(__file__) + + +@pytest.mark.datatest +@pytest.mark.embedded_remap_error +@pytest.mark.parametrize( + "experiment", + [ + test_defs.Experiments.JW, + ], +) +@pytest.mark.mpi +@pytest.mark.parametrize("processor_props", [True], indirect=True) +def test_initial_condition_jablonowski_williamson_compare_single_multi_rank( + experiment: test_defs.Experiment, + tmp_path: pathlib.Path, + processor_props: decomp_defs.ProcessProperties, + backend_like: model_backends.BackendLike, +) -> None: + if experiment.grid.params.limited_area: + pytest.xfail("Limited-area grids not yet supported") + + _log.info(f"running on {processor_props.comm} with {processor_props.comm_size} ranks") + + backend_name = "embedded" # shut up pyright/mypy + for k, v in model_backends.BACKENDS.items(): + if backend_like == v: + backend_name = k + + grid_file_path = grid_utils._download_grid_file(experiment.grid) + + single_rank_icon4py_driver: standalone_driver.Icon4pyDriver = ( + standalone_driver.initialize_driver( + output_path=tmp_path / f"ci_driver_output_for_backend_{backend_name}_serial_rank0", + grid_file_path=grid_file_path, + log_level="info", + backend_name=backend_name, + force_serial_run=True, + ) + ) + + single_rank_ds: driver_states.DriverStates = initial_condition.jablonowski_williamson( + grid=single_rank_icon4py_driver.grid, + geometry_field_source=single_rank_icon4py_driver.static_field_factories.geometry_field_source, + interpolation_field_source=single_rank_icon4py_driver.static_field_factories.interpolation_field_source, + metrics_field_source=single_rank_icon4py_driver.static_field_factories.metrics_field_source, + backend=single_rank_icon4py_driver.backend, + lowest_layer_thickness=single_rank_icon4py_driver.vertical_grid_config.lowest_layer_thickness, + model_top_height=single_rank_icon4py_driver.vertical_grid_config.model_top_height, + stretch_factor=single_rank_icon4py_driver.vertical_grid_config.stretch_factor, + damping_height=single_rank_icon4py_driver.vertical_grid_config.rayleigh_damping_height, + exchange=single_rank_icon4py_driver.exchange, + ) + + multi_rank_icon4py_driver: standalone_driver.Icon4pyDriver = ( + standalone_driver.initialize_driver( + output_path=tmp_path / f"ci_driver_output_for_backend_{backend_name}_serial_rank0", + grid_file_path=grid_file_path, + log_level="info", + backend_name=backend_name, + ) + ) + + multi_rank_ds: driver_states.DriverStates = initial_condition.jablonowski_williamson( + grid=multi_rank_icon4py_driver.grid, + geometry_field_source=multi_rank_icon4py_driver.static_field_factories.geometry_field_source, + interpolation_field_source=multi_rank_icon4py_driver.static_field_factories.interpolation_field_source, + metrics_field_source=multi_rank_icon4py_driver.static_field_factories.metrics_field_source, + backend=multi_rank_icon4py_driver.backend, + lowest_layer_thickness=multi_rank_icon4py_driver.vertical_grid_config.lowest_layer_thickness, + model_top_height=multi_rank_icon4py_driver.vertical_grid_config.model_top_height, + stretch_factor=multi_rank_icon4py_driver.vertical_grid_config.stretch_factor, + damping_height=multi_rank_icon4py_driver.vertical_grid_config.rayleigh_damping_height, + exchange=multi_rank_icon4py_driver.exchange, + ) + + # TODO (jcanton/msimberg): unify the two checks below and remove code duplication + fields = ["vn", "w", "exner", "theta_v", "rho"] + serial_reference_fields: dict[str, object] = { + field_name: getattr(single_rank_ds.prognostics.current, field_name).asnumpy() + for field_name in fields + } + + for field_name in fields: + print(f"verifying field {field_name}") + global_reference_field = processor_props.comm.bcast( + serial_reference_fields.get(field_name), + root=0, + ) + local_field = getattr(multi_rank_ds.prognostics.current, field_name) + dim = local_field.domain.dims[0] + parallel_helpers.check_local_global_field( + decomposition_info=multi_rank_icon4py_driver.decomposition_info, + processor_props=processor_props, + dim=dim, + global_reference_field=global_reference_field, + local_field=local_field.asnumpy(), + check_halos=True, + atol=0.0, # TODO (jcanton, msimberg): only on CPU (probably?) + ) + + fields = ["u", "v"] + serial_reference_fields: dict[str, object] = { + field_name: getattr(single_rank_ds.diagnostic, field_name).asnumpy() + for field_name in fields + } + for field_name in fields: + print(f"verifying diagnostic field {field_name}") + global_reference_field = processor_props.comm.bcast( + serial_reference_fields.get(field_name), + root=0, + ) + local_field = getattr(multi_rank_ds.diagnostic, field_name) + dim = local_field.domain.dims[0] + parallel_helpers.check_local_global_field( + decomposition_info=multi_rank_icon4py_driver.decomposition_info, + processor_props=processor_props, + dim=dim, + global_reference_field=global_reference_field, + local_field=local_field.asnumpy(), + check_halos=True, + atol=0.0, # TODO (jcanton, msimberg): only on CPU (probably?) + ) diff --git a/model/standalone_driver/tests/standalone_driver/mpi_tests/test_parallel_standalone_driver.py b/model/standalone_driver/tests/standalone_driver/mpi_tests/test_parallel_standalone_driver.py new file mode 100644 index 0000000000..d60a7dee8c --- /dev/null +++ b/model/standalone_driver/tests/standalone_driver/mpi_tests/test_parallel_standalone_driver.py @@ -0,0 +1,201 @@ +# ICON4Py - ICON inspired code in Python and GT4Py +# +# Copyright (c) 2022-2024, ETH Zurich and MeteoSwiss +# All rights reserved. +# +# Please, refer to the LICENSE file in the root directory. +# SPDX-License-Identifier: BSD-3-Clause + +import logging +import pathlib + +import pytest + +from icon4py.model.common import model_backends, model_options +from icon4py.model.common.decomposition import definitions as decomp_defs, mpi_decomposition +from icon4py.model.standalone_driver import driver_utils, main +from icon4py.model.testing import ( + data_handling, + datatest_utils as dt_utils, + definitions as test_defs, + grid_utils, + parallel_helpers, +) +from icon4py.model.testing.fixtures.datatest import backend_like, experiment, processor_props + + +if mpi_decomposition.mpi4py is None: + pytest.skip("Skipping parallel tests on single node installation", allow_module_level=True) + +_log = logging.getLogger(__file__) + + +@pytest.mark.datatest +@pytest.mark.embedded_remap_error +@pytest.mark.parametrize( + "experiment", + [ + test_defs.Experiments.JW, + ], +) +@pytest.mark.mpi +@pytest.mark.parametrize("processor_props", [True], indirect=True) +def test_standalone_driver_compare_single_multi_rank( + experiment: test_defs.Experiment, + tmp_path: pathlib.Path, + processor_props: decomp_defs.ProcessProperties, + backend_like: model_backends.BackendLike, +) -> None: + if experiment.grid.params.limited_area: + pytest.xfail("Limited-area grids not yet supported") + + _log.info(f"running on {processor_props.comm} with {processor_props.comm_size} ranks") + + backend_name = "embedded" # shut up pyright/mypy + for k, v in model_backends.BACKENDS.items(): + if backend_like == v: + backend_name = k + + grid_file_path = grid_utils._download_grid_file(experiment.grid) + + single_rank_ds, _ = main.main( + grid_file_path=grid_file_path, + icon4py_backend=backend_name, + output_path=tmp_path / f"ci_driver_output_for_backend_{backend_name}_serial_rank0", + force_serial_run=True, + ) + + multi_rank_ds, decomposition_info = main.main( + grid_file_path=grid_file_path, + icon4py_backend=backend_name, + output_path=tmp_path + / f"ci_driver_output_for_backend_{backend_name}_mpi_rank_{processor_props.rank}", + ) + + fields = ["vn", "w", "exner", "theta_v", "rho"] + serial_reference_fields: dict[str, object] = { + field_name: getattr(single_rank_ds.prognostics.current, field_name).asnumpy() + for field_name in fields + } + + for field_name in fields: + print(f"\nverifying field {field_name}") + global_reference_field = processor_props.comm.bcast( + serial_reference_fields.get(field_name), + root=0, + ) + local_field = getattr(multi_rank_ds.prognostics.current, field_name) + dim = local_field.domain.dims[0] + parallel_helpers.check_local_global_field( + decomposition_info=decomposition_info, + processor_props=processor_props, + dim=dim, + global_reference_field=global_reference_field, + local_field=local_field.asnumpy(), + check_halos=True, + atol=0.0, # TODO (jcanton, msimberg): only on CPU (probably?) + ) + + +@pytest.mark.datatest +@pytest.mark.embedded_remap_error +@pytest.mark.parametrize( + "experiment, istep_exit, substep_exit, step_date_exit, timeloop_diffusion_linit_exit", + [ + ( + test_defs.Experiments.JW, + 2, + 5, + "2008-09-01T00:05:00.000", + False, + ), + ], +) +@pytest.mark.mpi +@pytest.mark.parametrize("processor_props", [True], indirect=True) +def test_run_single_step_serialized_data( + experiment: test_defs.Experiment, + istep_exit: int, + substep_exit: int, + step_date_exit: str, + timeloop_diffusion_linit_exit: bool, + tmp_path: pathlib.Path, + processor_props: decomp_defs.ProcessProperties, + backend_like: model_backends.BackendLike, +) -> None: + if experiment.grid.params.limited_area: + pytest.xfail("Limited-area grids not yet supported") + + _log.info(f"running on {processor_props.comm} with {processor_props.comm_size} ranks") + + backend_name = "embedded" # shut up pyright/mypy + for k, v in model_backends.BACKENDS.items(): + if backend_like == v: + backend_name = k + + grid_file_path = grid_utils._download_grid_file(experiment.grid) + + multi_rank_ds, decomposition_info = main.main( + grid_file_path=grid_file_path, + icon4py_backend=backend_name, + output_path=tmp_path + / f"ci_driver_output_for_backend_{backend_name}_mpi_rank_{processor_props.rank}", + ) + + serial_reference_fields = None + if processor_props.rank == 0: + single_rank_processor_props = decomp_defs.get_processor_properties( + decomp_defs.get_runtype(with_mpi=False) + ) + root_url = test_defs.SERIALIZED_DATA_ROOT_URLS[single_rank_processor_props.comm_size] + archive_filename = dt_utils.get_experiment_archive_filename( + experiment, single_rank_processor_props.comm_size + ) + archive_path = f"{test_defs.SERIALIZED_DATA_DIR}/{archive_filename}" + uri = dt_utils.get_serialized_data_url(root_url, archive_path) + data_path = dt_utils.get_datapath_for_experiment(experiment, single_rank_processor_props) + data_handling.download_test_data(data_path.parent, uri) + + backend = model_options.customize_backend( + program=None, backend=driver_utils.get_backend_from_name(backend_name) + ) + data_provider = dt_utils.create_icon_serial_data_provider( + data_path, single_rank_processor_props.rank, backend + ) + savepoint_nonhydro_exit = data_provider.from_savepoint_nonhydro_exit( + istep=istep_exit, + date=step_date_exit, + substep=substep_exit, + ) + savepoint_diffusion_exit = data_provider.from_savepoint_diffusion_exit( + linit=timeloop_diffusion_linit_exit, + date=step_date_exit, + ) + serial_reference_fields = { + "vn": savepoint_diffusion_exit.vn().asnumpy(), + "w": savepoint_diffusion_exit.w().asnumpy(), + "exner": savepoint_diffusion_exit.exner().asnumpy(), + "theta_v": savepoint_diffusion_exit.theta_v().asnumpy(), + "rho": savepoint_nonhydro_exit.rho_new().asnumpy(), + } + + fields = ["vn", "w", "exner", "theta_v", "rho"] + for field_name in fields: + print(f"verifying field {field_name}") + global_reference_field = processor_props.comm.bcast( + serial_reference_fields.get(field_name) + if serial_reference_fields is not None + else None, + root=0, + ) + local_field = getattr(multi_rank_ds.prognostics.current, field_name) + dim = local_field.domain.dims[0] + parallel_helpers.check_local_global_field( + decomposition_info=decomposition_info, + processor_props=processor_props, + dim=dim, + global_reference_field=global_reference_field, + local_field=local_field.asnumpy(), + check_halos=True, + atol=1e-6, + ) diff --git a/model/testing/src/icon4py/model/testing/definitions.py b/model/testing/src/icon4py/model/testing/definitions.py index deffe01997..fafb6b29cf 100644 --- a/model/testing/src/icon4py/model/testing/definitions.py +++ b/model/testing/src/icon4py/model/testing/definitions.py @@ -301,6 +301,21 @@ def construct_diffusion_config( return diffusion.DiffusionConfig( n_substeps=ndyn_substeps, ) + elif experiment == Experiments.JW: + return diffusion.DiffusionConfig( + diffusion_type=diffusion.DiffusionType.SMAGORINSKY_4TH_ORDER, + hdiff_w=True, + hdiff_vn=True, + hdiff_temp=False, + n_substeps=5, + type_t_diffu=diffusion.TemperatureDiscretizationType.HETEROGENEOUS, + type_vn_diffu=diffusion.SmagorinskyStencilType.DIAMOND_VERTICES, + hdiff_efdt_ratio=10.0, + hdiff_w_efdt_ratio=15.0, + smagorinski_scaling_factor=0.025, + zdiffu_t=False, + velocity_boundary_diffusion_denom=200.0, + ) else: raise NotImplementedError( f"DiffusionConfig for experiment {experiment.name} not implemented." diff --git a/model/testing/src/icon4py/model/testing/parallel_helpers.py b/model/testing/src/icon4py/model/testing/parallel_helpers.py index 50dde45d3d..ceea0dd05f 100644 --- a/model/testing/src/icon4py/model/testing/parallel_helpers.py +++ b/model/testing/src/icon4py/model/testing/parallel_helpers.py @@ -14,7 +14,7 @@ from gt4py import next as gtx from icon4py.model.common import dimension as dims -from icon4py.model.common.decomposition import definitions +from icon4py.model.common.decomposition import definitions as decomp_defs from icon4py.model.common.utils import data_allocation as data_alloc from icon4py.model.testing import test_utils @@ -23,17 +23,17 @@ def check_comm_size( - props: definitions.ProcessProperties, sizes: tuple[int, ...] = (1, 2, 4) + props: decomp_defs.ProcessProperties, sizes: tuple[int, ...] = (1, 2, 4) ) -> None: if props.comm_size not in sizes: pytest.xfail(f"wrong comm size: {props.comm_size}: test only works for comm-sizes: {sizes}") -def log_process_properties(props: definitions.ProcessProperties) -> None: +def log_process_properties(props: decomp_defs.ProcessProperties) -> None: _log.info(f"rank={props.rank}/{props.comm_size}") -def log_local_field_size(decomposition_info: definitions.DecompositionInfo) -> None: +def log_local_field_size(decomposition_info: decomp_defs.DecompositionInfo) -> None: _log.info( f"local grid size: cells={decomposition_info.global_index(dims.CellDim).size}, " f"edges={decomposition_info.global_index(dims.EdgeDim).size}, " @@ -41,7 +41,7 @@ def log_local_field_size(decomposition_info: definitions.DecompositionInfo) -> N ) -def gather_field(field: np.ndarray, props: definitions.ProcessProperties) -> tuple: +def gather_field(field: np.ndarray, props: decomp_defs.ProcessProperties) -> tuple: constant_dims = tuple(field.shape[1:]) _log.info(f"gather_field on rank={props.rank} - gathering field of local shape {field.shape}") # Because of sparse indexing the field may have a non-contigous layout, @@ -71,8 +71,8 @@ def gather_field(field: np.ndarray, props: definitions.ProcessProperties) -> tup def check_local_global_field( - decomposition_info: definitions.DecompositionInfo, - processor_props: definitions.ProcessProperties, # F811 # fixture + decomposition_info: decomp_defs.DecompositionInfo, + processor_props: decomp_defs.ProcessProperties, # F811 # fixture dim: gtx.Dimension, global_reference_field: np.ndarray, local_field: np.ndarray, @@ -88,30 +88,79 @@ def check_local_global_field( ) assert ( local_field.shape[0] - == decomposition_info.global_index(dim, definitions.DecompositionInfo.EntryType.ALL).shape[ + == decomposition_info.global_index(dim, decomp_defs.DecompositionInfo.EntryType.ALL).shape[ 0 ] ) + def _non_blocking_allclose( + a: np.ndarray, b: np.ndarray, atol: float, verbose: bool, label: str = "" + ) -> None: + max_diff = np.max(np.abs(a - b)) + color = "\033[1;31m" if max_diff > 0 else "\033[32m" + print(f"{color}{label} max diff {max_diff}\033[0m") + # Compare halo against global reference field if check_halos: - test_utils.assert_dallclose( + # test_utils.assert_dallclose( + _non_blocking_allclose( global_reference_field[ data_alloc.as_numpy( decomposition_info.global_index( - dim, definitions.DecompositionInfo.EntryType.HALO + dim, decomp_defs.DecompositionInfo.EntryType.HALO ) ) ], local_field[ data_alloc.as_numpy( decomposition_info.local_index( - dim, definitions.DecompositionInfo.EntryType.HALO + dim, decomp_defs.DecompositionInfo.EntryType.HALO ) ) ], atol=atol, verbose=True, + label="halos", + ) + a = global_reference_field[ + data_alloc.as_numpy( + decomposition_info.global_index( + dim, decomp_defs.DecompositionInfo.EntryType.HALO_LEVEL_1 + ) + ) + ] + b = local_field[ + data_alloc.as_numpy( + decomposition_info.local_index( + dim, decomp_defs.DecompositionInfo.EntryType.HALO_LEVEL_1 + ) + ) + ] + if a.shape[0] > 0 and b.shape[0] > 0: + _non_blocking_allclose( + a, b, + atol=atol, + verbose=True, + label="halos_1", + ) + _non_blocking_allclose( + global_reference_field[ + data_alloc.as_numpy( + decomposition_info.global_index( + dim, decomp_defs.DecompositionInfo.EntryType.HALO_LEVEL_2 + ) + ) + ], + local_field[ + data_alloc.as_numpy( + decomposition_info.local_index( + dim, decomp_defs.DecompositionInfo.EntryType.HALO_LEVEL_2 + ) + ) + ], + atol=atol, + verbose=True, + label="halos_2", ) # Compare owned local field, excluding halos, against global reference @@ -119,14 +168,14 @@ def check_local_global_field( # total we have the full global field distributed on all ranks. owned_entries = local_field[ data_alloc.as_numpy( - decomposition_info.local_index(dim, definitions.DecompositionInfo.EntryType.OWNED) + decomposition_info.local_index(dim, decomp_defs.DecompositionInfo.EntryType.OWNED) ) ] gathered_sizes, gathered_field = gather_field(owned_entries, processor_props) global_index_sizes, gathered_global_indices = gather_field( data_alloc.as_numpy( - decomposition_info.global_index(dim, definitions.DecompositionInfo.EntryType.OWNED) + decomposition_info.global_index(dim, decomp_defs.DecompositionInfo.EntryType.OWNED) ), processor_props, ) @@ -149,4 +198,7 @@ def check_local_global_field( f" rank = {processor_props.rank}: SHAPES: global reference field {global_reference_field.shape}, gathered = {gathered_field.shape}" ) - test_utils.assert_dallclose(sorted_, global_reference_field, atol=atol, verbose=True) + # test_utils.assert_dallclose(sorted_, global_reference_field, atol=atol, verbose=True) + _non_blocking_allclose( + sorted_, global_reference_field, atol=atol, verbose=True, label="internal" + ) diff --git a/pyproject.toml b/pyproject.toml index 5d7c802741..5fec613b38 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -378,6 +378,7 @@ url = 'https://gridtools.github.io/pypi/' [tool.uv.sources] dace = {index = "gridtools"} +gt4py = {git = "https://github.com/GridTools/gt4py", branch = "workaround_caching_issue_with_embedded_inverse_image_caching"} # gt4py = {git = "https://github.com/GridTools/gt4py", branch = "main"} # gt4py = {index = "test.pypi"} icon4py-atmosphere-advection = {workspace = true} diff --git a/uv.lock b/uv.lock index 876ce4f82d..5a215aa1f5 100644 --- a/uv.lock +++ b/uv.lock @@ -1428,8 +1428,8 @@ wheels = [ [[package]] name = "gt4py" -version = "1.1.8" -source = { registry = "https://pypi.org/simple" } +version = "1.1.6.post10+8f3567da" +source = { git = "https://github.com/GridTools/gt4py?branch=workaround_caching_issue_with_embedded_inverse_image_caching#8f3567da4fba75ad9ec68c29409f785d9ea95333" } dependencies = [ { name = "array-api-compat" }, { name = "attrs" }, @@ -1460,10 +1460,6 @@ dependencies = [ { name = "versioningit" }, { name = "xxhash" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/5c/79/413b2dc0f3f2dc5d708ba77fba4bd8852a2e3ca716ef889bbebd3b6e567c/gt4py-1.1.8.tar.gz", hash = "sha256:6bcaea3553ecd11361c7b5e521fd8a1f472ee9398250fd787d8178163eaa9f00", size = 822711, upload-time = "2026-03-25T15:00:28.923Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/c8/0f/854c3bb18e61416536629dfc464fa0ea918b6113625a18778fcf20e9b30f/gt4py-1.1.8-py3-none-any.whl", hash = "sha256:0aee2da8d50a29212cd578ac996491c881e511664534ffc2899ce36fa2657b42", size = 1034528, upload-time = "2026-03-25T15:00:27.338Z" }, -] [package.optional-dependencies] cuda11 = [ @@ -1794,7 +1790,7 @@ dependencies = [ [package.metadata] requires-dist = [ - { name = "gt4py", specifier = "==1.1.8" }, + { name = "gt4py", git = "https://github.com/GridTools/gt4py?branch=workaround_caching_issue_with_embedded_inverse_image_caching" }, { name = "icon4py-common", editable = "model/common" }, { name = "packaging", specifier = ">=20.0" }, ] @@ -1811,7 +1807,7 @@ dependencies = [ [package.metadata] requires-dist = [ - { name = "gt4py", specifier = "==1.1.8" }, + { name = "gt4py", git = "https://github.com/GridTools/gt4py?branch=workaround_caching_issue_with_embedded_inverse_image_caching" }, { name = "icon4py-common", editable = "model/common" }, { name = "packaging", specifier = ">=20.0" }, ] @@ -1828,7 +1824,7 @@ dependencies = [ [package.metadata] requires-dist = [ - { name = "gt4py", specifier = "==1.1.8" }, + { name = "gt4py", git = "https://github.com/GridTools/gt4py?branch=workaround_caching_issue_with_embedded_inverse_image_caching" }, { name = "icon4py-common", editable = "model/common" }, { name = "packaging", specifier = ">=20.0" }, ] @@ -1845,7 +1841,7 @@ dependencies = [ [package.metadata] requires-dist = [ - { name = "gt4py", specifier = "==1.1.8" }, + { name = "gt4py", git = "https://github.com/GridTools/gt4py?branch=workaround_caching_issue_with_embedded_inverse_image_caching" }, { name = "icon4py-common", editable = "model/common" }, { name = "packaging", specifier = ">=20.0" }, ] @@ -1863,7 +1859,7 @@ dependencies = [ [package.metadata] requires-dist = [ - { name = "gt4py", specifier = "==1.1.8" }, + { name = "gt4py", git = "https://github.com/GridTools/gt4py?branch=workaround_caching_issue_with_embedded_inverse_image_caching" }, { name = "icon4py-common", extras = ["io"], editable = "model/common" }, { name = "numpy", specifier = ">=1.23.3" }, { name = "packaging", specifier = ">=20.0" }, @@ -1933,9 +1929,9 @@ requires-dist = [ { name = "dace", specifier = "==43!2026.2.12", index = "https://gridtools.github.io/pypi/" }, { name = "datashader", marker = "extra == 'io'", specifier = ">=0.16.1" }, { name = "ghex", marker = "extra == 'distributed'", specifier = ">=0.5.1" }, - { name = "gt4py", specifier = "==1.1.8" }, - { name = "gt4py", extras = ["cuda11"], marker = "extra == 'cuda11'" }, - { name = "gt4py", extras = ["cuda12"], marker = "extra == 'cuda12'" }, + { name = "gt4py", git = "https://github.com/GridTools/gt4py?branch=workaround_caching_issue_with_embedded_inverse_image_caching" }, + { name = "gt4py", extras = ["cuda11"], marker = "extra == 'cuda11'", git = "https://github.com/GridTools/gt4py?branch=workaround_caching_issue_with_embedded_inverse_image_caching" }, + { name = "gt4py", extras = ["cuda12"], marker = "extra == 'cuda12'", git = "https://github.com/GridTools/gt4py?branch=workaround_caching_issue_with_embedded_inverse_image_caching" }, { name = "holoviews", marker = "extra == 'io'", specifier = ">=1.16.0" }, { name = "icon4py-common", extras = ["distributed", "io"], marker = "extra == 'all'", editable = "model/common" }, { name = "mpi4py", marker = "extra == 'distributed'", specifier = ">=3.1.5" }, @@ -1972,7 +1968,7 @@ dependencies = [ requires-dist = [ { name = "click", specifier = ">=8.0.1" }, { name = "devtools", specifier = ">=0.12" }, - { name = "gt4py", specifier = "==1.1.8" }, + { name = "gt4py", git = "https://github.com/GridTools/gt4py?branch=workaround_caching_issue_with_embedded_inverse_image_caching" }, { name = "icon4py-atmosphere-diffusion", editable = "model/atmosphere/diffusion" }, { name = "icon4py-atmosphere-dycore", editable = "model/atmosphere/dycore" }, { name = "icon4py-common", editable = "model/common" }, @@ -2001,7 +1997,7 @@ dependencies = [ [package.metadata] requires-dist = [ { name = "devtools", specifier = ">=0.12" }, - { name = "gt4py", specifier = "==1.1.8" }, + { name = "gt4py", git = "https://github.com/GridTools/gt4py?branch=workaround_caching_issue_with_embedded_inverse_image_caching" }, { name = "icon4py-atmosphere-diffusion", editable = "model/atmosphere/diffusion" }, { name = "icon4py-atmosphere-dycore", editable = "model/atmosphere/dycore" }, { name = "icon4py-common", editable = "model/common" }, @@ -2031,7 +2027,7 @@ dependencies = [ [package.metadata] requires-dist = [ { name = "filelock", specifier = ">=3.18.0" }, - { name = "gt4py", specifier = "==1.1.8" }, + { name = "gt4py", git = "https://github.com/GridTools/gt4py?branch=workaround_caching_issue_with_embedded_inverse_image_caching" }, { name = "icon4py-common", extras = ["io"], editable = "model/common" }, { name = "numpy", specifier = ">=1.23.3" }, { name = "packaging", specifier = ">=20.0" }, @@ -2083,9 +2079,9 @@ requires-dist = [ { name = "cupy-cuda11x", marker = "extra == 'cuda11'", specifier = ">=13.0" }, { name = "cupy-cuda12x", marker = "extra == 'cuda12'", specifier = ">=13.0" }, { name = "fprettify", specifier = ">=0.3.7" }, - { name = "gt4py", specifier = "==1.1.8" }, - { name = "gt4py", extras = ["cuda11"], marker = "extra == 'cuda11'" }, - { name = "gt4py", extras = ["cuda12"], marker = "extra == 'cuda12'" }, + { name = "gt4py", git = "https://github.com/GridTools/gt4py?branch=workaround_caching_issue_with_embedded_inverse_image_caching" }, + { name = "gt4py", extras = ["cuda11"], marker = "extra == 'cuda11'", git = "https://github.com/GridTools/gt4py?branch=workaround_caching_issue_with_embedded_inverse_image_caching" }, + { name = "gt4py", extras = ["cuda12"], marker = "extra == 'cuda12'", git = "https://github.com/GridTools/gt4py?branch=workaround_caching_issue_with_embedded_inverse_image_caching" }, { name = "icon4py-atmosphere-advection", editable = "model/atmosphere/advection" }, { name = "icon4py-atmosphere-diffusion", editable = "model/atmosphere/diffusion" }, { name = "icon4py-atmosphere-dycore", editable = "model/atmosphere/dycore" },