Don't merge: Muphys staging branch

AMD_INTRODUCTION.md

@@ -0,0 +1,44 @@
+# Icon4py performance on MI300
+
+## Quickstart
+
+```
+# Connect to Beverin (CSCS system with MI300A)
+ssh beverin.cscs.ch
+```
+
+In Beverin:
+
+```
+# Enter scratch directory
+cd $SCRATCH
+
+# Clone icon4py and checkout the correct branch
+git clone git@github.com:C2SM/icon4py.git
+cd icon4py
+git checkout amd_profiling
+
+# Pull the correct `uenv` image. *!* NECESSARY ONLY ONCE *!*
+uenv image pull build::prgenv-gnu/25.12:2333839235
+
+# Start the uenv and mount the ROCm 7.1.0 environment. *!* This needs to be executed before running anything everytime *!*
+uenv start --view default prgenv-gnu/25.12:2333839235
+
+# Install the necessary venv
+bash amd_scripts/install_icon4py_venv.sh
+
+# Source venv
+source .venv/bin/activate
+
+# Source other necessary environment variables
+source amd_scripts/setup_env.sh
+
+# Set GT4Py related environment variables
+export GT4PY_UNSTRUCTURED_HORIZONTAL_HAS_UNIT_STRIDE="1"
+export GT4PY_BUILD_CACHE_LIFETIME=persistent
+export GT4PY_BUILD_CACHE_DIR=amd_profiling_granule
+export GT4PY_COLLECT_METRICS_LEVEL=10
+export GT4PY_DYCORE_ENABLE_METRICS="1"
+export GT4PY_ADD_GPU_TRACE_MARKERS="1"
+export HIPFLAGS="-std=c++17 -fPIC -O3 -march=native -Wno-unused-parameter -save-temps -Rpass-analysis=kernel-resource-usage"
+```

amd_scripts/install_icon4py_venv.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+
+set -e
+
+date
+
+# Go to the root of the icon4py repository to run the installation from there
+ICON4PY_GIT_ROOT=$(git rev-parse --show-toplevel)
+cd $ICON4PY_GIT_ROOT
+
+# Set necessasry flags for compilation
+source $ICON4PY_GIT_ROOT/amd_scripts/setup_env.sh
+
+# Install uv locally
+export PATH="$PWD/bin:$PATH"
+if [ ! -x "$PWD/bin/uv" ]; then
+    curl -LsSf https://astral.sh/uv/install.sh | UV_UNMANAGED_INSTALL="$PWD/bin" sh
+else
+    echo "# uv already installed at $PWD/bin/uv"
+fi
+
+# Install icon4py, gt4py, DaCe and other basic dependencies using uv
+uv sync --extra rocm7 --python $(which python3.12)
+
+# Activate virtual environment
+source .venv/bin/activate
+
+# Install the requirements for rocprofiler-compute so we can run the profiler from the same environment
+uv pip install -r /user-environment/linux-zen3/rocprofiler-compute-7.1.0-rjjjgkz67w66bp46jw7bvlfyduzr6vhv/libexec/rocprofiler-compute/requirements.txt
+
+echo "# install done"
+date

amd_scripts/setup_env.sh

@@ -0,0 +1,13 @@
+export CC="$(which gcc)"
+export MPICH_CC="$(which gcc)"
+export CXX="$(which g++)"
+export MPICH_CXX="$(which g++)"
+export HUGETLB_ELFMAP="no"
+export HUGETLB_MORECORE="no"
+export PYTHONOPTIMIZE="2"
+export HCC_AMDGPU_TARGET="gfx942"
+export ROCM_HOME="/user-environment/env/default"
+export HIPCC=$(which hipcc)
+export ROCM_VERSION="7.1.0"
+export LD_LIBRARY_PATH=/user-environment/linux-zen3/rocprofiler-dev-7.1.0-i7wbbbgrx7jjp4o2xroyj5j263dkzplv/lib:$LD_LIBRARY_PATH
+export LD_PRELOAD=/user-environment/env/default/lib/libomp.so:$LD_PRELOAD

model/atmosphere/advection/pyproject.toml

@@ -26,7 +26,7 @@ dependencies = [
   # workspace members
   "icon4py-common~=0.1.0",
   # external dependencies
-  "gt4py==1.1.8",
+  "gt4py==1.1.9",
   'packaging>=20.0'
 ]
 description = "ICON advection."

model/atmosphere/diffusion/pyproject.toml

@@ -26,7 +26,7 @@ dependencies = [
   # workspace members
   "icon4py-common~=0.1.0",
   # external dependencies
-  "gt4py==1.1.8",
+  "gt4py==1.1.9",
   'packaging>=20.0'
 ]
 description = "ICON diffusion."

model/atmosphere/dycore/pyproject.toml

@@ -26,7 +26,7 @@ dependencies = [
   # workspace members
   "icon4py-common~=0.1.0",
   # external dependencies
-  "gt4py==1.1.8",
+  "gt4py==1.1.9",
   'packaging>=20.0'
 ]
 description = "ICON dynamical core."

model/atmosphere/subgrid_scale_physics/microphysics/pyproject.toml

@@ -26,7 +26,7 @@ dependencies = [
   # workspace members
   "icon4py-common~=0.1.0",
   # external dependencies
-  "gt4py==1.1.8",
+  "gt4py==1.1.9",
   'packaging>=20.0'
 ]
 description = "ICON microphysics."

model/atmosphere/subgrid_scale_physics/muphys/pyproject.toml

@@ -26,7 +26,7 @@ dependencies = [
   "icon4py-common[io]~=0.1.0",
   # external dependencies
   "numpy>=1.23.3",
-  "gt4py==1.1.8",
+  "gt4py==1.1.9",
   "packaging>=20.0"
 ]
 description = "ICON subgrid scale muphys parameterization."

model/common/pyproject.toml

@@ -24,7 +24,7 @@ classifiers = [
 ]
 dependencies = [
   'array-api-compat>=1.13.0',
-  'gt4py==1.1.8',
+  'gt4py==1.1.9',
   'packaging>=20.0',
   'typing-extensions>=4.11.0',
   'numpy>=1.23.3',
@@ -43,6 +43,7 @@ version = "0.1.0"
 all = ["icon4py-common[distributed,io]"]
 cuda11 = ['cupy-cuda11x>=13.0', 'gt4py[cuda11]']
 cuda12 = ['cupy-cuda12x>=13.0', 'gt4py[cuda12]']
+rocm7 = ['gt4py[rocm7]']
 distributed = ["ghex>=0.5.1", "mpi4py>=3.1.5"]
 io = [
   # external dependencies

model/driver/pyproject.toml

@@ -31,7 +31,7 @@ dependencies = [
   # external dependencies
   "click>=8.0.1",
   "devtools>=0.12",
-  "gt4py==1.1.8",
+  "gt4py==1.1.9",
   "packaging>=20.0",
   "numpy>=1.23.3"
 ]

model/standalone_driver/pyproject.toml

@@ -32,7 +32,7 @@ dependencies = [
   "typer>=0.20.0",
   "pytest",
   "devtools>=0.12",
-  "gt4py==1.1.8",
+  "gt4py==1.1.9",
   "packaging>=20.0",
   "numpy>=1.23.3"
 ]

model/testing/pyproject.toml

@@ -27,7 +27,7 @@ dependencies = [
   'icon4py-common[io]~=0.1.0',
   # external dependencies
   "filelock>=3.18.0",
-  "gt4py==1.1.8",
+  "gt4py==1.1.9",
   "numpy>=1.23.3",
   'packaging>=20.0',
   "pytest>=8.0.1",

pyproject.toml

@@ -99,6 +99,7 @@ version = "0.1.0"
 all = ["icon4py[distributed,fortran,io,testing,profiling]"]
 cuda11 = ["icon4py-common[cuda11]"]
 cuda12 = ["icon4py-common[cuda12]"]
+rocm7 = ["icon4py-common[rocm7]"]
 distributed = ["icon4py-common[distributed]"]
 fortran = ["icon4py-tools~=0.1.0"]
 io = ["icon4py-common[io]"]
@@ -378,7 +379,7 @@ url = 'https://gridtools.github.io/pypi/'
 
 [tool.uv.sources]
 dace = {index = "gridtools"}
-# gt4py = {git = "https://github.com/GridTools/gt4py", branch = "main"}
+gt4py = {git = "https://github.com/havogt/gt4py", branch = "add_cuda13_rocm7_cupy14_testing"}
 # gt4py = {index = "test.pypi"}
 icon4py-atmosphere-advection = {workspace = true}
 icon4py-atmosphere-diffusion = {workspace = true}

tools/pyproject.toml

@@ -32,7 +32,7 @@ dependencies = [
   # external dependencies
   'clang-format>=22.1.0',
   'cffi>=1.5',
-  'gt4py==1.1.8',
+  'gt4py==1.1.9',
   "numpy>=1.23.3",
   'packaging>=20.0',
   'click>=8.1.7',