From 6b85cf86bbc36c837049babf437e3f3a703e9dc7 Mon Sep 17 00:00:00 2001 From: Sayan Saha Date: Tue, 12 Aug 2025 09:33:03 -0400 Subject: [PATCH 1/2] [CI] : Use llvm's python requirements.txt for common py packages. --- build-requirements.txt | 4 +--- build_tools/ci/install_python_deps.sh | 4 ---- build_tools/python_deploy/build_linux_packages.sh | 1 - 3 files changed, 1 insertion(+), 8 deletions(-) diff --git a/build-requirements.txt b/build-requirements.txt index f45b51399ac2..4279cb5a8075 100644 --- a/build-requirements.txt +++ b/build-requirements.txt @@ -1,11 +1,9 @@ -numpy -pybind11 +-r ./externals/llvm-project/mlir/python/requirements.txt wheel setuptools cmake ninja packaging -nanobind>=2.4, <3.0 # Workaround for what should be a torch dep # See discussion in #1174 diff --git a/build_tools/ci/install_python_deps.sh b/build_tools/ci/install_python_deps.sh index 6dca4ffe756c..8dfc99b1a472 100755 --- a/build_tools/ci/install_python_deps.sh +++ b/build_tools/ci/install_python_deps.sh @@ -6,10 +6,6 @@ this_dir="$(cd $(dirname $0) && pwd)" repo_root="$(cd $this_dir/../.. && pwd)" torch_version="${1:-unknown}" -echo "::group::installing llvm python deps" -python3 -m pip install --no-cache-dir -r $repo_root/externals/llvm-project/mlir/python/requirements.txt -echo "::endgroup::" - case $torch_version in nightly) echo "::group::installing nightly torch" diff --git a/build_tools/python_deploy/build_linux_packages.sh b/build_tools/python_deploy/build_linux_packages.sh index 094518de84a0..b53f7482b839 100755 --- a/build_tools/python_deploy/build_linux_packages.sh +++ b/build_tools/python_deploy/build_linux_packages.sh @@ -337,7 +337,6 @@ function setup_venv() { source /main_checkout/torch-mlir/docker_venv/bin/activate echo ":::: pip installing dependencies" - python3 -m pip install --no-cache-dir -r /main_checkout/torch-mlir/externals/llvm-project/mlir/python/requirements.txt case $torch_version in nightly) From 6c7664ba7d5f0cc0fe210a1cdb76968c6a255c2a Mon Sep 17 00:00:00 2001 From: Sayan Saha Date: Thu, 28 Aug 2025 09:38:00 -0400 Subject: [PATCH 2/2] [CI] : Add customizations for ci build/test scripts. --- build_tools/ci/build_posix.sh | 19 +++++++++++-------- build_tools/ci/check_generated_sources.sh | 12 ++++++------ build_tools/ci/install_python_deps.sh | 21 +++++++++++++++------ build_tools/ci/test_posix.sh | 9 +++++---- 4 files changed, 37 insertions(+), 24 deletions(-) diff --git a/build_tools/ci/build_posix.sh b/build_tools/ci/build_posix.sh index b9bb122acd37..b8dd3afa6753 100755 --- a/build_tools/ci/build_posix.sh +++ b/build_tools/ci/build_posix.sh @@ -2,19 +2,22 @@ set -eu -o errtrace -this_dir="$(cd $(dirname $0) && pwd)" -repo_root="$(cd $this_dir/../.. && pwd)" -build_dir="$repo_root/build" +this_dir="$(cd "$(dirname "$0")" && pwd)" +repo_root="$(cd "$this_dir"/../.. && pwd)" install_dir="$repo_root/install" + +# Setup build dir +build_dir="${build_dir:-${repo_root}/build}" mkdir -p "$build_dir" -build_dir="$(cd $build_dir && pwd)" -cache_dir="${cache_dir:-}" +build_dir="$(cd "${build_dir}" && pwd)" +echo "Building in ${build_dir}" # Setup cache dir. +cache_dir="${cache_dir:-}" if [ -z "${cache_dir}" ]; then cache_dir="${repo_root}/.build-cache" mkdir -p "${cache_dir}" - cache_dir="$(cd ${cache_dir} && pwd)" + cache_dir="$(cd "${cache_dir}" && pwd)" fi echo "Caching to ${cache_dir}" mkdir -p "${cache_dir}/ccache" @@ -34,7 +37,7 @@ export CMAKE_CXX_COMPILER_LAUNCHER=ccache # Clear ccache stats. ccache -z -cd $repo_root +cd "$repo_root" echo "::group::CMake configure" cmake -S "$repo_root/externals/llvm-project/llvm" -B "$build_dir" \ @@ -59,7 +62,7 @@ cmake --build "$build_dir" --target tools/torch-mlir/all -- -k 0 echo "::endgroup::" echo "::group::Unit tests" -cmake --build $repo_root/build --target check-torch-mlir-all +cmake --build "$build_dir" --target check-torch-mlir-all echo "::endgroup::" # Show ccache stats. diff --git a/build_tools/ci/check_generated_sources.sh b/build_tools/ci/check_generated_sources.sh index 719e221d71ba..81e86eab80f9 100755 --- a/build_tools/ci/check_generated_sources.sh +++ b/build_tools/ci/check_generated_sources.sh @@ -2,15 +2,15 @@ set -eu -o errtrace -this_dir="$(cd $(dirname $0) && pwd)" -repo_root="$(cd $this_dir/../.. && pwd)" +this_dir="$(cd "$(dirname "$0")" && pwd)" +repo_root="$(cd "$this_dir"/../.. && pwd)" function _check_file_not_changed_by() { # _check_file_not_changed_by cmd="$1" file="$2" - file_backup="$PWD/$(basename $file)" - file_new="$PWD/$(basename $file).new" + file_backup="$PWD/$(basename "$file")" + file_new="$PWD/$(basename "$file").new" # Save the original file. cp "$file" "$file_backup" # Run the command to regenerate it. @@ -37,9 +37,9 @@ function _check_file_not_changed_by() { } echo "::group:: Check that update_abstract_interp_lib.sh has been run" -_check_file_not_changed_by $repo_root/build_tools/update_abstract_interp_lib.sh $repo_root/lib/Dialect/Torch/Transforms/AbstractInterpLibrary.cpp +_check_file_not_changed_by "$repo_root"/build_tools/update_abstract_interp_lib.sh "$repo_root"/lib/Dialect/Torch/Transforms/AbstractInterpLibrary.cpp echo "::endgroup::" echo "::group:: Check that update_torch_ods.sh has been run" -_check_file_not_changed_by $repo_root/build_tools/update_torch_ods.sh $repo_root/include/torch-mlir/Dialect/Torch/IR/GeneratedTorchOps.td +_check_file_not_changed_by "$repo_root"/build_tools/update_torch_ods.sh "$repo_root"/include/torch-mlir/Dialect/Torch/IR/GeneratedTorchOps.td echo "::endgroup::" diff --git a/build_tools/ci/install_python_deps.sh b/build_tools/ci/install_python_deps.sh index 8dfc99b1a472..5c17f6ac144c 100755 --- a/build_tools/ci/install_python_deps.sh +++ b/build_tools/ci/install_python_deps.sh @@ -2,21 +2,30 @@ set -eu -o errtrace -this_dir="$(cd $(dirname $0) && pwd)" -repo_root="$(cd $this_dir/../.. && pwd)" +this_dir="$(cd "$(dirname "$0")" && pwd)" +repo_root="$(cd "$this_dir"/../.. && pwd)" torch_version="${1:-unknown}" + +echo "::group::upgrading pip" +python3 -m pip install --upgrade pip +echo "::endgroup::" + +echo "::group::installing llvm python deps" +python3 -m pip install --no-cache-dir -r "$repo_root"/externals/llvm-project/mlir/python/requirements.txt +echo "::endgroup::" + case $torch_version in nightly) echo "::group::installing nightly torch" - python3 -m pip install --no-cache-dir -r $repo_root/requirements.txt - python3 -m pip install --no-cache-dir -r $repo_root/torchvision-requirements.txt + python3 -m pip install --no-cache-dir -r "$repo_root"/requirements.txt + python3 -m pip install --no-cache-dir -r "$repo_root"/torchvision-requirements.txt echo "::endgroup::" ;; stable) echo "::group::installing stable torch" python3 -m pip install --no-cache-dir torch torchvision --index-url https://download.pytorch.org/whl/cpu - python3 -m pip install --no-cache-dir -r $repo_root/build-requirements.txt + python3 -m pip install --no-cache-dir -r "$repo_root"/build-requirements.txt echo "::endgroup::" ;; *) @@ -26,5 +35,5 @@ case $torch_version in esac echo "::group::installing test requirements" -python3 -m pip install --no-cache-dir -r $repo_root/test-requirements.txt +python3 -m pip install --no-cache-dir -r "$repo_root"/test-requirements.txt echo "::endgroup::" diff --git a/build_tools/ci/test_posix.sh b/build_tools/ci/test_posix.sh index bdeae5c7dfe4..66e8e8e43da4 100755 --- a/build_tools/ci/test_posix.sh +++ b/build_tools/ci/test_posix.sh @@ -2,11 +2,12 @@ set -eu -o errtrace -this_dir="$(cd $(dirname $0) && pwd)" -repo_root="$(cd $this_dir/../.. && pwd)" -torch_version="${1:-unknown}" +this_dir="$(cd "$(dirname "$0")" && pwd)" +repo_root="$(cd "$this_dir"/../.. && pwd)" +torch_version="${1:-nightly}" +build_dir="${build_dir:-${repo_root}/build}" -export PYTHONPATH="$repo_root/build/tools/torch-mlir/python_packages/torch_mlir:$repo_root/projects/pt1" +export PYTHONPATH="$build_dir/tools/torch-mlir/python_packages/torch_mlir:$repo_root/projects/pt1" echo "::group::Run ONNX e2e integration tests" python3 -m e2e_testing.main --config=onnx -v