[CI] : Use llvm's python requirements.txt for common py packages.

build-requirements.txt

@@ -1,11 +1,9 @@
-numpy
-pybind11
+-r ./externals/llvm-project/mlir/python/requirements.txt
 wheel
 setuptools
 cmake
 ninja
 packaging
-nanobind>=2.4, <3.0
 
 # Workaround for what should be a torch dep
 # See discussion in #1174

build_tools/ci/build_posix.sh

@@ -2,19 +2,22 @@
 
 set -eu -o errtrace
 
-this_dir="$(cd $(dirname $0) && pwd)"
-repo_root="$(cd $this_dir/../.. && pwd)"
-build_dir="$repo_root/build"
+this_dir="$(cd "$(dirname "$0")" && pwd)"
+repo_root="$(cd "$this_dir"/../.. && pwd)"
 install_dir="$repo_root/install"
+
+# Setup build dir
+build_dir="${build_dir:-${repo_root}/build}"
 mkdir -p "$build_dir"
-build_dir="$(cd $build_dir && pwd)"
-cache_dir="${cache_dir:-}"
+build_dir="$(cd "${build_dir}" && pwd)"
+echo "Building in ${build_dir}"
 
 # Setup cache dir.
+cache_dir="${cache_dir:-}"
 if [ -z "${cache_dir}" ]; then
   cache_dir="${repo_root}/.build-cache"
   mkdir -p "${cache_dir}"
-  cache_dir="$(cd ${cache_dir} && pwd)"
+  cache_dir="$(cd "${cache_dir}" && pwd)"
 fi
 echo "Caching to ${cache_dir}"
 mkdir -p "${cache_dir}/ccache"
@@ -34,7 +37,7 @@ export CMAKE_CXX_COMPILER_LAUNCHER=ccache
 # Clear ccache stats.
 ccache -z
 
-cd $repo_root
+cd "$repo_root"
 
 echo "::group::CMake configure"
 cmake -S "$repo_root/externals/llvm-project/llvm" -B "$build_dir" \
@@ -59,7 +62,7 @@ cmake --build "$build_dir" --target tools/torch-mlir/all -- -k 0
 echo "::endgroup::"
 
 echo "::group::Unit tests"
-cmake --build $repo_root/build --target check-torch-mlir-all
+cmake --build "$build_dir" --target check-torch-mlir-all
 echo "::endgroup::"
 
 # Show ccache stats.

build_tools/ci/check_generated_sources.sh

@@ -2,15 +2,15 @@
 
 set -eu -o errtrace
 
-this_dir="$(cd $(dirname $0) && pwd)"
-repo_root="$(cd $this_dir/../.. && pwd)"
+this_dir="$(cd "$(dirname "$0")" && pwd)"
+repo_root="$(cd "$this_dir"/../.. && pwd)"
 
 function _check_file_not_changed_by() {
   # _check_file_not_changed_by <cmd> <file>
   cmd="$1"
   file="$2"
-  file_backup="$PWD/$(basename $file)"
-  file_new="$PWD/$(basename $file).new"
+  file_backup="$PWD/$(basename "$file")"
+  file_new="$PWD/$(basename "$file").new"
   # Save the original file.
   cp "$file" "$file_backup"
   # Run the command to regenerate it.
@@ -37,9 +37,9 @@ function _check_file_not_changed_by() {
 }
 
 echo "::group:: Check that update_abstract_interp_lib.sh has been run"
-_check_file_not_changed_by $repo_root/build_tools/update_abstract_interp_lib.sh $repo_root/lib/Dialect/Torch/Transforms/AbstractInterpLibrary.cpp
+_check_file_not_changed_by "$repo_root"/build_tools/update_abstract_interp_lib.sh "$repo_root"/lib/Dialect/Torch/Transforms/AbstractInterpLibrary.cpp
 echo "::endgroup::"
 
 echo "::group:: Check that update_torch_ods.sh has been run"
-_check_file_not_changed_by $repo_root/build_tools/update_torch_ods.sh $repo_root/include/torch-mlir/Dialect/Torch/IR/GeneratedTorchOps.td
+_check_file_not_changed_by "$repo_root"/build_tools/update_torch_ods.sh "$repo_root"/include/torch-mlir/Dialect/Torch/IR/GeneratedTorchOps.td
 echo "::endgroup::"

build_tools/ci/install_python_deps.sh

@@ -2,25 +2,30 @@
 
 set -eu -o errtrace
 
-this_dir="$(cd $(dirname $0) && pwd)"
-repo_root="$(cd $this_dir/../.. && pwd)"
+this_dir="$(cd "$(dirname "$0")" && pwd)"
+repo_root="$(cd "$this_dir"/../.. && pwd)"
 torch_version="${1:-unknown}"
 
+
+echo "::group::upgrading pip"
+python3 -m pip install --upgrade pip
+echo "::endgroup::"
+
 echo "::group::installing llvm python deps"
-python3 -m pip install --no-cache-dir -r $repo_root/externals/llvm-project/mlir/python/requirements.txt
+python3 -m pip install --no-cache-dir -r "$repo_root"/externals/llvm-project/mlir/python/requirements.txt
 echo "::endgroup::"
 
 case $torch_version in
   nightly)
     echo "::group::installing nightly torch"
-    python3 -m pip install --no-cache-dir -r $repo_root/requirements.txt
-    python3 -m pip install --no-cache-dir -r $repo_root/torchvision-requirements.txt
+    python3 -m pip install --no-cache-dir -r "$repo_root"/requirements.txt
+    python3 -m pip install --no-cache-dir -r "$repo_root"/torchvision-requirements.txt
     echo "::endgroup::"
     ;;
   stable)
     echo "::group::installing stable torch"
     python3 -m pip install --no-cache-dir torch torchvision --index-url https://download.pytorch.org/whl/cpu
-    python3 -m pip install --no-cache-dir -r $repo_root/build-requirements.txt
+    python3 -m pip install --no-cache-dir -r "$repo_root"/build-requirements.txt
     echo "::endgroup::"
     ;;
   *)
@@ -30,5 +35,5 @@ case $torch_version in
 esac
 
 echo "::group::installing test requirements"
-python3 -m pip install --no-cache-dir -r $repo_root/test-requirements.txt
+python3 -m pip install --no-cache-dir -r "$repo_root"/test-requirements.txt
 echo "::endgroup::"

build_tools/ci/test_posix.sh

@@ -2,11 +2,12 @@
 
 set -eu -o errtrace
 
-this_dir="$(cd $(dirname $0) && pwd)"
-repo_root="$(cd $this_dir/../.. && pwd)"
-torch_version="${1:-unknown}"
+this_dir="$(cd "$(dirname "$0")" && pwd)"
+repo_root="$(cd "$this_dir"/../.. && pwd)"
+torch_version="${1:-nightly}"
+build_dir="${build_dir:-${repo_root}/build}"
 
-export PYTHONPATH="$repo_root/build/tools/torch-mlir/python_packages/torch_mlir:$repo_root/projects/pt1"
+export PYTHONPATH="$build_dir/tools/torch-mlir/python_packages/torch_mlir:$repo_root/projects/pt1"
 
 echo "::group::Run ONNX e2e integration tests"
 python3 -m e2e_testing.main --config=onnx -v

build_tools/python_deploy/build_linux_packages.sh

@@ -337,7 +337,6 @@ function setup_venv() {
   source /main_checkout/torch-mlir/docker_venv/bin/activate
 
   echo ":::: pip installing dependencies"
-  python3 -m pip install --no-cache-dir -r /main_checkout/torch-mlir/externals/llvm-project/mlir/python/requirements.txt
 
   case $torch_version in
     nightly)