diff --git a/CMake/resolve_dependency_modules/cudf.cmake b/CMake/resolve_dependency_modules/cudf.cmake index a945660f8e7..c80b71a0a0c 100644 --- a/CMake/resolve_dependency_modules/cudf.cmake +++ b/CMake/resolve_dependency_modules/cudf.cmake @@ -17,12 +17,12 @@ include_guard(GLOBAL) # 3.30.4 is the minimum version required by cudf cmake_minimum_required(VERSION 3.30.4) -# rapids_cmake commit d79e071 from 2026-05-01 +# rapids_cmake commit d69f757 from 2026-05-14 set(VELOX_rapids_cmake_VERSION 26.06) -set(VELOX_rapids_cmake_COMMIT d79e071f805e709771b80008d50a8b3a5bed93ca) +set(VELOX_rapids_cmake_COMMIT d69f7578b64e3ea86d881c2676d49b1c34605c09) set( VELOX_rapids_cmake_BUILD_SHA256_CHECKSUM - d0f9eea4feaef1cc325e86eac787052ec951659fdcf21abdfb06efc337a63179 + e05431e6e56949b24f68e54276b308df625e999a46f3a6ff699fd862d780dd86 ) set( VELOX_rapids_cmake_SOURCE_URL @@ -30,22 +30,22 @@ set( ) velox_resolve_dependency_url(rapids_cmake) -# rmm commit 2357ddd from 2026-05-04 +# rmm commit 0b710a9 from 2026-05-29 set(VELOX_rmm_VERSION 26.06) -set(VELOX_rmm_COMMIT 2357ddddcff042ba378e8de6f89e4a995a23b2db) +set(VELOX_rmm_COMMIT 0b710a93571115d6c3711932c4f9bcf51c23c82c) set( VELOX_rmm_BUILD_SHA256_CHECKSUM - 61492c2da88e7f6a6a4edc7101cce4698c156704d135db0b80119f4b9a2c575c + 4468c6cea82eae2f6ba18b187d365d07e8227874880fbad365e4537ae325f5ac ) set(VELOX_rmm_SOURCE_URL "https://github.com/rapidsai/rmm/archive/${VELOX_rmm_COMMIT}.tar.gz") velox_resolve_dependency_url(rmm) -# kvikio commit 247b64e from 2026-05-02 +# kvikio commit d9717c5 from 2026-05-29 set(VELOX_kvikio_VERSION 26.06) -set(VELOX_kvikio_COMMIT 247b64e97ecb7cb9ccb06ab123aea87ac571c5b4) +set(VELOX_kvikio_COMMIT d9717c56c8ad1fce9f57242e488191d38016b142) set( VELOX_kvikio_BUILD_SHA256_CHECKSUM - 6419490de95e412cefdbafffc73fac6b2162bc2200076611883fe41637028198 + 306d4213f56cae320286ead7b50a6004c3139d2948822faf548daee529e62025 ) set( VELOX_kvikio_SOURCE_URL @@ -53,16 +53,40 @@ set( ) velox_resolve_dependency_url(kvikio) -# cudf commit d09d10d from 2026-05-04 +# cudf commit 5ea7e1f from 2026-05-29 set(VELOX_cudf_VERSION 26.06 CACHE STRING "cudf version") -set(VELOX_cudf_COMMIT d09d10d14d3ed932b8de93638809101af5c7fec3) +set(VELOX_cudf_COMMIT 5ea7e1f25e6621b4f7256bc88c8bf1de62f2a292) set( VELOX_cudf_BUILD_SHA256_CHECKSUM - 5042ec46beb8260eb60d13b9cd44f26357b9756628f7d58659c77e88c67e15d5 + ec7726869c5cd43d793bef1e0b652d149870a4783fe601797ee8e8302d81cae1 ) set(VELOX_cudf_SOURCE_URL "https://github.com/rapidsai/cudf/archive/${VELOX_cudf_COMMIT}.tar.gz") velox_resolve_dependency_url(cudf) +# Probe for a system UCX install. The variables are used only to gate ucxx +# fetching below; nothing in Velox links against UCX directly yet. +find_library(UCX_LIBRARY NAMES ucp) +find_path(UCX_INCLUDE_DIR NAMES ucp/api/ucp.h) +if(UCX_LIBRARY AND UCX_INCLUDE_DIR) + set(UCX_FOUND TRUE) +else() + set(UCX_FOUND FALSE) +endif() +if(UCX_FOUND) + message(STATUS "Found UCX: ${UCX_LIBRARY} (headers: ${UCX_INCLUDE_DIR}) -- ucxx will be fetched") + # ucxx commit 2e37c84 from 2026-05-29 (release/0.50 branch) + set(VELOX_ucxx_VERSION 0.50) + set(VELOX_ucxx_COMMIT 2e37c8463544064e680e51820c47bfec69f55b69) + set( + VELOX_ucxx_BUILD_SHA256_CHECKSUM + b0172dd8278f95c85f68ba9d35a01445965f33edb6ea3e344c7cb7e0cb8ce3e2 + ) + set(VELOX_ucxx_SOURCE_URL "https://github.com/rapidsai/ucxx/archive/${VELOX_ucxx_COMMIT}.tar.gz") + velox_resolve_dependency_url(ucxx) +else() + message(STATUS "UCX not found -- ucxx will not be fetched") +endif() + # Use block so we don't leak variables block(SCOPE_FOR VARIABLES) # Setup libcudf build to not have testing components @@ -104,8 +128,23 @@ block(SCOPE_FOR VARIABLES) UPDATE_DISCONNECTED 1 ) + if(UCX_FOUND) + FetchContent_Declare( + ucxx + URL ${VELOX_ucxx_SOURCE_URL} + URL_HASH ${VELOX_ucxx_BUILD_SHA256_CHECKSUM} + SOURCE_SUBDIR + cpp + UPDATE_DISCONNECTED 1 + ) + endif() + FetchContent_MakeAvailable(cudf) + if(UCX_FOUND) + FetchContent_MakeAvailable(ucxx) + endif() + # cudf sets all warnings as errors, and therefore fails to compile with velox # expanded set of warnings. We selectively disable problematic warnings just for # cudf diff --git a/scripts/setup-centos-adapters.sh b/scripts/setup-centos-adapters.sh index ded2cd556d3..1eba8c1eebf 100755 --- a/scripts/setup-centos-adapters.sh +++ b/scripts/setup-centos-adapters.sh @@ -25,13 +25,13 @@ # Use "n" to never wipe directories. # * VELOX_CUDA_VERSION="12.9": Which version of CUDA to install, will pick up # CUDA_VERSION from the env -# * VELOX_UCX_VERSION="1.19.0": Which version of ucx to install, will pick up +# * VELOX_UCX_VERSION="1.20.1": Which version of ucx to install, will pick up # UCX_VERSION from the env set -efx -o pipefail VELOX_CUDA_VERSION=${CUDA_VERSION:-"12.9"} -VELOX_UCX_VERSION=${UCX_VERSION:-"1.19.0"} +VELOX_UCX_VERSION=${UCX_VERSION:-"1.20.1"} SCRIPT_DIR=$(dirname "${BASH_SOURCE[0]}") source "$SCRIPT_DIR"/setup-centos9.sh diff --git a/scripts/update-cudf-deps.sh b/scripts/update-cudf-deps.sh index a7048f23177..b6608537419 100755 --- a/scripts/update-cudf-deps.sh +++ b/scripts/update-cudf-deps.sh @@ -127,15 +127,38 @@ get_version() { fi } +get_ucxx_version() { + # UCXX uses its own library version instead of the cuDF release version. + local ref=$1 + local response + response=$(fetch_github_api "repos/rapidsai/ucxx/contents/VERSION?ref=${ref}") || return 1 + echo "$response" | jq -r '.content' | base64 -d | grep -oP '^[0-9]+\.[0-9]+' || { + echo "Error: Failed to parse VERSION file from ucxx:$ref" >&2 + return 1 + } +} + +get_ucxx_branch() { + local branch=$1 version=$2 + if [[ $branch =~ ^release/ ]]; then + local ucxx_version + # This maps from cuDF release branches (YY.MM) to the corresponding UCXX version. + ucxx_version=$(curl -sL --max-time 30 "https://version.gpuci.io/rapids/${version}") || return 1 + echo "release/${ucxx_version}" + else + echo "$branch" + fi +} + update_dependency() { - local var=$1 commit=$2 date=$3 checksum=$4 + local var=$1 commit=$2 date=$3 checksum=$4 version=$5 sed -i "s/# ${var} commit [a-f0-9]* from [0-9-]*/# ${var} commit ${commit:0:7} from ${date}/" "$CMAKE_FILE" sed -i "s/set(VELOX_${var}_COMMIT [a-f0-9]*)/set(VELOX_${var}_COMMIT ${commit})/" "$CMAKE_FILE" if [[ $var == "cudf" ]]; then - sed -i "s/set(VELOX_${var}_VERSION [0-9.]* CACHE/set(VELOX_${var}_VERSION ${VERSION} CACHE/" "$CMAKE_FILE" + sed -i "s/set(VELOX_${var}_VERSION [0-9.]* CACHE/set(VELOX_${var}_VERSION ${version} CACHE/" "$CMAKE_FILE" else - sed -i "s/set(VELOX_${var}_VERSION [0-9.]*)/set(VELOX_${var}_VERSION ${VERSION})/" "$CMAKE_FILE" + sed -i "s/set(VELOX_${var}_VERSION [0-9.]*)/set(VELOX_${var}_VERSION ${version})/" "$CMAKE_FILE" fi awk -v var="VELOX_${var}_BUILD_SHA256_CHECKSUM" -v sum="$checksum" ' @@ -160,7 +183,7 @@ if [[ $MODE == "--pr" ]]; then echo " SHA256: $CHECKSUM" echo - update_dependency "cudf" "$SHA" "$DATE" "$CHECKSUM" + update_dependency "cudf" "$SHA" "$DATE" "$CHECKSUM" "$VERSION" echo "Done! Updated cudf to PR #${ARG}: ${SHA:0:7} ($DATE)" elif [[ $MODE == "--commit" ]]; then @@ -175,14 +198,15 @@ elif [[ $MODE == "--commit" ]]; then echo " Version: $VERSION" echo - declare -A COMMITS DATES CHECKSUMS + declare -A COMMITS DATES CHECKSUMS VERSIONS COMMITS[cudf]=$SHA DATES[cudf]=$DATE + VERSIONS[cudf]=$VERSION echo "Finding compatible dependency versions (main branch commits before $TIMESTAMP)..." echo - for dep in rapids_cmake rmm kvikio; do + for dep in rapids_cmake rmm kvikio ucxx; do repo=${dep//_/-} echo "Fetching $repo..." read -r commit date < <(get_commit_before_date "$repo" "$TIMESTAMP") || exit 1 @@ -194,6 +218,10 @@ elif [[ $MODE == "--commit" ]]; then COMMITS[$dep]=$commit DATES[$dep]=$date CHECKSUMS[$dep]=$checksum + VERSIONS[$dep]=$VERSION + if [[ $dep == "ucxx" ]]; then + VERSIONS[$dep]=$(get_ucxx_version "$commit") || exit 1 + fi echo done @@ -203,27 +231,32 @@ elif [[ $MODE == "--commit" ]]; then echo echo "Updating $CMAKE_FILE..." - for dep in rapids_cmake rmm kvikio cudf; do - update_dependency "$dep" "${COMMITS[$dep]}" "${DATES[$dep]}" "${CHECKSUMS[$dep]}" + for dep in rapids_cmake rmm kvikio ucxx cudf; do + update_dependency "$dep" "${COMMITS[$dep]}" "${DATES[$dep]}" "${CHECKSUMS[$dep]}" "${VERSIONS[$dep]}" done echo "Done! Updated dependencies:" - for dep in rapids_cmake rmm kvikio cudf; do + for dep in rapids_cmake rmm kvikio ucxx cudf; do echo " $dep: ${COMMITS[$dep]:0:7} (${DATES[$dep]})" done elif [[ $MODE == "--branch" ]]; then VERSION=$(get_version "$ARG") || exit 1 + UCXX_BRANCH=$(get_ucxx_branch "$ARG" "$VERSION") || exit 1 echo "Updating cuDF dependencies from branch $ARG (version $VERSION)" echo - declare -A COMMITS DATES CHECKSUMS + declare -A COMMITS DATES CHECKSUMS VERSIONS - for dep in rapids_cmake rmm kvikio cudf; do + for dep in rapids_cmake rmm kvikio ucxx cudf; do repo=${dep//_/-} echo "Fetching $repo..." - read -r commit date < <(get_commit_info "$repo" "$ARG") || exit 1 + if [[ $dep == "ucxx" ]]; then + read -r commit date < <(get_commit_info "$repo" "$UCXX_BRANCH") || exit 1 + else + read -r commit date < <(get_commit_info "$repo" "$ARG") || exit 1 + fi echo " Commit: ${commit:0:7} from $date" echo " Computing SHA256..." checksum=$(get_sha256 "$repo" "$commit") || exit 1 @@ -232,16 +265,20 @@ elif [[ $MODE == "--branch" ]]; then COMMITS[$dep]=$commit DATES[$dep]=$date CHECKSUMS[$dep]=$checksum + VERSIONS[$dep]=$VERSION + if [[ $dep == "ucxx" ]]; then + VERSIONS[$dep]=$(get_ucxx_version "$commit") || exit 1 + fi echo done echo "Updating $CMAKE_FILE..." - for dep in rapids_cmake rmm kvikio cudf; do - update_dependency "$dep" "${COMMITS[$dep]}" "${DATES[$dep]}" "${CHECKSUMS[$dep]}" + for dep in rapids_cmake rmm kvikio ucxx cudf; do + update_dependency "$dep" "${COMMITS[$dep]}" "${DATES[$dep]}" "${CHECKSUMS[$dep]}" "${VERSIONS[$dep]}" done echo "Done! Updated dependencies:" - for dep in rapids_cmake rmm kvikio cudf; do + for dep in rapids_cmake rmm kvikio ucxx cudf; do echo " $dep: ${COMMITS[$dep]:0:7} (${DATES[$dep]})" done else