Skip to content

build(cudf): Bring in ucxx when ucx is present and update all RAPIDS dependencies to latest #17572

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
zoltan wants to merge 6 commits into
base: main
Choose a base branch
from
+105 −29
Open

build(cudf): Bring in ucxx when ucx is present and update all RAPIDS dependencies to latest #17572

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
95cc0d3
build: Fetch ucxx when system UCX is available
zoltan May 20, 2026
1cec669
build: Include ucxx in cuDF dependency updates
bdice Jun 2, 2026
a9cbed3
build: Refresh cuDF dependency pins
bdice Jun 2, 2026
82e20fe
Refine ucxx selection logic
bdice Jun 3, 2026
7df95a6
Add comments
bdice Jun 3, 2026
226990b
build: Use UCX_FOUND to gate ucxx fetch
zoltan Jun 3, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
63 changes: 51 additions & 12 deletions CMake/resolve_dependency_modules/cudf.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,52 +17,76 @@ include_guard(GLOBAL)
# 3.30.4 is the minimum version required by cudf
cmake_minimum_required(VERSION 3.30.4)

# rapids_cmake commit d79e071 from 2026-05-01
# rapids_cmake commit d69f757 from 2026-05-14
set(VELOX_rapids_cmake_VERSION 26.06)
set(VELOX_rapids_cmake_COMMIT d79e071f805e709771b80008d50a8b3a5bed93ca)
set(VELOX_rapids_cmake_COMMIT d69f7578b64e3ea86d881c2676d49b1c34605c09)
set(
VELOX_rapids_cmake_BUILD_SHA256_CHECKSUM
d0f9eea4feaef1cc325e86eac787052ec951659fdcf21abdfb06efc337a63179
e05431e6e56949b24f68e54276b308df625e999a46f3a6ff699fd862d780dd86
)
set(
VELOX_rapids_cmake_SOURCE_URL
"https://github.com/rapidsai/rapids-cmake/archive/${VELOX_rapids_cmake_COMMIT}.tar.gz"
)
velox_resolve_dependency_url(rapids_cmake)

# rmm commit 2357ddd from 2026-05-04
# rmm commit 0b710a9 from 2026-05-29
set(VELOX_rmm_VERSION 26.06)
set(VELOX_rmm_COMMIT 2357ddddcff042ba378e8de6f89e4a995a23b2db)
set(VELOX_rmm_COMMIT 0b710a93571115d6c3711932c4f9bcf51c23c82c)
set(
VELOX_rmm_BUILD_SHA256_CHECKSUM
61492c2da88e7f6a6a4edc7101cce4698c156704d135db0b80119f4b9a2c575c
4468c6cea82eae2f6ba18b187d365d07e8227874880fbad365e4537ae325f5ac
)
set(VELOX_rmm_SOURCE_URL "https://github.com/rapidsai/rmm/archive/${VELOX_rmm_COMMIT}.tar.gz")
velox_resolve_dependency_url(rmm)

# kvikio commit 247b64e from 2026-05-02
# kvikio commit d9717c5 from 2026-05-29
set(VELOX_kvikio_VERSION 26.06)
set(VELOX_kvikio_COMMIT 247b64e97ecb7cb9ccb06ab123aea87ac571c5b4)
set(VELOX_kvikio_COMMIT d9717c56c8ad1fce9f57242e488191d38016b142)
set(
VELOX_kvikio_BUILD_SHA256_CHECKSUM
6419490de95e412cefdbafffc73fac6b2162bc2200076611883fe41637028198
306d4213f56cae320286ead7b50a6004c3139d2948822faf548daee529e62025
)
set(
VELOX_kvikio_SOURCE_URL
"https://github.com/rapidsai/kvikio/archive/${VELOX_kvikio_COMMIT}.tar.gz"
)
velox_resolve_dependency_url(kvikio)

# cudf commit d09d10d from 2026-05-04
# cudf commit 5ea7e1f from 2026-05-29
set(VELOX_cudf_VERSION 26.06 CACHE STRING "cudf version")
set(VELOX_cudf_COMMIT d09d10d14d3ed932b8de93638809101af5c7fec3)
set(VELOX_cudf_COMMIT 5ea7e1f25e6621b4f7256bc88c8bf1de62f2a292)
set(
VELOX_cudf_BUILD_SHA256_CHECKSUM
5042ec46beb8260eb60d13b9cd44f26357b9756628f7d58659c77e88c67e15d5
ec7726869c5cd43d793bef1e0b652d149870a4783fe601797ee8e8302d81cae1
)
set(VELOX_cudf_SOURCE_URL "https://github.com/rapidsai/cudf/archive/${VELOX_cudf_COMMIT}.tar.gz")
velox_resolve_dependency_url(cudf)

# Probe for a system UCX install. The variables are used only to gate ucxx
# fetching below; nothing in Velox links against UCX directly yet.
find_library(UCX_LIBRARY NAMES ucp)
find_path(UCX_INCLUDE_DIR NAMES ucp/api/ucp.h)
Comment thread
zoltan marked this conversation as resolved.
if(UCX_LIBRARY AND UCX_INCLUDE_DIR)
set(UCX_FOUND TRUE)
else()
set(UCX_FOUND FALSE)
endif()
if(UCX_FOUND)
message(STATUS "Found UCX: ${UCX_LIBRARY} (headers: ${UCX_INCLUDE_DIR}) -- ucxx will be fetched")
# ucxx commit 2e37c84 from 2026-05-29 (release/0.50 branch)
set(VELOX_ucxx_VERSION 0.50)
set(VELOX_ucxx_COMMIT 2e37c8463544064e680e51820c47bfec69f55b69)
set(
VELOX_ucxx_BUILD_SHA256_CHECKSUM
b0172dd8278f95c85f68ba9d35a01445965f33edb6ea3e344c7cb7e0cb8ce3e2
)
set(VELOX_ucxx_SOURCE_URL "https://github.com/rapidsai/ucxx/archive/${VELOX_ucxx_COMMIT}.tar.gz")
velox_resolve_dependency_url(ucxx)
else()
message(STATUS "UCX not found -- ucxx will not be fetched")
endif()

# Use block so we don't leak variables
block(SCOPE_FOR VARIABLES)
# Setup libcudf build to not have testing components
Expand Down Expand Up @@ -104,8 +128,23 @@ block(SCOPE_FOR VARIABLES)
UPDATE_DISCONNECTED 1
)

if(UCX_FOUND)
FetchContent_Declare(
ucxx
URL ${VELOX_ucxx_SOURCE_URL}
URL_HASH ${VELOX_ucxx_BUILD_SHA256_CHECKSUM}
SOURCE_SUBDIR
cpp
UPDATE_DISCONNECTED 1
)
endif()

FetchContent_MakeAvailable(cudf)

if(UCX_FOUND)
FetchContent_MakeAvailable(ucxx)
endif()

# cudf sets all warnings as errors, and therefore fails to compile with velox
# expanded set of warnings. We selectively disable problematic warnings just for
# cudf
Expand Down
4 changes: 2 additions & 2 deletions scripts/setup-centos-adapters.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,13 +25,13 @@
# Use "n" to never wipe directories.
# * VELOX_CUDA_VERSION="12.9": Which version of CUDA to install, will pick up
# CUDA_VERSION from the env
# * VELOX_UCX_VERSION="1.19.0": Which version of ucx to install, will pick up
# * VELOX_UCX_VERSION="1.20.1": Which version of ucx to install, will pick up
# UCX_VERSION from the env

set -efx -o pipefail

VELOX_CUDA_VERSION=${CUDA_VERSION:-"12.9"}
VELOX_UCX_VERSION=${UCX_VERSION:-"1.19.0"}
VELOX_UCX_VERSION=${UCX_VERSION:-"1.20.1"}
SCRIPT_DIR=$(dirname "${BASH_SOURCE[0]}")
source "$SCRIPT_DIR"/setup-centos9.sh

Expand Down
67 changes: 52 additions & 15 deletions scripts/update-cudf-deps.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -127,15 +127,38 @@ get_version() {
fi
}

get_ucxx_version() {
# UCXX uses its own library version instead of the cuDF release version.
local ref=$1
local response
response=$(fetch_github_api "repos/rapidsai/ucxx/contents/VERSION?ref=${ref}") || return 1
echo "$response" | jq -r '.content' | base64 -d | grep -oP '^[0-9]+\.[0-9]+' || {
echo "Error: Failed to parse VERSION file from ucxx:$ref" >&2
return 1
}
}

get_ucxx_branch() {
local branch=$1 version=$2
if [[ $branch =~ ^release/ ]]; then
local ucxx_version
# This maps from cuDF release branches (YY.MM) to the corresponding UCXX version.
ucxx_version=$(curl -sL --max-time 30 "https://version.gpuci.io/rapids/${version}") || return 1
echo "release/${ucxx_version}"
else
echo "$branch"
fi
}

update_dependency() {
local var=$1 commit=$2 date=$3 checksum=$4
local var=$1 commit=$2 date=$3 checksum=$4 version=$5
sed -i "s/# ${var} commit [a-f0-9]* from [0-9-]*/# ${var} commit ${commit:0:7} from ${date}/" "$CMAKE_FILE"
sed -i "s/set(VELOX_${var}_COMMIT [a-f0-9]*)/set(VELOX_${var}_COMMIT ${commit})/" "$CMAKE_FILE"

if [[ $var == "cudf" ]]; then
sed -i "s/set(VELOX_${var}_VERSION [0-9.]* CACHE/set(VELOX_${var}_VERSION ${VERSION} CACHE/" "$CMAKE_FILE"
sed -i "s/set(VELOX_${var}_VERSION [0-9.]* CACHE/set(VELOX_${var}_VERSION ${version} CACHE/" "$CMAKE_FILE"
else
sed -i "s/set(VELOX_${var}_VERSION [0-9.]*)/set(VELOX_${var}_VERSION ${VERSION})/" "$CMAKE_FILE"
sed -i "s/set(VELOX_${var}_VERSION [0-9.]*)/set(VELOX_${var}_VERSION ${version})/" "$CMAKE_FILE"
fi

awk -v var="VELOX_${var}_BUILD_SHA256_CHECKSUM" -v sum="$checksum" '
Expand All @@ -160,7 +183,7 @@ if [[ $MODE == "--pr" ]]; then
echo " SHA256: $CHECKSUM"
echo

update_dependency "cudf" "$SHA" "$DATE" "$CHECKSUM"
update_dependency "cudf" "$SHA" "$DATE" "$CHECKSUM" "$VERSION"
echo "Done! Updated cudf to PR #${ARG}: ${SHA:0:7} ($DATE)"

elif [[ $MODE == "--commit" ]]; then
Expand All @@ -175,14 +198,15 @@ elif [[ $MODE == "--commit" ]]; then
echo " Version: $VERSION"
echo

declare -A COMMITS DATES CHECKSUMS
declare -A COMMITS DATES CHECKSUMS VERSIONS
COMMITS[cudf]=$SHA
DATES[cudf]=$DATE
VERSIONS[cudf]=$VERSION

echo "Finding compatible dependency versions (main branch commits before $TIMESTAMP)..."
echo

for dep in rapids_cmake rmm kvikio; do
for dep in rapids_cmake rmm kvikio ucxx; do
repo=${dep//_/-}
echo "Fetching $repo..."
read -r commit date < <(get_commit_before_date "$repo" "$TIMESTAMP") || exit 1
Expand All @@ -194,6 +218,10 @@ elif [[ $MODE == "--commit" ]]; then
COMMITS[$dep]=$commit
DATES[$dep]=$date
CHECKSUMS[$dep]=$checksum
VERSIONS[$dep]=$VERSION
if [[ $dep == "ucxx" ]]; then
VERSIONS[$dep]=$(get_ucxx_version "$commit") || exit 1
fi
echo
done

Expand All @@ -203,27 +231,32 @@ elif [[ $MODE == "--commit" ]]; then
echo

echo "Updating $CMAKE_FILE..."
for dep in rapids_cmake rmm kvikio cudf; do
update_dependency "$dep" "${COMMITS[$dep]}" "${DATES[$dep]}" "${CHECKSUMS[$dep]}"
for dep in rapids_cmake rmm kvikio ucxx cudf; do
update_dependency "$dep" "${COMMITS[$dep]}" "${DATES[$dep]}" "${CHECKSUMS[$dep]}" "${VERSIONS[$dep]}"
done

echo "Done! Updated dependencies:"
for dep in rapids_cmake rmm kvikio cudf; do
for dep in rapids_cmake rmm kvikio ucxx cudf; do
echo " $dep: ${COMMITS[$dep]:0:7} (${DATES[$dep]})"
done

elif [[ $MODE == "--branch" ]]; then
VERSION=$(get_version "$ARG") || exit 1
UCXX_BRANCH=$(get_ucxx_branch "$ARG" "$VERSION") || exit 1
echo "Updating cuDF dependencies from branch $ARG (version $VERSION)"
echo

declare -A COMMITS DATES CHECKSUMS
declare -A COMMITS DATES CHECKSUMS VERSIONS

for dep in rapids_cmake rmm kvikio cudf; do
for dep in rapids_cmake rmm kvikio ucxx cudf; do
repo=${dep//_/-}
echo "Fetching $repo..."

read -r commit date < <(get_commit_info "$repo" "$ARG") || exit 1
if [[ $dep == "ucxx" ]]; then
read -r commit date < <(get_commit_info "$repo" "$UCXX_BRANCH") || exit 1
else
read -r commit date < <(get_commit_info "$repo" "$ARG") || exit 1
fi
echo " Commit: ${commit:0:7} from $date"
echo " Computing SHA256..."
checksum=$(get_sha256 "$repo" "$commit") || exit 1
Expand All @@ -232,16 +265,20 @@ elif [[ $MODE == "--branch" ]]; then
COMMITS[$dep]=$commit
DATES[$dep]=$date
CHECKSUMS[$dep]=$checksum
VERSIONS[$dep]=$VERSION
if [[ $dep == "ucxx" ]]; then
VERSIONS[$dep]=$(get_ucxx_version "$commit") || exit 1
fi
echo
done

echo "Updating $CMAKE_FILE..."
for dep in rapids_cmake rmm kvikio cudf; do
update_dependency "$dep" "${COMMITS[$dep]}" "${DATES[$dep]}" "${CHECKSUMS[$dep]}"
for dep in rapids_cmake rmm kvikio ucxx cudf; do
update_dependency "$dep" "${COMMITS[$dep]}" "${DATES[$dep]}" "${CHECKSUMS[$dep]}" "${VERSIONS[$dep]}"
done

echo "Done! Updated dependencies:"
for dep in rapids_cmake rmm kvikio cudf; do
for dep in rapids_cmake rmm kvikio ucxx cudf; do
echo " $dep: ${COMMITS[$dep]:0:7} (${DATES[$dep]})"
done
else
Expand Down
Loading