Update dependency (#455) #25
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: release | |
| on: | |
| workflow_dispatch: | |
| push: | |
| branches: [master] | |
| paths: | |
| - "Cargo.toml" | |
| permissions: | |
| contents: write | |
| id-token: write | |
| packages: write | |
| env: | |
| CARGO_TERM_COLOR: always | |
| jobs: | |
| validate: | |
| name: validate-version | |
| runs-on: ubuntu-latest | |
| if: github.repository == 'EricLBuehler/candle-vllm' | |
| outputs: | |
| version: ${{ steps.version.outputs.version }} | |
| changed: ${{ steps.version.outputs.changed }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Detect version change in Cargo.toml | |
| id: version | |
| run: | | |
| cargo_version="$(sed -n 's/^version = "\(.*\)"/\1/p' Cargo.toml | head -n 1)" | |
| before_sha="${{ github.event.before }}" | |
| if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then | |
| prev_version="" | |
| elif [ -n "$before_sha" ] && [ "$before_sha" != "0000000000000000000000000000000000000000" ]; then | |
| prev_version="$(git show "${before_sha}:Cargo.toml" 2>/dev/null | sed -n 's/^version = "\(.*\)"/\1/p' | head -n 1)" || prev_version="" | |
| else | |
| prev_version="" | |
| fi | |
| echo "version=${cargo_version}" >> "$GITHUB_OUTPUT" | |
| if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then | |
| echo "Manual release requested for ${cargo_version}." | |
| echo "changed=true" >> "$GITHUB_OUTPUT" | |
| elif [ "$cargo_version" != "$prev_version" ] && [ -n "$prev_version" ]; then | |
| echo "Version changed: ${prev_version} -> ${cargo_version}" | |
| echo "changed=true" >> "$GITHUB_OUTPUT" | |
| else | |
| echo "Version unchanged (${cargo_version}), skipping release." | |
| echo "changed=false" >> "$GITHUB_OUTPUT" | |
| fi | |
| # --------------------------------------------------------------------------- | |
| # CUDA x64 builds: SM70/75/80/89 on GitHub free runners. | |
| # Build in Ubuntu 20.04 so these artifacts run on Ubuntu 20.04-era hosts. | |
| # SM80/89 use flashinfer/cutlass, so keep build parallelism low. | |
| # --------------------------------------------------------------------------- | |
| cuda-x64: | |
| name: cuda-${{ matrix.sm_name }} | |
| runs-on: ubuntu-latest | |
| needs: validate | |
| if: needs.validate.outputs.changed == 'true' | |
| strategy: | |
| fail-fast: false | |
| max-parallel: 1 | |
| matrix: | |
| include: | |
| - sm: "70" | |
| sm_name: sm70 | |
| cuda_version: "12.6.3" | |
| features: "cuda,nccl" | |
| - sm: "75" | |
| sm_name: sm75 | |
| cuda_version: "12.6.3" | |
| features: "cuda,nccl" | |
| - sm: "80" | |
| sm_name: sm80 | |
| cuda_version: "12.6.3" | |
| features: "cuda,nccl,flashinfer,cutlass" | |
| - sm: "89" | |
| sm_name: sm89 | |
| cuda_version: "12.6.3" | |
| features: "cuda,nccl,flashinfer,cutlass" | |
| container: | |
| image: docker.io/nvidia/cuda:${{ matrix.cuda_version }}-cudnn-devel-ubuntu20.04 | |
| env: | |
| CUDA_COMPUTE_CAP: ${{ matrix.sm }} | |
| DEBIAN_FRONTEND: noninteractive | |
| CARGO_BUILD_JOBS: "2" | |
| CUDAFORGE_THREADS: "2" | |
| RAYON_NUM_THREADS: "2" | |
| steps: | |
| - uses: actions/checkout@v4 | |
| # Ubuntu 20.04 ships git 2.25.1, whose sparse-checkout can fail during | |
| # cudaforge's `git clone --filter/--sparse` of flashinfer. SM80/SM89 hit | |
| # this path through the `flashinfer` feature; install a modern Git first. | |
| - name: Install system dependencies | |
| run: | | |
| apt-get update | |
| apt-get install -y --no-install-recommends --allow-change-held-packages \ | |
| software-properties-common ca-certificates curl gnupg | |
| add-apt-repository -y ppa:git-core/ppa | |
| apt-get update | |
| apt-get install -y --no-install-recommends --allow-change-held-packages \ | |
| libnccl-dev libnccl2 \ | |
| curl git ca-certificates \ | |
| clang libclang-dev \ | |
| patchelf dpkg-dev \ | |
| perl make | |
| rm -rf /var/lib/apt/lists/* | |
| git --version | |
| - name: Install Rust | |
| run: | | |
| curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y | |
| echo "$HOME/.cargo/bin" >> "$GITHUB_PATH" | |
| - name: Install cargo-deb | |
| run: | | |
| . "$HOME/.cargo/env" | |
| cargo install cargo-deb | |
| - name: Build candle-vllm binary | |
| run: | | |
| . "$HOME/.cargo/env" | |
| cargo build --release --features "${{ matrix.features }}" | |
| - name: Bundle runtime libraries with binary | |
| run: bash .github/scripts/bundle-binary-libs.sh target/release/candle-vllm target/release/lib | |
| - name: Package deb | |
| run: | | |
| . "$HOME/.cargo/env" | |
| cargo deb --no-build --output "dist/candle-vllm-linux-x64-${{ matrix.sm_name }}.deb" | |
| - name: Package tar.gz | |
| run: | | |
| mkdir -p dist/candle-vllm | |
| cp target/release/candle-vllm dist/candle-vllm/candle-vllm | |
| cp -a target/release/lib dist/candle-vllm/lib | |
| tar -C dist/candle-vllm -czf "dist/candle-vllm-${{ needs.validate.outputs.version }}-linux-x64-${{ matrix.sm_name }}.tar.gz" candle-vllm lib | |
| rm -rf dist/candle-vllm | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: candle-vllm-cuda-${{ matrix.sm_name }} | |
| path: | | |
| dist/*.tar.gz | |
| dist/*.deb | |
| # --------------------------------------------------------------------------- | |
| # CUDA 13 builds: SM90+ on GitHub free runners with limited parallelism | |
| # max-parallel: 1 to avoid OOM on shared runners. | |
| # --------------------------------------------------------------------------- | |
| cuda: | |
| name: cuda-${{ matrix.sm_name }} | |
| runs-on: ubuntu-latest | |
| needs: validate | |
| if: needs.validate.outputs.changed == 'true' | |
| strategy: | |
| fail-fast: false | |
| max-parallel: 1 | |
| matrix: | |
| include: | |
| - sm: "90" | |
| sm_name: sm90 | |
| cuda_version: "13.0.0" | |
| features: "cuda,nccl,flashinfer,cutlass" | |
| - sm: "100" | |
| sm_name: sm100 | |
| cuda_version: "13.0.0" | |
| features: "cuda,nccl,flashinfer,cutlass" | |
| - sm: "120" | |
| sm_name: sm120 | |
| cuda_version: "13.0.0" | |
| features: "cuda,nccl,flashinfer,cutlass" | |
| container: | |
| image: docker.io/nvidia/cuda:${{ matrix.cuda_version }}-cudnn-devel-ubuntu22.04 | |
| env: | |
| CUDA_COMPUTE_CAP: ${{ matrix.sm }} | |
| DEBIAN_FRONTEND: noninteractive | |
| CARGO_BUILD_JOBS: "2" | |
| CUDAFORGE_THREADS: "2" | |
| RAYON_NUM_THREADS: "2" | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Install system dependencies | |
| run: | | |
| apt-get update | |
| apt-get install -y --no-install-recommends --allow-change-held-packages \ | |
| libnccl-dev libnccl2 \ | |
| curl git ca-certificates \ | |
| clang libclang-dev \ | |
| patchelf dpkg-dev \ | |
| perl make | |
| rm -rf /var/lib/apt/lists/* | |
| - name: Install Rust | |
| run: | | |
| curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y | |
| echo "$HOME/.cargo/bin" >> "$GITHUB_PATH" | |
| - name: Install cargo-deb | |
| run: | | |
| . "$HOME/.cargo/env" | |
| cargo install cargo-deb | |
| - name: Build candle-vllm binary | |
| run: | | |
| . "$HOME/.cargo/env" | |
| cargo build --release --features "${{ matrix.features }}" | |
| - name: Bundle runtime libraries with binary | |
| run: bash .github/scripts/bundle-binary-libs.sh target/release/candle-vllm target/release/lib | |
| - name: Package deb | |
| run: | | |
| . "$HOME/.cargo/env" | |
| cargo deb --no-build --output "dist/candle-vllm-linux-x64-${{ matrix.sm_name }}.deb" | |
| - name: Package tar.gz | |
| run: | | |
| mkdir -p dist/candle-vllm | |
| cp target/release/candle-vllm dist/candle-vllm/candle-vllm | |
| cp -a target/release/lib dist/candle-vllm/lib | |
| tar -C dist/candle-vllm -czf "dist/candle-vllm-${{ needs.validate.outputs.version }}-linux-x64-${{ matrix.sm_name }}.tar.gz" candle-vllm lib | |
| rm -rf dist/candle-vllm | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: candle-vllm-cuda-${{ matrix.sm_name }} | |
| path: | | |
| dist/*.tar.gz | |
| dist/*.deb | |
| # --------------------------------------------------------------------------- | |
| # Metal build (macOS) — GitHub-hosted runner | |
| # --------------------------------------------------------------------------- | |
| metal: | |
| name: metal-darwin-arm64 | |
| runs-on: macos-latest | |
| needs: validate | |
| if: needs.validate.outputs.changed == 'true' | |
| env: | |
| # Metal BF16 kernels require macOS 14+ / Metal 3.1. Lower deployment | |
| # targets compile successfully but omit __HAVE_BFLOAT__ kernel variants. | |
| MACOSX_DEPLOYMENT_TARGET: "14.0" | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: dtolnay/rust-toolchain@stable | |
| with: | |
| targets: aarch64-apple-darwin | |
| - name: Build candle-vllm binary | |
| run: cargo build --release --target aarch64-apple-darwin --features "metal" | |
| - name: Package tar.gz | |
| run: | | |
| mkdir -p "dist/candle-vllm" | |
| cp "target/aarch64-apple-darwin/release/candle-vllm" "dist/candle-vllm/candle-vllm" | |
| tar -C dist/candle-vllm -czf "dist/candle-vllm-${{ needs.validate.outputs.version }}-darwin-arm64-metal.tar.gz" candle-vllm | |
| rm -rf dist/candle-vllm | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: candle-vllm-darwin-arm64-metal | |
| path: | | |
| dist/*.tar.gz | |
| # --------------------------------------------------------------------------- | |
| # Checksums + GitHub Release | |
| # --------------------------------------------------------------------------- | |
| checksums: | |
| name: checksums | |
| runs-on: ubuntu-latest | |
| needs: [validate, cuda-x64, cuda, metal] | |
| if: needs.validate.outputs.changed == 'true' | |
| steps: | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| path: dist | |
| merge-multiple: true | |
| - name: Generate SHA256SUMS | |
| run: | | |
| cd dist | |
| find . -type f \( -name "*.tar.gz" -o -name "*.deb" \) \ | |
| -exec sha256sum {} + > SHA256SUMS | |
| cat SHA256SUMS | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: candle-vllm-release-artifacts | |
| path: dist/* | |
| - name: Create tag and upload artifacts to release | |
| uses: softprops/action-gh-release@v2 | |
| with: | |
| tag_name: v${{ needs.validate.outputs.version }} | |
| name: candle-vllm v${{ needs.validate.outputs.version }} | |
| generate_release_notes: true | |
| files: | | |
| dist/*.tar.gz | |
| dist/*.deb | |
| dist/SHA256SUMS | |
| # --------------------------------------------------------------------------- | |
| # Publish documentation site + install script to GitHub Pages | |
| # --------------------------------------------------------------------------- | |
| publish-pages: | |
| name: publish-pages | |
| runs-on: ubuntu-latest | |
| needs: [validate, checksums] | |
| if: needs.validate.outputs.changed == 'true' | |
| permissions: | |
| contents: read | |
| pages: write | |
| id-token: write | |
| environment: | |
| name: github-pages | |
| url: ${{ steps.deploy.outputs.page_url }} | |
| concurrency: | |
| group: pages | |
| cancel-in-progress: false | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| sparse-checkout: paged | |
| - name: Build site | |
| env: | |
| VERSION: ${{ needs.validate.outputs.version }} | |
| REPO: ${{ github.repository }} | |
| run: | | |
| OWNER="$(echo "${REPO}" | cut -d/ -f1)" | |
| OWNER_LC="$(echo "${OWNER}" | tr '[:upper:]' '[:lower:]')" | |
| PAGES_HOST="${OWNER_LC}.github.io" | |
| # Copy the full documentation site | |
| cp -a paged site | |
| # Patch downloads.html with version/repo placeholders | |
| sed -i \ | |
| -e "s|var CANDLE_VLLM_VER='__VERSION__';|var CANDLE_VLLM_VER='${VERSION}';|" \ | |
| -e "s|__REPO__|${REPO}|g" \ | |
| site/downloads.html | |
| # Generate install.sh at site root (preserves existing URL) | |
| cat > site/install.sh << 'SCRIPT' | |
| #!/usr/bin/env bash | |
| set -euo pipefail | |
| VERSION="__VERSION__" | |
| PAGES_BASE="https://__OWNER__.github.io/candle-vllm" | |
| RELEASE_BASE="https://github.com/__REPO__/releases/download/v${VERSION}" | |
| detect_platform() { | |
| if [ "$(uname -s)" = "Darwin" ]; then | |
| echo "metal" | |
| return | |
| fi | |
| if ! command -v nvidia-smi &>/dev/null; then | |
| echo "ERROR: No NVIDIA GPU detected and no platform specified." >&2 | |
| echo " Set CANDLE_VLLM_PLATFORM=sm80 (or sm70/sm75/sm89/sm90/sm100/sm120/sm121) and re-run." >&2 | |
| exit 1 | |
| fi | |
| local cc | |
| cc="$(nvidia-smi --query-gpu=compute_cap --format=csv,noheader,nounits | head -1 | tr -d '[:space:]' | tr -d '.')" | |
| case "$cc" in | |
| 70|72) echo "sm70" ;; | |
| 75) echo "sm75" ;; | |
| 80|86|87) echo "sm80" ;; | |
| 89) echo "sm89" ;; | |
| 90|90a) echo "sm90" ;; | |
| 100|100a) echo "sm100" ;; | |
| 120|120a) echo "sm120" ;; | |
| 121|121a) echo "sm121" ;; | |
| *) echo "sm80" ;; | |
| esac | |
| } | |
| cuda_build_for_platform() { | |
| case "$1" in | |
| sm70|sm75|sm80|sm89) echo "12.6" ;; | |
| sm90|sm100|sm120|sm121) echo "13.0" ;; | |
| metal) echo "none" ;; | |
| *) echo "12.6" ;; | |
| esac | |
| } | |
| min_driver_for_cuda() { | |
| case "$1" in | |
| 13.*) echo "580.65.06" ;; | |
| 12.*) echo "525.60.13" ;; | |
| *) echo "525.60.13" ;; | |
| esac | |
| } | |
| version_ge() { | |
| [ "$1" = "$2" ] && return 0 | |
| [ "$(printf '%s\n%s\n' "$2" "$1" | sort -V | head -n1)" = "$2" ] | |
| } | |
| detect_driver_version() { | |
| if ! command -v nvidia-smi &>/dev/null; then | |
| return 1 | |
| fi | |
| nvidia-smi --query-gpu=driver_version --format=csv,noheader 2>/dev/null \ | |
| | head -1 | tr -d '[:space:]' | |
| } | |
| check_driver_compat() { | |
| local platform="$1" | |
| local cuda_build min_driver driver | |
| cuda_build="$(cuda_build_for_platform "$platform")" | |
| if [ "$cuda_build" = "none" ]; then | |
| return 0 | |
| fi | |
| if [ "${CANDLE_VLLM_SKIP_DRIVER_CHECK:-}" = "1" ]; then | |
| echo "Skipping NVIDIA driver compatibility check because CANDLE_VLLM_SKIP_DRIVER_CHECK=1." | |
| return 0 | |
| fi | |
| min_driver="$(min_driver_for_cuda "$cuda_build")" | |
| driver="$(detect_driver_version || true)" | |
| if [ -z "$driver" ]; then | |
| echo "ERROR: Could not detect NVIDIA driver with nvidia-smi." >&2 | |
| echo " ${platform} package is built with CUDA ${cuda_build} and requires driver >= ${min_driver}." >&2 | |
| echo " Set CANDLE_VLLM_SKIP_DRIVER_CHECK=1 only if you know the target runtime driver is compatible." >&2 | |
| exit 1 | |
| fi | |
| if ! version_ge "$driver" "$min_driver"; then | |
| echo "ERROR: NVIDIA driver ${driver} is too old for ${platform} package." >&2 | |
| echo " ${platform} package is built with CUDA ${cuda_build} and requires driver >= ${min_driver}." >&2 | |
| echo " Upgrade the NVIDIA driver or install a package built for an older CUDA family." >&2 | |
| exit 1 | |
| fi | |
| echo "NVIDIA driver ${driver} is compatible with ${platform} CUDA ${cuda_build} package." | |
| } | |
| PLATFORM="${CANDLE_VLLM_PLATFORM:-$(detect_platform)}" | |
| echo "Detected platform: ${PLATFORM}" | |
| check_driver_compat "$PLATFORM" | |
| echo "" | |
| need_sudo() { | |
| [ "$(id -u)" -ne 0 ] | |
| } | |
| SUDO="" | |
| if need_sudo; then | |
| if command -v sudo &>/dev/null; then | |
| SUDO="sudo" | |
| fi | |
| fi | |
| print_binary_hints() { | |
| echo "" | |
| echo "============================================" | |
| echo " candle-vllm installed successfully!" | |
| echo "============================================" | |
| echo "" | |
| echo " HuggingFace model:" | |
| echo " candle-vllm --p 2000 --m <model-id>" | |
| echo "" | |
| echo " Local weights + Web UI:" | |
| echo " candle-vllm --p 2000 --w /path/to/model/ --ui-server" | |
| echo "============================================" | |
| } | |
| install_deb() { | |
| local variant="$1" | |
| local cpu_arch="x64" | |
| if [ "$variant" = "sm121" ]; then cpu_arch="arm64"; fi | |
| local deb="candle-vllm-linux-${cpu_arch}-${variant}.deb" | |
| local url="${RELEASE_BASE}/${deb}" | |
| local tmp | |
| tmp="$(mktemp -d)" | |
| echo "Downloading ${deb}..." | |
| if ! curl -fSL -o "${tmp}/${deb}" "$url"; then | |
| echo "ERROR: Failed to download ${url}" >&2 | |
| rm -rf "$tmp" | |
| return 1 | |
| fi | |
| echo "Installing ${deb}..." | |
| if ! ${SUDO} dpkg -i "${tmp}/${deb}"; then | |
| echo "" | |
| echo "ERROR: dpkg -i failed. You may need root privileges:" >&2 | |
| echo " sudo dpkg -i ${tmp}/${deb}" >&2 | |
| echo " Or re-run with: curl -sSL ${PAGES_BASE}/install.sh | sudo bash" >&2 | |
| return 1 | |
| fi | |
| rm -rf "$tmp" | |
| print_binary_hints | |
| } | |
| install_tarball() { | |
| local tarball="$1" | |
| local url="${RELEASE_BASE}/${tarball}" | |
| local install_dir="/usr/local/bin" | |
| local lib_dir="/usr/local/lib/candle-vllm" | |
| local tmp | |
| tmp="$(mktemp -d)" | |
| echo "Downloading ${tarball}..." | |
| if ! curl -fSL -o "${tmp}/${tarball}" "$url"; then | |
| echo "ERROR: Failed to download ${url}" >&2 | |
| rm -rf "$tmp" | |
| return 1 | |
| fi | |
| echo "Extracting..." | |
| tar -xzf "${tmp}/${tarball}" -C "${tmp}" | |
| echo "Installing candle-vllm to ${install_dir}..." | |
| if ! ${SUDO} install -m 755 "${tmp}/candle-vllm" "${install_dir}/candle-vllm"; then | |
| echo "" | |
| echo "ERROR: Could not install to ${install_dir}." >&2 | |
| echo " Try: sudo install -m 755 ${tmp}/candle-vllm ${install_dir}/candle-vllm" >&2 | |
| echo " Or re-run with: curl -sSL ${PAGES_BASE}/install.sh | sudo bash" >&2 | |
| return 1 | |
| fi | |
| if [ -d "${tmp}/lib" ]; then | |
| ${SUDO} mkdir -p "${lib_dir}" | |
| ${SUDO} cp -a "${tmp}/lib/"* "${lib_dir}/" | |
| fi | |
| rm -rf "$tmp" | |
| print_binary_hints | |
| } | |
| if [ "$PLATFORM" = "metal" ]; then | |
| TARBALL="candle-vllm-${VERSION}-darwin-arm64-metal.tar.gz" | |
| echo "Installing candle-vllm (Metal/macOS)..." | |
| install_tarball "$TARBALL" | |
| else | |
| if [ "$PLATFORM" = "sm121" ]; then | |
| TARBALL="candle-vllm-${VERSION}-linux-arm64-sm121.tar.gz" | |
| else | |
| TARBALL="candle-vllm-${VERSION}-linux-x64-${PLATFORM}.tar.gz" | |
| fi | |
| echo "Install options:" | |
| echo " 1) deb package — installs 'candle-vllm' system-wide (Linux)" | |
| echo " 2) binary — installs 'candle-vllm' to /usr/local/bin" | |
| echo "" | |
| MODE="${CANDLE_VLLM_MODE:-}" | |
| if [ -z "$MODE" ]; then | |
| if [ -e /dev/tty ]; then | |
| printf "Choose [1/2] (default: 1): " | |
| read -r MODE < /dev/tty | |
| MODE="${MODE:-1}" | |
| else | |
| MODE="1" | |
| fi | |
| fi | |
| case "$MODE" in | |
| 1|deb) install_deb "$PLATFORM" ;; | |
| 2|binary) install_tarball "$TARBALL" ;; | |
| *) echo "Invalid choice: $MODE"; exit 1 ;; | |
| esac | |
| fi | |
| SCRIPT | |
| sed -i "s|__VERSION__|${VERSION}|g" site/install.sh | |
| sed -i "s|__REPO__|${REPO}|g" site/install.sh | |
| sed -i "s|__OWNER__|${OWNER_LC}|g" site/install.sh | |
| sed -i 's/^ //' site/install.sh | |
| chmod +x site/install.sh | |
| - uses: actions/configure-pages@v5 | |
| - uses: actions/upload-pages-artifact@v3 | |
| with: | |
| path: site | |
| - uses: actions/deploy-pages@v4 | |
| id: deploy |