Skip to content

Commit ec6de29

Browse files
guoqingbaoguoqingbao
authored
Fix gguf exccessive memory usage & nvfp4 model regression (#431)
* Fix gguf exccessive memory usage & nvfp4 model regression * Revise CI trigger from master branch & mention installation scirpt in ReadMe
1 parent 8c34057 commit ec6de29

13 files changed

Lines changed: 506 additions & 105 deletions

File tree

‎.github/workflows/release-sm121.yml‎

Lines changed: 168 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,168 @@
1+
name: release-sm121
2+
3+
on:
4+
push:
5+
branches: [master]
6+
paths:
7+
- "Cargo.toml"
8+
9+
permissions:
10+
contents: write
11+
12+
env:
13+
CARGO_TERM_COLOR: always
14+
15+
jobs:
16+
# ---------------------------------------------------------------------------
17+
# CUDA ARM build: SM121 (DGX Spark / GB10) — native aarch64 GitHub runner
18+
# Runs in parallel with the main release workflow on version change.
19+
# Uses GitHub's free ARM64 runner with CUDA 13 container.
20+
# Limited parallelism (2 threads) to fit free-tier runner resources.
21+
# Pages deployment is handled by the main release.yml workflow.
22+
# ---------------------------------------------------------------------------
23+
validate:
24+
name: validate-version
25+
runs-on: ubuntu-latest
26+
outputs:
27+
version: ${{ steps.version.outputs.version }}
28+
changed: ${{ steps.version.outputs.changed }}
29+
steps:
30+
- uses: actions/checkout@v4
31+
with:
32+
fetch-depth: 2
33+
- name: Detect version change in Cargo.toml
34+
id: version
35+
run: |
36+
cargo_version="$(sed -n 's/^version = "\(.*\)"/\1/p' Cargo.toml | head -n 1)"
37+
prev_version="$(git show HEAD~1:Cargo.toml 2>/dev/null | sed -n 's/^version = "\(.*\)"/\1/p' | head -n 1)" || prev_version=""
38+
echo "version=${cargo_version}" >> "$GITHUB_OUTPUT"
39+
if [ "$cargo_version" != "$prev_version" ] && [ -n "$prev_version" ]; then
40+
echo "Version changed: ${prev_version} -> ${cargo_version}"
41+
echo "changed=true" >> "$GITHUB_OUTPUT"
42+
else
43+
echo "Version unchanged (${cargo_version}), skipping release."
44+
echo "changed=false" >> "$GITHUB_OUTPUT"
45+
fi
46+
47+
build-sm121:
48+
name: cuda-sm121
49+
runs-on: ubuntu-24.04-arm
50+
needs: validate
51+
if: needs.validate.outputs.changed == 'true'
52+
container:
53+
image: docker.io/nvidia/cuda:13.0.0-cudnn-devel-ubuntu24.04
54+
env:
55+
CUDA_COMPUTE_CAP: "121"
56+
DEBIAN_FRONTEND: noninteractive
57+
CUDAFORGE_THREADS: "2"
58+
RAYON_NUM_THREADS: "2"
59+
steps:
60+
- uses: actions/checkout@v4
61+
62+
- name: Install system dependencies
63+
run: |
64+
apt-get update
65+
apt-get install -y --no-install-recommends --allow-change-held-packages \
66+
libnccl-dev libnccl2 \
67+
curl git ca-certificates \
68+
libssl-dev pkg-config \
69+
clang libclang-dev \
70+
patchelf dpkg-dev
71+
rm -rf /var/lib/apt/lists/*
72+
73+
- name: Install Rust
74+
run: |
75+
curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
76+
echo "$HOME/.cargo/bin" >> "$GITHUB_PATH"
77+
78+
- name: Install cargo-deb
79+
run: |
80+
. "$HOME/.cargo/env"
81+
cargo install cargo-deb
82+
83+
- name: Build candle-vllm binary
84+
run: |
85+
. "$HOME/.cargo/env"
86+
cargo build --release --features "cuda,nccl,flashinfer,cutlass"
87+
88+
- name: Bundle CUDA runtime with binary
89+
run: |
90+
CUDA_LIB="$(find /usr/local/cuda -follow -name 'libcudart.so.*.*' -not -name '*.a' 2>/dev/null | head -1)"
91+
if [ -z "$CUDA_LIB" ]; then CUDA_LIB="$(ldconfig -p | grep 'libcudart.so\.' | awk '{print $NF}' | head -1)"; fi
92+
mkdir -p target/release/lib
93+
if [ -n "$CUDA_LIB" ]; then
94+
cp "$CUDA_LIB" target/release/lib/
95+
CUDA_SONAME="$(objdump -p "$CUDA_LIB" 2>/dev/null | awk '/SONAME/{print $2}')"
96+
if [ -n "$CUDA_SONAME" ]; then
97+
ln -sf "$(basename "$CUDA_LIB")" "target/release/lib/$CUDA_SONAME"
98+
ln -sf "$CUDA_SONAME" target/release/lib/libcudart.so
99+
fi
100+
fi
101+
patchelf --set-rpath '$ORIGIN/lib:/usr/local/lib/candle-vllm' target/release/candle-vllm
102+
ls -la target/release/lib/
103+
104+
- name: Package deb
105+
run: |
106+
. "$HOME/.cargo/env"
107+
cargo deb --no-build --output "dist/candle-vllm-linux-arm64-sm121.deb"
108+
109+
- name: Package tar.gz
110+
run: |
111+
mkdir -p dist/candle-vllm
112+
cp target/release/candle-vllm dist/candle-vllm/candle-vllm
113+
cp -a target/release/lib dist/candle-vllm/lib
114+
tar -C dist/candle-vllm -czf "dist/candle-vllm-${{ needs.validate.outputs.version }}-linux-arm64-sm121.tar.gz" candle-vllm lib
115+
rm -rf dist/candle-vllm
116+
117+
- uses: actions/upload-artifact@v4
118+
with:
119+
name: candle-vllm-cuda-sm121
120+
path: |
121+
dist/*.tar.gz
122+
dist/*.deb
123+
124+
# ---------------------------------------------------------------------------
125+
# Append SM121 artifacts to the existing GitHub Release created by release.yml
126+
# ---------------------------------------------------------------------------
127+
append-release:
128+
name: append-to-release
129+
runs-on: ubuntu-latest
130+
needs: [validate, build-sm121]
131+
if: needs.validate.outputs.changed == 'true'
132+
steps:
133+
- uses: actions/download-artifact@v4
134+
with:
135+
name: candle-vllm-cuda-sm121
136+
path: dist
137+
138+
- name: Generate SHA256SUMS for SM121
139+
run: |
140+
cd dist
141+
find . -type f \( -name "*.tar.gz" -o -name "*.deb" \) \
142+
-exec sha256sum {} + > SHA256SUMS-sm121
143+
cat SHA256SUMS-sm121
144+
145+
- name: Wait for main release to be created
146+
env:
147+
GH_TOKEN: ${{ github.token }}
148+
run: |
149+
TAG="v${{ needs.validate.outputs.version }}"
150+
for i in $(seq 1 60); do
151+
if gh release view "$TAG" --repo "${{ github.repository }}" >/dev/null 2>&1; then
152+
echo "Release $TAG found."
153+
exit 0
154+
fi
155+
echo "Waiting for release $TAG... (attempt $i/60)"
156+
sleep 30
157+
done
158+
echo "Release $TAG not found after 30 minutes, uploading anyway."
159+
160+
- name: Upload SM121 artifacts to release
161+
uses: softprops/action-gh-release@v2
162+
with:
163+
tag_name: v${{ needs.validate.outputs.version }}
164+
generate_release_notes: false
165+
files: |
166+
dist/*.tar.gz
167+
dist/*.deb
168+
dist/SHA256SUMS-sm121

‎.github/workflows/release.yml‎

Lines changed: 43 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,9 @@ name: release
22

33
on:
44
push:
5-
tags:
6-
- "v*.*.*"
5+
branches: [master]
6+
paths:
7+
- "Cargo.toml"
78

89
permissions:
910
contents: write
@@ -20,18 +21,24 @@ jobs:
2021
if: github.repository == 'EricLBuehler/candle-vllm'
2122
outputs:
2223
version: ${{ steps.version.outputs.version }}
24+
changed: ${{ steps.version.outputs.changed }}
2325
steps:
2426
- uses: actions/checkout@v4
25-
- name: Validate tag matches Cargo.toml version
27+
with:
28+
fetch-depth: 2
29+
- name: Detect version change in Cargo.toml
2630
id: version
2731
run: |
2832
cargo_version="$(sed -n 's/^version = "\(.*\)"/\1/p' Cargo.toml | head -n 1)"
29-
expected_tag="v${cargo_version}"
30-
if [ "${{ github.ref_name }}" != "$expected_tag" ]; then
31-
echo "Tag ${{ github.ref_name }} must match Cargo.toml version $expected_tag" >&2
32-
exit 1
33-
fi
33+
prev_version="$(git show HEAD~1:Cargo.toml 2>/dev/null | sed -n 's/^version = "\(.*\)"/\1/p' | head -n 1)" || prev_version=""
3434
echo "version=${cargo_version}" >> "$GITHUB_OUTPUT"
35+
if [ "$cargo_version" != "$prev_version" ] && [ -n "$prev_version" ]; then
36+
echo "Version changed: ${prev_version} -> ${cargo_version}"
37+
echo "changed=true" >> "$GITHUB_OUTPUT"
38+
else
39+
echo "Version unchanged (${cargo_version}), skipping release."
40+
echo "changed=false" >> "$GITHUB_OUTPUT"
41+
fi
3542
3643
# ---------------------------------------------------------------------------
3744
# CUDA legacy builds: SM70/SM75 on GitHub free runners (no flashinfer/cutlass)
@@ -41,6 +48,7 @@ jobs:
4148
name: cuda-${{ matrix.sm_name }}
4249
runs-on: ubuntu-latest
4350
needs: validate
51+
if: needs.validate.outputs.changed == 'true'
4452
strategy:
4553
fail-fast: false
4654
matrix:
@@ -133,6 +141,7 @@ jobs:
133141
name: cuda-${{ matrix.sm_name }}
134142
runs-on: ubuntu-latest
135143
needs: validate
144+
if: needs.validate.outputs.changed == 'true'
136145
strategy:
137146
fail-fast: false
138147
max-parallel: 1
@@ -150,6 +159,10 @@ jobs:
150159
sm_name: sm90
151160
cuda_version: "13.0.0"
152161
features: "cuda,nccl,flashinfer,cutlass"
162+
- sm: "100"
163+
sm_name: sm100
164+
cuda_version: "13.0.0"
165+
features: "cuda,nccl,flashinfer,cutlass"
153166
- sm: "120"
154167
sm_name: sm120
155168
cuda_version: "13.0.0"
@@ -233,6 +246,7 @@ jobs:
233246
name: metal-darwin-arm64
234247
runs-on: macos-latest
235248
needs: validate
249+
if: needs.validate.outputs.changed == 'true'
236250
steps:
237251
- uses: actions/checkout@v4
238252
- uses: dtolnay/rust-toolchain@stable
@@ -262,6 +276,7 @@ jobs:
262276
name: checksums
263277
runs-on: ubuntu-latest
264278
needs: [validate, cuda-legacy, cuda, metal]
279+
if: needs.validate.outputs.changed == 'true'
265280
steps:
266281
- uses: actions/download-artifact@v4
267282
with:
@@ -280,10 +295,10 @@ jobs:
280295
name: candle-vllm-release-artifacts
281296
path: dist/*
282297

283-
- name: Upload artifacts to release
298+
- name: Create tag and upload artifacts to release
284299
uses: softprops/action-gh-release@v2
285300
with:
286-
tag_name: ${{ github.ref_name }}
301+
tag_name: v${{ needs.validate.outputs.version }}
287302
name: candle-vllm v${{ needs.validate.outputs.version }}
288303
generate_release_notes: true
289304
files: |
@@ -298,13 +313,17 @@ jobs:
298313
name: publish-pages
299314
runs-on: ubuntu-latest
300315
needs: [validate, checksums]
316+
if: needs.validate.outputs.changed == 'true'
301317
permissions:
302318
contents: read
303319
pages: write
304320
id-token: write
305321
environment:
306322
name: github-pages
307323
url: ${{ steps.deploy.outputs.page_url }}
324+
concurrency:
325+
group: pages
326+
cancel-in-progress: false
308327
steps:
309328
- uses: actions/checkout@v4
310329
with:
@@ -314,7 +333,6 @@ jobs:
314333
env:
315334
VERSION: ${{ needs.validate.outputs.version }}
316335
REPO: ${{ github.repository }}
317-
TAG: ${{ github.ref_name }}
318336
run: |
319337
mkdir -p site
320338
OWNER="$(echo "${REPO}" | cut -d/ -f1)"
@@ -340,7 +358,7 @@ jobs:
340358
fi
341359
if ! command -v nvidia-smi &>/dev/null; then
342360
echo "ERROR: No NVIDIA GPU detected and no platform specified." >&2
343-
echo " Set CANDLE_VLLM_PLATFORM=sm80 (or sm70/sm75/sm89/sm90/sm120) and re-run." >&2
361+
echo " Set CANDLE_VLLM_PLATFORM=sm80 (or sm70/sm75/sm89/sm90/sm100/sm120/sm121) and re-run." >&2
344362
exit 1
345363
fi
346364
local cc
@@ -351,8 +369,9 @@ jobs:
351369
80|86|87) echo "sm80" ;;
352370
89) echo "sm89" ;;
353371
90|90a) echo "sm90" ;;
354-
100|110) echo "sm90" ;;
355-
120) echo "sm120" ;;
372+
100|100a) echo "sm100" ;;
373+
120|120a) echo "sm120" ;;
374+
121|121a) echo "sm121" ;;
356375
*) echo "sm80" ;;
357376
esac
358377
}
@@ -388,7 +407,9 @@ jobs:
388407
389408
install_deb() {
390409
local variant="$1"
391-
local deb="candle-vllm-linux-x64-${variant}.deb"
410+
local cpu_arch="x64"
411+
if [ "$variant" = "sm121" ]; then cpu_arch="arm64"; fi
412+
local deb="candle-vllm-linux-${cpu_arch}-${variant}.deb"
392413
local url="${RELEASE_BASE}/${deb}"
393414
local tmp
394415
tmp="$(mktemp -d)"
@@ -446,8 +467,11 @@ jobs:
446467
echo "Installing candle-vllm (Metal/macOS)..."
447468
install_tarball "$TARBALL"
448469
else
449-
DEB="candle-vllm-linux-x64-${PLATFORM}.deb"
450-
TARBALL="candle-vllm-${VERSION}-linux-x64-${PLATFORM}.tar.gz"
470+
if [ "$PLATFORM" = "sm121" ]; then
471+
TARBALL="candle-vllm-${VERSION}-linux-arm64-sm121.tar.gz"
472+
else
473+
TARBALL="candle-vllm-${VERSION}-linux-x64-${PLATFORM}.tar.gz"
474+
fi
451475
echo "Install options:"
452476
echo " 1) deb package — installs 'candle-vllm' system-wide (Linux)"
453477
echo " 2) binary — installs 'candle-vllm' to /usr/local/bin"
@@ -472,9 +496,10 @@ jobs:
472496
fi
473497
SCRIPT
474498
499+
OWNER_LC="$(echo "${OWNER}" | tr '[:upper:]' '[:lower:]')"
475500
sed -i "s|__VERSION__|${VERSION}|g" site/install.sh
476501
sed -i "s|__REPO__|${REPO}|g" site/install.sh
477-
sed -i "s|__OWNER__|${OWNER}|g" site/install.sh
502+
sed -i "s|__OWNER__|${OWNER_LC}|g" site/install.sh
478503
sed -i 's/^ //' site/install.sh
479504
chmod +x site/install.sh
480505

‎Cargo.toml‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
[package]
22
name = "candle-vllm"
3-
version = "0.8.3"
3+
version = "0.8.4"
44
edition = "2021"
55
default-run = "candle-vllm"
66

@@ -22,10 +22,10 @@ anyhow = "1.0.75"
2222
rand = "0.9.0"
2323
rayon="1.10.0"
2424
hyper = { version = "0.14", features = ["full"] }
25-
candle-core = { git = "https://github.com/guoqingbao/candle.git", version = "0.8.3", rev = "3bcb218" }
26-
candle-nn = { git = "https://github.com/guoqingbao/candle.git", version = "0.8.3", rev = "3bcb218" }
25+
candle-core = { git = "https://github.com/guoqingbao/candle.git", version = "0.8.3", rev = "9202150" }
26+
candle-nn = { git = "https://github.com/guoqingbao/candle.git", version = "0.8.3", rev = "9202150" }
2727
dyn-fmt = "0.4.0"
28-
safetensors = "0.4"
28+
safetensors = "0.8"
2929
serde = { version = "1.0.190", features = ["serde_derive"] }
3030
tokenizers = "0.21.2"
3131
uuid = { version = "1.5.0", features = ["v4"] }
@@ -46,7 +46,7 @@ dirs = "5.0.1"
4646
minijinja = { version = "2.10.2", features = ["builtins", "json"] }
4747
minijinja-contrib = { version = "2.10.2", features = ["pycompat"] }
4848
thiserror = "1.0.58"
49-
attention-rs = { git = "https://github.com/guoqingbao/attention.rs.git", version="0.6.1", rev = "2cba5b9" }
49+
attention-rs = { git = "https://github.com/guoqingbao/attention.rs.git", version="0.6.2", rev = "8eb0f9e" }
5050
metal = { version = "0.27.0", features = ["mps"], optional = true }
5151
lazy_static = {version = "1.4.0"}
5252
interprocess = "2.2.2"
@@ -63,7 +63,7 @@ itertools = "0.13.0"
6363
chrono = "0.4.41"
6464
which = "8.0.0"
6565
ahash = "0.8.11"
66-
rustchatui = "0.3.1"
66+
rustchatui = "0.3.2"
6767
base64 = "0.22.1"
6868
bytes = "1.5"
6969
reqwest = { version = "0.12.24", features = ["blocking", "json", "rustls-tls"]}

0 commit comments

Comments
 (0)