Improve tool calling across various models

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "candle-vllm"
-version = "0.7.1"
+version = "0.7.2"
 edition = "2021"
 default-run = "candle-vllm"
 
@@ -46,7 +46,7 @@ dirs = "5.0.1"
 minijinja = { version = "2.10.2", features = ["builtins", "json"] }
 minijinja-contrib = { version = "2.10.2", features = ["pycompat"] }
 thiserror = "1.0.58"
-attention-rs = { git = "https://github.com/guoqingbao/attention.rs.git", version="0.5.2", rev = "36ba0bb" }
+attention-rs = { git = "https://github.com/guoqingbao/attention.rs.git", version="0.5.2", rev = "2ea587f" }
 metal = { version = "0.27.0", features = ["mps"], optional = true }
 lazy_static = {version = "1.4.0"}
 interprocess = "2.2.2"
@@ -85,8 +85,7 @@ metal = ["candle-core/metal", "candle-nn/metal", "dep:metal", "attention-rs/meta
 cudnn = ["candle-core/cudnn"]
 flashattn = ["attention-rs/flashattn", "attention-rs/no-fp8-kvcache"]
 flashinfer = ["attention-rs/flashinfer"]
-trtllm = ["flashinfer", "attention-rs/trtllm"]
 mkl = ["dep:intel-mkl-src", "candle-core/mkl", "candle-nn/mkl"]
 nccl = ["candle-core/nccl"]
 mpi = ["candle-core/nccl", "dep:mpi"]
-graph = ["attention-rs/graph", "candle-core/graph"]
+graph = ["cuda", "attention-rs/graph", "candle-core/graph"]

README-CN.md

@@ -41,7 +41,7 @@
     | #4 | **QWen2/Qwen3 Dense** |96 tks/s (8B)|135 tks/s **(8B, Q4k)**|
     | #5 | **QWen3 MoE** |92 tks/s **(30B)**|114 tks/s **(30B, Q4K)** |
     | #6 | **QWen3-Next MoE** |71 tks/s **(80B, BF16, tp=2)**|TBD|
-    | #7 | **QWen3.5 Dense** |30 tks/s **(27B, BF16)**|~42 tks/s **(27B, Q4K / FP8)** |
+    | #7 | **QWen3.5/3.6 Dense** |30 tks/s **(27B, BF16)**|~42 tks/s **(27B, Q4K / FP8)** |
     | #8 | **QWen3.5/3.6 MoE** |82 tks/s **(35B)**|93 tks/s **(35B, Q4K)** |
     | #9 | **Yi** |148 tks/s (6B)| 180 tks/s (6B, Q4k)|
     | #10 | **StableLM** |223 tks/s (3B)|-|
@@ -138,16 +138,16 @@ cargo install --features metal --path .
     **示例:**
 
     ```shell
-    [RUST_LOG=warn] cargo run [--release --features cuda,nccl,flashinfer,cutlass,graph] -- [--log --dtype bf16 --p 2000 --d 0,1 --gpu-memory-fraction 0.7 --isq q4k --prefill-chunk-size 8192 --frequency-penalty 1.1 --presence-penalty 1.1 --enforce-parser qwen_coder --yarn-scaling-factor 4.0] [--m Qwen/Qwen3.5-27B-FP8] [--fp8-kvcache] [--ui-server]
+    [RUST_LOG=warn] cargo run [--release --features cuda,nccl,flashinfer,cutlass,graph] -- [--log --dtype bf16 --p 2000 --d 0,1 --gpu-memory-fraction 0.5 --isq q4k --prefill-chunk-size 8192 --frequency-penalty 1.1 --presence-penalty 1.1 --enforce-parser qwen_coder --yarn-scaling-factor 4.0] [--m Qwen/Qwen3.6-27B-FP8] [--fp8-kvcache] [--ui-server]
     ```
 
     `ENV_PARAM`: RUST_LOG=warn
 
     `BUILD_PARAM`: --release --features cuda,nccl,flashinfer,cutlass,graph
 
-    `PROGRAM_PARAM`：--log --dtype bf16 --p 2000 --d 0,1 --gpu-memory-fraction 0.7 --isq q4k --prefill-chunk-size 8192 --frequency-penalty 1.1 --presence-penalty 1.1 --enforce-parser qwen_coder --yarn-scaling-factor 4.0
+    `PROGRAM_PARAM`：--log --dtype bf16 --p 2000 --d 0,1 --gpu-memory-fraction 0.5 --isq q4k --prefill-chunk-size 8192 --frequency-penalty 1.1 --presence-penalty 1.1 --enforce-parser qwen_coder --yarn-scaling-factor 4.0
 
-    `MODEL_ID/MODEL_WEIGHT_PATH`: --m Qwen/Qwen3.5-27B-FP8（或使用 `--w` 指定本地模型路径）
+    `MODEL_ID/MODEL_WEIGHT_PATH`: --m Qwen/Qwen3.6-27B-FP8（或使用 `--w` 指定本地模型路径）
 
     `CACHE CONFIG`: --fp8-kvcache
 
@@ -184,23 +184,23 @@ docker run --rm -it --gpus all --network host -v /home:/home -v /data:/data cand
 
     **本地路径 (ISQ量化, +UI Server)**
     ```shell
-    candle-vllm --p 8000 --d 0,1 --w /home/Qwen3.5-27B/ --isq q4k --ui-server --prefix-cache
+    candle-vllm --p 8000 --d 0,1 --w /home/Qwen3.6-27B/ --isq q4k --ui-server --prefix-cache
     ```
 
     **模型ID（从Huggingface下载）**
 
     ```shell
-    candle-vllm --m Qwen/Qwen3.5-35B-A3B --ui-server --prefix-cache
+    candle-vllm --m Qwen/Qwen3.6-35B-A3B --ui-server --prefix-cache
     ```
 
     **手动设置 YaRN 缩放**
     ```shell
-    candle-vllm --m Qwen/Qwen3.5-35B-A3B --yarn-scaling-factor 4.0 --ui-server --prefix-cache
+    candle-vllm --m Qwen/Qwen3.6-35B-A3B --yarn-scaling-factor 4.0 --ui-server --prefix-cache
     ```
 
     **FP8 模型** (block-wise量化, 通过增加`cutlass`特性构建)
     ```shell
-    candle-vllm --m Qwen/Qwen3.5-27B-FP8 --ui-server --prefix-cache
+    candle-vllm --m Qwen/Qwen3.6-27B-FP8 --ui-server --prefix-cache
     ```
     **FP4 模型** (MXFP4/NVFP4, 暂不支持MLX量化格式)
     ```shell
@@ -255,7 +255,7 @@ docker run --rm -it --gpus all --network host -v /home:/home -v /data:/data cand
     **只需在运行未量化模型时添加`isq`参数**
 
     ```shell
-    candle-vllm --m Qwen/Qwen3.5-27B --isq q4k
+    candle-vllm --m Qwen/Qwen3.6-27B --isq q4k
     ```
 
     注：原位量化加载可能需要更长的加载时间，原位`isq`参数选项：["q4_0", "q4_1", "q5_0", "q5_1", "q8_0", "q2k", "q3k","q4k","q5k","q6k"]
@@ -653,7 +653,7 @@ docker run --rm -it --gpus all --network host -v /home:/home -v /data:/data cand
     <summary>显示详情</summary>
     `--mem` (`kvcache-mem-gpu`) 用于以 MB 为单位设置固定 KV Cache 预算，默认值为 `4096` MB。
 
-    `--gpu-memory-fraction` 提供一个更轻量的自动模式。不显式指定时，默认值为 `0.7`。模型加载完成后，candle-vllm 会探测每张已加载的 CUDA 或 Metal 设备，并按以下公式计算 KV Cache 预算：
+    `--gpu-memory-fraction` 提供一个更轻量的自动模式。不显式指定时，默认值为 `0.5`。模型加载完成后，candle-vllm 会探测每张已加载的 CUDA 或 Metal 设备，并按以下公式计算 KV Cache 预算：
 
     ```
     gpu_memory_fraction * total_gpu_memory - current_memory_usage
@@ -662,7 +662,7 @@ docker run --rm -it --gpus all --network host -v /home:/home -v /data:/data cand
     多卡场景下，会取所有 rank 中最小的结果作为每个 rank 的 KV Cache 预算。例如：
 
     ```
-    candle-vllm --w /home/Qwen3-Coder-30B-A3B-Instruct-FP8 --d 0,1 --gpu-memory-fraction 0.7
+    candle-vllm --w /home/Qwen3-Coder-30B-A3B-Instruct-FP8 --d 0,1 --gpu-memory-fraction 0.5
     ```
 
     当你需要显式固定缓存预算时，用 `--mem`。当你希望服务根据模型加载后的可用显存自动调整时，用 `--gpu-memory-fraction`。

README.md

@@ -42,7 +42,7 @@ Efficient, easy-to-use platform for inference and serving local LLMs including a
     | #4 | **QWen2/Qwen3 Dense** |96 tks/s (8B)|135 tks/s **(8B, Q4k)**|
     | #5 | **QWen3 MoE** |92 tks/s **(30B)**|114 tks/s **(30B, Q4K)** |
     | #6 | **QWen3-Next MoE** |71 tks/s **(80B, BF16, tp=2)**|TBD|
-    | #7 | **QWen3.5 Dense** |30 tks/s **(27B, BF16)**|~42 tks/s **(27B, Q4K / FP8)** |
+    | #7 | **QWen3.5/3.6 Dense** |30 tks/s **(27B, BF16)**|~42 tks/s **(27B, Q4K / FP8)** |
     | #8 | **QWen3.5/3.6 MoE** |82 tks/s **(35B)**|93 tks/s **(35B, Q4K)** |
     | #9 | **Yi** |148 tks/s (6B)| 180 tks/s (6B, Q4k)|
     | #10 | **StableLM** |223 tks/s (3B)|-|
@@ -141,16 +141,16 @@ cargo install --features metal --path .
     **Example:**
 
     ```shell
-    [RUST_LOG=warn] cargo run [--release --features cuda,nccl,flashinfer,cutlass,graph] -- [--log --dtype bf16 --p 2000 --d 0,1 --gpu-memory-fraction 0.7 --isq q4k --prefill-chunk-size 8192 --frequency-penalty 1.1 --presence-penalty 1.1 --enforce-parser qwen_coder --yarn-scaling-factor 4.0] [--m Qwen/Qwen3.5-27B-FP8] [--fp8-kvcache] [--ui-server]
+    [RUST_LOG=warn] cargo run [--release --features cuda,nccl,flashinfer,cutlass,graph] -- [--log --dtype bf16 --p 2000 --d 0,1 --gpu-memory-fraction 0.5 --isq q4k --prefill-chunk-size 8192 --frequency-penalty 1.1 --presence-penalty 1.1 --enforce-parser qwen_coder --yarn-scaling-factor 4.0] [--m Qwen/Qwen3.6-27B-FP8] [--fp8-kvcache] [--ui-server]
     ```
 
     `ENV_PARAM`: RUST_LOG=warn
 
     `BUILD_PARAM`: --release --features cuda,nccl,flashinfer,cutlass,graph
 
-    `PROGRAM_PARAM`：--log --dtype bf16 --p 2000 --d 0,1 --gpu-memory-fraction 0.7 --isq q4k --prefill-chunk-size 8192 --frequency-penalty 1.1 --presence-penalty 1.1 --enforce-parser qwen_coder --yarn-scaling-factor 4.0
+    `PROGRAM_PARAM`：--log --dtype bf16 --p 2000 --d 0,1 --gpu-memory-fraction 0.5 --isq q4k --prefill-chunk-size 8192 --frequency-penalty 1.1 --presence-penalty 1.1 --enforce-parser qwen_coder --yarn-scaling-factor 4.0
 
-    `MODEL_ID/MODEL_WEIGHT_PATH`: --m Qwen/Qwen3.5-27B-FP8 (or `--w` specify local model path)
+    `MODEL_ID/MODEL_WEIGHT_PATH`: --m Qwen/Qwen3.6-27B-FP8 (or `--w` specify local model path)
 
     `CACHE CONFIG`: --fp8-kvcache
 
@@ -188,23 +188,23 @@ docker run --rm -it --gpus all --network host -v /home:/home -v /data:/data cand
 
     **Local Path (ISQ, +UI Server)**
     ```shell
-    candle-vllm --p 8000 --d 0,1 --w /home/Qwen3.5-27B/ --isq q4k --ui-server --prefix-cache
+    candle-vllm --p 8000 --d 0,1 --w /home/Qwen3.6-27B/ --isq q4k --ui-server --prefix-cache
     ```
 
     **Model-ID (download from Huggingface)**
 
     ```shell
-    candle-vllm --m Qwen/Qwen3.5-35B-A3B --ui-server --prefix-cache
+    candle-vllm --m Qwen/Qwen3.6-35B-A3B --ui-server --prefix-cache
     ```
 
     **Manual YaRN scaling**
     ```shell
-    candle-vllm --m Qwen/Qwen3.5-35B-A3B --yarn-scaling-factor 4.0 --ui-server --prefix-cache
+    candle-vllm --m Qwen/Qwen3.6-35B-A3B --yarn-scaling-factor 4.0 --ui-server --prefix-cache
     ```
 
     **FP8 Model** (block-wise quant, build with `cutlass` feature)
     ```shell
-    candle-vllm --m Qwen/Qwen3.5-27B-FP8 --ui-server --prefix-cache
+    candle-vllm --m Qwen/Qwen3.6-27B-FP8 --ui-server --prefix-cache
     ```
 
     ```shell
@@ -265,7 +265,7 @@ docker run --rm -it --gpus all --network host -v /home:/home -v /data:/data cand
     **Simply add `isq` parameter when running unquantized models**
 
     ```shell
-    candle-vllm --p 2000 --m Qwen/Qwen3.5-27B --isq q4k
+    candle-vllm --p 2000 --m Qwen/Qwen3.6-27B --isq q4k
     ```
 
     Options for in-site `isq` parameters: ["q4_0", "q4_1", "q5_0", "q5_1", "q8_0", "q2k", "q3k","q4k","q5k","q6k"]
@@ -659,7 +659,7 @@ Chat frontend (any frontend compatible with openai API, simple options available
     <summary>Show details</summary>
     The `--mem` (`kvcache-mem-gpu`) parameter sets a fixed KV cache budget in MB. By default this is `4096` MB.
 
-    The `--gpu-memory-fraction` parameter is a lighter-weight auto mode. When omitted, it defaults to `0.7`. After the model finishes loading, candle-vllm probes each loaded CUDA or Metal device and computes the KV cache budget as:
+    The `--gpu-memory-fraction` parameter is a lighter-weight auto mode. When omitted, it defaults to `0.5`. After the model finishes loading, candle-vllm probes each loaded CUDA or Metal device and computes the KV cache budget as:
 
     ```
     gpu_memory_fraction * remaining_gpu_memory_after_model_load
@@ -668,7 +668,7 @@ Chat frontend (any frontend compatible with openai API, simple options available
     This means the fraction directly controls how much of the free GPU memory left after model load can be used for the combined GPU cache budget. The minimum detected budget across ranks is used as the KV cache budget per rank. For example:
 
     ```
-    candle-vllm --w /home/Qwen3-Coder-30B-A3B-Instruct-FP8 --d 0,1 --gpu-memory-fraction 0.7
+    candle-vllm --w /home/Qwen3-Coder-30B-A3B-Instruct-FP8 --d 0,1 --gpu-memory-fraction 0.5
     ```
 
     Use `--mem` when you want an explicit fixed budget. Use `--gpu-memory-fraction` when you want the server to adapt to the currently available GPU memory after model load.

docs/kilocode.md

@@ -10,11 +10,11 @@ Kilo Code -> Candle-vLLM (OpenAI-compatible)
 
 ```bash
 cargo run --release --features cuda,nccl,graph,flashinfer,cutlass -- \
-  --m Qwen/Qwen3.5-27B-FP8 \
+  --m Qwen/Qwen3.6-27B-FP8 \
   --d 0 \
   --prefix-cache \
   --p 8000 \
-  --gpu-memory-fraction 0.7 \
+  --gpu-memory-fraction 0.5 \
   --enforce-parser qwen_coder
 ```
 
@@ -42,7 +42,7 @@ Create `~/.config/kilo/config.json`:
       },
       "models": {
         "qwen3-coder": {
-          "name": "Qwen/Qwen3.5-27B-FP8"
+          "name": "Qwen/Qwen3.6-27B-FP8"
         }
       }
     }

docs/opencode.md

@@ -10,11 +10,11 @@ OpenCode -> Candle-vLLM (OpenAI-compatible)
 
 ```bash
 cargo run --release --features cuda,nccl,graph,flashinfer,cutlass -- \
-  --m Qwen/Qwen3.5-27B-FP8 \
+  --m Qwen/Qwen3.6-27B-FP8 \
   --d 0 \
   --prefix-cache \
   --p 8000 \
-  --gpu-memory-fraction 0.7 \
+  --gpu-memory-fraction 0.5 \
   --enforce-parser qwen_coder
 ```
 
@@ -56,7 +56,7 @@ Create `~/.config/opencode/config.json`:
       },
       "models": {
         "qwen3-coder": {
-          "name": "Qwen/Qwen3.5-27B-FP8"
+          "name": "Qwen/Qwen3.6-27B-FP8"
         }
       }
     }

src/api.rs

@@ -9,7 +9,9 @@ use crate::openai::PipelineConfig;
 use crate::scheduler::cache_engine::{CacheConfig, CacheEngine};
 use crate::scheduler::prefix_cache::PrefixCacheConfig;
 use crate::scheduler::SchedulerConfig;
-use crate::tools::ToolFormat;
+use crate::tools::helpers::{
+    build_invalid_tool_call_feedback, build_tool_schema_map, filter_tool_calls,
+};
 use candle_core::{DType, Result};
 use parking_lot::RwLock;
 use std::collections::HashMap;
@@ -66,7 +68,7 @@ impl EngineBuilder {
             max_num_seqs: 16,
             block_size: if cfg!(feature = "cuda") { 64 } else { 32 },
             kvcache_mem_gpu: 4096,
-            gpu_memory_fraction: Some(0.7),
+            gpu_memory_fraction: Some(0.5),
             kvcache_mem_cpu: 128,
             temperature: None,
             top_p: None,
@@ -446,12 +448,13 @@ impl Engine {
                 .map_err(candle_core::Error::wrap)?;
         }
 
-        let (prompt, tokenizer, image_data) = {
+        let (prompt, tokenizer, image_data, resolved_tools) = {
             let e = self.engine.read();
             let (pipeline, _) = e.get_pipeline(0).unwrap();
 
             let tool_config = resolve_tools_for_request(&request.tools, &request.tool_choice, None)
                 .map_err(candle_core::Error::wrap)?;
+            let resolved_tools = tool_config.tools.clone();
 
             // tokenizer is inside DefaultPipeline
             let mut conversation = pipeline.conversation.clone();
@@ -499,35 +502,15 @@ impl Engine {
                 }
             };
 
-            if !tool_config.tools.is_empty() {
-                let mut tools_prompt = ToolFormat::get_tool_prompt(
-                    &pipeline.tool_config,
-                    &pipeline.tool_model_type,
-                    &pipeline.tool_parser_model_id,
-                    pipeline.enforce_parser.as_deref(),
-                );
-
-                // Enforce tool_choice=function
-                if let crate::openai::ToolChoiceKind::Function(name) = &tool_config.choice {
-                    tools_prompt = format!(
-                        "IMPORTANT: You MUST call the tool \"{}\". Do not respond with plain text.\n\n{}",
-                        name, tools_prompt
-                    );
-                }
-
-                let current_system = conversation.get_system_message().unwrap_or_default();
-                let new_system = if current_system.is_empty() {
-                    tools_prompt
-                } else {
-                    format!("{}\n\n{}", current_system, tools_prompt)
-                };
-                conversation.set_system_message(Some(new_system));
-            }
-
             let enable_thinking = request.thinking.unwrap_or(true);
             let prompt = conversation.get_prompt(enable_thinking, &tool_config.tools);
 
-            (prompt, pipeline.tokenizer.clone(), image_data)
+            (
+                prompt,
+                pipeline.tokenizer.clone(),
+                image_data,
+                resolved_tools,
+            )
         };
 
         let request_id = format!("cmpl-{}", uuid::Uuid::new_v4());
@@ -553,7 +536,7 @@ impl Engine {
         let prefilled_reasoning_end =
             crate::tools::stream_parser::detect_prefilled_reasoning_end_marker(&prompt);
 
-        let has_tools = request.tools.as_ref().is_some_and(|t| !t.is_empty());
+        let has_tools = !resolved_tools.is_empty();
         {
             let mut e = self.engine.write();
             let mut sampling_params = SamplingParams::new(
@@ -589,7 +572,7 @@ impl Engine {
                 false, // is_embedding
                 crate::openai::requests::EncodingFormat::default(),
                 crate::openai::requests::EmbeddingType::default(),
-                request.tools.clone().unwrap_or_default(),
+                resolved_tools.clone(),
                 image_data,
                 None, // streamer
                 Some(req_notify.clone()),
@@ -635,6 +618,45 @@ impl Engine {
                     }
                 }
             }
+            if has_tools {
+                let parser = crate::tools::parser::ToolParser::new();
+                let tool_schemas = build_tool_schema_map(&resolved_tools);
+                for choice in &mut choices {
+                    let parsed_calls = if let Some(calls) = choice.message.tool_calls.take() {
+                        calls
+                    } else if let Some(content) = &choice.message.content {
+                        parser.parse(content)
+                    } else {
+                        Vec::new()
+                    };
+
+                    if parsed_calls.is_empty() {
+                        continue;
+                    }
+
+                    let (valid_calls, invalid_calls) =
+                        filter_tool_calls(&parsed_calls, &tool_schemas);
+                    if !invalid_calls.is_empty() {
+                        tracing::warn!(
+                            "Dropped {} invalid tool call(s) before response",
+                            invalid_calls.len()
+                        );
+                    }
+                    if valid_calls.is_empty() {
+                        if let Some(feedback) =
+                            build_invalid_tool_call_feedback(&invalid_calls, &tool_schemas, None)
+                        {
+                            choice.message.content = Some(feedback);
+                        }
+                        choice.finish_reason = Some("stop".to_string());
+                        continue;
+                    }
+
+                    choice.message.tool_calls = Some(valid_calls);
+                    choice.message.content = None;
+                    choice.finish_reason = Some("tool_calls".to_string());
+                }
+            }
             Ok(ChatCompletionResponse {
                 id: request_id,
                 choices,