Merge branch 'main' into sufeng-buaa/sglang-tracing-part2

Author: ShangmingCai

.github/workflows/pr-test-rust.yml

@@ -86,7 +86,7 @@ jobs:
   pytest-rust:
     if: github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'run-ci')
     runs-on: 4-gpu-a10
-    timeout-minutes: 25
+    timeout-minutes: 32
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
@@ -144,6 +144,12 @@ jobs:
           python3 -m pip --no-cache-dir install --upgrade --break-system-packages genai-bench==0.0.2
           pytest -m e2e -s  -vv -o log_cli=true --log-cli-level=INFO
 
+      - name: Run Python E2E gRPC tests
+        run: |
+          bash scripts/killall_sglang.sh "nuk_gpus"
+          cd sgl-router
+          SHOW_ROUTER_LOGS=1 ROUTER_LOCAL_MODEL_PATH="/home/ubuntu/models" pytest py_test/e2e_grpc -s -vv -o log_cli=true --log-cli-level=INFO
+
       - name: Upload benchmark results
         if: success()
         uses: actions/upload-artifact@v4

benchmark/mmmu/data_utils.py

@@ -75,12 +75,6 @@
 }
 
 
-# DATA SAVING
-def save_json(filename, ds):
-    with open(filename, "w") as f:
-        json.dump(ds, f, indent=4)
-
-
 def get_multi_choice_info(options):
     """
     Given the list of options for multiple choice question

docker/Dockerfile

@@ -9,7 +9,7 @@ ARG DEEPEP_COMMIT=9af0e0d0e74f3577af1979c9b9e1ac2cad0104ee
 ARG FLASHMLA_COMMIT=1408756a88e52a25196b759eaf8db89d2b51b5a1
 ARG FAST_HADAMARD_TRANSFORM_COMMIT=7fd811c2b47f63b0b08d2582619f939e14dad77c
 ARG CMAKE_BUILD_PARALLEL_LEVEL=2
-ARG SGL_KERNEL_VERSION=0.3.15
+ARG SGL_KERNEL_VERSION=0.3.16.post3
 ENV DEBIAN_FRONTEND=noninteractive \
     CUDA_HOME=/usr/local/cuda \
     GDRCOPY_HOME=/usr/src/gdrdrv-2.4.4/ \
@@ -152,14 +152,6 @@ RUN if [ "$TARGETARCH" = "amd64" ]; then \
       pip install -v . ; \
     fi
 
-# Install fast-hadamard-transform
-RUN if [ "$TARGETARCH" = "amd64" ]; then \
-      git clone https://github.com/Dao-AILab/fast-hadamard-transform && \
-      cd fast-hadamard-transform && \
-      git checkout ${FAST_HADAMARD_TRANSFORM_COMMIT} && \
-      pip install . ; \
-    fi
-
 # Python tools
 RUN python3 -m pip install --no-cache-dir \
     datamodel_code_generator \

docs/advanced_features/lora.ipynb

@@ -59,6 +59,17 @@
     "### Serving Single Adaptor"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "**Note:** SGLang supports LoRA adapters through two APIs:\n",
+    "\n",
+    "1. **OpenAI-Compatible API** (`/v1/chat/completions`, `/v1/completions`): Use the `model:adapter-name` syntax. See [OpenAI API with LoRA](../basic_usage/openai_api_completions.ipynb#Using-LoRA-Adapters) for examples.\n",
+    "\n",
+    "2. **Native API** (`/generate`): Pass `lora_path` in the request body (shown below)."
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -379,6 +390,15 @@
     "print(f\"Output from lora1 (updated): \\n{response.json()[1]['text']}\\n\")"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### OpenAI-compatible API usage\n",
+    "\n",
+    "You can use LoRA adapters via the OpenAI-compatible APIs by specifying the adapter in the `model` field using the `base-model:adapter-name` syntax (for example, `qwen/qwen2.5-0.5b-instruct:adapter_a`). For more details and examples, see the “Using LoRA Adapters” section in the OpenAI API documentation: [openai_api_completions.ipynb](../basic_usage/openai_api_completions.ipynb).\n"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,

docs/advanced_features/server_arguments.md

@@ -228,6 +228,8 @@ Please consult the documentation below and [server_args.py](https://github.com/s
 | `--sampling-backend` | Choose the kernels for sampling layers. | None |
 | `--grammar-backend` | Choose the backend for grammar-guided decoding. | None |
 | `--mm-attention-backend` | Set multimodal attention backend. | None |
+| `--nsa-prefill-backend` | Prefill attention implementation for nsa backend. | `flashmla_sparse` |
+| `--nsa-decode-backend` | Decode attention implementation for nsa backend. | `flashmla_kv` |
 
 ## Speculative decoding
 

docs/basic_usage/deepseek.md

@@ -235,6 +235,44 @@ Important Notes:
 2. To receive more consistent tool call results, it is recommended to use `--chat-template examples/chat_template/tool_chat_template_deepseekv3.jinja`. It provides an improved unified prompt.
 
 
+### Thinking Budget for DeepSeek R1
+
+In SGLang, we can implement thinking budget with `CustomLogitProcessor`.
+
+Launch a server with `--enable-custom-logit-processor` flag on.
+
+```
+python3 -m sglang.launch_server --model deepseek-ai/DeepSeek-R1 --tp 8 --port 30000 --host 0.0.0.0 --mem-fraction-static 0.9 --disable-cuda-graph --reasoning-parser deepseek-r1 --enable-custom-logit-processor
+```
+
+Sample Request:
+
+```python
+import openai
+from rich.pretty import pprint
+from sglang.srt.sampling.custom_logit_processor import DeepSeekR1ThinkingBudgetLogitProcessor
+
+
+client = openai.Client(base_url="http://127.0.0.1:30000/v1", api_key="*")
+response = client.chat.completions.create(
+    model="deepseek-ai/DeepSeek-R1",
+    messages=[
+        {
+            "role": "user",
+            "content": "Question: Is Paris the Capital of France?",
+        }
+    ],
+    max_tokens=1024,
+    extra_body={
+        "custom_logit_processor": DeepSeekR1ThinkingBudgetLogitProcessor().to_str(),
+        "custom_params": {
+            "thinking_budget": 512,
+        },
+    },
+)
+pprint(response)
+```
+
 ## FAQ
 
 **Q: Model loading is taking too long, and I'm encountering an NCCL timeout. What should I do?**

docs/basic_usage/openai_api_completions.ipynb

@@ -361,6 +361,50 @@
     "For OpenAI compatible structured outputs API, refer to [Structured Outputs](../advanced_features/structured_outputs.ipynb) for more details.\n"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Using LoRA Adapters\n",
+    "\n",
+    "SGLang supports LoRA (Low-Rank Adaptation) adapters with OpenAI-compatible APIs. You can specify which adapter to use directly in the `model` parameter using the `base-model:adapter-name` syntax.\n",
+    "\n",
+    "**Server Setup:**\n",
+    "```bash\n",
+    "python -m sglang.launch_server \\\n",
+    "    --model-path qwen/qwen2.5-0.5b-instruct \\\n",
+    "    --enable-lora \\\n",
+    "    --lora-paths adapter_a=/path/to/adapter_a adapter_b=/path/to/adapter_b\n",
+    "```\n",
+    "\n",
+    "For more details on LoRA serving configuration, see the [LoRA documentation](../advanced_features/lora.ipynb).\n",
+    "\n",
+    "**API Call:**\n",
+    "\n",
+    "(Recommended) Use the `model:adapter` syntax to specify which adapter to use:\n",
+    "```python\n",
+    "response = client.chat.completions.create(\n",
+    "    model=\"qwen/qwen2.5-0.5b-instruct:adapter_a\",  # ← base-model:adapter-name\n",
+    "    messages=[{\"role\": \"user\", \"content\": \"Convert to SQL: show all users\"}],\n",
+    "    max_tokens=50,\n",
+    ")\n",
+    "```\n",
+    "\n",
+    "**Backward Compatible: Using `extra_body`**\n",
+    "\n",
+    "The old `extra_body` method is still supported for backward compatibility:\n",
+    "```python\n",
+    "# Backward compatible method\n",
+    "response = client.chat.completions.create(\n",
+    "    model=\"qwen/qwen2.5-0.5b-instruct\",\n",
+    "    messages=[{\"role\": \"user\", \"content\": \"Convert to SQL: show all users\"}],\n",
+    "    extra_body={\"lora_path\": \"adapter_a\"},  # ← old method\n",
+    "    max_tokens=50,\n",
+    ")\n",
+    "```\n",
+    "**Note:** When both `model:adapter` and `extra_body[\"lora_path\"]` are specified, the `model:adapter` syntax takes precedence."
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,

docs/basic_usage/sampling_params.md

@@ -319,3 +319,27 @@ response = requests.post(
 )
 print(response.json())
 ```
+
+Send an OpenAI chat completion request:
+
+```python
+import openai
+from sglang.utils import print_highlight
+
+client = openai.Client(base_url="http://127.0.0.1:30000/v1", api_key="None")
+
+response = client.chat.completions.create(
+    model="meta-llama/Meta-Llama-3-8B-Instruct",
+    messages=[
+        {"role": "user", "content": "List 3 countries and their capitals."},
+    ],
+    temperature=0.0,
+    max_tokens=32,
+    extra_body={
+        "custom_logit_processor": DeterministicLogitProcessor().to_str(),
+        "custom_params": {"token_id": 5},
+    },
+)
+
+print_highlight(f"Response: {response}")
+```

examples/runtime/lora.py

@@ -1,37 +1,67 @@
-# launch server
-# python -m sglang.launch_server --model mistralai/Mistral-7B-Instruct-v0.3 --lora-paths /home/ying/test_lora lora1=/home/ying/test_lora_1 lora2=/home/ying/test_lora_2 --disable-radix --disable-cuda-graph --max-loras-per-batch 4
-
-# send requests
-# lora_path[i] specifies the LoRA used for text[i], so make sure they have the same length
-# use None to specify base-only prompt, e.x. "lora_path": [None, "/home/ying/test_lora"]
-import json
-
-import requests
-
-url = "http://127.0.0.1:30000"
-json_data = {
-    "text": [
-        "prompt 1",
-        "prompt 2",
-        "prompt 3",
-        "prompt 4",
-        "prompt 5",
-        "prompt 6",
-        "prompt 7",
-    ],
-    "sampling_params": {"max_new_tokens": 32},
-    "lora_path": [
-        "/home/ying/test_lora",
-        "lora1",
-        "lora2",
-        "lora1",
-        "lora2",
-        None,
-        None,
-    ],
-}
-response = requests.post(
-    url + "/generate",
-    json=json_data,
-)
-print(json.dumps(response.json()))
+"""
+OpenAI-compatible LoRA adapter usage with SGLang.
+
+Server Setup:
+    python -m sglang.launch_server \\
+        --model meta-llama/Llama-3.1-8B-Instruct \\
+        --enable-lora \\
+        --lora-paths sql=/path/to/sql python=/path/to/python
+"""
+
+import openai
+
+client = openai.Client(base_url="http://127.0.0.1:30000/v1", api_key="EMPTY")
+
+
+def main():
+    print("SGLang OpenAI-Compatible LoRA Examples\n")
+
+    # Example 1: NEW - Adapter in model parameter (OpenAI-compatible)
+    print("1. Chat with LoRA adapter in model parameter:")
+    response = client.chat.completions.create(
+        model="meta-llama/Llama-3.1-8B-Instruct:sql",  # ← adapter:name syntax
+        messages=[{"role": "user", "content": "Convert to SQL: show all users"}],
+        max_tokens=50,
+    )
+    print(f"   Response: {response.choices[0].message.content}\n")
+
+    # Example 2: Completions API with adapter
+    print("2. Completion with LoRA adapter:")
+    response = client.completions.create(
+        model="meta-llama/Llama-3.1-8B-Instruct:python",
+        prompt="def fibonacci(n):",
+        max_tokens=50,
+    )
+    print(f"   Response: {response.choices[0].text}\n")
+
+    # Example 3: OLD - Backward compatible with explicit lora_path
+    print("3. Backward compatible (explicit lora_path):")
+    response = client.chat.completions.create(
+        model="meta-llama/Llama-3.1-8B-Instruct",
+        messages=[{"role": "user", "content": "Convert to SQL: show all users"}],
+        extra_body={"lora_path": "sql"},
+        max_tokens=50,
+    )
+    print(f"   Response: {response.choices[0].message.content}\n")
+
+    # Example 4: Base model (no adapter)
+    print("4. Base model without adapter:")
+    response = client.chat.completions.create(
+        model="meta-llama/Llama-3.1-8B-Instruct",
+        messages=[{"role": "user", "content": "Hello!"}],
+        max_tokens=30,
+    )
+    print(f"   Response: {response.choices[0].message.content}\n")
+
+    print("All examples completed!")
+
+
+if __name__ == "__main__":
+    try:
+        main()
+    except Exception as e:
+        print(f"Error: {e}")
+        print(
+            "\nEnsure server is running:\n"
+            "  python -m sglang.launch_server --model ... --enable-lora --lora-paths ..."
+        )

examples/runtime/multimodal/llava_onevision_server.py

@@ -6,14 +6,14 @@
 python3 llava_onevision_server.py
 """
 
-import base64
 import io
 import os
 import sys
 import time
 
 import numpy as np
 import openai
+import pybase64
 import requests
 from decord import VideoReader, cpu
 from PIL import Image
@@ -213,7 +213,7 @@ def prepare_video_messages(video_path):
         pil_img = Image.fromarray(frame)
         buff = io.BytesIO()
         pil_img.save(buff, format="JPEG")
-        base64_str = base64.b64encode(buff.getvalue()).decode("utf-8")
+        base64_str = pybase64.b64encode(buff.getvalue()).decode("utf-8")
         base64_frames.append(base64_str)
 
     messages = [{"role": "user", "content": []}]