Skip to content

fix:feature/sglang-provider #302

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
pointerhacker wants to merge 1 commit into
base: main
Choose a base branch
from
Open

fix:feature/sglang-provider #302

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions env.example
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,10 @@ AI_MODEL=global.anthropic.claude-sonnet-4-5-20250929-v1:0
# SILICONFLOW_API_KEY=sk-...
# SILICONFLOW_BASE_URL=https://api.siliconflow.com/v1 # Optional: switch to https://api.siliconflow.cn/v1 if needed

# SGLang Configuration (OpenAI-compatible)
# SGLANG_API_KEY=your-sglang-api-key
# SGLANG_BASE_URL=http://127.0.0.1:8000/v1 # Your SGLang endpoint

# Vercel AI Gateway Configuration
# Get your API key from: https://vercel.com/ai-gateway
# Model format: "provider/model" e.g., "openai/gpt-4o", "anthropic/claude-sonnet-4-5"
Expand Down
104 changes: 90 additions & 14 deletions lib/ai-providers.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@ import { createAmazonBedrock } from "@ai-sdk/amazon-bedrock"
import { createAnthropic } from "@ai-sdk/anthropic"
import { azure, createAzure } from "@ai-sdk/azure"
import { createDeepSeek, deepseek } from "@ai-sdk/deepseek"
import { gateway } from "@ai-sdk/gateway"
import { createGoogleGenerativeAI, google } from "@ai-sdk/google"
import { createOpenAI, openai } from "@ai-sdk/openai"
import { fromNodeProviderChain } from "@aws-sdk/credential-providers"
Expand All @@ -19,7 +18,7 @@ export type ProviderName =
| "openrouter"
| "deepseek"
| "siliconflow"
| "gateway"
| "sglang"

interface ModelConfig {
model: any
Expand All @@ -44,7 +43,7 @@ const ALLOWED_CLIENT_PROVIDERS: ProviderName[] = [
"openrouter",
"deepseek",
"siliconflow",
"gateway",
"sglang",
]

// Bedrock provider options for Anthropic beta features
Expand Down Expand Up @@ -337,9 +336,8 @@ function buildProviderOptions(
case "deepseek":
case "openrouter":
case "siliconflow":
case "gateway": {
case "sglang": {
// These providers don't have reasoning configs in AI SDK yet
// Gateway passes through to underlying providers which handle their own configs
break
}

Expand All @@ -361,7 +359,7 @@ const PROVIDER_ENV_VARS: Record<ProviderName, string | null> = {
openrouter: "OPENROUTER_API_KEY",
deepseek: "DEEPSEEK_API_KEY",
siliconflow: "SILICONFLOW_API_KEY",
gateway: "AI_GATEWAY_API_KEY",
sglang: "SGLANG_API_KEY",
}

/**
Expand Down Expand Up @@ -426,7 +424,7 @@ function validateProviderCredentials(provider: ProviderName): void {
* Get the AI model based on environment variables
*
* Environment variables:
* - AI_PROVIDER: The provider to use (bedrock, openai, anthropic, google, azure, ollama, openrouter, deepseek, siliconflow)
* - AI_PROVIDER: The provider to use (bedrock, openai, anthropic, google, azure, ollama, openrouter, deepseek, siliconflow, sglang)
* - AI_MODEL: The model ID/name for the selected provider
*
* Provider-specific env vars:
Expand All @@ -442,6 +440,8 @@ function validateProviderCredentials(provider: ProviderName): void {
* - DEEPSEEK_BASE_URL: DeepSeek endpoint (optional)
* - SILICONFLOW_API_KEY: SiliconFlow API key
* - SILICONFLOW_BASE_URL: SiliconFlow endpoint (optional, defaults to https://api.siliconflow.com/v1)
* - SGLANG_API_KEY: SGLang API key
* - SGLANG_BASE_URL: SGLang endpoint (optional)
*/
export function getAIModel(overrides?: ClientOverrides): ModelConfig {
// SECURITY: Prevent SSRF attacks (GHSA-9qf7-mprq-9qgm)
Expand Down Expand Up @@ -501,7 +501,6 @@ export function getAIModel(overrides?: ClientOverrides): ModelConfig {
if (configured.length === 0) {
throw new Error(
`No AI provider configured. Please set one of the following API keys in your .env.local file:\n` +
`- AI_GATEWAY_API_KEY for Vercel AI Gateway\n` +
`- DEEPSEEK_API_KEY for DeepSeek\n` +
`- OPENAI_API_KEY for OpenAI\n` +
`- ANTHROPIC_API_KEY for Anthropic\n` +
Expand All @@ -510,6 +509,7 @@ export function getAIModel(overrides?: ClientOverrides): ModelConfig {
`- OPENROUTER_API_KEY for OpenRouter\n` +
`- AZURE_API_KEY for Azure\n` +
`- SILICONFLOW_API_KEY for SiliconFlow\n` +
`- SGLANG_API_KEY for SGLang\n` +
`Or set AI_PROVIDER=ollama for local Ollama.`,
)
} else {
Expand Down Expand Up @@ -679,17 +679,93 @@ export function getAIModel(overrides?: ClientOverrides): ModelConfig {
break
}

case "gateway": {
// Vercel AI Gateway - unified access to multiple AI providers
// Model format: "provider/model" e.g., "openai/gpt-4o", "anthropic/claude-sonnet-4-5"
// See: https://vercel.com/ai-gateway
model = gateway(modelId)
case "sglang": {
const apiKey = overrides?.apiKey || process.env.SGLANG_API_KEY
const baseURL =
overrides?.baseUrl ||
process.env.SGLANG_BASE_URL

const sglangProvider = createOpenAI({
apiKey,
baseURL,
// Add a custom fetch wrapper to intercept and fix the stream from sglang
fetch: async (url, options) => {
const response = await fetch(url, options);
if (!response.body) {
return response;
}

// Create a transform stream to fix the non-compliant sglang stream
let buffer = '';
const decoder = new TextDecoder();

const transformStream = new TransformStream({
transform(chunk, controller) {
buffer += decoder.decode(chunk, { stream: true });
// Process all complete messages in the buffer
let messageEndPos;
while ((messageEndPos = buffer.indexOf('\n\n')) !== -1) {
const message = buffer.substring(0, messageEndPos);
buffer = buffer.substring(messageEndPos + 2); // Move past the '\n\n'

if (message.startsWith('data: ')) {
const jsonStr = message.substring(6).trim();
if (jsonStr === '[DONE]') {
controller.enqueue(new TextEncoder().encode(message + '\n\n'));
continue;
}
try {
const data = JSON.parse(jsonStr);
const delta = data.choices?.[0]?.delta;

if (delta) {
// Fix 1: remove invalid empty role
if (delta.role === '') {
delete delta.role;
}
// Fix 2: remove non-standard reasoning_content field
if ('reasoning_content' in delta) {
delete delta.reasoning_content;
}
}

// Re-serialize and forward the corrected data with the correct SSE format
controller.enqueue(new TextEncoder().encode(`data: ${JSON.stringify(data)}\n\n`));
} catch (e) {
// If parsing fails, forward the original message to avoid breaking the stream.
controller.enqueue(new TextEncoder().encode(message + '\n\n'));
}
} else if (message.trim() !== '') {
// Pass through other message types (e.g., 'event: ...')
controller.enqueue(new TextEncoder().encode(message + '\n\n'));
}
}
},
flush(controller) {
// If there's anything left in the buffer, forward it.
if (buffer.trim()) {
controller.enqueue(new TextEncoder().encode(buffer));
}
}
});

const transformedBody = response.body.pipeThrough(transformStream);

// Return a new response with the transformed body
return new Response(transformedBody, {
status: response.status,
statusText: response.statusText,
headers: response.headers,
});
}
});
model = sglangProvider.chat(modelId)
break
}

default:
throw new Error(
`Unknown AI provider: ${provider}. Supported providers: bedrock, openai, anthropic, google, azure, ollama, openrouter, deepseek, siliconflow, gateway`,
`Unknown AI provider: ${provider}. Supported providers: bedrock, openai, anthropic, google, azure, ollama, openrouter, deepseek, siliconflow, sglang`,
)
}

Expand Down