diff --git a/env.example b/env.example index 4992cf3e..feb5b342 100644 --- a/env.example +++ b/env.example @@ -68,6 +68,10 @@ AI_MODEL=global.anthropic.claude-sonnet-4-5-20250929-v1:0 # SILICONFLOW_API_KEY=sk-... # SILICONFLOW_BASE_URL=https://api.siliconflow.com/v1 # Optional: switch to https://api.siliconflow.cn/v1 if needed +# SGLang Configuration (OpenAI-compatible) +# SGLANG_API_KEY=your-sglang-api-key +# SGLANG_BASE_URL=http://127.0.0.1:8000/v1 # Your SGLang endpoint + # Vercel AI Gateway Configuration # Get your API key from: https://vercel.com/ai-gateway # Model format: "provider/model" e.g., "openai/gpt-4o", "anthropic/claude-sonnet-4-5" diff --git a/lib/ai-providers.ts b/lib/ai-providers.ts index 9ac7675d..f0857e3a 100644 --- a/lib/ai-providers.ts +++ b/lib/ai-providers.ts @@ -2,7 +2,6 @@ import { createAmazonBedrock } from "@ai-sdk/amazon-bedrock" import { createAnthropic } from "@ai-sdk/anthropic" import { azure, createAzure } from "@ai-sdk/azure" import { createDeepSeek, deepseek } from "@ai-sdk/deepseek" -import { gateway } from "@ai-sdk/gateway" import { createGoogleGenerativeAI, google } from "@ai-sdk/google" import { createOpenAI, openai } from "@ai-sdk/openai" import { fromNodeProviderChain } from "@aws-sdk/credential-providers" @@ -19,7 +18,7 @@ export type ProviderName = | "openrouter" | "deepseek" | "siliconflow" - | "gateway" + | "sglang" interface ModelConfig { model: any @@ -44,7 +43,7 @@ const ALLOWED_CLIENT_PROVIDERS: ProviderName[] = [ "openrouter", "deepseek", "siliconflow", - "gateway", + "sglang", ] // Bedrock provider options for Anthropic beta features @@ -337,9 +336,8 @@ function buildProviderOptions( case "deepseek": case "openrouter": case "siliconflow": - case "gateway": { + case "sglang": { // These providers don't have reasoning configs in AI SDK yet - // Gateway passes through to underlying providers which handle their own configs break } @@ -361,7 +359,7 @@ const PROVIDER_ENV_VARS: Record = { openrouter: "OPENROUTER_API_KEY", deepseek: "DEEPSEEK_API_KEY", siliconflow: "SILICONFLOW_API_KEY", - gateway: "AI_GATEWAY_API_KEY", + sglang: "SGLANG_API_KEY", } /** @@ -426,7 +424,7 @@ function validateProviderCredentials(provider: ProviderName): void { * Get the AI model based on environment variables * * Environment variables: - * - AI_PROVIDER: The provider to use (bedrock, openai, anthropic, google, azure, ollama, openrouter, deepseek, siliconflow) + * - AI_PROVIDER: The provider to use (bedrock, openai, anthropic, google, azure, ollama, openrouter, deepseek, siliconflow, sglang) * - AI_MODEL: The model ID/name for the selected provider * * Provider-specific env vars: @@ -442,6 +440,8 @@ function validateProviderCredentials(provider: ProviderName): void { * - DEEPSEEK_BASE_URL: DeepSeek endpoint (optional) * - SILICONFLOW_API_KEY: SiliconFlow API key * - SILICONFLOW_BASE_URL: SiliconFlow endpoint (optional, defaults to https://api.siliconflow.com/v1) + * - SGLANG_API_KEY: SGLang API key + * - SGLANG_BASE_URL: SGLang endpoint (optional) */ export function getAIModel(overrides?: ClientOverrides): ModelConfig { // SECURITY: Prevent SSRF attacks (GHSA-9qf7-mprq-9qgm) @@ -501,7 +501,6 @@ export function getAIModel(overrides?: ClientOverrides): ModelConfig { if (configured.length === 0) { throw new Error( `No AI provider configured. Please set one of the following API keys in your .env.local file:\n` + - `- AI_GATEWAY_API_KEY for Vercel AI Gateway\n` + `- DEEPSEEK_API_KEY for DeepSeek\n` + `- OPENAI_API_KEY for OpenAI\n` + `- ANTHROPIC_API_KEY for Anthropic\n` + @@ -510,6 +509,7 @@ export function getAIModel(overrides?: ClientOverrides): ModelConfig { `- OPENROUTER_API_KEY for OpenRouter\n` + `- AZURE_API_KEY for Azure\n` + `- SILICONFLOW_API_KEY for SiliconFlow\n` + + `- SGLANG_API_KEY for SGLang\n` + `Or set AI_PROVIDER=ollama for local Ollama.`, ) } else { @@ -679,17 +679,93 @@ export function getAIModel(overrides?: ClientOverrides): ModelConfig { break } - case "gateway": { - // Vercel AI Gateway - unified access to multiple AI providers - // Model format: "provider/model" e.g., "openai/gpt-4o", "anthropic/claude-sonnet-4-5" - // See: https://vercel.com/ai-gateway - model = gateway(modelId) + case "sglang": { + const apiKey = overrides?.apiKey || process.env.SGLANG_API_KEY + const baseURL = + overrides?.baseUrl || + process.env.SGLANG_BASE_URL + + const sglangProvider = createOpenAI({ + apiKey, + baseURL, + // Add a custom fetch wrapper to intercept and fix the stream from sglang + fetch: async (url, options) => { + const response = await fetch(url, options); + if (!response.body) { + return response; + } + + // Create a transform stream to fix the non-compliant sglang stream + let buffer = ''; + const decoder = new TextDecoder(); + + const transformStream = new TransformStream({ + transform(chunk, controller) { + buffer += decoder.decode(chunk, { stream: true }); + // Process all complete messages in the buffer + let messageEndPos; + while ((messageEndPos = buffer.indexOf('\n\n')) !== -1) { + const message = buffer.substring(0, messageEndPos); + buffer = buffer.substring(messageEndPos + 2); // Move past the '\n\n' + + if (message.startsWith('data: ')) { + const jsonStr = message.substring(6).trim(); + if (jsonStr === '[DONE]') { + controller.enqueue(new TextEncoder().encode(message + '\n\n')); + continue; + } + try { + const data = JSON.parse(jsonStr); + const delta = data.choices?.[0]?.delta; + + if (delta) { + // Fix 1: remove invalid empty role + if (delta.role === '') { + delete delta.role; + } + // Fix 2: remove non-standard reasoning_content field + if ('reasoning_content' in delta) { + delete delta.reasoning_content; + } + } + + // Re-serialize and forward the corrected data with the correct SSE format + controller.enqueue(new TextEncoder().encode(`data: ${JSON.stringify(data)}\n\n`)); + } catch (e) { + // If parsing fails, forward the original message to avoid breaking the stream. + controller.enqueue(new TextEncoder().encode(message + '\n\n')); + } + } else if (message.trim() !== '') { + // Pass through other message types (e.g., 'event: ...') + controller.enqueue(new TextEncoder().encode(message + '\n\n')); + } + } + }, + flush(controller) { + // If there's anything left in the buffer, forward it. + if (buffer.trim()) { + controller.enqueue(new TextEncoder().encode(buffer)); + } + } + }); + + const transformedBody = response.body.pipeThrough(transformStream); + + // Return a new response with the transformed body + return new Response(transformedBody, { + status: response.status, + statusText: response.statusText, + headers: response.headers, + }); + } + }); + model = sglangProvider.chat(modelId) break } default: throw new Error( - `Unknown AI provider: ${provider}. Supported providers: bedrock, openai, anthropic, google, azure, ollama, openrouter, deepseek, siliconflow, gateway`, + `Unknown AI provider: ${provider}. Supported providers: bedrock, openai, anthropic, google, azure, ollama, openrouter, deepseek, siliconflow, sglang`, ) }