diff --git a/core/llm/llms/Anthropic.ts b/core/llm/llms/Anthropic.ts index ff3f85e51eb..362c5559bf3 100644 --- a/core/llm/llms/Anthropic.ts +++ b/core/llm/llms/Anthropic.ts @@ -28,6 +28,7 @@ import { } from "../../index.js"; import { safeParseToolCallArgs } from "../../tools/parseArgs.js"; import { renderChatMessage, stripImages } from "../../util/messageContent.js"; +import { extractBase64FromDataUrl } from "../../util/url.js"; import { DEFAULT_REASONING_TOKENS } from "../constants.js"; import { BaseLLM } from "../index.js"; @@ -110,7 +111,7 @@ class Anthropic extends BaseLLM { source: { type: "base64", media_type: getAnthropicMediaTypeFromDataUrl(part.imageUrl.url), - data: part.imageUrl.url.split(",")[1], + data: extractBase64FromDataUrl(part.imageUrl.url), }, }); } diff --git a/core/llm/llms/Bedrock.ts b/core/llm/llms/Bedrock.ts index ea47b63ae1f..d51093db174 100644 --- a/core/llm/llms/Bedrock.ts +++ b/core/llm/llms/Bedrock.ts @@ -21,6 +21,7 @@ import type { CompletionOptions } from "../../index.js"; import { ChatMessage, Chunk, LLMOptions, MessageContent } from "../../index.js"; import { safeParseToolCallArgs } from "../../tools/parseArgs.js"; import { renderChatMessage, stripImages } from "../../util/messageContent.js"; +import { parseDataUrl } from "../../util/url.js"; import { BaseLLM } from "../index.js"; import { PROVIDER_TOOL_SUPPORT } from "../toolSupport.js"; import { getSecureID } from "../utils/getSecureID.js"; @@ -546,7 +547,7 @@ class Bedrock extends BaseLLM { blocks.push({ text: part.text }); } else if (part.type === "imageUrl" && part.imageUrl) { try { - const [mimeType, base64Data] = part.imageUrl.url.split(","); + const { mimeType, base64Data } = parseDataUrl(part.imageUrl.url); const format = mimeType.split("/")[1]?.split(";")[0] || "jpeg"; if ( format === ImageFormat.JPEG || diff --git a/core/llm/llms/Gemini.ts b/core/llm/llms/Gemini.ts index 7caf2db7d75..d63bc5e9d36 100644 --- a/core/llm/llms/Gemini.ts +++ b/core/llm/llms/Gemini.ts @@ -11,6 +11,7 @@ import { } from "../../index.js"; import { safeParseToolCallArgs } from "../../tools/parseArgs.js"; import { renderChatMessage, stripImages } from "../../util/messageContent.js"; +import { extractBase64FromDataUrl } from "../../util/url.js"; import { BaseLLM } from "../index.js"; import { GeminiChatContent, @@ -191,7 +192,9 @@ class Gemini extends BaseLLM { : { inlineData: { mimeType: "image/jpeg", - data: part.imageUrl?.url.split(",")[1], + data: part.imageUrl?.url + ? extractBase64FromDataUrl(part.imageUrl.url) + : "", }, }; } diff --git a/core/llm/llms/Ollama.ts b/core/llm/llms/Ollama.ts index 0a239fce909..96f169af045 100644 --- a/core/llm/llms/Ollama.ts +++ b/core/llm/llms/Ollama.ts @@ -13,6 +13,7 @@ import { } from "../../index.js"; import { renderChatMessage } from "../../util/messageContent.js"; import { getRemoteModelInfo } from "../../util/ollamaHelper.js"; +import { extractBase64FromDataUrl } from "../../util/url.js"; import { BaseLLM } from "../index.js"; type OllamaChatMessage = { @@ -303,7 +304,9 @@ class Ollama extends BaseLLM implements ModelInstaller { const images: string[] = []; message.content.forEach((part) => { if (part.type === "imageUrl" && part.imageUrl) { - const image = part.imageUrl?.url.split(",").at(-1); + const image = part.imageUrl?.url + ? extractBase64FromDataUrl(part.imageUrl.url) + : undefined; if (image) { images.push(image); } diff --git a/core/util/url.ts b/core/util/url.ts index 83e0edcba14..2c83e8f5999 100644 --- a/core/util/url.ts +++ b/core/util/url.ts @@ -9,3 +9,42 @@ export function canParseUrl(url: string): boolean { return false; } } + +export function parseDataUrl(dataUrl: string): { + mimeType: string; + base64Data: string; +} { + const urlParts = dataUrl.split(","); + + if (urlParts.length < 2) { + throw new Error( + "Invalid data URL format: expected 'data:type;base64,data' format", + ); + } + + const [mimeType, ...base64Parts] = urlParts; + const base64Data = base64Parts.join(","); + + return { mimeType, base64Data }; +} + +export function extractBase64FromDataUrl(dataUrl: string): string { + return parseDataUrl(dataUrl).base64Data; +} + +export function safeSplit( + input: string, + delimiter: string, + expectedParts: number, + errorContext: string = "input", +): string[] { + const parts = input.split(delimiter); + + if (parts.length !== expectedParts) { + throw new Error( + `Invalid ${errorContext} format: expected ${expectedParts} parts separated by "${delimiter}", got ${parts.length}`, + ); + } + + return parts; +} diff --git a/packages/openai-adapters/src/apis/Anthropic.ts b/packages/openai-adapters/src/apis/Anthropic.ts index abfe53446b9..2f2c2b23c51 100644 --- a/packages/openai-adapters/src/apis/Anthropic.ts +++ b/packages/openai-adapters/src/apis/Anthropic.ts @@ -25,6 +25,7 @@ import { CompletionUsage, } from "openai/resources/index"; import { ChatCompletionCreateParams } from "openai/resources/index.js"; +import { extractBase64FromDataUrl } from "../../../../core/util/url.js"; import { AnthropicConfig } from "../types.js"; import { chatChunk, @@ -199,7 +200,7 @@ export class AnthropicApi implements BaseLlmApi { source: { type: "base64", media_type: getAnthropicMediaTypeFromDataUrl(dataUrl), - data: dataUrl.split(",")[1], + data: extractBase64FromDataUrl(dataUrl), }, }); } diff --git a/packages/openai-adapters/src/apis/Bedrock.ts b/packages/openai-adapters/src/apis/Bedrock.ts index cf5588d686d..b39b5ebd6c3 100644 --- a/packages/openai-adapters/src/apis/Bedrock.ts +++ b/packages/openai-adapters/src/apis/Bedrock.ts @@ -31,6 +31,7 @@ import { import { fromNodeProviderChain } from "@aws-sdk/credential-providers"; import { fromStatic } from "@aws-sdk/token-providers"; +import { parseDataUrl } from "../../../../core/util/url.js"; import { BedrockConfig } from "../types.js"; import { chatChunk, chatChunkFromDelta, embedding, rerank } from "../util.js"; import { safeParseArgs } from "../util/parseArgs.js"; @@ -135,9 +136,9 @@ export class BedrockApi implements BaseLlmApi { case "image_url": default: try { - const [mimeType, base64Data] = ( - part as ChatCompletionContentPartImage - ).image_url.url.split(","); + const { mimeType, base64Data } = parseDataUrl( + (part as ChatCompletionContentPartImage).image_url.url, + ); const format = mimeType.split("/")[1]?.split(";")[0] || "jpeg"; if ( format === ImageFormat.JPEG ||