From a88a1386e9afedde69d30a18777c17fbd1e75a8e Mon Sep 17 00:00:00 2001 From: Marcello Fitton Date: Tue, 4 Aug 2026 15:51:16 -0700 Subject: [PATCH 01/11] accumulate and store usage for full agent run instead of just final completion call --- .../aibitat/providers/ai-provider.test.js | 149 ++++++++++++++++++ server/utils/agents/aibitat/index.js | 20 ++- .../agents/aibitat/plugins/chat-history.js | 4 +- .../agents/aibitat/providers/ai-provider.js | 87 +++++++++- .../agents/aibitat/providers/cerebras.js | 11 +- .../utils/agents/aibitat/providers/gemini.js | 9 +- 6 files changed, 260 insertions(+), 20 deletions(-) create mode 100644 server/__tests__/utils/agents/aibitat/providers/ai-provider.test.js diff --git a/server/__tests__/utils/agents/aibitat/providers/ai-provider.test.js b/server/__tests__/utils/agents/aibitat/providers/ai-provider.test.js new file mode 100644 index 00000000000..b3b93e35131 --- /dev/null +++ b/server/__tests__/utils/agents/aibitat/providers/ai-provider.test.js @@ -0,0 +1,149 @@ +const Provider = require("../../../../../utils/agents/aibitat/providers/ai-provider.js"); +const UnTooled = require("../../../../../utils/agents/aibitat/providers/helpers/untooled.js"); +const InheritMultiple = require("../../../../../utils/agents/aibitat/providers/helpers/classes.js"); + +class TestProvider extends Provider { + model = "test-model"; + + constructor() { + super(null); + } +} + +// Mirrors how the UnTooled providers (LM Studio, LocalAI, Cerebras, etc.) are +// declared - Provider's fields and methods arrive via the InheritMultiple mixin +// rather than a direct prototype chain. +class MixinProvider extends InheritMultiple([Provider, UnTooled]) { + model = "mixin-model"; +} + +describe("Provider usage tracking", () => { + test("recordUsage accumulates tokens across multiple completions", () => { + const provider = new TestProvider(); + + provider.resetUsage(); + provider.recordUsage({ + prompt_tokens: 100, + completion_tokens: 20, + total_tokens: 120, + }); + + provider.resetUsage(); + provider.recordUsage({ + prompt_tokens: 250, + completion_tokens: 40, + total_tokens: 290, + }); + + provider.resetUsage(); + provider.recordUsage({ + prompt_tokens: 400, + completion_tokens: 60, + total_tokens: 460, + }); + + // getUsage only reflects the most recent completion + const last = provider.getUsage(); + expect(last.prompt_tokens).toBe(400); + expect(last.completion_tokens).toBe(60); + expect(last.total_tokens).toBe(460); + + // getCumulativeUsage reflects the sum of all completions + const totals = provider.getCumulativeUsage(); + expect(totals.prompt_tokens).toBe(750); + expect(totals.completion_tokens).toBe(120); + expect(totals.total_tokens).toBe(870); + expect(totals.model).toBe("test-model"); + expect(totals.provider).toBe("TestProvider"); + }); + + test("resetUsage does not clear the accumulated totals", () => { + const provider = new TestProvider(); + + provider.resetUsage(); + provider.recordUsage({ + prompt_tokens: 100, + completion_tokens: 20, + total_tokens: 120, + }); + + provider.resetUsage(); + expect(provider.getUsage().total_tokens).toBe(0); + expect(provider.getCumulativeUsage().total_tokens).toBe(120); + }); + + test("resetCumulativeUsage zeroes the accumulated totals", () => { + const provider = new TestProvider(); + + provider.resetUsage(); + provider.recordUsage({ + prompt_tokens: 100, + completion_tokens: 20, + total_tokens: 120, + }); + + provider.resetCumulativeUsage(); + const totals = provider.getCumulativeUsage(); + expect(totals.prompt_tokens).toBe(0); + expect(totals.completion_tokens).toBe(0); + expect(totals.total_tokens).toBe(0); + expect(totals.model).toBe(null); + expect(totals.provider).toBe(null); + }); + + test("recordUsage normalizes Anthropic-style input/output token keys", () => { + const provider = new TestProvider(); + + provider.resetUsage(); + provider.recordUsage({ input_tokens: 30, output_tokens: 10 }); + + provider.resetUsage(); + provider.recordUsage({ input_tokens: 50, output_tokens: 15 }); + + const totals = provider.getCumulativeUsage(); + expect(totals.prompt_tokens).toBe(80); + expect(totals.completion_tokens).toBe(25); + expect(totals.total_tokens).toBe(105); + }); + + test("instances do not share an accumulator", () => { + const providerA = new TestProvider(); + const providerB = new TestProvider(); + + providerA.resetUsage(); + providerA.recordUsage({ + prompt_tokens: 100, + completion_tokens: 20, + total_tokens: 120, + }); + + expect(providerA.getCumulativeUsage().total_tokens).toBe(120); + expect(providerB.getCumulativeUsage().total_tokens).toBe(0); + }); + + test("accumulation works through InheritMultiple mixin providers", () => { + const providerA = new MixinProvider(); + const providerB = new MixinProvider(); + + providerA.resetUsage(); + providerA.recordUsage({ + prompt_tokens: 100, + completion_tokens: 10, + total_tokens: 110, + }); + + providerA.resetUsage(); + providerA.recordUsage({ + prompt_tokens: 300, + completion_tokens: 30, + total_tokens: 330, + }); + + expect(providerA.getCumulativeUsage().total_tokens).toBe(440); + expect(providerA.getUsage().total_tokens).toBe(330); + expect(providerB.getCumulativeUsage().total_tokens).toBe(0); + + providerA.resetCumulativeUsage(); + expect(providerA.getCumulativeUsage().total_tokens).toBe(0); + }); +}); diff --git a/server/utils/agents/aibitat/index.js b/server/utils/agents/aibitat/index.js index c87eac58d06..b73fe748730 100644 --- a/server/utils/agents/aibitat/index.js +++ b/server/utils/agents/aibitat/index.js @@ -1025,7 +1025,11 @@ https://docs.anythingllm.com/agent/intelligent-tool-selection }; // Emit routing notification before the first completion so it appears above the response - if (depth === 0) this?.flushRoutingMetadata?.(v4()); + // and reset the usage accumulator so metrics only cover this run's completions. + if (depth === 0) { + this?.flushRoutingMetadata?.(v4()); + this.providerInstance?.resetCumulativeUsage?.(); + } /** @type {{ functionCall: { name: string, arguments: string }, textResponse: string }} */ const completionStream = await this.#safeProviderCall(() => @@ -1111,7 +1115,7 @@ https://docs.anythingllm.com/agent/intelligent-tool-selection eventHandler?.("reportStreamEvent", { type: "usageMetrics", uuid: directOutputUUID, - metrics: this.providerInstance.getUsage(), + metrics: this.providerInstance.getCumulativeUsage(), }); this?.flushCitations?.(directOutputUUID); this?.emitChatId?.(directOutputUUID); @@ -1152,7 +1156,7 @@ https://docs.anythingllm.com/agent/intelligent-tool-selection eventHandler?.("reportStreamEvent", { type: "usageMetrics", uuid: responseUuid, - metrics: this.providerInstance.getUsage(), + metrics: this.providerInstance.getCumulativeUsage(), }); this?.flushCitations?.(responseUuid); this?.emitChatId?.(responseUuid); @@ -1187,7 +1191,11 @@ https://docs.anythingllm.com/agent/intelligent-tool-selection }; // Emit routing notification before the first completion so it appears above the response - if (depth === 0) this?.flushRoutingMetadata?.(msgUUID); + // and reset the usage accumulator so metrics only cover this run's completions. + if (depth === 0) { + this?.flushRoutingMetadata?.(msgUUID); + this.providerInstance?.resetCumulativeUsage?.(); + } // get the chat completion const completion = await this.#safeProviderCall(() => @@ -1262,7 +1270,7 @@ https://docs.anythingllm.com/agent/intelligent-tool-selection eventHandler?.("reportStreamEvent", { type: "usageMetrics", uuid: msgUUID, - metrics: this.providerInstance.getUsage(), + metrics: this.providerInstance.getCumulativeUsage(), }); this?.flushCitations?.(msgUUID); return result; @@ -1302,7 +1310,7 @@ https://docs.anythingllm.com/agent/intelligent-tool-selection eventHandler?.("reportStreamEvent", { type: "usageMetrics", uuid: msgUUID, - metrics: this.providerInstance.getUsage(), + metrics: this.providerInstance.getCumulativeUsage(), }); this?.flushCitations?.(msgUUID); this?.emitChatId?.(msgUUID); diff --git a/server/utils/agents/aibitat/plugins/chat-history.js b/server/utils/agents/aibitat/plugins/chat-history.js index 257a3bba706..c5bbee498b7 100644 --- a/server/utils/agents/aibitat/plugins/chat-history.js +++ b/server/utils/agents/aibitat/plugins/chat-history.js @@ -99,7 +99,7 @@ const chatHistory = { { prompt, response, attachments = [] } = {} ) { const invocation = aibitat.handlerProps.invocation; - const metrics = aibitat.providerInstance?.getUsage?.() ?? {}; + const metrics = aibitat.providerInstance?.getCumulativeUsage?.() ?? {}; const citations = aibitat._pendingCitations ?? []; const outputs = aibitat._pendingOutputs ?? []; const clarifyingQuestions = @@ -134,7 +134,7 @@ const chatHistory = { { prompt, response, attachments = [], options = {} } = {} ) { const invocation = aibitat.handlerProps.invocation; - const metrics = aibitat.providerInstance?.getUsage?.() ?? {}; + const metrics = aibitat.providerInstance?.getCumulativeUsage?.() ?? {}; const citations = aibitat._pendingCitations ?? []; const outputs = aibitat._pendingOutputs ?? []; const clarifyingQuestions = diff --git a/server/utils/agents/aibitat/providers/ai-provider.js b/server/utils/agents/aibitat/providers/ai-provider.js index 50f147f47b5..9f4dd99d4c7 100644 --- a/server/utils/agents/aibitat/providers/ai-provider.js +++ b/server/utils/agents/aibitat/providers/ai-provider.js @@ -51,6 +51,8 @@ const { bindAbortSignal } = require("../../../helpers/abortSignals"); * @property {(messages: Array, functions?: Array, eventHandler?: Function) => Promise<{functionCall: any, textResponse: string}>} stream - Stream a chat completion with tool calling. * @property {(messages: Array, functions?: Array) => Promise<{functionCall: any, textResponse: string, result?: string}>} complete - Non-streaming chat completion with tool calling. * @property {() => ProviderUsageMetrics} getUsage - Get usage metrics from the last completion. + * @property {() => ProviderUsageMetrics} getCumulativeUsage - Get usage metrics accumulated across all completions in the current run. + * @property {() => void} resetCumulativeUsage - Reset the accumulated usage metrics (call at the start of a run). */ class Provider { @@ -86,6 +88,24 @@ class Provider { timestamp: null, }; + /** + * Stores the usage metrics accumulated across every completion call in the + * current run. An agent loop makes one completion per tool call plus a final + * one for the response - this is the sum of all of them, whereas `lastUsage` + * only ever reflects the most recent call. + * @type {ProviderUsageMetrics} + */ + cumulativeUsage = { + prompt_tokens: 0, + completion_tokens: 0, + total_tokens: 0, + duration: 0, + outputTps: 0, + model: null, + provider: null, + timestamp: null, + }; + /** * Timestamp when the current request started (for duration calculation). * @type {number} @@ -639,16 +659,68 @@ class Provider { const completionTokens = usage.completion_tokens || usage.output_tokens || 0; - this.lastUsage = { + this.applyUsage({ prompt_tokens: promptTokens, completion_tokens: completionTokens, total_tokens: usage.total_tokens || promptTokens + completionTokens, + duration, + }); + } + + /** + * Stores a normalized usage record for the completion that just finished and + * adds it to the run-level accumulated totals. Subclasses that override + * `recordUsage` should normalize their provider-specific usage format and + * call this so accumulation still happens in one place. + * @param {{prompt_tokens?: number, completion_tokens?: number, total_tokens?: number, duration?: number}} usage + */ + applyUsage({ + prompt_tokens = 0, + completion_tokens = 0, + total_tokens = 0, + duration = 0, + } = {}) { + const timestamp = new Date(); + this.lastUsage = { + prompt_tokens, + completion_tokens, + total_tokens, outputTps: - completionTokens && duration > 0 ? completionTokens / duration : 0, + completion_tokens && duration > 0 ? completion_tokens / duration : 0, duration, model: this.model, provider: this.constructor.name, - timestamp: new Date(), + timestamp, + }; + + const totals = this.cumulativeUsage; + totals.prompt_tokens += prompt_tokens; + totals.completion_tokens += completion_tokens; + totals.total_tokens += total_tokens; + totals.duration += duration; + totals.outputTps = + totals.completion_tokens && totals.duration > 0 + ? totals.completion_tokens / totals.duration + : 0; + totals.model = this.model; + totals.provider = this.constructor.name; + totals.timestamp = timestamp; + } + + /** + * Resets the accumulated usage metrics. Call this at the start of an agent + * run so the totals only cover that run's completions. + */ + resetCumulativeUsage() { + this.cumulativeUsage = { + prompt_tokens: 0, + completion_tokens: 0, + total_tokens: 0, + duration: 0, + outputTps: 0, + model: null, + provider: null, + timestamp: null, }; } @@ -660,6 +732,15 @@ class Provider { return { ...this.lastUsage }; } + /** + * Get the usage metrics accumulated across all completions in the current + * run - one completion per tool call plus the final response. + * @returns {ProviderUsageMetrics} The accumulated usage metrics + */ + getCumulativeUsage() { + return { ...this.cumulativeUsage }; + } + /** * Stream a chat completion from the LLM with tool calling * Note: This using the OpenAI API format and may need to be adapted for other providers. diff --git a/server/utils/agents/aibitat/providers/cerebras.js b/server/utils/agents/aibitat/providers/cerebras.js index ba765b4d0fd..69a253853d4 100644 --- a/server/utils/agents/aibitat/providers/cerebras.js +++ b/server/utils/agents/aibitat/providers/cerebras.js @@ -181,17 +181,12 @@ class CerebrasProvider extends InheritMultiple([Provider, UnTooled]) { const completionTokens = usage.completion_tokens || 0; if (time_info?.completion_time) duration = time_info.completion_time; - this.lastUsage = { + this.applyUsage({ prompt_tokens: promptTokens, completion_tokens: completionTokens, - total_tokens: usage.total_tokens, - outputTps: - completionTokens && duration > 0 ? completionTokens / duration : 0, + total_tokens: usage.total_tokens || promptTokens + completionTokens, duration, - model: this.model, - provider: this.constructor.name, - timestamp: new Date(), - }; + }); } /** diff --git a/server/utils/agents/aibitat/providers/gemini.js b/server/utils/agents/aibitat/providers/gemini.js index 0777f2c4685..cc0581430be 100644 --- a/server/utils/agents/aibitat/providers/gemini.js +++ b/server/utils/agents/aibitat/providers/gemini.js @@ -274,12 +274,17 @@ class GeminiProvider extends Provider { functionCall: null, }; + // Gemini can attach usage to more than one chunk, so capture the latest + // and record exactly once after the stream ends - recordUsage accumulates + // into run totals, so calling it per-chunk would inflate them. + let usage = null; + for await (const streamEvent of response) { /** @type {OpenAI.OpenAI.Chat.ChatCompletionChunk} */ const chunk = streamEvent; // Capture usage from final chunk (when stream_options.include_usage is true) - if (chunk?.usage) this.recordUsage(chunk.usage); + if (chunk?.usage) usage = chunk.usage; const { content, tool_calls } = chunk?.choices?.[0]?.delta || {}; if (content) { @@ -316,6 +321,8 @@ class GeminiProvider extends Provider { } } + if (usage) this.recordUsage(usage); + if (completion.functionCall) { completion.functionCall.arguments = safeJsonParse( completion.functionCall.arguments, From 4f6d7b5d87c4c5ef71bbd89eb75d88296874d6a0 Mon Sep 17 00:00:00 2001 From: Marcello Fitton Date: Thu, 6 Aug 2026 12:52:32 -0700 Subject: [PATCH 02/11] add static helper for getting empty cumulative usage state --- .../agents/aibitat/providers/ai-provider.js | 50 ++++++++----------- 1 file changed, 20 insertions(+), 30 deletions(-) diff --git a/server/utils/agents/aibitat/providers/ai-provider.js b/server/utils/agents/aibitat/providers/ai-provider.js index 9f4dd99d4c7..c34919b67f7 100644 --- a/server/utils/agents/aibitat/providers/ai-provider.js +++ b/server/utils/agents/aibitat/providers/ai-provider.js @@ -77,16 +77,7 @@ class Provider { * Stores the usage metrics from the last completion call. * @type {ProviderUsageMetrics} */ - lastUsage = { - prompt_tokens: 0, - completion_tokens: 0, - total_tokens: 0, - duration: 0, - outputTps: 0, - model: null, - provider: null, - timestamp: null, - }; + lastUsage = Provider.#emptyUsage(); /** * Stores the usage metrics accumulated across every completion call in the @@ -95,16 +86,24 @@ class Provider { * only ever reflects the most recent call. * @type {ProviderUsageMetrics} */ - cumulativeUsage = { - prompt_tokens: 0, - completion_tokens: 0, - total_tokens: 0, - duration: 0, - outputTps: 0, - model: null, - provider: null, - timestamp: null, - }; + cumulativeUsage = Provider.#emptyUsage(); + + /** + * Zeroed usage metrics for initializing/resetting an accumulator. + * @returns {ProviderUsageMetrics} + */ + static #emptyUsage() { + return { + prompt_tokens: 0, + completion_tokens: 0, + total_tokens: 0, + duration: 0, + outputTps: 0, + model: null, + provider: null, + timestamp: null, + }; + } /** * Timestamp when the current request started (for duration calculation). @@ -712,16 +711,7 @@ class Provider { * run so the totals only cover that run's completions. */ resetCumulativeUsage() { - this.cumulativeUsage = { - prompt_tokens: 0, - completion_tokens: 0, - total_tokens: 0, - duration: 0, - outputTps: 0, - model: null, - provider: null, - timestamp: null, - }; + this.cumulativeUsage = Provider.#emptyUsage(); } /** From e2dd6b6ac3746d6da735a8e4e67e266477b46954 Mon Sep 17 00:00:00 2001 From: Marcello Fitton Date: Thu, 6 Aug 2026 12:54:10 -0700 Subject: [PATCH 03/11] remove optional chaining --- server/utils/agents/aibitat/index.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/utils/agents/aibitat/index.js b/server/utils/agents/aibitat/index.js index b73fe748730..e8cb12957c9 100644 --- a/server/utils/agents/aibitat/index.js +++ b/server/utils/agents/aibitat/index.js @@ -1028,7 +1028,7 @@ https://docs.anythingllm.com/agent/intelligent-tool-selection // and reset the usage accumulator so metrics only cover this run's completions. if (depth === 0) { this?.flushRoutingMetadata?.(v4()); - this.providerInstance?.resetCumulativeUsage?.(); + this.providerInstance.resetCumulativeUsage(); } /** @type {{ functionCall: { name: string, arguments: string }, textResponse: string }} */ @@ -1194,7 +1194,7 @@ https://docs.anythingllm.com/agent/intelligent-tool-selection // and reset the usage accumulator so metrics only cover this run's completions. if (depth === 0) { this?.flushRoutingMetadata?.(msgUUID); - this.providerInstance?.resetCumulativeUsage?.(); + this.providerInstance.resetCumulativeUsage(); } // get the chat completion From 8a48b5e4c67b2bbf134afcb2dac43de7a3a6e917 Mon Sep 17 00:00:00 2001 From: Marcello Fitton Date: Thu, 6 Aug 2026 16:47:18 -0700 Subject: [PATCH 04/11] add sanitation to usage and adversarial tests --- .../aibitat/providers/ai-provider.test.js | 135 ++++++++++++++++++ .../agents/aibitat/providers/ai-provider.js | 58 +++++--- .../agents/aibitat/providers/cerebras.js | 7 +- 3 files changed, 180 insertions(+), 20 deletions(-) diff --git a/server/__tests__/utils/agents/aibitat/providers/ai-provider.test.js b/server/__tests__/utils/agents/aibitat/providers/ai-provider.test.js index b3b93e35131..7e1a41c5789 100644 --- a/server/__tests__/utils/agents/aibitat/providers/ai-provider.test.js +++ b/server/__tests__/utils/agents/aibitat/providers/ai-provider.test.js @@ -147,3 +147,138 @@ describe("Provider usage tracking", () => { expect(providerA.getCumulativeUsage().total_tokens).toBe(0); }); }); + +describe("Provider usage robustness against malformed payloads", () => { + test.each([ + ["null", null], + ["undefined", undefined], + ["a string", "not-a-usage-object"], + ["a number", 42], + ["a boolean", true], + ["an array", [100, 20, 120]], + ["an empty object", {}], + ])("recordUsage does not crash when the payload is %s", (_label, payload) => { + const provider = new TestProvider(); + + provider.resetUsage(); + expect(() => provider.recordUsage(payload)).not.toThrow(); + + const totals = provider.getCumulativeUsage(); + expect(totals.prompt_tokens).toBe(0); + expect(totals.completion_tokens).toBe(0); + expect(totals.total_tokens).toBe(0); + }); + + test.each([ + ["null", null], + ["undefined", undefined], + ["a string", "not-a-usage-object"], + ["an array", [100, 20, 120]], + ])("applyUsage does not crash when the payload is %s", (_label, payload) => { + const provider = new TestProvider(); + expect(() => provider.applyUsage(payload)).not.toThrow(); + expect(provider.getCumulativeUsage().total_tokens).toBe(0); + }); + + test("coerces numeric strings instead of concatenating them", () => { + const provider = new TestProvider(); + + provider.resetUsage(); + provider.recordUsage({ + prompt_tokens: "100", + completion_tokens: "20", + total_tokens: "120", + }); + + provider.resetUsage(); + provider.recordUsage({ + prompt_tokens: "50", + completion_tokens: "5", + total_tokens: "55", + }); + + const totals = provider.getCumulativeUsage(); + expect(totals.prompt_tokens).toBe(150); + expect(totals.completion_tokens).toBe(25); + expect(totals.total_tokens).toBe(175); + expect(typeof totals.total_tokens).toBe("number"); + }); + + test("treats negative, NaN, and non-finite token counts as zero", () => { + const provider = new TestProvider(); + + provider.resetUsage(); + provider.recordUsage({ + prompt_tokens: -100, + completion_tokens: NaN, + total_tokens: Infinity, + }); + + const totals = provider.getCumulativeUsage(); + expect(totals.prompt_tokens).toBe(0); + expect(totals.completion_tokens).toBe(0); + expect(totals.total_tokens).toBe(0); + }); + + test("treats non-numeric token values as zero", () => { + const provider = new TestProvider(); + + provider.resetUsage(); + provider.recordUsage({ + prompt_tokens: { nested: 100 }, + completion_tokens: "twenty", + total_tokens: () => 120, + }); + + const totals = provider.getCumulativeUsage(); + expect(totals.prompt_tokens).toBe(0); + expect(totals.completion_tokens).toBe(0); + expect(totals.total_tokens).toBe(0); + }); + + test("garbage payloads between valid completions do not corrupt totals", () => { + const provider = new TestProvider(); + + provider.resetUsage(); + provider.recordUsage({ + prompt_tokens: 100, + completion_tokens: 20, + total_tokens: 120, + }); + + provider.resetUsage(); + provider.recordUsage(null); + + provider.resetUsage(); + provider.recordUsage({ prompt_tokens: "junk", completion_tokens: -5 }); + + provider.resetUsage(); + provider.recordUsage({ + prompt_tokens: 50, + completion_tokens: 10, + total_tokens: 60, + }); + + const totals = provider.getCumulativeUsage(); + expect(totals.prompt_tokens).toBe(150); + expect(totals.completion_tokens).toBe(30); + expect(totals.total_tokens).toBe(180); + }); + + test("mixin providers survive malformed payloads too", () => { + const provider = new MixinProvider(); + + provider.resetUsage(); + expect(() => provider.recordUsage(null)).not.toThrow(); + expect(() => provider.recordUsage([1, 2, 3])).not.toThrow(); + + provider.resetUsage(); + provider.recordUsage({ + prompt_tokens: 100, + completion_tokens: 10, + total_tokens: 110, + }); + + expect(provider.getCumulativeUsage().total_tokens).toBe(110); + }); +}); diff --git a/server/utils/agents/aibitat/providers/ai-provider.js b/server/utils/agents/aibitat/providers/ai-provider.js index c34919b67f7..0d90656f1f0 100644 --- a/server/utils/agents/aibitat/providers/ai-provider.js +++ b/server/utils/agents/aibitat/providers/ai-provider.js @@ -105,6 +105,20 @@ class Provider { }; } + /** + * Coerces a provider-reported metric into a safe, finite, non-negative + * number. Providers report usage in inconsistent shapes (missing keys, + * numeric strings, nulls, negative or non-finite values), so anything that + * does not resolve to a usable number becomes 0. + * @param {unknown} value + * @returns {number} + */ + static #toSafeMetric(value) { + const number = Number(value); + if (!Number.isFinite(number) || number < 0) return 0; + return number; + } + /** * Timestamp when the current request started (for duration calculation). * @type {number} @@ -654,14 +668,19 @@ class Provider { duration = (Date.now() - this._requestStartTime) / 1000; } - const promptTokens = usage.prompt_tokens || usage.input_tokens || 0; - const completionTokens = - usage.completion_tokens || usage.output_tokens || 0; + const safeUsage = usage && typeof usage === "object" ? usage : {}; + const promptTokens = Provider.#toSafeMetric( + safeUsage.prompt_tokens || safeUsage.input_tokens + ); + const completionTokens = Provider.#toSafeMetric( + safeUsage.completion_tokens || safeUsage.output_tokens + ); + const totalTokens = Provider.#toSafeMetric(safeUsage.total_tokens); this.applyUsage({ prompt_tokens: promptTokens, completion_tokens: completionTokens, - total_tokens: usage.total_tokens || promptTokens + completionTokens, + total_tokens: totalTokens || promptTokens + completionTokens, duration, }); } @@ -671,21 +690,26 @@ class Provider { * adds it to the run-level accumulated totals. Subclasses that override * `recordUsage` should normalize their provider-specific usage format and * call this so accumulation still happens in one place. + * Every value is coerced to a safe number so a malformed payload from any + * provider cannot crash the run or corrupt the accumulated totals. * @param {{prompt_tokens?: number, completion_tokens?: number, total_tokens?: number, duration?: number}} usage */ - applyUsage({ - prompt_tokens = 0, - completion_tokens = 0, - total_tokens = 0, - duration = 0, - } = {}) { + applyUsage(usage = {}) { + const safeUsage = usage && typeof usage === "object" ? usage : {}; + const promptTokens = Provider.#toSafeMetric(safeUsage.prompt_tokens); + const completionTokens = Provider.#toSafeMetric( + safeUsage.completion_tokens + ); + const totalTokens = Provider.#toSafeMetric(safeUsage.total_tokens); + const duration = Provider.#toSafeMetric(safeUsage.duration); + const timestamp = new Date(); this.lastUsage = { - prompt_tokens, - completion_tokens, - total_tokens, + prompt_tokens: promptTokens, + completion_tokens: completionTokens, + total_tokens: totalTokens, outputTps: - completion_tokens && duration > 0 ? completion_tokens / duration : 0, + completionTokens && duration > 0 ? completionTokens / duration : 0, duration, model: this.model, provider: this.constructor.name, @@ -693,9 +717,9 @@ class Provider { }; const totals = this.cumulativeUsage; - totals.prompt_tokens += prompt_tokens; - totals.completion_tokens += completion_tokens; - totals.total_tokens += total_tokens; + totals.prompt_tokens += promptTokens; + totals.completion_tokens += completionTokens; + totals.total_tokens += totalTokens; totals.duration += duration; totals.outputTps = totals.completion_tokens && totals.duration > 0 diff --git a/server/utils/agents/aibitat/providers/cerebras.js b/server/utils/agents/aibitat/providers/cerebras.js index 69a253853d4..d75e66ceda4 100644 --- a/server/utils/agents/aibitat/providers/cerebras.js +++ b/server/utils/agents/aibitat/providers/cerebras.js @@ -177,14 +177,15 @@ class CerebrasProvider extends InheritMultiple([Provider, UnTooled]) { recordUsage(usage = {}, time_info = {}) { // assume start time let duration = (Date.now() - this._requestStartTime) / 1000; - const promptTokens = usage.prompt_tokens || 0; - const completionTokens = usage.completion_tokens || 0; + const safeUsage = usage && typeof usage === "object" ? usage : {}; + const promptTokens = safeUsage.prompt_tokens || 0; + const completionTokens = safeUsage.completion_tokens || 0; if (time_info?.completion_time) duration = time_info.completion_time; this.applyUsage({ prompt_tokens: promptTokens, completion_tokens: completionTokens, - total_tokens: usage.total_tokens || promptTokens + completionTokens, + total_tokens: safeUsage.total_tokens || promptTokens + completionTokens, duration, }); } From cf565e0855a40bc59d4ff507af2b589bd4b72b95 Mon Sep 17 00:00:00 2001 From: Marcello Fitton Date: Fri, 7 Aug 2026 01:17:29 -0700 Subject: [PATCH 05/11] add cost feature --- .../Actions/RenderMetrics/index.jsx | 11 +- frontend/src/hooks/useCurrency.js | 43 ++ frontend/src/locales/en/common.js | 6 + frontend/src/models/appearance.js | 4 +- .../ScheduledJobs/RunDetailPage.jsx | 10 + .../Settings/Interface/index.jsx | 2 + .../components/CurrencyPreference/index.jsx | 45 ++ frontend/src/utils/currency.js | 110 +++++ frontend/src/utils/numbers.js | 7 - .../aibitat/providers/ai-provider.test.js | 68 +++ .../helpers/modelPricing/fixtures/api.json | 96 ++++ .../utils/helpers/modelPricing/index.test.js | 361 +++++++++++++++ server/index.js | 1 + server/package.json | 1 + server/storage/models/.gitignore | 1 + server/utils/agents/aibitat/index.js | 4 + .../agents/aibitat/providers/ai-provider.js | 27 ++ server/utils/chats/apiChatHandler.js | 54 ++- server/utils/chats/embed.js | 31 +- server/utils/chats/openaiCompatible.js | 75 ++-- server/utils/chats/stream.js | 13 +- server/utils/helpers/modelPricing/index.js | 415 ++++++++++++++++++ server/utils/telegramBot/chat/stream.js | 21 +- server/yarn.lock | 5 + 24 files changed, 1332 insertions(+), 79 deletions(-) create mode 100644 frontend/src/hooks/useCurrency.js create mode 100644 frontend/src/pages/GeneralSettings/Settings/components/CurrencyPreference/index.jsx create mode 100644 frontend/src/utils/currency.js create mode 100644 server/__tests__/utils/helpers/modelPricing/fixtures/api.json create mode 100644 server/__tests__/utils/helpers/modelPricing/index.test.js create mode 100644 server/utils/helpers/modelPricing/index.js diff --git a/frontend/src/components/WorkspaceChat/ChatContainer/ChatHistory/HistoricalMessage/Actions/RenderMetrics/index.jsx b/frontend/src/components/WorkspaceChat/ChatContainer/ChatHistory/HistoricalMessage/Actions/RenderMetrics/index.jsx index ea6dac5305a..6e174f0e755 100644 --- a/frontend/src/components/WorkspaceChat/ChatContainer/ChatHistory/HistoricalMessage/Actions/RenderMetrics/index.jsx +++ b/frontend/src/components/WorkspaceChat/ChatContainer/ChatHistory/HistoricalMessage/Actions/RenderMetrics/index.jsx @@ -1,5 +1,6 @@ import { formatDateTimeAsMoment } from "@/utils/directories"; import { formatDuration, numberWithCommas } from "@/utils/numbers"; +import useCurrency from "@/hooks/useCurrency"; import React, { useEffect, useState, useContext } from "react"; import { isMobile } from "react-device-detect"; const MetricsContext = React.createContext(); @@ -33,14 +34,17 @@ function getAutoShowMetrics() { * Build the metrics string for a given metrics object * - Model name * - Duration and output TPS + * - Cost (in the user's preferred currency, when known) * - Timestamp - * @param {metrics: {duration:number, outputTps: number, model?: string, timestamp?: number}} metrics + * @param {metrics: {duration:number, outputTps: number, model?: string, timestamp?: number, totalCost?: number}} metrics + * @param {(usd: number) => string} formatCost - formats a USD cost in the user's preferred currency * @returns {string} */ -function buildMetricsString(metrics = {}) { +function buildMetricsString(metrics = {}, formatCost = () => "") { return [ metrics?.model ? metrics.model : "", `${formatDuration(metrics.duration)} (${formatTps(metrics.outputTps)} tok/s)`, + typeof metrics?.totalCost === "number" ? formatCost(metrics.totalCost) : "", metrics?.timestamp ? formatDateTimeAsMoment(metrics.timestamp, "MMM D, h:mm A") : "", @@ -103,6 +107,7 @@ export default function RenderMetrics({ metrics = {} }) { // Inherit the showMetricsAutomatically state from the MetricsProvider so the state is shared across all chats const { showMetricsAutomatically, setShowMetricsAutomatically } = useContext(MetricsContext); + const { formatCost } = useCurrency(); if (!metrics?.duration || !metrics?.outputTps || isMobile) return null; return ( @@ -118,7 +123,7 @@ export default function RenderMetrics({ metrics = {} }) { className={`border-none flex md:justify-end items-center gap-x-[8px] -ml-7 ${showMetricsAutomatically ? "opacity-100" : "opacity-0"} md:group-hover:opacity-100 transition-all duration-300`} >

- {buildMetricsString(metrics)} + {buildMetricsString(metrics, formatCost)}

); diff --git a/frontend/src/hooks/useCurrency.js b/frontend/src/hooks/useCurrency.js new file mode 100644 index 00000000000..cf08d4a7c10 --- /dev/null +++ b/frontend/src/hooks/useCurrency.js @@ -0,0 +1,43 @@ +import { useCallback, useEffect, useState } from "react"; +import Appearance from "@/models/appearance"; +import { + CURRENCY_CHANGE_EVENT, + formatCost, + getExchangeRates, +} from "@/utils/currency"; + +/** + * Provides the user's preferred display currency and a formatter that + * converts stored USD costs into it. Falls back to USD display when no + * exchange rate is available for the preferred currency. + * @returns {{currency: string, formatCost: (usd: number) => string}} + */ +export default function useCurrency() { + const [currency, setCurrency] = useState( + Appearance.get("preferredCurrency") || "USD" + ); + const [rates, setRates] = useState(null); + + useEffect(() => { + getExchangeRates().then((record) => setRates(record?.rates ?? null)); + }, []); + + useEffect(() => { + function handleCurrencyChange(e) { + if (!e?.detail?.currency) return; + setCurrency(e.detail.currency); + } + window.addEventListener(CURRENCY_CHANGE_EVENT, handleCurrencyChange); + return () => + window.removeEventListener(CURRENCY_CHANGE_EVENT, handleCurrencyChange); + }, []); + + const rate = currency === "USD" ? 1 : rates?.[currency]; + const format = useCallback( + (usd) => + formatCost(usd, rate ? { currency, rate } : { currency: "USD", rate: 1 }), + [currency, rate] + ); + + return { currency, formatCost: format }; +} diff --git a/frontend/src/locales/en/common.js b/frontend/src/locales/en/common.js index d2f4a30c553..14c59804610 100644 --- a/frontend/src/locales/en/common.js +++ b/frontend/src/locales/en/common.js @@ -838,6 +838,11 @@ const TRANSLATIONS = { description: "Select the preferred language to render AnythingLLM's UI in - when translations are available.", }, + "preferred-currency": { + title: "Preferred Currency", + description: + "Currency used to display LLM usage costs. Costs are always recorded in USD and converted for display only.", + }, logo: { title: "Brand Logo", description: "Upload your custom logo to showcase on all pages.", @@ -1877,6 +1882,7 @@ const TRANSLATIONS = { metrics: { promptTokens: "Prompt tokens:", completionTokens: "Completion tokens:", + cost: "Cost:", }, }, toolCall: { diff --git a/frontend/src/models/appearance.js b/frontend/src/models/appearance.js index 1dce2e229cc..1384e39b3aa 100644 --- a/frontend/src/models/appearance.js +++ b/frontend/src/models/appearance.js @@ -7,7 +7,8 @@ import { safeJsonParse } from "@/utils/request"; * 'autoPlayAssistantTtsResponse' | * 'enableSpellCheck' | * 'renderHTML' | - * 'disableAutoScroll' + * 'disableAutoScroll' | + * 'preferredCurrency' * } AvailableSettings - The supported settings for the appearance model. */ @@ -19,6 +20,7 @@ const Appearance = { enableSpellCheck: true, renderHTML: false, disableAutoScroll: false, + preferredCurrency: "USD", }, /** diff --git a/frontend/src/pages/GeneralSettings/ScheduledJobs/RunDetailPage.jsx b/frontend/src/pages/GeneralSettings/ScheduledJobs/RunDetailPage.jsx index 6ae29835811..d0bfa5fb3bb 100644 --- a/frontend/src/pages/GeneralSettings/ScheduledJobs/RunDetailPage.jsx +++ b/frontend/src/pages/GeneralSettings/ScheduledJobs/RunDetailPage.jsx @@ -13,6 +13,7 @@ import { } from "@phosphor-icons/react"; import ScheduledJobs from "@/models/scheduledJobs"; import usePolling from "@/hooks/usePolling"; +import useCurrency from "@/hooks/useCurrency"; import showToast from "@/utils/toast"; import paths from "@/utils/paths"; import renderMarkdown from "@/utils/chat/markdown"; @@ -391,6 +392,7 @@ function FinalResponseSection({ t, result }) { } function MetricsSection({ t, metrics }) { + const { formatCost } = useCurrency(); if (!metrics || Object.keys(metrics).length === 0) return null; // Todo: there is a bug where if you create a job that has no tools, we wont get any metrics @@ -428,6 +430,14 @@ function MetricsSection({ t, metrics }) { )} + {typeof metrics.totalCost === "number" && ( + + {t("scheduledJobs.runDetail.metrics.cost")}{" "} + + {formatCost(metrics.totalCost)} + + + )} ); diff --git a/frontend/src/pages/GeneralSettings/Settings/Interface/index.jsx b/frontend/src/pages/GeneralSettings/Settings/Interface/index.jsx index 91fe30013b7..8f1773576fc 100644 --- a/frontend/src/pages/GeneralSettings/Settings/Interface/index.jsx +++ b/frontend/src/pages/GeneralSettings/Settings/Interface/index.jsx @@ -1,6 +1,7 @@ import Sidebar from "@/components/SettingsSidebar"; import { isMobile } from "react-device-detect"; import { useTranslation } from "react-i18next"; +import CurrencyPreference from "../components/CurrencyPreference"; import LanguagePreference from "../components/LanguagePreference"; import ThemePreference from "../components/ThemePreference"; @@ -27,6 +28,7 @@ export default function InterfaceSettings() { + diff --git a/frontend/src/pages/GeneralSettings/Settings/components/CurrencyPreference/index.jsx b/frontend/src/pages/GeneralSettings/Settings/components/CurrencyPreference/index.jsx new file mode 100644 index 00000000000..8f48f994fe6 --- /dev/null +++ b/frontend/src/pages/GeneralSettings/Settings/components/CurrencyPreference/index.jsx @@ -0,0 +1,45 @@ +import { useTranslation } from "react-i18next"; +import Appearance from "@/models/appearance"; +import { + CURRENCY_CHANGE_EVENT, + SUPPORTED_CURRENCIES, + currencyName, +} from "@/utils/currency"; + +export default function CurrencyPreference() { + const { t, i18n } = useTranslation(); + + function changeCurrency(currency) { + Appearance.set("preferredCurrency", currency); + window.dispatchEvent( + new CustomEvent(CURRENCY_CHANGE_EVENT, { detail: { currency } }) + ); + } + + return ( +
+

+ {t("customization.items.preferred-currency.title")} +

+

+ {t("customization.items.preferred-currency.description")} +

+
+ +
+
+ ); +} diff --git a/frontend/src/utils/currency.js b/frontend/src/utils/currency.js new file mode 100644 index 00000000000..4c74f699c75 --- /dev/null +++ b/frontend/src/utils/currency.js @@ -0,0 +1,110 @@ +import { safeJsonParse } from "@/utils/request"; + +const FX_CACHE_KEY = "anythingllm_fx_rates"; +const FX_TTL_MS = 24 * 60 * 60 * 1000; // 24 hours +const FX_RATES_URL = "https://api.frankfurter.dev/v1/latest?base=USD"; +export const CURRENCY_CHANGE_EVENT = "anythingllm_currency_change"; + +/** + * Currencies the Frankfurter API can convert USD into. Kept as a static list + * (rather than Intl.supportedValuesOf) so the picker never offers a currency + * we cannot actually convert to. + */ +export const SUPPORTED_CURRENCIES = [ + "USD", + "AUD", + "BGN", + "BRL", + "CAD", + "CHF", + "CNY", + "CZK", + "DKK", + "EUR", + "GBP", + "HKD", + "HUF", + "IDR", + "ILS", + "INR", + "ISK", + "JPY", + "KRW", + "MXN", + "MYR", + "NOK", + "NZD", + "PHP", + "PLN", + "RON", + "SEK", + "SGD", + "THB", + "TRY", + "ZAR", +]; + +/** + * Returns the display name of a currency code in the user's language, + * falling back to the code itself. + * @param {string} code - ISO 4217 currency code (eg: "EUR") + * @param {string} [locale] - BCP 47 locale tag (eg: "en") + * @returns {string} + */ +export function currencyName(code, locale = undefined) { + try { + return new Intl.DisplayNames(locale, { type: "currency" }).of(code) ?? code; + } catch { + return code; + } +} + +/** + * Fetches the latest USD-based exchange rates, cached in localStorage for + * 24 hours. Returns a stale cache when the API is unreachable, or null when + * no rates are available at all (callers should then display USD). + * Never throws. + * @returns {Promise<{fetchedAt: number, rates: Record}|null>} + */ +export async function getExchangeRates() { + const cached = safeJsonParse(window.localStorage.getItem(FX_CACHE_KEY), null); + if (cached?.fetchedAt && Date.now() - cached.fetchedAt < FX_TTL_MS) + return cached; + + try { + const res = await fetch(FX_RATES_URL); + if (!res.ok) throw new Error(`Bad response: ${res.status}`); + const { rates } = await res.json(); + if (!rates || typeof rates !== "object") + throw new Error("Malformed rates response"); + const record = { fetchedAt: Date.now(), rates }; + window.localStorage.setItem(FX_CACHE_KEY, JSON.stringify(record)); + return record; + } catch { + return cached ?? null; + } +} + +/** + * Formats a stored USD cost into the given display currency. + * Precision scales down with the value so tiny per-message costs stay legible: + * >= 1 shows 2 decimals, >= 0.01 shows up to 4, anything smaller up to 6. + * @param {number} usd - the cost in USD + * @param {{currency?: string, rate?: number}} [options] - display currency and its units-per-USD rate + * @returns {string} + */ +export function formatCost(usd, { currency = "USD", rate = 1 } = {}) { + if (typeof usd !== "number" || !isFinite(usd)) return ""; + const value = usd * (rate || 1); + const maxDigits = value >= 1 ? 2 : value >= 0.01 ? 4 : value > 0 ? 6 : 2; + try { + return new Intl.NumberFormat(undefined, { + style: "currency", + currency, + minimumFractionDigits: 2, + maximumFractionDigits: maxDigits, + }).format(value); + } catch { + return `$${value.toFixed(maxDigits)}`; + } +} diff --git a/frontend/src/utils/numbers.js b/frontend/src/utils/numbers.js index b3c13222b5b..bb019ecb567 100644 --- a/frontend/src/utils/numbers.js +++ b/frontend/src/utils/numbers.js @@ -8,13 +8,6 @@ export function nFormatter(input) { return Formatter.format(input); } -export function dollarFormat(input) { - return new Intl.NumberFormat("en-us", { - style: "currency", - currency: "USD", - }).format(input); -} - export function toPercentString(input = null, decimals = 0) { if (isNaN(input) || input === null) return ""; const percentage = Math.round(input * 100); diff --git a/server/__tests__/utils/agents/aibitat/providers/ai-provider.test.js b/server/__tests__/utils/agents/aibitat/providers/ai-provider.test.js index b3b93e35131..7a4fc55acc6 100644 --- a/server/__tests__/utils/agents/aibitat/providers/ai-provider.test.js +++ b/server/__tests__/utils/agents/aibitat/providers/ai-provider.test.js @@ -1,6 +1,7 @@ const Provider = require("../../../../../utils/agents/aibitat/providers/ai-provider.js"); const UnTooled = require("../../../../../utils/agents/aibitat/providers/helpers/untooled.js"); const InheritMultiple = require("../../../../../utils/agents/aibitat/providers/helpers/classes.js"); +const { MODEL_PRICING } = require("../../../../../utils/helpers/modelPricing"); class TestProvider extends Provider { model = "test-model"; @@ -147,3 +148,70 @@ describe("Provider usage tracking", () => { expect(providerA.getCumulativeUsage().total_tokens).toBe(0); }); }); + +describe("Provider cost accumulation", () => { + afterEach(() => jest.restoreAllMocks()); + + test("cost is priced per-call and summed even when the model changes mid-run", () => { + // Return a different rate per model so a sum over per-call breakdowns is + // distinguishable from pricing the summed totals at the final model's rate. + jest + .spyOn(MODEL_PRICING, "getCostBreakdown") + .mockImplementation((_slug, model, { prompt_tokens }) => { + const rate = model === "expensive-model" ? 10 : 1; + const inputCost = (prompt_tokens / 1_000_000) * rate; + return { inputCost, outputCost: 0, totalCost: inputCost }; + }); + + const provider = new TestProvider(); + provider.providerSlug = "openai"; + + provider.resetUsage(); + provider.recordUsage({ prompt_tokens: 1_000_000, completion_tokens: 10 }); + + provider.model = "expensive-model"; + provider.resetUsage(); + provider.recordUsage({ prompt_tokens: 1_000_000, completion_tokens: 10 }); + + expect(provider.getUsage().totalCost).toBe(10); + const totals = provider.getCumulativeUsage(); + expect(totals.inputCost).toBe(11); + expect(totals.outputCost).toBe(0); + expect(totals.totalCost).toBe(11); + }); + + test("cost fields stay absent when pricing is unknown", () => { + jest.spyOn(MODEL_PRICING, "getCostBreakdown").mockReturnValue(null); + + const provider = new TestProvider(); + provider.resetUsage(); + provider.recordUsage({ prompt_tokens: 100, completion_tokens: 10 }); + + expect(provider.getUsage()).not.toHaveProperty("totalCost"); + expect(provider.getCumulativeUsage()).not.toHaveProperty("totalCost"); + }); + + test("a partially priceable run sums only the priced calls", () => { + jest + .spyOn(MODEL_PRICING, "getCostBreakdown") + .mockImplementation((_slug, model) => + model === "unknown-model" + ? null + : { inputCost: 1, outputCost: 2, totalCost: 3 } + ); + + const provider = new TestProvider(); + provider.providerSlug = "openai"; + + provider.resetUsage(); + provider.recordUsage({ prompt_tokens: 100, completion_tokens: 10 }); + + provider.model = "unknown-model"; + provider.resetUsage(); + provider.recordUsage({ prompt_tokens: 100, completion_tokens: 10 }); + + // The unpriced call contributes nothing, but the priced call's cost survives. + expect(provider.getUsage()).not.toHaveProperty("totalCost"); + expect(provider.getCumulativeUsage().totalCost).toBe(3); + }); +}); diff --git a/server/__tests__/utils/helpers/modelPricing/fixtures/api.json b/server/__tests__/utils/helpers/modelPricing/fixtures/api.json new file mode 100644 index 00000000000..d83658690ab --- /dev/null +++ b/server/__tests__/utils/helpers/modelPricing/fixtures/api.json @@ -0,0 +1,96 @@ +{ + "openai": { + "id": "openai", + "name": "OpenAI", + "models": { + "gpt-4o": { + "id": "gpt-4o", + "name": "GPT-4o", + "cost": { "input": 2.5, "output": 10 } + }, + "gpt-4o-mini": { + "id": "gpt-4o-mini", + "name": "GPT-4o mini", + "cost": { "input": 0.15, "output": 0.6, "cache_read": 0.075 } + }, + "gpt-oss-free": { + "id": "gpt-oss-free", + "name": "Free model", + "cost": { "input": 0, "output": 0 } + }, + "gpt-subscription-only": { + "id": "gpt-subscription-only", + "name": "No published pricing" + } + } + }, + "google": { + "id": "google", + "name": "Google", + "models": { + "gemini-tiered": { + "id": "gemini-tiered", + "name": "Gemini with tiers", + "cost": { + "input": 1.25, + "output": 10, + "tiers": [ + { + "input": 2.5, + "output": 15, + "tier": { "type": "context", "size": 200000 } + } + ], + "context_over_200k": { "input": 2.5, "output": 15 } + } + }, + "gemini-legacy-200k": { + "id": "gemini-legacy-200k", + "name": "Gemini with legacy long-context pricing", + "cost": { + "input": 1, + "output": 5, + "context_over_200k": { "input": 2, "output": 10 } + } + } + } + }, + "amazon-bedrock": { + "id": "amazon-bedrock", + "name": "Amazon Bedrock", + "models": { + "anthropic.claude-sonnet-4-5-20250929-v1:0": { + "id": "anthropic.claude-sonnet-4-5-20250929-v1:0", + "name": "Claude Sonnet 4.5", + "cost": { "input": 3, "output": 15 } + }, + "eu.anthropic.claude-sonnet-4-5-20250929-v1:0": { + "id": "eu.anthropic.claude-sonnet-4-5-20250929-v1:0", + "name": "Claude Sonnet 4.5 (EU)", + "cost": { "input": 3.3, "output": 16.5 } + } + } + }, + "openrouter": { + "id": "openrouter", + "name": "OpenRouter", + "models": { + "anthropic/claude-sonnet-4.5": { + "id": "anthropic/claude-sonnet-4.5", + "name": "Claude Sonnet 4.5", + "cost": { "input": 3, "output": 15 } + } + } + }, + "ollama-cloud": { + "id": "ollama-cloud", + "name": "Ollama Cloud", + "models": { + "some-model": { + "id": "some-model", + "name": "Null-cost model", + "cost": null + } + } + } +} diff --git a/server/__tests__/utils/helpers/modelPricing/index.test.js b/server/__tests__/utils/helpers/modelPricing/index.test.js new file mode 100644 index 00000000000..32e1db5ac34 --- /dev/null +++ b/server/__tests__/utils/helpers/modelPricing/index.test.js @@ -0,0 +1,361 @@ +const fs = require("fs"); +const os = require("os"); +const path = require("path"); + +process.env.NODE_ENV = "test"; + +const FIXTURE = JSON.parse( + fs.readFileSync(path.resolve(__dirname, "fixtures/api.json"), "utf8") +); + +/** + * The module memoizes a singleton at require time, so every test builds its + * own instance against a fresh temp STORAGE_DIR and a mocked global fetch. + */ +function freshInstance() { + const { ModelPricing } = require("../../../../utils/helpers/modelPricing"); + ModelPricing.instance = null; + return new ModelPricing(); +} + +function mockFetchWith(response) { + global.fetch = jest.fn().mockImplementation(async () => response); +} + +function okResponse(data, { etag = null } = {}) { + return { + status: 200, + headers: { get: (key) => (key === "etag" ? etag : null) }, + json: async () => data, + }; +} + +/** Waits for the constructor's fire-and-forget refresh to settle. */ +async function flushRefresh() { + await new Promise((resolve) => setTimeout(resolve, 25)); +} + +describe("ModelPricing", () => { + let tempDir; + const originalFetch = global.fetch; + + beforeEach(() => { + jest.resetModules(); + tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "model-pricing-test-")); + process.env.STORAGE_DIR = tempDir; + }); + + afterEach(() => { + global.fetch = originalFetch; + fs.rmSync(tempDir, { recursive: true, force: true }); + }); + + describe("cache mechanics", () => { + it("fetches the remote pricing data and writes the disk cache", async () => { + mockFetchWith(okResponse(FIXTURE, { etag: '"abc123"' })); + const pricing = freshInstance(); + await flushRefresh(); + + const cacheDir = path.join(tempDir, "models", "pricing"); + expect(fs.existsSync(path.join(cacheDir, "model-pricing.json"))).toBe( + true + ); + expect(fs.existsSync(path.join(cacheDir, ".cached_at"))).toBe(true); + expect(fs.readFileSync(path.join(cacheDir, ".etag"), "utf8")).toBe( + '"abc123"' + ); + expect(pricing.isCacheStale).toBe(false); + + // The disk cache is slimmed to cost objects only, dropping models + // with absent or null cost. + const cached = JSON.parse( + fs.readFileSync(path.join(cacheDir, "model-pricing.json"), "utf8") + ); + expect(cached.openai["gpt-4o"]).toEqual({ input: 2.5, output: 10 }); + expect(cached.openai["gpt-subscription-only"]).toBeUndefined(); + expect(cached["ollama-cloud"]).toBeUndefined(); + }); + + it("serves pricing from the disk cache without refetching when fresh", async () => { + mockFetchWith(okResponse(FIXTURE)); + freshInstance(); + await flushRefresh(); + + jest.resetModules(); + const fetchSpy = jest.fn(); + global.fetch = fetchSpy; + const pricing = freshInstance(); + await flushRefresh(); + + expect(fetchSpy).not.toHaveBeenCalled(); + expect( + pricing.getCostBreakdown("openai", "gpt-4o", { + prompt_tokens: 1_000_000, + completion_tokens: 0, + }) + ).toEqual({ inputCost: 2.5, outputCost: 0, totalCost: 2.5 }); + }); + + it("only bumps the cache expiry on a 304 response", async () => { + mockFetchWith(okResponse(FIXTURE, { etag: '"abc123"' })); + freshInstance(); + await flushRefresh(); + + // Age the cache past expiry so the next boot refreshes, then 304 it. + const cacheDir = path.join(tempDir, "models", "pricing"); + fs.writeFileSync(path.join(cacheDir, ".cached_at"), "0"); + jest.resetModules(); + mockFetchWith({ + status: 304, + headers: { get: () => null }, + json: async () => { + throw new Error("304 has no body"); + }, + }); + const pricing = freshInstance(); + await flushRefresh(); + + expect(global.fetch).toHaveBeenCalledWith( + expect.any(String), + expect.objectContaining({ + headers: { "If-None-Match": '"abc123"' }, + }) + ); + expect(pricing.isCacheStale).toBe(false); + expect( + pricing.getCostBreakdown("openai", "gpt-4o", { + prompt_tokens: 1_000_000, + }) + ).toEqual({ inputCost: 2.5, outputCost: 0, totalCost: 2.5 }); + }); + + it("keeps serving the stale disk cache when the remote fetch fails", async () => { + mockFetchWith(okResponse(FIXTURE)); + freshInstance(); + await flushRefresh(); + + const cacheDir = path.join(tempDir, "models", "pricing"); + fs.writeFileSync(path.join(cacheDir, ".cached_at"), "0"); + jest.resetModules(); + global.fetch = jest.fn().mockRejectedValue(new Error("offline")); + const pricing = freshInstance(); + await flushRefresh(); + + expect( + pricing.getCostBreakdown("openai", "gpt-4o", { + prompt_tokens: 1_000_000, + }) + ).toEqual({ inputCost: 2.5, outputCost: 0, totalCost: 2.5 }); + }); + + it("falls back to the bundled snapshot when offline with no disk cache", async () => { + // Jest's VM cannot execute the real dynamic import of the ESM-only + // snapshot package, so inject a fake importer. The real import is + // exercised by the server at runtime. + global.fetch = jest.fn().mockRejectedValue(new Error("offline")); + const { ModelPricing } = require("../../../../utils/helpers/modelPricing"); + const originalImporter = ModelPricing.importSnapshot; + ModelPricing.importSnapshot = jest + .fn() + .mockResolvedValue({ providers: FIXTURE }); + + try { + ModelPricing.instance = null; + const pricing = new ModelPricing(); + await flushRefresh(); + + expect(ModelPricing.importSnapshot).toHaveBeenCalled(); + expect( + pricing.getCostBreakdown("openai", "gpt-4o", { + prompt_tokens: 1_000_000, + completion_tokens: 0, + }) + ).toEqual({ inputCost: 2.5, outputCost: 0, totalCost: 2.5 }); + + // No .cached_at is written so the next boot retries the remote. + expect( + fs.existsSync(path.join(tempDir, "models", "pricing", ".cached_at")) + ).toBe(false); + } finally { + ModelPricing.importSnapshot = originalImporter; + } + }); + }); + + describe("getCostBreakdown", () => { + let pricing; + + beforeEach(async () => { + mockFetchWith(okResponse(FIXTURE)); + pricing = freshInstance(); + await flushRefresh(); + }); + + it("computes exact input/output/total costs", () => { + expect( + pricing.getCostBreakdown("openai", "gpt-4o-mini", { + prompt_tokens: 1000, + completion_tokens: 500, + }) + ).toEqual({ + inputCost: (1000 / 1_000_000) * 0.15, + outputCost: (500 / 1_000_000) * 0.6, + totalCost: (1000 / 1_000_000) * 0.15 + (500 / 1_000_000) * 0.6, + }); + }); + + it("returns zeros for local/self-hosted providers without a lookup", () => { + for (const slug of ["ollama", "lmstudio", "koboldcpp"]) { + expect( + pricing.getCostBreakdown(slug, "whatever-model", { + prompt_tokens: 1000, + completion_tokens: 1000, + }) + ).toEqual({ inputCost: 0, outputCost: 0, totalCost: 0 }); + } + }); + + it("returns zeros for a model with published zero pricing", () => { + expect( + pricing.getCostBreakdown("openai", "gpt-oss-free", { + prompt_tokens: 1000, + completion_tokens: 1000, + }) + ).toEqual({ inputCost: 0, outputCost: 0, totalCost: 0 }); + }); + + it("returns null for unknown pricing", () => { + // Unmapped provider slug + expect(pricing.getCostBreakdown("generic-openai", "gpt-4o")).toBeNull(); + // Unknown model on a known provider + expect(pricing.getCostBreakdown("openai", "not-a-model")).toBeNull(); + // Model whose upstream cost is null (slimmed away) + expect(pricing.getCostBreakdown("openrouter", "some-model")).toBeNull(); + // Model with no published pricing (slimmed away) + expect( + pricing.getCostBreakdown("openai", "gpt-subscription-only") + ).toBeNull(); + // No provider at all + expect(pricing.getCostBreakdown(null, "gpt-4o")).toBeNull(); + }); + + it("matches model ids case-insensitively", () => { + expect( + pricing.getCostBreakdown("openai", "GPT-4o", { + prompt_tokens: 1_000_000, + }) + ).toEqual({ inputCost: 2.5, outputCost: 0, totalCost: 2.5 }); + }); + + it("normalizes bedrock region prefixes and version suffixes", () => { + // Region-prefixed user config matches the unprefixed dataset key, + // never the differently-priced eu. variant. + expect( + pricing.getCostBreakdown( + "bedrock", + "us.anthropic.claude-sonnet-4-5-20250929-v1:0", + { prompt_tokens: 1_000_000 } + ) + ).toEqual({ inputCost: 3, outputCost: 0, totalCost: 3 }); + expect( + pricing.getCostBreakdown( + "bedrock", + "anthropic.claude-sonnet-4-5-20250929", + { prompt_tokens: 1_000_000 } + ) + ).toEqual({ inputCost: 3, outputCost: 0, totalCost: 3 }); + }); + + it("applies long-context tier pricing above the tier threshold", () => { + expect( + pricing.getCostBreakdown("gemini", "gemini-tiered", { + prompt_tokens: 100_000, + completion_tokens: 1000, + }) + ).toEqual({ + inputCost: (100_000 / 1_000_000) * 1.25, + outputCost: (1000 / 1_000_000) * 10, + totalCost: (100_000 / 1_000_000) * 1.25 + (1000 / 1_000_000) * 10, + }); + expect( + pricing.getCostBreakdown("gemini", "gemini-tiered", { + prompt_tokens: 300_000, + completion_tokens: 1000, + }) + ).toEqual({ + inputCost: (300_000 / 1_000_000) * 2.5, + outputCost: (1000 / 1_000_000) * 15, + totalCost: (300_000 / 1_000_000) * 2.5 + (1000 / 1_000_000) * 15, + }); + }); + + it("applies legacy context_over_200k pricing when no tiers exist", () => { + expect( + pricing.getCostBreakdown("gemini", "gemini-legacy-200k", { + prompt_tokens: 300_000, + completion_tokens: 0, + }) + ).toEqual({ inputCost: (300_000 / 1_000_000) * 2, outputCost: 0, totalCost: (300_000 / 1_000_000) * 2 }); + }); + + it("resolves openrouter vendor/model ids directly", () => { + expect( + pricing.getCostBreakdown("openrouter", "anthropic/claude-sonnet-4.5", { + prompt_tokens: 1_000_000, + completion_tokens: 0, + }) + ).toEqual({ inputCost: 3, outputCost: 0, totalCost: 3 }); + }); + }); + + describe("addCostToMetrics", () => { + beforeEach(async () => { + mockFetchWith(okResponse(FIXTURE)); + freshInstance(); + await flushRefresh(); + }); + + it("decorates metrics when pricing is known", () => { + const { + addCostToMetrics, + } = require("../../../../utils/helpers/modelPricing"); + const metrics = { + prompt_tokens: 1_000_000, + completion_tokens: 0, + model: "gpt-4o", + }; + expect(addCostToMetrics(metrics, { provider: "openai" })).toEqual({ + ...metrics, + inputCost: 2.5, + outputCost: 0, + totalCost: 2.5, + }); + }); + + it("prefers an explicitly passed model over metrics.model", () => { + const { + addCostToMetrics, + } = require("../../../../utils/helpers/modelPricing"); + const decorated = addCostToMetrics( + { prompt_tokens: 1_000_000, completion_tokens: 0, model: "gpt-4o" }, + { provider: "openai", model: "gpt-4o-mini" } + ); + expect(decorated.inputCost).toBe(0.15); + }); + + it("returns metrics unchanged when pricing is unknown", () => { + const { + addCostToMetrics, + } = require("../../../../utils/helpers/modelPricing"); + const metrics = { + prompt_tokens: 100, + completion_tokens: 10, + model: "some-local-model", + }; + expect( + addCostToMetrics(metrics, { provider: "generic-openai" }) + ).toEqual(metrics); + expect(addCostToMetrics({}, { provider: "openai" })).toEqual({}); + }); + }); +}); diff --git a/server/index.js b/server/index.js index db2b6d1d6aa..66e2ef965f3 100644 --- a/server/index.js +++ b/server/index.js @@ -4,6 +4,7 @@ process.env.NODE_ENV === "development" require("./utils/logger")(); require("./utils/boot/patchSdkTimeouts")(); +require("./utils/helpers/modelPricing"); // boots the model pricing cache refresh const express = require("express"); const bodyParser = require("body-parser"); const cors = require("cors"); diff --git a/server/package.json b/server/package.json index db2cfe3a544..7a772e93792 100644 --- a/server/package.json +++ b/server/package.json @@ -34,6 +34,7 @@ "@mintplex-labs/express-ws": "^5.0.7", "@mintplex-labs/mdpdf": "^0.1.9", "@modelcontextprotocol/sdk": "^1.24.3", + "@opencode-ai/models": "^0.0.34", "@pinecone-database/pinecone": "^2.0.1", "@prisma/client": "5.3.1", "@qdrant/js-client-rest": "^1.9.0", diff --git a/server/storage/models/.gitignore b/server/storage/models/.gitignore index 2e6b5c3e9b6..19439d3bd17 100644 --- a/server/storage/models/.gitignore +++ b/server/storage/models/.gitignore @@ -10,6 +10,7 @@ togetherAi tesseract ppio context-windows/* +pricing/* MintplexLabs cometapi fireworks diff --git a/server/utils/agents/aibitat/index.js b/server/utils/agents/aibitat/index.js index b73fe748730..3885d0db564 100644 --- a/server/utils/agents/aibitat/index.js +++ b/server/utils/agents/aibitat/index.js @@ -1417,6 +1417,10 @@ https://docs.anythingllm.com/agent/intelligent-tool-selection */ getProviderForConfig(config) { const provider = this.#buildProviderForConfig(config); + // Record the slug the instance was built from so usage metrics can be + // priced - pre-built instances (config.provider as an object) keep theirs. + if (typeof config?.provider === "string") + provider.providerSlug ??= config.provider; provider.attachAbortSignal?.(this.abortController.signal); return provider; } diff --git a/server/utils/agents/aibitat/providers/ai-provider.js b/server/utils/agents/aibitat/providers/ai-provider.js index 9f4dd99d4c7..39e30df910e 100644 --- a/server/utils/agents/aibitat/providers/ai-provider.js +++ b/server/utils/agents/aibitat/providers/ai-provider.js @@ -16,6 +16,7 @@ const { ChatAnthropic } = require("@langchain/anthropic"); const { ChatOllama } = require("@langchain/community/chat_models/ollama"); const { toValidNumber, safeJsonParse } = require("../../../http"); const { getLLMProviderClass } = require("../../../helpers"); +const { MODEL_PRICING } = require("../../../helpers/modelPricing"); const { parseLMStudioBasePath } = require("../../../AiProviders/lmStudio"); const { parseDockerModelRunnerEndpoint, @@ -39,6 +40,9 @@ const { bindAbortSignal } = require("../../../helpers/abortSignals"); * @property {string|null} model - Model name * @property {string|null} provider - Provider class name * @property {Date|null} timestamp - Timestamp of the completion + * @property {number} [inputCost] - USD cost of the prompt tokens. Absent when pricing is unknown. + * @property {number} [outputCost] - USD cost of the completion tokens. Absent when pricing is unknown. + * @property {number} [totalCost] - USD sum of input and output costs. Absent when pricing is unknown. */ /** @@ -119,6 +123,15 @@ class Provider { */ providerTag = null; + /** + * The AnythingLLM provider slug this instance was built for (eg: "openai", + * "anthropic") - set by AIbitat when the provider is instantiated. Unlike + * `providerTag` or `constructor.name`, this matches the slugs used for + * model pricing lookups. Null when the origin of the instance is unknown. + * @type {string|null} + */ + providerSlug = null; + /** * Abort signal for the active agent session, attached by AIbitat. Bound to the * SDK client so every request this provider makes is cancelled when the session @@ -681,6 +694,14 @@ class Provider { duration = 0, } = {}) { const timestamp = new Date(); + // Cost is priced per-call (not derived from the summed totals) so the + // accumulated cost stays correct even if the model changes mid-run. + // A null breakdown (unknown pricing) leaves the cost fields absent. + const cost = MODEL_PRICING.getCostBreakdown(this.providerSlug, this.model, { + prompt_tokens, + completion_tokens, + }); + this.lastUsage = { prompt_tokens, completion_tokens, @@ -691,6 +712,7 @@ class Provider { model: this.model, provider: this.constructor.name, timestamp, + ...(cost ?? {}), }; const totals = this.cumulativeUsage; @@ -705,6 +727,11 @@ class Provider { totals.model = this.model; totals.provider = this.constructor.name; totals.timestamp = timestamp; + if (cost) { + totals.inputCost = (totals.inputCost ?? 0) + cost.inputCost; + totals.outputCost = (totals.outputCost ?? 0) + cost.outputCost; + totals.totalCost = (totals.totalCost ?? 0) + cost.totalCost; + } } /** diff --git a/server/utils/chats/apiChatHandler.js b/server/utils/chats/apiChatHandler.js index 805c1306d23..45f45fa3884 100644 --- a/server/utils/chats/apiChatHandler.js +++ b/server/utils/chats/apiChatHandler.js @@ -2,6 +2,7 @@ const { v4: uuidv4 } = require("uuid"); const { DocumentManager } = require("../DocumentManager"); const { WorkspaceChats } = require("../../models/workspaceChats"); const { getVectorDbClass, resolveProviderConnector } = require("../helpers"); +const { addChatCostToMetrics } = require("../helpers/modelPricing"); const { writeResponseChunk } = require("../helpers/chat/responses"); const { abortConnectorOnClientDisconnect } = require("../helpers/abortSignals"); const { @@ -224,14 +225,15 @@ async function chatSync({ }); } - const { connector: LLMConnector } = await resolveProviderConnector({ - workspace, - prompt: message, - user, - thread, - attachments, - apiSessionId: sessionId, - }); + const { connector: LLMConnector, routingMetadata } = + await resolveProviderConnector({ + workspace, + prompt: message, + user, + thread, + attachments, + apiSessionId: sessionId, + }); const VectorDb = getVectorDbClass(); const messageLimit = workspace?.openAiHistory || 20; @@ -418,11 +420,16 @@ async function chatSync({ ); // Send the text completion. - const { textResponse, metrics: performanceMetrics } = + const { textResponse, metrics: completionMetrics } = await LLMConnector.getChatCompletion(messages, { temperature: workspace?.openAiTemp ?? LLMConnector.defaultTemp, user: user, }); + const performanceMetrics = addChatCostToMetrics(completionMetrics, { + routingMetadata, + workspace, + connector: LLMConnector, + }); if (!textResponse) { return { @@ -591,14 +598,15 @@ async function streamChat({ }); } - const { connector: LLMConnector } = await resolveProviderConnector({ - workspace, - prompt: message, - user, - thread, - attachments, - apiSessionId: sessionId, - }); + const { connector: LLMConnector, routingMetadata } = + await resolveProviderConnector({ + workspace, + prompt: message, + user, + thread, + attachments, + apiSessionId: sessionId, + }); // A disconnected client (aborted request, closed connection) should stop the // provider generating too, not just stop us reading the response. @@ -811,7 +819,11 @@ async function streamChat({ user: user, }); completeText = textResponse; - metrics = performanceMetrics; + metrics = addChatCostToMetrics(performanceMetrics, { + routingMetadata, + workspace, + connector: LLMConnector, + }); writeResponseChunk(response, { uuid, sources, @@ -827,7 +839,11 @@ async function streamChat({ user: user, }); completeText = await LLMConnector.handleStream(response, stream, { uuid }); - metrics = stream.metrics; + metrics = addChatCostToMetrics(stream.metrics, { + routingMetadata, + workspace, + connector: LLMConnector, + }); } if (completeText?.length > 0) { diff --git a/server/utils/chats/embed.js b/server/utils/chats/embed.js index 8476311587a..dc023d81f5c 100644 --- a/server/utils/chats/embed.js +++ b/server/utils/chats/embed.js @@ -1,5 +1,6 @@ const { v4: uuidv4 } = require("uuid"); const { getVectorDbClass, resolveProviderConnector } = require("../helpers"); +const { addChatCostToMetrics } = require("../helpers/modelPricing"); const { chatPrompt, sourceIdentifier } = require("./index"); const { EmbedChats } = require("../../models/embedChats"); const { @@ -34,6 +35,7 @@ async function streamChatWithForEmbed( const uuid = uuidv4(); const { connector: LLMConnector, + routingMetadata, prefetchedContext, error: routerError, } = await resolveLLMConnectorForEmbed({ @@ -195,7 +197,11 @@ async function streamChatWithForEmbed( temperature: embed.workspace?.openAiTemp ?? LLMConnector.defaultTemp, }); completeText = textResponse; - metrics = performanceMetrics; + metrics = addChatCostToMetrics(performanceMetrics, { + routingMetadata, + workspace: embed.workspace, + connector: LLMConnector, + }); writeResponseChunk(response, { uuid, sources: [], @@ -212,7 +218,11 @@ async function streamChatWithForEmbed( uuid, sources: [], }); - metrics = stream.metrics; + metrics = addChatCostToMetrics(stream.metrics, { + routingMetadata, + workspace: embed.workspace, + connector: LLMConnector, + }); } await EmbedChats.new({ @@ -273,16 +283,18 @@ async function resolveLLMConnectorForEmbed({ include: true, }); - const { connector, prefetchedContext } = await resolveProviderConnector({ - workspace, - prompt: message, - chatHistoryOverride: embedHistory, - // +1 to include the current in-flight message to ensure routing rules are evaluated against the real total. - messageCountOverride: embedMessageCount + 1, - }); + const { connector, routingMetadata, prefetchedContext } = + await resolveProviderConnector({ + workspace, + prompt: message, + chatHistoryOverride: embedHistory, + // +1 to include the current in-flight message to ensure routing rules are evaluated against the real total. + messageCountOverride: embedMessageCount + 1, + }); return { connector, + routingMetadata, prefetchedContext: prefetchedContext ? { rawHistory: embedHistory.rawHistory, @@ -295,6 +307,7 @@ async function resolveLLMConnectorForEmbed({ } catch (routerError) { return { connector: null, + routingMetadata: null, prefetchedContext: null, error: `Model router error: ${routerError.message}`, }; diff --git a/server/utils/chats/openaiCompatible.js b/server/utils/chats/openaiCompatible.js index c6bf0e66ae9..a83811827a7 100644 --- a/server/utils/chats/openaiCompatible.js +++ b/server/utils/chats/openaiCompatible.js @@ -2,6 +2,7 @@ const { v4: uuidv4 } = require("uuid"); const { DocumentManager } = require("../DocumentManager"); const { WorkspaceChats } = require("../../models/workspaceChats"); const { getVectorDbClass, resolveProviderConnector } = require("../helpers"); +const { addChatCostToMetrics } = require("../helpers/modelPricing"); const { writeResponseChunk } = require("../helpers/chat/responses"); const { chatPrompt, sourceIdentifier } = require("./index"); const { abortConnectorOnClientDisconnect } = require("../helpers/abortSignals"); @@ -19,18 +20,19 @@ async function chatSync({ const uuid = uuidv4(); const chatMode = workspace?.chatMode ?? "automatic"; - const { connector: LLMConnector } = await resolveProviderConnector({ - workspace, - prompt, - attachments, - chatHistoryOverride: { - rawHistory: history, - chatHistory: history, - }, - // Do not +1 to this, since OAI message history ends in a user message - // and does not need to re-include an uncounted user message. - messageCountOverride: history.length, - }); + const { connector: LLMConnector, routingMetadata } = + await resolveProviderConnector({ + workspace, + prompt, + attachments, + chatHistoryOverride: { + rawHistory: history, + chatHistory: history, + }, + // Do not +1 to this, since OAI message history ends in a user message + // and does not need to re-include an uncounted user message. + messageCountOverride: history.length, + }); const VectorDb = getVectorDbClass(); const hasVectorizedSpace = await VectorDb.hasNamespace(workspace.slug); @@ -172,13 +174,16 @@ async function chatSync({ }); // Send the text completion. - const { textResponse, metrics } = await LLMConnector.getChatCompletion( - messages, - { + const { textResponse, metrics: completionMetrics } = + await LLMConnector.getChatCompletion(messages, { temperature: temperature ?? workspace?.openAiTemp ?? LLMConnector.defaultTemp, - } - ); + }); + const metrics = addChatCostToMetrics(completionMetrics, { + routingMetadata, + workspace, + connector: LLMConnector, + }); if (!textResponse) { return formatJSON( @@ -232,18 +237,19 @@ async function streamChat({ const uuid = uuidv4(); const chatMode = workspace?.chatMode ?? "automatic"; - const { connector: LLMConnector } = await resolveProviderConnector({ - workspace, - prompt, - attachments, - chatHistoryOverride: { - rawHistory: history, - chatHistory: history, - }, - // Do not +1 to this, since OAI message history ends in a user message - // and does not need to re-include an uncounted user message. - messageCountOverride: history.length, - }); + const { connector: LLMConnector, routingMetadata } = + await resolveProviderConnector({ + workspace, + prompt, + attachments, + chatHistoryOverride: { + rawHistory: history, + chatHistory: history, + }, + // Do not +1 to this, since OAI message history ends in a user message + // and does not need to re-include an uncounted user message. + messageCountOverride: history.length, + }); // A disconnected client (aborted request, closed connection) should stop the // provider generating too, not just stop us reading the response. @@ -453,6 +459,11 @@ async function streamChat({ sources, } ); + const metrics = addChatCostToMetrics(stream.metrics, { + routingMetadata, + workspace, + connector: LLMConnector, + }); if (completeText?.length > 0) { const { chat } = await WorkspaceChats.new({ @@ -462,7 +473,7 @@ async function streamChat({ text: completeText, sources, type: chatMode, - metrics: stream.metrics, + metrics, attachments, }, }); @@ -482,7 +493,7 @@ async function streamChat({ chunked: true, model: workspace.slug, finish_reason: "stop", - usage: stream.metrics, + usage: metrics, } ) ); @@ -503,7 +514,7 @@ async function streamChat({ chunked: true, model: workspace.slug, finish_reason: "stop", - usage: stream.metrics, + usage: metrics, } ) ); diff --git a/server/utils/chats/stream.js b/server/utils/chats/stream.js index 87e92cc6193..28253100f14 100644 --- a/server/utils/chats/stream.js +++ b/server/utils/chats/stream.js @@ -3,6 +3,7 @@ const { DocumentManager } = require("../DocumentManager"); const { WorkspaceChats } = require("../../models/workspaceChats"); const { WorkspaceParsedFiles } = require("../../models/workspaceParsedFiles"); const { getVectorDbClass, resolveProviderConnector } = require("../helpers"); +const { addChatCostToMetrics } = require("../helpers/modelPricing"); const { writeResponseChunk } = require("../helpers/chat/responses"); const { abortConnectorOnClientDisconnect } = require("../helpers/abortSignals"); const { grepAgents } = require("./agents"); @@ -291,7 +292,11 @@ async function streamChatWithWorkspace( }); completeText = textResponse; - metrics = performanceMetrics; + metrics = addChatCostToMetrics(performanceMetrics, { + routingMetadata, + workspace, + connector: LLMConnector, + }); writeResponseChunk(response, { uuid, sources, @@ -310,7 +315,11 @@ async function streamChatWithWorkspace( uuid, sources, }); - metrics = stream.metrics; + metrics = addChatCostToMetrics(stream.metrics, { + routingMetadata, + workspace, + connector: LLMConnector, + }); } if (completeText?.length > 0) { diff --git a/server/utils/helpers/modelPricing/index.js b/server/utils/helpers/modelPricing/index.js new file mode 100644 index 00000000000..cc2810a8b9b --- /dev/null +++ b/server/utils/helpers/modelPricing/index.js @@ -0,0 +1,415 @@ +const path = require("path"); +const fs = require("fs"); + +/** + * @typedef {Object} ModelCost - USD per 1,000,000 tokens (models.dev conventions) + * @property {number} input - cost per 1M prompt tokens + * @property {number} output - cost per 1M completion tokens + * @property {number} [cache_read] - cost per 1M cached prompt tokens read (falls back to `input`) + * @property {number} [cache_write] - cost per 1M prompt tokens written to cache (falls back to `input`) + * @property {number} [reasoning] - cost per 1M reasoning tokens (falls back to `output`) + * @property {Array} [tiers] - long-context pricing tiers + * @property {ModelCost} [context_over_200k] - legacy long-context pricing (prefer `tiers`) + */ + +/** + * @typedef {Object} CostBreakdown + * @property {number} inputCost - USD cost of the prompt tokens + * @property {number} outputCost - USD cost of the completion tokens + * @property {number} totalCost - USD sum of input and output costs + */ + +/** + * Providers that are local or self-hosted - inference is always free + * so we never consult the pricing data for them. + * @type {Set} + */ +const FREE_PROVIDERS = new Set([ + "ollama", + "lmstudio", + "localai", + "koboldcpp", + "textgenwebui", + "omlx", + "lemonade", + "docker-model-runner", +]); + +/** + * AnythingLLM provider slug -> models.dev provider id. + * Slugs absent from this map (and from FREE_PROVIDERS) resolve to "unknown cost" + * (eg: generic-openai, litellm, foundry, privatemode, nvidia-nim, apipie, + * cometapi, giteeai, ppio, sambanova - not present on models.dev as of Aug 2026). + * A missing/stale mapping degrades to "unknown", never a wrong price. + * @type {Record} + */ +const PROVIDER_ID_MAP = { + openai: "openai", + azure: "azure", // best-effort: only matches when deployment name === model id + anthropic: "anthropic", + gemini: "google", + togetherai: "togetherai", + fireworksai: "fireworks-ai", + mistral: "mistral", + perplexity: "perplexity", + openrouter: "openrouter", + novita: "novita-ai", + groq: "groq", + cohere: "cohere", + bedrock: "amazon-bedrock", + deepseek: "deepseek", + xai: "xai", + moonshotai: "moonshotai", + zai: "zai", + minimax: "minimax", + cerebras: "cerebras", +}; + +class ModelPricing { + static instance = null; + static expiryMs = 1000 * 60 * 60 * 24 * 3; // 3 days + static remoteUrl = "https://models.dev/api.json"; + + /** + * Loads the pricing snapshot bundled with `@opencode-ai/models`. The package + * is ESM-only so it must be dynamically imported - never require()'d. + * Static so tests can inject a fake importer (Jest's VM cannot execute + * dynamic import without --experimental-vm-modules). + * @returns {Promise<{providers: Record}>} + */ + static importSnapshot = () => import("@opencode-ai/models/snapshot"); + + cacheLocation = path.resolve( + process.env.STORAGE_DIR + ? path.resolve(process.env.STORAGE_DIR, "models", "pricing") + : path.resolve(__dirname, `../../../storage/models/pricing`) + ); + cacheFilePath = path.resolve(this.cacheLocation, "model-pricing.json"); + cacheFileExpiryPath = path.resolve(this.cacheLocation, ".cached_at"); + cacheEtagPath = path.resolve(this.cacheLocation, ".etag"); + + /** @type {Record>|null} - memoized pricing map keyed by models.dev provider id */ + #pricing = null; + /** @type {Record>} - lazy per-provider lowercased model id -> real model id */ + #lowercaseIndexes = {}; + /** @type {Record>} - lazy per-provider normalized model id -> real model id (bedrock) */ + #normalizedIndexes = {}; + + constructor() { + if (ModelPricing.instance) return ModelPricing.instance; + ModelPricing.instance = this; + if (!fs.existsSync(this.cacheLocation)) + fs.mkdirSync(this.cacheLocation, { recursive: true }); + + this.#loadFromDisk(); + if (this.isCacheStale || !this.#pricing) { + this.#refresh().catch((err) => + this.log("Background pricing refresh failed:", err?.message) + ); + } + } + + log(text, ...args) { + if (process.env.NODE_ENV === "test") return; + console.log(`\x1b[36m[ModelPricing]\x1b[0m ${text}`, ...args); + } + + /** + * Checks if the disk cache is older than the expiry time (or missing entirely). + * @returns {boolean} + */ + get isCacheStale() { + if (!fs.existsSync(this.cacheFileExpiryPath)) return true; + const cachedAt = fs.readFileSync(this.cacheFileExpiryPath, "utf8"); + return Date.now() - cachedAt > ModelPricing.expiryMs; + } + + /** Loads and memoizes the slimmed pricing map from the disk cache, if present. */ + #loadFromDisk() { + try { + if (!fs.existsSync(this.cacheFilePath)) return; + this.#pricing = JSON.parse( + fs.readFileSync(this.cacheFilePath, { encoding: "utf8" }) + ); + } catch (error) { + this.log("Failed to read pricing cache from disk", error?.message); + this.#pricing = null; + } + } + + /** + * Fetches the latest pricing data from models.dev, slims it, and caches it + * to disk + memory. Uses a conditional GET (ETag) so an unchanged upstream + * only bumps the cache expiry. Falls back to the bundled snapshot from + * `@opencode-ai/models` when the remote is unreachable and no disk cache exists. + */ + async #refresh() { + try { + const headers = {}; + if (this.#pricing && fs.existsSync(this.cacheEtagPath)) { + const etag = fs.readFileSync(this.cacheEtagPath, "utf8").trim(); + if (etag) headers["If-None-Match"] = etag; + } + + const response = await fetch(ModelPricing.remoteUrl, { headers }); + if (response.status === 304) { + await fs.promises.writeFile( + this.cacheFileExpiryPath, + Date.now().toString() + ); + this.log("Remote pricing unchanged (304) - cache expiry bumped."); + return; + } + + if (response.status !== 200) + throw new Error( + `Failed to fetch remote pricing data - status ${response.status}` + ); + + const data = await response.json(); + const pricing = this.#slim(data); + if (!Object.keys(pricing).length) + throw new Error("Remote pricing data contained no usable cost data"); + + this.#pricing = pricing; + this.#lowercaseIndexes = {}; + this.#normalizedIndexes = {}; + + const etag = response.headers.get("etag"); + await Promise.all([ + fs.promises.writeFile(this.cacheFilePath, JSON.stringify(pricing)), + fs.promises.writeFile(this.cacheFileExpiryPath, Date.now().toString()), + etag + ? fs.promises.writeFile(this.cacheEtagPath, etag) + : Promise.resolve(), + ]); + this.log("Remote pricing data synced and cached."); + } catch (error) { + this.log("Error syncing remote pricing data", error?.message); + if (!this.#pricing) await this.#loadSnapshot(); + } + } + + /** + * Loads the pricing data bundled with the `@opencode-ai/models` package as an + * offline/air-gapped fallback. We intentionally do not write `.cached_at` + * here so the next boot retries the remote source. + */ + async #loadSnapshot() { + try { + const { providers } = await ModelPricing.importSnapshot(); + this.#pricing = this.#slim(providers); + this.#lowercaseIndexes = {}; + this.#normalizedIndexes = {}; + this.log("Loaded bundled pricing snapshot (offline fallback)."); + } catch (error) { + this.log("Bundled pricing snapshot failed to load", error?.message); + } + } + + /** + * Slims the full models.dev provider map down to only cost data. + * Models with an absent or null cost object are dropped entirely so a + * lookup miss cleanly signals "unknown pricing". + * @param {Record}>} apiJson + * @returns {Record>} + */ + #slim(apiJson = {}) { + const slimmed = {}; + for (const [providerId, provider] of Object.entries(apiJson)) { + if (!provider?.models || typeof provider.models !== "object") continue; + for (const [modelId, model] of Object.entries(provider.models)) { + if (!model?.cost || typeof model.cost !== "object") continue; + if (typeof model.cost.input !== "number") continue; + slimmed[providerId] ??= {}; + slimmed[providerId][modelId] = model.cost; + } + } + return slimmed; + } + + /** + * Strips the region prefix and version suffix from a Bedrock model id since + * both our stored ids and models.dev's keys are inconsistently prefixed. + * eg: `us.anthropic.claude-sonnet-4-v1:0` -> `anthropic.claude-sonnet-4` + * @param {string} modelId + * @returns {string} + */ + #normalizeBedrockId(modelId = "") { + return modelId + .replace(/^(us|eu|ap|apac|jp|au|global)\./, "") + .replace(/-v\d+(:\d+)?$/, ""); + } + + /** + * Finds the cost object for a model under a models.dev provider id. + * Resolution order: exact match -> case-insensitive match -> bedrock + * region/version normalization (bedrock only). + * @param {string} providerId - models.dev provider id + * @param {string} providerSlug - AnythingLLM provider slug + * @param {string} model + * @returns {ModelCost|null} + */ + #findModelCost(providerId, providerSlug, model) { + const models = this.#pricing?.[providerId]; + if (!models) return null; + if (models[model]) return models[model]; + + if (!this.#lowercaseIndexes[providerId]) { + const index = {}; + for (const key of Object.keys(models)) index[key.toLowerCase()] = key; + this.#lowercaseIndexes[providerId] = index; + } + const caseMatch = this.#lowercaseIndexes[providerId][model.toLowerCase()]; + if (caseMatch) return models[caseMatch]; + + if (providerSlug === "bedrock") { + if (!this.#normalizedIndexes[providerId]) { + const index = {}; + for (const key of Object.keys(models)) { + const normalized = this.#normalizeBedrockId(key); + // Regional variants (au./eu./jp.) can be priced differently and all + // normalize to the same id - prefer the shortest (unprefixed) key. + if (!index[normalized] || key.length < index[normalized].length) + index[normalized] = key; + } + this.#normalizedIndexes[providerId] = index; + } + const normalizedMatch = + this.#normalizedIndexes[providerId][this.#normalizeBedrockId(model)]; + if (normalizedMatch) return models[normalizedMatch]; + } + + return null; + } + + /** + * Resolves the effective per-1M-token rates for a request, accounting for + * long-context pricing tiers (eg: Gemini >200k prompts). + * @param {ModelCost} cost + * @param {number} promptTokens + * @returns {{input: number, output: number}|null} + */ + #resolveRates(cost, promptTokens) { + let input = typeof cost.input === "number" ? cost.input : null; + let output = typeof cost.output === "number" ? cost.output : null; + + if (Array.isArray(cost.tiers)) { + const applicable = cost.tiers + .filter( + (t) => t?.tier?.type === "context" && promptTokens > t.tier.size + ) + .sort((a, b) => a.tier.size - b.tier.size) + .pop(); + if (applicable) { + input = applicable.input ?? input; + output = applicable.output ?? output; + } + } else if (cost.context_over_200k && promptTokens > 200_000) { + input = cost.context_over_200k.input ?? input; + output = cost.context_over_200k.output ?? output; + } + + if (typeof input !== "number" || typeof output !== "number") return null; + return { input, output }; + } + + /** + * Calculates the USD cost of a completion for a given provider slug + model. + * + * Returns zeros for local/self-hosted providers, `null` when pricing is + * unknown (unmapped provider, unknown model). Callers should treat `null` + * as "do not report cost" - absence is how the UI distinguishes unknown + * from free. + * + * Note: only flat prompt/completion rates are applied today since our + * metrics do not track cache or reasoning token counts. If they ever do, + * models.dev conventions are `reasoning ?? output` and + * `cache_read`/`cache_write ?? input` for the fallback rates. + * @param {string|null} providerSlug - AnythingLLM provider slug (eg: "openai" - NOT the class name) + * @param {string|null} model - model id used for the completion + * @param {{prompt_tokens?: number, completion_tokens?: number}} usage + * @returns {CostBreakdown|null} + */ + getCostBreakdown(providerSlug = null, model = null, usage = {}) { + if (!providerSlug) return null; + if (FREE_PROVIDERS.has(providerSlug)) + return { inputCost: 0, outputCost: 0, totalCost: 0 }; + + const providerId = PROVIDER_ID_MAP[providerSlug]; + if (!providerId || !model || !this.#pricing) return null; + + const cost = this.#findModelCost(providerId, providerSlug, model); + if (!cost) return null; + + const promptTokens = Number(usage?.prompt_tokens) || 0; + const completionTokens = Number(usage?.completion_tokens) || 0; + const rates = this.#resolveRates(cost, promptTokens); + if (!rates) return null; + + const inputCost = (promptTokens / 1_000_000) * rates.input; + const outputCost = (completionTokens / 1_000_000) * rates.output; + return { inputCost, outputCost, totalCost: inputCost + outputCost }; + } +} + +const MODEL_PRICING = new ModelPricing(); + +/** + * Returns a new metrics object with `inputCost`/`outputCost`/`totalCost` + * (USD) added when the price of the provider + model is known. Returns the + * metrics unchanged when pricing is unknown so the fields stay absent. + * @param {object} metrics - a chat metrics object with `prompt_tokens`/`completion_tokens` + * @param {{provider?: string|null, model?: string|null}} identity - provider slug + model used + * @returns {object} + */ +function addCostToMetrics( + metrics = {}, + { provider = null, model = null } = {} +) { + if (!metrics || typeof metrics !== "object" || !Object.keys(metrics).length) + return metrics; + + const breakdown = MODEL_PRICING.getCostBreakdown( + provider, + model ?? metrics.model, + { + prompt_tokens: metrics.prompt_tokens, + completion_tokens: metrics.completion_tokens, + } + ); + return breakdown ? { ...metrics, ...breakdown } : metrics; +} + +/** + * Convenience wrapper for the chat flows: resolves the provider slug + model + * that actually served the completion (router delegate first, then workspace + * overrides, then the system default) and decorates the metrics with cost. + * @param {object} metrics - a chat metrics object with `prompt_tokens`/`completion_tokens` + * @param {Object} opts + * @param {object|null} [opts.routingMetadata] - from `resolveProviderConnector` when the workspace uses the model router + * @param {object|null} [opts.workspace] - the workspace record the chat ran in + * @param {object|null} [opts.connector] - the LLM connector instance used + * @returns {object} + */ +function addChatCostToMetrics( + metrics = {}, + { routingMetadata = null, workspace = null, connector = null } = {} +) { + return addCostToMetrics(metrics, { + provider: + routingMetadata?.routedTo?.provider ?? + workspace?.chatProvider ?? + process.env.LLM_PROVIDER, + model: routingMetadata?.routedTo?.model ?? connector?.model, + }); +} + +module.exports = { + ModelPricing, + MODEL_PRICING, + addCostToMetrics, + addChatCostToMetrics, + FREE_PROVIDERS, + PROVIDER_ID_MAP, +}; diff --git a/server/utils/telegramBot/chat/stream.js b/server/utils/telegramBot/chat/stream.js index 4181a2fefac..cc48b87e41d 100644 --- a/server/utils/telegramBot/chat/stream.js +++ b/server/utils/telegramBot/chat/stream.js @@ -1,5 +1,6 @@ const { WorkspaceChats } = require("../../../models/workspaceChats"); const { getVectorDbClass, resolveProviderConnector } = require("../../helpers"); +const { addChatCostToMetrics } = require("../../helpers/modelPricing"); const { DocumentManager } = require("../../DocumentManager"); const { sourceIdentifier, @@ -97,12 +98,13 @@ async function streamResponse({ ctx.bot.sendChatAction(chatId, "typing").catch(() => {}); }, 4000); - const { connector: LLMConnector } = await resolveProviderConnector({ - workspace, - prompt: message, - thread, - attachments, - }); + const { connector: LLMConnector, routingMetadata } = + await resolveProviderConnector({ + workspace, + prompt: message, + thread, + attachments, + }); const VectorDb = getVectorDbClass(); const embeddingsCount = await VectorDb.namespaceCount(workspace.slug); @@ -147,6 +149,7 @@ async function streamResponse({ try { const { completeText, metrics } = await generateResponse({ LLMConnector, + routingMetadata, messages, workspace, ctx, @@ -260,6 +263,7 @@ async function buildSearchContext({ */ async function generateResponse({ LLMConnector, + routingMetadata = null, messages, workspace, ctx, @@ -296,6 +300,11 @@ async function generateResponse({ await sendFormattedMessage(ctx.bot, chatId, completeText); } + metrics = addChatCostToMetrics(metrics, { + routingMetadata, + workspace, + connector: LLMConnector, + }); return { completeText, metrics }; } diff --git a/server/yarn.lock b/server/yarn.lock index e900a3d9843..0f855f516c6 100644 --- a/server/yarn.lock +++ b/server/yarn.lock @@ -862,6 +862,11 @@ zod "^3.25 || ^4.0" zod-to-json-schema "^3.25.0" +"@opencode-ai/models@^0.0.34": + version "0.0.34" + resolved "https://registry.yarnpkg.com/@opencode-ai/models/-/models-0.0.34.tgz#0115cd4d098afb1421b178f8d28bb1959d81f168" + integrity sha512-+wpkzEikhaGG97z9XjbLTz9jBqOd5vGV+6LTFhAtIDXmGq+4WXN3bv6Yc/fcnebR4b7BYLCSyeyctweuSRzLSA== + "@pdf-lib/standard-fonts@^1.0.0": version "1.0.0" resolved "https://registry.yarnpkg.com/@pdf-lib/standard-fonts/-/standard-fonts-1.0.0.tgz#8ba691c4421f71662ed07c9a0294b44528af2d7f" From d7bd703da0019f49d17aa250d8a6c5bd34c7842e Mon Sep 17 00:00:00 2001 From: Marcello Fitton Date: Fri, 7 Aug 2026 12:49:44 -0700 Subject: [PATCH 06/11] strenghten tests --- .../aibitat/providers/ai-provider.test.js | 62 ++++ .../helpers/modelPricing/fixtures/api.json | 28 ++ .../utils/helpers/modelPricing/index.test.js | 346 ++++++++++++++++++ .../modelPricing/offline-snapshot.test.js | 147 ++++++++ server/utils/helpers/modelPricing/index.js | 25 +- 5 files changed, 605 insertions(+), 3 deletions(-) create mode 100644 server/__tests__/utils/helpers/modelPricing/offline-snapshot.test.js diff --git a/server/__tests__/utils/agents/aibitat/providers/ai-provider.test.js b/server/__tests__/utils/agents/aibitat/providers/ai-provider.test.js index 9d5508b8e81..ad0dd49ff81 100644 --- a/server/__tests__/utils/agents/aibitat/providers/ai-provider.test.js +++ b/server/__tests__/utils/agents/aibitat/providers/ai-provider.test.js @@ -122,6 +122,68 @@ describe("Provider usage tracking", () => { expect(providerB.getCumulativeUsage().total_tokens).toBe(0); }); + test("duration accumulates and outputTps is recomputed from run totals", () => { + const provider = new TestProvider(); + + provider.applyUsage({ + prompt_tokens: 100, + completion_tokens: 30, + total_tokens: 130, + duration: 2, + }); + provider.applyUsage({ + prompt_tokens: 200, + completion_tokens: 30, + total_tokens: 230, + duration: 4, + }); + + const totals = provider.getCumulativeUsage(); + expect(totals.duration).toBe(6); + // 60 tokens over 6 seconds - not an average of the per-call TPS values. + expect(totals.outputTps).toBe(10); + }); + + test("negative and non-finite durations do not poison the TPS math", () => { + const provider = new TestProvider(); + + provider.applyUsage({ + prompt_tokens: 100, + completion_tokens: 30, + total_tokens: 130, + duration: -5, + }); + provider.applyUsage({ + prompt_tokens: 100, + completion_tokens: 30, + total_tokens: 130, + duration: Infinity, + }); + + const totals = provider.getCumulativeUsage(); + expect(totals.duration).toBe(0); + expect(totals.outputTps).toBe(0); + }); + + test("returned usage snapshots are copies, not live references", () => { + const provider = new TestProvider(); + + provider.resetUsage(); + provider.recordUsage({ + prompt_tokens: 100, + completion_tokens: 20, + total_tokens: 120, + }); + + const cumulative = provider.getCumulativeUsage(); + const last = provider.getUsage(); + cumulative.prompt_tokens = 999_999; + last.prompt_tokens = 999_999; + + expect(provider.getCumulativeUsage().prompt_tokens).toBe(100); + expect(provider.getUsage().prompt_tokens).toBe(100); + }); + test("accumulation works through InheritMultiple mixin providers", () => { const providerA = new MixinProvider(); const providerB = new MixinProvider(); diff --git a/server/__tests__/utils/helpers/modelPricing/fixtures/api.json b/server/__tests__/utils/helpers/modelPricing/fixtures/api.json index d83658690ab..ec0076f4356 100644 --- a/server/__tests__/utils/helpers/modelPricing/fixtures/api.json +++ b/server/__tests__/utils/helpers/modelPricing/fixtures/api.json @@ -52,6 +52,34 @@ "output": 5, "context_over_200k": { "input": 2, "output": 10 } } + }, + "gemini-garbage-tiers": { + "id": "gemini-garbage-tiers", + "name": "Gemini with malformed tier entries", + "cost": { + "input": 1, + "output": 5, + "tiers": [ + null, + "junk", + { "tier": { "type": "context" } }, + { "input": 99, "output": 99, "tier": { "type": "other", "size": 0 } } + ] + } + }, + "gemini-corrupt-tier": { + "id": "gemini-corrupt-tier", + "name": "Gemini with a corrupt applicable tier", + "cost": { + "input": 1, + "output": 5, + "tiers": [ + { + "input": "not-a-number", + "tier": { "type": "context", "size": 10 } + } + ] + } } } }, diff --git a/server/__tests__/utils/helpers/modelPricing/index.test.js b/server/__tests__/utils/helpers/modelPricing/index.test.js index 32e1db5ac34..3ad56d3a9fd 100644 --- a/server/__tests__/utils/helpers/modelPricing/index.test.js +++ b/server/__tests__/utils/helpers/modelPricing/index.test.js @@ -148,6 +148,193 @@ describe("ModelPricing", () => { ).toEqual({ inputCost: 2.5, outputCost: 0, totalCost: 2.5 }); }); + it("walks the full retrieval lifecycle: cold fetch, warm cache, revalidation, upstream change", async () => { + // Boot 1 - cold: nothing on disk, fetch + cache the remote data. + mockFetchWith(okResponse(FIXTURE, { etag: '"v1"' })); + let pricing = freshInstance(); + await flushRefresh(); + expect(global.fetch).toHaveBeenCalled(); + expect( + pricing.getCostBreakdown("openai", "gpt-4o", { + prompt_tokens: 1_000_000, + }) + ).toEqual({ inputCost: 2.5, outputCost: 0, totalCost: 2.5 }); + + // Boot 2 - warm: cache is fresh, so no network call at all. + jest.resetModules(); + global.fetch = jest.fn(); + pricing = freshInstance(); + await flushRefresh(); + expect(global.fetch).not.toHaveBeenCalled(); + expect( + pricing.getCostBreakdown("openai", "gpt-4o", { + prompt_tokens: 1_000_000, + }) + ).toEqual({ inputCost: 2.5, outputCost: 0, totalCost: 2.5 }); + + // Boot 3 - expired: revalidates with the stored etag, gets a 304, and + // keeps serving the cached data. + const cacheDir = path.join(tempDir, "models", "pricing"); + fs.writeFileSync(path.join(cacheDir, ".cached_at"), "0"); + jest.resetModules(); + mockFetchWith({ + status: 304, + headers: { get: () => null }, + json: async () => { + throw new Error("304 has no body"); + }, + }); + pricing = freshInstance(); + await flushRefresh(); + expect(global.fetch).toHaveBeenCalledWith( + expect.any(String), + expect.objectContaining({ headers: { "If-None-Match": '"v1"' } }) + ); + expect(pricing.isCacheStale).toBe(false); + + // Boot 4 - expired again, but upstream pricing actually changed: the + // new rates and the new etag both land. + fs.writeFileSync(path.join(cacheDir, ".cached_at"), "0"); + const updatedFixture = JSON.parse(JSON.stringify(FIXTURE)); + updatedFixture.openai.models["gpt-4o"].cost = { input: 5, output: 20 }; + jest.resetModules(); + mockFetchWith(okResponse(updatedFixture, { etag: '"v2"' })); + pricing = freshInstance(); + await flushRefresh(); + expect( + pricing.getCostBreakdown("openai", "gpt-4o", { + prompt_tokens: 1_000_000, + }) + ).toEqual({ inputCost: 5, outputCost: 0, totalCost: 5 }); + expect(fs.readFileSync(path.join(cacheDir, ".etag"), "utf8")).toBe( + '"v2"' + ); + }); + + it("does a full GET (no etag) when the disk cache is unusable", async () => { + // If the cache body is corrupt, sending If-None-Match would risk a 304 + // against data we no longer have - the guard must skip the etag. + mockFetchWith(okResponse(FIXTURE, { etag: '"v1"' })); + freshInstance(); + await flushRefresh(); + + const cacheDir = path.join(tempDir, "models", "pricing"); + fs.writeFileSync( + path.join(cacheDir, "model-pricing.json"), + "not-json{{{" + ); + jest.resetModules(); + mockFetchWith(okResponse(FIXTURE, { etag: '"v1"' })); + const pricing = freshInstance(); + await flushRefresh(); + + expect(global.fetch).toHaveBeenCalledWith(expect.any(String), { + headers: {}, + }); + expect( + pricing.getCostBreakdown("openai", "gpt-4o", { + prompt_tokens: 1_000_000, + }) + ).toEqual({ inputCost: 2.5, outputCost: 0, totalCost: 2.5 }); + }); + + it("treats a corrupted .cached_at timestamp as stale and refetches", async () => { + mockFetchWith(okResponse(FIXTURE)); + freshInstance(); + await flushRefresh(); + + const cacheDir = path.join(tempDir, "models", "pricing"); + fs.writeFileSync(path.join(cacheDir, ".cached_at"), "garbage-timestamp"); + jest.resetModules(); + mockFetchWith(okResponse(FIXTURE)); + const pricing = freshInstance(); + await flushRefresh(); + + expect(global.fetch).toHaveBeenCalled(); + expect(pricing.isCacheStale).toBe(false); + }); + + it("recovers from a corrupted disk cache file by refetching", async () => { + mockFetchWith(okResponse(FIXTURE)); + freshInstance(); + await flushRefresh(); + + // Corrupt the cache body while its timestamp is still fresh - the boot + // must notice the unusable cache and refetch anyway. + const cacheDir = path.join(tempDir, "models", "pricing"); + fs.writeFileSync( + path.join(cacheDir, "model-pricing.json"), + "not-json{{{" + ); + jest.resetModules(); + mockFetchWith(okResponse(FIXTURE)); + const pricing = freshInstance(); + await flushRefresh(); + + expect(global.fetch).toHaveBeenCalled(); + expect( + pricing.getCostBreakdown("openai", "gpt-4o", { + prompt_tokens: 1_000_000, + }) + ).toEqual({ inputCost: 2.5, outputCost: 0, totalCost: 2.5 }); + }); + + it("keeps the existing cache when the remote returns unusable data", async () => { + mockFetchWith(okResponse(FIXTURE)); + freshInstance(); + await flushRefresh(); + + const cacheDir = path.join(tempDir, "models", "pricing"); + fs.writeFileSync(path.join(cacheDir, ".cached_at"), "0"); + jest.resetModules(); + mockFetchWith(okResponse({})); + const pricing = freshInstance(); + await flushRefresh(); + + expect( + pricing.getCostBreakdown("openai", "gpt-4o", { + prompt_tokens: 1_000_000, + }) + ).toEqual({ inputCost: 2.5, outputCost: 0, totalCost: 2.5 }); + }); + + it.each([ + ["a null body", okResponse(null)], + ["an array body", okResponse([1, 2, 3])], + ["a string body", okResponse("rate limited")], + [ + "a 500 status", + { status: 500, headers: { get: () => null }, json: async () => ({}) }, + ], + ])( + "survives a remote response with %s and falls back to the snapshot", + async (_label, response) => { + mockFetchWith(response); + const { + ModelPricing, + } = require("../../../../utils/helpers/modelPricing"); + const originalImporter = ModelPricing.importSnapshot; + ModelPricing.importSnapshot = jest + .fn() + .mockResolvedValue({ providers: FIXTURE }); + + try { + ModelPricing.instance = null; + const pricing = new ModelPricing(); + await flushRefresh(); + + expect(ModelPricing.importSnapshot).toHaveBeenCalled(); + expect( + pricing.getCostBreakdown("openai", "gpt-4o", { + prompt_tokens: 1_000_000, + }) + ).toEqual({ inputCost: 2.5, outputCost: 0, totalCost: 2.5 }); + } finally { + ModelPricing.importSnapshot = originalImporter; + } + } + ); + it("falls back to the bundled snapshot when offline with no disk cache", async () => { // Jest's VM cannot execute the real dynamic import of the ESM-only // snapshot package, so inject a fake importer. The real import is @@ -298,6 +485,68 @@ describe("ModelPricing", () => { ).toEqual({ inputCost: (300_000 / 1_000_000) * 2, outputCost: 0, totalCost: (300_000 / 1_000_000) * 2 }); }); + it("clamps negative and non-finite token counts to zero cost", () => { + // A provider misreporting counts must never produce a negative or + // infinite dollar amount. + expect( + pricing.getCostBreakdown("openai", "gpt-4o", { + prompt_tokens: -100_000, + completion_tokens: -50_000, + }) + ).toEqual({ inputCost: 0, outputCost: 0, totalCost: 0 }); + expect( + pricing.getCostBreakdown("openai", "gpt-4o", { + prompt_tokens: Infinity, + completion_tokens: NaN, + }) + ).toEqual({ inputCost: 0, outputCost: 0, totalCost: 0 }); + }); + + it("treats malformed usage payloads as zero tokens for a known model", () => { + for (const usage of [ + undefined, + null, + "not-usage", + [1, 2], + { prompt_tokens: "junk", completion_tokens: { nested: 5 } }, + ]) { + expect(pricing.getCostBreakdown("openai", "gpt-4o", usage)).toEqual({ + inputCost: 0, + outputCost: 0, + totalCost: 0, + }); + } + }); + + it("coerces numeric-string token counts instead of dropping them", () => { + expect( + pricing.getCostBreakdown("openai", "gpt-4o", { + prompt_tokens: "1000000", + completion_tokens: "0", + }) + ).toEqual({ inputCost: 2.5, outputCost: 0, totalCost: 2.5 }); + }); + + it("ignores malformed tier entries and falls back to base rates", () => { + expect( + pricing.getCostBreakdown("gemini", "gemini-garbage-tiers", { + prompt_tokens: 1_000_000, + completion_tokens: 0, + }) + ).toEqual({ inputCost: 1, outputCost: 0, totalCost: 1 }); + }); + + it("degrades a corrupt applicable tier to unknown, never a wrong price", () => { + // The tier applies (prompt > size) but its rate is garbage - report + // no cost rather than a number computed from junk. + expect( + pricing.getCostBreakdown("gemini", "gemini-corrupt-tier", { + prompt_tokens: 1_000_000, + completion_tokens: 0, + }) + ).toBeNull(); + }); + it("resolves openrouter vendor/model ids directly", () => { expect( pricing.getCostBreakdown("openrouter", "anthropic/claude-sonnet-4.5", { @@ -357,5 +606,102 @@ describe("ModelPricing", () => { ).toEqual(metrics); expect(addCostToMetrics({}, { provider: "openai" })).toEqual({}); }); + + it("passes non-object metrics through untouched without crashing", () => { + const { + addCostToMetrics, + } = require("../../../../utils/helpers/modelPricing"); + for (const metrics of [null, "metrics", 42]) { + expect(() => + addCostToMetrics(metrics, { provider: "openai" }) + ).not.toThrow(); + expect(addCostToMetrics(metrics, { provider: "openai" })).toBe(metrics); + } + // undefined falls back to the default parameter and comes back empty + expect(addCostToMetrics(undefined, { provider: "openai" })).toEqual({}); + }); + + it("does not mutate the metrics object it was given", () => { + const { + addCostToMetrics, + } = require("../../../../utils/helpers/modelPricing"); + const metrics = { + prompt_tokens: 1_000_000, + completion_tokens: 0, + model: "gpt-4o", + }; + const decorated = addCostToMetrics(metrics, { provider: "openai" }); + expect(decorated).not.toBe(metrics); + expect(metrics).not.toHaveProperty("totalCost"); + }); + }); + + describe("addChatCostToMetrics provider/model resolution", () => { + const METRICS = { + prompt_tokens: 1_000_000, + completion_tokens: 0, + model: "gpt-4o", + }; + let addChatCostToMetrics; + const originalLLMProvider = process.env.LLM_PROVIDER; + + beforeEach(async () => { + mockFetchWith(okResponse(FIXTURE)); + freshInstance(); + await flushRefresh(); + ({ + addChatCostToMetrics, + } = require("../../../../utils/helpers/modelPricing")); + delete process.env.LLM_PROVIDER; + }); + + afterEach(() => { + if (originalLLMProvider === undefined) delete process.env.LLM_PROVIDER; + else process.env.LLM_PROVIDER = originalLLMProvider; + }); + + it("prefers the router delegate over workspace and env settings", () => { + process.env.LLM_PROVIDER = "anthropic"; + const decorated = addChatCostToMetrics(METRICS, { + routingMetadata: { + routedTo: { provider: "openai", model: "gpt-4o-mini" }, + }, + workspace: { chatProvider: "generic-openai" }, + connector: { model: "gpt-4o" }, + }); + // gpt-4o-mini's rate, not gpt-4o's - both provider and model came + // from the router delegate. + expect(decorated.inputCost).toBe(0.15); + }); + + it("falls back to the workspace provider and connector model", () => { + const decorated = addChatCostToMetrics(METRICS, { + workspace: { chatProvider: "openai" }, + connector: { model: "gpt-4o-mini" }, + }); + expect(decorated.inputCost).toBe(0.15); + }); + + it("falls back to the env provider and metrics.model last", () => { + process.env.LLM_PROVIDER = "openai"; + const decorated = addChatCostToMetrics(METRICS, {}); + expect(decorated).toEqual({ + ...METRICS, + inputCost: 2.5, + outputCost: 0, + totalCost: 2.5, + }); + }); + + it("returns metrics unchanged when no provider can be resolved", () => { + expect(addChatCostToMetrics(METRICS, {})).toEqual(METRICS); + expect( + addChatCostToMetrics(METRICS, { + routingMetadata: { routedTo: null }, + workspace: { chatProvider: null }, + connector: null, + }) + ).toEqual(METRICS); + }); }); }); diff --git a/server/__tests__/utils/helpers/modelPricing/offline-snapshot.test.js b/server/__tests__/utils/helpers/modelPricing/offline-snapshot.test.js new file mode 100644 index 00000000000..e3e395d96e8 --- /dev/null +++ b/server/__tests__/utils/helpers/modelPricing/offline-snapshot.test.js @@ -0,0 +1,147 @@ +const fs = require("fs"); +const os = require("os"); +const path = require("path"); +const { execFileSync } = require("child_process"); + +/** + * Integration tests for the offline snapshot fallback. + * + * The unit suite (index.test.js) mocks `ModelPricing.importSnapshot` because + * Jest's CommonJS VM cannot execute a dynamic import of the ESM-only + * `@opencode-ai/models` package. These tests spawn a real Node process so the + * production import path actually runs - catching a broken `./snapshot` + * subpath, a changed export shape, or pricing data our slimmer cannot use. + */ + +jest.setTimeout(30_000); + +const MODULE_PATH = path.resolve( + __dirname, + "../../../../utils/helpers/modelPricing/index.js" +); + +/** + * Runs a probe script in a real Node process (fetch stubbed offline, fresh + * STORAGE_DIR) and returns its parsed JSON stdout. + * @param {string} probeBody - script body; runs after the offline stubs are set + * @param {string} storageDir - temp STORAGE_DIR for the child + * @returns {any} + */ +function runProbe(probeBody, storageDir) { + const script = ` + process.env.NODE_ENV = "test"; + process.env.STORAGE_DIR = ${JSON.stringify(storageDir)}; + // Stub fetch before the module loads so the singleton's boot refresh + // can never reach the network. + global.fetch = () => Promise.reject(new Error("offline-test")); + const fs = require("fs"); + const path = require("path"); + const { ModelPricing, MODEL_PRICING } = require(${JSON.stringify( + MODULE_PATH + )}); + (async () => { + ${probeBody} + })().catch((error) => { + console.error(error?.stack ?? error); + process.exit(1); + }); + `; + const scriptPath = path.join(storageDir, "probe.js"); + fs.writeFileSync(scriptPath, script); + const stdout = execFileSync(process.execPath, [scriptPath], { + encoding: "utf8", + timeout: 20_000, + }); + return JSON.parse(stdout); +} + +describe("ModelPricing offline snapshot fallback (real import)", () => { + let tempDir; + + beforeEach(() => { + tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "model-pricing-offline-")); + }); + + afterEach(() => { + fs.rmSync(tempDir, { recursive: true, force: true }); + }); + + test("the real @opencode-ai/models snapshot imports with the shape we depend on", () => { + const result = runProbe( + ` + const mod = await ModelPricing.importSnapshot(); + const providers = mod.providers ?? {}; + const openaiModels = providers.openai?.models ?? {}; + const sample = Object.values(openaiModels).find( + (m) => typeof m?.cost?.input === "number" + ); + const { PROVIDER_ID_MAP } = require(${JSON.stringify(MODULE_PATH)}); + console.log(JSON.stringify({ + hasProvidersExport: "providers" in mod, + providerCount: Object.keys(providers).length, + sampleCost: sample?.cost ?? null, + mappedIdsMissingFromSnapshot: Object.values(PROVIDER_ID_MAP).filter( + (id) => !providers[id] + ), + })); + `, + tempDir + ); + + expect(result.hasProvidersExport).toBe(true); + expect(result.providerCount).toBeGreaterThan(50); + expect(typeof result.sampleCost?.input).toBe("number"); + expect(typeof result.sampleCost?.output).toBe("number"); + // Every models.dev id we map to must exist in the bundled snapshot - + // a rename upstream would silently turn that provider's costs into + // "unknown" everywhere. + expect(result.mappedIdsMissingFromSnapshot).toEqual([]); + }); + + test("an offline boot with no disk cache prices real models from the snapshot", () => { + const result = runProbe( + ` + // Pick a sample model straight from the snapshot so this keeps working + // when the bundled catalog changes. + const { providers } = await ModelPricing.importSnapshot(); + const [sampleId] = Object.entries(providers.openai.models).find( + ([, m]) => typeof m?.cost?.input === "number" && m.cost.input > 0 + ); + + // The boot refresh (fired by the require above) fails offline and falls + // back to the snapshot asynchronously - poll until pricing resolves. + const deadline = Date.now() + 15000; + let breakdown = null; + while (!breakdown && Date.now() < deadline) { + breakdown = MODEL_PRICING.getCostBreakdown("openai", sampleId, { + prompt_tokens: 1_000_000, + completion_tokens: 1_000_000, + }); + if (!breakdown) await new Promise((r) => setTimeout(r, 50)); + } + + const cacheDir = path.join(process.env.STORAGE_DIR, "models", "pricing"); + console.log(JSON.stringify({ + sampleId, + breakdown, + wroteCachedAt: fs.existsSync(path.join(cacheDir, ".cached_at")), + wroteCacheFile: fs.existsSync(path.join(cacheDir, "model-pricing.json")), + })); + `, + tempDir + ); + + expect(result.breakdown).not.toBeNull(); + expect(result.breakdown.inputCost).toBeGreaterThan(0); + expect(result.breakdown.outputCost).toBeGreaterThanOrEqual(0); + expect(result.breakdown.totalCost).toBeCloseTo( + result.breakdown.inputCost + result.breakdown.outputCost, + 10 + ); + + // Snapshot pricing must never be persisted as if it were a fresh remote + // sync - the next boot has to retry the network. + expect(result.wroteCachedAt).toBe(false); + expect(result.wroteCacheFile).toBe(false); + }); +}); diff --git a/server/utils/helpers/modelPricing/index.js b/server/utils/helpers/modelPricing/index.js index cc2810a8b9b..05068cbee5f 100644 --- a/server/utils/helpers/modelPricing/index.js +++ b/server/utils/helpers/modelPricing/index.js @@ -120,7 +120,10 @@ class ModelPricing { */ get isCacheStale() { if (!fs.existsSync(this.cacheFileExpiryPath)) return true; - const cachedAt = fs.readFileSync(this.cacheFileExpiryPath, "utf8"); + const cachedAt = Number(fs.readFileSync(this.cacheFileExpiryPath, "utf8")); + // A corrupted timestamp would make every comparison false (NaN) and the + // cache permanently fresh - treat it as stale so it self-heals. + if (!Number.isFinite(cachedAt)) return true; return Date.now() - cachedAt > ModelPricing.expiryMs; } @@ -314,6 +317,20 @@ class ModelPricing { return { input, output }; } + /** + * Coerces a caller-supplied token count into a safe, finite, non-negative + * number. Chat metrics are not sanitized upstream like agent usage is, so a + * provider reporting a negative or non-finite count must not produce a + * negative or infinite cost. + * @param {unknown} value + * @returns {number} + */ + static #safeTokenCount(value) { + const number = Number(value); + if (!Number.isFinite(number) || number < 0) return 0; + return number; + } + /** * Calculates the USD cost of a completion for a given provider slug + model. * @@ -342,8 +359,10 @@ class ModelPricing { const cost = this.#findModelCost(providerId, providerSlug, model); if (!cost) return null; - const promptTokens = Number(usage?.prompt_tokens) || 0; - const completionTokens = Number(usage?.completion_tokens) || 0; + const promptTokens = ModelPricing.#safeTokenCount(usage?.prompt_tokens); + const completionTokens = ModelPricing.#safeTokenCount( + usage?.completion_tokens + ); const rates = this.#resolveRates(cost, promptTokens); if (!rates) return null; From 9939d0aa4bcd065acc8a6e2abef536e7398fc101 Mon Sep 17 00:00:00 2001 From: Marcello Fitton Date: Fri, 7 Aug 2026 18:03:14 -0700 Subject: [PATCH 07/11] move currency preference setting and FX rate fetching server side --- frontend/src/hooks/useCurrency.js | 38 +-- frontend/src/locales/en/common.js | 6 +- frontend/src/models/appearance.js | 4 +- frontend/src/models/system.js | 21 ++ .../components/CurrencyPreference/index.jsx | 45 +++- frontend/src/utils/currency.js | 65 +++-- server/.gitignore | 1 + .../helpers/currencyExchange/index.test.js | 248 ++++++++++++++++++ server/endpoints/admin.js | 5 + server/endpoints/system.js | 30 +++ server/models/systemSettings.js | 12 + .../utils/helpers/currencyExchange/index.js | 188 +++++++++++++ 12 files changed, 598 insertions(+), 65 deletions(-) create mode 100644 server/__tests__/utils/helpers/currencyExchange/index.test.js create mode 100644 server/utils/helpers/currencyExchange/index.js diff --git a/frontend/src/hooks/useCurrency.js b/frontend/src/hooks/useCurrency.js index cf08d4a7c10..8cdb610b906 100644 --- a/frontend/src/hooks/useCurrency.js +++ b/frontend/src/hooks/useCurrency.js @@ -1,35 +1,43 @@ import { useCallback, useEffect, useState } from "react"; -import Appearance from "@/models/appearance"; import { CURRENCY_CHANGE_EVENT, formatCost, - getExchangeRates, + getCurrencySettings, } from "@/utils/currency"; /** - * Provides the user's preferred display currency and a formatter that - * converts stored USD costs into it. Falls back to USD display when no - * exchange rate is available for the preferred currency. + * Provides the instance's display currency and a formatter that converts + * stored USD costs into it. Both come from the server (admin-set system + * setting + server-cached exchange rates). Falls back to USD display when no + * exchange rate is available for the display currency. * @returns {{currency: string, formatCost: (usd: number) => string}} */ export default function useCurrency() { - const [currency, setCurrency] = useState( - Appearance.get("preferredCurrency") || "USD" - ); + const [currency, setCurrency] = useState("USD"); const [rates, setRates] = useState(null); useEffect(() => { - getExchangeRates().then((record) => setRates(record?.rates ?? null)); - }, []); + let mounted = true; - useEffect(() => { - function handleCurrencyChange(e) { - if (!e?.detail?.currency) return; - setCurrency(e.detail.currency); + function applySettings(settings) { + if (!mounted || !settings) return; + setCurrency(settings.currency || "USD"); + setRates(settings.rates ?? null); + } + + getCurrencySettings().then(applySettings); + + // When an admin changes the display currency, `CurrencyPreference` + // invalidates the memoized settings and fires this event so every + // mounted consumer refetches without a page reload. + function handleCurrencyChange() { + getCurrencySettings().then(applySettings); } window.addEventListener(CURRENCY_CHANGE_EVENT, handleCurrencyChange); - return () => + return () => { + mounted = false; window.removeEventListener(CURRENCY_CHANGE_EVENT, handleCurrencyChange); + }; }, []); const rate = currency === "USD" ? 1 : rates?.[currency]; diff --git a/frontend/src/locales/en/common.js b/frontend/src/locales/en/common.js index 03c15b6a7dd..581ab237ffa 100644 --- a/frontend/src/locales/en/common.js +++ b/frontend/src/locales/en/common.js @@ -839,10 +839,10 @@ const TRANSLATIONS = { description: "Select the preferred language to render AnythingLLM's UI in - when translations are available.", }, - "preferred-currency": { - title: "Preferred Currency", + "display-currency": { + title: "Display Currency", description: - "Currency used to display LLM usage costs. Costs are always recorded in USD and converted for display only.", + "Currency used to display LLM usage costs across this instance. Costs are always recorded in USD and converted at current rates for display only.", }, logo: { title: "Brand Logo", diff --git a/frontend/src/models/appearance.js b/frontend/src/models/appearance.js index 1384e39b3aa..1dce2e229cc 100644 --- a/frontend/src/models/appearance.js +++ b/frontend/src/models/appearance.js @@ -7,8 +7,7 @@ import { safeJsonParse } from "@/utils/request"; * 'autoPlayAssistantTtsResponse' | * 'enableSpellCheck' | * 'renderHTML' | - * 'disableAutoScroll' | - * 'preferredCurrency' + * 'disableAutoScroll' * } AvailableSettings - The supported settings for the appearance model. */ @@ -20,7 +19,6 @@ const Appearance = { enableSpellCheck: true, renderHTML: false, disableAutoScroll: false, - preferredCurrency: "USD", }, /** diff --git a/frontend/src/models/system.js b/frontend/src/models/system.js index 3af36a3de55..6bf884a8a23 100644 --- a/frontend/src/models/system.js +++ b/frontend/src/models/system.js @@ -19,6 +19,27 @@ const System = { .then((res) => res?.online || false) .catch(() => false); }, + + /** + * Fetches the instance display currency and the server-cached USD exchange + * rates used to render stored USD costs. `rates` is null when the server + * has no rates available (callers should then display USD). + * @returns {Promise<{base: string, currency: string, rates: Record|null}|null>} + */ + exchangeRates: async function () { + return await fetch(`${API_BASE}/system/exchange-rates`, { + method: "GET", + headers: baseHeaders(), + }) + .then((res) => { + if (!res.ok) throw new Error("Could not fetch exchange rates."); + return res.json(); + }) + .catch((e) => { + console.error(e); + return null; + }); + }, totalIndexes: async function (slug = null) { const url = new URL(`${fullApiUrl()}/system/system-vectors`); if (!!slug) url.searchParams.append("slug", encodeURIComponent(slug)); diff --git a/frontend/src/pages/GeneralSettings/Settings/components/CurrencyPreference/index.jsx b/frontend/src/pages/GeneralSettings/Settings/components/CurrencyPreference/index.jsx index 8f48f994fe6..78b0c56cd38 100644 --- a/frontend/src/pages/GeneralSettings/Settings/components/CurrencyPreference/index.jsx +++ b/frontend/src/pages/GeneralSettings/Settings/components/CurrencyPreference/index.jsx @@ -1,35 +1,58 @@ +import { useEffect, useState } from "react"; import { useTranslation } from "react-i18next"; -import Appearance from "@/models/appearance"; +import Admin from "@/models/admin"; +import showToast from "@/utils/toast"; import { CURRENCY_CHANGE_EVENT, SUPPORTED_CURRENCIES, currencyName, + getCurrencySettings, + invalidateCurrencySettings, } from "@/utils/currency"; export default function CurrencyPreference() { const { t, i18n } = useTranslation(); + const [currency, setCurrency] = useState("USD"); - function changeCurrency(currency) { - Appearance.set("preferredCurrency", currency); - window.dispatchEvent( - new CustomEvent(CURRENCY_CHANGE_EVENT, { detail: { currency } }) - ); + useEffect(() => { + getCurrencySettings().then((settings) => { + if (settings?.currency) setCurrency(settings.currency); + }); + }, []); + + async function changeCurrency(e) { + const newCurrency = e.target.value; + const previousCurrency = currency; + setCurrency(newCurrency); + + const result = await Admin.updateSystemPreferences({ + display_currency: newCurrency, + }); + if (!result?.success) { + setCurrency(previousCurrency); + showToast("Failed to update display currency.", "error"); + return; + } + + // Refetch the memoized settings so every mounted cost display updates. + invalidateCurrencySettings(); + window.dispatchEvent(new CustomEvent(CURRENCY_CHANGE_EVENT)); } return (

- {t("customization.items.preferred-currency.title")} + {t("customization.items.display-currency.title")}

- {t("customization.items.preferred-currency.description")} + {t("customization.items.display-currency.description")}