diff --git a/frontend/src/hooks/useCurrency.js b/frontend/src/hooks/useCurrency.js new file mode 100644 index 00000000000..f6324311048 --- /dev/null +++ b/frontend/src/hooks/useCurrency.js @@ -0,0 +1,57 @@ +import { useCallback, useEffect, useState } from "react"; +import { + CURRENCY_CHANGE_EVENT, + formatCost, + getCurrencySettings, +} from "@/utils/currency"; + +/** + * Provides the instance's display currency and a formatter that converts + * stored USD costs into it. Both come from the server (admin-set system + * setting + server-cached exchange rates). Falls back to USD display when no + * exchange rate is available for the display currency. + * + * NOTE: Currently unused - cost display was intentionally pulled from the UI + * since per-chat/per-run costs are too small to be useful on their own. This + * hook is the intended entry point for upcoming cost features (eg: aggregate + * usage/spend views): call it in any component that renders a stored USD cost + * and pass the value through `formatCost`. + * @returns {{currency: string, formatCost: (usd: number) => string}} + */ +export default function useCurrency() { + const [currency, setCurrency] = useState("USD"); + const [rates, setRates] = useState(null); + + useEffect(() => { + let mounted = true; + + function applySettings(settings) { + if (!mounted || !settings) return; + setCurrency(settings.currency || "USD"); + setRates(settings.rates ?? null); + } + + getCurrencySettings().then(applySettings); + + // When an admin changes the display currency, `CurrencyPreference` + // invalidates the memoized settings and fires this event so every + // mounted consumer refetches without a page reload. + function handleCurrencyChange() { + getCurrencySettings().then(applySettings); + } + window.addEventListener(CURRENCY_CHANGE_EVENT, handleCurrencyChange); + return () => { + mounted = false; + window.removeEventListener(CURRENCY_CHANGE_EVENT, handleCurrencyChange); + }; + }, []); + + const rate = currency === "USD" ? 1 : rates?.[currency]; + const format = useCallback( + (usd) => + formatCost(usd, rate ? { currency, rate } : { currency: "USD", rate: 1 }), + [currency, rate] + ); + + return { currency, formatCost: format }; +} diff --git a/frontend/src/locales/en/common.js b/frontend/src/locales/en/common.js index 088156c6482..9c9dbc8187a 100644 --- a/frontend/src/locales/en/common.js +++ b/frontend/src/locales/en/common.js @@ -839,6 +839,11 @@ const TRANSLATIONS = { description: "Select the preferred language to render AnythingLLM's UI in - when translations are available.", }, + "display-currency": { + title: "Display Currency", + description: + "Currency used to display LLM usage costs across this instance. Costs are always recorded in USD and converted at current rates for display only.", + }, logo: { title: "Brand Logo", description: "Upload your custom logo to showcase on all pages.", diff --git a/frontend/src/models/system.js b/frontend/src/models/system.js index a9d424a9d2c..1440dd7fbdd 100644 --- a/frontend/src/models/system.js +++ b/frontend/src/models/system.js @@ -19,6 +19,27 @@ const System = { .then((res) => res?.online || false) .catch(() => false); }, + + /** + * Fetches the instance display currency and the server-cached USD exchange + * rates used to render stored USD costs. `rates` is null when the server + * has no rates available (callers should then display USD). + * @returns {Promise<{base: string, currency: string, rates: Record|null}|null>} + */ + exchangeRates: async function () { + return await fetch(`${API_BASE}/system/exchange-rates`, { + method: "GET", + headers: baseHeaders(), + }) + .then((res) => { + if (!res.ok) throw new Error("Could not fetch exchange rates."); + return res.json(); + }) + .catch((e) => { + console.error(e); + return null; + }); + }, totalIndexes: async function (slug = null) { const url = new URL(`${fullApiUrl()}/system/system-vectors`); if (!!slug) url.searchParams.append("slug", encodeURIComponent(slug)); diff --git a/frontend/src/pages/GeneralSettings/Settings/Interface/index.jsx b/frontend/src/pages/GeneralSettings/Settings/Interface/index.jsx index 91fe30013b7..e150e708295 100644 --- a/frontend/src/pages/GeneralSettings/Settings/Interface/index.jsx +++ b/frontend/src/pages/GeneralSettings/Settings/Interface/index.jsx @@ -1,6 +1,7 @@ import Sidebar from "@/components/SettingsSidebar"; import { isMobile } from "react-device-detect"; import { useTranslation } from "react-i18next"; +// import CurrencyPreference from "../components/CurrencyPreference"; import LanguagePreference from "../components/LanguagePreference"; import ThemePreference from "../components/ThemePreference"; @@ -27,6 +28,12 @@ export default function InterfaceSettings() { + {/* Hidden until cost display features ship. LLM usage costs are + already recorded (in USD) but not yet surfaced anywhere in the + UI, so the display currency setting has nothing to affect. + Re-enable this (and its import above) with the first cost + surface. */} + {/* */} diff --git a/frontend/src/pages/GeneralSettings/Settings/components/CurrencyPreference/index.jsx b/frontend/src/pages/GeneralSettings/Settings/components/CurrencyPreference/index.jsx new file mode 100644 index 00000000000..78b0c56cd38 --- /dev/null +++ b/frontend/src/pages/GeneralSettings/Settings/components/CurrencyPreference/index.jsx @@ -0,0 +1,68 @@ +import { useEffect, useState } from "react"; +import { useTranslation } from "react-i18next"; +import Admin from "@/models/admin"; +import showToast from "@/utils/toast"; +import { + CURRENCY_CHANGE_EVENT, + SUPPORTED_CURRENCIES, + currencyName, + getCurrencySettings, + invalidateCurrencySettings, +} from "@/utils/currency"; + +export default function CurrencyPreference() { + const { t, i18n } = useTranslation(); + const [currency, setCurrency] = useState("USD"); + + useEffect(() => { + getCurrencySettings().then((settings) => { + if (settings?.currency) setCurrency(settings.currency); + }); + }, []); + + async function changeCurrency(e) { + const newCurrency = e.target.value; + const previousCurrency = currency; + setCurrency(newCurrency); + + const result = await Admin.updateSystemPreferences({ + display_currency: newCurrency, + }); + if (!result?.success) { + setCurrency(previousCurrency); + showToast("Failed to update display currency.", "error"); + return; + } + + // Refetch the memoized settings so every mounted cost display updates. + invalidateCurrencySettings(); + window.dispatchEvent(new CustomEvent(CURRENCY_CHANGE_EVENT)); + } + + return ( +
+

+ {t("customization.items.display-currency.title")} +

+

+ {t("customization.items.display-currency.description")} +

+
+ +
+
+ ); +} diff --git a/frontend/src/utils/currency.js b/frontend/src/utils/currency.js new file mode 100644 index 00000000000..a3ffbdd5b9a --- /dev/null +++ b/frontend/src/utils/currency.js @@ -0,0 +1,109 @@ +import System from "@/models/system"; + +export const CURRENCY_CHANGE_EVENT = "anythingllm_currency_change"; + +/** + * Currencies the server's exchange rate source (Frankfurter) can convert USD + * into. Mirrored in `server/utils/helpers/currencyExchange/index.js` - keep + * the two lists in sync. + */ +export const SUPPORTED_CURRENCIES = [ + "USD", + "AUD", + "BGN", + "BRL", + "CAD", + "CHF", + "CNY", + "CZK", + "DKK", + "EUR", + "GBP", + "HKD", + "HUF", + "IDR", + "ILS", + "INR", + "ISK", + "JPY", + "KRW", + "MXN", + "MYR", + "NOK", + "NZD", + "PHP", + "PLN", + "RON", + "SEK", + "SGD", + "THB", + "TRY", + "ZAR", +]; + +/** + * Module-level cache of the exchange-rates fetch so the many components that + * render costs (one per chat message) share a single request per page load. + * @type {Promise<{base: string, currency: string, rates: Record|null}|null>|null} + */ +let currencySettingsPromise = null; + +/** + * Fetches the instance display currency and USD exchange rates from our own + * server (which caches the upstream source). The result is memoized for the + * lifetime of the page - call `invalidateCurrencySettings` after changing + * the display currency to force a refetch. + * @returns {Promise<{base: string, currency: string, rates: Record|null}|null>} + */ +export async function getCurrencySettings() { + currencySettingsPromise ??= System.exchangeRates().then((result) => { + // Never memoize a failed fetch - let the next caller retry. + if (!result) currencySettingsPromise = null; + return result; + }); + return currencySettingsPromise; +} + +/** Clears the memoized currency settings so the next read refetches. */ +export function invalidateCurrencySettings() { + currencySettingsPromise = null; +} + +/** + * Returns the display name of a currency code in the user's language, + * falling back to the code itself. + * @param {string} code - ISO 4217 currency code (eg: "EUR") + * @param {string} [locale] - BCP 47 locale tag (eg: "en") + * @returns {string} + */ +export function currencyName(code, locale = undefined) { + try { + return new Intl.DisplayNames(locale, { type: "currency" }).of(code) ?? code; + } catch { + return code; + } +} + +/** + * Formats a stored USD cost into the given display currency. + * Precision scales down with the value so tiny per-message costs stay legible: + * >= 1 shows 2 decimals, >= 0.01 shows up to 4, anything smaller up to 6. + * @param {number} usd - the cost in USD + * @param {{currency?: string, rate?: number}} [options] - display currency and its units-per-USD rate + * @returns {string} + */ +export function formatCost(usd, { currency = "USD", rate = 1 } = {}) { + if (typeof usd !== "number" || !isFinite(usd)) return ""; + const value = usd * (rate || 1); + const maxDigits = value >= 1 ? 2 : value >= 0.01 ? 4 : value > 0 ? 6 : 2; + try { + return new Intl.NumberFormat(undefined, { + style: "currency", + currency, + minimumFractionDigits: 2, + maximumFractionDigits: maxDigits, + }).format(value); + } catch { + return `$${value.toFixed(maxDigits)}`; + } +} diff --git a/frontend/src/utils/numbers.js b/frontend/src/utils/numbers.js index b3c13222b5b..bb019ecb567 100644 --- a/frontend/src/utils/numbers.js +++ b/frontend/src/utils/numbers.js @@ -8,13 +8,6 @@ export function nFormatter(input) { return Formatter.format(input); } -export function dollarFormat(input) { - return new Intl.NumberFormat("en-us", { - style: "currency", - currency: "USD", - }).format(input); -} - export function toPercentString(input = null, decimals = 0) { if (isNaN(input) || input === null) return ""; const percentage = Math.round(input * 100); diff --git a/server/.gitignore b/server/.gitignore index 874b2541b5f..bf66e8ec5ca 100644 --- a/server/.gitignore +++ b/server/.gitignore @@ -21,6 +21,7 @@ storage/plugins/anythingllm_mcp_servers.json !storage/documents/DOCUMENTS.md storage/push-notifications/* storage/direct-uploads +storage/currency logs/server.log *.db *.db-journal diff --git a/server/__tests__/utils/agents/aibitat/index.test.js b/server/__tests__/utils/agents/aibitat/index.test.js new file mode 100644 index 00000000000..0901e7cbf41 --- /dev/null +++ b/server/__tests__/utils/agents/aibitat/index.test.js @@ -0,0 +1,88 @@ +const AIbitat = require("../../../../utils/agents/aibitat"); +const { + MODEL_PRICING, +} = require("../../../../utils/helpers/modelPricing"); + +describe("AIbitat.getProviderForConfig providerSlug wiring", () => { + const originalOpenAiKey = process.env.OPEN_AI_KEY; + + beforeAll(() => { + // The OpenAI SDK refuses to construct without an api key - the tests + // never make a request, so any value works. + process.env.OPEN_AI_KEY = "test-key"; + }); + + afterAll(() => { + if (originalOpenAiKey === undefined) delete process.env.OPEN_AI_KEY; + else process.env.OPEN_AI_KEY = originalOpenAiKey; + }); + + afterEach(() => jest.restoreAllMocks()); + + test("stamps the instance with the AnythingLLM slug it was built from", () => { + const aibitat = new AIbitat({ provider: "openai", model: "gpt-4o" }); + const provider = aibitat.getProviderForConfig({ + provider: "openai", + model: "gpt-4o", + }); + + // The slug must be the pricing-map key ("openai"), not the class name + // ("OpenAIProvider") that goes into the metrics `provider` field. + expect(provider.providerSlug).toBe("openai"); + expect(provider.constructor.name).not.toBe(provider.providerSlug); + }); + + test("re-routing to a different slug stamps the new delegate's slug", () => { + // Mirrors a model router re-route: same aibitat, a new per-turn provider + // instance built from the resolved delegate's slug. + const aibitat = new AIbitat({ provider: "openai", model: "gpt-4o" }); + const first = aibitat.getProviderForConfig({ + provider: "openai", + model: "gpt-4o", + }); + const second = aibitat.getProviderForConfig({ + provider: "ollama", + model: "llama3:latest", + }); + + expect(first.providerSlug).toBe("openai"); + expect(second.providerSlug).toBe("ollama"); + }); + + test("a pre-built provider instance keeps its own slug", () => { + const aibitat = new AIbitat({ provider: "openai", model: "gpt-4o" }); + const prebuilt = aibitat.getProviderForConfig({ + provider: "openai", + model: "gpt-4o", + }); + prebuilt.providerSlug = "custom-slug"; + + // config.provider as an object bypasses construction entirely - the + // stamp must not overwrite the slug the instance already carries. + const returned = aibitat.getProviderForConfig({ provider: prebuilt }); + expect(returned).toBe(prebuilt); + expect(returned.providerSlug).toBe("custom-slug"); + }); + + test("the stamped slug is what reaches the pricing lookup", () => { + const getCostBreakdown = jest + .spyOn(MODEL_PRICING, "getCostBreakdown") + .mockReturnValue({ inputCost: 1, outputCost: 2, totalCost: 3 }); + + const aibitat = new AIbitat({ provider: "openai", model: "gpt-4o" }); + const provider = aibitat.getProviderForConfig({ + provider: "openai", + model: "gpt-4o", + }); + + provider.resetUsage(); + provider.recordUsage({ prompt_tokens: 100, completion_tokens: 10 }); + + expect(getCostBreakdown).toHaveBeenCalledWith( + "openai", + "gpt-4o", + expect.objectContaining({ prompt_tokens: 100, completion_tokens: 10 }) + ); + expect(provider.getCumulativeUsage().totalCost).toBe(3); + }); +}); diff --git a/server/__tests__/utils/agents/aibitat/providers/ai-provider.test.js b/server/__tests__/utils/agents/aibitat/providers/ai-provider.test.js new file mode 100644 index 00000000000..ad0dd49ff81 --- /dev/null +++ b/server/__tests__/utils/agents/aibitat/providers/ai-provider.test.js @@ -0,0 +1,414 @@ +const Provider = require("../../../../../utils/agents/aibitat/providers/ai-provider.js"); +const UnTooled = require("../../../../../utils/agents/aibitat/providers/helpers/untooled.js"); +const InheritMultiple = require("../../../../../utils/agents/aibitat/providers/helpers/classes.js"); +const { MODEL_PRICING } = require("../../../../../utils/helpers/modelPricing"); + +class TestProvider extends Provider { + model = "test-model"; + + constructor() { + super(null); + } +} + +// Mirrors how the UnTooled providers (LM Studio, LocalAI, Cerebras, etc.) are +// declared - Provider's fields and methods arrive via the InheritMultiple mixin +// rather than a direct prototype chain. +class MixinProvider extends InheritMultiple([Provider, UnTooled]) { + model = "mixin-model"; +} + +describe("Provider usage tracking", () => { + test("recordUsage accumulates tokens across multiple completions", () => { + const provider = new TestProvider(); + + provider.resetUsage(); + provider.recordUsage({ + prompt_tokens: 100, + completion_tokens: 20, + total_tokens: 120, + }); + + provider.resetUsage(); + provider.recordUsage({ + prompt_tokens: 250, + completion_tokens: 40, + total_tokens: 290, + }); + + provider.resetUsage(); + provider.recordUsage({ + prompt_tokens: 400, + completion_tokens: 60, + total_tokens: 460, + }); + + // getUsage only reflects the most recent completion + const last = provider.getUsage(); + expect(last.prompt_tokens).toBe(400); + expect(last.completion_tokens).toBe(60); + expect(last.total_tokens).toBe(460); + + // getCumulativeUsage reflects the sum of all completions + const totals = provider.getCumulativeUsage(); + expect(totals.prompt_tokens).toBe(750); + expect(totals.completion_tokens).toBe(120); + expect(totals.total_tokens).toBe(870); + expect(totals.model).toBe("test-model"); + expect(totals.provider).toBe("TestProvider"); + }); + + test("resetUsage does not clear the accumulated totals", () => { + const provider = new TestProvider(); + + provider.resetUsage(); + provider.recordUsage({ + prompt_tokens: 100, + completion_tokens: 20, + total_tokens: 120, + }); + + provider.resetUsage(); + expect(provider.getUsage().total_tokens).toBe(0); + expect(provider.getCumulativeUsage().total_tokens).toBe(120); + }); + + test("resetCumulativeUsage zeroes the accumulated totals", () => { + const provider = new TestProvider(); + + provider.resetUsage(); + provider.recordUsage({ + prompt_tokens: 100, + completion_tokens: 20, + total_tokens: 120, + }); + + provider.resetCumulativeUsage(); + const totals = provider.getCumulativeUsage(); + expect(totals.prompt_tokens).toBe(0); + expect(totals.completion_tokens).toBe(0); + expect(totals.total_tokens).toBe(0); + expect(totals.model).toBe(null); + expect(totals.provider).toBe(null); + }); + + test("recordUsage normalizes Anthropic-style input/output token keys", () => { + const provider = new TestProvider(); + + provider.resetUsage(); + provider.recordUsage({ input_tokens: 30, output_tokens: 10 }); + + provider.resetUsage(); + provider.recordUsage({ input_tokens: 50, output_tokens: 15 }); + + const totals = provider.getCumulativeUsage(); + expect(totals.prompt_tokens).toBe(80); + expect(totals.completion_tokens).toBe(25); + expect(totals.total_tokens).toBe(105); + }); + + test("instances do not share an accumulator", () => { + const providerA = new TestProvider(); + const providerB = new TestProvider(); + + providerA.resetUsage(); + providerA.recordUsage({ + prompt_tokens: 100, + completion_tokens: 20, + total_tokens: 120, + }); + + expect(providerA.getCumulativeUsage().total_tokens).toBe(120); + expect(providerB.getCumulativeUsage().total_tokens).toBe(0); + }); + + test("duration accumulates and outputTps is recomputed from run totals", () => { + const provider = new TestProvider(); + + provider.applyUsage({ + prompt_tokens: 100, + completion_tokens: 30, + total_tokens: 130, + duration: 2, + }); + provider.applyUsage({ + prompt_tokens: 200, + completion_tokens: 30, + total_tokens: 230, + duration: 4, + }); + + const totals = provider.getCumulativeUsage(); + expect(totals.duration).toBe(6); + // 60 tokens over 6 seconds - not an average of the per-call TPS values. + expect(totals.outputTps).toBe(10); + }); + + test("negative and non-finite durations do not poison the TPS math", () => { + const provider = new TestProvider(); + + provider.applyUsage({ + prompt_tokens: 100, + completion_tokens: 30, + total_tokens: 130, + duration: -5, + }); + provider.applyUsage({ + prompt_tokens: 100, + completion_tokens: 30, + total_tokens: 130, + duration: Infinity, + }); + + const totals = provider.getCumulativeUsage(); + expect(totals.duration).toBe(0); + expect(totals.outputTps).toBe(0); + }); + + test("returned usage snapshots are copies, not live references", () => { + const provider = new TestProvider(); + + provider.resetUsage(); + provider.recordUsage({ + prompt_tokens: 100, + completion_tokens: 20, + total_tokens: 120, + }); + + const cumulative = provider.getCumulativeUsage(); + const last = provider.getUsage(); + cumulative.prompt_tokens = 999_999; + last.prompt_tokens = 999_999; + + expect(provider.getCumulativeUsage().prompt_tokens).toBe(100); + expect(provider.getUsage().prompt_tokens).toBe(100); + }); + + test("accumulation works through InheritMultiple mixin providers", () => { + const providerA = new MixinProvider(); + const providerB = new MixinProvider(); + + providerA.resetUsage(); + providerA.recordUsage({ + prompt_tokens: 100, + completion_tokens: 10, + total_tokens: 110, + }); + + providerA.resetUsage(); + providerA.recordUsage({ + prompt_tokens: 300, + completion_tokens: 30, + total_tokens: 330, + }); + + expect(providerA.getCumulativeUsage().total_tokens).toBe(440); + expect(providerA.getUsage().total_tokens).toBe(330); + expect(providerB.getCumulativeUsage().total_tokens).toBe(0); + + providerA.resetCumulativeUsage(); + expect(providerA.getCumulativeUsage().total_tokens).toBe(0); + }); +}); + +describe("Provider cost accumulation", () => { + afterEach(() => jest.restoreAllMocks()); + + test("cost is priced per-call and summed even when the model changes mid-run", () => { + // Return a different rate per model so a sum over per-call breakdowns is + // distinguishable from pricing the summed totals at the final model's rate. + jest + .spyOn(MODEL_PRICING, "getCostBreakdown") + .mockImplementation((_slug, model, { prompt_tokens }) => { + const rate = model === "expensive-model" ? 10 : 1; + const inputCost = (prompt_tokens / 1_000_000) * rate; + return { inputCost, outputCost: 0, totalCost: inputCost }; + }); + + const provider = new TestProvider(); + provider.providerSlug = "openai"; + + provider.resetUsage(); + provider.recordUsage({ prompt_tokens: 1_000_000, completion_tokens: 10 }); + + provider.model = "expensive-model"; + provider.resetUsage(); + provider.recordUsage({ prompt_tokens: 1_000_000, completion_tokens: 10 }); + + expect(provider.getUsage().totalCost).toBe(10); + const totals = provider.getCumulativeUsage(); + expect(totals.inputCost).toBe(11); + expect(totals.outputCost).toBe(0); + expect(totals.totalCost).toBe(11); + }); + + test("cost fields stay absent when pricing is unknown", () => { + jest.spyOn(MODEL_PRICING, "getCostBreakdown").mockReturnValue(null); + + const provider = new TestProvider(); + provider.resetUsage(); + provider.recordUsage({ prompt_tokens: 100, completion_tokens: 10 }); + + expect(provider.getUsage()).not.toHaveProperty("totalCost"); + expect(provider.getCumulativeUsage()).not.toHaveProperty("totalCost"); + }); + + test("a partially priceable run sums only the priced calls", () => { + jest + .spyOn(MODEL_PRICING, "getCostBreakdown") + .mockImplementation((_slug, model) => + model === "unknown-model" + ? null + : { inputCost: 1, outputCost: 2, totalCost: 3 } + ); + + const provider = new TestProvider(); + provider.providerSlug = "openai"; + + provider.resetUsage(); + provider.recordUsage({ prompt_tokens: 100, completion_tokens: 10 }); + + provider.model = "unknown-model"; + provider.resetUsage(); + provider.recordUsage({ prompt_tokens: 100, completion_tokens: 10 }); + + // The unpriced call contributes nothing, but the priced call's cost survives. + expect(provider.getUsage()).not.toHaveProperty("totalCost"); + expect(provider.getCumulativeUsage().totalCost).toBe(3); + }); +}); + +describe("Provider usage robustness against malformed payloads", () => { + test.each([ + ["null", null], + ["undefined", undefined], + ["a string", "not-a-usage-object"], + ["a number", 42], + ["a boolean", true], + ["an array", [100, 20, 120]], + ["an empty object", {}], + ])("recordUsage does not crash when the payload is %s", (_label, payload) => { + const provider = new TestProvider(); + + provider.resetUsage(); + expect(() => provider.recordUsage(payload)).not.toThrow(); + + const totals = provider.getCumulativeUsage(); + expect(totals.prompt_tokens).toBe(0); + expect(totals.completion_tokens).toBe(0); + expect(totals.total_tokens).toBe(0); + }); + + test.each([ + ["null", null], + ["undefined", undefined], + ["a string", "not-a-usage-object"], + ["an array", [100, 20, 120]], + ])("applyUsage does not crash when the payload is %s", (_label, payload) => { + const provider = new TestProvider(); + expect(() => provider.applyUsage(payload)).not.toThrow(); + expect(provider.getCumulativeUsage().total_tokens).toBe(0); + }); + + test("coerces numeric strings instead of concatenating them", () => { + const provider = new TestProvider(); + + provider.resetUsage(); + provider.recordUsage({ + prompt_tokens: "100", + completion_tokens: "20", + total_tokens: "120", + }); + + provider.resetUsage(); + provider.recordUsage({ + prompt_tokens: "50", + completion_tokens: "5", + total_tokens: "55", + }); + + const totals = provider.getCumulativeUsage(); + expect(totals.prompt_tokens).toBe(150); + expect(totals.completion_tokens).toBe(25); + expect(totals.total_tokens).toBe(175); + expect(typeof totals.total_tokens).toBe("number"); + }); + + test("treats negative, NaN, and non-finite token counts as zero", () => { + const provider = new TestProvider(); + + provider.resetUsage(); + provider.recordUsage({ + prompt_tokens: -100, + completion_tokens: NaN, + total_tokens: Infinity, + }); + + const totals = provider.getCumulativeUsage(); + expect(totals.prompt_tokens).toBe(0); + expect(totals.completion_tokens).toBe(0); + expect(totals.total_tokens).toBe(0); + }); + + test("treats non-numeric token values as zero", () => { + const provider = new TestProvider(); + + provider.resetUsage(); + provider.recordUsage({ + prompt_tokens: { nested: 100 }, + completion_tokens: "twenty", + total_tokens: () => 120, + }); + + const totals = provider.getCumulativeUsage(); + expect(totals.prompt_tokens).toBe(0); + expect(totals.completion_tokens).toBe(0); + expect(totals.total_tokens).toBe(0); + }); + + test("garbage payloads between valid completions do not corrupt totals", () => { + const provider = new TestProvider(); + + provider.resetUsage(); + provider.recordUsage({ + prompt_tokens: 100, + completion_tokens: 20, + total_tokens: 120, + }); + + provider.resetUsage(); + provider.recordUsage(null); + + provider.resetUsage(); + provider.recordUsage({ prompt_tokens: "junk", completion_tokens: -5 }); + + provider.resetUsage(); + provider.recordUsage({ + prompt_tokens: 50, + completion_tokens: 10, + total_tokens: 60, + }); + + const totals = provider.getCumulativeUsage(); + expect(totals.prompt_tokens).toBe(150); + expect(totals.completion_tokens).toBe(30); + expect(totals.total_tokens).toBe(180); + }); + + test("mixin providers survive malformed payloads too", () => { + const provider = new MixinProvider(); + + provider.resetUsage(); + expect(() => provider.recordUsage(null)).not.toThrow(); + expect(() => provider.recordUsage([1, 2, 3])).not.toThrow(); + + provider.resetUsage(); + provider.recordUsage({ + prompt_tokens: 100, + completion_tokens: 10, + total_tokens: 110, + }); + + expect(provider.getCumulativeUsage().total_tokens).toBe(110); + }); +}); diff --git a/server/__tests__/utils/helpers/currencyExchange/index.test.js b/server/__tests__/utils/helpers/currencyExchange/index.test.js new file mode 100644 index 00000000000..fd624aaa364 --- /dev/null +++ b/server/__tests__/utils/helpers/currencyExchange/index.test.js @@ -0,0 +1,273 @@ +const fs = require("fs"); +const os = require("os"); +const path = require("path"); + +process.env.NODE_ENV = "test"; + +const FRANKFURTER_RESPONSE = { + amount: 1, + base: "USD", + date: "2026-08-06", + rates: { + EUR: 0.85, + GBP: 0.74, + JPY: 147.2, + CAD: 1.37, + }, +}; + +/** + * The module memoizes a singleton at require time, so every test builds its + * own instance against a fresh temp STORAGE_DIR and a mocked global fetch. + */ +function freshInstance() { + const { + CurrencyExchange, + } = require("../../../../utils/helpers/currencyExchange"); + CurrencyExchange.instance = null; + return new CurrencyExchange(); +} + +function mockFetchWith(response) { + global.fetch = jest.fn().mockImplementation(async () => response); +} + +function okResponse(data) { + return { + status: 200, + json: async () => data, + }; +} + +describe("CurrencyExchange", () => { + let tempDir; + const originalFetch = global.fetch; + + beforeEach(() => { + jest.resetModules(); + tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "currency-exchange-test-")); + process.env.STORAGE_DIR = tempDir; + }); + + afterEach(() => { + global.fetch = originalFetch; + fs.rmSync(tempDir, { recursive: true, force: true }); + }); + + describe("cache mechanics", () => { + it("fetches the remote rates and writes the disk cache", async () => { + mockFetchWith(okResponse(FRANKFURTER_RESPONSE)); + const exchange = freshInstance(); + const rates = await exchange.getRates(); + + expect(rates).toEqual({ + EUR: 0.85, + GBP: 0.74, + JPY: 147.2, + CAD: 1.37, + USD: 1, + }); + + const cacheDir = path.join(tempDir, "currency"); + expect(fs.existsSync(path.join(cacheDir, "exchange-rates.json"))).toBe( + true + ); + expect(fs.existsSync(path.join(cacheDir, ".cached_at"))).toBe(true); + expect(exchange.isCacheStale).toBe(false); + }); + + it("serves rates from the disk cache without refetching when fresh", async () => { + mockFetchWith(okResponse(FRANKFURTER_RESPONSE)); + await freshInstance().getRates(); + + jest.resetModules(); + const fetchSpy = jest.fn(); + global.fetch = fetchSpy; + const rates = await freshInstance().getRates(); + + expect(fetchSpy).not.toHaveBeenCalled(); + expect(rates.EUR).toBe(0.85); + }); + + it("refetches when the disk cache is older than the expiry", async () => { + mockFetchWith(okResponse(FRANKFURTER_RESPONSE)); + await freshInstance().getRates(); + + // Age the cache far past expiry, then serve different rates remotely. + fs.writeFileSync(path.join(tempDir, "currency", ".cached_at"), "0"); + jest.resetModules(); + mockFetchWith( + okResponse({ ...FRANKFURTER_RESPONSE, rates: { EUR: 0.9 } }) + ); + const rates = await freshInstance().getRates(); + + expect(global.fetch).toHaveBeenCalledTimes(1); + expect(rates).toEqual({ EUR: 0.9, USD: 1 }); + }); + + it("treats a corrupted .cached_at timestamp as stale", async () => { + mockFetchWith(okResponse(FRANKFURTER_RESPONSE)); + const exchange = freshInstance(); + await exchange.getRates(); + + fs.writeFileSync( + path.join(tempDir, "currency", ".cached_at"), + "not-a-number" + ); + expect(exchange.isCacheStale).toBe(true); + }); + + it("de-dupes concurrent refreshes into a single fetch", async () => { + mockFetchWith(okResponse(FRANKFURTER_RESPONSE)); + const exchange = freshInstance(); + const [a, b] = await Promise.all([ + exchange.getRates(), + exchange.getRates(), + ]); + + expect(global.fetch).toHaveBeenCalledTimes(1); + expect(a).toEqual(b); + }); + + it("fetches with an abort signal so a hung upstream cannot stall requests", async () => { + mockFetchWith(okResponse(FRANKFURTER_RESPONSE)); + await freshInstance().getRates(); + + const [, options] = global.fetch.mock.calls[0]; + expect(options?.signal).toBeInstanceOf(AbortSignal); + }); + + it("returns a copy so callers cannot mutate the memoized rates", async () => { + mockFetchWith(okResponse(FRANKFURTER_RESPONSE)); + const exchange = freshInstance(); + + const rates = await exchange.getRates(); + rates.EUR = 9999; + delete rates.USD; + + expect(await exchange.getRates()).toEqual({ + EUR: 0.85, + GBP: 0.74, + JPY: 147.2, + CAD: 1.37, + USD: 1, + }); + }); + }); + + describe("failure handling", () => { + it("returns null when no rates have ever been fetched and the remote fails", async () => { + global.fetch = jest.fn().mockRejectedValue(new Error("network down")); + const rates = await freshInstance().getRates(); + expect(rates).toBeNull(); + }); + + it("serves stale disk rates when the remote is unreachable", async () => { + mockFetchWith(okResponse(FRANKFURTER_RESPONSE)); + await freshInstance().getRates(); + + // Age the cache so a refresh is attempted, then kill the network. + fs.writeFileSync(path.join(tempDir, "currency", ".cached_at"), "0"); + jest.resetModules(); + global.fetch = jest.fn().mockRejectedValue(new Error("network down")); + const rates = await freshInstance().getRates(); + + expect(rates.EUR).toBe(0.85); + }); + + it("keeps existing rates when the remote returns a non-200 status", async () => { + mockFetchWith(okResponse(FRANKFURTER_RESPONSE)); + await freshInstance().getRates(); + + fs.writeFileSync(path.join(tempDir, "currency", ".cached_at"), "0"); + jest.resetModules(); + mockFetchWith({ status: 503, json: async () => ({}) }); + const rates = await freshInstance().getRates(); + + expect(rates.EUR).toBe(0.85); + }); + + it("retries the remote on the next call after a failed refresh", async () => { + global.fetch = jest.fn().mockRejectedValue(new Error("network down")); + const exchange = freshInstance(); + expect(await exchange.getRates()).toBeNull(); + + mockFetchWith(okResponse(FRANKFURTER_RESPONSE)); + const rates = await exchange.getRates(); + expect(rates.EUR).toBe(0.85); + }); + + it("survives a corrupted disk cache file", async () => { + mockFetchWith(okResponse(FRANKFURTER_RESPONSE)); + await freshInstance().getRates(); + + fs.writeFileSync( + path.join(tempDir, "currency", "exchange-rates.json"), + "{not json" + ); + jest.resetModules(); + mockFetchWith(okResponse(FRANKFURTER_RESPONSE)); + const rates = await freshInstance().getRates(); + + expect(rates.EUR).toBe(0.85); + }); + }); + + describe("payload sanitization", () => { + it("drops unsupported currencies and non-numeric or non-positive rates", async () => { + mockFetchWith( + okResponse({ + rates: { + EUR: 0.85, + FAKE: 2, // not a supported currency + GBP: "0.74", // numeric string is not trusted + JPY: -5, // negative + CAD: Infinity, // non-finite + CHF: 0, // zero would divide costs away + }, + }) + ); + const rates = await freshInstance().getRates(); + expect(rates).toEqual({ EUR: 0.85, USD: 1 }); + }); + + it("treats a payload with no usable rates as a failed refresh", async () => { + mockFetchWith(okResponse({ rates: { FAKE: 2 } })); + const rates = await freshInstance().getRates(); + expect(rates).toBeNull(); + }); + + it("treats a malformed payload as a failed refresh", async () => { + mockFetchWith(okResponse({ rates: "not-an-object" })); + expect(await freshInstance().getRates()).toBeNull(); + + jest.resetModules(); + mockFetchWith(okResponse(null)); + expect(await freshInstance().getRates()).toBeNull(); + }); + }); + + describe("isSupportedCurrency", () => { + it("accepts only known currency codes", () => { + const { + isSupportedCurrency, + } = require("../../../../utils/helpers/currencyExchange"); + expect(isSupportedCurrency("USD")).toBe(true); + expect(isSupportedCurrency("EUR")).toBe(true); + expect(isSupportedCurrency("FAKE")).toBe(false); + expect(isSupportedCurrency("usd")).toBe(false); + expect(isSupportedCurrency(null)).toBe(false); + expect(isSupportedCurrency(42)).toBe(false); + }); + }); + + describe("display_currency validation", () => { + it("accepts supported currencies and falls back to USD otherwise", () => { + const { SystemSettings } = require("../../../../models/systemSettings"); + expect(SystemSettings.validations.display_currency("EUR")).toBe("EUR"); + expect(SystemSettings.validations.display_currency("USD")).toBe("USD"); + expect(SystemSettings.validations.display_currency("FAKE")).toBe("USD"); + expect(SystemSettings.validations.display_currency(null)).toBe("USD"); + }); + }); +}); diff --git a/server/__tests__/utils/helpers/modelPricing/fixtures/api.json b/server/__tests__/utils/helpers/modelPricing/fixtures/api.json new file mode 100644 index 00000000000..ec0076f4356 --- /dev/null +++ b/server/__tests__/utils/helpers/modelPricing/fixtures/api.json @@ -0,0 +1,124 @@ +{ + "openai": { + "id": "openai", + "name": "OpenAI", + "models": { + "gpt-4o": { + "id": "gpt-4o", + "name": "GPT-4o", + "cost": { "input": 2.5, "output": 10 } + }, + "gpt-4o-mini": { + "id": "gpt-4o-mini", + "name": "GPT-4o mini", + "cost": { "input": 0.15, "output": 0.6, "cache_read": 0.075 } + }, + "gpt-oss-free": { + "id": "gpt-oss-free", + "name": "Free model", + "cost": { "input": 0, "output": 0 } + }, + "gpt-subscription-only": { + "id": "gpt-subscription-only", + "name": "No published pricing" + } + } + }, + "google": { + "id": "google", + "name": "Google", + "models": { + "gemini-tiered": { + "id": "gemini-tiered", + "name": "Gemini with tiers", + "cost": { + "input": 1.25, + "output": 10, + "tiers": [ + { + "input": 2.5, + "output": 15, + "tier": { "type": "context", "size": 200000 } + } + ], + "context_over_200k": { "input": 2.5, "output": 15 } + } + }, + "gemini-legacy-200k": { + "id": "gemini-legacy-200k", + "name": "Gemini with legacy long-context pricing", + "cost": { + "input": 1, + "output": 5, + "context_over_200k": { "input": 2, "output": 10 } + } + }, + "gemini-garbage-tiers": { + "id": "gemini-garbage-tiers", + "name": "Gemini with malformed tier entries", + "cost": { + "input": 1, + "output": 5, + "tiers": [ + null, + "junk", + { "tier": { "type": "context" } }, + { "input": 99, "output": 99, "tier": { "type": "other", "size": 0 } } + ] + } + }, + "gemini-corrupt-tier": { + "id": "gemini-corrupt-tier", + "name": "Gemini with a corrupt applicable tier", + "cost": { + "input": 1, + "output": 5, + "tiers": [ + { + "input": "not-a-number", + "tier": { "type": "context", "size": 10 } + } + ] + } + } + } + }, + "amazon-bedrock": { + "id": "amazon-bedrock", + "name": "Amazon Bedrock", + "models": { + "anthropic.claude-sonnet-4-5-20250929-v1:0": { + "id": "anthropic.claude-sonnet-4-5-20250929-v1:0", + "name": "Claude Sonnet 4.5", + "cost": { "input": 3, "output": 15 } + }, + "eu.anthropic.claude-sonnet-4-5-20250929-v1:0": { + "id": "eu.anthropic.claude-sonnet-4-5-20250929-v1:0", + "name": "Claude Sonnet 4.5 (EU)", + "cost": { "input": 3.3, "output": 16.5 } + } + } + }, + "openrouter": { + "id": "openrouter", + "name": "OpenRouter", + "models": { + "anthropic/claude-sonnet-4.5": { + "id": "anthropic/claude-sonnet-4.5", + "name": "Claude Sonnet 4.5", + "cost": { "input": 3, "output": 15 } + } + } + }, + "ollama-cloud": { + "id": "ollama-cloud", + "name": "Ollama Cloud", + "models": { + "some-model": { + "id": "some-model", + "name": "Null-cost model", + "cost": null + } + } + } +} diff --git a/server/__tests__/utils/helpers/modelPricing/index.test.js b/server/__tests__/utils/helpers/modelPricing/index.test.js new file mode 100644 index 00000000000..3ad56d3a9fd --- /dev/null +++ b/server/__tests__/utils/helpers/modelPricing/index.test.js @@ -0,0 +1,707 @@ +const fs = require("fs"); +const os = require("os"); +const path = require("path"); + +process.env.NODE_ENV = "test"; + +const FIXTURE = JSON.parse( + fs.readFileSync(path.resolve(__dirname, "fixtures/api.json"), "utf8") +); + +/** + * The module memoizes a singleton at require time, so every test builds its + * own instance against a fresh temp STORAGE_DIR and a mocked global fetch. + */ +function freshInstance() { + const { ModelPricing } = require("../../../../utils/helpers/modelPricing"); + ModelPricing.instance = null; + return new ModelPricing(); +} + +function mockFetchWith(response) { + global.fetch = jest.fn().mockImplementation(async () => response); +} + +function okResponse(data, { etag = null } = {}) { + return { + status: 200, + headers: { get: (key) => (key === "etag" ? etag : null) }, + json: async () => data, + }; +} + +/** Waits for the constructor's fire-and-forget refresh to settle. */ +async function flushRefresh() { + await new Promise((resolve) => setTimeout(resolve, 25)); +} + +describe("ModelPricing", () => { + let tempDir; + const originalFetch = global.fetch; + + beforeEach(() => { + jest.resetModules(); + tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "model-pricing-test-")); + process.env.STORAGE_DIR = tempDir; + }); + + afterEach(() => { + global.fetch = originalFetch; + fs.rmSync(tempDir, { recursive: true, force: true }); + }); + + describe("cache mechanics", () => { + it("fetches the remote pricing data and writes the disk cache", async () => { + mockFetchWith(okResponse(FIXTURE, { etag: '"abc123"' })); + const pricing = freshInstance(); + await flushRefresh(); + + const cacheDir = path.join(tempDir, "models", "pricing"); + expect(fs.existsSync(path.join(cacheDir, "model-pricing.json"))).toBe( + true + ); + expect(fs.existsSync(path.join(cacheDir, ".cached_at"))).toBe(true); + expect(fs.readFileSync(path.join(cacheDir, ".etag"), "utf8")).toBe( + '"abc123"' + ); + expect(pricing.isCacheStale).toBe(false); + + // The disk cache is slimmed to cost objects only, dropping models + // with absent or null cost. + const cached = JSON.parse( + fs.readFileSync(path.join(cacheDir, "model-pricing.json"), "utf8") + ); + expect(cached.openai["gpt-4o"]).toEqual({ input: 2.5, output: 10 }); + expect(cached.openai["gpt-subscription-only"]).toBeUndefined(); + expect(cached["ollama-cloud"]).toBeUndefined(); + }); + + it("serves pricing from the disk cache without refetching when fresh", async () => { + mockFetchWith(okResponse(FIXTURE)); + freshInstance(); + await flushRefresh(); + + jest.resetModules(); + const fetchSpy = jest.fn(); + global.fetch = fetchSpy; + const pricing = freshInstance(); + await flushRefresh(); + + expect(fetchSpy).not.toHaveBeenCalled(); + expect( + pricing.getCostBreakdown("openai", "gpt-4o", { + prompt_tokens: 1_000_000, + completion_tokens: 0, + }) + ).toEqual({ inputCost: 2.5, outputCost: 0, totalCost: 2.5 }); + }); + + it("only bumps the cache expiry on a 304 response", async () => { + mockFetchWith(okResponse(FIXTURE, { etag: '"abc123"' })); + freshInstance(); + await flushRefresh(); + + // Age the cache past expiry so the next boot refreshes, then 304 it. + const cacheDir = path.join(tempDir, "models", "pricing"); + fs.writeFileSync(path.join(cacheDir, ".cached_at"), "0"); + jest.resetModules(); + mockFetchWith({ + status: 304, + headers: { get: () => null }, + json: async () => { + throw new Error("304 has no body"); + }, + }); + const pricing = freshInstance(); + await flushRefresh(); + + expect(global.fetch).toHaveBeenCalledWith( + expect.any(String), + expect.objectContaining({ + headers: { "If-None-Match": '"abc123"' }, + }) + ); + expect(pricing.isCacheStale).toBe(false); + expect( + pricing.getCostBreakdown("openai", "gpt-4o", { + prompt_tokens: 1_000_000, + }) + ).toEqual({ inputCost: 2.5, outputCost: 0, totalCost: 2.5 }); + }); + + it("keeps serving the stale disk cache when the remote fetch fails", async () => { + mockFetchWith(okResponse(FIXTURE)); + freshInstance(); + await flushRefresh(); + + const cacheDir = path.join(tempDir, "models", "pricing"); + fs.writeFileSync(path.join(cacheDir, ".cached_at"), "0"); + jest.resetModules(); + global.fetch = jest.fn().mockRejectedValue(new Error("offline")); + const pricing = freshInstance(); + await flushRefresh(); + + expect( + pricing.getCostBreakdown("openai", "gpt-4o", { + prompt_tokens: 1_000_000, + }) + ).toEqual({ inputCost: 2.5, outputCost: 0, totalCost: 2.5 }); + }); + + it("walks the full retrieval lifecycle: cold fetch, warm cache, revalidation, upstream change", async () => { + // Boot 1 - cold: nothing on disk, fetch + cache the remote data. + mockFetchWith(okResponse(FIXTURE, { etag: '"v1"' })); + let pricing = freshInstance(); + await flushRefresh(); + expect(global.fetch).toHaveBeenCalled(); + expect( + pricing.getCostBreakdown("openai", "gpt-4o", { + prompt_tokens: 1_000_000, + }) + ).toEqual({ inputCost: 2.5, outputCost: 0, totalCost: 2.5 }); + + // Boot 2 - warm: cache is fresh, so no network call at all. + jest.resetModules(); + global.fetch = jest.fn(); + pricing = freshInstance(); + await flushRefresh(); + expect(global.fetch).not.toHaveBeenCalled(); + expect( + pricing.getCostBreakdown("openai", "gpt-4o", { + prompt_tokens: 1_000_000, + }) + ).toEqual({ inputCost: 2.5, outputCost: 0, totalCost: 2.5 }); + + // Boot 3 - expired: revalidates with the stored etag, gets a 304, and + // keeps serving the cached data. + const cacheDir = path.join(tempDir, "models", "pricing"); + fs.writeFileSync(path.join(cacheDir, ".cached_at"), "0"); + jest.resetModules(); + mockFetchWith({ + status: 304, + headers: { get: () => null }, + json: async () => { + throw new Error("304 has no body"); + }, + }); + pricing = freshInstance(); + await flushRefresh(); + expect(global.fetch).toHaveBeenCalledWith( + expect.any(String), + expect.objectContaining({ headers: { "If-None-Match": '"v1"' } }) + ); + expect(pricing.isCacheStale).toBe(false); + + // Boot 4 - expired again, but upstream pricing actually changed: the + // new rates and the new etag both land. + fs.writeFileSync(path.join(cacheDir, ".cached_at"), "0"); + const updatedFixture = JSON.parse(JSON.stringify(FIXTURE)); + updatedFixture.openai.models["gpt-4o"].cost = { input: 5, output: 20 }; + jest.resetModules(); + mockFetchWith(okResponse(updatedFixture, { etag: '"v2"' })); + pricing = freshInstance(); + await flushRefresh(); + expect( + pricing.getCostBreakdown("openai", "gpt-4o", { + prompt_tokens: 1_000_000, + }) + ).toEqual({ inputCost: 5, outputCost: 0, totalCost: 5 }); + expect(fs.readFileSync(path.join(cacheDir, ".etag"), "utf8")).toBe( + '"v2"' + ); + }); + + it("does a full GET (no etag) when the disk cache is unusable", async () => { + // If the cache body is corrupt, sending If-None-Match would risk a 304 + // against data we no longer have - the guard must skip the etag. + mockFetchWith(okResponse(FIXTURE, { etag: '"v1"' })); + freshInstance(); + await flushRefresh(); + + const cacheDir = path.join(tempDir, "models", "pricing"); + fs.writeFileSync( + path.join(cacheDir, "model-pricing.json"), + "not-json{{{" + ); + jest.resetModules(); + mockFetchWith(okResponse(FIXTURE, { etag: '"v1"' })); + const pricing = freshInstance(); + await flushRefresh(); + + expect(global.fetch).toHaveBeenCalledWith(expect.any(String), { + headers: {}, + }); + expect( + pricing.getCostBreakdown("openai", "gpt-4o", { + prompt_tokens: 1_000_000, + }) + ).toEqual({ inputCost: 2.5, outputCost: 0, totalCost: 2.5 }); + }); + + it("treats a corrupted .cached_at timestamp as stale and refetches", async () => { + mockFetchWith(okResponse(FIXTURE)); + freshInstance(); + await flushRefresh(); + + const cacheDir = path.join(tempDir, "models", "pricing"); + fs.writeFileSync(path.join(cacheDir, ".cached_at"), "garbage-timestamp"); + jest.resetModules(); + mockFetchWith(okResponse(FIXTURE)); + const pricing = freshInstance(); + await flushRefresh(); + + expect(global.fetch).toHaveBeenCalled(); + expect(pricing.isCacheStale).toBe(false); + }); + + it("recovers from a corrupted disk cache file by refetching", async () => { + mockFetchWith(okResponse(FIXTURE)); + freshInstance(); + await flushRefresh(); + + // Corrupt the cache body while its timestamp is still fresh - the boot + // must notice the unusable cache and refetch anyway. + const cacheDir = path.join(tempDir, "models", "pricing"); + fs.writeFileSync( + path.join(cacheDir, "model-pricing.json"), + "not-json{{{" + ); + jest.resetModules(); + mockFetchWith(okResponse(FIXTURE)); + const pricing = freshInstance(); + await flushRefresh(); + + expect(global.fetch).toHaveBeenCalled(); + expect( + pricing.getCostBreakdown("openai", "gpt-4o", { + prompt_tokens: 1_000_000, + }) + ).toEqual({ inputCost: 2.5, outputCost: 0, totalCost: 2.5 }); + }); + + it("keeps the existing cache when the remote returns unusable data", async () => { + mockFetchWith(okResponse(FIXTURE)); + freshInstance(); + await flushRefresh(); + + const cacheDir = path.join(tempDir, "models", "pricing"); + fs.writeFileSync(path.join(cacheDir, ".cached_at"), "0"); + jest.resetModules(); + mockFetchWith(okResponse({})); + const pricing = freshInstance(); + await flushRefresh(); + + expect( + pricing.getCostBreakdown("openai", "gpt-4o", { + prompt_tokens: 1_000_000, + }) + ).toEqual({ inputCost: 2.5, outputCost: 0, totalCost: 2.5 }); + }); + + it.each([ + ["a null body", okResponse(null)], + ["an array body", okResponse([1, 2, 3])], + ["a string body", okResponse("rate limited")], + [ + "a 500 status", + { status: 500, headers: { get: () => null }, json: async () => ({}) }, + ], + ])( + "survives a remote response with %s and falls back to the snapshot", + async (_label, response) => { + mockFetchWith(response); + const { + ModelPricing, + } = require("../../../../utils/helpers/modelPricing"); + const originalImporter = ModelPricing.importSnapshot; + ModelPricing.importSnapshot = jest + .fn() + .mockResolvedValue({ providers: FIXTURE }); + + try { + ModelPricing.instance = null; + const pricing = new ModelPricing(); + await flushRefresh(); + + expect(ModelPricing.importSnapshot).toHaveBeenCalled(); + expect( + pricing.getCostBreakdown("openai", "gpt-4o", { + prompt_tokens: 1_000_000, + }) + ).toEqual({ inputCost: 2.5, outputCost: 0, totalCost: 2.5 }); + } finally { + ModelPricing.importSnapshot = originalImporter; + } + } + ); + + it("falls back to the bundled snapshot when offline with no disk cache", async () => { + // Jest's VM cannot execute the real dynamic import of the ESM-only + // snapshot package, so inject a fake importer. The real import is + // exercised by the server at runtime. + global.fetch = jest.fn().mockRejectedValue(new Error("offline")); + const { ModelPricing } = require("../../../../utils/helpers/modelPricing"); + const originalImporter = ModelPricing.importSnapshot; + ModelPricing.importSnapshot = jest + .fn() + .mockResolvedValue({ providers: FIXTURE }); + + try { + ModelPricing.instance = null; + const pricing = new ModelPricing(); + await flushRefresh(); + + expect(ModelPricing.importSnapshot).toHaveBeenCalled(); + expect( + pricing.getCostBreakdown("openai", "gpt-4o", { + prompt_tokens: 1_000_000, + completion_tokens: 0, + }) + ).toEqual({ inputCost: 2.5, outputCost: 0, totalCost: 2.5 }); + + // No .cached_at is written so the next boot retries the remote. + expect( + fs.existsSync(path.join(tempDir, "models", "pricing", ".cached_at")) + ).toBe(false); + } finally { + ModelPricing.importSnapshot = originalImporter; + } + }); + }); + + describe("getCostBreakdown", () => { + let pricing; + + beforeEach(async () => { + mockFetchWith(okResponse(FIXTURE)); + pricing = freshInstance(); + await flushRefresh(); + }); + + it("computes exact input/output/total costs", () => { + expect( + pricing.getCostBreakdown("openai", "gpt-4o-mini", { + prompt_tokens: 1000, + completion_tokens: 500, + }) + ).toEqual({ + inputCost: (1000 / 1_000_000) * 0.15, + outputCost: (500 / 1_000_000) * 0.6, + totalCost: (1000 / 1_000_000) * 0.15 + (500 / 1_000_000) * 0.6, + }); + }); + + it("returns zeros for local/self-hosted providers without a lookup", () => { + for (const slug of ["ollama", "lmstudio", "koboldcpp"]) { + expect( + pricing.getCostBreakdown(slug, "whatever-model", { + prompt_tokens: 1000, + completion_tokens: 1000, + }) + ).toEqual({ inputCost: 0, outputCost: 0, totalCost: 0 }); + } + }); + + it("returns zeros for a model with published zero pricing", () => { + expect( + pricing.getCostBreakdown("openai", "gpt-oss-free", { + prompt_tokens: 1000, + completion_tokens: 1000, + }) + ).toEqual({ inputCost: 0, outputCost: 0, totalCost: 0 }); + }); + + it("returns null for unknown pricing", () => { + // Unmapped provider slug + expect(pricing.getCostBreakdown("generic-openai", "gpt-4o")).toBeNull(); + // Unknown model on a known provider + expect(pricing.getCostBreakdown("openai", "not-a-model")).toBeNull(); + // Model whose upstream cost is null (slimmed away) + expect(pricing.getCostBreakdown("openrouter", "some-model")).toBeNull(); + // Model with no published pricing (slimmed away) + expect( + pricing.getCostBreakdown("openai", "gpt-subscription-only") + ).toBeNull(); + // No provider at all + expect(pricing.getCostBreakdown(null, "gpt-4o")).toBeNull(); + }); + + it("matches model ids case-insensitively", () => { + expect( + pricing.getCostBreakdown("openai", "GPT-4o", { + prompt_tokens: 1_000_000, + }) + ).toEqual({ inputCost: 2.5, outputCost: 0, totalCost: 2.5 }); + }); + + it("normalizes bedrock region prefixes and version suffixes", () => { + // Region-prefixed user config matches the unprefixed dataset key, + // never the differently-priced eu. variant. + expect( + pricing.getCostBreakdown( + "bedrock", + "us.anthropic.claude-sonnet-4-5-20250929-v1:0", + { prompt_tokens: 1_000_000 } + ) + ).toEqual({ inputCost: 3, outputCost: 0, totalCost: 3 }); + expect( + pricing.getCostBreakdown( + "bedrock", + "anthropic.claude-sonnet-4-5-20250929", + { prompt_tokens: 1_000_000 } + ) + ).toEqual({ inputCost: 3, outputCost: 0, totalCost: 3 }); + }); + + it("applies long-context tier pricing above the tier threshold", () => { + expect( + pricing.getCostBreakdown("gemini", "gemini-tiered", { + prompt_tokens: 100_000, + completion_tokens: 1000, + }) + ).toEqual({ + inputCost: (100_000 / 1_000_000) * 1.25, + outputCost: (1000 / 1_000_000) * 10, + totalCost: (100_000 / 1_000_000) * 1.25 + (1000 / 1_000_000) * 10, + }); + expect( + pricing.getCostBreakdown("gemini", "gemini-tiered", { + prompt_tokens: 300_000, + completion_tokens: 1000, + }) + ).toEqual({ + inputCost: (300_000 / 1_000_000) * 2.5, + outputCost: (1000 / 1_000_000) * 15, + totalCost: (300_000 / 1_000_000) * 2.5 + (1000 / 1_000_000) * 15, + }); + }); + + it("applies legacy context_over_200k pricing when no tiers exist", () => { + expect( + pricing.getCostBreakdown("gemini", "gemini-legacy-200k", { + prompt_tokens: 300_000, + completion_tokens: 0, + }) + ).toEqual({ inputCost: (300_000 / 1_000_000) * 2, outputCost: 0, totalCost: (300_000 / 1_000_000) * 2 }); + }); + + it("clamps negative and non-finite token counts to zero cost", () => { + // A provider misreporting counts must never produce a negative or + // infinite dollar amount. + expect( + pricing.getCostBreakdown("openai", "gpt-4o", { + prompt_tokens: -100_000, + completion_tokens: -50_000, + }) + ).toEqual({ inputCost: 0, outputCost: 0, totalCost: 0 }); + expect( + pricing.getCostBreakdown("openai", "gpt-4o", { + prompt_tokens: Infinity, + completion_tokens: NaN, + }) + ).toEqual({ inputCost: 0, outputCost: 0, totalCost: 0 }); + }); + + it("treats malformed usage payloads as zero tokens for a known model", () => { + for (const usage of [ + undefined, + null, + "not-usage", + [1, 2], + { prompt_tokens: "junk", completion_tokens: { nested: 5 } }, + ]) { + expect(pricing.getCostBreakdown("openai", "gpt-4o", usage)).toEqual({ + inputCost: 0, + outputCost: 0, + totalCost: 0, + }); + } + }); + + it("coerces numeric-string token counts instead of dropping them", () => { + expect( + pricing.getCostBreakdown("openai", "gpt-4o", { + prompt_tokens: "1000000", + completion_tokens: "0", + }) + ).toEqual({ inputCost: 2.5, outputCost: 0, totalCost: 2.5 }); + }); + + it("ignores malformed tier entries and falls back to base rates", () => { + expect( + pricing.getCostBreakdown("gemini", "gemini-garbage-tiers", { + prompt_tokens: 1_000_000, + completion_tokens: 0, + }) + ).toEqual({ inputCost: 1, outputCost: 0, totalCost: 1 }); + }); + + it("degrades a corrupt applicable tier to unknown, never a wrong price", () => { + // The tier applies (prompt > size) but its rate is garbage - report + // no cost rather than a number computed from junk. + expect( + pricing.getCostBreakdown("gemini", "gemini-corrupt-tier", { + prompt_tokens: 1_000_000, + completion_tokens: 0, + }) + ).toBeNull(); + }); + + it("resolves openrouter vendor/model ids directly", () => { + expect( + pricing.getCostBreakdown("openrouter", "anthropic/claude-sonnet-4.5", { + prompt_tokens: 1_000_000, + completion_tokens: 0, + }) + ).toEqual({ inputCost: 3, outputCost: 0, totalCost: 3 }); + }); + }); + + describe("addCostToMetrics", () => { + beforeEach(async () => { + mockFetchWith(okResponse(FIXTURE)); + freshInstance(); + await flushRefresh(); + }); + + it("decorates metrics when pricing is known", () => { + const { + addCostToMetrics, + } = require("../../../../utils/helpers/modelPricing"); + const metrics = { + prompt_tokens: 1_000_000, + completion_tokens: 0, + model: "gpt-4o", + }; + expect(addCostToMetrics(metrics, { provider: "openai" })).toEqual({ + ...metrics, + inputCost: 2.5, + outputCost: 0, + totalCost: 2.5, + }); + }); + + it("prefers an explicitly passed model over metrics.model", () => { + const { + addCostToMetrics, + } = require("../../../../utils/helpers/modelPricing"); + const decorated = addCostToMetrics( + { prompt_tokens: 1_000_000, completion_tokens: 0, model: "gpt-4o" }, + { provider: "openai", model: "gpt-4o-mini" } + ); + expect(decorated.inputCost).toBe(0.15); + }); + + it("returns metrics unchanged when pricing is unknown", () => { + const { + addCostToMetrics, + } = require("../../../../utils/helpers/modelPricing"); + const metrics = { + prompt_tokens: 100, + completion_tokens: 10, + model: "some-local-model", + }; + expect( + addCostToMetrics(metrics, { provider: "generic-openai" }) + ).toEqual(metrics); + expect(addCostToMetrics({}, { provider: "openai" })).toEqual({}); + }); + + it("passes non-object metrics through untouched without crashing", () => { + const { + addCostToMetrics, + } = require("../../../../utils/helpers/modelPricing"); + for (const metrics of [null, "metrics", 42]) { + expect(() => + addCostToMetrics(metrics, { provider: "openai" }) + ).not.toThrow(); + expect(addCostToMetrics(metrics, { provider: "openai" })).toBe(metrics); + } + // undefined falls back to the default parameter and comes back empty + expect(addCostToMetrics(undefined, { provider: "openai" })).toEqual({}); + }); + + it("does not mutate the metrics object it was given", () => { + const { + addCostToMetrics, + } = require("../../../../utils/helpers/modelPricing"); + const metrics = { + prompt_tokens: 1_000_000, + completion_tokens: 0, + model: "gpt-4o", + }; + const decorated = addCostToMetrics(metrics, { provider: "openai" }); + expect(decorated).not.toBe(metrics); + expect(metrics).not.toHaveProperty("totalCost"); + }); + }); + + describe("addChatCostToMetrics provider/model resolution", () => { + const METRICS = { + prompt_tokens: 1_000_000, + completion_tokens: 0, + model: "gpt-4o", + }; + let addChatCostToMetrics; + const originalLLMProvider = process.env.LLM_PROVIDER; + + beforeEach(async () => { + mockFetchWith(okResponse(FIXTURE)); + freshInstance(); + await flushRefresh(); + ({ + addChatCostToMetrics, + } = require("../../../../utils/helpers/modelPricing")); + delete process.env.LLM_PROVIDER; + }); + + afterEach(() => { + if (originalLLMProvider === undefined) delete process.env.LLM_PROVIDER; + else process.env.LLM_PROVIDER = originalLLMProvider; + }); + + it("prefers the router delegate over workspace and env settings", () => { + process.env.LLM_PROVIDER = "anthropic"; + const decorated = addChatCostToMetrics(METRICS, { + routingMetadata: { + routedTo: { provider: "openai", model: "gpt-4o-mini" }, + }, + workspace: { chatProvider: "generic-openai" }, + connector: { model: "gpt-4o" }, + }); + // gpt-4o-mini's rate, not gpt-4o's - both provider and model came + // from the router delegate. + expect(decorated.inputCost).toBe(0.15); + }); + + it("falls back to the workspace provider and connector model", () => { + const decorated = addChatCostToMetrics(METRICS, { + workspace: { chatProvider: "openai" }, + connector: { model: "gpt-4o-mini" }, + }); + expect(decorated.inputCost).toBe(0.15); + }); + + it("falls back to the env provider and metrics.model last", () => { + process.env.LLM_PROVIDER = "openai"; + const decorated = addChatCostToMetrics(METRICS, {}); + expect(decorated).toEqual({ + ...METRICS, + inputCost: 2.5, + outputCost: 0, + totalCost: 2.5, + }); + }); + + it("returns metrics unchanged when no provider can be resolved", () => { + expect(addChatCostToMetrics(METRICS, {})).toEqual(METRICS); + expect( + addChatCostToMetrics(METRICS, { + routingMetadata: { routedTo: null }, + workspace: { chatProvider: null }, + connector: null, + }) + ).toEqual(METRICS); + }); + }); +}); diff --git a/server/__tests__/utils/helpers/modelPricing/offline-snapshot.test.js b/server/__tests__/utils/helpers/modelPricing/offline-snapshot.test.js new file mode 100644 index 00000000000..e3e395d96e8 --- /dev/null +++ b/server/__tests__/utils/helpers/modelPricing/offline-snapshot.test.js @@ -0,0 +1,147 @@ +const fs = require("fs"); +const os = require("os"); +const path = require("path"); +const { execFileSync } = require("child_process"); + +/** + * Integration tests for the offline snapshot fallback. + * + * The unit suite (index.test.js) mocks `ModelPricing.importSnapshot` because + * Jest's CommonJS VM cannot execute a dynamic import of the ESM-only + * `@opencode-ai/models` package. These tests spawn a real Node process so the + * production import path actually runs - catching a broken `./snapshot` + * subpath, a changed export shape, or pricing data our slimmer cannot use. + */ + +jest.setTimeout(30_000); + +const MODULE_PATH = path.resolve( + __dirname, + "../../../../utils/helpers/modelPricing/index.js" +); + +/** + * Runs a probe script in a real Node process (fetch stubbed offline, fresh + * STORAGE_DIR) and returns its parsed JSON stdout. + * @param {string} probeBody - script body; runs after the offline stubs are set + * @param {string} storageDir - temp STORAGE_DIR for the child + * @returns {any} + */ +function runProbe(probeBody, storageDir) { + const script = ` + process.env.NODE_ENV = "test"; + process.env.STORAGE_DIR = ${JSON.stringify(storageDir)}; + // Stub fetch before the module loads so the singleton's boot refresh + // can never reach the network. + global.fetch = () => Promise.reject(new Error("offline-test")); + const fs = require("fs"); + const path = require("path"); + const { ModelPricing, MODEL_PRICING } = require(${JSON.stringify( + MODULE_PATH + )}); + (async () => { + ${probeBody} + })().catch((error) => { + console.error(error?.stack ?? error); + process.exit(1); + }); + `; + const scriptPath = path.join(storageDir, "probe.js"); + fs.writeFileSync(scriptPath, script); + const stdout = execFileSync(process.execPath, [scriptPath], { + encoding: "utf8", + timeout: 20_000, + }); + return JSON.parse(stdout); +} + +describe("ModelPricing offline snapshot fallback (real import)", () => { + let tempDir; + + beforeEach(() => { + tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "model-pricing-offline-")); + }); + + afterEach(() => { + fs.rmSync(tempDir, { recursive: true, force: true }); + }); + + test("the real @opencode-ai/models snapshot imports with the shape we depend on", () => { + const result = runProbe( + ` + const mod = await ModelPricing.importSnapshot(); + const providers = mod.providers ?? {}; + const openaiModels = providers.openai?.models ?? {}; + const sample = Object.values(openaiModels).find( + (m) => typeof m?.cost?.input === "number" + ); + const { PROVIDER_ID_MAP } = require(${JSON.stringify(MODULE_PATH)}); + console.log(JSON.stringify({ + hasProvidersExport: "providers" in mod, + providerCount: Object.keys(providers).length, + sampleCost: sample?.cost ?? null, + mappedIdsMissingFromSnapshot: Object.values(PROVIDER_ID_MAP).filter( + (id) => !providers[id] + ), + })); + `, + tempDir + ); + + expect(result.hasProvidersExport).toBe(true); + expect(result.providerCount).toBeGreaterThan(50); + expect(typeof result.sampleCost?.input).toBe("number"); + expect(typeof result.sampleCost?.output).toBe("number"); + // Every models.dev id we map to must exist in the bundled snapshot - + // a rename upstream would silently turn that provider's costs into + // "unknown" everywhere. + expect(result.mappedIdsMissingFromSnapshot).toEqual([]); + }); + + test("an offline boot with no disk cache prices real models from the snapshot", () => { + const result = runProbe( + ` + // Pick a sample model straight from the snapshot so this keeps working + // when the bundled catalog changes. + const { providers } = await ModelPricing.importSnapshot(); + const [sampleId] = Object.entries(providers.openai.models).find( + ([, m]) => typeof m?.cost?.input === "number" && m.cost.input > 0 + ); + + // The boot refresh (fired by the require above) fails offline and falls + // back to the snapshot asynchronously - poll until pricing resolves. + const deadline = Date.now() + 15000; + let breakdown = null; + while (!breakdown && Date.now() < deadline) { + breakdown = MODEL_PRICING.getCostBreakdown("openai", sampleId, { + prompt_tokens: 1_000_000, + completion_tokens: 1_000_000, + }); + if (!breakdown) await new Promise((r) => setTimeout(r, 50)); + } + + const cacheDir = path.join(process.env.STORAGE_DIR, "models", "pricing"); + console.log(JSON.stringify({ + sampleId, + breakdown, + wroteCachedAt: fs.existsSync(path.join(cacheDir, ".cached_at")), + wroteCacheFile: fs.existsSync(path.join(cacheDir, "model-pricing.json")), + })); + `, + tempDir + ); + + expect(result.breakdown).not.toBeNull(); + expect(result.breakdown.inputCost).toBeGreaterThan(0); + expect(result.breakdown.outputCost).toBeGreaterThanOrEqual(0); + expect(result.breakdown.totalCost).toBeCloseTo( + result.breakdown.inputCost + result.breakdown.outputCost, + 10 + ); + + // Snapshot pricing must never be persisted as if it were a fresh remote + // sync - the next boot has to retry the network. + expect(result.wroteCachedAt).toBe(false); + expect(result.wroteCacheFile).toBe(false); + }); +}); diff --git a/server/endpoints/admin.js b/server/endpoints/admin.js index c817c67f04c..62cc04c6301 100644 --- a/server/endpoints/admin.js +++ b/server/endpoints/admin.js @@ -353,6 +353,7 @@ function adminEndpoints(app) { // These match the ManagerRoute pages in the frontend. const managerAllowedFields = [ "custom_app_name", + "display_currency", "footer_data", "support_email", "meta_page_title", @@ -386,6 +387,9 @@ function adminEndpoints(app) { case "support_email": requestedSettings[label] = setting?.value || null; break; + case "display_currency": + requestedSettings[label] = setting?.value || "USD"; + break; case "text_splitter_chunk_size": requestedSettings[label] = setting?.value || embedder?.embeddingMaxChunkLength || null; @@ -473,6 +477,7 @@ function adminEndpoints(app) { if (user?.role === ROLES.manager) { const managerAllowedFields = [ "custom_app_name", + "display_currency", "footer_data", "support_email", "meta_page_title", diff --git a/server/endpoints/system.js b/server/endpoints/system.js index e5c9e4988b7..8f5b2ff86a7 100644 --- a/server/endpoints/system.js +++ b/server/endpoints/system.js @@ -123,6 +123,36 @@ function systemEndpoints(app) { } }); + /** + * Returns the instance display currency and the cached USD exchange rates + * so any logged-in user can render stored USD costs in the display currency. + * `rates` is null when no rates have ever been fetched (eg: air-gapped + * install) - the frontend then falls back to displaying USD. + */ + app.get( + "/system/exchange-rates", + [validatedRequest], + async (_request, response) => { + try { + const { + CURRENCY_EXCHANGE, + } = require("../utils/helpers/currencyExchange"); + const setting = await SystemSettings.get({ + label: "display_currency", + }); + const rates = await CURRENCY_EXCHANGE.getRates(); + response.status(200).json({ + base: "USD", + currency: setting?.value || "USD", + rates, + }); + } catch (e) { + console.error(e.message, e); + response.sendStatus(500).end(); + } + } + ); + app.get( "/system/check-token", [validatedRequest], diff --git a/server/index.js b/server/index.js index db2b6d1d6aa..66e2ef965f3 100644 --- a/server/index.js +++ b/server/index.js @@ -4,6 +4,7 @@ process.env.NODE_ENV === "development" require("./utils/logger")(); require("./utils/boot/patchSdkTimeouts")(); +require("./utils/helpers/modelPricing"); // boots the model pricing cache refresh const express = require("express"); const bodyParser = require("body-parser"); const cors = require("cors"); diff --git a/server/models/systemSettings.js b/server/models/systemSettings.js index 06dc0278ebf..d5ef5df405a 100644 --- a/server/models/systemSettings.js +++ b/server/models/systemSettings.js @@ -60,6 +60,7 @@ const SystemSettings = { "agent_clarifying_questions_enabled", "agent_clarifying_questions_max_per_turn", "custom_app_name", + "display_currency", "feature_flags", "meta_page_title", "meta_page_favicon", @@ -90,6 +91,7 @@ const SystemSettings = { "agent_clarifying_questions_max_per_turn", "custom_app_name", "default_system_prompt", + "display_currency", // Meta page customization "meta_page_title", @@ -188,6 +190,16 @@ const SystemSettings = { return JSON.stringify([]); } }, + display_currency: (update) => { + const { + isSupportedCurrency, + } = require("../utils/helpers/currencyExchange"); + if (isSupportedCurrency(update)) return update; + console.error( + `Failed to run validation function on display_currency - unsupported currency "${update}"` + ); + return "USD"; + }, memory_enabled: async (update) => { try { const enabled = String(update) === "true"; diff --git a/server/package.json b/server/package.json index 7fb80513191..30b25db4ed2 100644 --- a/server/package.json +++ b/server/package.json @@ -34,6 +34,7 @@ "@mintplex-labs/express-ws": "^5.0.7", "@mintplex-labs/mdpdf": "^0.1.9", "@modelcontextprotocol/sdk": "^1.24.3", + "@opencode-ai/models": "^0.0.34", "@pinecone-database/pinecone": "^2.0.1", "@prisma/client": "5.3.1", "@qdrant/js-client-rest": "^1.9.0", diff --git a/server/storage/models/.gitignore b/server/storage/models/.gitignore index 4634bc3006c..70b7de0dfcc 100644 --- a/server/storage/models/.gitignore +++ b/server/storage/models/.gitignore @@ -10,6 +10,7 @@ togetherAi tesseract ppio context-windows/* +pricing/* MintplexLabs cometapi fireworks diff --git a/server/utils/agents/aibitat/index.js b/server/utils/agents/aibitat/index.js index c87eac58d06..78dc02e14c2 100644 --- a/server/utils/agents/aibitat/index.js +++ b/server/utils/agents/aibitat/index.js @@ -1025,7 +1025,11 @@ https://docs.anythingllm.com/agent/intelligent-tool-selection }; // Emit routing notification before the first completion so it appears above the response - if (depth === 0) this?.flushRoutingMetadata?.(v4()); + // and reset the usage accumulator so metrics only cover this run's completions. + if (depth === 0) { + this?.flushRoutingMetadata?.(v4()); + this.providerInstance.resetCumulativeUsage(); + } /** @type {{ functionCall: { name: string, arguments: string }, textResponse: string }} */ const completionStream = await this.#safeProviderCall(() => @@ -1111,7 +1115,7 @@ https://docs.anythingllm.com/agent/intelligent-tool-selection eventHandler?.("reportStreamEvent", { type: "usageMetrics", uuid: directOutputUUID, - metrics: this.providerInstance.getUsage(), + metrics: this.providerInstance.getCumulativeUsage(), }); this?.flushCitations?.(directOutputUUID); this?.emitChatId?.(directOutputUUID); @@ -1152,7 +1156,7 @@ https://docs.anythingllm.com/agent/intelligent-tool-selection eventHandler?.("reportStreamEvent", { type: "usageMetrics", uuid: responseUuid, - metrics: this.providerInstance.getUsage(), + metrics: this.providerInstance.getCumulativeUsage(), }); this?.flushCitations?.(responseUuid); this?.emitChatId?.(responseUuid); @@ -1187,7 +1191,11 @@ https://docs.anythingllm.com/agent/intelligent-tool-selection }; // Emit routing notification before the first completion so it appears above the response - if (depth === 0) this?.flushRoutingMetadata?.(msgUUID); + // and reset the usage accumulator so metrics only cover this run's completions. + if (depth === 0) { + this?.flushRoutingMetadata?.(msgUUID); + this.providerInstance.resetCumulativeUsage(); + } // get the chat completion const completion = await this.#safeProviderCall(() => @@ -1262,7 +1270,7 @@ https://docs.anythingllm.com/agent/intelligent-tool-selection eventHandler?.("reportStreamEvent", { type: "usageMetrics", uuid: msgUUID, - metrics: this.providerInstance.getUsage(), + metrics: this.providerInstance.getCumulativeUsage(), }); this?.flushCitations?.(msgUUID); return result; @@ -1302,7 +1310,7 @@ https://docs.anythingllm.com/agent/intelligent-tool-selection eventHandler?.("reportStreamEvent", { type: "usageMetrics", uuid: msgUUID, - metrics: this.providerInstance.getUsage(), + metrics: this.providerInstance.getCumulativeUsage(), }); this?.flushCitations?.(msgUUID); this?.emitChatId?.(msgUUID); @@ -1409,6 +1417,10 @@ https://docs.anythingllm.com/agent/intelligent-tool-selection */ getProviderForConfig(config) { const provider = this.#buildProviderForConfig(config); + // Record the slug the instance was built from so usage metrics can be + // priced - pre-built instances (config.provider as an object) keep theirs. + if (typeof config?.provider === "string") + provider.providerSlug ??= config.provider; provider.attachAbortSignal?.(this.abortController.signal); return provider; } diff --git a/server/utils/agents/aibitat/plugins/chat-history.js b/server/utils/agents/aibitat/plugins/chat-history.js index 257a3bba706..c5bbee498b7 100644 --- a/server/utils/agents/aibitat/plugins/chat-history.js +++ b/server/utils/agents/aibitat/plugins/chat-history.js @@ -99,7 +99,7 @@ const chatHistory = { { prompt, response, attachments = [] } = {} ) { const invocation = aibitat.handlerProps.invocation; - const metrics = aibitat.providerInstance?.getUsage?.() ?? {}; + const metrics = aibitat.providerInstance?.getCumulativeUsage?.() ?? {}; const citations = aibitat._pendingCitations ?? []; const outputs = aibitat._pendingOutputs ?? []; const clarifyingQuestions = @@ -134,7 +134,7 @@ const chatHistory = { { prompt, response, attachments = [], options = {} } = {} ) { const invocation = aibitat.handlerProps.invocation; - const metrics = aibitat.providerInstance?.getUsage?.() ?? {}; + const metrics = aibitat.providerInstance?.getCumulativeUsage?.() ?? {}; const citations = aibitat._pendingCitations ?? []; const outputs = aibitat._pendingOutputs ?? []; const clarifyingQuestions = diff --git a/server/utils/agents/aibitat/providers/ai-provider.js b/server/utils/agents/aibitat/providers/ai-provider.js index 50f147f47b5..b3513a3f165 100644 --- a/server/utils/agents/aibitat/providers/ai-provider.js +++ b/server/utils/agents/aibitat/providers/ai-provider.js @@ -16,6 +16,8 @@ const { ChatAnthropic } = require("@langchain/anthropic"); const { ChatOllama } = require("@langchain/community/chat_models/ollama"); const { toValidNumber, safeJsonParse } = require("../../../http"); const { getLLMProviderClass } = require("../../../helpers"); +const { MODEL_PRICING } = require("../../../helpers/modelPricing"); +const { toNonNegativeNumber } = require("../../../helpers/numbers"); const { parseLMStudioBasePath } = require("../../../AiProviders/lmStudio"); const { parseDockerModelRunnerEndpoint, @@ -39,6 +41,9 @@ const { bindAbortSignal } = require("../../../helpers/abortSignals"); * @property {string|null} model - Model name * @property {string|null} provider - Provider class name * @property {Date|null} timestamp - Timestamp of the completion + * @property {number} [inputCost] - USD cost of the prompt tokens. Absent when pricing is unknown. + * @property {number} [outputCost] - USD cost of the completion tokens. Absent when pricing is unknown. + * @property {number} [totalCost] - USD sum of input and output costs. Absent when pricing is unknown. */ /** @@ -51,6 +56,8 @@ const { bindAbortSignal } = require("../../../helpers/abortSignals"); * @property {(messages: Array, functions?: Array, eventHandler?: Function) => Promise<{functionCall: any, textResponse: string}>} stream - Stream a chat completion with tool calling. * @property {(messages: Array, functions?: Array) => Promise<{functionCall: any, textResponse: string, result?: string}>} complete - Non-streaming chat completion with tool calling. * @property {() => ProviderUsageMetrics} getUsage - Get usage metrics from the last completion. + * @property {() => ProviderUsageMetrics} getCumulativeUsage - Get usage metrics accumulated across all completions in the current run. + * @property {() => void} resetCumulativeUsage - Reset the accumulated usage metrics (call at the start of a run). */ class Provider { @@ -75,16 +82,33 @@ class Provider { * Stores the usage metrics from the last completion call. * @type {ProviderUsageMetrics} */ - lastUsage = { - prompt_tokens: 0, - completion_tokens: 0, - total_tokens: 0, - duration: 0, - outputTps: 0, - model: null, - provider: null, - timestamp: null, - }; + lastUsage = Provider.#emptyUsage(); + + /** + * Stores the usage metrics accumulated across every completion call in the + * current run. An agent loop makes one completion per tool call plus a final + * one for the response - this is the sum of all of them, whereas `lastUsage` + * only ever reflects the most recent call. + * @type {ProviderUsageMetrics} + */ + cumulativeUsage = Provider.#emptyUsage(); + + /** + * Zeroed usage metrics for initializing/resetting an accumulator. + * @returns {ProviderUsageMetrics} + */ + static #emptyUsage() { + return { + prompt_tokens: 0, + completion_tokens: 0, + total_tokens: 0, + duration: 0, + outputTps: 0, + model: null, + provider: null, + timestamp: null, + }; + } /** * Timestamp when the current request started (for duration calculation). @@ -99,6 +123,15 @@ class Provider { */ providerTag = null; + /** + * The AnythingLLM provider slug this instance was built for (eg: "openai", + * "anthropic") - set by AIbitat when the provider is instantiated. Unlike + * `providerTag` or `constructor.name`, this matches the slugs used for + * model pricing lookups. Null when the origin of the instance is unknown. + * @type {string|null} + */ + providerSlug = null; + /** * Abort signal for the active agent session, attached by AIbitat. Bound to the * SDK client so every request this provider makes is cancelled when the session @@ -635,21 +668,86 @@ class Provider { duration = (Date.now() - this._requestStartTime) / 1000; } - const promptTokens = usage.prompt_tokens || usage.input_tokens || 0; - const completionTokens = - usage.completion_tokens || usage.output_tokens || 0; + const safeUsage = usage && typeof usage === "object" ? usage : {}; + const promptTokens = toNonNegativeNumber( + safeUsage.prompt_tokens || safeUsage.input_tokens + ); + const completionTokens = toNonNegativeNumber( + safeUsage.completion_tokens || safeUsage.output_tokens + ); + const totalTokens = toNonNegativeNumber(safeUsage.total_tokens); + + this.applyUsage({ + prompt_tokens: promptTokens, + completion_tokens: completionTokens, + total_tokens: totalTokens || promptTokens + completionTokens, + duration, + }); + } + + /** + * Stores a normalized usage record for the completion that just finished and + * adds it to the run-level accumulated totals. Subclasses that override + * `recordUsage` should normalize their provider-specific usage format and + * call this so accumulation still happens in one place. + * Every value is coerced to a safe number so a malformed payload from any + * provider cannot crash the run or corrupt the accumulated totals. + * @param {{prompt_tokens?: number, completion_tokens?: number, total_tokens?: number, duration?: number}} usage + */ + applyUsage(usage = {}) { + const safeUsage = usage && typeof usage === "object" ? usage : {}; + const promptTokens = toNonNegativeNumber(safeUsage.prompt_tokens); + const completionTokens = toNonNegativeNumber(safeUsage.completion_tokens); + const totalTokens = toNonNegativeNumber(safeUsage.total_tokens); + const duration = toNonNegativeNumber(safeUsage.duration); + + const timestamp = new Date(); + // Cost is priced per-call (not derived from the summed totals) so the + // accumulated cost stays correct even if the model changes mid-run. + // A null breakdown (unknown pricing) leaves the cost fields absent. + const cost = MODEL_PRICING.getCostBreakdown(this.providerSlug, this.model, { + prompt_tokens: promptTokens, + completion_tokens: completionTokens, + }); this.lastUsage = { prompt_tokens: promptTokens, completion_tokens: completionTokens, - total_tokens: usage.total_tokens || promptTokens + completionTokens, + total_tokens: totalTokens, outputTps: completionTokens && duration > 0 ? completionTokens / duration : 0, duration, model: this.model, provider: this.constructor.name, - timestamp: new Date(), + timestamp, + ...(cost ?? {}), }; + + const totals = this.cumulativeUsage; + totals.prompt_tokens += promptTokens; + totals.completion_tokens += completionTokens; + totals.total_tokens += totalTokens; + totals.duration += duration; + totals.outputTps = + totals.completion_tokens && totals.duration > 0 + ? totals.completion_tokens / totals.duration + : 0; + totals.model = this.model; + totals.provider = this.constructor.name; + totals.timestamp = timestamp; + if (cost) { + totals.inputCost = (totals.inputCost ?? 0) + cost.inputCost; + totals.outputCost = (totals.outputCost ?? 0) + cost.outputCost; + totals.totalCost = (totals.totalCost ?? 0) + cost.totalCost; + } + } + + /** + * Resets the accumulated usage metrics. Call this at the start of an agent + * run so the totals only cover that run's completions. + */ + resetCumulativeUsage() { + this.cumulativeUsage = Provider.#emptyUsage(); } /** @@ -660,6 +758,15 @@ class Provider { return { ...this.lastUsage }; } + /** + * Get the usage metrics accumulated across all completions in the current + * run - one completion per tool call plus the final response. + * @returns {ProviderUsageMetrics} The accumulated usage metrics + */ + getCumulativeUsage() { + return { ...this.cumulativeUsage }; + } + /** * Stream a chat completion from the LLM with tool calling * Note: This using the OpenAI API format and may need to be adapted for other providers. diff --git a/server/utils/agents/aibitat/providers/cerebras.js b/server/utils/agents/aibitat/providers/cerebras.js index ba765b4d0fd..d75e66ceda4 100644 --- a/server/utils/agents/aibitat/providers/cerebras.js +++ b/server/utils/agents/aibitat/providers/cerebras.js @@ -177,21 +177,17 @@ class CerebrasProvider extends InheritMultiple([Provider, UnTooled]) { recordUsage(usage = {}, time_info = {}) { // assume start time let duration = (Date.now() - this._requestStartTime) / 1000; - const promptTokens = usage.prompt_tokens || 0; - const completionTokens = usage.completion_tokens || 0; + const safeUsage = usage && typeof usage === "object" ? usage : {}; + const promptTokens = safeUsage.prompt_tokens || 0; + const completionTokens = safeUsage.completion_tokens || 0; if (time_info?.completion_time) duration = time_info.completion_time; - this.lastUsage = { + this.applyUsage({ prompt_tokens: promptTokens, completion_tokens: completionTokens, - total_tokens: usage.total_tokens, - outputTps: - completionTokens && duration > 0 ? completionTokens / duration : 0, + total_tokens: safeUsage.total_tokens || promptTokens + completionTokens, duration, - model: this.model, - provider: this.constructor.name, - timestamp: new Date(), - }; + }); } /** diff --git a/server/utils/agents/aibitat/providers/gemini.js b/server/utils/agents/aibitat/providers/gemini.js index 0777f2c4685..cc0581430be 100644 --- a/server/utils/agents/aibitat/providers/gemini.js +++ b/server/utils/agents/aibitat/providers/gemini.js @@ -274,12 +274,17 @@ class GeminiProvider extends Provider { functionCall: null, }; + // Gemini can attach usage to more than one chunk, so capture the latest + // and record exactly once after the stream ends - recordUsage accumulates + // into run totals, so calling it per-chunk would inflate them. + let usage = null; + for await (const streamEvent of response) { /** @type {OpenAI.OpenAI.Chat.ChatCompletionChunk} */ const chunk = streamEvent; // Capture usage from final chunk (when stream_options.include_usage is true) - if (chunk?.usage) this.recordUsage(chunk.usage); + if (chunk?.usage) usage = chunk.usage; const { content, tool_calls } = chunk?.choices?.[0]?.delta || {}; if (content) { @@ -316,6 +321,8 @@ class GeminiProvider extends Provider { } } + if (usage) this.recordUsage(usage); + if (completion.functionCall) { completion.functionCall.arguments = safeJsonParse( completion.functionCall.arguments, diff --git a/server/utils/chats/apiChatHandler.js b/server/utils/chats/apiChatHandler.js index 0eebef9c0c8..2e66b3cfdd8 100644 --- a/server/utils/chats/apiChatHandler.js +++ b/server/utils/chats/apiChatHandler.js @@ -2,6 +2,7 @@ const { v4: uuidv4 } = require("uuid"); const { DocumentManager } = require("../DocumentManager"); const { WorkspaceChats } = require("../../models/workspaceChats"); const { getVectorDbClass, resolveProviderConnector } = require("../helpers"); +const { addChatCostToMetrics } = require("../helpers/modelPricing"); const { writeResponseChunk } = require("../helpers/chat/responses"); const { abortConnectorOnClientDisconnect } = require("../helpers/abortSignals"); const { @@ -226,14 +227,15 @@ async function chatSync({ }); } - const { connector: LLMConnector } = await resolveProviderConnector({ - workspace, - prompt: message, - user, - thread, - attachments, - apiSessionId: sessionId, - }); + const { connector: LLMConnector, routingMetadata } = + await resolveProviderConnector({ + workspace, + prompt: message, + user, + thread, + attachments, + apiSessionId: sessionId, + }); const VectorDb = getVectorDbClass(); const messageLimit = workspace?.openAiHistory || 20; @@ -420,11 +422,16 @@ async function chatSync({ ); // Send the text completion. - const { textResponse, metrics: performanceMetrics } = + const { textResponse, metrics: completionMetrics } = await LLMConnector.getChatCompletion(messages, { temperature: workspace?.openAiTemp ?? LLMConnector.defaultTemp, user: user, }); + const performanceMetrics = addChatCostToMetrics(completionMetrics, { + routingMetadata, + workspace, + connector: LLMConnector, + }); if (!textResponse) { return { @@ -594,14 +601,15 @@ async function streamChat({ }); } - const { connector: LLMConnector } = await resolveProviderConnector({ - workspace, - prompt: message, - user, - thread, - attachments, - apiSessionId: sessionId, - }); + const { connector: LLMConnector, routingMetadata } = + await resolveProviderConnector({ + workspace, + prompt: message, + user, + thread, + attachments, + apiSessionId: sessionId, + }); // A disconnected client (aborted request, closed connection) should stop the // provider generating too, not just stop us reading the response. @@ -814,7 +822,11 @@ async function streamChat({ user: user, }); completeText = textResponse; - metrics = performanceMetrics; + metrics = addChatCostToMetrics(performanceMetrics, { + routingMetadata, + workspace, + connector: LLMConnector, + }); writeResponseChunk(response, { uuid, sources, @@ -830,7 +842,11 @@ async function streamChat({ user: user, }); completeText = await LLMConnector.handleStream(response, stream, { uuid }); - metrics = stream.metrics; + metrics = addChatCostToMetrics(stream.metrics, { + routingMetadata, + workspace, + connector: LLMConnector, + }); } if (completeText?.length > 0) { diff --git a/server/utils/chats/embed.js b/server/utils/chats/embed.js index 8476311587a..dc023d81f5c 100644 --- a/server/utils/chats/embed.js +++ b/server/utils/chats/embed.js @@ -1,5 +1,6 @@ const { v4: uuidv4 } = require("uuid"); const { getVectorDbClass, resolveProviderConnector } = require("../helpers"); +const { addChatCostToMetrics } = require("../helpers/modelPricing"); const { chatPrompt, sourceIdentifier } = require("./index"); const { EmbedChats } = require("../../models/embedChats"); const { @@ -34,6 +35,7 @@ async function streamChatWithForEmbed( const uuid = uuidv4(); const { connector: LLMConnector, + routingMetadata, prefetchedContext, error: routerError, } = await resolveLLMConnectorForEmbed({ @@ -195,7 +197,11 @@ async function streamChatWithForEmbed( temperature: embed.workspace?.openAiTemp ?? LLMConnector.defaultTemp, }); completeText = textResponse; - metrics = performanceMetrics; + metrics = addChatCostToMetrics(performanceMetrics, { + routingMetadata, + workspace: embed.workspace, + connector: LLMConnector, + }); writeResponseChunk(response, { uuid, sources: [], @@ -212,7 +218,11 @@ async function streamChatWithForEmbed( uuid, sources: [], }); - metrics = stream.metrics; + metrics = addChatCostToMetrics(stream.metrics, { + routingMetadata, + workspace: embed.workspace, + connector: LLMConnector, + }); } await EmbedChats.new({ @@ -273,16 +283,18 @@ async function resolveLLMConnectorForEmbed({ include: true, }); - const { connector, prefetchedContext } = await resolveProviderConnector({ - workspace, - prompt: message, - chatHistoryOverride: embedHistory, - // +1 to include the current in-flight message to ensure routing rules are evaluated against the real total. - messageCountOverride: embedMessageCount + 1, - }); + const { connector, routingMetadata, prefetchedContext } = + await resolveProviderConnector({ + workspace, + prompt: message, + chatHistoryOverride: embedHistory, + // +1 to include the current in-flight message to ensure routing rules are evaluated against the real total. + messageCountOverride: embedMessageCount + 1, + }); return { connector, + routingMetadata, prefetchedContext: prefetchedContext ? { rawHistory: embedHistory.rawHistory, @@ -295,6 +307,7 @@ async function resolveLLMConnectorForEmbed({ } catch (routerError) { return { connector: null, + routingMetadata: null, prefetchedContext: null, error: `Model router error: ${routerError.message}`, }; diff --git a/server/utils/chats/openaiCompatible.js b/server/utils/chats/openaiCompatible.js index c6bf0e66ae9..a83811827a7 100644 --- a/server/utils/chats/openaiCompatible.js +++ b/server/utils/chats/openaiCompatible.js @@ -2,6 +2,7 @@ const { v4: uuidv4 } = require("uuid"); const { DocumentManager } = require("../DocumentManager"); const { WorkspaceChats } = require("../../models/workspaceChats"); const { getVectorDbClass, resolveProviderConnector } = require("../helpers"); +const { addChatCostToMetrics } = require("../helpers/modelPricing"); const { writeResponseChunk } = require("../helpers/chat/responses"); const { chatPrompt, sourceIdentifier } = require("./index"); const { abortConnectorOnClientDisconnect } = require("../helpers/abortSignals"); @@ -19,18 +20,19 @@ async function chatSync({ const uuid = uuidv4(); const chatMode = workspace?.chatMode ?? "automatic"; - const { connector: LLMConnector } = await resolveProviderConnector({ - workspace, - prompt, - attachments, - chatHistoryOverride: { - rawHistory: history, - chatHistory: history, - }, - // Do not +1 to this, since OAI message history ends in a user message - // and does not need to re-include an uncounted user message. - messageCountOverride: history.length, - }); + const { connector: LLMConnector, routingMetadata } = + await resolveProviderConnector({ + workspace, + prompt, + attachments, + chatHistoryOverride: { + rawHistory: history, + chatHistory: history, + }, + // Do not +1 to this, since OAI message history ends in a user message + // and does not need to re-include an uncounted user message. + messageCountOverride: history.length, + }); const VectorDb = getVectorDbClass(); const hasVectorizedSpace = await VectorDb.hasNamespace(workspace.slug); @@ -172,13 +174,16 @@ async function chatSync({ }); // Send the text completion. - const { textResponse, metrics } = await LLMConnector.getChatCompletion( - messages, - { + const { textResponse, metrics: completionMetrics } = + await LLMConnector.getChatCompletion(messages, { temperature: temperature ?? workspace?.openAiTemp ?? LLMConnector.defaultTemp, - } - ); + }); + const metrics = addChatCostToMetrics(completionMetrics, { + routingMetadata, + workspace, + connector: LLMConnector, + }); if (!textResponse) { return formatJSON( @@ -232,18 +237,19 @@ async function streamChat({ const uuid = uuidv4(); const chatMode = workspace?.chatMode ?? "automatic"; - const { connector: LLMConnector } = await resolveProviderConnector({ - workspace, - prompt, - attachments, - chatHistoryOverride: { - rawHistory: history, - chatHistory: history, - }, - // Do not +1 to this, since OAI message history ends in a user message - // and does not need to re-include an uncounted user message. - messageCountOverride: history.length, - }); + const { connector: LLMConnector, routingMetadata } = + await resolveProviderConnector({ + workspace, + prompt, + attachments, + chatHistoryOverride: { + rawHistory: history, + chatHistory: history, + }, + // Do not +1 to this, since OAI message history ends in a user message + // and does not need to re-include an uncounted user message. + messageCountOverride: history.length, + }); // A disconnected client (aborted request, closed connection) should stop the // provider generating too, not just stop us reading the response. @@ -453,6 +459,11 @@ async function streamChat({ sources, } ); + const metrics = addChatCostToMetrics(stream.metrics, { + routingMetadata, + workspace, + connector: LLMConnector, + }); if (completeText?.length > 0) { const { chat } = await WorkspaceChats.new({ @@ -462,7 +473,7 @@ async function streamChat({ text: completeText, sources, type: chatMode, - metrics: stream.metrics, + metrics, attachments, }, }); @@ -482,7 +493,7 @@ async function streamChat({ chunked: true, model: workspace.slug, finish_reason: "stop", - usage: stream.metrics, + usage: metrics, } ) ); @@ -503,7 +514,7 @@ async function streamChat({ chunked: true, model: workspace.slug, finish_reason: "stop", - usage: stream.metrics, + usage: metrics, } ) ); diff --git a/server/utils/chats/stream.js b/server/utils/chats/stream.js index 0bed748251e..5590c3a9b05 100644 --- a/server/utils/chats/stream.js +++ b/server/utils/chats/stream.js @@ -3,6 +3,7 @@ const { DocumentManager } = require("../DocumentManager"); const { WorkspaceChats } = require("../../models/workspaceChats"); const { WorkspaceParsedFiles } = require("../../models/workspaceParsedFiles"); const { getVectorDbClass, resolveProviderConnector } = require("../helpers"); +const { addChatCostToMetrics } = require("../helpers/modelPricing"); const { writeResponseChunk } = require("../helpers/chat/responses"); const { abortConnectorOnClientDisconnect } = require("../helpers/abortSignals"); const { grepAgents } = require("./agents"); @@ -293,7 +294,11 @@ async function streamChatWithWorkspace( }); completeText = textResponse; - metrics = performanceMetrics; + metrics = addChatCostToMetrics(performanceMetrics, { + routingMetadata, + workspace, + connector: LLMConnector, + }); writeResponseChunk(response, { uuid, sources, @@ -312,7 +317,11 @@ async function streamChatWithWorkspace( uuid, sources, }); - metrics = stream.metrics; + metrics = addChatCostToMetrics(stream.metrics, { + routingMetadata, + workspace, + connector: LLMConnector, + }); } if (completeText?.length > 0) { diff --git a/server/utils/helpers/currencyExchange/index.js b/server/utils/helpers/currencyExchange/index.js new file mode 100644 index 00000000000..da8f03f2be4 --- /dev/null +++ b/server/utils/helpers/currencyExchange/index.js @@ -0,0 +1,194 @@ +const path = require("path"); +const fs = require("fs"); + +/** + * Currencies the Frankfurter API can convert USD into. Kept as a static list + * so the frontend picker and the `display_currency` setting validation never + * accept a currency we cannot actually convert to. Mirrored in + * `frontend/src/utils/currency.js` - keep the two lists in sync. + * @type {string[]} + */ +const SUPPORTED_CURRENCIES = [ + "USD", + "AUD", + "BGN", + "BRL", + "CAD", + "CHF", + "CNY", + "CZK", + "DKK", + "EUR", + "GBP", + "HKD", + "HUF", + "IDR", + "ILS", + "INR", + "ISK", + "JPY", + "KRW", + "MXN", + "MYR", + "NOK", + "NZD", + "PHP", + "PLN", + "RON", + "SEK", + "SGD", + "THB", + "TRY", + "ZAR", +]; + +/** + * Checks that a currency code is one we can convert USD into. + * @param {unknown} code + * @returns {boolean} + */ +function isSupportedCurrency(code) { + return typeof code === "string" && SUPPORTED_CURRENCIES.includes(code); +} + +class CurrencyExchange { + static instance = null; + // FX rates move, but not enough to matter for an informational display - + // costs are stored in USD and only converted at render time. + static expiryMs = 1000 * 60 * 60 * 24 * 30; // 30 days + static remoteUrl = "https://api.frankfurter.dev/v1/latest?base=USD"; + // Unlike ModelPricing's boot-time refresh, this fetch runs inside the + // /system/exchange-rates request path - a hung upstream must fail fast + // (serving stale or null rates) instead of stalling user requests. + static fetchTimeoutMs = 10_000; + + cacheLocation = path.resolve( + process.env.STORAGE_DIR + ? path.resolve(process.env.STORAGE_DIR, "currency") + : path.resolve(__dirname, `../../../storage/currency`) + ); + cacheFilePath = path.resolve(this.cacheLocation, "exchange-rates.json"); + cacheFileExpiryPath = path.resolve(this.cacheLocation, ".cached_at"); + + /** @type {Record|null} - memoized units-per-USD rates keyed by currency code */ + #rates = null; + /** @type {Promise|null} - de-dupes concurrent refreshes */ + #inflightRefresh = null; + + constructor() { + if (CurrencyExchange.instance) return CurrencyExchange.instance; + CurrencyExchange.instance = this; + if (!fs.existsSync(this.cacheLocation)) + fs.mkdirSync(this.cacheLocation, { recursive: true }); + this.#loadFromDisk(); + } + + log(text, ...args) { + if (process.env.NODE_ENV === "test") return; + console.log(`\x1b[36m[CurrencyExchange]\x1b[0m ${text}`, ...args); + } + + /** + * Checks if the disk cache is older than the expiry time (or missing entirely). + * @returns {boolean} + */ + get isCacheStale() { + if (!fs.existsSync(this.cacheFileExpiryPath)) return true; + const cachedAt = Number(fs.readFileSync(this.cacheFileExpiryPath, "utf8")); + // A corrupted timestamp would make every comparison false (NaN) and the + // cache permanently fresh - treat it as stale so it self-heals. + if (!Number.isFinite(cachedAt)) return true; + return Date.now() - cachedAt > CurrencyExchange.expiryMs; + } + + /** Loads and memoizes the sanitized rates map from the disk cache, if present. */ + #loadFromDisk() { + try { + if (!fs.existsSync(this.cacheFilePath)) return; + this.#rates = this.#sanitize( + JSON.parse(fs.readFileSync(this.cacheFilePath, { encoding: "utf8" })) + ); + } catch (error) { + this.log("Failed to read exchange rate cache from disk", error?.message); + this.#rates = null; + } + } + + /** + * Keeps only supported currencies whose rate is a positive, finite number + * so a malformed payload can never produce a nonsense conversion. + * Returns null when nothing usable remains. + * @param {unknown} rates + * @returns {Record|null} + */ + #sanitize(rates) { + if (!rates || typeof rates !== "object") return null; + const sanitized = {}; + for (const code of SUPPORTED_CURRENCIES) { + const rate = rates[code]; + if (typeof rate === "number" && Number.isFinite(rate) && rate > 0) + sanitized[code] = rate; + } + // Frankfurter omits the base currency from its response. + sanitized.USD = 1; + return Object.keys(sanitized).length > 1 ? sanitized : null; + } + + /** + * Fetches the latest USD-based rates from Frankfurter and caches them to + * disk + memory. On failure the previously loaded (possibly stale) rates + * are left in place and `.cached_at` is not written, so the next call + * retries the remote source. + */ + async #refresh() { + try { + const response = await fetch(CurrencyExchange.remoteUrl, { + signal: AbortSignal.timeout(CurrencyExchange.fetchTimeoutMs), + }); + if (response.status !== 200) + throw new Error( + `Failed to fetch remote exchange rates - status ${response.status}` + ); + + const data = await response.json(); + const rates = this.#sanitize(data?.rates); + if (!rates) + throw new Error("Remote exchange rate data contained no usable rates"); + + this.#rates = rates; + await Promise.all([ + fs.promises.writeFile(this.cacheFilePath, JSON.stringify(rates)), + fs.promises.writeFile(this.cacheFileExpiryPath, Date.now().toString()), + ]); + this.log("Remote exchange rates synced and cached."); + } catch (error) { + this.log("Error syncing remote exchange rates", error?.message); + } + } + + /** + * Returns the units-per-USD exchange rates, refreshing from the remote + * source when the disk cache is stale or missing. Serves stale rates when + * the remote is unreachable, and null when no rates have ever been fetched + * (callers should then display USD). + * @returns {Promise|null>} + */ + async getRates() { + if (this.#rates && !this.isCacheStale) return { ...this.#rates }; + + this.#inflightRefresh ??= this.#refresh().finally( + () => (this.#inflightRefresh = null) + ); + await this.#inflightRefresh; + return this.#rates ? { ...this.#rates } : null; + } +} + +const CURRENCY_EXCHANGE = new CurrencyExchange(); + +module.exports = { + CurrencyExchange, + CURRENCY_EXCHANGE, + SUPPORTED_CURRENCIES, + isSupportedCurrency, +}; diff --git a/server/utils/helpers/modelPricing/index.js b/server/utils/helpers/modelPricing/index.js new file mode 100644 index 00000000000..9de658b3420 --- /dev/null +++ b/server/utils/helpers/modelPricing/index.js @@ -0,0 +1,422 @@ +const path = require("path"); +const fs = require("fs"); +const { toNonNegativeNumber } = require("../numbers"); + +/** + * @typedef {Object} ModelCost - USD per 1,000,000 tokens (models.dev conventions) + * @property {number} input - cost per 1M prompt tokens + * @property {number} output - cost per 1M completion tokens + * @property {number} [cache_read] - cost per 1M cached prompt tokens read (falls back to `input`) + * @property {number} [cache_write] - cost per 1M prompt tokens written to cache (falls back to `input`) + * @property {number} [reasoning] - cost per 1M reasoning tokens (falls back to `output`) + * @property {Array} [tiers] - long-context pricing tiers + * @property {ModelCost} [context_over_200k] - legacy long-context pricing (prefer `tiers`) + */ + +/** + * @typedef {Object} CostBreakdown + * @property {number} inputCost - USD cost of the prompt tokens + * @property {number} outputCost - USD cost of the completion tokens + * @property {number} totalCost - USD sum of input and output costs + */ + +/** + * Providers that are local or self-hosted - inference is always free + * so we never consult the pricing data for them. + * @type {Set} + */ +const FREE_PROVIDERS = new Set([ + "ollama", + "lmstudio", + "localai", + "koboldcpp", + "textgenwebui", + "omlx", + "lemonade", + "docker-model-runner", +]); + +/** + * AnythingLLM provider slug -> models.dev provider id. + * Slugs absent from this map (and from FREE_PROVIDERS) resolve to "unknown cost" + * (eg: generic-openai, litellm, foundry, privatemode, nvidia-nim, apipie, + * cometapi, giteeai, ppio, sambanova - not present on models.dev as of Aug 2026). + * A missing/stale mapping degrades to "unknown", never a wrong price. + * @type {Record} + */ +const PROVIDER_ID_MAP = { + openai: "openai", + azure: "azure", // best-effort: only matches when deployment name === model id + anthropic: "anthropic", + gemini: "google", + togetherai: "togetherai", + fireworksai: "fireworks-ai", + mistral: "mistral", + perplexity: "perplexity", + openrouter: "openrouter", + novita: "novita-ai", + groq: "groq", + cohere: "cohere", + bedrock: "amazon-bedrock", + deepseek: "deepseek", + xai: "xai", + moonshotai: "moonshotai", + zai: "zai", + minimax: "minimax", + cerebras: "cerebras", +}; + +class ModelPricing { + static instance = null; + static expiryMs = 1000 * 60 * 60 * 24 * 3; // 3 days + static remoteUrl = "https://models.dev/api.json"; + + /** + * Loads the pricing snapshot bundled with `@opencode-ai/models`. The package + * is ESM-only so it must be dynamically imported - never require()'d. + * Static so tests can inject a fake importer (Jest's VM cannot execute + * dynamic import without --experimental-vm-modules). + * @returns {Promise<{providers: Record}>} + */ + static importSnapshot = () => import("@opencode-ai/models/snapshot"); + + cacheLocation = path.resolve( + process.env.STORAGE_DIR + ? path.resolve(process.env.STORAGE_DIR, "models", "pricing") + : path.resolve(__dirname, `../../../storage/models/pricing`) + ); + cacheFilePath = path.resolve(this.cacheLocation, "model-pricing.json"); + cacheFileExpiryPath = path.resolve(this.cacheLocation, ".cached_at"); + cacheEtagPath = path.resolve(this.cacheLocation, ".etag"); + + /** @type {Record>|null} - memoized pricing map keyed by models.dev provider id */ + #pricing = null; + /** @type {Record>} - lazy per-provider lowercased model id -> real model id */ + #lowercaseIndexes = {}; + /** @type {Record>} - lazy per-provider normalized model id -> real model id (bedrock) */ + #normalizedIndexes = {}; + + constructor() { + if (ModelPricing.instance) return ModelPricing.instance; + ModelPricing.instance = this; + if (!fs.existsSync(this.cacheLocation)) + fs.mkdirSync(this.cacheLocation, { recursive: true }); + + this.#loadFromDisk(); + if (this.isCacheStale || !this.#pricing) { + this.#refresh().catch((err) => + this.log("Background pricing refresh failed:", err?.message) + ); + } + } + + log(text, ...args) { + if (process.env.NODE_ENV === "test") return; + console.log(`\x1b[36m[ModelPricing]\x1b[0m ${text}`, ...args); + } + + /** + * Checks if the disk cache is older than the expiry time (or missing entirely). + * @returns {boolean} + */ + get isCacheStale() { + if (!fs.existsSync(this.cacheFileExpiryPath)) return true; + const cachedAt = Number(fs.readFileSync(this.cacheFileExpiryPath, "utf8")); + // A corrupted timestamp would make every comparison false (NaN) and the + // cache permanently fresh - treat it as stale so it self-heals. + if (!Number.isFinite(cachedAt)) return true; + return Date.now() - cachedAt > ModelPricing.expiryMs; + } + + /** Loads and memoizes the slimmed pricing map from the disk cache, if present. */ + #loadFromDisk() { + try { + if (!fs.existsSync(this.cacheFilePath)) return; + this.#pricing = JSON.parse( + fs.readFileSync(this.cacheFilePath, { encoding: "utf8" }) + ); + } catch (error) { + this.log("Failed to read pricing cache from disk", error?.message); + this.#pricing = null; + } + } + + /** + * Fetches the latest pricing data from models.dev, slims it, and caches it + * to disk + memory. Uses a conditional GET (ETag) so an unchanged upstream + * only bumps the cache expiry. Falls back to the bundled snapshot from + * `@opencode-ai/models` when the remote is unreachable and no disk cache exists. + */ + async #refresh() { + try { + const headers = {}; + if (this.#pricing && fs.existsSync(this.cacheEtagPath)) { + const etag = fs.readFileSync(this.cacheEtagPath, "utf8").trim(); + if (etag) headers["If-None-Match"] = etag; + } + + const response = await fetch(ModelPricing.remoteUrl, { headers }); + if (response.status === 304) { + await fs.promises.writeFile( + this.cacheFileExpiryPath, + Date.now().toString() + ); + this.log("Remote pricing unchanged (304) - cache expiry bumped."); + return; + } + + if (response.status !== 200) + throw new Error( + `Failed to fetch remote pricing data - status ${response.status}` + ); + + const data = await response.json(); + const pricing = this.#slim(data); + if (!Object.keys(pricing).length) + throw new Error("Remote pricing data contained no usable cost data"); + + this.#pricing = pricing; + this.#lowercaseIndexes = {}; + this.#normalizedIndexes = {}; + + const etag = response.headers.get("etag"); + await Promise.all([ + fs.promises.writeFile(this.cacheFilePath, JSON.stringify(pricing)), + fs.promises.writeFile(this.cacheFileExpiryPath, Date.now().toString()), + etag + ? fs.promises.writeFile(this.cacheEtagPath, etag) + : Promise.resolve(), + ]); + this.log("Remote pricing data synced and cached."); + } catch (error) { + this.log("Error syncing remote pricing data", error?.message); + if (!this.#pricing) await this.#loadSnapshot(); + } + } + + /** + * Loads the pricing data bundled with the `@opencode-ai/models` package as an + * offline/air-gapped fallback. We intentionally do not write `.cached_at` + * here so the next boot retries the remote source. + */ + async #loadSnapshot() { + try { + const { providers } = await ModelPricing.importSnapshot(); + this.#pricing = this.#slim(providers); + this.#lowercaseIndexes = {}; + this.#normalizedIndexes = {}; + this.log("Loaded bundled pricing snapshot (offline fallback)."); + } catch (error) { + this.log("Bundled pricing snapshot failed to load", error?.message); + } + } + + /** + * Slims the full models.dev provider map down to only cost data. + * Models with an absent or null cost object are dropped entirely so a + * lookup miss cleanly signals "unknown pricing". + * @param {Record}>} apiJson + * @returns {Record>} + */ + #slim(apiJson = {}) { + const slimmed = {}; + for (const [providerId, provider] of Object.entries(apiJson)) { + if (!provider?.models || typeof provider.models !== "object") continue; + for (const [modelId, model] of Object.entries(provider.models)) { + if (!model?.cost || typeof model.cost !== "object") continue; + if (typeof model.cost.input !== "number") continue; + slimmed[providerId] ??= {}; + slimmed[providerId][modelId] = model.cost; + } + } + return slimmed; + } + + /** + * Strips the region prefix and version suffix from a Bedrock model id since + * both our stored ids and models.dev's keys are inconsistently prefixed. + * eg: `us.anthropic.claude-sonnet-4-v1:0` -> `anthropic.claude-sonnet-4` + * @param {string} modelId + * @returns {string} + */ + #normalizeBedrockId(modelId = "") { + return modelId + .replace(/^(us|eu|ap|apac|jp|au|global)\./, "") + .replace(/-v\d+(:\d+)?$/, ""); + } + + /** + * Finds the cost object for a model under a models.dev provider id. + * Resolution order: exact match -> case-insensitive match -> bedrock + * region/version normalization (bedrock only). + * @param {string} providerId - models.dev provider id + * @param {string} providerSlug - AnythingLLM provider slug + * @param {string} model + * @returns {ModelCost|null} + */ + #findModelCost(providerId, providerSlug, model) { + const models = this.#pricing?.[providerId]; + if (!models) return null; + if (models[model]) return models[model]; + + if (!this.#lowercaseIndexes[providerId]) { + const index = {}; + for (const key of Object.keys(models)) index[key.toLowerCase()] = key; + this.#lowercaseIndexes[providerId] = index; + } + const caseMatch = this.#lowercaseIndexes[providerId][model.toLowerCase()]; + if (caseMatch) return models[caseMatch]; + + if (providerSlug === "bedrock") { + if (!this.#normalizedIndexes[providerId]) { + const index = {}; + for (const key of Object.keys(models)) { + const normalized = this.#normalizeBedrockId(key); + // Regional variants (au./eu./jp.) can be priced differently and all + // normalize to the same id - prefer the shortest (unprefixed) key. + if (!index[normalized] || key.length < index[normalized].length) + index[normalized] = key; + } + this.#normalizedIndexes[providerId] = index; + } + const normalizedMatch = + this.#normalizedIndexes[providerId][this.#normalizeBedrockId(model)]; + if (normalizedMatch) return models[normalizedMatch]; + } + + return null; + } + + /** + * Resolves the effective per-1M-token rates for a request, accounting for + * long-context pricing tiers (eg: Gemini >200k prompts). + * @param {ModelCost} cost + * @param {number} promptTokens + * @returns {{input: number, output: number}|null} + */ + #resolveRates(cost, promptTokens) { + let input = typeof cost.input === "number" ? cost.input : null; + let output = typeof cost.output === "number" ? cost.output : null; + + if (Array.isArray(cost.tiers)) { + const applicable = cost.tiers + .filter( + (t) => t?.tier?.type === "context" && promptTokens > t.tier.size + ) + .sort((a, b) => a.tier.size - b.tier.size) + .pop(); + if (applicable) { + input = applicable.input ?? input; + output = applicable.output ?? output; + } + } else if (cost.context_over_200k && promptTokens > 200_000) { + input = cost.context_over_200k.input ?? input; + output = cost.context_over_200k.output ?? output; + } + + if (typeof input !== "number" || typeof output !== "number") return null; + return { input, output }; + } + + /** + * Calculates the USD cost of a completion for a given provider slug + model. + * + * Returns zeros for local/self-hosted providers, `null` when pricing is + * unknown (unmapped provider, unknown model). Callers should treat `null` + * as "do not report cost" - absence is how the UI distinguishes unknown + * from free. + * + * Note: only flat prompt/completion rates are applied today since our + * metrics do not track cache or reasoning token counts. If they ever do, + * models.dev conventions are `reasoning ?? output` and + * `cache_read`/`cache_write ?? input` for the fallback rates. + * @param {string|null} providerSlug - AnythingLLM provider slug (eg: "openai" - NOT the class name) + * @param {string|null} model - model id used for the completion + * @param {{prompt_tokens?: number, completion_tokens?: number}} usage + * @returns {CostBreakdown|null} + */ + getCostBreakdown(providerSlug = null, model = null, usage = {}) { + if (!providerSlug) return null; + if (FREE_PROVIDERS.has(providerSlug)) + return { inputCost: 0, outputCost: 0, totalCost: 0 }; + + const providerId = PROVIDER_ID_MAP[providerSlug]; + if (!providerId || !model || !this.#pricing) return null; + + const cost = this.#findModelCost(providerId, providerSlug, model); + if (!cost) return null; + + // Chat metrics are not sanitized upstream like agent usage is, so a + // provider reporting a negative or non-finite count must not produce a + // negative or infinite cost. + const promptTokens = toNonNegativeNumber(usage?.prompt_tokens); + const completionTokens = toNonNegativeNumber(usage?.completion_tokens); + const rates = this.#resolveRates(cost, promptTokens); + if (!rates) return null; + + const inputCost = (promptTokens / 1_000_000) * rates.input; + const outputCost = (completionTokens / 1_000_000) * rates.output; + return { inputCost, outputCost, totalCost: inputCost + outputCost }; + } +} + +const MODEL_PRICING = new ModelPricing(); + +/** + * Returns a new metrics object with `inputCost`/`outputCost`/`totalCost` + * (USD) added when the price of the provider + model is known. Returns the + * metrics unchanged when pricing is unknown so the fields stay absent. + * @param {object} metrics - a chat metrics object with `prompt_tokens`/`completion_tokens` + * @param {{provider?: string|null, model?: string|null}} identity - provider slug + model used + * @returns {object} + */ +function addCostToMetrics( + metrics = {}, + { provider = null, model = null } = {} +) { + if (!metrics || typeof metrics !== "object" || !Object.keys(metrics).length) + return metrics; + + const breakdown = MODEL_PRICING.getCostBreakdown( + provider, + model ?? metrics.model, + { + prompt_tokens: metrics.prompt_tokens, + completion_tokens: metrics.completion_tokens, + } + ); + return breakdown ? { ...metrics, ...breakdown } : metrics; +} + +/** + * Convenience wrapper for the chat flows: resolves the provider slug + model + * that actually served the completion (router delegate first, then workspace + * overrides, then the system default) and decorates the metrics with cost. + * @param {object} metrics - a chat metrics object with `prompt_tokens`/`completion_tokens` + * @param {Object} opts + * @param {object|null} [opts.routingMetadata] - from `resolveProviderConnector` when the workspace uses the model router + * @param {object|null} [opts.workspace] - the workspace record the chat ran in + * @param {object|null} [opts.connector] - the LLM connector instance used + * @returns {object} + */ +function addChatCostToMetrics( + metrics = {}, + { routingMetadata = null, workspace = null, connector = null } = {} +) { + return addCostToMetrics(metrics, { + provider: + routingMetadata?.routedTo?.provider ?? + workspace?.chatProvider ?? + process.env.LLM_PROVIDER, + model: routingMetadata?.routedTo?.model ?? connector?.model, + }); +} + +module.exports = { + ModelPricing, + MODEL_PRICING, + addCostToMetrics, + addChatCostToMetrics, + FREE_PROVIDERS, + PROVIDER_ID_MAP, +}; diff --git a/server/utils/helpers/numbers.js b/server/utils/helpers/numbers.js new file mode 100644 index 00000000000..9d6c21f8ccd --- /dev/null +++ b/server/utils/helpers/numbers.js @@ -0,0 +1,15 @@ +/** + * Coerces a value into a finite, non-negative number. Provider-reported + * usage metrics and token counts arrive in inconsistent shapes (missing + * keys, numeric strings, nulls, negative or non-finite values), so anything + * that does not resolve to a usable number becomes 0. + * @param {unknown} value + * @returns {number} + */ +function toNonNegativeNumber(value) { + const number = Number(value); + if (!Number.isFinite(number) || number < 0) return 0; + return number; +} + +module.exports = { toNonNegativeNumber }; diff --git a/server/utils/telegramBot/chat/stream.js b/server/utils/telegramBot/chat/stream.js index 4181a2fefac..cc48b87e41d 100644 --- a/server/utils/telegramBot/chat/stream.js +++ b/server/utils/telegramBot/chat/stream.js @@ -1,5 +1,6 @@ const { WorkspaceChats } = require("../../../models/workspaceChats"); const { getVectorDbClass, resolveProviderConnector } = require("../../helpers"); +const { addChatCostToMetrics } = require("../../helpers/modelPricing"); const { DocumentManager } = require("../../DocumentManager"); const { sourceIdentifier, @@ -97,12 +98,13 @@ async function streamResponse({ ctx.bot.sendChatAction(chatId, "typing").catch(() => {}); }, 4000); - const { connector: LLMConnector } = await resolveProviderConnector({ - workspace, - prompt: message, - thread, - attachments, - }); + const { connector: LLMConnector, routingMetadata } = + await resolveProviderConnector({ + workspace, + prompt: message, + thread, + attachments, + }); const VectorDb = getVectorDbClass(); const embeddingsCount = await VectorDb.namespaceCount(workspace.slug); @@ -147,6 +149,7 @@ async function streamResponse({ try { const { completeText, metrics } = await generateResponse({ LLMConnector, + routingMetadata, messages, workspace, ctx, @@ -260,6 +263,7 @@ async function buildSearchContext({ */ async function generateResponse({ LLMConnector, + routingMetadata = null, messages, workspace, ctx, @@ -296,6 +300,11 @@ async function generateResponse({ await sendFormattedMessage(ctx.bot, chatId, completeText); } + metrics = addChatCostToMetrics(metrics, { + routingMetadata, + workspace, + connector: LLMConnector, + }); return { completeText, metrics }; } diff --git a/server/yarn.lock b/server/yarn.lock index e900a3d9843..0f855f516c6 100644 --- a/server/yarn.lock +++ b/server/yarn.lock @@ -862,6 +862,11 @@ zod "^3.25 || ^4.0" zod-to-json-schema "^3.25.0" +"@opencode-ai/models@^0.0.34": + version "0.0.34" + resolved "https://registry.yarnpkg.com/@opencode-ai/models/-/models-0.0.34.tgz#0115cd4d098afb1421b178f8d28bb1959d81f168" + integrity sha512-+wpkzEikhaGG97z9XjbLTz9jBqOd5vGV+6LTFhAtIDXmGq+4WXN3bv6Yc/fcnebR4b7BYLCSyeyctweuSRzLSA== + "@pdf-lib/standard-fonts@^1.0.0": version "1.0.0" resolved "https://registry.yarnpkg.com/@pdf-lib/standard-fonts/-/standard-fonts-1.0.0.tgz#8ba691c4421f71662ed07c9a0294b44528af2d7f"