Expose hosted-proxy store size and flush-duration metrics

CHANGELOG.md

@@ -17,6 +17,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/)
 - The OAuth consent page now shows where the user will be sent after approving — the client's callback host, or "an application on this device" for local clients.
 - IPv6 loopback (`http://[::1]:…`) redirect URIs are now accepted at client registration, per RFC 8252.
 - Hosted OAuth proxy: support for Client ID Metadata Documents (CIMD). CIMD-capable clients connect using a stable, vendor-hosted client identity, with no per-client redirect entry to curate. The proxy trusts the verified first-party document hosts by default; add more with `MCP_OAUTH_CIMD_ALLOWED_HOSTS`.
+- Metrics: the `/metrics` endpoint now reports the hosted OAuth proxy store's size and flush cost — `mcp_proxy_store_upstream_tokens` and `mcp_proxy_store_clients` gauges, an `mcp_proxy_store_flush_duration_seconds` histogram, and an `mcp_proxy_store_flush_failures_total` counter — so operators can watch the store grow, see how long each persistence write takes, and alert when a write fails.
 
 ### Changed
 

docs/operations.md

@@ -80,6 +80,10 @@ Exposed series:
 - `mcp_upstream_status_total{tool,wiki,status}` — counter of upstream MediaWiki HTTP status codes.
 - `mcp_active_sessions` — gauge of active StreamableHTTP MCP sessions.
 - `mcp_ready_failures_total` — counter of `/ready` probes that returned non-200.
+- `mcp_proxy_store_upstream_tokens` — gauge of upstream MediaWiki tokens held in the hosted OAuth proxy store. This set grows with cumulative sign-ins over the process lifetime; watch it to size memory and the flush cost below.
+- `mcp_proxy_store_clients` — gauge of registered clients held in the hosted OAuth proxy store. FIFO-capped at 10,000, so this plateaus rather than growing without bound.
+- `mcp_proxy_store_flush_duration_seconds` — histogram of hosted-proxy store durable-flush durations (serialize + encrypt + write). Every upstream-token write flushes the whole store synchronously, so this scales with the token count above. Records successful flushes only.
+- `mcp_proxy_store_flush_failures_total` — counter of durable flushes that failed to write (a disk error). A failure means the most recent sign-in change is held in memory only and would be lost on restart; alert on any increase.
 
 The endpoint is **unauthenticated**. Restrict reverse-proxy access to your scrape network only — most Kubernetes-style deployments expose `/metrics` on a separate port or path that isn't routable from the public ingress.
 

src/auth/authorizationServer/proxyStore.ts

@@ -44,6 +44,13 @@ export interface DurableSnapshot {
 	upstream: Array<[string, UpstreamToken]>;
 }
 
+// Live counts of the durable slice, for observability. `upstreamTokens` is the
+// unbounded one (it grows with cumulative sign-ins); `clients` is FIFO-capped.
+export interface ProxyStoreStats {
+	upstreamTokens: number;
+	clients: number;
+}
+
 const TXN_TTL_MS = 15 * 60 * 1000;
 const CODE_TTL_MS = 5 * 60 * 1000;
 
@@ -67,6 +74,7 @@ export interface ProxyStore {
 	deleteUpstreamToken(id: string): void;
 	beginRefreshRotation(id: string, expectedRefreshId: string): boolean;
 	finishRefreshRotation(id: string, newRefreshId?: string): void;
+	stats(): ProxyStoreStats;
 }
 
 interface Expiring<T> {
@@ -108,6 +116,10 @@ export class InMemoryProxyStore implements ProxyStore {
 		return this.clients.get(id);
 	}
 
+	public stats(): ProxyStoreStats {
+		return { upstreamTokens: this.upstream.size, clients: this.clients.size };
+	}
+
 	public putTransaction(id: string, t: TransactionRecord, ttlMs = TXN_TTL_MS): void {
 		this.txns.set(id, { value: t, expiresAt: this.now() + ttlMs });
 	}

src/auth/authorizationServer/proxyStorePersistence.ts

@@ -1,6 +1,8 @@
 import { mkdirSync, readFileSync, renameSync, writeFileSync } from 'node:fs';
 import * as path from 'node:path';
+import { performance } from 'node:perf_hooks';
 import { isErrnoException } from '../../errors/isErrnoException.js';
+import { recordStoreFlush, recordStoreFlushFailure } from '../../runtime/metrics.js';
 import { getProxyStorePath } from '../paths.js';
 import type { ProxyConfig } from './proxyConfig.js';
 import { deriveKey, decrypt, encrypt } from './proxyStoreCrypto.js';
@@ -10,6 +12,7 @@ import {
 	type CodeRecord,
 	type DurableSnapshot,
 	type ProxyStore,
+	type ProxyStoreStats,
 	type TransactionRecord,
 	type UpstreamToken,
 } from './proxyStore.js';
@@ -91,6 +94,10 @@ export class PersistentProxyStore implements ProxyStore {
 		return this.inner.getClient(id);
 	}
 
+	public stats(): ProxyStoreStats {
+		return this.inner.stats();
+	}
+
 	public putTransaction(id: string, t: TransactionRecord, ttlMs?: number): void {
 		this.inner.putTransaction(id, t, ttlMs);
 	}
@@ -177,6 +184,7 @@ export class PersistentProxyStore implements ProxyStore {
 	}
 
 	private flushSync(): void {
+		const start = performance.now();
 		try {
 			const json = JSON.stringify(this.inner.snapshotDurable());
 			const blob = encrypt(this.key, Buffer.from(json, 'utf8'));
@@ -185,12 +193,16 @@ export class PersistentProxyStore implements ProxyStore {
 			writeFileSync(tmp, blob, { mode: 0o600 });
 			renameSync(tmp, this.file);
 			this.dirty = false;
+			recordStoreFlush(performance.now() - start);
 		} catch (err: unknown) {
 			// Best-effort durability: a disk failure must not break the live request. The
 			// record stays valid in memory; persistence is re-attempted by the next
 			// whole-snapshot flush — the deferred `dirty` flag drives it on the
 			// registration path, and any later token write-through re-persists the whole
 			// snapshot on the token path (where `dirty` is usually already clear).
+			// Count the failure so a slow-and-failing flush is visible to /metrics, not
+			// only in the onError log. inc() never throws, so it stays out of the way.
+			recordStoreFlushFailure();
 			try {
 				this.onError(err instanceof Error ? err : new Error(String(err)));
 			} catch {

src/runtime/metrics.ts

@@ -10,29 +10,48 @@ export interface RecordToolCallInput {
 	readonly upstreamStatus: number | undefined;
 }
 
+// Structural shape of the proxy-store size snapshot the gauges read on scrape.
+// Defined locally so runtime/ keeps no dependency on auth/; it matches
+// InMemoryProxyStore.stats() by structure at the wiring site.
+type ProxyStoreStats = { readonly upstreamTokens: number; readonly clients: number };
+
 interface Recorder {
 	recordToolCall(input: RecordToolCallInput): void;
 	recordReadyFailure(): void;
 	setSessionsProvider(fn: () => number): void;
+	recordStoreFlush(durationMs: number): void;
+	recordStoreFlushFailure(): void;
+	setProxyStoreStatsProvider(fn: () => ProxyStoreStats): void;
 	getMetricsHandler(): RequestHandler | undefined;
 }
 
 const DURATION_BUCKETS_SECONDS: readonly number[] = [
 	0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10,
 ];
 
+// A proxy-store flush is a synchronous serialize + AES-GCM encrypt + file write:
+// sub-millisecond for a small store, rising toward tens of ms as the durable slice
+// grows. Buckets start finer and below the tool-call set to resolve that range.
+const FLUSH_BUCKETS_SECONDS: readonly number[] = [
+	0.0005, 0.001, 0.0025, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1,
+];
+
 function makeDisabledRecorder(): Recorder {
 	return {
 		recordToolCall: () => {},
 		recordReadyFailure: () => {},
 		setSessionsProvider: () => {},
+		recordStoreFlush: () => {},
+		recordStoreFlushFailure: () => {},
+		setProxyStoreStatsProvider: () => {},
 		getMetricsHandler: () => undefined,
 	};
 }
 
 function makeLiveRecorder(): Recorder {
 	const registry = new Registry();
 	let sessionsProvider: (() => number) | undefined;
+	let storeStatsProvider: (() => ProxyStoreStats) | undefined;
 
 	const toolCalls = new Counter({
 		name: 'mcp_tool_calls_total',
@@ -71,6 +90,37 @@ function makeLiveRecorder(): Recorder {
 		},
 	});
 
+	const storeFlushDuration = new Histogram({
+		name: 'mcp_proxy_store_flush_duration_seconds',
+		help: 'Duration of a hosted OAuth proxy store durable flush (serialize + encrypt + write) in seconds.',
+		buckets: [...FLUSH_BUCKETS_SECONDS],
+		registers: [registry],
+	});
+
+	const storeFlushFailures = new Counter({
+		name: 'mcp_proxy_store_flush_failures_total',
+		help: 'Total number of hosted OAuth proxy store durable flushes that failed to write.',
+		registers: [registry],
+	});
+
+	new Gauge({
+		name: 'mcp_proxy_store_upstream_tokens',
+		help: 'Number of upstream MediaWiki tokens held in the hosted OAuth proxy store.',
+		registers: [registry],
+		collect() {
+			this.set(storeStatsProvider ? storeStatsProvider().upstreamTokens : 0);
+		},
+	});
+
+	new Gauge({
+		name: 'mcp_proxy_store_clients',
+		help: 'Number of registered clients held in the hosted OAuth proxy store.',
+		registers: [registry],
+		collect() {
+			this.set(storeStatsProvider ? storeStatsProvider().clients : 0);
+		},
+	});
+
 	const handler: RequestHandler = async (_req, res) => {
 		res.set('Content-Type', registry.contentType);
 		res.status(200).send(await registry.metrics());
@@ -94,6 +144,15 @@ function makeLiveRecorder(): Recorder {
 		setSessionsProvider(fn) {
 			sessionsProvider = fn;
 		},
+		recordStoreFlush(durationMs) {
+			storeFlushDuration.observe(durationMs / 1000);
+		},
+		recordStoreFlushFailure() {
+			storeFlushFailures.inc();
+		},
+		setProxyStoreStatsProvider(fn) {
+			storeStatsProvider = fn;
+		},
 		getMetricsHandler() {
 			return handler;
 		},
@@ -127,6 +186,18 @@ export function setSessionsProvider(fn: () => number): void {
 	recorder.setSessionsProvider(fn);
 }
 
+export function recordStoreFlush(durationMs: number): void {
+	recorder.recordStoreFlush(durationMs);
+}
+
+export function recordStoreFlushFailure(): void {
+	recorder.recordStoreFlushFailure();
+}
+
+export function setProxyStoreStatsProvider(fn: () => ProxyStoreStats): void {
+	recorder.setProxyStoreStatsProvider(fn);
+}
+
 export function getMetricsHandler(): RequestHandler | undefined {
 	return recorder.getMetricsHandler();
 }

src/transport/streamableHttp.ts

@@ -22,6 +22,7 @@ import {
 	isMetricsEnabled,
 	recordReadyFailure,
 	setSessionsProvider,
+	setProxyStoreStatsProvider,
 } from '../runtime/metrics.js';
 import { withRequestContext } from './requestContext.js';
 
@@ -1298,6 +1299,7 @@ export function buildApp(deps: BuildAppDeps): BuiltApp {
 	mountReadyEndpoint(app, { activeWiki: state.activeWiki, mwnProvider: state.mwnProvider });
 	mountMetricsEndpoint(app);
 	setSessionsProvider(() => Object.keys(sessions).length);
+	setProxyStoreStatsProvider(() => store.stats());
 
 	return { app, sessions, inFlight };
 }

tests/auth/authorizationServer/proxyStore.test.ts

@@ -1,6 +1,19 @@
 import { describe, it, expect } from 'vitest';
 import { InMemoryProxyStore } from '../../../src/auth/authorizationServer/proxyStore.js';
 
+describe('InMemoryProxyStore stats', () => {
+	it('reports upstream-token and client counts', () => {
+		const s = new InMemoryProxyStore();
+		expect(s.stats()).toEqual({ upstreamTokens: 0, clients: 0 });
+		s.putClient({ redirectUris: ['http://127.0.0.1/cb'], scopes: [], name: 'c' });
+		const id = s.putUpstreamToken({ accessToken: 'a', expiresAt: Date.now() + 1000 });
+		s.putUpstreamToken({ accessToken: 'b', expiresAt: Date.now() + 1000 });
+		expect(s.stats()).toEqual({ upstreamTokens: 2, clients: 1 });
+		s.deleteUpstreamToken(id);
+		expect(s.stats()).toEqual({ upstreamTokens: 1, clients: 1 });
+	});
+});
+
 describe('InMemoryProxyStore', () => {
 	it('registers and reads a client', () => {
 		const s = new InMemoryProxyStore();

tests/auth/authorizationServer/proxyStorePersistence.test.ts

@@ -2,6 +2,13 @@ import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';
 import * as fs from 'node:fs';
 import * as os from 'node:os';
 import * as path from 'node:path';
+import express from 'express';
+import request from 'supertest';
+import {
+	initMetrics,
+	getMetricsHandler,
+	__resetMetricsForTesting,
+} from '../../../src/runtime/metrics.js';
 import type { ProxyConfig } from '../../../src/auth/authorizationServer/proxyConfig.js';
 import { InMemoryProxyStore } from '../../../src/auth/authorizationServer/proxyStore.js';
 import {
@@ -45,6 +52,53 @@ describe('PersistentProxyStore', () => {
 		});
 	});
 
+	it('records a flush-duration metric on a durable write when metrics are enabled', async () => {
+		vi.stubEnv('MCP_METRICS', 'true');
+		__resetMetricsForTesting();
+		initMetrics();
+		try {
+			const s = make(file);
+			s.putUpstreamToken({ accessToken: 'a', expiresAt: 111 }); // write-through → flushSync
+			const handler = getMetricsHandler();
+			expect(handler).toBeDefined();
+			const app = express();
+			app.get('/metrics', handler!);
+			const res = await request(app).get('/metrics');
+			expect(res.text).toMatch(/mcp_proxy_store_flush_duration_seconds_count [1-9]/);
+		} finally {
+			__resetMetricsForTesting();
+		}
+	});
+
+	it('increments the flush-failure counter when a durable write cannot be persisted', async () => {
+		vi.stubEnv('MCP_METRICS', 'true');
+		__resetMetricsForTesting();
+		initMetrics();
+		try {
+			// Force flushSync to fail: put the store file under a path whose parent is a
+			// regular file, so the mkdirSync/write throws.
+			const blocker = path.join(dir, 'blocker');
+			fs.writeFileSync(blocker, 'x');
+			const onError = vi.fn();
+			const s = new PersistentProxyStore(
+				new InMemoryProxyStore(),
+				path.join(blocker, 's.enc'),
+				KEY,
+				onError,
+			);
+			s.putUpstreamToken({ accessToken: 'a', expiresAt: 111 }); // write-through flush fails
+			expect(onError).toHaveBeenCalledTimes(1); // the failure path ran, but did not throw
+			const handler = getMetricsHandler();
+			expect(handler).toBeDefined();
+			const app = express();
+			app.get('/metrics', handler!);
+			const res = await request(app).get('/metrics');
+			expect(res.text).toMatch(/mcp_proxy_store_flush_failures_total 1/);
+		} finally {
+			__resetMetricsForTesting();
+		}
+	});
+
 	it('persists a client registration via the coalesced deferred flush', async () => {
 		const s1 = make(file);
 		const c = s1.putClient({

tests/runtime/metrics.test.ts

@@ -9,6 +9,9 @@ import {
 	setSessionsProvider,
 	getMetricsHandler,
 	__resetMetricsForTesting,
+	recordStoreFlush,
+	recordStoreFlushFailure,
+	setProxyStoreStatsProvider,
 } from '../../src/runtime/metrics.js';
 
 describe('metrics module — disabled state', () => {
@@ -28,6 +31,11 @@ describe('metrics module — disabled state', () => {
 		).not.toThrow();
 		expect(() => recordReadyFailure()).not.toThrow();
 		expect(() => setSessionsProvider(() => 0)).not.toThrow();
+		expect(() => recordStoreFlush(1)).not.toThrow();
+		expect(() => recordStoreFlushFailure()).not.toThrow();
+		expect(() =>
+			setProxyStoreStatsProvider(() => ({ upstreamTokens: 0, clients: 0 })),
+		).not.toThrow();
 	});
 
 	it('getMetricsHandler returns undefined before init', () => {
@@ -147,6 +155,39 @@ describe('metrics module — enabled state', () => {
 		expect(body).toMatch(/mcp_active_sessions 0/);
 	});
 
+	it('observes mcp_proxy_store_flush_duration_seconds', async () => {
+		recordStoreFlush(5);
+		recordStoreFlush(5);
+		const body = await scrape();
+		expect(body).toContain('mcp_proxy_store_flush_duration_seconds_count 2');
+		expect(body).toContain('mcp_proxy_store_flush_duration_seconds_sum 0.01');
+	});
+
+	it('mcp_proxy_store gauges read the stats provider lazily at scrape', async () => {
+		let stats = { upstreamTokens: 2, clients: 1 };
+		setProxyStoreStatsProvider(() => stats);
+		let body = await scrape();
+		expect(body).toMatch(/mcp_proxy_store_upstream_tokens 2/);
+		expect(body).toMatch(/mcp_proxy_store_clients 1/);
+		stats = { upstreamTokens: 5, clients: 3 };
+		body = await scrape();
+		expect(body).toMatch(/mcp_proxy_store_upstream_tokens 5/);
+		expect(body).toMatch(/mcp_proxy_store_clients 3/);
+	});
+
+	it('counts mcp_proxy_store_flush_failures_total', async () => {
+		recordStoreFlushFailure();
+		recordStoreFlushFailure();
+		const body = await scrape();
+		expect(body).toMatch(/mcp_proxy_store_flush_failures_total 2/);
+	});
+
+	it('mcp_proxy_store gauges report 0 when no provider set', async () => {
+		const body = await scrape();
+		expect(body).toMatch(/mcp_proxy_store_upstream_tokens 0/);
+		expect(body).toMatch(/mcp_proxy_store_clients 0/);
+	});
+
 	it('initMetrics is idempotent', async () => {
 		expect(() => initMetrics()).not.toThrow();
 		expect(() => initMetrics()).not.toThrow();