HeyPuter · Salazareo · Jul 11, 2026 · Jul 11, 2026
diff --git a/src/backend/clients/database/MySQLDatabaseClient.ts b/src/backend/clients/database/MySQLDatabaseClient.ts
@@ -20,6 +20,7 @@
 import { readdirSync, readFileSync } from 'fs';
 import { isAbsolute, resolve as resolvePath } from 'path';
 import { createPool, type Pool } from 'mysql2';
+import { Span } from '../../util/span.js';
 import { AbstractDatabaseClient, type WriteResult } from './DatabaseClient';
 import { SQLBatcher } from './SQLBatcher.js';
 import { splitMysqlStatements } from './splitMysqlStatements.js';
@@ -135,6 +136,10 @@ export class MySQLDatabaseClient extends AbstractDatabaseClient {
     // Query interface
     // ------------------------------------------------------------------
 
+    // The db.* spans measure the logical query, including time queued in
+    // the SQLBatcher — the mysql2 auto-instrumentation only sees the
+    // coalesced multi-statement flush, so per-query latency lives here.
+    @Span('db.read', (query: string) => ({ 'db.statement': query }))
     override async read(
         query: string,
         params: unknown[] = [],
@@ -144,6 +149,7 @@ export class MySQLDatabaseClient extends AbstractDatabaseClient {
         return (result[0] as Record<string, unknown>[]) ?? [];
     }
 
+    @Span('db.pread', (query: string) => ({ 'db.statement': query }))
     override async pread(
         query: string,
         params: unknown[] = [],
@@ -153,6 +159,7 @@ export class MySQLDatabaseClient extends AbstractDatabaseClient {
         return (result[0] as Record<string, unknown>[]) ?? [];
     }
 
+    @Span('db.write', (query: string) => ({ 'db.statement': query }))
     override async write(
         query: string,
         params: unknown[] = [],
@@ -170,6 +177,9 @@ export class MySQLDatabaseClient extends AbstractDatabaseClient {
         };
     }
 
+    @Span('db.batchWrite', (entries: unknown[]) => ({
+        'db.batch_size': entries.length,
+    }))
     override async batchWrite(
         entries: { statement: string; values: unknown[] }[],
     ): Promise<void> {
@@ -195,6 +205,7 @@ export class MySQLDatabaseClient extends AbstractDatabaseClient {
         }
     }
 
+    @Span('db.tryHardRead', (query: string) => ({ 'db.statement': query }))
     override async tryHardRead(
         query: string,
         params: unknown[] = [],

diff --git a/src/backend/clients/database/PostgresDatabaseClient.ts b/src/backend/clients/database/PostgresDatabaseClient.ts
@@ -20,6 +20,7 @@
 import { readdirSync, readFileSync } from 'fs';
 import { isAbsolute, resolve as resolvePath } from 'path';
 import { Pool, type PoolConfig, type QueryResult } from 'pg';
+import { Span } from '../../util/span.js';
 import {
     AbstractDatabaseClient,
     type BatchEntry,
@@ -214,6 +215,7 @@ export class PostgresDatabaseClient extends AbstractDatabaseClient {
         return value;
     }
 
+    @Span('db.read', (query: string) => ({ 'db.statement': query }))
     override async read(
         query: string,
         params: unknown[] = [],
@@ -222,6 +224,7 @@ export class PostgresDatabaseClient extends AbstractDatabaseClient {
         return normalizePostgresRows(result);
     }
 
+    @Span('db.pread', (query: string) => ({ 'db.statement': query }))
     override async pread(
         query: string,
         params: unknown[] = [],
@@ -230,6 +233,7 @@ export class PostgresDatabaseClient extends AbstractDatabaseClient {
         return normalizePostgresRows(result);
     }
 
+    @Span('db.write', (query: string) => ({ 'db.statement': query }))
     override async write(
         query: string,
         params: unknown[] = [],
@@ -238,6 +242,9 @@ export class PostgresDatabaseClient extends AbstractDatabaseClient {
         return mapPostgresWriteResult(result);
     }
 
+    @Span('db.batchWrite', (entries: unknown[]) => ({
+        'db.batch_size': entries.length,
+    }))
     override async batchWrite(entries: BatchEntry[]): Promise<void> {
         if (entries.length === 0) return;
 
@@ -258,6 +265,7 @@ export class PostgresDatabaseClient extends AbstractDatabaseClient {
         }
     }
 
+    @Span('db.tryHardRead', (query: string) => ({ 'db.statement': query }))
     override async tryHardRead(
         query: string,
         params: unknown[] = [],

diff --git a/src/backend/clients/database/SqliteDatabaseClient.ts b/src/backend/clients/database/SqliteDatabaseClient.ts
@@ -21,6 +21,7 @@ import { existsSync, mkdirSync, readFileSync } from 'fs';
 import { basename, dirname, extname, join, resolve } from 'path';
 import { createContext, runInContext } from 'vm';
 import type { IConfig } from '../../types';
+import { Span } from '../../util/span.js';
 import { AbstractDatabaseClient, type WriteResult } from './DatabaseClient';
 
 const MIGRATIONS_DIR = resolve(__dirname, './migrations/sqlite');
@@ -136,6 +137,7 @@ export class SqliteDatabaseClient extends AbstractDatabaseClient {
     // Query interface
     // ------------------------------------------------------------------
 
+    @Span('db.read', (query: string) => ({ 'db.statement': query }))
     override async read(
         query: string,
         params: unknown[] = [],
@@ -156,6 +158,7 @@ export class SqliteDatabaseClient extends AbstractDatabaseClient {
         return this.read(query, params);
     }
 
+    @Span('db.write', (query: string) => ({ 'db.statement': query }))
     override async write(
         query: string,
         params: unknown[] = [],
@@ -172,6 +175,9 @@ export class SqliteDatabaseClient extends AbstractDatabaseClient {
         };
     }
 
+    @Span('db.batchWrite', (entries: unknown[]) => ({
+        'db.batch_size': entries.length,
+    }))
     override async batchWrite(
         entries: { statement: string; values: unknown[] }[],
     ): Promise<void> {

diff --git a/src/backend/clients/dynamodb/DDBClient.ts b/src/backend/clients/dynamodb/DDBClient.ts
@@ -43,6 +43,7 @@ import { once } from 'node:events';
 import { Agent as httpsAgent } from 'node:https';
 import { HttpError } from '../../core/http';
 import type { IConfig, IDynamoConfig } from '../../types';
+import { Span } from '../../util/span.js';
 import { PuterClient } from '../types';
 
 const LOCAL_DYNAMO_MEMORY_PREFIX = ':memory:';
@@ -151,6 +152,7 @@ export class DDBClient extends PuterClient {
         await this.#localInitPromise;
     }
 
+    @Span('ddb.get', (table: string) => ({ 'db.table': table }))
     async get<T extends Record<string, unknown>>(
         table: string,
         key: T,
@@ -167,6 +169,7 @@ export class DDBClient extends PuterClient {
         return client.send(command);
     }
 
+    @Span('ddb.put', (table: string) => ({ 'db.table': table }))
     async put<T extends Record<string, unknown>>(table: string, item: T) {
         const command = new PutCommand({
             TableName: table,
@@ -178,6 +181,9 @@ export class DDBClient extends PuterClient {
         return client.send(command);
     }
 
+    @Span('ddb.batchGet', (params: unknown[]) => ({
+        'db.batch_size': params.length,
+    }))
     async batchGet(
         params: { table: string; items: Record<string, unknown> }[],
         consistentRead = false,
@@ -212,6 +218,9 @@ export class DDBClient extends PuterClient {
         return client.send(command);
     }
 
+    @Span('ddb.batchPut', (params: unknown[]) => ({
+        'db.batch_size': params.length,
+    }))
     async batchPut(params: { table: string; item: Record<string, unknown> }[]) {
         const consumedCapacityByTable = new Map<string, number>();
         if (params.length === 0) {
@@ -318,6 +327,7 @@ export class DDBClient extends PuterClient {
         };
     }
 
+    @Span('ddb.del', (table: string) => ({ 'db.table': table }))
     async del<T extends Record<string, unknown>>(table: string, key: T) {
         const command = new DeleteCommand({
             TableName: table,
@@ -329,6 +339,7 @@ export class DDBClient extends PuterClient {
         return client.send(command);
     }
 
+    @Span('ddb.query', (table: string) => ({ 'db.table': table }))
     async query<T extends Record<string, unknown>>(
         table: string,
         keys: T,
@@ -383,6 +394,7 @@ export class DDBClient extends PuterClient {
         return client.send(command);
     }
 
+    @Span('ddb.update', (table: string) => ({ 'db.table': table }))
     async update<T extends Record<string, unknown>>(
         table: string,
         key: T,

diff --git a/src/backend/clients/event/EventClient.ts b/src/backend/clients/event/EventClient.ts
@@ -18,6 +18,7 @@
  */
 
 import { extensionStore } from '../../extensions';
+import { withSpan } from '../../util/span.js';
 import { PuterClient } from '../types';
 import {
     EventListener,
@@ -99,26 +100,34 @@ export class EventClient extends PuterClient {
         data: EventMap[T],
         meta: EventMetadata,
     ) {
-        const parts = key.split('.');
-        for (let i = 0; i < parts.length; i++) {
-            const matchKey = (
-                i === parts.length - 1
-                    ? key
-                    : `${parts.slice(0, i + 1).join('.')}.*`
-            ) as ListenKey;
-            const extensionListeners = extensionStore.events[matchKey];
-            const listeners = (this.#eventListeners[matchKey] || []).concat(
-                extensionListeners || [],
-            );
-            if (!listeners) continue;
-            for (const listener of listeners) {
-                try {
-                    await listener(key, data, meta);
-                } catch (e) {
-                    console.error('Error in event listener for event', key, e);
+        // Spanned because callers block on listeners — this is where time
+        // spent in extension hooks (e.g. `ip.validate`) gets attributed.
+        return withSpan('event.emitAndWait', { 'event.key': key }, async () => {
+            const parts = key.split('.');
+            for (let i = 0; i < parts.length; i++) {
+                const matchKey = (
+                    i === parts.length - 1
+                        ? key
+                        : `${parts.slice(0, i + 1).join('.')}.*`
+                ) as ListenKey;
+                const extensionListeners = extensionStore.events[matchKey];
+                const listeners = (this.#eventListeners[matchKey] || []).concat(
+                    extensionListeners || [],
+                );
+                if (!listeners) continue;
+                for (const listener of listeners) {
+                    try {
+                        await listener(key, data, meta);
+                    } catch (e) {
+                        console.error(
+                            'Error in event listener for event',
+                            key,
+                            e,
+                        );
+                    }
                 }
             }
-        }
+        });
     }
 
     /**

diff --git a/src/backend/controllers/drivers/DriverController.test.ts b/src/backend/controllers/drivers/DriverController.test.ts
@@ -19,6 +19,12 @@
 
 import { Readable } from 'node:stream';
 import type { Request, RequestHandler, Response } from 'express';
+import { trace } from '@opentelemetry/api';
+import {
+    BasicTracerProvider,
+    InMemorySpanExporter,
+    SimpleSpanProcessor,
+} from '@opentelemetry/sdk-trace-base';
 import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest';
 import { v4 as uuidv4 } from 'uuid';
 import { actorUid, type Actor } from '../../core/actor.js';
@@ -533,7 +539,7 @@
    });

    it('emits reject and answers 403 when a before listener vetoes', async () => {
        const actor = await makeUserActor();
        await grantKvPerm(actor);

        // Scope the veto to a sentinel key so it doesn't affect other calls
@@ -615,4 +621,65 @@
     });
 });
 
+describe('DriverController.#handleCall tracing', () => {
+    let routes: Captured;
+    const exporter = new InMemorySpanExporter();
+    const provider = new BasicTracerProvider({
+        spanProcessors: [new SimpleSpanProcessor(exporter)],
+    });
+
+    beforeAll(() => {
+        routes = captureRoutes(controller);
+        trace.setGlobalTracerProvider(provider);
+    });
+
+    afterAll(async () => {
+        await provider.shutdown();
+        trace.disable();
+    });
+
+    it('emits driver + downstream spans around a successful call', async () => {
+        const actor = await makeUserActor();
+        await server.stores.permission.setFlatUserPerm(
+            actor.user!.id!,
+            'service:puter-kvstore:ii:puter-kvstore',
+            {
+                permission: 'service:puter-kvstore:ii:puter-kvstore',
+                deleted: false,
+                issuer_user_id: actor.user!.id!,
+            } as never,
+        );
+        exporter.reset();
+
+        const res = makeRes();
+        const req = makeReq(
+            {
+                interface: 'puter-kvstore',
+                method: 'set',
+                args: { key: `k-${uuidv4()}`, value: 'v' },
+            },
+            actor,
+        );
+        await runWithContext({ actor }, () =>
+            routes['POST /call'](req, res as unknown as Response, () => {}),
+        );
+        expect((res.body as { success: boolean }).success).toBe(true);
+
+        const names = exporter.getFinishedSpans().map((s) => s.name);
+        const driverSpan = exporter
+            .getFinishedSpans()
+            .find((s) => s.name === 'driver.puter-kvstore.set');
+        expect(driverSpan).toBeDefined();
+        expect(driverSpan!.attributes).toMatchObject({
+            'driver.interface': 'puter-kvstore',
+            'driver.method': 'set',
+            'driver.name': 'puter-kvstore',
+        });
+        // The call path exercises the permission gate and the KV store's
+        // dynamo writes — both should have produced spans of their own.
+        expect(names).toContain('permission.scan');
+        expect(names.some((n) => n.startsWith('ddb.'))).toBe(true);
+    });
+});
+
 void vi; // unused but reserved for future expansion
diff --git a/src/backend/controllers/drivers/DriverController.ts b/src/backend/controllers/drivers/DriverController.ts
@@ -37,6 +37,7 @@ import {
 import type { PermissionService } from '../../services/permission/PermissionService.js';
 import { PermissionUtil } from '../../services/permission/permissionUtil.js';
 import type { WithLifecycle } from '../../types';
+import { withSpan } from '../../util/span.js';
 import { PuterController } from '../types.js';
 
 type DriverInstance = WithLifecycle & Record<string, unknown>;
@@ -382,11 +383,22 @@ export class DriverController extends PuterController {
         }
 
         // Drivers read actor/context via the Context API — no drilled args.
+        // The span ends when the method returns; for streamed results that
+        // is stream start, not stream drain (same window the lifecycle
+        // events below report as durationMs).
         const startedAt = Date.now();
         let result;
         try {
-            // eslint-disable-next-line @typescript-eslint/no-explicit-any
-            result = await (fn as (...x: unknown[]) => any).call(driver, args);
+            result = await withSpan(
+                `driver.${ifaceName}.${method}`,
+                {
+                    driver: ifaceName,
+                    'driver.method': method,
+                    'driver.name': resolved,
+                },
+                // eslint-disable-next-line @typescript-eslint/no-explicit-any
+                () => (fn as (...x: unknown[]) => any).call(driver, args),
+            );
         } catch (e) {
             this.clients.event?.emit(
                 `driver.${ifaceName}.${method}.error`,