diff --git a/.gitignore b/.gitignore index 4027660..6fd08c6 100644 --- a/.gitignore +++ b/.gitignore @@ -32,3 +32,4 @@ tests/.tmp/ *.log *.txt .kode/ +.kode-observability-http/ diff --git a/README.md b/README.md index e262e8a..7850fca 100644 --- a/README.md +++ b/README.md @@ -90,6 +90,48 @@ export OPEN_SANDBOX_ENDPOINT=http://127.0.0.1:8080 # optional export OPEN_SANDBOX_IMAGE=ubuntu # optional ``` +## Observability + +KODE keeps observability as an SDK-facing capability first: + +- runtime metrics via `agent.getMetricsSnapshot()` +- runtime observations via `agent.getObservationReader()` / `agent.subscribeObservations()` +- optional OTEL bridge via `observability.otel` +- optional persisted observation query via `observability.persistence` + +Minimal persisted-observation example: + +```typescript +import { + Agent, + JSONStore, + JSONStoreObservationBackend, + createStoreBackedObservationReader, +} from '@shareai-lab/kode-sdk'; + +const storeDir = './.kode'; +const observationBackend = new JSONStoreObservationBackend(storeDir); + +const agent = await Agent.create({ + templateId: 'assistant', + observability: { + persistence: { + backend: observationBackend, + }, + }, +}, deps); + +const runtimeSnapshot = agent.getMetricsSnapshot(); +const runtimeObservations = agent.getObservationReader().listObservations(); + +const persistedReader = createStoreBackedObservationReader(observationBackend); +const persistedObservations = await persistedReader.listObservations({ limit: 50 }); +``` + +If you want to expose these metrics or observations over HTTP, do it in your application on top of readers/backends, not inside `Agent` itself. `examples/08-observability-http.ts` is an application-layer example, not an SDK-owned HTTP feature. + +Run the full example locally with `npm run example:observability-http`. + ## Architecture for Scale For production deployments serving many users, we recommend the **Worker Microservice Pattern**: @@ -150,6 +192,7 @@ See [docs/en/guides/architecture.md](./docs/en/guides/architecture.md) for detai | [Concepts](./docs/en/getting-started/concepts.md) | Core concepts explained | | **Guides** | | | [Events](./docs/en/guides/events.md) | Three-channel event system | +| [Observability](./docs/en/guides/observability.md) | Metrics, observations, persistence, and app-layer exposure | | [Tools](./docs/en/guides/tools.md) | Built-in tools & custom tools | | [E2B Sandbox](./docs/en/guides/e2b-sandbox.md) | E2B cloud sandbox integration | | [OpenSandbox](./docs/en/guides/opensandbox-sandbox.md) | OpenSandbox self-hosted sandbox integration | diff --git a/README.zh-CN.md b/README.zh-CN.md index 3c95854..86d8740 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -90,6 +90,48 @@ export OPEN_SANDBOX_ENDPOINT=http://127.0.0.1:8080 # 可选 export OPEN_SANDBOX_IMAGE=ubuntu # 可选 ``` +## 可观测性 + +KODE 把可观测性优先作为 SDK 能力暴露： + +- 运行时指标：`agent.getMetricsSnapshot()` +- 运行时 observation：`agent.getObservationReader()` / `agent.subscribeObservations()` +- 可选 OTEL bridge：`observability.otel` +- 可选持久化 observation 查询：`observability.persistence` + +最小持久化 observation 示例： + +```typescript +import { + Agent, + JSONStore, + JSONStoreObservationBackend, + createStoreBackedObservationReader, +} from '@shareai-lab/kode-sdk'; + +const storeDir = './.kode'; +const observationBackend = new JSONStoreObservationBackend(storeDir); + +const agent = await Agent.create({ + templateId: 'assistant', + observability: { + persistence: { + backend: observationBackend, + }, + }, +}, deps); + +const runtimeSnapshot = agent.getMetricsSnapshot(); +const runtimeObservations = agent.getObservationReader().listObservations(); + +const persistedReader = createStoreBackedObservationReader(observationBackend); +const persistedObservations = await persistedReader.listObservations({ limit: 50 }); +``` + +如果你要通过 HTTP 对外暴露这些指标或 observation，应该在你的应用层基于 reader/backend 去包装，而不是让 `Agent` 自己直接监听端口。`examples/08-observability-http.ts` 只是应用层示例，不是 SDK 自带的 HTTP 能力。 + +可通过 `npm run example:observability-http` 本地运行完整示例。 + ## 支持的 Provider | Provider | 流式输出 | 工具调用 | 推理 | 文件 | @@ -110,6 +152,7 @@ export OPEN_SANDBOX_IMAGE=ubuntu # 可选 | [核心概念](./docs/zh-CN/getting-started/concepts.md) | 核心概念详解 | | **使用指南** | | | [事件系统](./docs/zh-CN/guides/events.md) | 三通道事件系统 | +| [可观测性](./docs/zh-CN/guides/observability.md) | 指标、observation、持久化与应用层暴露 | | [工具系统](./docs/zh-CN/guides/tools.md) | 内置工具与自定义工具 | | [E2B 沙箱](./docs/zh-CN/guides/e2b-sandbox.md) | E2B 云端沙箱接入 | | [OpenSandbox 沙箱](./docs/zh-CN/guides/opensandbox-sandbox.md) | OpenSandbox 自托管沙箱接入 | diff --git a/docs/en/examples/playbooks.md b/docs/en/examples/playbooks.md index 6f46117..579536d 100644 --- a/docs/en/examples/playbooks.md +++ b/docs/en/examples/playbooks.md @@ -153,7 +153,33 @@ const stats = await store.aggregateStats(agent.agentId); --- -## 6. Combined: Approval + Collaboration + Scheduling +## 6. Observability Readers + Application HTTP Wrapper + +- **Goal**: Read runtime/persisted observations from the SDK and optionally expose them through your own app-layer HTTP service. +- **Example**: `examples/08-observability-http.ts` +- **Run**: `npm run example:observability-http` +- **Key Steps**: + 1. Read point-in-time metrics with `agent.getMetricsSnapshot()`. + 2. Read live in-memory observations with `agent.getObservationReader()` or `agent.subscribeObservations()`. + 3. Configure `observability.persistence.backend` and query history with `createStoreBackedObservationReader(...)`. + 4. Map your own routes, auth, tenant checks, and response shaping in application code. +- **Considerations**: + - Prefer runtime reader for "what is happening now" and persisted reader for audit/history views. + - Treat `metadata.__debug` as internal/debug-only data; do not expose it blindly to external consumers. + - Keep HTTP, auth, rate limiting, and dashboard concerns outside SDK core. + +```typescript +const metrics = agent.getMetricsSnapshot(); +const runtimeReader = agent.getObservationReader(); +const persistedReader = createStoreBackedObservationReader(observationBackend); + +const runtime = runtimeReader.listObservations({ limit: 20 }); +const persisted = await persistedReader.listObservations({ agentIds: [agent.agentId], limit: 50 }); +``` + +--- + +## 7. Combined: Approval + Collaboration + Scheduling - **Scenario**: Code review bot, Planner splits tasks and assigns to Specialists, tool operations need approval, scheduled reminders ensure SLA. - **Implementation**: @@ -184,12 +210,13 @@ const stats = await store.aggregateStats(agent.agentId); - [Getting Started](../getting-started/quickstart.md) - [Events Guide](../guides/events.md) +- [Observability Guide](../guides/observability.md) - [Multi-Agent Systems](../advanced/multi-agent.md) - [Database Guide](../guides/database.md) --- -## 7. CLI Agent Application +## 8. CLI Agent Application Build command-line AI assistants like Claude Code or Cursor. diff --git a/docs/en/guides/observability.md b/docs/en/guides/observability.md new file mode 100644 index 0000000..d3a2c42 --- /dev/null +++ b/docs/en/guides/observability.md @@ -0,0 +1,166 @@ +# Observability Guide + +KODE exposes observability as SDK capabilities first, not as an application server. + +That means the SDK gives you structured metrics, observations, persistence hooks, and OTEL bridging. Your application decides whether to expose them through HTTP, dashboards, alerting, or internal admin tools. + +--- + +## What KODE Includes + +- Runtime metrics via `agent.getMetricsSnapshot()` +- Runtime observation reads via `agent.getObservationReader()` +- Runtime observation streaming via `agent.subscribeObservations()` +- Optional persisted observation queries via `observability.persistence` +- Optional OTEL export via `observability.otel` + +## What KODE Deliberately Does Not Include + +- Built-in HTTP server lifecycle +- Built-in auth, tenant isolation, or rate limiting +- Built-in observability dashboard UI +- Opinionated public API contracts for app delivery + +Those concerns belong in your application layer. + +--- + +## Runtime Metrics and Observations + +Use runtime readers when you want to inspect the current agent process without waiting for external exports. + +```typescript +const metrics = agent.getMetricsSnapshot(); +const reader = agent.getObservationReader(); + +const latest = reader.listObservations({ + kinds: ['generation', 'tool'], + limit: 20, +}); + +for await (const envelope of agent.subscribeObservations({ runId: metrics.currentRunId })) { + console.log(envelope.observation.kind, envelope.observation.name); +} +``` + +Typical runtime uses: + +- show "live now" generation/tool activity in an admin panel +- inspect approval waits, tool errors, and compression events +- derive counters without polling raw event buses + +--- + +## Persisted Observations + +Use persisted readers when you need history, audit views, or process-restart durability. + +```typescript +import { + Agent, + JSONStoreObservationBackend, + createStoreBackedObservationReader, +} from '@shareai-lab/kode-sdk'; + +const observationBackend = new JSONStoreObservationBackend('./.kode-observability'); + +const agent = await Agent.create({ + templateId: 'assistant', + observability: { + persistence: { + backend: observationBackend, + }, + }, +}, deps); + +const persistedReader = createStoreBackedObservationReader(observationBackend); +const history = await persistedReader.listObservations({ + agentIds: [agent.agentId], + kinds: ['agent_run', 'generation', 'tool'], + limit: 50, +}); +``` + +Use persisted storage for: + +- audit timelines +- run replay pages +- offline analytics jobs +- debugging after process restart + +--- + +## OTEL Bridge + +If your platform already standardizes on OpenTelemetry, enable the bridge and ship translated spans to your collector. + +```typescript +const agent = await Agent.create({ + templateId: 'assistant', + observability: { + otel: { + enabled: true, + serviceName: 'kode-agent', + exporter: { + protocol: 'http/json', + endpoint: process.env.OTEL_EXPORTER_OTLP_ENDPOINT!, + }, + }, + }, +}, deps); +``` + +Keep KODE's native observation model as your source of truth. OTEL is best treated as an interoperability/export path. + +--- + +## Data Safety and Capture Boundaries + +KODE supports configurable capture levels through `observability.capture`: + +- `off` +- `summary` +- `full` +- `redacted` + +Prefer `summary` or `redacted` for production unless you have a clear compliance reason to store more detail. + +Also note: + +- provider-specific raw payloads are not part of the public observation schema +- debug-only extensions may appear under `metadata.__debug` +- `metadata.__debug` should be treated as internal/private and filtered before external exposure + +This keeps the public observation model safer and more stable. + +--- + +## Exposing Observability over HTTP + +If you need HTTP endpoints, build them in your app on top of the SDK readers/backends. + +Reference example: + +- `examples/08-observability-http.ts` +- run with `npm run example:observability-http` + +That example demonstrates: + +- a normal app-owned HTTP server +- `POST /agents/demo/send` to drive an agent run +- `GET /api/observability/.../metrics` for runtime metrics +- `GET /api/observability/.../observations/runtime` for live observation reads +- `GET /api/observability/.../observations/persisted` for persisted history + +This boundary is intentional: the SDK provides observability primitives, while the app owns transport, auth, and presentation. + +--- + +## Recommended Rollout + +1. Start with runtime metrics and runtime observation readers. +2. Add persisted observation storage for auditability. +3. Add OTEL export only if your platform needs centralized telemetry. +4. Add app-layer HTTP or UI only after the data model and filtering policy are clear. + +This order keeps the SDK integration stable and avoids prematurely coupling KODE to one delivery surface. diff --git a/docs/zh-CN/examples/playbooks.md b/docs/zh-CN/examples/playbooks.md index ee0025d..ee7479b 100644 --- a/docs/zh-CN/examples/playbooks.md +++ b/docs/zh-CN/examples/playbooks.md @@ -153,7 +153,33 @@ const stats = await store.aggregateStats(agent.agentId); --- -## 6. 组合拳：审批 + 协作 + 调度 +## 6. 观测层读取与应用层 HTTP 包装 + +- **目标**：从 SDK 读取运行时/持久化 observation，并按你自己的应用边界选择是否通过 HTTP 暴露出去。 +- **示例**：`examples/08-observability-http.ts` +- **运行**：`npm run example:observability-http` +- **关键步骤**： + 1. 通过 `agent.getMetricsSnapshot()` 读取当前指标快照。 + 2. 通过 `agent.getObservationReader()` 或 `agent.subscribeObservations()` 读取运行时 observation。 + 3. 为 `observability.persistence.backend` 配置后端，并用 `createStoreBackedObservationReader(...)` 查询历史数据。 + 4. 在应用代码中自行定义路由、鉴权、租户隔离和响应裁剪。 +- **注意事项**： + - 运行时 reader 更适合“现在发生了什么”，持久化 reader 更适合审计与历史视图。 + - `metadata.__debug` 只能视为内部调试数据，不应直接原样对外暴露。 + - HTTP、鉴权、限流、Dashboard 都应留在 SDK 外部。 + +```typescript +const metrics = agent.getMetricsSnapshot(); +const runtimeReader = agent.getObservationReader(); +const persistedReader = createStoreBackedObservationReader(observationBackend); + +const runtime = runtimeReader.listObservations({ limit: 20 }); +const persisted = await persistedReader.listObservations({ agentIds: [agent.agentId], limit: 50 }); +``` + +--- + +## 7. 组合拳：审批 + 协作 + 调度 - **场景**：代码审查机器人，Planner 负责拆分任务并分配到不同 Specialist，工具操作需审批，定时提醒确保 SLA。 - **实现路径**： @@ -184,5 +210,6 @@ const stats = await store.aggregateStats(agent.agentId); - [快速开始](../getting-started/quickstart.md) - [事件指南](../guides/events.md) +- [可观测性指南](../guides/observability.md) - [多 Agent 系统](../advanced/multi-agent.md) - [数据库指南](../guides/database.md) diff --git a/docs/zh-CN/guides/observability.md b/docs/zh-CN/guides/observability.md new file mode 100644 index 0000000..56d0c9e --- /dev/null +++ b/docs/zh-CN/guides/observability.md @@ -0,0 +1,166 @@ +# 可观测性指南 + +KODE 将可观测性优先设计为 SDK 能力，而不是一个内置应用服务。 + +也就是说，SDK 负责提供结构化指标、observation、持久化接口与 OTEL bridge；至于是否通过 HTTP、Dashboard、告警系统或内部管理台暴露这些数据，应由你的应用层来决定。 + +--- + +## KODE 已提供什么 + +- 运行时指标：`agent.getMetricsSnapshot()` +- 运行时 observation 读取：`agent.getObservationReader()` +- 运行时 observation 订阅：`agent.subscribeObservations()` +- 可选持久化 observation 查询：`observability.persistence` +- 可选 OTEL 导出：`observability.otel` + +## KODE 有意不内置什么 + +- 内置 HTTP server 生命周期 +- 内置鉴权、租户隔离、限流 +- 内置观测 Dashboard UI +- 面向应用交付的固定公开 API 契约 + +这些都应该放在你的应用层。 + +--- + +## 运行时指标与 Observation + +如果你想直接观察当前 Agent 进程中的行为，而不是等待外部导出链路，优先使用运行时 reader。 + +```typescript +const metrics = agent.getMetricsSnapshot(); +const reader = agent.getObservationReader(); + +const latest = reader.listObservations({ + kinds: ['generation', 'tool'], + limit: 20, +}); + +for await (const envelope of agent.subscribeObservations({ runId: metrics.currentRunId })) { + console.log(envelope.observation.kind, envelope.observation.name); +} +``` + +常见用途： + +- 在管理台展示“当前正在发生什么” +- 观察审批等待、工具失败、上下文压缩事件 +- 在不轮询原始事件总线的情况下提取聚合指标 + +--- + +## 持久化 Observation + +如果你需要历史数据、审计视图，或者希望在进程重启后仍能查询 observation，就应配置持久化后端。 + +```typescript +import { + Agent, + JSONStoreObservationBackend, + createStoreBackedObservationReader, +} from '@shareai-lab/kode-sdk'; + +const observationBackend = new JSONStoreObservationBackend('./.kode-observability'); + +const agent = await Agent.create({ + templateId: 'assistant', + observability: { + persistence: { + backend: observationBackend, + }, + }, +}, deps); + +const persistedReader = createStoreBackedObservationReader(observationBackend); +const history = await persistedReader.listObservations({ + agentIds: [agent.agentId], + kinds: ['agent_run', 'generation', 'tool'], + limit: 50, +}); +``` + +适合的场景： + +- 审计时间线 +- run 回放页面 +- 离线分析任务 +- 进程重启后的问题排查 + +--- + +## OTEL Bridge + +如果你的平台已经统一使用 OpenTelemetry，可以启用 bridge，把 KODE 的 observation 转换后导出到已有 collector。 + +```typescript +const agent = await Agent.create({ + templateId: 'assistant', + observability: { + otel: { + enabled: true, + serviceName: 'kode-agent', + exporter: { + protocol: 'http/json', + endpoint: process.env.OTEL_EXPORTER_OTLP_ENDPOINT!, + }, + }, + }, +}, deps); +``` + +建议继续把 KODE 原生 observation 模型作为第一真相源，OTEL 更适合作为互操作/导出路径。 + +--- + +## 数据安全与采集边界 + +KODE 通过 `observability.capture` 支持不同采集等级： + +- `off` +- `summary` +- `full` +- `redacted` + +生产环境通常优先选择 `summary` 或 `redacted`，除非你有明确的合规或排障理由去保留更多细节。 + +另外还要注意： + +- provider 原始 payload 不属于公共 observation schema +- 调试扩展信息可能出现在 `metadata.__debug` +- `metadata.__debug` 应视为内部/私有字段，对外暴露前必须过滤 + +这样可以让公共 observation 模型更稳定，也更安全。 + +--- + +## 通过 HTTP 暴露观测数据 + +如果你确实需要 HTTP 接口，请在应用层基于 SDK reader/backend 自行封装。 + +参考示例： + +- `examples/08-observability-http.ts` +- 运行命令：`npm run example:observability-http` + +这个示例演示了： + +- 应用自己持有 HTTP server +- 用 `POST /agents/demo/send` 驱动一次 agent run +- 用 `GET /api/observability/.../metrics` 获取运行时指标 +- 用 `GET /api/observability/.../observations/runtime` 读取运行时 observation +- 用 `GET /api/observability/.../observations/persisted` 读取持久化历史 + +这个边界是刻意设计的：SDK 负责观测原语，应用负责传输层、鉴权和展示层。 + +--- + +## 推荐落地顺序 + +1. 先接入运行时 metrics 与 runtime observation reader。 +2. 再补持久化 observation 后端，保证可审计。 +3. 只有在平台需要统一遥测时再接 OTEL。 +4. 等数据模型与过滤策略稳定后，再补应用层 HTTP 或管理 UI。 + +按这个顺序推进，可以最大限度降低耦合，避免过早把 KODE 绑定到某一种交付形式。 diff --git a/examples/08-observability-http.ts b/examples/08-observability-http.ts new file mode 100644 index 0000000..8cad3bc --- /dev/null +++ b/examples/08-observability-http.ts @@ -0,0 +1,241 @@ +import './shared/load-env'; + +import http from 'node:http'; + +// This example intentionally keeps HTTP in application code to show +// how KODE observability readers can be wrapped without making HTTP +// part of the SDK core contract. +import { + Agent, + AgentConfig, + AgentTemplateRegistry, + ModelConfig, + JSONStore, + JSONStoreObservationBackend, + SandboxFactory, + ToolRegistry, + createStoreBackedObservationReader, +} from '@shareai-lab/kode-sdk'; +import { createDemoModelConfig, createDemoModelProvider } from './shared/demo-model'; +import { createExampleObservabilityHttpHandler } from './shared/observability-http'; + +const storeDir = './.kode-observability-http'; +const observationBackend = new JSONStoreObservationBackend(storeDir); +const persistedReader = createStoreBackedObservationReader(observationBackend); +const demoAgentId = 'agt-observability-http-demo'; +const defaultPort = Number(process.env.PORT || 3100); +const demoModelConfig = createDemoModelConfig(); +const liveAgents = new Map>(); + +const templates = new AgentTemplateRegistry(); +templates.register({ + id: 'obs-http-demo', + systemPrompt: 'You are an observability demo assistant.', + tools: [], + permission: { mode: 'auto' }, +}); + +const deps = { + store: new JSONStore(storeDir), + templateRegistry: templates, + sandboxFactory: new SandboxFactory(), + toolRegistry: new ToolRegistry(), + modelFactory: (config: any) => createDemoModelProvider(config), +}; + +async function createOrResumeAgent(agentId: string): Promise { + const exists = await deps.store.exists(agentId); + if (exists) { + return Agent.resumeFromStore(agentId, deps, { + overrides: { + modelConfig: demoModelConfig as ModelConfig, + observability: { + persistence: { backend: observationBackend }, + }, + }, + }); + } + + const config: AgentConfig = { + agentId, + templateId: 'obs-http-demo', + modelConfig: demoModelConfig, + sandbox: { kind: 'local', workDir: './workspace', enforceBoundary: true }, + observability: { + persistence: { backend: observationBackend }, + }, + }; + + return Agent.create(config, deps); +} + +function getLiveAgent(agentId: string): Promise { + const existing = liveAgents.get(agentId); + if (existing) { + return existing; + } + + const pending = createOrResumeAgent(agentId).catch((error) => { + liveAgents.delete(agentId); + throw error; + }); + liveAgents.set(agentId, pending); + return pending; +} + +function sendJson(res: http.ServerResponse, status: number, data: unknown): void { + res.statusCode = status; + res.setHeader('Content-Type', 'application/json; charset=utf-8'); + res.end(JSON.stringify(data, null, 2)); +} + +function parseJsonBody(body: string): any { + if (!body.trim()) { + return {}; + } + + try { + return JSON.parse(body); + } catch (error: any) { + throw new Error(`Request body must be valid JSON: ${error?.message || 'parse failed'}`); + } +} + +async function readBody(req: http.IncomingMessage): Promise { + const chunks: Buffer[] = []; + for await (const chunk of req) { + chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)); + } + return Buffer.concat(chunks).toString('utf-8'); +} + +function getExampleRoutes(port: number) { + const baseUrl = `http://127.0.0.1:${port}`; + const agentBase = `${baseUrl}/api/observability/agents/${demoAgentId}`; + + return { + baseUrl, + docs: `${baseUrl}/`, + sendPrompt: `${baseUrl}/agents/demo/send`, + metrics: `${agentBase}/metrics`, + runtimeObservations: `${agentBase}/observations/runtime`, + persistedObservations: `${agentBase}/observations/persisted`, + runtimeRun: `${agentBase}/observations/runtime/runs/`, + persistedRun: `${agentBase}/observations/persisted/runs/`, + healthz: `${baseUrl}/healthz`, + }; +} + +function getRequiredEnvHints(): string[] { + if (demoModelConfig.provider === 'anthropic') { + return ['ANTHROPIC_API_KEY']; + } + if (demoModelConfig.provider === 'gemini') { + return ['GEMINI_API_KEY']; + } + if (demoModelConfig.provider === 'glm') { + return ['OPENAI_API_KEY', 'OPENAI_MODEL_ID=glm-5', 'OPENAI_BASE_URL']; + } + return ['OPENAI_API_KEY']; +} + +function buildExampleIndex(port: number) { + const routes = getExampleRoutes(port); + return { + message: 'KODE observability demo server', + purpose: 'Application-layer HTTP wrapper around SDK observability readers and persistence.', + model: { + provider: demoModelConfig.provider, + model: demoModelConfig.model, + baseUrl: demoModelConfig.baseUrl, + }, + boundaries: [ + 'HTTP stays in the example application, not in Agent or SDK core.', + 'Use runtime reader for live state and persisted reader for history/audit.', + 'Filter internal/debug fields before exposing data outside trusted systems.', + ], + requiredEnv: getRequiredEnvHints(), + routes, + sampleCurl: { + sendPrompt: `curl -X POST ${routes.sendPrompt} -H 'content-type: application/json' -d '{"prompt":"Summarize KODE observability in one sentence."}'`, + metrics: `curl ${routes.metrics}`, + runtimeObservations: `curl '${routes.runtimeObservations}?limit=20'`, + persistedObservations: `curl '${routes.persistedObservations}?limit=20'`, + }, + }; +} + +const observabilityHandler = createExampleObservabilityHttpHandler({ + basePath: '/api/observability', + resolveRuntimeSource: async (agentId) => { + if (agentId !== demoAgentId) { + return undefined; + } + const agent = await getLiveAgent(agentId); + return { + getMetricsSnapshot: () => agent.getMetricsSnapshot(), + getObservationReader: () => agent.getObservationReader(), + }; + }, + resolvePersistedReader: async (agentId) => (agentId === demoAgentId ? persistedReader : undefined), +}); + +const server = http.createServer(async (req, res) => { + try { + const method = req.method || 'GET'; + const url = new URL(req.url || '/', `http://127.0.0.1:${defaultPort}`); + const path = url.pathname || '/'; + + if (method === 'GET' && (path === '/' || path === '/api/observability')) { + return sendJson(res, 200, buildExampleIndex(defaultPort)); + } + + if (method === 'POST' && path === '/agents/demo/send') { + const body = await readBody(req); + const payload = parseJsonBody(body); + const agent = await getLiveAgent(demoAgentId); + const result = await agent.chat(payload.prompt || 'say hello'); + return sendJson(res, 200, result); + } + + if (method === 'GET' && path === '/healthz') { + return sendJson(res, 200, { ok: true }); + } + + if (path.startsWith('/api/observability/')) { + const response = await observabilityHandler({ + method, + url: req.url || '/', + }); + + for (const [key, value] of Object.entries(response.headers)) { + res.setHeader(key, value); + } + return sendJson(res, response.status, response.body); + } + + return sendJson(res, 404, { + error: 'not_found', + ...buildExampleIndex(defaultPort), + }); + } catch (error: any) { + const message = error?.message || 'Unexpected server error'; + const status = message.startsWith('Request body must be valid JSON') ? 400 : 500; + return sendJson(res, status, { + error: status === 400 ? 'bad_request' : 'internal_error', + message, + }); + } +}); + +server.listen(defaultPort, '127.0.0.1', () => { + const routes = getExampleRoutes(defaultPort); + console.log(`Observability demo server listening on ${routes.baseUrl}`); + console.log(`Model provider: ${demoModelConfig.provider}`); + console.log(`Model id: ${demoModelConfig.model}`); + console.log(`Docs: ${routes.docs}`); + console.log(`POST prompt: ${routes.sendPrompt}`); + console.log(`Runtime metrics: ${routes.metrics}`); + console.log(`Runtime observations: ${routes.runtimeObservations}`); + console.log(`Persisted observations: ${routes.persistedObservations}`); +}); diff --git a/examples/README.en.md b/examples/README.en.md index 36fd381..41accf6 100644 --- a/examples/README.en.md +++ b/examples/README.en.md @@ -64,8 +64,45 @@ export POSTGRES_PASSWORD=your-password | openrouter | OpenRouter example | `npm run openrouter` | | openrouter-stream | OpenRouter streaming | `npm run openrouter-stream` | | openrouter-agent | OpenRouter Agent integration | `npm run openrouter-agent` | +| observability-http | App-layer HTTP example wrapping KODE observability | `npm run observability-http` | | nextjs | Next.js API route integration | `npm run nextjs` | +## Observability HTTP Example + +This example keeps HTTP in application code and uses SDK readers/backends underneath. + +```bash +# From repo root +npm run example:observability-http + +# Or from examples/ +cd examples +npm run observability-http +``` + +Required env: + +```bash +export KODE_EXAMPLE_PROVIDER=glm # optional, auto-detected when OPENAI_MODEL_ID starts with glm +export OPENAI_API_KEY=your-key +export OPENAI_MODEL_ID=glm-5 +export OPENAI_BASE_URL=https://open.bigmodel.cn/api/paas/v4/ +``` + +This follows the same `OPENAI_*` convention as `examples/openai-usage.ts`, so OpenAI-compatible providers such as GLM can be reused without hardcoding Anthropic-only settings. + +Suggested requests after startup: + +```bash +curl http://127.0.0.1:3100/ +curl -X POST http://127.0.0.1:3100/agents/demo/send \ + -H 'content-type: application/json' \ + -d '{"prompt":"Summarize observability in one sentence."}' +curl http://127.0.0.1:3100/api/observability/agents/agt-observability-http-demo/metrics +curl http://127.0.0.1:3100/api/observability/agents/agt-observability-http-demo/observations/runtime +curl http://127.0.0.1:3100/api/observability/agents/agt-observability-http-demo/observations/persisted +``` + ## E2B Cloud Sandbox Example ```bash @@ -105,3 +142,4 @@ examples/ 2. Some examples require network access to external APIs 3. The db-postgres example requires a running PostgreSQL database 4. E2B examples require an E2B account and API key +5. The `observability-http` example shows how an application can wrap SDK observability readers with HTTP; it is not an Agent-owned HTTP server and not a core SDK feature diff --git a/examples/README.md b/examples/README.md index 3a4406e..ca4d7ff 100644 --- a/examples/README.md +++ b/examples/README.md @@ -64,8 +64,45 @@ export POSTGRES_PASSWORD=your-password | openrouter | OpenRouter 完整示例 | `npm run openrouter` | | openrouter-stream | OpenRouter 流式输出 | `npm run openrouter-stream` | | openrouter-agent | OpenRouter Agent 集成 | `npm run openrouter-agent` | +| observability-http | 应用层包装 KODE 观测接口的 HTTP 示例 | `npm run observability-http` | | nextjs | Next.js API 路由集成 | `npm run nextjs` | +## 观测层 HTTP 示例 + +这个示例把 HTTP 放在应用层，底层仍然只使用 SDK 提供的 reader/backend 能力。 + +```bash +# 在仓库根目录运行 +npm run example:observability-http + +# 或在 examples/ 目录运行 +cd examples +npm run observability-http +``` + +需要的环境变量： + +```bash +export KODE_EXAMPLE_PROVIDER=glm # 可选；当 OPENAI_MODEL_ID 以 glm 开头时也会自动识别 +export OPENAI_API_KEY=your-key +export OPENAI_MODEL_ID=glm-5 +export OPENAI_BASE_URL=https://open.bigmodel.cn/api/paas/v4/ +``` + +这套约定与 `examples/openai-usage.ts` 保持一致，因此像 GLM 这样的 OpenAI-compatible provider 可以直接复用，不需要再写死 Anthropic 配置。 + +服务启动后可按下面顺序试用： + +```bash +curl http://127.0.0.1:3100/ +curl -X POST http://127.0.0.1:3100/agents/demo/send \ + -H 'content-type: application/json' \ + -d '{"prompt":"用一句话总结 KODE 的观测层。"}' +curl http://127.0.0.1:3100/api/observability/agents/agt-observability-http-demo/metrics +curl http://127.0.0.1:3100/api/observability/agents/agt-observability-http-demo/observations/runtime +curl http://127.0.0.1:3100/api/observability/agents/agt-observability-http-demo/observations/persisted +``` + ## E2B 云沙箱示例 ```bash @@ -105,3 +142,4 @@ examples/ 2. 部分示例需要网络访问外部 API 3. db-postgres 示例需要运行 PostgreSQL 数据库 4. E2B 示例需要 E2B 账号和 API Key +5. `observability-http` 示例演示的是“应用层自己包装 SDK 的观测接口并暴露 HTTP”，不是 `Agent` 内置 HTTP 服务，也不是 SDK 核心 feature diff --git a/examples/package.json b/examples/package.json index 54ed41c..81e18a4 100644 --- a/examples/package.json +++ b/examples/package.json @@ -13,6 +13,7 @@ "openrouter": "ts-node 05-openrouter-complete.ts", "openrouter-stream": "ts-node 06-openrouter-stream.ts", "openrouter-agent": "ts-node 07-openrouter-agent.ts", + "observability-http": "ts-node 08-observability-http.ts", "db-sqlite": "ts-node db-sqlite.ts", "db-postgres": "ts-node db-postgres.ts", "anthropic": "ts-node anthropic-usage.ts", diff --git a/examples/shared/demo-model.ts b/examples/shared/demo-model.ts index 543a0ac..2763fa3 100644 --- a/examples/shared/demo-model.ts +++ b/examples/shared/demo-model.ts @@ -1,17 +1,183 @@ -import { AnthropicProvider, ModelConfig, ModelProvider } from '@shareai-lab/kode-sdk'; +import { + AnthropicProvider, + GeminiProvider, + ModelConfig, + ModelProvider, + OpenAIProvider, +} from '@shareai-lab/kode-sdk'; + +type DemoProvider = 'anthropic' | 'openai' | 'gemini' | 'glm'; + +function pickString(...values: Array): string | undefined { + for (const value of values) { + if (typeof value === 'string' && value.trim()) { + return value.trim(); + } + } + return undefined; +} + +function normalizeProvider(value?: string): DemoProvider | undefined { + if (!value) { + return undefined; + } + const normalized = value.trim().toLowerCase(); + if (normalized === 'anthropic' || normalized === 'openai' || normalized === 'gemini' || normalized === 'glm') { + return normalized; + } + return undefined; +} + +function detectDemoProvider(explicit?: string): DemoProvider { + const preferred = normalizeProvider(explicit) ?? normalizeProvider(process.env.KODE_EXAMPLE_PROVIDER); + if (preferred) { + return preferred; + } + + const openAiApiKey = pickString(process.env.OPENAI_API_KEY, process.env.OPENAI_API_TOKEN); + const openAiModel = (process.env.OPENAI_MODEL_ID || '').toLowerCase(); + const openAiBaseUrl = (process.env.OPENAI_BASE_URL || '').toLowerCase(); + + if (openAiApiKey && (openAiModel.startsWith('glm') || openAiBaseUrl.includes('bigmodel') || openAiBaseUrl.includes('zhipu'))) { + return 'glm'; + } + if (pickString(process.env.ANTHROPIC_API_KEY, process.env.ANTHROPIC_API_TOKEN)) { + return 'anthropic'; + } + if (openAiApiKey) { + return 'openai'; + } + if (pickString(process.env.GEMINI_API_KEY, process.env.GEMINI_API_TOKEN)) { + return 'gemini'; + } + + throw new Error( + 'No demo model provider was configured. Set KODE_EXAMPLE_PROVIDER or provide ANTHROPIC_*/OPENAI_*/GEMINI_* env vars.' + ); +} + +export function createDemoModelConfig(overrides: Partial = {}): ModelConfig { + const provider = detectDemoProvider(overrides.provider); + + if (provider === 'anthropic') { + const apiKey = pickString(overrides.apiKey, process.env.ANTHROPIC_API_KEY, process.env.ANTHROPIC_API_TOKEN); + if (!apiKey) { + throw new Error('Anthropic API key/token is required. Set ANTHROPIC_API_KEY or ANTHROPIC_API_TOKEN.'); + } + return { + provider, + apiKey, + model: pickString(overrides.model, process.env.ANTHROPIC_MODEL_ID, 'claude-sonnet-4-5-20250929')!, + baseUrl: pickString(overrides.baseUrl, process.env.ANTHROPIC_BASE_URL), + proxyUrl: pickString(overrides.proxyUrl), + maxTokens: overrides.maxTokens, + temperature: overrides.temperature, + reasoningTransport: overrides.reasoningTransport, + extraHeaders: overrides.extraHeaders, + extraBody: overrides.extraBody, + providerOptions: overrides.providerOptions, + multimodal: overrides.multimodal, + thinking: overrides.thinking, + }; + } + + if (provider === 'gemini') { + const apiKey = pickString(overrides.apiKey, process.env.GEMINI_API_KEY, process.env.GEMINI_API_TOKEN); + if (!apiKey) { + throw new Error('Gemini API key/token is required. Set GEMINI_API_KEY or GEMINI_API_TOKEN.'); + } + return { + provider, + apiKey, + model: pickString(overrides.model, process.env.GEMINI_MODEL_ID, 'gemini-2.0-flash')!, + baseUrl: pickString(overrides.baseUrl, process.env.GEMINI_BASE_URL), + proxyUrl: pickString(overrides.proxyUrl), + maxTokens: overrides.maxTokens, + temperature: overrides.temperature, + reasoningTransport: overrides.reasoningTransport, + extraHeaders: overrides.extraHeaders, + extraBody: overrides.extraBody, + providerOptions: overrides.providerOptions, + multimodal: overrides.multimodal, + thinking: overrides.thinking, + }; + } + + const openAiApiKey = pickString(overrides.apiKey, process.env.OPENAI_API_KEY, process.env.OPENAI_API_TOKEN); + if (!openAiApiKey) { + throw new Error('OpenAI-compatible API key/token is required. Set OPENAI_API_KEY or OPENAI_API_TOKEN.'); + } + + const model = pickString(overrides.model, process.env.OPENAI_MODEL_ID, provider === 'glm' ? 'glm-5' : 'gpt-4o')!; + const baseUrl = pickString(overrides.baseUrl, process.env.OPENAI_BASE_URL); + + if (provider === 'glm' && !baseUrl) { + throw new Error('GLM requires OPENAI_BASE_URL (for example https://open.bigmodel.cn/api/paas/v4/).'); + } + + return { + provider, + apiKey: openAiApiKey, + model, + baseUrl, + proxyUrl: pickString(overrides.proxyUrl), + maxTokens: overrides.maxTokens, + temperature: overrides.temperature, + reasoningTransport: overrides.reasoningTransport, + extraHeaders: overrides.extraHeaders, + extraBody: overrides.extraBody, + providerOptions: overrides.providerOptions, + multimodal: overrides.multimodal, + thinking: overrides.thinking, + }; +} export function createDemoModelProvider(config: ModelConfig): ModelProvider { - const apiKey = - config.apiKey ?? - process.env.ANTHROPIC_API_KEY ?? - process.env.ANTHROPIC_API_TOKEN ?? - process.env.ANTHROPIC_API_Token; - if (!apiKey) { - throw new Error('Anthropic API key/token is required. Set ANTHROPIC_API_KEY or ANTHROPIC_API_TOKEN.'); + const resolved = createDemoModelConfig(config); + + if (resolved.provider === 'anthropic') { + return new AnthropicProvider(resolved.apiKey!, resolved.model, resolved.baseUrl, resolved.proxyUrl, { + reasoningTransport: resolved.reasoningTransport, + extraHeaders: resolved.extraHeaders, + extraBody: resolved.extraBody, + providerOptions: resolved.providerOptions, + multimodal: resolved.multimodal, + thinking: resolved.thinking, + }); + } + + if (resolved.provider === 'gemini') { + return new GeminiProvider(resolved.apiKey!, resolved.model, resolved.baseUrl, resolved.proxyUrl, { + reasoningTransport: resolved.reasoningTransport, + extraHeaders: resolved.extraHeaders, + extraBody: resolved.extraBody, + providerOptions: resolved.providerOptions, + multimodal: resolved.multimodal, + thinking: resolved.thinking, + }); } - const baseUrl = config.baseUrl ?? process.env.ANTHROPIC_BASE_URL; - const modelId = config.model ?? process.env.ANTHROPIC_MODEL_ID ?? 'claude-sonnet-4.5-20250929'; + if (resolved.provider === 'glm') { + return new OpenAIProvider(resolved.apiKey!, resolved.model, resolved.baseUrl, resolved.proxyUrl, { + reasoningTransport: resolved.reasoningTransport ?? 'provider', + reasoning: { + fieldName: 'reasoning_content', + requestParams: { thinking: { type: 'enabled', clear_thinking: false } }, + }, + extraHeaders: resolved.extraHeaders, + extraBody: resolved.extraBody, + providerOptions: resolved.providerOptions, + multimodal: resolved.multimodal, + thinking: resolved.thinking, + }); + } - return new AnthropicProvider(apiKey, modelId, baseUrl); + return new OpenAIProvider(resolved.apiKey!, resolved.model, resolved.baseUrl, resolved.proxyUrl, { + reasoningTransport: resolved.reasoningTransport, + extraHeaders: resolved.extraHeaders, + extraBody: resolved.extraBody, + providerOptions: resolved.providerOptions, + multimodal: resolved.multimodal, + thinking: resolved.thinking, + }); } diff --git a/examples/shared/load-env.ts b/examples/shared/load-env.ts index 132b7ab..07bd6a7 100644 --- a/examples/shared/load-env.ts +++ b/examples/shared/load-env.ts @@ -1,3 +1,15 @@ +import { existsSync } from 'node:fs'; import * as dotenv from 'dotenv'; -dotenv.config(); +const preferred = process.env.KODE_EXAMPLE_ENV_FILE; +const envFiles = preferred + ? [preferred] + : [ + '.env', + '.env.local', + ...(!existsSync('.env') && existsSync('.env.test') ? ['.env.test'] : []), + ]; + +for (const path of envFiles) { + dotenv.config({ path, override: false }); +} diff --git a/examples/shared/observability-http.ts b/examples/shared/observability-http.ts new file mode 100644 index 0000000..6e870fb --- /dev/null +++ b/examples/shared/observability-http.ts @@ -0,0 +1,274 @@ +import type { + ObservationEnvelope, + ObservationKind, + ObservationListOptions, + ObservationReader, + ObservationRunView, + ObservationStatus, + PersistedObservationListOptions, + PersistedObservationReader, +} from '@shareai-lab/kode-sdk'; + +export type ObservabilityHttpRequest = { + method?: string; + url: string; +}; + +export type ObservabilityHttpResponse = { + status: number; + headers: Record; + body: unknown; +}; + +export type RuntimeHttpSource = { + getMetricsSnapshot(): unknown | Promise; + getObservationReader(): ObservationReader | Promise; +}; + +export type ObservabilityHttpHandlerConfig = { + basePath?: string; + resolveRuntimeSource?: ( + agentId: string + ) => RuntimeHttpSource | Promise | undefined; + resolvePersistedReader?: ( + agentId: string + ) => PersistedObservationReader | Promise | undefined; +}; + +const OBSERVATION_KINDS: ObservationKind[] = ['agent_run', 'generation', 'tool', 'subagent', 'compression']; +const OBSERVATION_STATUSES: ObservationStatus[] = ['ok', 'error', 'cancelled']; + +function normalizeBasePath(basePath?: string): string { + if (!basePath || basePath === '/') { + return ''; + } + return `/${basePath.replace(/^\/+|\/+$/g, '')}`; +} + +function buildJsonResponse(status: number, body: unknown): ObservabilityHttpResponse { + return { + status, + headers: { + 'content-type': 'application/json; charset=utf-8', + }, + body, + }; +} + +function splitSegments(pathname: string): string[] { + return pathname.split('/').filter(Boolean); +} + +function stripBasePath(pathname: string, basePath: string): string[] | undefined { + const pathSegments = splitSegments(pathname); + const baseSegments = splitSegments(basePath); + + if (baseSegments.length > pathSegments.length) { + return undefined; + } + + for (let index = 0; index < baseSegments.length; index++) { + if (pathSegments[index] !== baseSegments[index]) { + return undefined; + } + } + + return pathSegments.slice(baseSegments.length); +} + +function readStringList(params: URLSearchParams, key: string): string[] | undefined { + const values = params + .getAll(key) + .flatMap((value) => value.split(',')) + .map((value) => value.trim()) + .filter(Boolean); + + return values.length > 0 ? [...new Set(values)] : undefined; +} + +function readNumberParam(params: URLSearchParams, key: string, opts?: { + integer?: boolean; + min?: number; +}): number | undefined { + const raw = params.get(key); + if (raw === null || raw.trim() === '') { + return undefined; + } + + const value = Number(raw); + if (!Number.isFinite(value)) { + throw new Error(`Invalid numeric query parameter "${key}"`); + } + if (opts?.integer && !Number.isInteger(value)) { + throw new Error(`Query parameter "${key}" must be an integer`); + } + if (opts?.min !== undefined && value < opts.min) { + throw new Error(`Query parameter "${key}" must be >= ${opts.min}`); + } + + return value; +} + +function readEnumList( + params: URLSearchParams, + key: string, + allowed: readonly T[] +): T[] | undefined { + const values = readStringList(params, key); + if (!values) { + return undefined; + } + + const invalid = values.filter((value) => !allowed.includes(value as T)); + if (invalid.length > 0) { + throw new Error(`Invalid "${key}" values: ${invalid.join(', ')}`); + } + + return values as T[]; +} + +function parseRuntimeListOptions(agentId: string, params: URLSearchParams): ObservationListOptions { + return { + agentId, + kinds: readEnumList(params, 'kinds', OBSERVATION_KINDS), + statuses: readEnumList(params, 'statuses', OBSERVATION_STATUSES), + limit: readNumberParam(params, 'limit', { integer: true, min: 1 }), + sinceSeq: readNumberParam(params, 'sinceSeq', { integer: true, min: 0 }), + runId: params.get('runId') || undefined, + traceId: params.get('traceId') || undefined, + parentSpanId: params.get('parentSpanId') || undefined, + }; +} + +function parsePersistedListOptions(agentId: string, params: URLSearchParams): PersistedObservationListOptions { + const options: PersistedObservationListOptions = { + ...parseRuntimeListOptions(agentId, params), + agentIds: [agentId], + templateIds: readStringList(params, 'templateIds'), + fromTimestamp: readNumberParam(params, 'fromTimestamp'), + toTimestamp: readNumberParam(params, 'toTimestamp'), + }; + + if ( + options.fromTimestamp !== undefined && + options.toTimestamp !== undefined && + options.fromTimestamp > options.toTimestamp + ) { + throw new Error('Query parameter "fromTimestamp" must be <= "toTimestamp"'); + } + + return options; +} + +function buildObservationListBody( + agentId: string, + source: 'runtime' | 'persisted', + observations: ObservationEnvelope[] +) { + return { agentId, source, observations }; +} + +function buildRunBody(agentId: string, source: 'runtime' | 'persisted', runView: ObservationRunView) { + return { + agentId, + source, + run: runView.run, + observations: runView.observations, + }; +} + +export function createExampleObservabilityHttpHandler( + config: ObservabilityHttpHandlerConfig +): (request: ObservabilityHttpRequest) => Promise { + const basePath = normalizeBasePath(config.basePath); + + return async (request) => { + const method = (request.method || 'GET').toUpperCase(); + const parsed = new URL(request.url, 'http://kode-observability.local'); + const relativeSegments = stripBasePath(parsed.pathname, basePath); + + if (!relativeSegments) { + return buildJsonResponse(404, { error: 'not_found' }); + } + + const [scope, agentId, resource, source, qualifier, targetId] = relativeSegments; + + if (method !== 'GET') { + return { + status: 405, + headers: { + 'content-type': 'application/json; charset=utf-8', + allow: 'GET', + }, + body: { + error: 'method_not_allowed', + message: `Method ${method} is not supported for ${parsed.pathname}`, + }, + }; + } + + if (scope !== 'agents' || !agentId) { + return buildJsonResponse(404, { error: 'not_found' }); + } + + try { + if (resource === 'metrics') { + const runtimeSource = config.resolveRuntimeSource ? await config.resolveRuntimeSource(agentId) : undefined; + if (!runtimeSource) { + return buildJsonResponse(404, { error: 'not_found', message: `Agent "${agentId}" was not found` }); + } + return buildJsonResponse(200, await runtimeSource.getMetricsSnapshot()); + } + + if (resource !== 'observations' || (source !== 'runtime' && source !== 'persisted')) { + return buildJsonResponse(404, { error: 'not_found' }); + } + + if (source === 'runtime') { + const runtimeSource = config.resolveRuntimeSource ? await config.resolveRuntimeSource(agentId) : undefined; + if (!runtimeSource) { + return buildJsonResponse(404, { error: 'not_found', message: `Agent "${agentId}" was not found` }); + } + + const reader = await runtimeSource.getObservationReader(); + + if (qualifier === 'runs' && targetId) { + const runView = reader.getRun(targetId); + return runView + ? buildJsonResponse(200, buildRunBody(agentId, 'runtime', runView)) + : buildJsonResponse(404, { error: 'not_found', message: `Run "${targetId}" was not found` }); + } + + return buildJsonResponse( + 200, + buildObservationListBody(agentId, 'runtime', reader.listObservations(parseRuntimeListOptions(agentId, parsed.searchParams))) + ); + } + + const reader = config.resolvePersistedReader ? await config.resolvePersistedReader(agentId) : undefined; + if (!reader) { + return buildJsonResponse(404, { + error: 'not_found', + message: `Persisted reader for agent "${agentId}" was not found`, + }); + } + + if (qualifier === 'runs' && targetId) { + const runView = await reader.getRun(targetId); + return runView + ? buildJsonResponse(200, buildRunBody(agentId, 'persisted', runView)) + : buildJsonResponse(404, { error: 'not_found', message: `Run "${targetId}" was not found` }); + } + + return buildJsonResponse( + 200, + buildObservationListBody(agentId, 'persisted', await reader.listObservations(parsePersistedListOptions(agentId, parsed.searchParams))) + ); + } catch (error: any) { + return buildJsonResponse(400, { + error: 'bad_request', + message: error?.message || 'Invalid request', + }); + } + }; +} diff --git a/package-lock.json b/package-lock.json index 12b3899..10bb48f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@shareai-lab/kode-sdk", - "version": "2.7.4", + "version": "2.7.5", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@shareai-lab/kode-sdk", - "version": "2.7.4", + "version": "2.7.5", "license": "MIT", "dependencies": { "@alibaba-group/opensandbox": "~0.1.4", diff --git a/package.json b/package.json index 588aaf1..3c137ae 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@shareai-lab/kode-sdk", - "version": "2.7.4", + "version": "2.7.5", "description": "Event-driven, long-running AI Agent development framework with enterprise-grade persistence and context management", "main": "dist/index.js", "types": "dist/index.d.ts", @@ -26,6 +26,7 @@ "example:openrouter": "ts-node examples/05-openrouter-complete.ts", "example:openrouter-stream": "ts-node examples/06-openrouter-stream.ts", "example:openrouter-agent": "ts-node examples/07-openrouter-agent.ts", + "example:observability-http": "ts-node examples/08-observability-http.ts", "example:db-sqlite": "ts-node examples/db-sqlite.ts", "example:db-postgres": "ts-node examples/db-postgres.ts" }, diff --git a/src/core/agent.ts b/src/core/agent.ts index 6273687..cf20a75 100644 --- a/src/core/agent.ts +++ b/src/core/agent.ts @@ -51,6 +51,28 @@ import { MessageQueue, SendOptions as QueueSendOptions } from './agent/message-q import { TodoManager } from './agent/todo-manager'; import { ToolRunner } from './agent/tool-runner'; import { logger } from '../utils/logger'; +import { + AgentMetricsSnapshot, + CompositeObservationSink, + CompressionObservation, + GenerationObservation, + ObservationCollector, + ObservationEnvelope, + ObservationKind, + ObservationPersistenceConfig, + ObservationReader, + ObservationSink, + PersistedObservationSink, + OTelObservationSink, + ObservationSubscribeOptions, + ObservabilityConfig, + ToolObservation, + createObservationReader, + generateRunId, + generateSpanId, + generateTraceId, +} from '../observability'; +import { UsageStatistics } from '../infra/providers/core/usage'; const CONFIG_VERSION = 'v2.7.0'; @@ -98,6 +120,7 @@ export interface AgentConfig { }; context?: ContextManagerOptions; metadata?: Record; + observability?: ObservabilityConfig; } interface AgentMetadata { @@ -131,6 +154,35 @@ interface SubAgentRuntime { depthRemaining: number; } +interface ActiveRunContext { + runId: string; + traceId: string; + spanId: string; + parentSpanId?: string; + startTime: number; + trigger: 'send' | 'complete' | 'resume' | 'scheduler' | 'delegate'; + step: number; + messageCountBefore: number; + scheduler?: { + taskId: string; + spec: string; + kind: 'steps' | 'time' | 'cron'; + triggeredAt: number; + }; +} + +interface PendingSchedulerRunContext { + taskId: string; + spec: string; + kind: 'steps' | 'time' | 'cron'; + triggeredAt: number; +} + +interface InheritedObservationContext { + traceId?: string; + parentSpanId?: string; +} + export interface CompleteResult { status: 'ok' | 'paused'; text?: string; @@ -183,6 +235,8 @@ export class Agent { private readonly pendingPermissions = new Map(); private readonly toolRunner: ToolRunner; + private readonly observationCollector: ObservationCollector; + private readonly observationReader: ObservationReader; private messages: Message[] = []; private state: AgentRuntimeState = 'READY'; @@ -204,6 +258,9 @@ export class Agent { private template: AgentTemplateDefinition; private lineage: string[] = []; private mediaCache = new Map(); + private activeRunContext?: ActiveRunContext; + private activeSchedulerTrigger?: PendingSchedulerRunContext; + private pendingSchedulerRunContext?: PendingSchedulerRunContext; private get persistentStore(): Store { if (!this.deps.store) { @@ -219,6 +276,45 @@ export class Agent { return deps.store; } + private buildObservationSink(): ObservationSink | undefined { + const observability = this.config.observability; + const sinks: ObservationSink[] = []; + + if (observability?.sink) { + sinks.push(observability.sink); + } + + if (observability?.otel && observability.otel.enabled !== false) { + sinks.push(new OTelObservationSink(observability.otel)); + } + + const persistence = observability?.persistence; + if (this.shouldEnableObservationPersistence(persistence)) { + sinks.push( + new PersistedObservationSink(persistence.backend!, { + retention: persistence.retention, + pruneIntervalMs: persistence.pruneIntervalMs, + }) + ); + } + + if (sinks.length === 0) { + return undefined; + } + + if (sinks.length === 1) { + return sinks[0]; + } + + return new CompositeObservationSink(sinks); + } + + private shouldEnableObservationPersistence( + persistence?: ObservationPersistenceConfig + ): persistence is ObservationPersistenceConfig & { backend: NonNullable } { + return !!persistence?.backend && persistence.enabled !== false; + } + constructor( private readonly config: AgentConfig, private readonly deps: AgentDependencies, @@ -275,6 +371,15 @@ export class Agent { triggeredAt: Date.now(), }); }, + onTriggerStart: (info) => { + this.activeSchedulerTrigger = { + ...info, + triggeredAt: Date.now(), + }; + }, + onTriggerEnd: () => { + this.activeSchedulerTrigger = undefined; + }, }); const runtimeMeta = { ...(this.template.runtime?.metadata || {}), ...(config.metadata || {}) } as Record; this.createdAt = new Date().toISOString(); @@ -336,6 +441,13 @@ export class Agent { remind: (content, options) => this.remind(content, options), }); + this.observationCollector = new ObservationCollector( + this.agentId, + config.observability?.enabled !== false, + this.buildObservationSink() + ); + this.observationReader = createObservationReader(this.observationCollector); + this.events.setStore(this.persistentStore, this.agentId); // 自动注入工具说明书到系统提示 @@ -501,6 +613,9 @@ export class Agent { } async send(message: string | ContentBlock[], options?: SendOptions): Promise { + if (this.activeSchedulerTrigger && (options?.kind ?? 'user') === 'user') { + this.pendingSchedulerRunContext = { ...this.activeSchedulerTrigger }; + } if (typeof message === 'string') { return this.messageQueue.send(message, options); } @@ -527,6 +642,18 @@ export class Agent { return this.events.subscribe(channels, { since: opts.since, kinds: opts.kinds }); } + getMetricsSnapshot(): AgentMetricsSnapshot { + return this.observationReader.getMetricsSnapshot(); + } + + subscribeObservations(opts?: ObservationSubscribeOptions): AsyncIterable { + return this.observationReader.subscribe(opts); + } + + getObservationReader(): ObservationReader { + return this.observationReader; + } + getTodos(): TodoItem[] { return this.todoManager.list(); } @@ -700,6 +827,10 @@ export class Agent { model?: string | ModelProvider | DelegateTaskModelConfig; tools?: string[]; }): Promise { + const subagentStart = Date.now(); + const traceId = this.activeRunContext?.traceId || generateTraceId(); + const subagentSpanId = generateSpanId(); + const parentSpanId = this.activeRunContext?.spanId; const parentModelConfig = this.model.toConfig(); const subAgentConfig: AgentConfig = { templateId: config.templateId, @@ -713,6 +844,10 @@ export class Agent { delegatedBy: 'task_tool', }, }; + this.setInheritedObservationContext(subAgentConfig, { + traceId, + parentSpanId: subagentSpanId, + }); if (typeof config.model === 'string') { subAgentConfig.modelConfig = { @@ -743,7 +878,32 @@ export class Agent { subAgent.lineage = [...this.lineage, this.agentId]; try { const result = await subAgent.complete(config.prompt); + const childRunId = subAgent.getMetricsSnapshot().currentRunId; + this.emitSubagentObservation({ + traceId, + parentSpanId, + spanId: subagentSpanId, + startTime: subagentStart, + childAgentId: subAgent.agentId, + childRunId, + templateId: config.templateId, + delegatedBy: 'task_tool', + status: result.status === 'ok' ? 'ok' : 'cancelled', + }); return result; + } catch (error: any) { + this.emitSubagentObservation({ + traceId, + parentSpanId, + spanId: subagentSpanId, + startTime: subagentStart, + childAgentId: subAgent.agentId, + templateId: config.templateId, + delegatedBy: 'task_tool', + status: 'error', + errorMessage: error?.message || 'Subagent execution failed', + }); + throw error; } finally { await (subAgent as any).sandbox?.dispose?.(); } @@ -1014,40 +1174,79 @@ export class Agent { this.setState('WORKING'); this.setBreakpoint('PRE_MODEL'); let doneEmitted = false; + const runContext = this.beginRunObservation('send'); + const generationStartTime = Date.now(); + let generationFirstTokenAt: number | undefined; + let generationStopReason: string | undefined; + let generationUsage: { inputTokens: number; outputTokens: number; totalTokens: number } | undefined; + let generationExtendedUsage: UsageStatistics | undefined; + let generationInputMessagesForObservation: Message[] = []; + let assistantBlocksForObservation: ContentBlock[] = []; try { await this.messageQueue.flush(); const usage = this.contextManager.analyze(this.messages); if (usage.shouldCompress) { + const compressionStartTime = Date.now(); + const messageCountBefore = this.messages.length; + const estimatedTokensBefore = usage.totalTokens; this.events.emitMonitor({ channel: 'monitor', type: 'context_compression', phase: 'start', }); - const compression = await this.contextManager.compress( - this.messages, - this.events.getTimeline(), - this.filePool, - this.sandbox - ); + try { + const compression = await this.contextManager.compress( + this.messages, + this.events.getTimeline(), + this.filePool, + this.sandbox + ); - if (compression) { - this.messages = [...compression.retainedMessages]; - this.messages.unshift(compression.summary); - this.lastSfpIndex = this.messages.length - 1; - await this.persistMessages(); - this.events.emitMonitor({ - channel: 'monitor', - type: 'context_compression', - phase: 'end', - summary: compression.summary.content.map((block) => (block.type === 'text' ? block.text : JSON.stringify(block))).join('\n'), - ratio: compression.ratio, + if (compression) { + const compressedMessages = [compression.summary, ...compression.retainedMessages]; + const afterUsage = this.contextManager.analyze(compressedMessages); + this.messages = compressedMessages; + this.lastSfpIndex = this.messages.length - 1; + await this.persistMessages(); + this.events.emitMonitor({ + channel: 'monitor', + type: 'context_compression', + phase: 'end', + summary: compression.summary.content.map((block) => (block.type === 'text' ? block.text : JSON.stringify(block))).join('\n'), + ratio: compression.ratio, + }); + this.emitCompressionObservation({ + context: runContext, + startTime: compressionStartTime, + endTime: Date.now(), + messageCountBefore, + messageCountAfter: compressedMessages.length, + estimatedTokensBefore, + estimatedTokensAfter: afterUsage.totalTokens, + ratio: compression.ratio, + summaryGenerated: true, + status: 'ok', + }); + } + } catch (error: any) { + this.emitCompressionObservation({ + context: runContext, + startTime: compressionStartTime, + endTime: Date.now(), + messageCountBefore, + estimatedTokensBefore, + summaryGenerated: false, + status: 'error', + errorMessage: error?.message || 'Context compression failed', }); + throw error; } } await this.hooks.runPreModel(this.messages); + generationInputMessagesForObservation = this.cloneMessages(this.messages); this.setBreakpoint('STREAMING_MODEL'); let assistantBlocks: ContentBlock[] = []; @@ -1065,6 +1264,7 @@ export class Agent { for await (const chunk of stream) { if (chunk.type === 'content_block_start') { + generationFirstTokenAt ??= Date.now(); if (chunk.content_block?.type === 'text') { currentBlockIndex = chunk.index ?? 0; textBuffers.set(currentBlockIndex, ''); @@ -1097,6 +1297,7 @@ export class Agent { }; } } else if (chunk.type === 'content_block_delta') { + generationFirstTokenAt ??= Date.now(); if (chunk.delta?.type === 'text_delta') { const text = chunk.delta.text ?? ''; const existing = textBuffers.get(currentBlockIndex) ?? ''; @@ -1129,6 +1330,11 @@ export class Agent { } else if (chunk.type === 'message_delta') { const inputTokens = (chunk.usage as any)?.input_tokens ?? 0; const outputTokens = (chunk.usage as any)?.output_tokens ?? 0; + generationUsage = { + inputTokens, + outputTokens, + totalTokens: inputTokens + outputTokens, + }; if (inputTokens || outputTokens) { this.events.emitMonitor({ channel: 'monitor', @@ -1138,6 +1344,9 @@ export class Agent { totalTokens: inputTokens + outputTokens, }); } + } else if (chunk.type === 'message_stop') { + generationStopReason = chunk.stop_reason ?? generationStopReason; + generationExtendedUsage = chunk.extendedUsage ?? generationExtendedUsage; } else if (chunk.type === 'content_block_stop') { if (assistantBlocks[currentBlockIndex]?.type === 'text') { const fullText = textBuffers.get(currentBlockIndex) ?? ''; @@ -1154,6 +1363,7 @@ export class Agent { assistantBlocks = this.compactContentBlocks(assistantBlocks); assistantBlocks = this.splitThinkBlocksIfNeeded(assistantBlocks); + assistantBlocksForObservation = assistantBlocks; await this.hooks.runPostModel({ role: 'assistant', content: assistantBlocks } as any); @@ -1167,6 +1377,18 @@ export class Agent { await this.persistMessages(); const toolBlocks = assistantBlocks.filter((block) => block.type === 'tool_use'); + this.emitGenerationObservation({ + context: runContext, + startTime: generationStartTime, + endTime: Date.now(), + firstTokenAt: generationFirstTokenAt, + inputMessages: generationInputMessagesForObservation, + assistantBlocks: assistantBlocksForObservation, + usage: generationUsage, + extendedUsage: generationExtendedUsage, + stopReason: generationStopReason, + status: 'ok', + }); if (toolBlocks.length > 0) { this.setBreakpoint('TOOL_PENDING'); const outcomes = await this.executeTools(toolBlocks); @@ -1177,6 +1399,7 @@ export class Agent { await this.persistMessages(); this.todoManager.onStep(); this.ensureProcessing(); + this.finishRunObservation(runContext, 'ok'); return; } } else { @@ -1199,7 +1422,21 @@ export class Agent { await this.persistInfo(); } this.events.emitMonitor({ channel: 'monitor', type: 'step_complete', step: this.stepCount, bookmark: envelope.bookmark }); + this.finishRunObservation(runContext, 'ok'); } catch (error: any) { + this.emitGenerationObservation({ + context: runContext, + startTime: generationStartTime, + endTime: Date.now(), + firstTokenAt: generationFirstTokenAt, + inputMessages: generationInputMessagesForObservation, + assistantBlocks: assistantBlocksForObservation, + usage: generationUsage, + extendedUsage: generationExtendedUsage, + stopReason: generationStopReason, + status: 'error', + errorMessage: error?.message || 'Model execution failed', + }); this.events.emitMonitor({ channel: 'monitor', type: 'error', @@ -1225,6 +1462,7 @@ export class Agent { await this.persistInfo(); } } + this.finishRunObservation(runContext, 'error', error?.message || 'Model execution failed'); } finally { this.setState('READY'); this.setBreakpoint('READY'); @@ -1277,6 +1515,7 @@ export class Agent { const message = `Tool not found: ${toolUse.name}`; this.updateToolRecord(record.id, { state: 'FAILED', error: message, isError: true }, 'tool missing'); this.events.emitMonitor({ channel: 'monitor', type: 'error', severity: 'warn', phase: 'tool', message }); + this.emitToolObservation(record.id); return this.makeToolResult(toolUse.id, { ok: false, error: message, @@ -1288,6 +1527,7 @@ export class Agent { if (!validation.ok) { const message = validation.error || 'Tool input validation failed'; this.updateToolRecord(record.id, { state: 'FAILED', error: message, isError: true }, 'input schema invalid'); + this.emitToolObservation(record.id); return this.makeToolResult(toolUse.id, { ok: false, error: message, @@ -1320,12 +1560,13 @@ export class Agent { isError: true, }, 'policy deny' - ); - this.setBreakpoint('POST_TOOL'); - this.events.emitProgress({ channel: 'progress', type: 'tool:end', call: this.snapshotToolRecord(record.id) }); - return this.makeToolResult(toolUse.id, { - ok: false, - error: message, + ); + this.setBreakpoint('POST_TOOL'); + this.events.emitProgress({ channel: 'progress', type: 'tool:end', call: this.snapshotToolRecord(record.id) }); + this.emitToolObservation(record.id); + return this.makeToolResult(toolUse.id, { + ok: false, + error: message, recommendations: ['检查模板或权限配置的 allow/deny 列表', '如需执行该工具，请调整权限模式或审批策略'], }); } @@ -1378,6 +1619,7 @@ export class Agent { this.events.emitMonitor({ channel: 'monitor', type: 'tool_executed', call: this.snapshotToolRecord(record.id) }); this.setBreakpoint('POST_TOOL'); this.events.emitProgress({ channel: 'progress', type: 'tool:end', call: this.snapshotToolRecord(record.id) }); + this.emitToolObservation(record.id); return this.makeToolResult(toolUse.id, { ok: true, data: decision.result }); } } @@ -1390,6 +1632,7 @@ export class Agent { this.updateToolRecord(record.id, { state: 'DENIED', error: message, isError: true }, 'approval denied'); this.setBreakpoint('POST_TOOL'); this.events.emitProgress({ channel: 'progress', type: 'tool:end', call: this.snapshotToolRecord(record.id) }); + this.emitToolObservation(record.id); return this.makeToolResult(toolUse.id, { ok: false, error: message }); } this.setBreakpoint('PRE_TOOL'); @@ -1443,6 +1686,7 @@ export class Agent { resultData = (outcome.content as any).data; } + this.emitToolObservation(record.id); return this.makeToolResult(toolUse.id, { ok: true, data: resultData }); } else { const errorContent = outcome.content as any; @@ -1482,6 +1726,7 @@ export class Agent { const recommendations = errorContent?.recommendations || this.getErrorRecommendations(errorType, toolUse.name); + this.emitToolObservation(record.id); return this.makeToolResult(toolUse.id, { ok: false, error: errorMessage, @@ -1522,6 +1767,7 @@ export class Agent { ? ['检查是否手动中断', '根据需要重新触发工具', '考虑调整超时时间'] : this.getErrorRecommendations('runtime', toolUse.name); + this.emitToolObservation(record.id); return this.makeToolResult(toolUse.id, { ok: false, error: message, @@ -1597,6 +1843,357 @@ export class Agent { return text.length > limit ? `${text.slice(0, limit)}…` : text; } + private beginRunObservation( + trigger: 'send' | 'complete' | 'resume' | 'scheduler' | 'delegate' = 'send' + ): ActiveRunContext { + const inherited = this.getInheritedObservationContext(); + + const schedulerContext = trigger === 'send' ? this.pendingSchedulerRunContext : undefined; + const resolvedTrigger = schedulerContext ? 'scheduler' : trigger; + + const context: ActiveRunContext = { + runId: generateRunId(), + traceId: inherited.traceId || generateTraceId(), + spanId: generateSpanId(), + parentSpanId: inherited.parentSpanId, + startTime: Date.now(), + trigger: resolvedTrigger, + step: this.stepCount, + messageCountBefore: this.messages.length, + scheduler: schedulerContext, + }; + + this.activeRunContext = context; + if (schedulerContext) { + this.pendingSchedulerRunContext = undefined; + } + return context; + } + + private finishRunObservation( + context: ActiveRunContext | undefined, + status: 'ok' | 'error' | 'cancelled', + errorMessage?: string + ): void { + if (!context) return; + this.observationCollector.record({ + kind: 'agent_run', + agentId: this.agentId, + runId: context.runId, + traceId: context.traceId, + spanId: context.spanId, + parentSpanId: context.parentSpanId, + name: `agent_run:${this.template.id}`, + status, + startTime: context.startTime, + endTime: Date.now(), + durationMs: Date.now() - context.startTime, + trigger: context.trigger, + step: context.step, + messageCountBefore: context.messageCountBefore, + messageCountAfter: this.messages.length, + errorMessage, + metadata: { + templateId: this.template.id, + ...(context.scheduler ? { scheduler: { ...context.scheduler } } : {}), + }, + }); + if (this.activeRunContext?.runId === context.runId) { + this.activeRunContext = undefined; + } + } + + private summarizeObservationValue(value: any, mode: 'off' | 'summary' | 'full' | 'redacted' = 'summary'): unknown { + if (mode === 'off' || value === undefined) { + return undefined; + } + if (mode === 'full') { + return value; + } + const text = typeof value === 'string' ? value : JSON.stringify(value); + const summary = text.length > 200 ? `${text.slice(0, 200)}…` : text; + if (mode === 'redacted') { + return summary + .replace(/sk-[a-zA-Z0-9_-]+/g, 'sk-***') + .replace(/Bearer\\s+[A-Za-z0-9._-]+/g, 'Bearer ***'); + } + return summary; + } + + private cloneMessages(messages: Message[]): Message[] { + return messages.map((message) => ({ + ...message, + content: message.content.map((block) => ({ ...block })) as ContentBlock[], + metadata: message.metadata ? { ...message.metadata } : undefined, + })); + } + + private sanitizePersistedMetadata(metadata?: Record): Record | undefined { + if (!metadata) { + return metadata; + } + + const sanitized = { ...metadata }; + delete sanitized.__observationTraceId; + delete sanitized.__observationParentSpanId; + return Object.keys(sanitized).length > 0 ? sanitized : undefined; + } + + private emitGenerationObservation(params: { + context?: ActiveRunContext; + startTime: number; + endTime: number; + firstTokenAt?: number; + inputMessages: Message[]; + assistantBlocks: ContentBlock[]; + usage?: { inputTokens: number; outputTokens: number; totalTokens: number }; + extendedUsage?: UsageStatistics; + stopReason?: string; + status: 'ok' | 'error'; + errorMessage?: string; + }): void { + const context = params.context; + if (!context) return; + + const config = this.model.toConfig(); + const capture = this.config.observability?.capture; + const extendedUsage = params.extendedUsage; + const usage = extendedUsage + ? { + inputTokens: extendedUsage.inputTokens, + outputTokens: extendedUsage.outputTokens, + totalTokens: extendedUsage.totalTokens, + reasoningTokens: extendedUsage.reasoningTokens, + cacheCreationTokens: extendedUsage.cache.cacheCreationTokens, + cacheReadTokens: extendedUsage.cache.cacheReadTokens, + } + : params.usage; + const durationMs = params.endTime - params.startTime; + const timeToFirstTokenMs = + params.firstTokenAt !== undefined ? Math.max(0, params.firstTokenAt - params.startTime) : undefined; + const metadata: Record = { + templateId: this.template.id, + }; + + if (extendedUsage) { + // Keep provider/debug usage available for troubleshooting, but do not + // expose it as a first-class stable observation field. + metadata.__debug = { + extendedUsage, + }; + } + + const observation: GenerationObservation = { + kind: 'generation', + agentId: this.agentId, + runId: context.runId, + traceId: context.traceId, + spanId: generateSpanId(), + parentSpanId: context.spanId, + name: `generation:${config.model}`, + status: params.status, + startTime: params.startTime, + endTime: params.endTime, + durationMs, + provider: config.provider, + model: config.model, + requestId: extendedUsage?.request.requestId, + inputSummary: this.summarizeObservationValue(params.inputMessages, capture?.generationInput), + outputSummary: this.summarizeObservationValue(params.assistantBlocks, capture?.generationOutput), + usage, + cost: extendedUsage?.cost, + request: { + latencyMs: extendedUsage?.request.latencyMs ?? durationMs, + timeToFirstTokenMs: extendedUsage?.request.timeToFirstTokenMs ?? timeToFirstTokenMs, + tokensPerSecond: extendedUsage?.request.tokensPerSecond, + stopReason: extendedUsage?.request.stopReason ?? params.stopReason, + retryCount: extendedUsage?.request.retryCount, + }, + errorMessage: params.errorMessage, + metadata, + }; + + this.observationCollector.record(observation); + } + + private getInheritedObservationContext(): InheritedObservationContext { + const configWithInternal = this.config as AgentConfig & { + __observationTraceId?: string; + __observationParentSpanId?: string; + }; + const metadata = this.config.metadata || {}; + return { + traceId: + typeof configWithInternal.__observationTraceId === 'string' + ? configWithInternal.__observationTraceId + : typeof metadata.__observationTraceId === 'string' + ? metadata.__observationTraceId + : undefined, + parentSpanId: + typeof configWithInternal.__observationParentSpanId === 'string' + ? configWithInternal.__observationParentSpanId + : typeof metadata.__observationParentSpanId === 'string' + ? metadata.__observationParentSpanId + : undefined, + }; + } + + private setInheritedObservationContext(config: AgentConfig, context: InheritedObservationContext): void { + const target = config as AgentConfig & { + __observationTraceId?: string; + __observationParentSpanId?: string; + }; + target.__observationTraceId = context.traceId; + target.__observationParentSpanId = context.parentSpanId; + } + + private emitToolObservation(recordId: string): void { + const record = this.toolRecords.get(recordId); + const context = this.activeRunContext; + if (!record || !context) return; + + const capture = this.config.observability?.capture; + const approval = this.buildToolApprovalObservation(record); + const observation: ToolObservation = { + kind: 'tool', + agentId: this.agentId, + runId: context.runId, + traceId: context.traceId, + spanId: generateSpanId(), + parentSpanId: context.spanId, + name: `tool:${record.name}`, + status: record.state === 'COMPLETED' ? 'ok' : 'error', + startTime: record.startedAt ?? record.createdAt, + endTime: record.completedAt ?? record.updatedAt, + durationMs: record.durationMs, + toolCallId: record.id, + toolName: record.name, + toolState: record.state, + approvalRequired: record.approval.required, + approval, + inputSummary: this.summarizeObservationValue(record.input, capture?.toolInput), + outputSummary: this.summarizeObservationValue(record.result, capture?.toolOutput), + errorMessage: record.error, + metadata: { + isError: record.isError ?? false, + }, + }; + + this.observationCollector.record(observation); + } + + private buildToolApprovalObservation(record: ToolCallRecord): ToolObservation['approval'] { + const approval = record.approval; + if (!approval.required) { + return { + required: false, + status: 'not_required', + }; + } + + let status: 'pending' | 'approved' | 'denied'; + if (!approval.decision) { + status = 'pending'; + } else if (approval.decision === 'allow') { + status = 'approved'; + } else { + status = 'denied'; + } + + const requestedAt = approval.requestedAt; + const decidedAt = approval.decidedAt; + const waitMs = + requestedAt !== undefined && decidedAt !== undefined ? Math.max(0, decidedAt - requestedAt) : undefined; + + return { + required: true, + status, + approvalId: record.id, + requestedAt, + decidedAt, + waitMs, + noteSummary: this.summarizeObservationValue(approval.note, 'redacted') as string | undefined, + }; + } + + private emitCompressionObservation(params: { + context?: ActiveRunContext; + startTime: number; + endTime: number; + messageCountBefore: number; + messageCountAfter?: number; + estimatedTokensBefore?: number; + estimatedTokensAfter?: number; + ratio?: number; + summaryGenerated: boolean; + status: 'ok' | 'error'; + errorMessage?: string; + }): void { + const context = params.context; + if (!context) return; + + const observation: CompressionObservation = { + kind: 'compression', + agentId: this.agentId, + runId: context.runId, + traceId: context.traceId, + spanId: generateSpanId(), + parentSpanId: context.spanId, + name: 'compression:context_window', + status: params.status, + startTime: params.startTime, + endTime: params.endTime, + durationMs: params.endTime - params.startTime, + policy: 'context_window', + reason: 'token_threshold', + messageCountBefore: params.messageCountBefore, + messageCountAfter: params.messageCountAfter, + estimatedTokensBefore: params.estimatedTokensBefore, + estimatedTokensAfter: params.estimatedTokensAfter, + ratio: params.ratio, + summaryGenerated: params.summaryGenerated, + errorMessage: params.errorMessage, + metadata: { + templateId: this.template.id, + }, + }; + + this.observationCollector.record(observation); + } + + private emitSubagentObservation(params: { + traceId: string; + parentSpanId?: string; + spanId: string; + startTime: number; + childAgentId: string; + childRunId?: string; + templateId: string; + delegatedBy?: string; + status: 'ok' | 'error' | 'cancelled'; + errorMessage?: string; + }): void { + const runId = this.activeRunContext?.runId ?? generateRunId(); + this.observationCollector.record({ + kind: 'subagent', + agentId: this.agentId, + runId, + traceId: params.traceId, + spanId: params.spanId, + parentSpanId: params.parentSpanId, + name: `subagent:${params.templateId}`, + status: params.status, + startTime: params.startTime, + endTime: Date.now(), + durationMs: Date.now() - params.startTime, + childAgentId: params.childAgentId, + childRunId: params.childRunId, + templateId: params.templateId, + delegatedBy: params.delegatedBy, + errorMessage: params.errorMessage, + }); + } + private validateBlocks(blocks: ContentBlock[]): void { const config = this.model.toConfig(); const multimodal = config.multimodal || {}; @@ -1910,6 +2507,7 @@ export class Agent { private async requestPermission(id: string, _toolName: string, _args: any, meta?: any): Promise<'allow' | 'deny'> { const approval: ToolCallApproval = { required: true, + requestedAt: Date.now(), decision: undefined, decidedAt: undefined, decidedBy: undefined, @@ -1925,7 +2523,7 @@ export class Agent { this.updateToolRecord( id, { - approval: buildApproval(decision, 'api', note), + approval: buildApproval(decision, 'api', note, this.toolRecords.get(id)?.approval), state: decision === 'allow' ? 'APPROVED' : 'DENIED', error: decision === 'deny' ? note : undefined, isError: decision === 'deny', @@ -2200,7 +2798,7 @@ export class Agent { createdAt: this.createdAt, updatedAt: new Date().toISOString(), configVersion: CONFIG_VERSION, - metadata: this.config.metadata, + metadata: this.sanitizePersistedMetadata(this.config.metadata), lineage: this.lineage, breakpoint: this.breakpoints.getCurrent(), }; @@ -2613,12 +3211,19 @@ function encodeUlid(time: number, length: number, chars: string): string { return encoded.join(''); } -function buildApproval(decision: 'allow' | 'deny', by: string, note?: string): ToolCallApproval { +function buildApproval( + decision: 'allow' | 'deny', + by: string, + note?: string, + existing?: ToolCallApproval +): ToolCallApproval { return { - required: true, + required: existing?.required ?? true, + requestedAt: existing?.requestedAt, decision, decidedBy: by, decidedAt: Date.now(), note, + meta: existing?.meta, }; } diff --git a/src/core/scheduler.ts b/src/core/scheduler.ts index 63296d2..cef5d3e 100644 --- a/src/core/scheduler.ts +++ b/src/core/scheduler.ts @@ -12,8 +12,16 @@ interface StepTask { type TriggerKind = 'steps' | 'time' | 'cron'; +export interface SchedulerTriggerInfo { + taskId: string; + spec: string; + kind: TriggerKind; +} + interface SchedulerOptions { - onTrigger?: (info: { taskId: string; spec: string; kind: TriggerKind }) => void; + onTrigger?: (info: SchedulerTriggerInfo) => void; + onTriggerStart?: (info: SchedulerTriggerInfo) => void; + onTriggerEnd?: (info: SchedulerTriggerInfo) => void; } export class Scheduler { @@ -21,9 +29,13 @@ export class Scheduler { private readonly listeners = new Set(); private queued: Promise = Promise.resolve(); private readonly onTrigger?: SchedulerOptions['onTrigger']; + private readonly onTriggerStart?: SchedulerOptions['onTriggerStart']; + private readonly onTriggerEnd?: SchedulerOptions['onTriggerEnd']; constructor(opts?: SchedulerOptions) { this.onTrigger = opts?.onTrigger; + this.onTriggerStart = opts?.onTriggerStart; + this.onTriggerEnd = opts?.onTriggerEnd; } everySteps(every: number, callback: StepCallback): AgentSchedulerHandle { @@ -58,8 +70,8 @@ export class Scheduler { const shouldTrigger = stepCount - task.lastTriggered >= task.every; if (!shouldTrigger) continue; task.lastTriggered = stepCount; - void Promise.resolve(task.callback({ stepCount })); - this.onTrigger?.({ taskId: task.id, spec: `steps:${task.every}`, kind: 'steps' }); + const info: SchedulerTriggerInfo = { taskId: task.id, spec: `steps:${task.every}`, kind: 'steps' }; + this.runTriggeredTask(info, () => task.callback({ stepCount }), { emitTrigger: 'before' }); } } @@ -76,7 +88,30 @@ export class Scheduler { this.onTrigger?.(info); } + runExternalTrigger(info: { taskId: string; spec: string; kind: 'time' | 'cron' }, callback: TaskCallback): void { + this.enqueue(() => this.runTriggeredTask(info, callback, { emitTrigger: 'afterSuccess' })); + } + private generateId(prefix: string): string { return `${prefix}-${Date.now()}-${Math.random().toString(36).slice(2, 9)}`; } + + private async runTriggeredTask( + info: SchedulerTriggerInfo, + callback: TaskCallback, + opts: { emitTrigger: 'before' | 'afterSuccess' } + ): Promise { + this.onTriggerStart?.(info); + try { + if (opts.emitTrigger === 'before') { + this.onTrigger?.(info); + } + await callback(); + if (opts.emitTrigger === 'afterSuccess') { + this.onTrigger?.(info); + } + } finally { + this.onTriggerEnd?.(info); + } + } } diff --git a/src/core/time-bridge.ts b/src/core/time-bridge.ts index cd5286a..51ec65b 100644 --- a/src/core/time-bridge.ts +++ b/src/core/time-bridge.ts @@ -39,13 +39,12 @@ export class TimeBridge { const spec = `every:${minutes}m`; const tick = (due: number) => { - this.scheduler.enqueue(async () => { + this.scheduler.runExternalTrigger({ taskId: id, spec, kind: 'time' }, async () => { const drift = Math.abs(Date.now() - due); if (drift > this.driftTolerance) { this.logger?.('timebridge:drift', { id, drift, expectedInterval: interval }); } await callback(); - this.scheduler.notifyExternalTrigger({ taskId: id, spec, kind: 'time' }); }); scheduleNext(); }; @@ -86,13 +85,12 @@ export class TimeBridge { }; const tick = (due: number) => { - this.scheduler.enqueue(async () => { + this.scheduler.runExternalTrigger({ taskId: id, spec: expr, kind: 'cron' }, async () => { const drift = Math.abs(Date.now() - due); if (drift > this.driftTolerance) { this.logger?.('timebridge:drift', { id, drift, spec: expr }); } await callback(); - this.scheduler.notifyExternalTrigger({ taskId: id, spec: expr, kind: 'cron' }); }); scheduleNext(); }; diff --git a/src/core/types.ts b/src/core/types.ts index a25c5c4..2d6ce45 100644 --- a/src/core/types.ts +++ b/src/core/types.ts @@ -98,6 +98,7 @@ export type ToolCallState = export interface ToolCallApproval { required: boolean; + requestedAt?: number; decision?: 'allow' | 'deny'; decidedBy?: string; decidedAt?: number; diff --git a/src/index.ts b/src/index.ts index 18ee26f..f72a5c7 100644 --- a/src/index.ts +++ b/src/index.ts @@ -24,6 +24,11 @@ export { } from './core/template'; export { TodoService, TodoItem, TodoSnapshot } from './core/todo'; export { TimeBridge } from './core/time-bridge'; +export type { + ObservabilityConfig, + ObservationPersistenceConfig, + OTelBridgeConfig, +} from './observability'; // Skills export { SkillsManager } from './core/skills'; @@ -123,5 +128,78 @@ export { extendSchema, } from './tools/type-inference'; +// Observability +export { + ObservationCollector, + CompositeObservationSink, + MemoryObservationSink, + MemoryObservationStore, + NoopObservationSink, + JSONStoreObservationBackend, + PostgresStoreObservationBackend, + OTLPHttpJsonExporter, + OTelObservationSink, + PersistedObservationSink, + SqliteStoreObservationBackend, + applyOTelPolicies, + applyObservationRetention, + buildBaseOTelAttributes, + buildOTLPTraceExportBody, + buildObservationSpecificAttributes, + createObservationReader, + createStoreBackedObservationReader, + createOTelExporter, + createOTelSpanTranslator, + generateRunId, + generateSpanId, + generateTraceId, + getOTelObservationMapping, + maskOTelSpan, + shouldExportOTelSpan, + shouldSampleOTelTrace, + toOTelSpanId, + toOTelTraceId, + translateObservationToOTelSpan, +} from './observability'; +export type { + AgentMetricsSnapshot, + AgentRunObservation, + BaseObservation, + CaptureMode, + CompressionObservation, + GenerationObservation, + ObservationEnvelope, + ObservationKind, + ObservationListOptions, + ObservationQueryOptions, + ObservationReader, + ObservationRecord, + ObservationRunView, + ObservationStatus, + ObservationSink, + ObservationSubscribeOptions, + ObservationPruneResult, + ObservationQueryBackend, + ObservationRetentionPolicy, + OTelAttributeNamespace, + OTelAttributeValue, + OTelExportMode, + OTelFetchLike, + OTelFilteringPolicy, + OTelHttpResponseLike, + OTelMaskingPolicy, + OTelObservationBridge, + OTelSamplingPolicy, + OTelSpanData, + OTelSpanEvent, + OTelSpanExporter, + OTelSpanKind, + OTLPHttpJsonExporterConfig, + PersistedObservationListOptions, + PersistedObservationReader, + SubagentObservation, + ToolObservation, +} from './observability'; + // Utils export { generateAgentId } from './utils/agent-id'; diff --git a/src/infra/providers/types.ts b/src/infra/providers/types.ts index e5e7ea7..4e28bbc 100644 --- a/src/infra/providers/types.ts +++ b/src/infra/providers/types.ts @@ -57,6 +57,8 @@ export interface ModelStreamChunk { input_tokens?: number; output_tokens: number; }; + extendedUsage?: UsageStatistics; + stop_reason?: string; } /** diff --git a/src/observability/collector.ts b/src/observability/collector.ts new file mode 100644 index 0000000..c973824 --- /dev/null +++ b/src/observability/collector.ts @@ -0,0 +1,265 @@ +import { logger } from '../utils/logger'; +import { buildObservationRunView, filterObservationEnvelopes, matchesObservationEnvelope } from './query'; +import { NoopObservationSink } from './sinks/noop'; +import { + AgentMetricsSnapshot, + GenerationObservation, + ObservationEnvelope, + ObservationKind, + ObservationListOptions, + ObservationReader, + ObservationRecord, + ObservationSink, + ObservationSubscribeOptions, + ObservationRunView, +} from './types'; + +class ObservationSubscriber { + private readonly queue: ObservationEnvelope[] = []; + private resolver?: (value: ObservationEnvelope | null) => void; + private closed = false; + + constructor(private readonly opts?: ObservationSubscribeOptions) {} + + push(envelope: ObservationEnvelope): void { + if (this.closed) return; + if (!matchesObservationEnvelope(envelope, this.opts)) return; + if (this.resolver) { + const resolve = this.resolver; + this.resolver = undefined; + resolve(envelope); + return; + } + this.queue.push(envelope); + } + + next(): Promise { + if (this.queue.length > 0) { + return Promise.resolve(this.queue.shift() || null); + } + if (this.closed) { + return Promise.resolve(null); + } + return new Promise((resolve) => { + this.resolver = resolve; + }); + } + + close(): void { + this.closed = true; + if (this.resolver) { + const resolve = this.resolver; + this.resolver = undefined; + resolve(null); + } + } +} + +export class ObservationCollector implements ObservationReader { + private seq = 0; + private readonly envelopes: ObservationEnvelope[] = []; + private readonly subscribers = new Set(); + private readonly sink: ObservationSink; + private snapshot: AgentMetricsSnapshot; + + constructor( + private readonly agentId: string, + private readonly enabled = true, + sink?: ObservationSink + ) { + this.sink = sink ?? new NoopObservationSink(); + this.snapshot = { + agentId, + totals: { + inputTokens: 0, + outputTokens: 0, + totalTokens: 0, + reasoningTokens: 0, + totalCostUsd: 0, + toolCalls: 0, + toolErrors: 0, + approvalRequests: 0, + approvalDenials: 0, + approvalWaitMsTotal: 0, + compressions: 0, + compressionErrors: 0, + tokensSavedEstimate: 0, + scheduledRuns: 0, + generations: 0, + generationErrors: 0, + subagents: 0, + }, + }; + } + + record(observation: ObservationRecord): void { + if (!this.enabled) return; + + const envelope: ObservationEnvelope = { + seq: this.seq++, + timestamp: Date.now(), + observation, + }; + + this.envelopes.push(envelope); + if (this.envelopes.length > 2000) { + this.envelopes.splice(0, this.envelopes.length - 1000); + } + + this.applyToSnapshot(observation); + + for (const subscriber of this.subscribers) { + subscriber.push(envelope); + } + + void Promise.resolve(this.sink.onObservation(envelope)).catch((error) => { + logger.warn('[Observability] Sink failed:', error); + }); + } + + subscribe(opts?: ObservationSubscribeOptions): AsyncIterable { + if (!this.enabled) { + return { + async *[Symbol.asyncIterator]() { + return; + }, + }; + } + + const subscriber = new ObservationSubscriber(opts); + this.subscribers.add(subscriber); + + for (const envelope of this.envelopes) { + subscriber.push(envelope); + } + + const self = this; + return { + [Symbol.asyncIterator](): AsyncIterator { + return { + async next() { + const value = await subscriber.next(); + if (!value) { + self.subscribers.delete(subscriber); + return { done: true, value: undefined as any }; + } + return { done: false, value }; + }, + async return() { + subscriber.close(); + self.subscribers.delete(subscriber); + return { done: true, value: undefined as any }; + }, + }; + }, + }; + } + + getMetricsSnapshot(): AgentMetricsSnapshot { + return JSON.parse(JSON.stringify(this.snapshot)); + } + + listObservations(opts?: ObservationListOptions): ObservationEnvelope[] { + if (!this.enabled) { + return []; + } + + return filterObservationEnvelopes(this.envelopes, opts); + } + + getRun(runId: string): ObservationRunView | undefined { + if (!this.enabled) { + return undefined; + } + + return buildObservationRunView(this.envelopes, runId); + } + + list(opts?: { kinds?: ObservationKind[]; limit?: number }): ObservationRecord[] { + return this.listObservations(opts).map((entry) => entry.observation); + } + + private applyToSnapshot(observation: ObservationRecord): void { + this.snapshot.currentRunId = observation.runId; + this.snapshot.traceId = observation.traceId; + + if (observation.kind === 'generation') { + this.applyGeneration(observation); + return; + } + + if (observation.kind === 'agent_run') { + if (observation.trigger === 'scheduler') { + this.snapshot.totals.scheduledRuns += 1; + } + return; + } + + if (observation.kind === 'tool') { + this.snapshot.totals.toolCalls += 1; + if (observation.status === 'error') { + this.snapshot.totals.toolErrors += 1; + } + if (observation.approval?.requestedAt !== undefined) { + this.snapshot.totals.approvalRequests += 1; + } + if (observation.approval?.requestedAt !== undefined && observation.approval.status === 'denied') { + this.snapshot.totals.approvalDenials += 1; + } + if (observation.approval?.waitMs !== undefined) { + this.snapshot.totals.approvalWaitMsTotal += observation.approval.waitMs; + } + return; + } + + if (observation.kind === 'subagent') { + this.snapshot.totals.subagents += 1; + return; + } + + if (observation.kind === 'compression') { + this.snapshot.totals.compressions += 1; + if (observation.status === 'error') { + this.snapshot.totals.compressionErrors += 1; + } + if ( + observation.estimatedTokensBefore !== undefined && + observation.estimatedTokensAfter !== undefined && + observation.estimatedTokensBefore > observation.estimatedTokensAfter + ) { + this.snapshot.totals.tokensSavedEstimate += + observation.estimatedTokensBefore - observation.estimatedTokensAfter; + } + } + } + + private applyGeneration(observation: GenerationObservation): void { + this.snapshot.totals.generations += 1; + if (observation.status === 'error') { + this.snapshot.totals.generationErrors += 1; + } + + const usage = observation.usage; + if (usage) { + this.snapshot.totals.inputTokens += usage.inputTokens; + this.snapshot.totals.outputTokens += usage.outputTokens; + this.snapshot.totals.totalTokens += usage.totalTokens; + this.snapshot.totals.reasoningTokens += usage.reasoningTokens ?? 0; + } + + const totalCost = observation.cost?.totalCost ?? 0; + this.snapshot.totals.totalCostUsd += totalCost; + + this.snapshot.lastGeneration = { + provider: observation.provider, + model: observation.model, + requestId: observation.requestId, + latencyMs: observation.request?.latencyMs, + timeToFirstTokenMs: observation.request?.timeToFirstTokenMs, + stopReason: observation.request?.stopReason, + retryCount: observation.request?.retryCount, + totalTokens: observation.usage?.totalTokens, + totalCostUsd: totalCost, + }; + } +} diff --git a/src/observability/ids.ts b/src/observability/ids.ts new file mode 100644 index 0000000..4e4b426 --- /dev/null +++ b/src/observability/ids.ts @@ -0,0 +1,15 @@ +function createId(prefix: string): string { + return `${prefix}-${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 10)}`; +} + +export function generateRunId(): string { + return createId('run'); +} + +export function generateTraceId(): string { + return createId('trc'); +} + +export function generateSpanId(): string { + return createId('spn'); +} diff --git a/src/observability/index.ts b/src/observability/index.ts new file mode 100644 index 0000000..c7a8a6e --- /dev/null +++ b/src/observability/index.ts @@ -0,0 +1,29 @@ +export { ObservationCollector } from './collector'; +export { generateRunId, generateSpanId, generateTraceId } from './ids'; +export * from './otel'; +export * from './persistence'; +export { createObservationReader } from './reader'; +export { CompositeObservationSink } from './sinks/composite'; +export { MemoryObservationSink, MemoryObservationStore } from './sinks/memory'; +export { NoopObservationSink } from './sinks/noop'; +export type { + AgentMetricsSnapshot, + AgentRunObservation, + BaseObservation, + CaptureMode, + CompressionObservation, + GenerationObservation, + ObservationEnvelope, + ObservationKind, + ObservationListOptions, + ObservationQueryOptions, + ObservationReader, + ObservationRecord, + ObservationRunView, + ObservationStatus, + ObservationSink, + ObservationSubscribeOptions, + ObservabilityConfig, + SubagentObservation, + ToolObservation, +} from './types'; diff --git a/src/observability/otel/attributes.ts b/src/observability/otel/attributes.ts new file mode 100644 index 0000000..6aea50a --- /dev/null +++ b/src/observability/otel/attributes.ts @@ -0,0 +1,229 @@ +import { createHash } from 'node:crypto'; + +import type { + AgentRunObservation, + CompressionObservation, + GenerationObservation, + ObservationEnvelope, + ObservationRecord, + ToolObservation, + SubagentObservation, +} from '../types'; +import type { OTelAttributeNamespace, OTelAttributeValue } from './types'; + +export function toOTelTraceId(value: string): string { + return createHash('sha256').update(`trace:${value}`).digest('hex').slice(0, 32); +} + +export function toOTelSpanId(value: string): string { + return createHash('sha256').update(`span:${value}`).digest('hex').slice(0, 16); +} + +export function toOTelAttributeValue(value: unknown): OTelAttributeValue | undefined { + if (typeof value === 'string' || typeof value === 'number' || typeof value === 'boolean') { + return value; + } + return undefined; +} + +export function buildBaseOTelAttributes( + envelope: ObservationEnvelope, + attributeNamespace: OTelAttributeNamespace +): Record { + const observation = envelope.observation; + const attributes: Record = { + 'kode.agent.id': observation.agentId, + 'kode.run.id': observation.runId, + 'kode.trace.id': observation.traceId, + 'kode.span.id': observation.spanId, + 'kode.observation.kind': observation.kind, + 'kode.observation.name': observation.name, + 'kode.observation.status': observation.status, + 'kode.envelope.seq': envelope.seq, + 'kode.envelope.timestamp': envelope.timestamp, + 'kode.duration.ms': observation.durationMs ?? Math.max(0, (observation.endTime ?? observation.startTime) - observation.startTime), + }; + + if (observation.parentSpanId) { + attributes['kode.parent_span.id'] = observation.parentSpanId; + } + + if (attributeNamespace === 'kode') { + return attributes; + } + + return { + ...attributes, + 'gen_ai.kode.agent_id': observation.agentId, + 'gen_ai.kode.run_id': observation.runId, + }; +} + +export function buildObservationSpecificAttributes( + observation: ObservationRecord, + attributeNamespace: OTelAttributeNamespace +): Record { + switch (observation.kind) { + case 'agent_run': + return buildAgentRunAttributes(observation); + case 'generation': + return buildGenerationAttributes(observation, attributeNamespace); + case 'tool': + return buildToolAttributes(observation); + case 'subagent': + return buildSubagentAttributes(observation); + case 'compression': + return buildCompressionAttributes(observation); + default: + return {}; + } +} + +function buildAgentRunAttributes(observation: AgentRunObservation): Record { + const attributes: Record = { + 'kode.run.trigger': observation.trigger, + 'kode.step': observation.step, + 'kode.message_count.before': observation.messageCountBefore, + }; + + if (observation.messageCountAfter !== undefined) { + attributes['kode.message_count.after'] = observation.messageCountAfter; + } + if (observation.metadata?.templateId && typeof observation.metadata.templateId === 'string') { + attributes['kode.template.id'] = observation.metadata.templateId; + } + if (observation.errorMessage) { + attributes['kode.error.message'] = observation.errorMessage; + } + + return attributes; +} + +function buildGenerationAttributes( + observation: GenerationObservation, + attributeNamespace: OTelAttributeNamespace +): Record { + const attributes: Record = {}; + + if (observation.provider) { + attributes['kode.generation.provider'] = observation.provider; + } + if (observation.model) { + attributes['kode.generation.model'] = observation.model; + } + if (observation.requestId) { + attributes['kode.generation.request_id'] = observation.requestId; + } + if (observation.request?.stopReason) { + attributes['kode.generation.stop_reason'] = observation.request.stopReason; + } + if (observation.request?.latencyMs !== undefined) { + attributes['kode.generation.latency_ms'] = observation.request.latencyMs; + } + if (observation.request?.timeToFirstTokenMs !== undefined) { + attributes['kode.generation.ttft_ms'] = observation.request.timeToFirstTokenMs; + } + if (observation.usage?.inputTokens !== undefined) { + attributes['kode.generation.input_tokens'] = observation.usage.inputTokens; + } + if (observation.usage?.outputTokens !== undefined) { + attributes['kode.generation.output_tokens'] = observation.usage.outputTokens; + } + if (observation.usage?.totalTokens !== undefined) { + attributes['kode.generation.total_tokens'] = observation.usage.totalTokens; + } + if (observation.cost?.totalCost !== undefined) { + attributes['kode.generation.total_cost_usd'] = observation.cost.totalCost; + } + + if (attributeNamespace !== 'kode') { + if (observation.provider) { + attributes['gen_ai.system'] = observation.provider; + } + if (observation.model) { + attributes['gen_ai.request.model'] = observation.model; + } + if (observation.usage?.inputTokens !== undefined) { + attributes['gen_ai.usage.input_tokens'] = observation.usage.inputTokens; + } + if (observation.usage?.outputTokens !== undefined) { + attributes['gen_ai.usage.output_tokens'] = observation.usage.outputTokens; + } + if (observation.usage?.totalTokens !== undefined) { + attributes['gen_ai.usage.total_tokens'] = observation.usage.totalTokens; + } + } + + if (observation.errorMessage) { + attributes['kode.error.message'] = observation.errorMessage; + } + + return attributes; +} + +function buildToolAttributes(observation: ToolObservation): Record { + const attributes: Record = { + 'kode.tool.call_id': observation.toolCallId, + 'kode.tool.name': observation.toolName, + 'kode.tool.state': observation.toolState, + 'kode.tool.approval_required': observation.approvalRequired, + }; + + if (observation.approval) { + attributes['kode.tool.approval.status'] = observation.approval.status; + if (observation.approval.waitMs !== undefined) { + attributes['kode.tool.approval.wait_ms'] = observation.approval.waitMs; + } + } + if (observation.errorMessage) { + attributes['kode.error.message'] = observation.errorMessage; + } + + return attributes; +} + +function buildSubagentAttributes(observation: SubagentObservation): Record { + const attributes: Record = { + 'kode.subagent.child_agent_id': observation.childAgentId, + 'kode.subagent.template_id': observation.templateId, + }; + + if (observation.childRunId) { + attributes['kode.subagent.child_run_id'] = observation.childRunId; + } + if (observation.delegatedBy) { + attributes['kode.subagent.delegated_by'] = observation.delegatedBy; + } + if (observation.errorMessage) { + attributes['kode.error.message'] = observation.errorMessage; + } + + return attributes; +} + +function buildCompressionAttributes(observation: CompressionObservation): Record { + const attributes: Record = { + 'kode.compression.policy': observation.policy, + 'kode.compression.reason': observation.reason, + 'kode.compression.message_count_before': observation.messageCountBefore, + 'kode.compression.summary_generated': observation.summaryGenerated, + }; + + if (observation.messageCountAfter !== undefined) { + attributes['kode.compression.message_count_after'] = observation.messageCountAfter; + } + if (observation.estimatedTokensBefore !== undefined) { + attributes['kode.compression.tokens_before'] = observation.estimatedTokensBefore; + } + if (observation.estimatedTokensAfter !== undefined) { + attributes['kode.compression.tokens_after'] = observation.estimatedTokensAfter; + } + if (observation.ratio !== undefined) { + attributes['kode.compression.ratio'] = observation.ratio; + } + if (observation.errorMessage) { + attributes['kode.error.message'] = observation.errorMessage; + } + + return attributes; +} diff --git a/src/observability/otel/exporter.ts b/src/observability/otel/exporter.ts new file mode 100644 index 0000000..bf09efd --- /dev/null +++ b/src/observability/otel/exporter.ts @@ -0,0 +1,134 @@ +import { fetch as undiciFetch } from 'undici'; + +import type { + OTLPHttpJsonExporterConfig, + OTelAttributeValue, + OTelSpanData, + OTelSpanExporter, +} from './types'; + +function toUnixNano(timestampMs: number): string { + return `${Math.max(0, Math.floor(timestampMs))}000000`; +} + +function mapSpanKind(kind: OTelSpanData['kind']): number { + switch (kind) { + case 'internal': + return 1; + case 'server': + return 2; + case 'client': + return 3; + default: + return 1; + } +} + +function mapStatusCode(status: OTelSpanData['status']): number { + switch (status) { + case 'ok': + return 1; + case 'error': + return 2; + case 'cancelled': + return 0; + default: + return 0; + } +} + +function attributeValueToOtlp(value: OTelAttributeValue): Record { + switch (typeof value) { + case 'string': + return { stringValue: value }; + case 'number': + return Number.isInteger(value) ? { intValue: String(value) } : { doubleValue: value }; + case 'boolean': + return { boolValue: value }; + default: + return { stringValue: String(value) }; + } +} + +export function buildOTLPTraceExportBody(spans: OTelSpanData[], serviceName = 'kode-observability'): Record { + return { + resourceSpans: [ + { + resource: { + attributes: [ + { + key: 'service.name', + value: { stringValue: serviceName }, + }, + ], + }, + scopeSpans: [ + { + scope: { + name: '@shareai-lab/kode-sdk/observability', + }, + spans: spans.map((span) => ({ + traceId: span.traceId, + spanId: span.spanId, + parentSpanId: span.parentSpanId, + name: span.name, + kind: mapSpanKind(span.kind), + startTimeUnixNano: toUnixNano(span.startTime), + endTimeUnixNano: toUnixNano(span.endTime ?? span.startTime), + attributes: Object.entries(span.attributes).map(([key, value]) => ({ + key, + value: attributeValueToOtlp(value), + })), + events: (span.events || []).map((event) => ({ + name: event.name, + timeUnixNano: toUnixNano(event.timestamp), + attributes: Object.entries(event.attributes || {}).map(([key, value]) => ({ + key, + value: attributeValueToOtlp(value), + })), + })), + status: { + code: mapStatusCode(span.status), + }, + })), + }, + ], + }, + ], + }; +} + +export class OTLPHttpJsonExporter implements OTelSpanExporter { + private readonly fetchImpl; + + constructor(private readonly config: OTLPHttpJsonExporterConfig) { + this.fetchImpl = config.fetch ?? undiciFetch; + } + + async export(spans: OTelSpanData[]): Promise { + if (spans.length === 0) { + return; + } + + const response = await this.fetchImpl(this.config.endpoint, { + method: 'POST', + headers: { + 'content-type': 'application/json', + ...(this.config.headers || {}), + }, + body: JSON.stringify(buildOTLPTraceExportBody(spans, this.config.serviceName)), + }); + + if (!response.ok) { + throw new Error(`OTLP export failed with status ${response.status}: ${await response.text()}`); + } + } + + async forceFlush(): Promise { + return; + } + + async shutdown(): Promise { + return; + } +} diff --git a/src/observability/otel/index.ts b/src/observability/otel/index.ts new file mode 100644 index 0000000..0379bc6 --- /dev/null +++ b/src/observability/otel/index.ts @@ -0,0 +1,23 @@ +export { buildBaseOTelAttributes, buildObservationSpecificAttributes, toOTelSpanId, toOTelTraceId } from './attributes'; +export { buildOTLPTraceExportBody, OTLPHttpJsonExporter } from './exporter'; +export { getOTelObservationMapping } from './mapping'; +export { applyOTelPolicies, maskOTelSpan, shouldExportOTelSpan, shouldSampleOTelTrace } from './policy'; +export { createOTelExporter, OTelObservationSink } from './sink'; +export { createOTelSpanTranslator, translateObservationToOTelSpan } from './translator'; +export type { + OTelAttributeNamespace, + OTelAttributeValue, + OTelBridgeConfig, + OTelExportMode, + OTelFetchLike, + OTelFilteringPolicy, + OTelHttpResponseLike, + OTelMaskingPolicy, + OTelObservationBridge, + OTelSamplingPolicy, + OTelSpanData, + OTelSpanEvent, + OTelSpanExporter, + OTelSpanKind, + OTLPHttpJsonExporterConfig, +} from './types'; diff --git a/src/observability/otel/mapping.ts b/src/observability/otel/mapping.ts new file mode 100644 index 0000000..8fd6c8d --- /dev/null +++ b/src/observability/otel/mapping.ts @@ -0,0 +1,19 @@ +import type { ObservationKind } from '../types'; +import type { OTelSpanKind } from './types'; + +export interface OTelObservationMapping { + spanName: string; + kind: OTelSpanKind; +} + +const OBSERVATION_MAPPINGS: Record = { + agent_run: { spanName: 'agent.run', kind: 'internal' }, + generation: { spanName: 'llm.generation', kind: 'client' }, + tool: { spanName: 'tool.call', kind: 'client' }, + subagent: { spanName: 'agent.delegate', kind: 'internal' }, + compression: { spanName: 'context.compression', kind: 'internal' }, +}; + +export function getOTelObservationMapping(kind: ObservationKind): OTelObservationMapping { + return OBSERVATION_MAPPINGS[kind]; +} diff --git a/src/observability/otel/policy.ts b/src/observability/otel/policy.ts new file mode 100644 index 0000000..d051b5f --- /dev/null +++ b/src/observability/otel/policy.ts @@ -0,0 +1,100 @@ +import type { ObservationEnvelope } from '../types'; +import type { + OTelAttributeValue, + OTelBridgeConfig, + OTelSamplingPolicy, + OTelSpanData, +} from './types'; +import { createHash } from 'node:crypto'; + +const DEFAULT_REDACT_PATTERNS = [ + /sk-[a-zA-Z0-9_-]+/g, + /Bearer\s+[A-Za-z0-9._-]+/g, + /api[_-]?key[=:]\s*[^,\s]+/gi, +]; + +function maskString(value: string, patterns: RegExp[]): string { + return patterns.reduce((current, pattern) => current.replace(pattern, (match) => { + if (match.startsWith('Bearer ')) return 'Bearer ***'; + if (match.startsWith('sk-')) return 'sk-***'; + const [prefix] = match.split(/[:=]/, 1); + return `${prefix}=***`; + }), value); +} + +export function shouldExportOTelSpan(envelope: ObservationEnvelope, span: OTelSpanData, config?: OTelBridgeConfig): boolean { + const filtering = config?.filtering; + if (!filtering) { + return true; + } + if (filtering.kinds && !filtering.kinds.includes(envelope.observation.kind)) { + return false; + } + if (filtering.statuses && !filtering.statuses.includes(envelope.observation.status)) { + return false; + } + if (filtering.predicate && !filtering.predicate({ envelope, span })) { + return false; + } + return true; +} + +export function shouldSampleOTelTrace(traceId: string, sampling?: OTelSamplingPolicy): boolean { + const resolved = sampling ?? { strategy: 'always_on' as const }; + if (resolved.strategy === 'always_on') { + return true; + } + if (resolved.strategy === 'always_off') { + return false; + } + + const ratio = Math.max(0, Math.min(1, resolved.ratio)); + if (ratio === 0) return false; + if (ratio === 1) return true; + + const bucket = createHash('sha256').update(traceId).digest().readUInt32BE(0) / 0xffffffff; + return bucket < ratio; +} + +export function maskOTelSpan(span: OTelSpanData, config?: OTelBridgeConfig): OTelSpanData { + const masking = config?.masking; + if (masking?.enabled === false) { + return span; + } + + const redactPatterns = masking?.redactPatterns ?? DEFAULT_REDACT_PATTERNS; + + const maskValue = (key: string, value: OTelAttributeValue): OTelAttributeValue => { + const custom = masking?.mask?.({ key, value }); + const masked = custom === undefined ? value : custom; + if (typeof masked === 'string') { + return maskString(masked, redactPatterns); + } + return masked; + }; + + return { + ...span, + attributes: Object.fromEntries( + Object.entries(span.attributes).map(([key, value]) => [key, maskValue(key, value)]) + ), + events: span.events?.map((event) => ({ + ...event, + attributes: event.attributes + ? Object.fromEntries( + Object.entries(event.attributes).map(([key, value]) => [key, maskValue(key, value)]) + ) + : undefined, + })), + }; +} + +export function applyOTelPolicies(envelope: ObservationEnvelope, span: OTelSpanData, config?: OTelBridgeConfig): OTelSpanData | undefined { + if (!shouldExportOTelSpan(envelope, span, config)) { + return undefined; + } + if (!shouldSampleOTelTrace(envelope.observation.traceId, config?.sampling)) { + return undefined; + } + return maskOTelSpan(span, config); +} diff --git a/src/observability/otel/sink.ts b/src/observability/otel/sink.ts new file mode 100644 index 0000000..3fca1bf --- /dev/null +++ b/src/observability/otel/sink.ts @@ -0,0 +1,112 @@ +import { logger } from '../../utils/logger'; +import type { ObservationEnvelope, ObservationSink } from '../types'; +import { OTLPHttpJsonExporter } from './exporter'; +import { applyOTelPolicies } from './policy'; +import { createOTelSpanTranslator } from './translator'; +import type { OTelBridgeConfig, OTelSpanData, OTelSpanExporter } from './types'; + +export class OTelObservationSink implements ObservationSink { + private readonly translator; + private readonly exporter?: OTelSpanExporter; + private readonly exportMode; + private readonly batchSize; + private readonly flushIntervalMs; + private readonly queue: OTelSpanData[] = []; + private flushTimer?: NodeJS.Timeout; + private pendingFlush: Promise = Promise.resolve(); + + constructor(private readonly config: OTelBridgeConfig = {}) { + this.translator = createOTelSpanTranslator(config); + this.exporter = createOTelExporter(config); + this.exportMode = config.exportMode ?? 'immediate'; + this.batchSize = Math.max(1, config.batchSize ?? 50); + this.flushIntervalMs = Math.max(10, config.flushIntervalMs ?? 1000); + } + + async onObservation(envelope: ObservationEnvelope): Promise { + if (this.config.enabled === false || !this.exporter) { + return; + } + + try { + const translated = this.translator(envelope); + const span = applyOTelPolicies(envelope, translated, this.config); + if (!span) { + return; + } + + if (this.exportMode === 'immediate') { + await this.exporter.export([span]); + return; + } + + this.queue.push(span); + if (this.queue.length >= this.batchSize) { + await this.forceFlush(); + return; + } + + this.ensureFlushTimer(); + } catch (error) { + logger.warn('[Observability] OTel bridge failed:', error); + } + } + + async forceFlush(): Promise { + if (!this.exporter) { + return; + } + + const spans = this.queue.splice(0, this.queue.length); + if (this.flushTimer) { + clearTimeout(this.flushTimer); + this.flushTimer = undefined; + } + + this.pendingFlush = this.pendingFlush.then(async () => { + if (spans.length > 0) { + await this.exporter!.export(spans); + } + await this.exporter!.forceFlush?.(); + }).catch((error) => { + logger.warn('[Observability] OTel bridge flush failed:', error); + }); + + await this.pendingFlush; + } + + async shutdown(): Promise { + if (this.flushTimer) { + clearTimeout(this.flushTimer); + this.flushTimer = undefined; + } + await this.forceFlush(); + await this.exporter?.shutdown?.(); + } + + private ensureFlushTimer(): void { + if (this.flushTimer) { + return; + } + + this.flushTimer = setTimeout(() => { + this.flushTimer = undefined; + void this.forceFlush(); + }, this.flushIntervalMs); + } +} + +export function createOTelExporter(config: OTelBridgeConfig): OTelSpanExporter | undefined { + if (config.exporter) { + return config.exporter; + } + if (config.endpoint) { + return new OTLPHttpJsonExporter({ + endpoint: config.endpoint, + headers: config.headers, + serviceName: config.serviceName, + fetch: config.fetch, + }); + } + return undefined; +} diff --git a/src/observability/otel/translator.ts b/src/observability/otel/translator.ts new file mode 100644 index 0000000..36f065b --- /dev/null +++ b/src/observability/otel/translator.ts @@ -0,0 +1,60 @@ +import type { ObservationEnvelope } from '../types'; +import { buildBaseOTelAttributes, buildObservationSpecificAttributes, toOTelSpanId, toOTelTraceId } from './attributes'; +import { getOTelObservationMapping } from './mapping'; +import type { OTelAttributeNamespace, OTelBridgeConfig, OTelSpanData, OTelSpanEvent } from './types'; + +function buildSpanEvents(envelope: ObservationEnvelope): OTelSpanEvent[] | undefined { + const observation = envelope.observation; + const events: OTelSpanEvent[] = [ + { + name: 'kode.observation', + timestamp: envelope.timestamp, + attributes: { + 'kode.observation.kind': observation.kind, + 'kode.observation.status': observation.status, + }, + }, + ]; + + if ('errorMessage' in observation && typeof observation.errorMessage === 'string' && observation.errorMessage) { + events.push({ + name: 'exception', + timestamp: observation.endTime ?? envelope.timestamp, + attributes: { + 'exception.message': observation.errorMessage, + }, + }); + } + + return events; +} + +export function translateObservationToOTelSpan( + envelope: ObservationEnvelope, + opts?: { attributeNamespace?: OTelAttributeNamespace } +): OTelSpanData { + const observation = envelope.observation; + const mapping = getOTelObservationMapping(observation.kind); + const attributeNamespace = opts?.attributeNamespace ?? 'dual'; + + return { + traceId: toOTelTraceId(observation.traceId), + spanId: toOTelSpanId(observation.spanId), + parentSpanId: observation.parentSpanId ? toOTelSpanId(observation.parentSpanId) : undefined, + name: mapping.spanName, + kind: mapping.kind, + startTime: observation.startTime, + endTime: observation.endTime, + status: observation.status, + attributes: { + ...buildBaseOTelAttributes(envelope, attributeNamespace), + ...buildObservationSpecificAttributes(observation, attributeNamespace), + }, + events: buildSpanEvents(envelope), + }; +} + +export function createOTelSpanTranslator(config?: OTelBridgeConfig) { + return (envelope: ObservationEnvelope): OTelSpanData => + translateObservationToOTelSpan(envelope, { attributeNamespace: config?.attributeNamespace }); +} diff --git a/src/observability/otel/types.ts b/src/observability/otel/types.ts new file mode 100644 index 0000000..95e0b56 --- /dev/null +++ b/src/observability/otel/types.ts @@ -0,0 +1,88 @@ +import type { ObservationEnvelope, ObservationKind, ObservationSink, ObservationStatus } from '../types'; + +export type OTelAttributeValue = string | number | boolean; +export type OTelSpanKind = 'internal' | 'client' | 'server'; +export type OTelExportMode = 'immediate' | 'batched'; +export type OTelAttributeNamespace = 'kode' | 'gen_ai' | 'dual'; + +export interface OTelSpanEvent { + name: string; + timestamp: number; + attributes?: Record; +} + +export interface OTelSpanData { + traceId: string; + spanId: string; + parentSpanId?: string; + name: string; + kind: OTelSpanKind; + startTime: number; + endTime?: number; + status: ObservationStatus; + attributes: Record; + events?: OTelSpanEvent[]; +} + +export interface OTelMaskingPolicy { + enabled?: boolean; + mask?: (params: { key: string; value: OTelAttributeValue }) => OTelAttributeValue; + redactPatterns?: RegExp[]; +} + +export interface OTelFilteringPolicy { + kinds?: ObservationKind[]; + statuses?: ObservationStatus[]; + predicate?: (params: { envelope: ObservationEnvelope; span: OTelSpanData }) => boolean; +} + +export type OTelSamplingPolicy = + | { strategy: 'always_on' } + | { strategy: 'always_off' } + | { strategy: 'trace_ratio'; ratio: number }; + +export interface OTelSpanExporter { + export(spans: OTelSpanData[]): void | Promise; + forceFlush?(): void | Promise; + shutdown?(): void | Promise; +} + +export interface OTelObservationBridge extends ObservationSink { + forceFlush?(): void | Promise; + shutdown?(): void | Promise; +} + +export interface OTelHttpResponseLike { + ok: boolean; + status: number; + text(): Promise; +} + +export type OTelFetchLike = (url: string, init: { + method: string; + headers?: Record; + body?: string; +}) => Promise; + +export interface OTLPHttpJsonExporterConfig { + endpoint: string; + headers?: Record; + serviceName?: string; + fetch?: OTelFetchLike; +} + +export interface OTelBridgeConfig { + enabled?: boolean; + exporter?: OTelSpanExporter; + endpoint?: string; + headers?: Record; + exportMode?: OTelExportMode; + batchSize?: number; + flushIntervalMs?: number; + masking?: OTelMaskingPolicy; + filtering?: OTelFilteringPolicy; + sampling?: OTelSamplingPolicy; + attributeNamespace?: OTelAttributeNamespace; + serviceName?: string; + fetch?: OTelFetchLike; +} diff --git a/src/observability/persistence/backends/jsonstore.ts b/src/observability/persistence/backends/jsonstore.ts new file mode 100644 index 0000000..3d6d19f --- /dev/null +++ b/src/observability/persistence/backends/jsonstore.ts @@ -0,0 +1,138 @@ +import { createHash } from 'node:crypto'; +import fs from 'node:fs'; +import path from 'node:path'; + +import type { ObservationEnvelope } from '../../types'; +import { + buildPersistedObservationRunView, + filterPersistedObservationEnvelopes, +} from '../reader'; +import { applyObservationRetention } from '../retention'; +import type { + ObservationPruneResult, + ObservationQueryBackend, + ObservationRetentionPolicy, + PersistedObservationListOptions, +} from '../types'; + +export class JSONStoreObservationBackend implements ObservationQueryBackend { + constructor(private readonly baseDir: string) {} + + async append(envelope: ObservationEnvelope): Promise { + const filePath = this.getObservationFilePath(envelope.observation.agentId); + await fs.promises.mkdir(path.dirname(filePath), { recursive: true }); + await fs.promises.appendFile(filePath, `${JSON.stringify(envelope)}\n`, 'utf-8'); + } + + async list(opts?: PersistedObservationListOptions): Promise { + const envelopes = await this.loadEnvelopes(opts); + return filterPersistedObservationEnvelopes(envelopes, opts); + } + + async getRun(runId: string) { + const envelopes = await this.loadEnvelopes({ runId }); + return buildPersistedObservationRunView(envelopes, runId); + } + + async prune(policy?: ObservationRetentionPolicy): Promise { + if (!policy) { + const retained = (await this.loadEnvelopes()).length; + return { deleted: 0, retained }; + } + + const files = await this.listObservationFiles(); + let deleted = 0; + let retained = 0; + + for (const filePath of files) { + const envelopes = await this.readObservationFile(filePath); + const next = applyObservationRetention(envelopes, policy); + deleted += next.result.deleted; + retained += next.result.retained; + + if (next.result.deleted === 0) { + continue; + } + + if (next.envelopes.length === 0) { + await fs.promises.rm(filePath, { force: true }); + continue; + } + + await this.writeObservationFile(filePath, next.envelopes); + } + + return { deleted, retained }; + } + + private async loadEnvelopes(opts?: PersistedObservationListOptions): Promise { + const filePaths = await this.resolveObservationFiles(opts); + const loaded = await Promise.all(filePaths.map((filePath) => this.readObservationFile(filePath))); + return loaded.flat(); + } + + private async resolveObservationFiles(opts?: PersistedObservationListOptions): Promise { + const agentIds = new Set(); + if (opts?.agentId) { + agentIds.add(opts.agentId); + } + if (opts?.agentIds) { + for (const agentId of opts.agentIds) { + agentIds.add(agentId); + } + } + + if (agentIds.size > 0) { + return [...agentIds].map((agentId) => this.getObservationFilePath(agentId)); + } + + return this.listObservationFiles(); + } + + private getObservationFilePath(agentId: string): string { + return path.join(this.baseDir, '_observations', `${this.hashAgentId(agentId)}.jsonl`); + } + + private hashAgentId(agentId: string): string { + return createHash('sha256').update(agentId).digest('hex'); + } + + private async listObservationFiles(): Promise { + const dir = path.join(this.baseDir, '_observations'); + try { + const entries = await fs.promises.readdir(dir, { withFileTypes: true }); + return entries + .filter((entry) => entry.isFile() && entry.name.endsWith('.jsonl')) + .map((entry) => path.join(dir, entry.name)); + } catch (error: any) { + if (error?.code === 'ENOENT') { + return []; + } + throw error; + } + } + + private async readObservationFile(filePath: string): Promise { + try { + const raw = await fs.promises.readFile(filePath, 'utf-8'); + return raw + .split(/\r?\n/) + .map((line) => line.trim()) + .filter(Boolean) + .map((line) => JSON.parse(line) as ObservationEnvelope); + } catch (error: any) { + if (error?.code === 'ENOENT') { + return []; + } + throw error; + } + } + + private async writeObservationFile(filePath: string, envelopes: ObservationEnvelope[]): Promise { + await fs.promises.mkdir(path.dirname(filePath), { recursive: true }); + const tempPath = `${filePath}.tmp`; + const payload = `${envelopes.map((envelope) => JSON.stringify(envelope)).join('\n')}\n`; + await fs.promises.writeFile(tempPath, payload, 'utf-8'); + await fs.promises.rename(tempPath, filePath); + } +} diff --git a/src/observability/persistence/backends/postgres.ts b/src/observability/persistence/backends/postgres.ts new file mode 100644 index 0000000..6fe9bd8 --- /dev/null +++ b/src/observability/persistence/backends/postgres.ts @@ -0,0 +1,219 @@ +import type { Pool } from 'pg'; + +import type { PostgresStore } from '../../../infra/db/postgres/postgres-store'; +import type { ObservationEnvelope } from '../../types'; +import { + buildPersistedObservationRunView, + filterPersistedObservationEnvelopes, +} from '../reader'; +import type { + ObservationPruneResult, + ObservationQueryBackend, + ObservationRetentionPolicy, + PersistedObservationListOptions, +} from '../types'; + +type PostgresObservationRow = { + id: number; + agent_id: string; + seq: number; + timestamp: number; + envelope: ObservationEnvelope; +}; + +export class PostgresStoreObservationBackend implements ObservationQueryBackend { + private readonly pool: Pool; + private readonly schemaReady: Promise; + + constructor(store: PostgresStore) { + const internal = store as any; + this.pool = internal.pool as Pool; + const initPromise = internal.initPromise as Promise; + this.schemaReady = initPromise.then(() => this.initialize()); + } + + async append(envelope: ObservationEnvelope): Promise { + await this.ensureReady(); + + const observation = envelope.observation; + const templateId = + observation.metadata?.templateId && typeof observation.metadata.templateId === 'string' + ? observation.metadata.templateId + : null; + + await this.pool.query( + `INSERT INTO observations ( + agent_id, run_id, trace_id, parent_span_id, kind, status, seq, timestamp, template_id, envelope + ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10::jsonb)`, + [ + observation.agentId, + observation.runId, + observation.traceId, + observation.parentSpanId ?? null, + observation.kind, + observation.status, + envelope.seq, + envelope.timestamp, + templateId, + JSON.stringify(envelope), + ] + ); + } + + async list(opts?: PersistedObservationListOptions): Promise { + await this.ensureReady(); + const rows = await this.queryRows(opts); + const envelopes = rows.map((row) => row.envelope); + return filterPersistedObservationEnvelopes(envelopes, opts); + } + + async getRun(runId: string) { + await this.ensureReady(); + const rows = await this.queryRows({ runId }); + const envelopes = rows.map((row) => row.envelope); + return buildPersistedObservationRunView(envelopes, runId); + } + + async prune(policy?: ObservationRetentionPolicy): Promise { + await this.ensureReady(); + + if (!policy) { + const retained = Number((await this.pool.query('SELECT COUNT(*)::int AS count FROM observations')).rows[0].count); + return { deleted: 0, retained }; + } + + let deleted = 0; + + if (policy.maxAgeMs !== undefined) { + const cutoff = Date.now() - Math.max(0, policy.maxAgeMs); + const result = await this.pool.query('DELETE FROM observations WHERE timestamp < $1', [cutoff]); + deleted += result.rowCount ?? 0; + } + + if (policy.maxEntriesPerAgent !== undefined) { + const maxEntries = Math.max(1, Math.floor(policy.maxEntriesPerAgent)); + const rows = await this.pool.query<{ + id: number; + agent_id: string; + }>( + `SELECT id, agent_id + FROM observations + ORDER BY agent_id ASC, timestamp DESC, seq DESC, id DESC` + ); + + const counts = new Map(); + const idsToDelete: number[] = []; + + for (const row of rows.rows) { + const next = (counts.get(row.agent_id) ?? 0) + 1; + counts.set(row.agent_id, next); + if (next > maxEntries) { + idsToDelete.push(row.id); + } + } + + if (idsToDelete.length > 0) { + const result = await this.pool.query('DELETE FROM observations WHERE id = ANY($1::bigint[])', [idsToDelete]); + deleted += result.rowCount ?? 0; + } + } + + const retained = Number((await this.pool.query('SELECT COUNT(*)::int AS count FROM observations')).rows[0].count); + return { deleted, retained }; + } + + private async ensureReady(): Promise { + await this.schemaReady; + } + + private async initialize(): Promise { + await this.pool.query(` + CREATE TABLE IF NOT EXISTS observations ( + id BIGSERIAL PRIMARY KEY, + agent_id TEXT NOT NULL, + run_id TEXT NOT NULL, + trace_id TEXT NOT NULL, + parent_span_id TEXT, + kind TEXT NOT NULL, + status TEXT NOT NULL, + seq INTEGER NOT NULL, + timestamp BIGINT NOT NULL, + template_id TEXT, + envelope JSONB NOT NULL + ); + + CREATE INDEX IF NOT EXISTS idx_observations_agent_id ON observations(agent_id); + CREATE INDEX IF NOT EXISTS idx_observations_run_id ON observations(run_id); + CREATE INDEX IF NOT EXISTS idx_observations_trace_id ON observations(trace_id); + CREATE INDEX IF NOT EXISTS idx_observations_kind ON observations(kind); + CREATE INDEX IF NOT EXISTS idx_observations_status ON observations(status); + CREATE INDEX IF NOT EXISTS idx_observations_template_id ON observations(template_id); + CREATE INDEX IF NOT EXISTS idx_observations_timestamp ON observations(timestamp DESC); + `); + } + + private async queryRows(opts?: PersistedObservationListOptions): Promise { + const clauses: string[] = []; + const params: unknown[] = []; + let index = 1; + + const agentIds = this.resolveAgentIds(opts); + if (agentIds.length > 0) { + clauses.push(`agent_id = ANY($${index++}::text[])`); + params.push(agentIds); + } + if (opts?.runId) { + clauses.push(`run_id = $${index++}`); + params.push(opts.runId); + } + if (opts?.traceId) { + clauses.push(`trace_id = $${index++}`); + params.push(opts.traceId); + } + if (opts?.parentSpanId) { + clauses.push(`parent_span_id = $${index++}`); + params.push(opts.parentSpanId); + } + if (opts?.kinds?.length) { + clauses.push(`kind = ANY($${index++}::text[])`); + params.push(opts.kinds); + } + if (opts?.statuses?.length) { + clauses.push(`status = ANY($${index++}::text[])`); + params.push(opts.statuses); + } + if (opts?.templateIds?.length) { + clauses.push(`template_id = ANY($${index++}::text[])`); + params.push(opts.templateIds); + } + if (opts?.fromTimestamp !== undefined) { + clauses.push(`timestamp >= $${index++}`); + params.push(opts.fromTimestamp); + } + if (opts?.toTimestamp !== undefined) { + clauses.push(`timestamp <= $${index++}`); + params.push(opts.toTimestamp); + } + + const where = clauses.length > 0 ? `WHERE ${clauses.join(' AND ')}` : ''; + const result = await this.pool.query( + `SELECT id, agent_id, seq, timestamp, envelope + FROM observations + ${where} + ORDER BY timestamp ASC, agent_id ASC, seq ASC, id ASC`, + params + ); + return result.rows; + } + + private resolveAgentIds(opts?: PersistedObservationListOptions): string[] { + const ids = new Set(); + if (opts?.agentId) { + ids.add(opts.agentId); + } + for (const agentId of opts?.agentIds ?? []) { + ids.add(agentId); + } + return [...ids]; + } +} diff --git a/src/observability/persistence/backends/sqlite.ts b/src/observability/persistence/backends/sqlite.ts new file mode 100644 index 0000000..2323b45 --- /dev/null +++ b/src/observability/persistence/backends/sqlite.ts @@ -0,0 +1,206 @@ +import type Database from 'better-sqlite3'; + +import type { SqliteStore } from '../../../infra/db/sqlite/sqlite-store'; +import type { ObservationEnvelope } from '../../types'; +import { + buildPersistedObservationRunView, + filterPersistedObservationEnvelopes, +} from '../reader'; +import type { + ObservationPruneResult, + ObservationQueryBackend, + ObservationRetentionPolicy, + PersistedObservationListOptions, +} from '../types'; + +type SqliteObservationRow = { + id: number; + agent_id: string; + seq: number; + timestamp: number; + envelope: string; +}; + +export class SqliteStoreObservationBackend implements ObservationQueryBackend { + private readonly db: Database.Database; + + constructor(store: SqliteStore) { + this.db = (store as any).db as Database.Database; + this.initialize(); + } + + async append(envelope: ObservationEnvelope): Promise { + const observation = envelope.observation; + const templateId = + observation.metadata?.templateId && typeof observation.metadata.templateId === 'string' + ? observation.metadata.templateId + : null; + + this.db + .prepare( + `INSERT INTO observations ( + agent_id, run_id, trace_id, parent_span_id, kind, status, seq, timestamp, template_id, envelope + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` + ) + .run( + observation.agentId, + observation.runId, + observation.traceId, + observation.parentSpanId ?? null, + observation.kind, + observation.status, + envelope.seq, + envelope.timestamp, + templateId, + JSON.stringify(envelope) + ); + } + + async list(opts?: PersistedObservationListOptions): Promise { + const rows = this.queryRows(opts); + const envelopes = rows.map((row) => JSON.parse(row.envelope) as ObservationEnvelope); + return filterPersistedObservationEnvelopes(envelopes, opts); + } + + async getRun(runId: string) { + const rows = this.queryRows({ runId }); + const envelopes = rows.map((row) => JSON.parse(row.envelope) as ObservationEnvelope); + return buildPersistedObservationRunView(envelopes, runId); + } + + async prune(policy?: ObservationRetentionPolicy): Promise { + if (!policy) { + const retained = (this.db.prepare('SELECT COUNT(*) as count FROM observations').get() as { count: number }).count; + return { deleted: 0, retained }; + } + + let deleted = 0; + + if (policy.maxAgeMs !== undefined) { + const cutoff = Date.now() - Math.max(0, policy.maxAgeMs); + const result = this.db.prepare('DELETE FROM observations WHERE timestamp < ?').run(cutoff); + deleted += result.changes; + } + + if (policy.maxEntriesPerAgent !== undefined) { + const maxEntries = Math.max(1, Math.floor(policy.maxEntriesPerAgent)); + const rows = this.db + .prepare( + `SELECT id, agent_id, timestamp, seq + FROM observations + ORDER BY agent_id ASC, timestamp DESC, seq DESC, id DESC` + ) + .all() as Array<{ id: number; agent_id: string; timestamp: number; seq: number }>; + + const counts = new Map(); + const idsToDelete: number[] = []; + + for (const row of rows) { + const next = (counts.get(row.agent_id) ?? 0) + 1; + counts.set(row.agent_id, next); + if (next > maxEntries) { + idsToDelete.push(row.id); + } + } + + if (idsToDelete.length > 0) { + const placeholders = idsToDelete.map(() => '?').join(', '); + const result = this.db.prepare(`DELETE FROM observations WHERE id IN (${placeholders})`).run(...idsToDelete); + deleted += result.changes; + } + } + + const retained = (this.db.prepare('SELECT COUNT(*) as count FROM observations').get() as { count: number }).count; + return { deleted, retained }; + } + + private initialize(): void { + this.db.exec(` + CREATE TABLE IF NOT EXISTS observations ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + agent_id TEXT NOT NULL, + run_id TEXT NOT NULL, + trace_id TEXT NOT NULL, + parent_span_id TEXT, + kind TEXT NOT NULL, + status TEXT NOT NULL, + seq INTEGER NOT NULL, + timestamp INTEGER NOT NULL, + template_id TEXT, + envelope TEXT NOT NULL + ); + + CREATE INDEX IF NOT EXISTS idx_observations_agent_id ON observations(agent_id); + CREATE INDEX IF NOT EXISTS idx_observations_run_id ON observations(run_id); + CREATE INDEX IF NOT EXISTS idx_observations_trace_id ON observations(trace_id); + CREATE INDEX IF NOT EXISTS idx_observations_kind ON observations(kind); + CREATE INDEX IF NOT EXISTS idx_observations_status ON observations(status); + CREATE INDEX IF NOT EXISTS idx_observations_template_id ON observations(template_id); + CREATE INDEX IF NOT EXISTS idx_observations_timestamp ON observations(timestamp); + `); + } + + private queryRows(opts?: PersistedObservationListOptions): SqliteObservationRow[] { + const clauses: string[] = []; + const params: unknown[] = []; + + const agentIds = this.resolveAgentIds(opts); + if (agentIds.length > 0) { + clauses.push(`agent_id IN (${agentIds.map(() => '?').join(', ')})`); + params.push(...agentIds); + } + if (opts?.runId) { + clauses.push('run_id = ?'); + params.push(opts.runId); + } + if (opts?.traceId) { + clauses.push('trace_id = ?'); + params.push(opts.traceId); + } + if (opts?.parentSpanId) { + clauses.push('parent_span_id = ?'); + params.push(opts.parentSpanId); + } + if (opts?.kinds?.length) { + clauses.push(`kind IN (${opts.kinds.map(() => '?').join(', ')})`); + params.push(...opts.kinds); + } + if (opts?.statuses?.length) { + clauses.push(`status IN (${opts.statuses.map(() => '?').join(', ')})`); + params.push(...opts.statuses); + } + if (opts?.templateIds?.length) { + clauses.push(`template_id IN (${opts.templateIds.map(() => '?').join(', ')})`); + params.push(...opts.templateIds); + } + if (opts?.fromTimestamp !== undefined) { + clauses.push('timestamp >= ?'); + params.push(opts.fromTimestamp); + } + if (opts?.toTimestamp !== undefined) { + clauses.push('timestamp <= ?'); + params.push(opts.toTimestamp); + } + + const where = clauses.length > 0 ? `WHERE ${clauses.join(' AND ')}` : ''; + return this.db + .prepare( + `SELECT id, agent_id, seq, timestamp, envelope + FROM observations + ${where} + ORDER BY timestamp ASC, agent_id ASC, seq ASC, id ASC` + ) + .all(...params) as SqliteObservationRow[]; + } + + private resolveAgentIds(opts?: PersistedObservationListOptions): string[] { + const ids = new Set(); + if (opts?.agentId) { + ids.add(opts.agentId); + } + for (const agentId of opts?.agentIds ?? []) { + ids.add(agentId); + } + return [...ids]; + } +} diff --git a/src/observability/persistence/index.ts b/src/observability/persistence/index.ts new file mode 100644 index 0000000..1ecc722 --- /dev/null +++ b/src/observability/persistence/index.ts @@ -0,0 +1,14 @@ +export { JSONStoreObservationBackend } from './backends/jsonstore'; +export { PostgresStoreObservationBackend } from './backends/postgres'; +export { SqliteStoreObservationBackend } from './backends/sqlite'; +export { createStoreBackedObservationReader, filterPersistedObservationEnvelopes } from './reader'; +export { applyObservationRetention } from './retention'; +export { PersistedObservationSink } from './sink'; +export type { + ObservationPersistenceConfig, + ObservationPruneResult, + ObservationQueryBackend, + ObservationRetentionPolicy, + PersistedObservationListOptions, + PersistedObservationReader, +} from './types'; diff --git a/src/observability/persistence/reader.ts b/src/observability/persistence/reader.ts new file mode 100644 index 0000000..81519d3 --- /dev/null +++ b/src/observability/persistence/reader.ts @@ -0,0 +1,81 @@ +import { + buildObservationRunView, + filterObservationEnvelopes, + matchesObservationEnvelope, +} from '../query'; +import type { ObservationEnvelope } from '../types'; +import type { + ObservationQueryBackend, + PersistedObservationListOptions, + PersistedObservationReader, +} from './types'; + +function sortObservationEnvelopes(envelopes: ObservationEnvelope[]): ObservationEnvelope[] { + return [...envelopes].sort((left, right) => { + if (left.timestamp !== right.timestamp) { + return left.timestamp - right.timestamp; + } + if (left.observation.agentId !== right.observation.agentId) { + return left.observation.agentId.localeCompare(right.observation.agentId); + } + return left.seq - right.seq; + }); +} + +function matchesPersistedObservationEnvelope( + envelope: ObservationEnvelope, + opts?: PersistedObservationListOptions +): boolean { + if (!matchesObservationEnvelope(envelope, opts)) { + return false; + } + + if (opts?.agentIds && !opts.agentIds.includes(envelope.observation.agentId)) { + return false; + } + + if (opts?.templateIds) { + const templateId = + envelope.observation.metadata?.templateId && + typeof envelope.observation.metadata.templateId === 'string' + ? envelope.observation.metadata.templateId + : undefined; + if (!templateId || !opts.templateIds.includes(templateId)) { + return false; + } + } + + if (opts?.fromTimestamp !== undefined && envelope.timestamp < opts.fromTimestamp) { + return false; + } + + if (opts?.toTimestamp !== undefined && envelope.timestamp > opts.toTimestamp) { + return false; + } + + return true; +} + +export function filterPersistedObservationEnvelopes( + envelopes: ObservationEnvelope[], + opts?: PersistedObservationListOptions +): ObservationEnvelope[] { + const filtered = envelopes.filter((envelope) => matchesPersistedObservationEnvelope(envelope, opts)); + return filterObservationEnvelopes(sortObservationEnvelopes(filtered), opts); +} + +export function buildPersistedObservationRunView( + envelopes: ObservationEnvelope[], + runId: string +) { + return buildObservationRunView(sortObservationEnvelopes(envelopes), runId); +} + +export function createStoreBackedObservationReader( + backend: ObservationQueryBackend +): PersistedObservationReader { + return { + listObservations: (opts) => backend.list(opts), + getRun: (runId) => backend.getRun(runId), + }; +} diff --git a/src/observability/persistence/retention.ts b/src/observability/persistence/retention.ts new file mode 100644 index 0000000..2633486 --- /dev/null +++ b/src/observability/persistence/retention.ts @@ -0,0 +1,52 @@ +import type { ObservationEnvelope } from '../types'; +import type { ObservationPruneResult, ObservationRetentionPolicy } from './types'; + +export function applyObservationRetention( + envelopes: ObservationEnvelope[], + policy?: ObservationRetentionPolicy, + now = Date.now() +): { envelopes: ObservationEnvelope[]; result: ObservationPruneResult } { + if (!policy) { + return { + envelopes: [...envelopes], + result: { + deleted: 0, + retained: envelopes.length, + }, + }; + } + + let retained = [...envelopes]; + + if (policy.maxAgeMs !== undefined) { + const cutoff = now - Math.max(0, policy.maxAgeMs); + retained = retained.filter((envelope) => envelope.timestamp >= cutoff); + } + + if (policy.maxEntriesPerAgent !== undefined) { + const maxEntries = Math.max(1, Math.floor(policy.maxEntriesPerAgent)); + const counts = new Map(); + const next: ObservationEnvelope[] = []; + + for (let index = retained.length - 1; index >= 0; index--) { + const envelope = retained[index]; + const agentId = envelope.observation.agentId; + const count = counts.get(agentId) ?? 0; + if (count >= maxEntries) { + continue; + } + counts.set(agentId, count + 1); + next.push(envelope); + } + + retained = next.reverse(); + } + + return { + envelopes: retained, + result: { + deleted: envelopes.length - retained.length, + retained: retained.length, + }, + }; +} diff --git a/src/observability/persistence/sink.ts b/src/observability/persistence/sink.ts new file mode 100644 index 0000000..5719c7f --- /dev/null +++ b/src/observability/persistence/sink.ts @@ -0,0 +1,57 @@ +import { logger } from '../../utils/logger'; +import type { ObservationEnvelope, ObservationSink } from '../types'; +import type { ObservationQueryBackend, ObservationRetentionPolicy } from './types'; + +export class PersistedObservationSink implements ObservationSink { + private readonly pruneIntervalMs: number; + private lastPruneAt = 0; + private prunePromise: Promise = Promise.resolve(); + + constructor( + private readonly backend: ObservationQueryBackend, + private readonly opts?: { + retention?: ObservationRetentionPolicy; + pruneIntervalMs?: number; + } + ) { + this.pruneIntervalMs = Math.max(1_000, opts?.pruneIntervalMs ?? 60_000); + } + + async onObservation(envelope: ObservationEnvelope): Promise { + try { + await this.backend.append(envelope); + } catch (error) { + logger.warn('[Observability] Persisted sink append failed:', error); + return; + } + + await this.maybePrune(); + } + + async shutdown(): Promise { + await this.prunePromise.catch(() => undefined); + await this.backend.close?.(); + } + + private async maybePrune(): Promise { + if (!this.opts?.retention || !this.backend.prune) { + return; + } + + const now = Date.now(); + if (now - this.lastPruneAt < this.pruneIntervalMs) { + return; + } + + this.lastPruneAt = now; + this.prunePromise = this.prunePromise + .then(async () => { + await this.backend.prune?.(this.opts?.retention); + }) + .catch((error) => { + logger.warn('[Observability] Persisted sink prune failed:', error); + }); + + await this.prunePromise; + } +} diff --git a/src/observability/persistence/types.ts b/src/observability/persistence/types.ts new file mode 100644 index 0000000..2560261 --- /dev/null +++ b/src/observability/persistence/types.ts @@ -0,0 +1,42 @@ +import type { + ObservationEnvelope, + ObservationListOptions, + ObservationRunView, +} from '../types'; + +export interface PersistedObservationListOptions extends ObservationListOptions { + agentIds?: string[]; + templateIds?: string[]; + fromTimestamp?: number; + toTimestamp?: number; +} + +export interface ObservationRetentionPolicy { + maxEntriesPerAgent?: number; + maxAgeMs?: number; +} + +export interface ObservationPruneResult { + deleted: number; + retained: number; +} + +export interface ObservationQueryBackend { + append(envelope: ObservationEnvelope): Promise; + list(opts?: PersistedObservationListOptions): Promise; + getRun(runId: string): Promise; + prune?(opts?: ObservationRetentionPolicy): Promise; + close?(): Promise; +} + +export interface PersistedObservationReader { + listObservations(opts?: PersistedObservationListOptions): Promise; + getRun(runId: string): Promise; +} + +export interface ObservationPersistenceConfig { + enabled?: boolean; + backend?: ObservationQueryBackend; + retention?: ObservationRetentionPolicy; + pruneIntervalMs?: number; +} diff --git a/src/observability/query.ts b/src/observability/query.ts new file mode 100644 index 0000000..f8bf37e --- /dev/null +++ b/src/observability/query.ts @@ -0,0 +1,78 @@ +import { + AgentRunObservation, + ObservationEnvelope, + ObservationListOptions, + ObservationQueryOptions, + ObservationRunView, +} from './types'; + +export function matchesObservationEnvelope( + envelope: ObservationEnvelope, + opts?: ObservationQueryOptions +): boolean { + if (!opts) { + return true; + } + + if (opts.sinceSeq !== undefined && envelope.seq <= opts.sinceSeq) { + return false; + } + + const observation = envelope.observation; + + if (opts.agentId !== undefined && observation.agentId !== opts.agentId) { + return false; + } + if (opts.kinds && !opts.kinds.includes(observation.kind)) { + return false; + } + if (opts.runId !== undefined && observation.runId !== opts.runId) { + return false; + } + if (opts.traceId !== undefined && observation.traceId !== opts.traceId) { + return false; + } + if (opts.parentSpanId !== undefined && observation.parentSpanId !== opts.parentSpanId) { + return false; + } + if (opts.statuses && !opts.statuses.includes(observation.status)) { + return false; + } + + return true; +} + +export function filterObservationEnvelopes( + envelopes: ObservationEnvelope[], + opts?: ObservationListOptions +): ObservationEnvelope[] { + let filtered = opts ? envelopes.filter((envelope) => matchesObservationEnvelope(envelope, opts)) : [...envelopes]; + + if (opts?.limit !== undefined) { + filtered = filtered.slice(-opts.limit); + } + + return filtered; +} + +export function buildObservationRunView( + envelopes: ObservationEnvelope[], + runId: string +): ObservationRunView | undefined { + const observations = envelopes.filter((envelope) => envelope.observation.runId === runId); + if (observations.length === 0) { + return undefined; + } + + const run = observations.find( + (envelope): envelope is ObservationEnvelope => envelope.observation.kind === 'agent_run' + ); + if (!run) { + return undefined; + } + + return { + run, + observations, + }; +} diff --git a/src/observability/reader.ts b/src/observability/reader.ts new file mode 100644 index 0000000..81a1c60 --- /dev/null +++ b/src/observability/reader.ts @@ -0,0 +1,10 @@ +import { ObservationReader } from './types'; + +export function createObservationReader(source: ObservationReader): ObservationReader { + return { + subscribe: (opts) => source.subscribe(opts), + getMetricsSnapshot: () => source.getMetricsSnapshot(), + listObservations: (opts) => source.listObservations(opts), + getRun: (runId) => source.getRun(runId), + }; +} diff --git a/src/observability/sinks/composite.ts b/src/observability/sinks/composite.ts new file mode 100644 index 0000000..0a59c0a --- /dev/null +++ b/src/observability/sinks/composite.ts @@ -0,0 +1,16 @@ +import { logger } from '../../utils/logger'; +import { ObservationEnvelope, ObservationSink } from '../types'; + +export class CompositeObservationSink implements ObservationSink { + constructor(private readonly sinks: ObservationSink[]) {} + + async onObservation(envelope: ObservationEnvelope): Promise { + for (const sink of this.sinks) { + try { + await sink.onObservation(envelope); + } catch (error) { + logger.warn('[Observability] Composite sink target failed:', error); + } + } + } +} diff --git a/src/observability/sinks/memory.ts b/src/observability/sinks/memory.ts new file mode 100644 index 0000000..e63982d --- /dev/null +++ b/src/observability/sinks/memory.ts @@ -0,0 +1,151 @@ +import { buildObservationRunView, filterObservationEnvelopes, matchesObservationEnvelope } from '../query'; +import { + ObservationEnvelope, + ObservationListOptions, + ObservationReader, + ObservationRunView, + ObservationSink, + ObservationSubscribeOptions, +} from '../types'; + +class ObservationSubscriber { + private readonly queue: ObservationEnvelope[] = []; + private resolver?: (value: ObservationEnvelope | null) => void; + private closed = false; + + constructor(private readonly opts?: ObservationSubscribeOptions) {} + + push(envelope: ObservationEnvelope): void { + if (this.closed) return; + if (!matchesObservationEnvelope(envelope, this.opts)) return; + if (this.resolver) { + const resolve = this.resolver; + this.resolver = undefined; + resolve(envelope); + return; + } + this.queue.push(envelope); + } + + next(): Promise { + if (this.queue.length > 0) { + return Promise.resolve(this.queue.shift() || null); + } + if (this.closed) { + return Promise.resolve(null); + } + return new Promise((resolve) => { + this.resolver = resolve; + }); + } + + close(): void { + this.closed = true; + if (this.resolver) { + const resolve = this.resolver; + this.resolver = undefined; + resolve(null); + } + } +} + +export interface MemoryObservationStoreOptions { + maxEntries?: number; +} + +export class MemoryObservationStore implements ObservationSink, ObservationReader { + private seq = 0; + private readonly envelopes: ObservationEnvelope[] = []; + private readonly subscribers = new Set(); + private readonly maxEntries: number; + + constructor(opts?: MemoryObservationStoreOptions) { + this.maxEntries = Math.max(1, opts?.maxEntries ?? 2000); + } + + onObservation(envelope: ObservationEnvelope): void { + const stored: ObservationEnvelope = { + seq: this.seq++, + timestamp: envelope.timestamp, + observation: envelope.observation, + }; + + this.envelopes.push(stored); + if (this.envelopes.length > this.maxEntries) { + this.envelopes.splice(0, this.envelopes.length - this.maxEntries); + } + + for (const subscriber of this.subscribers) { + subscriber.push(stored); + } + } + + subscribe(opts?: ObservationSubscribeOptions): AsyncIterable { + const subscriber = new ObservationSubscriber(opts); + this.subscribers.add(subscriber); + + for (const envelope of this.envelopes) { + subscriber.push(envelope); + } + + const self = this; + return { + [Symbol.asyncIterator](): AsyncIterator { + return { + async next() { + const value = await subscriber.next(); + if (!value) { + self.subscribers.delete(subscriber); + return { done: true, value: undefined as any }; + } + return { done: false, value }; + }, + async return() { + subscriber.close(); + self.subscribers.delete(subscriber); + return { done: true, value: undefined as any }; + }, + }; + }, + }; + } + + getMetricsSnapshot() { + return { + agentId: 'memory-observation-store', + totals: { + inputTokens: 0, + outputTokens: 0, + totalTokens: 0, + reasoningTokens: 0, + totalCostUsd: 0, + toolCalls: 0, + toolErrors: 0, + approvalRequests: 0, + approvalDenials: 0, + approvalWaitMsTotal: 0, + compressions: 0, + compressionErrors: 0, + tokensSavedEstimate: 0, + scheduledRuns: 0, + generations: 0, + generationErrors: 0, + subagents: 0, + }, + }; + } + + listObservations(opts?: ObservationListOptions): ObservationEnvelope[] { + return filterObservationEnvelopes(this.envelopes, opts); + } + + getRun(runId: string): ObservationRunView | undefined { + return buildObservationRunView(this.envelopes, runId); + } + + list(): ObservationEnvelope[] { + return this.listObservations(); + } +} + +export class MemoryObservationSink extends MemoryObservationStore {} diff --git a/src/observability/sinks/noop.ts b/src/observability/sinks/noop.ts new file mode 100644 index 0000000..32a0993 --- /dev/null +++ b/src/observability/sinks/noop.ts @@ -0,0 +1,7 @@ +import { ObservationEnvelope, ObservationSink } from '../types'; + +export class NoopObservationSink implements ObservationSink { + onObservation(_envelope: ObservationEnvelope): void { + // Intentionally empty. + } +} diff --git a/src/observability/types.ts b/src/observability/types.ts new file mode 100644 index 0000000..4731212 --- /dev/null +++ b/src/observability/types.ts @@ -0,0 +1,200 @@ +import type { ObservationPersistenceConfig } from './persistence/types'; +import type { OTelBridgeConfig } from './otel/types'; + +export type CaptureMode = 'off' | 'summary' | 'full' | 'redacted'; + +export type ObservationKind = 'agent_run' | 'generation' | 'tool' | 'subagent' | 'compression'; +export type ObservationStatus = 'ok' | 'error' | 'cancelled'; + +export interface BaseObservation { + kind: ObservationKind; + agentId: string; + runId: string; + traceId: string; + spanId: string; + parentSpanId?: string; + name: string; + status: ObservationStatus; + startTime: number; + endTime?: number; + durationMs?: number; + metadata?: Record; +} + +export interface GenerationObservation extends BaseObservation { + kind: 'generation'; + provider?: string; + model?: string; + requestId?: string; + inputSummary?: unknown; + outputSummary?: unknown; + usage?: { + inputTokens: number; + outputTokens: number; + totalTokens: number; + reasoningTokens?: number; + cacheCreationTokens?: number; + cacheReadTokens?: number; + }; + cost?: { + inputCost: number; + outputCost: number; + cacheWriteCost: number; + totalCost: number; + cacheSavings: number; + currency: 'USD'; + }; + request?: { + latencyMs: number; + timeToFirstTokenMs?: number; + tokensPerSecond?: number; + stopReason?: string; + retryCount?: number; + }; + errorMessage?: string; +} + +export interface ToolObservation extends BaseObservation { + kind: 'tool'; + toolCallId: string; + toolName: string; + toolState: string; + approvalRequired: boolean; + approval?: { + required: boolean; + status: 'not_required' | 'pending' | 'approved' | 'denied'; + approvalId?: string; + requestedAt?: number; + decidedAt?: number; + waitMs?: number; + noteSummary?: string; + }; + inputSummary?: unknown; + outputSummary?: unknown; + errorMessage?: string; +} + +export interface SubagentObservation extends BaseObservation { + kind: 'subagent'; + childAgentId: string; + childRunId?: string; + templateId: string; + delegatedBy?: string; + errorMessage?: string; +} + +export interface CompressionObservation extends BaseObservation { + kind: 'compression'; + policy: 'context_window'; + reason: 'token_threshold' | 'manual' | 'resume_recovery'; + messageCountBefore: number; + messageCountAfter?: number; + estimatedTokensBefore?: number; + estimatedTokensAfter?: number; + ratio?: number; + summaryGenerated: boolean; + errorMessage?: string; +} + +export interface AgentRunObservation extends BaseObservation { + kind: 'agent_run'; + trigger: 'send' | 'complete' | 'resume' | 'scheduler' | 'delegate'; + step: number; + messageCountBefore: number; + messageCountAfter?: number; + errorMessage?: string; +} + +export type ObservationRecord = + | AgentRunObservation + | GenerationObservation + | ToolObservation + | SubagentObservation + | CompressionObservation; + +export interface ObservationEnvelope { + seq: number; + timestamp: number; + observation: T; +} + +export interface ObservationQueryOptions { + agentId?: string; + kinds?: ObservationKind[]; + runId?: string; + traceId?: string; + parentSpanId?: string; + statuses?: ObservationStatus[]; + sinceSeq?: number; +} + +export interface ObservationListOptions extends ObservationQueryOptions { + limit?: number; +} + +export interface ObservationRunView { + run: ObservationEnvelope; + observations: ObservationEnvelope[]; +} + +export interface AgentMetricsSnapshot { + agentId: string; + currentRunId?: string; + traceId?: string; + totals: { + inputTokens: number; + outputTokens: number; + totalTokens: number; + reasoningTokens: number; + totalCostUsd: number; + toolCalls: number; + toolErrors: number; + approvalRequests: number; + approvalDenials: number; + approvalWaitMsTotal: number; + compressions: number; + compressionErrors: number; + tokensSavedEstimate: number; + scheduledRuns: number; + generations: number; + generationErrors: number; + subagents: number; + }; + lastGeneration?: { + provider?: string; + model?: string; + requestId?: string; + latencyMs?: number; + timeToFirstTokenMs?: number; + stopReason?: string; + retryCount?: number; + totalTokens?: number; + totalCostUsd?: number; + }; +} + +export interface ObservationSubscribeOptions extends ObservationQueryOptions {} + +export interface ObservationSink { + onObservation(envelope: ObservationEnvelope): void | Promise; +} + +export interface ObservationReader { + subscribe(opts?: ObservationSubscribeOptions): AsyncIterable; + getMetricsSnapshot(): AgentMetricsSnapshot; + listObservations(opts?: ObservationListOptions): ObservationEnvelope[]; + getRun(runId: string): ObservationRunView | undefined; +} + +export interface ObservabilityConfig { + enabled?: boolean; + sink?: ObservationSink; + otel?: OTelBridgeConfig; + persistence?: ObservationPersistenceConfig; + capture?: { + generationInput?: CaptureMode; + generationOutput?: CaptureMode; + toolInput?: CaptureMode; + toolOutput?: CaptureMode; + }; +} diff --git a/tests/unit/observability/approval-metadata.test.ts b/tests/unit/observability/approval-metadata.test.ts new file mode 100644 index 0000000..ed2af24 --- /dev/null +++ b/tests/unit/observability/approval-metadata.test.ts @@ -0,0 +1,255 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import { + Agent, + AgentTemplateRegistry, + JSONStore, + SandboxFactory, + ToolRegistry, +} from '../../../src'; +import { ModelConfig, ModelProvider, ModelResponse, ModelStreamChunk } from '../../../src/infra/provider'; +import { TestRunner, expect } from '../../helpers/utils'; +import { TEST_ROOT } from '../../helpers/fixtures'; +import { ensureCleanDir } from '../../helpers/setup'; + +const runner = new TestRunner('Observability Approval Metadata'); + +class QueueStreamProvider implements ModelProvider { + readonly model = 'queue-stream-provider'; + readonly maxWindowSize = 128000; + readonly maxOutputTokens = 4096; + readonly temperature = 0; + + constructor( + private readonly streams: Array<() => AsyncIterable>, + private readonly providerName = 'mock' + ) {} + + async complete(): Promise { + throw new Error('complete() should not be called'); + } + + async *stream(): AsyncIterable { + const next = this.streams.shift(); + if (!next) { + throw new Error('No scripted stream available'); + } + yield* next(); + } + + toConfig(): ModelConfig { + return { + provider: this.providerName, + model: this.model, + }; + } +} + +async function createObservedAgent(params: { + provider: ModelProvider; + template?: any; + registerTools?: (registry: ToolRegistry) => void; +}) { + const workDir = path.join(TEST_ROOT, `obs-approval-work-${Date.now()}-${Math.random().toString(36).slice(2, 7)}`); + const storeDir = path.join(TEST_ROOT, `obs-approval-store-${Date.now()}-${Math.random().toString(36).slice(2, 7)}`); + ensureCleanDir(workDir); + ensureCleanDir(storeDir); + + const templates = new AgentTemplateRegistry(); + templates.register( + params.template || { + id: 'obs-approval-agent', + systemPrompt: 'test approval observability', + tools: [], + permission: { mode: 'auto' }, + } + ); + + const tools = new ToolRegistry(); + params.registerTools?.(tools); + + const agent = await Agent.create( + { + templateId: params.template?.id || 'obs-approval-agent', + model: params.provider, + sandbox: { kind: 'local', workDir, enforceBoundary: true }, + }, + { + store: new JSONStore(storeDir), + templateRegistry: templates, + sandboxFactory: new SandboxFactory(), + toolRegistry: tools, + } + ); + + return { + agent, + cleanup: async () => { + await (agent as any).sandbox?.dispose?.(); + fs.rmSync(workDir, { recursive: true, force: true }); + fs.rmSync(storeDir, { recursive: true, force: true }); + }, + }; +} + +async function collectToolObservation(agent: Agent) { + for await (const envelope of agent.subscribeObservations({ kinds: ['tool'] })) { + return envelope.observation as any; + } + return undefined; +} + +function registerEchoTool(registry: ToolRegistry) { + registry.register('echo_tool', () => ({ + name: 'echo_tool', + description: 'echo value', + input_schema: { type: 'object', properties: { value: { type: 'string' } }, required: ['value'] }, + async exec(args: any) { + return { echoed: args.value }; + }, + toDescriptor() { + return { source: 'registered' as const, name: 'echo_tool', registryId: 'echo_tool' }; + }, + })); +} + +function createApprovalProvider(finalText: string): ModelProvider { + return new QueueStreamProvider([ + async function* () { + yield { + type: 'content_block_start', + index: 0, + content_block: { type: 'tool_use', id: 'tool-1', name: 'echo_tool', input: {} }, + }; + yield { + type: 'content_block_delta', + index: 0, + delta: { type: 'input_json_delta', partial_json: '{"value":"hi"}' }, + }; + yield { type: 'content_block_stop', index: 0 }; + yield { type: 'message_stop' }; + }, + async function* () { + yield { type: 'content_block_start', index: 0, content_block: { type: 'text', text: '' } }; + yield { type: 'content_block_delta', index: 0, delta: { type: 'text_delta', text: finalText } }; + yield { type: 'content_block_stop', index: 0 }; + yield { type: 'message_delta', usage: { input_tokens: 1, output_tokens: 1 } }; + yield { type: 'message_stop' }; + }, + ]); +} + +runner + .test('tool observation 记录 approved approval metadata 与快照聚合', async () => { + const { agent, cleanup } = await createObservedAgent({ + provider: createApprovalProvider('approval done'), + template: { + id: 'obs-approval-allow', + systemPrompt: 'use tools', + tools: ['echo_tool'], + permission: { mode: 'approval', requireApprovalTools: ['echo_tool'] as const }, + }, + registerTools: registerEchoTool, + }); + + const offPermissionRequired = agent.on('permission_required', async (evt: any) => { + await new Promise((resolve) => setTimeout(resolve, 10)); + await evt.respond('allow', { note: 'Bearer sk-secret-value approved' }); + }); + + try { + const result = await agent.chat('run tool'); + expect.toEqual(result.status, 'ok'); + + const toolObservation = await collectToolObservation(agent); + expect.toEqual(toolObservation.kind, 'tool'); + expect.toEqual(toolObservation.approval.status, 'approved'); + expect.toEqual(toolObservation.approval.required, true); + expect.toBeTruthy(toolObservation.approval.waitMs !== undefined); + expect.toBeGreaterThanOrEqual(toolObservation.approval.waitMs, 0); + expect.toBeTruthy(toolObservation.approval.noteSummary); + expect.toContain(toolObservation.approval.noteSummary, 'sk-***'); + expect.toBeFalsy(toolObservation.approval.noteSummary.includes('sk-secret-value')); + + const snapshot = agent.getMetricsSnapshot(); + expect.toEqual(snapshot.totals.approvalRequests, 1); + expect.toEqual(snapshot.totals.approvalDenials, 0); + expect.toBeGreaterThanOrEqual(snapshot.totals.approvalWaitMsTotal, 0); + } finally { + offPermissionRequired(); + await cleanup(); + } + }) + .test('tool observation 记录 denied approval metadata 与拒绝聚合', async () => { + const { agent, cleanup } = await createObservedAgent({ + provider: createApprovalProvider('approval denied'), + template: { + id: 'obs-approval-deny', + systemPrompt: 'use tools', + tools: ['echo_tool'], + permission: { mode: 'approval', requireApprovalTools: ['echo_tool'] as const }, + }, + registerTools: registerEchoTool, + }); + + const offPermissionRequired = agent.on('permission_required', async (evt: any) => { + await new Promise((resolve) => setTimeout(resolve, 10)); + await evt.respond('deny', { note: 'sk-secret-value deny this call' }); + }); + + try { + const result = await agent.chat('run tool'); + expect.toEqual(result.status, 'ok'); + + const toolObservation = await collectToolObservation(agent); + expect.toEqual(toolObservation.kind, 'tool'); + expect.toEqual(toolObservation.approval.status, 'denied'); + expect.toEqual(toolObservation.approval.required, true); + expect.toBeTruthy(toolObservation.approval.waitMs !== undefined); + expect.toBeGreaterThanOrEqual(toolObservation.approval.waitMs, 0); + expect.toBeTruthy(toolObservation.approval.noteSummary); + expect.toContain(toolObservation.approval.noteSummary, 'sk-***'); + expect.toBeFalsy(toolObservation.approval.noteSummary.includes('sk-secret-value')); + + const snapshot = agent.getMetricsSnapshot(); + expect.toEqual(snapshot.totals.approvalRequests, 1); + expect.toEqual(snapshot.totals.approvalDenials, 1); + expect.toBeGreaterThanOrEqual(snapshot.totals.approvalWaitMsTotal, 0); + } finally { + offPermissionRequired(); + await cleanup(); + } + }) + .test('无需审批的工具 observation 标记为 not_required', async () => { + const { agent, cleanup } = await createObservedAgent({ + provider: createApprovalProvider('no approval needed'), + template: { + id: 'obs-approval-not-required', + systemPrompt: 'use tools', + tools: ['echo_tool'], + permission: { mode: 'auto' }, + }, + registerTools: registerEchoTool, + }); + + try { + const result = await agent.chat('run tool'); + expect.toEqual(result.status, 'ok'); + + const toolObservation = await collectToolObservation(agent); + expect.toEqual(toolObservation.kind, 'tool'); + expect.toEqual(toolObservation.approval.status, 'not_required'); + expect.toEqual(toolObservation.approval.required, false); + + const snapshot = agent.getMetricsSnapshot(); + expect.toEqual(snapshot.totals.approvalRequests, 0); + expect.toEqual(snapshot.totals.approvalDenials, 0); + expect.toEqual(snapshot.totals.approvalWaitMsTotal, 0); + } finally { + await cleanup(); + } + }); + +export async function run() { + return runner.run(); +} diff --git a/tests/unit/observability/composite-sink.test.ts b/tests/unit/observability/composite-sink.test.ts new file mode 100644 index 0000000..a01f1e2 --- /dev/null +++ b/tests/unit/observability/composite-sink.test.ts @@ -0,0 +1,149 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import { + Agent, + AgentTemplateRegistry, + CompositeObservationSink, + JSONStore, + MemoryObservationStore, + SandboxFactory, + ToolRegistry, +} from '../../../src'; +import { ModelConfig, ModelProvider, ModelResponse, ModelStreamChunk } from '../../../src/infra/provider'; +import { TestRunner, expect } from '../../helpers/utils'; +import { TEST_ROOT } from '../../helpers/fixtures'; +import { ensureCleanDir, wait } from '../../helpers/setup'; + +const runner = new TestRunner('Observability Composite Sink'); + +class QueueStreamProvider implements ModelProvider { + readonly model = 'queue-stream-provider'; + readonly maxWindowSize = 128000; + readonly maxOutputTokens = 4096; + readonly temperature = 0; + + constructor( + private readonly streams: Array<() => AsyncIterable>, + private readonly providerName = 'mock' + ) {} + + async complete(): Promise { + throw new Error('complete() should not be called'); + } + + async *stream(): AsyncIterable { + const next = this.streams.shift(); + if (!next) { + throw new Error('No scripted stream available'); + } + yield* next(); + } + + toConfig(): ModelConfig { + return { + provider: this.providerName, + model: this.model, + }; + } +} + +async function createObservedAgent(params: { + provider: ModelProvider; + sink: CompositeObservationSink; +}) { + const workDir = path.join(TEST_ROOT, `obs-composite-work-${Date.now()}-${Math.random().toString(36).slice(2, 7)}`); + const storeDir = path.join(TEST_ROOT, `obs-composite-store-${Date.now()}-${Math.random().toString(36).slice(2, 7)}`); + ensureCleanDir(workDir); + ensureCleanDir(storeDir); + + const templates = new AgentTemplateRegistry(); + templates.register({ + id: 'obs-composite-agent', + systemPrompt: 'composite sink test', + tools: [], + permission: { mode: 'auto' }, + }); + + const agent = await Agent.create( + { + templateId: 'obs-composite-agent', + model: params.provider, + sandbox: { kind: 'local', workDir, enforceBoundary: true }, + observability: { sink: params.sink }, + }, + { + store: new JSONStore(storeDir), + templateRegistry: templates, + sandboxFactory: new SandboxFactory(), + toolRegistry: new ToolRegistry(), + } + ); + + return { + agent, + cleanup: async () => { + await (agent as any).sandbox?.dispose?.(); + fs.rmSync(workDir, { recursive: true, force: true }); + fs.rmSync(storeDir, { recursive: true, force: true }); + }, + }; +} + +runner.test('CompositeObservationSink 中单个 sink 失败不影响其他 sink 与主流程', async () => { + const received: any[] = []; + const memoryStore = new MemoryObservationStore(); + const composite = new CompositeObservationSink([ + { + onObservation(envelope) { + received.push(envelope); + throw new Error('sink boom'); + }, + }, + memoryStore, + ]); + + const provider = new QueueStreamProvider([ + async function* () { + yield { type: 'content_block_start', index: 0, content_block: { type: 'text', text: '' } }; + yield { type: 'content_block_delta', index: 0, delta: { type: 'text_delta', text: 'composite ok' } }; + yield { type: 'content_block_stop', index: 0 }; + yield { type: 'message_delta', usage: { input_tokens: 2, output_tokens: 3 } }; + yield { type: 'message_stop' }; + }, + ]); + + const { agent, cleanup } = await createObservedAgent({ provider, sink: composite }); + + try { + const result = await agent.chat('hello'); + expect.toEqual(result.status, 'ok'); + expect.toContain(result.text || '', 'composite ok'); + + for (let i = 0; i < 20; i++) { + if (received.length >= 2 && memoryStore.listObservations().length >= 2) { + break; + } + await wait(10); + } + + expect.toBeGreaterThanOrEqual(received.length, 2); + + const storeObservations = memoryStore.listObservations(); + expect.toHaveLength(storeObservations, 2); + expect.toEqual(storeObservations[0].observation.kind, 'generation'); + expect.toEqual(storeObservations[1].observation.kind, 'agent_run'); + + const snapshot = agent.getMetricsSnapshot(); + expect.toEqual(snapshot.totals.totalTokens, 5); + expect.toEqual(snapshot.totals.generations, 1); + + const readerObservations = agent.getObservationReader().listObservations(); + expect.toHaveLength(readerObservations, 2); + } finally { + await cleanup(); + } +}); + +export async function run() { + return runner.run(); +} diff --git a/tests/unit/observability/compression-observation.test.ts b/tests/unit/observability/compression-observation.test.ts new file mode 100644 index 0000000..fe87fb0 --- /dev/null +++ b/tests/unit/observability/compression-observation.test.ts @@ -0,0 +1,190 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import { + Agent, + AgentTemplateRegistry, + JSONStore, + SandboxFactory, + ToolRegistry, +} from '../../../src'; +import { ModelConfig, ModelProvider, ModelResponse, ModelStreamChunk } from '../../../src/infra/provider'; +import { TestRunner, expect } from '../../helpers/utils'; +import { TEST_ROOT } from '../../helpers/fixtures'; +import { ensureCleanDir } from '../../helpers/setup'; + +const runner = new TestRunner('Observability Compression'); + +class QueueStreamProvider implements ModelProvider { + readonly model = 'queue-stream-provider'; + readonly maxWindowSize = 128000; + readonly maxOutputTokens = 4096; + readonly temperature = 0; + + constructor( + private readonly streams: Array<() => AsyncIterable>, + private readonly providerName = 'mock' + ) {} + + async complete(): Promise { + throw new Error('complete() should not be called'); + } + + async *stream(): AsyncIterable { + const next = this.streams.shift(); + if (!next) { + throw new Error('No scripted stream available'); + } + yield* next(); + } + + toConfig(): ModelConfig { + return { + provider: this.providerName, + model: this.model, + }; + } +} + +async function createObservedAgent(params: { + provider: ModelProvider; + context?: { maxTokens?: number; compressToTokens?: number }; +}) { + const workDir = path.join(TEST_ROOT, `obs-compress-work-${Date.now()}-${Math.random().toString(36).slice(2, 7)}`); + const storeDir = path.join(TEST_ROOT, `obs-compress-store-${Date.now()}-${Math.random().toString(36).slice(2, 7)}`); + ensureCleanDir(workDir); + ensureCleanDir(storeDir); + + const templates = new AgentTemplateRegistry(); + templates.register({ + id: 'obs-compress-agent', + systemPrompt: 'test compression observability', + tools: [], + permission: { mode: 'auto' }, + }); + + const agent = await Agent.create( + { + templateId: 'obs-compress-agent', + model: params.provider, + sandbox: { kind: 'local', workDir, enforceBoundary: true }, + context: params.context, + }, + { + store: new JSONStore(storeDir), + templateRegistry: templates, + sandboxFactory: new SandboxFactory(), + toolRegistry: new ToolRegistry(), + } + ); + + return { + agent, + cleanup: async () => { + await (agent as any).sandbox?.dispose?.(); + fs.rmSync(workDir, { recursive: true, force: true }); + fs.rmSync(storeDir, { recursive: true, force: true }); + }, + }; +} + +async function listCompressionObservations(agent: Agent) { + return (agent as any).observationCollector.list({ kinds: ['compression'] }); +} + +function createTextStream(text: string): () => AsyncIterable { + return async function* () { + yield { type: 'content_block_start', index: 0, content_block: { type: 'text', text: '' } }; + yield { type: 'content_block_delta', index: 0, delta: { type: 'text_delta', text } }; + yield { type: 'content_block_stop', index: 0 }; + yield { type: 'message_delta', usage: { input_tokens: 1, output_tokens: 1 } }; + yield { type: 'message_stop' }; + }; +} + +runner + .test('触发 context compression 时生成 compression observation 与快照聚合', async () => { + const provider = new QueueStreamProvider([ + createTextStream('first'), + createTextStream('second'), + ]); + const { agent, cleanup } = await createObservedAgent({ + provider, + context: { maxTokens: 50, compressToTokens: 10 }, + }); + + try { + await agent.chat('a'.repeat(120)); + await agent.chat('b'.repeat(120)); + + const observations = await listCompressionObservations(agent); + expect.toHaveLength(observations, 1); + expect.toEqual(observations[0].kind, 'compression'); + expect.toEqual(observations[0].status, 'ok'); + expect.toEqual(observations[0].summaryGenerated, true); + expect.toBeTruthy(observations[0].messageCountBefore >= 1); + expect.toBeTruthy(observations[0].messageCountAfter !== undefined); + expect.toBeTruthy(observations[0].estimatedTokensBefore !== undefined); + expect.toBeTruthy(observations[0].estimatedTokensAfter !== undefined); + + const snapshot = agent.getMetricsSnapshot(); + expect.toEqual(snapshot.totals.compressions, 1); + expect.toEqual(snapshot.totals.compressionErrors, 0); + expect.toBeGreaterThanOrEqual(snapshot.totals.tokensSavedEstimate, 0); + } finally { + await cleanup(); + } + }) + .test('未触发压缩时不生成 compression observation', async () => { + const provider = new QueueStreamProvider([createTextStream('no compression')]); + const { agent, cleanup } = await createObservedAgent({ + provider, + context: { maxTokens: 1000, compressToTokens: 500 }, + }); + + try { + await agent.chat('short prompt'); + + const snapshot = agent.getMetricsSnapshot(); + expect.toEqual(snapshot.totals.compressions, 0); + expect.toEqual(snapshot.totals.compressionErrors, 0); + + const observations = await listCompressionObservations(agent); + expect.toHaveLength(observations, 0); + } finally { + await cleanup(); + } + }) + .test('压缩失败时记录 compression error observation', async () => { + const provider = new QueueStreamProvider([createTextStream('unused because compression fails')]); + const { agent, cleanup } = await createObservedAgent({ + provider, + context: { maxTokens: 50, compressToTokens: 10 }, + }); + + (agent as any).contextManager.compress = async () => { + throw new Error('compression boom'); + }; + + try { + const result = await agent.chat('z'.repeat(240)); + expect.toEqual(result.status, 'ok'); + + const observations = await listCompressionObservations(agent); + expect.toHaveLength(observations, 1); + expect.toEqual(observations[0].kind, 'compression'); + expect.toEqual(observations[0].status, 'error'); + expect.toEqual(observations[0].summaryGenerated, false); + expect.toContain(observations[0].errorMessage || '', 'compression boom'); + + const snapshot = agent.getMetricsSnapshot(); + expect.toEqual(snapshot.totals.compressions, 1); + expect.toEqual(snapshot.totals.compressionErrors, 1); + expect.toEqual(snapshot.totals.tokensSavedEstimate, 0); + } finally { + await cleanup(); + } + }); + +export async function run() { + return runner.run(); +} diff --git a/tests/unit/observability/core-observation.test.ts b/tests/unit/observability/core-observation.test.ts new file mode 100644 index 0000000..44a427b --- /dev/null +++ b/tests/unit/observability/core-observation.test.ts @@ -0,0 +1,420 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import { + Agent, + AgentTemplateRegistry, + ObservabilityConfig, + JSONStore, + SandboxFactory, + ToolRegistry, + AgentMetricsSnapshot, +} from '../../../src'; +import { ModelConfig, ModelProvider, ModelResponse, ModelStreamChunk } from '../../../src/infra/provider'; +import { TestRunner, expect } from '../../helpers/utils'; +import { TEST_ROOT } from '../../helpers/fixtures'; +import { ensureCleanDir } from '../../helpers/setup'; + +const runner = new TestRunner('Observability Core Observation'); + +class QueueStreamProvider implements ModelProvider { + readonly model = 'queue-stream-provider'; + readonly maxWindowSize = 128000; + readonly maxOutputTokens = 4096; + readonly temperature = 0; + + constructor( + private readonly streams: Array<() => AsyncIterable>, + private readonly providerName = 'mock' + ) {} + + async complete(): Promise { + throw new Error('complete() should not be called'); + } + + async *stream(): AsyncIterable { + const next = this.streams.shift(); + if (!next) { + throw new Error('No scripted stream available'); + } + yield* next(); + } + + toConfig(): ModelConfig { + return { + provider: this.providerName, + model: this.model, + }; + } +} + +function createGenerationUsage() { + return { + inputTokens: 11, + outputTokens: 7, + totalTokens: 18, + reasoningTokens: 3, + cache: { + cacheCreationTokens: 0, + cacheReadTokens: 0, + provider: {}, + }, + cost: { + inputCost: 0.00001, + outputCost: 0.00002, + cacheWriteCost: 0, + totalCost: 0.00003, + cacheSavings: 0, + currency: 'USD' as const, + }, + request: { + startTime: Date.now() - 20, + endTime: Date.now(), + latencyMs: 20, + timeToFirstTokenMs: 5, + requestId: 'req-observe-1', + modelUsed: 'queue-stream-provider', + stopReason: 'end_turn', + retryCount: 0, + }, + }; +} + +async function createObservedAgent(params: { + provider: ModelProvider; + template?: any; + registerTools?: (registry: ToolRegistry) => void; + observability?: ObservabilityConfig; +}) { + const workDir = path.join(TEST_ROOT, `obs-work-${Date.now()}-${Math.random().toString(36).slice(2, 7)}`); + const storeDir = path.join(TEST_ROOT, `obs-store-${Date.now()}-${Math.random().toString(36).slice(2, 7)}`); + ensureCleanDir(workDir); + ensureCleanDir(storeDir); + const store = new JSONStore(storeDir); + + const templates = new AgentTemplateRegistry(); + templates.register( + params.template || { + id: 'obs-agent', + systemPrompt: 'test observability', + tools: [], + permission: { mode: 'auto' }, + } + ); + + const tools = new ToolRegistry(); + params.registerTools?.(tools); + + const agent = await Agent.create( + { + templateId: params.template?.id || 'obs-agent', + model: params.provider, + sandbox: { kind: 'local', workDir, enforceBoundary: true }, + observability: params.observability, + }, + { + store, + templateRegistry: templates, + sandboxFactory: new SandboxFactory(), + toolRegistry: tools, + } + ); + + return { + agent, + store, + cleanup: async () => { + await (agent as any).sandbox?.dispose?.(); + fs.rmSync(workDir, { recursive: true, force: true }); + fs.rmSync(storeDir, { recursive: true, force: true }); + }, + }; +} + +async function collectObservations(agent: Agent, expected: number) { + const collected: any[] = []; + for await (const envelope of agent.subscribeObservations()) { + collected.push(envelope); + if (collected.length >= expected) { + break; + } + } + return collected; +} + +runner + .test('metrics snapshot 暴露 generation token/cost/latency', async () => { + const extendedUsage = createGenerationUsage(); + const provider = new QueueStreamProvider([ + async function* () { + yield { type: 'content_block_start', index: 0, content_block: { type: 'text', text: '' } }; + yield { type: 'content_block_delta', index: 0, delta: { type: 'text_delta', text: 'observed reply' } }; + yield { type: 'content_block_stop', index: 0 }; + yield { type: 'message_delta', usage: { input_tokens: 11, output_tokens: 7 } }; + yield { type: 'message_stop', stop_reason: 'end_turn', extendedUsage }; + }, + ]); + + const { agent, cleanup } = await createObservedAgent({ provider }); + try { + const result = await agent.chat('hello'); + expect.toEqual(result.status, 'ok'); + expect.toContain(result.text || '', 'observed reply'); + + const snapshot: AgentMetricsSnapshot = agent.getMetricsSnapshot(); + expect.toEqual(snapshot.totals.inputTokens, 11); + expect.toEqual(snapshot.totals.outputTokens, 7); + expect.toEqual(snapshot.totals.totalTokens, 18); + expect.toEqual(snapshot.totals.reasoningTokens, 3); + expect.toEqual(snapshot.totals.totalCostUsd, 0.00003); + expect.toEqual(snapshot.totals.generations, 1); + expect.toEqual(snapshot.lastGeneration?.requestId, 'req-observe-1'); + expect.toEqual(snapshot.lastGeneration?.stopReason, 'end_turn'); + expect.toEqual(snapshot.lastGeneration?.latencyMs, 20); + + const observations = await collectObservations(agent, 2); + expect.toEqual(observations[0].observation.kind, 'generation'); + expect.toEqual(observations[1].observation.kind, 'agent_run'); + expect.toEqual((observations[0].observation as any).rawUsage, undefined); + expect.toEqual( + (observations[0].observation as any).metadata?.__debug?.extendedUsage?.request?.requestId, + 'req-observe-1' + ); + } finally { + await cleanup(); + } + }) + .test('generation observation 的 inputSummary 使用真实模型输入而不是 assistant 输出', async () => { + const provider = new QueueStreamProvider([ + async function* () { + yield { type: 'content_block_start', index: 0, content_block: { type: 'text', text: '' } }; + yield { type: 'content_block_delta', index: 0, delta: { type: 'text_delta', text: 'assistant final reply' } }; + yield { type: 'content_block_stop', index: 0 }; + yield { type: 'message_delta', usage: { input_tokens: 2, output_tokens: 4 } }; + yield { type: 'message_stop' }; + }, + ]); + + const { agent, cleanup } = await createObservedAgent({ + provider, + observability: { + capture: { + generationInput: 'full', + generationOutput: 'full', + }, + }, + }); + + try { + const result = await agent.chat('user asks for summary'); + expect.toEqual(result.status, 'ok'); + + const observations = await collectObservations(agent, 2); + const generation = observations.find((entry) => entry.observation.kind === 'generation')?.observation as any; + expect.toBeTruthy(generation); + expect.toEqual(generation.inputSummary[generation.inputSummary.length - 1].role, 'user'); + expect.toContain(generation.inputSummary[generation.inputSummary.length - 1].content[0].text, 'user asks for summary'); + expect.toContain(generation.outputSummary[0].text, 'assistant final reply'); + } finally { + await cleanup(); + } + }) + .test('subscribeObservations 支持 kind 过滤与历史回放', async () => { + const provider = new QueueStreamProvider([ + async function* () { + yield { type: 'content_block_start', index: 0, content_block: { type: 'text', text: '' } }; + yield { type: 'content_block_delta', index: 0, delta: { type: 'text_delta', text: 'filter test' } }; + yield { type: 'content_block_stop', index: 0 }; + yield { type: 'message_delta', usage: { input_tokens: 2, output_tokens: 2 } }; + yield { type: 'message_stop' }; + }, + ]); + + const { agent, cleanup } = await createObservedAgent({ provider }); + try { + await agent.chat('hello'); + + const filtered: any[] = []; + for await (const envelope of agent.subscribeObservations({ kinds: ['generation'] })) { + filtered.push(envelope); + break; + } + + expect.toHaveLength(filtered, 1); + expect.toEqual(filtered[0].observation.kind, 'generation'); + } finally { + await cleanup(); + } + }) + .test('tool observation 会进入 metrics snapshot 与 observation 流', async () => { + const provider = new QueueStreamProvider([ + async function* () { + yield { + type: 'content_block_start', + index: 0, + content_block: { type: 'tool_use', id: 'tool-1', name: 'echo_tool', input: {} }, + }; + yield { + type: 'content_block_delta', + index: 0, + delta: { type: 'input_json_delta', partial_json: '{"value":"hi"}' }, + }; + yield { type: 'content_block_stop', index: 0 }; + yield { type: 'message_stop' }; + }, + async function* () { + yield { type: 'content_block_start', index: 0, content_block: { type: 'text', text: '' } }; + yield { type: 'content_block_delta', index: 0, delta: { type: 'text_delta', text: 'tool finished' } }; + yield { type: 'content_block_stop', index: 0 }; + yield { type: 'message_delta', usage: { input_tokens: 1, output_tokens: 1 } }; + yield { type: 'message_stop' }; + }, + ]); + + const { agent, cleanup } = await createObservedAgent({ + provider, + template: { + id: 'obs-tool-agent', + systemPrompt: 'use tools', + tools: ['echo_tool'], + permission: { mode: 'auto' }, + }, + registerTools: (registry) => { + registry.register('echo_tool', () => ({ + name: 'echo_tool', + description: 'echo value', + input_schema: { type: 'object', properties: { value: { type: 'string' } }, required: ['value'] }, + async exec(args: any) { + return { echoed: args.value }; + }, + toDescriptor() { + return { source: 'registered' as const, name: 'echo_tool', registryId: 'echo_tool' }; + }, + })); + }, + }); + + try { + const result = await agent.chat('run tool'); + expect.toEqual(result.status, 'ok'); + expect.toContain(result.text || '', 'tool finished'); + + const snapshot = agent.getMetricsSnapshot(); + expect.toEqual(snapshot.totals.toolCalls, 1); + expect.toEqual(snapshot.totals.toolErrors, 0); + + const observations = await collectObservations(agent, 4); + expect.toContain(observations.map((o) => o.observation.kind), 'tool'); + } finally { + await cleanup(); + } + }) + .test('subagent observation 会记录 childRunId', async () => { + const parentProvider = new QueueStreamProvider([]); + const { agent, store, cleanup } = await createObservedAgent({ + provider: parentProvider, + template: { + id: 'obs-parent-agent', + systemPrompt: 'parent', + tools: [], + permission: { mode: 'auto' }, + }, + }); + + const templates = (agent as any).deps.templateRegistry as AgentTemplateRegistry; + templates.register({ id: 'obs-child-agent', systemPrompt: 'child' }); + + try { + const result = await agent.delegateTask({ + templateId: 'obs-child-agent', + prompt: 'child work', + model: new QueueStreamProvider([ + async function* () { + yield { type: 'content_block_start', index: 0, content_block: { type: 'text', text: '' } }; + yield { type: 'content_block_delta', index: 0, delta: { type: 'text_delta', text: 'child ok' } }; + yield { type: 'content_block_stop', index: 0 }; + yield { type: 'message_delta', usage: { input_tokens: 1, output_tokens: 1 } }; + yield { type: 'message_stop' }; + }, + ]), + }); + expect.toEqual(result.status, 'ok'); + + const observations = await collectObservations(agent, 1); + expect.toEqual(observations[0].observation.kind, 'subagent'); + expect.toBeTruthy(observations[0].observation.childRunId); + const childInfo = await store.loadInfo(observations[0].observation.childAgentId); + expect.toBeTruthy(childInfo); + expect.toEqual(childInfo?.metadata?.metadata?.__observationTraceId, undefined); + expect.toEqual(childInfo?.metadata?.metadata?.__observationParentSpanId, undefined); + } finally { + await cleanup(); + } + }) + .test('sink 失败不会影响主流程与 snapshot', async () => { + const provider = new QueueStreamProvider([ + async function* () { + yield { type: 'content_block_start', index: 0, content_block: { type: 'text', text: '' } }; + yield { type: 'content_block_delta', index: 0, delta: { type: 'text_delta', text: 'sink safe' } }; + yield { type: 'content_block_stop', index: 0 }; + yield { type: 'message_delta', usage: { input_tokens: 3, output_tokens: 4 } }; + yield { type: 'message_stop' }; + }, + ]); + + const { agent, cleanup } = await createObservedAgent({ + provider, + observability: { + sink: { + onObservation() { + throw new Error('sink failure'); + }, + }, + }, + }); + + try { + const result = await agent.chat('hello'); + expect.toEqual(result.status, 'ok'); + expect.toContain(result.text || '', 'sink safe'); + + const snapshot = agent.getMetricsSnapshot(); + expect.toEqual(snapshot.totals.totalTokens, 7); + expect.toEqual(snapshot.totals.generations, 1); + } finally { + await cleanup(); + } + }) + .test('enabled=false 时不记录 snapshot 与 observation', async () => { + const provider = new QueueStreamProvider([ + async function* () { + yield { type: 'content_block_start', index: 0, content_block: { type: 'text', text: '' } }; + yield { type: 'content_block_delta', index: 0, delta: { type: 'text_delta', text: 'disabled' } }; + yield { type: 'content_block_stop', index: 0 }; + yield { type: 'message_delta', usage: { input_tokens: 5, output_tokens: 6 } }; + yield { type: 'message_stop' }; + }, + ]); + + const { agent, cleanup } = await createObservedAgent({ + provider, + observability: { enabled: false }, + }); + + try { + const result = await agent.chat('hello'); + expect.toEqual(result.status, 'ok'); + + const snapshot = agent.getMetricsSnapshot(); + expect.toEqual(snapshot.totals.totalTokens, 0); + expect.toEqual(snapshot.totals.generations, 0); + + const observations = await collectObservations(agent, 1); + expect.toHaveLength(observations, 0); + } finally { + await cleanup(); + } + }); + +export async function run() { + return runner.run(); +} diff --git a/tests/unit/observability/http-example.test.ts b/tests/unit/observability/http-example.test.ts new file mode 100644 index 0000000..f11a494 --- /dev/null +++ b/tests/unit/observability/http-example.test.ts @@ -0,0 +1,240 @@ +import fs from 'node:fs'; +import path from 'node:path'; + +import { + Agent, + AgentTemplateRegistry, + JSONStore, + JSONStoreObservationBackend, + SandboxFactory, + ToolRegistry, + createStoreBackedObservationReader, + type ModelConfig, + type ModelProvider, + type ModelResponse, + type ModelStreamChunk, +} from '../../../src'; +import { createExampleObservabilityHttpHandler } from '../../../examples/shared/observability-http'; +import { TEST_ROOT } from '../../helpers/fixtures'; +import { ensureCleanDir, wait } from '../../helpers/setup'; +import { TestRunner, expect, retry } from '../../helpers/utils'; + +const runner = new TestRunner('Observability HTTP Example Skeleton'); + +class QueueStreamProvider implements ModelProvider { + readonly model = 'queue-stream-provider'; + readonly maxWindowSize = 128000; + readonly maxOutputTokens = 4096; + readonly temperature = 0; + + constructor(private readonly streams: Array<() => AsyncIterable>) {} + + async complete(): Promise { + throw new Error('complete() should not be called'); + } + + async *stream(): AsyncIterable { + const next = this.streams.shift(); + if (!next) { + throw new Error('No scripted stream available'); + } + yield* next(); + } + + toConfig(): ModelConfig { + return { + provider: 'mock', + model: this.model, + }; + } +} + +async function createObservedAgent(suffix: string) { + const workDir = path.join(TEST_ROOT, `obs-http-example-work-${suffix}`); + const storeDir = path.join(TEST_ROOT, `obs-http-example-store-${suffix}`); + ensureCleanDir(workDir); + ensureCleanDir(storeDir); + + const templates = new AgentTemplateRegistry(); + templates.register({ + id: 'obs-http-example-agent', + systemPrompt: 'observability http example test', + tools: [], + permission: { mode: 'auto' }, + }); + + const observationBackend = new JSONStoreObservationBackend(storeDir); + const agent = await Agent.create( + { + agentId: `agt-obs-http-example-${suffix}`, + templateId: 'obs-http-example-agent', + model: new QueueStreamProvider([ + async function* () { + yield { type: 'content_block_start', index: 0, content_block: { type: 'text', text: '' } }; + yield { type: 'content_block_delta', index: 0, delta: { type: 'text_delta', text: 'observability-example-ok' } }; + yield { type: 'content_block_stop', index: 0 }; + yield { type: 'message_delta', usage: { input_tokens: 5, output_tokens: 7 } }; + yield { type: 'message_stop' }; + }, + ]), + sandbox: { kind: 'local', workDir, enforceBoundary: true }, + observability: { + persistence: { + enabled: true, + backend: observationBackend, + }, + }, + }, + { + store: new JSONStore(storeDir), + templateRegistry: templates, + sandboxFactory: new SandboxFactory(), + toolRegistry: new ToolRegistry(), + } + ); + + return { + agent, + observationBackend, + cleanup: async () => { + await (agent as any).sandbox?.dispose?.(); + await wait(10); + fs.rmSync(workDir, { recursive: true, force: true }); + fs.rmSync(storeDir, { recursive: true, force: true }); + }, + }; +} + +runner + .test('handler exposes runtime metrics and runtime observations from the live agent instance', async () => { + const suffix = `${Date.now()}-${Math.random().toString(36).slice(2, 7)}`; + const { agent, observationBackend, cleanup } = await createObservedAgent(suffix); + + try { + const result = await agent.chat('hello observability'); + expect.toEqual(result.status, 'ok'); + + const persistedReader = createStoreBackedObservationReader(observationBackend); + const handler = createExampleObservabilityHttpHandler({ + basePath: '/api/observability', + resolveRuntimeSource: async (agentId) => + agentId === agent.agentId + ? { + getMetricsSnapshot: () => agent.getMetricsSnapshot(), + getObservationReader: () => agent.getObservationReader(), + } + : undefined, + resolvePersistedReader: async (agentId) => (agentId === agent.agentId ? persistedReader : undefined), + }); + + const metricsResponse = await handler({ + method: 'GET', + url: `/api/observability/agents/${agent.agentId}/metrics`, + }); + expect.toEqual(metricsResponse.status, 200); + const metricsBody = metricsResponse.body as any; + expect.toEqual(metricsBody.agentId, agent.agentId); + expect.toEqual(metricsBody.totals.generations, 1); + expect.toBeGreaterThan(metricsBody.totals.totalTokens, 0); + + const runtimeResponse = await retry(async () => { + const response = await handler({ + method: 'GET', + url: `/api/observability/agents/${agent.agentId}/observations/runtime?limit=20`, + }); + const body = response.body as any; + const kinds = Array.isArray(body?.observations) + ? body.observations.map((entry: any) => entry.observation.kind) + : []; + if (response.status !== 200 || !kinds.includes('generation') || !kinds.includes('agent_run')) { + throw new Error('runtime observations not ready'); + } + return response; + }, 10, 20); + expect.toEqual(runtimeResponse.status, 200); + const runtimeBody = runtimeResponse.body as any; + const runtimeKinds = runtimeBody.observations.map((entry: any) => entry.observation.kind); + expect.toContain(runtimeKinds, 'generation'); + expect.toContain(runtimeKinds, 'agent_run'); + + const generationEnvelope = runtimeBody.observations.find((entry: any) => entry.observation.kind === 'generation'); + expect.toBeTruthy(generationEnvelope); + + const runtimeRunResponse = await handler({ + method: 'GET', + url: `/api/observability/agents/${agent.agentId}/observations/runtime/runs/${generationEnvelope.observation.runId}`, + }); + expect.toEqual(runtimeRunResponse.status, 200); + const runtimeRunBody = runtimeRunResponse.body as any; + expect.toEqual(runtimeRunBody.run.observation.kind, 'agent_run'); + expect.toContain( + runtimeRunBody.observations.map((entry: any) => entry.observation.kind), + 'generation' + ); + } finally { + await cleanup(); + } + }) + .test('handler exposes persisted observations and rejects invalid app-layer requests', async () => { + const suffix = `${Date.now()}-${Math.random().toString(36).slice(2, 7)}`; + const { agent, observationBackend, cleanup } = await createObservedAgent(suffix); + + try { + await agent.chat('hello persisted'); + + const persistedReader = createStoreBackedObservationReader(observationBackend); + const handler = createExampleObservabilityHttpHandler({ + basePath: '/api/observability', + resolveRuntimeSource: async (agentId) => + agentId === agent.agentId + ? { + getMetricsSnapshot: () => agent.getMetricsSnapshot(), + getObservationReader: () => agent.getObservationReader(), + } + : undefined, + resolvePersistedReader: async (agentId) => (agentId === agent.agentId ? persistedReader : undefined), + }); + + const persistedResponse = await retry(async () => { + const response = await handler({ + method: 'GET', + url: `/api/observability/agents/${agent.agentId}/observations/persisted?limit=20`, + }); + const body = response.body as any; + if (response.status !== 200 || !Array.isArray(body.observations) || body.observations.length < 2) { + throw new Error('persisted observations not ready'); + } + return response; + }, 10, 20); + + expect.toEqual(persistedResponse.status, 200); + const persistedBody = persistedResponse.body as any; + const persistedKinds = persistedBody.observations.map((entry: any) => entry.observation.kind); + expect.toContain(persistedKinds, 'generation'); + expect.toContain(persistedKinds, 'agent_run'); + + const methodResponse = await handler({ + method: 'POST', + url: `/api/observability/agents/${agent.agentId}/observations/runtime`, + }); + expect.toEqual(methodResponse.status, 405); + + const invalidQueryResponse = await handler({ + method: 'GET', + url: `/api/observability/agents/${agent.agentId}/observations/persisted?limit=0`, + }); + expect.toEqual(invalidQueryResponse.status, 400); + + const missingAgentResponse = await handler({ + method: 'GET', + url: '/api/observability/agents/missing-agent/metrics', + }); + expect.toEqual(missingAgentResponse.status, 404); + } finally { + await cleanup(); + } + }); + +export async function run() { + return runner.run(); +} diff --git a/tests/unit/observability/memory-store.test.ts b/tests/unit/observability/memory-store.test.ts new file mode 100644 index 0000000..38d9453 --- /dev/null +++ b/tests/unit/observability/memory-store.test.ts @@ -0,0 +1,213 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import { + Agent, + AgentTemplateRegistry, + JSONStore, + MemoryObservationStore, + SandboxFactory, + ToolRegistry, +} from '../../../src'; +import { ModelConfig, ModelProvider, ModelResponse, ModelStreamChunk } from '../../../src/infra/provider'; +import { TestRunner, expect } from '../../helpers/utils'; +import { TEST_ROOT } from '../../helpers/fixtures'; +import { ensureCleanDir, wait } from '../../helpers/setup'; + +const runner = new TestRunner('Observability Memory Store'); + +class QueueStreamProvider implements ModelProvider { + readonly model = 'queue-stream-provider'; + readonly maxWindowSize = 128000; + readonly maxOutputTokens = 4096; + readonly temperature = 0; + + constructor( + private readonly streams: Array<() => AsyncIterable>, + private readonly providerName = 'mock' + ) {} + + async complete(): Promise { + throw new Error('complete() should not be called'); + } + + async *stream(): AsyncIterable { + const next = this.streams.shift(); + if (!next) { + throw new Error('No scripted stream available'); + } + yield* next(); + } + + toConfig(): ModelConfig { + return { + provider: this.providerName, + model: this.model, + }; + } +} + +function createTextStream(text: string): () => AsyncIterable { + return async function* () { + yield { type: 'content_block_start', index: 0, content_block: { type: 'text', text: '' } }; + yield { type: 'content_block_delta', index: 0, delta: { type: 'text_delta', text } }; + yield { type: 'content_block_stop', index: 0 }; + yield { type: 'message_delta', usage: { input_tokens: 1, output_tokens: 1 } }; + yield { type: 'message_stop' }; + }; +} + +async function createObservedAgent(params: { + id: string; + provider: ModelProvider; + store: MemoryObservationStore; +}) { + const workDir = path.join(TEST_ROOT, `obs-memory-work-${params.id}-${Date.now()}-${Math.random().toString(36).slice(2, 7)}`); + const storeDir = path.join(TEST_ROOT, `obs-memory-store-${params.id}-${Date.now()}-${Math.random().toString(36).slice(2, 7)}`); + ensureCleanDir(workDir); + ensureCleanDir(storeDir); + + const templates = new AgentTemplateRegistry(); + templates.register({ + id: params.id, + systemPrompt: 'memory store test', + tools: [], + permission: { mode: 'auto' }, + }); + + const agent = await Agent.create( + { + templateId: params.id, + model: params.provider, + sandbox: { kind: 'local', workDir, enforceBoundary: true }, + observability: { sink: params.store }, + }, + { + store: new JSONStore(storeDir), + templateRegistry: templates, + sandboxFactory: new SandboxFactory(), + toolRegistry: new ToolRegistry(), + } + ); + + return { + agent, + cleanup: async () => { + await (agent as any).sandbox?.dispose?.(); + fs.rmSync(workDir, { recursive: true, force: true }); + fs.rmSync(storeDir, { recursive: true, force: true }); + }, + }; +} + +runner + .test('MemoryObservationStore 支持跨 agent 聚合与过滤', async () => { + const store = new MemoryObservationStore(); + const first = await createObservedAgent({ + id: 'obs-memory-agent-a', + provider: new QueueStreamProvider([createTextStream('agent a')]), + store, + }); + const second = await createObservedAgent({ + id: 'obs-memory-agent-b', + provider: new QueueStreamProvider([createTextStream('agent b')]), + store, + }); + + try { + await first.agent.chat('hello a'); + await second.agent.chat('hello b'); + + for (let i = 0; i < 20; i++) { + if (store.listObservations().length >= 4) { + break; + } + await wait(10); + } + + const all = store.listObservations(); + expect.toHaveLength(all, 4); + expect.toEqual(all[0].seq, 0); + expect.toEqual(all[3].seq, 3); + + const agentA = store.listObservations({ agentId: first.agent.agentId }); + expect.toHaveLength(agentA, 2); + + const runId = agentA[1].observation.runId; + const byRun = store.listObservations({ runId }); + expect.toHaveLength(byRun, 2); + + const traceId = byRun[0].observation.traceId; + const byTrace = store.listObservations({ traceId }); + expect.toHaveLength(byTrace, 2); + + const runView = store.getRun(runId); + expect.toBeTruthy(runView); + expect.toEqual(runView?.run.observation.kind, 'agent_run'); + expect.toHaveLength(runView?.observations || [], 2); + } finally { + await first.cleanup(); + await second.cleanup(); + } + }) + .test('MemoryObservationStore.subscribe 支持 store 级 sinceSeq 补读', async () => { + const store = new MemoryObservationStore(); + const agent = await createObservedAgent({ + id: 'obs-memory-replay', + provider: new QueueStreamProvider([createTextStream('memory replay')]), + store, + }); + + try { + await agent.agent.chat('hello'); + + for (let i = 0; i < 20; i++) { + if (store.listObservations().length >= 2) { + break; + } + await wait(10); + } + + const existing = store.listObservations(); + expect.toHaveLength(existing, 2); + + const replayed: any[] = []; + for await (const envelope of store.subscribe({ sinceSeq: existing[0].seq })) { + replayed.push(envelope); + break; + } + + expect.toHaveLength(replayed, 1); + expect.toEqual(replayed[0].seq, existing[1].seq); + expect.toEqual(replayed[0].observation.kind, 'agent_run'); + } finally { + await agent.cleanup(); + } + }) + .test('MemoryObservationStore 超过上限时淘汰最旧数据', async () => { + const store = new MemoryObservationStore({ maxEntries: 2 }); + const baseObservation = { + kind: 'generation' as const, + agentId: 'agent-x', + runId: 'run-x', + traceId: 'trace-x', + spanId: 'span-x', + name: 'generation:test', + status: 'ok' as const, + startTime: 1, + endTime: 2, + durationMs: 1, + }; + + store.onObservation({ seq: 0, timestamp: 1, observation: { ...baseObservation, spanId: 'span-1' } }); + store.onObservation({ seq: 0, timestamp: 2, observation: { ...baseObservation, spanId: 'span-2' } }); + store.onObservation({ seq: 0, timestamp: 3, observation: { ...baseObservation, spanId: 'span-3' } }); + + const all = store.listObservations(); + expect.toHaveLength(all, 2); + expect.toEqual(all[0].observation.spanId, 'span-2'); + expect.toEqual(all[1].observation.spanId, 'span-3'); + }); + +export async function run() { + return runner.run(); +} diff --git a/tests/unit/observability/otel/exporter.test.ts b/tests/unit/observability/otel/exporter.test.ts new file mode 100644 index 0000000..09d820f --- /dev/null +++ b/tests/unit/observability/otel/exporter.test.ts @@ -0,0 +1,95 @@ +import { + OTLPHttpJsonExporter, + buildOTLPTraceExportBody, + type OTelSpanData, +} from '../../../../src'; +import { TestRunner, expect } from '../../../helpers/utils'; + +const runner = new TestRunner('Observability OTel Exporter'); + +function createSpan(): OTelSpanData { + return { + traceId: '0123456789abcdef0123456789abcdef', + spanId: '0123456789abcdef', + parentSpanId: 'fedcba9876543210', + name: 'agent.run', + kind: 'internal', + startTime: 1710000000000, + endTime: 1710000000500, + status: 'ok', + attributes: { + 'kode.step': 2, + 'kode.duration.ms': 500, + 'kode.agent.id': 'agent-1', + }, + events: [ + { + name: 'kode.observation', + timestamp: 1710000000500, + attributes: { + 'kode.observation.kind': 'agent_run', + }, + }, + ], + }; +} + +runner + .test('buildOTLPTraceExportBody 会生成 OTLP JSON payload', async () => { + const body = buildOTLPTraceExportBody([createSpan()], 'kode-test-service') as any; + const resourceSpan = body.resourceSpans[0]; + const span = resourceSpan.scopeSpans[0].spans[0]; + + expect.toEqual(resourceSpan.resource.attributes[0].key, 'service.name'); + expect.toEqual(resourceSpan.resource.attributes[0].value.stringValue, 'kode-test-service'); + expect.toEqual(span.name, 'agent.run'); + expect.toEqual(span.kind, 1); + expect.toEqual(span.attributes[0].key, 'kode.step'); + expect.toEqual(span.attributes[0].value.intValue, '2'); + expect.toEqual(span.events[0].name, 'kode.observation'); + }) + .test('OTLPHttpJsonExporter 会发送 JSON 并在失败时抛错', async () => { + const requests: Array<{ url: string; init: any }> = []; + const exporter = new OTLPHttpJsonExporter({ + endpoint: 'https://otel.example/v1/traces', + headers: { authorization: 'Bearer token' }, + serviceName: 'kode-exporter-test', + fetch: async (url, init) => { + requests.push({ url, init }); + return { + ok: true, + status: 200, + async text() { + return 'ok'; + }, + }; + }, + }); + + await exporter.export([createSpan()]); + + expect.toHaveLength(requests, 1); + expect.toEqual(requests[0].url, 'https://otel.example/v1/traces'); + expect.toEqual(requests[0].init.method, 'POST'); + expect.toEqual(requests[0].init.headers.authorization, 'Bearer token'); + expect.toContain(requests[0].init.body, 'kode-exporter-test'); + + const failingExporter = new OTLPHttpJsonExporter({ + endpoint: 'https://otel.example/v1/traces', + fetch: async () => ({ + ok: false, + status: 500, + async text() { + return 'boom'; + }, + }), + }); + + await expect.toThrow(async () => { + await failingExporter.export([createSpan()]); + }, 'OTLP export failed with status 500: boom'); + }); + +export async function run() { + return runner.run(); +} diff --git a/tests/unit/observability/otel/policy.test.ts b/tests/unit/observability/otel/policy.test.ts new file mode 100644 index 0000000..2dccd7f --- /dev/null +++ b/tests/unit/observability/otel/policy.test.ts @@ -0,0 +1,126 @@ +import { + applyOTelPolicies, + maskOTelSpan, + shouldExportOTelSpan, + shouldSampleOTelTrace, + type ObservationEnvelope, + type ToolObservation, + type OTelSpanData, +} from '../../../../src'; +import { TestRunner, expect } from '../../../helpers/utils'; + +const runner = new TestRunner('Observability OTel Policy'); + +function createEnvelope(overrides?: Partial): ObservationEnvelope { + return { + seq: 1, + timestamp: 1710000000000, + observation: { + kind: 'tool', + agentId: 'agent-1', + runId: 'run-1', + traceId: 'trace-1', + spanId: 'span-1', + parentSpanId: 'parent-1', + name: 'tool:fs_read', + status: 'error', + startTime: 1710000000000, + endTime: 1710000000100, + durationMs: 100, + toolCallId: 'call-1', + toolName: 'fs_read', + toolState: 'FAILED', + approvalRequired: false, + errorMessage: 'Bearer secret-token', + ...overrides, + }, + }; +} + +function createSpan(): OTelSpanData { + return { + traceId: '0123456789abcdef0123456789abcdef', + spanId: '0123456789abcdef', + parentSpanId: 'fedcba9876543210', + name: 'tool.call', + kind: 'client', + startTime: 1710000000000, + endTime: 1710000000100, + status: 'error', + attributes: { + authorization: 'Bearer secret-token', + api_key: 'api_key=secret', + normal: 'keep-me', + }, + events: [ + { + name: 'exception', + timestamp: 1710000000100, + attributes: { + details: 'sk-secret-value', + }, + }, + ], + }; +} + +runner + .test('filtering policy 可按 kind 和 status 拦截导出', async () => { + const envelope = createEnvelope(); + const span = createSpan(); + + expect.toEqual( + shouldExportOTelSpan(envelope, span, { + filtering: { + kinds: ['tool'], + statuses: ['error'], + }, + }), + true + ); + expect.toEqual( + shouldExportOTelSpan(envelope, span, { + filtering: { + kinds: ['generation'], + }, + }), + false + ); + }) + .test('sampling policy 对同一 trace 保持稳定且支持边界值', async () => { + expect.toEqual(shouldSampleOTelTrace('trace-a', { strategy: 'always_on' }), true); + expect.toEqual(shouldSampleOTelTrace('trace-a', { strategy: 'always_off' }), false); + expect.toEqual(shouldSampleOTelTrace('trace-a', { strategy: 'trace_ratio', ratio: 0 }), false); + expect.toEqual(shouldSampleOTelTrace('trace-a', { strategy: 'trace_ratio', ratio: 1 }), true); + + const first = shouldSampleOTelTrace('trace-stable', { strategy: 'trace_ratio', ratio: 0.5 }); + const second = shouldSampleOTelTrace('trace-stable', { strategy: 'trace_ratio', ratio: 0.5 }); + expect.toEqual(first, second); + }) + .test('masking policy 会在 attributes 与 events 上执行脱敏', async () => { + const masked = maskOTelSpan(createSpan(), { + masking: { + mask: ({ key, value }) => (key === 'normal' ? 'custom-mask' : value), + }, + }); + + expect.toEqual(masked.attributes.authorization, 'Bearer ***'); + expect.toEqual(masked.attributes.api_key, 'api_key=***'); + expect.toEqual(masked.attributes.normal, 'custom-mask'); + expect.toEqual(masked.events?.[0].attributes?.details, 'sk-***'); + + const applied = applyOTelPolicies(createEnvelope(), createSpan(), { + filtering: { kinds: ['tool'] }, + sampling: { strategy: 'always_on' }, + }); + expect.toBeTruthy(applied); + + const dropped = applyOTelPolicies(createEnvelope(), createSpan(), { + sampling: { strategy: 'always_off' }, + }); + expect.toEqual(dropped, undefined); + }); + +export async function run() { + return runner.run(); +} diff --git a/tests/unit/observability/otel/sink.test.ts b/tests/unit/observability/otel/sink.test.ts new file mode 100644 index 0000000..4036ac8 --- /dev/null +++ b/tests/unit/observability/otel/sink.test.ts @@ -0,0 +1,223 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import { + Agent, + AgentTemplateRegistry, + JSONStore, + MemoryObservationStore, + OTelObservationSink, + SandboxFactory, + ToolRegistry, + type ModelConfig, + type ModelProvider, + type ModelResponse, + type ModelStreamChunk, + type ObservationEnvelope, + type OTelSpanData, + type OTelSpanExporter, +} from '../../../../src'; +import { TEST_ROOT } from '../../../helpers/fixtures'; +import { ensureCleanDir, wait } from '../../../helpers/setup'; +import { TestRunner, expect } from '../../../helpers/utils'; + +const runner = new TestRunner('Observability OTel Sink'); + +class QueueStreamProvider implements ModelProvider { + readonly model = 'queue-stream-provider'; + readonly maxWindowSize = 128000; + readonly maxOutputTokens = 4096; + readonly temperature = 0; + + constructor( + private readonly streams: Array<() => AsyncIterable>, + private readonly providerName = 'mock' + ) {} + + async complete(): Promise { + throw new Error('complete() should not be called'); + } + + async *stream(): AsyncIterable { + const next = this.streams.shift(); + if (!next) { + throw new Error('No scripted stream available'); + } + yield* next(); + } + + toConfig(): ModelConfig { + return { + provider: this.providerName, + model: this.model, + }; + } +} + +class RecordingExporter implements OTelSpanExporter { + readonly batches: OTelSpanData[][] = []; + shutdownCalls = 0; + + async export(spans: OTelSpanData[]): Promise { + this.batches.push(spans); + } + + async shutdown(): Promise { + this.shutdownCalls += 1; + } +} + +function createGenerationEnvelope(seq: number): ObservationEnvelope { + return { + seq, + timestamp: 1710000000000 + seq, + observation: { + kind: 'generation', + agentId: 'agent-1', + runId: 'run-1', + traceId: 'trace-1', + spanId: 'span-' + seq, + parentSpanId: 'parent-1', + name: 'generation:gpt-4.1', + status: 'ok', + startTime: 1710000000000 + seq, + endTime: 1710000000050 + seq, + durationMs: 50, + provider: 'openai', + model: 'gpt-4.1', + usage: { + inputTokens: 2, + outputTokens: 3, + totalTokens: 5, + }, + request: { + latencyMs: 50, + }, + }, + }; +} + +async function createObservedAgent(params: { + provider: ModelProvider; + memoryStore: MemoryObservationStore; + exporter: OTelSpanExporter; +}) { + const suffix = Date.now() + '-' + Math.random().toString(36).slice(2, 7); + const workDir = path.join(TEST_ROOT, 'obs-otel-work-' + suffix); + const storeDir = path.join(TEST_ROOT, 'obs-otel-store-' + suffix); + ensureCleanDir(workDir); + ensureCleanDir(storeDir); + + const templates = new AgentTemplateRegistry(); + templates.register({ + id: 'obs-otel-agent', + systemPrompt: 'otel sink test', + tools: [], + permission: { mode: 'auto' }, + }); + + const agent = await Agent.create( + { + templateId: 'obs-otel-agent', + model: params.provider, + sandbox: { kind: 'local', workDir, enforceBoundary: true }, + observability: { + sink: params.memoryStore, + otel: { + exporter: params.exporter, + attributeNamespace: 'dual', + }, + }, + }, + { + store: new JSONStore(storeDir), + templateRegistry: templates, + sandboxFactory: new SandboxFactory(), + toolRegistry: new ToolRegistry(), + } + ); + + return { + agent, + cleanup: async () => { + await (agent as any).sandbox?.dispose?.(); + fs.rmSync(workDir, { recursive: true, force: true }); + fs.rmSync(storeDir, { recursive: true, force: true }); + }, + }; +} + +runner + .test('OTelObservationSink supports batched flush', async () => { + const exporter = new RecordingExporter(); + const sink = new OTelObservationSink({ + exporter, + exportMode: 'batched', + batchSize: 2, + flushIntervalMs: 1000, + }); + + await sink.onObservation(createGenerationEnvelope(1)); + expect.toHaveLength(exporter.batches, 0); + + await sink.onObservation(createGenerationEnvelope(2)); + expect.toHaveLength(exporter.batches, 1); + expect.toHaveLength(exporter.batches[0], 2); + + await sink.onObservation(createGenerationEnvelope(3)); + expect.toHaveLength(exporter.batches, 1); + + await sink.forceFlush(); + expect.toHaveLength(exporter.batches, 2); + expect.toHaveLength(exporter.batches[1], 1); + + await sink.shutdown(); + expect.toEqual(exporter.shutdownCalls, 1); + }) + .test('Agent observability can fan out to native sink and OTel bridge sink', async () => { + const memoryStore = new MemoryObservationStore(); + const exporter = new RecordingExporter(); + const provider = new QueueStreamProvider([ + async function* () { + yield { type: 'content_block_start', index: 0, content_block: { type: 'text', text: '' } }; + yield { type: 'content_block_delta', index: 0, delta: { type: 'text_delta', text: 'otel bridge ok' } }; + yield { type: 'content_block_stop', index: 0 }; + yield { type: 'message_delta', usage: { input_tokens: 2, output_tokens: 3 } }; + yield { type: 'message_stop' }; + }, + ]); + + const { agent, cleanup } = await createObservedAgent({ + provider, + memoryStore, + exporter, + }); + + try { + const result = await agent.chat('hello'); + expect.toEqual(result.status, 'ok'); + + for (let i = 0; i < 20; i++) { + if (memoryStore.listObservations().length >= 2 && exporter.batches.length >= 2) { + break; + } + await wait(10); + } + + const observations = memoryStore.listObservations(); + expect.toHaveLength(observations, 2); + expect.toEqual(observations[0].observation.kind, 'generation'); + expect.toEqual(observations[1].observation.kind, 'agent_run'); + + const exportedSpans = exporter.batches.flat(); + expect.toHaveLength(exportedSpans, 2); + expect.toEqual(exportedSpans[0].name, 'llm.generation'); + expect.toEqual(exportedSpans[1].name, 'agent.run'); + expect.toEqual(exportedSpans[0].attributes['gen_ai.system'], 'mock'); + } finally { + await cleanup(); + } + }); + +export async function run() { + return runner.run(); +} diff --git a/tests/unit/observability/otel/translator.test.ts b/tests/unit/observability/otel/translator.test.ts new file mode 100644 index 0000000..ce7ae01 --- /dev/null +++ b/tests/unit/observability/otel/translator.test.ts @@ -0,0 +1,84 @@ +import { + translateObservationToOTelSpan, + type GenerationObservation, + type ObservationEnvelope, +} from '../../../../src'; +import { TestRunner, expect } from '../../../helpers/utils'; + +const runner = new TestRunner('Observability OTel Translator'); + +function createGenerationEnvelope(overrides?: Partial): ObservationEnvelope { + return { + seq: 7, + timestamp: 1710000000123, + observation: { + kind: 'generation', + agentId: 'agent-1', + runId: 'run-1', + traceId: 'trace-1', + spanId: 'span-1', + parentSpanId: 'parent-1', + name: 'generation:gpt-4.1', + status: 'ok', + startTime: 1710000000000, + endTime: 1710000000100, + durationMs: 100, + provider: 'openai', + model: 'gpt-4.1', + requestId: 'req-1', + usage: { + inputTokens: 10, + outputTokens: 5, + totalTokens: 15, + }, + request: { + latencyMs: 100, + timeToFirstTokenMs: 25, + stopReason: 'end_turn', + }, + metadata: { + templateId: 'template-1', + }, + ...overrides, + }, + }; +} + +runner + .test('generation observation 可映射为带 dual attributes 的 OTEL span', async () => { + const envelope = createGenerationEnvelope(); + const span = translateObservationToOTelSpan(envelope, { attributeNamespace: 'dual' }); + + expect.toEqual(span.name, 'llm.generation'); + expect.toEqual(span.kind, 'client'); + expect.toEqual(span.traceId.length, 32); + expect.toEqual(span.spanId.length, 16); + expect.toEqual(span.parentSpanId?.length, 16); + expect.toEqual(span.attributes['kode.agent.id'], 'agent-1'); + expect.toEqual(span.attributes['kode.generation.model'], 'gpt-4.1'); + expect.toEqual(span.attributes['gen_ai.system'], 'openai'); + expect.toEqual(span.attributes['gen_ai.request.model'], 'gpt-4.1'); + expect.toEqual(span.attributes['gen_ai.kode.agent_id'], 'agent-1'); + expect.toEqual(span.attributes['kode.duration.ms'], 100); + expect.toHaveLength(span.events || [], 1); + expect.toEqual(span.events?.[0].name, 'kode.observation'); + }) + .test('error observation 会生成 exception event', async () => { + const envelope = createGenerationEnvelope({ + status: 'error', + errorMessage: 'provider failed', + }); + + const span = translateObservationToOTelSpan(envelope, { attributeNamespace: 'kode' }); + + expect.toEqual(span.status, 'error'); + expect.toEqual(span.attributes['kode.error.message'], 'provider failed'); + expect.toEqual(span.attributes['gen_ai.system'], undefined); + expect.toHaveLength(span.events || [], 2); + expect.toEqual(span.events?.[1].name, 'exception'); + expect.toEqual(span.events?.[1].attributes?.['exception.message'], 'provider failed'); + }); + +export async function run() { + return runner.run(); +} diff --git a/tests/unit/observability/persistence/agent-integration.test.ts b/tests/unit/observability/persistence/agent-integration.test.ts new file mode 100644 index 0000000..ccdcef2 --- /dev/null +++ b/tests/unit/observability/persistence/agent-integration.test.ts @@ -0,0 +1,197 @@ +import fs from 'node:fs'; +import path from 'node:path'; + +import { + Agent, + AgentTemplateRegistry, + JSONStore, + JSONStoreObservationBackend, + MemoryObservationStore, + SandboxFactory, + ToolRegistry, + createStoreBackedObservationReader, + type ModelConfig, + type ModelProvider, + type ModelResponse, + type ModelStreamChunk, + type OTelSpanData, + type OTelSpanExporter, +} from '../../../../src'; +import { TEST_ROOT } from '../../../helpers/fixtures'; +import { ensureCleanDir, wait } from '../../../helpers/setup'; +import { TestRunner, expect } from '../../../helpers/utils'; + +const runner = new TestRunner('Observability Persistence Agent Integration'); + +class QueueStreamProvider implements ModelProvider { + readonly model = 'queue-stream-provider'; + readonly maxWindowSize = 128000; + readonly maxOutputTokens = 4096; + readonly temperature = 0; + + constructor(private readonly streams: Array<() => AsyncIterable>) {} + + async complete(): Promise { + throw new Error('complete() should not be called'); + } + + async *stream(): AsyncIterable { + const next = this.streams.shift(); + if (!next) { + throw new Error('No scripted stream available'); + } + yield* next(); + } + + toConfig(): ModelConfig { + return { + provider: 'mock', + model: this.model, + }; + } +} + +class RecordingExporter implements OTelSpanExporter { + readonly batches: OTelSpanData[][] = []; + + async export(spans: OTelSpanData[]): Promise { + this.batches.push(spans); + } +} + +async function createObservedAgent(params: { + storeDir: string; + workDir: string; + backend: JSONStoreObservationBackend; + provider: ModelProvider; + memoryStore?: MemoryObservationStore; + exporter?: RecordingExporter; +}) { + const templates = new AgentTemplateRegistry(); + templates.register({ + id: 'obs-persist-agent', + systemPrompt: 'persistence integration test', + tools: [], + permission: { mode: 'auto' }, + }); + + return Agent.create( + { + templateId: 'obs-persist-agent', + model: params.provider, + sandbox: { kind: 'local', workDir: params.workDir, enforceBoundary: true }, + observability: { + sink: params.memoryStore, + otel: params.exporter ? { exporter: params.exporter, attributeNamespace: 'dual' } : undefined, + persistence: { + backend: params.backend, + enabled: true, + }, + }, + }, + { + store: new JSONStore(params.storeDir), + templateRegistry: templates, + sandboxFactory: new SandboxFactory(), + toolRegistry: new ToolRegistry(), + } + ); +} + +runner + .test('persisted observation backend survives agent restart-style recreation', async () => { + const suffix = `${Date.now()}-${Math.random().toString(36).slice(2, 7)}`; + const workDir = path.join(TEST_ROOT, `obs-persist-work-${suffix}`); + const storeDir = path.join(TEST_ROOT, `obs-persist-store-${suffix}`); + ensureCleanDir(workDir); + ensureCleanDir(storeDir); + + try { + const backend = new JSONStoreObservationBackend(storeDir); + const agent = await createObservedAgent({ + storeDir, + workDir, + backend, + provider: new QueueStreamProvider([ + async function* () { + yield { type: 'content_block_start', index: 0, content_block: { type: 'text', text: '' } }; + yield { type: 'content_block_delta', index: 0, delta: { type: 'text_delta', text: 'persist me' } }; + yield { type: 'content_block_stop', index: 0 }; + yield { type: 'message_delta', usage: { input_tokens: 2, output_tokens: 3 } }; + yield { type: 'message_stop' }; + }, + ]), + }); + + const result = await agent.chat('hello'); + expect.toEqual(result.status, 'ok'); + await wait(20); + await (agent as any).sandbox?.dispose?.(); + + const reader = createStoreBackedObservationReader(new JSONStoreObservationBackend(storeDir)); + const observations = await reader.listObservations(); + expect.toHaveLength(observations, 2); + expect.toEqual(observations[0].observation.kind, 'generation'); + expect.toEqual(observations[1].observation.kind, 'agent_run'); + } finally { + fs.rmSync(workDir, { recursive: true, force: true }); + fs.rmSync(storeDir, { recursive: true, force: true }); + } + }) + .test('persistence can coexist with native sink and otel sink', async () => { + const suffix = `${Date.now()}-${Math.random().toString(36).slice(2, 7)}`; + const workDir = path.join(TEST_ROOT, `obs-persist-work-${suffix}`); + const storeDir = path.join(TEST_ROOT, `obs-persist-store-${suffix}`); + ensureCleanDir(workDir); + ensureCleanDir(storeDir); + + try { + const memoryStore = new MemoryObservationStore(); + const exporter = new RecordingExporter(); + const backend = new JSONStoreObservationBackend(storeDir); + const agent = await createObservedAgent({ + storeDir, + workDir, + backend, + memoryStore, + exporter, + provider: new QueueStreamProvider([ + async function* () { + yield { type: 'content_block_start', index: 0, content_block: { type: 'text', text: '' } }; + yield { type: 'content_block_delta', index: 0, delta: { type: 'text_delta', text: 'fanout ok' } }; + yield { type: 'content_block_stop', index: 0 }; + yield { type: 'message_delta', usage: { input_tokens: 1, output_tokens: 1 } }; + yield { type: 'message_stop' }; + }, + ]), + }); + + await agent.chat('hello'); + + let persisted: any[] = []; + for (let i = 0; i < 20; i++) { + persisted = await createStoreBackedObservationReader(backend).listObservations(); + if ( + memoryStore.listObservations().length >= 2 && + exporter.batches.flat().length >= 2 && + persisted.length >= 2 + ) { + break; + } + await wait(10); + } + + expect.toHaveLength(memoryStore.listObservations(), 2); + expect.toHaveLength(exporter.batches.flat(), 2); + expect.toHaveLength(persisted, 2); + + await (agent as any).sandbox?.dispose?.(); + } finally { + fs.rmSync(workDir, { recursive: true, force: true }); + fs.rmSync(storeDir, { recursive: true, force: true }); + } + }); + +export async function run() { + return runner.run(); +} diff --git a/tests/unit/observability/persistence/backend-contract.test.ts b/tests/unit/observability/persistence/backend-contract.test.ts new file mode 100644 index 0000000..450678a --- /dev/null +++ b/tests/unit/observability/persistence/backend-contract.test.ts @@ -0,0 +1,112 @@ +import fs from 'node:fs'; +import path from 'node:path'; + +import { + JSONStoreObservationBackend, + type ObservationEnvelope, + type ObservationRecord, +} from '../../../../src'; +import { TEST_ROOT } from '../../../helpers/fixtures'; +import { ensureCleanDir } from '../../../helpers/setup'; +import { TestRunner, expect } from '../../../helpers/utils'; + +const runner = new TestRunner('Observability Persistence Backend Contract'); + +function makeEnvelope( + seq: number, + observation: ObservationRecord, + timestamp = 1_710_000_000_000 + seq * 10 +): ObservationEnvelope { + return { + seq, + timestamp, + observation, + }; +} + +function makeAgentRun(agentId: string, runId: string, templateId: string, seq: number): ObservationEnvelope { + return makeEnvelope(seq, { + kind: 'agent_run', + agentId, + runId, + traceId: `trace-${runId}`, + spanId: `span-run-${seq}`, + name: 'agent.run', + status: 'ok', + startTime: 1_710_000_000_000 + seq * 10, + endTime: 1_710_000_000_010 + seq * 10, + durationMs: 10, + trigger: 'send', + step: seq, + messageCountBefore: seq, + metadata: { templateId }, + }); +} + +function makeGeneration(agentId: string, runId: string, seq: number): ObservationEnvelope { + return makeEnvelope(seq, { + kind: 'generation', + agentId, + runId, + traceId: `trace-${runId}`, + spanId: `span-gen-${seq}`, + parentSpanId: `span-run-${seq - 1}`, + name: 'generation:gpt', + status: 'ok', + startTime: 1_710_000_000_000 + seq * 10, + endTime: 1_710_000_000_015 + seq * 10, + durationMs: 15, + provider: 'mock', + model: 'gpt', + usage: { + inputTokens: 2, + outputTokens: 3, + totalTokens: 5, + }, + metadata: { templateId: 'template-a' }, + }); +} + +runner.test('JSONStoreObservationBackend supports append/list/getRun filters', async () => { + const dir = path.join(TEST_ROOT, `obs-persist-backend-${Date.now()}-${Math.random().toString(36).slice(2, 7)}`); + ensureCleanDir(dir); + const backend = new JSONStoreObservationBackend(dir); + + try { + await backend.append(makeAgentRun('agent-a', 'run-a', 'template-a', 0)); + await backend.append(makeGeneration('agent-a', 'run-a', 1)); + await backend.append(makeAgentRun('agent-b', 'run-b', 'template-b', 0)); + + const all = await backend.list(); + expect.toHaveLength(all, 3); + expect.toEqual(all[0].observation.agentId, 'agent-a'); + expect.toEqual(all[1].observation.agentId, 'agent-b'); + expect.toEqual(all[2].observation.kind, 'generation'); + + const byRun = await backend.list({ runId: 'run-a' }); + expect.toHaveLength(byRun, 2); + + const byKinds = await backend.list({ kinds: ['generation'] }); + expect.toHaveLength(byKinds, 1); + expect.toEqual(byKinds[0].observation.kind, 'generation'); + + const byTemplate = await backend.list({ templateIds: ['template-b'] }); + expect.toHaveLength(byTemplate, 1); + expect.toEqual(byTemplate[0].observation.runId, 'run-b'); + + const byAgents = await backend.list({ agentIds: ['agent-b'] }); + expect.toHaveLength(byAgents, 1); + expect.toEqual(byAgents[0].observation.agentId, 'agent-b'); + + const runView = await backend.getRun('run-a'); + expect.toBeTruthy(runView); + expect.toEqual(runView?.run.observation.kind, 'agent_run'); + expect.toHaveLength(runView?.observations || [], 2); + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } +}); + +export async function run() { + return runner.run(); +} diff --git a/tests/unit/observability/persistence/postgres-backend.test.ts b/tests/unit/observability/persistence/postgres-backend.test.ts new file mode 100644 index 0000000..c0d48b9 --- /dev/null +++ b/tests/unit/observability/persistence/postgres-backend.test.ts @@ -0,0 +1,98 @@ +import path from 'node:path'; + +import { + PostgresStore, + PostgresStoreObservationBackend, +} from '../../../../src'; +import { TestRunner, expect } from '../../../helpers/utils'; + +const runner = new TestRunner('Observability Persistence Postgres Backend'); + +const TEST_STORE_DIR = path.join(__dirname, '../../../.tmp/postgres-observation-backend'); +const PG_CONFIG = { + host: process.env.POSTGRES_HOST || 'localhost', + port: parseInt(process.env.POSTGRES_PORT || '5433'), + database: process.env.POSTGRES_DB || 'kode_test', + user: process.env.POSTGRES_USER || 'postgres', + password: process.env.POSTGRES_PASSWORD || 'testpass123', +}; + +let store: PostgresStore | null = null; +let backend: PostgresStoreObservationBackend | null = null; +let skipTests = false; + +async function checkPostgresAvailable(): Promise { + let testStore: PostgresStore | null = null; + try { + testStore = new PostgresStore(PG_CONFIG, TEST_STORE_DIR); + await (testStore as any).initPromise; + await testStore.list(); + await testStore.close(); + return true; + } catch (error: any) { + if (testStore) { + try { + await testStore.close(); + } catch { + // ignore close error + } + } + console.log(` ⚠️ PostgreSQL observation backend 测试跳过: ${error.message}`); + return false; + } +} + +runner + .beforeAll(async () => { + skipTests = !(await checkPostgresAvailable()); + if (skipTests) { + return; + } + + store = new PostgresStore(PG_CONFIG, TEST_STORE_DIR); + backend = new PostgresStoreObservationBackend(store); + await backend.prune(); + await (store as any).pool.query('DELETE FROM observations'); + }) + .afterAll(async () => { + if (!store) { + return; + } + await (store as any).pool.query('DELETE FROM observations'); + await store.close(); + }); + +runner.test('PostgresStoreObservationBackend persists and queries observations when PostgreSQL is available', async () => { + if (skipTests || !store || !backend) { + return; + } + + await backend.append({ + seq: 0, + timestamp: 1_710_000_000_000, + observation: { + kind: 'agent_run', + agentId: 'agent-pg', + runId: 'run-pg', + traceId: 'trace-pg', + spanId: 'span-pg', + name: 'agent.run', + status: 'ok', + startTime: 1_710_000_000_000, + endTime: 1_710_000_000_010, + durationMs: 10, + trigger: 'send', + step: 1, + messageCountBefore: 1, + metadata: { templateId: 'template-pg' }, + }, + }); + + const listed = await backend.list({ agentIds: ['agent-pg'] }); + expect.toHaveLength(listed, 1); + expect.toEqual(listed[0].observation.runId, 'run-pg'); +}); + +export async function run() { + return runner.run(); +} diff --git a/tests/unit/observability/persistence/reader.test.ts b/tests/unit/observability/persistence/reader.test.ts new file mode 100644 index 0000000..72a06ea --- /dev/null +++ b/tests/unit/observability/persistence/reader.test.ts @@ -0,0 +1,64 @@ +import { + createStoreBackedObservationReader, + type ObservationEnvelope, + type ObservationQueryBackend, +} from '../../../../src'; +import { TestRunner, expect } from '../../../helpers/utils'; + +const runner = new TestRunner('Observability Persistence Reader'); + +runner.test('createStoreBackedObservationReader delegates to backend methods', async () => { + const listCalls: any[] = []; + const runCalls: string[] = []; + const expected: ObservationEnvelope[] = [ + { + seq: 1, + timestamp: 1_710_000_000_001, + observation: { + kind: 'agent_run', + agentId: 'agent-1', + runId: 'run-1', + traceId: 'trace-1', + spanId: 'span-1', + name: 'agent.run', + status: 'ok', + startTime: 1_710_000_000_001, + endTime: 1_710_000_000_011, + durationMs: 10, + trigger: 'send', + step: 1, + messageCountBefore: 1, + }, + }, + ]; + + const backend: ObservationQueryBackend = { + async append() { + return; + }, + async list(opts) { + listCalls.push(opts); + return expected; + }, + async getRun(runId) { + runCalls.push(runId); + return { + run: expected[0] as any, + observations: expected, + }; + }, + }; + + const reader = createStoreBackedObservationReader(backend); + const listed = await reader.listObservations({ runId: 'run-1' }); + const runView = await reader.getRun('run-1'); + + expect.toHaveLength(listed, 1); + expect.toEqual(listCalls[0].runId, 'run-1'); + expect.toEqual(runCalls[0], 'run-1'); + expect.toEqual(runView?.run.observation.runId, 'run-1'); +}); + +export async function run() { + return runner.run(); +} diff --git a/tests/unit/observability/persistence/retention.test.ts b/tests/unit/observability/persistence/retention.test.ts new file mode 100644 index 0000000..40d4c2f --- /dev/null +++ b/tests/unit/observability/persistence/retention.test.ts @@ -0,0 +1,61 @@ +import { + applyObservationRetention, + type ObservationEnvelope, +} from '../../../../src'; +import { TestRunner, expect } from '../../../helpers/utils'; + +const runner = new TestRunner('Observability Persistence Retention'); + +function makeEnvelope(agentId: string, seq: number, timestamp: number): ObservationEnvelope { + return { + seq, + timestamp, + observation: { + kind: 'agent_run', + agentId, + runId: `${agentId}-run-${seq}`, + traceId: `${agentId}-trace-${seq}`, + spanId: `${agentId}-span-${seq}`, + name: 'agent.run', + status: 'ok', + startTime: timestamp, + endTime: timestamp + 10, + durationMs: 10, + trigger: 'send', + step: seq, + messageCountBefore: 1, + }, + }; +} + +runner + .test('applyObservationRetention enforces maxEntriesPerAgent', async () => { + const now = 2_000; + const input = [ + makeEnvelope('agent-a', 1, 1_000), + makeEnvelope('agent-a', 2, 1_100), + makeEnvelope('agent-a', 3, 1_200), + makeEnvelope('agent-b', 1, 1_300), + makeEnvelope('agent-b', 2, 1_400), + ]; + + const retained = applyObservationRetention(input, { maxEntriesPerAgent: 2 }, now); + expect.toHaveLength(retained.envelopes, 4); + expect.toEqual(retained.result.deleted, 1); + expect.toEqual(retained.envelopes[0].observation.runId, 'agent-a-run-2'); + }) + .test('applyObservationRetention enforces maxAgeMs before count pruning', async () => { + const input = [ + makeEnvelope('agent-a', 1, 1_000), + makeEnvelope('agent-a', 2, 1_500), + makeEnvelope('agent-a', 3, 1_900), + ]; + + const retained = applyObservationRetention(input, { maxAgeMs: 300 }, 2_000); + expect.toHaveLength(retained.envelopes, 1); + expect.toEqual(retained.envelopes[0].observation.runId, 'agent-a-run-3'); + }); + +export async function run() { + return runner.run(); +} diff --git a/tests/unit/observability/persistence/sink.test.ts b/tests/unit/observability/persistence/sink.test.ts new file mode 100644 index 0000000..312ab79 --- /dev/null +++ b/tests/unit/observability/persistence/sink.test.ts @@ -0,0 +1,96 @@ +import { + PersistedObservationSink, + type ObservationEnvelope, + type ObservationQueryBackend, +} from '../../../../src'; +import { TestRunner, expect } from '../../../helpers/utils'; + +const runner = new TestRunner('Observability Persistence Sink'); + +function makeEnvelope(seq: number): ObservationEnvelope { + return { + seq, + timestamp: 1_710_000_000_000 + seq, + observation: { + kind: 'agent_run', + agentId: 'agent-1', + runId: `run-${seq}`, + traceId: `trace-${seq}`, + spanId: `span-${seq}`, + name: 'agent.run', + status: 'ok', + startTime: 1_710_000_000_000 + seq, + endTime: 1_710_000_000_010 + seq, + durationMs: 10, + trigger: 'send', + step: seq, + messageCountBefore: 1, + }, + }; +} + +runner + .test('PersistedObservationSink writes envelopes and prunes lazily', async () => { + const appended: ObservationEnvelope[] = []; + let pruneCalls = 0; + const backend: ObservationQueryBackend = { + async append(envelope) { + appended.push(envelope); + }, + async list() { + return appended; + }, + async getRun() { + return undefined; + }, + async prune() { + pruneCalls += 1; + return { deleted: 0, retained: appended.length }; + }, + }; + + const sink = new PersistedObservationSink(backend, { + retention: { maxEntriesPerAgent: 10 }, + pruneIntervalMs: 1_000, + }); + + await sink.onObservation(makeEnvelope(1)); + await sink.onObservation(makeEnvelope(2)); + + expect.toHaveLength(appended, 2); + expect.toEqual(pruneCalls, 1); + }) + .test('backend append/prune failures do not escape the sink', async () => { + let appendCalls = 0; + const backend: ObservationQueryBackend = { + async append() { + appendCalls += 1; + if (appendCalls === 1) { + throw new Error('append failed'); + } + }, + async list() { + return []; + }, + async getRun() { + return undefined; + }, + async prune() { + throw new Error('prune failed'); + }, + }; + + const sink = new PersistedObservationSink(backend, { + retention: { maxEntriesPerAgent: 1 }, + pruneIntervalMs: 1_000, + }); + + await sink.onObservation(makeEnvelope(1)); + await sink.onObservation(makeEnvelope(2)); + + expect.toEqual(appendCalls, 2); + }); + +export async function run() { + return runner.run(); +} diff --git a/tests/unit/observability/persistence/sqlite-backend.test.ts b/tests/unit/observability/persistence/sqlite-backend.test.ts new file mode 100644 index 0000000..99b8b5f --- /dev/null +++ b/tests/unit/observability/persistence/sqlite-backend.test.ts @@ -0,0 +1,115 @@ +import fs from 'node:fs'; +import path from 'node:path'; + +import { + SqliteStore, + SqliteStoreObservationBackend, +} from '../../../../src'; +import { TEST_ROOT } from '../../../helpers/fixtures'; +import { ensureCleanDir } from '../../../helpers/setup'; +import { TestRunner, expect } from '../../../helpers/utils'; + +const runner = new TestRunner('Observability Persistence Sqlite Backend'); + +runner.test('SqliteStoreObservationBackend persists and prunes observations', async () => { + const suffix = `${Date.now()}-${Math.random().toString(36).slice(2, 7)}`; + const dir = path.join(TEST_ROOT, `obs-persist-sqlite-${suffix}`); + ensureCleanDir(dir); + + const dbPath = path.join(dir, 'agents.db'); + const store = new SqliteStore(dbPath, dir); + const backend = new SqliteStoreObservationBackend(store); + + try { + await backend.append({ + seq: 0, + timestamp: 1_710_000_000_000, + observation: { + kind: 'agent_run', + agentId: 'agent-a', + runId: 'run-a', + traceId: 'trace-a', + spanId: 'span-a', + name: 'agent.run', + status: 'ok', + startTime: 1_710_000_000_000, + endTime: 1_710_000_000_010, + durationMs: 10, + trigger: 'send', + step: 1, + messageCountBefore: 1, + metadata: { templateId: 'template-a' }, + }, + }); + + await backend.append({ + seq: 1, + timestamp: 1_710_000_000_100, + observation: { + kind: 'generation', + agentId: 'agent-a', + runId: 'run-a', + traceId: 'trace-a', + spanId: 'span-b', + parentSpanId: 'span-a', + name: 'generation:gpt', + status: 'ok', + startTime: 1_710_000_000_100, + endTime: 1_710_000_000_110, + durationMs: 10, + provider: 'mock', + model: 'gpt', + usage: { + inputTokens: 1, + outputTokens: 1, + totalTokens: 2, + }, + metadata: { templateId: 'template-a' }, + }, + }); + + await backend.append({ + seq: 2, + timestamp: 1_710_000_000_200, + observation: { + kind: 'agent_run', + agentId: 'agent-a', + runId: 'run-b', + traceId: 'trace-b', + spanId: 'span-c', + name: 'agent.run', + status: 'error', + startTime: 1_710_000_000_200, + endTime: 1_710_000_000_220, + durationMs: 20, + trigger: 'send', + step: 2, + messageCountBefore: 2, + metadata: { templateId: 'template-b' }, + }, + }); + + const runView = await backend.getRun('run-a'); + expect.toBeTruthy(runView); + expect.toHaveLength(runView?.observations || [], 2); + + const byTemplate = await backend.list({ templateIds: ['template-b'] }); + expect.toHaveLength(byTemplate, 1); + expect.toEqual(byTemplate[0].observation.runId, 'run-b'); + + const pruned = await backend.prune({ maxEntriesPerAgent: 2 }); + expect.toEqual(pruned.deleted, 1); + + const remaining = await backend.list(); + expect.toHaveLength(remaining, 2); + expect.toEqual(remaining[0].observation.runId, 'run-a'); + expect.toEqual(remaining[1].observation.runId, 'run-b'); + } finally { + await store.close(); + fs.rmSync(dir, { recursive: true, force: true }); + } +}); + +export async function run() { + return runner.run(); +} diff --git a/tests/unit/observability/reader-api.test.ts b/tests/unit/observability/reader-api.test.ts new file mode 100644 index 0000000..b3832e2 --- /dev/null +++ b/tests/unit/observability/reader-api.test.ts @@ -0,0 +1,270 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import { + Agent, + AgentTemplateRegistry, + JSONStore, + SandboxFactory, + ToolRegistry, +} from '../../../src'; +import { ModelConfig, ModelProvider, ModelResponse, ModelStreamChunk } from '../../../src/infra/provider'; +import { TestRunner, expect } from '../../helpers/utils'; +import { TEST_ROOT } from '../../helpers/fixtures'; +import { ensureCleanDir } from '../../helpers/setup'; + +const runner = new TestRunner('Observability Reader API'); + +class QueueStreamProvider implements ModelProvider { + readonly model = 'queue-stream-provider'; + readonly maxWindowSize = 128000; + readonly maxOutputTokens = 4096; + readonly temperature = 0; + + constructor( + private readonly streams: Array<() => AsyncIterable>, + private readonly providerName = 'mock' + ) {} + + async complete(): Promise { + throw new Error('complete() should not be called'); + } + + async *stream(): AsyncIterable { + const next = this.streams.shift(); + if (!next) { + throw new Error('No scripted stream available'); + } + yield* next(); + } + + toConfig(): ModelConfig { + return { + provider: this.providerName, + model: this.model, + }; + } +} + +async function createObservedAgent(params: { + provider: ModelProvider; + template?: any; + registerTools?: (registry: ToolRegistry) => void; + observability?: { enabled?: boolean }; +}) { + const workDir = path.join(TEST_ROOT, `obs-reader-work-${Date.now()}-${Math.random().toString(36).slice(2, 7)}`); + const storeDir = path.join(TEST_ROOT, `obs-reader-store-${Date.now()}-${Math.random().toString(36).slice(2, 7)}`); + ensureCleanDir(workDir); + ensureCleanDir(storeDir); + + const templates = new AgentTemplateRegistry(); + templates.register( + params.template || { + id: 'obs-reader-agent', + systemPrompt: 'test reader api', + tools: [], + permission: { mode: 'auto' }, + } + ); + + const tools = new ToolRegistry(); + params.registerTools?.(tools); + + const agent = await Agent.create( + { + templateId: params.template?.id || 'obs-reader-agent', + model: params.provider, + sandbox: { kind: 'local', workDir, enforceBoundary: true }, + observability: params.observability, + }, + { + store: new JSONStore(storeDir), + templateRegistry: templates, + sandboxFactory: new SandboxFactory(), + toolRegistry: tools, + } + ); + + return { + agent, + cleanup: async () => { + await (agent as any).sandbox?.dispose?.(); + fs.rmSync(workDir, { recursive: true, force: true }); + fs.rmSync(storeDir, { recursive: true, force: true }); + }, + }; +} + +function registerEchoTool(registry: ToolRegistry) { + registry.register('echo_tool', () => ({ + name: 'echo_tool', + description: 'echo value', + input_schema: { type: 'object', properties: { value: { type: 'string' } }, required: ['value'] }, + async exec(args: any) { + return { echoed: args.value }; + }, + toDescriptor() { + return { source: 'registered' as const, name: 'echo_tool', registryId: 'echo_tool' }; + }, + })); +} + +function createApprovalProvider(finalText: string): ModelProvider { + return new QueueStreamProvider([ + async function* () { + yield { + type: 'content_block_start', + index: 0, + content_block: { type: 'tool_use', id: 'tool-1', name: 'echo_tool', input: {} }, + }; + yield { + type: 'content_block_delta', + index: 0, + delta: { type: 'input_json_delta', partial_json: '{"value":"hi"}' }, + }; + yield { type: 'content_block_stop', index: 0 }; + yield { type: 'message_stop' }; + }, + async function* () { + yield { type: 'content_block_start', index: 0, content_block: { type: 'text', text: '' } }; + yield { type: 'content_block_delta', index: 0, delta: { type: 'text_delta', text: finalText } }; + yield { type: 'content_block_stop', index: 0 }; + yield { type: 'message_delta', usage: { input_tokens: 1, output_tokens: 1 } }; + yield { type: 'message_stop' }; + }, + ]); +} + +runner + .test('getObservationReader 暴露只读查询能力', async () => { + const { agent, cleanup } = await createObservedAgent({ + provider: createApprovalProvider('reader done'), + template: { + id: 'obs-reader-deny', + systemPrompt: 'use tools', + tools: ['echo_tool'], + permission: { mode: 'approval', requireApprovalTools: ['echo_tool'] as const }, + }, + registerTools: registerEchoTool, + }); + + const offPermissionRequired = agent.on('permission_required', async (evt: any) => { + await evt.respond('deny', { note: 'reader api denied' }); + }); + + try { + const result = await agent.chat('run tool'); + expect.toEqual(result.status, 'ok'); + + const reader = agent.getObservationReader() as any; + expect.toBeTruthy(reader); + expect.toEqual(typeof reader.getMetricsSnapshot, 'function'); + expect.toEqual(typeof reader.subscribe, 'function'); + expect.toEqual(typeof reader.listObservations, 'function'); + expect.toEqual(typeof reader.getRun, 'function'); + expect.toEqual(reader.record, undefined); + + const all = reader.listObservations(); + expect.toBeGreaterThanOrEqual(all.length, 3); + expect.toEqual(all[0].seq, 0); + expect.toEqual(all[all.length - 1].seq, all.length - 1); + + const runEnvelope = all.find((entry: any) => entry.observation.kind === 'agent_run'); + const toolEnvelope = all.find((entry: any) => entry.observation.kind === 'tool'); + expect.toBeTruthy(runEnvelope); + expect.toBeTruthy(toolEnvelope); + + const byRun = reader.listObservations({ runId: runEnvelope.observation.runId }); + expect.toHaveLength(byRun, all.length); + + const byTrace = reader.listObservations({ traceId: runEnvelope.observation.traceId }); + expect.toHaveLength(byTrace, all.length); + + const byParent = reader.listObservations({ parentSpanId: runEnvelope.observation.spanId }); + expect.toHaveLength(byParent, all.length - 1); + expect.toEqual(byParent[0].observation.kind, 'generation'); + expect.toEqual(byParent[1].observation.kind, 'tool'); + + const errors = reader.listObservations({ statuses: ['error'] }); + expect.toHaveLength(errors, 1); + expect.toEqual(errors[0].observation.kind, 'tool'); + + const limited = reader.listObservations({ limit: 1 }); + expect.toHaveLength(limited, 1); + expect.toEqual(limited[0].observation.kind, 'agent_run'); + + const runView = reader.getRun(runEnvelope.observation.runId); + expect.toBeTruthy(runView); + expect.toEqual(runView.run.observation.kind, 'agent_run'); + expect.toHaveLength(runView.observations, all.length); + } finally { + offPermissionRequired(); + await cleanup(); + } + }) + .test('reader.subscribe 支持 sinceSeq 历史补读', async () => { + const provider = new QueueStreamProvider([ + async function* () { + yield { type: 'content_block_start', index: 0, content_block: { type: 'text', text: '' } }; + yield { type: 'content_block_delta', index: 0, delta: { type: 'text_delta', text: 'reader subscribe' } }; + yield { type: 'content_block_stop', index: 0 }; + yield { type: 'message_delta', usage: { input_tokens: 1, output_tokens: 2 } }; + yield { type: 'message_stop' }; + }, + ]); + + const { agent, cleanup } = await createObservedAgent({ provider }); + + try { + await agent.chat('hello'); + + const reader = agent.getObservationReader(); + const envelopes = reader.listObservations(); + expect.toHaveLength(envelopes, 2); + + const replayed: any[] = []; + for await (const envelope of reader.subscribe({ sinceSeq: envelopes[0].seq })) { + replayed.push(envelope); + break; + } + + expect.toHaveLength(replayed, 1); + expect.toEqual(replayed[0].seq, envelopes[1].seq); + expect.toEqual(replayed[0].observation.kind, 'agent_run'); + } finally { + await cleanup(); + } + }) + .test('enabled=false 时 reader 仍可用但不会暴露观测数据', async () => { + const provider = new QueueStreamProvider([ + async function* () { + yield { type: 'content_block_start', index: 0, content_block: { type: 'text', text: '' } }; + yield { type: 'content_block_delta', index: 0, delta: { type: 'text_delta', text: 'disabled reader' } }; + yield { type: 'content_block_stop', index: 0 }; + yield { type: 'message_delta', usage: { input_tokens: 1, output_tokens: 1 } }; + yield { type: 'message_stop' }; + }, + ]); + + const { agent, cleanup } = await createObservedAgent({ + provider, + observability: { enabled: false }, + }); + + try { + await agent.chat('hello'); + + const reader = agent.getObservationReader(); + expect.toHaveLength(reader.listObservations(), 0); + expect.toEqual(reader.getRun('missing'), undefined); + + const snapshot = reader.getMetricsSnapshot(); + expect.toEqual(snapshot.totals.totalTokens, 0); + expect.toEqual(snapshot.totals.generations, 0); + } finally { + await cleanup(); + } + }); + +export async function run() { + return runner.run(); +} diff --git a/tests/unit/observability/scheduler-metadata.test.ts b/tests/unit/observability/scheduler-metadata.test.ts new file mode 100644 index 0000000..0494186 --- /dev/null +++ b/tests/unit/observability/scheduler-metadata.test.ts @@ -0,0 +1,211 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import { + Agent, + AgentTemplateRegistry, + JSONStore, + SandboxFactory, + ToolRegistry, +} from '../../../src'; +import { ModelConfig, ModelProvider, ModelResponse, ModelStreamChunk } from '../../../src/infra/provider'; +import { TestRunner, expect } from '../../helpers/utils'; +import { TEST_ROOT } from '../../helpers/fixtures'; +import { ensureCleanDir, wait } from '../../helpers/setup'; + +const runner = new TestRunner('Observability Scheduler Metadata'); + +class QueueStreamProvider implements ModelProvider { + readonly model = 'queue-stream-provider'; + readonly maxWindowSize = 128000; + readonly maxOutputTokens = 4096; + readonly temperature = 0; + + constructor( + private readonly streams: Array<() => AsyncIterable>, + private readonly providerName = 'mock' + ) {} + + async complete(): Promise { + throw new Error('complete() should not be called'); + } + + async *stream(): AsyncIterable { + const next = this.streams.shift(); + if (!next) { + throw new Error('No scripted stream available'); + } + yield* next(); + } + + toConfig(): ModelConfig { + return { + provider: this.providerName, + model: this.model, + }; + } +} + +async function createObservedAgent(provider: ModelProvider) { + const workDir = path.join(TEST_ROOT, `obs-scheduler-work-${Date.now()}-${Math.random().toString(36).slice(2, 7)}`); + const storeDir = path.join(TEST_ROOT, `obs-scheduler-store-${Date.now()}-${Math.random().toString(36).slice(2, 7)}`); + ensureCleanDir(workDir); + ensureCleanDir(storeDir); + + const templates = new AgentTemplateRegistry(); + templates.register({ + id: 'obs-scheduler-agent', + systemPrompt: 'test scheduler observability', + tools: [], + permission: { mode: 'auto' }, + }); + + const agent = await Agent.create( + { + templateId: 'obs-scheduler-agent', + model: provider, + sandbox: { kind: 'local', workDir, enforceBoundary: true }, + }, + { + store: new JSONStore(storeDir), + templateRegistry: templates, + sandboxFactory: new SandboxFactory(), + toolRegistry: new ToolRegistry(), + } + ); + + return { + agent, + cleanup: async () => { + await (agent as any).sandbox?.dispose?.(); + fs.rmSync(workDir, { recursive: true, force: true }); + fs.rmSync(storeDir, { recursive: true, force: true }); + }, + }; +} + +function createTextStream(text: string): () => AsyncIterable { + return async function* () { + yield { type: 'content_block_start', index: 0, content_block: { type: 'text', text: '' } }; + yield { type: 'content_block_delta', index: 0, delta: { type: 'text_delta', text } }; + yield { type: 'content_block_stop', index: 0 }; + yield { type: 'message_delta', usage: { input_tokens: 1, output_tokens: 1 } }; + yield { type: 'message_stop' }; + }; +} + +async function waitForAgentRuns(agent: Agent, count: number) { + for (let i = 0; i < 50; i++) { + const runs = (agent as any).observationCollector.list({ kinds: ['agent_run'] }); + if (runs.length >= count) { + return runs; + } + await wait(20); + } + return (agent as any).observationCollector.list({ kinds: ['agent_run'] }); +} + +runner + .test('scheduler 触发的后续 run 会带 agent_run metadata.scheduler', async () => { + const provider = new QueueStreamProvider([ + createTextStream('user run'), + createTextStream('scheduler run'), + ]); + const { agent, cleanup } = await createObservedAgent(provider); + + const schedulerEvents: any[] = []; + const offScheduler = agent.on('scheduler_triggered', (evt: any) => { + schedulerEvents.push(evt); + }); + + const scheduler = agent.schedule(); + scheduler.everySteps(1, async () => { + scheduler.clear(); + await agent.send('scheduled follow-up'); + }); + + try { + const result = await agent.chat('initial'); + expect.toEqual(result.status, 'ok'); + + const runs = await waitForAgentRuns(agent, 2); + expect.toHaveLength(runs, 2); + + expect.toEqual(runs[0].trigger, 'send'); + expect.toEqual((runs[0].metadata || {}).scheduler, undefined); + + expect.toEqual(runs[1].trigger, 'scheduler'); + expect.toBeTruthy(runs[1].metadata?.scheduler); + expect.toContain(runs[1].metadata?.scheduler?.taskId || '', 'steps-'); + expect.toEqual(runs[1].metadata?.scheduler?.kind, 'steps'); + expect.toEqual(runs[1].metadata?.scheduler?.spec, 'steps:1'); + + const snapshot = agent.getMetricsSnapshot(); + expect.toEqual(snapshot.totals.scheduledRuns, 1); + + expect.toHaveLength(schedulerEvents, 1); + expect.toEqual(schedulerEvents[0].kind, 'steps'); + expect.toEqual(schedulerEvents[0].spec, 'steps:1'); + } finally { + offScheduler(); + scheduler.clear(); + await cleanup(); + } + }) + .test('普通 run 不应带 scheduler metadata', async () => { + const provider = new QueueStreamProvider([createTextStream('plain run')]); + const { agent, cleanup } = await createObservedAgent(provider); + + try { + const result = await agent.chat('hello'); + expect.toEqual(result.status, 'ok'); + + const runs = await waitForAgentRuns(agent, 1); + expect.toHaveLength(runs, 1); + expect.toEqual(runs[0].trigger, 'send'); + expect.toEqual((runs[0].metadata || {}).scheduler, undefined); + + const snapshot = agent.getMetricsSnapshot(); + expect.toEqual(snapshot.totals.scheduledRuns, 0); + } finally { + await cleanup(); + } + }) + .test('scheduler reminder 不应污染下一次用户 run 的 trigger', async () => { + const provider = new QueueStreamProvider([ + createTextStream('first run'), + createTextStream('second run'), + ]); + const { agent, cleanup } = await createObservedAgent(provider); + + const scheduler = agent.schedule(); + scheduler.everySteps(1, async () => { + scheduler.clear(); + await agent.send('scheduled reminder', { kind: 'reminder' }); + }); + + try { + const first = await agent.chat('initial'); + expect.toEqual(first.status, 'ok'); + + await wait(50); + + const second = await agent.chat('after reminder'); + expect.toEqual(second.status, 'ok'); + + const runs = await waitForAgentRuns(agent, 2); + expect.toHaveLength(runs, 2); + expect.toEqual(runs[0].trigger, 'send'); + expect.toEqual(runs[1].trigger, 'send'); + expect.toEqual((runs[1].metadata || {}).scheduler, undefined); + + const snapshot = agent.getMetricsSnapshot(); + expect.toEqual(snapshot.totals.scheduledRuns, 0); + } finally { + scheduler.clear(); + await cleanup(); + } + }); + +export async function run() { + return runner.run(); +}