From 51d0c7b9f08fa850880874af85fbdbc57024ebde Mon Sep 17 00:00:00 2001 From: Mirko Kiefer Date: Tue, 5 May 2026 11:25:19 +0700 Subject: [PATCH 1/3] docs: README rewrite, protocol doc, open-questions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Repositions the README around builder pain (state-mutation drift) before introducing vocabulary. Pulls the philosophical thesis out of the README into a dedicated PROTOCOL.md; surfaces unsolved problems as collaboration invitations in docs/open-questions.md. README changes: - New tagline: "Trace your agent's world, not just its tool calls." - Hero replaced: 60-second drift-caught story + 2-line drop-in code + sub-line about the CLI viewer. - New "Five words you'll meet" section teaches scene / diff / goal / gradient / predictor by example, not definition. Each term anchored in a concrete picture or sentence. - Five-tier ladder reframed: each tier leads with the dev pain it solves, not the framework feature it adds. Tier 4 in particular framed as "predict before commit for irreversible actions." - "Why this isn't another agent framework" trimmed to 3 rows; "What scenegrad does NOT do" cut entirely; "Thinking framework" prose relocated to PROTOCOL.md. - "Scenes are the unit of agent work" thesis section before the stack, three sentences, links to PROTOCOL.md and open-questions.md. - Stack section promoted from "Related" — scenecast first as the foundational vocabulary. PROTOCOL.md (new): - Five primitives (Scene, Assertion, Goal, Patch, ToolCall, Predictor) with TypeScript signatures and one-paragraph contracts. - Two derivations (distance, gradient) explained as functions of the primitives. - Trajectory wire format + WorldModelMetrics scoring surface. - Pointer to open-questions.md for unsolved frontier. docs/open-questions.md (new): - Seven open frontiers: distance functions, semantic diffs, predictor scoring, multi-actor gradients, scene design as craft, cold-start moat, adversarial scenes. - Framed as invitations to collaborate, with explicit "want to push back?" closing section. --- PROTOCOL.md | 192 +++++++++++++++++++++++++++++++++ README.md | 235 ++++++++++++++++++++++++++--------------- docs/open-questions.md | 78 ++++++++++++++ 3 files changed, 420 insertions(+), 85 deletions(-) create mode 100644 PROTOCOL.md create mode 100644 docs/open-questions.md diff --git a/PROTOCOL.md b/PROTOCOL.md new file mode 100644 index 0000000..76e45b2 --- /dev/null +++ b/PROTOCOL.md @@ -0,0 +1,192 @@ +# Protocol + +The shapes scenegrad runs on. Five primitives, two derivations, one open frontier. + +This document is the reference for *what* a scene is, *what* a gradient is, *what* a predictor returns. The library is one implementation of these shapes; future implementations (other languages, other runtimes, learned components) target the same shapes. + +--- + +## The thesis in one paragraph + +Your agent doesn't act on raw state; it acts on a **scene** — a typed, job-shaped view of the world that makes the next action obvious. Its loop is closing the diff between scene-now and the goal scene, with assertions defining "done," tools moving the world, and predictors imagining consequences before commit. Scene design is the craft. + +--- + +## Primitives + +### `Scene` + +The right rendering of the world for a specific job. Whatever your `snapshot()` function returns. Typed; the type parameter `S` is your domain shape. + +```ts +type Scene = unknown; // shape determined by your job +``` + +A scene is *not* the same as raw state. Same database powers many scenes; the scene a job uses is the one that makes its next action obvious. (See [scenecast](https://github.com/daslabhq/scenecast) for canonical scene shapes — Email, Message, Contact, Event, Task, Document — that domain-specific scenes extend.) + +### `Assertion` + +A predicate over a scene with a graduated gap measure. + +```ts +interface Assertion { + name: string; + check(scene: S): { satisfied: boolean; gap: number; weight?: number }; +} +``` + +Why graduated, not boolean: "almost done" needs to be a real number for distance to be meaningful. Binary assertions are gap=1 when unmet, gap=0 when satisfied; richer assertions (e.g. "12 cells off in this grid") return the count. + +### `Goal` + +A list of assertions, optionally combined with a non-default reduction. + +```ts +interface Goal { + assertions: Assertion[]; + reduce?: (gaps: number[]) => number; // default: weighted sum +} +``` + +### `Patch` — semantic diff between two scenes + +```ts +interface ScenePatch { + changed_keys: string[]; + added_keys: string[]; + removed_keys: string[]; +} +``` + +v0 ships a top-level key-set diff. Richer typed diffs land alongside scenecast canonical types — `EmailPatch`, `TicketPatch`, `RowPatch` — that respect domain meaning rather than byte-level structure. + +### `ToolCall` + +A typed action on a scene. + +```ts +interface ToolCall { + name: string; + args: Record; +} +``` + +Domains usually narrow this with a discriminated union per tool. + +### `Predictor` + +A learned-or-LLM model of `predict(scene, action) → consequence`. The interface every world-model implementation targets. + +```ts +interface Predictor { + readonly name: string; + predict(scene: S, tool: T): Promise>; +} + +interface Consequence { + scene_after: S; + outcome: { ok: boolean; error_class?: string; p: number }; + delta: ScenePatch; + blast_radius: BlastEdge[]; // downstream effects (often empty in v0) + confidence: number; // calibrated [0,1] + analogues: Analogue[]; // historical neighbours + reasoning?: string; +} +``` + +--- + +## Derivations + +These two functions follow from the primitives. The framework computes them; users rarely override. + +### `distance(scene, goal): number` + +Default: weighted sum of unmet-assertion gaps. Satisfied assertions contribute 0. + +```ts +function distance(scene: S, goal: Goal): number { + const gaps = goal.assertions.map(a => { + const r = a.check(scene); + if (r.satisfied) return 0; + return (r.gap ?? 1) * (r.weight ?? 1); + }); + return goal.reduce ? goal.reduce(gaps) : gaps.reduce((a, b) => a + b, 0); +} +``` + +### `gradient` — the action sequence that minimizes distance + +There is no closed-form `gradient()` function. The gradient is *the work an agent does*. Solvers approximate it: + +``` +GreedySolver : pick the action that maximizes Δdistance via env.simulate +LLMSolver : ask the LLM to pick +DreamerSolver : ask a Predictor what each action would do, pick best, commit +``` + +Different solvers = different policies for following the gradient. They produce comparable `SolveResult` shapes so you can benchmark them on the same env. + +--- + +## The trajectory + +The wire format scenegrad emits — and the substrate every benchmark, viewer, and eval reads. + +```ts +interface TrajectoryStep { + step: number; + tool: T | null; + scene_before?: unknown; + scene_after?: unknown; + d_before: number; + d_after: number; + delta: number; // d_before - d_after; positive = closer to goal + predicted_delta?: number; // solvers that predict gradient closure + reasoning?: string; + ok: boolean; + error?: string; + assertions_after: AssertionState[]; + ts_ms: number; +} +``` + +Trajectories serialize as one JSON object per file (or JSONL across runs), with the inner shape compatible with [scene-otel](https://github.com/daslabhq/scene-otel) span events. The viewer reads JSONL; benches read JSONL; predictors train on JSONL. + +--- + +## The evaluation surface + +A predictor's quality is measurable: replay a known action sequence, compare predicted scene_after to actual scene_after, score. + +```ts +interface WorldModelMetrics { + outcome_acc: number; // P(predicted_ok == actual_ok) + scene_match: number; // P(predicted scene deep-equals actual) + delta_match: number; // P(predicted change-keys equal actual) + avg_confidence: number; + ece: number; // expected calibration error, 10-bin +} +``` + +This is the world-model-accuracy benchmark a predictor is judged against. Same shape can run inside scenebench as a leaderboard column across AutomationBench / τ-bench / LeRobot. + +--- + +## What this protocol claims, in one line per primitive + +- **Scene:** the right view of the world for the job — not raw state. +- **Assertion:** a graduated check over a scene; the unit "done" is built from. +- **Goal:** a set of assertions; aggregates to a scalar distance. +- **Patch:** semantic diff over the scene's typed shape, not bytes. +- **ToolCall:** a typed action whose effect is observable in the scene. +- **Predictor:** any function `(scene, action) → consequence` — LLM, kNN, distilled, learned. +- **Trajectory:** the wire-format record of an agent (or solver) closing a gradient. + +Five primitives. Two derivations. One protocol surface that benchmarks, viewers, predictors, and evaluators all share. + +--- + +## Open frontier + +The interesting unsolved problems live at [`docs/open-questions.md`](./docs/open-questions.md). They're invitations, not gaps in the implementation. diff --git a/README.md b/README.md index 393ac4a..255575c 100644 --- a/README.md +++ b/README.md @@ -1,46 +1,34 @@ # scenegrad -> **Drop into your agent in 2 lines. Get a scrubbable replay. Level up when you're ready.** +> **Trace your agent's world, not just its tool calls.** +> Drop in 2 lines. Scrub a timeline of typed scenes — what your agent saw, what changed, where it drifted. [![v0.0.1](https://img.shields.io/badge/version-v0.0.1--alpha-orange)](https://github.com/daslabhq/scenegrad) [![license](https://img.shields.io/badge/license-MIT-blue)](./LICENSE) [![scenegrad viewer](./docs/demo.gif)](https://daslabhq.github.io/scenegrad/bulk.html) -*Above: 12 support-triage trajectories at a glance, then drill into one. The bulk view shows the distribution (3 escalated-vip in red, 6 escalated-t2 in amber, 3 auto-resolved in emerald). Filter chips narrow the grid; click any card to scrub through the full trajectory with typed widgets — Ticket card on the left shows the world morphing (`status: new` → `investigating` → `escalated-vip`, `[CRITICAL]` reply appearing), Customer card on the right materializes with ENTERPRISE badge + LTV when the agent runs `enrich_with_account`. All 12 runs cost ~$0.02 in Haiku tokens.* - **[→ Try the live demo](https://daslabhq.github.io/scenegrad/bulk.html)** · **[Single-trace viewer](https://daslabhq.github.io/scenegrad/)** --- -## Is this for you? - -scenegrad pays off when your agent does **multi-step work that mutates state**. Specifically: - -- ✅ Your agent calls multiple tools across multiple turns -- ✅ Those tools change something — a CRM record, a ticket status, a database row, a scheduled job -- ✅ You can't tell from the tool-call log alone whether the *world* ended up in the right state -- ✅ You want to compare runs across models or prompts and see *trajectories*, not just final scores -- ✅ You're using Vercel AI SDK, Anthropic SDK, LangChain, or your own loop (5-line drop-in for any) - -scenegrad is **not** the right fit if: - -- ❌ Your agent is single-call chat / RAG with no state mutation — Phoenix or Helicone will serve you better -- ❌ You only need output scoring (LLM-as-judge) — Braintrust does that better; consider scenegrad alongside, not instead -- ❌ You need a full eval platform with annotation queues, datasets-as-a-service, etc. — that's not the scope here +## The bug logs hide -We're explicitly *not* trying to be a replacement for those tools. scenegrad fills the gap they don't: visualizing typed scene state across multi-step trajectories, with a viewer that scales from single-trace to bulk grid. +Your agent ran a 6-step onboarding flow. The trace says it succeeded. The user complains next morning that no welcome email arrived. ---- - -scenegrad is a tiny observability + evaluation substrate for AI agents. Pay only for what you use: +With logs, you dig for an hour. With scenegrad, you scrub the trajectory and see step 4 — agent claimed `welcome_sent: true` in its working memory; the world snapshot says `welcome_at: null`. Drift caught in 30 seconds. ```ts import { trace } from "scenegrad"; -const t = trace.start(); +const t = trace.start({ + snapshot: async () => ({ + user: await db.users.findOne({ session_id }), + welcome_sent: await emails.exists({ template: "welcome" }), + }), +}); -// your existing Vercel AI SDK / LangChain / custom loop, untouched: +// your existing Vercel AI SDK / Anthropic SDK / LangChain loop, untouched: const result = await generateText({ model: anthropic("claude-haiku-4-5"), tools: { /* your tools, unchanged */ }, @@ -51,68 +39,119 @@ const result = await generateText({ t.dump("./traces/run.jsonl"); ``` -Then view it: - ```bash npx scenegrad view ./traces # bulk view of every JSONL in ./traces -npx scenegrad view ./traces/run.jsonl # single-trace view of one file +npx scenegrad view ./traces/run.jsonl # single-trace view +``` + +> **v0.0.1 note:** the CLI requires [Bun](https://bun.sh) on your PATH. Node-compatible bin compilation lands in v0.0.2. + +--- + +## Five words you'll meet, all you need + +scenegrad has a small vocabulary. Each word is just the obvious name for what you're already looking at. + +### Scene — the world your agent acts on + +Same database. Three jobs. Three scenes: + ``` +support-triage scene → ticket card + customer LTV + recent thread +inbox triage scene → message list + sender importance + status +data-migration scene → source schema + target schema + row diff +``` + +A **scene** is the right rendering of the world for a specific job. Your `snapshot()` function returns it. Typed shapes for common scenes (Email, Message, Contact, Event, Task, Document) live in [`scenecast`](https://github.com/daslabhq/scenecast); your domain-specific ones extend them. + +### Diff — what changed, semantically + +Step 3 archived message #2. The scene before vs after differs in one field: `status: unread → archived`. That reads as one semantic change, not a byte-level mess. scenegrad diffs over the scene's typed shape, not over JSON bytes. + +### Goal + assertions — what "done" means + +Done is a list of assertions on the scene. *No unread messages. Important mail not archived. Welcome email sent.* An assertion has a `satisfied` flag and a `gap` measure, so "almost done" is a real number, not vibes. + +### Gradient — the work that closes the diff -> **v0.0.1 note:** the CLI requires [Bun](https://bun.sh) on your PATH (the bin scripts are TypeScript with a `#!/usr/bin/env bun` shebang). Node-compatible bin compilation lands in v0.0.2. +Goal scene minus current scene = a gap. The action sequence that closes the gap is the gradient. **Gradient = the work the agent has to do.** No backprop, no math — just "the actions from now to done." When the gradient stops shrinking, your agent is stuck and you can see it. -That's it. Drop in, run, view your runs in a browser. No goal to design. No assertions to write. No restructuring of your agent. +### Predictor — imagine before commit -When you want more — add a `snapshot()` to capture world state between calls. Add a `goal()` of assertions to measure gap closure. Each level is opt-in. +For irreversible actions (real APIs, prod databases, outbound email), you want to know what'll happen *before* committing. A predictor takes `(scene, action)` and returns the imagined next scene + confidence. v0 ships an LLM predictor; the abstraction is built so kNN, distilled, and learned predictors swap in later. + +That's the whole vocabulary. Five words. --- ## The five-tier ladder -| Tier | What you write | What you get | +Adopt at the tier that matches your pain. Same trajectory format flows through all five — you can level up later without restructuring. + +| Tier | The pain it solves | What you write | |---|---|---| -| **0 — trace** | `trace.start()` + one hook | Tool-call timeline, scrubbable. Replaces logs. | -| **1 — + snapshot** | Add `snapshot: () => fetchWorld()` | World deltas between calls. See *what changed*, not just what was called. | -| **2 — + goal** | Add `goal: (s) => [...assertions]` | Gap-closure curve, drift detection, runtime `status()` for agent guidance. | -| **3 — + solver** | Use `defineEnv` + `LLMSolver` / `GreedySolver` | scenegrad drives the loop — for benches, comparison, leaderboards. | -| **4 — + predictor** | Add `Predictor` + `DreamerSolver` | Plan in imagination, act in the env. World-model accuracy as a measurable benchmark metric. | +| **0 — trace** | "I can't tell from logs whether my agent worked" | `trace.start()` + 1 hook | +| **1 — + snapshot** | "I want to see what changed in the *world*, not just what was *called*" | `snapshot: () => fetchWorld()` | +| **2 — + goal** | "I want my agent grounded in actual world state, not its working memory" | `goal: (s) => [...assertions]` | +| **3 — + solver** | "I want a typed env to benchmark agents in" | `defineEnv` + `LLMSolver` / `GreedySolver` | +| **4 — + predictor** | "My agent's about to call an irreversible API. I want it to think first." | `Predictor` + `DreamerSolver` | + +--- + +## Tier 0 — drop in, get a scrubbable replay -The same trajectory format flows through all five tiers. You can adopt at tier 0, level up months later as you understand your agent's failure modes. +Two lines. Works alongside Vercel AI SDK, Anthropic SDK, LangChain, or your own loop. + +```ts +import { trace } from "scenegrad"; +const t = trace.start(); + +const result = await generateText({ + model: anthropic("claude-haiku-4-5"), + tools: { /* unchanged */ }, + onStepFinish: t.captureStep, + prompt: "...", +}); + +t.dump("./traces/run.jsonl"); +``` + +That's it. No goal to design. No assertions to write. No restructuring of your agent. You get a scrubbable timeline that replaces logs. --- -## Tier 1 — add a snapshot +## Tier 1 — add a snapshot, see what changed -When tools mutate external state (DBs, APIs, CRMs, queues), seeing what *changed* is more useful than seeing what was *called*. Pass a `snapshot` function: +When tools mutate external state, seeing the *delta* is more useful than seeing the *call*. ```ts const t = trace.start({ snapshot: async () => ({ - user: await db.users.findOne({ session_id }), + user: await db.users.findOne({ session_id }), welcome_sent: await emails.exists({ template: "welcome" }), }), }); ``` -Now each step in the trajectory captures both `scene_before` and `scene_after`. The viewer can render the world delta per step. +Now each step captures `scene_before` and `scene_after`. The viewer renders the world delta per step — and surfaces the moment the agent's belief diverges from the world. --- -## Tier 2 — add a goal, get drift detection + status() +## Tier 2 — add a goal, get drift detection + status() at runtime -Define what "done" looks like as assertions: +Define what "done" looks like as assertions. The same spec serves both runtime guidance and post-hoc evaluation. ```ts const watcher = observe({ snapshot: async () => fetchWorld(), goal: (s) => [ - { name: "name_collected", check: (s) => ({ satisfied: !!s.user?.name, gap: 1 }) }, - { name: "email_collected", check: (s) => ({ satisfied: !!s.user?.email, gap: 1 }) }, - { name: "role_specified", check: (s) => ({ satisfied: !!s.user?.role, gap: 1 }) }, - { name: "welcome_email_sent", check: (s) => ({ satisfied: s.welcome_sent, gap: 1 }) }, + { name: "name_collected", check: (s) => ({ satisfied: !!s.user?.name, gap: 1 }) }, + { name: "email_collected", check: (s) => ({ satisfied: !!s.user?.email, gap: 1 }) }, + { name: "role_specified", check: (s) => ({ satisfied: !!s.user?.role, gap: 1 }) }, + { name: "welcome_email_sent", check: (s) => ({ satisfied: s.welcome_sent, gap: 1 }) }, ], }); -// In your existing loop, each turn: const status = await watcher.status(); const result = await generateText({ @@ -126,15 +165,15 @@ const result = await generateText({ }); ``` -The agent's checklist is now **grounded in actual world state — not its working memory.** When you evaluate post-hoc, you read the same assertions back through `watcher.trajectory()`. Spec written once; serves both runtime guidance and evaluation. +The agent's checklist is now grounded in *actual world state*, not its working memory. Post-hoc, you read the same assertions back through `watcher.trajectory()`. Spec written once; serves both runtime guidance and evaluation. This is also TDD-shaped agent development: write the assertion → run → watch the gap → tighten. See `examples/inbox.ts` for the canonical three-iteration progression. --- -## Tier 3 — solver mode, for benches +## Tier 3 — drive the loop yourself, for benches -When you want scenegrad to drive the loop (for benchmarks, comparison across models, controlled tests): +When you want scenegrad to drive the agent (for benchmarks, comparing models, controlled tests): ```ts import { defineEnv, LLMSolver, GreedySolver } from "scenegrad"; @@ -147,11 +186,38 @@ const task = defineEnv({ step: (s, t) => t.name === "inc" ? { count: s.count + 1 } : { count: s.count - 1 }, }); -await new GreedySolver().solve(task, "default"); // baseline -await new LLMSolver({ model: "claude-haiku-4-5" }).solve(task, "default"); // LLM-driven +await new GreedySolver().solve(task, "default"); // optimal baseline +await new LLMSolver({ model: "claude-haiku-4-5" }).solve(task, "default"); // LLM-driven ``` -Both solvers produce the same `SolveResult` shape. Compare them on the same env to see how much the LLM drifts from the optimal greedy baseline. +Both produce the same `SolveResult` shape. Compare them on the same env to see how much the LLM drifts from the optimal greedy baseline. + +--- + +## Tier 4 — predict before commit (for irreversible actions) + +Your agent's about to call an irreversible API — write to prod, send an email, charge a card. You want it to *think* before picking. Tier 4 wraps every candidate action in a predictor first; the agent commits only the action whose imagined outcome closes the most distance. + +```ts +import { defineEnv, LLMPredictor, DreamerSolver, evalWorldModel } from "scenegrad"; + +const env = defineEnv({ /* same as tier 3 */ }); +const predictor = new LLMPredictor({ model: "claude-haiku-4-5" }); + +// Plans in the predictor, commits one action at a time. +await new DreamerSolver({ predictor, lookahead: 1 }).solve(env, "default"); + +// Measure how good the predictor actually is — predicted vs actual scene_after. +const metrics = await evalWorldModel({ + env, predictor, + tasks: [{ taskId: "default", actions: [/* known action sequence */] }], +}); +// → outcome_acc, scene_match, delta_match, avg_confidence, ece (calibration) +``` + +`Predictor` is a one-method interface. v0 ships `LLMPredictor` as a placeholder; future predictors (kNN over a trace store, distilled-from-traces, fully learned) drop in via the same API — `new DreamerSolver({ predictor })` doesn't change. + +The `evalWorldModel` metric scores predictors against real env traces — outcome accuracy, scene-match rate, delta-match rate, calibration. Ship a predictor → score it on any tier-3 env → publish the leaderboard column. --- @@ -184,35 +250,38 @@ The `evalWorldModel` metric is the world-model-accuracy benchmark a predictor is ## Why this isn't another agent framework -If you already use… | scenegrad adds… ----|--- -LangChain / LangGraph | Drift measurement and per-task evaluation. LangChain orchestrates; scenegrad measures. -OpenTelemetry / Phoenix | A vocabulary for *what* to observe (scene + goal + diff), not just *how* to ship spans. -Custom eval scripts | A common shape so your evals are comparable across runs, models, teams. -System prompts to constrain behavior | A way to say "done" the framework can VERIFY, not just hope the LLM honors. +| If you already use… | scenegrad adds… | +|---|---| +| LangChain / LangGraph | Drift measurement and per-task evaluation. LangChain orchestrates; scenegrad measures. | +| OpenTelemetry / Phoenix | A vocabulary for *what* to observe (scene + goal + diff), not just *how* to ship spans. | +| Custom eval scripts | A common shape so your evals are comparable across runs, models, teams. | -scenegrad doesn't replace your agent. It instruments your task so behavior becomes visible, comparable, and refinable. +scenegrad doesn't replace your agent. It instruments your task so behavior becomes visible, comparable, refinable. + +**Not the right fit if** your agent is single-call chat / RAG with no state mutation (Phoenix or Helicone serve that better), or you only need output scoring (Braintrust does that better — consider scenegrad alongside, not instead). --- -## What scenegrad does NOT do +## Scenes are the unit of agent work + +Your agent doesn't act on raw state; it acts on a **scene** — a typed, job-shaped view of the world that makes the next action obvious. Its loop is closing the diff between scene-now and the goal scene, with assertions defining "done," tools moving the world, and predictors imagining consequences before commit. -- **Doesn't write your distance function.** Domain-specific. Sometimes hard. -- **Doesn't induce your toolkit.** You author it; [autocompile](https://github.com/mirkokiefer/autocompile) refines it over time. -- **Doesn't solve local-optima / dead-end paths.** It exposes them; your solver picks the search strategy. -- **Doesn't replace your agent.** Your agent runs whatever loop it runs; scenegrad measures it. -- **Doesn't unify cross-domain distance.** ARC's "12 cells off" and SAP's "3 audit controls violated" aren't directly comparable — each domain owns its units. -- **Doesn't force the agent to predict gradients.** The natural drift signals (gap-not-closing, goal-claimed-but-unmet, vs-baseline) work without polluting the prompt. +Scene design is the craft: a well-designed scene makes hard jobs solvable; a wrong one makes them impossible. The whole stack exists to make that craft tractable. + +Read the protocol → [`PROTOCOL.md`](./PROTOCOL.md). Open questions → [`docs/open-questions.md`](./docs/open-questions.md). --- -## The thinking framework +## The stack -We sense the scene / world now. We make assertions on it — sensor values, query responses, photos, descriptions. **It's always an *image of* the scene, not the scene itself.** +scenegrad sits in a four-repo stack. Each has a single job: -We have a future scene vaguely in mind. We assert that as best we can. The diff between now and then is the gradient. Tools are actions that close it. We make progress, re-evaluate, sometimes redefine the goal as we learn. Reality is complex; the loop is simple. +- **[`scenecast`](https://github.com/daslabhq/scenecast)** — typed scene shapes + multi-format renderers. *The vocabulary of scenes.* +- **[`scene-otel`](https://github.com/daslabhq/scene-otel)** — wire format. Every snapshot becomes an OTel span event readable in Phoenix, Honeycomb, Braintrust, etc. +- **`scenegrad`** *(this repo)* — gradients, solvers, predictors. *The verbs.* +- **[`scenebench`](https://github.com/daslabhq/scenebench)** — benchmarks built on the stack (AutomationBench, τ-bench, LeRobot adapters; S4Bench, more native benches coming). -Two gradients flow through this: **the agent's** (closing scene-now to scene-then per step) and **yours** (closing the spec to reality, by tightening assertions when behavior surprises you). Both are gradient descent. Both happen in the same framework. +Related: [`autocompile`](https://github.com/mirkokiefer/autocompile) — observes accumulated trajectories and hardens patterns to code. --- @@ -220,7 +289,7 @@ Two gradients flow through this: **the agent's** (closing scene-now to scene-the ```bash npm install scenegrad -# optional, for LLMSolver: +# optional, for LLMSolver / LLMPredictor: npm install @anthropic-ai/sdk # optional, for tier-0 with Vercel AI SDK: npm install ai @ai-sdk/anthropic zod @@ -229,27 +298,23 @@ npm install ai @ai-sdk/anthropic zod ANTHROPIC_API_KEY=... bun examples/trace-only-aisdk.ts # tier 2 — observer mode with goal + status injection -ANTHROPIC_API_KEY=... bun examples/onboarding.ts # Anthropic SDK -ANTHROPIC_API_KEY=... bun examples/support-triage-aisdk.ts # Vercel AI SDK +ANTHROPIC_API_KEY=... bun examples/onboarding.ts # Anthropic SDK +ANTHROPIC_API_KEY=... bun examples/support-triage-aisdk.ts # Vercel AI SDK # tier 3 — solver mode (for benches) -bun examples/counter.ts # no LLM -ANTHROPIC_API_KEY=... bun examples/inbox.ts # LLMSolver, TDD progression +bun examples/counter.ts # no LLM +ANTHROPIC_API_KEY=... bun examples/inbox.ts # LLMSolver, TDD progression + +# tier 4 — predictor + dreamer +ANTHROPIC_API_KEY=... bun examples/dreamer-inbox.ts # LLMSolver vs DreamerSolver, plus eval ``` ## Status -v0.0.1 — substrate types, `defineEnv`, `observe` (tiers 0/1/2), `trace.start()` (tier-0 alias), `GreedySolver`, `LLMSolver`, JSONL trace format, viewer scaffold. +v0.0.1 — substrate types, `defineEnv`, `observe` (tiers 0/1/2), `trace.start()`, `GreedySolver`, `LLMSolver`, JSONL trace format, viewer scaffold. Tier 4 (`Predictor`, `DreamerSolver`, `evalWorldModel`) lands in v0.0.2. Reference benches live in [scenebench](https://github.com/daslabhq/scenebench): ARC-trajectory ships first; AutomationBench (806 real tasks); S4Bench (SAP) and LeRobot (robotics) follow. -## Related - -- [`scene-otel`](https://github.com/daslabhq/scene-otel) — wire format scenegrad emits trajectories in -- [`scenecast`](https://github.com/daslabhq/scenecast) — typed scene shapes + multi-size widgets that render trajectories visually -- [`scenebench`](https://github.com/daslabhq/scenebench) — benchmarks built on scenegrad -- [`autocompile`](https://github.com/mirkokiefer/autocompile) — observes accumulated trajectories, hardens patterns to code - ## License MIT. diff --git a/docs/open-questions.md b/docs/open-questions.md new file mode 100644 index 0000000..b01dcab --- /dev/null +++ b/docs/open-questions.md @@ -0,0 +1,78 @@ +# Open questions + +The protocol is small; the unsolved problems around it are not. These are invitations, not gaps in the v0.0.1 implementation. + +If you have an opinion on any of these, open an issue or PR. The protocol document is meant to evolve with the conversation. + +--- + +## 1. What's a *good* distance function over a typed scene? + +Default: weighted sum of unmet-assertion gaps. It works for most demos. It breaks the moment assertions are correlated, when satisfied-but-shaky beats unsatisfied-but-stable, or when the agent should be rewarded for *partial* progress on hard assertions vs *complete* progress on easy ones. + +Open questions: +- Should the framework offer principled aggregations (max, log-sum-exp, learned-from-data) as built-ins? +- How do you handle assertions whose gap units differ (rows vs cells vs cosine similarity)? +- Is there a domain-independent distance, or is distance fundamentally a domain-author craft? + +## 2. How do you compute *semantic* diffs? + +v0 ships a top-level key-set diff. That's clearly insufficient for typed scenes (you want `EmailPatch` not `Patch{changed_keys: ["draft"]}`). [scenecast](https://github.com/daslabhq/scenecast) defines canonical scene shapes; the diff types should hang off those. + +Open questions: +- What's the right per-canonical-asset diff shape? (`EmailPatch`, `MessagePatch`, `RowPatch`, `EventPatch`, …) +- Should diff types support partial / approximate matches (e.g. for LLM-rewritten text)? +- How do you diff scenes whose shape *itself* changed (schema migration mid-trajectory)? + +## 3. What's the right scoring rubric for a Predictor? + +`evalWorldModel` v0 reports outcome accuracy, scene-match (deep equal), delta-match (key-set), confidence, and 10-bin ECE. Deep equality is brittle; a predictor that gets the structure right but a value slightly wrong scores 0%. + +Open questions: +- Should scene-match be replaced with a typed equivalence (using scenecast canonical shapes)? +- What's the right metric for "predicted the right *reason* something happened" (a richer success signal than scene equality)? +- How do you compose accuracy across heterogeneous task suites (AutomationBench + τ-bench + LeRobot)? +- Is there a calibration metric better suited to this domain than ECE? + +## 4. How do gradients compose across multi-actor scenes? + +Today, one agent acts on one scene. Real systems are multi-agent (orchestrator + sub-agents), multi-actor (humans + agents share the scene), and multi-time (some assertions only become checkable later). + +Open questions: +- What's the protocol shape for "this assertion can only be evaluated 24h later"? +- How does the trajectory format handle two actors interleaving steps on the same scene? +- Can predictors model *human* responses as part of the scene? (Reply prediction, approval prediction, …) + +## 5. Scene design as a teachable craft + +Scenes are the unit of agent work, but there's no equivalent of "single responsibility principle" or "keep components small" for them yet. + +Open questions: +- What heuristics actually work? (We have intuitions: "include only what's needed for the next action," "don't conflate snapshot with derived view." We don't have laws.) +- Is there a pattern language? (Trigger-scene, summary-scene, diff-scene, plan-scene, …?) +- When should a scene split into two? When should two scenes merge? +- Is there an analog of "code smells" — *scene smells* — that flag a poorly-designed scene before it causes failures? + +## 6. The cold-start moat + +The thesis is that per-org learned predictors compound — more traces → better predictions → defensible flywheel. But on day one a customer has zero traces. + +Open questions: +- What's the best cross-org prior to bootstrap from? (Trained on canonical scene shapes? On benchmark trajectories?) +- How do you transfer a predictor from a Gmail-rich org to an Outlook org? +- What's the minimum trace volume for a kNN predictor to beat an LLM predictor on the same env? + +## 7. Adversarial scenes + +If a tool can affect a scene field that an assertion checks, an adversarial agent (or an over-eager one) can satisfy assertions without doing the work. + +Open questions: +- Is there a notion of "tamper-evident" scenes — fields that prove their own provenance? +- Should assertions distinguish "achieved by intended means" from "achieved at all"? +- How do you stress-test an assertion suite against gaming? + +--- + +## Want to push back? + +These are working-state, not committed. If you've thought hard about any of these, the issues tab is the right place. Especially welcome: counter-examples that break a current default, or pattern languages you've found load-bearing in your own agent work. From f24e8aa90e231fce6c8a8a068a51addfb4410f22 Mon Sep 17 00:00:00 2001 From: Mirko Kiefer Date: Sat, 9 May 2026 23:07:41 +0700 Subject: [PATCH 2/3] feat(env): accept mark.Predicate as a Goal alternative Increment 1 of the mark integration: scenegrad's Goal is now a union of mark.Predicate (preferred, going forward) and the legacy AssertionGoal shape. distance() and checkAll() handle either; existing scenebench paths keep working unchanged. This unblocks downstream benches (arcbench, future SAPBench) that want to pass mark Predicates directly without wrapping them in scenegrad's Assertion interface. Increments 2-4 (migrate scenebench's adapter to pass Predicates directly, delete Assertion) follow when needed. Also: remove a hardcoded local path from scripts/record-demo.ts. --- package.json | 9 +++--- scripts/record-demo.ts | 4 ++- src/env.ts | 73 +++++++++++++++++++++++++++++++++++++++--- src/observe.ts | 31 ++++++++++++------ 4 files changed, 98 insertions(+), 19 deletions(-) diff --git a/package.json b/package.json index 8789a22..ddf2bb7 100644 --- a/package.json +++ b/package.json @@ -51,18 +51,19 @@ "scene" ], "peerDependencies": { - "@anthropic-ai/sdk": "^0.39.0" + "@anthropic-ai/sdk": "^0.39.0", + "mark": "*" }, "peerDependenciesMeta": { - "@anthropic-ai/sdk": { - "optional": true - } + "@anthropic-ai/sdk": { "optional": true }, + "mark": { "optional": true } }, "devDependencies": { "@ai-sdk/anthropic": "^3.0.74", "@anthropic-ai/sdk": "^0.39.0", "@types/bun": "latest", "ai": "^6.0.174", + "mark": "file:../mark", "playwright": "^1.59.1", "typescript": "^5.6.0", "zod": "^4.4.2" diff --git a/scripts/record-demo.ts b/scripts/record-demo.ts index 19fe723..49cb95d 100644 --- a/scripts/record-demo.ts +++ b/scripts/record-demo.ts @@ -15,9 +15,11 @@ import { chromium } from "playwright"; import { mkdirSync } from "node:fs"; +import { dirname, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; const URL = "http://localhost:7401/bulk.html"; -const OUT_DIR = "/Users/fm/git/daslab/ios2/Daslab/oss/scenegrad/docs/demo-recording"; +const OUT_DIR = resolve(dirname(fileURLToPath(import.meta.url)), "..", "docs", "demo-recording"); mkdirSync(OUT_DIR, { recursive: true }); const browser = await chromium.launch({ headless: true }); diff --git a/src/env.ts b/src/env.ts index 934eadc..ab3acd0 100644 --- a/src/env.ts +++ b/src/env.ts @@ -2,13 +2,22 @@ * SceneGradEnv — the contract every domain implements. * * scenegrad reframe: an agent's job is to close the gap between - * scene_now and scene_then, where scene_then is a set of assertions. + * scene_now and scene_then, where scene_then is described by a goal. * * Domain author writes: scene(), goal(), tools(), step(). * Framework derives: distance, simulate, solver dispatch, telemetry, * scrubbable trajectories — all from the same four methods. + * + * Goals come in two shapes (since we're mid-migration to mark predicates): + * 1. mark.Predicate — the canonical shape going forward + * 2. { assertions: Assertion[] } — legacy scenegrad shape, still supported + * + * `distance()` and `checkAll()` accept either; new code should pass Predicate. */ +import type { Predicate } from "mark"; +import { evaluate } from "mark"; + export type Distance = number; export interface Assertion { @@ -22,13 +31,25 @@ export interface Assertion { check(scene: S): { satisfied: boolean; gap: number; weight?: number }; } -export interface Goal { +export interface AssertionGoal { assertions: Assertion[]; /** Optional: combine per-assertion gaps into a scalar. * Default: weighted sum. */ reduce?: (gaps: number[]) => number; } +/** + * A goal is either a mark Predicate (preferred) or a legacy AssertionGoal. + * Polymorphic so existing scenebench code keeps working unchanged while new + * benches (arcbench, future SAPBench) can pass Predicates directly. + */ +export type Goal = Predicate | AssertionGoal; + +/** True when the value is a mark Predicate (vs an AssertionGoal). */ +export function isPredicate(g: Goal): g is Predicate { + return typeof (g as any).op === "string"; +} + export interface StepResult { scene_after: S; ok: boolean; @@ -76,10 +97,14 @@ export interface SceneGradEnv { // --------------------------------------------------------------------------- /** - * Default distance: weighted sum of unmet-assertion gaps. - * Satisfied assertions contribute 0. + * Default distance: weighted sum of unmet-assertion gaps. Predicate goals + * delegate to mark.evaluate(); the resulting gap is the scalar. + * Satisfied conditions contribute 0. */ export function distance(scene: S, goal: Goal): Distance { + if (isPredicate(goal)) { + return evaluate(scene, goal).gap; + } const gaps = goal.assertions.map(a => { const r = a.check(scene); if (r.satisfied) return 0; @@ -90,7 +115,7 @@ export function distance(scene: S, goal: Goal): Distance { /** * Per-assertion check result — surfaces in trajectories so we can see - * which specific goals are unmet at each step. + * which specific sub-goals are unmet at each step. */ export interface AssertionState { name: string; @@ -99,12 +124,50 @@ export interface AssertionState { } export function checkAll(scene: S, goal: Goal): AssertionState[] { + if (isPredicate(goal)) { + // Decompose top-level AND so each child surfaces as its own assertion-state + // row. Anything else evaluates as a single-element list. + const subs = goal.op === "and" ? goal.of : [goal]; + return subs.map((sub, i) => { + const r = evaluate(scene, sub); + return { + name: describePredicate(sub) ?? `assertion_${i}`, + satisfied: r.satisfied, + gap: r.gap, + }; + }); + } return goal.assertions.map(a => { const r = a.check(scene); return { name: a.name, satisfied: r.satisfied, gap: r.satisfied ? 0 : (r.gap ?? 1) }; }); } +/** Cheap human-readable summary of a Predicate, for display in trajectories. */ +function describePredicate(p: Predicate): string { + switch (p.op) { + case "eq": return `${p.path} = ${jsonShort(p.value)}`; + case "neq": return `${p.path} ≠ ${jsonShort(p.value)}`; + case "contains": return `${p.path} contains "${p.substring}"`; + case "exists": return `${p.path} exists`; + case "missing": return `${p.path} missing`; + case "find": return `find in ${p.collection}`; + case "count": return `count(${p.collection})`; + case "and": return `all(${p.of.length})`; + case "or": return `any(${p.of.length})`; + case "not": return `not ${describePredicate(p.of)}`; + } +} + +function jsonShort(v: unknown): string { + try { + const s = JSON.stringify(v); + return s.length > 40 ? s.slice(0, 37) + "..." : s; + } catch { + return String(v); + } +} + /** * Default simulate: clone via JSON, apply step, restore. Works for any * env with JSON-serializable scenes and pure-functional tools. diff --git a/src/observe.ts b/src/observe.ts index 86d6bc3..61800d4 100644 --- a/src/observe.ts +++ b/src/observe.ts @@ -13,7 +13,7 @@ * One API; opt into more by passing more fields. Pay only for what you use. */ -import type { Assertion, AssertionState, ToolCall } from "./env.js"; +import type { Assertion, AssertionState, Goal, ToolCall } from "./env.js"; import { distance, checkAll } from "./env.js"; export interface ObserveSpec { @@ -21,9 +21,11 @@ export interface ObserveSpec { * Without this: only tool-call timing is captured. */ snapshot?: () => Promise | S; - /** Optional. Assertions defining "done." - * Without this: no gap measurement, no status() — pure trace. */ - goal?: (s: S) => Assertion[]; + /** Optional. Goal defining "done." + * Accepts either a mark Predicate (preferred, going forward) or a list + * of legacy Assertions. Without it: no gap measurement, no status() — + * pure trace. */ + goal?: (s: S) => Goal | Assertion[]; /** Optional. Side-channel handlers for emitted events. */ exporters?: ((event: ObserverEvent) => void | Promise)[]; @@ -92,8 +94,7 @@ export class Watcher { if (!this.spec.goal || scene === undefined) { return { scene, assertions: [], satisfied: [], unmet: [], gap: 0, done: true }; } - const assertions_arr = this.spec.goal(scene); - const goal = { assertions: assertions_arr }; + const goal = normalizeGoal(this.spec.goal(scene)); const all = checkAll(scene, goal); return { scene, @@ -128,12 +129,12 @@ export class Watcher { this.lastScene = after; if (this.spec.goal && before !== undefined) { - const goalBefore = { assertions: this.spec.goal(before) }; + const goalBefore = normalizeGoal(this.spec.goal(before)); d_before = distance(before, goalBefore); assertions_before = checkAll(before, goalBefore); } if (this.spec.goal && after !== undefined) { - const goalAfter = { assertions: this.spec.goal(after) }; + const goalAfter = normalizeGoal(this.spec.goal(after)); d_after = distance(after, goalAfter); assertions_after = checkAll(after, goalAfter); } @@ -166,7 +167,8 @@ export class Watcher { async done(): Promise { if (!this.spec.goal || !this.spec.snapshot) return true; const s = await this.spec.snapshot(); - return this.spec.goal(s).every(a => a.check(s).satisfied); + const goal = normalizeGoal(this.spec.goal(s)); + return checkAll(s, goal).every(a => a.satisfied); } /** Full trajectory so far. */ @@ -193,3 +195,14 @@ export class Watcher { export function observe(spec: ObserveSpec = {}): Watcher { return new Watcher(spec); } + +/** + * Coerce whatever the user's `goal()` callback returns into the canonical + * Goal shape. Accepts: + * - mark Predicate → returned as-is (preferred) + * - { assertions: [...] } → returned as-is (legacy AssertionGoal) + * - Assertion[] → wrapped as { assertions: [...] } (legacy direct) + */ +function normalizeGoal(g: Goal | Assertion[]): Goal { + return Array.isArray(g) ? { assertions: g } : g; +} From 6ca3dbc055846d5118649bbb7bd80b99c7de98a5 Mon Sep 17 00:00:00 2001 From: Mirko Kiefer Date: Tue, 12 May 2026 18:59:01 +0700 Subject: [PATCH 3/3] Adopt autocheck (rename from mark) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Dep: file:../mark → file:../autocheck - Imports: `Predicate, evaluate` → `CheckExpr, runCheck` - isPredicate() → isCheckExpr(); describePredicate() → describeCheckExpr() - Goal = CheckExpr | AssertionGoal - Bridge: autocheck.pass → assertionState.satisfied / Assertion contract (scenegrad's downstream Assertion / AssertionState shapes unchanged) 8 scenegrad tests still green. --- package.json | 6 +++--- src/env.ts | 45 +++++++++++++++++++++++---------------------- 2 files changed, 26 insertions(+), 25 deletions(-) diff --git a/package.json b/package.json index ddf2bb7..aee3c52 100644 --- a/package.json +++ b/package.json @@ -52,18 +52,18 @@ ], "peerDependencies": { "@anthropic-ai/sdk": "^0.39.0", - "mark": "*" + "autocheck": "*" }, "peerDependenciesMeta": { "@anthropic-ai/sdk": { "optional": true }, - "mark": { "optional": true } + "autocheck": { "optional": true } }, "devDependencies": { "@ai-sdk/anthropic": "^3.0.74", "@anthropic-ai/sdk": "^0.39.0", "@types/bun": "latest", "ai": "^6.0.174", - "mark": "file:../mark", + "autocheck": "file:../autocheck", "playwright": "^1.59.1", "typescript": "^5.6.0", "zod": "^4.4.2" diff --git a/src/env.ts b/src/env.ts index ab3acd0..464fd87 100644 --- a/src/env.ts +++ b/src/env.ts @@ -8,15 +8,15 @@ * Framework derives: distance, simulate, solver dispatch, telemetry, * scrubbable trajectories — all from the same four methods. * - * Goals come in two shapes (since we're mid-migration to mark predicates): - * 1. mark.Predicate — the canonical shape going forward + * Goals come in two shapes (since we're mid-migration to autocheck): + * 1. autocheck.CheckExpr — the canonical shape going forward * 2. { assertions: Assertion[] } — legacy scenegrad shape, still supported * - * `distance()` and `checkAll()` accept either; new code should pass Predicate. + * `distance()` and `checkAll()` accept either; new code should pass CheckExpr. */ -import type { Predicate } from "mark"; -import { evaluate } from "mark"; +import type { CheckExpr } from "autocheck"; +import { runCheck } from "autocheck"; export type Distance = number; @@ -39,14 +39,15 @@ export interface AssertionGoal { } /** - * A goal is either a mark Predicate (preferred) or a legacy AssertionGoal. - * Polymorphic so existing scenebench code keeps working unchanged while new - * benches (arcbench, future SAPBench) can pass Predicates directly. + * A goal is either an autocheck CheckExpr (preferred) or a legacy + * AssertionGoal. Polymorphic so existing scenebench code keeps working + * unchanged while new benches (arc-bench, future SAPBench) can pass + * CheckExprs directly. */ -export type Goal = Predicate | AssertionGoal; +export type Goal = CheckExpr | AssertionGoal; -/** True when the value is a mark Predicate (vs an AssertionGoal). */ -export function isPredicate(g: Goal): g is Predicate { +/** True when the value is an autocheck CheckExpr (vs an AssertionGoal). */ +export function isCheckExpr(g: Goal): g is CheckExpr { return typeof (g as any).op === "string"; } @@ -97,13 +98,13 @@ export interface SceneGradEnv { // --------------------------------------------------------------------------- /** - * Default distance: weighted sum of unmet-assertion gaps. Predicate goals - * delegate to mark.evaluate(); the resulting gap is the scalar. + * Default distance: weighted sum of unmet-assertion gaps. CheckExpr goals + * delegate to autocheck.runCheck(); the resulting gap is the scalar. * Satisfied conditions contribute 0. */ export function distance(scene: S, goal: Goal): Distance { - if (isPredicate(goal)) { - return evaluate(scene, goal).gap; + if (isCheckExpr(goal)) { + return runCheck(scene, goal).gap; } const gaps = goal.assertions.map(a => { const r = a.check(scene); @@ -124,15 +125,15 @@ export interface AssertionState { } export function checkAll(scene: S, goal: Goal): AssertionState[] { - if (isPredicate(goal)) { + if (isCheckExpr(goal)) { // Decompose top-level AND so each child surfaces as its own assertion-state // row. Anything else evaluates as a single-element list. const subs = goal.op === "and" ? goal.of : [goal]; return subs.map((sub, i) => { - const r = evaluate(scene, sub); + const r = runCheck(scene, sub); return { - name: describePredicate(sub) ?? `assertion_${i}`, - satisfied: r.satisfied, + name: describeCheckExpr(sub) ?? `assertion_${i}`, + satisfied: r.pass, gap: r.gap, }; }); @@ -143,8 +144,8 @@ export function checkAll(scene: S, goal: Goal): AssertionState[] { }); } -/** Cheap human-readable summary of a Predicate, for display in trajectories. */ -function describePredicate(p: Predicate): string { +/** Cheap human-readable summary of a CheckExpr, for display in trajectories. */ +function describeCheckExpr(p: CheckExpr): string { switch (p.op) { case "eq": return `${p.path} = ${jsonShort(p.value)}`; case "neq": return `${p.path} ≠ ${jsonShort(p.value)}`; @@ -155,7 +156,7 @@ function describePredicate(p: Predicate): string { case "count": return `count(${p.collection})`; case "and": return `all(${p.of.length})`; case "or": return `any(${p.of.length})`; - case "not": return `not ${describePredicate(p.of)}`; + case "not": return `not ${describeCheckExpr(p.of)}`; } }