From b2de63642e4ff18d13ce4e37af92ca5f3a6f45fa Mon Sep 17 00:00:00 2001 From: tautvydasLiekis Date: Thu, 8 Oct 2026 22:21:02 +0300 Subject: [PATCH 01/19] refactor: merge work summary records one row at a time --- src/extensions/work-attribution/summary.ts | 81 ++++++++++++++-------- 1 file changed, 54 insertions(+), 27 deletions(-) diff --git a/src/extensions/work-attribution/summary.ts b/src/extensions/work-attribution/summary.ts index d73dbe59d..fcd53f66d 100644 --- a/src/extensions/work-attribution/summary.ts +++ b/src/extensions/work-attribution/summary.ts @@ -426,25 +426,48 @@ function recordKey(type: WorkRecord["type"], row: SummaryEntry): string { return JSON.stringify(row.transitionId) } } -async function merge(summary: WorkSummary, records: WorkRecord[]): Promise { - const sessions = new Set(summary.sessions) - const links = new Map(summary.workLinks.map((row) => [recordKey("work_link", row), row])) - const requests = new Map(summary.requests.map((row) => [JSON.stringify(row.requestId), row])) - const plans = new Map(summary.plans.map((row) => [planKey(row), row])) - const commits = new Map(summary.commits.map((row) => [commitKey(row), row])) - const transitions = new Map(summary.fileTransitions.map((row) => [JSON.stringify(row.transitionId), row])) - const observations = new Map(summary.fileObservations.map((row) => [JSON.stringify(row.observationId), row])) - const continuations = new Map(summary.continuations.map((row) => [continuationKey(row), row])) +/** Rows by key. A store that reads saved rows on demand can stand in for a Map. */ +interface RowMap { + get(key: string): SummaryEntry | undefined + has(key: string): boolean + set(key: string, row: SummaryEntry): unknown +} +/** One work's rows while records merge: sessions in first-seen order, each collection by row key. */ +interface SummaryRows { + sessions: Set + workLinks: T + requests: T + plans: T + commits: T + fileTransitions: T + fileObservations: T + continuations: T +} +function summaryRows(summary: WorkSummary): SummaryRows> { + return { + sessions: new Set(summary.sessions), + workLinks: new Map(summary.workLinks.map((row) => [recordKey("work_link", row), row])), + requests: new Map(summary.requests.map((row) => [JSON.stringify(row.requestId), row])), + plans: new Map(summary.plans.map((row) => [planKey(row), row])), + commits: new Map(summary.commits.map((row) => [commitKey(row), row])), + fileTransitions: new Map(summary.fileTransitions.map((row) => [JSON.stringify(row.transitionId), row])), + fileObservations: new Map(summary.fileObservations.map((row) => [JSON.stringify(row.observationId), row])), + continuations: new Map(summary.continuations.map((row) => [continuationKey(row), row])), + } +} +/** Merge rules for one batch of records, applied row by row. */ +async function merge(summary: SummaryRows, records: WorkRecord[]): Promise { + const { sessions, continuations } = summary const entriesByType = { - work_link: links, - request: requests, - request_dispatch: requests, - request_response: requests, - request_cost: requests, - plan: plans, - commit: commits, - file_transition: transitions, - file_observation: observations, + work_link: summary.workLinks, + request: summary.requests, + request_dispatch: summary.requests, + request_response: summary.requests, + request_cost: summary.requests, + plan: summary.plans, + commit: summary.commits, + file_transition: summary.fileTransitions, + file_observation: summary.fileObservations, } for (let index = 0; index < records.length; index++) { if (index % MERGE_BATCH_SIZE === 0) await setImmediate() @@ -498,14 +521,6 @@ async function merge(summary: WorkSummary, records: WorkRecord[]): Promise if (transitionIds.length) existing.transitionIds = transitionIds } else entries.set(key, item) } - summary.sessions = [...sessions] - summary.workLinks = [...links.values()] - summary.requests = [...requests.values()] - summary.plans = [...plans.values()] - summary.commits = [...commits.values()] - summary.fileTransitions = [...transitions.values()] - summary.fileObservations = [...observations.values()] - summary.continuations = [...continuations.values()] } function publish(directory: string, summary: WorkSummary, assertLease: () => void): Promise { return writeFileDurably(join(directory, "work.json"), `${JSON.stringify(summary, null, 2)}\n`, assertLease) @@ -534,7 +549,19 @@ async function update( continuations: [], } const history = !summary && !complete ? readWorkRecords(agentDir).filter((row) => row.workId === workId) : [] - await merge(value, history.concat(records)) + const rows = summaryRows(value) + await merge(rows, history.concat(records)) + value.sessions = [...rows.sessions] + for (const collection of [ + "workLinks", + "requests", + "plans", + "commits", + "fileTransitions", + "fileObservations", + "continuations", + ] as const) + value[collection] = [...rows[collection].values()] if (published === JSON.stringify(value)) return assertLease() await publish(directory, value, assertLease) From 265b8e70af8d2ff0548e39cd96f1d37a66f67c64 Mon Sep 17 00:00:00 2001 From: tautvydasLiekis Date: Fri, 9 Oct 2026 03:03:07 +0300 Subject: [PATCH 02/19] feat: store work summaries as row logs --- src/extensions/__mocks__/work-summary.ts | 8 + src/extensions/work-attribution.test.ts | 13 +- .../__fixtures__/work-summary-v1-legacy.json | 37 + .../__fixtures__/work-summary-v1-stack.json | 315 ++++++ .../__fixtures__/work-summary-v1.7.1.json | 127 +++ .../work-attribution/commits.test.ts | 40 + src/extensions/work-attribution/commits.ts | 4 +- .../continuation.integration.test.ts | 10 +- .../work-attribution/cost-sync.test.ts | 25 +- .../work-attribution/file-transitions.test.ts | 9 +- .../work-attribution/row-log.test.ts | 783 +++++++++++++++ src/extensions/work-attribution/row-log.ts | 918 ++++++++++++++++++ .../semantic.integration.test.ts | 12 +- .../work-attribution/summary.test.ts | 30 +- src/extensions/work-attribution/summary.ts | 223 ++--- tests/e2e/acp/work-continuation.test.ts | 28 +- tests/e2e/acp/work-pr-discovery.test.ts | 3 +- tests/e2e/tui/support/work-summary.ts | 58 ++ tests/e2e/tui/work-attribution.test.ts | 10 +- tests/e2e/tui/work-pr-discovery.test.ts | 7 +- 20 files changed, 2443 insertions(+), 217 deletions(-) create mode 100644 src/extensions/__mocks__/work-summary.ts create mode 100644 src/extensions/work-attribution/__fixtures__/work-summary-v1-legacy.json create mode 100644 src/extensions/work-attribution/__fixtures__/work-summary-v1-stack.json create mode 100644 src/extensions/work-attribution/__fixtures__/work-summary-v1.7.1.json create mode 100644 src/extensions/work-attribution/row-log.test.ts create mode 100644 src/extensions/work-attribution/row-log.ts create mode 100644 tests/e2e/tui/support/work-summary.ts diff --git a/src/extensions/__mocks__/work-summary.ts b/src/extensions/__mocks__/work-summary.ts new file mode 100644 index 000000000..693e43e46 --- /dev/null +++ b/src/extensions/__mocks__/work-summary.ts @@ -0,0 +1,8 @@ +import { readWorkSummary, type WorkSummaryView } from "../work-attribution/row-log.js" + +/** A work's saved summary in the version 1 shape; fails the test when it is missing or damaged. */ +export function savedWorkSummary(agentDir: string, workId: string): WorkSummaryView { + const summary = readWorkSummary(agentDir, workId) + if (!summary) throw new Error(`No readable work summary for ${workId}`) + return summary +} diff --git a/src/extensions/work-attribution.test.ts b/src/extensions/work-attribution.test.ts index 5b28181ea..def10ba8c 100644 --- a/src/extensions/work-attribution.test.ts +++ b/src/extensions/work-attribution.test.ts @@ -25,6 +25,7 @@ import { createCommandContext, createContext } from "./__mocks__/context.js" import { createExtensionApi } from "./__mocks__/extension-api.js" import { testTheme } from "./__mocks__/theme.js" import { createWorkScopeSnapshot } from "./__mocks__/work-scope.js" +import { savedWorkSummary } from "./__mocks__/work-summary.js" import requestTimingExtension from "./request-timing.js" import { copySessionFileAndAddHandoffNote, removeTempDir } from "./teleport/provisioning/handoff-note.js" import * as billingSource from "./work-attribution/billing-source.js" @@ -207,11 +208,11 @@ describe("local work attribution", () => { expect(getWorkId(ctx)).toBe(target.workId) await flushWorkSummaries() const path = join(dir, "work", target.workId, "work.json") - expect(JSON.parse(readFileSync(path, "utf8")).workLinks).toHaveLength(1) + expect(savedWorkSummary(dir, target.workId).workLinks).toHaveLength(1) writeFileSync(path, "damaged") recoverWorkSummaries() await flushWorkSummaries() - expect(JSON.parse(readFileSync(path, "utf8")).workLinks).toHaveLength(1) + expect(savedWorkSummary(dir, target.workId).workLinks).toHaveLength(1) await command.handler(`unlink ${links[0].linkId}`, ctx) expect(records().filter((row) => row.type === "work_link")).toEqual([ links[0], @@ -459,11 +460,11 @@ describe("local work attribution", () => { requestId: calls[1].requestId, }) await flushWorkSummaries() - const requests = JSON.parse(readFileSync(join(dir, "work", getWorkId(ctx), "work.json"), "utf8")).requests + const { requests } = savedWorkSummary(dir, getWorkId(ctx)) expect(requests).toHaveLength(2) expect(records().filter((row) => row.type === "request")).toHaveLength(2) expect(records().filter((row) => row.type === "request_dispatch")).toHaveLength(2) - expect(requests[0].billingSelector.tag).toBe(calls[0].tag) + expect(requests[0].billingSelector).toMatchObject({ tag: calls[0].tag }) expect(requests[1]).toMatchObject({ requestId: calls[1].requestId, parentRequestId: logicalId, @@ -602,7 +603,7 @@ describe("local work attribution", () => { }) recordProviderResponse(attempt.requestId, { status: 500, headers: new Headers() }) await flushWorkSummaries() - const summary = JSON.parse(readFileSync(join(dir, "work", attempt.workId, "work.json"), "utf8")) + const summary = savedWorkSummary(dir, attempt.workId) expect(summary.requests).toHaveLength(1) expect(summary.requests[0]).toMatchObject({ requestId: attempt.requestId, @@ -625,7 +626,7 @@ describe("local work attribution", () => { }) recordProviderResponse("unknown-request", { status: 200, headers: new Headers() }) await flushWorkSummaries() - const summary = JSON.parse(readFileSync(join(dir, "work", attempt.workId, "work.json"), "utf8")) + const summary = savedWorkSummary(dir, attempt.workId) expect(summary.requests).toHaveLength(1) expect(summary.requests[0].response).toEqual({ status: 503, receivedAt: expect.any(String) }) }) diff --git a/src/extensions/work-attribution/__fixtures__/work-summary-v1-legacy.json b/src/extensions/work-attribution/__fixtures__/work-summary-v1-legacy.json new file mode 100644 index 000000000..9150de2ac --- /dev/null +++ b/src/extensions/work-attribution/__fixtures__/work-summary-v1-legacy.json @@ -0,0 +1,37 @@ +{ + "version": 1, + "workId": "8b304360-39a9-4d6e-8ac7-984c97c0e315", + "sessions": ["01a10dc0-7ea4-74d8-9b87-9a5026c7b05c"], + "requests": [ + { + "requestId": "00000001-1111-4111-8111-111111111111", + "sessionId": "01a10dc0-7ea4-74d8-9b87-9a5026c7b05c", + "cwd": "/src/example", + "provider": "kimchi-dev", + "model": "glm-5.3", + "modelSource": "context", + "recordedAt": "2026-09-01T10:00:00.000Z" + }, + { + "requestId": "00000002-1111-4111-8111-111111111111", + "sessionId": "01a10dc0-7ea4-74d8-9b87-9a5026c7b05c", + "cwd": "/src/example", + "provider": "kimchi-dev", + "model": "glm-5.3", + "modelSource": "context", + "recordedAt": "2026-09-02T10:00:00.000Z" + } + ], + "plans": [], + "commits": [ + { + "sha": "cccccccccccccccccccccccccccccccccccccccc", + "repository": "/src/example/.git", + "worktree": "/src/example", + "sessionId": "01a10dc0-7ea4-74d8-9b87-9a5026c7b05c", + "cwd": "/src/example", + "recordedAt": "2026-08-02T11:00:00.000Z", + "paths": ["a.ts"] + } + ] +} diff --git a/src/extensions/work-attribution/__fixtures__/work-summary-v1-stack.json b/src/extensions/work-attribution/__fixtures__/work-summary-v1-stack.json new file mode 100644 index 000000000..4c951a3d7 --- /dev/null +++ b/src/extensions/work-attribution/__fixtures__/work-summary-v1-stack.json @@ -0,0 +1,315 @@ +{ + "version": 1, + "workId": "7a203360-39a9-4d6e-8ac7-984c97c0e314", + "sessions": ["01a10dc0-7ea4-74d8-9b87-9a5026c7b05c", "02b20dc0-7ea4-74d8-9b87-9a5026c7b05c"], + "workLinks": [ + { + "linkId": "88888888-2222-4333-8444-555555555555", + "revision": 1, + "sourceWorkId": "6f102360-39a9-4d6e-8ac7-984c97c0e313", + "targetWorkId": "7a203360-39a9-4d6e-8ac7-984c97c0e314", + "requestIds": ["00000001-1111-4111-8111-111111111111"], + "scope": { + "account": { + "apiUrl": "https://api.example.ai/v1", + "organizationId": "33333333-2222-4333-8444-555555555555", + "userId": "44444444-2222-4333-8444-555555555555" + }, + "repository": "/src/example/.git" + }, + "status": "active", + "evidence": { + "source": "work-command" + }, + "sessionId": "02b20dc0-7ea4-74d8-9b87-9a5026c7b05c", + "cwd": "/src/example", + "recordedAt": "2026-10-02T09:00:00.000Z" + } + ], + "requests": [ + { + "requestId": "00000001-3333-4333-8333-333333333333", + "sessionId": "01a10dc0-7ea4-74d8-9b87-9a5026c7b05c", + "cwd": "/src/example", + "startedAt": "2026-10-01T10:00:00.000Z", + "provider": "kimchi-dev", + "model": "glm-5.3", + "modelSource": "context", + "segment": { + "id": "00000001-4444-4444-8444-444444444444", + "attribution": "explicit", + "reason": "work-command" + }, + "scope": { + "account": { + "apiUrl": "https://api.example.ai/v1", + "organizationId": "33333333-2222-4333-8444-555555555555", + "userId": "44444444-2222-4333-8444-555555555555" + }, + "repository": "/src/example/.git" + }, + "recordedAt": "2026-10-01T10:01:00.000Z", + "dispatchedAt": "2026-10-01T10:00:00.003Z", + "billingSource": { + "apiUrl": "https://api.example.ai/v1", + "gatewayUrl": "https://llm.example.ai/openai/v1/chat/completions", + "credentialHash": "eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee" + }, + "billingSelector": { + "type": "tag", + "tag": "kimchi-request:00000001-3333-4333-8333-333333333333", + "startTime": "2026-09-30T22:00:00.003Z", + "endTime": "2026-11-01T10:00:00.003Z" + }, + "response": { + "status": 200, + "receivedAt": "2026-10-01T10:00:01.000Z", + "traceId": "ffffffffffffffffffffffffffffffff" + }, + "billingRows": [ + { + "id": "00000001-5555-4555-8555-555555555555", + "costUsd": "0.012345678", + "promptTokens": "1200", + "completionTokens": "40", + "model": "glm-5.3" + } + ], + "billingLookup": { + "status": "priced", + "checkedAt": "2026-10-01T10:01:00.000Z", + "organizationId": "33333333-2222-4333-8444-555555555555", + "userId": "44444444-2222-4333-8444-555555555555" + } + }, + { + "requestId": "00000002-3333-4333-8333-333333333333", + "sessionId": "01a10dc0-7ea4-74d8-9b87-9a5026c7b05c", + "cwd": "/src/example", + "startedAt": "2026-10-02T10:00:00.000Z", + "provider": "kimchi-dev", + "model": "glm-5.3", + "modelSource": "context", + "segment": { + "id": "00000002-4444-4444-8444-444444444444", + "attribution": "explicit", + "reason": "work-command" + }, + "scope": { + "account": { + "apiUrl": "https://api.example.ai/v1", + "organizationId": "33333333-2222-4333-8444-555555555555", + "userId": "44444444-2222-4333-8444-555555555555" + }, + "repository": "/src/example/.git" + }, + "recordedAt": "2026-10-02T10:01:00.000Z", + "dispatchedAt": "2026-10-02T10:00:00.003Z", + "billingSource": { + "apiUrl": "https://api.example.ai/v1", + "gatewayUrl": "https://llm.example.ai/openai/v1/chat/completions", + "credentialHash": "eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee" + }, + "billingSelector": { + "type": "tag", + "tag": "kimchi-request:00000002-3333-4333-8333-333333333333", + "startTime": "2026-09-30T22:00:00.003Z", + "endTime": "2026-11-02T10:00:00.003Z" + }, + "response": { + "status": 200, + "receivedAt": "2026-10-02T10:00:01.000Z", + "traceId": "ffffffffffffffffffffffffffffffff" + }, + "billingRows": [ + { + "id": "00000002-5555-4555-8555-555555555555", + "costUsd": "0.012345678", + "promptTokens": "1200", + "completionTokens": "40", + "model": "glm-5.3" + } + ], + "billingLookup": { + "status": "priced", + "checkedAt": "2026-10-02T10:01:00.000Z", + "organizationId": "33333333-2222-4333-8444-555555555555", + "userId": "44444444-2222-4333-8444-555555555555" + } + }, + { + "requestId": "00000003-3333-4333-8333-333333333333", + "sessionId": "02b20dc0-7ea4-74d8-9b87-9a5026c7b05c", + "cwd": "/src/example", + "startedAt": "2026-10-03T10:00:00.000Z", + "provider": "kimchi-dev", + "model": "glm-5.3", + "modelSource": "context", + "segment": { + "id": "00000003-4444-4444-8444-444444444444", + "attribution": "explicit", + "reason": "work-command" + }, + "scope": { + "account": { + "apiUrl": "https://api.example.ai/v1", + "organizationId": "33333333-2222-4333-8444-555555555555", + "userId": "44444444-2222-4333-8444-555555555555" + }, + "repository": "/src/example/.git" + }, + "recordedAt": "2026-10-03T10:01:00.000Z", + "dispatchedAt": "2026-10-03T10:00:00.003Z", + "billingSource": { + "apiUrl": "https://api.example.ai/v1", + "gatewayUrl": "https://llm.example.ai/openai/v1/chat/completions", + "credentialHash": "eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee" + }, + "billingSelector": { + "type": "tag", + "tag": "kimchi-request:00000003-3333-4333-8333-333333333333", + "startTime": "2026-09-30T22:00:00.003Z", + "endTime": "2026-11-03T10:00:00.003Z" + }, + "response": { + "status": 200, + "receivedAt": "2026-10-03T10:00:01.000Z", + "traceId": "ffffffffffffffffffffffffffffffff" + }, + "billingRows": [ + { + "id": "00000003-5555-4555-8555-555555555555", + "costUsd": "0.012345678", + "promptTokens": "1200", + "completionTokens": "40", + "model": "glm-5.3" + } + ], + "billingLookup": { + "status": "priced", + "checkedAt": "2026-10-03T10:01:00.000Z", + "organizationId": "33333333-2222-4333-8444-555555555555", + "userId": "44444444-2222-4333-8444-555555555555" + } + } + ], + "plans": [ + { + "sessionId": "01a10dc0-7ea4-74d8-9b87-9a5026c7b05c", + "path": "/src/example/.kimchi/plans/search.md", + "snapshotPath": "/home/user/.config/kimchi/harness/work/7a203360-39a9-4d6e-8ac7-984c97c0e314/plans/search-3d4e5f.md", + "contentHash": "9999999999999999999999999999999999999999999999999999999999999999", + "requestId": "00000001-3333-4333-8333-333333333333", + "cwd": "/src/example", + "recordedAt": "2026-10-01T09:00:00.000Z" + } + ], + "commits": [ + { + "sha": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "repository": "/src/example/.git", + "worktree": "/src/example", + "sessionId": "01a10dc0-7ea4-74d8-9b87-9a5026c7b05c", + "cwd": "/src/example", + "recordedAt": "2026-10-02T11:00:00.000Z", + "source": "bash-git-trace-commits", + "toolCallId": "call_9", + "requestId": "00000002-3333-4333-8333-333333333333", + "paths": ["src/search.ts"], + "transitionIds": ["00000003-2222-4222-8222-222222222222"], + "fileMatches": [ + { + "path": "src/search.ts", + "worktree": "/src/example", + "method": "file-hunks", + "transitionIds": ["00000003-2222-4222-8222-222222222222"] + } + ], + "pullRequests": [ + { + "provider": "github", + "host": "github.com", + "id": "5731", + "repositoryId": "901", + "repository": "example/kimchi", + "number": 731, + "url": "https://github.com/example/kimchi/pull/731", + "state": "open", + "headSha": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "mergeCommitSha": null, + "mergedAt": null, + "closedAt": null, + "checkedAt": "2026-10-03T12:00:00Z" + } + ], + "prLookup": { + "status": "linked", + "checkedAt": "2026-10-03T12:00:00Z" + } + } + ], + "fileTransitions": [ + { + "transitionId": "00000003-2222-4222-8222-222222222222", + "toolCallId": "call_3", + "requestId": "00000003-1111-4111-8111-111111111111", + "repository": "/src/example/.git", + "worktree": "/src/example", + "path": "src/file3.ts", + "baseline": null, + "baselineFile": null, + "branch": "feat/search", + "before": null, + "after": { + "blob": "3333333333333333333333333333333333333333", + "mode": "100644" + }, + "cursor": { + "bytes": 0, + "digest": "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc" + }, + "historyBoundaryId": "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", + "sessionId": "01a10dc0-7ea4-74d8-9b87-9a5026c7b05c", + "cwd": "/src/example", + "recordedAt": "2026-09-03T10:05:00.000Z" + } + ], + "fileObservations": [ + { + "observationId": "99999999-2222-4333-8444-555555555555", + "source": "bash", + "toolCallId": "call_8", + "requestId": "00000002-3333-4333-8333-333333333333", + "repository": "/src/example/.git", + "worktree": "/src/example", + "startedAt": "2026-10-02T10:30:00.000Z", + "complete": true, + "files": [ + { + "path": "README.md", + "before": null, + "after": { + "blob": "1111111111111111111111111111111111111111", + "mode": "100644" + } + } + ], + "sessionId": "01a10dc0-7ea4-74d8-9b87-9a5026c7b05c", + "cwd": "/src/example", + "recordedAt": "2026-10-02T10:30:01.000Z" + } + ], + "continuations": [ + { + "sessionId": "02b20dc0-7ea4-74d8-9b87-9a5026c7b05c", + "cwd": "/src/example", + "recordedAt": "2026-10-03T08:00:00.000Z", + "source": "pasted-plan", + "evidence": { + "path": "/home/user/.config/kimchi/harness/work/7a203360-39a9-4d6e-8ac7-984c97c0e314/plans/search-3d4e5f.md", + "contentHash": "9999999999999999999999999999999999999999999999999999999999999999", + "segmentId": "00000003-4444-4444-8444-444444444444" + } + } + ] +} diff --git a/src/extensions/work-attribution/__fixtures__/work-summary-v1.7.1.json b/src/extensions/work-attribution/__fixtures__/work-summary-v1.7.1.json new file mode 100644 index 000000000..da30a1f0f --- /dev/null +++ b/src/extensions/work-attribution/__fixtures__/work-summary-v1.7.1.json @@ -0,0 +1,127 @@ +{ + "version": 1, + "workId": "6f102360-39a9-4d6e-8ac7-984c97c0e313", + "sessions": ["01a10dc0-7ea4-74d8-9b87-9a5026c7b05c", "02b20dc0-7ea4-74d8-9b87-9a5026c7b05c"], + "requests": [ + { + "requestId": "00000001-1111-4111-8111-111111111111", + "sessionId": "01a10dc0-7ea4-74d8-9b87-9a5026c7b05c", + "cwd": "/src/example", + "provider": "kimchi-dev", + "model": "glm-5.3", + "modelSource": "context", + "recordedAt": "2026-09-01T10:00:00.000Z" + }, + { + "requestId": "00000002-1111-4111-8111-111111111111", + "sessionId": "01a10dc0-7ea4-74d8-9b87-9a5026c7b05c", + "cwd": "/src/example", + "provider": "kimchi-dev", + "model": "glm-5.3", + "modelSource": "context", + "recordedAt": "2026-09-02T10:00:00.000Z" + }, + { + "requestId": "00000003-1111-4111-8111-111111111111", + "sessionId": "02b20dc0-7ea4-74d8-9b87-9a5026c7b05c", + "cwd": "/src/example", + "provider": "kimchi-dev", + "model": "glm-5.3", + "modelSource": "context", + "recordedAt": "2026-09-03T10:00:00.000Z" + } + ], + "plans": [ + { + "sessionId": "01a10dc0-7ea4-74d8-9b87-9a5026c7b05c", + "path": "/src/example/.kimchi/plans/search.md", + "snapshotPath": "/home/user/.config/kimchi/harness/work/6f102360-39a9-4d6e-8ac7-984c97c0e313/plans/search-0a1b2c.md", + "cwd": "/src/example", + "recordedAt": "2026-09-01T09:00:00.000Z" + } + ], + "commits": [ + { + "sha": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "repository": "/src/example/.git", + "worktree": "/src/example", + "sessionId": "01a10dc0-7ea4-74d8-9b87-9a5026c7b05c", + "cwd": "/src/example", + "recordedAt": "2026-09-02T11:00:00.000Z", + "source": "bash-git-trace-commits", + "paths": ["src/file1.ts"], + "transitionIds": ["00000001-2222-4222-8222-222222222222"], + "fileMatches": [ + { + "path": "src/file1.ts", + "worktree": "/src/example", + "method": "file-chain", + "transitionIds": ["00000001-2222-4222-8222-222222222222"] + } + ] + } + ], + "fileTransitions": [ + { + "transitionId": "00000001-2222-4222-8222-222222222222", + "toolCallId": "call_1", + "requestId": "00000001-1111-4111-8111-111111111111", + "repository": "/src/example/.git", + "worktree": "/src/example", + "path": "src/file1.ts", + "baseline": null, + "baselineFile": null, + "branch": "feat/search", + "before": null, + "after": { + "blob": "1111111111111111111111111111111111111111", + "mode": "100644" + }, + "cursor": { + "bytes": 0, + "digest": "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc" + }, + "historyBoundaryId": "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", + "sessionId": "01a10dc0-7ea4-74d8-9b87-9a5026c7b05c", + "cwd": "/src/example", + "recordedAt": "2026-09-01T10:05:00.000Z" + }, + { + "transitionId": "00000002-2222-4222-8222-222222222222", + "toolCallId": "call_2", + "requestId": "00000002-1111-4111-8111-111111111111", + "repository": "/src/example/.git", + "worktree": "/src/example", + "path": "src/file2.ts", + "baseline": null, + "baselineFile": null, + "branch": "main", + "before": null, + "after": { + "blob": "2222222222222222222222222222222222222222", + "mode": "100644" + }, + "cursor": { + "bytes": 0, + "digest": "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc" + }, + "historyBoundaryId": "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", + "sessionId": "02b20dc0-7ea4-74d8-9b87-9a5026c7b05c", + "cwd": "/src/example", + "recordedAt": "2026-09-02T10:05:00.000Z" + } + ], + "continuations": [ + { + "sessionId": "02b20dc0-7ea4-74d8-9b87-9a5026c7b05c", + "cwd": "/src/example", + "recordedAt": "2026-09-03T08:00:00.000Z", + "source": "recent-branch", + "evidence": { + "path": "/src/example/src/file1.ts", + "transitionId": "00000001-2222-4222-8222-222222222222", + "branch": "feat/search" + } + } + ] +} diff --git a/src/extensions/work-attribution/commits.test.ts b/src/extensions/work-attribution/commits.test.ts index caa4794ca..7c4e63592 100644 --- a/src/extensions/work-attribution/commits.test.ts +++ b/src/extensions/work-attribution/commits.test.ts @@ -464,6 +464,46 @@ await flushWorkSummaries();` ]) }, 15000) + it("follows another session's commit of the same work through a rebase", async () => { + vi.stubEnv("PI_CODING_AGENT_DIR", join(directory, "agent")) + const first = createContext({ cwd: repository, sessionManager: { getSessionId: () => "first" } }) + const workId = attribution.getWorkId(first) + git(repository, "commit", "--allow-empty", "-qm", "base") + await createCommitTrackingBashTool(first).execute( + "first-tool", + { command: "git checkout -qb feat && echo ours > ours.txt && git add ours.txt && git commit -qm ours" }, + undefined, + undefined, + first, + ) + const ours = git(repository, "rev-parse", "HEAD") + for (let index = 0; index < 50; index++) + attribution.appendWorkRecord(first, { type: "request", requestId: `r${index}` }) + await flushWorkSummaries() + git(repository, "checkout", "-q", "main") + git(repository, "commit", "--allow-empty", "-qm", "upstream") + git(repository, "checkout", "-q", "feat") + // A later session of the same work only knows the commit from the work's summary. + const second = createContext({ cwd: repository, sessionManager: { getSessionId: () => "second" } }) + attribution.setWorkId(second, workId) + await createCommitTrackingBashTool(second).execute( + "second-tool", + { command: "git rebase -q main" }, + undefined, + undefined, + second, + ) + const recorded = fs + .readFileSync(attribution.workLedgerPath(second), "utf8") + .trim() + .split("\n") + .map((line) => JSON.parse(line)) + .filter((row) => row.type === "commit") + expect(recorded).toEqual([ + expect.objectContaining({ sha: git(repository, "rev-parse", "HEAD"), rewrittenFrom: ours, workId }), + ]) + }) + it("records same-work cherry-picks, reverts and new merge commits", async () => { vi.stubEnv("PI_CODING_AGENT_DIR", join(directory, "agent")) const ctx = createContext({ cwd: repository, sessionManager: { getSessionId: () => "sequencer" } }) diff --git a/src/extensions/work-attribution/commits.ts b/src/extensions/work-attribution/commits.ts index 9144ce025..3afce3e5d 100644 --- a/src/extensions/work-attribution/commits.ts +++ b/src/extensions/work-attribution/commits.ts @@ -21,6 +21,7 @@ import { } from "../work-attribution.js" import { debugWorkAttribution } from "./diagnostics.js" import { observeToolFiles } from "./file-observations.js" +import { readWorkSummary } from "./row-log.js" import { COST_PER_PR_RESOURCE_ID } from "./resource.js" const GIT_LOOKUP_TIMEOUT_MS = 2000 @@ -310,7 +311,8 @@ function recordedCommits(ctx: WorkContext, workId: string): Set { } catch {} } try { - for (const row of JSON.parse(readText(join(getAgentDir(), "work", workId, "work.json"))).commits) shas.add(row.sha) + for (const row of readWorkSummary(getAgentDir(), workId)?.commits ?? []) + if (typeof row.sha === "string") shas.add(row.sha) } catch {} return shas } diff --git a/src/extensions/work-attribution/continuation.integration.test.ts b/src/extensions/work-attribution/continuation.integration.test.ts index ee5c5c5f2..dad5635db 100644 --- a/src/extensions/work-attribution/continuation.integration.test.ts +++ b/src/extensions/work-attribution/continuation.integration.test.ts @@ -19,6 +19,7 @@ import { savePlanMarkdown } from "../../shared/planning/plan-markdown.js" import { createContext } from "../__mocks__/context.js" import { createExtensionApi } from "../__mocks__/extension-api.js" import { createWorkScopeSnapshot } from "../__mocks__/work-scope.js" +import { savedWorkSummary } from "../__mocks__/work-summary.js" import { appendWorkRecord, createWorkAttributionExtension, getWorkId, getWorkSegment } from "../work-attribution.js" import { createCommitTrackingBashTool } from "./commits.js" import { findWorkContinuation } from "./continuation.js" @@ -92,13 +93,12 @@ it.each([0, 7])("records a candidate observation for an ADR written by Bash befo }), ) await flushWorkSummaries() - const summaryPath = join(agentDir, "work", workId, "work.json") - const saved = JSON.parse(readFileSync(summaryPath, "utf8")) + const saved = savedWorkSummary(agentDir, workId) expect(saved.fileObservations).toContainEqual(expect.objectContaining({ toolCallId, source: "bash" })) - rmSync(summaryPath) + rmSync(join(agentDir, "work", workId, "work.json")) recoverWorkSummaries() await flushWorkSummaries() - expect(JSON.parse(readFileSync(summaryPath, "utf8")).fileObservations).toEqual(saved.fileObservations) + expect(savedWorkSummary(agentDir, workId).fileObservations).toEqual(saved.fileObservations) expect(await findWorkContinuation({ cwd }, `Implement ${path}`, captured)).toEqual({ owned: false }) }) @@ -213,7 +213,7 @@ it.each(["interactive", "rpc"] as const)("adopts a pasted plan before the first expect.objectContaining({ type: "request", requestId: request.headers["X-Request-Id"], workId }), ) await flushWorkSummaries() - const summary = JSON.parse(readFileSync(join(agentDir, "work", workId, "work.json"), "utf8")) + const summary = savedWorkSummary(agentDir, workId) expect(summary.continuations).toContainEqual( expect.objectContaining({ sessionId: "pasted-session", diff --git a/src/extensions/work-attribution/cost-sync.test.ts b/src/extensions/work-attribution/cost-sync.test.ts index 939fa675e..87e17b9a6 100644 --- a/src/extensions/work-attribution/cost-sync.test.ts +++ b/src/extensions/work-attribution/cost-sync.test.ts @@ -7,6 +7,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest" import * as json from "../../config/json.js" import * as config from "../../config.js" import { createContext } from "../__mocks__/context.js" +import { savedWorkSummary } from "../__mocks__/work-summary.js" import { appendWorkRecord, getWorkId } from "../work-attribution.js" import { captureBillingSource, requestTagSelector } from "./billing-source.js" import { workCostDetails } from "./cost-details.js" @@ -598,12 +599,11 @@ describe("automatic exact work cost lookup", () => { const expected = { id: ROW, costUsd: "0.123456789", ...metadata } expect(readWorkRecords(dir).find((row) => row.type === "request_cost")?.billingRows).toEqual([expected]) await flushWorkSummaries() - const path = join(dir, "work", workId, "work.json") - expect(JSON.parse(readFileSync(path, "utf8")).requests[0].billingRows).toEqual([expected]) - writeFileSync(path, "damaged") + expect(savedWorkSummary(dir, workId).requests[0].billingRows).toEqual([expected]) + writeFileSync(join(dir, "work", workId, "work.json"), "damaged") recoverWorkSummaries() await flushWorkSummaries() - expect(JSON.parse(readFileSync(path, "utf8")).requests[0].billingRows).toEqual([expected]) + expect(savedWorkSummary(dir, workId).requests[0].billingRows).toEqual([expected]) expect(report(workId).pullRequests[0].totalCostUsd).toBe("0.123456789") expect(JSON.stringify(readWorkRecords(dir))).not.toContain("DO_NOT_SAVE") }) @@ -686,7 +686,7 @@ describe("automatic exact work cost lookup", () => { fetchMock.mockResolvedValueOnce(Response.json({ items: [enriched, enriched], totalCount: 1 })) await sync() await flushWorkSummaries() - const rows = JSON.parse(readFileSync(join(dir, "work", workId, "work.json"), "utf8")).requests[0].billingRows + const rows = savedWorkSummary(dir, workId).requests[0].billingRows expect(rows).toEqual([{ id: ROW, costUsd: "0.123456789", promptTokens: "100", cacheReadPrice: "0" }]) expect(report(workId).pullRequests[0].totalCostUsd).toBe("0.123456789") vi.mocked(Date.now).mockReturnValue(now + 2 * RECHECK_MS) @@ -694,9 +694,7 @@ describe("automatic exact work cost lookup", () => { fetchMock.mockResolvedValueOnce(Response.json({ items: [enriched], totalCount: 1 })) await sync() await flushWorkSummaries() - expect(JSON.parse(readFileSync(join(dir, "work", workId, "work.json"), "utf8")).requests[0].billingRows).toEqual( - rows, - ) + expect(savedWorkSummary(dir, workId).requests[0].billingRows).toEqual(rows) expect(report(workId).pullRequests[0].totalCostUsd).toBe("0.123456789") }) it.each([ @@ -712,7 +710,7 @@ describe("automatic exact work cost lookup", () => { else fetchMock.mockResolvedValueOnce(new Response(null, { status: 503 })) await sync() await flushWorkSummaries() - const request = JSON.parse(readFileSync(join(dir, "work", workId, "work.json"), "utf8")).requests[0] + const request = savedWorkSummary(dir, workId).requests[0] // A failed key check returns no billing page, and a changed key cannot withdraw a verified price. expect(request.billingLookup).toMatchObject({ organizationId: ORG, @@ -1234,12 +1232,11 @@ describe("automatic exact work cost lookup", () => { const { workId } = tagged() await sync() await flushWorkSummaries() - const path = join(dir, "work", workId, "work.json") - const before = JSON.parse(readFileSync(path, "utf8")) - writeFileSync(path, "broken") + const before = savedWorkSummary(dir, workId) + writeFileSync(join(dir, "work", workId, "work.json"), "broken") recoverWorkSummaries() await flushWorkSummaries() - expect(JSON.parse(readFileSync(path, "utf8"))).toEqual(before) + expect(savedWorkSummary(dir, workId)).toEqual(before) }) it("bounds requests per pass and visits the remaining requests on the next pass", async () => { for (let i = 0; i < 35; i++) tagged(`session-${i}`) @@ -1454,7 +1451,7 @@ describe("automatic exact work cost lookup", () => { expect(fetchMock.mock.calls[1][1]?.body).toBeUndefined() expect(report(workId).pullRequests[0]).toMatchObject({ knownCostUsd: "0.123456789", totalCostUsd: "0.123456789" }) await flushWorkSummaries() - const summary = JSON.parse(readFileSync(join(dir, "work", workId, "work.json"), "utf8")) + const summary = savedWorkSummary(dir, workId) expect(summary.requests).toHaveLength(1) expect(summary.requests[0].billingRows).toEqual([{ id: ROW, costUsd: "0.123456789" }]) expect(JSON.stringify(readWorkRecords(dir))).not.toContain(currentKey) diff --git a/src/extensions/work-attribution/file-transitions.test.ts b/src/extensions/work-attribution/file-transitions.test.ts index 9dff3d075..bbef529f4 100644 --- a/src/extensions/work-attribution/file-transitions.test.ts +++ b/src/extensions/work-attribution/file-transitions.test.ts @@ -24,6 +24,7 @@ import type { import { afterEach, beforeEach, describe, expect, it, vi } from "vitest" import { createContext } from "../__mocks__/context.js" import { createExtensionApi } from "../__mocks__/extension-api.js" +import { savedWorkSummary } from "../__mocks__/work-summary.js" import toolRenderingExtension from "../tool-rendering.js" import { createWorkAttributionExtension, getWorkId, setWorkId } from "../work-attribution.js" import { createCommitTrackingBashTool } from "./commits.js" @@ -510,8 +511,8 @@ describe("manual commit reconciliation", () => { }), ]) await flushWorkSummaries() - const summary = JSON.parse(readFileSync(join(root, "agent", "work", workId, "work.json"), "utf8")) - expect(summary.commits.map((row: { sessionId: string }) => row.sessionId).sort()).toEqual(["first", "second"]) + const summary = savedWorkSummary(join(root, "agent"), workId) + expect(summary.commits.map((row) => row.sessionId).sort()).toEqual(["first", "second"]) }) it("retains both sessions when same-work edits compose into one committed file", async () => { @@ -537,8 +538,8 @@ describe("manual commit reconciliation", () => { expect(match.transitionIds).toEqual(expected) } await flushWorkSummaries() - const summary = JSON.parse(readFileSync(join(root, "agent", "work", workId, "work.json"), "utf8")) - expect(summary.commits.map((row: { sessionId: string }) => row.sessionId).sort()).toEqual(["first", "second"]) + const summary = savedWorkSummary(join(root, "agent"), workId) + expect(summary.commits.map((row) => row.sessionId).sort()).toEqual(["first", "second"]) }) it("rechecks completed commits when additional transition evidence becomes available", async () => { diff --git a/src/extensions/work-attribution/row-log.test.ts b/src/extensions/work-attribution/row-log.test.ts new file mode 100644 index 000000000..6ab03e3e5 --- /dev/null +++ b/src/extensions/work-attribution/row-log.test.ts @@ -0,0 +1,783 @@ +import { spawn } from "node:child_process" +import { randomUUID } from "node:crypto" +import * as fs from "node:fs" +import * as asyncFs from "node:fs/promises" +import { tmpdir } from "node:os" +import { basename, join } from "node:path" +import { performance } from "node:perf_hooks" +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest" +import { createContext } from "../__mocks__/context.js" +import { appendWorkRecord } from "../work-attribution.js" +import { + COLLECTIONS, + type Collection, + commitPullRequests, + logPath, + ROW_KEYS, + readWorkHead, + readWorkRows, + readWorkSummary, + summaryLatest, + type WorkHead, + type WorkSummaryView, +} from "./row-log.js" +import { flushWorkSummaries, mergeWorkRecords, updateWorkSummary, type WorkRecord } from "./summary.js" + +vi.mock("node:fs/promises", async (importOriginal) => ({ ...(await importOriginal()) })) +vi.mock("node:fs", async (importOriginal) => ({ ...(await importOriginal()) })) + +let dir: string +beforeEach(() => { + dir = fs.mkdtempSync(join(tmpdir(), "kimchi-row-log-")) + vi.stubEnv("PI_CODING_AGENT_DIR", dir) +}) +afterEach(async () => { + await flushWorkSummaries() + vi.restoreAllMocks() + vi.unstubAllEnvs() + fs.rmSync(dir, { recursive: true, force: true }) +}) + +const BASE = Date.parse("2026-10-01T00:00:00.000Z") +const iso = (at: number) => new Date(at).toISOString() +function folder(workId: string): string { + return join(dir, "work", workId) +} +function manifest(workId: string): WorkHead { + return JSON.parse(fs.readFileSync(join(folder(workId), "work.json"), "utf8")) +} +function writeVersionOne(summary: WorkSummaryView): void { + fs.mkdirSync(folder(summary.workId), { recursive: true }) + fs.writeFileSync(join(folder(summary.workId), "work.json"), `${JSON.stringify(summary, null, 2)}\n`) +} +function fixture(name: string): WorkSummaryView { + return JSON.parse(fs.readFileSync(new URL(`./__fixtures__/${name}`, import.meta.url), "utf8")) +} +function emptySummary(workId: string): WorkSummaryView { + return { + version: 1, + workId, + sessions: [], + workLinks: [], + requests: [], + plans: [], + commits: [], + fileTransitions: [], + fileObservations: [], + continuations: [], + } +} +/** The version 1 shape with every collection, in the order the row logs fold into. */ +function withDefaults(value: WorkSummaryView): WorkSummaryView { + const empty = emptySummary(value.workId) + return { + ...empty, + sessions: value.sessions, + ...Object.fromEntries(COLLECTIONS.map((collection) => [collection, value[collection] ?? []])), + } +} + +/** How version 1 applied a batch: parse the whole summary, merge in memory, serialize it again. */ +async function versionOneUpdate(summary: WorkSummaryView, records: WorkRecord[]): Promise { + const copy = withDefaults(JSON.parse(JSON.stringify(summary))) + const keyed = (collection: Collection) => new Map(copy[collection].map((row) => [ROW_KEYS[collection](row), row])) + const rows = { + sessions: new Set(copy.sessions), + workLinks: keyed("workLinks"), + requests: keyed("requests"), + plans: keyed("plans"), + commits: keyed("commits"), + fileTransitions: keyed("fileTransitions"), + fileObservations: keyed("fileObservations"), + continuations: keyed("continuations"), + } + await mergeWorkRecords(rows, structuredClone(records)) + return JSON.parse( + JSON.stringify({ + version: 1, + workId: copy.workId, + sessions: [...rows.sessions], + ...Object.fromEntries(COLLECTIONS.map((collection) => [collection, [...rows[collection].values()]])), + }), + ) +} + +/** Feeds one batch through the real summary writer, as records appended in one tick are. */ +async function apply(records: WorkRecord[]): Promise { + for (const record of structuredClone(records)) updateWorkSummary(record) + await flushWorkSummaries() +} + +/** The committed files match the expected version 1 view, and no stray row file is left. */ +function expectCommitted(workId: string, expected: WorkSummaryView, { latest = true } = {}): WorkHead { + const head = manifest(workId) + expect(head.version).toBe(2) + expect(JSON.stringify(readWorkSummary(dir, workId))).toBe(JSON.stringify(expected)) + const named: string[] = [] + for (const collection of COLLECTIONS) { + const log = head.logs[collection] + if (!log) { + expect(expected[collection]).toEqual([]) + continue + } + const path = logPath(folder(workId), collection, log.generation) + named.push(basename(path)) + const bytes = fs.readFileSync(path) + expect(bytes.length).toBe(log.bytes) + expect(bytes[log.bytes - 1]).toBe(10) + expect(log.rows).toBe(expected[collection].length) + } + expect(fs.readdirSync(join(folder(workId), "rows")).sort()).toEqual(named.sort()) + expect(head.sessions).toEqual(expected.sessions) + expect(head.pullRequests).toEqual(commitPullRequests(expected.commits)) + if (latest) expect(head.latest).toEqual(summaryLatest((collection) => expected[collection]).latest) + return head +} + +/** Deterministic records over every type, with repeated keys, PR updates, billing unions and replays. */ +function randomRecords(seed: number, workId: string, count: number, { replays = false, large = false } = {}) { + let state = seed + const random = () => { + state = (state + 0x6d2b79f5) | 0 + let value = Math.imul(state ^ (state >>> 15), 1 | state) + value = (value + Math.imul(value ^ (value >>> 7), 61 | value)) ^ value + return ((value ^ (value >>> 14)) >>> 0) / 4294967296 + } + const pick = (values: readonly T[]): T => values[Math.floor(random() * values.length)] + const requestIds = Array.from({ length: 12 }, (_, index) => `${String(index).padStart(8, "0")}-1111-4111-8111-1111`) + const started = new Map() + const billingRows = [ + { id: "row-a", costUsd: "0.1" }, + { id: "row-a", costUsd: "0.1", promptTokens: "10" }, + { id: "row-b", costUsd: null }, + ] + const records: WorkRecord[] = [] + for (let index = 0; index < count; index++) { + const at = BASE + index * 1000 + if (replays && records.length && random() < 0.15) { + records.push(structuredClone(pick(records))) + continue + } + const base = { version: 1 as const, workId, sessionId: pick(["s1", "s2", "s3"]), cwd: "/tree", recordedAt: iso(at) } + const requestId = pick(requestIds) + const startedAt = started.get(requestId) ?? iso(at) + started.set(requestId, startedAt) + const type = pick([ + "work", + "work_link", + "request", + "request", + "request_dispatch", + "request_response", + "request_cost", + "request_cost", + "plan", + "commit", + "commit", + "file_transition", + "file_observation", + ] as const) + switch (type) { + case "work": + records.push({ + ...base, + type, + ...(random() < 0.6 + ? { + continuation: { + source: pick(["recent-branch", "named-artifact"]), + evidence: { path: pick(["/a.md", "/b.md"]) }, + }, + } + : { segment: { id: randomUUID(), attribution: "explicit", reason: "work-command" } }), + }) + break + case "work_link": + records.push({ + ...base, + type, + linkId: pick(["link-1", "link-2"]), + revision: pick([1, 2]), + sourceWorkId: workId, + targetWorkId: workId, + requestIds: [requestId], + scope: { repository: "/repo/.git" }, + status: pick(["active", "revoked"]), + evidence: { source: "work-command" }, + }) + break + case "request": + records.push({ + ...base, + type, + requestId, + startedAt, + model: pick(["a", "b"]), + scope: { repository: pick(["/repo/.git", "/other/.git"]) }, + ...(large && requestIds.indexOf(requestId) < 3 ? { note: "x".repeat(60_000) } : {}), + }) + break + case "request_dispatch": + records.push({ ...base, type, requestId, startedAt, dispatchedAt: iso(at + 3), billingTagSkipped: "fixture" }) + break + case "request_response": + records.push({ + ...base, + type, + requestId, + startedAt, + response: { status: pick([200, 503]), receivedAt: iso(at) }, + }) + break + case "request_cost": + records.push({ + ...base, + type, + requestId, + billingRows: billingRows.filter(() => random() < 0.5), + billingLookup: { + status: pick(["priced", "pending"]), + checkedAt: pick([iso(at), iso(at - 50_000), "invalid"]), + }, + }) + break + case "plan": + records.push({ + ...base, + type, + path: pick(["/tree/.kimchi/plans/a.md", "/tree/.kimchi/plans/b.md"]), + snapshotPath: pick(["/work/plans/a-1.md", "/work/plans/a-2.md", "/work/plans/b-1.md"]), + }) + break + case "commit": { + const number = pick([7, 8]) + records.push({ + ...base, + type, + sha: pick(["a", "b", "c", "d"]).repeat(40), + repository: pick(["/repo/.git", "/other/.git"]), + worktree: "/tree", + ...(random() < 0.5 + ? { + pullRequests: [ + { + provider: "github", + host: "github.com", + id: `70${number}`, + url: `https://github.com/example/kimchi/pull/${number}`, + number, + state: pick(["open", "merged"]), + checkedAt: iso(at - Math.floor(random() * 5) * 1000), + }, + ], + prLookup: { status: "linked", checkedAt: iso(at) }, + } + : {}), + ...(random() < 0.5 + ? { + fileMatches: [ + { + path: pick(["a.ts", "b.ts"]), + worktree: "/tree", + method: pick(["file-chain", "path-blob", "file-hunks"]), + transitionIds: [pick(["t1", "t2", "t3"])], + }, + ], + } + : {}), + ...(random() < 0.5 ? { paths: [pick(["a.ts", "b.ts"])], transitionIds: [pick(["t1", "t2"])] } : {}), + }) + break + } + case "file_transition": + records.push({ + ...base, + type, + transitionId: pick(["t1", "t2", "t3", "t4", "t5"]), + toolCallId: "call", + repository: pick(["/repo/.git", "/other/.git"]), + worktree: "/tree", + path: "a.ts", + ...(random() < 0.7 ? { branch: pick(["main", "feat/x"]) } : {}), + }) + break + case "file_observation": + records.push({ + ...base, + type, + observationId: pick(["o1", "o2"]), + toolCallId: "call", + repository: "/repo/.git", + worktree: "/tree", + source: "bash", + files: [{ path: pick(["a.ts", "b.ts"]) }], + complete: random() < 0.5, + }) + break + } + } + const batches: WorkRecord[][] = [] + for (let index = 0; index < records.length; ) { + const size = 1 + Math.floor(random() * 6) + batches.push(records.slice(index, index + size)) + index += size + } + return batches +} + +describe("version 1 summaries", () => { + it.each([ + "work-summary-v1.7.1.json", + "work-summary-v1-stack.json", + "work-summary-v1-legacy.json", + ])("reads %s unchanged and migrates it on the next update", async (name) => { + const saved = fixture(name) + writeVersionOne(saved) + const { workId } = saved + expect(readWorkSummary(dir, workId)).toEqual(withDefaults(saved)) + expect(readWorkRows(dir, workId, "commits")).toEqual(saved.commits) + const head = await readWorkHead(dir, workId) + expect(head).toMatchObject({ version: 2, logs: { requests: { rows: saved.requests.length } } }) + const existing = saved.requests[0] + const batch: WorkRecord[] = [ + { + version: 1, + type: "request_response", + workId, + sessionId: existing.sessionId, + requestId: String(existing.requestId), + response: { status: 200, receivedAt: "2026-10-08T12:00:00.000Z" }, + recordedAt: "2026-10-08T12:00:00.000Z", + }, + { + version: 1, + type: "request", + workId, + sessionId: "new-session", + requestId: "new-request", + startedAt: "2026-10-08T12:00:01.000Z", + recordedAt: "2026-10-08T12:00:01.000Z", + }, + ] + await apply(batch) + const expected = await versionOneUpdate(saved, batch) + expectCommitted(workId, expected) + expect(readWorkSummary(dir, workId)?.requests.at(-1)).toMatchObject({ requestId: "new-request" }) + }) + + it("reads a version 1 summary that grew past the listing limit only as too large", async () => { + const saved = fixture("work-summary-v1.7.1.json") + writeVersionOne(saved) + expect(await readWorkHead(dir, saved.workId, 100)).toBe("too-large") + expect(readWorkSummary(dir, saved.workId)).toEqual(withDefaults(saved)) + }) +}) + +describe("row logs", () => { + it.each<[number, { replays?: boolean; large?: boolean }]>([ + [1, {}], + [2, { replays: true }], + [3, { large: true }], + [4, { replays: true, large: true }], + ])( + "fold to the version 1 merge of random records (seed %i, %o)", + async (seed, options) => { + const workId = randomUUID() + const batches = randomRecords(seed, workId, 360, options) + let expected = await versionOneUpdate(emptySummary(workId), batches.slice(0, 5).flat()) + writeVersionOne(expected) + let compactions = 0 + let previous: WorkHead | undefined + for (const [index, batch] of batches.slice(5).entries()) { + let migrated = !previous + if (index === 30) { + // An older release rewrote the summary as version 1; its logs are left behind. + writeVersionOne(expected) + migrated = true + } + await apply(batch) + expected = await versionOneUpdate(expected, batch) + const generations = (head?: WorkHead) => + COLLECTIONS.map((collection) => head?.logs[collection]?.generation ?? 0) + // Replayed older records can move a row's time back; /work keeps the newer value until a recompute. + const head = expectCommitted(workId, expected, { latest: !options.replays || migrated }) + const advanced = generations(head).some((generation, at) => generation > generations(previous)[at]) + if (advanced && !migrated && previous) { + compactions++ + // Compaction recomputes what /work lists from every row. + expect(head.latest).toEqual(summaryLatest((collection) => expected[collection]).latest) + } + previous = head + } + if (options.large) expect(compactions).toBeGreaterThan(0) + }, + 60_000, + ) + + it("ignores an interrupted append and drops it on the next update", async () => { + const workId = randomUUID() + const [first, second] = randomRecords(5, workId, 40) + .flat() + .reduce( + (halves, record, index) => { + halves[index < 20 ? 0 : 1].push(record) + return halves + }, + [[], []], + ) + await apply(first) + const expected = await versionOneUpdate(emptySummary(workId), first) + const head = manifest(workId) + for (const collection of COLLECTIONS) { + const log = head.logs[collection] + if (log) + fs.appendFileSync(logPath(folder(workId), collection, log.generation), '{"sessionId":"cut","requestId":"x') + } + expect(readWorkSummary(dir, workId)).toEqual(expected) + await apply(second) + expectCommitted(workId, await versionOneUpdate(expected, second)) + }) + + it.each(["missing", "short", "garbled"])("rebuilds a work from its journals when a log is %s", async (damage) => { + const ctx = createContext({ cwd: "/project", sessionManager: { getSessionId: () => "journal" } }) + const workId = randomUUID() + for (let index = 0; index < 5; index++) + appendWorkRecord(ctx, { type: "request", requestId: `request-${index}`, startedAt: iso(BASE + index) }, workId) + appendWorkRecord( + ctx, + { type: "commit", sha: "a".repeat(40), repository: "/project/.git", worktree: "/project" }, + workId, + ) + await flushWorkSummaries() + const before = readWorkSummary(dir, workId) + const log = manifest(workId).logs.requests + if (!log) throw new Error("requests were not saved") + const path = logPath(folder(workId), "requests", log.generation) + if (damage === "missing") fs.rmSync(path) + if (damage === "short") fs.truncateSync(path, log.bytes - 5) + if (damage === "garbled") fs.writeFileSync(path, "x".repeat(log.bytes)) + // Readers report damage instead of a partial work. + expect(readWorkSummary(dir, workId)).toBeUndefined() + appendWorkRecord(ctx, { type: "request", requestId: "after-damage" }, workId) + await flushWorkSummaries() + const rebuilt = readWorkSummary(dir, workId) + expect(rebuilt?.requests.map((row) => row.requestId)).toEqual([ + ...(before?.requests.map((row) => row.requestId) ?? []), + "after-damage", + ]) + expect(rebuilt?.commits).toEqual(before?.commits) + expect(manifest(workId).logs.requests?.generation).toBeGreaterThan(log.generation) + expect(fs.readdirSync(join(folder(workId), "rows"))).not.toContain(basename(path)) + }) + + it("rebuilds from the journals when work.json is deleted and removes the old logs", async () => { + const ctx = createContext({ cwd: "/project", sessionManager: { getSessionId: () => "journal" } }) + const workId = randomUUID() + appendWorkRecord(ctx, { type: "request", requestId: "kept" }, workId) + await flushWorkSummaries() + const old = fs.readdirSync(join(folder(workId), "rows")) + fs.rmSync(join(folder(workId), "work.json")) + appendWorkRecord(ctx, { type: "request", requestId: "next" }, workId) + await flushWorkSummaries() + expect(readWorkSummary(dir, workId)?.requests.map((row) => row.requestId)).toEqual(["kept", "next"]) + expect(fs.readdirSync(join(folder(workId), "rows")).some((name) => old.includes(name))).toBe(false) + }) + + it("removes a compacted generation that a crash left uncommitted", async () => { + const workId = randomUUID() + const batches = randomRecords(3, workId, 300, { large: true }) + const records = batches.flat() + const original = asyncFs.rename + let renames = 0 + // The second rename of an update is the compaction's manifest: fail it once, like a crash would. + vi.spyOn(asyncFs, "rename").mockImplementation(async (...args) => { + if (String(args[1]).endsWith("work.json") && ++renames === 2) throw new Error("injected crash") + return original(...args) + }) + let expected = emptySummary(workId) + for (const batch of batches) { + renames = 0 + await apply(batch) + expected = await versionOneUpdate(expected, batch) + if (fs.readdirSync(join(folder(workId), "rows")).length > Object.keys(manifest(workId).logs).length) break + } + const stray = fs + .readdirSync(join(folder(workId), "rows")) + .filter( + (name) => + !Object.entries(manifest(workId).logs).some( + ([collection, log]) => name === `${collection}.${log.generation}.jsonl`, + ), + ) + expect(stray).toHaveLength(1) + // Committed rows survive; only the uncommitted copy is extra. + expect(JSON.stringify(readWorkSummary(dir, workId))).toBe(JSON.stringify(expected)) + vi.mocked(asyncFs.rename).mockRestore() + const next = records.slice(0, 1) + await apply(next) + expectCommitted(workId, await versionOneUpdate(expected, next)) + }, 60_000) + + it("starts a fresh generation after an older release rewrote the summary as version 1", async () => { + const workId = randomUUID() + const [first, ...rest] = randomRecords(6, workId, 30) + await apply(first) + const old = manifest(workId) + const expected = await versionOneUpdate(emptySummary(workId), first) + writeVersionOne(expected) + const next = rest.flat() + await apply(next) + const head = expectCommitted(workId, await versionOneUpdate(expected, next)) + for (const collection of COLLECTIONS) { + const generation = old.logs[collection]?.generation + if (generation) expect(head.logs[collection]?.generation).toBeGreaterThan(generation) + } + }) + + it("rereads the manifest once when a compaction replaces a log during a read", async () => { + const workId = randomUUID() + const [batch] = randomRecords(7, workId, 40) + await apply(batch) + const expected = readWorkSummary(dir, workId) + const head = manifest(workId) + const collection = COLLECTIONS.find((name) => head.logs[name]) + if (!collection) throw new Error("nothing was saved") + const log = head.logs[collection] + if (!log) throw new Error("nothing was saved") + const old = logPath(folder(workId), collection, log.generation) + const original = fs.openSync + let replaced = false + vi.spyOn(fs, "openSync").mockImplementation((...args) => { + if (!replaced && String(args[0]) === old) { + replaced = true + // What a compaction does between a reader's manifest read and its log open. + fs.copyFileSync(old, logPath(folder(workId), collection, log.generation + 1)) + const next = { ...head, logs: { ...head.logs, [collection]: { ...log, generation: log.generation + 1 } } } + fs.writeFileSync(join(folder(workId), "work.json"), JSON.stringify(next)) + fs.rmSync(old) + } + return original(...args) + }) + expect(readWorkSummary(dir, workId)).toEqual(expected) + expect(replaced).toBe(true) + }) + + it("never shows a partial work to a reader polling while another process appends and compacts", async () => { + const workId = randomUUID() + const batches = randomRecords(8, workId, 300, { large: true }) + const input = join(dir, "batches.json") + fs.writeFileSync(input, JSON.stringify(batches)) + const script = join(dir, "writer.mts") + fs.writeFileSync( + script, + `import { readFileSync } from "node:fs"; +const { flushWorkSummaries, updateWorkSummary } = await import(${JSON.stringify(new URL("./summary.ts", import.meta.url).pathname)}); +for (const batch of JSON.parse(readFileSync(process.argv[2], "utf8"))) { + for (const record of batch) updateWorkSummary(record); + await flushWorkSummaries(); +}`, + ) + const child = spawn(process.execPath, ["--import", "tsx", script, input], { + env: { ...process.env, PI_CODING_AGENT_DIR: dir }, + }) + let errors = "" + child.stderr.on("data", (chunk) => { + errors += chunk + }) + const failures: string[] = [] + let reads = 0 + let requests = 0 + const reading = setInterval(() => { + if (!fs.existsSync(join(folder(workId), "work.json"))) return + const view = readWorkSummary(dir, workId) + reads++ + if (!view) failures.push("unreadable") + else if (view.requests.length < requests) failures.push("requests went missing") + else requests = view.requests.length + }, 1) + try { + expect(await new Promise((resolve) => child.once("exit", resolve)), errors).toBe(0) + } finally { + clearInterval(reading) + child.kill() + } + expect(failures).toEqual([]) + expect(reads).toBeGreaterThan(10) + let expected = emptySummary(workId) + for (const batch of batches) expected = await versionOneUpdate(expected, batch) + const head = expectCommitted(workId, expected) + expect(COLLECTIONS.some((collection) => (head.logs[collection]?.generation ?? 0) > 1)).toBe(true) + }, 60_000) + + it("notices another process's appends and compactions before appending again", async () => { + const workId = randomUUID() + const batches = randomRecords(9, workId, 300, { large: true }) + await apply(batches[0]) + let expected = await versionOneUpdate(emptySummary(workId), batches[0]) + // A second module instance has its own cached positions, like another Kimchi process. + vi.resetModules() + const other = await import("./summary.js") + const before = manifest(workId) + for (const batch of batches.slice(1, -1)) { + for (const record of structuredClone(batch)) other.updateWorkSummary(record) + await other.flushWorkSummaries() + expected = await versionOneUpdate(expected, batch) + } + const changed = manifest(workId) + expect( + COLLECTIONS.some((name) => (changed.logs[name]?.generation ?? 0) > (before.logs[name]?.generation ?? 0)), + ).toBe(true) + const last = batches.at(-1) ?? [] + await apply(last) + expectCommitted(workId, await versionOneUpdate(expected, last)) + }, 60_000) +}) + +/** A billed request row as the stack writes it: about 2.2 KB of compact JSON. */ +function billedRequest(index: number, sessionId: string) { + const requestId = randomUUID() + const at = iso(BASE + index * 60_000) + return { + requestId, + sessionId, + cwd: "/Users/someone/src/an-example-repository-name-worktree", + startedAt: at, + provider: "kimchi-dev", + model: "glm-5.3", + modelSource: "context", + segment: { id: randomUUID(), attribution: "inferred", reason: "semantic-continue" }, + scope: { + account: { apiUrl: "https://api.example.ai/v1", organizationId: randomUUID(), userId: randomUUID() }, + repository: "/Users/someone/src/an-example-repository-name/.git", + }, + recordedAt: at, + dispatchedAt: at, + billingSource: { + apiUrl: "https://api.example.ai/v1", + gatewayUrl: "https://llm.example.ai/openai/v1/chat/completions", + credentialHash: "e".repeat(64), + }, + billingSelector: { type: "tag", tag: `kimchi-request:${requestId}`, startTime: at, endTime: at }, + response: { status: 200, receivedAt: at, traceId: "f".repeat(32) }, + billingRows: [ + { + id: randomUUID(), + costUsd: "0.00123456", + promptPrice: "0.0011234", + completionPrice: "0.000123", + cacheReadPrice: "0.00001234", + cacheCreationPrice: "0", + originalTotalPrice: "0.00123456", + recommendedTotalPrice: "0", + promptTokens: "12345", + completionTokens: "123", + totalTokens: "12468", + cacheReadInputTokens: "1234", + provider: "kimchi-dev", + model: "glm-5.3", + originalModel: "glm-5.3", + responseStatusCode: 200, + contextWindowSize: 1048576, + createTime: at, + }, + ], + billingLookup: { status: "priced", checkedAt: at, organizationId: randomUUID(), userId: randomUUID() }, + } +} + +describe("a large work", () => { + it("updates 20,000 requests in bounded time and bytes, reading one line per changed row", async () => { + const workId = randomUUID() + const sessions = Array.from({ length: 200 }, () => randomUUID()) + const summary = emptySummary(workId) + summary.sessions = sessions + summary.requests = Array.from({ length: 20_000 }, (_, index) => billedRequest(index, sessions[index % 200])) + summary.commits = Array.from({ length: 200 }, (_, index) => ({ + sha: index.toString(16).padStart(40, "0"), + repository: "/repo/.git", + worktree: "/tree", + sessionId: sessions[0], + recordedAt: iso(BASE + index * 3_600_000), + })) + writeVersionOne(summary) + const record = (index: number): WorkRecord => ({ + version: 1, + type: "request", + workId, + sessionId: sessions[0], + requestId: randomUUID(), + startedAt: iso(BASE + (20_000 + index) * 60_000), + recordedAt: iso(BASE + (20_000 + index) * 60_000), + }) + await apply([record(0)]) + expect(manifest(workId).logs.requests?.rows).toBe(20_001) + + const logBytes = () => + COLLECTIONS.reduce((sum, collection) => { + const log = manifest(workId).logs[collection] + return sum + (log ? fs.statSync(logPath(folder(workId), collection, log.generation)).size : 0) + }, 0) + let read = 0 + const rows = join(folder(workId), "rows") + const readSync = fs.readSync + vi.spyOn(fs, "readSync").mockImplementation((...args) => { + const bytes = readSync(...args) + read += bytes + return bytes + }) + const open = asyncFs.open + vi.spyOn(asyncFs, "open").mockImplementation(async (...args) => { + const handle = await open(...args) + if (String(args[0]).startsWith(rows)) { + const original = handle.read.bind(handle) + vi.spyOn(handle, "read").mockImplementation(async (...readArgs) => { + const result = await original(...readArgs) + read += result.bytesRead + return result + }) + } + return handle + }) + const times: number[] = [] + const appended: number[] = [] + for (let index = 1; index <= 20; index++) { + const before = logBytes() + const middle = summary.requests[index * 997] + const batch: WorkRecord[] = + index % 2 + ? [record(index)] + : [ + { + version: 1, + type: "request_cost", + workId, + sessionId: middle.sessionId, + requestId: middle.requestId, + billingLookup: { status: "priced", checkedAt: iso(BASE + 10 ** 9 + index) }, + recordedAt: iso(BASE + 10 ** 9 + index), + }, + ] + const started = performance.now() + await apply(batch) + times.push(performance.now() - started) + appended.push(logBytes() - before) + } + const median = [...times].sort((left, right) => left - right)[10] + expect(median).toBeLessThan(50) + expect(Math.max(...appended)).toBeLessThanOrEqual(8 * 1024) + expect(fs.statSync(join(folder(workId), "work.json")).size).toBeLessThanOrEqual(16 * 1024) + // No ordinary update scans a log: only the line of each changed row is read. + expect(read).toBeLessThan(20 * 8 * 1024) + + vi.mocked(asyncFs.open).mockClear() + vi.mocked(fs.readSync).mockClear() + const opened = vi.spyOn(fs, "openSync") + const started = performance.now() + const head = await readWorkHead(dir, workId) + expect(performance.now() - started).toBeLessThan(5) + expect(head).toMatchObject({ version: 2, logs: { requests: { rows: 20_011 } } }) + expect(opened).not.toHaveBeenCalled() + expect(vi.mocked(asyncFs.open)).not.toHaveBeenCalled() + expect(vi.mocked(fs.readSync)).not.toHaveBeenCalled() + }, 120_000) +}) diff --git a/src/extensions/work-attribution/row-log.ts b/src/extensions/work-attribution/row-log.ts new file mode 100644 index 000000000..d499ef611 --- /dev/null +++ b/src/extensions/work-attribution/row-log.ts @@ -0,0 +1,918 @@ +import { closeSync, openSync, readFileSync, readSync } from "node:fs" +import { type FileHandle, mkdir, open, readdir, readFile, rm, stat } from "node:fs/promises" +import { join } from "node:path" +import { setImmediate } from "node:timers/promises" +import { writeFileDurably } from "../../config/json.js" +import { mergePullRequestLinks } from "../pull-request-status/links.js" + +/* + * Work summary storage. Version 2 `work//work.json` is a small manifest. Each collection's rows live in + * `rows/..jsonl`, one version 1 row per line; a later line with the same key replaces an + * earlier one, and the first line of a key fixes its position. A writer appends changed rows, syncs them, then + * renames the manifest, which commits them. This module uses plain `fs` so tests and tools can read summaries. + */ + +/** The manifest, and the commit point for every row log. */ +export const WORK_FILE = "work.json" +const ROWS = "rows" +/** Version 1 summaries larger than this are not read just to list a work. */ +export const MAX_SUMMARY_BYTES = 8 * 1024 * 1024 +/** Appends and copies are written in chunks of at most this size. */ +const CHUNK_BYTES = 4 * 1024 * 1024 +/** A log is compacted once superseded lines exceed half its live rows and this many bytes. */ +const COMPACT_GARBAGE_BYTES = 1024 * 1024 +/** Row positions kept in memory per process across works; about 100 bytes each. */ +const MAX_INDEXED_ROWS = 200_000 +const YIELD_ROWS = 1000 + +/** One row of a summary collection, as version 1 `work.json` stores it. */ +export interface Row { + sessionId: string + [key: string]: unknown +} + +/** Summary collections, named as in version 1 `work.json`. Each one is a row log in version 2. */ +export const COLLECTIONS = [ + "workLinks", + "requests", + "plans", + "commits", + "fileTransitions", + "fileObservations", + "continuations", +] as const +export type Collection = (typeof COLLECTIONS)[number] +const COLLECTION_NAMES = new Set(COLLECTIONS) +function isCollection(name: string): name is Collection { + return COLLECTION_NAMES.has(name) +} + +/** Row identity per collection. */ +export const ROW_KEYS: Record string> = { + workLinks: (row) => + JSON.stringify([ + row.linkId, + row.revision, + row.sourceWorkId, + row.targetWorkId, + row.requestIds, + row.scope, + row.status, + row.evidence, + ]), + requests: (row) => JSON.stringify(row.requestId), + plans: (row) => JSON.stringify([row.sessionId, row.path, row.snapshotPath]), + commits: (row) => JSON.stringify([row.sessionId, row.sha, row.repository, row.worktree]), + fileTransitions: (row) => JSON.stringify(row.transitionId), + fileObservations: (row) => JSON.stringify(row.observationId), + continuations: (row) => JSON.stringify([row.sessionId, row.source, row.evidence]), +} + +/** `rows/..jsonl`. Bytes past `bytes` belong to an interrupted append and are ignored. */ +export interface RowLog { + /** Positive for a log file. `readWorkHead` reports rows still inside a version 1 `work.json` as generation 0. */ + generation: number + /** Committed length. */ + bytes: number + /** Distinct keys: the length of the version 1 array. */ + rows: number +} + +/** What `/work` lists. Each value is the newest by its timestamp; later writes win ties. */ +export interface WorkLatest { + /** Newest request start, native edit or retained plan. */ + activityAt?: string + request?: { startedAt?: string; repository?: string; cwd?: string } + edit?: { recordedAt?: string; repository: string } + branch?: { recordedAt?: string; branch: string } + commit?: { recordedAt?: string; repository: string } + plan?: { recordedAt?: string; snapshotPath: string } +} + +/** Version 2 `work.json`. */ +export interface WorkHead { + version: 2 + workId: string + /** Time of the last committed change. */ + updatedAt: string + /** Sessions in first-seen order. */ + sessions: string[] + /** A missing log is an empty collection. */ + logs: Partial> + latest: WorkLatest + /** PR links of every commit row, merged. */ + pullRequests: Record[] +} + +/** The version 1 `work.json` shape, which folding the row logs returns. */ +export type WorkSummaryView = { version: 1; workId: string; sessions: string[] } & Record + +/** Validates one committed row; a failure means the logs are damaged and the work is rebuilt from its journals. */ +export type RowCheck = (collection: Collection, row: Row) => boolean + +/** A log that does not match its manifest. */ +export class RowLogDamage extends Error {} + +function object(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value) +} +function code(error: unknown): unknown { + return object(error) ? error.code : undefined +} +function count(value: unknown, minimum = 0): value is number { + return typeof value === "number" && Number.isSafeInteger(value) && value >= minimum +} +function isRow(value: unknown): value is Row { + return object(value) && typeof value.sessionId === "string" +} +function text(value: unknown): string | undefined { + return typeof value === "string" ? value : undefined +} + +export function logPath(folder: string, collection: Collection, generation: number): string { + return join(folder, ROWS, `${collection}.${generation}.jsonl`) +} + +/** A version 2 manifest for `workId`, checked without opening its logs. */ +export function isWorkHead(value: unknown, workId: string): value is WorkHead { + return ( + object(value) && + value.version === 2 && + value.workId === workId && + typeof value.updatedAt === "string" && + Array.isArray(value.sessions) && + value.sessions.every((session) => typeof session === "string") && + object(value.logs) && + Object.entries(value.logs).every( + ([name, log]) => + isCollection(name) && + object(log) && + count(log.generation, 1) && + count(log.rows, 1) && + count(log.bytes, log.rows + 1), + ) && + object(value.latest) && + Array.isArray(value.pullRequests) + ) +} + +function summaryView(workId: string, sessions: string[], rows: (collection: Collection) => Row[]): WorkSummaryView { + return { + version: 1, + workId, + sessions, + workLinks: rows("workLinks"), + requests: rows("requests"), + plans: rows("plans"), + commits: rows("commits"), + fileTransitions: rows("fileTransitions"), + fileObservations: rows("fileObservations"), + continuations: rows("continuations"), + } +} + +/** A version 1 summary read as leniently as `/work` always read it; older ones lack later collections. */ +function versionOneView(value: unknown, workId: string): WorkSummaryView | undefined { + if (!object(value) || value.version !== 1 || value.workId !== workId) return undefined + const sessions = Array.isArray(value.sessions) ? value.sessions.filter((session) => typeof session === "string") : [] + return summaryView(workId, sessions, (collection) => { + const rows = value[collection] + return Array.isArray(rows) ? rows.filter(isRow) : [] + }) +} + +function parseRow(line: string, collection: Collection): Row { + let row: unknown + try { + row = JSON.parse(line) + } catch { + throw new RowLogDamage(`A ${collection} row is not JSON`) + } + if (!isRow(row)) throw new RowLogDamage(`A ${collection} row has no session`) + return row +} + +/** Rows of one committed range, folded by key. */ +function foldLog(collection: Collection, buffer: Buffer): Row[] { + const rows = new Map() + let start = 0 + for (let end = buffer.indexOf(10); end !== -1; end = buffer.indexOf(10, start)) { + const row = parseRow(buffer.toString("utf8", start, end), collection) + rows.set(ROW_KEYS[collection](row), row) + start = end + 1 + } + if (start !== buffer.length) throw new RowLogDamage(`The ${collection} log does not end at a line break`) + return [...rows.values()] +} + +/** The committed bytes of one log. A compaction may delete it after its manifest was read (ENOENT). */ +function readLogSync(folder: string, collection: Collection, generation: number, bytes: number): Buffer { + const fd = openSync(logPath(folder, collection, generation), "r") + try { + const buffer = Buffer.allocUnsafe(bytes) + for (let read = 0; read < bytes; ) { + const length = readSync(fd, buffer, read, bytes - read, read) + if (!length) throw new RowLogDamage(`The ${collection} log is shorter than its committed length`) + read += length + } + return buffer + } finally { + closeSync(fd) + } +} + +/** Reads the manifest and folds what `read` needs; retries once when a compaction replaced a log. */ +function readCommitted( + agentDir: string, + workId: string, + fromVersionOne: (view: WorkSummaryView) => T, + read: (folder: string, head: WorkHead) => T, +): T | undefined { + const folder = join(agentDir, "work", workId) + for (let attempt = 0; ; attempt++) { + let value: unknown + try { + value = JSON.parse(readFileSync(join(folder, WORK_FILE), "utf8")) + } catch { + return undefined + } + const view = versionOneView(value, workId) + if (view) return fromVersionOne(view) + if (!isWorkHead(value, workId)) return undefined + try { + return read(folder, value) + } catch (error) { + if (attempt === 0 && code(error) === "ENOENT") continue + if (error instanceof RowLogDamage || code(error) === "ENOENT") return undefined + throw error + } + } +} + +function foldCommitted(folder: string, head: WorkHead, collection: Collection): Row[] { + const log = head.logs[collection] + return log ? foldLog(collection, readLogSync(folder, collection, log.generation, log.bytes)) : [] +} + +/** The whole summary in the version 1 shape, from either version; undefined when missing or damaged. */ +export function readWorkSummary(agentDir: string, workId: string): WorkSummaryView | undefined { + return readCommitted( + agentDir, + workId, + (view) => view, + (folder, head) => summaryView(workId, head.sessions, (collection) => foldCommitted(folder, head, collection)), + ) +} + +/** One collection's rows from either version, without reading the other logs. */ +export function readWorkRows(agentDir: string, workId: string, collection: Collection): Row[] | undefined { + return readCommitted( + agentDir, + workId, + (view) => view[collection], + (folder, head) => foldCommitted(folder, head, collection), + ) +} + +/** + * The manifest, or the same values computed from a version 1 summary small enough to read. Opens no row log. + * "too-large" is a version 1 summary over `maxBytes`; its next update migrates it. + */ +export async function readWorkHead( + agentDir: string, + workId: string, + maxBytes = MAX_SUMMARY_BYTES, +): Promise { + const path = join(agentDir, "work", workId, WORK_FILE) + let value: unknown + let modifiedAt: number + try { + const info = await stat(path) + if (!info.isFile()) return undefined + if (info.size > maxBytes) return "too-large" + modifiedAt = info.mtimeMs + value = JSON.parse(await readFile(path, "utf8")) + } catch { + return undefined + } + if (isWorkHead(value, workId)) return value + const view = versionOneView(value, workId) + if (!view) return undefined + const logs: Partial> = {} + for (const collection of COLLECTIONS) + if (view[collection].length) logs[collection] = { generation: 0, bytes: 0, rows: view[collection].length } + return { + version: 2, + workId, + updatedAt: new Date(modifiedAt).toISOString(), + sessions: view.sessions, + logs, + ...summaryLatest((collection) => view[collection]), + } +} + +/** Milliseconds of an ISO 8601 timestamp, as `/work` orders rows; impossible dates are not times. */ +function timestamp(value: unknown): number | undefined { + if ( + typeof value !== "string" || + !/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d{1,9})?(?:Z|[+-]\d{2}:\d{2})$/.test(value) + ) + return undefined + const parsed = Date.parse(value) + if (!Number.isFinite(parsed)) return undefined + const day = value.slice(0, 10) + return new Date(`${day}T00:00:00Z`).toISOString().startsWith(day) ? parsed : undefined +} +/** Whether a row stamped `at` replaces a latest value stamped `current`. Rows without a valid time sort first. */ +function replaces(at: unknown, current: { at?: string } | undefined): boolean { + return !current || (timestamp(at) ?? 0) >= (timestamp(current.at) ?? 0) +} +/** Folds one changed or saved row into what `/work` lists. */ +export function observeLatest(latest: WorkLatest, collection: Collection, row: Row): void { + const at = collection === "requests" ? row.startedAt : row.recordedAt + const time = timestamp(at) + if ( + time !== undefined && + (collection === "requests" || collection === "fileTransitions" || collection === "plans") && + time > (timestamp(latest.activityAt) ?? Number.NEGATIVE_INFINITY) + ) + latest.activityAt = text(at) + const recordedAt = text(row.recordedAt) + if (collection === "requests") { + if (replaces(at, latest.request && { at: latest.request.startedAt })) { + const scope = object(row.scope) ? row.scope : undefined + latest.request = { startedAt: text(at), repository: text(scope?.repository), cwd: text(row.cwd) } + } + } else if (collection === "fileTransitions") { + if (typeof row.repository === "string" && replaces(at, latest.edit && { at: latest.edit.recordedAt })) + latest.edit = { recordedAt, repository: row.repository } + if (typeof row.branch === "string" && replaces(at, latest.branch && { at: latest.branch.recordedAt })) + latest.branch = { recordedAt, branch: row.branch } + } else if (collection === "commits") { + if (typeof row.repository === "string" && replaces(at, latest.commit && { at: latest.commit.recordedAt })) + latest.commit = { recordedAt, repository: row.repository } + } else if (collection === "plans") { + if (typeof row.snapshotPath === "string" && replaces(at, latest.plan && { at: latest.plan.recordedAt })) + latest.plan = { recordedAt, snapshotPath: row.snapshotPath } + } +} +/** PR links of every commit row, merged; empty or failed lookups never remove a link. */ +export function commitPullRequests(commits: Iterable): Record[] { + return mergePullRequestLinks( + ...[...commits].map((row) => (Array.isArray(row.pullRequests) ? row.pullRequests.filter(object) : [])), + ) +} +/** `latest` and `pullRequests` recomputed from every row, in log order. */ +export function summaryLatest( + rows: (collection: Collection) => Iterable, +): Pick { + const latest: WorkLatest = {} + for (const collection of COLLECTIONS) for (const row of rows(collection)) observeLatest(latest, collection, row) + return { latest, pullRequests: commitPullRequests(rows("commits")) } +} + +interface Line { + offset: number + length: number +} +interface LogIndex { + generation: number + /** File identity: a replaced file is scanned again. */ + ino: number + /** Modification time when this process last read or wrote the log; an unexplained change means a full scan. */ + mtimeMs: number + /** Committed bytes already indexed. */ + bytes: number + /** Bytes of each key's newest line, line breaks included. */ + live: number + /** Newest line per key, in first-seen order. */ + lines: Map +} +/** Row positions of logs this process has read, most recently used last. */ +const indexes = new Map() +let indexedRows = 0 +function cached(folder: string, collection: Collection): LogIndex | undefined { + return indexes.get(`${folder}\0${collection}`)?.index +} +function remember(folder: string, collection: Collection, index: LogIndex): void { + const key = `${folder}\0${collection}` + const previous = indexes.get(key) + if (previous) { + indexes.delete(key) + indexedRows -= previous.rows + } + indexes.set(key, { index, rows: index.lines.size }) + indexedRows += index.lines.size + for (const [oldest, entry] of indexes) { + if (indexedRows <= MAX_INDEXED_ROWS || oldest === key) break + indexes.delete(oldest) + indexedRows -= entry.rows + } +} +/** Drops this work's positions; its next update scans the logs again. */ +function forget(folder: string): void { + for (const collection of COLLECTIONS) { + const key = `${folder}\0${collection}` + const entry = indexes.get(key) + if (!entry) continue + indexes.delete(key) + indexedRows -= entry.rows + } +} + +async function readExactly(handle: FileHandle, buffer: Buffer, position: number): Promise { + for (let read = 0; read < buffer.length; ) { + const { bytesRead } = await handle.read(buffer, read, buffer.length - read, position + read) + if (!bytesRead) throw new RowLogDamage("A row log is shorter than its committed length") + read += bytesRead + } +} + +/** Indexes committed lines from `index.bytes` to `end`; each must be a valid row, and the range must end a line. */ +async function scanLog( + handle: FileHandle, + collection: Collection, + index: LogIndex, + end: number, + check: RowCheck, +): Promise { + let carry = Buffer.alloc(0) + let rows = 0 + for (let position = index.bytes; position < end; ) { + const chunk = Buffer.allocUnsafe(Math.min(CHUNK_BYTES, end - position)) + await readExactly(handle, chunk, position) + const buffer = carry.length ? Buffer.concat([carry, chunk]) : chunk + const base = position - carry.length + let start = 0 + for (let stop = buffer.indexOf(10); stop !== -1; stop = buffer.indexOf(10, start)) { + const row = parseRow(buffer.toString("utf8", start, stop), collection) + if (!check(collection, row)) throw new RowLogDamage(`A ${collection} row is invalid`) + const key = ROW_KEYS[collection](row) + const previous = index.lines.get(key) + if (previous) index.live -= previous.length + 1 + index.lines.set(key, { offset: base + start, length: stop - start }) + index.live += stop - start + 1 + start = stop + 1 + if (++rows % YIELD_ROWS === 0) await setImmediate() + } + carry = buffer.subarray(start) + position += chunk.length + } + if (carry.length) throw new RowLogDamage(`The ${collection} log does not end at a line break`) + index.bytes = end +} + +/** Next unused generation of a collection, past any file a crash or another version left behind. */ +async function nextGeneration(folder: string, collection: Collection, current = 0): Promise { + let highest = current + for (const name of await readdir(join(folder, ROWS)).catch(() => [])) { + const match = /^(\w+)\.(\d+)\.jsonl$/.exec(name) + if (match?.[1] === collection) highest = Math.max(highest, Number(match[2])) + } + return highest + 1 +} + +/** Deletes row files the manifest does not name: leftovers of crashes, compactions and other versions. */ +async function removeStrays(folder: string, head: WorkHead): Promise { + for (const name of await readdir(join(folder, ROWS)).catch(() => [])) { + const match = /^(\w+)\.(\d+)\.jsonl$/.exec(name) + const stray = + match && isCollection(match[1]) ? head.logs[match[1]]?.generation !== Number(match[2]) : /^\..*\.tmp$/.test(name) + // A reader may still hold an old generation open. Windows then refuses, and a later update retries. + if (stray) await rm(join(folder, ROWS, name), { force: true }).catch(() => {}) + } +} + +/** Writes a new log from `[key, line]` pairs; returns its index. */ +async function writeLog( + folder: string, + collection: Collection, + generation: number, + lines: Iterable<[string, string]>, + assertLease: () => void, +): Promise { + const index: LogIndex = { generation, ino: 0, mtimeMs: 0, bytes: 0, live: 0, lines: new Map() } + const handle = await open(logPath(folder, collection, generation), "wx", 0o600) + try { + let chunk: string[] = [] + let chunkBytes = 0 + const flush = async () => { + assertLease() + await handle.write(chunk.join("")) + chunk = [] + chunkBytes = 0 + } + let rows = 0 + for (const [key, line] of lines) { + const length = Buffer.byteLength(line) + index.lines.set(key, { offset: index.bytes, length }) + index.bytes += length + 1 + chunk.push(line, "\n") + chunkBytes += length + 1 + if (chunkBytes >= CHUNK_BYTES) await flush() + if (++rows % YIELD_ROWS === 0) await setImmediate() + } + if (chunk.length) await flush() + await handle.sync() + const { size, ino, mtimeMs } = await handle.stat() + if (size !== index.bytes) throw new RowLogDamage(`The new ${collection} log changed while it was written`) + index.ino = ino + index.mtimeMs = mtimeMs + index.live = index.bytes + return index + } finally { + await handle.close() + } +} + +/** Appends lines at the committed end and syncs them; another writer's bytes in between fail the update. */ +async function appendLog( + path: string, + from: number, + lines: string[], + assertLease: () => void, +): Promise<{ ino: number; mtimeMs: number }> { + const data = Buffer.from(lines.join("")) + const handle = await open(path, from ? "a" : "ax", 0o600) + try { + for (let start = 0; start < data.length; start += CHUNK_BYTES) { + assertLease() + await handle.write(data, start, Math.min(CHUNK_BYTES, data.length - start)) + } + await handle.sync() + const { size, ino, mtimeMs } = await handle.stat() + if (size !== from + data.length) throw new RowLogDamage("A row log changed during an append") + return { ino, mtimeMs } + } finally { + await handle.close() + } +} + +/** One collection's rows during a locked update. Saved rows load on first use; changed and new rows are written. */ +export class RowStore { + /** Rows read or set in this update, in first-touch order, with the saved line each was read from. */ + readonly touched = new Map() + constructor( + private readonly collection: Collection, + private readonly index?: LogIndex, + private readonly readLine?: (line: Line) => string, + ) {} + get(key: string): Row | undefined { + const touched = this.touched.get(key) + if (touched) return touched.row + const at = this.index?.lines.get(key) + if (!at || !this.readLine) return undefined + const line = this.readLine(at) + const row = parseRow(line, this.collection) + if (ROW_KEYS[this.collection](row) !== key) throw new RowLogDamage(`A ${this.collection} row moved`) + this.touched.set(key, { row, line }) + return row + } + has(key: string): boolean { + return this.touched.has(key) || this.index?.lines.has(key) === true + } + set(key: string, row: Row): void { + const touched = this.touched.get(key) + if (touched) touched.row = row + else this.touched.set(key, { row }) + } +} + +/** Sessions and rows of one work while records merge into it. */ +export type WorkRows = { sessions: Set } & Record + +function workRows(sessions: Iterable, store: (collection: Collection) => RowStore): WorkRows { + return { + sessions: new Set(sessions), + workLinks: store("workLinks"), + requests: store("requests"), + plans: store("plans"), + commits: store("commits"), + fileTransitions: store("fileTransitions"), + fileObservations: store("fileObservations"), + continuations: store("continuations"), + } +} + +interface OpenWork { + /** The committed manifest when rows are appended; absent when a version 1 summary migrates or the work is rebuilt. */ + head?: WorkHead + /** Nothing usable was saved: the whole history must be merged again. */ + rebuild: boolean + /** A damaged manifest being rebuilt; new generations go past the ones it named. */ + damaged?: WorkHead + indexes: Partial> + rows: WorkRows + close(): void +} + +/** Opens and checks the committed logs, dropping an interrupted append; damage throws. */ +async function openLogs(folder: string, head: WorkHead, check: RowCheck): Promise { + const indexes: Partial> = {} + for (const collection of COLLECTIONS) { + const log = head.logs[collection] + if (!log) continue + const handle = await open(logPath(folder, collection, log.generation), "r+") + try { + const { size, ino, mtimeMs } = await handle.stat() + if (size < log.bytes) throw new RowLogDamage(`The ${collection} log is shorter than its committed length`) + let index = cached(folder, collection) + // Another process may have committed more lines since this one indexed the log; only that tail is read. + // The same length with a different modification time means the committed bytes themselves changed. + const grew = index !== undefined && index.bytes < log.bytes + if ( + !index || + index.generation !== log.generation || + index.ino !== ino || + index.bytes > log.bytes || + (!grew && index.mtimeMs !== mtimeMs) + ) + index = { generation: log.generation, ino, mtimeMs, bytes: 0, live: 0, lines: new Map() } + // Bytes past the committed length are an interrupted append; this writer holds the lock. + if (size > log.bytes) await handle.truncate(log.bytes) + if (index.bytes < log.bytes) await scanLog(handle, collection, index, log.bytes, check) + index.mtimeMs = (await handle.stat()).mtimeMs + if (index.lines.size !== log.rows) throw new RowLogDamage(`The ${collection} log has the wrong row count`) + indexes[collection] = index + } finally { + await handle.close() + } + } + const files = new Map() + const reader = (collection: Collection, index: LogIndex) => (line: Line) => { + let fd = files.get(collection) + if (fd === undefined) { + fd = openSync(logPath(folder, collection, index.generation), "r") + files.set(collection, fd) + } + const buffer = Buffer.allocUnsafe(line.length) + if (readSync(fd, buffer, 0, line.length, line.offset) !== line.length) + throw new RowLogDamage(`A ${collection} row is missing`) + return buffer.toString("utf8") + } + return { + head, + rebuild: false, + indexes, + rows: workRows(head.sessions, (collection) => { + const index = indexes[collection] + return new RowStore(collection, index, index && reader(collection, index)) + }), + close() { + for (const fd of files.values()) closeSync(fd) + files.clear() + }, + } +} + +async function openWork( + folder: string, + workId: string, + summary: (value: unknown) => WorkSummaryView | undefined, + check: RowCheck, +): Promise { + let value: unknown + try { + value = JSON.parse(await readFile(join(folder, WORK_FILE), "utf8")) + } catch (error) { + if (!(error instanceof SyntaxError) && code(error) !== "ENOENT") throw error + } + let damaged: WorkHead | undefined + if (isWorkHead(value, workId)) { + try { + return await openLogs(folder, value, check) + } catch (error) { + if (!(error instanceof RowLogDamage) && code(error) !== "ENOENT") throw error + forget(folder) + damaged = value + } + } + // A version 1 summary migrates with every row; anything else is rebuilt from the journals. + const saved = damaged ? undefined : summary(value) + return { + rebuild: !saved, + damaged, + indexes: {}, + rows: workRows(saved?.sessions ?? [], (collection) => { + const store = new RowStore(collection) + for (const row of saved?.[collection] ?? []) store.set(ROW_KEYS[collection](row), row) + return store + }), + close() {}, + } +} + +function* serialized(touched: RowStore["touched"]): Generator<[string, string]> { + for (const [key, { row }] of touched) yield [key, JSON.stringify(row)] +} +/** Each touched row's new line, when it differs from the saved one. */ +async function changedLines(store: RowStore): Promise<[string, string, Row][]> { + const changed: [string, string, Row][] = [] + let rows = 0 + for (const [key, { row, line }] of store.touched) { + const next = JSON.stringify(row) + if (next !== line) changed.push([key, next, row]) + if (++rows % YIELD_ROWS === 0) await setImmediate() + } + return changed +} + +/** Commits changed rows: appended to the current logs, or as new generations on migration and rebuild. */ +async function commitWork( + folder: string, + workId: string, + work: OpenWork, + assertLease: () => void, +): Promise { + const { head: previous } = work + const changes = new Map() + if (previous) + for (const collection of COLLECTIONS) { + const changed = await changedLines(work.rows[collection]) + if (changed.length) changes.set(collection, changed) + } + const sessions = [...work.rows.sessions] + if (previous && !changes.size && sessions.length === previous.sessions.length) return undefined + assertLease() + await mkdir(join(folder, ROWS), { recursive: true, mode: 0o700 }) + const logs: Partial> = {} + const written: Partial> = {} + let values: Pick + let generations = !previous + if (previous) { + Object.assign(logs, previous.logs) + const latest = structuredClone(previous.latest) + for (const [collection, rows] of changes) { + const log = previous.logs[collection] + const generation = log?.generation ?? (await nextGeneration(folder, collection)) + generations ||= !log + const index = work.indexes[collection] ?? { generation, ino: 0, mtimeMs: 0, bytes: 0, live: 0, lines: new Map() } + const from = log?.bytes ?? 0 + const lines = rows.map(([, line]) => `${line}\n`) + Object.assign(index, await appendLog(logPath(folder, collection, generation), from, lines, assertLease)) + let offset = from + for (const [key, line, row] of rows) { + const saved = index.lines.get(key) + if (saved) index.live -= saved.length + 1 + const length = Buffer.byteLength(line) + index.lines.set(key, { offset, length }) + index.live += length + 1 + offset += length + 1 + observeLatest(latest, collection, row) + } + index.bytes = offset + written[collection] = index + logs[collection] = { generation, bytes: offset, rows: index.lines.size } + } + values = { + latest, + pullRequests: written.commits + ? commitPullRequests(liveRows(folder, "commits", written.commits)) + : previous.pullRequests, + } + } else { + for (const collection of COLLECTIONS) { + const { touched } = work.rows[collection] + if (!touched.size) continue + const generation = await nextGeneration(folder, collection, work.damaged?.logs[collection]?.generation) + written[collection] = await writeLog(folder, collection, generation, serialized(touched), assertLease) + logs[collection] = { generation, bytes: written[collection].bytes, rows: written[collection].lines.size } + } + values = summaryLatest((collection) => [...work.rows[collection].touched.values()].map(({ row }) => row)) + } + const head: WorkHead = { version: 2, workId, updatedAt: new Date().toISOString(), sessions, logs, ...values } + await writeFileDurably(join(folder, WORK_FILE), `${JSON.stringify(head, null, 2)}\n`, assertLease) + for (const collection of COLLECTIONS) { + const index = written[collection] + if (index) remember(folder, collection, index) + } + const compacted = await compactLogs(folder, head, { ...work.indexes, ...written }, assertLease) + if (generations || compacted || !cleaned.has(folder)) { + await removeStrays(folder, compacted ?? head) + cleaned.add(folder) + } + return compacted ?? head +} + +/** Folders whose stray row files this process already removed. */ +const cleaned = new Set() + +/** Each key's newest row in a log this process indexed, in first-seen order. */ +function liveRows(folder: string, collection: Collection, index: LogIndex): Row[] { + const buffer = readLogSync(folder, collection, index.generation, index.bytes) + return [...index.lines.values()].map(({ offset, length }) => + parseRow(buffer.toString("utf8", offset, offset + length), collection), + ) +} + +/** + * Rewrites logs whose superseded lines outweigh half their live rows: each key's newest line is copied in + * first-seen order to the next generation, committed with `latest` recomputed from every row, then the old + * generation is deleted. A reader holding the old manifest sees it missing once and rereads the manifest. + */ +async function compactLogs( + folder: string, + head: WorkHead, + indexes: Partial>, + assertLease: () => void, +): Promise { + const due = new Set( + COLLECTIONS.filter((collection) => { + const log = head.logs[collection] + const index = indexes[collection] + return log && index && log.bytes > 1.5 * index.live && log.bytes - index.live > COMPACT_GARBAGE_BYTES + }), + ) + if (!due.size) return undefined + const logs = { ...head.logs } + const latest: WorkLatest = {} + let pullRequests: Record[] = [] + const replaced: [Collection, LogIndex, number][] = [] + for (const collection of COLLECTIONS) { + const index = indexes[collection] + if (!index) continue + const buffer = readLogSync(folder, collection, index.generation, index.bytes) + const commits: Row[] = [] + const live = function* (): Generator<[string, string]> { + for (const [key, { offset, length }] of index.lines) { + const line = buffer.toString("utf8", offset, offset + length) + const row = parseRow(line, collection) + observeLatest(latest, collection, row) + if (collection === "commits") commits.push(row) + yield [key, line] + } + } + if (due.has(collection)) { + const generation = await nextGeneration(folder, collection, index.generation) + const next = await writeLog(folder, collection, generation, live(), assertLease) + replaced.push([collection, next, index.generation]) + logs[collection] = { generation, bytes: next.bytes, rows: next.lines.size } + } else { + let rows = 0 + for (const _ of live()) if (++rows % YIELD_ROWS === 0) await setImmediate() + } + if (collection === "commits") pullRequests = commitPullRequests(commits) + } + const compacted: WorkHead = { ...head, logs, latest, pullRequests } + await writeFileDurably(join(folder, WORK_FILE), `${JSON.stringify(compacted, null, 2)}\n`, assertLease) + for (const [collection, index, old] of replaced) { + remember(folder, collection, index) + await rm(logPath(folder, collection, old), { force: true }).catch(() => {}) + } + return compacted +} + +/** + * One locked update: opens the work's summary in either version, lets `apply` merge records into its rows, then + * commits what changed. A version 1 summary migrates here; a missing, invalid or damaged one is rebuilt, and + * `apply` learns that it must merge the work's whole history. Returns the committed manifest, or undefined + * when nothing changed. + */ +export async function updateWorkRows( + folder: string, + workId: string, + options: { + /** A valid version 1 summary with every collection, or undefined. */ + summary: (value: unknown) => WorkSummaryView | undefined + check: RowCheck + assertLease: () => void + }, + apply: (rows: WorkRows, rebuild: boolean) => Promise, +): Promise { + const work = await openWork(folder, workId, options.summary, options.check) + try { + await apply(work.rows, work.rebuild) + // Saved rows are loaded; a compaction may delete the file they came from. + work.close() + return await commitWork(folder, workId, work, options.assertLease) + } catch (error) { + // Cached positions may include rows that were never committed, and a new generation may be left behind. + forget(folder) + cleaned.delete(folder) + throw error + } finally { + work.close() + } +} + +/** Cheap validity for launch recovery: a version 1 summary, or a manifest whose logs hold their committed bytes. */ +export async function validWorkFiles( + folder: string, + workId: string, + summary: (value: unknown) => boolean, +): Promise { + let value: unknown + try { + value = JSON.parse(await readFile(join(folder, WORK_FILE), "utf8")) + } catch { + return false + } + if (!isWorkHead(value, workId)) return summary(value) + for (const collection of COLLECTIONS) { + const log = value.logs[collection] + if (!log) continue + const info = await stat(logPath(folder, collection, log.generation)).catch(() => undefined) + if (!info || info.size < log.bytes) return false + } + return true +} diff --git a/src/extensions/work-attribution/semantic.integration.test.ts b/src/extensions/work-attribution/semantic.integration.test.ts index c90d568d4..e5caead59 100644 --- a/src/extensions/work-attribution/semantic.integration.test.ts +++ b/src/extensions/work-attribution/semantic.integration.test.ts @@ -1,6 +1,6 @@ import { execFileSync } from "node:child_process" import { randomUUID } from "node:crypto" -import { mkdirSync, mkdtempSync, readFileSync, realpathSync, rmSync, writeFileSync } from "node:fs" +import { mkdirSync, mkdtempSync, realpathSync, rmSync, writeFileSync } from "node:fs" import { tmpdir } from "node:os" import { join } from "node:path" import type { @@ -17,6 +17,7 @@ import { createCommandContext, createContext } from "../__mocks__/context.js" import { createExtensionApi } from "../__mocks__/extension-api.js" import { createModel, createModelRegistry } from "../__mocks__/model-registry.js" import { createWorkScopeSnapshot } from "../__mocks__/work-scope.js" +import { savedWorkSummary } from "../__mocks__/work-summary.js" import * as prCostTelemetry from "../telemetry/pr-cost.js" import { appendWorkRecord, @@ -159,10 +160,11 @@ it.each([ readWorkRecords(join(root, "agent")).find((row) => row.requestId === request.headers["X-Request-Id"])?.workId, ).toBe(planned) await flushWorkSummaries() - const summary = JSON.parse(readFileSync(join(root, "agent", "work", planned, "work.json"), "utf8")) - expect( - summary.requests.find((row: { requestId: string }) => row.requestId === request.headers["X-Request-Id"]).segment, - ).toMatchObject({ attribution: "inferred", reason: "model-continue" }) + const summary = savedWorkSummary(join(root, "agent"), planned) + expect(summary.requests.find((row) => row.requestId === request.headers["X-Request-Id"])?.segment).toMatchObject({ + attribution: "inferred", + reason: "model-continue", + }) expect(summary.continuations).toContainEqual( expect.objectContaining({ source: "semantic", diff --git a/src/extensions/work-attribution/summary.test.ts b/src/extensions/work-attribution/summary.test.ts index 98eee8078..9ac82c131 100644 --- a/src/extensions/work-attribution/summary.test.ts +++ b/src/extensions/work-attribution/summary.test.ts @@ -14,6 +14,7 @@ import { lockSync } from "proper-lockfile" import { afterEach, beforeEach, describe, expect, it, vi } from "vitest" import { createContext } from "../__mocks__/context.js" import { createExtensionApi } from "../__mocks__/extension-api.js" +import { savedWorkSummary } from "../__mocks__/work-summary.js" import { appendWorkRecord, createWorkAttributionExtension, @@ -53,7 +54,7 @@ function path(workId: string) { return join(dir, "work", workId, "work.json") } function summary(workId: string) { - return JSON.parse(fs.readFileSync(path(workId), "utf8")) + return savedWorkSummary(dir, workId) } describe("readable work summaries", () => { @@ -397,10 +398,7 @@ describe("readable work summaries", () => { for (const snapshotPath of ["/work/plans/v1.md", "/work/plans/v2.md", "/work/plans/v2.md"]) appendWorkRecord(ctx, { type: "plan", path: "/project/.kimchi/plans/feature.md", snapshotPath }) await flushWorkSummaries() - expect(summary(workId).plans.map((plan: { snapshotPath: string }) => plan.snapshotPath)).toEqual([ - "/work/plans/v1.md", - "/work/plans/v2.md", - ]) + expect(summary(workId).plans.map((plan) => plan.snapshotPath)).toEqual(["/work/plans/v1.md", "/work/plans/v2.md"]) }) it("finishes the turn while summary publication is pending, but drains it on shutdown", async () => { const originalRename = asyncFs.rename @@ -509,7 +507,7 @@ describe("readable work summaries", () => { await flushWorkSummaries() expect( summary(workId) - .commits.map((entry: { sessionId: string }) => entry.sessionId) + .commits.map((entry) => entry.sessionId) .sort(), ).toEqual(["child", "parent"]) expect(summary(workId).fileTransitions).toEqual([ @@ -552,8 +550,8 @@ describe("readable work summaries", () => { const ctx = context() const request = recordProviderRequest(ctx) await flushWorkSummaries() - const existing = summary(request.workId) - existing.fileTransitions = undefined + // A version 1 summary written before file transitions were recorded. + const { fileTransitions: _, ...existing } = summary(request.workId) fs.writeFileSync(path(request.workId), JSON.stringify(existing)) const journal = join(dir, "work-attribution", "transitions", "old.jsonl") fs.mkdirSync(dirname(journal), { recursive: true }) @@ -605,8 +603,7 @@ describe("readable work summaries", () => { complete: false, }) await flushWorkSummaries() - const old = summary(workId) - old.fileObservations = undefined + const { fileObservations: _, ...old } = summary(workId) old.commits[0].fileMatches = [] fs.writeFileSync(path(workId), JSON.stringify(old)) const { size, mtimeMs } = fs.statSync(path(workId)) @@ -727,7 +724,7 @@ describe("readable work summaries", () => { `${JSON.stringify({ type: "request", requestId: "unpublished", version: 1, sessionId: "parent", workId, cwd: "/project", recordedAt: new Date().toISOString() })}\n`, ) expect((await launch()).ledgerReads).toBe(1) - expect(summary(workId).requests.map((row: { requestId: string }) => row.requestId)).toContain("unpublished") + expect(summary(workId).requests.map((row) => row.requestId)).toContain("unpublished") // Recovered ledgers may be unchanged even when their derived output disappears or is damaged. fs.utimesSync(ledger, past, past) for (const damaged of [undefined, "{"]) { @@ -793,10 +790,7 @@ describe("readable work summaries", () => { appendWorkRecord(ctx, { type: "request", requestId: "queued" }, workId) release() await flushWorkSummaries() - expect(summary(workId).requests.map((row: { requestId: string }) => row.requestId)).toEqual([ - "queued", - "during-release", - ]) + expect(summary(workId).requests.map((row) => row.requestId)).toEqual(["queued", "during-release"]) }) it("does one recovery scan per agent directory despite child session fanout", async () => { const workId = getWorkId(context()) @@ -993,6 +987,12 @@ console.log("ready"); await flushWorkSummaries();`, expect(errors).toEqual([]) expect(summary(workId).requests).toHaveLength(24) expect(summary(workId).sessions.sort()).toEqual(["session-0", "session-1", "session-2"]) + // The version 1 file migrated once; every append landed on a committed line boundary. + const head = JSON.parse(fs.readFileSync(path(workId), "utf8")) + const log = fs.readFileSync(join(directory, "rows", `requests.${head.logs.requests.generation}.jsonl`)) + expect(head).toMatchObject({ version: 2, logs: { requests: { rows: 24, bytes: log.length } } }) + expect(log.at(-1)).toBe(10) + expect(fs.readdirSync(join(directory, "rows"))).toEqual([`requests.${head.logs.requests.generation}.jsonl`]) } finally { if (reading) clearInterval(reading) for (const { child } of children) child.kill() diff --git a/src/extensions/work-attribution/summary.ts b/src/extensions/work-attribution/summary.ts index fcd53f66d..78d7fead1 100644 --- a/src/extensions/work-attribution/summary.ts +++ b/src/extensions/work-attribution/summary.ts @@ -1,13 +1,20 @@ import { createReadStream, existsSync, readdirSync, readFileSync, statSync, writeFileSync } from "node:fs" -import { mkdir, readdir, readFile, stat } from "node:fs/promises" +import { mkdir, readdir, stat } from "node:fs/promises" import { basename, dirname, join } from "node:path" import { setImmediate } from "node:timers/promises" import { getAgentDir } from "@earendil-works/pi-coding-agent" import { lock } from "proper-lockfile" -import { writeFileDurably } from "../../config/json.js" import { isWorkId } from "../../shared/work-id.js" import { mergePullRequestLinks } from "../pull-request-status/links.js" import { debugWorkAttribution } from "./diagnostics.js" +import { + COLLECTIONS, + type Collection, + ROW_KEYS, + updateWorkRows, + validWorkFiles, + type WorkSummaryView, +} from "./row-log.js" const LOCK_STALE_MS = 5000 const MERGE_BATCH_SIZE = 1000 @@ -19,6 +26,8 @@ const RECOVERY_MTIME_SLACK_MS = 2000 /** Background reads yield to the event loop after each chunk of this size. */ const READ_CHUNK_BYTES = 1_048_576 const BUDGET_CHECK_LINES = 1024 +/** The manifest stays small unless something grows without a bound; say so in diagnostics. */ +const MANIFEST_WARNING_BYTES = 256 * 1024 interface SummaryEntry { sessionId: string [key: string]: unknown @@ -50,18 +59,6 @@ export interface WorkRecord extends SummaryEntry { export type WorkRecordProblem = | { kind: "invalid"; path: string; line: number } | { kind: "unknown-type"; path: string; line: number; record: Record } -interface WorkSummary { - version: 1 - workId: string - sessions: string[] - workLinks: SummaryEntry[] - requests: SummaryEntry[] - plans: SummaryEntry[] - commits: SummaryEntry[] - fileTransitions: SummaryEntry[] - fileObservations: SummaryEntry[] - continuations: SummaryEntry[] -} interface PendingUpdate { records: WorkRecord[] complete: boolean @@ -125,51 +122,44 @@ function unknownType(value: unknown): value is Record { !KNOWN_RECORD_TYPES.has(value.type) ) } -function validSummary(value: unknown, workId: string): value is WorkSummary { +/** Field checks for one row of each collection, in version 1 `work.json` and in the row logs. */ +const ROW_CHECKS: Record boolean> = { + workLinks: (row) => entry(row, ["linkId", "sourceWorkId", "targetWorkId"]), + requests: (row) => entry(row, ["requestId"]), + plans: (row) => entry(row, ["path"]), + commits: (row) => entry(row, ["sha", "repository", "worktree"]), + fileTransitions: (row) => entry(row, ["transitionId", "toolCallId", "repository", "worktree", "path"]), + fileObservations: (row) => + entry(row, ["observationId", "toolCallId", "repository", "worktree", "source"]) && Array.isArray(row.files), + continuations: (row) => entry(row, ["source"]) && object(row.evidence), +} +/** Collections that summaries written before them lack. */ +const LATER_COLLECTIONS = new Set(["workLinks", "fileTransitions", "fileObservations", "continuations"]) +function validSummary(value: unknown, workId: string): value is WorkSummaryView { return ( object(value) && value.version === 1 && value.workId === workId && - (value.workLinks === undefined || - (Array.isArray(value.workLinks) && - value.workLinks.every((row) => entry(row, ["linkId", "sourceWorkId", "targetWorkId"])))) && Array.isArray(value.sessions) && value.sessions.every((session) => typeof session === "string") && - Array.isArray(value.requests) && - value.requests.every((row) => entry(row, ["requestId"])) && - Array.isArray(value.plans) && - value.plans.every((row) => entry(row, ["path"])) && - Array.isArray(value.commits) && - value.commits.every((row) => entry(row, ["sha", "repository", "worktree"])) && - (value.fileTransitions === undefined || - (Array.isArray(value.fileTransitions) && - value.fileTransitions.every((row) => - entry(row, ["transitionId", "toolCallId", "repository", "worktree", "path"]), - ))) && - (value.fileObservations === undefined || - (Array.isArray(value.fileObservations) && - value.fileObservations.every( - (row) => - entry(row, ["observationId", "toolCallId", "repository", "worktree", "source"]) && Array.isArray(row.files), - ))) && - (value.continuations === undefined || - (Array.isArray(value.continuations) && - value.continuations.every((row) => entry(row, ["source"]) && object(row.evidence)))) + COLLECTIONS.every((collection) => { + const rows = value[collection] + return ( + (rows === undefined && LATER_COLLECTIONS.has(collection)) || + (Array.isArray(rows) && rows.every(ROW_CHECKS[collection])) + ) + }) ) } -async function readSummary(path: string, workId: string): Promise { - try { - const value = JSON.parse(await readFile(path, "utf8")) - if (validSummary(value, workId)) - return { - ...value, - workLinks: value.workLinks ?? [], - fileTransitions: value.fileTransitions ?? [], - fileObservations: value.fileObservations ?? [], - continuations: value.continuations ?? [], - } - } catch (error) { - if (!(error instanceof SyntaxError) && (!object(error) || error.code !== "ENOENT")) throw error +/** A valid version 1 summary, with collections it predates as empty ones. */ +function versionOneSummary(value: unknown, workId: string): WorkSummaryView | undefined { + if (!validSummary(value, workId)) return undefined + return { + ...value, + workLinks: value.workLinks ?? [], + fileTransitions: value.fileTransitions ?? [], + fileObservations: value.fileObservations ?? [], + continuations: value.continuations ?? [], } } /** An interrupted append leaves a record cut off mid-value; any other unparseable line is damage. */ @@ -340,15 +330,7 @@ function strings(...values: unknown[]): string[] { ), ] } -function planKey(row: SummaryEntry): string { - return JSON.stringify([row.sessionId, row.path, row.snapshotPath]) -} -function commitKey(row: SummaryEntry): string { - return JSON.stringify([row.sessionId, row.sha, row.repository, row.worktree]) -} -function continuationKey(row: SummaryEntry): string { - return JSON.stringify([row.sessionId, row.source, row.evidence]) -} +const continuationKey = ROW_KEYS.continuations function latestObservation(previous: unknown, current: unknown): Record | undefined { if (!object(current) || typeof current.checkedAt !== "string" || !Number.isFinite(Date.parse(current.checkedAt))) return object(previous) ? previous : undefined @@ -398,33 +380,20 @@ function fileMatches(...values: unknown[]) { } return [...matches.values()] } -function recordKey(type: WorkRecord["type"], row: SummaryEntry): string { - switch (type) { - case "work_link": - return JSON.stringify([ - row.linkId, - row.revision, - row.sourceWorkId, - row.targetWorkId, - row.requestIds, - row.scope, - row.status, - row.evidence, - ]) - case "request": - case "request_dispatch": - case "request_response": - case "request_cost": - return JSON.stringify(row.requestId) - case "plan": - return planKey(row) - case "commit": - return commitKey(row) - case "file_observation": - return JSON.stringify(row.observationId) - default: - return JSON.stringify(row.transitionId) - } +/** The summary collection each record type merges into. */ +const RECORD_COLLECTIONS: Record, Collection> = { + work_link: "workLinks", + request: "requests", + request_dispatch: "requests", + request_response: "requests", + request_cost: "requests", + plan: "plans", + commit: "commits", + file_transition: "fileTransitions", + file_observation: "fileObservations", +} +function recordKey(type: Exclude, row: SummaryEntry): string { + return ROW_KEYS[RECORD_COLLECTIONS[type]](row) } /** Rows by key. A store that reads saved rows on demand can stand in for a Map. */ interface RowMap { @@ -433,30 +402,9 @@ interface RowMap { set(key: string, row: SummaryEntry): unknown } /** One work's rows while records merge: sessions in first-seen order, each collection by row key. */ -interface SummaryRows { - sessions: Set - workLinks: T - requests: T - plans: T - commits: T - fileTransitions: T - fileObservations: T - continuations: T -} -function summaryRows(summary: WorkSummary): SummaryRows> { - return { - sessions: new Set(summary.sessions), - workLinks: new Map(summary.workLinks.map((row) => [recordKey("work_link", row), row])), - requests: new Map(summary.requests.map((row) => [JSON.stringify(row.requestId), row])), - plans: new Map(summary.plans.map((row) => [planKey(row), row])), - commits: new Map(summary.commits.map((row) => [commitKey(row), row])), - fileTransitions: new Map(summary.fileTransitions.map((row) => [JSON.stringify(row.transitionId), row])), - fileObservations: new Map(summary.fileObservations.map((row) => [JSON.stringify(row.observationId), row])), - continuations: new Map(summary.continuations.map((row) => [continuationKey(row), row])), - } -} +export type SummaryRows = { sessions: Set } & Record /** Merge rules for one batch of records, applied row by row. */ -async function merge(summary: SummaryRows, records: WorkRecord[]): Promise { +export async function mergeWorkRecords(summary: SummaryRows, records: WorkRecord[]): Promise { const { sessions, continuations } = summary const entriesByType = { work_link: summary.workLinks, @@ -522,9 +470,6 @@ async function merge(summary: SummaryRows, records: WorkRecord[]): Promise } else entries.set(key, item) } } -function publish(directory: string, summary: WorkSummary, assertLease: () => void): Promise { - return writeFileDurably(join(directory, "work.json"), `${JSON.stringify(summary, null, 2)}\n`, assertLease) -} async function update( agentDir: string, workId: string, @@ -533,38 +478,22 @@ async function update( assertLease: () => void, ): Promise { assertLease() - const directory = join(agentDir, "work", workId) - const summary = await readSummary(join(directory, "work.json"), workId) - const published = summary && JSON.stringify(summary) - const value = summary ?? { - version: 1, + const head = await updateWorkRows( + join(agentDir, "work", workId), workId, - sessions: [], - workLinks: [], - requests: [], - plans: [], - commits: [], - fileTransitions: [], - fileObservations: [], - continuations: [], - } - const history = !summary && !complete ? readWorkRecords(agentDir).filter((row) => row.workId === workId) : [] - const rows = summaryRows(value) - await merge(rows, history.concat(records)) - value.sessions = [...rows.sessions] - for (const collection of [ - "workLinks", - "requests", - "plans", - "commits", - "fileTransitions", - "fileObservations", - "continuations", - ] as const) - value[collection] = [...rows[collection].values()] - if (published === JSON.stringify(value)) return - assertLease() - await publish(directory, value, assertLease) + { + summary: (value) => versionOneSummary(value, workId), + check: (collection, row) => ROW_CHECKS[collection](row), + assertLease, + }, + async (rows, rebuild) => { + const history = rebuild && !complete ? readWorkRecords(agentDir).filter((row) => row.workId === workId) : [] + await mergeWorkRecords(rows, history.concat(records)) + }, + ) + const bytes = head && Buffer.byteLength(JSON.stringify(head, null, 2)) + if (bytes && bytes > MANIFEST_WARNING_BYTES) + debugWorkAttribution(`Work summary manifest for ${workId} is ${bytes} bytes`) } function refresh(agentDir: string, workId: string, records: WorkRecord[], complete = false): Promise { const directory = join(agentDir, "work", workId) @@ -646,11 +575,13 @@ async function readRecovery(agentDir: string, stamp: string) { const summaries: Record = {} for (const [workId, fingerprint] of Object.entries(saved.summaries)) { if (!isWorkId(workId) || typeof fingerprint !== "string") return undefined - const path = join(agentDir, "work", workId, "work.json") + const directory = join(agentDir, "work", workId) + const path = join(directory, "work.json") if (!existsSync(path)) return undefined const current = summaryFingerprint(path) // Missing or damaged output requires all source ledgers, even when none changed. - if (current !== fingerprint && !(await readSummary(path, workId))) return undefined + if (current !== fingerprint && !(await validWorkFiles(directory, workId, (value) => validSummary(value, workId)))) + return undefined summaries[workId] = current } return { startedAt: saved.startedAt, summaries } diff --git a/tests/e2e/acp/work-continuation.test.ts b/tests/e2e/acp/work-continuation.test.ts index e5ad97e10..06d889ee6 100644 --- a/tests/e2e/acp/work-continuation.test.ts +++ b/tests/e2e/acp/work-continuation.test.ts @@ -4,6 +4,7 @@ import { mkdirSync, readdirSync, readFileSync, realpathSync, writeFileSync } fro import { join } from "node:path" import { afterEach, expect, it } from "vitest" import { isWorkMatchingRequest } from "../tui/support/fake-openai-server.js" +import { readWorkSummary } from "../tui/support/work-summary.js" import { type AcpFixture, startAcpFixture } from "./support/acp-fixture.js" import { newSession, prompt } from "./support/scenarios.js" @@ -313,20 +314,25 @@ it("uses separate calls to the selected model to continue and split Studio work expect(request.body).toMatchObject({ model: "basic" }) } await expect - .poll(() => { - const summary = JSON.parse(readFileSync(join(agentDir, "work", planned.workId, "work.json"), "utf8")) - return summary.requests.map((row: { requestId: string }) => row.requestId).sort() - }) + .poll(() => + readWorkSummary(agentDir, planned.workId) + ?.requests.map((row) => row.requestId) + .sort(), + ) .toEqual([planned.requestId, requestRecord(1).requestId, judgeRequests[2].headers["x-request-id"]].sort()) await expect - .poll(() => { - const summary = JSON.parse(readFileSync(join(agentDir, "work", unrelated.workId, "work.json"), "utf8")) - return summary.requests.map((row: { requestId: string }) => row.requestId) - }) + .poll(() => readWorkSummary(agentDir, unrelated.workId)?.requests.map((row) => row.requestId)) .toEqual([unrelated.requestId]) + // Every derived file of a work stays free of prompt text; only its intent and retained plans hold user text. for (const workId of [planned.workId, unrelated.workId]) { - const summary = readFileSync(join(agentDir, "work", workId, "work.json"), "utf8") - for (const privateText of [planningText, continuationText, unrelatedText, "Decide whether a user's"]) - expect(summary).not.toContain(privateText) + const folder = join(agentDir, "work", workId) + const files = readdirSync(folder, { recursive: true, withFileTypes: true }) + .filter((entry) => entry.isFile()) + .map((entry) => join(entry.parentPath, entry.name)) + .filter((path) => path !== join(folder, "intent.json") && !path.startsWith(join(folder, "plans"))) + expect(files.some((path) => path.startsWith(join(folder, "rows")))).toBe(true) + for (const path of files) + for (const privateText of [planningText, continuationText, unrelatedText, "Decide whether a user's"]) + expect(readFileSync(path, "utf8"), path).not.toContain(privateText) } }) diff --git a/tests/e2e/acp/work-pr-discovery.test.ts b/tests/e2e/acp/work-pr-discovery.test.ts index 5afe7f80f..bbb868172 100644 --- a/tests/e2e/acp/work-pr-discovery.test.ts +++ b/tests/e2e/acp/work-pr-discovery.test.ts @@ -5,6 +5,7 @@ import { tmpdir } from "node:os" import { join } from "node:path" import { fileURLToPath } from "node:url" import { afterEach, expect, it, vi } from "vitest" +import { readWorkSummary } from "../tui/support/work-summary.js" import { type AcpFixture, startAcpFixture } from "./support/acp-fixture.js" import { newSession, prompt } from "./support/scenarios.js" @@ -149,7 +150,7 @@ it("shows pending and discovered GitLab MR details over ACP without a model requ beforeLinked, ) await expect - .poll(() => JSON.parse(readFileSync(join(agentDir, "work", workId, "work.json"), "utf8")).commits) + .poll(() => readWorkSummary(agentDir, workId)?.commits) .toEqual([ expect.objectContaining({ sha, diff --git a/tests/e2e/tui/support/work-summary.ts b/tests/e2e/tui/support/work-summary.ts new file mode 100644 index 000000000..e3b6fe69b --- /dev/null +++ b/tests/e2e/tui/support/work-summary.ts @@ -0,0 +1,58 @@ +/** + * Reads a work's local summary the way docs/work-attribution.md describes it, independently of the product reader. + * Version 1 `work.json` holds every row. Version 2 `work.json` is a manifest whose `logs` name + * `rows/..jsonl`; only the first `bytes` of each log are committed, and a later row with the + * same key replaces an earlier one in place. + */ +import { readFileSync } from "node:fs" +import { join } from "node:path" + +type Row = Record + +const KEYS: Record string> = { + workLinks: (row) => + JSON.stringify([ + row.linkId, + row.revision, + row.sourceWorkId, + row.targetWorkId, + row.requestIds, + row.scope, + row.status, + row.evidence, + ]), + requests: (row) => JSON.stringify(row.requestId), + plans: (row) => JSON.stringify([row.sessionId, row.path, row.snapshotPath]), + commits: (row) => JSON.stringify([row.sessionId, row.sha, row.repository, row.worktree]), + fileTransitions: (row) => JSON.stringify(row.transitionId), + fileObservations: (row) => JSON.stringify(row.observationId), + continuations: (row) => JSON.stringify([row.sessionId, row.source, row.evidence]), +} + +/** The summary in the version 1 shape, or undefined while it is missing or a writer is replacing a log. */ +export function readWorkSummary(agentDir: string, workId: string): Record | undefined { + const folder = join(agentDir, "work", workId) + try { + const manifest = JSON.parse(readFileSync(join(folder, "work.json"), "utf8")) + if (manifest.version !== 2) return manifest + const view: Record = { version: 1, workId: manifest.workId, sessions: manifest.sessions } + for (const [collection, key] of Object.entries(KEYS)) { + const log = manifest.logs[collection] + const rows = new Map() + if (log) { + const text = readFileSync(join(folder, "rows", `${collection}.${log.generation}.jsonl`)) + .subarray(0, log.bytes) + .toString("utf8") + for (const line of text.split("\n")) { + if (!line) continue + const row = JSON.parse(line) + rows.set(key(row), row) + } + } + view[collection] = [...rows.values()] + } + return view + } catch { + return undefined + } +} diff --git a/tests/e2e/tui/work-attribution.test.ts b/tests/e2e/tui/work-attribution.test.ts index 321ab2cdc..3c33f12d8 100644 --- a/tests/e2e/tui/work-attribution.test.ts +++ b/tests/e2e/tui/work-attribution.test.ts @@ -8,6 +8,7 @@ import { check } from "proper-lockfile" import { fullText, viewText, waitForText } from "./support/assertions.js" import { type FakeResponseRequest, isWorkMatchingRequest } from "./support/fake-openai-server.js" import { launchKimchi, PROMPT_READY, runKimchiSession, TUI_TEST_CONFIG } from "./support/kimchi-fixture.js" +import { readWorkSummary } from "./support/work-summary.js" test.use(TUI_TEST_CONFIG) const models = [{ slug: "basic", displayName: "Fake Basic", contextWindow: 200_000, maxTokens: 8192 }] @@ -234,16 +235,13 @@ async function waitForSummary( workId: string, minimum: Partial>, ) { - const path = join(agentDir, "work", workId, "work.json") const deadline = Date.now() + 15_000 while (Date.now() < deadline) { - try { - const summary = JSON.parse(readFileSync(path, "utf8")) - if (Object.entries(minimum).every(([key, count]) => summary[key]?.length >= count)) return summary - } catch {} + const summary = readWorkSummary(agentDir, workId) + if (summary && Object.entries(minimum).every(([key, count]) => summary[key]?.length >= count)) return summary await sleep(50) } - throw new Error(`Work summary did not become ready: ${path}`) + throw new Error(`Work summary did not become ready: ${join(agentDir, "work", workId, "work.json")}`) } test("the next message recovers account tracking after its saved scope is lost", async ({ terminal }) => { diff --git a/tests/e2e/tui/work-pr-discovery.test.ts b/tests/e2e/tui/work-pr-discovery.test.ts index 9b01ec786..6b4ec1ff7 100644 --- a/tests/e2e/tui/work-pr-discovery.test.ts +++ b/tests/e2e/tui/work-pr-discovery.test.ts @@ -6,6 +6,7 @@ import { SessionManager } from "@earendil-works/pi-coding-agent" import { expect, Key, test } from "@microsoft/tui-test" import { viewText, waitForText } from "./support/assertions.js" import { PROMPT_READY, runKimchiSession, TUI_TEST_CONFIG } from "./support/kimchi-fixture.js" +import { readWorkSummary } from "./support/work-summary.js" test.use(TUI_TEST_CONFIG) @@ -133,9 +134,9 @@ test("pending work survives a GitHub login error and finds a PR while coding con expect(printed).toContain(`Work ID: ${workId}`) expect(printed).toContain("PR #731 open: https://github.com/example/kimchi-lab/pull/731") trace.step("the next background check finds the PR; Enter prints the work's details") - const summary = JSON.parse(readFileSync(join(fixture.agentDir, "work", workId, "work.json"), "utf8")) - expect(summary.commits).toHaveLength(1) - expect(summary.commits[0]).toMatchObject({ + const summary = readWorkSummary(fixture.agentDir, workId) + expect(summary?.commits).toHaveLength(1) + expect(summary?.commits[0]).toMatchObject({ sha: headSha, prLookup: { status: "linked" }, pullRequests: [{ number: 731, state: "open", headSha, repository: "example/kimchi-lab", host: "github.com" }], From a3c8511850fd17eaf322a3aefb734f9d885cdbe2 Mon Sep 17 00:00:00 2001 From: tautvydasLiekis Date: Fri, 9 Oct 2026 03:06:13 +0300 Subject: [PATCH 03/19] perf: read only commit rows when following rewritten commits --- src/extensions/work-attribution/commits.test.ts | 8 ++++++-- src/extensions/work-attribution/commits.ts | 5 +++-- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/src/extensions/work-attribution/commits.test.ts b/src/extensions/work-attribution/commits.test.ts index 7c4e63592..baf2a9977 100644 --- a/src/extensions/work-attribution/commits.test.ts +++ b/src/extensions/work-attribution/commits.test.ts @@ -2,7 +2,7 @@ import { execFileSync } from "node:child_process" import * as fs from "node:fs" import { mkdtempSync, realpathSync, rmSync, writeFileSync } from "node:fs" import { tmpdir } from "node:os" -import { join } from "node:path" +import { basename, join, sep } from "node:path" import { afterEach, beforeEach, describe, expect, it, vi } from "vitest" import { createContext } from "../__mocks__/context.js" import { createExtensionApi } from "../__mocks__/extension-api.js" @@ -464,7 +464,7 @@ await flushWorkSummaries();` ]) }, 15000) - it("follows another session's commit of the same work through a rebase", async () => { + it("follows another session's commit of the same work through a rebase, reading only commit rows", async () => { vi.stubEnv("PI_CODING_AGENT_DIR", join(directory, "agent")) const first = createContext({ cwd: repository, sessionManager: { getSessionId: () => "first" } }) const workId = attribution.getWorkId(first) @@ -486,6 +486,7 @@ await flushWorkSummaries();` // A later session of the same work only knows the commit from the work's summary. const second = createContext({ cwd: repository, sessionManager: { getSessionId: () => "second" } }) attribution.setWorkId(second, workId) + const opened = vi.spyOn(fs, "openSync") await createCommitTrackingBashTool(second).execute( "second-tool", { command: "git rebase -q main" }, @@ -493,6 +494,9 @@ await flushWorkSummaries();` undefined, second, ) + const logs = opened.mock.calls.map(([path]) => String(path)).filter((path) => path.includes(`${sep}rows${sep}`)) + expect(logs.length).toBeGreaterThan(0) + expect(logs.every((path) => basename(path).startsWith("commits."))).toBe(true) const recorded = fs .readFileSync(attribution.workLedgerPath(second), "utf8") .trim() diff --git a/src/extensions/work-attribution/commits.ts b/src/extensions/work-attribution/commits.ts index 3afce3e5d..eab5a4cc0 100644 --- a/src/extensions/work-attribution/commits.ts +++ b/src/extensions/work-attribution/commits.ts @@ -21,7 +21,7 @@ import { } from "../work-attribution.js" import { debugWorkAttribution } from "./diagnostics.js" import { observeToolFiles } from "./file-observations.js" -import { readWorkSummary } from "./row-log.js" +import { readWorkRows } from "./row-log.js" import { COST_PER_PR_RESOURCE_ID } from "./resource.js" const GIT_LOOKUP_TIMEOUT_MS = 2000 @@ -311,7 +311,8 @@ function recordedCommits(ctx: WorkContext, workId: string): Set { } catch {} } try { - for (const row of readWorkSummary(getAgentDir(), workId)?.commits ?? []) + // Only the commit rows: a long work's summary holds many more request rows. + for (const row of readWorkRows(getAgentDir(), workId, "commits") ?? []) if (typeof row.sha === "string") shas.add(row.sha) } catch {} return shas From eccc217d4b65c23f4dc7dd89db545ed5019435e7 Mon Sep 17 00:00:00 2001 From: tautvydasLiekis Date: Fri, 9 Oct 2026 03:21:22 +0300 Subject: [PATCH 04/19] perf: browse works from summary heads and cost totals --- .../work-attribution/browser.test.ts | 158 +++++++++-- src/extensions/work-attribution/browser.ts | 240 ++++++++-------- .../work-attribution/cost-details.ts | 257 +++++++++++++----- .../work-attribution/cost-sync.test.ts | 164 ++++++++++- src/extensions/work-attribution/cost-sync.ts | 19 +- .../work-attribution/row-log.test.ts | 19 ++ src/extensions/work-attribution/row-log.ts | 21 +- tests/e2e/tui/work-attribution.test.ts | 82 ++++++ 8 files changed, 735 insertions(+), 225 deletions(-) diff --git a/src/extensions/work-attribution/browser.test.ts b/src/extensions/work-attribution/browser.test.ts index 66afba9e5..dafc65c21 100644 --- a/src/extensions/work-attribution/browser.test.ts +++ b/src/extensions/work-attribution/browser.test.ts @@ -1,9 +1,14 @@ +import * as fs from "node:fs" import { mkdirSync, mkdtempSync, rmSync, utimesSync, writeFileSync } from "node:fs" +import * as asyncFs from "node:fs/promises" import { tmpdir } from "node:os" import { join } from "node:path" import { afterEach, beforeEach, describe, expect, it, vi } from "vitest" import { buildWorkBrowser, readWorkBrowser, type SavedWork } from "./browser.js" -import { workCostDetails } from "./cost-details.js" +import { workCostDetails, workCostTotals } from "./cost-details.js" +import { summaryHead, type WorkHead } from "./row-log.js" + +vi.mock("node:fs/promises", async (importOriginal) => ({ ...(await importOriginal()) })) const NOW = Date.parse("2026-10-08T12:00:00Z") const HOUR = 60 * 60_000 @@ -15,6 +20,7 @@ beforeEach(() => { vi.stubEnv("TZ", "UTC") }) afterEach(() => { + vi.restoreAllMocks() vi.unstubAllEnvs() rmSync(agentDir, { recursive: true, force: true }) }) @@ -40,11 +46,12 @@ function request(requestId: string, startedAt = "2026-10-08T09:00:00Z") { function costs( id: string, requests: { requestId: string; usd?: string; priced?: boolean; workIds?: string[]; linkedWorkIds?: string[] }[], + pullRequests: unknown[] = [], ) { return { version: 1, workId: id, - pullRequests: [], + pullRequests, requests: requests.map(({ requestId, usd = "0", priced = true, workIds = [id], linkedWorkIds }) => ({ requestId, workIds, @@ -85,20 +92,34 @@ function commit(sha: string, pullRequests: unknown[]) { pullRequests, } } -function save(id: string, files: { summary?: unknown; costs?: unknown; plans?: Record }, at = NOW) { +function save( + id: string, + files: { summary?: unknown; costs?: unknown; totals?: string; plans?: Record }, + at = NOW, +) { const folder = join(agentDir, "work", id) mkdirSync(join(folder, "plans"), { recursive: true }) + const totals = files.totals ?? (files.costs === undefined ? undefined : JSON.stringify(workCostTotals(files.costs))) for (const [name, value] of [ ["work.json", files.summary], ["costs.json", files.costs], + ["cost-totals.json", totals], ] as const) if (value !== undefined) writeFileSync(join(folder, name), typeof value === "string" ? value : JSON.stringify(value)) for (const [name, text] of Object.entries(files.plans ?? {})) writeFileSync(join(folder, "plans", name), text) if (files.summary !== undefined) utimesSync(join(folder, "work.json"), new Date(at), new Date(at)) } -function browse(works: SavedWork[], lines: string[] = []) { - return buildWorkBrowser(agentDir, { workId: works[0].workId, lines }, works) +function browse( + works: { workId: string; summary?: ReturnType; costs?: unknown }[], + lines: string[] = [], +) { + const saved: SavedWork[] = works.map((work) => ({ + workId: work.workId, + head: work.summary && summaryHead(work.summary, work.workId, new Date(NOW).toISOString()), + totals: work.costs && workCostTotals(work.costs), + })) + return buildWorkBrowser(agentDir, { workId: works[0].workId, lines }, saved) } describe("work browser rows", () => { @@ -205,22 +226,48 @@ describe("work browser rows", () => { ) }) - it("counts requests that connected works both saved once", () => { + it("shows each connected work's own spend and counts their shared report once", () => { const [first, second] = [1, 2].map(workId) - const shared = { requestId: "shared", usd: "0.250000000" } - const { spend } = browse([ + // The cost pass saves the whole connected group in each member's costs.json. + const group = (id: string) => + costs( + id, + [ + { requestId: "own", usd: "0.500000000", workIds: [first] }, + // A correction linked this planning request into the second work. + { requestId: "linked", usd: "0.250000000", workIds: [first], linkedWorkIds: [second] }, + { requestId: "other", usd: "0.125000000", workIds: [second] }, + ], + [ + { + key: '["github","github.com","7"]', + account: null, + pullRequest: pullRequest(7, "merged"), + workIds: [first, second], + knownCostUsd: "0.875000000", + totalCostUsd: "0.875000000", + explicit: { knownCostUsd: "0.875000000" }, + inferred: { knownCostUsd: "0.000000000" }, + }, + ], + ) + const { rows, spend } = browse([ { workId: first, - summary: summary(first, { requests: [request("own"), request("shared")] }), - costs: costs(first, [{ requestId: "own", usd: "0.500000000" }, shared]), - }, - { - workId: second, - summary: summary(second, { requests: [request("other")] }), - costs: costs(second, [shared, { requestId: "other", usd: "0.125000000" }]), + summary: summary(first, { requests: [request("own"), request("linked")] }), + costs: group(first), }, + { workId: second, summary: summary(second, { requests: [request("other")] }), costs: group(second) }, ]) + const row = (id: string) => rows.find((candidate) => candidate.workId === id) + expect(row(first)?.value).toBe("$0.7500 · no PR") + expect(row(second)?.value).toBe("$0.3750 · no PR") + expect(row(first)?.description).toContain("Prices: 2/2 requests priced, $0.750000000 USD.") + expect(row(second)?.description).toContain("Prices: 2/2 requests priced, $0.375000000 USD.") + // The PR's total covers every contributing work. + expect(row(second)?.description).toContain("Cost: $0.875000000 USD — https://github.com/example/kimchi/pull/7") + // The header counts requests the group shares once, as deduplicating by request ID did. expect(spend).toBe("$0.8750") }) @@ -343,7 +390,7 @@ describe("work browser rows", () => { requests: [request("y1")], plans: [{ sessionId: "session", path: "/gone.md", snapshotPath: "/gone/plans/missing.md" }], }), - costs: '{"requests":"not a list"}', + totals: '{"requests":"not a list"}', }) const { rows, spend } = await readWorkBrowser(agentDir, { workId: missing, lines: [] }, NOW) @@ -365,6 +412,85 @@ describe("work browser rows", () => { expect(spend).toBe("$0.1000 known so far") }) + it("labels an oversized version 1 summary as too large and keeps its PRs and partial spend", async () => { + const id = workId(1) + const pull = pullRequest(7, "open") + const requests = Array.from({ length: 3600 }, (_, index) => ({ + ...request(`r${index}`), + billingRows: [{ id: `bill-${index}`, costUsd: "0.001", note: "x".repeat(2400) }], + })) + save(id, { + summary: summary(id, { requests, commits: [commit("1", [pull])] }), + costs: costs( + id, + [{ requestId: "r0", usd: "0.250000000" }], + [ + { + key: '["github","github.com","7"]', + account: null, + pullRequest: pull, + workIds: [id], + knownCostUsd: "0.000000000", + totalCostUsd: null, + }, + ], + ), + }) + expect(fs.statSync(join(agentDir, "work", id, "work.json")).size).toBeGreaterThan(8 * 1024 * 1024) + + const { rows } = await readWorkBrowser(agentDir, { workId: workId(2), lines: [] }, NOW) + + expect(rows[1]).toMatchObject({ + label: " 00000001 summary too large", + // Requests newer than the last cost pass may be missing from the totals. + value: "$0.2500 known so far · PR #7 open", + }) + expect(rows[1].description).toContain( + `Summary too large to list until the work's next update: ${join(agentDir, "work", id, "work.json")}`, + ) + expect(rows[1].description).toContain("PR #7 open: https://github.com/example/kimchi/pull/7") + }) + + it("opens 30 long works from their manifests and cost totals alone", async () => { + const ids = Array.from({ length: 30 }, (_, index) => workId(index + 1)) + for (const [index, id] of ids.entries()) { + const folder = join(agentDir, "work", id) + mkdirSync(join(folder, "rows"), { recursive: true }) + const manifest: WorkHead = { + version: 2, + workId: id, + updatedAt: new Date(NOW).toISOString(), + sessions: ["session"], + logs: { requests: { generation: 3, bytes: 33_441_207, rows: 15_000 } }, + latest: { + activityAt: new Date(NOW - index * HOUR).toISOString(), + request: { startedAt: new Date(NOW - index * HOUR).toISOString(), repository: "/src/kimchi/.git" }, + branch: { recordedAt: "2026-10-08T08:00:00Z", branch: "feat/search" }, + }, + pullRequests: [pullRequest(700 + index, "open")], + } + writeFileSync(join(folder, "work.json"), JSON.stringify(manifest)) + // The logs and the full report are large; the panel must not open them. + writeFileSync(join(folder, "rows", "requests.3.jsonl"), "") + writeFileSync(join(folder, "costs.json"), "") + const priced = Array.from({ length: 14_990 }, (_, at) => ({ requestId: `${id}-${at}`, usd: "0.001000000" })) + writeFileSync(join(folder, "cost-totals.json"), JSON.stringify(workCostTotals(costs(id, priced)))) + } + const reads = vi.spyOn(asyncFs, "readFile") + const opened = vi.spyOn(asyncFs, "open") + + const { rows, spend } = await readWorkBrowser(agentDir, { workId: ids[0], lines: [] }, NOW) + + expect(rows).toHaveLength(30) + expect(rows[0].value).toBe("$14.99 known so far · PR #700 open") + expect(rows[0].description).toContain("Repository: kimchi · branch feat/search") + expect(rows[0].description).toContain("· 15000 requests") + expect(spend).toBe("$449.70 known so far") + const files = reads.mock.calls.map(([path]) => String(path)) + expect(files.every((path) => path.endsWith("work.json") || path.endsWith("cost-totals.json"))).toBe(true) + expect(opened).not.toHaveBeenCalled() + }) + it("describes a work with the details /work prints plus where and when it ran", async () => { const id = workId(1) save(id, { diff --git a/src/extensions/work-attribution/browser.ts b/src/extensions/work-attribution/browser.ts index 37cc00f9d..9c205b2f3 100644 --- a/src/extensions/work-attribution/browser.ts +++ b/src/extensions/work-attribution/browser.ts @@ -7,27 +7,27 @@ import { mergePullRequestLinks, pullRequestLabel } from "../pull-request-status/ import { storedPullRequests } from "../pull-request-status/provider-records.js" import type { WorkPullRequest } from "../pull-request-status/pull-requests.js" import type { WorkDetailsRequest } from "../work-attribution.js" -import { costDetailLines, knownSpend, ownRequests } from "./cost-details.js" -import { time } from "./costs.js" +import { costDetailLines, type knownSpend } from "./cost-details.js" +import { decimalNanos, time } from "./costs.js" +import { MAX_SUMMARY_BYTES, readWorkHead, type WorkHead } from "./row-log.js" import { object } from "./summary.js" -/** Besides the current work, the panel lists works with activity this recent. */ +/** Besides the current work, the panel lists works whose summary changed recently. */ const RECENT_MS = 35 * 24 * 60 * 60_000 const MAX_WORKS = 30 -/** Larger files are skipped rather than read; a plan is read only for its title. */ -const MAX_SUMMARY_BYTES = 8 * 1024 * 1024 +/** A plan is read only for its title. */ const MAX_PLAN_BYTES = 256 * 1024 -type Entry = Record type Spend = ReturnType +type Link = Pick -/** One work's saved files; any of them can be missing or damaged. */ +/** One work's saved files; any of them can be missing or damaged. Each stays small however long the work runs. */ export interface SavedWork { workId: string - /** Parsed `work.json`. */ - summary?: unknown - /** Parsed `costs.json`. */ - costs?: unknown + /** The `work.json` manifest, or "too-large" for a version 1 summary over the read limit. */ + head?: WorkHead | "too-large" + /** Parsed `cost-totals.json`. */ + totals?: unknown planTitle?: string /** `work.json` modification time, used when the summary has no timestamps. */ modifiedAt?: number @@ -57,7 +57,7 @@ export async function readWorkBrowser( now = Date.now(), ): Promise { const directory = join(agentDir, "work") - // Recording any activity rewrites work.json, so its modification time bounds what is read. + // Recording any activity commits a new work.json, so its modification time bounds what is read. const modified = new Map() await Promise.all( (await readdir(directory).catch(() => [])).filter(isWorkId).map(async (workId) => { @@ -66,53 +66,30 @@ export async function readWorkBrowser( }), ) - const candidates = [...modified] + const recent = [...modified] .filter(([workId, at]) => workId !== current.workId && at >= now - RECENT_MS) - .map(([workId]) => workId) - const [own, ...others] = await Promise.all( - [current.workId, ...candidates].map( - async (workId): Promise => ({ - workId, - summary: await readJson(join(directory, workId, "work.json")), - modifiedAt: modified.get(workId), - }), - ), - ) - // Cost passes rewrite work.json weeks after the last request, so saved activity times pick the works listed. - const recent = others - .map((work) => ({ work, activity: lastActivity(work) ?? 0 })) - .filter(({ activity }) => activity >= now - RECENT_MS) - .sort((left, right) => right.activity - left.activity || left.work.workId.localeCompare(right.work.workId)) + .sort(([, left], [, right]) => right - left) .slice(0, MAX_WORKS - 1) - .map(({ work }) => work) + .map(([workId]) => workId) const works = await Promise.all( - [own, ...recent].map(async (work): Promise => { - const folder = join(directory, work.workId) - const [costs, planTitle] = await Promise.all([ - readJson(join(folder, "costs.json")), - readPlanTitle(folder, work.summary), + [current.workId, ...recent].map(async (workId): Promise => { + const folder = join(directory, workId) + const [head, totals] = await Promise.all([ + readWorkHead(agentDir, workId), + readJson(join(folder, "cost-totals.json")), ]) - return { ...work, costs, planTitle } + return { + workId, + head, + totals, + planTitle: typeof head === "object" ? await readPlanTitle(folder, head) : undefined, + modifiedAt: modified.get(workId), + } }), ) return buildWorkBrowser(agentDir, current, works) } -/** The parsed `work.json`, when it is this work's. */ -function savedSummary({ workId, summary }: SavedWork): Entry | undefined { - return object(summary) && summary.workId === workId ? summary : undefined -} - -/** Newest request start, native edit or retained plan; the file time when the summary has none. */ -function lastActivity(work: SavedWork): number | undefined { - const summary = savedSummary(work) - const times = [ - ...entries(summary, "requests").map((row) => time(row.startedAt)), - ...[...entries(summary, "fileTransitions"), ...entries(summary, "plans")].map((row) => time(row.recordedAt)), - ].filter((at) => at !== undefined) - return times.length ? Math.max(...times) : work.modifiedAt -} - /** Rows from saved summaries. Only the current work has live PR lookup lines; the rest use their saved links. */ export function buildWorkBrowser(agentDir: string, current: WorkDetailsRequest, works: SavedWork[]): WorkBrowser { const built = works @@ -123,83 +100,113 @@ export function buildWorkBrowser(agentDir: string, current: WorkDetailsRequest, (right.activity ?? 0) - (left.activity ?? 0) || left.row.workId.localeCompare(right.row.workId), ) - // Connected works save the same requests in each costs.json; a saved price replaces a placeholder. - const requests = new Map() - for (const { spent } of built) - for (const request of spent ?? []) - if (requests.get(request.requestId)?.priceStatus === undefined) requests.set(request.requestId, request) + // Connected works' totals cover the same requests: count each group once, plus requests not priced yet. + const groups = new Map() + const spend: Spend = { priced: 0, total: 0, nanos: 0n } + for (const { group, unpriced } of built) { + spend.total += unpriced + if (group && !groups.has(group.key)) groups.set(group.key, group.spend) + } + for (const group of groups.values()) { + spend.priced += group.priced + spend.total += group.total + spend.nanos += group.nanos + } return { rows: built.map(({ row }) => row), spend: spendText( - knownSpend([...requests.values()]), - built.some(({ spent }) => !spent), + spend, + built.some(({ partial }) => partial), ), } } +function pullRequestLink(value: unknown): Link | undefined { + if ( + !object(value) || + (value.provider !== "github" && value.provider !== "gitlab") || + typeof value.number !== "number" || + (value.state !== "open" && value.state !== "closed" && value.state !== "merged") || + typeof value.url !== "string" + ) + return undefined + return { provider: value.provider, number: value.number, state: value.state, url: value.url } +} + +function spendOf(value: unknown): Spend | undefined { + return object(value) && typeof value.priced === "number" && typeof value.total === "number" + ? { priced: value.priced, total: value.total, nanos: decimalNanos(value.knownCostUsd) ?? 0n } + : undefined +} + +/** A work's saved `cost-totals.json`, when it is readable. */ +function costTotals(value: unknown, workId: string) { + if (!object(value) || value.workId !== workId || !Array.isArray(value.group) || !Array.isArray(value.pullRequests)) + return undefined + const own = spendOf(value.requests) + const group = spendOf(value.groupRequests) + if (!own || !group || !object(value.requests) || typeof value.requests.recorded !== "number") return undefined + return { + own, + recorded: value.requests.recorded, + group: { key: JSON.stringify(value.group), spend: group }, + /** PRs this work committed to. */ + links: value.pullRequests.flatMap((row) => { + const link = object(row) && row.own === true ? pullRequestLink(row.pullRequest) : undefined + return link ? [link] : [] + }), + } +} + function workRow(agentDir: string, work: SavedWork, current?: WorkDetailsRequest) { const { workId } = work const folder = join(agentDir, "work", workId) - const summary = savedSummary(work) - const requests = entries(summary, "requests") - const edits = entries(summary, "fileTransitions") - const commits = entries(summary, "commits") - const costs = - object(work.costs) && Array.isArray(work.costs.pullRequests) && Array.isArray(work.costs.requests) - ? work.costs.requests.filter((row): row is Entry => object(row) && typeof row.requestId === "string") - : undefined - // Requests recorded after the last cost pass are not in costs.json yet; they count as unpriced. - const saved = new Set(costs?.map((row) => row.requestId)) - const unpriced = requests.flatMap((row) => - typeof row.requestId === "string" && !saved.has(row.requestId) ? [{ requestId: row.requestId }] : [], - ) - - // A saved report also lists connected works' requests; the row shows this work's own spend. - const spent = summary || costs ? [...(costs ? ownRequests(costs, workId) : []), ...unpriced] : undefined - const links = mergePullRequestLinks(...commits.map((commit) => storedPullRequests(commit.pullRequests))) - const activity = lastActivity(work) - const { name, branch } = location(edits, commits, requests) + const head = typeof work.head === "object" ? work.head : undefined + const tooLarge = work.head === "too-large" + const totals = costTotals(work.totals, workId) + const requests = head?.logs.requests?.rows ?? 0 + // Requests recorded after the last cost pass are not in the totals yet; they count as unpriced. + const unpriced = head ? Math.max(0, requests - (totals?.recorded ?? 0)) : 0 + const spent: Spend | undefined = totals + ? { ...totals.own, total: totals.own.total + unpriced } + : head && { priced: 0, total: unpriced, nanos: 0n } + // A summary too large to read still has its PRs in the totals, but its newest requests may be unpriced. + const links: Link[] = head + ? mergePullRequestLinks(storedPullRequests(head.pullRequests)) + : tooLarge + ? (totals?.links ?? []) + : [] + const latest = head?.latest + const activity = time(latest?.activityAt) ?? work.modifiedAt + const path = + latest?.edit?.repository ?? latest?.commit?.repository ?? latest?.request?.repository ?? latest?.request?.cwd + const name = path === undefined ? undefined : repositoryName(path) + const branch = latest?.branch?.branch const title = plain(work.planTitle ?? [name, branch].filter(Boolean).join(" · ")) const pullRequests = current ? current.lines : links.map((pr) => `${pullRequestLabel(pr)}: ${pr.url}`) - const costLines = costDetailLines(work.costs, join(folder, "costs.json")) - const context = summary + const costLines = costDetailLines(work.totals, join(folder, "costs.json")) + const context = head ? [ `Repository: ${name ? plain(name) : "unknown"}${branch ? ` · branch ${plain(branch)}` : ""}`, - `Last activity: ${activity === undefined ? "unknown" : localTime(activity)} · ${requests.length} request${requests.length === 1 ? "" : "s"}`, + `Last activity: ${activity === undefined ? "unknown" : localTime(activity)} · ${requests} request${requests === 1 ? "" : "s"}`, + ] + : [ + `${tooLarge ? "Summary too large to list until the work's next update" : "Summary unavailable"}: ${join(folder, "work.json")}`, ] - : [`Summary unavailable: ${join(folder, "work.json")}`] const row: WorkRow = { workId, - label: `${current ? "●" : " "} ${workId.slice(0, 8)} ${title || (summary ? "untitled work" : "summary unavailable")}`, - value: `${spendText(spent && knownSpend(spent))} · ${pullRequestState(links)}`, + label: `${current ? "●" : " "} ${workId.slice(0, 8)} ${title || (head ? "untitled work" : tooLarge ? "summary too large" : "summary unavailable")}`, + value: `${spendText(spent, tooLarge)} · ${pullRequestState(links)}`, description: [`Work ID: ${workId}`, ...context, ...pullRequests, ...costLines].join("\n"), details: [`Work ID: ${workId}`, ...pullRequests, ...costLines].join("\n"), } - return { row, current: current !== undefined, activity, spent } -} - -/** Repository and branch of the latest native edit, else the repository of the latest commit or request. */ -function location(edits: Entry[], commits: Entry[], requests: Entry[]): { name?: string; branch?: string } { - const edit = newest( - edits.filter((row) => typeof row.repository === "string"), - "recordedAt", - ) - - const commit = newest( - commits.filter((row) => typeof row.repository === "string"), - "recordedAt", - ) - - const request = newest(requests, "startedAt") - const scope = request && object(request.scope) ? request.scope : undefined - const path = edit?.repository ?? commit?.repository ?? scope?.repository ?? request?.cwd - const branch = newest( - edits.filter((row) => typeof row.branch === "string"), - "recordedAt", - )?.branch return { - name: typeof path === "string" ? repositoryName(path) : undefined, - branch: typeof branch === "string" ? branch : undefined, + row, + current: current !== undefined, + activity, + partial: !spent || tooLarge, + unpriced, + group: totals?.group, } } @@ -228,7 +235,7 @@ function roundedUsd(nanos: bigint): string { return `$${digits.slice(0, -places)}.${digits.slice(-places)}` } -function pullRequestState(links: WorkPullRequest[]): string { +function pullRequestState(links: Link[]): string { if (links.length === 1) return pullRequestLabel(links[0]) if (!links.length) return "no PR" return `${links.length} ${links.every((pr) => pr.provider === "gitlab") ? "MRs" : "PRs"}` @@ -247,18 +254,6 @@ function plain(text: string): string { .trim() } -/** Object entries of one summary collection; a damaged collection reads as empty. */ -function entries(summary: unknown, key: string): Entry[] { - return object(summary) && Array.isArray(summary[key]) ? summary[key].filter(object) : [] -} - -/** The entry with the latest valid timestamp in `field`; later entries win ties. */ -function newest(rows: Entry[], field: string): Entry | undefined { - let latest: Entry | undefined - for (const row of rows) if (!latest || (time(row[field]) ?? 0) >= (time(latest[field]) ?? 0)) latest = row - return latest -} - /** Missing, special and oversized files read as undefined. */ async function readText(path: string, limit: number): Promise { try { @@ -279,11 +274,8 @@ async function readJson(path: string): Promise { } /** The latest plan's title, read from its retained copy inside this work's folder. */ -async function readPlanTitle(folder: string, summary: unknown): Promise { - const snapshot = newest( - entries(summary, "plans").filter((row) => typeof row.snapshotPath === "string"), - "recordedAt", - )?.snapshotPath +async function readPlanTitle(folder: string, head: WorkHead): Promise { + const snapshot = head.latest.plan?.snapshotPath if (typeof snapshot !== "string") return undefined const text = await readText(join(folder, "plans", basename(snapshot)), MAX_PLAN_BYTES) const title = text === undefined ? undefined : derivePlanTitle(text) diff --git a/src/extensions/work-attribution/cost-details.ts b/src/extensions/work-attribution/cost-details.ts index be96824c0..461733a13 100644 --- a/src/extensions/work-attribution/cost-details.ts +++ b/src/extensions/work-attribution/cost-details.ts @@ -3,7 +3,7 @@ import { join } from "node:path" import { isWorkId } from "../../shared/work-id.js" import { decimalNanos, usd } from "./costs.js" import { PRICED_TAG_LIMIT } from "./request-tags.js" -import { isWorkAccount, sameWorkAccount } from "./scope.js" +import { isWorkAccount, sameWorkAccount, type WorkAccount } from "./scope.js" import { object } from "./summary.js" // The cost lines /work shows, read from the work's saved costs.json. @@ -24,22 +24,181 @@ export function knownSpend(requests: Record[]): { priced: numbe } } -/** /work reads the last durable result; opening the command never waits for the network. */ +/** Request counts and the known subtotal, as `/work` prints them. */ +interface SpendTotals { + total: number + priced: number + knownCostUsd: string +} + +/** What `/work` and its browser print for one work: a bounded digest of its `costs.json`, by PR. */ +export interface WorkCostTotals { + version: 1 + workId: string + /** Connected works whose requests and PR totals `costs.json` covers; every member's file names the same ones. */ + group: string[] + /** Every request `costs.json` covers, connected works' included. */ + groupRequests: SpendTotals + /** Requests this work recorded or that a correction linked into it. */ + requests: SpendTotals & { + /** Those recorded in this work. Newer requests in `work.json` are not priced yet. */ + recorded: number + unresolved: number + inferred: number + shared: number + /** Requests sent without a billing tag, by reason; such requests are never priced. */ + untagged?: Record + /** Requests whose last billing refresh failed before any page, with the newest reason. */ + failedRefresh?: { count: number; reason?: string } + } + pullRequests: { + key: string + account: WorkAccount | null + /** This work has commits on the PR. */ + own: boolean + pullRequest: { provider: string; number: number; state: string; url: string } | null + totalCostUsd: string | null + knownCostUsd: string + /** Sure and likely parts of a merged or closed PR's spend. */ + sure?: string + likely?: string + /** An unmerged PR's spend so far, and whether every request it may include is priced. */ + soFar?: { knownCostUsd: string; complete: boolean } + }[] + /** The full report these totals come from. */ + report: "costs.json" +} + +function spendTotals(rows: Record[]): SpendTotals { + const { priced, total, nanos } = knownSpend(rows) + return { total, priced, knownCostUsd: usd(nanos) } +} + +/** Totals for the work a saved report belongs to. Its requests may include connected works'; PR rows count them all. */ +function untaggedTotals(rows: Record[]): { untagged?: Record } { + const untagged: Record = {} + for (const row of rows) + if (typeof row.billingTagSkipped === "string") + untagged[row.billingTagSkipped] = (untagged[row.billingTagSkipped] ?? 0) + 1 + return Object.keys(untagged).length ? { untagged } : {} +} +function failedRefreshTotals(rows: Record[]): { failedRefresh?: { count: number; reason?: string } } { + const failed = rows.flatMap((row) => + object(row.billingLookup) && row.billingLookup.status === "unavailable" ? [row.billingLookup] : [], + ) + if (!failed.length) return {} + const latest = failed.reduce((left, right) => + String(right.checkedAt ?? "") > String(left.checkedAt ?? "") ? right : left, + ) + return { + failedRefresh: { count: failed.length, ...(typeof latest.reason === "string" ? { reason: latest.reason } : {}) }, + } +} +export function workCostTotals(report: unknown): WorkCostTotals | undefined { + if ( + !object(report) || + typeof report.workId !== "string" || + !Array.isArray(report.pullRequests) || + !Array.isArray(report.requests) + ) + return undefined + const { workId } = report + const ids = (value: unknown) => (Array.isArray(value) ? value.filter((id) => typeof id === "string") : []) + const requests = report.requests.filter(object) + const pullRequests = report.pullRequests.filter(object) + // Own spend: the work's requests and requests a correction linked into it, not other connected works'. + const own = ownRequests(requests, workId) + const assigned = (allocation: string) => own.filter((row) => row.allocation === allocation).length + return { + version: 1, + workId, + group: [ + ...new Set([ + workId, + ...requests.flatMap((row) => [...ids(row.workIds), ...ids(row.linkedWorkIds)]), + ...pullRequests.flatMap((row) => ids(row.workIds)), + ]), + ].sort(), + groupRequests: spendTotals(requests), + requests: { + ...spendTotals(own), + recorded: requests.filter((row) => ids(row.workIds).includes(workId)).length, + unresolved: assigned("unknown"), + inferred: assigned("inferred"), + shared: assigned("shared"), + ...untaggedTotals(own), + ...failedRefreshTotals(own), + }, + pullRequests: pullRequests.map((row) => { + const pull = object(row.pullRequest) ? row.pullRequest : undefined + // An unmerged PR's spend stays outside confirmed and inferred totals until it merges. + const unmerged = pull && pull.state !== "merged" + return { + key: String(row.key), + account: isWorkAccount(row.account) ? row.account : null, + own: ids(row.workIds).includes(workId), + pullRequest: + pull && + (pull.provider === "github" || pull.provider === "gitlab") && + typeof pull.number === "number" && + typeof pull.state === "string" && + typeof pull.url === "string" + ? { provider: pull.provider, number: pull.number, state: pull.state, url: pull.url } + : null, + totalCostUsd: typeof row.totalCostUsd === "string" ? row.totalCostUsd : null, + knownCostUsd: String(row.knownCostUsd), + ...(!unmerged && object(row.explicit) && object(row.inferred) + ? { sure: String(row.explicit.knownCostUsd), likely: String(row.inferred.knownCostUsd) } + : {}), + ...(unmerged ? { soFar: spendSoFar(row, requests) } : {}), + } + }), + report: "costs.json", + } +} + +/** An open or closed PR's priced spend over the requests counted for it so far. */ +function spendSoFar(row: Record, requests: Record[]) { + const counted = requests.filter( + (request) => + request.allocation === "unmerged" && + Array.isArray(request.pullRequestIds) && + request.pullRequestIds.includes(row.key) && + (isWorkAccount(request.account) && isWorkAccount(row.account) + ? sameWorkAccount(request.account, row.account) + : request.account === row.account), + ) + return { + knownCostUsd: usd(counted.reduce((sum, request) => sum + (decimalNanos(request.knownCostUsd) ?? 0n), 0n)), + // Unpriced, shared or unresolved requests may still belong to this PR. + complete: + isWorkAccount(row.account) && + counted.every((request) => request.priceStatus === "priced") && + [row.sharedRequestIds, row.inferredRequestIds, row.unknownRequestIds].every( + (ids) => !Array.isArray(ids) || !ids.length, + ), + } +} + +/** /work reads the last durable result; opening the command never waits for the network or reads the full report. */ export function workCostDetails(agentDir: string, workId: string): string[] { if (!isWorkId(workId)) return [] - const path = join(agentDir, "work", workId, "costs.json") + const folder = join(agentDir, "work", workId) try { - return costDetailLines(JSON.parse(readFileSync(path, "utf8")), path) + return costDetailLines( + JSON.parse(readFileSync(join(folder, "cost-totals.json"), "utf8")), + join(folder, "costs.json"), + ) } catch { return ["Cost: unknown; waiting for billing"] } } -/** `/work` cost lines for a saved `costs.json` value read from `path`. */ +/** `/work` cost lines for saved `cost-totals.json` content; `path` names the full report. */ export function costDetailLines(value: unknown, path: string): string[] { - if (!object(value) || !Array.isArray(value.pullRequests)) return ["Cost: unknown; waiting for billing"] + if (!object(value) || !Array.isArray(value.pullRequests) || !object(value.requests)) + return ["Cost: unknown; waiting for billing"] const lines: string[] = [] - const requests = Array.isArray(value.requests) ? value.requests.filter(object) : [] for (const row of value.pullRequests) { if (!object(row)) continue if (value.pullRequests.some((other) => object(other) && other !== row && other.key === row.key)) @@ -49,75 +208,45 @@ export function costDetailLines(value: unknown, path: string): string[] { : "Account: unknown", ) const label = object(row.pullRequest) ? row.pullRequest.url : row.key - // An unmerged PR's spend stays outside confirmed and inferred totals until it merges. const state = object(row.pullRequest) && row.pullRequest.state !== "merged" ? row.pullRequest.state : undefined - if (state) { - const counted = requests.filter( - (request) => - request.allocation === "unmerged" && - Array.isArray(request.pullRequestIds) && - request.pullRequestIds.includes(row.key) && - (isWorkAccount(request.account) && isWorkAccount(row.account) - ? sameWorkAccount(request.account, row.account) - : request.account === row.account), - ) - - const spent = usd(counted.reduce((sum, request) => sum + (decimalNanos(request.knownCostUsd) ?? 0n), 0n)) - // Unpriced, shared or unresolved requests may still belong to this PR. - const complete = - isWorkAccount(row.account) && - counted.every((request) => request.priceStatus === "priced") && - [row.sharedRequestIds, row.inferredRequestIds, row.unknownRequestIds].every( - (ids) => !Array.isArray(ids) || !ids.length, - ) + if (state && object(row.soFar)) lines.push( - complete - ? `Cost so far: $${spent} USD (${state}) — ${label}` - : `Cost so far: unknown; $${spent} USD priced (${state}) — ${label}`, + row.soFar.complete === true + ? `Cost so far: $${row.soFar.knownCostUsd} USD (${state}) — ${label}` + : `Cost so far: unknown; $${row.soFar.knownCostUsd} USD priced (${state}) — ${label}`, ) - } else if (typeof row.totalCostUsd === "string") lines.push(`Cost: $${row.totalCostUsd} USD — ${label}`) + else if (typeof row.totalCostUsd === "string") lines.push(`Cost: $${row.totalCostUsd} USD — ${label}`) else lines.push(`Cost: unknown; $${row.knownCostUsd} USD priced so far — ${label}`) - if (!state && object(row.explicit) && object(row.inferred)) + if (!state && typeof row.sure === "string" && typeof row.likely === "string") lines.push( - `Confirmed: $${row.explicit.knownCostUsd} USD; inferred: $${row.inferred.knownCostUsd} USD${row.totalCostUsd === null ? " known so far" : ""}.`, + `Confirmed: $${row.sure} USD; inferred: $${row.likely} USD${row.totalCostUsd === null ? " known so far" : ""}.`, ) } - if (Array.isArray(value.requests)) { - // PR lines above include connected works; this one is the work's own spend, even without a PR. - const own = typeof value.workId === "string" ? ownRequests(requests, value.workId) : requests - const spend = knownSpend(own) - const unresolved = own.filter((row) => row.allocation === "unknown").length - const inferred = own.filter((row) => row.allocation === "inferred").length - const shared = own.filter((row) => row.allocation === "shared").length + // Work without a PR has no cost line above, so its own priced spend is shown here. + const { priced, total, knownCostUsd, unresolved, inferred, shared } = value.requests + if (typeof priced === "number" && typeof total === "number") lines.push( - `Prices: ${spend.priced}/${spend.total} requests priced, $${usd(spend.nanos)} USD${spend.priced < spend.total ? " known so far" : ""}. PR assignments: ${unresolved} unresolved, ${inferred} inferred, ${shared} shared.`, + `Prices: ${priced}/${total} requests priced, $${knownCostUsd} USD${priced < total ? " known so far" : ""}. PR assignments: ${unresolved} unresolved, ${inferred} inferred, ${shared} shared.`, ) - - const untagged = new Map() - for (const row of own) - if (typeof row.billingTagSkipped === "string") - untagged.set(row.billingTagSkipped, (untagged.get(row.billingTagSkipped) ?? 0) + 1) - if (untagged.size) { - const count = [...untagged.values()].reduce((sum, value) => sum + value, 0) - const reasons = [...untagged] - .sort(([left], [right]) => left.localeCompare(right)) - .map(([reason, total]) => `${untagged.size > 1 ? `${total} ` : ""}${reason.replaceAll("-", " ")}`) - lines.push( - `${count} request${count === 1 ? "" : "s"} untagged: ${reasons.join(", ")}${untagged.has("tag-limit") ? ` (Kimchi adds model and phase tags; keep at most ${PRICED_TAG_LIMIT} in /tags)` : ""}.`, - ) - } - - const failed = own.flatMap((row) => - object(row.billingLookup) && row.billingLookup.status === "unavailable" ? [row.billingLookup] : [], + // Untagged requests and failed refreshes of the work's own requests explain unknown prices. + if (object(value.requests.untagged)) { + const untagged = Object.entries(value.requests.untagged).filter( + (entry): entry is [string, number] => typeof entry[1] === "number", ) - if (failed.length) { - const latest = failed.reduce((left, right) => - String(right.checkedAt ?? "") > String(left.checkedAt ?? "") ? right : left, - ) + const count = untagged.reduce((sum, [, total]) => sum + total, 0) + const reasons = untagged + .sort(([left], [right]) => left.localeCompare(right)) + .map(([reason, total]) => `${untagged.length > 1 ? `${total} ` : ""}${reason.replaceAll("-", " ")}`) + if (count) lines.push( - `Last billing refresh failed for ${failed.length} request${failed.length === 1 ? "" : "s"}${typeof latest.reason === "string" ? `: ${latest.reason}` : ""}.`, + `${count} request${count === 1 ? "" : "s"} untagged: ${reasons.join(", ")}${untagged.some(([reason]) => reason === "tag-limit") ? ` (Kimchi adds model and phase tags; keep at most ${PRICED_TAG_LIMIT} in /tags)` : ""}.`, ) - } + } + if (object(value.requests.failedRefresh) && typeof value.requests.failedRefresh.count === "number") { + const { count, reason } = value.requests.failedRefresh + lines.push( + `Last billing refresh failed for ${count} request${count === 1 ? "" : "s"}${typeof reason === "string" ? `: ${reason}` : ""}.`, + ) } return [...lines, `Cost details: ${path}`] } diff --git a/src/extensions/work-attribution/cost-sync.test.ts b/src/extensions/work-attribution/cost-sync.test.ts index 87e17b9a6..3edd79e68 100644 --- a/src/extensions/work-attribution/cost-sync.test.ts +++ b/src/extensions/work-attribution/cost-sync.test.ts @@ -1,21 +1,35 @@ import { randomUUID } from "node:crypto" import * as fs from "node:fs" -import { mkdirSync, mkdtempSync, readdirSync, readFileSync, rmSync, statSync, unlinkSync, writeFileSync } from "node:fs" +import { + existsSync, + mkdirSync, + mkdtempSync, + readdirSync, + readFileSync, + rmSync, + statSync, + unlinkSync, + writeFileSync, +} from "node:fs" import { tmpdir } from "node:os" import { join } from "node:path" import { afterEach, beforeEach, describe, expect, it, vi } from "vitest" import * as json from "../../config/json.js" import * as config from "../../config.js" +import { isWorkId } from "../../shared/work-id.js" import { createContext } from "../__mocks__/context.js" import { savedWorkSummary } from "../__mocks__/work-summary.js" import { appendWorkRecord, getWorkId } from "../work-attribution.js" import { captureBillingSource, requestTagSelector } from "./billing-source.js" -import { workCostDetails } from "./cost-details.js" +import { readWorkBrowser } from "./browser.js" +import { costDetailLines, knownSpend, ownRequests, workCostDetails, workCostTotals } from "./cost-details.js" import { readWorkCostReport, reconcileWorkCosts } from "./cost-sync.js" import * as costs from "./costs.js" -import { calculatePullRequestCosts } from "./costs.js" +import { calculatePullRequestCosts, decimalNanos, usd } from "./costs.js" +import { PRICED_TAG_LIMIT } from "./request-tags.js" +import { isWorkAccount, sameWorkAccount } from "./scope.js" import * as summary from "./summary.js" -import { flushWorkSummaries, readWorkRecords, recoverWorkSummaries, type WorkRecord } from "./summary.js" +import { flushWorkSummaries, object, readWorkRecords, recoverWorkSummaries, type WorkRecord } from "./summary.js" vi.mock("../../config.js", async (original) => ({ ...(await original()) })) vi.mock("../../config/json.js", async (original) => ({ ...(await original()) })) @@ -59,8 +73,104 @@ beforeEach(() => { ) vi.stubGlobal("fetch", fetchMock) }) +/** The `/work` cost lines the previous release printed from a whole `costs.json`. */ +function previousCostDetailLines(value: unknown, path: string): string[] { + if (!object(value) || !Array.isArray(value.pullRequests)) return ["Cost: unknown; waiting for billing"] + const lines: string[] = [] + const requests = Array.isArray(value.requests) ? value.requests.filter(object) : [] + for (const row of value.pullRequests) { + if (!object(row)) continue + if (value.pullRequests.some((other) => object(other) && other !== row && other.key === row.key)) + lines.push( + isWorkAccount(row.account) + ? `Account: ${row.account.organizationId} / ${row.account.userId} (${row.account.apiUrl})` + : "Account: unknown", + ) + const label = object(row.pullRequest) ? row.pullRequest.url : row.key + // An unmerged PR's spend stays outside sure and likely totals until it merges. + const state = object(row.pullRequest) && row.pullRequest.state !== "merged" ? row.pullRequest.state : undefined + if (state) { + const counted = requests.filter( + (request) => + request.allocation === "unmerged" && + Array.isArray(request.pullRequestIds) && + request.pullRequestIds.includes(row.key) && + (isWorkAccount(request.account) && isWorkAccount(row.account) + ? sameWorkAccount(request.account, row.account) + : request.account === row.account), + ) + const spent = usd(counted.reduce((sum, request) => sum + (decimalNanos(request.knownCostUsd) ?? 0n), 0n)) + // Unpriced, shared or unresolved requests may still belong to this PR. + const complete = + isWorkAccount(row.account) && + counted.every((request) => request.priceStatus === "priced") && + [row.sharedRequestIds, row.inferredRequestIds, row.unknownRequestIds].every( + (ids) => !Array.isArray(ids) || !ids.length, + ) + lines.push( + complete + ? `Cost so far: $${spent} USD (${state}) — ${label}` + : `Cost so far: unknown; $${spent} USD confirmed (${state}) — ${label}`, + ) + } else if (typeof row.totalCostUsd === "string") lines.push(`Cost: $${row.totalCostUsd} USD — ${label}`) + else lines.push(`Cost: unknown; $${row.knownCostUsd} USD confirmed so far — ${label}`) + if (!state && object(row.explicit) && object(row.inferred)) + lines.push( + `Sure: $${row.explicit.knownCostUsd} USD; likely: $${row.inferred.knownCostUsd} USD${row.totalCostUsd === null ? " known so far" : ""}.`, + ) + } + if (Array.isArray(value.requests)) { + // PR lines above include connected works; this one is the work's own spend, even without a PR. + const own = typeof value.workId === "string" ? ownRequests(requests, value.workId) : requests + const spend = knownSpend(own) + const unresolved = own.filter((row) => row.allocation === "unknown").length + const inferred = own.filter((row) => row.allocation === "inferred").length + const shared = own.filter((row) => row.allocation === "shared").length + lines.push( + `Prices: ${spend.priced}/${spend.total} requests priced, $${usd(spend.nanos)} USD${spend.priced < spend.total ? " known so far" : ""}. PR assignments: ${unresolved} unresolved, ${inferred} inferred, ${shared} shared.`, + ) + const untagged = new Map() + for (const row of own) + if (typeof row.billingTagSkipped === "string") + untagged.set(row.billingTagSkipped, (untagged.get(row.billingTagSkipped) ?? 0) + 1) + if (untagged.size) { + const count = [...untagged.values()].reduce((sum, value) => sum + value, 0) + const reasons = [...untagged] + .sort(([left], [right]) => left.localeCompare(right)) + .map(([reason, total]) => `${untagged.size > 1 ? `${total} ` : ""}${reason.replaceAll("-", " ")}`) + lines.push( + `${count} request${count === 1 ? "" : "s"} untagged: ${reasons.join(", ")}${untagged.has("tag-limit") ? ` (Kimchi adds model and phase tags; keep at most ${PRICED_TAG_LIMIT} in /tags)` : ""}.`, + ) + } + const failed = own.flatMap((row) => + object(row.billingLookup) && row.billingLookup.status === "unavailable" ? [row.billingLookup] : [], + ) + if (failed.length) { + const latest = failed.reduce((left, right) => + String(right.checkedAt ?? "") > String(left.checkedAt ?? "") ? right : left, + ) + lines.push( + `Last billing refresh failed for ${failed.length} request${failed.length === 1 ? "" : "s"}${typeof latest.reason === "string" ? `: ${latest.reason}` : ""}.`, + ) + } + } + return [...lines, `Cost details: ${path}`] +} + afterEach(async () => { await flushWorkSummaries() + // Every report a test saved prints the same `/work` lines from its bounded totals as from the whole file. + // Connected works differ only in the Prices line, which now counts the work's own requests. + for (const workId of existsSync(join(dir, "work")) ? readdirSync(join(dir, "work")).filter(isWorkId) : []) { + const path = join(dir, "work", workId, "costs.json") + if (!existsSync(path)) continue + const saved = JSON.parse(readFileSync(path, "utf8")) + const totals = workCostTotals(saved) + if (!totals) continue + const own = (lines: string[]) => + totals.group.length > 1 ? lines.filter((line) => !line.startsWith("Prices:")) : lines + expect(own(costDetailLines(totals, path))).toEqual(own(previousCostDetailLines(saved, path))) + } vi.restoreAllMocks() vi.unstubAllEnvs() vi.unstubAllGlobals() @@ -199,6 +309,45 @@ describe("automatic exact work cost lookup", () => { ) }) + it("prints each connected work's own spend while their PR total counts every contributor", async () => { + const first = tagged("first") + const second = tagged("second") + fetchMock.mockImplementation(async (input) => { + const url = new URL(String(input)) + if (url.pathname.endsWith("api-keys:verify")) return Response.json({ organizationId: ORG, userId: PROMPT }) + const isFirst = url.searchParams.get("tags") === `kimchi-request:${first.requestId}` + return Response.json({ items: [{ id: isFirst ? ROW : ORG, totalPrice: isFirst ? "1" : "2" }] }) + }) + await sync() + for (const [{ workId }, own] of [ + [first, "1.000000000"], + [second, "2.000000000"], + ] as const) { + const lines = workCostDetails(dir, workId) + expect(lines).toContain("Cost: $3.000000000 USD — https://github.com/example/repo/pull/1") + expect(lines.find((line) => line.startsWith("Prices:"))).toMatch( + new RegExp(`^Prices: 1/1 requests priced, \\$${own.replace(".", "\\.")} USD\\.`), + ) + // The bounded totals name the same connected group in each member's folder. + expect(JSON.parse(readFileSync(join(dir, "work", workId, "cost-totals.json"), "utf8")).group).toEqual( + [first.workId, second.workId].sort(), + ) + } + const totals = join(dir, "work", first.workId, "cost-totals.json") + const written = statSync(totals).mtimeMs + vi.spyOn(Date, "now").mockReturnValue(Date.now() + 10 * 60_000) + await sync() + expect(statSync(totals).mtimeMs).toBe(written) + await flushWorkSummaries() + // The browser shows the same own amounts and counts the connected works' requests once. + const { rows, spend } = await readWorkBrowser(dir, { workId: first.workId, lines: [] }) + expect(Object.fromEntries(rows.map((row) => [row.workId, row.value]))).toEqual({ + [first.workId]: "$1.00 · PR #1 merged", + [second.workId]: "$2.00 · PR #1 merged", + }) + expect(spend).toBe("$3.00") + }) + it("saves only the selected work's unrelated requests in its aggregate buckets", async () => { const works = ["first", "second"].map((sessionId) => { const ctx = createContext({ cwd: dir, sessionManager: { getSessionId: () => sessionId } }) @@ -243,7 +392,7 @@ describe("automatic exact work cost lookup", () => { knownCostUsd: "0.000000000", totalCostUsd: null, }) - writeFileSync(join(dir, "work", workId, "costs.json"), JSON.stringify(saved)) + writeFileSync(join(dir, "work", workId, "cost-totals.json"), JSON.stringify(workCostTotals(saved))) const lines = workCostDetails(dir, workId) expect(lines).toContain(`Account: ${ORG} / ${PROMPT} (${API})`) expect(lines).toContain("Account: unknown") @@ -290,7 +439,10 @@ describe("automatic exact work cost lookup", () => { const save = (observations: Parameters[1]) => { const costs = calculatePullRequestCosts(records, observations) mkdirSync(join(dir, "work", workId), { recursive: true }) - writeFileSync(join(dir, "work", workId, "costs.json"), JSON.stringify({ version: 1, workId, ...costs })) + writeFileSync( + join(dir, "work", workId, "cost-totals.json"), + JSON.stringify(workCostTotals({ version: 1, workId, ...costs })), + ) } save([]) expect(workCostDetails(dir, workId)).toContain( diff --git a/src/extensions/work-attribution/cost-sync.ts b/src/extensions/work-attribution/cost-sync.ts index 7215df2a4..a8bbf2d5b 100644 --- a/src/extensions/work-attribution/cost-sync.ts +++ b/src/extensions/work-attribution/cost-sync.ts @@ -19,6 +19,7 @@ import { openBilling, } from "./billing-evidence.js" import { type BillingSource, captureBillingSource, LOOKUP_WINDOW_MS, sameBillingSource } from "./billing-source.js" +import { workCostTotals } from "./cost-details.js" import { calculatePullRequestCosts, type PullRequestCost, @@ -197,6 +198,12 @@ function group(groups: Map, key: K, value: T): void { else groups.set(key, [value]) } +async function writeChanged(path: string, content: string, assertLease: () => void): Promise { + try { + if (readFileSync(path, "utf8") === content) return + } catch {} + await writeFileDurably(path, content, assertLease) +} async function publishReports( agentDir: string, { displays, report, workIds }: CostState, @@ -269,11 +276,13 @@ async function publishReports( await mkdir(directory, { recursive: true, mode: 0o700 }) // ponytail: connected works repeat in per-work files; use one shared snapshot if storage growth warrants it. const value = { version: 1, workId, ...content(component(workId)) } - const text = `${JSON.stringify(value, null, 2)}\n` - try { - if (readFileSync(join(directory, "costs.json"), "utf8") === text) continue - } catch {} - await writeFileDurably(join(directory, "costs.json"), text, assertLease) + // `/work` and its browser read only these bounded totals. + await writeChanged( + join(directory, "cost-totals.json"), + `${JSON.stringify(workCostTotals(value), null, 2)}\n`, + assertLease, + ) + await writeChanged(join(directory, "costs.json"), `${JSON.stringify(value, null, 2)}\n`, assertLease) } } diff --git a/src/extensions/work-attribution/row-log.test.ts b/src/extensions/work-attribution/row-log.test.ts index 6ab03e3e5..e4a5e5e6f 100644 --- a/src/extensions/work-attribution/row-log.test.ts +++ b/src/extensions/work-attribution/row-log.test.ts @@ -8,6 +8,8 @@ import { performance } from "node:perf_hooks" import { afterEach, beforeEach, describe, expect, it, vi } from "vitest" import { createContext } from "../__mocks__/context.js" import { appendWorkRecord } from "../work-attribution.js" +import { readWorkBrowser } from "./browser.js" +import { workCostTotals } from "./cost-details.js" import { COLLECTIONS, type Collection, @@ -779,5 +781,22 @@ describe("a large work", () => { expect(opened).not.toHaveBeenCalled() expect(vi.mocked(asyncFs.open)).not.toHaveBeenCalled() expect(vi.mocked(fs.readSync)).not.toHaveBeenCalled() + + // The /work browser lists it from the manifest and the bounded cost totals alone. + const priced = summary.requests.map((row) => ({ + requestId: row.requestId, + workIds: [workId], + priceStatus: "priced", + knownCostUsd: "0.001234560", + })) + fs.writeFileSync( + join(folder(workId), "cost-totals.json"), + JSON.stringify(workCostTotals({ workId, pullRequests: [], requests: priced })), + ) + const browser = await readWorkBrowser(dir, { workId, lines: [] }) + expect(browser.rows[0].value).toBe("$24.69 known so far · no PR") + expect(browser.rows[0].description).toContain("· 20011 requests") + expect(opened).not.toHaveBeenCalled() + expect(vi.mocked(fs.readSync)).not.toHaveBeenCalled() }, 120_000) }) diff --git a/src/extensions/work-attribution/row-log.ts b/src/extensions/work-attribution/row-log.ts index d499ef611..49f32db10 100644 --- a/src/extensions/work-attribution/row-log.ts +++ b/src/extensions/work-attribution/row-log.ts @@ -125,6 +125,10 @@ function count(value: unknown, minimum = 0): value is number { function isRow(value: unknown): value is Row { return object(value) && typeof value.sessionId === "string" } +/** Version 1 readers were lenient: any object in a collection is a row. The writer validates them. */ +function savedRow(value: unknown): value is Row { + return object(value) +} function text(value: unknown): string | undefined { return typeof value === "string" ? value : undefined } @@ -177,7 +181,7 @@ function versionOneView(value: unknown, workId: string): WorkSummaryView | undef const sessions = Array.isArray(value.sessions) ? value.sessions.filter((session) => typeof session === "string") : [] return summaryView(workId, sessions, (collection) => { const rows = value[collection] - return Array.isArray(rows) ? rows.filter(isRow) : [] + return Array.isArray(rows) ? rows.filter(savedRow) : [] }) } @@ -295,20 +299,17 @@ export async function readWorkHead( } catch { return undefined } - if (isWorkHead(value, workId)) return value + return isWorkHead(value, workId) ? value : summaryHead(value, workId, new Date(modifiedAt).toISOString()) +} + +/** The head of a version 1 summary, computed from its rows; they stay inside `work.json` (generation 0). */ +export function summaryHead(value: unknown, workId: string, updatedAt: string): WorkHead | undefined { const view = versionOneView(value, workId) if (!view) return undefined const logs: Partial> = {} for (const collection of COLLECTIONS) if (view[collection].length) logs[collection] = { generation: 0, bytes: 0, rows: view[collection].length } - return { - version: 2, - workId, - updatedAt: new Date(modifiedAt).toISOString(), - sessions: view.sessions, - logs, - ...summaryLatest((collection) => view[collection]), - } + return { version: 2, workId, updatedAt, sessions: view.sessions, logs, ...summaryLatest((name) => view[name]) } } /** Milliseconds of an ISO 8601 timestamp, as `/work` orders rows; impossible dates are not times. */ diff --git a/tests/e2e/tui/work-attribution.test.ts b/tests/e2e/tui/work-attribution.test.ts index 3c33f12d8..2720a7089 100644 --- a/tests/e2e/tui/work-attribution.test.ts +++ b/tests/e2e/tui/work-attribution.test.ts @@ -288,6 +288,88 @@ test("the next message recovers account tracking after its saved scope is lost", ) }) +test("/work keeps listing an oversized summary from an older build with its PR and spend so far", async ({ + terminal, +}) => { + const workId = randomUUID() + const url = "https://github.com/example/kimchi/pull/7" + await runKimchiSession( + terminal, + { + artifactName: "work-browser-oversized-summary", + account, + gitInit: true, + models, + responses: [], + seedHome(home) { + // An older build kept every row in work.json; this one is larger than the 8 MiB /work reads. + const folder = join(home, ".config/kimchi/harness/work", workId) + mkdirSync(folder, { recursive: true }) + const requests = Array.from({ length: 3600 }, (_, index) => ({ + requestId: String(index), + sessionId: "older", + billingRows: [{ id: String(index), costUsd: "0.001", note: "x".repeat(2500) }], + })) + const pullRequest = { provider: "github", host: "github.com", number: 7, url, state: "open" } + const commit = { + sha: "a".repeat(40), + repository: "/src/kimchi/.git", + worktree: "/src/kimchi", + sessionId: "older", + } + writeFileSync( + join(folder, "work.json"), + JSON.stringify({ + version: 1, + workId, + sessions: ["older"], + requests, + plans: [], + commits: [{ ...commit, pullRequests: [pullRequest] }], + }), + ) + // The cost pass's bounded totals still name the work's PR and its spend so far. + const spend = { total: 3600, priced: 3600, knownCostUsd: "3.600000000" } + writeFileSync( + join(folder, "cost-totals.json"), + JSON.stringify({ + version: 1, + workId, + group: [workId], + groupRequests: spend, + requests: { ...spend, recorded: 3600, unresolved: 0, inferred: 0, shared: 0 }, + pullRequests: [ + { + key: '["github","github.com","7"]', + account: null, + own: true, + pullRequest: { provider: "github", number: 7, state: "open", url }, + totalCostUsd: null, + knownCostUsd: "0.000000000", + soFar: { knownCostUsd: "3.600000000", complete: false }, + }, + ], + report: "costs.json", + }), + ) + }, + }, + async (_fixture, trace) => { + terminal.submit("/work") + await waitForText(terminal, "summary too large", { full: false }) + const listed = viewText(terminal) + .split("\n") + .find((line) => line.includes(workId.slice(0, 8))) + expect(listed).toContain("summary too large") + // Requests newer than the last cost pass may be missing, so the amount stays partial. + expect(listed).toContain("$3.60 known so far · PR #7 open") + trace.step("the list keeps an oversized older summary's PR and spend so far until its next update converts it") + terminal.keyEscape() + await waitForText(terminal, PROMPT_READY, { full: false }) + }, + ) +}) + test("the user can correct one earlier planning turn and revoke that correction", async ({ terminal }) => { await runKimchiSession( terminal, From e07383788cdc362755c075fcdfae5eb09fb6e598 Mon Sep 17 00:00:00 2001 From: tautvydasLiekis Date: Fri, 9 Oct 2026 03:24:58 +0300 Subject: [PATCH 05/19] perf: refresh the full cost report at most every five minutes --- .../work-attribution/cost-sync.test.ts | 33 +++++++++++++++++++ src/extensions/work-attribution/cost-sync.ts | 28 +++++++++++++--- 2 files changed, 57 insertions(+), 4 deletions(-) diff --git a/src/extensions/work-attribution/cost-sync.test.ts b/src/extensions/work-attribution/cost-sync.test.ts index 3edd79e68..28ae88625 100644 --- a/src/extensions/work-attribution/cost-sync.test.ts +++ b/src/extensions/work-attribution/cost-sync.test.ts @@ -54,6 +54,8 @@ beforeEach(() => { dir = mkdtempSync(join(tmpdir(), "kimchi-cost-sync-")) currentKey = "test-only-original-key" vi.stubEnv("PI_CODING_AGENT_DIR", dir) + // These cases read the full report after every pass; one case below keeps the five-minute refresh. + vi.stubEnv("KIMCHI_E2E_COST_REPORT_REFRESH_MS", "0") const originalConfig = config.loadConfig() const endpoints = config.resolveEndpoints() vi.spyOn(config, "loadConfig").mockImplementation(() => ({ ...originalConfig, apiKey: currentKey })) @@ -1666,6 +1668,37 @@ describe("automatic exact work cost lookup", () => { await sync() expect(report(workId).pullRequests[0].totalCostUsd).toBeNull() }) + it("refreshes the full report at most every five minutes while /work shows each new price", async () => { + vi.stubEnv("KIMCHI_E2E_COST_REPORT_REFRESH_MS", undefined) + const { workId } = tagged() + const path = join(dir, "work", workId, "costs.json") + let price: string | undefined + fetchMock.mockImplementation(async (input) => + String(input).endsWith("api-keys:verify") + ? Response.json({ organizationId: ORG, userId: PROMPT }) + : Response.json({ items: price ? [{ id: ROW, totalPrice: price }] : [] }), + ) + await sync() + // A process's first pass writes every report. + const pending = readFileSync(path, "utf8") + expect(JSON.parse(pending).pullRequests[0].totalCostUsd).toBeNull() + price = "0.5" + const now = Date.now() + const clock = vi.spyOn(Date, "now") + for (const minutes of [1, 4]) { + clock.mockReturnValue(now + minutes * 60_000) + await sync() + expect(readFileSync(path, "utf8")).toBe(pending) + expect(workCostDetails(dir, workId)).toContain("Cost: $0.500000000 USD — https://github.com/example/repo/pull/1") + } + clock.mockReturnValue(now + 5 * 60_000) + await sync() + expect(report(workId).pullRequests[0].totalCostUsd).toBe("0.500000000") + // A deleted report is rebuilt on the next pass. + unlinkSync(path) + await sync() + expect(report(workId).pullRequests[0].totalCostUsd).toBe("0.500000000") + }) it("keeps missing billing pending and accepts a late explicit zero", async () => { const { workId } = tagged() fetchMock.mockResolvedValueOnce(Response.json({ organizationId: ORG, userId: PROMPT })) diff --git a/src/extensions/work-attribution/cost-sync.ts b/src/extensions/work-attribution/cost-sync.ts index a8bbf2d5b..532f730de 100644 --- a/src/extensions/work-attribution/cost-sync.ts +++ b/src/extensions/work-attribution/cost-sync.ts @@ -6,6 +6,7 @@ import { type VerifyApiKeyResponse, verifyApiKey } from "../../api/organizations import { writeFileAtomic, writeFileDurably } from "../../config/json.js" import { loadConfig } from "../../config.js" import { isWorkId } from "../../shared/work-id.js" +import { readE2eSeam } from "../e2e-seam.js" import { trackPRCostMetric } from "../telemetry/pr-cost.js" import { appendWorkRecord } from "../work-attribution.js" import { BEFORE_PAGE_TIMEOUT_MESSAGE, type BillingRow, billingResponse, lookupRows } from "./billing-api.js" @@ -198,6 +199,14 @@ function group(groups: Map, key: K, value: T): void { else groups.set(key, [value]) } +/** A long work's full report is large; it is refreshed at most this often, and `/work` reads the totals instead. */ +const REPORT_REFRESH_MS = 5 * 60_000 +/** When this process last refreshed each work's full report. Its first pass refreshes every report. */ +const reportRefreshes = new Map() +function reportRefreshMs(): number { + const value = Number(readE2eSeam("KIMCHI_E2E_COST_REPORT_REFRESH_MS") ?? Number.NaN) + return Number.isSafeInteger(value) && value >= 0 ? value : REPORT_REFRESH_MS +} async function writeChanged(path: string, content: string, assertLease: () => void): Promise { try { if (readFileSync(path, "utf8") === content) return @@ -209,7 +218,7 @@ async function publishReports( { displays, report, workIds }: CostState, polls: Record, assertLease: () => void, -): Promise { +): Promise { trackPRCostMetric({ kind: "unpriced", value: report.requests.filter((request) => request.priceStatus !== "priced").length, @@ -270,6 +279,8 @@ async function publishReports( contents.set(included, found) return found } + // A full report left for a later pass keeps this calculation unpublished, so that pass still writes it. + let deferred = false for (const workId of workIds) { assertLease() const directory = join(agentDir, "work", workId) @@ -282,8 +293,17 @@ async function publishReports( `${JSON.stringify(workCostTotals(value), null, 2)}\n`, assertLease, ) - await writeChanged(join(directory, "costs.json"), `${JSON.stringify(value, null, 2)}\n`, assertLease) + // The accuracy CLI and lab read the full report. A missing one is rebuilt at once. + const path = join(directory, "costs.json") + const now = Date.now() + if (now - (reportRefreshes.get(workId) ?? Number.NEGATIVE_INFINITY) < reportRefreshMs() && existsSync(path)) { + deferred = true + continue + } + await writeChanged(path, `${JSON.stringify(value, null, 2)}\n`, assertLease) + reportRefreshes.set(workId, now) } + return !deferred } /** Called by the shared supervisor's lease holder; model inference never waits for this pass. */ @@ -501,11 +521,11 @@ export async function reconcileWorkCosts( latest.workIds.every((workId) => existsSync(join(agentDir, "work", workId, "costs.json"))) ) return - await publishReports(agentDir, latest, polls, () => { + const complete = await publishReports(agentDir, latest, polls, () => { signal.throwIfAborted() assertLease() }) - latest.published = failures + if (complete) latest.published = failures } finally { clearTimeout(timeout) } From 3d94a2e2f62a2135c5d88acf910af12de24fba1a Mon Sep 17 00:00:00 2001 From: tautvydasLiekis Date: Fri, 9 Oct 2026 03:26:28 +0300 Subject: [PATCH 06/19] docs: describe work summary row logs --- docs/work-attribution-accuracy.md | 4 +- docs/work-attribution.md | 68 ++++++++++++++++++++++--------- 2 files changed, 50 insertions(+), 22 deletions(-) diff --git a/docs/work-attribution-accuracy.md b/docs/work-attribution-accuracy.md index 042fb8af1..0606b1347 100644 --- a/docs/work-attribution-accuracy.md +++ b/docs/work-attribution-accuracy.md @@ -8,11 +8,11 @@ pnpm exec tsx src/extensions/work-attribution/accuracy-cli.ts report.json refere The command reads these two files and prints the differences. It makes no network or model calls, reads no credentials and writes nothing. Keep real receipts and labels outside the repository. -By default, Kimchi saves the report at `~/.config/kimchi/harness/work//costs.json`. It contains request rows, PR totals and all six unallocated buckets for that work and every work connected to it. Works are connected when they contribute to the same PR or when a work link, such as a `/work link` correction, moves requests between them. Connections are transitive: if work A shares a PR with work B, and B shares another PR with work C, A's report also includes C's requests and PR totals. Prepare reference entries for every request in that file. Do not add per-work reports together: connected works repeat the same requests. +By default, Kimchi saves the report at `~/.config/kimchi/harness/work//costs.json`. It contains request rows, PR totals and all six unallocated buckets for that work and every work connected to it. Kimchi refreshes it at most every five minutes per work, and at once when it is missing, so a report saved right after a price arrived can lag `/work` by a few minutes. Works are connected when they contribute to the same PR or when a work link, such as a `/work link` correction, moves requests between them. Connections are transitive: if work A shares a PR with work B, and B shares another PR with work C, A's report also includes C's requests and PR totals. Prepare reference entries for every request in that file. Do not add per-work reports together: connected works repeat the same requests. ## Prepare the reference before inspecting matcher output -Use provider or gateway receipts to list requests, their billing accounts and exact billed prices. Label which PR each request belongs to using the task history and Git evidence. Do not generate this reference from `work.json`, `costs.json`, matcher confidence or the allocator under test: that would repeat the same mistakes on both sides. +Use provider or gateway receipts to list requests, their billing accounts and exact billed prices. Label which PR each request belongs to using the task history and Git evidence. Do not generate this reference from the work summary, `costs.json`, matcher confidence or the allocator under test: that would repeat the same mistakes on both sides. The version 1 reference has one entry per request: diff --git a/docs/work-attribution.md b/docs/work-attribution.md index 4d9e47708..3b010e937 100644 --- a/docs/work-attribution.md +++ b/docs/work-attribution.md @@ -2,19 +2,21 @@ Kimchi keeps a local record of the model requests, file edits, plans and commits that belong to the same work. A `workId` connects them, even when planning and implementation happen in different sessions or worktrees of the same repository. -The result is `~/.config/kimchi/harness/work//work.json` in the user's home directory. Kimchi finds GitHub pull requests and GitLab merge requests for recorded commits, then looks up their request costs in the background. `/work` shows one total split into confirmed and inferred spend, or an open PR's spend so far, plus the work's priced spend; `costs.json` keeps the calculation. Incomplete billing stays unknown. These files are not uploaded. +The result is a summary in `~/.config/kimchi/harness/work//` in the user's home directory: `work.json` and its row logs. Kimchi finds GitHub pull requests and GitLab merge requests for recorded commits, then looks up their request costs in the background. `/work` shows one total split into confirmed and inferred spend, or an open PR's spend so far, plus the work's own priced spend; `costs.json` keeps the calculation. Incomplete billing stays unknown. These files are not uploaded. ## Where the files live -**`work.json` is stored per user, outside the project.** All projects use the same `~/.config/kimchi/harness/` directory, with a separate folder for each work ID. `~` means the home directory of the user running Kimchi. +**The summary is stored per user, outside the project.** All projects use the same `~/.config/kimchi/harness/` directory, with a separate folder for each work ID. `~` means the home directory of the user running Kimchi. ```text # Project: editable plan ~/src/example/.kimchi/plans/add-search.md -# User: summary and retained plan versions +# User: summary, cost reports and retained plan versions ~/.config/kimchi/harness/work//work.json +~/.config/kimchi/harness/work//rows/..jsonl ~/.config/kimchi/harness/work//scope.json +~/.config/kimchi/harness/work//cost-totals.json ~/.config/kimchi/harness/work//costs.json ~/.config/kimchi/harness/work//plans/add-search-.md @@ -42,7 +44,7 @@ Deleting a worktree removes its editable plan but leaves the user-level summary flowchart LR A["Session A: plan"] -->|save native plan or write Markdown| P["Saved planning evidence"] P -->|name its path or paste the saved native plan| B["Fresh session B: implement"] - A -->|requests| W["Local work.json"] + A -->|requests| W["Local work summary"] B -->|requests and commits| W ``` @@ -53,11 +55,11 @@ flowchart LR ## Browse works with `/work` -In the terminal, `/work` opens a list of works. The current work comes first, marked `●`, followed by up to 29 works with activity in the last 35 days, newest first. Each row names the work by its saved plan's title, or else by repository and branch, with a short work ID. It also shows the work's priced spend and PR state, for example `$0.0623 · PR #731 open` or `$0.0123 known so far · 2 PRs`. +In the terminal, `/work` opens a list of works. The current work comes first, marked `●`, followed by up to 29 works with activity in the last 35 days, newest first. Each row names the work by its saved plan's title, or else by repository and branch, with a short work ID. It also shows the work's own priced spend and PR state, for example `$0.0623 · PR #731 open` or `$0.0123 known so far · 2 PRs`. Own spend covers the work's requests and requests a correction linked into it, not other works on the same PR. -Amounts in the list are rounded; the details keep exact values. `known so far` means some requests still have no price, including requests made since the last cost check. A work whose requests have no price yet shows `cost unknown`, never `$0`. The header adds up the listed works; a request saved by several connected works counts once. +Amounts in the list are rounded; the details keep exact values. `known so far` means some requests still have no price, including requests made since the last cost check. A work whose requests have no price yet shows `cost unknown`, never `$0`. The header adds up the listed works; connected works share one cost report, and its requests count once. -The selected row shows what `/work` prints, plus its repository, branch, last activity and request count. Only the current work has live PR lookup lines; other works show their saved PR links. Enter prints the selected work's details into the conversation, and Esc closes the list. The list reads only local files, so it never waits for the network. +The selected row shows what `/work` prints, plus its repository, branch, last activity and request count. Only the current work has live PR lookup lines; other works show their saved PR links. Enter prints the selected work's details into the conversation, and Esc closes the list. The list reads only each work's small `work.json` and `cost-totals.json`, so it opens quickly for long works and never waits for the network. A summary saved by an older build that is larger than 8 MiB shows `summary too large` with its saved spend and PRs until the work's next update converts it. ACP, RPC and print mode keep printing the current work's details as text. So do `/work new`, `/work `, `/work matching on|off` and `/work link|unlink`. @@ -117,7 +119,7 @@ Model matching is off by default. `/work matching on` allows saved task text to Kimchi captures the model selected when you submit a message, then uses that model for separate matching calls before the main reply. These calls use its normal authentication and provider connection. They have no tools and do not enter the chat history. No local model, extra model setting or `judge` role is required. With no saved task to compare, the first message needs no matching call. -While matching is enabled, the first user message is kept in `work//intent.json` after its account is verified. Later checks use that text and the latest retained native plan. Candidates must belong to the same Git repository, API endpoint, organization and authenticated user. Rotating a key for the same account keeps that identity. Missing user identity or legacy text without an account prevents automatic semantic continuation; old text is not assigned to whoever is logged in now. Enabled redaction applies before sending. Raw text and credentials are not copied into `work.json`. +While matching is enabled, the first user message is kept in `work//intent.json` after its account is verified. Later checks use that text and the latest retained native plan. Candidates must belong to the same Git repository, API endpoint, organization and authenticated user. Rotating a key for the same account keeps that identity. Missing user identity or legacy text without an account prevents automatic semantic continuation; old text is not assigned to whoever is logged in now. Enabled redaction applies before sending. Raw text and credentials are not copied into the work summary. The model checks the current task first. To find earlier work, it first reads the message alone, then compares each eligible saved task separately. Exactly one must match and every competitor must be clearly different. Each check is a separate paid request with its own ID, saved before dispatch. Matching requests stay with the work active before the decision; adopting another work never moves those earlier requests. @@ -163,11 +165,11 @@ If planning and implementation already ended up in separate works, open the impl /work link ``` -The planning work's `work.json` lists `requests[].segment.id`. The command selects requests already recorded for that input, including its retries and side calls. Other inputs in the same conversation stay separate. Both works and the selected requests must have matching saved account and repository scope. Older records without that evidence cannot be repaired this way. +The planning work's request rows list `segment.id`. The command selects requests already recorded for that input, including its retries and side calls. Other inputs in the same conversation stay separate. Both works and the selected requests must have matching saved account and repository scope. Older records without that evidence cannot be repaired this way. If an uncertain input already belongs to the right work, use that current work ID as the source. This records your confirmation without changing its original request or work IDs. `/work unlink ` also revokes this confirmation. -The command saves a `work_link` revision in the implementing work. It names the selected requests and their intended work; original request and work IDs stay unchanged. The target work's `work.json` keeps these revisions in `workLinks` for cost calculations. Model decisions do not retroactively rewrite earlier requests. +The command saves a `work_link` revision in the implementing work. It names the selected requests and their intended work; original request and work IDs stay unchanged. The target work's summary keeps these revisions in its `workLinks` rows for cost calculations. Model decisions do not retroactively rewrite earlier requests. To withdraw it, run `/work unlink ` in the implementing work. A revoked or conflicting correction leaves the affected assignment unknown until corrected again. Linking those requests from another work replaces the earlier link with a newer revision, including a revoked automatic confirmation. Only the work with the newest revision can then revoke it. @@ -178,7 +180,7 @@ flowchart LR R["Model request with requestId"] --> A["Assistant returns tool calls"] A --> T["Native write or edit"] T --> F["fileTransitions: requestId,
toolCallId, repository and path"] - F --> W["Same work.json"] + F --> W["Same work summary"] ``` One request can produce edits in several repositories. Each edit keeps the ID of the request that produced its tool call, even if another request runs before the file write finishes. The request's `cwd` remains the session's working directory; each edit records the repository and worktree it actually changed. @@ -303,9 +305,31 @@ Kimchi tries credentials in this order: an environment token for the selected ho Before sending a covered model request, Kimchi saves and flushes its request ID, work ID, session ID, input segment and account/repository scope. Missing scope is saved as `null`. It then sends `X-Request-Id`. This also works with telemetry disabled. -Records with the same work ID go into the same `work.json`. Requests are deduplicated by request ID; commit records keep the contributing sessions. `fileTransitions` keeps native edits, `fileObservations` keeps candidate Bash/MCP changes, `continuations` explains why an input continued the current or another saved work, and `workLinks` keeps correction revisions. IDs identify records; array positions have no meaning. +Records with the same work ID go into the same summary. Requests are deduplicated by request ID; commit records keep the contributing sessions. `fileTransitions` keeps native edits, `fileObservations` keeps candidate Bash/MCP changes, `continuations` explains why an input continued the current or another saved work, and `workLinks` keeps correction revisions. IDs identify records; positions have no meaning. -A request record describes an attempt. It does not prove a successful response or a charge. Covered HTTP replies add their status and safe response IDs. Billing lookups later add exact prices when available. `work.json` contains paths, request metadata, PR links and billing rows, with no prompts or file contents. Retained native plans contain the plan text and stay local. PR lookup sends repository and commit identifiers to the repository's GitHub or GitLab API. +`work.json` is a small manifest. It holds the sessions, what `/work` lists (latest activity, repository, branch, retained plan and every commit's PR links) and one entry per collection. Each collection is an append-only log, `rows/..jsonl`, with one JSON row per line. A later line with the same key replaces an earlier one and keeps its position. Keys are the request, transition and observation IDs; a commit's key is its session, SHA, repository and worktree. Only the first `bytes` of each log, as `work.json` records them, are committed: + +```json +{ + "version": 2, + "workId": "6f102360-39a9-4d6e-8ac7-984c97c0e313", + "sessions": ["01a10dc0-7ea4-74d8-9b87-9a5026c7b05c"], + "logs": { + "requests": { "generation": 3, "bytes": 33441207, "rows": 15012 }, + "commits": { "generation": 1, "bytes": 41877, "rows": 38 } + } +} +``` + +To read one collection by hand, take its committed bytes and keep each key's last row: + +```sh +cd ~/.config/kimchi/harness/work/ +g=$(jq .logs.requests.generation work.json); n=$(jq .logs.requests.bytes work.json) +head -c "$n" "rows/requests.$g.jsonl" | jq -sc 'reduce .[] as $r ({}; .[$r.requestId] = $r) | [.[]]' +``` + +A request record describes an attempt. It does not prove a successful response or a charge. Covered HTTP replies add their status and safe response IDs. Billing lookups later add exact prices when available. The summary contains paths, request metadata, PR links and billing rows, with no prompts or file contents. Retained native plans contain the plan text and stay local. PR lookup sends repository and commit identifiers to the repository's GitHub or GitLab API.
Local files and recovery @@ -314,8 +338,10 @@ All paths below are inside the agent directory. | File | What it is for | | --- | --- | -| `work//work.json` | Readable summary of sessions, request attempts, plans, file edits, continuation evidence, commits and PR links. | -| `work//costs.json` | Derived PR totals and each request's allocation, price and billing lookup status. Rebuilt from source records when missing, and rewritten only after a journal or refresh failure changes. | +| `work//work.json` | Manifest: sessions, what `/work` lists and the committed length of each row log. Replacing it commits new rows. | +| `work//rows/..jsonl` | Readable summary rows: request attempts, plans, file edits, continuation evidence, commits and PR links. A later line with the same key replaces an earlier one. | +| `work//cost-totals.json` | What `/work` prints: PR totals, and the work's own requests and priced spend. Rewritten when it changes. | +| `work//costs.json` | Derived PR totals and each request's allocation, price and billing lookup status, for all connected works. Refreshed at most every five minutes after a journal or refresh failure changes, and rebuilt from source records when missing. | | `work-attribution/billing-polls.json` | Latest polling times and the last refresh that failed before any billing page, only for requests whose billing window is still open. Replaced after checks, so unchanged results and failed refreshes do not grow the journal. | | `work//scope.json` | Original API endpoint, organization, user and Git common-directory identity. Credentials are not saved here. | | `work//intent.json` | Saved task text for opt-in model matching, bound to its original account and repository. | @@ -326,10 +352,12 @@ All paths below are inside the agent directory. | `work-attribution/objects//` | Kimchi's own Git object store of native edit text kept for `file-hunks` matching. Nothing is deleted by age; it grows by one object per distinct edited content, without copies of the committed version an edit starts from. | | `work-attribution/pr-checks.json` | When each recorded repository and commit was last checked for a PR or MR, including checks whose unchanged result was not appended. A restart therefore keeps the lookup schedule. Kimchi replaces the file at most once per pass and only after a change. It drops entries for commits it no longer records and for checks more than a day old. Deleting it costs at most one extra check per commit. | -- Each plan record has `path` for the editable file, `snapshotPath` for its retained version and `contentHash` to detect changes. Plans produced by an attributed tool also name its `requestId`. If saving the retained copy fails, Kimchi shows a warning and leaves the local plan usable. Naming or pasting that plan later then does not continue its work or confirm its producing input; `/work ` still selects it. -- Kimchi flushes source records before updating `work.json`. Writers merge under a lock and replace the summary atomically. Shutdown waits for queued summary writes. -- Recovery saves the time of its last successful scan plus each summary's size and modification time. It reads source logs changed since then and skips writing unchanged results. -- A missing or invalid known summary triggers a full replay of the source logs, including edit journals. A failed log read or summary write leaves the checkpoint unchanged. Older summaries remain readable when the new collections are absent. +- Each plan record has `path` for the editable file, `snapshotPath` for its retained version and `contentHash` to detect later changes to either copy. Plans produced by an attributed tool also name the originating `requestId`, even if the session changed work before that tool ran. If saving the retained copy fails, Kimchi shows a warning and leaves the local plan usable. Naming or pasting that plan later then does not continue its work or confirm its producing input; `/work ` still selects it. +- Kimchi flushes source records before updating the summary. Writers merge under a per-work lock, append only changed rows, sync them, then replace `work.json`, which commits them. Readers stop at each log's committed length, so an interrupted append is ignored and the next writer removes it. Shutdown waits for queued summary writes. +- When a log's superseded lines outweigh half its current rows and exceed 1 MiB, the writer copies each key's latest row into the next generation, commits it and deletes the old file. A reader that loaded the old manifest rereads it once. +- Recovery saves the time of its last successful scan plus each manifest's size and modification time. It reads source logs changed since then and skips writing unchanged results. +- A missing or invalid summary, a log shorter than its committed length, or a damaged row triggers a full replay of the work's source logs, including edit journals, into a new generation. A failed log read or summary write leaves the checkpoint unchanged. +- Summaries saved before row logs keep everything in `work.json` (`"version": 1`). They stay readable and convert on the work's next update; idle works are not rewritten. Releases up to v1.7.1 do not read the new format: they rebuild a version 1 summary from the source logs, without record types they do not know, and the next newer release converts it again and removes the stale logs.
@@ -407,7 +435,7 @@ An exact price does not confirm a model's task match. Inferred requests remain l A refresh that fails before any billing page arrives adds no evidence. This covers being offline, DNS or TLS errors, a failed API key check, an HTTP error such as 429 or 5xx on the first page, and Kimchi's pass deadline. The last known price stays usable and nothing is added to the journal. `billing-polls.json` keeps the failure, the request's `billingLookup` in `costs.json` shows it with its real timestamp, and `/work` says that the last refresh failed. Errors after the first page, including partial pagination, keep the full total unknown. Failure records saved by earlier versions also stay unknown until a complete lookup succeeds, because they do not record whether a page had arrived; their pass-deadline records marked as happening before any page are the exception. -Billing details appear on the matching `work.json` request. This excerpt uses example IDs; comments explain the fields and are not part of the stored JSON: +Billing details appear on the matching request row in `rows/requests..jsonl`. This excerpt uses example IDs; comments explain the fields and are not part of the stored JSON: ```jsonc { From fb523e9c4169898c1df5b0e0c2ce05e14bc81c3f Mon Sep 17 00:00:00 2001 From: tautvydasLiekis Date: Fri, 9 Oct 2026 10:05:07 +0300 Subject: [PATCH 07/19] test: wait for the cost totals that /work reads --- tests/e2e/acp/work-costs.test.ts | 11 ++++++----- tests/e2e/tui/work-costs.test.ts | 20 +++++++++++++------- 2 files changed, 19 insertions(+), 12 deletions(-) diff --git a/tests/e2e/acp/work-costs.test.ts b/tests/e2e/acp/work-costs.test.ts index 66571f051..486849b4c 100644 --- a/tests/e2e/acp/work-costs.test.ts +++ b/tests/e2e/acp/work-costs.test.ts @@ -120,10 +120,11 @@ it("shows an open MR's pending cost, then its confirmed and inferred total once }) mkdirSync(join(cwd, ".kimchi", "plans"), { recursive: true }) writeFileSync(join(cwd, ".kimchi", "plans", "costs.md"), `\n# Add a greeting\n`) - const costsPath = join(agentDir, "work", workId, "costs.json") - const costs = () => { + // `/work` reads these totals; the full `costs.json` is refreshed at most every five minutes. + const totalsPath = join(agentDir, "work", workId, "cost-totals.json") + const totals = () => { try { - return JSON.parse(readFileSync(costsPath, "utf8")) + return JSON.parse(readFileSync(totalsPath, "utf8")) } catch { return undefined } @@ -131,7 +132,7 @@ it("shows an open MR's pending cost, then its confirmed and inferred total once const sessionId = await newSession(fixture, cwd) expect((await prompt(fixture, sessionId, "/work .kimchi/plans/costs.md")).stopReason).toBe("end_turn") - await waitFor(costs, (value) => value?.requests?.length === 1, 45_000) + await waitFor(totals, (value) => value?.requests?.total === 1, 45_000) const pending = await workDetails(fixture, sessionId) expect(pending).toContain("Cost so far: unknown; $0.000000000 USD priced (open)") expect(pending).toContain("Prices: 0/1 requests priced, $0.000000000 USD known so far.") @@ -139,7 +140,7 @@ it("shows an open MR's pending cost, then its confirmed and inferred total once billed = true setGitLab("merged") await waitFor( - costs, + totals, (value) => value?.pullRequests?.some((row: { totalCostUsd?: unknown }) => row.totalCostUsd === "0.000166000"), 75_000, ) diff --git a/tests/e2e/tui/work-costs.test.ts b/tests/e2e/tui/work-costs.test.ts index 9f92886e0..37b033c9b 100644 --- a/tests/e2e/tui/work-costs.test.ts +++ b/tests/e2e/tui/work-costs.test.ts @@ -11,8 +11,14 @@ import { BILLING_ACCOUNT, seedPendingWorkCost } from "./support/work-costs.js" test.use(TUI_TEST_CONFIG) -/** Background passes run every 30 seconds; wait for the one that publishes the expected costs. */ -async function waitForCosts(path: string, ready: (costs: Record) => boolean): Promise { +/** The cost totals `/work` reads; the full `costs.json` is refreshed at most every five minutes. */ +interface CostTotals { + requests?: { total?: unknown } + pullRequests?: { totalCostUsd?: unknown }[] +} + +/** Background passes run every 30 seconds; wait for the one that publishes the expected totals. */ +async function waitForTotals(path: string, ready: (totals: CostTotals) => boolean): Promise { const deadline = Date.now() + 45_000 while (Date.now() < deadline) { try { @@ -33,7 +39,7 @@ test("an open PR's pending cost becomes a confirmed and inferred total once it i const extraArgs: string[] = [] let billed = false let statePath = "" - let costsPath = "" + let totalsPath = "" let headSha = "" const setGitHub = (mode: string) => { writeFileSync(`${statePath}.next`, JSON.stringify({ mode, headSha })) @@ -91,7 +97,7 @@ test("an open PR's pending cost becomes a confirmed and inferred total once it i checkedAt: new Date().toISOString(), }, }) - costsPath = join(agentDir, "work", workId, "costs.json") + totalsPath = join(agentDir, "work", workId, "cost-totals.json") const fixtureDir = join(home, "pr-api-fixture") mkdirSync(fixtureDir) statePath = join(fixtureDir, "state.json") @@ -129,7 +135,7 @@ test("an open PR's pending cost becomes a confirmed and inferred total once it i }, async (_fixture, trace) => { await waitForText(terminal, "PR #731 open", { full: false, timeoutMs: 10_000 }) - await waitForCosts(costsPath, (costs) => costs.requests?.length === 1) + await waitForTotals(totalsPath, (totals) => totals.requests?.total === 1) terminal.submit("/work") await waitForText(terminal, "Cost so far: unknown; $0.000000000 USD priced (open)") await waitForText(terminal, "Prices: 0/1 requests priced, $0.000000000 USD known so far.") @@ -139,8 +145,8 @@ test("an open PR's pending cost becomes a confirmed and inferred total once it i trace.step("an open PR's spend stays unknown while its request has no bill") billed = true setGitHub("merged") - await waitForCosts(costsPath, (costs) => - costs.pullRequests?.some((row) => (row as { totalCostUsd?: unknown }).totalCostUsd === "0.000166000"), + await waitForTotals(totalsPath, (totals) => + Boolean(totals.pullRequests?.some((row) => row.totalCostUsd === "0.000166000")), ) terminal.submit("/work") await waitForText(terminal, `Cost: $0.000166000 USD — ${url}`) From c2fb4d852a89877a40bc5670ae226829950f30cd Mon Sep 17 00:00:00 2001 From: tautvydasLiekis Date: Fri, 9 Oct 2026 11:14:44 +0300 Subject: [PATCH 08/19] style: separate declarations in row logs and cost details --- .../work-attribution/cost-details.ts | 4 +++ src/extensions/work-attribution/row-log.ts | 28 +++++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/src/extensions/work-attribution/cost-details.ts b/src/extensions/work-attribution/cost-details.ts index 461733a13..2a1401a7d 100644 --- a/src/extensions/work-attribution/cost-details.ts +++ b/src/extensions/work-attribution/cost-details.ts @@ -82,6 +82,7 @@ function untaggedTotals(rows: Record[]): { untagged?: Record[]): { failedRefresh?: { count: number; reason?: string } } { const failed = rows.flatMap((row) => object(row.billingLookup) && row.billingLookup.status === "unavailable" ? [row.billingLookup] : [], @@ -94,6 +95,7 @@ function failedRefreshTotals(rows: Record[]): { failedRefresh?: failedRefresh: { count: failed.length, ...(typeof latest.reason === "string" ? { reason: latest.reason } : {}) }, } } + export function workCostTotals(report: unknown): WorkCostTotals | undefined { if ( !object(report) || @@ -222,6 +224,7 @@ export function costDetailLines(value: unknown, path: string): string[] { `Confirmed: $${row.sure} USD; inferred: $${row.likely} USD${row.totalCostUsd === null ? " known so far" : ""}.`, ) } + // Work without a PR has no cost line above, so its own priced spend is shown here. const { priced, total, knownCostUsd, unresolved, inferred, shared } = value.requests if (typeof priced === "number" && typeof total === "number") @@ -233,6 +236,7 @@ export function costDetailLines(value: unknown, path: string): string[] { const untagged = Object.entries(value.requests.untagged).filter( (entry): entry is [string, number] => typeof entry[1] === "number", ) + const count = untagged.reduce((sum, [, total]) => sum + total, 0) const reasons = untagged .sort(([left], [right]) => left.localeCompare(right)) diff --git a/src/extensions/work-attribution/row-log.ts b/src/extensions/work-attribution/row-log.ts index 49f32db10..efae1fb73 100644 --- a/src/extensions/work-attribution/row-log.ts +++ b/src/extensions/work-attribution/row-log.ts @@ -43,6 +43,7 @@ export const COLLECTIONS = [ ] as const export type Collection = (typeof COLLECTIONS)[number] const COLLECTION_NAMES = new Set(COLLECTIONS) + function isCollection(name: string): name is Collection { return COLLECTION_NAMES.has(name) } @@ -116,19 +117,24 @@ export class RowLogDamage extends Error {} function object(value: unknown): value is Record { return typeof value === "object" && value !== null && !Array.isArray(value) } + function code(error: unknown): unknown { return object(error) ? error.code : undefined } + function count(value: unknown, minimum = 0): value is number { return typeof value === "number" && Number.isSafeInteger(value) && value >= minimum } + function isRow(value: unknown): value is Row { return object(value) && typeof value.sessionId === "string" } + /** Version 1 readers were lenient: any object in a collection is a row. The writer validates them. */ function savedRow(value: unknown): value is Row { return object(value) } + function text(value: unknown): string | undefined { return typeof value === "string" ? value : undefined } @@ -240,6 +246,7 @@ function readCommitted( } catch { return undefined } + const view = versionOneView(value, workId) if (view) return fromVersionOne(view) if (!isWorkHead(value, workId)) return undefined @@ -282,6 +289,7 @@ export function readWorkRows(agentDir: string, workId: string, collection: Colle * The manifest, or the same values computed from a version 1 summary small enough to read. Opens no row log. * "too-large" is a version 1 summary over `maxBytes`; its next update migrates it. */ + export async function readWorkHead( agentDir: string, workId: string, @@ -324,10 +332,12 @@ function timestamp(value: unknown): number | undefined { const day = value.slice(0, 10) return new Date(`${day}T00:00:00Z`).toISOString().startsWith(day) ? parsed : undefined } + /** Whether a row stamped `at` replaces a latest value stamped `current`. Rows without a valid time sort first. */ function replaces(at: unknown, current: { at?: string } | undefined): boolean { return !current || (timestamp(at) ?? 0) >= (timestamp(current.at) ?? 0) } + /** Folds one changed or saved row into what `/work` lists. */ export function observeLatest(latest: WorkLatest, collection: Collection, row: Row): void { const at = collection === "requests" ? row.startedAt : row.recordedAt @@ -357,12 +367,14 @@ export function observeLatest(latest: WorkLatest, collection: Collection, row: R latest.plan = { recordedAt, snapshotPath: row.snapshotPath } } } + /** PR links of every commit row, merged; empty or failed lookups never remove a link. */ export function commitPullRequests(commits: Iterable): Record[] { return mergePullRequestLinks( ...[...commits].map((row) => (Array.isArray(row.pullRequests) ? row.pullRequests.filter(object) : [])), ) } + /** `latest` and `pullRequests` recomputed from every row, in log order. */ export function summaryLatest( rows: (collection: Collection) => Iterable, @@ -376,6 +388,7 @@ interface Line { offset: number length: number } + interface LogIndex { generation: number /** File identity: a replaced file is scanned again. */ @@ -389,12 +402,15 @@ interface LogIndex { /** Newest line per key, in first-seen order. */ lines: Map } + /** Row positions of logs this process has read, most recently used last. */ const indexes = new Map() let indexedRows = 0 + function cached(folder: string, collection: Collection): LogIndex | undefined { return indexes.get(`${folder}\0${collection}`)?.index } + function remember(folder: string, collection: Collection, index: LogIndex): void { const key = `${folder}\0${collection}` const previous = indexes.get(key) @@ -410,6 +426,7 @@ function remember(folder: string, collection: Collection, index: LogIndex): void indexedRows -= entry.rows } } + /** Drops this work's positions; its next update scans the logs again. */ function forget(folder: string): void { for (const collection of COLLECTIONS) { @@ -503,6 +520,7 @@ async function writeLog( chunk = [] chunkBytes = 0 } + let rows = 0 for (const [key, line] of lines) { const length = Buffer.byteLength(line) @@ -639,6 +657,7 @@ async function openLogs(folder: string, head: WorkHead, check: RowCheck): Promis await handle.close() } } + const files = new Map() const reader = (collection: Collection, index: LogIndex) => (line: Line) => { let fd = files.get(collection) @@ -646,6 +665,7 @@ async function openLogs(folder: string, head: WorkHead, check: RowCheck): Promis fd = openSync(logPath(folder, collection, index.generation), "r") files.set(collection, fd) } + const buffer = Buffer.allocUnsafe(line.length) if (readSync(fd, buffer, 0, line.length, line.offset) !== line.length) throw new RowLogDamage(`A ${collection} row is missing`) @@ -678,6 +698,7 @@ async function openWork( } catch (error) { if (!(error instanceof SyntaxError) && code(error) !== "ENOENT") throw error } + let damaged: WorkHead | undefined if (isWorkHead(value, workId)) { try { @@ -688,6 +709,7 @@ async function openWork( damaged = value } } + // A version 1 summary migrates with every row; anything else is rebuilt from the journals. const saved = damaged ? undefined : summary(value) return { @@ -706,6 +728,7 @@ async function openWork( function* serialized(touched: RowStore["touched"]): Generator<[string, string]> { for (const [key, { row }] of touched) yield [key, JSON.stringify(row)] } + /** Each touched row's new line, when it differs from the saved one. */ async function changedLines(store: RowStore): Promise<[string, string, Row][]> { const changed: [string, string, Row][] = [] @@ -781,12 +804,14 @@ async function commitWork( } values = summaryLatest((collection) => [...work.rows[collection].touched.values()].map(({ row }) => row)) } + const head: WorkHead = { version: 2, workId, updatedAt: new Date().toISOString(), sessions, logs, ...values } await writeFileDurably(join(folder, WORK_FILE), `${JSON.stringify(head, null, 2)}\n`, assertLease) for (const collection of COLLECTIONS) { const index = written[collection] if (index) remember(folder, collection, index) } + const compacted = await compactLogs(folder, head, { ...work.indexes, ...written }, assertLease) if (generations || compacted || !cleaned.has(folder)) { await removeStrays(folder, compacted ?? head) @@ -811,6 +836,7 @@ function liveRows(folder: string, collection: Collection, index: LogIndex): Row[ * first-seen order to the next generation, committed with `latest` recomputed from every row, then the old * generation is deleted. A reader holding the old manifest sees it missing once and rereads the manifest. */ + async function compactLogs( folder: string, head: WorkHead, @@ -854,6 +880,7 @@ async function compactLogs( } if (collection === "commits") pullRequests = commitPullRequests(commits) } + const compacted: WorkHead = { ...head, logs, latest, pullRequests } await writeFileDurably(join(folder, WORK_FILE), `${JSON.stringify(compacted, null, 2)}\n`, assertLease) for (const [collection, index, old] of replaced) { @@ -869,6 +896,7 @@ async function compactLogs( * `apply` learns that it must merge the work's whole history. Returns the committed manifest, or undefined * when nothing changed. */ + export async function updateWorkRows( folder: string, workId: string, From e6ccd821abe0db6a8fea0335d7b87b0e8948c4c2 Mon Sep 17 00:00:00 2001 From: tautvydasLiekis Date: Fri, 9 Oct 2026 13:08:06 +0300 Subject: [PATCH 09/19] perf: keep positions of row logs an update leaves unchanged --- .../work-attribution/row-log.test.ts | 63 +++++++++++++++++++ src/extensions/work-attribution/row-log.ts | 2 + 2 files changed, 65 insertions(+) diff --git a/src/extensions/work-attribution/row-log.test.ts b/src/extensions/work-attribution/row-log.test.ts index e4a5e5e6f..64b5ee9c6 100644 --- a/src/extensions/work-attribution/row-log.test.ts +++ b/src/extensions/work-attribution/row-log.test.ts @@ -799,4 +799,67 @@ describe("a large work", () => { expect(opened).not.toHaveBeenCalled() expect(vi.mocked(fs.readSync)).not.toHaveBeenCalled() }, 120_000) + + it("scans a log that a restarted process leaves unchanged once, not on every update", async () => { + const workId = randomUUID() + await apply( + Array.from({ length: 2000 }, (_, index) => ({ + version: 1, + type: "file_transition", + workId, + sessionId: "edits", + transitionId: `transition-${index}`, + toolCallId: "call", + repository: "/repo/.git", + worktree: "/tree", + path: `src/file-${index}.ts`, + recordedAt: iso(BASE + index * 1000), + })), + ) + const edits = manifest(workId).logs.fileTransitions + if (!edits) throw new Error("edits were not saved") + // A new module instance has no cached positions, like Kimchi after a restart. + vi.resetModules() + const restarted = await import("./summary.js") + const syncFs = await import("node:fs") + const promises = await import("node:fs/promises") + const request = (index: number): WorkRecord => ({ + version: 1, + type: "request", + workId, + sessionId: "requests", + requestId: `request-${index}`, + startedAt: iso(BASE + 10 ** 7 + index), + recordedAt: iso(BASE + 10 ** 7 + index), + }) + // The first update reads every committed log once. + restarted.updateWorkSummary(request(0)) + await restarted.flushWorkSummaries() + + let read = 0 + const readSync = syncFs.readSync + vi.spyOn(syncFs, "readSync").mockImplementation((...args) => { + const bytes = readSync(...args) + read += bytes + return bytes + }) + const open = promises.open + vi.spyOn(promises, "open").mockImplementation(async (...args) => { + const handle = await open(...args) + const original = handle.read.bind(handle) + vi.spyOn(handle, "read").mockImplementation(async (...readArgs) => { + const result = await original(...readArgs) + read += result.bytesRead + return result + }) + return handle + }) + for (let index = 1; index <= 10; index++) { + restarted.updateWorkSummary(request(index)) + await restarted.flushWorkSummaries() + } + + expect(read).toBeLessThan(edits.bytes) + expect(manifest(workId).logs.requests?.rows).toBe(11) + }) }) diff --git a/src/extensions/work-attribution/row-log.ts b/src/extensions/work-attribution/row-log.ts index efae1fb73..b287625d4 100644 --- a/src/extensions/work-attribution/row-log.ts +++ b/src/extensions/work-attribution/row-log.ts @@ -652,6 +652,8 @@ async function openLogs(folder: string, head: WorkHead, check: RowCheck): Promis if (index.bytes < log.bytes) await scanLog(handle, collection, index, log.bytes, check) index.mtimeMs = (await handle.stat()).mtimeMs if (index.lines.size !== log.rows) throw new RowLogDamage(`The ${collection} log has the wrong row count`) + // Kept for later updates too, so a log this update leaves unchanged is not scanned again. + remember(folder, collection, index) indexes[collection] = index } finally { await handle.close() From 6fe5d0f20001b119607d95225b6e0fd6b5c841f0 Mon Sep 17 00:00:00 2001 From: tautvydasLiekis Date: Fri, 9 Oct 2026 13:08:06 +0300 Subject: [PATCH 10/19] fix: write deleted cost totals again on an idle cost pass --- src/extensions/work-attribution/cost-sync.test.ts | 7 +++++++ src/extensions/work-attribution/cost-sync.ts | 6 ++++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/src/extensions/work-attribution/cost-sync.test.ts b/src/extensions/work-attribution/cost-sync.test.ts index 28ae88625..e8989c562 100644 --- a/src/extensions/work-attribution/cost-sync.test.ts +++ b/src/extensions/work-attribution/cost-sync.test.ts @@ -2128,6 +2128,13 @@ describe("idle cost passes", () => { expect(report(workId)).toEqual(saved) expect(calculate).not.toHaveBeenCalled() expect(reports(write.mock.calls)).toHaveLength(1) + // So are the deleted totals that /work reads. + const totals = join(dir, "work", workId, "cost-totals.json") + const shown = workCostDetails(dir, workId) + unlinkSync(totals) + await sync() + expect(workCostDetails(dir, workId)).toEqual(shown) + expect(calculate).not.toHaveBeenCalled() // Any append changes the journals' fingerprint. appendWorkRecord(ctx, { type: "request_response", requestId, billingSource: source, response: { status: 200 } }) await sync() diff --git a/src/extensions/work-attribution/cost-sync.ts b/src/extensions/work-attribution/cost-sync.ts index 532f730de..b6d7e7530 100644 --- a/src/extensions/work-attribution/cost-sync.ts +++ b/src/extensions/work-attribution/cost-sync.ts @@ -515,10 +515,12 @@ export async function reconcileWorkCosts( const currentPolls = JSON.stringify(polls) if (currentPolls !== previousPolls) writeFileAtomic(pollingPath, `${currentPolls}\n`) const failures = refreshFailures(latest, polls) - // Unchanged inputs give unchanged reports; only a deleted report needs writing again. + // Unchanged inputs give unchanged reports; only a deleted report or totals file needs writing again. if ( latest.published === failures && - latest.workIds.every((workId) => existsSync(join(agentDir, "work", workId, "costs.json"))) + latest.workIds.every((workId) => + ["costs.json", "cost-totals.json"].every((name) => existsSync(join(agentDir, "work", workId, name))), + ) ) return const complete = await publishReports(agentDir, latest, polls, () => { From f9c908745d29b2ccb3a54b3d2d90359a9461d6eb Mon Sep 17 00:00:00 2001 From: tautvydasLiekis Date: Fri, 9 Oct 2026 13:09:47 +0300 Subject: [PATCH 11/19] test: compare every /work cost line of connected works with the previous output --- src/extensions/work-attribution/cost-sync.test.ts | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/extensions/work-attribution/cost-sync.test.ts b/src/extensions/work-attribution/cost-sync.test.ts index e8989c562..c119b05e6 100644 --- a/src/extensions/work-attribution/cost-sync.test.ts +++ b/src/extensions/work-attribution/cost-sync.test.ts @@ -162,16 +162,13 @@ function previousCostDetailLines(value: unknown, path: string): string[] { afterEach(async () => { await flushWorkSummaries() // Every report a test saved prints the same `/work` lines from its bounded totals as from the whole file. - // Connected works differ only in the Prices line, which now counts the work's own requests. for (const workId of existsSync(join(dir, "work")) ? readdirSync(join(dir, "work")).filter(isWorkId) : []) { const path = join(dir, "work", workId, "costs.json") if (!existsSync(path)) continue const saved = JSON.parse(readFileSync(path, "utf8")) const totals = workCostTotals(saved) if (!totals) continue - const own = (lines: string[]) => - totals.group.length > 1 ? lines.filter((line) => !line.startsWith("Prices:")) : lines - expect(own(costDetailLines(totals, path))).toEqual(own(previousCostDetailLines(saved, path))) + expect(costDetailLines(totals, path)).toEqual(previousCostDetailLines(saved, path)) } vi.restoreAllMocks() vi.unstubAllEnvs() From 7b41f3e060f7acb6ec6e0601ae9adc3ddb77e712 Mon Sep 17 00:00:00 2001 From: tautvydasLiekis Date: Fri, 9 Oct 2026 13:16:30 +0300 Subject: [PATCH 12/19] docs: describe the /work header and every row log key --- docs/work-attribution.md | 6 +++--- .../work-attribution/browser.test.ts | 18 ++++++++++++++++++ 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/docs/work-attribution.md b/docs/work-attribution.md index 3b010e937..16b1612d4 100644 --- a/docs/work-attribution.md +++ b/docs/work-attribution.md @@ -57,7 +57,7 @@ flowchart LR In the terminal, `/work` opens a list of works. The current work comes first, marked `●`, followed by up to 29 works with activity in the last 35 days, newest first. Each row names the work by its saved plan's title, or else by repository and branch, with a short work ID. It also shows the work's own priced spend and PR state, for example `$0.0623 · PR #731 open` or `$0.0123 known so far · 2 PRs`. Own spend covers the work's requests and requests a correction linked into it, not other works on the same PR. -Amounts in the list are rounded; the details keep exact values. `known so far` means some requests still have no price, including requests made since the last cost check. A work whose requests have no price yet shows `cost unknown`, never `$0`. The header adds up the listed works; connected works share one cost report, and its requests count once. +Amounts in the list are rounded; the details keep exact values. `known so far` means some requests still have no price, including requests made since the last cost check. A work whose requests have no price yet shows `cost unknown`, never `$0`. The header adds up the cost reports of the listed works. Connected works share one report, so its requests count once, including those of connected works that are not listed. The selected row shows what `/work` prints, plus its repository, branch, last activity and request count. Only the current work has live PR lookup lines; other works show their saved PR links. Enter prints the selected work's details into the conversation, and Esc closes the list. The list reads only each work's small `work.json` and `cost-totals.json`, so it opens quickly for long works and never waits for the network. A summary saved by an older build that is larger than 8 MiB shows `summary too large` with its saved spend and PRs until the work's next update converts it. @@ -307,7 +307,7 @@ Before sending a covered model request, Kimchi saves and flushes its request ID, Records with the same work ID go into the same summary. Requests are deduplicated by request ID; commit records keep the contributing sessions. `fileTransitions` keeps native edits, `fileObservations` keeps candidate Bash/MCP changes, `continuations` explains why an input continued the current or another saved work, and `workLinks` keeps correction revisions. IDs identify records; positions have no meaning. -`work.json` is a small manifest. It holds the sessions, what `/work` lists (latest activity, repository, branch, retained plan and every commit's PR links) and one entry per collection. Each collection is an append-only log, `rows/..jsonl`, with one JSON row per line. A later line with the same key replaces an earlier one and keeps its position. Keys are the request, transition and observation IDs; a commit's key is its session, SHA, repository and worktree. Only the first `bytes` of each log, as `work.json` records them, are committed: +`work.json` is a small manifest. It holds the sessions, what `/work` lists (latest activity, repository, branch, retained plan and every commit's PR links) and one entry per collection. Each collection is an append-only log, `rows/..jsonl`, with one JSON row per line. A later line with the same key replaces an earlier one and keeps its position. Keys are the request, transition and observation IDs; a commit's key is its session, SHA, repository and worktree; a plan's is its session, path and retained copy; a continuation's is its session, source and evidence; and a work link's is the whole revision. Only the first `bytes` of each log, as `work.json` records them, are committed: ```json { @@ -352,7 +352,7 @@ All paths below are inside the agent directory. | `work-attribution/objects//` | Kimchi's own Git object store of native edit text kept for `file-hunks` matching. Nothing is deleted by age; it grows by one object per distinct edited content, without copies of the committed version an edit starts from. | | `work-attribution/pr-checks.json` | When each recorded repository and commit was last checked for a PR or MR, including checks whose unchanged result was not appended. A restart therefore keeps the lookup schedule. Kimchi replaces the file at most once per pass and only after a change. It drops entries for commits it no longer records and for checks more than a day old. Deleting it costs at most one extra check per commit. | -- Each plan record has `path` for the editable file, `snapshotPath` for its retained version and `contentHash` to detect later changes to either copy. Plans produced by an attributed tool also name the originating `requestId`, even if the session changed work before that tool ran. If saving the retained copy fails, Kimchi shows a warning and leaves the local plan usable. Naming or pasting that plan later then does not continue its work or confirm its producing input; `/work ` still selects it. +- Each plan record has `path` for the editable file, `snapshotPath` for its retained version and `contentHash` to detect changes. Plans produced by an attributed tool also name its `requestId`. If saving the retained copy fails, Kimchi shows a warning and leaves the local plan usable. Naming or pasting that plan later then does not continue its work or confirm its producing input; `/work ` still selects it. - Kimchi flushes source records before updating the summary. Writers merge under a per-work lock, append only changed rows, sync them, then replace `work.json`, which commits them. Readers stop at each log's committed length, so an interrupted append is ignored and the next writer removes it. Shutdown waits for queued summary writes. - When a log's superseded lines outweigh half its current rows and exceed 1 MiB, the writer copies each key's latest row into the next generation, commits it and deletes the old file. A reader that loaded the old manifest rereads it once. - Recovery saves the time of its last successful scan plus each manifest's size and modification time. It reads source logs changed since then and skips writing unchanged results. diff --git a/src/extensions/work-attribution/browser.test.ts b/src/extensions/work-attribution/browser.test.ts index dafc65c21..0ec8e7c97 100644 --- a/src/extensions/work-attribution/browser.test.ts +++ b/src/extensions/work-attribution/browser.test.ts @@ -296,6 +296,24 @@ describe("work browser rows", () => { expect(spend).toBe("$0.8750") }) + it("counts a listed work's whole connected report in the header, unlisted works included", () => { + const [listed, older] = [1, 2].map(workId) + const { rows, spend } = browse([ + { + workId: listed, + summary: summary(listed, { requests: [request("listed")] }), + costs: costs(listed, [ + { requestId: "listed", usd: "0.500000000", workIds: [listed] }, + // A connected work too old to be listed. + { requestId: "older", usd: "0.125000000", workIds: [older] }, + ]), + }, + ]) + + expect(rows[0].value).toBe("$0.5000 · no PR") + expect(spend).toBe("$0.6250") + }) + it("names each work's PR state from its saved links", () => { const ids = [1, 2, 3, 4, 5].map(workId) const { rows } = browse( From ce19bc759957eaa1f4120a273aa7807b69161397 Mon Sep 17 00:00:00 2001 From: tautvydasLiekis Date: Fri, 9 Oct 2026 13:16:30 +0300 Subject: [PATCH 13/19] refactor: drop unused temporary-file cleanup from row logs --- src/extensions/work-attribution/cost-details.ts | 4 +++- src/extensions/work-attribution/row-log.ts | 5 ++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/extensions/work-attribution/cost-details.ts b/src/extensions/work-attribution/cost-details.ts index 2a1401a7d..1d40a352c 100644 --- a/src/extensions/work-attribution/cost-details.ts +++ b/src/extensions/work-attribution/cost-details.ts @@ -74,7 +74,7 @@ function spendTotals(rows: Record[]): SpendTotals { return { total, priced, knownCostUsd: usd(nanos) } } -/** Totals for the work a saved report belongs to. Its requests may include connected works'; PR rows count them all. */ +/** Requests sent without a billing tag, by reason. */ function untaggedTotals(rows: Record[]): { untagged?: Record } { const untagged: Record = {} for (const row of rows) @@ -83,6 +83,7 @@ function untaggedTotals(rows: Record[]): { untagged?: Record[]): { failedRefresh?: { count: number; reason?: string } } { const failed = rows.flatMap((row) => object(row.billingLookup) && row.billingLookup.status === "unavailable" ? [row.billingLookup] : [], @@ -96,6 +97,7 @@ function failedRefreshTotals(rows: Record[]): { failedRefresh?: } } +/** Totals for the work a saved report belongs to. Its requests may include connected works'; PR rows count them all. */ export function workCostTotals(report: unknown): WorkCostTotals | undefined { if ( !object(report) || diff --git a/src/extensions/work-attribution/row-log.ts b/src/extensions/work-attribution/row-log.ts index b287625d4..28fed1024 100644 --- a/src/extensions/work-attribution/row-log.ts +++ b/src/extensions/work-attribution/row-log.ts @@ -494,10 +494,9 @@ async function nextGeneration(folder: string, collection: Collection, current = async function removeStrays(folder: string, head: WorkHead): Promise { for (const name of await readdir(join(folder, ROWS)).catch(() => [])) { const match = /^(\w+)\.(\d+)\.jsonl$/.exec(name) - const stray = - match && isCollection(match[1]) ? head.logs[match[1]]?.generation !== Number(match[2]) : /^\..*\.tmp$/.test(name) // A reader may still hold an old generation open. Windows then refuses, and a later update retries. - if (stray) await rm(join(folder, ROWS, name), { force: true }).catch(() => {}) + if (match && isCollection(match[1]) && head.logs[match[1]]?.generation !== Number(match[2])) + await rm(join(folder, ROWS, name), { force: true }).catch(() => {}) } } From 7045f98105e34a6052ccb7be681560cbb2e38952 Mon Sep 17 00:00:00 2001 From: tautvydasLiekis Date: Fri, 9 Oct 2026 15:53:07 +0300 Subject: [PATCH 14/19] style: sort work attribution imports --- src/extensions/work-attribution/commits.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/extensions/work-attribution/commits.ts b/src/extensions/work-attribution/commits.ts index eab5a4cc0..dd86f4e86 100644 --- a/src/extensions/work-attribution/commits.ts +++ b/src/extensions/work-attribution/commits.ts @@ -21,8 +21,8 @@ import { } from "../work-attribution.js" import { debugWorkAttribution } from "./diagnostics.js" import { observeToolFiles } from "./file-observations.js" -import { readWorkRows } from "./row-log.js" import { COST_PER_PR_RESOURCE_ID } from "./resource.js" +import { readWorkRows } from "./row-log.js" const GIT_LOOKUP_TIMEOUT_MS = 2000 From 222fe5083adc4de32b4129b9a311920f9318df69 Mon Sep 17 00:00:00 2001 From: tautvydasLiekis Date: Sat, 10 Oct 2026 00:40:21 +0300 Subject: [PATCH 15/19] refactor: name cost total portions explicit and inferred --- src/extensions/work-attribution/cost-details.ts | 12 ++++++------ src/extensions/work-attribution/cost-sync.test.ts | 8 ++++---- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/extensions/work-attribution/cost-details.ts b/src/extensions/work-attribution/cost-details.ts index 1d40a352c..f781423d8 100644 --- a/src/extensions/work-attribution/cost-details.ts +++ b/src/extensions/work-attribution/cost-details.ts @@ -59,9 +59,9 @@ export interface WorkCostTotals { pullRequest: { provider: string; number: number; state: string; url: string } | null totalCostUsd: string | null knownCostUsd: string - /** Sure and likely parts of a merged or closed PR's spend. */ - sure?: string - likely?: string + /** Confirmed (`explicit`) and inferred parts of a merged or closed PR's spend, as in `costs.json`. */ + explicit?: string + inferred?: string /** An unmerged PR's spend so far, and whether every request it may include is priced. */ soFar?: { knownCostUsd: string; complete: boolean } }[] @@ -152,7 +152,7 @@ export function workCostTotals(report: unknown): WorkCostTotals | undefined { totalCostUsd: typeof row.totalCostUsd === "string" ? row.totalCostUsd : null, knownCostUsd: String(row.knownCostUsd), ...(!unmerged && object(row.explicit) && object(row.inferred) - ? { sure: String(row.explicit.knownCostUsd), likely: String(row.inferred.knownCostUsd) } + ? { explicit: String(row.explicit.knownCostUsd), inferred: String(row.inferred.knownCostUsd) } : {}), ...(unmerged ? { soFar: spendSoFar(row, requests) } : {}), } @@ -221,9 +221,9 @@ export function costDetailLines(value: unknown, path: string): string[] { ) else if (typeof row.totalCostUsd === "string") lines.push(`Cost: $${row.totalCostUsd} USD — ${label}`) else lines.push(`Cost: unknown; $${row.knownCostUsd} USD priced so far — ${label}`) - if (!state && typeof row.sure === "string" && typeof row.likely === "string") + if (!state && typeof row.explicit === "string" && typeof row.inferred === "string") lines.push( - `Confirmed: $${row.sure} USD; inferred: $${row.likely} USD${row.totalCostUsd === null ? " known so far" : ""}.`, + `Confirmed: $${row.explicit} USD; inferred: $${row.inferred} USD${row.totalCostUsd === null ? " known so far" : ""}.`, ) } diff --git a/src/extensions/work-attribution/cost-sync.test.ts b/src/extensions/work-attribution/cost-sync.test.ts index c119b05e6..83bfdbe8b 100644 --- a/src/extensions/work-attribution/cost-sync.test.ts +++ b/src/extensions/work-attribution/cost-sync.test.ts @@ -89,7 +89,7 @@ function previousCostDetailLines(value: unknown, path: string): string[] { : "Account: unknown", ) const label = object(row.pullRequest) ? row.pullRequest.url : row.key - // An unmerged PR's spend stays outside sure and likely totals until it merges. + // An unmerged PR's spend stays outside confirmed and inferred totals until it merges. const state = object(row.pullRequest) && row.pullRequest.state !== "merged" ? row.pullRequest.state : undefined if (state) { const counted = requests.filter( @@ -112,13 +112,13 @@ function previousCostDetailLines(value: unknown, path: string): string[] { lines.push( complete ? `Cost so far: $${spent} USD (${state}) — ${label}` - : `Cost so far: unknown; $${spent} USD confirmed (${state}) — ${label}`, + : `Cost so far: unknown; $${spent} USD priced (${state}) — ${label}`, ) } else if (typeof row.totalCostUsd === "string") lines.push(`Cost: $${row.totalCostUsd} USD — ${label}`) - else lines.push(`Cost: unknown; $${row.knownCostUsd} USD confirmed so far — ${label}`) + else lines.push(`Cost: unknown; $${row.knownCostUsd} USD priced so far — ${label}`) if (!state && object(row.explicit) && object(row.inferred)) lines.push( - `Sure: $${row.explicit.knownCostUsd} USD; likely: $${row.inferred.knownCostUsd} USD${row.totalCostUsd === null ? " known so far" : ""}.`, + `Confirmed: $${row.explicit.knownCostUsd} USD; inferred: $${row.inferred.knownCostUsd} USD${row.totalCostUsd === null ? " known so far" : ""}.`, ) } if (Array.isArray(value.requests)) { From 8f6418328ebc26c5b5b07fb2d649e72434b3ad3c Mon Sep 17 00:00:00 2001 From: tautvydasLiekis Date: Sat, 10 Oct 2026 05:56:54 +0300 Subject: [PATCH 16/19] perf: keep work sessions in a row log and type the cost totals --- docs/work-attribution.md | 12 +- .../work-attribution/browser.test.ts | 54 ++++--- src/extensions/work-attribution/browser.ts | 45 +++--- .../work-attribution/cost-details.ts | 124 ++++++++-------- .../work-attribution/cost-sync.test.ts | 24 ++-- src/extensions/work-attribution/cost-sync.ts | 10 +- .../work-attribution/row-log.test.ts | 68 ++++++--- src/extensions/work-attribution/row-log.ts | 134 ++++++++++-------- .../work-attribution/summary.test.ts | 11 +- src/extensions/work-attribution/summary.ts | 4 +- tests/e2e/tui/support/work-summary.ts | 9 +- 11 files changed, 290 insertions(+), 205 deletions(-) diff --git a/docs/work-attribution.md b/docs/work-attribution.md index 16b1612d4..056330be9 100644 --- a/docs/work-attribution.md +++ b/docs/work-attribution.md @@ -307,16 +307,16 @@ Before sending a covered model request, Kimchi saves and flushes its request ID, Records with the same work ID go into the same summary. Requests are deduplicated by request ID; commit records keep the contributing sessions. `fileTransitions` keeps native edits, `fileObservations` keeps candidate Bash/MCP changes, `continuations` explains why an input continued the current or another saved work, and `workLinks` keeps correction revisions. IDs identify records; positions have no meaning. -`work.json` is a small manifest. It holds the sessions, what `/work` lists (latest activity, repository, branch, retained plan and every commit's PR links) and one entry per collection. Each collection is an append-only log, `rows/..jsonl`, with one JSON row per line. A later line with the same key replaces an earlier one and keeps its position. Keys are the request, transition and observation IDs; a commit's key is its session, SHA, repository and worktree; a plan's is its session, path and retained copy; a continuation's is its session, source and evidence; and a work link's is the whole revision. Only the first `bytes` of each log, as `work.json` records them, are committed: +`work.json` is a small manifest written on one line. It holds what `/work` lists (latest activity, repository, branch, retained plan and every commit's PR links) and one entry per log. Each collection is an append-only log, `rows/..jsonl`, with one JSON row per line. The work's sessions are a log too, `rows/sessions..jsonl`, which gains a row when a session first contributes. A later line with the same key replaces an earlier one and keeps its position. Keys are the request, transition, observation and session IDs; a commit's key is its session, SHA, repository and worktree; a plan's is its session, path and retained copy; a continuation's is its session, source and evidence; and a work link's is the whole revision. Only the first `bytes` of each log, as `work.json` records them, are committed (formatted here): ```json { "version": 2, "workId": "6f102360-39a9-4d6e-8ac7-984c97c0e313", - "sessions": ["01a10dc0-7ea4-74d8-9b87-9a5026c7b05c"], "logs": { "requests": { "generation": 3, "bytes": 33441207, "rows": 15012 }, - "commits": { "generation": 1, "bytes": 41877, "rows": 38 } + "commits": { "generation": 1, "bytes": 41877, "rows": 38 }, + "sessions": { "generation": 1, "bytes": 106, "rows": 2 } } } ``` @@ -338,8 +338,8 @@ All paths below are inside the agent directory. | File | What it is for | | --- | --- | -| `work//work.json` | Manifest: sessions, what `/work` lists and the committed length of each row log. Replacing it commits new rows. | -| `work//rows/..jsonl` | Readable summary rows: request attempts, plans, file edits, continuation evidence, commits and PR links. A later line with the same key replaces an earlier one. | +| `work//work.json` | Manifest: what `/work` lists and the committed length of each row log. Replacing it commits new rows. | +| `work//rows/..jsonl` | Readable summary rows: request attempts, plans, file edits, continuation evidence, commits, PR links and, in `sessions`, the contributing sessions. A later line with the same key replaces an earlier one. | | `work//cost-totals.json` | What `/work` prints: PR totals, and the work's own requests and priced spend. Rewritten when it changes. | | `work//costs.json` | Derived PR totals and each request's allocation, price and billing lookup status, for all connected works. Refreshed at most every five minutes after a journal or refresh failure changes, and rebuilt from source records when missing. | | `work-attribution/billing-polls.json` | Latest polling times and the last refresh that failed before any billing page, only for requests whose billing window is still open. Replaced after checks, so unchanged results and failed refreshes do not grow the journal. | @@ -357,7 +357,7 @@ All paths below are inside the agent directory. - When a log's superseded lines outweigh half its current rows and exceed 1 MiB, the writer copies each key's latest row into the next generation, commits it and deletes the old file. A reader that loaded the old manifest rereads it once. - Recovery saves the time of its last successful scan plus each manifest's size and modification time. It reads source logs changed since then and skips writing unchanged results. - A missing or invalid summary, a log shorter than its committed length, or a damaged row triggers a full replay of the work's source logs, including edit journals, into a new generation. A failed log read or summary write leaves the checkpoint unchanged. -- Summaries saved before row logs keep everything in `work.json` (`"version": 1`). They stay readable and convert on the work's next update; idle works are not rewritten. Releases up to v1.7.1 do not read the new format: they rebuild a version 1 summary from the source logs, without record types they do not know, and the next newer release converts it again and removes the stale logs. +- Summaries saved before row logs keep everything in `work.json` (`"version": 1`). They stay readable and convert on the work's next update; idle works are not rewritten. Releases up to v1.7.1 do not read the new format: they rebuild a version 1 summary from the source logs, without record types they do not know, and the next newer release converts it again and removes the stale logs. Earlier version 2 manifests that list `sessions` themselves stay readable; the work's next update moves them into the sessions log. diff --git a/src/extensions/work-attribution/browser.test.ts b/src/extensions/work-attribution/browser.test.ts index 0ec8e7c97..365b1e2ec 100644 --- a/src/extensions/work-attribution/browser.test.ts +++ b/src/extensions/work-attribution/browser.test.ts @@ -5,7 +5,8 @@ import { tmpdir } from "node:os" import { join } from "node:path" import { afterEach, beforeEach, describe, expect, it, vi } from "vitest" import { buildWorkBrowser, readWorkBrowser, type SavedWork } from "./browser.js" -import { workCostDetails, workCostTotals } from "./cost-details.js" +import { type WorkCostReport, workCostDetails, workCostTotals } from "./cost-details.js" +import type { PullRequestCost } from "./costs.js" import { summaryHead, type WorkHead } from "./row-log.js" vi.mock("node:fs/promises", async (importOriginal) => ({ ...(await importOriginal()) })) @@ -46,24 +47,44 @@ function request(requestId: string, startedAt = "2026-10-08T09:00:00Z") { function costs( id: string, requests: { requestId: string; usd?: string; priced?: boolean; workIds?: string[]; linkedWorkIds?: string[] }[], - pullRequests: unknown[] = [], -) { + pullRequests: PullRequestCost[] = [], +): WorkCostReport { return { - version: 1, workId: id, pullRequests, requests: requests.map(({ requestId, usd = "0", priced = true, workIds = [id], linkedWorkIds }) => ({ requestId, + account: null, workIds, ...(linkedWorkIds ? { linkedWorkIds } : {}), - allocation: "unlinked", + sessionIds: ["session"], + startedAt: null, pullRequestIds: [], + allocation: "unlinked", + billingRecordIds: [], priceStatus: priced ? "priced" : "missing", knownCostUsd: usd, totalCostUsd: priced ? usd : null, })), } } +/** A saved PR total; request lists and portions not given are empty. */ +function pullRequestCost( + row: Pick & + Partial, +): PullRequestCost { + const empty = { requestIds: [], knownCostUsd: "0.000000000", totalCostUsd: "0.000000000" } + return { + account: null, + requestIds: [], + explicit: empty, + inferred: empty, + sharedRequestIds: [], + inferredRequestIds: [], + unknownRequestIds: [], + ...row, + } +} function pullRequest(number: number, state: "open" | "closed" | "merged", provider: "github" | "gitlab" = "github") { const finished = state === "open" ? null : "2026-10-07T10:00:00Z" const path = @@ -94,7 +115,7 @@ function commit(sha: string, pullRequests: unknown[]) { } function save( id: string, - files: { summary?: unknown; costs?: unknown; totals?: string; plans?: Record }, + files: { summary?: unknown; costs?: WorkCostReport; totals?: string; plans?: Record }, at = NOW, ) { const folder = join(agentDir, "work", id) @@ -111,7 +132,7 @@ function save( if (files.summary !== undefined) utimesSync(join(folder, "work.json"), new Date(at), new Date(at)) } function browse( - works: { workId: string; summary?: ReturnType; costs?: unknown }[], + works: { workId: string; summary?: ReturnType; costs?: WorkCostReport }[], lines: string[] = [], ) { const saved: SavedWork[] = works.map((work) => ({ @@ -239,16 +260,19 @@ describe("work browser rows", () => { { requestId: "other", usd: "0.125000000", workIds: [second] }, ], [ - { + pullRequestCost({ key: '["github","github.com","7"]', - account: null, pullRequest: pullRequest(7, "merged"), workIds: [first, second], + requestIds: ["own", "linked", "other"], knownCostUsd: "0.875000000", totalCostUsd: "0.875000000", - explicit: { knownCostUsd: "0.875000000" }, - inferred: { knownCostUsd: "0.000000000" }, - }, + explicit: { + requestIds: ["own", "linked", "other"], + knownCostUsd: "0.875000000", + totalCostUsd: "0.875000000", + }, + }), ], ) const { rows, spend } = browse([ @@ -443,14 +467,13 @@ describe("work browser rows", () => { id, [{ requestId: "r0", usd: "0.250000000" }], [ - { + pullRequestCost({ key: '["github","github.com","7"]', - account: null, pullRequest: pull, workIds: [id], knownCostUsd: "0.000000000", totalCostUsd: null, - }, + }), ], ), }) @@ -478,7 +501,6 @@ describe("work browser rows", () => { version: 2, workId: id, updatedAt: new Date(NOW).toISOString(), - sessions: ["session"], logs: { requests: { generation: 3, bytes: 33_441_207, rows: 15_000 } }, latest: { activityAt: new Date(NOW - index * HOUR).toISOString(), diff --git a/src/extensions/work-attribution/browser.ts b/src/extensions/work-attribution/browser.ts index 9c205b2f3..c530c2a08 100644 --- a/src/extensions/work-attribution/browser.ts +++ b/src/extensions/work-attribution/browser.ts @@ -12,7 +12,7 @@ import { decimalNanos, time } from "./costs.js" import { MAX_SUMMARY_BYTES, readWorkHead, type WorkHead } from "./row-log.js" import { object } from "./summary.js" -/** Besides the current work, the panel lists works whose summary changed recently. */ +/** Besides the current work, the panel lists works with activity this recent. */ const RECENT_MS = 35 * 24 * 60 * 60_000 const MAX_WORKS = 30 /** A plan is read only for its title. */ @@ -66,30 +66,43 @@ export async function readWorkBrowser( }), ) - const recent = [...modified] + const candidates = [...modified] .filter(([workId, at]) => workId !== current.workId && at >= now - RECENT_MS) - .sort(([, left], [, right]) => right - left) - .slice(0, MAX_WORKS - 1) .map(([workId]) => workId) + const [own, ...others] = await Promise.all( + [current.workId, ...candidates].map( + async (workId): Promise => ({ + workId, + head: await readWorkHead(agentDir, workId), + modifiedAt: modified.get(workId), + }), + ), + ) + // Cost passes rewrite work.json weeks after the last request, so saved activity times pick the works listed. + const recent = others + .map((work) => ({ work, activity: lastActivity(work) ?? 0 })) + .filter(({ activity }) => activity >= now - RECENT_MS) + .sort((left, right) => right.activity - left.activity || left.work.workId.localeCompare(right.work.workId)) + .slice(0, MAX_WORKS - 1) + .map(({ work }) => work) const works = await Promise.all( - [current.workId, ...recent].map(async (workId): Promise => { - const folder = join(directory, workId) - const [head, totals] = await Promise.all([ - readWorkHead(agentDir, workId), + [own, ...recent].map(async (work): Promise => { + const folder = join(directory, work.workId) + const [totals, planTitle] = await Promise.all([ readJson(join(folder, "cost-totals.json")), + typeof work.head === "object" ? readPlanTitle(folder, work.head) : undefined, ]) - return { - workId, - head, - totals, - planTitle: typeof head === "object" ? await readPlanTitle(folder, head) : undefined, - modifiedAt: modified.get(workId), - } + return { ...work, totals, planTitle } }), ) return buildWorkBrowser(agentDir, current, works) } +/** Newest request start, native edit or retained plan; the file time when the manifest has none. */ +function lastActivity(work: SavedWork): number | undefined { + return (typeof work.head === "object" ? time(work.head.latest.activityAt) : undefined) ?? work.modifiedAt +} + /** Rows from saved summaries. Only the current work has live PR lookup lines; the rest use their saved links. */ export function buildWorkBrowser(agentDir: string, current: WorkDetailsRequest, works: SavedWork[]): WorkBrowser { const built = works @@ -177,7 +190,7 @@ function workRow(agentDir: string, work: SavedWork, current?: WorkDetailsRequest ? (totals?.links ?? []) : [] const latest = head?.latest - const activity = time(latest?.activityAt) ?? work.modifiedAt + const activity = lastActivity(work) const path = latest?.edit?.repository ?? latest?.commit?.repository ?? latest?.request?.repository ?? latest?.request?.cwd const name = path === undefined ? undefined : repositoryName(path) diff --git a/src/extensions/work-attribution/cost-details.ts b/src/extensions/work-attribution/cost-details.ts index f781423d8..da0141420 100644 --- a/src/extensions/work-attribution/cost-details.ts +++ b/src/extensions/work-attribution/cost-details.ts @@ -1,22 +1,35 @@ import { readFileSync } from "node:fs" import { join } from "node:path" import { isWorkId } from "../../shared/work-id.js" -import { decimalNanos, usd } from "./costs.js" +import type { BillingDisplay } from "./billing-evidence.js" +import { decimalNanos, type PullRequestCost, type RequestCostAllocation, usd } from "./costs.js" import { PRICED_TAG_LIMIT } from "./request-tags.js" import { isWorkAccount, sameWorkAccount, type WorkAccount } from "./scope.js" import { object } from "./summary.js" -// The cost lines /work shows, read from the work's saved costs.json. +// The cost lines /work shows, read from the work's saved cost-totals.json. + +/** A request of a saved report, with the billing state `costs.json` shows for it. */ +export type SavedRequestCost = RequestCostAllocation & { + /** Why the request was sent without a billing tag; such a request is never priced. */ + billingTagSkipped?: string + billingLookup?: BillingDisplay["billingLookup"] +} + +/** The parts of a work's `costs.json` its totals come from; PRs and requests include connected works'. */ +export interface WorkCostReport { + workId: string + pullRequests: PullRequestCost[] + requests: SavedRequestCost[] +} /** A saved report also lists connected works' requests; a work's own are its requests and those linked into it. */ -export function ownRequests(rows: readonly Record[], workId: string): Record[] { - return rows.filter((row) => - [row.workIds, row.linkedWorkIds].some((ids) => Array.isArray(ids) && ids.includes(workId)), - ) +function ownRequests(rows: SavedRequestCost[], workId: string): SavedRequestCost[] { + return rows.filter((row) => row.workIds.includes(workId) || row.linkedWorkIds?.includes(workId)) } /** Priced requests and the known subtotal of saved request costs; unpriced requests add nothing. */ -export function knownSpend(requests: Record[]): { priced: number; total: number; nanos: bigint } { +export function knownSpend(requests: SavedRequestCost[]): { priced: number; total: number; nanos: bigint } { return { priced: requests.filter((row) => row.priceStatus === "priced").length, total: requests.length, @@ -48,7 +61,7 @@ export interface WorkCostTotals { shared: number /** Requests sent without a billing tag, by reason; such requests are never priced. */ untagged?: Record - /** Requests whose last billing refresh failed before any page, with the newest reason. */ + /** Requests whose last billing refresh failed, with the newest reason. */ failedRefresh?: { count: number; reason?: string } } pullRequests: { @@ -59,7 +72,7 @@ export interface WorkCostTotals { pullRequest: { provider: string; number: number; state: string; url: string } | null totalCostUsd: string | null knownCostUsd: string - /** Confirmed (`explicit`) and inferred parts of a merged or closed PR's spend, as in `costs.json`. */ + /** Confirmed (`explicit`) and inferred known subtotals of a merged PR's spend. */ explicit?: string inferred?: string /** An unmerged PR's spend so far, and whether every request it may include is priced. */ @@ -69,64 +82,49 @@ export interface WorkCostTotals { report: "costs.json" } -function spendTotals(rows: Record[]): SpendTotals { +function spendTotals(rows: SavedRequestCost[]): SpendTotals { const { priced, total, nanos } = knownSpend(rows) return { total, priced, knownCostUsd: usd(nanos) } } /** Requests sent without a billing tag, by reason. */ -function untaggedTotals(rows: Record[]): { untagged?: Record } { +function untaggedTotals(rows: SavedRequestCost[]): { untagged?: Record } { const untagged: Record = {} - for (const row of rows) - if (typeof row.billingTagSkipped === "string") - untagged[row.billingTagSkipped] = (untagged[row.billingTagSkipped] ?? 0) + 1 + for (const { billingTagSkipped: reason } of rows) + if (reason !== undefined) untagged[reason] = (untagged[reason] ?? 0) + 1 return Object.keys(untagged).length ? { untagged } : {} } -/** Requests whose last billing refresh failed before any page, with the newest reason. */ -function failedRefreshTotals(rows: Record[]): { failedRefresh?: { count: number; reason?: string } } { - const failed = rows.flatMap((row) => - object(row.billingLookup) && row.billingLookup.status === "unavailable" ? [row.billingLookup] : [], - ) +/** Requests whose last billing refresh failed, with the newest reason. */ +function failedRefreshTotals(rows: SavedRequestCost[]): { failedRefresh?: { count: number; reason?: string } } { + const failed = rows.flatMap(({ billingLookup }) => (billingLookup?.status === "unavailable" ? [billingLookup] : [])) if (!failed.length) return {} - const latest = failed.reduce((left, right) => - String(right.checkedAt ?? "") > String(left.checkedAt ?? "") ? right : left, - ) + const latest = failed.reduce((left, right) => ((right.checkedAt ?? "") > (left.checkedAt ?? "") ? right : left)) return { - failedRefresh: { count: failed.length, ...(typeof latest.reason === "string" ? { reason: latest.reason } : {}) }, + failedRefresh: { count: failed.length, ...(latest.reason === undefined ? {} : { reason: latest.reason }) }, } } /** Totals for the work a saved report belongs to. Its requests may include connected works'; PR rows count them all. */ -export function workCostTotals(report: unknown): WorkCostTotals | undefined { - if ( - !object(report) || - typeof report.workId !== "string" || - !Array.isArray(report.pullRequests) || - !Array.isArray(report.requests) - ) - return undefined - const { workId } = report - const ids = (value: unknown) => (Array.isArray(value) ? value.filter((id) => typeof id === "string") : []) - const requests = report.requests.filter(object) - const pullRequests = report.pullRequests.filter(object) +export function workCostTotals({ workId, pullRequests, requests }: WorkCostReport): WorkCostTotals { // Own spend: the work's requests and requests a correction linked into it, not other connected works'. const own = ownRequests(requests, workId) - const assigned = (allocation: string) => own.filter((row) => row.allocation === allocation).length + const assigned = (allocation: RequestCostAllocation["allocation"]) => + own.filter((row) => row.allocation === allocation).length return { version: 1, workId, group: [ ...new Set([ workId, - ...requests.flatMap((row) => [...ids(row.workIds), ...ids(row.linkedWorkIds)]), - ...pullRequests.flatMap((row) => ids(row.workIds)), + ...requests.flatMap((row) => [...row.workIds, ...(row.linkedWorkIds ?? [])]), + ...pullRequests.flatMap((row) => row.workIds), ]), ].sort(), groupRequests: spendTotals(requests), requests: { ...spendTotals(own), - recorded: requests.filter((row) => ids(row.workIds).includes(workId)).length, + recorded: requests.filter((row) => row.workIds.includes(workId)).length, unresolved: assigned("unknown"), inferred: assigned("inferred"), shared: assigned("shared"), @@ -134,27 +132,24 @@ export function workCostTotals(report: unknown): WorkCostTotals | undefined { ...failedRefreshTotals(own), }, pullRequests: pullRequests.map((row) => { - const pull = object(row.pullRequest) ? row.pullRequest : undefined - // An unmerged PR's spend stays outside confirmed and inferred totals until it merges. - const unmerged = pull && pull.state !== "merged" + const pull = row.pullRequest return { - key: String(row.key), - account: isWorkAccount(row.account) ? row.account : null, - own: ids(row.workIds).includes(workId), - pullRequest: - pull && - (pull.provider === "github" || pull.provider === "gitlab") && - typeof pull.number === "number" && - typeof pull.state === "string" && - typeof pull.url === "string" - ? { provider: pull.provider, number: pull.number, state: pull.state, url: pull.url } - : null, - totalCostUsd: typeof row.totalCostUsd === "string" ? row.totalCostUsd : null, - knownCostUsd: String(row.knownCostUsd), - ...(!unmerged && object(row.explicit) && object(row.inferred) - ? { explicit: String(row.explicit.knownCostUsd), inferred: String(row.inferred.knownCostUsd) } - : {}), - ...(unmerged ? { soFar: spendSoFar(row, requests) } : {}), + key: row.key, + account: row.account, + own: row.workIds.includes(workId), + // A PR link without a provider is GitHub's, as cost reports read stored links. + pullRequest: pull && { + provider: pull.provider ?? "github", + number: pull.number, + state: pull.state, + url: pull.url, + }, + totalCostUsd: row.totalCostUsd, + knownCostUsd: row.knownCostUsd, + // An unmerged PR's spend stays outside confirmed and inferred totals until it merges. + ...(pull && pull.state !== "merged" + ? { soFar: spendSoFar(row, requests) } + : { explicit: row.explicit.knownCostUsd, inferred: row.inferred.knownCostUsd }), } }), report: "costs.json", @@ -162,13 +157,12 @@ export function workCostTotals(report: unknown): WorkCostTotals | undefined { } /** An open or closed PR's priced spend over the requests counted for it so far. */ -function spendSoFar(row: Record, requests: Record[]) { +function spendSoFar(row: PullRequestCost, requests: SavedRequestCost[]) { const counted = requests.filter( (request) => request.allocation === "unmerged" && - Array.isArray(request.pullRequestIds) && request.pullRequestIds.includes(row.key) && - (isWorkAccount(request.account) && isWorkAccount(row.account) + (request.account && row.account ? sameWorkAccount(request.account, row.account) : request.account === row.account), ) @@ -176,11 +170,9 @@ function spendSoFar(row: Record, requests: Record sum + (decimalNanos(request.knownCostUsd) ?? 0n), 0n)), // Unpriced, shared or unresolved requests may still belong to this PR. complete: - isWorkAccount(row.account) && + row.account !== null && counted.every((request) => request.priceStatus === "priced") && - [row.sharedRequestIds, row.inferredRequestIds, row.unknownRequestIds].every( - (ids) => !Array.isArray(ids) || !ids.length, - ), + [row.sharedRequestIds, row.inferredRequestIds, row.unknownRequestIds].every((ids) => !ids.length), } } diff --git a/src/extensions/work-attribution/cost-sync.test.ts b/src/extensions/work-attribution/cost-sync.test.ts index 83bfdbe8b..a14701069 100644 --- a/src/extensions/work-attribution/cost-sync.test.ts +++ b/src/extensions/work-attribution/cost-sync.test.ts @@ -22,7 +22,7 @@ import { savedWorkSummary } from "../__mocks__/work-summary.js" import { appendWorkRecord, getWorkId } from "../work-attribution.js" import { captureBillingSource, requestTagSelector } from "./billing-source.js" import { readWorkBrowser } from "./browser.js" -import { costDetailLines, knownSpend, ownRequests, workCostDetails, workCostTotals } from "./cost-details.js" +import { costDetailLines, workCostDetails, workCostTotals } from "./cost-details.js" import { readWorkCostReport, reconcileWorkCosts } from "./cost-sync.js" import * as costs from "./costs.js" import { calculatePullRequestCosts, decimalNanos, usd } from "./costs.js" @@ -123,13 +123,20 @@ function previousCostDetailLines(value: unknown, path: string): string[] { } if (Array.isArray(value.requests)) { // PR lines above include connected works; this one is the work's own spend, even without a PR. - const own = typeof value.workId === "string" ? ownRequests(requests, value.workId) : requests - const spend = knownSpend(own) + const { workId } = value + const own = + typeof workId === "string" + ? requests.filter((row) => + [row.workIds, row.linkedWorkIds].some((ids) => Array.isArray(ids) && ids.includes(workId)), + ) + : requests + const priced = own.filter((row) => row.priceStatus === "priced").length + const nanos = own.reduce((sum, row) => sum + (decimalNanos(row.knownCostUsd) ?? 0n), 0n) const unresolved = own.filter((row) => row.allocation === "unknown").length const inferred = own.filter((row) => row.allocation === "inferred").length const shared = own.filter((row) => row.allocation === "shared").length lines.push( - `Prices: ${spend.priced}/${spend.total} requests priced, $${usd(spend.nanos)} USD${spend.priced < spend.total ? " known so far" : ""}. PR assignments: ${unresolved} unresolved, ${inferred} inferred, ${shared} shared.`, + `Prices: ${priced}/${own.length} requests priced, $${usd(nanos)} USD${priced < own.length ? " known so far" : ""}. PR assignments: ${unresolved} unresolved, ${inferred} inferred, ${shared} shared.`, ) const untagged = new Map() for (const row of own) @@ -166,9 +173,7 @@ afterEach(async () => { const path = join(dir, "work", workId, "costs.json") if (!existsSync(path)) continue const saved = JSON.parse(readFileSync(path, "utf8")) - const totals = workCostTotals(saved) - if (!totals) continue - expect(costDetailLines(totals, path)).toEqual(previousCostDetailLines(saved, path)) + expect(costDetailLines(workCostTotals(saved), path)).toEqual(previousCostDetailLines(saved, path)) } vi.restoreAllMocks() vi.unstubAllEnvs() @@ -438,10 +443,7 @@ describe("automatic exact work cost lookup", () => { const save = (observations: Parameters[1]) => { const costs = calculatePullRequestCosts(records, observations) mkdirSync(join(dir, "work", workId), { recursive: true }) - writeFileSync( - join(dir, "work", workId, "cost-totals.json"), - JSON.stringify(workCostTotals({ version: 1, workId, ...costs })), - ) + writeFileSync(join(dir, "work", workId, "cost-totals.json"), JSON.stringify(workCostTotals({ workId, ...costs }))) } save([]) expect(workCostDetails(dir, workId)).toContain( diff --git a/src/extensions/work-attribution/cost-sync.ts b/src/extensions/work-attribution/cost-sync.ts index b6d7e7530..eb7776817 100644 --- a/src/extensions/work-attribution/cost-sync.ts +++ b/src/extensions/work-attribution/cost-sync.ts @@ -20,8 +20,9 @@ import { openBilling, } from "./billing-evidence.js" import { type BillingSource, captureBillingSource, LOOKUP_WINDOW_MS, sameBillingSource } from "./billing-source.js" -import { workCostTotals } from "./cost-details.js" +import { type SavedRequestCost, workCostTotals } from "./cost-details.js" import { + type CostTotal, calculatePullRequestCosts, type PullRequestCost, type PullRequestCostReport, @@ -249,7 +250,7 @@ async function publishReports( for (const row of report.pullRequests) if (row.workIds.length) group(pullRequests, component(row.workIds[0]), row) const requests = new Map, RequestCostAllocation[]>() for (const row of report.requests) if (row.workIds.length) group(requests, component(row.workIds[0]), row) - const shown = (row: RequestCostAllocation) => { + const shown = (row: RequestCostAllocation): SavedRequestCost => { const display = displays.get(row.requestId) const failure = shownFailure(display, polls[row.requestId]) return { @@ -261,7 +262,10 @@ async function publishReports( } } // Members of one component share the same rows, so build them once per component. - const contents = new Map, object>() + const contents = new Map< + Set, + { pullRequests: PullRequestCost[]; requests: SavedRequestCost[]; unallocated: Record } + >() const content = (included: Set) => { let found = contents.get(included) if (found) return found diff --git a/src/extensions/work-attribution/row-log.test.ts b/src/extensions/work-attribution/row-log.test.ts index 64b5ee9c6..45e84753d 100644 --- a/src/extensions/work-attribution/row-log.test.ts +++ b/src/extensions/work-attribution/row-log.test.ts @@ -10,10 +10,12 @@ import { createContext } from "../__mocks__/context.js" import { appendWorkRecord } from "../work-attribution.js" import { readWorkBrowser } from "./browser.js" import { workCostTotals } from "./cost-details.js" +import type { RequestCostAllocation } from "./costs.js" import { COLLECTIONS, type Collection, commitPullRequests, + LOGS, logPath, ROW_KEYS, readWorkHead, @@ -116,21 +118,23 @@ function expectCommitted(workId: string, expected: WorkSummaryView, { latest = t expect(head.version).toBe(2) expect(JSON.stringify(readWorkSummary(dir, workId))).toBe(JSON.stringify(expected)) const named: string[] = [] - for (const collection of COLLECTIONS) { - const log = head.logs[collection] + for (const name of LOGS) { + const log = head.logs[name] if (!log) { - expect(expected[collection]).toEqual([]) + expect(expected[name]).toEqual([]) continue } - const path = logPath(folder(workId), collection, log.generation) + const path = logPath(folder(workId), name, log.generation) named.push(basename(path)) const bytes = fs.readFileSync(path) expect(bytes.length).toBe(log.bytes) expect(bytes[log.bytes - 1]).toBe(10) - expect(log.rows).toBe(expected[collection].length) + expect(log.rows).toBe(expected[name].length) } expect(fs.readdirSync(join(folder(workId), "rows")).sort()).toEqual(named.sort()) - expect(head.sessions).toEqual(expected.sessions) + // Sessions live in their log, and the manifest is compact: every update rewrites it. + expect(head.sessions).toBeUndefined() + expect(fs.readFileSync(join(folder(workId), "work.json"), "utf8")).toBe(`${JSON.stringify(head)}\n`) expect(head.pullRequests).toEqual(commitPullRequests(expected.commits)) if (latest) expect(head.latest).toEqual(summaryLatest((collection) => expected[collection]).latest) return head @@ -430,10 +434,9 @@ describe("row logs", () => { await apply(first) const expected = await versionOneUpdate(emptySummary(workId), first) const head = manifest(workId) - for (const collection of COLLECTIONS) { - const log = head.logs[collection] - if (log) - fs.appendFileSync(logPath(folder(workId), collection, log.generation), '{"sessionId":"cut","requestId":"x') + for (const name of LOGS) { + const log = head.logs[name] + if (log) fs.appendFileSync(logPath(folder(workId), name, log.generation), '{"sessionId":"cut","requestId":"x') } expect(readWorkSummary(dir, workId)).toEqual(expected) await apply(second) @@ -536,6 +539,25 @@ describe("row logs", () => { } }) + it("reads a manifest that still lists its sessions and moves them into a log on the next update", async () => { + const workId = randomUUID() + const [first, ...rest] = randomRecords(10, workId, 40) + await apply(first) + const expected = await versionOneUpdate(emptySummary(workId), first) + // Builds before the sessions log listed them in the manifest. + const { sessions: log, ...logs } = manifest(workId).logs + if (!log) throw new Error("sessions were not saved") + fs.rmSync(logPath(folder(workId), "sessions", log.generation)) + fs.writeFileSync( + join(folder(workId), "work.json"), + JSON.stringify({ ...manifest(workId), sessions: expected.sessions, logs }, null, 2), + ) + expect(readWorkSummary(dir, workId)).toEqual(expected) + const next = rest.flat() + await apply(next) + expectCommitted(workId, await versionOneUpdate(expected, next)) + }) + it("rereads the manifest once when a compaction replaces a log during a read", async () => { const workId = randomUUID() const [batch] = randomRecords(7, workId, 40) @@ -767,28 +789,36 @@ describe("a large work", () => { const median = [...times].sort((left, right) => left - right)[10] expect(median).toBeLessThan(50) expect(Math.max(...appended)).toBeLessThanOrEqual(8 * 1024) - expect(fs.statSync(join(folder(workId), "work.json")).size).toBeLessThanOrEqual(16 * 1024) + // Its 200 sessions are in their own log, so each update rewrites a small manifest. + expect(fs.statSync(join(folder(workId), "work.json")).size).toBeLessThanOrEqual(2 * 1024) // No ordinary update scans a log: only the line of each changed row is read. expect(read).toBeLessThan(20 * 8 * 1024) vi.mocked(asyncFs.open).mockClear() vi.mocked(fs.readSync).mockClear() const opened = vi.spyOn(fs, "openSync") - const started = performance.now() const head = await readWorkHead(dir, workId) - expect(performance.now() - started).toBeLessThan(5) expect(head).toMatchObject({ version: 2, logs: { requests: { rows: 20_011 } } }) expect(opened).not.toHaveBeenCalled() expect(vi.mocked(asyncFs.open)).not.toHaveBeenCalled() expect(vi.mocked(fs.readSync)).not.toHaveBeenCalled() // The /work browser lists it from the manifest and the bounded cost totals alone. - const priced = summary.requests.map((row) => ({ - requestId: row.requestId, - workIds: [workId], - priceStatus: "priced", - knownCostUsd: "0.001234560", - })) + const priced = summary.requests.map( + (row): RequestCostAllocation => ({ + requestId: String(row.requestId), + account: null, + workIds: [workId], + sessionIds: [row.sessionId], + startedAt: String(row.startedAt), + pullRequestIds: [], + allocation: "unlinked", + billingRecordIds: [], + priceStatus: "priced", + knownCostUsd: "0.001234560", + totalCostUsd: "0.001234560", + }), + ) fs.writeFileSync( join(folder(workId), "cost-totals.json"), JSON.stringify(workCostTotals({ workId, pullRequests: [], requests: priced })), diff --git a/src/extensions/work-attribution/row-log.ts b/src/extensions/work-attribution/row-log.ts index 28fed1024..610554bff 100644 --- a/src/extensions/work-attribution/row-log.ts +++ b/src/extensions/work-attribution/row-log.ts @@ -8,8 +8,9 @@ import { mergePullRequestLinks } from "../pull-request-status/links.js" /* * Work summary storage. Version 2 `work//work.json` is a small manifest. Each collection's rows live in * `rows/..jsonl`, one version 1 row per line; a later line with the same key replaces an - * earlier one, and the first line of a key fixes its position. A writer appends changed rows, syncs them, then - * renames the manifest, which commits them. This module uses plain `fs` so tests and tools can read summaries. + * earlier one, and the first line of a key fixes its position. Sessions are a log too, one `{"sessionId"}` row each. + * A writer appends changed rows, syncs them, then renames the manifest, which commits them. This module uses plain + * `fs` so tests and tools can read summaries. */ /** The manifest, and the commit point for every row log. */ @@ -42,14 +43,17 @@ export const COLLECTIONS = [ "continuations", ] as const export type Collection = (typeof COLLECTIONS)[number] -const COLLECTION_NAMES = new Set(COLLECTIONS) +/** Every row log: the collections and the work's sessions in first-seen order. */ +export const LOGS = [...COLLECTIONS, "sessions"] as const +export type LogName = (typeof LOGS)[number] +const LOG_NAMES = new Set(LOGS) -function isCollection(name: string): name is Collection { - return COLLECTION_NAMES.has(name) +function isLog(name: string): name is LogName { + return LOG_NAMES.has(name) } -/** Row identity per collection. */ -export const ROW_KEYS: Record string> = { +/** Row identity per log. */ +export const ROW_KEYS: Record string> = { workLinks: (row) => JSON.stringify([ row.linkId, @@ -67,9 +71,10 @@ export const ROW_KEYS: Record string> = { fileTransitions: (row) => JSON.stringify(row.transitionId), fileObservations: (row) => JSON.stringify(row.observationId), continuations: (row) => JSON.stringify([row.sessionId, row.source, row.evidence]), + sessions: (row) => JSON.stringify(row.sessionId), } -/** `rows/..jsonl`. Bytes past `bytes` belong to an interrupted append and are ignored. */ +/** `rows/..jsonl`. Bytes past `bytes` belong to an interrupted append and are ignored. */ export interface RowLog { /** Positive for a log file. `readWorkHead` reports rows still inside a version 1 `work.json` as generation 0. */ generation: number @@ -96,10 +101,10 @@ export interface WorkHead { workId: string /** Time of the last committed change. */ updatedAt: string - /** Sessions in first-seen order. */ - sessions: string[] + /** Sessions in first-seen order, in manifests written before the sessions log; the next update moves them. */ + sessions?: string[] /** A missing log is an empty collection. */ - logs: Partial> + logs: Partial> latest: WorkLatest /** PR links of every commit row, merged. */ pullRequests: Record[] @@ -139,7 +144,7 @@ function text(value: unknown): string | undefined { return typeof value === "string" ? value : undefined } -export function logPath(folder: string, collection: Collection, generation: number): string { +export function logPath(folder: string, collection: LogName, generation: number): string { return join(folder, ROWS, `${collection}.${generation}.jsonl`) } @@ -150,16 +155,12 @@ export function isWorkHead(value: unknown, workId: string): value is WorkHead { value.version === 2 && value.workId === workId && typeof value.updatedAt === "string" && - Array.isArray(value.sessions) && - value.sessions.every((session) => typeof session === "string") && + (value.sessions === undefined || + (Array.isArray(value.sessions) && value.sessions.every((session) => typeof session === "string"))) && object(value.logs) && Object.entries(value.logs).every( ([name, log]) => - isCollection(name) && - object(log) && - count(log.generation, 1) && - count(log.rows, 1) && - count(log.bytes, log.rows + 1), + isLog(name) && object(log) && count(log.generation, 1) && count(log.rows, 1) && count(log.bytes, log.rows + 1), ) && object(value.latest) && Array.isArray(value.pullRequests) @@ -191,7 +192,7 @@ function versionOneView(value: unknown, workId: string): WorkSummaryView | undef }) } -function parseRow(line: string, collection: Collection): Row { +function parseRow(line: string, collection: LogName): Row { let row: unknown try { row = JSON.parse(line) @@ -203,7 +204,7 @@ function parseRow(line: string, collection: Collection): Row { } /** Rows of one committed range, folded by key. */ -function foldLog(collection: Collection, buffer: Buffer): Row[] { +function foldLog(collection: LogName, buffer: Buffer): Row[] { const rows = new Map() let start = 0 for (let end = buffer.indexOf(10); end !== -1; end = buffer.indexOf(10, start)) { @@ -216,7 +217,7 @@ function foldLog(collection: Collection, buffer: Buffer): Row[] { } /** The committed bytes of one log. A compaction may delete it after its manifest was read (ENOENT). */ -function readLogSync(folder: string, collection: Collection, generation: number, bytes: number): Buffer { +function readLogSync(folder: string, collection: LogName, generation: number, bytes: number): Buffer { const fd = openSync(logPath(folder, collection, generation), "r") try { const buffer = Buffer.allocUnsafe(bytes) @@ -260,7 +261,7 @@ function readCommitted( } } -function foldCommitted(folder: string, head: WorkHead, collection: Collection): Row[] { +function foldCommitted(folder: string, head: WorkHead, collection: LogName): Row[] { const log = head.logs[collection] return log ? foldLog(collection, readLogSync(folder, collection, log.generation, log.bytes)) : [] } @@ -271,7 +272,12 @@ export function readWorkSummary(agentDir: string, workId: string): WorkSummaryVi agentDir, workId, (view) => view, - (folder, head) => summaryView(workId, head.sessions, (collection) => foldCommitted(folder, head, collection)), + (folder, head) => + summaryView( + workId, + head.sessions ?? foldCommitted(folder, head, "sessions").map((row) => row.sessionId), + (collection) => foldCommitted(folder, head, collection), + ), ) } @@ -314,10 +320,9 @@ export async function readWorkHead( export function summaryHead(value: unknown, workId: string, updatedAt: string): WorkHead | undefined { const view = versionOneView(value, workId) if (!view) return undefined - const logs: Partial> = {} - for (const collection of COLLECTIONS) - if (view[collection].length) logs[collection] = { generation: 0, bytes: 0, rows: view[collection].length } - return { version: 2, workId, updatedAt, sessions: view.sessions, logs, ...summaryLatest((name) => view[name]) } + const logs: Partial> = {} + for (const name of LOGS) if (view[name].length) logs[name] = { generation: 0, bytes: 0, rows: view[name].length } + return { version: 2, workId, updatedAt, logs, ...summaryLatest((name) => view[name]) } } /** Milliseconds of an ISO 8601 timestamp, as `/work` orders rows; impossible dates are not times. */ @@ -407,11 +412,11 @@ interface LogIndex { const indexes = new Map() let indexedRows = 0 -function cached(folder: string, collection: Collection): LogIndex | undefined { +function cached(folder: string, collection: LogName): LogIndex | undefined { return indexes.get(`${folder}\0${collection}`)?.index } -function remember(folder: string, collection: Collection, index: LogIndex): void { +function remember(folder: string, collection: LogName, index: LogIndex): void { const key = `${folder}\0${collection}` const previous = indexes.get(key) if (previous) { @@ -429,7 +434,7 @@ function remember(folder: string, collection: Collection, index: LogIndex): void /** Drops this work's positions; its next update scans the logs again. */ function forget(folder: string): void { - for (const collection of COLLECTIONS) { + for (const collection of LOGS) { const key = `${folder}\0${collection}` const entry = indexes.get(key) if (!entry) continue @@ -449,7 +454,7 @@ async function readExactly(handle: FileHandle, buffer: Buffer, position: number) /** Indexes committed lines from `index.bytes` to `end`; each must be a valid row, and the range must end a line. */ async function scanLog( handle: FileHandle, - collection: Collection, + collection: LogName, index: LogIndex, end: number, check: RowCheck, @@ -464,7 +469,8 @@ async function scanLog( let start = 0 for (let stop = buffer.indexOf(10); stop !== -1; stop = buffer.indexOf(10, start)) { const row = parseRow(buffer.toString("utf8", start, stop), collection) - if (!check(collection, row)) throw new RowLogDamage(`A ${collection} row is invalid`) + // A session row is valid once it names its session. + if (collection !== "sessions" && !check(collection, row)) throw new RowLogDamage(`A ${collection} row is invalid`) const key = ROW_KEYS[collection](row) const previous = index.lines.get(key) if (previous) index.live -= previous.length + 1 @@ -480,8 +486,8 @@ async function scanLog( index.bytes = end } -/** Next unused generation of a collection, past any file a crash or another version left behind. */ -async function nextGeneration(folder: string, collection: Collection, current = 0): Promise { +/** Next unused generation of a log, past any file a crash or another version left behind. */ +async function nextGeneration(folder: string, collection: LogName, current = 0): Promise { let highest = current for (const name of await readdir(join(folder, ROWS)).catch(() => [])) { const match = /^(\w+)\.(\d+)\.jsonl$/.exec(name) @@ -495,7 +501,7 @@ async function removeStrays(folder: string, head: WorkHead): Promise { for (const name of await readdir(join(folder, ROWS)).catch(() => [])) { const match = /^(\w+)\.(\d+)\.jsonl$/.exec(name) // A reader may still hold an old generation open. Windows then refuses, and a later update retries. - if (match && isCollection(match[1]) && head.logs[match[1]]?.generation !== Number(match[2])) + if (match && isLog(match[1]) && head.logs[match[1]]?.generation !== Number(match[2])) await rm(join(folder, ROWS, name), { force: true }).catch(() => {}) } } @@ -503,7 +509,7 @@ async function removeStrays(folder: string, head: WorkHead): Promise { /** Writes a new log from `[key, line]` pairs; returns its index. */ async function writeLog( folder: string, - collection: Collection, + collection: LogName, generation: number, lines: Iterable<[string, string]>, assertLease: () => void, @@ -571,7 +577,7 @@ export class RowStore { /** Rows read or set in this update, in first-touch order, with the saved line each was read from. */ readonly touched = new Map() constructor( - private readonly collection: Collection, + private readonly collection: LogName, private readonly index?: LogIndex, private readonly readLine?: (line: Line) => string, ) {} @@ -596,12 +602,22 @@ export class RowStore { } } +/** The work's sessions; one first seen in this update is appended to their log. */ +class SessionStore extends RowStore { + add(sessionId: string): void { + const key = ROW_KEYS.sessions({ sessionId }) + if (!this.has(key)) this.set(key, { sessionId }) + } +} + /** Sessions and rows of one work while records merge into it. */ -export type WorkRows = { sessions: Set } & Record +export type WorkRows = { sessions: SessionStore } & Record -function workRows(sessions: Iterable, store: (collection: Collection) => RowStore): WorkRows { +/** `saved` lists sessions that no log holds yet. */ +function workRows(sessions: SessionStore, saved: string[], store: (collection: Collection) => RowStore): WorkRows { + for (const sessionId of saved) sessions.add(sessionId) return { - sessions: new Set(sessions), + sessions, workLinks: store("workLinks"), requests: store("requests"), plans: store("plans"), @@ -619,15 +635,15 @@ interface OpenWork { rebuild: boolean /** A damaged manifest being rebuilt; new generations go past the ones it named. */ damaged?: WorkHead - indexes: Partial> + indexes: Partial> rows: WorkRows close(): void } /** Opens and checks the committed logs, dropping an interrupted append; damage throws. */ async function openLogs(folder: string, head: WorkHead, check: RowCheck): Promise { - const indexes: Partial> = {} - for (const collection of COLLECTIONS) { + const indexes: Partial> = {} + for (const collection of LOGS) { const log = head.logs[collection] if (!log) continue const handle = await open(logPath(folder, collection, log.generation), "r+") @@ -676,7 +692,8 @@ async function openLogs(folder: string, head: WorkHead, check: RowCheck): Promis head, rebuild: false, indexes, - rows: workRows(head.sessions, (collection) => { + // A manifest written before the sessions log lists them itself; this update moves them into the log. + rows: workRows(new SessionStore("sessions", indexes.sessions), head.sessions ?? [], (collection) => { const index = indexes[collection] return new RowStore(collection, index, index && reader(collection, index)) }), @@ -717,7 +734,7 @@ async function openWork( rebuild: !saved, damaged, indexes: {}, - rows: workRows(saved?.sessions ?? [], (collection) => { + rows: workRows(new SessionStore("sessions"), saved?.sessions ?? [], (collection) => { const store = new RowStore(collection) for (const row of saved?.[collection] ?? []) store.set(ROW_KEYS[collection](row), row) return store @@ -750,18 +767,17 @@ async function commitWork( assertLease: () => void, ): Promise { const { head: previous } = work - const changes = new Map() + const changes = new Map() if (previous) - for (const collection of COLLECTIONS) { + for (const collection of LOGS) { const changed = await changedLines(work.rows[collection]) if (changed.length) changes.set(collection, changed) } - const sessions = [...work.rows.sessions] - if (previous && !changes.size && sessions.length === previous.sessions.length) return undefined + if (previous && !changes.size) return undefined assertLease() await mkdir(join(folder, ROWS), { recursive: true, mode: 0o700 }) - const logs: Partial> = {} - const written: Partial> = {} + const logs: Partial> = {} + const written: Partial> = {} let values: Pick let generations = !previous if (previous) { @@ -783,7 +799,7 @@ async function commitWork( index.lines.set(key, { offset, length }) index.live += length + 1 offset += length + 1 - observeLatest(latest, collection, row) + if (collection !== "sessions") observeLatest(latest, collection, row) } index.bytes = offset written[collection] = index @@ -796,7 +812,7 @@ async function commitWork( : previous.pullRequests, } } else { - for (const collection of COLLECTIONS) { + for (const collection of LOGS) { const { touched } = work.rows[collection] if (!touched.size) continue const generation = await nextGeneration(folder, collection, work.damaged?.logs[collection]?.generation) @@ -806,9 +822,9 @@ async function commitWork( values = summaryLatest((collection) => [...work.rows[collection].touched.values()].map(({ row }) => row)) } - const head: WorkHead = { version: 2, workId, updatedAt: new Date().toISOString(), sessions, logs, ...values } - await writeFileDurably(join(folder, WORK_FILE), `${JSON.stringify(head, null, 2)}\n`, assertLease) - for (const collection of COLLECTIONS) { + const head: WorkHead = { version: 2, workId, updatedAt: new Date().toISOString(), logs, ...values } + await writeFileDurably(join(folder, WORK_FILE), `${JSON.stringify(head)}\n`, assertLease) + for (const collection of LOGS) { const index = written[collection] if (index) remember(folder, collection, index) } @@ -883,7 +899,7 @@ async function compactLogs( } const compacted: WorkHead = { ...head, logs, latest, pullRequests } - await writeFileDurably(join(folder, WORK_FILE), `${JSON.stringify(compacted, null, 2)}\n`, assertLease) + await writeFileDurably(join(folder, WORK_FILE), `${JSON.stringify(compacted)}\n`, assertLease) for (const [collection, index, old] of replaced) { remember(folder, collection, index) await rm(logPath(folder, collection, old), { force: true }).catch(() => {}) @@ -938,7 +954,7 @@ export async function validWorkFiles( return false } if (!isWorkHead(value, workId)) return summary(value) - for (const collection of COLLECTIONS) { + for (const collection of LOGS) { const log = value.logs[collection] if (!log) continue const info = await stat(logPath(folder, collection, log.generation)).catch(() => undefined) diff --git a/src/extensions/work-attribution/summary.test.ts b/src/extensions/work-attribution/summary.test.ts index 9ac82c131..1df571f24 100644 --- a/src/extensions/work-attribution/summary.test.ts +++ b/src/extensions/work-attribution/summary.test.ts @@ -478,7 +478,6 @@ describe("readable work summaries", () => { expect(value.requests[0]).not.toHaveProperty("version") expect(value.requests[0]).not.toHaveProperty("type") expect(summary(next).requests).toHaveLength(1) - expect(fs.readFileSync(path(workId), "utf8")).toContain('\n "workId":') }) it("retains request-to-file links and different originating sessions for the same commit", async () => { const workId = getWorkId(context()) @@ -990,9 +989,15 @@ console.log("ready"); await flushWorkSummaries();`, // The version 1 file migrated once; every append landed on a committed line boundary. const head = JSON.parse(fs.readFileSync(path(workId), "utf8")) const log = fs.readFileSync(join(directory, "rows", `requests.${head.logs.requests.generation}.jsonl`)) - expect(head).toMatchObject({ version: 2, logs: { requests: { rows: 24, bytes: log.length } } }) + expect(head).toMatchObject({ + version: 2, + logs: { requests: { rows: 24, bytes: log.length }, sessions: { rows: 3 } }, + }) expect(log.at(-1)).toBe(10) - expect(fs.readdirSync(join(directory, "rows"))).toEqual([`requests.${head.logs.requests.generation}.jsonl`]) + expect(fs.readdirSync(join(directory, "rows")).sort()).toEqual([ + `requests.${head.logs.requests.generation}.jsonl`, + `sessions.${head.logs.sessions.generation}.jsonl`, + ]) } finally { if (reading) clearInterval(reading) for (const { child } of children) child.kill() diff --git a/src/extensions/work-attribution/summary.ts b/src/extensions/work-attribution/summary.ts index 78d7fead1..690cd8079 100644 --- a/src/extensions/work-attribution/summary.ts +++ b/src/extensions/work-attribution/summary.ts @@ -402,7 +402,7 @@ interface RowMap { set(key: string, row: SummaryEntry): unknown } /** One work's rows while records merge: sessions in first-seen order, each collection by row key. */ -export type SummaryRows = { sessions: Set } & Record +export type SummaryRows = { sessions: { add(sessionId: string): unknown } } & Record /** Merge rules for one batch of records, applied row by row. */ export async function mergeWorkRecords(summary: SummaryRows, records: WorkRecord[]): Promise { const { sessions, continuations } = summary @@ -491,7 +491,7 @@ async function update( await mergeWorkRecords(rows, history.concat(records)) }, ) - const bytes = head && Buffer.byteLength(JSON.stringify(head, null, 2)) + const bytes = head && Buffer.byteLength(JSON.stringify(head)) if (bytes && bytes > MANIFEST_WARNING_BYTES) debugWorkAttribution(`Work summary manifest for ${workId} is ${bytes} bytes`) } diff --git a/tests/e2e/tui/support/work-summary.ts b/tests/e2e/tui/support/work-summary.ts index e3b6fe69b..99c848fe9 100644 --- a/tests/e2e/tui/support/work-summary.ts +++ b/tests/e2e/tui/support/work-summary.ts @@ -1,8 +1,8 @@ /** * Reads a work's local summary the way docs/work-attribution.md describes it, independently of the product reader. * Version 1 `work.json` holds every row. Version 2 `work.json` is a manifest whose `logs` name - * `rows/..jsonl`; only the first `bytes` of each log are committed, and a later row with the - * same key replaces an earlier one in place. + * `rows/..jsonl`, sessions included; only the first `bytes` of each log are committed, and a + * later row with the same key replaces an earlier one in place. */ import { readFileSync } from "node:fs" import { join } from "node:path" @@ -27,6 +27,7 @@ const KEYS: Record string> = { fileTransitions: (row) => JSON.stringify(row.transitionId), fileObservations: (row) => JSON.stringify(row.observationId), continuations: (row) => JSON.stringify([row.sessionId, row.source, row.evidence]), + sessions: (row) => JSON.stringify(row.sessionId), } /** The summary in the version 1 shape, or undefined while it is missing or a writer is replacing a log. */ @@ -35,7 +36,7 @@ export function readWorkSummary(agentDir: string, workId: string): Record = { version: 1, workId: manifest.workId, sessions: manifest.sessions } + const view: Record = { version: 1, workId: manifest.workId } for (const [collection, key] of Object.entries(KEYS)) { const log = manifest.logs[collection] const rows = new Map() @@ -49,7 +50,7 @@ export function readWorkSummary(agentDir: string, workId: string): Record row.sessionId) : [...rows.values()] } return view } catch { From 95614e7c6788e2228620a9600bfeca1c35afb72e Mon Sep 17 00:00:00 2001 From: tautvydasLiekis Date: Sat, 10 Oct 2026 11:00:51 +0300 Subject: [PATCH 17/19] refactor: pick merge collections from the record type map --- src/extensions/work-attribution/summary.ts | 19 +++---------------- 1 file changed, 3 insertions(+), 16 deletions(-) diff --git a/src/extensions/work-attribution/summary.ts b/src/extensions/work-attribution/summary.ts index 690cd8079..5c91f1efd 100644 --- a/src/extensions/work-attribution/summary.ts +++ b/src/extensions/work-attribution/summary.ts @@ -392,9 +392,6 @@ const RECORD_COLLECTIONS: Record, Collection file_transition: "fileTransitions", file_observation: "fileObservations", } -function recordKey(type: Exclude, row: SummaryEntry): string { - return ROW_KEYS[RECORD_COLLECTIONS[type]](row) -} /** Rows by key. A store that reads saved rows on demand can stand in for a Map. */ interface RowMap { get(key: string): SummaryEntry | undefined @@ -406,17 +403,6 @@ export type SummaryRows = { sessions: { add(sessionId: string): unknown } } & Re /** Merge rules for one batch of records, applied row by row. */ export async function mergeWorkRecords(summary: SummaryRows, records: WorkRecord[]): Promise { const { sessions, continuations } = summary - const entriesByType = { - work_link: summary.workLinks, - request: summary.requests, - request_dispatch: summary.requests, - request_response: summary.requests, - request_cost: summary.requests, - plan: summary.plans, - commit: summary.commits, - file_transition: summary.fileTransitions, - file_observation: summary.fileObservations, - } for (let index = 0; index < records.length; index++) { if (index % MERGE_BATCH_SIZE === 0) await setImmediate() const { type, version: _version, workId: _workId, ...item } = records[index] @@ -439,8 +425,9 @@ export async function mergeWorkRecords(summary: SummaryRows, records: WorkRecord } continue } - const entries = entriesByType[type] - const key = recordKey(type, item) + const collection = RECORD_COLLECTIONS[type] + const entries = summary[collection] + const key = ROW_KEYS[collection](item) const existing = entries.get(key) if (type === "request_cost" && existing?.billingRows !== undefined && Array.isArray(item.billingRows)) { // Each billing ID is listed once, as its latest observation describes it. From 289de5c77832de5d1e53d20fc0476c5f34eb5fb5 Mon Sep 17 00:00:00 2001 From: tautvydasLiekis Date: Mon, 12 Oct 2026 09:54:46 +0300 Subject: [PATCH 18/19] refactor: drop version 1 work summaries --- docs/work-attribution.md | 4 +- src/extensions/__mocks__/work-summary.ts | 30 +- .../__fixtures__/work-summary-v1-legacy.json | 37 -- .../__fixtures__/work-summary-v1-stack.json | 315 ------------------ .../__fixtures__/work-summary-v1.7.1.json | 127 ------- .../work-attribution/browser.test.ts | 172 +++++----- src/extensions/work-attribution/browser.ts | 54 +-- .../work-attribution/row-log.test.ts | 183 ++++------ src/extensions/work-attribution/row-log.ts | 195 ++++------- .../work-attribution/summary.test.ts | 20 +- src/extensions/work-attribution/summary.ts | 49 +-- tests/e2e/tui/support/work-summary.ts | 11 +- tests/e2e/tui/work-attribution.test.ts | 82 ----- 13 files changed, 277 insertions(+), 1002 deletions(-) delete mode 100644 src/extensions/work-attribution/__fixtures__/work-summary-v1-legacy.json delete mode 100644 src/extensions/work-attribution/__fixtures__/work-summary-v1-stack.json delete mode 100644 src/extensions/work-attribution/__fixtures__/work-summary-v1.7.1.json diff --git a/docs/work-attribution.md b/docs/work-attribution.md index 056330be9..2abd98b4d 100644 --- a/docs/work-attribution.md +++ b/docs/work-attribution.md @@ -59,7 +59,7 @@ In the terminal, `/work` opens a list of works. The current work comes first, ma Amounts in the list are rounded; the details keep exact values. `known so far` means some requests still have no price, including requests made since the last cost check. A work whose requests have no price yet shows `cost unknown`, never `$0`. The header adds up the cost reports of the listed works. Connected works share one report, so its requests count once, including those of connected works that are not listed. -The selected row shows what `/work` prints, plus its repository, branch, last activity and request count. Only the current work has live PR lookup lines; other works show their saved PR links. Enter prints the selected work's details into the conversation, and Esc closes the list. The list reads only each work's small `work.json` and `cost-totals.json`, so it opens quickly for long works and never waits for the network. A summary saved by an older build that is larger than 8 MiB shows `summary too large` with its saved spend and PRs until the work's next update converts it. +The selected row shows what `/work` prints, plus its repository, branch, last activity and request count. Only the current work has live PR lookup lines; other works show their saved PR links. Enter prints the selected work's details into the conversation, and Esc closes the list. The list reads only each work's small `work.json` and `cost-totals.json`, so it opens quickly for long works and never waits for the network. Other works without a readable `work.json` are left out until an update rebuilds their summary. ACP, RPC and print mode keep printing the current work's details as text. So do `/work new`, `/work `, `/work matching on|off` and `/work link|unlink`. @@ -357,7 +357,7 @@ All paths below are inside the agent directory. - When a log's superseded lines outweigh half its current rows and exceed 1 MiB, the writer copies each key's latest row into the next generation, commits it and deletes the old file. A reader that loaded the old manifest rereads it once. - Recovery saves the time of its last successful scan plus each manifest's size and modification time. It reads source logs changed since then and skips writing unchanged results. - A missing or invalid summary, a log shorter than its committed length, or a damaged row triggers a full replay of the work's source logs, including edit journals, into a new generation. A failed log read or summary write leaves the checkpoint unchanged. -- Summaries saved before row logs keep everything in `work.json` (`"version": 1`). They stay readable and convert on the work's next update; idle works are not rewritten. Releases up to v1.7.1 do not read the new format: they rebuild a version 1 summary from the source logs, without record types they do not know, and the next newer release converts it again and removes the stale logs. Earlier version 2 manifests that list `sessions` themselves stay readable; the work's next update moves them into the sessions log. +- A `work.json` that is not a version 2 manifest, such as a `"version": 1` summary saved by v1.7.1 or earlier, is deleted the next time Kimchi lists or updates that work; no other file is touched. The summary is then rebuilt like a missing one, and `/work` leaves the work out until it is. `costs.json`, `cost-totals.json` and PR cost reports come from the source logs, so they do not change. Earlier version 2 manifests that list `sessions` themselves stay readable; the work's next update moves them into the sessions log. diff --git a/src/extensions/__mocks__/work-summary.ts b/src/extensions/__mocks__/work-summary.ts index 693e43e46..e6a2dbb4c 100644 --- a/src/extensions/__mocks__/work-summary.ts +++ b/src/extensions/__mocks__/work-summary.ts @@ -1,8 +1,34 @@ -import { readWorkSummary, type WorkSummaryView } from "../work-attribution/row-log.js" +import { join } from "node:path" +import { + COLLECTIONS, + type Collection, + ROW_KEYS, + type Row, + readWorkSummary, + updateWorkRows, + type WorkSummaryView, +} from "../work-attribution/row-log.js" -/** A work's saved summary in the version 1 shape; fails the test when it is missing or damaged. */ +/** A work's saved rows, folded from its logs; fails the test when the summary is missing or damaged. */ export function savedWorkSummary(agentDir: string, workId: string): WorkSummaryView { const summary = readWorkSummary(agentDir, workId) if (!summary) throw new Error(`No readable work summary for ${workId}`) return summary } + +/** Saves rows as the summary of a work that has none yet, as the writer commits merged records. */ +export async function saveWorkSummary( + agentDir: string, + { workId, sessions, ...rows }: Pick & Partial>, +): Promise { + await updateWorkRows( + join(agentDir, "work", workId), + workId, + { check: () => true, assertLease: () => {} }, + async (summary) => { + for (const sessionId of sessions) summary.sessions.add(sessionId) + for (const collection of COLLECTIONS) + for (const row of rows[collection] ?? []) summary[collection].set(ROW_KEYS[collection](row), row) + }, + ) +} diff --git a/src/extensions/work-attribution/__fixtures__/work-summary-v1-legacy.json b/src/extensions/work-attribution/__fixtures__/work-summary-v1-legacy.json deleted file mode 100644 index 9150de2ac..000000000 --- a/src/extensions/work-attribution/__fixtures__/work-summary-v1-legacy.json +++ /dev/null @@ -1,37 +0,0 @@ -{ - "version": 1, - "workId": "8b304360-39a9-4d6e-8ac7-984c97c0e315", - "sessions": ["01a10dc0-7ea4-74d8-9b87-9a5026c7b05c"], - "requests": [ - { - "requestId": "00000001-1111-4111-8111-111111111111", - "sessionId": "01a10dc0-7ea4-74d8-9b87-9a5026c7b05c", - "cwd": "/src/example", - "provider": "kimchi-dev", - "model": "glm-5.3", - "modelSource": "context", - "recordedAt": "2026-09-01T10:00:00.000Z" - }, - { - "requestId": "00000002-1111-4111-8111-111111111111", - "sessionId": "01a10dc0-7ea4-74d8-9b87-9a5026c7b05c", - "cwd": "/src/example", - "provider": "kimchi-dev", - "model": "glm-5.3", - "modelSource": "context", - "recordedAt": "2026-09-02T10:00:00.000Z" - } - ], - "plans": [], - "commits": [ - { - "sha": "cccccccccccccccccccccccccccccccccccccccc", - "repository": "/src/example/.git", - "worktree": "/src/example", - "sessionId": "01a10dc0-7ea4-74d8-9b87-9a5026c7b05c", - "cwd": "/src/example", - "recordedAt": "2026-08-02T11:00:00.000Z", - "paths": ["a.ts"] - } - ] -} diff --git a/src/extensions/work-attribution/__fixtures__/work-summary-v1-stack.json b/src/extensions/work-attribution/__fixtures__/work-summary-v1-stack.json deleted file mode 100644 index 4c951a3d7..000000000 --- a/src/extensions/work-attribution/__fixtures__/work-summary-v1-stack.json +++ /dev/null @@ -1,315 +0,0 @@ -{ - "version": 1, - "workId": "7a203360-39a9-4d6e-8ac7-984c97c0e314", - "sessions": ["01a10dc0-7ea4-74d8-9b87-9a5026c7b05c", "02b20dc0-7ea4-74d8-9b87-9a5026c7b05c"], - "workLinks": [ - { - "linkId": "88888888-2222-4333-8444-555555555555", - "revision": 1, - "sourceWorkId": "6f102360-39a9-4d6e-8ac7-984c97c0e313", - "targetWorkId": "7a203360-39a9-4d6e-8ac7-984c97c0e314", - "requestIds": ["00000001-1111-4111-8111-111111111111"], - "scope": { - "account": { - "apiUrl": "https://api.example.ai/v1", - "organizationId": "33333333-2222-4333-8444-555555555555", - "userId": "44444444-2222-4333-8444-555555555555" - }, - "repository": "/src/example/.git" - }, - "status": "active", - "evidence": { - "source": "work-command" - }, - "sessionId": "02b20dc0-7ea4-74d8-9b87-9a5026c7b05c", - "cwd": "/src/example", - "recordedAt": "2026-10-02T09:00:00.000Z" - } - ], - "requests": [ - { - "requestId": "00000001-3333-4333-8333-333333333333", - "sessionId": "01a10dc0-7ea4-74d8-9b87-9a5026c7b05c", - "cwd": "/src/example", - "startedAt": "2026-10-01T10:00:00.000Z", - "provider": "kimchi-dev", - "model": "glm-5.3", - "modelSource": "context", - "segment": { - "id": "00000001-4444-4444-8444-444444444444", - "attribution": "explicit", - "reason": "work-command" - }, - "scope": { - "account": { - "apiUrl": "https://api.example.ai/v1", - "organizationId": "33333333-2222-4333-8444-555555555555", - "userId": "44444444-2222-4333-8444-555555555555" - }, - "repository": "/src/example/.git" - }, - "recordedAt": "2026-10-01T10:01:00.000Z", - "dispatchedAt": "2026-10-01T10:00:00.003Z", - "billingSource": { - "apiUrl": "https://api.example.ai/v1", - "gatewayUrl": "https://llm.example.ai/openai/v1/chat/completions", - "credentialHash": "eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee" - }, - "billingSelector": { - "type": "tag", - "tag": "kimchi-request:00000001-3333-4333-8333-333333333333", - "startTime": "2026-09-30T22:00:00.003Z", - "endTime": "2026-11-01T10:00:00.003Z" - }, - "response": { - "status": 200, - "receivedAt": "2026-10-01T10:00:01.000Z", - "traceId": "ffffffffffffffffffffffffffffffff" - }, - "billingRows": [ - { - "id": "00000001-5555-4555-8555-555555555555", - "costUsd": "0.012345678", - "promptTokens": "1200", - "completionTokens": "40", - "model": "glm-5.3" - } - ], - "billingLookup": { - "status": "priced", - "checkedAt": "2026-10-01T10:01:00.000Z", - "organizationId": "33333333-2222-4333-8444-555555555555", - "userId": "44444444-2222-4333-8444-555555555555" - } - }, - { - "requestId": "00000002-3333-4333-8333-333333333333", - "sessionId": "01a10dc0-7ea4-74d8-9b87-9a5026c7b05c", - "cwd": "/src/example", - "startedAt": "2026-10-02T10:00:00.000Z", - "provider": "kimchi-dev", - "model": "glm-5.3", - "modelSource": "context", - "segment": { - "id": "00000002-4444-4444-8444-444444444444", - "attribution": "explicit", - "reason": "work-command" - }, - "scope": { - "account": { - "apiUrl": "https://api.example.ai/v1", - "organizationId": "33333333-2222-4333-8444-555555555555", - "userId": "44444444-2222-4333-8444-555555555555" - }, - "repository": "/src/example/.git" - }, - "recordedAt": "2026-10-02T10:01:00.000Z", - "dispatchedAt": "2026-10-02T10:00:00.003Z", - "billingSource": { - "apiUrl": "https://api.example.ai/v1", - "gatewayUrl": "https://llm.example.ai/openai/v1/chat/completions", - "credentialHash": "eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee" - }, - "billingSelector": { - "type": "tag", - "tag": "kimchi-request:00000002-3333-4333-8333-333333333333", - "startTime": "2026-09-30T22:00:00.003Z", - "endTime": "2026-11-02T10:00:00.003Z" - }, - "response": { - "status": 200, - "receivedAt": "2026-10-02T10:00:01.000Z", - "traceId": "ffffffffffffffffffffffffffffffff" - }, - "billingRows": [ - { - "id": "00000002-5555-4555-8555-555555555555", - "costUsd": "0.012345678", - "promptTokens": "1200", - "completionTokens": "40", - "model": "glm-5.3" - } - ], - "billingLookup": { - "status": "priced", - "checkedAt": "2026-10-02T10:01:00.000Z", - "organizationId": "33333333-2222-4333-8444-555555555555", - "userId": "44444444-2222-4333-8444-555555555555" - } - }, - { - "requestId": "00000003-3333-4333-8333-333333333333", - "sessionId": "02b20dc0-7ea4-74d8-9b87-9a5026c7b05c", - "cwd": "/src/example", - "startedAt": "2026-10-03T10:00:00.000Z", - "provider": "kimchi-dev", - "model": "glm-5.3", - "modelSource": "context", - "segment": { - "id": "00000003-4444-4444-8444-444444444444", - "attribution": "explicit", - "reason": "work-command" - }, - "scope": { - "account": { - "apiUrl": "https://api.example.ai/v1", - "organizationId": "33333333-2222-4333-8444-555555555555", - "userId": "44444444-2222-4333-8444-555555555555" - }, - "repository": "/src/example/.git" - }, - "recordedAt": "2026-10-03T10:01:00.000Z", - "dispatchedAt": "2026-10-03T10:00:00.003Z", - "billingSource": { - "apiUrl": "https://api.example.ai/v1", - "gatewayUrl": "https://llm.example.ai/openai/v1/chat/completions", - "credentialHash": "eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee" - }, - "billingSelector": { - "type": "tag", - "tag": "kimchi-request:00000003-3333-4333-8333-333333333333", - "startTime": "2026-09-30T22:00:00.003Z", - "endTime": "2026-11-03T10:00:00.003Z" - }, - "response": { - "status": 200, - "receivedAt": "2026-10-03T10:00:01.000Z", - "traceId": "ffffffffffffffffffffffffffffffff" - }, - "billingRows": [ - { - "id": "00000003-5555-4555-8555-555555555555", - "costUsd": "0.012345678", - "promptTokens": "1200", - "completionTokens": "40", - "model": "glm-5.3" - } - ], - "billingLookup": { - "status": "priced", - "checkedAt": "2026-10-03T10:01:00.000Z", - "organizationId": "33333333-2222-4333-8444-555555555555", - "userId": "44444444-2222-4333-8444-555555555555" - } - } - ], - "plans": [ - { - "sessionId": "01a10dc0-7ea4-74d8-9b87-9a5026c7b05c", - "path": "/src/example/.kimchi/plans/search.md", - "snapshotPath": "/home/user/.config/kimchi/harness/work/7a203360-39a9-4d6e-8ac7-984c97c0e314/plans/search-3d4e5f.md", - "contentHash": "9999999999999999999999999999999999999999999999999999999999999999", - "requestId": "00000001-3333-4333-8333-333333333333", - "cwd": "/src/example", - "recordedAt": "2026-10-01T09:00:00.000Z" - } - ], - "commits": [ - { - "sha": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", - "repository": "/src/example/.git", - "worktree": "/src/example", - "sessionId": "01a10dc0-7ea4-74d8-9b87-9a5026c7b05c", - "cwd": "/src/example", - "recordedAt": "2026-10-02T11:00:00.000Z", - "source": "bash-git-trace-commits", - "toolCallId": "call_9", - "requestId": "00000002-3333-4333-8333-333333333333", - "paths": ["src/search.ts"], - "transitionIds": ["00000003-2222-4222-8222-222222222222"], - "fileMatches": [ - { - "path": "src/search.ts", - "worktree": "/src/example", - "method": "file-hunks", - "transitionIds": ["00000003-2222-4222-8222-222222222222"] - } - ], - "pullRequests": [ - { - "provider": "github", - "host": "github.com", - "id": "5731", - "repositoryId": "901", - "repository": "example/kimchi", - "number": 731, - "url": "https://github.com/example/kimchi/pull/731", - "state": "open", - "headSha": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", - "mergeCommitSha": null, - "mergedAt": null, - "closedAt": null, - "checkedAt": "2026-10-03T12:00:00Z" - } - ], - "prLookup": { - "status": "linked", - "checkedAt": "2026-10-03T12:00:00Z" - } - } - ], - "fileTransitions": [ - { - "transitionId": "00000003-2222-4222-8222-222222222222", - "toolCallId": "call_3", - "requestId": "00000003-1111-4111-8111-111111111111", - "repository": "/src/example/.git", - "worktree": "/src/example", - "path": "src/file3.ts", - "baseline": null, - "baselineFile": null, - "branch": "feat/search", - "before": null, - "after": { - "blob": "3333333333333333333333333333333333333333", - "mode": "100644" - }, - "cursor": { - "bytes": 0, - "digest": "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc" - }, - "historyBoundaryId": "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", - "sessionId": "01a10dc0-7ea4-74d8-9b87-9a5026c7b05c", - "cwd": "/src/example", - "recordedAt": "2026-09-03T10:05:00.000Z" - } - ], - "fileObservations": [ - { - "observationId": "99999999-2222-4333-8444-555555555555", - "source": "bash", - "toolCallId": "call_8", - "requestId": "00000002-3333-4333-8333-333333333333", - "repository": "/src/example/.git", - "worktree": "/src/example", - "startedAt": "2026-10-02T10:30:00.000Z", - "complete": true, - "files": [ - { - "path": "README.md", - "before": null, - "after": { - "blob": "1111111111111111111111111111111111111111", - "mode": "100644" - } - } - ], - "sessionId": "01a10dc0-7ea4-74d8-9b87-9a5026c7b05c", - "cwd": "/src/example", - "recordedAt": "2026-10-02T10:30:01.000Z" - } - ], - "continuations": [ - { - "sessionId": "02b20dc0-7ea4-74d8-9b87-9a5026c7b05c", - "cwd": "/src/example", - "recordedAt": "2026-10-03T08:00:00.000Z", - "source": "pasted-plan", - "evidence": { - "path": "/home/user/.config/kimchi/harness/work/7a203360-39a9-4d6e-8ac7-984c97c0e314/plans/search-3d4e5f.md", - "contentHash": "9999999999999999999999999999999999999999999999999999999999999999", - "segmentId": "00000003-4444-4444-8444-444444444444" - } - } - ] -} diff --git a/src/extensions/work-attribution/__fixtures__/work-summary-v1.7.1.json b/src/extensions/work-attribution/__fixtures__/work-summary-v1.7.1.json deleted file mode 100644 index da30a1f0f..000000000 --- a/src/extensions/work-attribution/__fixtures__/work-summary-v1.7.1.json +++ /dev/null @@ -1,127 +0,0 @@ -{ - "version": 1, - "workId": "6f102360-39a9-4d6e-8ac7-984c97c0e313", - "sessions": ["01a10dc0-7ea4-74d8-9b87-9a5026c7b05c", "02b20dc0-7ea4-74d8-9b87-9a5026c7b05c"], - "requests": [ - { - "requestId": "00000001-1111-4111-8111-111111111111", - "sessionId": "01a10dc0-7ea4-74d8-9b87-9a5026c7b05c", - "cwd": "/src/example", - "provider": "kimchi-dev", - "model": "glm-5.3", - "modelSource": "context", - "recordedAt": "2026-09-01T10:00:00.000Z" - }, - { - "requestId": "00000002-1111-4111-8111-111111111111", - "sessionId": "01a10dc0-7ea4-74d8-9b87-9a5026c7b05c", - "cwd": "/src/example", - "provider": "kimchi-dev", - "model": "glm-5.3", - "modelSource": "context", - "recordedAt": "2026-09-02T10:00:00.000Z" - }, - { - "requestId": "00000003-1111-4111-8111-111111111111", - "sessionId": "02b20dc0-7ea4-74d8-9b87-9a5026c7b05c", - "cwd": "/src/example", - "provider": "kimchi-dev", - "model": "glm-5.3", - "modelSource": "context", - "recordedAt": "2026-09-03T10:00:00.000Z" - } - ], - "plans": [ - { - "sessionId": "01a10dc0-7ea4-74d8-9b87-9a5026c7b05c", - "path": "/src/example/.kimchi/plans/search.md", - "snapshotPath": "/home/user/.config/kimchi/harness/work/6f102360-39a9-4d6e-8ac7-984c97c0e313/plans/search-0a1b2c.md", - "cwd": "/src/example", - "recordedAt": "2026-09-01T09:00:00.000Z" - } - ], - "commits": [ - { - "sha": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", - "repository": "/src/example/.git", - "worktree": "/src/example", - "sessionId": "01a10dc0-7ea4-74d8-9b87-9a5026c7b05c", - "cwd": "/src/example", - "recordedAt": "2026-09-02T11:00:00.000Z", - "source": "bash-git-trace-commits", - "paths": ["src/file1.ts"], - "transitionIds": ["00000001-2222-4222-8222-222222222222"], - "fileMatches": [ - { - "path": "src/file1.ts", - "worktree": "/src/example", - "method": "file-chain", - "transitionIds": ["00000001-2222-4222-8222-222222222222"] - } - ] - } - ], - "fileTransitions": [ - { - "transitionId": "00000001-2222-4222-8222-222222222222", - "toolCallId": "call_1", - "requestId": "00000001-1111-4111-8111-111111111111", - "repository": "/src/example/.git", - "worktree": "/src/example", - "path": "src/file1.ts", - "baseline": null, - "baselineFile": null, - "branch": "feat/search", - "before": null, - "after": { - "blob": "1111111111111111111111111111111111111111", - "mode": "100644" - }, - "cursor": { - "bytes": 0, - "digest": "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc" - }, - "historyBoundaryId": "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", - "sessionId": "01a10dc0-7ea4-74d8-9b87-9a5026c7b05c", - "cwd": "/src/example", - "recordedAt": "2026-09-01T10:05:00.000Z" - }, - { - "transitionId": "00000002-2222-4222-8222-222222222222", - "toolCallId": "call_2", - "requestId": "00000002-1111-4111-8111-111111111111", - "repository": "/src/example/.git", - "worktree": "/src/example", - "path": "src/file2.ts", - "baseline": null, - "baselineFile": null, - "branch": "main", - "before": null, - "after": { - "blob": "2222222222222222222222222222222222222222", - "mode": "100644" - }, - "cursor": { - "bytes": 0, - "digest": "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc" - }, - "historyBoundaryId": "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", - "sessionId": "02b20dc0-7ea4-74d8-9b87-9a5026c7b05c", - "cwd": "/src/example", - "recordedAt": "2026-09-02T10:05:00.000Z" - } - ], - "continuations": [ - { - "sessionId": "02b20dc0-7ea4-74d8-9b87-9a5026c7b05c", - "cwd": "/src/example", - "recordedAt": "2026-09-03T08:00:00.000Z", - "source": "recent-branch", - "evidence": { - "path": "/src/example/src/file1.ts", - "transitionId": "00000001-2222-4222-8222-222222222222", - "branch": "feat/search" - } - } - ] -} diff --git a/src/extensions/work-attribution/browser.test.ts b/src/extensions/work-attribution/browser.test.ts index 365b1e2ec..3bc603066 100644 --- a/src/extensions/work-attribution/browser.test.ts +++ b/src/extensions/work-attribution/browser.test.ts @@ -4,10 +4,11 @@ import * as asyncFs from "node:fs/promises" import { tmpdir } from "node:os" import { join } from "node:path" import { afterEach, beforeEach, describe, expect, it, vi } from "vitest" +import { saveWorkSummary } from "../__mocks__/work-summary.js" import { buildWorkBrowser, readWorkBrowser, type SavedWork } from "./browser.js" import { type WorkCostReport, workCostDetails, workCostTotals } from "./cost-details.js" import type { PullRequestCost } from "./costs.js" -import { summaryHead, type WorkHead } from "./row-log.js" +import { type Collection, type Row, readWorkHead, type WorkHead } from "./row-log.js" vi.mock("node:fs/promises", async (importOriginal) => ({ ...(await importOriginal()) })) @@ -29,17 +30,9 @@ afterEach(() => { function workId(index: number): string { return `${String(index).padStart(8, "0")}-0000-4000-8000-000000000000` } -function summary(id: string, fields: Record = {}) { - return { - version: 1, - workId: id, - sessions: ["session"], - requests: [], - plans: [], - commits: [], - fileTransitions: [], - ...fields, - } +type Summary = Parameters[1] +function summary(id: string, rows: Partial> = {}): Summary { + return { workId: id, sessions: ["session"], ...rows } } function request(requestId: string, startedAt = "2026-10-08T09:00:00Z") { return { requestId, sessionId: "session", startedAt, cwd: "/src/kimchi", scope: { repository: "/src/kimchi/.git" } } @@ -113,16 +106,18 @@ function commit(sha: string, pullRequests: unknown[]) { pullRequests, } } -function save( +/** Saves a work's files; a summary given as text is written to work.json as is. */ +async function save( id: string, - files: { summary?: unknown; costs?: WorkCostReport; totals?: string; plans?: Record }, + files: { summary?: Summary | string; costs?: WorkCostReport; totals?: string; plans?: Record }, at = NOW, ) { const folder = join(agentDir, "work", id) mkdirSync(join(folder, "plans"), { recursive: true }) + if (typeof files.summary === "string") writeFileSync(join(folder, "work.json"), files.summary) + else if (files.summary) await saveWorkSummary(agentDir, files.summary) const totals = files.totals ?? (files.costs === undefined ? undefined : JSON.stringify(workCostTotals(files.costs))) for (const [name, value] of [ - ["work.json", files.summary], ["costs.json", files.costs], ["cost-totals.json", totals], ] as const) @@ -131,28 +126,37 @@ function save( for (const [name, text] of Object.entries(files.plans ?? {})) writeFileSync(join(folder, "plans", name), text) if (files.summary !== undefined) utimesSync(join(folder, "work.json"), new Date(at), new Date(at)) } -function browse( - works: { workId: string; summary?: ReturnType; costs?: WorkCostReport }[], - lines: string[] = [], -) { - const saved: SavedWork[] = works.map((work) => ({ - workId: work.workId, - head: work.summary && summaryHead(work.summary, work.workId, new Date(NOW).toISOString()), - totals: work.costs && workCostTotals(work.costs), - })) +async function browse(works: { workId: string; summary?: Summary; costs?: WorkCostReport }[], lines: string[] = []) { + const saved: SavedWork[] = [] + for (const work of works) { + if (work.summary) await saveWorkSummary(agentDir, work.summary) + saved.push({ + workId: work.workId, + head: await readWorkHead(agentDir, work.workId), + totals: work.costs && workCostTotals(work.costs), + }) + } return buildWorkBrowser(agentDir, { workId: works[0].workId, lines }, saved) } describe("work browser rows", () => { it("lists the current work first, then recent works by last activity, and leaves out old works", async () => { const [current, older, newer, stale, idle] = [1, 2, 3, 4, 5].map(workId) - save(current, { summary: summary(current, { requests: [request("r1", "2026-09-20T09:00:00Z")] }) }, NOW - DAY) + await save(current, { summary: summary(current, { requests: [request("r1", "2026-09-20T09:00:00Z")] }) }, NOW - DAY) // A later background update does not make older work more recent than newer activity. - save(older, { summary: summary(older, { requests: [request("r2", "2026-10-07T10:00:00Z")] }) }, NOW - HOUR) - save(newer, { summary: summary(newer, { requests: [request("r3", "2026-10-08T09:00:00Z")] }) }, NOW - 2 * HOUR) - save(stale, { summary: summary(stale, { requests: [request("r4", "2026-08-20T09:00:00Z")] }) }, NOW - 40 * DAY) + await save(older, { summary: summary(older, { requests: [request("r2", "2026-10-07T10:00:00Z")] }) }, NOW - HOUR) + await save( + newer, + { summary: summary(newer, { requests: [request("r3", "2026-10-08T09:00:00Z")] }) }, + NOW - 2 * HOUR, + ) + await save( + stale, + { summary: summary(stale, { requests: [request("r4", "2026-08-20T09:00:00Z")] }) }, + NOW - 40 * DAY, + ) // A request's final cost record rewrites the summary of work idle for over a month. - save(idle, { summary: summary(idle, { requests: [request("r5", "2026-08-19T09:00:00Z")] }) }, NOW - HOUR) + await save(idle, { summary: summary(idle, { requests: [request("r5", "2026-08-19T09:00:00Z")] }) }, NOW - HOUR) mkdirSync(join(agentDir, "work", "not-a-work")) const { rows } = await readWorkBrowser(agentDir, { workId: current, lines: [] }, NOW) @@ -164,7 +168,7 @@ describe("work browser rows", () => { it("reads at most 30 works", async () => { for (let index = 1; index <= 35; index++) - save(workId(index), { summary: summary(workId(index)) }, NOW - index * HOUR) + await save(workId(index), { summary: summary(workId(index)) }, NOW - index * HOUR) const { rows } = await readWorkBrowser(agentDir, { workId: workId(35), lines: [] }, NOW) @@ -176,16 +180,20 @@ describe("work browser rows", () => { it("keeps the most recently active works when more works changed recently", async () => { const current = workId(1) - save(current, { summary: summary(current) }) + await save(current, { summary: summary(current) }) // Background cost and PR updates rewrote these summaries after their last activity. for (let index = 2; index <= 30; index++) - save( + await save( workId(index), { summary: summary(workId(index), { requests: [request(`r${index}`, "2026-09-28T09:00:00Z")] }) }, NOW - HOUR, ) const active = workId(31) - save(active, { summary: summary(active, { requests: [request("active", "2026-10-07T09:00:00Z")] }) }, NOW - DAY) + await save( + active, + { summary: summary(active, { requests: [request("active", "2026-10-07T09:00:00Z")] }) }, + NOW - DAY, + ) const { rows } = await readWorkBrowser(agentDir, { workId: current, lines: [] }, NOW) @@ -194,9 +202,9 @@ describe("work browser rows", () => { expect(rows.map((row) => row.workId)).not.toContain(workId(30)) }) - it("shows priced spend and never shows unpriced spend as $0", () => { + it("shows priced spend and never shows unpriced spend as $0", async () => { const ids = [1, 2, 3, 4, 5, 6].map(workId) - const { rows, spend } = browse([ + const { rows, spend } = await browse([ { workId: ids[0], summary: summary(ids[0], { requests: [request("a1"), request("a2")] }), @@ -247,7 +255,7 @@ describe("work browser rows", () => { ) }) - it("shows each connected work's own spend and counts their shared report once", () => { + it("shows each connected work's own spend and counts their shared report once", async () => { const [first, second] = [1, 2].map(workId) // The cost pass saves the whole connected group in each member's costs.json. const group = (id: string) => @@ -275,7 +283,7 @@ describe("work browser rows", () => { }), ], ) - const { rows, spend } = browse([ + const { rows, spend } = await browse([ { workId: first, summary: summary(first, { requests: [request("own"), request("linked")] }), @@ -295,7 +303,7 @@ describe("work browser rows", () => { expect(spend).toBe("$0.8750") }) - it("shows each work's own spend while the saved reports repeat their connected works", () => { + it("shows each work's own spend while the saved reports repeat their connected works", async () => { const [first, second] = [1, 2].map(workId) // The cost pass saves the whole connected group in each member's costs.json. const group = [ @@ -303,7 +311,7 @@ describe("work browser rows", () => { { requestId: "linked", usd: "0.250000000", workIds: [first], linkedWorkIds: [second] }, { requestId: "implementation", usd: "0.125000000", workIds: [second] }, ] - const { rows, spend } = browse([ + const { rows, spend } = await browse([ { workId: first, summary: summary(first, { requests: [request("planning"), request("linked")] }), @@ -320,9 +328,9 @@ describe("work browser rows", () => { expect(spend).toBe("$0.8750") }) - it("counts a listed work's whole connected report in the header, unlisted works included", () => { + it("counts a listed work's whole connected report in the header, unlisted works included", async () => { const [listed, older] = [1, 2].map(workId) - const { rows, spend } = browse([ + const { rows, spend } = await browse([ { workId: listed, summary: summary(listed, { requests: [request("listed")] }), @@ -338,9 +346,9 @@ describe("work browser rows", () => { expect(spend).toBe("$0.6250") }) - it("names each work's PR state from its saved links", () => { + it("names each work's PR state from its saved links", async () => { const ids = [1, 2, 3, 4, 5].map(workId) - const { rows } = browse( + const { rows } = await browse( [ { workId: ids[0], summary: summary(ids[0], { commits: [commit("1", [pullRequest(731, "open")])] }) }, { workId: ids[1], summary: summary(ids[1], { commits: [commit("2", [])] }) }, @@ -389,21 +397,25 @@ describe("work browser rows", () => { snapshotPath: `/old/agent/work/${planned}/plans/csv-abc.md`, recordedAt: "2026-10-08T08:00:00Z", } - save(planned, { + await save(planned, { summary: summary(planned, { plans: [planRecord] }), plans: { "csv-abc.md": `\n# Add CSV\u001b[31m export\n\nSteps\n` }, }) - save(edited, { + await save(edited, { summary: summary(edited, { requests: [request("e1")], fileTransitions: [ { + sessionId: "session", + transitionId: "t1", repository: "/src/kimchi/.git", worktree: "/src/kimchi-csv", branch: "main", recordedAt: "2026-10-07T08:00:00Z", }, { + sessionId: "session", + transitionId: "t2", repository: "/src/kimchi/.git", worktree: "/src/kimchi-csv", branch: "feat/csv", @@ -412,7 +424,7 @@ describe("work browser rows", () => { ], }), }) - save(asked, { + await save(asked, { summary: summary(asked, { requests: [{ ...request("a1"), scope: { repository: "/srv/app.git" } }] }), }) @@ -424,10 +436,10 @@ describe("work browser rows", () => { expect(label(asked)).toBe(" 00000003 app") }) - it("keeps missing and damaged files visible as unknown", async () => { + it("shows a missing summary and damaged totals as unknown, and deletes a damaged work.json", async () => { const [missing, damaged, wrongCosts] = [1, 2, 3].map(workId) - save(damaged, { summary: "{", costs: costs(damaged, [{ requestId: "x1", usd: "0.100000000" }]) }) - save(wrongCosts, { + await save(damaged, { summary: "{", costs: costs(damaged, [{ requestId: "x1", usd: "0.100000000" }]) }) + await save(wrongCosts, { summary: summary(wrongCosts, { requests: [request("y1")], plans: [{ sessionId: "session", path: "/gone.md", snapshotPath: "/gone/plans/missing.md" }], @@ -438,8 +450,9 @@ describe("work browser rows", () => { const { rows, spend } = await readWorkBrowser(agentDir, { workId: missing, lines: [] }, NOW) const row = (id: string) => rows.find((candidate) => candidate.workId === id) - // A damaged summary has no activity times, so its file time orders it. - expect(rows.map((candidate) => candidate.workId)).toEqual([missing, damaged, wrongCosts]) + // The damaged work has no summary now and is listed again once an update rebuilds it; its costs stay. + expect(rows.map((candidate) => candidate.workId)).toEqual([missing, wrongCosts]) + expect(fs.readdirSync(join(agentDir, "work", damaged)).sort()).toEqual(["cost-totals.json", "costs.json", "plans"]) expect(row(missing)).toMatchObject({ label: "● 00000001 summary unavailable", value: "cost unknown · no PR" }) expect(row(missing)?.description).toBe( [ @@ -448,48 +461,9 @@ describe("work browser rows", () => { "Cost: unknown; waiting for billing", ].join("\n"), ) - expect(row(damaged)).toMatchObject({ label: " 00000002 summary unavailable", value: "$0.1000 · no PR" }) expect(row(wrongCosts)).toMatchObject({ label: " 00000003 kimchi", value: "cost unknown · no PR" }) expect(row(wrongCosts)?.description).toContain("Cost: unknown; waiting for billing") - expect(spend).toBe("$0.1000 known so far") - }) - - it("labels an oversized version 1 summary as too large and keeps its PRs and partial spend", async () => { - const id = workId(1) - const pull = pullRequest(7, "open") - const requests = Array.from({ length: 3600 }, (_, index) => ({ - ...request(`r${index}`), - billingRows: [{ id: `bill-${index}`, costUsd: "0.001", note: "x".repeat(2400) }], - })) - save(id, { - summary: summary(id, { requests, commits: [commit("1", [pull])] }), - costs: costs( - id, - [{ requestId: "r0", usd: "0.250000000" }], - [ - pullRequestCost({ - key: '["github","github.com","7"]', - pullRequest: pull, - workIds: [id], - knownCostUsd: "0.000000000", - totalCostUsd: null, - }), - ], - ), - }) - expect(fs.statSync(join(agentDir, "work", id, "work.json")).size).toBeGreaterThan(8 * 1024 * 1024) - - const { rows } = await readWorkBrowser(agentDir, { workId: workId(2), lines: [] }, NOW) - - expect(rows[1]).toMatchObject({ - label: " 00000001 summary too large", - // Requests newer than the last cost pass may be missing from the totals. - value: "$0.2500 known so far · PR #7 open", - }) - expect(rows[1].description).toContain( - `Summary too large to list until the work's next update: ${join(agentDir, "work", id, "work.json")}`, - ) - expect(rows[1].description).toContain("PR #7 open: https://github.com/example/kimchi/pull/7") + expect(spend).toBe("cost unknown") }) it("opens 30 long works from their manifests and cost totals alone", async () => { @@ -533,10 +507,18 @@ describe("work browser rows", () => { it("describes a work with the details /work prints plus where and when it ran", async () => { const id = workId(1) - save(id, { + await save(id, { summary: summary(id, { requests: [request("r1", "2026-10-08T08:00:00Z"), request("r2", "2026-10-08T09:30:00Z")], - fileTransitions: [{ repository: "/src/kimchi/.git", branch: "feat/csv", recordedAt: "2026-10-08T09:00:00Z" }], + fileTransitions: [ + { + sessionId: "session", + transitionId: "t1", + repository: "/src/kimchi/.git", + branch: "feat/csv", + recordedAt: "2026-10-08T09:00:00Z", + }, + ], }), costs: costs(id, [{ requestId: "r1", usd: "0.031133200" }]), }) @@ -556,7 +538,7 @@ describe("work browser rows", () => { ) }) - it("explains unknown prices with the work's own untagged requests and failed refreshes only", () => { + it("explains unknown prices with the work's own untagged requests and failed refreshes only", async () => { const [first, second] = [1, 2].map(workId) // The cost pass saves the whole connected group in each member's costs.json. const group = (id: string) => { @@ -574,7 +556,7 @@ describe("work browser rows", () => { }) return report } - const { rows } = browse([ + const { rows } = await browse([ { workId: first, summary: summary(first, { requests: [request("planning")] }), costs: group(first) }, { workId: second, summary: summary(second, { requests: [request("untagged")] }), costs: group(second) }, ]) diff --git a/src/extensions/work-attribution/browser.ts b/src/extensions/work-attribution/browser.ts index c530c2a08..ed0eddb3a 100644 --- a/src/extensions/work-attribution/browser.ts +++ b/src/extensions/work-attribution/browser.ts @@ -9,13 +9,14 @@ import type { WorkPullRequest } from "../pull-request-status/pull-requests.js" import type { WorkDetailsRequest } from "../work-attribution.js" import { costDetailLines, type knownSpend } from "./cost-details.js" import { decimalNanos, time } from "./costs.js" -import { MAX_SUMMARY_BYTES, readWorkHead, type WorkHead } from "./row-log.js" +import { readWorkHead, type WorkHead } from "./row-log.js" import { object } from "./summary.js" /** Besides the current work, the panel lists works with activity this recent. */ const RECENT_MS = 35 * 24 * 60 * 60_000 const MAX_WORKS = 30 -/** A plan is read only for its title. */ +/** Larger files are skipped rather than read; a plan is read only for its title. */ +const MAX_TOTALS_BYTES = 8 * 1024 * 1024 const MAX_PLAN_BYTES = 256 * 1024 type Spend = ReturnType @@ -24,8 +25,8 @@ type Link = Pick /** One work's saved files; any of them can be missing or damaged. Each stays small however long the work runs. */ export interface SavedWork { workId: string - /** The `work.json` manifest, or "too-large" for a version 1 summary over the read limit. */ - head?: WorkHead | "too-large" + /** The `work.json` manifest. */ + head?: WorkHead /** Parsed `cost-totals.json`. */ totals?: unknown planTitle?: string @@ -79,7 +80,9 @@ export async function readWorkBrowser( ), ) // Cost passes rewrite work.json weeks after the last request, so saved activity times pick the works listed. + // Reading deletes a work.json that is not a manifest; that work is listed again once an update rebuilds it. const recent = others + .filter((work) => work.head) .map((work) => ({ work, activity: lastActivity(work) ?? 0 })) .filter(({ activity }) => activity >= now - RECENT_MS) .sort((left, right) => right.activity - left.activity || left.work.workId.localeCompare(right.work.workId)) @@ -90,7 +93,7 @@ export async function readWorkBrowser( const folder = join(directory, work.workId) const [totals, planTitle] = await Promise.all([ readJson(join(folder, "cost-totals.json")), - typeof work.head === "object" ? readPlanTitle(folder, work.head) : undefined, + work.head && readPlanTitle(folder, work.head), ]) return { ...work, totals, planTitle } }), @@ -100,7 +103,7 @@ export async function readWorkBrowser( /** Newest request start, native edit or retained plan; the file time when the manifest has none. */ function lastActivity(work: SavedWork): number | undefined { - return (typeof work.head === "object" ? time(work.head.latest.activityAt) : undefined) ?? work.modifiedAt + return time(work.head?.latest.activityAt) ?? work.modifiedAt } /** Rows from saved summaries. Only the current work has live PR lookup lines; the rest use their saved links. */ @@ -134,18 +137,6 @@ export function buildWorkBrowser(agentDir: string, current: WorkDetailsRequest, } } -function pullRequestLink(value: unknown): Link | undefined { - if ( - !object(value) || - (value.provider !== "github" && value.provider !== "gitlab") || - typeof value.number !== "number" || - (value.state !== "open" && value.state !== "closed" && value.state !== "merged") || - typeof value.url !== "string" - ) - return undefined - return { provider: value.provider, number: value.number, state: value.state, url: value.url } -} - function spendOf(value: unknown): Spend | undefined { return object(value) && typeof value.priced === "number" && typeof value.total === "number" ? { priced: value.priced, total: value.total, nanos: decimalNanos(value.knownCostUsd) ?? 0n } @@ -163,19 +154,13 @@ function costTotals(value: unknown, workId: string) { own, recorded: value.requests.recorded, group: { key: JSON.stringify(value.group), spend: group }, - /** PRs this work committed to. */ - links: value.pullRequests.flatMap((row) => { - const link = object(row) && row.own === true ? pullRequestLink(row.pullRequest) : undefined - return link ? [link] : [] - }), } } function workRow(agentDir: string, work: SavedWork, current?: WorkDetailsRequest) { const { workId } = work const folder = join(agentDir, "work", workId) - const head = typeof work.head === "object" ? work.head : undefined - const tooLarge = work.head === "too-large" + const { head } = work const totals = costTotals(work.totals, workId) const requests = head?.logs.requests?.rows ?? 0 // Requests recorded after the last cost pass are not in the totals yet; they count as unpriced. @@ -183,12 +168,7 @@ function workRow(agentDir: string, work: SavedWork, current?: WorkDetailsRequest const spent: Spend | undefined = totals ? { ...totals.own, total: totals.own.total + unpriced } : head && { priced: 0, total: unpriced, nanos: 0n } - // A summary too large to read still has its PRs in the totals, but its newest requests may be unpriced. - const links: Link[] = head - ? mergePullRequestLinks(storedPullRequests(head.pullRequests)) - : tooLarge - ? (totals?.links ?? []) - : [] + const links: Link[] = head ? mergePullRequestLinks(storedPullRequests(head.pullRequests)) : [] const latest = head?.latest const activity = lastActivity(work) const path = @@ -203,13 +183,11 @@ function workRow(agentDir: string, work: SavedWork, current?: WorkDetailsRequest `Repository: ${name ? plain(name) : "unknown"}${branch ? ` · branch ${plain(branch)}` : ""}`, `Last activity: ${activity === undefined ? "unknown" : localTime(activity)} · ${requests} request${requests === 1 ? "" : "s"}`, ] - : [ - `${tooLarge ? "Summary too large to list until the work's next update" : "Summary unavailable"}: ${join(folder, "work.json")}`, - ] + : [`Summary unavailable: ${join(folder, "work.json")}`] const row: WorkRow = { workId, - label: `${current ? "●" : " "} ${workId.slice(0, 8)} ${title || (head ? "untitled work" : tooLarge ? "summary too large" : "summary unavailable")}`, - value: `${spendText(spent, tooLarge)} · ${pullRequestState(links)}`, + label: `${current ? "●" : " "} ${workId.slice(0, 8)} ${title || (head ? "untitled work" : "summary unavailable")}`, + value: `${spendText(spent)} · ${pullRequestState(links)}`, description: [`Work ID: ${workId}`, ...context, ...pullRequests, ...costLines].join("\n"), details: [`Work ID: ${workId}`, ...pullRequests, ...costLines].join("\n"), } @@ -217,7 +195,7 @@ function workRow(agentDir: string, work: SavedWork, current?: WorkDetailsRequest row, current: current !== undefined, activity, - partial: !spent || tooLarge, + partial: !spent, unpriced, group: totals?.group, } @@ -278,7 +256,7 @@ async function readText(path: string, limit: number): Promise { - const text = await readText(path, MAX_SUMMARY_BYTES) + const text = await readText(path, MAX_TOTALS_BYTES) try { return text === undefined ? undefined : JSON.parse(text) } catch { diff --git a/src/extensions/work-attribution/row-log.test.ts b/src/extensions/work-attribution/row-log.test.ts index 45e84753d..473bc19bc 100644 --- a/src/extensions/work-attribution/row-log.test.ts +++ b/src/extensions/work-attribution/row-log.test.ts @@ -7,6 +7,7 @@ import { basename, join } from "node:path" import { performance } from "node:perf_hooks" import { afterEach, beforeEach, describe, expect, it, vi } from "vitest" import { createContext } from "../__mocks__/context.js" +import { saveWorkSummary } from "../__mocks__/work-summary.js" import { appendWorkRecord } from "../work-attribution.js" import { readWorkBrowser } from "./browser.js" import { workCostTotals } from "./cost-details.js" @@ -19,7 +20,6 @@ import { logPath, ROW_KEYS, readWorkHead, - readWorkRows, readWorkSummary, summaryLatest, type WorkHead, @@ -50,16 +50,8 @@ function folder(workId: string): string { function manifest(workId: string): WorkHead { return JSON.parse(fs.readFileSync(join(folder(workId), "work.json"), "utf8")) } -function writeVersionOne(summary: WorkSummaryView): void { - fs.mkdirSync(folder(summary.workId), { recursive: true }) - fs.writeFileSync(join(folder(summary.workId), "work.json"), `${JSON.stringify(summary, null, 2)}\n`) -} -function fixture(name: string): WorkSummaryView { - return JSON.parse(fs.readFileSync(new URL(`./__fixtures__/${name}`, import.meta.url), "utf8")) -} function emptySummary(workId: string): WorkSummaryView { return { - version: 1, workId, sessions: [], workLinks: [], @@ -71,19 +63,9 @@ function emptySummary(workId: string): WorkSummaryView { continuations: [], } } -/** The version 1 shape with every collection, in the order the row logs fold into. */ -function withDefaults(value: WorkSummaryView): WorkSummaryView { - const empty = emptySummary(value.workId) - return { - ...empty, - sessions: value.sessions, - ...Object.fromEntries(COLLECTIONS.map((collection) => [collection, value[collection] ?? []])), - } -} - -/** How version 1 applied a batch: parse the whole summary, merge in memory, serialize it again. */ -async function versionOneUpdate(summary: WorkSummaryView, records: WorkRecord[]): Promise { - const copy = withDefaults(JSON.parse(JSON.stringify(summary))) +/** The reference: the same merge applied to a copy of every row in memory, then serialized. */ +async function mergeInMemory(summary: WorkSummaryView, records: WorkRecord[]): Promise { + const copy: WorkSummaryView = JSON.parse(JSON.stringify(summary)) const keyed = (collection: Collection) => new Map(copy[collection].map((row) => [ROW_KEYS[collection](row), row])) const rows = { sessions: new Set(copy.sessions), @@ -98,7 +80,6 @@ async function versionOneUpdate(summary: WorkSummaryView, records: WorkRecord[]) await mergeWorkRecords(rows, structuredClone(records)) return JSON.parse( JSON.stringify({ - version: 1, workId: copy.workId, sessions: [...rows.sessions], ...Object.fromEntries(COLLECTIONS.map((collection) => [collection, [...rows[collection].values()]])), @@ -112,7 +93,7 @@ async function apply(records: WorkRecord[]): Promise { await flushWorkSummaries() } -/** The committed files match the expected version 1 view, and no stray row file is left. */ +/** The committed logs fold to the expected rows, and no stray row file is left. */ function expectCommitted(workId: string, expected: WorkSummaryView, { latest = true } = {}): WorkHead { const head = manifest(workId) expect(head.version).toBe(2) @@ -331,51 +312,51 @@ function randomRecords(seed: number, workId: string, count: number, { replays = return batches } -describe("version 1 summaries", () => { - it.each([ - "work-summary-v1.7.1.json", - "work-summary-v1-stack.json", - "work-summary-v1-legacy.json", - ])("reads %s unchanged and migrates it on the next update", async (name) => { - const saved = fixture(name) - writeVersionOne(saved) - const { workId } = saved - expect(readWorkSummary(dir, workId)).toEqual(withDefaults(saved)) - expect(readWorkRows(dir, workId, "commits")).toEqual(saved.commits) - const head = await readWorkHead(dir, workId) - expect(head).toMatchObject({ version: 2, logs: { requests: { rows: saved.requests.length } } }) - const existing = saved.requests[0] - const batch: WorkRecord[] = [ - { - version: 1, - type: "request_response", - workId, - sessionId: existing.sessionId, - requestId: String(existing.requestId), - response: { status: 200, receivedAt: "2026-10-08T12:00:00.000Z" }, - recordedAt: "2026-10-08T12:00:00.000Z", - }, - { - version: 1, - type: "request", - workId, - sessionId: "new-session", - requestId: "new-request", - startedAt: "2026-10-08T12:00:01.000Z", - recordedAt: "2026-10-08T12:00:01.000Z", - }, - ] - await apply(batch) - const expected = await versionOneUpdate(saved, batch) - expectCommitted(workId, expected) - expect(readWorkSummary(dir, workId)?.requests.at(-1)).toMatchObject({ requestId: "new-request" }) +describe("a work.json from an older release", () => { + const older = () => createContext({ cwd: "/project", sessionManager: { getSessionId: () => "older" } }) + /** A work its journals record, whose work.json a release before row logs saved with every row. */ + async function olderWork(): Promise { + const workId = randomUUID() + appendWorkRecord(older(), { type: "request", requestId: "journaled", startedAt: iso(BASE) }, workId) + await flushWorkSummaries() + fs.rmSync(join(folder(workId), "rows"), { recursive: true }) + // No journal has this row; only the old file does. + const row = { requestId: "only-in-work-json", sessionId: "older", startedAt: iso(BASE) } + const summary = { version: 1, workId, sessions: ["older"], requests: [row], plans: [], commits: [] } + fs.writeFileSync(join(folder(workId), "work.json"), `${JSON.stringify(summary, null, 2)}\n`) + fs.writeFileSync( + join(folder(workId), "cost-totals.json"), + JSON.stringify(workCostTotals({ workId, pullRequests: [], requests: [] })), + ) + return workId + } + + it("is deleted by /work, which leaves the work out until its next update rebuilds it", async () => { + const workId = await olderWork() + const kept = fs + .readdirSync(folder(workId)) + .filter((name) => name !== "work.json") + .sort() + const current = randomUUID() + + const { rows } = await readWorkBrowser(dir, { workId: current, lines: [] }, BASE + 60_000) + + expect(rows.map((row) => row.workId)).toEqual([current]) + expect(fs.readdirSync(folder(workId)).sort()).toEqual(kept) + appendWorkRecord(older(), { type: "request", requestId: "next", startedAt: iso(BASE + 1000) }, workId) + await flushWorkSummaries() + const listed = await readWorkBrowser(dir, { workId: current, lines: [] }, BASE + 60_000) + expect(listed.rows.find((row) => row.workId === workId)?.description).toContain("· 2 requests") }) - it("reads a version 1 summary that grew past the listing limit only as too large", async () => { - const saved = fixture("work-summary-v1.7.1.json") - writeVersionOne(saved) - expect(await readWorkHead(dir, saved.workId, 100)).toBe("too-large") - expect(readWorkSummary(dir, saved.workId)).toEqual(withDefaults(saved)) + it("is deleted by the next update, which rebuilds the work from its journals", async () => { + const workId = await olderWork() + + appendWorkRecord(older(), { type: "request", requestId: "next", startedAt: iso(BASE + 1000) }, workId) + await flushWorkSummaries() + + expect(manifest(workId).version).toBe(2) + expect(readWorkSummary(dir, workId)?.requests.map((row) => row.requestId)).toEqual(["journaled", "next"]) }) }) @@ -386,31 +367,25 @@ describe("row logs", () => { [3, { large: true }], [4, { replays: true, large: true }], ])( - "fold to the version 1 merge of random records (seed %i, %o)", + "fold to an in-memory merge of random records (seed %i, %o)", async (seed, options) => { const workId = randomUUID() - const batches = randomRecords(seed, workId, 360, options) - let expected = await versionOneUpdate(emptySummary(workId), batches.slice(0, 5).flat()) - writeVersionOne(expected) + let expected = emptySummary(workId) let compactions = 0 let previous: WorkHead | undefined - for (const [index, batch] of batches.slice(5).entries()) { - let migrated = !previous - if (index === 30) { - // An older release rewrote the summary as version 1; its logs are left behind. - writeVersionOne(expected) - migrated = true - } + for (const batch of randomRecords(seed, workId, 360, options)) { await apply(batch) - expected = await versionOneUpdate(expected, batch) - const generations = (head?: WorkHead) => - COLLECTIONS.map((collection) => head?.logs[collection]?.generation ?? 0) + expected = await mergeInMemory(expected, batch) // Replayed older records can move a row's time back; /work keeps the newer value until a recompute. - const head = expectCommitted(workId, expected, { latest: !options.replays || migrated }) - const advanced = generations(head).some((generation, at) => generation > generations(previous)[at]) - if (advanced && !migrated && previous) { + // The first commit computes it from every row. + const head = expectCommitted(workId, expected, { latest: !options.replays || !previous }) + // A compaction moves a log to its next generation and recomputes what /work lists from every row. + const compacted = COLLECTIONS.some((collection) => { + const log = previous?.logs[collection] + return log && (head.logs[collection]?.generation ?? 0) > log.generation + }) + if (compacted) { compactions++ - // Compaction recomputes what /work lists from every row. expect(head.latest).toEqual(summaryLatest((collection) => expected[collection]).latest) } previous = head @@ -432,7 +407,7 @@ describe("row logs", () => { [[], []], ) await apply(first) - const expected = await versionOneUpdate(emptySummary(workId), first) + const expected = await mergeInMemory(emptySummary(workId), first) const head = manifest(workId) for (const name of LOGS) { const log = head.logs[name] @@ -440,7 +415,7 @@ describe("row logs", () => { } expect(readWorkSummary(dir, workId)).toEqual(expected) await apply(second) - expectCommitted(workId, await versionOneUpdate(expected, second)) + expectCommitted(workId, await mergeInMemory(expected, second)) }) it.each(["missing", "short", "garbled"])("rebuilds a work from its journals when a log is %s", async (damage) => { @@ -503,7 +478,7 @@ describe("row logs", () => { for (const batch of batches) { renames = 0 await apply(batch) - expected = await versionOneUpdate(expected, batch) + expected = await mergeInMemory(expected, batch) if (fs.readdirSync(join(folder(workId), "rows")).length > Object.keys(manifest(workId).logs).length) break } const stray = fs @@ -520,30 +495,14 @@ describe("row logs", () => { vi.mocked(asyncFs.rename).mockRestore() const next = records.slice(0, 1) await apply(next) - expectCommitted(workId, await versionOneUpdate(expected, next)) + expectCommitted(workId, await mergeInMemory(expected, next)) }, 60_000) - it("starts a fresh generation after an older release rewrote the summary as version 1", async () => { - const workId = randomUUID() - const [first, ...rest] = randomRecords(6, workId, 30) - await apply(first) - const old = manifest(workId) - const expected = await versionOneUpdate(emptySummary(workId), first) - writeVersionOne(expected) - const next = rest.flat() - await apply(next) - const head = expectCommitted(workId, await versionOneUpdate(expected, next)) - for (const collection of COLLECTIONS) { - const generation = old.logs[collection]?.generation - if (generation) expect(head.logs[collection]?.generation).toBeGreaterThan(generation) - } - }) - it("reads a manifest that still lists its sessions and moves them into a log on the next update", async () => { const workId = randomUUID() const [first, ...rest] = randomRecords(10, workId, 40) await apply(first) - const expected = await versionOneUpdate(emptySummary(workId), first) + const expected = await mergeInMemory(emptySummary(workId), first) // Builds before the sessions log listed them in the manifest. const { sessions: log, ...logs } = manifest(workId).logs if (!log) throw new Error("sessions were not saved") @@ -555,7 +514,7 @@ describe("row logs", () => { expect(readWorkSummary(dir, workId)).toEqual(expected) const next = rest.flat() await apply(next) - expectCommitted(workId, await versionOneUpdate(expected, next)) + expectCommitted(workId, await mergeInMemory(expected, next)) }) it("rereads the manifest once when a compaction replaces a log during a read", async () => { @@ -628,7 +587,7 @@ for (const batch of JSON.parse(readFileSync(process.argv[2], "utf8"))) { expect(failures).toEqual([]) expect(reads).toBeGreaterThan(10) let expected = emptySummary(workId) - for (const batch of batches) expected = await versionOneUpdate(expected, batch) + for (const batch of batches) expected = await mergeInMemory(expected, batch) const head = expectCommitted(workId, expected) expect(COLLECTIONS.some((collection) => (head.logs[collection]?.generation ?? 0) > 1)).toBe(true) }, 60_000) @@ -637,7 +596,7 @@ for (const batch of JSON.parse(readFileSync(process.argv[2], "utf8"))) { const workId = randomUUID() const batches = randomRecords(9, workId, 300, { large: true }) await apply(batches[0]) - let expected = await versionOneUpdate(emptySummary(workId), batches[0]) + let expected = await mergeInMemory(emptySummary(workId), batches[0]) // A second module instance has its own cached positions, like another Kimchi process. vi.resetModules() const other = await import("./summary.js") @@ -645,7 +604,7 @@ for (const batch of JSON.parse(readFileSync(process.argv[2], "utf8"))) { for (const batch of batches.slice(1, -1)) { for (const record of structuredClone(batch)) other.updateWorkSummary(record) await other.flushWorkSummaries() - expected = await versionOneUpdate(expected, batch) + expected = await mergeInMemory(expected, batch) } const changed = manifest(workId) expect( @@ -653,7 +612,7 @@ for (const batch of JSON.parse(readFileSync(process.argv[2], "utf8"))) { ).toBe(true) const last = batches.at(-1) ?? [] await apply(last) - expectCommitted(workId, await versionOneUpdate(expected, last)) + expectCommitted(workId, await mergeInMemory(expected, last)) }, 60_000) }) @@ -723,7 +682,7 @@ describe("a large work", () => { sessionId: sessions[0], recordedAt: iso(BASE + index * 3_600_000), })) - writeVersionOne(summary) + await saveWorkSummary(dir, summary) const record = (index: number): WorkRecord => ({ version: 1, type: "request", diff --git a/src/extensions/work-attribution/row-log.ts b/src/extensions/work-attribution/row-log.ts index 610554bff..39f4902f8 100644 --- a/src/extensions/work-attribution/row-log.ts +++ b/src/extensions/work-attribution/row-log.ts @@ -1,4 +1,4 @@ -import { closeSync, openSync, readFileSync, readSync } from "node:fs" +import { closeSync, openSync, readFileSync, readSync, type Stats } from "node:fs" import { type FileHandle, mkdir, open, readdir, readFile, rm, stat } from "node:fs/promises" import { join } from "node:path" import { setImmediate } from "node:timers/promises" @@ -7,17 +7,16 @@ import { mergePullRequestLinks } from "../pull-request-status/links.js" /* * Work summary storage. Version 2 `work//work.json` is a small manifest. Each collection's rows live in - * `rows/..jsonl`, one version 1 row per line; a later line with the same key replaces an - * earlier one, and the first line of a key fixes its position. Sessions are a log too, one `{"sessionId"}` row each. - * A writer appends changed rows, syncs them, then renames the manifest, which commits them. This module uses plain - * `fs` so tests and tools can read summaries. + * `rows/..jsonl`, one row per line; a later line with the same key replaces an earlier one, + * and the first line of a key fixes its position. Sessions are a log too, one `{"sessionId"}` row each. A writer + * appends changed rows, syncs them, then renames the manifest, which commits them. Any other `work.json`, such as a + * version 1 summary from an older release, is deleted when the work is next listed or updated. This module uses + * plain `fs` so tests and tools can read summaries. */ /** The manifest, and the commit point for every row log. */ export const WORK_FILE = "work.json" const ROWS = "rows" -/** Version 1 summaries larger than this are not read just to list a work. */ -export const MAX_SUMMARY_BYTES = 8 * 1024 * 1024 /** Appends and copies are written in chunks of at most this size. */ const CHUNK_BYTES = 4 * 1024 * 1024 /** A log is compacted once superseded lines exceed half its live rows and this many bytes. */ @@ -26,13 +25,13 @@ const COMPACT_GARBAGE_BYTES = 1024 * 1024 const MAX_INDEXED_ROWS = 200_000 const YIELD_ROWS = 1000 -/** One row of a summary collection, as version 1 `work.json` stores it. */ +/** One row of a summary collection. */ export interface Row { sessionId: string [key: string]: unknown } -/** Summary collections, named as in version 1 `work.json`. Each one is a row log in version 2. */ +/** Summary collections; each one is a row log. */ export const COLLECTIONS = [ "workLinks", "requests", @@ -76,11 +75,10 @@ export const ROW_KEYS: Record string> = { /** `rows/..jsonl`. Bytes past `bytes` belong to an interrupted append and are ignored. */ export interface RowLog { - /** Positive for a log file. `readWorkHead` reports rows still inside a version 1 `work.json` as generation 0. */ generation: number /** Committed length. */ bytes: number - /** Distinct keys: the length of the version 1 array. */ + /** Distinct keys. */ rows: number } @@ -110,8 +108,8 @@ export interface WorkHead { pullRequests: Record[] } -/** The version 1 `work.json` shape, which folding the row logs returns. */ -export type WorkSummaryView = { version: 1; workId: string; sessions: string[] } & Record +/** Every row of a work, as folding its logs returns them. */ +export type WorkSummaryView = { workId: string; sessions: string[] } & Record /** Validates one committed row; a failure means the logs are damaged and the work is rebuilt from its journals. */ export type RowCheck = (collection: Collection, row: Row) => boolean @@ -135,11 +133,6 @@ function isRow(value: unknown): value is Row { return object(value) && typeof value.sessionId === "string" } -/** Version 1 readers were lenient: any object in a collection is a row. The writer validates them. */ -function savedRow(value: unknown): value is Row { - return object(value) -} - function text(value: unknown): string | undefined { return typeof value === "string" ? value : undefined } @@ -169,7 +162,6 @@ export function isWorkHead(value: unknown, workId: string): value is WorkHead { function summaryView(workId: string, sessions: string[], rows: (collection: Collection) => Row[]): WorkSummaryView { return { - version: 1, workId, sessions, workLinks: rows("workLinks"), @@ -182,16 +174,6 @@ function summaryView(workId: string, sessions: string[], rows: (collection: Coll } } -/** A version 1 summary read as leniently as `/work` always read it; older ones lack later collections. */ -function versionOneView(value: unknown, workId: string): WorkSummaryView | undefined { - if (!object(value) || value.version !== 1 || value.workId !== workId) return undefined - const sessions = Array.isArray(value.sessions) ? value.sessions.filter((session) => typeof session === "string") : [] - return summaryView(workId, sessions, (collection) => { - const rows = value[collection] - return Array.isArray(rows) ? rows.filter(savedRow) : [] - }) -} - function parseRow(line: string, collection: LogName): Row { let row: unknown try { @@ -236,7 +218,6 @@ function readLogSync(folder: string, collection: LogName, generation: number, by function readCommitted( agentDir: string, workId: string, - fromVersionOne: (view: WorkSummaryView) => T, read: (folder: string, head: WorkHead) => T, ): T | undefined { const folder = join(agentDir, "work", workId) @@ -248,8 +229,6 @@ function readCommitted( return undefined } - const view = versionOneView(value, workId) - if (view) return fromVersionOne(view) if (!isWorkHead(value, workId)) return undefined try { return read(folder, value) @@ -266,63 +245,52 @@ function foldCommitted(folder: string, head: WorkHead, collection: LogName): Row return log ? foldLog(collection, readLogSync(folder, collection, log.generation, log.bytes)) : [] } -/** The whole summary in the version 1 shape, from either version; undefined when missing or damaged. */ +/** The whole summary; undefined when missing or damaged. */ export function readWorkSummary(agentDir: string, workId: string): WorkSummaryView | undefined { - return readCommitted( - agentDir, - workId, - (view) => view, - (folder, head) => - summaryView( - workId, - head.sessions ?? foldCommitted(folder, head, "sessions").map((row) => row.sessionId), - (collection) => foldCommitted(folder, head, collection), - ), + return readCommitted(agentDir, workId, (folder, head) => + summaryView( + workId, + head.sessions ?? foldCommitted(folder, head, "sessions").map((row) => row.sessionId), + (collection) => foldCommitted(folder, head, collection), + ), ) } -/** One collection's rows from either version, without reading the other logs. */ +/** One collection's rows, without reading the other logs. */ export function readWorkRows(agentDir: string, workId: string, collection: Collection): Row[] | undefined { - return readCommitted( - agentDir, - workId, - (view) => view[collection], - (folder, head) => foldCommitted(folder, head, collection), - ) + return readCommitted(agentDir, workId, (folder, head) => foldCommitted(folder, head, collection)) } /** - * The manifest, or the same values computed from a version 1 summary small enough to read. Opens no row log. - * "too-large" is a version 1 summary over `maxBytes`; its next update migrates it. + * The work's manifest. Any other `work.json`, such as a version 1 summary or a damaged file, is deleted: the work + * then has no summary until its next update rebuilds one from the journals. */ - -export async function readWorkHead( - agentDir: string, - workId: string, - maxBytes = MAX_SUMMARY_BYTES, -): Promise { - const path = join(agentDir, "work", workId, WORK_FILE) - let value: unknown - let modifiedAt: number +async function readManifest(folder: string, workId: string): Promise { + const path = join(folder, WORK_FILE) + let read: Stats + let text: string try { - const info = await stat(path) - if (!info.isFile()) return undefined - if (info.size > maxBytes) return "too-large" - modifiedAt = info.mtimeMs - value = JSON.parse(await readFile(path, "utf8")) - } catch { - return undefined + read = await stat(path) + text = await readFile(path, "utf8") + } catch (error) { + if (code(error) === "ENOENT") return undefined + throw error } - return isWorkHead(value, workId) ? value : summaryHead(value, workId, new Date(modifiedAt).toISOString()) + let value: unknown + try { + value = JSON.parse(text) + } catch {} + if (isWorkHead(value, workId)) return value + // `/work` reads without the lock: remove only the file read here, never a manifest committed since. A later read + // retries a removal that fails. + const current = await stat(path).catch(() => undefined) + if (current?.ino === read.ino && current.mtimeMs === read.mtimeMs) await rm(path, { force: true }).catch(() => {}) + return undefined } -/** The head of a version 1 summary, computed from its rows; they stay inside `work.json` (generation 0). */ -export function summaryHead(value: unknown, workId: string, updatedAt: string): WorkHead | undefined { - const view = versionOneView(value, workId) - if (!view) return undefined - const logs: Partial> = {} - for (const name of LOGS) if (view[name].length) logs[name] = { generation: 0, bytes: 0, rows: view[name].length } - return { version: 2, workId, updatedAt, logs, ...summaryLatest((name) => view[name]) } +/** The manifest alone, without opening a row log; undefined when the work has no summary. */ +export async function readWorkHead(agentDir: string, workId: string): Promise { + return readManifest(join(agentDir, "work", workId), workId).catch(() => undefined) } /** Milliseconds of an ISO 8601 timestamp, as `/work` orders rows; impossible dates are not times. */ @@ -486,7 +454,7 @@ async function scanLog( index.bytes = end } -/** Next unused generation of a log, past any file a crash or another version left behind. */ +/** Next unused generation of a log, past any file a crash or a rebuild left behind. */ async function nextGeneration(folder: string, collection: LogName, current = 0): Promise { let highest = current for (const name of await readdir(join(folder, ROWS)).catch(() => [])) { @@ -496,7 +464,7 @@ async function nextGeneration(folder: string, collection: LogName, current = 0): return highest + 1 } -/** Deletes row files the manifest does not name: leftovers of crashes, compactions and other versions. */ +/** Deletes row files the manifest does not name: leftovers of crashes, compactions and rebuilds. */ async function removeStrays(folder: string, head: WorkHead): Promise { for (const name of await readdir(join(folder, ROWS)).catch(() => [])) { const match = /^(\w+)\.(\d+)\.jsonl$/.exec(name) @@ -629,7 +597,7 @@ function workRows(sessions: SessionStore, saved: string[], store: (collection: C } interface OpenWork { - /** The committed manifest when rows are appended; absent when a version 1 summary migrates or the work is rebuilt. */ + /** The committed manifest when rows are appended; absent when the work is rebuilt. */ head?: WorkHead /** Nothing usable was saved: the whole history must be merged again. */ rebuild: boolean @@ -704,41 +672,25 @@ async function openLogs(folder: string, head: WorkHead, check: RowCheck): Promis } } -async function openWork( - folder: string, - workId: string, - summary: (value: unknown) => WorkSummaryView | undefined, - check: RowCheck, -): Promise { - let value: unknown - try { - value = JSON.parse(await readFile(join(folder, WORK_FILE), "utf8")) - } catch (error) { - if (!(error instanceof SyntaxError) && code(error) !== "ENOENT") throw error - } - +async function openWork(folder: string, workId: string, check: RowCheck): Promise { + const head = await readManifest(folder, workId) let damaged: WorkHead | undefined - if (isWorkHead(value, workId)) { + if (head) { try { - return await openLogs(folder, value, check) + return await openLogs(folder, head, check) } catch (error) { if (!(error instanceof RowLogDamage) && code(error) !== "ENOENT") throw error forget(folder) - damaged = value + damaged = head } } - // A version 1 summary migrates with every row; anything else is rebuilt from the journals. - const saved = damaged ? undefined : summary(value) + // Without a usable manifest, the work is rebuilt from the journals. return { - rebuild: !saved, + rebuild: true, damaged, indexes: {}, - rows: workRows(new SessionStore("sessions"), saved?.sessions ?? [], (collection) => { - const store = new RowStore(collection) - for (const row of saved?.[collection] ?? []) store.set(ROW_KEYS[collection](row), row) - return store - }), + rows: workRows(new SessionStore("sessions"), [], (collection) => new RowStore(collection)), close() {}, } } @@ -759,7 +711,7 @@ async function changedLines(store: RowStore): Promise<[string, string, Row][]> { return changed } -/** Commits changed rows: appended to the current logs, or as new generations on migration and rebuild. */ +/** Commits changed rows: appended to the current logs, or as new generations on a rebuild. */ async function commitWork( folder: string, workId: string, @@ -908,24 +860,18 @@ async function compactLogs( } /** - * One locked update: opens the work's summary in either version, lets `apply` merge records into its rows, then - * commits what changed. A version 1 summary migrates here; a missing, invalid or damaged one is rebuilt, and - * `apply` learns that it must merge the work's whole history. Returns the committed manifest, or undefined - * when nothing changed. + * One locked update: opens the work's summary, lets `apply` merge records into its rows, then commits what changed. + * A missing, invalid or damaged summary is rebuilt, and `apply` learns that it must merge the work's whole history. + * Returns the committed manifest, or undefined when nothing changed. */ export async function updateWorkRows( folder: string, workId: string, - options: { - /** A valid version 1 summary with every collection, or undefined. */ - summary: (value: unknown) => WorkSummaryView | undefined - check: RowCheck - assertLease: () => void - }, + options: { check: RowCheck; assertLease: () => void }, apply: (rows: WorkRows, rebuild: boolean) => Promise, ): Promise { - const work = await openWork(folder, workId, options.summary, options.check) + const work = await openWork(folder, workId, options.check) try { await apply(work.rows, work.rebuild) // Saved rows are loaded; a compaction may delete the file they came from. @@ -941,21 +887,12 @@ export async function updateWorkRows( } } -/** Cheap validity for launch recovery: a version 1 summary, or a manifest whose logs hold their committed bytes. */ -export async function validWorkFiles( - folder: string, - workId: string, - summary: (value: unknown) => boolean, -): Promise { - let value: unknown - try { - value = JSON.parse(await readFile(join(folder, WORK_FILE), "utf8")) - } catch { - return false - } - if (!isWorkHead(value, workId)) return summary(value) +/** Cheap validity for launch recovery: a manifest whose logs hold their committed bytes. */ +export async function validWorkFiles(folder: string, workId: string): Promise { + const head = await readManifest(folder, workId).catch(() => undefined) + if (!head) return false for (const collection of LOGS) { - const log = value.logs[collection] + const log = head.logs[collection] if (!log) continue const info = await stat(logPath(folder, collection, log.generation)).catch(() => undefined) if (!info || info.size < log.bytes) return false diff --git a/src/extensions/work-attribution/summary.test.ts b/src/extensions/work-attribution/summary.test.ts index 1df571f24..9067ef220 100644 --- a/src/extensions/work-attribution/summary.test.ts +++ b/src/extensions/work-attribution/summary.test.ts @@ -14,7 +14,7 @@ import { lockSync } from "proper-lockfile" import { afterEach, beforeEach, describe, expect, it, vi } from "vitest" import { createContext } from "../__mocks__/context.js" import { createExtensionApi } from "../__mocks__/extension-api.js" -import { savedWorkSummary } from "../__mocks__/work-summary.js" +import { savedWorkSummary, saveWorkSummary } from "../__mocks__/work-summary.js" import { appendWorkRecord, createWorkAttributionExtension, @@ -457,7 +457,7 @@ describe("readable work summaries", () => { recordProviderRequest(parent) await flushWorkSummaries() const value = summary(workId) - expect(value).toMatchObject({ version: 1, workId, sessions: expect.arrayContaining(["parent", "child"]) }) + expect(value).toMatchObject({ workId, sessions: expect.arrayContaining(["parent", "child"]) }) expect(value.requests).toHaveLength(2) expect(value.requests).toEqual( expect.arrayContaining([ @@ -549,9 +549,7 @@ describe("readable work summaries", () => { const ctx = context() const request = recordProviderRequest(ctx) await flushWorkSummaries() - // A version 1 summary written before file transitions were recorded. - const { fileTransitions: _, ...existing } = summary(request.workId) - fs.writeFileSync(path(request.workId), JSON.stringify(existing)) + const existing = summary(request.workId) const journal = join(dir, "work-attribution", "transitions", "old.jsonl") fs.mkdirSync(dirname(journal), { recursive: true }) fs.writeFileSync( @@ -602,9 +600,11 @@ describe("readable work summaries", () => { complete: false, }) await flushWorkSummaries() + // A summary saved by a build that did not merge observations or hunk evidence. const { fileObservations: _, ...old } = summary(workId) old.commits[0].fileMatches = [] - fs.writeFileSync(path(workId), JSON.stringify(old)) + fs.rmSync(dirname(path(workId)), { recursive: true }) + await saveWorkSummary(dir, old) const { size, mtimeMs } = fs.statSync(path(workId)) const ledger = join(dir, "work-attribution", "parent.jsonl") const past = new Date(Date.now() - 60 * 60 * 1000) @@ -931,11 +931,7 @@ console.log("durable"); await flushWorkSummaries();`, it("serializes simultaneous processes without lost updates or truncated publication", async () => { const workId = randomUUID() const directory = dirname(path(workId)) - fs.mkdirSync(directory, { recursive: true }) - fs.writeFileSync( - path(workId), - JSON.stringify({ version: 1, workId, sessions: [], requests: [], plans: [], commits: [] }), - ) + await saveWorkSummary(dir, { workId, sessions: [] }) const release = lockSync(directory) const script = join(dir, "writer.mts") fs.writeFileSync( @@ -986,7 +982,7 @@ console.log("ready"); await flushWorkSummaries();`, expect(errors).toEqual([]) expect(summary(workId).requests).toHaveLength(24) expect(summary(workId).sessions.sort()).toEqual(["session-0", "session-1", "session-2"]) - // The version 1 file migrated once; every append landed on a committed line boundary. + // Every append landed on a committed line boundary. const head = JSON.parse(fs.readFileSync(path(workId), "utf8")) const log = fs.readFileSync(join(directory, "rows", `requests.${head.logs.requests.generation}.jsonl`)) expect(head).toMatchObject({ diff --git a/src/extensions/work-attribution/summary.ts b/src/extensions/work-attribution/summary.ts index 5c91f1efd..48d4f1519 100644 --- a/src/extensions/work-attribution/summary.ts +++ b/src/extensions/work-attribution/summary.ts @@ -7,14 +7,7 @@ import { lock } from "proper-lockfile" import { isWorkId } from "../../shared/work-id.js" import { mergePullRequestLinks } from "../pull-request-status/links.js" import { debugWorkAttribution } from "./diagnostics.js" -import { - COLLECTIONS, - type Collection, - ROW_KEYS, - updateWorkRows, - validWorkFiles, - type WorkSummaryView, -} from "./row-log.js" +import { type Collection, ROW_KEYS, updateWorkRows, validWorkFiles } from "./row-log.js" const LOCK_STALE_MS = 5000 const MERGE_BATCH_SIZE = 1000 @@ -122,7 +115,7 @@ function unknownType(value: unknown): value is Record { !KNOWN_RECORD_TYPES.has(value.type) ) } -/** Field checks for one row of each collection, in version 1 `work.json` and in the row logs. */ +/** Field checks for one committed row of each collection's log. */ const ROW_CHECKS: Record boolean> = { workLinks: (row) => entry(row, ["linkId", "sourceWorkId", "targetWorkId"]), requests: (row) => entry(row, ["requestId"]), @@ -133,35 +126,6 @@ const ROW_CHECKS: Record boolean> = { entry(row, ["observationId", "toolCallId", "repository", "worktree", "source"]) && Array.isArray(row.files), continuations: (row) => entry(row, ["source"]) && object(row.evidence), } -/** Collections that summaries written before them lack. */ -const LATER_COLLECTIONS = new Set(["workLinks", "fileTransitions", "fileObservations", "continuations"]) -function validSummary(value: unknown, workId: string): value is WorkSummaryView { - return ( - object(value) && - value.version === 1 && - value.workId === workId && - Array.isArray(value.sessions) && - value.sessions.every((session) => typeof session === "string") && - COLLECTIONS.every((collection) => { - const rows = value[collection] - return ( - (rows === undefined && LATER_COLLECTIONS.has(collection)) || - (Array.isArray(rows) && rows.every(ROW_CHECKS[collection])) - ) - }) - ) -} -/** A valid version 1 summary, with collections it predates as empty ones. */ -function versionOneSummary(value: unknown, workId: string): WorkSummaryView | undefined { - if (!validSummary(value, workId)) return undefined - return { - ...value, - workLinks: value.workLinks ?? [], - fileTransitions: value.fileTransitions ?? [], - fileObservations: value.fileObservations ?? [], - continuations: value.continuations ?? [], - } -} /** An interrupted append leaves a record cut off mid-value; any other unparseable line is damage. */ function truncated(line: string): boolean { const closing: string[] = [] @@ -468,11 +432,7 @@ async function update( const head = await updateWorkRows( join(agentDir, "work", workId), workId, - { - summary: (value) => versionOneSummary(value, workId), - check: (collection, row) => ROW_CHECKS[collection](row), - assertLease, - }, + { check: (collection, row) => ROW_CHECKS[collection](row), assertLease }, async (rows, rebuild) => { const history = rebuild && !complete ? readWorkRecords(agentDir).filter((row) => row.workId === workId) : [] await mergeWorkRecords(rows, history.concat(records)) @@ -567,8 +527,7 @@ async function readRecovery(agentDir: string, stamp: string) { if (!existsSync(path)) return undefined const current = summaryFingerprint(path) // Missing or damaged output requires all source ledgers, even when none changed. - if (current !== fingerprint && !(await validWorkFiles(directory, workId, (value) => validSummary(value, workId)))) - return undefined + if (current !== fingerprint && !(await validWorkFiles(directory, workId))) return undefined summaries[workId] = current } return { startedAt: saved.startedAt, summaries } diff --git a/tests/e2e/tui/support/work-summary.ts b/tests/e2e/tui/support/work-summary.ts index 99c848fe9..3e6fff2b4 100644 --- a/tests/e2e/tui/support/work-summary.ts +++ b/tests/e2e/tui/support/work-summary.ts @@ -1,8 +1,7 @@ /** * Reads a work's local summary the way docs/work-attribution.md describes it, independently of the product reader. - * Version 1 `work.json` holds every row. Version 2 `work.json` is a manifest whose `logs` name - * `rows/..jsonl`, sessions included; only the first `bytes` of each log are committed, and a - * later row with the same key replaces an earlier one in place. + * `work.json` is a manifest whose `logs` name `rows/..jsonl`, sessions included; only the + * first `bytes` of each log are committed, and a later row with the same key replaces an earlier one in place. */ import { readFileSync } from "node:fs" import { join } from "node:path" @@ -30,13 +29,13 @@ const KEYS: Record string> = { sessions: (row) => JSON.stringify(row.sessionId), } -/** The summary in the version 1 shape, or undefined while it is missing or a writer is replacing a log. */ +/** Every row of the summary, or undefined while it is missing or a writer is replacing a log. */ export function readWorkSummary(agentDir: string, workId: string): Record | undefined { const folder = join(agentDir, "work", workId) try { const manifest = JSON.parse(readFileSync(join(folder, "work.json"), "utf8")) - if (manifest.version !== 2) return manifest - const view: Record = { version: 1, workId: manifest.workId } + if (manifest.version !== 2) return undefined + const view: Record = { workId: manifest.workId } for (const [collection, key] of Object.entries(KEYS)) { const log = manifest.logs[collection] const rows = new Map() diff --git a/tests/e2e/tui/work-attribution.test.ts b/tests/e2e/tui/work-attribution.test.ts index 2720a7089..3c33f12d8 100644 --- a/tests/e2e/tui/work-attribution.test.ts +++ b/tests/e2e/tui/work-attribution.test.ts @@ -288,88 +288,6 @@ test("the next message recovers account tracking after its saved scope is lost", ) }) -test("/work keeps listing an oversized summary from an older build with its PR and spend so far", async ({ - terminal, -}) => { - const workId = randomUUID() - const url = "https://github.com/example/kimchi/pull/7" - await runKimchiSession( - terminal, - { - artifactName: "work-browser-oversized-summary", - account, - gitInit: true, - models, - responses: [], - seedHome(home) { - // An older build kept every row in work.json; this one is larger than the 8 MiB /work reads. - const folder = join(home, ".config/kimchi/harness/work", workId) - mkdirSync(folder, { recursive: true }) - const requests = Array.from({ length: 3600 }, (_, index) => ({ - requestId: String(index), - sessionId: "older", - billingRows: [{ id: String(index), costUsd: "0.001", note: "x".repeat(2500) }], - })) - const pullRequest = { provider: "github", host: "github.com", number: 7, url, state: "open" } - const commit = { - sha: "a".repeat(40), - repository: "/src/kimchi/.git", - worktree: "/src/kimchi", - sessionId: "older", - } - writeFileSync( - join(folder, "work.json"), - JSON.stringify({ - version: 1, - workId, - sessions: ["older"], - requests, - plans: [], - commits: [{ ...commit, pullRequests: [pullRequest] }], - }), - ) - // The cost pass's bounded totals still name the work's PR and its spend so far. - const spend = { total: 3600, priced: 3600, knownCostUsd: "3.600000000" } - writeFileSync( - join(folder, "cost-totals.json"), - JSON.stringify({ - version: 1, - workId, - group: [workId], - groupRequests: spend, - requests: { ...spend, recorded: 3600, unresolved: 0, inferred: 0, shared: 0 }, - pullRequests: [ - { - key: '["github","github.com","7"]', - account: null, - own: true, - pullRequest: { provider: "github", number: 7, state: "open", url }, - totalCostUsd: null, - knownCostUsd: "0.000000000", - soFar: { knownCostUsd: "3.600000000", complete: false }, - }, - ], - report: "costs.json", - }), - ) - }, - }, - async (_fixture, trace) => { - terminal.submit("/work") - await waitForText(terminal, "summary too large", { full: false }) - const listed = viewText(terminal) - .split("\n") - .find((line) => line.includes(workId.slice(0, 8))) - expect(listed).toContain("summary too large") - // Requests newer than the last cost pass may be missing, so the amount stays partial. - expect(listed).toContain("$3.60 known so far · PR #7 open") - trace.step("the list keeps an oversized older summary's PR and spend so far until its next update converts it") - terminal.keyEscape() - await waitForText(terminal, PROMPT_READY, { full: false }) - }, - ) -}) - test("the user can correct one earlier planning turn and revoke that correction", async ({ terminal }) => { await runKimchiSession( terminal, From 456c3bf949e42ab6f0cb4ddb136ace08d07ced2a Mon Sep 17 00:00:00 2001 From: tautvydasLiekis Date: Mon, 12 Oct 2026 10:22:00 +0300 Subject: [PATCH 19/19] test: expect work summaries without a version --- tests/e2e/tui/work-attribution.test.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/e2e/tui/work-attribution.test.ts b/tests/e2e/tui/work-attribution.test.ts index 3c33f12d8..21b805126 100644 --- a/tests/e2e/tui/work-attribution.test.ts +++ b/tests/e2e/tui/work-attribution.test.ts @@ -482,7 +482,6 @@ test("work and request IDs are durable before the first reply, and commits belon } const summary = await waitForSummary(fixture.agentDir, started[0].workId, { requests: sent.length, commits: 1 }) expect(summary).toMatchObject({ - version: 1, workId: started[0].workId, sessions: [started[0].sessionId], plans: [],