Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions docs/work-attribution-accuracy.md
Original file line number Diff line number Diff line change
Expand Up @@ -8,11 +8,11 @@ pnpm exec tsx src/extensions/work-attribution/accuracy-cli.ts report.json refere

The command reads these two files and prints the differences. It makes no network or model calls, reads no credentials and writes nothing. Keep real receipts and labels outside the repository.

By default, Kimchi saves the report at `~/.config/kimchi/harness/work/<workId>/costs.json`. It contains request rows, PR totals and all six unallocated buckets for that work and every work connected to it. Works are connected when they contribute to the same PR or when a work link, such as a `/work link` correction, moves requests between them. Connections are transitive: if work A shares a PR with work B, and B shares another PR with work C, A's report also includes C's requests and PR totals. Prepare reference entries for every request in that file. Do not add per-work reports together: connected works repeat the same requests.
By default, Kimchi saves the report at `~/.config/kimchi/harness/work/<workId>/costs.json`. It contains request rows, PR totals and all six unallocated buckets for that work and every work connected to it. Kimchi refreshes it at most every five minutes per work, and at once when it is missing, so a report saved right after a price arrived can lag `/work` by a few minutes. Works are connected when they contribute to the same PR or when a work link, such as a `/work link` correction, moves requests between them. Connections are transitive: if work A shares a PR with work B, and B shares another PR with work C, A's report also includes C's requests and PR totals. Prepare reference entries for every request in that file. Do not add per-work reports together: connected works repeat the same requests.

## Prepare the reference before inspecting matcher output

Use provider or gateway receipts to list requests, their billing accounts and exact billed prices. Label which PR each request belongs to using the task history and Git evidence. Do not generate this reference from `work.json`, `costs.json`, matcher confidence or the allocator under test: that would repeat the same mistakes on both sides.
Use provider or gateway receipts to list requests, their billing accounts and exact billed prices. Label which PR each request belongs to using the task history and Git evidence. Do not generate this reference from the work summary, `costs.json`, matcher confidence or the allocator under test: that would repeat the same mistakes on both sides.

The version 1 reference has one entry per request:

Expand Down
66 changes: 47 additions & 19 deletions docs/work-attribution.md

Large diffs are not rendered by default.

8 changes: 8 additions & 0 deletions src/extensions/__mocks__/work-summary.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
import { readWorkSummary, type WorkSummaryView } from "../work-attribution/row-log.js"

/** A work's saved summary in the version 1 shape; fails the test when it is missing or damaged. */
export function savedWorkSummary(agentDir: string, workId: string): WorkSummaryView {
const summary = readWorkSummary(agentDir, workId)
if (!summary) throw new Error(`No readable work summary for ${workId}`)
return summary
}
13 changes: 7 additions & 6 deletions src/extensions/work-attribution.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ import { createCommandContext, createContext } from "./__mocks__/context.js"
import { createExtensionApi } from "./__mocks__/extension-api.js"
import { testTheme } from "./__mocks__/theme.js"
import { createWorkScopeSnapshot } from "./__mocks__/work-scope.js"
import { savedWorkSummary } from "./__mocks__/work-summary.js"
import requestTimingExtension from "./request-timing.js"
import { copySessionFileAndAddHandoffNote, removeTempDir } from "./teleport/provisioning/handoff-note.js"
import * as billingSource from "./work-attribution/billing-source.js"
Expand Down Expand Up @@ -207,11 +208,11 @@ describe("local work attribution", () => {
expect(getWorkId(ctx)).toBe(target.workId)
await flushWorkSummaries()
const path = join(dir, "work", target.workId, "work.json")
expect(JSON.parse(readFileSync(path, "utf8")).workLinks).toHaveLength(1)
expect(savedWorkSummary(dir, target.workId).workLinks).toHaveLength(1)
writeFileSync(path, "damaged")
recoverWorkSummaries()
await flushWorkSummaries()
expect(JSON.parse(readFileSync(path, "utf8")).workLinks).toHaveLength(1)
expect(savedWorkSummary(dir, target.workId).workLinks).toHaveLength(1)
await command.handler(`unlink ${links[0].linkId}`, ctx)
expect(records().filter((row) => row.type === "work_link")).toEqual([
links[0],
Expand Down Expand Up @@ -459,11 +460,11 @@ describe("local work attribution", () => {
requestId: calls[1].requestId,
})
await flushWorkSummaries()
const requests = JSON.parse(readFileSync(join(dir, "work", getWorkId(ctx), "work.json"), "utf8")).requests
const { requests } = savedWorkSummary(dir, getWorkId(ctx))
expect(requests).toHaveLength(2)
expect(records().filter((row) => row.type === "request")).toHaveLength(2)
expect(records().filter((row) => row.type === "request_dispatch")).toHaveLength(2)
expect(requests[0].billingSelector.tag).toBe(calls[0].tag)
expect(requests[0].billingSelector).toMatchObject({ tag: calls[0].tag })
expect(requests[1]).toMatchObject({
requestId: calls[1].requestId,
parentRequestId: logicalId,
Expand Down Expand Up @@ -602,7 +603,7 @@ describe("local work attribution", () => {
})
recordProviderResponse(attempt.requestId, { status: 500, headers: new Headers() })
await flushWorkSummaries()
const summary = JSON.parse(readFileSync(join(dir, "work", attempt.workId, "work.json"), "utf8"))
const summary = savedWorkSummary(dir, attempt.workId)
expect(summary.requests).toHaveLength(1)
expect(summary.requests[0]).toMatchObject({
requestId: attempt.requestId,
Expand All @@ -625,7 +626,7 @@ describe("local work attribution", () => {
})
recordProviderResponse("unknown-request", { status: 200, headers: new Headers() })
await flushWorkSummaries()
const summary = JSON.parse(readFileSync(join(dir, "work", attempt.workId, "work.json"), "utf8"))
const summary = savedWorkSummary(dir, attempt.workId)
expect(summary.requests).toHaveLength(1)
expect(summary.requests[0].response).toEqual({ status: 503, receivedAt: expect.any(String) })
})
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
{
"version": 1,
"workId": "8b304360-39a9-4d6e-8ac7-984c97c0e315",
"sessions": ["01a10dc0-7ea4-74d8-9b87-9a5026c7b05c"],
"requests": [
{
"requestId": "00000001-1111-4111-8111-111111111111",
"sessionId": "01a10dc0-7ea4-74d8-9b87-9a5026c7b05c",
"cwd": "/src/example",
"provider": "kimchi-dev",
"model": "glm-5.3",
"modelSource": "context",
"recordedAt": "2026-09-01T10:00:00.000Z"
},
{
"requestId": "00000002-1111-4111-8111-111111111111",
"sessionId": "01a10dc0-7ea4-74d8-9b87-9a5026c7b05c",
"cwd": "/src/example",
"provider": "kimchi-dev",
"model": "glm-5.3",
"modelSource": "context",
"recordedAt": "2026-09-02T10:00:00.000Z"
}
],
"plans": [],
"commits": [
{
"sha": "cccccccccccccccccccccccccccccccccccccccc",
"repository": "/src/example/.git",
"worktree": "/src/example",
"sessionId": "01a10dc0-7ea4-74d8-9b87-9a5026c7b05c",
"cwd": "/src/example",
"recordedAt": "2026-08-02T11:00:00.000Z",
"paths": ["a.ts"]
}
]
}
Loading
Loading