Skip to content

Commit c305788

Browse files
authored
diagnostics(query): trace interruption causality (#2111)
* diagnostics(issue-1830): trace interruption causality * test(issue-1830): lock interruption ownership matrix * fix(codex): preserve stream deadline contract * fix(diagnostics): harden interruption trace lifecycle Refs #1830 * fix(diagnostics): harden interruption trace settlement Refs #1830 * fix(diagnostics): preserve interruption causality * fix(diagnostics): address interruption trace review * fix(diagnostics): preserve tracing observer contracts * fix(diagnostics): preserve interruption trace contracts * test(permissions): cover interactive hook interrupts
1 parent ea65516 commit c305788

91 files changed

Lines changed: 7545 additions & 389 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

docs/advanced-setup.md

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -481,6 +481,8 @@ host. Without this variable the behavior is unchanged.
481481
| `OPENCLAUDE_MAX_TURNS` | No | Per-prompt **local** interactive REPL turn cap for the in-process query loop. Defaults to `50`. Set a larger positive integer for long autonomous local interactive sessions (for example models that take many small tool steps). CLI `--max-turns 0` explicitly disables this cap and prints a cautionary warning. Precedence for a valid override: CLI `--max-turns` → this env var → legacy `CLAUDE_CODE_MAX_TURNS` (only when this var is unset/empty) → `/config` → Max turns (interactive) → `50`. If this env var is set but invalid (zero, negative, non-integer), the default `50` is used and lower layers are not consulted — same pattern as `OPENCLAUDE_MAX_RETRIES`. Does not apply to remote-backed interactive sessions (`connect` / `ssh` / `--remote`). |
482482
| `OPENCLAUDE_RETRY_DELAY_MS` | No | Base retry delay in milliseconds for APIs that do not send `Retry-After`; exponential backoff starts from this value, capped at 60000 (default: 500) |
483483
| `OPENCLAUDE_QUERY_HARD_MAX_MS` | No | Foreground query hard maximum in milliseconds. Defaults to 1800000 (30 minutes). Use a larger positive integer for long autonomous sessions; invalid, zero, negative, fractional, or timer-overflow values are ignored with a warning. |
484+
| `OPENCLAUDE_INTERRUPT_TRACE` | No | Set to `1` or `true` to retain a bounded, privacy-safe interruption lifecycle trace in memory. Disabled by default. The trace contains only allowlisted lifecycle metadata—never prompts, responses, tool arguments, credentials, or raw error messages. |
485+
| `OPENCLAUDE_INTERRUPT_TRACE_FILE` | No | Optional absolute JSONL output path used only when `OPENCLAUDE_INTERRUPT_TRACE` is enabled. On Linux, missing parent directories are created privately and every parent is opened through `/proc/self/fd` without following symbolic links before the final regular file is appended. If the file already exists, its mode is reset to `0600` on every append, so do not configure a shared file. Other platforms retain the bounded trace in memory but do not write this file because Node does not expose an equivalent safe descriptor-relative traversal API there. Writes are best-effort and never change request behavior. Use a separate path per OpenClaude process and keep the resulting diagnostic file private. |
484486
| `OPENCLAUDE_DISABLE_CO_AUTHORED_BY` | No | Suppress the default `Co-Authored-By` trailer in generated git commits |
485487
| `OPENCLAUDE_LOG_TOKEN_USAGE` | No | When truthy (e.g. `verbose`), emits one JSON line on stderr per API request with input/output/cache tokens and the resolved provider. **User-facing debug output** — complements the REPL display controlled by `/config showCacheStats`. Distinct from `CLAUDE_CODE_ENABLE_TOKEN_USAGE_ATTACHMENT`, which is **model-facing** (injects context usage info into the prompt itself). Both can run together. |
486488

Lines changed: 183 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,183 @@
1+
import { afterEach, beforeEach, describe, expect, test } from 'bun:test'
2+
import {
3+
acquireSharedMutationLock,
4+
releaseSharedMutationLock,
5+
} from './test/sharedMutationLock.js'
6+
import { QueryEngine } from './QueryEngine.js'
7+
import {
8+
__getInterruptionTraceSnapshotForTests,
9+
__resetInterruptionTraceForTests,
10+
__waitForInterruptionTraceFlushForTests,
11+
registerInterruptionController,
12+
} from './utils/interruptionTrace.js'
13+
14+
const originalTrace = process.env.OPENCLAUDE_INTERRUPT_TRACE
15+
16+
beforeEach(async () => {
17+
await acquireSharedMutationLock('QueryEngine.interruptionTrace.test.ts')
18+
})
19+
20+
afterEach(async () => {
21+
try {
22+
await __waitForInterruptionTraceFlushForTests()
23+
__resetInterruptionTraceForTests()
24+
if (originalTrace === undefined) delete process.env.OPENCLAUDE_INTERRUPT_TRACE
25+
else process.env.OPENCLAUDE_INTERRUPT_TRACE = originalTrace
26+
} finally {
27+
releaseSharedMutationLock()
28+
}
29+
})
30+
31+
describe('QueryEngine interruption tracing', () => {
32+
test('does not record lifecycle entries while tracing is disabled', async () => {
33+
delete process.env.OPENCLAUDE_INTERRUPT_TRACE
34+
const engine = Object.create(QueryEngine.prototype) as QueryEngine
35+
const controller = new AbortController()
36+
;(engine as unknown as { abortController: AbortController }).abortController =
37+
controller
38+
;(engine as unknown as {
39+
submitMessageImpl(): AsyncGenerator<never, void, unknown>
40+
}).submitMessageImpl = async function* () {}
41+
42+
for await (const _message of engine.submitMessage('hello')) {
43+
// The stub deliberately yields nothing.
44+
}
45+
46+
expect(__getInterruptionTraceSnapshotForTests()).toEqual([])
47+
})
48+
49+
test('records a programmatic query-root interruption before aborting', () => {
50+
process.env.OPENCLAUDE_INTERRUPT_TRACE = '1'
51+
const controller = new AbortController()
52+
const engine = Object.create(QueryEngine.prototype) as QueryEngine
53+
;(engine as unknown as {
54+
abortController: AbortController
55+
}).abortController = controller
56+
57+
engine.interrupt('sdk_interrupt')
58+
59+
const requested = __getInterruptionTraceSnapshotForTests().find(
60+
entry => entry.event === 'abort.requested',
61+
)
62+
expect(controller.signal.aborted).toBe(true)
63+
expect(requested).toMatchObject({
64+
source: 'sdk_interrupt',
65+
subsystem: 'query_engine',
66+
controllerRole: 'query-root',
67+
})
68+
})
69+
70+
test('records start and terminal lifecycle for successful SDK turns', async () => {
71+
process.env.OPENCLAUDE_INTERRUPT_TRACE = '1'
72+
const engine = Object.create(QueryEngine.prototype) as QueryEngine
73+
const controller = new AbortController()
74+
;(engine as unknown as { abortController: AbortController }).abortController =
75+
controller
76+
;(engine as unknown as {
77+
submitMessageImpl(): AsyncGenerator<never, void, unknown>
78+
}).submitMessageImpl = async function* () {}
79+
80+
for await (const _message of engine.submitMessage('hello')) {
81+
// The stub deliberately yields nothing.
82+
}
83+
84+
const trace = __getInterruptionTraceSnapshotForTests()
85+
const started = trace.find(entry => entry.event === 'query.started')
86+
const terminal = trace.find(entry => entry.event === 'query.terminal')
87+
expect(started).toMatchObject({
88+
subsystem: 'query_engine',
89+
querySource: 'sdk',
90+
controllerRole: 'query-root',
91+
})
92+
expect(terminal).toMatchObject({
93+
subsystem: 'query_engine',
94+
queryId: started?.queryId,
95+
outcome: 'completed',
96+
})
97+
expect(typeof started?.eventId).toBe('string')
98+
expect(typeof terminal?.causalEventId).toBe('string')
99+
expect(terminal!.causalEventId).toBe(started!.eventId)
100+
})
101+
102+
test('records aborted and failed SDK turn terminals', async () => {
103+
process.env.OPENCLAUDE_INTERRUPT_TRACE = '1'
104+
105+
for (const scenario of ['aborted', 'failed'] as const) {
106+
__resetInterruptionTraceForTests()
107+
const engine = Object.create(QueryEngine.prototype) as QueryEngine
108+
const controller = new AbortController()
109+
;(engine as unknown as { abortController: AbortController }).abortController =
110+
controller
111+
;(engine as unknown as {
112+
submitMessageImpl(): AsyncGenerator<never, void, unknown>
113+
}).submitMessageImpl = async function* () {
114+
if (scenario === 'aborted') {
115+
controller.abort('interrupt')
116+
return
117+
}
118+
throw new Error('turn failed')
119+
}
120+
121+
const drain = async () => {
122+
for await (const _message of engine.submitMessage('hello')) {
123+
// The stub deliberately yields nothing.
124+
}
125+
}
126+
if (scenario === 'failed') await expect(drain()).rejects.toThrow('turn failed')
127+
else await drain()
128+
129+
const trace = __getInterruptionTraceSnapshotForTests()
130+
const started = trace.find(entry => entry.event === 'query.started')
131+
const terminal = trace.find(entry => entry.event === 'query.terminal')
132+
expect(terminal?.outcome).toBe(scenario)
133+
expect(typeof started?.eventId).toBe('string')
134+
expect(typeof terminal?.eventId).toBe('string')
135+
if (scenario === 'aborted') {
136+
const observed = trace.find(
137+
entry => entry.event === 'signal.observed',
138+
)
139+
expect(typeof observed?.eventId).toBe('string')
140+
expect(terminal?.causalEventId).toBe(observed!.eventId)
141+
} else {
142+
expect(terminal?.causalEventId).toBe(started!.eventId)
143+
}
144+
}
145+
})
146+
147+
test('registers the query root when tracing is enabled at the turn boundary', async () => {
148+
delete process.env.OPENCLAUDE_INTERRUPT_TRACE
149+
const engine = Object.create(QueryEngine.prototype) as QueryEngine
150+
const controller = new AbortController()
151+
;(engine as unknown as { abortController: AbortController }).abortController =
152+
controller
153+
registerInterruptionController(controller, {
154+
subsystem: 'query_engine',
155+
controllerRole: 'query-root',
156+
})
157+
process.env.OPENCLAUDE_INTERRUPT_TRACE = '1'
158+
;(engine as unknown as {
159+
submitMessageImpl(): AsyncGenerator<never, void, unknown>
160+
}).submitMessageImpl = async function* () {
161+
controller.abort()
162+
}
163+
164+
for await (const _message of engine.submitMessage('hello')) {
165+
// The stub deliberately yields nothing.
166+
}
167+
168+
const trace = __getInterruptionTraceSnapshotForTests()
169+
const registered = trace.find(
170+
entry =>
171+
entry.event === 'controller.registered' &&
172+
entry.controllerRole === 'query-root',
173+
)
174+
const observed = trace.find(entry => entry.event === 'signal.observed')
175+
const terminal = trace.find(entry => entry.event === 'query.terminal')
176+
expect(registered).toBeDefined()
177+
expect(typeof observed?.eventId).toBe('string')
178+
expect(terminal).toMatchObject({
179+
outcome: 'aborted',
180+
causalEventId: observed!.eventId,
181+
})
182+
})
183+
})

src/QueryEngine.ts

Lines changed: 68 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,14 @@ import { SYNTHETIC_OUTPUT_TOOL_NAME } from './tools/SyntheticOutputTool/Syntheti
4545
import type { Message } from './types/message.js'
4646
import type { OrphanedPermission } from './types/textInputTypes.js'
4747
import { createAbortController } from './utils/abortController.js'
48+
import {
49+
flushInterruptionTrace,
50+
getInterruptionSignalAbortEventId,
51+
isInterruptionTraceEnabled,
52+
registerInterruptionController,
53+
requestAbort,
54+
traceInterruptionEvent,
55+
} from './utils/interruptionTrace.js'
4856
import { validateArrayOf, assertNonEmptyString, assertObject, assertFunction } from './utils/validation.js'
4957
import { invalidateRemovedToolSchemas } from './utils/toolSchemaCache.js'
5058
import type { AttributionState } from './utils/commitAttribution.js'
@@ -205,6 +213,10 @@ export class QueryEngine {
205213
this.config = config
206214
this.mutableMessages = config.initialMessages ?? []
207215
this.abortController = config.abortController ?? createAbortController()
216+
registerInterruptionController(this.abortController, {
217+
subsystem: 'query_engine',
218+
controllerRole: 'query-root',
219+
})
208220
this.permissionDenials = []
209221
this.readFileState = config.readFileCache
210222
this.totalUsage = EMPTY_USAGE
@@ -213,6 +225,56 @@ export class QueryEngine {
213225
async *submitMessage(
214226
prompt: string | ContentBlockParam[],
215227
options?: { uuid?: string; isMeta?: boolean },
228+
): AsyncGenerator<SDKMessage, void, unknown> {
229+
const queryId = isInterruptionTraceEnabled() ? randomUUID() : undefined
230+
registerInterruptionController(this.abortController, {
231+
subsystem: 'query_engine',
232+
controllerRole: 'query-root',
233+
queryId,
234+
querySource: 'sdk',
235+
}, { refreshQueryContext: true })
236+
const startedEventId = traceInterruptionEvent('query.started', {
237+
subsystem: 'query_engine',
238+
phase: 'running',
239+
queryId,
240+
querySource: 'sdk',
241+
controllerRole: 'query-root',
242+
})
243+
let outcome = 'consumer_closed'
244+
let terminalError: unknown
245+
try {
246+
yield* this.submitMessageImpl(prompt, options)
247+
outcome = this.abortController.signal.aborted ? 'aborted' : 'completed'
248+
} catch (error) {
249+
terminalError = error
250+
outcome = this.abortController.signal.aborted ? 'aborted' : 'failed'
251+
throw error
252+
} finally {
253+
const terminalOutcome = this.abortController.signal.aborted
254+
? 'aborted'
255+
: outcome
256+
traceInterruptionEvent('query.terminal', {
257+
subsystem: 'query_engine',
258+
phase: terminalOutcome,
259+
queryId,
260+
querySource: 'sdk',
261+
controllerRole: 'query-root',
262+
outcome: terminalOutcome,
263+
reason: this.abortController.signal.reason,
264+
error: terminalError,
265+
causalEventId: terminalOutcome === 'aborted'
266+
? getInterruptionSignalAbortEventId(this.abortController.signal)
267+
: startedEventId,
268+
})
269+
if (this.abortController.signal.aborted) {
270+
flushInterruptionTrace('query_terminal')
271+
}
272+
}
273+
}
274+
275+
private async *submitMessageImpl(
276+
prompt: string | ContentBlockParam[],
277+
options?: { uuid?: string; isMeta?: boolean },
216278
): AsyncGenerator<SDKMessage, void, unknown> {
217279
const {
218280
cwd,
@@ -1212,8 +1274,12 @@ export class QueryEngine {
12121274
}
12131275
}
12141276

1215-
interrupt(): void {
1216-
this.abortController.abort()
1277+
interrupt(source = 'programmatic_interrupt'): void {
1278+
requestAbort(this.abortController, undefined, {
1279+
source,
1280+
subsystem: 'query_engine',
1281+
controllerRole: 'query-root',
1282+
})
12171283
}
12181284

12191285
getMessages(): readonly Message[] {
Lines changed: 108 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,108 @@
1+
import { afterEach, beforeEach, describe, expect, test } from 'bun:test'
2+
import {
3+
acquireSharedMutationLock,
4+
releaseSharedMutationLock,
5+
} from '../test/sharedMutationLock.js'
6+
import {
7+
__getInterruptionTraceSnapshotForTests,
8+
__resetInterruptionTraceForTests,
9+
__waitForInterruptionTraceFlushForTests,
10+
} from '../utils/interruptionTrace.js'
11+
import {
12+
abortPrintModeControlRequest,
13+
type PrintModeControlAbortSource,
14+
} from './printInterruption.js'
15+
16+
const originalInterruptionTrace = process.env.OPENCLAUDE_INTERRUPT_TRACE
17+
let hasSharedMutationLock = false
18+
19+
beforeEach(async () => {
20+
await acquireSharedMutationLock('cli/print.interruptionTrace.test.ts')
21+
hasSharedMutationLock = true
22+
})
23+
24+
afterEach(async () => {
25+
try {
26+
await __waitForInterruptionTraceFlushForTests()
27+
__resetInterruptionTraceForTests()
28+
if (originalInterruptionTrace === undefined) {
29+
delete process.env.OPENCLAUDE_INTERRUPT_TRACE
30+
} else {
31+
process.env.OPENCLAUDE_INTERRUPT_TRACE = originalInterruptionTrace
32+
}
33+
} finally {
34+
if (hasSharedMutationLock) {
35+
releaseSharedMutationLock()
36+
hasSharedMutationLock = false
37+
}
38+
}
39+
})
40+
41+
describe('print-mode interruption tracing', () => {
42+
test.each([
43+
['sdk_control_interrupt', 'interrupt'],
44+
['sdk_end_session', undefined],
45+
] as const)(
46+
'links %s input to the query and speculation aborts',
47+
(source: PrintModeControlAbortSource, queryReason: unknown) => {
48+
process.env.OPENCLAUDE_INTERRUPT_TRACE = '1'
49+
__resetInterruptionTraceForTests()
50+
const queryController = new AbortController()
51+
const suggestionController = new AbortController()
52+
53+
const causalEventId = abortPrintModeControlRequest(
54+
queryController,
55+
suggestionController,
56+
source,
57+
queryReason,
58+
)
59+
60+
expect(queryController.signal.aborted).toBe(true)
61+
expect(suggestionController.signal.aborted).toBe(true)
62+
const trace = __getInterruptionTraceSnapshotForTests()
63+
expect(trace.find(entry => entry.eventId === causalEventId)).toMatchObject({
64+
event: `input.${source}`,
65+
source,
66+
subsystem: 'print_mode',
67+
})
68+
expect(
69+
trace.find(
70+
entry =>
71+
entry.event === 'abort.requested' &&
72+
entry.controllerRole === 'query-root',
73+
),
74+
).toMatchObject({ source, causalEventId, subsystem: 'print_mode' })
75+
expect(
76+
trace.find(
77+
entry =>
78+
entry.event === 'abort.requested' &&
79+
entry.controllerRole === 'speculation',
80+
),
81+
).toMatchObject({
82+
source,
83+
causalEventId,
84+
subsystem: 'prompt_suggestion',
85+
})
86+
},
87+
)
88+
89+
test('preserves native abort behavior when tracing is disabled', () => {
90+
delete process.env.OPENCLAUDE_INTERRUPT_TRACE
91+
__resetInterruptionTraceForTests()
92+
const queryController = new AbortController()
93+
const suggestionController = new AbortController()
94+
95+
const causalEventId = abortPrintModeControlRequest(
96+
queryController,
97+
suggestionController,
98+
'sdk_control_interrupt',
99+
'interrupt',
100+
)
101+
102+
expect(causalEventId).toBeUndefined()
103+
expect(queryController.signal.reason).toBe('interrupt')
104+
expect(suggestionController.signal.reason).toBeInstanceOf(DOMException)
105+
expect(suggestionController.signal.reason.name).toBe('AbortError')
106+
expect(__getInterruptionTraceSnapshotForTests()).toEqual([])
107+
})
108+
})

0 commit comments

Comments
 (0)