From e265fa40987f76ce73e507519ba03e1648972c4c Mon Sep 17 00:00:00 2001 From: Steve Morgan <73501129+rebelopsio@users.noreply.github.com> Date: Wed, 13 May 2026 13:27:48 -0400 Subject: [PATCH] fix(daily): surface turns, duration, cost in verification-failure error MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When the agent makes zero tool calls AND produces empty text, we need to know whether the model was ever consulted. Surface Turns, Duration, and CostUSD from the agent run, and explicitly show "" when Text is empty. Renames explainToolCalls → explainAgentOutcome to reflect broader scope. --- cmd/daily_run.go | 27 ++++++++++++++++----------- cmd/daily_run_test.go | 30 +++++++++++++++++++++++++++--- 2 files changed, 43 insertions(+), 14 deletions(-) diff --git a/cmd/daily_run.go b/cmd/daily_run.go index 8481430..9b17183 100644 --- a/cmd/daily_run.go +++ b/cmd/daily_run.go @@ -205,7 +205,7 @@ func runDaily(ctx context.Context, deps dailyDeps, opts dailyOptions) (dailyResu return dailyResult{}, fmt.Errorf( "daily: agent claimed success but no file at %s: %w", absPath, - explainToolCalls(runRes.ToolCalls, runRes.Text, statErr), + explainAgentOutcome(runRes, statErr), ) } @@ -217,17 +217,18 @@ func runDaily(ctx context.Context, deps dailyDeps, opts dailyOptions) (dailyResu }, nil } -// explainToolCalls returns an error summarizing what the agent did, -// so the operator can diagnose why the expected file wasn't written. -// agentText is the concatenated assistant output from the run; -// truncated to keep the error message bounded. -func explainToolCalls(calls []agent.ToolCallRecord, agentText string, statErr error) error { +// explainAgentOutcome returns an error summarizing what the agent did +// (and didn't do) so the operator can diagnose why the expected file +// wasn't written. Surfaces tool calls, turns/duration/cost from the +// SDK, and the agent's text output. Empty text is reported as +// "" explicitly because it's a meaningful signal, not noise. +func explainAgentOutcome(res *agent.RunResult, statErr error) error { var b strings.Builder - if len(calls) == 0 { + if len(res.ToolCalls) == 0 { fmt.Fprintf(&b, "agent made zero tool calls (stat: %v)", statErr) } else { - fmt.Fprintf(&b, "agent made %d tool call(s):", len(calls)) - for i, c := range calls { + fmt.Fprintf(&b, "agent made %d tool call(s):", len(res.ToolCalls)) + for i, c := range res.ToolCalls { outcome := "ok" if c.Error != "" { outcome = "error: " + c.Error @@ -236,8 +237,12 @@ func explainToolCalls(calls []agent.ToolCallRecord, agentText string, statErr er } fmt.Fprintf(&b, "\n(stat: %v)", statErr) } - if agentText != "" { - fmt.Fprintf(&b, "\nagent said: %s", truncateText(agentText, 1000)) + fmt.Fprintf(&b, "\nturns=%d duration=%s cost_usd=%.6f", + res.Turns, res.Duration, res.CostUSD) + if res.Text != "" { + fmt.Fprintf(&b, "\nagent said: %s", truncateText(res.Text, 1000)) + } else { + fmt.Fprintf(&b, "\nagent said: ") } return errors.New(b.String()) } diff --git a/cmd/daily_run_test.go b/cmd/daily_run_test.go index 006b96f..5004172 100644 --- a/cmd/daily_run_test.go +++ b/cmd/daily_run_test.go @@ -410,8 +410,9 @@ func TestRunDaily_VerifiesFileAfterRun(t *testing.T) { } // Missing file with no tool calls: error includes the path, the -// "zero tool calls" call-out, and the stat error. Empty agent text -// must not produce an "agent said:" line. +// "zero tool calls" call-out, the stat error, the SDK signal line +// (turns/duration/cost), and an explicit "" marker for the +// missing text so the operator can tell signal from noise. func TestRunDaily_FailsLoudlyWhenFileMissing(t *testing.T) { deps, gatherer, runtime, _ := fixtureDeps(t) gatherer.issues = []domain.Issue{{Ref: domain.ExternalRef{Provider: "linear", ID: "ENG-1"}, Title: "x"}} @@ -425,7 +426,8 @@ func TestRunDaily_FailsLoudlyWhenFileMissing(t *testing.T) { msg := err.Error() assert.Contains(t, msg, "no file at") assert.Contains(t, msg, "zero tool calls") - assert.NotContains(t, msg, "agent said:", "empty text must not emit an agent-said line") + assert.Contains(t, msg, "turns=0") + assert.Contains(t, msg, "agent said: ") } // Missing file with tool calls: every tool call's name and outcome @@ -489,6 +491,28 @@ func TestRunDaily_AgentTextTruncated(t *testing.T) { assert.Less(t, len(msg), 2000, "error message should be bounded") } +// Turns, duration, and cost from the SDK are surfaced so the operator +// can tell whether the model was consulted at all when both tool +// calls and text are empty. +func TestRunDaily_MissingFileIncludesTurnsAndDuration(t *testing.T) { + deps, gatherer, runtime, _ := fixtureDeps(t) + gatherer.issues = []domain.Issue{{Ref: domain.ExternalRef{Provider: "linear", ID: "ENG-1"}, Title: "x"}} + + runtime.result = &agent.RunResult{ + Turns: 3, + Duration: 2500 * time.Millisecond, + CostUSD: 0.0042, + } + + _, err := runDaily(context.Background(), deps, dailyOptions{}) + require.Error(t, err) + msg := err.Error() + assert.Contains(t, msg, "turns=3") + assert.Contains(t, msg, "duration=2.5s") + assert.Contains(t, msg, "cost_usd=0.004200") + assert.Contains(t, msg, "agent said: ") +} + func TestSummarizeToolCalls(t *testing.T) { assert.Equal(t, "0 tool calls", summarizeToolCalls(nil)) assert.Equal(t, "1 tool call(s): mcp__archy__archy_write_vault_note",