-
Notifications
You must be signed in to change notification settings - Fork 0
Guide chargeable goal resumptions to change strategy #1156
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: agent/daemon-live-absorb-fleet-soak-harness
Are you sure you want to change the base?
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -11,9 +11,10 @@ use signalbox_application::{ | |
| }; | ||
| use signalbox_domain::{ | ||
| AcceptedInputId, DurableCommandId, Goal, GoalBlockProvenance, GoalCommandResult, GoalEvent, | ||
| GoalEventKind, GoalEventOrdinal, GoalModelBlockedReasonKind, GoalModelProvenance, GoalNeed, | ||
| GoalReport, GoalSchedulerProvenance, GoalUserAction, GoalUserCommand, NormalizedToolArguments, | ||
| SessionId, ToolEffectClass, ToolExecutionErrorDetail, ToolName, ToolPermissionDefault, TurnId, | ||
| GoalEventKind, GoalEventOrdinal, GoalGuidance, GoalModelBlockedReasonKind, GoalModelProvenance, | ||
| GoalNeed, GoalReport, GoalSchedulerProvenance, GoalTextError, GoalUserAction, GoalUserCommand, | ||
| NormalizedToolArguments, SessionId, ToolEffectClass, ToolExecutionErrorDetail, ToolName, | ||
| ToolPermissionDefault, TurnId, | ||
| }; | ||
| use signalbox_persistence::{ | ||
| goal::{ | ||
|
|
@@ -69,6 +70,8 @@ const EXECUTION_FAILURE_NEED: &str = | |
| /// a durably rejected command, a daemon restart, an unreachable database — | ||
| /// leaves this text as what the operator reads. | ||
| const EXECUTION_FAILURE_RESUMING_PREAMBLE: &str = "The goal turn failed to execute and automatic resumption is scheduled. If the goal is still blocked here once resumption ends, it is waiting for an operator."; | ||
| /// Guidance for a failure the session caused and should not repeat unchanged. | ||
| const CHARGEABLE_FAILURE_RESUME_GUIDANCE: &str = "Continue pursuing the commissioned goal. The preceding turn failed to execute. Inspect the durable session state and choose a different safe approach before repeating the failed operation."; | ||
| /// Retries one armed attempt may spend on a database that answers nothing. | ||
| /// | ||
| /// These are not resumptions and do not spend the attempt budget: nothing was | ||
|
|
@@ -633,13 +636,57 @@ impl PostgresGoalPassDisposition { | |
| return ResumeAttempt::Unsettled; | ||
| } | ||
| }; | ||
| if !reread.is_some_and(|goal| awaits_automatic_resumption(&goal, blocked)) { | ||
| let Some(goal) = reread else { | ||
| return ResumeAttempt::Settled; | ||
| }; | ||
| if !awaits_automatic_resumption(&goal, blocked) { | ||
| return ResumeAttempt::Settled; | ||
| } | ||
| let Some(failed_turn) = goal.events().last().and_then(execution_failure_turn) else { | ||
| tracing::error!( | ||
| session = %session.into_uuid(), | ||
| event_ordinal = blocked.get(), | ||
| cause_code = "goal_automatic_resume_failure_turn_missing", | ||
| "automatic goal resumption could not identify its blocked turn" | ||
| ); | ||
| return ResumeAttempt::Unsettled; | ||
| }; | ||
| let unchargeable = match self | ||
| .repository | ||
| .unchargeable_automatic_resume_turns(session, &[failed_turn]) | ||
| .await | ||
| { | ||
| Ok(turns) => turns.contains(&failed_turn), | ||
| Err(error) => { | ||
| tracing::error!( | ||
| session = %session.into_uuid(), | ||
| turn = %failed_turn.into_uuid(), | ||
| event_ordinal = blocked.get(), | ||
| cause_code = "goal_automatic_resume_failure_classification_failed", | ||
| cause = %error, | ||
| "automatic goal resumption could not classify its failed turn" | ||
| ); | ||
| return ResumeAttempt::Unsettled; | ||
| } | ||
| }; | ||
| let guidance = match automatic_resume_guidance(unchargeable) { | ||
| Ok(guidance) => guidance, | ||
| Err(error) => { | ||
| tracing::error!( | ||
| session = %session.into_uuid(), | ||
| event_ordinal = blocked.get(), | ||
| cause_code = "goal_automatic_resume_guidance_invalid", | ||
| cause = %error, | ||
| "automatic goal resumption could not construct its static guidance" | ||
| ); | ||
| return ResumeAttempt::Unsettled; | ||
| } | ||
| }; | ||
| let strategy_guidance = guidance.is_some(); | ||
| let command = GoalUserCommand::new( | ||
| automatic_resume_command(session, blocked), | ||
| session, | ||
| GoalUserAction::Resume(None), | ||
| GoalUserAction::Resume(guidance), | ||
| ); | ||
| let candidates = GoalTurnCandidates::new( | ||
| AcceptedInputId::from_uuid(Uuid::now_v7()), | ||
|
|
@@ -663,6 +710,7 @@ impl PostgresGoalPassDisposition { | |
| session = %session.into_uuid(), | ||
| event_ordinal = event.ordinal().get(), | ||
| blocked_event_ordinal = blocked.get(), | ||
| strategy_guidance, | ||
| "automatically resumed a goal blocked by execution failure" | ||
| ); | ||
| ResumeAttempt::Settled | ||
|
|
@@ -1024,6 +1072,13 @@ fn chargeable_automatic_resume_attempts( | |
| u32::try_from(spent).unwrap_or(u32::MAX) | ||
| } | ||
|
|
||
| fn automatic_resume_guidance(unchargeable: bool) -> Result<Option<GoalGuidance>, GoalTextError> { | ||
| if unchargeable { | ||
| return Ok(None); | ||
| } | ||
| GoalGuidance::try_new(String::from(CHARGEABLE_FAILURE_RESUME_GUIDANCE)).map(Some) | ||
| } | ||
|
|
||
| /// Whether the goal is still blocked by exactly the named failure event. | ||
| fn awaits_automatic_resumption(goal: &Goal, blocked: GoalEventOrdinal) -> bool { | ||
| goal.events() | ||
|
|
@@ -1366,6 +1421,23 @@ mod tests { | |
| ); | ||
| } | ||
|
|
||
| #[test] | ||
| fn a_chargeable_failure_changes_the_next_turn_input() { | ||
| let guidance = automatic_resume_guidance(false) | ||
| .expect("the static guidance is admitted") | ||
| .expect("a chargeable failure carries guidance"); | ||
|
|
||
| assert_eq!(guidance.as_str(), CHARGEABLE_FAILURE_RESUME_GUIDANCE); | ||
|
Comment on lines
+1426
to
+1430
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
If AGENTS.md reference: AGENTS.md:L231-L237 Useful? React with 👍 / 👎. |
||
| } | ||
|
|
||
| #[test] | ||
| fn an_unchargeable_failure_reuses_the_commissioned_statement() { | ||
| assert_eq!( | ||
| automatic_resume_guidance(true).expect("no guidance needs admission"), | ||
| None | ||
| ); | ||
| } | ||
|
|
||
| #[test] | ||
| fn an_operator_resume_restarts_the_attempt_budget() { | ||
| let after_operator = failed( | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -22,7 +22,9 @@ parking of an operator-commissioned escalation is verified against this PR | |
| automatic resumptions is verified against this PR | ||
| (`agent/daemon-live-goal-resume-rearm`). Automatic-resume failure accounting and | ||
| its twenty-attempt ceiling are verified against this PR | ||
| (`agent/daemon-live-goal-resume-failure-budget`). Identity and durable-command | ||
| (`agent/daemon-live-goal-resume-failure-budget`). Chargeable-failure resume | ||
| guidance is verified against this PR | ||
| (`agent/daemon-live-chargeable-resume-guidance`). Identity and durable-command | ||
| mechanics remain owned by [identity and commands](identity-and-commands.md), | ||
| turn execution by | ||
| [turn lifecycle and scheduling](turn-lifecycle-and-scheduling.md), tool dispatch | ||
|
|
@@ -205,22 +207,19 @@ and therefore independently eligible to continue. | |
| automatic resumption. The daemon derives from the goal event history how many | ||
| consecutive automatic resumptions the current run has already spent: the run is | ||
| the trailing alternation of execution-failure blocks and the resumptions that | ||
| answered them, and every other event ends it. Below a budget of twenty | ||
| chargeable consecutive attempts, the appended need text states that automatic | ||
| answered them, and every other event ends it. Below the required configured | ||
| chargeable-attempt budget, the appended need text states that automatic | ||
|
Comment on lines
+210
to
+211
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
When AGENTS.md reference: AGENTS.md:L46-L49 Useful? React with 👍 / 👎. |
||
| resumption is scheduled and names the operator repair for a goal still blocked | ||
| once resumption ends, and exactly one resume follows after a backoff of two | ||
| minutes doubled per attempt already spent, to a thirty-minute maximum. At the | ||
| budget the goal stays blocked, and its need text states that automatic | ||
| resumption is exhausted and states the operator repair. All three bounds are | ||
| fixed in source and no configuration reads them: an automatic resumption spends | ||
| provider budget on a session no operator asked about, so its cadence and its end | ||
| are product decisions rather than deployment ones. Every need text an | ||
| execution-failure block carries names the operator repair, because an armed | ||
| attempt can also fail to resume by being durably rejected, by losing its | ||
| process, or by never reaching the database, and in each case that text is what | ||
| an operator reads. Resumption does not bypass execution-failure blocking or make | ||
| a failure a silent retry — the block is appended first, and every attempt is an | ||
| ordinary recorded `resumed` event. | ||
| once resumption ends, and exactly one resume follows after the required | ||
| configured backoff doubled per attempt already spent, up to its required | ||
| configured cap. At the budget the goal stays blocked, and its need text states | ||
| that automatic resumption is exhausted and states the operator repair. Every | ||
| need text an execution-failure block carries names the operator repair, because | ||
| an armed attempt can also fail to resume by being durably rejected, by losing | ||
| its process, or by never reaching the database, and in each case that text is | ||
| what an operator reads. Resumption does not bypass execution-failure blocking or | ||
| make a failure a silent retry — the block is appended first, and every attempt | ||
| is an ordinary recorded `resumed` event. | ||
|
|
||
| A resumed turn does not spend that twenty-attempt goal budget when durable | ||
| evidence attributes its failure outside the session: its exact model-call or | ||
|
|
@@ -235,6 +234,16 @@ transient provider-availability condition. Typed records rather than a log line | |
| remain authority, so deploys, reconciliation deadlines, and transient provider | ||
| availability cannot exhaust work the session did not fail. | ||
|
|
||
| A chargeable failure resumes with fixed guidance to inspect durable state and | ||
| choose a different safe approach before repeating the failed operation, making | ||
| the resumed run reconsider its strategy rather than simply replaying the | ||
| immutable commissioned statement. An unchargeable failure resumes without | ||
| guidance and therefore reuses that statement: infrastructure recovery must not | ||
| invent a new model instruction. The exact failed-turn evidence used for budget | ||
| charging selects between those inputs; inability to read it leaves the | ||
| resumption unsettled for the existing bounded database retry rather than | ||
| guessing. | ||
|
|
||
| **Implemented behavior.** An automatic resumption's durable command identity is | ||
| derived from the session and the exact blocked event it answers rather than | ||
| minted. A repeated attempt is therefore an exact command replay rather than a | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
At the added call sites,
falsemeans chargeable andtruemeans unchargeable, so reversing either literal silently changes whether strategy guidance is injected. This is the boolean-blindness case prohibited bydocs/style.md; represent the classification with a named two-variant enum so the polarity is visible and checked at every call.AGENTS.md reference: AGENTS.md:L14-L15
Useful? React with 👍 / 👎.