Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions .gaai/core/scripts/daemon-start.sh
Original file line number Diff line number Diff line change
Expand Up @@ -465,6 +465,8 @@ do_start() {
[[ -n "${GAAI_CODEX_IGNORE_USER_CONFIG:-}" ]] && tmux_env_args+=(-e "GAAI_CODEX_IGNORE_USER_CONFIG=${GAAI_CODEX_IGNORE_USER_CONFIG}")
[[ -n "${GAAI_DAEMON_HOME:-}" ]] && tmux_env_args+=(-e "GAAI_DAEMON_HOME=${GAAI_DAEMON_HOME}")
[[ -n "${GAAI_REPO_ROOT:-}" ]] && tmux_env_args+=(-e "GAAI_REPO_ROOT=${GAAI_REPO_ROOT}")
[[ -n "${GAAI_CI_TEST_GATE_TIMEOUT_SEC:-}" ]] && tmux_env_args+=(-e "GAAI_CI_TEST_GATE_TIMEOUT_SEC=${GAAI_CI_TEST_GATE_TIMEOUT_SEC}")
[[ -n "${GAAI_CI_TEST_GATE_MATERIALIZE_SEC:-}" ]] && tmux_env_args+=(-e "GAAI_CI_TEST_GATE_MATERIALIZE_SEC=${GAAI_CI_TEST_GATE_MATERIALIZE_SEC}")
tmux new-session -d -s gaai-daemon ${tmux_env_args[@]+"${tmux_env_args[@]}"} "$daemon_cmd"

# Give it a moment to start, then grab the PID
Expand Down
147 changes: 99 additions & 48 deletions .gaai/core/scripts/lib/test-gate.sh
Original file line number Diff line number Diff line change
Expand Up @@ -49,9 +49,27 @@
# _test_gate_ci_scope_touched, _test_gate_poll_ci_verdict.
#
# CI-delegated merge gate env vars (all optional, have defaults):
# GAAI_CI_TEST_GATE_TIMEOUT_SEC — bounded poll wait (default: 1200,
# same order of magnitude as
# GAAI_AGENT_HANG_THRESHOLD_SEC)
# GAAI_CI_TEST_GATE_TIMEOUT_SEC — bounded poll wait (default: 2700 =
# 45 min, derived from the CI
# workflow's own declared job
# timeouts along its critical path:
# detect -> max(head, baseline) ->
# test-gate. Covers the workflow's
# full declared execution budget;
# queue latency is absorbed on top
# of it by this same bounded wait)
# GAAI_CI_TEST_GATE_MATERIALIZE_SEC — bounded sub-wait for a workflow run
# to first appear for the expected
# SHA (default: 300 = 5 min). If no
# run has been observed within this
# sub-budget, the poll returns
# "unavailable:no_run" immediately
# instead of waiting out the full
# budget — distinguishes "CI will
# never run" from "CI is still
# running". Once a run has been
# observed in any state, the full
# budget governs.
# GAAI_CI_TEST_GATE_POLL_INTERVAL_SEC — sleep between polls (default: 20;
# keeps wrapper.log fresh so the
# hang-detector never fires on a
Expand Down Expand Up @@ -567,26 +585,40 @@ _test_gate_run_with_timeout() {
# also sends that SHA to the REST merge endpoint as its atomic TOCTOU backstop.
#
# Bounded and sleep-based — never busy-waits. Prints exactly one line to
# STDOUT: "pass", "fail", "blocked:head_moved", or
# "unavailable:<reason>" (reason in timeout|api_error). Progress logging goes
# to stderr because stdout is the signal captured by the caller. No scheduler
# or notification side effects occur here.
# STDOUT: "pass", "fail", "blocked:head_moved", or "unavailable:<reason>"
# (reason in timeout|api_error|no_run — no_run means the materialize
# sub-budget expired before any workflow run was ever observed for the
# expected SHA; timeout means a run WAS observed but no decisive verdict
# arrived before the full budget expired). Progress logging goes to stderr
# because stdout is the signal captured by the caller. No scheduler or
# notification side effects occur here.
_test_gate_poll_ci_verdict() {
local story_id="$1" pr_url="$2" expected_head_sha="$3"
local timeout_sec poll_interval api_timeout_sec
timeout_sec=$(_test_gate_positive_int "${GAAI_CI_TEST_GATE_TIMEOUT_SEC:-1200}" 1200 GAAI_CI_TEST_GATE_TIMEOUT_SEC)
local timeout_sec materialize_sec poll_interval api_timeout_sec
# Default derived from the CI workflow's own declared job timeouts along
# its critical path (detect -> max(head, baseline) -> test-gate) — see the
# file header for the full derivation. Not a wall-clock sample: it stays
# correct independent of runner performance drift.
timeout_sec=$(_test_gate_positive_int "${GAAI_CI_TEST_GATE_TIMEOUT_SEC:-2700}" 2700 GAAI_CI_TEST_GATE_TIMEOUT_SEC)
materialize_sec=$(_test_gate_positive_int "${GAAI_CI_TEST_GATE_MATERIALIZE_SEC:-300}" 300 GAAI_CI_TEST_GATE_MATERIALIZE_SEC)
poll_interval=$(_test_gate_positive_int "${GAAI_CI_TEST_GATE_POLL_INTERVAL_SEC:-20}" 20 GAAI_CI_TEST_GATE_POLL_INTERVAL_SEC)
api_timeout_sec=$(_test_gate_positive_int "${GAAI_CI_TEST_GATE_API_TIMEOUT_SEC:-30}" 30 GAAI_CI_TEST_GATE_API_TIMEOUT_SEC)

local started_at deadline now remaining call_timeout sleep_for
local started_at deadline materialize_deadline now remaining call_timeout sleep_for
started_at=$(date +%s)
deadline=$(( started_at + timeout_sec ))
materialize_deadline=$(( started_at + materialize_sec ))
local run_observed=0
local api_err_streak=0
local jq_expr='if (.workflow_runs | length) == 0 then "missing" else (.workflow_runs | sort_by(.created_at) | last | [(.status // "unknown"), (.conclusion // "pending"), (.head_sha // "missing")] | @tsv) end'

while :; do
now=$(date +%s)
(( now >= deadline )) && break
if [[ "$run_observed" -eq 0 && "$now" -ge "$materialize_deadline" ]]; then
echo "unavailable:no_run"
return 0
fi
remaining=$(( deadline - now ))
call_timeout="$api_timeout_sec"
(( call_timeout > remaining )) && call_timeout="$remaining"
Expand All @@ -604,46 +636,52 @@ _test_gate_poll_ci_verdict() {
IFS=$'\t' read -r run_status run_conclusion observed_head_sha <<< "$run_line"
if [[ "$observed_head_sha" != "$expected_head_sha" ]]; then
api_err_streak=$(( api_err_streak + 1 ))
elif [[ "$run_status" == "completed" ]]; then
# A completed blocking conclusion for the exact pushed SHA is already
# decisive. Do not let an unrelated PR-head lookup outage downgrade a
# known CI failure into the local fallback path.
if [[ "$run_conclusion" != "success" && "$run_conclusion" != "cancelled" ]]; then
echo "fail"
return 0
fi

# The workflow lookup may have consumed most of this iteration's
# budget. Recompute the remaining wall clock before the second call.
# Both success and cancellation need this check: success may authorize
# only the current head, while cancellation caused by a superseding
# push must report the moved head rather than masquerading as failure.
now=$(date +%s)
(( now >= deadline )) && break
remaining=$(( deadline - now ))
local head_call_timeout="$api_timeout_sec"
(( head_call_timeout > remaining )) && head_call_timeout="$remaining"

local current_head_sha head_rc=0
current_head_sha=$(_test_gate_run_with_timeout "$head_call_timeout" gh pr view "$pr_url" \
--json headRefOid --jq .headRefOid) || head_rc=$?
if [[ "$head_rc" -ne 0 || ! "$current_head_sha" =~ ^[0-9a-fA-F]{40}$ ]]; then
api_err_streak=$(( api_err_streak + 1 ))
elif [[ "$current_head_sha" != "$expected_head_sha" ]]; then
echo "blocked:head_moved"
return 0
elif [[ "$run_conclusion" == "success" ]]; then
echo "pass"
return 0
else
# A run for the expected SHA has now been seen in SOME state — this
# is "materialized" regardless of whether it is queued, in progress,
# or already completed. From here on the full budget governs.
run_observed=1
if [[ "$run_status" == "completed" ]]; then
# A completed blocking conclusion for the exact pushed SHA is already
# decisive. Do not let an unrelated PR-head lookup outage downgrade a
# known CI failure into the local fallback path.
if [[ "$run_conclusion" != "success" && "$run_conclusion" != "cancelled" ]]; then
echo "fail"
return 0
fi

# The workflow lookup may have consumed most of this iteration's
# budget. Recompute the remaining wall clock before the second call.
# Both success and cancellation need this check: success may authorize
# only the current head, while cancellation caused by a superseding
# push must report the moved head rather than masquerading as failure.
now=$(date +%s)
(( now >= deadline )) && break
remaining=$(( deadline - now ))
local head_call_timeout="$api_timeout_sec"
(( head_call_timeout > remaining )) && head_call_timeout="$remaining"

local current_head_sha head_rc=0
current_head_sha=$(_test_gate_run_with_timeout "$head_call_timeout" gh pr view "$pr_url" \
--json headRefOid --jq .headRefOid) || head_rc=$?
if [[ "$head_rc" -ne 0 || ! "$current_head_sha" =~ ^[0-9a-fA-F]{40}$ ]]; then
api_err_streak=$(( api_err_streak + 1 ))
elif [[ "$current_head_sha" != "$expected_head_sha" ]]; then
echo "blocked:head_moved"
return 0
elif [[ "$run_conclusion" == "success" ]]; then
echo "pass"
return 0
else
# A same-head cancelled run is non-decisive: a manual re-run may
# supersede it. Keep polling until a decisive run appears or the
# bounded wait falls back locally.
api_err_streak=0
fi
else
# A same-head cancelled run is non-decisive: a manual re-run may
# supersede it. Keep polling until a decisive run appears or the
# bounded wait falls back locally.
# queued/in_progress/waiting/requested/pending are valid observations.
api_err_streak=0
fi
else
# queued/in_progress/waiting/requested/pending are valid observations.
api_err_streak=0
fi
else
api_err_streak=$(( api_err_streak + 1 ))
Expand All @@ -656,9 +694,17 @@ _test_gate_poll_ci_verdict() {

now=$(date +%s)
(( now >= deadline )) && break
if [[ "$run_observed" -eq 0 && "$now" -ge "$materialize_deadline" ]]; then
echo "unavailable:no_run"
return 0
fi
remaining=$(( deadline - now ))
sleep_for="$poll_interval"
(( sleep_for > remaining )) && sleep_for="$remaining"
if [[ "$run_observed" -eq 0 ]]; then
local materialize_remaining=$(( materialize_deadline - now ))
(( sleep_for > materialize_remaining )) && sleep_for="$materialize_remaining"
fi
echo "[TEST-GATE-CI] ${story_id} : waiting on ${pr_url}@${expected_head_sha} (elapsed=$(( now - started_at ))s/${timeout_sec}s)" >&2
sleep "$sleep_for"
done
Expand Down Expand Up @@ -718,7 +764,12 @@ _run_merge_test_gate() {
;;
*)
reason="${verdict#unavailable:}"
echo "[WARN] ${story_id} handle_commit_phase: CI test-gate result unavailable (${reason}) — falling back to local gate [class=TEST_GATE_CI_UNAVAILABLE_FALLBACK]"
local reason_detail=""
case "$reason" in
timeout) reason_detail=" — CI run observed but still in progress at cutoff" ;;
no_run) reason_detail=" — no CI run observed for this SHA" ;;
esac
echo "[WARN] ${story_id} handle_commit_phase: CI test-gate result unavailable (${reason})${reason_detail} — falling back to local gate [class=TEST_GATE_CI_UNAVAILABLE_FALLBACK]"
_run_deterministic_test_gate "$story_id" "$worktree_path" "$qa_report_path"
return $?
;;
Expand Down
80 changes: 80 additions & 0 deletions .gaai/core/scripts/tests/ci-merge-test-gate.test.sh
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,14 @@
# lookup is unavailable; it can never be downgraded to local fallback.
# T11/T12 (cancellation): same-head cancellation is non-decisive and falls back
# on timeout, while cancellation caused by a superseding head blocks as moved.
# T13 (materialize sub-budget + default, AC1+AC5+AC6): GAAI_CI_TEST_GATE_TIMEOUT_SEC
# left unset (exercises the 2700s default) with a permanently-"missing" run —
# the much shorter materialize sub-budget fires first, returning
# "unavailable:no_run" well inside it rather than waiting out the full budget.
# T14 (timeout classification, AC3): a run observed in_progress on every poll —
# materialize sub-budget is satisfied immediately, the full (short, pinned)
# budget then expires and reports "unavailable:timeout" with the
# still-in-progress detail, distinct from T13's no_run detail.
#
# Run: bash .gaai/core/scripts/tests/ci-merge-test-gate.test.sh
# Exit 0 = all pass.
Expand Down Expand Up @@ -206,9 +214,11 @@ OUT_T2="$SANDBOX/T2.out.log"
: > "$GH_CALL_LOG"
set_gh_run_lines "completed"$'\t'"success"$'\t'"$HEAD_T2"
export GH_PR_HEAD_SHA="$HEAD_T2"
export GAAI_CI_TEST_GATE_TIMEOUT_SEC=5
unset GH_API_EXIT GH_API_HANG_SEC
_run_merge_test_gate "T2" "$REPO_T2" "$QA_T2" "$PR_URL" "$HEAD_T2" > "$OUT_T2" 2>&1
RC_T2=$?
unset GAAI_CI_TEST_GATE_TIMEOUT_SEC

[[ "$RC_T2" -eq 0 ]] && pass "T2: returns 0 (CI pass)" || fail "T2: expected rc=0, got ${RC_T2}"
grep -q "gh api.*head_sha=${HEAD_T2}" "$GH_CALL_LOG" \
Expand All @@ -235,9 +245,11 @@ OUT_T3="$SANDBOX/T3.out.log"
: > "$NOTIFY_CALLS_LOG"
set_gh_run_lines "completed"$'\t'"failure"$'\t'"$HEAD_T3"
export GH_PR_HEAD_SHA="$HEAD_T3"
export GAAI_CI_TEST_GATE_TIMEOUT_SEC=5
unset GH_API_EXIT GH_API_HANG_SEC
_run_merge_test_gate "T3" "$REPO_T3" "$QA_T3" "$PR_URL" "$HEAD_T3" > "$OUT_T3" 2>&1
RC_T3=$?
unset GAAI_CI_TEST_GATE_TIMEOUT_SEC

[[ "$RC_T3" -eq 1 ]] && pass "T3: returns 1 (CI fail blocks)" || fail "T3: expected rc=1, got ${RC_T3}"
grep -q -- "--set-phase-status T3 failed" "$SCHEDULER_CALLS_LOG" \
Expand Down Expand Up @@ -357,10 +369,12 @@ OUT_T7="$SANDBOX/T7.out.log"
: > "$NOTIFY_CALLS_LOG"
set_gh_run_lines "completed"$'\t'"success"$'\t'"$HEAD_T7"
export GH_PR_HEAD_SHA="$(printf '%040d' 1)"
export GAAI_CI_TEST_GATE_TIMEOUT_SEC=5
unset GH_API_EXIT GH_API_HANG_SEC

_run_merge_test_gate "T7" "$REPO_T7" "$QA_T7" "$PR_URL" "$HEAD_T7" > "$OUT_T7" 2>&1
RC_T7=$?
unset GAAI_CI_TEST_GATE_TIMEOUT_SEC

[[ "$RC_T7" -eq 1 ]] && pass "T7: moved PR head blocks" || fail "T7: expected rc=1, got ${RC_T7}"
grep -q "test_gate_head_moved" "$NOTIFY_CALLS_LOG" \
Expand Down Expand Up @@ -504,9 +518,11 @@ OUT_T12="$SANDBOX/T12.out.log"
: > "$NOTIFY_CALLS_LOG"
set_gh_run_lines "completed"$'\t'"cancelled"$'\t'"$HEAD_T12"
export GH_PR_HEAD_SHA="$(printf '%040d' 2)"
export GAAI_CI_TEST_GATE_TIMEOUT_SEC=5

_run_merge_test_gate "T12" "$REPO_T12" "$QA_T12" "$PR_URL" "$HEAD_T12" > "$OUT_T12" 2>&1
RC_T12=$?
unset GAAI_CI_TEST_GATE_TIMEOUT_SEC

[[ "$RC_T12" -eq 1 ]] \
&& pass "T12: superseding push blocks the cancelled old head" \
Expand All @@ -517,6 +533,70 @@ grep -q "test_gate_head_moved" "$NOTIFY_CALLS_LOG" \

unset GH_PR_HEAD_SHA

# ── T13: materialize sub-budget fast-fail + unset-default resolution (AC1+AC5+AC6) ─
echo "--- T13: no run ever materialises, default budget unset ---"
REPO_T13="$(setup_repo T13 "workers/fake-pkg/file.txt")"
HEAD_T13=$(git -C "$REPO_T13" rev-parse HEAD)
QA_T13="$SANDBOX/T13.qa-report.md"
OUT_T13="$SANDBOX/T13.out.log"
: > "$GH_CALL_LOG"
set_gh_run_lines "missing"
unset GAAI_CI_TEST_GATE_TIMEOUT_SEC
export GAAI_CI_TEST_GATE_MATERIALIZE_SEC=1
export GAAI_CI_TEST_GATE_POLL_INTERVAL_SEC=1
export GAAI_CI_TEST_GATE_API_TIMEOUT_SEC=1
unset GH_API_EXIT GH_API_HANG_SEC

_t13_start=$(date +%s)
_run_merge_test_gate "T13" "$REPO_T13" "$QA_T13" "$PR_URL" "$HEAD_T13" > "$OUT_T13" 2>&1
RC_T13=$?
_t13_end=$(date +%s)
_t13_elapsed=$(( _t13_end - _t13_start ))

[[ "$RC_T13" -eq 0 ]] && pass "T13: returns 0 (local fallback PASS)" || fail "T13: expected rc=0, got ${RC_T13}"
[[ "$_t13_elapsed" -lt 10 ]] \
&& pass "T13: no_run resolved well inside the materialize sub-budget (${_t13_elapsed}s < 10s)" \
|| fail "T13: took ${_t13_elapsed}s — materialize sub-budget did not fast-fail"
grep -q "unavailable (no_run)" "$OUT_T13" \
&& pass "T13: fallback reason=no_run (distinct token, AC6)" \
|| fail "T13: expected reason=no_run in output"
grep -q "no CI run observed for this SHA" "$OUT_T13" \
&& pass "T13: no_run detail present (AC3)" \
|| fail "T13: no_run detail missing from output"
grep -q '/2700s)' "$OUT_T13" \
&& pass "T13: progress line reflects the unset-default full budget of 2700s (AC1+AC5)" \
|| fail "T13: progress line does not show the 2700s default"

unset GAAI_CI_TEST_GATE_MATERIALIZE_SEC GAAI_CI_TEST_GATE_POLL_INTERVAL_SEC GAAI_CI_TEST_GATE_API_TIMEOUT_SEC

# ── T14: run observed but never decisive → timeout classification (AC3) ─────
echo "--- T14: run observed in_progress throughout, budget exhausts ---"
REPO_T14="$(setup_repo T14 "workers/fake-pkg/file.txt")"
HEAD_T14=$(git -C "$REPO_T14" rev-parse HEAD)
QA_T14="$SANDBOX/T14.qa-report.md"
OUT_T14="$SANDBOX/T14.out.log"
: > "$GH_CALL_LOG"
set_gh_run_lines "in_progress"$'\t'"pending"$'\t'"$HEAD_T14"
export GAAI_CI_TEST_GATE_TIMEOUT_SEC=2
export GAAI_CI_TEST_GATE_MATERIALIZE_SEC=1
export GAAI_CI_TEST_GATE_POLL_INTERVAL_SEC=1
export GAAI_CI_TEST_GATE_API_TIMEOUT_SEC=1
unset GH_API_EXIT GH_API_HANG_SEC

_run_merge_test_gate "T14" "$REPO_T14" "$QA_T14" "$PR_URL" "$HEAD_T14" > "$OUT_T14" 2>&1
RC_T14=$?

[[ "$RC_T14" -eq 0 ]] && pass "T14: returns 0 (local fallback PASS)" || fail "T14: expected rc=0, got ${RC_T14}"
grep -q "unavailable (timeout)" "$OUT_T14" \
&& pass "T14: fallback reason=timeout (verdict shape unchanged)" \
|| fail "T14: expected reason=timeout in output"
grep -q "CI run observed but still in progress at cutoff" "$OUT_T14" \
&& pass "T14: timeout detail present (AC3's second class)" \
|| fail "T14: timeout detail missing from output"

unset GAAI_CI_TEST_GATE_TIMEOUT_SEC GAAI_CI_TEST_GATE_MATERIALIZE_SEC \
GAAI_CI_TEST_GATE_POLL_INTERVAL_SEC GAAI_CI_TEST_GATE_API_TIMEOUT_SEC

# ── Summary ──────────────────────────────────────────────────────────────────
echo ""
echo " ${PASS_COUNT} passed, ${FAIL_COUNT} failed"
Expand Down
10 changes: 10 additions & 0 deletions .gaai/core/scripts/tests/daemon-operator-state-reporoot.test.sh
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,16 @@ if grep -qE 'tmux_env_args\+=\(-e "GAAI_REPO_ROOT=' "$START"; then
else
fail "TC3-2: daemon-start.sh does not forward GAAI_REPO_ROOT via tmux env"
fi
if grep -qE 'tmux_env_args\+=\(-e "GAAI_CI_TEST_GATE_TIMEOUT_SEC=' "$START"; then
pass "TC3-3: daemon-start.sh forwards GAAI_CI_TEST_GATE_TIMEOUT_SEC via tmux env"
else
fail "TC3-3: daemon-start.sh does not forward GAAI_CI_TEST_GATE_TIMEOUT_SEC via tmux env"
fi
if grep -qE 'tmux_env_args\+=\(-e "GAAI_CI_TEST_GATE_MATERIALIZE_SEC=' "$START"; then
pass "TC3-4: daemon-start.sh forwards GAAI_CI_TEST_GATE_MATERIALIZE_SEC via tmux env"
else
fail "TC3-4: daemon-start.sh does not forward GAAI_CI_TEST_GATE_MATERIALIZE_SEC via tmux env"
fi

echo "=== TC4: delivery-daemon.sh operator-state paths use _STATE_PROJECT_DIR (not raw GAAI_PROJECT_DIR) ==="
for var in LOG_FILE LOG_DIR LOCK_DIR; do
Expand Down
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
## [Unreleased]

### Changed
-: 'The CI test-gate wait outlasts a normal CI run, so local
-: Produce and validate the two-axis QA handoff

## [2.49.0] - 2026-08-11
Expand Down
Loading