From f635ff00aa08035561887895c15f9c7477a51cb8 Mon Sep 17 00:00:00 2001 From: David Crowe Date: Wed, 7 Oct 2026 17:33:48 -0700 Subject: [PATCH] harness canary: post the verdict on the drift issue that dispatched the run The drift scout dispatches harness-release with client_payload {harness, drift_issue, version, drift_kind, close_on_pass}. The workflow passes those through to canary-report.mjs, which comments on the drift issue in davidcrowe/gatewaystack-connect: a pass says safe to close and closes it (on by default for release drift, off for docs drift, close_on_pass:false overrides); a failure names the step and links the canary issue or run. Best effort: a drift-side failure never fails the reporter. --- .github/workflows/harness-canary.yml | 25 +++++++++- docs/harness-canary.md | 25 ++++++++++ scripts/canary-report.mjs | 69 +++++++++++++++++++++++++--- 3 files changed, 111 insertions(+), 8 deletions(-) diff --git a/.github/workflows/harness-canary.yml b/.github/workflows/harness-canary.yml index 9a169d2..cf5e39b 100644 --- a/.github/workflows/harness-canary.yml +++ b/.github/workflows/harness-canary.yml @@ -13,6 +13,12 @@ # issue for that leg. Coverage table, secrets and the workspace flag: # docs/harness-canary.md. # +# Drift pairing: the drift scout (gatewaystack-connect, apps/tenant-gateway/ +# src/drift) dispatches `harness-release` with client_payload +# {harness, drift_issue, version, drift_kind, close_on_pass} when an upstream +# release signal moves; the reporter then comments the verdict on that drift +# issue and, for release-type drift, closes it on a pass. +# # Live legs (one governed `echo `, then the audit-row assert): # claude-code claude-acp -p, BYO subscription OAuth (CLAUDE_CODE_OAUTH_TOKEN) # forwarded by the proxy; no ANTHROPIC_API_KEY anywhere. @@ -246,6 +252,13 @@ jobs: CANARY_ISSUES_TOKEN: ${{ secrets.CANARY_ISSUES_TOKEN }} RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} HARNESS_VERSION: ${{ steps.harness.outputs.version }} + # Drift-scout pairing (repository_dispatch harness-release): the + # verdict is also commented on the drift issue, which is closed on + # a pass for release-type drift (docs/harness-canary.md). + DRIFT_ISSUE: ${{ github.event.client_payload.drift_issue || '' }} + DRIFT_VERSION: ${{ github.event.client_payload.version || '' }} + DRIFT_KIND: ${{ github.event.client_payload.drift_kind || '' }} + DRIFT_CLOSE: ${{ github.event.client_payload.drift_issue && github.event.client_payload.close_on_pass == false && 'off' || '' }} O_HARNESS: ${{ steps.harness.outcome }} O_SEED: ${{ steps.seed.outcome }} O_INSTALL: ${{ steps.acp_install.outcome }} @@ -268,7 +281,9 @@ jobs: fi done node scripts/canary-report.mjs --harness "$HARNESS" --status "$status" --version "$HARNESS_VERSION" \ - --run-url "$RUN_URL" --failed-step "${failed:-none}" ${log:+--log "$log"} + --run-url "$RUN_URL" --failed-step "${failed:-none}" ${log:+--log "$log"} \ + ${DRIFT_ISSUE:+--drift-issue "$DRIFT_ISSUE"} ${DRIFT_VERSION:+--drift-version "$DRIFT_VERSION"} \ + ${DRIFT_KIND:+--drift-kind "$DRIFT_KIND"} ${DRIFT_CLOSE:+--drift-close "$DRIFT_CLOSE"} # SDK legs: install the PUBLISHED package (npm / PyPI, latest), make one # governed tool-call check (POST /govern/tool-use with tool `shell`, @@ -357,6 +372,10 @@ jobs: CANARY_ISSUES_TOKEN: ${{ secrets.CANARY_ISSUES_TOKEN }} RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} PKG_VERSION: ${{ steps.pkg.outputs.version }} + DRIFT_ISSUE: ${{ github.event.client_payload.drift_issue || '' }} + DRIFT_VERSION: ${{ github.event.client_payload.version || '' }} + DRIFT_KIND: ${{ github.event.client_payload.drift_kind || '' }} + DRIFT_CLOSE: ${{ github.event.client_payload.drift_issue && github.event.client_payload.close_on_pass == false && 'off' || '' }} O_PKG: ${{ steps.pkg.outcome }} O_CALL: ${{ steps.call.outcome }} O_AUDIT: ${{ steps.audit.outcome }} @@ -374,4 +393,6 @@ jobs: fi done node scripts/canary-report.mjs --harness "$LEG" --status "$status" --version "$PKG_VERSION" \ - --run-url "$RUN_URL" --failed-step "${failed:-none}" ${log:+--log "$log"} + --run-url "$RUN_URL" --failed-step "${failed:-none}" ${log:+--log "$log"} \ + ${DRIFT_ISSUE:+--drift-issue "$DRIFT_ISSUE"} ${DRIFT_VERSION:+--drift-version "$DRIFT_VERSION"} \ + ${DRIFT_KIND:+--drift-kind "$DRIFT_KIND"} ${DRIFT_CLOSE:+--drift-close "$DRIFT_CLOSE"} diff --git a/docs/harness-canary.md b/docs/harness-canary.md index 43825d1..410b268 100644 --- a/docs/harness-canary.md +++ b/docs/harness-canary.md @@ -114,6 +114,31 @@ Triggers: `schedule: 13 */6 * * *`, `workflow_dispatch` (input `harness`: `harness-release` (optional `client_payload.harness`, same values). Concurrency is per leg; legs do not cancel each other. +### Drift pairing + +The drift scout (gatewaystack-connect, `apps/tenant-gateway/src/drift/`, +`docs/drift-scout.md` there) sends the `harness-release` dispatch when an +upstream release signal (npm latest, GitHub release/tag, PyPI) of a harness +or SDK moves, with `client_payload` +`{harness, drift_issue, version, drift_kind: "release"|"docs", close_on_pass}`. +The workflow passes those through to the reporter as `--drift-issue`, +`--drift-version`, `--drift-kind`, `--drift-close`, and the reporter comments +the verdict on that drift issue in `davidcrowe/gatewaystack-connect`: + +- pass: `Canary on : ✅ passed: safe to close` and, when + closing is on, closes the drift issue (`state_reason: completed`); +- fail: `Canary on : ❌ failed at : see ` + (the run URL when no canary issue exists). + +Closing on pass defaults ON for `drift_kind: release` (the only kind the scout +dispatches) and OFF for `drift_kind: docs`; `close_on_pass: false` in the +payload (or `--drift-close off`) turns it off either way. The drift comment is +best effort: a failure there is logged and never fails the reporter, so the +canary issue stays the primary record. `CANARY_ISSUES_TOKEN` already covers +it (same repo, Issues: read and write). The scout's own dispatch credential +(`CANARY_DISPATCH_TOKEN`, Actions: read and write on this repo) is documented +in gatewaystack-connect `docs/drift-scout.md`. + ## Secrets (repository secrets on agentic-control-plane/acp-install) | Secret | Used by | Notes | diff --git a/scripts/canary-report.mjs b/scripts/canary-report.mjs index bcc727f..65e940d 100644 --- a/scripts/canary-report.mjs +++ b/scripts/canary-report.mjs @@ -3,7 +3,9 @@ // davidcrowe/gatewaystack-connect. // // canary-report.mjs --harness --status --version -// --run-url [--failed-step ] [--log ] [--dry-run] +// --run-url [--failed-step ] [--log ] +// [--drift-issue [--drift-version ] [--drift-kind release|docs] [--drift-close on|off]] +// [--dry-run] // // Dedup: the open issue labelled `canary` + `harness:` (and, as a // belt-and-braces check, carrying the `` @@ -11,6 +13,16 @@ // with an open issue -> comment the run URL and version. Success with an // open issue -> comment and close. Success with none -> nothing to do. // +// Drift pairing: when the run was dispatched by the drift scout +// (repository_dispatch harness-release with client_payload.drift_issue), +// the verdict is also posted as a comment on that drift issue: +// "Canary on : ✅ passed: safe to close", or +// "Canary on : ❌ failed at : see ". +// On a pass the drift issue is closed too when --drift-close is on. Default: +// on for release-type drift (`--drift-kind release`, the only kind the scout +// dispatches today), off for docs-page drift (`--drift-kind docs`), since a +// green canary says nothing about a changed docs page. +// // Auth: CANARY_ISSUES_TOKEN (a fine-grained PAT with Issues: read/write on // the target repo). --dry-run prints the requests instead of sending them. @@ -32,6 +44,11 @@ const version = opt("version", "unknown"); const runUrl = opt("run-url", ""); const failedStep = opt("failed-step", "none"); const logFile = opt("log", ""); +const driftIssue = Number(opt("drift-issue", "")) || 0; +const driftVersion = opt("drift-version", "") || version; +const driftKind = opt("drift-kind", "release") === "docs" ? "docs" : "release"; +const driftCloseOpt = opt("drift-close", ""); +const driftClose = driftCloseOpt ? /^(on|true|1|yes)$/i.test(driftCloseOpt) : driftKind === "release"; // Leg ids: every harness leg and every sdk-* leg the workflow defines (the // list lives in canary-assert.mjs; here any well-formed id is accepted so a @@ -95,7 +112,10 @@ async function findOpenIssue() { const runLine = `Run: ${runUrl || "(no run url)"} · harness version: \`${version}\` · ${stamp}`; -async function main() { +// Files/updates/closes the canary issue; returns the canary issue URL the +// drift comment should point at (null when the run is green and no issue +// was open). +async function reportCanaryIssue() { const open = await findOpenIssue(); if (status === "failure") { if (open) { @@ -103,7 +123,7 @@ async function main() { body: `Still failing at step \`${failedStep}\`.\n\n${runLine}${logExcerpt()}`, }); console.log(`updated #${open.number}`); - return; + return open.html_url; } const created = await gh("POST", `/repos/${REPO}/issues`, { title: `harness canary: ${harness} failing at ${failedStep}`, @@ -115,6 +135,7 @@ async function main() { `- Failing step: \`${failedStep}\``, `- Harness version: \`${version}\``, `- ${runLine}`, + driftIssue ? `- Triggered by drift issue #${driftIssue} (upstream ${driftVersion})` : "", "", isSdk ? `The canary installs the published ${harness.slice(4)} package at latest, makes one governed tool-call check against the canary workspace (POST /govern/tool-use, tool \`shell\`, \`echo \`; the proxy SDK makes one acpFetch through ACP's egress instead) and, except for the proxy leg, asserts the audit row landed. Steps: install-package, run-check, audit-row. This issue is updated on every failing run and closed automatically on the next run where every step is green.` @@ -122,16 +143,52 @@ async function main() { ? `The canary installs the current released harness, installs ACP via the live installer with a seeded key, asserts the install invariants, pushes one governed \`echo\` through the \`${harness}\` launcher (model traffic through ACP's proxy: Gemini on the platform key, or the canary's own subscription OAuth for claude-code) and asserts the audit row landed. A transcript containing a rejection (\`auto-rejecting\`, \`rejected permission\`, \`Denied at approval\`, ...) fails the governed-call step (gsc#1380); the audit rows in the window are printed with their \`decision\` so it can be classified. This issue is updated on every failing run and closed automatically on the next run where every step is green.` : "The canary installs the current released harness, installs ACP via the live installer with a seeded key and asserts the install invariants (plugin/hook/provider wired exactly once, launcher executable, directive once, no key literal in any config). This leg has no live governed call: the harness cannot run headless on a model that needs no vendor key (docs/harness-canary.md has the per-leg reason). This issue is updated on every failing run and closed automatically on the next run where every step is green.", logExcerpt(), - ].join("\n"), + ].filter((l, i, a) => !(l === "" && a[i - 1] === "")).join("\n"), }); console.log(`created ${created.html_url}`); - return; + return created.html_url; } // success - if (!open) { console.log("green, no open canary issue"); return; } + if (!open) { console.log("green, no open canary issue"); return null; } await gh("POST", `/repos/${REPO}/issues/${open.number}/comments`, { body: `Green again. ${runLine}` }); await gh("PATCH", `/repos/${REPO}/issues/${open.number}`, { state: "closed", state_reason: "completed" }); console.log(`closed #${open.number}`); + return null; +} + +// Posts the verdict on the drift issue that dispatched this run. Best +// effort: a failure here is logged and does not fail the reporter, so the +// canary issue (the primary record) is never lost to a drift-side hiccup. +async function reportToDriftIssue(canaryIssueUrl) { + if (!driftIssue) return; + const head = `Canary \`${harness}\` on \`${driftVersion}\``; + const tail = `\n\n${runLine}`; + try { + if (status === "success") { + await gh("POST", `/repos/${REPO}/issues/${driftIssue}/comments`, { + body: `${head}: ✅ passed: safe to close${driftClose ? " — closing." : "."}${tail}`, + }); + if (driftClose) { + await gh("PATCH", `/repos/${REPO}/issues/${driftIssue}`, { state: "closed", state_reason: "completed" }); + console.log(`drift #${driftIssue}: passed, closed`); + } else { + console.log(`drift #${driftIssue}: passed, left open (${driftKind} drift)`); + } + return; + } + const see = canaryIssueUrl ?? runUrl ?? "(no link)"; + await gh("POST", `/repos/${REPO}/issues/${driftIssue}/comments`, { + body: `${head}: ❌ failed at \`${failedStep}\`: see ${see}${tail}`, + }); + console.log(`drift #${driftIssue}: failed at ${failedStep}, commented`); + } catch (e) { + console.error(`drift #${driftIssue}: could not post the verdict: ${e.message ?? e}`); + } +} + +async function main() { + const canaryIssueUrl = await reportCanaryIssue(); + await reportToDriftIssue(canaryIssueUrl); } main().catch((e) => { console.error(e.message ?? e); process.exit(1); });