diff --git a/.github/workflows/harness-canary.yml b/.github/workflows/harness-canary.yml index cf5e39b..1b2e76d 100644 --- a/.github/workflows/harness-canary.yml +++ b/.github/workflows/harness-canary.yml @@ -13,11 +13,17 @@ # issue for that leg. Coverage table, secrets and the workspace flag: # docs/harness-canary.md. # -# Drift pairing: the drift scout (gatewaystack-connect, apps/tenant-gateway/ -# src/drift) dispatches `harness-release` with client_payload -# {harness, drift_issue, version, drift_kind, close_on_pass} when an upstream -# release signal moves; the reporter then comments the verdict on that drift -# issue and, for release-type drift, closes it on a pass. +# Drift sweep: the drift scout (gatewaystack-connect, apps/tenant-gateway/ +# src/drift) files one `drift` issue per upstream release with the marker +# ``. No dispatch token: +# after each leg reports, scripts/canary-report.mjs lists the open `drift` +# issues and resolves the ones for this leg's registry entries at the +# version the run just tested (pass -> comment + close; fail -> one comment +# per failure). The scheduled run therefore answers a release issue within +# ~6 h. SDK legs pass the upstream versions they installed as --tested +# pairs, since their own --version is the ACP package's. Dispatch pairing +# (repository_dispatch harness-release with client_payload.drift_issue) is +# still honoured for runs triggered by hand. # # Live legs (one governed `echo `, then the audit-row assert): # claude-code claude-acp -p, BYO subscription OAuth (CLAUDE_CODE_OAUTH_TOKEN) @@ -340,6 +346,19 @@ jobs: esac echo "resolved: $v" echo "version=$v" >> "$GITHUB_OUTPUT" + # Upstream versions this leg actually installed, as drift-registry + # entryId=version pairs, so the reporter's drift sweep can match + # the scout's release issues (its own version above is the ACP + # package's). Entry ids and leg mapping: registry.ts canaryLeg. + pyver() { pip show "$1" 2>/dev/null | sed -n 's/^Version: //p' || true; } + u="" + case "$LEG" in + sdk-langchain) u="langchain-core=$(pyver langchain-core),langgraph=$(pyver langgraph)" ;; + sdk-crewai) u="crewai=$(pyver crewai)" ;; + sdk-governance-anthropic) u="anthropic-sdk-typescript=$(npm ls @anthropic-ai/sdk --depth=1 2>/dev/null | grep -o '@anthropic-ai/sdk@[0-9.]*' | head -1 | sed 's/.*@//' || true)" ;; + esac + echo "upstream: ${u:-(none mapped)}" + echo "upstream=$u" >> "$GITHUB_OUTPUT" echo "CANARY_SINCE=$(date -u +%Y-%m-%dT%H:%M:%S.000Z)" >> "$GITHUB_ENV" - name: One governed tool-call check @@ -372,6 +391,7 @@ jobs: CANARY_ISSUES_TOKEN: ${{ secrets.CANARY_ISSUES_TOKEN }} RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} PKG_VERSION: ${{ steps.pkg.outputs.version }} + UPSTREAM_VERSIONS: ${{ steps.pkg.outputs.upstream }} DRIFT_ISSUE: ${{ github.event.client_payload.drift_issue || '' }} DRIFT_VERSION: ${{ github.event.client_payload.version || '' }} DRIFT_KIND: ${{ github.event.client_payload.drift_kind || '' }} @@ -394,5 +414,6 @@ jobs: done node scripts/canary-report.mjs --harness "$LEG" --status "$status" --version "$PKG_VERSION" \ --run-url "$RUN_URL" --failed-step "${failed:-none}" ${log:+--log "$log"} \ + ${UPSTREAM_VERSIONS:+--tested "$UPSTREAM_VERSIONS"} \ ${DRIFT_ISSUE:+--drift-issue "$DRIFT_ISSUE"} ${DRIFT_VERSION:+--drift-version "$DRIFT_VERSION"} \ ${DRIFT_KIND:+--drift-kind "$DRIFT_KIND"} ${DRIFT_CLOSE:+--drift-close "$DRIFT_CLOSE"} diff --git a/docs/harness-canary.md b/docs/harness-canary.md index 410b268..96869f6 100644 --- a/docs/harness-canary.md +++ b/docs/harness-canary.md @@ -114,30 +114,61 @@ Triggers: `schedule: 13 */6 * * *`, `workflow_dispatch` (input `harness`: `harness-release` (optional `client_payload.harness`, same values). Concurrency is per leg; legs do not cancel each other. -### Drift pairing +### Drift sweep (how a drift issue gets resolved) The drift scout (gatewaystack-connect, `apps/tenant-gateway/src/drift/`, -`docs/drift-scout.md` there) sends the `harness-release` dispatch when an -upstream release signal (npm latest, GitHub release/tag, PyPI) of a harness -or SDK moves, with `client_payload` -`{harness, drift_issue, version, drift_kind: "release"|"docs", close_on_pass}`. -The workflow passes those through to the reporter as `--drift-issue`, -`--drift-version`, `--drift-kind`, `--drift-close`, and the reporter comments -the verdict on that drift issue in `davidcrowe/gatewaystack-connect`: - -- pass: `Canary on : ✅ passed: safe to close` and, when - closing is on, closes the drift issue (`state_reason: completed`); -- fail: `Canary on : ❌ failed at : see ` - (the run URL when no canary issue exists). - -Closing on pass defaults ON for `drift_kind: release` (the only kind the scout -dispatches) and OFF for `drift_kind: docs`; `close_on_pass: false` in the -payload (or `--drift-close off`) turns it off either way. The drift comment is -best effort: a failure there is logged and never fails the reporter, so the -canary issue stays the primary record. `CANARY_ISSUES_TOKEN` already covers -it (same repo, Issues: read and write). The scout's own dispatch credential -(`CANARY_DISPATCH_TOKEN`, Actions: read and write on this repo) is documented -in gatewaystack-connect `docs/drift-scout.md`. +`docs/drift-scout.md` there) files **one issue per upstream release** in +`davidcrowe/gatewaystack-connect` when a release signal (npm latest, GitHub +release/tag, PyPI) of a harness or SDK moves: label `drift`, body marker +``. It does not trigger +the canary. **The scheduled canary resolves matching drift issues within +~6 h; no extra token.** Each leg installs `@latest`, so by the next scheduled +run it is testing exactly the version the scout saw. + +After every leg reports, `scripts/canary-report.mjs` lists the open `drift` +issues, keeps the ones whose release marker names a registry entry mapped to +this leg (`LEG_ENTRIES` in the script mirrors `canaryLeg` in the scout's +`registry.ts`: `claude-code`, `codex`, `opencode`, `qwen-code`, `pi`, +`prime-agent`, `grok`, `dsh`, `hermes`, `openclaw`, `muse` map to the leg of +the same name; `crewai` to `sdk-crewai`; `langchain-core` and `langgraph` to +`sdk-langchain`; `anthropic-sdk-typescript` to `sdk-governance-anthropic`) +and whose normalised version (first dotted number, the scout's +`normalizeVersion` rule, so `v1.1.0`, `rust-v1.1.0`, `1.1.0 (Claude Code)` +all read `1.1.0`) equals the version this run tested, and posts the verdict: + +- pass: `Canary on : ✅ passed — closing.` and closes the + issue (`state_reason: completed`); +- fail: `Canary on : ❌ failed at : `, once per failure: when the reporter's last verdict comment on + that issue already says failed for the same version, a later red run on + the same version does not repeat it. + +Docs-type drift issues (markers that are not `release:`) are left alone: a +green canary says nothing about a changed docs page. For harness legs the +tested version is the harness's own `--version` output. For SDK legs the +`--version` is the ACP package's, so the `install-package` step also records +the upstream versions it installed (`langchain-core`, `langgraph`, `crewai`, +`@anthropic-ai/sdk`) and passes them as `--tested entryId=version,...`; an +SDK entry with no recorded version is skipped. The sweep is best effort: a +failure there is logged and never fails the reporter, so the canary issue +stays the primary record. `CANARY_ISSUES_TOKEN` covers it (same repo, +Issues: read and write). + +Dispatch pairing is kept for hand-triggered runs: a `repository_dispatch` +`harness-release` with `client_payload` +`{harness, drift_issue, version, drift_kind: "release"|"docs", close_on_pass}` +reaches the reporter as `--drift-issue`, `--drift-version`, `--drift-kind`, +`--drift-close`, and the verdict is posted on that issue (closed on pass for +`release` drift unless `close_on_pass: false`); the sweep skips that issue +so it is not commented twice. + +Preview the sweep locally with a fixture (`{drift: [...], canary: [...], +comments: [...]}` is what dry-run `GET`s return): + +``` +node scripts/canary-report.mjs --harness qwen-code --status success --version "0.9.0 (Qwen Code)" \ + --run-url https://example.invalid/run --dry-run --fixture fixture.json +``` ## Secrets (repository secrets on agentic-control-plane/acp-install) diff --git a/scripts/canary-report.mjs b/scripts/canary-report.mjs index 65e940d..28ebc1f 100644 --- a/scripts/canary-report.mjs +++ b/scripts/canary-report.mjs @@ -1,11 +1,13 @@ #!/usr/bin/env node // Harness canary reporter: one deduped GitHub issue per harness in -// davidcrowe/gatewaystack-connect. +// davidcrowe/gatewaystack-connect, plus the sweep that resolves the drift +// scout's release issues for the version this run tested. // // canary-report.mjs --harness --status --version // --run-url [--failed-step ] [--log ] +// [--tested ] // [--drift-issue [--drift-version ] [--drift-kind release|docs] [--drift-close on|off]] -// [--dry-run] +// [--dry-run [--fixture ]] // // Dedup: the open issue labelled `canary` + `harness:` (and, as a // belt-and-braces check, carrying the `` @@ -13,18 +15,36 @@ // with an open issue -> comment the run URL and version. Success with an // open issue -> comment and close. Success with none -> nothing to do. // -// Drift pairing: when the run was dispatched by the drift scout -// (repository_dispatch harness-release with client_payload.drift_issue), -// the verdict is also posted as a comment on that drift issue: -// "Canary on : ✅ passed: safe to close", or -// "Canary on : ❌ failed at : see ". -// On a pass the drift issue is closed too when --drift-close is on. Default: -// on for release-type drift (`--drift-kind release`, the only kind the scout -// dispatches today), off for docs-page drift (`--drift-kind docs`), since a -// green canary says nothing about a changed docs page. +// Drift sweep (no dispatch token needed): the drift scout (gatewaystack- +// connect, apps/tenant-gateway/src/drift) files ONE issue per upstream +// release, label `drift`, body marker +// ``. After reporting +// the leg, this script lists the open `drift` issues, keeps those whose +// marker names a registry entry mapped to this leg (LEG_ENTRIES below +// mirrors `canaryLeg` in the scout's registry.ts) and whose normalised +// version equals the version this run tested, and posts the verdict: +// pass: "Canary on : ✅ passed — closing." + close (completed) +// fail: "Canary on : ❌ failed at : " once per +// failure (not repeated when the reporter's last comment on that +// issue already says failed for the same version). +// Docs-type drift issues (non-release markers) are never touched. The +// scheduled run (every 6 h, latest versions) is what resolves a release +// issue, within ~6 h of the scout filing it. +// +// Drift pairing (kept): when the run was dispatched by the scout with +// client_payload.drift_issue, the verdict is posted on that issue too, and +// the sweep skips it so it is not commented twice. +// +// --tested: for SDK legs `--version` is the ACP package's version, not the +// upstream's; the workflow passes the upstream versions it actually +// installed as `entryId=version` pairs (e.g. `langchain-core=1.6.4`). An +// SDK entry without a pair is skipped by the sweep. Harness legs use +// --version (the harness's own `--version` output, normalised). // // Auth: CANARY_ISSUES_TOKEN (a fine-grained PAT with Issues: read/write on -// the target repo). --dry-run prints the requests instead of sending them. +// the target repo). --dry-run prints the requests instead of sending them; +// --fixture ({drift: [...issues], canary: [...issues], comments: +// [...]}) is what dry-run GETs return, so the sweep can be previewed. import fs from "node:fs"; @@ -38,12 +58,14 @@ const opt = (name, dflt) => { return i >= 0 && i + 1 < args.length ? args[i + 1] : dflt; }; const dryRun = args.includes("--dry-run"); +const fixtureFile = opt("fixture", ""); const harness = opt("harness"); const status = opt("status"); const version = opt("version", "unknown"); const runUrl = opt("run-url", ""); const failedStep = opt("failed-step", "none"); const logFile = opt("log", ""); +const tested = opt("tested", ""); const driftIssue = Number(opt("drift-issue", "")) || 0; const driftVersion = opt("drift-version", "") || version; const driftKind = opt("drift-kind", "release") === "docs" ? "docs" : "release"; @@ -65,14 +87,66 @@ if (!token && !dryRun) { process.exit(2); } +// Canary leg -> drift-registry entry ids. Mirror of `canaryLeg` in +// gatewaystack-connect apps/tenant-gateway/src/drift/registry.ts; keep in +// step when an entry gains or changes its leg. +const LEG_ENTRIES = { + "claude-code": ["claude-code"], + codex: ["codex"], + opencode: ["opencode"], + "qwen-code": ["qwen-code"], + pi: ["pi"], + "prime-agent": ["prime-agent"], + grok: ["grok"], + dsh: ["dsh"], + hermes: ["hermes"], + openclaw: ["openclaw"], + muse: ["muse"], + "sdk-crewai": ["crewai"], + "sdk-langchain": ["langchain-core", "langgraph"], + "sdk-governance-anthropic": ["anthropic-sdk-typescript"], +}; +const DRIFT_LABEL = "drift"; +const RELEASE_MARKER = /drift-scout:key=release:([^@\s]+)@([^\s<]*)/; +const VERDICT_MARKER = (v, result) => ``; + +// Same rule as normalizeVersion in the scout's issue.ts: "v1.1.0", +// "rust-v0.150.0", "langchain-core==1.6.4", "2.0.14 (Claude Code)" -> +// "1.1.0", "0.150.0", "1.6.4", "2.0.14". Pre-release suffixes survive. +function normalizeVersion(s) { + const t = (s ?? "").trim(); + const m = t.match(/\d+(?:\.\d+)+(?:[-+][0-9A-Za-z.+-]*)?/); + return m ? m[0] : t.replace(/^v(?=\d)/i, ""); +} + +const testedPairs = Object.fromEntries( + tested.split(",").map((s) => s.trim()).filter((s) => s.includes("=")) + .map((s) => [s.slice(0, s.indexOf("=")).trim(), s.slice(s.indexOf("=") + 1).trim()]) + .filter(([, v]) => v), +); +// The normalised version this run tested for one registry entry, or null +// when it cannot be known (an SDK entry the workflow passed no pair for). +function testedVersionFor(entryId) { + if (testedPairs[entryId]) return normalizeVersion(testedPairs[entryId]); + if (isSdk) return null; + return normalizeVersion(version); +} + const marker = ``; const labels = ["canary", `harness:${harness}`]; const stamp = new Date().toISOString(); +let fixture = null; +if (dryRun && fixtureFile) fixture = JSON.parse(fs.readFileSync(fixtureFile, "utf8")); + async function gh(method, path, body) { if (dryRun) { console.log(`[dry-run] ${method} ${path}${body ? "\n" + JSON.stringify(body, null, 2) : ""}`); - return method === "GET" ? [] : { html_url: "(dry-run)" }; + if (method !== "GET") return { html_url: "(dry-run)" }; + if (!fixture) return []; + if (path.includes("/comments")) return fixture.comments ?? []; + if (path.includes(`labels=${DRIFT_LABEL}&`) || path.endsWith(`labels=${DRIFT_LABEL}`)) return fixture.drift ?? []; + return fixture.canary ?? []; } const res = await fetch(`${API}${path}`, { method, @@ -156,6 +230,19 @@ async function reportCanaryIssue() { return null; } +// The verdict comment for one drift issue. `v` is the version as the drift +// issue states it (normalised); the hidden marker is what the sweep's +// dedupe reads back. +function verdictComment(v, canaryIssueUrl) { + const head = `Canary \`${harness}\` on \`${v}\``; + const tail = `\n\n${runLine}`; + if (status === "success") { + return { pass: true, body: `${VERDICT_MARKER(v, "passed")}\n${head}: ✅ passed — closing.${tail}` }; + } + const see = canaryIssueUrl || runUrl || "(no link)"; + return { pass: false, body: `${VERDICT_MARKER(v, "failed")}\n${head}: ❌ failed at \`${failedStep}\`: ${see}${tail}` }; +} + // Posts the verdict on the drift issue that dispatched this run. Best // effort: a failure here is logged and does not fail the reporter, so the // canary issue (the primary record) is never lost to a drift-side hiccup. @@ -166,7 +253,7 @@ async function reportToDriftIssue(canaryIssueUrl) { try { if (status === "success") { await gh("POST", `/repos/${REPO}/issues/${driftIssue}/comments`, { - body: `${head}: ✅ passed: safe to close${driftClose ? " — closing." : "."}${tail}`, + body: `${VERDICT_MARKER(normalizeVersion(driftVersion), "passed")}\n${head}: ✅ passed: safe to close${driftClose ? " — closing." : "."}${tail}`, }); if (driftClose) { await gh("PATCH", `/repos/${REPO}/issues/${driftIssue}`, { state: "closed", state_reason: "completed" }); @@ -178,7 +265,7 @@ async function reportToDriftIssue(canaryIssueUrl) { } const see = canaryIssueUrl ?? runUrl ?? "(no link)"; await gh("POST", `/repos/${REPO}/issues/${driftIssue}/comments`, { - body: `${head}: ❌ failed at \`${failedStep}\`: see ${see}${tail}`, + body: `${VERDICT_MARKER(normalizeVersion(driftVersion), "failed")}\n${head}: ❌ failed at \`${failedStep}\`: see ${see}${tail}`, }); console.log(`drift #${driftIssue}: failed at ${failedStep}, commented`); } catch (e) { @@ -186,9 +273,67 @@ async function reportToDriftIssue(canaryIssueUrl) { } } +// True when the reporter's most recent verdict on this issue already says +// "failed" for the same version: a later red run on the same version adds +// nothing, so it is not repeated. A pass, or a verdict for another version, +// does not suppress. +async function alreadyFailedHere(issueNumber, v) { + const q = new URLSearchParams({ per_page: "100" }); + const list = await gh("GET", `/repos/${REPO}/issues/${issueNumber}/comments?${q}`); + const mine = (Array.isArray(list) ? list : []).filter((c) => String(c.body ?? "").includes(`harness-canary:verdict leg=${harness} `)); + const last = mine[mine.length - 1]; + return !!last && String(last.body).includes(VERDICT_MARKER(v, "failed")); +} + +// Finds every open release-type drift issue for an entry mapped to this leg +// at the version this run tested and posts the verdict (close on pass). +// Best effort, same as reportToDriftIssue. +async function sweepDriftIssues(canaryIssueUrl) { + const entries = LEG_ENTRIES[harness]; + if (!entries) { console.log(`drift sweep: no registry entry maps to leg ${harness}`); return; } + const want = new Map(); + for (const id of entries) { + const v = testedVersionFor(id); + if (v) want.set(id, v); + } + if (want.size === 0) { console.log(`drift sweep: no tested upstream version known for ${entries.join(", ")} (pass --tested)`); return; } + console.log(`drift sweep: ${[...want].map(([id, v]) => `${id}@${v}`).join(", ")}`); + try { + const q = new URLSearchParams({ state: "open", labels: DRIFT_LABEL, per_page: "100" }); + const list = await gh("GET", `/repos/${REPO}/issues?${q}`); + const issues = (Array.isArray(list) ? list : []).filter((i) => !i.pull_request); + let touched = 0; + for (const issue of issues) { + if (issue.number === driftIssue) continue; // already handled by the pairing path + const m = String(issue.body ?? "").match(RELEASE_MARKER); + if (!m) continue; // docs-type drift: a green canary says nothing about it + const [, entryId, markerVersion] = m; + const v = want.get(entryId); + if (!v || normalizeVersion(markerVersion) !== v) continue; + touched++; + const { pass, body } = verdictComment(v, canaryIssueUrl); + if (!pass && await alreadyFailedHere(issue.number, v)) { + console.log(`drift #${issue.number} (${entryId}@${v}): already marked failed on ${v}, not repeating`); + continue; + } + await gh("POST", `/repos/${REPO}/issues/${issue.number}/comments`, { body }); + if (pass) { + await gh("PATCH", `/repos/${REPO}/issues/${issue.number}`, { state: "closed", state_reason: "completed" }); + console.log(`drift #${issue.number} (${entryId}@${v}): passed, closed`); + } else { + console.log(`drift #${issue.number} (${entryId}@${v}): failed at ${failedStep}, commented`); + } + } + if (touched === 0) console.log(`drift sweep: no open release drift issue matches (${issues.length} open drift issues scanned)`); + } catch (e) { + console.error(`drift sweep: ${e.message ?? e}`); + } +} + async function main() { const canaryIssueUrl = await reportCanaryIssue(); await reportToDriftIssue(canaryIssueUrl); + await sweepDriftIssues(canaryIssueUrl); } main().catch((e) => { console.error(e.message ?? e); process.exit(1); });