From 1337d7754e0751c91b204f222c98101f75fd4930 Mon Sep 17 00:00:00 2001 From: Ovtcharov Date: Tue, 18 Aug 2026 01:06:10 -0700 Subject: [PATCH 1/2] ci(hub): deploy the Agent Hub Worker before publishing to it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Publishing hub components has been broken since July and the error blamed the wrong thing. Every run died on: invalid_manifest: language "go" is not supported. Use one of: cpp, python but that list is rendered from VALID_LANGUAGES, which has included `go` and `typescript` since #2530 — the commit that added terminal-hub and agent-ui as R2 packages in the first place. The message quoted code that no longer exists, because the deployed Worker was months behind the manifests it was validating. There was no way for it not to drift. agent_hub_worker_ci.yml only type-checks and runs vitest; the sole deploy path is a human running `wrangler deploy`. So the validator silently ages until a release trips over it, which is what happened three times before someone disabled the workflow. release_components.yml now deploys the Worker before it uploads anything, and both publish jobs gate on that. Deploying here rather than on push-to-main is deliberate: the job sits in the agent-publish environment, so a production Worker deploy still needs a reviewer and lands in the same approval as the upload it has to match. Dry runs skip the deploy and still run every build and validation step — the publish jobs admit a *skipped* dependency but never a failed one. Needs CLOUDFLARE_API_TOKEN and CLOUDFLARE_ACCOUNT_ID on the agent-publish environment; the job fails with both names and where to get them rather than publishing to a stale Worker. A post-deploy health check on hub.amd-gaia.ai asserts what is actually live, since the whole failure mode was that nobody ever checked. --- .github/workflows/release_components.yml | 84 +++++++++++++++++++++++- workers/agent-hub/README.md | 17 +++++ 2 files changed, 99 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release_components.yml b/.github/workflows/release_components.yml index a339f2128..647ee13d2 100644 --- a/.github/workflows/release_components.yml +++ b/.github/workflows/release_components.yml @@ -115,10 +115,84 @@ jobs: --verify-released \ --release-version "${VERSION}" + # ── Bring the Worker up to date before anything is uploaded to it ─────────── + # The Worker validates every manifest it accepts, so a Worker older than the + # manifests rejects a release that is perfectly valid. That is not theoretical: + # `go` and `typescript` were added to VALID_LANGUAGES in July, the Worker was + # never redeployed, and every publish since died on + # invalid_manifest: language "go" is not supported. Use one of: cpp, python + # — an error quoting a list that no longer exists in the source. There is no + # other deploy path (agent_hub_worker_ci.yml only type-checks and tests), so + # the Worker drifts until someone runs `wrangler deploy` by hand. + # + # Deploying here rather than on push-to-main is deliberate: this job inherits + # the agent-publish reviewer gate, so a production Worker deploy still needs a + # human, and it lands in the same approval as the upload it has to match. + deploy-worker: + name: Deploy Agent Hub Worker + needs: version + if: needs.version.outputs.dry_run == 'false' + runs-on: ubuntu-latest + environment: agent-publish + steps: + - uses: actions/checkout@v7 + + - uses: actions/setup-node@v6 + with: + node-version: '20' + + - name: Install Worker dependencies + working-directory: workers/agent-hub + run: npm ci + + # Type-check before deploying: agent_hub_worker_ci.yml gates PRs, but a + # dispatch can run from any ref, and shipping a Worker that fails tsc + # would break publishing for every agent, not just this release. + - name: Type-check + working-directory: workers/agent-hub + run: npx tsc --noEmit + + - name: Deploy + working-directory: workers/agent-hub + env: + CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} + CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} + run: | + set -euo pipefail + if [ -z "${CLOUDFLARE_API_TOKEN:-}" ]; then + echo "::error::missing environment secret CLOUDFLARE_API_TOKEN on the agent-publish environment. The Agent Hub Worker validates the manifests this workflow uploads, so publishing without deploying it first is how a stale validator rejects a valid release. Create a Cloudflare API token with the 'Edit Cloudflare Workers' template and add it (plus CLOUDFLARE_ACCOUNT_ID) to the agent-publish environment. See workers/agent-hub/README.md." + exit 1 + fi + npx wrangler deploy + echo "✓ agent-hub Worker deployed from this ref" + + # Prove the deployed Worker is the one we just built, not a cached edge + # version — the failure this job exists to prevent was invisible until a + # publish failed, because nothing ever asserted what was live. + - name: Verify the live Worker answers + run: | + set -euo pipefail + for i in 1 2 3 4 5; do + if curl -fsS --max-time 15 https://hub.amd-gaia.ai/health >/dev/null; then + echo "✓ hub.amd-gaia.ai/health OK" + exit 0 + fi + echo "health check attempt $i failed; retrying in 5s" + sleep 5 + done + echo "::error::the Worker deployed but hub.amd-gaia.ai/health did not answer. Check the Cloudflare dashboard before re-running the publish — uploading to a half-deployed Worker is how partial state gets into an immutable path." + exit 1 + # ── terminal-hub: build the 6 targets and publish them ────────────────────── terminal-hub: name: Publish terminal-hub - needs: version + needs: [version, deploy-worker] + # deploy-worker is skipped on a dry run; a skipped dependency would skip this + # job too, so admit 'skipped' explicitly while still refusing 'failure'. + if: | + always() + && needs.version.result == 'success' + && (needs.deploy-worker.result == 'success' || needs.deploy-worker.result == 'skipped') runs-on: ubuntu-latest # agent-publish gates on a deployment tag allowlist that lives in repo settings, # not here: it must include v* or every release tag is rejected before any step @@ -225,7 +299,13 @@ jobs: # ── agent-ui: republish the installers build-installers.yml already made ──── agent-ui: name: Publish agent-ui - needs: version + needs: [version, deploy-worker] + # deploy-worker is skipped on a dry run; a skipped dependency would skip this + # job too, so admit 'skipped' explicitly while still refusing 'failure'. + if: | + always() + && needs.version.result == 'success' + && (needs.deploy-worker.result == 'success' || needs.deploy-worker.result == 'skipped') runs-on: ubuntu-latest # Same tag-allowlist requirement as terminal-hub above (#2935). environment: agent-publish diff --git a/workers/agent-hub/README.md b/workers/agent-hub/README.md index 9a35ba5fc..257272c5d 100644 --- a/workers/agent-hub/README.md +++ b/workers/agent-hub/README.md @@ -244,6 +244,23 @@ checked into the repo: npx wrangler deploy ``` + CI does this for you on a real publish. `release_components.yml`'s + `deploy-worker` job deploys this Worker before it uploads anything, because + the Worker *validates* the manifests being uploaded — a Worker older than the + manifests rejects a valid release. That is not hypothetical: `go` and + `typescript` were added to `VALID_LANGUAGES` and the Worker was not + redeployed, so every publish failed with `language "go" is not supported` + while the source said otherwise. + + The job needs two secrets on the **`agent-publish`** environment: + + | Secret | How to get it | + |---|---| + | `CLOUDFLARE_API_TOKEN` | Cloudflare dashboard → My Profile → API Tokens → Create Token → **Edit Cloudflare Workers** template. Must also cover R2 for the bucket binding. | + | `CLOUDFLARE_ACCOUNT_ID` | Cloudflare dashboard → Workers & Pages → Account ID | + + Without them the job fails loudly rather than publishing to a stale Worker. + 4. **(Optional) Bind the route** by uncommenting the `routes` line in `wrangler.toml` to serve the API under `hub.amd-gaia.ai/*`. From 39cb4f48e6b55543abca8468c1f2a2a6238e38f5 Mon Sep 17 00:00:00 2001 From: Ovtcharov Date: Tue, 18 Aug 2026 09:29:07 -0700 Subject: [PATCH 2/2] =?UTF-8?q?ci(hub):=20address=20review=20=E2=80=94=20c?= =?UTF-8?q?ancel-safety,=20a=20real=20deploy=20check,=20and=20a=20rehearsa?= =?UTF-8?q?l?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Five fixes from review, one of which was a genuine hole. `always()` on the publish jobs survives a cancelled run, so hitting Cancel mid-release could still POST to /publish — into an immutable path. Now `!cancelled()`, which keeps the skipped-dependency handling while restoring Cancel as a stop. The post-deploy check could not detect the failure it was written for. /health returned a fixed `{"status":"ok"}`, which the months-stale Worker would have answered exactly the same way. Deploys now stamp the commit into WORKER_BUILD, /health reports it, and the workflow asserts the live build equals the one it just pushed. It also curls the same base URL the publish steps resolve rather than the custom domain, because CI publishes through workers.dev — the WAF on hub.amd-gaia.ai blocks large multipart uploads — so that is the origin whose freshness actually matters. The deploy job could never be rehearsed: it is skipped on a dry run, so its first execution would have been a live release. Split out a worker-check job that runs on every dispatch, unapproved, and does typecheck + the vitest suite + `deploy:dry-run`. That also answers the review's point that gating on `tsc` while skipping the suite guarding the /publish contract was backwards; the deploy job now inherits a checked bundle instead of re-deriving a weaker one. Smaller: validate both Cloudflare secrets rather than one and let wrangler fail obscurely on the other; setup-node v7 with npm caching to match the sibling workflow; and correct agent_hub_worker_ci.yml's header, which still claimed deploys were manual. --- .github/workflows/agent_hub_worker_ci.yml | 5 +- .github/workflows/release_components.yml | 106 ++++++++++++++++------ workers/agent-hub/README.md | 9 ++ workers/agent-hub/src/index.ts | 5 +- workers/agent-hub/src/types.ts | 7 ++ workers/agent-hub/test/routes.test.ts | 15 +++ 6 files changed, 115 insertions(+), 32 deletions(-) diff --git a/.github/workflows/agent_hub_worker_ci.yml b/.github/workflows/agent_hub_worker_ci.yml index d2dac88e6..773f2f4be 100644 --- a/.github/workflows/agent_hub_worker_ci.yml +++ b/.github/workflows/agent_hub_worker_ci.yml @@ -5,7 +5,10 @@ # The worker's test suite guards the shared catalog contract (POST /publish, # index.json shape) that every hub agent release and the website depend on — # previously it was local-only, so a contract regression could reach main -# without any CI ever running the suite. Deploys stay manual; this only tests. +# without any CI ever running the suite. This workflow only tests; the Worker +# is deployed by release_components.yml's deploy-worker job, immediately +# before it publishes to it (a Worker older than the manifests it validates +# rejects valid releases). name: Agent Hub Worker CI diff --git a/.github/workflows/release_components.yml b/.github/workflows/release_components.yml index 647ee13d2..3d24a5539 100644 --- a/.github/workflows/release_components.yml +++ b/.github/workflows/release_components.yml @@ -115,6 +115,42 @@ jobs: --verify-released \ --release-version "${VERSION}" + # ── Prove the Worker builds, before anything decides to deploy it ─────────── + # Runs on dry runs too, which is the point: the deploy job below is skipped on + # a dry run, so without this its code path would first execute during a live + # release. `deploy:dry-run` bundles and resolves wrangler.toml exactly as a + # real deploy does, minus the upload. + worker-check: + name: Check Agent Hub Worker + needs: version + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + + - uses: actions/setup-node@v7 + with: + node-version: '20' + cache: npm + cache-dependency-path: workers/agent-hub/package-lock.json + + - working-directory: workers/agent-hub + run: npm ci + + # `npm run typecheck` not `tsc --noEmit`: the script also checks + # tsconfig.test.json, and the suite below is what actually guards the + # /publish contract every hub release depends on. + - name: Type-check + working-directory: workers/agent-hub + run: npm run typecheck + + - name: Unit tests (vitest) + working-directory: workers/agent-hub + run: npm test + + - name: Bundle (dry-run deploy) + working-directory: workers/agent-hub + run: npm run deploy:dry-run + # ── Bring the Worker up to date before anything is uploaded to it ─────────── # The Worker validates every manifest it accepts, so a Worker older than the # manifests rejects a release that is perfectly valid. That is not theoretical: @@ -122,36 +158,30 @@ jobs: # never redeployed, and every publish since died on # invalid_manifest: language "go" is not supported. Use one of: cpp, python # — an error quoting a list that no longer exists in the source. There is no - # other deploy path (agent_hub_worker_ci.yml only type-checks and tests), so - # the Worker drifts until someone runs `wrangler deploy` by hand. + # other deploy path (agent_hub_worker_ci.yml only tests), so the Worker drifts + # until someone runs `wrangler deploy` by hand. # # Deploying here rather than on push-to-main is deliberate: this job inherits # the agent-publish reviewer gate, so a production Worker deploy still needs a # human, and it lands in the same approval as the upload it has to match. deploy-worker: name: Deploy Agent Hub Worker - needs: version + needs: [version, worker-check] if: needs.version.outputs.dry_run == 'false' runs-on: ubuntu-latest environment: agent-publish steps: - uses: actions/checkout@v7 - - uses: actions/setup-node@v6 + - uses: actions/setup-node@v7 with: node-version: '20' + cache: npm + cache-dependency-path: workers/agent-hub/package-lock.json - - name: Install Worker dependencies - working-directory: workers/agent-hub + - working-directory: workers/agent-hub run: npm ci - # Type-check before deploying: agent_hub_worker_ci.yml gates PRs, but a - # dispatch can run from any ref, and shipping a Worker that fails tsc - # would break publishing for every agent, not just this release. - - name: Type-check - working-directory: workers/agent-hub - run: npx tsc --noEmit - - name: Deploy working-directory: workers/agent-hub env: @@ -159,39 +189,52 @@ jobs: CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} run: | set -euo pipefail - if [ -z "${CLOUDFLARE_API_TOKEN:-}" ]; then - echo "::error::missing environment secret CLOUDFLARE_API_TOKEN on the agent-publish environment. The Agent Hub Worker validates the manifests this workflow uploads, so publishing without deploying it first is how a stale validator rejects a valid release. Create a Cloudflare API token with the 'Edit Cloudflare Workers' template and add it (plus CLOUDFLARE_ACCOUNT_ID) to the agent-publish environment. See workers/agent-hub/README.md." + missing="" + [ -n "${CLOUDFLARE_API_TOKEN:-}" ] || missing="${missing} CLOUDFLARE_API_TOKEN" + [ -n "${CLOUDFLARE_ACCOUNT_ID:-}" ] || missing="${missing} CLOUDFLARE_ACCOUNT_ID" + if [ -n "${missing}" ]; then + echo "::error::missing environment secret(s) on agent-publish:${missing}. The Agent Hub Worker validates the manifests this workflow uploads, so publishing without deploying it first is how a stale validator rejects a valid release. Create a Cloudflare API token from the 'Edit Cloudflare Workers' template (it must also cover R2 for the bucket binding) and add both secrets to the agent-publish environment. See workers/agent-hub/README.md." exit 1 fi - npx wrangler deploy - echo "✓ agent-hub Worker deployed from this ref" + # Stamp the commit so the check below can prove WHICH build went live. + npx wrangler deploy --var WORKER_BUILD:"${GITHUB_SHA}" + echo "✓ deployed ${GITHUB_SHA}" - # Prove the deployed Worker is the one we just built, not a cached edge - # version — the failure this job exists to prevent was invisible until a - # publish failed, because nothing ever asserted what was live. - - name: Verify the live Worker answers + # Assert the live Worker is the build we just pushed. Curling the same base + # URL the publish steps use, not the custom domain: CI publishes through + # workers.dev because the WAF on hub.amd-gaia.ai blocks large multipart + # uploads (wrangler.toml), so that is the origin whose freshness matters. + - name: Verify the deployed build is live + env: + GAIA_HUB_PUBLISH_URL: ${{ vars.GAIA_HUB_PUBLISH_URL }} + GAIA_HUB_BASE_URL: ${{ vars.GAIA_HUB_BASE_URL }} run: | set -euo pipefail - for i in 1 2 3 4 5; do - if curl -fsS --max-time 15 https://hub.amd-gaia.ai/health >/dev/null; then - echo "✓ hub.amd-gaia.ai/health OK" + base="${GAIA_HUB_PUBLISH_URL:-${GAIA_HUB_BASE_URL:-https://hub.amd-gaia.ai}}" + for i in 1 2 3 4 5 6; do + live="$(curl -fsS --max-time 15 "${base}/health" | python -c 'import json,sys; print(json.load(sys.stdin).get("build",""))' 2>/dev/null || true)" + if [ "${live}" = "${GITHUB_SHA}" ]; then + echo "✓ ${base} is serving ${GITHUB_SHA}" exit 0 fi - echo "health check attempt $i failed; retrying in 5s" + echo "attempt ${i}: ${base}/health reports build='${live:-}', want '${GITHUB_SHA}'; retrying in 5s" sleep 5 done - echo "::error::the Worker deployed but hub.amd-gaia.ai/health did not answer. Check the Cloudflare dashboard before re-running the publish — uploading to a half-deployed Worker is how partial state gets into an immutable path." + echo "::error::${base} is not serving the build just deployed (wanted ${GITHUB_SHA}). Publishing now would upload against a Worker whose manifest validator is not the one in this ref — the exact failure this job exists to prevent. Check the Cloudflare dashboard before re-running." exit 1 # ── terminal-hub: build the 6 targets and publish them ────────────────────── terminal-hub: name: Publish terminal-hub - needs: [version, deploy-worker] + needs: [version, worker-check, deploy-worker] # deploy-worker is skipped on a dry run; a skipped dependency would skip this # job too, so admit 'skipped' explicitly while still refusing 'failure'. + # !cancelled() rather than always(): always() survives a cancelled run, and + # an upload that outlives Cancel lands in an immutable path. if: | - always() + !cancelled() && needs.version.result == 'success' + && needs.worker-check.result == 'success' && (needs.deploy-worker.result == 'success' || needs.deploy-worker.result == 'skipped') runs-on: ubuntu-latest # agent-publish gates on a deployment tag allowlist that lives in repo settings, @@ -299,12 +342,15 @@ jobs: # ── agent-ui: republish the installers build-installers.yml already made ──── agent-ui: name: Publish agent-ui - needs: [version, deploy-worker] + needs: [version, worker-check, deploy-worker] # deploy-worker is skipped on a dry run; a skipped dependency would skip this # job too, so admit 'skipped' explicitly while still refusing 'failure'. + # !cancelled() rather than always(): always() survives a cancelled run, and + # an upload that outlives Cancel lands in an immutable path. if: | - always() + !cancelled() && needs.version.result == 'success' + && needs.worker-check.result == 'success' && (needs.deploy-worker.result == 'success' || needs.deploy-worker.result == 'skipped') runs-on: ubuntu-latest # Same tag-allowlist requirement as terminal-hub above (#2935). diff --git a/workers/agent-hub/README.md b/workers/agent-hub/README.md index 257272c5d..599d4f0dc 100644 --- a/workers/agent-hub/README.md +++ b/workers/agent-hub/README.md @@ -261,6 +261,15 @@ checked into the repo: Without them the job fails loudly rather than publishing to a stale Worker. + The deploy stamps the commit into `WORKER_BUILD`, which `GET /health` + returns, so the workflow can assert *which* build went live instead of + assuming. Check it by hand any time: + + ```bash + curl -s https://hub.amd-gaia.ai/health + # {"status":"ok","build":""} — "unknown" means a hand-run deploy + ``` + 4. **(Optional) Bind the route** by uncommenting the `routes` line in `wrangler.toml` to serve the API under `hub.amd-gaia.ai/*`. diff --git a/workers/agent-hub/src/index.ts b/workers/agent-hub/src/index.ts index 5bfa5a3f8..904e035b8 100644 --- a/workers/agent-hub/src/index.ts +++ b/workers/agent-hub/src/index.ts @@ -39,7 +39,10 @@ async function route(request: Request, env: Env): Promise { const method = request.method.toUpperCase(); if (path === "/health") { - return json({ status: "ok" }); + // `build` is the commit this Worker was deployed from. A fixed "ok" + // cannot tell a current Worker from a months-stale one — which is how an + // outdated manifest validator went unnoticed until every release failed. + return json({ status: "ok", build: env.WORKER_BUILD ?? "unknown" }); } if (path === "/publish") { diff --git a/workers/agent-hub/src/types.ts b/workers/agent-hub/src/types.ts index 003c6a7d3..44edc186e 100644 --- a/workers/agent-hub/src/types.ts +++ b/workers/agent-hub/src/types.ts @@ -29,6 +29,13 @@ export interface Env { */ PUBLISH_TOKENS?: string; + /** + * Commit this Worker was deployed from, surfaced by `GET /health` so a + * deploy can be verified rather than assumed. Set at deploy time with + * `wrangler deploy --var WORKER_BUILD:`; absent in local dev. + */ + WORKER_BUILD?: string; + /** * Bearer token for the maintainer-only `POST /reindex` endpoint, which * rebuilds index.json from the immutable R2 objects (idempotent). Separate diff --git a/workers/agent-hub/test/routes.test.ts b/workers/agent-hub/test/routes.test.ts index 704997cab..83d7397bd 100644 --- a/workers/agent-hub/test/routes.test.ts +++ b/workers/agent-hub/test/routes.test.ts @@ -30,6 +30,21 @@ describe("GET routes", () => { expect(((await res.json()) as any).status).toBe("ok"); }); + it("reports the deployed build on /health so a stale Worker is detectable", async () => { + // The release workflow greps this for the commit it just deployed. A fixed + // "ok" would let a months-old Worker pass the post-deploy check, which is + // how an outdated manifest validator broke every publish unnoticed. + const env = { ...makeEnv(), WORKER_BUILD: "abc123" }; + const res = await worker.fetch(get("/health"), env as never); + expect(((await res.json()) as any).build).toBe("abc123"); + }); + + it("says 'unknown' rather than omitting the build when it is unset", async () => { + const env = makeEnv(); + const res = await worker.fetch(get("/health"), env as never); + expect(((await res.json()) as any).build).toBe("unknown"); + }); + it("serves /index.json after a publish", async () => { const env = makeEnv(); await seed(env);