diff --git a/.github/workflows/release_components.yml b/.github/workflows/release_components.yml index bc5267474..ae1cd61f5 100644 --- a/.github/workflows/release_components.yml +++ b/.github/workflows/release_components.yml @@ -331,7 +331,7 @@ jobs: - name: Install publisher deps if: needs.version.outputs.dry_run == 'false' - run: python -m pip install --upgrade requests pyyaml + run: python -m pip install --upgrade requests pyyaml boto3 - name: Publish to the Agent Hub (POST /publish) if: needs.version.outputs.dry_run == 'false' @@ -340,6 +340,13 @@ jobs: AGENT_HUB_PUBLISH_TOKEN: ${{ secrets.GAIA_HUB_TOKEN }} GAIA_HUB_PUBLISH_URL: ${{ vars.GAIA_HUB_PUBLISH_URL }} GAIA_HUB_BASE_URL: ${{ vars.GAIA_HUB_BASE_URL }} + # Artifacts at/over 90 MiB cannot travel through the Worker at all + # (Cloudflare caps request bodies at 100 MB on Free/Pro), so the + # publisher PUTs them straight into R2 and publishes them by + # reference. Only the Agent UI installers hit this today. + R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }} + R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }} + CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} run: | set -euo pipefail # Explicit = on every artifact: the publisher's @@ -385,6 +392,30 @@ jobs: # yet. Wait for the full set, bounded — and fail loudly on timeout rather # than publishing a partial platform set that the catalog would then # advertise as complete. + # Before the wait, not after: this job can sit for up to an hour on the + # installer assets and then pull ~400 MB, and discovering a missing secret + # at the end wastes all of it. Covers the R2 credentials too — the + # installers are large enough to take the direct-to-R2 lane, so those are + # as load-bearing here as the hub token. + - name: Require the publish credentials + if: needs.version.outputs.dry_run == 'false' + env: + GAIA_HUB_TOKEN: ${{ secrets.GAIA_HUB_TOKEN }} + R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }} + R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }} + CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} + run: | + set -euo pipefail + missing="" + [ -n "${GAIA_HUB_TOKEN:-}" ] || missing="${missing} GAIA_HUB_TOKEN" + [ -n "${R2_ACCESS_KEY_ID:-}" ] || missing="${missing} R2_ACCESS_KEY_ID" + [ -n "${R2_SECRET_ACCESS_KEY:-}" ] || missing="${missing} R2_SECRET_ACCESS_KEY" + [ -n "${CLOUDFLARE_ACCOUNT_ID:-}" ] || missing="${missing} CLOUDFLARE_ACCOUNT_ID" + if [ -n "${missing}" ]; then + echo "::error::missing secret(s):${missing}. GAIA_HUB_TOKEN must match the Worker's PUBLISH_TOKENS; the R2_* pair are R2 S3 credentials (Cloudflare -> R2 -> Manage API Tokens, Object Read & Write) needed because these installers exceed the Worker request-body cap and go straight to R2. See workers/agent-hub/README.md." + exit 1 + fi + - name: Wait for the installer assets on the release if: needs.version.outputs.dry_run == 'false' shell: bash @@ -431,20 +462,9 @@ jobs: --pattern "gaia-agent-ui-${VERSION}-x86_64.AppImage" ls -la dist - - name: Require the publish token - if: needs.version.outputs.dry_run == 'false' - env: - GAIA_HUB_TOKEN: ${{ secrets.GAIA_HUB_TOKEN }} - run: | - set -euo pipefail - if [ -z "${GAIA_HUB_TOKEN:-}" ]; then - echo "::error::missing environment secret GAIA_HUB_TOKEN on the agent-publish environment (must match the Worker's PUBLISH_TOKENS). See workers/agent-hub/README.md." - exit 1 - fi - - name: Install publisher deps if: needs.version.outputs.dry_run == 'false' - run: python -m pip install --upgrade requests pyyaml + run: python -m pip install --upgrade requests pyyaml boto3 - name: Publish to the Agent Hub (POST /publish) if: needs.version.outputs.dry_run == 'false' @@ -453,6 +473,13 @@ jobs: AGENT_HUB_PUBLISH_TOKEN: ${{ secrets.GAIA_HUB_TOKEN }} GAIA_HUB_PUBLISH_URL: ${{ vars.GAIA_HUB_PUBLISH_URL }} GAIA_HUB_BASE_URL: ${{ vars.GAIA_HUB_BASE_URL }} + # Artifacts at/over 90 MiB cannot travel through the Worker at all + # (Cloudflare caps request bodies at 100 MB on Free/Pro), so the + # publisher PUTs them straight into R2 and publishes them by + # reference. Only the Agent UI installers hit this today. + R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }} + R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }} + CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} VERSION: ${{ needs.version.outputs.version }} run: | set -euo pipefail diff --git a/hub/agents/email/python/packaging/publish_to_r2.py b/hub/agents/email/python/packaging/publish_to_r2.py index e2fc82b0a..613ba3871 100644 --- a/hub/agents/email/python/packaging/publish_to_r2.py +++ b/hub/agents/email/python/packaging/publish_to_r2.py @@ -42,6 +42,8 @@ from __future__ import annotations import argparse +import base64 +import contextlib import hashlib import json import os @@ -118,6 +120,74 @@ def _download_sha256(base_url: str, agent_id: str, version: str, filename: str) return hashlib.sha256(resp.content).hexdigest() +# Cloudflare caps a Worker request body by plan — 100 MB on Free/Pro. Anything +# at or above this goes straight to R2 over the S3 API instead, and is published +# by reference. Deliberately below the real cap so an artifact that grows into +# the limit switches lanes before it starts 413ing mid-release. +DIRECT_UPLOAD_THRESHOLD = 90 * 1024 * 1024 + + +def _r2_credentials() -> tuple[str, str, str] | None: + """R2 S3 credentials, or None when direct upload is not configured.""" + key = os.environ.get("R2_ACCESS_KEY_ID") + secret = os.environ.get("R2_SECRET_ACCESS_KEY") + account = os.environ.get("CLOUDFLARE_ACCOUNT_ID") + if key and secret and account: + return key, secret, account + return None + + +def _upload_to_r2(artifact_path: Path, key: str, sha_hex: str) -> None: + """PUT an artifact straight into the hub bucket, bypassing the Worker. + + Single-part on purpose: R2 records a whole-object SHA-256 only for + non-multipart uploads, and the Worker refuses to publish an object it cannot + verify. ``put_object`` is always single-part (``upload_file`` would switch to + multipart above its threshold and silently strip the checksum). + """ + try: + import boto3 # imported lazily: only the direct-upload path needs it + except ImportError as e: # pragma: no cover - environment problem, not logic + raise SystemExit( + "error: boto3 is required to upload artifacts larger than " + f"{DIRECT_UPLOAD_THRESHOLD} bytes directly to R2. " + "Install it (pip install boto3) and re-run." + ) from e + + creds = _r2_credentials() + if creds is None: + raise SystemExit( + f"error: {artifact_path.name} is too large to publish through the " + "Worker (Cloudflare caps request bodies at 100 MB on Free/Pro), so it " + "must go directly to R2 — but R2_ACCESS_KEY_ID, R2_SECRET_ACCESS_KEY " + "and CLOUDFLARE_ACCOUNT_ID are not all set. Create an R2 API token " + "(Cloudflare dashboard -> R2 -> Manage API Tokens) with Object " + "Read & Write on the hub bucket and set all three." + ) + access_key, secret_key, account_id = creds + bucket = os.environ.get("R2_BUCKET", "gaia-hub") + + client = boto3.client( + "s3", + endpoint_url=f"https://{account_id}.r2.cloudflarestorage.com", + aws_access_key_id=access_key, + aws_secret_access_key=secret_key, + region_name="auto", + ) + print( + f"[publish] uploading {artifact_path.name} -> r2://{bucket}/{key}", flush=True + ) + with artifact_path.open("rb") as fh: + client.put_object( + Bucket=bucket, + Key=key, + Body=fh, + ContentType="application/octet-stream", + # Base64, not hex — and this is what makes the object verifiable. + ChecksumSHA256=base64.b64encode(bytes.fromhex(sha_hex)).decode("ascii"), + ) + + def publish_one( base_url: str, manifest_path: Path, @@ -148,15 +218,48 @@ def publish_one( flush=True, ) - with artifact_path.open("rb") as fh: + # Oversized artifacts cannot travel through the Worker at all (Cloudflare + # caps the request body at 100 MB on Free/Pro), so they go straight to R2 and + # the POST below only carries their coordinates. The Worker verifies the + # stored object's size and SHA-256 before recording it, so this is a + # different transport, not a weaker guarantee. + by_reference = size >= DIRECT_UPLOAD_THRESHOLD + if by_reference: + # Check BEFORE uploading. The Worker's by-reference immutability guard + # keys on the agent manifest and therefore fires only AFTER this PUT + # would already have replaced the published bytes — leaving the catalog + # describing the old artifact while R2 serves the new one, and the 409 + # handler below re-downloading the bytes it just overwrote and happily + # agreeing with itself. Skipping the upload keeps that 409 meaningful. + download_url = f"{base_url.rstrip('/')}/agents/{agent_id}/{version}/{filename}" + head = requests.head(download_url, timeout=60, allow_redirects=True) + if head.status_code == 200: + print( + f"[publish] {filename} is already in R2 — not overwriting it; " + "the POST below verifies the stored bytes against this build.", + flush=True, + ) + else: + _upload_to_r2( + artifact_path, f"agents/{agent_id}/{version}/{filename}", local_sha + ) + + with contextlib.ExitStack() as stack: files = { "manifest": ( "gaia-agent.yaml", manifest_path.read_bytes(), "application/x-yaml", ), - "artifact": (filename, fh, "application/octet-stream"), } + if by_reference: + files["artifact_ref_filename"] = (None, filename) + files["artifact_ref_sha256"] = (None, local_sha) + files["artifact_ref_size"] = (None, str(size)) + files["artifact_ref_content_type"] = (None, "application/octet-stream") + else: + fh = stack.enter_context(artifact_path.open("rb")) + files["artifact"] = (filename, fh, "application/octet-stream") # Same multipart field name + shape the Worker expects from # `gaia agent publish` (src/gaia/hub/publisher.py): the README becomes # the catalog entry's `readme` (rendered as sanitized markdown on the diff --git a/hub/agents/email/python/tests/test_publish_to_r2_by_reference.py b/hub/agents/email/python/tests/test_publish_to_r2_by_reference.py new file mode 100644 index 000000000..813bb3114 --- /dev/null +++ b/hub/agents/email/python/tests/test_publish_to_r2_by_reference.py @@ -0,0 +1,248 @@ +# Copyright(C) 2025-2026 Advanced Micro Devices, Inc. All rights reserved. +# SPDX-License-Identifier: MIT +"""Which lane an artifact takes to the hub, and what actually goes on the wire. + +Cloudflare caps a Worker request body at 100 MB on Free/Pro, so an artifact +above that cannot be published through the Worker at all — it 413s at the edge. +The publisher therefore routes large artifacts straight into R2 and sends only +their coordinates. + +These tests assert the *shape* of both calls, not merely that they happened. A +mock that records "put_object was invoked" would still pass if the upload went +multipart, or if the checksum were omitted, or if the hex digest were sent where +base64 belongs — and every one of those makes the Worker refuse the publish, +during a release, after the binaries have been built. +""" + +from __future__ import annotations + +import base64 +import hashlib +import importlib.util +from pathlib import Path + +import pytest + +PACKAGING = Path(__file__).resolve().parents[1] / "packaging" +_spec = importlib.util.spec_from_file_location( + "email_publish_to_r2", PACKAGING / "publish_to_r2.py" +) +assert _spec and _spec.loader +pub = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(pub) + +MANIFEST_YAML = """\ +id: demo +name: Demo +version: 1.2.3 +description: "Demo agent" +author: AMD +license: MIT +language: python +category: conversation +""" + + +@pytest.fixture +def manifest(tmp_path: Path) -> Path: + p = tmp_path / "gaia-agent.yaml" + p.write_text(MANIFEST_YAML, encoding="utf-8") + return p + + +def _artifact( + tmp_path: Path, size: int, name: str = "demo-1.2.3-x64-setup.exe" +) -> Path: + p = tmp_path / name + p.write_bytes(b"x" * size) + return p + + +class _Resp: + """Stands in for the Worker, echoing the sha it would have recorded. + + The publisher re-checks that value against its own digest, so returning a + placeholder would mask a real mismatch rather than exercise the check. + """ + + status_code = 201 + + def __init__(self, sha: str) -> None: + self._sha = sha + + def json(self) -> dict: + return {"published": {"artifact": {"sha256": self._sha}}} + + +@pytest.fixture +def captured(monkeypatch): + """Capture the outgoing POST and any R2 upload, without performing either.""" + seen: dict = {"post": None, "put": None, "head": None, "already_published": False} + + def fake_post(url, headers=None, files=None, timeout=None): + # requests encodes each value as (filename, body[, content_type]); read + # the file handle now, before the caller's context manager closes it. + flat = {} + for k, v in (files or {}).items(): + body = v[1] + flat[k] = body.read() if hasattr(body, "read") else body + seen["post"] = {"url": url, "files": flat} + # By reference the Worker verifies R2's stored digest and returns it; + # inline it hashes the bytes it received. Model both. + sha = flat.get("artifact_ref_sha256") + if sha is None: + sha = hashlib.sha256(flat["artifact"]).hexdigest() + return _Resp(sha) + + class _S3: + def put_object(self, **kw): + seen["put"] = kw + + def fake_head(url, timeout=None, allow_redirects=None): + seen["head"] = url + return type( + "H", (), {"status_code": seen["already_published"] and 200 or 404} + )() + + monkeypatch.setattr(pub.requests, "head", fake_head) + + monkeypatch.setattr(pub.requests, "post", fake_post) + monkeypatch.setattr(pub, "_download_sha256", lambda *a, **k: None) + monkeypatch.setitem( + __import__("sys").modules, + "boto3", + type("m", (), {"client": lambda *a, **k: _S3()}), + ) + return seen + + +def _publish(manifest: Path, artifact: Path): + return pub.publish_one( + base_url="https://hub.example", + manifest_path=manifest, + manifest={"id": "demo", "version": "1.2.3"}, + artifact_path=artifact, + platform_key="win-x64", + token="tok", + ) + + +def test_a_small_artifact_still_rides_inline(manifest, tmp_path, captured, monkeypatch): + """The existing path must not change — email and terminal-hub depend on it.""" + monkeypatch.setenv("R2_ACCESS_KEY_ID", "k") + monkeypatch.setenv("R2_SECRET_ACCESS_KEY", "s") + monkeypatch.setenv("CLOUDFLARE_ACCOUNT_ID", "acct") + + _publish(manifest, _artifact(tmp_path, 1024)) + + assert captured["put"] is None, "a small artifact must not touch the S3 API" + files = captured["post"]["files"] + assert "artifact" in files + assert not any(k.startswith("artifact_ref_") for k in files) + + +def test_an_oversized_artifact_goes_to_r2_and_is_published_by_reference( + manifest, tmp_path, captured, monkeypatch +): + monkeypatch.setenv("R2_ACCESS_KEY_ID", "k") + monkeypatch.setenv("R2_SECRET_ACCESS_KEY", "s") + monkeypatch.setenv("CLOUDFLARE_ACCOUNT_ID", "acct") + size = pub.DIRECT_UPLOAD_THRESHOLD + 1 + art = _artifact(tmp_path, size) + sha = hashlib.sha256(art.read_bytes()).hexdigest() + + _publish(manifest, art) + + put = captured["put"] + assert put is not None, "an oversized artifact must be uploaded to R2" + assert put["Key"] == f"agents/demo/1.2.3/{art.name}" + # Base64 of the raw digest. Sending hex here is accepted by boto3 and then + # rejected by R2, which is a failure that only shows up mid-release. + assert put["ChecksumSHA256"] == base64.b64encode(bytes.fromhex(sha)).decode() + + files = captured["post"]["files"] + assert "artifact" not in files, "the bytes must not also travel through the Worker" + assert files["artifact_ref_filename"] == art.name + assert files["artifact_ref_sha256"] == sha + assert files["artifact_ref_size"] == str(size) + + +def test_put_object_is_used_so_the_upload_stays_single_part( + manifest, tmp_path, captured, monkeypatch +): + """R2 records a whole-object SHA-256 only for single-part uploads. + + `upload_file`/`upload_fileobj` switch to multipart above their threshold and + the checksum is lost, after which the Worker refuses the publish as + unverifiable. Pinning the call keeps that from regressing quietly. + """ + monkeypatch.setenv("R2_ACCESS_KEY_ID", "k") + monkeypatch.setenv("R2_SECRET_ACCESS_KEY", "s") + monkeypatch.setenv("CLOUDFLARE_ACCOUNT_ID", "acct") + + _publish(manifest, _artifact(tmp_path, pub.DIRECT_UPLOAD_THRESHOLD + 1)) + + assert set(captured["put"]) >= {"Bucket", "Key", "Body", "ChecksumSHA256"} + + +@pytest.mark.parametrize( + "missing", ["R2_ACCESS_KEY_ID", "R2_SECRET_ACCESS_KEY", "CLOUDFLARE_ACCOUNT_ID"] +) +def test_missing_r2_credentials_fail_loudly( + manifest, tmp_path, captured, monkeypatch, missing +): + """Never fall back to the Worker — that path 413s and wastes a release.""" + for name in ("R2_ACCESS_KEY_ID", "R2_SECRET_ACCESS_KEY", "CLOUDFLARE_ACCOUNT_ID"): + monkeypatch.setenv(name, "v") + monkeypatch.delenv(missing, raising=False) + + with pytest.raises(SystemExit) as e: + _publish(manifest, _artifact(tmp_path, pub.DIRECT_UPLOAD_THRESHOLD + 1)) + + msg = str(e.value) + assert "R2_ACCESS_KEY_ID" in msg and "100 MB" in msg + assert ( + captured["post"] is None + ), "nothing may be published when the upload cannot run" + + +def test_the_threshold_sits_below_cloudflares_real_cap(): + """Switch lanes before the cap, not at it, so growth never 413s a release.""" + assert pub.DIRECT_UPLOAD_THRESHOLD < 100 * 1024 * 1024 + + +def test_an_already_published_object_is_never_overwritten( + manifest, tmp_path, captured, monkeypatch +): + """The failure this guards is silent, which is what makes it dangerous. + + The Worker's by-reference immutability check keys on the agent manifest, so + it fires only after the PUT. Upload first and a re-publish with different + bytes replaces the stored artifact, the catalog keeps the OLD hash, and the + 409 handler then re-downloads the bytes it just wrote — agreeing with itself + and exiting green while install-time verification is broken for everyone. + """ + monkeypatch.setenv("R2_ACCESS_KEY_ID", "k") + monkeypatch.setenv("R2_SECRET_ACCESS_KEY", "s") + monkeypatch.setenv("CLOUDFLARE_ACCOUNT_ID", "acct") + captured["already_published"] = True + + _publish(manifest, _artifact(tmp_path, pub.DIRECT_UPLOAD_THRESHOLD + 1)) + + assert captured["head"] is not None, "must check before uploading" + assert captured["put"] is None, "published bytes must never be overwritten" + + +def test_the_check_runs_before_the_upload_not_after( + manifest, tmp_path, captured, monkeypatch +): + """A first publish still uploads — the guard must not block the normal path.""" + monkeypatch.setenv("R2_ACCESS_KEY_ID", "k") + monkeypatch.setenv("R2_SECRET_ACCESS_KEY", "s") + monkeypatch.setenv("CLOUDFLARE_ACCOUNT_ID", "acct") + captured["already_published"] = False + + _publish(manifest, _artifact(tmp_path, pub.DIRECT_UPLOAD_THRESHOLD + 1)) + + assert captured["head"] is not None + assert captured["put"] is not None diff --git a/hub/agents/gaia/python/packaging/publish_to_r2.py b/hub/agents/gaia/python/packaging/publish_to_r2.py index 58ff7e602..895c95485 100644 --- a/hub/agents/gaia/python/packaging/publish_to_r2.py +++ b/hub/agents/gaia/python/packaging/publish_to_r2.py @@ -171,6 +171,12 @@ def _download_sha256(base_url: str, agent_id: str, version: str, filename: str) return hashlib.sha256(resp.content).hexdigest() +# Cloudflare's Worker request-body cap on Free/Pro. Mirrors +# DIRECT_UPLOAD_THRESHOLD's rationale in the email publisher, which has the +# direct-to-R2 lane this script lacks. +WORKER_BODY_LIMIT = 90 * 1024 * 1024 + + def publish_one( base_url: str, manifest_path: Path, @@ -185,6 +191,20 @@ def publish_one( raise SystemExit(f"error: artifact not found: {artifact_path}") filename = artifact_path.name local_sha, size = _sha256_file(artifact_path) + # Cloudflare caps a Worker request body by plan (100 MB on Free/Pro), so an + # artifact above it is rejected at the edge before the Worker runs — a bare + # 413 with an HTML body, mid-release, after the freeze has already been paid + # for. The email publisher has a direct-to-R2 lane for this; this one does + # not, so say so plainly rather than letting the release discover it. + if size >= WORKER_BODY_LIMIT: + raise SystemExit( + f"error: {artifact_path.name} is {size} bytes, at or over the " + f"{WORKER_BODY_LIMIT}-byte Cloudflare request-body cap, and this " + "publisher can only POST through the Worker. Port the direct-to-R2 " + "lane from hub/agents/email/python/packaging/publish_to_r2.py " + "(_upload_to_r2 + the artifact_ref_* fields) before releasing a " + "sidecar this large." + ) agent_id = str(manifest["id"]) version = str(manifest["version"]) publish_url = f"{base_url.rstrip('/')}{PUBLISH_PATH}" diff --git a/workers/agent-hub/README.md b/workers/agent-hub/README.md index bb3a6756b..09969586c 100644 --- a/workers/agent-hub/README.md +++ b/workers/agent-hub/README.md @@ -287,6 +287,38 @@ checked into the repo: `MAX_ARTIFACT_BYTES` (a plain var, default 250 MiB) caps artifact size and can be overridden per environment without a secret. +## Publishing artifacts larger than 100 MB + +A Worker request body is capped by the Cloudflare **account plan** — 100 MB on +Free and Pro, 200 MB Business, 500 MB Enterprise. `POST /publish` therefore +cannot carry the Agent UI installers (106-135 MiB); they are rejected with a +`413` by Cloudflare's edge before the Worker executes, so `MAX_ARTIFACT_BYTES` +is not involved and raising it changes nothing. + +Artifacts at or above 90 MiB are instead PUT straight into the bucket over R2's +S3 API and published **by reference**: the POST carries +`artifact_ref_{filename,sha256,size,content_type}` in place of the file part. + +Integrity is not relaxed. Before recording anything the Worker heads the object +and checks its size and SHA-256 against what R2 stored at PUT time. R2 keeps a +whole-object SHA-256 only for **single-part** uploads, so an object without one +is refused (`artifact_unverifiable`) rather than accepted on the publisher's +word — the uploader must use `put_object` with `ChecksumSHA256`, never +`upload_file`, which switches to multipart and drops the checksum. + +The publisher needs three extra secrets for this path: + +| Secret | How to get it | +|---|---| +| `R2_ACCESS_KEY_ID` | Cloudflare dashboard → R2 → **Manage API Tokens** → create a token with **Object Read & Write** on the hub bucket | +| `R2_SECRET_ACCESS_KEY` | Shown once alongside the access key id | +| `CLOUDFLARE_ACCOUNT_ID` | Same value the Worker deploy uses | + +These are R2 S3 credentials and are **not** the same as `CLOUDFLARE_API_TOKEN`, +which deploys the Worker. Missing any of them is a loud failure naming all +three; the publisher never silently falls back to the Worker path, because that +path 413s and would waste the release. + ## Deploying on Railway (demo) For demo/staging only: [`Dockerfile`](./Dockerfile) runs `wrangler dev` diff --git a/workers/agent-hub/src/publish.ts b/workers/agent-hub/src/publish.ts index a7427571f..a4450a6ae 100644 --- a/workers/agent-hub/src/publish.ts +++ b/workers/agent-hub/src/publish.ts @@ -82,6 +82,152 @@ async function optionalPackageFiles(form: FormData): Promise { return JSON.stringify({ files }); } +/** + * Pick the artifact source for this publish: an inline `artifact` file part, or + * a by-reference upload named by `artifact_ref_filename`. Exactly one is valid. + */ +function selectArtifactSource( + form: FormData, + byReference: boolean, + refFilename: string | null +): { artifactFile: File | null; filename: string } { + const part = form.get("artifact"); + if (byReference) { + if (part != null) { + throw new HttpError( + 400, + "invalid_request", + "Send either an 'artifact' file part or 'artifact_ref_*' fields, not both." + ); + } + return { artifactFile: null, filename: refFilename as string }; + } + if (part == null || typeof part === "string") { + throw new HttpError( + 400, + "invalid_request", + "Missing 'artifact' file part (the wheel or binary to publish), and no " + + "'artifact_ref_filename' for a by-reference publish." + ); + } + // workers-types declares FormData.get() as `string | null`, so the guard above + // narrows to `never`; the cast is how the rest of this file already bridges + // that gap between the declared type and the runtime File. + const file = part as File; + return { artifactFile: file, filename: file.name }; +} + +/** Hex-encode an ArrayBuffer (R2 returns checksums as raw bytes). */ +function hex(buf: ArrayBuffer): string { + return [...new Uint8Array(buf)].map((b) => b.toString(16).padStart(2, "0")).join(""); +} + +/** + * Verify an artifact CI uploaded straight to R2, and build its catalog record. + * + * The Worker never sees these bytes — that is the whole point, since anything + * over the plan's request-body cap (100 MB on Free/Pro) 413s at the edge before + * this code runs. So "trust the publisher's claim" is not good enough: the + * caller states a size and SHA-256, and both are checked against what R2 itself + * recorded when it accepted the PUT. + * + * R2 only stores a whole-object SHA-256 for NON-multipart uploads. A missing + * checksum is therefore refused rather than waved through — a multipart upload + * would otherwise silently downgrade this to an unverified claim, which is + * exactly the guarantee the inline path never gives up. CI must force a + * single-part PUT with x-amz-checksum-sha256. + */ +async function verifyUploadedArtifact( + env: Env, + form: FormData, + key: string, + filename: string +): Promise { + const claimedSha = (await optionalTextPart(form, "artifact_ref_sha256", "artifact_ref_sha256")) + ?.trim() + .toLowerCase(); + const claimedSizeText = await optionalTextPart(form, "artifact_ref_size", "artifact_ref_size"); + const contentType = + (await optionalTextPart(form, "artifact_ref_content_type", "artifact_ref_content_type")) ?? + "application/octet-stream"; + + if (!claimedSha || !/^[0-9a-f]{64}$/.test(claimedSha)) { + throw new HttpError( + 400, + "invalid_request", + "A by-reference publish needs 'artifact_ref_sha256' as 64 lowercase hex characters." + ); + } + const claimedSize = Number(claimedSizeText); + if (!Number.isInteger(claimedSize) || claimedSize <= 0) { + throw new HttpError( + 400, + "invalid_request", + "A by-reference publish needs 'artifact_ref_size' as the object's byte count." + ); + } + + const head = await env.BUCKET.head(key); + if (!head) { + throw new HttpError( + 404, + "artifact_not_uploaded", + `No object at ${key}. Upload the artifact to R2 first (S3 API), then publish ` + + `it by reference. Nothing has been recorded in the catalog.` + ); + } + if (head.size !== claimedSize) { + throw new HttpError( + 409, + "artifact_mismatch", + `Object at ${key} is ${head.size} bytes but the publish claims ${claimedSize}. ` + + `Re-upload the artifact; the catalog was not modified.` + ); + } + + // The inline lane enforces this before storing; the by-reference lane must + // too, or MAX_ARTIFACT_BYTES silently stops being the artifact size cap the + // README says it is. The 250 MiB default clears the 135 MiB installers, so + // this costs nothing today and keeps one ceiling rather than two. + const limit = maxBytes(env); + if (head.size > limit) { + throw new HttpError( + 413, + "artifact_too_large", + `Object at ${key} is ${head.size} bytes, over the ${limit}-byte limit. ` + + `The catalog was not modified.` + ); + } + + const stored = head.checksums?.sha256; + if (!stored) { + throw new HttpError( + 409, + "artifact_unverifiable", + `Object at ${key} has no SHA-256 recorded by R2, so its integrity cannot be ` + + `confirmed. R2 stores a whole-object SHA-256 only for single-part uploads — ` + + `re-upload without multipart and with x-amz-checksum-sha256 set.` + ); + } + const actualSha = hex(stored); + if (actualSha !== claimedSha) { + throw new HttpError( + 409, + "artifact_mismatch", + `Object at ${key} hashes to ${actualSha} but the publish claims ${claimedSha}. ` + + `The catalog was not modified.` + ); + } + + return { + filename, + path: key, + size_bytes: head.size, + sha256: actualSha, + content_type: contentType, + }; +} + export async function handlePublish( request: Request, env: Env, @@ -114,15 +260,24 @@ export async function handlePublish( const manifestText = typeof manifestPart === "string" ? manifestPart : await (manifestPart as Blob).text(); - const artifactPart = form.get("artifact"); - if (artifactPart == null || typeof artifactPart === "string") { - throw new HttpError( - 400, - "invalid_request", - "Missing 'artifact' file part (the wheel or binary to publish)." - ); - } - const artifactFile = artifactPart as File; + // Two ways to supply the artifact: + // + // inline — an `artifact` file part. The Worker hashes and stores it. + // by-reference — `artifact_ref_*` text parts naming an object CI already + // PUT straight into R2 over the S3 API. + // + // by-reference exists because a Worker request body is capped by the + // Cloudflare plan (100 MB on Free/Pro) and the Agent UI installers are + // 106-135 MiB, so they 413 at the edge before the Worker ever runs. Uploading + // to R2 directly has no such cap. The integrity guarantees are NOT relaxed: + // the object is verified below against the size and SHA-256 the caller + // claims, using the checksum R2 itself recorded at PUT time. + const refFilename = await optionalTextPart(form, "artifact_ref_filename", "artifact_ref_filename"); + const byReference = refFilename != null; + + // Exactly one of the two must be present. Resolved through a small helper so + // the File narrowing stays local and cannot leak into the rest of the handler. + const { artifactFile, filename } = selectArtifactSource(form, byReference, refFilename); // Optional README + CHANGELOG markdown for this version (rendered on the Hub // pages). Both are optional; an empty part is rejected (omit it instead). @@ -155,7 +310,6 @@ export async function handlePublish( const manifest = parseManifest(manifestText); assertAuthorAllowed(publisher, manifest.author); - const filename = artifactFile.name; if (!ARTIFACT_FILENAME_RE.test(filename)) { throw new HttpError( 400, @@ -192,49 +346,66 @@ export async function handlePublish( } const versionExists = Boolean(existing?.versions[manifest.version]); - const bytes = new Uint8Array(await artifactFile.arrayBuffer()); - const limit = maxBytes(env); - if (bytes.byteLength === 0) { - throw new HttpError(400, "invalid_artifact", "Artifact is empty (0 bytes)."); - } - if (bytes.byteLength > limit) { - throw new HttpError( - 413, - "artifact_too_large", - `Artifact is ${bytes.byteLength} bytes, over the ${limit}-byte limit.` - ); - } - const key = artifactKey(manifest.id, manifest.version, filename); // Per-filename immutability: a published artifact is never overwritten. A new // platform binary under an existing version uses a distinct filename and is - // allowed; re-uploading the same filename is rejected. (Idempotent re-runs of + // allowed; re-publishing the same filename is rejected. (Idempotent re-runs of // a release job should treat this 409 as "already published" — success.) - if (await env.BUCKET.head(key)) { + // + // What counts as "published" differs by mode, and the distinction matters: + // inline, the R2 object only exists once the Worker has stored it, so its + // presence IS the record. By-reference, CI has already PUT the object before + // calling this, so the object always exists and heading it would 409 every + // time — the record is the AGENT MANIFEST, which only lists artifacts this + // endpoint accepted. + const alreadyPublished = byReference + ? Boolean(existing?.versions[manifest.version]?.artifacts?.some((a) => a.filename === filename)) + : Boolean(await env.BUCKET.head(key)); + if (alreadyPublished) { throw new HttpError( 409, "version_exists", - `Artifact already exists at ${key} and is immutable. To add another ` + - `platform binary use a distinct filename; to change this one, bump the version.` + `Artifact ${filename} is already published under ${manifest.id}@${manifest.version} ` + + `and is immutable. To add another platform binary use a distinct filename; ` + + `to change this one, bump the version.` ); } - const sha256 = await sha256Hex(bytes); - const artifact: ArtifactInfo = { - filename, - path: key, - size_bytes: bytes.byteLength, - sha256, - content_type: artifactFile.type || "application/octet-stream", - }; - - // Store the artifact. The raw gaia-agent.yaml is written only on the first - // publish of a version so it stays the immutable record of that release; a - // later platform binary joining the same version must not rewrite it. - await env.BUCKET.put(key, bytes, { - httpMetadata: { contentType: artifact.content_type }, - sha256, - }); + let artifact: ArtifactInfo; + if (byReference) { + artifact = await verifyUploadedArtifact(env, form, key, filename); + } else { + const file = artifactFile!; + const bytes = new Uint8Array(await file.arrayBuffer()); + const limit = maxBytes(env); + if (bytes.byteLength === 0) { + throw new HttpError(400, "invalid_artifact", "Artifact is empty (0 bytes)."); + } + if (bytes.byteLength > limit) { + throw new HttpError( + 413, + "artifact_too_large", + `Artifact is ${bytes.byteLength} bytes, over the ${limit}-byte limit. ` + + `Artifacts above the Cloudflare request-body cap must be uploaded to R2 ` + + `directly and published by reference (artifact_ref_* fields).` + ); + } + const sha256 = await sha256Hex(bytes); + artifact = { + filename, + path: key, + size_bytes: bytes.byteLength, + sha256, + content_type: file.type || "application/octet-stream", + }; + // Store the artifact. The raw gaia-agent.yaml is written only on the first + // publish of a version so it stays the immutable record of that release; a + // later platform binary joining the same version must not rewrite it. + await env.BUCKET.put(key, bytes, { + httpMetadata: { contentType: artifact.content_type }, + sha256, + }); + } // The raw gaia-agent.yaml, README, and CHANGELOG are per-version records: // write them only on the first publish of a version so a later platform binary // joining the same version cannot rewrite them. diff --git a/workers/agent-hub/test/fake-r2.ts b/workers/agent-hub/test/fake-r2.ts index 7fdb7b615..4a2fabc2f 100644 --- a/workers/agent-hub/test/fake-r2.ts +++ b/workers/agent-hub/test/fake-r2.ts @@ -53,6 +53,13 @@ function makeBody(obj: StoredObject) { }; } +/** hex -> ArrayBuffer, matching how R2 returns stored checksums. */ +function hexToBuffer(hex: string): ArrayBuffer { + const out = new Uint8Array(hex.length / 2); + for (let i = 0; i < out.length; i++) out[i] = parseInt(hex.slice(i * 2, i * 2 + 2), 16); + return out.buffer; +} + export class FakeR2 { private store = new Map(); @@ -71,6 +78,10 @@ export class FakeR2 { } } const obj: StoredObject = { key, bytes, contentType, uploaded: new Date() }; + // R2 records a whole-object SHA-256 only when one was supplied (binding + // `sha256` option, or x-amz-checksum-sha256 on a single-part S3 PUT). A + // multipart upload has none — modelled by simply not setting it. + if (options?.sha256) (obj as StoredObject & { sha256?: string }).sha256 = options.sha256; this.store.set(key, obj); return makeBody(obj); } @@ -86,7 +97,15 @@ export class FakeR2 { const obj = this.store.get(key); if (!obj) return null; const body = makeBody(obj); - return { key: body.key, size: body.size, httpEtag: body.httpEtag, uploaded: body.uploaded }; + const sha = (obj as StoredObject & { sha256?: string }).sha256; + return { + key: body.key, + size: body.size, + httpEtag: body.httpEtag, + uploaded: body.uploaded, + // Real R2 hands back raw bytes, not hex — the Worker hex-encodes them. + checksums: sha ? { sha256: hexToBuffer(sha) } : {}, + }; } async delete(key: string): Promise { @@ -152,6 +171,41 @@ export function makeEnv(overrides?: { tokens?: unknown; maxBytes?: string }): { } /** Build a POST /publish multipart request. */ +/** + * A by-reference publish: CI has already PUT the bytes into R2 over the S3 API, + * and the Worker is only asked to verify and record them. + */ +export function publishByRefRequest(opts: { + token?: string; + manifestYaml: string; + filename: string; + sha256: string; + size: number; + contentType?: string; + readme?: string; + changelog?: string; +}): Request { + const form = new FormData(); + form.set("manifest", opts.manifestYaml); + if (opts.readme !== undefined) form.set("readme", opts.readme); + if (opts.changelog !== undefined) form.set("changelog", opts.changelog); + form.set("artifact_ref_filename", opts.filename); + form.set("artifact_ref_sha256", opts.sha256); + form.set("artifact_ref_size", String(opts.size)); + if (opts.contentType) form.set("artifact_ref_content_type", opts.contentType); + return new Request("https://hub.amd-gaia.ai/publish", { + method: "POST", + headers: { authorization: `Bearer ${opts.token ?? "tok_amd"}` }, + body: form, + }); +} + +/** sha256 hex over bytes, for tests that stage an object then publish it. */ +export async function sha256Of(bytes: Uint8Array): Promise { + const d = await crypto.subtle.digest("SHA-256", bytes); + return [...new Uint8Array(d)].map((b) => b.toString(16).padStart(2, "0")).join(""); +} + export function publishRequest(opts: { token?: string; manifestYaml: string; diff --git a/workers/agent-hub/test/publish-by-reference.test.ts b/workers/agent-hub/test/publish-by-reference.test.ts new file mode 100644 index 000000000..71cdc27e2 --- /dev/null +++ b/workers/agent-hub/test/publish-by-reference.test.ts @@ -0,0 +1,181 @@ +// Copyright(C) 2025-2026 Advanced Micro Devices, Inc. All rights reserved. +// SPDX-License-Identifier: MIT +// +// Publishing an artifact that CI uploaded straight to R2. +// +// Why this path exists: a Worker request body is capped by the Cloudflare plan +// (100 MB on Free/Pro), and the Agent UI installers are 106-135 MiB, so they +// 413 at the edge before the Worker runs at all. Uploading to R2 over the S3 +// API has no such cap. +// +// The risk this introduces is that the Worker never sees the bytes, so it +// cannot hash them itself. These tests exist to pin the compensating control: +// the object is verified against the size and SHA-256 R2 recorded at PUT time, +// and anything unverifiable is REFUSED rather than trusted. A regression here +// would silently turn "the hub checked this artifact" into "the publisher said +// so", which is the difference between an integrity guarantee and a claim. + +import { describe, expect, it } from "vitest"; + +import worker from "../src/index"; +import type { AgentManifest } from "../src/types"; +import { makeEnv, publishByRefRequest, sampleManifest, sha256Of } from "./fake-r2"; + +const BYTES = new TextEncoder().encode("pretend this is a 130 MiB installer"); +const FILENAME = "gaia-agent-ui-1.0.0-x64-setup.exe"; +const KEY = "agents/chat/0.1.0/" + FILENAME; + +/** Stage an object in R2 the way CI's S3 upload would, with or without a checksum. */ +async function stage( + env: ReturnType, + opts: { bytes?: Uint8Array; withChecksum?: boolean } = {} +) { + const bytes = opts.bytes ?? BYTES; + const sha = await sha256Of(bytes); + await env.BUCKET.put(KEY, bytes, { + httpMetadata: { contentType: "application/octet-stream" }, + ...(opts.withChecksum === false ? {} : { sha256: sha }), + }); + return { bytes, sha }; +} + +async function publishRef( + env: ReturnType, + over: Partial[0]> = {}, + sha = "", + size = BYTES.byteLength +) { + return worker.fetch( + publishByRefRequest({ + manifestYaml: sampleManifest(), + filename: FILENAME, + sha256: sha, + size, + ...over, + }), + env as never + ); +} + +describe("publish by reference", () => { + it("records an artifact the Worker never received", async () => { + const env = makeEnv(); + const { sha, bytes } = await stage(env); + + const res = await publishRef(env, {}, sha, bytes.byteLength); + expect(res.status).toBe(201); + + // The catalog must carry the same facts the inline path would have written, + // or downloads and the install-time lock check have nothing to verify against. + const manifest = JSON.parse( + await (await env.BUCKET.get("agents/chat/manifest.json")).text() + ) as AgentManifest; + const artifact = manifest.versions["0.1.0"].artifacts[0]; + expect(artifact.filename).toBe(FILENAME); + expect(artifact.sha256).toBe(sha); + expect(artifact.size_bytes).toBe(bytes.byteLength); + expect(artifact.path).toBe(KEY); + }); + + it("refuses when the object was never uploaded", async () => { + const env = makeEnv(); + const res = await publishRef(env, {}, await sha256Of(BYTES)); + expect(res.status).toBe(404); + expect((await res.json() as any).error.code).toBe("artifact_not_uploaded"); + }); + + it("refuses when the stored bytes hash differently than claimed", async () => { + // The case that matters: a publisher claiming a hash for bytes it did not + // upload would otherwise poison the catalog for every future download. + const env = makeEnv(); + await stage(env); + const lie = "f".repeat(64); + const res = await publishRef(env, {}, lie); + expect(res.status).toBe(409); + expect((await res.json() as any).error.code).toBe("artifact_mismatch"); + }); + + it("refuses when the stored size differs from the claim", async () => { + const env = makeEnv(); + const { sha } = await stage(env); + const res = await publishRef(env, {}, sha, BYTES.byteLength + 1); + expect(res.status).toBe(409); + expect((await res.json() as any).error.code).toBe("artifact_mismatch"); + }); + + it("refuses an object R2 has no checksum for, rather than trusting the claim", async () => { + // R2 records a whole-object SHA-256 only for single-part uploads. A + // multipart upload leaves none — and accepting the publisher's word there + // would quietly reduce this to an unverified claim. + const env = makeEnv(); + const { sha } = await stage(env, { withChecksum: false }); + const res = await publishRef(env, {}, sha); + expect(res.status).toBe(409); + expect((await res.json() as any).error.code).toBe("artifact_unverifiable"); + }); + + it("rejects a malformed sha256 before touching the bucket", async () => { + const env = makeEnv(); + await stage(env); + for (const bad of ["", "abc", "G".repeat(64), "a".repeat(63)]) { + const res = await publishRef(env, {}, bad); + expect(res.status, `sha ${JSON.stringify(bad)} was accepted`).toBe(400); + } + }); + + it("rejects a non-positive size", async () => { + const env = makeEnv(); + const { sha } = await stage(env); + for (const bad of [0, -1]) { + const res = await publishRef(env, {}, sha, bad); + expect(res.status, `size ${bad} was accepted`).toBe(400); + } + }); + + it("keeps per-filename immutability, keyed on the manifest not the bucket", async () => { + // Inline, the object's presence is the record. By-reference the object is + // ALWAYS present by the time we are called, so the manifest is the record — + // otherwise every by-reference publish would 409 against its own upload. + const env = makeEnv(); + const { sha, bytes } = await stage(env); + expect((await publishRef(env, {}, sha, bytes.byteLength)).status).toBe(201); + + const again = await publishRef(env, {}, sha, bytes.byteLength); + expect(again.status).toBe(409); + expect((await again.json() as any).error.code).toBe("version_exists"); + }); + + it("refuses a request carrying both an inline artifact and a reference", async () => { + const env = makeEnv(); + const form = new FormData(); + form.set("manifest", sampleManifest()); + form.set("artifact_ref_filename", FILENAME); + form.set("artifact", new Blob([BYTES]), FILENAME); + const res = await worker.fetch( + new Request("https://hub.amd-gaia.ai/publish", { + method: "POST", + headers: { authorization: "Bearer tok_amd" }, + body: form, + }), + env as never + ); + expect(res.status).toBe(400); + }); + + it("still requires authentication", async () => { + const env = makeEnv(); + const { sha } = await stage(env); + const res = await publishRef(env, { token: "not-a-real-token" }, sha); + expect(res.status).toBe(401); + }); + + it("applies MAX_ARTIFACT_BYTES to a by-reference publish too", async () => { + // Otherwise the documented artifact ceiling quietly stops covering the one + // lane that exists specifically for the largest artifacts. + const env = makeEnv({ maxBytes: "16" }); + const { sha, bytes } = await stage(env); + const res = await publishRef(env, {}, sha, bytes.byteLength); + expect(res.status).toBe(413); + expect(((await res.json()) as any).error.code).toBe("artifact_too_large"); + }); +});