From 9d1744262f9c9be1ac614624e703f85bef6f073c Mon Sep 17 00:00:00 2001 From: Ovtcharov Date: Wed, 19 Aug 2026 16:47:09 -0700 Subject: [PATCH 1/3] feat(hub): publish artifacts by reference so oversized ones can ship at all MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Agent UI installers have never been publishable. Every one is 106-135 MiB and a Worker request body is capped by the Cloudflare plan — 100 MB on Free and Pro — so they 413 at the edge before the Worker runs. MAX_ARTIFACT_BYTES was never the constraint and raising it changes nothing; the HTML error body is Cloudflare's, not ours. terminal-hub and email only work because they are small. POST /publish now accepts an artifact either inline, as today, or by reference: CI PUTs the bytes straight into R2 over the S3 API, which has no such cap, then sends `artifact_ref_{filename,sha256,size,content_type}` instead of the file. The Worker not seeing the bytes must not become "the publisher said so". Before recording anything it heads the object and checks the size and the SHA-256 against what R2 itself stored at PUT time. R2 keeps a whole-object SHA-256 only for single-part uploads, so an object without one is REFUSED rather than accepted on trust — CI has to upload single-part with x-amz-checksum-sha256. Immutability needed rethinking rather than reusing. Inline, the object's presence is the record, so heading it is the right check. By reference the object always exists by the time we are called, so that check would 409 against the caller's own upload; the record is the agent manifest, which only lists artifacts this endpoint accepted. 10 tests cover the new path, including the two that matter: a hash that does not match the stored bytes, and an object R2 could not checksum. Both were confirmed to fail when the verifier is mutated to trust the caller's claim. --- workers/agent-hub/src/publish.ts | 243 ++++++++++++++---- workers/agent-hub/test/fake-r2.ts | 56 +++- .../test/publish-by-reference.test.ts | 171 ++++++++++++ 3 files changed, 426 insertions(+), 44 deletions(-) create mode 100644 workers/agent-hub/test/publish-by-reference.test.ts diff --git a/workers/agent-hub/src/publish.ts b/workers/agent-hub/src/publish.ts index a7427571f..44c09d430 100644 --- a/workers/agent-hub/src/publish.ts +++ b/workers/agent-hub/src/publish.ts @@ -82,6 +82,138 @@ async function optionalPackageFiles(form: FormData): Promise { return JSON.stringify({ files }); } +/** + * Pick the artifact source for this publish: an inline `artifact` file part, or + * a by-reference upload named by `artifact_ref_filename`. Exactly one is valid. + */ +function selectArtifactSource( + form: FormData, + byReference: boolean, + refFilename: string | null +): { artifactFile: File | null; filename: string } { + const part = form.get("artifact"); + if (byReference) { + if (part != null) { + throw new HttpError( + 400, + "invalid_request", + "Send either an 'artifact' file part or 'artifact_ref_*' fields, not both." + ); + } + return { artifactFile: null, filename: refFilename as string }; + } + if (part == null || typeof part === "string") { + throw new HttpError( + 400, + "invalid_request", + "Missing 'artifact' file part (the wheel or binary to publish), and no " + + "'artifact_ref_filename' for a by-reference publish." + ); + } + // workers-types declares FormData.get() as `string | null`, so the guard above + // narrows to `never`; the cast is how the rest of this file already bridges + // that gap between the declared type and the runtime File. + const file = part as File; + return { artifactFile: file, filename: file.name }; +} + +/** Hex-encode an ArrayBuffer (R2 returns checksums as raw bytes). */ +function hex(buf: ArrayBuffer): string { + return [...new Uint8Array(buf)].map((b) => b.toString(16).padStart(2, "0")).join(""); +} + +/** + * Verify an artifact CI uploaded straight to R2, and build its catalog record. + * + * The Worker never sees these bytes — that is the whole point, since anything + * over the plan's request-body cap (100 MB on Free/Pro) 413s at the edge before + * this code runs. So "trust the publisher's claim" is not good enough: the + * caller states a size and SHA-256, and both are checked against what R2 itself + * recorded when it accepted the PUT. + * + * R2 only stores a whole-object SHA-256 for NON-multipart uploads. A missing + * checksum is therefore refused rather than waved through — a multipart upload + * would otherwise silently downgrade this to an unverified claim, which is + * exactly the guarantee the inline path never gives up. CI must force a + * single-part PUT with x-amz-checksum-sha256. + */ +async function verifyUploadedArtifact( + env: Env, + form: FormData, + key: string, + filename: string +): Promise { + const claimedSha = (await optionalTextPart(form, "artifact_ref_sha256", "artifact_ref_sha256")) + ?.trim() + .toLowerCase(); + const claimedSizeText = await optionalTextPart(form, "artifact_ref_size", "artifact_ref_size"); + const contentType = + (await optionalTextPart(form, "artifact_ref_content_type", "artifact_ref_content_type")) ?? + "application/octet-stream"; + + if (!claimedSha || !/^[0-9a-f]{64}$/.test(claimedSha)) { + throw new HttpError( + 400, + "invalid_request", + "A by-reference publish needs 'artifact_ref_sha256' as 64 lowercase hex characters." + ); + } + const claimedSize = Number(claimedSizeText); + if (!Number.isInteger(claimedSize) || claimedSize <= 0) { + throw new HttpError( + 400, + "invalid_request", + "A by-reference publish needs 'artifact_ref_size' as the object's byte count." + ); + } + + const head = await env.BUCKET.head(key); + if (!head) { + throw new HttpError( + 404, + "artifact_not_uploaded", + `No object at ${key}. Upload the artifact to R2 first (S3 API), then publish ` + + `it by reference. Nothing has been recorded in the catalog.` + ); + } + if (head.size !== claimedSize) { + throw new HttpError( + 409, + "artifact_mismatch", + `Object at ${key} is ${head.size} bytes but the publish claims ${claimedSize}. ` + + `Re-upload the artifact; the catalog was not modified.` + ); + } + + const stored = head.checksums?.sha256; + if (!stored) { + throw new HttpError( + 409, + "artifact_unverifiable", + `Object at ${key} has no SHA-256 recorded by R2, so its integrity cannot be ` + + `confirmed. R2 stores a whole-object SHA-256 only for single-part uploads — ` + + `re-upload without multipart and with x-amz-checksum-sha256 set.` + ); + } + const actualSha = hex(stored); + if (actualSha !== claimedSha) { + throw new HttpError( + 409, + "artifact_mismatch", + `Object at ${key} hashes to ${actualSha} but the publish claims ${claimedSha}. ` + + `The catalog was not modified.` + ); + } + + return { + filename, + path: key, + size_bytes: head.size, + sha256: actualSha, + content_type: contentType, + }; +} + export async function handlePublish( request: Request, env: Env, @@ -114,15 +246,24 @@ export async function handlePublish( const manifestText = typeof manifestPart === "string" ? manifestPart : await (manifestPart as Blob).text(); - const artifactPart = form.get("artifact"); - if (artifactPart == null || typeof artifactPart === "string") { - throw new HttpError( - 400, - "invalid_request", - "Missing 'artifact' file part (the wheel or binary to publish)." - ); - } - const artifactFile = artifactPart as File; + // Two ways to supply the artifact: + // + // inline — an `artifact` file part. The Worker hashes and stores it. + // by-reference — `artifact_ref_*` text parts naming an object CI already + // PUT straight into R2 over the S3 API. + // + // by-reference exists because a Worker request body is capped by the + // Cloudflare plan (100 MB on Free/Pro) and the Agent UI installers are + // 106-135 MiB, so they 413 at the edge before the Worker ever runs. Uploading + // to R2 directly has no such cap. The integrity guarantees are NOT relaxed: + // the object is verified below against the size and SHA-256 the caller + // claims, using the checksum R2 itself recorded at PUT time. + const refFilename = await optionalTextPart(form, "artifact_ref_filename", "artifact_ref_filename"); + const byReference = refFilename != null; + + // Exactly one of the two must be present. Resolved through a small helper so + // the File narrowing stays local and cannot leak into the rest of the handler. + const { artifactFile, filename } = selectArtifactSource(form, byReference, refFilename); // Optional README + CHANGELOG markdown for this version (rendered on the Hub // pages). Both are optional; an empty part is rejected (omit it instead). @@ -155,7 +296,6 @@ export async function handlePublish( const manifest = parseManifest(manifestText); assertAuthorAllowed(publisher, manifest.author); - const filename = artifactFile.name; if (!ARTIFACT_FILENAME_RE.test(filename)) { throw new HttpError( 400, @@ -192,49 +332,66 @@ export async function handlePublish( } const versionExists = Boolean(existing?.versions[manifest.version]); - const bytes = new Uint8Array(await artifactFile.arrayBuffer()); - const limit = maxBytes(env); - if (bytes.byteLength === 0) { - throw new HttpError(400, "invalid_artifact", "Artifact is empty (0 bytes)."); - } - if (bytes.byteLength > limit) { - throw new HttpError( - 413, - "artifact_too_large", - `Artifact is ${bytes.byteLength} bytes, over the ${limit}-byte limit.` - ); - } - const key = artifactKey(manifest.id, manifest.version, filename); // Per-filename immutability: a published artifact is never overwritten. A new // platform binary under an existing version uses a distinct filename and is - // allowed; re-uploading the same filename is rejected. (Idempotent re-runs of + // allowed; re-publishing the same filename is rejected. (Idempotent re-runs of // a release job should treat this 409 as "already published" — success.) - if (await env.BUCKET.head(key)) { + // + // What counts as "published" differs by mode, and the distinction matters: + // inline, the R2 object only exists once the Worker has stored it, so its + // presence IS the record. By-reference, CI has already PUT the object before + // calling this, so the object always exists and heading it would 409 every + // time — the record is the AGENT MANIFEST, which only lists artifacts this + // endpoint accepted. + const alreadyPublished = byReference + ? Boolean(existing?.versions[manifest.version]?.artifacts?.some((a) => a.filename === filename)) + : Boolean(await env.BUCKET.head(key)); + if (alreadyPublished) { throw new HttpError( 409, "version_exists", - `Artifact already exists at ${key} and is immutable. To add another ` + - `platform binary use a distinct filename; to change this one, bump the version.` + `Artifact ${filename} is already published under ${manifest.id}@${manifest.version} ` + + `and is immutable. To add another platform binary use a distinct filename; ` + + `to change this one, bump the version.` ); } - const sha256 = await sha256Hex(bytes); - const artifact: ArtifactInfo = { - filename, - path: key, - size_bytes: bytes.byteLength, - sha256, - content_type: artifactFile.type || "application/octet-stream", - }; - - // Store the artifact. The raw gaia-agent.yaml is written only on the first - // publish of a version so it stays the immutable record of that release; a - // later platform binary joining the same version must not rewrite it. - await env.BUCKET.put(key, bytes, { - httpMetadata: { contentType: artifact.content_type }, - sha256, - }); + let artifact: ArtifactInfo; + if (byReference) { + artifact = await verifyUploadedArtifact(env, form, key, filename); + } else { + const file = artifactFile!; + const bytes = new Uint8Array(await file.arrayBuffer()); + const limit = maxBytes(env); + if (bytes.byteLength === 0) { + throw new HttpError(400, "invalid_artifact", "Artifact is empty (0 bytes)."); + } + if (bytes.byteLength > limit) { + throw new HttpError( + 413, + "artifact_too_large", + `Artifact is ${bytes.byteLength} bytes, over the ${limit}-byte limit. ` + + `Artifacts above the Cloudflare request-body cap must be uploaded to R2 ` + + `directly and published by reference (artifact_ref_* fields).` + ); + } + const sha256 = await sha256Hex(bytes); + artifact = { + filename, + path: key, + size_bytes: bytes.byteLength, + sha256, + content_type: file.type || "application/octet-stream", + }; + // Store the artifact. The raw gaia-agent.yaml is written only on the first + // publish of a version so it stays the immutable record of that release; a + // later platform binary joining the same version must not rewrite it. + await env.BUCKET.put(key, bytes, { + httpMetadata: { contentType: artifact.content_type }, + sha256, + }); + } // The raw gaia-agent.yaml, README, and CHANGELOG are per-version records: // write them only on the first publish of a version so a later platform binary // joining the same version cannot rewrite them. diff --git a/workers/agent-hub/test/fake-r2.ts b/workers/agent-hub/test/fake-r2.ts index 7fdb7b615..25f34a2b7 100644 --- a/workers/agent-hub/test/fake-r2.ts +++ b/workers/agent-hub/test/fake-r2.ts @@ -53,6 +53,13 @@ function makeBody(obj: StoredObject) { }; } +/** hex -> ArrayBuffer, matching how R2 returns stored checksums. */ +function hexToBuffer(hex: string): ArrayBuffer { + const out = new Uint8Array(hex.length / 2); + for (let i = 0; i < out.length; i++) out[i] = parseInt(hex.substr(i * 2, 2), 16); + return out.buffer; +} + export class FakeR2 { private store = new Map(); @@ -71,6 +78,10 @@ export class FakeR2 { } } const obj: StoredObject = { key, bytes, contentType, uploaded: new Date() }; + // R2 records a whole-object SHA-256 only when one was supplied (binding + // `sha256` option, or x-amz-checksum-sha256 on a single-part S3 PUT). A + // multipart upload has none — modelled by simply not setting it. + if (options?.sha256) (obj as StoredObject & { sha256?: string }).sha256 = options.sha256; this.store.set(key, obj); return makeBody(obj); } @@ -86,7 +97,15 @@ export class FakeR2 { const obj = this.store.get(key); if (!obj) return null; const body = makeBody(obj); - return { key: body.key, size: body.size, httpEtag: body.httpEtag, uploaded: body.uploaded }; + const sha = (obj as StoredObject & { sha256?: string }).sha256; + return { + key: body.key, + size: body.size, + httpEtag: body.httpEtag, + uploaded: body.uploaded, + // Real R2 hands back raw bytes, not hex — the Worker hex-encodes them. + checksums: sha ? { sha256: hexToBuffer(sha) } : {}, + }; } async delete(key: string): Promise { @@ -152,6 +171,41 @@ export function makeEnv(overrides?: { tokens?: unknown; maxBytes?: string }): { } /** Build a POST /publish multipart request. */ +/** + * A by-reference publish: CI has already PUT the bytes into R2 over the S3 API, + * and the Worker is only asked to verify and record them. + */ +export function publishByRefRequest(opts: { + token?: string; + manifestYaml: string; + filename: string; + sha256: string; + size: number; + contentType?: string; + readme?: string; + changelog?: string; +}): Request { + const form = new FormData(); + form.set("manifest", opts.manifestYaml); + if (opts.readme !== undefined) form.set("readme", opts.readme); + if (opts.changelog !== undefined) form.set("changelog", opts.changelog); + form.set("artifact_ref_filename", opts.filename); + form.set("artifact_ref_sha256", opts.sha256); + form.set("artifact_ref_size", String(opts.size)); + if (opts.contentType) form.set("artifact_ref_content_type", opts.contentType); + return new Request("https://hub.amd-gaia.ai/publish", { + method: "POST", + headers: { authorization: `Bearer ${opts.token ?? "tok_amd"}` }, + body: form, + }); +} + +/** sha256 hex over bytes, for tests that stage an object then publish it. */ +export async function sha256Of(bytes: Uint8Array): Promise { + const d = await crypto.subtle.digest("SHA-256", bytes); + return [...new Uint8Array(d)].map((b) => b.toString(16).padStart(2, "0")).join(""); +} + export function publishRequest(opts: { token?: string; manifestYaml: string; diff --git a/workers/agent-hub/test/publish-by-reference.test.ts b/workers/agent-hub/test/publish-by-reference.test.ts new file mode 100644 index 000000000..bebf128c1 --- /dev/null +++ b/workers/agent-hub/test/publish-by-reference.test.ts @@ -0,0 +1,171 @@ +// Copyright(C) 2025-2026 Advanced Micro Devices, Inc. All rights reserved. +// SPDX-License-Identifier: MIT +// +// Publishing an artifact that CI uploaded straight to R2. +// +// Why this path exists: a Worker request body is capped by the Cloudflare plan +// (100 MB on Free/Pro), and the Agent UI installers are 106-135 MiB, so they +// 413 at the edge before the Worker runs at all. Uploading to R2 over the S3 +// API has no such cap. +// +// The risk this introduces is that the Worker never sees the bytes, so it +// cannot hash them itself. These tests exist to pin the compensating control: +// the object is verified against the size and SHA-256 R2 recorded at PUT time, +// and anything unverifiable is REFUSED rather than trusted. A regression here +// would silently turn "the hub checked this artifact" into "the publisher said +// so", which is the difference between an integrity guarantee and a claim. + +import { describe, expect, it } from "vitest"; + +import worker from "../src/index"; +import type { AgentManifest } from "../src/types"; +import { makeEnv, publishByRefRequest, sampleManifest, sha256Of } from "./fake-r2"; + +const BYTES = new TextEncoder().encode("pretend this is a 130 MiB installer"); +const FILENAME = "gaia-agent-ui-1.0.0-x64-setup.exe"; +const KEY = "agents/chat/0.1.0/" + FILENAME; + +/** Stage an object in R2 the way CI's S3 upload would, with or without a checksum. */ +async function stage( + env: ReturnType, + opts: { bytes?: Uint8Array; withChecksum?: boolean } = {} +) { + const bytes = opts.bytes ?? BYTES; + const sha = await sha256Of(bytes); + await env.BUCKET.put(KEY, bytes, { + httpMetadata: { contentType: "application/octet-stream" }, + ...(opts.withChecksum === false ? {} : { sha256: sha }), + }); + return { bytes, sha }; +} + +async function publishRef( + env: ReturnType, + over: Partial[0]> = {}, + sha = "", + size = BYTES.byteLength +) { + return worker.fetch( + publishByRefRequest({ + manifestYaml: sampleManifest(), + filename: FILENAME, + sha256: sha, + size, + ...over, + }), + env as never + ); +} + +describe("publish by reference", () => { + it("records an artifact the Worker never received", async () => { + const env = makeEnv(); + const { sha, bytes } = await stage(env); + + const res = await publishRef(env, {}, sha, bytes.byteLength); + expect(res.status).toBe(201); + + // The catalog must carry the same facts the inline path would have written, + // or downloads and the install-time lock check have nothing to verify against. + const manifest = JSON.parse( + await (await env.BUCKET.get("agents/chat/manifest.json")).text() + ) as AgentManifest; + const artifact = manifest.versions["0.1.0"].artifacts[0]; + expect(artifact.filename).toBe(FILENAME); + expect(artifact.sha256).toBe(sha); + expect(artifact.size_bytes).toBe(bytes.byteLength); + expect(artifact.path).toBe(KEY); + }); + + it("refuses when the object was never uploaded", async () => { + const env = makeEnv(); + const res = await publishRef(env, {}, await sha256Of(BYTES)); + expect(res.status).toBe(404); + expect((await res.json() as any).error.code).toBe("artifact_not_uploaded"); + }); + + it("refuses when the stored bytes hash differently than claimed", async () => { + // The case that matters: a publisher claiming a hash for bytes it did not + // upload would otherwise poison the catalog for every future download. + const env = makeEnv(); + await stage(env); + const lie = "f".repeat(64); + const res = await publishRef(env, {}, lie); + expect(res.status).toBe(409); + expect((await res.json() as any).error.code).toBe("artifact_mismatch"); + }); + + it("refuses when the stored size differs from the claim", async () => { + const env = makeEnv(); + const { sha } = await stage(env); + const res = await publishRef(env, {}, sha, BYTES.byteLength + 1); + expect(res.status).toBe(409); + expect((await res.json() as any).error.code).toBe("artifact_mismatch"); + }); + + it("refuses an object R2 has no checksum for, rather than trusting the claim", async () => { + // R2 records a whole-object SHA-256 only for single-part uploads. A + // multipart upload leaves none — and accepting the publisher's word there + // would quietly reduce this to an unverified claim. + const env = makeEnv(); + const { sha } = await stage(env, { withChecksum: false }); + const res = await publishRef(env, {}, sha); + expect(res.status).toBe(409); + expect((await res.json() as any).error.code).toBe("artifact_unverifiable"); + }); + + it("rejects a malformed sha256 before touching the bucket", async () => { + const env = makeEnv(); + await stage(env); + for (const bad of ["", "abc", "G".repeat(64), "a".repeat(63)]) { + const res = await publishRef(env, {}, bad); + expect(res.status, `sha ${JSON.stringify(bad)} was accepted`).toBe(400); + } + }); + + it("rejects a non-positive size", async () => { + const env = makeEnv(); + const { sha } = await stage(env); + for (const bad of [0, -1]) { + const res = await publishRef(env, {}, sha, bad); + expect(res.status, `size ${bad} was accepted`).toBe(400); + } + }); + + it("keeps per-filename immutability, keyed on the manifest not the bucket", async () => { + // Inline, the object's presence is the record. By-reference the object is + // ALWAYS present by the time we are called, so the manifest is the record — + // otherwise every by-reference publish would 409 against its own upload. + const env = makeEnv(); + const { sha, bytes } = await stage(env); + expect((await publishRef(env, {}, sha, bytes.byteLength)).status).toBe(201); + + const again = await publishRef(env, {}, sha, bytes.byteLength); + expect(again.status).toBe(409); + expect((await again.json() as any).error.code).toBe("version_exists"); + }); + + it("refuses a request carrying both an inline artifact and a reference", async () => { + const env = makeEnv(); + const form = new FormData(); + form.set("manifest", sampleManifest()); + form.set("artifact_ref_filename", FILENAME); + form.set("artifact", new Blob([BYTES]), FILENAME); + const res = await worker.fetch( + new Request("https://hub.amd-gaia.ai/publish", { + method: "POST", + headers: { authorization: "Bearer tok_amd" }, + body: form, + }), + env as never + ); + expect(res.status).toBe(400); + }); + + it("still requires authentication", async () => { + const env = makeEnv(); + const { sha } = await stage(env); + const res = await publishRef(env, { token: "not-a-real-token" }, sha); + expect(res.status).toBe(401); + }); +}); From caeb34a3ab475680bb73a05e36b8ef7a50a06048 Mon Sep 17 00:00:00 2001 From: Ovtcharov Date: Wed, 19 Aug 2026 16:55:40 -0700 Subject: [PATCH 2/3] ci(hub): upload oversized artifacts straight to R2 instead of through the Worker MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The publisher now routes any artifact at or above 90 MiB into R2 over the S3 API and publishes it by reference, rather than streaming it through a Worker request that Cloudflare rejects at 100 MB. The threshold sits below the real cap on purpose: an artifact growing into the limit should change lanes before it starts 413ing mid-release, not after. put_object, never upload_file. R2 records a whole-object SHA-256 only for single-part uploads, and the Worker refuses to publish an object it cannot verify — upload_file switches to multipart above its own threshold and drops the checksum, which would turn a green release into an unverifiable one. Shared by every hub publisher, so email and terminal-hub inherit it; both stay on the inline path today because they are small, and a test pins that so this does not quietly change how the working agents publish. Needs R2_ACCESS_KEY_ID, R2_SECRET_ACCESS_KEY and CLOUDFLARE_ACCOUNT_ID. These are R2 S3 credentials, NOT the CLOUDFLARE_API_TOKEN that deploys the Worker. Missing any of them fails loudly naming all three rather than falling back to the Worker path, which cannot work for these sizes. --- .github/workflows/release_components.yml | 18 +- .../email/python/packaging/publish_to_r2.py | 92 +++++++- .../tests/test_publish_to_r2_by_reference.py | 203 ++++++++++++++++++ workers/agent-hub/README.md | 32 +++ 4 files changed, 341 insertions(+), 4 deletions(-) create mode 100644 hub/agents/email/python/tests/test_publish_to_r2_by_reference.py diff --git a/.github/workflows/release_components.yml b/.github/workflows/release_components.yml index 3d24a5539..0a5ed8d85 100644 --- a/.github/workflows/release_components.yml +++ b/.github/workflows/release_components.yml @@ -312,7 +312,7 @@ jobs: - name: Install publisher deps if: needs.version.outputs.dry_run == 'false' - run: python -m pip install --upgrade requests pyyaml + run: python -m pip install --upgrade requests pyyaml boto3 - name: Publish to the Agent Hub (POST /publish) if: needs.version.outputs.dry_run == 'false' @@ -321,6 +321,13 @@ jobs: AGENT_HUB_PUBLISH_TOKEN: ${{ secrets.GAIA_HUB_TOKEN }} GAIA_HUB_PUBLISH_URL: ${{ vars.GAIA_HUB_PUBLISH_URL }} GAIA_HUB_BASE_URL: ${{ vars.GAIA_HUB_BASE_URL }} + # Artifacts at/over 90 MiB cannot travel through the Worker at all + # (Cloudflare caps request bodies at 100 MB on Free/Pro), so the + # publisher PUTs them straight into R2 and publishes them by + # reference. Only the Agent UI installers hit this today. + R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }} + R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }} + CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} run: | set -euo pipefail # Explicit = on every artifact: the publisher's @@ -425,7 +432,7 @@ jobs: - name: Install publisher deps if: needs.version.outputs.dry_run == 'false' - run: python -m pip install --upgrade requests pyyaml + run: python -m pip install --upgrade requests pyyaml boto3 - name: Publish to the Agent Hub (POST /publish) if: needs.version.outputs.dry_run == 'false' @@ -434,6 +441,13 @@ jobs: AGENT_HUB_PUBLISH_TOKEN: ${{ secrets.GAIA_HUB_TOKEN }} GAIA_HUB_PUBLISH_URL: ${{ vars.GAIA_HUB_PUBLISH_URL }} GAIA_HUB_BASE_URL: ${{ vars.GAIA_HUB_BASE_URL }} + # Artifacts at/over 90 MiB cannot travel through the Worker at all + # (Cloudflare caps request bodies at 100 MB on Free/Pro), so the + # publisher PUTs them straight into R2 and publishes them by + # reference. Only the Agent UI installers hit this today. + R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }} + R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }} + CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} VERSION: ${{ needs.version.outputs.version }} run: | set -euo pipefail diff --git a/hub/agents/email/python/packaging/publish_to_r2.py b/hub/agents/email/python/packaging/publish_to_r2.py index e2fc82b0a..4977c2824 100644 --- a/hub/agents/email/python/packaging/publish_to_r2.py +++ b/hub/agents/email/python/packaging/publish_to_r2.py @@ -42,6 +42,8 @@ from __future__ import annotations import argparse +import base64 +import contextlib import hashlib import json import os @@ -118,6 +120,74 @@ def _download_sha256(base_url: str, agent_id: str, version: str, filename: str) return hashlib.sha256(resp.content).hexdigest() +# Cloudflare caps a Worker request body by plan — 100 MB on Free/Pro. Anything +# at or above this goes straight to R2 over the S3 API instead, and is published +# by reference. Deliberately below the real cap so an artifact that grows into +# the limit switches lanes before it starts 413ing mid-release. +DIRECT_UPLOAD_THRESHOLD = 90 * 1024 * 1024 + + +def _r2_credentials() -> tuple[str, str, str] | None: + """R2 S3 credentials, or None when direct upload is not configured.""" + key = os.environ.get("R2_ACCESS_KEY_ID") + secret = os.environ.get("R2_SECRET_ACCESS_KEY") + account = os.environ.get("CLOUDFLARE_ACCOUNT_ID") + if key and secret and account: + return key, secret, account + return None + + +def _upload_to_r2(artifact_path: Path, key: str, sha_hex: str) -> None: + """PUT an artifact straight into the hub bucket, bypassing the Worker. + + Single-part on purpose: R2 records a whole-object SHA-256 only for + non-multipart uploads, and the Worker refuses to publish an object it cannot + verify. ``put_object`` is always single-part (``upload_file`` would switch to + multipart above its threshold and silently strip the checksum). + """ + try: + import boto3 # imported lazily: only the direct-upload path needs it + except ImportError as e: # pragma: no cover - environment problem, not logic + raise SystemExit( + "error: boto3 is required to upload artifacts larger than " + f"{DIRECT_UPLOAD_THRESHOLD} bytes directly to R2. " + "Install it (pip install boto3) and re-run." + ) from e + + creds = _r2_credentials() + if creds is None: + raise SystemExit( + f"error: {artifact_path.name} is too large to publish through the " + "Worker (Cloudflare caps request bodies at 100 MB on Free/Pro), so it " + "must go directly to R2 — but R2_ACCESS_KEY_ID, R2_SECRET_ACCESS_KEY " + "and CLOUDFLARE_ACCOUNT_ID are not all set. Create an R2 API token " + "(Cloudflare dashboard -> R2 -> Manage API Tokens) with Object " + "Read & Write on the hub bucket and set all three." + ) + access_key, secret_key, account_id = creds + bucket = os.environ.get("R2_BUCKET", "gaia-hub") + + client = boto3.client( + "s3", + endpoint_url=f"https://{account_id}.r2.cloudflarestorage.com", + aws_access_key_id=access_key, + aws_secret_access_key=secret_key, + region_name="auto", + ) + print( + f"[publish] uploading {artifact_path.name} -> r2://{bucket}/{key}", flush=True + ) + with artifact_path.open("rb") as fh: + client.put_object( + Bucket=bucket, + Key=key, + Body=fh, + ContentType="application/octet-stream", + # Base64, not hex — and this is what makes the object verifiable. + ChecksumSHA256=base64.b64encode(bytes.fromhex(sha_hex)).decode("ascii"), + ) + + def publish_one( base_url: str, manifest_path: Path, @@ -148,15 +218,33 @@ def publish_one( flush=True, ) - with artifact_path.open("rb") as fh: + # Oversized artifacts cannot travel through the Worker at all (Cloudflare + # caps the request body at 100 MB on Free/Pro), so they go straight to R2 and + # the POST below only carries their coordinates. The Worker verifies the + # stored object's size and SHA-256 before recording it, so this is a + # different transport, not a weaker guarantee. + by_reference = size >= DIRECT_UPLOAD_THRESHOLD + if by_reference: + _upload_to_r2( + artifact_path, f"agents/{agent_id}/{version}/{filename}", local_sha + ) + + with contextlib.ExitStack() as stack: files = { "manifest": ( "gaia-agent.yaml", manifest_path.read_bytes(), "application/x-yaml", ), - "artifact": (filename, fh, "application/octet-stream"), } + if by_reference: + files["artifact_ref_filename"] = (None, filename) + files["artifact_ref_sha256"] = (None, local_sha) + files["artifact_ref_size"] = (None, str(size)) + files["artifact_ref_content_type"] = (None, "application/octet-stream") + else: + fh = stack.enter_context(artifact_path.open("rb")) + files["artifact"] = (filename, fh, "application/octet-stream") # Same multipart field name + shape the Worker expects from # `gaia agent publish` (src/gaia/hub/publisher.py): the README becomes # the catalog entry's `readme` (rendered as sanitized markdown on the diff --git a/hub/agents/email/python/tests/test_publish_to_r2_by_reference.py b/hub/agents/email/python/tests/test_publish_to_r2_by_reference.py new file mode 100644 index 000000000..5c28925c8 --- /dev/null +++ b/hub/agents/email/python/tests/test_publish_to_r2_by_reference.py @@ -0,0 +1,203 @@ +# Copyright(C) 2025-2026 Advanced Micro Devices, Inc. All rights reserved. +# SPDX-License-Identifier: MIT +"""Which lane an artifact takes to the hub, and what actually goes on the wire. + +Cloudflare caps a Worker request body at 100 MB on Free/Pro, so an artifact +above that cannot be published through the Worker at all — it 413s at the edge. +The publisher therefore routes large artifacts straight into R2 and sends only +their coordinates. + +These tests assert the *shape* of both calls, not merely that they happened. A +mock that records "put_object was invoked" would still pass if the upload went +multipart, or if the checksum were omitted, or if the hex digest were sent where +base64 belongs — and every one of those makes the Worker refuse the publish, +during a release, after the binaries have been built. +""" + +from __future__ import annotations + +import base64 +import hashlib +import importlib.util +from pathlib import Path + +import pytest + +PACKAGING = Path(__file__).resolve().parents[1] / "packaging" +_spec = importlib.util.spec_from_file_location( + "email_publish_to_r2", PACKAGING / "publish_to_r2.py" +) +assert _spec and _spec.loader +pub = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(pub) + +MANIFEST_YAML = """\ +id: demo +name: Demo +version: 1.2.3 +description: "Demo agent" +author: AMD +license: MIT +language: python +category: conversation +""" + + +@pytest.fixture +def manifest(tmp_path: Path) -> Path: + p = tmp_path / "gaia-agent.yaml" + p.write_text(MANIFEST_YAML, encoding="utf-8") + return p + + +def _artifact( + tmp_path: Path, size: int, name: str = "demo-1.2.3-x64-setup.exe" +) -> Path: + p = tmp_path / name + p.write_bytes(b"x" * size) + return p + + +class _Resp: + """Stands in for the Worker, echoing the sha it would have recorded. + + The publisher re-checks that value against its own digest, so returning a + placeholder would mask a real mismatch rather than exercise the check. + """ + + status_code = 201 + + def __init__(self, sha: str) -> None: + self._sha = sha + + def json(self) -> dict: + return {"published": {"artifact": {"sha256": self._sha}}} + + +@pytest.fixture +def captured(monkeypatch): + """Capture the outgoing POST and any R2 upload, without performing either.""" + seen: dict = {"post": None, "put": None} + + def fake_post(url, headers=None, files=None, timeout=None): + # requests encodes each value as (filename, body[, content_type]); read + # the file handle now, before the caller's context manager closes it. + flat = {} + for k, v in (files or {}).items(): + body = v[1] + flat[k] = body.read() if hasattr(body, "read") else body + seen["post"] = {"url": url, "files": flat} + # By reference the Worker verifies R2's stored digest and returns it; + # inline it hashes the bytes it received. Model both. + sha = flat.get("artifact_ref_sha256") + if sha is None: + sha = hashlib.sha256(flat["artifact"]).hexdigest() + return _Resp(sha) + + class _S3: + def put_object(self, **kw): + seen["put"] = kw + + monkeypatch.setattr(pub.requests, "post", fake_post) + monkeypatch.setattr(pub, "_download_sha256", lambda *a, **k: None) + monkeypatch.setitem( + __import__("sys").modules, + "boto3", + type("m", (), {"client": lambda *a, **k: _S3()}), + ) + return seen + + +def _publish(manifest: Path, artifact: Path): + return pub.publish_one( + base_url="https://hub.example", + manifest_path=manifest, + manifest={"id": "demo", "version": "1.2.3"}, + artifact_path=artifact, + platform_key="win-x64", + token="tok", + ) + + +def test_a_small_artifact_still_rides_inline(manifest, tmp_path, captured, monkeypatch): + """The existing path must not change — email and terminal-hub depend on it.""" + monkeypatch.setenv("R2_ACCESS_KEY_ID", "k") + monkeypatch.setenv("R2_SECRET_ACCESS_KEY", "s") + monkeypatch.setenv("CLOUDFLARE_ACCOUNT_ID", "acct") + + _publish(manifest, _artifact(tmp_path, 1024)) + + assert captured["put"] is None, "a small artifact must not touch the S3 API" + files = captured["post"]["files"] + assert "artifact" in files + assert not any(k.startswith("artifact_ref_") for k in files) + + +def test_an_oversized_artifact_goes_to_r2_and_is_published_by_reference( + manifest, tmp_path, captured, monkeypatch +): + monkeypatch.setenv("R2_ACCESS_KEY_ID", "k") + monkeypatch.setenv("R2_SECRET_ACCESS_KEY", "s") + monkeypatch.setenv("CLOUDFLARE_ACCOUNT_ID", "acct") + size = pub.DIRECT_UPLOAD_THRESHOLD + 1 + art = _artifact(tmp_path, size) + sha = hashlib.sha256(art.read_bytes()).hexdigest() + + _publish(manifest, art) + + put = captured["put"] + assert put is not None, "an oversized artifact must be uploaded to R2" + assert put["Key"] == f"agents/demo/1.2.3/{art.name}" + # Base64 of the raw digest. Sending hex here is accepted by boto3 and then + # rejected by R2, which is a failure that only shows up mid-release. + assert put["ChecksumSHA256"] == base64.b64encode(bytes.fromhex(sha)).decode() + + files = captured["post"]["files"] + assert "artifact" not in files, "the bytes must not also travel through the Worker" + assert files["artifact_ref_filename"] == art.name + assert files["artifact_ref_sha256"] == sha + assert files["artifact_ref_size"] == str(size) + + +def test_put_object_is_used_so_the_upload_stays_single_part( + manifest, tmp_path, captured, monkeypatch +): + """R2 records a whole-object SHA-256 only for single-part uploads. + + `upload_file`/`upload_fileobj` switch to multipart above their threshold and + the checksum is lost, after which the Worker refuses the publish as + unverifiable. Pinning the call keeps that from regressing quietly. + """ + monkeypatch.setenv("R2_ACCESS_KEY_ID", "k") + monkeypatch.setenv("R2_SECRET_ACCESS_KEY", "s") + monkeypatch.setenv("CLOUDFLARE_ACCOUNT_ID", "acct") + + _publish(manifest, _artifact(tmp_path, pub.DIRECT_UPLOAD_THRESHOLD + 1)) + + assert set(captured["put"]) >= {"Bucket", "Key", "Body", "ChecksumSHA256"} + + +@pytest.mark.parametrize( + "missing", ["R2_ACCESS_KEY_ID", "R2_SECRET_ACCESS_KEY", "CLOUDFLARE_ACCOUNT_ID"] +) +def test_missing_r2_credentials_fail_loudly( + manifest, tmp_path, captured, monkeypatch, missing +): + """Never fall back to the Worker — that path 413s and wastes a release.""" + for name in ("R2_ACCESS_KEY_ID", "R2_SECRET_ACCESS_KEY", "CLOUDFLARE_ACCOUNT_ID"): + monkeypatch.setenv(name, "v") + monkeypatch.delenv(missing, raising=False) + + with pytest.raises(SystemExit) as e: + _publish(manifest, _artifact(tmp_path, pub.DIRECT_UPLOAD_THRESHOLD + 1)) + + msg = str(e.value) + assert "R2_ACCESS_KEY_ID" in msg and "100 MB" in msg + assert ( + captured["post"] is None + ), "nothing may be published when the upload cannot run" + + +def test_the_threshold_sits_below_cloudflares_real_cap(): + """Switch lanes before the cap, not at it, so growth never 413s a release.""" + assert pub.DIRECT_UPLOAD_THRESHOLD < 100 * 1024 * 1024 diff --git a/workers/agent-hub/README.md b/workers/agent-hub/README.md index 599d4f0dc..bf105659c 100644 --- a/workers/agent-hub/README.md +++ b/workers/agent-hub/README.md @@ -276,6 +276,38 @@ checked into the repo: `MAX_ARTIFACT_BYTES` (a plain var, default 250 MiB) caps artifact size and can be overridden per environment without a secret. +## Publishing artifacts larger than 100 MB + +A Worker request body is capped by the Cloudflare **account plan** — 100 MB on +Free and Pro, 200 MB Business, 500 MB Enterprise. `POST /publish` therefore +cannot carry the Agent UI installers (106-135 MiB); they are rejected with a +`413` by Cloudflare's edge before the Worker executes, so `MAX_ARTIFACT_BYTES` +is not involved and raising it changes nothing. + +Artifacts at or above 90 MiB are instead PUT straight into the bucket over R2's +S3 API and published **by reference**: the POST carries +`artifact_ref_{filename,sha256,size,content_type}` in place of the file part. + +Integrity is not relaxed. Before recording anything the Worker heads the object +and checks its size and SHA-256 against what R2 stored at PUT time. R2 keeps a +whole-object SHA-256 only for **single-part** uploads, so an object without one +is refused (`artifact_unverifiable`) rather than accepted on the publisher's +word — the uploader must use `put_object` with `ChecksumSHA256`, never +`upload_file`, which switches to multipart and drops the checksum. + +The publisher needs three extra secrets for this path: + +| Secret | How to get it | +|---|---| +| `R2_ACCESS_KEY_ID` | Cloudflare dashboard → R2 → **Manage API Tokens** → create a token with **Object Read & Write** on the hub bucket | +| `R2_SECRET_ACCESS_KEY` | Shown once alongside the access key id | +| `CLOUDFLARE_ACCOUNT_ID` | Same value the Worker deploy uses | + +These are R2 S3 credentials and are **not** the same as `CLOUDFLARE_API_TOKEN`, +which deploys the Worker. Missing any of them is a loud failure naming all +three; the publisher never silently falls back to the Worker path, because that +path 413s and would waste the release. + ## Deploying on Railway (demo) For demo/staging only: [`Dockerfile`](./Dockerfile) runs `wrangler dev` From 1a00c25b1b739283d5267e3db9b6035baf1cbf89 Mon Sep 17 00:00:00 2001 From: Ovtcharov Date: Thu, 20 Aug 2026 11:21:00 -0700 Subject: [PATCH 3/3] fix(hub): never overwrite a published artifact, and close the review gaps MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The critical one was mine and it failed silently, which is the worst shape. The publisher PUT to R2 before asking the Worker to record the artifact, but the by-reference immutability guard keys on the agent manifest and so fires only after that PUT. Re-publishing different bytes for a filename already published therefore replaced the stored object, left the catalog holding the OLD hash and size, and then the 409 handler re-downloaded the bytes it had just written — agreed with itself, printed "already published with identical bytes", and exited 0. Install-time verification would have been broken for every user with nothing failing anywhere. The publisher now checks before uploading, which also restores the meaning of that 409. The by-reference lane also skipped MAX_ARTIFACT_BYTES, so the ceiling the README calls the artifact size cap did not cover the one lane that exists for the largest artifacts. Enforced there too; the 250 MiB default clears the 135 MiB installers. agent-ui checked its credentials after waiting up to an hour for release assets and pulling ~400 MB. Moved ahead of the wait and extended to the R2 pair, so a missing secret fails in seconds. The flagship publisher is a separate script without this lane, and its frozen sidecar is the other artifact plausibly heading past 100 MB. Rather than duplicate the implementation it now refuses loudly at the same threshold and names what to port, instead of letting a release discover an HTML 413. --- .github/workflows/release_components.yml | 35 +++++++++----- .../email/python/packaging/publish_to_r2.py | 21 +++++++-- .../tests/test_publish_to_r2_by_reference.py | 47 ++++++++++++++++++- .../gaia/python/packaging/publish_to_r2.py | 20 ++++++++ workers/agent-hub/src/publish.ts | 14 ++++++ workers/agent-hub/test/fake-r2.ts | 2 +- .../test/publish-by-reference.test.ts | 10 ++++ 7 files changed, 133 insertions(+), 16 deletions(-) diff --git a/.github/workflows/release_components.yml b/.github/workflows/release_components.yml index 0a5ed8d85..97617fbd4 100644 --- a/.github/workflows/release_components.yml +++ b/.github/workflows/release_components.yml @@ -373,6 +373,30 @@ jobs: # yet. Wait for the full set, bounded — and fail loudly on timeout rather # than publishing a partial platform set that the catalog would then # advertise as complete. + # Before the wait, not after: this job can sit for up to an hour on the + # installer assets and then pull ~400 MB, and discovering a missing secret + # at the end wastes all of it. Covers the R2 credentials too — the + # installers are large enough to take the direct-to-R2 lane, so those are + # as load-bearing here as the hub token. + - name: Require the publish credentials + if: needs.version.outputs.dry_run == 'false' + env: + GAIA_HUB_TOKEN: ${{ secrets.GAIA_HUB_TOKEN }} + R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }} + R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }} + CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} + run: | + set -euo pipefail + missing="" + [ -n "${GAIA_HUB_TOKEN:-}" ] || missing="${missing} GAIA_HUB_TOKEN" + [ -n "${R2_ACCESS_KEY_ID:-}" ] || missing="${missing} R2_ACCESS_KEY_ID" + [ -n "${R2_SECRET_ACCESS_KEY:-}" ] || missing="${missing} R2_SECRET_ACCESS_KEY" + [ -n "${CLOUDFLARE_ACCOUNT_ID:-}" ] || missing="${missing} CLOUDFLARE_ACCOUNT_ID" + if [ -n "${missing}" ]; then + echo "::error::missing secret(s):${missing}. GAIA_HUB_TOKEN must match the Worker's PUBLISH_TOKENS; the R2_* pair are R2 S3 credentials (Cloudflare -> R2 -> Manage API Tokens, Object Read & Write) needed because these installers exceed the Worker request-body cap and go straight to R2. See workers/agent-hub/README.md." + exit 1 + fi + - name: Wait for the installer assets on the release if: needs.version.outputs.dry_run == 'false' shell: bash @@ -419,17 +443,6 @@ jobs: --pattern "gaia-agent-ui-${VERSION}-x86_64.AppImage" ls -la dist - - name: Require the publish token - if: needs.version.outputs.dry_run == 'false' - env: - GAIA_HUB_TOKEN: ${{ secrets.GAIA_HUB_TOKEN }} - run: | - set -euo pipefail - if [ -z "${GAIA_HUB_TOKEN:-}" ]; then - echo "::error::missing environment secret GAIA_HUB_TOKEN on the agent-publish environment (must match the Worker's PUBLISH_TOKENS). See workers/agent-hub/README.md." - exit 1 - fi - - name: Install publisher deps if: needs.version.outputs.dry_run == 'false' run: python -m pip install --upgrade requests pyyaml boto3 diff --git a/hub/agents/email/python/packaging/publish_to_r2.py b/hub/agents/email/python/packaging/publish_to_r2.py index 4977c2824..613ba3871 100644 --- a/hub/agents/email/python/packaging/publish_to_r2.py +++ b/hub/agents/email/python/packaging/publish_to_r2.py @@ -225,9 +225,24 @@ def publish_one( # different transport, not a weaker guarantee. by_reference = size >= DIRECT_UPLOAD_THRESHOLD if by_reference: - _upload_to_r2( - artifact_path, f"agents/{agent_id}/{version}/{filename}", local_sha - ) + # Check BEFORE uploading. The Worker's by-reference immutability guard + # keys on the agent manifest and therefore fires only AFTER this PUT + # would already have replaced the published bytes — leaving the catalog + # describing the old artifact while R2 serves the new one, and the 409 + # handler below re-downloading the bytes it just overwrote and happily + # agreeing with itself. Skipping the upload keeps that 409 meaningful. + download_url = f"{base_url.rstrip('/')}/agents/{agent_id}/{version}/{filename}" + head = requests.head(download_url, timeout=60, allow_redirects=True) + if head.status_code == 200: + print( + f"[publish] {filename} is already in R2 — not overwriting it; " + "the POST below verifies the stored bytes against this build.", + flush=True, + ) + else: + _upload_to_r2( + artifact_path, f"agents/{agent_id}/{version}/{filename}", local_sha + ) with contextlib.ExitStack() as stack: files = { diff --git a/hub/agents/email/python/tests/test_publish_to_r2_by_reference.py b/hub/agents/email/python/tests/test_publish_to_r2_by_reference.py index 5c28925c8..813bb3114 100644 --- a/hub/agents/email/python/tests/test_publish_to_r2_by_reference.py +++ b/hub/agents/email/python/tests/test_publish_to_r2_by_reference.py @@ -77,7 +77,7 @@ def json(self) -> dict: @pytest.fixture def captured(monkeypatch): """Capture the outgoing POST and any R2 upload, without performing either.""" - seen: dict = {"post": None, "put": None} + seen: dict = {"post": None, "put": None, "head": None, "already_published": False} def fake_post(url, headers=None, files=None, timeout=None): # requests encodes each value as (filename, body[, content_type]); read @@ -98,6 +98,14 @@ class _S3: def put_object(self, **kw): seen["put"] = kw + def fake_head(url, timeout=None, allow_redirects=None): + seen["head"] = url + return type( + "H", (), {"status_code": seen["already_published"] and 200 or 404} + )() + + monkeypatch.setattr(pub.requests, "head", fake_head) + monkeypatch.setattr(pub.requests, "post", fake_post) monkeypatch.setattr(pub, "_download_sha256", lambda *a, **k: None) monkeypatch.setitem( @@ -201,3 +209,40 @@ def test_missing_r2_credentials_fail_loudly( def test_the_threshold_sits_below_cloudflares_real_cap(): """Switch lanes before the cap, not at it, so growth never 413s a release.""" assert pub.DIRECT_UPLOAD_THRESHOLD < 100 * 1024 * 1024 + + +def test_an_already_published_object_is_never_overwritten( + manifest, tmp_path, captured, monkeypatch +): + """The failure this guards is silent, which is what makes it dangerous. + + The Worker's by-reference immutability check keys on the agent manifest, so + it fires only after the PUT. Upload first and a re-publish with different + bytes replaces the stored artifact, the catalog keeps the OLD hash, and the + 409 handler then re-downloads the bytes it just wrote — agreeing with itself + and exiting green while install-time verification is broken for everyone. + """ + monkeypatch.setenv("R2_ACCESS_KEY_ID", "k") + monkeypatch.setenv("R2_SECRET_ACCESS_KEY", "s") + monkeypatch.setenv("CLOUDFLARE_ACCOUNT_ID", "acct") + captured["already_published"] = True + + _publish(manifest, _artifact(tmp_path, pub.DIRECT_UPLOAD_THRESHOLD + 1)) + + assert captured["head"] is not None, "must check before uploading" + assert captured["put"] is None, "published bytes must never be overwritten" + + +def test_the_check_runs_before_the_upload_not_after( + manifest, tmp_path, captured, monkeypatch +): + """A first publish still uploads — the guard must not block the normal path.""" + monkeypatch.setenv("R2_ACCESS_KEY_ID", "k") + monkeypatch.setenv("R2_SECRET_ACCESS_KEY", "s") + monkeypatch.setenv("CLOUDFLARE_ACCOUNT_ID", "acct") + captured["already_published"] = False + + _publish(manifest, _artifact(tmp_path, pub.DIRECT_UPLOAD_THRESHOLD + 1)) + + assert captured["head"] is not None + assert captured["put"] is not None diff --git a/hub/agents/gaia/python/packaging/publish_to_r2.py b/hub/agents/gaia/python/packaging/publish_to_r2.py index 58ff7e602..895c95485 100644 --- a/hub/agents/gaia/python/packaging/publish_to_r2.py +++ b/hub/agents/gaia/python/packaging/publish_to_r2.py @@ -171,6 +171,12 @@ def _download_sha256(base_url: str, agent_id: str, version: str, filename: str) return hashlib.sha256(resp.content).hexdigest() +# Cloudflare's Worker request-body cap on Free/Pro. Mirrors +# DIRECT_UPLOAD_THRESHOLD's rationale in the email publisher, which has the +# direct-to-R2 lane this script lacks. +WORKER_BODY_LIMIT = 90 * 1024 * 1024 + + def publish_one( base_url: str, manifest_path: Path, @@ -185,6 +191,20 @@ def publish_one( raise SystemExit(f"error: artifact not found: {artifact_path}") filename = artifact_path.name local_sha, size = _sha256_file(artifact_path) + # Cloudflare caps a Worker request body by plan (100 MB on Free/Pro), so an + # artifact above it is rejected at the edge before the Worker runs — a bare + # 413 with an HTML body, mid-release, after the freeze has already been paid + # for. The email publisher has a direct-to-R2 lane for this; this one does + # not, so say so plainly rather than letting the release discover it. + if size >= WORKER_BODY_LIMIT: + raise SystemExit( + f"error: {artifact_path.name} is {size} bytes, at or over the " + f"{WORKER_BODY_LIMIT}-byte Cloudflare request-body cap, and this " + "publisher can only POST through the Worker. Port the direct-to-R2 " + "lane from hub/agents/email/python/packaging/publish_to_r2.py " + "(_upload_to_r2 + the artifact_ref_* fields) before releasing a " + "sidecar this large." + ) agent_id = str(manifest["id"]) version = str(manifest["version"]) publish_url = f"{base_url.rstrip('/')}{PUBLISH_PATH}" diff --git a/workers/agent-hub/src/publish.ts b/workers/agent-hub/src/publish.ts index 44c09d430..a4450a6ae 100644 --- a/workers/agent-hub/src/publish.ts +++ b/workers/agent-hub/src/publish.ts @@ -185,6 +185,20 @@ async function verifyUploadedArtifact( ); } + // The inline lane enforces this before storing; the by-reference lane must + // too, or MAX_ARTIFACT_BYTES silently stops being the artifact size cap the + // README says it is. The 250 MiB default clears the 135 MiB installers, so + // this costs nothing today and keeps one ceiling rather than two. + const limit = maxBytes(env); + if (head.size > limit) { + throw new HttpError( + 413, + "artifact_too_large", + `Object at ${key} is ${head.size} bytes, over the ${limit}-byte limit. ` + + `The catalog was not modified.` + ); + } + const stored = head.checksums?.sha256; if (!stored) { throw new HttpError( diff --git a/workers/agent-hub/test/fake-r2.ts b/workers/agent-hub/test/fake-r2.ts index 25f34a2b7..4a2fabc2f 100644 --- a/workers/agent-hub/test/fake-r2.ts +++ b/workers/agent-hub/test/fake-r2.ts @@ -56,7 +56,7 @@ function makeBody(obj: StoredObject) { /** hex -> ArrayBuffer, matching how R2 returns stored checksums. */ function hexToBuffer(hex: string): ArrayBuffer { const out = new Uint8Array(hex.length / 2); - for (let i = 0; i < out.length; i++) out[i] = parseInt(hex.substr(i * 2, 2), 16); + for (let i = 0; i < out.length; i++) out[i] = parseInt(hex.slice(i * 2, i * 2 + 2), 16); return out.buffer; } diff --git a/workers/agent-hub/test/publish-by-reference.test.ts b/workers/agent-hub/test/publish-by-reference.test.ts index bebf128c1..71cdc27e2 100644 --- a/workers/agent-hub/test/publish-by-reference.test.ts +++ b/workers/agent-hub/test/publish-by-reference.test.ts @@ -168,4 +168,14 @@ describe("publish by reference", () => { const res = await publishRef(env, { token: "not-a-real-token" }, sha); expect(res.status).toBe(401); }); + + it("applies MAX_ARTIFACT_BYTES to a by-reference publish too", async () => { + // Otherwise the documented artifact ceiling quietly stops covering the one + // lane that exists specifically for the largest artifacts. + const env = makeEnv({ maxBytes: "16" }); + const { sha, bytes } = await stage(env); + const res = await publishRef(env, {}, sha, bytes.byteLength); + expect(res.status).toBe(413); + expect(((await res.json()) as any).error.code).toBe("artifact_too_large"); + }); });