diff --git a/.github/workflows/release_agent_email.yml b/.github/workflows/release_agent_email.yml index f865c273b..a0903a769 100644 --- a/.github/workflows/release_agent_email.yml +++ b/.github/workflows/release_agent_email.yml @@ -503,6 +503,9 @@ jobs: npm run build - name: Assemble + publish the whole-package zip (all platforms + client + docs) + # DISABLED: the ~177 MB all-platforms zip exceeds Cloudflare's edge upload + # limit (POST returns 413). Revive via presigned-to-R2 or per-platform zips. + if: false shell: bash env: AGENT_HUB_PUBLISH_TOKEN: ${{ secrets.GAIA_HUB_TOKEN }} @@ -564,6 +567,7 @@ jobs: done - name: Verify the published package zip is fetchable at the edge + if: false # disabled together with the whole-package zip publish above shell: bash env: GAIA_HUB_BASE_URL: ${{ vars.GAIA_HUB_BASE_URL }} diff --git a/hub/agents/npm/agent-email/CHANGELOG.md b/hub/agents/npm/agent-email/CHANGELOG.md index 81e9be714..051e1a0db 100644 --- a/hub/agents/npm/agent-email/CHANGELOG.md +++ b/hub/agents/npm/agent-email/CHANGELOG.md @@ -5,6 +5,16 @@ follows [SemVer](https://semver.org/): the **MAJOR** of the on-the-wire `SCHEMA_VERSION` is what `checkVersion` enforces at startup, so a contract MAJOR bump is always at least a package MINOR bump with a migration note. +## 0.2.4 + +First fully-published release of this feature set. The whole-package zip download +(#1843) is temporarily disabled: the ~177 MB all-platforms zip exceeds Cloudflare's +edge upload limit, so the publish step rejected it (413) and blocked every prior +attempt (0.2.1–0.2.3). The worker-side streaming approach was reverted; 0.2.4 ships +the per-platform binaries + this npm client without the combined zip. Per-platform +binaries remain individually downloadable from the Hub. No agent wire-contract +change — `SCHEMA_VERSION` stays `2.0`. + ## 0.2.3 Re-cut of 0.2.2 after the Agent Hub worker was redeployed with the large-artifact diff --git a/hub/agents/npm/agent-email/assets/architecture.html b/hub/agents/npm/agent-email/assets/architecture.html index f24506523..ec326c017 100644 --- a/hub/agents/npm/agent-email/assets/architecture.html +++ b/hub/agents/npm/agent-email/assets/architecture.html @@ -83,7 +83,7 @@
@amd-gaia/agent-email - v0.2.3 + v0.2.4 architecture
$npm i @amd-gaia/agent-email
diff --git a/hub/agents/npm/agent-email/binaries.lock.json b/hub/agents/npm/agent-email/binaries.lock.json index 20d7a96b1..a2d0a8828 100644 --- a/hub/agents/npm/agent-email/binaries.lock.json +++ b/hub/agents/npm/agent-email/binaries.lock.json @@ -1,7 +1,7 @@ { "schemaVersion": "1.0", - "agentVersion": "0.2.3", - "baseUrl": "https://hub.amd-gaia.ai/agents/email/0.2.3", + "agentVersion": "0.2.4", + "baseUrl": "https://hub.amd-gaia.ai/agents/email/0.2.4", "binaries": { "win32-x64": { "filename": "email-agent-win32-x64.exe", diff --git a/hub/agents/npm/agent-email/package.json b/hub/agents/npm/agent-email/package.json index cff9a4580..fc3ad9be5 100644 --- a/hub/agents/npm/agent-email/package.json +++ b/hub/agents/npm/agent-email/package.json @@ -1,6 +1,6 @@ { "name": "@amd-gaia/agent-email", - "version": "0.2.3", + "version": "0.2.4", "type": "module", "description": "Thin JS/TS client + build-time binary fetcher + sidecar lifecycle helpers for the GAIA email agent (frozen, no-Python REST sidecar). Runs 100% locally on AMD Ryzen AI.", "author": "AMD AI Group", diff --git a/hub/agents/python/email/gaia-agent.yaml b/hub/agents/python/email/gaia-agent.yaml index b503ddbd7..0841d5ccb 100644 --- a/hub/agents/python/email/gaia-agent.yaml +++ b/hub/agents/python/email/gaia-agent.yaml @@ -1,6 +1,6 @@ id: email name: Email Triage -version: 0.2.3 +version: 0.2.4 description: "GAIA email triage agent — read, triage, organize, and reply to Gmail/Outlook locally" author: AMD license: MIT diff --git a/hub/agents/python/email/gaia_agent_email/version.py b/hub/agents/python/email/gaia_agent_email/version.py index 7f9d20081..1f34f568a 100644 --- a/hub/agents/python/email/gaia_agent_email/version.py +++ b/hub/agents/python/email/gaia_agent_email/version.py @@ -28,7 +28,7 @@ # Package build version. Keep in sync with ``pyproject.toml``'s ``version`` — # ``test_rest_contract.test_agent_version_matches_package_metadata`` asserts the # installed distribution metadata agrees with this literal so the two never drift. -AGENT_VERSION = "0.2.3" +AGENT_VERSION = "0.2.4" # REST/contract version exposed to hosts. Aliased to the frozen contract's # SCHEMA_VERSION so a contract bump is an API bump — no second number to forget. diff --git a/hub/agents/python/email/packaging/publish_to_r2.py b/hub/agents/python/email/packaging/publish_to_r2.py index 38b80604d..7ef7b7be9 100644 --- a/hub/agents/python/email/packaging/publish_to_r2.py +++ b/hub/agents/python/email/packaging/publish_to_r2.py @@ -42,7 +42,6 @@ from __future__ import annotations import argparse -import base64 import hashlib import json import os @@ -54,21 +53,11 @@ PUBLISH_PATH = "/publish" TOKEN_ENV = "AGENT_HUB_PUBLISH_TOKEN" -_CHUNK = 1 << 20 # 1 MiB read chunks for streaming sha256 def _sha256_file(path: Path) -> tuple[str, int]: - """Hash a file in chunks — never loads the full content into memory.""" - h = hashlib.sha256() - size = 0 - with path.open("rb") as fh: - while True: - chunk = fh.read(_CHUNK) - if not chunk: - break - h.update(chunk) - size += len(chunk) - return h.hexdigest(), size + data = path.read_bytes() + return hashlib.sha256(data).hexdigest(), len(data) def _read_token() -> str: @@ -129,109 +118,6 @@ def _download_sha256(base_url: str, agent_id: str, version: str, filename: str) return hashlib.sha256(resp.content).hexdigest() -def _b64(data: bytes) -> str: - """Base64-encode bytes to ASCII string.""" - return base64.b64encode(data).decode("ascii") - - -def publish_streaming( - base_url: str, - manifest_path: Path, - manifest: dict, - artifact_path: Path, - platform_key: str, - token: str, - package_files_bytes: bytes | None = None, -) -> dict: - """Stream the artifact as a raw request body (application/octet-stream). - - Used for the whole-package zip (~177 MB) to avoid buffering the full file - in memory, which would exceed Cloudflare's ~128 MB per-Worker memory limit. - Metadata travels in X-Gaia-* headers. The file handle is passed directly to - requests so it streams chunk-by-chunk without loading the full content. - """ - if not artifact_path.exists(): - raise SystemExit(f"error: artifact not found: {artifact_path}") - filename = artifact_path.name - local_sha, size = _sha256_file(artifact_path) - agent_id = str(manifest["id"]) - version = str(manifest["version"]) - publish_url = f"{base_url.rstrip('/')}{PUBLISH_PATH}" - - print( - f"[publish] {filename} ({size} bytes, sha256={local_sha[:12]}…) " - f"-> {agent_id}@{version} [streaming]", - flush=True, - ) - - manifest_bytes = manifest_path.read_bytes() - headers = { - "authorization": f"Bearer {token}", - "content-type": "application/octet-stream", - "content-length": str(size), - # Stored object content-type (the package artifact is a zip). The - # request body itself is octet-stream; this is the metadata the Worker - # persists on the R2 object. - "x-gaia-content-type": "application/zip", - "x-gaia-manifest": _b64(manifest_bytes), - "x-gaia-filename": filename, - "x-gaia-sha256": local_sha, - } - if package_files_bytes is not None: - headers["x-gaia-package-files"] = _b64(package_files_bytes) - - with artifact_path.open("rb") as fh: - resp = requests.post( - publish_url, - headers=headers, - data=fh, # streams chunk-by-chunk; never loads full file - timeout=600, - ) - - if resp.status_code == 201: - body = resp.json() - server_sha = body.get("published", {}).get("artifact", {}).get("sha256") - if server_sha != local_sha: - raise SystemExit( - f"error: integrity check FAILED for {filename}: Worker stored " - f"sha256={server_sha} but local sha256={local_sha}. The upload was " - "corrupted in transit; failing loudly." - ) - n = body.get("published", {}).get("version_artifacts", "?") - print( - f"[publish] OK 201 — stored, server sha256 verified. " - f"{agent_id}@{version} now has {n} artifact(s).", - flush=True, - ) - elif resp.status_code == 409: - remote_sha = _download_sha256(base_url, agent_id, version, filename) - if remote_sha != local_sha: - raise SystemExit( - f"error: {filename} is already published at {agent_id}@{version} " - f"with a DIFFERENT sha256 (remote={remote_sha}, local={local_sha}). " - "Published artifacts are immutable — bump the version to change it." - ) - print( - f"[publish] OK 409 — already published with identical bytes " - f"(idempotent no-op).", - flush=True, - ) - else: - raise SystemExit( - f"error: publish of {filename} failed: HTTP {resp.status_code} " - f"{resp.text[:500]}" - ) - - executable = "email-agent.exe" if filename.endswith(".exe") else "email-agent" - return { - "platform": platform_key, - "filename": filename, - "executable": executable, - "sha256": local_sha, - "size": size, - } - - def publish_one( base_url: str, manifest_path: Path, @@ -432,34 +318,19 @@ def main(argv=None) -> int: if path.name.lower().endswith(".zip") else _infer_platform_key(path.name) ) - if platform_key == "package": - # Whole-package zip: stream directly to R2 to avoid buffering ~177 MB - # in memory (Cloudflare Workers hard ~128 MB per-request limit). - results.append( - publish_streaming( - args.base_url, - args.manifest, - manifest, - path, - platform_key, - token, - package_files_bytes=package_files_bytes, - ) - ) - else: - results.append( - publish_one( - args.base_url, - args.manifest, - manifest, - path, - platform_key, - token, - readme_bytes=readme_bytes, - changelog_bytes=changelog_bytes, - package_files_bytes=package_files_bytes, - ) + results.append( + publish_one( + args.base_url, + args.manifest, + manifest, + path, + platform_key, + token, + readme_bytes=readme_bytes, + changelog_bytes=changelog_bytes, + package_files_bytes=package_files_bytes, ) + ) if args.summary_out: args.summary_out.write_text(json.dumps(results, indent=2), encoding="utf-8") diff --git a/hub/agents/python/email/pyproject.toml b/hub/agents/python/email/pyproject.toml index ccd1ea94d..a657c20b3 100644 --- a/hub/agents/python/email/pyproject.toml +++ b/hub/agents/python/email/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "gaia-agent-email" -version = "0.2.3" +version = "0.2.4" description = "GAIA email triage agent — read, triage, organize, and reply to Gmail/Outlook locally" authors = [{ name = "AMD" }] license = { text = "MIT" } diff --git a/tests/unit/test_publish_to_r2_stream.py b/tests/unit/test_publish_to_r2_stream.py deleted file mode 100644 index 4ce17013f..000000000 --- a/tests/unit/test_publish_to_r2_stream.py +++ /dev/null @@ -1,432 +0,0 @@ -# Copyright(C) 2025-2026 Advanced Micro Devices, Inc. All rights reserved. -# SPDX-License-Identifier: MIT -""" -Tests for hub/agents/python/email/packaging/publish_to_r2.py. - -Focuses on the call-validity contract at the HTTP boundary: - - package (platform_key == "package") → application/octet-stream with X-Gaia-* headers, - body via data= (streaming, NOT files=) - - non-package binaries → multipart/form-data via files=, no X-Gaia-* headers - - 201 with matching sha → success - - 409 with matching remote sha → idempotent success - - 409 with differing sha → SystemExit - - non-201/409 → SystemExit -""" - -from __future__ import annotations - -import hashlib -import json -import sys -from pathlib import Path -from unittest.mock import MagicMock, patch - -import pytest - -# --------------------------------------------------------------------------- -# Locate the module under test (not installed as a package). -# --------------------------------------------------------------------------- -_MODULE_PATH = ( - Path(__file__).parent.parent.parent - / "hub" - / "agents" - / "python" - / "email" - / "packaging" - / "publish_to_r2" -) -sys.path.insert(0, str(_MODULE_PATH.parent)) -import publish_to_r2 as pub # noqa: E402 - -# --------------------------------------------------------------------------- -# Helpers -# --------------------------------------------------------------------------- - -SAMPLE_MANIFEST_YAML = """\ -id: email -name: Email -version: 0.1.0 -description: "Email triage agent" -author: AMD -license: MIT -language: python -category: productivity -icon: mail -security_tier: verified -models: [Gemma-4-E4B-it-GGUF] -tags: [email, triage] -requirements: - min_memory_gb: 8 - platforms: [win-x64, linux-x64, darwin-arm64] -interfaces: - cli: true -""" - -SAMPLE_FILES_JSON = json.dumps( - { - "files": [ - {"name": "binaries/email-agent-linux-x64", "size_bytes": 35_000_000}, - {"name": "README.md", "size_bytes": 13_000}, - ] - } -) - - -def _sha256(data: bytes) -> str: - return hashlib.sha256(data).hexdigest() - - -def _make_zip(tmp_path: Path, content: bytes = b"fake-zip-bytes") -> Path: - p = tmp_path / "agent-email-0.1.0.zip" - p.write_bytes(content) - return p - - -def _make_binary( - tmp_path: Path, name: str = "email-agent-linux-x64", content: bytes = b"bin" -) -> Path: - p = tmp_path / name - p.write_bytes(content) - return p - - -def _make_manifest(tmp_path: Path) -> Path: - p = tmp_path / "gaia-agent.yaml" - p.write_text(SAMPLE_MANIFEST_YAML, encoding="utf-8") - return p - - -def _make_resp(status: int, body: dict | None = None) -> MagicMock: - r = MagicMock() - r.status_code = status - r.json.return_value = body or {} - r.text = json.dumps(body or {})[:500] - return r - - -# --------------------------------------------------------------------------- -# publish_streaming — call validity -# --------------------------------------------------------------------------- - - -class TestPublishStreamingCallValidity: - """Assert the exact shape of the HTTP call the streaming function makes.""" - - def test_uses_octet_stream_content_type(self, tmp_path): - """application/octet-stream with X-Gaia-* headers, not multipart.""" - zip_path = _make_zip(tmp_path) - manifest_path = _make_manifest(tmp_path) - manifest = {"id": "email", "version": "0.1.0"} - zip_sha = _sha256(zip_path.read_bytes()) - - fake_resp = _make_resp( - 201, - {"published": {"artifact": {"sha256": zip_sha}, "version_artifacts": 1}}, - ) - with patch("requests.post", return_value=fake_resp) as mock_post: - pub.publish_streaming( - "https://hub.example.com", - manifest_path, - manifest, - zip_path, - "package", - "tok_test", - ) - - mock_post.assert_called_once() - _, kwargs = mock_post.call_args - headers = kwargs["headers"] - - # Must be octet-stream, NOT multipart. - assert headers["content-type"] == "application/octet-stream" - # Required X-Gaia-* headers must be present. - assert "x-gaia-manifest" in headers - assert "x-gaia-filename" in headers - assert headers["x-gaia-filename"] == zip_path.name - assert "x-gaia-sha256" in headers - assert headers["x-gaia-sha256"] == zip_sha - # Content-Length must match file size. - assert headers["content-length"] == str(zip_path.stat().st_size) - # Body must be via data= (file handle for streaming), NOT files=. - assert "data" in kwargs, "expected data= kwarg for streaming" - assert "files" not in kwargs, "must NOT use files= on the streaming path" - - def test_includes_package_files_header_when_provided(self, tmp_path): - """X-Gaia-Package-Files header present when package_files_bytes passed.""" - zip_path = _make_zip(tmp_path) - manifest_path = _make_manifest(tmp_path) - manifest = {"id": "email", "version": "0.1.0"} - zip_sha = _sha256(zip_path.read_bytes()) - - fake_resp = _make_resp( - 201, - {"published": {"artifact": {"sha256": zip_sha}, "version_artifacts": 1}}, - ) - with patch("requests.post", return_value=fake_resp) as mock_post: - pub.publish_streaming( - "https://hub.example.com", - manifest_path, - manifest, - zip_path, - "package", - "tok_test", - package_files_bytes=SAMPLE_FILES_JSON.encode(), - ) - - _, kwargs = mock_post.call_args - assert "x-gaia-package-files" in kwargs["headers"] - - def test_omits_package_files_header_when_not_provided(self, tmp_path): - """X-Gaia-Package-Files absent when package_files_bytes not passed.""" - zip_path = _make_zip(tmp_path) - manifest_path = _make_manifest(tmp_path) - manifest = {"id": "email", "version": "0.1.0"} - zip_sha = _sha256(zip_path.read_bytes()) - - fake_resp = _make_resp( - 201, - {"published": {"artifact": {"sha256": zip_sha}, "version_artifacts": 1}}, - ) - with patch("requests.post", return_value=fake_resp) as mock_post: - pub.publish_streaming( - "https://hub.example.com", - manifest_path, - manifest, - zip_path, - "package", - "tok_test", - ) - - _, kwargs = mock_post.call_args - assert "x-gaia-package-files" not in kwargs["headers"] - - def test_201_matching_sha_returns_summary(self, tmp_path): - """201 with matching server sha → returns the summary dict.""" - zip_path = _make_zip(tmp_path) - manifest_path = _make_manifest(tmp_path) - manifest = {"id": "email", "version": "0.1.0"} - zip_sha = _sha256(zip_path.read_bytes()) - - fake_resp = _make_resp( - 201, - {"published": {"artifact": {"sha256": zip_sha}, "version_artifacts": 1}}, - ) - with patch("requests.post", return_value=fake_resp): - result = pub.publish_streaming( - "https://hub.example.com", - manifest_path, - manifest, - zip_path, - "package", - "tok_test", - ) - - assert result["platform"] == "package" - assert result["filename"] == zip_path.name - assert result["sha256"] == zip_sha - - def test_201_mismatched_sha_raises(self, tmp_path): - """201 with non-matching server sha → SystemExit (integrity failure).""" - zip_path = _make_zip(tmp_path) - manifest_path = _make_manifest(tmp_path) - manifest = {"id": "email", "version": "0.1.0"} - - fake_resp = _make_resp( - 201, - {"published": {"artifact": {"sha256": "a" * 64}, "version_artifacts": 1}}, - ) - with patch("requests.post", return_value=fake_resp): - with pytest.raises(SystemExit, match="integrity check FAILED"): - pub.publish_streaming( - "https://hub.example.com", - manifest_path, - manifest, - zip_path, - "package", - "tok_test", - ) - - def test_409_matching_remote_is_idempotent_success(self, tmp_path): - """409 with matching remote sha → idempotent no-op, returns summary.""" - content = b"zip-bytes" - zip_path = _make_zip(tmp_path, content) - manifest_path = _make_manifest(tmp_path) - manifest = {"id": "email", "version": "0.1.0"} - zip_sha = _sha256(content) - - fake_post_resp = _make_resp(409) - fake_get_resp = MagicMock() - fake_get_resp.status_code = 200 - fake_get_resp.content = content - - with patch("requests.post", return_value=fake_post_resp): - with patch("requests.get", return_value=fake_get_resp): - result = pub.publish_streaming( - "https://hub.example.com", - manifest_path, - manifest, - zip_path, - "package", - "tok_test", - ) - - assert result["sha256"] == zip_sha - - def test_409_differing_remote_raises(self, tmp_path): - """409 with different remote sha → SystemExit (immutability violation).""" - zip_path = _make_zip(tmp_path, b"local-bytes") - manifest_path = _make_manifest(tmp_path) - manifest = {"id": "email", "version": "0.1.0"} - - fake_post_resp = _make_resp(409) - fake_get_resp = MagicMock() - fake_get_resp.status_code = 200 - fake_get_resp.content = b"different-remote-bytes" - - with patch("requests.post", return_value=fake_post_resp): - with patch("requests.get", return_value=fake_get_resp): - with pytest.raises(SystemExit, match="DIFFERENT sha256"): - pub.publish_streaming( - "https://hub.example.com", - manifest_path, - manifest, - zip_path, - "package", - "tok_test", - ) - - def test_500_raises_systemexit(self, tmp_path): - """Non-201/409 response → SystemExit with actionable message.""" - zip_path = _make_zip(tmp_path) - manifest_path = _make_manifest(tmp_path) - manifest = {"id": "email", "version": "0.1.0"} - - fake_resp = _make_resp(500) - fake_resp.text = "Internal Server Error" - with patch("requests.post", return_value=fake_resp): - with pytest.raises(SystemExit, match="HTTP 500"): - pub.publish_streaming( - "https://hub.example.com", - manifest_path, - manifest, - zip_path, - "package", - "tok_test", - ) - - -# --------------------------------------------------------------------------- -# publish_one — regression: non-package binary stays multipart -# --------------------------------------------------------------------------- - - -class TestPublishOneBinaryStaysMultipart: - """The multipart path must be untouched for per-platform binaries.""" - - def test_binary_uses_multipart_files_kwarg(self, tmp_path): - """Per-platform binary uses files= (multipart), no X-Gaia-* headers.""" - binary = _make_binary(tmp_path) - manifest_path = _make_manifest(tmp_path) - manifest = {"id": "email", "version": "0.1.0"} - bin_sha = _sha256(binary.read_bytes()) - - fake_resp = _make_resp( - 201, - {"published": {"artifact": {"sha256": bin_sha}, "version_artifacts": 1}}, - ) - with patch("requests.post", return_value=fake_resp) as mock_post: - pub.publish_one( - "https://hub.example.com", - manifest_path, - manifest, - binary, - "linux-x64", - "tok_test", - ) - - _, kwargs = mock_post.call_args - # Must use files= (multipart), not data=. - assert "files" in kwargs, "expected files= kwarg for multipart path" - assert "data" not in kwargs, "must NOT use data= on the multipart path" - # Must NOT have X-Gaia-* headers. - headers = kwargs.get("headers", {}) - for key in headers: - assert not key.lower().startswith( - "x-gaia-" - ), f"Unexpected X-Gaia-* header on multipart path: {key}" - - -# --------------------------------------------------------------------------- -# main() routing: package → streaming, binary → multipart -# --------------------------------------------------------------------------- - - -class TestMainRouting: - """Integration-level test: main() routes zip to streaming, binary to multipart.""" - - def _run_main(self, argv: list[str], post_resp, get_resp=None) -> None: - with patch("requests.post", return_value=post_resp) as mock_post: - if get_resp is not None: - with patch("requests.get", return_value=get_resp): - with patch.dict( - "os.environ", {"AGENT_HUB_PUBLISH_TOKEN": "tok_test"} - ): - pub.main(argv) - else: - with patch.dict("os.environ", {"AGENT_HUB_PUBLISH_TOKEN": "tok_test"}): - pub.main(argv) - return mock_post - - def test_zip_routes_to_streaming(self, tmp_path): - zip_path = _make_zip(tmp_path) - manifest_path = _make_manifest(tmp_path) - zip_sha = _sha256(zip_path.read_bytes()) - - post_resp = _make_resp( - 201, - {"published": {"artifact": {"sha256": zip_sha}, "version_artifacts": 1}}, - ) - with patch("requests.post", return_value=post_resp) as mock_post: - with patch.dict("os.environ", {"AGENT_HUB_PUBLISH_TOKEN": "tok_test"}): - pub.main( - [ - "--base-url", - "https://hub.example.com", - "--manifest", - str(manifest_path), - "--artifact", - f"{zip_path}=package", - ] - ) - - _, kwargs = mock_post.call_args - # Zip → streaming path → data=, NOT files=. - assert "data" in kwargs - assert "files" not in kwargs - - def test_binary_routes_to_multipart(self, tmp_path): - binary = _make_binary(tmp_path) - manifest_path = _make_manifest(tmp_path) - bin_sha = _sha256(binary.read_bytes()) - - post_resp = _make_resp( - 201, - {"published": {"artifact": {"sha256": bin_sha}, "version_artifacts": 1}}, - ) - with patch("requests.post", return_value=post_resp) as mock_post: - with patch.dict("os.environ", {"AGENT_HUB_PUBLISH_TOKEN": "tok_test"}): - pub.main( - [ - "--base-url", - "https://hub.example.com", - "--manifest", - str(manifest_path), - "--artifact", - f"{binary}=linux-x64", - ] - ) - - _, kwargs = mock_post.call_args - # Binary → multipart path → files=, NOT data=. - assert "files" in kwargs - assert "data" not in kwargs diff --git a/workers/agent-hub/README.md b/workers/agent-hub/README.md index eeb9d99b2..5fc49cb5d 100644 --- a/workers/agent-hub/README.md +++ b/workers/agent-hub/README.md @@ -14,91 +14,12 @@ depend on any `src/gaia` code. | Route | Auth | Purpose | |-------|------|---------| -| `POST /publish` | Bearer | Publish a new agent version. Two Content-Type paths — see *Publish protocol* below. | +| `POST /publish` | Bearer | Publish a new agent version (validate → scope-check → immutability-check → checksum → store → rebuild index). Form parts: `manifest` (gaia-agent.yaml text), `artifact` (wheel/binary/zip file), optional `readme` + `changelog` (markdown, rendered on the Hub pages), and optional `package_files` (JSON `{files:[{name,size_bytes}]}` listing the contents of a whole-package `.zip` artifact — surfaced as the catalog's `package`) | | `GET /index.json` | none | Catalog of every agent (latest version only), including the latest README + CHANGELOG markdown | | `GET /agents//manifest.json` | none | Per-agent aggregate manifest (all versions) | | `GET /agents///` | none | Download an artifact, the raw `gaia-agent.yaml`, `README.md`, or `CHANGELOG.md` | | `GET /health` | none | Liveness probe | -### Publish protocol - -`POST /publish` accepts two Content-Type variants, routed by the first bytes of -the header value: - -#### `multipart/form-data` — per-platform binaries and wheels (existing path) - -Used for artifacts that fit comfortably under Cloudflare's per-Worker memory -budget (~40 MB platform binaries, Python wheels). The artifact is buffered -server-side to compute the SHA-256. - -Form parts: -| Part | Required | Description | -|------|----------|-------------| -| `manifest` | ✓ | `gaia-agent.yaml` text | -| `artifact` | ✓ | The wheel or binary file | -| `readme` | | README.md markdown (rendered on Hub pages) | -| `changelog` | | CHANGELOG.md markdown (rendered on Hub pages) | -| `package_files` | | JSON `{"files":[{"name":"…","size_bytes":0}]}` — file listing of a whole-package zip | - -Example: -```bash -curl -X POST https://hub.amd-gaia.ai/publish \ - -H "Authorization: Bearer $TOKEN" \ - -F "manifest=@gaia-agent.yaml" \ - -F "artifact=@email-agent-linux-x64" -``` - -#### `application/octet-stream` — whole-package zip (streaming path) - -Used for large artifacts (the ~177 MB whole-package zip) that would exceed -Cloudflare's ~128 MB per-Worker memory limit if buffered. The raw request body -is streamed directly to R2 without buffering; metadata travels in headers. - -Required headers: -| Header | Description | -|--------|-------------| -| `X-Gaia-Manifest` | Base64 of `gaia-agent.yaml` bytes (UTF-8-safe) | -| `X-Gaia-Filename` | Artifact filename (validated against `FILENAME_RE`) | -| `X-Gaia-Sha256` | Client-computed lowercase hex SHA-256 (R2 verifies on store) | -| `Content-Length` | Exact byte count (used for the pre-flight size guard) | - -Optional headers: -| Header | Description | -|--------|-------------| -| `X-Gaia-Package-Files` | Base64 of `package-files.json` bytes | -| `X-Gaia-Content-Type` | Artifact content-type (default `application/octet-stream`) | - -Integrity is enforced by passing `sha256` to `R2.put()` — R2 verifies the -streamed bytes against the provided hash and throws if they disagree; the Worker -catches this and returns `400 integrity_check_failed`. - -Example: -```bash -SHA=$(sha256sum agent-email-0.1.0.zip | awk '{print $1}') -curl -X POST https://hub.amd-gaia.ai/publish \ - -H "Authorization: Bearer $TOKEN" \ - -H "Content-Type: application/octet-stream" \ - -H "Content-Length: $(stat -c%s agent-email-0.1.0.zip)" \ - -H "X-Gaia-Manifest: $(base64 -w0 gaia-agent.yaml)" \ - -H "X-Gaia-Filename: agent-email-0.1.0.zip" \ - -H "X-Gaia-Sha256: $SHA" \ - --data-binary @agent-email-0.1.0.zip -``` - -Both paths return the same `201` body shape: -```json -{ - "published": { - "id": "email", - "version": "0.1.0", - "artifact": { "filename": "…", "path": "…", "size_bytes": 0, "sha256": "…", "content_type": "…" }, - "version_artifacts": 1, - "latest_version": "0.1.0" - }, - "catalog_agents": 1 -} -``` - ### Publish guarantees - **Per-publisher auth.** Bearer token resolved against the `PUBLISH_TOKENS` @@ -209,7 +130,7 @@ in plain Node without Miniflare. # Terminal 1 PUBLISH_TOKENS='{"dev-token":{"publisher":"AMD","authors":["AMD"]}}' npm run dev -# Terminal 2 — multipart (per-platform binary or wheel) +# Terminal 2 curl -X POST http://localhost:8787/publish \ -H "Authorization: Bearer dev-token" \ -F "manifest=@hub/agents/python/chat/gaia-agent.yaml" \ @@ -217,17 +138,6 @@ curl -X POST http://localhost:8787/publish \ -F "readme=@hub/agents/python/chat/README.md;type=text/markdown" \ -F "changelog=@hub/agents/python/chat/CHANGELOG.md;type=text/markdown" -# Terminal 2 — streaming octet-stream (whole-package zip) -SHA=$(sha256sum dist/agent-email-0.1.0.zip | awk '{print $1}') -curl -X POST http://localhost:8787/publish \ - -H "Authorization: Bearer dev-token" \ - -H "Content-Type: application/octet-stream" \ - -H "Content-Length: $(stat -f%z dist/agent-email-0.1.0.zip)" \ - -H "X-Gaia-Manifest: $(base64 -i hub/agents/python/email/gaia-agent.yaml)" \ - -H "X-Gaia-Filename: agent-email-0.1.0.zip" \ - -H "X-Gaia-Sha256: $SHA" \ - --data-binary @dist/agent-email-0.1.0.zip - curl http://localhost:8787/index.json ``` diff --git a/workers/agent-hub/src/publish.ts b/workers/agent-hub/src/publish.ts index 117b6f4fa..2e7ea6299 100644 --- a/workers/agent-hub/src/publish.ts +++ b/workers/agent-hub/src/publish.ts @@ -4,26 +4,11 @@ /** * POST /publish handler. * - * Two upload paths, selected by Content-Type: - * - * 1. multipart/form-data (EXISTING) — per-platform binaries and wheels. The - * artifact rides in the `artifact` form part (buffered). README, CHANGELOG, - * and package_files are optional form parts. All artifacts under ~128 MB - * (the Cloudflare Worker per-request memory limit). - * - * 2. application/octet-stream (NEW, streaming) — the whole-package zip (~177 MB). - * The raw request body is the artifact; metadata travels in X-Gaia-* headers. - * The body is passed directly to R2 without buffering, so memory usage is - * independent of artifact size. R2-side sha256 integrity is enforced via the - * X-Gaia-Sha256 header. - * - * Both paths share the same tail: makeVersionEntry → upsertVersion → - * writeAgentManifest → rebuildIndex → 201. - * - * Flow for each path: - * authenticate → parse inputs → validate manifest → enforce publisher scope - * → enforce version immutability → store artifact → write ancillary objects - * → rebuild index. Every guard fails loudly with a structured error. + * Flow: authenticate -> parse multipart -> validate manifest -> enforce + * publisher scope -> enforce version immutability -> generate server-side + * SHA-256 -> store artifact + raw manifest + optional README + optional + * CHANGELOG + per-agent manifest -> rebuild index.json. Every guard fails + * loudly with a structured error. */ import { assertAuthorAllowed, authenticate } from "./auth"; @@ -39,13 +24,11 @@ import { readmeKey, writeAgentManifest, } from "./storage"; -import type { ArtifactInfo, AgentManifest, Env, ParsedManifest } from "./types"; +import type { ArtifactInfo, Env } from "./types"; const DEFAULT_MAX_BYTES = 262_144_000; // 250 MiB // Artifact filename: a single safe path segment (no traversal, no separators). const FILENAME_RE = /^[A-Za-z0-9][A-Za-z0-9._+-]*$/; -// Lowercase hex sha256: exactly 64 hex digits. -const SHA256_RE = /^[0-9a-f]{64}$/; /** Lowercase hex SHA-256 of the given bytes, computed in the Worker. */ export async function sha256Hex(bytes: ArrayBuffer | Uint8Array): Promise { @@ -95,11 +78,15 @@ async function optionalMarkdownPart( } /** - * Validate the package_files JSON text: must be `{ files: [{ name, size_bytes }] }`. - * Returns the canonically re-serialized text on success. Throws HttpError on malformed input. - * Used by both the multipart and streaming paths. + * Read + validate the optional `package_files` part: the listing of files inside + * the published whole-package zip. Must be JSON of shape + * `{ files: [{ name, size_bytes }] }`. Absent → null (no package zip). A + * present-but-malformed part fails loudly rather than storing junk. */ -function validatePackageFilesJson(text: string): string { +async function optionalPackageFiles(form: FormData): Promise { + const part = form.get("package_files"); + if (part == null) return null; + const text = typeof part === "string" ? part : await (part as Blob).text(); let parsed: unknown; try { parsed = JSON.parse(text); @@ -107,8 +94,8 @@ function validatePackageFilesJson(text: string): string { throw new HttpError( 400, "invalid_request", - `The 'package_files' value is not valid JSON: ${(e as Error).message}. Expected ` + - `{ "files": [{ "name": "...", "size_bytes": 0 }] }, or omit it.` + `The 'package_files' part is not valid JSON: ${(e as Error).message}. Expected ` + + `{ "files": [{ "name": "...", "size_bytes": 0 }] }, or omit the part.` ); } const files = (parsed as { files?: unknown }).files; @@ -125,7 +112,7 @@ function validatePackageFilesJson(text: string): string { throw new HttpError( 400, "invalid_request", - "The 'package_files' value must be { \"files\": [{ \"name\": string, " + + "The 'package_files' part must be { \"files\": [{ \"name\": string, " + '"size_bytes": number }, ...] } with at least one file.' ); } @@ -133,235 +120,6 @@ function validatePackageFilesJson(text: string): string { return JSON.stringify({ files }); } -/** - * Read + validate the optional `package_files` form part. Returns null when absent. - */ -async function optionalPackageFiles(form: FormData): Promise { - const part = form.get("package_files"); - if (part == null) return null; - const text = typeof part === "string" ? part : await (part as Blob).text(); - return validatePackageFilesJson(text); -} - -/** - * UTF-8-safe base64 decode: handles non-ASCII characters in the encoded payload. - */ -function decodeBase64Utf8(b64: string): string { - return new TextDecoder().decode(Uint8Array.from(atob(b64), (c) => c.charCodeAt(0))); -} - -/** - * Shared publish tail: write ancillary objects → update catalog → rebuild index → 201. - * - * Currently called only by the streaming path; the multipart path inlines an - * equivalent tail (which also writes the optional README/CHANGELOG parts). - * Keep the two in sync until the multipart path is migrated here. - */ -async function finishPublish( - env: Env, - manifest: ParsedManifest, - artifact: ArtifactInfo, - manifestText: string, - packageFilesText: string | null, - existing: AgentManifest | null, - versionExists: boolean, - publisher: string, - now: Date -): Promise { - // The raw gaia-agent.yaml, README, and CHANGELOG are per-version records. - // Write only on the first publish of a version; a later artifact in the same - // version must not overwrite them. - if (!versionExists) { - await env.BUCKET.put(rawManifestKey(manifest.id, manifest.version), manifestText, { - httpMetadata: { contentType: "application/x-yaml; charset=utf-8" }, - }); - } - - // The package file listing rides the whole-package zip POST. It must not be - // gated on !versionExists: the zip arrives AFTER per-platform binaries have - // already created the version. Write it once (head check prevents re-write). - if ( - packageFilesText != null && - !(await env.BUCKET.head(packageFilesKey(manifest.id, manifest.version))) - ) { - await env.BUCKET.put(packageFilesKey(manifest.id, manifest.version), packageFilesText, { - httpMetadata: { contentType: "application/json; charset=utf-8" }, - }); - } - - const versionEntry = makeVersionEntry(manifest, artifact, publisher, now.toISOString()); - const updated = upsertVersion(existing, manifest, versionEntry); - await writeAgentManifest(env.BUCKET, updated); - - const index = await rebuildIndex(env.BUCKET, now); - - return json( - { - published: { - id: manifest.id, - version: manifest.version, - artifact, - version_artifacts: updated.versions[manifest.version].artifacts.length, - latest_version: updated.latest_version, - }, - catalog_agents: index.agents.length, - }, - 201 - ); -} - -/** - * Handle a streaming application/octet-stream publish. - * - * Metadata travels in X-Gaia-* headers; the raw request body is passed - * directly to R2 without buffering. - */ -async function handleStreamingPublish( - request: Request, - env: Env, - publisher: { publisher: string; authors: string[] }, - now: Date -): Promise { - // Read required headers. - const manifestB64 = request.headers.get("x-gaia-manifest"); - if (!manifestB64) { - throw new HttpError(400, "invalid_request", "Missing required header X-Gaia-Manifest."); - } - const filename = request.headers.get("x-gaia-filename"); - if (!filename) { - throw new HttpError(400, "invalid_request", "Missing required header X-Gaia-Filename."); - } - const sha256 = request.headers.get("x-gaia-sha256"); - if (!sha256) { - throw new HttpError(400, "invalid_request", "Missing required header X-Gaia-Sha256."); - } - if (!SHA256_RE.test(sha256)) { - throw new HttpError( - 400, - "invalid_request", - `X-Gaia-Sha256 must be a lowercase 64-hex-digit SHA-256 hash; got: ${JSON.stringify(sha256)}.` - ); - } - - let manifestText: string; - try { - manifestText = decodeBase64Utf8(manifestB64); - } catch { - throw new HttpError(400, "invalid_request", "X-Gaia-Manifest is not valid base64."); - } - - if (!FILENAME_RE.test(filename)) { - throw new HttpError( - 400, - "invalid_artifact", - `Artifact filename ${JSON.stringify(filename)} is invalid. Use a single path ` + - `segment of letters, digits, '.', '_', '+', '-' (e.g. 'agent-email-0.1.0.zip').` - ); - } - - // Optional X-Gaia-Package-Files: base64 of package-files.json text. - let packageFilesText: string | null = null; - const packageFilesB64 = request.headers.get("x-gaia-package-files"); - if (packageFilesB64) { - let raw: string; - try { - raw = decodeBase64Utf8(packageFilesB64); - } catch { - throw new HttpError(400, "invalid_request", "X-Gaia-Package-Files is not valid base64."); - } - packageFilesText = validatePackageFilesJson(raw); - } - - const contentType = - request.headers.get("x-gaia-content-type") ?? "application/octet-stream"; - - const manifest = parseManifest(manifestText); - assertAuthorAllowed(publisher, manifest.author); - - // Size guard without buffering. Content-Length is advisory (a client could - // under-declare it); R2's own object-size limit is the real ceiling, and the - // streamed body is never buffered in the Worker regardless. - const declaredLenRaw = request.headers.get("content-length"); - const declaredLen = declaredLenRaw ? Number(declaredLenRaw) : NaN; - if (!declaredLen || declaredLen <= 0 || !Number.isFinite(declaredLen)) { - throw new HttpError(400, "invalid_artifact", "Content-Length is missing or zero; cannot determine artifact size."); - } - const limit = maxBytes(env); - if (declaredLen > limit) { - throw new HttpError( - 413, - "artifact_too_large", - `Content-Length ${declaredLen} bytes exceeds the ${limit}-byte limit.` - ); - } - - // Ownership check. - const existing = await readAgentManifest(env.BUCKET, manifest.id); - if (existing && existing.author !== manifest.author) { - throw new HttpError( - 403, - "forbidden_scope", - `Agent '${manifest.id}' is owned by author '${existing.author}'. A publish ` + - `with author '${manifest.author}' cannot update it.` - ); - } - const versionExists = Boolean(existing?.versions[manifest.version]); - - const key = artifactKey(manifest.id, manifest.version, filename); - - // Per-filename immutability. - if (await env.BUCKET.head(key)) { - throw new HttpError( - 409, - "version_exists", - `Artifact already exists at ${key} and is immutable. To add another ` + - `platform binary use a distinct filename; to change this one, bump the version.` - ); - } - - // Stream-store with R2-side integrity — no buffering. - if (!request.body) { - throw new HttpError(400, "invalid_artifact", "Request body is empty."); - } - let stored: { size: number } | null = null; - try { - stored = await env.BUCKET.put(key, request.body, { - httpMetadata: { contentType }, - sha256, - }); - } catch (e) { - throw new HttpError( - 400, - "integrity_check_failed", - `Stored bytes did not match X-Gaia-Sha256=${sha256} (or the R2 put failed): ${(e as Error).message}.` - ); - } - if (!stored) { - throw new HttpError(500, "store_error", "R2 put returned null; artifact status unknown."); - } - - const sizeBytes = stored.size; - const artifact: ArtifactInfo = { - filename, - path: key, - size_bytes: sizeBytes, - sha256, - content_type: contentType, - }; - - return finishPublish( - env, - manifest, - artifact, - manifestText, - packageFilesText, - existing, - versionExists, - publisher.publisher, - now - ); -} - export async function handlePublish( request: Request, env: Env, @@ -370,26 +128,16 @@ export async function handlePublish( const publisher = authenticate(request, env); const contentType = request.headers.get("content-type") ?? ""; - - // Route to the streaming path for application/octet-stream. - if (contentType.toLowerCase().startsWith("application/octet-stream")) { - return handleStreamingPublish(request, env, publisher, now); - } - if (!contentType.toLowerCase().includes("multipart/form-data")) { throw new HttpError( 415, "unsupported_media_type", - "POST /publish expects either multipart/form-data (per-platform binaries/wheels) " + - "or application/octet-stream (streaming whole-package zip with X-Gaia-* headers). " + - "See workers/agent-hub/README.md for the full protocol." + "POST /publish expects multipart/form-data with 'manifest' (gaia-agent.yaml " + + "text), 'artifact' (the wheel or binary file), and optionally 'readme' " + + "(README.md markdown text) and 'changelog' (CHANGELOG.md markdown text) parts." ); } - // -------------------------------------------------------------------------- - // Multipart path (EXISTING — unchanged behavior) - // -------------------------------------------------------------------------- - let form: FormData; try { form = await request.formData(); @@ -494,7 +242,6 @@ export async function handlePublish( httpMetadata: { contentType: artifact.content_type }, sha256, }); - // The raw gaia-agent.yaml, README, and CHANGELOG are per-version records: // write them only on the first publish of a version so a later platform binary // joining the same version cannot rewrite them. diff --git a/workers/agent-hub/test/fake-r2.ts b/workers/agent-hub/test/fake-r2.ts index feb0c3d19..e3e7d8fd5 100644 --- a/workers/agent-hub/test/fake-r2.ts +++ b/workers/agent-hub/test/fake-r2.ts @@ -14,37 +14,16 @@ interface StoredObject { uploaded: Date; } -async function toBytesAsync( - value: string | ArrayBuffer | ArrayBufferView | Uint8Array | ReadableStream -): Promise { +function toBytes(value: string | ArrayBuffer | ArrayBufferView | Uint8Array): Uint8Array { if (typeof value === "string") return new TextEncoder().encode(value); if (value instanceof Uint8Array) return new Uint8Array(value); if (value instanceof ArrayBuffer) return new Uint8Array(value); if (ArrayBuffer.isView(value)) { return new Uint8Array(value.buffer, value.byteOffset, value.byteLength); } - // ReadableStream — consume chunk-by-chunk (streaming path). - if (value instanceof ReadableStream) { - const reader = value.getReader(); - const chunks: Uint8Array[] = []; - for (;;) { - const { done, value: chunk } = await reader.read(); - if (done) break; - chunks.push(chunk instanceof Uint8Array ? chunk : new Uint8Array(chunk)); - } - const total = chunks.reduce((n, c) => n + c.byteLength, 0); - const out = new Uint8Array(total); - let offset = 0; - for (const c of chunks) { - out.set(c, offset); - offset += c.byteLength; - } - return out; - } throw new TypeError("Unsupported R2 put value type in fake-r2."); } - function makeBody(obj: StoredObject) { const bytes = obj.bytes; return { @@ -76,7 +55,7 @@ export class FakeR2 { // eslint-disable-next-line @typescript-eslint/no-explicit-any async put(key: string, value: any, options?: any): Promise { - const bytes = await toBytesAsync(value); + const bytes = toBytes(value); const contentType = options?.httpMetadata?.contentType ?? "application/octet-stream"; // Honour R2's optional sha256 integrity check so tests catch mismatches. if (options?.sha256) { @@ -200,44 +179,6 @@ export function publishRequest(opts: { }); } -/** - * Build a POST /publish application/octet-stream (streaming) request. - * The artifact bytes become the raw body; metadata travels in X-Gaia-* headers. - */ -export function streamPublishRequest(opts: { - token?: string; - manifestYaml: string; - bytes: Uint8Array | string; - filename: string; - sha256: string; - packageFiles?: string; - contentType?: string; -}): Request { - const bodyBytes = - typeof opts.bytes === "string" ? new TextEncoder().encode(opts.bytes) : opts.bytes; - - function b64(text: string): string { - return btoa(unescape(encodeURIComponent(text))); - } - - const headers = new Headers(); - if (opts.token) headers.set("authorization", `Bearer ${opts.token}`); - headers.set("content-type", opts.contentType ?? "application/octet-stream"); - headers.set("content-length", String(bodyBytes.byteLength)); - headers.set("x-gaia-manifest", b64(opts.manifestYaml)); - headers.set("x-gaia-filename", opts.filename); - headers.set("x-gaia-sha256", opts.sha256); - if (opts.packageFiles !== undefined) headers.set("x-gaia-package-files", b64(opts.packageFiles)); - - return new Request("https://hub.amd-gaia.ai/publish", { - method: "POST", - headers, - body: bodyBytes, - // @ts-expect-error — duplex is needed for streaming in some environments - duplex: "half", - }); -} - /** A valid sample gaia-agent.yaml for tests. */ export function sampleManifest(overrides: Partial> = {}): string { const id = overrides.id ?? "chat"; diff --git a/workers/agent-hub/test/publish.test.ts b/workers/agent-hub/test/publish.test.ts index 5b445f7b4..0dff23c72 100644 --- a/workers/agent-hub/test/publish.test.ts +++ b/workers/agent-hub/test/publish.test.ts @@ -5,7 +5,7 @@ import { describe, expect, it } from "vitest"; import worker from "../src/index"; import type { AgentManifest, CatalogIndex } from "../src/types"; -import { makeEnv, publishRequest, sampleManifest, streamPublishRequest } from "./fake-r2"; +import { makeEnv, publishRequest, sampleManifest } from "./fake-r2"; async function publish(env: ReturnType, opts: Parameters[0]) { return worker.fetch(publishRequest(opts), env as never); @@ -744,272 +744,3 @@ describe("POST /publish — security tier & deprecation", () => { expect(index.agents.find((a) => a.id === "old")!.deprecated).toBe(true); }); }); - -// --------------------------------------------------------------------------- -// Streaming path (application/octet-stream with X-Gaia-* headers) -// --------------------------------------------------------------------------- - -async function hexSha256(data: Uint8Array | string): Promise { - const bytes = typeof data === "string" ? new TextEncoder().encode(data) : data; - const digest = await crypto.subtle.digest("SHA-256", bytes); - return [...new Uint8Array(digest)].map((b) => b.toString(16).padStart(2, "0")).join(""); -} - -async function streamPublish( - env: ReturnType, - opts: Parameters[0] -) { - return worker.fetch(streamPublishRequest(opts), env as never); -} - -describe("POST /publish — streaming (octet-stream) path", () => { - it("(a) stores artifact, returns 201, artifact.sha256 matches, size correct, object present", async () => { - const env = makeEnv(); - const artifactContent = "large-zip-bytes-streamed"; - const sha256 = await hexSha256(artifactContent); - const res = await streamPublish(env, { - token: "tok_amd", - manifestYaml: sampleManifest({ id: "email", name: "Email" }), - bytes: artifactContent, - filename: "agent-email-0.1.0.zip", - sha256, - }); - expect(res.status).toBe(201); - const body = (await res.json()) as any; - expect(body.published.artifact.sha256).toBe(sha256); - expect(body.published.artifact.size_bytes).toBe(artifactContent.length); - expect(env.bucket.keys()).toContain("agents/email/0.1.0/agent-email-0.1.0.zip"); - // Artifact bytes must be stored correctly. - const stored = await env.bucket.get("agents/email/0.1.0/agent-email-0.1.0.zip"); - expect(await stored!.text()).toBe(artifactContent); - }); - - it("(b) WRONG X-Gaia-Sha256 → R2 integrity check fails → non-201 with integrity_check_failed", async () => { - const env = makeEnv(); - const artifactContent = "real-bytes"; - const wrongSha = "a".repeat(64); // wrong hex sha256 - const res = await streamPublish(env, { - token: "tok_amd", - manifestYaml: sampleManifest({ id: "email", name: "Email" }), - bytes: artifactContent, - filename: "agent-email-0.1.0.zip", - sha256: wrongSha, - }); - expect(res.status).not.toBe(201); - const body = (await res.json()) as any; - expect(body.error.code).toBe("integrity_check_failed"); - // Nothing must be stored. - expect(env.bucket.keys()).toEqual([]); - }); - - it("(c) Content-Length exceeds MAX_ARTIFACT_BYTES → 413 artifact_too_large", async () => { - const env = makeEnv({ maxBytes: "4" }); - const artifactContent = "way-too-many-bytes"; - const sha256 = await hexSha256(artifactContent); - const res = await streamPublish(env, { - token: "tok_amd", - manifestYaml: sampleManifest({ id: "email", name: "Email" }), - bytes: artifactContent, - filename: "agent-email-0.1.0.zip", - sha256, - }); - expect(res.status).toBe(413); - expect(((await res.json()) as any).error.code).toBe("artifact_too_large"); - }); - - it("(d) writes package-files.json when X-Gaia-Package-Files header present", async () => { - const env = makeEnv(); - const filesJson = JSON.stringify({ - files: [ - { name: "binaries/email-agent-linux-x64", size_bytes: 35000000 }, - { name: "README.md", size_bytes: 13000 }, - ], - }); - const artifactContent = "zip-content"; - const sha256 = await hexSha256(artifactContent); - const res = await streamPublish(env, { - token: "tok_amd", - manifestYaml: sampleManifest({ id: "email", name: "Email" }), - bytes: artifactContent, - filename: "agent-email-0.1.0.zip", - sha256, - packageFiles: filesJson, - }); - expect(res.status).toBe(201); - const keys = env.bucket.keys(); - expect(keys).toContain("agents/email/0.1.0/package-files.json"); - const pf = (await (await env.bucket.get("agents/email/0.1.0/package-files.json"))!.json()) as any; - expect(pf.files).toHaveLength(2); - }); - - it("(d2) does NOT write package-files.json when X-Gaia-Package-Files header is absent", async () => { - const env = makeEnv(); - const artifactContent = "zip-content"; - const sha256 = await hexSha256(artifactContent); - const res = await streamPublish(env, { - token: "tok_amd", - manifestYaml: sampleManifest({ id: "email", name: "Email" }), - bytes: artifactContent, - filename: "agent-email-0.1.0.zip", - sha256, - }); - expect(res.status).toBe(201); - expect(env.bucket.keys()).not.toContain("agents/email/0.1.0/package-files.json"); - }); - - it("(e) re-POST same filename → 409 version_exists", async () => { - const env = makeEnv(); - const artifactContent = "zip-bytes"; - const sha256 = await hexSha256(artifactContent); - const first = await streamPublish(env, { - token: "tok_amd", - manifestYaml: sampleManifest({ id: "email", name: "Email" }), - bytes: artifactContent, - filename: "agent-email-0.1.0.zip", - sha256, - }); - expect(first.status).toBe(201); - - const second = await streamPublish(env, { - token: "tok_amd", - manifestYaml: sampleManifest({ id: "email", name: "Email" }), - bytes: artifactContent, - filename: "agent-email-0.1.0.zip", - sha256, - }); - expect(second.status).toBe(409); - expect(((await second.json()) as any).error.code).toBe("version_exists"); - }); - - it("(f) missing required header X-Gaia-Manifest → 400 invalid_request", async () => { - const env = makeEnv(); - // Build request manually without X-Gaia-Manifest. - const artifactContent = "zip-bytes"; - const sha256 = await hexSha256(artifactContent); - const req = new Request("https://hub.amd-gaia.ai/publish", { - method: "POST", - headers: { - authorization: "Bearer tok_amd", - "content-type": "application/octet-stream", - "content-length": String(artifactContent.length), - "x-gaia-filename": "agent-email-0.1.0.zip", - "x-gaia-sha256": sha256, - // X-Gaia-Manifest intentionally omitted - }, - body: artifactContent, - }); - const res = await worker.fetch(req, env as never); - expect(res.status).toBe(400); - expect(((await res.json()) as any).error.code).toBe("invalid_request"); - }); - - it("(f2) missing required header X-Gaia-Filename → 400 invalid_request", async () => { - const env = makeEnv(); - const artifactContent = "zip-bytes"; - const sha256 = await hexSha256(artifactContent); - const manifestYaml = sampleManifest({ id: "email", name: "Email" }); - function b64(text: string): string { - return btoa(unescape(encodeURIComponent(text))); - } - const req = new Request("https://hub.amd-gaia.ai/publish", { - method: "POST", - headers: { - authorization: "Bearer tok_amd", - "content-type": "application/octet-stream", - "content-length": String(artifactContent.length), - "x-gaia-manifest": b64(manifestYaml), - "x-gaia-sha256": sha256, - // X-Gaia-Filename intentionally omitted - }, - body: artifactContent, - }); - const res = await worker.fetch(req, env as never); - expect(res.status).toBe(400); - expect(((await res.json()) as any).error.code).toBe("invalid_request"); - }); - - it("(f3) missing required header X-Gaia-Sha256 → 400 invalid_request", async () => { - const env = makeEnv(); - const artifactContent = "zip-bytes"; - const manifestYaml = sampleManifest({ id: "email", name: "Email" }); - function b64(text: string): string { - return btoa(unescape(encodeURIComponent(text))); - } - const req = new Request("https://hub.amd-gaia.ai/publish", { - method: "POST", - headers: { - authorization: "Bearer tok_amd", - "content-type": "application/octet-stream", - "content-length": String(artifactContent.length), - "x-gaia-manifest": b64(manifestYaml), - "x-gaia-filename": "agent-email-0.1.0.zip", - // X-Gaia-Sha256 intentionally omitted - }, - body: artifactContent, - }); - const res = await worker.fetch(req, env as never); - expect(res.status).toBe(400); - expect(((await res.json()) as any).error.code).toBe("invalid_request"); - }); - - it("(g) regression — existing multipart path still passes (checksum test)", async () => { - // This is a quick regression guard: the multipart path must be untouched. - const env = makeEnv(); - const artifact = "deterministic-artifact-content"; - const res = await worker.fetch( - publishRequest({ - token: "tok_amd", - manifestYaml: sampleManifest(), - artifact, - filename: "gaia_agent_chat-0.1.0-py3-none-any.whl", - }), - env as never - ); - expect(res.status).toBe(201); - const digest = await crypto.subtle.digest("SHA-256", new TextEncoder().encode(artifact)); - const expected = [...new Uint8Array(digest)] - .map((b) => b.toString(16).padStart(2, "0")) - .join(""); - expect(((await res.json()) as any).published.artifact.sha256).toBe(expected); - }); - - it("streaming path writes manifest.json and index.json (end-to-end)", async () => { - const env = makeEnv(); - const artifactContent = "whole-package-zip-bytes"; - const sha256 = await hexSha256(artifactContent); - const res = await streamPublish(env, { - token: "tok_amd", - manifestYaml: sampleManifest({ id: "email", name: "Email" }), - bytes: artifactContent, - filename: "agent-email-0.1.0.zip", - sha256, - }); - expect(res.status).toBe(201); - const keys = env.bucket.keys(); - expect(keys).toContain("agents/email/manifest.json"); - expect(keys).toContain("index.json"); - const index = (await (await env.bucket.get("index.json"))!.json()) as CatalogIndex; - expect(index.agents.map((a) => a.id)).toContain("email"); - }); - - it("streaming path does NOT write gaia-agent.yaml, README, or CHANGELOG (no drive-by)", async () => { - // The streaming path is zip-only: no README/CHANGELOG, and gaia-agent.yaml only on first publish. - const env = makeEnv(); - const artifactContent = "zip-content"; - const sha256 = await hexSha256(artifactContent); - const res = await streamPublish(env, { - token: "tok_amd", - manifestYaml: sampleManifest({ id: "email", name: "Email" }), - bytes: artifactContent, - filename: "agent-email-0.1.0.zip", - sha256, - }); - expect(res.status).toBe(201); - const keys = env.bucket.keys(); - // gaia-agent.yaml IS written on first publish. - expect(keys).toContain("agents/email/0.1.0/gaia-agent.yaml"); - // README and CHANGELOG must NOT be written (streaming path has no form parts for them). - expect(keys).not.toContain("agents/email/0.1.0/README.md"); - expect(keys).not.toContain("agents/email/0.1.0/CHANGELOG.md"); - }); -});