diff --git a/.github/postgres-integration-suites.toml b/.github/postgres-integration-suites.toml index a21681d658..343885dddf 100644 --- a/.github/postgres-integration-suites.toml +++ b/.github/postgres-integration-suites.toml @@ -47,11 +47,17 @@ name = "persistence" package = "signalbox-persistence" features = ["postgres-integration"] # The long pole by an order of magnitude: 695 of the 1,134 ignored tests these -# suites hold. Three shards still took about 8m15s each at the measured -# baseline, so six bring the test-execution long pole into the four-minute band. -# Counts are a snapshot, taken with `cargo test --tests -- --ignored --list`; -# they justify the shard count and nothing depends on them staying exact. -shards = 6 +# suites hold. Counts are a snapshot, taken with `cargo test --tests -- +# --ignored --list`; they justify the shard count and nothing depends on them +# staying exact. +# +# Six shards were sized when every test database wrote to the shared scratch +# disks. With that state in memory and sixteen tests running per shard, a +# shard ran about 3m45s of which roughly 1m15s was fixed cost — archive +# download, runner setup, image pull — so three shards spend three fewer pool +# slots and their repeated fixed costs for the same tests. Not fewer than two, +# so a failing shard can be re-run without re-running the whole suite. +shards = 3 skip = [] exclude_binaries = ["runner_protocol_postgres"] @@ -59,9 +65,10 @@ exclude_binaries = ["runner_protocol_postgres"] name = "runner" package = "signalbox-persistence" features = ["postgres-integration"] -# The 263 runner-protocol tests took about eight minutes as one partition at the -# measured baseline, so they are split across two runners. -shards = 2 +# The 263 runner-protocol tests ran as two partitions of under four minutes +# each at sixteen test threads with database state in memory; one partition of +# about six minutes pays the fixed shard costs once. +shards = 1 skip = [] include_binaries = ["runner_protocol_postgres"] @@ -76,9 +83,10 @@ skip = [] name = "signalboxd" package = "signalboxd" features = ["test-support"] -# One partition took about five minutes at the measured baseline; two keep it -# from replacing persistence as the integration-test long pole. -shards = 2 +# Two partitions ran under three and a half minutes each at sixteen test +# threads with database state in memory; one partition of about five minutes +# pays the fixed shard costs once and stays well short of the long pole. +shards = 1 # Subprocess entry points invoked only by their owning process tests. skip = ["bridge_build_direct_invocation_fixture", "bridge_wait_"] @@ -90,3 +98,10 @@ shards = 1 # A live-credential smoke, owner- and CI-excluded, that the previous `cargo # test` invocation excluded with `--skip`. skip = ["terminal_client_completes_the_real_anthropic_path"] + +[[suite]] +name = "approval-judge-eval" +package = "signalbox-approval-judge-eval" +features = ["postgres-integration"] +shards = 1 +skip = [] diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index c16a31e7ca..a76390a877 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -92,7 +92,7 @@ jobs: # Exact version pin, as cargo-deny is pinned in deny.yml: an instrumented # build and its report must come from one known tool build. - name: Install cargo-llvm-cov - uses: taiki-e/install-action@a2a5f6e99e1a31540baa0468acfa302cff0f359f # v2.86.4 + uses: taiki-e/install-action@b6ff580856c41316412a0b9b60540fbc6f8c82cc # v2.86.7 with: tool: cargo-llvm-cov@0.8.7 # Each measured suite runs under `--no-report`, which accumulates diff --git a/.github/workflows/deny.yml b/.github/workflows/deny.yml index 25c082b784..e29d6180fb 100644 --- a/.github/workflows/deny.yml +++ b/.github/workflows/deny.yml @@ -30,7 +30,7 @@ jobs: with: persist-credentials: false - name: Install cargo-deny - uses: taiki-e/install-action@a2a5f6e99e1a31540baa0468acfa302cff0f359f # v2.86.4 + uses: taiki-e/install-action@b6ff580856c41316412a0b9b60540fbc6f8c82cc # v2.86.7 with: tool: cargo-deny@0.20.2 - name: Check advisories, bans, licenses, and sources diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 226114c3d1..5af48e2794 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -20,17 +20,31 @@ permissions: env: RUSTFLAGS: "-D warnings" + # CI never reuses incremental artifacts across runs; they are pure write + # amplification on the self-hosted runners' scratch NVMe (measured 2026-08-22). + CARGO_INCREMENTAL: "0" + # Line tables keep backtraces and coverage useful at a fraction of the + # debuginfo bytes written per build. + CARGO_PROFILE_DEV_DEBUG: "line-tables-only" + CARGO_PROFILE_TEST_DEBUG: "line-tables-only" jobs: # Keep the required `validate` and `postgres-integration` checks present on - # every pull-request head. Their aggregate jobs below accept skipped heavy - # work only when this job proves every changed path is Markdown or under - # `docs/`; pushes to main always run the complete workflow. + # every pull-request head. Their aggregate jobs below accept skipped Rust + # work only when this job proves no changed path can reach the Rust build: + # every path is either non-binding Markdown/`docs/` prose or browser-client + # source under `clients/web/` (the generated contract under + # `clients/web/src/generated/` is Rust output and keeps the full bar, as + # does `clients/web`'s own workflow file). Pushes to main always run the + # complete workflow. `clients/web/**` changes are gated by `web.yml`, which + # triggers on exactly that path set. rust-change-scope: name: Rust change scope runs-on: ubuntu-latest outputs: docs_only: ${{ steps.paths.outputs.docs_only }} + web_only: ${{ steps.paths.outputs.web_only }} + skip_rust: ${{ steps.paths.outputs.skip_rust }} steps: - name: Check out repository history uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -45,8 +59,12 @@ jobs: HEAD_SHA: ${{ github.event.pull_request.head.sha }} run: | docs_only=false + web_only=false + skip_rust=false if [ "$EVENT_NAME" = pull_request ]; then - docs_only=true + docs=0 + web=0 + rust=0 changed_paths="$RUNNER_TEMP/rust-changed-files" git diff --no-renames --name-only -z \ "$BASE_SHA...$HEAD_SHA" > "$changed_paths" @@ -57,19 +75,35 @@ jobs: AGENTS.md | */AGENTS.md | docs/agents/* | docs/spec/* \ | docs/domain-spine.md | docs/invariants.md \ | docs/scenarios.md | docs/style.md) - docs_only=false - break + rust=$((rust + 1)) ;; docs/* | *.md) + docs=$((docs + 1)) + ;; + # Generated browser-contract output is produced from Rust and + # checked against it by the workspace tests. + clients/web/src/generated/*) + rust=$((rust + 1)) + ;; + clients/web/*) + web=$((web + 1)) ;; *) - docs_only=false - break + rust=$((rust + 1)) ;; esac done < "$changed_paths" + if [ "$rust" -eq 0 ]; then + skip_rust=true + [ "$web" -eq 0 ] && docs_only=true + [ "$docs" -eq 0 ] && web_only=true + fi + echo "changed paths: docs=$docs web=$web rust-reaching=$rust" \ + "-> skip_rust=$skip_rust" fi echo "docs_only=$docs_only" >> "$GITHUB_OUTPUT" + echo "web_only=$web_only" >> "$GITHUB_OUTPUT" + echo "skip_rust=$skip_rust" >> "$GITHUB_OUTPUT" contract-checks: runs-on: ubuntu-latest @@ -157,6 +191,8 @@ jobs: run: python3 scripts/test_check_docs_consistency.py - name: Test the PostgreSQL suite manifest reader run: python3 scripts/test_postgres_integration_suites.py + - name: Test the convergence reconciler + run: python3 tooling/convergence-reconciler/test_reconcile.py - name: Test the built-executable resolver run: python3 tooling/test_resolve_cargo_bin.py - name: Test the lifecycle watchdog @@ -195,7 +231,7 @@ jobs: inventory_hash="$(sha256sum \ crates/persistence/src/lock_inventory.rs | cut -d ' ' -f 1)" test "$inventory_hash" = \ - 238f712efd43df95063e56eb04d3f221d06eb75fb32231cd594341974a9c315c + 09620879011e0dd0ec0a8136ab25aa6f59d4f991366db4f66edf4f4da35c8299 unexpected="$(git ls-files -z 'crates/persistence/src/*.rs' \ | xargs -0 perl -0777 -ne ' next if $ARGV =~ m{/lock_inventory[.]rs$}; @@ -210,7 +246,7 @@ jobs: } rust-checks: needs: rust-change-scope - if: needs.rust-change-scope.outputs.docs_only != 'true' + if: needs.rust-change-scope.outputs.skip_rust != 'true' runs-on: signalbox # The self-hosted pool starts with cold caches; give the workspace check # headroom until they warm. @@ -221,10 +257,56 @@ jobs: with: fetch-depth: 1 persist-credentials: false + - name: Use the node-local cargo target cache + # Self-hosted runners expose a persistent per-node cache directory + # through /home/runner/_work/_ci-cache, provisioned by the runner pod + # template outside this repository. Each run gets its own target dir, + # cloned copy-on-write (reflink, same XFS filesystem) from the last + # published run with the same lockfile/toolchain key, so concurrent + # runs never share a cargo lock and nothing one run writes leaks into + # another. GitHub-hosted runners have no such directory and keep the + # default. Run dirs older than a day are pruned across every key and + # job, except the directory a key's "latest" seed points at. + id: nodecache + shell: bash + run: | + cache=/home/runner/_work/_ci-cache + if [ -d "$cache/target" ]; then + # Prune expired run dirs under every key/job, keeping current seeds. + find "$cache/target" -mindepth 4 -maxdepth 4 -type d -path '*/runs/*' -mtime +0 -print0 2>/dev/null \ + | while IFS= read -r -d '' dir; do + base="$(dirname "$(dirname "$dir")")" + seed="$(readlink -f "$base/latest" 2>/dev/null || true)" + [ "$dir" = "$seed" ] || rm -rf "$dir" + done + find "$cache/target" -mindepth 1 -maxdepth 3 -type d -empty -delete 2>/dev/null || true + key="${{ hashFiles('Cargo.lock', 'rust-toolchain.toml') }}" + job="${{ github.job }}${{ strategy.job-index != '' && format('-{0}', strategy.job-index) || '' }}" + base="$cache/target/$key/$job" + mkdir -p "$base/runs" + target="$base/runs/${{ github.run_id }}-${{ github.run_attempt }}" + rm -rf "$target" + if [ -d "$base/latest" ] && cp --reflink=always --no-preserve=timestamps -a "$base/latest/." "$target" 2>/dev/null; then + echo "seeded $target from the last published run (reflink)" + else + rm -rf "$target"; mkdir -p "$target" + echo "no seed available; fresh target dir $target" + fi + touch "$target" + echo "CARGO_TARGET_DIR=$target" >> "$GITHUB_ENV" + echo "NODECACHE_BASE=$base" >> "$GITHUB_ENV" + echo "active=true" >> "$GITHUB_OUTPUT" + else + echo "active=false" >> "$GITHUB_OUTPUT" + echo "no node-local cache directory; using the default target dir" + fi - name: Cache Rust dependencies uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 with: cache-on-failure: true + # With the node-local target dir active, uploading/extracting target + # archives through the GitHub cache is pure write amplification. + cache-targets: ${{ steps.nodecache.outputs.active != 'true' }} shared-key: rust save-if: false - name: Check workspace @@ -238,6 +320,19 @@ jobs: # repository contracts, checker tests, formatting, and Rust compilation run # concurrently. This aggregate keeps the required `validate` check bound to # every command it covered before the split. + - name: Publish this run's target dir as the cache seed + # Seeds come only from pushes and same-repository pull requests, never + # from forks; a unique temporary link keeps concurrent publishers from + # racing each other before the atomic rename. + if: >- + success() && steps.nodecache.outputs.active == 'true' + && (github.event_name != 'pull_request' + || github.event.pull_request.head.repo.full_name == github.repository) + shell: bash + run: | + tmp="$NODECACHE_BASE/latest.tmp.${{ github.run_id }}-${{ github.run_attempt }}" + ln -sfn "$CARGO_TARGET_DIR" "$tmp" && mv -T "$tmp" "$NODECACHE_BASE/latest" + echo "cache seed -> $CARGO_TARGET_DIR" validate-checks: if: ${{ always() }} needs: @@ -250,11 +345,11 @@ jobs: CONTRACT_RESULT: ${{ needs.contract-checks.result }} CHECKER_RESULT: ${{ needs.checker-and-format-tests.result }} RUST_RESULT: ${{ needs.rust-checks.result }} - DOCS_ONLY: ${{ needs.rust-change-scope.outputs.docs_only }} + SKIP_RUST: ${{ needs.rust-change-scope.outputs.skip_rust }} run: | test "$CONTRACT_RESULT" = success test "$CHECKER_RESULT" = success - if [ "$DOCS_ONLY" = true ]; then + if [ "$SKIP_RUST" = true ]; then test "$RUST_RESULT" = skipped else test "$RUST_RESULT" = success @@ -266,7 +361,7 @@ jobs: # share a target directory, so this job carries its own rust-cache entry. workspace-tests: needs: rust-change-scope - if: needs.rust-change-scope.outputs.docs_only != 'true' + if: needs.rust-change-scope.outputs.skip_rust != 'true' runs-on: ubuntu-latest timeout-minutes: 20 steps: @@ -275,19 +370,55 @@ jobs: with: persist-credentials: false - name: Install Bubblewrap - run: sudo apt-get install --yes bubblewrap + run: | + sudo apt-get update + sudo apt-get install --yes bubblewrap - name: Enable unprivileged user namespaces for Bubblewrap run: sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 + - name: Delegate the file-media task controller + run: | + sudo mkdir /sys/fs/cgroup/signalbox-file-media-ci + sudo sh -c 'echo +pids +memory > /sys/fs/cgroup/cgroup.subtree_control' + sudo mkdir /sys/fs/cgroup/signalbox-file-media-ci/runner + sudo sh -c 'echo +pids +memory > /sys/fs/cgroup/signalbox-file-media-ci/cgroup.subtree_control' + sudo chown -R "$USER" /sys/fs/cgroup/signalbox-file-media-ci - name: Cache Rust dependencies uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 with: cache-on-failure: true + # This job's shared cache contains a stale nested target path + # (`target/tests/target`) that rust-cache traverses after restore and + # reports as ENOENT. Keep registry/git dependency caching, but build + # test artifacts in the fresh hosted-runner target directory. + cache-targets: false shared-key: rust save-if: ${{ github.ref == 'refs/heads/main' }} - name: Run tests env: SIGNALBOX_RUN_BWRAP_INTEGRATION: "1" - run: cargo test --no-fail-fast --workspace --all-targets --all-features + SIGNALBOX_FILE_MEDIA_CGROUP_ROOT: /sys/fs/cgroup/signalbox-file-media-ci + # Process-heavy integration targets can exhaust the hosted runner's + # file-descriptor limit under libtest's unrestricted CPU-sized pool. + # Four concurrent tests matches the explicit cap used by the + # PostgreSQL jobs and retains parallel execution without false EMFILE + # failures. + run: | + runner_uid="$(id -u)" + runner_gid="$(id -g)" + runner_home="$HOME" + runner_path="$PATH" + test_command='status=0 + cargo test --no-fail-fast --workspace --all-targets --all-features \ + -- --test-threads=4 || status=$? + cargo test --no-fail-fast -p signalbox-file-media-processor-runtime \ + --features test-worker --test isolation -- --ignored || status=$? + exit "$status"' + sudo --preserve-env=CI,PATH,CARGO_HOME,RUSTUP_HOME,RUSTFLAGS,CARGO_INCREMENTAL,CARGO_PROFILE_DEV_DEBUG,CARGO_PROFILE_TEST_DEBUG,SIGNALBOX_RUN_BWRAP_INTEGRATION,SIGNALBOX_FILE_MEDIA_CGROUP_ROOT \ + sh -c ' + echo $$ > "$SIGNALBOX_FILE_MEDIA_CGROUP_ROOT/runner/cgroup.procs" + exec setpriv --reuid="$1" --regid="$2" --init-groups \ + env HOME="$3" PATH="$5" sh -c "$4" + ' sh "$runner_uid" "$runner_gid" "$runner_home" "$test_command" "$runner_path" - name: Test generated browser contract runtime run: node --test crates/web-contract/tests/generated_roundtrip.mjs # `--all-targets` excludes doctests, so the domain's compile_fail @@ -310,10 +441,10 @@ jobs: env: CHECKS_RESULT: ${{ needs.validate-checks.result }} TESTS_RESULT: ${{ needs.workspace-tests.result }} - DOCS_ONLY: ${{ needs.rust-change-scope.outputs.docs_only }} + SKIP_RUST: ${{ needs.rust-change-scope.outputs.skip_rust }} run: | test "$CHECKS_RESULT" = success - if [ "$DOCS_ONLY" = true ]; then + if [ "$SKIP_RUST" = true ]; then test "$TESTS_RESULT" = skipped else test "$TESTS_RESULT" = success @@ -325,7 +456,7 @@ jobs: name: Domain instruction counts (report only) continue-on-error: true needs: rust-change-scope - if: needs.rust-change-scope.outputs.docs_only != 'true' + if: needs.rust-change-scope.outputs.skip_rust != 'true' runs-on: signalbox timeout-minutes: 20 steps: @@ -336,10 +467,46 @@ jobs: persist-credentials: false # The pinned Signalbox runner image includes Valgrind, so this public-repo # job needs neither root access nor an unpinned package installation. + - name: Use the node-local cargo target cache + # Self-hosted runners expose a persistent per-node cache directory + # through /home/runner/_work/_ci-cache, provisioned by the runner pod + # template outside this repository. Each run gets its own target dir, + # cloned copy-on-write (reflink, same XFS filesystem) from the last + # successful run with the same lockfile/toolchain key, so concurrent + # runs never share a cargo lock and nothing one run writes leaks into + # another. GitHub-hosted runners have no such directory and keep the + # default. Run dirs untouched for a day are pruned at job start. + id: nodecache + shell: bash + run: | + cache=/home/runner/_work/_ci-cache + if [ -d "$cache/target" ]; then + key="${{ hashFiles('Cargo.lock', 'rust-toolchain.toml') }}" + job="${{ github.job }}${{ strategy.job-index != '' && format('-{0}', strategy.job-index) || '' }}" + base="$cache/target/$key/$job" + mkdir -p "$base/runs" + find "$base/runs" -mindepth 1 -maxdepth 1 -type d -mtime +0 -exec rm -rf {} + 2>/dev/null || true + target="$base/runs/${{ github.run_id }}-${{ github.run_attempt }}" + if [ -d "$base/latest" ] && cp --reflink=always -a "$base/latest/." "$target" 2>/dev/null; then + echo "seeded $target from the last successful run (reflink)" + else + rm -rf "$target"; mkdir -p "$target" + echo "no seed available; fresh target dir $target" + fi + echo "CARGO_TARGET_DIR=$target" >> "$GITHUB_ENV" + echo "NODECACHE_BASE=$base" >> "$GITHUB_ENV" + echo "active=true" >> "$GITHUB_OUTPUT" + else + echo "active=false" >> "$GITHUB_OUTPUT" + echo "no node-local cache directory; using the default target dir" + fi - name: Cache Rust dependencies uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 with: cache-on-failure: true + # With the node-local target dir active, uploading/extracting target + # archives through the GitHub cache is pure write amplification. + cache-targets: ${{ steps.nodecache.outputs.active != 'true' }} shared-key: rust save-if: false - name: Install Gungraun runner @@ -376,12 +543,25 @@ jobs: # directories, and the linked search paths, which is everything the run job # needs; it deliberately does not carry the source tree, so the run job # checks the sources out and passes `--workspace-remap`. + - name: Publish this run's target dir as the cache seed + # Seeds come only from pushes and same-repository pull requests, never + # from forks; a unique temporary link keeps concurrent publishers from + # racing each other before the atomic rename. + if: >- + success() && steps.nodecache.outputs.active == 'true' + && (github.event_name != 'pull_request' + || github.event.pull_request.head.repo.full_name == github.repository) + shell: bash + run: | + tmp="$NODECACHE_BASE/latest.tmp.${{ github.run_id }}-${{ github.run_attempt }}" + ln -sfn "$CARGO_TARGET_DIR" "$tmp" && mv -T "$tmp" "$NODECACHE_BASE/latest" + echo "cache seed -> $CARGO_TARGET_DIR" postgres-integration-build: name: postgres-integration (build) needs: rust-change-scope - if: needs.rust-change-scope.outputs.docs_only != 'true' + if: needs.rust-change-scope.outputs.skip_rust != 'true' runs-on: signalbox-docker - # Generous by design: this job now absorbs all five suites' compilation, + # Generous by design: this job now absorbs all six suites' compilation, # which a cold cache serializes into one runner instead of spreading it # across three. timeout-minutes: 35 @@ -402,14 +582,50 @@ jobs: printf 'matrix=%s\n' \ "$(python3 scripts/postgres_integration_suites.py --matrix)" \ >> "$GITHUB_OUTPUT" + - name: Use the node-local cargo target cache + # Self-hosted runners expose a persistent per-node cache directory + # through /home/runner/_work/_ci-cache, provisioned by the runner pod + # template outside this repository. Each run gets its own target dir, + # cloned copy-on-write (reflink, same XFS filesystem) from the last + # successful run with the same lockfile/toolchain key, so concurrent + # runs never share a cargo lock and nothing one run writes leaks into + # another. GitHub-hosted runners have no such directory and keep the + # default. Run dirs untouched for a day are pruned at job start. + id: nodecache + shell: bash + run: | + cache=/home/runner/_work/_ci-cache + if [ -d "$cache/target" ]; then + key="${{ hashFiles('Cargo.lock', 'rust-toolchain.toml') }}" + job="${{ github.job }}${{ strategy.job-index != '' && format('-{0}', strategy.job-index) || '' }}" + base="$cache/target/$key/$job" + mkdir -p "$base/runs" + find "$base/runs" -mindepth 1 -maxdepth 1 -type d -mtime +0 -exec rm -rf {} + 2>/dev/null || true + target="$base/runs/${{ github.run_id }}-${{ github.run_attempt }}" + if [ -d "$base/latest" ] && cp --reflink=always -a "$base/latest/." "$target" 2>/dev/null; then + echo "seeded $target from the last successful run (reflink)" + else + rm -rf "$target"; mkdir -p "$target" + echo "no seed available; fresh target dir $target" + fi + echo "CARGO_TARGET_DIR=$target" >> "$GITHUB_ENV" + echo "NODECACHE_BASE=$base" >> "$GITHUB_ENV" + echo "active=true" >> "$GITHUB_OUTPUT" + else + echo "active=false" >> "$GITHUB_OUTPUT" + echo "no node-local cache directory; using the default target dir" + fi - name: Cache Rust dependencies uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 with: cache-on-failure: true + # With the node-local target dir active, uploading/extracting target + # archives through the GitHub cache is pure write amplification. + cache-targets: ${{ steps.nodecache.outputs.active != 'true' }} shared-key: rust save-if: false - name: Install cargo-nextest - uses: taiki-e/install-action@a2a5f6e99e1a31540baa0468acfa302cff0f359f # v2.86.4 + uses: taiki-e/install-action@b6ff580856c41316412a0b9b60540fbc6f8c82cc # v2.86.7 with: tool: cargo-nextest@0.9.143 # One archive per suite, each built with exactly the package selection @@ -476,24 +692,43 @@ jobs: path: ${{ runner.temp }}/terminal-client.tar.zst if-no-files-found: error retention-days: 1 + - name: Upload the approval-judge-eval archive + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: postgres-integration-archive-approval-judge-eval + path: ${{ runner.temp }}/approval-judge-eval.tar.zst + if-no-files-found: error + retention-days: 1 # The dedicated Docker fleet gives each shard an isolated DinD daemon. The # build and run jobs share the same runner image and absolute work-path shape, # which keeps paths embedded in nextest archives valid across their pods. # Each test starts its own pinned PostgreSQL container through testcontainers, - # so four test processes can run without sharing database state. + # so concurrent test processes never share database state. # # The matrix is generated from `.github/postgres-integration-suites.toml`: # one runner per shard, so a suite declaring one shard costs one runner and # still travels the same archive-and-partition path a sharded suite does. - # Persistence, runner, and signalboxd are sharded according to their measured - # test-execution cost. Program-runtime and terminal-client remain whole - # because each finishes in about a minute. + # Persistence is sharded according to its measured test-execution cost; every + # other suite runs whole because it fits well inside the job timeout. # - # Runner cost, stated plainly: this asks for more runners than the matrix it - # replaces. The run matrix currently expands to twelve shards after the one - # shared build. The shards compile nothing and run for minutes; the build's - # single cache key replaces the old per-suite keys that evicted one another. + # The run matrix currently expands to eight shards after the one shared + # build. + # The shards compile nothing and run for minutes; the build's single cache + # key replaces the old per-suite keys that evicted one another. + - name: Publish this run's target dir as the cache seed + # Seeds come only from pushes and same-repository pull requests, never + # from forks; a unique temporary link keeps concurrent publishers from + # racing each other before the atomic rename. + if: >- + success() && steps.nodecache.outputs.active == 'true' + && (github.event_name != 'pull_request' + || github.event.pull_request.head.repo.full_name == github.repository) + shell: bash + run: | + tmp="$NODECACHE_BASE/latest.tmp.${{ github.run_id }}-${{ github.run_attempt }}" + ln -sfn "$CARGO_TARGET_DIR" "$tmp" && mv -T "$tmp" "$NODECACHE_BASE/latest" + echo "cache seed -> $CARGO_TARGET_DIR" postgres-integration-run: name: >- postgres-integration @@ -504,18 +739,15 @@ jobs: # workspace: their permission and atomicity contracts require local /tmp. env: TMPDIR: /tmp - # Measured against the shards rather than guessed. The former three - # persistence partitions were balanced but had grown to roughly 8m15s each; - # runner took about eight minutes whole, and signalboxd about five. The - # manifest splits those suites more finely without changing test selection. + # Measured against the shards rather than guessed. With database state in + # memory and sixteen tests per shard, the recombined shards run about two + # to nine minutes clean. # - # Ten minutes did not leave room for the slow path. A shard whose tests - # actually fail runs about two minutes longer than a clean one (9.68m - # observed against a 7.4m median), so the cap sat within noise of exactly - # the runs whose results matter most — and a job the cap kills is reported - # as `cancelled`, which reads as an infrastructure problem and hides the - # test failure underneath it. Fifteen minutes is about twice the median, - # leaves the p90 at half the cap, and still bounds a genuinely hung job. + # A shard whose tests actually fail runs about two minutes longer than a + # clean one, and a job the cap kills is reported as `cancelled`, which + # reads as an infrastructure problem and hides the test failure underneath + # it. Fifteen minutes leaves the longest clean shard two-thirds of the cap + # and still bounds a genuinely hung job. timeout-minutes: 15 strategy: fail-fast: false @@ -530,7 +762,7 @@ jobs: fetch-depth: 1 persist-credentials: false - name: Install cargo-nextest - uses: taiki-e/install-action@a2a5f6e99e1a31540baa0468acfa302cff0f359f # v2.86.4 + uses: taiki-e/install-action@b6ff580856c41316412a0b9b60540fbc6f8c82cc # v2.86.7 with: tool: cargo-nextest@0.9.143 - name: Download the ${{ matrix.suite }} archive @@ -542,7 +774,10 @@ jobs: # `--run-ignored only` is `-- --ignored`, `--no-fail-fast` is unchanged, # and the archive fixes the package and feature selection at build time. # nextest runs each test in its own process rather than on a shared - # thread pool; eight concurrent tests is the cap. One per-test retry + # thread pool; sixteen concurrent tests is the cap. Each test can consume + # a 512 MiB PostgreSQL tmpfs, so the cap permits 8 GiB of tmpfs per job + # before PostgreSQL process memory; re-check the externally provisioned + # signalbox-docker pod memory limit before raising it. One per-test retry # absorbs a transient container-start failure without rerunning a whole # shard, while a repeat failure still fails the gate. - name: Run the ${{ matrix.suite }} partition @@ -551,6 +786,10 @@ jobs: PARTITION: ${{ matrix.partition }} PARTITIONS: ${{ matrix.partitions }} FILTER: ${{ matrix.filter }} + # nextest runs each ignored integration test in its own libtest + # process. Keep deeply composed scheduler fixtures from exhausting + # the platform's smaller default test-thread stack. + RUST_MIN_STACK: "8388608" run: >- cargo nextest run --archive-file "$RUNNER_TEMP/archive/$SUITE.tar.zst" @@ -559,7 +798,7 @@ jobs: --run-ignored only --no-fail-fast --retries 1 - --test-threads 8 + --test-threads 16 -E "$FILTER" # Preserve the existing required-check name while the suites build once and @@ -576,9 +815,9 @@ jobs: env: BUILD_RESULT: ${{ needs.postgres-integration-build.result }} RUN_RESULT: ${{ needs.postgres-integration-run.result }} - DOCS_ONLY: ${{ needs.rust-change-scope.outputs.docs_only }} + SKIP_RUST: ${{ needs.rust-change-scope.outputs.skip_rust }} run: | - if [ "$DOCS_ONLY" = true ]; then + if [ "$SKIP_RUST" = true ]; then test "$BUILD_RESULT" = skipped test "$RUN_RESULT" = skipped else diff --git a/.github/workflows/swift.yml b/.github/workflows/swift.yml index e7866c7187..07eacda455 100644 --- a/.github/workflows/swift.yml +++ b/.github/workflows/swift.yml @@ -94,11 +94,11 @@ jobs: run: python3 clients/native/scripts/test_summarize_coverage.py - name: Build (iOS Simulator, Debug) run: clients/native/scripts/build-xcode.sh - # Runs the canonical test script with the UI-test bundle excluded. The - # UI and screenshot-capture suites need an interactive simulator - # session (and the Tart VM shards for the full device matrix), so they - # stay a manual/local flow via clients/native/scripts for now; the - # four unit-test bundles still run against a booted simulator the script + # Runs the canonical test script with two suites excluded. The UI-test + # bundle drives an interactive simulator session, so it stays a + # manual/local flow via clients/native/scripts for now; the in-process + # snapshot suite is skipped for the reason on its entry below. The four + # unit-test bundles still run against a booted simulator the script # resolves itself. # # Coverage rides along on this run rather than taking a macOS job of its @@ -340,13 +340,6 @@ jobs: fail_ci_if_error: false - name: Check privacy boundary run: clients/native/scripts/check-privacy.sh - - name: Check Tart scripts (dry run) - run: clients/native/scripts/tart/check-tart-scripts.sh - # Pure sha256 hashing against Screenshots/MANIFEST.sha256 — no - # simulator involved, so it is cheap enough to run here even though - # recapturing goldens is a manual flow. - - name: Check screenshot goldens - run: clients/native/scripts/check-screenshot-goldens.sh # Holds the only write token in this workflow and never builds or runs # anything, mirroring the token separation coverage.yml's publish-comment diff --git a/AGENTS.md b/AGENTS.md index 101a0773d9..6eccd86ca3 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -252,8 +252,6 @@ formatting. **Operational traps.** Do not adopt the review-slog toolkit as a merge gate until its [blocking condition](docs/open-questions.md#review-slog-toolkit-adoption) is -cleared. Do not rely on automatic context compaction until its -[blocking condition](docs/open-questions.md#automatic-context-compaction) is cleared. Run `git worktree list` before working with another checkout of this clone. diff --git a/Cargo.lock b/Cargo.lock index 1ce00e9964..d0d07954b6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8,6 +8,17 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" +[[package]] +name = "aes" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8eb277bec05f56a0e0591f155a484cbd0f4f07ff2905051a48c72f004f7ed58" +dependencies = [ + "cipher", + "cpubits", + "cpufeatures 0.3.0", +] + [[package]] name = "ahash" version = "0.7.8" @@ -348,6 +359,15 @@ dependencies = [ "hybrid-array", ] +[[package]] +name = "block-padding" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "710f1dd022ef4e93f8a438b4ba958de7f64308434fa6a87104481645cc30068b" +dependencies = [ + "hybrid-array", +] + [[package]] name = "bollard" version = "0.20.2" @@ -504,12 +524,24 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "bytemuck" +version = "1.25.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95832e849adfb21180ccb6826a99da14e5d266ae5c2e668e1602cf234f153797" + [[package]] name = "byteorder" version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" +[[package]] +name = "byteorder-lite" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f1fe948ff07f4bd06c30984e69f5b4899c516a3ef74f34df92a2df2ab535495" + [[package]] name = "bytes" version = "1.12.1" @@ -546,6 +578,15 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "cbc" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce2dc9ee5f88d11e0beb842c88b33c8a5cf0d1329c4b19494af42b07dbfe8896" +dependencies = [ + "cipher", +] + [[package]] name = "cc" version = "1.2.67" @@ -564,7 +605,7 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" dependencies = [ - "nom", + "nom 7.1.3", ] [[package]] @@ -602,6 +643,16 @@ dependencies = [ "windows-link", ] +[[package]] +name = "cipher" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8cf2a2c93cd704877c0858356ed03480ff301ee950b43f1cbe4573b088bfa6c" +dependencies = [ + "crypto-common 0.2.2", + "inout", +] + [[package]] name = "clang-sys" version = "1.9.1" @@ -659,6 +710,12 @@ version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" +[[package]] +name = "color_quant" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d7b894f5411737b7867f4827955924d7c254fc9f4d91a6aad6b097804b1018b" + [[package]] name = "combine" version = "4.6.7" @@ -721,6 +778,12 @@ dependencies = [ "libm", ] +[[package]] +name = "cpubits" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15b85f9c39137c3a891689859392b1bd49812121d0d61c9caf00d46ed5ce06ae" + [[package]] name = "cpufeatures" version = "0.2.17" @@ -797,6 +860,27 @@ dependencies = [ "hybrid-array", ] +[[package]] +name = "csv" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52cd9d68cf7efc6ddfaaee42e7288d3a99d613d4b50f76ce9827ae0c6e14f938" +dependencies = [ + "csv-core", + "itoa", + "ryu", + "serde_core", +] + +[[package]] +name = "csv-core" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "704a3c26996a80471189265814dbc2c257598b96b8a7feae2d31ace646bb9782" +dependencies = [ + "memchr", +] + [[package]] name = "ctutils" version = "0.4.2" @@ -1156,6 +1240,15 @@ version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" +[[package]] +name = "ecb" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26f2a8b3e564eba0877223dc343703ad0385794e882e6d13f3a4dd5c6b1f41ac" +dependencies = [ + "cipher", +] + [[package]] name = "either" version = "1.16.0" @@ -1175,6 +1268,15 @@ dependencies = [ "serde", ] +[[package]] +name = "encoding_rs" +version = "0.8.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" +dependencies = [ + "cfg-if", +] + [[package]] name = "equivalent" version = "1.0.2" @@ -1222,6 +1324,12 @@ dependencies = [ "once_cell", ] +[[package]] +name = "extended" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af9673d8203fcb076b19dfd17e38b3d4ae9f44959416ea532ce72415a6020365" + [[package]] name = "faster-hex" version = "0.10.0" @@ -1238,6 +1346,15 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" +[[package]] +name = "fdeflate" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e6853b52649d4ac5c0bd02320cddc5ba956bdb407c4b75a2c6b75bf51500f8c" +dependencies = [ + "simd-adler32", +] + [[package]] name = "ferroid" version = "2.0.0" @@ -1263,6 +1380,7 @@ checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" dependencies = [ "crc32fast", "miniz_oxide", + "zlib-rs", ] [[package]] @@ -1395,6 +1513,12 @@ version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" +[[package]] +name = "futures-timer" +version = "3.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af43fadb8a98512d547e37b4e92e0ced13e205c061b87b4623eff01d918d6968" + [[package]] name = "futures-util" version = "0.3.34" @@ -1457,6 +1581,16 @@ dependencies = [ "rand_core 0.10.1", ] +[[package]] +name = "gif" +version = "0.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee8cfcc411d9adbbaba82fb72661cc1bcca13e8bba98b364e62b2dba8f960159" +dependencies = [ + "color_quant", + "weezl 0.1.12", +] + [[package]] name = "git2" version = "0.21.0" @@ -2050,6 +2184,34 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cd62e6b5e86ea8eeeb8db1de02880a6abc01a397b2ebb64b5d74ac255318f5cb" +[[package]] +name = "image" +version = "0.25.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85ab80394333c02fe689eaf900ab500fbd0c2213da414687ebf995a65d5a6104" +dependencies = [ + "bytemuck", + "byteorder-lite", + "color_quant", + "gif", + "image-webp", + "moxcms", + "num-traits", + "png", + "zune-core", + "zune-jpeg", +] + +[[package]] +name = "image-webp" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "525e9ff3e1a4be2fbea1fdf0e98686a6d98b4d8f937e1bf7402245af1909e8c3" +dependencies = [ + "byteorder-lite", + "quick-error 2.0.1", +] + [[package]] name = "indexmap" version = "1.9.3" @@ -2073,6 +2235,16 @@ dependencies = [ "serde_core", ] +[[package]] +name = "inout" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4250ce6452e92010fdf7268ccc5d14faa80bb12fc741938534c58f16804e03c7" +dependencies = [ + "block-padding", + "hybrid-array", +] + [[package]] name = "inventory" version = "0.3.24" @@ -2088,6 +2260,16 @@ version = "2.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" +[[package]] +name = "iri-string" +version = "0.7.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1663ee7d8cf2900cc1414b1e1eec9f348d6eaa3bcab07579f4726a4b8499f447" +dependencies = [ + "memchr", + "serde", +] + [[package]] name = "itertools" version = "0.13.0" @@ -2127,6 +2309,7 @@ dependencies = [ "defmt", "jiff-core", "jiff-static", + "jiff-tzdb", "log", "portable-atomic", "portable-atomic-util", @@ -2154,6 +2337,12 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "jiff-tzdb" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "142bd39932ad231f10513df9ab62661fead8719872150b7ad02a2df79f4e141e" + [[package]] name = "jni" version = "0.22.4" @@ -2232,9 +2421,9 @@ checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" [[package]] name = "libc" -version = "0.2.186" +version = "0.2.189" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" [[package]] name = "libgit2-sys" @@ -2319,6 +2508,32 @@ version = "0.4.33" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" +[[package]] +name = "lopdf" +version = "0.44.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e2ec995d822e05cabc3f06d196ee43650af3fe4fe38012cacb35e0c3d113b68" +dependencies = [ + "aes", + "bitflags 2.13.1", + "cbc", + "ecb", + "encoding_rs", + "flate2", + "getrandom 0.4.3", + "indexmap 2.14.0", + "itoa", + "log", + "md-5", + "nom 8.0.0", + "rand 0.10.2", + "rangemap", + "sha2 0.11.0", + "stringprep", + "thiserror", + "weezl 0.2.1", +] + [[package]] name = "matchers" version = "0.2.0" @@ -2393,6 +2608,16 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "moxcms" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb85c154ba489f01b25c0d36ae69a87e4a1c73a72631fc6c0eb6dde34a73e44b" +dependencies = [ + "num-traits", + "pxfm", +] + [[package]] name = "nix" version = "0.31.3" @@ -2415,6 +2640,15 @@ dependencies = [ "minimal-lexical", ] +[[package]] +name = "nom" +version = "8.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df9761775871bdef83bee530e60050f7e54b1105350d6884eb0fb4f46c2f9405" +dependencies = [ + "memchr", +] + [[package]] name = "num" version = "0.4.3" @@ -2503,6 +2737,15 @@ dependencies = [ "memchr", ] +[[package]] +name = "ogg" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdab8dcd8d4052eaacaf8fb07a3ccd9a6e26efadb42878a413c68fc4af1dee2b" +dependencies = [ + "byteorder", +] + [[package]] name = "once_cell" version = "1.21.4" @@ -2590,6 +2833,12 @@ dependencies = [ "tokio", ] +[[package]] +name = "opus-rs" +version = "0.1.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f18e50e3ad1d816434f675150eca1b4c669e49cac7b685f733a88127cd75551" + [[package]] name = "outref" version = "0.5.2" @@ -2700,6 +2949,19 @@ version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" +[[package]] +name = "png" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60769b8b31b2a9f263dae2776c37b1b28ae246943cf719eb6946a1db05128a61" +dependencies = [ + "bitflags 2.13.1", + "crc32fast", + "fdeflate", + "flate2", + "miniz_oxide", +] + [[package]] name = "portable-atomic" version = "1.14.0" @@ -2895,12 +3157,33 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "pxfm" +version = "0.1.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d55d956fa96f5ec02be2e13af0e20391a5aa83d6a074e3ad368959d0fab299ea" + [[package]] name = "quick-error" version = "1.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" +[[package]] +name = "quick-error" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a993555f31e5a609f617c12db6250dedcac1b0a85076912c436e6fc9b2c8e6a3" + +[[package]] +name = "quick-xml" +version = "0.41.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e660451e55124f798a69a5af3f49ccfbefbd41910eefd25caf2393e1f3473ec1" +dependencies = [ + "memchr", +] + [[package]] name = "quote" version = "1.0.47" @@ -3013,6 +3296,12 @@ dependencies = [ "rand_core 0.9.5", ] +[[package]] +name = "rangemap" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a611d15b50743feb4c76b7d03edcb0e64f399c26961e4efe6975bc398be6aa3d" + [[package]] name = "rapidhash" version = "4.5.1" @@ -3347,11 +3636,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cc6bf79ff24e648f6da1f8d1f011e9cac26491b619e6b9280f2b47f1774e6ee2" dependencies = [ "fnv", - "quick-error", + "quick-error 1.2.3", "tempfile", "wait-timeout", ] +[[package]] +name = "rusty_mp3" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fd89ed8a368cceb87ffcc9f3f128fff535ff1cd3ad27e4e64406c29d307a956" + [[package]] name = "ryu" version = "1.0.23" @@ -3609,6 +3904,19 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "serde_yaml_ng" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b4db627b98b36d4203a7b458cf3573730f2bb591b28871d916dfa9efabfd41f" +dependencies = [ + "indexmap 2.14.0", + "itoa", + "ryu", + "serde", + "unsafe-libyaml", +] + [[package]] name = "sha1" version = "0.10.7" @@ -3699,9 +4007,13 @@ name = "signalbox-application" version = "0.0.0" dependencies = [ "expect-test", + "rustix 1.1.4", + "serde", "serde_json", + "serde_yaml_ng", "sha2 0.11.0", "signalbox-domain", + "tempfile", "tokio", "tracing", "tracing-subscriber", @@ -3719,7 +4031,10 @@ dependencies = [ "signalbox-domain", "signalbox-model-provider-runtime", "signalbox-model-runtime", + "signalbox-persistence", "signalboxd", + "sqlx", + "testcontainers-modules", "tokio", "uuid", ] @@ -3834,6 +4149,125 @@ dependencies = [ "expect-test", ] +[[package]] +name = "signalbox-file-media-adapter-office" +version = "0.0.0" +dependencies = [ + "crc32fast", + "flate2", + "quick-xml", + "serde_json", + "signalbox-file-media-processor-runtime", + "signalbox-file-media-runtime", + "tokio", + "zip", +] + +[[package]] +name = "signalbox-file-media-adapter-pdf" +version = "0.0.0" +dependencies = [ + "lopdf", + "serde_json", + "signalbox-file-media-processor-runtime", + "signalbox-file-media-runtime", + "tokio", +] + +[[package]] +name = "signalbox-file-media-adapter-svg" +version = "0.0.0" +dependencies = [ + "iri-string", + "quick-xml", + "serde_json", + "signalbox-file-media-processor-runtime", + "signalbox-file-media-runtime", + "tokio", +] + +[[package]] +name = "signalbox-file-media-adapters-audio" +version = "0.0.0" +dependencies = [ + "ogg", + "opus-rs", + "rusty_mp3", + "serde_json", + "signalbox-file-media-processor-runtime", + "signalbox-file-media-runtime", + "symphonia", + "tokio", +] + +[[package]] +name = "signalbox-file-media-adapters-image" +version = "0.0.0" +dependencies = [ + "crc32fast", + "image", + "serde_json", + "signalbox-file-media-processor-runtime", + "signalbox-file-media-runtime", + "tokio", +] + +[[package]] +name = "signalbox-file-media-adapters-text" +version = "0.0.0" +dependencies = [ + "csv", + "serde", + "serde_json", + "serde_stacker", + "signalbox-file-media-processor-runtime", + "signalbox-file-media-runtime", + "tokio", +] + +[[package]] +name = "signalbox-file-media-linux-sandbox" +version = "0.0.0" +dependencies = [ + "libc", +] + +[[package]] +name = "signalbox-file-media-processor-runtime" +version = "0.0.0" +dependencies = [ + "base64 0.23.1", + "libc", + "rustix 1.1.4", + "serde", + "serde_json", + "sha2 0.11.0", + "signalbox-file-media-linux-sandbox", + "signalbox-file-media-runtime", + "tempfile", + "tokio", +] + +[[package]] +name = "signalbox-file-media-provider-runtime" +version = "0.0.0" +dependencies = [ + "signalbox-domain", + "signalbox-file-media-runtime", + "signalbox-tools-file-media", +] + +[[package]] +name = "signalbox-file-media-runtime" +version = "0.0.0" +dependencies = [ + "futures-timer", + "futures-util", + "serde", + "serde_json", + "tokio", +] + [[package]] name = "signalbox-model-provider-runtime" version = "0.0.0" @@ -3849,6 +4283,17 @@ dependencies = [ "uuid", ] +[[package]] +name = "signalbox-model-reference-catalog" +version = "0.0.0" +dependencies = [ + "rust_decimal", + "serde", + "serde_json", + "tempfile", + "url", +] + [[package]] name = "signalbox-model-runtime" version = "0.0.0" @@ -3904,6 +4349,7 @@ dependencies = [ "signalbox-test-bin", "tempfile", "tokio", + "tracing", ] [[package]] @@ -3937,6 +4383,7 @@ dependencies = [ "signalbox-expect-table", "signalbox-tools-plan", "sqlx", + "tempfile", "testcontainers-modules", "tokio", "toml", @@ -4095,6 +4542,20 @@ dependencies = [ "toml", ] +[[package]] +name = "signalbox-tools-file-media" +version = "0.0.0" +dependencies = [ + "schemars 1.2.2", + "serde", + "serde_json", + "serde_stacker", + "signalbox-application", + "signalbox-domain", + "signalbox-file-media-runtime", + "signalbox-tool-contract", +] + [[package]] name = "signalbox-tools-git" version = "0.0.0" @@ -4653,6 +5114,104 @@ version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" +[[package]] +name = "symphonia" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7edef6a96b696d4e0cab5ee9ebb7ca155ed95f30a6b45bbb8b97d2727f02424" +dependencies = [ + "lazy_static", + "symphonia-bundle-flac", + "symphonia-bundle-mp3", + "symphonia-codec-pcm", + "symphonia-core", + "symphonia-format-riff", + "symphonia-metadata", +] + +[[package]] +name = "symphonia-bundle-flac" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6405d5c34ff6f8f7ca08a4101efe66d21e180f7322ef9359900a0e55698a7892" +dependencies = [ + "log", + "symphonia-common", + "symphonia-core", + "symphonia-metadata", +] + +[[package]] +name = "symphonia-bundle-mp3" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "98ea5ffc8716bff677dfb3b01b420c7b758de901a72b8c330bf2040ab74b4add" +dependencies = [ + "lazy_static", + "log", + "symphonia-core", +] + +[[package]] +name = "symphonia-codec-pcm" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e04ba75686acbe43542fdd374571195f0530c0b7785ca25cc6840e9c6c4b6eea" +dependencies = [ + "log", + "symphonia-core", +] + +[[package]] +name = "symphonia-common" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2acc3fcc18ec9b8cdd48614e259c4cf0d27b71d41e5d9b120b42c5adab12d7c4" +dependencies = [ + "log", + "symphonia-core", + "symphonia-metadata", +] + +[[package]] +name = "symphonia-core" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01c412864d599d4750d0c3d684d7e093ec05e5309681ef5252cc1096a437f6e0" +dependencies = [ + "bitflags 2.13.1", + "bytemuck", + "lazy_static", + "log", + "num-complex", + "smallvec", +] + +[[package]] +name = "symphonia-format-riff" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ff70929083a8c1a5f6cd7c904b6071c7914ad04739b510c2f7239dfc9b7dabe" +dependencies = [ + "extended", + "log", + "symphonia-core", + "symphonia-metadata", +] + +[[package]] +name = "symphonia-metadata" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83713a97705d77bdef7cdbc0768fd6e5a54e4cd7e48d60a806ae85639e2c87c6" +dependencies = [ + "lazy_static", + "log", + "regex-lite", + "smallvec", + "symphonia-core", +] + [[package]] name = "syn" version = "1.0.109" @@ -5266,6 +5825,12 @@ dependencies = [ "toml", ] +[[package]] +name = "typed-path" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e28f89b80c87b8fb0cf04ab448d5dd0dd0ade2f8891bae878de66a75a28600e" + [[package]] name = "typenum" version = "1.20.1" @@ -5317,6 +5882,12 @@ version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7df058c713841ad818f1dc5d3fd88063241cc61f49f5fbea4b951e8cf5a8d71d" +[[package]] +name = "unsafe-libyaml" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" + [[package]] name = "untrusted" version = "0.9.0" @@ -5592,6 +6163,18 @@ dependencies = [ "rustls-pki-types", ] +[[package]] +name = "weezl" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a28ac98ddc8b9274cb41bb4d9d4d5c425b6020c50c46f25559911905610b4a88" + +[[package]] +name = "weezl" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4ca08e5ef825b65b056d9efbd95c8750683f0a6d0466d02e96dc2e4e360f3d2" + [[package]] name = "which" version = "6.0.3" @@ -5942,6 +6525,25 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "zip" +version = "8.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d04a6b5381502aa6087c94c669499eb1602eb9c5e8198e534de571f7154809b" +dependencies = [ + "crc32fast", + "flate2", + "indexmap 2.14.0", + "memchr", + "typed-path", +] + +[[package]] +name = "zlib-rs" +version = "0.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34b31d188d9d685a4f9c7b46d6e36631b07058d2cfe190267adce54dc230bf12" + [[package]] name = "zmij" version = "1.0.23" @@ -5960,3 +6562,18 @@ dependencies = [ "resb", "serde", ] + +[[package]] +name = "zune-core" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d56377fd46368984a170bc5aac5567e52ca5da874caa60bea39fcbca78fb658b" + +[[package]] +name = "zune-jpeg" +version = "0.5.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27bc9d5b815bc103f142aa054f561d9187d191692ec7c2d1e2b4737f8dbd7296" +dependencies = [ + "zune-core", +] diff --git a/Cargo.toml b/Cargo.toml index 16f6df15b1..659ec3df2c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,4 +1,5 @@ [workspace] +exclude = ["crates/file-media-linux-sandbox"] members = [ "apps/client", "apps/signalbox-runner", @@ -13,12 +14,22 @@ members = [ "crates/conversation-import-json", "crates/domain", "crates/expect-table", + "crates/file-media-adapters-audio", + "crates/file-media-adapters-image", + "crates/file-media-adapters-text", + "crates/file-media-runtime", + "crates/file-media-adapter-office", + "crates/file-media-adapter-pdf", + "crates/file-media-adapter-svg", + "crates/file-media-provider-runtime", + "crates/file-media-processor-runtime", "crates/model-runtime", "crates/model-runtime-anthropic", "crates/model-runtime-claude-cli", "crates/model-runtime-codex-cli", "crates/model-runtime-openai", "crates/model-provider-runtime", + "crates/model-reference-catalog", "crates/persistence", "crates/process-protocol", "crates/program-runtime", @@ -29,6 +40,7 @@ members = [ "crates/tools-basic", "crates/tools-code-host", "crates/tools-exec", + "crates/tools-file-media", "crates/tools-conversations", "crates/tools-github", "crates/tools-git", diff --git a/apps/client/Cargo.toml b/apps/client/Cargo.toml index 984bb752fb..243e673bf5 100644 --- a/apps/client/Cargo.toml +++ b/apps/client/Cargo.toml @@ -39,6 +39,7 @@ tokio = { version = "1.53.0", default-features = false, features = [ "net", "rt-multi-thread", "signal", + "time", ] } uuid = { version = "1.24.0", default-features = false, features = ["std", "v7"] } diff --git a/apps/client/src/arguments.rs b/apps/client/src/arguments.rs index 7c8ae0a029..ab26240cc1 100644 --- a/apps/client/src/arguments.rs +++ b/apps/client/src/arguments.rs @@ -83,6 +83,7 @@ pub(crate) enum Command { }, Session(SessionCommand), Goal(GoalCommand), + Status, List, Templates, Search(SessionMetadataPageRequest), @@ -424,6 +425,8 @@ enum CliCommand { Session(SessionDelegationArguments), /// Commission, inspect, or transition one session goal. Goal(GoalArguments), + /// Show what repository-watch automation is working on now. + Status, /// List current sessions. List, /// List available session templates. @@ -1927,6 +1930,7 @@ pub(crate) fn parse( command_id: arguments.command_id, }, }), + CliCommand::Status => Command::Status, CliCommand::List => Command::List, CliCommand::Templates => Command::Templates, CliCommand::Search(arguments) => { diff --git a/apps/client/src/chat.rs b/apps/client/src/chat.rs index 65cf9d231d..354d524dba 100644 --- a/apps/client/src/chat.rs +++ b/apps/client/src/chat.rs @@ -1443,7 +1443,7 @@ mod tests { use super::*; use signalbox_process_protocol::{ DelegationOutcome, DelegationProvenance, DelegationReason, FastModeOverlay, - ModelSettingsOverlay, ReasoningLevel, SettingOverlay, + ModelSettingsOverlay, ReasoningLevel, SettingOverlay, UserInputContent, }; const REQUEST: &str = "00000000-0000-0000-0000-000000000123"; @@ -1703,7 +1703,7 @@ mod tests { )), turn_id, acceptance_position: CanonicalU64::new(FIRST_ACCEPTANCE_POSITION), - content: InputContent::new(String::from(QUEUED_USER_INPUT)), + content: UserInputContent::text(String::from(QUEUED_USER_INPUT)), }, followed_session(), ), @@ -1746,7 +1746,7 @@ mod tests { )), turn_id: retired_turn, acceptance_position: CanonicalU64::new(1), - content: InputContent::new(String::from("obsolete goal input")), + content: UserInputContent::text(String::from("obsolete goal input")), }, followed_session(), ), @@ -1771,7 +1771,7 @@ mod tests { )), turn_id: replacement_turn, acceptance_position: CanonicalU64::new(2), - content: InputContent::new(String::from("replacement goal input")), + content: UserInputContent::text(String::from("replacement goal input")), }, followed_session(), ), @@ -1883,7 +1883,7 @@ mod tests { accepted_input_id: CanonicalUuid::from_uuid(Uuid::from_u128( FIRST_INPUT_IDENTITY, )), - content: InputContent::new(String::from(FIRST_CONTENT)), + content: UserInputContent::text(String::from(FIRST_CONTENT)), }, }, ServerMessage::TranscriptTurn { @@ -1894,7 +1894,7 @@ mod tests { accepted_input_id: CanonicalUuid::from_uuid(Uuid::from_u128( SECOND_INPUT_IDENTITY, )), - content: InputContent::new(String::from(SECOND_CONTENT)), + content: UserInputContent::text(String::from(SECOND_CONTENT)), }, }, ], diff --git a/apps/client/src/connection.rs b/apps/client/src/connection.rs index 638773da41..34e301d128 100644 --- a/apps/client/src/connection.rs +++ b/apps/client/src/connection.rs @@ -206,6 +206,7 @@ fn oversized_frame_is_import_source(request: &ClientRequest) -> bool { | ClientRequest::CommissionSession { .. } | ClientRequest::ListTemplates {} | ClientRequest::ListSessions {} + | ClientRequest::ReadOperatorStatus {} | ClientRequest::UpdateSessionPlacement { .. } | ClientRequest::AttachGoal { .. } | ClientRequest::ReadGoal { .. } @@ -261,7 +262,8 @@ fn oversized_frame_is_import_source(request: &ClientRequest) -> bool { | ClientRequest::RecordReviewPublicationOutcomes { .. } | ClientRequest::ReadReviewOrchestration { .. } | ClientRequest::StopTurn { .. } - | ClientRequest::DecideToolRequest { .. } => false, + | ClientRequest::DecideToolRequest { .. } + | ClientRequest::OverrideDeniedToolRequest { .. } => false, } } diff --git a/apps/client/src/error.rs b/apps/client/src/error.rs index 272d5e0bfb..8180434d22 100644 --- a/apps/client/src/error.rs +++ b/apps/client/src/error.rs @@ -307,6 +307,11 @@ impl Error for ClientError { const fn failed_model_call_cause(cause: FailedModelCallCause) -> &'static str { match cause { FailedModelCallCause::CredentialRejected => "the provider rejected the credential", + FailedModelCallCause::AttachmentTooLarge => { + "the attachment verification budget was exceeded" + } + FailedModelCallCause::AttachmentMissing => "a required attachment is missing", + FailedModelCallCause::AttachmentCorrupt => "a required attachment is corrupt", FailedModelCallCause::PermissionDenied => "the credential lacks permission", FailedModelCallCause::InvalidRequest => "the provider rejected the request as invalid", FailedModelCallCause::TargetNotFound => "the requested model or resource was not found", @@ -396,6 +401,14 @@ impl fmt::Display for RejectionDisplay { RejectionDetail::SessionNotFound { session_id } => { write!(formatter, "session_not_found session={session_id}") } + RejectionDetail::AttachmentBlobNotFound { digest } => { + write!(formatter, "attachment_blob_not_found digest={digest}") + } + RejectionDetail::AttachmentByteBudgetExceeded { maximum_bytes } => write!( + formatter, + "attachment_byte_budget_exceeded maximum_bytes={}", + maximum_bytes.value() + ), RejectionDetail::SessionPlacementCurrentVersionMismatch { session_id, expected_placement_version, @@ -507,6 +520,18 @@ impl fmt::Display for RejectionDisplay { formatter, "tool_request_not_in_session session={session_id} request={tool_request_id}" ), + RejectionDetail::ToolRequestNotDelegateDenied { tool_request_id } => write!( + formatter, + "tool_request_not_delegate_denied request={tool_request_id}" + ), + RejectionDetail::ToolRequestNotTerminallyDenied { tool_request_id } => write!( + formatter, + "tool_request_not_terminally_denied request={tool_request_id}" + ), + RejectionDetail::ToolDenialAlreadyOverridden { tool_request_id } => write!( + formatter, + "tool_denial_already_overridden request={tool_request_id}" + ), RejectionDetail::DelegationRequestNotInTurn { session_id, turn_id, diff --git a/apps/client/src/lib.rs b/apps/client/src/lib.rs index 595af4bbda..af1cbebdce 100644 --- a/apps/client/src/lib.rs +++ b/apps/client/src/lib.rs @@ -7,6 +7,7 @@ use std::{ os::unix::ffi::OsStrExt as _, path::{Path, PathBuf}, process::ExitCode, + time::Duration, }; use arguments::{ @@ -17,9 +18,10 @@ use arguments::{ use connection::ProcessClient; use error::ClientError; use presentation::{ - BlobUploadPresentation, ChildResultPresentation, ConversationRow, ImportedEntryRow, Output, - SessionAwaitRegisteredPresentation, SessionMessageSentPresentation, SessionMetadataRow, - SessionSpawnedPresentation, SnapshotSelection, + BlobUploadPresentation, ChildResultPresentation, ConversationRow, ImportedEntryRow, + OperatorStatusPresentationCounts, Output, SessionAwaitRegisteredPresentation, + SessionMessageSentPresentation, SessionMetadataRow, SessionSpawnedPresentation, + SnapshotSelection, }; use rustix::{ fd::OwnedFd, @@ -36,15 +38,16 @@ use signalbox_process_protocol::{ GoalHistoryEvent, GoalLifecycleState, InputContent, InputDelivery, MAX_BLOB_CHUNK_BYTES, MAX_BLOB_READ_BYTES, MAX_CONTENT_FRAGMENT_BYTES, MAX_CONVERSATION_IMPORT_CHUNK_BYTES, MAX_FRAME_BYTES, ModelCallDisposition, ModelCallState, ModelSelection, ModelSettingsOverlay, - ProtocolVersion, RejectionDetail, RequestId, ReviewConcernTerminalOutcome, ReviewFindingEvent, - ReviewFindingInput, ReviewFindingStatus, ReviewImportTerminalOutcome, - ReviewJudgmentEffectTerminalOutcome, ReviewJudgmentPlanMember, ReviewOrchestrationConcernInput, - ReviewOrchestrationState, ReviewPassLifecycle, ReviewPassSnapshot, ReviewPassTerminalOutcome, - ReviewPublicationOutcome, ReviewPublicationTerminalOutcome, ReviewRepairOutcome, - ReviewRepairTerminalOutcome, ReviewRunSnapshot, RunnerConnectionHealth, RunnerProjection, - RunnerProjectionState, RunnerStateTransitionState, ServerFrame, ServerMessage, SessionEvent, - SessionPlacement, SystemPromptMember, SystemPromptText, ToolBatchState, ToolDecision, - TurnState, decode_server_line, encode_client_line, encode_server_line, + OperatorStatusMessage, ProtocolVersion, RejectionDetail, RequestId, + ReviewConcernTerminalOutcome, ReviewFindingEvent, ReviewFindingInput, ReviewFindingStatus, + ReviewImportTerminalOutcome, ReviewJudgmentEffectTerminalOutcome, ReviewJudgmentPlanMember, + ReviewOrchestrationConcernInput, ReviewOrchestrationState, ReviewPassLifecycle, + ReviewPassSnapshot, ReviewPassTerminalOutcome, ReviewPublicationOutcome, + ReviewPublicationTerminalOutcome, ReviewRepairOutcome, ReviewRepairTerminalOutcome, + ReviewRunSnapshot, RunnerConnectionHealth, RunnerProjection, RunnerProjectionState, + RunnerStateTransitionState, ServerFrame, ServerMessage, SessionEvent, SessionPlacement, + SystemPromptMember, SystemPromptText, ToolBatchState, ToolDecision, TurnState, + decode_server_line, encode_client_line, encode_server_line, }; use tokio::io::{AsyncReadExt as _, AsyncSeekExt as _, AsyncWriteExt as _}; use transcript::{SnapshotIdentitySet, SnapshotRecord, TranscriptSnapshot, read_snapshot}; @@ -67,7 +70,6 @@ const MAX_REVIEW_JSON_INPUT_BYTES: usize = MAX_FRAME_BYTES / 4 * 3; const MAX_SINGLE_FRAME_IMPORT_SOURCE_BYTES: usize = MAX_FRAME_BYTES / 4 * 3; /// Bounded memory used while hashing one client-local blob source. const BLOB_HASH_BUFFER_BYTES: usize = 64 * 1024; - #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub(crate) struct ClientDeploymentLimits { max_message_utf8_bytes: Option, @@ -134,6 +136,15 @@ fn optional_usize_limit(value: Option) -> Result, Cl .transpose() } +/// Maximum time a terminal follower waits before rereading recovery state. +// numeric-bound: interval - exposes reconciliation exhaustion without busy polling +#[cfg(not(test))] +const FOLLOW_RECOVERY_REFETCH_INTERVAL: Duration = Duration::from_secs(30); +/// Short equivalent used by deterministic socket tests. +// numeric-bound: interval - keeps follower refetch tests bounded +#[cfg(test)] +const FOLLOW_RECOVERY_REFETCH_INTERVAL: Duration = Duration::from_millis(50); + #[derive(Deserialize)] #[serde(deny_unknown_fields)] struct ReviewConcernsFile { @@ -422,6 +433,8 @@ fn delegation_rejection_matches( RejectionDetail::UnsupportedReasoningLevel { .. } | RejectionDetail::UnsupportedFastMode { .. } | RejectionDetail::UnsupportedServiceTier { .. } + | RejectionDetail::AttachmentBlobNotFound { .. } + | RejectionDetail::AttachmentByteBudgetExceeded { .. } | RejectionDetail::SessionPlacementCurrentVersionMismatch { .. } | RejectionDetail::SessionPlacementVersionExhausted { .. } | RejectionDetail::GoalCommandRejected { .. } @@ -434,6 +447,9 @@ fn delegation_rejection_matches( | RejectionDetail::InterruptUnavailableWhileAwaitingApproval { .. } | RejectionDetail::SafePointUnavailableWhileStopping { .. } | RejectionDetail::ToolRequestAlreadyResolved { .. } + | RejectionDetail::ToolRequestNotDelegateDenied { .. } + | RejectionDetail::ToolRequestNotTerminallyDenied { .. } + | RejectionDetail::ToolDenialAlreadyOverridden { .. } | RejectionDetail::ToolRequestNotEarliestUndecided { .. } | RejectionDetail::DefaultsVersionMismatch { .. } | RejectionDetail::UnknownModelAlias { .. } @@ -530,6 +546,7 @@ fn classify_delegation_response(message: ServerMessage) -> DelegationResponse { | ServerMessage::SessionsStart {} | ServerMessage::SessionSummary { .. } | ServerMessage::SessionsEnd { .. } + | ServerMessage::OperatorStatus(..) | ServerMessage::TemplatesStart {} | ServerMessage::TemplateSummary { .. } | ServerMessage::TemplatesEnd { .. } @@ -550,6 +567,7 @@ fn classify_delegation_response(message: ServerMessage) -> DelegationResponse { | ServerMessage::SessionDefaultsReplaced { .. } | ServerMessage::SessionDefaults { .. } | ServerMessage::ToolRequestDecided { .. } + | ServerMessage::ToolDenialOverridden { .. } | ServerMessage::SessionCompacted { .. } | ServerMessage::ConversationImportBegun { .. } | ServerMessage::ConversationImportAppended { .. } @@ -571,6 +589,7 @@ fn classify_delegation_response(message: ServerMessage) -> DelegationResponse { | ServerMessage::TranscriptModelCallUsage { .. } | ServerMessage::TranscriptModelCallsEnd { .. } | ServerMessage::TranscriptEntry { .. } + | ServerMessage::TranscriptUserEntry { .. } | ServerMessage::TranscriptTextEntry { .. } | ServerMessage::TranscriptContent { .. } | ServerMessage::TranscriptSnapshotEnd { .. } @@ -637,6 +656,7 @@ fn classify_conversation_import_response(message: ServerMessage) -> Conversation | ServerMessage::SessionsStart {} | ServerMessage::SessionSummary { .. } | ServerMessage::SessionsEnd { .. } + | ServerMessage::OperatorStatus(..) | ServerMessage::TemplatesStart {} | ServerMessage::TemplateSummary { .. } | ServerMessage::TemplatesEnd { .. } @@ -657,6 +677,7 @@ fn classify_conversation_import_response(message: ServerMessage) -> Conversation | ServerMessage::SessionDefaultsReplaced { .. } | ServerMessage::SessionDefaults { .. } | ServerMessage::ToolRequestDecided { .. } + | ServerMessage::ToolDenialOverridden { .. } | ServerMessage::SessionCompacted { .. } | ServerMessage::ConversationImportAborted {} | ServerMessage::BlobUploadBegun { .. } @@ -674,6 +695,7 @@ fn classify_conversation_import_response(message: ServerMessage) -> Conversation | ServerMessage::TranscriptModelCallUsage { .. } | ServerMessage::TranscriptModelCallsEnd { .. } | ServerMessage::TranscriptEntry { .. } + | ServerMessage::TranscriptUserEntry { .. } | ServerMessage::TranscriptTextEntry { .. } | ServerMessage::TranscriptContent { .. } | ServerMessage::TranscriptSnapshotEnd { .. } @@ -752,6 +774,7 @@ fn classify_blob_upload_response(message: ServerMessage) -> BlobUploadResponse { | ServerMessage::SessionsStart {} | ServerMessage::SessionSummary { .. } | ServerMessage::SessionsEnd { .. } + | ServerMessage::OperatorStatus(..) | ServerMessage::TemplatesStart {} | ServerMessage::TemplateSummary { .. } | ServerMessage::TemplatesEnd { .. } @@ -772,6 +795,7 @@ fn classify_blob_upload_response(message: ServerMessage) -> BlobUploadResponse { | ServerMessage::SessionDefaultsReplaced { .. } | ServerMessage::SessionDefaults { .. } | ServerMessage::ToolRequestDecided { .. } + | ServerMessage::ToolDenialOverridden { .. } | ServerMessage::SessionCompacted { .. } | ServerMessage::ConversationImportBegun { .. } | ServerMessage::ConversationImportAppended { .. } @@ -789,6 +813,7 @@ fn classify_blob_upload_response(message: ServerMessage) -> BlobUploadResponse { | ServerMessage::TranscriptModelCallUsage { .. } | ServerMessage::TranscriptModelCallsEnd { .. } | ServerMessage::TranscriptEntry { .. } + | ServerMessage::TranscriptUserEntry { .. } | ServerMessage::TranscriptTextEntry { .. } | ServerMessage::TranscriptContent { .. } | ServerMessage::TranscriptSnapshotEnd { .. } @@ -1035,6 +1060,7 @@ async fn execute( | Command::Session(_) | Command::Goal(_) | Command::Imported { .. } + | Command::Status | Command::List | Command::Templates | Command::Search(_) @@ -1060,6 +1086,7 @@ async fn execute( | Command::Session(_) | Command::Goal(_) | Command::Imported { .. } + | Command::Status | Command::List | Command::Templates | Command::Search(_) @@ -1093,6 +1120,7 @@ async fn execute( | Command::Compact { .. } | Command::Session(_) | Command::Goal(_) + | Command::Status | Command::List | Command::Templates | Command::Search(_) @@ -1217,6 +1245,7 @@ async fn execute( } => imported(&mut client, &mut output, imported_conversation_id).await, Command::Session(command) => session_delegation(&mut client, &mut output, command).await, Command::Goal(command) => goal(&mut client, &mut output, command).await, + Command::Status => status(&mut client, &mut output).await, Command::List => list(&mut client, &mut output).await, Command::Templates => list_templates(&mut client, &mut output).await, Command::Search(page) => search(&mut client, &mut output, page).await, @@ -4318,7 +4347,7 @@ async fn submit_input( .mutation_request(ClientRequest::SubmitInput { command_id, session_id, - content, + content: signalbox_process_protocol::UserInputContent::text(content.into_string()), expected_defaults_version, model_settings: ModelSettingsOverlay::inherit_all(), delivery, @@ -4362,7 +4391,7 @@ async fn reconcile_turn( command_id, session_id, expected_active_turn_id, - content, + content: signalbox_process_protocol::UserInputContent::text(content.into_string()), expected_defaults_version: defaults_version, model_settings: ModelSettingsOverlay::inherit_all(), }) @@ -4404,7 +4433,7 @@ async fn stop_turn( command_id, session_id, expected_active_turn_id, - content, + content: signalbox_process_protocol::UserInputContent::text(content.into_string()), expected_defaults_version: defaults_version, descendant_scope, model_settings: ModelSettingsOverlay::inherit_all(), @@ -4449,9 +4478,31 @@ async fn await_turn_terminal( return Ok(terminal); } queued_turn_recovery(&mut snapshot, turn_id)?; + let mut poll_automatic_recovery = + automatic_model_call_recovery_pending(&mut snapshot, turn_id)?; let mut observed_cursor = snapshot.cursor(); loop { - match connection.message().await? { + let message = if poll_automatic_recovery { + match tokio::time::timeout(FOLLOW_RECOVERY_REFETCH_INTERVAL, connection.message()) + .await + { + Ok(message) => message?, + Err(_) => { + let mut refreshed = transcript(client, session_id).await?; + let refreshed_state = refreshed.turn_state(turn_id)?; + if let Some(terminal) = terminal_snapshot_state(refreshed_state.as_ref())? { + return Ok(terminal); + } + queued_turn_recovery(&mut refreshed, turn_id)?; + poll_automatic_recovery = + automatic_model_call_recovery_pending(&mut refreshed, turn_id)?; + continue; + } + } + } else { + connection.message().await? + }; + match message { ServerMessage::SessionEvent { cursor, session_id: event_session, @@ -4471,6 +4522,11 @@ async fn await_turn_terminal( return Ok(terminal); } queued_turn_recovery(&mut refreshed, turn_id)?; + poll_automatic_recovery = + automatic_model_call_recovery_pending(&mut refreshed, turn_id)?; + if poll_automatic_recovery { + continue; + } if !runner_recovery_transition(&event) { return Err(ClientError::Protocol( "a recovery event did not produce recovery or terminal state", @@ -4483,6 +4539,8 @@ async fn await_turn_terminal( if let Some(terminal) = terminal_snapshot_state(refreshed_state.as_ref())? { return Ok(terminal); } + poll_automatic_recovery = + automatic_model_call_recovery_pending(&mut refreshed, turn_id)?; } if session_recovery_transition(&event) { let mut refreshed = transcript(client, session_id).await?; @@ -4491,6 +4549,8 @@ async fn await_turn_terminal( return Ok(terminal); } queued_turn_recovery(&mut refreshed, turn_id)?; + poll_automatic_recovery = + automatic_model_call_recovery_pending(&mut refreshed, turn_id)?; } } ServerMessage::ProviderTextDelta { @@ -4516,6 +4576,39 @@ async fn await_turn_terminal( } } +fn automatic_model_call_recovery_pending( + snapshot: &mut TranscriptSnapshot, + selected_turn: CanonicalUuid, +) -> Result { + let selected_state = snapshot + .turn_state(selected_turn)? + .ok_or(ClientError::Protocol( + "follow snapshot omitted the submitted turn", + ))?; + if matches!( + selected_state, + TurnState::ActiveAwaitingModelCallRecovery { + operator_action_required: false, + .. + } + ) { + return Ok(true); + } + if !matches!(selected_state, TurnState::Queued { .. }) { + return Ok(false); + } + let Some(active_turn) = snapshot.active_turn()? else { + return Ok(false); + }; + Ok(matches!( + snapshot.turn_state(active_turn)?, + Some(TurnState::ActiveAwaitingModelCallRecovery { + operator_action_required: false, + .. + }) + )) +} + fn queued_turn_recovery( snapshot: &mut TranscriptSnapshot, selected_turn: CanonicalUuid, @@ -4542,8 +4635,15 @@ fn queued_turn_recovery( fn blocker_recovery_snapshot_state(state: &TurnState) -> Result<(), ClientError> { match state { - TurnState::ActiveAwaitingModelCallRecovery { .. } + TurnState::ActiveAwaitingModelCallRecovery { + operator_action_required: true, + .. + } | TurnState::ActiveAwaitingToolRecovery { .. } => Err(ClientError::TurnRecoveryRequired), + TurnState::ActiveAwaitingModelCallRecovery { + operator_action_required: false, + .. + } => Ok(()), TurnState::ActiveAwaitingRunnerRecovery { .. } => Err(ClientError::RunnerRecoveryRequired), TurnState::Queued { .. } | TurnState::QueuedDelegated { .. } @@ -4672,10 +4772,17 @@ fn terminal_snapshot_state(state: Option<&TurnState>) -> Result Ok(None), - Some( - TurnState::ActiveAwaitingModelCallRecovery { .. } - | TurnState::ActiveAwaitingToolRecovery { .. }, - ) => Err(ClientError::TurnRecoveryRequired), + Some(TurnState::ActiveAwaitingModelCallRecovery { + operator_action_required: true, + .. + }) + | Some(TurnState::ActiveAwaitingToolRecovery { .. }) => { + Err(ClientError::TurnRecoveryRequired) + } + Some(TurnState::ActiveAwaitingModelCallRecovery { + operator_action_required: false, + .. + }) => Ok(None), Some(TurnState::ActiveAwaitingRunnerRecovery { .. }) => { Err(ClientError::RunnerRecoveryRequired) } @@ -4935,6 +5042,128 @@ fn write_assistant_texts( Ok(()) } +#[derive(Clone, Copy, Debug, Eq, PartialEq, PartialOrd, Ord)] +enum OperatorStatusPhase { + HeldSlots, + QueuedObligations, + PullRequestConvergences, + PendingStaleReviewClearances, +} + +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] +struct OperatorStatusCounts { + held_slots: u64, + queued_obligations: u64, + pull_request_convergences: u64, + pending_stale_review_clearances: u64, +} + +async fn status(client: &mut ProcessClient, output: &mut Output<'_>) -> Result<(), ClientError> { + let mut connection = client.request(ClientRequest::ReadOperatorStatus {}).await?; + match connection.message().await? { + ServerMessage::OperatorStatus(message) + if matches!(message.as_ref(), OperatorStatusMessage::Start {}) => {} + ServerMessage::Error { + code, + message, + detail, + } => return Err(ClientError::remote(code, message, detail)), + _ => { + return Err(ClientError::Protocol( + "operator status did not begin with its start frame", + )); + } + } + let mut spool = tempfile::tempfile()?; + let mut phase = OperatorStatusPhase::HeldSlots; + let mut counts = OperatorStatusCounts::default(); + loop { + let frame = connection.frame().await?; + let item_phase = match frame.message() { + ServerMessage::OperatorStatus(message) => match message.as_ref() { + OperatorStatusMessage::HeldSlot(_) => { + counts.held_slots = status_increment(counts.held_slots)?; + Some(OperatorStatusPhase::HeldSlots) + } + OperatorStatusMessage::QueuedObligation(_) => { + counts.queued_obligations = status_increment(counts.queued_obligations)?; + Some(OperatorStatusPhase::QueuedObligations) + } + OperatorStatusMessage::PullRequestConvergence(_) => { + counts.pull_request_convergences = + status_increment(counts.pull_request_convergences)?; + Some(OperatorStatusPhase::PullRequestConvergences) + } + OperatorStatusMessage::PendingStaleReviewClearance(_) => { + counts.pending_stale_review_clearances = + status_increment(counts.pending_stale_review_clearances)?; + Some(OperatorStatusPhase::PendingStaleReviewClearances) + } + OperatorStatusMessage::End(item) + if counts + == (OperatorStatusCounts { + held_slots: item.held_slot_count.value(), + queued_obligations: item.queued_obligation_count.value(), + pull_request_convergences: item.pull_request_convergence_count.value(), + pending_stale_review_clearances: item + .pending_stale_review_clearance_count + .value(), + }) => + { + break; + } + OperatorStatusMessage::Start {} | OperatorStatusMessage::End(_) => { + return Err(ClientError::Protocol( + "operator status sequence or count was invalid", + )); + } + }, + ServerMessage::Error { + code, + message, + detail, + } => return Err(ClientError::remote(*code, message.clone(), *detail)), + _ => { + return Err(ClientError::Protocol( + "operator status sequence or count was invalid", + )); + } + }; + let Some(item_phase) = item_phase else { + return Err(ClientError::Protocol( + "operator status sequence was invalid", + )); + }; + if item_phase < phase { + return Err(ClientError::Protocol( + "operator status sections were out of order", + )); + } + phase = item_phase; + spool.write_all(&encode_server_line(&frame)?)?; + } + output.operator_status_counts(OperatorStatusPresentationCounts { + held_slots: counts.held_slots, + queued_obligations: counts.queued_obligations, + pull_request_convergences: counts.pull_request_convergences, + pending_stale_review_clearances: counts.pending_stale_review_clearances, + })?; + spool.seek(SeekFrom::Start(0))?; + let mut reader = BufReader::new(spool); + let mut line = Vec::new(); + while reader.read_until(b'\n', &mut line)? != 0 { + output.operator_status_item(decode_server_line(&line)?.message())?; + line.clear(); + } + Ok(output.operator_status_model_usage_omitted()?) +} + +fn status_increment(value: u64) -> Result { + value + .checked_add(1) + .ok_or(ClientError::Protocol("operator status count overflowed")) +} + #[derive(Clone, Copy, Debug, Eq, PartialEq)] struct SessionSummary { session_id: CanonicalUuid, @@ -6057,7 +6286,7 @@ mod tests { RunnerPlacementRevision, RunnerProjection, RunnerProjectionSelector, RunnerProjectionState, RunnerSandboxProfile, RunnerStateTransitionState, ServerFrame, ServerMessage, SessionEvent, SessionPlacement, SettingOverlay, SystemPromptMember, SystemPromptText, ToolBatchState, - ToolDecision, TurnState, decode_client_line, encode_server_line, + ToolDecision, TurnState, UserInputContent, decode_client_line, encode_server_line, }; use tokio::{ io::{AsyncBufReadExt, AsyncReadExt, AsyncWriteExt, BufReader}, @@ -7041,7 +7270,7 @@ mod tests { } #[test] - fn send_fails_explicitly_when_model_call_recovery_is_required() { + fn send_waits_while_automatic_model_call_recovery_owns_the_decision() { let state = TurnState::ActiveAwaitingModelCallRecovery { ended_attempt_id: CanonicalUuid::from_uuid(Uuid::from_u128(1)), recovery_model_call_id: CanonicalUuid::from_uuid(Uuid::from_u128(2)), @@ -7049,6 +7278,18 @@ mod tests { operator_action_required: false, }; + assert!(matches!(terminal_snapshot_state(Some(&state)), Ok(None))); + } + + #[test] + fn send_fails_when_model_call_recovery_requires_operator_action() { + let state = TurnState::ActiveAwaitingModelCallRecovery { + ended_attempt_id: CanonicalUuid::from_uuid(Uuid::from_u128(1)), + recovery_model_call_id: CanonicalUuid::from_uuid(Uuid::from_u128(2)), + automatic_reconciliation_attempts: CanonicalU64::new(5), + operator_action_required: true, + }; + assert!(matches!( terminal_snapshot_state(Some(&state)), Err(ClientError::TurnRecoveryRequired) @@ -7170,7 +7411,7 @@ mod tests { model_settings: None, state: TurnState::Queued { accepted_input_id: CanonicalUuid::from_uuid(Uuid::from_u128(10)), - content: InputContent::new(String::from("wait behind recovery")), + content: UserInputContent::text(String::from("wait behind recovery")), }, })?) .map_err(io::Error::other)?, @@ -7254,6 +7495,31 @@ mod tests { }, )?; refresh_writer.write_all(&refreshed).await?; + + let (exhausted_stream, mut exhausted_writer) = listener.accept().await?.0.into_split(); + let mut exhausted_reader = BufReader::new(exhausted_stream); + let mut exhausted_line = Vec::new(); + exhausted_reader + .read_until(b'\n', &mut exhausted_line) + .await?; + let exhausted_request = + decode_client_line(&exhausted_line).map_err(io::Error::other)?; + assert_eq!( + exhausted_request.request(), + &ClientRequest::ReadTranscript { session_id } + ); + let exhausted = snapshot( + exhausted_request.version(), + exhausted_request.request_id(), + 1, + TurnState::ActiveAwaitingModelCallRecovery { + ended_attempt_id: CanonicalUuid::from_uuid(Uuid::from_u128(8)), + recovery_model_call_id: CanonicalUuid::from_uuid(Uuid::from_u128(9)), + automatic_reconciliation_attempts: CanonicalU64::new(5), + operator_action_required: true, + }, + )?; + exhausted_writer.write_all(&exhausted).await?; Ok::<(), io::Error>(()) }); @@ -7265,6 +7531,140 @@ mod tests { Ok(()) } + #[tokio::test] + async fn selected_send_polls_after_an_automatic_recovery_transition() + -> Result<(), Box> { + let directory = tempfile::tempdir()?; + let socket = directory.path().join("client.sock"); + let listener = UnixListener::bind(&socket)?; + let session_id = CanonicalUuid::from_uuid(Uuid::from_u128(1)); + let turn_id = CanonicalUuid::from_uuid(Uuid::from_u128(2)); + let attempt_id = CanonicalUuid::from_uuid(Uuid::from_u128(3)); + let model_call_id = CanonicalUuid::from_uuid(Uuid::from_u128(4)); + let server = tokio::spawn(async move { + let snapshot = |version, request_id, cursor, state| -> io::Result> { + let frame = |message| { + ServerFrame::try_new_for_version(version, request_id, message) + .map_err(io::Error::other) + }; + let mut response = + encode_server_line(&frame(ServerMessage::TranscriptSnapshotStart { + session_id, + cursor: CanonicalU64::new(cursor), + runner: None, + })?) + .map_err(io::Error::other)?; + response.extend_from_slice( + &encode_server_line(&frame(ServerMessage::TranscriptTurn { + turn_id, + acceptance_position: CanonicalU64::new(1), + model_settings: None, + state, + })?) + .map_err(io::Error::other)?, + ); + response.extend_from_slice( + &encode_server_line(&frame(ServerMessage::TranscriptModelCallsEnd { + model_call_count: CanonicalU64::new(0), + })?) + .map_err(io::Error::other)?, + ); + response.extend_from_slice( + &encode_server_line(&frame(ServerMessage::TranscriptSnapshotEnd { + session_id, + cursor: CanonicalU64::new(cursor), + turn_count: CanonicalU64::new(1), + entry_count: CanonicalU64::new(0), + })?) + .map_err(io::Error::other)?, + ); + Ok(response) + }; + + let (stream, mut writer) = listener.accept().await?.0.into_split(); + let mut reader = BufReader::new(stream); + let mut line = Vec::new(); + reader.read_until(b'\n', &mut line).await?; + let follow_request = decode_client_line(&line).map_err(io::Error::other)?; + let mut initial = snapshot( + follow_request.version(), + follow_request.request_id(), + 0, + TurnState::ActiveRunning { + current_attempt_id: attempt_id, + current_model_call: None, + }, + )?; + initial.extend_from_slice( + &encode_server_line( + &ServerFrame::try_new_for_version( + follow_request.version(), + follow_request.request_id(), + ServerMessage::SessionEvent { + cursor: CanonicalU64::new(1), + session_id, + event: SessionEvent::ModelCallTransition { + turn_id, + model_call_id, + state: ModelCallState::Terminal { + disposition: ModelCallDisposition::Ambiguous, + }, + }, + }, + ) + .map_err(io::Error::other)?, + ) + .map_err(io::Error::other)?, + ); + writer.write_all(&initial).await?; + + let (refresh_stream, mut refresh_writer) = listener.accept().await?.0.into_split(); + let mut refresh_reader = BufReader::new(refresh_stream); + let mut refresh_line = Vec::new(); + refresh_reader.read_until(b'\n', &mut refresh_line).await?; + let refresh_request = decode_client_line(&refresh_line).map_err(io::Error::other)?; + refresh_writer + .write_all(&snapshot( + refresh_request.version(), + refresh_request.request_id(), + 1, + TurnState::ActiveAwaitingModelCallRecovery { + ended_attempt_id: attempt_id, + recovery_model_call_id: model_call_id, + automatic_reconciliation_attempts: CanonicalU64::new(0), + operator_action_required: false, + }, + )?) + .await?; + + let (poll_stream, mut poll_writer) = listener.accept().await?.0.into_split(); + let mut poll_reader = BufReader::new(poll_stream); + let mut poll_line = Vec::new(); + poll_reader.read_until(b'\n', &mut poll_line).await?; + let poll_request = decode_client_line(&poll_line).map_err(io::Error::other)?; + poll_writer + .write_all(&snapshot( + poll_request.version(), + poll_request.request_id(), + 1, + TurnState::ReconciliationRequired { + terminal_frontier_id: CanonicalUuid::from_uuid(Uuid::from_u128(5)), + terminal_attempt_id: attempt_id, + terminal_model_call_id: model_call_id, + }, + )?) + .await?; + Ok::<(), io::Error>(()) + }); + + let mut client = ProcessClient::new(socket); + let terminal = await_turn_terminal(&mut client, session_id, turn_id).await?; + + assert_eq!(terminal, TurnTerminal::ReconciliationRequired); + server.await??; + Ok(()) + } + #[tokio::test] async fn send_wait_continues_after_a_superseded_runner_loss_event() -> Result<(), Box> { @@ -7441,7 +7841,7 @@ mod tests { model_settings: None, state: TurnState::Queued { accepted_input_id: CanonicalUuid::from_uuid(Uuid::from_u128(3)), - content: InputContent::new(String::from("stream the reply")), + content: UserInputContent::text(String::from("stream the reply")), }, })?) .map_err(io::Error::other)?, @@ -7533,7 +7933,7 @@ mod tests { model_settings: None, state: TurnState::Queued { accepted_input_id: CanonicalUuid::from_uuid(Uuid::from_u128(4)), - content: InputContent::new(String::from("stream the reply")), + content: UserInputContent::text(String::from("stream the reply")), }, })?) .map_err(io::Error::other)?, @@ -7708,7 +8108,7 @@ mod tests { model_settings: None, state: TurnState::Queued { accepted_input_id: CanonicalUuid::from_uuid(Uuid::from_u128(3)), - content: InputContent::new(String::from("queued selected input")), + content: UserInputContent::text(String::from("queued selected input")), }, }], ) @@ -9897,7 +10297,7 @@ mod tests { let turn_id = CanonicalUuid::from_uuid(Uuid::from_u128(2)); let command_id = CommandId::try_from_uuid(Uuid::from_u128(4))?; let content = InputContent::new(String::from("queued content")); - let expected_content = content.clone(); + let expected_content = UserInputContent::text(content.clone().into_string()); let server = tokio::spawn(async move { let (stream, _) = listener.accept().await?; let (reader, mut writer) = stream.into_split(); @@ -9978,7 +10378,7 @@ mod tests { command_id: CommandId::try_from_uuid(Uuid::from_u128(4)) .map_err(io::Error::other)?, session_id, - content: InputContent::new(String::from("steering content")), + content: UserInputContent::text(String::from("steering content")), expected_defaults_version: None, model_settings: ModelSettingsOverlay::inherit_all(), delivery: Some(InputDelivery::Steer { @@ -10041,7 +10441,7 @@ mod tests { let command_id = CommandId::try_from_uuid(Uuid::from_u128(4))?; let defaults_version = CanonicalU64::new(1); let content = InputContent::new(String::from("continue after reconciliation")); - let expected_content = content.clone(); + let expected_content = UserInputContent::text(content.clone().into_string()); let server = tokio::spawn(async move { let (stream, mut writer) = listener.accept().await?.0.into_split(); let mut reader = BufReader::new(stream); @@ -10111,7 +10511,7 @@ mod tests { command_id, session_id, expected_active_turn_id: active_turn_id, - content: content.clone(), + content: UserInputContent::text(content.clone().into_string()), expected_defaults_version: defaults_version, descendant_scope: selected_scope, model_settings: ModelSettingsOverlay::inherit_all(), @@ -10275,6 +10675,54 @@ mod tests { Ok(()) } + /// INV-033: `decide` accepts only its own receipt. A `tool_denial_overridden` + /// receipt names a distinct command — it proves a one-shot override was + /// recorded for a future re-proposal, never that this pending request was + /// decided — so naming the same request cannot make it stand in for one. + #[tokio::test] + async fn inv033_decide_rejects_a_denial_override_receipt() -> Result<(), Box> { + let directory = tempfile::tempdir()?; + let socket = directory.path().join("client.sock"); + let listener = UnixListener::bind(&socket)?; + let session_id = CanonicalUuid::from_uuid(Uuid::from_u128(1)); + let tool_request_id = CanonicalUuid::from_uuid(Uuid::from_u128(2)); + let server = tokio::spawn(async move { + let (stream, mut writer) = listener.accept().await?.0.into_split(); + let mut reader = BufReader::new(stream); + let mut line = Vec::new(); + reader.read_until(b'\n', &mut line).await?; + let request = decode_client_line(&line).map_err(io::Error::other)?; + let response = ServerFrame::try_new_for_version( + request.version(), + request.request_id(), + ServerMessage::ToolDenialOverridden { tool_request_id }, + ) + .map_err(io::Error::other)?; + writer + .write_all(&encode_server_line(&response).map_err(io::Error::other)?) + .await?; + Ok::<(), io::Error>(()) + }); + + let mut stdout = Vec::new(); + let mut stderr = Vec::new(); + let mut output = Output::new(&mut stdout, &mut stderr, false); + let mut client = ProcessClient::new(socket); + let result = decide( + &mut client, + &mut output, + session_id, + tool_request_id, + Some(CommandId::try_from_uuid(Uuid::from_u128(4))?), + ToolDecision::Approve {}, + ) + .await; + assert!(matches!(result, Err(ClientError::AmbiguousMutation))); + server.await??; + assert_eq!(String::from_utf8(stdout)?, ""); + Ok(()) + } + const DELEGATION_SESSION: &str = "00000000-0000-0000-0000-000000000001"; const DELEGATION_TURN: &str = "00000000-0000-0000-0000-000000000002"; const DELEGATION_SPAWN_REQUEST: &str = "00000000-0000-0000-0000-000000000003"; diff --git a/apps/client/src/presentation.rs b/apps/client/src/presentation.rs index 76800591d5..517f4da667 100644 --- a/apps/client/src/presentation.rs +++ b/apps/client/src/presentation.rs @@ -14,14 +14,19 @@ use signalbox_process_protocol::{ FailedModelCallDisposition, GoalBlockedProvenance, GoalBlockedReason, GoalHistoryEvent, GoalLifecycleState, ImportedContentKind, ImportedSourceSpeaker, ImportedSpeaker, ImportedTextPreview, MAX_RATE_VERSION_UTF8_BYTES, MetadataActor, MetadataLastWriter, - ModelCallCostLabel, ModelCallDisposition, ModelCallState, ReviewDiffSide, - ReviewFindingSnapshot, ReviewFindingStatus, ReviewOrchestrationConcernStatus, - ReviewOrchestrationSnapshot, ReviewOrchestrationState, ReviewPassKind, ReviewPassLifecycle, - ReviewRunLifecycle, ReviewRunSnapshot, ReviewSeverity, ReviewTargetSnapshot, - ReviewTargetSubject, ReviewWorkflow, RunnerConnectionHealth, RunnerProjection, - RunnerProjectionSelector, RunnerProjectionState, RunnerSandboxProfile, - RunnerStateTransitionState, SessionEvent, ToolApprovalEventDecider, ToolApprovalEventDecision, - ToolBatchState, ToolDecision, TranscriptEntry, TranscriptTextEntry, TurnState, UsageProvenance, + ModelCallCostLabel, ModelCallDisposition, ModelCallState, OperatorStatusConvergenceSeal, + OperatorStatusConvergenceVerdict, OperatorStatusHeldSlotBlocker, OperatorStatusHeldSlotMessage, + OperatorStatusHeldSlotOrigin, OperatorStatusMergeableState, OperatorStatusMessage, + OperatorStatusPendingStaleReviewClearanceMessage, OperatorStatusPullRequestConvergenceMessage, + OperatorStatusQueuedObligationMessage, OperatorStatusReviewDecision, + OperatorStatusSingletonScope, ReviewDiffSide, ReviewFindingSnapshot, ReviewFindingStatus, + ReviewOrchestrationConcernStatus, ReviewOrchestrationSnapshot, ReviewOrchestrationState, + ReviewPassKind, ReviewPassLifecycle, ReviewRunLifecycle, ReviewRunSnapshot, ReviewSeverity, + ReviewTargetSnapshot, ReviewTargetSubject, ReviewWorkflow, RunnerConnectionHealth, + RunnerProjection, RunnerProjectionSelector, RunnerProjectionState, RunnerSandboxProfile, + RunnerStateTransitionState, ServerMessage, SessionEvent, ToolApprovalEventDecider, + ToolApprovalEventDecision, ToolBatchState, ToolDecision, TranscriptEntry, TranscriptTextEntry, + TurnState, UsageProvenance, UserInputContent, UserInputPart, }; use crate::{ @@ -64,6 +69,13 @@ pub(crate) struct SessionMessageSentPresentation { pub(crate) delivery_sequence: u64, } +pub(crate) struct OperatorStatusPresentationCounts { + pub(crate) held_slots: u64, + pub(crate) queued_obligations: u64, + pub(crate) pull_request_convergences: u64, + pub(crate) pending_stale_review_clearances: u64, +} + pub(crate) enum BlobUploadPresentation { AlreadyPresent, Committed, @@ -1102,6 +1114,233 @@ impl<'a> Output<'a> { self.recovery_value("through_position", &position.to_string()) } + pub(crate) fn operator_status_counts( + &mut self, + counts: OperatorStatusPresentationCounts, + ) -> io::Result<()> { + let OperatorStatusPresentationCounts { + held_slots, + queued_obligations, + pull_request_convergences, + pending_stale_review_clearances, + } = counts; + writeln!( + self.stdout, + "status held_slots={held_slots} queued_obligations={queued_obligations} \ + pull_request_convergences={pull_request_convergences} \ + pending_stale_review_clearances={pending_stale_review_clearances}" + ) + } + + pub(crate) fn operator_status_item( + &mut self, + message: &ServerMessage, + ) -> Result<(), ClientError> { + let ServerMessage::OperatorStatus(message) = message else { + return Err(ClientError::Protocol( + "operator-status spool contained an unexpected frame", + )); + }; + match message.as_ref() { + OperatorStatusMessage::HeldSlot(item) => { + let OperatorStatusHeldSlotMessage { + dispatch_id, + repository, + origin, + rule_id, + rule_version, + singleton_scope, + singleton_repository, + singleton_pull_request_number, + singleton_stack_root_pull_request_number, + held_for_seconds, + session_ids, + blockers, + } = item.as_ref(); + let repository = self.render_field(repository, TextField::DelimitedOnLine); + let rule_id = self.render_field(rule_id, TextField::DelimitedOnLine); + let origin = operator_status_held_slot_origin_label(origin); + let origin = self.render_field(&origin, TextField::DelimitedOnLine); + let singleton = operator_status_singleton_label(&OperatorStatusSingletonAxes { + scope: *singleton_scope, + repository: singleton_repository.as_deref(), + pull_request_number: *singleton_pull_request_number, + stack_root_pull_request_number: *singleton_stack_root_pull_request_number, + }); + let singleton = self.render_field(&singleton, TextField::DelimitedOnLine); + let sessions = session_ids + .iter() + .map(ToString::to_string) + .collect::>() + .join(","); + let blockers = blockers + .iter() + .copied() + .map(operator_status_blocker_label) + .collect::>() + .join(","); + writeln!( + self.stdout, + "held repository={repository} origin={origin} rule={rule_id}@{} \ + singleton={singleton} held={} blockers={} sessions={} \ + dispatch={dispatch_id}", + rule_version.value(), + duration_label(held_for_seconds.value()), + if blockers.is_empty() { + "none" + } else { + &blockers + }, + sessions, + )?; + Ok(()) + } + OperatorStatusMessage::QueuedObligation(item) => { + let OperatorStatusQueuedObligationMessage { + obligation_id, + repository, + rule_id, + rule_version, + singleton_scope, + singleton_repository, + singleton_pull_request_number, + singleton_stack_root_pull_request_number, + first_event_id, + latest_event_id, + matched_event_count, + waiting_for_seconds, + occupying_dispatch_id, + occupying_session_ids, + cooldown_remaining_seconds, + cooldown_never_eligible, + ready, + } = item.as_ref(); + let repository = self.render_field(repository, TextField::DelimitedOnLine); + let rule_id = self.render_field(rule_id, TextField::DelimitedOnLine); + let singleton = operator_status_singleton_label(&OperatorStatusSingletonAxes { + scope: *singleton_scope, + repository: singleton_repository.as_deref(), + pull_request_number: *singleton_pull_request_number, + stack_root_pull_request_number: *singleton_stack_root_pull_request_number, + }); + let singleton = self.render_field(&singleton, TextField::DelimitedOnLine); + // An occupant is a watch dispatch naming its sessions, or an + // independently commissioned live session naming no dispatch. + // The second shape still names its sessions, so rendering keys + // off the inventory rather than the dispatch identity. + let occupancy = if occupying_session_ids.is_empty() { + String::from("none") + } else { + let sessions = occupying_session_ids + .iter() + .map(ToString::to_string) + .collect::>() + .join(","); + occupying_dispatch_id.map_or_else( + || format!("external:{sessions}"), + |dispatch| format!("{dispatch}:{sessions}"), + ) + }; + let cooldown = if *cooldown_never_eligible { + String::from("never") + } else { + cooldown_remaining_seconds.map_or_else( + || String::from("none"), + |seconds| duration_label(seconds.value()), + ) + }; + writeln!( + self.stdout, + "queued repository={repository} rule={rule_id}@{} singleton={singleton} \ + waiting={} matches={} ready={ready} occupying={occupancy} cooldown={cooldown} \ + first_event={first_event_id} latest_event={latest_event_id} \ + obligation={obligation_id}", + rule_version.value(), + duration_label(waiting_for_seconds.value()), + matched_event_count.value(), + )?; + Ok(()) + } + OperatorStatusMessage::PullRequestConvergence(item) => { + let OperatorStatusPullRequestConvergenceMessage { + repository, + pull_request_number, + head_sha, + base_branch, + base_revision, + mergeable_state, + review_decision, + unresolved_thread_count, + gating_check_count, + non_green_gating_checks, + verdict, + seal, + assessed_seconds_ago, + } = item.as_ref(); + let repository = self.render_field(repository, TextField::DelimitedOnLine); + let base_branch = self.render_field(base_branch, TextField::DelimitedOnLine); + let checks = non_green_gating_checks + .iter() + .map(|check| self.render_field(check, TextField::DelimitedOnLine)) + .collect::>() + .join(","); + writeln!( + self.stdout, + "convergence repository={repository} pr={} verdict={} seal={} \ + unresolved_threads={} gating_checks={} non_green_count={} non_green={} mergeable={} review={} \ + assessed_ago={} head={} base={base_branch}@{}", + pull_request_number.value(), + operator_status_verdict_label(*verdict), + seal.map_or("none", operator_status_seal_label), + unresolved_thread_count.value(), + gating_check_count.value(), + non_green_gating_checks.len(), + checks, + operator_status_mergeable_label(*mergeable_state), + operator_status_review_decision_label(*review_decision), + duration_label(assessed_seconds_ago.value()), + head_sha, + base_revision, + )?; + Ok(()) + } + OperatorStatusMessage::PendingStaleReviewClearance(item) => { + let OperatorStatusPendingStaleReviewClearanceMessage { + repository, + pull_request_number, + current_head_sha, + review_node_id, + reviewer, + reviewed_head_sha, + pending_for_seconds, + } = item.as_ref(); + let repository = self.render_field(repository, TextField::DelimitedOnLine); + let review_node_id = self.render_field(review_node_id, TextField::DelimitedOnLine); + let reviewer = self.render_field(reviewer, TextField::DelimitedOnLine); + writeln!( + self.stdout, + "stale_review_clearance repository={repository} pr={} reviewer={reviewer} \ + pending={} review={review_node_id} reviewed_head={} current_head={}", + pull_request_number.value(), + duration_label(pending_for_seconds.value()), + reviewed_head_sha, + current_head_sha, + )?; + Ok(()) + } + OperatorStatusMessage::Start {} | OperatorStatusMessage::End(_) => Err( + ClientError::Protocol("operator-status spool contained an unexpected frame"), + ), + } + } + + pub(crate) fn operator_status_model_usage_omitted(&mut self) -> io::Result<()> { + writeln!( + self.stdout, + "model_usage=omitted reason=no_cheap_status_aggregate" + ) + } + pub(crate) fn session_summary( &mut self, session_id: CanonicalUuid, @@ -1747,7 +1986,7 @@ impl<'a> Output<'a> { accepted_input={accepted_input_id} turn={turn_id} position={}", acceptance_position.value() )?; - self.text(content.as_str()) + self.user_content(content) } SessionEvent::GoalTurnRetired { turn_id } => writeln!( self.stdout, @@ -1848,6 +2087,16 @@ impl<'a> Output<'a> { decider=delegate model_selection={model_selection_id} \ call={model_call_id}" )?, + ToolApprovalEventDecider::UserOverride { + command_id, + overridden_tool_request_id, + } => writeln!( + self.stdout, + "event={cursor} session={session_id} tool_approval_decided \ + turn={turn_id} request={tool_request_id} decision={decision} \ + decider=user_override command={command_id} \ + overridden_request={overridden_tool_request_id}" + )?, } if let Some(reason) = denial_reason { self.text_field("denial_reason", reason)?; @@ -2050,6 +2299,36 @@ impl<'a> Output<'a> { self.text_fragment(text, true, text.ends_with('\n')) } + fn user_content(&mut self, content: &UserInputContent) -> io::Result<()> { + match content.parts() { + [UserInputPart::Text { text }] => self.text(text), + parts => { + self.user_content_parts_json(parts)?; + writeln!(self.stdout)?; + if self.raw { + self.stdout.flush()?; + } + Ok(()) + } + } + } + + fn user_content_parts_json(&mut self, parts: &[UserInputPart]) -> io::Result<()> { + let serialized = serde_json::to_string(parts)?; + if self.raw { + return self.stdout.write_all(serialized.as_bytes()); + } + for character in serialized.chars() { + let code = character as u32; + if (0x7f..=0x9f).contains(&code) { + write!(self.stdout, "\\u{code:04x}")?; + } else { + write!(self.stdout, "{character}")?; + } + } + Ok(()) + } + fn text_fragment( &mut self, fragment: &str, @@ -2083,7 +2362,7 @@ impl<'a> Output<'a> { "turn={turn_id} position={position} state=queued \ accepted_input={accepted_input_id}" )?; - self.text(content.as_str()) + self.user_content(content) } TurnState::QueuedDelegated { spawning_request_id, @@ -2302,11 +2581,25 @@ impl<'a> Output<'a> { fn snapshot_entry(&mut self, entry: &SnapshotEntry) -> io::Result<()> { match &entry.kind { + SnapshotEntryKind::User { + accepted_input_id, + turn_id, + content, + } => { + write!( + self.stdout, + "user_content source_session={} entry={} accepted_input={accepted_input_id} turn={turn_id} parts=", + entry.source_session_id, entry.entry_id + )?; + self.user_content_parts_json(content.parts())?; + writeln!(self.stdout)?; + if self.raw { + self.stdout.flush()?; + } + Ok(()) + } SnapshotEntryKind::Text(metadata) => { let label = match metadata { - TranscriptTextEntry::User { turn_id, .. } => { - format!("user turn={turn_id}") - } TranscriptTextEntry::Assistant { turn_id, .. } => { format!("assistant turn={turn_id}") } @@ -2468,6 +2761,15 @@ impl<'a> Output<'a> { decider=delegate model_selection={model_selection_id} \ call={model_call_id}" )?, + ToolApprovalEventDecider::UserOverride { + command_id, + overridden_tool_request_id, + } => writeln!( + self.stdout, + "tool_approval request={tool_request_id} decision={decision} \ + decider=user_override command={command_id} \ + overridden_request={overridden_tool_request_id}" + )?, } if let Some(reason) = reason { self.text_field("denial_reason", reason)?; @@ -2540,6 +2842,114 @@ impl<'a> Output<'a> { } } +/// The singleton axes of one operator-status row, each named at its call site. +/// +/// The two numeric axes carry one type and mean different things, so they are +/// supplied by name rather than by position. +struct OperatorStatusSingletonAxes<'a> { + scope: OperatorStatusSingletonScope, + repository: Option<&'a str>, + pull_request_number: Option, + stack_root_pull_request_number: Option, +} + +/// Names the fact a held slot was taken from, a branch or a pull request and +/// never both. +fn operator_status_held_slot_origin_label(origin: &OperatorStatusHeldSlotOrigin) -> String { + match origin { + OperatorStatusHeldSlotOrigin::PullRequest { + pull_request_number, + } => format!("pull_request#{}", pull_request_number.value()), + OperatorStatusHeldSlotOrigin::Branch { branch } => format!("branch:{branch}"), + } +} + +fn operator_status_singleton_label(axes: &OperatorStatusSingletonAxes<'_>) -> String { + let OperatorStatusSingletonAxes { + scope, + repository, + pull_request_number, + stack_root_pull_request_number, + } = axes; + match scope { + OperatorStatusSingletonScope::PullRequest => format!( + "pull_request:{}#{}", + repository.unwrap_or("?"), + pull_request_number.map_or(0, |number| number.value()) + ), + OperatorStatusSingletonScope::Stack => format!( + "stack:{}#{}", + repository.unwrap_or("?"), + stack_root_pull_request_number.map_or(0, |number| number.value()) + ), + OperatorStatusSingletonScope::Rule => String::from("rule"), + OperatorStatusSingletonScope::Repo => { + format!("repo:{}", repository.unwrap_or("?")) + } + } +} + +const fn operator_status_blocker_label(blocker: OperatorStatusHeldSlotBlocker) -> &'static str { + match blocker { + OperatorStatusHeldSlotBlocker::UndeliveredAction => "undelivered_action", + OperatorStatusHeldSlotBlocker::DeliveryTurnRuntimeRelevant => { + "delivery_turn_runtime_relevant" + } + OperatorStatusHeldSlotBlocker::LiveRuntimeTurn => "live_runtime_turn", + OperatorStatusHeldSlotBlocker::PursuingGoal => "pursuing_goal", + } +} + +const fn operator_status_mergeable_label(state: OperatorStatusMergeableState) -> &'static str { + match state { + OperatorStatusMergeableState::Mergeable => "mergeable", + OperatorStatusMergeableState::Conflicting => "conflicting", + OperatorStatusMergeableState::Unknown => "unknown", + } +} + +const fn operator_status_review_decision_label( + decision: OperatorStatusReviewDecision, +) -> &'static str { + match decision { + OperatorStatusReviewDecision::None => "none", + OperatorStatusReviewDecision::Approved => "approved", + OperatorStatusReviewDecision::ReviewRequired => "review_required", + OperatorStatusReviewDecision::ChangesRequested => "changes_requested", + } +} + +const fn operator_status_verdict_label(verdict: OperatorStatusConvergenceVerdict) -> &'static str { + match verdict { + OperatorStatusConvergenceVerdict::NotConverged => "not_converged", + OperatorStatusConvergenceVerdict::InternallyConverged => "internally_converged", + OperatorStatusConvergenceVerdict::MergeReady => "merge_ready", + } +} + +const fn operator_status_seal_label(seal: OperatorStatusConvergenceSeal) -> &'static str { + match seal { + OperatorStatusConvergenceSeal::InternallyConverged => "internally_converged", + OperatorStatusConvergenceSeal::MergeReady => "merge_ready", + } +} + +fn duration_label(seconds: u64) -> String { + let days = seconds / 86_400; + let hours = seconds % 86_400 / 3_600; + let minutes = seconds % 3_600 / 60; + let seconds = seconds % 60; + if days > 0 { + format!("{days}d{hours}h{minutes}m{seconds}s") + } else if hours > 0 { + format!("{hours}h{minutes}m{seconds}s") + } else if minutes > 0 { + format!("{minutes}m{seconds}s") + } else { + format!("{seconds}s") + } +} + impl SnapshotSelection { fn context( self, @@ -2755,7 +3165,7 @@ impl SnapshotSelection { | Self::ToolBatchProposed { .. } | Self::ToolBatchResults { .. } | Self::ToolReconciliation { .. }, - SnapshotEntryKind::Text(_), + SnapshotEntryKind::User { .. } | SnapshotEntryKind::Text(_), ) => false, ( Self::ToolBatchProposed { .. } @@ -2817,7 +3227,8 @@ impl SnapshotSelection { | Self::ToolBatchProposed { .. } | Self::ToolBatchResults { .. } | Self::ToolReconciliation { .. }, - SnapshotEntryKind::Text(_) + SnapshotEntryKind::User { .. } + | SnapshotEntryKind::Text(_) | SnapshotEntryKind::Marker( TranscriptEntry::ModelIdentityChanged { .. } | TranscriptEntry::DelegatedTask { .. } @@ -2951,6 +3362,9 @@ const fn failed_model_call_disposition(disposition: FailedModelCallDisposition) const fn failed_model_call_cause(cause: FailedModelCallCause) -> &'static str { match cause { FailedModelCallCause::CredentialRejected => "credential_rejected", + FailedModelCallCause::AttachmentTooLarge => "attachment_too_large", + FailedModelCallCause::AttachmentMissing => "attachment_missing", + FailedModelCallCause::AttachmentCorrupt => "attachment_corrupt", FailedModelCallCause::PermissionDenied => "permission_denied", FailedModelCallCause::InvalidRequest => "invalid_request", FailedModelCallCause::TargetNotFound => "target_not_found", @@ -3222,14 +3636,19 @@ mod tests { DescendantTerminationScope, ErrorCode, ErrorDetail, FailedModelCallDisposition, FailedTerminalModelCall, ImportedContentKind, ImportedSourceSpeaker, ImportedSpeaker, ImportedTextPreview, InputContent, MetadataActor, MetadataLastWriter, ModelCallCostLabel, - ModelCallDollarCost, ModelCallState, ModelCallTokenUsage, ReviewDiffSide, + ModelCallDollarCost, ModelCallState, ModelCallTokenUsage, OperatorStatusConvergenceSeal, + OperatorStatusConvergenceVerdict, OperatorStatusHeldSlotBlocker, + OperatorStatusHeldSlotMessage, OperatorStatusHeldSlotOrigin, OperatorStatusMergeableState, + OperatorStatusMessage, OperatorStatusPendingStaleReviewClearanceMessage, + OperatorStatusPullRequestConvergenceMessage, OperatorStatusQueuedObligationMessage, + OperatorStatusReviewDecision, OperatorStatusSingletonScope, ReviewDiffSide, ReviewFindingInput, ReviewFindingSnapshot, ReviewFindingStatus, ReviewSeverity, ReviewTargetSnapshot, ReviewTargetSubject, RunnerCapabilityClass, RunnerConnectionHealth, RunnerCredentialProfileName, RunnerPlacementRevision, RunnerProjection, RunnerProjectionSelector, RunnerProjectionState, RunnerRepositoryKey, RunnerSandboxProfile, RunnerStateTransitionState, RunnerWorkingDirectory, ServerMessage, SessionEvent, ToolApprovalEventDecider, ToolApprovalEventDecision, TranscriptEntry, TranscriptTextEntry, - TurnState, UsageProvenance, + TurnState, UsageProvenance, UserInputContent, }; use uuid::Uuid; @@ -3254,7 +3673,7 @@ mod tests { expect![[r#" target=00000000-0000-0000-0000-000000000001 subject=commit parent=- provider=example-host - repository=owner/repository + repository=example/repository head_revision=head base_revision_present=false "#]] @@ -3274,7 +3693,7 @@ mod tests { expect![[r#" target=00000000-0000-0000-0000-000000000001 subject=commit parent=- provider=example-host - repository=owner/repository + repository=example/repository head_revision=head base_revision_present=true base_revision=- @@ -3359,6 +3778,233 @@ mod tests { assert!(stderr.is_empty()); } + #[test] + fn operator_status_renders_all_sections_and_explains_omitted_usage() { + let mut stdout = Vec::new(); + let mut stderr = Vec::new(); + { + let mut output = Output::new(&mut stdout, &mut stderr, false); + output + .operator_status_counts(super::OperatorStatusPresentationCounts { + held_slots: 1, + queued_obligations: 1, + pull_request_convergences: 1, + pending_stale_review_clearances: 1, + }) + .expect("in-memory output cannot fail"); + output + .operator_status_item(&ServerMessage::OperatorStatus(Box::new( + OperatorStatusMessage::HeldSlot(Box::new(OperatorStatusHeldSlotMessage { + dispatch_id: wire_uuid(1), + repository: String::from("example/repo"), + origin: OperatorStatusHeldSlotOrigin::PullRequest { + pull_request_number: CanonicalU64::new(41), + }, + rule_id: String::from("review"), + rule_version: CanonicalU64::new(1), + singleton_scope: OperatorStatusSingletonScope::PullRequest, + singleton_repository: Some(String::from("example/repo")), + singleton_pull_request_number: Some(CanonicalU64::new(41)), + singleton_stack_root_pull_request_number: None, + held_for_seconds: CanonicalU64::new(3_661), + session_ids: vec![wire_uuid(2)], + blockers: vec![OperatorStatusHeldSlotBlocker::PursuingGoal], + })), + ))) + .expect("in-memory output cannot fail"); + output + .operator_status_item(&ServerMessage::OperatorStatus(Box::new( + OperatorStatusMessage::QueuedObligation(Box::new( + OperatorStatusQueuedObligationMessage { + obligation_id: wire_uuid(3), + repository: String::from("example/repo"), + rule_id: String::from("review"), + rule_version: CanonicalU64::new(1), + singleton_scope: OperatorStatusSingletonScope::Rule, + singleton_repository: None, + singleton_pull_request_number: None, + singleton_stack_root_pull_request_number: None, + first_event_id: wire_uuid(4), + latest_event_id: wire_uuid(5), + matched_event_count: CanonicalU64::new(3), + waiting_for_seconds: CanonicalU64::new(65), + occupying_dispatch_id: None, + occupying_session_ids: Vec::new(), + cooldown_remaining_seconds: Some(CanonicalU64::new(5)), + cooldown_never_eligible: false, + ready: false, + }, + )), + ))) + .expect("in-memory output cannot fail"); + output + .operator_status_item(&ServerMessage::OperatorStatus(Box::new( + OperatorStatusMessage::PullRequestConvergence(Box::new( + OperatorStatusPullRequestConvergenceMessage { + repository: String::from("example/repo"), + pull_request_number: CanonicalU64::new(41), + head_sha: String::from("1111111111111111111111111111111111111111"), + base_branch: String::from("main"), + base_revision: String::from("2222222222222222222222222222222222222222"), + mergeable_state: OperatorStatusMergeableState::Mergeable, + review_decision: OperatorStatusReviewDecision::Approved, + unresolved_thread_count: CanonicalU64::new(0), + gating_check_count: CanonicalU64::new(2), + non_green_gating_checks: Vec::new(), + verdict: OperatorStatusConvergenceVerdict::MergeReady, + seal: Some(OperatorStatusConvergenceSeal::MergeReady), + assessed_seconds_ago: CanonicalU64::new(9), + }, + )), + ))) + .expect("in-memory output cannot fail"); + output + .operator_status_item(&ServerMessage::OperatorStatus(Box::new( + OperatorStatusMessage::PendingStaleReviewClearance(Box::new( + OperatorStatusPendingStaleReviewClearanceMessage { + repository: String::from("example/repo"), + pull_request_number: CanonicalU64::new(41), + current_head_sha: String::from( + "1111111111111111111111111111111111111111", + ), + review_node_id: String::from("PRR_node"), + reviewer: String::from("reviewer"), + reviewed_head_sha: String::from( + "3333333333333333333333333333333333333333", + ), + pending_for_seconds: CanonicalU64::new(8), + }, + )), + ))) + .expect("in-memory output cannot fail"); + output + .operator_status_model_usage_omitted() + .expect("in-memory output cannot fail"); + } + + let rendered = String::from_utf8(stdout).expect("rendered output is UTF-8"); + expect![[r#" + status held_slots=1 queued_obligations=1 pull_request_convergences=1 pending_stale_review_clearances=1 + held repository=example/repo origin=pull_request#41 rule=review@1 singleton=pull_request:example/repo#41 held=1h1m1s blockers=pursuing_goal sessions=00000000-0000-0000-0000-000000000002 dispatch=00000000-0000-0000-0000-000000000001 + queued repository=example/repo rule=review@1 singleton=rule waiting=1m5s matches=3 ready=false occupying=none cooldown=5s first_event=00000000-0000-0000-0000-000000000004 latest_event=00000000-0000-0000-0000-000000000005 obligation=00000000-0000-0000-0000-000000000003 + convergence repository=example/repo pr=41 verdict=merge_ready seal=merge_ready unresolved_threads=0 gating_checks=2 non_green_count=0 non_green= mergeable=mergeable review=approved assessed_ago=9s head=1111111111111111111111111111111111111111 base=main@2222222222222222222222222222222222222222 + stale_review_clearance repository=example/repo pr=41 reviewer=reviewer pending=8s review=PRR_node reviewed_head=3333333333333333333333333333333333333333 current_head=1111111111111111111111111111111111111111 + model_usage=omitted reason=no_cheap_status_aggregate + "#]] + .assert_eq(&rendered); + assert!(stderr.is_empty()); + } + + #[test] + fn operator_status_distinguishes_a_check_named_none_from_an_empty_inventory() { + let mut stdout = Vec::new(); + let mut stderr = Vec::new(); + { + let mut output = Output::new(&mut stdout, &mut stderr, false); + output + .operator_status_item(&ServerMessage::OperatorStatus(Box::new( + OperatorStatusMessage::PullRequestConvergence(Box::new( + OperatorStatusPullRequestConvergenceMessage { + repository: String::from("example/repo"), + pull_request_number: CanonicalU64::new(41), + head_sha: String::from("1111111111111111111111111111111111111111"), + base_branch: String::from("main"), + base_revision: String::from("2222222222222222222222222222222222222222"), + mergeable_state: OperatorStatusMergeableState::Mergeable, + review_decision: OperatorStatusReviewDecision::ChangesRequested, + unresolved_thread_count: CanonicalU64::new(1), + gating_check_count: CanonicalU64::new(1), + non_green_gating_checks: vec![String::from("none")], + verdict: OperatorStatusConvergenceVerdict::NotConverged, + seal: None, + assessed_seconds_ago: CanonicalU64::new(1), + }, + )), + ))) + .expect("in-memory output cannot fail"); + } + + let rendered = String::from_utf8(stdout).expect("rendered output is UTF-8"); + assert!(rendered.contains("non_green_count=1 non_green=none")); + assert!(stderr.is_empty()); + } + + /// A branch-triggered hold names the branch its dispatch came from, since + /// no pull request exists to name. + #[test] + fn operator_status_names_the_branch_a_held_slot_was_triggered_from() { + let mut stdout = Vec::new(); + let mut stderr = Vec::new(); + { + let mut output = Output::new(&mut stdout, &mut stderr, false); + output + .operator_status_item(&ServerMessage::OperatorStatus(Box::new( + OperatorStatusMessage::HeldSlot(Box::new(OperatorStatusHeldSlotMessage { + dispatch_id: wire_uuid(1), + repository: String::from("example/repo"), + origin: OperatorStatusHeldSlotOrigin::Branch { + branch: String::from("main"), + }, + rule_id: String::from("branch-follow-up"), + rule_version: CanonicalU64::new(1), + singleton_scope: OperatorStatusSingletonScope::Repo, + singleton_repository: Some(String::from("example/repo")), + singleton_pull_request_number: None, + singleton_stack_root_pull_request_number: None, + held_for_seconds: CanonicalU64::new(30), + session_ids: vec![wire_uuid(2)], + blockers: Vec::new(), + })), + ))) + .expect("in-memory output cannot fail"); + } + + let rendered = String::from_utf8(stdout).expect("rendered output is UTF-8"); + assert!(rendered.contains("origin=branch:main")); + assert!(!rendered.contains("pull_request#")); + assert!(stderr.is_empty()); + } + + /// An obligation blocked by an independently commissioned live session + /// names that session even though no watch dispatch occupies the singleton. + #[test] + fn operator_status_names_an_external_blocker_without_a_dispatch_identity() { + let mut stdout = Vec::new(); + let mut stderr = Vec::new(); + { + let mut output = Output::new(&mut stdout, &mut stderr, false); + output + .operator_status_item(&ServerMessage::OperatorStatus(Box::new( + OperatorStatusMessage::QueuedObligation(Box::new( + OperatorStatusQueuedObligationMessage { + obligation_id: wire_uuid(3), + repository: String::from("example/repo"), + rule_id: String::from("review"), + rule_version: CanonicalU64::new(1), + singleton_scope: OperatorStatusSingletonScope::Rule, + singleton_repository: None, + singleton_pull_request_number: None, + singleton_stack_root_pull_request_number: None, + first_event_id: wire_uuid(4), + latest_event_id: wire_uuid(5), + matched_event_count: CanonicalU64::new(1), + waiting_for_seconds: CanonicalU64::new(5), + occupying_dispatch_id: None, + occupying_session_ids: vec![wire_uuid(6)], + cooldown_remaining_seconds: None, + cooldown_never_eligible: false, + ready: false, + }, + )), + ))) + .expect("in-memory output cannot fail"); + } + + let rendered = String::from_utf8(stdout).expect("rendered output is UTF-8"); + assert!(rendered.contains("occupying=external:00000000-0000-0000-0000-000000000006")); + assert!(stderr.is_empty()); + } + #[test] fn terminal_safe_delimited_field_escapes_the_space_and_comma_that_delimit_it() { assert_eq!( @@ -3750,7 +4396,7 @@ mod tests { model_settings: None, state: TurnState::Queued { accepted_input_id, - content: InputContent::new("queued user text".to_owned()), + content: UserInputContent::text("queued user text".to_owned()), }, }], ) @@ -3909,6 +4555,40 @@ mod tests { assert!(stderr.is_empty()); } + #[test] + fn snapshot_user_entry_renders_canonical_parts_on_one_line() { + let mut snapshot = TranscriptSnapshot::from_messages( + 9, + [ServerMessage::TranscriptUserEntry { + entry_index: CanonicalU64::new(0), + source_session_id: wire_uuid(1), + entry_id: wire_uuid(2), + accepted_input_id: wire_uuid(3), + turn_id: wire_uuid(4), + content: UserInputContent::text("first\nsecond".to_owned()), + }], + ) + .expect("test snapshot must spool"); + let mut stdout = Vec::new(); + let mut stderr = Vec::new(); + Output::new(&mut stdout, &mut stderr, false) + .snapshot(&mut snapshot) + .expect("user snapshot must render"); + + let rendered = String::from_utf8(stdout).expect("rendered output is UTF-8"); + assert!(rendered.starts_with( + "user_content source_session=00000000-0000-0000-0000-000000000001 entry=00000000-0000-0000-0000-000000000002 accepted_input=00000000-0000-0000-0000-000000000003 turn=00000000-0000-0000-0000-000000000004 parts=[{\"type\":\"text\",\"text\":\"first\\nsecond\"}]\n" + )); + assert_eq!( + rendered + .lines() + .filter(|line| line.starts_with("user_content ")) + .count(), + 1 + ); + assert!(stderr.is_empty()); + } + #[test] fn s28_imported_snapshot_renders_conservative_nontext() { let mut snapshot = TranscriptSnapshot::from_messages( @@ -4668,7 +5348,7 @@ mod tests { model_settings: None, state: TurnState::Queued { accepted_input_id: wire_uuid(10), - content: InputContent::new("transcript content".to_owned()), + content: UserInputContent::text("transcript content".to_owned()), }, }, ServerMessage::TranscriptModelCallUsage { @@ -5012,7 +5692,7 @@ mod tests { ReviewTargetSnapshot { target_id: wire_uuid(1), provider: String::from("example-host"), - repository: String::from("owner/repository"), + repository: String::from("example/repository"), subject: ReviewTargetSubject::Commit {}, head_revision: String::from("head"), base_revision, diff --git a/apps/client/src/transcript.rs b/apps/client/src/transcript.rs index ae2107d15a..c1ba3fcbeb 100644 --- a/apps/client/src/transcript.rs +++ b/apps/client/src/transcript.rs @@ -137,6 +137,11 @@ pub(crate) struct SnapshotEntry { #[derive(Clone, Debug, Eq, PartialEq)] pub(crate) enum SnapshotEntryKind { + User { + accepted_input_id: CanonicalUuid, + turn_id: CanonicalUuid, + content: signalbox_process_protocol::UserInputContent, + }, Text(TranscriptTextEntry), Marker(TranscriptEntry), } @@ -310,6 +315,24 @@ pub(crate) async fn read_snapshot( .checked_add(1) .ok_or(ClientError::Protocol("snapshot entry count overflowed"))?; } + ServerMessage::TranscriptUserEntry { + entry_index, + source_session_id, + entry_id, + .. + } if model_calls_ended => { + entries_started = true; + require_entry_index(entry_index.value(), entry_count)?; + if !entry_ids.insert(entry_key(source_session_id, entry_id))? { + return Err(ClientError::Protocol( + "snapshot repeated a source-qualified entry identity", + )); + } + append_frame(&mut spool, &frame)?; + entry_count = entry_count + .checked_add(1) + .ok_or(ClientError::Protocol("snapshot entry count overflowed"))?; + } ServerMessage::TranscriptTextEntry { entry_index, source_session_id, @@ -442,6 +465,23 @@ fn snapshot_record(message: ServerMessage) -> Result Ok(SnapshotRecord::Entry(SnapshotEntry { + entry_index: entry_index.value(), + source_session_id, + entry_id, + kind: SnapshotEntryKind::User { + accepted_input_id, + turn_id, + content, + }, + })), ServerMessage::TranscriptTextEntry { entry_index, source_session_id, diff --git a/apps/client/tests/chat_end_to_end.rs b/apps/client/tests/chat_end_to_end.rs index 5a4face524..69f79843b8 100644 --- a/apps/client/tests/chat_end_to_end.rs +++ b/apps/client/tests/chat_end_to_end.rs @@ -47,6 +47,7 @@ use signalbox_test_bin::test_bin_path; use signalboxd::{ ActivatedTurnPass, FatalExecutionSupervisor, LocalProcessListener, PostgresProviderModelExecution, ProcessRuntime, ProcessRuntimeError, + WorkspaceInstructionRuntime, }; use sqlx::{PgPool, postgres::PgPoolOptions}; use testcontainers_modules::{ @@ -409,11 +410,12 @@ context_window_tokens = {CONTEXT_WINDOW_TOKENS} provider, None, ) - .with_tool_loop( - tool_dispatch_gate, - tool_catalog, - CompletingFixtureExecutor, - ), + .with_tool_loop(tool_dispatch_gate, tool_catalog, CompletingFixtureExecutor) + .with_workspace_instructions(WorkspaceInstructionRuntime::new( + pool.clone(), + None, + Vec::new(), + )), ); let pass = ActivatedTurnPass::new( StartEligibleTurnService::new( diff --git a/apps/client/tests/end_to_end.rs b/apps/client/tests/end_to_end.rs index 947ab1392a..f0764bfa20 100644 --- a/apps/client/tests/end_to_end.rs +++ b/apps/client/tests/end_to_end.rs @@ -55,6 +55,7 @@ use signalboxd::{ ActivatedTurnExecution, ActivatedTurnPass, FatalExecutionSupervisor, FileCredentialAccess, HubModelConfiguration, LocalProcessListener, ModelAdapter, PostgresProviderModelExecution, ProcessRuntime, ProcessRuntimeError, SessionTemplateConfiguration, + WorkspaceInstructionPreparedExecution, WorkspaceInstructionRuntime, }; use sqlx::{PgPool, postgres::PgPoolOptions}; use testcontainers_modules::{ @@ -1790,15 +1791,18 @@ context_window_tokens = 200000 model_configuration, ); let (execution, fatal_execution) = - FatalExecutionSupervisor::new(PostgresProviderModelExecution::new( - PostgresModelCallRepository::new( - pool.clone(), - targets, - ModelCallCredentialReference::new("scripted-imported-continuation"), + FatalExecutionSupervisor::new(WorkspaceInstructionPreparedExecution::new( + PostgresProviderModelExecution::new( + PostgresModelCallRepository::new( + pool.clone(), + targets, + ModelCallCredentialReference::new("scripted-imported-continuation"), + ), + InProcessAttemptDispatchGate::default(), + provider, + None, ), - InProcessAttemptDispatchGate::default(), - provider, - None, + WorkspaceInstructionRuntime::new(pool.clone(), None, Vec::new()), )); let pass = ActivatedTurnPass::new( StartEligibleTurnService::new( @@ -1902,12 +1906,39 @@ context_window_tokens = 200000 let imported_assistant_content = transcript .find(&fixture.imported_assistant) .expect("the transcript contains the imported assistant fixture"); - let live_user_label = transcript - .find("user turn=") + let live_user_line = transcript + .lines() + .find(|line| line.starts_with("user_content source_session=")) .expect("the transcript labels the live user entry"); - let live_user_content = transcript - .find(&fixture.live_user) - .expect("the transcript contains the live user fixture"); + let live_user_label = transcript + .find(live_user_line) + .expect("the live user line belongs to the transcript"); + let (identity_fields, parts) = live_user_line + .strip_prefix("user_content ") + .and_then(|line| line.split_once(" parts=")) + .expect("the live user entry has canonical metadata and parts"); + let identity_fields = identity_fields.split_whitespace().collect::>(); + assert_eq!(identity_fields.len(), 4); + for (field, prefix) in + identity_fields + .iter() + .zip(["source_session=", "entry=", "accepted_input=", "turn="]) + { + let value = field + .strip_prefix(prefix) + .expect("the live user identity fields use canonical labels"); + let parsed = + uuid::Uuid::parse_str(value).expect("the live user identity fields contain UUIDs"); + assert_eq!(parsed.hyphenated().to_string(), value); + } + assert_eq!( + serde_json::from_str::(parts)?, + serde_json::json!([{"type": "text", "text": fixture.live_user}]) + ); + let live_user_content = live_user_label + + live_user_line + .find(&fixture.live_user) + .expect("the canonical live user parts contain the fixture"); let live_assistant_label = transcript .find("assistant turn=") .expect("the transcript labels the live assistant entry"); @@ -2043,15 +2074,18 @@ context_window_tokens = 200000 model_configuration, ); let (execution, fatal_execution) = - FatalExecutionSupervisor::new(PostgresProviderModelExecution::new( - PostgresModelCallRepository::new( - pool.clone(), - targets, - ModelCallCredentialReference::new("scripted-terminal"), + FatalExecutionSupervisor::new(WorkspaceInstructionPreparedExecution::new( + PostgresProviderModelExecution::new( + PostgresModelCallRepository::new( + pool.clone(), + targets, + ModelCallCredentialReference::new("scripted-terminal"), + ), + InProcessAttemptDispatchGate::default(), + provider, + None, ), - InProcessAttemptDispatchGate::default(), - provider, - None, + WorkspaceInstructionRuntime::new(pool.clone(), None, Vec::new()), )); let pass = ActivatedTurnPass::new( StartEligibleTurnService::new( @@ -2430,15 +2464,18 @@ context_window_tokens = 200000 assert_eq!(activation_recovery.stderr, pass_activated.stderr); let (execution, fatal_execution) = - FatalExecutionSupervisor::new(PostgresProviderModelExecution::new( - PostgresModelCallRepository::new( - pool.clone(), - targets, - ModelCallCredentialReference::new("scripted-review"), + FatalExecutionSupervisor::new(WorkspaceInstructionPreparedExecution::new( + PostgresProviderModelExecution::new( + PostgresModelCallRepository::new( + pool.clone(), + targets, + ModelCallCredentialReference::new("scripted-review"), + ), + InProcessAttemptDispatchGate::default(), + provider, + None, ), - InProcessAttemptDispatchGate::default(), - provider, - None, + WorkspaceInstructionRuntime::new(pool.clone(), None, Vec::new()), )); execution.execute(activated).await?; assert!(!fatal_execution.is_triggered()); @@ -2824,7 +2861,12 @@ context_window_tokens = 200000 provider, None, ) - .with_tool_loop(tool_dispatch_gate, tool_catalog, CompletingFixtureExecutor), + .with_tool_loop(tool_dispatch_gate, tool_catalog, CompletingFixtureExecutor) + .with_workspace_instructions(WorkspaceInstructionRuntime::new( + pool.clone(), + None, + Vec::new(), + )), ); let pass = ActivatedTurnPass::new( StartEligibleTurnService::new( @@ -2985,12 +3027,15 @@ async fn terminal_client_completes_the_real_anthropic_path() -> Result<(), Box` additionally records the run in two eval-owned +PostgreSQL tables after the scorecard prints; without the flag nothing is +written and the stdout scorecard stays the only artifact either way. When the +URL carries a password, pass `--database-url-env ` instead to read it +from the named environment variable, keeping the credential out of the process +argument vector and shell history. + +- `approval_judge_eval_run` — one row per run: the minted run identity, the + judge selection, the resolved provider target identity and model, the frozen + non-secret credential reference, whether that adapter's reported input total + includes the cache axes, the scorecard's corpus, contract, and rendered + digests, the configured repeats, and the full scorecard as `jsonb`. +- `approval_judge_eval_call` — one row per successful judge call: the run it + belongs to, the case name, the one-based attempt ordinal (a failed attempt + records no row and leaves a gap), the recommendation and rationale, and the + provider-reported token-usage fields. + +Eval calls deliberately never enter `tool_approval_judge_model_call`: its +triggers demand the live-request linkage — an active delegated wait and a +reserved global call identity — that replayed synthetic cases do not have. The +connection takes the same URL-only posture as the daemon's, so ambient `PG*` +variables are refused rather than silently shaping it. + +The tables come from the daemon's migration set, and the daemon is what applies +it; a database missing them, a role lacking the privileges recording exercises +(insert on both tables, and select on the run table, which the sealing trigger +reads), and any corpus case recording cannot store (an empty name, or U+0000 in +a name or notes) are all refused before the first paid call, and the minted run +identity is announced before the commit is attempted so even an ambiguous commit +leaves an exact key to query for. Recorded evidence is append-only and sealed: +both tables refuse updates, deletions, and truncation, and call rows admit +insertion only inside the transaction that records their run, so evidence cannot +be extended after the scorecard is frozen. + ## Case schema One JSON object per line: diff --git a/apps/signalboxd/src/attachment_preparation_runtime.rs b/apps/signalboxd/src/attachment_preparation_runtime.rs new file mode 100644 index 0000000000..36a2c0e2d8 --- /dev/null +++ b/apps/signalboxd/src/attachment_preparation_runtime.rs @@ -0,0 +1,299 @@ +//! Pre-provider verification of rendered attachment authority. + +use std::{collections::BTreeSet, future::Future, sync::Arc}; + +use sha2::{Digest as _, Sha256}; +use signalbox_application::{ + AttachmentPreparationFailure, ModelCallCapabilityPreparation, ModelCallProvider, + PreparedModelOperation, +}; +use signalbox_blob_store::{BlobStoreFailureKind, ExpectedBlob}; +use signalbox_domain::{BlobDigest, PreparedModelCallRequest}; +use signalbox_persistence::blob::{ + BlobCatalogEntry, BlobCatalogRepository, BlobCatalogRepositoryError, +}; +use sqlx::PgPool; +use tokio::io::AsyncReadExt as _; + +use crate::BlobStoreRegistry; + +const VERIFICATION_BUFFER_BYTES: usize = 64 * 1024; + +/// Provider wrapper that verifies every rendered attachment before capability +/// preparation or send authorization can begin. +#[derive(Clone, Debug)] +pub struct AttachmentPreparingModelCallProvider { + inner: Provider, + catalog: BlobCatalogRepository, + registry: Option>, +} + +impl AttachmentPreparingModelCallProvider { + /// Composes attachment preparation over one provider adapter. + pub fn new(inner: Provider, pool: PgPool, registry: Option>) -> Self { + Self { + inner, + catalog: BlobCatalogRepository::new(pool), + registry, + } + } +} + +impl ModelCallProvider for AttachmentPreparingModelCallProvider +where + Provider: ModelCallProvider + Send, + Provider::Capability: Send, +{ + type Capability = Provider::Capability; + type Error = Provider::Error; + + async fn prepare_capability( + &mut self, + operation: PreparedModelOperation, + cancellation: Cancellation, + ) -> Result, Self::Error> + where + Cancellation: Future + Send + 'static, + { + let digests = operation.attachment_digests().collect::>(); + if digests.is_empty() { + return self.inner.prepare_capability(operation, cancellation).await; + } + + let mut cancellation = Box::pin(cancellation); + let prepared = { + let preparation = prepare_attachments( + &self.catalog, + self.registry.as_deref(), + operation.request(), + digests, + ); + tokio::pin!(preparation); + tokio::select! { + biased; + () = &mut cancellation => { + return Ok(ModelCallCapabilityPreparation::Cancelled); + } + prepared = &mut preparation => prepared, + } + }; + if let Err(failure) = prepared { + return Ok(ModelCallCapabilityPreparation::AttachmentFailure(failure)); + } + self.inner.prepare_capability(operation, cancellation).await + } + + async fn invoke( + &mut self, + authorized: signalbox_domain::AuthorizedModelCall, + capability: Self::Capability, + acceptance_possible: AcceptancePossible, + cancellation: Cancellation, + ) -> Result + where + AcceptancePossible: FnOnce() + Send, + Cancellation: Future + Send + 'static, + { + self.inner + .invoke(authorized, capability, acceptance_possible, cancellation) + .await + } +} + +async fn prepare_attachments( + catalog: &BlobCatalogRepository, + registry: Option<&BlobStoreRegistry>, + request: &PreparedModelCallRequest, + digests: BTreeSet, +) -> Result<(), AttachmentPreparationFailure> { + let Some(registry) = registry else { + return Err(AttachmentPreparationFailure::Corrupt); + }; + let mut entries = Vec::with_capacity(digests.len()); + let mut total = 0_u64; + for digest in digests { + let entry = catalog + .find(digest) + .await + .map_err(map_catalog_failure)? + .ok_or(AttachmentPreparationFailure::Missing)?; + let expected = entry.expected(); + if request + .attachment_byte_length(digest) + .map(|length| length.get()) + != Some(expected.byte_length()) + { + return Err(AttachmentPreparationFailure::Corrupt); + } + total = total.checked_add(expected.byte_length()).ok_or( + AttachmentPreparationFailure::TooLarge { + maximum_bytes: registry.max_blob_bytes(), + }, + )?; + entries.push(entry); + } + if total > registry.max_blob_bytes() { + return Err(AttachmentPreparationFailure::TooLarge { + maximum_bytes: registry.max_blob_bytes(), + }); + } + for entry in &entries { + verify_entry(registry, entry).await?; + } + Ok(()) +} + +async fn verify_entry( + registry: &BlobStoreRegistry, + entry: &BlobCatalogEntry, +) -> Result<(), AttachmentPreparationFailure> { + let expected = entry.expected(); + let mut saw_missing = false; + let mut saw_corrupt = false; + let mut saw_unavailable = false; + for replica in entry.replicas() { + let Some(store) = registry.recorded_store(replica.store()) else { + return Err(AttachmentPreparationFailure::Corrupt); + }; + match store.open(replica.object_key()).await { + Ok(opened) => { + if opened.byte_length() != expected.byte_length() { + saw_corrupt = true; + continue; + } + match verify_stream(opened.into_reader(), expected).await { + Ok(()) => return Ok(()), + Err(StreamVerificationFailure::Corrupt) => saw_corrupt = true, + Err(StreamVerificationFailure::Unavailable) => saw_unavailable = true, + } + } + Err(error) => match error.kind() { + BlobStoreFailureKind::NotFound => saw_missing = true, + BlobStoreFailureKind::VerificationFailed => saw_corrupt = true, + BlobStoreFailureKind::Unavailable => saw_unavailable = true, + }, + } + } + if saw_unavailable { + Err(AttachmentPreparationFailure::Unavailable) + } else if saw_corrupt { + Err(AttachmentPreparationFailure::Corrupt) + } else if saw_missing || entry.replicas().is_empty() { + Err(AttachmentPreparationFailure::Missing) + } else { + Err(AttachmentPreparationFailure::Corrupt) + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum StreamVerificationFailure { + Corrupt, + Unavailable, +} + +async fn verify_stream( + mut reader: signalbox_blob_store::BlobReader, + expected: ExpectedBlob, +) -> Result<(), StreamVerificationFailure> { + let mut digest = Sha256::new(); + let mut observed = 0_u64; + let mut buffer = vec![0_u8; VERIFICATION_BUFFER_BYTES]; + loop { + let read = reader + .read(&mut buffer) + .await + .map_err(|_| StreamVerificationFailure::Unavailable)?; + if read == 0 { + break; + } + observed = observed + .checked_add(u64::try_from(read).map_err(|_| StreamVerificationFailure::Corrupt)?) + .ok_or(StreamVerificationFailure::Corrupt)?; + if observed > expected.byte_length() { + return Err(StreamVerificationFailure::Corrupt); + } + digest.update(&buffer[..read]); + } + let observed_digest = BlobDigest::from_bytes(digest.finalize().into()); + if observed == expected.byte_length() && observed_digest == expected.digest() { + Ok(()) + } else { + Err(StreamVerificationFailure::Corrupt) + } +} + +fn map_catalog_failure(error: BlobCatalogRepositoryError) -> AttachmentPreparationFailure { + match error { + BlobCatalogRepositoryError::Database(_) + | BlobCatalogRepositoryError::CommitAmbiguous(_) => { + AttachmentPreparationFailure::Unavailable + } + BlobCatalogRepositoryError::Corruption(_) => AttachmentPreparationFailure::Corrupt, + } +} + +#[cfg(test)] +mod tests { + use std::{io::Cursor, num::NonZeroU64}; + + use signalbox_blob_store::ExpectedBlob; + use signalbox_domain::BlobDigest; + + use super::{StreamVerificationFailure, verify_stream}; + + fn expected(bytes: &[u8]) -> ExpectedBlob { + ExpectedBlob::new( + BlobDigest::digest(bytes), + NonZeroU64::new(u64::try_from(bytes.len()).expect("fixture length fits u64")) + .expect("fixtures are nonempty"), + ) + } + + #[tokio::test] + async fn exact_stream_verifies_without_retaining_attachment_bytes() { + let bytes = b"canonical attachment bytes"; + + assert_eq!( + verify_stream(Box::new(Cursor::new(bytes.to_vec())), expected(bytes)).await, + Ok(()) + ); + } + + #[tokio::test] + async fn short_stream_is_corrupt() { + let bytes = b"canonical attachment bytes"; + + assert_eq!( + verify_stream( + Box::new(Cursor::new(bytes[..bytes.len() - 1].to_vec())), + expected(bytes), + ) + .await, + Err(StreamVerificationFailure::Corrupt) + ); + } + + #[tokio::test] + async fn long_stream_is_rejected_when_catalog_length_is_exceeded() { + let bytes = b"canonical attachment bytes"; + let mut longer = bytes.to_vec(); + longer.push(b'!'); + + assert_eq!( + verify_stream(Box::new(Cursor::new(longer)), expected(bytes)).await, + Err(StreamVerificationFailure::Corrupt) + ); + } + + #[tokio::test] + async fn equal_length_digest_mismatch_is_corrupt() { + let bytes = b"canonical attachment bytes"; + let different = b"canonical attachment byte!"; + assert_eq!(bytes.len(), different.len()); + + assert_eq!( + verify_stream(Box::new(Cursor::new(different.to_vec())), expected(bytes)).await, + Err(StreamVerificationFailure::Corrupt) + ); + } +} diff --git a/apps/signalboxd/src/bin/approval-judge-eval.rs b/apps/signalboxd/src/bin/approval-judge-eval.rs index 1b9d5edb80..50bc9f4ecf 100644 --- a/apps/signalboxd/src/bin/approval-judge-eval.rs +++ b/apps/signalboxd/src/bin/approval-judge-eval.rs @@ -27,6 +27,11 @@ use signalbox_model_provider_runtime::{ use signalbox_model_runtime::CredentialReference; use signalbox_model_runtime_anthropic::{AnthropicConfig, AnthropicRuntime}; use signalbox_model_runtime_openai::{OpenAiConfig, OpenAiRuntime}; +use signalbox_persistence::approval_judge_eval::{ + APPROVAL_JUDGE_EVAL_CASE_CATEGORIES, APPROVAL_JUDGE_EVAL_SCORING_SEMANTICS_VERSION, + ApprovalJudgeEvalCallRecord, ApprovalJudgeEvalRecordingSchema, ApprovalJudgeEvalRunId, + ApprovalJudgeEvalRunRecord, record_eval_run, verify_recording_schema, +}; use signalboxd::{ CredentialDelivery, DaemonToolCatalog, DaemonToolComposition, FileCredentialAccess, HubModelConfiguration, ModelAdapter, @@ -35,7 +40,7 @@ use signalboxd::{ ApprovalJudgeEvalVerdict, judge_eval_case, judge_system_prompt, render_eval_case, }, model_adapter::ConfiguredModelRuntime, - usage_limits, + provider_reported_usage, usage_limits, }; fn help_text() -> String { @@ -54,6 +59,13 @@ Options: --repeats Judge calls per case, n >= 1. Default 3; repeats measure verdict stability. --filter Keep only cases whose name or category contains . Default: all cases. --limit Stop after selecting n cases, n >= 1. Default: no bound. + --database-url + Also record the run and each verdict in the named PostgreSQL + database's eval-owned tables. Default: stdout scorecard only. + --database-url-env + Like --database-url, but read the URL from the named + environment variable, keeping a password-bearing URL out of + the process argument vector and shell history. --help Print this reference and exit without spending quota." ) } @@ -65,8 +77,6 @@ const MAX_PAID_CALLS: usize = 1_000; /// Bumped whenever the majority, tie, or stability algorithms change, so /// before/after scorecards with identical replay metadata still declare /// which analysis produced their summaries. -const SCORING_SEMANTICS_VERSION: u32 = 3; - /// Closed scorecard grouping; deserialization is the single source of truth, /// so an unknown spelling fails the corpus load and a new variant fails /// compilation anywhere a match is not exhaustive. @@ -85,8 +95,8 @@ enum CaseCategory { } impl CaseCategory { - const fn as_str(self) -> &'static str { - match self { + fn as_str(self) -> &'static str { + let category = match self { Self::GitPush => "git_push", Self::ThreadOps => "thread_ops", Self::NetworkEgress => "network_egress", @@ -96,7 +106,9 @@ impl CaseCategory { Self::InjectionResistance => "injection_resistance", Self::ContextAbsent => "context_absent", Self::UndecodableArguments => "undecodable_arguments", - } + }; + debug_assert!(APPROVAL_JUDGE_EVAL_CASE_CATEGORIES.contains(&category)); + category } } @@ -159,6 +171,7 @@ struct RunOptions { repeats: usize, filter: Option, limit: Option, + database_url: Option, } enum ParsedArguments { @@ -166,12 +179,20 @@ enum ParsedArguments { Help, } +struct EvalRecording { + schema: ApprovalJudgeEvalRecordingSchema, + repeats: u32, + usage_input_includes_cache_tokens: bool, +} + fn parse_arguments() -> Result { let mut configuration = None; let mut cases = None; let mut repeats = 3_usize; let mut filter = None; let mut limit = None; + let mut database_url = None; + let mut database_url_from_environment = None; let mut arguments = env::args().skip(1); while let Some(flag) = arguments.next() { let mut value = |flag: &str| { @@ -194,6 +215,19 @@ fn parse_arguments() -> Result { } } "--filter" => filter = Some(value("--filter")?), + "--database-url" => database_url = Some(value("--database-url")?), + "--database-url-env" => { + let variable = value("--database-url-env")?; + let url = env::var(&variable).map_err(|_| { + format!("--database-url-env names {variable}, which is unset or not text") + })?; + if url.is_empty() { + return Err(format!( + "--database-url-env names {variable}, which is empty" + )); + } + database_url_from_environment = Some(url); + } "--limit" => { let bound: usize = value("--limit")? .parse() @@ -208,12 +242,18 @@ fn parse_arguments() -> Result { other => return Err(format!("unknown flag: {other}")), } } + if database_url.is_some() && database_url_from_environment.is_some() { + return Err(String::from( + "--database-url and --database-url-env both name a recording database; pass one", + )); + } Ok(ParsedArguments::Run(RunOptions { configuration: configuration.ok_or_else(|| String::from("--config is required"))?, cases: cases.ok_or_else(|| String::from("--cases is required"))?, repeats, filter, limit, + database_url: database_url.or(database_url_from_environment), })) } @@ -324,7 +364,7 @@ fn render_scorecard( .sum::(), "failed_calls": scores.values().map(|score| score.failed_calls).sum::(), "escalation_calibration": escalation, - "scoring_semantics_version": SCORING_SEMANTICS_VERSION, + "scoring_semantics_version": APPROVAL_JUDGE_EVAL_SCORING_SEMANTICS_VERSION, "categories": categories, "cases": case_reports, }); @@ -507,6 +547,44 @@ async fn run(options: RunOptions) -> Result<(), String> { }) .ok_or_else(|| String::from("approval_judge target has no runtime model definition"))?; + // Recording admission, the database connection, and the schema check all + // resolve before the first paid call, so neither an oversized --repeats + // nor an unreachable or unmigrated database can surface only after quota + // is already spent. + let recording = match &options.database_url { + Some(database_url) => { + let repeats = u32::try_from(options.repeats).map_err(|_| { + String::from("--repeats exceeds the range --database-url recording stores") + })?; + // Both strings are persisted as text and inside the scorecard + // jsonb, neither of which admits U+0000, and configuration + // admission does not reject it there. + if provider_model.contains('\u{0}') || binding.credential_reference.contains('\u{0}') { + return Err(String::from( + "the resolved provider model or credential reference contains U+0000, \ + which --database-url recording cannot store", + )); + } + let pool = signalbox_persistence::connect_production(database_url) + .await + .map_err(|error| format!("database connection failed: {error}"))?; + // The eval tables must already exist: schema application belongs + // to the daemon, and a measurement tool never migrates a live + // database out from under it. + let schema = verify_recording_schema(&pool) + .await + .map_err(|error| format!("database recording is unavailable: {error}"))?; + Some(EvalRecording { + schema, + repeats, + usage_input_includes_cache_tokens: configuration + .cache_inclusive_input_targets() + .contains(&binding.target), + }) + } + None => None, + }; + let corpus = fs::read_to_string(&options.cases) .map_err(|error| format!("corpus read failed: {error}"))?; let digest = stable_digest(corpus.as_bytes()); @@ -534,6 +612,28 @@ async fn run(options: RunOptions) -> Result<(), String> { { continue; } + // The recording schema stores case names non-empty, and PostgreSQL + // text and jsonb admit no U+0000, so a selected case recording + // cannot store must fail here, before any paid call, rather than + // after the whole run's quota is spent. Name and notes are the only + // persisted case fields this can reach: category and expected are + // closed sets, tool names admit no control characters, and the other + // fields are persisted only as digests. Without --database-url the + // corpus admission is unchanged. + if recording.is_some() { + let name_storable = !case.name.is_empty() && !case.name.contains('\u{0}'); + let notes_storable = case + .notes + .as_deref() + .is_none_or(|notes| !notes.contains('\u{0}')); + if !name_storable || !notes_storable { + return Err(format!( + "corpus line {} carries a name or notes --database-url recording cannot \ + store: names are non-empty and neither field may contain U+0000", + index + 1 + )); + } + } cases.push(case); } if cases.is_empty() { @@ -674,11 +774,16 @@ async fn run(options: RunOptions) -> Result<(), String> { let mut scores: BTreeMap = BTreeMap::new(); let mut case_reports = Vec::new(); + let mut recorded_calls: Vec = Vec::new(); for (case, eval_case) in cases.iter().zip(&eval_cases) { let mut verdicts: Vec = Vec::new(); let mut failures = 0_usize; + // Counts every attempt, so a failed call leaves a gap in the recorded + // ordinals rather than shifting later verdicts onto its position. + let mut attempt_ordinal = 0_u32; let mut failure_causes: Vec = Vec::new(); for _ in 0..options.repeats { + attempt_ordinal = attempt_ordinal.saturating_add(1); match judge_eval_case(&model, &binding, eval_case).await { Ok(verdict) => { // The daemon rejects verdicts whose reported usage exceeds @@ -694,7 +799,25 @@ async fn run(options: RunOptions) -> Result<(), String> { let cause = String::from("reported usage exceeds configured limits"); eprintln!("call failed for {}: {cause}", case.name); failure_causes.push(cause); + } else if recording.is_some() + && !recording_rationale_is_storable(&verdict.rationale) + { + failures += 1; + let cause = String::from( + "provider rationale contains U+0000, which database recording cannot store", + ); + eprintln!("call failed for {}: {cause}", case.name); + failure_causes.push(cause); } else { + if recording.is_some() { + recorded_calls.push(ApprovalJudgeEvalCallRecord { + case_name: case.name.clone(), + repeat_ordinal: attempt_ordinal, + recommendation: verdict.recommendation, + rationale: verdict.rationale.clone(), + usage: provider_reported_usage(verdict.usage), + }); + } verdicts.push(verdict); } } @@ -771,7 +894,11 @@ async fn run(options: RunOptions) -> Result<(), String> { "repeats": verdicts.iter().map(|verdict| serde_json::json!({ "recommendation": recommendation_label(verdict.recommendation), "rationale": verdict.rationale, - "provider_reported_model": verdict.provider_reported_model, + "provider_reported_model": if recording.is_some() { + storable_provider_reported_model(verdict.provider_reported_model.as_deref()) + } else { + verdict.provider_reported_model.clone() + }, })).collect::>(), "notes": case.notes, })); @@ -780,23 +907,87 @@ async fn run(options: RunOptions) -> Result<(), String> { let rendered = render_scorecard( ScorecardMetadata { judge_selection: selection.into_uuid().to_string(), - provider_model, - corpus_digest: digest, - contract_digest, - rendered_digest, + provider_model: provider_model.clone(), + corpus_digest: digest.clone(), + contract_digest: contract_digest.clone(), + rendered_digest: rendered_digest.clone(), repeats: options.repeats, speculative_tools, }, &scores, case_reports, )?; + let scorecard = serde_json::from_str(&rendered) + .map_err(|error| format!("scorecard parsing failed: {error}"))?; println!("{rendered}"); + // Recording follows the print, so a database failure can cost only the + // stored copy and never the primary stdout artifact. + if let Some(recording) = recording { + let run = ApprovalJudgeEvalRunRecord { + run: ApprovalJudgeEvalRunId::from_uuid(uuid::Uuid::now_v7()), + selection, + target: binding.target, + provider_model, + credential_reference: binding.credential_reference.clone(), + usage_input_includes_cache_tokens: recording.usage_input_includes_cache_tokens, + corpus_digest: digest, + contract_digest, + rendered_digest, + repeats: recording.repeats, + scorecard, + }; + let run_identity = run.run.into_uuid(); + // The identity is announced before the commit is attempted, so an + // ambiguous commit outcome still leaves the exact key to query for. + eprintln!( + "recording eval run {run_identity} holding {} calls", + recorded_calls.len() + ); + record_eval_run(&recording.schema, &run, &recorded_calls) + .await + .map_err(|error| { + format!("database recording failed for eval run {run_identity}: {error}") + })?; + eprintln!("recorded eval run {run_identity}"); + } Ok(()) } +/// PostgreSQL JSONB cannot represent U+0000. Provider-controlled model text +/// containing it is encoded as a versioned UTF-8 hex string; ordinary model +/// text remains unchanged, and the prefix makes decoding unambiguous. +fn storable_provider_reported_model(model: Option<&str>) -> Option { + const ENCODED_PREFIX: &str = "signalbox:utf8-hex-v1:"; + let model = model?; + if !model.contains('\u{0}') && !model.starts_with(ENCODED_PREFIX) { + return Some(String::from(model)); + } + let mut encoded = String::with_capacity(ENCODED_PREFIX.len() + model.len() * 2); + encoded.push_str(ENCODED_PREFIX); + let hex_digit = |nibble: u8| { + char::from(if nibble < 10 { + b'0' + nibble + } else { + b'a' + (nibble - 10) + }) + }; + for byte in model.as_bytes() { + encoded.push(hex_digit(byte >> 4)); + encoded.push(hex_digit(byte & 0x0f)); + } + Some(encoded) +} + +fn recording_rationale_is_storable(rationale: &str) -> bool { + !rationale.contains('\u{0}') +} + #[cfg(test)] mod tests { - use super::{MAX_PAID_CALLS, paid_call_count}; + use super::{ + MAX_PAID_CALLS, paid_call_count, recording_rationale_is_storable, + storable_provider_reported_model, + }; #[test] fn paid_call_count_accepts_the_safety_ceiling() { @@ -812,4 +1003,42 @@ mod tests { fn paid_call_count_rejects_arithmetic_overflow() { assert!(paid_call_count(usize::MAX, 2).is_err()); } + + #[test] + fn provider_reported_model_with_nul_is_reversibly_encoded() { + assert_eq!( + storable_provider_reported_model(Some("model\u{0}revision")), + Some(String::from( + "signalbox:utf8-hex-v1:6d6f64656c007265766973696f6e" + )) + ); + } + + #[test] + fn ordinary_provider_reported_model_is_unchanged() { + assert_eq!( + storable_provider_reported_model(Some("provider/model\\revision")), + Some(String::from("provider/model\\revision")) + ); + } + + #[test] + fn provider_reported_model_using_encoding_prefix_is_escaped() { + assert_eq!( + storable_provider_reported_model(Some("signalbox:utf8-hex-v1:literal")), + Some(String::from( + "signalbox:utf8-hex-v1:7369676e616c626f783a757466382d6865782d76313a6c69746572616c" + )) + ); + } + + #[test] + fn provider_rationale_with_nul_is_not_storable_for_recording() { + assert!(!recording_rationale_is_storable("because\u{0}details")); + } + + #[test] + fn ordinary_provider_rationale_is_storable_for_recording() { + assert!(recording_rationale_is_storable("because details")); + } } diff --git a/apps/signalboxd/src/bin/signalbox-debug.rs b/apps/signalboxd/src/bin/signalbox-debug.rs index 97e5d27cf0..5f3cfe487e 100644 --- a/apps/signalboxd/src/bin/signalbox-debug.rs +++ b/apps/signalboxd/src/bin/signalbox-debug.rs @@ -42,7 +42,8 @@ use signalbox_persistence::{ use signalboxd::{ ActivatedTurnPass, FatalExecutionSignal, FatalExecutionSupervisor, FileCredentialAccess, HubModelConfiguration, ModelAdapter, PostgresProviderModelExecution, - PostgresScriptedModelExecution, + PostgresScriptedModelExecution, WorkspaceInstructionPreparedExecution, + WorkspaceInstructionRuntime, }; use sqlx::postgres::PgPoolOptions; use tokio::{ @@ -286,7 +287,7 @@ async fn poll_terminal_transcript( loop { let rows = sqlx::query_as::<_, TranscriptRow>( "SELECT entry.payload_kind, - accepted.content_text, + accepted_part.text_value, entry.assistant_text_value FROM turn_lifecycle AS lifecycle JOIN context_frontier_member AS member @@ -298,6 +299,10 @@ async fn poll_terminal_transcript( LEFT JOIN accepted_input AS accepted ON accepted.session_id = entry.source_session_id AND accepted.accepted_input_id = entry.origin_accepted_input_id + LEFT JOIN accepted_input_content_part AS accepted_part + ON accepted_part.accepted_input_id = accepted.accepted_input_id + AND accepted_part.position = 0 + AND accepted_part.part_kind = 'text' WHERE lifecycle.session_id = $1 AND lifecycle.turn_id = $2 AND lifecycle.state_kind = 'terminal' @@ -407,6 +412,7 @@ async fn run(arguments: DebugArguments) -> Result<(), DebugDriverError> { credential_pin, credential_families, automatic_tool_round_limit, + instruction_roots, provider, ) = match provider { DebugProvider::Scripted { reply } => { @@ -431,6 +437,7 @@ async fn run(arguments: DebugArguments) -> Result<(), DebugDriverError> { // `None` is what selects the fallback reference. None, None, + Vec::new(), DebugProviderRuntime::Scripted( AssistantText::try_new(reply).map_err(|_| DebugDriverError::InvalidText)?, ), @@ -489,6 +496,7 @@ async fn run(arguments: DebugArguments) -> Result<(), DebugDriverError> { configuration.runtime_model_catalog(), diagnostic_model_identity_limit, ); + let instruction_roots = configuration.workspace_instructions().roots().to_vec(); ( selection, configuration.target_catalog(), @@ -496,6 +504,7 @@ async fn run(arguments: DebugArguments) -> Result<(), DebugDriverError> { configuration.session_credential_pin(), Some(configuration.credential_family_catalog()), automatic_tool_round_limit, + instruction_roots, DebugProviderRuntime::Anthropic(provider), ) } @@ -575,13 +584,18 @@ async fn run(arguments: DebugArguments) -> Result<(), DebugDriverError> { UuidV7StartEligibleTurnIdGenerator, StartEligibleTurnRepository::new(pool.clone()), ); + let workspace_instructions = + WorkspaceInstructionRuntime::new(pool.clone(), None, instruction_roots); let transcript = match provider { DebugProviderRuntime::Scripted(reply) => { let (execution, fatal_execution) = - FatalExecutionSupervisor::new(PostgresScriptedModelExecution::new( - repository, - InProcessAttemptDispatchGate::default(), - reply, + FatalExecutionSupervisor::new(WorkspaceInstructionPreparedExecution::new( + PostgresScriptedModelExecution::new( + repository, + InProcessAttemptDispatchGate::default(), + reply, + ), + workspace_instructions, )); let (pass, pass_failure) = ObservableDebugPass::new(ActivatedTurnPass::new(activation, execution)); @@ -597,11 +611,14 @@ async fn run(arguments: DebugArguments) -> Result<(), DebugDriverError> { } DebugProviderRuntime::Anthropic(provider) => { let (execution, fatal_execution) = - FatalExecutionSupervisor::new(PostgresProviderModelExecution::new( - repository, - InProcessAttemptDispatchGate::default(), - provider, - automatic_tool_round_limit, + FatalExecutionSupervisor::new(WorkspaceInstructionPreparedExecution::new( + PostgresProviderModelExecution::new( + repository, + InProcessAttemptDispatchGate::default(), + provider, + automatic_tool_round_limit, + ), + workspace_instructions, )); let (pass, pass_failure) = ObservableDebugPass::new(ActivatedTurnPass::new(activation, execution)); diff --git a/apps/signalboxd/src/blob_storage_runtime.rs b/apps/signalboxd/src/blob_storage_runtime.rs index ac52cf0111..2155924f2a 100644 --- a/apps/signalboxd/src/blob_storage_runtime.rs +++ b/apps/signalboxd/src/blob_storage_runtime.rs @@ -198,7 +198,7 @@ fn open_blob_root( OpenedFilesystemBlobRoot::open_without_locality_check_for_test(root) } #[cfg(not(feature = "test-support"))] - unreachable!("production initialization always requires local backing") + OpenedFilesystemBlobRoot::open(root) } #[derive(Clone, Debug)] diff --git a/apps/signalboxd/src/configuration.rs b/apps/signalboxd/src/configuration.rs index 795e4af18f..9883e1dc5f 100644 --- a/apps/signalboxd/src/configuration.rs +++ b/apps/signalboxd/src/configuration.rs @@ -15,16 +15,17 @@ use std::{ use rust_decimal::Decimal; use signalbox_domain::{ AnthropicServiceTier, BranchName, CheckConclusion, CodexCliServiceTier, DirectModelSelection, - FastMode, FastModeOverlay, FastModeSupport, FrozenAliasDefinition, LabelName, MergeableState, - ModelAlias, ModelCapabilities, ModelCapabilityCatalog, ModelCapabilityDefinition, - ModelSelectionRequest, ModelSettingsOverlay, ModelSettingsPrecedence, ModelTargetCatalog, - ModelTargetDefinition, OpenAiServiceTier, ProviderModelIdentity, PullRequestNumber, - ReasoningLevel, RepoWatchAuthorLogin, RepoWatchEventKindNameV1, RepoWatchLabelMatcher, - RepoWatchLabelMatcherInput, RepoWatchMatcherV1, RepoWatchMatcherV1Input, RepoWatchPattern, - RepoWatchRule, RepoWatchRuleActionV1, RepoWatchRuleId, RepoWatchRuleVersion, - RepoWatchSingletonScope, RepoWatchTemplateContextDeclaration, RepositorySlug, - ResolvedProviderTarget, ServiceTier, SessionTemplateName, SettingOverlay, ToolApprovalPosture, - ToolName, UnsupportedModelSetting, ValidatedModelSettings, + FastMode, FastModeOverlay, FastModeSupport, FrozenAliasDefinition, InstructionPath, LabelName, + MergeableState, ModelAlias, ModelCapabilities, ModelCapabilityCatalog, + ModelCapabilityDefinition, ModelSelectionRequest, ModelSettingsOverlay, + ModelSettingsPrecedence, ModelTargetCatalog, ModelTargetDefinition, OpenAiServiceTier, + ProviderModelIdentity, PullRequestNumber, ReasoningLevel, RepoWatchAuthorLogin, + RepoWatchEventKindNameV1, RepoWatchLabelMatcher, RepoWatchLabelMatcherInput, + RepoWatchMatcherV1, RepoWatchMatcherV1Input, RepoWatchPattern, RepoWatchRule, + RepoWatchRuleActionV1, RepoWatchRuleId, RepoWatchRuleVersion, RepoWatchSingletonScope, + RepoWatchTemplateContextDeclaration, RepositorySlug, ResolvedProviderTarget, ServiceTier, + SessionTemplateName, SettingOverlay, ToolApprovalPosture, ToolName, UnsupportedModelSetting, + ValidatedModelSettings, }; use signalbox_model_provider_runtime::{RuntimeModelCatalog, RuntimeModelDefinition}; use signalbox_model_runtime::{ @@ -174,7 +175,7 @@ impl ModelAdapter { match self { Self::Anthropic | Self::OpenAi => matches!(delivery, "file"), Self::ClaudeCli => matches!(delivery, "ambient" | "file"), - Self::CodexCli => matches!(delivery, "ambient"), + Self::CodexCli => matches!(delivery, "ambient" | "codex_home"), } } @@ -433,6 +434,19 @@ pub struct DaemonToolConfiguration { cargo_registry_cache: Option, } +/// Explicit non-workspace instruction roots registered by deployment configuration. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct WorkspaceInstructionConfiguration { + roots: Box<[InstructionPath]>, +} + +impl WorkspaceInstructionConfiguration { + /// Returns explicit roots in deterministic configuration order. + pub fn roots(&self) -> &[InstructionPath] { + &self.roots + } +} + impl DaemonToolConfiguration { /// Absolute root pinned into both workspace tool families. pub fn workspace_root(&self) -> &Path { @@ -714,6 +728,16 @@ const REQUIRED_NUMERIC_BOUNDS: &[(&str, NumericBoundKind)] = &[ "repository_reconciliation_quantum", NumericBoundKind::Integer, ), + ("webhook_drain_work_budget", NumericBoundKind::Duration), + ("fenced_pool_min_connections", NumericBoundKind::Integer), + ( + "fenced_pool_floor_reconciliation_interval", + NumericBoundKind::Duration, + ), + ( + "fenced_pool_floor_reconciliation_attempt_bound", + NumericBoundKind::Duration, + ), ("max_concurrent_snapshot_readers", NumericBoundKind::Integer), ("max_blob_replica_count", NumericBoundKind::Integer), ("max_session_metadata_tags", NumericBoundKind::Integer), @@ -740,6 +764,7 @@ const REQUIRED_NUMERIC_BOUNDS: &[(&str, NumericBoundKind)] = &[ NumericBoundKind::Duration, ), ("model_exchange_timeout", NumericBoundKind::Duration), + ("codex_cli_version_probe_bound", NumericBoundKind::Duration), ("expired_pass_recovery_attempts", NumericBoundKind::Integer), ( "expired_pass_recovery_attempt_bound", @@ -984,6 +1009,7 @@ pub struct HubModelConfiguration { approval_judge_selection: Option, repository_watch: Option, blob_storage: Option, + workspace_instructions: WorkspaceInstructionConfiguration, } #[derive(Clone, Copy, Debug, Eq, PartialEq)] @@ -1028,6 +1054,7 @@ impl HubModelConfiguration { "approval_judge", "repository_watch", "blob_storage", + "workspace_instructions", ], )?; if document.get("version").and_then(|item| item.as_integer()) != Some(1) { @@ -1120,6 +1147,8 @@ impl HubModelConfiguration { .get("repository_watch") .map(|item| parse_repository_watch_configuration(item, &numeric_bounds)) .transpose()?; + let workspace_instructions = + parse_workspace_instruction_configuration(document.get("workspace_instructions"))?; let models = document .get("models") .and_then(|item| item.as_array_of_tables()) @@ -1184,21 +1213,9 @@ impl HubModelConfiguration { continue; } }; - // Codex still carries one credential reference into its runtime, - // so two families preferring different profiles cannot both be - // served. Claude now receives the complete adapter-scoped catalog - // and resolves each operation's pinned reference, so differing - // preferences are admitted; the retained value is only the - // runtime's default for an operation that pins nothing. - if adapter == ModelAdapter::CodexCli - && adapter_profile - .as_ref() - .is_some_and(|profile| profile != &credential_profile) - { - return Err( - HubModelConfigurationError::ConflictingAdapterCredentialProfiles { adapter }, - ); - } + // CLI runtimes receive their complete adapter-scoped delivery + // catalogs. The retained value is only the default for an ambient + // operation that pins no catalog member. adapter_profile.get_or_insert_with(|| Arc::clone(&credential_profile)); let entry = AdapterMapping { adapter, @@ -1667,6 +1684,7 @@ impl HubModelConfiguration { approval_judge_selection, repository_watch, blob_storage, + workspace_instructions, }) } @@ -1951,6 +1969,14 @@ impl HubModelConfiguration { post_kill_reap_bound, ); runtime_configuration.exchange_timeout = model_exchange_timeout; + runtime_configuration = runtime_configuration.with_credential_homes( + self.credential_profiles.values().filter_map(|profile| { + let CredentialDelivery::CodexHome { path, .. } = profile.delivery() else { + return None; + }; + Some((CredentialReference::new(profile.name()), path.to_path_buf())) + }), + ); runtime_configuration.model_capabilities = self.runtime_model_capability_catalog(); CodexCliRuntime::new(runtime_configuration) }) @@ -2086,6 +2112,11 @@ impl HubModelConfiguration { self.daemon_tools.as_ref() } + /// Returns explicit roots whose content is discoverable but not eligible by default. + pub const fn workspace_instructions(&self) -> &WorkspaceInstructionConfiguration { + &self.workspace_instructions + } + /// Reports whether the configuration contains one direct selection key. pub fn contains_selection(&self, selection: DirectModelSelection) -> bool { self.direct_selections.contains(&selection) @@ -2125,6 +2156,47 @@ pub(crate) fn checked_in_example_configuration() )) } +fn parse_workspace_instruction_configuration( + item: Option<&Item>, +) -> Result { + let Some(item) = item else { + return Ok(WorkspaceInstructionConfiguration { + roots: Box::new([]), + }); + }; + let table = item + .as_table() + .ok_or(HubModelConfigurationError::InvalidWorkspaceInstructionConfiguration)?; + reject_unknown_fields(table, &["version", "registered_roots"]) + .map_err(|_| HubModelConfigurationError::InvalidWorkspaceInstructionConfiguration)?; + if table.get("version").and_then(Item::as_integer) != Some(1) { + return Err(HubModelConfigurationError::InvalidWorkspaceInstructionConfiguration); + } + let values = table + .get("registered_roots") + .and_then(Item::as_array) + .ok_or(HubModelConfigurationError::InvalidWorkspaceInstructionConfiguration)?; + if values.len() > 64 { + return Err(HubModelConfigurationError::InvalidWorkspaceInstructionConfiguration); + } + let mut roots = Vec::with_capacity(values.len()); + let mut unique = BTreeSet::new(); + for value in values { + let value = value + .as_str() + .ok_or(HubModelConfigurationError::InvalidWorkspaceInstructionConfiguration)?; + let root = InstructionPath::try_new(value.to_owned()) + .map_err(|_| HubModelConfigurationError::InvalidWorkspaceInstructionConfiguration)?; + if !unique.insert(root.clone()) { + return Err(HubModelConfigurationError::InvalidWorkspaceInstructionConfiguration); + } + roots.push(root); + } + Ok(WorkspaceInstructionConfiguration { + roots: roots.into_boxed_slice(), + }) +} + fn parse_repository_watch_configuration( item: &Item, numeric_bounds: &NumericBoundsConfiguration, @@ -2375,6 +2447,7 @@ fn parse_convergence_pull_requests( .as_integer() .and_then(|value| u64::try_from(value).ok()) .and_then(NonZeroU64::new) + .filter(|value| value.get() <= i32::MAX as u64) .map(PullRequestNumber::new) .ok_or(HubModelConfigurationError::InvalidRepositoryWatchConfiguration)?; if parsed.contains(&number) { @@ -3202,9 +3275,11 @@ fn validate_github_tool_mapping(mapping: &Table) -> Result<(), HubModelConfigura } fn validate_workspace_tool_mapping(mapping: &Table) -> Result<(), HubModelConfigurationError> { - let root = Path::new(required_string(mapping, "workspace_root")?); + let root_value = required_string(mapping, "workspace_root")?; + let root = Path::new(root_value); if required_string(mapping, "adapter")? != "local" || !root.is_absolute() + || InstructionPath::try_new(root_value.to_owned()).is_err() || mapping.get("credential_profile").is_some() || mapping.get("egress_policy").is_some() { @@ -3730,6 +3805,13 @@ pub enum HubModelConfigurationError { /// A credential profile named no delivery, or its delivery's own fields /// were absent or malformed. InvalidCredentialDelivery, + /// One member's Codex home failed path/directory admission. + InvalidCredentialHome { + /// Non-secret profile reference identifying the failed member. + credential_profile: Arc, + /// Closed startup failure class; never path or auth material. + failure: crate::credential_pools::CredentialHomeAdmissionFailure, + }, /// A credential profile named a delivery its adapter does not admit. UnsupportedCredentialDelivery { /// Build-provided adapter whose admitted deliveries were checked. @@ -3888,6 +3970,8 @@ pub enum HubModelConfigurationError { InvalidWebFetchPolicy, /// The optional version-one repository-watch section was malformed. InvalidRepositoryWatchConfiguration, + /// The optional version-one workspace-instruction section was malformed. + InvalidWorkspaceInstructionConfiguration, /// The convergence sweep names no loaded session template. UnknownConvergenceSweepTemplate { /// Exact missing template name. @@ -3956,6 +4040,20 @@ impl fmt::Display for HubModelConfigurationError { "model configuration names unknown convergence template `{template}`" ); } + // Startup telemetry formats this value, so the failing member and the + // closed admission cause must both survive. The path never appears, as + // `configuration-and-credentials.md#the-codex_home-delivery` requires. + if let Self::InvalidCredentialHome { + credential_profile, + failure, + } = self + { + return write!( + formatter, + "model configuration credential profile `{credential_profile}` names an unavailable Codex credential home: {}", + failure.cause() + ); + } formatter.write_str(match self { Self::Read => "model configuration file could not be read", Self::InvalidDocument => "model configuration is not valid TOML", @@ -3992,6 +4090,9 @@ impl fmt::Display for HubModelConfigurationError { Self::InvalidCredentialDelivery => { "model configuration contains an invalid credential delivery" } + Self::InvalidCredentialHome { .. } => { + "model configuration contains an unavailable Codex credential home" + } Self::UnsupportedCredentialDelivery { .. } => { "model configuration names a credential delivery its adapter does not admit" } @@ -4107,6 +4208,9 @@ impl fmt::Display for HubModelConfigurationError { Self::InvalidRepositoryWatchConfiguration => { "model configuration contains invalid repository-watch settings" } + Self::InvalidWorkspaceInstructionConfiguration => { + "model configuration contains invalid workspace-instruction settings" + } Self::UnknownConvergenceSweepTemplate { .. } => { "model configuration names an unknown convergence template" } @@ -4398,11 +4502,16 @@ members = [{ profile = "codex-subscription-primary", priority = 1 }]"#; const EAGER_WATCH_RULE_ID: &str = "merge-forward-on-base-advance"; const EAGER_WATCH_HEAD_PATTERN: &str = "^agent/.+$"; const WATCH_TEMPLATE: &str = "merge-forward"; + const REGISTERED_INSTRUCTION_ROOT: &str = "/srv/signalbox/instruction-library"; const CONFIGURATION: &str = r#" version = 1 [numeric_bounds] repository_reconciliation_quantum = 16 +webhook_drain_work_budget = "45s" +fenced_pool_min_connections = 48 +fenced_pool_floor_reconciliation_interval = "5s" +fenced_pool_floor_reconciliation_attempt_bound = "30s" max_concurrent_snapshot_readers = 8 max_blob_replica_count = 32 max_session_metadata_tags = 256 @@ -4414,6 +4523,7 @@ max_review_orchestration_concerns = 32 max_imported_conversation_display_title_scalars = 256 graceful_shutdown_cleanup_window = "30s" model_exchange_timeout = "600s" +codex_cli_version_probe_bound = "10s" expired_pass_recovery_attempts = 4 expired_pass_recovery_attempt_bound = "3s" expired_pass_recovery_lock_retry_delay = "6s" @@ -4964,7 +5074,6 @@ template = "{WATCH_TEMPLATE}" .expect("configured judge fixture UUID is valid"), ) } - #[test] fn configured_tool_postures_are_typed() { let configured = HubModelConfiguration::parse(&format!( @@ -4989,7 +5098,6 @@ template = "{WATCH_TEMPLATE}" assert_eq!(postures[2].0.as_str(), WEB_FETCH_NAME); assert_eq!(postures[2].1, ToolApprovalPosture::Human); } - #[test] fn configured_judge_selection_is_typed() { let configured = HubModelConfiguration::parse(&format!( @@ -5650,6 +5758,19 @@ cool_off_seconds = {} ); } + #[test] + fn repository_watch_rejects_a_convergence_pull_request_above_graphql_int() { + let configured = configuration_with_convergence_sweep().replace( + &format!("convergence_pull_requests = [{CONVERGENCE_PULL_REQUEST}]"), + &format!("convergence_pull_requests = [{}]", i64::from(i32::MAX) + 1), + ); + + assert_eq!( + HubModelConfiguration::parse(&configured).err(), + Some(HubModelConfigurationError::InvalidRepositoryWatchConfiguration) + ); + } + #[test] fn repository_watch_accepts_a_positive_rule_revision() { let revision = @@ -6600,6 +6721,27 @@ context_window_tokens = 200000 ); } + #[test] + fn tool_mapping_registry_rejects_noncanonical_workspace_root_spellings() { + let trailing_separator = CONFIGURATION.replace( + "workspace_root = \"/srv/signalbox/workspace\"", + "workspace_root = \"/srv/signalbox/workspace/\"", + ); + let dot_component = CONFIGURATION.replace( + "workspace_root = \"/srv/signalbox/workspace\"", + "workspace_root = \"/srv/signalbox/./workspace\"", + ); + + assert_eq!( + HubModelConfiguration::parse(&trailing_separator).err(), + Some(HubModelConfigurationError::InvalidToolMappings) + ); + assert_eq!( + HubModelConfiguration::parse(&dot_component).err(), + Some(HubModelConfigurationError::InvalidToolMappings) + ); + } + #[test] fn tool_mapping_registry_requires_git_identity() { let missing = CONFIGURATION.replace( @@ -7447,59 +7589,124 @@ members = [{ profile = "anthropic-primary", priority = 1, weight = 3 }]"#, } #[test] - fn configuration_rejects_a_delivery_this_build_supplies_no_surface_for() { + fn configuration_admits_an_existing_nonempty_credential_home() { + let temporary = tempfile::tempdir().expect("synthetic home root is created"); + let home = temporary.path().join("account-a"); + std::fs::create_dir(&home).expect("synthetic home is created"); + std::fs::write(home.join("fixture-marker"), "synthetic") + .expect("synthetic home is nonempty"); + let credential_home = CONFIGURATION.replace( + "delivery = \"ambient\"", + &format!( + "delivery = \"codex_home\"\ncodex_home = {:?}", + home.to_string_lossy() + ), + ); + + let parsed = HubModelConfiguration::parse(&credential_home) + .expect("existing nonempty synthetic home is admitted"); + assert_eq!( + parsed + .credential_profile(CODEX_SUBSCRIPTION_PROFILE) + .expect("Codex profile remains present") + .delivery() + .path(), + Some(&home) + ); + } + + #[test] + fn configuration_rejects_a_relative_credential_home_with_a_typed_member_error() { let credential_home = CONFIGURATION.replace( "delivery = \"ambient\"", - "delivery = \"codex_home\"\ncodex_home = \"/var/lib/signalbox/codex/account-a\"", + "delivery = \"codex_home\"\ncodex_home = \"relative/account-a\"", ); assert_eq!( HubModelConfiguration::parse(&credential_home).err(), - Some(HubModelConfigurationError::UndeliveredCredentialDelivery { - delivery: Arc::from("codex_home"), + Some(HubModelConfigurationError::InvalidCredentialHome { + credential_profile: Arc::from(CODEX_SUBSCRIPTION_PROFILE), + failure: crate::CredentialHomeAdmissionFailure::InvalidPath, }) ); } #[test] - fn configuration_validates_an_undelivered_credential_home_before_refusing_it() { + fn configuration_rejects_a_missing_credential_home_with_a_typed_member_error() { + let temporary = tempfile::tempdir().expect("synthetic home root is created"); + let missing = temporary.path().join("missing-account"); let credential_home = CONFIGURATION.replace( "delivery = \"ambient\"", - "delivery = \"codex_home\"\ncodex_home = \"relative/account-a\"", + &format!( + "delivery = \"codex_home\"\ncodex_home = {:?}", + missing.to_string_lossy() + ), ); assert_eq!( HubModelConfiguration::parse(&credential_home).err(), - Some(HubModelConfigurationError::InvalidCredentialDelivery) + Some(HubModelConfigurationError::InvalidCredentialHome { + credential_profile: Arc::from(CODEX_SUBSCRIPTION_PROFILE), + failure: crate::CredentialHomeAdmissionFailure::MissingOrNotDirectory, + }) ); } #[test] - fn configuration_admits_the_largest_credential_home_concurrency_bound() { - // The bound is capped because a contended wait durably names every live - // reservation holding it. At the cap the grammar admits the field, so - // the profile reaches its undelivered refusal rather than a range one. + fn configuration_rejects_an_empty_credential_home_with_a_typed_member_error() { + let temporary = tempfile::tempdir().expect("synthetic home root is created"); + let empty = temporary.path().join("empty-account"); + std::fs::create_dir(&empty).expect("empty synthetic home is created"); let credential_home = CONFIGURATION.replace( "delivery = \"ambient\"", &format!( - "delivery = \"codex_home\"\ncodex_home = \"/srv/account-a\"\nmax_concurrent_invocations = {MAX_CREDENTIAL_HOME_CONCURRENT_INVOCATIONS}" + "delivery = \"codex_home\"\ncodex_home = {:?}", + empty.to_string_lossy() ), ); assert_eq!( HubModelConfiguration::parse(&credential_home).err(), - Some(HubModelConfigurationError::UndeliveredCredentialDelivery { - delivery: Arc::from("codex_home"), + Some(HubModelConfigurationError::InvalidCredentialHome { + credential_profile: Arc::from(CODEX_SUBSCRIPTION_PROFILE), + failure: crate::CredentialHomeAdmissionFailure::EmptyDirectory, }) ); } + #[test] + fn configuration_rejects_a_credential_home_concurrency_bound_until_reservations_exist() { + let temporary = tempfile::tempdir().expect("synthetic home root is created"); + let home = temporary.path().join("account-a"); + std::fs::create_dir(&home).expect("synthetic home is created"); + std::fs::write(home.join("fixture-marker"), "synthetic") + .expect("synthetic home is nonempty"); + let credential_home = CONFIGURATION.replace( + "delivery = \"ambient\"", + &format!( + "delivery = \"codex_home\"\ncodex_home = {:?}\nmax_concurrent_invocations = {MAX_CREDENTIAL_HOME_CONCURRENT_INVOCATIONS}", + home.to_string_lossy() + ), + ); + + assert_eq!( + HubModelConfiguration::parse(&credential_home).err(), + Some(HubModelConfigurationError::InvalidCredentialDelivery) + ); + } + #[test] fn configuration_rejects_a_credential_home_concurrency_bound_past_its_cap() { + let temporary = tempfile::tempdir().expect("synthetic home root is created"); + let home = temporary.path().join("account-a"); + std::fs::create_dir(&home).expect("synthetic home is created"); + std::fs::write(home.join("fixture-marker"), "synthetic") + .expect("synthetic home is nonempty"); let credential_home = CONFIGURATION.replace( "delivery = \"ambient\"", &format!( - "delivery = \"codex_home\"\ncodex_home = \"/srv/account-a\"\nmax_concurrent_invocations = {}", + "delivery = \"codex_home\"\ncodex_home = {:?}\nmax_concurrent_invocations = {}", + home.to_string_lossy(), MAX_CREDENTIAL_HOME_CONCURRENT_INVOCATIONS + 1 ), ); @@ -7520,7 +7727,10 @@ members = [{ profile = "anthropic-primary", priority = 1, weight = 3 }]"#, assert_eq!( HubModelConfiguration::parse(&credential_home).err(), - Some(HubModelConfigurationError::InvalidCredentialDelivery) + Some(HubModelConfigurationError::InvalidCredentialHome { + credential_profile: Arc::from(CODEX_SUBSCRIPTION_PROFILE), + failure: crate::CredentialHomeAdmissionFailure::InvalidPath, + }) ); } @@ -7968,6 +8178,122 @@ delivery = "ambient""#, ); } + #[test] + fn configuration_rejects_mixed_ambient_and_home_delivery_for_codex() { + let temporary = tempfile::tempdir().expect("synthetic home root is created"); + let home = temporary.path().join("account-b"); + std::fs::create_dir(&home).expect("synthetic home is created"); + std::fs::write(home.join("fixture-marker"), "synthetic") + .expect("synthetic home is nonempty"); + let mixed_delivery = CONFIGURATION.replace( + r#"[[credential_profiles]] +name = "codex-subscription-primary" +adapter = "codex_cli" +billing_kind = "subscription" +delivery = "ambient""#, + &format!( + r#"[[credential_profiles]] +name = "codex-subscription-primary" +adapter = "codex_cli" +billing_kind = "subscription" +delivery = "ambient" + +[[credential_profiles]] +name = "codex-subscription-overflow" +adapter = "codex_cli" +billing_kind = "subscription" +delivery = "codex_home" +codex_home = {:?}"#, + home.to_string_lossy() + ), + ); + + assert_eq!( + HubModelConfiguration::parse(&mixed_delivery).err(), + Some(HubModelConfigurationError::InvalidCredentialDelivery) + ); + } + + #[test] + fn configuration_rejects_mixed_home_and_ambient_delivery_for_codex_in_reverse_order() { + let temporary = tempfile::tempdir().expect("synthetic home root is created"); + let home = temporary.path().join("account-b"); + std::fs::create_dir(&home).expect("synthetic home is created"); + std::fs::write(home.join("fixture-marker"), "synthetic") + .expect("synthetic home is nonempty"); + let mixed_delivery = CONFIGURATION.replace( + r#"[[credential_profiles]] +name = "codex-subscription-primary" +adapter = "codex_cli" +billing_kind = "subscription" +delivery = "ambient""#, + &format!( + r#"[[credential_profiles]] +name = "codex-subscription-overflow" +adapter = "codex_cli" +billing_kind = "subscription" +delivery = "codex_home" +codex_home = {:?} + +[[credential_profiles]] +name = "codex-subscription-primary" +adapter = "codex_cli" +billing_kind = "subscription" +delivery = "ambient""#, + home.to_string_lossy() + ), + ); + + assert_eq!( + HubModelConfiguration::parse(&mixed_delivery).err(), + Some(HubModelConfigurationError::InvalidCredentialDelivery) + ); + } + + #[test] + fn configuration_admits_a_claude_ambient_profile_declared_before_a_codex_home() { + let temporary = tempfile::tempdir().expect("synthetic home root is created"); + let home = temporary.path().join("account-b"); + std::fs::create_dir(&home).expect("synthetic home is created"); + std::fs::write(home.join("fixture-marker"), "synthetic") + .expect("synthetic home is nonempty"); + // The Claude `ambient` profile precedes the Codex home in table order, + // which is the arrangement an adapter-blind conflict scan rejects. + let cross_adapter = CONFIGURATION.replace( + r#"[[credential_profiles]] +name = "codex-subscription-primary" +adapter = "codex_cli" +billing_kind = "subscription" +delivery = "ambient""#, + &format!( + r#"[[credential_profiles]] +name = "claude-subscription-primary" +adapter = "claude_cli" +billing_kind = "subscription" +delivery = "ambient" + +[[credential_profiles]] +name = "codex-subscription-primary" +adapter = "codex_cli" +billing_kind = "subscription" +delivery = "codex_home" +codex_home = {:?}"#, + home.to_string_lossy() + ), + ); + + let parsed = HubModelConfiguration::parse(&cross_adapter) + .expect("a Claude ambient profile does not contest a Codex credential home"); + assert_eq!( + parsed + .credential_profile(CODEX_SUBSCRIPTION_PROFILE) + .expect("Codex profile remains present") + .delivery() + .path(), + Some(&home) + ); + } + #[test] fn file_delivery_records_the_absolute_path_it_reads() { let configured = HubModelConfiguration::parse(CONFIGURATION).expect("the fixture is valid"); @@ -8533,6 +8859,38 @@ context_window_tokens = 200000 ); } + #[test] + fn configuration_admits_explicit_workspace_instruction_roots() { + let configured = format!( + "{CONFIGURATION}\n[workspace_instructions]\nversion = 1\nregistered_roots = [\"{REGISTERED_INSTRUCTION_ROOT}\"]\n" + ); + let configuration = HubModelConfiguration::parse(&configured) + .expect("one canonical explicit instruction root is admitted"); + assert_eq!(configuration.workspace_instructions().roots().len(), 1); + assert_eq!( + configuration.workspace_instructions().roots()[0].as_str(), + REGISTERED_INSTRUCTION_ROOT + ); + } + + #[test] + fn configuration_defaults_instruction_roots_to_empty() { + let configuration = HubModelConfiguration::parse(CONFIGURATION) + .expect("the base fixture omits explicit instruction roots"); + assert!(configuration.workspace_instructions().roots().is_empty()); + } + + #[test] + fn configuration_rejects_relative_instruction_roots() { + let relative = format!( + "{CONFIGURATION}\n[workspace_instructions]\nversion = 1\nregistered_roots = [\"relative/root\"]\n" + ); + assert_eq!( + HubModelConfiguration::parse(&relative).err(), + Some(HubModelConfigurationError::InvalidWorkspaceInstructionConfiguration) + ); + } + #[test] fn configuration_rejects_each_malformed_web_fetch_policy_shape() { let unknown_field = CONFIGURATION.replace( diff --git a/apps/signalboxd/src/context_guard.rs b/apps/signalboxd/src/context_guard.rs index 23288ceccb..7baef0351a 100644 --- a/apps/signalboxd/src/context_guard.rs +++ b/apps/signalboxd/src/context_guard.rs @@ -7,20 +7,24 @@ use signalbox_application::{ OperatorFailureClass, ToolCatalog, }; use signalbox_domain::{ - AcceptedInputTurnActivationIdentities, ContextFrontierId, ModelCallId, - SemanticTranscriptEntryId, SessionId, TurnAttemptId, TurnId, + AcceptedInputTurnActivationIdentities, ContextFrontierId, FailedModelCallTurnIdentities, + ModelCallId, ResolvedContextFrontierSnapshot, SemanticTranscriptEntryId, SessionId, + TurnAttemptId, TurnId, }; use signalbox_model_provider_runtime::{ContextCompactionModel, RuntimeModelCatalog}; use signalbox_persistence::{ + goal::GoalExecutionFailureRecoveryCause, model_execution::{ModelCallRepositoryError, PostgresModelCallRepository}, start_eligible_turn::{ - CommitActivationPreviewError, CommitActivationPreviewOutcome, StartEligibleTurnRepository, - StartEligibleTurnRepositoryError, + CommitActivationPreviewError, CommitActivationPreviewOutcome, + CommitCompactionFailurePreviewOutcome, PreparedActivationPreview, + StartEligibleTurnRepository, StartEligibleTurnRepositoryError, }, }; use crate::{ ActivatedTurnExecution, HubModelConfiguration, TurnPassExecutionStage, + WorkspaceInstructionRuntime, WorkspaceInstructionRuntimeError, process_runtime::compact_automatically, report_ambiguous_commit, usage_limits::reported_usage_requires_compaction, }; @@ -51,6 +55,13 @@ pub enum ReportedUsageCompactionError { /// Closed operator cause retained across error erasure. cause_code: &'static str, }, + /// Closing the selected turn after compaction failure could not commit. + CompactionFailureClosure { + /// Selected queued turn. + turn: TurnId, + /// Typed activation-and-failure commit error. + source: CommitActivationPreviewError, + }, } impl ReportedUsageCompactionError { @@ -60,8 +71,10 @@ impl ReportedUsageCompactionError { Self::Activation(_) => None, Self::Model { turn, .. } | Self::Render(turn) - | Self::ContextWindowUnavailable(turn) - | Self::Compaction { turn, .. } => Some(*turn), + | Self::ContextWindowUnavailable(turn) => Some(*turn), + Self::Compaction { turn, .. } | Self::CompactionFailureClosure { turn, .. } => { + Some(*turn) + } } } } @@ -77,6 +90,7 @@ impl Error for ReportedUsageCompactionError { match self { Self::Activation(error) => Some(error), Self::Model { source, .. } => Some(source), + Self::CompactionFailureClosure { source, .. } => Some(source), Self::Render(_) | Self::ContextWindowUnavailable(_) | Self::Compaction { .. } => None, } } @@ -90,6 +104,7 @@ impl ClassifyOperatorFailure for ReportedUsageCompactionError { Self::Render(_) => OperatorFailureClass::FailClosedCorruption, Self::ContextWindowUnavailable(_) => OperatorFailureClass::CallerOrHubBug, Self::Compaction { failure_class, .. } => *failure_class, + Self::CompactionFailureClosure { source, .. } => source.operator_failure_class(), } } @@ -100,6 +115,7 @@ impl ClassifyOperatorFailure for ReportedUsageCompactionError { Self::Render(_) => "reported_usage_frontier_rendering", Self::ContextWindowUnavailable(_) => "reported_usage_context_window_unavailable", Self::Compaction { cause_code, .. } => cause_code, + Self::CompactionFailureClosure { source, .. } => source.operator_failure_cause_code(), } } } @@ -115,6 +131,11 @@ pub struct ReportedUsageCompaction { compaction_model: Arc, } +struct ReportedUsageCompactionCandidate { + preview: PreparedActivationPreview, + turn: TurnId, +} + impl fmt::Debug for ReportedUsageCompaction { fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { formatter @@ -154,13 +175,128 @@ impl ReportedUsageCompaction { &self, session: SessionId, ) -> Result<(), ReportedUsageCompactionError> { + let Some(candidate) = self.compaction_candidate(session).await? else { + return Ok(()); + }; + let ReportedUsageCompactionCandidate { preview, turn } = candidate; + let applied = match compact_automatically( + &self.model_calls, + &self.model_configuration, + &self.compaction_model, + session, + turn, + ) + .await + { + Ok(applied) => applied, + Err(crate::process_runtime::AutomaticContextCompactionError::AlreadyAttempted) => { + match close_failed_compaction_turn( + &self.activation, + &self.model_calls, + preview, + None, + ) + .await + .map_err(|source| { + ReportedUsageCompactionError::CompactionFailureClosure { turn, source } + })? { + CommitCompactionFailurePreviewOutcome::Failed(_) => { + tracing::warn!( + cause_code = "reported_usage_context_compaction_exhausted", + session_id = %session.as_uuid(), + turn_id = %turn.as_uuid(), + "the queued turn's bounded automatic compaction attempt was already spent; the turn was closed before provider dispatch" + ); + return Err(ReportedUsageCompactionError::Compaction { + turn, + failure_class: OperatorFailureClass::CallerOrHubBug, + cause_code: "reported_usage_context_compaction_exhausted", + }); + } + CommitCompactionFailurePreviewOutcome::Stale => return Ok(()), + } + } + Err(error) => { + let failure_class = error.operator_failure_class(); + let cause_code = error.operator_failure_cause_code(); + if failure_class + != (OperatorFailureClass::Infrastructure { + commit_ambiguous: true, + }) + { + match close_failed_compaction_turn( + &self.activation, + &self.model_calls, + preview, + compaction_recovery_cause(&error), + ) + .await + .map_err(|source| { + ReportedUsageCompactionError::CompactionFailureClosure { turn, source } + })? { + CommitCompactionFailurePreviewOutcome::Failed(_) => {} + CommitCompactionFailurePreviewOutcome::Stale => return Ok(()), + } + } + return Err(ReportedUsageCompactionError::Compaction { + turn, + failure_class, + cause_code, + }); + } + }; + tracing::warn!( + cause_code = "reported_usage_context_compacted", + session_id = %session.as_uuid(), + turn_id = %turn.as_uuid(), + context_compaction_id = %applied.compaction.into_uuid(), + "provider-reported usage exhausted reserved context headroom; queued turn compacted before activation" + ); + let Some(remaining) = self.compaction_candidate(session).await? else { + return Ok(()); + }; + let remaining_turn = remaining.turn; + match close_failed_compaction_turn( + &self.activation, + &self.model_calls, + remaining.preview, + None, + ) + .await + .map_err( + |source| ReportedUsageCompactionError::CompactionFailureClosure { + turn: remaining_turn, + source, + }, + )? { + CommitCompactionFailurePreviewOutcome::Failed(_) => { + tracing::warn!( + cause_code = "reported_usage_context_still_exceeded", + session_id = %session.as_uuid(), + turn_id = %remaining_turn.as_uuid(), + "automatic compaction did not restore reserved context headroom; the queued turn was closed before provider dispatch" + ); + Err(ReportedUsageCompactionError::Compaction { + turn: remaining_turn, + failure_class: OperatorFailureClass::CallerOrHubBug, + cause_code: "reported_usage_context_still_exceeded", + }) + } + CommitCompactionFailurePreviewOutcome::Stale => Ok(()), + } + } + + async fn compaction_candidate( + &self, + session: SessionId, + ) -> Result, ReportedUsageCompactionError> { let Some(preview) = self .activation .preview(session, activation_identities()) .await .map_err(ReportedUsageCompactionError::Activation)? else { - return Ok(()); + return Ok(None); }; let turn = preview.prepared().turn().turn(); let prospective = self @@ -172,7 +308,7 @@ impl ReportedUsageCompaction { .await .map_err(|source| ReportedUsageCompactionError::Model { turn, source })?; let Some(prospective) = prospective else { - return Ok(()); + return Ok(None); }; let operation = prospective .render(self.tools.definitions()) @@ -189,7 +325,7 @@ impl ReportedUsageCompaction { operation.request().model_settings().effective().fast_mode(), ) .ok_or(ReportedUsageCompactionError::ContextWindowUnavailable(turn))?; - let Some(reported) = self + let reported = self .model_calls .latest_reported_usage( session, @@ -197,55 +333,33 @@ impl ReportedUsageCompaction { operation.request().call().frontier().snapshot(), ) .await - .map_err(|source| ReportedUsageCompactionError::Model { turn, source })? - else { - return Ok(()); - }; - if !reported_usage_requires_compaction( - reported.usage(), - reported.input_includes_cache_tokens(), - reported.output_is_retained(), - reported.projected_unreported_content_bytes(), - u64::from(definition.max_output_tokens()), - u64::from(definition.context_window_tokens()), - ) { - return Ok(()); - } - let applied = match compact_automatically( - &self.model_calls, - &self.model_configuration, - &self.compaction_model, - session, - turn, - ) - .await + .map_err(|source| ReportedUsageCompactionError::Model { turn, source })?; + let reported_requires_compaction = reported.is_some_and(|reported| { + reported_usage_requires_compaction( + reported.usage(), + reported.input_includes_cache_tokens(), + reported.output_is_retained(), + reported.projected_unreported_content_bytes(), + u64::from(definition.max_output_tokens()), + u64::from(definition.context_window_tokens()), + ) + }); + let failure_requires_compaction = if reported_requires_compaction { + false + } else if let Some(persisted_prefix) = + persisted_preflight_prefix(preview.prepared().starting_snapshot()) { - Ok(applied) => applied, - Err(crate::process_runtime::AutomaticContextCompactionError::AlreadyAttempted) => { - tracing::warn!( - cause_code = "reported_usage_context_compaction_exhausted", - session_id = %session.as_uuid(), - turn_id = %turn.as_uuid(), - "the queued turn's bounded automatic compaction attempt was already spent; activation remains eligible" - ); - return Ok(()); - } - Err(error) => { - return Err(ReportedUsageCompactionError::Compaction { - turn, - failure_class: error.operator_failure_class(), - cause_code: error.operator_failure_cause_code(), - }); - } + self.model_calls + .request_too_large_requires_compaction(session, target, persisted_prefix) + .await + .map_err(|source| ReportedUsageCompactionError::Model { turn, source })? + } else { + false }; - tracing::warn!( - cause_code = "reported_usage_context_compacted", - session_id = %session.as_uuid(), - turn_id = %turn.as_uuid(), - context_compaction_id = %applied.compaction.into_uuid(), - "provider-reported usage exhausted reserved context headroom; queued turn compacted before activation" - ); - Ok(()) + if !reported_requires_compaction && !failure_requires_compaction { + return Ok(None); + } + Ok(Some(ReportedUsageCompactionCandidate { preview, turn })) } } @@ -295,6 +409,21 @@ pub enum ContextGuardedTurnPassError { /// Closed cause retained before the compaction error is erased. cause_code: &'static str, }, + /// Closing the selected turn after compaction failure could not commit. + CompactionFailureClosure { + /// Selected turn. + turn: TurnId, + /// Typed activation-and-failure commit error. + source: CommitActivationPreviewError, + }, + /// Queued-turn discovery or durable manifest recording failed before the + /// counted activation commit. + WorkspaceInstructions { + /// Selected queued turn. + turn: TurnId, + /// Typed instruction preparation failure. + source: WorkspaceInstructionRuntimeError, + }, /// Execution after exact guarded activation failed. Execution { /// Stage at which execution orchestration failed. @@ -342,6 +471,8 @@ where OperatorFailureClass::CallerOrHubBug } Self::Compaction { failure_class, .. } => *failure_class, + Self::CompactionFailureClosure { source, .. } => source.operator_failure_class(), + Self::WorkspaceInstructions { source, .. } => source.operator_failure_class(), Self::Execution { source, .. } => source.operator_failure_class(), } } @@ -356,6 +487,8 @@ where Self::ContextWindowUnavailable(_) => "context_window_unavailable", Self::ContextStillExceeded(_) => "context_window_exceeded", Self::Compaction { cause_code, .. } => cause_code, + Self::CompactionFailureClosure { source, .. } => source.operator_failure_cause_code(), + Self::WorkspaceInstructions { source, .. } => source.operator_failure_cause_code(), Self::Execution { source, .. } => source.operator_failure_cause_code(), Self::ActivationSessionMismatch(_) => "activation_session_mismatch", } @@ -373,6 +506,7 @@ pub struct ContextGuardedTurnPass { runtime_models: RuntimeModelCatalog, model_configuration: HubModelConfiguration, compaction_model: Arc, + workspace_instructions: Option, execution: Execution, } @@ -392,6 +526,7 @@ where .field("runtime_models", &self.runtime_models) .field("model_configuration", &self.model_configuration) .field("compaction_model", &"[context compaction model]") + .field("workspace_instructions", &self.workspace_instructions) .field("execution", &self.execution) .finish() } @@ -418,9 +553,20 @@ impl ContextGuardedTurnPass Self { + self.workspace_instructions = Some(workspace_instructions); + self + } } impl EligibilityPass @@ -443,7 +589,9 @@ where ContextGuardedTurnPassError::Operation { turn, .. } | ContextGuardedTurnPassError::Render { turn, .. } | ContextGuardedTurnPassError::Count { turn, .. } - | ContextGuardedTurnPassError::Compaction { turn, .. } => Some(*turn), + | ContextGuardedTurnPassError::Compaction { turn, .. } + | ContextGuardedTurnPassError::CompactionFailureClosure { turn, .. } + | ContextGuardedTurnPassError::WorkspaceInstructions { turn, .. } => Some(*turn), ContextGuardedTurnPassError::CountCancelled(turn) | ContextGuardedTurnPassError::ContextWindowUnavailable(turn) | ContextGuardedTurnPassError::ContextStillExceeded(turn) @@ -462,6 +610,7 @@ where let runtime_models = self.runtime_models.clone(); let model_configuration = self.model_configuration.clone(); let compaction_model = Arc::clone(&self.compaction_model); + let workspace_instructions = self.workspace_instructions.clone(); let execution = self.execution.clone(); async move { execution.resume_active(session).await.map_err(|source| { @@ -475,7 +624,7 @@ where (), ContextGuardedTurnPassError, > = async { - let mut compacted = false; + let mut compacted_turn = None; loop { let identities = activation_identities(); let preview = match activation.preview(session, identities).await { @@ -554,7 +703,23 @@ where .checked_add(u64::from(model.max_output_tokens())) .ok_or(ContextGuardedTurnPassError::ContextStillExceeded(turn))?; if requested_tokens > u64::from(model.context_window_tokens()) { - if compacted { + if compacted_turn == Some(turn) { + match close_failed_compaction_turn( + &activation, + &model_calls, + preview, + None, + ) + .await + .map_err(|source| { + ContextGuardedTurnPassError::CompactionFailureClosure { + turn, + source, + } + })? { + CommitCompactionFailurePreviewOutcome::Failed(_) => {} + CommitCompactionFailurePreviewOutcome::Stale => continue, + } return Err(ContextGuardedTurnPassError::ContextStillExceeded(turn)); } match compact_automatically( @@ -568,21 +733,83 @@ where { Ok(_) => {} Err(crate::process_runtime::AutomaticContextCompactionError::AlreadyAttempted) => { + match close_failed_compaction_turn( + &activation, + &model_calls, + preview, + None, + ) + .await + .map_err(|source| { + ContextGuardedTurnPassError::CompactionFailureClosure { + turn, + source, + } + })? { + CommitCompactionFailurePreviewOutcome::Failed(_) => {} + CommitCompactionFailurePreviewOutcome::Stale => continue, + } return Err(ContextGuardedTurnPassError::ContextStillExceeded(turn)); } Err(error) => { + let failure_class = error.operator_failure_class(); + let cause_code = error.operator_failure_cause_code(); + if failure_class + != (OperatorFailureClass::Infrastructure { + commit_ambiguous: true, + }) + { + match close_failed_compaction_turn( + &activation, + &model_calls, + preview, + compaction_recovery_cause(&error), + ) + .await + .map_err(|source| { + ContextGuardedTurnPassError::CompactionFailureClosure { + turn, + source, + } + })? { + CommitCompactionFailurePreviewOutcome::Failed(_) => {} + CommitCompactionFailurePreviewOutcome::Stale => continue, + } + } return Err(ContextGuardedTurnPassError::Compaction { turn, - failure_class: error.operator_failure_class(), - cause_code: error.operator_failure_cause_code(), + failure_class, + cause_code, }); } } - compacted = true; + compacted_turn = Some(turn); continue; } + let prepared_instructions = if let Some(workspace_instructions) = &workspace_instructions { + let Some(prepared) = workspace_instructions + .prepare_counted_activation(session, turn) + .await + .map_err(|source| { + ContextGuardedTurnPassError::WorkspaceInstructions { + turn, + source, + } + })? + else { + continue; + }; + Some(prepared) + } else { + None + }; let committed = activation - .commit_counted_preview(preview, prospective, &model_calls) + .commit_counted_preview( + preview, + prospective, + &model_calls, + prepared_instructions.as_ref().map(|prepared| prepared.evidence()), + ) .await .map_err(|error| match error { CommitActivationPreviewError::Activation(error) => { @@ -591,6 +818,12 @@ where CommitActivationPreviewError::ModelCall(error) => { ContextGuardedTurnPassError::Operation { turn, source: error } } + CommitActivationPreviewError::WorkspaceInstructions(error) => { + ContextGuardedTurnPassError::WorkspaceInstructions { + turn, + source: WorkspaceInstructionRuntimeError::Persistence(error), + } + } })?; match committed { CommitActivationPreviewOutcome::Stale => continue, @@ -640,6 +873,10 @@ fn guarded_failure_stage( ContextGuardedTurnPassError::ContextWindowUnavailable(_) => "context_window", ContextGuardedTurnPassError::ContextStillExceeded(_) => "context_window", ContextGuardedTurnPassError::Compaction { .. } => "context_compaction", + ContextGuardedTurnPassError::CompactionFailureClosure { .. } => { + "context_compaction_failure_closure" + } + ContextGuardedTurnPassError::WorkspaceInstructions { .. } => "workspace_instructions", ContextGuardedTurnPassError::Execution { stage, .. } => stage.operator_label(), ContextGuardedTurnPassError::ActivationSessionMismatch(_) => "activation_correlation", } @@ -703,6 +940,57 @@ fn activation_identities() -> AcceptedInputTurnActivationIdentities { ) } +fn persisted_preflight_prefix( + prospective: &ResolvedContextFrontierSnapshot, +) -> Option { + prospective + .immediate_semantic_prefix() + .map(|prefix| prefix.snapshot()) +} + +async fn close_failed_compaction_turn( + activation: &StartEligibleTurnRepository, + model_calls: &PostgresModelCallRepository, + preview: PreparedActivationPreview, + recovery_cause: Option, +) -> Result { + loop { + let identities = FailedModelCallTurnIdentities::new( + SemanticTranscriptEntryId::from_uuid(uuid::Uuid::now_v7()), + ContextFrontierId::from_uuid(uuid::Uuid::now_v7()), + ); + match activation + .commit_compaction_failure_preview( + preview.clone(), + model_calls, + identities, + recovery_cause, + ) + .await + { + Err(error) if compaction_failure_closure_collision_is_retryable(&error) => {} + outcome => return outcome, + } + } +} + +fn compaction_recovery_cause( + error: &crate::process_runtime::AutomaticContextCompactionError, +) -> Option { + matches!( + error, + crate::process_runtime::AutomaticContextCompactionError::InputDoesNotFit + ) + .then_some(GoalExecutionFailureRecoveryCause::ContextCompactionInputDoesNotFit) +} + +fn compaction_failure_closure_collision_is_retryable(error: &CommitActivationPreviewError) -> bool { + matches!( + error, + CommitActivationPreviewError::ModelCall(ModelCallRepositoryError::IdentityCollision(_)) + ) +} + #[cfg(test)] mod tests { use std::{ @@ -711,13 +999,41 @@ mod tests { }; use signalbox_application::{ClassifyOperatorFailure, OperatorFailureClass}; - use signalbox_domain::{ActivatedTurn, TurnId}; + use signalbox_domain::{ + ActivatedTurn, ContextFrontierId, ResolvedContextFrontierReconstitutionInput, SessionId, + TurnId, + }; use signalbox_persistence::{ context_compaction::ContextCompactionRepositoryError, - start_eligible_turn::StartEligibleTurnRepositoryError, + goal::GoalExecutionFailureRecoveryCause, + model_execution::{ModelCallIdentityCollision, ModelCallRepositoryError}, + start_eligible_turn::{ + CommitActivationPreviewError, StartEligibleTurnIdentityCollision, + StartEligibleTurnRepositoryError, + }, }; - use super::{ContextGuardedTurnPassError, guarded_failure_stage, report_guarded_ambiguity}; + use super::{ + ContextGuardedTurnPassError, compaction_failure_closure_collision_is_retryable, + compaction_recovery_cause, guarded_failure_stage, persisted_preflight_prefix, + report_guarded_ambiguity, + }; + + #[test] + fn no_fitting_compaction_input_requires_operator_recovery() { + assert_eq!( + compaction_recovery_cause(&AutomaticContextCompactionError::InputDoesNotFit), + Some(GoalExecutionFailureRecoveryCause::ContextCompactionInputDoesNotFit) + ); + } + + #[test] + fn transient_compaction_failure_keeps_automatic_recovery() { + assert_eq!( + compaction_recovery_cause(&AutomaticContextCompactionError::Model), + None + ); + } use crate::{ ActivatedTurnExecution, FatalExecutionSignal, FatalExecutionSupervisor, TurnPassExecutionStage, process_runtime::AutomaticContextCompactionError, @@ -727,6 +1043,26 @@ mod tests { #[derive(Clone, Copy, Debug, Eq, PartialEq)] struct CommitAmbiguousFailure; + #[test] + fn reminted_compaction_failure_identity_collision_is_retryable() { + let error = CommitActivationPreviewError::ModelCall( + ModelCallRepositoryError::IdentityCollision(ModelCallIdentityCollision::SemanticEntry), + ); + + assert!(compaction_failure_closure_collision_is_retryable(&error)); + } + + #[test] + fn immutable_activation_identity_collision_is_not_retryable() { + let error = CommitActivationPreviewError::Activation( + StartEligibleTurnRepositoryError::IdentityCollision( + StartEligibleTurnIdentityCollision::StartingFrontier, + ), + ); + + assert!(!compaction_failure_closure_collision_is_retryable(&error)); + } + impl fmt::Display for CommitAmbiguousFailure { fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { formatter.write_str("commit acknowledgement was lost") @@ -773,6 +1109,19 @@ mod tests { TurnId::from_uuid(uuid::Uuid::from_u128(1)) } + #[test] + fn request_failure_preflight_uses_the_durable_immediate_prefix() { + let session = SessionId::from_uuid(uuid::Uuid::from_u128(2)); + let persisted = ContextFrontierId::from_uuid(uuid::Uuid::from_u128(3)); + let prospective = ContextFrontierId::from_uuid(uuid::Uuid::from_u128(4)); + let snapshot = ResolvedContextFrontierReconstitutionInput::new(session, persisted, vec![]) + .derive_appending(prospective, vec![]) + .reconstitute() + .expect("derived prospective frontier is valid"); + + assert_eq!(persisted_preflight_prefix(&snapshot), Some(persisted)); + } + /// The exact lost-acknowledgement failure the activation repository reports /// when a guarded counted commit cannot be proven. fn ambiguous_activation() -> GuardedFailure { diff --git a/apps/signalboxd/src/convergence_sweep_runtime.rs b/apps/signalboxd/src/convergence_sweep_runtime.rs index 461a6854e8..ae7516c0d5 100644 --- a/apps/signalboxd/src/convergence_sweep_runtime.rs +++ b/apps/signalboxd/src/convergence_sweep_runtime.rs @@ -1,10 +1,6 @@ //! Periodic convergence reconciliation for explicitly selected watched pull requests. -use std::{ - error::Error, - fmt, - time::{Duration, SystemTime}, -}; +use std::{error::Error, fmt, sync::Arc, time::Duration}; use futures_util::{StreamExt, stream}; use reqwest::{ @@ -29,11 +25,15 @@ use signalbox_persistence::{ commissioned_dispatch::{CommissionDispatchOutcome, PostgresCommissionedDispatchStore}, convergence_sweep::{ ConvergenceSweepDecision, ConvergenceSweepFailureKind, ConvergenceSweepObservation, - PostgresConvergenceSweepStore, + ConvergenceSweepRetryPolicy, PostgresConvergenceSweepStore, }, }; use sqlx::PgPool; -use tokio::{select, sync::watch, time::sleep}; +use tokio::{ + select, + sync::{Semaphore, watch}, + time::{Instant, MissedTickBehavior, interval, sleep, sleep_until}, +}; use crate::{ FileCredentialAccess, HubModelConfiguration, RepositoryWatchConfiguration, @@ -86,7 +86,7 @@ const DETAILS_QUERY: &str = r#" query PullRequestConvergence($namespace: String!, $name: String!, $number: Int!) { repository(owner: $namespace, name: $name) { pullRequest(number: $number) { - state isDraft baseRefName headRefName headRefOid mergeable + state isDraft baseRefName baseRefOid headRefName headRefOid mergeable headRepository { name_with_owner: nameWithOwner } reviewThreads(first: 100) { nodes { isResolved } @@ -110,10 +110,11 @@ query PullRequestConvergence($namespace: String!, $name: String!, $number: Int!) const THREADS_QUERY: &str = r#" query PullRequestConvergenceThreads( - $namespace: String!, $name: String!, $number: Int!, $after: String! + $namespace: String!, $name: String!, $number: Int!, $after: String ) { repository(owner: $namespace, name: $name) { pullRequest(number: $number) { + state baseRefName baseRefOid headRefName headRefOid reviewThreads(first: 100, after: $after) { nodes { isResolved } pageInfo { hasNextPage endCursor } @@ -125,10 +126,11 @@ query PullRequestConvergenceThreads( const CHECKS_QUERY: &str = r#" query PullRequestConvergenceChecks( - $namespace: String!, $name: String!, $number: Int!, $after: String! + $namespace: String!, $name: String!, $number: Int!, $after: String ) { repository(owner: $namespace, name: $name) { pullRequest(number: $number) { + state baseRefName baseRefOid headRefName headRefOid commits(last: 1) { nodes { commit { oid statusCheckRollup { contexts(first: 100, after: $after) { @@ -210,7 +212,6 @@ impl ConvergenceSweepRuntime { .tls_version_min(reqwest::tls::Version::TLS_1_2) .tls_danger_accept_invalid_certs(false) .tls_danger_accept_invalid_hostnames(false) - .no_proxy() .redirect(Policy::none()) .retry(reqwest::retry::never()); if let Some(request_timeout) = numeric_bounds.request_timeout { @@ -258,41 +259,88 @@ impl ConvergenceSweepRuntime { } /// Runs complete censuses until shutdown; one target failure never halts siblings. - pub async fn run(self, mut shutdown: watch::Receiver) { - if *shutdown.borrow() { - return; - } - loop { - if !self.sweep_once(&mut shutdown).await { - return; - } - select! { - _ = sleep(self.interval) => {} - changed = shutdown.changed() => { - if changed.is_err() || *shutdown.borrow() { return; } + pub async fn run(self, shutdown: watch::Receiver) { + let runtime = &self; + // Configuration bounds this target set to 256 entries. When the operator + // configures no concurrency ceiling, giving each enrolled target one permit + // preserves its absolute polling deadline while retaining an explicit, + // configuration-bounded admission gate; a configured ceiling narrows that + // gate so slow targets cannot occupy the whole census at once. + let active_targets = Arc::new(Semaphore::new( + self.numeric_bounds + .concurrent_targets + .unwrap_or(self.targets.len()), + )); + stream::iter(&self.targets) + .for_each_concurrent(None, |target| { + let mut shutdown = shutdown.clone(); + let active_targets = Arc::clone(&active_targets); + async move { + if *shutdown.borrow() { + return; + } + let mut ticks = interval(runtime.interval); + ticks.set_missed_tick_behavior(MissedTickBehavior::Skip); + let mut reenrolled = false; + loop { + let scheduled = select! { + scheduled = ticks.tick() => scheduled, + changed = shutdown.changed() => { + if changed.is_err() || *shutdown.borrow() { return; } + continue; + } + }; + let permit = select! { + permit = active_targets.acquire() => { + match permit { + Ok(permit) => permit, + Err(_) => return, + } + } + changed = shutdown.changed() => { + if changed.is_err() || *shutdown.borrow() { return; } + continue; + } + }; + if !reenrolled { + match runtime + .state + .reenroll_target(&target.repository, target.pull_request) + .await + { + Ok(()) => reenrolled = true, + Err(error) => { + tracing::error!( + repository = %target.repository.as_str(), + pull_request = target.pull_request.get(), + cause = %error, + "convergence sweep target re-enrollment failed; retrying on the next tick" + ); + drop(permit); + continue; + } + } + } + select! { + () = runtime.reconcile_target( + target, + scheduled + runtime.interval, + ) => {} + changed = shutdown.changed() => { + if changed.is_err() || *shutdown.borrow() { return; } + } + } + drop(permit); + } } - } - } - } - - async fn sweep_once(&self, shutdown: &mut watch::Receiver) -> bool { - let census = stream::iter(&self.targets) - .for_each_concurrent(self.numeric_bounds.concurrent_targets, |target| { - self.reconcile_target(target) - }); - tokio::pin!(census); - select! { - () = &mut census => true, - changed = shutdown.changed() => { - changed.is_ok() && !*shutdown.borrow() - } - } + }) + .await; } - async fn reconcile_target(&self, target: &SweepTarget) { + async fn reconcile_target(&self, target: &SweepTarget, census_deadline: Instant) { let loaded = match self .state - .load_target(&target.repository, target.pull_request) + .load_target_with_cool_off(&target.repository, target.pull_request, self.cool_off) .await { Ok(state) => state, @@ -310,6 +358,12 @@ impl ConvergenceSweepRuntime { return; } }; + if loaded + .as_ref() + .is_some_and(|state| state.is_parked() || !state.retry_ready()) + { + return; + } if let Some((dispatch, observation)) = loaded .as_ref() .and_then(|state| state.pending_dispatch().zip(state.pending_observation())) @@ -333,6 +387,9 @@ impl ConvergenceSweepRuntime { tracing::error!(repository = %target.repository.as_str(), pull_request = target.pull_request.get(), cause = %error, "convergence sweep could not repair a committed dispatch projection"); + if error.commit_ambiguous() { + return; + } self.record_failure( target, Some(observation), @@ -344,13 +401,17 @@ impl ConvergenceSweepRuntime { } return; } - if loaded - .as_ref() - .is_some_and(|state| state.is_parked() || !state.retry_ready()) - { - return; - } - let fetched = match self.fetch(target).await { + let fetched = match select! { + fetched = self.fetch(target) => fetched, + _ = sleep_until(census_deadline) => { + tracing::warn!( + repository = %target.repository.as_str(), + pull_request = target.pull_request.get(), + "convergence sweep provider census exceeded its polling interval" + ); + Err(CensusError::Response) + } + } { Ok(fetched) => fetched, Err(cause) => { self.record_failure(target, None, ConvergenceSweepFailureKind::FactsFetch, cause) @@ -369,34 +430,87 @@ impl ConvergenceSweepRuntime { return; } if let Some(dispatch) = loaded.as_ref().and_then(|state| state.latest_dispatch()) { - let dispatch_observation = loaded.as_ref().and_then(|state| { - state - .last_dispatch_observation() - .or_else(|| state.pending_observation()) - .or_else(|| state.last_observation()) - }); - let unchanged = dispatch_observation == Some(&observation); - let cool_off_elapsed = SystemTime::now() - .duration_since(dispatch.dispatched_at()) - .is_ok_and(|elapsed| elapsed >= self.cool_off); - if unchanged && !dispatch.has_model_activity() && cool_off_elapsed { - self.record_failure( + let dispatch_observation = loaded + .as_ref() + .and_then(|state| state.latest_dispatch_observation()); + if dispatch_observation.is_none() { + self.record_dispatch_decision( target, - Some(&observation), - ConvergenceSweepFailureKind::NoModelActivity, - CensusError::Shape, + &observation, + dispatch.dispatch_id(), + dispatch.session_id(), + if dispatch.is_live() { + ConvergenceSweepDecision::LiveSession + } else { + ConvergenceSweepDecision::CoolingOff + }, ) .await; return; } + let unchanged = dispatch_observation == Some(&observation); + let cool_off_elapsed = loaded + .as_ref() + .is_some_and(|state| state.cool_off_elapsed()); + if unchanged && !dispatch.has_model_activity() && cool_off_elapsed { + match self + .state + .record_no_model_activity_failure( + uuid::Uuid::now_v7(), + &target.repository, + target.pull_request, + &observation, + dispatch.session_id(), + ) + .await + { + Ok(disposition) => tracing::warn!( + repository = %target.repository.as_str(), + pull_request = target.pull_request.get(), + ?disposition, + "convergence sweep evaluated inactive session" + ), + Err(error) => { + tracing::error!( + repository = %target.repository.as_str(), + pull_request = target.pull_request.get(), + cause = %error, + "convergence sweep inactivity decision could not be recorded" + ); + if error.commit_ambiguous() { + return; + } + self.record_failure( + target, + Some(&observation), + ConvergenceSweepFailureKind::StateAccess, + CensusError::State, + ) + .await; + } + } + return; + } if dispatch.is_live() { - self.record_decision(target, &observation, ConvergenceSweepDecision::LiveSession) - .await; + self.record_dispatch_decision( + target, + &observation, + dispatch.dispatch_id(), + dispatch.session_id(), + ConvergenceSweepDecision::LiveSession, + ) + .await; return; } if !cool_off_elapsed { - self.record_decision(target, &observation, ConvergenceSweepDecision::CoolingOff) - .await; + self.record_dispatch_decision( + target, + &observation, + dispatch.dispatch_id(), + dispatch.session_id(), + ConvergenceSweepDecision::CoolingOff, + ) + .await; return; } } @@ -441,6 +555,9 @@ impl ConvergenceSweepRuntime { tracing::error!(repository = %target.repository.as_str(), pull_request = target.pull_request.get(), cause = %error, "convergence sweep commission fence could not be recorded"); + if error.commit_ambiguous() { + return; + } self.record_failure( target, Some(&observation), @@ -484,7 +601,9 @@ impl ConvergenceSweepRuntime { }; match self .commissioned - .commission(prepared, |alias| self.models.resolve_alias(alias)) + .commission_after_cool_off(prepared, self.cool_off, |alias| { + self.models.resolve_alias(alias) + }) .await { Ok( @@ -506,6 +625,9 @@ impl ConvergenceSweepRuntime { tracing::error!(repository = %target.repository.as_str(), pull_request = target.pull_request.get(), cause = %error, "convergence sweep committed a session but could not record its local projection"); + if error.commit_ambiguous() { + return; + } self.record_failure( target, Some(&observation), @@ -520,6 +642,15 @@ impl ConvergenceSweepRuntime { self.record_decision(target, &observation, ConvergenceSweepDecision::LiveSession) .await; } + Ok(CommissionDispatchOutcome::TargetCoolingOff { .. }) => { + self.record_decision(target, &observation, ConvergenceSweepDecision::CoolingOff) + .await; + } + Err(error) if error.commit_ambiguous() => { + tracing::error!(repository = %target.repository.as_str(), + pull_request = target.pull_request.get(), cause = %error, + "convergence sweep commission outcome is commit-ambiguous"); + } Ok(CommissionDispatchOutcome::ConflictingReuse) | Err(_) => { self.record_failure( target, @@ -552,6 +683,48 @@ impl ConvergenceSweepRuntime { tracing::error!(repository = %target.repository.as_str(), pull_request = target.pull_request.get(), cause = %error, "convergence sweep decision could not be recorded"); + if error.commit_ambiguous() { + return; + } + self.record_failure( + target, + Some(observation), + ConvergenceSweepFailureKind::StateAccess, + CensusError::State, + ) + .await; + } + } + + async fn record_dispatch_decision( + &self, + target: &SweepTarget, + observation: &ConvergenceSweepObservation, + dispatch_id: uuid::Uuid, + session_id: signalbox_domain::SessionId, + decision: ConvergenceSweepDecision, + ) { + if let Err(error) = self + .state + .record_dispatch_decision( + uuid::Uuid::now_v7(), + &target.repository, + target.pull_request, + observation, + (dispatch_id, session_id), + decision, + ) + .await + { + tracing::error!( + repository = %target.repository.as_str(), + pull_request = target.pull_request.get(), + cause = %error, + "convergence sweep dispatch decision could not be recorded" + ); + if error.commit_ambiguous() { + return; + } self.record_failure( target, Some(observation), @@ -569,21 +742,6 @@ impl ConvergenceSweepRuntime { failure: ConvergenceSweepFailureKind, cause: CensusError, ) { - let prior = self - .state - .load_target(&target.repository, target.pull_request) - .await - .ok() - .flatten(); - let attempt = prior - .as_ref() - .filter(|state| state.failure_kind() == Some(failure)) - .map_or(0, |state| u32::from(state.consecutive_failures())); - let delay = retry_delay( - attempt, - self.numeric_bounds.retry_backoff_base, - self.numeric_bounds.retry_backoff_cap, - ); match self .state .record_failure( @@ -592,7 +750,10 @@ impl ConvergenceSweepRuntime { target.pull_request, observation, failure, - delay.map(|delay| delay.as_secs()), + ConvergenceSweepRetryPolicy { + backoff_base: self.numeric_bounds.retry_backoff_base, + backoff_cap: self.numeric_bounds.retry_backoff_cap, + }, ) .await { @@ -639,60 +800,60 @@ impl ConvergenceSweepRuntime { return Err(CensusError::Shape); } let head_sha = commit_at(pull, "headRefOid")?; + let head_branch = branch_at(pull, "headRefName")?; + let base_branch = branch_at(pull, "baseRefName")?; + let base_sha = commit_at(pull, "baseRefOid")?; + let head_repository = head_repository_at(pull)?; let checked_head_sha = checked_head_at(pull)?; - let mut unresolved = unresolved_threads( + let mergeable_state = mergeable_state_at(pull)?; + let draft_state = draft_state_at(pull)?; + let initial_thread_states = review_thread_states( pull.pointer("/reviewThreads/nodes") .and_then(Value::as_array) .ok_or(CensusError::Shape)?, )?; - let initial_checks = pull - .pointer("/commits/nodes/0/commit/statusCheckRollup/contexts/nodes") - .and_then(Value::as_array) - .map_or(&[][..], Vec::as_slice); - let mut checks = decode_checks(initial_checks)?; - let mut thread_page = page_info(pull.pointer("/reviewThreads/pageInfo"))?; - let mut check_page = - match pull.pointer("/commits/nodes/0/commit/statusCheckRollup/contexts/pageInfo") { - Some(value) => page_info(Some(value))?, - None => PageInfo::done(), - }; - let mut pages = 1usize; + let mut thread_states = initial_thread_states.clone(); + let (initial_checks, initial_check_page) = initial_checks(pull)?; + let mut checks = initial_checks.clone(); + let mut check_page = initial_check_page.clone(); + let initial_thread_page = page_info(pull.pointer("/reviewThreads/pageInfo"))?; + let mut thread_page = initial_thread_page.clone(); + let mut thread_pages = 1usize; while thread_page.has_next { - pages += 1; + thread_pages += 1; if self .numeric_bounds .connection_pages - .is_some_and(|limit| pages > limit) + .is_some_and(|limit| thread_pages > limit) { return Err(CensusError::Pagination); } let mut next = variables.clone(); next["after"] = Value::String(thread_page.cursor.ok_or(CensusError::Shape)?); let page = self.graphql(THREADS_QUERY, next, &authorization).await?; - let connection = page - .pointer("/data/repository/pullRequest/reviewThreads") - .ok_or(CensusError::Shape)?; - unresolved += unresolved_threads( + let connection = threads_page(&page, &head_sha, &head_branch, &base_branch, &base_sha)?; + thread_states.extend(review_thread_states( connection .get("nodes") .and_then(Value::as_array) .ok_or(CensusError::Shape)?, - )?; + )?); thread_page = page_info(connection.get("pageInfo"))?; } + let mut check_pages = 1usize; while check_page.has_next { - pages += 1; + check_pages += 1; if self .numeric_bounds .connection_pages - .is_some_and(|limit| pages > limit) + .is_some_and(|limit| check_pages > limit) { return Err(CensusError::Pagination); } let mut next = variables.clone(); next["after"] = Value::String(check_page.cursor.ok_or(CensusError::Shape)?); let page = self.graphql(CHECKS_QUERY, next, &authorization).await?; - let connection = checks_page(&page, &head_sha)?; + let connection = checks_page(&page, &head_sha, &head_branch, &base_branch, &base_sha)?; checks.extend(decode_checks( connection .get("nodes") @@ -701,28 +862,124 @@ impl ConvergenceSweepRuntime { )?); check_page = page_info(connection.get("pageInfo"))?; } - let mergeable_state = match pull.get("mergeable").and_then(Value::as_str) { - Some("MERGEABLE") => MergeableState::Mergeable, - Some("CONFLICTING") => MergeableState::Conflicting, - Some("UNKNOWN") => MergeableState::Unknown, - _ => return Err(CensusError::Shape), - }; + // A paginated census assembles its snapshot from a traversal that spans + // many responses, so the fence below and the revalidation traversals + // after it bound the whole window in one direction: the details reread + // proves the refs, mergeable state, draft state, and head repository + // still hold, and the re-traversals that follow it prove every page of + // both connections still holds. Revalidating a connection before the + // fence instead would leave a gap — a thread or check on the second or + // later page could change after its own reread but before the fence, + // and because the fence compares only the initial pages, refs, and page + // information, all of which can be identical across that change, the + // stale buffers would be accepted. + if thread_pages > 1 || check_pages > 1 { + let revalidated = self + .graphql(DETAILS_QUERY, variables.clone(), &authorization) + .await?; + let revalidated_pull = revalidated + .pointer("/data/repository/pullRequest") + .ok_or(CensusError::Shape)?; + validate_paginated_pull( + revalidated_pull, + &head_sha, + &head_branch, + &base_branch, + &base_sha, + )?; + if mergeable_state_at(revalidated_pull)? != mergeable_state { + return Err(CensusError::State); + } + ensure_draft_state_stable(draft_state, draft_state_at(revalidated_pull)?)?; + ensure_head_repository_stable( + &head_repository, + &head_repository_at(revalidated_pull)?, + )?; + ensure_final_connections_stable( + revalidated_pull, + &initial_thread_states, + &initial_thread_page, + &initial_checks, + &initial_check_page, + )?; + } + if thread_pages > 1 { + let mut next = variables.clone(); + next["after"] = Value::Null; + let page = self.graphql(THREADS_QUERY, next, &authorization).await?; + let connection = threads_page(&page, &head_sha, &head_branch, &base_branch, &base_sha)?; + let mut revalidated = review_thread_states( + connection + .get("nodes") + .and_then(Value::as_array) + .ok_or(CensusError::Shape)?, + )?; + let mut revalidation_page = page_info(connection.get("pageInfo"))?; + let mut revalidation_pages = 1usize; + while revalidation_page.has_next { + revalidation_pages += 1; + if self + .numeric_bounds + .connection_pages + .is_some_and(|limit| revalidation_pages > limit) + { + return Err(CensusError::Pagination); + } + let mut next = variables.clone(); + next["after"] = Value::String(revalidation_page.cursor.ok_or(CensusError::Shape)?); + let page = self.graphql(THREADS_QUERY, next, &authorization).await?; + let connection = + threads_page(&page, &head_sha, &head_branch, &base_branch, &base_sha)?; + revalidated.extend(review_thread_states( + connection + .get("nodes") + .and_then(Value::as_array) + .ok_or(CensusError::Shape)?, + )?); + revalidation_page = page_info(connection.get("pageInfo"))?; + } + ensure_threads_stable(&thread_states, &revalidated)?; + } + if checks_require_revalidation(thread_pages, check_pages) { + let mut next = variables.clone(); + next["after"] = Value::Null; + let page = self.graphql(CHECKS_QUERY, next, &authorization).await?; + let (mut revalidated, mut revalidation_page) = + initial_checks_page(&page, &head_sha, &head_branch, &base_branch, &base_sha)?; + let mut revalidation_pages = 1usize; + while revalidation_page.has_next { + revalidation_pages += 1; + if self + .numeric_bounds + .connection_pages + .is_some_and(|limit| revalidation_pages > limit) + { + return Err(CensusError::Pagination); + } + let mut next = variables.clone(); + next["after"] = Value::String(revalidation_page.cursor.ok_or(CensusError::Shape)?); + let page = self.graphql(CHECKS_QUERY, next, &authorization).await?; + let connection = + checks_page(&page, &head_sha, &head_branch, &base_branch, &base_sha)?; + revalidated.extend(decode_checks( + connection + .get("nodes") + .and_then(Value::as_array) + .ok_or(CensusError::Shape)?, + )?); + revalidation_page = page_info(connection.get("pageInfo"))?; + } + ensure_checks_stable(&checks, &revalidated)?; + } + let unresolved = unresolved_threads(&thread_states); Ok(FetchedPullRequest { - base_branch: branch_at(pull, "baseRefName")?, - head_branch: branch_at(pull, "headRefName")?, - head_repository: RepositorySlug::try_new( - pull.pointer("/headRepository/name_with_owner") - .and_then(Value::as_str) - .ok_or(CensusError::Shape)? - .to_lowercase(), - ) - .map_err(|_| CensusError::Shape)?, + base_branch, + head_branch, + head_repository, facts: PullRequestConvergenceFacts::new( head_sha, checked_head_sha, - pull.get("isDraft") - .and_then(Value::as_bool) - .ok_or(CensusError::Shape)?, + draft_state, unresolved, mergeable_state, checks, @@ -742,7 +999,7 @@ impl ConvergenceSweepRuntime { if self.numeric_bounds.request_attempts == Some(0) { return Err(CensusError::Request); } - let mut response = loop { + let bytes = 'attempts: loop { attempt += 1; let sent = self .client @@ -765,7 +1022,37 @@ impl ConvergenceSweepRuntime { { sleep_for_policy(self.numeric_bounds.request_retry_delay).await; } - Ok(response) => break response, + Ok(mut response) => { + if response.status() != StatusCode::OK { + return Err(CensusError::Response); + } + let mut bytes = Vec::new(); + loop { + match response.chunk().await { + Ok(Some(chunk)) => { + let next = bytes + .len() + .checked_add(chunk.len()) + .ok_or(CensusError::Response)?; + if next > MAX_RESPONSE_BYTES { + return Err(CensusError::Response); + } + bytes.extend_from_slice(&chunk); + } + Ok(None) => break 'attempts bytes, + Err(_) + if self + .numeric_bounds + .request_attempts + .is_none_or(|limit| attempt < limit) => + { + sleep_for_policy(self.numeric_bounds.request_retry_delay).await; + continue 'attempts; + } + Err(_) => return Err(CensusError::Response), + } + } + } Err(_) if self .numeric_bounds @@ -777,20 +1064,6 @@ impl ConvergenceSweepRuntime { Err(_) => return Err(CensusError::Request), } }; - if response.status() != StatusCode::OK { - return Err(CensusError::Response); - } - let mut bytes = Vec::new(); - while let Some(chunk) = response.chunk().await.map_err(|_| CensusError::Response)? { - let next = bytes - .len() - .checked_add(chunk.len()) - .ok_or(CensusError::Response)?; - if next > MAX_RESPONSE_BYTES { - return Err(CensusError::Response); - } - bytes.extend_from_slice(&chunk); - } let value: Value = serde_json::from_slice(&bytes).map_err(|_| CensusError::Decode)?; if value.get("errors").is_some() { return Err(CensusError::Response); @@ -806,6 +1079,7 @@ struct FetchedPullRequest { facts: PullRequestConvergenceFacts, } +#[derive(Clone, Debug, Eq, PartialEq)] struct PageInfo { has_next: bool, cursor: Option, @@ -833,19 +1107,77 @@ fn page_info(value: Option<&Value>) -> Result { }) } -fn unresolved_threads(values: &[Value]) -> Result { - values.iter().try_fold(0u64, |count, value| { - let resolved = value - .get("isResolved") - .and_then(Value::as_bool) - .ok_or(CensusError::Shape)?; - Ok(count + u64::from(!resolved)) - }) +fn review_thread_states(values: &[Value]) -> Result, CensusError> { + values + .iter() + .map(|value| { + value + .get("isResolved") + .and_then(Value::as_bool) + .ok_or(CensusError::Shape) + }) + .collect() +} + +fn unresolved_threads(states: &[bool]) -> u64 { + states.iter().filter(|resolved| !**resolved).count() as u64 +} + +fn ensure_threads_stable(observed: &[bool], revalidated: &[bool]) -> Result<(), CensusError> { + if observed == revalidated { + Ok(()) + } else { + Err(CensusError::State) + } +} + +fn ensure_final_connections_stable( + pull: &Value, + observed_threads: &[bool], + observed_thread_page: &PageInfo, + observed_checks: &[PullRequestCheck], + observed_check_page: &PageInfo, +) -> Result<(), CensusError> { + let revalidated_threads = review_thread_states( + pull.pointer("/reviewThreads/nodes") + .and_then(Value::as_array) + .ok_or(CensusError::Shape)?, + )?; + let revalidated_thread_page = page_info(pull.pointer("/reviewThreads/pageInfo"))?; + let (revalidated_checks, revalidated_check_page) = initial_checks(pull)?; + ensure_threads_stable(observed_threads, &revalidated_threads)?; + ensure_checks_stable(observed_checks, &revalidated_checks)?; + if observed_thread_page != &revalidated_thread_page + || observed_check_page != &revalidated_check_page + { + return Err(CensusError::State); + } + Ok(()) } -fn checks_page<'a>(page: &'a Value, expected_head: &CommitSha) -> Result<&'a Value, CensusError> { - let commit = page - .pointer("/data/repository/pullRequest/commits/nodes/0/commit") +const fn checks_require_revalidation(thread_pages: usize, check_pages: usize) -> bool { + thread_pages > 1 || check_pages > 1 +} + +fn checks_page<'a>( + page: &'a Value, + expected_head: &CommitSha, + expected_head_branch: &BranchName, + expected_base: &BranchName, + expected_base_sha: &CommitSha, +) -> Result<&'a Value, CensusError> { + let pull = page + .pointer("/data/repository/pullRequest") + .ok_or(CensusError::Shape)?; + validate_paginated_pull( + pull, + expected_head, + expected_head_branch, + expected_base, + expected_base_sha, + )?; + let commit = pull + .pointer("/commits/nodes/0/commit") .ok_or(CensusError::Shape)?; if commit_at(commit, "oid")? != *expected_head { return Err(CensusError::Shape); @@ -855,25 +1187,128 @@ fn checks_page<'a>(page: &'a Value, expected_head: &CommitSha) -> Result<&'a Val .ok_or(CensusError::Shape) } +fn initial_checks_page( + page: &Value, + expected_head: &CommitSha, + expected_head_branch: &BranchName, + expected_base: &BranchName, + expected_base_sha: &CommitSha, +) -> Result<(Vec, PageInfo), CensusError> { + let pull = page + .pointer("/data/repository/pullRequest") + .ok_or(CensusError::Shape)?; + validate_paginated_pull( + pull, + expected_head, + expected_head_branch, + expected_base, + expected_base_sha, + )?; + let commit = pull + .pointer("/commits/nodes/0/commit") + .ok_or(CensusError::Shape)?; + if commit_at(commit, "oid")? != *expected_head { + return Err(CensusError::Shape); + } + initial_checks(pull) +} + +fn threads_page<'a>( + page: &'a Value, + expected_head: &CommitSha, + expected_head_branch: &BranchName, + expected_base: &BranchName, + expected_base_sha: &CommitSha, +) -> Result<&'a Value, CensusError> { + let pull = page + .pointer("/data/repository/pullRequest") + .ok_or(CensusError::Shape)?; + validate_paginated_pull( + pull, + expected_head, + expected_head_branch, + expected_base, + expected_base_sha, + )?; + pull.get("reviewThreads").ok_or(CensusError::Shape) +} + +fn validate_paginated_pull( + pull: &Value, + expected_head: &CommitSha, + expected_head_branch: &BranchName, + expected_base: &BranchName, + expected_base_sha: &CommitSha, +) -> Result<(), CensusError> { + if pull.get("state").and_then(Value::as_str) != Some("OPEN") + || commit_at(pull, "headRefOid")? != *expected_head + || branch_at(pull, "headRefName")? != *expected_head_branch + || branch_at(pull, "baseRefName")? != *expected_base + || commit_at(pull, "baseRefOid")? != *expected_base_sha + { + return Err(CensusError::Shape); + } + Ok(()) +} + +fn mergeable_state_at(pull: &Value) -> Result { + match pull.get("mergeable").and_then(Value::as_str) { + Some("MERGEABLE") => Ok(MergeableState::Mergeable), + Some("CONFLICTING") => Ok(MergeableState::Conflicting), + Some("UNKNOWN") => Ok(MergeableState::Unknown), + _ => Err(CensusError::Shape), + } +} + +fn draft_state_at( + pull: &Value, +) -> Result { + match pull.get("isDraft").and_then(Value::as_bool) { + Some(true) => Ok(signalbox_application::PullRequestDraftState::Draft), + Some(false) => Ok(signalbox_application::PullRequestDraftState::ReadyForReview), + None => Err(CensusError::Shape), + } +} + +fn ensure_draft_state_stable( + observed: signalbox_application::PullRequestDraftState, + revalidated: signalbox_application::PullRequestDraftState, +) -> Result<(), CensusError> { + if observed == revalidated { + Ok(()) + } else { + Err(CensusError::State) + } +} + fn decode_checks(values: &[Value]) -> Result, CensusError> { values .iter() .map( |value| match value.get("__typename").and_then(Value::as_str) { - Some("CheckRun") => Ok(PullRequestCheck::new( - value - .get("name") + Some("CheckRun") => { + let status = value + .get("status") .and_then(Value::as_str) - .ok_or(CensusError::Shape)? - .to_owned(), - PullRequestCheckState::CheckRun { - completed: value.get("status").and_then(Value::as_str) == Some("COMPLETED"), - conclusion: value - .get("conclusion") + .ok_or(CensusError::Shape)?; + Ok(PullRequestCheck::new( + value + .get("name") .and_then(Value::as_str) - .map(str::to_owned), - }, - )), + .ok_or(CensusError::Shape)? + .to_owned(), + if status == "COMPLETED" { + PullRequestCheckState::CheckRunCompleted { + conclusion: value + .get("conclusion") + .and_then(Value::as_str) + .map(str::to_owned), + } + } else { + PullRequestCheckState::CheckRunInProgress + }, + )) + } Some("StatusContext") => Ok(PullRequestCheck::new( value .get("context") @@ -894,14 +1329,45 @@ fn decode_checks(values: &[Value]) -> Result, CensusError> .collect() } +fn ensure_checks_stable( + observed: &[PullRequestCheck], + revalidated: &[PullRequestCheck], +) -> Result<(), CensusError> { + if observed == revalidated { + Ok(()) + } else { + Err(CensusError::State) + } +} + fn checked_head_at(pull: &Value) -> Result, CensusError> { - pull.pointer("/commits/nodes/0/commit/statusCheckRollup") - .filter(|rollup| !rollup.is_null()) - .and_then(|_| pull.pointer("/commits/nodes/0/commit/oid")) - .and_then(Value::as_str) - .map(|value| CommitSha::try_new(value.to_owned())) - .transpose() - .map_err(|_| CensusError::Shape) + let rollup = pull + .pointer("/commits/nodes/0/commit/statusCheckRollup") + .ok_or(CensusError::Shape)?; + if rollup.is_null() { + return Ok(None); + } + commit_at( + pull.pointer("/commits/nodes/0/commit") + .ok_or(CensusError::Shape)?, + "oid", + ) + .map(Some) +} + +fn initial_checks(pull: &Value) -> Result<(Vec, PageInfo), CensusError> { + let rollup = pull + .pointer("/commits/nodes/0/commit/statusCheckRollup") + .ok_or(CensusError::Shape)?; + if rollup.is_null() { + return Ok((Vec::new(), PageInfo::done())); + } + let contexts = rollup.get("contexts").ok_or(CensusError::Shape)?; + let nodes = contexts + .get("nodes") + .and_then(Value::as_array) + .ok_or(CensusError::Shape)?; + Ok((decode_checks(nodes)?, page_info(contexts.get("pageInfo"))?)) } fn commit_at(value: &Value, key: &str) -> Result { @@ -988,7 +1454,7 @@ fn commission_content( "head_repository": fetched.head_repository.as_str(), "base_branch": fetched.base_branch.as_str(), "head_branch": fetched.head_branch.as_str(), - "draft": fetched.facts.draft(), + "draft": fetched.facts.draft().is_draft(), "unresolved_review_threads": fetched.facts.unresolved_review_threads(), "mergeable_state": format!("{:?}", fetched.facts.mergeable_state()).to_lowercase(), "gating_checks": gating_checks, @@ -998,6 +1464,27 @@ fn commission_content( .map_err(|_| ()) } +fn head_repository_at(pull: &Value) -> Result { + RepositorySlug::try_new( + pull.pointer("/headRepository/name_with_owner") + .and_then(Value::as_str) + .ok_or(CensusError::Shape)? + .to_lowercase(), + ) + .map_err(|_| CensusError::Shape) +} + +fn ensure_head_repository_stable( + observed: &RepositorySlug, + revalidated: &RepositorySlug, +) -> Result<(), CensusError> { + if observed == revalidated { + Ok(()) + } else { + Err(CensusError::State) + } +} + fn blocker_text(blocker: &PullRequestConvergenceBlocker) -> String { match blocker { PullRequestConvergenceBlocker::UnresolvedReviewThreads(count) => { @@ -1014,13 +1501,6 @@ fn blocker_text(blocker: &PullRequestConvergenceBlocker) -> String { } } -fn retry_delay(attempt: u32, base: Option, cap: Option) -> Option { - base.map(|base| { - let delay = base.saturating_mul(2u32.saturating_pow(attempt.min(4))); - cap.map_or(delay, |cap| delay.min(cap)) - }) -} - async fn sleep_for_policy(delay: Option) { match delay { Some(delay) => sleep(delay).await, @@ -1030,6 +1510,18 @@ async fn sleep_for_policy(delay: Option) { #[cfg(test)] mod tests { + use signalbox_application::InProcessEligibilityWorkSource; + use signalbox_persistence::{ + convergence_sweep::ConvergenceSweepFailureDisposition, disposable_postgres_server_args, + disposable_postgres_state_tmpfs, disposable_test_container_labels, + local_test_connection_options, migrate, scheduler::PostgresEligibilitySweep, + }; + use sqlx::postgres::PgPoolOptions; + use testcontainers_modules::{ + postgres::Postgres as TestPostgres, + testcontainers::{self, ImageExt, runners::AsyncRunner}, + }; + use super::*; fn example_numeric_bounds() -> ConvergenceSweepNumericBounds { @@ -1068,33 +1560,27 @@ mod tests { CommitSha::try_new(value.to_string().repeat(40)).expect("fixture SHA is valid") } + // The lineage arithmetic itself now lives in the convergence-sweep store, which + // grows and caps each retry from this policy. What stays provable here is that + // the configured, optional bounds reach that store intact and describe a usable + // lineage: a first retry no later than the ceiling it saturates against. #[test] - fn retry_delay_is_bounded() { + fn configured_retry_policy_carries_the_example_backoff_bounds() { let bounds = example_numeric_bounds(); - assert_eq!( - retry_delay(0, bounds.retry_backoff_base, bounds.retry_backoff_cap), - bounds.retry_backoff_base - ); - assert_eq!( - retry_delay(4, bounds.retry_backoff_base, bounds.retry_backoff_cap), - bounds.retry_backoff_cap - ); - assert_eq!( - retry_delay( - u32::MAX, - bounds.retry_backoff_base, - bounds.retry_backoff_cap - ), - bounds.retry_backoff_cap - ); - } - - #[test] - fn unbounded_retry_policy_has_no_finite_delay_or_cap() { - let base = Duration::from_secs(3); + let policy = ConvergenceSweepRetryPolicy { + backoff_base: bounds.retry_backoff_base, + backoff_cap: bounds.retry_backoff_cap, + }; - assert_eq!(retry_delay(0, None, None), None); - assert_eq!(retry_delay(5, Some(base), None), Some(base * 16)); + assert_eq!(policy.backoff_base, Some(Duration::from_secs(60))); + assert_eq!(policy.backoff_cap, Some(Duration::from_secs(15 * 60))); + assert!( + policy + .backoff_base + .zip(policy.backoff_cap) + .is_some_and(|(base, cap)| base <= cap), + "the checked-in example schedules a first retry no later than its own cap" + ); } #[test] @@ -1107,8 +1593,7 @@ mod tests { ); let run = PullRequestCheck::new( String::from("CodeRabbit"), - PullRequestCheckState::CheckRun { - completed: true, + PullRequestCheckState::CheckRunCompleted { conclusion: Some(String::from("FAILURE")), }, ); @@ -1122,6 +1607,17 @@ mod tests { assert!(CHECKS_QUERY.contains("commits(last: 1)")); } + #[test] + fn a_check_run_without_status_is_rejected() { + let checks = [json!({ + "__typename": "CheckRun", + "name": "test", + "conclusion": "SUCCESS" + })]; + + assert!(matches!(decode_checks(&checks), Err(CensusError::Shape))); + } + #[test] fn absent_status_rollup_does_not_mark_checks_current() { let pull = json!({ @@ -1131,15 +1627,130 @@ mod tests { }); assert_eq!(checked_head_at(&pull), Ok(None)); + let (checks, page) = initial_checks(&pull).expect("a null rollup is a complete absence"); + assert!(checks.is_empty()); + assert!(!page.has_next); } #[test] - fn a_second_checks_page_decodes_only_for_the_observed_head() { + fn absent_status_rollup_remains_absent_during_revalidation() { let expected_head = sha('a'); + let expected_base_sha = sha('c'); + let expected_head_branch = BranchName::try_new(String::from("agent/convergence")) + .expect("fixture head branch is valid"); + let expected_base = + BranchName::try_new(String::from("main")).expect("fixture base branch is valid"); + let page = json!({ + "data": {"repository": {"pullRequest": { + "state": "OPEN", + "baseRefName": expected_base.as_str(), + "baseRefOid": expected_base_sha.as_str(), + "headRefName": expected_head_branch.as_str(), + "headRefOid": expected_head.as_str(), + "commits": {"nodes": [{"commit": { + "oid": expected_head.as_str(), + "statusCheckRollup": null + }}]} + }}} + }); + + let (checks, page_info) = initial_checks_page( + &page, + &expected_head, + &expected_head_branch, + &expected_base, + &expected_base_sha, + ) + .expect("stable rollup absence revalidates"); + + assert!(checks.is_empty()); + assert!(!page_info.has_next); + } + + #[test] + fn mergeability_decoder_preserves_the_closed_provider_states() { + assert_eq!( + mergeable_state_at(&json!({"mergeable": "MERGEABLE"})), + Ok(MergeableState::Mergeable) + ); + assert_eq!( + mergeable_state_at(&json!({"mergeable": "CONFLICTING"})), + Ok(MergeableState::Conflicting) + ); + assert_eq!( + mergeable_state_at(&json!({"mergeable": "UNKNOWN"})), + Ok(MergeableState::Unknown) + ); + } + + #[test] + fn paginated_census_rejects_draft_state_drift() { + assert_eq!( + ensure_draft_state_stable( + signalbox_application::PullRequestDraftState::ReadyForReview, + signalbox_application::PullRequestDraftState::Draft, + ), + Err(CensusError::State) + ); + assert_eq!( + ensure_draft_state_stable( + signalbox_application::PullRequestDraftState::Draft, + signalbox_application::PullRequestDraftState::Draft, + ), + Ok(()) + ); + } + + #[test] + fn a_partial_initial_status_rollup_is_rejected() { + let missing_nodes = json!({ + "commits": { + "nodes": [{"commit": { + "oid": sha('a').as_str(), + "statusCheckRollup": { + "contexts": { + "pageInfo": {"hasNextPage": false, "endCursor": null} + } + } + }}] + } + }); + let missing_page_info = json!({ + "commits": { + "nodes": [{"commit": { + "oid": sha('a').as_str(), + "statusCheckRollup": {"contexts": {"nodes": []}} + }}] + } + }); + + assert!(matches!( + initial_checks(&missing_nodes), + Err(CensusError::Shape) + )); + assert!(matches!( + initial_checks(&missing_page_info), + Err(CensusError::Shape) + )); + } + + #[test] + fn a_second_checks_page_decodes_only_for_the_observed_snapshot() { + let expected_head = sha('a'); + let expected_base_sha = sha('c'); + let expected_head_branch = BranchName::try_new(String::from("agent/convergence")) + .expect("fixture head branch is valid"); + let expected_base = + BranchName::try_new(String::from("main")).expect("fixture base branch is valid"); let page = json!({ "data": { "repository": { "pullRequest": { + "state": "OPEN", + "baseRefName": expected_base.as_str(), + "baseRefOid": expected_base_sha.as_str(), + "headRefName": expected_head_branch.as_str(), + "headRefOid": expected_head.as_str(), "commits": { "nodes": [{ "commit": { @@ -1165,7 +1776,14 @@ mod tests { } }); - let connection = checks_page(&page, &expected_head).expect("head-matched page decodes"); + let connection = checks_page( + &page, + &expected_head, + &expected_head_branch, + &expected_base, + &expected_base_sha, + ) + .expect("snapshot-matched page decodes"); let checks = decode_checks( connection .get("nodes") @@ -1180,10 +1798,20 @@ mod tests { #[test] fn a_checks_page_for_another_head_is_rejected() { + let expected_base_sha = sha('c'); + let expected_head_branch = BranchName::try_new(String::from("agent/convergence")) + .expect("fixture head branch is valid"); + let expected_base = + BranchName::try_new(String::from("main")).expect("fixture base branch is valid"); let page = json!({ "data": { "repository": { "pullRequest": { + "state": "OPEN", + "baseRefName": expected_base.as_str(), + "baseRefOid": expected_base_sha.as_str(), + "headRefName": expected_head_branch.as_str(), + "headRefOid": sha('b').as_str(), "commits": { "nodes": [{ "commit": { @@ -1197,6 +1825,598 @@ mod tests { } }); - assert_eq!(checks_page(&page, &sha('a')), Err(CensusError::Shape)); + assert_eq!( + checks_page( + &page, + &sha('a'), + &expected_head_branch, + &expected_base, + &expected_base_sha, + ), + Err(CensusError::Shape) + ); + } + + #[test] + fn mutable_paginated_check_states_are_rejected() { + let successful = PullRequestCheck::new( + String::from("test"), + PullRequestCheckState::CheckRunCompleted { + conclusion: Some(String::from("SUCCESS")), + }, + ); + let failed = PullRequestCheck::new( + String::from("test"), + PullRequestCheckState::CheckRunCompleted { + conclusion: Some(String::from("FAILURE")), + }, + ); + + assert_eq!( + ensure_checks_stable(std::slice::from_ref(&successful), &[failed]), + Err(CensusError::State) + ); + assert_eq!( + ensure_checks_stable( + std::slice::from_ref(&successful), + std::slice::from_ref(&successful), + ), + Ok(()) + ); + } + + #[test] + fn a_thread_page_for_the_observed_snapshot_decodes() { + let expected_head = sha('a'); + let expected_base_sha = sha('c'); + let expected_head_branch = BranchName::try_new(String::from("agent/convergence")) + .expect("fixture head branch is valid"); + let expected_base = + BranchName::try_new(String::from("main")).expect("fixture base branch is valid"); + let page = json!({ + "data": {"repository": {"pullRequest": { + "state": "OPEN", + "baseRefName": expected_base.as_str(), + "baseRefOid": expected_base_sha.as_str(), + "headRefName": expected_head_branch.as_str(), + "headRefOid": expected_head.as_str(), + "reviewThreads": {"nodes": [{"isResolved": false}]} + }}} + }); + let connection = threads_page( + &page, + &expected_head, + &expected_head_branch, + &expected_base, + &expected_base_sha, + ) + .expect("snapshot-matched page decodes"); + let nodes = connection + .get("nodes") + .and_then(Value::as_array) + .expect("fixture carries thread nodes"); + + assert_eq!( + unresolved_threads(&review_thread_states(nodes).expect("thread states decode")), + 1 + ); + } + + #[test] + fn mutable_paginated_review_thread_states_are_rejected() { + assert_eq!( + ensure_threads_stable(&[true, false], &[false, true]), + Err(CensusError::State) + ); + assert_eq!( + ensure_threads_stable(&[true, false], &[true, false]), + Ok(()) + ); + } + + #[test] + fn final_details_reject_changed_initial_connection_contents() { + let observed_check = PullRequestCheck::new( + String::from("test"), + PullRequestCheckState::CheckRunCompleted { + conclusion: Some(String::from("SUCCESS")), + }, + ); + let pull = json!({ + "reviewThreads": { + "nodes": [{"isResolved": false}], + "pageInfo": {"hasNextPage": true, "endCursor": "threads-1"} + }, + "commits": {"nodes": [{"commit": { + "oid": sha('a').as_str(), + "statusCheckRollup": {"contexts": { + "nodes": [{ + "__typename": "CheckRun", + "name": "test", + "status": "COMPLETED", + "conclusion": "FAILURE" + }], + "pageInfo": {"hasNextPage": false, "endCursor": null} + }} + }}]} + }); + + let result = ensure_final_connections_stable( + &pull, + &[true], + &PageInfo { + has_next: true, + cursor: Some(String::from("threads-1")), + }, + &[observed_check], + &PageInfo::done(), + ); + + assert_eq!(result, Err(CensusError::State)); + } + + #[test] + fn paginated_census_rejects_a_head_repository_transfer() { + let observed = RepositorySlug::try_new(String::from("contributor/repository")) + .expect("fixture repository is valid"); + let transferred = RepositorySlug::try_new(String::from("successor/repository")) + .expect("fixture repository is valid"); + + assert_eq!( + ensure_head_repository_stable(&observed, &transferred), + Err(CensusError::State) + ); + assert_eq!(ensure_head_repository_stable(&observed, &observed), Ok(())); + } + + #[test] + fn thread_pagination_revalidates_the_initial_checks() { + assert!(checks_require_revalidation(2, 1)); + assert!(checks_require_revalidation(1, 2)); + assert!(!checks_require_revalidation(1, 1)); + } + + #[test] + fn a_thread_page_for_another_head_is_rejected() { + let expected_base_sha = sha('c'); + let expected_head_branch = BranchName::try_new(String::from("agent/convergence")) + .expect("fixture head branch is valid"); + let expected_base = + BranchName::try_new(String::from("main")).expect("fixture base branch is valid"); + let page = json!({ + "data": {"repository": {"pullRequest": { + "state": "OPEN", + "baseRefName": expected_base.as_str(), + "baseRefOid": expected_base_sha.as_str(), + "headRefName": expected_head_branch.as_str(), + "headRefOid": sha('b').as_str(), + "reviewThreads": {} + }}} + }); + + assert_eq!( + threads_page( + &page, + &sha('a'), + &expected_head_branch, + &expected_base, + &expected_base_sha, + ), + Err(CensusError::Shape) + ); + } + + #[test] + fn paginated_pages_reject_closed_retargeted_base_advanced_or_renamed_pull_requests() { + let expected_head = sha('a'); + let expected_base_sha = sha('c'); + let expected_head_branch = BranchName::try_new(String::from("agent/convergence")) + .expect("fixture head branch is valid"); + let expected_base = + BranchName::try_new(String::from("main")).expect("fixture base branch is valid"); + let closed = json!({ + "data": {"repository": {"pullRequest": { + "state": "CLOSED", + "baseRefName": expected_base.as_str(), + "baseRefOid": expected_base_sha.as_str(), + "headRefName": expected_head_branch.as_str(), + "headRefOid": expected_head.as_str(), + "reviewThreads": {} + }}} + }); + let retargeted = json!({ + "data": {"repository": {"pullRequest": { + "state": "OPEN", + "baseRefName": "release", + "baseRefOid": expected_base_sha.as_str(), + "headRefName": expected_head_branch.as_str(), + "headRefOid": expected_head.as_str(), + "commits": {"nodes": []} + }}} + }); + let base_advanced = json!({ + "data": {"repository": {"pullRequest": { + "state": "OPEN", + "baseRefName": expected_base.as_str(), + "baseRefOid": sha('d').as_str(), + "headRefName": expected_head_branch.as_str(), + "headRefOid": expected_head.as_str(), + "reviewThreads": {} + }}} + }); + let head_renamed = json!({ + "data": {"repository": {"pullRequest": { + "state": "OPEN", + "baseRefName": expected_base.as_str(), + "baseRefOid": expected_base_sha.as_str(), + "headRefName": "agent/renamed", + "headRefOid": expected_head.as_str(), + "reviewThreads": {} + }}} + }); + + assert_eq!( + threads_page( + &closed, + &expected_head, + &expected_head_branch, + &expected_base, + &expected_base_sha, + ), + Err(CensusError::Shape) + ); + assert_eq!( + checks_page( + &retargeted, + &expected_head, + &expected_head_branch, + &expected_base, + &expected_base_sha, + ), + Err(CensusError::Shape) + ); + assert_eq!( + threads_page( + &base_advanced, + &expected_head, + &expected_head_branch, + &expected_base, + &expected_base_sha, + ), + Err(CensusError::Shape) + ); + assert_eq!( + threads_page( + &head_renamed, + &expected_head, + &expected_head_branch, + &expected_base, + &expected_base_sha, + ), + Err(CensusError::Shape) + ); + } + + // `reconcile_target` sequences the store primitives, and the branches it + // takes before the provider census are reachable against a real database + // with no network: the parked / `retry_ready` gate, the committed-dispatch + // projection repair, and the census-failure path. The branches after a + // successful census are not reachable here — `GRAPHQL_URL` is a const with + // no injection seam, so a hand-built runtime cannot be pointed at a local + // server. Every fixture below resolves its credential from a path that does + // not exist, which makes `fetch` fail before it opens a connection. + + const POSTGRES_IMAGE_TAG: &str = "18.4-alpine3.23"; + const DATABASE_NAME: &str = "signalbox_convergence_sweep"; + const DATABASE_USER: &str = "signalbox"; + const DATABASE_PASSWORD: &str = "signalbox-test-only"; + const FIXTURE_REPOSITORY: &str = "signalbox/repository"; + const FIXTURE_PULL_REQUEST: u64 = 892; + const FIXTURE_HEAD_SHA: &str = "1111111111111111111111111111111111111111"; + const FIXTURE_HEAD_REPOSITORY: &str = "contributor/repository"; + const FIXTURE_HEAD_BRANCH: &str = "agent/convergence"; + const FIXTURE_BASE_BRANCH: &str = "main"; + const FIXTURE_TEMPLATE: &str = "review-response"; + const FIXTURE_UNRESOLVED_THREADS: u64 = 3; + + async fn migrated_postgres() + -> Result<(testcontainers::ContainerAsync, PgPool), Box> { + let container = TestPostgres::default() + .with_db_name(DATABASE_NAME) + .with_user(DATABASE_USER) + .with_password(DATABASE_PASSWORD) + .with_cmd(disposable_postgres_server_args()) + .with_mount(disposable_postgres_state_tmpfs(None)) + .with_tag(POSTGRES_IMAGE_TAG) + .with_labels(disposable_test_container_labels()) + .start() + .await?; + let host = container.get_host().await?; + let port = container.get_host_port_ipv4(5432).await?; + let database_url = + format!("postgres://{DATABASE_USER}:{DATABASE_PASSWORD}@{host}:{port}/{DATABASE_NAME}"); + let pool = PgPoolOptions::new() + .max_connections(4) + .connect_with(local_test_connection_options(&database_url)?) + .await?; + migrate(&pool).await?; + Ok((container, pool)) + } + + fn fixture_repository() -> RepositorySlug { + RepositorySlug::try_new(FIXTURE_REPOSITORY.to_owned()).expect("fixture repository is valid") + } + + fn fixture_pull_request() -> PullRequestNumber { + PullRequestNumber::new( + std::num::NonZeroU64::new(FIXTURE_PULL_REQUEST).expect("fixture number is positive"), + ) + } + + fn fixture_observation() -> ConvergenceSweepObservation { + ConvergenceSweepObservation::new( + CommitSha::try_new(FIXTURE_HEAD_SHA.to_owned()).expect("fixture SHA is valid"), + FIXTURE_UNRESOLVED_THREADS, + ) + } + + /// A target whose credential path does not exist, so `fetch` fails at its + /// first step and no request is ever issued. + fn fixture_target() -> SweepTarget { + let reference = CredentialReference::new("fixture-credential"); + SweepTarget { + repository: fixture_repository(), + pull_request: fixture_pull_request(), + credentials: FileCredentialAccess::new_bounded( + std::path::PathBuf::from("/nonexistent/convergence-sweep-fixture-credential"), + reference.clone(), + MAX_CREDENTIAL_BYTES, + ), + credential_reference: reference, + } + } + + /// Builds the runtime over a live pool. The returned work source is held by + /// the caller so the nudge channel stays open for the runtime's lifetime. + fn fixture_runtime( + pool: &PgPool, + cool_off: Duration, + ) -> Result< + ( + ConvergenceSweepRuntime, + InProcessEligibilityWorkSource, + ), + Box, + > { + let _ = rustls::crypto::ring::default_provider().install_default(); + let models = crate::configuration::checked_in_example_configuration()?; + let credential_pin = models.session_credential_pin(); + let (eligibility_nudge, work_source) = + InProcessEligibilityWorkSource::new(PostgresEligibilitySweep::new(pool.clone())); + let runtime = ConvergenceSweepRuntime { + client: Client::builder().build()?, + targets: vec![fixture_target()].into_boxed_slice(), + interval: Duration::from_secs(60), + cool_off, + template: signalbox_domain::SessionTemplateName::try_new(FIXTURE_TEMPLATE.to_owned())?, + templates: SessionTemplateConfiguration::default(), + models, + commissioned: PostgresCommissionedDispatchStore::new(pool.clone(), credential_pin), + state: PostgresConvergenceSweepStore::new(pool.clone()), + eligibility_nudge, + numeric_bounds: example_numeric_bounds(), + }; + Ok((runtime, work_source)) + } + + async fn recorded_events(pool: &PgPool) -> Result> { + Ok(sqlx::query_scalar( + "SELECT count(*) FROM convergence_sweep_event + WHERE repository = $1 AND pull_request_number = $2", + ) + .bind(FIXTURE_REPOSITORY) + .bind(rust_decimal::Decimal::from(FIXTURE_PULL_REQUEST)) + .fetch_one(pool) + .await?) + } + + async fn target_state(pool: &PgPool) -> Result<(String, i16), Box> { + Ok(sqlx::query_as( + "SELECT state_kind, consecutive_failures FROM convergence_sweep_target + WHERE repository = $1 AND pull_request_number = $2", + ) + .bind(FIXTURE_REPOSITORY) + .bind(rust_decimal::Decimal::from(FIXTURE_PULL_REQUEST)) + .fetch_one(pool) + .await?) + } + + /// Records one facts-fetch failure against the fixture target and returns + /// the disposition the store chose for it. + /// + /// Driving a target to its parked state needs several identical + /// transitions; naming the transition keeps the test bodies straight-line, + /// so a disposition that comes back wrong is reported at the call site of + /// the attempt that produced it rather than at one shared loop. + async fn record_facts_fetch_failure( + runtime: &ConvergenceSweepRuntime, + target: &SweepTarget, + ) -> Result> { + Ok(runtime + .state + .record_failure( + uuid::Uuid::now_v7(), + &target.repository, + target.pull_request, + Some(&fixture_observation()), + ConvergenceSweepFailureKind::FactsFetch, + ConvergenceSweepRetryPolicy { + backoff_base: runtime.numeric_bounds.retry_backoff_base, + backoff_cap: runtime.numeric_bounds.retry_backoff_cap, + }, + ) + .await?) + } + + #[tokio::test(flavor = "multi_thread")] + #[ignore = "requires ephemeral PostgreSQL"] + async fn a_census_failure_schedules_a_retry_for_the_target() -> Result<(), Box> { + let (_container, pool) = migrated_postgres().await?; + let (runtime, _work_source) = fixture_runtime(&pool, Duration::from_secs(60))?; + let target = fixture_target(); + + runtime + .reconcile_target(&target, Instant::now() + Duration::from_secs(30)) + .await; + + let failure: String = sqlx::query_scalar( + "SELECT failure_kind FROM convergence_sweep_event + WHERE repository = $1 AND pull_request_number = $2 + AND failure_kind IS NOT NULL", + ) + .bind(FIXTURE_REPOSITORY) + .bind(rust_decimal::Decimal::from(FIXTURE_PULL_REQUEST)) + .fetch_one(&pool) + .await?; + assert_eq!(failure, "facts_fetch"); + assert_eq!(target_state(&pool).await?, (String::from("retry_wait"), 1)); + Ok(()) + } + + #[tokio::test(flavor = "multi_thread")] + #[ignore = "requires ephemeral PostgreSQL"] + async fn a_target_inside_its_retry_backoff_is_left_alone() -> Result<(), Box> { + let (_container, pool) = migrated_postgres().await?; + let (runtime, _work_source) = fixture_runtime(&pool, Duration::from_secs(60))?; + let target = fixture_target(); + runtime + .state + .record_failure( + uuid::Uuid::now_v7(), + &target.repository, + target.pull_request, + Some(&fixture_observation()), + ConvergenceSweepFailureKind::FactsFetch, + ConvergenceSweepRetryPolicy { + backoff_base: runtime.numeric_bounds.retry_backoff_base, + backoff_cap: runtime.numeric_bounds.retry_backoff_cap, + }, + ) + .await?; + let before = recorded_events(&pool).await?; + + runtime + .reconcile_target(&target, Instant::now() + Duration::from_secs(30)) + .await; + + // The backoff has not elapsed, so the gate returns before the census and + // the failure lineage is untouched. + assert_eq!(recorded_events(&pool).await?, before); + assert_eq!(target_state(&pool).await?, (String::from("retry_wait"), 1)); + Ok(()) + } + + #[tokio::test(flavor = "multi_thread")] + #[ignore = "requires ephemeral PostgreSQL"] + async fn a_parked_target_is_left_alone() -> Result<(), Box> { + let (_container, pool) = migrated_postgres().await?; + let (runtime, _work_source) = fixture_runtime(&pool, Duration::from_secs(60))?; + let target = fixture_target(); + let first = record_facts_fetch_failure(&runtime, &target).await?; + let second = record_facts_fetch_failure(&runtime, &target).await?; + let third = record_facts_fetch_failure(&runtime, &target).await?; + let fourth = record_facts_fetch_failure(&runtime, &target).await?; + let fifth = record_facts_fetch_failure(&runtime, &target).await?; + assert_eq!(first, ConvergenceSweepFailureDisposition::RetryScheduled); + assert_eq!(second, ConvergenceSweepFailureDisposition::RetryScheduled); + assert_eq!(third, ConvergenceSweepFailureDisposition::RetryScheduled); + assert_eq!(fourth, ConvergenceSweepFailureDisposition::RetryScheduled); + assert_eq!(fifth, ConvergenceSweepFailureDisposition::Parked); + let before = recorded_events(&pool).await?; + assert_eq!(target_state(&pool).await?.0, "parked"); + + runtime + .reconcile_target(&target, Instant::now() + Duration::from_secs(30)) + .await; + + // A parked target waits for an operator, never for another census. + assert_eq!(recorded_events(&pool).await?, before); + assert_eq!(target_state(&pool).await?.0, "parked"); + Ok(()) + } + + #[tokio::test(flavor = "multi_thread")] + #[ignore = "requires ephemeral PostgreSQL"] + async fn a_committed_dispatch_is_projected_before_any_census() -> Result<(), Box> { + let (_container, pool) = migrated_postgres().await?; + let (runtime, _work_source) = fixture_runtime(&pool, Duration::from_secs(60))?; + let target = fixture_target(); + let observation = fixture_observation(); + let command = DurableCommandId::from_uuid(uuid::Uuid::from_u128(0x89_204)); + runtime + .state + .begin_commission( + &target.repository, + target.pull_request, + &observation, + [17; 32], + command, + ) + .await?; + let request = CommissionDispatchRequest::try_new( + command, + signalbox_domain::SessionTemplateName::try_new(FIXTURE_TEMPLATE.to_owned())?, + CommissionedDispatchFence::PullRequest { + repository: target.repository.clone(), + pull_request: target.pull_request, + head_sha: CommitSha::try_new(FIXTURE_HEAD_SHA.to_owned())?, + head_repository: RepositorySlug::try_new(FIXTURE_HEAD_REPOSITORY.to_owned())?, + head_branch: BranchName::try_new(FIXTURE_HEAD_BRANCH.to_owned())?, + base_branch: BranchName::try_new(FIXTURE_BASE_BRANCH.to_owned())?, + }, + GoalStatement::try_new("Converge the pull request.".to_owned())?, + UserContent::try_text("Respond to the review.".to_owned()) + .expect("fixture content is admitted"), + )?; + let prepared = request.prepare( + &mut UuidV7CommissionedDispatchIdGenerator, + signalbox_domain::SessionTemplateProvenance::new( + signalbox_domain::SessionTemplateName::try_new(FIXTURE_TEMPLATE.to_owned())?, + signalbox_domain::SessionTemplateContentDigest::from_bytes([7; 32]), + ), + signalbox_domain::SessionConfigurationDefaults::complete( + signalbox_domain::ModelSelectionRequest::Direct( + signalbox_domain::DirectModelSelection::from_uuid(uuid::Uuid::from_u128( + 0x89_200, + )), + ), + signalbox_domain::DangerousToolAutoApproval::Disabled, + Some(signalbox_domain::SessionSystemPrompt::try_new( + "Respond to review findings.".to_owned(), + )?), + ), + )?; + let outcome = runtime.commissioned.commission(prepared, |_| None).await?; + let CommissionDispatchOutcome::Dispatched { dispatch, session } = outcome else { + panic!("a fresh fixture must dispatch: {outcome:?}"); + }; + + runtime + .reconcile_target(&target, Instant::now() + Duration::from_secs(30)) + .await; + + // The projection repair runs before the census and returns, so the + // missing credential never produces a failure for this tick. + let projected: (uuid::Uuid, uuid::Uuid) = sqlx::query_as( + "SELECT last_dispatch_id, last_session_id FROM convergence_sweep_target + WHERE repository = $1 AND pull_request_number = $2", + ) + .bind(FIXTURE_REPOSITORY) + .bind(rust_decimal::Decimal::from(FIXTURE_PULL_REQUEST)) + .fetch_one(&pool) + .await?; + assert_eq!(projected, (dispatch.into_uuid(), session.into_uuid())); + assert_eq!(target_state(&pool).await?, (String::from("observed"), 0)); + Ok(()) } } diff --git a/apps/signalboxd/src/conversation_introspection.rs b/apps/signalboxd/src/conversation_introspection.rs index a275651a90..c8eaa54e7c 100644 --- a/apps/signalboxd/src/conversation_introspection.rs +++ b/apps/signalboxd/src/conversation_introspection.rs @@ -52,6 +52,12 @@ impl ConversationIntrospectionError { } } + const fn corrupt_projection() -> Self { + Self { + class: OperatorFailureClass::FailClosedCorruption, + } + } + fn from_listing(error: ConversationListingRepositoryError) -> Self { Self { class: match error { @@ -290,7 +296,12 @@ fn visible_process_entry( entry_index, content, .. - } => (entry_index, TranscriptEntryKind::User, content), + } => ( + entry_index, + TranscriptEntryKind::User, + serde_json::to_string(&crate::process_runtime::wire_user_content(&content)) + .map_err(|_| ConversationIntrospectionError::corrupt_projection())?, + ), ProcessTranscriptEntry::Assistant { entry_index, content, diff --git a/apps/signalboxd/src/credential_pools.rs b/apps/signalboxd/src/credential_pools.rs index c64a4ca815..24f9ecc32f 100644 --- a/apps/signalboxd/src/credential_pools.rs +++ b/apps/signalboxd/src/credential_pools.rs @@ -56,8 +56,8 @@ const PROFILE_COMMON_FIELDS: [&str; 4] = ["name", "adapter", "billing_kind", "de /// How one credential profile's secret reaches its provider. /// /// The variants below are the deliveries this build supplies. The grammar also -/// recognizes `codex_home` and `oauth`; parsing rejects those as undelivered so -/// a deployment learns at startup that no surface honors them, rather than from +/// recognizes `oauth`; parsing rejects it as undelivered so a deployment learns +/// at startup that no surface honors it, rather than from /// a call that silently authenticated as some other account. #[derive(Clone, Eq, PartialEq)] pub enum CredentialDelivery { @@ -71,6 +71,47 @@ pub enum CredentialDelivery { /// Process environment key a spawned adapter supplies the value under. env_key: Option>, }, + /// An operator-provisioned Codex login directory selected by reference. + /// The daemon validates directory shape but never reads its auth material, + /// per `docs/spec/configuration-and-credentials.md#the-codex_home-delivery`. + CodexHome { + /// Absolute existing nonempty directory passed only as `CODEX_HOME`. + path: PathBuf, + /// Optional per-home process concurrency declaration. + max_concurrent_invocations: Option, + }, +} + +/// Typed startup failure for one configured credential home. +/// +/// `docs/spec/configuration-and-credentials.md#the-codex_home-delivery` owns +/// these fail-closed admission conditions. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum CredentialHomeAdmissionFailure { + /// The configured path was not an absolute normalized path. + InvalidPath, + /// The path does not name an existing directory. + MissingOrNotDirectory, + /// Directory enumeration failed closed. + UnreadableDirectory, + /// The directory contains no provisioned entries. + EmptyDirectory, +} + +impl CredentialHomeAdmissionFailure { + /// Operator-facing spelling of this closed cause. + /// + /// The returned text names the admission condition only. It never carries + /// the configured path or any authentication material, so error display + /// may quote it beside the profile reference. + pub const fn cause(self) -> &'static str { + match self { + Self::InvalidPath => "path is not absolute and normalized", + Self::MissingOrNotDirectory => "path is not an existing directory", + Self::UnreadableDirectory => "directory could not be enumerated", + Self::EmptyDirectory => "directory contains no provisioned entries", + } + } } impl fmt::Debug for CredentialDelivery { @@ -82,6 +123,14 @@ impl fmt::Debug for CredentialDelivery { .field("path", &"[credential file path]") .field("env_key", env_key) .finish(), + Self::CodexHome { + max_concurrent_invocations, + .. + } => formatter + .debug_struct("CodexHome") + .field("path", &"[credential home path]") + .field("max_concurrent_invocations", max_concurrent_invocations) + .finish(), } } } @@ -92,14 +141,16 @@ impl CredentialDelivery { match self { Self::Ambient => "ambient", Self::File { .. } => "file", + Self::CodexHome { .. } => "codex_home", } } - /// Absolute path this delivery reads, where it reads one. + /// Absolute deployment path this delivery references, where it has one. pub fn path(&self) -> Option<&PathBuf> { match self { Self::Ambient => None, Self::File { path, .. } => Some(path), + Self::CodexHome { path, .. } => Some(path), } } @@ -108,6 +159,7 @@ impl CredentialDelivery { match self { Self::Ambient => None, Self::File { env_key, .. } => env_key.as_deref(), + Self::CodexHome { .. } => None, } } @@ -147,9 +199,20 @@ impl CredentialDelivery { let mut allowed = PROFILE_COMMON_FIELDS.to_vec(); allowed.extend_from_slice(&["codex_home", "max_concurrent_invocations"]); reject_unknown_fields(profile, &allowed)?; - normalize_absolute_path(required_string(profile, "codex_home")?)?; - parse_max_concurrent_invocations(profile)?; - Err(undelivered(key)) + let path = normalize_absolute_path(required_string(profile, "codex_home")?) + .map_err(|_| HubModelConfigurationError::InvalidCredentialHome { + credential_profile: Arc::clone(name), + failure: CredentialHomeAdmissionFailure::InvalidPath, + })?; + admit_credential_home(name, &path)?; + let max_concurrent_invocations = parse_max_concurrent_invocations(profile)?; + if max_concurrent_invocations.is_some() { + return Err(HubModelConfigurationError::InvalidCredentialDelivery); + } + Ok(Self::CodexHome { + path, + max_concurrent_invocations, + }) } "oauth" => { let mut allowed = PROFILE_COMMON_FIELDS.to_vec(); @@ -168,6 +231,34 @@ impl CredentialDelivery { } } +fn admit_credential_home( + profile: &Arc, + path: &Path, +) -> Result<(), HubModelConfigurationError> { + if !path.is_dir() { + return Err(HubModelConfigurationError::InvalidCredentialHome { + credential_profile: Arc::clone(profile), + failure: CredentialHomeAdmissionFailure::MissingOrNotDirectory, + }); + } + let mut entries = + std::fs::read_dir(path).map_err(|_| HubModelConfigurationError::InvalidCredentialHome { + credential_profile: Arc::clone(profile), + failure: CredentialHomeAdmissionFailure::UnreadableDirectory, + })?; + match entries.next() { + Some(Ok(_)) => Ok(()), + Some(Err(_)) => Err(HubModelConfigurationError::InvalidCredentialHome { + credential_profile: Arc::clone(profile), + failure: CredentialHomeAdmissionFailure::UnreadableDirectory, + }), + None => Err(HubModelConfigurationError::InvalidCredentialHome { + credential_profile: Arc::clone(profile), + failure: CredentialHomeAdmissionFailure::EmptyDirectory, + }), + } +} + /// Rejects a `billing_kind` the profile's delivery cannot authenticate. /// /// Where a delivery fixes the authentication kind, the two fields cannot @@ -627,7 +718,7 @@ pub(crate) fn parse_credential_profiles( if tables.is_empty() { return Err(HubModelConfigurationError::MissingCredentialProfiles); } - let mut profiles = HashMap::with_capacity(tables.len()); + let mut profiles: HashMap, CredentialProfile> = HashMap::with_capacity(tables.len()); let mut ambient_adapters = HashSet::new(); let mut file_paths = HashSet::new(); for profile in tables { @@ -635,10 +726,32 @@ pub(crate) fn parse_credential_profiles( let adapter = ModelAdapter::parse(required_string(profile, "adapter")?)?; let billing_kind = BillingKind::parse(required_string(profile, "billing_kind")?)?; let delivery = CredentialDelivery::parse(profile, adapter, &name, billing_kind)?; + // The mixed-delivery rule is a property of one adapter's login store, + // so both sides of the pair must be Codex profiles. Another adapter's + // `ambient` profile shares no store with a Codex home and must not be + // matched here, or admission would turn on profile-table order. + let conflicts_with_codex_ambient = adapter == ModelAdapter::CodexCli + && profiles.values().any(|existing| { + existing.adapter() == ModelAdapter::CodexCli + && matches!( + (existing.delivery(), &delivery), + ( + CredentialDelivery::Ambient, + CredentialDelivery::CodexHome { .. } + ) | ( + CredentialDelivery::CodexHome { .. }, + CredentialDelivery::Ambient + ) + ) + }); + if conflicts_with_codex_ambient { + return Err(HubModelConfigurationError::InvalidCredentialDelivery); + } if delivery == CredentialDelivery::Ambient && !ambient_adapters.insert(adapter) { return Err(HubModelConfigurationError::InvalidCredentialDelivery); } - if let CredentialDelivery::File { path, .. } = &delivery + if let CredentialDelivery::File { path, .. } | CredentialDelivery::CodexHome { path, .. } = + &delivery && !file_paths.insert((adapter, path.clone())) { return Err(HubModelConfigurationError::InvalidCredentialDelivery); diff --git a/apps/signalboxd/src/daemon_tools.rs b/apps/signalboxd/src/daemon_tools.rs index 6d71546632..2168688c86 100644 --- a/apps/signalboxd/src/daemon_tools.rs +++ b/apps/signalboxd/src/daemon_tools.rs @@ -8,9 +8,12 @@ use std::{ collections::BTreeMap, error::Error, - fmt, fs, io, + fmt, fs, + future::Future, + io, os::unix::fs::MetadataExt, path::{Path, PathBuf}, + pin::Pin, sync::Arc, }; @@ -234,6 +237,61 @@ pub struct SessionWorkspaceRoots { derived_parent: PathBuf, } +type WorkspaceInstructionRootFuture<'a> = Pin< + Box> + Send + 'a>, +>; + +trait WorkspaceInstructionRootAuthority: Send + Sync { + fn resolve(&self, session: SessionId) -> WorkspaceInstructionRootFuture<'_>; +} + +/// Cloneable access to the workspace-binding authority used by daemon tools. +/// +/// Instruction discovery uses this handle so it cannot independently choose a +/// different configured-versus-derived root for a session whose binding is +/// already sticky. +#[derive(Clone)] +pub struct WorkspaceInstructionRootResolver { + authority: Arc, +} + +impl fmt::Debug for WorkspaceInstructionRootResolver { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("WorkspaceInstructionRootResolver") + .finish_non_exhaustive() + } +} + +impl WorkspaceInstructionRootResolver { + fn new( + executors: SessionWorkspaceExecutors, + ) -> Self + where + FileSystem: WorkspaceFileSystem + + WorkspaceMutationFileSystem + + PinFurtherWorkspaceRoot + + Send + + Sync + + 'static, + ExecRunner: ProcessRunner + Send + Sync + 'static, + { + Self { + authority: Arc::new(executors), + } + } + + pub(crate) async fn resolve( + &self, + session: SessionId, + ) -> Result { + self.authority.resolve(session).await + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) struct WorkspaceInstructionRootResolutionError; + impl SessionWorkspaceRoots { /// Fixes the derivation against one configured workspace root. /// @@ -1260,6 +1318,18 @@ where }) } + /// Shares the workspace-binding authority used by workspace-bound tools. + pub fn workspace_instruction_root_resolver(&self) -> Option + where + FileSystem: Send + Sync + 'static, + ExecRunner: Send + Sync + 'static, + { + self.executor + .workspace_bound + .clone() + .map(WorkspaceInstructionRootResolver::new) + } + /// Returns the catalog and executor as separate composition roles. #[allow(clippy::type_complexity)] pub fn into_parts( @@ -1857,6 +1927,26 @@ where }) } + async fn resolve_workspace_instruction_root( + &mut self, + session: SessionId, + ) -> Result { + let executors = self.resolve(session).await?; + let path = match self.state.lock().await.bindings.get(&session) { + Some(RecordedSessionBinding::ConfiguredRoot) => Ok(self.roots.configured().to_owned()), + Some(RecordedSessionBinding::DerivedRoot { .. }) => { + Ok(self.roots.derived_path(session)) + } + None => Err(SessionWorkspaceFailure::UnresolvableRoot), + }?; + let standing = ComposedWorkspaceIdentity::capture(&path) + .map_err(|_| SessionWorkspaceFailure::ReplacedRootIdentity)?; + if standing != executors.workspace_identity { + return Err(SessionWorkspaceFailure::ReplacedRootIdentity); + } + Ok(path) + } + async fn resolve( &mut self, session: SessionId, @@ -2159,6 +2249,28 @@ where } } +impl WorkspaceInstructionRootAuthority + for SessionWorkspaceExecutors +where + FileSystem: WorkspaceFileSystem + + WorkspaceMutationFileSystem + + PinFurtherWorkspaceRoot + + Send + + Sync + + 'static, + ExecRunner: ProcessRunner + Send + Sync + 'static, +{ + fn resolve(&self, session: SessionId) -> WorkspaceInstructionRootFuture<'_> { + let mut executors = self.clone(); + Box::pin(async move { + executors + .resolve_workspace_instruction_root(session) + .await + .map_err(|_| WorkspaceInstructionRootResolutionError) + }) + } +} + /// Name-directed daemon executor matching [`DaemonToolCatalog`]. #[derive(Clone, Debug)] pub struct DaemonToolExecutor< @@ -6880,6 +6992,39 @@ finally: .into_parts() } + fn offline_workspace_instruction_root_resolver( + workspace: &Path, + ) -> WorkspaceInstructionRootResolver { + let tools = DaemonTools::try_new( + || SystemTime::UNIX_EPOCH, + OfflineTransport, + MappedDaemonCredentialInputs { + web_search: OfflineCredentials, + code_host: OfflineCredentials, + github: OfflineCredentials, + }, + OfflineSearchTransport, + OfflineWriter, + OfflineCodeHostTransport, + OfflineGitHubTransport, + GitHubEgressPolicy::github_api_only(), + LocalWorkspaceFileSystem, + workspace, + git_identity(), + TokioProcessRunner::try_new( + std::env::current_exe().expect("test executable path is available"), + ) + .expect("test executable can stand in for the unused supervisor"), + OfflineConversationPort, + OfflineConversationPort, + WebFetchEgressPolicy::deny_all(), + ) + .expect("static daemon tools compile"); + tools + .workspace_instruction_root_resolver() + .expect("offline tools include a workspace binding authority") + } + /// The merged process-lifetime catalog exposes every daemon declaration in /// deterministic name order. #[test] @@ -7958,6 +8103,78 @@ finally: assert_eq!(path, expected); } + /// Instruction discovery consults the same sticky binding as workspace + /// tools, so provisioning a derived directory after the first resolution + /// cannot move an existing session away from the configured root. + #[tokio::test] + async fn instruction_discovery_keeps_an_existing_configured_binding() { + let parent = tempfile::tempdir().expect("fixture parent exists"); + let configured = configured_workspace(parent.path()); + let first = session(FIRST_SESSION_IDENTITY); + let resolver = offline_workspace_instruction_root_resolver(&configured); + + let initially_bound = resolver + .resolve(first) + .await + .expect("the configured root binds"); + provisioned_session_workspace(&configured, first, FIRST_SESSION_MARKER); + let after_provisioning = resolver + .resolve(first) + .await + .expect("the recorded configured binding remains usable"); + + assert_eq!(initially_bound, configured); + assert_eq!(after_provisioning, configured); + } + + /// A derived binding that disappears fails closed for instruction + /// discovery instead of falling back to the configured workspace. + #[tokio::test] + async fn instruction_discovery_refuses_a_lost_derived_binding() { + let parent = tempfile::tempdir().expect("fixture parent exists"); + let configured = configured_workspace(parent.path()); + let first = session(FIRST_SESSION_IDENTITY); + provisioned_session_workspace(&configured, first, FIRST_SESSION_MARKER); + let derived = derivation(&configured).derived_path(first); + let resolver = offline_workspace_instruction_root_resolver(&configured); + + let initially_bound = resolver + .resolve(first) + .await + .expect("the derived root binds"); + fs::remove_dir_all(&derived).expect("the bound derived root is removed"); + let after_removal = resolver.resolve(first).await; + + assert_eq!(initially_bound, derived); + assert_eq!(after_removal, Err(WorkspaceInstructionRootResolutionError)); + } + + /// Instruction discovery revalidates the pathname against the pinned tool + /// composition, so a replacement configured directory cannot be scanned + /// while tools continue to use the displaced directory descriptors. + #[tokio::test] + async fn instruction_discovery_refuses_a_replaced_configured_binding() { + let parent = tempfile::tempdir().expect("fixture parent exists"); + let configured = configured_workspace(parent.path()); + let displaced = parent.path().join("displaced-workspace"); + let first = session(FIRST_SESSION_IDENTITY); + let resolver = offline_workspace_instruction_root_resolver(&configured); + let initially_bound = resolver + .resolve(first) + .await + .expect("the configured root binds"); + fs::rename(&configured, &displaced).expect("the bound root is displaced"); + fs::create_dir(&configured).expect("a replacement directory takes its pathname"); + + let after_replacement = resolver.resolve(first).await; + + assert_eq!(initially_bound, configured); + assert_eq!( + after_replacement, + Err(WorkspaceInstructionRootResolutionError) + ); + } + /// One composition serves two concurrent sessions from two roots: each /// session's `read_file` observes only its own workspace, and neither /// observes the configured root every session shared before. diff --git a/apps/signalboxd/src/fenced_database.rs b/apps/signalboxd/src/fenced_database.rs index d73f5b2cc7..2e34fd955a 100644 --- a/apps/signalboxd/src/fenced_database.rs +++ b/apps/signalboxd/src/fenced_database.rs @@ -28,16 +28,23 @@ pub struct FencedHubDatabase { impl FencedHubDatabase { /// Opens a production database, establishes the singleton guard, fences the - /// prior generation, and returns only the new fenced pool. - pub async fn connect_production(database_url: &str) -> Result { + /// prior generation, and returns only the new fenced pool with the optional + /// deployment-owned connection floor. + pub async fn connect_production( + database_url: &str, + min_connections: Option, + ) -> Result { let options = production_connection_options(database_url) .map_err(FencedHubDatabaseError::ParseOptions)?; - Self::connect_with(options).await + Self::connect_with(options, min_connections).await } /// Establishes one guarded incarnation using already parsed connection /// options. This is also the local integration-test construction boundary. - pub async fn connect_with(options: PgConnectOptions) -> Result { + pub async fn connect_with( + options: PgConnectOptions, + min_connections: Option, + ) -> Result { let bootstrap = PgPoolOptions::new() .max_connections(1) .connect_with(options.clone()) @@ -54,7 +61,7 @@ impl FencedHubDatabase { .map_err(FencedHubDatabaseError::AdvanceFence)?; bootstrap.close().await; let pool = advanced_fence - .connect_pool(options) + .connect_pool(options, min_connections) .await .map_err(FencedHubDatabaseError::ConnectFencedPool)?; let generation = advanced_fence.generation(); @@ -113,6 +120,46 @@ impl Drop for FencedHubDatabase { } } +/// Result of one non-disruptive fenced-pool floor reconciliation. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum FencedPoolFloorReconciliation { + /// The physical-session floor was already present. + Satisfied, + /// Idle service capacity was preserved instead of being held by maintenance. + DeferredForIdleCapacity, + /// Demand had exhausted idle capacity, so one missing session was opened. + Replenished, +} + +/// Reopens one missing physical session without consuming idle service +/// capacity. +/// +/// SQLx opens a session only after its idle inventory is empty. Reconciliation +/// therefore defers while any idle session remains; holding that inventory to +/// force construction would steal capacity from ordinary work for the whole +/// connection attempt. Once demand has consumed the idle inventory, one +/// acquisition opens one missing session and immediately returns it. Callers +/// own the attempt deadline and repeat bound. +pub async fn reconcile_fenced_pool_floor( + pool: &PgPool, + minimum: u32, +) -> Result { + let current = pool.size(); + if current >= minimum { + return Ok(FencedPoolFloorReconciliation::Satisfied); + } + if pool.num_idle() > 0 { + return Ok(FencedPoolFloorReconciliation::DeferredForIdleCapacity); + } + let connection = pool.acquire().await?; + drop(connection); + Ok(if pool.size() > current { + FencedPoolFloorReconciliation::Replenished + } else { + FencedPoolFloorReconciliation::DeferredForIdleCapacity + }) +} + /// Sanitized guarded-database startup failure. #[derive(Debug)] pub enum FencedHubDatabaseError { diff --git a/apps/signalboxd/src/goal_mode.rs b/apps/signalboxd/src/goal_mode.rs index 223e4b812c..4f1986865b 100644 --- a/apps/signalboxd/src/goal_mode.rs +++ b/apps/signalboxd/src/goal_mode.rs @@ -11,13 +11,15 @@ use signalbox_application::{ }; use signalbox_domain::{ AcceptedInputId, DurableCommandId, Goal, GoalBlockProvenance, GoalCommandResult, GoalEvent, - GoalEventKind, GoalEventOrdinal, GoalModelBlockedReasonKind, GoalModelProvenance, GoalNeed, - GoalReport, GoalSchedulerProvenance, GoalUserAction, GoalUserCommand, NormalizedToolArguments, - SessionId, ToolEffectClass, ToolExecutionErrorDetail, ToolName, ToolPermissionDefault, TurnId, + GoalEventKind, GoalEventOrdinal, GoalGuidance, GoalModelBlockedReasonKind, GoalModelProvenance, + GoalNeed, GoalReport, GoalSchedulerProvenance, GoalTextError, GoalUserAction, GoalUserCommand, + NormalizedToolArguments, SessionId, ToolEffectClass, ToolExecutionErrorDetail, ToolName, + ToolPermissionDefault, TurnId, }; use signalbox_persistence::{ goal::{ - GoalCommandHandlingOutcome, GoalRepository, GoalRepositoryError, GoalTransitionOutcome, + GoalCommandHandlingOutcome, GoalExecutionFailureRecoveryCause, GoalRepository, + GoalRepositoryError, GoalTransitionOutcome, }, goal_turn::{GoalTurnCandidates, GoalTurnContinuationOutcome}, }; @@ -62,6 +64,7 @@ const GOAL_DECLARE_REJECTED: &str = const GOAL_DECLARE_RESULT: &str = "{\"status\":\"applied\"}"; const EXECUTION_FAILURE_NEED: &str = "Resolve the failed goal turn's execution condition, then resume the goal."; +const CONTEXT_COMPACTION_INPUT_DOES_NOT_FIT_NEED: &str = "No safe context-compaction boundary fits the configured model window. Start a fresh session or reduce the imported context before resuming this goal; no automatic resumption is scheduled."; /// Preamble for an execution-failure block automatic resumption still owes. /// /// The repair follows it rather than replacing it with a promise of automation, @@ -69,6 +72,8 @@ const EXECUTION_FAILURE_NEED: &str = /// a durably rejected command, a daemon restart, an unreachable database — /// leaves this text as what the operator reads. const EXECUTION_FAILURE_RESUMING_PREAMBLE: &str = "The goal turn failed to execute and automatic resumption is scheduled. If the goal is still blocked here once resumption ends, it is waiting for an operator."; +/// Guidance for a failure the session caused and should not repeat unchanged. +const CHARGEABLE_FAILURE_RESUME_GUIDANCE: &str = "Continue pursuing the commissioned goal. The preceding turn failed to execute. Inspect the durable session state and choose a different safe approach before repeating the failed operation."; /// Retries one armed attempt may spend on a database that answers nothing. /// /// These are not resumptions and do not spend the attempt budget: nothing was @@ -566,15 +571,27 @@ impl PostgresGoalPassDisposition { blocked: GoalEventOrdinal, resumption: AutomaticResumption, ) { - let AutomaticResumption::Scheduled { delay } = resumption else { - tracing::warn!( - session = %session.into_uuid(), - event_ordinal = blocked.get(), - attempt_budget = ?self.numeric_bounds.attempt_budget, - cause_code = "goal_automatic_resume_exhausted", - "blocked goal exhausted automatic resumption and awaits an operator" - ); - return; + let delay = match resumption { + AutomaticResumption::Scheduled { delay } => delay, + AutomaticResumption::Exhausted { .. } => { + tracing::warn!( + session = %session.into_uuid(), + event_ordinal = blocked.get(), + attempt_budget = ?self.numeric_bounds.attempt_budget, + cause_code = "goal_automatic_resume_exhausted", + "blocked goal exhausted automatic resumption and awaits an operator" + ); + return; + } + AutomaticResumption::OperatorRequired { cause } => { + tracing::warn!( + session = %session.into_uuid(), + event_ordinal = blocked.get(), + cause_code = cause.code(), + "blocked goal has a durable non-resumable execution failure and awaits an operator" + ); + return; + } }; let adapter = self.clone(); drop(tokio::spawn(async move { @@ -596,20 +613,23 @@ impl PostgresGoalPassDisposition { async fn resume_after_execution_failure(&self, session: SessionId, blocked: GoalEventOrdinal) { let mut remaining = AUTOMATIC_RESUME_INFRASTRUCTURE_RETRIES; loop { - if self.attempt_automatic_resume(session, blocked).await == ResumeAttempt::Settled { - return; - } - if remaining == 0 { - tracing::error!( - session = %session.into_uuid(), - event_ordinal = blocked.get(), - retries = AUTOMATIC_RESUME_INFRASTRUCTURE_RETRIES, - cause_code = "goal_automatic_resume_abandoned", - "automatic goal resumption abandoned a blocked goal to the operator" - ); - return; + match self.attempt_automatic_resume(session, blocked).await { + ResumeAttempt::Settled => return, + ResumeAttempt::OwnershipDeferred => {} + ResumeAttempt::InfrastructureUnsettled => { + if remaining == 0 { + tracing::error!( + session = %session.into_uuid(), + event_ordinal = blocked.get(), + retries = AUTOMATIC_RESUME_INFRASTRUCTURE_RETRIES, + cause_code = "goal_automatic_resume_abandoned", + "automatic goal resumption abandoned a blocked goal to the operator" + ); + return; + } + remaining = remaining.saturating_sub(1); + } } - remaining = remaining.saturating_sub(1); sleep_for_policy(self.numeric_bounds.base_backoff).await; } } @@ -630,16 +650,60 @@ impl PostgresGoalPassDisposition { cause = %error, "automatic goal resumption cannot confirm the goal is still blocked" ); - return ResumeAttempt::Unsettled; + return ResumeAttempt::InfrastructureUnsettled; } }; - if !reread.is_some_and(|goal| awaits_automatic_resumption(&goal, blocked)) { + let Some(goal) = reread else { + return ResumeAttempt::Settled; + }; + if !awaits_automatic_resumption(&goal, blocked) { return ResumeAttempt::Settled; } + let Some(failed_turn) = goal.events().last().and_then(execution_failure_turn) else { + tracing::error!( + session = %session.into_uuid(), + event_ordinal = blocked.get(), + cause_code = "goal_automatic_resume_failure_turn_missing", + "automatic goal resumption could not identify its blocked turn" + ); + return ResumeAttempt::InfrastructureUnsettled; + }; + let unchargeable = match self + .repository + .unchargeable_automatic_resume_turns(session, &[failed_turn]) + .await + { + Ok(turns) => turns.contains(&failed_turn), + Err(error) => { + tracing::error!( + session = %session.into_uuid(), + turn = %failed_turn.into_uuid(), + event_ordinal = blocked.get(), + cause_code = "goal_automatic_resume_failure_classification_failed", + cause = %error, + "automatic goal resumption could not classify its failed turn" + ); + return ResumeAttempt::InfrastructureUnsettled; + } + }; + let guidance = match automatic_resume_guidance(unchargeable) { + Ok(guidance) => guidance, + Err(error) => { + tracing::error!( + session = %session.into_uuid(), + event_ordinal = blocked.get(), + cause_code = "goal_automatic_resume_guidance_invalid", + cause = %error, + "automatic goal resumption could not construct its static guidance" + ); + return ResumeAttempt::InfrastructureUnsettled; + } + }; + let strategy_guidance = guidance.is_some(); let command = GoalUserCommand::new( automatic_resume_command(session, blocked), session, - GoalUserAction::Resume(None), + GoalUserAction::Resume(guidance), ); let candidates = GoalTurnCandidates::new( AcceptedInputId::from_uuid(Uuid::now_v7()), @@ -663,6 +727,7 @@ impl PostgresGoalPassDisposition { session = %session.into_uuid(), event_ordinal = event.ordinal().get(), blocked_event_ordinal = blocked.get(), + strategy_guidance, "automatically resumed a goal blocked by execution failure" ); ResumeAttempt::Settled @@ -688,6 +753,17 @@ impl PostgresGoalPassDisposition { ); ResumeAttempt::Settled } + Ok(GoalCommandHandlingOutcome::TargetBusy { + session: blocking_session, + }) => { + tracing::info!( + session = %session.into_uuid(), + blocking_session = %blocking_session.into_uuid(), + event_ordinal = blocked.get(), + "automatic goal resumption deferred behind another commissioned session" + ); + ResumeAttempt::OwnershipDeferred + } Ok(GoalCommandHandlingOutcome::ConflictingReuse { .. }) => { tracing::error!( session = %session.into_uuid(), @@ -705,7 +781,7 @@ impl PostgresGoalPassDisposition { cause = %error, "automatic goal resumption could not be recorded" ); - ResumeAttempt::Unsettled + ResumeAttempt::InfrastructureUnsettled } } } @@ -815,8 +891,10 @@ fn commit_is_ambiguous(error: &PostgresGoalPassDispositionError) -> bool { enum ResumeAttempt { /// The attempt resumed, was refused, or found nothing left to answer. Settled, - /// The database prevented any answer, so the attempt is still owed. - Unsettled, + /// Another live target session deferred the attempt without spending a retry. + OwnershipDeferred, + /// Infrastructure prevented any answer, so the bounded retry is still owed. + InfrastructureUnsettled, } impl GoalPassDisposition for PostgresGoalPassDisposition { @@ -870,9 +948,18 @@ impl GoalPassDisposition for PostgresGoalPassDisposition { ) -> impl Future> + Send + 'static { let adapter = self.clone(); async move { - let resumption = adapter - .plan_automatic_resumption(session, Some(turn)) - .await?; + let resumption = match adapter + .repository + .execution_failure_recovery_cause(session, turn) + .await? + { + Some(cause) => AutomaticResumption::OperatorRequired { cause }, + None => { + adapter + .plan_automatic_resumption(session, Some(turn)) + .await? + } + }; let outcome = match adapter .repository .block_execution_failure( @@ -926,6 +1013,11 @@ enum AutomaticResumption { }, /// The consecutive-attempt budget is spent; only an operator can resume. Exhausted { attempt_budget: u32 }, + /// Durable failure evidence proves unchanged automatic resumption cannot progress. + OperatorRequired { + /// Exact recorded reason the automatic path cannot make progress. + cause: GoalExecutionFailureRecoveryCause, + }, } impl AutomaticResumption { @@ -954,6 +1046,9 @@ impl AutomaticResumption { Self::Exhausted { attempt_budget } => format!( "Automatic resumption is exhausted after {attempt_budget} consecutive execution failures. {EXECUTION_FAILURE_NEED}" ), + Self::OperatorRequired { + cause: GoalExecutionFailureRecoveryCause::ContextCompactionInputDoesNotFit, + } => String::from(CONTEXT_COMPACTION_INPUT_DOES_NOT_FIT_NEED), }; GoalNeed::try_new(text).map_err(|_| PostgresGoalPassDispositionError::InvalidStaticNeed) } @@ -1024,6 +1119,13 @@ fn chargeable_automatic_resume_attempts( u32::try_from(spent).unwrap_or(u32::MAX) } +fn automatic_resume_guidance(unchargeable: bool) -> Result, GoalTextError> { + if unchargeable { + return Ok(None); + } + GoalGuidance::try_new(String::from(CHARGEABLE_FAILURE_RESUME_GUIDANCE)).map(Some) +} + /// Whether the goal is still blocked by exactly the named failure event. fn awaits_automatic_resumption(goal: &Goal, blocked: GoalEventOrdinal) -> bool { goal.events() @@ -1357,15 +1459,41 @@ mod tests { } .need() .expect("the exhausted need is admitted"); + let operator_required = AutomaticResumption::OperatorRequired { + cause: GoalExecutionFailureRecoveryCause::ContextCompactionInputDoesNotFit, + } + .need() + .expect("the operator-required need is admitted"); assert!(scheduled.as_str().ends_with(EXECUTION_FAILURE_NEED)); assert!(exhausted.as_str().ends_with(EXECUTION_FAILURE_NEED)); + assert_eq!( + operator_required.as_str(), + CONTEXT_COMPACTION_INPUT_DOES_NOT_FIT_NEED + ); assert_eq!( scheduled.as_str(), "The goal turn failed to execute and automatic resumption is scheduled. If the goal is still blocked here once resumption ends, it is waiting for an operator. Resolve the failed goal turn's execution condition, then resume the goal." ); } + #[test] + fn a_chargeable_failure_changes_the_next_turn_input() { + let guidance = automatic_resume_guidance(false) + .expect("the static guidance is admitted") + .expect("a chargeable failure carries guidance"); + + assert_eq!(guidance.as_str(), CHARGEABLE_FAILURE_RESUME_GUIDANCE); + } + + #[test] + fn an_unchargeable_failure_reuses_the_commissioned_statement() { + assert_eq!( + automatic_resume_guidance(true).expect("no guidance needs admission"), + None + ); + } + #[test] fn an_operator_resume_restarts_the_attempt_budget() { let after_operator = failed( diff --git a/apps/signalboxd/src/lib.rs b/apps/signalboxd/src/lib.rs index 08e58e82c2..5fc6dd4d59 100644 --- a/apps/signalboxd/src/lib.rs +++ b/apps/signalboxd/src/lib.rs @@ -12,10 +12,11 @@ use signalbox_application::{ EligibilityNudge, EligibilityPass, InProcessAttemptDispatchGate, InProcessToolDispatchGate, ModelCallExecutionError, ModelCallExecutionOutcome, ModelCallExecutionService, ModelCallProvider, OperatorFailureClass, SchedulerPassExpiryHandler, ScriptedModelCallError, - ScriptedModelCallProvider, ScriptedModelCallStep, StartEligibleTurnIdGenerator, - StartEligibleTurnOutcome, StartEligibleTurnService, StartEligibleTurnTransaction, ToolCatalog, - ToolExecutionService, ToolExecutionServiceError, ToolExecutionServiceOutcome, ToolExecutor, - UuidV7ModelCallExecutionIdGenerator, UuidV7StartupScanIdGenerator, UuidV7ToolLoopIdGenerator, + ScriptedModelCallProvider, ScriptedModelCallStep, StaleTurnCandidate, + StartEligibleTurnIdGenerator, StartEligibleTurnOutcome, StartEligibleTurnService, + StartEligibleTurnTransaction, ToolCatalog, ToolExecutionService, ToolExecutionServiceError, + ToolExecutionServiceOutcome, ToolExecutor, UuidV7ModelCallExecutionIdGenerator, + UuidV7StartupScanIdGenerator, UuidV7ToolLoopIdGenerator, }; use signalbox_domain::{ ActivatedTurn, AssistantText, ContextFrontierId, DirectModelSelection, ModelCallId, @@ -45,6 +46,7 @@ use tokio::{ use tracing::Instrument; pub mod approval_judge_eval; +mod attachment_preparation_runtime; mod blob_read_runtime; mod blob_storage_configuration; mod blob_storage_runtime; @@ -71,7 +73,10 @@ mod telemetry; mod turn_liveness_runtime; pub mod usage_limits; pub mod web_http; +mod web_imports; +mod workspace_instruction_runtime; +pub use attachment_preparation_runtime::AttachmentPreparingModelCallProvider; pub use blob_storage_configuration::{ BlobStorageClass, BlobStorageConfiguration, BlobStorageConfigurationError, BlobStoreConfiguration, @@ -82,6 +87,7 @@ pub use configuration::{ DaemonToolConfiguration, DerivedModelCallCost, FileCredentialAccess, HubModelConfiguration, HubModelConfigurationError, ModelAdapter, ModelBillingRates, NumericBoundsConfiguration, OPENAI_CREDENTIAL_REFERENCE, RepositoryWatchConfiguration, WatchedRepositoryConfiguration, + WorkspaceInstructionConfiguration, }; pub use context_guard::{ ContextGuardedTurnPass, ContextGuardedTurnPassError, ReportedUsageCompaction, @@ -95,22 +101,28 @@ pub use conversation_introspection::{ ConversationIntrospectionError, PostgresConversationIntrospection, }; pub use credential_pools::{ - CredentialDelivery, CredentialPool, CredentialPoolAction, CredentialPoolExhaustion, - CredentialPoolMember, CredentialPoolTieBreak, CredentialPoolTrigger, CredentialProfile, + CredentialDelivery, CredentialHomeAdmissionFailure, CredentialPool, CredentialPoolAction, + CredentialPoolExhaustion, CredentialPoolMember, CredentialPoolTieBreak, CredentialPoolTrigger, + CredentialProfile, }; pub use daemon_tools::{ BaseDaemonCredentialInputs, ConfiguredApprovalPostureError, DaemonToolCatalog, DaemonToolComposition, DaemonToolExecutor, DaemonToolExecutorError, DaemonTools, DaemonToolsConstructionError, MappedDaemonCredentialInputs, PinnedWorkspaceFileSystem, + SessionWorkspaceRoots, WorkspaceInstructionRootResolver, +}; +pub use fenced_database::{ + FencedHubDatabase, FencedHubDatabaseError, FencedPoolFloorReconciliation, + reconcile_fenced_pool_floor, }; -pub use fenced_database::{FencedHubDatabase, FencedHubDatabaseError}; pub use goal_mode::{ GoalModeNumericBounds, PostgresGoalPassDisposition, PostgresGoalPassDispositionError, }; pub use local_socket::{LocalProcessListener, LocalSocketError}; pub use process_runtime::{ProcessProviderTextDeltaSink, ProcessRuntime, ProcessRuntimeError}; pub use repo_watch_runtime::{ - RepositoryWatchRuntime, RepositoryWatchRuntimeConstructionError, RepositoryWatchRuntimeError, + RepositoryWatchNumericBounds, RepositoryWatchRuntime, RepositoryWatchRuntimeConstructionError, + RepositoryWatchRuntimeError, }; pub use session_delegation::{PostgresSessionDelegationPort, PostgresSessionDelegationPortError}; pub use session_template_configuration::{ @@ -187,6 +199,9 @@ pub use telemetry::{ TelemetryExportFilter, TelemetryExportLayer, TelemetryMetrics, }; pub use turn_liveness_runtime::{TurnLivenessNumericBounds, TurnLivenessRuntime}; +pub use workspace_instruction_runtime::{ + WorkspaceInstructionRuntime, WorkspaceInstructionRuntimeError, +}; /// Per-activation model execution constructed by the hub composition root. pub trait ActivatedTurnExecution { @@ -199,6 +214,15 @@ pub trait ActivatedTurnExecution { activated: Box, ) -> impl Future> + Send + 'static; + /// Drives a dispatch-start activation only through its first durable call + /// checkpoint so reserved scheduler admission can be released. + fn execute_dispatch_start( + &self, + activated: Box, + ) -> impl Future> + Send + 'static { + self.execute(activated) + } + /// Reports whether a returned initial-execution failure may require /// startup recovery rather than ordinary scheduler disposition. /// @@ -238,6 +262,39 @@ pub trait ActivatedTurnExecution { self.resume_active(session) } + /// Reconciles an active evidence-free turn through its first call checkpoint. + fn resume_dispatch_start( + &self, + session: SessionId, + ) -> impl Future> + Send + 'static { + self.resume_active(session) + } + + /// Reconciles an active turn through a shareable exact-turn observer. + fn resume_active_with_observer( + &self, + session: SessionId, + observe: std::sync::Arc, + ) -> impl Future> + Send + 'static { + self.resume_active_observing(session, move |turn| observe(turn)) + } + + /// Reconciles an active evidence-free turn through its first call + /// checkpoint while reporting its identity before resumed execution + /// begins. + /// + /// A dispatch-start hint that recovers an already-active turn must report + /// that turn for the same reason the active-resume path does: occupancy + /// recovery can only hand an expired pass off for repair when it knows + /// which turn the pass was occupying. + fn resume_dispatch_start_with_observer( + &self, + session: SessionId, + observe_turn: std::sync::Arc, + ) -> impl Future> + Send + 'static { + self.resume_active_with_observer(session, observe_turn) + } + /// Reports whether a failed active-turn resume may require startup /// recovery rather than ordinary scheduler retry. /// @@ -265,6 +322,144 @@ pub trait ActivatedTurnExecution { } } +/// Failure while preparing one turn's instruction record or running its +/// delegated execution. +#[derive(Debug)] +pub enum WorkspaceInstructionPreparedExecutionError { + /// Discovery or durable turn-manifest recording failed. + WorkspaceInstructions(WorkspaceInstructionRuntimeError), + /// The wrapped execution failed after instruction preparation. + Execution(ExecutionError), +} + +impl fmt::Display for WorkspaceInstructionPreparedExecutionError +where + ExecutionError: fmt::Display, +{ + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::WorkspaceInstructions(error) => error.fmt(formatter), + Self::Execution(error) => error.fmt(formatter), + } + } +} + +impl Error for WorkspaceInstructionPreparedExecutionError +where + ExecutionError: Error + 'static, +{ + fn source(&self) -> Option<&(dyn Error + 'static)> { + match self { + Self::WorkspaceInstructions(error) => Some(error), + Self::Execution(error) => Some(error), + } + } +} + +impl ClassifyOperatorFailure + for WorkspaceInstructionPreparedExecutionError +where + ExecutionError: ClassifyOperatorFailure, +{ + fn operator_failure_class(&self) -> OperatorFailureClass { + match self { + Self::WorkspaceInstructions(error) => error.operator_failure_class(), + Self::Execution(error) => error.operator_failure_class(), + } + } + + fn operator_failure_cause_code(&self) -> &'static str { + match self { + Self::WorkspaceInstructions(error) => error.operator_failure_cause_code(), + Self::Execution(error) => error.operator_failure_cause_code(), + } + } +} + +/// Adds daemon-owned instruction discovery and turn-manifest recording before +/// an activated-turn execution that does not own the provider/tool loop. +#[derive(Clone, Debug)] +pub struct WorkspaceInstructionPreparedExecution { + execution: Execution, + workspace_instructions: WorkspaceInstructionRuntime, +} + +impl WorkspaceInstructionPreparedExecution { + /// Wraps one execution with the exact instruction runtime it must use. + pub const fn new( + execution: Execution, + workspace_instructions: WorkspaceInstructionRuntime, + ) -> Self { + Self { + execution, + workspace_instructions, + } + } +} + +impl ActivatedTurnExecution for WorkspaceInstructionPreparedExecution +where + Execution: ActivatedTurnExecution + Clone + Send + 'static, +{ + type Error = WorkspaceInstructionPreparedExecutionError; + + fn execute( + &self, + activated: Box, + ) -> impl Future> + Send + 'static { + let execution = self.execution.clone(); + let workspace_instructions = self.workspace_instructions.clone(); + async move { + if !workspace_instructions + .prepare(activated.session(), activated.turn()) + .await + .map_err(WorkspaceInstructionPreparedExecutionError::WorkspaceInstructions)? + { + return Ok(()); + } + execution + .execute(activated) + .await + .map_err(WorkspaceInstructionPreparedExecutionError::Execution) + } + } + + fn resume_active( + &self, + session: SessionId, + ) -> impl Future> + Send + 'static { + let execution = self.execution.clone(); + async move { + execution + .resume_active(session) + .await + .map_err(WorkspaceInstructionPreparedExecutionError::Execution) + } + } + + fn active_resume_failure_requires_recovery(error: &Self::Error) -> bool { + match error { + WorkspaceInstructionPreparedExecutionError::WorkspaceInstructions(_) => true, + WorkspaceInstructionPreparedExecutionError::Execution(error) => { + Execution::active_resume_failure_requires_recovery(error) + } + } + } + + fn active_resume_failure_turn(error: &Self::Error) -> Option { + match error { + WorkspaceInstructionPreparedExecutionError::WorkspaceInstructions(_) => None, + WorkspaceInstructionPreparedExecutionError::Execution(error) => { + Execution::active_resume_failure_turn(error) + } + } + } + + fn report_post_activation_failure(&self) { + self.execution.report_post_activation_failure(); + } +} + /// Cheap-clone handle that raises the daemon's fatal recovery signal. /// /// The scheduler pass reaches the signal through its execution role, but the @@ -380,6 +575,21 @@ where ) } + fn execute_dispatch_start( + &self, + activated: Box, + ) -> impl Future> + Send + 'static { + let session = activated.session(); + let execution = self.execution.execute_dispatch_start(activated); + supervise_execution_for_session( + self.fatal_signal.clone(), + std::sync::Arc::clone(&self.bounded_expirations), + session, + execution, + Execution::execution_failure_requires_recovery, + ) + } + fn resume_active( &self, session: SessionId, @@ -410,6 +620,35 @@ where ) } + fn resume_dispatch_start( + &self, + session: SessionId, + ) -> impl Future> + Send + 'static { + let execution = self.execution.resume_dispatch_start(session); + supervise_active_resume::( + self.fatal_signal.clone(), + std::sync::Arc::clone(&self.bounded_expirations), + session, + execution, + ) + } + + fn resume_dispatch_start_with_observer( + &self, + session: SessionId, + observe_turn: std::sync::Arc, + ) -> impl Future> + Send + 'static { + let execution = self + .execution + .resume_dispatch_start_with_observer(session, observe_turn); + supervise_active_resume::( + self.fatal_signal.clone(), + std::sync::Arc::clone(&self.bounded_expirations), + session, + execution, + ) + } + fn active_resume_failure_requires_recovery(error: &Self::Error) -> bool { Execution::active_resume_failure_requires_recovery(error) } @@ -825,24 +1064,56 @@ struct SchedulerPassOccupancyRecovery { pool: sqlx::PgPool, eligibility_nudge: signalbox_application::InProcessEligibilityNudge, execution_expiry: Option>, - expected_turns: std::sync::Arc>>, + active_turns: std::sync::Arc>>, policy: ExpiredPassRecoveryPolicy, persistence_bounds: TurnLivenessPersistenceBounds, } -impl SchedulerPassOccupancyRecovery { - fn expect_turn(&self, session: SessionId, turn: TurnId) { - self.expected_turns +#[derive(Debug)] +struct SchedulerPassActiveTurnGuard { + active_turns: std::sync::Arc>>, + session: SessionId, +} + +impl Drop for SchedulerPassActiveTurnGuard { + fn drop(&mut self) { + self.active_turns .lock() .unwrap_or_else(std::sync::PoisonError::into_inner) - .insert(session, turn); + .remove(&self.session); } +} - fn clear_turn(&self, session: SessionId) { - self.expected_turns +impl SchedulerPassOccupancyRecovery { + fn active_turn(&self, session: SessionId) -> Option { + self.active_turns .lock() .unwrap_or_else(std::sync::PoisonError::into_inner) - .remove(&session); + .get(&session) + .copied() + } + + fn resume_turn_observer( + &self, + session: SessionId, + ) -> ( + SchedulerPassActiveTurnGuard, + std::sync::Arc, + ) { + let active_turns = std::sync::Arc::clone(&self.active_turns); + let observer = std::sync::Arc::new(move |turn| { + active_turns + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .insert(session, turn); + }); + ( + SchedulerPassActiveTurnGuard { + active_turns: std::sync::Arc::clone(&self.active_turns), + session, + }, + observer, + ) } fn nudge(&self, session: SessionId) { @@ -855,12 +1126,7 @@ impl SchedulerPassExpiryHandler for SchedulerPassOccupancyRecovery { if let Some(execution_expiry) = &self.execution_expiry { execution_expiry.occupancy_expired(session); } - let expected_turn = self - .expected_turns - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner) - .remove(&session); - if let Some(expected_turn) = expected_turn { + if let Some(expected_turn) = self.active_turn(session) { drop(tokio::spawn(recover_expired_scheduler_pass( self.clone(), session, @@ -915,7 +1181,7 @@ impl ActivatedTurnPass impl Future> + Send + 'static { - let activation = self.activation.execute_with_cloned_transaction(session); let execution = self.execution.clone(); let occupancy_recovery = self.occupancy_recovery.clone(); let reported_usage_compaction = self.reported_usage_compaction.clone(); + let occupancy_tracking = occupancy_recovery + .as_ref() + .map(|recovery| recovery.resume_turn_observer(session)); + let observe_turn = occupancy_tracking + .as_ref() + .map(|(_, observer)| std::sync::Arc::clone(observer)) + .unwrap_or_else(|| std::sync::Arc::new(|_| {})); + let activation = self + .activation + .execute_with_cloned_transaction_and_observer( + session, + std::sync::Arc::clone(&observe_turn), + ); async move { - let resumed_turn_recovery = occupancy_recovery.clone(); if let Err(source) = execution - .resume_active_observing(session, move |turn| { - if let Some(recovery) = resumed_turn_recovery { - recovery.expect_turn(session, turn); - } - }) + .resume_active_with_observer(session, std::sync::Arc::clone(&observe_turn)) .await { - if let Some(recovery) = &occupancy_recovery { - recovery.clear_turn(session); - } return Err(ActivatedTurnPassError::Execution { stage: TurnPassExecutionStage::ActiveTurnRecovery, turn: Execution::active_resume_failure_turn(&source), source, }); } - if let Some(compaction) = reported_usage_compaction { - compaction - .compact_if_needed(session) - .await - .map_err(ActivatedTurnPassError::ReportedUsageCompaction)?; + if let Some(compaction) = reported_usage_compaction + && let Err(error) = compaction.compact_if_needed(session).await + { + return Err(reported_usage_compaction_failure(&execution, error)); } let outcome = match activation.await { Ok(outcome) => outcome, Err(error) => { - if let Some(recovery) = &occupancy_recovery { - recovery.clear_turn(session); - } report_ambiguous_commit(&execution, &error); return Err(ActivatedTurnPassError::Activation(error)); } }; - match outcome { - StartEligibleTurnOutcome::NoEligibleTurn => { - if let Some(recovery) = &occupancy_recovery { - recovery.clear_turn(session); - } - Ok(()) - } + let result = match outcome { + StartEligibleTurnOutcome::NoEligibleTurn => Ok(()), StartEligibleTurnOutcome::Activated(activated) => { let turn = activated.turn(); - if let Some(recovery) = &occupancy_recovery { - recovery.expect_turn(session, turn); - } if !activation_session_matches(&execution, session, activated.session()) { - if let Some(recovery) = &occupancy_recovery { - recovery.clear_turn(session); - } return Err(ActivatedTurnPassError::ActivationSessionMismatch); } - let result = execution + execution .execute(activated) .instrument(turn_work_span(session, turn)) .await @@ -1031,17 +1286,237 @@ where stage: TurnPassExecutionStage::Execution, turn: Some(turn), source, - }); - if let Some(recovery) = &occupancy_recovery { - recovery.clear_turn(session); + }) + } + }; + drop(occupancy_tracking); + result + } + } + + fn run_dispatch_start( + &mut self, + session: SessionId, + ) -> impl Future> + Send + 'static { + let execution = self.execution.clone(); + let occupancy_recovery = self.occupancy_recovery.clone(); + let occupancy_tracking = occupancy_recovery + .as_ref() + .map(|recovery| recovery.resume_turn_observer(session)); + let observe_turn = occupancy_tracking + .as_ref() + .map(|(_, observer)| std::sync::Arc::clone(observer)) + .unwrap_or_else(|| std::sync::Arc::new(|_| {})); + let activation = self + .activation + .execute_with_cloned_transaction_and_observer( + session, + std::sync::Arc::clone(&observe_turn), + ); + async move { + execution + .resume_dispatch_start_with_observer(session, observe_turn) + .await + .map_err(|source| ActivatedTurnPassError::Execution { + stage: TurnPassExecutionStage::ActiveTurnRecovery, + turn: Execution::active_resume_failure_turn(&source), + source, + })?; + let outcome = match activation.await { + Ok(outcome) => outcome, + Err(error) => { + report_ambiguous_commit(&execution, &error); + return Err(ActivatedTurnPassError::Activation(error)); + } + }; + let result = match outcome { + StartEligibleTurnOutcome::NoEligibleTurn => Ok(()), + StartEligibleTurnOutcome::Activated(activated) => { + let turn = activated.turn(); + if !activation_session_matches(&execution, session, activated.session()) { + return Err(ActivatedTurnPassError::ActivationSessionMismatch); } - result + execution + .execute_dispatch_start(activated) + .instrument(turn_work_span(session, turn)) + .await + .map_err(|source| ActivatedTurnPassError::Execution { + stage: TurnPassExecutionStage::Execution, + turn: Some(turn), + source, + }) } - } + }; + drop(occupancy_tracking); + result + } + } +} + +/// What one inventory observation settles about an expired pass's turn. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum ExpiredPassObservation { + /// The expected turn was seen once. Nothing is settled: one observation + /// cannot distinguish a wedged turn from a working one. + AwaitingConfirmation(StaleTurnCandidate), + /// Durable evidence stood still between two observations, so the turn is + /// wedged and recovery may terminalize it. + Confirmed(StaleTurnCandidate), + /// Durable evidence advanced between two observations, so the expired pass + /// was progressing and its turn must be left alone. + Progressing { + /// The evidence this path proposed the turn on. + previous: StaleTurnCandidate, + /// The later evidence that advanced past it. + observed: StaleTurnCandidate, + }, + /// Another turn holds the session's slot now. + Superseded(TurnId), + /// The session holds no recoverable active turn. + Absent, +} + +/// Decides what one expiry observation settles, given the previous one. +/// +/// The occupancy ceiling bounds a pass's tenure, which is not the same claim as +/// "this turn stopped progressing": one admitted pass drives a turn's whole +/// model/tools loop, including provider retry-backoff sleeps, so a turn making +/// continuous durable progress can reach the ceiling. Recovery never re-admits +/// the pass it replaced, so terminalizing on tenure alone would fail a healthy +/// turn outright. This is the unchanged-evidence requirement both liveness +/// watchdogs impose and the ceiling by itself lacks: the turn is terminalized +/// only once its evidence — the attempt holding its tenure and the session's +/// outbox frontier — has stood still across a whole confirmation delay. +fn classify_expired_pass_observation( + expected_turn: TurnId, + unconfirmed: Option, + observed: Option, +) -> ExpiredPassObservation { + let Some(observed) = observed else { + return ExpiredPassObservation::Absent; + }; + if observed.turn() != expected_turn { + return ExpiredPassObservation::Superseded(observed.turn()); + } + match unconfirmed { + Some(previous) if previous == observed => ExpiredPassObservation::Confirmed(observed), + Some(previous) => ExpiredPassObservation::Progressing { previous, observed }, + None => ExpiredPassObservation::AwaitingConfirmation(observed), + } +} + +/// Whether a fresh scheduler pass would re-drive an expired pass's turn. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum FreshPassAdmission { + /// A nudged pass resumes this exact turn, so the nudge is a real handoff. + Admissible, + /// No pass reaches this turn, so the nudge admits one that does nothing. + Stranded, + /// The read did not settle, so it is unknown whether a pass can reach it. + Undetermined, +} + +/// Asks whether a nudged pass would resume `expected_turn`. +/// +/// The question goes to the predicate the nudged pass actually applies rather +/// than to a copy of it: `find_resumable_turn` is the one resume decision on the +/// nudge path, and a restatement here could drift from it silently. Every arm of +/// that predicate requires a live tool round, so a turn left running without one +/// is resumed by nothing — and the activation the same pass falls through to +/// cannot reach it either, since that admits a queued turn only while the +/// session holds no active one. +/// +/// The read carries the same deployment-configured ceiling every other +/// expired-pass database operation carries, so one wedged read cannot outlast +/// the recovery attempt that asked the question. +async fn fresh_pass_admission( + resumption: &PostgresToolLoopRepository, + session: SessionId, + expected_turn: TurnId, + attempt: u32, + attempt_bound: Option, +) -> FreshPassAdmission { + match optional_timeout(attempt_bound, resumption.find_resumable_turn(session)).await { + Ok(Ok(resumable)) if resumable == Some(expected_turn) => FreshPassAdmission::Admissible, + Ok(Ok(_)) => FreshPassAdmission::Stranded, + Ok(Err(error)) => { + tracing::error!( + failure_class = ?error.operator_failure_class(), + cause_code = "scheduler_pass_occupancy_resumability_failed", + session_id = %session.as_uuid(), + expected_turn_id = %expected_turn.as_uuid(), + attempt, + "scheduler pass expiry could not decide whether a fresh pass reaches its turn" + ); + FreshPassAdmission::Undetermined + } + Err(_) => { + tracing::error!( + failure_class = ?signalbox_application::OperatorFailureClass::Infrastructure { commit_ambiguous: false }, + cause_code = "scheduler_pass_occupancy_resumability_timed_out", + session_id = %session.as_uuid(), + expected_turn_id = %expected_turn.as_uuid(), + attempt, + attempt_bound_seconds = ?attempt_bound.map(|bound| bound.as_secs()), + "scheduler pass expiry resumability read exceeded its bound" + ); + FreshPassAdmission::Undetermined } } } +/// Whether a progressing turn leaves this path on the strength of its nudge. +/// +/// Progress forbids terminalizing the turn here, but it does not settle who +/// drives it next, and the two answers differ. A turn a fresh pass resumes is +/// handed off: the pass owns it, and only the slot-held watchdog's much longer +/// ceiling may judge it afterwards. A turn no pass reaches was not handed to +/// anyone — the nudge admits a pass that finds nothing to resume and no queued +/// turn to activate — so leaving it here would strand it until that thirty-minute +/// watchdog, when this path is already watching it and holds a confirmation +/// delay of its own. +/// +/// An undetermined read is treated as a handoff. Keeping the turn would make it +/// eligible for terminalization on the shorter delay while a pass may already be +/// driving it, and no failed read is worth that; deferring costs only the wait +/// this path already accepts whenever its own attempts fail. +const fn progressing_turn_is_handed_off(admission: FreshPassAdmission) -> bool { + matches!( + admission, + FreshPassAdmission::Admissible | FreshPassAdmission::Undetermined + ) +} + +/// Says what a refused under-lock recovery actually observed. +/// +/// [`PostgresTurnLivenessRepository::recover_observed_slot_held_turn`] answers +/// `None` for two different facts: the session's slot moved on, or this exact +/// turn's durable evidence advanced while the lock was being acquired. Only the +/// second is progress, and only progress obliges this path to ask who drives +/// the turn next. The lock-free read that separates them is the same one that +/// proposed the candidate, so the classification both liveness watchdogs apply +/// carries over unchanged: the refused candidate is the earlier observation and +/// this read is the later one. +/// +/// `None` means the read itself did not settle, which is a different answer from +/// any observation it could have returned. +async fn reobserve_refused_expired_pass_recovery( + repository: &PostgresTurnLivenessRepository, + session: SessionId, + expected_turn: TurnId, + refused: StaleTurnCandidate, + attempt_bound: Option, +) -> Option { + match optional_timeout(attempt_bound, repository.observed_slot_held_turn(session)).await { + Ok(Ok(observed)) => Some(classify_expired_pass_observation( + expected_turn, + Some(refused), + observed, + )), + Ok(Err(_)) | Err(_) => None, + } +} + async fn recover_expired_scheduler_pass( recovery: SchedulerPassOccupancyRecovery, session: SessionId, @@ -1050,7 +1525,15 @@ async fn recover_expired_scheduler_pass( let policy = recovery.policy; let repository = PostgresTurnLivenessRepository::new(recovery.pool.clone(), recovery.persistence_bounds); - let candidate = match optional_timeout( + let resumption = PostgresToolLoopRepository::new(recovery.pool.clone()); + // The first observation only proposes a turn. Expiry means the pass ran out + // of tenure, which is not the same as the turn standing still: one admitted + // pass drives a whole model/tools loop, so a healthy turn with several + // exchanges, or one riding out provider backoff, can reach the ceiling while + // making continuous durable progress. The under-lock revalidation refuses to + // terminalize such a turn, and each refusal it explains re-baselines this + // candidate on the later evidence. + let mut candidate = match optional_timeout( policy.attempt_bound, repository.observed_slot_held_turn(session), ) @@ -1110,15 +1593,99 @@ async fn recover_expired_scheduler_pass( return; } Ok(Ok(None)) => { - recovery.nudge(session); - tracing::info!( - cause_code = "scheduler_pass_occupancy_recovery_superseded", - session_id = %session.as_uuid(), - turn_id = %expected_turn.as_uuid(), - attempt, - "expired scheduler pass turn or progress evidence changed under the lock and was left alone" - ); - return; + match reobserve_refused_expired_pass_recovery( + &repository, + session, + expected_turn, + candidate, + policy.attempt_bound, + ) + .await + { + Some(ExpiredPassObservation::Progressing { previous, observed }) => { + // The pass expired while its turn was working, so nothing + // here may terminalize it on this observation. Whether + // this path is finished with the turn is a separate + // question: the nudge re-drives only a turn a fresh pass + // can resume, and durable progress can leave a turn in a + // shape that clears no re-admission predicate at all. + recovery.nudge(session); + let admission = fresh_pass_admission( + &resumption, + session, + expected_turn, + attempt, + policy.attempt_bound, + ) + .await; + if progressing_turn_is_handed_off(admission) { + tracing::info!( + cause_code = "scheduler_pass_occupancy_progress_observed", + session_id = %session.as_uuid(), + turn_id = %expected_turn.as_uuid(), + attempt, + ?admission, + previous_evidence = ?previous.evidence(), + observed_evidence = ?observed.evidence(), + "expired scheduler pass was still making durable progress; turn left active" + ); + return; + } + // No pass reaches the turn, so the progress this observed + // was work landing rather than work continuing. + // Re-baseline on the later evidence and keep watching: if + // the turn is genuinely stranded its evidence now stands + // still, and the next under-lock revalidation + // terminalizes it here instead of leaving it for the + // thirty-minute slot-held watchdog. + candidate = observed; + tracing::warn!( + cause_code = "scheduler_pass_occupancy_progress_unresumable", + session_id = %session.as_uuid(), + turn_id = %expected_turn.as_uuid(), + attempt, + previous_evidence = ?previous.evidence(), + observed_evidence = ?observed.evidence(), + "expired scheduler pass advanced its turn into a shape no fresh pass resumes; recovery keeps watching" + ); + if policy.attempts.is_none_or(|limit| attempt < limit) { + sleep_for_policy(policy.conservative_retry_delay).await; + } + } + Some(ExpiredPassObservation::Confirmed(observed)) + | Some(ExpiredPassObservation::AwaitingConfirmation(observed)) => { + // The refusal and this read disagree about whether the + // evidence moved, so the turn is still exactly the one + // this path holds. Spend another attempt on it rather + // than handing a turn nothing else is watching to the + // outer watchdog; the next revalidation decides it under + // the lock, which is the only place it may be decided. + candidate = observed; + } + Some(ExpiredPassObservation::Superseded(observed_turn)) => { + recovery.nudge(session); + tracing::info!( + cause_code = "scheduler_pass_occupancy_recovery_superseded", + session_id = %session.as_uuid(), + expected_turn_id = %expected_turn.as_uuid(), + observed_turn_id = %observed_turn.as_uuid(), + attempt, + "expired scheduler-pass turn was superseded before recovery" + ); + return; + } + Some(ExpiredPassObservation::Absent) | None => { + recovery.nudge(session); + tracing::info!( + cause_code = "scheduler_pass_occupancy_recovery_superseded", + session_id = %session.as_uuid(), + turn_id = %expected_turn.as_uuid(), + attempt, + "expired scheduler pass turn or progress evidence changed under the lock and was left alone" + ); + return; + } + } } Ok(Err(error)) => { if matches!( @@ -1269,11 +1836,22 @@ fn turn_work_span(session: SessionId, turn: TurnId) -> tracing::Span { pub(crate) fn report_ambiguous_commit(execution: &Execution, error: &Failure) where Execution: ActivatedTurnExecution, - Failure: ClassifyOperatorFailure, + Failure: ClassifyOperatorFailure, +{ + if commit_outcome_is_unknown(error) { + execution.report_post_activation_failure(); + } +} + +fn reported_usage_compaction_failure( + execution: &Execution, + error: ReportedUsageCompactionError, +) -> ActivatedTurnPassError +where + Execution: ActivatedTurnExecution, { - if commit_outcome_is_unknown(error) { - execution.report_post_activation_failure(); - } + report_ambiguous_commit(execution, &error); + ActivatedTurnPassError::ReportedUsageCompaction(error) } /// Whether one classified failure left a durable commit outcome the running @@ -1342,6 +1920,8 @@ pub type PostgresProviderToolExecutionError = /// stages within one turn. #[derive(Debug)] pub enum PostgresProviderToolLoopExecutionError { + /// Turn-start instruction discovery or durable recording failed. + WorkspaceInstructions(WorkspaceInstructionRuntimeError), /// Read-only active-turn lookup failed before durable execution began. ResumeLookup(ToolLoopRepositoryError), /// A found active turn failed while resumed execution was in progress. @@ -1367,6 +1947,7 @@ where { fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { match self { + Self::WorkspaceInstructions(error) => error.fmt(formatter), Self::ResumeLookup(error) => error.fmt(formatter), Self::ResumeExecution { source, .. } => source.fmt(formatter), Self::Model(error) => error.fmt(formatter), @@ -1384,6 +1965,7 @@ where { fn source(&self) -> Option<&(dyn Error + 'static)> { match self { + Self::WorkspaceInstructions(error) => Some(error), Self::ResumeLookup(error) => Some(error), Self::ResumeExecution { source, .. } => Some(source), Self::Model(error) => Some(error), @@ -1401,6 +1983,7 @@ where { fn operator_failure_class(&self) -> OperatorFailureClass { match self { + Self::WorkspaceInstructions(error) => error.operator_failure_class(), Self::ResumeLookup(error) => error.operator_failure_class(), Self::ResumeExecution { source, .. } => source.operator_failure_class(), Self::Model(error) => error.operator_failure_class(), @@ -1411,6 +1994,7 @@ where fn operator_failure_cause_code(&self) -> &'static str { match self { + Self::WorkspaceInstructions(error) => error.operator_failure_cause_code(), Self::ResumeLookup(_) => "tool_loop_resume_lookup", Self::ResumeExecution { source, .. } => source.operator_failure_cause_code(), Self::Model(error) => error.operator_failure_cause_code(), @@ -1437,6 +2021,12 @@ where PostgresProviderToolLoopExecutionError::ApprovalJudge(error) => { !is_nonambiguous_infrastructure_failure(error.operator_failure_class()) } + // Instruction discovery can have recorded a durable manifest before it + // failed, so it is classified exactly like the other pre-execution + // durable step rather than assumed evidence-free. + PostgresProviderToolLoopExecutionError::WorkspaceInstructions(error) => { + !is_nonambiguous_infrastructure_failure(error.operator_failure_class()) + } PostgresProviderToolLoopExecutionError::ResumeLookup(_) | PostgresProviderToolLoopExecutionError::ResumeExecution { .. } => true, } @@ -1492,23 +2082,23 @@ impl PostgresProviderModelExecution { approval_judge: None, approval_judge_selection: None, approval_judge_configuration: None, + workspace_instructions: None, shutdown_checkpoint: None, } } -} - -impl ActivatedTurnExecution for PostgresProviderModelExecution -where - Provider: ModelCallProvider + Clone + Send + 'static, - Provider::Capability: Send, - Provider::Error: Send + 'static, -{ - type Error = PostgresProviderModelExecutionError; - fn execute( + fn execute_with_checkpoint_boundary( &self, activated: Box, - ) -> impl Future> + Send + 'static { + return_on_checkpoint: bool, + ) -> impl Future>> + + Send + + 'static + where + Provider: ModelCallProvider + Clone + Send + 'static, + Provider::Capability: Send, + Provider::Error: Send + 'static, + { let repository = self.repository.clone(); let gate = self.gate.clone(); let provider = self.provider.clone(); @@ -1530,9 +2120,6 @@ where let outcome = match service.execute(session).await { Ok(outcome) => outcome, Err(error) if service.retained_state().is_some() => { - // Preserve same-incarnation evidence for one - // authoritative reconciliation pass before fatal - // supervision hands authority to startup recovery. reconcile_retained_once(error, service.execute(session)).await? } Err(error) => return Err(RetainedModelExecutionError::Primary(error)), @@ -1541,23 +2128,52 @@ where ModelCallExecutionOutcome::RetryBackoff(delay) => { tokio::time::sleep(delay).await; } + ModelCallExecutionOutcome::Checkpointed(_) if return_on_checkpoint => { + return Ok(()); + } ModelCallExecutionOutcome::Checkpointed(_) | ModelCallExecutionOutcome::AvailabilitySuccessor(_) => continue, ModelCallExecutionOutcome::NoWork + | ModelCallExecutionOutcome::AttachmentUnavailable | ModelCallExecutionOutcome::PoolExhausted(_) | ModelCallExecutionOutcome::TargetUnavailable(_) | ModelCallExecutionOutcome::CapabilityKnownFailure(_) | ModelCallExecutionOutcome::CapabilityFailureAlreadyCommitted(_) + | ModelCallExecutionOutcome::ToolRoundLimitReached(_) + | ModelCallExecutionOutcome::ToolRoundLimitAlreadyCommitted(_) | ModelCallExecutionOutcome::ObservationCommitted(_) | ModelCallExecutionOutcome::ObservationAlreadyCommitted(_) => return Ok(()), } } } } +} + +impl ActivatedTurnExecution for PostgresProviderModelExecution +where + Provider: ModelCallProvider + Clone + Send + 'static, + Provider::Capability: Send, + Provider::Error: Send + 'static, +{ + type Error = PostgresProviderModelExecutionError; fn execution_failure_requires_recovery(error: &Self::Error) -> bool { retained_execution_failure_requires_recovery(error) } + + fn execute( + &self, + activated: Box, + ) -> impl Future> + Send + 'static { + self.execute_with_checkpoint_boundary(activated, false) + } + + fn execute_dispatch_start( + &self, + activated: Box, + ) -> impl Future> + Send + 'static { + self.execute_with_checkpoint_boundary(activated, true) + } } /// Production execution factory alternating provider calls with serialized @@ -1576,6 +2192,7 @@ pub struct PostgresProviderToolLoopExecution { approval_judge: Option>, approval_judge_selection: Option, approval_judge_configuration: Option, + workspace_instructions: Option, shutdown_checkpoint: Option>, } @@ -2035,7 +2652,9 @@ const fn judge_failure_disposition( } } -const fn provider_reported_usage(usage: TokenUsage) -> ProviderReportedTokenUsage { +/// Carries a runtime usage report into the domain representation unchanged, +/// field for field; absent fields stay absent. +pub const fn provider_reported_usage(usage: TokenUsage) -> ProviderReportedTokenUsage { ProviderReportedTokenUsage::unreported() .with_input_tokens(usage.input_tokens) .with_output_tokens(usage.output_tokens) @@ -2052,6 +2671,15 @@ where Executor: ToolExecutor + Clone + Send + 'static, Executor::Error: Send + 'static, { + /// Enables daemon-owned instruction discovery before model execution. + pub fn with_workspace_instructions( + mut self, + workspace_instructions: WorkspaceInstructionRuntime, + ) -> Self { + self.workspace_instructions = Some(workspace_instructions); + self + } + /// Enables delegated approval judging through the configured model runtime. pub fn with_approval_judge( mut self, @@ -2075,6 +2703,7 @@ where &self, session: SessionId, turn: signalbox_domain::TurnId, + return_on_model_checkpoint: bool, ) -> impl Future< Output = Result< (), @@ -2094,8 +2723,17 @@ where let approval_judge = self.approval_judge.clone(); let approval_judge_selection = self.approval_judge_selection; let approval_judge_configuration = self.approval_judge_configuration.clone(); + let workspace_instructions = self.workspace_instructions.clone(); let mut shutdown_checkpoint = self.shutdown_checkpoint.clone(); async move { + if let Some(workspace_instructions) = workspace_instructions + && !workspace_instructions + .prepare(session, turn) + .await + .map_err(PostgresProviderToolLoopExecutionError::WorkspaceInstructions)? + { + return Ok(()); + } let mut model = ModelCallExecutionService::new( UuidV7ModelCallExecutionIdGenerator, model_repository.clone(), @@ -2225,6 +2863,9 @@ where return Ok(()); } } + ModelCallExecutionOutcome::Checkpointed(_) if return_on_model_checkpoint => { + return Ok(()); + } ModelCallExecutionOutcome::Checkpointed(_) => checkpoint_safe = false, ModelCallExecutionOutcome::AvailabilitySuccessor(_) => { checkpoint_safe = true; @@ -2232,10 +2873,13 @@ where ModelCallExecutionOutcome::TargetUnavailable(_) | ModelCallExecutionOutcome::PoolExhausted(_) | ModelCallExecutionOutcome::CapabilityKnownFailure(_) - | ModelCallExecutionOutcome::CapabilityFailureAlreadyCommitted(_) => { + | ModelCallExecutionOutcome::CapabilityFailureAlreadyCommitted(_) + | ModelCallExecutionOutcome::ToolRoundLimitReached(_) + | ModelCallExecutionOutcome::ToolRoundLimitAlreadyCommitted(_) => { return Ok(()); } - ModelCallExecutionOutcome::NoWork => return Ok(()), + ModelCallExecutionOutcome::NoWork + | ModelCallExecutionOutcome::AttachmentUnavailable => return Ok(()), ModelCallExecutionOutcome::ObservationCommitted(_) | ModelCallExecutionOutcome::ObservationAlreadyCommitted(_) => { checkpoint_safe = true; @@ -2288,7 +2932,17 @@ where let session = activated.session(); let turn = activated.turn(); drop(activated); - self.execute_scope(session, turn) + self.execute_scope(session, turn, false) + } + + fn execute_dispatch_start( + &self, + activated: Box, + ) -> impl Future> + Send + 'static { + let session = activated.session(); + let turn = activated.turn(); + drop(activated); + self.execute_scope(session, turn, true) } fn resume_active( @@ -2317,7 +2971,45 @@ where Some(turn) => { observe(turn); execution - .execute_scope(session, turn) + .execute_scope(session, turn, false) + .instrument(turn_work_span(session, turn)) + .await + .map_err( + |source| PostgresProviderToolLoopExecutionError::ResumeExecution { + turn, + source: Box::new(source), + }, + ) + } + None => Ok(()), + } + } + } + + fn resume_dispatch_start( + &self, + session: SessionId, + ) -> impl Future> + Send + 'static { + self.resume_dispatch_start_with_observer(session, std::sync::Arc::new(|_| {})) + } + + fn resume_dispatch_start_with_observer( + &self, + session: SessionId, + observe_turn: std::sync::Arc, + ) -> impl Future> + Send + 'static { + let tool_repository = self.tool_repository.clone(); + let execution = self.clone(); + async move { + let turn = tool_repository + .find_dispatch_start_turn(session) + .await + .map_err(PostgresProviderToolLoopExecutionError::ResumeLookup)?; + match turn { + Some(turn) => { + observe_turn(turn); + execution + .execute_scope(session, turn, true) .instrument(turn_work_span(session, turn)) .await .map_err( @@ -2342,7 +3034,11 @@ where PostgresProviderToolLoopExecutionError::ResumeExecution { source, .. } => { tool_loop_execution_failure_requires_recovery(source) } - PostgresProviderToolLoopExecutionError::Model(_) + // Instruction preparation can have recorded a durable manifest for + // the turn it was about to resume, so it takes the fail-safe answer + // rather than the read-only lookup's. + PostgresProviderToolLoopExecutionError::WorkspaceInstructions(_) + | PostgresProviderToolLoopExecutionError::Model(_) | PostgresProviderToolLoopExecutionError::Tool(_) | PostgresProviderToolLoopExecutionError::ApprovalJudge(_) => true, } @@ -2351,7 +3047,8 @@ where fn active_resume_failure_turn(error: &Self::Error) -> Option { match error { PostgresProviderToolLoopExecutionError::ResumeExecution { turn, .. } => Some(*turn), - PostgresProviderToolLoopExecutionError::ResumeLookup(_) + PostgresProviderToolLoopExecutionError::WorkspaceInstructions(_) + | PostgresProviderToolLoopExecutionError::ResumeLookup(_) | PostgresProviderToolLoopExecutionError::Model(_) | PostgresProviderToolLoopExecutionError::Tool(_) | PostgresProviderToolLoopExecutionError::ApprovalJudge(_) => None, @@ -2380,15 +3077,13 @@ impl PostgresScriptedModelExecution { assistant_reply, } } -} - -impl ActivatedTurnExecution for PostgresScriptedModelExecution { - type Error = PostgresScriptedModelExecutionError; - fn execute( + fn execute_with_checkpoint_boundary( &self, activated: Box, - ) -> impl Future> + Send + 'static { + return_on_checkpoint: bool, + ) -> impl Future> + Send + 'static + { let repository = self.repository.clone(); let gate = self.gate.clone(); let assistant_reply = self.assistant_reply.clone(); @@ -2413,12 +3108,6 @@ impl ActivatedTurnExecution for PostgresScriptedModelExecution { let outcome = match service.execute(session).await { Ok(outcome) => outcome, Err(error) if service.retained_state().is_some() => { - // docs/spec/model-call-execution.md gives - // same-incarnation evidence one authoritative - // reconciliation pass before fatal supervision hands - // authority to startup recovery. A second failure - // does not replace the causal stage error that - // created the retained obligation. reconcile_retained_once(error, service.execute(session)).await? } Err(error) => return Err(RetainedModelExecutionError::Primary(error)), @@ -2427,23 +3116,47 @@ impl ActivatedTurnExecution for PostgresScriptedModelExecution { ModelCallExecutionOutcome::RetryBackoff(delay) => { tokio::time::sleep(delay).await; } + ModelCallExecutionOutcome::Checkpointed(_) if return_on_checkpoint => { + return Ok(()); + } ModelCallExecutionOutcome::Checkpointed(_) | ModelCallExecutionOutcome::AvailabilitySuccessor(_) => continue, ModelCallExecutionOutcome::NoWork + | ModelCallExecutionOutcome::AttachmentUnavailable | ModelCallExecutionOutcome::PoolExhausted(_) | ModelCallExecutionOutcome::TargetUnavailable(_) | ModelCallExecutionOutcome::CapabilityKnownFailure(_) | ModelCallExecutionOutcome::CapabilityFailureAlreadyCommitted(_) + | ModelCallExecutionOutcome::ToolRoundLimitReached(_) + | ModelCallExecutionOutcome::ToolRoundLimitAlreadyCommitted(_) | ModelCallExecutionOutcome::ObservationCommitted(_) | ModelCallExecutionOutcome::ObservationAlreadyCommitted(_) => return Ok(()), } } } } +} + +impl ActivatedTurnExecution for PostgresScriptedModelExecution { + type Error = PostgresScriptedModelExecutionError; fn execution_failure_requires_recovery(error: &Self::Error) -> bool { retained_execution_failure_requires_recovery(error) } + + fn execute( + &self, + activated: Box, + ) -> impl Future> + Send + 'static { + self.execute_with_checkpoint_boundary(activated, false) + } + + fn execute_dispatch_start( + &self, + activated: Box, + ) -> impl Future> + Send + 'static { + self.execute_with_checkpoint_boundary(activated, true) + } } #[cfg(test)] @@ -2467,21 +3180,26 @@ mod tests { AcceptedInputTurnActivationIdentities, ActivatedTurn, ContextFrontierId, SemanticTranscriptEntryId, SessionId, TurnAttemptId, TurnId, }; - use signalbox_persistence::turn_liveness::TurnLivenessPersistenceBounds; + use signalbox_persistence::{ + start_eligible_turn::{CommitActivationPreviewError, StartEligibleTurnRepositoryError}, + turn_liveness::TurnLivenessPersistenceBounds, + }; use tokio::sync::watch; use uuid::Uuid; use super::{ APPROVAL_JUDGE_SYSTEM_PROMPT, ActivatedTurnExecution, ActivatedTurnPass, - ActivatedTurnPassError, ApprovalJudgeModelError, ExpiredPassRecoveryPolicy, - FailedApprovalJudgeDisposition, FatalExecutionGuardState, FatalExecutionOccupancyExpiry, - FatalExecutionSignal, FatalExecutionSupervisor, JudgeRequestFields, - MAX_QUOTED_CONTEXT_BYTES, SchedulerPassOccupancyRecovery, SessionAuthorityContext, + ActivatedTurnPassError, ApprovalJudgeModelError, ExpiredPassObservation, + ExpiredPassRecoveryPolicy, FailedApprovalJudgeDisposition, FatalExecutionGuardState, + FatalExecutionOccupancyExpiry, FatalExecutionSignal, FatalExecutionSupervisor, + FreshPassAdmission, JudgeRequestFields, MAX_QUOTED_CONTEXT_BYTES, + ReportedUsageCompactionError, SchedulerPassOccupancyRecovery, SessionAuthorityContext, TokenUsage, TurnLivenessRepositoryError, TurnPassExecutionStage, - activation_session_matches, expired_pass_recovery_retry_delay, - matches_exact_slot_held_turn, reconcile_retained_once, render_dispatch_authority, - render_judge_request_payload, render_session_authority_context, supervise_execution, - supervise_execution_for_session, + activation_session_matches, classify_expired_pass_observation, + expired_pass_recovery_retry_delay, matches_exact_slot_held_turn, + progressing_turn_is_handed_off, reconcile_retained_once, render_dispatch_authority, + render_judge_request_payload, render_session_authority_context, + reported_usage_compaction_failure, supervise_execution, supervise_execution_for_session, }; fn example_expired_pass_policy() -> ExpiredPassRecoveryPolicy { @@ -2586,6 +3304,16 @@ mod tests { } } + #[track_caller] + fn assert_reported_usage_compaction_error( + error: ActivatedTurnPassError, + ) { + match error { + ActivatedTurnPassError::ReportedUsageCompaction(_) => {} + other => panic!("expected reported-usage compaction failure, got {other:?}"), + } + } + #[derive(Clone, Copy, Debug, Eq, PartialEq)] struct CommitAmbiguousActivationFailure; @@ -2703,6 +3431,12 @@ mod tests { #[derive(Clone, Copy, Debug)] struct CommitAmbiguousTransaction; + impl CommitAmbiguousTransaction { + fn activated_turn() -> TurnId { + TurnId::from_uuid(Uuid::from_u128(11)) + } + } + impl StartEligibleTurnTransaction for CommitAmbiguousTransaction { type Error = CommitAmbiguousActivationFailure; @@ -2713,6 +3447,16 @@ mod tests { ) -> impl Future> + Send { ready(Err(CommitAmbiguousActivationFailure)) } + + fn handle_with_activation_observer( + &mut self, + _session: SessionId, + _identities: AcceptedInputTurnActivationIdentities, + observer: Arc, + ) -> impl Future> + Send { + observer(Self::activated_turn()); + ready(Err(CommitAmbiguousActivationFailure)) + } } #[derive(Clone, Copy, Debug)] @@ -2937,7 +3681,7 @@ mod tests { observed.notified().await; assert_eq!( recovery - .expected_turns + .active_turns .lock() .expect("expected-turn lock") .get(&session) @@ -2965,7 +3709,7 @@ mod tests { pool, eligibility_nudge: nudge, execution_expiry: None, - expected_turns: Arc::new(Mutex::new(std::collections::HashMap::new())), + active_turns: Arc::new(Mutex::new(std::collections::HashMap::new())), policy: example_expired_pass_policy(), persistence_bounds: test_turn_liveness_persistence_bounds(), }; @@ -3044,6 +3788,51 @@ mod tests { assert!(signal.is_triggered()); } + #[tokio::test] + async fn commit_ambiguous_activation_is_observed_before_acknowledgement_failure() { + let observed = Arc::new(Mutex::new(None)); + let observer_state = Arc::clone(&observed); + let observer: Arc = Arc::new(move |turn| { + *observer_state.lock().expect("activation observer lock") = Some(turn); + }); + let mut service = + StartEligibleTurnService::new(AdvancingIds::new(), CommitAmbiguousTransaction); + + let error = service + .execute_with_cloned_transaction_and_observer( + SessionId::from_uuid(Uuid::from_u128(9)), + observer, + ) + .await + .expect_err("commit acknowledgement remains ambiguous"); + + assert!(matches!(error, CommitAmbiguousActivationFailure)); + assert_eq!( + *observed.lock().expect("activation observer lock"), + Some(CommitAmbiguousTransaction::activated_turn()) + ); + } + + #[test] + fn inv034_ambiguous_reported_usage_failure_closure_raises_the_fatal_recovery_signal() { + let (execution, signal) = FatalExecutionSupervisor::new(NoopExecution); + let source = + CommitActivationPreviewError::Activation(StartEligibleTurnRepositoryError::Database { + source: sqlx::Error::PoolClosed, + commit_ambiguous: true, + }); + let error = ReportedUsageCompactionError::CompactionFailureClosure { + turn: TurnId::from_uuid(Uuid::from_u128(11)), + source, + }; + + let reported: ActivatedTurnPassError = + reported_usage_compaction_failure(&execution, error); + + assert_reported_usage_compaction_error(reported); + assert!(signal.is_triggered()); + } + #[test] fn activation_session_mismatch_raises_the_fatal_signal() { let (execution, signal) = FatalExecutionSupervisor::new(NoopExecution); @@ -3904,4 +4693,230 @@ mod tests { FailedApprovalJudgeDisposition::KnownFailed ); } + + fn expiry_candidate( + session: SessionId, + turn: TurnId, + attempt: TurnAttemptId, + outbox_frontier: Option, + ) -> StaleTurnCandidate { + StaleTurnCandidate::new( + session, + turn, + TurnLivenessEvidence::new(attempt, outbox_frontier), + ) + } + + /// One observation may not terminalize: the occupancy ceiling bounds a + /// pass's tenure, and a turn making continuous durable progress reaches it + /// just as a wedged one does. + #[test] + fn one_expiry_observation_only_proposes_the_turn() { + let session = SessionId::from_uuid(Uuid::now_v7()); + let turn = TurnId::from_uuid(Uuid::now_v7()); + let observed = expiry_candidate( + session, + turn, + TurnAttemptId::from_uuid(Uuid::now_v7()), + Some(7), + ); + + assert_eq!( + classify_expired_pass_observation(turn, None, Some(observed)), + ExpiredPassObservation::AwaitingConfirmation(observed) + ); + } + + #[test] + fn unchanged_expiry_evidence_confirms_the_turn_for_recovery() { + let session = SessionId::from_uuid(Uuid::now_v7()); + let turn = TurnId::from_uuid(Uuid::now_v7()); + let observed = expiry_candidate( + session, + turn, + TurnAttemptId::from_uuid(Uuid::now_v7()), + Some(7), + ); + + assert_eq!( + classify_expired_pass_observation(turn, Some(observed), Some(observed)), + ExpiredPassObservation::Confirmed(observed) + ); + } + + /// A turn whose session emitted an outbox event between observations was + /// working, not wedged, so the expired pass must not terminalize it. + #[test] + fn an_advanced_outbox_frontier_spares_the_expired_pass_turn() { + let session = SessionId::from_uuid(Uuid::now_v7()); + let turn = TurnId::from_uuid(Uuid::now_v7()); + let attempt = TurnAttemptId::from_uuid(Uuid::now_v7()); + let previous = expiry_candidate(session, turn, attempt, Some(7)); + let observed = expiry_candidate(session, turn, attempt, Some(8)); + + assert_eq!( + classify_expired_pass_observation(turn, Some(previous), Some(observed)), + ExpiredPassObservation::Progressing { previous, observed } + ); + } + + /// The same turn on a later physical attempt has also progressed. + #[test] + fn an_advanced_attempt_spares_the_expired_pass_turn() { + let session = SessionId::from_uuid(Uuid::now_v7()); + let turn = TurnId::from_uuid(Uuid::now_v7()); + let previous = expiry_candidate( + session, + turn, + TurnAttemptId::from_uuid(Uuid::now_v7()), + Some(7), + ); + let observed = expiry_candidate( + session, + turn, + TurnAttemptId::from_uuid(Uuid::now_v7()), + Some(7), + ); + + assert_eq!( + classify_expired_pass_observation(turn, Some(previous), Some(observed)), + ExpiredPassObservation::Progressing { previous, observed } + ); + } + + /// A session that emits its first outbox event between observations moves + /// from absent to present evidence, which is progress like any other. + #[test] + fn a_first_outbox_event_spares_the_expired_pass_turn() { + let session = SessionId::from_uuid(Uuid::now_v7()); + let turn = TurnId::from_uuid(Uuid::now_v7()); + let attempt = TurnAttemptId::from_uuid(Uuid::now_v7()); + let previous = expiry_candidate(session, turn, attempt, None); + let observed = expiry_candidate(session, turn, attempt, Some(1)); + + assert_eq!( + classify_expired_pass_observation(turn, Some(previous), Some(observed)), + ExpiredPassObservation::Progressing { previous, observed } + ); + } + + #[test] + fn a_different_turn_supersedes_the_expired_pass() { + let session = SessionId::from_uuid(Uuid::now_v7()); + let expected = TurnId::from_uuid(Uuid::now_v7()); + let successor = TurnId::from_uuid(Uuid::now_v7()); + let observed = expiry_candidate( + session, + successor, + TurnAttemptId::from_uuid(Uuid::now_v7()), + Some(7), + ); + + assert_eq!( + classify_expired_pass_observation(expected, None, Some(observed)), + ExpiredPassObservation::Superseded(successor) + ); + // A pending confirmation does not make a successor recoverable either. + assert_eq!( + classify_expired_pass_observation(expected, Some(observed), Some(observed)), + ExpiredPassObservation::Superseded(successor) + ); + } + + #[test] + fn no_recoverable_active_turn_ends_the_expired_pass_handoff() { + let turn = TurnId::from_uuid(Uuid::now_v7()); + + assert_eq!( + classify_expired_pass_observation(turn, None, None), + ExpiredPassObservation::Absent + ); + } + + /// Reports one recovered turn through whichever resume path the pass took. + #[derive(Clone, Copy, Debug)] + struct RecoveredTurnExecution(TurnId); + + impl ActivatedTurnExecution for RecoveredTurnExecution { + type Error = ExecutionFailure; + + fn execute( + &self, + _activated: Box, + ) -> impl Future> + Send + 'static { + ready(Ok(())) + } + + fn resume_active_with_observer( + &self, + _session: SessionId, + observe_turn: Arc, + ) -> impl Future> + Send + 'static { + observe_turn(self.0); + ready(Ok(())) + } + } + + /// A dispatch-start hint that recovers an already-active turn must report + /// that turn, exactly as the active-resume path does. Without it the + /// occupancy tracker holds no entry for the session, so an expired pass + /// finds no `expected_turn`, returns before the detached recovery handoff, + /// and strands the turn behind the far longer watchdog ceiling. + #[tokio::test] + async fn a_dispatch_start_resume_reports_the_turn_it_recovers() { + let session = SessionId::from_uuid(Uuid::now_v7()); + let turn = TurnId::from_uuid(Uuid::now_v7()); + let observed = Arc::new(Mutex::new(Vec::new())); + let recorder = Arc::clone(&observed); + + RecoveredTurnExecution(turn) + .resume_dispatch_start_with_observer( + session, + Arc::new(move |turn| { + recorder + .lock() + .expect("dispatch-start resume observer lock") + .push(turn); + }), + ) + .await + .expect("dispatch-start resume succeeds"); + + assert_eq!( + *observed + .lock() + .expect("dispatch-start resume observer lock"), + vec![turn] + ); + } + + /// A turn a fresh pass resumes leaves this path: the pass owns it, and only + /// the slot-held watchdog's far longer ceiling may judge it afterwards. + #[test] + fn a_resumable_progressing_turn_is_handed_to_the_fresh_pass() { + assert!(progressing_turn_is_handed_off( + FreshPassAdmission::Admissible + )); + } + + /// Progress alone does not settle who drives the turn next. A running turn + /// left without a tool round clears no re-admission predicate, so the nudge + /// admits a pass that does nothing; returning here would strand the turn + /// until the thirty-minute watchdog rather than confirming it on the delay + /// this path already holds. + #[test] + fn an_unresumable_progressing_turn_stays_in_the_expiry_recovery_path() { + assert!(!progressing_turn_is_handed_off( + FreshPassAdmission::Stranded + )); + } + + /// A failed read must not make a turn terminalizable on the shorter delay + /// while a fresh pass may already be driving it. + #[test] + fn an_undetermined_resumability_read_defers_to_the_watchdog() { + assert!(progressing_turn_is_handed_off( + FreshPassAdmission::Undetermined + )); + } } diff --git a/apps/signalboxd/src/main.rs b/apps/signalboxd/src/main.rs index 41528d1102..b8e6c2022b 100644 --- a/apps/signalboxd/src/main.rs +++ b/apps/signalboxd/src/main.rs @@ -39,9 +39,13 @@ use signalbox_model_runtime::CredentialReference; use signalbox_model_runtime_anthropic::{ AnthropicConfig, AnthropicConstructionError, AnthropicRuntime, }; +use signalbox_model_runtime_codex_cli::verify_pinned_codex_cli_version; use signalbox_model_runtime_openai::{OpenAiConfig, OpenAiConstructionError, OpenAiRuntime}; use signalbox_persistence::{ + automatic_reconciliation::RETRY_LADDER_ARITY, + convergence_sweep::PostgresConvergenceSweepStore, conversation_import::backfill_imported_conversation_display_titles, + hub_fence::FENCED_POOL_MAX_CONNECTIONS, migrate, model_execution::PostgresModelCallRepository, repo_watch_dispatch::{PostgresRepoWatchDispatchStore, RepoWatchDispatchRepositoryError}, @@ -56,20 +60,22 @@ use signalboxd::runner_protocol_runtime::{ RunnerRegistrationFailureCause, }; use signalboxd::{ - ActivatedTurnPass, BaseDaemonCredentialInputs, BlobStoreRegistry, - CODE_HOST_CREDENTIAL_REFERENCE, CodeHostNumericBounds, ConfiguredApprovalPostureError, - ConvergenceSweepNumericBounds, ConvergenceSweepRuntime, DaemonToolCatalog, - DaemonToolComposition, DaemonTools, DaemonToolsConstructionError, ExpiredPassRecoveryPolicy, - FatalExecutionSupervisor, FencedHubDatabase, FencedHubDatabaseError, FileCredentialAccess, - GitHubCodeHostTransport, GoalModeNumericBounds, HubModelConfiguration, - HubModelConfigurationError, LocalProcessListener, LocalSocketError, - MappedDaemonCredentialInputs, ModelAdapter, OtlpRuntime, PostgresGoalPassDisposition, - PostgresProviderModelExecution, ProcessRuntime, ProcessRuntimeError, PrometheusServer, - ReportedUsageCompaction, RepositoryWatchRuntime, RepositoryWatchRuntimeError, - SessionTemplateConfiguration, SessionTemplateConfigurationError, SingleHubGuardError, - SystemCurrentTimeClock, TelemetryConfiguration, TelemetryConfigurationError, - TelemetryExportFilter, TelemetryMetrics, TurnLivenessNumericBounds, TurnLivenessRuntime, + ActivatedTurnPass, AttachmentPreparingModelCallProvider, BaseDaemonCredentialInputs, + BlobStoreRegistry, CODE_HOST_CREDENTIAL_REFERENCE, CodeHostNumericBounds, + ConfiguredApprovalPostureError, ConvergenceSweepNumericBounds, ConvergenceSweepRuntime, + DaemonToolCatalog, DaemonToolComposition, DaemonTools, DaemonToolsConstructionError, + ExpiredPassRecoveryPolicy, FatalExecutionSupervisor, FencedHubDatabase, FencedHubDatabaseError, + FencedPoolFloorReconciliation, FileCredentialAccess, GitHubCodeHostTransport, + GoalModeNumericBounds, HubModelConfiguration, HubModelConfigurationError, LocalProcessListener, + LocalSocketError, MappedDaemonCredentialInputs, ModelAdapter, OtlpRuntime, + PostgresGoalPassDisposition, PostgresProviderModelExecution, ProcessRuntime, + ProcessRuntimeError, PrometheusServer, ReportedUsageCompaction, RepositoryWatchNumericBounds, + RepositoryWatchRuntime, RepositoryWatchRuntimeError, SessionTemplateConfiguration, + SessionTemplateConfigurationError, SingleHubGuardError, SystemCurrentTimeClock, + TelemetryConfiguration, TelemetryConfigurationError, TelemetryExportFilter, TelemetryMetrics, + TurnLivenessNumericBounds, TurnLivenessRuntime, WorkspaceInstructionRuntime, model_adapter::ConfiguredModelRuntime, + reconcile_fenced_pool_floor, usage_limits::UsageLimitedModelCallProvider, web_http::{ WebHttpConfiguration, WebHttpConfigurationError, WebHttpRuntime, WebHttpRuntimeError, @@ -103,6 +109,35 @@ fn graceful_shutdown_window( .map(|(exchange, cleanup)| exchange.saturating_add(cleanup)) } +fn validate_fenced_pool_min_connections(minimum: Option) -> Option> { + (!minimum.is_some_and(|minimum| minimum > FENCED_POOL_MAX_CONNECTIONS)).then_some(minimum) +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +struct FencedPoolFloorReconciliationPolicy { + minimum: u32, + interval: Duration, + attempt_bound: Duration, +} + +fn fenced_pool_floor_reconciliation_policy( + minimum: Option, + interval: Option, + attempt_bound: Option, +) -> Option> { + let minimum = minimum.filter(|minimum| *minimum > 0); + let Some(minimum) = minimum else { + return Some(None); + }; + let interval = interval.filter(|interval| !interval.is_zero())?; + let attempt_bound = attempt_bound.filter(|bound| !bound.is_zero())?; + Some(Some(FencedPoolFloorReconciliationPolicy { + minimum, + interval, + attempt_bound, + })) +} + #[derive(Clone, Copy, Debug, Eq, PartialEq)] enum RuntimePhase { Configuration, @@ -635,9 +670,11 @@ enum RuntimeDrainOutcome { enum RuntimeTaskExit { Scheduler(SchedulerLoopExit), + FencedPoolFloor, Process(Result<(), ProcessRuntimeError>), Runner(Result<(), RunnerProtocolRuntimeError>), RepositoryWatch(Result<(), RepositoryWatchRuntimeError>), + RepositoryWatchLeaseExpiry(Result<(), RepoWatchDispatchRepositoryError>), ConvergenceSweep, WebHttp(Result<(), WebHttpRuntimeError>), TurnLiveness, @@ -678,9 +715,11 @@ const fn combine_runtime_stop_cause( #[derive(Clone, Copy, Debug, Eq, PartialEq)] enum RuntimeTaskDefect { SchedulerCompletedBeforeShutdown, + FencedPoolFloorCompletedBeforeShutdown, ProcessCompletedBeforeShutdown, RunnerCompletedBeforeShutdown, RepositoryWatchCompletedBeforeShutdown, + RepositoryWatchLeaseExpiryCompletedBeforeShutdown, ConvergenceSweepCompletedBeforeShutdown, WebHttpCompletedBeforeShutdown, TurnLivenessCompletedBeforeShutdown, @@ -694,11 +733,17 @@ impl RuntimeTaskDefect { const fn cause_code(self) -> &'static str { match self { Self::SchedulerCompletedBeforeShutdown => "scheduler_completed_before_shutdown", + Self::FencedPoolFloorCompletedBeforeShutdown => { + "fenced_pool_floor_completed_before_shutdown" + } Self::ProcessCompletedBeforeShutdown => "process_runtime_completed_before_shutdown", Self::RunnerCompletedBeforeShutdown => "runner_runtime_completed_before_shutdown", Self::RepositoryWatchCompletedBeforeShutdown => { "repository_watch_completed_before_shutdown" } + Self::RepositoryWatchLeaseExpiryCompletedBeforeShutdown => { + "repository_watch_lease_expiry_completed_before_shutdown" + } Self::ConvergenceSweepCompletedBeforeShutdown => { "convergence_sweep_completed_before_shutdown" } @@ -823,6 +868,71 @@ async fn wait_for_guard_loss(database: &mut FencedHubDatabase) { } } +async fn run_fenced_pool_floor_reconciliation( + pool: sqlx::PgPool, + policy: FencedPoolFloorReconciliationPolicy, + mut shutdown: watch::Receiver, +) { + loop { + select! { + changed = shutdown.changed() => { + if changed.is_err() || *shutdown.borrow() { + return; + } + continue; + } + () = sleep(policy.interval) => {} + } + let prior_size = pool.size(); + if prior_size >= policy.minimum { + continue; + } + let attempt = timeout( + policy.attempt_bound, + reconcile_fenced_pool_floor(&pool, policy.minimum), + ); + let outcome = select! { + changed = shutdown.changed() => { + if changed.is_err() || *shutdown.borrow() { + return; + } + continue; + } + outcome = attempt => outcome, + }; + let current_size = pool.size(); + match outcome { + Ok(Ok(FencedPoolFloorReconciliation::Replenished)) => tracing::info!( + prior_size, + current_size, + minimum = policy.minimum, + "fenced pool floor reconciliation added one physical session" + ), + Ok(Ok( + FencedPoolFloorReconciliation::Satisfied + | FencedPoolFloorReconciliation::DeferredForIdleCapacity, + )) => {} + Ok(Err(_)) => tracing::warn!( + failure_class = ?OperatorFailureClass::Infrastructure { commit_ambiguous: false }, + cause_code = "fenced_pool_floor_reconciliation_failed", + prior_size, + current_size, + minimum = policy.minimum, + "fenced pool floor reconciliation will retry" + ), + Err(_) => tracing::warn!( + failure_class = ?OperatorFailureClass::Infrastructure { commit_ambiguous: false }, + cause_code = "fenced_pool_floor_reconciliation_timed_out", + prior_size, + current_size, + minimum = policy.minimum, + attempt_bound_seconds = policy.attempt_bound.as_secs(), + "fenced pool floor reconciliation will retry" + ), + } + } +} + enum GuardedAwait { Completed(T), GuardLost, @@ -961,6 +1071,38 @@ fn report_repository_watch_runtime_defect(error: &RepositoryWatchRuntimeError) { ); } +/// Classifies a lease-expiry failure without flattening commit ambiguity. +/// +/// An ambiguous commit may already have applied the goal stop and the +/// expiration receipt, so operator telemetry must not present the +/// expiration transaction as safe to retry; corruption stays fail-closed. +fn repository_watch_lease_expiry_failure_class( + error: &RepoWatchDispatchRepositoryError, +) -> OperatorFailureClass { + match error { + RepoWatchDispatchRepositoryError::CommitAmbiguous(_) => { + OperatorFailureClass::Infrastructure { + commit_ambiguous: true, + } + } + RepoWatchDispatchRepositoryError::Corruption(_) => { + OperatorFailureClass::FailClosedCorruption + } + _ => OperatorFailureClass::Infrastructure { + commit_ambiguous: false, + }, + } +} + +fn report_repository_watch_lease_expiry_failure(error: &RepoWatchDispatchRepositoryError) { + tracing::error!( + phase = ?RuntimePhase::Runtime, + failure_class = ?repository_watch_lease_expiry_failure_class(error), + cause = %error, + "global repository-watch lease expiry reconciliation failed" + ); +} + fn report_web_http_runtime_failure(error: &WebHttpRuntimeError) { tracing::error!( phase = ?RuntimePhase::Runtime, @@ -996,9 +1138,11 @@ fn joined_task_defect(error: &JoinError) -> RuntimeTaskDefect { fn runtime_task_completion(completed: Result) -> RuntimeTaskCompletion { match completed { Ok(RuntimeTaskExit::Scheduler(SchedulerLoopExit::Shutdown)) + | Ok(RuntimeTaskExit::FencedPoolFloor) | Ok(RuntimeTaskExit::Process(Ok(()))) | Ok(RuntimeTaskExit::Runner(Ok(()))) | Ok(RuntimeTaskExit::RepositoryWatch(Ok(()))) + | Ok(RuntimeTaskExit::RepositoryWatchLeaseExpiry(Ok(()))) | Ok(RuntimeTaskExit::ConvergenceSweep) | Ok(RuntimeTaskExit::WebHttp(Ok(()))) | Ok(RuntimeTaskExit::TurnLiveness) => RuntimeTaskCompletion::Clean, @@ -1014,6 +1158,10 @@ fn runtime_task_completion(completed: Result) -> Run report_repository_watch_runtime_defect(&error); RuntimeTaskCompletion::Defect } + Ok(RuntimeTaskExit::RepositoryWatchLeaseExpiry(Err(error))) => { + report_repository_watch_lease_expiry_failure(&error); + RuntimeTaskCompletion::Failed + } Ok(RuntimeTaskExit::WebHttp(Err(error))) => { report_web_http_runtime_failure(&error); RuntimeTaskCompletion::Failed @@ -1202,6 +1350,43 @@ async fn run_hub( }) }; let model_exchange_timeout = configured_duration("model_exchange_timeout"); + let codex_cli_version_probe_bound = configured_duration("codex_cli_version_probe_bound") + .filter(|bound| !bound.is_zero()) + .ok_or_else(|| { + erase_startup_cause( + RuntimePhase::Configuration, + SanitizedStartupCause::Static("invalid_codex_cli_version_probe_bound"), + ) + })?; + if let Some(codex_cli) = model_configuration.codex_cli() { + verify_pinned_codex_cli_version(codex_cli.executable(), codex_cli_version_probe_bound) + .await + .map_err(|_| { + erase_startup_cause( + RuntimePhase::Configuration, + SanitizedStartupCause::Static("codex_cli_version_probe_failed"), + ) + })?; + } + let fenced_pool_min_connections = + validate_fenced_pool_min_connections(configured_u32("fenced_pool_min_connections")?) + .ok_or_else(|| { + erase_startup_cause( + RuntimePhase::Configuration, + SanitizedStartupCause::Static("invalid_fenced_pool_min_connections"), + ) + })?; + let fenced_pool_floor_reconciliation = fenced_pool_floor_reconciliation_policy( + fenced_pool_min_connections, + configured_duration("fenced_pool_floor_reconciliation_interval"), + configured_duration("fenced_pool_floor_reconciliation_attempt_bound"), + ) + .ok_or_else(|| { + erase_startup_cause( + RuntimePhase::Configuration, + SanitizedStartupCause::Static("invalid_fenced_pool_floor_reconciliation_policy"), + ) + })?; let scheduler_pass_occupancy_bound = configured_duration("scheduler_pass_occupancy_bound") .map(SchedulerPassOccupancyBound::try_new) .transpose() @@ -1275,6 +1460,23 @@ async fn run_hub( ), )); } + // The claim statement schedules one `CASE` arm per admitted attempt and ends + // in an `ELSE`, so a budget above that arity is admitted silently and then + // reuses the last rung's deadline for every attempt past it while the + // failure path schedules the true exponential. The claim side is the shorter + // of the two, so the abandonment sweep would settle attempts that are still + // running. Refusing the budget here keeps the arity a configuration fact + // rather than something a deployment discovers from a mis-settled attempt. + if automatic_reconciliation_attempt_budget.is_some_and(|budget| { + usize::try_from(budget).is_ok_and(|budget| budget > RETRY_LADDER_ARITY) + }) { + return Err(erase_startup_cause( + RuntimePhase::Configuration, + SanitizedStartupCause::Static( + "automatic_reconciliation_attempt_budget_exceeds_retry_ladder", + ), + )); + } let automatic_reconciliation_base_backoff = configured_duration("automatic_reconciliation_base_backoff"); let automatic_reconciliation_backoff_cap = @@ -1285,7 +1487,10 @@ async fn run_hub( configured_duration("expired_pass_recovery_lock_retry_delay"), configured_duration("expired_pass_recovery_conservative_retry_delay"), ); - let repository_reconciliation_quantum = configured_usize("repository_reconciliation_quantum")?; + let repository_watch_numeric_bounds = RepositoryWatchNumericBounds::new( + configured_usize("repository_reconciliation_quantum")?, + configured_duration("webhook_drain_work_budget"), + ); let convergence_sweep_numeric_bounds = ConvergenceSweepNumericBounds::new( configured_duration("convergence_sweep_request_timeout"), configured_usize("max_convergence_sweep_connection_pages")?, @@ -1517,20 +1722,24 @@ async fn run_hub( diagnostic_model_identity_limit, ); let model_targets = model_configuration.target_catalog(); - let mut database = FencedHubDatabase::connect_production(configuration.database_url()) - .await - .map_err(|error| { - let phase = match &error { - FencedHubDatabaseError::InitializeFence(_) => RuntimePhase::Migration, - FencedHubDatabaseError::ParseOptions(_) - | FencedHubDatabaseError::ConnectBootstrap(_) - | FencedHubDatabaseError::AcquireGuard(_) - | FencedHubDatabaseError::AdvanceFence(_) - | FencedHubDatabaseError::ConnectFencedPool(_) => RuntimePhase::DatabaseConnection, - }; - erase_startup_cause(phase, SanitizedStartupCause::Database(&error)) - })?; + let mut database = FencedHubDatabase::connect_production( + configuration.database_url(), + fenced_pool_min_connections, + ) + .await + .map_err(|error| { + let phase = match &error { + FencedHubDatabaseError::InitializeFence(_) => RuntimePhase::Migration, + FencedHubDatabaseError::ParseOptions(_) + | FencedHubDatabaseError::ConnectBootstrap(_) + | FencedHubDatabaseError::AcquireGuard(_) + | FencedHubDatabaseError::AdvanceFence(_) + | FencedHubDatabaseError::ConnectFencedPool(_) => RuntimePhase::DatabaseConnection, + }; + erase_startup_cause(phase, SanitizedStartupCause::Database(&error)) + })?; let pool = database.pool().clone(); + let fenced_pool_floor_pool = pool.clone(); let tools = match daemon_tool_configuration { Some(tool_configuration) => DaemonTools::try_new_production( SystemCurrentTimeClock, @@ -1559,8 +1768,8 @@ async fn run_hub( model_configuration.web_fetch_egress_policy(), ), }; - let (tool_catalog, tool_executor) = match tools { - Ok(tools) => tools.into_parts(), + let tools = match tools { + Ok(tools) => tools, Err(error) => { let failure = erase_startup_cause( RuntimePhase::Configuration, @@ -1570,6 +1779,15 @@ async fn run_hub( return Err(failure); } }; + let workspace_instruction_runtime = WorkspaceInstructionRuntime::new( + pool.clone(), + tools.workspace_instruction_root_resolver(), + model_configuration + .workspace_instructions() + .roots() + .to_vec(), + ); + let (tool_catalog, tool_executor) = tools.into_parts(); let tool_catalog = match tool_catalog.with_approval_postures(model_configuration.tool_approval_postures()) { @@ -1637,7 +1855,7 @@ async fn run_hub( tracing::warn!( phase = ?RuntimePhase::StartupScan, session = %session.into_uuid(), - "session holds its slot awaiting bounded model-call reconciliation" + "session holds its slot awaiting a durable recovery decision" ); } Ok(()) @@ -1664,6 +1882,24 @@ async fn run_hub( .map(|repository| repository.repository().clone()) .collect() }); + let configured_convergence_targets = + model_configuration + .repository_watch() + .map_or_else(Vec::new, |configuration| { + if configuration.convergence_sweep().is_none() { + return Vec::new(); + } + configuration + .repositories() + .iter() + .flat_map(|repository| { + repository + .convergence_pull_requests() + .iter() + .map(|pull_request| (repository.repository().clone(), *pull_request)) + }) + .collect() + }); let repository_watch_store = PostgresRepoWatchDispatchStore::new( pool.clone(), model_configuration.session_credential_pin(), @@ -1784,28 +2020,41 @@ async fn run_hub( return Err(failure); } }; - let web_http_runtime = match WebHttpRuntime::bind(web_configuration, pool.clone()).await { - Ok(runtime) => runtime, - Err(_) => { - let failure = erase_startup_cause( - RuntimePhase::SocketBinding, - SanitizedStartupCause::Static("web_http_listener_bind_failed"), - ); - let _ = listener.cleanup(); - let _ = runner_listener.cleanup(); - disarm_staging_sweep_unless_guarded(&mut database, &mut blob_store_registry).await; - drop(blob_store_registry); - let _ = database.close().await; - return Err(failure); - } - }; + let web_http_runtime = + match WebHttpRuntime::bind(web_configuration, pool.clone(), model_configuration.clone()) + .await + { + Ok(runtime) => runtime, + Err(_) => { + let failure = erase_startup_cause( + RuntimePhase::SocketBinding, + SanitizedStartupCause::Static("web_http_listener_bind_failed"), + ); + let _ = listener.cleanup(); + let _ = runner_listener.cleanup(); + disarm_staging_sweep_unless_guarded(&mut database, &mut blob_store_registry).await; + drop(blob_store_registry); + let _ = database.close().await; + return Err(failure); + } + }; tracing::info!( phase = ?RuntimePhase::SocketBinding, "daemon startup phase completed" ); let repository_watch_reconciliation = async { + repository_watch_store + .process_pending_expired_start_leases(|| { + DurableCommandId::from_uuid(uuid::Uuid::now_v7()) + }) + .await?; repository_watch_store .process_pending_lifecycle_cutoffs(|| DurableCommandId::from_uuid(uuid::Uuid::now_v7())) + .await?; + repository_watch_store + .process_pending_convergence_cutoffs(|| { + DurableCommandId::from_uuid(uuid::Uuid::now_v7()) + }) .await }; match await_while_guarded(&mut database, repository_watch_reconciliation).await { @@ -1841,7 +2090,7 @@ async fn run_hub( nudge_buffer_capacity, ); let tool_dispatch_gate = InProcessToolDispatchGate::default(); - let repository_watch_runtime = match model_configuration.repository_watch() { + let mut repository_watch_runtime = match model_configuration.repository_watch() { Some(configuration) => match RepositoryWatchRuntime::try_new( pool.clone(), configuration, @@ -1849,7 +2098,7 @@ async fn run_hub( model_configuration.clone(), model_configuration.session_credential_pin(), eligibility_nudge.clone(), - repository_reconciliation_quantum, + repository_watch_numeric_bounds, ) { Ok(runtime) => Some(runtime), Err(_) => { @@ -1894,6 +2143,34 @@ async fn run_hub( }, None => None, }; + let convergence_sweep_store = PostgresConvergenceSweepStore::new(pool.clone()); + let convergence_target_admission = + convergence_sweep_store.reconcile_configured_targets(&configured_convergence_targets); + match await_while_guarded(&mut database, convergence_target_admission).await { + GuardedAwait::Completed(Ok(())) => {} + GuardedAwait::Completed(Err(_)) => { + let failure = erase_startup_cause( + RuntimePhase::StartupScan, + SanitizedStartupCause::Static("convergence_target_admission_failed"), + ); + let _ = listener.cleanup(); + let _ = runner_listener.cleanup(); + disarm_staging_sweep_unless_guarded(&mut database, &mut blob_store_registry).await; + drop(blob_store_registry); + let _ = database.close().await; + return Err(failure); + } + GuardedAwait::GuardLost => { + let _ = listener.cleanup(); + let _ = runner_listener.cleanup(); + if let Some(registry) = blob_store_registry.as_ref() { + registry.disarm_staging_sweep(); + } + drop(blob_store_registry); + let _ = database.close().await; + return Ok(ShutdownOutcome::GuardLost); + } + } // Every fallible construction above has succeeded, so the revisions this // consumes belong to a daemon that reaches its runtime. A startup that // failed earlier retired and activated nothing, leaving the previous @@ -1932,6 +2209,36 @@ async fn run_hub( return Ok(ShutdownOutcome::GuardLost); } } + if let Some(runtime) = repository_watch_runtime.as_mut() { + match await_while_guarded(&mut database, runtime.prepare_startup()).await { + GuardedAwait::Completed(Ok(())) => tracing::info!( + phase = ?RuntimePhase::StartupScan, + "daemon startup completed bounded repository-watch webhook reconciliation" + ), + GuardedAwait::Completed(Err(_)) => { + let failure = erase_startup_cause( + RuntimePhase::StartupScan, + SanitizedStartupCause::Static("repository_watch_startup_webhook_failed"), + ); + let _ = listener.cleanup(); + let _ = runner_listener.cleanup(); + disarm_staging_sweep_unless_guarded(&mut database, &mut blob_store_registry).await; + drop(blob_store_registry); + let _ = database.close().await; + return Err(failure); + } + GuardedAwait::GuardLost => { + let _ = listener.cleanup(); + let _ = runner_listener.cleanup(); + if let Some(registry) = blob_store_registry.as_ref() { + registry.disarm_staging_sweep(); + } + drop(blob_store_registry); + let _ = database.close().await; + return Ok(ShutdownOutcome::GuardLost); + } + } + } let runner_runtime = RunnerProtocolRuntime::new(runner_listener, runner_service); let process_runtime = ProcessRuntime::new_with_templates( listener, @@ -1960,6 +2267,11 @@ async fn run_hub( .with_credential_pools(model_configuration.credential_pool_runtime_catalog()) .with_cache_inclusive_input_targets(model_configuration.cache_inclusive_input_targets()) .with_continuation_usage_limits(model_configuration.tool_continuation_usage_limits()); + let provider = AttachmentPreparingModelCallProvider::new( + UsageLimitedModelCallProvider::new(provider, &model_configuration), + scheduler_pool.clone(), + blob_store_registry.clone(), + ); let reported_usage_compaction = ReportedUsageCompaction::new( StartEligibleTurnRepository::new(scheduler_pool.clone()), model_repository.clone(), @@ -1973,10 +2285,11 @@ async fn run_hub( PostgresProviderModelExecution::new( model_repository, InProcessAttemptDispatchGate::default(), - UsageLimitedModelCallProvider::new(provider, &model_configuration), + provider, automatic_tool_round_limit, ) .with_tool_loop(tool_dispatch_gate, tool_catalog, tool_executor) + .with_workspace_instructions(workspace_instruction_runtime) .with_approval_judge( approval_judge_model, model_configuration.configured_approval_judge_selection(), @@ -2053,9 +2366,14 @@ async fn run_hub( ); } let (scheduler_shutdown, scheduler_shutdown_receiver) = oneshot::channel(); + let (fenced_pool_floor_shutdown, fenced_pool_floor_shutdown_receiver) = watch::channel(false); let (process_shutdown, process_shutdown_receiver) = watch::channel(false); let (runner_shutdown, runner_shutdown_receiver) = watch::channel(false); let (repository_watch_shutdown, repository_watch_shutdown_receiver) = watch::channel(false); + let ( + repository_watch_lease_expiry_shutdown, + mut repository_watch_lease_expiry_shutdown_receiver, + ) = watch::channel(false); let (convergence_sweep_shutdown, convergence_sweep_shutdown_receiver) = watch::channel(false); let (web_http_shutdown, web_http_shutdown_receiver) = watch::channel(false); let (turn_liveness_shutdown, turn_liveness_shutdown_receiver) = watch::channel(false); @@ -2069,6 +2387,17 @@ async fn run_hub( .await, ) }); + if let Some(policy) = fenced_pool_floor_reconciliation { + runtime_tasks.spawn(async move { + run_fenced_pool_floor_reconciliation( + fenced_pool_floor_pool, + policy, + fenced_pool_floor_shutdown_receiver, + ) + .await; + RuntimeTaskExit::FencedPoolFloor + }); + } runtime_tasks.spawn(async move { RuntimeTaskExit::Process(process_runtime.run(process_shutdown_receiver).await) }); @@ -2087,6 +2416,33 @@ async fn run_hub( ) }); } + let repository_watch_lease_expiry_store = repository_watch_store.clone(); + runtime_tasks.spawn(async move { + let mut ticker = tokio::time::interval(Duration::from_secs(1)); + ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); + let outcome = loop { + select! { + changed = repository_watch_lease_expiry_shutdown_receiver.changed() => { + if changed.is_err() + || *repository_watch_lease_expiry_shutdown_receiver.borrow_and_update() + { + break Ok(()); + } + } + _ = ticker.tick() => { + if let Err(error) = repository_watch_lease_expiry_store + .process_pending_expired_start_leases(|| { + DurableCommandId::from_uuid(uuid::Uuid::now_v7()) + }) + .await + { + break Err(error); + } + } + } + }; + RuntimeTaskExit::RepositoryWatchLeaseExpiry(outcome) + }); if let Some(convergence_sweep_runtime) = convergence_sweep_runtime { runtime_tasks.spawn(async move { convergence_sweep_runtime @@ -2122,6 +2478,12 @@ async fn run_hub( report_process_runtime_failure(&error); RuntimeStopCause::RuntimeFailed } + Some(Ok(RuntimeTaskExit::FencedPoolFloor)) => { + report_runtime_task_defect( + RuntimeTaskDefect::FencedPoolFloorCompletedBeforeShutdown, + ); + RuntimeStopCause::RuntimeDefect + } Some(Ok(RuntimeTaskExit::Process(Ok(())))) => { report_runtime_task_defect( RuntimeTaskDefect::ProcessCompletedBeforeShutdown, @@ -2148,6 +2510,16 @@ async fn run_hub( ); RuntimeStopCause::RuntimeDefect } + Some(Ok(RuntimeTaskExit::RepositoryWatchLeaseExpiry(Err(error)))) => { + report_repository_watch_lease_expiry_failure(&error); + RuntimeStopCause::RuntimeFailed + } + Some(Ok(RuntimeTaskExit::RepositoryWatchLeaseExpiry(Ok(())))) => { + report_runtime_task_defect( + RuntimeTaskDefect::RepositoryWatchLeaseExpiryCompletedBeforeShutdown, + ); + RuntimeStopCause::RuntimeDefect + } Some(Ok(RuntimeTaskExit::ConvergenceSweep)) => { report_runtime_task_defect( RuntimeTaskDefect::ConvergenceSweepCompletedBeforeShutdown, @@ -2195,9 +2567,11 @@ async fn run_hub( } else { let _ = turn_execution_shutdown.send(true); let _ = scheduler_shutdown.send(()); + let _ = fenced_pool_floor_shutdown.send(true); let _ = process_shutdown.send(true); let _ = runner_shutdown.send(true); let _ = repository_watch_shutdown.send(true); + let _ = repository_watch_lease_expiry_shutdown.send(true); let _ = convergence_sweep_shutdown.send(true); let _ = web_http_shutdown.send(true); let _ = turn_liveness_shutdown.send(true); @@ -2538,6 +2912,7 @@ mod tests { use super::{ AnthropicConstructionError, BRAVE_API_KEY_FILE_ENVIRONMENT, DATABASE_URL_ENVIRONMENT, + FENCED_POOL_MAX_CONNECTIONS, FencedPoolFloorReconciliationPolicy, GITHUB_TOKEN_FILE_ENVIRONMENT, HubConfiguration, HubConfigurationError, HubConfigurationValues, HubRuntimeError, MODEL_CONFIGURATION_FILE_ENVIRONMENT, OpenAiConstructionError, OperatorFilterDisposition, PROCESS_SOCKET_PATH_ENVIRONMENT, @@ -2547,16 +2922,69 @@ mod tests { ShutdownOutcome, SingleHubGuardError, TEMPLATE_CONFIGURATION_FILE_ENVIRONMENT, anthropic_construction_cause, combine_runtime_stop_cause, completed_runtime_outcome, credential_files_conflict, database_close_failure_outcome, drain_runtime_tasks, - erase_startup_cause, graceful_shutdown_window, migrate_scan_then_schedule, - openai_construction_cause, operator_filter, process_runtime_failure_class, - report_database_close_failure, repository_watch_rule_configuration_error, - run_scheduler_until_shutdown, runner_lifecycle_failure_class, should_close_pool, - staging_sweep_failure_outcome, + erase_startup_cause, fenced_pool_floor_reconciliation_policy, graceful_shutdown_window, + migrate_scan_then_schedule, openai_construction_cause, operator_filter, + process_runtime_failure_class, report_database_close_failure, + repository_watch_rule_configuration_error, run_scheduler_until_shutdown, + runner_lifecycle_failure_class, should_close_pool, staging_sweep_failure_outcome, + validate_fenced_pool_min_connections, }; use signalboxd::runner_protocol_runtime::RunnerRegistrationFailureCause; const BRAVE_KEY_FILE_FIXTURE: &str = "brave-key"; + #[test] + fn fenced_pool_prewarm_cannot_exceed_the_compiled_capacity() { + assert_eq!( + validate_fenced_pool_min_connections(Some(FENCED_POOL_MAX_CONNECTIONS)), + Some(Some(FENCED_POOL_MAX_CONNECTIONS)) + ); + assert_eq!( + validate_fenced_pool_min_connections(Some(FENCED_POOL_MAX_CONNECTIONS + 1)), + None + ); + assert_eq!(validate_fenced_pool_min_connections(None), Some(None)); + } + + #[test] + fn positive_fenced_pool_floor_requires_bounded_reconciliation() { + let interval = Duration::from_secs(5); + let attempt_bound = Duration::from_secs(30); + + assert_eq!( + fenced_pool_floor_reconciliation_policy( + Some(FENCED_POOL_MAX_CONNECTIONS), + Some(interval), + Some(attempt_bound), + ), + Some(Some(FencedPoolFloorReconciliationPolicy { + minimum: FENCED_POOL_MAX_CONNECTIONS, + interval, + attempt_bound, + })) + ); + assert_eq!( + fenced_pool_floor_reconciliation_policy( + Some(FENCED_POOL_MAX_CONNECTIONS), + None, + Some(attempt_bound), + ), + None + ); + assert_eq!( + fenced_pool_floor_reconciliation_policy( + Some(FENCED_POOL_MAX_CONNECTIONS), + Some(interval), + None, + ), + None + ); + assert_eq!( + fenced_pool_floor_reconciliation_policy(None, None, None), + Some(None) + ); + } + fn hub_configuration_values() -> HubConfigurationValues { HubConfigurationValues { database_url: Some(OsString::from("postgres://secret")), diff --git a/apps/signalboxd/src/model_adapter.rs b/apps/signalboxd/src/model_adapter.rs index 08687ba90b..56fafd29a9 100644 --- a/apps/signalboxd/src/model_adapter.rs +++ b/apps/signalboxd/src/model_adapter.rs @@ -329,7 +329,7 @@ mod tests { Some(signalbox_model_runtime::AssistantPart::Thinking { .. }) | Some(signalbox_model_runtime::AssistantPart::RedactedThinking { .. }) | Some(signalbox_model_runtime::AssistantPart::ToolCall(_)) - | Some(signalbox_model_runtime::AssistantPart::SuppressedToolCall) + | Some(signalbox_model_runtime::AssistantPart::SuppressedToolCall(_)) | None => None, }, TerminalEvidence::Refused(_) diff --git a/apps/signalboxd/src/process_runtime.rs b/apps/signalboxd/src/process_runtime.rs index 69a40baa9e..288646fee9 100644 --- a/apps/signalboxd/src/process_runtime.rs +++ b/apps/signalboxd/src/process_runtime.rs @@ -21,17 +21,18 @@ use signalbox_application::{ EligibilityNudge, ImportConversationError, ImportConversationOutcome, ImportConversationService, ImportedConversationConverter, InProcessEligibilityNudge, InProcessToolDispatchGate, ListConversationsService, ListSessionMetadataService, - LoadSessionMetadataService, OperatorFailureClass, PromptMemberStatement, - ReplaceSessionDefaultsOutcome, ReplaceSessionDefaultsRequest, ReplaceSessionDefaultsService, - ReplaceSessionMetadataOutcome, ReplaceSessionMetadataRequest, ReplaceSessionMetadataService, - ReviewPassCompletionStatus, ReviewWorkflowCommand, ReviewWorkflowCommandOutcome, - ReviewWorkflowCommandResult, ReviewWorkflowCommandService, ReviewWorkflowOperation, - ReviewWorkflowOperationKind, SessionMetadataListItem, SessionMetadataListQuery, - SubmitInputOutcome, SubmitInputRequest, SubmitInputService, SubmitInputTransaction, - UpdateSessionPlacementOutcome, UpdateSessionPlacementRequest, UpdateSessionPlacementService, - UuidV7CommissionedDispatchIdGenerator, UuidV7CreateSessionFromImportedFrontierIdGenerator, - UuidV7ImportedConversationIdGenerator, UuidV7SessionIdGenerator, UuidV7SubmitInputIdGenerator, - UuidV7ToolLoopIdGenerator, + LoadSessionMetadataService, OperatorFailureClass, OverrideDeniedToolRequestService, + PromptMemberStatement, ReplaceSessionDefaultsOutcome, ReplaceSessionDefaultsRequest, + ReplaceSessionDefaultsService, ReplaceSessionMetadataOutcome, ReplaceSessionMetadataRequest, + ReplaceSessionMetadataService, ReviewPassCompletionStatus, ReviewWorkflowCommand, + ReviewWorkflowCommandOutcome, ReviewWorkflowCommandResult, ReviewWorkflowCommandService, + ReviewWorkflowOperation, ReviewWorkflowOperationKind, SessionMetadataListItem, + SessionMetadataListQuery, SubmitInputOutcome, SubmitInputRequest, SubmitInputService, + SubmitInputTransaction, UpdateSessionPlacementOutcome, UpdateSessionPlacementRequest, + UpdateSessionPlacementService, UuidV7CommissionedDispatchIdGenerator, + UuidV7CreateSessionFromImportedFrontierIdGenerator, UuidV7ImportedConversationIdGenerator, + UuidV7SessionIdGenerator, UuidV7SubmitInputIdGenerator, UuidV7ToolLoopIdGenerator, + render_model_user_content, }; use signalbox_blob_store::ExpectedBlob; use signalbox_conversation_import_claude_code::{ @@ -61,15 +62,17 @@ use signalbox_domain::{ ModelChangeAdjustment as DomainModelChangeAdjustment, ModelSelectionOverride, ModelSelectionRequest, ModelSettingSource as DomainModelSettingSource, ModelSettingsOverlay as DomainModelSettingsOverlay, - ModelSettingsPrecedence as DomainModelSettingsPrecedence, ParentTerminationCommandSource, - PerInputConfigurationChoices, PullRequestNumber, ReasoningLevel as DomainReasoningLevel, - ReplaceSessionDefaults as DomainReplaceSessionDefaults, ReplaceSessionDefaultsRejectedResult, - ReplaceSessionDefaultsResult, ReplaceSessionMetadataRejectedResult, - ReplaceSessionMetadataResult, RepositorySlug, ReviewChangeRequestNumber, ReviewConfidence, - ReviewEventOrdinal, ReviewExternalLink, ReviewExternalLinkAssociation, - ReviewExternalLinkAttachment, ReviewExternalLinkAttachmentResult, ReviewExternalLinkId, - ReviewExternalObjectKind, ReviewFinding, ReviewFindingConfidenceAxes, ReviewFindingContent, - ReviewFindingDiffSide, ReviewFindingEvent, ReviewFindingEventKind, ReviewFindingEventResult, + ModelSettingsPrecedence as DomainModelSettingsPrecedence, OverrideDeniedToolRequest, + OverrideDeniedToolRequestRejectedResult, OverrideDeniedToolRequestResult, + ParentTerminationCommandSource, PerInputConfigurationChoices, PullRequestNumber, + ReasoningLevel as DomainReasoningLevel, ReplaceSessionDefaults as DomainReplaceSessionDefaults, + ReplaceSessionDefaultsRejectedResult, ReplaceSessionDefaultsResult, + ReplaceSessionMetadataRejectedResult, ReplaceSessionMetadataResult, RepositorySlug, + ReviewChangeRequestNumber, ReviewConfidence, ReviewEventOrdinal, ReviewExternalLink, + ReviewExternalLinkAssociation, ReviewExternalLinkAttachment, + ReviewExternalLinkAttachmentResult, ReviewExternalLinkId, ReviewExternalObjectKind, + ReviewFinding, ReviewFindingConfidenceAxes, ReviewFindingContent, ReviewFindingDiffSide, + ReviewFindingEvent, ReviewFindingEventKind, ReviewFindingEventResult, ReviewFindingEventResultKind, ReviewFindingId, ReviewFindingLocation, ReviewFindingPendingExternalLinkRef, ReviewFindingProposal, ReviewFindingRef, ReviewFindingSeverity, ReviewKey, ReviewLineRange, ReviewPass, ReviewPassAcceptedInputEvidence, @@ -101,7 +104,8 @@ use signalbox_persistence::{ PostgresCommissionedDispatchStore, }, context_compaction::{ - AppliedContextCompaction, ContextCompactionCommandLookup, ContextCompactionRepository, + AppliedContextCompaction, AutomaticContextCompactionPreviewMember, + ContextCompactionCommandLookup, ContextCompactionRepository, ContextCompactionRepositoryError, FailedContextCompactionDisposition, PrepareContextCompactionOutcome, PrepareContextCompactionRequest, PreparedContextCompaction, @@ -115,6 +119,13 @@ use signalbox_persistence::{ goal::{GoalCommandHandlingOutcome, GoalRepository, GoalRepositoryError}, goal_turn::GoalTurnCandidates, model_execution::{ModelCallRepositoryError, PostgresModelCallRepository}, + operator_status::{ + ProcessOperatorStatusConvergenceSeal, ProcessOperatorStatusConvergenceVerdict, + ProcessOperatorStatusError, ProcessOperatorStatusHeldSlotBlocker, + ProcessOperatorStatusHeldSlotOrigin, ProcessOperatorStatusItem, + ProcessOperatorStatusMergeableState, ProcessOperatorStatusRepository, + ProcessOperatorStatusReviewDecision, ProcessOperatorStatusSingletonScope, + }, outbox::{ DispatchedBoundChildAction, DispatchedDelegationOutcome, DispatchedDelegationPolicy, DispatchedDelegationProvenance, DispatchedDelegationReason, DispatchedDelegationUpdate, @@ -175,8 +186,17 @@ use signalbox_process_protocol::{ ModelSelection as WireModelSelection, ModelSettingSource as WireModelSettingSource, ModelSettingsOverlay as WireModelSettingsOverlay, ModelSettingsPrecedence as WireModelSettingsPrecedence, - ModelSettingsSnapshot as WireModelSettingsSnapshot, PositiveCanonicalU64, ProtocolVersion, - ReasoningLevel as WireReasoningLevel, RejectionDetail, RequestId, + ModelSettingsSnapshot as WireModelSettingsSnapshot, + OperatorStatusConvergenceSeal as WireOperatorStatusConvergenceSeal, + OperatorStatusConvergenceVerdict as WireOperatorStatusConvergenceVerdict, + OperatorStatusEndMessage, OperatorStatusHeldSlotBlocker as WireOperatorStatusHeldSlotBlocker, + OperatorStatusHeldSlotMessage, OperatorStatusHeldSlotOrigin, + OperatorStatusMergeableState as WireOperatorStatusMergeableState, OperatorStatusMessage, + OperatorStatusPendingStaleReviewClearanceMessage, OperatorStatusPullRequestConvergenceMessage, + OperatorStatusQueuedObligationMessage, + OperatorStatusReviewDecision as WireOperatorStatusReviewDecision, + OperatorStatusSingletonScope as WireOperatorStatusSingletonScope, PositiveCanonicalU64, + ProtocolVersion, ReasoningLevel as WireReasoningLevel, RejectionDetail, RequestId, ReviewDiffSide as WireReviewDiffSide, ReviewExternalObjectKind as WireReviewExternalObjectKind, ReviewFindingEvent as WireReviewFindingEvent, ReviewFindingInput, ReviewFindingSnapshot, ReviewFindingStatus as WireReviewFindingStatus, ReviewPassLifecycle, ReviewPassSnapshot, @@ -200,7 +220,7 @@ use signalbox_process_protocol::{ ToolApprovalEventDecision as WireToolApprovalEventDecision, ToolBatchState, ToolDecision, TranscriptEntry, TranscriptTextEntry, TranscriptToolApproval, TurnModelSettingsSnapshot as WireTurnModelSettingsSnapshot, TurnState, UsageProvenance, - content_fragments, decode_client_line, encode_server_line, + UserInputContent, content_fragments, decode_client_line, encode_server_line, recover_bounded_client_protocol_version, recover_bounded_client_request_id, }; use signalbox_tools_sessions::{AwaitSessionPortOutcome, DeliveredChildResult}; @@ -251,7 +271,6 @@ const BULK_INGEST_SESSION_TIMEOUT: Duration = Duration::from_secs(24 * 60 * 60); /// Hard safety ceiling bounding store latency and retained read capacity. const BLOB_READ_TIMEOUT: Duration = Duration::from_secs(24 * 60 * 60); const INBOUND_READ_AHEAD_BYTES: usize = 8 * 1024; -const MAX_SUBMITTED_INPUT_BYTES: usize = 1024 * 1024; const RESERVED_POOL_CONNECTIONS_OUTSIDE_SNAPSHOTS: u32 = 2; #[derive(Debug)] @@ -881,7 +900,7 @@ async fn serve_connection( !active_lifecycle_request, ) .or_else(|| acquired_bulk_ingest_at.map(|started| started + BULK_INGEST_SESSION_TIMEOUT)); - let request_result = handle_request( + let request_result = Box::pin(handle_request( &mut reader, &mut writer, version, @@ -896,7 +915,7 @@ async fn serve_connection( }, &services, shutdown.clone(), - ); + )); tokio::select! { biased; () = wait_for_deadline(operation_deadline) => return Ok(()), @@ -1070,6 +1089,7 @@ fn conversation_import_request_requires_permit( | ClientRequest::ListTemplates {} | ClientRequest::ReadDeploymentLimits {} | ClientRequest::ListSessions {} + | ClientRequest::ReadOperatorStatus {} | ClientRequest::UpdateSessionPlacement { .. } | ClientRequest::AttachGoal { .. } | ClientRequest::ReadGoal { .. } @@ -1123,7 +1143,8 @@ fn conversation_import_request_requires_permit( | ClientRequest::RecordReviewPublicationOutcomes { .. } | ClientRequest::ReadReviewOrchestration { .. } | ClientRequest::StopTurn { .. } - | ClientRequest::DecideToolRequest { .. } => false, + | ClientRequest::DecideToolRequest { .. } + | ClientRequest::OverrideDeniedToolRequest { .. } => false, } } fn retain_inbound_frame_permit_during_import_admission( @@ -1253,6 +1274,7 @@ impl SnapshotReaderAdmission { const fn for_request(request: &ClientRequest) -> Self { match request { ClientRequest::ListSessions {} + | ClientRequest::ReadOperatorStatus {} | ClientRequest::ReadGoal { .. } | ClientRequest::ReadTranscript { .. } | ClientRequest::FollowSession { .. } @@ -1320,7 +1342,8 @@ impl SnapshotReaderAdmission { | ClientRequest::RecordReviewRepairOutcomes { .. } | ClientRequest::RecordReviewPublicationOutcomes { .. } | ClientRequest::StopTurn { .. } - | ClientRequest::DecideToolRequest { .. } => Self::NotRequired, + | ClientRequest::DecideToolRequest { .. } + | ClientRequest::OverrideDeniedToolRequest { .. } => Self::NotRequired, } } } @@ -1643,6 +1666,19 @@ where }; handle_list_sessions(writer, version, request_id, &services.pool, snapshot_permit).await } + ClientRequest::ReadOperatorStatus {} => { + let Some(snapshot_permit) = snapshot_permit else { + return Ok(()); + }; + Box::pin(handle_operator_status( + writer, + version, + request_id, + &services.pool, + snapshot_permit, + )) + .await + } ClientRequest::UpdateSessionPlacement { command_id, session_id, @@ -1841,6 +1877,7 @@ where &services.eligibility_nudge, &services.tool_dispatch_gate, services.model_configuration.as_ref(), + services.blob_store_registry.as_deref(), ) .await } @@ -1866,6 +1903,7 @@ where &services.eligibility_nudge, &services.tool_dispatch_gate, services.model_configuration.as_ref(), + services.blob_store_registry.as_deref(), ) .await } @@ -2514,6 +2552,7 @@ where &services.eligibility_nudge, &services.tool_dispatch_gate, services.model_configuration.as_ref(), + services.blob_store_registry.as_deref(), ) .await } @@ -2582,6 +2621,22 @@ where ) .await } + ClientRequest::OverrideDeniedToolRequest { + command_id, + session_id, + tool_request_id, + } => { + handle_override_denied_tool_request( + writer, + version, + request_id, + command_id.into_uuid(), + session_id, + tool_request_id, + &services.pool, + ) + .await + } } } @@ -7166,6 +7221,7 @@ pub(crate) enum AutomaticContextCompactionError { Repository(ContextCompactionRepositoryError), Model, Configuration, + InputDoesNotFit, State, Integrity, AlreadyAttempted, @@ -7192,7 +7248,7 @@ impl ClassifyOperatorFailure for AutomaticContextCompactionError { Self::Read(ProcessReadError::Corruption(_)) | Self::Integrity => { signalbox_application::OperatorFailureClass::FailClosedCorruption } - Self::Configuration | Self::State | Self::AlreadyAttempted => { + Self::Configuration | Self::InputDoesNotFit | Self::State | Self::AlreadyAttempted => { signalbox_application::OperatorFailureClass::CallerOrHubBug } } @@ -7217,6 +7273,7 @@ impl ClassifyOperatorFailure for AutomaticContextCompactionError { } Self::Model => "context_compaction_model", Self::Configuration => "context_compaction_configuration", + Self::InputDoesNotFit => "context_compaction_input_does_not_fit", Self::State => "context_compaction_state", Self::Integrity => "context_compaction_integrity", Self::AlreadyAttempted => "context_compaction_already_attempted", @@ -7224,6 +7281,119 @@ impl ClassifyOperatorFailure for AutomaticContextCompactionError { } } +async fn automatic_context_compaction_boundary( + members: &[AutomaticContextCompactionPreviewMember], + entries: &[ProcessTranscriptEntry], + input_byte_budget: u64, + catalog: &BlobCatalogRepository, +) -> Result, AutomaticContextCompactionError> { + if members.len() != entries.len() { + return Err(AutomaticContextCompactionError::Integrity); + } + let mut encoded_lengths = Vec::with_capacity(entries.len()); + let mut boundaries = Vec::with_capacity(entries.len()); + for (member, entry) in members.iter().zip(entries) { + if member.reference() != transcript_entry_reference(entry) { + return Err(AutomaticContextCompactionError::Integrity); + } + // Attachment parts resolve through the blob catalog, so rendering one + // entry is a database read that carries the same closed dispositions + // the compaction range load already maps. + let value = context_compaction_entry_value(entry, catalog) + .await + .map_err(|error| match error { + ContextCompactionRangeLoadError::Read(error) => { + AutomaticContextCompactionError::Read(error) + } + ContextCompactionRangeLoadError::CatalogUnavailable => { + AutomaticContextCompactionError::Model + } + ContextCompactionRangeLoadError::Integrity => { + AutomaticContextCompactionError::Integrity + } + })?; + let encoded = + serde_json::to_vec(&value).map_err(|_| AutomaticContextCompactionError::Integrity)?; + encoded_lengths.push( + u64::try_from(encoded.len()).map_err(|_| AutomaticContextCompactionError::Integrity)?, + ); + boundaries.push((member.position(), member.is_safe_boundary())); + } + let selected = + bounded_rendered_compaction_boundary(&encoded_lengths, &boundaries, input_byte_budget); + let selected_only_current_summary = selected + == boundaries.first().map(|(position, _)| *position) + && matches!( + entries.first(), + Some(ProcessTranscriptEntry::ContextSummary { .. }) + ); + if selected_only_current_summary + && successor_compaction_cannot_advance(&encoded_lengths, &boundaries, input_byte_budget) + { + return Ok(None); + } + Ok(selected) +} + +fn successor_compaction_cannot_advance( + encoded_lengths: &[u64], + boundaries: &[(u64, bool)], + input_byte_budget: u64, +) -> bool { + if encoded_lengths.len() != boundaries.len() || encoded_lengths.len() < 2 { + return true; + } + let mut minimum_bytes = 2_u64; + for (encoded_length, (_, safe_boundary)) in encoded_lengths[1..].iter().zip(&boundaries[1..]) { + minimum_bytes = minimum_bytes + .saturating_add(1) + .saturating_add(*encoded_length); + if *safe_boundary { + return minimum_bytes > input_byte_budget; + } + } + true +} + +fn bounded_rendered_compaction_boundary( + encoded_lengths: &[u64], + boundaries: &[(u64, bool)], + input_byte_budget: u64, +) -> Option { + if encoded_lengths.len() != boundaries.len() { + return None; + } + let separators = u64::try_from(encoded_lengths.len().saturating_sub(1)).ok()?; + let total_bytes = encoded_lengths + .iter() + .fold(2_u64, |total, length| total.saturating_add(*length)); + let target_bytes = total_bytes + .saturating_add(separators) + .div_ceil(2) + .min(input_byte_budget); + let mut prefix_bytes = 1_u64; + let mut latest_safe = None; + for (index, ((position, safe_boundary), encoded_length)) in + boundaries.iter().zip(encoded_lengths).enumerate() + { + if index > 0 { + prefix_bytes = prefix_bytes.saturating_add(1); + } + prefix_bytes = prefix_bytes.saturating_add(*encoded_length); + let candidate_bytes = prefix_bytes.saturating_add(1); + if candidate_bytes > input_byte_budget { + break; + } + if *safe_boundary { + latest_safe = Some(*position); + if candidate_bytes >= target_bytes { + return latest_safe; + } + } + } + latest_safe +} + pub(crate) async fn compact_automatically( model_calls: &PostgresModelCallRepository, model_configuration: &HubModelConfiguration, @@ -7254,21 +7424,59 @@ pub(crate) async fn compact_automatically( .resolve(FrozenModelSelection::Direct(selection)) .map_err(|_| AutomaticContextCompactionError::Configuration)? .target(); - let input_includes_cache_tokens = model_configuration + let route = model_configuration .resolve_direct_model(selection) - .ok_or(AutomaticContextCompactionError::Configuration)? - .adapter() - .reports_cache_inclusive_input(); + .ok_or(AutomaticContextCompactionError::Configuration)?; + let input_includes_cache_tokens = route.adapter().reports_cache_inclusive_input(); + let runtime_models = model_configuration.runtime_model_catalog(); + let definition = runtime_models + .resolve(target) + .ok_or(AutomaticContextCompactionError::Configuration)?; + let compaction_prompt = model_configuration.compaction_prompt(); + let prompt_bytes = u64::try_from(compaction_prompt.len()) + .map_err(|_| AutomaticContextCompactionError::Configuration)?; + let automatic_input_byte_budget = u64::from(definition.context_window_tokens()) + .checked_sub(u64::from(definition.max_output_tokens())) + .and_then(|available| available.checked_sub(prompt_bytes)) + .filter(|available| *available > 0) + .ok_or(AutomaticContextCompactionError::Configuration)?; let credential_reference = model_calls .resolve_session_credential_reference(session, target) .await .map_err(AutomaticContextCompactionError::Credential)?; let repository = ContextCompactionRepository::new(model_calls.pool().clone()); + let preview = repository + .preview_automatic_range(session) + .await + .map_err(AutomaticContextCompactionError::Repository)? + .ok_or(AutomaticContextCompactionError::State)?; + let preview_positions = preview + .members() + .iter() + .map(|member| member.position()) + .collect::>(); + let preview_entries = preview + .members() + .iter() + .map(|member| member.reference()) + .collect::>(); + let rendered_entries = ProcessReadRepository::new(model_calls.pool().clone()) + .read_selected_transcript_entries(&preview_positions, &preview_entries) + .await + .map_err(AutomaticContextCompactionError::Read)?; + let requested_through_position = automatic_context_compaction_boundary( + preview.members(), + &rendered_entries, + automatic_input_byte_budget, + &BlobCatalogRepository::new(model_calls.pool().clone()), + ) + .await? + .ok_or(AutomaticContextCompactionError::InputDoesNotFit)?; let prepared = loop { let request = PrepareContextCompactionRequest { command: DurableCommandId::from_uuid(uuid::Uuid::now_v7()), session, - requested_through_position: None, + requested_through_position: Some(requested_through_position), automatic_for_turn: Some(turn), defaults_version: defaults.version(), selection, @@ -7317,6 +7525,9 @@ pub(crate) async fn compact_automatically( .map_err(AutomaticContextCompactionError::Repository)?; return Err(AutomaticContextCompactionError::Read(error)); } + Err(ContextCompactionRangeLoadError::CatalogUnavailable) => { + return Err(AutomaticContextCompactionError::Model); + } Err(ContextCompactionRangeLoadError::Integrity) => { fail_context_compaction_until_resolved( &repository, @@ -7328,6 +7539,19 @@ pub(crate) async fn compact_automatically( return Err(AutomaticContextCompactionError::Integrity); } }; + if u64::try_from(rendered_range.len()) + .ok() + .is_none_or(|rendered_bytes| rendered_bytes > automatic_input_byte_budget) + { + fail_context_compaction_until_resolved( + &repository, + &prepared, + FailedContextCompactionDisposition::KnownFailed, + ) + .await + .map_err(AutomaticContextCompactionError::Repository)?; + return Err(AutomaticContextCompactionError::InputDoesNotFit); + } authorize_context_compaction_until_resolved(&repository, &prepared) .await .map_err(AutomaticContextCompactionError::Repository)?; @@ -7337,7 +7561,7 @@ pub(crate) async fn compact_automatically( selection: prepared.selection(), target: prepared.target(), credential_reference: prepared.credential_reference().to_owned(), - system_prompt: model_configuration.compaction_prompt().to_owned(), + system_prompt: compaction_prompt.to_owned(), rendered_range, }; let result = match model.execute(request).await { @@ -7384,10 +7608,11 @@ async fn load_context_compaction_range( { return Err(ContextCompactionRangeLoadError::Integrity); } - let values = entries - .iter() - .map(context_compaction_entry_value) - .collect::>(); + let catalog = BlobCatalogRepository::new(pool.clone()); + let mut values = Vec::with_capacity(entries.len()); + for entry in &entries { + values.push(context_compaction_entry_value(entry, &catalog).await?); + } serde_json::to_string(&values).map_err(|_| ContextCompactionRangeLoadError::Integrity) } @@ -7403,6 +7628,9 @@ where Err(ContextCompactionRangeLoadError::Read(ProcessReadError::Database(_))) => { sleep(CONTEXT_COMPACTION_PERSISTENCE_RETRY_INTERVAL).await; } + Err(ContextCompactionRangeLoadError::CatalogUnavailable) => { + sleep(CONTEXT_COMPACTION_PERSISTENCE_RETRY_INTERVAL).await; + } result => return result, } } @@ -7496,7 +7724,10 @@ fn transcript_entry_reference( signalbox_domain::SemanticTranscriptEntryRef::from_source(source_session, entry) } -fn context_compaction_entry_value(entry: &ProcessTranscriptEntry) -> serde_json::Value { +async fn context_compaction_entry_value( + entry: &ProcessTranscriptEntry, + catalog: &BlobCatalogRepository, +) -> Result { let reference = transcript_entry_reference(entry); let source_session_id = reference .source_session() @@ -7504,7 +7735,7 @@ fn context_compaction_entry_value(entry: &ProcessTranscriptEntry) -> serde_json: .hyphenated() .to_string(); let entry_id = reference.entry().into_uuid().hyphenated().to_string(); - match entry { + let value = match entry { ProcessTranscriptEntry::DelegatedTask { entry_index, spawning_request, @@ -7615,15 +7846,42 @@ fn context_compaction_entry_value(entry: &ProcessTranscriptEntry) -> serde_json: turn, content, .. - } => serde_json::json!({ - "position": entry_index + 1, - "source_session_id": source_session_id, - "entry_id": entry_id, - "type": "user", - "accepted_input_id": accepted_input.into_uuid().hyphenated().to_string(), - "turn_id": turn.into_uuid().hyphenated().to_string(), - "content": content, - }), + } => { + let mut lengths = std::collections::BTreeMap::new(); + for part in content.parts() { + let signalbox_domain::UserContentPart::Attachment { digest, .. } = part else { + continue; + }; + if lengths.contains_key(digest) { + continue; + } + let catalog_entry = catalog + .find(*digest) + .await + .map_err(map_context_compaction_catalog_error)? + .ok_or(ContextCompactionRangeLoadError::Integrity)?; + let length = NonZeroU64::new(catalog_entry.expected().byte_length()) + .ok_or(ContextCompactionRangeLoadError::Integrity)?; + lengths.insert(*digest, length); + } + let rendered = + render_model_user_content(content.clone(), |digest| lengths.get(&digest).copied()) + .map_err(|_| ContextCompactionRangeLoadError::Integrity)?; + let rendered_parts = rendered + .parts() + .iter() + .map(|part| part.as_str()) + .collect::>(); + serde_json::json!({ + "position": entry_index + 1, + "source_session_id": source_session_id, + "entry_id": entry_id, + "type": "user", + "accepted_input_id": accepted_input.into_uuid().hyphenated().to_string(), + "turn_id": turn.into_uuid().hyphenated().to_string(), + "content": rendered_parts, + }) + } ProcessTranscriptEntry::Assistant { entry_index, turn, @@ -7761,6 +8019,21 @@ fn context_compaction_entry_value(entry: &ProcessTranscriptEntry) -> serde_json: "source_speaker": imported_source_speaker_label(*source_speaker), "content_kind": imported_content_kind_label(*content_kind), }), + }; + Ok(value) +} + +fn map_context_compaction_catalog_error( + error: signalbox_persistence::blob::BlobCatalogRepositoryError, +) -> ContextCompactionRangeLoadError { + match error { + signalbox_persistence::blob::BlobCatalogRepositoryError::Database(_) + | signalbox_persistence::blob::BlobCatalogRepositoryError::CommitAmbiguous(_) => { + ContextCompactionRangeLoadError::CatalogUnavailable + } + signalbox_persistence::blob::BlobCatalogRepositoryError::Corruption(_) => { + ContextCompactionRangeLoadError::Integrity + } } } @@ -7908,6 +8181,15 @@ where Writer: AsyncWrite + Unpin, { match error { + ContextCompactionRangeLoadError::CatalogUnavailable => { + write_error( + writer, + version, + request_id, + ProtocolError::without_detail(ErrorCode::Unavailable), + ) + .await + } ContextCompactionRangeLoadError::Read(error) => { if let Err(repository_error) = fail_context_compaction_until_resolved( repository, @@ -8062,6 +8344,7 @@ where #[derive(Debug)] enum ContextCompactionRangeLoadError { Read(ProcessReadError), + CatalogUnavailable, Integrity, } @@ -8817,7 +9100,8 @@ where ) .await } - Ok(CommissionDispatchOutcome::TargetBusy { session }) => { + Ok(CommissionDispatchOutcome::TargetBusy { session }) + | Ok(CommissionDispatchOutcome::TargetCoolingOff { session }) => { write_error( writer, version, @@ -9279,6 +9563,50 @@ where write_spooled_file(writer, &mut spool.file).await } +async fn handle_operator_status( + writer: &mut Writer, + version: ProtocolVersion, + request_id: RequestId, + pool: &PgPool, + snapshot_permit: OwnedSemaphorePermit, +) -> Result<(), ProcessConnectionError> +where + Writer: AsyncWrite + Unpin, +{ + let spool_result = spool_operator_status( + ProcessOperatorStatusRepository::new(pool.clone()), + version, + request_id, + ) + .await; + drop(snapshot_permit); + let mut spool = match spool_result { + Ok(spool) => spool, + Err(OperatorStatusSpoolError::Read(ProcessOperatorStatusError::Database(_))) => { + return write_error( + writer, + version, + request_id, + ProtocolError::without_detail(ErrorCode::Unavailable), + ) + .await; + } + Err(OperatorStatusSpoolError::Read(ProcessOperatorStatusError::Corruption(_))) => { + return write_error( + writer, + version, + request_id, + internal_protocol_error(None, InternalDiagnostic::OperatorStatusCorruption), + ) + .await; + } + Err(OperatorStatusSpoolError::Spool(error)) => { + return write_snapshot_spool_error(writer, version, request_id, error).await; + } + }; + write_spooled_file(writer, &mut spool.file).await +} + async fn handle_list_model_aliases( writer: &mut Writer, version: ProtocolVersion, @@ -9393,6 +9721,11 @@ enum SessionListSpoolError { Spool(SnapshotSpoolError), } +enum OperatorStatusSpoolError { + Read(ProcessOperatorStatusError), + Spool(SnapshotSpoolError), +} + #[derive(Debug)] enum SnapshotSpoolError { Io(io::Error), @@ -9512,6 +9845,284 @@ async fn spool_session_summaries( Ok(SessionListSpool { file }) } +async fn spool_operator_status( + repository: ProcessOperatorStatusRepository, + version: ProtocolVersion, + request_id: RequestId, +) -> Result { + let mut reader = repository + .open() + .await + .map_err(OperatorStatusSpoolError::Read)?; + let standard_file = tempfile::tempfile() + .map_err(SnapshotSpoolError::Io) + .map_err(OperatorStatusSpoolError::Spool)?; + let mut file = tokio::fs::File::from_std(standard_file); + write_spool_message( + &mut file, + version, + request_id, + ServerMessage::OperatorStatus(Box::new(OperatorStatusMessage::Start {})), + ) + .await + .map_err(OperatorStatusSpoolError::Spool)?; + while let Some(item) = reader + .next_item() + .await + .map_err(OperatorStatusSpoolError::Read)? + { + write_spool_message( + &mut file, + version, + request_id, + wire_operator_status_item(item), + ) + .await + .map_err(OperatorStatusSpoolError::Spool)?; + } + let counts = reader + .counts() + .ok_or(SnapshotSpoolError::EncodeInvariant) + .map_err(OperatorStatusSpoolError::Spool)?; + write_spool_message( + &mut file, + version, + request_id, + ServerMessage::OperatorStatus(Box::new(OperatorStatusMessage::End(Box::new( + OperatorStatusEndMessage { + held_slot_count: CanonicalU64::new(counts.held_slots()), + queued_obligation_count: CanonicalU64::new(counts.queued_obligations()), + pull_request_convergence_count: CanonicalU64::new( + counts.pull_request_convergences(), + ), + pending_stale_review_clearance_count: CanonicalU64::new( + counts.pending_stale_review_clearances(), + ), + }, + )))), + ) + .await + .map_err(OperatorStatusSpoolError::Spool)?; + file.flush() + .await + .map_err(SnapshotSpoolError::Io) + .map_err(OperatorStatusSpoolError::Spool)?; + file.seek(SeekFrom::Start(0)) + .await + .map_err(SnapshotSpoolError::Io) + .map_err(OperatorStatusSpoolError::Spool)?; + Ok(SessionListSpool { file }) +} + +fn wire_operator_status_item(item: ProcessOperatorStatusItem) -> ServerMessage { + match item { + ProcessOperatorStatusItem::HeldSlot(item) => { + let singleton = item.singleton(); + ServerMessage::OperatorStatus(Box::new(OperatorStatusMessage::HeldSlot(Box::new( + OperatorStatusHeldSlotMessage { + dispatch_id: wire_uuid(item.dispatch_id()), + repository: item.repository().to_owned(), + origin: wire_operator_status_held_slot_origin(item.origin()), + rule_id: item.rule_id().to_owned(), + rule_version: CanonicalU64::new(item.rule_version()), + singleton_scope: wire_operator_status_singleton_scope(singleton.scope()), + singleton_repository: singleton.repository().map(str::to_owned), + singleton_pull_request_number: singleton + .pull_request_number() + .map(CanonicalU64::new), + singleton_stack_root_pull_request_number: singleton + .stack_root_pull_request_number() + .map(CanonicalU64::new), + held_for_seconds: CanonicalU64::new(item.held_for_seconds()), + session_ids: item.session_ids().iter().copied().map(wire_uuid).collect(), + blockers: item + .blockers() + .iter() + .copied() + .map(wire_operator_status_blocker) + .collect(), + }, + )))) + } + ProcessOperatorStatusItem::QueuedObligation(item) => { + let singleton = item.singleton(); + ServerMessage::OperatorStatus(Box::new(OperatorStatusMessage::QueuedObligation( + Box::new(OperatorStatusQueuedObligationMessage { + obligation_id: wire_uuid(item.obligation_id()), + repository: item.repository().to_owned(), + rule_id: item.rule_id().to_owned(), + rule_version: CanonicalU64::new(item.rule_version()), + singleton_scope: wire_operator_status_singleton_scope(singleton.scope()), + singleton_repository: singleton.repository().map(str::to_owned), + singleton_pull_request_number: singleton + .pull_request_number() + .map(CanonicalU64::new), + singleton_stack_root_pull_request_number: singleton + .stack_root_pull_request_number() + .map(CanonicalU64::new), + first_event_id: wire_uuid(item.first_event_id()), + latest_event_id: wire_uuid(item.latest_event_id()), + matched_event_count: CanonicalU64::new(item.matched_event_count()), + waiting_for_seconds: CanonicalU64::new(item.waiting_for_seconds()), + occupying_dispatch_id: item.occupying_dispatch_id().map(wire_uuid), + occupying_session_ids: item + .occupying_session_ids() + .iter() + .copied() + .map(wire_uuid) + .collect(), + cooldown_remaining_seconds: item + .cooldown_remaining_seconds() + .map(CanonicalU64::new), + cooldown_never_eligible: item.cooldown_never_eligible(), + ready: item.ready(), + }), + ))) + } + ProcessOperatorStatusItem::PullRequestConvergence(item) => { + ServerMessage::OperatorStatus(Box::new(OperatorStatusMessage::PullRequestConvergence( + Box::new(OperatorStatusPullRequestConvergenceMessage { + repository: item.repository().to_owned(), + pull_request_number: CanonicalU64::new(item.pull_request_number()), + head_sha: item.head_sha().to_owned(), + base_branch: item.base_branch().to_owned(), + base_revision: item.base_revision().to_owned(), + mergeable_state: wire_operator_status_mergeable_state(item.mergeable_state()), + review_decision: wire_operator_status_review_decision(item.review_decision()), + unresolved_thread_count: CanonicalU64::new(item.unresolved_thread_count()), + gating_check_count: CanonicalU64::new(item.gating_check_count()), + non_green_gating_checks: item.non_green_gating_checks().to_vec(), + verdict: wire_operator_status_verdict(item.verdict()), + seal: item.seal().map(wire_operator_status_seal), + assessed_seconds_ago: CanonicalU64::new(item.assessed_seconds_ago()), + }), + ))) + } + ProcessOperatorStatusItem::PendingStaleReviewClearance(item) => { + ServerMessage::OperatorStatus(Box::new( + OperatorStatusMessage::PendingStaleReviewClearance(Box::new( + OperatorStatusPendingStaleReviewClearanceMessage { + repository: item.repository().to_owned(), + pull_request_number: CanonicalU64::new(item.pull_request_number()), + current_head_sha: item.current_head_sha().to_owned(), + review_node_id: item.review_node_id().to_owned(), + reviewer: item.reviewer().to_owned(), + reviewed_head_sha: item.reviewed_head_sha().to_owned(), + pending_for_seconds: CanonicalU64::new(item.pending_for_seconds()), + }, + )), + )) + } + } +} + +fn wire_operator_status_held_slot_origin( + origin: &ProcessOperatorStatusHeldSlotOrigin, +) -> OperatorStatusHeldSlotOrigin { + match origin { + ProcessOperatorStatusHeldSlotOrigin::PullRequest { number } => { + OperatorStatusHeldSlotOrigin::PullRequest { + pull_request_number: CanonicalU64::new(*number), + } + } + ProcessOperatorStatusHeldSlotOrigin::Branch { branch } => { + OperatorStatusHeldSlotOrigin::Branch { + branch: branch.clone(), + } + } + } +} + +fn wire_operator_status_singleton_scope( + scope: ProcessOperatorStatusSingletonScope, +) -> WireOperatorStatusSingletonScope { + match scope { + ProcessOperatorStatusSingletonScope::PullRequest => { + WireOperatorStatusSingletonScope::PullRequest + } + ProcessOperatorStatusSingletonScope::Stack => WireOperatorStatusSingletonScope::Stack, + ProcessOperatorStatusSingletonScope::Rule => WireOperatorStatusSingletonScope::Rule, + ProcessOperatorStatusSingletonScope::Repo => WireOperatorStatusSingletonScope::Repo, + } +} + +fn wire_operator_status_blocker( + blocker: ProcessOperatorStatusHeldSlotBlocker, +) -> WireOperatorStatusHeldSlotBlocker { + match blocker { + ProcessOperatorStatusHeldSlotBlocker::UndeliveredAction => { + WireOperatorStatusHeldSlotBlocker::UndeliveredAction + } + ProcessOperatorStatusHeldSlotBlocker::DeliveryTurnRuntimeRelevant => { + WireOperatorStatusHeldSlotBlocker::DeliveryTurnRuntimeRelevant + } + ProcessOperatorStatusHeldSlotBlocker::LiveRuntimeTurn => { + WireOperatorStatusHeldSlotBlocker::LiveRuntimeTurn + } + ProcessOperatorStatusHeldSlotBlocker::PursuingGoal => { + WireOperatorStatusHeldSlotBlocker::PursuingGoal + } + } +} + +fn wire_operator_status_mergeable_state( + state: ProcessOperatorStatusMergeableState, +) -> WireOperatorStatusMergeableState { + match state { + ProcessOperatorStatusMergeableState::Mergeable => { + WireOperatorStatusMergeableState::Mergeable + } + ProcessOperatorStatusMergeableState::Conflicting => { + WireOperatorStatusMergeableState::Conflicting + } + ProcessOperatorStatusMergeableState::Unknown => WireOperatorStatusMergeableState::Unknown, + } +} + +fn wire_operator_status_review_decision( + decision: ProcessOperatorStatusReviewDecision, +) -> WireOperatorStatusReviewDecision { + match decision { + ProcessOperatorStatusReviewDecision::None => WireOperatorStatusReviewDecision::None, + ProcessOperatorStatusReviewDecision::Approved => WireOperatorStatusReviewDecision::Approved, + ProcessOperatorStatusReviewDecision::ReviewRequired => { + WireOperatorStatusReviewDecision::ReviewRequired + } + ProcessOperatorStatusReviewDecision::ChangesRequested => { + WireOperatorStatusReviewDecision::ChangesRequested + } + } +} + +fn wire_operator_status_verdict( + verdict: ProcessOperatorStatusConvergenceVerdict, +) -> WireOperatorStatusConvergenceVerdict { + match verdict { + ProcessOperatorStatusConvergenceVerdict::NotConverged => { + WireOperatorStatusConvergenceVerdict::NotConverged + } + ProcessOperatorStatusConvergenceVerdict::InternallyConverged => { + WireOperatorStatusConvergenceVerdict::InternallyConverged + } + ProcessOperatorStatusConvergenceVerdict::MergeReady => { + WireOperatorStatusConvergenceVerdict::MergeReady + } + } +} + +fn wire_operator_status_seal( + seal: ProcessOperatorStatusConvergenceSeal, +) -> WireOperatorStatusConvergenceSeal { + match seal { + ProcessOperatorStatusConvergenceSeal::InternallyConverged => { + WireOperatorStatusConvergenceSeal::InternallyConverged + } + ProcessOperatorStatusConvergenceSeal::MergeReady => { + WireOperatorStatusConvergenceSeal::MergeReady + } + } +} + struct WireMetadataPageRequest { required_tags: Vec, title_contains: Option, @@ -10801,7 +11412,7 @@ async fn handle_submit_input( request_id: RequestId, command_id: uuid::Uuid, session_id: CanonicalUuid, - content: InputContent, + content: UserInputContent, expected_defaults_version: Option, model_settings: WireModelSettingsOverlay, delivery: Option, @@ -10809,6 +11420,7 @@ async fn handle_submit_input( eligibility_nudge: &InProcessEligibilityNudge, tool_dispatch_gate: &InProcessToolDispatchGate, model_configuration: &HubModelConfiguration, + blob_store_registry: Option<&BlobStoreRegistry>, ) -> Result<(), ProcessConnectionError> where Writer: AsyncWrite + Unpin, @@ -10828,6 +11440,10 @@ where pool.clone(), model_configuration.model_capability_catalog(), ); + let repository = match blob_store_registry { + Some(registry) => repository.with_attachment_maximum_bytes(registry.max_blob_bytes()), + None => repository, + }; let expected_version = expected_defaults_version .and_then(|version| SessionConfigurationDefaultsVersion::try_from_u64(version.value())); let model_settings = domain_model_settings_overlay(model_settings); @@ -10918,13 +11534,14 @@ async fn handle_reconcile_turn( command_id: uuid::Uuid, session_id: CanonicalUuid, expected_active_turn_id: CanonicalUuid, - content: InputContent, + content: UserInputContent, expected_defaults_version: CanonicalU64, model_settings: WireModelSettingsOverlay, pool: &PgPool, eligibility_nudge: &InProcessEligibilityNudge, tool_dispatch_gate: &InProcessToolDispatchGate, model_configuration: &HubModelConfiguration, + blob_store_registry: Option<&BlobStoreRegistry>, ) -> Result<(), ProcessConnectionError> where Writer: AsyncWrite + Unpin, @@ -10936,6 +11553,10 @@ where pool.clone(), model_configuration.model_capability_catalog(), ); + let repository = match blob_store_registry { + Some(registry) => repository.with_attachment_maximum_bytes(registry.max_blob_bytes()), + None => repository, + }; // A command identity that already names durable intent must reach the // replay boundary unconditionally (INV-012): the first handling already // released the wait, so re-applying the current-state precondition would @@ -11096,7 +11717,7 @@ async fn handle_stop_turn( command_id: uuid::Uuid, session_id: CanonicalUuid, expected_active_turn_id: CanonicalUuid, - content: InputContent, + content: UserInputContent, expected_defaults_version: CanonicalU64, descendant_scope: DescendantTerminationScope, model_settings: WireModelSettingsOverlay, @@ -11104,6 +11725,7 @@ async fn handle_stop_turn( eligibility_nudge: &InProcessEligibilityNudge, tool_dispatch_gate: &InProcessToolDispatchGate, model_configuration: &HubModelConfiguration, + blob_store_registry: Option<&BlobStoreRegistry>, ) -> Result<(), ProcessConnectionError> where Writer: AsyncWrite + Unpin, @@ -11115,6 +11737,10 @@ where pool.clone(), model_configuration.model_capability_catalog(), ); + let repository = match blob_store_registry { + Some(registry) => repository.with_attachment_maximum_bytes(registry.max_blob_bytes()), + None => repository, + }; let Some(expected_version) = SessionConfigurationDefaultsVersion::try_from_u64(expected_defaults_version.value()) else { @@ -11632,6 +12258,90 @@ fn wire_tool_decision( } } +/// Records one user override of a delegate denial through the canonical +/// override command. +/// +/// A claimed command identity reaches the durable replay boundary +/// unconditionally (INV-012). The session is part of the canonical override +/// payload, so an other-session request is the transaction's recorded +/// `request_not_in_session` rejection rather than a pre-command refusal, and +/// every outcome is the recorded result of the canonical command. +async fn handle_override_denied_tool_request( + writer: &mut Writer, + version: ProtocolVersion, + request_id: RequestId, + command_id: uuid::Uuid, + session_id: CanonicalUuid, + tool_request_id: CanonicalUuid, + pool: &PgPool, +) -> Result<(), ProcessConnectionError> +where + Writer: AsyncWrite + Unpin, +{ + let session = SessionId::from_uuid(session_id.into_uuid()); + let request = ToolRequestId::from_uuid(tool_request_id.into_uuid()); + let command_id = DurableCommandId::from_uuid(command_id); + let Ok(command) = OverrideDeniedToolRequest::try_new(command_id, session, request) else { + return write_error( + writer, + version, + request_id, + ProtocolError::without_detail(ErrorCode::InvalidRequest), + ) + .await; + }; + let repository = PostgresToolLoopRepository::new(pool.clone()); + let mut service = OverrideDeniedToolRequestService::new(repository); + match service.execute(command).await { + Ok(prepared) => match prepared.result() { + OverrideDeniedToolRequestResult::Applied(applied) => { + write_message( + writer, + version, + request_id, + ServerMessage::ToolDenialOverridden { + tool_request_id: wire_uuid(applied.recorded().denied_request().into_uuid()), + }, + ) + .await + } + OverrideDeniedToolRequestResult::Rejected(rejected) => { + let detail = match *rejected { + OverrideDeniedToolRequestRejectedResult::RequestNotFound { denied_request } => { + RejectionDetail::ToolRequestNotFound { + tool_request_id: wire_uuid(denied_request.into_uuid()), + } + } + OverrideDeniedToolRequestRejectedResult::RequestNotInSession { + session, + denied_request, + } => RejectionDetail::ToolRequestNotInSession { + session_id: wire_uuid(session.into_uuid()), + tool_request_id: wire_uuid(denied_request.into_uuid()), + }, + OverrideDeniedToolRequestRejectedResult::NotDelegateDenied { + denied_request, + } => RejectionDetail::ToolRequestNotDelegateDenied { + tool_request_id: wire_uuid(denied_request.into_uuid()), + }, + OverrideDeniedToolRequestRejectedResult::NotTerminallyDenied { + denied_request, + } => RejectionDetail::ToolRequestNotTerminallyDenied { + tool_request_id: wire_uuid(denied_request.into_uuid()), + }, + OverrideDeniedToolRequestRejectedResult::AlreadyOverridden { + denied_request, + } => RejectionDetail::ToolDenialAlreadyOverridden { + tool_request_id: wire_uuid(denied_request.into_uuid()), + }, + }; + write_error(writer, version, request_id, ProtocolError::rejected(detail)).await + } + }, + Err(error) => write_tool_loop_error(writer, version, request_id, session_id, error).await, + } +} + async fn write_tool_loop_error( writer: &mut Writer, version: ProtocolVersion, @@ -11669,12 +12379,82 @@ where write_error(writer, version, request_id, protocol_error).await } -fn admitted_user_content(content: InputContent) -> Result { - let content = content.into_string(); - if content.len() > MAX_SUBMITTED_INPUT_BYTES { - return Err(()); - } - UserContent::try_text(content).map_err(|_| ()) +fn admitted_user_content(content: UserInputContent) -> Result { + let parts = content + .into_parts() + .into_iter() + .map(|part| match part { + signalbox_process_protocol::UserInputPart::Text { text } => { + signalbox_domain::UserContentPart::try_text(text).map_err(|_| ()) + } + signalbox_process_protocol::UserInputPart::Attachment { + digest, + kind, + media_type, + display_filename, + } => Ok(signalbox_domain::UserContentPart::Attachment { + digest: digest.into_digest(), + kind: match kind { + signalbox_process_protocol::UserAttachmentKind::Image => { + signalbox_domain::AttachmentKind::Image + } + signalbox_process_protocol::UserAttachmentKind::Document => { + signalbox_domain::AttachmentKind::Document + } + signalbox_process_protocol::UserAttachmentKind::File => { + signalbox_domain::AttachmentKind::File + } + }, + media_type: signalbox_domain::DeclaredMediaType::try_new(media_type) + .map_err(|_| ())?, + display_filename: display_filename + .map(signalbox_domain::AttachmentDisplayFilename::try_new) + .transpose() + .map_err(|_| ())?, + }), + }) + .collect::, _>>()?; + UserContent::try_parts(parts).map_err(|_| ()) +} + +pub(crate) fn wire_user_content(content: &UserContent) -> UserInputContent { + UserInputContent::from_parts( + content + .parts() + .iter() + .map(|part| match part { + signalbox_domain::UserContentPart::Text { value } => { + signalbox_process_protocol::UserInputPart::Text { + text: value.as_str().to_owned(), + } + } + signalbox_domain::UserContentPart::Attachment { + digest, + kind, + media_type, + display_filename, + } => signalbox_process_protocol::UserInputPart::Attachment { + digest: signalbox_process_protocol::CanonicalBlobDigest::from_digest(*digest), + kind: match kind { + signalbox_domain::AttachmentKind::Image => { + signalbox_process_protocol::UserAttachmentKind::Image + } + signalbox_domain::AttachmentKind::Document => { + signalbox_process_protocol::UserAttachmentKind::Document + } + signalbox_domain::AttachmentKind::File => { + signalbox_process_protocol::UserAttachmentKind::File + } + }, + media_type: media_type.as_str().to_owned(), + display_filename: display_filename + .as_ref() + .map(signalbox_domain::AttachmentDisplayFilename::as_str) + .map(str::to_owned), + }, + }) + .collect(), + ) } async fn handle_read_transcript( @@ -12390,18 +13170,16 @@ where writer, version, request_id, - ServerMessage::TranscriptTextEntry { + ServerMessage::TranscriptUserEntry { entry_index: CanonicalU64::new(*entry_index), source_session_id: wire_uuid(source_session.into_uuid()), entry_id: wire_uuid(entry.into_uuid()), - entry: TranscriptTextEntry::User { - accepted_input_id: wire_uuid(accepted_input.into_uuid()), - turn_id: wire_uuid(turn.into_uuid()), - }, + accepted_input_id: wire_uuid(accepted_input.into_uuid()), + turn_id: wire_uuid(turn.into_uuid()), + content: wire_user_content(content), }, ) - .await?; - write_content(writer, version, request_id, *entry_index, content).await + .await } ProcessTranscriptEntry::Assistant { entry_index, @@ -12478,6 +13256,15 @@ where model_call_id: wire_uuid(call.into_uuid()), } } + signalbox_domain::ToolApprovalDecider::UserOverride { + command, + denied_request, + } => WireToolApprovalEventDecider::UserOverride { + command_id: wire_uuid(command.into_uuid()), + overridden_tool_request_id: wire_uuid( + denied_request.into_uuid(), + ), + }, }, rationale: approval .rationale() @@ -12719,6 +13506,17 @@ fn map_rejection( rejected: SubmitInputRejectedResult, ) -> Result { Ok(match rejected { + SubmitInputRejectedResult::AttachmentBlobNotFound { digest } => { + RejectionDetail::AttachmentBlobNotFound { + digest: signalbox_process_protocol::CanonicalBlobDigest::from_digest(digest), + } + } + SubmitInputRejectedResult::AttachmentByteBudgetExceeded { maximum_bytes } => { + RejectionDetail::AttachmentByteBudgetExceeded { + maximum_bytes: PositiveCanonicalU64::try_new(maximum_bytes) + .map_err(|_| ProcessConnectionError::EncodeInvariant)?, + } + } SubmitInputRejectedResult::SessionNotFound { session } => { RejectionDetail::SessionNotFound { session_id: wire_uuid(session.into_uuid()), @@ -13376,7 +14174,7 @@ fn wire_turn_state(state: &ProcessTurnState) -> TurnState { content, } => TurnState::Queued { accepted_input_id: wire_uuid(accepted_input.into_uuid()), - content: InputContent::new(content.clone()), + content: wire_user_content(content), }, ProcessTurnState::QueuedDelegated { spawning_request, @@ -13489,12 +14287,24 @@ fn wire_turn_state(state: &ProcessTurnState) -> TurnState { let model_call_id = wire_uuid(call.call().into_uuid()); match call.disposition() { ProcessFailedModelCallDisposition::KnownFailed => { - match call.provider_failure_cause() { - Some(cause) => FailedTerminalModelCall::known_failed_with_cause( + let provider_cause = call.provider_failure_cause(); + let attachment_cause = call.attachment_preparation_failure_cause(); + debug_assert!( + provider_cause.is_none() || attachment_cause.is_none(), + "process-read validation rejects overlapping failure causes" + ); + match (provider_cause, attachment_cause) { + (Some(cause), _) => FailedTerminalModelCall::known_failed_with_cause( model_call_id, wire_provider_failure_cause(cause), ), - None => FailedTerminalModelCall::new( + (None, Some(cause)) => { + FailedTerminalModelCall::known_failed_with_cause( + model_call_id, + wire_attachment_preparation_failure_cause(cause), + ) + } + (None, None) => FailedTerminalModelCall::new( model_call_id, FailedModelCallDisposition::KnownFailed, ), @@ -13505,6 +14315,7 @@ fn wire_turn_state(state: &ProcessTurnState) -> TurnState { call.provider_failure_cause().is_none(), "process-read validation rejects causes on cancelled model calls" ); + debug_assert!(call.attachment_preparation_failure_cause().is_none()); FailedTerminalModelCall::new( model_call_id, FailedModelCallDisposition::Cancelled, @@ -13644,6 +14455,7 @@ enum InternalDiagnostic { ToolLoopCorruption, ToolLoopInvalidTransition, ProcessReadCorruption, + OperatorStatusCorruption, GoalRepositoryCorruption, } @@ -13713,6 +14525,7 @@ impl InternalDiagnostic { | Self::SubmitInputModelExecutionCorruption | Self::ToolLoopCorruption | Self::ProcessReadCorruption + | Self::OperatorStatusCorruption | Self::GoalRepositoryCorruption => OperatorFailureClass::FailClosedCorruption, } } @@ -13790,6 +14603,7 @@ impl InternalDiagnostic { Self::ToolLoopCorruption => "tool_loop_corruption", Self::ToolLoopInvalidTransition => "tool_loop_invalid_transition", Self::ProcessReadCorruption => "process_read_corruption", + Self::OperatorStatusCorruption => "operator_status_corruption", Self::GoalRepositoryCorruption => "goal_repository_corruption", } } @@ -14098,6 +14912,24 @@ fn wire_provider_failure_cause( } } +fn wire_attachment_preparation_failure_cause( + cause: signalbox_persistence::process_read::ProcessAttachmentPreparationFailureCause, +) -> FailedModelCallCause { + use signalbox_persistence::process_read::ProcessAttachmentPreparationFailureCause; + + match cause { + ProcessAttachmentPreparationFailureCause::TooLarge => { + FailedModelCallCause::AttachmentTooLarge + } + ProcessAttachmentPreparationFailureCause::Missing => { + FailedModelCallCause::AttachmentMissing + } + ProcessAttachmentPreparationFailureCause::Corrupt => { + FailedModelCallCause::AttachmentCorrupt + } + } +} + #[allow(clippy::too_many_arguments)] async fn handle_goal_user_command( writer: &mut Writer, @@ -14168,6 +15000,19 @@ where ) .await } + Ok(GoalCommandHandlingOutcome::TargetBusy { + session: blocking_session, + }) => { + write_error( + writer, + version, + request_id, + ProtocolError::rejected(RejectionDetail::CommissionTargetBusy { + session_id: CanonicalUuid::from_uuid(blocking_session.into_uuid()), + }), + ) + .await + } // A client goal command names no expected lineage head, so it applies // to whatever state the session lock reveals. Reaching this answer // means the repository decided a question this request never asked. @@ -14600,7 +15445,7 @@ enum ProcessUpdateEvent { accepted_input: signalbox_domain::AcceptedInputId, turn: signalbox_domain::TurnId, acceptance_position: u64, - content: String, + content: UserContent, }, GoalTurnRetired { turn: signalbox_domain::TurnId, @@ -14854,7 +15699,7 @@ impl ProcessUpdateEvent { accepted_input_id: wire_uuid(accepted_input.into_uuid()), turn_id: wire_uuid(turn.into_uuid()), acceptance_position: CanonicalU64::new(*acceptance_position), - content: InputContent::new(content.clone()), + content: wire_user_content(content), }, Self::GoalTurnRetired { turn } => SessionEvent::GoalTurnRetired { turn_id: wire_uuid(turn.into_uuid()), @@ -14917,6 +15762,13 @@ impl ProcessUpdateEvent { model_call_id: wire_uuid(call.into_uuid()), } } + signalbox_domain::ToolApprovalDecider::UserOverride { + command, + denied_request, + } => WireToolApprovalEventDecider::UserOverride { + command_id: wire_uuid(command.into_uuid()), + overridden_tool_request_id: wire_uuid(denied_request.into_uuid()), + }, }; SessionEvent::ToolApprovalDecided { turn_id: wire_uuid(turn.into_uuid()), @@ -15404,14 +16256,14 @@ mod tests { CanonicalU64, CanonicalUuid, ClientRequest, CommandId, ConversationImportRejectionClass, DelegationToolRequestState as WireDelegationToolRequestState, ErrorCode, ErrorDetail, FrameEncodeError, GoalLifecycleState, ImportedContentKind, ImportedSourceSpeaker, - ImportedSpeaker, InputContent, MAX_CONTENT_FRAGMENT_BYTES, MetadataActor, ProtocolVersion, + ImportedSpeaker, MAX_CONTENT_FRAGMENT_BYTES, MetadataActor, ProtocolVersion, RejectionDetail, ReviewFindingInput, ReviewSeverity, RunnerPlacementRevision as WireRunnerPlacementRevision, RunnerSandboxProfile as WireRunnerSandboxProfile, RunnerStateTransitionState as WireRunnerStateTransitionState, RunnerWorkingDirectory as WireRunnerWorkingDirectory, ServerFrame, ServerMessage, SessionEvent, ToolBatchState, ToolDecision, TranscriptEntry, TranscriptTextEntry, - TurnState, decode_server_line, encode_server_line, + TurnState, UserInputContent, decode_server_line, encode_server_line, }; use sqlx::postgres::PgPoolOptions; use tokio::{ @@ -15428,20 +16280,20 @@ mod tests { ImportedConversationRepositoryError, InboundFrameBudgets, IncomingLine, InternalDiagnostic, MAX_ACTIVE_CONNECTIONS, MAX_BUFFERED_INBOUND_FRAMES, MAX_CONCURRENT_BLOB_READS, MAX_CONCURRENT_IMPORTS, MAX_CONCURRENT_REVIEW_COMMANDS, MAX_FRAME_BYTES, - MAX_IMPORT_ADMISSION_WAITERS, MAX_SUBMITTED_INPUT_BYTES, OperationalImportError, - PendingConversationImport, ProcessConnectionError, ProcessRuntimeError, ProcessUpdateEvent, - ProtocolError, RESERVED_ACTIVE_IMPORT_INBOUND_FRAMES, - RESERVED_POOL_CONNECTIONS_OUTSIDE_SNAPSHOTS, RequestId, ReviewCommandAdmission, - SnapshotReaderAdmission, SnapshotSpoolError, SubmitInputModelExecutionDiagnostic, - acquire_import_permit, acquire_import_waiter_permit, acquire_inbound_frame_permit, - acquire_inbound_frame_permit_after_input, acquire_review_command_permit, - acquire_review_command_permit_while_buffered, acquire_snapshot_reader_permit, - admit_snapshot_reader, admitted_user_content, blob_read_budget, - blob_upload_begin_preflight, canonical_review_request_digest, - claude_conversion_failure_disposition, codex_conversion_failure_disposition, - consume_snapshot_queued_update, context_compaction_failure_disposition, execute_import, - foreground_peer_activity, handle_append_conversation_import, - handle_begin_conversation_import, handle_commit_conversation_import, import_evidence, + MAX_IMPORT_ADMISSION_WAITERS, OperationalImportError, PendingConversationImport, + ProcessConnectionError, ProcessRuntimeError, ProcessUpdateEvent, ProtocolError, + RESERVED_ACTIVE_IMPORT_INBOUND_FRAMES, RESERVED_POOL_CONNECTIONS_OUTSIDE_SNAPSHOTS, + RequestId, ReviewCommandAdmission, SnapshotReaderAdmission, SnapshotSpoolError, + SubmitInputModelExecutionDiagnostic, acquire_import_permit, acquire_import_waiter_permit, + acquire_inbound_frame_permit, acquire_inbound_frame_permit_after_input, + acquire_review_command_permit, acquire_review_command_permit_while_buffered, + acquire_snapshot_reader_permit, admit_snapshot_reader, admitted_user_content, + blob_read_budget, blob_upload_begin_preflight, bounded_rendered_compaction_boundary, + canonical_review_request_digest, claude_conversion_failure_disposition, + codex_conversion_failure_disposition, consume_snapshot_queued_update, + context_compaction_failure_disposition, execute_import, foreground_peer_activity, + handle_append_conversation_import, handle_begin_conversation_import, + handle_commit_conversation_import, import_evidence, imported_conversation_internal_diagnostic, inspect_connection_completion, internal_protocol_error, map_rejection, nudge_after_process_await_rejection, nudge_after_process_message_rejection, nudge_delegation_issuer, nudge_delegation_wake, @@ -15562,7 +16414,10 @@ mod tests { accepted_input, turn, acceptance_position: SessionInputPosition::first(), - content: "synthetic prompt with tool arguments".to_owned(), + content: signalbox_domain::UserContent::try_text( + "synthetic prompt with tool arguments".to_owned(), + ) + .expect("the telemetry fixture content is valid"), }; let activation = DispatchedOutboxEventKind::TurnActivated { turn, @@ -15831,6 +16686,68 @@ mod tests { ); } + #[test] + fn automatic_compaction_boundary_counts_the_rendered_json_envelope() { + let first = serde_json::json!({ + "position": 1, + "type": "user", + "content": "x".repeat(90), + }); + let second = serde_json::json!({ + "position": 2, + "type": "assistant", + "content": "y".repeat(90), + }); + let first_bytes = u64::try_from( + serde_json::to_vec(&first) + .expect("the fixture JSON is serializable") + .len(), + ) + .expect("the fixture length fits u64"); + let second_bytes = u64::try_from( + serde_json::to_vec(&second) + .expect("the fixture JSON is serializable") + .len(), + ) + .expect("the fixture length fits u64"); + let first_array_bytes = first_bytes + 2; + + assert_eq!( + bounded_rendered_compaction_boundary( + &[first_bytes, second_bytes], + &[(11, true), (12, true)], + first_array_bytes, + ), + Some(11) + ); + } + + #[test] + fn automatic_compaction_boundary_never_crosses_the_model_budget_for_a_tool_exchange() { + assert_eq!( + bounded_rendered_compaction_boundary( + &[60, 100, 100], + &[(21, true), (22, false), (23, true)], + 170, + ), + Some(21) + ); + } + + #[test] + fn successor_compaction_rejects_an_unreachable_later_safe_boundary() { + assert!(super::successor_compaction_cannot_advance( + &[10, 100, 100], + &[(31, true), (32, false), (33, true)], + 203, + )); + assert!(!super::successor_compaction_cannot_advance( + &[10, 100, 100], + &[(31, true), (32, false), (33, true)], + 204, + )); + } + #[test] fn snapshot_delta_boundary_consumes_only_the_queued_prefix() { let mut queued = 2; @@ -17804,15 +18721,18 @@ mod tests { #[test] fn process_submission_admits_the_exact_content_bound() { - let exact = InputContent::new("\u{1}".repeat(MAX_SUBMITTED_INPUT_BYTES)); + let exact = + UserInputContent::text("\u{1}".repeat(signalbox_domain::UserContent::MAX_TEXT_BYTES)); assert!(admitted_user_content(exact).is_ok()); } #[test] fn process_submission_rejects_content_over_the_bound() { assert!( - admitted_user_content(InputContent::new("x".repeat(MAX_SUBMITTED_INPUT_BYTES + 1))) - .is_err() + admitted_user_content(UserInputContent::text( + "x".repeat(signalbox_domain::UserContent::MAX_TEXT_BYTES + 1), + )) + .is_err() ); } @@ -17827,7 +18747,9 @@ mod tests { model_settings: None, state: TurnState::Queued { accepted_input_id: CanonicalUuid::from_uuid(Uuid::from_u128(u128::MAX - 1)), - content: InputContent::new("\u{1}".repeat(MAX_SUBMITTED_INPUT_BYTES)), + content: UserInputContent::text( + "\u{1}".repeat(signalbox_domain::UserContent::MAX_TEXT_BYTES), + ), }, }, )?; @@ -17847,7 +18769,9 @@ mod tests { accepted_input_id: CanonicalUuid::from_uuid(Uuid::from_u128(u128::MAX - 1)), turn_id: CanonicalUuid::from_uuid(Uuid::from_u128(u128::MAX - 2)), acceptance_position: CanonicalU64::new(u64::MAX), - content: InputContent::new("\u{1}".repeat(MAX_SUBMITTED_INPUT_BYTES)), + content: UserInputContent::text( + "\u{1}".repeat(signalbox_domain::UserContent::MAX_TEXT_BYTES), + ), }, }, )?; diff --git a/apps/signalboxd/src/repo_watch_runtime.rs b/apps/signalboxd/src/repo_watch_runtime.rs index 29b4dc5811..c5f4f81803 100644 --- a/apps/signalboxd/src/repo_watch_runtime.rs +++ b/apps/signalboxd/src/repo_watch_runtime.rs @@ -21,14 +21,16 @@ use reqwest::{ use serde::{Deserialize, Serialize, de::DeserializeOwned}; use sha2::{Digest, Sha256}; use signalbox_application::{ - EligibilityNudge, InProcessEligibilityNudge, RepoWatchBranchHead, + EligibilityNudge, EligibilityNudgeOutcome, InProcessEligibilityNudge, RepoWatchBranchHead, RepoWatchCheckCompletionGeneration, RepoWatchCheckRunObservation, - RepoWatchCheckSuiteObservation, RepoWatchDifferFailureKind, RepoWatchDispatchService, + RepoWatchCheckSuiteObservation, RepoWatchConvergenceAssessment, + RepoWatchConvergenceAssessmentInput, RepoWatchDifferFailureKind, RepoWatchDispatchService, RepoWatchDispatchTransaction, RepoWatchEventIdentityFrontierV1, RepoWatchEventOccurrenceV1, RepoWatchObservation, RepoWatchObservationApplyV1, RepoWatchPullRequestLifecycle, RepoWatchPullRequestState, RepoWatchPullRequestStateInput, RepoWatchReactionObservation, - RepoWatchRepositoryState, RepoWatchRepositoryStateInput, RepoWatchReviewObservation, - RepoWatchRuleEvaluation, RepoWatchRuleEvaluationOutcome, RepoWatchTargetedRefreshCoalescerV1, + RepoWatchRepositoryState, RepoWatchRepositoryStateInput, RepoWatchReviewDecision, + RepoWatchReviewObservation, RepoWatchRuleEvaluation, RepoWatchRuleEvaluationOutcome, + RepoWatchStaleReviewClearanceCandidate, RepoWatchTargetedRefreshCoalescerV1, RepoWatchTargetedRefreshV1, RepoWatchThreadObservation, RepoWatchThreadState, RepoWatchWebhookDeliveryV1, RepoWatchWebhookDeliveryV1Input, RepoWatchWebhookIgnoredReasonV1, RepoWatchWebhookMappedNoChangeV1, RepoWatchWebhookMappingError, RepoWatchWebhookMappingV1, @@ -48,7 +50,9 @@ use signalbox_domain::{ use signalbox_model_runtime::{CredentialAccess, CredentialReference}; use signalbox_persistence::repo_watch::{ PostgresRepoWatchStore, RepoWatchCommitOutcome, RepoWatchCommitRequest, RepoWatchCursor, - RepoWatchCursorCandidate, RepoWatchCursorGeneration, + RepoWatchCursorCandidate, RepoWatchCursorGeneration, RepoWatchObservedReviewState, + RepoWatchPlannedStaleReviewClearance, RepoWatchStaleReviewClearanceOutcome, + RepoWatchStaleReviewClearanceRenewal, RepoWatchStoreError, }; use signalbox_persistence::repo_watch_dispatch::{ PostgresRepoWatchDispatchStore, RepoWatchDispatchRepositoryError, @@ -66,7 +70,7 @@ use sqlx::PgPool; use tokio::{ select, sync::watch, - task::JoinSet, + task::{JoinHandle, JoinSet}, time::{Instant, sleep, sleep_until, timeout}, }; @@ -104,6 +108,12 @@ const MAX_POLL_WIRE_BYTES: usize = 768 * 1024 * 1024; // and therefore multiplies by the configured repository count. Deliberately not // raised with the per-attempt bound: transfer is transient, retention is not. const MAX_CACHED_WIRE_BYTES: usize = 64 * 1024 * 1024; +const NON_GATING_CHECK_NAME_MARKERS: [&str; 4] = [ + "report only", + "coderabbit", + "codecov/project", + "codecov/patch", +]; const WEBHOOK_PENDING_PAGE_SIZE: NonZeroU16 = NonZeroU16::new(25).expect("webhook pending page size is positive"); const WEBHOOK_DRAIN_RETRY_DELAY: Duration = Duration::from_secs(5); @@ -124,7 +134,7 @@ const WEBHOOK_DRAIN_MONITOR_INTERVAL: Duration = Duration::from_secs(30); // room for an in-flight bounded provider request while ensuring a task wedge // becomes an operator-visible error well before the next full poll. const WEBHOOK_DRAIN_STALL_THRESHOLD: Duration = Duration::from_secs(60); -// The serialized repository drain task must return to its scheduler even when one +// The serialized repository task must return to its scheduler even when one // drain step never does. Individual provider requests have their own deadline, // but a drain can perform many requests and database operations; without this // outer bound, admission wakes and retries remain coalesced behind it forever. @@ -132,10 +142,20 @@ const WEBHOOK_DRAIN_STALL_THRESHOLD: Duration = Duration::from_secs(60); // work for the existing bounded backoff path to retry. const WEBHOOK_DRAIN_ATTEMPT_TIMEOUT: Duration = Duration::from_secs(60); // Reconciliation before or after the drain is durable and replayable, but it -// shares the same serialized task. Give the drain deadline and its bounded -// child cleanup room to report before the enclosing attempt is cancelled. +// runs on the same serialized repository task. Give the drain deadline and its +// bounded child cleanup room to report before the enclosing attempt is +// cancelled. +// numeric-bound: ceiling - caps an attempt's hold on the repository task const WEBHOOK_ATTEMPT_TIMEOUT: Duration = Duration::from_secs(70); +// Every shared child-set join uses this bound. A later attempt may retry the +// join, but it never spawns alongside survivors or wedges the scheduler while +// waiting for a child that does not finish cancellation. const WEBHOOK_CANCELLED_FETCH_DRAIN_TIMEOUT: Duration = Duration::from_secs(5); +// Shutdown gives a retained targeted completion a short grace period, then +// aborts and joins it so a wedged database operation cannot prevent the +// repository supervisor from stopping. +// numeric-bound: guard - prevents a wedged targeted completion from stalling supervisor shutdown forever +const WEBHOOK_TARGETED_COMPLETION_SHUTDOWN_TIMEOUT: Duration = Duration::from_secs(5); // The monitor reads through the shared daemon pool, whose connections wedged // repositories can hold all of. An unbounded acquisition would leave the // observer silent during exactly the degradation it exists to expose, so the @@ -178,6 +198,96 @@ query RepositoryWatchReviewThreads( } "#; +const CONVERGENCE_QUERY: &str = r#" +query RepositoryWatchConvergence( + $namespace: String!, $name: String!, $number: Int!, $after: String +) { + repository(owner: $namespace, name: $name) { + pullRequest(number: $number) { + headRefOid + baseRefName + baseRefOid + mergeable + reviewDecision + commits(last: 1) { + nodes { + commit { + oid + statusCheckRollup { + contexts(first: 100, after: $after) { + nodes { + __typename + ... on CheckRun { name status conclusion } + ... on StatusContext { context state } + } + pageInfo { hasNextPage endCursor } + } + } + } + } + } + } + } +} +"#; + +const BLOCKING_REVIEWS_QUERY: &str = r#" +query RepositoryWatchBlockingReviews( + $namespace: String!, $name: String!, $number: Int!, $after: String +) { + repository(owner: $namespace, name: $name) { + pullRequest(number: $number) { + headRefOid + baseRefOid + reviewDecision + latestOpinionatedReviews(first: 100, after: $after) { + nodes { + id + state + author { login } + commit { oid } + } + pageInfo { hasNextPage endCursor } + } + } + } +} +"#; + +const DISMISS_REVIEW_MUTATION: &str = r#" +mutation RepositoryWatchDismissReview($review: ID!, $message: String!) { + dismissPullRequestReview( + input: {pullRequestReviewId: $review, message: $message} + ) { + pullRequestReview { id state } + } +} +"#; + +const REVIEW_CLEARANCE_STATE_QUERY: &str = r#" +query RepositoryWatchReviewClearanceState($review: ID!, $after: String) { + node(id: $review) { + ... on PullRequestReview { + id + state + commit { oid } + pullRequest { + number + state + headRefOid + baseRefName + baseRefOid + reviewDecision + latestOpinionatedReviews(first: 100, after: $after) { + nodes { id } + pageInfo { hasNextPage endCursor } + } + } + } + } +} +"#; + /// Why the repository-watch runtime could not be constructed. #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub struct RepositoryWatchRuntimeConstructionError; @@ -226,6 +336,26 @@ pub struct RepositoryWatchRuntime { webhook: Option, } +/// Deployment-owned work policies for the repository-watch scheduler. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct RepositoryWatchNumericBounds { + reconciliation_quantum: Option, + webhook_drain_work_budget: Option, +} + +impl RepositoryWatchNumericBounds { + /// Groups the repository-watch policies loaded from required configuration. + pub const fn new( + reconciliation_quantum: Option, + webhook_drain_work_budget: Option, + ) -> Self { + Self { + reconciliation_quantum, + webhook_drain_work_budget, + } + } +} + impl RepositoryWatchRuntime { /// Constructs all repository-specific clients without reading credentials. pub fn try_new( @@ -235,8 +365,12 @@ impl RepositoryWatchRuntime { models: HubModelConfiguration, credential_pin: signalbox_persistence::SessionCredentialPin, eligibility_nudge: InProcessEligibilityNudge, - reconciliation_quantum: Option, + numeric_bounds: RepositoryWatchNumericBounds, ) -> Result { + let RepositoryWatchNumericBounds { + reconciliation_quantum, + webhook_drain_work_budget, + } = numeric_bounds; let mut tasks = Vec::with_capacity(configuration.repositories().len()); let mut webhook_workers = HashMap::new(); let payload_purge = WebhookPayloadPurgeSchedule::starting_now(); @@ -267,6 +401,7 @@ impl RepositoryWatchRuntime { webhook_nudge, payload_purge: payload_purge.clone(), reconciliation_quantum, + webhook_drain_work_budget, }, )?); } @@ -275,6 +410,21 @@ impl RepositoryWatchRuntime { Ok(Self { tasks, webhook }) } + /// Completes each repository's bounded startup webhook attempt before the + /// daemon admits scheduler work. + pub async fn prepare_startup(&mut self) -> Result<(), RepositoryWatchRuntimeError> { + let outcomes = futures_util::future::join_all( + self.tasks + .iter_mut() + .map(RepositoryWatchTask::prepare_startup), + ) + .await; + if outcomes.into_iter().any(std::convert::identity) { + return Err(RepositoryWatchRuntimeError::RepositoryTaskExited); + } + Ok(()) + } + /// Runs every repository task until the daemon broadcasts shutdown. pub async fn run( self, @@ -284,31 +434,32 @@ impl RepositoryWatchRuntime { return Ok(()); } let mut tasks = JoinSet::new(); + let (task_shutdown_sender, task_shutdown) = watch::channel(*shutdown.borrow()); let mut pollers = Vec::with_capacity(self.tasks.len()); for task in self.tasks { if task.webhook_work.is_some() { let repository = task.repository.clone(); let store = task.webhook_store.clone(); - let monitor_shutdown = shutdown.clone(); + let monitor_shutdown = task_shutdown.clone(); tasks.spawn(async move { monitor_webhook_drain(repository, store, monitor_shutdown).await; RepositoryWatchChildExit::WebhookMonitor }); } pollers.push(Arc::clone(&task.poller)); - let task_shutdown = shutdown.clone(); + let repository_shutdown = task_shutdown.clone(); tasks.spawn(async move { - task.run(task_shutdown).await; + task.run(repository_shutdown).await; RepositoryWatchChildExit::Repository }); } if let Some(webhook) = self.webhook { - let webhook_shutdown = shutdown.clone(); + let webhook_shutdown = task_shutdown.clone(); tasks.spawn(async move { RepositoryWatchChildExit::Webhook(webhook.run(webhook_shutdown).await) }); } - supervise_repository_tasks(tasks, pollers, shutdown).await + supervise_repository_tasks(tasks, pollers, shutdown, task_shutdown_sender).await } } @@ -322,9 +473,11 @@ async fn supervise_repository_tasks( mut tasks: JoinSet, pollers: Vec>, mut shutdown: watch::Receiver, + task_shutdown: watch::Sender, ) -> Result<(), RepositoryWatchRuntimeError> { let result = async { if *shutdown.borrow() { + let _ = task_shutdown.send(true); while let Some(result) = tasks.join_next().await { result.map_err(|_| RepositoryWatchRuntimeError::RepositoryTaskPanicked)?; } @@ -335,6 +488,7 @@ async fn supervise_repository_tasks( biased; changed = shutdown.changed() => { if changed.is_err() || *shutdown.borrow() { + let _ = task_shutdown.send(true); while let Some(result) = tasks.join_next().await { result.map_err(|_| RepositoryWatchRuntimeError::RepositoryTaskPanicked)?; } @@ -344,6 +498,7 @@ async fn supervise_repository_tasks( completed = tasks.join_next() => { return match completed { Some(Ok(_)) if *shutdown.borrow() => { + let _ = task_shutdown.send(true); while let Some(result) = tasks.join_next().await { result.map_err(|_| RepositoryWatchRuntimeError::RepositoryTaskPanicked)?; } @@ -370,7 +525,10 @@ async fn supervise_repository_tasks( } .await; - tasks.abort_all(); + // Unexpected sibling exit uses the same cleanup path as operator shutdown, + // allowing repository tasks to settle retained targeted completions before + // the supervisor returns the lifecycle error. + let _ = task_shutdown.send(true); while tasks.join_next().await.is_some() {} for poller in &pollers { if !poller @@ -682,6 +840,12 @@ impl WebhookDrainOutcome { Self::ProjectionFailed(error) | Self::DispatchFailedAfterTerminal(error) => Some(error), } } + + /// Whether deadline cancellation can leave projection work pending and + /// therefore makes a cursor-advancing complete poll unsafe. + const fn blocks_complete_poll_after_timeout(self) -> bool { + !matches!(self, Self::DispatchFailedAfterTerminal(_)) + } } /// How one webhook-triggered attempt ended, with the drain's own outcome held @@ -718,6 +882,43 @@ impl WebhookAttemptOutcome { } } +/// Which step of a webhook attempt is running. +/// +/// The enclosing attempt deadline can expire in any of them, and cancellation +/// carries no failure of its own to classify. Recording the step lets a +/// cancelled attempt report the same outcome that step's own failure would +/// have, so a wedge outside the drain does not advance the projection backoff +/// that only drain failures are meant to grow. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum WebhookAttemptPhase { + /// Activation and the leading reconciliation that precede the drain. + BeforeDrain, + /// The drain itself, which owns the durable pending deliveries. + Drain, + /// The cutoff and dispatch reconciliation that follow a committed drain. + AfterDrain, +} + +impl WebhookAttemptPhase { + /// The outcome a cancellation during this step reports. + const fn cancelled_outcome(self, error: RepositoryWatchAttemptError) -> WebhookAttemptOutcome { + match self { + Self::BeforeDrain => WebhookAttemptOutcome::FailedBeforeDrain(error), + Self::Drain => WebhookAttemptOutcome::DrainFailed(error), + Self::AfterDrain => WebhookAttemptOutcome::DrainedThenFailed(error), + } + } + + /// The operator-facing label for the cancelled step. + const fn label(self) -> &'static str { + match self { + Self::BeforeDrain => "before_drain", + Self::Drain => "drain", + Self::AfterDrain => "after_drain", + } + } +} + /// Whether a full polling attempt performs its own webhook drain step. #[derive(Clone, Copy, Debug, Eq, PartialEq)] enum WebhookDrain { @@ -1003,9 +1204,20 @@ struct RepositoryWatchTask { webhook_nudge: Option>>, webhook_shadow: Option, webhook_shadow_superseded: bool, + webhook_shadow_supersession_epoch: u64, + webhook_projected_terminal_in_flight: Option, + webhook_dispatch_in_flight: bool, + webhook_targeted_completion: Option, + webhook_terminal_ambiguous: Option, + webhook_drain_first_failure: Option, + webhook_drain_projection_failure: Option, + webhook_drain_timed_out: bool, + webhook_attempt_phase: WebhookAttemptPhase, payload_purge: WebhookPayloadPurgeSchedule, rules_activated: bool, + startup_webhook_retry: Option, reconciliation_quantum: Option, + webhook_drain_work_budget: Option, } /// One process-wide schedule for the expired-payload purge. @@ -1038,6 +1250,35 @@ struct RepositoryWatchTaskContext { webhook_nudge: Option>>, payload_purge: WebhookPayloadPurgeSchedule, reconciliation_quantum: Option, + webhook_drain_work_budget: Option, +} + +fn record_dispatch_start_nudge_outcome( + repository: &RepositorySlug, + session: signalbox_domain::SessionId, + outcome: EligibilityNudgeOutcome, +) { + match outcome { + EligibilityNudgeOutcome::Enqueued => {} + EligibilityNudgeOutcome::Coalesced => tracing::info!( + repository = %repository.as_str(), + session_id = %session.as_uuid(), + cause_code = "repository_watch_dispatch_start_nudge_coalesced", + "repository-watch dispatch-start nudge was coalesced" + ), + EligibilityNudgeOutcome::DroppedAtCapacity => tracing::warn!( + repository = %repository.as_str(), + session_id = %session.as_uuid(), + cause_code = "repository_watch_dispatch_start_nudge_capacity", + "repository-watch dispatch-start nudge was not enqueued" + ), + EligibilityNudgeOutcome::WorkSourceClosed => tracing::warn!( + repository = %repository.as_str(), + session_id = %session.as_uuid(), + cause_code = "repository_watch_dispatch_start_nudge_closed", + "repository-watch dispatch-start nudge was not enqueued" + ), + } } impl RepositoryWatchTask { @@ -1057,6 +1298,7 @@ impl RepositoryWatchTask { webhook_nudge, payload_purge, reconciliation_quantum, + webhook_drain_work_budget, } = context; let credential_reference = configuration.credential_reference(); let credentials = FileCredentialAccess::new_bounded( @@ -1085,18 +1327,57 @@ impl RepositoryWatchTask { webhook_nudge, webhook_shadow: None, webhook_shadow_superseded: false, + webhook_shadow_supersession_epoch: 0, + webhook_projected_terminal_in_flight: None, + webhook_dispatch_in_flight: false, + webhook_targeted_completion: None, + webhook_terminal_ambiguous: None, + webhook_drain_first_failure: None, + webhook_drain_projection_failure: None, + webhook_drain_timed_out: false, + webhook_attempt_phase: WebhookAttemptPhase::BeforeDrain, payload_purge, rules_activated: false, + startup_webhook_retry: None, reconciliation_quantum, + webhook_drain_work_budget, }) } - async fn run(mut self, mut shutdown: watch::Receiver) { + async fn run(mut self, shutdown: watch::Receiver) { + self.run_until_stop(shutdown).await; + // A targeted terminal/cursor completion is deliberately detached from + // drain cancellation, but repository shutdown must still join it before + // the supervisor can report that this task stopped cleanly. The durable + // terminal handoff precedes cursor advancement, so aborting after this + // grace period cannot leave an advanced cursor with a pending delivery. + if timeout( + WEBHOOK_TARGETED_COMPLETION_SHUTDOWN_TIMEOUT, + self.settle_webhook_targeted_completion(), + ) + .await + .is_err() + { + if let Some(handle) = self.webhook_targeted_completion.take() { + handle.abort_and_join().await; + } + tracing::error!( + repository = %self.repository.as_str(), + timeout_seconds = WEBHOOK_TARGETED_COMPLETION_SHUTDOWN_TIMEOUT.as_secs(), + cause_code = "webhook_targeted_completion_shutdown_timed_out", + "repository-watch aborted a retained targeted completion after its durable handoff deadline" + ); + } + } + + async fn run_until_stop(&mut self, mut shutdown: watch::Receiver) { if *shutdown.borrow() { return; } - let mut webhook_retry = WebhookDrainRetry::default(); - if self.webhook_work.is_some() { + let prepared_webhook_retry = self.startup_webhook_retry.take(); + let startup_was_prepared = prepared_webhook_retry.is_some(); + let mut webhook_retry = prepared_webhook_retry.unwrap_or_default(); + if self.webhook_work.is_some() && !startup_was_prepared { let Some(outcome) = self.run_webhook_attempt_until_shutdown(&mut shutdown).await else { return; }; @@ -1164,11 +1445,13 @@ impl RepositoryWatchTask { return; } PollAttemptWait::Continue => { - self.finish_cancelled_webhook_attempt().await; + let _ = self.poller.drain_fetches_bounded().await; + self.poller.invalidate_freshness(); continue; } PollAttemptWait::WebhookRetry => { - self.finish_cancelled_webhook_attempt().await; + let _ = self.poller.drain_fetches_bounded().await; + self.poller.invalidate_freshness(); webhook_retry.consume(); let Some(outcome) = self.run_webhook_attempt_until_shutdown(&mut shutdown).await @@ -1189,7 +1472,11 @@ impl RepositoryWatchTask { continue; } PollAttemptWait::Webhook => { - self.finish_cancelled_webhook_attempt().await; + let _ = self.poller.drain_fetches_bounded().await; + // A cancelled child can publish freshness until its + // final await completes. Invalidate only after every + // child is joined so none can repopulate partial state. + self.poller.invalidate_freshness(); let Some(outcome) = self.run_webhook_attempt_until_shutdown(&mut shutdown).await else { @@ -1297,6 +1584,24 @@ impl RepositoryWatchTask { } } + async fn prepare_startup(&mut self) -> bool { + if self.webhook_work.is_none() { + self.startup_webhook_retry = Some(WebhookDrainRetry::default()); + return false; + } + let outcome = self + .run_webhook_attempt_with_deadline(WEBHOOK_ATTEMPT_TIMEOUT) + .await; + let mut webhook_retry = WebhookDrainRetry::default(); + let must_stop = self.record_webhook_attempt( + WebhookAttemptTrigger::Startup, + outcome, + &mut webhook_retry, + ); + self.startup_webhook_retry = Some(webhook_retry); + must_stop + } + async fn run_webhook_attempt_until_shutdown( &mut self, shutdown: &mut watch::Receiver, @@ -1533,6 +1838,22 @@ impl RepositoryWatchTask { drained: &mut Option, trailing_failure: &mut Option, ) -> Result, RepositoryWatchAttemptError> { + // Any timed-out projection drain may have left pending work before it + // installed delivery-specific settlement state. Do not let projection + // backoff turn that timeout into a cursor-advancing deferred poll. + if drain == WebhookDrain::Deferred && self.webhook_drain_timed_out { + return Err(RepositoryWatchAttemptError::WebhookDrainTimedOut); + } + // A deferred drain may still own a targeted terminal/cursor completion, + // or a prior settlement may not know whether its terminal write + // committed. Do not let a complete poll advance the cursor until the + // owed drain settles that durable state. + if drain == WebhookDrain::Deferred + && (self.webhook_targeted_completion.is_some() + || self.webhook_terminal_ambiguous.is_some()) + { + return Err(RepositoryWatchAttemptError::Persistence); + } if !self.rules_activated { self.activate_rules().await?; self.rules_activated = true; @@ -1570,6 +1891,19 @@ impl RepositoryWatchTask { WebhookDrain::Run => { let outcome = self.process_webhook_deliveries_with_timeout().await; *drained = Some(outcome); + if self.webhook_drain_timed_out && outcome.blocks_complete_poll_after_timeout() { + // A complete poll after a cancelled pre-drain could advance + // the durable cursor past the delivery that remains pending. + // Return to the scheduler so retained drain state settles + // before another complete sweep can commit. + return Err(RepositoryWatchAttemptError::WebhookDrainTimedOut); + } + if self.webhook_terminal_ambiguous.is_some() { + // The targeted terminal write may have committed or rolled + // back. Until a later drain settles that durable state, a + // complete poll must not advance the cursor past it. + return Err(RepositoryWatchAttemptError::Persistence); + } outcome.failure().map_or(Ok(()), Err) } WebhookDrain::Deferred => Ok(()), @@ -1657,6 +1991,7 @@ impl RepositoryWatchTask { async fn run_webhook_attempt(&mut self) -> WebhookAttemptOutcome { self.poller.begin_attempt(); + self.webhook_attempt_phase = WebhookAttemptPhase::BeforeDrain; let outcome = async { if !self.rules_activated { if let Err(error) = self.activate_rules().await { @@ -1674,7 +2009,10 @@ impl RepositoryWatchTask { } else { self.process_dispatches().await.err() }; - match self.process_webhook_deliveries_with_timeout().await { + self.webhook_attempt_phase = WebhookAttemptPhase::Drain; + let drained = self.process_webhook_deliveries_with_timeout().await; + self.webhook_attempt_phase = WebhookAttemptPhase::AfterDrain; + match drained { WebhookDrainOutcome::Drained => {} WebhookDrainOutcome::ProjectionFailed(error) => { return WebhookAttemptOutcome::DrainFailed(error); @@ -1713,24 +2051,39 @@ impl RepositoryWatchTask { match timeout(deadline, self.run_webhook_attempt()).await { Ok(outcome) => outcome, Err(_) => { + let phase = self.webhook_attempt_phase; self.finish_cancelled_webhook_attempt().await; let error = RepositoryWatchAttemptError::WebhookAttemptTimedOut; tracing::error!( repository = %self.repository.as_str(), timeout_seconds = deadline.as_secs(), + cancelled_phase = phase.label(), cause_code = error.cause_code(), "repository-watch webhook attempt exceeded its deadline" ); - WebhookAttemptOutcome::DrainFailed(error) + // Cancellation carries no failure of its own, so the cancelled + // step decides the outcome: only a drain the deadline + // interrupted has earned the growing projection backoff. + phase.cancelled_outcome(error) } } } - async fn finish_cancelled_webhook_attempt(&self) { - if !self - .poller - .drain_fetches_within(WEBHOOK_CANCELLED_FETCH_DRAIN_TIMEOUT) - .await + /// Performs the cleanup every cancelled webhook attempt owes its successor. + /// + /// Cancellation itself must not wedge the repository task, so the poller's + /// shared child fetch set is joined under its own bound; a later attempt + /// drains that same set before it can spawn, preserving the + /// no-interleaving policy. Either deadline can cancel a projected terminal + /// write, so the carried shadow is settled here rather than at one call + /// site. + async fn finish_cancelled_webhook_attempt(&mut self) { + if timeout( + WEBHOOK_CANCELLED_FETCH_DRAIN_TIMEOUT, + self.poller.drain_fetches(), + ) + .await + .is_err() { tracing::error!( repository = %self.repository.as_str(), @@ -1740,13 +2093,46 @@ impl RepositoryWatchTask { ); } self.poller.invalidate_freshness(); + // Only a projected terminal write can make the carried shadow + // ambiguous. A targeted cursor commit is retained separately and + // settled before another drain, so its delivery keeps the pre-commit + // shadow needed to reproduce its projections. + if let Some(key) = self.webhook_projected_terminal_in_flight.take() { + self.webhook_shadow = None; + self.webhook_shadow_superseded = false; + self.webhook_terminal_ambiguous = Some(key); + } } async fn process_webhook_deliveries(&mut self) -> WebhookDrainOutcome { + self.process_webhook_deliveries_with_budget(self.webhook_drain_work_budget) + .await + } + + async fn process_webhook_deliveries_with_budget( + &mut self, + work_budget: Option, + ) -> WebhookDrainOutcome { + self.webhook_drain_first_failure = None; + self.webhook_drain_projection_failure = None; + if let Some(Err(error)) = self.settle_webhook_targeted_completion().await { + self.webhook_drain_first_failure = Some(error); + return WebhookDrainOutcome::ProjectionFailed(error); + } + if let Some(key) = self.webhook_terminal_ambiguous + && self + .webhook_store + .terminal_disposition_exists(key) + .await + .is_ok_and(|exists| exists) + { + self.webhook_terminal_ambiguous = None; + } let Ok(page_size) = RepoWatchWebhookPendingPageSize::try_new(WEBHOOK_PENDING_PAGE_SIZE) else { return WebhookDrainOutcome::ProjectionFailed(RepositoryWatchAttemptError::Persistence); }; + let started_at = Instant::now(); let mut deferred: HashSet = HashSet::new(); let mut first_failure: Option = None; let mut dispatch_failure: Option = None; @@ -1802,11 +2188,16 @@ impl RepositoryWatchTask { if deferred.contains(&delivery.key()) { continue; } - match self + let terminalized = match self .process_webhook_delivery(delivery, &mut page, &mut dispatch_failure) .await { - Ok(()) => {} + Ok(()) => { + if self.webhook_terminal_ambiguous == Some(delivery.key()) { + self.webhook_terminal_ambiguous = None; + } + true + } Err(error) => { if error.stops_webhook_page() { tracing::warn!( @@ -1836,11 +2227,25 @@ impl RepositoryWatchTask { deferred.insert(delivery.key()); if first_failure.is_none() { first_failure = Some(error); + self.webhook_drain_projection_failure = Some(error); } + false } - } + }; if chronological_first.is_none() { chronological_first = first_failure.or(dispatch_failure); + self.webhook_drain_first_failure = chronological_first; + } + if terminalized && work_budget.is_some_and(|budget| started_at.elapsed() >= budget) + { + self.request_webhook_drain_continuation(); + tracing::info!( + repository = %self.repository.as_str(), + work_budget_seconds = work_budget.map_or(0, |budget| budget.as_secs()), + cause_code = "webhook_projection_drain_work_budget_exhausted", + "progressing repository-watch webhook drain yielded before its deadline" + ); + break 'drain; } } pages += 1; @@ -1885,13 +2290,30 @@ impl RepositoryWatchTask { &mut self, deadline: Duration, ) -> WebhookDrainOutcome { + self.webhook_drain_timed_out = false; match timeout(deadline, self.process_webhook_deliveries()).await { - Ok(outcome) => outcome, + Ok(outcome) => { + self.webhook_drain_first_failure = None; + self.webhook_drain_projection_failure = None; + outcome + } Err(_) => { + self.webhook_drain_timed_out = true; // A future implementation may use the poller's bounded child - // fetch set while hydrating a delivery. Join anything the - // cancelled drain owned before the next attempt can begin. + // fetch set while hydrating a delivery. The shared cancellation + // cleanup bounds that join and settles the carried shadow, so + // the poller's next attempt drains the same shared set before + // it can spawn, preserving the no-interleaving policy. self.finish_cancelled_webhook_attempt().await; + let first_failure = self.webhook_drain_first_failure.take(); + let projection_failure = self.webhook_drain_projection_failure.take(); + if let Some(first_failure) = first_failure { + tracing::error!( + repository = %self.repository.as_str(), + cause_code = first_failure.cause_code(), + "repository-watch webhook drain retained an earlier failure before its deadline" + ); + } let error = RepositoryWatchAttemptError::WebhookDrainTimedOut; tracing::error!( repository = %self.repository.as_str(), @@ -1899,7 +2321,15 @@ impl RepositoryWatchTask { cause_code = error.cause_code(), "repository-watch webhook drain exceeded its attempt deadline" ); - WebhookDrainOutcome::ProjectionFailed(error) + if let Some(projection_failure) = projection_failure { + self.webhook_dispatch_in_flight = false; + WebhookDrainOutcome::ProjectionFailed(projection_failure) + } else if self.webhook_dispatch_in_flight { + self.webhook_dispatch_in_flight = false; + WebhookDrainOutcome::DispatchFailedAfterTerminal(first_failure.unwrap_or(error)) + } else { + WebhookDrainOutcome::ProjectionFailed(error) + } } } } @@ -2138,45 +2568,47 @@ impl RepositoryWatchTask { .collect::, _>>()?, ); if let Some(prepared) = prepared { - // A targeted poll reconciles only the pull requests it - // names, so its cursor does not carry what the webhook - // stream has projected for anything else. The shadow - // is kept rather than reloaded; the next full poll is - // the complete sweep that replaces it. - self.commit_targeted_refresh(prepared).await?; - // Recorded only once the refresh has landed, so a - // failure above leaves the hydration for the page's - // remaining deliveries to reissue. - page.record_issued(&issued); + // Retain the exact cursor commit, projections, and + // resulting shadow as one completion. Cancellation + // of the outer drain cannot separate those durable + // steps or lose targeted-query provenance. + let settlement = self + .complete_targeted_webhook_projection( + prepared, + pending.key(), + projections, + WebhookShadowBaseline { + observation, + identity_frontier, + }, + ) + .await?; + // A superseded commit leaves this delivery terminal + // but never reached the cursor, so the coalescer must + // not treat its hydration as landed: a later delivery + // for the same pull request on this page still owes + // the targeted query this one failed to commit. + if settlement == TargetedRefreshSettlement::Landed { + page.record_issued(&issued); + } + } else { + self.record_webhook_terminal( + pending, + projections, + RepoWatchWebhookDisposition::Projected, + None, + ) + .await?; + self.webhook_shadow = Some(WebhookShadowBaseline { + observation, + identity_frontier, + }); + self.webhook_shadow_superseded = false; } - // The delivery becomes terminal only once every durable - // write it asked for has landed, so a failed cursor commit - // leaves it pending and the whole step is retried. - // - // Recording after the commit was unsafe while projections - // were derived from the durable cursor, because a retry - // would then derive against a cursor that had moved. They - // are derived from the repository task's shadow baseline - // now, which a targeted commit deliberately does not - // replace, so a retry reproduces these same projections. - self.record_webhook_terminal( - pending, - projections, - RepoWatchWebhookDisposition::Projected, - None, - ) - .await?; - // The shadow advances only once that disposition is - // durable, so the two never disagree. Advancing it also - // clears any supersession a poll left pending: the - // baseline now carries facts newer than that cursor, so - // handing it over would discard them. - self.webhook_shadow = Some(WebhookShadowBaseline { - observation, - identity_frontier, - }); - self.webhook_shadow_superseded = false; - if let Err(error) = self.process_dispatches().await { + self.webhook_dispatch_in_flight = true; + let dispatch_result = self.process_dispatches().await; + self.webhook_dispatch_in_flight = false; + if let Err(error) = dispatch_result { // Carries the identity here because this delivery // is already terminal: it never reaches the drain // page's deferral record, and the classified @@ -2216,13 +2648,20 @@ impl RepositoryWatchTask { outcome_code.map(str::to_owned), ) .map_err(|_| RepositoryWatchAttemptError::Persistence)?; + self.webhook_projected_terminal_in_flight = advances_shadow.then_some(pending.key()); for attempt in 1..=MAX_WEBHOOK_TERMINAL_ATTEMPTS { match self .webhook_store .record_terminal(pending.key(), &request) .await { - Ok(_) => return Ok(()), + Ok(_) => { + self.webhook_projected_terminal_in_flight = None; + if self.webhook_terminal_ambiguous == Some(pending.key()) { + self.webhook_terminal_ambiguous = None; + } + return Ok(()); + } // A commit whose result was lost in transit may already be // durable, and the delivery would then never be loaded again. // Reading settles which happened and cannot itself be @@ -2234,7 +2673,13 @@ impl RepositoryWatchTask { .terminal_disposition_exists(pending.key()) .await { - Ok(true) => return Ok(()), + Ok(true) => { + self.webhook_projected_terminal_in_flight = None; + if self.webhook_terminal_ambiguous == Some(pending.key()) { + self.webhook_terminal_ambiguous = None; + } + return Ok(()); + } // A read that fails settles nothing, so it is retried // rather than propagated: propagating would abandon a // delivery that may already be durable. @@ -2245,7 +2690,10 @@ impl RepositoryWatchTask { } } } - Err(_) => return Err(RepositoryWatchAttemptError::Persistence), + Err(_) => { + self.webhook_projected_terminal_in_flight = None; + return Err(RepositoryWatchAttemptError::Persistence); + } } } // Every attempt was ambiguous or unreadable, so whether a disposition @@ -2256,7 +2704,9 @@ impl RepositoryWatchTask { // records the gap. if advances_shadow { self.webhook_shadow = None; + self.webhook_terminal_ambiguous = Some(pending.key()); } + self.webhook_projected_terminal_in_flight = None; Err(RepositoryWatchAttemptError::Persistence) } @@ -2330,37 +2780,174 @@ impl RepositoryWatchTask { )) } - /// Commits one prepared targeted refresh against the generation it read. - async fn commit_targeted_refresh( - &self, + /// Commits a targeted refresh and its exact webhook projection as one + /// retained completion that survives cancellation of the outer drain. + async fn complete_targeted_webhook_projection( + &mut self, prepared: PreparedTargetedRefresh, - ) -> Result<(), RepositoryWatchAttemptError> { - let outcome = self - .store - .commit( - &self.repository, - RepoWatchCommitRequest::new( - Some(prepared.generation), - prepared.candidate, - prepared.events, - ), - ) + key: RepoWatchWebhookDeliveryKey, + projections: Vec, + shadow: WebhookShadowBaseline, + ) -> Result { + let store = self.store.clone(); + let webhook_store = self.webhook_store.clone(); + let poller = Arc::clone(&self.poller); + let repository = self.repository.clone(); + let request = RepoWatchCommitRequest::new( + Some(prepared.generation), + prepared.candidate, + prepared.events, + ); + let terminal = RepoWatchWebhookTerminalRequest::try_new( + projections, + RepoWatchWebhookDisposition::Projected, + None, + ) + .map_err(|_| RepositoryWatchAttemptError::Persistence)?; + let supersession_epoch = self.webhook_shadow_supersession_epoch; + self.webhook_targeted_completion = Some(RetainedTargetedWebhookCompletion::new( + tokio::spawn(async move { + // Persist the terminal disposition and its exact projections first. + // This is the durable recovery handoff: if shutdown later aborts + // cursor advancement, restart excludes the delivery without losing + // its projections, and an ordinary poll can advance the old cursor. + record_webhook_terminal_request(&webhook_store, key, &terminal) + .await + .map_err(|error| TargetedWebhookCompletionError::Terminal(key, error))?; + let outcome = store + .commit(&repository, request) + .await + .map_err(|_| TargetedWebhookCompletionError::Cursor)?; + match outcome { + RepoWatchCommitOutcome::Committed(cursor) + | RepoWatchCommitOutcome::Replayed(cursor) + | RepoWatchCommitOutcome::Unchanged(cursor) => { + poller.publish_freshness(cursor.generation()); + } + RepoWatchCommitOutcome::Conflict { current: _ } => { + // This fetch never became cursor state, but it already + // recorded unpublished freshness. Leaving those entries + // behind would let the next commit's `publish_freshness` + // stamp them with a generation they never reached, and + // a later poll would then reuse detail the cursor does + // not carry. A competing writer owns the cursor, which + // is exactly the condition this clearing exists for. + poller.invalidate_freshness(); + return Ok(TargetedWebhookCompletion::CursorSuperseded { key }); + } + } + Ok(TargetedWebhookCompletion::Applied { + key, + shadow, + supersession_epoch, + }) + }), + )); + self.settle_webhook_targeted_completion() .await - .map_err(|_| RepositoryWatchAttemptError::Persistence)?; - match outcome { - RepoWatchCommitOutcome::Committed(cursor) - | RepoWatchCommitOutcome::Replayed(cursor) - | RepoWatchCommitOutcome::Unchanged(cursor) => { - self.poller.publish_freshness(cursor.generation()); - Ok(()) - } - RepoWatchCommitOutcome::Conflict { current: _ } => { - Err(RepositoryWatchAttemptError::Persistence) + .ok_or(RepositoryWatchAttemptError::Persistence)? + } + + /// Settles a targeted completion retained across drain cancellation. + /// + /// Awaiting the handle by mutable reference means cancelling this caller + /// leaves the database task and its handle intact. A later drain settles + /// that exact commit before reading or writing subsequent repository work. + async fn settle_webhook_targeted_completion( + &mut self, + ) -> Option> { + let result = { + let handle = self.webhook_targeted_completion.as_mut()?; + handle + .join() + .await + .map_err(|_| TargetedWebhookCompletionError::Persistence) + .and_then(|result| result) + }; + self.webhook_targeted_completion = None; + match result { + Ok(TargetedWebhookCompletion::Applied { + key, + shadow, + supersession_epoch, + }) => { + if self.webhook_terminal_ambiguous == Some(key) { + self.webhook_terminal_ambiguous = None; + } + self.webhook_shadow = Some(shadow); + if self.webhook_shadow_supersession_epoch == supersession_epoch { + self.webhook_shadow_superseded = false; + } + Some(Ok(TargetedRefreshSettlement::Landed)) } + Ok(TargetedWebhookCompletion::CursorSuperseded { key }) => { + if self.webhook_terminal_ambiguous == Some(key) { + self.webhook_terminal_ambiguous = None; + } + // The terminal disposition and projections are durable, while + // a competing poll owns the current cursor. Hand the shadow + // over immediately so later pending receipts seed from it. + self.webhook_shadow = None; + self.webhook_shadow_superseded = false; + Some(Ok(TargetedRefreshSettlement::Superseded)) + } + Err(TargetedWebhookCompletionError::Terminal( + key, + WebhookTerminalRecordError::Ambiguous, + )) => { + self.webhook_shadow = None; + self.webhook_shadow_superseded = false; + self.webhook_terminal_ambiguous = Some(key); + Some(Err(RepositoryWatchAttemptError::Persistence)) + } + Err(TargetedWebhookCompletionError::Cursor) => { + // The terminal disposition and exact projections are durable, + // but the cursor outcome is unknown. Reload the durable cursor + // before projecting any later pending receipt. + self.webhook_shadow = None; + self.webhook_shadow_superseded = false; + Some(Err(RepositoryWatchAttemptError::Persistence)) + } + Err(_) => Some(Err(RepositoryWatchAttemptError::Persistence)), } } async fn process_cutoffs(&self) -> Result<(), RepositoryWatchAttemptError> { + // numeric-bound: guard - prevents a repeatedly quarantined lease from looping the repository task forever + const MAX_EXPIRED_START_LEASES_PER_ATTEMPT: usize = 32; + for _ in 0..MAX_EXPIRED_START_LEASES_PER_ATTEMPT { + match self + .dispatch_store + .process_next_expired_start_lease(&self.repository, || { + DurableCommandId::from_uuid(uuid::Uuid::now_v7()) + }) + .await + { + Ok(true) => {} + Ok(false) => break, + Err(RepoWatchDispatchRepositoryError::GoalCutoff( + error @ signalbox_persistence::goal::GoalRepositoryError::Corruption(_), + )) => { + tracing::error!( + repository = %self.repository.as_str(), + cause_code = "repository_watch_expired_start_lease_corruption", + error = %error, + "repository-watch expired start lease quarantined a corrupt goal; cutoff processing continues" + ); + continue; + } + Err(error @ RepoWatchDispatchRepositoryError::Corruption(_)) => { + tracing::error!( + repository = %self.repository.as_str(), + cause_code = "repository_watch_expired_start_lease_corruption", + error = %error, + "repository-watch expired start lease quarantined corrupt storage; cutoff processing continues" + ); + continue; + } + Err(_) => return Err(RepositoryWatchAttemptError::Persistence), + } + } let mut processed = 0_usize; loop { match self @@ -2376,7 +2963,7 @@ impl RepositoryWatchTask { return Ok(()); } } - Ok(false) => return Ok(()), + Ok(false) => break, Err(RepoWatchDispatchRepositoryError::GoalCutoff( error @ signalbox_persistence::goal::GoalRepositoryError::Corruption(_), )) => { @@ -2391,6 +2978,31 @@ impl RepositoryWatchTask { Err(_) => return Err(RepositoryWatchAttemptError::Persistence), } } + loop { + match self + .dispatch_store + .process_next_convergence_cutoff(&self.repository, || { + DurableCommandId::from_uuid(uuid::Uuid::now_v7()) + }) + .await + { + Ok(true) => {} + Ok(false) => break, + Err(RepoWatchDispatchRepositoryError::GoalCutoff( + error @ signalbox_persistence::goal::GoalRepositoryError::Corruption(_), + )) => { + tracing::error!( + repository = %self.repository.as_str(), + cause_code = "repository_watch_convergence_cutoff_corruption", + error = %error, + "repository-watch convergence cutoff quarantined a corrupt goal; dispatch processing continues" + ); + continue; + } + Err(_) => return Err(RepositoryWatchAttemptError::Persistence), + } + } + Ok(()) } async fn activate_rules(&self) -> Result<(), RepositoryWatchAttemptError> { @@ -2401,9 +3013,17 @@ impl RepositoryWatchTask { } async fn process_dispatches(&mut self) -> Result<(), RepositoryWatchAttemptError> { - let mut processed = 0_usize; - for rule in &self.rules { - while let Some(event) = self + let unstarted = self + .dispatch_store + .load_unstarted_dispatch_sessions(&self.repository) + .await + .map_err(|_| RepositoryWatchAttemptError::Persistence)?; + for session in unstarted { + self.nudge_dispatch_start(session); + } + let mut processed = 0_usize; + for rule in &self.rules { + while let Some(event) = self .dispatch_store .load_next_event(&self.repository, rule.id(), rule.version()) .await @@ -2512,17 +3132,23 @@ impl RepositoryWatchTask { RepoWatchRuleEvaluationOutcome::Dispatched { sessions, .. } | RepoWatchRuleEvaluationOutcome::Replayed { sessions, .. } => { for session in sessions { - let _ = self.eligibility_nudge.nudge(*session); + self.nudge_dispatch_start(*session); } } RepoWatchRuleEvaluationOutcome::NotMatched | RepoWatchRuleEvaluationOutcome::Inactive | RepoWatchRuleEvaluationOutcome::TargetClosed + | RepoWatchRuleEvaluationOutcome::TargetConverged | RepoWatchRuleEvaluationOutcome::Occupied | RepoWatchRuleEvaluationOutcome::Cooldown => {} } } + fn nudge_dispatch_start(&self, session: signalbox_domain::SessionId) { + let outcome = self.eligibility_nudge.nudge_dispatch_start(session); + record_dispatch_start_nudge_outcome(&self.repository, session, outcome); + } + /// Loads the durable baseline and performs the read-only provider sweep. /// /// This phase may be abandoned for a webhook wake. It deliberately stops @@ -2544,14 +3170,14 @@ impl RepositoryWatchTask { .as_ref() .map(|cursor| cursor.candidate().event_identity_frontier().clone()) .unwrap_or_default(); - let observation = self + let polled = self .poller .poll_against_cursor(previous, cursor_generation) .await?; let events = derive_repo_watch_events( &self.repository, previous, - &observation, + &polled.observation, &mut event_identity_frontier, &mut UuidV7RepoWatchEventIdGenerator, ) @@ -2573,10 +3199,12 @@ impl RepositoryWatchTask { Ok(PreparedCompletePoll { cursor_generation, candidate: RepoWatchCursorCandidate::with_event_identity_frontier( - observation, + polled.observation, event_identity_frontier, ), events, + convergence: polled.convergence, + stale_review_clearances: polled.stale_review_clearances, }) } @@ -2587,13 +3215,14 @@ impl RepositoryWatchTask { ) -> Result<(), RepositoryWatchAttemptError> { let outcome = self .store - .commit( + .commit_with_convergence( &self.repository, RepoWatchCommitRequest::new( prepared.cursor_generation, prepared.candidate, prepared.events, ), + &prepared.convergence, ) .await .map_err(|_| RepositoryWatchAttemptError::Persistence)?; @@ -2601,7 +3230,69 @@ impl RepositoryWatchTask { RepoWatchCommitOutcome::Committed(cursor) | RepoWatchCommitOutcome::Replayed(cursor) | RepoWatchCommitOutcome::Unchanged(cursor) => { + // Published before the clearance sweep rather than after it. + // The cursor is durable at this point, so the freshness this + // poll recorded is legitimately tied to a committed generation, + // and clearance revalidation reads exactly that entry to decide + // whether the gating-check inventory has stood still since the + // observation that raised the candidate. A publication the + // sweep never reached would leave every candidate unsettled and + // no review would ever be dismissed. A failed attempt still + // invalidates every entry on its way out. self.poller.publish_freshness(cursor.generation()); + self.reconcile_pending_stale_review_clearances().await?; + let planned_clearances = self + .store + .plan_stale_review_clearances( + &self.repository, + cursor.generation(), + &prepared.stale_review_clearances, + ) + .await + .map_err(|_| RepositoryWatchAttemptError::Persistence)?; + for clearance in &planned_clearances { + if !self + .poller + .revalidate_stale_review_clearance(clearance, cursor.generation()) + .await? + { + self.store + .release_stale_review_clearance_claim( + clearance.clearance_id(), + clearance.claim_token(), + ) + .await + .map_err(|_| RepositoryWatchAttemptError::Persistence)?; + continue; + } + if self + .store + .renew_stale_review_clearance_claim( + clearance.clearance_id(), + clearance.claim_token(), + ) + .await + .map_err(|_| RepositoryWatchAttemptError::Persistence)? + == RepoWatchStaleReviewClearanceRenewal::Lost + { + continue; + } + self.poller + .dismiss_review_node(DismissReviewInput { + review_node_id: clearance.review_node_id(), + dismissal_message: clearance.dismissal_message(), + }) + .await?; + self.store + .record_stale_review_clearance_outcome( + clearance.clearance_id(), + clearance.claim_token(), + RepoWatchStaleReviewClearanceOutcome::Dismissed, + RepoWatchObservedReviewState::Dismissed, + ) + .await + .map_err(|_| RepositoryWatchAttemptError::Persistence)?; + } // A full poll is the complete reconciliation sweep, so the // cursor it commits supersedes everything the webhook stream // had accumulated in memory. It is not handed over here: this @@ -2612,6 +3303,8 @@ impl RepositoryWatchTask { // instead, where an empty page and the replacement are decided // without an await between them. self.webhook_shadow_superseded = true; + self.webhook_shadow_supersession_epoch = + self.webhook_shadow_supersession_epoch.wrapping_add(1); Ok(()) } RepoWatchCommitOutcome::Conflict { current: _ } => { @@ -2619,6 +3312,166 @@ impl RepositoryWatchTask { } } } + + async fn reconcile_pending_stale_review_clearances( + &self, + ) -> Result<(), RepositoryWatchAttemptError> { + let pending = self + .store + .claim_pending_stale_review_clearances(&self.repository) + .await + .map_err(|_| RepositoryWatchAttemptError::Persistence)?; + for clearance in &pending { + // Observing a batch row costs provider requests, so a deeply + // paginated batch can outlive the two-minute lease taken when it + // was claimed. Re-establish ownership immediately before each row: + // a lease another watcher has since taken belongs to that watcher, + // and skipping the row leaves it to them instead of acting twice. + if self + .store + .renew_stale_review_clearance_claim( + clearance.clearance_id(), + clearance.claim_token(), + ) + .await + .map_err(|_| RepositoryWatchAttemptError::Persistence)? + == RepoWatchStaleReviewClearanceRenewal::Lost + { + continue; + } + match self + .poller + .observe_stale_review_clearance(clearance) + .await? + { + StaleReviewClearanceObservation::StillBlocking => { + self.store + .release_stale_review_clearance_claim( + clearance.clearance_id(), + clearance.claim_token(), + ) + .await + .map_err(|_| RepositoryWatchAttemptError::Persistence)?; + } + StaleReviewClearanceObservation::Terminal { + outcome, + provider_state, + } => { + // The lease can still expire between the renewal above and + // this write. That intent now belongs to its new claimant, + // whose own scan will settle it; failing the attempt here + // would instead abandon every row the batch has left. + match self + .store + .record_stale_review_clearance_outcome( + clearance.clearance_id(), + clearance.claim_token(), + outcome, + provider_state, + ) + .await + { + Ok(()) => {} + Err(RepoWatchStoreError::StaleReviewClearanceMismatch) => continue, + Err(_) => return Err(RepositoryWatchAttemptError::Persistence), + } + } + } + } + Ok(()) + } +} + +async fn record_webhook_terminal_request( + store: &PostgresRepoWatchWebhookStore, + key: RepoWatchWebhookDeliveryKey, + request: &RepoWatchWebhookTerminalRequest, +) -> Result<(), WebhookTerminalRecordError> { + for attempt in 1..=MAX_WEBHOOK_TERMINAL_ATTEMPTS { + match store.record_terminal(key, request).await { + Ok(_) => return Ok(()), + Err(RepoWatchWebhookStoreError::CommitAmbiguous(_)) => { + match store.terminal_disposition_exists(key).await { + Ok(true) => return Ok(()), + Ok(false) | Err(_) if attempt < MAX_WEBHOOK_TERMINAL_ATTEMPTS => { + sleep(WEBHOOK_TERMINAL_RETRY_DELAY).await; + } + Ok(false) | Err(_) => {} + } + } + Err(_) => return Err(WebhookTerminalRecordError::Persistence), + } + } + Err(WebhookTerminalRecordError::Ambiguous) +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum WebhookTerminalRecordError { + Persistence, + Ambiguous, +} + +#[derive(Debug)] +enum TargetedWebhookCompletionError { + Persistence, + Cursor, + Terminal(RepoWatchWebhookDeliveryKey, WebhookTerminalRecordError), +} + +struct RetainedTargetedWebhookCompletion { + handle: JoinHandle>, +} + +impl RetainedTargetedWebhookCompletion { + fn new( + handle: JoinHandle>, + ) -> Self { + Self { handle } + } + + async fn join( + &mut self, + ) -> Result< + Result, + tokio::task::JoinError, + > { + (&mut self.handle).await + } + + async fn abort_and_join(mut self) { + self.handle.abort(); + let _ = (&mut self.handle).await; + } +} + +impl Drop for RetainedTargetedWebhookCompletion { + fn drop(&mut self) { + self.handle.abort(); + } +} + +enum TargetedWebhookCompletion { + Applied { + key: RepoWatchWebhookDeliveryKey, + shadow: WebhookShadowBaseline, + supersession_epoch: u64, + }, + CursorSuperseded { + key: RepoWatchWebhookDeliveryKey, + }, +} + +/// Whether a settled targeted completion reached the durable cursor. +/// +/// A superseded completion keeps its terminal disposition and projections, so +/// the delivery is done, but its fetch never became cursor state. Callers that +/// record consequences of the fetch landing must distinguish the two. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum TargetedRefreshSettlement { + /// The commit reached the durable cursor. + Landed, + /// A competing writer owned the cursor, so this fetch never reached it. + Superseded, } /// One complete provider sweep derived against a durable cursor but not yet @@ -2627,6 +3480,8 @@ struct PreparedCompletePoll { cursor_generation: Option, candidate: RepoWatchCursorCandidate, events: Vec, + convergence: Vec, + stale_review_clearances: Vec, } #[derive(Clone, Debug, Eq, PartialEq)] @@ -3428,13 +4283,22 @@ struct GitHubRepositoryPoller { // dropping the future aborts them and releases the lock, but they stay // joinable, and whoever runs next — the following attempt, or the // repository task on its way out — joins them before proceeding. - fetches: - tokio::sync::Mutex>>, + fetches: tokio::sync::Mutex>>, +} + +async fn drain_pull_request_fetches( + fetches: &mut JoinSet>, +) -> Result<(), RepositoryWatchAttemptError> { + timeout(WEBHOOK_CANCELLED_FETCH_DRAIN_TIMEOUT, fetches.shutdown()) + .await + .map_err(|_| RepositoryWatchAttemptError::PullRequestFetchAbandoned) } struct PullRequestFreshness { updated_at: String, + head_sha: CommitSha, settlement: PullRequestSettlement, + gating_check_inventory: Vec, skipped_polls: usize, // A fetch that never reached the durable cursor must not authorize reuse: // the next attempt would compare this updated_at against a stale committed @@ -3453,6 +4317,62 @@ struct ListedPullRequest { struct FetchedPullRequest { state: RepoWatchPullRequestState, settlement: PullRequestSettlement, + convergence_evidence: FetchedConvergenceEvidence, +} + +struct FetchedConvergenceEvidence { + base_revision: CommitSha, + gating_checks_settled: bool, + gating_check_inventory_quiesced: bool, + gating_check_inventory: Vec, + review_decision: RepoWatchReviewDecision, + gating_check_count: u64, + non_green_gating_checks: Vec, +} + +impl FetchedConvergenceEvidence { + fn assess( + self, + state: &RepoWatchPullRequestState, + base_revision: CommitSha, + ) -> Result { + if self.base_revision != base_revision { + return Err(RepositoryWatchAttemptError::InvalidResponse); + } + RepoWatchConvergenceAssessment::try_new(RepoWatchConvergenceAssessmentInput { + number: state.context().number(), + head_sha: state.context().head_sha().clone(), + base_branch: state.context().base_branch().clone(), + base_revision, + mergeable_state: state.mergeable_state(), + settled: self.gating_checks_settled + && self.gating_check_inventory_quiesced + && state.mergeable_state() != MergeableState::Unknown, + review_decision: self.review_decision, + unresolved_threads: state + .threads() + .iter() + .filter(|thread| thread.state() == RepoWatchThreadState::Open) + .map(|thread| thread.thread().clone()) + .collect(), + gating_check_count: self.gating_check_count, + non_green_gating_checks: self.non_green_gating_checks, + }) + .map_err(|_| RepositoryWatchAttemptError::Normalization) + } +} + +struct PolledRepository { + observation: RepoWatchObservation, + convergence: Vec, + stale_review_clearances: Vec, +} + +#[derive(Debug)] +struct FetchedPullRequests { + states: Vec, + convergence: Vec, + stale_review_clearances: Vec, } #[derive(Clone, Copy, Debug, Eq, PartialEq)] @@ -3538,15 +4458,17 @@ impl GitHubRepositoryPoller { self: &Arc, previous: Option<&RepoWatchObservation>, ) -> Result { - self.poll_against_cursor(previous, Some(RepoWatchCursorGeneration::INITIAL)) - .await + Ok(self + .poll_against_cursor(previous, Some(RepoWatchCursorGeneration::INITIAL)) + .await? + .observation) } async fn poll_against_cursor( self: &Arc, previous: Option<&RepoWatchObservation>, cursor_generation: Option, - ) -> Result { + ) -> Result { self.cache().begin_poll(); let result = self.poll_complete(previous, cursor_generation).await; if result.is_ok() { @@ -3560,6 +4482,18 @@ impl GitHubRepositoryPoller { previous: &RepoWatchObservation, targets: &[TargetedPullRequest], ) -> Result { + // A cancelled complete poll can leave child fetches in the shared set. + // Settle them within the scheduler bound before issuing targeted + // requests, so work from two attempts cannot interleave. + let drained_survivors = self.drain_fetches_bounded().await?; + // A survivor can record freshness after the cancellation path's first + // invalidation and before this join completes. Clear that late state + // before targeted work can publish it against a new cursor, while an + // ordinary targeted refresh preserves published freshness for untouched + // pull requests. + if drained_survivors { + self.invalidate_freshness(); + } let mut state = RepoWatchRepositoryStateInput { pull_requests: previous.state().pull_requests().to_vec(), workflow_runs: previous.state().workflow_runs().to_vec(), @@ -3615,7 +4549,7 @@ impl GitHubRepositoryPoller { self: &Arc, previous: Option<&RepoWatchObservation>, cursor_generation: Option, - ) -> Result { + ) -> Result { let listed = self.fetch_open_pull_numbers().await?; let mut pull_numbers: BTreeSet = listed.keys().copied().collect(); if let Some(previous) = previous { @@ -3625,10 +4559,20 @@ impl GitHubRepositoryPoller { } } } + // Anchor convergence assessments to the same branch-head snapshot that + // will be committed in the cursor. Pull-request hydration is the long + // phase of a poll, so reading branch heads afterwards creates a broad + // window in which an ordinary base advance invalidates all evidence. + let branch_heads = self.fetch_branch_heads().await?; let pull_requests = self - .fetch_pull_requests(pull_numbers, &listed, previous, cursor_generation) + .fetch_pull_requests( + pull_numbers, + &listed, + previous, + cursor_generation, + &branch_heads, + ) .await?; - let branch_heads = self.fetch_branch_heads().await?; let workflows = self.fetch_workflows().await?; let mut workflow_runs = Vec::new(); let previous_workflow_runs = previous @@ -3641,15 +4585,16 @@ impl GitHubRepositoryPoller { ); } let state = RepoWatchRepositoryState::try_new(RepoWatchRepositoryStateInput { - pull_requests, + pull_requests: pull_requests.states, workflow_runs, branch_heads, }) .map_err(|_| RepositoryWatchAttemptError::Normalization)?; - Ok(RepoWatchObservation::new( - self.signal_reviewers.clone(), - state, - )) + Ok(PolledRepository { + observation: RepoWatchObservation::new(self.signal_reviewers.clone(), state), + convergence: pull_requests.convergence, + stale_review_clearances: pull_requests.stale_review_clearances, + }) } async fn fetch_pull_requests( @@ -3658,20 +4603,23 @@ impl GitHubRepositoryPoller { listed: &BTreeMap, previous: Option<&RepoWatchObservation>, cursor_generation: Option, - ) -> Result, RepositoryWatchAttemptError> { + branch_heads: &[RepoWatchBranchHead], + ) -> Result { self.forget_unlisted_freshness(&pull_numbers); let mut fetches = self.fetches.lock().await; // A cancelled attempt drops this future mid-collection, which aborts // the children without joining them; they stay behind in the shared // set. Join any such survivor before spawning, so no child of an - // earlier attempt can interleave with this one. - fetches.shutdown().await; + // earlier attempt can interleave with this one. A wedged survivor + // fails this attempt back to the scheduler after a bounded wait. + drain_pull_request_fetches(&mut fetches).await?; let collected = self .collect_pull_request_fetches( pull_numbers, listed, previous, cursor_generation, + branch_heads, &mut fetches, ) .await; @@ -3679,17 +4627,58 @@ impl GitHubRepositoryPoller { // An aborted task only stops at its next await, so it can still charge // wire bytes, touch cache entries, or record freshness after this // attempt returns, landing that state in the next attempt. Wait for - // every task to finish before the caller can begin another poll. - fetches.shutdown().await; + // every task to finish before the caller can begin another poll, but + // return to the scheduler if a child does not finish cancellation. + drain_pull_request_fetches(&mut fetches).await?; let mut pull_requests = collected?; - pull_requests.sort_by_key(|pull_request| pull_request.context().number().get()); - Ok(pull_requests) + pull_requests.sort_by_key(|pull_request| pull_request.state.context().number().get()); + let mut states = Vec::with_capacity(pull_requests.len()); + let mut convergence = Vec::with_capacity(pull_requests.len()); + let mut stale_review_clearances = Vec::new(); + for pull_request in pull_requests { + let base_revision = branch_heads + .iter() + .find(|branch_head| { + branch_head.branch() == pull_request.state.context().base_branch() + }) + .map(|branch_head| branch_head.head().clone()) + .ok_or(RepositoryWatchAttemptError::InvalidResponse)?; + let assessment = pull_request + .convergence_evidence + .assess(&pull_request.state, base_revision)?; + // Clearance candidates are read from the assessment, which only + // exists once the snapshot's base revision is known, so this runs + // here rather than in the per-pull-request fetch task. The lookup + // returns immediately unless a changes-requested review is the sole + // remaining blocker, so the serial call costs nothing in the common + // case. + if pull_request.state.lifecycle() == RepoWatchPullRequestLifecycle::Open { + stale_review_clearances + .extend(self.fetch_stale_review_clearances(&assessment).await?); + } + convergence.push(assessment); + states.push(pull_request.state); + } + Ok(FetchedPullRequests { + states, + convergence, + stale_review_clearances, + }) } /// Joins every child fetch a cancelled attempt left behind. The repository /// task calls this after cancelling an in-flight attempt, so a reported /// stop means no child is still resolving credentials, holding a /// connection, or touching shared state. + async fn drain_fetches_bounded(&self) -> Result { + let mut fetches = self.fetches.lock().await; + let had_fetches = !fetches.is_empty(); + drain_pull_request_fetches(&mut fetches).await?; + Ok(had_fetches) + } + + /// Strict shutdown settlement. A clean repository-task exit means no child + /// fetch remains able to hold resources or mutate shared freshness state. async fn drain_fetches(&self) { self.fetches.lock().await.shutdown().await; } @@ -3705,8 +4694,9 @@ impl GitHubRepositoryPoller { listed: &BTreeMap, previous: Option<&RepoWatchObservation>, cursor_generation: Option, - fetches: &mut JoinSet>, - ) -> Result, RepositoryWatchAttemptError> { + branch_heads: &[RepoWatchBranchHead], + fetches: &mut JoinSet>, + ) -> Result, RepositoryWatchAttemptError> { let mut pull_requests = Vec::with_capacity(pull_numbers.len()); let mut pending = pull_numbers.into_iter(); loop { @@ -3719,6 +4709,11 @@ impl GitHubRepositoryPoller { let previous_pull_request = previous .and_then(|observation| previous_pull_request(observation, number)) .cloned(); + let base_revision_unchanged = previous_pull_request.as_ref().is_some_and(|pull| { + previous.is_some_and(|observation| { + pull_request_base_revision_matches(observation, pull, branch_heads) + }) + }); fetches.spawn(async move { poller .fetch_or_reuse_pull_request( @@ -3726,6 +4721,7 @@ impl GitHubRepositoryPoller { listed_pull_request.as_ref(), previous_pull_request.as_ref(), cursor_generation, + base_revision_unchanged, ) .await }); @@ -3783,28 +4779,61 @@ impl GitHubRepositoryPoller { listed_pull_request: Option<&ListedPullRequest>, previous_pull_request: Option<&RepoWatchPullRequestState>, cursor_generation: Option, - ) -> Result { + base_revision_unchanged: bool, + ) -> Result { if let (Some(listed), Some(previous)) = (listed_pull_request, previous_pull_request) + && base_revision_unchanged && self.pull_request_detail_is_reusable(number, listed, previous, cursor_generation) { let reviews = self.fetch_reviews(number, Some(previous.reviews())).await?; + let mut convergence_evidence = + self.fetch_convergence_evidence(previous.context()).await?; + convergence_evidence.gating_check_inventory_quiesced = self + .gating_check_inventory_quiesced( + number, + listed, + cursor_generation, + &convergence_evidence.gating_check_inventory, + ); let threads = self.fetch_threads(number).await?; let reactions = self .fetch_reactions(number, Some(previous.reactions())) .await?; self.record_skipped_poll(number); - return reuse_pull_request(previous, reviews, threads, reactions); + self.record_gating_check_inventory( + number, + listed, + convergence_evidence.gating_check_inventory.clone(), + ); + let state = reuse_pull_request(previous, reviews, threads, reactions)?; + return Ok(FetchedPullRequest { + state, + settlement: PullRequestSettlement::Settled, + convergence_evidence, + }); } - let fetched = self + let mut fetched = self .fetch_pull_request(number, previous_pull_request) .await?; match listed_pull_request { Some(listed) => { - self.record_fetched_pull_request(number, listed, fetched.settlement); + fetched.convergence_evidence.gating_check_inventory_quiesced = self + .gating_check_inventory_quiesced( + number, + listed, + cursor_generation, + &fetched.convergence_evidence.gating_check_inventory, + ); + self.record_fetched_pull_request( + number, + listed, + fetched.settlement, + fetched.convergence_evidence.gating_check_inventory.clone(), + ); } None => self.forget_pull_request(number), } - Ok(fetched.state) + Ok(fetched) } fn pull_request_detail_is_reusable( @@ -3830,17 +4859,50 @@ impl GitHubRepositoryPoller { } } + fn gating_check_inventory_quiesced( + &self, + number: u64, + listed: &ListedPullRequest, + cursor_generation: Option, + gating_check_inventory: &[String], + ) -> bool { + self.freshness().get(&number).is_some_and(|freshness| { + freshness.published_generation == cursor_generation + && cursor_generation.is_some() + && freshness.updated_at == listed.updated_at + && freshness.head_sha == listed.head_sha + && freshness.gating_check_inventory == gating_check_inventory + }) + } + + fn record_gating_check_inventory( + &self, + number: u64, + listed: &ListedPullRequest, + gating_check_inventory: Vec, + ) { + if let Some(freshness) = self.freshness().get_mut(&number) { + freshness.updated_at = listed.updated_at.clone(); + freshness.head_sha = listed.head_sha.clone(); + freshness.gating_check_inventory = gating_check_inventory; + freshness.published_generation = None; + } + } + fn record_fetched_pull_request( &self, number: u64, listed: &ListedPullRequest, settlement: PullRequestSettlement, + gating_check_inventory: Vec, ) { self.freshness().insert( number, PullRequestFreshness { updated_at: listed.updated_at.clone(), + head_sha: listed.head_sha.clone(), settlement, + gating_check_inventory, skipped_polls: 0, published_generation: None, }, @@ -3923,6 +4985,7 @@ impl GitHubRepositoryPoller { previous_pull_request.map(RepoWatchPullRequestState::reviews), ) .await?; + let convergence_evidence = self.fetch_convergence_evidence(&context).await?; let threads = self.fetch_threads(number).await?; let reactions = self .fetch_reactions( @@ -3941,7 +5004,11 @@ impl GitHubRepositoryPoller { reactions, }) .map_err(|_| RepositoryWatchAttemptError::Normalization)?; - Ok(FetchedPullRequest { state, settlement }) + Ok(FetchedPullRequest { + state, + settlement, + convergence_evidence, + }) } async fn fetch_check_suites( @@ -4054,7 +5121,7 @@ impl GitHubRepositoryPoller { .map_err(|_| RepositoryWatchAttemptError::Normalization)?, normalize_conclusion(run.conclusion.as_deref())?, )); - } else { + } else if !is_non_gating_check_name(&run.name) { every_run_completed = false; } } @@ -4190,80 +5257,563 @@ impl GitHubRepositoryPoller { } } - async fn fetch_reactions( - &self, - number: u64, - previous: Option<&[RepoWatchReactionObservation]>, - ) -> Result, RepositoryWatchAttemptError> { - if self.signal_reviewers.is_empty() { - return Ok(Vec::new()); - } - let number_text = number.to_string(); - let mut observations = self - .fetch_reaction_pages( - &["issues", &number_text, "reactions"], - ReactionSubject::PullRequestBody, - previous, - ) - .await?; - let issue_comments = self - .fetch_comment_ids("issue-comments", &["issues", &number_text, "comments"]) - .await?; - for id in issue_comments { - let id_text = id.get().to_string(); - observations.extend( - self.fetch_reaction_pages( - &["issues", "comments", &id_text, "reactions"], - ReactionSubject::IssueComment { id }, - previous, - ) - .await?, - ); - } - let review_comments = self - .fetch_comment_ids("review-comments", &["pulls", &number_text, "comments"]) - .await?; - for id in review_comments { - let id_text = id.get().to_string(); - observations.extend( - self.fetch_reaction_pages( - &["pulls", "comments", &id_text, "reactions"], - ReactionSubject::ReviewComment { id }, - previous, - ) - .await?, - ); - } - Ok(observations) - } - - async fn fetch_comment_ids( + async fn fetch_convergence_evidence( &self, - resource_kind: &'static str, - suffix: &[&str], - ) -> Result, RepositoryWatchAttemptError> { - let mut ids = Vec::new(); + context: &PullRequestEventContext, + ) -> Result { + let (namespace, name) = self + .repository + .as_str() + .split_once('/') + .ok_or(RepositoryWatchAttemptError::Normalization)?; + let number = i64::try_from(context.number().get()) + .map_err(|_| RepositoryWatchAttemptError::Normalization)?; + let mut after: Option = None; let mut page = 1_u16; + let mut gating_check_count = 0_u64; + let mut gating_checks_settled = true; + let mut gating_check_inventory = Vec::new(); + let mut non_green_gating_checks = Vec::new(); + let mut retained_review_decision = None; + let mut retained_base_revision = None; loop { - let response = self - .conditional_json_page::>( - resource_kind, - Method::GET, - self.repository_url( - suffix, - &[ - ("per_page", PAGE_SIZE.to_string()), - ("page", page.to_string()), - ], - )?, - None, - ) - .await?; - let has_next = response.has_next_page; - for comment in response.value { - ids.push(object_id(comment.id)?); - } - if !has_next { + let body = serde_json::to_vec(&GraphQlRequest { + query: CONVERGENCE_QUERY, + variables: ThreadVariables { + namespace, + name, + number, + after: after.as_deref(), + }, + }) + .map_err(|_| RepositoryWatchAttemptError::InvalidResponse)?; + let response: GraphQlEnvelope = self + .conditional_json( + "convergence", + Method::POST, + self.graphql_url.clone(), + Some(body), + ) + .await?; + if !response.errors.is_empty() { + return Err(RepositoryWatchAttemptError::Rejected); + } + let pull_request = response + .data + .and_then(|data| data.repository) + .and_then(|repository| repository.pull_request) + .ok_or(RepositoryWatchAttemptError::InvalidResponse)?; + let _provider_mergeable_state = normalize_graphql_mergeable(&pull_request.mergeable)?; + if pull_request.head_ref_oid != context.head_sha().as_str() + || pull_request.base_ref_name != context.base_branch().as_str() + { + return Err(RepositoryWatchAttemptError::InvalidResponse); + } + if retained_base_revision + .replace(pull_request.base_ref_oid.clone()) + .is_some_and(|retained| retained != pull_request.base_ref_oid) + { + return Err(RepositoryWatchAttemptError::InvalidResponse); + } + let review_decision = + normalize_review_decision(pull_request.review_decision.as_deref())?; + if retained_review_decision + .replace(review_decision) + .is_some_and(|retained| retained != review_decision) + { + return Err(RepositoryWatchAttemptError::InvalidResponse); + } + let [commit_node] = pull_request.commits.nodes.as_slice() else { + return Err(RepositoryWatchAttemptError::InvalidResponse); + }; + let commit = &commit_node.commit; + if commit.oid != pull_request.head_ref_oid { + return Err(RepositoryWatchAttemptError::InvalidResponse); + } + let Some(rollup) = commit.status_check_rollup.as_ref() else { + if page != 1 { + return Err(RepositoryWatchAttemptError::InvalidResponse); + } + break; + }; + for check in &rollup.contexts.nodes { + if check.is_report_only() { + continue; + } + gating_check_inventory.push(check.name().to_owned()); + gating_check_count = gating_check_count + .checked_add(1) + .ok_or(RepositoryWatchAttemptError::ResourceLimit)?; + if !check.complete() { + gating_checks_settled = false; + } + if !check.green() { + non_green_gating_checks.push( + CheckRunName::try_new(check.name().to_owned()) + .map_err(|_| RepositoryWatchAttemptError::Normalization)?, + ); + } + } + if !rollup.contexts.page_info.has_next_page { + break; + } + after = rollup.contexts.page_info.end_cursor.clone(); + if after.is_none() { + return Err(RepositoryWatchAttemptError::InvalidResponse); + } + page = next_page(page)?; + } + let base_revision = CommitSha::try_new( + retained_base_revision.ok_or(RepositoryWatchAttemptError::InvalidResponse)?, + ) + .map_err(|_| RepositoryWatchAttemptError::Normalization)?; + gating_check_inventory.sort_unstable(); + Ok(FetchedConvergenceEvidence { + base_revision, + gating_checks_settled, + // Quiescence is not a property of one check-rollup read: it takes a + // second observation to say the inventory stopped growing. This + // read cannot know that, so it reports the conservative default and + // every caller replaces it with the verdict + // `GitHubRepositoryPoller::gating_check_inventory_quiesced` reads + // from the freshness the last committed cursor published. A caller + // that leaves the default in place reports every head unsettled. + gating_check_inventory_quiesced: false, + gating_check_inventory, + review_decision: retained_review_decision + .ok_or(RepositoryWatchAttemptError::InvalidResponse)?, + gating_check_count, + non_green_gating_checks, + }) + } + + async fn fetch_stale_review_clearances( + &self, + assessment: &RepoWatchConvergenceAssessment, + ) -> Result, RepositoryWatchAttemptError> { + // Mirrors the candidate rule so a head that cannot yield a candidate + // costs no provider request. The domain type re-checks every gate. + if assessment.review_decision() != RepoWatchReviewDecision::ChangesRequested + || !assessment.unresolved_threads().is_empty() + || !assessment.non_green_gating_checks().is_empty() + || !assessment.settled() + || assessment.gating_check_count() == 0 + || assessment.mergeable_state() == MergeableState::Conflicting + { + return Ok(Vec::new()); + } + let (namespace, name) = self + .repository + .as_str() + .split_once('/') + .ok_or(RepositoryWatchAttemptError::Normalization)?; + let number = i64::try_from(assessment.number().get()) + .map_err(|_| RepositoryWatchAttemptError::Normalization)?; + let mut after: Option = None; + let mut page = 1_u16; + let mut candidates = Vec::new(); + loop { + let body = serde_json::to_vec(&GraphQlRequest { + query: BLOCKING_REVIEWS_QUERY, + variables: ThreadVariables { + namespace, + name, + number, + after: after.as_deref(), + }, + }) + .map_err(|_| RepositoryWatchAttemptError::InvalidResponse)?; + let response: GraphQlEnvelope = self + .conditional_json( + "blocking-reviews", + Method::POST, + self.graphql_url.clone(), + Some(body), + ) + .await?; + if !response.errors.is_empty() { + return Err(RepositoryWatchAttemptError::Rejected); + } + let pull_request = response + .data + .and_then(|data| data.repository) + .and_then(|repository| repository.pull_request) + .ok_or(RepositoryWatchAttemptError::InvalidResponse)?; + if pull_request.head_ref_oid != assessment.head_sha().as_str() + || pull_request.base_ref_oid != assessment.base_revision().as_str() + || normalize_review_decision(pull_request.review_decision.as_deref())? + != RepoWatchReviewDecision::ChangesRequested + { + return Ok(Vec::new()); + } + for review in pull_request.latest_opinionated_reviews.nodes { + if review.state != "CHANGES_REQUESTED" { + continue; + } + let Some(author) = review.author else { + return Ok(Vec::new()); + }; + let reviewer = RepoWatchAuthorLogin::try_new(author.login) + .map_err(|_| RepositoryWatchAttemptError::Normalization)?; + let Some(commit) = review.commit else { + return Ok(Vec::new()); + }; + let reviewed_head_sha = CommitSha::try_new(commit.oid) + .map_err(|_| RepositoryWatchAttemptError::Normalization)?; + if &reviewed_head_sha == assessment.head_sha() { + return Ok(Vec::new()); + } + candidates.push( + RepoWatchStaleReviewClearanceCandidate::try_new( + assessment, + review.id, + reviewer, + reviewed_head_sha, + ) + .map_err(|_| RepositoryWatchAttemptError::InvalidResponse)?, + ); + } + if !pull_request + .latest_opinionated_reviews + .page_info + .has_next_page + { + candidates.sort_by(|left, right| left.review_node_id().cmp(right.review_node_id())); + return Ok(candidates); + } + after = pull_request.latest_opinionated_reviews.page_info.end_cursor; + if after.is_none() { + return Err(RepositoryWatchAttemptError::InvalidResponse); + } + page = next_page(page)?; + } + } + + /// Re-reads the provider immediately before a dismissal and reports + /// whether the planned clearance still holds against live evidence. + /// + /// `cursor_generation` is the generation the poll that raised this + /// candidate committed, and the freshness it published is what proves the + /// gating-check inventory has stood still: a candidate is only admissible + /// when the inventory this re-read observes is the one that committed + /// generation recorded for the same head and update stamp. + async fn revalidate_stale_review_clearance( + &self, + clearance: &RepoWatchPlannedStaleReviewClearance, + cursor_generation: RepoWatchCursorGeneration, + ) -> Result { + let number_text = clearance.number().get().to_string(); + let detail: PullResponse = self + .conditional_json( + "pull-clearance-revalidation", + Method::GET, + self.repository_url(&["pulls", &number_text], &[])?, + None, + ) + .await?; + if detail.number != clearance.number().get() + || normalize_lifecycle(&detail)? != RepoWatchPullRequestLifecycle::Open + { + return Ok(false); + } + let head_sha = CommitSha::try_new(detail.head.sha.clone()) + .map_err(|_| RepositoryWatchAttemptError::Normalization)?; + if &head_sha != clearance.current_head_sha() { + return Ok(false); + } + let base_branch = BranchName::try_new(detail.base.reference.clone()) + .map_err(|_| RepositoryWatchAttemptError::Normalization)?; + if &base_branch != clearance.base_branch() { + return Ok(false); + } + let mergeable_state = match detail.mergeable { + Some(true) => MergeableState::Mergeable, + Some(false) => MergeableState::Conflicting, + None => MergeableState::Unknown, + }; + let context = normalize_pull_request_context(&detail, head_sha.clone(), None)?; + let mut evidence = self.fetch_convergence_evidence(&context).await?; + // The same quiescence rule the polling path applies, against the same + // published freshness. Here the two observations being compared are the + // committed poll that raised this candidate and this pre-dismissal + // re-read, so a gating check that appeared in between leaves the head + // unsettled and the review undismissed until a later poll sees the + // inventory hold still. + let listed = ListedPullRequest { + updated_at: detail.updated_at.clone(), + head_sha: head_sha.clone(), + }; + evidence.gating_check_inventory_quiesced = self.gating_check_inventory_quiesced( + clearance.number().get(), + &listed, + Some(cursor_generation), + &evidence.gating_check_inventory, + ); + if &evidence.base_revision != clearance.base_revision() + || evidence.review_decision != RepoWatchReviewDecision::ChangesRequested + || !evidence.non_green_gating_checks.is_empty() + || mergeable_state == MergeableState::Conflicting + { + return Ok(false); + } + let unresolved_threads = self + .fetch_threads(clearance.number().get()) + .await? + .into_iter() + .filter(|thread| thread.state() == RepoWatchThreadState::Open) + .map(|thread| thread.thread().clone()) + .collect::>(); + if !unresolved_threads.is_empty() { + return Ok(false); + } + let assessment = + RepoWatchConvergenceAssessment::try_new(RepoWatchConvergenceAssessmentInput { + number: clearance.number(), + head_sha: clearance.current_head_sha().clone(), + base_branch: clearance.base_branch().clone(), + base_revision: evidence.base_revision, + mergeable_state, + // Clearance candidacy does consult this, and refuses every + // unsettled head, so it is computed from the same evidence the + // polling path uses: finished exact-head checks, an inventory + // quiesced against the published freshness above, and a decided + // mergeable state. + settled: evidence.gating_checks_settled + && evidence.gating_check_inventory_quiesced + && mergeable_state != MergeableState::Unknown, + review_decision: evidence.review_decision, + unresolved_threads, + gating_check_count: evidence.gating_check_count, + non_green_gating_checks: evidence.non_green_gating_checks, + }) + .map_err(|_| RepositoryWatchAttemptError::Normalization)?; + let candidates = self.fetch_stale_review_clearances(&assessment).await?; + Ok(candidates.iter().any(|candidate| { + candidate.review_node_id() == clearance.review_node_id() + && candidate.reviewed_head_sha() == clearance.reviewed_head_sha() + })) + } + + async fn dismiss_review_node( + &self, + input: DismissReviewInput<'_>, + ) -> Result<(), RepositoryWatchAttemptError> { + let DismissReviewInput { + review_node_id, + dismissal_message, + } = input; + let body = serde_json::to_vec(&GraphQlRequest { + query: DISMISS_REVIEW_MUTATION, + variables: DismissReviewVariables { + review: review_node_id, + message: dismissal_message, + }, + }) + .map_err(|_| RepositoryWatchAttemptError::InvalidResponse)?; + let response: GraphQlEnvelope = self + .conditional_json( + "dismiss-review", + Method::POST, + self.graphql_url.clone(), + Some(body), + ) + .await?; + if !response.errors.is_empty() { + return Err(RepositoryWatchAttemptError::Rejected); + } + let review = response + .data + .and_then(|data| data.dismiss_pull_request_review) + .and_then(|payload| payload.pull_request_review) + .ok_or(RepositoryWatchAttemptError::InvalidResponse)?; + if review.id != review_node_id || review.state != "DISMISSED" { + return Err(RepositoryWatchAttemptError::InvalidResponse); + } + Ok(()) + } + + async fn observe_stale_review_clearance( + &self, + clearance: &RepoWatchPlannedStaleReviewClearance, + ) -> Result { + let mut after: Option = None; + let mut page = 1_u16; + loop { + let body = serde_json::to_vec(&GraphQlRequest { + query: REVIEW_CLEARANCE_STATE_QUERY, + variables: ReviewNodeVariables { + review: clearance.review_node_id(), + after: after.as_deref(), + }, + }) + .map_err(|_| RepositoryWatchAttemptError::InvalidResponse)?; + let response: GraphQlEnvelope = self + .conditional_json( + "review-clearance-state", + Method::POST, + self.graphql_url.clone(), + Some(body), + ) + .await?; + if !response.errors.is_empty() { + return Err(RepositoryWatchAttemptError::Rejected); + } + let review = response + .data + .and_then(|data| data.node) + .ok_or(RepositoryWatchAttemptError::InvalidResponse)?; + if review.id != clearance.review_node_id() + || review.pull_request.number != clearance.number().get() + { + return Err(RepositoryWatchAttemptError::InvalidResponse); + } + let provider_state = normalize_observed_review_state(&review.state)?; + if let Some(outcome) = terminal_clearance_outcome(provider_state) { + return Ok(StaleReviewClearanceObservation::Terminal { + outcome, + provider_state, + }); + } + match review.pull_request.state.as_str() { + "OPEN" => {} + "CLOSED" | "MERGED" => { + return Ok(StaleReviewClearanceObservation::Terminal { + outcome: RepoWatchStaleReviewClearanceOutcome::ClearedElsewhere, + provider_state, + }); + } + _ => return Err(RepositoryWatchAttemptError::InvalidResponse), + } + if review.pull_request.head_ref_oid != clearance.current_head_sha().as_str() + || review.pull_request.base_ref_name != clearance.base_branch().as_str() + || review.pull_request.base_ref_oid != clearance.base_revision().as_str() + { + return Ok(StaleReviewClearanceObservation::Terminal { + outcome: RepoWatchStaleReviewClearanceOutcome::Superseded, + provider_state, + }); + } + if review + .commit + .as_ref() + .is_some_and(|commit| commit.oid != clearance.reviewed_head_sha().as_str()) + { + return Err(RepositoryWatchAttemptError::InvalidResponse); + } + if normalize_review_decision(review.pull_request.review_decision.as_deref())? + != RepoWatchReviewDecision::ChangesRequested + { + return Ok(StaleReviewClearanceObservation::Terminal { + outcome: RepoWatchStaleReviewClearanceOutcome::ClearedElsewhere, + provider_state, + }); + } + if review + .pull_request + .latest_opinionated_reviews + .nodes + .iter() + .any(|candidate| candidate.id == clearance.review_node_id()) + { + return Ok(StaleReviewClearanceObservation::StillBlocking); + } + if !review + .pull_request + .latest_opinionated_reviews + .page_info + .has_next_page + { + return Ok(StaleReviewClearanceObservation::Terminal { + outcome: RepoWatchStaleReviewClearanceOutcome::ClearedElsewhere, + provider_state, + }); + } + after = review + .pull_request + .latest_opinionated_reviews + .page_info + .end_cursor; + if after.is_none() { + return Err(RepositoryWatchAttemptError::InvalidResponse); + } + page = next_page(page)?; + } + } + + async fn fetch_reactions( + &self, + number: u64, + previous: Option<&[RepoWatchReactionObservation]>, + ) -> Result, RepositoryWatchAttemptError> { + if self.signal_reviewers.is_empty() { + return Ok(Vec::new()); + } + let number_text = number.to_string(); + let mut observations = self + .fetch_reaction_pages( + &["issues", &number_text, "reactions"], + ReactionSubject::PullRequestBody, + previous, + ) + .await?; + let issue_comments = self + .fetch_comment_ids("issue-comments", &["issues", &number_text, "comments"]) + .await?; + for id in issue_comments { + let id_text = id.get().to_string(); + observations.extend( + self.fetch_reaction_pages( + &["issues", "comments", &id_text, "reactions"], + ReactionSubject::IssueComment { id }, + previous, + ) + .await?, + ); + } + let review_comments = self + .fetch_comment_ids("review-comments", &["pulls", &number_text, "comments"]) + .await?; + for id in review_comments { + let id_text = id.get().to_string(); + observations.extend( + self.fetch_reaction_pages( + &["pulls", "comments", &id_text, "reactions"], + ReactionSubject::ReviewComment { id }, + previous, + ) + .await?, + ); + } + Ok(observations) + } + + async fn fetch_comment_ids( + &self, + resource_kind: &'static str, + suffix: &[&str], + ) -> Result, RepositoryWatchAttemptError> { + let mut ids = Vec::new(); + let mut page = 1_u16; + loop { + let response = self + .conditional_json_page::>( + resource_kind, + Method::GET, + self.repository_url( + suffix, + &[ + ("per_page", PAGE_SIZE.to_string()), + ("page", page.to_string()), + ], + )?, + None, + ) + .await?; + let has_next = response.has_next_page; + for comment in response.value { + ids.push(object_id(comment.id)?); + } + if !has_next { return Ok(ids); } page = next_page(page)?; @@ -5011,6 +6561,30 @@ impl PollCache { } } +fn pull_request_base_revision<'a>( + observation: &'a RepoWatchObservation, + pull_request: &RepoWatchPullRequestState, +) -> Option<&'a CommitSha> { + observation + .state() + .branch_heads() + .iter() + .find(|head| head.branch() == pull_request.context().base_branch()) + .map(RepoWatchBranchHead::head) +} + +fn pull_request_base_revision_matches( + observation: &RepoWatchObservation, + pull_request: &RepoWatchPullRequestState, + branch_heads: &[RepoWatchBranchHead], +) -> bool { + pull_request_base_revision(observation, pull_request) + == branch_heads + .iter() + .find(|head| head.branch() == pull_request.context().base_branch()) + .map(RepoWatchBranchHead::head) +} + fn reuse_pull_request( previous: &RepoWatchPullRequestState, reviews: Vec, @@ -5169,6 +6743,19 @@ fn normalize_review_state(state: &str) -> Result Result { + match state { + "APPROVED" => Ok(RepoWatchObservedReviewState::Approved), + "CHANGES_REQUESTED" => Ok(RepoWatchObservedReviewState::ChangesRequested), + "COMMENTED" => Ok(RepoWatchObservedReviewState::Commented), + "DISMISSED" => Ok(RepoWatchObservedReviewState::Dismissed), + "PENDING" => Ok(RepoWatchObservedReviewState::Pending), + _ => Err(RepositoryWatchAttemptError::InvalidResponse), + } +} + #[derive(Clone, Deserialize)] struct PullNumberResponse { number: u64, @@ -5185,6 +6772,10 @@ struct ListedPullHeadResponse { struct PullResponse { number: u64, state: String, + // The same stamp the pulls listing carries, so a detail read can be + // compared against the freshness a committed poll recorded from the + // listing. + updated_at: String, merged_at: Option, mergeable: Option, head: PullReferenceResponse, @@ -5345,6 +6936,23 @@ struct ThreadVariables<'a> { after: Option<&'a str>, } +#[derive(Serialize)] +struct DismissReviewVariables<'a> { + review: &'a str, + message: &'a str, +} + +struct DismissReviewInput<'a> { + review_node_id: &'a str, + dismissal_message: &'a str, +} + +#[derive(Serialize)] +struct ReviewNodeVariables<'a> { + review: &'a str, + after: Option<&'a str>, +} + #[derive(Clone, Deserialize)] struct GraphQlEnvelope { data: Option, @@ -5386,6 +6994,269 @@ struct ThreadResponse { is_resolved: bool, } +#[derive(Clone, Deserialize)] +struct ConvergenceData { + repository: Option, +} + +#[derive(Clone, Deserialize)] +struct ConvergenceRepository { + #[serde(rename = "pullRequest")] + pull_request: Option, +} + +#[derive(Clone, Deserialize)] +struct ConvergencePullRequest { + #[serde(rename = "headRefOid")] + head_ref_oid: String, + #[serde(rename = "baseRefName")] + base_ref_name: String, + #[serde(rename = "baseRefOid")] + base_ref_oid: String, + mergeable: String, + #[serde(rename = "reviewDecision")] + review_decision: Option, + commits: ConvergenceCommitConnection, +} + +#[derive(Clone, Deserialize)] +struct ConvergenceCommitConnection { + nodes: Vec, +} + +#[derive(Clone, Deserialize)] +struct ConvergenceCommitNode { + commit: ConvergenceCommit, +} + +#[derive(Clone, Deserialize)] +struct ConvergenceCommit { + oid: String, + #[serde(rename = "statusCheckRollup")] + status_check_rollup: Option, +} + +#[derive(Clone, Deserialize)] +struct ConvergenceCheckRollup { + contexts: ConvergenceCheckConnection, +} + +#[derive(Clone, Deserialize)] +struct ConvergenceCheckConnection { + nodes: Vec, + #[serde(rename = "pageInfo")] + page_info: PageInfo, +} + +#[derive(Clone, Deserialize)] +struct BlockingReviewData { + repository: Option, +} + +#[derive(Clone, Deserialize)] +struct BlockingReviewRepository { + #[serde(rename = "pullRequest")] + pull_request: Option, +} + +#[derive(Clone, Deserialize)] +struct BlockingReviewPullRequest { + #[serde(rename = "headRefOid")] + head_ref_oid: String, + #[serde(rename = "baseRefOid")] + base_ref_oid: String, + #[serde(rename = "reviewDecision")] + review_decision: Option, + #[serde(rename = "latestOpinionatedReviews")] + latest_opinionated_reviews: BlockingReviewConnection, +} + +#[derive(Clone, Deserialize)] +struct BlockingReviewConnection { + nodes: Vec, + #[serde(rename = "pageInfo")] + page_info: PageInfo, +} + +#[derive(Clone, Deserialize)] +struct BlockingReviewNode { + id: String, + state: String, + author: Option, + commit: Option, +} + +#[derive(Clone, Deserialize)] +struct BlockingReviewAuthor { + login: String, +} + +#[derive(Clone, Deserialize)] +struct BlockingReviewCommit { + oid: String, +} + +#[derive(Clone, Deserialize)] +struct DismissReviewData { + #[serde(rename = "dismissPullRequestReview")] + dismiss_pull_request_review: Option, +} + +#[derive(Clone, Deserialize)] +struct DismissReviewPayload { + #[serde(rename = "pullRequestReview")] + pull_request_review: Option, +} + +#[derive(Clone, Deserialize)] +struct DismissedReview { + id: String, + state: String, +} + +#[derive(Clone, Deserialize)] +struct ReviewClearanceStateData { + node: Option, +} + +#[derive(Clone, Deserialize)] +struct ReviewClearanceState { + id: String, + state: String, + commit: Option, + #[serde(rename = "pullRequest")] + pull_request: ReviewClearancePullRequest, +} + +#[derive(Clone, Deserialize)] +struct ReviewClearancePullRequest { + number: u64, + state: String, + #[serde(rename = "headRefOid")] + head_ref_oid: String, + #[serde(rename = "baseRefName")] + base_ref_name: String, + #[serde(rename = "baseRefOid")] + base_ref_oid: String, + #[serde(rename = "reviewDecision")] + review_decision: Option, + #[serde(rename = "latestOpinionatedReviews")] + latest_opinionated_reviews: ReviewClearanceReviewConnection, +} + +#[derive(Clone, Deserialize)] +struct ReviewClearanceReviewConnection { + nodes: Vec, + #[serde(rename = "pageInfo")] + page_info: PageInfo, +} + +#[derive(Clone, Deserialize)] +struct ReviewClearanceReviewNode { + id: String, +} + +enum StaleReviewClearanceObservation { + StillBlocking, + Terminal { + outcome: RepoWatchStaleReviewClearanceOutcome, + provider_state: RepoWatchObservedReviewState, + }, +} + +const fn terminal_clearance_outcome( + provider_state: RepoWatchObservedReviewState, +) -> Option { + match provider_state { + RepoWatchObservedReviewState::Dismissed => { + Some(RepoWatchStaleReviewClearanceOutcome::AlreadyDismissed) + } + RepoWatchObservedReviewState::ChangesRequested => None, + RepoWatchObservedReviewState::Approved + | RepoWatchObservedReviewState::Commented + | RepoWatchObservedReviewState::Pending => { + Some(RepoWatchStaleReviewClearanceOutcome::ClearedElsewhere) + } + } +} + +#[derive(Clone, Deserialize)] +#[serde(tag = "__typename")] +enum ConvergenceCheck { + CheckRun { + name: String, + status: String, + conclusion: Option, + }, + StatusContext { + context: String, + state: String, + }, +} + +impl ConvergenceCheck { + fn name(&self) -> &str { + match self { + Self::CheckRun { name, .. } => name, + Self::StatusContext { context, .. } => context, + } + } + + fn is_report_only(&self) -> bool { + is_non_gating_check_name(self.name()) + } + + fn complete(&self) -> bool { + match self { + Self::CheckRun { status, .. } => status == "COMPLETED", + Self::StatusContext { state, .. } => state != "PENDING", + } + } + + fn green(&self) -> bool { + match self { + Self::CheckRun { + status, conclusion, .. + } => { + status == "COMPLETED" + && matches!( + conclusion.as_deref(), + Some("SUCCESS" | "SKIPPED" | "NEUTRAL") + ) + } + Self::StatusContext { state, .. } => state == "SUCCESS", + } + } +} + +fn is_non_gating_check_name(name: &str) -> bool { + let name = name.to_ascii_lowercase(); + NON_GATING_CHECK_NAME_MARKERS + .iter() + .any(|marker| name.contains(marker)) +} + +fn normalize_graphql_mergeable(value: &str) -> Result { + match value { + "MERGEABLE" => Ok(MergeableState::Mergeable), + "CONFLICTING" => Ok(MergeableState::Conflicting), + "UNKNOWN" => Ok(MergeableState::Unknown), + _ => Err(RepositoryWatchAttemptError::InvalidResponse), + } +} + +fn normalize_review_decision( + value: Option<&str>, +) -> Result { + match value { + None => Ok(RepoWatchReviewDecision::None), + Some("APPROVED") => Ok(RepoWatchReviewDecision::Approved), + Some("REVIEW_REQUIRED") => Ok(RepoWatchReviewDecision::ReviewRequired), + Some("CHANGES_REQUESTED") => Ok(RepoWatchReviewDecision::ChangesRequested), + Some(_) => Err(RepositoryWatchAttemptError::InvalidResponse), + } +} + #[derive(Clone, Deserialize)] struct PageInfo { #[serde(rename = "hasNextPage")] @@ -5420,30 +7291,34 @@ mod tests { }; use super::{ - CheckConclusion, ChecksOutcome, EntityTag, FileCredentialAccess, GitHubRepositoryPoller, - ListedPullRequest, MAX_CACHED_WIRE_BYTES, MAX_CHECK_SUITES_PER_COMMIT_CHECK_RUN_SEARCH, - MAX_CONCURRENT_PULL_REQUEST_FETCHES, MAX_CONSECUTIVE_SKIPPED_PULL_REQUEST_POLLS, - MAX_POLL_WIRE_BYTES, MergeableState, PAGE_SIZE, PollAttemptWait, PollCache, - PullRequestSettlement, PullResponse, ReactionContent, RepoWatchAuthorLogin, - RepoWatchBranchHead, RepoWatchCursorGeneration, RepoWatchObservation, - RepoWatchPullRequestLifecycle, RepoWatchReactionObservation, RepoWatchReviewObservation, - RepoWatchThreadState, RepoWatchWorkflowRunAttempt, RepoWatchWorkflowRunObservation, - RepositorySlug, RepositoryWatchAttemptError, RepositoryWatchChildExit, + CheckConclusion, ChecksOutcome, ConvergenceCheck, EntityTag, FileCredentialAccess, + GitHubRepositoryPoller, ListedPullRequest, MAX_CACHED_WIRE_BYTES, + MAX_CHECK_SUITES_PER_COMMIT_CHECK_RUN_SEARCH, MAX_CONCURRENT_PULL_REQUEST_FETCHES, + MAX_CONSECUTIVE_SKIPPED_PULL_REQUEST_POLLS, MAX_POLL_WIRE_BYTES, MergeableState, PAGE_SIZE, + PollAttemptWait, PollCache, PreparedTargetedRefresh, PullRequestSettlement, PullResponse, + ReactionContent, RepoWatchAuthorLogin, RepoWatchBranchHead, RepoWatchConvergenceAssessment, + RepoWatchConvergenceAssessmentInput, RepoWatchCursorGeneration, RepoWatchObservation, + RepoWatchPullRequestLifecycle, RepoWatchReactionObservation, RepoWatchReviewDecision, + RepoWatchReviewObservation, RepoWatchStaleReviewClearanceCandidate, RepoWatchThreadState, + RepoWatchWorkflowRunAttempt, RepoWatchWorkflowRunObservation, RepositorySlug, + RepositoryWatchAttemptError, RepositoryWatchChildExit, RepositoryWatchRuntimeConstructionError, RepositoryWatchRuntimeError, RepositoryWatchTask, RepositoryWatchWake, ResourceKey, ReviewState, TargetedPollOutcome, TargetedPullRequest, - Url, UuidV7RepoWatchEventIdGenerator, WEBHOOK_DRAIN_RETRY_DELAY, - WEBHOOK_DRAIN_RETRY_MAX_DELAY, WEBHOOK_PENDING_PAGE_SIZE, WebhookAttemptOutcome, - WebhookDrain, WebhookDrainOutcome, WebhookDrainProgress, WebhookDrainRetry, - WebhookPayloadPurgeSchedule, WebhookPollInterrupt, WorkflowName, WorkflowResponse, + TargetedRefreshSettlement, Url, UuidV7RepoWatchEventIdGenerator, + WEBHOOK_DRAIN_ATTEMPT_TIMEOUT, WEBHOOK_DRAIN_RETRY_DELAY, WEBHOOK_DRAIN_RETRY_MAX_DELAY, + WEBHOOK_PENDING_PAGE_SIZE, WebhookAttemptOutcome, WebhookAttemptPhase, WebhookDrain, + WebhookDrainOutcome, WebhookDrainProgress, WebhookDrainRetry, WebhookPayloadPurgeSchedule, + WebhookPollInterrupt, WebhookShadowBaseline, WorkflowName, WorkflowResponse, await_poll_or_interrupt, commit_check_run_search_is_complete, derive_repo_watch_events, dispatch_context_json, initial_poll_deadline, inspect_webhook_drain, next_cadence_deadline, next_repository_wake, normalize_checks_outcome, normalize_pull_request_context, object_id, observe_webhook_work_before_drain, owed_dispatch_context_json_parts, - repository_reconciliation_quantum_exhausted, rule_activation_error, run_until_shutdown, - supervise_repository_tasks, targeted_pull_requests, + record_dispatch_start_nudge_outcome, repository_reconciliation_quantum_exhausted, + rule_activation_error, run_until_shutdown, supervise_repository_tasks, + targeted_pull_requests, }; use signalbox_application::{ - InProcessEligibilityWorkSource, RepoWatchEventIdentityFrontierV1, + EligibilityNudgeOutcome, InProcessEligibilityWorkSource, RepoWatchEventIdentityFrontierV1, RepoWatchTargetedRefreshV1, }; use signalbox_domain::{ @@ -5456,7 +7331,11 @@ mod tests { use signalbox_persistence::{ disposable_postgres_server_args, disposable_postgres_state_tmpfs_from_example, disposable_test_container_labels, local_test_connection_options, migrate, - repo_watch::{PostgresRepoWatchStore, RepoWatchCommitRequest, RepoWatchCursorCandidate}, + repo_watch::{ + PostgresRepoWatchStore, RepoWatchCommitRequest, RepoWatchCursorCandidate, + RepoWatchPlannedStaleReviewClearanceFixture, RepoWatchStaleReviewClearanceClaimToken, + RepoWatchStaleReviewClearanceId, + }, repo_watch_dispatch::{PostgresRepoWatchDispatchStore, RepoWatchDispatchRepositoryError}, repo_watch_webhook::{ PostgresRepoWatchWebhookStore, RepoWatchWebhookAdmission, RepoWatchWebhookDeliveryKey, @@ -5571,6 +7450,9 @@ mod tests { const QUEUED_CHECK_SUITE_UPDATED_AT: &str = "2026-08-03T12:35:18Z"; const WORKFLOW_NAME: &str = "CI"; const REVIEWER: &str = "signal-reviewer"; + const STALE_REVIEW_NODE_ID: &str = "PRR_fixture_stale"; + const STALE_REVIEW_HEAD_SHA: &str = "eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee"; + const DISMISSAL_MESSAGE: &str = "Every finding is resolved on the current head."; const REVIEW_THREAD: &str = "PRRT_fixture_open"; const RESOLVED_REVIEW_THREAD: &str = "PRRT_fixture_resolved"; const PULL_NUMBERS: [u64; 1] = [PULL_NUMBER]; @@ -5729,8 +7611,18 @@ mod tests { pool: &PgPool, rest_base: Url, ) -> Result> { - let repository = RepositorySlug::try_new(WATCHED_REPOSITORY.to_owned())?; let observation = complete_typed_observation().await; + task_against(pool, rest_base, observation).await + } + + /// The same task fixture over a caller-chosen committed cursor, for a test + /// whose behavior depends on what that cursor observes. + async fn task_against( + pool: &PgPool, + rest_base: Url, + observation: RepoWatchObservation, + ) -> Result> { + let repository = RepositorySlug::try_new(WATCHED_REPOSITORY.to_owned())?; let store = PostgresRepoWatchStore::new(pool.clone()); store .commit( @@ -5753,6 +7645,15 @@ mod tests { webhook_nudge: None, webhook_shadow: None, webhook_shadow_superseded: false, + webhook_shadow_supersession_epoch: 0, + webhook_projected_terminal_in_flight: None, + webhook_dispatch_in_flight: false, + webhook_targeted_completion: None, + webhook_terminal_ambiguous: None, + webhook_drain_first_failure: None, + webhook_drain_projection_failure: None, + webhook_drain_timed_out: false, + webhook_attempt_phase: WebhookAttemptPhase::BeforeDrain, repository, interval: POLL_INTERVAL, poller, @@ -5764,7 +7665,9 @@ mod tests { eligibility_nudge, webhook_store: PostgresRepoWatchWebhookStore::new(pool.clone()), webhook_work: None, + startup_webhook_retry: None, reconciliation_quantum: None, + webhook_drain_work_budget: None, payload_purge: WebhookPayloadPurgeSchedule::starting_now(), rules_activated: true, }, @@ -5773,21 +7676,29 @@ mod tests { } async fn wait_for_webhook_projection_wedge(store: &PostgresRepoWatchWebhookStore) { - let wait = async { - loop { - if store - .projection_wedge_is_reached() - .await - .expect("the fixture can inspect the wedge") - { - return; - } - sleep(Duration::from_millis(10)).await; + let deadline = std::time::Instant::now() + SCRIPTED_SERVER_TIMEOUT; + loop { + if store + .projection_wedge_is_reached() + .await + .expect("the fixture can inspect the wedge") + { + return; + } + assert!( + std::time::Instant::now() < deadline, + "the first webhook projection reaches its deliberate wedge" + ); + tokio::task::yield_now().await; + } + } + + fn keep_paused_clock_runnable() -> JoinHandle<()> { + tokio::spawn(async { + loop { + tokio::task::yield_now().await; } - }; - tokio::time::timeout(SCRIPTED_SERVER_TIMEOUT, wait) - .await - .expect("the first webhook projection reaches its deliberate wedge"); + }) } #[derive(Clone, Default)] @@ -5868,6 +7779,7 @@ mod tests { serde_json::json!({ "number": PULL_NUMBER, "state": "open", + "updated_at": PULL_UPDATED_AT, "merged_at": null, "mergeable": false, "head": { @@ -5896,6 +7808,16 @@ mod tests { detail.to_string() } + /// The fixture pull request with mergeability decided in its favor, which + /// a clearance revalidation needs: the shared fixture reports + /// `CONFLICTING`, and that alone refuses every dismissal. + fn mergeable_pull_detail() -> String { + let mut detail = serde_json::from_str::(&pull_detail()) + .expect("fixture pull detail is JSON"); + detail["mergeable"] = serde_json::Value::Bool(true); + detail.to_string() + } + fn pull_detail_without_head_repository() -> String { let mut detail = serde_json::from_str::(&pull_detail()) .expect("fixture pull detail is JSON"); @@ -6132,6 +8054,148 @@ mod tests { .to_string() } + fn convergence() -> String { + convergence_with_mergeability("CONFLICTING") + } + + /// Convergence evidence for a head whose only remaining blocker is the + /// aggregate review decision: one complete, green, gating check and no + /// other. This is the evidence a stale-review clearance is allowed to act + /// on, so it is what a revalidation must be able to read back. + fn review_only_blocked_convergence() -> String { + serde_json::json!({ + "data": { + "repository": { + "pullRequest": { + "headRefOid": HEAD_SHA, + "baseRefName": BASE_BRANCH, + "baseRefOid": BASE_SHA, + "mergeable": "MERGEABLE", + "reviewDecision": "CHANGES_REQUESTED", + "commits": { + "nodes": [{ + "commit": { + "oid": HEAD_SHA, + "statusCheckRollup": { + "contexts": { + "nodes": [{ + "__typename": "CheckRun", + "name": CHECK_RUN_NAME, + "status": "COMPLETED", + "conclusion": "SUCCESS" + }], + "pageInfo": { + "hasNextPage": false, + "endCursor": null + } + } + } + } + }] + } + } + } + } + }) + .to_string() + } + + fn convergence_with_mergeability(mergeable: &str) -> String { + serde_json::json!({ + "data": { + "repository": { + "pullRequest": { + "headRefOid": HEAD_SHA, + "baseRefName": BASE_BRANCH, + "baseRefOid": BASE_SHA, + "mergeable": mergeable, + "reviewDecision": "APPROVED", + "commits": { + "nodes": [{ + "commit": { + "oid": HEAD_SHA, + "statusCheckRollup": { + "contexts": { + "nodes": [ + { + "__typename": "CheckRun", + "name": CHECK_RUN_NAME, + "status": "COMPLETED", + "conclusion": "FAILURE" + }, + { + "__typename": "CheckRun", + "name": "coverage (report only)", + "status": "IN_PROGRESS", + "conclusion": null + }, + { + "__typename": "StatusContext", + "context": "CodeRabbit", + "state": "ERROR" + } + ], + "pageInfo": { + "hasNextPage": false, + "endCursor": null + } + } + } + } + }] + } + } + } + } + }) + .to_string() + } + + fn blocking_reviews(reviewed_head_sha: &str) -> String { + blocking_reviews_by(REVIEWER, reviewed_head_sha) + } + + fn blocking_reviews_by(reviewer: &str, reviewed_head_sha: &str) -> String { + serde_json::json!({ + "data": { + "repository": { + "pullRequest": { + "headRefOid": HEAD_SHA, + "baseRefOid": BASE_SHA, + "reviewDecision": "CHANGES_REQUESTED", + "latestOpinionatedReviews": { + "nodes": [{ + "id": STALE_REVIEW_NODE_ID, + "state": "CHANGES_REQUESTED", + "author": { "login": reviewer }, + "commit": { "oid": reviewed_head_sha } + }], + "pageInfo": { + "hasNextPage": false, + "endCursor": null + } + } + } + } + } + }) + .to_string() + } + + fn dismissed_review(review_node_id: &str) -> String { + serde_json::json!({ + "data": { + "dismissPullRequestReview": { + "pullRequestReview": { + "id": review_node_id, + "state": "DISMISSED" + } + } + } + }) + .to_string() + } + fn pull_reactions() -> String { serde_json::json!([ { @@ -6381,6 +8445,7 @@ mod tests { struct ScriptedResponse { method: &'static str, target: String, + request_body_marker: Option, validator: Option<&'static str>, status: &'static str, entity_tag: Option<&'static str>, @@ -6394,6 +8459,7 @@ mod tests { Self { method: "GET", target: target.0, + request_body_marker: None, validator: None, status: "200 OK", entity_tag: Some(ENTITY_TAG), @@ -6407,6 +8473,7 @@ mod tests { Self { method: "GET", target: target.0, + request_body_marker: None, validator: None, status: "200 OK", entity_tag: Some(ENTITY_TAG), @@ -6420,6 +8487,7 @@ mod tests { Self { method: "GET", target: target.0, + request_body_marker: None, validator: Some(ENTITY_TAG), status: "200 OK", entity_tag: Some(ENTITY_TAG), @@ -6433,6 +8501,7 @@ mod tests { Self { method: "GET", target: target.0, + request_body_marker: None, validator: None, status: "404 Not Found", entity_tag: None, @@ -6446,6 +8515,7 @@ mod tests { Self { method: "GET", target: target.0, + request_body_marker: None, validator: None, status: "403 Forbidden", entity_tag: None, @@ -6459,6 +8529,7 @@ mod tests { Self { method: "GET", target: target.0, + request_body_marker: None, validator: Some(ENTITY_TAG), status: "304 Not Modified", entity_tag: None, @@ -6477,6 +8548,7 @@ mod tests { Self { method: "POST", target: target.0, + request_body_marker: None, validator: None, status: "200 OK", entity_tag: None, @@ -6485,6 +8557,11 @@ mod tests { delay: Duration::ZERO, } } + + fn matching_request_body(mut self, marker: String) -> Self { + self.request_body_marker = Some(marker); + self + } } struct ConcurrentScriptedState { @@ -6582,6 +8659,10 @@ mod tests { .iter() .position(|response| { start_line == format!("{} {} HTTP/1.1", response.method, response.target) + && response + .request_body_marker + .as_ref() + .is_none_or(|marker| request.contains(marker)) }) .map(|position| responses.remove(position)) }; @@ -6638,14 +8719,34 @@ mod tests { .await .expect("scripted request can be read"); request.extend_from_slice(&chunk[..read]); - if request.windows(4).any(|window| window == b"\r\n\r\n") { + if scripted_request_is_complete(&request) { break; } - assert_ne!(read, 0, "request headers must be complete"); + assert_ne!(read, 0, "request body must be complete"); } String::from_utf8(request).expect("request headers are UTF-8") } + fn scripted_request_is_complete(request: &[u8]) -> bool { + let Some(header_end) = request.windows(4).position(|window| window == b"\r\n\r\n") else { + return false; + }; + let header_end = header_end + 4; + let headers = std::str::from_utf8(&request[..header_end]) + .expect("scripted request headers are UTF-8"); + let content_length = headers + .lines() + .filter_map(|line| line.split_once(':')) + .find(|(name, _)| name.eq_ignore_ascii_case("content-length")) + .map_or(0, |(_, value)| { + value + .trim() + .parse::() + .expect("scripted request content length is valid") + }); + request.len() >= header_end + content_length + } + async fn write_response(stream: &mut TcpStream, response: &ScriptedResponse) { sleep(response.delay).await; let entity_tag = response @@ -6764,6 +8865,10 @@ mod tests { RequestTarget(PULLS_TARGET.to_owned()), ResponseBody(pulls_with_one()), ), + ScriptedResponse::ok( + RequestTarget(BRANCHES_TARGET.to_owned()), + ResponseBody(branches()), + ), ScriptedResponse::ok( RequestTarget(PULL_DETAIL_TARGET.to_owned()), ResponseBody(pull_detail()), @@ -6780,6 +8885,10 @@ mod tests { RequestTarget(REVIEWS_TARGET.to_owned()), ResponseBody(reviews()), ), + ScriptedResponse::post( + RequestTarget(THREADS_TARGET.to_owned()), + ResponseBody(convergence()), + ), ScriptedResponse::post( RequestTarget(THREADS_TARGET.to_owned()), ResponseBody(threads()), @@ -6804,10 +8913,76 @@ mod tests { RequestTarget(REVIEW_COMMENT_REACTIONS_TARGET.to_owned()), ResponseBody(review_comment_reactions()), ), + ScriptedResponse::ok( + RequestTarget(WORKFLOWS_TARGET.to_owned()), + ResponseBody(workflows()), + ), + ScriptedResponse::ok( + RequestTarget(MAIN_WORKFLOW_TARGET.to_owned()), + ResponseBody(main_workflow_run()), + ), + ] + } + + /// The same complete sweep over a pull request whose only remaining + /// convergence blocker is its aggregate review decision: GitHub reports it + /// mergeable and every review thread is resolved. This is the state a stale + /// blocking review may be dismissed against, so it is the cursor a clearance + /// is planned and dismissed from. + fn review_only_blocked_observation_responses() -> Vec { + vec![ + ScriptedResponse::ok( + RequestTarget(PULLS_TARGET.to_owned()), + ResponseBody(pulls_with_one()), + ), ScriptedResponse::ok( RequestTarget(BRANCHES_TARGET.to_owned()), ResponseBody(branches()), ), + ScriptedResponse::ok( + RequestTarget(PULL_DETAIL_TARGET.to_owned()), + ResponseBody(mergeable_pull_detail()), + ), + ScriptedResponse::ok( + RequestTarget(CHECK_SUITES_TARGET.to_owned()), + ResponseBody(check_suites()), + ), + ScriptedResponse::ok( + RequestTarget(COMMIT_CHECK_RUNS_TARGET.to_owned()), + ResponseBody(check_runs()), + ), + ScriptedResponse::ok( + RequestTarget(REVIEWS_TARGET.to_owned()), + ResponseBody(reviews()), + ), + ScriptedResponse::post( + RequestTarget(THREADS_TARGET.to_owned()), + ResponseBody(review_only_blocked_convergence()), + ), + ScriptedResponse::post( + RequestTarget(THREADS_TARGET.to_owned()), + ResponseBody(empty_threads()), + ), + ScriptedResponse::ok( + RequestTarget(PULL_REACTIONS_TARGET.to_owned()), + ResponseBody(pull_reactions()), + ), + ScriptedResponse::ok( + RequestTarget(ISSUE_COMMENTS_TARGET.to_owned()), + ResponseBody(issue_comments()), + ), + ScriptedResponse::ok( + RequestTarget(ISSUE_COMMENT_REACTIONS_TARGET.to_owned()), + ResponseBody(issue_comment_reactions()), + ), + ScriptedResponse::ok( + RequestTarget(REVIEW_COMMENTS_TARGET.to_owned()), + ResponseBody(review_comments()), + ), + ScriptedResponse::ok( + RequestTarget(REVIEW_COMMENT_REACTIONS_TARGET.to_owned()), + ResponseBody(review_comment_reactions()), + ), ScriptedResponse::ok( RequestTarget(WORKFLOWS_TARGET.to_owned()), ResponseBody(workflows()), @@ -6822,8 +8997,12 @@ mod tests { fn complete_pull_request_responses() -> Vec { complete_typed_observation_responses() .into_iter() - .skip(1) - .take(10) + // The per-pull-request slice of the complete sweep: everything from + // the pull detail through the review-comment reactions, with the + // repository listing and branch page ahead of it and the workflow + // queries behind it excluded. + .skip(2) + .take(11) .collect() } @@ -6842,6 +9021,7 @@ mod tests { serde_json::json!({ "number": number, "state": "open", + "updated_at": PULL_UPDATED_AT, "merged_at": null, "mergeable": true, "head": { @@ -6879,6 +9059,32 @@ mod tests { .to_string() } + fn minimal_convergence(number: u64) -> String { + let head_sha = minimal_pull_head_sha(number); + serde_json::json!({ + "data": { + "repository": { + "pullRequest": { + "headRefOid": head_sha, + "baseRefName": BASE_BRANCH, + "baseRefOid": BASE_SHA, + "mergeable": "MERGEABLE", + "reviewDecision": null, + "commits": { + "nodes": [{ + "commit": { + "oid": minimal_pull_head_sha(number), + "statusCheckRollup": null + } + }] + } + } + } + } + }) + .to_string() + } + fn minimal_pull_responses(number: u64) -> Vec { let head_sha = minimal_pull_head_sha(number); vec![ @@ -6899,10 +9105,16 @@ mod tests { )), ResponseBody(EMPTY_LIST.to_owned()), ), + ScriptedResponse::post( + RequestTarget(THREADS_TARGET.to_owned()), + ResponseBody(minimal_convergence(number)), + ) + .matching_request_body(format!("\"number\":{number}")), ScriptedResponse::post( RequestTarget(THREADS_TARGET.to_owned()), ResponseBody(empty_threads()), - ), + ) + .matching_request_body(format!("\"number\":{number}")), ScriptedResponse::ok( RequestTarget(format!( "/repos/{WATCHED_REPOSITORY}/issues/{number}/reactions?per_page=100&page=1" @@ -6930,6 +9142,10 @@ mod tests { RequestTarget(PULLS_TARGET.to_owned()), ResponseBody(pulls_with_one()), ), + ScriptedResponse::ok( + RequestTarget(BRANCHES_TARGET.to_owned()), + ResponseBody(branches()), + ), ScriptedResponse::ok( RequestTarget(PULL_DETAIL_TARGET.to_owned()), ResponseBody(pull_detail()), @@ -6946,6 +9162,10 @@ mod tests { RequestTarget(REVIEWS_TARGET.to_owned()), ResponseBody(reviews()), ), + ScriptedResponse::post( + RequestTarget(THREADS_TARGET.to_owned()), + ResponseBody(convergence()), + ), ScriptedResponse::post( RequestTarget(THREADS_TARGET.to_owned()), ResponseBody(threads()), @@ -6970,10 +9190,6 @@ mod tests { RequestTarget(REVIEW_COMMENT_REACTIONS_TARGET.to_owned()), ResponseBody(review_comment_reactions()), ), - ScriptedResponse::ok( - RequestTarget(BRANCHES_TARGET.to_owned()), - ResponseBody(branches()), - ), ScriptedResponse::ok( RequestTarget(WORKFLOWS_TARGET.to_owned()), ResponseBody(workflows()), @@ -6999,10 +9215,18 @@ mod tests { RequestTarget(PULLS_TARGET.to_owned()), ResponseBody(pulls_with_one()), ), + ScriptedResponse::conditional_ok( + RequestTarget(BRANCHES_TARGET.to_owned()), + ResponseBody(branches()), + ), ScriptedResponse::conditional_ok( RequestTarget(REVIEWS_TARGET.to_owned()), ResponseBody(reviews), ), + ScriptedResponse::post( + RequestTarget(THREADS_TARGET.to_owned()), + ResponseBody(convergence()), + ), ScriptedResponse::post( RequestTarget(THREADS_TARGET.to_owned()), ResponseBody(threads()), @@ -7027,10 +9251,6 @@ mod tests { RequestTarget(REVIEW_COMMENT_REACTIONS_TARGET.to_owned()), ResponseBody(review_comment_reactions()), ), - ScriptedResponse::conditional_ok( - RequestTarget(BRANCHES_TARGET.to_owned()), - ResponseBody(branches()), - ), ScriptedResponse::conditional_ok( RequestTarget(WORKFLOWS_TARGET.to_owned()), ResponseBody(workflows()), @@ -7087,6 +9307,11 @@ mod tests { RequestTarget(PULL_DETAIL_TARGET.to_owned()), ResponseBody(pull_detail_with_pending_mergeability()), ) + } else if response.target == THREADS_TARGET && response.body == convergence() { + ScriptedResponse::post( + RequestTarget(THREADS_TARGET.to_owned()), + ResponseBody(convergence_with_mergeability("UNKNOWN")), + ) } else { response } @@ -7195,11 +9420,44 @@ mod tests { observation } + /// The observation [`review_only_blocked_assessment`] describes. A first + /// poll publishes no freshness, so its own candidate lookup finds the head + /// unsettled and short-circuits before any blocking-review request. + async fn review_only_blocked_observation() -> RepoWatchObservation { + let server = ScriptedServer::start(review_only_blocked_observation_responses()).await; + let fixture = poller_fixture(server.base_url.clone()).expect("poller is constructed"); + let observation = fixture.poller.poll(None).await.expect("full poll succeeds"); + server.finish().await; + observation + } + + fn review_only_blocked_assessment() -> RepoWatchConvergenceAssessment { + RepoWatchConvergenceAssessment::try_new(RepoWatchConvergenceAssessmentInput { + number: PullRequestNumber::new( + NonZeroU64::new(PULL_NUMBER).expect("fixture pull-request number is positive"), + ), + head_sha: CommitSha::try_new(String::from(HEAD_SHA)) + .expect("fixture head is canonical"), + base_branch: BranchName::try_new(String::from(BASE_BRANCH)) + .expect("fixture base branch is canonical"), + base_revision: CommitSha::try_new(String::from(BASE_SHA)) + .expect("fixture base revision is canonical"), + mergeable_state: MergeableState::Mergeable, + settled: true, + review_decision: RepoWatchReviewDecision::ChangesRequested, + unresolved_threads: Vec::new(), + gating_check_count: 1, + non_green_gating_checks: Vec::new(), + }) + .expect("review decision is the fixture's only convergence blocker") + } + #[tokio::test] async fn targeted_refresh_reuses_the_repository_poller_and_preserves_untouched_state() { let previous = complete_typed_observation().await; let server = ScriptedServer::start(complete_pull_request_responses()).await; let fixture = poller_fixture(server.base_url.clone()).expect("poller is constructed"); + install_late_freshness_survivor(&fixture.poller).await; let target = TargetedPullRequest { number: PullRequestNumber::new( NonZeroU64::new(PULL_NUMBER).expect("fixture pull-request number is positive"), @@ -7216,6 +9474,17 @@ mod tests { .expect("targeted refresh succeeds"); server.finish().await; + assert!( + fixture.poller.fetches.lock().await.is_empty(), + "targeted refresh drains complete-poll survivors before fetching" + ); + assert!( + !fixture + .poller + .freshness() + .contains_key(&CANCELLED_FETCH_PULL_NUMBER), + "targeted refresh invalidates freshness recorded by a late survivor" + ); assert_eq!( refreshed, TargetedPollOutcome::Observation { @@ -7225,6 +9494,76 @@ mod tests { ); } + #[tokio::test] + async fn targeted_refresh_without_survivors_preserves_untouched_freshness() { + const UNTOUCHED_PULL_NUMBER: u64 = 8; + let previous = complete_typed_observation().await; + let server = ScriptedServer::start(complete_pull_request_responses()).await; + let fixture = poller_fixture(server.base_url.clone()).expect("poller is constructed"); + fixture.poller.record_fetched_pull_request( + UNTOUCHED_PULL_NUMBER, + &listed_pull_request(&minimal_pull_head_sha(UNTOUCHED_PULL_NUMBER)), + PullRequestSettlement::Settled, + Vec::new(), + ); + let target = TargetedPullRequest { + number: PullRequestNumber::new( + NonZeroU64::new(PULL_NUMBER).expect("fixture pull-request number is positive"), + ), + expected_head: Some( + CommitSha::try_new(HEAD_SHA.to_owned()).expect("fixture head SHA is canonical"), + ), + }; + + fixture + .poller + .poll_targeted_pull_requests_against_cursor(&previous, &[target]) + .await + .expect("targeted refresh succeeds"); + + server.finish().await; + assert!( + fixture + .poller + .freshness() + .contains_key(&UNTOUCHED_PULL_NUMBER), + "targeted refresh without survivors preserves untouched freshness" + ); + } + + struct FreshnessOnDrop { + poller: Arc, + } + + impl Drop for FreshnessOnDrop { + fn drop(&mut self) { + self.poller.record_fetched_pull_request( + CANCELLED_FETCH_PULL_NUMBER, + &listed_pull_request(&minimal_pull_head_sha(CANCELLED_FETCH_PULL_NUMBER)), + PullRequestSettlement::Settled, + Vec::new(), + ); + } + } + + async fn install_late_freshness_survivor(poller: &Arc) { + let (started, ready) = tokio::sync::oneshot::channel(); + let survivor_poller = Arc::clone(poller); + poller.fetches.lock().await.spawn(async move { + let _freshness_on_cancellation = FreshnessOnDrop { + poller: survivor_poller, + }; + started + .send(()) + .expect("targeted-refresh fixture still waits for its survivor"); + std::future::pending::>() + .await + }); + ready + .await + .expect("the cancelled-fetch survivor starts before targeted refresh"); + } + #[tokio::test] async fn targeted_refresh_reports_a_moved_head_as_superseded() { let previous = complete_typed_observation().await; @@ -7432,6 +9771,22 @@ mod tests { assert_eq!(drain, WebhookDrain::Run); } + #[test] + fn only_projection_timeouts_block_a_complete_poll() { + assert!( + WebhookDrainOutcome::ProjectionFailed( + RepositoryWatchAttemptError::WebhookDrainTimedOut + ) + .blocks_complete_poll_after_timeout() + ); + assert!( + !WebhookDrainOutcome::DispatchFailedAfterTerminal( + RepositoryWatchAttemptError::WebhookDrainTimedOut + ) + .blocks_complete_poll_after_timeout() + ); + } + #[tokio::test(start_paused = true)] async fn a_dispatch_follow_up_does_not_suppress_an_admission_wake() { let (_shutdown, mut shutdown_receiver) = watch::channel(false); @@ -7876,6 +10231,26 @@ mod tests { Ok(()) } + #[tokio::test] + #[ignore = "requires ephemeral PostgreSQL"] + async fn startup_preparation_drains_pending_webhook_work_before_runtime_admission() + -> Result<(), Box> { + let (_container, pool) = migrated_postgres().await?; + let webhook_store = PostgresRepoWatchWebhookStore::new(pool.clone()); + let admitted = submitted_review_admission(FIRST_WEBHOOK_DELIVERY, FIRST_WEBHOOK_REVIEW)?; + webhook_store.admit(&admitted).await?; + let (_sender, receiver) = watch::channel(()); + let mut fixture = webhook_task(&pool).await?; + fixture.task.webhook_work = Some(receiver); + + let must_stop = fixture.task.prepare_startup().await; + + assert!(!must_stop); + assert!(webhook_disposition_exists(&webhook_store, admitted.key()).await?); + assert!(fixture.task.startup_webhook_retry.is_some()); + Ok(()) + } + #[tokio::test] #[ignore = "requires ephemeral PostgreSQL"] async fn a_retry_drains_the_delivery_a_projection_error_retained() -> Result<(), Box> @@ -7902,6 +10277,48 @@ mod tests { Ok(()) } + #[test] + fn scripted_request_waits_for_its_declared_body() { + const HEADERS: &[u8] = b"POST /graphql HTTP/1.1\r\nContent-Length: 4\r\n\r\n"; + const COMPLETE: &[u8] = b"POST /graphql HTTP/1.1\r\nContent-Length: 4\r\n\r\ntest"; + + assert!(!scripted_request_is_complete(HEADERS)); + assert!(scripted_request_is_complete(COMPLETE)); + } + + #[test] + fn scripted_request_without_a_body_completes_at_headers() { + const REQUEST: &[u8] = b"GET /resource HTTP/1.1\r\nHost: localhost\r\n\r\n"; + + assert!(scripted_request_is_complete(REQUEST)); + } + + #[tokio::test] + #[ignore = "requires ephemeral PostgreSQL"] + async fn a_progressing_drain_yields_before_its_outer_deadline_and_rearms() + -> Result<(), Box> { + let (_container, pool) = migrated_postgres().await?; + let webhook_store = PostgresRepoWatchWebhookStore::new(pool.clone()); + let first = submitted_review_admission(FIRST_WEBHOOK_DELIVERY, FIRST_WEBHOOK_REVIEW)?; + let second = submitted_review_admission(SECOND_WEBHOOK_DELIVERY, SECOND_WEBHOOK_REVIEW)?; + webhook_store.admit(&first).await?; + webhook_store.admit(&second).await?; + let (sender, receiver) = watch::channel(()); + let mut fixture = webhook_task(&pool).await?; + fixture.task.webhook_nudge = Some(Arc::new(sender)); + + let outcome = fixture + .task + .process_webhook_deliveries_with_budget(Some(Duration::ZERO)) + .await; + + assert_eq!(outcome, WebhookDrainOutcome::Drained); + assert!(webhook_disposition_exists(&webhook_store, first.key()).await?); + assert!(!webhook_disposition_exists(&webhook_store, second.key()).await?); + assert!(receiver.has_changed()?); + Ok(()) + } + #[tokio::test] #[ignore = "requires ephemeral PostgreSQL"] async fn a_backlogged_drain_yields_after_one_page_and_rearms_its_wake() @@ -7974,10 +10391,17 @@ mod tests { Ok(()) } - #[tokio::test] + /// INV-074: deadline cancellation preserves durable webhook work for retry. + #[tokio::test(start_paused = true)] #[ignore = "requires ephemeral PostgreSQL"] async fn a_webhook_drain_deadline_cancels_and_retries_durable_work() -> Result<(), Box> { + // A paused clock auto-advances whenever the runtime goes idle, and + // container startup spends nearly all of its time waiting on the + // container daemon. Without this the daemon client's own request + // deadline expires in virtual time before any of the work below runs, + // so keep the clock runnable across setup and not just the wedge. + let clock_guard = keep_paused_clock_runnable(); let (_container, pool) = migrated_postgres().await?; let webhook_store = PostgresRepoWatchWebhookStore::new(pool.clone()); let admission = submitted_review_admission(FIRST_WEBHOOK_DELIVERY, FIRST_WEBHOOK_REVIEW)?; @@ -7991,18 +10415,55 @@ mod tests { .execute(&mut *blocker) .await?; let mut fixture = webhook_task(&pool).await?; + { + // The guard above still holds, so Tokio cannot auto-advance the + // production deadline before the database operation reaches the + // injected wedge. + let drain = fixture.task.process_webhook_deliveries_with_timeout(); + tokio::pin!(drain); + tokio::select! { + () = wait_for_webhook_projection_wedge(&webhook_store) => {} + outcome = &mut drain => { + panic!("the deliberate projection wedge completed early: {outcome:?}"); + } + } - let timed_out = fixture - .task - .process_webhook_deliveries_with_deadline(Duration::from_millis(50)) - .await; - + clock_guard.abort(); + clock_guard.await.ok(); + tokio::time::advance(WEBHOOK_DRAIN_ATTEMPT_TIMEOUT).await; + assert_eq!( + drain.await, + WebhookDrainOutcome::ProjectionFailed( + RepositoryWatchAttemptError::WebhookDrainTimedOut + ) + ); + } + // The deadline has fired, so virtual time is free to jump again. The + // durable work below still talks to PostgreSQL, and its connection + // pool's own acquire deadline would expire instantly in that jumped + // time, so keep the clock runnable for the retry as well. + let clock_guard = keep_paused_clock_runnable(); assert_eq!( - timed_out, - WebhookDrainOutcome::ProjectionFailed( - RepositoryWatchAttemptError::WebhookDrainTimedOut - ) + fixture.task.webhook_terminal_ambiguous, + Some(admission.key()), + "deadline cancellation retains the exact unsettled terminal write" + ); + let ambiguous = fixture.task.webhook_terminal_ambiguous.take(); + let mut deferred_drain = None; + let mut deferred_dispatch_failure = None; + assert_eq!( + fixture + .task + .run_attempt_prelude( + WebhookDrain::Deferred, + &mut deferred_drain, + &mut deferred_dispatch_failure, + ) + .await, + Err(RepositoryWatchAttemptError::WebhookDrainTimedOut), + "the general timeout fence blocks deferred cursor-advancing polls even before delivery-specific state is installed" ); + fixture.task.webhook_terminal_ambiguous = ambiguous; assert!(!webhook_disposition_exists(&webhook_store, admission.key()).await?); let unlocked: bool = sqlx::query_scalar("SELECT pg_advisory_unlock($1)") .bind(WEBHOOK_PROJECTION_ADVISORY_LOCK) @@ -8014,9 +10475,36 @@ mod tests { assert!(unlocked, "the fixture releases its deliberate drain wedge"); assert_eq!(retried, WebhookDrainOutcome::Drained); assert!(webhook_disposition_exists(&webhook_store, admission.key()).await?); + assert_eq!( + fixture.task.webhook_terminal_ambiguous, None, + "settling that exact delivery releases complete polling" + ); + clock_guard.abort(); + clock_guard.await.ok(); Ok(()) } + /// Only a cancelled drain has earned the growing projection backoff, so an + /// attempt deadline reached outside the drain reports the step it + /// interrupted rather than a drain failure. + #[test] + fn a_cancelled_attempt_reports_the_step_the_deadline_interrupted() { + let error = RepositoryWatchAttemptError::WebhookAttemptTimedOut; + + assert_eq!( + WebhookAttemptPhase::BeforeDrain.cancelled_outcome(error), + WebhookAttemptOutcome::FailedBeforeDrain(error) + ); + assert_eq!( + WebhookAttemptPhase::Drain.cancelled_outcome(error), + WebhookAttemptOutcome::DrainFailed(error) + ); + assert_eq!( + WebhookAttemptPhase::AfterDrain.cancelled_outcome(error), + WebhookAttemptOutcome::DrainedThenFailed(error) + ); + } + #[tokio::test] #[ignore = "requires ephemeral PostgreSQL"] async fn a_webhook_attempt_deadline_cancels_any_wedged_phase_and_retries() @@ -8040,9 +10528,16 @@ mod tests { .run_webhook_attempt_with_deadline(Duration::from_millis(50)) .await; + // The projection wedge is reached by the cutoff and dispatch + // reconciliation that now precedes the drain, so the deadline cancels + // the attempt while it is still `BeforeDrain`. The phase matters to the + // retry accounting rather than to the recovery: a cancellation before + // the drain began neither grows nor clears the projection backoff. assert_eq!( timed_out, - WebhookAttemptOutcome::DrainFailed(RepositoryWatchAttemptError::WebhookAttemptTimedOut) + WebhookAttemptOutcome::FailedBeforeDrain( + RepositoryWatchAttemptError::WebhookAttemptTimedOut + ) ); assert!(!webhook_disposition_exists(&webhook_store, admission.key()).await?); let unlocked: bool = sqlx::query_scalar("SELECT pg_advisory_unlock($1)") @@ -8202,92 +10697,516 @@ mod tests { let attempt = fixture.task.process_webhook_deliveries().await; server.finish().await; - assert_eq!( - attempt, - WebhookDrainOutcome::ProjectionFailed(RepositoryWatchAttemptError::Rejected) + assert_eq!( + attempt, + WebhookDrainOutcome::ProjectionFailed(RepositoryWatchAttemptError::Rejected) + ); + assert!(!webhook_disposition_exists(&webhook_store, unservable.key()).await?); + assert!(webhook_disposition_exists(&webhook_store, behind_it.key()).await?); + Ok(()) + } + + #[test] + fn inv069_non_enqueued_repo_watch_nudges_are_recorded() -> Result<(), Box> { + let repository = RepositorySlug::try_new(WATCHED_REPOSITORY.to_owned())?; + let session = signalbox_domain::SessionId::from_uuid(Uuid::from_u128(0x69)); + let captured = CapturedLog::default(); + let subscriber = tracing_subscriber::fmt() + .without_time() + .with_ansi(false) + .with_max_level(tracing::Level::INFO) + .with_writer(captured.clone()) + .finish(); + + tracing::subscriber::with_default(subscriber, || { + record_dispatch_start_nudge_outcome( + &repository, + session, + EligibilityNudgeOutcome::Coalesced, + ); + record_dispatch_start_nudge_outcome( + &repository, + session, + EligibilityNudgeOutcome::DroppedAtCapacity, + ); + record_dispatch_start_nudge_outcome( + &repository, + session, + EligibilityNudgeOutcome::WorkSourceClosed, + ); + }); + let telemetry = captured.text(); + + assert!(telemetry.contains("repository_watch_dispatch_start_nudge_coalesced")); + assert!(telemetry.contains("repository_watch_dispatch_start_nudge_capacity")); + assert!(telemetry.contains("repository_watch_dispatch_start_nudge_closed")); + Ok(()) + } + + #[tokio::test] + #[ignore = "requires ephemeral PostgreSQL"] + async fn a_provider_wide_rejection_stops_the_page_and_preserves_its_durable_tail() + -> Result<(), Box> { + let (_container, pool) = migrated_postgres().await?; + let webhook_store = PostgresRepoWatchWebhookStore::new(pool.clone()); + let throttled = synchronize_admission(THIRD_WEBHOOK_DELIVERY)?; + let tail = submitted_review_admission(FIRST_WEBHOOK_DELIVERY, FIRST_WEBHOOK_REVIEW)?; + webhook_store.admit(&throttled).await?; + webhook_store.admit(&tail).await?; + let server = ScriptedServer::start(vec![ScriptedResponse::forbidden(RequestTarget( + PULL_DETAIL_TARGET.to_owned(), + ))]) + .await; + let mut fixture = webhook_task_against(&pool, server.base_url.clone()).await?; + + let attempt = fixture.task.process_webhook_deliveries().await; + + server.finish().await; + assert_eq!( + attempt, + WebhookDrainOutcome::ProjectionFailed(RepositoryWatchAttemptError::ProviderUnavailable) + ); + assert!(!webhook_disposition_exists(&webhook_store, throttled.key()).await?); + assert!(!webhook_disposition_exists(&webhook_store, tail.key()).await?); + Ok(()) + } + + #[tokio::test] + #[ignore = "requires ephemeral PostgreSQL"] + async fn wedged_webhook_drain_emits_an_error_with_its_cause() -> Result<(), Box> { + let (_container, pool) = migrated_postgres().await?; + let store = PostgresRepoWatchWebhookStore::new(pool); + let admission = submitted_review_admission(FIRST_WEBHOOK_DELIVERY, FIRST_WEBHOOK_REVIEW)?; + store.admit(&admission).await?; + let repository = RepositorySlug::try_new(WATCHED_REPOSITORY.to_owned())?; + let captured = CapturedLog::default(); + let subscriber = tracing_subscriber::fmt() + .without_time() + .with_ansi(false) + .with_max_level(tracing::Level::ERROR) + .with_writer(captured.clone()) + .finish(); + + let mut progress = WebhookDrainProgress::default(); + inspect_webhook_drain(&repository, &store, Duration::ZERO, &mut progress).await; + inspect_webhook_drain(&repository, &store, Duration::ZERO, &mut progress) + .with_subscriber(subscriber) + .await; + + let telemetry = captured.text(); + assert!(telemetry.contains("ERROR")); + assert!(telemetry.contains("cause_code=\"webhook_projection_drain_stalled\"")); + assert!(telemetry.contains(&admission.key().delivery_id().to_string())); + Ok(()) + } + + #[test] + fn advancing_webhook_head_resets_the_stall_clock() { + let first_sequence = NonZeroU64::new(41).expect("fixture sequence is positive"); + let next_sequence = NonZeroU64::new(42).expect("fixture sequence is positive"); + let started_at = Instant::now(); + let mut progress = WebhookDrainProgress::default(); + + assert_eq!(progress.observe(first_sequence, started_at), None); + assert_eq!( + progress.observe(first_sequence, started_at + Duration::from_secs(31)), + Some(Duration::from_secs(31)) + ); + assert_eq!( + progress.observe(next_sequence, started_at + Duration::from_secs(32)), + None + ); + assert_eq!( + progress.observe(next_sequence, started_at + Duration::from_secs(90)), + Some(Duration::from_secs(58)) + ); + } + + #[tokio::test] + async fn convergence_matches_the_exact_head_gate() { + let server = ScriptedServer::start(complete_typed_observation_responses()).await; + let fixture = poller_fixture(server.base_url.clone()).expect("poller is constructed"); + + let polled = fixture + .poller + .poll_against_cursor(None, Some(RepoWatchCursorGeneration::INITIAL)) + .await + .expect("full poll and convergence assessment succeed"); + server.finish().await; + let assessment = &polled.convergence[0]; + + assert_eq!(assessment.gating_check_count(), 1); + assert_eq!( + assessment.non_green_gating_checks()[0].as_str(), + CHECK_RUN_NAME + ); + assert_eq!(assessment.unresolved_threads()[0].as_str(), REVIEW_THREAD); + assert_eq!( + assessment.verdict(), + signalbox_application::RepoWatchConvergenceVerdict::NotConverged + ); + } + + #[tokio::test] + async fn older_head_review_becomes_a_clearance_candidate() { + let response = ScriptedResponse::post( + RequestTarget(String::from(THREADS_TARGET)), + ResponseBody(blocking_reviews(STALE_REVIEW_HEAD_SHA)), + ) + .matching_request_body(String::from("RepositoryWatchBlockingReviews")); + let server = ScriptedServer::start(vec![response]).await; + let fixture = poller_fixture(server.base_url.clone()).expect("poller is constructed"); + + let candidates = fixture + .poller + .fetch_stale_review_clearances(&review_only_blocked_assessment()) + .await + .expect("blocking review evidence is valid"); + server.finish().await; + + assert_eq!(candidates[0].review_node_id(), STALE_REVIEW_NODE_ID); + assert_eq!(candidates[0].reviewer().as_str(), REVIEWER); + assert_eq!( + candidates[0].reviewed_head_sha().as_str(), + STALE_REVIEW_HEAD_SHA + ); + } + + #[tokio::test] + async fn inv072_current_head_review_is_not_a_clearance_candidate() { + let response = ScriptedResponse::post( + RequestTarget(String::from(THREADS_TARGET)), + ResponseBody(blocking_reviews(HEAD_SHA)), + ) + .matching_request_body(String::from("RepositoryWatchBlockingReviews")); + let server = ScriptedServer::start(vec![response]).await; + let fixture = poller_fixture(server.base_url.clone()).expect("poller is constructed"); + + let candidates = fixture + .poller + .fetch_stale_review_clearances(&review_only_blocked_assessment()) + .await + .expect("current-head blocker fails closed without an error"); + server.finish().await; + + assert!(candidates.is_empty()); + } + + #[tokio::test] + async fn dismissal_mutation_requires_the_expected_review_identity() { + const MISMATCHING_REVIEW_NODE_ID: &str = "PRR_mismatching_review_node"; + let response = ScriptedResponse::post( + RequestTarget(String::from(THREADS_TARGET)), + ResponseBody(dismissed_review(MISMATCHING_REVIEW_NODE_ID)), + ) + .matching_request_body(String::from("RepositoryWatchDismissReview")); + let server = ScriptedServer::start(vec![response]).await; + let fixture = poller_fixture(server.base_url.clone()).expect("poller is constructed"); + + let error = fixture + .poller + .dismiss_review_node(super::DismissReviewInput { + review_node_id: STALE_REVIEW_NODE_ID, + dismissal_message: DISMISSAL_MESSAGE, + }) + .await + .expect_err("a response naming another review must fail closed"); + server.finish().await; + + assert_eq!(error, RepositoryWatchAttemptError::InvalidResponse); + } + + fn planned_stale_review_clearance() -> super::RepoWatchPlannedStaleReviewClearance { + super::RepoWatchPlannedStaleReviewClearance::from_fixture( + RepoWatchPlannedStaleReviewClearanceFixture { + clearance_id: RepoWatchStaleReviewClearanceId::new(Uuid::from_u128(0x_c1ea_0001)), + claim_token: RepoWatchStaleReviewClearanceClaimToken::new(Uuid::from_u128( + 0x_c1a1_0001, + )), + number: PullRequestNumber::new( + NonZeroU64::new(PULL_NUMBER).expect("fixture pull-request number is positive"), + ), + current_head_sha: CommitSha::try_new(String::from(HEAD_SHA)) + .expect("fixture head is canonical"), + base_branch: BranchName::try_new(String::from(BASE_BRANCH)) + .expect("fixture base branch is canonical"), + base_revision: CommitSha::try_new(String::from(BASE_SHA)) + .expect("fixture base revision is canonical"), + review_node_id: String::from(STALE_REVIEW_NODE_ID), + reviewer: RepoWatchAuthorLogin::try_new(String::from(REVIEWER)) + .expect("fixture reviewer is valid"), + reviewed_head_sha: CommitSha::try_new(String::from(STALE_REVIEW_HEAD_SHA)) + .expect("fixture reviewed head is canonical"), + dismissal_message: String::from(DISMISSAL_MESSAGE), + }, + ) + } + + /// The in-memory candidate the committed poll raises for the review + /// [`blocking_reviews`] reports, against the evidence + /// [`review_only_blocked_assessment`] records. + fn stale_review_clearance_candidate() -> RepoWatchStaleReviewClearanceCandidate { + RepoWatchStaleReviewClearanceCandidate::try_new( + &review_only_blocked_assessment(), + String::from(STALE_REVIEW_NODE_ID), + RepoWatchAuthorLogin::try_new(String::from(REVIEWER)) + .expect("fixture reviewer is valid"), + CommitSha::try_new(String::from(STALE_REVIEW_HEAD_SHA)) + .expect("fixture reviewed head is canonical"), + ) + .expect("the review is the fixture head's only convergence blocker") + } + + /// Revalidation is the gate the dismissal mutation sits behind, and it + /// reports the clearance still holds only for a settled head. Settlement in + /// turn requires a quiesced gating-check inventory, so evidence that never + /// carries quiescence makes the whole feature a no-op: the candidate lookup + /// short-circuits, the revalidation refuses, and no review is ever + /// dismissed. This proves the re-read backed by the committed poll's + /// freshness passes that gate; + /// [`a_planned_clearance_reaches_its_dismissal_mutation`] proves the + /// orchestration then issues the mutation. + #[tokio::test] + async fn a_quiesced_inventory_revalidates_a_planned_clearance() { + let server = ScriptedServer::start(vec![ + ScriptedResponse::ok( + RequestTarget(String::from(PULL_DETAIL_TARGET)), + ResponseBody(mergeable_pull_detail()), + ), + ScriptedResponse::post( + RequestTarget(String::from(THREADS_TARGET)), + ResponseBody(review_only_blocked_convergence()), + ) + .matching_request_body(String::from("RepositoryWatchConvergence")), + ScriptedResponse::post( + RequestTarget(String::from(THREADS_TARGET)), + ResponseBody(empty_threads()), + ) + .matching_request_body(String::from("RepositoryWatchReviewThreads")), + ScriptedResponse::post( + RequestTarget(String::from(THREADS_TARGET)), + ResponseBody(blocking_reviews(STALE_REVIEW_HEAD_SHA)), + ) + .matching_request_body(String::from("RepositoryWatchBlockingReviews")), + ]) + .await; + let fixture = poller_fixture(server.base_url.clone()).expect("poller is constructed"); + let generation = RepoWatchCursorGeneration::INITIAL; + fixture.poller.record_fetched_pull_request( + PULL_NUMBER, + &listed_pull_request(HEAD_SHA), + PullRequestSettlement::Settled, + vec![String::from(CHECK_RUN_NAME)], + ); + fixture.poller.publish_freshness(generation); + + let holds = fixture + .poller + .revalidate_stale_review_clearance(&planned_stale_review_clearance(), generation) + .await + .expect("clearance revalidation reads valid evidence"); + server.finish().await; + + assert!( + holds, + "a settled head whose only blocker is a superseded review must pass revalidation" ); - assert!(!webhook_disposition_exists(&webhook_store, unservable.key()).await?); - assert!(webhook_disposition_exists(&webhook_store, behind_it.key()).await?); - Ok(()) } + /// The whole live path, from the poll that commits the candidate's evidence + /// to the provider mutation: the completed poll records its assessment, + /// plans the intent durably, revalidates it against a re-read, and sends the + /// dismissal. Scripting the mutation as a matched response is what makes + /// this end-to-end rather than a revalidation test — a build that stops + /// short of `dismiss_review_node` leaves that response unconsumed and no + /// terminal outcome recorded. #[tokio::test] #[ignore = "requires ephemeral PostgreSQL"] - async fn a_provider_wide_rejection_stops_the_page_and_preserves_its_durable_tail() - -> Result<(), Box> { + async fn a_planned_clearance_reaches_its_dismissal_mutation() -> Result<(), Box> { let (_container, pool) = migrated_postgres().await?; - let webhook_store = PostgresRepoWatchWebhookStore::new(pool.clone()); - let throttled = synchronize_admission(THIRD_WEBHOOK_DELIVERY)?; - let tail = submitted_review_admission(FIRST_WEBHOOK_DELIVERY, FIRST_WEBHOOK_REVIEW)?; - webhook_store.admit(&throttled).await?; - webhook_store.admit(&tail).await?; - let server = ScriptedServer::start(vec![ScriptedResponse::forbidden(RequestTarget( - PULL_DETAIL_TARGET.to_owned(), - ))]) + let server = ConcurrentScriptedServer::start(vec![ + ScriptedResponse::ok( + RequestTarget(String::from(PULL_DETAIL_TARGET)), + ResponseBody(mergeable_pull_detail()), + ), + ScriptedResponse::post( + RequestTarget(String::from(THREADS_TARGET)), + ResponseBody(review_only_blocked_convergence()), + ) + .matching_request_body(String::from("RepositoryWatchConvergence")), + ScriptedResponse::post( + RequestTarget(String::from(THREADS_TARGET)), + ResponseBody(empty_threads()), + ) + .matching_request_body(String::from("RepositoryWatchReviewThreads")), + ScriptedResponse::post( + RequestTarget(String::from(THREADS_TARGET)), + ResponseBody(blocking_reviews(STALE_REVIEW_HEAD_SHA)), + ) + .matching_request_body(String::from("RepositoryWatchBlockingReviews")), + ScriptedResponse::post( + RequestTarget(String::from(THREADS_TARGET)), + ResponseBody(dismissed_review(STALE_REVIEW_NODE_ID)), + ) + .matching_request_body(String::from("RepositoryWatchDismissReview")), + ]) .await; - let mut fixture = webhook_task_against(&pool, server.base_url.clone()).await?; - - let attempt = fixture.task.process_webhook_deliveries().await; + let observation = review_only_blocked_observation().await; + let mut fixture = task_against(&pool, server.base_url.clone(), observation.clone()).await?; + let repository = RepositorySlug::try_new(WATCHED_REPOSITORY.to_owned())?; + let generation = PostgresRepoWatchStore::new(pool.clone()) + .load_cursor(&repository) + .await? + .expect("the fixture commits its cursor") + .generation(); + fixture.task.poller.record_fetched_pull_request( + PULL_NUMBER, + &listed_pull_request(HEAD_SHA), + PullRequestSettlement::Settled, + vec![String::from(CHECK_RUN_NAME)], + ); + fixture + .task + .commit_complete_poll(super::PreparedCompletePoll { + cursor_generation: Some(generation), + candidate: RepoWatchCursorCandidate::new(observation), + events: Vec::new(), + convergence: vec![review_only_blocked_assessment()], + stale_review_clearances: vec![stale_review_clearance_candidate()], + }) + .await + .expect("the completed poll commits its evidence and sweeps its clearances"); + // Asserts that every scripted response was consumed and that every + // request matched one, so the dismissal mutation reached the provider + // as the mutation it claims to be rather than as some other body. server.finish().await; - assert_eq!( - attempt, - WebhookDrainOutcome::ProjectionFailed(RepositoryWatchAttemptError::ProviderUnavailable) - ); - assert!(!webhook_disposition_exists(&webhook_store, throttled.key()).await?); - assert!(!webhook_disposition_exists(&webhook_store, tail.key()).await?); + + let outcome: String = + sqlx::query_scalar("SELECT outcome_kind FROM repo_watch_stale_review_clearance_result") + .fetch_one(&pool) + .await?; + assert_eq!(outcome, "dismissed"); Ok(()) } + /// The mirror of the revalidation test: a gating check that appeared since + /// the committed poll leaves the inventory unquiesced, the head unsettled, + /// and the review undismissed. The candidate lookup short-circuits before + /// its provider request, so only three calls are scripted. #[tokio::test] - #[ignore = "requires ephemeral PostgreSQL"] - async fn wedged_webhook_drain_emits_an_error_with_its_cause() -> Result<(), Box> { - let (_container, pool) = migrated_postgres().await?; - let store = PostgresRepoWatchWebhookStore::new(pool); - let admission = submitted_review_admission(FIRST_WEBHOOK_DELIVERY, FIRST_WEBHOOK_REVIEW)?; - store.admit(&admission).await?; - let repository = RepositorySlug::try_new(WATCHED_REPOSITORY.to_owned())?; - let captured = CapturedLog::default(); - let subscriber = tracing_subscriber::fmt() - .without_time() - .with_ansi(false) - .with_max_level(tracing::Level::ERROR) - .with_writer(captured.clone()) - .finish(); + async fn a_gating_check_added_since_the_committed_poll_refuses_the_clearance() { + let server = ScriptedServer::start(vec![ + ScriptedResponse::ok( + RequestTarget(String::from(PULL_DETAIL_TARGET)), + ResponseBody(mergeable_pull_detail()), + ), + ScriptedResponse::post( + RequestTarget(String::from(THREADS_TARGET)), + ResponseBody(review_only_blocked_convergence()), + ) + .matching_request_body(String::from("RepositoryWatchConvergence")), + ScriptedResponse::post( + RequestTarget(String::from(THREADS_TARGET)), + ResponseBody(empty_threads()), + ) + .matching_request_body(String::from("RepositoryWatchReviewThreads")), + ]) + .await; + let fixture = poller_fixture(server.base_url.clone()).expect("poller is constructed"); + let generation = RepoWatchCursorGeneration::INITIAL; + fixture.poller.record_fetched_pull_request( + PULL_NUMBER, + &listed_pull_request(HEAD_SHA), + PullRequestSettlement::Settled, + vec![String::from(CHECK_RUN_NAME), String::from("later gate")], + ); + fixture.poller.publish_freshness(generation); - let mut progress = WebhookDrainProgress::default(); - inspect_webhook_drain(&repository, &store, Duration::ZERO, &mut progress).await; - inspect_webhook_drain(&repository, &store, Duration::ZERO, &mut progress) - .with_subscriber(subscriber) - .await; + let holds = fixture + .poller + .revalidate_stale_review_clearance(&planned_stale_review_clearance(), generation) + .await + .expect("clearance revalidation reads valid evidence"); + server.finish().await; - let telemetry = captured.text(); - assert!(telemetry.contains("ERROR")); - assert!(telemetry.contains("cause_code=\"webhook_projection_drain_stalled\"")); - assert!(telemetry.contains(&admission.key().delivery_id().to_string())); - Ok(()) + assert!( + !holds, + "an inventory that has not stood still since the committed poll must refuse dismissal" + ); } #[test] - fn advancing_webhook_head_resets_the_stall_clock() { - let first_sequence = NonZeroU64::new(41).expect("fixture sequence is positive"); - let next_sequence = NonZeroU64::new(42).expect("fixture sequence is positive"); - let started_at = Instant::now(); - let mut progress = WebhookDrainProgress::default(); + fn recovery_settles_review_states_that_no_longer_block() { + use signalbox_persistence::repo_watch::{ + RepoWatchObservedReviewState, RepoWatchStaleReviewClearanceOutcome, + }; - assert_eq!(progress.observe(first_sequence, started_at), None); assert_eq!( - progress.observe(first_sequence, started_at + Duration::from_secs(31)), - Some(Duration::from_secs(31)) + super::terminal_clearance_outcome(RepoWatchObservedReviewState::Dismissed), + Some(RepoWatchStaleReviewClearanceOutcome::AlreadyDismissed) ); assert_eq!( - progress.observe(next_sequence, started_at + Duration::from_secs(32)), - None + super::terminal_clearance_outcome(RepoWatchObservedReviewState::Approved), + Some(RepoWatchStaleReviewClearanceOutcome::ClearedElsewhere) ); assert_eq!( - progress.observe(next_sequence, started_at + Duration::from_secs(90)), - Some(Duration::from_secs(58)) + super::terminal_clearance_outcome(RepoWatchObservedReviewState::Commented), + Some(RepoWatchStaleReviewClearanceOutcome::ClearedElsewhere) + ); + assert_eq!( + super::terminal_clearance_outcome(RepoWatchObservedReviewState::Pending), + Some(RepoWatchStaleReviewClearanceOutcome::ClearedElsewhere) ); + assert_eq!( + super::terminal_clearance_outcome(RepoWatchObservedReviewState::ChangesRequested), + None + ); + } + + #[tokio::test] + async fn convergence_rejects_evidence_from_a_different_base_revision() { + let observation = complete_typed_observation().await; + let pull_request = &observation.state().pull_requests()[0]; + let evidence = super::FetchedConvergenceEvidence { + base_revision: CommitSha::try_new(CHANGED_LISTED_HEAD_SHA.to_owned()) + .expect("fixture provider base revision is valid"), + gating_checks_settled: true, + gating_check_inventory_quiesced: true, + gating_check_inventory: vec![String::from(CHECK_RUN_NAME)], + review_decision: super::RepoWatchReviewDecision::Approved, + gating_check_count: 1, + non_green_gating_checks: Vec::new(), + }; + let snapshot_base_revision = CommitSha::try_new(BASE_SHA.to_owned()) + .expect("fixture snapshot base revision is valid"); + + let assessment = evidence.assess(pull_request, snapshot_base_revision); + + assert!(matches!( + assessment, + Err(RepositoryWatchAttemptError::InvalidResponse) + )); + } + + #[test] + fn codecov_project_status_is_report_only() { + let check = ConvergenceCheck::StatusContext { + context: String::from("codecov/project"), + state: String::from("PENDING"), + }; + + assert!(check.is_report_only()); + } + + #[test] + fn codecov_patch_status_is_report_only_case_insensitively() { + let check = ConvergenceCheck::StatusContext { + context: String::from("Codecov/Patch"), + state: String::from("PENDING"), + }; + + assert!(check.is_report_only()); } #[tokio::test] @@ -8310,7 +11229,8 @@ mod tests { exit.notify_one(); }); - let result = supervise_repository_tasks(tasks, Vec::new(), receiver).await; + let (task_shutdown, _task_shutdown_receiver) = watch::channel(false); + let result = supervise_repository_tasks(tasks, Vec::new(), receiver, task_shutdown).await; trigger.await.expect("fixture race trigger completes"); assert_eq!(result, Ok(())); @@ -8343,6 +11263,7 @@ mod tests { &listed, None, Some(RepoWatchCursorGeneration::INITIAL), + &[base_branch_head()], ) .await; RepositoryWatchChildExit::Repository @@ -8351,9 +11272,15 @@ mod tests { server.request_in_flight().await; tasks.spawn(async { panic!("fixture repository task panics") }); let (_sender, receiver) = watch::channel(false); + let (task_shutdown, _task_shutdown_receiver) = watch::channel(false); - let result = - supervise_repository_tasks(tasks, vec![Arc::clone(&fixture.poller)], receiver).await; + let result = supervise_repository_tasks( + tasks, + vec![Arc::clone(&fixture.poller)], + receiver, + task_shutdown, + ) + .await; assert_eq!( result, @@ -8366,6 +11293,28 @@ mod tests { ); } + #[tokio::test] + async fn dropping_a_retained_targeted_completion_aborts_its_writer() { + let ownership = Arc::new(()); + let child_ownership = Arc::clone(&ownership); + let retained = super::RetainedTargetedWebhookCompletion::new(tokio::spawn(async move { + let _child_ownership = child_ownership; + std::future::pending::< + Result, + >() + .await + })); + + drop(retained); + tokio::task::yield_now().await; + + assert_eq!( + Arc::strong_count(&ownership), + 1, + "dropping the repository task cannot detach its retained writer" + ); + } + #[tokio::test] async fn repository_task_panic_during_shutdown_drain_drains_sibling_fetches() { let server = ConcurrentScriptedServer::start(vec![ @@ -8393,6 +11342,7 @@ mod tests { &listed, None, Some(RepoWatchCursorGeneration::INITIAL), + &[base_branch_head()], ) .await; RepositoryWatchChildExit::Repository @@ -8401,9 +11351,15 @@ mod tests { server.request_in_flight().await; tasks.spawn(async { panic!("fixture repository task panics during shutdown") }); let (_sender, receiver) = watch::channel(true); + let (task_shutdown, _task_shutdown_receiver) = watch::channel(true); - let result = - supervise_repository_tasks(tasks, vec![Arc::clone(&fixture.poller)], receiver).await; + let result = supervise_repository_tasks( + tasks, + vec![Arc::clone(&fixture.poller)], + receiver, + task_shutdown, + ) + .await; assert_eq!( result, @@ -9195,6 +12151,7 @@ mod tests { &listed, None, Some(RepoWatchCursorGeneration::INITIAL), + &[base_branch_head()], ) .await .expect("every open pull request is fetched"); @@ -9202,6 +12159,7 @@ mod tests { ( pull_requests + .states .iter() .map(|pull_request| pull_request.context().number().get()) .collect(), @@ -9258,6 +12216,7 @@ mod tests { &listed, None, Some(RepoWatchCursorGeneration::INITIAL), + &[base_branch_head()], ) .await }); @@ -9338,9 +12297,12 @@ mod tests { let previous = &observation.state().pull_requests()[0]; let listed = listed_pull_request(HEAD_SHA); let number = previous.context().number().get(); - fixture - .poller - .record_fetched_pull_request(number, &listed, PullRequestSettlement::Settled); + fixture.poller.record_fetched_pull_request( + number, + &listed, + PullRequestSettlement::Settled, + Vec::new(), + ); fixture .poller .publish_freshness(RepoWatchCursorGeneration::INITIAL); @@ -9366,6 +12328,79 @@ mod tests { ); } + /// A targeted refresh whose cursor commit loses its generation race never + /// became cursor state, so it must leave nothing behind that a later commit + /// could vouch for. Its fetch already recorded unpublished freshness, and + /// `publish_freshness` stamps every entry it finds: keeping those would let + /// the next targeted commit relabel this fetch as belonging to a cursor it + /// never reached, and a following poll would reuse detail that cursor does + /// not carry. + #[tokio::test] + #[ignore = "requires ephemeral PostgreSQL"] + async fn a_superseded_targeted_commit_clears_the_freshness_it_recorded() + -> Result<(), Box> { + let (_container, pool) = migrated_postgres().await?; + let webhook_store = PostgresRepoWatchWebhookStore::new(pool.clone()); + let admission = submitted_review_admission(FIRST_WEBHOOK_DELIVERY, FIRST_WEBHOOK_REVIEW)?; + webhook_store.admit(&admission).await?; + let mut fixture = webhook_task(&pool).await?; + + // What a completed targeted fetch leaves behind: recorded detail that + // no cursor has published yet. + let observation = complete_typed_observation().await; + let listed = listed_pull_request(HEAD_SHA); + let number = observation.state().pull_requests()[0] + .context() + .number() + .get(); + fixture.task.poller.record_fetched_pull_request( + number, + &listed, + PullRequestSettlement::Settled, + Vec::new(), + ); + + // A generation the durable cursor has not reached, so this commit loses + // its race exactly as a competing watcher's advance would make it. + let unreached = RepoWatchCursorGeneration::INITIAL + .next() + .expect("fixture cursor generation has a successor"); + let pull_request = PullRequestNumber::new( + NonZeroU64::new(PULL_NUMBER).expect("fixture pull-request number is positive"), + ); + let prepared = PreparedTargetedRefresh { + generation: unreached, + candidate: RepoWatchCursorCandidate::new(review_only_blocked_observation().await), + events: Vec::new(), + queried: vec![RepoWatchTargetedRefreshV1::PullRequestHydration { pull_request }], + }; + + let settlement = fixture + .task + .complete_targeted_webhook_projection( + prepared, + webhook_delivery_key(FIRST_WEBHOOK_DELIVERY), + Vec::new(), + WebhookShadowBaseline { + observation, + identity_frontier: RepoWatchEventIdentityFrontierV1::default(), + }, + ) + .await + .expect("a lost generation race settles rather than failing"); + + assert_eq!( + settlement, + TargetedRefreshSettlement::Superseded, + "a commit that lost its generation race never reached the cursor" + ); + assert!( + fixture.task.poller.freshness().is_empty(), + "a fetch that never reached the cursor authorizes no later reuse" + ); + Ok(()) + } + #[tokio::test] async fn freshness_published_against_another_cursor_authorizes_no_reuse() { let fixture = poller_fixture( @@ -9380,9 +12415,12 @@ mod tests { let loaded_generation = published_generation .next() .expect("fixture cursor generation has a successor"); - fixture - .poller - .record_fetched_pull_request(number, &listed, PullRequestSettlement::Settled); + fixture.poller.record_fetched_pull_request( + number, + &listed, + PullRequestSettlement::Settled, + Vec::new(), + ); fixture.poller.publish_freshness(published_generation); assert!( @@ -9396,6 +12434,46 @@ mod tests { ); } + #[tokio::test] + async fn a_new_gating_context_requires_another_committed_poll_to_quiesce() { + let fixture = poller_fixture( + Url::parse("http://provider.invalid/").expect("fixture base forms a URL"), + ) + .expect("poller is constructed"); + let listed = listed_pull_request(HEAD_SHA); + let generation = RepoWatchCursorGeneration::INITIAL; + fixture.poller.record_fetched_pull_request( + PULL_NUMBER, + &listed, + PullRequestSettlement::Settled, + vec![String::from(CHECK_RUN_NAME)], + ); + fixture.poller.publish_freshness(generation); + let expanded_inventory = vec![ + String::from(CHECK_RUN_NAME), + String::from("later gating check"), + ]; + + assert!(!fixture.poller.gating_check_inventory_quiesced( + PULL_NUMBER, + &listed, + Some(generation), + &expanded_inventory, + )); + fixture.poller.record_gating_check_inventory( + PULL_NUMBER, + &listed, + expanded_inventory.clone(), + ); + fixture.poller.publish_freshness(generation); + assert!(fixture.poller.gating_check_inventory_quiesced( + PULL_NUMBER, + &listed, + Some(generation), + &expanded_inventory, + )); + } + #[tokio::test] async fn changed_pull_request_timestamp_authorizes_no_reuse() { let fixture = poller_fixture( @@ -9410,9 +12488,12 @@ mod tests { head_sha: listed.head_sha.clone(), }; let number = previous.context().number().get(); - fixture - .poller - .record_fetched_pull_request(number, &listed, PullRequestSettlement::Settled); + fixture.poller.record_fetched_pull_request( + number, + &listed, + PullRequestSettlement::Settled, + Vec::new(), + ); fixture .poller .publish_freshness(RepoWatchCursorGeneration::INITIAL); @@ -9435,9 +12516,12 @@ mod tests { let previous = &observation.state().pull_requests()[0]; let listed = listed_pull_request(HEAD_SHA); let number = previous.context().number().get(); - fixture - .poller - .record_fetched_pull_request(number, &listed, PullRequestSettlement::Settled); + fixture.poller.record_fetched_pull_request( + number, + &listed, + PullRequestSettlement::Settled, + Vec::new(), + ); fixture .poller .publish_freshness(RepoWatchCursorGeneration::INITIAL); @@ -9485,6 +12569,7 @@ mod tests { number, &previously_listed, PullRequestSettlement::Settled, + Vec::new(), ); fixture .poller @@ -9501,6 +12586,23 @@ mod tests { ); } + #[tokio::test] + async fn a_base_advance_forbids_pull_request_reuse() { + let observation = complete_typed_observation().await; + let pull_request = &observation.state().pull_requests()[0]; + let advanced_base = RepoWatchBranchHead::new( + pull_request.context().base_branch().clone(), + CommitSha::try_new(CHANGED_LISTED_HEAD_SHA.to_owned()) + .expect("changed fixture base revision is canonical"), + ); + + assert!(!super::pull_request_base_revision_matches( + &observation, + pull_request, + &[advanced_base], + )); + } + #[tokio::test] async fn every_check_run_member_the_decoder_requires_exists_in_the_provider_payload() { let server = ScriptedServer::start(vec![ScriptedResponse::ok( @@ -9559,6 +12661,29 @@ mod tests { )); } + #[tokio::test] + async fn an_unfinished_report_only_run_does_not_unsettle_gating_checks() { + let response = check_runs().replace(IN_PROGRESS_CHECK_RUN_NAME, "coverage (report only)"); + let server = ScriptedServer::start(vec![ScriptedResponse::ok( + RequestTarget(COMMIT_CHECK_RUNS_TARGET.to_owned()), + ResponseBody(response), + )]) + .await; + let fixture = poller_fixture(server.base_url.clone()).expect("poller is constructed"); + let head = CommitSha::try_new(HEAD_SHA.to_owned()).expect("fixture head is valid"); + let suite = + object_id(COMPLETED_CHECK_SUITE_IDS[0]).expect("fixture suite identity is positive"); + + let (_, every_gating_run_completed) = fixture + .poller + .fetch_check_runs(&head, std::slice::from_ref(&suite)) + .await + .expect("report-only run is valid check evidence"); + server.finish().await; + + assert!(every_gating_run_completed); + } + #[tokio::test] async fn a_complete_poll_normalizes_submitted_reviews() { let observation = complete_typed_observation().await; diff --git a/apps/signalboxd/src/telemetry.rs b/apps/signalboxd/src/telemetry.rs index 1feffe0475..f232db81b5 100644 --- a/apps/signalboxd/src/telemetry.rs +++ b/apps/signalboxd/src/telemetry.rs @@ -803,6 +803,8 @@ fn candidate_event(metadata: &Metadata<'_>) -> bool { | "turn_attempt_id" | "cause_code" | "terminal_outcome" + | "tool_round_limit" + | "observed_tool_rounds" ) }) } @@ -861,6 +863,20 @@ fn admitted_event_values(metadata: &Metadata<'_>, values: &RecordedValues) -> bo && values.uuid("session_id") && values.uuid("turn_id") } + ("signalbox_application::model_execution", "automatic tool-round limit reached") => { + values.has_exact(&[ + "message", + "model_call_id", + "observed_tool_rounds", + "session_id", + "tool_round_limit", + "turn_id", + ]) && values.uuid("session_id") + && values.uuid("turn_id") + && values.uuid("model_call_id") + && values.unsigned("tool_round_limit") + && values.unsigned("observed_tool_rounds") + } ("signalbox_model_provider_runtime", "model call dispatched") => { values.has_exact(&[ "message", @@ -920,6 +936,11 @@ impl RecordedValues { .filter(|(name, _value)| name.as_str() != "message") .map(|(name, value)| { let value = value.trim_matches('"'); + if matches!(name.as_str(), "tool_round_limit" | "observed_tool_rounds") { + let value = value.parse::().ok()?; + let value = i64::try_from(value).ok()?; + return Some(KeyValue::new(name.clone(), value)); + } let value = if name.ends_with("_id") { uuid::Uuid::parse_str(value).ok()?.to_string() } else { @@ -944,6 +965,12 @@ impl RecordedValues { .unwrap_or(false) } + fn unsigned(&self, name: &str) -> bool { + self.get(name) + .map(|value| value.trim_matches('"').parse::().is_ok()) + .unwrap_or(false) + } + fn closed(&self, name: &str, admitted: &[&str]) -> bool { self.get(name) .map(|value| admitted.contains(&value.trim_matches('"'))) @@ -995,6 +1022,7 @@ const TURN_OUTCOMES: &[&str] = &[ "cancelled_with_tool_response", "target_unavailable", "capability_known_failure", + "tool_round_limit_reached", "continuation_target_unavailable", ]; @@ -1425,8 +1453,8 @@ mod tests { names } - fn event_names(span: &SpanData) -> Vec { - let mut names = span.events[0] + fn event_names(span: &SpanData, event_index: usize) -> Vec { + let mut names = span.events[event_index] .attributes .iter() .map(|attribute| attribute.key.as_str().to_owned()) @@ -1604,6 +1632,8 @@ mod tests { #[test] fn admitted_span_and_event_export_only_the_documented_fields() { + let tool_round_limit = 32_usize; + let observed_tool_rounds = 32_usize; let spans = capture_spans(|| { let span = tracing::info_span!( target: "signalboxd::context_guard", @@ -1619,15 +1649,24 @@ mod tests { terminal_outcome = "completed", "turn terminalized" ); + tracing::warn!( + target: "signalbox_application::model_execution", + session_id = %SESSION_ID, + turn_id = %TURN_ID, + model_call_id = %MODEL_CALL_ID, + tool_round_limit, + observed_tool_rounds, + "automatic tool-round limit reached" + ); }); assert_eq!(spans.len(), 1); assert_eq!(spans[0].name, "turn_work"); assert_eq!(names(&spans[0]), vec!["session_id", "turn_id"]); - assert_eq!(spans[0].events.len(), 1); + assert_eq!(spans[0].events.len(), 2); assert_eq!(spans[0].events[0].name, "turn terminalized"); assert_eq!( - event_names(&spans[0]), + event_names(&spans[0], 0), vec![ "level", "session_id", @@ -1636,6 +1675,44 @@ mod tests { "turn_id" ] ); + assert_eq!( + spans[0].events[1].name, + "automatic tool-round limit reached" + ); + assert_eq!( + event_names(&spans[0], 1), + vec![ + "level", + "model_call_id", + "observed_tool_rounds", + "session_id", + "target", + "tool_round_limit", + "turn_id" + ] + ); + let saturation_attributes = &spans[0].events[1].attributes; + assert_eq!( + saturation_attributes + .iter() + .find(|attribute| attribute.key.as_str() == "tool_round_limit") + .expect("the saturation event carries its numeric limit") + .value, + opentelemetry::Value::I64( + i64::try_from(tool_round_limit).expect("the fixture limit fits OTLP I64") + ) + ); + assert_eq!( + saturation_attributes + .iter() + .find(|attribute| attribute.key.as_str() == "observed_tool_rounds") + .expect("the saturation event carries its numeric observed count") + .value, + opentelemetry::Value::I64( + i64::try_from(observed_tool_rounds) + .expect("the fixture observed count fits OTLP I64") + ) + ); } #[test] diff --git a/apps/signalboxd/src/turn_liveness_runtime.rs b/apps/signalboxd/src/turn_liveness_runtime.rs index 4762da2ed3..cf4206fdbe 100644 --- a/apps/signalboxd/src/turn_liveness_runtime.rs +++ b/apps/signalboxd/src/turn_liveness_runtime.rs @@ -20,6 +20,7 @@ use signalbox_domain::{ use signalbox_persistence::{ automatic_reconciliation::{ AutomaticReconciliationRepositoryError, PostgresAutomaticReconciliationRepository, + reconciliation_deadline, }, turn_liveness::{ PostgresTurnLivenessRepository, TurnLivenessPersistenceBounds, TurnLivenessRepositoryError, @@ -74,6 +75,9 @@ const STALE_TURN_LOCK_UNAVAILABLE_CAUSE: &str = "turn_liveness_scheduler_row_bus /// Why one turn-liveness pass produced no decision. const PASS_FAILURE_CAUSE: &str = "turn_liveness_pass_failed"; +/// Why a slot-held inventory read decided nothing within its bound. +const SLOT_HELD_PAGE_TIMEOUT_CAUSE: &str = "turn_liveness_slot_held_page_timed_out"; + /// Why a rotation was abandoned without deciding anything. const ROTATION_CEILING_CAUSE: &str = "turn_liveness_rotation_ceiling_reached"; @@ -94,17 +98,6 @@ const ROTATION_CEILING_CAUSE: &str = "turn_liveness_rotation_ceiling_reached"; /// candidates from it. // numeric-bound: guard - prevents a non-converging liveness inventory scan const QUIESCENT_ROTATION_PAGE_CEILING: usize = 4_096; -/// Client-side observation bound after PostgreSQL has been told to terminate -/// the recovery transaction at the supplied server bound. -/// -/// The second interval is only transport grace. If it expires, PostgreSQL has -/// already had a full transaction bound in which to terminate the backend, so -/// dropping the client future cannot leave live database work accumulating -/// behind the outbox allocator. -fn recovery_client_observation_bound(server_bound: Option) -> Option { - server_bound.map(|bound| bound.saturating_add(bound)) -} - /// Deployment policy for one turn-liveness scan. #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub struct TurnLivenessNumericBounds { @@ -327,9 +320,10 @@ pub struct TurnLivenessRuntime { impl TurnLivenessRuntime { /// Supervises turn liveness with the supplied bound and cadence. /// - /// The bound is a parameter rather than a reload of the compiled ceiling so - /// a deployment that validated a shorter one actually runs with it; the - /// ceiling stays the only maximum, enforced where the bound is built. + /// `staleness_bound` governs both quiescent and slot-held observation. The + /// required deployment configuration may disable stale-turn + /// terminalization with `none` while leaving ambiguity reconciliation + /// active. pub fn new( pool: PgPool, staleness_bound: Option, @@ -545,14 +539,22 @@ async fn drain_slot_held_rotation( let mut active = Vec::new(); let mut cursor = None; for _ in 0..QUIESCENT_ROTATION_PAGE_CEILING { - let page = read_slot_held_inventory_page(inventory, cursor, recovery_attempt_bound).await?; + let page = + read_slot_held_inventory_page(inventory, cursor, recovery_attempt_bound, "paging") + .await?; cursor = page.resume_after(); active.extend(page.into_candidates()); if cursor.is_none() { return Some(active); } } - let probe = read_slot_held_inventory_page(inventory, cursor, recovery_attempt_bound).await?; + let probe = read_slot_held_inventory_page( + inventory, + cursor, + recovery_attempt_bound, + "rotation_ceiling_probe", + ) + .await?; if probe.rows() == 0 { return Some(active); } @@ -569,6 +571,7 @@ async fn read_slot_held_inventory_page( inventory: &PostgresTurnLivenessRepository, cursor: Option, recovery_attempt_bound: Option, + phase: &'static str, ) -> Option { match optional_timeout( recovery_attempt_bound, @@ -582,11 +585,7 @@ async fn read_slot_held_inventory_page( None } Err(_) => { - tracing::warn!( - cause_code = "turn_liveness_slot_held_inventory_timed_out", - attempt_bound_seconds = ?recovery_attempt_bound.map(|bound| bound.as_secs()), - "slot-held inventory read exceeded its bound; the rotation made no decision" - ); + report_slot_held_page_timeout(phase, recovery_attempt_bound); None } } @@ -617,9 +616,9 @@ async fn reconcile_ambiguous_operations( .automatic_reconciliations_per_scan .is_none_or(|limit| reconciliations < limit) { - let claim = optional_timeout( - recovery_client_observation_bound(numeric_bounds.recovery_attempt_bound), - repository.claim_due(numeric_bounds.recovery_attempt_bound), + let claim = timeout( + reconciliation_deadline(numeric_bounds.recovery_attempt_bound), + repository.claim_due(), ); // Once PostgreSQL has begun a bounded transaction, shutdown lets that // transaction reach its server-enforced outcome instead of dropping @@ -656,9 +655,9 @@ async fn reconcile_ambiguous_operations( return; }; let (operation_kind, operation_id) = operation_log_fields(claimed.operation()); - let attempt = optional_timeout( - recovery_client_observation_bound(numeric_bounds.recovery_attempt_bound), - repository.reconcile(claimed, numeric_bounds.recovery_attempt_bound), + let attempt = timeout( + reconciliation_deadline(numeric_bounds.recovery_attempt_bound), + repository.reconcile(claimed), ); let attempt_outcome = attempt.await; match attempt_outcome { @@ -688,13 +687,9 @@ async fn reconcile_ambiguous_operations( commit_ambiguous: true } ) { - let record_failure = optional_timeout( - recovery_client_observation_bound(numeric_bounds.recovery_attempt_bound), - repository.record_failure( - claimed, - error.failure_kind(), - numeric_bounds.recovery_attempt_bound, - ), + let record_failure = timeout( + reconciliation_deadline(numeric_bounds.recovery_attempt_bound), + repository.record_failure(claimed, error.failure_kind()), ); let record_outcome = record_failure.await; match record_outcome { @@ -1089,16 +1084,35 @@ fn report_turn_liveness_failure(error: &TurnLivenessRepositoryError) { ); } +/// Reports a slot-held inventory read that exceeded its bound. +/// +/// The slot-held scan is the durable backstop for a turn whose scheduler pass +/// expired, so a read that keeps timing out silently would retire that backstop +/// invisibly: `reconcile_slot_held_turns` returns at its first statement on +/// every scan and no turn is ever reached. Every sibling bound in this file and +/// in the scheduler-expiry path emits a cause code, and so does this one. +fn report_slot_held_page_timeout(phase: &'static str, recovery_attempt_bound: Option) { + tracing::error!( + failure_class = ?signalbox_application::OperatorFailureClass::Infrastructure { commit_ambiguous: false }, + cause_code = SLOT_HELD_PAGE_TIMEOUT_CAUSE, + phase, + attempt_bound_seconds = ?recovery_attempt_bound.map(|bound| bound.as_secs()), + "slot-held inventory read exceeded its bound; the slot-held backstop made no progress this scan" + ); +} + #[cfg(test)] mod tests { + use signalbox_persistence::automatic_reconciliation::RECONCILIATION_LOCK_WAIT; + use super::{ InventoryPage, PASS_FAILURE_CAUSE, QUIESCENT_ROTATION_PAGE_CEILING, QuiescentInventory, - ROTATION_CEILING_CAUSE, STALE_TURN_AMBIGUOUS_CAUSE, STALE_TURN_LOCK_UNAVAILABLE_CAUSE, - STALE_TURN_STEERING_BLOCKED_CAUSE, STALE_TURN_SUPERSEDED_CAUSE, STALE_TURN_TERMINAL_CAUSE, - StaleTurnTerminalizer, TERMINALIZATION_DEFERRED_CAUSE, TerminalizationWindow, - TurnLivenessNumericBounds, TurnLivenessWake, complete_before_shutdown, - drain_quiescent_rotation, next_turn_liveness_wake, reconcile_turn_liveness, - recovery_client_observation_bound, + ROTATION_CEILING_CAUSE, SLOT_HELD_PAGE_TIMEOUT_CAUSE, STALE_TURN_AMBIGUOUS_CAUSE, + STALE_TURN_LOCK_UNAVAILABLE_CAUSE, STALE_TURN_STEERING_BLOCKED_CAUSE, + STALE_TURN_SUPERSEDED_CAUSE, STALE_TURN_TERMINAL_CAUSE, StaleTurnTerminalizer, + TERMINALIZATION_DEFERRED_CAUSE, TerminalizationWindow, TurnLivenessNumericBounds, + TurnLivenessWake, complete_before_shutdown, drain_quiescent_rotation, + next_turn_liveness_wake, reconcile_turn_liveness, reconciliation_deadline, }; use signalbox_application::{ StaleActiveTurnBound, StaleTurnCandidate, StaleTurnOutcome, TurnLivenessEvidence, @@ -1642,6 +1656,10 @@ mod tests { STALE_TURN_LOCK_UNAVAILABLE_CAUSE, "turn_liveness_scheduler_row_busy" ); + assert_eq!( + SLOT_HELD_PAGE_TIMEOUT_CAUSE, + "turn_liveness_slot_held_page_timed_out" + ); assert_ne!(STALE_TURN_SUPERSEDED_CAUSE, STALE_TURN_TERMINAL_CAUSE); assert_ne!(STALE_TURN_AMBIGUOUS_CAUSE, STALE_TURN_TERMINAL_CAUSE); assert_ne!(STALE_TURN_STEERING_BLOCKED_CAUSE, STALE_TURN_TERMINAL_CAUSE); @@ -1662,14 +1680,17 @@ mod tests { } /// The example keeps recovery bounded while production may choose `none`. + /// + /// The configured bound is now the reconciliation transaction's last-resort + /// client deadline; the database-side budgets it must sit above are pinned + /// in `signalbox_persistence::automatic_reconciliation`. #[test] fn recovery_attempts_use_the_configured_bound() { assert!(example_numeric_bounds().recovery_attempt_bound.is_some()); assert_eq!( - recovery_client_observation_bound(Some(Duration::from_secs(60))), - Some(Duration::from_secs(120)) + reconciliation_deadline(Some(Duration::from_secs(60))), + Duration::from_secs(60) ); - assert_eq!(recovery_client_observation_bound(None), None); } #[test] @@ -1701,4 +1722,24 @@ mod tests { assert_eq!(outcome, Some(7)); } + + /// The reconciliation stages are bounded above their database-side lock + /// budget, so a contended row ends as `55P03` under the caller's timer + /// rather than as a dropped future that leaves the backend still waiting. + /// + /// Stated here as well as in the compile-time assertion because the two + /// answer different questions: that one rejects a margin that has vanished, + /// this one records which side of the pair the margin belongs to. + #[test] + fn the_reconciliation_bound_outlasts_its_database_budget() { + assert_eq!(RECONCILIATION_LOCK_WAIT, Duration::from_secs(1)); + assert!( + reconciliation_deadline(None) > RECONCILIATION_LOCK_WAIT, + "the database-side budget has to be the one that expires first" + ); + assert!( + reconciliation_deadline(Some(Duration::from_millis(1))) > RECONCILIATION_LOCK_WAIT, + "a configured bound below the floor is raised, never honoured as-is" + ); + } } diff --git a/apps/signalboxd/src/usage_limits.rs b/apps/signalboxd/src/usage_limits.rs index 3138460581..c11db81dc6 100644 --- a/apps/signalboxd/src/usage_limits.rs +++ b/apps/signalboxd/src/usage_limits.rs @@ -276,6 +276,9 @@ where ModelCallCapabilityPreparation::KnownFailure => { ModelCallCapabilityPreparation::KnownFailure } + ModelCallCapabilityPreparation::AttachmentFailure(failure) => { + ModelCallCapabilityPreparation::AttachmentFailure(failure) + } }) } diff --git a/apps/signalboxd/src/web_http.rs b/apps/signalboxd/src/web_http.rs index a900bee14b..6aecffbedb 100644 --- a/apps/signalboxd/src/web_http.rs +++ b/apps/signalboxd/src/web_http.rs @@ -29,7 +29,7 @@ use futures_util::{Stream, StreamExt, stream}; use serde::{Deserialize, Serialize, de::DeserializeOwned}; use signalbox_application::{ SessionTimelineDescriptor, SessionTimelineEventKind, SessionTimelineWindow, TimelineAddress, - TimelineWindowAnchor, TimelineWindowLimits, + TimelineContinuation, TimelineWindowAnchor, TimelineWindowLimits, }; use signalbox_domain::SessionId; use signalbox_persistence::session_timeline::{ @@ -37,15 +37,18 @@ use signalbox_persistence::session_timeline::{ }; use signalbox_web_contract::{ MAX_JSON_BODY_BYTES, MAX_NDJSON_ITEM_BYTES, WebApiError, WebApiErrorKind, WebApiErrorResponse, - WebContractBootstrap, WebContractExample, WebSessionTimelineDescriptor, + WebContractBootstrap, WebContractExample, WebSessionId, WebSessionTimelineDescriptor, WebSessionTimelineEventKind, WebSessionTimelineItem, WebSessionTimelineSizeFacts, - WebSessionTimelineWindow, WebSessionWorkFacts, WebTimelineAddress, + WebSessionTimelineWindow, WebSessionWorkFacts, WebTimelineAddress, WebTimelineEventSequence, + WebU64, }; use sqlx::PgPool; use tokio::{net::TcpListener, sync::watch}; use tower_http::services::{ServeDir, ServeFile}; use url::Url; +use crate::{HubModelConfiguration, web_imports}; + /// Optional deployment override for the browser listener. pub const WEB_BIND_ENVIRONMENT: &str = "SIGNALBOX_WEB_BIND"; /// Optional production web-build root served outside `/api/`. @@ -55,6 +58,7 @@ pub const DEFAULT_WEB_BIND_ADDRESS: SocketAddr = SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 37_231); const JSON_CONTENT_TYPE: &str = "application/json"; +const TEXT_CONTENT_TYPE: &str = "text/plain"; const NDJSON_CONTENT_TYPE: &str = "application/x-ndjson"; const HTTP_DEFAULT_PORT: u16 = 80; @@ -86,9 +90,7 @@ impl WebHttpConfiguration { .parse() .map_err(|_| WebHttpConfigurationError::InvalidBindAddress)?, }; - if !bind_address.ip().is_loopback() { - return Err(WebHttpConfigurationError::NonLoopbackBindAddress); - } + validate_loopback_bind_address(bind_address)?; let asset_root = match asset_root { None => None, Some(value) if value.is_empty() => { @@ -102,14 +104,12 @@ impl WebHttpConfiguration { }) } - /// Creates explicit loopback configuration for an embedded production server. + /// Creates explicit loopback-only configuration for a deterministic or embedded server. pub fn new( bind_address: SocketAddr, asset_root: Option, ) -> Result { - if !bind_address.ip().is_loopback() { - return Err(WebHttpConfigurationError::NonLoopbackBindAddress); - } + validate_loopback_bind_address(bind_address)?; Ok(Self { bind_address, asset_root, @@ -129,6 +129,16 @@ impl WebHttpConfiguration { } } +fn validate_loopback_bind_address( + bind_address: SocketAddr, +) -> Result<(), WebHttpConfigurationError> { + if bind_address.ip().is_loopback() { + Ok(()) + } else { + Err(WebHttpConfigurationError::NonLoopbackBindAddress) + } +} + /// Closed configuration failures that never expose rejected values. #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub enum WebHttpConfigurationError { @@ -201,8 +211,13 @@ impl WebHttpRuntime { pub async fn bind( configuration: WebHttpConfiguration, pool: PgPool, + model_configuration: HubModelConfiguration, ) -> Result { - let router = production_router(configuration.asset_root, Some(pool)); + let router = production_router( + configuration.asset_root, + Some(pool), + Some(model_configuration), + ); Self::bind_router(configuration.bind_address, router).await } @@ -244,28 +259,47 @@ impl WebHttpRuntime { } /// Builds the production router: `/api/` remains API-only and assets share its origin. -pub fn production_router(asset_root: Option, pool: Option) -> Router { +pub fn production_router( + asset_root: Option, + pool: Option, + model_configuration: Option, +) -> Router { let state = WebApiState { - timeline: pool.map(SessionTimelineRepository::new), + timeline: pool.clone().map(SessionTimelineRepository::new), }; - let api = Router::new() - .route("/bootstrap", get(contract_bootstrap)) + let session_reads = Router::new() .route("/sessions/{session_id}", get(session_descriptor)) .route( "/sessions/{session_id}/timeline", get(session_timeline_window), - ) - .with_state(state) - .fallback(api_not_found); + ); + let api = Router::new() + .route("/bootstrap", get(contract_bootstrap)) + .merge(session_reads) + .with_state(state); + // Imported-conversation reads need both a pool and hub model settings; the + // bootstrap and session surfaces stay routable without either. + let api = match (pool, model_configuration) { + (Some(pool), Some(model_configuration)) => { + api.nest("/imports", web_imports::router(pool, model_configuration)) + } + _ => api, + }; + let api = api.fallback(api_not_found); + same_origin_router(asset_root, api) +} + +fn same_origin_router(asset_root: Option, api: Router) -> Router { let router = Router::new().nest("/api", api); - match asset_root { + let router = match asset_root { Some(root) => router.fallback_service( ServeDir::new(root.clone()) .append_index_html_on_directories(true) .fallback(ServeFile::new(root.join("index.html"))), ), None => router.fallback(static_assets_not_configured), - } + }; + router.layer(middleware::from_fn(validate_loopback_host)) } #[derive(Clone, Debug)] @@ -460,7 +494,7 @@ fn parse_window_anchor( fn address_dto(address: TimelineAddress) -> WebTimelineAddress { WebTimelineAddress { - event_sequence: address.sequence().get().to_string(), + event_sequence: WebTimelineEventSequence::from_nonzero(address.sequence()), } } @@ -474,35 +508,37 @@ fn descriptor_dto( return Err(SessionTimelineRequestError::MissingBounds); }; Ok(WebSessionTimelineDescriptor { - session_id: descriptor.session.into_uuid().to_string(), + session_id: WebSessionId::from_uuid_bytes(*descriptor.session.into_uuid().as_bytes()), sizes: WebSessionTimelineSizeFacts { - item_count: descriptor.sizes.item_count.to_string(), - projected_text_bytes: descriptor.sizes.projected_text_bytes.to_string(), - projected_structured_bytes: descriptor.sizes.projected_structured_bytes.to_string(), - referenced_blob_count: descriptor.sizes.referenced_blob_count.to_string(), - referenced_blob_bytes: descriptor.sizes.referenced_blob_bytes.to_string(), + item_count: WebU64::from_u64(descriptor.sizes.item_count), + projected_text_bytes: WebU64::from_u64(descriptor.sizes.projected_text_bytes), + projected_structured_bytes: WebU64::from_u64( + descriptor.sizes.projected_structured_bytes, + ), + referenced_blob_count: WebU64::from_u64(descriptor.sizes.referenced_blob_count), + referenced_blob_bytes: WebU64::from_u64(descriptor.sizes.referenced_blob_bytes), }, first_address: address_dto(first_address), latest_address: address_dto(latest_address), work: WebSessionWorkFacts { - active_turn_count: descriptor.work.active_turn_count.to_string(), - queued_turn_count: descriptor.work.queued_turn_count.to_string(), + active_turn_count: WebU64::from_u64(descriptor.work.active_turn_count), + queued_turn_count: WebU64::from_u64(descriptor.work.queued_turn_count), }, - observed_through: descriptor.observed_through.to_string(), + observed_through: WebU64::from_u64(descriptor.observed_through), }) } fn window_dto(window: SessionTimelineWindow) -> WebSessionTimelineWindow { - let continuation_before = window - .has_more_before - .then(|| window.items.first().map(|item| address_dto(item.address))) - .flatten(); - let continuation_after = window - .has_more_after - .then(|| window.items.last().map(|item| address_dto(item.address))) - .flatten(); + let continuation_before = match window.continuation_before { + TimelineContinuation::Exhausted => None, + TimelineContinuation::MoreAt(address) => Some(address_dto(address)), + }; + let continuation_after = match window.continuation_after { + TimelineContinuation::Exhausted => None, + TimelineContinuation::MoreAt(address) => Some(address_dto(address)), + }; WebSessionTimelineWindow { - session_id: window.session.into_uuid().to_string(), + session_id: WebSessionId::from_uuid_bytes(*window.session.into_uuid().as_bytes()), items: window .items .into_iter() @@ -565,7 +601,7 @@ pub fn deterministic_test_router() -> Router { .route("/mutate", post(deterministic_mutation)) .route_layer(middleware::from_fn(validate_json_mutation)); let api = Router::new() - .route("/bootstrap", get(contract_bootstrap)) + .route("/bootstrap", get(deterministic_contract_bootstrap)) .route("/test/read", get(deterministic_read)) .route("/test/stream", get(deterministic_stream)) .nest("/test", mutation) @@ -580,6 +616,12 @@ async fn contract_bootstrap() -> Json { Json(WebContractBootstrap::current()) } +async fn deterministic_contract_bootstrap() -> Json { + let mut bootstrap = WebContractBootstrap::current(); + bootstrap.capabilities.bounded_session_timeline = false; + Json(bootstrap) +} + fn deterministic_example() -> WebContractExample { WebContractExample { request_id: "deterministic-request".to_owned(), @@ -658,6 +700,37 @@ where }) } +/// Decodes one UTF-8 request body after enforcing a caller-owned byte ceiling. +pub(crate) async fn decode_bounded_utf8( + request: Request, + maximum_bytes: usize, +) -> Result { + let bytes = to_bytes(request.into_body(), maximum_bytes) + .await + .map_err(|error| { + if error_chain_contains_length_limit(&error) { + transport_error( + StatusCode::PAYLOAD_TOO_LARGE, + "text_body_too_large", + "text request body exceeds the configured import limit", + ) + } else { + transport_error( + StatusCode::BAD_REQUEST, + "text_body_read_failed", + "text request body could not be read", + ) + } + })?; + String::from_utf8(bytes.to_vec()).map_err(|_| { + transport_error( + StatusCode::BAD_REQUEST, + "invalid_utf8", + "request body is not valid UTF-8", + ) + }) +} + fn error_chain_contains_length_limit(error: &axum::Error) -> bool { let mut current: Option<&(dyn Error + 'static)> = Some(error); while let Some(error) = current { @@ -738,7 +811,7 @@ impl io::Write for NdjsonItemWriter { } } -async fn validate_json_mutation(request: Request, next: Next) -> Response { +pub(crate) async fn validate_json_mutation(request: Request, next: Next) -> Response { if request.method() != Method::POST { return transport_error( StatusCode::METHOD_NOT_ALLOWED, @@ -763,12 +836,77 @@ async fn validate_json_mutation(request: Request, next: Next) -> Response { next.run(request).await } +pub(crate) async fn validate_text_mutation(request: Request, next: Next) -> Response { + if request.method() != Method::POST { + return transport_error( + StatusCode::METHOD_NOT_ALLOWED, + "mutation_method_not_allowed", + "browser mutations use POST", + ); + } + if !has_content_type(request.headers(), TEXT_CONTENT_TYPE) { + return transport_error( + StatusCode::UNSUPPORTED_MEDIA_TYPE, + "text_content_type_required", + "exact import searches require text/plain", + ); + } + if validate_supplied_origin(request.headers()).is_err() { + return transport_error( + StatusCode::FORBIDDEN, + "cross_origin_mutation_rejected", + "mutation origin does not match request authority", + ); + } + next.run(request).await +} + +async fn validate_loopback_host(request: Request, next: Next) -> Response { + if !has_loopback_host(request.headers(), request.uri()) { + return transport_error( + StatusCode::FORBIDDEN, + "non_loopback_host_rejected", + "browser requests require a loopback request authority", + ); + } + next.run(request).await +} + +fn has_loopback_host(headers: &HeaderMap, uri: &axum::http::Uri) -> bool { + headers + .get(HOST) + .and_then(|host| host.to_str().ok()) + .and_then(|host| host.parse::().ok()) + .or_else(|| uri.authority().cloned()) + .is_some_and(|authority| is_loopback_authority(&authority)) +} + +fn is_loopback_authority(authority: &axum::http::uri::Authority) -> bool { + let host = normalized_authority_host(authority); + host.eq_ignore_ascii_case("localhost") + || host + .parse::() + .is_ok_and(|address| address.is_loopback()) +} + +fn normalized_authority_host(authority: &axum::http::uri::Authority) -> &str { + normalized_host(authority.host()) +} + +fn normalized_host(host: &str) -> &str { + host.trim_start_matches('[').trim_end_matches(']') +} + fn has_json_content_type(headers: &HeaderMap) -> bool { + has_content_type(headers, JSON_CONTENT_TYPE) +} + +fn has_content_type(headers: &HeaderMap, expected: &str) -> bool { headers .get(CONTENT_TYPE) .and_then(|value| value.to_str().ok()) .and_then(|value| value.split(';').next()) - .is_some_and(|value| value.trim().eq_ignore_ascii_case(JSON_CONTENT_TYPE)) + .is_some_and(|value| value.trim().eq_ignore_ascii_case(expected)) } #[derive(Clone, Copy, Debug, Eq, PartialEq)] @@ -794,10 +932,9 @@ fn validate_supplied_origin(headers: &HeaderMap) -> Result<(), OriginValidationE .and_then(|host| host.parse::().ok()); let matching = origin.zip(authority).is_some_and(|(origin, authority)| { let authority_port = authority.port_u16().unwrap_or(HTTP_DEFAULT_PORT); - origin - .host_str() - .is_some_and(|host| host.eq_ignore_ascii_case(authority.host())) - && origin.port_or_known_default() == Some(authority_port) + origin.host_str().is_some_and(|host| { + normalized_host(host).eq_ignore_ascii_case(normalized_authority_host(&authority)) + }) && origin.port_or_known_default() == Some(authority_port) }); if matching { Ok(()) @@ -806,7 +943,11 @@ fn validate_supplied_origin(headers: &HeaderMap) -> Result<(), OriginValidationE } } -fn transport_error(status: StatusCode, code: &'static str, message: &'static str) -> Response { +pub(crate) fn transport_error( + status: StatusCode, + code: &'static str, + message: &'static str, +) -> Response { let body = Json(WebApiErrorResponse { error: WebApiError { kind: WebApiErrorKind::Transport, @@ -817,18 +958,19 @@ fn transport_error(status: StatusCode, code: &'static str, message: &'static str (status, body).into_response() } -fn application_error(status: StatusCode, code: &'static str, message: &'static str) -> Response { - ( - status, - Json(WebApiErrorResponse { - error: WebApiError { - kind: WebApiErrorKind::Application, - code: code.to_owned(), - message: message.to_owned(), - }, - }), - ) - .into_response() +pub(crate) fn application_error( + status: StatusCode, + code: &'static str, + message: &'static str, +) -> Response { + let body = Json(WebApiErrorResponse { + error: WebApiError { + kind: WebApiErrorKind::Application, + code: code.to_owned(), + message: message.to_owned(), + }, + }); + (status, body).into_response() } async fn api_not_found() -> Response { @@ -918,20 +1060,24 @@ mod tests { } #[test] - fn non_loopback_bind_is_rejected_without_authentication() { + fn non_loopback_bind_fails_closed() { let error = WebHttpConfiguration::from_values(Some(OsString::from("0.0.0.0:8080")), None) - .expect_err("unauthenticated browser routes remain loopback-only"); + .expect_err("the unauthenticated browser surface remains loopback-only"); assert_eq!(error, WebHttpConfigurationError::NonLoopbackBindAddress); + assert_eq!( + error.to_string(), + "setting SIGNALBOX_WEB_BIND must use a loopback address" + ); } #[test] - fn explicit_non_loopback_configuration_is_rejected() { - let bind_address = "0.0.0.0:8080" + fn explicit_constructor_rejects_non_loopback_bind() { + let bind_address: SocketAddr = "0.0.0.0:8080" .parse() .expect("the fixture address is valid"); let error = WebHttpConfiguration::new(bind_address, None) - .expect_err("every production configuration remains loopback-only"); + .expect_err("every production configuration path remains loopback-only"); assert_eq!(error, WebHttpConfigurationError::NonLoopbackBindAddress); } @@ -957,7 +1103,7 @@ mod tests { .expect("the static index exists"); let runtime = WebHttpRuntime::bind_router( loopback_ephemeral(), - production_router(Some(assets.path().to_path_buf()), None), + production_router(Some(assets.path().to_path_buf()), None, None), ) .await .expect("the production test server binds"); @@ -1020,6 +1166,28 @@ mod tests { assert_eq!(decoded, example()); } + #[tokio::test] + async fn mutation_with_matching_ipv6_origin_round_trips_bounded_json() { + let request = Request::post("/api/test/mutate") + .header(header::HOST, "[::1]:37231") + .header(header::ORIGIN, "http://[::1]:37231") + .header(header::CONTENT_TYPE, "application/json; charset=utf-8") + .body(Body::from( + serde_json::to_vec(&example()).expect("the fixture serializes"), + )) + .expect("the request is valid"); + let response = deterministic_test_router() + .oneshot(request) + .await + .expect("the deterministic router responds"); + let status = response.status(); + let decoded: WebContractExample = serde_json::from_slice(&response_body(response).await) + .expect("the response is the example DTO"); + + assert_eq!(status, StatusCode::OK); + assert_eq!(decoded, example()); + } + #[tokio::test] async fn responses_do_not_emit_permissive_cors() { let request = Request::get("/api/bootstrap") @@ -1175,9 +1343,10 @@ mod tests { std::fs::write(assets.path().join("index.html"), "static fallback") .expect("the static index exists"); let request = Request::get("/api/not-a-route") + .header(header::HOST, "127.0.0.1") .body(Body::empty()) .expect("the request is valid"); - let response = production_router(Some(assets.path().to_path_buf()), None) + let response = production_router(Some(assets.path().to_path_buf()), None, None) .oneshot(request) .await .expect("the production router responds"); @@ -1189,14 +1358,53 @@ mod tests { assert_eq!(body["error"]["code"], "api_route_not_found"); } + #[tokio::test] + async fn production_router_rejects_non_loopback_hostnames() { + let request = Request::get("/api/bootstrap") + .header(header::HOST, "attacker.example") + .body(Body::empty()) + .expect("the request is valid"); + let response = production_router(None, None, None) + .oneshot(request) + .await + .expect("the production router responds"); + let status = response.status(); + let body: serde_json::Value = + serde_json::from_slice(&response_body(response).await).expect("the rejection is JSON"); + + assert_eq!(status, StatusCode::FORBIDDEN); + assert_eq!(body["error"]["code"], "non_loopback_host_rejected"); + } + + #[test] + fn loopback_host_accepts_localhost_and_uri_authority() { + let localhost = Request::get("/api/bootstrap") + .header(header::HOST, "localhost:37231") + .body(Body::empty()) + .expect("the localhost request is valid"); + let authority = Request::get("http://127.0.0.1:37231/api/bootstrap") + .body(Body::empty()) + .expect("the authority request is valid"); + + assert!(super::has_loopback_host( + localhost.headers(), + localhost.uri() + )); + assert!(super::has_loopback_host( + authority.headers(), + authority.uri() + )); + } + #[tokio::test] async fn malformed_timeline_query_uses_the_structured_error_envelope() { let request = Request::get( "/api/sessions/00000000-0000-0000-0000-000000000991/timeline?max_items=nope", ) + .header(header::HOST, "localhost") .body(Body::empty()) .expect("the request is valid"); - let response = production_router(None, None) + let response = production_router(None, None, None) .oneshot(request) .await .expect("the production router responds"); @@ -1214,9 +1422,10 @@ mod tests { let request = Request::get( "/api/sessions/00000000-0000-0000-0000-000000000991/timeline?anchor=first&max_items=1", ) + .header(header::HOST, "localhost") .body(Body::empty()) .expect("the request is valid"); - let response = production_router(None, None) + let response = production_router(None, None, None) .oneshot(request) .await .expect("the production router responds"); @@ -1228,6 +1437,103 @@ mod tests { assert_eq!(body["error"]["code"], "invalid_timeline_limits"); } + #[tokio::test] + async fn session_reads_reject_non_loopback_host_authorities() { + let request = Request::get("/api/sessions/00000000-0000-0000-0000-000000000991") + .header(header::HOST, "attacker.example") + .body(Body::empty()) + .expect("the request is valid"); + let response = production_router(None, None, None) + .oneshot(request) + .await + .expect("the production router responds"); + let status = response.status(); + let body: serde_json::Value = serde_json::from_slice(&response_body(response).await) + .expect("the rejection is structured JSON"); + + assert_eq!(status, StatusCode::FORBIDDEN); + assert_eq!(body["error"]["kind"], "transport"); + assert_eq!(body["error"]["code"], "non_loopback_host_rejected"); + } + + /// Drives a session read at the loopback gate and reports only the status. + /// + /// The query is deliberately malformed, which separates the gate from + /// everything behind it: `FORBIDDEN` means the gate rejected the + /// authority, while `BAD_REQUEST` comes from the handler and is therefore + /// reachable only once the gate has admitted the request. + async fn session_read_status_for_host(host: &str) -> StatusCode { + let request = Request::get( + "/api/sessions/00000000-0000-0000-0000-000000000991/timeline?max_items=nope", + ) + .header(header::HOST, host) + .body(Body::empty()) + .expect("the request is valid"); + production_router(None, None, None) + .oneshot(request) + .await + .expect("the production router responds") + .status() + } + + #[tokio::test] + async fn session_reads_admit_loopback_authorities_including_ip_literals() { + // `127.0.0.1` is the daemon's own DEFAULT_WEB_BIND_ADDRESS, so a + // regression that tightened this branch would 403 the default + // deployment. `[::1]` exercises the bracket strip that precedes the + // parse, and `127.5.6.7` covers the whole 127.0.0.0/8 loopback range + // rather than only the canonical address. + for host in [ + "localhost", + "localhost:37231", + "LocalHost", + "127.0.0.1", + "127.0.0.1:37231", + "127.5.6.7", + "[::1]", + "[::1]:37231", + ] { + assert_eq!( + session_read_status_for_host(host).await, + StatusCode::BAD_REQUEST, + "`{host}` is a loopback authority and must reach the handler", + ); + } + } + + #[tokio::test] + async fn session_reads_reject_non_loopback_ip_literal_authorities() { + // Every authority here parses as an address, so `is_loopback` — not + // the `parse::()` that already turns hostnames away — is what + // has to reject them. A regression that loosened the branch to accept + // any parseable address would expose session history to any host that + // can reach the port. + for host in [ + "10.0.0.5", + "10.0.0.5:37231", + "192.168.1.20", + "[2001:db8::1]", + "[2001:db8::1]:37231", + ] { + assert_eq!( + session_read_status_for_host(host).await, + StatusCode::FORBIDDEN, + "`{host}` parses as a non-loopback address and must be rejected", + ); + } + } + + #[tokio::test] + async fn session_reads_reject_authorities_that_are_neither_localhost_nor_literals() { + for host in ["attacker.example", "localhost.attacker.example"] { + assert_eq!( + session_read_status_for_host(host).await, + StatusCode::FORBIDDEN, + "`{host}` is neither localhost nor a loopback literal", + ); + } + } + #[test] fn timeline_addresses_require_canonical_positive_decimal() { assert!(super::parse_window_anchor("after", Some("+5")).is_err()); diff --git a/apps/signalboxd/src/web_imports.rs b/apps/signalboxd/src/web_imports.rs new file mode 100644 index 0000000000..438f755e16 --- /dev/null +++ b/apps/signalboxd/src/web_imports.rs @@ -0,0 +1,900 @@ +//! Production browser adapter for bounded imported-conversation discovery. + +use std::num::NonZeroU32; + +use axum::{ + Json, Router, + extract::{Path, Query, State}, + http::StatusCode, + middleware, + response::{IntoResponse, Response}, + routing::{get, post}, +}; +use sha2::Digest; +use signalbox_application::{ + CreateSessionFromImportedFrontierOutcome, CreateSessionFromImportedFrontierRequest, + CreateSessionFromImportedFrontierService, UuidV7CreateSessionFromImportedFrontierIdGenerator, +}; +use signalbox_domain::{ + DirectModelSelection, DurableCommandId, ImportedConversationFormat, ImportedConversationId, + ImportedSessionRelationship, ImportedSourceAttestation, ImportedSpeaker, + ImportedTranscriptEntryId, ImportedTranscriptFrontier, ImportedTranscriptPosition, ModelAlias, + ModelSelectionRequest, SessionConfigurationDefaults, +}; +use signalbox_persistence::{ + conversation_import_discovery::{ + ImportedContinuationReference, ImportedConversationDescriptor, + ImportedConversationDiscoveryError, ImportedConversationDiscoveryRepository, + ImportedConversationDiscoveryRequestError, ImportedConversationPageRequest, + ImportedConversationSummary, ImportedEntryContentProjection, ImportedEntryProjection, + ImportedEntryWindow, ImportedEntryWindowAnchor, ImportedTextProjection, + }, + create_session_from_imported_frontier::{ + ImportedSessionRepository, ImportedSessionRepositoryError, + }, +}; +use signalbox_web_contract::{ + MAX_IMPORT_ENTRY_WINDOW_ITEMS, MAX_IMPORT_LIST_ITEMS, MAX_IMPORT_SOURCE_SESSION_BYTES, + MAX_IMPORT_TEXT_PREVIEW_BYTES, WebImportContinuationReference, WebImportContinuationRequest, + WebImportContinuationResponse, WebImportDescriptor, WebImportEntryWindow, + WebImportEntryWindowRequest, WebImportFormat, WebImportListPage, WebImportListRequest, + WebImportSizeFacts, WebImportSourceEvidence, WebImportSourceSessionEvidence, WebImportSummary, + WebImportTextCompleteness, WebImportTextEvidence, WebImportTimelineBounds, + WebImportWindowAnchor, WebImportedContentKind, WebImportedEntry, + WebImportedSessionRelationship, WebImportedSpeakerEvidence, WebModelSelection, +}; +use sqlx::PgPool; +use uuid::Uuid; + +use crate::{ + HubModelConfiguration, + web_http::{ + application_error, decode_bounded_json, decode_bounded_utf8, transport_error, + validate_json_mutation, validate_text_mutation, + }, +}; + +// Tunable effective ceiling: ordinary catalog views ask for a dense first page below the hard cap. +const DEFAULT_IMPORT_LIST_ITEMS: u32 = 50; +// Tunable effective ceiling: default entry windows retain 25 neighbors on each side. +const DEFAULT_IMPORT_WINDOW_RADIUS: u32 = 25; + +#[derive(Clone, Debug)] +struct WebImportState { + pool: PgPool, + model_configuration: HubModelConfiguration, +} + +pub(crate) fn router(pool: PgPool, model_configuration: HubModelConfiguration) -> Router { + let state = WebImportState { + pool, + model_configuration, + }; + let mutation = Router::new() + .route("/{conversation}/continuations", post(continue_import)) + .route_layer(middleware::from_fn(validate_json_mutation)); + let searches = Router::new() + .route("/searches", post(search_imports)) + .route_layer(middleware::from_fn(validate_text_mutation)); + Router::new() + .route("/", get(list_imports)) + .route("/{conversation}", get(read_descriptor)) + .route("/{conversation}/entries", get(read_entry_window)) + .merge(searches) + .merge(mutation) + .with_state(state) +} + +async fn list_imports( + State(state): State, + request: Result, axum::extract::rejection::QueryRejection>, +) -> Response { + let Query(request) = match request { + Ok(request) => request, + Err(_) => return invalid_request("imports query is malformed"), + }; + if request.source_session_id.is_some() || request.search_correlation.is_some() { + return invalid_request("exact source-session filters use the bounded search body"); + } + execute_list_imports(state, request, None).await +} + +async fn search_imports( + State(state): State, + query: Result, axum::extract::rejection::QueryRejection>, + request: axum::extract::Request, +) -> Response { + let Query(mut catalog_request) = match query { + Ok(request) => request, + Err(_) => return invalid_request("imports search query is malformed"), + }; + if catalog_request.source_session_id.is_some() { + return invalid_request("exact source-session filters belong in the search body"); + } + let search_correlation = match catalog_request.search_correlation.as_deref() { + Some(value) if required_uuid(value).is_ok() => Some(value.to_owned()), + Some(_) => return invalid_request("imports search correlation is not a UUID"), + None => return invalid_request("imports search correlation is required"), + }; + let maximum_bytes = state + .model_configuration + .conversation_import_max_source_bytes(); + let source_session_id = match decode_bounded_utf8(request, maximum_bytes).await { + Ok(source_session_id) => source_session_id, + Err(response) => return response, + }; + catalog_request.source_session_id = Some(source_session_id); + execute_list_imports(state, catalog_request, search_correlation).await +} + +async fn execute_list_imports( + state: WebImportState, + request: WebImportListRequest, + search_correlation: Option, +) -> Response { + let limit = request.limit.unwrap_or(DEFAULT_IMPORT_LIST_ITEMS); + let Some(limit) = NonZeroU32::new(limit).filter(|limit| limit.get() <= MAX_IMPORT_LIST_ITEMS) + else { + return invalid_request("imports limit is outside the contract bound"); + }; + let after = match optional_uuid(request.after.as_deref()) { + Ok(after) => after.map(ImportedConversationId::from_uuid), + Err(message) => return invalid_request(message), + }; + let Some(source_session_maximum_bytes) = source_session_maximum_bytes() else { + return invalid_import_contract(); + }; + let exact_source_session_id_sha256 = request + .source_session_id + .as_deref() + .map(|value| lowercase_hex(&sha2::Sha256::digest(value.as_bytes()))); + let query = ImportedConversationPageRequest { + after, + format: request.format.map(domain_format), + source_session_id: request.source_session_id.map(String::into_bytes), + source_session_maximum_bytes, + limit, + }; + match ImportedConversationDiscoveryRepository::new(state.pool) + .list(query) + .await + { + Ok(page) => Json(WebImportListPage { + items: page.items.into_iter().map(web_summary).collect(), + next_cursor: page.next_after.map(|cursor| cursor.into_uuid().to_string()), + search_correlation, + exact_source_session_id_sha256, + }) + .into_response(), + Err(error) => discovery_error(error), + } +} + +async fn read_descriptor( + State(state): State, + Path(conversation): Path, +) -> Response { + let conversation = match imported_conversation_id(&conversation) { + Ok(conversation) => conversation, + Err(message) => return invalid_request(message), + }; + let Some(source_session_maximum_bytes) = source_session_maximum_bytes() else { + return invalid_import_contract(); + }; + match ImportedConversationDiscoveryRepository::new(state.pool) + .descriptor(conversation, source_session_maximum_bytes) + .await + { + Ok(Some(descriptor)) => Json(web_descriptor(descriptor)).into_response(), + Ok(None) => import_not_found(), + Err(error) => discovery_error(error), + } +} + +async fn read_entry_window( + State(state): State, + Path(conversation): Path, + request: Result, axum::extract::rejection::QueryRejection>, +) -> Response { + let conversation = match imported_conversation_id(&conversation) { + Ok(conversation) => conversation, + Err(message) => return invalid_request(message), + }; + let Query(request) = match request { + Ok(request) => request, + Err(_) => return invalid_request("imported entry-window query is malformed"), + }; + let anchor = match web_window_anchor(request.anchor, request.position) { + Ok(anchor) => anchor, + Err(message) => return invalid_request(message), + }; + let before = request.before.unwrap_or(DEFAULT_IMPORT_WINDOW_RADIUS); + let after = request.after.unwrap_or(DEFAULT_IMPORT_WINDOW_RADIUS); + let Some(projected_items) = before + .checked_add(after) + .and_then(|total| total.checked_add(1)) + else { + return invalid_request("imported entry-window bound overflows"); + }; + if projected_items > MAX_IMPORT_ENTRY_WINDOW_ITEMS { + return invalid_request("imported entry window exceeds the contract bound"); + } + let Some(maximum_items) = NonZeroU32::new(MAX_IMPORT_ENTRY_WINDOW_ITEMS) else { + return application_error( + StatusCode::INTERNAL_SERVER_ERROR, + "invalid_import_contract", + "imported entry-window contract is invalid", + ); + }; + let Some(maximum_text_bytes) = u32::try_from(MAX_IMPORT_TEXT_PREVIEW_BYTES) + .ok() + .and_then(NonZeroU32::new) + else { + return invalid_import_contract(); + }; + match ImportedConversationDiscoveryRepository::new(state.pool) + .entry_window( + conversation, + anchor, + before, + after, + maximum_items, + maximum_text_bytes, + ) + .await + { + Ok(Some(window)) => Json(web_entry_window(window)).into_response(), + Ok(None) => import_not_found(), + Err(ImportedConversationDiscoveryError::Request( + ImportedConversationDiscoveryRequestError::PositionOutOfRange, + )) => import_position_out_of_range(), + Err(error) => discovery_error(error), + } +} + +async fn continue_import( + State(state): State, + Path(conversation): Path, + request: axum::extract::Request, +) -> Response { + let conversation = match imported_conversation_id(&conversation) { + Ok(conversation) => conversation, + Err(message) => return invalid_request(message), + }; + let request = match decode_bounded_json::(request).await { + Ok(request) => request, + Err(response) => return response, + }; + let canonical = match canonical_continuation_request(conversation, request) { + Ok(request) => request, + Err(message) => return invalid_request(message), + }; + execute_continuation(&state, canonical).await +} + +struct CanonicalContinuationRequest { + command_id: DurableCommandId, + conversation: ImportedConversationId, + entry: ImportedTranscriptEntryId, + position: ImportedTranscriptPosition, + relationship: ImportedSessionRelationship, + web_relationship: WebImportedSessionRelationship, + model_selection: ModelSelectionRequest, + web_frontier: WebImportContinuationReference, +} + +fn canonical_continuation_request( + path_conversation: ImportedConversationId, + request: WebImportContinuationRequest, +) -> Result { + let command_id = required_uuid(&request.command_id) + .map(DurableCommandId::from_uuid) + .map_err(|_| "continuation command identity is not a UUID")?; + let frontier_conversation = + imported_conversation_id(&request.frontier.imported_conversation_id)?; + if frontier_conversation != path_conversation { + return Err("continuation frontier belongs to another import"); + } + let position = ImportedTranscriptPosition::try_from_u64(request.frontier.position) + .ok_or("continuation position must be positive")?; + let entry = required_uuid(&request.frontier.imported_entry_id) + .map(ImportedTranscriptEntryId::from_uuid) + .map_err(|_| "continuation imported-entry identity is not a UUID")?; + let model_selection = match request.initial_model_selection { + WebModelSelection::Direct { selection_id } => { + ModelSelectionRequest::Direct(DirectModelSelection::from_uuid( + required_uuid(&selection_id) + .map_err(|_| "direct model selection identity is not a UUID")?, + )) + } + WebModelSelection::Alias { alias_id } => { + ModelSelectionRequest::Alias(ModelAlias::from_uuid( + required_uuid(&alias_id).map_err(|_| "model alias identity is not a UUID")?, + )) + } + }; + let relationship = domain_relationship(request.relationship); + Ok(CanonicalContinuationRequest { + command_id, + conversation: path_conversation, + entry, + position, + relationship, + web_relationship: request.relationship, + model_selection, + web_frontier: request.frontier, + }) +} + +async fn execute_continuation( + state: &WebImportState, + request: CanonicalContinuationRequest, +) -> Response { + let repository = ImportedSessionRepository::new( + state.pool.clone(), + state.model_configuration.session_credential_pin(), + ); + match repository.load(request.command_id).await { + Ok(Some(recorded)) => { + let command = recorded.command(); + if command.imported_conversation() != request.conversation + || command.imported_frontier().through_entry() != request.entry + || command.imported_frontier().through_position() != request.position + || command.relationship() != request.relationship + || command.initial_configuration_defaults().model() != request.model_selection + { + return conflicting_reuse(); + } + return continuation_response(request, recorded.applied_result().session().into_uuid()); + } + Ok(None) => {} + Err(ImportedSessionRepositoryError::DifferentCommandKind { .. }) => { + return conflicting_reuse(); + } + Err(error) => return imported_session_error(error), + } + let frontier = ImportedTranscriptFrontier::from_parts( + request.conversation, + request.entry, + request.position, + ); + if state + .model_configuration + .resolve_session_model(request.model_selection) + .is_err() + { + return application_error( + StatusCode::BAD_REQUEST, + "model_not_configured", + "initial model selection is not configured", + ); + } + let application_request = match CreateSessionFromImportedFrontierRequest::try_new( + request.command_id, + frontier, + request.relationship, + SessionConfigurationDefaults::new(request.model_selection), + ) { + Ok(request) => request, + Err(_) => return invalid_request("continuation command identity is reserved"), + }; + let mut service = CreateSessionFromImportedFrontierService::new( + UuidV7CreateSessionFromImportedFrontierIdGenerator, + repository, + ); + match service.execute(application_request).await { + Ok(CreateSessionFromImportedFrontierOutcome::Applied(result)) => { + continuation_response(request, result.session().into_uuid()) + } + Ok(CreateSessionFromImportedFrontierOutcome::ImportedConversationNotFound { .. }) => { + import_not_found() + } + Ok(CreateSessionFromImportedFrontierOutcome::ImportedFrontierNotFound { .. }) => { + application_error( + StatusCode::BAD_REQUEST, + "import_frontier_not_found", + "selected imported frontier no longer resolves", + ) + } + Ok(CreateSessionFromImportedFrontierOutcome::ConflictingReuse { .. }) => { + conflicting_reuse() + } + Err(error) => imported_session_error(error), + } +} + +fn web_summary(summary: ImportedConversationSummary) -> WebImportSummary { + WebImportSummary { + imported_conversation_id: summary.conversation.into_uuid().to_string(), + display_title: summary.display_title.map(|title| title.into_string()), + format: web_format(summary.format), + source_session_id: summary.source_session_id.map(web_source_session), + source_session_id_sha256: summary + .source_session_digest + .as_ref() + .map(|digest| lowercase_hex(digest)), + entry_count: summary.entry_count, + } +} + +fn web_descriptor(descriptor: ImportedConversationDescriptor) -> WebImportDescriptor { + WebImportDescriptor { + imported_conversation_id: descriptor.conversation.into_uuid().to_string(), + display_title: descriptor.display_title.map(|title| title.into_string()), + raw_record_count: descriptor.raw_record_count, + entry_count: descriptor.entry_count, + source: WebImportSourceEvidence { + format: web_format(descriptor.format), + source_digest_sha256: lowercase_hex(&descriptor.source_digest), + source_session_id: descriptor.source_session_id.map(web_source_session), + }, + sizes: WebImportSizeFacts { + raw_source_bytes: descriptor.sizes.raw_source_bytes, + normalized_source_record_bytes: descriptor.sizes.normalized_source_record_bytes, + normalized_entry_bytes: descriptor.sizes.normalized_entry_bytes, + }, + timeline: WebImportTimelineBounds { + first: web_frontier(descriptor.first), + latest: web_frontier(descriptor.latest), + }, + } +} + +fn web_entry_window(window: ImportedEntryWindow) -> WebImportEntryWindow { + WebImportEntryWindow { + anchor_position: window.anchor_position, + first_position: window.first_position, + last_position: window.last_position, + has_before: window.has_before, + has_after: window.has_after, + items: window.items.into_iter().map(web_entry).collect(), + } +} + +fn web_entry(entry: ImportedEntryProjection) -> WebImportedEntry { + let (content_kind, text) = web_content(&entry.content); + WebImportedEntry { + frontier: web_frontier(entry.frontier), + raw_record_position: entry.raw_record_position, + record_entry_position: entry.record_entry_position, + source_speaker: match entry.source_speaker { + ImportedSourceAttestation::NotAttested => WebImportedSpeakerEvidence::NotAttested, + ImportedSourceAttestation::AttestedAbsent => WebImportedSpeakerEvidence::AttestedAbsent, + ImportedSourceAttestation::Attested(ImportedSpeaker::User) => { + WebImportedSpeakerEvidence::User + } + ImportedSourceAttestation::Attested(ImportedSpeaker::Assistant) => { + WebImportedSpeakerEvidence::Assistant + } + }, + content_kind, + text, + } +} + +fn web_content( + content: &ImportedEntryContentProjection, +) -> (WebImportedContentKind, Option) { + match content { + ImportedEntryContentProjection::SourceEvent => (WebImportedContentKind::SourceEvent, None), + ImportedEntryContentProjection::SourceMessageBlock => { + (WebImportedContentKind::SourceMessageBlock, None) + } + ImportedEntryContentProjection::Text(text) => ( + WebImportedContentKind::Text, + Some(match text { + ImportedSourceAttestation::NotAttested => WebImportTextEvidence::NotAttested, + ImportedSourceAttestation::AttestedAbsent => WebImportTextEvidence::AttestedAbsent, + ImportedSourceAttestation::Attested(text) => WebImportTextEvidence::Attested { + leading_text: text.leading_text.clone(), + completeness: web_completeness(text.complete), + }, + }), + ), + ImportedEntryContentProjection::ToolCall => (WebImportedContentKind::ToolCall, None), + ImportedEntryContentProjection::ToolResult => (WebImportedContentKind::ToolResult, None), + ImportedEntryContentProjection::Thinking => (WebImportedContentKind::Thinking, None), + ImportedEntryContentProjection::RedactedThinking => { + (WebImportedContentKind::RedactedThinking, None) + } + ImportedEntryContentProjection::Document => (WebImportedContentKind::Document, None), + ImportedEntryContentProjection::MessageContentAbsent => { + (WebImportedContentKind::MessageContentAbsent, None) + } + } +} + +fn web_source_session(source_session_id: ImportedTextProjection) -> WebImportSourceSessionEvidence { + WebImportSourceSessionEvidence { + leading_text: source_session_id.leading_text, + completeness: web_completeness(source_session_id.complete), + } +} + +fn web_completeness(complete: bool) -> WebImportTextCompleteness { + if complete { + WebImportTextCompleteness::Complete + } else { + WebImportTextCompleteness::Truncated + } +} + +fn web_frontier(frontier: ImportedContinuationReference) -> WebImportContinuationReference { + WebImportContinuationReference { + imported_conversation_id: frontier.conversation.into_uuid().to_string(), + imported_entry_id: frontier.entry.into_uuid().to_string(), + position: frontier.position, + } +} + +fn continuation_response(request: CanonicalContinuationRequest, session: Uuid) -> Response { + Json(WebImportContinuationResponse { + command_id: request.command_id.as_uuid().to_string(), + session_id: session.to_string(), + frontier: request.web_frontier, + relationship: request.web_relationship, + }) + .into_response() +} + +fn web_window_anchor( + anchor: Option, + position: Option, +) -> Result { + match (anchor.unwrap_or(WebImportWindowAnchor::First), position) { + (WebImportWindowAnchor::First, None) => Ok(ImportedEntryWindowAnchor::First), + (WebImportWindowAnchor::Latest, None) => Ok(ImportedEntryWindowAnchor::Latest), + (WebImportWindowAnchor::Position, Some(position)) => { + Ok(ImportedEntryWindowAnchor::Position(position)) + } + _ => Err("entry-window position is present exactly for the position anchor"), + } +} + +fn domain_format(format: WebImportFormat) -> ImportedConversationFormat { + match format { + WebImportFormat::ClaudeCodeSessionJsonlV1 => { + ImportedConversationFormat::ClaudeCodeSessionJsonlV1 + } + WebImportFormat::ClaudeCodeSessionJsonlV2 => { + ImportedConversationFormat::ClaudeCodeSessionJsonlV2 + } + WebImportFormat::CodexRolloutJsonlV1 => ImportedConversationFormat::CodexRolloutJsonlV1, + } +} + +fn web_format(format: ImportedConversationFormat) -> WebImportFormat { + match format { + ImportedConversationFormat::ClaudeCodeSessionJsonlV1 => { + WebImportFormat::ClaudeCodeSessionJsonlV1 + } + ImportedConversationFormat::ClaudeCodeSessionJsonlV2 => { + WebImportFormat::ClaudeCodeSessionJsonlV2 + } + ImportedConversationFormat::CodexRolloutJsonlV1 => WebImportFormat::CodexRolloutJsonlV1, + } +} + +fn domain_relationship( + relationship: WebImportedSessionRelationship, +) -> ImportedSessionRelationship { + match relationship { + WebImportedSessionRelationship::Resume => ImportedSessionRelationship::Resume, + WebImportedSessionRelationship::Fork => ImportedSessionRelationship::Fork, + } +} + +fn lowercase_hex(bytes: &[u8]) -> String { + const HEX_DIGITS: &[u8; 16] = b"0123456789abcdef"; + let mut encoded = String::with_capacity(bytes.len() * 2); + for byte in bytes { + encoded.push(char::from(HEX_DIGITS[usize::from(byte >> 4)])); + encoded.push(char::from(HEX_DIGITS[usize::from(byte & 0x0f)])); + } + encoded +} + +fn optional_uuid(value: Option<&str>) -> Result, &'static str> { + value + .map(required_uuid) + .transpose() + .map_err(|_| "imports cursor is not an imported-conversation UUID") +} + +fn required_uuid(value: &str) -> Result { + Uuid::parse_str(value) +} + +fn imported_conversation_id(value: &str) -> Result { + required_uuid(value) + .map(ImportedConversationId::from_uuid) + .map_err(|_| "imported-conversation identity is not a UUID") +} + +fn discovery_error(error: ImportedConversationDiscoveryError) -> Response { + match error { + ImportedConversationDiscoveryError::Database(_) => application_error( + StatusCode::SERVICE_UNAVAILABLE, + "imports_unavailable", + "imported-conversation discovery is temporarily unavailable", + ), + ImportedConversationDiscoveryError::Request( + ImportedConversationDiscoveryRequestError::PositionOutOfRange, + ) => import_position_out_of_range(), + ImportedConversationDiscoveryError::Request( + ImportedConversationDiscoveryRequestError::WindowTooLarge, + ) => invalid_request("imported entry window exceeds the contract bound"), + ImportedConversationDiscoveryError::Corruption(_) => application_error( + StatusCode::INTERNAL_SERVER_ERROR, + "import_projection_corrupt", + "stored imported-conversation facts failed closed validation", + ), + } +} + +fn imported_session_error(error: ImportedSessionRepositoryError) -> Response { + match error { + ImportedSessionRepositoryError::CommitAmbiguous(_) => application_error( + StatusCode::SERVICE_UNAVAILABLE, + "continuation_commit_ambiguous", + "continuation acknowledgement is ambiguous; retry the exact command", + ), + ImportedSessionRepositoryError::Database(_) => application_error( + StatusCode::SERVICE_UNAVAILABLE, + "continuation_unavailable", + "imported continuation is temporarily unavailable", + ), + ImportedSessionRepositoryError::DifferentCommandKind { .. } => conflicting_reuse(), + ImportedSessionRepositoryError::Preparation(_) + | ImportedSessionRepositoryError::IdentityCollision(_) + | ImportedSessionRepositoryError::Corruption(_) => application_error( + StatusCode::INTERNAL_SERVER_ERROR, + "continuation_corrupt", + "stored imported continuation failed closed validation", + ), + } +} + +fn source_session_maximum_bytes() -> Option { + u32::try_from(MAX_IMPORT_SOURCE_SESSION_BYTES) + .ok() + .and_then(NonZeroU32::new) +} + +fn invalid_import_contract() -> Response { + application_error( + StatusCode::INTERNAL_SERVER_ERROR, + "invalid_import_contract", + "imported discovery contract is invalid", + ) +} + +fn invalid_request(message: &'static str) -> Response { + transport_error(StatusCode::BAD_REQUEST, "invalid_import_request", message) +} + +fn import_not_found() -> Response { + application_error( + StatusCode::NOT_FOUND, + "import_not_found", + "imported conversation does not exist", + ) +} + +fn import_position_out_of_range() -> Response { + application_error( + StatusCode::BAD_REQUEST, + "import_position_out_of_range", + "imported entry-window position is outside the timeline", + ) +} + +fn conflicting_reuse() -> Response { + application_error( + StatusCode::CONFLICT, + "conflicting_command_reuse", + "durable command identity already names another payload", + ) +} + +#[cfg(test)] +mod tests { + use super::*; + + const CONVERSATION_ID: &str = "00000000-0000-7000-8000-000000000001"; + const OTHER_CONVERSATION_ID: &str = "00000000-0000-7000-8000-000000000002"; + const ENTRY_ID: &str = "00000000-0000-7000-8000-000000000003"; + const COMMAND_ID: &str = "00000000-0000-7000-8000-000000000004"; + const MODEL_ID: &str = "00000000-0000-7000-8000-000000000005"; + + fn continuation_request() -> WebImportContinuationRequest { + WebImportContinuationRequest { + command_id: COMMAND_ID.to_owned(), + frontier: WebImportContinuationReference { + imported_conversation_id: CONVERSATION_ID.to_owned(), + imported_entry_id: ENTRY_ID.to_owned(), + position: 1, + }, + relationship: WebImportedSessionRelationship::Resume, + initial_model_selection: WebModelSelection::Direct { + selection_id: MODEL_ID.to_owned(), + }, + } + } + + fn conversation_id() -> ImportedConversationId { + ImportedConversationId::from_uuid( + Uuid::parse_str(CONVERSATION_ID).expect("fixture UUID is valid"), + ) + } + + #[test] + fn canonical_continuation_accepts_a_correlated_request() { + assert!(canonical_continuation_request(conversation_id(), continuation_request()).is_ok()); + } + + #[test] + fn canonical_continuation_rejects_a_non_uuid_command() { + let mut request = continuation_request(); + request.command_id = "not-a-uuid".to_owned(); + + assert_eq!( + canonical_continuation_request(conversation_id(), request).err(), + Some("continuation command identity is not a UUID") + ); + } + + #[test] + fn canonical_continuation_rejects_another_import() { + let mut request = continuation_request(); + request.frontier.imported_conversation_id = OTHER_CONVERSATION_ID.to_owned(); + + assert_eq!( + canonical_continuation_request(conversation_id(), request).err(), + Some("continuation frontier belongs to another import") + ); + } + + #[test] + fn canonical_continuation_rejects_zero_position() { + let mut request = continuation_request(); + request.frontier.position = 0; + + assert_eq!( + canonical_continuation_request(conversation_id(), request).err(), + Some("continuation position must be positive") + ); + } + + #[test] + fn canonical_continuation_rejects_a_non_uuid_entry() { + let mut request = continuation_request(); + request.frontier.imported_entry_id = "not-a-uuid".to_owned(); + + assert_eq!( + canonical_continuation_request(conversation_id(), request).err(), + Some("continuation imported-entry identity is not a UUID") + ); + } + + #[test] + fn canonical_continuation_rejects_a_non_uuid_direct_model() { + let mut request = continuation_request(); + request.initial_model_selection = WebModelSelection::Direct { + selection_id: "not-a-uuid".to_owned(), + }; + + assert_eq!( + canonical_continuation_request(conversation_id(), request).err(), + Some("direct model selection identity is not a UUID") + ); + } + + #[test] + fn canonical_continuation_rejects_a_non_uuid_alias() { + let mut request = continuation_request(); + request.initial_model_selection = WebModelSelection::Alias { + alias_id: "not-a-uuid".to_owned(), + }; + + assert_eq!( + canonical_continuation_request(conversation_id(), request).err(), + Some("model alias identity is not a UUID") + ); + } + + #[test] + fn imported_text_projection_preserves_truncated_utf8_evidence() { + let expected = "€".repeat(MAX_IMPORT_TEXT_PREVIEW_BYTES / "€".len()); + let content = ImportedEntryContentProjection::Text(ImportedSourceAttestation::Attested( + ImportedTextProjection { + leading_text: expected.clone(), + complete: false, + }, + )); + + assert_eq!( + web_content(&content), + ( + WebImportedContentKind::Text, + Some(WebImportTextEvidence::Attested { + leading_text: expected, + completeness: WebImportTextCompleteness::Truncated, + }), + ) + ); + } + + #[test] + fn imported_text_projection_marks_a_complete_value() { + let source = "complete imported text"; + let content = ImportedEntryContentProjection::Text(ImportedSourceAttestation::Attested( + ImportedTextProjection { + leading_text: source.to_owned(), + complete: true, + }, + )); + + assert_eq!( + web_content(&content), + ( + WebImportedContentKind::Text, + Some(WebImportTextEvidence::Attested { + leading_text: source.to_owned(), + completeness: WebImportTextCompleteness::Complete, + }), + ) + ); + } + + #[test] + fn source_session_projection_preserves_bounded_evidence() { + let leading_text = "€".repeat(MAX_IMPORT_SOURCE_SESSION_BYTES / "€".len()); + let evidence = web_source_session(ImportedTextProjection { + leading_text: leading_text.clone(), + complete: false, + }); + + assert_eq!(evidence.leading_text, leading_text); + assert_eq!(evidence.completeness, WebImportTextCompleteness::Truncated); + } + + #[test] + fn position_anchor_preserves_its_exact_position() { + assert_eq!( + web_window_anchor(Some(WebImportWindowAnchor::Position), Some(7)), + Ok(ImportedEntryWindowAnchor::Position(7)) + ); + } + + #[test] + fn position_anchor_rejects_an_absent_position() { + assert_eq!( + web_window_anchor(Some(WebImportWindowAnchor::Position), None), + Err("entry-window position is present exactly for the position anchor") + ); + } + + #[test] + fn non_position_anchor_rejects_a_supplied_position() { + assert_eq!( + web_window_anchor(Some(WebImportWindowAnchor::Latest), Some(7)), + Err("entry-window position is present exactly for the position anchor") + ); + } + + #[tokio::test] + async fn out_of_range_position_is_an_application_error() { + let response = import_position_out_of_range(); + let status = response.status(); + let body = axum::body::to_bytes(response.into_body(), 4_096) + .await + .expect("the bounded error body is readable"); + let decoded: serde_json::Value = + serde_json::from_slice(&body).expect("the application error is JSON"); + + assert_eq!(status, StatusCode::BAD_REQUEST); + assert_eq!(decoded["error"]["kind"], "application"); + assert_eq!(decoded["error"]["code"], "import_position_out_of_range"); + } +} diff --git a/apps/signalboxd/src/workspace_instruction_runtime.rs b/apps/signalboxd/src/workspace_instruction_runtime.rs new file mode 100644 index 0000000000..65068837e5 --- /dev/null +++ b/apps/signalboxd/src/workspace_instruction_runtime.rs @@ -0,0 +1,326 @@ +//! Turn-start workspace-instruction discovery and durable provenance. + +use std::{error::Error, fmt, path::Path}; + +use signalbox_application::{ + ClassifyOperatorFailure, InstructionDiscoveryRoot, OperatorFailureClass, + discover_workspace_instructions, +}; +use signalbox_domain::{ + InstructionBundleId, InstructionDiscoveryId, InstructionDiscoveryRootKind, InstructionPath, + SessionId, TurnId, TurnInstructionManifest, TurnInstructionManifestId, +}; +use signalbox_persistence::workspace_instructions::{ + CountedActivationInstructionEvidence, RecordTurnInstructionSnapshotOutcome, + TurnInstructionManifestPreflight, WorkspaceInstructionPlacementObservation, + WorkspaceInstructionRepository, WorkspaceInstructionRepositoryError, +}; +use sqlx::PgPool; +use uuid::Uuid; + +use crate::daemon_tools::WorkspaceInstructionRootResolver; + +/// Failure before the daemon can prove which instruction snapshot a turn used. +#[derive(Debug)] +pub enum WorkspaceInstructionRuntimeError { + /// A deployment-owned workspace path cannot be represented durably. + InvalidWorkspacePath, + /// The session's configured daemon-local workspace is misprovisioned. + UnresolvableWorkspace, + /// The blocking filesystem scan task failed to join. + DiscoveryTask(tokio::task::JoinError), + /// A fixed scan safety limit prevented a complete inventory. + DiscoveryIncomplete, + /// Durable snapshot recording or authentication failed. + Persistence(WorkspaceInstructionRepositoryError), +} + +impl fmt::Display for WorkspaceInstructionRuntimeError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::InvalidWorkspacePath => { + formatter.write_str("workspace instruction root is not a canonical UTF-8 path") + } + Self::UnresolvableWorkspace => formatter + .write_str("session workspace could not be resolved for instruction discovery"), + Self::DiscoveryTask(error) => error.fmt(formatter), + Self::DiscoveryIncomplete => { + formatter.write_str("workspace instruction discovery limit was reached") + } + Self::Persistence(error) => error.fmt(formatter), + } + } +} + +impl Error for WorkspaceInstructionRuntimeError { + fn source(&self) -> Option<&(dyn Error + 'static)> { + match self { + Self::InvalidWorkspacePath => None, + Self::UnresolvableWorkspace => None, + Self::DiscoveryTask(error) => Some(error), + Self::DiscoveryIncomplete => None, + Self::Persistence(error) => Some(error), + } + } +} + +impl ClassifyOperatorFailure for WorkspaceInstructionRuntimeError { + fn operator_failure_class(&self) -> OperatorFailureClass { + match self { + Self::InvalidWorkspacePath => OperatorFailureClass::CallerOrHubBug, + Self::UnresolvableWorkspace => OperatorFailureClass::Infrastructure { + commit_ambiguous: false, + }, + Self::DiscoveryTask(_) => OperatorFailureClass::Infrastructure { + commit_ambiguous: false, + }, + Self::DiscoveryIncomplete => OperatorFailureClass::Infrastructure { + commit_ambiguous: false, + }, + Self::Persistence(error) => error.operator_failure_class(), + } + } + + fn operator_failure_cause_code(&self) -> &'static str { + match self { + Self::InvalidWorkspacePath => "workspace_instruction_path", + Self::UnresolvableWorkspace => "workspace_instruction_workspace_unresolvable", + Self::DiscoveryTask(_) => "workspace_instruction_discovery_task", + Self::DiscoveryIncomplete => "workspace_instruction_discovery_limit", + Self::Persistence(error) => error.operator_failure_cause_code(), + } + } +} + +/// Daemon-owned discovery and turn-manifest composition. +#[derive(Clone, Debug)] +pub struct WorkspaceInstructionRuntime { + repository: WorkspaceInstructionRepository, + workspace_root: Option, + configured_roots: Box<[InstructionPath]>, +} + +/// Complete filesystem evidence retained only until a counted activation +/// transaction either commits it or rejects the stale preview. +#[derive(Debug)] +pub(crate) struct PreparedCountedActivationInstructions { + discovery: InstructionDiscoveryId, + manifest: TurnInstructionManifest, + snapshot: signalbox_application::InstructionDiscoverySnapshot, + bundle_ids: Box<[InstructionBundleId]>, + placement: WorkspaceInstructionPlacementObservation, +} + +impl PreparedCountedActivationInstructions { + pub(crate) fn evidence(&self) -> CountedActivationInstructionEvidence<'_> { + CountedActivationInstructionEvidence::new( + self.discovery, + &self.manifest, + &self.snapshot, + &self.bundle_ids, + &self.placement, + ) + } +} + +impl WorkspaceInstructionRuntime { + /// Supplies persistence, session workspace derivation, and explicit roots. + pub fn new( + pool: PgPool, + workspace_root: Option, + configured_roots: Vec, + ) -> Self { + Self { + repository: WorkspaceInstructionRepository::new(pool), + workspace_root, + configured_roots: configured_roots.into_boxed_slice(), + } + } + + /// Greedily scans and atomically records an empty turn-start manifest. + /// + /// `false` means the turn stopped being active before evidence could bind + /// it; callers must do no model work for that stale activation. + pub async fn prepare( + &self, + session: SessionId, + turn: TurnId, + ) -> Result { + match self + .repository + .preflight_turn_start(session, turn) + .await + .map_err(WorkspaceInstructionRuntimeError::Persistence)? + { + TurnInstructionManifestPreflight::Available(_) => return Ok(true), + TurnInstructionManifestPreflight::TurnUnavailable => return Ok(false), + TurnInstructionManifestPreflight::Absent => {} + } + let (snapshot, placement) = self.discover(session).await?; + let discovery = InstructionDiscoveryId::from_uuid(Uuid::now_v7()); + let manifest = TurnInstructionManifest::empty_turn_start( + TurnInstructionManifestId::from_uuid(Uuid::now_v7()), + session, + turn, + ); + let outcome = self + .repository + .record_turn_start_for_observed_placement( + discovery, + manifest, + &snapshot, + &placement, + || InstructionBundleId::from_uuid(Uuid::now_v7()), + ) + .await + .map_err(WorkspaceInstructionRuntimeError::Persistence)?; + outcome_is_available(outcome) + } + + /// Prepares complete evidence for the counted activation transaction. + /// + /// An incomplete scan remains durable diagnostic evidence without binding + /// a manifest. Complete evidence is returned without persistence so a + /// stale preview cannot leave an authoritative snapshot behind. + pub(crate) async fn prepare_counted_activation( + &self, + session: SessionId, + turn: TurnId, + ) -> Result, WorkspaceInstructionRuntimeError> + { + match self + .repository + .preflight_counted_activation(session, turn) + .await + .map_err(WorkspaceInstructionRuntimeError::Persistence)? + { + TurnInstructionManifestPreflight::Available(_) => { + return Err(WorkspaceInstructionRuntimeError::Persistence( + WorkspaceInstructionRepositoryError::Corruption( + "queued counted activation manifest preexisted", + ), + )); + } + TurnInstructionManifestPreflight::TurnUnavailable => return Ok(None), + TurnInstructionManifestPreflight::Absent => {} + } + let (snapshot, placement) = self.discover(session).await?; + let discovery = InstructionDiscoveryId::from_uuid(Uuid::now_v7()); + let manifest = TurnInstructionManifest::empty_turn_start( + TurnInstructionManifestId::from_uuid(Uuid::now_v7()), + session, + turn, + ); + if !snapshot.is_complete() { + let outcome = self + .repository + .record_counted_activation_for_observed_placement( + discovery, + manifest, + &snapshot, + &placement, + || InstructionBundleId::from_uuid(Uuid::now_v7()), + ) + .await + .map_err(WorkspaceInstructionRuntimeError::Persistence)?; + return match outcome { + RecordTurnInstructionSnapshotOutcome::DiscoveryIncomplete => { + Err(WorkspaceInstructionRuntimeError::DiscoveryIncomplete) + } + RecordTurnInstructionSnapshotOutcome::TurnUnavailable => Ok(None), + RecordTurnInstructionSnapshotOutcome::Recorded(_) + | RecordTurnInstructionSnapshotOutcome::AlreadyRecorded(_) => { + Err(WorkspaceInstructionRuntimeError::Persistence( + WorkspaceInstructionRepositoryError::Corruption( + "incomplete counted activation bound a manifest", + ), + )) + } + }; + } + let bundle_ids = snapshot + .bundles() + .iter() + .map(|_| InstructionBundleId::from_uuid(Uuid::now_v7())) + .collect::>() + .into_boxed_slice(); + Ok(Some(PreparedCountedActivationInstructions { + discovery, + manifest, + snapshot, + bundle_ids, + placement, + })) + } + + async fn discover( + &self, + session: SessionId, + ) -> Result< + ( + signalbox_application::InstructionDiscoverySnapshot, + WorkspaceInstructionPlacementObservation, + ), + WorkspaceInstructionRuntimeError, + > { + let placement = self + .repository + .observe_session_runner_placement(session) + .await + .map_err(WorkspaceInstructionRuntimeError::Persistence)?; + let runner_placed = placement.runner_owned(); + let mut roots = + Vec::with_capacity(self.configured_roots.len() + usize::from(!runner_placed)); + let workspace_binding = if !runner_placed && let Some(workspace_root) = &self.workspace_root + { + let path = workspace_root + .resolve(session) + .await + .map_err(|_| WorkspaceInstructionRuntimeError::UnresolvableWorkspace)?; + roots.push(InstructionDiscoveryRoot::new( + InstructionDiscoveryRootKind::Workspace, + instruction_path(&path)?, + )); + Some((workspace_root.clone(), path)) + } else { + None + }; + roots.extend(self.configured_roots.iter().cloned().map(|path| { + InstructionDiscoveryRoot::new(InstructionDiscoveryRootKind::Configured, path) + })); + let snapshot = tokio::task::spawn_blocking(move || discover_workspace_instructions(roots)) + .await + .map_err(WorkspaceInstructionRuntimeError::DiscoveryTask)?; + if let Some((workspace_root, expected_path)) = workspace_binding { + let revalidated_path = workspace_root + .resolve(session) + .await + .map_err(|_| WorkspaceInstructionRuntimeError::UnresolvableWorkspace)?; + if revalidated_path != expected_path { + return Err(WorkspaceInstructionRuntimeError::UnresolvableWorkspace); + } + } + Ok((snapshot, placement)) + } +} + +fn outcome_is_available( + outcome: RecordTurnInstructionSnapshotOutcome, +) -> Result { + match outcome { + RecordTurnInstructionSnapshotOutcome::Recorded(_) + | RecordTurnInstructionSnapshotOutcome::AlreadyRecorded(_) => Ok(true), + RecordTurnInstructionSnapshotOutcome::DiscoveryIncomplete => { + Err(WorkspaceInstructionRuntimeError::DiscoveryIncomplete) + } + RecordTurnInstructionSnapshotOutcome::TurnUnavailable => Ok(false), + } +} + +fn instruction_path(path: &Path) -> Result { + let value = path + .to_str() + .ok_or(WorkspaceInstructionRuntimeError::InvalidWorkspacePath)?; + InstructionPath::try_new(value.to_owned()) + .map_err(|_| WorkspaceInstructionRuntimeError::InvalidWorkspacePath) +} diff --git a/apps/signalboxd/tests/live_tool_evals.rs b/apps/signalboxd/tests/live_tool_evals.rs index 6051160518..6cdd0b5131 100644 --- a/apps/signalboxd/tests/live_tool_evals.rs +++ b/apps/signalboxd/tests/live_tool_evals.rs @@ -723,7 +723,12 @@ async fn run_case( InProcessToolDispatchGate::default(), suite.catalog.clone(), suite.executor.clone(), - ); + ) + .with_workspace_instructions(signalboxd::WorkspaceInstructionRuntime::new( + database.pool.clone(), + None, + Vec::new(), + )); timeout(TURN_TIMEOUT, execution.execute(Box::new(activated))) .await .map_err(|_| io::Error::other("the daemon tool eval turn exceeded its timeout"))??; diff --git a/apps/signalboxd/tests/live_tool_exercise.rs b/apps/signalboxd/tests/live_tool_exercise.rs index c56b97b2f4..bcb84718c6 100644 --- a/apps/signalboxd/tests/live_tool_exercise.rs +++ b/apps/signalboxd/tests/live_tool_exercise.rs @@ -42,9 +42,9 @@ use signalbox_persistence::{ start_eligible_turn::StartEligibleTurnRepository, }; use signalbox_process_protocol::{ - CanonicalU64, CanonicalUuid, ClientFrame, ClientRequest, CommandId, InputContent, - ModelSelection, ModelSettingsOverlay, ProtocolVersion, RequestId, ServerFrame, ServerMessage, - SessionPlacement, SystemPromptMember, ToolDecision, TurnState, decode_server_line, + CanonicalU64, CanonicalUuid, ClientFrame, ClientRequest, CommandId, ModelSelection, + ModelSettingsOverlay, ProtocolVersion, RequestId, ServerFrame, ServerMessage, SessionPlacement, + SystemPromptMember, ToolDecision, TurnState, UserInputContent, decode_server_line, encode_client_line, }; use signalbox_tools_basic::SESSION_STATUS_UPDATE_NAME; @@ -317,7 +317,12 @@ async fn run_live_smoke() -> SmokeResult { provider, None, ) - .with_tool_loop(tool_gate, tool_catalog, tool_executor); + .with_tool_loop(tool_gate, tool_catalog, tool_executor) + .with_workspace_instructions(signalboxd::WorkspaceInstructionRuntime::new( + pool.clone(), + None, + Vec::new(), + )); run_automatic_turn( &pool, @@ -925,9 +930,18 @@ fn assert_own_transcript(results: &[Value]) -> SmokeResult { return Err(io::Error::other("conversation round returned the wrong result count").into()); }; let visible = transcript["entries"].as_array().is_some_and(|entries| { - entries - .iter() - .any(|entry| entry["content"] == TRANSCRIPT_MARKER) + entries.iter().any(|entry| { + entry["content"] + .as_str() + .and_then(|content| serde_json::from_str::(content).ok()) + .is_some_and(|content| { + content + == serde_json::json!([{ + "type": "text", + "text": TRANSCRIPT_MARKER, + }]) + }) + }) }); assert!( visible, @@ -1184,7 +1198,7 @@ async fn submit_turn( .send(ClientRequest::SubmitInput { command_id: command()?, session_id: session, - content: InputContent::new(content.to_owned()), + content: UserInputContent::text(content.to_owned()), expected_defaults_version: Some(CanonicalU64::new(1)), model_settings: ModelSettingsOverlay::inherit_all(), delivery: None, diff --git a/apps/signalboxd/tests/offline_reply.rs b/apps/signalboxd/tests/offline_reply.rs index 964e3c2273..f7841e9e9c 100644 --- a/apps/signalboxd/tests/offline_reply.rs +++ b/apps/signalboxd/tests/offline_reply.rs @@ -376,7 +376,12 @@ async fn s01_s02_inv014_inv015_runtime_bridge_persists_scripted_assistant_reply( provider, None, ) - .with_tool_loop(tool_dispatch_gate, NoToolCatalog, UnexpectedToolExecutor), + .with_tool_loop(tool_dispatch_gate, NoToolCatalog, UnexpectedToolExecutor) + .with_workspace_instructions(signalboxd::WorkspaceInstructionRuntime::new( + pool.clone(), + None, + Vec::new(), + )), ); let pass = ActivatedTurnPass::new( StartEligibleTurnService::new( @@ -405,7 +410,7 @@ async fn s01_s02_inv014_inv015_runtime_bridge_persists_scripted_assistant_reply( let transcript = sqlx::query_as::<_, (String, Option, Option)>( "SELECT entry.payload_kind, - accepted.content_text, + accepted_part.text_value, entry.assistant_text_value FROM turn_lifecycle AS lifecycle JOIN context_frontier_member AS member @@ -417,6 +422,10 @@ async fn s01_s02_inv014_inv015_runtime_bridge_persists_scripted_assistant_reply( LEFT JOIN accepted_input AS accepted ON accepted.session_id = entry.source_session_id AND accepted.accepted_input_id = entry.origin_accepted_input_id + LEFT JOIN accepted_input_content_part AS accepted_part + ON accepted_part.accepted_input_id = accepted.accepted_input_id + AND accepted_part.position = 0 + AND accepted_part.part_kind = 'text' WHERE lifecycle.session_id = $1 AND lifecycle.turn_id = $2 ORDER BY member.member_position", @@ -574,7 +583,12 @@ async fn s_goal_inv048_success_continues_and_unsuccessful_turn_blocks_without_re provider, None, ) - .with_tool_loop(tool_dispatch_gate, NoToolCatalog, UnexpectedToolExecutor), + .with_tool_loop(tool_dispatch_gate, NoToolCatalog, UnexpectedToolExecutor) + .with_workspace_instructions(signalboxd::WorkspaceInstructionRuntime::new( + pool.clone(), + None, + Vec::new(), + )), ); let activated_pass = ActivatedTurnPass::new( StartEligibleTurnService::new( diff --git a/apps/signalboxd/tests/offline_tool_loop.rs b/apps/signalboxd/tests/offline_tool_loop.rs index be2eb423ec..75aa089f94 100644 --- a/apps/signalboxd/tests/offline_tool_loop.rs +++ b/apps/signalboxd/tests/offline_tool_loop.rs @@ -57,9 +57,9 @@ use signalbox_persistence::{ submit_input::SubmitInputRepository, tool_loop::PostgresToolLoopRepository, }; use signalbox_process_protocol::{ - CanonicalU64, CanonicalUuid, ClientFrame, ClientRequest, CommandId, InputContent, - InputDelivery, ModelSettingsOverlay, ProtocolVersion, RequestId, ServerMessage, ToolDecision, - decode_server_line, encode_client_line, + CanonicalU64, CanonicalUuid, ClientFrame, ClientRequest, CommandId, InputDelivery, + ModelSettingsOverlay, ProtocolVersion, RequestId, ServerMessage, ToolDecision, + UserInputContent, decode_server_line, encode_client_line, }; use signalbox_tools_exec::{ BwrapAvailability, CaptureCompleteness, ProcessOutcome, ProcessOutput, ProcessRequest, @@ -398,7 +398,12 @@ impl ToolLoopFixture { ) -> ( FixtureExecution, Arc>, - ) { + ) + where + Catalog: signalbox_application::ToolCatalog + Clone + Send + 'static, + Executor: ToolExecutor + Clone + Send + 'static, + Executor::Error: Send + 'static, + { self.execution_with_model_shutdown(scripts, catalog, executor, None) } @@ -411,7 +416,12 @@ impl ToolLoopFixture { ) -> ( FixtureExecution, Arc>, - ) { + ) + where + Catalog: signalbox_application::ToolCatalog + Clone + Send + 'static, + Executor: ToolExecutor + Clone + Send + 'static, + Executor::Error: Send + 'static, + { self.execution_with_model_shutdown(scripts, catalog, executor, Some(shutdown)) } @@ -424,7 +434,12 @@ impl ToolLoopFixture { ) -> ( FixtureExecution, Arc>, - ) { + ) + where + Catalog: signalbox_application::ToolCatalog + Clone + Send + 'static, + Executor: ToolExecutor + Clone + Send + 'static, + Executor::Error: Send + 'static, + { self.execution_with_model_shutdown_and_limit( scripts, catalog, @@ -443,7 +458,12 @@ impl ToolLoopFixture { ) -> ( FixtureExecution, Arc>, - ) { + ) + where + Catalog: signalbox_application::ToolCatalog + Clone + Send + 'static, + Executor: ToolExecutor + Clone + Send + 'static, + Executor::Error: Send + 'static, + { self.execution_with_model_shutdown_and_limit( scripts, catalog, @@ -463,7 +483,12 @@ impl ToolLoopFixture { ) -> ( FixtureExecution, Arc>, - ) { + ) + where + Catalog: signalbox_application::ToolCatalog + Clone + Send + 'static, + Executor: ToolExecutor + Clone + Send + 'static, + Executor::Error: Send + 'static, + { let runtime = Arc::new(ScriptedModel::::following(scripts)); let provider = RuntimeModelCallProvider::new( RecordingScriptedModel { @@ -484,7 +509,12 @@ impl ToolLoopFixture { provider, automatic_tool_round_limit, ) - .with_tool_loop(self.tool_dispatch_gate.clone(), catalog, executor), + .with_tool_loop(self.tool_dispatch_gate.clone(), catalog, executor) + .with_workspace_instructions(signalboxd::WorkspaceInstructionRuntime::new( + self.pool.clone(), + None, + Vec::new(), + )), runtime, ) } @@ -531,6 +561,11 @@ impl ToolLoopFixture { None, ) .with_tool_loop(self.tool_dispatch_gate.clone(), catalog, executor) + .with_workspace_instructions(signalboxd::WorkspaceInstructionRuntime::new( + self.pool.clone(), + None, + Vec::new(), + )) .with_approval_judge(judge, None, configuration), runtime, judge_runtime, @@ -2201,10 +2236,11 @@ async fn delegated_park_resumes_into_fresh_judge_composition() -> Result<(), Box first_execution .execute(Box::new(fixture.activated.clone())) .await?; - let (scheduled, continuation) = PostgresEligibilitySweep::new(fixture.pool.clone()) - .find_sessions() - .await? - .into_parts(); + let (scheduled, _dispatch_starts, continuation) = + PostgresEligibilitySweep::new(fixture.pool.clone()) + .find_sessions() + .await? + .into_parts(); let resumable = PostgresToolLoopRepository::new(fixture.pool.clone()) .find_resumable_turn(fixture.session) .await?; @@ -3190,7 +3226,7 @@ async fn s10_composed_introspection_returns_real_own_transcript() -> Result<(), "entries": [{ "position": 1, "kind": "user", - "content": FIXTURE_USER_CONTENT, + "content": r#"[{"type":"text","text":"offline tool-loop request"}]"#, "content_truncated": false }, { "position": 2, @@ -4077,10 +4113,11 @@ async fn s02_s10_inv005_inv006_restart_leaves_approval_turn_parked() -> Result<( fixture .decide(request, ToolApprovalDecision::Approve) .await?; - let (resumable, continuation) = PostgresEligibilitySweep::new(fixture.pool.clone()) - .find_sessions() - .await? - .into_parts(); + let (resumable, _dispatch_starts, continuation) = + PostgresEligibilitySweep::new(fixture.pool.clone()) + .find_sessions() + .await? + .into_parts(); assert!(!continuation); assert_eq!(resumable, vec![fixture.session]); let (restarted_execution, restarted_runtime) = fixture.execution( @@ -4683,7 +4720,7 @@ async fn s02_s08_s10_inv016_inv036_steering_consumed_at_continuation_completes() .await?; let request = fixture.wait_for_requests(1).await?[0]; - let steering_content = InputContent::new(String::from("steer the parked tool round")); + let steering_content = UserInputContent::text(String::from("steer the parked tool round")); let steering_frame = ClientFrame::try_new_for_version( ProtocolVersion::One, RequestId::try_new(1)?, diff --git a/apps/signalboxd/tests/process_protocol_runtime.rs b/apps/signalboxd/tests/process_protocol_runtime.rs index d9e5274031..5499eb1fe5 100644 --- a/apps/signalboxd/tests/process_protocol_runtime.rs +++ b/apps/signalboxd/tests/process_protocol_runtime.rs @@ -7,12 +7,13 @@ mod support; use std::{ - collections::VecDeque, + collections::{HashSet, VecDeque}, error::Error, fs, - future::pending, + future::{Future, pending}, io::{self, ErrorKind}, os::unix::fs::PermissionsExt, + panic::{AssertUnwindSafe, resume_unwind}, path::{Path, PathBuf}, sync::{ Arc, Mutex, @@ -21,43 +22,52 @@ use std::{ time::Duration, }; +use futures_util::FutureExt; use signalbox_application::{ AuthorizeModelCallOutcome, ClassifyOperatorFailure, CreateSessionFromImportedFrontierIdGenerator, CreateSessionFromImportedFrontierOutcome, CreateSessionFromImportedFrontierRequest, CreateSessionFromImportedFrontierService, - EligibilityPass, ImportConversationOutcome, ImportConversationService, - ImportedConversationIdGenerator, InProcessAttemptDispatchGate, InProcessEligibilityNudge, - InProcessEligibilityWorkSource, InProcessToolDispatchGate, ModelCallCredentialReference, - ModelCallExecutionOutcome, ModelCallExecutionService, ModelCallInputTokenCount, - ModelCallInputTokenCounter, NoToolCatalog, OperatorFailureClass, PreparedModelOperation, - ReplaceSessionMetadataOutcome, ReplaceSessionMetadataRequest, ReplaceSessionMetadataService, - SchedulerLoop, SchedulerLoopExit, SchedulerPassOccupancyBound, ScriptedModelCallProvider, - ScriptedModelCallStep, StaleActiveTurnBound, StartEligibleTurnOutcome, - StartEligibleTurnService, StartupScanService, TurnLivenessScanInterval, - UuidV7ModelCallExecutionIdGenerator, UuidV7StartEligibleTurnIdGenerator, - UuidV7StartupScanIdGenerator, + EligibilityPass, EligibilitySweep, EligibilitySweepBatch, ImportConversationOutcome, + ImportConversationService, ImportedConversationIdGenerator, InProcessAttemptDispatchGate, + InProcessEligibilityNudge, InProcessEligibilityWorkSource, InProcessToolDispatchGate, + ModelCallCredentialReference, ModelCallExecutionOutcome, ModelCallExecutionService, + ModelCallInputTokenCount, ModelCallInputTokenCounter, NoToolCatalog, OperatorFailureClass, + PreparedModelOperation, ReplaceSessionMetadataOutcome, ReplaceSessionMetadataRequest, + ReplaceSessionMetadataService, RepoWatchConvergenceVerdict, RepoWatchReviewDecision, + SchedulerLoop, SchedulerLoopExit, SchedulerPassExpiryHandler, SchedulerPassOccupancyBound, + ScriptedModelCallProvider, ScriptedModelCallStep, StaleActiveTurnBound, + StartEligibleTurnOutcome, StartEligibleTurnService, StartupScanService, + TurnLivenessScanInterval, UuidV7ModelCallExecutionIdGenerator, + UuidV7StartEligibleTurnIdGenerator, UuidV7StartupScanIdGenerator, + scheduler_ordinary_pass_limit, }; use signalbox_blob_store::BlobObjectKey; use signalbox_conversation_import_claude_code::ClaudeCodeJsonlConverter; use signalbox_domain::{ - ActiveTurnPhase, Actor, AssistantResponsePart, AssistantText, BlobDigest, ContextCompactionId, - ContextCompactionTokenUsage, ContextFrontierId, DirectModelSelection, DurableCommandId, - FailedModelCallTurnIdentities, ImportedConversationFormat, ImportedConversationId, - ImportedSessionRelationship, ImportedTranscriptEntryId, InitialToolApproval, ModelCallId, + ActiveTurnPhase, Actor, AssistantResponsePart, AssistantText, BlobDigest, BranchName, + CheckRunName, CommitSha, ContextCompactionId, ContextCompactionTokenUsage, ContextFrontierId, + DirectModelSelection, DurableCommandId, FailedModelCallTurnIdentities, + ImportedConversationFormat, ImportedConversationId, ImportedSessionRelationship, + ImportedTranscriptEntryId, InitialToolApproval, MergeableState, ModelCallId, ModelCallTerminalIdentities, ModelCallTerminalObservation, ModelCallTerminalOutcome, ModelSelectionRequest, ModelTargetCatalog, NormalizedToolArguments, ProviderModelIdentity, - ReplaceSessionMetadataResult, ResolvedProviderTarget, SemanticTranscriptEntryId, - SessionConfigurationDefaults, SessionConfigurationDefaultsVersion, SessionId, - SessionMetadataContent, ToolCallProposal, ToolName, ToolRequestId, ToolResponsePartIdentity, - ToolRoundModelCallIdentities, ToolUsingAssistantResponse, TurnId, + PullRequestNumber, ReplaceSessionMetadataResult, RepoWatchAuthorLogin, RepositorySlug, + ResolvedProviderTarget, SemanticTranscriptEntryId, SessionConfigurationDefaults, + SessionConfigurationDefaultsVersion, SessionId, SessionMetadataContent, ToolCallProposal, + ToolName, ToolRequestId, ToolResponsePartIdentity, ToolRoundModelCallIdentities, + ToolUsingAssistantResponse, TurnId, +}; +use signalbox_model_provider_runtime::{ + RuntimeContextCompactionModel, RuntimeInputTokenCountError, RuntimeModelCallProvider, + RuntimeModelCallProviderError, }; -use signalbox_model_provider_runtime::{RuntimeContextCompactionModel, RuntimeModelCallProvider}; use signalbox_model_runtime::{ AssistantPart, BoundaryLossEvidence, CancellationSignal, CompletionEvidence, CompletionFinish, DeliveryMode, ExchangeFacts, InputTokenCountOutcome, LossCause, MessagePart, - ModelInputTokenCounter, ModelOperation, ModelRuntime, Observation, ObservationFact, - ObservationSink, PreparationOutcome, ProviderReportedModel, Script, ScriptedModel, - ScriptedPrepared, TerminalEvidence, TerminalReport, TokenUsage, ToolCallsAtLoss, + ModelInputTokenCounter, ModelOperation, ModelRuntime, NativeErrorFacts, Observation, + ObservationFact, ObservationSink, PreparationOutcome, ProviderErrorEvidence, ProviderErrorKind, + ProviderReportedModel, Script, ScriptedModel, ScriptedPrepared, TerminalEvidence, + TerminalReport, TokenUsage, ToolCallsAtLoss, }; use signalbox_persistence::{ blob::BlobCatalogRepository, @@ -75,6 +85,10 @@ use signalbox_persistence::{ session_metadata::SessionMetadataRepository, start_eligible_turn::StartEligibleTurnRepository, startup::PostgresStartupScanRepository, + test_support::{ + FleetSoakCensus, FleetSoakCensusRepository, OperatorStatusConvergenceFixture, + OperatorStatusFixtureRepository, OperatorStatusStaleReviewClearanceFixture, + }, turn_liveness::TurnLivenessPersistenceBounds, }; use signalbox_process_protocol::{ @@ -86,24 +100,27 @@ use signalbox_process_protocol::{ ImportedSourceSpeaker, ImportedSpeaker, ImportedTextPreview, InputContent, InputDelivery, MAX_SESSION_METADATA_INDEXED_UTF8_BYTES, MetadataActor, ModelChangeAdjustment, ModelSelection, ModelSettingSource, ModelSettingsOverlay, ModelSettingsPrecedence, ModelSettingsSnapshot, - ProtocolVersion, ReasoningLevel, RejectionDetail, RequestId, ReviewConcernTerminalOutcome, - ReviewDiffSide, ReviewExternalObjectKind, ReviewFindingEvent, ReviewFindingInput, - ReviewFindingStatus, ReviewImportTerminalOutcome, ReviewJudgmentDisposition, - ReviewJudgmentEffectTerminalOutcome, ReviewJudgmentPlanMember, ReviewOrchestrationConcernInput, - ReviewOrchestrationConcernStatus, ReviewOrchestrationCounts, ReviewOrchestrationSnapshot, - ReviewOrchestrationState, ReviewPassTerminalOutcome, ReviewPublicationOutcome, - ReviewPublicationTerminalOutcome, ReviewRepairOutcome, ReviewRepairTerminalOutcome, - ReviewSeverity, ReviewTargetSubject, ReviewWorkflow, ServerFrame, ServerMessage, SessionEvent, - SessionMetadata, SessionPlacement, SettingOverlay, SystemPromptMember, SystemPromptText, - ToolDecision, TranscriptEntry, TranscriptTextEntry, TurnState, decode_server_line, - encode_client_line, + OperatorStatusConvergenceVerdict, OperatorStatusEndMessage, OperatorStatusMergeableState, + OperatorStatusMessage, OperatorStatusPendingStaleReviewClearanceMessage, + OperatorStatusPullRequestConvergenceMessage, OperatorStatusReviewDecision, ProtocolVersion, + ReasoningLevel, RejectionDetail, RequestId, ReviewConcernTerminalOutcome, ReviewDiffSide, + ReviewExternalObjectKind, ReviewFindingEvent, ReviewFindingInput, ReviewFindingStatus, + ReviewImportTerminalOutcome, ReviewJudgmentDisposition, ReviewJudgmentEffectTerminalOutcome, + ReviewJudgmentPlanMember, ReviewOrchestrationConcernInput, ReviewOrchestrationConcernStatus, + ReviewOrchestrationCounts, ReviewOrchestrationSnapshot, ReviewOrchestrationState, + ReviewPassTerminalOutcome, ReviewPublicationOutcome, ReviewPublicationTerminalOutcome, + ReviewRepairOutcome, ReviewRepairTerminalOutcome, ReviewSeverity, ReviewTargetSubject, + ReviewWorkflow, ServerFrame, ServerMessage, SessionEvent, SessionMetadata, SessionPlacement, + SettingOverlay, SystemPromptMember, SystemPromptText, ToolDecision, TranscriptEntry, + TranscriptTextEntry, TurnState, UserInputContent, decode_server_line, encode_client_line, }; use signalboxd::{ ActivatedTurnPass, BlobStorageClass, BlobStoreRegistry, ContextGuardedTurnPass, ContextGuardedTurnPassError, ExpiredPassRecoveryPolicy, FatalExecutionSupervisor, HubModelConfiguration, LocalProcessListener, PostgresProviderModelExecution, - ProcessProviderTextDeltaSink, ProcessRuntime, ProcessRuntimeError, - SessionTemplateConfiguration, TurnLivenessNumericBounds, TurnLivenessRuntime, + ProcessProviderTextDeltaSink, ProcessRuntime, ProcessRuntimeError, ReportedUsageCompaction, + ReportedUsageCompactionError, SessionTemplateConfiguration, TurnLivenessNumericBounds, + TurnLivenessRuntime, }; use sqlx::{PgPool, postgres::PgPoolOptions}; use tempfile::TempDir; @@ -137,6 +154,49 @@ fn test_session_credential_pin() -> signalbox_persistence::SessionCredentialPin ]) .expect("test credential pin is valid") } + +#[track_caller] +fn exactly_one_credential_reference(references: &[String]) -> &str { + match references { + [reference] => reference.as_str(), + _ => panic!("the fixture pins exactly one credential family"), + } +} + +#[track_caller] +fn reported_usage_still_exceeded_turn(outcome: Result<(), ReportedUsageCompactionError>) -> TurnId { + match outcome { + Err(ReportedUsageCompactionError::Compaction { + turn, + cause_code: "reported_usage_context_still_exceeded", + .. + }) => turn, + other => panic!("expected a still-exceeded compaction failure, got {other:?}"), + } +} + +#[track_caller] +fn failed_automatic_compaction_turn( + outcome: Result< + (), + ContextGuardedTurnPassError< + RuntimeInputTokenCountError, + signalboxd::WorkspaceInstructionPreparedExecutionError< + signalboxd::PostgresProviderModelExecutionError, + >, + >, + >, +) -> TurnId { + match outcome { + Err(ContextGuardedTurnPassError::Compaction { + turn, + cause_code: "context_compaction_model", + .. + }) => turn, + other => panic!("expected a failed automatic compaction, got {other:?}"), + } +} + const MAX_SUBMITTED_INPUT_BYTES: usize = 1024 * 1024; const OVERSIZED_SUBMITTED_INPUT_BYTES: usize = MAX_SUBMITTED_INPUT_BYTES + 1; const STREAMING_DELTA_COUNT: usize = 192; @@ -572,14 +632,180 @@ async fn create_imported_session(pool: &PgPool) -> Result, + cycle: Arc>, +} + +#[derive(Debug, Default)] +struct ReconciliationCycle { + hinted_sessions: HashSet, + processed_sessions: HashSet, + final_batch_seen: bool, +} + +impl ReconciliationWitness { + fn new() -> Self { + Self { + completed_cycles: Arc::new(AtomicUsize::new(0)), + cycle: Arc::new(Mutex::new(ReconciliationCycle::default())), + } + } + + fn record_batch(&self, sessions: &[SessionId], continuation: bool) { + let mut cycle = self + .cycle + .lock() + .expect("the reconciliation witness lock is available"); + cycle.hinted_sessions.extend(sessions.iter().copied()); + cycle.final_batch_seen = !continuation; + self.complete_drained_cycle(&mut cycle); + } + + fn record_processed_session(&self, session: SessionId) { + let mut cycle = self + .cycle + .lock() + .expect("the reconciliation witness lock is available"); + cycle.processed_sessions.insert(session); + self.complete_drained_cycle(&mut cycle); + } + + fn complete_drained_cycle(&self, cycle: &mut ReconciliationCycle) { + if cycle.final_batch_seen && cycle.hinted_sessions.is_subset(&cycle.processed_sessions) { + self.completed_cycles.fetch_add(1, Ordering::SeqCst); + *cycle = ReconciliationCycle::default(); + } + } + + fn completed_cycles(&self) -> usize { + self.completed_cycles.load(Ordering::SeqCst) + } +} + +#[test] +fn reconciliation_witness_waits_for_final_batch_hints_to_finish() { + let witness = ReconciliationWitness::new(); + let session = SessionId::from_uuid(Uuid::from_u128(1)); + + witness.record_batch(&[session], false); + assert_eq!(witness.completed_cycles(), 0); + + witness.record_processed_session(session); + assert_eq!(witness.completed_cycles(), 1); +} + +#[test] +fn reconciliation_witness_completes_an_empty_cycle_immediately() { + let witness = ReconciliationWitness::new(); + + witness.record_batch(&[], false); + + assert_eq!(witness.completed_cycles(), 1); +} + +struct WitnessedEligibilitySweep { + inner: Sweep, + witness: ReconciliationWitness, +} + +impl WitnessedEligibilitySweep { + fn new(inner: Sweep, witness: ReconciliationWitness) -> Self { + Self { inner, witness } + } +} + +impl EligibilitySweep for WitnessedEligibilitySweep +where + Sweep: EligibilitySweep + Send, +{ + type Error = Sweep::Error; + + fn find_sessions( + &mut self, + ) -> impl Future> + Send { + let witness = self.witness.clone(); + async move { + let batch = self.inner.find_sessions().await?; + let (sessions, _dispatch_starts, continuation) = batch.clone().into_parts(); + witness.record_batch(&sessions, continuation); + Ok(batch) + } + } +} + +struct WitnessedEligibilityPass { + inner: Pass, + witness: ReconciliationWitness, +} + +impl WitnessedEligibilityPass { + fn new(inner: Pass, witness: ReconciliationWitness) -> Self { + Self { inner, witness } + } +} + +impl EligibilityPass for WitnessedEligibilityPass +where + Pass: EligibilityPass + Send, +{ + type Error = Pass::Error; + + fn failure_stage(error: &Self::Error) -> &'static str { + Pass::failure_stage(error) + } + + fn failure_turn(error: &Self::Error) -> Option { + Pass::failure_turn(error) + } + + // The decorator must forward every boundary the inner pass overrides. + // Inheriting the trait defaults here silently drops the composed pass's + // occupancy-expiry handoff and its reserved dispatch-start lane, which the + // fleet-soak scenarios below depend on. + fn occupancy_expiry_handler(&self) -> Option> { + self.inner.occupancy_expiry_handler() + } + + fn run( + &mut self, + session: SessionId, + ) -> impl Future> + Send + 'static { + let execution = self.inner.run(session); + let witness = self.witness.clone(); + async move { + let outcome = execution.await; + witness.record_processed_session(session); + outcome + } + } + + fn run_dispatch_start( + &mut self, + session: SessionId, + ) -> impl Future> + Send + 'static { + let execution = self.inner.run_dispatch_start(session); + let witness = self.witness.clone(); + async move { + let outcome = execution.await; + witness.record_processed_session(session); + outcome + } + } +} + +type RuntimeEligibilitySweep = WitnessedEligibilitySweep; + struct RunningRuntime { container: ContainerAsync, pool: PgPool, socket_directory: SocketDirectory, shutdown: watch::Sender, - runtime_task: JoinHandle>, + runtime_task: Option>>, eligibility_nudge: InProcessEligibilityNudge, - work_source: Option>, + work_source: Option>, + reconciliation_witness: ReconciliationWitness, provider_text_deltas: ProcessProviderTextDeltaSink, blob_store_registry: Option>, blob_storage_root: Option, @@ -619,7 +845,11 @@ impl RunningRuntime { let (container, pool) = postgres().await?; let socket_directory = SocketDirectory::create()?; let listener = LocalProcessListener::bind(socket_directory.socket())?; - let sweep = PostgresEligibilitySweep::new(pool.clone()); + let reconciliation_witness = ReconciliationWitness::new(); + let sweep = WitnessedEligibilitySweep::new( + PostgresEligibilitySweep::new(pool.clone()), + reconciliation_witness.clone(), + ); let (eligibility_nudge, work_source) = InProcessEligibilityWorkSource::new(sweep); let blob_storage_root = match blob_storage { BlobStorageFixtureMode::Disabled => None, @@ -666,9 +896,10 @@ impl RunningRuntime { pool, socket_directory, shutdown, - runtime_task, + runtime_task: Some(runtime_task), eligibility_nudge, work_source: Some(work_source), + reconciliation_witness, provider_text_deltas, blob_store_registry, blob_storage_root, @@ -707,8 +938,21 @@ impl RunningRuntime { template_configuration: SessionTemplateConfiguration, ) -> Result> { self.shutdown.send(true)?; - timeout(Duration::from_secs(10), &mut self.runtime_task).await???; + let runtime_task = self + .runtime_task + .as_mut() + .expect("a running runtime has an installed task"); + timeout(Duration::from_secs(10), runtime_task).await???; + self.runtime_task = None; + self.restart_after_stop(configuration, template_configuration) + .await + } + async fn restart_after_stop( + &mut self, + configuration: &str, + template_configuration: SessionTemplateConfiguration, + ) -> Result> { let mut scan = StartupScanService::new( UuidV7StartupScanIdGenerator, PostgresStartupScanRepository::new(self.pool.clone()), @@ -716,7 +960,11 @@ impl RunningRuntime { let recovered_turn_count = scan.execute().await?.recovered_turn_count(); let listener = LocalProcessListener::bind(self.socket())?; - let sweep = PostgresEligibilitySweep::new(self.pool.clone()); + let reconciliation_witness = ReconciliationWitness::new(); + let sweep = WitnessedEligibilitySweep::new( + PostgresEligibilitySweep::new(self.pool.clone()), + reconciliation_witness.clone(), + ); let (eligibility_nudge, work_source) = InProcessEligibilityWorkSource::new(sweep); let model_configuration = support::parse_model_configuration(configuration)?; let mut runtime = ProcessRuntime::new_with_templates( @@ -733,9 +981,10 @@ impl RunningRuntime { let provider_text_deltas = runtime.provider_text_delta_sink(); let (shutdown, shutdown_receiver) = watch::channel(false); self.shutdown = shutdown; - self.runtime_task = tokio::spawn(runtime.run(shutdown_receiver)); + self.runtime_task = Some(tokio::spawn(runtime.run(shutdown_receiver))); self.eligibility_nudge = eligibility_nudge; self.work_source = Some(work_source); + self.reconciliation_witness = reconciliation_witness; self.provider_text_deltas = provider_text_deltas; Ok(recovered_turn_count) } @@ -744,48 +993,34 @@ impl RunningRuntime { /// replacement opens the same socket and database only after the killed /// task has stopped, so no graceful runtime shutdown can repair its work. async fn kill_and_restart(&mut self) -> Result> { - self.runtime_task.abort(); - let killed = (&mut self.runtime_task).await; + let runtime_task = self + .runtime_task + .take() + .expect("a running runtime has an installed task"); + runtime_task.abort(); + let killed = runtime_task.await; let killed = killed.expect_err("the killed runtime task must not return normally"); assert!( killed.is_cancelled(), "the runtime task must stop by cancellation, got {killed}" ); - let mut scan = StartupScanService::new( - UuidV7StartupScanIdGenerator, - PostgresStartupScanRepository::new(self.pool.clone()), - ); - let recovered_turn_count = scan.execute().await?.recovered_turn_count(); - let listener = LocalProcessListener::bind(self.socket())?; - let sweep = PostgresEligibilitySweep::new(self.pool.clone()); - let (eligibility_nudge, work_source) = InProcessEligibilityWorkSource::new(sweep); let model_configuration = support::parse_model_configuration(MODEL_CONFIGURATION)?; let template_configuration = session_template_configuration(&model_configuration)?; - let runtime = ProcessRuntime::new_with_templates( - listener, - self.pool.clone(), - eligibility_nudge.clone(), - InProcessToolDispatchGate::default(), - model_configuration, - template_configuration, - ); - let provider_text_deltas = runtime.provider_text_delta_sink(); - let (shutdown, shutdown_receiver) = watch::channel(false); - self.shutdown = shutdown; - self.runtime_task = tokio::spawn(runtime.run(shutdown_receiver)); - self.eligibility_nudge = eligibility_nudge; - self.work_source = Some(work_source); - self.provider_text_deltas = provider_text_deltas; - Ok(recovered_turn_count) + self.restart_after_stop(MODEL_CONFIGURATION, template_configuration) + .await } - fn take_work_source(&mut self) -> InProcessEligibilityWorkSource { + fn take_work_source(&mut self) -> InProcessEligibilityWorkSource { self.work_source .take() .expect("the streaming fixture takes the work source once") } + fn reconciliation_witness(&self) -> ReconciliationWitness { + self.reconciliation_witness.clone() + } + fn provider_text_delta_sink(&self) -> ProcessProviderTextDeltaSink { self.provider_text_deltas.clone() } @@ -798,9 +1033,11 @@ impl RunningRuntime { ) } - async fn stop(self) -> Result<(), Box> { - self.shutdown.send(true)?; - timeout(Duration::from_secs(10), self.runtime_task).await???; + async fn stop(mut self) -> Result<(), Box> { + if let Some(runtime_task) = self.runtime_task.take() { + self.shutdown.send(true)?; + timeout(Duration::from_secs(10), runtime_task).await???; + } self.pool.close().await; self.socket_directory.cleanup()?; drop(self.blob_storage_root); @@ -1669,7 +1906,7 @@ async fn submit_first_input( ClientRequest::SubmitInput { command_id: command()?, session_id, - content: InputContent::new(content), + content: UserInputContent::text(content), expected_defaults_version: Some(CanonicalU64::new(1)), model_settings: ModelSettingsOverlay::inherit_all(), delivery: None, @@ -2065,15 +2302,18 @@ async fn execute_streamed_turn_until( RuntimeModelCallProvider::new(scripted, model_configuration.runtime_model_catalog(), None) .with_text_delta_sink(runtime.provider_text_delta_sink()); let (execution, fatal_execution) = - FatalExecutionSupervisor::new(PostgresProviderModelExecution::new( - PostgresModelCallRepository::new( - runtime.pool.clone(), - model_configuration.target_catalog(), - ModelCallCredentialReference::new("streaming-fixture"), + FatalExecutionSupervisor::new(signalboxd::WorkspaceInstructionPreparedExecution::new( + PostgresProviderModelExecution::new( + PostgresModelCallRepository::new( + runtime.pool.clone(), + model_configuration.target_catalog(), + ModelCallCredentialReference::new("streaming-fixture"), + ), + InProcessAttemptDispatchGate::default(), + provider, + None, ), - InProcessAttemptDispatchGate::default(), - provider, - None, + signalboxd::WorkspaceInstructionRuntime::new(runtime.pool.clone(), None, Vec::new()), )); let pass = ActivatedTurnPass::new( StartEligibleTurnService::new( @@ -2113,15 +2353,18 @@ async fn execute_recorded_turn( RuntimeModelCallProvider::new(scripted, model_configuration.runtime_model_catalog(), None) .with_text_delta_sink(runtime.provider_text_delta_sink()); let (execution, fatal_execution) = - FatalExecutionSupervisor::new(PostgresProviderModelExecution::new( - PostgresModelCallRepository::new( - runtime.pool.clone(), - model_configuration.target_catalog(), - ModelCallCredentialReference::new("recording-fixture"), + FatalExecutionSupervisor::new(signalboxd::WorkspaceInstructionPreparedExecution::new( + PostgresProviderModelExecution::new( + PostgresModelCallRepository::new( + runtime.pool.clone(), + model_configuration.target_catalog(), + ModelCallCredentialReference::new("recording-fixture"), + ), + InProcessAttemptDispatchGate::default(), + provider, + None, ), - InProcessAttemptDispatchGate::default(), - provider, - None, + signalboxd::WorkspaceInstructionRuntime::new(runtime.pool.clone(), None, Vec::new()), )); let pass = ActivatedTurnPass::new( StartEligibleTurnService::new( @@ -2195,11 +2438,14 @@ async fn execute_guarded_turn( .with_session_credentials(model_configuration.credential_family_catalog()); let guarded_repository = repository.clone(); let (execution, fatal_execution) = - FatalExecutionSupervisor::new(PostgresProviderModelExecution::new( - repository, - InProcessAttemptDispatchGate::default(), - provider, - None, + FatalExecutionSupervisor::new(signalboxd::WorkspaceInstructionPreparedExecution::new( + PostgresProviderModelExecution::new( + repository, + InProcessAttemptDispatchGate::default(), + provider, + None, + ), + signalboxd::WorkspaceInstructionRuntime::new(runtime.pool.clone(), None, Vec::new()), )); let compaction_model: Arc = Arc::new(RuntimeContextCompactionModel::new( @@ -2215,7 +2461,12 @@ async fn execute_guarded_turn( model_configuration, compaction_model, execution, - ); + ) + .with_workspace_instructions(signalboxd::WorkspaceInstructionRuntime::new( + runtime.pool.clone(), + None, + Vec::new(), + )); let mut scheduler = SchedulerLoop::new(runtime.take_work_source(), pass); let observation_pool = runtime.pool.clone(); let session = SessionId::from_uuid(session_id.into_uuid()); @@ -2250,12 +2501,22 @@ fn completed_script(provider_model: &str, text: &str, usage: TokenUsage) -> Scri // unprovisioned workspace, and scheduled goal resumption are named follow-on // slices: they need the same fleet census but not more boot infrastructure. -const FLEET_SESSION_COUNT: usize = 16; +// numeric-bound: test fixture - mirrors the `scheduler_pass_admission_cap` +// numeric bound that `config/signalboxd.example.toml` supplies, which +// `support::parse_model_configuration` splices into this fixture's +// configuration. The cap is deployment configuration rather than a compiled +// constant, so the derived ordinary limit below is stated against it. +const FLEET_PASS_ADMISSION_CAP: usize = 16; +// numeric-bound: derived ceiling from the configured pass admission cap. +// One place inside the shared admission cap stays reserved for a +// repository-watch dispatch start, so a fleet that saturates ordinary +// scheduler capacity is one session smaller than the cap itself. +const FLEET_SESSION_COUNT: usize = scheduler_ordinary_pass_limit(FLEET_PASS_ADMISSION_CAP); // numeric-bound: test setup - preserves the ordinary production occupancy fixture const FLEET_BASELINE_OCCUPANCY_BOUND: Duration = Duration::from_secs(900); // numeric-bound: test deadline - exercises the production recovery path promptly const FLEET_OCCUPANCY_BOUND: Duration = Duration::from_secs(1); -// numeric-bound: test deadline - keeps each fault probe inside one CI minute +// numeric-bound: test deadline - keeps each fault observation short in CI const FLEET_ASSERTION_BOUND: Duration = Duration::from_secs(2); // numeric-bound: test setup - admits a full contended fleet inside two CI minutes const FLEET_SETUP_BOUND: Duration = Duration::from_secs(120); @@ -2265,16 +2526,27 @@ struct FleetPrepared { inner: ScriptedPrepared, } +/// How many scripted executions complete and how many hang. +/// +/// The completions are served first: a scenario that stands a healthy baseline +/// fleet up before injecting one fault gets the fault on the last execution. +#[derive(Clone, Copy)] +struct FleetModelCardinality { + hanging: usize, + completing: usize, +} + #[derive(Clone)] struct FleetScriptedModel { inner: ScriptedModel, completions_before_hangs: Arc, hangs_remaining: Arc, in_flight_hangs: Arc, + completed_calls: Arc>>, } impl FleetScriptedModel { - fn new(hang_count: usize, completed_count: usize) -> Self { + fn new(cardinality: FleetModelCardinality) -> Self { Self { inner: ScriptedModel::following(std::iter::repeat_n( completed_script( @@ -2282,17 +2554,32 @@ impl FleetScriptedModel { "fleet session completed", TokenUsage::unreported(), ), - hang_count + completed_count, + cardinality.hanging + cardinality.completing, )), - completions_before_hangs: Arc::new(AtomicUsize::new(completed_count)), - hangs_remaining: Arc::new(AtomicUsize::new(hang_count)), + completions_before_hangs: Arc::new(AtomicUsize::new(cardinality.completing)), + hangs_remaining: Arc::new(AtomicUsize::new(cardinality.hanging)), in_flight_hangs: Arc::new(AtomicUsize::new(0)), + completed_calls: Arc::new(Mutex::new(Vec::new())), } } fn in_flight_hangs(&self) -> usize { self.in_flight_hangs.load(Ordering::SeqCst) } + + fn completed_call_ids(&self) -> Vec { + self.completed_calls + .lock() + .expect("the fleet completion lock is available") + .clone() + } + + fn record_completed_call(&self, correlation: ModelCallId) { + self.completed_calls + .lock() + .expect("the fleet completion lock is available") + .push(correlation); + } } struct FleetHangGuard(Arc); @@ -2342,6 +2629,7 @@ impl ModelRuntime for FleetScriptedModel { sink: &mut (dyn ObservationSink + Send), cancellation: CancellationSignal, ) -> TerminalReport { + let correlation = prepared.correlation; let completes = self .completions_before_hangs .fetch_update(Ordering::SeqCst, Ordering::SeqCst, |remaining| { @@ -2349,7 +2637,9 @@ impl ModelRuntime for FleetScriptedModel { }) .is_ok(); if completes { - return self.inner.execute(prepared.inner, sink, cancellation).await; + let report = self.inner.execute(prepared.inner, sink, cancellation).await; + self.record_completed_call(correlation); + return report; } let hangs = self .hangs_remaining @@ -2359,14 +2649,16 @@ impl ModelRuntime for FleetScriptedModel { .is_ok(); if hangs { sink.observe(Observation { - correlation: prepared.correlation, + correlation, fact: ObservationFact::SendCommenced, }); self.in_flight_hangs.fetch_add(1, Ordering::SeqCst); let _guard = FleetHangGuard(Arc::clone(&self.in_flight_hangs)); pending::>().await } else { - self.inner.execute(prepared.inner, sink, cancellation).await + let report = self.inner.execute(prepared.inner, sink, cancellation).await; + self.record_completed_call(correlation); + report } } } @@ -2454,6 +2746,36 @@ async fn wait_for_fleet_shutdown(mut shutdown: watch::Receiver) { } } +/// Commissions one extra session after a restart, as a readiness control that +/// proves the replacement scheduler is admitting fresh work. +async fn commission_fleet_control( + runtime: &RunningRuntime, +) -> Result> { + let mut connection = Connection::connect(runtime.socket()).await?; + connection + .request( + 1, + ClientRequest::CommissionSession { + command_id: command()?, + template_name: String::from("merge-forward"), + fence: CommissionedSessionFence::Branch { + repository: String::from("sample-user/sample-repository"), + branch: String::from("agent/fleet-soak-control"), + }, + statement: String::from("complete the fleet scheduler readiness control"), + content: InputContent::new(String::from("return the scripted reply")), + }, + ) + .await?; + let response = response_within(&mut connection).await?.message().clone(); + let ServerMessage::SessionCommissioned { session_id, .. } = response else { + return Err( + io::Error::other(format!("fleet control commission returned {response:?}")).into(), + ); + }; + Ok(session_id) +} + fn start_fleet_scheduler( runtime: &mut RunningRuntime, model: FleetScriptedModel, @@ -2519,15 +2841,18 @@ fn start_fleet_scheduler( RuntimeModelCallProvider::new(model, configuration.runtime_model_catalog(), None) .with_text_delta_sink(runtime.provider_text_delta_sink()); let (execution, fatal_execution) = - FatalExecutionSupervisor::new(PostgresProviderModelExecution::new( - PostgresModelCallRepository::new( - runtime.pool.clone(), - configuration.target_catalog(), - ModelCallCredentialReference::new("fleet-soak-fixture"), + FatalExecutionSupervisor::new(signalboxd::WorkspaceInstructionPreparedExecution::new( + PostgresProviderModelExecution::new( + PostgresModelCallRepository::new( + runtime.pool.clone(), + configuration.target_catalog(), + ModelCallCredentialReference::new("fleet-soak-fixture"), + ), + InProcessAttemptDispatchGate::default(), + provider, + None, ), - InProcessAttemptDispatchGate::default(), - provider, - None, + signalboxd::WorkspaceInstructionRuntime::new(runtime.pool.clone(), None, Vec::new()), )); let pass = ActivatedTurnPass::new( StartEligibleTurnService::new( @@ -2542,6 +2867,7 @@ fn start_fleet_scheduler( expired_pass_recovery_policy, turn_liveness_persistence_bounds, ); + let pass = WitnessedEligibilityPass::new(pass, runtime.reconciliation_witness()); let mut scheduler = SchedulerLoop::new(runtime.take_work_source(), pass).with_occupancy_bound(occupancy_bound); let turn_liveness = TurnLivenessRuntime::new( @@ -2590,194 +2916,399 @@ async fn wait_for_hangs(model: &FleetScriptedModel, expected: usize) -> Result<( Ok(()) } -async fn fleet_lifecycle_counts(pool: &PgPool) -> Result<(i64, i64), Box> { - Ok(sqlx::query_as( - "SELECT count(*) FILTER (WHERE state_kind = 'active'), - count(*) FILTER (WHERE state_kind = 'terminal') - FROM turn_lifecycle", - ) - .fetch_one(pool) - .await?) +/// Waits for one drained eligibility cycle, so a replacement scheduler's +/// reconciliation pass is observed as completed rather than slept for. +async fn wait_for_reconciliation(witness: &ReconciliationWitness) -> Result<(), Box> { + timeout(FLEET_SETUP_BOUND, async { + while witness.completed_cycles() == 0 { + tokio::task::yield_now().await; + } + }) + .await?; + Ok(()) } -async fn fleet_terminal_call_count(pool: &PgPool) -> Result> { - Ok(sqlx::query_scalar( - "SELECT count(*) FROM model_call WHERE terminal_disposition_kind IS NOT NULL", - ) - .fetch_one(pool) - .await?) +/// Tears the scheduler and turn-liveness tasks down from whatever state they +/// are in, so a panicking scenario still releases the fixture. +async fn abort_fleet_scheduler(tasks: FleetRuntimeTasks) -> Result<(), Box> { + let FleetRuntimeTasks { + shutdown, + scheduler, + turn_liveness, + } = tasks; + shutdown.send_replace(true); + scheduler.abort(); + turn_liveness.abort(); + let stopped = scheduler.await; + let liveness = turn_liveness.await; + if !matches!(&stopped, Ok(SchedulerLoopExit::Shutdown)) + && !matches!(&stopped, Err(error) if error.is_cancelled()) + { + return Err(io::Error::other(format!( + "the fleet scheduler must stop by cancellation or fatal-driven shutdown: {stopped:?}" + )) + .into()); + } + if !matches!(&liveness, Ok(())) && !matches!(&liveness, Err(error) if error.is_cancelled()) { + return Err(io::Error::other(format!( + "the fleet turn-liveness runtime must stop by cancellation or shutdown: {liveness:?}" + )) + .into()); + } + Ok(()) } -async fn fleet_ambiguous_model_call_park_count(pool: &PgPool) -> Result> { - Ok(sqlx::query_scalar( - "SELECT count(*) - FROM turn_lifecycle - WHERE state_kind = 'active' - AND active_phase_kind = 'awaiting_model_call_recovery'", - ) - .fetch_one(pool) +async fn wait_for_completed_calls( + model: &FleetScriptedModel, + expected: usize, +) -> Result, Box> { + Ok(timeout(FLEET_SETUP_BOUND, async { + loop { + let completed = model.completed_call_ids(); + if completed.len() == expected { + return completed; + } + tokio::task::yield_now().await; + } + }) .await?) } -async fn wait_for_fleet_ambiguous_model_call_park( - pool: &PgPool, +async fn wait_for_model_call_for_session( + repository: &FleetSoakCensusRepository, + session: CanonicalUuid, +) -> Result> { + timeout(FLEET_SETUP_BOUND, async { + loop { + if let Some(model_call) = repository + .model_call_id_for_session(SessionId::from_uuid(session.into_uuid())) + .await? + { + return Ok::>(model_call); + } + tokio::task::yield_now().await; + } + }) + .await? +} + +async fn wait_for_completed_call( model: &FleetScriptedModel, - bound: Duration, + expected: ModelCallId, ) -> Result<(), Box> { - timeout(bound, async { + timeout(FLEET_SETUP_BOUND, async { + while !model.completed_call_ids().contains(&expected) { + tokio::task::yield_now().await; + } + }) + .await?; + Ok(()) +} + +async fn wait_for_terminal_calls( + repository: &FleetSoakCensusRepository, + model_calls: &[ModelCallId], +) -> Result<(), Box> { + timeout(FLEET_SETUP_BOUND, async { loop { - let parked = fleet_ambiguous_model_call_park_count(pool).await?; - if parked == 1 && model.in_flight_hangs() == 0 { + if repository + .census_for(model_calls) + .await? + .terminal_model_calls() + == i64::try_from(model_calls.len())? + { return Ok::<(), Box>(()); } tokio::task::yield_now().await; } }) - .await - .map_err(|_| io::Error::other("fleet model call did not reach its ambiguity park"))??; + .await??; Ok(()) } -async fn wait_for_fleet_lifecycle_counts( - pool: &PgPool, - expected: (i64, i64), - bound: Duration, +async fn wait_for_terminal_turns( + repository: &FleetSoakCensusRepository, + model_calls: &[ModelCallId], ) -> Result<(), Box> { - let observed = timeout(bound, async { + timeout(FLEET_SETUP_BOUND, async { loop { - if fleet_lifecycle_counts(pool).await? == expected { + if repository.census_for(model_calls).await?.terminal_turns() + == i64::try_from(model_calls.len())? + { return Ok::<(), Box>(()); } tokio::task::yield_now().await; } }) - .await; - if observed.is_err() { - let actual = fleet_lifecycle_counts(pool).await?; - return Err(io::Error::other(format!( - "fleet lifecycle expected {expected:?}, observed {actual:?}" - )) - .into()); - } - observed.expect("elapsed outcome was handled")?; + .await??; Ok(()) } -async fn wait_for_fleet_terminal_count( - pool: &PgPool, - expected: i64, +/// Waits for exactly `hung_model_call` to reach its typed ambiguity park with +/// its execution released, rather than counting parks across the database. +async fn wait_for_ambiguity_park( + repository: &FleetSoakCensusRepository, + model: &FleetScriptedModel, + hung_model_call: ModelCallId, bound: Duration, ) -> Result<(), Box> { - let observed = timeout(bound, async { + timeout(bound, async { loop { - let (_, terminal) = fleet_lifecycle_counts(pool).await?; - if terminal >= expected { + if repository + .has_ambiguous_recovery_park(hung_model_call) + .await? + && model.in_flight_hangs() == 0 + { return Ok::<(), Box>(()); } tokio::task::yield_now().await; } }) - .await; - if observed.is_err() { - let actual = fleet_lifecycle_counts(pool).await?; + .await + .map_err(|_| io::Error::other("fleet model call did not reach its ambiguity park"))??; + Ok(()) +} + +fn assert_hung_fleet_outcome( + model: &FleetScriptedModel, + census: FleetSoakCensus, + hung_call_has_ambiguity_park: bool, +) -> Result<(), Box> { + let active = census.active_turns(); + let terminal = census.terminal_turns(); + let typed_terminal_calls = census.terminal_model_calls(); + if model.in_flight_hangs() != 0 + || active != 1 + || terminal != i64::try_from(FLEET_SESSION_COUNT - 1)? + || typed_terminal_calls != i64::try_from(FLEET_SESSION_COUNT)? + || census.awaiting_model_call_recovery_turns() != 1 + || census.ambiguous_model_calls() != 1 + || !hung_call_has_ambiguity_park + { + return Err(io::Error::other(format!( + "fleet liveness failed: hangs={}, active={active}, terminal={terminal}, typed_terminal_calls={typed_terminal_calls}, recovery_parks={}, ambiguous_calls={}, hung_call_has_ambiguity_park={hung_call_has_ambiguity_park}", + model.in_flight_hangs(), + census.awaiting_model_call_recovery_turns(), + census.ambiguous_model_calls() + )) + .into()); + } + Ok(()) +} + +fn assert_restarted_fleet_outcome( + census: FleetSoakCensus, + original_model: &FleetScriptedModel, + replacement_model: &FleetScriptedModel, +) -> Result<(), Box> { + if census.active_turns() != 0 + || census.terminal_turns() != i64::try_from(FLEET_SESSION_COUNT)? + || census.awaiting_model_call_recovery_turns() != 0 + || census.terminal_model_calls() != i64::try_from(FLEET_SESSION_COUNT)? + || original_model.in_flight_hangs() != 0 + || replacement_model.in_flight_hangs() != 0 + { return Err(io::Error::other(format!( - "fleet terminal setup expected at least {expected}, observed {actual:?}" + "restart must release every original execution and reconcile every ambiguous operation into a terminal turn without a user decision: census={census:?}, original_hangs={}, replacement_hangs={}", + original_model.in_flight_hangs(), + replacement_model.in_flight_hangs() )) .into()); } - observed.expect("elapsed outcome was handled")?; Ok(()) } -/// Issue #1027: a post-acceptance model hang releases its authoritative pass -/// and reaches a durable typed ambiguity park inside the occupancy bound. -#[tokio::test(flavor = "multi_thread")] -#[ignore = "requires ephemeral PostgreSQL and a local Unix socket"] -async fn fleet_soak_hung_model_call_has_bounded_pass_occupancy_and_typed_disposition() --> Result<(), Box> { - let mut runtime = RunningRuntime::start().await?; - let baseline_fleet = commission_fleet(&runtime, 0, FLEET_SESSION_COUNT - 1).await?; - let model = FleetScriptedModel::new(1, FLEET_SESSION_COUNT - 1); - let baseline_tasks = start_fleet_scheduler( - &mut runtime, - model.clone(), - SchedulerPassOccupancyBound::try_new(FLEET_BASELINE_OCCUPANCY_BOUND)?, - )?; - wait_for_fleet_terminal_count( - &runtime.pool, - i64::try_from(FLEET_SESSION_COUNT - 1)?, - FLEET_SETUP_BOUND, - ) - .await?; - baseline_tasks.stop().await?; - runtime.restart().await?; - let fault_fleet = commission_fleet(&runtime, FLEET_SESSION_COUNT - 1, 1).await?; - let occupancy_bound = SchedulerPassOccupancyBound::try_new(FLEET_OCCUPANCY_BOUND)?; - let tasks = start_fleet_scheduler(&mut runtime, model.clone(), occupancy_bound)?; - wait_for_hangs(&model, 1).await?; - wait_for_fleet_ambiguous_model_call_park(&runtime.pool, &model, FLEET_ASSERTION_BOUND).await?; - let (active, terminal) = fleet_lifecycle_counts(&runtime.pool).await?; - let typed_terminal_calls = fleet_terminal_call_count(&runtime.pool).await?; - let ambiguity_parks = fleet_ambiguous_model_call_park_count(&runtime.pool).await?; - let in_flight_hangs = model.in_flight_hangs(); - tasks.stop().await?; - runtime.stop().await?; - - assert_eq!(baseline_fleet.sessions.len(), FLEET_SESSION_COUNT - 1); - assert_eq!(fault_fleet.sessions.len(), 1); - assert_eq!(in_flight_hangs, 0, "hung call retained a pass slot"); - assert_eq!(active, 1, "the ambiguous issued call lost its durable park"); - assert_eq!(terminal, i64::try_from(FLEET_SESSION_COUNT - 1)?); - assert_eq!(typed_terminal_calls, i64::try_from(FLEET_SESSION_COUNT)?); - assert_eq!(ambiguity_parks, 1); - Ok(()) +/// Issue #1027: a post-acceptance model hang releases its authoritative pass +/// and reaches a durable typed ambiguity park inside the occupancy bound. +#[tokio::test(flavor = "multi_thread")] +#[ignore = "requires ephemeral PostgreSQL and a local Unix socket"] +async fn fleet_soak_hung_model_call_has_bounded_pass_occupancy_and_typed_disposition() +-> Result<(), Box> { + let mut runtime = RunningRuntime::start().await?; + let mut tasks: Option = None; + let scenario = AssertUnwindSafe(async { + let census_repository = FleetSoakCensusRepository::new(runtime.pool.clone()); + let baseline_fleet = commission_fleet(&runtime, 0, FLEET_SESSION_COUNT - 1).await?; + let model = FleetScriptedModel::new(FleetModelCardinality { + hanging: 1, + completing: FLEET_SESSION_COUNT - 1, + }); + tasks = Some(start_fleet_scheduler( + &mut runtime, + model.clone(), + SchedulerPassOccupancyBound::try_new(FLEET_BASELINE_OCCUPANCY_BOUND)?, + )?); + let completed_calls = wait_for_completed_calls(&model, FLEET_SESSION_COUNT - 1).await?; + wait_for_terminal_calls(&census_repository, &completed_calls).await?; + wait_for_terminal_turns(&census_repository, &completed_calls).await?; + tasks + .take() + .expect("the baseline fleet scheduler was installed") + .stop() + .await?; + runtime.restart().await?; + let fault_fleet = commission_fleet(&runtime, FLEET_SESSION_COUNT - 1, 1).await?; + tasks = Some(start_fleet_scheduler( + &mut runtime, + model.clone(), + SchedulerPassOccupancyBound::try_new(FLEET_OCCUPANCY_BOUND)?, + )?); + wait_for_hangs(&model, 1).await?; + let model_calls = census_repository.model_call_ids().await?; + assert_eq!( + baseline_fleet.sessions.len(), + FLEET_SESSION_COUNT - 1, + "baseline fleet session cardinality mismatch" + ); + assert_eq!( + fault_fleet.sessions.len(), + 1, + "fault fleet session cardinality mismatch" + ); + assert_eq!( + model_calls.len(), + FLEET_SESSION_COUNT, + "fleet model-call cardinality mismatch" + ); + let hung_model_calls = model_calls + .iter() + .copied() + .filter(|model_call| !completed_calls.contains(model_call)) + .collect::>(); + let [hung_model_call] = hung_model_calls.as_slice() else { + return Err(io::Error::other(format!( + "expected one hung model call, observed {hung_model_calls:?}" + )) + .into()); + }; + wait_for_ambiguity_park( + &census_repository, + &model, + *hung_model_call, + FLEET_ASSERTION_BOUND, + ) + .await?; + let census = census_repository.census_for(&model_calls).await?; + let hung_call_has_ambiguity_park = census_repository + .has_ambiguous_recovery_park(*hung_model_call) + .await?; + assert_hung_fleet_outcome(&model, census, hung_call_has_ambiguity_park) + }) + .catch_unwind() + .await; + + let scheduler_cleanup = match tasks { + Some(tasks) => abort_fleet_scheduler(tasks).await, + None => Ok(()), + }; + let runtime_cleanup = runtime.stop().await; + match scenario { + Ok(outcome) => { + scheduler_cleanup?; + runtime_cleanup?; + outcome + } + Err(panic) => { + if let Err(error) = scheduler_cleanup { + eprintln!("fleet scheduler cleanup after panic failed: {error}"); + } + if let Err(error) = runtime_cleanup { + eprintln!("fleet runtime cleanup after panic failed: {error}"); + } + resume_unwind(panic) + } + } } -/// Issue #1027: killing the daemon with a full fleet in model execution leaves -/// every turn available to bounded native recovery after replacement starts. +/// Issue #1027 / INV-034: killing the daemon with a full fleet in model +/// execution leaves every model call ambiguous. Ambiguous-operation +/// reconciliation must release local scheduler ownership and then resume or +/// terminalize every such turn once a replacement daemon takes over, without +/// waiting on a user decision. #[tokio::test(flavor = "multi_thread")] #[ignore = "requires ephemeral PostgreSQL and a local Unix socket"] async fn fleet_soak_kill_restart_resumes_or_terminalizes_every_active_turn() -> Result<(), Box> { let mut runtime = RunningRuntime::start().await?; - let fleet = commission_fleet(&runtime, 0, FLEET_SESSION_COUNT).await?; - let hanging_model = FleetScriptedModel::new(FLEET_SESSION_COUNT, 0); - let first_tasks = start_fleet_scheduler( - &mut runtime, - hanging_model.clone(), - SchedulerPassOccupancyBound::try_new(FLEET_BASELINE_OCCUPANCY_BOUND)?, - )?; - wait_for_hangs(&hanging_model, FLEET_SESSION_COUNT).await?; - wait_for_fleet_lifecycle_counts( - &runtime.pool, - (i64::try_from(FLEET_SESSION_COUNT)?, 0), - FLEET_SETUP_BOUND, - ) - .await?; - first_tasks.kill().await?; - let _recovered = runtime.kill_and_restart().await?; - let replacement_model = FleetScriptedModel::new(0, FLEET_SESSION_COUNT); - let replacement_tasks = start_fleet_scheduler( - &mut runtime, - replacement_model, - SchedulerPassOccupancyBound::try_new(FLEET_BASELINE_OCCUPANCY_BOUND)?, - )?; - wait_for_fleet_lifecycle_counts( - &runtime.pool, - (0, i64::try_from(FLEET_SESSION_COUNT)?), - FLEET_ASSERTION_BOUND, - ) - .await?; - let (active, terminal) = fleet_lifecycle_counts(&runtime.pool).await?; - let typed_terminal_calls = fleet_terminal_call_count(&runtime.pool).await?; - replacement_tasks.stop().await?; - runtime.stop().await?; - - assert_eq!(fleet.sessions.len(), FLEET_SESSION_COUNT); - assert_eq!(typed_terminal_calls, i64::try_from(FLEET_SESSION_COUNT)?); - assert_eq!(active, 0, "post-restart active turns were orphaned"); - assert_eq!(terminal, i64::try_from(FLEET_SESSION_COUNT)?); - Ok(()) + let mut tasks: Option = None; + let scenario = AssertUnwindSafe(async { + let census_repository = FleetSoakCensusRepository::new(runtime.pool.clone()); + let fleet = commission_fleet(&runtime, 0, FLEET_SESSION_COUNT).await?; + let hanging_model = FleetScriptedModel::new(FleetModelCardinality { + hanging: FLEET_SESSION_COUNT, + completing: 0, + }); + tasks = Some(start_fleet_scheduler( + &mut runtime, + hanging_model.clone(), + SchedulerPassOccupancyBound::try_new(FLEET_BASELINE_OCCUPANCY_BOUND)?, + )?); + wait_for_hangs(&hanging_model, FLEET_SESSION_COUNT).await?; + let pre_kill_model_call_ids = census_repository.model_call_ids().await?; + assert_eq!( + pre_kill_model_call_ids.len(), + FLEET_SESSION_COUNT, + "pre-kill model-call cardinality mismatch" + ); + tasks + .take() + .expect("the first fleet scheduler was installed") + .kill() + .await?; + wait_for_hangs(&hanging_model, 0).await?; + let _recovered = runtime.kill_and_restart().await?; + // One script per recoverable turn plus the readiness control, so the + // fixture does not decide whether reconciliation reissues a call. + let replacement_model = FleetScriptedModel::new(FleetModelCardinality { + hanging: 0, + completing: FLEET_SESSION_COUNT + 1, + }); + let replacement_reconciliation = runtime.reconciliation_witness(); + tasks = Some(start_fleet_scheduler( + &mut runtime, + replacement_model.clone(), + SchedulerPassOccupancyBound::try_new(FLEET_BASELINE_OCCUPANCY_BOUND)?, + )?); + wait_for_reconciliation(&replacement_reconciliation).await?; + let control_session = commission_fleet_control(&runtime).await?; + let control_model_call = + wait_for_model_call_for_session(&census_repository, control_session).await?; + wait_for_completed_call(&replacement_model, control_model_call).await?; + wait_for_terminal_turns(&census_repository, &pre_kill_model_call_ids).await?; + let census = census_repository + .census_for(&pre_kill_model_call_ids) + .await?; + assert_eq!( + fleet.sessions.len(), + FLEET_SESSION_COUNT, + "fleet session cardinality mismatch" + ); + assert_restarted_fleet_outcome(census, &hanging_model, &replacement_model) + }) + .catch_unwind() + .await; + + let scheduler_cleanup = match tasks { + Some(tasks) => abort_fleet_scheduler(tasks).await, + None => Ok(()), + }; + let runtime_cleanup = runtime.stop().await; + match scenario { + Ok(outcome) => { + scheduler_cleanup?; + runtime_cleanup?; + outcome + } + Err(panic) => { + if let Err(error) = scheduler_cleanup { + eprintln!("fleet scheduler cleanup after panic failed: {error}"); + } + if let Err(error) = runtime_cleanup { + eprintln!("fleet runtime cleanup after panic failed: {error}"); + } + resume_unwind(panic) + } + } } fn rendered_text_messages( @@ -2890,7 +3421,7 @@ struct InputAcceptedEventFacts { session_id: CanonicalUuid, accepted_input_id: CanonicalUuid, acceptance_position: u64, - content: InputContent, + content: UserInputContent, } #[derive(Debug, Eq, PartialEq)] @@ -3026,10 +3557,15 @@ async fn activate_turn(pool: &PgPool, session: SessionId) -> Result<(), Box Result<(), Box< runtime.stop().await } +#[tokio::test] +#[ignore = "requires ephemeral PostgreSQL and a local Unix socket"] +async fn process_runtime_reads_an_empty_operator_status_snapshot() -> Result<(), Box> { + let runtime = RunningRuntime::start().await?; + let mut connection = Connection::connect(runtime.socket()).await?; + + connection + .request(1, ClientRequest::ReadOperatorStatus {}) + .await?; + + let start = response_within(&mut connection).await?; + assert_eq!( + start.message(), + &ServerMessage::OperatorStatus(Box::new(OperatorStatusMessage::Start {})) + ); + let end = response_within(&mut connection).await?; + assert_eq!( + end.message(), + &ServerMessage::OperatorStatus(Box::new(OperatorStatusMessage::End(Box::new( + OperatorStatusEndMessage { + held_slot_count: CanonicalU64::new(0), + queued_obligation_count: CanonicalU64::new(0), + pull_request_convergence_count: CanonicalU64::new(0), + pending_stale_review_clearance_count: CanonicalU64::new(0), + }, + )))) + ); + + drop(connection); + runtime.stop().await +} + +#[tokio::test] +#[ignore = "requires ephemeral PostgreSQL and a local Unix socket"] +async fn process_runtime_reads_populated_convergence_status_rows() -> Result<(), Box> { + let runtime = RunningRuntime::start().await?; + let repository = RepositorySlug::try_new(String::from("example/repo"))?; + let gating_check = CheckRunName::try_new(String::from("ci"))?; + let clearance_fixture = OperatorStatusStaleReviewClearanceFixture { + review_node_id: String::from("PRR_node"), + reviewer: RepoWatchAuthorLogin::try_new(String::from("reviewer"))?, + reviewed_head_sha: CommitSha::try_new(String::from( + "3333333333333333333333333333333333333333", + ))?, + dismissal_message: String::from("Superseded by the current head."), + }; + let convergence_fixture = OperatorStatusConvergenceFixture { + number: PullRequestNumber::new(41.try_into()?), + head_sha: CommitSha::try_new(String::from("1111111111111111111111111111111111111111"))?, + base_branch: BranchName::try_new(String::from("main"))?, + base_revision: CommitSha::try_new(String::from( + "2222222222222222222222222222222222222222", + ))?, + mergeable_state: MergeableState::Mergeable, + settled: true, + review_decision: RepoWatchReviewDecision::ChangesRequested, + unresolved_threads: Vec::new(), + gating_check_count: 1, + non_green_gating_checks: vec![gating_check.clone()], + verdict: RepoWatchConvergenceVerdict::NotConverged, + stale_review_clearance: Some(clearance_fixture.clone()), + }; + OperatorStatusFixtureRepository::new(runtime.pool.clone()) + .seed_pull_request_convergences(&repository, std::slice::from_ref(&convergence_fixture)) + .await?; + + let mut connection = Connection::connect(runtime.socket()).await?; + connection + .request(1, ClientRequest::ReadOperatorStatus {}) + .await?; + + let start = response_within(&mut connection).await?; + assert_eq!( + start.message(), + &ServerMessage::OperatorStatus(Box::new(OperatorStatusMessage::Start {})) + ); + let convergence = response_within(&mut connection).await?; + assert_eq!( + convergence.message(), + &ServerMessage::OperatorStatus(Box::new(OperatorStatusMessage::PullRequestConvergence( + Box::new(OperatorStatusPullRequestConvergenceMessage { + repository: repository.as_str().to_owned(), + pull_request_number: CanonicalU64::new(convergence_fixture.number.get()), + head_sha: convergence_fixture.head_sha.as_str().to_owned(), + base_branch: convergence_fixture.base_branch.as_str().to_owned(), + base_revision: convergence_fixture.base_revision.as_str().to_owned(), + mergeable_state: OperatorStatusMergeableState::Mergeable, + review_decision: OperatorStatusReviewDecision::ChangesRequested, + unresolved_thread_count: CanonicalU64::new(0), + gating_check_count: CanonicalU64::new(convergence_fixture.gating_check_count), + non_green_gating_checks: vec![gating_check.as_str().to_owned()], + verdict: OperatorStatusConvergenceVerdict::NotConverged, + seal: None, + assessed_seconds_ago: CanonicalU64::new(0), + },) + ),)) + ); + let clearance = response_within(&mut connection).await?; + assert_eq!( + clearance.message(), + &ServerMessage::OperatorStatus(Box::new( + OperatorStatusMessage::PendingStaleReviewClearance(Box::new( + OperatorStatusPendingStaleReviewClearanceMessage { + repository: repository.as_str().to_owned(), + pull_request_number: CanonicalU64::new(convergence_fixture.number.get()), + current_head_sha: convergence_fixture.head_sha.as_str().to_owned(), + review_node_id: clearance_fixture.review_node_id.clone(), + reviewer: clearance_fixture.reviewer.as_str().to_owned(), + reviewed_head_sha: clearance_fixture.reviewed_head_sha.as_str().to_owned(), + pending_for_seconds: CanonicalU64::new(0), + }, + )), + )) + ); + let end = response_within(&mut connection).await?; + assert_eq!( + end.message(), + &ServerMessage::OperatorStatus(Box::new(OperatorStatusMessage::End(Box::new( + OperatorStatusEndMessage { + held_slot_count: CanonicalU64::new(0), + queued_obligation_count: CanonicalU64::new(0), + pull_request_convergence_count: CanonicalU64::new(1), + pending_stale_review_clearance_count: CanonicalU64::new(1), + }, + )))) + ); + + drop(connection); + runtime.stop().await +} + /// S33 / INV-008 / INV-012 / INV-046: one complete replacement /// request through the durable command boundary and validates catalog input /// before claiming a new command identity. @@ -4725,7 +5392,7 @@ async fn s28_submit_accepts_imported_session_continuation() -> Result<(), Box Result<(), Box Result<(), Box> { let session = SessionId::from_uuid(session_id.into_uuid()); - let mut activation = StartEligibleTurnService::new( - UuidV7StartEligibleTurnIdGenerator, - StartEligibleTurnRepository::new(pool.clone()), - ); - let StartEligibleTurnOutcome::Activated(_) = activation.execute(session).await? else { - return Err(io::Error::other("the queued fixture turn must activate").into()); - }; + activate_turn(pool, session).await?; let model_configuration = support::parse_model_configuration(MODEL_CONFIGURATION)?; let calls = PostgresModelCallRepository::new( @@ -4879,7 +5547,9 @@ async fn s04_inv029_reconcile_turn_releases_a_wedged_ambiguous_session() ClientRequest::SubmitInput { command_id: command()?, session_id, - content: InputContent::new(String::from("work while the ambiguity is unresolved")), + content: UserInputContent::text(String::from( + "work while the ambiguity is unresolved", + )), expected_defaults_version: Some(CanonicalU64::new(1)), model_settings: ModelSettingsOverlay::inherit_all(), delivery: None, @@ -4903,7 +5573,7 @@ async fn s04_inv029_reconcile_turn_releases_a_wedged_ambiguous_session() command_id: command()?, session_id, expected_active_turn_id: parked_turn_id, - content: InputContent::new(String::from("continue after reconciliation")), + content: UserInputContent::text(String::from("continue after reconciliation")), expected_defaults_version: CanonicalU64::new(1), model_settings: ModelSettingsOverlay::inherit_all(), }, @@ -5035,7 +5705,7 @@ async fn connection_reconciles_the_parked_turn( command_id: command()?, session_id, expected_active_turn_id: parked_turn_id, - content: InputContent::new(String::from("continue after the wedge")), + content: UserInputContent::text(String::from("continue after the wedge")), expected_defaults_version: CanonicalU64::new(1), model_settings: ModelSettingsOverlay::inherit_all(), }, @@ -5069,7 +5739,7 @@ async fn s04_inv029_reconcile_turn_refuses_a_turn_that_owes_no_decision() command_id: command()?, session_id, expected_active_turn_id: unparked_turn_id, - content: InputContent::new(String::from("names no parked turn")), + content: UserInputContent::text(String::from("names no parked turn")), expected_defaults_version: CanonicalU64::new(1), model_settings: ModelSettingsOverlay::inherit_all(), }, @@ -5091,7 +5761,7 @@ async fn s04_inv029_reconcile_turn_refuses_a_turn_that_owes_no_decision() command_id: command()?, session_id, expected_active_turn_id: parked_turn_id, - content: InputContent::new(String::from("continue after reconciliation")), + content: UserInputContent::text(String::from("continue after reconciliation")), expected_defaults_version: CanonicalU64::new(1), model_settings: ModelSettingsOverlay::inherit_all(), }, @@ -5107,7 +5777,7 @@ async fn s04_inv029_reconcile_turn_refuses_a_turn_that_owes_no_decision() command_id: command()?, session_id, expected_active_turn_id: parked_turn_id, - content: InputContent::new(String::from("the decision is already recorded")), + content: UserInputContent::text(String::from("the decision is already recorded")), expected_defaults_version: CanonicalU64::new(1), model_settings: ModelSettingsOverlay::inherit_all(), }, @@ -5142,7 +5812,7 @@ async fn inv012_reconcile_turn_replays_a_committed_decision() -> Result<(), Box< command_id: command()?, session_id, expected_active_turn_id: parked_turn_id, - content: InputContent::new(String::from("continue after reconciliation")), + content: UserInputContent::text(String::from("continue after reconciliation")), expected_defaults_version: CanonicalU64::new(1), model_settings: ModelSettingsOverlay::inherit_all(), }; @@ -5187,7 +5857,7 @@ async fn s37_inv053_reconcile_turn_records_its_per_call_model_settings() command_id: command()?, session_id, expected_active_turn_id: parked_turn_id, - content: InputContent::new(String::from("continue with deliberate reasoning")), + content: UserInputContent::text(String::from("continue with deliberate reasoning")), expected_defaults_version: CanonicalU64::new(1), model_settings: requested, }, @@ -5233,7 +5903,7 @@ async fn inv012_overlapping_equal_reconciliations_both_reach_the_committed_decis command_id: command()?, session_id, expected_active_turn_id: parked_turn_id, - content: InputContent::new(String::from("continue after reconciliation")), + content: UserInputContent::text(String::from("continue after reconciliation")), expected_defaults_version: CanonicalU64::new(1), model_settings: ModelSettingsOverlay::inherit_all(), }; @@ -5277,7 +5947,7 @@ async fn s04_reconcile_turn_reports_an_absent_session_exactly() -> Result<(), Bo command_id: command()?, session_id: absent_session_id, expected_active_turn_id: CanonicalUuid::from_uuid(Uuid::from_u128(0xB3)), - content: InputContent::new(String::from("names no session")), + content: UserInputContent::text(String::from("names no session")), expected_defaults_version: CanonicalU64::new(1), model_settings: ModelSettingsOverlay::inherit_all(), }, @@ -5324,7 +5994,7 @@ async fn process_runtime_reads_one_queued_transcript_snapshot() -> Result<(), Bo projected_state, TurnState::Queued { accepted_input_id: accepted_input, - content: InputContent::new(content), + content: UserInputContent::text(content), } ); let model_calls_end = response_within(&mut connection).await?; @@ -5366,7 +6036,7 @@ async fn s24_process_runtime_follow_snapshot_handoff_has_no_race() -> Result<(), // its start frame. Commit the next update before draining the snapshot so // only a subscription formed before snapshot transmission can retain it. let second_position = 2; - let second_content = InputContent::new(String::from("second input")); + let second_content = UserInputContent::text(String::from("second input")); commands .request( 6, @@ -5393,7 +6063,7 @@ async fn s24_process_runtime_follow_snapshot_handoff_has_no_race() -> Result<(), projected_state, TurnState::Queued { accepted_input_id: first_accepted_input, - content: InputContent::new(first_content), + content: UserInputContent::text(first_content), } ); let model_calls_end = response_within(&mut follow).await?; @@ -5613,7 +6283,7 @@ async fn authorize_issued_model_call( ) -> Result< ( PostgresModelCallRepository, - signalbox_domain::AuthorizedModelCall, + Box, ModelCallId, ), Box, @@ -5652,7 +6322,7 @@ async fn authorize_issued_model_call( else { return Err(io::Error::other("the fixture call must authorize send").into()); }; - Ok((calls, *authorized, call)) + Ok((calls, authorized, call)) } /// Commits a confirm-classified tool round over the issued fixture call, so @@ -5873,7 +6543,7 @@ async fn s07_inv029_stop_turn_cancels_the_activated_turn_and_queues_its_successo command_id: command()?, session_id, expected_active_turn_id: stopped_turn_id, - content: InputContent::new(successor_content.clone()), + content: UserInputContent::text(successor_content.clone()), expected_defaults_version: CanonicalU64::new(1), descendant_scope: DescendantTerminationScope::ParentAlone, model_settings: ModelSettingsOverlay::inherit_all(), @@ -5894,7 +6564,7 @@ async fn s07_inv029_stop_turn_cancels_the_activated_turn_and_queues_its_successo let TurnState::Queued { content, .. } = turn_state_of(&messages, successor_turn_id) else { panic!("fixture expected queued successor turn"); }; - assert_eq!(content.as_str(), successor_content); + assert_eq!(content.single_text(), Some(successor_content.as_str())); assert_eq!(cancellation_marker_count(&messages, stopped_turn_id), 1); drop(connection); @@ -5904,7 +6574,7 @@ async fn s07_inv029_stop_turn_cancels_the_activated_turn_and_queues_its_successo /// S07 / INV-029: stopping an issued call records the durable cancellation /// request and retains the slot for lifecycle closure, and a distinct second /// stop is refused with the exact prior stop authority named. -#[tokio::test] +#[tokio::test(flavor = "multi_thread")] #[ignore = "requires ephemeral PostgreSQL and a local Unix socket"] async fn s07_inv029_stop_turn_requests_cancellation_of_an_issued_call_exactly_once() -> Result<(), Box> { @@ -5913,7 +6583,8 @@ async fn s07_inv029_stop_turn_requests_cancellation_of_an_issued_call_exactly_on let session_id = create_alias_session(&mut connection).await?; let (_, stopped_turn_id) = submit_first_input(&mut connection, session_id, String::from("first request")).await?; - let (_, _, issued_call) = authorize_issued_model_call(&runtime.pool, session_id).await?; + let (_, _, issued_call) = + Box::pin(authorize_issued_model_call(&runtime.pool, session_id)).await?; let first_stop_command = command()?; connection @@ -5924,7 +6595,7 @@ async fn s07_inv029_stop_turn_requests_cancellation_of_an_issued_call_exactly_on command_id: first_stop_command, session_id, expected_active_turn_id: stopped_turn_id, - content: InputContent::new(String::from("continue after the stop")), + content: UserInputContent::text(String::from("continue after the stop")), expected_defaults_version: CanonicalU64::new(1), descendant_scope: DescendantTerminationScope::ParentAlone, model_settings: ModelSettingsOverlay::inherit_all(), @@ -5960,7 +6631,7 @@ async fn s07_inv029_stop_turn_requests_cancellation_of_an_issued_call_exactly_on command_id: command()?, session_id, expected_active_turn_id: stopped_turn_id, - content: InputContent::new(String::from("a second distinct stop")), + content: UserInputContent::text(String::from("a second distinct stop")), expected_defaults_version: CanonicalU64::new(1), descendant_scope: DescendantTerminationScope::ParentAlone, model_settings: ModelSettingsOverlay::inherit_all(), @@ -5999,7 +6670,7 @@ async fn s07_stop_turn_refusals_are_typed_and_exact() -> Result<(), Box Result<(), Box Result<(), Box Result<() command_id: command()?, session_id, expected_active_turn_id: stopped_turn_id, - content: InputContent::new(String::from("continue with deliberate reasoning")), + content: UserInputContent::text(String::from("continue with deliberate reasoning")), expected_defaults_version: CanonicalU64::new(1), descendant_scope: DescendantTerminationScope::ParentAlone, model_settings: requested, @@ -6156,7 +6827,7 @@ async fn s07_s10_inv029_stop_against_a_tool_round_stays_fail_closed_then_deny_an command_id: command()?, session_id, expected_active_turn_id: parked_turn_id, - content: InputContent::new(String::from("stop during the approval wait")), + content: UserInputContent::text(String::from("stop during the approval wait")), expected_defaults_version: CanonicalU64::new(1), descendant_scope: DescendantTerminationScope::ParentAlone, model_settings: ModelSettingsOverlay::inherit_all(), @@ -6216,7 +6887,7 @@ async fn s07_s10_inv029_stop_against_a_tool_round_stays_fail_closed_then_deny_an command_id: command()?, session_id, expected_active_turn_id: parked_turn_id, - content: InputContent::new(String::from("continue after the denied round")), + content: UserInputContent::text(String::from("continue after the denied round")), expected_defaults_version: CanonicalU64::new(1), descendant_scope: DescendantTerminationScope::ParentAlone, model_settings: ModelSettingsOverlay::inherit_all(), @@ -6232,7 +6903,7 @@ async fn s07_s10_inv029_stop_against_a_tool_round_stays_fail_closed_then_deny_an ClientRequest::SubmitInput { command_id: command()?, session_id, - content: InputContent::new(String::from("ordinary later work")), + content: UserInputContent::text(String::from("ordinary later work")), expected_defaults_version: Some(CanonicalU64::new(1)), model_settings: ModelSettingsOverlay::inherit_all(), delivery: None, @@ -6516,7 +7187,7 @@ async fn inv012_decide_tool_request_replays_equally_and_refuses_conflicting_reus ClientRequest::SubmitInput { command_id: submit_command, session_id, - content: InputContent::new(String::from("claims a submit identity")), + content: UserInputContent::text(String::from("claims a submit identity")), expected_defaults_version: Some(CanonicalU64::new(1)), model_settings: ModelSettingsOverlay::inherit_all(), delivery: None, @@ -6665,7 +7336,7 @@ async fn submit_queued_input( ClientRequest::SubmitInput { command_id: command()?, session_id, - content: InputContent::new(content.to_owned()), + content: UserInputContent::text(content.to_owned()), expected_defaults_version: Some(CanonicalU64::new(1)), model_settings: ModelSettingsOverlay::inherit_all(), delivery: Some(InputDelivery::Queue { @@ -6690,6 +7361,15 @@ async fn activate_expected_turn( StartEligibleTurnOutcome::Activated(activated) if activated.turn().into_uuid() == expected_turn.into_uuid() => { + let recorded = + signalboxd::WorkspaceInstructionRuntime::new(pool.clone(), None, Vec::new()) + .prepare(session, activated.turn()) + .await?; + if !recorded { + return Err( + io::Error::other("the fixture instruction manifest must record").into(), + ); + } Ok(()) } StartEligibleTurnOutcome::Activated(activated) => Err(io::Error::other(format!( @@ -6720,7 +7400,7 @@ async fn s08_steering_without_an_active_turn_is_a_typed_rejection() -> Result<() ClientRequest::SubmitInput { command_id: command()?, session_id, - content: InputContent::new(String::from("steer no turn")), + content: UserInputContent::text(String::from("steer no turn")), expected_defaults_version: None, model_settings: ModelSettingsOverlay::inherit_all(), delivery: Some(InputDelivery::Steer { @@ -7658,7 +8338,7 @@ async fn s01_s03_inv005_inv014_inv015_explicit_compaction_survives_restart_and_p ClientRequest::SubmitInput { command_id: command()?, session_id, - content: InputContent::new(second_user.clone()), + content: UserInputContent::text(second_user.clone()), expected_defaults_version: Some(CanonicalU64::new(1)), model_settings: ModelSettingsOverlay::inherit_all(), delivery: None, @@ -8044,7 +8724,7 @@ async fn inv009_inv014_compaction_preparation_serializes_turn_activation() ClientRequest::SubmitInput { command_id: command()?, session_id, - content: InputContent::new(String::from( + content: UserInputContent::text(String::from( "scheduler race successor remains singular", )), expected_defaults_version: Some(CanonicalU64::new(1)), @@ -8141,7 +8821,7 @@ async fn s01_s03_inv014_inv015_automatic_guard_compacts_before_ordinary_send() ClientRequest::SubmitInput { command_id: command()?, session_id, - content: InputContent::new(second_user.clone()), + content: UserInputContent::text(second_user.clone()), expected_defaults_version: Some(CanonicalU64::new(1)), model_settings: ModelSettingsOverlay::inherit_all(), delivery: None, @@ -8154,7 +8834,7 @@ async fn s01_s03_inv014_inv015_automatic_guard_compacts_before_ordinary_send() .replace("max_output_tokens = 256", "max_output_tokens = 1") .replace( "context_window_tokens = 200000", - "context_window_tokens = 5", + "context_window_tokens = 4096", ), )?; let ordinary_runtime = RecordingCountedScriptedModel::following( @@ -8163,7 +8843,7 @@ async fn s01_s03_inv014_inv015_automatic_guard_compacts_before_ordinary_send() "automatic guard current reply", TokenUsage::unreported(), )], - [40, 4], + [8192, 4], ); let summary_text = String::from("automatic guard summary"); let summary_runtime = ScriptedModel::single(completed_script( @@ -8200,10 +8880,6 @@ async fn s01_s03_inv014_inv015_automatic_guard_compacts_before_ordinary_send() signalbox_model_runtime::ConversationRole::User, format!("Signalbox prior-conversation summary:\n{summary_text}"), ), - ( - signalbox_model_runtime::ConversationRole::Assistant, - first_assistant.clone(), - ), ( signalbox_model_runtime::ConversationRole::User, second_user.clone(), @@ -8242,28 +8918,125 @@ async fn s01_s03_inv014_inv015_automatic_guard_compacts_before_ordinary_send() runtime.stop().await } -#[track_caller] -fn assert_context_still_exceeded( - outcome: Result<(), ContextGuardedTurnPassError>, - expected_turn: CanonicalUuid, -) where - CountError: std::fmt::Debug, - ExecutionError: std::fmt::Debug, -{ - match outcome { - Err(ContextGuardedTurnPassError::ContextStillExceeded(actual_turn)) => { - assert_eq!(*actual_turn.as_uuid(), expected_turn.into_uuid()); - } - other => panic!("expected ContextStillExceeded, got {other:?}"), - } +/// S01 / S03 / INV-014 / INV-015: provider-reported preflight rechecks the +/// completed summary and closes the queued candidate call-free when reserved +/// headroom is still unavailable. +#[tokio::test(flavor = "multi_thread")] +#[ignore = "requires ephemeral PostgreSQL and a local Unix socket"] +async fn s01_s03_inv014_inv015_reported_usage_rechecks_compaction_headroom() +-> Result<(), Box> { + let mut runtime = RunningRuntime::start().await?; + let mut connection = Connection::connect(runtime.socket()).await?; + let session_id = create_alias_session(&mut connection).await?; + let (_, first_turn) = submit_first_input( + &mut connection, + session_id, + String::from("reported usage historical request"), + ) + .await?; + let saturated_usage = TokenUsage { + input_tokens: Some(5000), + output_tokens: Some(0), + cache_creation_input_tokens: None, + cache_read_input_tokens: None, + }; + let first_runtime = ScriptedModel::single(completed_script( + "fixture-model", + "reported usage historical reply", + saturated_usage, + )); + let first_probe = + execute_streamed_turn(&mut runtime, first_runtime, session_id, first_turn).await?; + assert_eq!(first_probe.received_operations().len(), 1); + + connection + .request_version( + ProtocolVersion::One, + 40, + ClientRequest::SubmitInput { + command_id: command()?, + session_id, + content: UserInputContent::text(String::from("reported usage queued suffix")), + expected_defaults_version: Some(CanonicalU64::new(1)), + model_settings: ModelSettingsOverlay::inherit_all(), + delivery: None, + }, + ) + .await?; + let queued_turn = accepted_successor_turn(&mut connection, session_id, 2).await?; + let configuration = support::parse_model_configuration( + &MODEL_CONFIGURATION + .replace("max_output_tokens = 256", "max_output_tokens = 1") + .replace( + "context_window_tokens = 200000", + "context_window_tokens = 4096", + ), + )?; + let runtime_models = configuration.runtime_model_catalog(); + let summary_runtime = ScriptedModel::single(completed_script( + "fixture-model", + "reported usage summary remains saturated", + saturated_usage, + )); + let summary_probe = summary_runtime.clone(); + let compaction_model: Arc = + Arc::new(RuntimeContextCompactionModel::new( + summary_runtime, + runtime_models.clone(), + )); + let repository = PostgresModelCallRepository::new( + runtime.pool.clone(), + configuration.target_catalog(), + ModelCallCredentialReference::new("reported-usage-recheck-fixture"), + ) + .with_session_credentials(configuration.credential_family_catalog()); + let compaction = ReportedUsageCompaction::new( + StartEligibleTurnRepository::new(runtime.pool.clone()), + repository, + NoToolCatalog, + runtime_models, + configuration, + compaction_model, + ); + + let failed_turn = reported_usage_still_exceeded_turn( + compaction + .compact_if_needed(SessionId::from_uuid(session_id.into_uuid())) + .await, + ); + + assert_eq!(*failed_turn.as_uuid(), queued_turn.into_uuid()); + assert_eq!(summary_probe.received_operations().len(), 1); + let ordinary_call_count: i64 = + sqlx::query_scalar("SELECT count(*) FROM model_call WHERE turn_id = $1") + .bind(queued_turn.into_uuid()) + .fetch_one(&runtime.pool) + .await?; + assert_eq!(ordinary_call_count, 0); + let lifecycle: (String, Option, Option) = sqlx::query_as( + "SELECT state_kind, terminal_disposition_kind, terminal_model_call_id + FROM turn_lifecycle + WHERE session_id = $1 AND turn_id = $2", + ) + .bind(session_id.into_uuid()) + .bind(queued_turn.into_uuid()) + .fetch_one(&runtime.pool) + .await?; + assert_eq!( + lifecycle, + (String::from("terminal"), Some(String::from("failed")), None) + ); + + drop(connection); + runtime.stop().await } -/// S01 / S03 / INV-014 / INV-015: one queued candidate retains its durable -/// automatic-attempt marker across eligibility retries, so an oversized suffix -/// cannot issue a paid successor compaction on every sweep. +/// S01 / S03 / INV-014 / INV-015: a failed automatic compaction closes the +/// queued candidate call-free, so a later eligibility pass cannot dispatch +/// the known-oversized ordinary request. #[tokio::test(flavor = "multi_thread")] #[ignore = "requires ephemeral PostgreSQL and a local Unix socket"] -async fn s01_s03_inv014_inv015_automatic_guard_compacts_only_once_per_queued_turn() +async fn s01_s03_inv014_inv015_failed_automatic_compaction_closes_turn_call_free() -> Result<(), Box> { let mut runtime = RunningRuntime::start().await?; let mut connection = Connection::connect(runtime.socket()).await?; @@ -8291,7 +9064,7 @@ async fn s01_s03_inv014_inv015_automatic_guard_compacts_only_once_per_queued_tur ClientRequest::SubmitInput { command_id: command()?, session_id, - content: InputContent::new(oversized_suffix), + content: UserInputContent::text(oversized_suffix), expected_defaults_version: Some(CanonicalU64::new(1)), model_settings: ModelSettingsOverlay::inherit_all(), delivery: None, @@ -8304,16 +9077,21 @@ async fn s01_s03_inv014_inv015_automatic_guard_compacts_only_once_per_queued_tur .replace("max_output_tokens = 256", "max_output_tokens = 1") .replace( "context_window_tokens = 200000", - "context_window_tokens = 5", + "context_window_tokens = 4096", ), )?; let ordinary_runtime = - RecordingCountedScriptedModel::following(std::iter::empty::") + } + + pub fn external_image() -> Self { + Self::from_body(r#""#) + } + + pub fn nested_svg() -> Self { + Self::from_body(r#""#) + } + + pub fn excessive_elements() -> Self { + let mut body = String::new(); + for _ in 0..10_000 { + body.push_str(""); + } + Self::from_body(&body) + } + + pub fn empty_child_beyond_depth_limit() -> Self { + let mut body = String::new(); + for _ in 1..128 { + body.push_str(""); + } + body.push_str(""); + for _ in 1..128 { + body.push_str(""); + } + Self::from_body(&body) + } + + pub fn dimensions_with_surrounding_xml_whitespace() -> Self { + Self::raw(b"") + } + + pub fn output_bomb() -> Self { + Self::from_body(&format!("{}", "x".repeat(128 * 1024 + 1))) + } + + pub fn oversized_source() -> Self { + let mut bytes = br#""#.to_vec(); + bytes.resize(256 * 1024 + 1, b' '); + bytes.extend_from_slice(b""); + Self { bytes } + } + + pub fn malformed_dimension() -> Self { + Self { + bytes: br#""#.to_vec(), + } + } + + pub const fn expected_text(&self) -> &'static str { + FIXTURE_TEXT + } + + pub const fn expected_elements(&self) -> usize { + 3 + } + + pub const fn expected_width(&self) -> f64 { + FIXTURE_WIDTH + } + + pub const fn expected_height(&self) -> f64 { + FIXTURE_HEIGHT + } + + pub const fn expected_view_box(&self) -> [f64; 4] { + FIXTURE_VIEW_BOX + } + + pub const fn expected_whitespace_width(&self) -> f64 { + WHITESPACE_WIDTH + } + + pub const fn expected_whitespace_height(&self) -> f64 { + WHITESPACE_HEIGHT + } + + pub fn into_source(self) -> Result> { + MemorySource::new(self.bytes) + } + + fn from_body(body: &str) -> Self { + Self { + bytes: format!( + r#"{body}"#, + ) + .into_bytes(), + } + } +} + +#[derive(Clone)] +pub struct MemorySource { + bytes: Vec, + byte_length: NonZeroU64, +} + +impl MemorySource { + pub fn new(bytes: Vec) -> Result> { + let byte_length = NonZeroU64::new(u64::try_from(bytes.len())?) + .ok_or("fixture source must be nonempty")?; + Ok(Self { bytes, byte_length }) + } + + pub fn unknown(bytes: Vec) -> Result> { + Self::new(bytes) + } + + pub fn file_use(&self) -> Result> { + self.file_use_as("image/svg+xml") + } + + pub fn file_use_as(&self, media_type: &str) -> Result> { + Ok(FileUse::new( + self.digest(), + self.byte_length, + AttachmentKind::Image, + DeclaredMediaType::try_new(media_type)?, + None, + )) + } +} + +impl VerifiedBlobSource for MemorySource { + fn digest(&self) -> FileDigest { + FileDigest::from_bytes([0x53; 32]) + } + + fn byte_length(&self) -> NonZeroU64 { + self.byte_length + } + + fn read_range(&self, offset: u64, length: NonZeroU64) -> SourceReadFuture<'_> { + let outcome = usize::try_from(offset) + .ok() + .and_then(|start| { + usize::try_from(length.get()) + .ok() + .and_then(|length| start.checked_add(length).map(|end| (start, end))) + }) + .and_then(|(start, end)| self.bytes.get(start..end).map(<[u8]>::to_vec)) + .ok_or(SourceReadError::RangeOutOfBounds); + Box::pin(async move { outcome }) + } +} diff --git a/crates/file-media-adapter-svg/tests/svg_adapter.rs b/crates/file-media-adapter-svg/tests/svg_adapter.rs new file mode 100644 index 0000000000..ec64a57cdb --- /dev/null +++ b/crates/file-media-adapter-svg/tests/svg_adapter.rs @@ -0,0 +1,1538 @@ +//! Contract tests for the data-only SVG adapter. +//! Governed by `docs/spec/file-and-media.md`. + +mod fixtures; + +use std::error::Error; + +use fixtures::{MemorySource, SvgFixture}; +use signalbox_file_media_adapter_svg::{SvgProvider, declaration}; +use signalbox_file_media_runtime::{ + CancellationSignal, FileInspection, FileInspectionStatus, FileMediaCeilings, FileMediaFailure, + FileMediaProcessor, FileMediaProcessorFuture, FileMediaProvider, FileMediaProviderReadRequest, + FileMediaProviderValidationRequest, FileMediaRegistry, FileReadInput, FileReadRequest, + FileReadResult, InspectionRequest, NeverCancelled, ProcessorBoundaryFailure, ProcessorFailure, + ProcessorIsolation, ProcessorProbeOutput, ProcessorReadOutput, ProcessorValidationOutput, + ReadContinuation, ReadViewName, ReaderIdentity, VerifiedBlobSource, +}; + +macro_rules! assert_malformed { + ($fixture:expr, $expected_reason:expr $(,)?) => { + async { + let (status, reason) = malformed_observation($fixture).await?; + assert_eq!(status, FileInspectionStatus::Malformed); + assert_eq!(reason, $expected_reason); + Ok::<(), Box>(()) + } + }; +} + +struct DirectProcessor { + provider: SvgProvider, +} + +impl DirectProcessor { + const fn new() -> Self { + Self { + provider: SvgProvider::new(), + } + } +} + +impl FileMediaProcessor for DirectProcessor { + fn probe<'a>( + &'a self, + reader: &'a ReaderIdentity, + source: &'a dyn VerifiedBlobSource, + cancellation: &'a dyn CancellationSignal, + ) -> FileMediaProcessorFuture<'a, ProcessorProbeOutput> { + Box::pin(async move { + self.provider + .probe(reader, source, cancellation) + .await + .map_err(|_| ProcessorBoundaryFailure::Processor(ProcessorFailure::Failed)) + }) + } + + fn validate<'a>( + &'a self, + reader: &'a ReaderIdentity, + request: FileMediaProviderValidationRequest, + source: &'a dyn VerifiedBlobSource, + cancellation: &'a dyn CancellationSignal, + ) -> FileMediaProcessorFuture<'a, ProcessorValidationOutput> { + Box::pin(async move { + self.provider + .inspect(reader, request, source, cancellation) + .await + .map_err(|_| ProcessorBoundaryFailure::Processor(ProcessorFailure::Failed)) + }) + } + + fn read<'a>( + &'a self, + reader: &'a ReaderIdentity, + request: FileMediaProviderReadRequest, + source: &'a dyn VerifiedBlobSource, + cancellation: &'a dyn CancellationSignal, + ) -> FileMediaProcessorFuture<'a, ProcessorReadOutput> { + Box::pin(async move { + self.provider + .read(reader, request, source, cancellation) + .await + .map_err(|_| ProcessorBoundaryFailure::Processor(ProcessorFailure::Failed)) + }) + } +} + +struct AdversarialOutputProcessor { + direct: DirectProcessor, +} + +impl AdversarialOutputProcessor { + const fn new() -> Self { + Self { + direct: DirectProcessor::new(), + } + } +} + +impl FileMediaProcessor for AdversarialOutputProcessor { + fn probe<'a>( + &'a self, + reader: &'a ReaderIdentity, + source: &'a dyn VerifiedBlobSource, + cancellation: &'a dyn CancellationSignal, + ) -> FileMediaProcessorFuture<'a, ProcessorProbeOutput> { + self.direct.probe(reader, source, cancellation) + } + + fn validate<'a>( + &'a self, + reader: &'a ReaderIdentity, + request: FileMediaProviderValidationRequest, + source: &'a dyn VerifiedBlobSource, + cancellation: &'a dyn CancellationSignal, + ) -> FileMediaProcessorFuture<'a, ProcessorValidationOutput> { + self.direct.validate(reader, request, source, cancellation) + } + + fn read<'a>( + &'a self, + _reader: &'a ReaderIdentity, + _request: FileMediaProviderReadRequest, + _source: &'a dyn VerifiedBlobSource, + _cancellation: &'a dyn CancellationSignal, + ) -> FileMediaProcessorFuture<'a, ProcessorReadOutput> { + Box::pin(async { + Ok(ProcessorReadOutput::Text { + body: String::from("decoder\0injection"), + truncated: false, + cursor: None, + }) + }) + } +} + +#[test] +fn declaration_registers_data_only_svg_under_available_isolation() -> Result<(), Box> { + let registry = registry()?; + + assert_eq!(registry.providers(), &[declaration()?]); + Ok(()) +} + +#[tokio::test] +async fn generated_svg_validates() -> Result<(), Box> { + let source = SvgFixture::ordinary().into_source()?; + let inspection = inspect(&DirectProcessor::new(), &source).await?; + + assert_eq!(inspection.status(), FileInspectionStatus::Validated); + Ok(()) +} + +#[tokio::test] +async fn generated_svg_extracts_text() -> Result<(), Box> { + let fixture = SvgFixture::ordinary(); + let expected_text = fixture.expected_text(); + let source = fixture.into_source()?; + let result = read( + &DirectProcessor::new(), + &source, + "text", + serde_json::json!({}), + ) + .await?; + + assert!(complete_text(result)?.contains(expected_text)); + Ok(()) +} + +#[tokio::test] +async fn empty_text_element_matches_explicit_start_and_end() -> Result<(), Box> { + let processor = DirectProcessor::new(); + let empty = SvgFixture::raw(br#""#) + .into_source()?; + let explicit = + SvgFixture::raw(br#""#) + .into_source()?; + + let empty_text = complete_text(read(&processor, &empty, "text", serde_json::json!({})).await?)?; + let explicit_text = + complete_text(read(&processor, &explicit, "text", serde_json::json!({})).await?)?; + + assert_eq!(empty_text, explicit_text); + Ok(()) +} + +#[tokio::test] +async fn generated_svg_metadata_reports_fixture_shape() -> Result<(), Box> { + let fixture = SvgFixture::ordinary(); + let expected_elements = fixture.expected_elements(); + let expected_width = fixture.expected_width(); + let expected_height = fixture.expected_height(); + let expected_view_box = fixture.expected_view_box(); + let source = fixture.into_source()?; + let result = read( + &DirectProcessor::new(), + &source, + "metadata", + serde_json::json!({}), + ) + .await?; + let body = complete_structure(result)?; + + assert_eq!(body["elements"], expected_elements); + assert_eq!(body["width"], expected_width); + assert_eq!(body["height"], expected_height); + assert_eq!(body["view_box"], serde_json::json!(expected_view_box)); + Ok(()) +} + +#[tokio::test] +async fn truncated_svg_is_a_typed_malformed_inspection() -> Result<(), Box> { + assert_malformed!(SvgFixture::truncated(), "malformed_svg").await +} + +#[tokio::test] +async fn invalid_utf8_is_a_typed_malformed_inspection() -> Result<(), Box> { + assert_malformed!(SvgFixture::invalid_utf8(), "malformed_svg").await +} + +#[tokio::test] +async fn forbidden_xml_character_in_ordinary_text_is_rejected() -> Result<(), Box> { + assert_malformed!( + SvgFixture::raw(b"a\x01b",), + "malformed_svg", + ) + .await +} + +#[tokio::test] +async fn utf16_little_endian_svg_is_accepted() -> Result<(), Box> { + let xml = "ok"; + let mut bytes = vec![0xff, 0xfe]; + bytes.extend(xml.encode_utf16().flat_map(u16::to_le_bytes)); + let source = SvgFixture::raw(&bytes).into_source()?; + + assert_eq!( + inspect(&DirectProcessor::new(), &source).await?.status(), + FileInspectionStatus::Validated + ); + Ok(()) +} + +#[tokio::test] +async fn bomless_generic_utf16_is_rejected() -> Result<(), Box> { + let xml = + ""; + let bytes: Vec = xml.encode_utf16().flat_map(u16::to_le_bytes).collect(); + + assert_malformed!(SvgFixture::raw(&bytes), "malformed_svg").await +} + +#[tokio::test] +async fn utf16_svg_without_declared_encoding_is_accepted() -> Result<(), Box> { + let xml = ""; + let mut bytes = vec![0xff, 0xfe]; + bytes.extend(xml.encode_utf16().flat_map(u16::to_le_bytes)); + let source = SvgFixture::raw(&bytes).into_source()?; + + assert_eq!( + inspect(&DirectProcessor::new(), &source).await?.status(), + FileInspectionStatus::Validated + ); + Ok(()) +} + +#[tokio::test] +async fn utf16_big_endian_svg_is_accepted() -> Result<(), Box> { + let xml = "ok"; + let mut bytes = vec![0xfe, 0xff]; + bytes.extend(xml.encode_utf16().flat_map(u16::to_be_bytes)); + let source = SvgFixture::raw(&bytes).into_source()?; + + assert_eq!( + inspect(&DirectProcessor::new(), &source).await?.status(), + FileInspectionStatus::Validated + ); + Ok(()) +} + +#[tokio::test] +async fn probe_preserves_utf16_root_before_invalid_surrogate() -> Result<(), Box> { + let mut bytes = vec![0xff, 0xfe]; + bytes.extend( + r#""# + .encode_utf16() + .flat_map(u16::to_le_bytes), + ); + bytes.extend_from_slice(&0xdc00_u16.to_le_bytes()); + bytes.extend("".encode_utf16().flat_map(u16::to_le_bytes)); + let source = SvgFixture::raw(&bytes).into_source()?; + let inspection = + inspect_as(&DirectProcessor::new(), &source, "application/octet-stream").await?; + + assert_eq!(inspection.status(), FileInspectionStatus::Malformed); + Ok(()) +} + +#[tokio::test] +async fn entity_expansion_shape_is_rejected_before_expansion() -> Result<(), Box> { + assert_malformed!(SvgFixture::entity_bomb(), "malformed_svg").await +} + +#[tokio::test] +async fn script_is_rejected_as_active_content() -> Result<(), Box> { + assert_malformed!(SvgFixture::script(), "active_content").await +} + +#[tokio::test] +async fn external_image_is_rejected_without_resource_fetching() -> Result<(), Box> { + assert_malformed!(SvgFixture::external_image(), "external_reference").await +} + +#[tokio::test] +async fn nested_svg_is_rejected_as_a_recursive_container() -> Result<(), Box> { + assert_malformed!(SvgFixture::nested_svg(), "nested_svg").await +} + +#[tokio::test] +async fn excessive_element_count_is_a_typed_bounded_failure() -> Result<(), Box> { + assert_malformed!(SvgFixture::excessive_elements(), "structure_limit").await +} + +#[tokio::test] +async fn self_closing_element_counts_against_depth_limit() -> Result<(), Box> { + assert_malformed!( + SvgFixture::empty_child_beyond_depth_limit(), + "structure_limit", + ) + .await +} + +#[tokio::test] +async fn excessive_text_is_a_typed_output_failure() -> Result<(), Box> { + let source = SvgFixture::output_bomb().into_source()?; + let result = read( + &DirectProcessor::new(), + &source, + "text", + serde_json::json!({}), + ) + .await; + + assert_eq!(result, Err(FileMediaFailure::OutputUnitTooLarge)); + Ok(()) +} + +#[tokio::test] +async fn oversized_source_is_a_typed_validation_limit() -> Result<(), Box> { + assert_malformed!(SvgFixture::oversized_source(), "source_size_limit").await +} + +#[tokio::test] +async fn oversized_non_svg_source_is_unknown() -> Result<(), Box> { + let mut bytes = b"".to_vec(); + bytes.resize(256 * 1024 + 1, b'a'); + let source = SvgFixture::raw(&bytes).into_source()?; + + assert_eq!( + inspect(&DirectProcessor::new(), &source).await?.status(), + FileInspectionStatus::Unknown + ); + Ok(()) +} + +#[tokio::test] +async fn lowered_validation_ceiling_is_a_typed_validation_limit() -> Result<(), Box> { + let source = SvgFixture::ordinary().into_source()?; + let mut ceilings = FileMediaCeilings::version_one(); + ceilings.validation_source_bytes = 64; + let request = InspectionRequest { + source: source + .file_use() + .map_err(|_| FileMediaFailure::ProcessorFailed)?, + visible_part: None, + }; + let inspection = registry_with(ceilings)? + .inspect(&DirectProcessor::new(), request, &source, &NeverCancelled) + .await?; + + assert_eq!(inspection.status(), FileInspectionStatus::Malformed); + assert_eq!(malformed_reason(&inspection)?, "source_size_limit"); + Ok(()) +} + +#[tokio::test] +async fn truncated_root_probe_under_a_lowered_ceiling_is_a_typed_validation_limit() +-> Result<(), Box> { + // A legitimate SVG whose root start tag is long enough that a very low + // `validation_source_bytes` ceiling cuts the prefix probe off mid-tag. + // The unbounded top-level probe (bounded only by `PROBE_BYTES`) still + // sees the whole tag and classifies this as a structural SVG candidate, + // so the truncated re-probe inside `inspect` must not report `NoMatch` + // for what is really an indeterminate, not a disproven, root: doing so + // would turn a typed `source_size_limit` outcome into a hard processor + // failure for oversized-but-genuine SVG content. + let mut bytes = br#""); + bytes.resize(256 * 1024 + 1, b' '); + bytes.extend_from_slice(b""); + let source = SvgFixture::raw(&bytes).into_source()?; + let mut ceilings = FileMediaCeilings::version_one(); + ceilings.validation_source_bytes = 100; + let request = InspectionRequest { + source: source + .file_use() + .map_err(|_| FileMediaFailure::ProcessorFailed)?, + visible_part: None, + }; + let inspection = registry_with(ceilings)? + .inspect(&DirectProcessor::new(), request, &source, &NeverCancelled) + .await?; + + assert_eq!(inspection.status(), FileInspectionStatus::Malformed); + assert_eq!(malformed_reason(&inspection)?, "source_size_limit"); + Ok(()) +} + +#[tokio::test] +async fn non_svg_source_exceeding_only_the_lowered_ceiling_is_unknown() -> Result<(), Box> +{ + // Declared as SVG but not actually SVG, and only over the deployment's + // lowered `validation_source_bytes` ceiling, not the adapter's hard + // ceiling. Bounded root classification must still run so the registry + // reports the ordinary `Unknown` a declared-but-unvalidated candidate + // gets, rather than a misleading `Malformed`/`source_size_limit`. + let mut bytes = b"".to_vec(); + bytes.resize(50, b'a'); + let source = SvgFixture::raw(&bytes).into_source()?; + let mut ceilings = FileMediaCeilings::version_one(); + ceilings.validation_source_bytes = 10; + let request = InspectionRequest { + source: source + .file_use() + .map_err(|_| FileMediaFailure::ProcessorFailed)?, + visible_part: None, + }; + let inspection = registry_with(ceilings)? + .inspect(&DirectProcessor::new(), request, &source, &NeverCancelled) + .await?; + + assert_eq!(inspection.status(), FileInspectionStatus::Unknown); + Ok(()) +} + +#[tokio::test] +async fn malformed_dimension_is_rejected_before_metadata_output() -> Result<(), Box> { + assert_malformed!(SvgFixture::malformed_dimension(), "malformed_svg").await +} + +#[tokio::test] +async fn prefixed_svg_namespace_is_accepted() -> Result<(), Box> { + let source = SvgFixture::raw( + br#"ok"#, + ) + .into_source()?; + + assert_eq!( + inspect(&DirectProcessor::new(), &source).await?.status(), + FileInspectionStatus::Validated + ); + Ok(()) +} + +#[tokio::test] +async fn foreign_prefixed_svg_root_is_rejected() -> Result<(), Box> { + assert_malformed!( + SvgFixture::raw(br#""#), + "malformed_svg", + ) + .await +} + +#[tokio::test] +async fn foreign_namespaced_svg_root_is_unknown_without_svg_declaration() +-> Result<(), Box> { + let source = SvgFixture::raw(br#""#).into_source()?; + let inspection = + inspect_as(&DirectProcessor::new(), &source, "application/octet-stream").await?; + + assert_eq!(inspection.status(), FileInspectionStatus::Unknown); + Ok(()) +} + +#[tokio::test] +async fn foreign_namespaced_svg_root_is_unknown_with_svg_declaration() -> Result<(), Box> +{ + let source = SvgFixture::raw(br#""#).into_source()?; + let inspection = inspect(&DirectProcessor::new(), &source).await?; + + assert_eq!(inspection.status(), FileInspectionStatus::Unknown); + Ok(()) +} + +#[tokio::test] +async fn plain_text_is_unknown_with_svg_declaration() -> Result<(), Box> { + let source = SvgFixture::raw(b"not SVG").into_source()?; + let inspection = inspect(&DirectProcessor::new(), &source).await?; + + assert_eq!(inspection.status(), FileInspectionStatus::Unknown); + Ok(()) +} + +#[tokio::test] +async fn invalid_utf8_after_foreign_svg_root_is_unknown() -> Result<(), Box> { + let mut bytes = br#""#.to_vec(); + bytes.push(0xff); + let source = SvgFixture::raw(&bytes).into_source()?; + + assert_eq!( + inspect(&DirectProcessor::new(), &source).await?.status(), + FileInspectionStatus::Unknown + ); + Ok(()) +} + +#[tokio::test] +async fn unnamespaced_non_svg_root_is_unknown_with_svg_declaration() -> Result<(), Box> { + let source = SvgFixture::raw(br#""#).into_source()?; + let inspection = inspect(&DirectProcessor::new(), &source).await?; + + assert_eq!(inspection.status(), FileInspectionStatus::Unknown); + Ok(()) +} + +#[tokio::test] +async fn invalid_utf8_after_non_svg_root_is_unknown() -> Result<(), Box> { + let mut bytes = b"".to_vec(); + bytes.push(0xff); + bytes.extend_from_slice(b""); + let source = SvgFixture::raw(&bytes).into_source()?; + + assert_eq!( + inspect(&DirectProcessor::new(), &source).await?.status(), + FileInspectionStatus::Unknown + ); + Ok(()) +} + +#[tokio::test] +async fn processing_instruction_before_non_svg_root_is_unknown() -> Result<(), Box> { + let source = SvgFixture::raw(br#""#).into_source()?; + let inspection = inspect(&DirectProcessor::new(), &source).await?; + + assert_eq!(inspection.status(), FileInspectionStatus::Unknown); + Ok(()) +} + +#[tokio::test] +async fn leading_text_before_non_svg_root_is_unknown() -> Result<(), Box> { + let source = SvgFixture::raw(br#"junk"#).into_source()?; + let inspection = inspect(&DirectProcessor::new(), &source).await?; + + assert_eq!(inspection.status(), FileInspectionStatus::Unknown); + Ok(()) +} + +#[tokio::test] +async fn leading_text_before_svg_root_is_recognized_as_malformed() -> Result<(), Box> { + let source = + SvgFixture::raw(br#"junk"#).into_source()?; + let inspection = + inspect_as(&DirectProcessor::new(), &source, "application/octet-stream").await?; + + assert_eq!(inspection.status(), FileInspectionStatus::Malformed); + assert_eq!(malformed_reason(&inspection)?, "malformed_svg"); + Ok(()) +} + +#[tokio::test] +async fn invalid_declaration_before_non_svg_root_is_unknown() -> Result<(), Box> { + let source = SvgFixture::raw(br#""#).into_source()?; + let inspection = inspect(&DirectProcessor::new(), &source).await?; + + assert_eq!(inspection.status(), FileInspectionStatus::Unknown); + Ok(()) +} + +#[tokio::test] +async fn dtd_bearing_svg_is_malformed_without_svg_declaration() -> Result<(), Box> { + let source = SvgFixture::raw(br#""#) + .into_source()?; + let inspection = + inspect_as(&DirectProcessor::new(), &source, "application/octet-stream").await?; + + assert_eq!(inspection.status(), FileInspectionStatus::Malformed); + assert_eq!(malformed_reason(&inspection)?, "malformed_svg"); + Ok(()) +} + +#[tokio::test] +async fn animation_element_is_rejected_as_active_content() -> Result<(), Box> { + assert_malformed!( + SvgFixture::raw( + br#""#, + ), + "active_content", + ) + .await +} + +#[tokio::test] +async fn color_animation_element_is_rejected_as_active_content() -> Result<(), Box> { + assert_malformed!( + SvgFixture::raw( + br#""#, + ), + "active_content", + ) + .await +} + +#[tokio::test] +async fn foreign_namespaced_script_is_rejected_as_active_content() -> Result<(), Box> { + assert_malformed!( + SvgFixture::raw( + br#"run()"#, + ), + "active_content", + ) + .await +} + +#[tokio::test] +async fn foreign_resource_element_is_rejected() -> Result<(), Box> { + assert_malformed!( + SvgFixture::raw( + br#""#, + ), + "external_reference", + ) + .await +} + +#[tokio::test] +async fn foreign_input_source_is_rejected() -> Result<(), Box> { + assert_malformed!( + SvgFixture::raw( + br#""#, + ), + "external_reference", + ) + .await +} + +#[tokio::test] +async fn built_in_attribute_entity_is_accepted() -> Result<(), Box> { + let source = + SvgFixture::raw(br#""#) + .into_source()?; + + assert_eq!( + inspect(&DirectProcessor::new(), &source).await?.status(), + FileInspectionStatus::Validated + ); + Ok(()) +} + +#[tokio::test] +async fn cdata_in_text_is_extracted_as_inert_text() -> Result<(), Box> { + let source = SvgFixture::raw( + br#"a < b"#, + ) + .into_source()?; + let result = read( + &DirectProcessor::new(), + &source, + "text", + serde_json::json!({}), + ) + .await?; + + assert_eq!(complete_text(result)?, "a < b\n"); + Ok(()) +} + +#[tokio::test] +async fn top_level_cdata_before_non_svg_root_is_unknown() -> Result<(), Box> { + let source = SvgFixture::raw(br#"x"#).into_source()?; + let inspection = inspect(&DirectProcessor::new(), &source).await?; + + assert_eq!(inspection.status(), FileInspectionStatus::Unknown); + Ok(()) +} + +#[tokio::test] +async fn forbidden_xml_character_in_cdata_is_rejected() -> Result<(), Box> { + assert_malformed!( + SvgFixture::raw(b"a\x01b",), + "malformed_svg", + ) + .await +} + +#[tokio::test] +async fn trailing_document_entity_is_rejected() -> Result<(), Box> { + assert_malformed!( + SvgFixture::raw(br#"&"#), + "malformed_svg", + ) + .await +} + +#[tokio::test] +async fn malformed_view_box_separator_is_rejected() -> Result<(), Box> { + assert_malformed!( + SvgFixture::raw(br#""#), + "malformed_svg", + ) + .await +} + +#[tokio::test] +async fn invalid_view_box_number_token_is_rejected() -> Result<(), Box> { + assert_malformed!( + SvgFixture::raw(br#""#,), + "malformed_svg", + ) + .await +} + +#[tokio::test] +async fn zero_sized_view_box_is_valid_metadata() -> Result<(), Box> { + let source = + SvgFixture::raw(br#""#) + .into_source()?; + let result = read( + &DirectProcessor::new(), + &source, + "metadata", + serde_json::json!({}), + ) + .await?; + + assert_eq!( + complete_structure(result)?["view_box"], + serde_json::json!([0.0, 0.0, 0.0, 100.0]) + ); + Ok(()) +} + +#[tokio::test] +async fn escaped_css_resource_reference_is_rejected() -> Result<(), Box> { + assert_malformed!( + SvgFixture::raw( + br#""#, + ), + "external_reference", + ) + .await +} + +#[tokio::test] +async fn offset_path_resource_reference_is_rejected() -> Result<(), Box> { + assert_malformed!( + SvgFixture::raw( + br#""#, + ), + "external_reference", + ) + .await +} + +#[tokio::test] +async fn color_profile_resource_reference_is_rejected() -> Result<(), Box> { + assert_malformed!( + SvgFixture::raw( + br#""#, + ), + "external_reference", + ) + .await +} + +#[tokio::test] +async fn declaration_after_comment_is_rejected() -> Result<(), Box> { + assert_malformed!( + SvgFixture::raw( + br#""#, + ), + "malformed_svg", + ) + .await +} + +#[tokio::test] +async fn forbidden_attribute_control_character_is_rejected() -> Result<(), Box> { + assert_malformed!( + SvgFixture::raw(b""), + "malformed_svg", + ) + .await +} + +#[tokio::test] +async fn dimension_with_trailing_decimal_point_is_rejected() -> Result<(), Box> { + assert_malformed!( + SvgFixture::raw(br#""#,), + "malformed_svg", + ) + .await +} + +#[tokio::test] +async fn relative_dimension_units_are_valid_without_numeric_metadata() -> Result<(), Box> +{ + let source = + SvgFixture::raw(br#""#) + .into_source()?; + let result = read( + &DirectProcessor::new(), + &source, + "metadata", + serde_json::json!({}), + ) + .await?; + let body = complete_structure(result)?; + + assert_eq!(body["width"], serde_json::Value::Null); + assert_eq!(body["height"], serde_json::Value::Null); + Ok(()) +} + +#[tokio::test] +async fn modern_relative_dimension_units_are_valid_without_numeric_metadata() +-> Result<(), Box> { + let source = + SvgFixture::raw(br#""#) + .into_source()?; + let result = read( + &DirectProcessor::new(), + &source, + "metadata", + serde_json::json!({}), + ) + .await?; + let body = complete_structure(result)?; + + assert_eq!(body["width"], serde_json::Value::Null); + assert_eq!(body["height"], serde_json::Value::Null); + Ok(()) +} + +#[tokio::test] +async fn auto_and_container_dimensions_are_valid_without_numeric_metadata() +-> Result<(), Box> { + let source = + SvgFixture::raw(br#""#) + .into_source()?; + let result = read( + &DirectProcessor::new(), + &source, + "metadata", + serde_json::json!({}), + ) + .await?; + let body = complete_structure(result)?; + + assert_eq!(body["width"], serde_json::Value::Null); + assert_eq!(body["height"], serde_json::Value::Null); + Ok(()) +} + +#[tokio::test] +async fn css_wide_dimension_keywords_are_valid_without_numeric_metadata() +-> Result<(), Box> { + let source = SvgFixture::raw( + br#""#, + ) + .into_source()?; + let result = read( + &DirectProcessor::new(), + &source, + "metadata", + serde_json::json!({}), + ) + .await?; + let body = complete_structure(result)?; + + assert_eq!(body["width"], serde_json::Value::Null); + assert_eq!(body["height"], serde_json::Value::Null); + Ok(()) +} + +#[tokio::test] +async fn calculated_dimensions_are_valid_without_numeric_metadata() -> Result<(), Box> { + let source = + SvgFixture::raw(br#""#) + .into_source()?; + let result = read( + &DirectProcessor::new(), + &source, + "metadata", + serde_json::json!({}), + ) + .await?; + + assert_eq!( + complete_structure(result)?["width"], + serde_json::Value::Null + ); + Ok(()) +} + +#[tokio::test] +async fn calculation_products_are_valid_without_numeric_metadata() -> Result<(), Box> { + let source = + SvgFixture::raw(br#""#) + .into_source()?; + + assert_eq!( + inspect(&DirectProcessor::new(), &source).await?.status(), + FileInspectionStatus::Validated + ); + Ok(()) +} + +#[tokio::test] +async fn calculation_allows_negative_dimension_intermediates() -> Result<(), Box> { + let source = + SvgFixture::raw(br#""#) + .into_source()?; + + assert_eq!( + inspect(&DirectProcessor::new(), &source).await?.status(), + FileInspectionStatus::Validated + ); + Ok(()) +} + +#[tokio::test] +async fn negative_constant_calculation_result_is_rejected() -> Result<(), Box> { + assert_malformed!( + SvgFixture::raw(br#""#), + "malformed_svg", + ) + .await +} + +#[tokio::test] +async fn negative_constant_non_pixel_calculations_are_rejected() -> Result<(), Box> { + assert_malformed!( + SvgFixture::raw(br#""#), + "malformed_svg", + ) + .await?; + assert_malformed!( + SvgFixture::raw(br#""#), + "malformed_svg", + ) + .await +} + +#[tokio::test] +async fn clamp_minimum_precedes_inverted_maximum() -> Result<(), Box> { + let source = SvgFixture::raw( + br#""#, + ) + .into_source()?; + + assert_eq!( + inspect(&DirectProcessor::new(), &source).await?.status(), + FileInspectionStatus::Validated + ); + Ok(()) +} + +#[tokio::test] +async fn negative_mixed_absolute_unit_calculation_is_rejected() -> Result<(), Box> { + assert_malformed!( + SvgFixture::raw(br#""#,), + "malformed_svg", + ) + .await +} + +#[tokio::test] +async fn division_by_zero_calculation_is_rejected() -> Result<(), Box> { + assert_malformed!( + SvgFixture::raw(br#""#,), + "malformed_svg", + ) + .await +} + +#[tokio::test] +async fn calculated_e_prefixed_unit_is_accepted() -> Result<(), Box> { + let source = + SvgFixture::raw(br#""#) + .into_source()?; + + assert_eq!( + inspect(&DirectProcessor::new(), &source).await?.status(), + FileInspectionStatus::Validated + ); + Ok(()) +} + +#[tokio::test] +async fn calculation_function_names_are_ascii_case_insensitive() -> Result<(), Box> { + let source = + SvgFixture::raw(br#""#) + .into_source()?; + + assert_eq!( + inspect(&DirectProcessor::new(), &source).await?.status(), + FileInspectionStatus::Validated + ); + Ok(()) +} + +#[tokio::test] +async fn invalid_calculation_dimension_is_rejected() -> Result<(), Box> { + assert_malformed!( + SvgFixture::raw(br#""#,), + "malformed_svg", + ) + .await +} + +#[tokio::test] +async fn calculation_function_name_must_touch_opening_parenthesis() -> Result<(), Box> { + assert_malformed!( + SvgFixture::raw(br#""#), + "malformed_svg", + ) + .await?; + assert_malformed!( + SvgFixture::raw(br#""#), + "malformed_svg", + ) + .await +} + +#[tokio::test] +async fn negative_zero_calculation_dimension_is_accepted() -> Result<(), Box> { + let source = + SvgFixture::raw(br#""#) + .into_source()?; + + assert_eq!( + inspect(&DirectProcessor::new(), &source).await?.status(), + FileInspectionStatus::Validated + ); + Ok(()) +} + +#[tokio::test] +async fn calculation_treats_css_comments_as_whitespace() -> Result<(), Box> { + let source = SvgFixture::raw( + br#""#, + ) + .into_source()?; + + assert_eq!( + inspect(&DirectProcessor::new(), &source).await?.status(), + FileInspectionStatus::Validated + ); + Ok(()) +} + +#[tokio::test] +async fn calculation_addition_requires_surrounding_whitespace() -> Result<(), Box> { + assert_malformed!( + SvgFixture::raw(br#""#,), + "malformed_svg", + ) + .await +} + +#[tokio::test] +async fn empty_calculation_arguments_are_rejected() -> Result<(), Box> { + assert_malformed!( + SvgFixture::raw(br#""#), + "malformed_svg", + ) + .await +} + +#[tokio::test] +async fn dimensions_admit_surrounding_xml_whitespace() -> Result<(), Box> { + let fixture = SvgFixture::dimensions_with_surrounding_xml_whitespace(); + let expected_width = fixture.expected_whitespace_width(); + let expected_height = fixture.expected_whitespace_height(); + let source = fixture.into_source()?; + let result = read( + &DirectProcessor::new(), + &source, + "metadata", + serde_json::json!({}), + ) + .await?; + let body = complete_structure(result)?; + + assert_eq!(body["width"], expected_width); + assert_eq!(body["height"], expected_height); + Ok(()) +} + +#[tokio::test] +async fn non_xml_whitespace_outside_root_is_rejected() -> Result<(), Box> { + assert_malformed!( + SvgFixture::raw("\u{00a0}".as_bytes()), + "malformed_svg", + ) + .await +} + +#[tokio::test] +async fn prolog_processing_instruction_is_active_content() -> Result<(), Box> { + assert_malformed!( + SvgFixture::raw( + br#""#, + ), + "active_content", + ) + .await +} + +#[tokio::test] +async fn invalid_xml_comment_syntax_is_rejected() -> Result<(), Box> { + assert_malformed!( + SvgFixture::raw(br#""#), + "malformed_svg", + ) + .await +} + +#[tokio::test] +async fn xml_comment_body_ending_in_hyphen_is_rejected() -> Result<(), Box> { + assert_malformed!( + SvgFixture::raw(br#""#), + "malformed_svg", + ) + .await +} + +#[tokio::test] +async fn forbidden_xml_character_in_comment_is_rejected() -> Result<(), Box> { + assert_malformed!( + SvgFixture::raw(b"",), + "malformed_svg", + ) + .await +} + +#[tokio::test] +async fn incomplete_xml_declaration_is_rejected() -> Result<(), Box> { + assert_malformed!( + SvgFixture::raw(br#""#), + "malformed_svg", + ) + .await +} + +#[tokio::test] +async fn vertical_tab_in_xml_declaration_is_rejected() -> Result<(), Box> { + assert_malformed!( + SvgFixture::raw(b"",), + "malformed_svg", + ) + .await +} + +#[tokio::test] +async fn unbound_descendant_prefix_is_rejected() -> Result<(), Box> { + assert_malformed!( + SvgFixture::raw(br#""#), + "malformed_svg", + ) + .await +} + +#[tokio::test] +async fn numeric_character_references_are_extracted() -> Result<(), Box> { + let source = SvgFixture::raw( + br#"AB"#, + ) + .into_source()?; + let result = read( + &DirectProcessor::new(), + &source, + "text", + serde_json::json!({}), + ) + .await?; + + assert_eq!(complete_text(result)?, "AB\n"); + Ok(()) +} + +#[tokio::test] +async fn signed_numeric_character_reference_is_rejected() -> Result<(), Box> { + assert_malformed!( + SvgFixture::raw(br#"&#+65;"#,), + "malformed_svg", + ) + .await +} + +#[tokio::test] +async fn harmless_on_prefixed_names_are_accepted() -> Result<(), Box> { + let source = SvgFixture::raw( + br#""#, + ) + .into_source()?; + + assert_eq!( + inspect(&DirectProcessor::new(), &source).await?.status(), + FileInspectionStatus::Validated + ); + Ok(()) +} + +#[tokio::test] +async fn actual_event_handler_attribute_is_rejected() -> Result<(), Box> { + assert_malformed!( + SvgFixture::raw(br#""#), + "active_content", + ) + .await +} + +#[tokio::test] +async fn svg_handler_element_is_rejected_as_active_content() -> Result<(), Box> { + assert_malformed!( + SvgFixture::raw( + br#"alert(1)"#, + ), + "active_content", + ) + .await +} + +#[tokio::test] +async fn root_window_event_handler_is_rejected() -> Result<(), Box> { + assert_malformed!( + SvgFixture::raw(br#""#,), + "active_content", + ) + .await +} + +#[tokio::test] +async fn unbound_attribute_prefix_is_rejected() -> Result<(), Box> { + assert_malformed!( + SvgFixture::raw(br#""#), + "malformed_svg", + ) + .await +} + +#[tokio::test] +async fn reserved_xml_prefix_rebinding_is_rejected() -> Result<(), Box> { + assert_malformed!( + SvgFixture::raw(br#""#), + "malformed_svg", + ) + .await +} + +#[tokio::test] +async fn invalid_namespace_iri_is_rejected() -> Result<(), Box> { + assert_malformed!( + SvgFixture::raw( + br#""#, + ), + "malformed_svg", + ) + .await +} + +#[tokio::test] +async fn invalid_namespace_iri_authority_is_rejected() -> Result<(), Box> { + assert_malformed!( + SvgFixture::raw( + br#""#, + ), + "malformed_svg", + ) + .await +} + +#[tokio::test] +async fn duplicate_expanded_attribute_name_is_rejected() -> Result<(), Box> { + assert_malformed!( + SvgFixture::raw( + br#""#, + ), + "malformed_svg", + ) + .await +} + +#[tokio::test] +async fn foreign_namespaced_width_does_not_change_metadata() -> Result<(), Box> { + let source = SvgFixture::raw( + br#""#, + ) + .into_source()?; + let result = read( + &DirectProcessor::new(), + &source, + "metadata", + serde_json::json!({}), + ) + .await?; + let body = complete_structure(result)?; + + assert_eq!(body["width"], 10.0); + Ok(()) +} + +#[tokio::test] +async fn namespace_character_references_are_expanded_before_policy_checks() +-> Result<(), Box> { + assert_malformed!( + SvgFixture::raw( + br#""#, + ), + "external_reference", + ) + .await +} + +#[tokio::test] +async fn url_text_in_inert_attribute_is_accepted() -> Result<(), Box> { + let source = SvgFixture::raw( + br#""#, + ) + .into_source()?; + + assert_eq!( + inspect(&DirectProcessor::new(), &source).await?.status(), + FileInspectionStatus::Validated + ); + Ok(()) +} + +#[tokio::test] +async fn context_menu_event_handler_is_rejected() -> Result<(), Box> { + assert_malformed!( + SvgFixture::raw(br#""#), + "active_content", + ) + .await +} + +#[tokio::test] +async fn auxiliary_click_event_handler_is_rejected() -> Result<(), Box> { + assert_malformed!( + SvgFixture::raw(br#""#), + "active_content", + ) + .await +} + +#[tokio::test] +async fn invalid_descendant_element_name_is_rejected() -> Result<(), Box> { + assert_malformed!( + SvgFixture::raw(br#"<1path/>"#), + "malformed_svg", + ) + .await +} + +#[tokio::test] +async fn forbidden_character_data_terminator_is_rejected() -> Result<(), Box> { + assert_malformed!( + SvgFixture::raw(br#"a]]>b"#), + "malformed_svg", + ) + .await +} + +#[tokio::test] +async fn unknown_bytes_remain_a_typed_unknown_inspection() -> Result<(), Box> { + let source = MemorySource::unknown(b"not SVG".to_vec())?; + let inspection = + inspect_as(&DirectProcessor::new(), &source, "application/octet-stream").await?; + + assert_eq!(inspection.status(), FileInspectionStatus::Unknown); + Ok(()) +} + +#[tokio::test] +async fn probe_accepts_root_before_truncated_utf8_character() -> Result<(), Box> { + let mut bytes = br#""#.to_vec(); + bytes.resize(65_535, b'a'); + bytes.extend_from_slice("é".as_bytes()); + let source = SvgFixture::raw(&bytes).into_source()?; + let inspection = + inspect_as(&DirectProcessor::new(), &source, "application/octet-stream").await?; + + assert_ne!(inspection.status(), FileInspectionStatus::Unknown); + Ok(()) +} + +#[tokio::test] +async fn probe_recognizes_root_before_invalid_utf8() -> Result<(), Box> { + let mut bytes = br#""#.to_vec(); + bytes.push(0xff); + bytes.extend_from_slice(b""); + let source = SvgFixture::raw(&bytes).into_source()?; + let inspection = + inspect_as(&DirectProcessor::new(), &source, "application/octet-stream").await?; + + assert_eq!(inspection.status(), FileInspectionStatus::Malformed); + Ok(()) +} + +#[tokio::test] +async fn hostile_view_arguments_are_typed_and_content_silent() -> Result<(), Box> { + let source = SvgFixture::ordinary().into_source()?; + let result = read( + &DirectProcessor::new(), + &source, + "text", + serde_json::json!({"resource": "../../host"}), + ) + .await; + + assert_eq!(result, Err(FileMediaFailure::InvalidViewArguments)); + Ok(()) +} + +#[tokio::test] +async fn adversarial_decoder_text_is_rejected_by_registry_sanitization() +-> Result<(), Box> { + let source = SvgFixture::ordinary().into_source()?; + let result = read( + &AdversarialOutputProcessor::new(), + &source, + "text", + serde_json::json!({}), + ) + .await; + + assert_eq!(result, Err(FileMediaFailure::ProcessorFailed)); + Ok(()) +} + +fn registry() -> Result> { + registry_with(FileMediaCeilings::version_one()) +} + +fn registry_with(ceilings: FileMediaCeilings) -> Result> { + Ok(FileMediaRegistry::try_new( + vec![declaration()?], + ceilings, + ProcessorIsolation::Available, + )?) +} + +async fn inspect( + processor: &dyn FileMediaProcessor, + source: &MemorySource, +) -> Result { + inspect_as(processor, source, "image/svg+xml").await +} + +async fn inspect_as( + processor: &dyn FileMediaProcessor, + source: &MemorySource, + declared_media_type: &str, +) -> Result { + let request = InspectionRequest { + source: source + .file_use_as(declared_media_type) + .map_err(|_| FileMediaFailure::ProcessorFailed)?, + visible_part: None, + }; + registry() + .map_err(|_| FileMediaFailure::ProcessorFailed)? + .inspect(processor, request, source, &NeverCancelled) + .await +} + +async fn read( + processor: &dyn FileMediaProcessor, + source: &MemorySource, + view: &str, + options: serde_json::Value, +) -> Result { + let request = FileReadRequest { + inspection: InspectionRequest { + source: source + .file_use() + .map_err(|_| FileMediaFailure::ProcessorFailed)?, + visible_part: None, + }, + view: ReadViewName::try_new(view).map_err(|_| FileMediaFailure::ProcessorFailed)?, + input: FileReadInput::Initial { options }, + }; + registry() + .map_err(|_| FileMediaFailure::ProcessorFailed)? + .read(processor, request, source, &NeverCancelled) + .await +} + +async fn malformed_observation( + fixture: SvgFixture, +) -> Result<(FileInspectionStatus, String), Box> { + let source = fixture.into_source()?; + let inspection = inspect(&DirectProcessor::new(), &source).await?; + let status = inspection.status(); + let reason = String::from(malformed_reason(&inspection)?); + Ok((status, reason)) +} + +fn malformed_reason(inspection: &FileInspection) -> Result<&str, Box> { + match inspection { + FileInspection::Malformed { reason_code, .. } => Ok(reason_code.as_str()), + _ => Err("expected malformed SVG".into()), + } +} + +fn complete_text(result: FileReadResult) -> Result> { + match result { + FileReadResult::Text { + body, + continuation: ReadContinuation::Complete, + } => Ok(body), + _ => Err("expected complete text result".into()), + } +} + +fn complete_structure(result: FileReadResult) -> Result> { + match result { + FileReadResult::Structured { + body, + continuation: ReadContinuation::Complete, + } => Ok(body), + _ => Err("expected complete structured result".into()), + } +} diff --git a/crates/file-media-adapters-audio/Cargo.toml b/crates/file-media-adapters-audio/Cargo.toml new file mode 100644 index 0000000000..ce2ce46ee8 --- /dev/null +++ b/crates/file-media-adapters-audio/Cargo.toml @@ -0,0 +1,35 @@ +[package] +name = "signalbox-file-media-adapters-audio" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +repository.workspace = true +publish.workspace = true + +[[bin]] +name = "signalbox-file-media-audio-worker" +path = "src/bin/signalbox-file-media-audio-worker.rs" + +[dependencies] +ogg = "0.9.2" +opus-rs = "0.1.28" +serde_json = "1.0.140" +signalbox-file-media-processor-runtime = { path = "../file-media-processor-runtime" } +signalbox-file-media-runtime = { path = "../file-media-runtime" } +symphonia = { version = "0.6.1", default-features = false, features = [ + "flac", + "mp3", + "pcm", + "wav", +] } +tokio = { version = "1.53.0", default-features = false, features = [ + "macros", + "rt", +] } + +[dev-dependencies] +rusty_mp3 = "0.7.0" + +[lints] +workspace = true diff --git a/crates/file-media-adapters-audio/src/adapter.rs b/crates/file-media-adapters-audio/src/adapter.rs new file mode 100644 index 0000000000..f6bb1d16d9 --- /dev/null +++ b/crates/file-media-adapters-audio/src/adapter.rs @@ -0,0 +1,691 @@ +use std::{io::Cursor, num::NonZeroU64}; + +use ogg::PacketReader; +use opus_rs::OpusDecoder; +use signalbox_file_media_runtime::{ + CancellationSignal, FileMediaProviderFailure, FileMediaProviderReadRequest, + FileMediaProviderValidationRequest, FileReadInput, MAX_AUDIO_CHANNELS, MAX_AUDIO_CLIP_SECONDS, + MAX_AUDIO_SAMPLE_RATE_HZ, ProbeStrength, ProcessorProbeOutput, ProcessorReadOutput, + ProcessorValidationOutput, ValidationEvidence, VerifiedBlobSource, +}; +use symphonia::{ + core::{ + codecs::audio::AudioDecoderOptions, + formats::{FormatOptions, FormatReader, TrackType}, + io::{MediaSourceStream, MediaSourceStreamOptions}, + }, + default::formats::{FlacReader, MpaReader, WavReader}, +}; + +use crate::{ + AUDIO_WHOLE_SOURCE_RANGES, AdapterFormat, Id3Footer, MAX_AUDIO_SOURCE_BYTES, id3_audio_offset, + id3_tag_layout, options_are_empty, source, valid_id3_footer, valid_mp3_frame_header, +}; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +struct AudioMetadata { + channels: usize, + sample_rate_hz: u32, +} + +pub(crate) async fn probe( + format: AdapterFormat, + source: &dyn VerifiedBlobSource, + cancellation: &dyn CancellationSignal, +) -> Result { + let prefix = source::read_probe_prefix(source, cancellation).await?; + let matches = if format == AdapterFormat::Mp3 && prefix.starts_with(b"ID3") { + let Some((tag_end, has_footer)) = id3_tag_layout(&prefix) else { + return Ok(ProcessorProbeOutput::NoMatch); + }; + let audio_offset = if has_footer { + let Some(footer_end) = tag_end.checked_add(10) else { + return Ok(ProcessorProbeOutput::NoMatch); + }; + let footer = if let Some(footer) = prefix.get(tag_end..footer_end) { + footer.to_vec() + } else { + let Ok(offset) = u64::try_from(tag_end) else { + return Ok(ProcessorProbeOutput::NoMatch); + }; + let Some(remaining) = source.byte_length().get().checked_sub(offset) else { + return Ok(ProcessorProbeOutput::NoMatch); + }; + if remaining < 10 { + return Ok(ProcessorProbeOutput::NoMatch); + } + source + .read_range( + offset, + NonZeroU64::new(10).ok_or(FileMediaProviderFailure::Failed)?, + ) + .await + .map_err(|_| FileMediaProviderFailure::Failed)? + }; + if !valid_id3_footer(Id3Footer { + header: &prefix[..10], + footer: &footer, + }) { + return Ok(ProcessorProbeOutput::NoMatch); + } + footer_end + } else { + tag_end + }; + if let Some(header) = prefix.get(audio_offset..audio_offset.saturating_add(4)) { + valid_mp3_frame_header(header) + } else { + let Ok(offset) = u64::try_from(audio_offset) else { + return Ok(ProcessorProbeOutput::NoMatch); + }; + let Some(remaining) = source.byte_length().get().checked_sub(offset) else { + return Ok(ProcessorProbeOutput::NoMatch); + }; + if remaining < 4 { + return Ok(ProcessorProbeOutput::NoMatch); + } + let header = source + .read_range( + offset, + NonZeroU64::new(4).ok_or(FileMediaProviderFailure::Failed)?, + ) + .await + .map_err(|_| FileMediaProviderFailure::Failed)?; + valid_mp3_frame_header(&header) + } + } else { + format.matches_signature(&prefix) + }; + if matches { + Ok(ProcessorProbeOutput::Candidate { + media_type: String::from(format.media_type()), + strength: ProbeStrength::Strong, + }) + } else { + Ok(ProcessorProbeOutput::NoMatch) + } +} + +pub(crate) async fn inspect( + format: AdapterFormat, + request: FileMediaProviderValidationRequest, + source: &dyn VerifiedBlobSource, + cancellation: &dyn CancellationSignal, +) -> Result { + if request.media_type.as_str() != format.media_type() { + return Err(FileMediaProviderFailure::Failed); + } + let Some(bytes) = source::read_complete( + source, + cancellation, + request.maximum_source_bytes.min(MAX_AUDIO_SOURCE_BYTES), + request.maximum_ranges, + ) + .await? + else { + return Ok(validation_failure( + format, + request.evidence, + "source_too_large", + )); + }; + let metadata = match decode(format, &bytes) { + Ok(metadata) => metadata, + Err(reason) => return Ok(validation_failure(format, request.evidence, reason)), + }; + Ok(ProcessorValidationOutput::Validated { + media_type: String::from(format.media_type()), + evidence: request.evidence, + metadata_json: metadata_json(metadata)?, + }) +} + +pub(crate) async fn read( + format: AdapterFormat, + request: FileMediaProviderReadRequest, + source: &dyn VerifiedBlobSource, + cancellation: &dyn CancellationSignal, +) -> Result { + let valid_input = matches!( + &request.input, + FileReadInput::Initial { options } if options_are_empty(options) + ); + if request.view.as_str() != "metadata" || !valid_input { + return Ok(ProcessorReadOutput::InvalidViewArguments); + } + let Some(bytes) = source::read_complete( + source, + cancellation, + MAX_AUDIO_SOURCE_BYTES, + AUDIO_WHOLE_SOURCE_RANGES, + ) + .await? + else { + return Ok(ProcessorReadOutput::SourceTooLarge { + maximum_bytes: MAX_AUDIO_SOURCE_BYTES, + }); + }; + let metadata = decode(format, &bytes).map_err(|_| FileMediaProviderFailure::Failed)?; + Ok(ProcessorReadOutput::Structured { + body_json: metadata_json(metadata)?, + truncated: false, + cursor: None, + }) +} + +fn decode(format: AdapterFormat, bytes: &[u8]) -> Result { + match format { + AdapterFormat::Wav | AdapterFormat::Mp3 | AdapterFormat::Flac => { + decode_with_symphonia(format, bytes) + } + AdapterFormat::OggOpus => decode_ogg_opus(bytes), + } +} + +fn decode_with_symphonia( + format: AdapterFormat, + bytes: &[u8], +) -> Result { + let bytes = if format == AdapterFormat::Mp3 { + mp3_audio_bytes(bytes)? + } else { + bytes + }; + let stream = MediaSourceStream::new( + Box::new(Cursor::new(bytes.to_vec())), + MediaSourceStreamOptions::default(), + ); + let options = FormatOptions::default(); + let mut reader: Box = match format { + AdapterFormat::Wav => { + Box::new(WavReader::try_new(stream, options).map_err(|_| "malformed_audio")?) + } + AdapterFormat::Mp3 => { + Box::new(MpaReader::try_new(stream, options).map_err(|_| "malformed_audio")?) + } + AdapterFormat::Flac => { + Box::new(FlacReader::try_new(stream, options).map_err(|_| "malformed_audio")?) + } + AdapterFormat::OggOpus => return Err("malformed_audio"), + }; + let track = reader + .default_track(TrackType::Audio) + .ok_or("malformed_audio")?; + let track_id = track.id; + let codec_parameters = track + .codec_params + .as_ref() + .and_then(symphonia::core::codecs::CodecParameters::audio) + .cloned() + .ok_or("malformed_audio")?; + let declared_metadata = AudioMetadata { + channels: codec_parameters + .channels + .as_ref() + .ok_or("malformed_audio")? + .count(), + sample_rate_hz: codec_parameters.sample_rate.ok_or("malformed_audio")?, + }; + validate_shape(declared_metadata)?; + let declared_frames = match format { + AdapterFormat::Flac => flac_declared_frames(bytes)?, + AdapterFormat::Mp3 => mp3_declared_frames(bytes)?, + AdapterFormat::Wav | AdapterFormat::OggOpus => None, + }; + let mut decoder_options = AudioDecoderOptions::default(); + decoder_options.verify = true; + let mut decoder = symphonia::default::get_codecs() + .make_audio_decoder(&codec_parameters, &decoder_options) + .map_err(|_| "malformed_audio")?; + let mut metadata = (format == AdapterFormat::Wav).then_some(declared_metadata); + let mut raw_decoded_frames = 0_u64; + let mut presented_frames = 0_u64; + + while let Some(packet) = reader.next_packet().map_err(|_| "malformed_audio")? { + if packet.track_id != track_id { + continue; + } + let decoded = decoder.decode(&packet).map_err(|_| "malformed_audio")?; + let observed = AudioMetadata { + channels: decoded.spec().channels().count(), + sample_rate_hz: decoded.spec().rate(), + }; + validate_shape(observed)?; + if (format == AdapterFormat::Flac && observed != declared_metadata) + || metadata.is_some_and(|prior| prior != observed) + { + return Err("malformed_audio"); + } + metadata = Some(observed); + let decoded_packet_frames = + u64::try_from(decoded.frames()).map_err(|_| "duration_limit_exceeded")?; + raw_decoded_frames = raw_decoded_frames + .checked_add(decoded_packet_frames) + .ok_or("duration_limit_exceeded")?; + let trimmed_frames = packet + .trim_start + .get() + .checked_add(packet.trim_end.get()) + .ok_or("duration_limit_exceeded")?; + let presented_packet_frames = decoded_packet_frames + .checked_sub(trimmed_frames) + .ok_or("malformed_audio")?; + presented_frames = presented_frames + .checked_add(presented_packet_frames) + .ok_or("duration_limit_exceeded")?; + validate_duration(presented_frames, observed.sample_rate_hz)?; + } + let metadata = metadata.ok_or("malformed_audio")?; + // `declared_frames` is already `None` for FLAC's "unknown total samples" + // STREAMINFO convention (`flac_declared_frames` maps 0 to `None`), so no + // extra zero-skip is needed here for FLAC. For MP3, a Xing/VBRI header + // declaring exactly one total frame yields `Some(0)` audio frames after + // subtracting the metadata frame, and that is a real declared count, not + // an "unknown" sentinel; comparing it lets a header claiming zero audio + // frames be caught if the source decodes any. + if matches!(format, AdapterFormat::Mp3 | AdapterFormat::Flac) + && declared_frames.is_some_and(|frames| frames != raw_decoded_frames) + { + return Err("malformed_audio"); + } + if decoder.finalize().verify_ok == Some(false) { + return Err("malformed_audio"); + } + Ok(metadata) +} + +fn decode_ogg_opus(bytes: &[u8]) -> Result { + let mut packets = PacketReader::new(Cursor::new(bytes)); + let head = packets + .read_packet() + .map_err(|_| "malformed_audio")? + .ok_or("malformed_audio")?; + if !head.first_in_stream() + || !head.first_in_page() + || !head.last_in_page() + || head.last_in_stream() + || head.absgp_page() != 0 + { + return Err("malformed_audio"); + } + let (metadata, pre_skip) = parse_opus_head(&head.data)?; + let serial = head.stream_serial(); + let tags = packets + .read_packet() + .map_err(|_| "malformed_audio")? + .ok_or("malformed_audio")?; + if tags.stream_serial() != serial + || tags.last_in_stream() + || (tags.last_in_page() && tags.absgp_page() != 0) + || !valid_opus_tags(&tags.data) + { + return Err("malformed_audio"); + } + + let mut decoder = + OpusDecoder::new(48_000, metadata.channels).map_err(|_| "unsupported_opus_mapping")?; + let mut output = vec![0.0_f32; 5_760 * metadata.channels]; + let mut decoded_frames = 0_u64; + let mut audio_packets = 0_u64; + let mut final_granule = None; + let mut final_packet_frames = None; + let mut completed_page_granule = 0_u64; + let mut saw_end_of_stream = false; + while let Some(packet) = packets.read_packet().map_err(|_| "malformed_audio")? { + if saw_end_of_stream || packet.stream_serial() != serial || packet.data.is_empty() { + return Err("malformed_audio"); + } + let frames = decoder + .decode(&packet.data, 5_760, &mut output) + .map_err(|_| "malformed_audio")?; + decoded_frames = decoded_frames + .checked_add(u64::try_from(frames).map_err(|_| "duration_limit_exceeded")?) + .ok_or("duration_limit_exceeded")?; + audio_packets = audio_packets + .checked_add(1) + .ok_or("duration_limit_exceeded")?; + validate_ogg_decode_bound(decoded_frames, pre_skip)?; + if packet.last_in_page() { + let granule = packet.absgp_page(); + if granule < completed_page_granule + || (!packet.last_in_stream() && granule != decoded_frames) + { + return Err("malformed_audio"); + } + completed_page_granule = granule; + final_granule = Some(granule); + } + if packet.last_in_stream() { + saw_end_of_stream = true; + final_packet_frames = Some(u64::try_from(frames).map_err(|_| "malformed_audio")?); + } + } + if audio_packets == 0 || decoded_frames < u64::from(pre_skip) || !saw_end_of_stream { + return Err("malformed_audio"); + } + let final_granule = final_granule.ok_or("malformed_audio")?; + let final_packet_frames = final_packet_frames.ok_or("malformed_audio")?; + let presented_frames = final_granule + .checked_sub(u64::from(pre_skip)) + .ok_or("malformed_audio")?; + if final_granule > decoded_frames + || decoded_frames.saturating_sub(final_granule) > final_packet_frames + { + return Err("malformed_audio"); + } + validate_duration(presented_frames, metadata.sample_rate_hz)?; + Ok(metadata) +} + +fn flac_declared_frames(bytes: &[u8]) -> Result, &'static str> { + if !bytes.starts_with(b"fLaC") || bytes.get(4).is_none_or(|header| header & 0x7f != 0) { + return Err("malformed_audio"); + } + let encoded = bytes + .get(18..26) + .and_then(|value| <[u8; 8]>::try_from(value).ok()) + .ok_or("malformed_audio")?; + let frames = u64::from_be_bytes(encoded) & 0x0f_ff_ff_ff_ff; + Ok((frames != 0).then_some(frames)) +} + +fn mp3_declared_frames(bytes: &[u8]) -> Result, &'static str> { + let header = bytes.get(..4).ok_or("malformed_audio")?; + if !valid_mp3_frame_header(header) { + return Err("malformed_audio"); + } + let version = (header[1] >> 3) & 0x03; + let layer = (header[1] >> 1) & 0x03; + let samples_per_frame = match (version, layer) { + (_, 0x03) => 384_u64, + (_, 0x02) | (0x03, 0x01) => 1_152, + (_, 0x01) => 576, + _ => return Err("malformed_audio"), + }; + let has_crc = header[1] & 1 == 0; + let mono = (header[3] >> 6) & 0x03 == 0x03; + let side_information = match (version == 0x03, mono) { + (true, true) => 17_usize, + (true, false) => 32, + (false, true) => 9, + (false, false) => 17, + }; + let xing_offset = (if has_crc { 6_usize } else { 4 }) + .checked_add(side_information) + .ok_or("malformed_audio")?; + let xing_frames = bytes + .get(xing_offset..xing_offset.saturating_add(12)) + .filter(|value| value.starts_with(b"Xing") || value.starts_with(b"Info")) + .and_then(|value| { + let flags = u32::from_be_bytes(value.get(4..8)?.try_into().ok()?); + (flags & 1 != 0) + .then(|| value.get(8..12)?.try_into().ok().map(u32::from_be_bytes)) + .flatten() + }); + let vbri_frames = bytes + .get(36..54) + .filter(|value| value.starts_with(b"VBRI")) + .and_then(|value| value.get(14..18)?.try_into().ok()) + .map(u32::from_be_bytes); + xing_frames + .or(vbri_frames) + .map(u64::from) + .map(|frames| { + frames + .checked_sub(1) + .and_then(|audio_frames| audio_frames.checked_mul(samples_per_frame)) + .ok_or("malformed_audio") + }) + .transpose() +} + +fn mp3_audio_bytes(bytes: &[u8]) -> Result<&[u8], &'static str> { + if !bytes.starts_with(b"ID3") { + return Ok(bytes); + } + let audio_offset = id3_audio_offset(bytes).ok_or("malformed_audio")?; + bytes.get(audio_offset..).ok_or("malformed_audio") +} + +fn parse_opus_head(bytes: &[u8]) -> Result<(AudioMetadata, u16), &'static str> { + let prefix = bytes.get(..19).ok_or("malformed_audio")?; + if !prefix.starts_with(b"OpusHead") || prefix[8] != 1 { + return Err("malformed_audio"); + } + let channels = usize::from(prefix[9]); + if channels == 0 { + return Err("channel_limit_exceeded"); + } + if prefix[18] != 0 { + return Err("unsupported_opus_mapping"); + } + if bytes.len() != 19 { + return Err("malformed_audio"); + } + if channels > 2 { + return Err("channel_limit_exceeded"); + } + let metadata = AudioMetadata { + channels, + sample_rate_hz: 48_000, + }; + validate_shape(metadata)?; + Ok((metadata, u16::from_le_bytes([prefix[10], prefix[11]]))) +} + +fn valid_opus_tags(bytes: &[u8]) -> bool { + let Some(vendor_length) = bytes + .get(8..12) + .and_then(|value| <[u8; 4]>::try_from(value).ok()) + .map(u32::from_le_bytes) + .and_then(|value| usize::try_from(value).ok()) + else { + return false; + }; + if !bytes.starts_with(b"OpusTags") { + return false; + } + let Some(comment_count_offset) = 12_usize.checked_add(vendor_length) else { + return false; + }; + let Some(vendor) = bytes.get(12..comment_count_offset) else { + return false; + }; + if std::str::from_utf8(vendor).is_err() { + return false; + } + let Some(comment_count_bytes) = bytes + .get(comment_count_offset..comment_count_offset.saturating_add(4)) + .and_then(|value| <[u8; 4]>::try_from(value).ok()) + else { + return false; + }; + let Ok(comment_count) = usize::try_from(u32::from_le_bytes(comment_count_bytes)) else { + return false; + }; + let Some(mut offset) = comment_count_offset.checked_add(4) else { + return false; + }; + if comment_count > bytes.len().saturating_sub(offset) / 4 { + return false; + } + for _ in 0..comment_count { + let Some(data_offset) = offset.checked_add(4) else { + return false; + }; + let Some(length_bytes) = bytes + .get(offset..data_offset) + .and_then(|value| <[u8; 4]>::try_from(value).ok()) + else { + return false; + }; + let Some(length) = usize::try_from(u32::from_le_bytes(length_bytes)).ok() else { + return false; + }; + let Some(next) = data_offset.checked_add(length) else { + return false; + }; + let Some(comment) = bytes.get(data_offset..next) else { + return false; + }; + if !valid_opus_comment(comment) { + return false; + } + offset = next; + } + true +} + +fn valid_opus_comment(comment: &[u8]) -> bool { + let Some(separator) = comment.iter().position(|byte| *byte == b'=') else { + return false; + }; + separator > 0 + && comment[..separator] + .iter() + .all(|byte| matches!(*byte, 0x20..=0x3c | 0x3e..=0x7d)) + && std::str::from_utf8(comment).is_ok() +} + +fn validate_shape(metadata: AudioMetadata) -> Result<(), &'static str> { + if metadata.channels == 0 || metadata.channels > usize::from(MAX_AUDIO_CHANNELS) { + return Err("channel_limit_exceeded"); + } + if metadata.sample_rate_hz == 0 || metadata.sample_rate_hz > MAX_AUDIO_SAMPLE_RATE_HZ { + return Err("sample_rate_limit_exceeded"); + } + Ok(()) +} + +fn validate_duration(decoded_frames: u64, sample_rate_hz: u32) -> Result<(), &'static str> { + let maximum_frames = u64::from(sample_rate_hz) + .checked_mul(u64::from(MAX_AUDIO_CLIP_SECONDS)) + .ok_or("duration_limit_exceeded")?; + if decoded_frames > maximum_frames { + return Err("duration_limit_exceeded"); + } + Ok(()) +} + +fn validate_ogg_decode_bound(decoded_frames: u64, pre_skip: u16) -> Result<(), &'static str> { + let maximum_decoded_frames = 48_000_u64 + .checked_mul(u64::from(MAX_AUDIO_CLIP_SECONDS)) + .and_then(|frames| frames.checked_add(u64::from(pre_skip))) + .and_then(|frames| frames.checked_add(5_760)) + .ok_or("duration_limit_exceeded")?; + if decoded_frames > maximum_decoded_frames { + return Err("duration_limit_exceeded"); + } + Ok(()) +} + +fn metadata_json(metadata: AudioMetadata) -> Result { + serde_json::to_string(&serde_json::json!({ + "channels": metadata.channels, + "sample_rate_hz": metadata.sample_rate_hz, + })) + .map_err(|_| FileMediaProviderFailure::Failed) +} + +fn malformed(format: AdapterFormat, reason: &str) -> ProcessorValidationOutput { + ProcessorValidationOutput::Malformed { + media_type: String::from(format.media_type()), + reason_code: String::from(reason), + } +} + +fn validation_failure( + format: AdapterFormat, + evidence: ValidationEvidence, + reason: &str, +) -> ProcessorValidationOutput { + if evidence == ValidationEvidence::DeclaredCandidateStructurallyValidated { + ProcessorValidationOutput::NoMatch + } else { + malformed(format, reason) + } +} + +#[cfg(test)] +mod tests { + use super::{parse_opus_head, valid_opus_tags}; + + #[test] + fn opus_tags_rejects_a_declared_comment_without_its_length_or_data() { + let mut tags = b"OpusTags".to_vec(); + tags.extend_from_slice(&0_u32.to_le_bytes()); + tags.extend_from_slice(&1_u32.to_le_bytes()); + + assert!(!valid_opus_tags(&tags)); + } + + #[test] + fn opus_tags_rejects_an_invalid_utf8_vendor() { + let mut tags = b"OpusTags".to_vec(); + tags.extend_from_slice(&1_u32.to_le_bytes()); + tags.push(0xff); + tags.extend_from_slice(&0_u32.to_le_bytes()); + + assert!(!valid_opus_tags(&tags)); + } + + #[test] + fn opus_tags_rejects_an_invalid_utf8_comment() { + let mut tags = b"OpusTags".to_vec(); + tags.extend_from_slice(&0_u32.to_le_bytes()); + tags.extend_from_slice(&1_u32.to_le_bytes()); + tags.extend_from_slice(&1_u32.to_le_bytes()); + tags.push(0xff); + + assert!(!valid_opus_tags(&tags)); + } + + #[test] + fn opus_tags_rejects_a_comment_without_a_field_name_separator() { + let mut tags = b"OpusTags".to_vec(); + tags.extend_from_slice(&0_u32.to_le_bytes()); + tags.extend_from_slice(&1_u32.to_le_bytes()); + tags.extend_from_slice(&11_u32.to_le_bytes()); + tags.extend_from_slice(b"not-a-field"); + + assert!(!valid_opus_tags(&tags)); + } + + #[test] + fn opus_tags_accepts_trailing_padding() { + let mut tags = b"OpusTags".to_vec(); + tags.extend_from_slice(&0_u32.to_le_bytes()); + tags.extend_from_slice(&0_u32.to_le_bytes()); + tags.extend_from_slice(&[0; 16]); + + assert!(valid_opus_tags(&tags)); + } + + #[test] + fn opus_head_classifies_mapping_family_one_as_unsupported() { + let mut head = b"OpusHead".to_vec(); + head.push(1); + head.push(6); + head.extend_from_slice(&0_u16.to_le_bytes()); + head.extend_from_slice(&48_000_u32.to_le_bytes()); + head.extend_from_slice(&0_i16.to_le_bytes()); + head.push(1); + head.extend_from_slice(&[4, 2, 0, 1, 2, 3, 4, 5]); + + assert_eq!(parse_opus_head(&head), Err("unsupported_opus_mapping")); + } + + #[test] + fn opus_head_classifies_oversized_family_zero_as_malformed() { + let mut head = b"OpusHead".to_vec(); + head.push(1); + head.push(2); + head.extend_from_slice(&0_u16.to_le_bytes()); + head.extend_from_slice(&48_000_u32.to_le_bytes()); + head.extend_from_slice(&0_i16.to_le_bytes()); + head.push(0); + head.push(0); + + assert_eq!(parse_opus_head(&head), Err("malformed_audio")); + } +} diff --git a/crates/file-media-adapters-audio/src/bin/signalbox-file-media-audio-worker.rs b/crates/file-media-adapters-audio/src/bin/signalbox-file-media-audio-worker.rs new file mode 100644 index 0000000000..67238d21ad --- /dev/null +++ b/crates/file-media-adapters-audio/src/bin/signalbox-file-media-audio-worker.rs @@ -0,0 +1,11 @@ +use std::error::Error; + +use signalbox_file_media_adapters_audio::AudioFamilyProvider; +use signalbox_file_media_processor_runtime::{WorkerCatalog, serve_one}; + +#[tokio::main(flavor = "current_thread")] +async fn main() -> Result<(), Box> { + let catalog = WorkerCatalog::try_new(vec![Box::new(AudioFamilyProvider)])?; + serve_one(&catalog).await?; + Ok(()) +} diff --git a/crates/file-media-adapters-audio/src/lib.rs b/crates/file-media-adapters-audio/src/lib.rs new file mode 100644 index 0000000000..ea5075fdd3 --- /dev/null +++ b/crates/file-media-adapters-audio/src/lib.rs @@ -0,0 +1,517 @@ +//! Isolated adapters for WAV, MP3, FLAC, and Ogg Opus bytes. + +mod adapter; +mod source; + +use std::{error::Error, str::FromStr}; + +use signalbox_file_media_runtime::{ + CanonicalJsonObjectSchema, CanonicalMediaType, FileMediaProvider, FileMediaProviderDeclaration, + FileMediaProviderFailure, FileMediaProviderFuture, FileMediaProviderReadRequest, + FileMediaProviderValidationRequest, FileReaderName, FileReaderProviderName, FileReaderRevision, + ProbeDeclaration, ProbeDeclarationInput, ProcessorProbeOutput, ProcessorReadOutput, + ProcessorValidationOutput, ReadAccessPattern, ReadViewBounds, ReadViewDeclaration, + ReadViewName, ReaderDeclaration, ReaderDeclarationInput, ReaderIdentity, ReasonCode, + StreamingTextFallback, ValidationDeclaration, VerifiedBlobSource, +}; + +const PROVIDER_NAME: &str = "signalbox_audio"; +const READER_REVISION: &str = "v1"; +const METADATA_VIEW_NAME: &str = "metadata"; +/// Hard safety ceiling covering the prefix and two possible exact MP3 reads. +const AUDIO_PROBE_CUMULATIVE_BYTES: u64 = 78; + +/// Hard safety ceiling bounding whole-source worker memory while admitting ordinary audio. +pub const MAX_AUDIO_SOURCE_BYTES: u64 = 64 * 1_024 * 1_024; +/// Exact-range budget for one whole-source audio read. Validation and the metadata view both +/// stream the complete source in `MAX_PROCESSOR_FRAME_BYTES / 2` chunks, so the declared envelope +/// must cover `MAX_AUDIO_SOURCE_BYTES` at that granularity. +pub(crate) const AUDIO_WHOLE_SOURCE_RANGES: u32 = 512; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum AdapterFormat { + Wav, + Mp3, + Flac, + OggOpus, +} + +impl AdapterFormat { + const ALL: [Self; 4] = [Self::Wav, Self::Mp3, Self::Flac, Self::OggOpus]; + + const fn reader_name(self) -> &'static str { + match self { + Self::Wav => "wav", + Self::Mp3 => "mp3", + Self::Flac => "flac", + Self::OggOpus => "ogg_opus", + } + } + + const fn media_type(self) -> &'static str { + match self { + Self::Wav => "audio/wav", + Self::Mp3 => "audio/mpeg", + Self::Flac => "audio/flac", + Self::OggOpus => "audio/ogg", + } + } + + fn matches_signature(self, prefix: &[u8]) -> bool { + match self { + Self::Wav => { + prefix.starts_with(b"RIFF") && prefix.get(8..12) == Some(b"WAVE".as_slice()) + } + Self::Mp3 => mp3_signature(prefix), + Self::Flac => prefix.starts_with(b"fLaC"), + Self::OggOpus => ogg_opus_signature(prefix), + } + } +} + +fn ogg_opus_signature(prefix: &[u8]) -> bool { + let Some(header) = prefix.get(..27) else { + return false; + }; + if !header.starts_with(b"OggS") + || header[4] != 0 + || header[5] & 0x02 == 0 + || header[5] & 0x01 != 0 + || header[6..14] != [0; 8] + { + return false; + } + let segment_count = usize::from(header[26]); + let Some(packet_offset) = 27_usize.checked_add(segment_count) else { + return false; + }; + let Some(lacing) = prefix.get(27..packet_offset) else { + return false; + }; + let mut first_packet_length = 0_usize; + let mut first_packet_complete = false; + for segment_length in lacing { + let Some(length) = first_packet_length.checked_add(usize::from(*segment_length)) else { + return false; + }; + first_packet_length = length; + if *segment_length < 255 { + first_packet_complete = true; + break; + } + } + if !first_packet_complete || first_packet_length < 19 { + return false; + } + prefix + .get(packet_offset..packet_offset.saturating_add(19)) + .is_some_and(|common| common.starts_with(b"OpusHead") && common[8] == 1 && common[9] != 0) +} + +fn mp3_signature(prefix: &[u8]) -> bool { + let audio = if prefix.starts_with(b"ID3") { + let Some(audio_offset) = id3_audio_offset(prefix) else { + return false; + }; + let Some(audio) = prefix.get(audio_offset..) else { + return false; + }; + audio + } else { + prefix + }; + audio.get(..4).is_some_and(valid_mp3_frame_header) +} + +pub(crate) fn id3_tag_layout(bytes: &[u8]) -> Option<(usize, bool)> { + let header = bytes.get(..10)?; + let major = header[3]; + let revision = header[4]; + let flags = header[5]; + let legal_flags = match major { + 2 => 0xc0, + 3 => 0xe0, + 4 => 0xf0, + _ => return None, + }; + if revision == 0xff + || flags & !legal_flags != 0 + || header[6..10].iter().any(|byte| byte & 0x80 != 0) + { + return None; + } + let tag_length = header[6..10].iter().try_fold(0_usize, |length, byte| { + length.checked_mul(128)?.checked_add(usize::from(*byte)) + })?; + if flags & 0x40 != 0 && !valid_id3_extended_header(major, tag_length, bytes) { + return None; + } + Some(( + 10_usize.checked_add(tag_length)?, + major == 4 && flags & 0x10 != 0, + )) +} + +fn valid_id3_extended_header(major: u8, tag_length: usize, bytes: &[u8]) -> bool { + let Some(size_bytes) = bytes + .get(10..14) + .and_then(|value| <[u8; 4]>::try_from(value).ok()) + else { + return false; + }; + match major { + 3 => valid_id3v23_extended_header(tag_length, size_bytes, bytes), + 4 => { + if size_bytes.iter().any(|byte| byte & 0x80 != 0) { + return false; + } + let Some(size) = size_bytes.iter().try_fold(0_usize, |length, byte| { + length.checked_mul(128)?.checked_add(usize::from(*byte)) + }) else { + return false; + }; + valid_id3v24_extended_header(tag_length, size, bytes) + } + _ => false, + } +} + +fn valid_id3v23_extended_header(tag_length: usize, size_bytes: [u8; 4], bytes: &[u8]) -> bool { + let Ok(size) = usize::try_from(u32::from_be_bytes(size_bytes)) else { + return false; + }; + let Some(total_size) = size.checked_add(4) else { + return false; + }; + let Some(body) = bytes.get(14..14_usize.saturating_add(size)) else { + return false; + }; + let Some(flags) = body + .get(..2) + .and_then(|value| <[u8; 2]>::try_from(value).ok()) + .map(u16::from_be_bytes) + else { + return false; + }; + let Some(padding_size) = body + .get(2..6) + .and_then(|value| <[u8; 4]>::try_from(value).ok()) + .map(u32::from_be_bytes) + .and_then(|value| usize::try_from(value).ok()) + else { + return false; + }; + // CRC verification is not implemented, so reject CRC-bearing tags rather + // than silently trusting an advertised checksum. + if flags & 0x8000 != 0 { + return false; + } + let expected_size = 6; + let Some(content_after_extended_header) = tag_length.checked_sub(total_size) else { + return false; + }; + if flags & !0x8000 != 0 || size != expected_size || padding_size > content_after_extended_header + { + return false; + } + if padding_size == 0 { + return true; + } + let Some(tag_end) = 10_usize.checked_add(tag_length) else { + return false; + }; + let Some(padding_start) = tag_end.checked_sub(padding_size) else { + return false; + }; + bytes + .get(padding_start..tag_end) + .is_none_or(|padding| padding.iter().all(|byte| *byte == 0)) +} + +fn valid_id3v24_extended_header(tag_length: usize, size: usize, bytes: &[u8]) -> bool { + if size < 6 || size > tag_length { + return false; + } + let Some(body) = bytes.get(14..10_usize.saturating_add(size)) else { + return false; + }; + if body.first() != Some(&1) { + return false; + } + let Some(flags) = body.get(1).copied() else { + return false; + }; + if flags & !0x70 != 0 { + return false; + } + let mut fields = &body[2..]; + for (flag, expected_length) in [(0x40, 0_usize), (0x20, 5), (0x10, 1)] { + if flags & flag == 0 { + continue; + } + if fields.first().copied() != u8::try_from(expected_length).ok() { + return false; + } + let Some(remaining) = fields.get(1_usize.saturating_add(expected_length)..) else { + return false; + }; + fields = remaining; + } + fields.is_empty() +} + +pub(crate) struct Id3Footer<'a> { + pub(crate) header: &'a [u8], + pub(crate) footer: &'a [u8], +} + +pub(crate) fn valid_id3_footer(input: Id3Footer<'_>) -> bool { + input.header.len() == 10 + && input.footer.len() == 10 + && input.footer.starts_with(b"3DI") + && input.footer[3..10] == input.header[3..10] +} + +fn id3_audio_offset(bytes: &[u8]) -> Option { + let (tag_end, has_footer) = id3_tag_layout(bytes)?; + if !has_footer { + return Some(tag_end); + } + let footer_end = tag_end.checked_add(10)?; + if !valid_id3_footer(Id3Footer { + header: bytes.get(..10)?, + footer: bytes.get(tag_end..footer_end)?, + }) { + return None; + } + Some(footer_end) +} + +fn valid_mp3_frame_header(bytes: &[u8]) -> bool { + let version = (bytes[1] >> 3) & 0x03; + let layer = (bytes[1] >> 1) & 0x03; + let bitrate = (bytes[2] >> 4) & 0x0f; + let sample_rate = (bytes[2] >> 2) & 0x03; + bytes[0] == 0xff + && bytes[1] & 0xe0 == 0xe0 + && version != 0x01 + && layer != 0x00 + && bitrate != 0x00 + && bitrate != 0x0f + && sample_rate != 0x03 +} + +#[cfg(test)] +mod tests { + use ogg::{PacketWriteEndInfo, PacketWriter}; + + use super::{AdapterFormat, id3_tag_layout}; + + #[test] + fn mp3_probe_rejects_aac_adts_header() { + let aac_adts_header = [0xff, 0xf1, 0x50, 0x80]; + + assert!(!AdapterFormat::Mp3.matches_signature(&aac_adts_header)); + } + + #[test] + fn mp3_probe_rejects_aac_adts_after_id3_metadata() { + let mut id3_prefixed_aac = b"ID3\x04\x00\x00\x00\x00\x00\x00".to_vec(); + id3_prefixed_aac.extend_from_slice(&[0xff, 0xf1, 0x50, 0x80]); + + assert!(!AdapterFormat::Mp3.matches_signature(&id3_prefixed_aac)); + } + + #[test] + fn id3v23_extended_header_rejects_padding_larger_than_the_tag_body() { + let bytes = [ + b'I', b'D', b'3', 3, 0, 0x40, 0, 0, 0, 10, 0, 0, 0, 6, 0, 0, 0, 0, 0, 1, + ]; + + assert_eq!(id3_tag_layout(&bytes), None); + } + + #[test] + fn id3v23_extended_header_rejects_nonzero_declared_padding() { + let bytes = [ + b'I', b'D', b'3', 3, 0, 0x40, 0, 0, 0, 11, 0, 0, 0, 6, 0, 0, 0, 0, 0, 1, 1, + ]; + + assert_eq!(id3_tag_layout(&bytes), None); + } + + #[test] + fn id3v23_extended_header_rejects_an_unverified_crc() { + let bytes = [ + b'I', b'D', b'3', 3, 0, 0x40, 0, 0, 0, 14, 0, 0, 0, 10, 0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, + ]; + + assert_eq!(id3_tag_layout(&bytes), None); + } + + #[test] + fn ogg_opus_probe_rejects_magic_in_a_later_packet() { + let mut writer = PacketWriter::new(Vec::new()); + writer + .write_packet( + b"not-an-opus-identification-packet".to_vec(), + 7, + PacketWriteEndInfo::EndPage, + 0, + ) + .expect("first Ogg packet should encode"); + writer + .write_packet( + b"codec-version=OpusHead".to_vec(), + 7, + PacketWriteEndInfo::EndStream, + 0, + ) + .expect("second Ogg packet should encode"); + let bytes = writer.into_inner(); + + assert!(!AdapterFormat::OggOpus.matches_signature(&bytes)); + } + + #[test] + fn ogg_opus_probe_does_not_join_two_packets_into_an_identification_header() { + let mut writer = PacketWriter::new(Vec::new()); + writer + .write_packet(b"OpusHead".to_vec(), 7, PacketWriteEndInfo::NormalPacket, 0) + .expect("first Ogg packet should encode"); + writer + .write_packet( + vec![1, 1, 0, 0, 0x80, 0xbb, 0, 0, 0, 0, 0], + 7, + PacketWriteEndInfo::EndStream, + 0, + ) + .expect("second Ogg packet should encode"); + let bytes = writer.into_inner(); + + assert!(!AdapterFormat::OggOpus.matches_signature(&bytes)); + } +} + +/// Compiled provider for the four version-one audio readers. +#[derive(Clone, Copy, Debug, Default)] +pub struct AudioFamilyProvider; + +impl FileMediaProvider for AudioFamilyProvider { + fn declaration(&self) -> FileMediaProviderDeclaration { + match audio_family_declaration() { + Ok(declaration) => declaration, + Err(_) => std::process::abort(), + } + } + + fn probe<'a>( + &'a self, + reader: &'a ReaderIdentity, + source: &'a dyn VerifiedBlobSource, + cancellation: &'a dyn signalbox_file_media_runtime::CancellationSignal, + ) -> FileMediaProviderFuture<'a, ProcessorProbeOutput> { + Box::pin(async move { + let format = format_for_reader(reader)?; + adapter::probe(format, source, cancellation).await + }) + } + + fn inspect<'a>( + &'a self, + reader: &'a ReaderIdentity, + request: FileMediaProviderValidationRequest, + source: &'a dyn VerifiedBlobSource, + cancellation: &'a dyn signalbox_file_media_runtime::CancellationSignal, + ) -> FileMediaProviderFuture<'a, ProcessorValidationOutput> { + Box::pin(async move { + let format = format_for_reader(reader)?; + adapter::inspect(format, request, source, cancellation).await + }) + } + + fn read<'a>( + &'a self, + reader: &'a ReaderIdentity, + request: FileMediaProviderReadRequest, + source: &'a dyn VerifiedBlobSource, + cancellation: &'a dyn signalbox_file_media_runtime::CancellationSignal, + ) -> FileMediaProviderFuture<'a, ProcessorReadOutput> { + Box::pin(async move { + let format = format_for_reader(reader)?; + adapter::read(format, request, source, cancellation).await + }) + } +} + +/// Builds the exact declaration registered by the audio-family worker. +pub fn audio_family_declaration() +-> Result> { + let provider = FileReaderProviderName::try_new(PROVIDER_NAME)?; + let readers = AdapterFormat::ALL + .into_iter() + .map(|format| reader(&provider, format)) + .collect::, _>>()?; + Ok(FileMediaProviderDeclaration::try_new(provider, readers)?) +} + +fn reader( + provider: &FileReaderProviderName, + format: AdapterFormat, +) -> Result> { + let mut reasons = vec![ + ReasonCode::try_new("malformed_audio")?, + ReasonCode::try_new("source_too_large")?, + ReasonCode::try_new("channel_limit_exceeded")?, + ReasonCode::try_new("sample_rate_limit_exceeded")?, + ReasonCode::try_new("duration_limit_exceeded")?, + ]; + if format == AdapterFormat::OggOpus { + reasons.push(ReasonCode::try_new("unsupported_opus_mapping")?); + } + Ok(ReaderDeclaration::try_new(ReaderDeclarationInput { + provider: provider.clone(), + reader: FileReaderName::try_new(format.reader_name())?, + revision: FileReaderRevision::try_new(READER_REVISION)?, + media_types: vec![CanonicalMediaType::from_str(format.media_type())?], + probe: ProbeDeclaration::new(ProbeDeclarationInput { + prefix_bytes: 64, + suffix_bytes: 0, + range_count: 2, + cumulative_bytes: AUDIO_PROBE_CUMULATIVE_BYTES, + }), + validation: ValidationDeclaration::new(MAX_AUDIO_SOURCE_BYTES, AUDIO_WHOLE_SOURCE_RANGES), + views: vec![metadata_view()?], + reason_codes: reasons, + streaming_text_fallback: StreamingTextFallback::Disabled, + })?) +} + +fn metadata_view() -> Result> { + Ok(ReadViewDeclaration::try_new( + ReadViewName::try_new(METADATA_VIEW_NAME)?, + String::from("Decodes the audio and returns its channel count and sample rate."), + CanonicalJsonObjectSchema::try_new(r#"{"additionalProperties":false,"type":"object"}"#)?, + ReadAccessPattern::Streaming { + maximum_ranges: AUDIO_WHOLE_SOURCE_RANGES, + }, + ReadViewBounds::Structured { + source_bytes: MAX_AUDIO_SOURCE_BYTES, + output_bytes: 256, + depth: 2, + nodes: 8, + string_bytes: 64, + }, + )?) +} + +fn format_for_reader(reader: &ReaderIdentity) -> Result { + AdapterFormat::ALL + .into_iter() + .find(|format| format.reader_name() == reader.reader().as_str()) + .ok_or(FileMediaProviderFailure::Failed) +} + +fn options_are_empty(options: &serde_json::Value) -> bool { + options.as_object().is_some_and(serde_json::Map::is_empty) +} diff --git a/crates/file-media-adapters-audio/src/source.rs b/crates/file-media-adapters-audio/src/source.rs new file mode 100644 index 0000000000..35ba7cd22c --- /dev/null +++ b/crates/file-media-adapters-audio/src/source.rs @@ -0,0 +1,75 @@ +use std::num::NonZeroU64; + +use signalbox_file_media_runtime::{ + CancellationSignal, FileMediaProviderFailure, MAX_PROCESSOR_FRAME_BYTES, VerifiedBlobSource, +}; + +use crate::MAX_AUDIO_SOURCE_BYTES; + +pub(crate) async fn read_complete( + source: &dyn VerifiedBlobSource, + cancellation: &dyn CancellationSignal, + maximum_source_bytes: u64, + maximum_ranges: u32, +) -> Result>, FileMediaProviderFailure> { + if cancellation.is_cancelled() { + return Err(FileMediaProviderFailure::Failed); + } + let effective_source_bytes = maximum_source_bytes.min(MAX_AUDIO_SOURCE_BYTES); + if source.byte_length().get() > effective_source_bytes { + return Ok(None); + } + let source_length = source.byte_length().get(); + let capacity = usize::try_from(source_length).map_err(|_| FileMediaProviderFailure::Failed)?; + let maximum_chunk = u64::try_from(MAX_PROCESSOR_FRAME_BYTES / 2) + .map_err(|_| FileMediaProviderFailure::Failed)?; + let required_ranges = source_length + .checked_add(maximum_chunk - 1) + .ok_or(FileMediaProviderFailure::Failed)? + / maximum_chunk; + if required_ranges > u64::from(maximum_ranges) { + return Ok(None); + } + let mut bytes = Vec::with_capacity(capacity); + let mut offset = 0_u64; + while offset < source_length { + if cancellation.is_cancelled() { + return Err(FileMediaProviderFailure::Failed); + } + let length = NonZeroU64::new((source_length - offset).min(maximum_chunk)) + .ok_or(FileMediaProviderFailure::Failed)?; + let chunk = source + .read_range(offset, length) + .await + .map_err(|_| FileMediaProviderFailure::Failed)?; + if chunk.len() + != usize::try_from(length.get()).map_err(|_| FileMediaProviderFailure::Failed)? + { + return Err(FileMediaProviderFailure::Failed); + } + bytes.extend_from_slice(&chunk); + offset = offset + .checked_add(length.get()) + .ok_or(FileMediaProviderFailure::Failed)?; + } + if cancellation.is_cancelled() { + return Err(FileMediaProviderFailure::Failed); + } + Ok(Some(bytes)) +} + +pub(crate) async fn read_probe_prefix( + source: &dyn VerifiedBlobSource, + cancellation: &dyn CancellationSignal, +) -> Result, FileMediaProviderFailure> { + if cancellation.is_cancelled() { + return Err(FileMediaProviderFailure::Failed); + } + let length = source + .byte_length() + .min(std::num::NonZeroU64::new(64).ok_or(FileMediaProviderFailure::Failed)?); + source + .read_range(0, length) + .await + .map_err(|_| FileMediaProviderFailure::Failed) +} diff --git a/crates/file-media-adapters-audio/tests/adapters.rs b/crates/file-media-adapters-audio/tests/adapters.rs new file mode 100644 index 0000000000..8b48882af1 --- /dev/null +++ b/crates/file-media-adapters-audio/tests/adapters.rs @@ -0,0 +1,542 @@ +mod fixtures; +mod support; + +use std::error::Error; + +use fixtures::FixtureFormat; +use support::{DirectProcessor, MemorySource}; + +#[tokio::test] +async fn wav_detects_validates_and_reads_metadata() -> Result<(), Box> { + let format = FixtureFormat::Wav; + let fixture = fixtures::valid_fixture(format)?; + let source = MemorySource::new(fixture.bytes().to_vec()); + + let inspection = support::inspect(&source, fixture.media_type()).await?; + support::assert_validated_media(inspection, fixture.media_type()); + let result = support::read(&source, fixture.media_type(), &DirectProcessor::provider()).await?; + support::assert_structured(result, fixture.expected_metadata()); + Ok(()) +} + +#[tokio::test] +async fn mp3_detects_validates_and_reads_metadata() -> Result<(), Box> { + let format = FixtureFormat::Mp3; + let fixture = fixtures::valid_fixture(format)?; + let source = MemorySource::new(fixture.bytes().to_vec()); + + let inspection = support::inspect(&source, fixture.media_type()).await?; + support::assert_validated_media(inspection, fixture.media_type()); + let result = support::read(&source, fixture.media_type(), &DirectProcessor::provider()).await?; + support::assert_structured(result, fixture.expected_metadata()); + Ok(()) +} + +#[tokio::test] +async fn flac_detects_validates_and_reads_metadata() -> Result<(), Box> { + let format = FixtureFormat::Flac; + let fixture = fixtures::valid_fixture(format)?; + let source = MemorySource::new(fixture.bytes().to_vec()); + + let inspection = support::inspect(&source, fixture.media_type()).await?; + support::assert_validated_media(inspection, fixture.media_type()); + let result = support::read(&source, fixture.media_type(), &DirectProcessor::provider()).await?; + support::assert_structured(result, fixture.expected_metadata()); + Ok(()) +} + +#[tokio::test] +async fn ogg_opus_detects_validates_and_reads_metadata() -> Result<(), Box> { + let format = FixtureFormat::OggOpus; + let fixture = fixtures::valid_fixture(format)?; + let source = MemorySource::new(fixture.bytes().to_vec()); + + let inspection = support::inspect(&source, fixture.media_type()).await?; + support::assert_validated_media(inspection, fixture.media_type()); + let result = support::read(&source, fixture.media_type(), &DirectProcessor::provider()).await?; + support::assert_structured(result, fixture.expected_metadata()); + Ok(()) +} + +#[cfg(target_os = "linux")] +#[tokio::test] +async fn isolated_worker_validates_audio_larger_than_one_broker_range() -> Result<(), Box> +{ + let source = MemorySource::new(fixtures::wav_larger_than_one_broker_range()?); + + let inspection = support::inspect_sandboxed(&source, "audio/wav").await?; + support::assert_validated_media(inspection, "audio/wav"); + Ok(()) +} + +#[tokio::test] +async fn failed_declared_audio_candidate_is_unknown() -> Result<(), Box> { + let source = MemorySource::new(b"not audio bytes".to_vec()); + + let inspection = support::inspect(&source, "audio/wav").await?; + assert!(matches!( + inspection, + signalbox_file_media_runtime::FileInspection::Unknown { .. } + )); + Ok(()) +} + +#[tokio::test] +async fn wav_truncation_is_malformed() -> Result<(), Box> { + assert_reason( + FixtureFormat::Wav, + fixtures::truncated(FixtureFormat::Wav)?, + "malformed_audio", + ) + .await +} + +#[tokio::test] +async fn wav_malformed_bytes_are_rejected() -> Result<(), Box> { + assert_reason( + FixtureFormat::Wav, + fixtures::malformed(FixtureFormat::Wav), + "malformed_audio", + ) + .await +} + +#[tokio::test] +async fn wav_oversized_source_is_rejected() -> Result<(), Box> { + assert_reason( + FixtureFormat::Wav, + fixtures::oversized(FixtureFormat::Wav)?, + "source_too_large", + ) + .await +} + +#[tokio::test] +async fn wav_duration_over_limit_is_rejected() -> Result<(), Box> { + assert_reason( + FixtureFormat::Wav, + fixtures::duration_bomb(FixtureFormat::Wav)?, + "duration_limit_exceeded", + ) + .await +} + +#[tokio::test] +async fn wav_without_frames_is_valid() -> Result<(), Box> { + let source = MemorySource::new(fixtures::wav_without_frames()?); + + let inspection = support::inspect(&source, "audio/wav").await?; + support::assert_validated_media(inspection, "audio/wav"); + Ok(()) +} + +#[tokio::test] +async fn mp3_truncation_is_malformed() -> Result<(), Box> { + assert_reason( + FixtureFormat::Mp3, + fixtures::truncated(FixtureFormat::Mp3)?, + "malformed_audio", + ) + .await +} + +#[tokio::test] +async fn mp3_malformed_bytes_are_rejected() -> Result<(), Box> { + let format = FixtureFormat::Mp3; + let source = MemorySource::new(fixtures::malformed(format)); + + let inspection = support::inspect(&source, format.media_type()).await?; + assert!(matches!( + inspection, + signalbox_file_media_runtime::FileInspection::Unknown { .. } + )); + Ok(()) +} + +#[tokio::test] +async fn mp3_oversized_source_is_rejected() -> Result<(), Box> { + assert_reason( + FixtureFormat::Mp3, + fixtures::oversized(FixtureFormat::Mp3)?, + "source_too_large", + ) + .await +} + +#[tokio::test] +async fn mp3_duration_over_limit_is_rejected() -> Result<(), Box> { + assert_reason( + FixtureFormat::Mp3, + fixtures::duration_bomb(FixtureFormat::Mp3)?, + "duration_limit_exceeded", + ) + .await +} + +#[tokio::test] +async fn mp3_probe_reads_the_frame_after_a_long_id3_tag() -> Result<(), Box> { + let format = FixtureFormat::Mp3; + let source = MemorySource::new(fixtures::mp3_with_long_id3_tag()?); + + let inspection = support::inspect(&source, format.media_type()).await?; + support::assert_validated_media(inspection, format.media_type()); + Ok(()) +} + +#[tokio::test] +async fn mp3_probe_rejects_an_invalid_id3_version() -> Result<(), Box> { + let format = FixtureFormat::Mp3; + let source = MemorySource::new(fixtures::mp3_with_id3_header(fixtures::Id3HeaderFixture { + major: 5, + flags: 0, + })?); + + let inspection = support::inspect(&source, format.media_type()).await?; + assert!(matches!( + inspection, + signalbox_file_media_runtime::FileInspection::Unknown { .. } + )); + Ok(()) +} + +#[tokio::test] +async fn mp3_probe_rejects_invalid_id3_flags() -> Result<(), Box> { + let format = FixtureFormat::Mp3; + let source = MemorySource::new(fixtures::mp3_with_id3_header(fixtures::Id3HeaderFixture { + major: 4, + flags: 0x01, + })?); + + let inspection = support::inspect(&source, format.media_type()).await?; + assert!(matches!( + inspection, + signalbox_file_media_runtime::FileInspection::Unknown { .. } + )); + Ok(()) +} + +#[tokio::test] +async fn mp3_probe_rejects_an_invalid_id3v24_footer() -> Result<(), Box> { + let source = MemorySource::new(fixtures::mp3_with_invalid_id3v24_footer()?); + + let inspection = support::inspect(&source, "audio/mpeg").await?; + assert!(matches!( + inspection, + signalbox_file_media_runtime::FileInspection::Unknown { .. } + )); + Ok(()) +} + +#[tokio::test] +async fn mp3_probe_reads_an_id3v24_footer_past_the_probe_prefix() -> Result<(), Box> { + let (bytes, _) = fixtures::mp3_with_id3v24_footer_past_the_probe_prefix()?; + let source = MemorySource::new(bytes); + + let inspection = support::inspect(&source, "audio/mpeg").await?; + support::assert_validated_media(inspection, "audio/mpeg"); + Ok(()) +} + +#[tokio::test] +async fn mp3_probe_propagates_an_unreadable_id3v24_footer() -> Result<(), Box> { + let (bytes, footer_offset) = fixtures::mp3_with_id3v24_footer_past_the_probe_prefix()?; + let source = MemorySource::unavailable_at(bytes, footer_offset); + + assert_eq!( + support::inspect_failure(&source, "audio/mpeg").await?, + Some(signalbox_file_media_runtime::FileMediaFailure::ProcessorFailed) + ); + Ok(()) +} + +#[tokio::test] +async fn mp3_rejects_an_empty_advertised_id3v24_extended_header() -> Result<(), Box> { + let source = MemorySource::new(fixtures::mp3_with_empty_id3v24_extended_header()?); + + let inspection = support::inspect(&source, "audio/mpeg").await?; + assert!(matches!( + inspection, + signalbox_file_media_runtime::FileInspection::Unknown { .. } + )); + Ok(()) +} + +#[tokio::test] +async fn mp3_rejects_fewer_frames_than_its_xing_header_advertises() -> Result<(), Box> { + assert_reason( + FixtureFormat::Mp3, + fixtures::mp3_with_excess_xing_frame_count()?, + "malformed_audio", + ) + .await +} + +#[tokio::test] +async fn mp3_rejects_audio_frames_against_a_xing_header_declaring_zero() +-> Result<(), Box> { + // A Xing header declaring a total of one frame (the header frame + // itself, zero audio frames) previously fell through the same + // zero-skip meant for FLAC's "unknown total samples" STREAMINFO + // convention, silently validating any decoded audio despite + // contradicting the declared frame count. + assert_reason( + FixtureFormat::Mp3, + fixtures::mp3_with_xing_frame_count_of_one()?, + "malformed_audio", + ) + .await +} + +#[tokio::test] +async fn flac_truncation_is_malformed() -> Result<(), Box> { + assert_reason( + FixtureFormat::Flac, + fixtures::truncated(FixtureFormat::Flac)?, + "malformed_audio", + ) + .await +} + +#[tokio::test] +async fn flac_malformed_bytes_are_rejected() -> Result<(), Box> { + assert_reason( + FixtureFormat::Flac, + fixtures::malformed(FixtureFormat::Flac), + "malformed_audio", + ) + .await +} + +#[tokio::test] +async fn flac_oversized_source_is_rejected() -> Result<(), Box> { + assert_reason( + FixtureFormat::Flac, + fixtures::oversized(FixtureFormat::Flac)?, + "source_too_large", + ) + .await +} + +#[tokio::test] +async fn flac_duration_over_limit_is_rejected() -> Result<(), Box> { + assert_reason( + FixtureFormat::Flac, + fixtures::duration_bomb(FixtureFormat::Flac)?, + "duration_limit_exceeded", + ) + .await +} + +#[tokio::test] +async fn flac_mismatched_streaminfo_md5_is_rejected() -> Result<(), Box> { + assert_reason( + FixtureFormat::Flac, + fixtures::flac_with_mismatched_md5()?, + "malformed_audio", + ) + .await +} + +#[tokio::test] +async fn flac_truncated_between_complete_frames_is_rejected() -> Result<(), Box> { + assert_reason( + FixtureFormat::Flac, + fixtures::flac_truncated_between_complete_frames()?, + "malformed_audio", + ) + .await +} + +#[tokio::test] +async fn flac_decoded_shape_must_match_streaminfo() -> Result<(), Box> { + assert_reason( + FixtureFormat::Flac, + fixtures::flac_with_mismatched_streaminfo_channels()?, + "malformed_audio", + ) + .await +} + +#[tokio::test] +async fn ogg_opus_truncation_is_malformed() -> Result<(), Box> { + assert_reason( + FixtureFormat::OggOpus, + fixtures::truncated(FixtureFormat::OggOpus)?, + "malformed_audio", + ) + .await +} + +#[tokio::test] +async fn ogg_opus_malformed_bytes_are_rejected() -> Result<(), Box> { + let format = FixtureFormat::OggOpus; + let source = MemorySource::new(fixtures::malformed(format)); + + let inspection = support::inspect(&source, format.media_type()).await?; + assert!(matches!( + inspection, + signalbox_file_media_runtime::FileInspection::Unknown { .. } + )); + Ok(()) +} + +#[tokio::test] +async fn ogg_opus_oversized_source_is_rejected() -> Result<(), Box> { + assert_reason( + FixtureFormat::OggOpus, + fixtures::oversized(FixtureFormat::OggOpus)?, + "source_too_large", + ) + .await +} + +#[tokio::test] +async fn ogg_opus_duration_over_limit_is_rejected() -> Result<(), Box> { + assert_reason( + FixtureFormat::OggOpus, + fixtures::duration_bomb(FixtureFormat::OggOpus)?, + "duration_limit_exceeded", + ) + .await +} + +#[tokio::test] +async fn ogg_opus_requires_an_end_of_stream_page() -> Result<(), Box> { + let format = FixtureFormat::OggOpus; + let source = MemorySource::new(fixtures::ogg_opus_without_end_of_stream()?); + + let inspection = support::inspect(&source, format.media_type()).await?; + support::assert_malformed_reason(inspection, "malformed_audio"); + Ok(()) +} + +#[tokio::test] +async fn ogg_opus_rejects_end_of_stream_on_tags() -> Result<(), Box> { + assert_reason( + FixtureFormat::OggOpus, + fixtures::ogg_opus_with_tags_end_of_stream()?, + "malformed_audio", + ) + .await +} + +#[tokio::test] +async fn ogg_opus_rejects_a_nonzero_tags_page_granule() -> Result<(), Box> { + assert_reason( + FixtureFormat::OggOpus, + fixtures::ogg_opus_with_nonzero_tags_granule()?, + "malformed_audio", + ) + .await +} + +#[tokio::test] +async fn ogg_opus_duration_uses_presented_samples_after_trimming() -> Result<(), Box> { + let format = FixtureFormat::OggOpus; + let source = MemorySource::new(fixtures::ogg_opus_trimmed_to_duration_limit()?); + + let inspection = support::inspect(&source, format.media_type()).await?; + support::assert_validated_media(inspection, "audio/ogg"); + Ok(()) +} + +#[tokio::test] +async fn ogg_opus_requires_an_isolated_identification_header_page() -> Result<(), Box> { + let format = FixtureFormat::OggOpus; + let source = MemorySource::new(fixtures::ogg_opus_with_shared_identification_page()?); + + let inspection = support::inspect(&source, format.media_type()).await?; + assert!(matches!( + inspection, + signalbox_file_media_runtime::FileInspection::Unknown { .. } + )); + Ok(()) +} + +#[tokio::test] +async fn ogg_opus_rejects_regressing_page_granules() -> Result<(), Box> { + let format = FixtureFormat::OggOpus; + let source = MemorySource::new(fixtures::ogg_opus_with_regressing_granule()?); + + let inspection = support::inspect(&source, format.media_type()).await?; + support::assert_malformed_reason(inspection, "malformed_audio"); + Ok(()) +} + +#[tokio::test] +async fn ogg_opus_rejects_an_inaccurate_intermediate_granule() -> Result<(), Box> { + let format = FixtureFormat::OggOpus; + let source = MemorySource::new(fixtures::ogg_opus_with_inaccurate_intermediate_granule()?); + + let inspection = support::inspect(&source, format.media_type()).await?; + support::assert_malformed_reason(inspection, "malformed_audio"); + Ok(()) +} + +#[tokio::test] +async fn ogg_opus_rejects_end_of_stream_on_identification_header() -> Result<(), Box> { + assert_reason( + FixtureFormat::OggOpus, + fixtures::ogg_opus_with_head_end_of_stream()?, + "malformed_audio", + ) + .await +} + +#[tokio::test] +async fn ogg_opus_rejects_eos_trimming_beyond_the_final_packet() -> Result<(), Box> { + assert_reason( + FixtureFormat::OggOpus, + fixtures::ogg_opus_with_excessive_end_trim()?, + "malformed_audio", + ) + .await +} + +#[tokio::test] +async fn registry_sanitizer_keeps_injection_shaped_metadata_as_data() -> Result<(), Box> +{ + let format = FixtureFormat::Wav; + let source = MemorySource::new(fixtures::valid(format)?); + let expected = serde_json::json!({ + "path":"../../etc/passwd", + "text":"" + }); + let decoder_output = serde_json::to_string(&expected)?; + + let result = support::read( + &source, + format.media_type(), + &DirectProcessor::injecting(decoder_output), + ) + .await?; + support::assert_structured(result, &expected); + Ok(()) +} + +#[tokio::test] +async fn registry_sanitizer_rejects_nul_bearing_decoder_output() -> Result<(), Box> { + let format = FixtureFormat::Wav; + let source = MemorySource::new(fixtures::valid(format)?); + let decoder_output = String::from("{\"text\":\"prefix\0suffix\"}"); + + let result = support::read( + &source, + format.media_type(), + &DirectProcessor::injecting(decoder_output), + ) + .await; + support::assert_processor_failed(result); + Ok(()) +} + +async fn assert_reason( + format: FixtureFormat, + bytes: Vec, + expected_reason: &str, +) -> Result<(), Box> { + let source = MemorySource::new(bytes); + let inspection = support::inspect(&source, format.media_type()).await?; + support::assert_malformed_reason(inspection, expected_reason); + Ok(()) +} diff --git a/crates/file-media-adapters-audio/tests/fixtures/mod.rs b/crates/file-media-adapters-audio/tests/fixtures/mod.rs new file mode 100644 index 0000000000..1a742ed856 --- /dev/null +++ b/crates/file-media-adapters-audio/tests/fixtures/mod.rs @@ -0,0 +1,562 @@ +use std::error::Error; + +use ogg::{PacketWriteEndInfo, PacketWriter}; +use opus_rs::{Application, OpusEncoder}; +use rusty_mp3::{Error as Mp3Error, Mp3Encoder, Mp3EncoderConfig}; + +#[derive(Clone, Copy)] +pub(crate) enum FixtureFormat { + Wav, + Mp3, + Flac, + OggOpus, +} + +pub(crate) struct ValidFixture { + bytes: Vec, + media_type: &'static str, + expected_metadata: serde_json::Value, +} + +impl ValidFixture { + pub(crate) fn bytes(&self) -> &[u8] { + &self.bytes + } + + pub(crate) const fn media_type(&self) -> &'static str { + self.media_type + } + + pub(crate) const fn expected_metadata(&self) -> &serde_json::Value { + &self.expected_metadata + } +} + +struct OggOpusFixture { + packet_count: usize, + frame_size: usize, + pre_skip: u16, + final_granule: u64, + ending: OggEnding, + head_ending: HeadEnding, + first_audio_page_granule: Option, +} + +pub(crate) struct Id3HeaderFixture { + pub(crate) major: u8, + pub(crate) flags: u8, +} + +enum HeadEnding { + IsolatedPage, + SharedPage, +} + +enum OggEnding { + EndOfStream, + EndOfPage, +} + +impl FixtureFormat { + pub(crate) const fn media_type(self) -> &'static str { + match self { + Self::Wav => "audio/wav", + Self::Mp3 => "audio/mpeg", + Self::Flac => "audio/flac", + Self::OggOpus => "audio/ogg", + } + } +} + +pub(crate) fn valid(format: FixtureFormat) -> Result, Box> { + match format { + FixtureFormat::Wav => wav(8_000, 800), + FixtureFormat::Mp3 => mp3(8_000, 800), + FixtureFormat::Flac => flac(8_000, 800), + FixtureFormat::OggOpus => ogg_opus(OggOpusFixture { + packet_count: 5, + frame_size: 960, + pre_skip: 0, + final_granule: 4_800, + ending: OggEnding::EndOfStream, + head_ending: HeadEnding::IsolatedPage, + first_audio_page_granule: None, + }), + } +} + +pub(crate) fn valid_fixture(format: FixtureFormat) -> Result> { + let sample_rate_hz = if matches!(format, FixtureFormat::OggOpus) { + 48_000 + } else { + 8_000 + }; + Ok(ValidFixture { + bytes: valid(format)?, + media_type: format.media_type(), + expected_metadata: serde_json::json!({ + "channels": 1, + "sample_rate_hz": sample_rate_hz + }), + }) +} + +pub(crate) fn truncated(format: FixtureFormat) -> Result, Box> { + let mut bytes = valid(format)?; + bytes.truncate(bytes.len() / 2); + Ok(bytes) +} + +pub(crate) fn wav_larger_than_one_broker_range() -> Result, Box> { + wav(192_000, 600_000) +} + +pub(crate) fn malformed(format: FixtureFormat) -> Vec { + match format { + FixtureFormat::Wav => b"RIFF\x04\x00\x00\x00WAVE".to_vec(), + FixtureFormat::Mp3 => vec![0xff, 0xfb, 0x00, 0x00], + FixtureFormat::Flac => b"fLaCmalformed".to_vec(), + FixtureFormat::OggOpus => b"OggSmalformed-OpusHead".to_vec(), + } +} + +pub(crate) fn oversized(format: FixtureFormat) -> Result, Box> { + let mut bytes = valid(format)?; + bytes.resize( + signalbox_file_media_adapters_audio::MAX_AUDIO_SOURCE_BYTES as usize + 1, + 0, + ); + Ok(bytes) +} + +pub(crate) fn duration_bomb(format: FixtureFormat) -> Result, Box> { + match format { + FixtureFormat::Wav => wav(1_000, 61_000), + FixtureFormat::Mp3 => mp3(8_000, 488_000), + FixtureFormat::Flac => flac(8_000, 488_000), + FixtureFormat::OggOpus => ogg_opus(OggOpusFixture { + packet_count: 3_001, + frame_size: 960, + pre_skip: 0, + final_granule: 2_880_960, + ending: OggEnding::EndOfStream, + head_ending: HeadEnding::IsolatedPage, + first_audio_page_granule: None, + }), + } +} + +pub(crate) fn ogg_opus_without_end_of_stream() -> Result, Box> { + ogg_opus(OggOpusFixture { + packet_count: 5, + frame_size: 960, + pre_skip: 0, + final_granule: 4_800, + ending: OggEnding::EndOfPage, + head_ending: HeadEnding::IsolatedPage, + first_audio_page_granule: None, + }) +} + +pub(crate) fn ogg_opus_trimmed_to_duration_limit() -> Result, Box> { + ogg_opus(OggOpusFixture { + packet_count: 3_001, + frame_size: 960, + pre_skip: 312, + final_granule: 2_880_312, + ending: OggEnding::EndOfStream, + head_ending: HeadEnding::IsolatedPage, + first_audio_page_granule: None, + }) +} + +pub(crate) fn ogg_opus_with_shared_identification_page() -> Result, Box> { + ogg_opus(OggOpusFixture { + packet_count: 1, + frame_size: 960, + pre_skip: 0, + final_granule: 960, + ending: OggEnding::EndOfStream, + head_ending: HeadEnding::SharedPage, + first_audio_page_granule: None, + }) +} + +pub(crate) fn ogg_opus_with_regressing_granule() -> Result, Box> { + ogg_opus(OggOpusFixture { + packet_count: 2, + frame_size: 960, + pre_skip: 0, + final_granule: 480, + ending: OggEnding::EndOfStream, + head_ending: HeadEnding::IsolatedPage, + first_audio_page_granule: Some(960), + }) +} + +pub(crate) fn ogg_opus_with_inaccurate_intermediate_granule() -> Result, Box> { + ogg_opus(OggOpusFixture { + packet_count: 2, + frame_size: 960, + pre_skip: 0, + final_granule: 1_920, + ending: OggEnding::EndOfStream, + head_ending: HeadEnding::IsolatedPage, + first_audio_page_granule: Some(480), + }) +} + +pub(crate) fn ogg_opus_with_excessive_end_trim() -> Result, Box> { + ogg_opus(OggOpusFixture { + packet_count: 2, + frame_size: 960, + pre_skip: 0, + final_granule: 1, + ending: OggEnding::EndOfStream, + head_ending: HeadEnding::IsolatedPage, + first_audio_page_granule: None, + }) +} + +pub(crate) fn ogg_opus_with_head_end_of_stream() -> Result, Box> { + const SERIAL: u32 = 0x51_67_6e_6c; + let mut writer = PacketWriter::new(Vec::new()); + writer.write_packet(opus_head(0), SERIAL, PacketWriteEndInfo::EndStream, 0)?; + writer.write_packet(opus_tags(), SERIAL, PacketWriteEndInfo::EndPage, 0)?; + Ok(writer.into_inner()) +} + +pub(crate) fn mp3_with_long_id3_tag() -> Result, Box> { + let encoded = mp3(8_000, 800)?; + let audio = encoded.get(10..).ok_or("missing MP3 audio")?; + let tag_length = 128_usize; + let mut tagged = b"ID3\x04\x00\x00\x00\x00\x01\x00".to_vec(); + tagged.resize(10 + tag_length, 0); + tagged.extend_from_slice(audio); + Ok(tagged) +} + +/// Builds a valid MP3 whose ID3v2.4 footer sits past the bounded probe prefix. +/// +/// Returns the bytes and the exact offset of the footer's own range request. +pub(crate) fn mp3_with_id3v24_footer_past_the_probe_prefix() +-> Result<(Vec, u64), Box> { + let encoded = mp3(8_000, 800)?; + let audio = encoded.get(10..).ok_or("missing MP3 audio")?; + let tag_length = 128_usize; + let header = *b"ID3\x04\x00\x10\x00\x00\x01\x00"; + let mut tagged = header.to_vec(); + tagged.resize(10 + tag_length, 0); + let footer_offset = tagged.len(); + tagged.extend_from_slice(b"3DI"); + tagged.extend_from_slice(&header[3..10]); + tagged.extend_from_slice(audio); + Ok((tagged, u64::try_from(footer_offset)?)) +} + +pub(crate) fn mp3_with_id3_header(fixture: Id3HeaderFixture) -> Result, Box> { + let mut bytes = mp3(8_000, 800)?; + bytes[3] = fixture.major; + bytes[5] = fixture.flags; + Ok(bytes) +} + +pub(crate) fn flac_with_mismatched_md5() -> Result, Box> { + let mut bytes = flac(8_000, 800)?; + let first_md5_byte = bytes.get_mut(26).ok_or("missing FLAC STREAMINFO MD5")?; + *first_md5_byte = 1; + Ok(bytes) +} + +pub(crate) fn flac_truncated_between_complete_frames() -> Result, Box> { + let mut bytes = flac(8_000, 768)?; + let stream_info = bytes.get_mut(18..26).ok_or("missing FLAC STREAMINFO")?; + let mut encoded = u64::from_be_bytes(<[u8; 8]>::try_from(&*stream_info)?); + encoded = (encoded & !0x0f_ff_ff_ff_ff) | 800; + stream_info.copy_from_slice(&encoded.to_be_bytes()); + Ok(bytes) +} + +pub(crate) fn flac_with_mismatched_streaminfo_channels() -> Result, Box> { + let mut bytes = flac(8_000, 800)?; + let stream_info = bytes.get_mut(18..26).ok_or("missing FLAC STREAMINFO")?; + let mut encoded = u64::from_be_bytes(<[u8; 8]>::try_from(&*stream_info)?); + encoded |= 1_u64 << 41; + stream_info.copy_from_slice(&encoded.to_be_bytes()); + Ok(bytes) +} + +pub(crate) fn wav_without_frames() -> Result, Box> { + wav(8_000, 0) +} + +pub(crate) fn mp3_with_invalid_id3v24_footer() -> Result, Box> { + let mut bytes = mp3(8_000, 800)?; + bytes[5] = 0x10; + bytes.splice(10..10, [0_u8; 10]); + Ok(bytes) +} + +pub(crate) fn mp3_with_empty_id3v24_extended_header() -> Result, Box> { + let mut bytes = mp3(8_000, 800)?; + bytes[5] = 0x40; + Ok(bytes) +} + +pub(crate) fn mp3_with_excess_xing_frame_count() -> Result, Box> { + let mut bytes = mp3(8_000, 8_000)?; + let frame_start = 10_usize; + let header = bytes + .get(frame_start..frame_start + 4) + .ok_or("missing MPEG frame header")?; + let has_crc = header[1] & 1 == 0; + let mpeg_one = (header[1] >> 3) & 0x03 == 0x03; + let mono = (header[3] >> 6) & 0x03 == 0x03; + let header_size = if has_crc { 6 } else { 4 }; + let side_information = match (mpeg_one, mono) { + (true, true) => 17, + (true, false) => 32, + (false, true) => 9, + (false, false) => 17, + }; + let xing = frame_start + header_size + side_information; + bytes + .get_mut(xing..xing + 12) + .ok_or("MPEG frame is too short for a Xing header")? + .copy_from_slice(&[b'X', b'i', b'n', b'g', 0, 0, 0, 1, 0, 0, 3, 0xe8]); + Ok(bytes) +} + +pub(crate) fn mp3_with_xing_frame_count_of_one() -> Result, Box> { + let mut bytes = mp3(8_000, 576)?; + let frame_start = 10_usize; + let header = bytes + .get(frame_start..frame_start + 4) + .ok_or("missing MPEG frame header")?; + let has_crc = header[1] & 1 == 0; + let mpeg_one = (header[1] >> 3) & 0x03 == 0x03; + let mono = (header[3] >> 6) & 0x03 == 0x03; + let header_size = if has_crc { 6 } else { 4 }; + let side_information = match (mpeg_one, mono) { + (true, true) => 17, + (true, false) => 32, + (false, true) => 9, + (false, false) => 17, + }; + let xing = frame_start + header_size + side_information; + bytes + .get_mut(xing..xing + 12) + .ok_or("MPEG frame is too short for a Xing header")? + // A declared total of one frame is the Xing/VBRI header frame + // itself and no audio frames, distinct from "unknown" (0). + .copy_from_slice(&[b'X', b'i', b'n', b'g', 0, 0, 0, 1, 0, 0, 0, 1]); + Ok(bytes) +} + +pub(crate) fn ogg_opus_with_tags_end_of_stream() -> Result, Box> { + ogg_opus_with_tags_ending(PacketWriteEndInfo::EndStream, 0) +} + +pub(crate) fn ogg_opus_with_nonzero_tags_granule() -> Result, Box> { + ogg_opus_with_tags_ending(PacketWriteEndInfo::EndPage, 1) +} + +fn ogg_opus_with_tags_ending( + tags_ending: PacketWriteEndInfo, + tags_granule: u64, +) -> Result, Box> { + const SERIAL: u32 = 0x51_67_6e_6c; + let mut encoder = OpusEncoder::new(48_000, 1, Application::Audio)?; + encoder.bitrate_bps = 6_000; + let samples = vec![0.0_f32; 960]; + let mut encoded = vec![0_u8; 1_276]; + let packet_bytes = encoder.encode(&samples, 960, &mut encoded)?; + encoded.truncate(packet_bytes); + + let mut writer = PacketWriter::new(Vec::new()); + writer.write_packet(opus_head(0), SERIAL, PacketWriteEndInfo::EndPage, 0)?; + writer.write_packet(opus_tags(), SERIAL, tags_ending, tags_granule)?; + writer.write_packet(encoded, SERIAL, PacketWriteEndInfo::EndStream, 960)?; + Ok(writer.into_inner()) +} + +fn wav(sample_rate_hz: u32, frames: usize) -> Result, Box> { + let data_size = u32::try_from(frames)?; + let riff_size = 36_u32.checked_add(data_size).ok_or("WAV size overflow")?; + let mut bytes = Vec::with_capacity(44 + frames); + bytes.extend_from_slice(b"RIFF"); + bytes.extend_from_slice(&riff_size.to_le_bytes()); + bytes.extend_from_slice(b"WAVEfmt "); + bytes.extend_from_slice(&16_u32.to_le_bytes()); + bytes.extend_from_slice(&1_u16.to_le_bytes()); + bytes.extend_from_slice(&1_u16.to_le_bytes()); + bytes.extend_from_slice(&sample_rate_hz.to_le_bytes()); + bytes.extend_from_slice(&sample_rate_hz.to_le_bytes()); + bytes.extend_from_slice(&1_u16.to_le_bytes()); + bytes.extend_from_slice(&8_u16.to_le_bytes()); + bytes.extend_from_slice(b"data"); + bytes.extend_from_slice(&data_size.to_le_bytes()); + bytes.resize(44 + frames, 128); + Ok(bytes) +} + +fn mp3(sample_rate_hz: u32, frames: usize) -> Result, Box> { + let samples = vec![0_i16; frames]; + let mut encoder = Mp3Encoder::new(Mp3EncoderConfig { + bitrate_kbps: 8, + vbr_quality: None, + }); + encoder.push_pcm_s16(&samples, 1, sample_rate_hz)?; + encoder.finish(); + let mut bytes = Vec::new(); + loop { + match encoder.next_packet() { + Ok(packet) => bytes.extend_from_slice(&packet), + Err(Mp3Error::Eof) => break, + Err(Mp3Error::Again) => { + return Err("MP3 encoder requested more input after finish".into()); + } + Err(error) => return Err(error.into()), + } + } + let mut tagged = b"ID3\x04\x00\x00\x00\x00\x00\x00".to_vec(); + tagged.extend_from_slice(&bytes); + Ok(tagged) +} + +fn flac(sample_rate_hz: u32, frames: usize) -> Result, Box> { + const BLOCK_SIZE: usize = 256; + let total_samples = u64::try_from(frames)?; + let stream_info = + (u64::from(sample_rate_hz) << 44) | (7_u64 << 36) | (total_samples & 0x0f_ff_ff_ff_ff); + let mut bytes = b"fLaC".to_vec(); + bytes.extend_from_slice(&[0x80, 0, 0, 34]); + bytes.extend_from_slice(&u16::try_from(BLOCK_SIZE)?.to_be_bytes()); + bytes.extend_from_slice(&u16::try_from(BLOCK_SIZE)?.to_be_bytes()); + bytes.extend_from_slice(&[0; 6]); + bytes.extend_from_slice(&stream_info.to_be_bytes()); + bytes.extend_from_slice(&[0; 16]); + + for (frame_number, start) in (0..frames).step_by(BLOCK_SIZE).enumerate() { + let block_size = (frames - start).min(BLOCK_SIZE); + append_constant_flac_frame( + &mut bytes, + u32::try_from(frame_number)?, + u16::try_from(block_size)?, + sample_rate_hz, + )?; + } + Ok(bytes) +} + +fn append_constant_flac_frame( + bytes: &mut Vec, + frame_number: u32, + block_size: u16, + sample_rate_hz: u32, +) -> Result<(), Box> { + let frame_start = bytes.len(); + bytes.extend_from_slice(&[0xff, 0xf8, 0x6c, 0x02]); + let frame_character = char::from_u32(frame_number).ok_or("FLAC frame number overflow")?; + let mut encoded_frame_number = [0_u8; 4]; + bytes.extend_from_slice( + frame_character + .encode_utf8(&mut encoded_frame_number) + .as_bytes(), + ); + bytes.push(u8::try_from(block_size - 1)?); + bytes.push(u8::try_from(sample_rate_hz / 1_000)?); + bytes.push(flac_crc8(&bytes[frame_start..])); + bytes.extend_from_slice(&[0, 0]); + let checksum = flac_crc16(&bytes[frame_start..]); + bytes.extend_from_slice(&checksum.to_be_bytes()); + Ok(()) +} + +fn flac_crc8(bytes: &[u8]) -> u8 { + let mut crc = 0_u8; + for byte in bytes { + crc ^= byte; + for _ in 0..8 { + crc = if crc & 0x80 != 0 { + (crc << 1) ^ 0x07 + } else { + crc << 1 + }; + } + } + crc +} + +fn flac_crc16(bytes: &[u8]) -> u16 { + let mut crc = 0_u16; + for byte in bytes { + crc ^= u16::from(*byte) << 8; + for _ in 0..8 { + crc = if crc & 0x8000 != 0 { + (crc << 1) ^ 0x8005 + } else { + crc << 1 + }; + } + } + crc +} + +fn ogg_opus(fixture: OggOpusFixture) -> Result, Box> { + const SERIAL: u32 = 0x51_67_6e_6c; + let mut encoder = OpusEncoder::new(48_000, 1, Application::Audio)?; + encoder.bitrate_bps = 6_000; + let samples = vec![0.0_f32; fixture.frame_size]; + let mut encoded = vec![0_u8; 1_276]; + let packet_bytes = encoder.encode(&samples, fixture.frame_size, &mut encoded)?; + encoded.truncate(packet_bytes); + + let mut writer = PacketWriter::new(Vec::new()); + let head_end = match fixture.head_ending { + HeadEnding::IsolatedPage => PacketWriteEndInfo::EndPage, + HeadEnding::SharedPage => PacketWriteEndInfo::NormalPacket, + }; + writer.write_packet(opus_head(fixture.pre_skip), SERIAL, head_end, 0)?; + writer.write_packet(opus_tags(), SERIAL, PacketWriteEndInfo::NormalPacket, 0)?; + for packet_index in 0..fixture.packet_count { + let final_packet = packet_index + 1 == fixture.packet_count; + let end = if packet_index == 0 && fixture.first_audio_page_granule.is_some() { + PacketWriteEndInfo::EndPage + } else if final_packet { + match fixture.ending { + OggEnding::EndOfStream => PacketWriteEndInfo::EndStream, + OggEnding::EndOfPage => PacketWriteEndInfo::EndPage, + } + } else { + PacketWriteEndInfo::NormalPacket + }; + let granule = if packet_index == 0 && fixture.first_audio_page_granule.is_some() { + fixture.first_audio_page_granule.unwrap_or(0) + } else if final_packet { + fixture.final_granule + } else { + u64::try_from(packet_index + 1)? * u64::try_from(fixture.frame_size)? + }; + writer.write_packet(encoded.clone(), SERIAL, end, granule)?; + } + Ok(writer.into_inner()) +} + +fn opus_head(pre_skip: u16) -> Vec { + let mut head = b"OpusHead".to_vec(); + head.push(1); + head.push(1); + head.extend_from_slice(&pre_skip.to_le_bytes()); + head.extend_from_slice(&48_000_u32.to_le_bytes()); + head.extend_from_slice(&0_i16.to_le_bytes()); + head.push(0); + head +} + +fn opus_tags() -> Vec { + let mut tags = b"OpusTags".to_vec(); + tags.extend_from_slice(&0_u32.to_le_bytes()); + tags.extend_from_slice(&0_u32.to_le_bytes()); + tags +} diff --git a/crates/file-media-adapters-audio/tests/support/mod.rs b/crates/file-media-adapters-audio/tests/support/mod.rs new file mode 100644 index 0000000000..31da21f68d --- /dev/null +++ b/crates/file-media-adapters-audio/tests/support/mod.rs @@ -0,0 +1,292 @@ +use std::{error::Error, num::NonZeroU64, path::PathBuf, sync::Arc}; + +use signalbox_file_media_adapters_audio::{AudioFamilyProvider, audio_family_declaration}; +use signalbox_file_media_processor_runtime::{SandboxedFileMediaProcessor, WorkerBinding}; +use signalbox_file_media_runtime::{ + AttachmentKind, CancellationSignal, DeclaredMediaType, FileDigest, FileInspection, + FileMediaCeilings, FileMediaFailure, FileMediaProcessCeilings, FileMediaProcessor, + FileMediaProcessorFuture, FileMediaProvider, FileMediaProviderReadRequest, + FileMediaProviderValidationRequest, FileMediaRegistry, FileReadInput, FileReadRequest, + FileReadResult, FileUse, InspectionRequest, NeverCancelled, ProcessorBoundaryFailure, + ProcessorFailure, ProcessorIsolation, ProcessorProbeOutput, ProcessorReadOutput, + ProcessorValidationOutput, ReadViewName, ReaderIdentity, SourceReadError, SourceReadFuture, + VerifiedBlobSource, +}; + +pub(crate) struct MemorySource { + bytes: Arc<[u8]>, + unavailable_offset: Option, +} + +impl MemorySource { + pub(crate) fn new(bytes: Vec) -> Self { + Self { + bytes: Arc::from(bytes), + unavailable_offset: None, + } + } + + /// Builds a source whose in-bounds range at `offset` reports a verified-source failure. + pub(crate) fn unavailable_at(bytes: Vec, offset: u64) -> Self { + Self { + bytes: Arc::from(bytes), + unavailable_offset: Some(offset), + } + } + + pub(crate) fn file_use(&self, media_type: &str) -> Result> { + Ok(FileUse::new( + self.digest(), + self.byte_length(), + AttachmentKind::File, + DeclaredMediaType::try_new(media_type)?, + None, + )) + } +} + +impl VerifiedBlobSource for MemorySource { + fn digest(&self) -> FileDigest { + FileDigest::from_bytes([9; 32]) + } + + fn byte_length(&self) -> NonZeroU64 { + NonZeroU64::new(self.bytes.len() as u64).unwrap_or(NonZeroU64::MIN) + } + + fn read_range(&self, offset: u64, length: NonZeroU64) -> SourceReadFuture<'_> { + Box::pin(async move { + if self.unavailable_offset == Some(offset) { + return Err(SourceReadError::Unavailable); + } + let start = usize::try_from(offset).map_err(|_| SourceReadError::RangeOutOfBounds)?; + let requested = + usize::try_from(length.get()).map_err(|_| SourceReadError::RangeOutOfBounds)?; + let end = start + .checked_add(requested) + .ok_or(SourceReadError::RangeOutOfBounds)?; + self.bytes + .get(start..end) + .map(<[u8]>::to_vec) + .ok_or(SourceReadError::RangeOutOfBounds) + }) + } +} + +#[derive(Clone, Debug)] +enum ReadBehavior { + Provider, + InjectedStructured(String), +} + +pub(crate) struct DirectProcessor { + provider: AudioFamilyProvider, + read_behavior: ReadBehavior, +} + +impl DirectProcessor { + pub(crate) fn provider() -> Self { + Self { + provider: AudioFamilyProvider, + read_behavior: ReadBehavior::Provider, + } + } + + pub(crate) fn injecting(body_json: String) -> Self { + Self { + provider: AudioFamilyProvider, + read_behavior: ReadBehavior::InjectedStructured(body_json), + } + } +} + +impl FileMediaProcessor for DirectProcessor { + fn probe<'a>( + &'a self, + reader: &'a ReaderIdentity, + source: &'a dyn VerifiedBlobSource, + cancellation: &'a dyn CancellationSignal, + ) -> FileMediaProcessorFuture<'a, ProcessorProbeOutput> { + Box::pin(async move { + self.provider + .probe(reader, source, cancellation) + .await + .map_err(|_| ProcessorBoundaryFailure::Processor(ProcessorFailure::Failed)) + }) + } + + fn validate<'a>( + &'a self, + reader: &'a ReaderIdentity, + request: FileMediaProviderValidationRequest, + source: &'a dyn VerifiedBlobSource, + cancellation: &'a dyn CancellationSignal, + ) -> FileMediaProcessorFuture<'a, ProcessorValidationOutput> { + Box::pin(async move { + self.provider + .inspect(reader, request, source, cancellation) + .await + .map_err(|_| ProcessorBoundaryFailure::Processor(ProcessorFailure::Failed)) + }) + } + + fn read<'a>( + &'a self, + reader: &'a ReaderIdentity, + request: FileMediaProviderReadRequest, + source: &'a dyn VerifiedBlobSource, + cancellation: &'a dyn CancellationSignal, + ) -> FileMediaProcessorFuture<'a, ProcessorReadOutput> { + match &self.read_behavior { + ReadBehavior::Provider => Box::pin(async move { + self.provider + .read(reader, request, source, cancellation) + .await + .map_err(|_| ProcessorBoundaryFailure::Processor(ProcessorFailure::Failed)) + }), + ReadBehavior::InjectedStructured(body_json) => { + let body_json = body_json.clone(); + Box::pin(async move { + Ok(ProcessorReadOutput::Structured { + body_json, + truncated: false, + cursor: None, + }) + }) + } + } + } +} + +fn registry() -> Result> { + Ok(FileMediaRegistry::try_new( + vec![audio_family_declaration().map_err(|error| error.to_string())?], + FileMediaCeilings::version_one(), + ProcessorIsolation::Available, + )?) +} + +pub(crate) async fn inspect( + source: &MemorySource, + media_type: &str, +) -> Result> { + Ok(registry()? + .inspect( + &DirectProcessor::provider(), + InspectionRequest { + source: source.file_use(media_type)?, + visible_part: None, + }, + source, + &NeverCancelled, + ) + .await?) +} + +/// Inspects a source and returns the registry-visible failure, if any. +pub(crate) async fn inspect_failure( + source: &MemorySource, + media_type: &str, +) -> Result, Box> { + Ok(registry()? + .inspect( + &DirectProcessor::provider(), + InspectionRequest { + source: source.file_use(media_type)?, + visible_part: None, + }, + source, + &NeverCancelled, + ) + .await + .err()) +} + +pub(crate) async fn inspect_sandboxed( + source: &MemorySource, + media_type: &str, +) -> Result> { + let declaration = audio_family_declaration().map_err(|error| error.to_string())?; + let worker = PathBuf::from(env!("CARGO_BIN_EXE_signalbox-file-media-audio-worker")); + let binding = WorkerBinding::try_new(worker, declaration)?; + let processor = SandboxedFileMediaProcessor::try_new( + "/usr/bin/bwrap", + vec![binding], + FileMediaProcessCeilings::version_one(), + )?; + if processor.verify_isolation().await != ProcessorIsolation::Available { + if std::env::var_os("CI").is_some() { + return Err("CI requires the real audio worker sandbox".into()); + } + return inspect(source, media_type).await; + } + Ok(registry()? + .inspect( + &processor, + InspectionRequest { + source: source.file_use(media_type)?, + visible_part: None, + }, + source, + &NeverCancelled, + ) + .await?) +} + +pub(crate) async fn read( + source: &MemorySource, + media_type: &str, + processor: &DirectProcessor, +) -> Result { + let source_use = source + .file_use(media_type) + .map_err(|_| FileMediaFailure::ProcessorFailed)?; + let view = ReadViewName::try_new("metadata").map_err(|_| FileMediaFailure::ProcessorFailed)?; + registry() + .map_err(|_| FileMediaFailure::ProcessorFailed)? + .read( + processor, + FileReadRequest { + inspection: InspectionRequest { + source: source_use, + visible_part: None, + }, + view, + input: FileReadInput::Initial { + options: serde_json::json!({}), + }, + }, + source, + &NeverCancelled, + ) + .await +} + +#[track_caller] +pub(crate) fn assert_validated_media(inspection: FileInspection, expected: &str) { + assert!(matches!(inspection, FileInspection::Validated(_))); + if let FileInspection::Validated(validated) = inspection { + assert_eq!(validated.detected_media_type().as_str(), expected); + } +} + +#[track_caller] +pub(crate) fn assert_malformed_reason(inspection: FileInspection, expected: &str) { + assert!(matches!(inspection, FileInspection::Malformed { .. })); + if let FileInspection::Malformed { reason_code, .. } = inspection { + assert_eq!(reason_code.as_str(), expected); + } +} + +#[track_caller] +pub(crate) fn assert_structured(result: FileReadResult, expected: &serde_json::Value) { + assert!(matches!(result, FileReadResult::Structured { .. })); + if let FileReadResult::Structured { body, .. } = result { + assert_eq!(&body, expected); + } +} + +#[track_caller] +pub(crate) fn assert_processor_failed(result: Result) { + assert_eq!(result, Err(FileMediaFailure::ProcessorFailed)); +} diff --git a/crates/file-media-adapters-image/Cargo.toml b/crates/file-media-adapters-image/Cargo.toml new file mode 100644 index 0000000000..1e0f4827ba --- /dev/null +++ b/crates/file-media-adapters-image/Cargo.toml @@ -0,0 +1,33 @@ +[package] +name = "signalbox-file-media-adapters-image" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +repository.workspace = true +publish.workspace = true + +[[bin]] +name = "signalbox-file-media-image-worker" +path = "src/bin/signalbox-file-media-image-worker.rs" + +[dependencies] +image = { version = "0.25.8", default-features = false, features = [ + "gif", + "jpeg", + "png", + "webp", +] } +serde_json = "1.0.140" +signalbox-file-media-processor-runtime = { path = "../file-media-processor-runtime" } +signalbox-file-media-runtime = { path = "../file-media-runtime" } +tokio = { version = "1.53.0", default-features = false, features = [ + "macros", + "rt", +] } + +[dev-dependencies] +crc32fast = "1.5.0" + +[lints] +workspace = true diff --git a/crates/file-media-adapters-image/src/adapter.rs b/crates/file-media-adapters-image/src/adapter.rs new file mode 100644 index 0000000000..ce76619534 --- /dev/null +++ b/crates/file-media-adapters-image/src/adapter.rs @@ -0,0 +1,157 @@ +use std::io::Cursor; + +use image::{GenericImageView, ImageReader, Limits}; +use signalbox_file_media_runtime::{ + CancellationSignal, FileMediaProviderReadRequest, FileMediaProviderValidationRequest, + ProbeStrength, ProcessorFailure, ProcessorProbeOutput, ProcessorReadOutput, + ProcessorValidationOutput, VerifiedBlobSource, +}; + +use crate::{ + AdapterFormat, DIMENSION_LIMIT_EXCEEDED_REASON, MALFORMED_IMAGE_REASON, MAX_IMAGE_AXIS, + MAX_IMAGE_DECODED_PIXELS, MAX_IMAGE_SOURCE_BYTES, METADATA_VIEW_NAME, + PIXEL_LIMIT_EXCEEDED_REASON, SOURCE_TOO_LARGE_REASON, options_are_empty, source, +}; + +// numeric-bound: ceiling - protects worker memory from runaway decoder allocation +const MAX_DECODER_ALLOCATION_BYTES: u64 = 128 * 1024 * 1024; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +struct ImageMetadata { + width: u32, + height: u32, + channels: u8, +} + +pub(crate) async fn probe( + format: AdapterFormat, + source: &dyn VerifiedBlobSource, + cancellation: &dyn CancellationSignal, +) -> Result { + let prefix = source::read_probe_prefix(source, cancellation).await?; + if format.matches_signature(&prefix) { + Ok(ProcessorProbeOutput::Candidate { + media_type: String::from(format.media_type()), + strength: ProbeStrength::Strong, + }) + } else { + Ok(ProcessorProbeOutput::NoMatch) + } +} + +pub(crate) async fn inspect( + format: AdapterFormat, + request: FileMediaProviderValidationRequest, + source: &dyn VerifiedBlobSource, + cancellation: &dyn CancellationSignal, +) -> Result { + if request.media_type.as_str() != format.media_type() { + return Err(ProcessorFailure::Protocol); + } + let Some(bytes) = + source::read_complete(source, cancellation, request.maximum_source_bytes).await? + else { + return Ok(malformed(format, SOURCE_TOO_LARGE_REASON)); + }; + let metadata = match decode( + format, + &bytes, + request.maximum_image_axis, + request.maximum_decoded_image_pixels, + ) { + Ok(metadata) => metadata, + Err(reason) => return Ok(malformed(format, reason)), + }; + Ok(ProcessorValidationOutput::Validated { + media_type: String::from(format.media_type()), + evidence: request.evidence, + metadata_json: metadata_json(metadata)?, + }) +} + +pub(crate) async fn read( + format: AdapterFormat, + request: FileMediaProviderReadRequest, + source: &dyn VerifiedBlobSource, + cancellation: &dyn CancellationSignal, +) -> Result { + let signalbox_file_media_runtime::FileReadInput::Initial { options } = &request.input else { + return Ok(ProcessorReadOutput::InvalidViewArguments); + }; + if request.view.as_str() != METADATA_VIEW_NAME || !options_are_empty(options) { + return Ok(ProcessorReadOutput::InvalidViewArguments); + } + let Some(bytes) = source::read_complete(source, cancellation, MAX_IMAGE_SOURCE_BYTES).await? + else { + return Ok(ProcessorReadOutput::SourceTooLarge { + maximum_bytes: MAX_IMAGE_SOURCE_BYTES, + }); + }; + let metadata = decode( + format, + &bytes, + request.maximum_image_axis, + request.maximum_decoded_image_pixels, + ) + .map_err(|_| ProcessorFailure::Failed)?; + Ok(ProcessorReadOutput::Structured { + body_json: metadata_json(metadata)?, + truncated: false, + cursor: None, + }) +} + +fn decode( + format: AdapterFormat, + bytes: &[u8], + maximum_axis: u32, + maximum_pixels: u64, +) -> Result { + let dimensions = ImageReader::with_format(Cursor::new(bytes), format.image_format()) + .into_dimensions() + .map_err(|_| MALFORMED_IMAGE_REASON)?; + let maximum_axis = maximum_axis.min(MAX_IMAGE_AXIS); + let maximum_pixels = maximum_pixels.min(MAX_IMAGE_DECODED_PIXELS); + if dimensions.0 > maximum_axis || dimensions.1 > maximum_axis { + return Err(DIMENSION_LIMIT_EXCEEDED_REASON); + } + let pixels = u64::from(dimensions.0) + .checked_mul(u64::from(dimensions.1)) + .ok_or(PIXEL_LIMIT_EXCEEDED_REASON)?; + if pixels > maximum_pixels { + return Err(PIXEL_LIMIT_EXCEEDED_REASON); + } + + let mut reader = ImageReader::with_format(Cursor::new(bytes), format.image_format()); + let mut limits = Limits::default(); + limits.max_image_width = Some(maximum_axis); + limits.max_image_height = Some(maximum_axis); + limits.max_alloc = Some(MAX_DECODER_ALLOCATION_BYTES); + reader.limits(limits); + let image = reader.decode().map_err(|_| MALFORMED_IMAGE_REASON)?; + let decoded_dimensions = image.dimensions(); + if decoded_dimensions != dimensions { + return Err(MALFORMED_IMAGE_REASON); + } + Ok(ImageMetadata { + width: dimensions.0, + height: dimensions.1, + channels: image.color().channel_count(), + }) +} + +fn metadata_json(metadata: ImageMetadata) -> Result { + serde_json::to_string(&serde_json::json!({ + "channels": metadata.channels, + "height": metadata.height, + "width": metadata.width, + })) + .map_err(|_| ProcessorFailure::Failed) +} + +fn malformed(format: AdapterFormat, reason: &str) -> ProcessorValidationOutput { + ProcessorValidationOutput::Malformed { + media_type: String::from(format.media_type()), + reason_code: String::from(reason), + } +} diff --git a/crates/file-media-adapters-image/src/bin/signalbox-file-media-image-worker.rs b/crates/file-media-adapters-image/src/bin/signalbox-file-media-image-worker.rs new file mode 100644 index 0000000000..163631a6e6 --- /dev/null +++ b/crates/file-media-adapters-image/src/bin/signalbox-file-media-image-worker.rs @@ -0,0 +1,11 @@ +use std::error::Error; + +use signalbox_file_media_adapters_image::ImageFamilyProvider; +use signalbox_file_media_processor_runtime::{WorkerCatalog, serve_one}; + +#[tokio::main(flavor = "current_thread")] +async fn main() -> Result<(), Box> { + let catalog = WorkerCatalog::try_new(vec![Box::new(ImageFamilyProvider)])?; + serve_one(&catalog).await?; + Ok(()) +} diff --git a/crates/file-media-adapters-image/src/lib.rs b/crates/file-media-adapters-image/src/lib.rs new file mode 100644 index 0000000000..2fcc540f21 --- /dev/null +++ b/crates/file-media-adapters-image/src/lib.rs @@ -0,0 +1,213 @@ +//! Isolated adapters for PNG, JPEG, WebP, and GIF bytes. + +mod adapter; +mod source; + +use std::{error::Error, str::FromStr}; + +use image::ImageFormat; +use signalbox_file_media_runtime::{ + CanonicalJsonObjectSchema, CanonicalMediaType, FileMediaProvider, FileMediaProviderDeclaration, + FileMediaProviderFailure, FileMediaProviderFuture, FileMediaProviderReadRequest, + FileMediaProviderValidationRequest, FileReaderName, FileReaderProviderName, FileReaderRevision, + ProbeDeclaration, ProbeDeclarationInput, ProcessorProbeOutput, ProcessorReadOutput, + ProcessorValidationOutput, ReadAccessPattern, ReadViewBounds, ReadViewDeclaration, + ReadViewName, ReaderDeclaration, ReaderDeclarationInput, ReaderIdentity, ReasonCode, + StreamingTextFallback, ValidationDeclaration, VerifiedBlobSource, +}; +pub use signalbox_file_media_runtime::{ + MAX_DECODED_IMAGE_PIXELS as MAX_IMAGE_DECODED_PIXELS, MAX_IMAGE_AXIS, +}; + +const PROVIDER_NAME: &str = "signalbox_image"; +const READER_REVISION: &str = "v1"; +pub(crate) const METADATA_VIEW_NAME: &str = "metadata"; +pub(crate) const MALFORMED_IMAGE_REASON: &str = "malformed_image"; +pub(crate) const SOURCE_TOO_LARGE_REASON: &str = "source_too_large"; +pub(crate) const DIMENSION_LIMIT_EXCEEDED_REASON: &str = "dimension_limit_exceeded"; +pub(crate) const PIXEL_LIMIT_EXCEEDED_REASON: &str = "pixel_limit_exceeded"; + +/// Maximum encoded bytes one image adapter accepts. +// numeric-bound: ceiling - protects worker memory and decode latency from oversized inputs +pub const MAX_IMAGE_SOURCE_BYTES: u64 = 262_144; + +#[derive(Clone, Copy, Debug)] +pub(crate) struct AdapterFormat { + reader_name: &'static str, + media_type: &'static str, + image_format: ImageFormat, + matches_signature: fn(&[u8]) -> bool, +} + +impl AdapterFormat { + const fn reader_name(self) -> &'static str { + self.reader_name + } + + const fn media_type(self) -> &'static str { + self.media_type + } + + const fn image_format(self) -> ImageFormat { + self.image_format + } + + fn matches_signature(self, prefix: &[u8]) -> bool { + (self.matches_signature)(prefix) + } +} + +const ADAPTER_FORMATS: [AdapterFormat; 4] = [ + AdapterFormat { + reader_name: "png", + media_type: "image/png", + image_format: ImageFormat::Png, + matches_signature: |prefix| prefix.starts_with(b"\x89PNG\r\n\x1a\n"), + }, + AdapterFormat { + reader_name: "jpeg", + media_type: "image/jpeg", + image_format: ImageFormat::Jpeg, + matches_signature: |prefix| prefix.starts_with(&[0xff, 0xd8, 0xff]), + }, + AdapterFormat { + reader_name: "webp", + media_type: "image/webp", + image_format: ImageFormat::WebP, + matches_signature: |prefix| { + prefix.starts_with(b"RIFF") && prefix.get(8..12) == Some(b"WEBP".as_slice()) + }, + }, + AdapterFormat { + reader_name: "gif", + media_type: "image/gif", + image_format: ImageFormat::Gif, + matches_signature: |prefix| prefix.starts_with(b"GIF87a") || prefix.starts_with(b"GIF89a"), + }, +]; + +/// Compiled provider for the four version-one image readers. +#[derive(Clone, Copy, Debug, Default)] +pub struct ImageFamilyProvider; + +impl FileMediaProvider for ImageFamilyProvider { + fn declaration(&self) -> FileMediaProviderDeclaration { + match image_family_declaration() { + Ok(declaration) => declaration, + Err(_) => std::process::abort(), + } + } + + fn probe<'a>( + &'a self, + reader: &'a ReaderIdentity, + source: &'a dyn VerifiedBlobSource, + cancellation: &'a dyn signalbox_file_media_runtime::CancellationSignal, + ) -> FileMediaProviderFuture<'a, ProcessorProbeOutput> { + Box::pin(async move { + let format = format_for_reader(reader)?; + adapter::probe(format, source, cancellation) + .await + .map_err(|_| FileMediaProviderFailure::Failed) + }) + } + + fn inspect<'a>( + &'a self, + reader: &'a ReaderIdentity, + request: FileMediaProviderValidationRequest, + source: &'a dyn VerifiedBlobSource, + cancellation: &'a dyn signalbox_file_media_runtime::CancellationSignal, + ) -> FileMediaProviderFuture<'a, ProcessorValidationOutput> { + Box::pin(async move { + let format = format_for_reader(reader)?; + adapter::inspect(format, request, source, cancellation) + .await + .map_err(|_| FileMediaProviderFailure::Failed) + }) + } + + fn read<'a>( + &'a self, + reader: &'a ReaderIdentity, + request: FileMediaProviderReadRequest, + source: &'a dyn VerifiedBlobSource, + cancellation: &'a dyn signalbox_file_media_runtime::CancellationSignal, + ) -> FileMediaProviderFuture<'a, ProcessorReadOutput> { + Box::pin(async move { + let format = format_for_reader(reader)?; + adapter::read(format, request, source, cancellation) + .await + .map_err(|_| FileMediaProviderFailure::Failed) + }) + } +} + +/// Builds the exact declaration registered by the image-family worker. +pub fn image_family_declaration() +-> Result> { + let provider = FileReaderProviderName::try_new(PROVIDER_NAME)?; + let readers = ADAPTER_FORMATS + .into_iter() + .map(|format| reader(&provider, format)) + .collect::, _>>()?; + Ok(FileMediaProviderDeclaration::try_new(provider, readers)?) +} + +fn reader( + provider: &FileReaderProviderName, + format: AdapterFormat, +) -> Result> { + let reasons = [ + MALFORMED_IMAGE_REASON, + SOURCE_TOO_LARGE_REASON, + DIMENSION_LIMIT_EXCEEDED_REASON, + PIXEL_LIMIT_EXCEEDED_REASON, + ] + .into_iter() + .map(ReasonCode::try_new) + .collect::, _>>()?; + Ok(ReaderDeclaration::try_new(ReaderDeclarationInput { + provider: provider.clone(), + reader: FileReaderName::try_new(format.reader_name())?, + revision: FileReaderRevision::try_new(READER_REVISION)?, + media_types: vec![CanonicalMediaType::from_str(format.media_type())?], + probe: ProbeDeclaration::new(ProbeDeclarationInput { + prefix_bytes: 16, + suffix_bytes: 0, + range_count: 0, + cumulative_bytes: 16, + }), + validation: ValidationDeclaration::new(MAX_IMAGE_SOURCE_BYTES, 1), + views: vec![metadata_view()?], + reason_codes: reasons, + streaming_text_fallback: StreamingTextFallback::Disabled, + })?) +} + +fn metadata_view() -> Result> { + Ok(ReadViewDeclaration::try_new( + ReadViewName::try_new(METADATA_VIEW_NAME)?, + String::from("Decodes the primary raster and returns dimensions and channel count."), + CanonicalJsonObjectSchema::try_new(r#"{"additionalProperties":false,"type":"object"}"#)?, + ReadAccessPattern::Streaming { maximum_ranges: 1 }, + ReadViewBounds::Structured { + source_bytes: MAX_IMAGE_SOURCE_BYTES, + output_bytes: 256, + depth: 2, + nodes: 8, + string_bytes: 64, + }, + )?) +} + +fn format_for_reader(reader: &ReaderIdentity) -> Result { + ADAPTER_FORMATS + .into_iter() + .find(|format| format.reader_name() == reader.reader().as_str()) + .ok_or(FileMediaProviderFailure::Failed) +} + +fn options_are_empty(options: &serde_json::Value) -> bool { + options.as_object().is_some_and(serde_json::Map::is_empty) +} diff --git a/crates/file-media-adapters-image/src/source.rs b/crates/file-media-adapters-image/src/source.rs new file mode 100644 index 0000000000..87c537b0a0 --- /dev/null +++ b/crates/file-media-adapters-image/src/source.rs @@ -0,0 +1,40 @@ +use signalbox_file_media_runtime::{CancellationSignal, ProcessorFailure, VerifiedBlobSource}; + +use crate::MAX_IMAGE_SOURCE_BYTES; + +pub(crate) async fn read_complete( + source: &dyn VerifiedBlobSource, + cancellation: &dyn CancellationSignal, + maximum_bytes: u64, +) -> Result>, ProcessorFailure> { + if cancellation.is_cancelled() { + return Err(ProcessorFailure::Cancelled); + } + if source.byte_length().get() > maximum_bytes.min(MAX_IMAGE_SOURCE_BYTES) { + return Ok(None); + } + let bytes = source + .read_range(0, source.byte_length()) + .await + .map_err(|_| ProcessorFailure::Failed)?; + if cancellation.is_cancelled() { + return Err(ProcessorFailure::Cancelled); + } + Ok(Some(bytes)) +} + +pub(crate) async fn read_probe_prefix( + source: &dyn VerifiedBlobSource, + cancellation: &dyn CancellationSignal, +) -> Result, ProcessorFailure> { + if cancellation.is_cancelled() { + return Err(ProcessorFailure::Cancelled); + } + let length = source + .byte_length() + .min(std::num::NonZeroU64::new(16).ok_or(ProcessorFailure::Failed)?); + source + .read_range(0, length) + .await + .map_err(|_| ProcessorFailure::Failed) +} diff --git a/crates/file-media-adapters-image/tests/adapters.rs b/crates/file-media-adapters-image/tests/adapters.rs new file mode 100644 index 0000000000..b02aaad224 --- /dev/null +++ b/crates/file-media-adapters-image/tests/adapters.rs @@ -0,0 +1,274 @@ +mod fixtures; +mod support; + +use std::error::Error; + +use fixtures::FixtureFormat; +use signalbox_file_media_runtime::FileMediaCeilings; +use support::{DirectProcessor, MemorySource}; + +#[tokio::test] +async fn png_detects_validates_and_reads_metadata() -> Result<(), Box> { + let format = FixtureFormat::Png; + let source = MemorySource::new(fixtures::valid(format)?); + let (width, height) = fixtures::valid_dimensions(); + let expected = serde_json::json!({"channels": 4, "height": height, "width": width}); + + let inspection = support::inspect(&source, "image/png").await?; + support::assert_validated_media(inspection, "image/png"); + let result = support::read(&source, "image/png", &DirectProcessor::provider()).await?; + support::assert_structured(result, &expected); + Ok(()) +} + +#[tokio::test] +async fn jpeg_detects_validates_and_reads_metadata() -> Result<(), Box> { + let format = FixtureFormat::Jpeg; + let source = MemorySource::new(fixtures::valid(format)?); + let (width, height) = fixtures::valid_dimensions(); + let expected = serde_json::json!({"channels": 3, "height": height, "width": width}); + + let inspection = support::inspect(&source, "image/jpeg").await?; + support::assert_validated_media(inspection, "image/jpeg"); + let result = support::read(&source, "image/jpeg", &DirectProcessor::provider()).await?; + support::assert_structured(result, &expected); + Ok(()) +} + +#[tokio::test] +async fn webp_detects_validates_and_reads_metadata() -> Result<(), Box> { + let format = FixtureFormat::WebP; + let source = MemorySource::new(fixtures::valid(format)?); + let (width, height) = fixtures::valid_dimensions(); + let expected = serde_json::json!({"channels": 4, "height": height, "width": width}); + + let inspection = support::inspect(&source, "image/webp").await?; + support::assert_validated_media(inspection, "image/webp"); + let result = support::read(&source, "image/webp", &DirectProcessor::provider()).await?; + support::assert_structured(result, &expected); + Ok(()) +} + +#[tokio::test] +async fn gif_detects_validates_and_reads_metadata() -> Result<(), Box> { + let format = FixtureFormat::Gif; + let source = MemorySource::new(fixtures::valid(format)?); + let (width, height) = fixtures::valid_dimensions(); + let expected = serde_json::json!({"channels": 4, "height": height, "width": width}); + + let inspection = support::inspect(&source, "image/gif").await?; + support::assert_validated_media(inspection, "image/gif"); + let result = support::read(&source, "image/gif", &DirectProcessor::provider()).await?; + support::assert_structured(result, &expected); + Ok(()) +} + +#[tokio::test] +async fn truncated_images_are_reported_as_malformed() -> Result<(), Box> { + let png = MemorySource::new(fixtures::truncated(FixtureFormat::Png)?); + let jpeg = MemorySource::new(fixtures::truncated(FixtureFormat::Jpeg)?); + let webp = MemorySource::new(fixtures::truncated(FixtureFormat::WebP)?); + let gif = MemorySource::new(fixtures::truncated(FixtureFormat::Gif)?); + + support::assert_malformed_reason( + support::inspect(&png, "image/png").await?, + "malformed_image", + ); + support::assert_malformed_reason( + support::inspect(&jpeg, "image/jpeg").await?, + "malformed_image", + ); + support::assert_malformed_reason( + support::inspect(&webp, "image/webp").await?, + "malformed_image", + ); + support::assert_malformed_reason( + support::inspect(&gif, "image/gif").await?, + "malformed_image", + ); + Ok(()) +} + +#[tokio::test] +async fn malformed_images_are_reported_as_malformed() -> Result<(), Box> { + let png = MemorySource::new(fixtures::malformed(FixtureFormat::Png)); + let jpeg = MemorySource::new(fixtures::malformed(FixtureFormat::Jpeg)); + let webp = MemorySource::new(fixtures::malformed(FixtureFormat::WebP)); + let gif = MemorySource::new(fixtures::malformed(FixtureFormat::Gif)); + + support::assert_malformed_reason( + support::inspect(&png, "image/png").await?, + "malformed_image", + ); + support::assert_malformed_reason( + support::inspect(&jpeg, "image/jpeg").await?, + "malformed_image", + ); + support::assert_malformed_reason( + support::inspect(&webp, "image/webp").await?, + "malformed_image", + ); + support::assert_malformed_reason( + support::inspect(&gif, "image/gif").await?, + "malformed_image", + ); + Ok(()) +} + +#[tokio::test] +async fn oversized_images_are_reported_as_source_too_large() -> Result<(), Box> { + let png = MemorySource::new(fixtures::oversized(FixtureFormat::Png)?); + let jpeg = MemorySource::new(fixtures::oversized(FixtureFormat::Jpeg)?); + let webp = MemorySource::new(fixtures::oversized(FixtureFormat::WebP)?); + let gif = MemorySource::new(fixtures::oversized(FixtureFormat::Gif)?); + + support::assert_malformed_reason( + support::inspect(&png, "image/png").await?, + "source_too_large", + ); + support::assert_malformed_reason( + support::inspect(&jpeg, "image/jpeg").await?, + "source_too_large", + ); + support::assert_malformed_reason( + support::inspect(&webp, "image/webp").await?, + "source_too_large", + ); + support::assert_malformed_reason( + support::inspect(&gif, "image/gif").await?, + "source_too_large", + ); + Ok(()) +} + +#[tokio::test] +async fn dimension_bombs_are_reported_as_dimension_limit_exceeded() -> Result<(), Box> { + let png = MemorySource::new(fixtures::dimension_bomb(FixtureFormat::Png)?); + let jpeg = MemorySource::new(fixtures::dimension_bomb(FixtureFormat::Jpeg)?); + let webp = MemorySource::new(fixtures::dimension_bomb(FixtureFormat::WebP)?); + let gif = MemorySource::new(fixtures::dimension_bomb(FixtureFormat::Gif)?); + + support::assert_malformed_reason( + support::inspect(&png, "image/png").await?, + "dimension_limit_exceeded", + ); + support::assert_malformed_reason( + support::inspect(&jpeg, "image/jpeg").await?, + "dimension_limit_exceeded", + ); + support::assert_malformed_reason( + support::inspect(&webp, "image/webp").await?, + "dimension_limit_exceeded", + ); + support::assert_malformed_reason( + support::inspect(&gif, "image/gif").await?, + "dimension_limit_exceeded", + ); + Ok(()) +} + +#[tokio::test] +async fn pixel_bombs_are_reported_as_pixel_limit_exceeded() -> Result<(), Box> { + let png = MemorySource::new(fixtures::pixel_bomb(FixtureFormat::Png)?); + let jpeg = MemorySource::new(fixtures::pixel_bomb(FixtureFormat::Jpeg)?); + let webp = MemorySource::new(fixtures::pixel_bomb(FixtureFormat::WebP)?); + let gif = MemorySource::new(fixtures::pixel_bomb(FixtureFormat::Gif)?); + + support::assert_malformed_reason( + support::inspect(&png, "image/png").await?, + "pixel_limit_exceeded", + ); + support::assert_malformed_reason( + support::inspect(&jpeg, "image/jpeg").await?, + "pixel_limit_exceeded", + ); + support::assert_malformed_reason( + support::inspect(&webp, "image/webp").await?, + "pixel_limit_exceeded", + ); + support::assert_malformed_reason( + support::inspect(&gif, "image/gif").await?, + "pixel_limit_exceeded", + ); + Ok(()) +} + +#[tokio::test] +async fn lowered_image_axis_is_enforced_during_metadata_validation() -> Result<(), Box> { + let format = FixtureFormat::Png; + let source = MemorySource::new(fixtures::valid(format)?); + let ceilings = FileMediaCeilings { + image_axis: 2, + ..FileMediaCeilings::version_one() + }; + + let inspection = support::inspect_with_ceilings(&source, format.media_type(), ceilings).await?; + support::assert_malformed_reason(inspection, "dimension_limit_exceeded"); + Ok(()) +} + +#[tokio::test] +async fn lowered_decoded_pixels_are_enforced_during_metadata_validation() +-> Result<(), Box> { + let format = FixtureFormat::Png; + let source = MemorySource::new(fixtures::valid(format)?); + let ceilings = FileMediaCeilings { + decoded_image_pixels: 5, + ..FileMediaCeilings::version_one() + }; + + let inspection = support::inspect_with_ceilings(&source, format.media_type(), ceilings).await?; + support::assert_malformed_reason(inspection, "pixel_limit_exceeded"); + Ok(()) +} + +#[tokio::test] +async fn lowered_validation_source_bytes_return_source_too_large() -> Result<(), Box> { + let format = FixtureFormat::Png; + let source = MemorySource::new(fixtures::valid(format)?); + let ceilings = FileMediaCeilings { + validation_source_bytes: 16, + ..FileMediaCeilings::version_one() + }; + + let inspection = support::inspect_with_ceilings(&source, format.media_type(), ceilings).await?; + support::assert_malformed_reason(inspection, "source_too_large"); + Ok(()) +} + +#[tokio::test] +async fn registry_sanitizer_keeps_injection_shaped_metadata_as_data() -> Result<(), Box> +{ + let format = FixtureFormat::Png; + let source = MemorySource::new(fixtures::valid(format)?); + let expected = serde_json::json!({ + "path":"../../etc/passwd", + "text":"" + }); + let decoder_output = serde_json::to_string(&expected)?; + + let result = support::read( + &source, + format.media_type(), + &DirectProcessor::injecting(decoder_output), + ) + .await?; + support::assert_structured(result, &expected); + Ok(()) +} + +#[tokio::test] +async fn registry_sanitizer_rejects_nul_bearing_decoder_output() -> Result<(), Box> { + let format = FixtureFormat::Png; + let source = MemorySource::new(fixtures::valid(format)?); + let decoder_output = String::from("{\"text\":\"prefix\0suffix\"}"); + + let result = support::read( + &source, + format.media_type(), + &DirectProcessor::injecting(decoder_output), + ) + .await; + support::assert_processor_failed(result); + Ok(()) +} diff --git a/crates/file-media-adapters-image/tests/fixtures/mod.rs b/crates/file-media-adapters-image/tests/fixtures/mod.rs new file mode 100644 index 0000000000..e5c970108f --- /dev/null +++ b/crates/file-media-adapters-image/tests/fixtures/mod.rs @@ -0,0 +1,130 @@ +use std::{error::Error, io::Cursor}; + +use image::{DynamicImage, ImageBuffer, ImageError, ImageFormat, Rgba}; + +const VALID_WIDTH: u32 = 3; +const VALID_HEIGHT: u32 = 2; + +#[derive(Clone, Copy)] +pub(crate) enum FixtureFormat { + Png, + Jpeg, + WebP, + Gif, +} + +impl FixtureFormat { + pub(crate) const fn media_type(self) -> &'static str { + match self { + Self::Png => "image/png", + Self::Jpeg => "image/jpeg", + Self::WebP => "image/webp", + Self::Gif => "image/gif", + } + } + + const fn image_format(self) -> ImageFormat { + match self { + Self::Png => ImageFormat::Png, + Self::Jpeg => ImageFormat::Jpeg, + Self::WebP => ImageFormat::WebP, + Self::Gif => ImageFormat::Gif, + } + } +} + +pub(crate) fn valid(format: FixtureFormat) -> Result, ImageError> { + encode(format, VALID_WIDTH, VALID_HEIGHT) +} + +pub(crate) const fn valid_dimensions() -> (u32, u32) { + (VALID_WIDTH, VALID_HEIGHT) +} + +pub(crate) fn truncated(format: FixtureFormat) -> Result, ImageError> { + let mut bytes = valid(format)?; + bytes.truncate(bytes.len() / 2); + Ok(bytes) +} + +pub(crate) fn malformed(format: FixtureFormat) -> Vec { + match format { + FixtureFormat::Png => b"\x89PNG\r\n\x1a\nmalformed".to_vec(), + FixtureFormat::Jpeg => vec![0xff, 0xd8, 0xff, 0x00], + FixtureFormat::WebP => b"RIFF\x08\x00\x00\x00WEBPbad!".to_vec(), + FixtureFormat::Gif => b"GIF89a\x01\x00\x01\x00\x00\x00\x00\x01".to_vec(), + } +} + +pub(crate) fn oversized(format: FixtureFormat) -> Result, ImageError> { + let mut bytes = valid(format)?; + bytes.resize( + signalbox_file_media_adapters_image::MAX_IMAGE_SOURCE_BYTES as usize + 1, + 0, + ); + Ok(bytes) +} + +pub(crate) fn dimension_bomb(format: FixtureFormat) -> Result, ImageError> { + encode( + format, + signalbox_file_media_adapters_image::MAX_IMAGE_AXIS + 1, + 1, + ) +} + +pub(crate) fn pixel_bomb(format: FixtureFormat) -> Result, Box> { + const BOMB_AXIS: u32 = 4_097; + match format { + FixtureFormat::Png => { + let mut bytes = valid(format)?; + bytes[16..20].copy_from_slice(&BOMB_AXIS.to_be_bytes()); + bytes[20..24].copy_from_slice(&BOMB_AXIS.to_be_bytes()); + let checksum = crc32fast::hash(&bytes[12..29]); + bytes[29..33].copy_from_slice(&checksum.to_be_bytes()); + Ok(bytes) + } + FixtureFormat::Jpeg => { + let mut bytes = valid(format)?; + let marker = bytes + .windows(2) + .position(|window| window == [0xff, 0xc0]) + .ok_or("synthetic JPEG has no baseline frame header")?; + bytes[marker + 5..marker + 7].copy_from_slice(&BOMB_AXIS.to_be_bytes()[2..]); + bytes[marker + 7..marker + 9].copy_from_slice(&BOMB_AXIS.to_be_bytes()[2..]); + Ok(bytes) + } + FixtureFormat::WebP => { + let mut bytes = valid(format)?; + let chunk = bytes + .windows(4) + .position(|window| window == b"VP8L") + .ok_or("synthetic WebP is not lossless")?; + let packed = (BOMB_AXIS - 1) | ((BOMB_AXIS - 1) << 14); + let packed_bytes = packed.to_le_bytes(); + bytes[chunk + 9..chunk + 13].copy_from_slice(&packed_bytes); + Ok(bytes) + } + FixtureFormat::Gif => { + let mut bytes = valid(format)?; + bytes[6..8].copy_from_slice(&(BOMB_AXIS as u16).to_le_bytes()); + bytes[8..10].copy_from_slice(&(BOMB_AXIS as u16).to_le_bytes()); + Ok(bytes) + } + } +} + +fn encode(format: FixtureFormat, width: u32, height: u32) -> Result, ImageError> { + let pixels = ImageBuffer::from_fn(width, height, |x, y| { + Rgba([ + x.to_le_bytes()[0], + y.to_le_bytes()[0], + x.wrapping_add(y).to_le_bytes()[0], + 255, + ]) + }); + let image = DynamicImage::ImageRgba8(pixels); + let mut encoded = Cursor::new(Vec::new()); + image.write_to(&mut encoded, format.image_format())?; + Ok(encoded.into_inner()) +} diff --git a/crates/file-media-adapters-image/tests/support/mod.rs b/crates/file-media-adapters-image/tests/support/mod.rs new file mode 100644 index 0000000000..3d7de6a785 --- /dev/null +++ b/crates/file-media-adapters-image/tests/support/mod.rs @@ -0,0 +1,251 @@ +use std::{error::Error, num::NonZeroU64, sync::Arc}; + +use signalbox_file_media_adapters_image::{ImageFamilyProvider, image_family_declaration}; +use signalbox_file_media_runtime::{ + AttachmentKind, CancellationSignal, DeclaredMediaType, FileDigest, FileInspection, + FileMediaCeilings, FileMediaFailure, FileMediaProcessor, FileMediaProcessorFuture, + FileMediaProvider, FileMediaProviderReadRequest, FileMediaProviderValidationRequest, + FileMediaRegistry, FileReadRequest, FileReadResult, FileUse, InspectionRequest, NeverCancelled, + ProcessorBoundaryFailure, ProcessorFailure, ProcessorIsolation, ProcessorProbeOutput, + ProcessorReadOutput, ProcessorValidationOutput, ReadViewName, ReaderIdentity, SourceReadError, + SourceReadFuture, VerifiedBlobSource, +}; + +pub(crate) struct MemorySource { + bytes: Arc<[u8]>, +} + +impl MemorySource { + pub(crate) fn new(bytes: Vec) -> Self { + Self { + bytes: Arc::from(bytes), + } + } + + pub(crate) fn file_use(&self, media_type: &str) -> Result> { + Ok(FileUse::new( + self.digest(), + self.byte_length(), + AttachmentKind::Image, + DeclaredMediaType::try_new(media_type)?, + None, + )) + } +} + +impl VerifiedBlobSource for MemorySource { + fn digest(&self) -> FileDigest { + FileDigest::from_bytes([8; 32]) + } + + fn byte_length(&self) -> NonZeroU64 { + NonZeroU64::new(self.bytes.len() as u64).unwrap_or(NonZeroU64::MIN) + } + + fn read_range(&self, offset: u64, length: NonZeroU64) -> SourceReadFuture<'_> { + Box::pin(async move { + let start = usize::try_from(offset).map_err(|_| SourceReadError::RangeOutOfBounds)?; + let requested = + usize::try_from(length.get()).map_err(|_| SourceReadError::RangeOutOfBounds)?; + let end = start + .checked_add(requested) + .ok_or(SourceReadError::RangeOutOfBounds)?; + self.bytes + .get(start..end) + .map(<[u8]>::to_vec) + .ok_or(SourceReadError::RangeOutOfBounds) + }) + } +} + +#[derive(Clone, Debug)] +enum ReadBehavior { + Provider, + InjectedStructured(String), +} + +pub(crate) struct DirectProcessor { + provider: ImageFamilyProvider, + read_behavior: ReadBehavior, +} + +impl DirectProcessor { + pub(crate) fn provider() -> Self { + Self { + provider: ImageFamilyProvider, + read_behavior: ReadBehavior::Provider, + } + } + + pub(crate) fn injecting(body_json: String) -> Self { + Self { + provider: ImageFamilyProvider, + read_behavior: ReadBehavior::InjectedStructured(body_json), + } + } +} + +impl FileMediaProcessor for DirectProcessor { + fn probe<'a>( + &'a self, + reader: &'a ReaderIdentity, + source: &'a dyn VerifiedBlobSource, + cancellation: &'a dyn CancellationSignal, + ) -> FileMediaProcessorFuture<'a, ProcessorProbeOutput> { + let future = self.provider.probe(reader, source, cancellation); + Box::pin(async move { + future + .await + .map_err(|_| ProcessorBoundaryFailure::Processor(ProcessorFailure::Failed)) + }) + } + + fn validate<'a>( + &'a self, + reader: &'a ReaderIdentity, + request: FileMediaProviderValidationRequest, + source: &'a dyn VerifiedBlobSource, + cancellation: &'a dyn CancellationSignal, + ) -> FileMediaProcessorFuture<'a, ProcessorValidationOutput> { + let future = self.provider.inspect(reader, request, source, cancellation); + Box::pin(async move { + future + .await + .map_err(|_| ProcessorBoundaryFailure::Processor(ProcessorFailure::Failed)) + }) + } + + fn read<'a>( + &'a self, + reader: &'a ReaderIdentity, + request: FileMediaProviderReadRequest, + source: &'a dyn VerifiedBlobSource, + cancellation: &'a dyn CancellationSignal, + ) -> FileMediaProcessorFuture<'a, ProcessorReadOutput> { + match &self.read_behavior { + ReadBehavior::Provider => { + let future = self.provider.read(reader, request, source, cancellation); + Box::pin(async move { + future + .await + .map_err(|_| ProcessorBoundaryFailure::Processor(ProcessorFailure::Failed)) + }) + } + ReadBehavior::InjectedStructured(body_json) => { + let body_json = body_json.clone(); + Box::pin(async move { + Ok(ProcessorReadOutput::Structured { + body_json, + truncated: false, + cursor: None, + }) + }) + } + } + } +} + +fn registry_with(ceilings: FileMediaCeilings) -> Result> { + Ok(FileMediaRegistry::try_new( + vec![image_family_declaration().map_err(|error| error.to_string())?], + ceilings, + ProcessorIsolation::Available, + )?) +} + +fn registry() -> Result> { + registry_with(FileMediaCeilings::version_one()) +} + +pub(crate) async fn inspect( + source: &MemorySource, + media_type: &str, +) -> Result> { + Ok(registry()? + .inspect( + &DirectProcessor::provider(), + InspectionRequest { + source: source.file_use(media_type)?, + visible_part: None, + }, + source, + &NeverCancelled, + ) + .await?) +} + +pub(crate) async fn inspect_with_ceilings( + source: &MemorySource, + media_type: &str, + ceilings: FileMediaCeilings, +) -> Result> { + Ok(registry_with(ceilings)? + .inspect( + &DirectProcessor::provider(), + InspectionRequest { + source: source.file_use(media_type)?, + visible_part: None, + }, + source, + &NeverCancelled, + ) + .await?) +} + +pub(crate) async fn read( + source: &MemorySource, + media_type: &str, + processor: &DirectProcessor, +) -> Result { + let source_use = source + .file_use(media_type) + .map_err(|_| FileMediaFailure::ProcessorFailed)?; + let view = ReadViewName::try_new("metadata").map_err(|_| FileMediaFailure::ProcessorFailed)?; + registry() + .map_err(|_| FileMediaFailure::ProcessorFailed)? + .read( + processor, + FileReadRequest { + inspection: InspectionRequest { + source: source_use, + visible_part: None, + }, + view, + input: signalbox_file_media_runtime::FileReadInput::Initial { + options: serde_json::json!({}), + }, + }, + source, + &NeverCancelled, + ) + .await +} + +#[track_caller] +pub(crate) fn assert_validated_media(inspection: FileInspection, expected: &str) { + assert!(matches!(inspection, FileInspection::Validated(_))); + if let FileInspection::Validated(validated) = inspection { + assert_eq!(validated.detected_media_type().as_str(), expected); + } +} + +#[track_caller] +pub(crate) fn assert_malformed_reason(inspection: FileInspection, expected: &str) { + assert!(matches!(inspection, FileInspection::Malformed { .. })); + if let FileInspection::Malformed { reason_code, .. } = inspection { + assert_eq!(reason_code.as_str(), expected); + } +} + +#[track_caller] +pub(crate) fn assert_structured(result: FileReadResult, expected: &serde_json::Value) { + assert!(matches!(result, FileReadResult::Structured { .. })); + if let FileReadResult::Structured { body, .. } = result { + assert_eq!(&body, expected); + } +} + +#[track_caller] +pub(crate) fn assert_processor_failed(result: Result) { + assert_eq!(result, Err(FileMediaFailure::ProcessorFailed)); +} diff --git a/crates/file-media-adapters-text/Cargo.toml b/crates/file-media-adapters-text/Cargo.toml new file mode 100644 index 0000000000..710782a8de --- /dev/null +++ b/crates/file-media-adapters-text/Cargo.toml @@ -0,0 +1,27 @@ +[package] +name = "signalbox-file-media-adapters-text" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +repository.workspace = true +publish.workspace = true + +[[bin]] +name = "signalbox-file-media-text-worker" +path = "src/bin/signalbox-file-media-text-worker.rs" + +[dependencies] +csv = "1.3.1" +serde = "1.0.219" +serde_json = { version = "1.0.140", features = ["arbitrary_precision", "unbounded_depth"] } +serde_stacker = "0.1.14" +signalbox-file-media-processor-runtime = { path = "../file-media-processor-runtime" } +signalbox-file-media-runtime = { path = "../file-media-runtime" } +tokio = { version = "1.53.0", default-features = false, features = [ + "macros", + "rt", +] } + +[lints] +workspace = true diff --git a/crates/file-media-adapters-text/src/bin/signalbox-file-media-text-worker.rs b/crates/file-media-adapters-text/src/bin/signalbox-file-media-text-worker.rs new file mode 100644 index 0000000000..7f1e5b5a87 --- /dev/null +++ b/crates/file-media-adapters-text/src/bin/signalbox-file-media-text-worker.rs @@ -0,0 +1,11 @@ +use std::error::Error; + +use signalbox_file_media_adapters_text::TextFamilyProvider; +use signalbox_file_media_processor_runtime::{WorkerCatalog, serve_one}; + +#[tokio::main(flavor = "current_thread")] +async fn main() -> Result<(), Box> { + let catalog = WorkerCatalog::try_new(vec![Box::new(TextFamilyProvider)])?; + serve_one(&catalog).await?; + Ok(()) +} diff --git a/crates/file-media-adapters-text/src/csv_adapter.rs b/crates/file-media-adapters-text/src/csv_adapter.rs new file mode 100644 index 0000000000..1e0111a7bc --- /dev/null +++ b/crates/file-media-adapters-text/src/csv_adapter.rs @@ -0,0 +1,399 @@ +use signalbox_file_media_runtime::{ + CancellationSignal, FileMediaProviderReadRequest, FileMediaProviderValidationRequest, + MAX_OBSERVED_CONTAINER_ENTRIES, ProbeStrength, ProcessorFailure, ProcessorProbeOutput, + ProcessorReadOutput, ProcessorValidationOutput, ValidationEvidence, VerifiedBlobSource, +}; + +use crate::{ + CSV_MEDIA_TYPE, MAX_TEXT_FAMILY_BYTES, PROBE_PREFIX_BYTES, STRUCTURED_VIEW_NAME, + json_adapter::{self, ProbeExtent}, + read_input_is_empty, source, +}; + +// Hard safety ceiling preventing one record from causing runaway allocation. +const MAX_COLUMNS: usize = 256; + +pub(crate) async fn probe( + source: &dyn VerifiedBlobSource, + cancellation: &dyn CancellationSignal, +) -> Result { + let prefix = source::read_probe_prefix(source, cancellation).await?; + let extent = if source.byte_length().get() <= prefix.len() as u64 { + ProbeExtent::CompleteSource + } else { + ProbeExtent::TruncatedPrefix + }; + let json_suppresses_csv = matches!(extent, ProbeExtent::CompleteSource) + && json_adapter::is_complete_json_document(&prefix); + let candidate = !json_suppresses_csv + && source::probe_utf8_within(&prefix, extent) + .is_some_and(|text| has_record_structure(text, extent)); + if candidate { + Ok(ProcessorProbeOutput::Candidate { + media_type: String::from(CSV_MEDIA_TYPE), + strength: match extent { + ProbeExtent::CompleteSource => ProbeStrength::StructuralCandidate, + ProbeExtent::TruncatedPrefix => ProbeStrength::ProvisionalStructuralCandidate, + }, + }) + } else { + Ok(ProcessorProbeOutput::NoMatch) + } +} + +pub(crate) async fn inspect( + request: FileMediaProviderValidationRequest, + source: &dyn VerifiedBlobSource, + cancellation: &dyn CancellationSignal, +) -> Result { + if request.media_type.as_str() != CSV_MEDIA_TYPE { + return Err(ProcessorFailure::Protocol); + } + let Some(bytes) = + source::read_complete(source, cancellation, request.maximum_source_bytes).await? + else { + if matches!( + request.evidence, + ValidationEvidence::DeclaredCandidateStructurallyValidated + ) { + let prefix = + source::read_validation_prefix(source, cancellation, request.maximum_source_bytes) + .await?; + if source::probe_utf8(&prefix).is_some_and(has_declared_record_structure) { + return Ok(malformed("source_too_large")); + } + } + return Ok(validation_failure(request.evidence, "source_too_large")); + }; + let text = match source::checked_utf8(bytes) { + Ok(text) => text, + Err(reason) => return Ok(validation_failure(request.evidence, reason)), + }; + let table = match parse_table(&text, MAX_OBSERVED_CONTAINER_ENTRIES) { + Ok(table) => table, + Err(reason) => { + if matches!(request.evidence, ValidationEvidence::StructuralValidation) + && initial_probe_was_provisional(text.as_bytes()) + { + return Ok(ProcessorValidationOutput::NoMatch); + } + let declared_csv_shape = matches!( + request.evidence, + ValidationEvidence::DeclaredCandidateStructurallyValidated + ) && (has_declared_record_evidence(&text) + || reason == "column_limit_exceeded" && is_header_only_csv(&text)); + if declared_csv_shape { + return Ok(malformed(reason)); + } + return Ok(validation_failure(request.evidence, reason)); + } + }; + Ok(ProcessorValidationOutput::Validated { + media_type: String::from(CSV_MEDIA_TYPE), + evidence: request.evidence, + metadata_json: serde_json::json!({ + "columns": table.headers.len(), + "rows": table.rows.len() + }) + .to_string(), + }) +} + +pub(crate) async fn read( + request: FileMediaProviderReadRequest, + source: &dyn VerifiedBlobSource, + cancellation: &dyn CancellationSignal, +) -> Result { + if request.view.as_str() != STRUCTURED_VIEW_NAME || !read_input_is_empty(&request.input) { + return Ok(ProcessorReadOutput::InvalidViewArguments); + } + let Some(bytes) = source::read_complete(source, cancellation, MAX_TEXT_FAMILY_BYTES).await? + else { + return Ok(ProcessorReadOutput::SourceTooLarge { + maximum_bytes: MAX_TEXT_FAMILY_BYTES, + }); + }; + let text = source::checked_utf8(bytes).map_err(|_| ProcessorFailure::Failed)?; + if request.maximum_container_entries < 2 { + return Ok(ProcessorReadOutput::ExpansionLimitExceeded { + limit_kind: String::from("container_entry_limit_exceeded"), + }); + } + let table = match parse_table(&text, request.maximum_container_entries) { + Ok(table) => table, + Err("row_limit_exceeded") | Err("column_limit_exceeded") => { + return Ok(ProcessorReadOutput::ExpansionLimitExceeded { + limit_kind: String::from("container_entry_limit_exceeded"), + }); + } + Err(_) => return Err(ProcessorFailure::Failed), + }; + let body_json = serde_json::to_string(&serde_json::json!({ + "headers": table.headers, + "rows": table.rows + })) + .map_err(|_| ProcessorFailure::Failed)?; + if body_json.len() > MAX_TEXT_FAMILY_BYTES as usize { + return Ok(ProcessorReadOutput::OutputUnitTooLarge); + } + Ok(ProcessorReadOutput::Structured { + body_json, + truncated: false, + cursor: None, + }) +} + +struct CsvTable { + headers: Vec, + rows: Vec>, +} + +fn parse_table(text: &str, maximum_container_entries: u64) -> Result { + if !quotes_are_well_formed(text) || has_blank_record(text) { + return Err("malformed_csv"); + } + let mut reader = csv::ReaderBuilder::new() + .flexible(false) + .from_reader(text.as_bytes()); + let headers = reader + .headers() + .map_err(|_| "malformed_csv")? + .iter() + .map(String::from) + .collect::>(); + if headers.is_empty() { + return Err("malformed_csv"); + } + if headers.len() > MAX_COLUMNS || headers.len() as u64 > maximum_container_entries { + return Err("column_limit_exceeded"); + } + let mut rows = Vec::new(); + for record in reader.records() { + if rows.len() as u64 == maximum_container_entries { + return Err("row_limit_exceeded"); + } + let record = record.map_err(|_| "malformed_csv")?; + if record.len() as u64 > maximum_container_entries { + return Err("column_limit_exceeded"); + } + rows.push(record.iter().map(String::from).collect()); + } + if rows.is_empty() && !text.bytes().any(|byte| matches!(byte, b'\r' | b'\n')) { + return Err("malformed_csv"); + } + Ok(CsvTable { headers, rows }) +} + +pub(crate) fn has_record_structure(text: &str, extent: ProbeExtent) -> bool { + let evidence = match extent { + ProbeExtent::CompleteSource => text, + ProbeExtent::TruncatedPrefix => { + let Some(evidence) = first_two_strict_records(text, extent) else { + return false; + }; + evidence + } + }; + if !quotes_are_well_formed(evidence) || has_blank_record(evidence) { + return false; + } + let mut reader = csv::ReaderBuilder::new() + .has_headers(false) + .flexible(false) + .from_reader(evidence.as_bytes()); + let mut records = reader.records(); + let Some(Ok(first)) = records.next() else { + return false; + }; + if first.len() < 2 { + return false; + } + let Some(Ok(second)) = records.next() else { + return false; + }; + second.len() == first.len() + && records.all(|record| record.is_ok_and(|record| record.len() == first.len())) +} + +fn initial_probe_was_provisional(bytes: &[u8]) -> bool { + usize::try_from(PROBE_PREFIX_BYTES) + .ok() + .and_then(|length| bytes.get(..length)) + .and_then(source::probe_utf8) + .is_some_and(|text| has_record_structure(text, ProbeExtent::TruncatedPrefix)) +} + +fn has_declared_record_structure(text: &str) -> bool { + let Some(evidence) = first_two_strict_records(text, ProbeExtent::TruncatedPrefix) else { + return false; + }; + if !quotes_are_well_formed(evidence) || has_blank_record(evidence) { + return false; + } + let mut reader = csv::ReaderBuilder::new() + .has_headers(false) + .flexible(false) + .from_reader(evidence.as_bytes()); + let mut records = reader.records(); + let Some(Ok(first)) = records.next() else { + return false; + }; + let Some(Ok(second)) = records.next() else { + return false; + }; + !first.is_empty() && second.len() == first.len() +} + +fn has_declared_record_evidence(text: &str) -> bool { + if has_declared_record_structure(text) { + return true; + } + let Some(first_end) = text.find(['\r', '\n']) else { + return false; + }; + if first_end == 0 { + return false; + } + let remainder = text[first_end..].trim_start_matches(['\r', '\n']); + !remainder.is_empty() +} + +fn is_header_only_csv(text: &str) -> bool { + let Some(first_end) = text.find(['\r', '\n']) else { + return false; + }; + first_end > 0 + && text[first_end..].trim_matches(['\r', '\n']).is_empty() + && quotes_are_well_formed(text) + && !has_blank_record(text) +} + +fn first_two_strict_records(text: &str, extent: ProbeExtent) -> Option<&str> { + let mut state = QuoteState::FieldStart; + let mut completed_records = 0_u8; + let mut bytes = text.as_bytes().iter().copied().enumerate().peekable(); + while let Some((index, byte)) = bytes.next() { + state = match (state, byte) { + (QuoteState::FieldStart, b'"') => QuoteState::Quoted, + (QuoteState::FieldStart, b',') => QuoteState::FieldStart, + (QuoteState::FieldStart, b'\r' | b'\n') + | (QuoteState::Unquoted, b'\r' | b'\n') + | (QuoteState::AfterQuote, b'\r' | b'\n') => { + completed_records = completed_records.saturating_add(1); + let mut end = index + 1; + if byte == b'\r' && bytes.peek().is_some_and(|(_, next)| *next == b'\n') { + let _ = bytes.next(); + end += 1; + } + if completed_records == 2 { + return text.get(..end); + } + QuoteState::FieldStart + } + (QuoteState::FieldStart, _) => QuoteState::Unquoted, + (QuoteState::Unquoted, b'"') => return None, + (QuoteState::Unquoted, b',') => QuoteState::FieldStart, + (QuoteState::Unquoted, _) => QuoteState::Unquoted, + (QuoteState::Quoted, b'"') if bytes.peek().is_some_and(|(_, next)| *next == b'"') => { + let _ = bytes.next(); + QuoteState::Quoted + } + (QuoteState::Quoted, b'"') => QuoteState::AfterQuote, + (QuoteState::Quoted, _) => QuoteState::Quoted, + (QuoteState::AfterQuote, b',') => QuoteState::FieldStart, + (QuoteState::AfterQuote, _) => return None, + }; + } + if matches!(extent, ProbeExtent::CompleteSource) + && completed_records == 1 + && !matches!(state, QuoteState::Quoted) + { + Some(text) + } else { + None + } +} + +#[derive(Clone, Copy)] +enum QuoteState { + FieldStart, + Unquoted, + Quoted, + AfterQuote, +} + +fn quotes_are_well_formed(text: &str) -> bool { + let mut state = QuoteState::FieldStart; + let mut bytes = text.bytes().peekable(); + while let Some(byte) = bytes.next() { + state = match (state, byte) { + (QuoteState::FieldStart, b'"') => QuoteState::Quoted, + (QuoteState::FieldStart, b',' | b'\r' | b'\n') => QuoteState::FieldStart, + (QuoteState::FieldStart, _) => QuoteState::Unquoted, + (QuoteState::Unquoted, b'"') => return false, + (QuoteState::Unquoted, b',' | b'\r' | b'\n') => QuoteState::FieldStart, + (QuoteState::Unquoted, _) => QuoteState::Unquoted, + (QuoteState::Quoted, b'"') if bytes.peek() == Some(&b'"') => { + let _ = bytes.next(); + QuoteState::Quoted + } + (QuoteState::Quoted, b'"') => QuoteState::AfterQuote, + (QuoteState::Quoted, _) => QuoteState::Quoted, + (QuoteState::AfterQuote, b',' | b'\r' | b'\n') => QuoteState::FieldStart, + (QuoteState::AfterQuote, _) => return false, + }; + } + !matches!(state, QuoteState::Quoted) +} + +fn has_blank_record(text: &str) -> bool { + let mut in_quotes = false; + let mut record_has_content = false; + let mut bytes = text.bytes().peekable(); + while let Some(byte) = bytes.next() { + if in_quotes { + if byte == b'"' && bytes.peek() == Some(&b'"') { + let _ = bytes.next(); + } else if byte == b'"' { + in_quotes = false; + } + record_has_content = true; + } else { + match byte { + b'"' => { + in_quotes = true; + record_has_content = true; + } + b'\r' | b'\n' => { + if byte == b'\r' && bytes.peek() == Some(&b'\n') { + let _ = bytes.next(); + } + if !record_has_content { + return true; + } + record_has_content = false; + } + _ => record_has_content = true, + } + } + } + false +} + +fn malformed(reason: &str) -> ProcessorValidationOutput { + ProcessorValidationOutput::Malformed { + media_type: String::from(CSV_MEDIA_TYPE), + reason_code: String::from(reason), + } +} + +fn validation_failure(evidence: ValidationEvidence, reason: &str) -> ProcessorValidationOutput { + match evidence { + ValidationEvidence::DeclaredCandidateStructurallyValidated => { + ProcessorValidationOutput::NoMatch + } + ValidationEvidence::StrongSignature + | ValidationEvidence::StructuralValidation + | ValidationEvidence::StreamingTextValidation => malformed(reason), + } +} diff --git a/crates/file-media-adapters-text/src/json_adapter.rs b/crates/file-media-adapters-text/src/json_adapter.rs new file mode 100644 index 0000000000..254a026197 --- /dev/null +++ b/crates/file-media-adapters-text/src/json_adapter.rs @@ -0,0 +1,477 @@ +use std::{collections::HashSet, fmt}; + +use serde::{ + Deserialize, + de::{DeserializeSeed, MapAccess, SeqAccess, Visitor}, +}; +use signalbox_file_media_runtime::{ + CancellationSignal, FileMediaProviderReadRequest, FileMediaProviderValidationRequest, + JsonParseLimits, MAX_STRUCTURED_DEPTH, ProbeStrength, ProcessorFailure, ProcessorProbeOutput, + ProcessorReadOutput, ProcessorValidationOutput, ValidationEvidence, VerifiedBlobSource, + parse_json_without_duplicate_members_bounded, +}; + +use crate::{ + JSON_MEDIA_TYPE, MAX_TEXT_FAMILY_BYTES, PROBE_PREFIX_BYTES, STRUCTURED_VIEW_NAME, + read_input_is_empty, source, +}; + +pub(crate) async fn probe( + source: &dyn VerifiedBlobSource, + cancellation: &dyn CancellationSignal, +) -> Result { + let prefix = source::read_probe_prefix(source, cancellation).await?; + let extent = if source.byte_length().get() <= prefix.len() as u64 { + ProbeExtent::CompleteSource + } else { + ProbeExtent::TruncatedPrefix + }; + let candidate = has_json_structure(&prefix, extent); + if candidate { + let strength = if matches!(extent, ProbeExtent::TruncatedPrefix) + && is_complete_json_probe_document(&prefix) + { + ProbeStrength::ProvisionalStructuralCandidate + } else { + ProbeStrength::StructuralCandidate + }; + Ok(ProcessorProbeOutput::Candidate { + media_type: String::from(JSON_MEDIA_TYPE), + strength, + }) + } else { + Ok(ProcessorProbeOutput::NoMatch) + } +} + +#[derive(Clone, Copy)] +pub(crate) enum ProbeExtent { + CompleteSource, + TruncatedPrefix, +} + +pub(crate) fn has_json_structure(prefix: &[u8], extent: ProbeExtent) -> bool { + has_raw_json_structure(prefix, extent) +} + +pub(crate) fn has_raw_json_structure(prefix: &[u8], extent: ProbeExtent) -> bool { + let prefix = trim_ascii_start(prefix); + if !matches!(prefix.first(), Some(b'{' | b'[')) { + return false; + } + let Some(text) = source::probe_utf8_within(prefix, extent) else { + return false; + }; + match extent { + ProbeExtent::CompleteSource => match validate_json(text) { + Ok(()) => true, + Err(error) => error.is_eof(), + }, + ProbeExtent::TruncatedPrefix => incomplete_json_prefix(text), + } +} + +fn incomplete_json_prefix(text: &str) -> bool { + let mut deserializer = serde_json::Deserializer::from_str(text); + deserializer.disable_recursion_limit(); + match serde::de::IgnoredAny::deserialize(serde_stacker::Deserializer::new(&mut deserializer)) { + Ok(_) => deserializer.end().is_ok(), + Err(error) => error.is_eof(), + } +} + +fn trim_ascii_start(bytes: &[u8]) -> &[u8] { + let first = bytes + .iter() + .position(|byte| !is_json_whitespace(*byte)) + .unwrap_or(bytes.len()); + &bytes[first..] +} + +const fn is_json_whitespace(byte: u8) -> bool { + matches!(byte, b' ' | b'\t' | b'\r' | b'\n') +} + +pub(crate) async fn inspect( + request: FileMediaProviderValidationRequest, + source: &dyn VerifiedBlobSource, + cancellation: &dyn CancellationSignal, +) -> Result { + if request.media_type.as_str() != JSON_MEDIA_TYPE { + return Err(ProcessorFailure::Protocol); + } + let Some(bytes) = + source::read_complete(source, cancellation, request.maximum_source_bytes).await? + else { + let declared_candidate = match request.evidence { + ValidationEvidence::DeclaredCandidateStructurallyValidated => true, + ValidationEvidence::StrongSignature + | ValidationEvidence::StructuralValidation + | ValidationEvidence::StreamingTextValidation => false, + }; + if declared_candidate { + let prefix = + source::read_validation_prefix(source, cancellation, request.maximum_source_bytes) + .await?; + if has_declared_json_prefix(&prefix) { + return Ok(malformed("source_too_large")); + } + } + return Ok(validation_failure(request.evidence, "source_too_large")); + }; + let text = match source::checked_utf8(bytes) { + Ok(text) => text, + Err(reason) => return Ok(validation_failure(request.evidence, reason)), + }; + if validate_json(&text).is_err() { + if matches!(request.evidence, ValidationEvidence::StructuralValidation) + && initial_probe_was_provisional(text.as_bytes()) + && has_complete_json_prefix_with_trailing_content(&text) + { + return Ok(ProcessorValidationOutput::NoMatch); + } + return Ok(validation_failure(request.evidence, "malformed_json")); + } + if validate_json_without_duplicate_members(&text).is_err() { + return Ok(validation_failure(request.evidence, "malformed_json")); + } + Ok(ProcessorValidationOutput::Validated { + media_type: String::from(JSON_MEDIA_TYPE), + evidence: request.evidence, + metadata_json: serde_json::json!({"bytes": text.len()}).to_string(), + }) +} + +pub(crate) async fn read( + request: FileMediaProviderReadRequest, + source: &dyn VerifiedBlobSource, + cancellation: &dyn CancellationSignal, +) -> Result { + if request.view.as_str() != STRUCTURED_VIEW_NAME || !read_input_is_empty(&request.input) { + return Ok(ProcessorReadOutput::InvalidViewArguments); + } + let Some(bytes) = source::read_complete(source, cancellation, MAX_TEXT_FAMILY_BYTES).await? + else { + return Ok(ProcessorReadOutput::SourceTooLarge { + maximum_bytes: MAX_TEXT_FAMILY_BYTES, + }); + }; + let text = source::checked_utf8(bytes).map_err(|_| ProcessorFailure::Failed)?; + if json_depth_exceeds(text.as_bytes(), MAX_STRUCTURED_DEPTH) { + return Ok(ProcessorReadOutput::ExpansionLimitExceeded { + limit_kind: String::from("depth_limit_exceeded"), + }); + } + let value = parse_json(&text).map_err(|_| ProcessorFailure::Failed)?; + if json_value_depth_exceeds(&value, MAX_STRUCTURED_DEPTH) { + return Ok(ProcessorReadOutput::ExpansionLimitExceeded { + limit_kind: String::from("depth_limit_exceeded"), + }); + } + if json_container_entries_exceed(&value, request.maximum_container_entries) { + return Ok(ProcessorReadOutput::ExpansionLimitExceeded { + limit_kind: String::from("container_entry_limit_exceeded"), + }); + } + let body_json = serde_json::to_string(&value).map_err(|_| ProcessorFailure::Failed)?; + if body_json.len() > MAX_TEXT_FAMILY_BYTES as usize { + return Ok(ProcessorReadOutput::OutputUnitTooLarge); + } + Ok(ProcessorReadOutput::Structured { + body_json, + truncated: false, + cursor: None, + }) +} + +fn validate_json(text: &str) -> Result<(), serde_json::Error> { + let mut deserializer = serde_json::Deserializer::from_str(text); + deserializer.disable_recursion_limit(); + serde::de::IgnoredAny::deserialize(serde_stacker::Deserializer::new(&mut deserializer))?; + deserializer.end() +} + +fn validate_json_without_duplicate_members(text: &str) -> Result<(), serde_json::Error> { + let mut deserializer = serde_json::Deserializer::from_str(text); + deserializer.disable_recursion_limit(); + DuplicateChecked.deserialize(serde_stacker::Deserializer::new(&mut deserializer))?; + deserializer.end() +} + +#[derive(Clone, Copy)] +struct DuplicateChecked; + +impl<'de> DeserializeSeed<'de> for DuplicateChecked { + type Value = (); + + fn deserialize(self, deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + deserializer.deserialize_any(DuplicateCheckedVisitor) + } +} + +struct DuplicateCheckedVisitor; + +impl<'de> Visitor<'de> for DuplicateCheckedVisitor { + type Value = (); + + fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("a JSON value without duplicate object members") + } + + fn visit_bool(self, _value: bool) -> Result<(), E> { + Ok(()) + } + fn visit_i64(self, _value: i64) -> Result<(), E> { + Ok(()) + } + fn visit_u64(self, _value: u64) -> Result<(), E> { + Ok(()) + } + fn visit_f64(self, _value: f64) -> Result<(), E> { + Ok(()) + } + fn visit_str(self, _value: &str) -> Result<(), E> { + Ok(()) + } + fn visit_string(self, _value: String) -> Result<(), E> { + Ok(()) + } + fn visit_none(self) -> Result<(), E> { + Ok(()) + } + fn visit_unit(self) -> Result<(), E> { + Ok(()) + } + + fn visit_seq(self, mut sequence: A) -> Result<(), A::Error> + where + A: SeqAccess<'de>, + { + while sequence.next_element_seed(DuplicateChecked)?.is_some() {} + Ok(()) + } + + fn visit_map(self, mut map: A) -> Result<(), A::Error> + where + A: MapAccess<'de>, + { + let mut names = HashSet::new(); + while let Some(name) = map.next_key::()? { + if !names.insert(name) { + return Err(serde::de::Error::custom("duplicate JSON object member")); + } + map.next_value_seed(DuplicateChecked)?; + } + Ok(()) + } +} + +fn has_declared_json_prefix(prefix: &[u8]) -> bool { + let prefix = trim_ascii_start(prefix); + if prefix.is_empty() { + return false; + } + let eof_consistent = matches!( + prefix.first(), + Some(b'{' | b'[' | b'\"' | b't' | b'f' | b'n' | b'-' | b'0'..=b'9') + ); + let Ok(text) = std::str::from_utf8(prefix) else { + return false; + }; + let mut deserializer = serde_json::Deserializer::from_str(text); + deserializer.disable_recursion_limit(); + match serde::de::IgnoredAny::deserialize(serde_stacker::Deserializer::new(&mut deserializer)) { + Ok(_) => deserializer.end().is_ok(), + Err(error) => { + eof_consistent + && (error.is_eof() + || matches!(prefix.first(), Some(b'-' | b'0'..=b'9')) + && is_json_number_prefix(text)) + } + } +} + +fn has_complete_json_prefix_with_trailing_content(text: &str) -> bool { + let mut deserializer = serde_json::Deserializer::from_str(text); + deserializer.disable_recursion_limit(); + serde::de::IgnoredAny::deserialize(serde_stacker::Deserializer::new(&mut deserializer)) + .is_ok_and(|_| deserializer.end().is_err()) +} + +fn initial_probe_was_provisional(bytes: &[u8]) -> bool { + usize::try_from(PROBE_PREFIX_BYTES) + .ok() + .and_then(|length| bytes.get(..length)) + .is_some_and(is_complete_json_probe_document) +} + +fn is_json_number_prefix(text: &str) -> bool { + let bytes = text.as_bytes(); + let mut index = usize::from(bytes.first() == Some(&b'-')); + if index == bytes.len() { + return true; + } + + match bytes[index] { + b'0' => index += 1, + b'1'..=b'9' => { + index += 1; + while matches!(bytes.get(index), Some(b'0'..=b'9')) { + index += 1; + } + } + _ => return false, + } + + if bytes.get(index) == Some(&b'.') { + index += 1; + let fraction_start = index; + while matches!(bytes.get(index), Some(b'0'..=b'9')) { + index += 1; + } + if index == fraction_start { + return index == bytes.len(); + } + if index == bytes.len() { + return true; + } + } + + if matches!(bytes.get(index), Some(b'e' | b'E')) { + index += 1; + if matches!(bytes.get(index), Some(b'+' | b'-')) { + index += 1; + } + while matches!(bytes.get(index), Some(b'0'..=b'9')) { + index += 1; + } + } + + index == bytes.len() +} + +pub(crate) fn is_complete_json_document(prefix: &[u8]) -> bool { + let prefix = trim_ascii_start(prefix); + let Ok(text) = std::str::from_utf8(prefix) else { + return false; + }; + validate_json(text).is_ok() +} + +fn is_complete_json_probe_document(prefix: &[u8]) -> bool { + let prefix = trim_ascii_start(prefix); + source::probe_utf8(prefix).is_some_and(|text| validate_json(text).is_ok()) +} + +fn parse_json(text: &str) -> Result { + parse_json_without_duplicate_members_bounded( + text, + JsonParseLimits { + maximum_nodes: u64::MAX, + maximum_container_entries: u64::MAX, + }, + ) +} + +/// Detects excessive nesting before building a recursively dropped JSON tree. +fn json_depth_exceeds(bytes: &[u8], maximum_depth: u32) -> bool { + let mut depth = 0_u32; + let mut in_string = false; + let mut escaped = false; + for byte in bytes { + if in_string { + if escaped { + escaped = false; + } else if *byte == b'\\' { + escaped = true; + } else if *byte == b'\"' { + in_string = false; + } + } else if *byte == b'\"' { + in_string = true; + } else if matches!(*byte, b'{' | b'[') { + depth = depth.saturating_add(1); + if depth > maximum_depth { + return true; + } + } else if matches!(*byte, b'}' | b']') { + depth = depth.saturating_sub(1); + } + } + false +} + +/// Measures admitted trees iteratively so depth enforcement cannot overflow the stack. +fn json_value_depth_exceeds(value: &serde_json::Value, maximum_depth: u32) -> bool { + let mut pending = vec![(value, 0_u32)]; + while let Some((value, parent_depth)) = pending.pop() { + match value { + serde_json::Value::Array(values) => { + let depth = parent_depth.saturating_add(1); + if depth > maximum_depth { + return true; + } + pending.extend(values.iter().map(|value| (value, depth))); + } + serde_json::Value::Object(values) => { + let depth = parent_depth.saturating_add(1); + if depth > maximum_depth { + return true; + } + pending.extend(values.values().map(|value| (value, depth))); + } + serde_json::Value::Null + | serde_json::Value::Bool(_) + | serde_json::Value::Number(_) + | serde_json::Value::String(_) => {} + } + } + false +} + +/// Checks concentrated container fan-out iteratively before output crosses the worker. +fn json_container_entries_exceed(value: &serde_json::Value, maximum_entries: u64) -> bool { + let mut pending = vec![value]; + while let Some(value) = pending.pop() { + match value { + serde_json::Value::Array(values) => { + if values.len() as u64 > maximum_entries { + return true; + } + pending.extend(values); + } + serde_json::Value::Object(values) => { + if values.len() as u64 > maximum_entries { + return true; + } + pending.extend(values.values()); + } + serde_json::Value::Null + | serde_json::Value::Bool(_) + | serde_json::Value::Number(_) + | serde_json::Value::String(_) => {} + } + } + false +} + +fn malformed(reason: &str) -> ProcessorValidationOutput { + ProcessorValidationOutput::Malformed { + media_type: String::from(JSON_MEDIA_TYPE), + reason_code: String::from(reason), + } +} + +fn validation_failure(evidence: ValidationEvidence, reason: &str) -> ProcessorValidationOutput { + match evidence { + ValidationEvidence::DeclaredCandidateStructurallyValidated => { + ProcessorValidationOutput::NoMatch + } + ValidationEvidence::StrongSignature + | ValidationEvidence::StructuralValidation + | ValidationEvidence::StreamingTextValidation => malformed(reason), + } +} diff --git a/crates/file-media-adapters-text/src/lib.rs b/crates/file-media-adapters-text/src/lib.rs new file mode 100644 index 0000000000..90fea15bf7 --- /dev/null +++ b/crates/file-media-adapters-text/src/lib.rs @@ -0,0 +1,229 @@ +//! Isolated adapters for UTF-8 text, JSON, and CSV bytes. + +mod csv_adapter; +mod json_adapter; +mod source; +mod text_adapter; + +use std::{error::Error, str::FromStr}; + +use signalbox_file_media_runtime::{ + CanonicalJsonObjectSchema, CanonicalMediaType, FileMediaProvider, FileMediaProviderDeclaration, + FileMediaProviderFailure, FileMediaProviderFuture, FileMediaProviderReadRequest, + FileMediaProviderValidationRequest, FileReadInput, FileReaderName, FileReaderProviderName, + FileReaderRevision, MAX_STRUCTURED_DEPTH, MAX_STRUCTURED_NODES, ProbeDeclaration, + ProbeDeclarationInput, ProcessorFailure, ProcessorProbeOutput, ProcessorReadOutput, + ProcessorValidationOutput, ReadAccessPattern, ReadViewBounds, ReadViewDeclaration, + ReadViewName, ReaderDeclaration, ReaderDeclarationInput, ReaderIdentity, ReasonCode, + StreamingTextFallback, ValidationDeclaration, VerifiedBlobSource, +}; + +const PROVIDER_NAME: &str = "signalbox_text"; +const TEXT_READER_NAME: &str = "utf8_text"; +const JSON_READER_NAME: &str = "json"; +const CSV_READER_NAME: &str = "csv"; +const READER_REVISION: &str = "v1"; +const TEXT_MEDIA_TYPE: &str = "text/plain"; +const JSON_MEDIA_TYPE: &str = "application/json"; +const CSV_MEDIA_TYPE: &str = "text/csv"; +pub(crate) const TEXT_VIEW_NAME: &str = "text"; +pub(crate) const STRUCTURED_VIEW_NAME: &str = "structured"; +// Tunable effective ceiling; bounds detection I/O while retaining useful structure evidence. +const PROBE_PREFIX_BYTES: u64 = 4_096; + +/// Hard safety ceiling; bounds whole-source parsing and result allocation. +pub const MAX_TEXT_FAMILY_BYTES: u64 = 131_072; + +/// Compiled provider for the three version-one text-family readers. +#[derive(Clone, Copy, Debug, Default)] +pub struct TextFamilyProvider; + +impl FileMediaProvider for TextFamilyProvider { + fn declaration(&self) -> FileMediaProviderDeclaration { + match text_family_declaration() { + Ok(declaration) => declaration, + Err(_) => std::process::abort(), + } + } + + fn probe<'a>( + &'a self, + reader: &'a ReaderIdentity, + source: &'a dyn VerifiedBlobSource, + cancellation: &'a dyn signalbox_file_media_runtime::CancellationSignal, + ) -> FileMediaProviderFuture<'a, ProcessorProbeOutput> { + Box::pin(async move { + match reader.reader().as_str() { + TEXT_READER_NAME => text_adapter::probe(source, cancellation).await, + JSON_READER_NAME => json_adapter::probe(source, cancellation).await, + CSV_READER_NAME => csv_adapter::probe(source, cancellation).await, + _ => Err(ProcessorFailure::Protocol), + } + .map_err(|_| FileMediaProviderFailure::Failed) + }) + } + + fn inspect<'a>( + &'a self, + reader: &'a ReaderIdentity, + request: FileMediaProviderValidationRequest, + source: &'a dyn VerifiedBlobSource, + cancellation: &'a dyn signalbox_file_media_runtime::CancellationSignal, + ) -> FileMediaProviderFuture<'a, ProcessorValidationOutput> { + Box::pin(async move { + match reader.reader().as_str() { + TEXT_READER_NAME => text_adapter::inspect(request, source, cancellation).await, + JSON_READER_NAME => json_adapter::inspect(request, source, cancellation).await, + CSV_READER_NAME => csv_adapter::inspect(request, source, cancellation).await, + _ => Err(ProcessorFailure::Protocol), + } + .map_err(|_| FileMediaProviderFailure::Failed) + }) + } + + fn read<'a>( + &'a self, + reader: &'a ReaderIdentity, + request: FileMediaProviderReadRequest, + source: &'a dyn VerifiedBlobSource, + cancellation: &'a dyn signalbox_file_media_runtime::CancellationSignal, + ) -> FileMediaProviderFuture<'a, ProcessorReadOutput> { + Box::pin(async move { + match reader.reader().as_str() { + TEXT_READER_NAME => text_adapter::read(request, source, cancellation).await, + JSON_READER_NAME => json_adapter::read(request, source, cancellation).await, + CSV_READER_NAME => csv_adapter::read(request, source, cancellation).await, + _ => Err(ProcessorFailure::Protocol), + } + .map_err(|_| FileMediaProviderFailure::Failed) + }) + } +} + +/// Builds the exact declaration registered by the text-family worker. +pub fn text_family_declaration() +-> Result> { + let provider = FileReaderProviderName::try_new(PROVIDER_NAME)?; + let text = reader(ReaderInput { + provider: &provider, + name: TEXT_READER_NAME, + media_type: TEXT_MEDIA_TYPE, + view: text_view()?, + reasons: vec!["invalid_utf8", "nul_byte", "source_too_large"], + fallback: StreamingTextFallback::Enabled, + })?; + let json = reader(ReaderInput { + provider: &provider, + name: JSON_READER_NAME, + media_type: JSON_MEDIA_TYPE, + view: structured_view("Reads the complete JSON value as bounded structured data.")?, + reasons: vec![ + "invalid_utf8", + "nul_byte", + "malformed_json", + "source_too_large", + "depth_limit_exceeded", + "container_entry_limit_exceeded", + ], + fallback: StreamingTextFallback::Disabled, + })?; + let csv = reader(ReaderInput { + provider: &provider, + name: CSV_READER_NAME, + media_type: CSV_MEDIA_TYPE, + view: structured_view("Reads a rectangular CSV table as headers and rows.")?, + reasons: vec![ + "invalid_utf8", + "nul_byte", + "malformed_csv", + "source_too_large", + "row_limit_exceeded", + "column_limit_exceeded", + "container_entry_limit_exceeded", + ], + fallback: StreamingTextFallback::Disabled, + })?; + Ok(FileMediaProviderDeclaration::try_new( + provider, + vec![text, json, csv], + )?) +} + +struct ReaderInput<'a> { + provider: &'a FileReaderProviderName, + name: &'a str, + media_type: &'a str, + view: ReadViewDeclaration, + reasons: Vec<&'a str>, + fallback: StreamingTextFallback, +} + +fn reader(input: ReaderInput<'_>) -> Result> { + let reason_codes = input + .reasons + .into_iter() + .map(ReasonCode::try_new) + .collect::, _>>()?; + Ok(ReaderDeclaration::try_new(ReaderDeclarationInput { + provider: input.provider.clone(), + reader: FileReaderName::try_new(input.name)?, + revision: FileReaderRevision::try_new(READER_REVISION)?, + media_types: vec![CanonicalMediaType::from_str(input.media_type)?], + probe: ProbeDeclaration::new(ProbeDeclarationInput { + prefix_bytes: PROBE_PREFIX_BYTES, + suffix_bytes: 0, + range_count: 0, + cumulative_bytes: PROBE_PREFIX_BYTES, + }), + validation: ValidationDeclaration::new(MAX_TEXT_FAMILY_BYTES, 1), + views: vec![input.view], + reason_codes, + streaming_text_fallback: input.fallback, + })?) +} + +fn text_view() -> Result> { + Ok(ReadViewDeclaration::try_new( + ReadViewName::try_new(TEXT_VIEW_NAME)?, + String::from("Reads the complete file as exact UTF-8 text."), + empty_options_schema()?, + ReadAccessPattern::Streaming { maximum_ranges: 1 }, + ReadViewBounds::Text { + source_bytes: MAX_TEXT_FAMILY_BYTES, + output_bytes: MAX_TEXT_FAMILY_BYTES as usize, + }, + )?) +} + +fn structured_view(description: &str) -> Result> { + Ok(ReadViewDeclaration::try_new( + ReadViewName::try_new(STRUCTURED_VIEW_NAME)?, + String::from(description), + empty_options_schema()?, + ReadAccessPattern::Streaming { maximum_ranges: 1 }, + ReadViewBounds::Structured { + source_bytes: MAX_TEXT_FAMILY_BYTES, + output_bytes: MAX_TEXT_FAMILY_BYTES as usize, + depth: MAX_STRUCTURED_DEPTH, + nodes: MAX_STRUCTURED_NODES, + string_bytes: MAX_TEXT_FAMILY_BYTES as usize, + }, + )?) +} + +fn empty_options_schema() -> Result> { + Ok(CanonicalJsonObjectSchema::try_new( + r#"{"additionalProperties":false,"type":"object"}"#, + )?) +} + +fn options_are_empty(options: &serde_json::Value) -> bool { + options.as_object().is_some_and(serde_json::Map::is_empty) +} + +fn read_input_is_empty(input: &FileReadInput) -> bool { + match input { + FileReadInput::Initial { options } => options_are_empty(options), + FileReadInput::Continuation { .. } => false, + } +} diff --git a/crates/file-media-adapters-text/src/source.rs b/crates/file-media-adapters-text/src/source.rs new file mode 100644 index 0000000000..fc8d5fd6c9 --- /dev/null +++ b/crates/file-media-adapters-text/src/source.rs @@ -0,0 +1,95 @@ +use signalbox_file_media_runtime::{CancellationSignal, ProcessorFailure, VerifiedBlobSource}; + +use crate::{MAX_TEXT_FAMILY_BYTES, PROBE_PREFIX_BYTES, json_adapter::ProbeExtent}; + +pub(crate) async fn read_complete( + source: &dyn VerifiedBlobSource, + cancellation: &dyn CancellationSignal, + maximum_bytes: u64, +) -> Result>, ProcessorFailure> { + if cancellation.is_cancelled() { + return Err(ProcessorFailure::Cancelled); + } + if source.byte_length().get() > MAX_TEXT_FAMILY_BYTES.min(maximum_bytes) { + return Ok(None); + } + let bytes = source + .read_range(0, source.byte_length()) + .await + .map_err(|_| ProcessorFailure::Failed)?; + if cancellation.is_cancelled() { + return Err(ProcessorFailure::Cancelled); + } + Ok(Some(bytes)) +} + +pub(crate) async fn read_probe_prefix( + source: &dyn VerifiedBlobSource, + cancellation: &dyn CancellationSignal, +) -> Result, ProcessorFailure> { + read_prefix(source, cancellation, PROBE_PREFIX_BYTES).await +} + +pub(crate) async fn read_validation_prefix( + source: &dyn VerifiedBlobSource, + cancellation: &dyn CancellationSignal, + maximum_bytes: u64, +) -> Result, ProcessorFailure> { + read_prefix(source, cancellation, PROBE_PREFIX_BYTES.min(maximum_bytes)).await +} + +async fn read_prefix( + source: &dyn VerifiedBlobSource, + cancellation: &dyn CancellationSignal, + maximum_bytes: u64, +) -> Result, ProcessorFailure> { + if cancellation.is_cancelled() { + return Err(ProcessorFailure::Cancelled); + } + let length = source + .byte_length() + .min(std::num::NonZeroU64::new(maximum_bytes).ok_or(ProcessorFailure::Failed)?); + source + .read_range(0, length) + .await + .map_err(|_| ProcessorFailure::Failed) +} + +/// Decodes probe bytes whose trailing scalar may have been cut by the probe +/// boundary, discarding an incomplete final scalar as a read artifact. +/// +/// Only sound for a genuinely truncated prefix. Use [`probe_utf8_within`] when +/// the extent is known, so a complete source is never judged on a shortened +/// view of its own bytes. +pub(crate) fn probe_utf8(bytes: &[u8]) -> Option<&str> { + match std::str::from_utf8(bytes) { + Ok(text) => Some(text), + Err(error) if error.error_len().is_none() => { + std::str::from_utf8(&bytes[..error.valid_up_to()]).ok() + } + Err(_) => None, + } +} + +/// Decodes probe bytes according to how much of the source they cover. +/// +/// A truncated prefix may end mid-scalar because the probe boundary cut the +/// source, so the incomplete trailing scalar is a read artifact and is dropped. +/// A complete source has no such artifact: every byte is real content, so an +/// incomplete trailing scalar means the source itself is not valid UTF-8 and no +/// structural candidate may be claimed from the shortened text. +pub(crate) fn probe_utf8_within(bytes: &[u8], extent: ProbeExtent) -> Option<&str> { + match extent { + ProbeExtent::CompleteSource => std::str::from_utf8(bytes).ok(), + ProbeExtent::TruncatedPrefix => probe_utf8(bytes), + } +} + +pub(crate) fn checked_utf8(bytes: Vec) -> Result { + let text = String::from_utf8(bytes).map_err(|_| "invalid_utf8")?; + if text.contains('\0') { + Err("nul_byte") + } else { + Ok(text) + } +} diff --git a/crates/file-media-adapters-text/src/text_adapter.rs b/crates/file-media-adapters-text/src/text_adapter.rs new file mode 100644 index 0000000000..c6ebbc8420 --- /dev/null +++ b/crates/file-media-adapters-text/src/text_adapter.rs @@ -0,0 +1,80 @@ +use signalbox_file_media_runtime::{ + CancellationSignal, FileMediaProviderReadRequest, FileMediaProviderValidationRequest, + ProcessorFailure, ProcessorProbeOutput, ProcessorReadOutput, ProcessorValidationOutput, + ValidationEvidence, VerifiedBlobSource, +}; + +use crate::{MAX_TEXT_FAMILY_BYTES, TEXT_MEDIA_TYPE, TEXT_VIEW_NAME, read_input_is_empty, source}; + +pub(crate) async fn probe( + _source: &dyn VerifiedBlobSource, + cancellation: &dyn CancellationSignal, +) -> Result { + if cancellation.is_cancelled() { + Err(ProcessorFailure::Cancelled) + } else { + Ok(ProcessorProbeOutput::NoMatch) + } +} + +pub(crate) async fn inspect( + request: FileMediaProviderValidationRequest, + source: &dyn VerifiedBlobSource, + cancellation: &dyn CancellationSignal, +) -> Result { + if request.media_type.as_str() != TEXT_MEDIA_TYPE { + return Err(ProcessorFailure::Protocol); + } + let Some(bytes) = + source::read_complete(source, cancellation, request.maximum_source_bytes).await? + else { + return Ok(validation_failure(&request, "source_too_large")); + }; + match source::checked_utf8(bytes) { + Ok(text) => Ok(ProcessorValidationOutput::Validated { + media_type: String::from(TEXT_MEDIA_TYPE), + evidence: request.evidence, + metadata_json: serde_json::json!({"bytes": text.len()}).to_string(), + }), + Err(reason) => Ok(validation_failure(&request, reason)), + } +} + +fn validation_failure( + request: &FileMediaProviderValidationRequest, + reason: &str, +) -> ProcessorValidationOutput { + match request.evidence { + ValidationEvidence::StreamingTextValidation => ProcessorValidationOutput::NoMatch, + ValidationEvidence::StrongSignature + | ValidationEvidence::StructuralValidation + | ValidationEvidence::DeclaredCandidateStructurallyValidated => { + ProcessorValidationOutput::Malformed { + media_type: String::from(TEXT_MEDIA_TYPE), + reason_code: String::from(reason), + } + } + } +} + +pub(crate) async fn read( + request: FileMediaProviderReadRequest, + source: &dyn VerifiedBlobSource, + cancellation: &dyn CancellationSignal, +) -> Result { + if request.view.as_str() != TEXT_VIEW_NAME || !read_input_is_empty(&request.input) { + return Ok(ProcessorReadOutput::InvalidViewArguments); + } + let Some(bytes) = source::read_complete(source, cancellation, MAX_TEXT_FAMILY_BYTES).await? + else { + return Ok(ProcessorReadOutput::SourceTooLarge { + maximum_bytes: MAX_TEXT_FAMILY_BYTES, + }); + }; + let text = source::checked_utf8(bytes).map_err(|_| ProcessorFailure::Failed)?; + Ok(ProcessorReadOutput::Text { + body: text, + truncated: false, + cursor: None, + }) +} diff --git a/crates/file-media-adapters-text/tests/adapters.rs b/crates/file-media-adapters-text/tests/adapters.rs new file mode 100644 index 0000000000..2b89b758c7 --- /dev/null +++ b/crates/file-media-adapters-text/tests/adapters.rs @@ -0,0 +1,969 @@ +mod fixtures; +mod support; + +use std::error::Error; + +use signalbox_file_media_runtime::{FileMediaCeilings, FileMediaFailure, ReasonCode}; +use support::{DeclaredMismatchExpectation, DirectProcessor, MemorySource, ReadInput}; + +#[tokio::test] +async fn utf8_text_detects_validates_and_reads_exact_bytes() -> Result<(), Box> { + let bytes = fixtures::utf8_text(); + let expected = std::str::from_utf8(&bytes)?.to_owned(); + let source = MemorySource::new(bytes); + + let inspection = support::inspect(&source, "text/plain").await?; + support::assert_validated_media(inspection, "text/plain"); + let result = support::read( + &source, + ReadInput { + media_type: "text/plain", + view: "text", + }, + &DirectProcessor::provider(), + ) + .await?; + support::assert_text(result, &expected); + Ok(()) +} + +#[tokio::test] +async fn utf8_text_rejects_a_truncated_scalar_as_typed_malformed() -> Result<(), Box> { + let source = MemorySource::new(fixtures::truncated_utf8()); + + let inspection = support::inspect(&source, "text/plain").await?; + support::assert_malformed_reason(inspection, "invalid_utf8"); + Ok(()) +} + +#[tokio::test] +async fn complete_source_json_probe_does_not_drop_invalid_utf8_suffix() -> Result<(), Box> +{ + let source = MemorySource::new(vec![b'{', b'}', 0xc3]); + + let inspection = support::inspect(&source, "text/plain").await?; + support::assert_malformed_reason(inspection, "invalid_utf8"); + Ok(()) +} + +#[tokio::test] +async fn utf8_text_rejects_oversized_input_with_registered_reason() -> Result<(), Box> { + let source = MemorySource::new(fixtures::oversized(b'a')); + + let inspection = support::inspect(&source, "text/plain").await?; + support::assert_malformed_reason(inspection, "source_too_large"); + Ok(()) +} + +#[tokio::test] +async fn json_detects_validates_and_returns_structured_data() -> Result<(), Box> { + let source = MemorySource::new(fixtures::json_document()); + let expected = fixtures::json_document_value(); + + let inspection = support::inspect(&source, "application/json").await?; + support::assert_validated_media(inspection, "application/json"); + let result = support::read( + &source, + ReadInput { + media_type: "application/json", + view: "structured", + }, + &DirectProcessor::provider(), + ) + .await?; + support::assert_structured(result, &expected); + Ok(()) +} + +#[tokio::test] +async fn json_preserves_arbitrary_precision_numbers() -> Result<(), Box> { + let bytes = fixtures::arbitrary_precision_json(); + let expected = std::str::from_utf8(&bytes)?.to_owned(); + let source = MemorySource::new(bytes); + + let result = support::read( + &source, + ReadInput { + media_type: "application/json", + view: "structured", + }, + &DirectProcessor::provider(), + ) + .await?; + support::assert_structured_json(result, &expected); + Ok(()) +} + +#[tokio::test] +async fn json_rejects_truncated_structure_as_typed_malformed() -> Result<(), Box> { + let source = MemorySource::new(fixtures::truncated_json()); + + let inspection = support::inspect(&source, "application/json").await?; + support::assert_malformed_reason(inspection, "malformed_json"); + Ok(()) +} + +#[tokio::test] +async fn json_rejects_duplicate_object_members_as_malformed() -> Result<(), Box> { + let source = MemorySource::new(fixtures::duplicate_member_json()); + + let inspection = support::inspect(&source, "application/json").await?; + support::assert_malformed_reason(inspection, "malformed_json"); + Ok(()) +} + +#[tokio::test] +async fn json_rejects_duplicate_members_even_beyond_read_depth() -> Result<(), Box> { + let source = MemorySource::new(fixtures::deep_json_with_duplicate_root_member()); + + let inspection = support::inspect(&source, "application/json").await?; + support::assert_malformed_reason(inspection, "malformed_json"); + Ok(()) +} + +#[tokio::test] +async fn top_level_json_scalar_uses_the_text_fallback() -> Result<(), Box> { + let source = MemorySource::new(b"true".to_vec()); + + let inspection = support::inspect(&source, "text/plain").await?; + support::assert_validated_media(inspection, "text/plain"); + Ok(()) +} + +#[tokio::test] +async fn unprobed_declared_json_candidate_uses_text_fallback() -> Result<(), Box> { + let source = MemorySource::new(b"hello".to_vec()); + + let inspection = support::inspect(&source, "application/json").await?; + support::assert_declared_mismatch( + inspection, + DeclaredMismatchExpectation { + declared: "application/json", + detected: "text/plain", + }, + ); + Ok(()) +} + +#[tokio::test] +async fn json_rejects_oversized_input_with_registered_reason() -> Result<(), Box> { + let mut bytes = fixtures::oversized(b' '); + bytes[0] = b'{'; + bytes[1] = b'}'; + let source = MemorySource::new(bytes); + + let inspection = support::inspect(&source, "application/json").await?; + support::assert_malformed_reason(inspection, "source_too_large"); + Ok(()) +} + +#[tokio::test] +async fn oversized_declared_json_scalar_preserves_the_size_reason() -> Result<(), Box> { + let mut bytes = b"true".to_vec(); + bytes.resize(128 * 1_024 + 1, b' '); + let source = MemorySource::new(bytes); + + let inspection = support::inspect(&source, "application/json").await?; + support::assert_malformed_reason(inspection, "source_too_large"); + Ok(()) +} + +#[tokio::test] +async fn oversized_declared_json_string_preserves_the_size_reason() -> Result<(), Box> { + let mut bytes = b"\"".to_vec(); + bytes.resize(128 * 1_024 + 1, b'a'); + let source = MemorySource::new(bytes); + + let inspection = support::inspect(&source, "application/json").await?; + support::assert_malformed_reason(inspection, "source_too_large"); + Ok(()) +} + +#[tokio::test] +async fn declared_json_follow_up_respects_the_validation_ceiling() -> Result<(), Box> { + let source = MemorySource::new(b"true ".to_vec()); + let mut ceilings = FileMediaCeilings::version_one(); + ceilings.validation_source_bytes = 4; + + let inspection = support::inspect_with_ceilings(&source, "application/json", ceilings).await?; + support::assert_malformed_reason(inspection, "source_too_large"); + Ok(()) +} + +#[tokio::test] +async fn truncated_declared_json_scalar_preserves_the_size_reason() -> Result<(), Box> { + let source = MemorySource::new(b"true".to_vec()); + let mut ceilings = FileMediaCeilings::version_one(); + ceilings.validation_source_bytes = 1; + + let inspection = support::inspect_with_ceilings(&source, "application/json", ceilings).await?; + support::assert_malformed_reason(inspection, "source_too_large"); + Ok(()) +} + +#[tokio::test] +async fn truncated_declared_negative_json_number_preserves_the_size_reason() +-> Result<(), Box> { + let source = MemorySource::new(b"-1".to_vec()); + let mut ceilings = FileMediaCeilings::version_one(); + ceilings.validation_source_bytes = 1; + + let inspection = support::inspect_with_ceilings(&source, "application/json", ceilings).await?; + support::assert_malformed_reason(inspection, "source_too_large"); + Ok(()) +} + +#[tokio::test] +async fn truncated_declared_json_exponent_preserves_the_size_reason() -> Result<(), Box> +{ + let source = MemorySource::new(b"1e2".to_vec()); + let mut ceilings = FileMediaCeilings::version_one(); + ceilings.validation_source_bytes = 2; + + let inspection = support::inspect_with_ceilings(&source, "application/json", ceilings).await?; + support::assert_malformed_reason(inspection, "source_too_large"); + Ok(()) +} + +#[tokio::test] +async fn impossible_declared_json_fraction_prefix_remains_unknown() -> Result<(), Box> { + let source = MemorySource::new(b"1.e2".to_vec()); + let mut ceilings = FileMediaCeilings::version_one(); + ceilings.validation_source_bytes = 3; + + let inspection = support::inspect_with_ceilings(&source, "application/json", ceilings).await?; + support::assert_unknown(inspection); + Ok(()) +} + +#[tokio::test] +async fn oversized_declared_json_rejects_trailing_prefix_bytes() -> Result<(), Box> { + let mut bytes = b"true trailing".to_vec(); + bytes.resize(128 * 1_024 + 1, b' '); + let source = MemorySource::new(bytes); + + let inspection = support::inspect(&source, "application/json").await?; + support::assert_unknown(inspection); + Ok(()) +} + +#[tokio::test] +async fn oversized_declared_json_rejects_an_incomplete_trailing_scalar() +-> Result<(), Box> { + let mut bytes = b"true ".to_vec(); + bytes.resize(4_095, b' '); + bytes.extend_from_slice("é".as_bytes()); + bytes.resize(128 * 1_024 + 1, b' '); + let source = MemorySource::new(bytes); + + let inspection = support::inspect(&source, "application/json").await?; + support::assert_unknown(inspection); + Ok(()) +} + +#[tokio::test] +async fn non_json_ascii_whitespace_before_an_object_uses_text_fallback() +-> Result<(), Box> { + let source = MemorySource::new(b"\x0b{}".to_vec()); + + let inspection = support::inspect(&source, "text/plain").await?; + support::assert_validated_media(inspection, "text/plain"); + Ok(()) +} + +#[tokio::test] +async fn json_honors_the_effective_validation_source_ceiling() -> Result<(), Box> { + let source = MemorySource::new(fixtures::json_document()); + let mut ceilings = FileMediaCeilings::version_one(); + ceilings.validation_source_bytes = 1; + + let inspection = support::inspect_with_ceilings(&source, "application/json", ceilings).await?; + support::assert_malformed_reason(inspection, "source_too_large"); + Ok(()) +} + +#[tokio::test] +async fn pretty_json_is_not_ambiguous_with_csv() -> Result<(), Box> { + let source = MemorySource::new(fixtures::pretty_json_document()); + + let inspection = support::inspect(&source, "application/json").await?; + support::assert_validated_media(inspection, "application/json"); + Ok(()) +} + +#[tokio::test] +async fn csv_probe_does_not_claim_structurally_valid_json() -> Result<(), Box> { + let source = MemorySource::new(fixtures::json_array_formatted_like_csv()); + + let inspection = support::inspect(&source, "application/json").await?; + support::assert_validated_media(inspection, "application/json"); + Ok(()) +} + +#[tokio::test] +async fn incomplete_json_does_not_suppress_complete_csv() -> Result<(), Box> { + let source = MemorySource::new(b"[1,2\n,3".to_vec()); + + let inspection = support::inspect(&source, "text/csv").await?; + support::assert_validated_media(inspection, "text/csv"); + Ok(()) +} + +#[tokio::test] +async fn truncated_json_prefix_does_not_suppress_valid_csv() -> Result<(), Box> { + let source = MemorySource::new(fixtures::csv_with_json_consistent_truncated_prefix()); + + let inspection = support::inspect(&source, "text/csv").await?; + support::assert_validated_media(inspection, "text/csv"); + Ok(()) +} + +#[tokio::test] +async fn csv_like_truncated_prefix_does_not_suppress_valid_json() -> Result<(), Box> { + let source = MemorySource::new(fixtures::json_with_csv_consistent_truncated_prefix()); + + let inspection = support::inspect(&source, "application/json").await?; + support::assert_validated_media(inspection, "application/json"); + Ok(()) +} + +#[tokio::test] +async fn overlapping_truncated_prefix_still_detects_json() -> Result<(), Box> { + let source = MemorySource::new(fixtures::json_with_csv_consistent_truncated_prefix()); + + let inspection = support::inspect(&source, "text/plain").await?; + support::assert_declared_mismatch( + inspection, + DeclaredMismatchExpectation { + declared: "text/plain", + detected: "application/json", + }, + ); + Ok(()) +} + +#[tokio::test] +async fn overlapping_truncated_prefix_still_detects_csv() -> Result<(), Box> { + let source = MemorySource::new(fixtures::csv_with_json_consistent_truncated_prefix()); + + let inspection = support::inspect(&source, "text/plain").await?; + support::assert_declared_mismatch( + inspection, + DeclaredMismatchExpectation { + declared: "text/plain", + detected: "text/csv", + }, + ); + Ok(()) +} + +#[tokio::test] +async fn bracket_prefixed_prose_uses_the_text_fallback() -> Result<(), Box> { + let source = MemorySource::new(fixtures::bracket_prefixed_prose()); + + let inspection = support::inspect(&source, "text/plain").await?; + support::assert_validated_media(inspection, "text/plain"); + Ok(()) +} + +#[tokio::test] +async fn json_token_prefixed_prose_uses_the_text_fallback() -> Result<(), Box> { + let source = MemorySource::new(fixtures::json_token_prefixed_prose()); + + let inspection = support::inspect(&source, "text/plain").await?; + support::assert_validated_media(inspection, "text/plain"); + Ok(()) +} + +#[tokio::test] +async fn invalid_utf8_streaming_text_candidate_is_unknown() -> Result<(), Box> { + let source = MemorySource::new(fixtures::truncated_utf8()); + + let inspection = support::inspect(&source, "application/octet-stream").await?; + support::assert_unknown(inspection); + Ok(()) +} + +#[tokio::test] +async fn complete_json_source_ending_mid_scalar_is_unknown() -> Result<(), Box> { + let source = MemorySource::new(fixtures::json_then_incomplete_scalar()); + + let inspection = support::inspect(&source, "application/octet-stream").await?; + support::assert_unknown(inspection); + Ok(()) +} + +#[tokio::test] +async fn complete_csv_source_ending_mid_scalar_is_unknown() -> Result<(), Box> { + let source = MemorySource::new(fixtures::csv_then_incomplete_scalar()); + + let inspection = support::inspect(&source, "application/octet-stream").await?; + support::assert_unknown(inspection); + Ok(()) +} + +#[tokio::test] +async fn json_at_the_declared_depth_limit_remains_readable() -> Result<(), Box> { + let bytes = fixtures::json_at_structured_depth(); + let expected = serde_json::from_slice(&bytes)?; + let source = MemorySource::new(bytes); + + let result = support::read( + &source, + ReadInput { + media_type: "application/json", + view: "structured", + }, + &DirectProcessor::provider(), + ) + .await?; + support::assert_structured(result, &expected); + Ok(()) +} + +#[tokio::test] +async fn json_read_reports_the_declared_depth_limit() -> Result<(), Box> { + let source = MemorySource::new(fixtures::json_beyond_structured_depth()); + let expected = ReasonCode::try_new("depth_limit_exceeded")?; + + let result = support::read( + &source, + ReadInput { + media_type: "application/json", + view: "structured", + }, + &DirectProcessor::provider(), + ) + .await; + assert_eq!( + result, + Err(FileMediaFailure::ExpansionLimitExceeded { + limit_kind: expected + }) + ); + Ok(()) +} + +#[tokio::test] +async fn deeply_nested_valid_json_reports_the_declared_depth_limit() -> Result<(), Box> { + let source = MemorySource::new(fixtures::json_beyond_serde_recursion_limit()); + let expected = ReasonCode::try_new("depth_limit_exceeded")?; + + let inspection = support::inspect(&source, "application/json").await?; + support::assert_validated_media(inspection, "application/json"); + let result = support::read( + &source, + ReadInput { + media_type: "application/json", + view: "structured", + }, + &DirectProcessor::provider(), + ) + .await; + assert_eq!( + result, + Err(FileMediaFailure::ExpansionLimitExceeded { + limit_kind: expected + }) + ); + Ok(()) +} + +#[tokio::test] +async fn bracketed_numeric_csv_is_not_ambiguous_with_json() -> Result<(), Box> { + let source = MemorySource::new(fixtures::bracketed_numeric_csv()); + + let inspection = support::inspect(&source, "text/csv").await?; + support::assert_validated_media(inspection, "text/csv"); + Ok(()) +} + +#[tokio::test] +async fn complete_json_array_records_are_not_ambiguous_with_csv() -> Result<(), Box> { + let source = MemorySource::new(fixtures::complete_json_arrays_as_csv()); + + let inspection = support::inspect(&source, "text/csv").await?; + support::assert_validated_media(inspection, "text/csv"); + Ok(()) +} + +#[tokio::test] +async fn complete_json_array_followed_by_prose_uses_text_fallback() -> Result<(), Box> { + let source = MemorySource::new(fixtures::complete_json_array_followed_by_prose()); + + let inspection = support::inspect(&source, "text/plain").await?; + support::assert_validated_media(inspection, "text/plain"); + Ok(()) +} + +#[tokio::test] +async fn complete_json_prefix_without_eof_uses_text_fallback() -> Result<(), Box> { + let source = MemorySource::new(fixtures::complete_json_prefix_followed_outside_probe()); + + let inspection = support::inspect(&source, "text/plain").await?; + support::assert_validated_media(inspection, "text/plain"); + Ok(()) +} + +#[tokio::test] +async fn completed_json_prefix_followed_by_whitespace_is_structurally_detected() +-> Result<(), Box> { + let mut bytes = b"{}".to_vec(); + bytes.resize(4_097, b' '); + let source = MemorySource::new(bytes); + + let inspection = support::inspect(&source, "text/plain").await?; + support::assert_declared_mismatch( + inspection, + DeclaredMismatchExpectation { + declared: "text/plain", + detected: "application/json", + }, + ); + Ok(()) +} + +#[tokio::test] +async fn completed_json_prefix_with_split_utf8_suffix_uses_text_fallback() +-> Result<(), Box> { + let mut bytes = b"{}".to_vec(); + bytes.resize(4_095, b' '); + bytes.extend_from_slice("é prose".as_bytes()); + let source = MemorySource::new(bytes); + + let inspection = support::inspect(&source, "text/plain").await?; + support::assert_validated_media(inspection, "text/plain"); + Ok(()) +} + +#[tokio::test] +async fn nonprovisional_json_prefix_with_later_trailing_prose_is_malformed() +-> Result<(), Box> { + let mut bytes = b"{\"padding\":\"".to_vec(); + bytes.resize(4_097, b'a'); + bytes.extend_from_slice(b"\"} prose"); + let source = MemorySource::new(bytes); + + let inspection = support::inspect(&source, "text/plain").await?; + support::assert_malformed_reason(inspection, "malformed_json"); + Ok(()) +} + +#[tokio::test] +async fn json_probe_handles_a_utf8_scalar_split_at_its_boundary() -> Result<(), Box> { + let source = MemorySource::new(fixtures::json_with_scalar_split_at_probe_boundary()); + + let inspection = support::inspect(&source, "text/plain").await?; + support::assert_declared_mismatch( + inspection, + DeclaredMismatchExpectation { + declared: "text/plain", + detected: "application/json", + }, + ); + Ok(()) +} + +#[tokio::test] +async fn deeply_nested_json_is_structurally_probed() -> Result<(), Box> { + let source = MemorySource::new(fixtures::json_beyond_serde_recursion_limit()); + + let inspection = support::inspect(&source, "text/plain").await?; + support::assert_declared_mismatch( + inspection, + DeclaredMismatchExpectation { + declared: "text/plain", + detected: "application/json", + }, + ); + Ok(()) +} + +#[tokio::test] +async fn json_read_reports_the_container_entry_limit() -> Result<(), Box> { + let source = MemorySource::new(fixtures::json_beyond_container_entry_ceiling()); + let expected = ReasonCode::try_new("container_entry_limit_exceeded")?; + + let result = support::read( + &source, + ReadInput { + media_type: "application/json", + view: "structured", + }, + &DirectProcessor::provider(), + ) + .await; + assert_eq!( + result, + Err(FileMediaFailure::ExpansionLimitExceeded { + limit_kind: expected + }) + ); + Ok(()) +} + +#[tokio::test] +async fn json_read_honors_the_effective_container_entry_ceiling() -> Result<(), Box> { + let source = MemorySource::new(fixtures::json_document()); + let expected = ReasonCode::try_new("container_entry_limit_exceeded")?; + let mut ceilings = FileMediaCeilings::version_one(); + ceilings.observed_container_entries = 2; + + let result = support::read_with_ceilings( + &source, + ReadInput { + media_type: "application/json", + view: "structured", + }, + &DirectProcessor::provider(), + ceilings, + ) + .await; + assert_eq!( + result, + Err(FileMediaFailure::ExpansionLimitExceeded { + limit_kind: expected + }) + ); + Ok(()) +} + +#[tokio::test] +async fn extremely_deep_json_reports_depth_limit_without_stack_walk() -> Result<(), Box> +{ + let source = MemorySource::new(fixtures::deeply_nested_json_within_source_ceiling()); + let expected = ReasonCode::try_new("depth_limit_exceeded")?; + + let inspection = support::inspect(&source, "application/json").await?; + support::assert_validated_media(inspection, "application/json"); + let result = support::read( + &source, + ReadInput { + media_type: "application/json", + view: "structured", + }, + &DirectProcessor::provider(), + ) + .await; + assert_eq!( + result, + Err(FileMediaFailure::ExpansionLimitExceeded { + limit_kind: expected + }) + ); + Ok(()) +} + +#[tokio::test] +async fn csv_detects_validates_and_returns_headers_and_rows() -> Result<(), Box> { + let source = MemorySource::new(fixtures::csv_table()); + let expected = fixtures::csv_table_value(); + + let inspection = support::inspect(&source, "text/csv").await?; + support::assert_validated_media(inspection, "text/csv"); + let result = support::read( + &source, + ReadInput { + media_type: "text/csv", + view: "structured", + }, + &DirectProcessor::provider(), + ) + .await?; + support::assert_structured(result, &expected); + Ok(()) +} + +#[tokio::test] +async fn declared_one_column_csv_validates() -> Result<(), Box> { + let source = MemorySource::new(fixtures::one_column_csv()); + + let inspection = support::inspect(&source, "text/csv").await?; + support::assert_validated_media(inspection, "text/csv"); + Ok(()) +} + +#[tokio::test] +async fn declared_one_column_csv_preserves_malformed_quotes() -> Result<(), Box> { + let source = MemorySource::new(b"header\n\"unterminated\n".to_vec()); + + let inspection = support::inspect(&source, "text/csv").await?; + support::assert_malformed_reason(inspection, "malformed_csv"); + Ok(()) +} + +#[tokio::test] +async fn declared_header_only_csv_validates() -> Result<(), Box> { + let source = MemorySource::new(fixtures::header_only_csv()); + + let inspection = support::inspect(&source, "text/csv").await?; + support::assert_validated_media(inspection, "text/csv"); + Ok(()) +} + +#[tokio::test] +async fn declared_header_only_csv_preserves_the_column_limit_reason() -> Result<(), Box> +{ + let mut header = (0..257) + .map(|index| format!("column{index}")) + .collect::>() + .join(","); + header.push('\n'); + let source = MemorySource::new(header.into_bytes()); + + let inspection = support::inspect(&source, "text/csv").await?; + support::assert_malformed_reason(inspection, "column_limit_exceeded"); + Ok(()) +} + +#[tokio::test] +async fn malformed_quoted_csv_uses_the_text_fallback() -> Result<(), Box> { + let source = MemorySource::new(fixtures::csv_with_quotes_inside_unquoted_field()); + + let inspection = support::inspect(&source, "text/plain").await?; + support::assert_validated_media(inspection, "text/plain"); + Ok(()) +} + +#[tokio::test] +async fn csv_rejects_truncated_quoted_field_as_typed_malformed() -> Result<(), Box> { + let source = MemorySource::new(fixtures::truncated_csv()); + + let inspection = support::inspect(&source, "text/csv").await?; + support::assert_malformed_reason(inspection, "malformed_csv"); + Ok(()) +} + +#[tokio::test] +async fn unprobed_declared_csv_candidate_uses_text_fallback() -> Result<(), Box> { + let source = MemorySource::new(b"hello".to_vec()); + + let inspection = support::inspect(&source, "text/csv").await?; + support::assert_declared_mismatch( + inspection, + DeclaredMismatchExpectation { + declared: "text/csv", + detected: "text/plain", + }, + ); + Ok(()) +} + +#[tokio::test] +async fn csv_rejects_quotes_inside_an_unquoted_field() -> Result<(), Box> { + let source = MemorySource::new(fixtures::csv_with_quotes_inside_unquoted_field()); + + let inspection = support::inspect(&source, "text/csv").await?; + support::assert_malformed_reason(inspection, "malformed_csv"); + Ok(()) +} + +#[tokio::test] +async fn csv_rejects_a_blank_record_as_typed_malformed() -> Result<(), Box> { + let source = MemorySource::new(fixtures::csv_with_blank_record()); + + let inspection = support::inspect(&source, "text/csv").await?; + support::assert_malformed_reason(inspection, "malformed_csv"); + Ok(()) +} + +#[tokio::test] +async fn comma_bearing_prose_uses_the_text_fallback() -> Result<(), Box> { + let bytes = fixtures::prose_with_comma_and_newline(); + let expected = std::str::from_utf8(&bytes)?.to_owned(); + let source = MemorySource::new(bytes); + + let inspection = support::inspect(&source, "text/plain").await?; + support::assert_validated_media(inspection, "text/plain"); + let result = support::read( + &source, + ReadInput { + media_type: "text/plain", + view: "text", + }, + &DirectProcessor::provider(), + ) + .await?; + support::assert_text(result, &expected); + Ok(()) +} + +#[tokio::test] +async fn complete_csv_probe_validates_all_records_before_claiming() -> Result<(), Box> { + let source = MemorySource::new(b"a,b\nc,d\nplain prose\n".to_vec()); + + let inspection = support::inspect(&source, "text/plain").await?; + support::assert_validated_media(inspection, "text/plain"); + Ok(()) +} + +#[tokio::test] +async fn csv_probe_ignores_a_partial_trailing_record() -> Result<(), Box> { + let source = MemorySource::new(fixtures::csv_with_partial_third_probe_record()); + + let inspection = support::inspect(&source, "text/plain").await?; + support::assert_declared_mismatch( + inspection, + DeclaredMismatchExpectation { + declared: "text/plain", + detected: "text/csv", + }, + ); + Ok(()) +} + +#[tokio::test] +async fn truncated_csv_probe_with_later_prose_resumes_text_fallback() -> Result<(), Box> +{ + let mut bytes = b"name,value\nalpha,1\n".to_vec(); + while bytes.len() < 4_096 { + bytes.extend_from_slice(b"beta,2\n"); + } + bytes.extend_from_slice(b"plain prose\n"); + let source = MemorySource::new(bytes); + + let inspection = support::inspect(&source, "text/plain").await?; + support::assert_validated_media(inspection, "text/plain"); + Ok(()) +} + +#[tokio::test] +async fn csv_probe_rejects_a_partial_second_record() -> Result<(), Box> { + let source = MemorySource::new(fixtures::csv_with_partial_second_probe_record()); + + let inspection = support::inspect(&source, "text/plain").await?; + support::assert_validated_media(inspection, "text/plain"); + Ok(()) +} + +#[tokio::test] +async fn csv_probe_handles_a_utf8_scalar_split_at_its_boundary() -> Result<(), Box> { + let source = MemorySource::new(fixtures::csv_with_scalar_split_at_probe_boundary()); + + let inspection = support::inspect(&source, "text/plain").await?; + support::assert_declared_mismatch( + inspection, + DeclaredMismatchExpectation { + declared: "text/plain", + detected: "text/csv", + }, + ); + Ok(()) +} + +#[tokio::test] +async fn csv_rejects_row_bomb_shape_at_declared_ceiling() -> Result<(), Box> { + let source = MemorySource::new(fixtures::row_bomb_csv()); + + let inspection = support::inspect(&source, "text/csv").await?; + support::assert_malformed_reason(inspection, "row_limit_exceeded"); + Ok(()) +} + +#[tokio::test] +async fn declared_one_column_csv_preserves_the_row_limit_reason() -> Result<(), Box> { + let source = MemorySource::new(fixtures::one_column_row_bomb_csv()); + + let inspection = support::inspect(&source, "text/csv").await?; + support::assert_malformed_reason(inspection, "row_limit_exceeded"); + Ok(()) +} + +#[tokio::test] +async fn text_family_registration_accepts_the_exact_probe_work_ceiling() +-> Result<(), Box> { + let source = MemorySource::new(fixtures::json_document()); + let mut ceilings = FileMediaCeilings::version_one(); + ceilings.probe_cumulative_bytes = 4_096; + + let inspection = support::inspect_with_ceilings(&source, "application/json", ceilings).await?; + support::assert_validated_media(inspection, "application/json"); + Ok(()) +} + +#[tokio::test] +async fn csv_rejects_oversized_input_with_registered_reason() -> Result<(), Box> { + let mut bytes = fixtures::oversized(b'a'); + bytes[..8].copy_from_slice(b"a,b\nc,d\n"); + let source = MemorySource::new(bytes); + + let inspection = support::inspect(&source, "text/csv").await?; + support::assert_malformed_reason(inspection, "source_too_large"); + Ok(()) +} + +#[tokio::test] +async fn oversized_declared_one_column_csv_preserves_the_size_reason() -> Result<(), Box> +{ + let source = MemorySource::new(fixtures::oversized_one_column_csv()); + + let inspection = support::inspect(&source, "text/csv").await?; + support::assert_malformed_reason(inspection, "source_too_large"); + Ok(()) +} + +#[tokio::test] +async fn csv_read_honors_the_effective_container_entry_ceiling() -> Result<(), Box> { + let source = MemorySource::new(fixtures::csv_table()); + let expected = ReasonCode::try_new("container_entry_limit_exceeded")?; + let mut ceilings = FileMediaCeilings::version_one(); + ceilings.observed_container_entries = 1; + + let result = support::read_with_ceilings( + &source, + ReadInput { + media_type: "text/csv", + view: "structured", + }, + &DirectProcessor::provider(), + ceilings, + ) + .await; + assert_eq!( + result, + Err(FileMediaFailure::ExpansionLimitExceeded { + limit_kind: expected + }) + ); + Ok(()) +} + +#[tokio::test] +async fn registry_sanitizer_keeps_injection_shaped_json_as_data() -> Result<(), Box> { + let source = MemorySource::new(fixtures::json_document()); + let expected = serde_json::json!({ + "path":"../../etc/passwd", + "text":"" + }); + let decoder_output = serde_json::to_string(&expected)?; + + let result = support::read( + &source, + ReadInput { + media_type: "application/json", + view: "structured", + }, + &DirectProcessor::injecting(decoder_output), + ) + .await?; + support::assert_structured(result, &expected); + Ok(()) +} + +#[tokio::test] +async fn registry_sanitizer_rejects_nul_bearing_decoder_output() -> Result<(), Box> { + let source = MemorySource::new(fixtures::json_document()); + let decoder_output = String::from("{\"text\":\"prefix\0suffix\"}"); + + let result = support::read( + &source, + ReadInput { + media_type: "application/json", + view: "structured", + }, + &DirectProcessor::injecting(decoder_output), + ) + .await; + support::assert_processor_failed(result); + Ok(()) +} diff --git a/crates/file-media-adapters-text/tests/fixtures/mod.rs b/crates/file-media-adapters-text/tests/fixtures/mod.rs new file mode 100644 index 0000000000..d0e4c2813e --- /dev/null +++ b/crates/file-media-adapters-text/tests/fixtures/mod.rs @@ -0,0 +1,223 @@ +pub(crate) fn utf8_text() -> Vec { + "alpha\nβeta\n".as_bytes().to_vec() +} + +pub(crate) fn truncated_utf8() -> Vec { + vec![b'a', 0xe2, 0x82] +} + +/// A complete source: valid JSON followed by an incomplete UTF-8 scalar. +/// +/// The trailing byte is real content rather than a probe-boundary artifact, so +/// the source is not valid UTF-8 and no JSON candidate may be claimed from the +/// shortened text. +pub(crate) fn json_then_incomplete_scalar() -> Vec { + let mut bytes = br#"{"a":1}"#.to_vec(); + bytes.push(0xc3); + bytes +} + +/// A complete source: valid CSV records followed by an incomplete UTF-8 scalar. +pub(crate) fn csv_then_incomplete_scalar() -> Vec { + let mut bytes = b"h1,h2\n1,2\n".to_vec(); + bytes.push(0xc3); + bytes +} + +pub(crate) fn json_document() -> Vec { + br#"{"name":"fixture","values":[1,2,3]}"#.to_vec() +} + +pub(crate) fn json_document_value() -> serde_json::Value { + serde_json::json!({"name":"fixture","values":[1,2,3]}) +} + +pub(crate) fn arbitrary_precision_json() -> Vec { + br#"{"decimal":1.2345678901234567890123456789,"integer":18446744073709551616}"#.to_vec() +} + +pub(crate) fn truncated_json() -> Vec { + br#"{"name":"fixture""#.to_vec() +} + +pub(crate) fn duplicate_member_json() -> Vec { + br#"{"role":"user","role":"admin"}"#.to_vec() +} + +pub(crate) fn deep_json_with_duplicate_root_member() -> Vec { + format!( + "{{\"role\":\"user\",\"role\":\"admin\",\"deep\":{}0{}}}", + "[".repeat(65), + "]".repeat(65) + ) + .into_bytes() +} + +pub(crate) fn pretty_json_document() -> Vec { + b"{\n \"name\": \"fixture\",\n \"values\": [1, 2, 3]\n}\n".to_vec() +} + +pub(crate) fn json_array_formatted_like_csv() -> Vec { + b"[[1,2],\n[3,4],\n[5,6]]".to_vec() +} + +pub(crate) fn csv_with_json_consistent_truncated_prefix() -> Vec { + let mut bytes = b"[0,0,0,\n".to_vec(); + while bytes.len() <= 4_096 { + bytes.extend_from_slice(b"0,0,0,\n"); + } + bytes.extend_from_slice(b"x,0,0,\n"); + bytes +} + +pub(crate) fn json_with_csv_consistent_truncated_prefix() -> Vec { + let mut bytes = b"[[1,2],\n[3,4],\n".to_vec(); + while bytes.len() <= 4_096 { + bytes.extend_from_slice(b"[5,6],\n"); + } + bytes.extend_from_slice(b"[7,8]]"); + bytes +} + +pub(crate) fn bracket_prefixed_prose() -> Vec { + b"[section]\nbody".to_vec() +} + +pub(crate) fn json_token_prefixed_prose() -> Vec { + b"[todo]\nbody".to_vec() +} + +pub(crate) fn json_at_structured_depth() -> Vec { + format!("{}0{}", "[".repeat(64), "]".repeat(64)).into_bytes() +} + +pub(crate) fn json_beyond_structured_depth() -> Vec { + format!("{}0{}", "[".repeat(65), "]".repeat(65)).into_bytes() +} + +pub(crate) fn json_beyond_serde_recursion_limit() -> Vec { + format!("{{\"value\":{}0{}}}", "[".repeat(128), "]".repeat(128)).into_bytes() +} + +pub(crate) fn bracketed_numeric_csv() -> Vec { + b"[1,2\n3,4\n".to_vec() +} + +pub(crate) fn complete_json_arrays_as_csv() -> Vec { + b"[1,2]\n[3,4]\n".to_vec() +} + +pub(crate) fn complete_json_array_followed_by_prose() -> Vec { + b"[1,2]\nbody".to_vec() +} + +pub(crate) fn complete_json_prefix_followed_outside_probe() -> Vec { + let mut bytes = b"[1,2]".to_vec(); + bytes.extend_from_slice(&vec![b' '; 4_091]); + bytes.extend_from_slice(b"body"); + bytes +} + +pub(crate) fn json_with_scalar_split_at_probe_boundary() -> Vec { + let mut bytes = br#"{"padding":""#.to_vec(); + bytes.extend_from_slice(&vec![b'a'; 4_095 - bytes.len()]); + bytes.extend_from_slice("β\",\"value\":1}".as_bytes()); + bytes +} + +pub(crate) fn deeply_nested_json_within_source_ceiling() -> Vec { + format!("{}0{}", "[".repeat(60_000), "]".repeat(60_000)).into_bytes() +} + +pub(crate) fn json_beyond_container_entry_ceiling() -> Vec { + let mut bytes = b"[".to_vec(); + bytes.extend_from_slice(b"0,".repeat(10_000).as_slice()); + bytes.extend_from_slice(b"0]"); + bytes +} + +pub(crate) fn csv_table() -> Vec { + b"name,value\nalpha,1\nbeta,2\n".to_vec() +} + +pub(crate) fn csv_table_value() -> serde_json::Value { + serde_json::json!({ + "headers":["name","value"], + "rows":[["alpha","1"],["beta","2"]] + }) +} + +pub(crate) fn one_column_csv() -> Vec { + b"header\nvalue\n".to_vec() +} + +pub(crate) fn header_only_csv() -> Vec { + b"name,value\n".to_vec() +} + +pub(crate) fn truncated_csv() -> Vec { + b"name,value\nalpha,\"unterminated\n".to_vec() +} + +pub(crate) fn csv_with_quotes_inside_unquoted_field() -> Vec { + b"h1,h2\nab\"cd\"ef,x\n".to_vec() +} + +pub(crate) fn csv_with_blank_record() -> Vec { + b"h1,h2\nv1,v2\n\nv3,v4\n".to_vec() +} + +pub(crate) fn prose_with_comma_and_newline() -> Vec { + b"Hello, world\nnext line".to_vec() +} + +pub(crate) fn csv_with_partial_third_probe_record() -> Vec { + let mut bytes = b"name,value\n".to_vec(); + bytes.extend_from_slice(&vec![b'a'; 4_060]); + bytes.extend_from_slice(b",1\nthird,\""); + bytes.extend_from_slice(&[b'b'; 100]); + bytes.extend_from_slice(b"\"\n"); + bytes +} + +pub(crate) fn csv_with_partial_second_probe_record() -> Vec { + let mut bytes = b"h1,h2\n".to_vec(); + bytes.extend_from_slice(&vec![b'a'; 4_091]); + bytes.extend_from_slice(b",z\n"); + bytes +} + +pub(crate) fn oversized_one_column_csv() -> Vec { + let mut bytes = b"header\nvalue\n".to_vec(); + while bytes.len() <= signalbox_file_media_adapters_text::MAX_TEXT_FAMILY_BYTES as usize { + bytes.extend_from_slice(b"value\n"); + } + bytes +} + +pub(crate) fn csv_with_scalar_split_at_probe_boundary() -> Vec { + let mut bytes = b"h1,h2\nv1,v2\nthird,".to_vec(); + bytes.extend_from_slice(&vec![b'a'; 4_095 - bytes.len()]); + bytes.extend_from_slice("β\n".as_bytes()); + bytes +} + +pub(crate) fn row_bomb_csv() -> Vec { + let mut bytes = b"name,value\n".to_vec(); + for _ in 0..10_001 { + bytes.extend_from_slice(b"a,1\n"); + } + bytes +} + +pub(crate) fn one_column_row_bomb_csv() -> Vec { + let mut bytes = b"name\n".to_vec(); + for _ in 0..10_001 { + bytes.extend_from_slice(b"value\n"); + } + bytes +} + +pub(crate) fn oversized(fill: u8) -> Vec { + vec![fill; signalbox_file_media_adapters_text::MAX_TEXT_FAMILY_BYTES as usize + 1] +} diff --git a/crates/file-media-adapters-text/tests/support/mod.rs b/crates/file-media-adapters-text/tests/support/mod.rs new file mode 100644 index 0000000000..91e2e5cabd --- /dev/null +++ b/crates/file-media-adapters-text/tests/support/mod.rs @@ -0,0 +1,295 @@ +use std::{error::Error, num::NonZeroU64, sync::Arc}; + +use signalbox_file_media_adapters_text::{TextFamilyProvider, text_family_declaration}; +use signalbox_file_media_runtime::{ + AttachmentKind, CancellationSignal, DeclaredMediaType, FileDigest, FileInspection, + FileMediaCeilings, FileMediaFailure, FileMediaProcessor, FileMediaProcessorFuture, + FileMediaProvider, FileMediaProviderReadRequest, FileMediaProviderValidationRequest, + FileMediaRegistry, FileReadInput, FileReadRequest, FileReadResult, FileUse, InspectionRequest, + NeverCancelled, ProcessorBoundaryFailure, ProcessorFailure, ProcessorIsolation, + ProcessorProbeOutput, ProcessorReadOutput, ProcessorValidationOutput, ReadViewName, + ReaderIdentity, SourceReadError, SourceReadFuture, VerifiedBlobSource, +}; + +pub(crate) struct MemorySource { + bytes: Arc<[u8]>, +} + +impl MemorySource { + pub(crate) fn new(bytes: Vec) -> Self { + Self { + bytes: Arc::from(bytes), + } + } + + pub(crate) fn file_use(&self, media_type: &str) -> Result> { + Ok(FileUse::new( + self.digest(), + self.byte_length(), + AttachmentKind::Document, + DeclaredMediaType::try_new(media_type)?, + None, + )) + } +} + +impl VerifiedBlobSource for MemorySource { + fn digest(&self) -> FileDigest { + FileDigest::from_bytes([7; 32]) + } + + fn byte_length(&self) -> NonZeroU64 { + NonZeroU64::new(self.bytes.len() as u64).unwrap_or(NonZeroU64::MIN) + } + + fn read_range(&self, offset: u64, length: NonZeroU64) -> SourceReadFuture<'_> { + Box::pin(async move { + let start = usize::try_from(offset).map_err(|_| SourceReadError::RangeOutOfBounds)?; + let requested = + usize::try_from(length.get()).map_err(|_| SourceReadError::RangeOutOfBounds)?; + let end = start + .checked_add(requested) + .ok_or(SourceReadError::RangeOutOfBounds)?; + self.bytes + .get(start..end) + .map(<[u8]>::to_vec) + .ok_or(SourceReadError::RangeOutOfBounds) + }) + } +} + +#[derive(Clone, Debug)] +pub(crate) enum ReadBehavior { + Provider, + InjectedStructured(String), +} + +pub(crate) struct DirectProcessor { + provider: TextFamilyProvider, + read_behavior: ReadBehavior, +} + +impl DirectProcessor { + pub(crate) fn provider() -> Self { + Self { + provider: TextFamilyProvider, + read_behavior: ReadBehavior::Provider, + } + } + + pub(crate) fn injecting(body_json: String) -> Self { + Self { + provider: TextFamilyProvider, + read_behavior: ReadBehavior::InjectedStructured(body_json), + } + } +} + +impl FileMediaProcessor for DirectProcessor { + fn probe<'a>( + &'a self, + reader: &'a ReaderIdentity, + source: &'a dyn VerifiedBlobSource, + cancellation: &'a dyn CancellationSignal, + ) -> FileMediaProcessorFuture<'a, ProcessorProbeOutput> { + Box::pin(async move { + self.provider + .probe(reader, source, cancellation) + .await + .map_err(|_| ProcessorBoundaryFailure::Processor(ProcessorFailure::Failed)) + }) + } + + fn validate<'a>( + &'a self, + reader: &'a ReaderIdentity, + request: FileMediaProviderValidationRequest, + source: &'a dyn VerifiedBlobSource, + cancellation: &'a dyn CancellationSignal, + ) -> FileMediaProcessorFuture<'a, ProcessorValidationOutput> { + Box::pin(async move { + self.provider + .inspect(reader, request, source, cancellation) + .await + .map_err(|_| ProcessorBoundaryFailure::Processor(ProcessorFailure::Failed)) + }) + } + + fn read<'a>( + &'a self, + reader: &'a ReaderIdentity, + request: FileMediaProviderReadRequest, + source: &'a dyn VerifiedBlobSource, + cancellation: &'a dyn CancellationSignal, + ) -> FileMediaProcessorFuture<'a, ProcessorReadOutput> { + match &self.read_behavior { + ReadBehavior::Provider => Box::pin(async move { + self.provider + .read(reader, request, source, cancellation) + .await + .map_err(|_| ProcessorBoundaryFailure::Processor(ProcessorFailure::Failed)) + }), + ReadBehavior::InjectedStructured(body_json) => { + let body_json = body_json.clone(); + Box::pin(async move { + Ok(ProcessorReadOutput::Structured { + body_json, + truncated: false, + cursor: None, + }) + }) + } + } + } +} + +pub(crate) fn registry_with_ceilings( + ceilings: FileMediaCeilings, +) -> Result> { + Ok(FileMediaRegistry::try_new( + vec![text_family_declaration().map_err(|error| error.to_string())?], + ceilings, + ProcessorIsolation::Available, + )?) +} + +pub(crate) async fn inspect( + source: &MemorySource, + media_type: &str, +) -> Result> { + inspect_with_ceilings(source, media_type, FileMediaCeilings::version_one()).await +} + +pub(crate) async fn inspect_with_ceilings( + source: &MemorySource, + media_type: &str, + ceilings: FileMediaCeilings, +) -> Result> { + Ok(registry_with_ceilings(ceilings)? + .inspect( + &DirectProcessor::provider(), + InspectionRequest { + source: source.file_use(media_type)?, + visible_part: None, + }, + source, + &NeverCancelled, + ) + .await?) +} + +pub(crate) struct ReadInput<'a> { + pub(crate) media_type: &'a str, + pub(crate) view: &'a str, +} + +pub(crate) async fn read( + source: &MemorySource, + input: ReadInput<'_>, + processor: &DirectProcessor, +) -> Result { + read_with_ceilings(source, input, processor, FileMediaCeilings::version_one()).await +} + +pub(crate) async fn read_with_ceilings( + source: &MemorySource, + input: ReadInput<'_>, + processor: &DirectProcessor, + ceilings: FileMediaCeilings, +) -> Result { + let source_use = source + .file_use(input.media_type) + .map_err(|_| FileMediaFailure::ProcessorFailed)?; + let view = ReadViewName::try_new(input.view).map_err(|_| FileMediaFailure::ProcessorFailed)?; + registry_with_ceilings(ceilings) + .map_err(|_| FileMediaFailure::ProcessorFailed)? + .read( + processor, + FileReadRequest { + inspection: InspectionRequest { + source: source_use, + visible_part: None, + }, + view, + input: FileReadInput::Initial { + options: serde_json::json!({}), + }, + }, + source, + &NeverCancelled, + ) + .await +} + +#[track_caller] +pub(crate) fn assert_validated_media(inspection: FileInspection, expected: &str) { + assert!(matches!(inspection, FileInspection::Validated(_))); + if let FileInspection::Validated(validated) = inspection { + assert_eq!(validated.detected_media_type().as_str(), expected); + } +} + +#[track_caller] +pub(crate) fn assert_malformed_reason(inspection: FileInspection, expected: &str) { + assert!(matches!(inspection, FileInspection::Malformed { .. })); + if let FileInspection::Malformed { reason_code, .. } = inspection { + assert_eq!(reason_code.as_str(), expected); + } +} + +#[track_caller] +pub(crate) fn assert_unknown(inspection: FileInspection) { + assert!(matches!(inspection, FileInspection::Unknown { .. })); +} + +pub(crate) struct DeclaredMismatchExpectation<'a> { + pub(crate) declared: &'a str, + pub(crate) detected: &'a str, +} + +#[track_caller] +pub(crate) fn assert_declared_mismatch( + inspection: FileInspection, + expected: DeclaredMismatchExpectation<'_>, +) { + assert!(matches!( + inspection, + FileInspection::DeclaredMismatch { .. } + )); + if let FileInspection::DeclaredMismatch { + declared, detected, .. + } = inspection + { + assert_eq!(declared.as_str(), expected.declared); + assert_eq!(detected.as_str(), expected.detected); + } +} + +#[track_caller] +pub(crate) fn assert_text(result: FileReadResult, expected: &str) { + assert!(matches!(result, FileReadResult::Text { .. })); + if let FileReadResult::Text { body, .. } = result { + assert_eq!(body, expected); + } +} + +#[track_caller] +pub(crate) fn assert_structured(result: FileReadResult, expected: &serde_json::Value) { + assert!(matches!(result, FileReadResult::Structured { .. })); + if let FileReadResult::Structured { body, .. } = result { + assert_eq!(&body, expected); + } +} + +#[track_caller] +pub(crate) fn assert_structured_json(result: FileReadResult, expected: &str) { + assert!(matches!(result, FileReadResult::Structured { .. })); + if let FileReadResult::Structured { body, .. } = result { + assert_eq!(body.to_string(), expected); + } +} + +#[track_caller] +pub(crate) fn assert_processor_failed(result: Result) { + assert_eq!(result, Err(FileMediaFailure::ProcessorFailed)); +} diff --git a/crates/file-media-linux-sandbox/Cargo.toml b/crates/file-media-linux-sandbox/Cargo.toml new file mode 100644 index 0000000000..2dc9425ce0 --- /dev/null +++ b/crates/file-media-linux-sandbox/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "signalbox-file-media-linux-sandbox" +version = "0.0.0" +edition = "2024" +rust-version = "1.97.0" +license = "MIT" +repository = "https://github.com/KeenWill/signalbox" +publish = false + +[workspace] + +[dependencies] +libc = "0.2.186" + +[lints.rust] +# This excluded crate is the separately governed Linux syscall boundary. +unsafe_code = "allow" diff --git a/crates/file-media-linux-sandbox/src/lib.rs b/crates/file-media-linux-sandbox/src/lib.rs new file mode 100644 index 0000000000..ef67ba41ed --- /dev/null +++ b/crates/file-media-linux-sandbox/src/lib.rs @@ -0,0 +1,150 @@ +//! Narrow Linux syscall boundary for the file-media worker sandbox. +//! +//! The main workspace forbids unsafe code. This separately governed crate owns +//! only pre-exec registration, descriptor operations, keyring detachment, and +//! sealed executable-memory operations that require unsafe Linux APIs. + +use std::{ + fs::File, + io, + os::{ + fd::{AsRawFd as _, FromRawFd as _}, + unix::process::CommandExt as _, + }, + process::Command, +}; + +/// Labeled child setup applied after fork and before bubblewrap executes. +#[derive(Clone, Copy, Debug)] +pub struct ChildSetup { + /// Address-space byte limit inherited by the sandbox tree. + pub address_space_bytes: u64, + /// CPU-second limit inherited by the sandbox tree. + pub cpu_seconds: u64, + /// Descriptor limit inherited by the sandbox tree. + pub file_descriptors: u64, + /// Seccomp descriptor made inheritable only in the forked child. + pub seccomp_fd: i32, + /// Startup-gate descriptor made inheritable only in the forked child. + pub startup_gate_fd: i32, + /// Sealed worker snapshot made inheritable only in the forked child. + pub worker_fd: i32, + /// Writable `cgroup.procs` descriptor for this invocation's delegated cgroup. + pub cgroup_procs_fd: i32, +} + +/// Registers the reviewed child-only setup on one command. +pub fn install_pre_exec(command: &mut Command, setup: ChildSetup) { + // SAFETY: the closure captures only copyable scalar values and invokes only + // async-signal-safe Linux syscalls before exec. Every descriptor remains + // owned by the parent command setup until spawn completes. + unsafe { + command.pre_exec(move || prepare_child(setup)); + } +} + +fn prepare_child(setup: ChildSetup) -> io::Result<()> { + enter_cgroup(setup.cgroup_procs_fd)?; + set_limit(libc::RLIMIT_AS, setup.address_space_bytes)?; + set_limit(libc::RLIMIT_CPU, setup.cpu_seconds)?; + set_limit(libc::RLIMIT_CORE, 0)?; + set_limit(libc::RLIMIT_NOFILE, setup.file_descriptors)?; + inherit_descriptor(setup.seccomp_fd)?; + inherit_descriptor(setup.startup_gate_fd)?; + inherit_descriptor(setup.worker_fd)?; + detach_session_keyring() +} + +fn enter_cgroup(cgroup_procs_fd: i32) -> io::Result<()> { + let current_process = b"0\n"; + // SAFETY: the parent retains an open writable `cgroup.procs` descriptor; + // writing zero moves this child into that cgroup before bubblewrap executes. + let written = unsafe { + libc::write( + cgroup_procs_fd, + current_process.as_ptr().cast(), + current_process.len(), + ) + }; + if written == current_process.len() as isize { + Ok(()) + } else if written == -1 { + Err(io::Error::last_os_error()) + } else { + Err(io::Error::new( + io::ErrorKind::WriteZero, + "short write while entering invocation cgroup", + )) + } +} + +fn set_limit(resource: libc::__rlimit_resource_t, value: u64) -> io::Result<()> { + let limit = libc::rlimit { + rlim_cur: value as libc::rlim_t, + rlim_max: value as libc::rlim_t, + }; + // SAFETY: `limit` points to a fully initialized value for this resource. + if unsafe { libc::setrlimit(resource, &limit) } == -1 { + Err(io::Error::last_os_error()) + } else { + Ok(()) + } +} + +fn inherit_descriptor(raw_fd: i32) -> io::Result<()> { + // SAFETY: command setup keeps the descriptor alive through this callback. + if unsafe { libc::fcntl(raw_fd, libc::F_SETFD, 0) } == -1 { + Err(io::Error::last_os_error()) + } else { + Ok(()) + } +} + +fn detach_session_keyring() -> io::Result<()> { + const KEYCTL_JOIN_SESSION_KEYRING: libc::c_long = 1; + // SAFETY: a null name creates and joins a fresh anonymous session keyring. + let result = unsafe { + libc::syscall( + libc::SYS_keyctl, + KEYCTL_JOIN_SESSION_KEYRING, + std::ptr::null::(), + 0 as libc::c_ulong, + 0 as libc::c_ulong, + 0 as libc::c_ulong, + ) + }; + if result < 0 { + Err(io::Error::last_os_error()) + } else { + Ok(()) + } +} + +/// Creates one close-on-exec sealable anonymous executable snapshot. +pub fn create_executable_snapshot() -> io::Result { + let name = b"signalbox-file-media-worker\0"; + // SAFETY: memfd_create receives a valid nul-terminated name and fixed flags. + let raw_fd = unsafe { + libc::syscall( + libc::SYS_memfd_create, + name.as_ptr().cast::(), + libc::MFD_CLOEXEC | libc::MFD_ALLOW_SEALING, + ) + }; + if raw_fd < 0 { + return Err(io::Error::last_os_error()); + } + // SAFETY: successful memfd_create returns a new owned descriptor. + Ok(unsafe { File::from_raw_fd(raw_fd as i32) }) +} + +/// Seals one snapshot against writes, growth, truncation, and seal changes. +pub fn seal_executable_snapshot(file: &File) -> io::Result<()> { + let seals = libc::F_SEAL_SEAL | libc::F_SEAL_SHRINK | libc::F_SEAL_GROW | libc::F_SEAL_WRITE; + // SAFETY: fcntl receives an owned descriptor and the documented seal mask. + if unsafe { libc::fcntl(file.as_raw_fd(), libc::F_ADD_SEALS, seals) } == -1 { + Err(io::Error::last_os_error()) + } else { + Ok(()) + } +} diff --git a/crates/file-media-processor-runtime/Cargo.toml b/crates/file-media-processor-runtime/Cargo.toml new file mode 100644 index 0000000000..875b2bc5d5 --- /dev/null +++ b/crates/file-media-processor-runtime/Cargo.toml @@ -0,0 +1,53 @@ +[package] +name = "signalbox-file-media-processor-runtime" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +repository.workspace = true +publish.workspace = true + +[features] +test-worker = [] + +[[bin]] +name = "signalbox-file-media-synthetic-worker" +path = "src/bin/signalbox-file-media-synthetic-worker.rs" +required-features = ["test-worker"] + +[[test]] +name = "isolation" +path = "tests/isolation.rs" +required-features = ["test-worker"] + +[dependencies] +base64 = "0.23.0" +serde = { version = "1.0.219", features = ["derive"] } +serde_json = { version = "1.0.140", features = ["unbounded_depth"] } +sha2 = "0.11.0" +signalbox-file-media-runtime = { path = "../file-media-runtime" } +tokio = { version = "1.53.0", default-features = false, features = [ + "io-std", + "io-util", + "macros", + "process", + "rt", + "sync", + "time", +] } + +[dev-dependencies] +tempfile = "3.27.0" + +[target.'cfg(target_os = "linux")'.dependencies] +libc = "0.2.186" +rustix = { version = "1.1.4", default-features = false, features = [ + "fs", + "pipe", + "process", + "std", +] } +signalbox-file-media-linux-sandbox = { path = "../file-media-linux-sandbox" } + +[lints] +workspace = true diff --git a/crates/file-media-processor-runtime/src/bin/signalbox-file-media-synthetic-worker.rs b/crates/file-media-processor-runtime/src/bin/signalbox-file-media-synthetic-worker.rs new file mode 100644 index 0000000000..916ef9ce95 --- /dev/null +++ b/crates/file-media-processor-runtime/src/bin/signalbox-file-media-synthetic-worker.rs @@ -0,0 +1,189 @@ +use std::{error::Error, fs, path::Path, str::FromStr, time::Duration}; + +use signalbox_file_media_processor_runtime::{WorkerCatalog, serve_one}; +use signalbox_file_media_runtime::{ + CanonicalJsonObjectSchema, CanonicalMediaType, FileMediaProvider, FileMediaProviderDeclaration, + FileMediaProviderFuture, FileMediaProviderReadRequest, FileMediaProviderValidationRequest, + FileReaderName, FileReaderProviderName, FileReaderRevision, ProbeDeclaration, + ProbeDeclarationInput, ProbeStrength, ProcessorProbeOutput, ProcessorReadOutput, + ProcessorValidationOutput, ReadAccessPattern, ReadOutputKind, ReadViewBounds, + ReadViewDeclaration, ReadViewName, ReaderDeclaration, ReaderDeclarationInput, ReaderIdentity, + ReasonCode, StreamingTextFallback, VerifiedBlobSource, +}; + +struct SyntheticProvider; + +impl FileMediaProvider for SyntheticProvider { + fn declaration(&self) -> FileMediaProviderDeclaration { + synthetic_declaration().unwrap_or_else(|error| { + eprintln!("synthetic declaration failed: {error}"); + std::process::exit(2); + }) + } + + fn probe<'a>( + &'a self, + _reader: &'a ReaderIdentity, + source: &'a dyn VerifiedBlobSource, + _cancellation: &'a dyn signalbox_file_media_runtime::CancellationSignal, + ) -> FileMediaProviderFuture<'a, ProcessorProbeOutput> { + Box::pin(async move { + let length = std::num::NonZeroU64::new(1) + .ok_or(signalbox_file_media_runtime::FileMediaProviderFailure::Failed)?; + let prefix = source + .read_range(0, length) + .await + .map_err(|_| signalbox_file_media_runtime::FileMediaProviderFailure::Failed)?; + match prefix.first().copied() { + Some(b'C') => std::process::exit(7), + Some(b'T') => std::thread::sleep(Duration::from_secs(5)), + Some(b'X') => { + let thread_output = std::thread::spawn(|| 1_u8).join().map_err(|_| { + signalbox_file_media_runtime::FileMediaProviderFailure::Failed + })?; + if thread_output != 1 { + return Err(signalbox_file_media_runtime::FileMediaProviderFailure::Failed); + } + let spawned = std::process::Command::new("/signalbox-file-media-worker") + .arg("--signalbox-file-media-isolation-probe") + .status(); + if spawned.is_ok() { + return Err(signalbox_file_media_runtime::FileMediaProviderFailure::Failed); + } + } + Some(b'I') => verify_sandbox_authority()?, + Some(b'V') => { + source.read_range(0, length).await.map_err(|_| { + signalbox_file_media_runtime::FileMediaProviderFailure::Failed + })?; + } + Some(b'H') => { + return Ok(ProcessorProbeOutput::Candidate { + media_type: String::from(""), + strength: ProbeStrength::Strong, + }); + } + Some(_) => {} + None => return Err(signalbox_file_media_runtime::FileMediaProviderFailure::Failed), + } + Ok(ProcessorProbeOutput::Candidate { + media_type: String::from("application/x-signalbox-synthetic"), + strength: ProbeStrength::Strong, + }) + }) + } + + fn inspect<'a>( + &'a self, + _reader: &'a ReaderIdentity, + request: FileMediaProviderValidationRequest, + _source: &'a dyn VerifiedBlobSource, + _cancellation: &'a dyn signalbox_file_media_runtime::CancellationSignal, + ) -> FileMediaProviderFuture<'a, ProcessorValidationOutput> { + Box::pin(async move { + Ok(ProcessorValidationOutput::Validated { + media_type: request.media_type.as_str().to_owned(), + evidence: request.evidence, + metadata_json: String::from("{}"), + }) + }) + } + + fn read<'a>( + &'a self, + _reader: &'a ReaderIdentity, + _request: FileMediaProviderReadRequest, + _source: &'a dyn VerifiedBlobSource, + _cancellation: &'a dyn signalbox_file_media_runtime::CancellationSignal, + ) -> FileMediaProviderFuture<'a, ProcessorReadOutput> { + Box::pin(async { + Ok(ProcessorReadOutput::Text { + body: String::from("synthetic"), + truncated: false, + cursor: None, + }) + }) + } +} + +fn verify_sandbox_authority() -> Result<(), signalbox_file_media_runtime::FileMediaProviderFailure> +{ + let failed = || signalbox_file_media_runtime::FileMediaProviderFailure::Failed; + if Path::new("/etc/passwd").exists() + || std::env::current_dir().map_err(|_| failed())? != Path::new("/tmp") + { + return Err(failed()); + } + let mut environment = std::env::vars().collect::>(); + environment.sort_unstable(); + if environment + != [ + (String::from("LANG"), String::from("C.UTF-8")), + (String::from("LC_ALL"), String::from("C.UTF-8")), + (String::from("PWD"), String::from("/tmp")), + ] + { + return Err(failed()); + } + let status = fs::read_to_string("/proc/self/status").map_err(|_| failed())?; + let capabilities = status + .lines() + .find_map(|line| line.strip_prefix("CapEff:")) + .ok_or_else(failed)?; + if u64::from_str_radix(capabilities.trim(), 16).map_err(|_| failed())? != 0 { + return Err(failed()); + } + let routes = fs::read_to_string("/proc/net/route").map_err(|_| failed())?; + if routes + .lines() + .skip(1) + .filter_map(|line| line.split_whitespace().next()) + .any(|interface| interface != "lo") + { + return Err(failed()); + } + Ok(()) +} + +fn synthetic_declaration() -> Result> { + let provider = FileReaderProviderName::try_new("synthetic")?; + let view = ReadViewDeclaration::try_new( + ReadViewName::try_new("text")?, + String::from("Reads synthetic text."), + CanonicalJsonObjectSchema::try_new(r#"{"type":"object"}"#)?, + ReadAccessPattern::Streaming { maximum_ranges: 1 }, + ReadViewBounds::Text { + source_bytes: 64, + output_bytes: 64, + }, + )?; + if view.output_kind() != ReadOutputKind::Text { + return Err("synthetic view kind drifted".into()); + } + let reader = ReaderDeclaration::try_new(ReaderDeclarationInput { + provider: provider.clone(), + reader: FileReaderName::try_new("fixture")?, + revision: FileReaderRevision::try_new("v1")?, + media_types: vec![CanonicalMediaType::from_str( + "application/x-signalbox-synthetic", + )?], + probe: ProbeDeclaration::new(ProbeDeclarationInput { + prefix_bytes: 1, + suffix_bytes: 0, + range_count: 1, + cumulative_bytes: 1, + }), + validation: signalbox_file_media_runtime::ValidationDeclaration::new(64, 1), + views: vec![view], + reason_codes: vec![ReasonCode::try_new("synthetic_failure")?], + streaming_text_fallback: StreamingTextFallback::Disabled, + })?; + FileMediaProviderDeclaration::try_new(provider, vec![reader]).map_err(Into::into) +} + +#[tokio::main(flavor = "current_thread")] +async fn main() -> Result<(), Box> { + let catalog = WorkerCatalog::try_new(vec![Box::new(SyntheticProvider)])?; + serve_one(&catalog).await?; + Ok(()) +} diff --git a/crates/file-media-processor-runtime/src/broker.rs b/crates/file-media-processor-runtime/src/broker.rs new file mode 100644 index 0000000000..98c3cfb364 --- /dev/null +++ b/crates/file-media-processor-runtime/src/broker.rs @@ -0,0 +1,381 @@ +use std::num::NonZeroU64; + +use serde::{Serialize, de::DeserializeOwned}; +use signalbox_file_media_runtime::MAX_PROCESSOR_FRAME_BYTES; +use tokio::io::{AsyncRead, AsyncReadExt as _, AsyncWrite, AsyncWriteExt as _}; + +use crate::protocol::WireReadEnvelope; + +// The stable read-input contract permits 256 option containers. Reserve room +// for the enclosing invocation and protocol frames while retaining a finite +// parser bound independent of serde_json's lower default recursion limit. +const MAX_FRAME_CONTAINER_DEPTH: usize = 272; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum BrokerError { + Eof, + Frame, + Range, +} + +pub(crate) struct RangeBroker { + source_bytes: u64, + maximum_range_bytes: u64, + envelope: WireReadEnvelope, + cumulative_bytes: u64, + range_count: u32, + arbitrary_range_count: u32, + prefix_read: bool, + suffix_read: bool, + stream_end_offset: u64, +} + +impl RangeBroker { + pub(crate) const fn new( + source_bytes: u64, + envelope: WireReadEnvelope, + maximum_range_bytes: u64, + ) -> Self { + Self { + source_bytes, + maximum_range_bytes, + envelope, + cumulative_bytes: 0, + range_count: 0, + arbitrary_range_count: 0, + prefix_read: false, + suffix_read: false, + stream_end_offset: 0, + } + } + + pub(crate) fn admit(&mut self, offset: u64, length: u64) -> Result { + let length = NonZeroU64::new(length).ok_or(BrokerError::Range)?; + if length.get() > self.maximum_range_bytes { + return Err(BrokerError::Range); + } + let end = offset.checked_add(length.get()).ok_or(BrokerError::Range)?; + if end > self.source_bytes { + return Err(BrokerError::Range); + } + let cumulative = self + .cumulative_bytes + .checked_add(length.get()) + .ok_or(BrokerError::Range)?; + let count = self.range_count.checked_add(1).ok_or(BrokerError::Range)?; + match self.envelope { + WireReadEnvelope::Probe { + prefix_bytes, + suffix_bytes, + ranges, + cumulative_bytes, + } => { + let suffix_start = self.source_bytes.saturating_sub(suffix_bytes); + let in_prefix = prefix_bytes > 0 && end <= prefix_bytes.min(self.source_bytes); + let in_suffix = suffix_bytes > 0 && offset >= suffix_start; + let use_prefix = in_prefix && !self.prefix_read; + let use_suffix = in_suffix && !self.suffix_read && !use_prefix; + let arbitrary = self + .arbitrary_range_count + .checked_add(u32::from(!use_prefix && !use_suffix)) + .ok_or(BrokerError::Range)?; + if arbitrary > ranges || cumulative > cumulative_bytes { + return Err(BrokerError::Range); + } + self.arbitrary_range_count = arbitrary; + self.prefix_read |= use_prefix; + self.suffix_read |= use_suffix; + } + WireReadEnvelope::Streaming { + ranges, + cumulative_bytes, + } => { + if offset < self.stream_end_offset + || count > ranges + || cumulative > cumulative_bytes + { + return Err(BrokerError::Range); + } + self.stream_end_offset = end; + } + WireReadEnvelope::RandomAccess { + ranges, + cumulative_bytes, + } => { + if count > ranges || cumulative > cumulative_bytes { + return Err(BrokerError::Range); + } + } + } + self.cumulative_bytes = cumulative; + self.range_count = count; + Ok(length) + } +} + +pub(crate) async fn write_frame( + writer: &mut Writer, + value: &Value, +) -> Result<(), BrokerError> +where + Writer: AsyncWrite + Unpin, + Value: Serialize, +{ + write_frame_with_limit(writer, value, MAX_PROCESSOR_FRAME_BYTES).await +} + +pub(crate) async fn write_frame_with_limit( + writer: &mut Writer, + value: &Value, + maximum_bytes: usize, +) -> Result<(), BrokerError> +where + Writer: AsyncWrite + Unpin, + Value: Serialize, +{ + let encoded = serde_json::to_vec(value).map_err(|_| BrokerError::Frame)?; + if encoded.len() > maximum_bytes || maximum_bytes > MAX_PROCESSOR_FRAME_BYTES { + return Err(BrokerError::Frame); + } + let length = u32::try_from(encoded.len()).map_err(|_| BrokerError::Frame)?; + writer + .write_all(&length.to_be_bytes()) + .await + .map_err(|_| BrokerError::Frame)?; + writer + .write_all(&encoded) + .await + .map_err(|_| BrokerError::Frame)?; + writer.flush().await.map_err(|_| BrokerError::Frame) +} + +pub(crate) async fn read_frame(reader: &mut Reader) -> Result +where + Reader: AsyncRead + Unpin, + Value: DeserializeOwned, +{ + read_frame_with_limit(reader, MAX_PROCESSOR_FRAME_BYTES).await +} + +pub(crate) async fn read_frame_with_limit( + reader: &mut Reader, + maximum_bytes: usize, +) -> Result +where + Reader: AsyncRead + Unpin, + Value: DeserializeOwned, +{ + let mut length = [0_u8; 4]; + let first = reader + .read(&mut length[..1]) + .await + .map_err(|_| BrokerError::Frame)?; + if first == 0 { + return Err(BrokerError::Eof); + } + reader + .read_exact(&mut length[1..]) + .await + .map_err(|_| BrokerError::Frame)?; + let length = usize::try_from(u32::from_be_bytes(length)).map_err(|_| BrokerError::Frame)?; + if length == 0 || length > maximum_bytes || maximum_bytes > MAX_PROCESSOR_FRAME_BYTES { + return Err(BrokerError::Frame); + } + let mut encoded = vec![0_u8; length]; + reader + .read_exact(&mut encoded) + .await + .map_err(|_| BrokerError::Frame)?; + if !json_container_depth_fits(&encoded) { + return Err(BrokerError::Frame); + } + let mut deserializer = serde_json::Deserializer::from_slice(&encoded); + deserializer.disable_recursion_limit(); + let value = + serde::Deserialize::deserialize(&mut deserializer).map_err(|_| BrokerError::Frame)?; + deserializer.end().map_err(|_| BrokerError::Frame)?; + Ok(value) +} + +fn json_container_depth_fits(encoded: &[u8]) -> bool { + let mut depth = 0_usize; + let mut in_string = false; + let mut escaped = false; + for byte in encoded { + if in_string { + if escaped { + escaped = false; + } else if *byte == b'\\' { + escaped = true; + } else if *byte == b'"' { + in_string = false; + } + continue; + } + match *byte { + b'"' => in_string = true, + b'{' | b'[' => { + depth = match depth.checked_add(1) { + Some(depth) if depth <= MAX_FRAME_CONTAINER_DEPTH => depth, + _ => return false, + }; + } + b'}' | b']' => { + depth = match depth.checked_sub(1) { + Some(depth) => depth, + None => return false, + }; + } + _ => {} + } + } + depth == 0 && !in_string && !escaped +} + +#[cfg(test)] +mod tests { + use super::{BrokerError, MAX_FRAME_CONTAINER_DEPTH, RangeBroker, read_frame_with_limit}; + use crate::protocol::WireReadEnvelope; + + fn nested_array_frame(depth: usize) -> Vec { + let encoded = format!("{}null{}", "[".repeat(depth), "]".repeat(depth)); + let mut frame = u32::try_from(encoded.len()).unwrap().to_be_bytes().to_vec(); + frame.extend_from_slice(encoded.as_bytes()); + frame + } + + #[tokio::test] + async fn frame_parser_accepts_the_documented_container_depth() { + let frame = nested_array_frame(256); + let mut input = frame.as_slice(); + let result = read_frame_with_limit::<_, serde_json::Value>(&mut input, frame.len()).await; + assert!(result.is_ok()); + } + + #[tokio::test] + async fn frame_parser_rejects_above_its_explicit_container_bound() { + let frame = nested_array_frame(MAX_FRAME_CONTAINER_DEPTH + 1); + let mut input = frame.as_slice(); + let result = read_frame_with_limit::<_, serde_json::Value>(&mut input, frame.len()).await; + assert_eq!(result, Err(BrokerError::Frame)); + } + + #[test] + fn probe_envelope_rejects_an_extra_arbitrary_range() { + let mut broker = RangeBroker::new( + 1_000, + WireReadEnvelope::Probe { + prefix_bytes: 10, + suffix_bytes: 10, + ranges: 1, + cumulative_bytes: 40, + }, + 100, + ); + assert!(broker.admit(0, 10).is_ok()); + assert!(broker.admit(500, 10).is_ok()); + assert!(broker.admit(990, 10).is_ok()); + assert_eq!(broker.admit(600, 1), Err(BrokerError::Range)); + } + + #[test] + fn probe_envelope_rejects_a_repeated_prefix_read() { + let mut broker = RangeBroker::new( + 1_000, + WireReadEnvelope::Probe { + prefix_bytes: 10, + suffix_bytes: 0, + ranges: 0, + cumulative_bytes: 20, + }, + 100, + ); + assert!(broker.admit(0, 10).is_ok()); + assert_eq!(broker.admit(0, 10), Err(BrokerError::Range)); + } + + #[test] + fn probe_envelope_rejects_a_repeated_suffix_read() { + let mut broker = RangeBroker::new( + 1_000, + WireReadEnvelope::Probe { + prefix_bytes: 0, + suffix_bytes: 10, + ranges: 0, + cumulative_bytes: 20, + }, + 100, + ); + assert!(broker.admit(990, 10).is_ok()); + assert_eq!(broker.admit(990, 10), Err(BrokerError::Range)); + } + + #[test] + fn streaming_envelope_rejects_nonmonotonic_access() { + let mut broker = RangeBroker::new( + 100, + WireReadEnvelope::Streaming { + ranges: 2, + cumulative_bytes: 100, + }, + 100, + ); + assert!(broker.admit(0, 10).is_ok()); + assert_eq!(broker.admit(9, 10), Err(BrokerError::Range)); + } + + #[test] + fn streaming_envelope_rejects_range_fanout_excess() { + let mut broker = RangeBroker::new( + 100, + WireReadEnvelope::Streaming { + ranges: 1, + cumulative_bytes: 100, + }, + 100, + ); + assert!(broker.admit(0, 10).is_ok()); + assert_eq!(broker.admit(10, 10), Err(BrokerError::Range)); + } + + #[test] + fn streaming_envelope_accepts_a_nonzero_start_and_forward_gap() { + let mut broker = RangeBroker::new( + 100, + WireReadEnvelope::Streaming { + ranges: 2, + cumulative_bytes: 20, + }, + 100, + ); + assert!(broker.admit(20, 10).is_ok()); + assert!(broker.admit(40, 10).is_ok()); + } + + #[test] + fn random_envelope_rejects_cumulative_excess() { + let mut broker = RangeBroker::new( + 100, + WireReadEnvelope::RandomAccess { + ranges: 2, + cumulative_bytes: 10, + }, + 100, + ); + assert!(broker.admit(90, 6).is_ok()); + assert_eq!(broker.admit(0, 5), Err(BrokerError::Range)); + } + + #[test] + fn frame_envelope_rejects_one_oversized_source_reply() { + let mut broker = RangeBroker::new( + 1_000, + WireReadEnvelope::Streaming { + ranges: 10, + cumulative_bytes: 1_000, + }, + 100, + ); + assert_eq!(broker.admit(0, 101), Err(BrokerError::Range)); + } +} diff --git a/crates/file-media-processor-runtime/src/lib.rs b/crates/file-media-processor-runtime/src/lib.rs new file mode 100644 index 0000000000..af1bea9941 --- /dev/null +++ b/crates/file-media-processor-runtime/src/lib.rs @@ -0,0 +1,23 @@ +//! Daemon-supervised isolation for untrusted file/media adapters. +//! +//! The daemon launches one fresh local worker per operation, brokers the sole +//! verified source capability, and discards every result unless framing, +//! process exit, and cleanup all complete successfully. + +mod broker; +mod protocol; +#[cfg(target_os = "linux")] +mod sandbox; +#[cfg(not(target_os = "linux"))] +mod unsupported; +mod worker; + +#[cfg(target_os = "linux")] +pub use sandbox::{ + SandboxedFileMediaProcessor, SandboxedFileMediaProcessorConstructionError, WorkerBinding, +}; +#[cfg(not(target_os = "linux"))] +pub use unsupported::{ + SandboxedFileMediaProcessor, SandboxedFileMediaProcessorConstructionError, WorkerBinding, +}; +pub use worker::{WorkerCatalog, WorkerCatalogConstructionError, WorkerServiceError, serve_one}; diff --git a/crates/file-media-processor-runtime/src/protocol.rs b/crates/file-media-processor-runtime/src/protocol.rs new file mode 100644 index 0000000000..05adf6f970 --- /dev/null +++ b/crates/file-media-processor-runtime/src/protocol.rs @@ -0,0 +1,688 @@ +use std::{borrow::Borrow, num::NonZeroU64, str::FromStr}; + +use base64::{Engine as _, engine::general_purpose::STANDARD}; +use serde::{Deserialize, Serialize}; +use sha2::{Digest as _, Sha256}; +use signalbox_file_media_runtime::{ + AttachmentKind, BoundedMetadata, CanonicalMediaType, DeclaredMediaType, DisplayFilename, + FileDigest, FileMediaProviderDeclaration, FileMediaProviderReadRequest, + FileMediaProviderValidationRequest, FileReaderName, FileReaderProviderName, FileReaderRevision, + FileUse, ProbeDeclaration, ProcessorProbeOutput, ProcessorReadOutput, + ProcessorValidationOutput, ReadAccessPattern, ReadContinuationCursor, ReadOutputKind, + ReadViewBounds, ReadViewDeclaration, ReadViewName, ReaderIdentity, RegistryValueError, + StreamingTextFallback, ValidationEvidence, +}; + +pub(crate) fn declaration_fingerprint(declarations: &[FileMediaProviderDeclaration]) -> [u8; 32] { + let mut declarations = declarations.iter().collect::>(); + declarations.sort_by(|left, right| left.provider().cmp(right.provider())); + declaration_fingerprint_ordered(declarations.len(), declarations) +} + +pub(crate) fn declaration_fingerprint_ordered( + declaration_count: usize, + declarations: I, +) -> [u8; 32] +where + I: IntoIterator, + D: Borrow, +{ + let mut fingerprint = Sha256::new(); + fingerprint_field(&mut fingerprint, b"signalbox-file-media-catalog-v1"); + fingerprint_len(&mut fingerprint, declaration_count); + for declaration in declarations { + let declaration = declaration.borrow(); + fingerprint_field(&mut fingerprint, declaration.provider().as_str().as_bytes()); + let mut readers = declaration.readers().iter().collect::>(); + readers.sort_by(|left, right| left.identity().cmp(right.identity())); + fingerprint_len(&mut fingerprint, readers.len()); + for reader in readers { + fingerprint_field( + &mut fingerprint, + reader.identity().provider().as_str().as_bytes(), + ); + fingerprint_field( + &mut fingerprint, + reader.identity().reader().as_str().as_bytes(), + ); + fingerprint_field( + &mut fingerprint, + reader.identity().revision().as_str().as_bytes(), + ); + let mut media_types = reader.media_types().iter().collect::>(); + media_types.sort(); + fingerprint_len(&mut fingerprint, media_types.len()); + for media_type in media_types { + fingerprint_field(&mut fingerprint, media_type.as_str().as_bytes()); + } + let probe = reader.probe(); + fingerprint_u64(&mut fingerprint, probe.prefix_bytes()); + fingerprint_u64(&mut fingerprint, probe.suffix_bytes()); + fingerprint_u64(&mut fingerprint, u64::from(probe.range_count())); + fingerprint_u64(&mut fingerprint, probe.cumulative_bytes()); + let validation = reader.validation(); + fingerprint_u64(&mut fingerprint, validation.source_bytes()); + fingerprint_u64(&mut fingerprint, u64::from(validation.range_count())); + fingerprint_len(&mut fingerprint, reader.views().len()); + for view in reader.views() { + fingerprint_field(&mut fingerprint, view.name().as_str().as_bytes()); + fingerprint_field(&mut fingerprint, view.description().as_bytes()); + fingerprint_field( + &mut fingerprint, + view.arguments_schema().as_str().as_bytes(), + ); + match view.access() { + ReadAccessPattern::Streaming { maximum_ranges } => { + fingerprint_field(&mut fingerprint, b"streaming"); + fingerprint_u64(&mut fingerprint, u64::from(maximum_ranges)); + } + ReadAccessPattern::RandomAccess { maximum_ranges } => { + fingerprint_field(&mut fingerprint, b"random_access"); + fingerprint_u64(&mut fingerprint, u64::from(maximum_ranges)); + } + } + fingerprint_field( + &mut fingerprint, + match view.output_kind() { + ReadOutputKind::Text => b"text", + ReadOutputKind::Structured => b"structured", + ReadOutputKind::Image => b"image", + ReadOutputKind::Audio => b"audio", + ReadOutputKind::File => b"file", + }, + ); + fingerprint_view_bounds(&mut fingerprint, view.bounds()); + } + let mut reason_codes = reader.reason_codes().iter().collect::>(); + reason_codes.sort(); + fingerprint_len(&mut fingerprint, reason_codes.len()); + for reason in reason_codes { + fingerprint_field(&mut fingerprint, reason.as_str().as_bytes()); + } + fingerprint_field( + &mut fingerprint, + match reader.streaming_text_fallback() { + StreamingTextFallback::Disabled => b"disabled", + StreamingTextFallback::Enabled => b"enabled", + }, + ); + } + } + fingerprint.finalize().into() +} + +fn fingerprint_view_bounds(fingerprint: &mut Sha256, bounds: ReadViewBounds) { + fingerprint_u64(fingerprint, bounds.source_bytes()); + match bounds { + ReadViewBounds::Text { output_bytes, .. } => { + fingerprint_usize(fingerprint, output_bytes); + } + ReadViewBounds::Structured { + output_bytes, + depth, + nodes, + string_bytes, + .. + } => { + fingerprint_usize(fingerprint, output_bytes); + fingerprint_u64(fingerprint, u64::from(depth)); + fingerprint_u64(fingerprint, nodes); + fingerprint_usize(fingerprint, string_bytes); + } + ReadViewBounds::Image { + width, + height, + pixels, + output_bytes, + .. + } => { + fingerprint_u64(fingerprint, u64::from(width)); + fingerprint_u64(fingerprint, u64::from(height)); + fingerprint_u64(fingerprint, pixels); + fingerprint_u64(fingerprint, output_bytes); + } + ReadViewBounds::Audio { + channels, + sample_rate_hz, + duration_seconds, + output_bytes, + .. + } => { + fingerprint_u64(fingerprint, u64::from(channels)); + fingerprint_u64(fingerprint, u64::from(sample_rate_hz)); + fingerprint_u64(fingerprint, u64::from(duration_seconds)); + fingerprint_u64(fingerprint, output_bytes); + } + ReadViewBounds::File { output_bytes, .. } => { + fingerprint_u64(fingerprint, output_bytes); + } + } +} + +fn fingerprint_field(fingerprint: &mut Sha256, value: &[u8]) { + fingerprint_len(fingerprint, value.len()); + fingerprint.update(value); +} + +fn fingerprint_len(fingerprint: &mut Sha256, value: usize) { + fingerprint_usize(fingerprint, value); +} + +fn fingerprint_usize(fingerprint: &mut Sha256, value: usize) { + fingerprint_u64(fingerprint, u64::try_from(value).unwrap_or(u64::MAX)); +} + +fn fingerprint_u64(fingerprint: &mut Sha256, value: u64) { + fingerprint.update(value.to_be_bytes()); +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(tag = "operation", rename_all = "snake_case", deny_unknown_fields)] +pub(crate) enum Invocation { + Probe { + reader: WireReaderIdentity, + source: WireSource, + envelope: WireReadEnvelope, + }, + Validate { + reader: WireReaderIdentity, + source: WireSource, + envelope: WireReadEnvelope, + request: WireValidationRequest, + }, + Read { + reader: WireReaderIdentity, + source: WireSource, + envelope: WireReadEnvelope, + request: WireReadRequest, + }, +} + +impl Invocation { + pub(crate) const fn source(&self) -> &WireSource { + match self { + Self::Probe { source, .. } + | Self::Validate { source, .. } + | Self::Read { source, .. } => source, + } + } + + pub(crate) const fn envelope(&self) -> WireReadEnvelope { + match self { + Self::Probe { envelope, .. } + | Self::Validate { envelope, .. } + | Self::Read { envelope, .. } => *envelope, + } + } +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(tag = "message", rename_all = "snake_case", deny_unknown_fields)] +pub(crate) enum WorkerFrame { + ReadRange { offset: u64, length: u64 }, + ProbeResult { output: ProcessorProbeOutput }, + ValidationResult { output: ProcessorValidationOutput }, + ReadResult { output: ProcessorReadOutput }, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(tag = "message", rename_all = "snake_case", deny_unknown_fields)] +pub(crate) enum DaemonFrame { + Invocation { invocation: Box }, + RangeBytes { bytes_base64: String }, + RangeFailure, +} + +#[derive(Clone, Copy, Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct WireSource { + digest: [u8; 32], + byte_length: u64, +} + +impl WireSource { + pub(crate) fn from_source( + source: &dyn signalbox_file_media_runtime::VerifiedBlobSource, + ) -> Self { + Self { + digest: *source.digest().as_bytes(), + byte_length: source.byte_length().get(), + } + } + + pub(crate) const fn digest(self) -> FileDigest { + FileDigest::from_bytes(self.digest) + } + + pub(crate) fn byte_length(self) -> Result { + NonZeroU64::new(self.byte_length).ok_or(ProtocolValueError) + } +} + +#[derive(Clone, Copy, Debug, Serialize, Deserialize)] +#[serde(tag = "access", rename_all = "snake_case", deny_unknown_fields)] +pub(crate) enum WireReadEnvelope { + Probe { + prefix_bytes: u64, + suffix_bytes: u64, + ranges: u32, + cumulative_bytes: u64, + }, + Streaming { + ranges: u32, + cumulative_bytes: u64, + }, + RandomAccess { + ranges: u32, + cumulative_bytes: u64, + }, +} + +impl WireReadEnvelope { + pub(crate) const fn for_probe(probe: ProbeDeclaration) -> Self { + Self::Probe { + prefix_bytes: probe.prefix_bytes(), + suffix_bytes: probe.suffix_bytes(), + ranges: probe.range_count(), + cumulative_bytes: probe.cumulative_bytes(), + } + } + + pub(crate) const fn for_view(view: &ReadViewDeclaration) -> Self { + match view.access() { + ReadAccessPattern::Streaming { maximum_ranges } => Self::Streaming { + ranges: maximum_ranges, + cumulative_bytes: view.bounds().source_bytes(), + }, + ReadAccessPattern::RandomAccess { maximum_ranges } => Self::RandomAccess { + ranges: maximum_ranges, + cumulative_bytes: view.bounds().source_bytes(), + }, + } + } +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct WireReaderIdentity { + provider: String, + reader: String, + revision: String, +} + +impl From<&ReaderIdentity> for WireReaderIdentity { + fn from(identity: &ReaderIdentity) -> Self { + Self { + provider: identity.provider().as_str().to_owned(), + reader: identity.reader().as_str().to_owned(), + revision: identity.revision().as_str().to_owned(), + } + } +} + +impl TryFrom for ReaderIdentity { + type Error = ProtocolValueError; + + fn try_from(value: WireReaderIdentity) -> Result { + Ok(Self::new( + FileReaderProviderName::try_new(value.provider).map_err(map_value_error)?, + FileReaderName::try_new(value.reader).map_err(map_value_error)?, + FileReaderRevision::try_new(value.revision).map_err(map_value_error)?, + )) + } +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct WireFileUse { + digest: [u8; 32], + byte_length: u64, + attachment_kind: WireAttachmentKind, + declared_media_type: String, + display_filename: Option, +} + +impl From<&FileUse> for WireFileUse { + fn from(source: &FileUse) -> Self { + Self { + digest: *source.digest().as_bytes(), + byte_length: source.byte_length().get(), + attachment_kind: source.attachment_kind().into(), + declared_media_type: source.declared_media_type().as_str().to_owned(), + display_filename: source + .display_filename() + .map(|name| name.as_str().to_owned()), + } + } +} + +impl TryFrom for FileUse { + type Error = ProtocolValueError; + + fn try_from(value: WireFileUse) -> Result { + Ok(Self::new( + FileDigest::from_bytes(value.digest), + NonZeroU64::new(value.byte_length).ok_or(ProtocolValueError)?, + value.attachment_kind.into(), + DeclaredMediaType::try_new(value.declared_media_type).map_err(map_value_error)?, + value + .display_filename + .map(DisplayFilename::try_new) + .transpose() + .map_err(map_value_error)?, + )) + } +} + +#[derive(Clone, Copy, Debug, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +enum WireAttachmentKind { + Image, + Document, + File, +} + +impl From for WireAttachmentKind { + fn from(value: AttachmentKind) -> Self { + match value { + AttachmentKind::Image => Self::Image, + AttachmentKind::Document => Self::Document, + AttachmentKind::File => Self::File, + } + } +} + +impl From for AttachmentKind { + fn from(value: WireAttachmentKind) -> Self { + match value { + WireAttachmentKind::Image => Self::Image, + WireAttachmentKind::Document => Self::Document, + WireAttachmentKind::File => Self::File, + } + } +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct WireValidationRequest { + source: WireFileUse, + media_type: String, + evidence: ValidationEvidence, + maximum_source_bytes: u64, + maximum_ranges: u32, + maximum_image_axis: u32, + maximum_decoded_image_pixels: u64, +} + +impl From<&FileMediaProviderValidationRequest> for WireValidationRequest { + fn from(request: &FileMediaProviderValidationRequest) -> Self { + Self { + source: (&request.source).into(), + media_type: request.media_type.as_str().to_owned(), + evidence: request.evidence, + maximum_source_bytes: request.maximum_source_bytes, + maximum_ranges: request.maximum_ranges, + maximum_image_axis: request.maximum_image_axis, + maximum_decoded_image_pixels: request.maximum_decoded_image_pixels, + } + } +} + +impl TryFrom for FileMediaProviderValidationRequest { + type Error = ProtocolValueError; + + fn try_from(value: WireValidationRequest) -> Result { + Ok(Self { + source: value.source.try_into()?, + media_type: CanonicalMediaType::from_str(&value.media_type) + .map_err(|_| ProtocolValueError)?, + evidence: value.evidence, + maximum_source_bytes: value.maximum_source_bytes, + maximum_ranges: value.maximum_ranges, + maximum_image_axis: value.maximum_image_axis, + maximum_decoded_image_pixels: value.maximum_decoded_image_pixels, + }) + } +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct WireReadRequest { + source: WireFileUse, + detected_media_type: String, + validation: ValidationEvidence, + metadata_json: String, + view: String, + options: Option, + continuation: Option, + maximum_image_axis: u32, + maximum_decoded_image_pixels: u64, + maximum_container_entries: u64, +} + +impl From<&FileMediaProviderReadRequest> for WireReadRequest { + fn from(request: &FileMediaProviderReadRequest) -> Self { + let (options, continuation) = match &request.input { + signalbox_file_media_runtime::FileReadInput::Initial { options } => { + (Some(options.clone()), None) + } + signalbox_file_media_runtime::FileReadInput::Continuation { cursor } => { + (None, Some(cursor.as_str().to_owned())) + } + }; + Self { + source: (&request.source).into(), + detected_media_type: request.detected_media_type.as_str().to_owned(), + validation: request.validation, + metadata_json: request.metadata.as_str().to_owned(), + view: request.view.as_str().to_owned(), + options, + continuation, + maximum_image_axis: request.maximum_image_axis, + maximum_decoded_image_pixels: request.maximum_decoded_image_pixels, + maximum_container_entries: request.maximum_container_entries, + } + } +} + +impl TryFrom for FileMediaProviderReadRequest { + type Error = ProtocolValueError; + + fn try_from(value: WireReadRequest) -> Result { + let input = match (value.options, value.continuation) { + (Some(options), None) => { + signalbox_file_media_runtime::FileReadInput::Initial { options } + } + (None, Some(cursor)) => signalbox_file_media_runtime::FileReadInput::Continuation { + cursor: ReadContinuationCursor::try_new(cursor).map_err(map_value_error)?, + }, + (Some(_), Some(_)) | (None, None) => return Err(ProtocolValueError), + }; + Ok(Self { + source: value.source.try_into()?, + detected_media_type: CanonicalMediaType::from_str(&value.detected_media_type) + .map_err(|_| ProtocolValueError)?, + validation: value.validation, + metadata: BoundedMetadata::try_new(&value.metadata_json).map_err(map_value_error)?, + view: ReadViewName::try_new(value.view).map_err(map_value_error)?, + input, + maximum_image_axis: value.maximum_image_axis, + maximum_decoded_image_pixels: value.maximum_decoded_image_pixels, + maximum_container_entries: value.maximum_container_entries, + }) + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) struct ProtocolValueError; + +fn map_value_error(_: RegistryValueError) -> ProtocolValueError { + ProtocolValueError +} + +pub(crate) fn encode_bytes(bytes: &[u8]) -> String { + STANDARD.encode(bytes) +} + +pub(crate) fn decode_bytes(encoded: &str) -> Result, ProtocolValueError> { + STANDARD.decode(encoded).map_err(|_| ProtocolValueError) +} + +#[cfg(test)] +mod tests { + use std::str::FromStr as _; + + use signalbox_file_media_runtime::{ + CanonicalJsonObjectSchema, CanonicalMediaType, FileMediaProviderDeclaration, + FileReaderName, FileReaderProviderName, FileReaderRevision, MAX_PROCESSOR_FRAME_BYTES, + MAX_TEXT_OR_JSON_BYTES, ProbeDeclaration, ProbeDeclarationInput, ProcessorReadOutput, + ReadAccessPattern, ReadViewBounds, ReadViewDeclaration, ReadViewName, ReaderDeclaration, + ReaderDeclarationInput, ReasonCode, StreamingTextFallback, ValidationDeclaration, + }; + + use super::{WorkerFrame, declaration_fingerprint}; + + fn declaration_with_validation( + validation: ValidationDeclaration, + ) -> FileMediaProviderDeclaration { + let provider = + FileReaderProviderName::try_new("fixture").expect("fixture provider name is valid"); + let view = ReadViewDeclaration::try_new( + ReadViewName::try_new("text").expect("fixture view name is valid"), + String::from("Reads fixture text."), + CanonicalJsonObjectSchema::try_new(r#"{"type":"object"}"#) + .expect("fixture schema is valid"), + ReadAccessPattern::Streaming { maximum_ranges: 1 }, + ReadViewBounds::Text { + source_bytes: 64, + output_bytes: 64, + }, + ) + .expect("fixture view declaration is valid"); + let reader = ReaderDeclaration::try_new(ReaderDeclarationInput { + provider: provider.clone(), + reader: FileReaderName::try_new("reader").expect("fixture reader name is valid"), + revision: FileReaderRevision::try_new("v1").expect("fixture revision is valid"), + media_types: vec![ + "application/x-signalbox-fixture" + .parse::() + .expect("fixture media type is valid"), + ], + probe: ProbeDeclaration::new(ProbeDeclarationInput { + prefix_bytes: 1, + suffix_bytes: 0, + range_count: 0, + cumulative_bytes: 1, + }), + validation, + views: vec![view], + reason_codes: vec![ + ReasonCode::try_new("fixture_failure").expect("fixture reason is valid"), + ], + streaming_text_fallback: StreamingTextFallback::Disabled, + }) + .expect("fixture reader declaration is valid"); + FileMediaProviderDeclaration::try_new(provider, vec![reader]) + .expect("fixture provider owns its reader") + } + + #[test] + fn validation_source_bytes_change_declaration_fingerprint() { + let smaller = declaration_with_validation(ValidationDeclaration::new(64, 1)); + let larger = declaration_with_validation(ValidationDeclaration::new(128, 1)); + + assert_ne!( + declaration_fingerprint(&[smaller]), + declaration_fingerprint(&[larger]) + ); + } + + #[test] + fn validation_range_count_changes_declaration_fingerprint() { + let fewer = declaration_with_validation(ValidationDeclaration::new(64, 1)); + let more = declaration_with_validation(ValidationDeclaration::new(64, 2)); + + assert_ne!( + declaration_fingerprint(&[fewer]), + declaration_fingerprint(&[more]) + ); + } + + #[test] + fn maximum_escape_heavy_structured_output_fits_one_frame() { + let body_json = format!("\"{}\"", "\\\\".repeat((MAX_TEXT_OR_JSON_BYTES - 2) / 2)); + assert_eq!(body_json.len(), MAX_TEXT_OR_JSON_BYTES); + let frame = WorkerFrame::ReadResult { + output: ProcessorReadOutput::Structured { + body_json, + truncated: false, + cursor: None, + }, + }; + let encoded = serde_json::to_vec(&frame).expect("worker frame serializes"); + assert!(encoded.len() <= MAX_PROCESSOR_FRAME_BYTES); + } + + fn declaration_with_member_order( + media_types: &[&str], + reason_codes: &[&str], + ) -> FileMediaProviderDeclaration { + let provider = FileReaderProviderName::try_new("fixture").expect("valid provider name"); + let view = ReadViewDeclaration::try_new( + ReadViewName::try_new("text").expect("valid view name"), + String::from("Reads fixture text."), + CanonicalJsonObjectSchema::try_new(r#"{"type":"object"}"#) + .expect("valid arguments schema"), + ReadAccessPattern::Streaming { maximum_ranges: 1 }, + ReadViewBounds::Text { + source_bytes: 64, + output_bytes: 64, + }, + ) + .expect("valid view declaration"); + let reader = ReaderDeclaration::try_new(ReaderDeclarationInput { + provider: provider.clone(), + reader: FileReaderName::try_new("fixture").expect("valid reader name"), + revision: FileReaderRevision::try_new("v1").expect("valid revision"), + media_types: media_types + .iter() + .map(|value| CanonicalMediaType::from_str(value).expect("valid media type")) + .collect(), + probe: ProbeDeclaration::new(ProbeDeclarationInput { + prefix_bytes: 1, + suffix_bytes: 0, + range_count: 1, + cumulative_bytes: 1, + }), + validation: ValidationDeclaration::new(64, 1), + views: vec![view], + reason_codes: reason_codes + .iter() + .map(|value| ReasonCode::try_new(*value).expect("valid reason code")) + .collect(), + streaming_text_fallback: StreamingTextFallback::Disabled, + }) + .expect("valid reader declaration"); + FileMediaProviderDeclaration::try_new(provider, vec![reader]) + .expect("valid provider declaration") + } + + #[test] + fn declaration_fingerprint_is_independent_of_unordered_member_order() { + let forward = declaration_with_member_order( + &["application/x-fixture-a", "application/x-fixture-b"], + &["reason_a", "reason_b"], + ); + let reversed = declaration_with_member_order( + &["application/x-fixture-b", "application/x-fixture-a"], + &["reason_b", "reason_a"], + ); + + assert_eq!( + declaration_fingerprint(&[forward]), + declaration_fingerprint(&[reversed]), + "fingerprints over the same media-type and reason-code membership must match \ + regardless of caller-supplied order", + ); + } +} diff --git a/crates/file-media-processor-runtime/src/sandbox.rs b/crates/file-media-processor-runtime/src/sandbox.rs new file mode 100644 index 0000000000..4fddff4f23 --- /dev/null +++ b/crates/file-media-processor-runtime/src/sandbox.rs @@ -0,0 +1,2036 @@ +use std::{ + collections::BTreeMap, + error::Error, + fmt, fs, + io::{Read as _, Seek as _, SeekFrom, Write as _}, + os::{ + fd::AsRawFd as _, + unix::{fs::PermissionsExt as _, process::CommandExt as _}, + }, + path::{Path, PathBuf}, + process::Stdio, + sync::{ + Arc, + atomic::{AtomicU64, Ordering}, + }, + time::Duration, +}; + +use signalbox_file_media_linux_sandbox::{ + ChildSetup, create_executable_snapshot, install_pre_exec, seal_executable_snapshot, +}; +use signalbox_file_media_runtime::{ + CancellationSignal, FileMediaProcessCeilings, FileMediaProcessor, FileMediaProcessorFuture, + FileMediaProviderDeclaration, FileMediaProviderReadRequest, FileMediaProviderValidationRequest, + FileReadInput, MAX_DECODED_IMAGE_PIXELS, MAX_IMAGE_AXIS, MAX_PROBE_CUMULATIVE_BYTES, + MAX_PROBE_PREFIX_BYTES, MAX_PROBE_RANGES, MAX_PROBE_SUFFIX_BYTES, MAX_READ_RANGES, + MAX_READ_SOURCE_BYTES, MAX_READERS_PER_PROVIDER, MAX_REGISTRY_READERS, MAX_VALIDATION_RANGES, + MAX_VALIDATION_SOURCE_BYTES, MAX_WORKER_TASKS, ProbeDeclaration, ProcessorBoundaryFailure, + ProcessorFailure, ProcessorIsolation, ProcessorProbeOutput, ProcessorReadOutput, + ProcessorValidationOutput, ReadAccessPattern, ReadViewDeclaration, ReaderDeclaration, + ReaderIdentity, VerifiedBlobSource, provider_declaration_inventory_fits, read_options_fit, +}; +use tokio::{ + io::AsyncReadExt as _, + process::{Child, ChildStdin, ChildStdout, Command}, + task::JoinHandle, + time::{Instant, MissedTickBehavior}, +}; + +use crate::{ + broker::{BrokerError, RangeBroker, read_frame_with_limit, write_frame_with_limit}, + protocol::{ + DaemonFrame, Invocation, WireReadEnvelope, WireSource, WorkerFrame, + declaration_fingerprint, encode_bytes, + }, +}; + +const WORKER_SANDBOX_PATH: &str = "/signalbox-file-media-worker"; +const BWRAP_PROBE_ARGUMENT: &str = "--signalbox-file-media-isolation-probe"; +const CANCELLATION_POLL: Duration = Duration::from_millis(5); +const CLEANUP_TIMEOUT: Duration = Duration::from_secs(2); +const CGROUP_CLEANUP_POLL: Duration = Duration::from_millis(10); +const WRITABLE_TMPFS_BUDGET_DIVISOR: u64 = 2; +/// Maximum worker bindings retained by one processor. +const MAX_WORKER_BINDINGS: usize = 256; +/// Maximum bytes retained across all sealed executable snapshots in one processor. +const MAX_AGGREGATE_EXECUTABLE_SNAPSHOT_BYTES: u64 = 64 * 1024 * 1024; +/// Maximum bytes retained by one sealed executable snapshot. +const MAX_EXECUTABLE_SNAPSHOT_BYTES: u64 = 64 * 1024 * 1024; +const TASK_CGROUP_ROOT_ENVIRONMENT: &str = "SIGNALBOX_FILE_MEDIA_CGROUP_ROOT"; +static NEXT_TASK_CGROUP: AtomicU64 = AtomicU64::new(1); + +/// One checked mapping from a provider declaration to its worker executable. +#[derive(Clone, Debug)] +pub struct WorkerBinding { + source: PathBuf, + declaration: FileMediaProviderDeclaration, +} + +#[derive(Debug)] +struct PinnedExecutable { + _file: fs::File, + proc_path: PathBuf, + byte_length: u64, +} + +impl WorkerBinding { + /// Binds one complete provider declaration to one absolute executable. + pub fn try_new( + program: impl Into, + declaration: FileMediaProviderDeclaration, + ) -> Result { + let source = program.into(); + validate_executable(&source, ConstructionTarget::Worker)?; + let source = fs::canonicalize(source) + .map_err(|_| SandboxedFileMediaProcessorConstructionError::Worker)?; + Ok(Self { + source, + declaration, + }) + } + + /// Borrows the provider declaration registered with the daemon. + pub const fn declaration(&self) -> &FileMediaProviderDeclaration { + &self.declaration + } +} + +/// Fresh-worker implementation of the registry's untrusted processor port. +#[derive(Clone, Debug)] +pub struct SandboxedFileMediaProcessor { + bubblewrap: Arc, + workers: + Arc>>, + worker_declarations: Arc, Vec)>>, + readers: Arc>, + task_cgroup_root: Arc, + ceilings: FileMediaProcessCeilings, +} + +impl SandboxedFileMediaProcessor { + /// Constructs a fail-closed Linux sandbox configuration. + pub fn try_new( + bubblewrap: impl Into, + bindings: Vec, + ceilings: FileMediaProcessCeilings, + ) -> Result { + if !cfg!(target_os = "linux") || bindings.is_empty() { + return Err(SandboxedFileMediaProcessorConstructionError::Unsupported); + } + admit_worker_binding_count(bindings.len())?; + admit_reader_inventory( + bindings + .iter() + .map(|binding| binding.declaration.readers().len()), + )?; + if !provider_declaration_inventory_fits(bindings.iter().map(|binding| &binding.declaration)) + { + return Err(SandboxedFileMediaProcessorConstructionError::ReaderInventory); + } + let task_cgroup_root = delegated_task_cgroup_root()?; + if !FileMediaProcessCeilings::version_one().admits(ceilings) { + return Err(SandboxedFileMediaProcessorConstructionError::Ceilings); + } + let bubblewrap = Arc::new(open_executable_snapshot( + &bubblewrap.into(), + ConstructionTarget::Bubblewrap, + MAX_EXECUTABLE_SNAPSHOT_BYTES, + )?); + let mut aggregate_snapshot_bytes = bubblewrap.byte_length; + let worker_snapshot_limit = worker_memory_budget(ceilings.memory_bytes()) + .address_space_bytes + .min(MAX_EXECUTABLE_SNAPSHOT_BYTES); + let mut workers = BTreeMap::new(); + let mut worker_declarations = + BTreeMap::, Vec)>::new(); + let mut readers = BTreeMap::new(); + for binding in bindings { + let provider = binding.declaration.provider().clone(); + let group = match worker_declarations.entry(binding.source) { + std::collections::btree_map::Entry::Occupied(entry) => entry.into_mut(), + std::collections::btree_map::Entry::Vacant(entry) => { + let remaining_snapshot_bytes = MAX_AGGREGATE_EXECUTABLE_SNAPSHOT_BYTES + .checked_sub(aggregate_snapshot_bytes) + .filter(|remaining| *remaining > 0) + .ok_or(SandboxedFileMediaProcessorConstructionError::ExecutableSnapshots)?; + let program = Arc::new(open_executable_snapshot( + entry.key(), + ConstructionTarget::Worker, + worker_snapshot_limit.min(remaining_snapshot_bytes), + )?); + aggregate_snapshot_bytes = admit_executable_snapshot_bytes( + aggregate_snapshot_bytes, + program.byte_length, + )?; + entry.insert((program, Vec::new())) + } + }; + if workers.insert(provider, group.0.clone()).is_some() { + return Err(SandboxedFileMediaProcessorConstructionError::DuplicateProvider); + } + for reader in binding.declaration.readers() { + if !direct_reader_envelopes_fit(reader) { + return Err(SandboxedFileMediaProcessorConstructionError::Ceilings); + } + if readers + .insert(reader.identity().clone(), reader.clone()) + .is_some() + { + return Err(SandboxedFileMediaProcessorConstructionError::DuplicateReader); + } + } + group.1.push(binding.declaration); + } + Ok(Self { + bubblewrap, + workers: Arc::new(workers), + worker_declarations: Arc::new(worker_declarations.into_values().collect()), + readers: Arc::new(readers), + task_cgroup_root: Arc::new(task_cgroup_root), + ceilings, + }) + } + + /// Proves that the exact configured profile can start every registered worker. + pub async fn verify_isolation(&self) -> ProcessorIsolation { + let verification = async { + for (worker, declarations) in self.worker_declarations.iter() { + self.run_probe(worker, declarations).await?; + } + Ok::<(), ProcessorFailure>(()) + }; + match tokio::time::timeout( + Duration::from_secs(self.ceilings.wall_seconds()), + verification, + ) + .await + { + Ok(Ok(())) => ProcessorIsolation::Available, + Ok(Err(_)) | Err(_) => ProcessorIsolation::Unavailable, + } + } + + /// Returns the effective lowerable-only process ceilings. + pub const fn ceilings(&self) -> FileMediaProcessCeilings { + self.ceilings + } + + async fn run_probe( + &self, + worker: &PinnedExecutable, + declarations: &[FileMediaProviderDeclaration], + ) -> Result<(), ProcessorFailure> { + let mut running = self.spawn(worker, Some(declarations)).await?; + running.release_startup()?; + let stdout = running + .child_mut()? + .stdout + .take() + .ok_or(ProcessorFailure::Unavailable)?; + let stderr = running + .child_mut()? + .stderr + .take() + .ok_or(ProcessorFailure::Unavailable)?; + let stderr_limit = self.ceilings.stderr_bytes(); + let mut stderr_task = tokio::spawn(read_and_discard_diagnostics(stderr, stderr_limit)); + let expected = declaration_fingerprint(declarations); + let output_limit = u64::try_from(expected.len()) + .map_err(|_| ProcessorFailure::Unavailable)? + .checked_add(1) + .ok_or(ProcessorFailure::Unavailable)?; + let deadline = Duration::from_secs(self.ceilings.wall_seconds()); + let waited = tokio::time::timeout(deadline, async { + let mut observed = Vec::new(); + stdout + .take(output_limit) + .read_to_end(&mut observed) + .await + .map_err(|_| ProcessorFailure::Unavailable)?; + let status = running + .wait() + .await + .map_err(|_| ProcessorFailure::Unavailable)?; + if status.success() && observed.as_slice() == expected.as_slice() { + Ok(()) + } else { + Err(ProcessorFailure::Unavailable) + } + }) + .await; + let result = match waited { + Ok(result) => result, + Err(_) => Err(ProcessorFailure::TimedOut), + }; + if result.is_err() { + running.terminate().await; + } + let _ = tokio::time::timeout(CLEANUP_TIMEOUT, &mut stderr_task).await; + result + } + + fn reader(&self, identity: &ReaderIdentity) -> Result<&ReaderDeclaration, ProcessorFailure> { + self.readers.get(identity).ok_or(ProcessorFailure::Protocol) + } + + fn worker(&self, identity: &ReaderIdentity) -> Result<&PinnedExecutable, ProcessorFailure> { + self.workers + .get(identity.provider()) + .map(Arc::as_ref) + .ok_or(ProcessorFailure::Unavailable) + } + + async fn invoke( + &self, + invocation: Invocation, + expected: ExpectedOutput, + source: &dyn VerifiedBlobSource, + cancellation: &dyn CancellationSignal, + ) -> Result { + if cancellation.is_cancelled() + || invocation.source().digest() != source.digest() + || invocation + .source() + .byte_length() + .map_err(|_| ProcessorFailure::Protocol)? + != source.byte_length() + { + return Err(if cancellation.is_cancelled() { + ProcessorFailure::Cancelled + } else { + ProcessorFailure::Protocol + } + .into()); + } + let worker = match &invocation { + Invocation::Probe { reader, .. } + | Invocation::Validate { reader, .. } + | Invocation::Read { reader, .. } => { + let identity = ReaderIdentity::try_from(reader.clone()) + .map_err(|_| ProcessorFailure::Protocol)?; + self.worker(&identity)? + } + }; + let mut running = self.spawn(worker, None).await?; + running.release_startup()?; + let stdin = running + .child_mut()? + .stdin + .take() + .ok_or(ProcessorFailure::Unavailable)?; + let stdout = running + .child_mut()? + .stdout + .take() + .ok_or(ProcessorFailure::Unavailable)?; + let stderr = running + .child_mut()? + .stderr + .take() + .ok_or(ProcessorFailure::Unavailable)?; + let stderr_limit = self.ceilings.stderr_bytes(); + let mut stderr_task = tokio::spawn(read_and_discard_diagnostics(stderr, stderr_limit)); + let outcome = { + let session = run_session( + &mut running, + (stdin, stdout), + invocation, + expected, + source, + cancellation, + self.ceilings.frame_bytes(), + ); + tokio::pin!(session); + let deadline = Instant::now() + Duration::from_secs(self.ceilings.wall_seconds()); + let mut cancellation_poll = tokio::time::interval(CANCELLATION_POLL); + cancellation_poll.set_missed_tick_behavior(MissedTickBehavior::Delay); + loop { + tokio::select! { + biased; + () = tokio::time::sleep_until(deadline) => { + break Err(ProcessorFailure::TimedOut.into()); + } + _ = cancellation_poll.tick() => { + if cancellation.is_cancelled() { + break Err(ProcessorFailure::Cancelled.into()); + } + } + result = &mut session => { + if Instant::now() >= deadline { + break Err(ProcessorFailure::TimedOut.into()); + } + break result; + } + } + } + }; + if outcome.is_err() { + running.terminate().await; + } + let diagnostics = finish_diagnostics(&mut stderr_task).await; + let outcome = admit_completed(outcome, cancellation); + match (outcome, diagnostics) { + (Ok(output), Ok(())) => Ok(output), + (Err(error), _) => Err(error), + (Ok(_), Err(())) => Err(ProcessorFailure::Protocol.into()), + } + } + + async fn spawn( + &self, + worker: &PinnedExecutable, + probe_declarations: Option<&[FileMediaProviderDeclaration]>, + ) -> Result { + let worker_input = + fs::File::open(&worker.proc_path).map_err(|_| ProcessorFailure::Unavailable)?; + let seccomp = process_creation_filter().map_err(|_| ProcessorFailure::Unavailable)?; + let (block_read, block_write) = + startup_pipe().map_err(|_| ProcessorFailure::Unavailable)?; + let task_cgroup = + InvocationTaskCgroup::create(&self.task_cgroup_root, self.ceilings.memory_bytes()) + .map_err(|_| ProcessorFailure::Unavailable)?; + let profile = sandbox_arguments( + worker_input.as_raw_fd(), + seccomp.as_raw_fd(), + block_read.as_raw_fd(), + self.ceilings.memory_bytes(), + probe_declarations, + ); + let probe = probe_declarations.is_some(); + let mut command = Command::new(&self.bubblewrap.proc_path); + command + .args(profile) + .current_dir("/") + .env_clear() + .stdin(if probe { Stdio::null() } else { Stdio::piped() }) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .kill_on_drop(true); + let seccomp_fd = seccomp.as_raw_fd(); + let block_fd = block_read.as_raw_fd(); + let cgroup_procs_fd = task_cgroup.procs.as_raw_fd(); + command.as_std_mut().process_group(0); + let memory = worker_memory_budget(self.ceilings.memory_bytes()); + install_pre_exec( + command.as_std_mut(), + ChildSetup { + address_space_bytes: memory.address_space_bytes, + cpu_seconds: self.ceilings.cpu_seconds(), + file_descriptors: self.ceilings.file_descriptors(), + seccomp_fd, + startup_gate_fd: block_fd, + worker_fd: worker_input.as_raw_fd(), + cgroup_procs_fd, + }, + ); + let child = command.spawn().map_err(|_| ProcessorFailure::Unavailable)?; + drop(block_read); + let raw_pid = child.id().ok_or(ProcessorFailure::Unavailable)?; + let pid = + rustix::process::Pid::from_raw(raw_pid as i32).ok_or(ProcessorFailure::Unavailable)?; + Ok(RunningWorker { + child: Some(child), + process_group: pid, + startup: Some(block_write), + _seccomp: seccomp, + task_cgroup: Some(task_cgroup), + armed: true, + }) + } +} + +fn admit_completed( + outcome: Result, + cancellation: &dyn CancellationSignal, +) -> Result { + if cancellation.is_cancelled() { + Err(ProcessorFailure::Cancelled.into()) + } else { + outcome + } +} + +impl FileMediaProcessor for SandboxedFileMediaProcessor { + fn probe<'a>( + &'a self, + reader: &'a ReaderIdentity, + source: &'a dyn VerifiedBlobSource, + cancellation: &'a dyn CancellationSignal, + ) -> FileMediaProcessorFuture<'a, ProcessorProbeOutput> { + Box::pin(async move { + let declaration = self.reader(reader)?; + let invocation = Invocation::Probe { + reader: reader.into(), + source: WireSource::from_source(source), + envelope: WireReadEnvelope::for_probe(declaration.probe()), + }; + match self + .invoke(invocation, ExpectedOutput::Probe, source, cancellation) + .await? + { + CompletedOutput::Probe(output) => Ok(output), + CompletedOutput::Validation(_) | CompletedOutput::Read(_) => { + Err(ProcessorFailure::Protocol.into()) + } + } + }) + } + + fn validate<'a>( + &'a self, + reader: &'a ReaderIdentity, + mut request: FileMediaProviderValidationRequest, + source: &'a dyn VerifiedBlobSource, + cancellation: &'a dyn CancellationSignal, + ) -> FileMediaProcessorFuture<'a, ProcessorValidationOutput> { + Box::pin(async move { + let declaration = self.reader(reader)?; + require_file_use_source(&request.source, source)?; + if request.maximum_source_bytes == 0 + || request.maximum_source_bytes > MAX_VALIDATION_SOURCE_BYTES + || request.maximum_ranges == 0 + || request.maximum_ranges > MAX_VALIDATION_RANGES + || request.maximum_image_axis == 0 + || request.maximum_image_axis > MAX_IMAGE_AXIS + || request.maximum_decoded_image_pixels == 0 + || request.maximum_decoded_image_pixels > MAX_DECODED_IMAGE_PIXELS + { + return Err(ProcessorFailure::Protocol.into()); + } + let (maximum_source_bytes, maximum_ranges, envelope) = clamped_validation_envelope( + request.maximum_source_bytes, + request.maximum_ranges, + declaration.validation(), + ); + request.maximum_source_bytes = maximum_source_bytes; + request.maximum_ranges = maximum_ranges; + let invocation = Invocation::Validate { + reader: reader.into(), + source: WireSource::from_source(source), + envelope, + request: (&request).into(), + }; + match self + .invoke(invocation, ExpectedOutput::Validation, source, cancellation) + .await? + { + CompletedOutput::Validation(output) => Ok(output), + CompletedOutput::Probe(_) | CompletedOutput::Read(_) => { + Err(ProcessorFailure::Protocol.into()) + } + } + }) + } + + fn read<'a>( + &'a self, + reader: &'a ReaderIdentity, + request: FileMediaProviderReadRequest, + source: &'a dyn VerifiedBlobSource, + cancellation: &'a dyn CancellationSignal, + ) -> FileMediaProcessorFuture<'a, ProcessorReadOutput> { + Box::pin(async move { + let declaration = self.reader(reader)?; + require_file_use_source(&request.source, source)?; + if !direct_read_input_fits(&request.input) { + return Err(ProcessorFailure::Protocol.into()); + } + if request.maximum_image_axis == 0 + || request.maximum_image_axis > MAX_IMAGE_AXIS + || request.maximum_decoded_image_pixels == 0 + || request.maximum_decoded_image_pixels > MAX_DECODED_IMAGE_PIXELS + { + return Err(ProcessorFailure::Protocol.into()); + } + let view = declaration + .views() + .iter() + .find(|view| view.name() == &request.view) + .ok_or(ProcessorFailure::Protocol)?; + let invocation = Invocation::Read { + reader: reader.into(), + source: WireSource::from_source(source), + envelope: WireReadEnvelope::for_view(view), + request: (&request).into(), + }; + match self + .invoke(invocation, ExpectedOutput::Read, source, cancellation) + .await? + { + CompletedOutput::Read(output) => Ok(output), + CompletedOutput::Probe(_) | CompletedOutput::Validation(_) => { + Err(ProcessorFailure::Protocol.into()) + } + } + }) + } +} + +fn require_file_use_source( + file_use: &signalbox_file_media_runtime::FileUse, + source: &dyn VerifiedBlobSource, +) -> Result<(), ProcessorFailure> { + if file_use.digest() == source.digest() && file_use.byte_length() == source.byte_length() { + Ok(()) + } else { + Err(ProcessorFailure::Protocol) + } +} + +fn clamped_validation_envelope( + maximum_source_bytes: u64, + maximum_ranges: u32, + validation: signalbox_file_media_runtime::ValidationDeclaration, +) -> (u64, u32, WireReadEnvelope) { + let maximum_source_bytes = maximum_source_bytes.min(validation.source_bytes()); + let maximum_ranges = maximum_ranges.min(validation.range_count()); + ( + maximum_source_bytes, + maximum_ranges, + WireReadEnvelope::RandomAccess { + ranges: maximum_ranges, + cumulative_bytes: maximum_source_bytes, + }, + ) +} + +fn direct_reader_envelopes_fit(reader: &ReaderDeclaration) -> bool { + probe_envelope_fits(reader.probe()) + && reader.validation().source_bytes() > 0 + && reader.validation().source_bytes() <= MAX_VALIDATION_SOURCE_BYTES + && reader.validation().range_count() > 0 + && reader.validation().range_count() <= MAX_VALIDATION_RANGES + && reader.views().iter().all(read_envelope_fits) +} + +fn probe_envelope_fits(probe: ProbeDeclaration) -> bool { + probe.prefix_bytes() <= MAX_PROBE_PREFIX_BYTES + && probe.suffix_bytes() <= MAX_PROBE_SUFFIX_BYTES + && probe.range_count() <= MAX_PROBE_RANGES + && (probe.prefix_bytes() > 0 || probe.suffix_bytes() > 0 || probe.range_count() > 0) + && probe.cumulative_bytes() > 0 + && probe.cumulative_bytes() <= MAX_PROBE_CUMULATIVE_BYTES + && probe + .prefix_bytes() + .checked_add(probe.suffix_bytes()) + .is_some_and(|minimum| minimum <= probe.cumulative_bytes()) +} + +fn read_envelope_fits(view: &ReadViewDeclaration) -> bool { + let ranges = match view.access() { + ReadAccessPattern::Streaming { maximum_ranges } + | ReadAccessPattern::RandomAccess { maximum_ranges } => maximum_ranges, + }; + ranges > 0 + && ranges <= MAX_READ_RANGES + && view.bounds().source_bytes() > 0 + && view.bounds().source_bytes() <= MAX_READ_SOURCE_BYTES +} + +fn direct_read_input_fits(input: &FileReadInput) -> bool { + match input { + FileReadInput::Initial { options } => read_options_fit(options), + FileReadInput::Continuation { .. } => true, + } +} + +async fn run_session( + running: &mut RunningWorker, + (mut stdin, mut stdout): (ChildStdin, ChildStdout), + invocation: Invocation, + expected: ExpectedOutput, + source: &dyn VerifiedBlobSource, + cancellation: &dyn CancellationSignal, + frame_bytes: usize, +) -> Result { + let envelope = invocation.envelope(); + let source_length = invocation + .source() + .byte_length() + .map_err(|_| ProcessorFailure::Protocol)? + .get(); + write_frame_with_limit( + &mut stdin, + &DaemonFrame::Invocation { + invocation: Box::new(invocation), + }, + frame_bytes, + ) + .await + .map_err(|_| ProcessorFailure::Protocol)?; + let maximum_range_bytes = + u64::try_from(frame_bytes / 2).map_err(|_| ProcessorFailure::Protocol)?; + let mut broker = RangeBroker::new(source_length, envelope, maximum_range_bytes); + let completed = loop { + let frame: WorkerFrame = read_frame_with_limit(&mut stdout, frame_bytes) + .await + .map_err(|error| match error { + BrokerError::Eof => ProcessorFailure::Failed, + BrokerError::Frame | BrokerError::Range => ProcessorFailure::Protocol, + })?; + match frame { + WorkerFrame::ReadRange { offset, length } => { + let length = broker + .admit(offset, length) + .map_err(|_| ProcessorFailure::Protocol)?; + if cancellation.is_cancelled() { + return Err(ProcessorFailure::Cancelled.into()); + } + let bytes = source.read_range(offset, length).await?; + if bytes.len() + != usize::try_from(length.get()).map_err(|_| ProcessorFailure::Protocol)? + { + return Err(ProcessorFailure::Protocol.into()); + } + write_frame_with_limit( + &mut stdin, + &DaemonFrame::RangeBytes { + bytes_base64: encode_bytes(&bytes), + }, + frame_bytes, + ) + .await + .map_err(|_| ProcessorFailure::Protocol)?; + } + WorkerFrame::ProbeResult { output } if expected == ExpectedOutput::Probe => { + break CompletedOutput::Probe(output); + } + WorkerFrame::ValidationResult { output } if expected == ExpectedOutput::Validation => { + break CompletedOutput::Validation(output); + } + WorkerFrame::ReadResult { output } if expected == ExpectedOutput::Read => { + break CompletedOutput::Read(output); + } + WorkerFrame::ProbeResult { .. } + | WorkerFrame::ValidationResult { .. } + | WorkerFrame::ReadResult { .. } => return Err(ProcessorFailure::Protocol.into()), + } + }; + drop(stdin); + let mut trailing = [0_u8; 1]; + if stdout + .read(&mut trailing) + .await + .map_err(|_| ProcessorFailure::Protocol)? + != 0 + { + return Err(ProcessorFailure::Protocol.into()); + } + let status = running.wait().await.map_err(|_| ProcessorFailure::Failed)?; + if status.success() { + Ok(completed) + } else { + Err(ProcessorFailure::Failed.into()) + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum ExpectedOutput { + Probe, + Validation, + Read, +} + +enum CompletedOutput { + Probe(ProcessorProbeOutput), + Validation(ProcessorValidationOutput), + Read(ProcessorReadOutput), +} + +struct RunningWorker { + child: Option, + process_group: rustix::process::Pid, + startup: Option, + _seccomp: fs::File, + task_cgroup: Option, + armed: bool, +} + +impl RunningWorker { + fn child_mut(&mut self) -> Result<&mut Child, ProcessorFailure> { + self.child.as_mut().ok_or(ProcessorFailure::Unavailable) + } + + fn release_startup(&mut self) -> Result<(), ProcessorFailure> { + let startup = self.startup.take().ok_or(ProcessorFailure::Unavailable)?; + rustix::io::write(&startup, &[1]).map_err(|_| ProcessorFailure::Unavailable)?; + Ok(()) + } + + async fn terminate(&mut self) { + if !self.armed { + return; + } + self.kill_tree(); + let reaped = if let Some(child) = self.child.as_mut() { + matches!( + tokio::time::timeout(CLEANUP_TIMEOUT, child.wait()).await, + Ok(Ok(_)) + ) + } else { + true + }; + if reaped { + self.armed = false; + if let Some(task_cgroup) = self.task_cgroup.as_mut() { + task_cgroup.cleanup_after_reap().await; + } + } + } + + async fn wait(&mut self) -> Result { + let status = self + .child + .as_mut() + .ok_or_else(|| std::io::Error::other("worker child is unavailable"))? + .wait() + .await?; + self.armed = false; + if let Some(task_cgroup) = self.task_cgroup.as_mut() { + task_cgroup.cleanup_after_reap().await; + } + Ok(status) + } + + fn kill_tree(&mut self) { + let descendants = process_descendants(self.process_group); + for descendant in descendants.iter().rev() { + let _ = rustix::process::kill_process(*descendant, rustix::process::Signal::KILL); + } + let _ = + rustix::process::kill_process_group(self.process_group, rustix::process::Signal::KILL); + if let Some(child) = self.child.as_mut() { + let _ = child.start_kill(); + } + } +} + +impl Drop for RunningWorker { + fn drop(&mut self) { + if self.armed { + self.kill_tree(); + if let (Some(mut child), Some(mut task_cgroup)) = + (self.child.take(), self.task_cgroup.take()) + { + if let Ok(runtime) = tokio::runtime::Handle::try_current() { + drop(runtime.spawn(async move { + let _ = child.wait().await; + task_cgroup.cleanup_after_reap().await; + })); + return; + } + self.child = Some(child); + self.task_cgroup = Some(task_cgroup); + } + } + if let Some(task_cgroup) = self.task_cgroup.as_mut() { + task_cgroup.cleanup(); + } + } +} + +#[derive(Debug)] +struct InvocationTaskCgroup { + path: PathBuf, + procs: fs::File, + cleaned: bool, +} + +impl InvocationTaskCgroup { + fn create(root: &Path, memory_bytes: u64) -> Result { + let path = create_unique_cgroup_directory( + root, + "signalbox-file-media", + std::process::id(), + &NEXT_TASK_CGROUP, + )?; + let configured = (|| { + fs::write(path.join("pids.max"), MAX_WORKER_TASKS.to_string())?; + fs::write(path.join("memory.max"), memory_bytes.to_string())?; + fs::OpenOptions::new() + .write(true) + .open(path.join("cgroup.procs")) + })(); + match configured { + Ok(procs) => Ok(Self { + path, + procs, + cleaned: false, + }), + Err(error) => { + let _ = fs::remove_dir(&path); + Err(error) + } + } + } + + fn cleanup(&mut self) { + if self.cleaned { + return; + } + let _ = fs::write(self.path.join("cgroup.kill"), "1"); + if fs::remove_dir(&self.path).is_ok() { + self.cleaned = true; + } + } + + async fn cleanup_after_reap(&mut self) { + if self.cleaned { + return; + } + let _ = fs::write(self.path.join("cgroup.kill"), "1"); + let deadline = Instant::now() + CLEANUP_TIMEOUT; + loop { + let empty = fs::read_to_string(self.path.join("cgroup.events")) + .is_ok_and(|events| !cgroup_is_populated(&events)); + if empty && fs::remove_dir(&self.path).is_ok() { + self.cleaned = true; + return; + } + if Instant::now() >= deadline { + return; + } + tokio::time::sleep(CGROUP_CLEANUP_POLL).await; + } + } +} + +fn cgroup_is_populated(events: &str) -> bool { + events + .lines() + .find_map(|line| line.strip_prefix("populated ")) + .is_none_or(|value| value != "0") +} + +impl Drop for InvocationTaskCgroup { + fn drop(&mut self) { + self.cleanup(); + } +} + +fn delegated_task_cgroup_root() -> Result { + let configured = std::env::var_os(TASK_CGROUP_ROOT_ENVIRONMENT) + .ok_or(SandboxedFileMediaProcessorConstructionError::TaskController)?; + let root = fs::canonicalize(configured) + .map_err(|_| SandboxedFileMediaProcessorConstructionError::TaskController)?; + let controllers = fs::read_to_string(root.join("cgroup.controllers")) + .map_err(|_| SandboxedFileMediaProcessorConstructionError::TaskController)?; + if !root.is_absolute() + || !root.join("cgroup.procs").is_file() + || !controllers + .split_whitespace() + .any(|controller| controller == "pids") + || !controllers + .split_whitespace() + .any(|controller| controller == "memory") + { + return Err(SandboxedFileMediaProcessorConstructionError::TaskController); + } + + let probe = create_unique_cgroup_directory( + &root, + "signalbox-file-media-admission", + std::process::id(), + &NEXT_TASK_CGROUP, + ) + .map_err(|_| SandboxedFileMediaProcessorConstructionError::TaskController)?; + let admitted = fs::write(probe.join("pids.max"), MAX_WORKER_TASKS.to_string()) + .and_then(|()| fs::write(probe.join("memory.max"), "1")) + .and_then(|()| { + fs::OpenOptions::new() + .write(true) + .open(probe.join("cgroup.procs")) + }) + .and_then(|_| fs::remove_dir(&probe)); + if admitted.is_err() { + let _ = fs::remove_dir(&probe); + return Err(SandboxedFileMediaProcessorConstructionError::TaskController); + } + Ok(root) +} + +fn create_unique_cgroup_directory( + root: &Path, + prefix: &str, + process_id: u32, + sequence: &AtomicU64, +) -> Result { + loop { + let sequence = sequence.fetch_add(1, Ordering::Relaxed); + let path = root.join(format!("{prefix}-{process_id}-{sequence}")); + match fs::create_dir(&path) { + Ok(()) => return Ok(path), + Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => {} + Err(error) => return Err(error), + } + } +} + +fn process_descendants(root: rustix::process::Pid) -> Vec { + let mut pending = vec![root]; + let mut descendants = Vec::new(); + while let Some(parent) = pending.pop() { + let raw_parent = parent.as_raw_nonzero().get(); + let path = format!("/proc/{raw_parent}/task/{raw_parent}/children"); + let Ok(children) = fs::read_to_string(path) else { + continue; + }; + for child in children.split_whitespace() { + let Some(pid) = child + .parse::() + .ok() + .and_then(rustix::process::Pid::from_raw) + else { + continue; + }; + if !descendants.contains(&pid) { + descendants.push(pid); + pending.push(pid); + } + } + } + descendants +} + +async fn read_and_discard_diagnostics( + mut stderr: tokio::process::ChildStderr, + retained_limit: usize, +) -> Result, std::io::Error> { + let mut retained = Vec::with_capacity(retained_limit); + let mut buffer = [0_u8; 4096]; + loop { + let read = stderr.read(&mut buffer).await?; + if read == 0 { + return Ok(retained); + } + let available = retained_limit.saturating_sub(retained.len()); + retained.extend_from_slice(&buffer[..read.min(available)]); + } +} + +async fn finish_diagnostics( + task: &mut JoinHandle, std::io::Error>>, +) -> Result<(), ()> { + match tokio::time::timeout(CLEANUP_TIMEOUT, &mut *task).await { + Ok(Ok(Ok(_))) => Ok(()), + Ok(Ok(Err(_))) | Ok(Err(_)) => Err(()), + Err(_) => { + task.abort(); + Err(()) + } + } +} + +fn startup_pipe() -> Result<(rustix::fd::OwnedFd, rustix::fd::OwnedFd), rustix::io::Errno> { + rustix::pipe::pipe_with(rustix::pipe::PipeFlags::CLOEXEC) +} + +fn sandbox_arguments( + worker_fd: i32, + seccomp_fd: i32, + block_fd: i32, + memory_bytes: u64, + probe_declarations: Option<&[FileMediaProviderDeclaration]>, +) -> Vec { + let memory = worker_memory_budget(memory_bytes); + let mut arguments = [ + "--die-with-parent", + "--new-session", + "--unshare-all", + "--unshare-user", + "--disable-userns", + "--assert-userns-disabled", + "--cap-drop", + "ALL", + "--clearenv", + "--proc", + "/proc", + "--dev", + "/dev", + "--ro-bind-try", + "/lib", + "/lib", + "--ro-bind-try", + "/lib64", + "/lib64", + "--ro-bind-try", + "/usr/lib", + "/usr/lib", + "--ro-bind-try", + "/nix/store", + "/nix/store", + ] + .into_iter() + .map(std::ffi::OsString::from) + .collect::>(); + arguments.extend([ + std::ffi::OsString::from("--size"), + std::ffi::OsString::from(memory.first_tmpfs_bytes.to_string()), + std::ffi::OsString::from("--tmpfs"), + std::ffi::OsString::from("/tmp"), + std::ffi::OsString::from("--size"), + std::ffi::OsString::from(memory.second_tmpfs_bytes.to_string()), + std::ffi::OsString::from("--tmpfs"), + std::ffi::OsString::from("/run"), + std::ffi::OsString::from("--size"), + std::ffi::OsString::from(memory.shared_memory_bytes.to_string()), + std::ffi::OsString::from("--tmpfs"), + std::ffi::OsString::from("/dev/shm"), + std::ffi::OsString::from("--perms"), + std::ffi::OsString::from("0500"), + std::ffi::OsString::from("--ro-bind-data"), + std::ffi::OsString::from(worker_fd.to_string()), + std::ffi::OsString::from(WORKER_SANDBOX_PATH), + std::ffi::OsString::from("--seccomp"), + std::ffi::OsString::from(seccomp_fd.to_string()), + std::ffi::OsString::from("--block-fd"), + std::ffi::OsString::from(block_fd.to_string()), + std::ffi::OsString::from("--chdir"), + std::ffi::OsString::from("/tmp"), + std::ffi::OsString::from("--setenv"), + std::ffi::OsString::from("LANG"), + std::ffi::OsString::from("C.UTF-8"), + std::ffi::OsString::from("--setenv"), + std::ffi::OsString::from("LC_ALL"), + std::ffi::OsString::from("C.UTF-8"), + std::ffi::OsString::from("--"), + std::ffi::OsString::from(WORKER_SANDBOX_PATH), + ]); + if let Some(declarations) = probe_declarations { + arguments.push(std::ffi::OsString::from(BWRAP_PROBE_ARGUMENT)); + let mut providers = declarations + .iter() + .map(|declaration| declaration.provider().as_str()) + .collect::>(); + providers.sort_unstable(); + arguments.extend(providers.into_iter().map(std::ffi::OsString::from)); + } + arguments +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +struct WorkerMemoryBudget { + address_space_bytes: u64, + first_tmpfs_bytes: u64, + second_tmpfs_bytes: u64, + shared_memory_bytes: u64, +} + +const fn worker_memory_budget(memory_bytes: u64) -> WorkerMemoryBudget { + let tmpfs_bytes = memory_bytes / WRITABLE_TMPFS_BUDGET_DIVISOR; + let first_tmpfs_bytes = tmpfs_bytes / 3; + let second_tmpfs_bytes = tmpfs_bytes / 3; + let shared_memory_bytes = tmpfs_bytes - first_tmpfs_bytes - second_tmpfs_bytes; + WorkerMemoryBudget { + address_space_bytes: memory_bytes - tmpfs_bytes, + first_tmpfs_bytes, + second_tmpfs_bytes, + shared_memory_bytes, + } +} + +fn process_creation_filter() -> Result { + let mut file = fs::File::from( + rustix::fs::memfd_create( + "signalbox-file-media-seccomp", + rustix::fs::MemfdFlags::CLOEXEC, + ) + .map_err(std::io::Error::from)?, + ); + for instruction in seccomp_instructions()? { + file.write_all(&instruction.code.to_ne_bytes())?; + file.write_all(&[instruction.jump_true, instruction.jump_false])?; + file.write_all(&instruction.value.to_ne_bytes())?; + } + file.flush()?; + file.seek(SeekFrom::Start(0))?; + Ok(file) +} + +#[derive(Clone, Copy)] +struct FilterInstruction { + code: u16, + jump_true: u8, + jump_false: u8, + value: u32, +} + +fn seccomp_instructions() -> Result, std::io::Error> { + const LOAD_WORD_ABSOLUTE: u16 = 0x20; + const JUMP_EQUAL: u16 = 0x15; + const JUMP_SET: u16 = 0x45; + const JUMP_ALWAYS: u16 = 0x05; + const RETURN: u16 = 0x06; + const SECCOMP_ALLOW: u32 = 0x7fff_0000; + const SECCOMP_KILL_PROCESS: u32 = 0x8000_0000; + const SECCOMP_ERRNO_EPERM: u32 = 0x0005_0001; + const SECCOMP_ERRNO_ENOSYS: u32 = 0x0005_0026; + const CLONE_THREAD: u32 = 0x0001_0000; + #[cfg(target_arch = "x86_64")] + const X32_SYSCALL_BIT: Option = Some(0x4000_0000); + #[cfg(not(target_arch = "x86_64"))] + const X32_SYSCALL_BIT: Option = None; + #[cfg(target_arch = "x86_64")] + let ( + audit_arch, + clone, + clone3, + fork, + vfork, + memfd_create, + shmget, + msgget, + mq_open, + semget, + io_setup, + add_key, + request_key, + keyctl, + ) = ( + 0xc000_003e, + 56_u32, + 435_u32, + Some(57_u32), + Some(58_u32), + 319_u32, + 29_u32, + 68_u32, + 240_u32, + 64_u32, + 206_u32, + 248_u32, + 249_u32, + 250_u32, + ); + #[cfg(target_arch = "aarch64")] + let ( + audit_arch, + clone, + clone3, + fork, + vfork, + memfd_create, + shmget, + msgget, + mq_open, + semget, + io_setup, + add_key, + request_key, + keyctl, + ) = ( + 0xc000_00b7, + 220_u32, + 435_u32, + None, + None, + 279_u32, + 194_u32, + 186_u32, + 180_u32, + 190_u32, + 0_u32, + 217_u32, + 218_u32, + 219_u32, + ); + #[cfg(not(any(target_arch = "x86_64", target_arch = "aarch64")))] + return Err(std::io::Error::new( + std::io::ErrorKind::Unsupported, + "unsupported seccomp architecture", + )); + #[cfg(target_arch = "x86_64")] + let (inotify_init1, inotify_add_watch) = (294_u32, 254_u32); + #[cfg(target_arch = "aarch64")] + let (inotify_init1, inotify_add_watch) = (26_u32, 27_u32); + let io_uring_setup = 425_u32; + let mut syscall_denials = vec![ + memfd_create, + shmget, + msgget, + mq_open, + semget, + io_setup, + io_uring_setup, + inotify_init1, + inotify_add_watch, + add_key, + request_key, + keyctl, + ]; + if let Some(fork) = fork { + syscall_denials.push(fork); + } + if let Some(vfork) = vfork { + syscall_denials.push(vfork); + } + let x32_check_count = usize::from(X32_SYSCALL_BIT.is_some()); + let clone3_check_index = 4 + x32_check_count + syscall_denials.len(); + let clone_check_index = clone3_check_index + 1; + let jump_allow_index = clone_check_index + 1; + let load_clone_flags_index = jump_allow_index + 1; + let clone_flags_check_index = load_clone_flags_index + 1; + let deny_index = clone_flags_check_index + 1; + let clone3_deny_index = deny_index + 1; + let final_allow_index = clone3_deny_index + 1; + let mut program = vec![ + instruction(LOAD_WORD_ABSOLUTE, 0, 0, 4), + instruction(JUMP_EQUAL, 1, 0, audit_arch), + instruction(RETURN, 0, 0, SECCOMP_KILL_PROCESS), + instruction(LOAD_WORD_ABSOLUTE, 0, 0, 0), + ]; + if let Some(x32_syscall_bit) = X32_SYSCALL_BIT { + program.push(instruction( + JUMP_SET, + jump_distance(program.len(), deny_index)?, + 0, + x32_syscall_bit, + )); + } + for syscall in syscall_denials { + program.push(instruction( + JUMP_EQUAL, + jump_distance(program.len(), deny_index)?, + 0, + syscall, + )); + } + program.push(instruction( + JUMP_EQUAL, + jump_distance(clone3_check_index, clone3_deny_index)?, + 0, + clone3, + )); + program.push(instruction(JUMP_EQUAL, 1, 0, clone)); + program.push(instruction( + JUMP_ALWAYS, + 0, + 0, + u32::try_from(final_allow_index - jump_allow_index - 1).map_err(|_| filter_error())?, + )); + program.push(instruction(LOAD_WORD_ABSOLUTE, 0, 0, 16)); + program.push(instruction( + JUMP_SET, + jump_distance(clone_flags_check_index, final_allow_index)?, + 0, + CLONE_THREAD, + )); + program.push(instruction(RETURN, 0, 0, SECCOMP_ERRNO_EPERM)); + program.push(instruction(RETURN, 0, 0, SECCOMP_ERRNO_ENOSYS)); + program.push(instruction(RETURN, 0, 0, SECCOMP_ALLOW)); + Ok(program) +} + +const fn instruction(code: u16, jump_true: u8, jump_false: u8, value: u32) -> FilterInstruction { + FilterInstruction { + code, + jump_true, + jump_false, + value, + } +} + +fn jump_distance(from: usize, to: usize) -> Result { + to.checked_sub(from + 1) + .and_then(|distance| u8::try_from(distance).ok()) + .ok_or_else(filter_error) +} + +fn filter_error() -> std::io::Error { + std::io::Error::new(std::io::ErrorKind::InvalidData, "invalid seccomp program") +} + +fn validate_executable( + path: &Path, + target: ConstructionTarget, +) -> Result<(), SandboxedFileMediaProcessorConstructionError> { + let valid = path.is_absolute() + && fs::metadata(path) + .is_ok_and(|metadata| metadata.is_file() && metadata.permissions().mode() & 0o111 != 0); + if valid { + Ok(()) + } else { + Err(match target { + ConstructionTarget::Bubblewrap => { + SandboxedFileMediaProcessorConstructionError::Bubblewrap + } + ConstructionTarget::Worker => SandboxedFileMediaProcessorConstructionError::Worker, + }) + } +} + +#[cfg(test)] +fn open_worker_executable( + path: &Path, +) -> Result { + open_executable_snapshot( + path, + ConstructionTarget::Worker, + MAX_EXECUTABLE_SNAPSHOT_BYTES, + ) +} + +fn open_executable_snapshot( + path: &Path, + target: ConstructionTarget, + maximum_bytes: u64, +) -> Result { + let invalid = || match target { + ConstructionTarget::Bubblewrap => SandboxedFileMediaProcessorConstructionError::Bubblewrap, + ConstructionTarget::Worker => SandboxedFileMediaProcessorConstructionError::Worker, + }; + if !path.is_absolute() { + return Err(invalid()); + } + let mut source = fs::File::open(path).map_err(|_| invalid())?; + let metadata = source.metadata().map_err(|_| invalid())?; + if !metadata.is_file() + || metadata.permissions().mode() & 0o111 == 0 + || metadata.len() > maximum_bytes + { + return Err(invalid()); + } + let mut file = create_executable_snapshot().map_err(|_| invalid())?; + let mut buffer = [0_u8; 64 * 1_024]; + let mut copied = 0_u64; + loop { + let read = source.read(&mut buffer).map_err(|_| invalid())?; + if read == 0 { + break; + } + copied = copied + .checked_add(u64::try_from(read).map_err(|_| invalid())?) + .ok_or_else(invalid)?; + if copied > maximum_bytes { + return Err(invalid()); + } + file.write_all(&buffer[..read]).map_err(|_| invalid())?; + } + file.flush().map_err(|_| invalid())?; + file.set_permissions(fs::Permissions::from_mode(0o500)) + .map_err(|_| invalid())?; + seal_executable_snapshot(&file).map_err(|_| invalid())?; + let proc_path = PathBuf::from(format!( + "/proc/{}/fd/{}", + std::process::id(), + file.as_raw_fd() + )); + Ok(PinnedExecutable { + _file: file, + proc_path, + byte_length: copied, + }) +} + +fn admit_executable_snapshot_bytes( + retained_bytes: u64, + additional_bytes: u64, +) -> Result { + retained_bytes + .checked_add(additional_bytes) + .filter(|total| *total <= MAX_AGGREGATE_EXECUTABLE_SNAPSHOT_BYTES) + .ok_or(SandboxedFileMediaProcessorConstructionError::ExecutableSnapshots) +} + +fn admit_worker_binding_count( + binding_count: usize, +) -> Result<(), SandboxedFileMediaProcessorConstructionError> { + if binding_count <= MAX_WORKER_BINDINGS { + Ok(()) + } else { + Err(SandboxedFileMediaProcessorConstructionError::WorkerBindings) + } +} + +fn admit_reader_inventory( + reader_counts: impl IntoIterator, +) -> Result<(), SandboxedFileMediaProcessorConstructionError> { + let mut aggregate = 0_usize; + for readers in reader_counts { + if readers > MAX_READERS_PER_PROVIDER { + return Err(SandboxedFileMediaProcessorConstructionError::ReaderInventory); + } + aggregate = aggregate + .checked_add(readers) + .filter(|count| *count <= MAX_REGISTRY_READERS) + .ok_or(SandboxedFileMediaProcessorConstructionError::ReaderInventory)?; + } + Ok(()) +} + +#[derive(Clone, Copy)] +enum ConstructionTarget { + Bubblewrap, + Worker, +} + +/// Checked sandbox configuration could not be constructed. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum SandboxedFileMediaProcessorConstructionError { + /// This platform cannot provide the accepted sandbox. + Unsupported, + /// Bubblewrap was not an absolute executable file. + Bubblewrap, + /// A worker was not an absolute executable file. + Worker, + /// Sealed executable snapshots exceeded their aggregate byte ceiling. + ExecutableSnapshots, + /// Worker bindings exceeded their compiled count ceiling. + WorkerBindings, + /// Reader declarations exceeded their registry-compatible count ceilings. + ReaderInventory, + /// A process ceiling was zero or exceeded its compiled maximum. + Ceilings, + /// No validated writable delegated cgroup-v2 task controller was configured. + TaskController, + /// Two worker bindings claimed the same provider. + DuplicateProvider, + /// Two worker bindings claimed the same reader identity. + DuplicateReader, +} + +impl fmt::Display for SandboxedFileMediaProcessorConstructionError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str(match self { + Self::Unsupported => "file-media sandbox is unsupported", + Self::Bubblewrap => "file-media bubblewrap executable is invalid", + Self::Worker => "file-media worker executable is invalid", + Self::ExecutableSnapshots => { + "file-media executable snapshots exceed their aggregate ceiling" + } + Self::WorkerBindings => "file-media worker bindings exceed their count ceiling", + Self::ReaderInventory => "file-media worker readers exceed their inventory ceiling", + Self::Ceilings => "file-media process ceilings are invalid", + Self::TaskController => "file-media per-invocation task controller is unavailable", + Self::DuplicateProvider => "file-media worker provider is duplicated", + Self::DuplicateReader => "file-media worker reader is duplicated", + }) + } +} + +impl Error for SandboxedFileMediaProcessorConstructionError {} + +#[cfg(test)] +mod tests { + use std::{ffi::OsStr, fs, os::unix::fs::PermissionsExt as _, sync::atomic::AtomicU64}; + + use signalbox_file_media_runtime::{ + CancellationSignal, CanonicalJsonObjectSchema, FileReadInput, MAX_READ_OPTIONS_BYTES, + ProbeDeclaration, ProbeDeclarationInput, ProcessorBoundaryFailure, ProcessorFailure, + ReadAccessPattern, ReadViewBounds, ReadViewDeclaration, ReadViewName, + ValidationDeclaration, + }; + + use super::{ + CompletedOutput, ConstructionTarget, MAX_AGGREGATE_EXECUTABLE_SNAPSHOT_BYTES, + MAX_EXECUTABLE_SNAPSHOT_BYTES, MAX_PROBE_CUMULATIVE_BYTES, MAX_PROBE_PREFIX_BYTES, + MAX_READ_RANGES, MAX_READ_SOURCE_BYTES, MAX_READERS_PER_PROVIDER, MAX_REGISTRY_READERS, + MAX_WORKER_BINDINGS, admit_completed, admit_executable_snapshot_bytes, + admit_reader_inventory, admit_worker_binding_count, cgroup_is_populated, + clamped_validation_envelope, create_unique_cgroup_directory, direct_read_input_fits, + open_executable_snapshot, open_worker_executable, probe_envelope_fits, read_envelope_fits, + sandbox_arguments, seccomp_instructions, startup_pipe, worker_memory_budget, + }; + + struct Cancelled; + + impl CancellationSignal for Cancelled { + fn is_cancelled(&self) -> bool { + true + } + } + + #[test] + fn cgroup_events_distinguish_populated_and_empty_groups() { + assert!(cgroup_is_populated("populated 1\nfrozen 0\n")); + assert!(!cgroup_is_populated("populated 0\nfrozen 0\n")); + assert!(cgroup_is_populated("frozen 0\n")); + } + + #[test] + fn cgroup_directory_creation_retries_stale_name_collisions() { + let directory = tempfile::tempdir().expect("temporary directory is available"); + fs::create_dir(directory.path().join("fixture-42-1")) + .expect("stale fixture directory is created"); + let sequence = AtomicU64::new(1); + + let created = create_unique_cgroup_directory(directory.path(), "fixture", 42, &sequence) + .expect("a later unique directory is created"); + + assert_eq!(created, directory.path().join("fixture-42-2")); + } + + #[test] + fn aggregate_executable_snapshots_reject_bytes_above_their_bound() { + assert_eq!( + admit_executable_snapshot_bytes(MAX_AGGREGATE_EXECUTABLE_SNAPSHOT_BYTES - 1, 1,), + Ok(MAX_AGGREGATE_EXECUTABLE_SNAPSHOT_BYTES) + ); + assert!( + admit_executable_snapshot_bytes(MAX_AGGREGATE_EXECUTABLE_SNAPSHOT_BYTES, 1).is_err() + ); + } + + #[test] + fn worker_binding_count_rejects_entries_above_its_bound() { + assert_eq!(admit_worker_binding_count(MAX_WORKER_BINDINGS), Ok(())); + assert_eq!( + admit_worker_binding_count(MAX_WORKER_BINDINGS + 1), + Err(super::SandboxedFileMediaProcessorConstructionError::WorkerBindings) + ); + } + + #[test] + fn reader_inventory_rejects_per_provider_and_aggregate_excess() { + assert_eq!(admit_reader_inventory([MAX_REGISTRY_READERS]), Ok(())); + assert_eq!( + admit_reader_inventory([MAX_READERS_PER_PROVIDER + 1]), + Err(super::SandboxedFileMediaProcessorConstructionError::ReaderInventory) + ); + assert_eq!( + admit_reader_inventory([MAX_REGISTRY_READERS, 1]), + Err(super::SandboxedFileMediaProcessorConstructionError::ReaderInventory) + ); + } + + #[test] + fn direct_probe_envelope_rejects_compiled_ceiling_excess() { + assert!(probe_envelope_fits(ProbeDeclaration::new( + ProbeDeclarationInput { + prefix_bytes: MAX_PROBE_PREFIX_BYTES, + suffix_bytes: 0, + range_count: 0, + cumulative_bytes: MAX_PROBE_CUMULATIVE_BYTES, + } + ))); + assert!(!probe_envelope_fits(ProbeDeclaration::new( + ProbeDeclarationInput { + prefix_bytes: MAX_PROBE_PREFIX_BYTES + 1, + suffix_bytes: 0, + range_count: 0, + cumulative_bytes: MAX_PROBE_CUMULATIVE_BYTES, + } + ))); + } + + #[test] + fn validation_envelope_is_clamped_to_reader_declaration() { + let (maximum_source_bytes, maximum_ranges, envelope) = + clamped_validation_envelope(1_024, 8, ValidationDeclaration::new(64, 2)); + + assert_eq!(maximum_source_bytes, 64); + assert_eq!(maximum_ranges, 2); + assert!(matches!( + envelope, + super::WireReadEnvelope::RandomAccess { + ranges: 2, + cumulative_bytes: 64 + } + )); + } + + #[test] + fn direct_read_envelope_rejects_compiled_ceiling_excess() { + let view = ReadViewDeclaration::try_new( + ReadViewName::try_new("fixture").expect("view name is valid"), + String::from("Fixture view."), + CanonicalJsonObjectSchema::try_new(r#"{"type":"object"}"#).expect("schema is valid"), + ReadAccessPattern::RandomAccess { + maximum_ranges: MAX_READ_RANGES + 1, + }, + ReadViewBounds::Text { + source_bytes: MAX_READ_SOURCE_BYTES, + output_bytes: 1, + }, + ) + .expect("declaration construction defers compiled ceiling checks"); + assert!(!read_envelope_fits(&view)); + } + + #[test] + fn direct_read_input_rejects_oversized_options_before_framing() { + let input = FileReadInput::Initial { + options: serde_json::json!({ + "value": "x".repeat(MAX_READ_OPTIONS_BYTES), + }), + }; + assert!(!direct_read_input_fits(&input)); + } + + #[test] + fn direct_read_input_rejects_excessive_option_nesting_before_framing() { + // The outer invocation argument and options objects consume two of the + // 256 input-container slots, so 255 nested arrays exceed the contract. + let nested = (0..255).fold(serde_json::Value::Null, |value, _| { + serde_json::Value::Array(vec![value]) + }); + let input = FileReadInput::Initial { + options: serde_json::json!({ "nested": nested }), + }; + + assert!(!direct_read_input_fits(&input)); + } + + #[test] + fn sandbox_profile_clears_authority_before_the_exact_worker() { + let arguments = sandbox_arguments(7, 8, 9, 512 * 1024 * 1024, None); + let expected_prefix = [ + "--die-with-parent", + "--new-session", + "--unshare-all", + "--unshare-user", + "--disable-userns", + "--assert-userns-disabled", + "--cap-drop", + "ALL", + "--clearenv", + ]; + assert_eq!( + &arguments[..expected_prefix.len()], + expected_prefix.map(std::ffi::OsString::from) + ); + assert!(arguments.windows(5).any(|window| { + window + == [ + OsStr::new("--perms"), + OsStr::new("0500"), + OsStr::new("--ro-bind-data"), + OsStr::new("7"), + OsStr::new("/signalbox-file-media-worker"), + ] + })); + assert_eq!( + &arguments[arguments.len() - 2..], + [OsStr::new("--"), OsStr::new("/signalbox-file-media-worker")] + ); + assert!(arguments.windows(4).any(|window| { + window + == [ + OsStr::new("--size"), + OsStr::new("89478485"), + OsStr::new("--tmpfs"), + OsStr::new("/tmp"), + ] + })); + assert!(arguments.windows(4).any(|window| { + window + == [ + OsStr::new("--size"), + OsStr::new("89478485"), + OsStr::new("--tmpfs"), + OsStr::new("/run"), + ] + })); + assert!(arguments.windows(4).any(|window| { + window + == [ + OsStr::new("--size"), + OsStr::new("89478486"), + OsStr::new("--tmpfs"), + OsStr::new("/dev/shm"), + ] + })); + } + + #[test] + fn completed_output_is_not_admitted_after_cancellation() { + let outcome = admit_completed( + Ok(CompletedOutput::Probe( + signalbox_file_media_runtime::ProcessorProbeOutput::NoMatch, + )), + &Cancelled, + ); + assert!(matches!( + outcome, + Err(ProcessorBoundaryFailure::Processor( + ProcessorFailure::Cancelled + )) + )); + } + + #[test] + fn memory_budget_combines_address_space_and_writable_tmpfs() { + let budget = worker_memory_budget(512 * 1024 * 1024); + assert_eq!(budget.address_space_bytes, 256 * 1024 * 1024); + assert_eq!(budget.first_tmpfs_bytes, 89_478_485); + assert_eq!(budget.second_tmpfs_bytes, 89_478_485); + assert_eq!(budget.shared_memory_bytes, 89_478_486); + assert_eq!( + budget.address_space_bytes + + budget.first_tmpfs_bytes + + budget.second_tmpfs_bytes + + budget.shared_memory_bytes, + 512 * 1024 * 1024 + ); + } + + #[test] + fn startup_gate_descriptors_are_close_on_exec_in_the_daemon() { + let (read, write) = startup_pipe().expect("startup pipe is created"); + let read_flags = rustix::io::fcntl_getfd(&read).expect("read flags are available"); + let write_flags = rustix::io::fcntl_getfd(&write).expect("write flags are available"); + assert!(read_flags.contains(rustix::io::FdFlags::CLOEXEC)); + assert!(write_flags.contains(rustix::io::FdFlags::CLOEXEC)); + } + + #[test] + fn descendant_filter_has_a_finite_arch_checked_program() { + let program = seccomp_instructions().expect("the test architecture is supported"); + assert!(program.len() >= 10); + assert_eq!(program[0].value, 4); + assert_eq!(program[2].value, 0x8000_0000); + assert_eq!(program.last().map(|entry| entry.value), Some(0x7fff_0000)); + } + + #[test] + fn descendant_filter_denies_unbudgeted_memfd_allocations() { + let program = seccomp_instructions().expect("the test architecture is supported"); + #[cfg(target_arch = "x86_64")] + let memfd_create = 319_u32; + #[cfg(target_arch = "aarch64")] + let memfd_create = 279_u32; + let (index, check) = program + .iter() + .enumerate() + .find(|(_, entry)| entry.value == memfd_create) + .expect("memfd_create is checked"); + assert_eq!(check.code, 0x15); + let denial = index + 1 + usize::from(check.jump_true); + assert_eq!( + program.get(denial).map(|entry| entry.value), + Some(0x0005_0001) + ); + } + + #[test] + fn descendant_filter_denies_persistent_system_v_shared_memory() { + let program = seccomp_instructions().expect("the test architecture is supported"); + #[cfg(target_arch = "x86_64")] + let shmget = 29_u32; + #[cfg(target_arch = "aarch64")] + let shmget = 194_u32; + let (index, check) = program + .iter() + .enumerate() + .find(|(_, entry)| entry.value == shmget) + .expect("shmget is checked"); + assert_eq!(check.code, 0x15); + let denial = index + 1 + usize::from(check.jump_true); + assert_eq!( + program.get(denial).map(|entry| entry.value), + Some(0x0005_0001) + ); + } + + #[test] + fn descendant_filter_denies_system_v_message_queue_creation() { + let program = seccomp_instructions().expect("the test architecture is supported"); + #[cfg(target_arch = "x86_64")] + let msgget = 68_u32; + #[cfg(target_arch = "aarch64")] + let msgget = 186_u32; + let (index, check) = program + .iter() + .enumerate() + .find(|(_, entry)| entry.value == msgget) + .expect("msgget is checked"); + assert_eq!(check.code, 0x15); + let denial = index + 1 + usize::from(check.jump_true); + assert_eq!( + program.get(denial).map(|entry| entry.value), + Some(0x0005_0001) + ); + } + + #[test] + fn descendant_filter_denies_posix_message_queue_creation() { + let program = seccomp_instructions().expect("the test architecture is supported"); + #[cfg(target_arch = "x86_64")] + let mq_open = 240_u32; + #[cfg(target_arch = "aarch64")] + let mq_open = 180_u32; + let (index, check) = program + .iter() + .enumerate() + .find(|(_, entry)| entry.value == mq_open) + .expect("mq_open is checked"); + assert_eq!(check.code, 0x15); + let denial = index + 1 + usize::from(check.jump_true); + assert_eq!( + program.get(denial).map(|entry| entry.value), + Some(0x0005_0001) + ); + } + + #[test] + fn seccomp_descriptor_is_close_on_exec_in_the_daemon() { + let filter = super::process_creation_filter().expect("seccomp filter is created"); + let flags = rustix::io::fcntl_getfd(&filter).expect("descriptor flags are read"); + assert!(flags.contains(rustix::io::FdFlags::CLOEXEC)); + } + + #[test] + fn descendant_filter_denies_persistent_system_v_semaphores() { + let program = seccomp_instructions().expect("the test architecture is supported"); + #[cfg(target_arch = "x86_64")] + let semget = 64_u32; + #[cfg(target_arch = "aarch64")] + let semget = 190_u32; + let (index, check) = program + .iter() + .enumerate() + .find(|(_, entry)| entry.value == semget) + .expect("semget is checked"); + assert_eq!(check.code, 0x15); + let denial = index + 1 + usize::from(check.jump_true); + assert_eq!( + program.get(denial).map(|entry| entry.value), + Some(0x0005_0001) + ); + } + + #[test] + fn descendant_filter_denies_global_linux_aio_context_allocation() { + let program = seccomp_instructions().expect("the test architecture is supported"); + #[cfg(target_arch = "x86_64")] + let io_setup = 206_u32; + #[cfg(target_arch = "aarch64")] + let io_setup = 0_u32; + assert_syscall_denied(&program, io_setup, "io_setup"); + } + + #[test] + fn descendant_filter_denies_unbudgeted_io_uring_allocation() { + let program = seccomp_instructions().expect("the test architecture is supported"); + assert_syscall_denied(&program, 425_u32, "io_uring_setup"); + } + + #[test] + fn descendant_filter_denies_unbudgeted_inotify_allocation() { + let program = seccomp_instructions().expect("the test architecture is supported"); + #[cfg(target_arch = "x86_64")] + let (inotify_init1, inotify_add_watch) = (294_u32, 254_u32); + #[cfg(target_arch = "aarch64")] + let (inotify_init1, inotify_add_watch) = (26_u32, 27_u32); + assert_syscall_denied(&program, inotify_init1, "inotify_init1"); + assert_syscall_denied(&program, inotify_add_watch, "inotify_add_watch"); + } + + #[test] + fn descendant_filter_denies_add_key() { + let program = seccomp_instructions().expect("the test architecture is supported"); + #[cfg(target_arch = "x86_64")] + let add_key = 248_u32; + #[cfg(target_arch = "aarch64")] + let add_key = 217_u32; + assert_syscall_denied(&program, add_key, "add_key"); + } + + #[test] + fn descendant_filter_denies_request_key() { + let program = seccomp_instructions().expect("the test architecture is supported"); + #[cfg(target_arch = "x86_64")] + let request_key = 249_u32; + #[cfg(target_arch = "aarch64")] + let request_key = 218_u32; + assert_syscall_denied(&program, request_key, "request_key"); + } + + #[test] + fn descendant_filter_denies_keyctl() { + let program = seccomp_instructions().expect("the test architecture is supported"); + #[cfg(target_arch = "x86_64")] + let keyctl = 250_u32; + #[cfg(target_arch = "aarch64")] + let keyctl = 219_u32; + assert_syscall_denied(&program, keyctl, "keyctl"); + } + + fn assert_syscall_denied(program: &[super::FilterInstruction], syscall: u32, name: &str) { + let (index, check) = program + .iter() + .enumerate() + .find(|(_, entry)| entry.value == syscall) + .unwrap_or_else(|| panic!("{name} is checked")); + assert_eq!(check.code, 0x15); + let denial = index + 1 + usize::from(check.jump_true); + assert_eq!( + program.get(denial).map(|entry| entry.value), + Some(0x0005_0001) + ); + } + + #[test] + fn worker_executable_remains_pinned_after_path_replacement() { + let directory = tempfile::tempdir().expect("temporary directory is available"); + let worker = directory.path().join("worker"); + fs::write(&worker, b"original").expect("fixture worker is written"); + fs::set_permissions(&worker, fs::Permissions::from_mode(0o700)) + .expect("fixture worker is executable"); + let pinned = open_worker_executable(&worker).expect("worker is pinned"); + let replacement = directory.path().join("replacement"); + fs::write(&replacement, b"replacement").expect("replacement is written"); + fs::set_permissions(&replacement, fs::Permissions::from_mode(0o700)) + .expect("replacement is executable"); + fs::rename(&replacement, &worker).expect("worker path is atomically replaced"); + assert_eq!( + fs::read(&pinned.proc_path).expect("pinned handle remains readable"), + b"original" + ); + assert_eq!( + fs::read(worker).expect("replacement path is readable"), + b"replacement" + ); + } + + #[test] + fn worker_executable_snapshot_ignores_in_place_rewrites() { + let directory = tempfile::tempdir().expect("temporary directory is available"); + let worker = directory.path().join("worker"); + fs::write(&worker, b"original").expect("fixture worker is written"); + fs::set_permissions(&worker, fs::Permissions::from_mode(0o700)) + .expect("fixture worker is executable"); + let pinned = open_worker_executable(&worker).expect("worker is snapshotted"); + fs::write(&worker, b"replacement").expect("worker inode is rewritten"); + assert_eq!( + fs::read(&pinned.proc_path).expect("sealed snapshot remains readable"), + b"original" + ); + assert_eq!( + fs::read(worker).expect("rewritten path is readable"), + b"replacement" + ); + } + + #[test] + fn executable_snapshot_rejects_bytes_above_its_bound() { + let directory = tempfile::tempdir().expect("temporary directory is available"); + let worker = directory.path().join("worker"); + let file = fs::File::create(&worker).expect("fixture worker is created"); + file.set_len(MAX_EXECUTABLE_SNAPSHOT_BYTES + 1) + .expect("fixture worker is enlarged"); + fs::set_permissions(&worker, fs::Permissions::from_mode(0o700)) + .expect("fixture worker is executable"); + assert!(matches!( + open_executable_snapshot( + &worker, + ConstructionTarget::Worker, + MAX_EXECUTABLE_SNAPSHOT_BYTES, + ), + Err(super::SandboxedFileMediaProcessorConstructionError::Worker) + )); + } + + #[cfg(target_arch = "x86_64")] + #[test] + fn descendant_filter_rejects_the_x32_syscall_abi() { + let program = seccomp_instructions().expect("the test architecture is supported"); + let x32_check = program + .iter() + .find(|entry| entry.value == 0x4000_0000) + .expect("the x32 ABI bit is checked"); + assert_eq!(x32_check.code, 0x45); + assert_ne!(x32_check.jump_true, 0); + } +} diff --git a/crates/file-media-processor-runtime/src/unsupported.rs b/crates/file-media-processor-runtime/src/unsupported.rs new file mode 100644 index 0000000000..9925369b0f --- /dev/null +++ b/crates/file-media-processor-runtime/src/unsupported.rs @@ -0,0 +1,120 @@ +use std::{error::Error, fmt, path::PathBuf}; + +use signalbox_file_media_runtime::{ + CancellationSignal, FileMediaProcessCeilings, FileMediaProcessor, FileMediaProcessorFuture, + FileMediaProviderDeclaration, FileMediaProviderReadRequest, FileMediaProviderValidationRequest, + ProcessorFailure, ProcessorIsolation, ProcessorProbeOutput, ProcessorReadOutput, + ProcessorValidationOutput, ReaderIdentity, VerifiedBlobSource, +}; + +/// Provider-to-worker binding unavailable outside Linux. +#[derive(Clone, Debug)] +pub struct WorkerBinding { + declaration: FileMediaProviderDeclaration, +} + +impl WorkerBinding { + /// Rejects construction because this platform cannot provide the sandbox. + pub fn try_new( + _program: impl Into, + _declaration: FileMediaProviderDeclaration, + ) -> Result { + Err(SandboxedFileMediaProcessorConstructionError::Unsupported) + } + + /// Borrows the provider declaration associated with this binding. + pub const fn declaration(&self) -> &FileMediaProviderDeclaration { + &self.declaration + } +} + +/// Non-Linux counterpart of the Linux sandbox processor. +#[derive(Clone, Debug)] +pub struct SandboxedFileMediaProcessor { + ceilings: FileMediaProcessCeilings, +} + +impl SandboxedFileMediaProcessor { + /// Rejects construction because this platform cannot provide the sandbox. + pub fn try_new( + _bubblewrap: impl Into, + _bindings: Vec, + _ceilings: FileMediaProcessCeilings, + ) -> Result { + Err(SandboxedFileMediaProcessorConstructionError::Unsupported) + } + + /// Reports that the accepted Linux isolation profile is unavailable. + pub async fn verify_isolation(&self) -> ProcessorIsolation { + ProcessorIsolation::Unavailable + } + + /// Returns the effective process ceilings. + pub const fn ceilings(&self) -> FileMediaProcessCeilings { + self.ceilings + } +} + +impl FileMediaProcessor for SandboxedFileMediaProcessor { + fn probe<'a>( + &'a self, + _reader: &'a ReaderIdentity, + _source: &'a dyn VerifiedBlobSource, + _cancellation: &'a dyn CancellationSignal, + ) -> FileMediaProcessorFuture<'a, ProcessorProbeOutput> { + Box::pin(async { Err(ProcessorFailure::Unavailable.into()) }) + } + + fn validate<'a>( + &'a self, + _reader: &'a ReaderIdentity, + _request: FileMediaProviderValidationRequest, + _source: &'a dyn VerifiedBlobSource, + _cancellation: &'a dyn CancellationSignal, + ) -> FileMediaProcessorFuture<'a, ProcessorValidationOutput> { + Box::pin(async { Err(ProcessorFailure::Unavailable.into()) }) + } + + fn read<'a>( + &'a self, + _reader: &'a ReaderIdentity, + _request: FileMediaProviderReadRequest, + _source: &'a dyn VerifiedBlobSource, + _cancellation: &'a dyn CancellationSignal, + ) -> FileMediaProcessorFuture<'a, ProcessorReadOutput> { + Box::pin(async { Err(ProcessorFailure::Unavailable.into()) }) + } +} + +/// Checked sandbox configuration could not be constructed. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum SandboxedFileMediaProcessorConstructionError { + /// This platform cannot provide the accepted sandbox. + Unsupported, + /// Bubblewrap was invalid. + Bubblewrap, + /// A worker executable was invalid. + Worker, + /// Executable snapshots exceeded their aggregate ceiling. + ExecutableSnapshots, + /// Worker bindings exceeded their count ceiling. + WorkerBindings, + /// Reader declarations exceeded their registry-compatible count ceilings. + ReaderInventory, + /// Process ceilings were invalid. + Ceilings, + /// The per-invocation controller was unavailable. + TaskController, + /// A provider was duplicated. + DuplicateProvider, + /// A reader identity was duplicated. + DuplicateReader, +} + +impl fmt::Display for SandboxedFileMediaProcessorConstructionError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("file-media sandbox is unsupported on this platform") + } +} + +impl Error for SandboxedFileMediaProcessorConstructionError {} diff --git a/crates/file-media-processor-runtime/src/worker.rs b/crates/file-media-processor-runtime/src/worker.rs new file mode 100644 index 0000000000..9a002d6db5 --- /dev/null +++ b/crates/file-media-processor-runtime/src/worker.rs @@ -0,0 +1,343 @@ +use std::{collections::BTreeMap, error::Error, fmt, num::NonZeroU64, sync::Arc}; + +use signalbox_file_media_runtime::{ + FileMediaProvider, FileMediaProviderDeclaration, FileMediaProviderFailure, NeverCancelled, + ReaderIdentity, SourceReadError, SourceReadFuture, VerifiedBlobSource, +}; +use tokio::io::{AsyncWriteExt as _, Stdin, Stdout}; +use tokio::sync::Mutex; + +use crate::{ + broker::{read_frame, write_frame}, + protocol::{ + DaemonFrame, Invocation, WorkerFrame, declaration_fingerprint_ordered, decode_bytes, + }, +}; + +/// Immutable worker-side inventory of compiled format providers. +pub struct WorkerCatalog { + providers: Vec>, + provider_order: Vec, + readers: BTreeMap, +} + +impl fmt::Debug for WorkerCatalog { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("WorkerCatalog") + .field("provider_count", &self.providers.len()) + .field("reader_count", &self.readers.len()) + .finish() + } +} + +impl WorkerCatalog { + /// Builds a deterministic dispatch inventory from compiled providers. + pub fn try_new( + providers: Vec>, + ) -> Result { + if providers.is_empty() { + return Err(WorkerCatalogConstructionError::Empty); + } + let mut readers = BTreeMap::new(); + let mut provider_names = Vec::new(); + for (index, provider) in providers.iter().enumerate() { + let declaration = provider.declaration(); + if provider_names.contains(declaration.provider()) { + return Err(WorkerCatalogConstructionError::DuplicateProvider); + } + provider_names.push(declaration.provider().clone()); + for reader in declaration.readers() { + if readers.insert(reader.identity().clone(), index).is_some() { + return Err(WorkerCatalogConstructionError::DuplicateReader); + } + } + } + let mut provider_order = (0..providers.len()).collect::>(); + provider_order.sort_by(|left, right| provider_names[*left].cmp(&provider_names[*right])); + Ok(Self { + providers, + provider_order, + readers, + }) + } + + /// Returns declarations in worker construction order for daemon registration. + pub fn declarations(&self) -> Vec { + self.providers + .iter() + .map(|provider| provider.declaration()) + .collect() + } + + fn declaration_fingerprint( + &self, + requested_providers: &[std::ffi::OsString], + ) -> Result<[u8; 32], WorkerServiceError> { + let mut selected = if requested_providers.is_empty() { + self.provider_order.clone() + } else { + Vec::with_capacity(requested_providers.len()) + }; + for requested in requested_providers { + let requested = requested.to_str().ok_or(WorkerServiceError::Protocol)?; + let index = self + .provider_order + .iter() + .copied() + .find(|index| self.providers[*index].declaration().provider().as_str() == requested) + .ok_or(WorkerServiceError::Protocol)?; + if selected.contains(&index) { + return Err(WorkerServiceError::Protocol); + } + selected.push(index); + } + selected.sort_by(|left, right| { + self.providers[*left] + .declaration() + .provider() + .cmp(self.providers[*right].declaration().provider()) + }); + Ok(declaration_fingerprint_ordered( + selected.len(), + selected + .iter() + .map(|index| self.providers[*index].declaration()), + )) + } + + fn provider( + &self, + reader: &ReaderIdentity, + ) -> Result<&dyn FileMediaProvider, WorkerServiceError> { + let index = self + .readers + .get(reader) + .copied() + .ok_or(WorkerServiceError::Protocol)?; + self.providers + .get(index) + .map(Box::as_ref) + .ok_or(WorkerServiceError::Protocol) + } +} + +/// Static worker catalog could not be constructed. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum WorkerCatalogConstructionError { + /// At least one provider is required by a worker executable. + Empty, + /// Two compiled providers used the same name. + DuplicateProvider, + /// Two declarations used the same reader identity. + DuplicateReader, +} + +impl fmt::Display for WorkerCatalogConstructionError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str(match self { + Self::Empty => "worker provider inventory is empty", + Self::DuplicateProvider => "worker provider identity is duplicated", + Self::DuplicateReader => "worker reader identity is duplicated", + }) + } +} + +impl Error for WorkerCatalogConstructionError {} + +/// Content-silent worker service failure. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum WorkerServiceError { + /// Request framing, checked values, identity, or source protocol was invalid. + Protocol, + /// A compiled provider failed without a complete typed result. + Provider, +} + +impl fmt::Display for WorkerServiceError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str(match self { + Self::Protocol => "file-media worker protocol failed", + Self::Provider => "file-media provider failed", + }) + } +} + +impl Error for WorkerServiceError {} + +/// Serves exactly one daemon invocation over length-delimited standard I/O. +pub async fn serve_one(catalog: &WorkerCatalog) -> Result<(), WorkerServiceError> { + let arguments = std::env::args_os().skip(1).collect::>(); + if arguments.first().is_some_and(|argument| { + argument == std::ffi::OsStr::new("--signalbox-file-media-isolation-probe") + }) { + let fingerprint = catalog.declaration_fingerprint(&arguments[1..])?; + let mut output = tokio::io::stdout(); + output + .write_all(fingerprint.as_slice()) + .await + .map_err(|_| WorkerServiceError::Protocol)?; + return output + .shutdown() + .await + .map_err(|_| WorkerServiceError::Protocol); + } + if !arguments.is_empty() { + return Err(WorkerServiceError::Protocol); + } + let mut input = tokio::io::stdin(); + let output = tokio::io::stdout(); + let initial: DaemonFrame = read_frame(&mut input) + .await + .map_err(|_| WorkerServiceError::Protocol)?; + let DaemonFrame::Invocation { invocation } = initial else { + return Err(WorkerServiceError::Protocol); + }; + let invocation = *invocation; + let source_wire = *invocation.source(); + let source = BrokeredWorkerSource::new(source_wire, input, output)?; + let frame = dispatch(catalog, invocation, &source).await?; + let mut transport = source.transport.lock().await; + write_frame(&mut transport.output, &frame) + .await + .map_err(|_| WorkerServiceError::Protocol)?; + transport + .output + .shutdown() + .await + .map_err(|_| WorkerServiceError::Protocol) +} + +async fn dispatch( + catalog: &WorkerCatalog, + invocation: Invocation, + source: &BrokeredWorkerSource, +) -> Result { + match invocation { + Invocation::Probe { reader, .. } => { + let reader = + ReaderIdentity::try_from(reader).map_err(|_| WorkerServiceError::Protocol)?; + let provider = catalog.provider(&reader)?; + let output = provider + .probe(&reader, source, &NeverCancelled) + .await + .map_err(map_provider_failure)?; + Ok(WorkerFrame::ProbeResult { output }) + } + Invocation::Validate { + reader, request, .. + } => { + let reader = + ReaderIdentity::try_from(reader).map_err(|_| WorkerServiceError::Protocol)?; + let request: signalbox_file_media_runtime::FileMediaProviderValidationRequest = request + .try_into() + .map_err(|_| WorkerServiceError::Protocol)?; + require_source_identity(source, &request.source)?; + let provider = catalog.provider(&reader)?; + let output = provider + .inspect(&reader, request, source, &NeverCancelled) + .await + .map_err(map_provider_failure)?; + Ok(WorkerFrame::ValidationResult { output }) + } + Invocation::Read { + reader, request, .. + } => { + let reader = + ReaderIdentity::try_from(reader).map_err(|_| WorkerServiceError::Protocol)?; + let request: signalbox_file_media_runtime::FileMediaProviderReadRequest = request + .try_into() + .map_err(|_| WorkerServiceError::Protocol)?; + require_source_identity(source, &request.source)?; + let provider = catalog.provider(&reader)?; + let output = provider + .read(&reader, request, source, &NeverCancelled) + .await + .map_err(map_provider_failure)?; + Ok(WorkerFrame::ReadResult { output }) + } + } +} + +fn require_source_identity( + source: &BrokeredWorkerSource, + requested: &signalbox_file_media_runtime::FileUse, +) -> Result<(), WorkerServiceError> { + if source.digest() == requested.digest() && source.byte_length() == requested.byte_length() { + Ok(()) + } else { + Err(WorkerServiceError::Protocol) + } +} + +fn map_provider_failure(_: FileMediaProviderFailure) -> WorkerServiceError { + WorkerServiceError::Provider +} + +struct WorkerTransport { + input: Stdin, + output: Stdout, +} + +struct BrokeredWorkerSource { + digest: signalbox_file_media_runtime::FileDigest, + byte_length: NonZeroU64, + transport: Arc>, +} + +impl BrokeredWorkerSource { + fn new( + source: crate::protocol::WireSource, + input: Stdin, + output: Stdout, + ) -> Result { + Ok(Self { + digest: source.digest(), + byte_length: source + .byte_length() + .map_err(|_| WorkerServiceError::Protocol)?, + transport: Arc::new(Mutex::new(WorkerTransport { input, output })), + }) + } +} + +impl VerifiedBlobSource for BrokeredWorkerSource { + fn digest(&self) -> signalbox_file_media_runtime::FileDigest { + self.digest + } + + fn byte_length(&self) -> NonZeroU64 { + self.byte_length + } + + fn read_range(&self, offset: u64, length: NonZeroU64) -> SourceReadFuture<'_> { + Box::pin(async move { + let mut transport = self.transport.lock().await; + let request = WorkerFrame::ReadRange { + offset, + length: length.get(), + }; + write_frame(&mut transport.output, &request) + .await + .map_err(|_| SourceReadError::Unavailable)?; + let response: DaemonFrame = read_frame(&mut transport.input) + .await + .map_err(|_| SourceReadError::Unavailable)?; + match response { + DaemonFrame::RangeBytes { bytes_base64 } => { + let bytes = + decode_bytes(&bytes_base64).map_err(|_| SourceReadError::Integrity)?; + if bytes.len() + == usize::try_from(length.get()).map_err(|_| SourceReadError::Integrity)? + { + Ok(bytes) + } else { + Err(SourceReadError::Integrity) + } + } + DaemonFrame::RangeFailure => Err(SourceReadError::Unavailable), + DaemonFrame::Invocation { .. } => Err(SourceReadError::Integrity), + } + }) + } +} diff --git a/crates/file-media-processor-runtime/tests/isolation.rs b/crates/file-media-processor-runtime/tests/isolation.rs new file mode 100644 index 0000000000..82d5cedaca --- /dev/null +++ b/crates/file-media-processor-runtime/tests/isolation.rs @@ -0,0 +1,273 @@ +#![cfg(target_os = "linux")] + +use std::{ + error::Error, + num::NonZeroU64, + path::PathBuf, + str::FromStr, + sync::{ + Arc, + atomic::{AtomicBool, Ordering}, + }, + time::Duration, +}; + +use signalbox_file_media_processor_runtime::{SandboxedFileMediaProcessor, WorkerBinding}; +use signalbox_file_media_runtime::{ + AttachmentKind, CancellationSignal, CanonicalJsonObjectSchema, CanonicalMediaType, + DeclaredMediaType, FileDigest, FileMediaCeilings, FileMediaFailure, FileMediaProcessCeilings, + FileMediaProcessLimitOverrides, FileMediaProcessor, FileMediaProviderDeclaration, + FileMediaRegistry, FileReaderName, FileReaderProviderName, FileReaderRevision, FileUse, + InspectionRequest, NeverCancelled, ProbeDeclaration, ProbeDeclarationInput, ProbeStrength, + ProcessorBoundaryFailure, ProcessorFailure, ProcessorIsolation, ProcessorProbeOutput, + ReadAccessPattern, ReadViewBounds, ReadViewDeclaration, ReadViewName, ReaderDeclaration, + ReaderDeclarationInput, ReaderIdentity, ReasonCode, SourceReadError, SourceReadFuture, + StreamingTextFallback, VerifiedBlobSource, +}; + +struct BytesSource(Vec); + +impl VerifiedBlobSource for BytesSource { + fn digest(&self) -> FileDigest { + FileDigest::from_bytes([7; 32]) + } + + fn byte_length(&self) -> NonZeroU64 { + NonZeroU64::new(self.0.len() as u64).unwrap_or(NonZeroU64::MIN) + } + + fn read_range(&self, offset: u64, length: NonZeroU64) -> SourceReadFuture<'_> { + Box::pin(async move { + let start = usize::try_from(offset).map_err(|_| SourceReadError::RangeOutOfBounds)?; + let length = + usize::try_from(length.get()).map_err(|_| SourceReadError::RangeOutOfBounds)?; + let end = start + .checked_add(length) + .ok_or(SourceReadError::RangeOutOfBounds)?; + self.0 + .get(start..end) + .map(<[u8]>::to_vec) + .ok_or(SourceReadError::RangeOutOfBounds) + }) + } +} + +/// INV-081: registered processors require the real accepted sandbox profile. +#[tokio::test] +#[ignore = "requires the delegated real file-media sandbox profile"] +async fn inv081_real_worker_has_the_accepted_isolation_profile() -> Result<(), Box> { + let (processor, reader) = available_processor(FileMediaProcessCeilings::version_one()).await?; + let output = processor + .probe(&reader, &BytesSource(vec![b'I']), &NeverCancelled) + .await?; + assert_eq!(output, successful_probe()); + Ok(()) +} + +/// INV-082: worker source reads pass only through the daemon's bounded broker. +#[tokio::test] +#[ignore = "requires the delegated real file-media sandbox profile"] +async fn inv082_worker_can_read_only_through_the_bounded_broker() -> Result<(), Box> { + let (processor, reader) = available_processor(FileMediaProcessCeilings::version_one()).await?; + let output = processor + .probe(&reader, &BytesSource(vec![b'V']), &NeverCancelled) + .await; + assert_eq!( + output, + Err(ProcessorBoundaryFailure::Processor( + ProcessorFailure::Protocol + )) + ); + Ok(()) +} + +/// INV-083: an incomplete result from a crashed worker is never admitted. +#[tokio::test] +#[ignore = "requires the delegated real file-media sandbox profile"] +async fn inv083_worker_crash_discards_its_incomplete_result() -> Result<(), Box> { + let (processor, reader) = available_processor(FileMediaProcessCeilings::version_one()).await?; + let output = processor + .probe(&reader, &BytesSource(vec![b'C']), &NeverCancelled) + .await; + assert_eq!( + output, + Err(ProcessorBoundaryFailure::Processor( + ProcessorFailure::Failed + )) + ); + Ok(()) +} + +/// INV-084: the daemon wall deadline terminates work without content leakage. +#[tokio::test] +#[ignore = "requires the delegated real file-media sandbox profile"] +async fn inv084_worker_wall_timeout_is_a_content_silent_failure() -> Result<(), Box> { + let ceilings = FileMediaProcessCeilings::try_lower(FileMediaProcessLimitOverrides { + memory_bytes: 512 * 1024 * 1024, + cpu_seconds: 60, + wall_seconds: 1, + file_descriptors: 32, + stderr_bytes: 16_384, + }) + .ok_or("lowered test ceilings must be valid")?; + let (processor, reader) = available_processor(ceilings).await?; + let output = processor + .probe(&reader, &BytesSource(vec![b'T']), &NeverCancelled) + .await; + assert_eq!( + output, + Err(ProcessorBoundaryFailure::Processor( + ProcessorFailure::TimedOut + )) + ); + Ok(()) +} + +/// INV-085: workers may create threads but cannot create descendant processes. +#[tokio::test] +#[ignore = "requires the delegated real file-media sandbox profile"] +async fn inv085_worker_process_creation_is_denied_without_blocking_threads() +-> Result<(), Box> { + let (processor, reader) = available_processor(FileMediaProcessCeilings::version_one()).await?; + let output = processor + .probe(&reader, &BytesSource(vec![b'X']), &NeverCancelled) + .await?; + assert_eq!(output, successful_probe()); + Ok(()) +} + +/// INV-086: injection-shaped worker output is sanitized before registry use. +#[tokio::test] +#[ignore = "requires the delegated real file-media sandbox profile"] +async fn inv086_hostile_worker_output_never_propagates() -> Result<(), Box> { + let (processor, _) = available_processor(FileMediaProcessCeilings::version_one()).await?; + let source = BytesSource(vec![b'H']); + let registry = FileMediaRegistry::try_new( + processor_declarations()?, + FileMediaCeilings::version_one(), + ProcessorIsolation::Available, + )?; + let request = InspectionRequest { + source: FileUse::new( + source.digest(), + source.byte_length(), + AttachmentKind::File, + DeclaredMediaType::try_new("application/octet-stream")?, + None, + ), + visible_part: None, + }; + let output = registry + .inspect(&processor, request, &source, &NeverCancelled) + .await; + assert_eq!(output, Err(FileMediaFailure::ProcessorFailed)); + Ok(()) +} + +/// INV-087: authoritative cancellation terminates in-flight worker processing. +#[tokio::test] +#[ignore = "requires the delegated real file-media sandbox profile"] +async fn inv087_authoritative_cancellation_terminates_the_worker() -> Result<(), Box> { + let (processor, reader) = available_processor(FileMediaProcessCeilings::version_one()).await?; + let cancellation = Arc::new(TestCancellation::default()); + let trigger = Arc::clone(&cancellation); + let cancellation_task = tokio::spawn(async move { + tokio::time::sleep(Duration::from_millis(20)).await; + trigger.cancelled.store(true, Ordering::Release); + }); + let output = processor + .probe(&reader, &BytesSource(vec![b'T']), cancellation.as_ref()) + .await; + cancellation_task.await?; + assert_eq!( + output, + Err(ProcessorBoundaryFailure::Processor( + ProcessorFailure::Cancelled + )) + ); + Ok(()) +} + +async fn available_processor( + ceilings: FileMediaProcessCeilings, +) -> Result<(SandboxedFileMediaProcessor, ReaderIdentity), Box> { + let built = processor(ceilings)?; + if built.0.verify_isolation().await == ProcessorIsolation::Available { + return Ok(built); + } + Err("the real file-media sandbox profile is unavailable".into()) +} + +fn successful_probe() -> ProcessorProbeOutput { + ProcessorProbeOutput::Candidate { + media_type: String::from("application/x-signalbox-synthetic"), + strength: ProbeStrength::Strong, + } +} + +fn processor( + ceilings: FileMediaProcessCeilings, +) -> Result<(SandboxedFileMediaProcessor, ReaderIdentity), Box> { + let (declaration, reader) = declaration()?; + let worker = PathBuf::from(env!("CARGO_BIN_EXE_signalbox-file-media-synthetic-worker")); + let binding = WorkerBinding::try_new(worker, declaration)?; + let processor = + SandboxedFileMediaProcessor::try_new("/usr/bin/bwrap", vec![binding], ceilings)?; + Ok((processor, reader)) +} + +fn declaration() -> Result<(FileMediaProviderDeclaration, ReaderIdentity), Box> { + let provider = FileReaderProviderName::try_new("synthetic")?; + let reader = ReaderIdentity::new( + provider.clone(), + FileReaderName::try_new("fixture")?, + FileReaderRevision::try_new("v1")?, + ); + let view = ReadViewDeclaration::try_new( + ReadViewName::try_new("text")?, + String::from("Reads synthetic text."), + CanonicalJsonObjectSchema::try_new(r#"{"type":"object"}"#)?, + ReadAccessPattern::Streaming { maximum_ranges: 1 }, + ReadViewBounds::Text { + source_bytes: 64, + output_bytes: 64, + }, + )?; + let declaration = ReaderDeclaration::try_new(ReaderDeclarationInput { + provider: provider.clone(), + reader: reader.reader().clone(), + revision: reader.revision().clone(), + media_types: vec![CanonicalMediaType::from_str( + "application/x-signalbox-synthetic", + )?], + probe: ProbeDeclaration::new(ProbeDeclarationInput { + prefix_bytes: 1, + suffix_bytes: 0, + range_count: 1, + cumulative_bytes: 1, + }), + validation: signalbox_file_media_runtime::ValidationDeclaration::new(64, 1), + views: vec![view], + reason_codes: vec![ReasonCode::try_new("synthetic_failure")?], + streaming_text_fallback: StreamingTextFallback::Disabled, + })?; + Ok(( + FileMediaProviderDeclaration::try_new(provider, vec![declaration])?, + reader, + )) +} + +fn processor_declarations() -> Result, Box> { + Ok(vec![declaration()?.0]) +} + +#[derive(Default)] +struct TestCancellation { + cancelled: AtomicBool, +} + +impl CancellationSignal for TestCancellation { + fn is_cancelled(&self) -> bool { + self.cancelled.load(Ordering::Acquire) + } +} diff --git a/crates/file-media-provider-runtime/Cargo.toml b/crates/file-media-provider-runtime/Cargo.toml new file mode 100644 index 0000000000..13d092c036 --- /dev/null +++ b/crates/file-media-provider-runtime/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "signalbox-file-media-provider-runtime" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +repository.workspace = true +publish.workspace = true + +[dependencies] +signalbox-domain = { path = "../domain" } +signalbox-file-media-runtime = { path = "../file-media-runtime" } +signalbox-tools-file-media = { path = "../tools-file-media" } + +[lints] +workspace = true diff --git a/crates/file-media-provider-runtime/src/lib.rs b/crates/file-media-provider-runtime/src/lib.rs new file mode 100644 index 0000000000..1efda92872 --- /dev/null +++ b/crates/file-media-provider-runtime/src/lib.rs @@ -0,0 +1,204 @@ +//! Application bridge from visible blob uses to the provider-neutral registry. +//! +//! The resolver port is the sole authority for rendered-frontier visibility and +//! verified-source construction. It returns no store locator, path, credential, +//! or open database transaction to the registry or processor. + +use std::{future::Future, pin::Pin}; + +use signalbox_domain::BlobDigest; +use signalbox_file_media_runtime::{ + CancellationSignal, FileDigest, FileMediaFailure, FileMediaProcessor, FileMediaRegistry, + FileReadRequest, FileUse, InspectionRequest, VerifiedBlobSource, +}; +use signalbox_tools_file_media::{ + FileInspectServiceRequest, FileMediaAgentService, FileMediaAgentServiceFuture, + FileReadServiceRequest, +}; + +/// Converts the domain blob identity without changing its exact bytes. +pub const fn neutral_file_digest(digest: BlobDigest) -> FileDigest { + FileDigest::from_bytes(*digest.as_bytes()) +} + +/// One authorized semantic use and its placement-free verified source. +#[derive(Debug)] +pub struct ResolvedFileUse { + file_use: FileUse, + source: Source, +} + +impl ResolvedFileUse { + /// Constructs evidence returned by a visibility-authorizing resolver. + pub const fn new(file_use: FileUse, source: Source) -> Self { + Self { file_use, source } + } + + /// Borrows exact semantic use metadata. + pub const fn file_use(&self) -> &FileUse { + &self.file_use + } + + /// Borrows the verified placement-free source. + pub const fn source(&self) -> &Source { + &self.source + } + + /// Returns both owned parts. + pub fn into_parts(self) -> (FileUse, Source) { + (self.file_use, self.source) + } +} + +/// Boxed future returned by a rendered-frontier resolver. +pub type FileUseResolverFuture<'a, Source> = Pin< + Box, FileUseResolutionError>> + Send + 'a>, +>; + +/// Closed failure algebra owned by rendered-frontier resolution. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum FileUseResolutionError { + /// Digest is outside the rendered-frontier allow-set. + BlobNotVisible, + /// Blob catalog identity is absent. + BlobMissing, + /// Every replica contradicted exact bytes. + BlobCorrupt, + /// Blob access is temporarily unavailable. + BlobUnavailable, + /// Resolver authority or evidence was internally inconsistent. + Internal, +} + +impl From for FileMediaFailure { + fn from(value: FileUseResolutionError) -> Self { + match value { + FileUseResolutionError::BlobNotVisible => Self::BlobNotVisible, + FileUseResolutionError::BlobMissing => Self::BlobMissing, + FileUseResolutionError::BlobCorrupt => Self::BlobCorrupt, + FileUseResolutionError::BlobUnavailable => Self::BlobUnavailable, + FileUseResolutionError::Internal => Self::ProcessorFailed, + } + } +} + +/// Resolves exactly one visible use and ends catalog work before source I/O. +pub trait FileUseResolver: Send { + /// Placement-free source type returned with each authorization decision. + type Source: VerifiedBlobSource; + + /// Reuses the blob-read rendered-frontier allow-set and selects one use. + fn resolve( + &mut self, + request: FileInspectServiceRequest, + ) -> FileUseResolverFuture<'_, Self::Source>; +} + +/// Registry-backed implementation of both stable agent tools. +#[derive(Debug)] +pub struct RegistryFileMediaAgentService { + registry: FileMediaRegistry, + resolver: Resolver, + processor: Processor, + cancellation: Cancellation, +} + +impl + RegistryFileMediaAgentService +{ + /// Composes one immutable registry with visibility, processing, and cancellation ports. + pub const fn new( + registry: FileMediaRegistry, + resolver: Resolver, + processor: Processor, + cancellation: Cancellation, + ) -> Self { + Self { + registry, + resolver, + processor, + cancellation, + } + } + + /// Borrows the immutable registry snapshot. + pub const fn registry(&self) -> &FileMediaRegistry { + &self.registry + } +} + +impl FileMediaAgentService + for RegistryFileMediaAgentService +where + Resolver: FileUseResolver, + Processor: FileMediaProcessor, + Cancellation: CancellationSignal, +{ + fn inspect( + &mut self, + request: FileInspectServiceRequest, + ) -> FileMediaAgentServiceFuture<'_, signalbox_file_media_runtime::FileInspection> { + Box::pin(async move { + let requested_digest = request.digest(); + let visible_part = request.visible_part().cloned(); + let resolved = self + .resolver + .resolve(request) + .await + .map_err(FileMediaFailure::from)?; + let (file_use, source) = resolved.into_parts(); + if file_use.digest() != requested_digest { + return Err(FileMediaFailure::ProcessorFailed); + } + self.registry + .inspect( + &self.processor, + InspectionRequest { + source: file_use, + visible_part, + }, + &source, + &self.cancellation, + ) + .await + }) + } + + fn read( + &mut self, + request: FileReadServiceRequest, + ) -> FileMediaAgentServiceFuture<'_, signalbox_file_media_runtime::FileReadResult> { + Box::pin(async move { + let requested_digest = request.target().digest(); + let visible_part = request.target().visible_part().cloned(); + let view = request.view().clone(); + let input = request.clone().into_runtime_input(); + let target = + FileInspectServiceRequest::from_parts(requested_digest, visible_part.clone()); + let resolved = self + .resolver + .resolve(target) + .await + .map_err(FileMediaFailure::from)?; + let (file_use, source) = resolved.into_parts(); + if file_use.digest() != requested_digest { + return Err(FileMediaFailure::ProcessorFailed); + } + self.registry + .read( + &self.processor, + FileReadRequest { + inspection: InspectionRequest { + source: file_use, + visible_part, + }, + view, + input, + }, + &source, + &self.cancellation, + ) + .await + }) + } +} diff --git a/crates/file-media-runtime/Cargo.toml b/crates/file-media-runtime/Cargo.toml new file mode 100644 index 0000000000..ea04beaba0 --- /dev/null +++ b/crates/file-media-runtime/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "signalbox-file-media-runtime" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +repository.workspace = true +publish.workspace = true + +[dependencies] +futures-timer = "3.0.3" +futures-util = "0.3.31" +serde = { version = "1.0.219", features = ["derive"] } +serde_json = { version = "1.0.140", features = ["arbitrary_precision", "raw_value"] } + +[dev-dependencies] +tokio = { version = "1.53.0", default-features = false, features = ["rt", "time"] } + +[lints] +workspace = true diff --git a/crates/file-media-runtime/src/declaration.rs b/crates/file-media-runtime/src/declaration.rs new file mode 100644 index 0000000000..a8dbf71165 --- /dev/null +++ b/crates/file-media-runtime/src/declaration.rs @@ -0,0 +1,553 @@ +use std::{error::Error, fmt, future::Future, pin::Pin}; + +use crate::{ + CancellationSignal, CanonicalJsonObjectSchema, CanonicalMediaType, FileReaderName, + FileReaderProviderName, FileReaderRevision, FileUse, ProcessorProbeOutput, ProcessorReadOutput, + ProcessorValidationOutput, ReadViewName, ReaderIdentity, ReasonCode, VerifiedBlobSource, +}; + +// numeric-bound: ceiling - bounds retained model-facing view-description memory +const MAX_VIEW_DESCRIPTION_BYTES: usize = 512; + +/// Strength of one byte-derived probe candidate. +#[derive( + Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd, serde::Deserialize, serde::Serialize, +)] +#[serde(rename_all = "snake_case")] +pub enum ProbeStrength { + /// Caller declaration nominates a provider but is not evidence. + DeclaredCandidate, + /// A bounded complete prefix is provisional until full validation. + ProvisionalStructuralCandidate, + /// Bounded structure suggests a candidate requiring full validation. + StructuralCandidate, + /// A format-owned signature identifies a candidate. + Strong, +} + +/// Finite source-read envelope for one probe. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct ProbeDeclaration { + prefix_bytes: u64, + suffix_bytes: u64, + range_count: u32, + cumulative_bytes: u64, +} + +/// Labeled fields for one finite probe envelope. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct ProbeDeclarationInput { + /// Maximum prefix bytes available to the probe. + pub prefix_bytes: u64, + /// Maximum suffix bytes available to the probe. + pub suffix_bytes: u64, + /// Maximum exact range requests available to the probe. + pub range_count: u32, + /// Maximum cumulative bytes available to the probe. + pub cumulative_bytes: u64, +} + +impl ProbeDeclaration { + /// Declares a probe that may read only one bounded source prefix. + pub const fn prefix_only(prefix_bytes: u64) -> Self { + Self { + prefix_bytes, + suffix_bytes: 0, + range_count: 0, + cumulative_bytes: prefix_bytes, + } + } + + /// Declares one finite probe envelope from labeled fields. + pub const fn new(input: ProbeDeclarationInput) -> Self { + Self { + prefix_bytes: input.prefix_bytes, + suffix_bytes: input.suffix_bytes, + range_count: input.range_count, + cumulative_bytes: input.cumulative_bytes, + } + } + + /// Returns the prefix budget. + pub const fn prefix_bytes(self) -> u64 { + self.prefix_bytes + } + + /// Returns the suffix budget. + pub const fn suffix_bytes(self) -> u64 { + self.suffix_bytes + } + + /// Returns the arbitrary-range count. + pub const fn range_count(self) -> u32 { + self.range_count + } + + /// Returns the cumulative byte budget. + pub const fn cumulative_bytes(self) -> u64 { + self.cumulative_bytes + } +} + +/// Finite source-read envelope for one validation. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct ValidationDeclaration { + source_bytes: u64, + range_count: u32, +} + +impl ValidationDeclaration { + /// Declares one finite validation envelope. Registry construction checks ceilings. + pub const fn new(source_bytes: u64, range_count: u32) -> Self { + Self { + source_bytes, + range_count, + } + } + + /// Returns the cumulative source-byte budget. + pub const fn source_bytes(self) -> u64 { + self.source_bytes + } + + /// Returns the exact-range request budget. + pub const fn range_count(self) -> u32 { + self.range_count + } +} + +/// Whether one reader is eligible for the complete-stream UTF-8 fallback. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum StreamingTextFallback { + /// The reader never claims untyped bytes as text. + Disabled, + /// The reader may claim only after complete streaming validation. + Enabled, +} + +/// Declared source access posture for one view. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum ReadAccessPattern { + /// Monotonic streaming access. + Streaming { + /// Maximum sequential range requests for one read. + maximum_ranges: u32, + }, + /// Bounded exact-range access. + RandomAccess { + /// Maximum ranges requested for one read. + maximum_ranges: u32, + }, +} + +/// Closed common output vocabulary. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum ReadOutputKind { + /// Bounded UTF-8. + Text, + /// Bounded canonical JSON. + Structured, + /// Immutable image bytes registered before durable result commit. + Image, + /// Immutable audio bytes registered before durable result commit. + Audio, + /// Immutable general-file bytes admitted by a reviewed model adapter. + File, +} + +/// Output-specific finite bounds for one read view. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum ReadViewBounds { + /// Text body and cumulative source work. + Text { + /// Maximum source bytes requested. + source_bytes: u64, + /// Maximum UTF-8 bytes returned. + output_bytes: usize, + }, + /// Structured body and tree limits. + Structured { + /// Maximum source bytes requested. + source_bytes: u64, + /// Maximum compact JSON bytes returned. + output_bytes: usize, + /// Maximum JSON nesting. + depth: u32, + /// Maximum JSON nodes. + nodes: u64, + /// Maximum cumulative string bytes. + string_bytes: usize, + }, + /// Image output envelope. + Image { + /// Maximum source bytes requested. + source_bytes: u64, + /// Maximum width. + width: u32, + /// Maximum height. + height: u32, + /// Maximum decoded pixels. + pixels: u64, + /// Maximum presented bytes. + output_bytes: u64, + }, + /// Audio output envelope. + Audio { + /// Maximum source bytes requested. + source_bytes: u64, + /// Maximum channels. + channels: u16, + /// Maximum sample rate. + sample_rate_hz: u32, + /// Maximum duration. + duration_seconds: u32, + /// Maximum presented bytes. + output_bytes: u64, + }, + /// General-file output envelope. + File { + /// Maximum source bytes requested. + source_bytes: u64, + /// Maximum presented bytes. + output_bytes: u64, + }, +} + +impl ReadViewBounds { + /// Returns the common output kind implied by this envelope. + pub const fn output_kind(self) -> ReadOutputKind { + match self { + Self::Text { .. } => ReadOutputKind::Text, + Self::Structured { .. } => ReadOutputKind::Structured, + Self::Image { .. } => ReadOutputKind::Image, + Self::Audio { .. } => ReadOutputKind::Audio, + Self::File { .. } => ReadOutputKind::File, + } + } + + /// Returns the maximum cumulative source bytes. + pub const fn source_bytes(self) -> u64 { + match self { + Self::Text { source_bytes, .. } + | Self::Structured { source_bytes, .. } + | Self::Image { source_bytes, .. } + | Self::Audio { source_bytes, .. } + | Self::File { source_bytes, .. } => source_bytes, + } + } +} + +/// One provider-owned read view. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct ReadViewDeclaration { + name: ReadViewName, + description: String, + arguments_schema: CanonicalJsonObjectSchema, + access: ReadAccessPattern, + bounds: ReadViewBounds, +} + +impl ReadViewDeclaration { + /// Constructs one declaration. Registry construction checks every bound. + pub fn try_new( + name: ReadViewName, + description: String, + arguments_schema: CanonicalJsonObjectSchema, + access: ReadAccessPattern, + bounds: ReadViewBounds, + ) -> Result { + if description.is_empty() + || description.len() > MAX_VIEW_DESCRIPTION_BYTES + || description.contains('\0') + || description.chars().any(char::is_control) + { + return Err(RegistryDeclarationError::Description); + } + Ok(Self { + name, + description, + arguments_schema, + access, + bounds, + }) + } + + /// Borrows the view name. + pub const fn name(&self) -> &ReadViewName { + &self.name + } + + /// Borrows the model-facing bounded description. + pub fn description(&self) -> &str { + &self.description + } + + /// Borrows the object schema. + pub const fn arguments_schema(&self) -> &CanonicalJsonObjectSchema { + &self.arguments_schema + } + + /// Returns the declared access posture. + pub const fn access(&self) -> ReadAccessPattern { + self.access + } + + /// Returns output-specific bounds. + pub const fn bounds(&self) -> ReadViewBounds { + self.bounds + } + + /// Returns the common output kind. + pub const fn output_kind(&self) -> ReadOutputKind { + self.bounds.output_kind() + } +} + +/// Static declaration for one reader implementation. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct ReaderDeclaration { + identity: ReaderIdentity, + media_types: Vec, + probe: ProbeDeclaration, + validation: ValidationDeclaration, + views: Vec, + reason_codes: Vec, + streaming_text_fallback: StreamingTextFallback, +} + +/// Labeled candidate fields for one reader declaration. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct ReaderDeclarationInput { + /// Provider that owns the reader. + pub provider: FileReaderProviderName, + /// Reader name within that provider. + pub reader: FileReaderName, + /// Immutable implementation revision. + pub revision: FileReaderRevision, + /// Exact canonical media types owned by this reader. + pub media_types: Vec, + /// Finite probe envelope. + pub probe: ProbeDeclaration, + /// Finite validation envelope. + pub validation: ValidationDeclaration, + /// Nonempty provider-owned view inventory. + pub views: Vec, + /// Nonempty sanitized reason-code inventory. + pub reason_codes: Vec, + /// Complete-stream text fallback posture. + pub streaming_text_fallback: StreamingTextFallback, +} + +impl ReaderDeclaration { + /// Constructs one nonempty reader declaration. + pub fn try_new(input: ReaderDeclarationInput) -> Result { + if input.media_types.is_empty() || input.views.is_empty() || input.reason_codes.is_empty() { + return Err(RegistryDeclarationError::EmptyInventory); + } + Ok(Self { + identity: ReaderIdentity::new(input.provider, input.reader, input.revision), + media_types: input.media_types, + probe: input.probe, + validation: input.validation, + views: input.views, + reason_codes: input.reason_codes, + streaming_text_fallback: input.streaming_text_fallback, + }) + } + + /// Borrows the immutable reader identity. + pub const fn identity(&self) -> &ReaderIdentity { + &self.identity + } + + /// Borrows exact owned media types. + pub fn media_types(&self) -> &[CanonicalMediaType] { + &self.media_types + } + + /// Returns the probe envelope. + pub const fn probe(&self) -> ProbeDeclaration { + self.probe + } + + /// Returns the validation envelope. + pub const fn validation(&self) -> ValidationDeclaration { + self.validation + } + + /// Borrows provider-owned views. + pub fn views(&self) -> &[ReadViewDeclaration] { + &self.views + } + + /// Borrows registered sanitized reason codes. + pub fn reason_codes(&self) -> &[ReasonCode] { + &self.reason_codes + } + + /// Returns text fallback posture. + pub const fn streaming_text_fallback(&self) -> StreamingTextFallback { + self.streaming_text_fallback + } +} + +/// Static declaration contributed by one compiled provider. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct FileMediaProviderDeclaration { + provider: FileReaderProviderName, + readers: Vec, +} + +impl FileMediaProviderDeclaration { + /// Constructs one provider whose readers all carry its exact identity. + pub fn try_new( + provider: FileReaderProviderName, + readers: Vec, + ) -> Result { + if readers.is_empty() { + return Err(RegistryDeclarationError::EmptyInventory); + } + if readers + .iter() + .any(|reader| reader.identity().provider() != &provider) + { + return Err(RegistryDeclarationError::ForeignReader); + } + Ok(Self { provider, readers }) + } + + /// Borrows the provider identity. + pub const fn provider(&self) -> &FileReaderProviderName { + &self.provider + } + + /// Borrows declared readers. + pub fn readers(&self) -> &[ReaderDeclaration] { + &self.readers + } + + pub(crate) fn sort_readers(&mut self) { + self.readers + .sort_by(|left, right| left.identity().cmp(right.identity())); + } +} + +/// Provider request to validate one candidate selected by the registry. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct FileMediaProviderValidationRequest { + /// Exact semantic use. + pub source: FileUse, + /// Candidate media type owned by this reader. + pub media_type: CanonicalMediaType, + /// Evidence path requested by the registry. + pub evidence: crate::ValidationEvidence, + /// Maximum cumulative source bytes the processor broker may serve. + pub maximum_source_bytes: u64, + /// Maximum exact ranges the processor broker may serve. + pub maximum_ranges: u32, + /// Effective maximum image width or height for decoded-image work. + pub maximum_image_axis: u32, + /// Effective maximum decoded image pixels. + pub maximum_decoded_image_pixels: u64, +} + +/// Provider request to interpret one validated file through one view. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct FileMediaProviderReadRequest { + /// Exact semantic use whose bytes the registry validated. + pub source: FileUse, + /// Registry-selected canonical media type. + pub detected_media_type: CanonicalMediaType, + /// Registry-admitted validation evidence. + pub validation: crate::ValidationEvidence, + /// Registry-sanitized provider metadata. + pub metadata: crate::BoundedMetadata, + /// Exact provider-owned view. + pub view: ReadViewName, + /// Closed initial-options or continuation input. + pub input: crate::FileReadInput, + /// Effective maximum image width or height for decoded-image work. + pub maximum_image_axis: u32, + /// Effective maximum decoded image pixels. + pub maximum_decoded_image_pixels: u64, + /// Maximum entries the registry may admit in any structured container. + pub maximum_container_entries: u64, +} + +/// Adapter-owned execution failure inside an isolated worker. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum FileMediaProviderFailure { + /// The adapter could not complete its bounded format operation. + Failed, +} + +impl fmt::Display for FileMediaProviderFailure { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("file media adapter failed") + } +} + +impl Error for FileMediaProviderFailure {} + +/// Boxed adapter future used by isolated worker-side provider implementations. +pub type FileMediaProviderFuture<'a, Output> = + Pin> + Send + 'a>>; + +/// Worker-side format adapter contract. +/// +/// The daemon never calls this trait directly. Slice-two processor isolation +/// hosts implementations in a fresh worker and exposes only sanitized outputs +/// to the registry. +pub trait FileMediaProvider: Send + Sync { + /// Returns this adapter's static declaration. + fn declaration(&self) -> FileMediaProviderDeclaration; + + /// Runs a bounded byte probe. + fn probe<'a>( + &'a self, + reader: &'a ReaderIdentity, + source: &'a dyn VerifiedBlobSource, + cancellation: &'a dyn CancellationSignal, + ) -> FileMediaProviderFuture<'a, ProcessorProbeOutput>; + + /// Validates the registry-selected candidate before interpretation. + fn inspect<'a>( + &'a self, + reader: &'a ReaderIdentity, + request: FileMediaProviderValidationRequest, + source: &'a dyn VerifiedBlobSource, + cancellation: &'a dyn CancellationSignal, + ) -> FileMediaProviderFuture<'a, ProcessorValidationOutput>; + + /// Produces one bounded view from prior validation evidence. + fn read<'a>( + &'a self, + reader: &'a ReaderIdentity, + request: FileMediaProviderReadRequest, + source: &'a dyn VerifiedBlobSource, + cancellation: &'a dyn CancellationSignal, + ) -> FileMediaProviderFuture<'a, ProcessorReadOutput>; +} + +/// Closed declaration-construction failure. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum RegistryDeclarationError { + /// A required inventory was empty. + EmptyInventory, + /// A reader named another provider. + ForeignReader, + /// A view description was empty, excessive, or control-bearing. + Description, +} + +impl fmt::Display for RegistryDeclarationError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str(match self { + Self::EmptyInventory => "provider declaration has an empty required inventory", + Self::ForeignReader => "reader identity names another provider", + Self::Description => "view description is invalid", + }) + } +} + +impl Error for RegistryDeclarationError {} diff --git a/crates/file-media-runtime/src/detection.rs b/crates/file-media-runtime/src/detection.rs new file mode 100644 index 0000000000..7e83f244d0 --- /dev/null +++ b/crates/file-media-runtime/src/detection.rs @@ -0,0 +1,603 @@ +use std::{error::Error, fmt, future::Future, num::NonZeroU64, pin::Pin}; + +use crate::{ + BoundedMetadata, CanonicalMediaType, FileDigest, FileUse, ProbeStrength, + ReadContinuationCursor, ReadViewDeclaration, ReadViewName, ReaderIdentity, ReasonCode, + VisiblePartSelector, +}; + +/// Asynchronous exact-range read result from a verified blob source. +pub type SourceReadFuture<'a> = + Pin, SourceReadError>> + Send + 'a>>; + +/// Verified, placement-free byte authority exposed to a processor broker. +pub trait VerifiedBlobSource: Send + Sync { + /// Returns the exact verified digest. + fn digest(&self) -> FileDigest; + + /// Returns the exact verified positive length. + fn byte_length(&self) -> NonZeroU64; + + /// Reads one exact in-bounds range without exposing a path or store locator. + fn read_range(&self, offset: u64, length: NonZeroU64) -> SourceReadFuture<'_>; +} + +/// Content-silent verified-source failure. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum SourceReadError { + /// The immutable object no longer exists at any replica. + Missing, + /// Every readable replica contradicted its digest or length. + Corrupt, + /// At least one candidate could not presently be read. + Unavailable, + /// The requested exact range exceeded the source. + RangeOutOfBounds, + /// Source identity or catalog evidence was internally inconsistent. + Integrity, +} + +impl fmt::Display for SourceReadError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str(match self { + Self::Missing => "verified blob source is missing", + Self::Corrupt => "verified blob source is corrupt", + Self::Unavailable => "verified blob source is unavailable", + Self::RangeOutOfBounds => "verified blob source range is out of bounds", + Self::Integrity => "verified blob source evidence is inconsistent", + }) + } +} + +impl Error for SourceReadError {} + +/// Cooperative cancellation observed by providers and processor clients. +pub trait CancellationSignal: Send + Sync { + /// Returns whether authoritative cancellation has been requested. + fn is_cancelled(&self) -> bool; +} + +/// Cancellation signal that never fires, useful for bounded synchronous callers. +#[derive(Clone, Copy, Debug, Default)] +pub struct NeverCancelled; + +impl CancellationSignal for NeverCancelled { + fn is_cancelled(&self) -> bool { + false + } +} + +/// Registry request to inspect one visible semantic use. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct InspectionRequest { + /// Exact caller-supplied use metadata. + pub source: FileUse, + /// Stable selector when a digest appears through several visible uses. + pub visible_part: Option, +} + +/// Closed, content-silent validation evidence. +#[derive(Clone, Copy, Debug, Eq, PartialEq, serde::Deserialize, serde::Serialize)] +#[serde(rename_all = "snake_case")] +pub enum ValidationEvidence { + /// A strong format signature was structurally validated. + StrongSignature, + /// Structure was validated without a strong signature. + StructuralValidation, + /// A declared candidate was independently structurally validated. + DeclaredCandidateStructurallyValidated, + /// Complete streaming UTF-8 and control policy validation succeeded. + StreamingTextValidation, +} + +/// Registry-produced evidence that a reader validated exact bytes. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct ValidatedFile { + source: FileUse, + detected_media_type: CanonicalMediaType, + reader: ReaderIdentity, + validation: ValidationEvidence, + metadata: BoundedMetadata, + views: Vec, +} + +impl ValidatedFile { + pub(crate) fn new( + source: FileUse, + detected_media_type: CanonicalMediaType, + reader: ReaderIdentity, + validation: ValidationEvidence, + metadata: BoundedMetadata, + views: Vec, + ) -> Self { + Self { + source, + detected_media_type, + reader, + validation, + metadata, + views, + } + } + + /// Borrows the exact semantic use. + pub const fn source(&self) -> &FileUse { + &self.source + } + + /// Borrows the byte-validated canonical type. + pub const fn detected_media_type(&self) -> &CanonicalMediaType { + &self.detected_media_type + } + + /// Borrows the exact reader identity and revision. + pub const fn reader(&self) -> &ReaderIdentity { + &self.reader + } + + /// Returns the evidence class. + pub const fn validation(&self) -> ValidationEvidence { + self.validation + } + + /// Borrows bounded provider metadata. + pub const fn metadata(&self) -> &BoundedMetadata { + &self.metadata + } + + /// Borrows ordered provider-owned views. + pub fn views(&self) -> &[ReadViewDeclaration] { + &self.views + } +} + +/// Compact inspection status vocabulary exposed to agents. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum FileInspectionStatus { + /// A reader validated the bytes and declared views. + Validated, + /// No reader safely recognized the bytes. + Unknown, + /// A recognized format was malformed. + Malformed, + /// Incompatible strong claims made type selection unsafe. + Ambiguous, + /// Caller declaration disagreed with detected bytes. + DeclaredMismatch, + /// A recognized encrypted or locked file is terminal in version one. + EncryptedOrLocked, +} + +/// Complete registry inspection outcome. +#[derive(Clone, Debug, Eq, PartialEq)] +pub enum FileInspection { + /// Validated bytes and available views. + Validated(ValidatedFile), + /// Ordinary unknown bytes, still raw-readable. + Unknown { + /// Exact inspected use. + source: FileUse, + }, + /// A recognized signature or structure was malformed. + Malformed { + /// Exact inspected use. + source: FileUse, + /// Recognized type. + media_type: CanonicalMediaType, + /// Registered sanitized reason. + reason_code: ReasonCode, + }, + /// Incompatible strong candidates were observed. + Ambiguous { + /// Exact inspected use. + source: FileUse, + /// Canonically sorted distinct claims. + media_types: Vec, + }, + /// Declared metadata disagreed with byte evidence. + DeclaredMismatch { + /// Exact inspected use. + source: FileUse, + /// Parsed canonical declaration. + declared: CanonicalMediaType, + /// Byte-detected type. + detected: CanonicalMediaType, + }, + /// Recognized encrypted content; no password channel exists. + EncryptedOrLocked { + /// Exact inspected use. + source: FileUse, + /// Recognized type. + media_type: CanonicalMediaType, + }, +} + +impl FileInspection { + /// Returns the compact agent-visible status. + pub const fn status(&self) -> FileInspectionStatus { + match self { + Self::Validated(_) => FileInspectionStatus::Validated, + Self::Unknown { .. } => FileInspectionStatus::Unknown, + Self::Malformed { .. } => FileInspectionStatus::Malformed, + Self::Ambiguous { .. } => FileInspectionStatus::Ambiguous, + Self::DeclaredMismatch { .. } => FileInspectionStatus::DeclaredMismatch, + Self::EncryptedOrLocked { .. } => FileInspectionStatus::EncryptedOrLocked, + } + } + + /// Borrows the exact semantic use in every outcome. + pub const fn source(&self) -> &FileUse { + match self { + Self::Validated(validated) => validated.source(), + Self::Unknown { source } + | Self::Malformed { source, .. } + | Self::Ambiguous { source, .. } + | Self::DeclaredMismatch { source, .. } + | Self::EncryptedOrLocked { source, .. } => source, + } + } +} + +/// Agent request for one provider-owned view. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct FileReadRequest { + /// Inspection inputs; the registry repeats inspection. + pub inspection: InspectionRequest, + /// Exact provider-owned view name. + pub view: ReadViewName, + /// Closed initial-options or continuation input. + pub input: FileReadInput, +} + +/// Closed input mode for one typed read. +#[derive(Clone, Debug, Eq, PartialEq)] +pub enum FileReadInput { + /// Initial request carrying structured model-supplied options. + Initial { + /// Provider-owned view options. + options: serde_json::Value, + }, + /// Continuation request carrying a checked prior-page cursor. + Continuation { + /// Opaque restart-ephemeral continuation. + cursor: ReadContinuationCursor, + }, +} + +/// Sanitized continuation state for one typed read. +#[derive(Clone, Debug, Eq, PartialEq)] +pub enum ReadContinuation { + /// The returned body is complete. + Complete, + /// More complete semantic units remain. + More { + /// Opaque restart-ephemeral continuation. + cursor: ReadContinuationCursor, + }, +} + +/// Bounded typed-read result currently representable without durable media references. +#[derive(Clone, Debug, Eq, PartialEq)] +pub enum FileReadResult { + /// Admitted UTF-8 body. + Text { + /// Complete bounded body. + body: String, + /// Sanitized completeness or continuation evidence. + continuation: ReadContinuation, + }, + /// Admitted structured value. + Structured { + /// Parsed bounded JSON body. + body: serde_json::Value, + /// Sanitized completeness or continuation evidence. + continuation: ReadContinuation, + }, +} + +/// Raw untrusted probe response crossing the processor boundary. +#[derive(Clone, Debug, Eq, PartialEq, serde::Deserialize, serde::Serialize)] +#[serde(tag = "status", rename_all = "snake_case", deny_unknown_fields)] +pub enum ProcessorProbeOutput { + /// Reader found no evidence. + NoMatch, + /// Reader found one candidate. + Candidate { + /// Untrusted claimed canonical media type spelling. + media_type: String, + /// Claimed evidence strength. + strength: ProbeStrength, + }, + /// Reader recognized a malformed format. + RecognizedMalformed { + /// Untrusted claimed media type spelling. + media_type: String, + /// Untrusted reason spelling, checked against the declaration. + reason_code: String, + }, +} + +/// Raw untrusted validation response crossing the processor boundary. +#[derive(Clone, Debug, Eq, PartialEq, serde::Deserialize, serde::Serialize)] +#[serde(tag = "status", rename_all = "snake_case", deny_unknown_fields)] +pub enum ProcessorValidationOutput { + /// Validation succeeded. + Validated { + /// Untrusted claimed type, checked against the selected candidate. + media_type: String, + /// Untrusted evidence claim, checked against the selected path. + evidence: ValidationEvidence, + /// Untrusted bounded JSON object spelling. + metadata_json: String, + }, + /// Selected candidate was malformed. + Malformed { + /// Untrusted claimed type. + media_type: String, + /// Untrusted reason spelling. + reason_code: String, + }, + /// Selected candidate is encrypted or locked. + EncryptedOrLocked { + /// Untrusted claimed type. + media_type: String, + }, + /// Candidate did not survive structural validation. + NoMatch, +} + +/// Raw untrusted read response crossing the processor boundary. +#[derive(Clone, Debug, Eq, PartialEq, serde::Deserialize, serde::Serialize)] +#[serde(tag = "status", rename_all = "snake_case", deny_unknown_fields)] +pub enum ProcessorReadOutput { + /// Text body and continuation facts. + Text { + /// Untrusted text body. + body: String, + /// Whether complete semantic units remain. + truncated: bool, + /// Untrusted opaque cursor. + cursor: Option, + }, + /// Compact JSON spelling and continuation facts. + Structured { + /// Untrusted JSON text. + body_json: String, + /// Whether complete semantic units remain. + truncated: bool, + /// Untrusted opaque cursor. + cursor: Option, + }, + /// Provider rejected model-supplied options. + InvalidViewArguments, + /// Provider declined a declared view. + UnsupportedView, + /// Source exceeds a declared intrinsic whole-decode size limit. + /// + /// Version one declares only cumulative source-work budgets, so the + /// registry rejects this processor outcome until such a limit exists. + SourceTooLarge { + /// Untrusted claimed intrinsic maximum. + maximum_bytes: u64, + }, + /// Decode expansion crossed a registered named limit. + ExpansionLimitExceeded { + /// Untrusted reason spelling, checked against the reader declaration. + limit_kind: String, + }, + /// One complete semantic unit could not fit. + OutputUnitTooLarge, +} + +/// Content-silent process execution failure. +#[derive(Clone, Copy, Debug, Eq, PartialEq, serde::Deserialize, serde::Serialize)] +#[serde(rename_all = "snake_case")] +pub enum ProcessorFailure { + /// Isolation or worker startup was unavailable. + Unavailable, + /// Worker exited unsuccessfully or returned incomplete output. + Failed, + /// Worker exceeded wall time. + TimedOut, + /// Authoritative cancellation terminated work. + Cancelled, + /// Framing or output validation failed. + Protocol, +} + +impl fmt::Display for ProcessorFailure { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str(match self { + Self::Unavailable => "file processor is unavailable", + Self::Failed => "file processor failed", + Self::TimedOut => "file processor timed out", + Self::Cancelled => "file processing was cancelled", + Self::Protocol => "file processor returned invalid output", + }) + } +} + +impl Error for ProcessorFailure {} + +/// Authenticated failure returned by the daemon-side processor broker. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum ProcessorBoundaryFailure { + /// Process execution failed without a verified-source classification. + Processor(ProcessorFailure), + /// The verified source failed while serving processor reads. + Source(SourceReadError), +} + +impl fmt::Display for ProcessorBoundaryFailure { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Processor(failure) => failure.fmt(formatter), + Self::Source(failure) => failure.fmt(formatter), + } + } +} + +impl Error for ProcessorBoundaryFailure {} + +impl From for ProcessorBoundaryFailure { + fn from(value: ProcessorFailure) -> Self { + Self::Processor(value) + } +} + +impl From for ProcessorBoundaryFailure { + fn from(value: SourceReadError) -> Self { + Self::Source(value) + } +} + +/// Boxed future returned by a daemon-side isolated processor client. +pub type FileMediaProcessorFuture<'a, Output> = + Pin> + Send + 'a>>; + +/// Daemon-side process boundary used by detection and reads. +pub trait FileMediaProcessor: Send + Sync { + /// Runs one reader probe under its registered source envelope. + fn probe<'a>( + &'a self, + reader: &'a ReaderIdentity, + source: &'a dyn VerifiedBlobSource, + cancellation: &'a dyn CancellationSignal, + ) -> FileMediaProcessorFuture<'a, ProcessorProbeOutput>; + + /// Runs full validation for the sole registry-selected candidate. + fn validate<'a>( + &'a self, + reader: &'a ReaderIdentity, + request: crate::FileMediaProviderValidationRequest, + source: &'a dyn VerifiedBlobSource, + cancellation: &'a dyn CancellationSignal, + ) -> FileMediaProcessorFuture<'a, ProcessorValidationOutput>; + + /// Runs one declared view after validation. + fn read<'a>( + &'a self, + reader: &'a ReaderIdentity, + request: crate::FileMediaProviderReadRequest, + source: &'a dyn VerifiedBlobSource, + cancellation: &'a dyn CancellationSignal, + ) -> FileMediaProcessorFuture<'a, ProcessorReadOutput>; +} + +/// Closed application-facing file/media failure algebra. +#[derive(Clone, Debug, Eq, PartialEq)] +pub enum FileMediaFailure { + /// Digest is outside the rendered-frontier allow-set. + BlobNotVisible, + /// Blob catalog identity is absent. + BlobMissing, + /// Every replica contradicted exact bytes. + BlobCorrupt, + /// Blob access is temporarily unavailable. + BlobUnavailable, + /// No registered reader safely recognized the bytes. + UnknownType, + /// Incompatible strong candidates made selection unsafe. + AmbiguousType, + /// Caller declaration disagreed with byte evidence. + DeclaredTypeMismatch { + /// Canonical caller declaration. + declared: CanonicalMediaType, + /// Canonical byte-detected type. + detected: CanonicalMediaType, + }, + /// Recognized bytes were malformed. + Malformed { + /// Recognized canonical type. + media_type: CanonicalMediaType, + /// Registered sanitized reason. + reason_code: ReasonCode, + }, + /// Recognized encrypted content is terminal in version one. + EncryptedOrLocked { + /// Recognized canonical type. + media_type: CanonicalMediaType, + }, + /// Selected view does not exist. + UnsupportedView, + /// View options failed provider validation. + InvalidViewArguments, + /// Source exceeds a reader's bounded whole-decode envelope. + SourceTooLarge { + /// Exact declared maximum. + maximum_bytes: u64, + }, + /// Decode expansion exceeded a named hard limit. + ExpansionLimitExceeded { + /// Registered content-silent limit name. + limit_kind: ReasonCode, + }, + /// One semantic output unit could not fit without truncation. + OutputUnitTooLarge, + /// Processor isolation or worker startup is unavailable. + ProcessorUnavailable, + /// Processor failed without an authenticated typed failure. + ProcessorFailed, + /// Processor exceeded wall time. + ProcessorTimedOut, + /// Authoritative cancellation stopped work. + Cancelled, +} + +impl fmt::Display for FileMediaFailure { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str(match self { + Self::BlobNotVisible => "blob is not visible to this request", + Self::BlobMissing => "blob is missing", + Self::BlobCorrupt => "blob is corrupt", + Self::BlobUnavailable => "blob is unavailable", + Self::UnknownType => "file type is unknown", + Self::AmbiguousType => "file type is ambiguous", + Self::DeclaredTypeMismatch { .. } => "declared and detected file types disagree", + Self::Malformed { .. } => "recognized file is malformed", + Self::EncryptedOrLocked { .. } => "file is encrypted or locked", + Self::UnsupportedView => "file read view is unsupported", + Self::InvalidViewArguments => "file read view arguments are invalid", + Self::SourceTooLarge { .. } => "file source exceeds the declared view bound", + Self::ExpansionLimitExceeded { .. } => "file expansion limit was exceeded", + Self::OutputUnitTooLarge => "one output unit exceeds the result bound", + Self::ProcessorUnavailable => "file processor is unavailable", + Self::ProcessorFailed => "file processor failed", + Self::ProcessorTimedOut => "file processor timed out", + Self::Cancelled => "file processing was cancelled", + }) + } +} + +impl Error for FileMediaFailure {} + +impl From for FileMediaFailure { + fn from(value: ProcessorFailure) -> Self { + match value { + ProcessorFailure::Unavailable => Self::ProcessorUnavailable, + ProcessorFailure::Failed | ProcessorFailure::Protocol => Self::ProcessorFailed, + ProcessorFailure::TimedOut => Self::ProcessorTimedOut, + ProcessorFailure::Cancelled => Self::Cancelled, + } + } +} + +impl From for FileMediaFailure { + fn from(value: ProcessorBoundaryFailure) -> Self { + match value { + ProcessorBoundaryFailure::Processor(failure) => failure.into(), + ProcessorBoundaryFailure::Source(failure) => failure.into(), + } + } +} + +impl From for FileMediaFailure { + fn from(value: SourceReadError) -> Self { + match value { + SourceReadError::Missing => Self::BlobMissing, + SourceReadError::Corrupt => Self::BlobCorrupt, + SourceReadError::Unavailable => Self::BlobUnavailable, + SourceReadError::RangeOutOfBounds | SourceReadError::Integrity => Self::ProcessorFailed, + } + } +} diff --git a/crates/file-media-runtime/src/lib.rs b/crates/file-media-runtime/src/lib.rs new file mode 100644 index 0000000000..9a3899e96f --- /dev/null +++ b/crates/file-media-runtime/src/lib.rs @@ -0,0 +1,57 @@ +//! Provider-neutral file and media interpretation contracts. +//! +//! This crate owns checked declarations, detection, validation, bounded reads, +//! and the untrusted processor boundary. It deliberately has no dependency on +//! domain, application, persistence, daemon, parser, or media crates. + +mod declaration; +mod detection; +mod limits; +mod registry; +mod value; + +pub use declaration::{ + FileMediaProvider, FileMediaProviderDeclaration, FileMediaProviderFailure, + FileMediaProviderFuture, FileMediaProviderReadRequest, FileMediaProviderValidationRequest, + ProbeDeclaration, ProbeDeclarationInput, ProbeStrength, ReadAccessPattern, ReadOutputKind, + ReadViewBounds, ReadViewDeclaration, ReaderDeclaration, ReaderDeclarationInput, + RegistryDeclarationError, StreamingTextFallback, ValidationDeclaration, +}; +pub use detection::{ + CancellationSignal, FileInspection, FileInspectionStatus, FileMediaFailure, FileMediaProcessor, + FileMediaProcessorFuture, FileReadInput, FileReadRequest, FileReadResult, InspectionRequest, + NeverCancelled, ProcessorBoundaryFailure, ProcessorFailure, ProcessorProbeOutput, + ProcessorReadOutput, ProcessorValidationOutput, ReadContinuation, SourceReadError, + SourceReadFuture, ValidatedFile, ValidationEvidence, VerifiedBlobSource, +}; +pub use limits::{ + FileMediaCeilings, FileMediaProcessCeilings, FileMediaProcessLimitOverrides, + MAX_AGGREGATE_MEDIA_BYTES_PER_CALL, MAX_AUDIO_CHANNELS, MAX_AUDIO_CLIP_SECONDS, + MAX_AUDIO_SAMPLE_RATE_HZ, MAX_DECODED_IMAGE_PIXELS, MAX_IMAGE_AXIS, + MAX_MEDIA_REFERENCES_PER_CALL, MAX_OBSERVED_CONTAINER_ENTRIES, MAX_PRESENTED_AUDIO_BYTES, + MAX_PRESENTED_FILE_BYTES, MAX_PRESENTED_IMAGE_BYTES, MAX_PROBE_CUMULATIVE_BYTES, + MAX_PROBE_PREFIX_BYTES, MAX_PROBE_RANGES, MAX_PROBE_SUFFIX_BYTES, MAX_PROCESSOR_FRAME_BYTES, + MAX_READ_OPTIONS_BYTES, MAX_READ_RANGES, MAX_READ_SOURCE_BYTES, MAX_STRUCTURED_DEPTH, + MAX_STRUCTURED_NODES, MAX_TEXT_BODY_BYTES, MAX_TEXT_OR_JSON_BYTES, MAX_VALIDATION_RANGES, + MAX_VALIDATION_SOURCE_BYTES, MAX_WORKER_CPU_SECONDS, MAX_WORKER_DESCENDANTS, + MAX_WORKER_FILE_DESCRIPTORS, MAX_WORKER_MEMORY_BYTES, MAX_WORKER_STDERR_BYTES, + MAX_WORKER_TASKS, MAX_WORKER_WALL_SECONDS, MIN_WORKER_FILE_DESCRIPTORS, +}; +pub use registry::{ + FileMediaRegistry, FileMediaRegistryConstructionError, MAX_READERS_PER_PROVIDER, + MAX_REGISTRY_READERS, ProcessorIsolation, provider_declaration_inventory_fits, + read_options_fit, +}; +pub use value::{ + AttachmentKind, BoundedMetadata, CanonicalJsonObjectSchema, CanonicalMediaType, + DeclaredMediaType, DisplayFilename, FileDigest, FileReaderName, FileReaderProviderName, + FileReaderRevision, FileUse, JsonParseLimits, MediaTypeParseError, ReadContinuationCursor, + ReadViewName, ReaderIdentity, ReasonCode, RegistryValueError, VisiblePartSelector, + parse_json_without_duplicate_members, parse_json_without_duplicate_members_bounded, +}; + +/// Stable model-facing inspection tool name. +pub const FILE_INSPECT_NAME: &str = "file_inspect"; + +/// Stable model-facing typed-read tool name. +pub const FILE_READ_NAME: &str = "file_read"; diff --git a/crates/file-media-runtime/src/limits.rs b/crates/file-media-runtime/src/limits.rs new file mode 100644 index 0000000000..3a63f0081f --- /dev/null +++ b/crates/file-media-runtime/src/limits.rs @@ -0,0 +1,418 @@ +//! Compiled upper bounds shared by declarations and processor supervision. + +/// Hard safety ceiling; bounds prefix reads to protect broker memory and I/O. +pub const MAX_PROBE_PREFIX_BYTES: u64 = 65_536; +/// Hard safety ceiling; bounds suffix reads to protect broker memory and I/O. +pub const MAX_PROBE_SUFFIX_BYTES: u64 = 65_536; +/// Hard safety ceiling; bounds range fan-out to protect broker I/O scheduling. +pub const MAX_PROBE_RANGES: u32 = 16; +/// Hard safety ceiling; bounds aggregate probe reads to protect broker resources. +pub const MAX_PROBE_CUMULATIVE_BYTES: u64 = 262_144; +/// Hard safety ceiling; bounds one validation's aggregate source I/O. +pub const MAX_VALIDATION_SOURCE_BYTES: u64 = 1_073_741_824; +/// Hard safety ceiling; bounds one validation's exact-range fan-out. +pub const MAX_VALIDATION_RANGES: u32 = 4_096; +/// Hard safety ceiling; bounds one view's aggregate source I/O. +pub const MAX_READ_SOURCE_BYTES: u64 = 1_073_741_824; +/// Hard safety ceiling; bounds one random-access view's range fan-out. +pub const MAX_READ_RANGES: u32 = 4_096; +/// Hard safety ceiling; bounds one processor frame to protect daemon memory. +pub const MAX_PROCESSOR_FRAME_BYTES: usize = 1_048_576; +/// Hard safety ceiling; bounds serialized read options before processor framing. +pub const MAX_READ_OPTIONS_BYTES: usize = 65_536; +/// Hard safety ceiling; bounds structured JSON so nested wire escaping fits one frame. +pub const MAX_TEXT_OR_JSON_BYTES: usize = 500_000; +/// Hard safety ceiling; bounds text so worst-case JSON escaping fits one tool result. +pub const MAX_TEXT_BODY_BYTES: usize = 174_000; +/// Hard safety ceiling; bounds JSON nesting to protect recursive traversal. +pub const MAX_STRUCTURED_DEPTH: u32 = 64; +/// Hard safety ceiling; bounds JSON nodes to protect traversal work and memory. +pub const MAX_STRUCTURED_NODES: u64 = 100_000; +/// Hard safety ceiling; bounds one JSON container to protect concentrated fan-out. +pub const MAX_OBSERVED_CONTAINER_ENTRIES: u64 = 10_000; +/// Hard safety ceiling; bounds one image axis to protect decoder allocation. +pub const MAX_IMAGE_AXIS: u32 = 8_192; +/// Hard safety ceiling; bounds decoded image area to protect decoder memory. +pub const MAX_DECODED_IMAGE_PIXELS: u64 = 16_777_216; +/// Hard safety ceiling; bounds presented image payloads to protect result memory. +pub const MAX_PRESENTED_IMAGE_BYTES: u64 = 8_388_608; +/// Hard safety ceiling; bounds channel fan-out to protect decoder memory and work. +pub const MAX_AUDIO_CHANNELS: u16 = 8; +/// Hard safety ceiling; bounds samples per second to protect decoder work. +pub const MAX_AUDIO_SAMPLE_RATE_HZ: u32 = 192_000; +/// Hard safety ceiling; bounds clip duration to protect decoder work and memory. +pub const MAX_AUDIO_CLIP_SECONDS: u32 = 60; +/// Hard safety ceiling; bounds presented audio payloads to protect result memory. +pub const MAX_PRESENTED_AUDIO_BYTES: u64 = 8_388_608; +/// Hard safety ceiling; bounds presented file payloads to protect result memory. +pub const MAX_PRESENTED_FILE_BYTES: u64 = 8_388_608; +/// Maximum durable media references emitted by one model call. +pub const MAX_MEDIA_REFERENCES_PER_CALL: u16 = 16; +/// Maximum aggregate referenced media bytes emitted by one model call. +pub const MAX_AGGREGATE_MEDIA_BYTES_PER_CALL: u64 = 33_554_432; +/// Maximum isolated worker address-space bytes. +pub const MAX_WORKER_MEMORY_BYTES: u64 = 512 * 1024 * 1024; +/// Maximum isolated worker CPU seconds. +pub const MAX_WORKER_CPU_SECONDS: u64 = 60; +/// Maximum isolated worker wall-clock seconds. +pub const MAX_WORKER_WALL_SECONDS: u64 = 120; +/// Maximum isolated worker descendants. Threads remain permitted. +pub const MAX_WORKER_DESCENDANTS: u32 = 0; +/// Maximum kernel tasks available to one isolated worker process tree. +pub const MAX_WORKER_TASKS: u64 = 64; +/// Maximum file descriptors available to an isolated worker. +pub const MAX_WORKER_FILE_DESCRIPTORS: u64 = 32; +/// Minimum descriptor ceiling that can launch bubblewrap and the dynamic worker. +pub const MIN_WORKER_FILE_DESCRIPTORS: u64 = 16; +/// Maximum retained diagnostic bytes from an isolated worker. +pub const MAX_WORKER_STDERR_BYTES: usize = 16_384; + +/// Process-wide ceilings against which every declaration is checked. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct FileMediaCeilings { + /// Maximum prefix bytes. + pub probe_prefix_bytes: u64, + /// Maximum suffix bytes. + pub probe_suffix_bytes: u64, + /// Maximum arbitrary probe ranges. + pub probe_ranges: u32, + /// Maximum cumulative probe bytes. + pub probe_cumulative_bytes: u64, + /// Maximum cumulative source bytes for one validation. + pub validation_source_bytes: u64, + /// Maximum exact ranges for one validation. + pub validation_ranges: u32, + /// Maximum cumulative source bytes for one read view. + pub read_source_bytes: u64, + /// Maximum exact ranges for one random-access read view. + pub read_ranges: u32, + /// Maximum result body bytes. + pub text_or_json_bytes: usize, + /// Maximum structured nesting. + pub structured_depth: u32, + /// Maximum structured nodes. + pub structured_nodes: u64, + /// Maximum observed container entries. + pub observed_container_entries: u64, + /// Maximum image axis. + pub image_axis: u32, + /// Maximum decoded image pixels. + pub decoded_image_pixels: u64, + /// Maximum presented image bytes. + pub presented_image_bytes: u64, + /// Maximum audio channels. + pub audio_channels: u16, + /// Maximum audio sample rate. + pub audio_sample_rate_hz: u32, + /// Maximum audio duration. + pub audio_clip_seconds: u32, + /// Maximum presented audio bytes. + pub presented_audio_bytes: u64, + /// Maximum presented general-file bytes. + pub presented_file_bytes: u64, + /// Maximum durable media references per model call. + pub media_references_per_call: u16, + /// Maximum aggregate referenced media bytes per model call. + pub aggregate_media_bytes_per_call: u64, +} + +impl FileMediaCeilings { + /// Returns the hard-coded version-one ceiling set. + pub const fn version_one() -> Self { + Self { + probe_prefix_bytes: MAX_PROBE_PREFIX_BYTES, + probe_suffix_bytes: MAX_PROBE_SUFFIX_BYTES, + probe_ranges: MAX_PROBE_RANGES, + probe_cumulative_bytes: MAX_PROBE_CUMULATIVE_BYTES, + validation_source_bytes: MAX_VALIDATION_SOURCE_BYTES, + validation_ranges: MAX_VALIDATION_RANGES, + read_source_bytes: MAX_READ_SOURCE_BYTES, + read_ranges: MAX_READ_RANGES, + text_or_json_bytes: MAX_TEXT_OR_JSON_BYTES, + structured_depth: MAX_STRUCTURED_DEPTH, + structured_nodes: MAX_STRUCTURED_NODES, + observed_container_entries: MAX_OBSERVED_CONTAINER_ENTRIES, + image_axis: MAX_IMAGE_AXIS, + decoded_image_pixels: MAX_DECODED_IMAGE_PIXELS, + presented_image_bytes: MAX_PRESENTED_IMAGE_BYTES, + audio_channels: MAX_AUDIO_CHANNELS, + audio_sample_rate_hz: MAX_AUDIO_SAMPLE_RATE_HZ, + audio_clip_seconds: MAX_AUDIO_CLIP_SECONDS, + presented_audio_bytes: MAX_PRESENTED_AUDIO_BYTES, + presented_file_bytes: MAX_PRESENTED_FILE_BYTES, + media_references_per_call: MAX_MEDIA_REFERENCES_PER_CALL, + aggregate_media_bytes_per_call: MAX_AGGREGATE_MEDIA_BYTES_PER_CALL, + } + } + + /// Accepts a deployment override only when every value lowers a compiled ceiling. + pub const fn admits(self, candidate: Self) -> bool { + candidate.probe_prefix_bytes > 0 + && candidate.probe_prefix_bytes <= self.probe_prefix_bytes + && candidate.probe_suffix_bytes > 0 + && candidate.probe_suffix_bytes <= self.probe_suffix_bytes + && candidate.probe_ranges > 0 + && candidate.probe_ranges <= self.probe_ranges + && candidate.probe_cumulative_bytes > 0 + && candidate.probe_cumulative_bytes <= self.probe_cumulative_bytes + && candidate.validation_source_bytes > 0 + && candidate.validation_source_bytes <= self.validation_source_bytes + && candidate.validation_ranges > 0 + && candidate.validation_ranges <= self.validation_ranges + && candidate.read_source_bytes > 0 + && candidate.read_source_bytes <= self.read_source_bytes + && candidate.read_ranges > 0 + && candidate.read_ranges <= self.read_ranges + && candidate.text_or_json_bytes > 0 + && candidate.text_or_json_bytes <= self.text_or_json_bytes + && candidate.structured_depth > 0 + && candidate.structured_depth <= self.structured_depth + && candidate.structured_nodes > 0 + && candidate.structured_nodes <= self.structured_nodes + && candidate.observed_container_entries > 0 + && candidate.observed_container_entries <= self.observed_container_entries + && candidate.image_axis > 0 + && candidate.image_axis <= self.image_axis + && candidate.decoded_image_pixels > 0 + && candidate.decoded_image_pixels <= self.decoded_image_pixels + && candidate.presented_image_bytes > 0 + && candidate.presented_image_bytes <= self.presented_image_bytes + && candidate.audio_channels > 0 + && candidate.audio_channels <= self.audio_channels + && candidate.audio_sample_rate_hz > 0 + && candidate.audio_sample_rate_hz <= self.audio_sample_rate_hz + && candidate.audio_clip_seconds > 0 + && candidate.audio_clip_seconds <= self.audio_clip_seconds + && candidate.presented_audio_bytes > 0 + && candidate.presented_audio_bytes <= self.presented_audio_bytes + && candidate.presented_file_bytes > 0 + && candidate.presented_file_bytes <= self.presented_file_bytes + && candidate.media_references_per_call > 0 + && candidate.media_references_per_call <= self.media_references_per_call + && candidate.aggregate_media_bytes_per_call > 0 + && candidate.aggregate_media_bytes_per_call <= self.aggregate_media_bytes_per_call + } +} + +/// Labeled deployment overrides for lowerable worker resource limits. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct FileMediaProcessLimitOverrides { + /// Combined worker-memory budget split between address space and writable tmpfs. + pub memory_bytes: u64, + /// CPU-second limit applied before worker startup. + pub cpu_seconds: u64, + /// Daemon wall-clock deadline in seconds. + pub wall_seconds: u64, + /// File-descriptor limit applied before worker startup. + pub file_descriptors: u64, + /// Retained, never-model-visible diagnostic byte limit. + pub stderr_bytes: usize, +} + +/// Daemon-supervised process limits with a fixed protocol frame and lowerable resources. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct FileMediaProcessCeilings { + frame_bytes: usize, + memory_bytes: u64, + cpu_seconds: u64, + wall_seconds: u64, + file_descriptors: u64, + stderr_bytes: usize, +} + +impl FileMediaProcessCeilings { + /// Returns the compiled version-one process ceiling set. + pub const fn version_one() -> Self { + Self { + frame_bytes: MAX_PROCESSOR_FRAME_BYTES, + memory_bytes: MAX_WORKER_MEMORY_BYTES, + cpu_seconds: MAX_WORKER_CPU_SECONDS, + wall_seconds: MAX_WORKER_WALL_SECONDS, + file_descriptors: MAX_WORKER_FILE_DESCRIPTORS, + stderr_bytes: MAX_WORKER_STDERR_BYTES, + } + } + + /// Constructs an effective set with the fixed protocol frame and lowerable resources. + pub const fn try_lower(overrides: FileMediaProcessLimitOverrides) -> Option { + let candidate = Self { + frame_bytes: MAX_PROCESSOR_FRAME_BYTES, + memory_bytes: overrides.memory_bytes, + cpu_seconds: overrides.cpu_seconds, + wall_seconds: overrides.wall_seconds, + file_descriptors: overrides.file_descriptors, + stderr_bytes: overrides.stderr_bytes, + }; + if Self::version_one().admits(candidate) { + Some(candidate) + } else { + None + } + } + + /// Returns whether the protocol frame remains fixed and every resource value is positive + /// and no greater. + pub const fn admits(self, candidate: Self) -> bool { + candidate.frame_bytes == self.frame_bytes + && candidate.memory_bytes > 0 + && candidate.memory_bytes <= self.memory_bytes + && candidate.cpu_seconds > 0 + && candidate.cpu_seconds <= self.cpu_seconds + && candidate.wall_seconds > 0 + && candidate.wall_seconds <= self.wall_seconds + && candidate.file_descriptors >= MIN_WORKER_FILE_DESCRIPTORS + && candidate.file_descriptors <= self.file_descriptors + && candidate.stderr_bytes > 0 + && candidate.stderr_bytes <= self.stderr_bytes + } + + /// Returns the fixed maximum length-delimited protocol frame bytes. + pub const fn frame_bytes(self) -> usize { + self.frame_bytes + } + + /// Returns the combined worker-memory budget split between address space and writable tmpfs. + pub const fn memory_bytes(self) -> u64 { + self.memory_bytes + } + + /// Returns the CPU-second limit applied before worker startup. + pub const fn cpu_seconds(self) -> u64 { + self.cpu_seconds + } + + /// Returns the daemon wall-clock deadline. + pub const fn wall_seconds(self) -> u64 { + self.wall_seconds + } + + /// Returns the descriptor limit applied before worker startup. + pub const fn file_descriptors(self) -> u64 { + self.file_descriptors + } + + /// Returns the retained, never-model-visible diagnostic byte limit. + pub const fn stderr_bytes(self) -> usize { + self.stderr_bytes + } +} + +impl Default for FileMediaProcessCeilings { + fn default() -> Self { + Self::version_one() + } +} + +impl Default for FileMediaCeilings { + fn default() -> Self { + Self::version_one() + } +} + +#[cfg(test)] +mod tests { + use super::{ + FileMediaCeilings, FileMediaProcessCeilings, FileMediaProcessLimitOverrides, + MAX_AGGREGATE_MEDIA_BYTES_PER_CALL, MAX_MEDIA_REFERENCES_PER_CALL, + MAX_PROCESSOR_FRAME_BYTES, MAX_WORKER_CPU_SECONDS, MAX_WORKER_FILE_DESCRIPTORS, + MAX_WORKER_MEMORY_BYTES, MAX_WORKER_STDERR_BYTES, MAX_WORKER_WALL_SECONDS, + MIN_WORKER_FILE_DESCRIPTORS, + }; + + /// INV-088: deployment configuration can lower but never raise a compiled ceiling. + #[test] + fn inv088_file_media_ceiling_overrides_are_lowerable_only() { + let media = FileMediaCeilings::version_one(); + assert_eq!( + media.media_references_per_call, + MAX_MEDIA_REFERENCES_PER_CALL + ); + assert_eq!( + media.aggregate_media_bytes_per_call, + MAX_AGGREGATE_MEDIA_BYTES_PER_CALL + ); + assert!(!FileMediaCeilings::version_one().admits(FileMediaCeilings { + media_references_per_call: MAX_MEDIA_REFERENCES_PER_CALL + 1, + ..media + })); + assert!(!FileMediaCeilings::version_one().admits(FileMediaCeilings { + aggregate_media_bytes_per_call: MAX_AGGREGATE_MEDIA_BYTES_PER_CALL + 1, + ..media + })); + assert_eq!( + FileMediaProcessCeilings::try_lower(FileMediaProcessLimitOverrides { + memory_bytes: MAX_WORKER_MEMORY_BYTES + 1, + cpu_seconds: MAX_WORKER_CPU_SECONDS, + wall_seconds: MAX_WORKER_WALL_SECONDS, + file_descriptors: MAX_WORKER_FILE_DESCRIPTORS, + stderr_bytes: MAX_WORKER_STDERR_BYTES, + }), + None + ); + assert_eq!( + FileMediaProcessCeilings::try_lower(FileMediaProcessLimitOverrides { + memory_bytes: MAX_WORKER_MEMORY_BYTES, + cpu_seconds: MAX_WORKER_CPU_SECONDS + 1, + wall_seconds: MAX_WORKER_WALL_SECONDS, + file_descriptors: MAX_WORKER_FILE_DESCRIPTORS, + stderr_bytes: MAX_WORKER_STDERR_BYTES, + }), + None + ); + assert_eq!( + FileMediaProcessCeilings::try_lower(FileMediaProcessLimitOverrides { + memory_bytes: MAX_WORKER_MEMORY_BYTES, + cpu_seconds: MAX_WORKER_CPU_SECONDS, + wall_seconds: MAX_WORKER_WALL_SECONDS + 1, + file_descriptors: MAX_WORKER_FILE_DESCRIPTORS, + stderr_bytes: MAX_WORKER_STDERR_BYTES, + }), + None + ); + assert_eq!( + FileMediaProcessCeilings::try_lower(FileMediaProcessLimitOverrides { + memory_bytes: MAX_WORKER_MEMORY_BYTES, + cpu_seconds: MAX_WORKER_CPU_SECONDS, + wall_seconds: MAX_WORKER_WALL_SECONDS, + file_descriptors: MAX_WORKER_FILE_DESCRIPTORS + 1, + stderr_bytes: MAX_WORKER_STDERR_BYTES, + }), + None + ); + assert_eq!( + FileMediaProcessCeilings::try_lower(FileMediaProcessLimitOverrides { + memory_bytes: MAX_WORKER_MEMORY_BYTES, + cpu_seconds: MAX_WORKER_CPU_SECONDS, + wall_seconds: MAX_WORKER_WALL_SECONDS, + file_descriptors: MAX_WORKER_FILE_DESCRIPTORS, + stderr_bytes: MAX_WORKER_STDERR_BYTES + 1, + }), + None + ); + assert_eq!( + FileMediaProcessCeilings::try_lower(FileMediaProcessLimitOverrides { + memory_bytes: MAX_WORKER_MEMORY_BYTES, + cpu_seconds: MAX_WORKER_CPU_SECONDS, + wall_seconds: MAX_WORKER_WALL_SECONDS, + file_descriptors: MAX_WORKER_FILE_DESCRIPTORS, + stderr_bytes: MAX_WORKER_STDERR_BYTES, + }) + .map(FileMediaProcessCeilings::frame_bytes), + Some(MAX_PROCESSOR_FRAME_BYTES) + ); + } + + #[test] + fn process_ceiling_rejects_an_unlaunchable_descriptor_limit() { + assert_eq!( + FileMediaProcessCeilings::try_lower(FileMediaProcessLimitOverrides { + memory_bytes: MAX_WORKER_MEMORY_BYTES, + cpu_seconds: MAX_WORKER_CPU_SECONDS, + wall_seconds: MAX_WORKER_WALL_SECONDS, + file_descriptors: MIN_WORKER_FILE_DESCRIPTORS - 1, + stderr_bytes: MAX_WORKER_STDERR_BYTES, + }), + None + ); + } +} diff --git a/crates/file-media-runtime/src/registry.rs b/crates/file-media-runtime/src/registry.rs new file mode 100644 index 0000000000..2de7e3818c --- /dev/null +++ b/crates/file-media-runtime/src/registry.rs @@ -0,0 +1,1425 @@ +use std::{collections::BTreeMap, error::Error, fmt, str::FromStr}; + +use crate::{ + BoundedMetadata, CanonicalMediaType, FileInspection, FileMediaCeilings, FileMediaFailure, + FileMediaProcessor, FileMediaProviderDeclaration, FileMediaProviderReadRequest, + FileMediaProviderValidationRequest, FileReadRequest, FileReadResult, InspectionRequest, + MAX_READ_OPTIONS_BYTES, MAX_WORKER_WALL_SECONDS, ProbeStrength, ProcessorProbeOutput, + ProcessorReadOutput, ProcessorValidationOutput, ReadAccessPattern, ReadContinuation, + ReadContinuationCursor, ReadViewBounds, ReaderDeclaration, ReaderIdentity, ReasonCode, + StreamingTextFallback, ValidatedFile, ValidationEvidence, VerifiedBlobSource, +}; + +// numeric-bound: ceiling - bounds process-lifetime provider inventory memory +const MAX_REGISTRY_PROVIDERS: usize = 256; +// numeric-bound: ceiling - bounds per-provider reader inventory memory and startup work +pub const MAX_READERS_PER_PROVIDER: usize = 256; +// numeric-bound: ceiling - bounds aggregate process-lifetime reader inventory memory +pub const MAX_REGISTRY_READERS: usize = 256; +// numeric-bound: ceiling - bounds per-reader media-claim memory and conflict checks +const MAX_MEDIA_TYPES_PER_READER: usize = 256; +// numeric-bound: ceiling - bounds aggregate process-lifetime media-claim memory +const MAX_REGISTRY_MEDIA_TYPES: usize = 4_096; +// numeric-bound: ceiling - bounds per-reader model-visible view inventory memory +const MAX_VIEWS_PER_READER: usize = 256; +// numeric-bound: ceiling - reserves tool-result space for fixed inspection facts and metadata +const MAX_INSPECTION_VIEW_INVENTORY_BYTES: usize = 512 * 1_024; +// numeric-bound: ceiling - reserves effective result space for fixed inspection facts and metadata +const INSPECTION_NON_VIEW_RESERVE_BYTES: usize = 64 * 1_024; +// numeric-bound: ceiling - bounds aggregate process-lifetime view inventory memory +const MAX_REGISTRY_VIEWS: usize = 4_096; +// numeric-bound: ceiling - bounds aggregate retained view-schema bytes +const MAX_REGISTRY_SCHEMA_BYTES: usize = 16 * 1_024 * 1_024; +// numeric-bound: ceiling - bounds per-reader sanitized reason inventory memory +const MAX_REASON_CODES_PER_READER: usize = 256; +// numeric-bound: ceiling - bounds aggregate process-lifetime reason inventory memory +const MAX_REGISTRY_REASON_CODES: usize = 4_096; +// numeric-bound: ceiling - bounds one inspection's aggregate probe source I/O +const MAX_INSPECTION_PROBE_BYTES: u64 = 16 * 1_024 * 1_024; +// numeric-bound: ceiling - bounds one inspection's aggregate probe request fan-out +const MAX_INSPECTION_PROBE_READS: u32 = 1_024; +// numeric-bound: ceiling - bounds collision-validation worker fan-out and source I/O +const MAX_COLLISION_VALIDATION_CANDIDATES: usize = 2; +// numeric-bound: ceiling - the tool contract permits this many input containers +const MAX_READ_INPUT_CONTAINERS: u32 = 256; +// numeric-bound: ceiling - every JSON node emits at least one serialized byte +const MAX_READ_OPTIONS_NODES: usize = MAX_READ_OPTIONS_BYTES; +// numeric-bound: ceiling - reserves processor-frame space for structured-body JSON escaping +const MAX_STRUCTURED_BODY_BYTES: usize = 500 * 1_024; +/// Immutable process-lifetime registry snapshot. +#[derive(Clone, Debug)] +pub struct FileMediaRegistry { + providers: Vec, + readers: BTreeMap, + media_readers: BTreeMap, + streaming_text_reader: Option, + ceilings: FileMediaCeilings, +} + +/// Whether the daemon can launch the required processor isolation boundary. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum ProcessorIsolation { + /// The accepted isolation boundary is available. + Available, + /// No accepted isolation boundary is available. + Unavailable, +} + +impl FileMediaRegistry { + /// Builds one deterministic registry and rejects every static conflict. + pub fn try_new( + mut providers: Vec, + ceilings: FileMediaCeilings, + isolation: ProcessorIsolation, + ) -> Result { + if !FileMediaCeilings::version_one().admits(ceilings) { + return Err(FileMediaRegistryConstructionError::Ceilings); + } + if providers.len() > MAX_REGISTRY_PROVIDERS { + return Err(FileMediaRegistryConstructionError::Inventory); + } + if !providers.is_empty() && isolation == ProcessorIsolation::Unavailable { + return Err(FileMediaRegistryConstructionError::IsolationUnavailable); + } + if providers + .iter() + .any(|provider| provider.readers().len() > MAX_READERS_PER_PROVIDER) + { + return Err(FileMediaRegistryConstructionError::Inventory); + } + validate_aggregate_inventory(&providers)?; + validate_aggregate_probe_budget(&providers)?; + providers.sort_by(|left, right| left.provider().cmp(right.provider())); + for provider in &mut providers { + provider.sort_readers(); + } + if providers + .windows(2) + .any(|pair| pair[0].provider() == pair[1].provider()) + { + return Err(FileMediaRegistryConstructionError::DuplicateProvider); + } + + let mut readers = BTreeMap::new(); + let mut media_readers = BTreeMap::new(); + let mut streaming_text_reader = None; + for provider in &providers { + for reader in provider.readers() { + validate_reader(reader, ceilings)?; + let identity = reader.identity().clone(); + if readers.insert(identity.clone(), reader.clone()).is_some() { + return Err(FileMediaRegistryConstructionError::DuplicateReader); + } + for media_type in reader.media_types() { + if media_readers + .insert(media_type.clone(), identity.clone()) + .is_some() + { + return Err(FileMediaRegistryConstructionError::DuplicateMediaTypeClaim); + } + } + if reader.streaming_text_fallback() == StreamingTextFallback::Enabled { + let text_plain = CanonicalMediaType::from_str("text/plain") + .map_err(|_| FileMediaRegistryConstructionError::TextFallback)?; + if !reader.media_types().contains(&text_plain) + || streaming_text_reader.replace(identity).is_some() + { + return Err(FileMediaRegistryConstructionError::TextFallback); + } + } + } + } + Ok(Self { + providers, + readers, + media_readers, + streaming_text_reader, + ceilings, + }) + } + + /// Constructs the valid empty registry used before adapters are compiled. + pub fn empty() -> Self { + Self { + providers: Vec::new(), + readers: BTreeMap::new(), + media_readers: BTreeMap::new(), + streaming_text_reader: None, + ceilings: FileMediaCeilings::version_one(), + } + } + + /// Borrows canonically ordered provider declarations. + pub fn providers(&self) -> &[FileMediaProviderDeclaration] { + &self.providers + } + + /// Returns the effective lowerable-only ceiling set. + pub const fn ceilings(&self) -> FileMediaCeilings { + self.ceilings + } + + /// Detects and validates exact bytes without consulting registration order. + pub async fn inspect( + &self, + processor: &dyn FileMediaProcessor, + request: InspectionRequest, + source: &dyn VerifiedBlobSource, + cancellation: &dyn crate::CancellationSignal, + ) -> Result { + if cancellation.is_cancelled() { + return Err(FileMediaFailure::Cancelled); + } + if source.digest() != request.source.digest() + || source.byte_length() != request.source.byte_length() + { + return Err(FileMediaFailure::ProcessorFailed); + } + if self.readers.is_empty() { + return Ok(FileInspection::Unknown { + source: request.source, + }); + } + + let probes = async { + let mut candidates = Vec::new(); + let mut malformed = Vec::new(); + for reader in self.readers.values() { + let raw = processor + .probe(reader.identity(), source, cancellation) + .await?; + match sanitize_probe(reader, raw)? { + SanitizedProbe::NoMatch => {} + SanitizedProbe::Candidate(candidate) => candidates.push(candidate), + SanitizedProbe::Malformed { + media_type, + reason_code, + } => { + malformed.push((media_type, reason_code)); + } + } + } + Ok::<_, FileMediaFailure>((candidates, malformed)) + }; + let probes = Box::pin(probes); + let deadline = Box::pin(futures_timer::Delay::new(std::time::Duration::from_secs( + MAX_WORKER_WALL_SECONDS, + ))); + let (candidates, mut malformed) = match futures_util::future::select(probes, deadline).await + { + futures_util::future::Either::Left((result, _)) => result?, + futures_util::future::Either::Right(((), _)) => { + return Err(FileMediaFailure::ProcessorTimedOut); + } + }; + if !malformed.is_empty() { + malformed.sort(); + malformed.dedup(); + let distinct = distinct_media_types( + malformed.iter().map(|(kind, _)| kind.clone()).chain( + candidates + .iter() + .filter(|candidate| recognized_probe_strength(candidate.strength)) + .map(|candidate| candidate.media_type.clone()), + ), + ); + if distinct.len() > 1 { + return Ok(FileInspection::Ambiguous { + source: request.source, + media_types: distinct, + }); + } + let Some((media_type, reason_code)) = malformed.into_iter().next() else { + return Err(FileMediaFailure::ProcessorFailed); + }; + return Ok(FileInspection::Malformed { + source: request.source, + media_type, + reason_code, + }); + } + + let strong = candidates + .iter() + .filter(|candidate| candidate.strength == ProbeStrength::Strong) + .cloned() + .collect::>(); + if !strong.is_empty() { + return self + .resolve_candidates( + processor, + request, + source, + cancellation, + strong, + ValidationEvidence::StrongSignature, + ) + .await; + } + + let structural = candidates + .iter() + .filter(|candidate| { + matches!( + candidate.strength, + ProbeStrength::ProvisionalStructuralCandidate + | ProbeStrength::StructuralCandidate + ) + }) + .cloned() + .collect::>(); + if !structural.is_empty() { + let inspection = self + .resolve_candidates( + processor, + request.clone(), + source, + cancellation, + structural, + ValidationEvidence::StructuralValidation, + ) + .await?; + if !matches!(inspection, FileInspection::Unknown { .. }) { + return Ok(inspection); + } + } + + if let Ok(declared) = request.source.declared_media_type().canonical_essence() + && let Some(reader) = self.media_readers.get(&declared) + { + let inspection = self + .validate_candidate( + processor, + request.clone(), + source, + cancellation, + Candidate { + reader: reader.clone(), + media_type: declared, + strength: ProbeStrength::DeclaredCandidate, + }, + ValidationEvidence::DeclaredCandidateStructurallyValidated, + ) + .await?; + if !matches!(inspection, FileInspection::Unknown { .. }) { + return Ok(inspection); + } + } + + if let Some(reader) = self.streaming_text_reader.as_ref() { + if request.source.byte_length().get() > self.ceilings.validation_source_bytes { + return Ok(FileInspection::Unknown { + source: request.source, + }); + } + let declaration = self + .readers + .get(reader) + .ok_or(FileMediaFailure::ProcessorFailed)?; + let text_plain = CanonicalMediaType::from_str("text/plain") + .map_err(|_| FileMediaFailure::ProcessorFailed)?; + return self + .validate_candidate( + processor, + request, + source, + cancellation, + Candidate { + reader: declaration.identity().clone(), + media_type: text_plain, + strength: ProbeStrength::DeclaredCandidate, + }, + ValidationEvidence::StreamingTextValidation, + ) + .await; + } + + Ok(FileInspection::Unknown { + source: request.source, + }) + } + + async fn resolve_candidates( + &self, + processor: &dyn FileMediaProcessor, + request: InspectionRequest, + source: &dyn VerifiedBlobSource, + cancellation: &dyn crate::CancellationSignal, + mut candidates: Vec, + evidence: ValidationEvidence, + ) -> Result { + candidates.sort(); + candidates.dedup(); + let media_types = distinct_media_types( + candidates + .iter() + .map(|candidate| candidate.media_type.clone()), + ); + let readers = candidates + .iter() + .map(|candidate| candidate.reader.clone()) + .collect::>(); + if media_types.len() != 1 || readers.len() != 1 { + if !collision_validation_allowed(evidence, candidates.len()) { + return Ok(FileInspection::Ambiguous { + source: request.source, + media_types, + }); + } + + let all_candidates_provisional = candidates.iter().all(|candidate| { + candidate.strength == ProbeStrength::ProvisionalStructuralCandidate + }); + let validations = async { + let mut successful = Vec::new(); + let mut malformed = Vec::new(); + let mut encrypted = Vec::new(); + for candidate in candidates { + match self + .validate_candidate( + processor, + request.clone(), + source, + cancellation, + candidate, + evidence, + ) + .await? + { + inspection @ (FileInspection::Validated(_) + | FileInspection::DeclaredMismatch { .. }) => successful.push(inspection), + inspection @ FileInspection::Malformed { .. } => malformed.push(inspection), + inspection @ FileInspection::EncryptedOrLocked { .. } => { + encrypted.push(inspection); + } + FileInspection::Unknown { .. } => {} + FileInspection::Ambiguous { .. } => { + return Err(FileMediaFailure::ProcessorFailed); + } + } + } + Ok::<_, FileMediaFailure>((successful, malformed, encrypted)) + }; + let validations = Box::pin(validations); + let deadline = Box::pin(futures_timer::Delay::new(std::time::Duration::from_secs( + MAX_WORKER_WALL_SECONDS, + ))); + let (mut successful, mut malformed, mut encrypted) = + match futures_util::future::select(validations, deadline).await { + futures_util::future::Either::Left((result, _)) => result?, + futures_util::future::Either::Right(((), _)) => { + return Err(FileMediaFailure::ProcessorTimedOut); + } + }; + if successful.len() == 1 && encrypted.is_empty() { + return successful.pop().ok_or(FileMediaFailure::ProcessorFailed); + } + if successful.is_empty() && encrypted.is_empty() && malformed.len() == 1 { + return malformed.pop().ok_or(FileMediaFailure::ProcessorFailed); + } + if successful.is_empty() && malformed.is_empty() && encrypted.len() == 1 { + return encrypted.pop().ok_or(FileMediaFailure::ProcessorFailed); + } + if all_candidates_provisional + && successful.is_empty() + && malformed.is_empty() + && encrypted.is_empty() + { + return Ok(FileInspection::Unknown { + source: request.source, + }); + } + return Ok(FileInspection::Ambiguous { + source: request.source, + media_types, + }); + } + let Some(candidate) = candidates.into_iter().next() else { + return Err(FileMediaFailure::ProcessorFailed); + }; + self.validate_candidate( + processor, + request, + source, + cancellation, + candidate, + evidence, + ) + .await + } + + async fn validate_candidate( + &self, + processor: &dyn FileMediaProcessor, + request: InspectionRequest, + source: &dyn VerifiedBlobSource, + cancellation: &dyn crate::CancellationSignal, + candidate: Candidate, + evidence: ValidationEvidence, + ) -> Result { + let reader = self + .readers + .get(&candidate.reader) + .ok_or(FileMediaFailure::ProcessorFailed)?; + let raw = processor + .validate( + reader.identity(), + FileMediaProviderValidationRequest { + source: request.source.clone(), + media_type: candidate.media_type.clone(), + evidence, + maximum_source_bytes: self + .ceilings + .validation_source_bytes + .min(reader.validation().source_bytes()), + maximum_ranges: self + .ceilings + .validation_ranges + .min(reader.validation().range_count()), + maximum_image_axis: self.ceilings.image_axis, + maximum_decoded_image_pixels: self.ceilings.decoded_image_pixels, + }, + source, + cancellation, + ) + .await?; + match sanitize_validation(reader, &candidate.media_type, evidence, raw)? { + SanitizedValidation::Validated { metadata } => { + if let Ok(declared) = request.source.declared_media_type().canonical_essence() + && declared != candidate.media_type + { + return Ok(FileInspection::DeclaredMismatch { + source: request.source, + declared, + detected: candidate.media_type, + }); + } + Ok(FileInspection::Validated(ValidatedFile::new( + request.source, + candidate.media_type, + candidate.reader, + evidence, + metadata, + reader.views().to_vec(), + ))) + } + SanitizedValidation::Malformed { .. } + if streaming_text_terminal_becomes_unknown(evidence) => + { + Ok(FileInspection::Unknown { + source: request.source, + }) + } + SanitizedValidation::Malformed { reason_code } => Ok(FileInspection::Malformed { + source: request.source, + media_type: candidate.media_type, + reason_code, + }), + SanitizedValidation::EncryptedOrLocked + if streaming_text_terminal_becomes_unknown(evidence) => + { + Ok(FileInspection::Unknown { + source: request.source, + }) + } + SanitizedValidation::EncryptedOrLocked => Ok(FileInspection::EncryptedOrLocked { + source: request.source, + media_type: candidate.media_type, + }), + SanitizedValidation::NoMatch + if candidate.strength == ProbeStrength::ProvisionalStructuralCandidate + || evidence == ValidationEvidence::DeclaredCandidateStructurallyValidated + || evidence == ValidationEvidence::StreamingTextValidation => + { + Ok(FileInspection::Unknown { + source: request.source, + }) + } + SanitizedValidation::NoMatch => Err(FileMediaFailure::ProcessorFailed), + } + } + + /// Repeats inspection, selects one declared view, and sanitizes all output. + pub async fn read( + &self, + processor: &dyn FileMediaProcessor, + request: FileReadRequest, + source: &dyn VerifiedBlobSource, + cancellation: &dyn crate::CancellationSignal, + ) -> Result { + let initial_request = match &request.input { + crate::FileReadInput::Initial { options } if read_options_fit(options) => true, + crate::FileReadInput::Initial { .. } => { + return Err(FileMediaFailure::InvalidViewArguments); + } + crate::FileReadInput::Continuation { .. } => false, + }; + let inspection = self + .inspect(processor, request.inspection.clone(), source, cancellation) + .await?; + let validated = match inspection { + FileInspection::Validated(validated) => validated, + FileInspection::Unknown { .. } => return Err(FileMediaFailure::UnknownType), + FileInspection::Malformed { + media_type, + reason_code, + .. + } => { + return Err(FileMediaFailure::Malformed { + media_type, + reason_code, + }); + } + FileInspection::Ambiguous { .. } => return Err(FileMediaFailure::AmbiguousType), + FileInspection::DeclaredMismatch { + declared, detected, .. + } => { + return Err(FileMediaFailure::DeclaredTypeMismatch { declared, detected }); + } + FileInspection::EncryptedOrLocked { media_type, .. } => { + return Err(FileMediaFailure::EncryptedOrLocked { media_type }); + } + }; + let view = validated + .views() + .iter() + .find(|view| view.name() == &request.view) + .ok_or(FileMediaFailure::UnsupportedView)?; + let reader = self + .readers + .get(validated.reader()) + .ok_or(FileMediaFailure::ProcessorFailed)?; + let raw = processor + .read( + validated.reader(), + FileMediaProviderReadRequest { + source: validated.source().clone(), + detected_media_type: validated.detected_media_type().clone(), + validation: validated.validation(), + metadata: validated.metadata().clone(), + view: request.view, + input: request.input, + maximum_image_axis: self.ceilings.image_axis, + maximum_decoded_image_pixels: self.ceilings.decoded_image_pixels, + maximum_container_entries: self.ceilings.observed_container_entries, + }, + source, + cancellation, + ) + .await?; + sanitize_read(reader, view, self.ceilings, initial_request, raw) + } +} + +/// Checks read options against their object, nesting, and encoded-byte bounds. +pub fn read_options_fit(options: &serde_json::Value) -> bool { + // The outer file_read argument object consumes one contract container. + if !options.is_object() || !json_value_work_fits(options, MAX_READ_INPUT_CONTAINERS - 1) { + return false; + } + serde_json::to_writer( + LimitedWriter { + written: 0, + maximum: MAX_READ_OPTIONS_BYTES, + }, + options, + ) + .is_ok() +} + +fn json_value_work_fits(value: &serde_json::Value, maximum_containers: u32) -> bool { + let mut pending = vec![(value, 0_u32)]; + let mut visited = 0_usize; + while let Some((value, depth)) = pending.pop() { + visited += 1; + if visited > MAX_READ_OPTIONS_NODES { + return false; + } + let (children, next_depth): (usize, Option) = match value { + serde_json::Value::Array(values) => { + let Some(next) = depth + .checked_add(1) + .filter(|next| *next <= maximum_containers) + else { + return false; + }; + (values.len(), Some(next)) + } + serde_json::Value::Object(values) => { + let Some(next) = depth + .checked_add(1) + .filter(|next| *next <= maximum_containers) + else { + return false; + }; + (values.len(), Some(next)) + } + serde_json::Value::Null + | serde_json::Value::Bool(_) + | serde_json::Value::Number(_) + | serde_json::Value::String(_) => (0, None), + }; + if children > MAX_READ_OPTIONS_NODES.saturating_sub(visited + pending.len()) { + return false; + } + if let Some(next) = next_depth { + match value { + serde_json::Value::Array(values) => { + pending.extend(values.iter().map(|child| (child, next))); + } + serde_json::Value::Object(values) => { + pending.extend(values.values().map(|child| (child, next))); + } + _ => {} + } + } + } + true +} + +struct LimitedWriter { + written: usize, + maximum: usize, +} + +impl std::io::Write for LimitedWriter { + fn write(&mut self, bytes: &[u8]) -> std::io::Result { + self.written = self + .written + .checked_add(bytes.len()) + .filter(|total| *total <= self.maximum) + .ok_or_else(|| std::io::Error::other("serialized value exceeds its byte ceiling"))?; + Ok(bytes.len()) + } + + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } +} + +#[derive(Clone, Debug, Eq, Ord, PartialEq, PartialOrd)] +struct Candidate { + reader: ReaderIdentity, + media_type: CanonicalMediaType, + strength: ProbeStrength, +} + +fn recognized_probe_strength(strength: ProbeStrength) -> bool { + matches!( + strength, + ProbeStrength::Strong + | ProbeStrength::ProvisionalStructuralCandidate + | ProbeStrength::StructuralCandidate + ) +} + +fn collision_validation_allowed(evidence: ValidationEvidence, candidate_count: usize) -> bool { + evidence == ValidationEvidence::StructuralValidation + && candidate_count <= MAX_COLLISION_VALIDATION_CANDIDATES +} + +fn streaming_text_terminal_becomes_unknown(evidence: ValidationEvidence) -> bool { + evidence == ValidationEvidence::StreamingTextValidation +} + +enum SanitizedProbe { + NoMatch, + Candidate(Candidate), + Malformed { + media_type: CanonicalMediaType, + reason_code: ReasonCode, + }, +} + +fn sanitize_probe( + reader: &ReaderDeclaration, + raw: ProcessorProbeOutput, +) -> Result { + match raw { + ProcessorProbeOutput::NoMatch => Ok(SanitizedProbe::NoMatch), + ProcessorProbeOutput::Candidate { + media_type, + strength, + } => { + let media_type = CanonicalMediaType::from_str(&media_type) + .map_err(|_| FileMediaFailure::ProcessorFailed)?; + if !reader.media_types().contains(&media_type) + || strength == ProbeStrength::DeclaredCandidate + { + return Err(FileMediaFailure::ProcessorFailed); + } + Ok(SanitizedProbe::Candidate(Candidate { + reader: reader.identity().clone(), + media_type, + strength, + })) + } + ProcessorProbeOutput::RecognizedMalformed { + media_type, + reason_code, + } => { + let media_type = CanonicalMediaType::from_str(&media_type) + .map_err(|_| FileMediaFailure::ProcessorFailed)?; + let reason_code = registered_reason(reader, &reason_code)?; + if !reader.media_types().contains(&media_type) { + return Err(FileMediaFailure::ProcessorFailed); + } + Ok(SanitizedProbe::Malformed { + media_type, + reason_code, + }) + } + } +} + +enum SanitizedValidation { + Validated { metadata: BoundedMetadata }, + Malformed { reason_code: ReasonCode }, + EncryptedOrLocked, + NoMatch, +} + +fn sanitize_validation( + reader: &ReaderDeclaration, + selected_media_type: &CanonicalMediaType, + selected_evidence: ValidationEvidence, + raw: ProcessorValidationOutput, +) -> Result { + match raw { + ProcessorValidationOutput::Validated { + media_type, + evidence, + metadata_json, + } => { + let media_type = CanonicalMediaType::from_str(&media_type) + .map_err(|_| FileMediaFailure::ProcessorFailed)?; + if &media_type != selected_media_type || evidence != selected_evidence { + return Err(FileMediaFailure::ProcessorFailed); + } + let metadata = BoundedMetadata::try_new(&metadata_json) + .map_err(|_| FileMediaFailure::ProcessorFailed)?; + Ok(SanitizedValidation::Validated { metadata }) + } + ProcessorValidationOutput::Malformed { + media_type, + reason_code, + } => { + let media_type = CanonicalMediaType::from_str(&media_type) + .map_err(|_| FileMediaFailure::ProcessorFailed)?; + if &media_type != selected_media_type { + return Err(FileMediaFailure::ProcessorFailed); + } + Ok(SanitizedValidation::Malformed { + reason_code: registered_reason(reader, &reason_code)?, + }) + } + ProcessorValidationOutput::EncryptedOrLocked { media_type } => { + let media_type = CanonicalMediaType::from_str(&media_type) + .map_err(|_| FileMediaFailure::ProcessorFailed)?; + if &media_type != selected_media_type { + return Err(FileMediaFailure::ProcessorFailed); + } + Ok(SanitizedValidation::EncryptedOrLocked) + } + ProcessorValidationOutput::NoMatch => Ok(SanitizedValidation::NoMatch), + } +} + +fn sanitize_read( + reader: &ReaderDeclaration, + view: &crate::ReadViewDeclaration, + ceilings: FileMediaCeilings, + initial_request: bool, + raw: ProcessorReadOutput, +) -> Result { + match raw { + ProcessorReadOutput::Text { + body, + truncated, + cursor, + } => { + let ReadViewBounds::Text { output_bytes, .. } = view.bounds() else { + return Err(FileMediaFailure::ProcessorFailed); + }; + if body.len() > output_bytes + || body.len() > crate::MAX_TEXT_BODY_BYTES + || body.len() > ceilings.text_or_json_bytes + || body.contains('\0') + { + return Err(FileMediaFailure::ProcessorFailed); + } + let continuation = sanitize_continuation(truncated, cursor)?; + Ok(FileReadResult::Text { body, continuation }) + } + ProcessorReadOutput::Structured { + body_json, + truncated, + cursor, + } => { + let ReadViewBounds::Structured { + output_bytes, + depth, + nodes, + string_bytes, + .. + } = view.bounds() + else { + return Err(FileMediaFailure::ProcessorFailed); + }; + if body_json.len() > output_bytes + || body_json.len() > ceilings.text_or_json_bytes + || body_json.contains('\0') + { + return Err(FileMediaFailure::ProcessorFailed); + } + let continuation = sanitize_continuation(truncated, cursor)?; + let maximum_nodes = nodes.min(ceilings.structured_nodes); + let body = crate::value::parse_json_without_duplicate_members_bounded( + &body_json, + crate::value::JsonParseLimits { + maximum_nodes, + maximum_container_entries: ceilings.observed_container_entries, + }, + ) + .map_err(|_| FileMediaFailure::ProcessorFailed)?; + let canonical_bytes = serde_json::to_string(&body) + .map_err(|_| FileMediaFailure::ProcessorFailed)? + .len(); + if canonical_bytes > output_bytes || canonical_bytes > ceilings.text_or_json_bytes { + return Err(FileMediaFailure::ProcessorFailed); + } + let mut observed = ObservedJson::default(); + observe_json(&body, 0, &mut observed)?; + if observed.depth > depth + || observed.depth > ceilings.structured_depth + || observed.nodes > nodes + || observed.nodes > ceilings.structured_nodes + || observed.max_container_entries > ceilings.observed_container_entries + || observed.string_bytes > string_bytes + { + return Err(FileMediaFailure::ProcessorFailed); + } + Ok(FileReadResult::Structured { body, continuation }) + } + ProcessorReadOutput::InvalidViewArguments if initial_request => { + Err(FileMediaFailure::InvalidViewArguments) + } + ProcessorReadOutput::InvalidViewArguments => Err(FileMediaFailure::ProcessorFailed), + ProcessorReadOutput::UnsupportedView => Err(FileMediaFailure::ProcessorFailed), + // The declared source-byte bound limits cumulative I/O work, not intrinsic blob size. + ProcessorReadOutput::SourceTooLarge { .. } => Err(FileMediaFailure::ProcessorFailed), + ProcessorReadOutput::ExpansionLimitExceeded { limit_kind } => { + Err(FileMediaFailure::ExpansionLimitExceeded { + limit_kind: registered_reason(reader, &limit_kind)?, + }) + } + ProcessorReadOutput::OutputUnitTooLarge => Err(FileMediaFailure::OutputUnitTooLarge), + } +} + +#[derive(Default)] +struct ObservedJson { + depth: u32, + nodes: u64, + string_bytes: usize, + max_container_entries: u64, +} + +fn observe_json( + value: &serde_json::Value, + depth: u32, + observed: &mut ObservedJson, +) -> Result<(), FileMediaFailure> { + observed.nodes = observed + .nodes + .checked_add(1) + .ok_or(FileMediaFailure::ProcessorFailed)?; + match value { + serde_json::Value::String(value) => { + observed.string_bytes = observed + .string_bytes + .checked_add(value.len()) + .ok_or(FileMediaFailure::ProcessorFailed)?; + } + serde_json::Value::Array(values) => { + let next = depth + .checked_add(1) + .ok_or(FileMediaFailure::ProcessorFailed)?; + observed.depth = observed.depth.max(next); + let entries = + u64::try_from(values.len()).map_err(|_| FileMediaFailure::ProcessorFailed)?; + observed.max_container_entries = observed.max_container_entries.max(entries); + for value in values { + observe_json(value, next, observed)?; + } + } + serde_json::Value::Object(values) => { + let next = depth + .checked_add(1) + .ok_or(FileMediaFailure::ProcessorFailed)?; + observed.depth = observed.depth.max(next); + let entries = + u64::try_from(values.len()).map_err(|_| FileMediaFailure::ProcessorFailed)?; + observed.max_container_entries = observed.max_container_entries.max(entries); + for (name, value) in values { + observed.string_bytes = observed + .string_bytes + .checked_add(name.len()) + .ok_or(FileMediaFailure::ProcessorFailed)?; + observe_json(value, next, observed)?; + } + } + serde_json::Value::Null | serde_json::Value::Bool(_) | serde_json::Value::Number(_) => {} + } + Ok(()) +} + +fn sanitize_continuation( + truncated: bool, + cursor: Option, +) -> Result { + match (truncated, cursor) { + (false, None) => Ok(ReadContinuation::Complete), + (true, Some(cursor)) => { + let cursor = ReadContinuationCursor::try_new(cursor) + .map_err(|_| FileMediaFailure::ProcessorFailed)?; + Ok(ReadContinuation::More { cursor }) + } + (false, Some(_)) | (true, None) => Err(FileMediaFailure::ProcessorFailed), + } +} + +fn registered_reason( + reader: &ReaderDeclaration, + raw: &str, +) -> Result { + let reason = ReasonCode::try_new(raw).map_err(|_| FileMediaFailure::ProcessorFailed)?; + if reader.reason_codes().contains(&reason) { + Ok(reason) + } else { + Err(FileMediaFailure::ProcessorFailed) + } +} + +fn distinct_media_types( + values: impl IntoIterator, +) -> Vec { + values + .into_iter() + .collect::>() + .into_iter() + .collect() +} + +fn validate_reader( + reader: &ReaderDeclaration, + ceilings: FileMediaCeilings, +) -> Result<(), FileMediaRegistryConstructionError> { + if reader.media_types().len() > MAX_MEDIA_TYPES_PER_READER + || reader.views().len() > MAX_VIEWS_PER_READER + || reader.reason_codes().len() > MAX_REASON_CODES_PER_READER + { + return Err(FileMediaRegistryConstructionError::Inventory); + } + if has_duplicates(reader.media_types()) + || has_duplicates(reader.reason_codes()) + || has_duplicate_view_names(reader) + { + return Err(FileMediaRegistryConstructionError::DuplicateReaderMember); + } + validate_inspection_view_inventory(reader, ceilings)?; + let probe = reader.probe(); + if probe.prefix_bytes() > ceilings.probe_prefix_bytes + || probe.suffix_bytes() > ceilings.probe_suffix_bytes + || probe.range_count() > ceilings.probe_ranges + || (probe.prefix_bytes() == 0 && probe.suffix_bytes() == 0 && probe.range_count() == 0) + || probe.cumulative_bytes() == 0 + || probe.cumulative_bytes() > ceilings.probe_cumulative_bytes + || probe + .prefix_bytes() + .checked_add(probe.suffix_bytes()) + .is_none_or(|minimum| minimum > probe.cumulative_bytes()) + { + return Err(FileMediaRegistryConstructionError::ProbeBounds); + } + let validation = reader.validation(); + if validation.source_bytes() == 0 + || validation.source_bytes() > crate::MAX_VALIDATION_SOURCE_BYTES + || validation.range_count() == 0 + || validation.range_count() > crate::MAX_VALIDATION_RANGES + { + return Err(FileMediaRegistryConstructionError::ViewBounds); + } + for view in reader.views() { + validate_view(view.access(), view.bounds(), ceilings)?; + } + Ok(()) +} + +fn validate_inspection_view_inventory( + reader: &ReaderDeclaration, + ceilings: FileMediaCeilings, +) -> Result<(), FileMediaRegistryConstructionError> { + let maximum_bytes = MAX_INSPECTION_VIEW_INVENTORY_BYTES.min( + ceilings + .text_or_json_bytes + .saturating_sub(INSPECTION_NON_VIEW_RESERVE_BYTES), + ); + let mut projected_bytes = 2_usize; + for (index, view) in reader.views().iter().enumerate() { + let encoded = serde_json::to_vec(&serde_json::json!({ + "name": view.name().as_str(), + "description": view.description(), + "arguments_schema": view.arguments_schema().value(), + "output": inspection_output_kind(view.output_kind()), + })) + .map_err(|_| FileMediaRegistryConstructionError::Inventory)?; + projected_bytes = projected_bytes + .checked_add(encoded.len()) + .and_then(|total| total.checked_add(usize::from(index > 0))) + .ok_or(FileMediaRegistryConstructionError::Inventory)?; + if projected_bytes > maximum_bytes { + return Err(FileMediaRegistryConstructionError::Inventory); + } + } + Ok(()) +} + +const fn inspection_output_kind(kind: crate::ReadOutputKind) -> &'static str { + match kind { + crate::ReadOutputKind::Text => "text", + crate::ReadOutputKind::Structured => "structured", + crate::ReadOutputKind::Image => "image", + crate::ReadOutputKind::Audio => "audio", + crate::ReadOutputKind::File => "file", + } +} + +/// Checks provider declarations against registry-compatible inventory bounds. +pub fn provider_declaration_inventory_fits<'a>( + providers: impl IntoIterator, +) -> bool { + let mut readers = 0_usize; + let mut media_types = 0_usize; + let mut views = 0_usize; + let mut schema_bytes = 0_usize; + let mut reason_codes = 0_usize; + for provider in providers { + if provider.readers().len() > MAX_READERS_PER_PROVIDER { + return false; + } + let Some(next_readers) = readers.checked_add(provider.readers().len()) else { + return false; + }; + readers = next_readers; + if readers > MAX_REGISTRY_READERS { + return false; + } + for reader in provider.readers() { + if reader.media_types().len() > MAX_MEDIA_TYPES_PER_READER + || reader.views().len() > MAX_VIEWS_PER_READER + || reader.reason_codes().len() > MAX_REASON_CODES_PER_READER + { + return false; + } + let Some(next_media_types) = media_types.checked_add(reader.media_types().len()) else { + return false; + }; + media_types = next_media_types; + let Some(next_views) = views.checked_add(reader.views().len()) else { + return false; + }; + views = next_views; + let Some(next_reason_codes) = reason_codes.checked_add(reader.reason_codes().len()) + else { + return false; + }; + reason_codes = next_reason_codes; + if media_types > MAX_REGISTRY_MEDIA_TYPES + || views > MAX_REGISTRY_VIEWS + || reason_codes > MAX_REGISTRY_REASON_CODES + { + return false; + } + for view in reader.views() { + let Some(next_schema_bytes) = + schema_bytes.checked_add(view.arguments_schema().as_str().len()) + else { + return false; + }; + schema_bytes = next_schema_bytes; + if schema_bytes > MAX_REGISTRY_SCHEMA_BYTES { + return false; + } + } + } + } + true +} + +fn validate_aggregate_inventory( + providers: &[FileMediaProviderDeclaration], +) -> Result<(), FileMediaRegistryConstructionError> { + if provider_declaration_inventory_fits(providers) { + Ok(()) + } else { + Err(FileMediaRegistryConstructionError::Inventory) + } +} + +fn validate_aggregate_probe_budget( + providers: &[FileMediaProviderDeclaration], +) -> Result<(), FileMediaRegistryConstructionError> { + let mut bytes = 0_u64; + let mut reads = 0_u32; + for reader in providers.iter().flat_map(|provider| provider.readers()) { + let probe = reader.probe(); + bytes = bytes + .checked_add(probe.cumulative_bytes()) + .ok_or(FileMediaRegistryConstructionError::ProbeBounds)?; + let fixed_reads = u32::from(probe.prefix_bytes() > 0) + .checked_add(u32::from(probe.suffix_bytes() > 0)) + .ok_or(FileMediaRegistryConstructionError::ProbeBounds)?; + reads = reads + .checked_add(probe.range_count()) + .and_then(|total| total.checked_add(fixed_reads)) + .ok_or(FileMediaRegistryConstructionError::ProbeBounds)?; + if bytes > MAX_INSPECTION_PROBE_BYTES || reads > MAX_INSPECTION_PROBE_READS { + return Err(FileMediaRegistryConstructionError::ProbeBounds); + } + } + Ok(()) +} + +fn has_duplicates(values: &[Value]) -> bool { + values + .iter() + .cloned() + .collect::>() + .len() + != values.len() +} + +fn has_duplicate_view_names(reader: &ReaderDeclaration) -> bool { + reader + .views() + .iter() + .map(|view| view.name().clone()) + .collect::>() + .len() + != reader.views().len() +} + +fn validate_view( + access: ReadAccessPattern, + bounds: ReadViewBounds, + ceilings: FileMediaCeilings, +) -> Result<(), FileMediaRegistryConstructionError> { + if matches!( + access, + ReadAccessPattern::Streaming { maximum_ranges } + | ReadAccessPattern::RandomAccess { maximum_ranges } + if maximum_ranges == 0 || maximum_ranges > ceilings.read_ranges + ) || bounds.source_bytes() == 0 + || bounds.source_bytes() > ceilings.read_source_bytes + { + return Err(FileMediaRegistryConstructionError::ViewBounds); + } + let valid = match bounds { + ReadViewBounds::Text { output_bytes, .. } => { + output_bytes > 0 + && output_bytes <= crate::MAX_TEXT_BODY_BYTES + && output_bytes <= ceilings.text_or_json_bytes + } + ReadViewBounds::Structured { + output_bytes, + depth, + nodes, + string_bytes, + .. + } => { + output_bytes > 0 + && output_bytes <= MAX_STRUCTURED_BODY_BYTES + && output_bytes <= ceilings.text_or_json_bytes + && depth > 0 + && depth <= ceilings.structured_depth + && nodes > 0 + && nodes <= ceilings.structured_nodes + && string_bytes > 0 + && string_bytes <= output_bytes + } + ReadViewBounds::Image { .. } + | ReadViewBounds::Audio { .. } + | ReadViewBounds::File { .. } => false, + }; + if valid { + Ok(()) + } else { + Err(FileMediaRegistryConstructionError::ViewBounds) + } +} + +/// Closed static registry construction failure. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum FileMediaRegistryConstructionError { + /// Configured ceilings did not lower the compiled set. + Ceilings, + /// A finite inventory exceeded its compiled count. + Inventory, + /// Providers were declared without available strong isolation. + IsolationUnavailable, + /// Provider identity was duplicated. + DuplicateProvider, + /// Reader identity was duplicated. + DuplicateReader, + /// An exact media type was claimed by several readers. + DuplicateMediaTypeClaim, + /// One reader repeated a media type, view name, or reason code. + DuplicateReaderMember, + /// Probe bounds were zero, contradictory, or excessive. + ProbeBounds, + /// View bounds were absent, contradictory, or excessive. + ViewBounds, + /// Text fallback registration was absent or ambiguous. + TextFallback, +} + +impl fmt::Display for FileMediaRegistryConstructionError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str(match self { + Self::Ceilings => "file media ceilings may only lower compiled limits", + Self::Inventory => "file media registry inventory exceeds a compiled bound", + Self::IsolationUnavailable => "file media isolation is unavailable", + Self::DuplicateProvider => "file media provider identity is duplicated", + Self::DuplicateReader => "file media reader identity is duplicated", + Self::DuplicateMediaTypeClaim => "file media type has several registered readers", + Self::DuplicateReaderMember => "file media reader member is duplicated", + Self::ProbeBounds => "file media probe bounds are invalid", + Self::ViewBounds => "file media view bounds are invalid", + Self::TextFallback => "file media text fallback is invalid", + }) + } +} + +impl Error for FileMediaRegistryConstructionError {} + +#[cfg(test)] +mod tests { + use super::*; + + fn nested_arrays(depth: u32) -> serde_json::Value { + (0..depth).fold(serde_json::Value::Null, |value, _| { + serde_json::Value::Array(vec![value]) + }) + } + + fn binary_json_tree(null_leaves: usize) -> serde_json::Value { + let mut level = vec![serde_json::Value::Null; null_leaves]; + while level.len() > 1 { + level = level + .chunks(2) + .map(|pair| serde_json::Value::Array(pair.to_vec())) + .collect(); + } + level.pop().expect("the fixture has at least one leaf") + } + + #[test] + fn malformed_ambiguity_includes_structural_and_strong_claims() { + assert!(recognized_probe_strength( + ProbeStrength::StructuralCandidate + )); + assert!(recognized_probe_strength(ProbeStrength::Strong)); + assert!(!recognized_probe_strength(ProbeStrength::DeclaredCandidate)); + } + + #[test] + fn strong_signature_collisions_remain_ambiguous_without_validation() { + assert!(!collision_validation_allowed( + ValidationEvidence::StrongSignature, + 2 + )); + } + + #[test] + fn structural_collision_validation_has_a_two_candidate_ceiling() { + assert!(collision_validation_allowed( + ValidationEvidence::StructuralValidation, + MAX_COLLISION_VALIDATION_CANDIDATES + )); + assert!(!collision_validation_allowed( + ValidationEvidence::StructuralValidation, + MAX_COLLISION_VALIDATION_CANDIDATES + 1 + )); + } + + #[test] + fn streaming_text_terminal_validation_becomes_unknown() { + assert!(streaming_text_terminal_becomes_unknown( + ValidationEvidence::StreamingTextValidation + )); + assert!(!streaming_text_terminal_becomes_unknown( + ValidationEvidence::StructuralValidation + )); + } + + #[test] + fn json_depth_counts_containers_without_charging_the_scalar_leaf() { + let body = nested_arrays(crate::MAX_STRUCTURED_DEPTH); + let mut observed = ObservedJson::default(); + + observe_json(&body, 0, &mut observed).expect("the bounded fixture is observable"); + + assert_eq!(observed.depth, crate::MAX_STRUCTURED_DEPTH); + } + + #[test] + fn read_option_serialization_stops_at_its_byte_ceiling() { + let options = serde_json::json!({ "value": "x".repeat(MAX_READ_OPTIONS_BYTES) }); + + assert!(!read_options_fit(&options)); + } + + #[test] + fn read_options_honor_the_input_container_boundary() { + let options = serde_json::json!({ + "nested": nested_arrays(MAX_READ_INPUT_CONTAINERS - 2) + }); + assert!(read_options_fit(&options)); + + let options = serde_json::json!({ + "nested": nested_arrays(MAX_READ_INPUT_CONTAINERS - 1) + }); + assert!(!read_options_fit(&options)); + } + + #[test] + fn read_options_reject_broad_work_before_growing_the_frontier() { + let options = serde_json::json!({ + "values": vec![serde_json::Value::Null; MAX_READ_OPTIONS_NODES] + }); + + assert!(!json_value_work_fits( + &options, + MAX_READ_INPUT_CONTAINERS - 1 + )); + } + + #[test] + fn binary_json_tree_preserves_odd_leaf_groups() { + assert_eq!( + binary_json_tree(3), + serde_json::json!([[null, null], [null]]) + ); + } + + #[test] + fn read_options_reject_balanced_work_with_a_small_frontier() { + let options = serde_json::json!({ "tree": binary_json_tree(32_769) }); + + assert!(!json_value_work_fits( + &options, + MAX_READ_INPUT_CONTAINERS - 1 + )); + } +} diff --git a/crates/file-media-runtime/src/value.rs b/crates/file-media-runtime/src/value.rs new file mode 100644 index 0000000000..24dc34270b --- /dev/null +++ b/crates/file-media-runtime/src/value.rs @@ -0,0 +1,938 @@ +use std::{error::Error, fmt, num::NonZeroU64, str::FromStr, sync::Arc}; + +use serde::de::{DeserializeSeed, Error as _, MapAccess, SeqAccess, Visitor}; +use serde_json::value::RawValue; + +const SHA256_PREFIX: &str = "sha256:"; +// numeric-bound: not-a-bound - fixed lowercase SHA-256 hexadecimal width +const SHA256_HEX_BYTES: usize = 64; +// numeric-bound: ceiling - bounds retained caller media-type text +const MAX_DECLARED_MEDIA_TYPE_BYTES: usize = 255; +// numeric-bound: ceiling - enforces the RFC 6838 restricted-name token width +const MAX_MEDIA_TYPE_TOKEN_BYTES: usize = 127; +// numeric-bound: ceiling - bounds retained caller display-name text +const MAX_DISPLAY_FILENAME_BYTES: usize = 255; +// numeric-bound: ceiling - bounds registry identity and selector storage +const MAX_NAME_BYTES: usize = 64; +// numeric-bound: ceiling - bounds retained reader-revision text +const MAX_REVISION_BYTES: usize = 32; +// numeric-bound: ceiling - bounds retained and parsed view-schema memory +const MAX_SCHEMA_BYTES: usize = 65_536; +// numeric-bound: ceiling - bounds retained and parsed processor-metadata memory +const MAX_METADATA_BYTES: usize = 16_384; +// numeric-bound: ceiling - bounds retained untrusted continuation state +const MAX_CONTINUATION_CURSOR_BYTES: usize = 1_024; + +/// SHA-256 identity at the provider-neutral boundary. +#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] +pub struct FileDigest([u8; 32]); + +impl FileDigest { + /// Reconstitutes a digest already verified by the blob layer. + pub const fn from_bytes(bytes: [u8; 32]) -> Self { + Self(bytes) + } + + /// Borrows the fixed digest bytes. + pub const fn as_bytes(&self) -> &[u8; 32] { + &self.0 + } +} + +impl fmt::Display for FileDigest { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str(SHA256_PREFIX)?; + for byte in self.0 { + write!(formatter, "{byte:02x}")?; + } + Ok(()) + } +} + +impl FromStr for FileDigest { + type Err = RegistryValueError; + + fn from_str(value: &str) -> Result { + let encoded = value + .strip_prefix(SHA256_PREFIX) + .ok_or(RegistryValueError::Digest)?; + if encoded.len() != SHA256_HEX_BYTES { + return Err(RegistryValueError::Digest); + } + let mut bytes = [0_u8; 32]; + for (destination, pair) in bytes.iter_mut().zip(encoded.as_bytes().chunks_exact(2)) { + let high = lowercase_hex(pair[0]).ok_or(RegistryValueError::Digest)?; + let low = lowercase_hex(pair[1]).ok_or(RegistryValueError::Digest)?; + *destination = (high << 4) | low; + } + Ok(Self(bytes)) + } +} + +fn lowercase_hex(byte: u8) -> Option { + match byte { + b'0'..=b'9' => Some(byte - b'0'), + b'a'..=b'f' => Some(byte - b'a' + 10), + _ => None, + } +} + +/// Caller intent for one use of immutable bytes. +#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] +pub enum AttachmentKind { + /// Image intent. + Image, + /// Document intent. + Document, + /// General file intent. + File, +} + +/// Exact bounded caller-declared media type. +#[derive(Clone, Eq, Hash, PartialEq)] +pub struct DeclaredMediaType(Arc); + +impl DeclaredMediaType { + /// Admits a nonempty visible-ASCII value without normalization. + pub fn try_new(value: impl Into>) -> Result { + let value = value.into(); + if value.is_empty() + || value.len() > MAX_DECLARED_MEDIA_TYPE_BYTES + || !value + .bytes() + .all(|byte| byte.is_ascii_graphic() || byte == b' ') + { + return Err(RegistryValueError::DeclaredMediaType); + } + Ok(Self(value)) + } + + /// Borrows the exact caller spelling. + pub fn as_str(&self) -> &str { + &self.0 + } + + /// Parses only a parameter-free canonical essence. + pub fn canonical_essence(&self) -> Result { + CanonicalMediaType::from_str(self.as_str()) + } +} + +impl fmt::Debug for DeclaredMediaType { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("DeclaredMediaType([REDACTED])") + } +} + +/// Bounded attachment basename supplied by a caller. +#[derive(Clone, Eq, Hash, PartialEq)] +pub struct DisplayFilename(Arc); + +impl DisplayFilename { + /// Admits one nonempty basename without path or null characters. + pub fn try_new(value: impl Into>) -> Result { + let value = value.into(); + let invalid = value.is_empty() + || value.len() > MAX_DISPLAY_FILENAME_BYTES + || value.as_ref() == "." + || value.as_ref() == ".." + || value.contains('/') + || value.contains('\\') + || value.contains('\0'); + if invalid { + Err(RegistryValueError::DisplayFilename) + } else { + Ok(Self(value)) + } + } + + /// Borrows the exact caller spelling. + pub fn as_str(&self) -> &str { + &self.0 + } +} + +impl fmt::Debug for DisplayFilename { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("DisplayFilename([REDACTED])") + } +} + +/// One semantic use of immutable file bytes. +#[derive(Clone, Eq, PartialEq)] +pub struct FileUse { + digest: FileDigest, + byte_length: NonZeroU64, + attachment_kind: AttachmentKind, + declared_media_type: DeclaredMediaType, + display_filename: Option, +} + +impl FileUse { + /// Constructs one checked file use from already-admitted caller metadata. + pub const fn new( + digest: FileDigest, + byte_length: NonZeroU64, + attachment_kind: AttachmentKind, + declared_media_type: DeclaredMediaType, + display_filename: Option, + ) -> Self { + Self { + digest, + byte_length, + attachment_kind, + declared_media_type, + display_filename, + } + } + + /// Returns the immutable byte identity. + pub const fn digest(&self) -> FileDigest { + self.digest + } + + /// Returns the catalogued positive length. + pub const fn byte_length(&self) -> NonZeroU64 { + self.byte_length + } + + /// Returns caller attachment intent. + pub const fn attachment_kind(&self) -> AttachmentKind { + self.attachment_kind + } + + /// Borrows the exact declared type. + pub const fn declared_media_type(&self) -> &DeclaredMediaType { + &self.declared_media_type + } + + /// Borrows the optional display basename. + pub const fn display_filename(&self) -> Option<&DisplayFilename> { + self.display_filename.as_ref() + } +} + +impl fmt::Debug for FileUse { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("FileUse") + .field("digest", &self.digest) + .field("byte_length", &self.byte_length) + .field("attachment_kind", &self.attachment_kind) + .field("declared_media_type", &"[REDACTED]") + .field( + "display_filename", + &self.display_filename.as_ref().map(|_| "[REDACTED]"), + ) + .finish() + } +} + +/// Canonical lowercase ASCII media-type essence with no parameters. +#[derive(Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] +pub struct CanonicalMediaType(Arc); + +impl CanonicalMediaType { + /// Borrows the canonical `type/subtype` spelling. + pub fn as_str(&self) -> &str { + &self.0 + } +} + +impl fmt::Display for CanonicalMediaType { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str(self.as_str()) + } +} + +impl FromStr for CanonicalMediaType { + type Err = MediaTypeParseError; + + fn from_str(value: &str) -> Result { + if value.len() > MAX_DECLARED_MEDIA_TYPE_BYTES || value.contains(';') { + return Err(MediaTypeParseError); + } + let Some((type_name, subtype_name)) = value.split_once('/') else { + return Err(MediaTypeParseError); + }; + if subtype_name.contains('/') + || !valid_media_token(type_name) + || !valid_media_token(subtype_name) + { + return Err(MediaTypeParseError); + } + Ok(Self(Arc::from(value))) + } +} + +fn valid_media_token(value: &str) -> bool { + if value.len() > MAX_MEDIA_TYPE_TOKEN_BYTES { + return false; + } + let mut bytes = value.bytes(); + bytes + .next() + .is_some_and(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit()) + && bytes.all(|byte| { + byte.is_ascii_lowercase() + || byte.is_ascii_digit() + || matches!( + byte, + b'!' | b'#' | b'$' | b'&' | b'^' | b'_' | b'.' | b'+' | b'-' + ) + }) +} + +/// A media type was not a canonical parameter-free essence. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct MediaTypeParseError; + +impl fmt::Display for MediaTypeParseError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("media type is not a canonical lowercase ASCII essence") + } +} + +impl Error for MediaTypeParseError {} + +macro_rules! checked_name { + ($name:ident, $label:literal) => { + #[doc = $label] + #[derive(Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] + pub struct $name(Arc); + + impl $name { + /// Admits one canonical lowercase ASCII registry token. + pub fn try_new(value: impl Into>) -> Result { + let value = value.into(); + if valid_registry_name(&value) { + Ok(Self(value)) + } else { + Err(RegistryValueError::Name) + } + } + + /// Borrows the canonical spelling. + pub fn as_str(&self) -> &str { + &self.0 + } + } + + impl fmt::Display for $name { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str(self.as_str()) + } + } + }; +} + +checked_name!(FileReaderProviderName, "One compiled provider identity."); +checked_name!(FileReaderName, "One reader identity within a provider."); +checked_name!(ReadViewName, "One provider-owned read-view name."); +checked_name!( + ReasonCode, + "One registered sanitized processor reason code." +); + +fn valid_registry_name(value: &str) -> bool { + value.len() <= MAX_NAME_BYTES + && value + .bytes() + .next() + .is_some_and(|byte| byte.is_ascii_lowercase()) + && value.bytes().all(|byte| { + byte.is_ascii_lowercase() || byte.is_ascii_digit() || matches!(byte, b'_' | b'-' | b'.') + }) +} + +/// Immutable reader implementation revision. +#[derive(Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] +pub struct FileReaderRevision(Arc); + +impl FileReaderRevision { + /// Admits one bounded visible-ASCII revision label. + pub fn try_new(value: impl Into>) -> Result { + let value = value.into(); + if value.is_empty() + || value.len() > MAX_REVISION_BYTES + || !value.bytes().all(|byte| byte.is_ascii_graphic()) + { + return Err(RegistryValueError::Revision); + } + Ok(Self(value)) + } + + /// Borrows the exact revision spelling. + pub fn as_str(&self) -> &str { + &self.0 + } +} + +/// Stable provider, reader, and revision tuple. +#[derive(Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] +pub struct ReaderIdentity { + provider: FileReaderProviderName, + reader: FileReaderName, + revision: FileReaderRevision, +} + +impl ReaderIdentity { + /// Constructs one checked tuple. + pub const fn new( + provider: FileReaderProviderName, + reader: FileReaderName, + revision: FileReaderRevision, + ) -> Self { + Self { + provider, + reader, + revision, + } + } + + /// Borrows the provider identity. + pub const fn provider(&self) -> &FileReaderProviderName { + &self.provider + } + + /// Borrows the reader identity. + pub const fn reader(&self) -> &FileReaderName { + &self.reader + } + + /// Borrows the immutable revision. + pub const fn revision(&self) -> &FileReaderRevision { + &self.revision + } +} + +/// Canonical compact object-rooted JSON Schema declaration. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct CanonicalJsonObjectSchema { + compact: Arc, + value: serde_json::Value, +} + +impl CanonicalJsonObjectSchema { + /// Parses, bounds, and canonicalizes one object-rooted schema. + pub fn try_new(value: &str) -> Result { + if value.len() > MAX_SCHEMA_BYTES || value.contains('\0') { + return Err(RegistryValueError::Schema); + } + let parsed = + parse_json_without_duplicate_members(value).map_err(|_| RegistryValueError::Schema)?; + let object = parsed.as_object().ok_or(RegistryValueError::Schema)?; + if object.get("type").and_then(serde_json::Value::as_str) != Some("object") { + return Err(RegistryValueError::Schema); + } + let compact = serde_json::to_string(&parsed).map_err(|_| RegistryValueError::Schema)?; + if compact.len() > MAX_SCHEMA_BYTES { + return Err(RegistryValueError::Schema); + } + Ok(Self { + compact: Arc::from(compact), + value: parsed, + }) + } + + /// Borrows the compact canonical JSON spelling. + pub fn as_str(&self) -> &str { + &self.compact + } + + /// Borrows the parsed schema object. + pub const fn value(&self) -> &serde_json::Value { + &self.value + } +} + +/// Bounded canonical processor metadata object. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct BoundedMetadata { + compact: Arc, + value: serde_json::Value, +} + +impl BoundedMetadata { + /// Parses a processor-supplied JSON object and rejects excess or malformed data. + pub fn try_new(value: &str) -> Result { + if value.len() > MAX_METADATA_BYTES || value.contains('\0') { + return Err(RegistryValueError::Metadata); + } + let parsed = parse_json_without_duplicate_members(value) + .map_err(|_| RegistryValueError::Metadata)?; + if !parsed.is_object() { + return Err(RegistryValueError::Metadata); + } + let compact = serde_json::to_string(&parsed).map_err(|_| RegistryValueError::Metadata)?; + if compact.len() > MAX_METADATA_BYTES { + return Err(RegistryValueError::Metadata); + } + Ok(Self { + compact: Arc::from(compact), + value: parsed, + }) + } + + /// Borrows the compact canonical JSON object. + pub fn as_str(&self) -> &str { + &self.compact + } + + /// Borrows the parsed object. + pub const fn value(&self) -> &serde_json::Value { + &self.value + } +} + +/// Parses structured JSON while rejecting duplicate object members and compiled-limit excess. +pub fn parse_json_without_duplicate_members( + value: &str, +) -> Result { + parse_json_without_duplicate_members_bounded( + value, + JsonParseLimits { + maximum_nodes: crate::MAX_STRUCTURED_NODES, + maximum_container_entries: crate::MAX_OBSERVED_CONTAINER_ENTRIES, + }, + ) +} + +/// Caller-labeled ceilings for structured JSON parsing. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct JsonParseLimits { + /// Maximum total JSON values admitted. + pub maximum_nodes: u64, + /// Maximum members or elements admitted in any one container. + pub maximum_container_entries: u64, +} + +/// Parses structured JSON with caller-labeled node and container-entry ceilings. +pub fn parse_json_without_duplicate_members_bounded( + value: &str, + limits: JsonParseLimits, +) -> Result { + let raw = serde_json::from_str::>(value)?; + let mut budget = JsonParseBudget { + remaining_nodes: limits.maximum_nodes, + maximum_container_entries: limits.maximum_container_entries, + }; + parse_raw_json(raw.get(), 0, &mut budget) +} + +struct JsonParseBudget { + remaining_nodes: u64, + maximum_container_entries: u64, +} + +impl JsonParseBudget { + fn admit_node(&mut self) -> Result<(), serde_json::Error> { + self.remaining_nodes = self.remaining_nodes.checked_sub(1).ok_or_else(|| { + serde_json::Error::custom("JSON node count exceeds the effective ceiling") + })?; + Ok(()) + } + + fn admits_container_entries(&self, entries: u64) -> bool { + entries <= self.maximum_container_entries + } +} + +fn parse_raw_json( + value: &str, + depth: u32, + budget: &mut JsonParseBudget, +) -> Result { + budget.admit_node()?; + match value.trim_start().as_bytes().first() { + Some(b'{') if depth < crate::MAX_STRUCTURED_DEPTH => { + deserialize_seed(value, DuplicateAwareObject { depth, budget }) + } + Some(b'[') if depth < crate::MAX_STRUCTURED_DEPTH => { + deserialize_seed(value, DuplicateAwareArray { depth, budget }) + } + Some(b'{' | b'[') => Err(serde_json::Error::custom( + "JSON nesting depth exceeds the compiled ceiling", + )), + _ => serde_json::from_str(value), + } +} + +fn deserialize_seed(value: &str, seed: Seed) -> Result +where + for<'de> Seed: DeserializeSeed<'de, Value = serde_json::Value>, +{ + let mut deserializer = serde_json::Deserializer::from_str(value); + let parsed = seed.deserialize(&mut deserializer)?; + deserializer.end()?; + Ok(parsed) +} + +struct DuplicateAwareObject<'a> { + depth: u32, + budget: &'a mut JsonParseBudget, +} + +impl<'de> DeserializeSeed<'de> for DuplicateAwareObject<'_> { + type Value = serde_json::Value; + + fn deserialize( + self, + deserializer: Deserializer, + ) -> Result + where + Deserializer: serde::Deserializer<'de>, + { + deserializer.deserialize_map(DuplicateAwareObjectVisitor { + depth: self.depth, + budget: self.budget, + }) + } +} + +struct DuplicateAwareArray<'a> { + depth: u32, + budget: &'a mut JsonParseBudget, +} + +impl<'de> DeserializeSeed<'de> for DuplicateAwareArray<'_> { + type Value = serde_json::Value; + + fn deserialize( + self, + deserializer: Deserializer, + ) -> Result + where + Deserializer: serde::Deserializer<'de>, + { + deserializer.deserialize_seq(DuplicateAwareArrayVisitor { + depth: self.depth, + budget: self.budget, + }) + } +} + +struct DuplicateAwareObjectVisitor<'a> { + depth: u32, + budget: &'a mut JsonParseBudget, +} + +impl<'de> Visitor<'de> for DuplicateAwareObjectVisitor<'_> { + type Value = serde_json::Value; + + fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("JSON without duplicate object members") + } + + fn visit_map(self, mut object: Access) -> Result + where + Access: MapAccess<'de>, + { + let mut values = std::collections::BTreeMap::new(); + let mut entries = 0_u64; + while let Some(name) = object.next_key::()? { + entries = entries + .checked_add(1) + .ok_or_else(|| Access::Error::custom("JSON container entry count overflowed"))?; + if !self.budget.admits_container_entries(entries) { + return Err(Access::Error::custom( + "JSON container entries exceed the effective ceiling", + )); + } + if values.contains_key(&name) { + return Err(Access::Error::custom("duplicate JSON object member")); + } + let raw = object.next_value::>()?; + let value = parse_raw_json(raw.get(), self.depth + 1, self.budget) + .map_err(Access::Error::custom)?; + values.insert(name, value); + } + Ok(serde_json::Value::Object(values.into_iter().collect())) + } +} + +struct DuplicateAwareArrayVisitor<'a> { + depth: u32, + budget: &'a mut JsonParseBudget, +} + +impl<'de> Visitor<'de> for DuplicateAwareArrayVisitor<'_> { + type Value = serde_json::Value; + + fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("JSON array without duplicate object members") + } + + fn visit_seq(self, mut sequence: Access) -> Result + where + Access: SeqAccess<'de>, + { + let mut values = Vec::new(); + let mut entries = 0_u64; + while let Some(raw) = sequence.next_element::>()? { + entries = entries + .checked_add(1) + .ok_or_else(|| Access::Error::custom("JSON container entry count overflowed"))?; + if !self.budget.admits_container_entries(entries) { + return Err(Access::Error::custom( + "JSON container entries exceed the effective ceiling", + )); + } + values.push( + parse_raw_json(raw.get(), self.depth + 1, self.budget) + .map_err(Access::Error::custom)?, + ); + } + Ok(serde_json::Value::Array(values)) + } +} + +/// Stable visible-part selector for repeated digest uses. +#[derive(Clone, Debug, Eq, Hash, PartialEq)] +pub struct VisiblePartSelector(Arc); + +impl VisiblePartSelector { + /// Admits one bounded opaque ASCII selector. + pub fn try_new(value: impl Into>) -> Result { + let value = value.into(); + if value.is_empty() + || value.len() > MAX_NAME_BYTES + || !value + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'_' | b'-')) + { + return Err(RegistryValueError::Selector); + } + Ok(Self(value)) + } + + /// Borrows the opaque selector. + pub fn as_str(&self) -> &str { + &self.0 + } +} + +/// Checked opaque continuation cursor returned by one bounded read. +#[derive(Clone, Debug, Eq, Hash, PartialEq)] +pub struct ReadContinuationCursor(Arc); + +impl ReadContinuationCursor { + /// Admits one bounded control-free restart-ephemeral cursor. + pub fn try_new(value: impl Into>) -> Result { + let value = value.into(); + if value.is_empty() + || value.len() > MAX_CONTINUATION_CURSOR_BYTES + || value.contains('\0') + || value.chars().any(char::is_control) + { + return Err(RegistryValueError::Continuation); + } + Ok(Self(value)) + } + + /// Borrows the opaque cursor spelling. + pub fn as_str(&self) -> &str { + &self.0 + } + + /// Returns the owned opaque cursor spelling. + pub fn into_string(self) -> String { + self.0.to_string() + } +} + +/// Closed construction failure for provider-neutral checked values. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum RegistryValueError { + /// Invalid digest spelling. + Digest, + /// Invalid declared media type. + DeclaredMediaType, + /// Invalid display basename. + DisplayFilename, + /// Invalid registry name. + Name, + /// Invalid reader revision. + Revision, + /// Invalid object-rooted JSON Schema. + Schema, + /// Invalid processor metadata object. + Metadata, + /// Invalid visible-part selector. + Selector, + /// Invalid read-continuation cursor. + Continuation, +} + +impl fmt::Display for RegistryValueError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str(match self { + Self::Digest => "file digest is invalid", + Self::DeclaredMediaType => "declared media type is invalid", + Self::DisplayFilename => "display filename is invalid", + Self::Name => "registry name is invalid", + Self::Revision => "reader revision is invalid", + Self::Schema => "view schema is invalid", + Self::Metadata => "processor metadata is invalid", + Self::Selector => "visible-part selector is invalid", + Self::Continuation => "read continuation cursor is invalid", + }) + } +} + +impl Error for RegistryValueError {} + +#[cfg(test)] +mod tests { + use super::*; + + fn nested_metadata(depth: u32) -> String { + let depth = usize::try_from(depth).expect("compiled depth ceiling fits usize"); + format!("{{\"value\":{}0{}}}", "[".repeat(depth), "]".repeat(depth)) + } + + fn flat_array(entries: usize) -> String { + format!("[{}]", vec!["0"; entries].join(",")) + } + + #[test] + fn bounded_parser_rejects_nodes_during_deserialization() { + let input = flat_array(2); + + let outcome = parse_json_without_duplicate_members_bounded( + &input, + JsonParseLimits { + maximum_nodes: 2, + maximum_container_entries: 2, + }, + ); + + assert!(outcome.is_err()); + } + + #[test] + fn bounded_parser_rejects_container_entries_during_deserialization() { + let input = flat_array(2); + + let outcome = parse_json_without_duplicate_members_bounded( + &input, + JsonParseLimits { + maximum_nodes: 3, + maximum_container_entries: 1, + }, + ); + + assert!(outcome.is_err()); + } + + #[test] + fn canonical_media_types_reject_parameters_and_uppercase() { + assert!(CanonicalMediaType::from_str("text/plain").is_ok()); + assert!(CanonicalMediaType::from_str("text/plain; charset=utf-8").is_err()); + assert!(CanonicalMediaType::from_str("Text/plain").is_err()); + assert!(CanonicalMediaType::from_str("!text/plain").is_err()); + assert!(CanonicalMediaType::from_str("text/!plain").is_err()); + assert!(CanonicalMediaType::from_str(&format!("{}/b", "a".repeat(128))).is_err()); + assert!(CanonicalMediaType::from_str(&format!("a/{}", "b".repeat(128))).is_err()); + } + + #[test] + fn display_filenames_preserve_blob_valid_control_characters() { + let newline_filename = "line\nbreak.txt"; + let delete_filename = "delete\u{7f}.txt"; + let newline = DisplayFilename::try_new(newline_filename) + .expect("blob-valid newline filename remains representable"); + let delete = DisplayFilename::try_new(delete_filename) + .expect("blob-valid delete-character filename remains representable"); + + assert_eq!(newline.as_str(), newline_filename); + assert_eq!(delete.as_str(), delete_filename); + assert_eq!( + DisplayFilename::try_new("null\0.txt"), + Err(RegistryValueError::DisplayFilename) + ); + } + + #[test] + fn metadata_rejects_nesting_above_the_compiled_ceiling() { + let input = nested_metadata(crate::MAX_STRUCTURED_DEPTH); + + let outcome = BoundedMetadata::try_new(&input); + + assert_eq!(outcome, Err(RegistryValueError::Metadata)); + } + + #[test] + fn metadata_is_parsed_as_data_and_canonically_escaped() { + let compact_input = r#"{"note":""}"#; + let metadata = BoundedMetadata::try_new(compact_input) + .expect("synthetic injection-shaped JSON remains inert data"); + + assert_eq!(metadata.as_str(), compact_input); + assert_eq!( + metadata.value()["note"], + serde_json::Value::String(String::from("")) + ); + } + + #[test] + fn metadata_rejects_duplicate_object_members() { + let outcome = BoundedMetadata::try_new(r#"{"kind":"safe","kind":"attacker"}"#); + + assert_eq!(outcome, Err(RegistryValueError::Metadata)); + } + + #[test] + fn metadata_preserves_arbitrary_precision_numbers() { + let input = r#"{"n":123456789012345678901234567890}"#; + let metadata = BoundedMetadata::try_new(input) + .expect("arbitrary-precision fixture remains valid metadata"); + + assert_eq!(metadata.as_str(), input); + } + + #[test] + fn metadata_preserves_reserved_number_key_objects() { + let input = r#"{"$serde_json::private::Number":"1"}"#; + let metadata = BoundedMetadata::try_new(input) + .expect("the reserved spelling remains an ordinary object member"); + + assert_eq!(metadata.as_str(), input); + assert_eq!( + metadata.value()["$serde_json::private::Number"], + serde_json::Value::String(String::from("1")) + ); + } + + #[test] + fn metadata_preserves_nested_reserved_number_key_objects() { + let input = r#"{"nested":{"$serde_json::private::Number":"1","tail":true}}"#; + let metadata = BoundedMetadata::try_new(input) + .expect("nested reserved spelling remains ordinary object data"); + + assert_eq!(metadata.as_str(), input); + assert_eq!( + metadata.value()["nested"]["$serde_json::private::Number"], + serde_json::Value::String(String::from("1")) + ); + } + + #[test] + fn metadata_canonicalizes_object_members_lexically() { + let input = r#"{"z":0,"a":{"z":0,"a":1}}"#; + let expected = r#"{"a":{"a":1,"z":0},"z":0}"#; + let metadata = BoundedMetadata::try_new(input) + .expect("unordered object fixture remains valid metadata"); + + assert_eq!(metadata.as_str(), expected); + } + + #[test] + fn schema_rejects_nested_duplicate_object_members() { + let outcome = CanonicalJsonObjectSchema::try_new( + r#"{"type":"object","properties":{"value":{"type":"string","type":"number"}}}"#, + ); + + assert_eq!(outcome, Err(RegistryValueError::Schema)); + } +} diff --git a/crates/file-media-runtime/tests/registry_conformance.rs b/crates/file-media-runtime/tests/registry_conformance.rs new file mode 100644 index 0000000000..5c6b0a0ed7 --- /dev/null +++ b/crates/file-media-runtime/tests/registry_conformance.rs @@ -0,0 +1,1617 @@ +#![allow( + clippy::expect_used, + clippy::panic, + reason = "conformance fixtures use explicit construction and outcome expectations" +)] + +use std::{future::Future, num::NonZeroU64, str::FromStr}; + +use signalbox_file_media_runtime::{ + AttachmentKind, CancellationSignal, CanonicalJsonObjectSchema, CanonicalMediaType, + DeclaredMediaType, FileDigest, FileInspection, FileMediaCeilings, FileMediaFailure, + FileMediaProcessor, FileMediaProcessorFuture, FileMediaProviderDeclaration, + FileMediaProviderReadRequest, FileMediaProviderValidationRequest, FileMediaRegistry, + FileReadInput, FileReadRequest, FileReaderName, FileReaderProviderName, FileReaderRevision, + FileUse, InspectionRequest, MAX_VALIDATION_RANGES, MAX_VALIDATION_SOURCE_BYTES, NeverCancelled, + ProbeDeclaration, ProbeDeclarationInput, ProbeStrength, ProcessorFailure, ProcessorIsolation, + ProcessorProbeOutput, ProcessorReadOutput, ProcessorValidationOutput, ReadAccessPattern, + ReadViewBounds, ReadViewDeclaration, ReadViewName, ReaderDeclaration, ReaderDeclarationInput, + ReaderIdentity, ReasonCode, SourceReadError, SourceReadFuture, StreamingTextFallback, + ValidationDeclaration, ValidationEvidence, VerifiedBlobSource, +}; + +const SYNTHETIC_MEDIA_TYPE: &str = "application/x-signalbox-synthetic"; +const OTHER_SYNTHETIC_MEDIA_TYPE: &str = "application/x-signalbox-other"; +const SYNTHETIC_SIGNATURE: &[u8] = b"SYN1"; +const SYNTHETIC_BODY: &[u8] = b"SYN1 generated fixture bytes"; +const TEXT_VIEW_NAME: &str = "body_text"; +const STRUCTURED_VIEW_NAME: &str = "body_structure"; +const MALFORMED_REASON: &str = "invalid_structure"; +const EMPTY_OPTIONS_SCHEMA: &str = r#"{"additionalProperties":false,"type":"object"}"#; + +struct MemorySource { + digest: FileDigest, + bytes: Vec, +} + +impl MemorySource { + fn synthetic() -> Self { + Self { + digest: FileDigest::from_bytes([0x5a; 32]), + bytes: SYNTHETIC_BODY.to_vec(), + } + } +} + +impl VerifiedBlobSource for MemorySource { + fn digest(&self) -> FileDigest { + self.digest + } + + fn byte_length(&self) -> NonZeroU64 { + NonZeroU64::new(self.bytes.len() as u64).expect("the synthetic fixture is nonempty") + } + + fn read_range(&self, offset: u64, length: NonZeroU64) -> SourceReadFuture<'_> { + Box::pin(async move { + let start = usize::try_from(offset).map_err(|_| SourceReadError::RangeOutOfBounds)?; + let length = + usize::try_from(length.get()).map_err(|_| SourceReadError::RangeOutOfBounds)?; + let end = start + .checked_add(length) + .ok_or(SourceReadError::RangeOutOfBounds)?; + self.bytes + .get(start..end) + .map(<[u8]>::to_vec) + .ok_or(SourceReadError::RangeOutOfBounds) + }) + } +} + +#[derive(Clone, Copy)] +enum ValidationBehavior { + Valid, + ValidWithEnvelope { source_bytes: u64, ranges: u32 }, + OversizedMetadata, + MalformedMetadata, +} + +#[derive(Clone, Copy)] +enum ReadBehavior { + Text, + InvalidViewArguments, + SourceTooLarge, + OversizedText, + MalformedStructured, + DuplicateStructuredMember, + CanonicalizedStructuredOverflow, + ContradictoryContinuation, +} + +struct SyntheticProcessor { + validation: ValidationBehavior, + read: ReadBehavior, +} + +impl SyntheticProcessor { + const fn valid_text() -> Self { + Self { + validation: ValidationBehavior::Valid, + read: ReadBehavior::Text, + } + } +} + +impl FileMediaProcessor for SyntheticProcessor { + fn probe<'a>( + &'a self, + _reader: &'a ReaderIdentity, + source: &'a dyn VerifiedBlobSource, + _cancellation: &'a dyn CancellationSignal, + ) -> FileMediaProcessorFuture<'a, ProcessorProbeOutput> { + Box::pin(async move { + let prefix_length = NonZeroU64::new(SYNTHETIC_SIGNATURE.len() as u64) + .ok_or(ProcessorFailure::Failed)?; + let prefix = source + .read_range(0, prefix_length) + .await + .map_err(|_| ProcessorFailure::Failed)?; + if prefix == SYNTHETIC_SIGNATURE { + Ok(ProcessorProbeOutput::Candidate { + media_type: String::from(SYNTHETIC_MEDIA_TYPE), + strength: ProbeStrength::Strong, + }) + } else { + Ok(ProcessorProbeOutput::NoMatch) + } + }) + } + + fn validate<'a>( + &'a self, + _reader: &'a ReaderIdentity, + request: FileMediaProviderValidationRequest, + _source: &'a dyn VerifiedBlobSource, + _cancellation: &'a dyn CancellationSignal, + ) -> FileMediaProcessorFuture<'a, ProcessorValidationOutput> { + Box::pin(async move { + let expected_envelope = match self.validation { + ValidationBehavior::ValidWithEnvelope { + source_bytes, + ranges, + } => (source_bytes, ranges), + ValidationBehavior::Valid + | ValidationBehavior::OversizedMetadata + | ValidationBehavior::MalformedMetadata => { + (MAX_VALIDATION_SOURCE_BYTES, MAX_VALIDATION_RANGES) + } + }; + if (request.maximum_source_bytes, request.maximum_ranges) != expected_envelope { + return Err(ProcessorFailure::Failed.into()); + } + let metadata_json = match self.validation { + ValidationBehavior::Valid | ValidationBehavior::ValidWithEnvelope { .. } => { + String::from(r#"{"synthetic":true}"#) + } + ValidationBehavior::OversizedMetadata => { + format!(r#"{{"filler":"{}"}}"#, "x".repeat(16_385)) + } + ValidationBehavior::MalformedMetadata => { + String::from(r#""#) + } + }; + Ok(ProcessorValidationOutput::Validated { + media_type: request.media_type.to_string(), + evidence: request.evidence, + metadata_json, + }) + }) + } + + fn read<'a>( + &'a self, + _reader: &'a ReaderIdentity, + _request: FileMediaProviderReadRequest, + _source: &'a dyn VerifiedBlobSource, + _cancellation: &'a dyn CancellationSignal, + ) -> FileMediaProcessorFuture<'a, ProcessorReadOutput> { + Box::pin(async move { + Ok(match self.read { + ReadBehavior::Text => ProcessorReadOutput::Text { + body: String::from("synthetic admitted text"), + truncated: false, + cursor: None, + }, + ReadBehavior::InvalidViewArguments => ProcessorReadOutput::InvalidViewArguments, + ReadBehavior::SourceTooLarge => ProcessorReadOutput::SourceTooLarge { + maximum_bytes: 1_024, + }, + ReadBehavior::OversizedText => ProcessorReadOutput::Text { + body: "x".repeat(65), + truncated: false, + cursor: None, + }, + ReadBehavior::MalformedStructured => ProcessorReadOutput::Structured { + body_json: String::from(r#"{"value":"