backend-hermetic-e2e.yml

name: Backend Hermetic E2E

on:
  pull_request:
    branches: main
  merge_group:
    types: [checks_requested]
  workflow_dispatch:

permissions:
  contents: read

concurrency:
  group: backend-hermetic-e2e-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: true

jobs:
  scope:
    name: Detect Hermetic Backend Scope
    runs-on: ubuntu-latest
    # The full-history fetch (fetch-depth: 0, required to resolve the PR base
    # live for the scope diff) can exceed two minutes on hosted runners for
    # large merge refs; the old 2-minute job cap killed the fetch mid-transfer
    # and the merge gate correctly failed closed on "scope cancelled" with no
    # code failure. Bound generously like the apt steps below: the cap is a
    # hang backstop, not a budget the legitimate path must fit under.
    timeout-minutes: 10
    outputs:
      applies: ${{ steps.scope.outputs.applies }}
    steps:
      - name: Checkout code
        uses: actions/checkout@v7
        with:
          fetch-depth: 0

      - name: Decide whether hermetic backend coverage applies
        id: scope
        shell: bash
        env:
          EVENT_NAME: ${{ github.event_name }}
          # Resolve the PR base live against the checkout (matching
          # detect-changes) rather than the event-payload base.sha, which can be
          # stale for a queued run (FC-stale-event-payload-diff-base, #10785).
          PR_BASE_REF: origin/${{ github.base_ref }}
          MERGE_GROUP_BASE_SHA: ${{ github.event.merge_group.base_sha }}
        run: |
          case "$EVENT_NAME" in
            pull_request) base_sha="$PR_BASE_REF" ;;
            merge_group) base_sha="$MERGE_GROUP_BASE_SHA" ;;
            workflow_dispatch)
              echo "applies=true" >> "$GITHUB_OUTPUT"
              exit 0
              ;;
            *)
              echo "Unsupported event for hermetic backend scope: $EVENT_NAME" >&2
              exit 1
              ;;
          esac

          if [[ -z "$base_sha" ]] || ! git cat-file -e "${base_sha}^{commit}"; then
            echo "Cannot resolve hermetic backend scope base: ${base_sha:-<empty>}" >&2
            exit 1
          fi

          changed_files="$(git diff --name-only "$base_sha"...HEAD)"
          printf '%s\n' "$changed_files"
          if grep -Eq '^(backend/|package\.json$|package-lock\.json$|\.github/workflows/backend-hermetic-e2e\.yml$)' <<<"$changed_files"; then
            echo "applies=true" >> "$GITHUB_OUTPUT"
          else
            echo "applies=false" >> "$GITHUB_OUTPUT"
          fi

  hermetic-e2e:
    name: Hermetic Backend E2E
    runs-on: ubuntu-latest
    timeout-minutes: 10
    needs: scope
    if: needs.scope.outputs.applies == 'true'

    steps:
      - name: Checkout code
        uses: actions/checkout@v7

      - name: Set up Python
        uses: actions/setup-python@v6
        with:
          python-version-file: backend/.python-version

      - name: Set up uv
        # astral-sh/setup-uv v7, pinned to an immutable commit for CI supply-chain stability.
        uses: astral-sh/setup-uv@ecd24dd710f2fb0dca1693a67af11fc4a5c5ec84
        with:
          # Exact version resolves locally; an unset version means "latest",
          # which fetches the astral-sh/versions manifest and has hard-failed
          # CI on that network call. Matches backend/Dockerfile UV_VERSION.
          version: "0.11.13"
          enable-cache: true
          cache-dependency-glob: backend/pylock*.toml

      - name: Install backend and e2e dependencies
        working-directory: backend
        run: uv pip sync pylock.toml --system

      - name: Restore tokenizer cache
        uses: actions/cache@v6
        with:
          path: ${{ runner.temp }}/tiktoken-cache
          key: tiktoken-${{ runner.os }}-${{ hashFiles('backend/pylock.toml') }}

      - name: Prewarm tokenizer cache
        env:
          TIKTOKEN_CACHE_DIR: ${{ runner.temp }}/tiktoken-cache
        run: python backend/scripts/prewarm_tiktoken_cache.py

      - name: Run hermetic backend e2e harness
        env:
          E2E_PYTEST_TIMEOUT: 120s
          TIKTOKEN_CACHE_DIR: ${{ runner.temp }}/tiktoken-cache
        run: bash backend/testing/e2e/run.sh -q --tb=short

  listen-pusher-stack-gauntlet:
    name: Listen Pusher Stack Gauntlet
    runs-on: ubuntu-latest
    timeout-minutes: 20
    needs: scope
    if: needs.scope.outputs.applies == 'true'

    steps:
      - name: Checkout code
        uses: actions/checkout@v7

      - name: Set up Python
        uses: actions/setup-python@v6
        with:
          python-version-file: backend/.python-version

      - name: Set up uv
        uses: astral-sh/setup-uv@ecd24dd710f2fb0dca1693a67af11fc4a5c5ec84
        with:
          # Exact version resolves locally; an unset version means "latest",
          # which fetches the astral-sh/versions manifest and has hard-failed
          # CI on that network call. Matches backend/Dockerfile UV_VERSION.
          version: "0.11.13"
          enable-cache: true
          cache-dependency-glob: backend/pylock*.toml

      - name: Install backend dependencies in the gauntlet virtualenv
        working-directory: backend
        run: |
          uv venv .venv
          uv pip sync pylock.toml --python .venv/bin/python

      - name: Set up Node.js for the Firestore emulator
        uses: actions/setup-node@v7
        with:
          node-version: '22'
          cache: npm
          cache-dependency-path: package-lock.json

      - name: Install pinned Firebase CLI dependencies
        run: |
          for attempt in 1 2 3; do
            if npm ci --ignore-scripts --prefer-offline --no-audit --fund=false; then
              exit 0
            fi
            if [[ "$attempt" == 3 ]]; then
              echo "Pinned Firebase CLI dependency install failed after 3 attempts." >&2
              npm cache verify || true
              exit 1
            fi
            echo "npm ci failed on attempt $attempt/3; verifying cache before bounded retry." >&2
            npm cache verify
            sleep "$attempt"
          done

      - name: Set up Java for the Firestore emulator
        uses: actions/setup-java@v5
        with:
          distribution: temurin
          java-version: '21'

      - name: Install Redis server
        # The harness spawns its own `redis-server` binary per test on an
        # ephemeral port (see backend/testing/*/run.py), so this needs the
        # actual package installed, not a `services:` container.
        #
        # apt against the default Azure-mirror apt source on hosted runners
        # can silently stall (connects but never completes the transfer)
        # instead of failing, which previously let this step hang until the
        # job's 20-minute timeout killed it with an opaque "operation was
        # canceled". Bound each apt network attempt and retry a few times so
        # a stalled mirror fails over quickly, and cap the step itself as a
        # backstop.
        timeout-minutes: 5
        run: |
          sudo apt-get -o Acquire::Retries=3 -o Acquire::http::Timeout=10 -o Acquire::https::Timeout=10 update
          sudo apt-get -o Acquire::Retries=3 -o Acquire::http::Timeout=10 -o Acquire::https::Timeout=10 install --yes redis-server

      - name: Run listen to pusher stack gauntlet
        run: npm run test:listen-pusher-stack:emulator -- --state-dir "$RUNNER_TEMP/listen-pusher-stack"

      - name: Show listen gauntlet process diagnostics on failure
        if: failure()
        run: |
          state_dir="$RUNNER_TEMP/listen-pusher-stack"
          if [[ -d "$state_dir" ]]; then
            find "$state_dir" -type f \( -name '*.log' -o -name '*.jsonl' \) -print -exec tail -n 160 {} \;
          else
            echo "Listen gauntlet state directory was not retained."
          fi

  sync-cloud-tasks-stack-gauntlet:
    name: Sync Cloud Tasks Stack Gauntlet
    runs-on: ubuntu-latest
    timeout-minutes: 20
    needs: scope
    if: needs.scope.outputs.applies == 'true'

    steps:
      - name: Checkout code
        uses: actions/checkout@v7

      - name: Set up Python
        uses: actions/setup-python@v6
        with:
          python-version-file: backend/.python-version

      - name: Set up uv
        uses: astral-sh/setup-uv@ecd24dd710f2fb0dca1693a67af11fc4a5c5ec84
        with:
          # Exact version resolves locally; an unset version means "latest",
          # which fetches the astral-sh/versions manifest and has hard-failed
          # CI on that network call. Matches backend/Dockerfile UV_VERSION.
          version: "0.11.13"
          enable-cache: true
          cache-dependency-glob: backend/pylock*.toml

      - name: Install backend dependencies in the gauntlet virtualenv
        working-directory: backend
        run: |
          uv venv .venv
          uv pip sync pylock.toml --python .venv/bin/python

      - name: Set up Node.js for the Firestore emulator
        uses: actions/setup-node@v7
        with:
          node-version: '22'
          cache: npm
          cache-dependency-path: package-lock.json

      - name: Install pinned Firebase CLI dependencies
        run: |
          for attempt in 1 2 3; do
            if npm ci --ignore-scripts --prefer-offline --no-audit --fund=false; then
              exit 0
            fi
            if [[ "$attempt" == 3 ]]; then
              echo "Pinned Firebase CLI dependency install failed after 3 attempts." >&2
              npm cache verify || true
              exit 1
            fi
            echo "npm ci failed on attempt $attempt/3; verifying cache before bounded retry." >&2
            npm cache verify
            sleep "$attempt"
          done

      - name: Set up Java for the Firestore emulator
        uses: actions/setup-java@v5
        with:
          distribution: temurin
          java-version: '21'

      - name: Install Redis server
        # The harness spawns its own `redis-server` binary per test on an
        # ephemeral port (see backend/testing/*/run.py), so this needs the
        # actual package installed, not a `services:` container.
        #
        # apt against the default Azure-mirror apt source on hosted runners
        # can silently stall (connects but never completes the transfer)
        # instead of failing, which previously let this step hang until the
        # job's 20-minute timeout killed it with an opaque "operation was
        # canceled". Bound each apt network attempt and retry a few times so
        # a stalled mirror fails over quickly, and cap the step itself as a
        # backstop.
        timeout-minutes: 5
        run: |
          sudo apt-get -o Acquire::Retries=3 -o Acquire::http::Timeout=10 -o Acquire::https::Timeout=10 update
          sudo apt-get -o Acquire::Retries=3 -o Acquire::http::Timeout=10 -o Acquire::https::Timeout=10 install --yes redis-server

      - name: Run Sync Cloud Tasks stack gauntlet
        run: npm run test:sync-cloud-tasks-stack:emulator

  replay-harness-phase0a-gauntlet:
    name: Replay Harness Phase 0A Feasibility (advisory)
    runs-on: ubuntu-latest
    timeout-minutes: 20
    needs: scope
    if: needs.scope.outputs.applies == 'true'

    steps:
      - name: Checkout code
        uses: actions/checkout@v7

      - name: Set up Python
        uses: actions/setup-python@v6
        with:
          python-version-file: backend/.python-version

      - name: Set up uv
        uses: astral-sh/setup-uv@ecd24dd710f2fb0dca1693a67af11fc4a5c5ec84
        with:
          # Exact version resolves locally; an unset version means "latest",
          # which fetches the astral-sh/versions manifest and has hard-failed
          # CI on that network call. Matches backend/Dockerfile UV_VERSION.
          version: "0.11.13"
          enable-cache: true
          cache-dependency-glob: backend/pylock*.toml

      - name: Install backend dependencies in the gauntlet virtualenv
        working-directory: backend
        run: |
          uv venv .venv
          uv pip sync pylock.toml --python .venv/bin/python

      - name: Set up Node.js for the Firestore emulator
        uses: actions/setup-node@v7
        with:
          node-version: '22'
          cache: npm
          cache-dependency-path: package-lock.json

      - name: Install pinned Firebase CLI dependencies
        run: npm ci --ignore-scripts

      - name: Set up Java for the Firestore emulator
        uses: actions/setup-java@v5
        with:
          distribution: temurin
          java-version: '21'

      - name: Install Redis server
        # The harness spawns its own `redis-server` binary per test on an
        # ephemeral port (see backend/testing/*/run.py), so this needs the
        # actual package installed, not a `services:` container.
        #
        # apt against the default Azure-mirror apt source on hosted runners
        # can silently stall (connects but never completes the transfer)
        # instead of failing, which previously let this step hang until the
        # job's 20-minute timeout killed it with an opaque "operation was
        # canceled". Bound each apt network attempt and retry a few times so
        # a stalled mirror fails over quickly, and cap the step itself as a
        # backstop.
        timeout-minutes: 5
        run: |
          sudo apt-get -o Acquire::Retries=3 -o Acquire::http::Timeout=10 -o Acquire::https::Timeout=10 update
          sudo apt-get -o Acquire::Retries=3 -o Acquire::http::Timeout=10 -o Acquire::https::Timeout=10 install --yes redis-server

      - name: Run Phase 0A replay harness feasibility experiment
        env:
          # Fixed state root so failed-job evidence (logs/attestations) is retained
          # at a known path for the failure step below.
          OMI_REPLAY_STATE_ROOT: ${{ runner.temp }}/omi-replay-harness
        run: npm run test:replay-harness-phase0a:emulator

      - name: Show replay harness logs on failure
        if: failure()
        run: |
          state_root="$RUNNER_TEMP/omi-replay-harness"
          if [[ -d "$state_root" ]]; then
            find "$state_root" -type f \( -name '*.log' -o -name 'attestation.json' \) -print -exec sh -c 'echo "===== $1 ====="; tail -n 160 "$1"' _ {} \;
          else
            echo "Replay harness state directory was not retained."
          fi

  merge-gate:
    name: Backend Hermetic Merge Gate
    runs-on: ubuntu-latest
    timeout-minutes: 2
    # `!cancelled()` rather than `always()`: with cancel-in-progress above, a
    # superseding push cancels this run mid-flight, and under `always()` the
    # gate still executed against `cancelled` upstream results and reported a
    # hard red with no code cause (3 of the last 4 re-run-to-green cycles on
    # this workflow were exactly that). A cancelled run's verdict belongs to
    # the superseding run, so the gate must go neutral, not red.
    if: ${{ !cancelled() }}
    needs: [scope, hermetic-e2e, listen-pusher-stack-gauntlet, sync-cloud-tasks-stack-gauntlet]
    steps:
      - name: Fail closed on selected hermetic checks
        env:
          SCOPE_RESULT: ${{ needs.scope.result }}
          SCOPE_APPLIES: ${{ needs.scope.outputs.applies }}
          HERMETIC_E2E_RESULT: ${{ needs.hermetic-e2e.result }}
          LISTEN_PUSHER_RESULT: ${{ needs.listen-pusher-stack-gauntlet.result }}
          SYNC_CLOUD_TASKS_RESULT: ${{ needs.sync-cloud-tasks-stack-gauntlet.result }}
        run: |
          # Belt to the `!cancelled()` suspenders: a partially-cancelled run
          # (one upstream job cancelled without the gate itself being
          # cancelled) is still superseded work, not a failure.
          for result_name in SCOPE_RESULT HERMETIC_E2E_RESULT LISTEN_PUSHER_RESULT SYNC_CLOUD_TASKS_RESULT; do
            if [[ "${!result_name}" == 'cancelled' ]]; then
              echo "$result_name was cancelled by a superseding run; deferring to that run's gate."
              exit 0
            fi
          done

          if [[ "$SCOPE_RESULT" != 'success' ]]; then
            echo "Hermetic backend scope failed with result: $SCOPE_RESULT" >&2
            exit 1
          fi

          case "$SCOPE_APPLIES" in
            true) required_result='success' ;;
            false) required_result='skipped' ;;
            *)
              echo "Hermetic backend scope produced an invalid result: ${SCOPE_APPLIES:-<empty>}" >&2
              exit 1
              ;;
          esac

          failed=0
          for result_name in HERMETIC_E2E_RESULT LISTEN_PUSHER_RESULT SYNC_CLOUD_TASKS_RESULT; do
            result="${!result_name}"
            if [[ "$result" != "$required_result" ]]; then
              echo "$result_name must be $required_result, got ${result:-<empty>}" >&2
              failed=1
            fi
          done
          exit "$failed"