fix(deploy): chown read-only-rootfs tmpfs in root-init; drop podman-r… #123
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Backend CI and Deploy | |
| on: | |
| push: | |
| branches: | |
| - main | |
| # Allowlist the backend image's actual inputs so unrelated changes (heartbeat, docs, | |
| # the other deploy workflows) can't rebuild/restart the backend. The image is built | |
| # solely from Main/backend/ (Dockerfile + that build context); the workflow file is | |
| # included so its own edits still ship. Previously a broad paths-ignore let any | |
| # non-heartbeat/docs change (e.g. a concierge workflow edit) bounce the backend. | |
| paths: | |
| - "Main/backend/**" | |
| - ".github/workflows/backend-deploy.yml" | |
| workflow_dispatch: | |
| env: | |
| BACKEND_DIR: Main/backend | |
| PYTHON_VERSION: "3.12" | |
| REGISTRY: ghcr.io | |
| # Shared droplet coordinates as repo-level Actions variables (see heartbeat/concierge), | |
| # so a host/user change is one repo setting, not an edit across three workflows. | |
| DROPLET_HOST: ${{ vars.DROPLET_HOST }} | |
| DROPLET_USER: ${{ vars.DROPLET_USER }} | |
| SYSTEMD_UNIT: fingpt-api | |
| jobs: | |
| build: | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| packages: write | |
| outputs: | |
| image: ${{ steps.meta.outputs.image }} | |
| sha_tag: ${{ steps.meta.outputs.sha_tag }} | |
| main_tag: ${{ steps.meta.outputs.main_tag }} | |
| latest_tag: ${{ steps.meta.outputs.latest_tag }} | |
| digest: ${{ steps.digest.outputs.digest }} | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 | |
| - name: Set up uv | |
| uses: astral-sh/setup-uv@caf0cab7a618c569241d31dcd442f54681755d39 # v3.2.4 | |
| - name: Install Python ${{ env.PYTHON_VERSION }} | |
| run: uv python install ${{ env.PYTHON_VERSION }} | |
| - name: Sync backend dependencies | |
| working-directory: ${{ env.BACKEND_DIR }} | |
| run: uv sync --frozen --python ${{ env.PYTHON_VERSION }} | |
| - name: Run Django system checks | |
| working-directory: ${{ env.BACKEND_DIR }} | |
| run: uv run python manage.py check | |
| - name: Verify deployment readiness | |
| working-directory: ${{ env.BACKEND_DIR }} | |
| run: uv run python verify_deployment.py | |
| - name: Compute image tags | |
| id: meta | |
| run: | | |
| REPO_SLUG=$(echo "${{ github.repository }}" | tr '[:upper:]' '[:lower:]') | |
| IMAGE_ID="${{ env.REGISTRY }}/${REPO_SLUG}-backend" | |
| echo "image=$IMAGE_ID" >> "$GITHUB_OUTPUT" | |
| echo "sha_tag=${IMAGE_ID}:${{ github.sha }}" >> "$GITHUB_OUTPUT" | |
| echo "main_tag=${IMAGE_ID}:main" >> "$GITHUB_OUTPUT" | |
| echo "latest_tag=${IMAGE_ID}:latest" >> "$GITHUB_OUTPUT" | |
| - name: Log in to GHCR | |
| uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0 | |
| with: | |
| registry: ${{ env.REGISTRY }} | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Build backend image | |
| run: | | |
| docker build \ | |
| --pull \ | |
| --file ${{ env.BACKEND_DIR }}/Dockerfile \ | |
| --tag ${{ steps.meta.outputs.sha_tag }} \ | |
| --tag ${{ steps.meta.outputs.main_tag }} \ | |
| --tag ${{ steps.meta.outputs.latest_tag }} \ | |
| ${{ env.BACKEND_DIR }} | |
| - name: Push backend image | |
| run: | | |
| docker push ${{ steps.meta.outputs.sha_tag }} | |
| docker push ${{ steps.meta.outputs.main_tag }} | |
| docker push ${{ steps.meta.outputs.latest_tag }} | |
| - name: Capture pushed image digest | |
| id: digest | |
| run: | | |
| DIGEST=$(docker inspect --format='{{index .RepoDigests 0}}' ${{ steps.meta.outputs.sha_tag }}) | |
| echo "digest=$DIGEST" >> "$GITHUB_OUTPUT" | |
| test: | |
| runs-on: ubuntu-latest | |
| needs: build | |
| permissions: | |
| contents: read | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 | |
| - name: Set up uv | |
| uses: astral-sh/setup-uv@caf0cab7a618c569241d31dcd442f54681755d39 # v3.2.4 | |
| - name: Install Python ${{ env.PYTHON_VERSION }} | |
| run: uv python install ${{ env.PYTHON_VERSION }} | |
| - name: Sync backend dependencies | |
| working-directory: ${{ env.BACKEND_DIR }} | |
| run: uv sync --frozen --python ${{ env.PYTHON_VERSION }} | |
| - name: Run backend tests | |
| working-directory: ${{ env.BACKEND_DIR }} | |
| run: uv run pytest tests -q | |
| deploy: | |
| runs-on: ubuntu-latest | |
| if: ${{ github.ref == 'refs/heads/main' }} | |
| # DEPLOY_SSH_KEY / GHCR_READ_TOKEN are ENVIRONMENT secrets (Production has a | |
| # deployment branch policy of main only), not repo-scoped: even a workflow | |
| # edited or dispatched on a non-main ref cannot release the prod SSH key. | |
| # This gates the secret itself, independently of the ref-gate above -- the | |
| # two layers fail separately (same stanza as the heartbeat/concierge jobs). | |
| environment: | |
| name: Production | |
| url: https://agenticfinsearch.org | |
| # Serialize deploys so two merges in quick succession can't land out of order, | |
| # leave the droplet on the older commit, or -- now that superseded images are | |
| # retired below -- rmi an in-flight peer's freshly pulled, still container-less | |
| # image mid-cutover (same stanza as the concierge/heartbeat deploy jobs). | |
| concurrency: | |
| group: backend-deploy | |
| cancel-in-progress: false | |
| needs: | |
| - build | |
| - test | |
| permissions: | |
| contents: read | |
| steps: | |
| - name: Determine deploy eligibility | |
| id: deploy-gate | |
| env: | |
| DEPLOY_SSH_KEY: ${{ secrets.DEPLOY_SSH_KEY }} | |
| GHCR_READ_TOKEN: ${{ secrets.GHCR_READ_TOKEN }} | |
| run: | | |
| set -euo pipefail | |
| if [[ -n "${DEPLOY_SSH_KEY:-}" && -n "${GHCR_READ_TOKEN:-}" ]]; then | |
| echo "enabled=true" >> "$GITHUB_OUTPUT" | |
| else | |
| echo "enabled=false" >> "$GITHUB_OUTPUT" | |
| fi | |
| - name: Deploy to Fedora droplet | |
| if: ${{ steps.deploy-gate.outputs.enabled == 'true' }} | |
| uses: appleboy/ssh-action@334f9259f2f8eb3376d33fa4c684fff373f2c2a6 # v0.1.10, SHA-pinned (holds the prod SSH key) | |
| env: | |
| REMOTE_IMAGE: ${{ needs.build.outputs.digest }} | |
| GHCR_USERNAME: ${{ github.repository_owner }} | |
| GHCR_TOKEN: ${{ secrets.GHCR_READ_TOKEN }} | |
| with: | |
| host: ${{ env.DROPLET_HOST }} | |
| username: ${{ env.DROPLET_USER }} | |
| key: ${{ secrets.DEPLOY_SSH_KEY }} | |
| port: 22 | |
| # The default 10m is now shared by the multi-GB image pull, the pre-cutover | |
| # FULL-boot gate (store build + collectstatic + gunicorn up + /health/ poll, | |
| # ~2-3min), cutover, and image retirement -- give it explicit headroom. | |
| command_timeout: 20m | |
| envs: REMOTE_IMAGE,GHCR_USERNAME,GHCR_TOKEN,SYSTEMD_UNIT | |
| script: | | |
| set -euo pipefail | |
| export XDG_RUNTIME_DIR=/run/user/$(id -u) | |
| podman login ${{ env.REGISTRY }} -u "$GHCR_USERNAME" -p "$GHCR_TOKEN" | |
| podman pull "$REMOTE_IMAGE" | |
| # Shared rootless network so the api container reaches redis by name. | |
| podman network exists fingpt-net || podman network create fingpt-net | |
| # Redis counter store (atomic incr/decr for agent budget + ratelimit). | |
| # Ephemeral cache: no persistence. Started idempotently (start-if-absent, not | |
| # --replace) BEFORE the pre-cutover gate so (a) the firewall self-test has redis | |
| # up on fingpt-net, and (b) an ABORTED deploy does not bounce the live redis / | |
| # wipe counters for the still-serving old container. | |
| # TRADE-OFF of start-if-absent: the running container stays FROZEN on whatever | |
| # redis:7-alpine image it was created from -- deploys never pick up redis image | |
| # updates (security patches included). To upgrade redis, run on the droplet: | |
| # podman pull docker.io/library/redis:7-alpine && podman rm -f fingpt-redis | |
| # then let the next deploy (or a re-run of this one) recreate it. Counters are | |
| # ephemeral by design, so the wipe is acceptable when done deliberately. | |
| if ! podman container exists fingpt-redis; then | |
| podman run -d --name fingpt-redis --restart=always \ | |
| --network fingpt-net \ | |
| --health-cmd 'redis-cli ping' --health-interval 10s --health-retries 5 \ | |
| docker.io/library/redis:7-alpine redis-server --save '' --appendonly no | |
| else | |
| podman start fingpt-redis >/dev/null 2>&1 || true | |
| fi | |
| # Pre-cutover FULL-BOOT validation: run the NEW image's REAL entrypoint on | |
| # fingpt-net BEFORE the systemctl restart's --replace kills the currently- | |
| # serving container. BOOT_CHECK_ONLY=1 (mode lives at the END of | |
| # Main/backend/entrypoint.sh, wrapping the final exec) exercises the byte- | |
| # identical PID1 path that runs at cutover: the full root-init (generate + | |
| # sentinel checks + nft load + transition-proof self-test + setpriv drop -- | |
| # so this SUBSUMES the old --egress-check-only gate, which stays in the | |
| # entrypoint for manual/local use), then the full app phase (store build, | |
| # collectstatic, Playwright verify), then explicit writable-surface probes | |
| # over the tmpfs set (these cover the fail-SOFT MCP-child $HOME writes a | |
| # /health/ poll can never catch), then backgrounds the image's own CMD and | |
| # polls /health/ INSIDE the container until gunicorn answers (hence no | |
| # --publish: the gate's :8000 stays netns-private). No positional args | |
| # after the image ref, so the Dockerfile CMD flows through byte-identical | |
| # (zero-drift property). On failure, abort so the OLD container keeps | |
| # serving (there is no auto-rollback). This closes the CI-invisible gap: | |
| # the test job is pytest-only and never touches nft/netns/--read-only. | |
| # The gate runs the IDENTICAL frozen cap/pids/read-only/tmpfs/memory flags | |
| # as the ExecStart below (cap-to-step map + tmpfs map at the override; | |
| # parity pinned in Main/backend/tests/test_dockerfile_nonroot.py), with | |
| # exactly one deliberate asymmetry: /app/runtime here is a THROWAWAY | |
| # tmpfs, never the real /home/deploy/fingpt/runtime bind mount. Mounting | |
| # the real dir pre-cutover is dangerous: :U would re-chown it under the | |
| # serving container, and a new-recipe store build would os.replace() the | |
| # shared DuckDB while the old container still serves it | |
| # (truthlayer/retrieve.py). Same isolation for redis: DB15 is a throwaway | |
| # keyspace so gate boot-time writes can never touch the serving | |
| # container's rate-limit/budget counters in DB0 (only settings_prod.py | |
| # and ops/egress_firewall.py consume REDIS_URL, and the firewall | |
| # self-test checks host:port reachability only, DB-agnostic). | |
| echo "Validating full boot (root-init + read-only rootfs + /health/) on the new image (pre-cutover)..." | |
| podman run --rm --cap-drop=ALL --cap-add=NET_ADMIN --cap-add=CHOWN --cap-add=SETUID --cap-add=SETGID --cap-add=SETPCAP --pids-limit=1024 --read-only --tmpfs=/tmp:rw,size=512m,mode=1777 --tmpfs=/app/staticfiles:rw,mode=0755 --tmpfs=/home/fingpt:rw,mode=0755 --tmpfs=/app/runtime:rw,size=512m --memory=1.7g --memory-swap=2g --network fingpt-net \ | |
| --env-file /home/deploy/fingpt/envs/.env.production \ | |
| --env REDIS_URL=redis://fingpt-redis:6379/15 \ | |
| --env BOOT_CHECK_ONLY=1 \ | |
| "$REMOTE_IMAGE" \ | |
| || { echo "ERROR: full-boot pre-cutover validation FAILED; aborting deploy (old container left serving)"; exit 1; } | |
| echo "Full-boot pre-cutover validation passed." | |
| # Record the currently-deployed image for manual rollback (no auto-rollback). | |
| PREV_IMAGE=$(podman inspect --format '{{.ImageName}}' "$SYSTEMD_UNIT" 2>/dev/null || echo "unknown") | |
| echo "Previous image (manual rollback target): $PREV_IMAGE" | |
| # Update systemd override to run the image we just pulled. | |
| # PID1 is now root-in-userns (no Dockerfile USER line); it loads the egress | |
| # firewall, chowns /app/runtime to fingpt, then setpriv-drops to uid1001 for | |
| # the app. The runtime bind mount still needs BOTH relabel suffixes on this | |
| # SELinux-enforcing Fedora host: | |
| # :U rootless podman recursively chowns the host dir to the in-container uid | |
| # (now root/0, since PID1 is root); entrypoint then re-chowns it to fingpt | |
| # so the dropped app user can write it (validated: spec Appendix A probe 3). | |
| # :Z relabels the (private) host dir to container_file_t (MAC/SELinux); | |
| # without it container_t is denied and the write is EACCES -- which | |
| # crash-looped the truth-layer store build the first time a deploy actually | |
| # wrote this mount (#322). Ownership alone is not enough; both are required. | |
| # Frozen capability set: --cap-drop=ALL, then add back ONLY the five caps the | |
| # root-init phase of Main/backend/entrypoint.sh actually uses. Each cap maps | |
| # 1:1 to a root-init step, so nothing in the set is discretionary: | |
| # NET_ADMIN nft loads the SSRF egress firewall -- rootless podman's | |
| # default set excludes it; without it nft EPERMs, entrypoint | |
| # fail-closes, and the container never serves; | |
| # CHOWN the /app/runtime re-chown to fingpt after the :U mount chown; | |
| # SETUID/SETGID setpriv's drop to uid/gid 1001 (+ --init-groups); | |
| # SETPCAP setpriv's --bounding-set=-net_admin drop for the app phase | |
| # (the app runs as uid1001 with NET_ADMIN out of its bounding set). | |
| # The pre-cutover gate above runs the IDENTICAL flags -- that is the safety | |
| # property: all five caps get exercised end-to-end BEFORE cutover, so a | |
| # too-narrow set aborts the deploy rather than crash-looping the unit. Set + | |
| # gate/ExecStart parity are pinned by Main/backend/tests/test_dockerfile_nonroot.py; | |
| # widen only by editing BOTH podman run invocations AND the sentinel, with the | |
| # new cap's root-init step documented here. | |
| # --pids-limit=1024 is the fork-bomb ceiling: worst case today is ~400-600 | |
| # tasks (gunicorn workers x threads + agent fan-out + Playwright/Chromium); | |
| # revisit if AGENT_MAX_CONCURRENCY or GUNICORN_WORKERS is raised. | |
| # --read-only rootfs: the ONLY writable surfaces are the tmpfs set below, | |
| # /dev/shm (podman-managed 64MB tmpfs; gunicorn --worker-tmp-dir, untouched | |
| # by --read-only), and the /app/runtime bind mount. The tmpfs map: | |
| # /tmp Chromium temp profiles + shm redirect | |
| # (--disable-dev-shm-usage), nft mktemp at root-init, | |
| # /tmp/fingpt_cache; sized + world-writable-sticky; | |
| # /app/staticfiles collectstatic writes it at boot (0 files today, kept | |
| # writable as future-proofing); mode=0755 tmpfs, | |
| # chowned to fingpt by root-init (see below); | |
| # /home/fingpt fontconfig cache, edgartools ~/.edgar import-time | |
| # marker, yfinance cache; MUST end up fingpt-owned or | |
| # the MCP-child EACCES bug (#331 class) returns. podman | |
| # 5.6.2 rejects tmpfs uid=/gid= mount options outright, | |
| # so ownership is NOT set here -- entrypoint.sh root-init | |
| # chowns both dirs while PID1 still holds CAP_CHOWN. | |
| # mode=0755 (not the tmpfs default 1777) lands them | |
| # owner-writable, not world-writable. | |
| # /app/logs and /app/media are vestigial with ZERO writers -- deliberately | |
| # NOT tmpfs, so a future stray write fails LOUDLY instead of vanishing | |
| # into RAM. Playwright's DEPENDENCIES_VALIDATED marker is pre-baked at | |
| # image build (Dockerfile) so first browser launch needs no rootfs write. | |
| # The pre-cutover gate above boots the FULL app under these exact flags | |
| # (BOOT_CHECK_ONLY=1) before cutover, so a missing/mis-owned tmpfs aborts | |
| # the deploy with the old container still serving. Accepted residual: the | |
| # gate's /app/runtime is a throwaway tmpfs, so the :U chown / :Z SELinux | |
| # relabel path of the REAL bind mount (#322 class) is NOT exercised | |
| # pre-cutover; it stays covered only by the post-cutover ~130s health | |
| # window in the Verify step below. | |
| OVERRIDE_DIR="$HOME/.config/systemd/user/${SYSTEMD_UNIT}.service.d" | |
| mkdir -p "$OVERRIDE_DIR" | |
| cat > "$OVERRIDE_DIR/override.conf" <<EOF | |
| [Service] | |
| ExecStart= | |
| ExecStart=/usr/bin/podman run --name ${SYSTEMD_UNIT} --replace --rm --cap-drop=ALL --cap-add=NET_ADMIN --cap-add=CHOWN --cap-add=SETUID --cap-add=SETGID --cap-add=SETPCAP --pids-limit=1024 --read-only --tmpfs=/tmp:rw,size=512m,mode=1777 --tmpfs=/app/staticfiles:rw,mode=0755 --tmpfs=/home/fingpt:rw,mode=0755 --cgroups=split --sdnotify=conmon -d --memory=1.7g --memory-swap=2g --network fingpt-net -v /home/deploy/fingpt/runtime:/app/runtime:U,Z --publish 127.0.0.1:8000:8000 --env-file /home/deploy/fingpt/envs/.env.production --env REDIS_URL=redis://fingpt-redis:6379/0 ${REMOTE_IMAGE} | |
| EOF | |
| systemctl --user daemon-reload | |
| systemctl --user restart "$SYSTEMD_UNIT" | |
| # Retire superseded backend images. The pull above is BY DIGEST, so every | |
| # old image keeps a repo@sha256 name, is never "dangling", and the plain | |
| # image prune below skips it forever (~3.3GB accumulated per deploy; 10 | |
| # deep when first cleaned by hand, 2026-07-03). Keep exactly two: the | |
| # image just deployed and PREV_IMAGE (the manual rollback target recorded | |
| # above). Do NOT swap this for pruning ALL unused images or a time-window | |
| # filter: the first deletes the rollback target outright, the second ages | |
| # it by BUILD time, so any deploy gap longer than the window deletes it | |
| # too. Retirement runs ONLY when the keep set has two DISTINCT resolvable | |
| # members: on a first deploy / absent container (PREV_IMAGE=unknown -- a | |
| # crash-looped unit's --rm leaves no container), a same-digest job re-run | |
| # (KEEP_NEW == KEEP_PREV), or a failed keep resolution, deleting | |
| # "everything else" would take the last-known-good image precisely when a | |
| # rollback is most likely needed -- those paths skip, loudly. Removals | |
| # are unforced (force can remove RUNNING containers) and fail OPEN to | |
| # accumulation: a refusal warns and continues, never failing a deploy | |
| # that has already cut over (set -e is in force). reference= matching | |
| # digest-named images, short-ID resolution, and multi-name rmi refusal | |
| # were validated live on the droplet's podman 5.6.2 before this shipped. | |
| REPO="${REMOTE_IMAGE%%@*}" | |
| if [ "$PREV_IMAGE" = "unknown" ]; then | |
| echo "No previous container recorded; skipping image retirement" | |
| elif ! KEEP_NEW=$(podman image inspect --format '{{.Id}}' "$REMOTE_IMAGE"); then | |
| echo "WARN: cannot resolve just-deployed image ${REMOTE_IMAGE}; skipping image retirement" | |
| elif ! KEEP_PREV=$(podman image inspect --format '{{.Id}}' "$PREV_IMAGE"); then | |
| echo "WARN: cannot resolve rollback target ${PREV_IMAGE}; skipping image retirement" | |
| elif [ "$KEEP_NEW" = "$KEEP_PREV" ]; then | |
| echo "Redeploy of the already-running image; skipping image retirement" | |
| else | |
| for short in $(podman images --filter "reference=${REPO}" --format '{{.ID}}' | sort -u); do | |
| img=$(podman image inspect --format '{{.Id}}' "$short" 2>/dev/null) || continue | |
| case "$img" in | |
| "$KEEP_NEW"|"$KEEP_PREV") ;; | |
| *) | |
| echo "Retiring superseded image ${short}" | |
| podman rmi "$short" || echo "WARN: could not remove ${short}; leaving in place" | |
| ;; | |
| esac | |
| done | |
| fi | |
| podman image prune -f | |
| - name: Verify deployment health | |
| if: ${{ steps.deploy-gate.outputs.enabled == 'true' }} | |
| uses: appleboy/ssh-action@334f9259f2f8eb3376d33fa4c684fff373f2c2a6 # v0.1.10, SHA-pinned (holds the prod SSH key) | |
| with: | |
| host: ${{ env.DROPLET_HOST }} | |
| username: ${{ env.DROPLET_USER }} | |
| key: ${{ secrets.DEPLOY_SSH_KEY }} | |
| port: 22 | |
| envs: SYSTEMD_UNIT | |
| script: | | |
| set -euo pipefail | |
| export XDG_RUNTIME_DIR=/run/user/$(id -u) | |
| echo "Checking service status..." | |
| systemctl --user is-active "$SYSTEMD_UNIT" || { | |
| echo "ERROR: Service is not active" | |
| systemctl --user status "$SYSTEMD_UNIT" || true | |
| exit 1 | |
| } | |
| # Cold start is dominated by entrypoint.sh building the XBRL truth-layer | |
| # DuckDB store BEFORE gunicorn forks (~45s observed), on top of collectstatic | |
| # + Playwright verification. `systemctl is-active` only proves the container | |
| # launched, not that gunicorn is listening -- so poll /health/ across a window | |
| # comfortably above the measured cold start (~130s) instead of a fixed ~45s | |
| # wait, which raced the store build and failed the deploy even though the | |
| # container was healthy. Succeeds as soon as the endpoint answers. | |
| echo "Waiting 10s before first health probe..." | |
| sleep 10 | |
| echo "Checking health endpoint (up to ~130s for cold start)..." | |
| for i in $(seq 1 24); do | |
| if curl -sf -m 10 http://localhost:8000/health/; then | |
| echo "" | |
| echo "Deployment verification successful (attempt $i)!" | |
| exit 0 | |
| fi | |
| echo "Attempt $i/24 failed, retrying in 5s..." | |
| sleep 5 | |
| done | |
| echo "ERROR: Health check failed" | |
| systemctl --user status "$SYSTEMD_UNIT" || true | |
| journalctl --user -u "$SYSTEMD_UNIT" --no-pager -n 80 || true | |
| exit 1 |