feat(security): Root G hygiene batch — env allow-list, log redaction,… #118
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Backend CI and Deploy | |
| on: | |
| push: | |
| branches: | |
| - main | |
| # Allowlist the backend image's actual inputs so unrelated changes (heartbeat, docs, | |
| # the other deploy workflows) can't rebuild/restart the backend. The image is built | |
| # solely from Main/backend/ (Dockerfile + that build context); the workflow file is | |
| # included so its own edits still ship. Previously a broad paths-ignore let any | |
| # non-heartbeat/docs change (e.g. a concierge workflow edit) bounce the backend. | |
| paths: | |
| - "Main/backend/**" | |
| - ".github/workflows/backend-deploy.yml" | |
| workflow_dispatch: | |
| env: | |
| BACKEND_DIR: Main/backend | |
| PYTHON_VERSION: "3.12" | |
| REGISTRY: ghcr.io | |
| # Shared droplet coordinates as repo-level Actions variables (see heartbeat/concierge), | |
| # so a host/user change is one repo setting, not an edit across three workflows. | |
| DROPLET_HOST: ${{ vars.DROPLET_HOST }} | |
| DROPLET_USER: ${{ vars.DROPLET_USER }} | |
| SYSTEMD_UNIT: fingpt-api | |
| jobs: | |
| build: | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| packages: write | |
| outputs: | |
| image: ${{ steps.meta.outputs.image }} | |
| sha_tag: ${{ steps.meta.outputs.sha_tag }} | |
| main_tag: ${{ steps.meta.outputs.main_tag }} | |
| latest_tag: ${{ steps.meta.outputs.latest_tag }} | |
| digest: ${{ steps.digest.outputs.digest }} | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 | |
| - name: Set up uv | |
| uses: astral-sh/setup-uv@caf0cab7a618c569241d31dcd442f54681755d39 # v3.2.4 | |
| - name: Install Python ${{ env.PYTHON_VERSION }} | |
| run: uv python install ${{ env.PYTHON_VERSION }} | |
| - name: Sync backend dependencies | |
| working-directory: ${{ env.BACKEND_DIR }} | |
| run: uv sync --frozen --python ${{ env.PYTHON_VERSION }} | |
| - name: Run Django system checks | |
| working-directory: ${{ env.BACKEND_DIR }} | |
| run: uv run python manage.py check | |
| - name: Verify deployment readiness | |
| working-directory: ${{ env.BACKEND_DIR }} | |
| run: uv run python verify_deployment.py | |
| - name: Compute image tags | |
| id: meta | |
| run: | | |
| REPO_SLUG=$(echo "${{ github.repository }}" | tr '[:upper:]' '[:lower:]') | |
| IMAGE_ID="${{ env.REGISTRY }}/${REPO_SLUG}-backend" | |
| echo "image=$IMAGE_ID" >> "$GITHUB_OUTPUT" | |
| echo "sha_tag=${IMAGE_ID}:${{ github.sha }}" >> "$GITHUB_OUTPUT" | |
| echo "main_tag=${IMAGE_ID}:main" >> "$GITHUB_OUTPUT" | |
| echo "latest_tag=${IMAGE_ID}:latest" >> "$GITHUB_OUTPUT" | |
| - name: Log in to GHCR | |
| uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0 | |
| with: | |
| registry: ${{ env.REGISTRY }} | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Build backend image | |
| run: | | |
| docker build \ | |
| --pull \ | |
| --file ${{ env.BACKEND_DIR }}/Dockerfile \ | |
| --tag ${{ steps.meta.outputs.sha_tag }} \ | |
| --tag ${{ steps.meta.outputs.main_tag }} \ | |
| --tag ${{ steps.meta.outputs.latest_tag }} \ | |
| ${{ env.BACKEND_DIR }} | |
| - name: Push backend image | |
| run: | | |
| docker push ${{ steps.meta.outputs.sha_tag }} | |
| docker push ${{ steps.meta.outputs.main_tag }} | |
| docker push ${{ steps.meta.outputs.latest_tag }} | |
| - name: Capture pushed image digest | |
| id: digest | |
| run: | | |
| DIGEST=$(docker inspect --format='{{index .RepoDigests 0}}' ${{ steps.meta.outputs.sha_tag }}) | |
| echo "digest=$DIGEST" >> "$GITHUB_OUTPUT" | |
| test: | |
| runs-on: ubuntu-latest | |
| needs: build | |
| permissions: | |
| contents: read | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 | |
| - name: Set up uv | |
| uses: astral-sh/setup-uv@caf0cab7a618c569241d31dcd442f54681755d39 # v3.2.4 | |
| - name: Install Python ${{ env.PYTHON_VERSION }} | |
| run: uv python install ${{ env.PYTHON_VERSION }} | |
| - name: Sync backend dependencies | |
| working-directory: ${{ env.BACKEND_DIR }} | |
| run: uv sync --frozen --python ${{ env.PYTHON_VERSION }} | |
| - name: Run backend tests | |
| working-directory: ${{ env.BACKEND_DIR }} | |
| run: uv run pytest tests -q | |
| deploy: | |
| runs-on: ubuntu-latest | |
| if: ${{ github.ref == 'refs/heads/main' }} | |
| # DEPLOY_SSH_KEY / GHCR_READ_TOKEN are ENVIRONMENT secrets (Production has a | |
| # deployment branch policy of main only), not repo-scoped: even a workflow | |
| # edited or dispatched on a non-main ref cannot release the prod SSH key. | |
| # This gates the secret itself, independently of the ref-gate above -- the | |
| # two layers fail separately (same stanza as the heartbeat/concierge jobs). | |
| environment: | |
| name: Production | |
| url: https://agenticfinsearch.org | |
| # Serialize deploys so two merges in quick succession can't land out of order, | |
| # leave the droplet on the older commit, or -- now that superseded images are | |
| # retired below -- rmi an in-flight peer's freshly pulled, still container-less | |
| # image mid-cutover (same stanza as the concierge/heartbeat deploy jobs). | |
| concurrency: | |
| group: backend-deploy | |
| cancel-in-progress: false | |
| needs: | |
| - build | |
| - test | |
| permissions: | |
| contents: read | |
| steps: | |
| - name: Determine deploy eligibility | |
| id: deploy-gate | |
| env: | |
| DEPLOY_SSH_KEY: ${{ secrets.DEPLOY_SSH_KEY }} | |
| GHCR_READ_TOKEN: ${{ secrets.GHCR_READ_TOKEN }} | |
| run: | | |
| set -euo pipefail | |
| if [[ -n "${DEPLOY_SSH_KEY:-}" && -n "${GHCR_READ_TOKEN:-}" ]]; then | |
| echo "enabled=true" >> "$GITHUB_OUTPUT" | |
| else | |
| echo "enabled=false" >> "$GITHUB_OUTPUT" | |
| fi | |
| - name: Deploy to Fedora droplet | |
| if: ${{ steps.deploy-gate.outputs.enabled == 'true' }} | |
| uses: appleboy/ssh-action@334f9259f2f8eb3376d33fa4c684fff373f2c2a6 # v0.1.10, SHA-pinned (holds the prod SSH key) | |
| env: | |
| REMOTE_IMAGE: ${{ needs.build.outputs.digest }} | |
| GHCR_USERNAME: ${{ github.repository_owner }} | |
| GHCR_TOKEN: ${{ secrets.GHCR_READ_TOKEN }} | |
| with: | |
| host: ${{ env.DROPLET_HOST }} | |
| username: ${{ env.DROPLET_USER }} | |
| key: ${{ secrets.DEPLOY_SSH_KEY }} | |
| port: 22 | |
| envs: REMOTE_IMAGE,GHCR_USERNAME,GHCR_TOKEN,SYSTEMD_UNIT | |
| script: | | |
| set -euo pipefail | |
| export XDG_RUNTIME_DIR=/run/user/$(id -u) | |
| podman login ${{ env.REGISTRY }} -u "$GHCR_USERNAME" -p "$GHCR_TOKEN" | |
| podman pull "$REMOTE_IMAGE" | |
| # Shared rootless network so the api container reaches redis by name. | |
| podman network exists fingpt-net || podman network create fingpt-net | |
| # Redis counter store (atomic incr/decr for agent budget + ratelimit). | |
| # Ephemeral cache: no persistence. Started idempotently (start-if-absent, not | |
| # --replace) BEFORE the pre-cutover gate so (a) the firewall self-test has redis | |
| # up on fingpt-net, and (b) an ABORTED deploy does not bounce the live redis / | |
| # wipe counters for the still-serving old container. | |
| # TRADE-OFF of start-if-absent: the running container stays FROZEN on whatever | |
| # redis:7-alpine image it was created from -- deploys never pick up redis image | |
| # updates (security patches included). To upgrade redis, run on the droplet: | |
| # podman pull docker.io/library/redis:7-alpine && podman rm -f fingpt-redis | |
| # then let the next deploy (or a re-run of this one) recreate it. Counters are | |
| # ephemeral by design, so the wipe is acceptable when done deliberately. | |
| if ! podman container exists fingpt-redis; then | |
| podman run -d --name fingpt-redis --restart=always \ | |
| --network fingpt-net \ | |
| --health-cmd 'redis-cli ping' --health-interval 10s --health-retries 5 \ | |
| docker.io/library/redis:7-alpine redis-server --save '' --appendonly no | |
| else | |
| podman start fingpt-redis >/dev/null 2>&1 || true | |
| fi | |
| # Pre-cutover firewall validation: run the NEW image's REAL entrypoint in | |
| # check-only mode on fingpt-net BEFORE the systemctl restart's --replace kills | |
| # the currently-serving container. --egress-check-only exercises the byte- | |
| # identical root-init PID1 runs at cutover (generate + sentinel checks + nft | |
| # load + transition-proof self-test + setpriv drop), then exits before the app | |
| # phase; an inline re-copy of that sequence (this step's previous shape) can | |
| # silently drift from the boot path it is supposed to prove. On failure, abort | |
| # so the OLD container keeps serving (there is no auto-rollback). This closes | |
| # the CI-invisible gap: the test job is pytest-only and never touches nft/netns. | |
| # The gate runs under the IDENTICAL frozen cap/pids flags as the ExecStart | |
| # below (cap-to-step map at the override; parity pinned in | |
| # Main/backend/tests/test_dockerfile_nonroot.py). That parity IS the safety | |
| # property: check-only mode exits right after the setpriv drop, so all five | |
| # caps the root-init needs (nft + chown + setuid/setgid + bounding-set drop) | |
| # are exercised HERE -- a too-narrow set aborts the deploy with the old | |
| # container still serving, instead of crash-looping the new unit at cutover. | |
| echo "Validating egress firewall on the new image (pre-cutover)..." | |
| podman run --rm --cap-drop=ALL --cap-add=NET_ADMIN --cap-add=CHOWN --cap-add=SETUID --cap-add=SETGID --cap-add=SETPCAP --pids-limit=1024 --network fingpt-net \ | |
| --env REDIS_URL=redis://fingpt-redis:6379/0 \ | |
| "$REMOTE_IMAGE" --egress-check-only \ | |
| || { echo "ERROR: egress firewall pre-cutover validation FAILED; aborting deploy (old container left serving)"; exit 1; } | |
| echo "Egress firewall pre-cutover validation passed." | |
| # Record the currently-deployed image for manual rollback (no auto-rollback). | |
| PREV_IMAGE=$(podman inspect --format '{{.ImageName}}' "$SYSTEMD_UNIT" 2>/dev/null || echo "unknown") | |
| echo "Previous image (manual rollback target): $PREV_IMAGE" | |
| # Update systemd override to run the image we just pulled. | |
| # PID1 is now root-in-userns (no Dockerfile USER line); it loads the egress | |
| # firewall, chowns /app/runtime to fingpt, then setpriv-drops to uid1001 for | |
| # the app. The runtime bind mount still needs BOTH relabel suffixes on this | |
| # SELinux-enforcing Fedora host: | |
| # :U rootless podman recursively chowns the host dir to the in-container uid | |
| # (now root/0, since PID1 is root); entrypoint then re-chowns it to fingpt | |
| # so the dropped app user can write it (validated: spec Appendix A probe 3). | |
| # :Z relabels the (private) host dir to container_file_t (MAC/SELinux); | |
| # without it container_t is denied and the write is EACCES -- which | |
| # crash-looped the truth-layer store build the first time a deploy actually | |
| # wrote this mount (#322). Ownership alone is not enough; both are required. | |
| # Frozen capability set: --cap-drop=ALL, then add back ONLY the five caps the | |
| # root-init phase of Main/backend/entrypoint.sh actually uses. Each cap maps | |
| # 1:1 to a root-init step, so nothing in the set is discretionary: | |
| # NET_ADMIN nft loads the SSRF egress firewall -- rootless podman's | |
| # default set excludes it; without it nft EPERMs, entrypoint | |
| # fail-closes, and the container never serves; | |
| # CHOWN the /app/runtime re-chown to fingpt after the :U mount chown; | |
| # SETUID/SETGID setpriv's drop to uid/gid 1001 (+ --init-groups); | |
| # SETPCAP setpriv's --bounding-set=-net_admin drop for the app phase | |
| # (the app runs as uid1001 with NET_ADMIN out of its bounding set). | |
| # The pre-cutover gate above runs the IDENTICAL flags -- that is the safety | |
| # property: all five caps get exercised end-to-end BEFORE cutover, so a | |
| # too-narrow set aborts the deploy rather than crash-looping the unit. Set + | |
| # gate/ExecStart parity are pinned by Main/backend/tests/test_dockerfile_nonroot.py; | |
| # widen only by editing BOTH podman run invocations AND the sentinel, with the | |
| # new cap's root-init step documented here. | |
| # --pids-limit=1024 is the fork-bomb ceiling: worst case today is ~400-600 | |
| # tasks (gunicorn workers x threads + agent fan-out + Playwright/Chromium); | |
| # revisit if AGENT_MAX_CONCURRENCY or GUNICORN_WORKERS is raised. | |
| # A read-only rootfs is DELIBERATELY deferred: the gate exits before the app | |
| # phase, so --read-only would ship unvalidated pre-cutover -- and collectstatic | |
| # writes /app/staticfiles at boot, which needs a tmpfs/volume design first. | |
| OVERRIDE_DIR="$HOME/.config/systemd/user/${SYSTEMD_UNIT}.service.d" | |
| mkdir -p "$OVERRIDE_DIR" | |
| cat > "$OVERRIDE_DIR/override.conf" <<EOF | |
| [Service] | |
| ExecStart= | |
| ExecStart=/usr/bin/podman run --name ${SYSTEMD_UNIT} --replace --rm --cap-drop=ALL --cap-add=NET_ADMIN --cap-add=CHOWN --cap-add=SETUID --cap-add=SETGID --cap-add=SETPCAP --pids-limit=1024 --cgroups=split --sdnotify=conmon -d --memory=1.7g --memory-swap=2g --network fingpt-net -v /home/deploy/fingpt/runtime:/app/runtime:U,Z --publish 127.0.0.1:8000:8000 --env-file /home/deploy/fingpt/envs/.env.production --env REDIS_URL=redis://fingpt-redis:6379/0 ${REMOTE_IMAGE} | |
| EOF | |
| systemctl --user daemon-reload | |
| systemctl --user restart "$SYSTEMD_UNIT" | |
| # Retire superseded backend images. The pull above is BY DIGEST, so every | |
| # old image keeps a repo@sha256 name, is never "dangling", and the plain | |
| # image prune below skips it forever (~3.3GB accumulated per deploy; 10 | |
| # deep when first cleaned by hand, 2026-07-03). Keep exactly two: the | |
| # image just deployed and PREV_IMAGE (the manual rollback target recorded | |
| # above). Do NOT swap this for pruning ALL unused images or a time-window | |
| # filter: the first deletes the rollback target outright, the second ages | |
| # it by BUILD time, so any deploy gap longer than the window deletes it | |
| # too. Retirement runs ONLY when the keep set has two DISTINCT resolvable | |
| # members: on a first deploy / absent container (PREV_IMAGE=unknown -- a | |
| # crash-looped unit's --rm leaves no container), a same-digest job re-run | |
| # (KEEP_NEW == KEEP_PREV), or a failed keep resolution, deleting | |
| # "everything else" would take the last-known-good image precisely when a | |
| # rollback is most likely needed -- those paths skip, loudly. Removals | |
| # are unforced (force can remove RUNNING containers) and fail OPEN to | |
| # accumulation: a refusal warns and continues, never failing a deploy | |
| # that has already cut over (set -e is in force). reference= matching | |
| # digest-named images, short-ID resolution, and multi-name rmi refusal | |
| # were validated live on the droplet's podman 5.6.2 before this shipped. | |
| REPO="${REMOTE_IMAGE%%@*}" | |
| if [ "$PREV_IMAGE" = "unknown" ]; then | |
| echo "No previous container recorded; skipping image retirement" | |
| elif ! KEEP_NEW=$(podman image inspect --format '{{.Id}}' "$REMOTE_IMAGE"); then | |
| echo "WARN: cannot resolve just-deployed image ${REMOTE_IMAGE}; skipping image retirement" | |
| elif ! KEEP_PREV=$(podman image inspect --format '{{.Id}}' "$PREV_IMAGE"); then | |
| echo "WARN: cannot resolve rollback target ${PREV_IMAGE}; skipping image retirement" | |
| elif [ "$KEEP_NEW" = "$KEEP_PREV" ]; then | |
| echo "Redeploy of the already-running image; skipping image retirement" | |
| else | |
| for short in $(podman images --filter "reference=${REPO}" --format '{{.ID}}' | sort -u); do | |
| img=$(podman image inspect --format '{{.Id}}' "$short" 2>/dev/null) || continue | |
| case "$img" in | |
| "$KEEP_NEW"|"$KEEP_PREV") ;; | |
| *) | |
| echo "Retiring superseded image ${short}" | |
| podman rmi "$short" || echo "WARN: could not remove ${short}; leaving in place" | |
| ;; | |
| esac | |
| done | |
| fi | |
| podman image prune -f | |
| - name: Verify deployment health | |
| if: ${{ steps.deploy-gate.outputs.enabled == 'true' }} | |
| uses: appleboy/ssh-action@334f9259f2f8eb3376d33fa4c684fff373f2c2a6 # v0.1.10, SHA-pinned (holds the prod SSH key) | |
| with: | |
| host: ${{ env.DROPLET_HOST }} | |
| username: ${{ env.DROPLET_USER }} | |
| key: ${{ secrets.DEPLOY_SSH_KEY }} | |
| port: 22 | |
| envs: SYSTEMD_UNIT | |
| script: | | |
| set -euo pipefail | |
| export XDG_RUNTIME_DIR=/run/user/$(id -u) | |
| echo "Checking service status..." | |
| systemctl --user is-active "$SYSTEMD_UNIT" || { | |
| echo "ERROR: Service is not active" | |
| systemctl --user status "$SYSTEMD_UNIT" || true | |
| exit 1 | |
| } | |
| # Cold start is dominated by entrypoint.sh building the XBRL truth-layer | |
| # DuckDB store BEFORE gunicorn forks (~45s observed), on top of collectstatic | |
| # + Playwright verification. `systemctl is-active` only proves the container | |
| # launched, not that gunicorn is listening -- so poll /health/ across a window | |
| # comfortably above the measured cold start (~130s) instead of a fixed ~45s | |
| # wait, which raced the store build and failed the deploy even though the | |
| # container was healthy. Succeeds as soon as the endpoint answers. | |
| echo "Waiting 10s before first health probe..." | |
| sleep 10 | |
| echo "Checking health endpoint (up to ~130s for cold start)..." | |
| for i in $(seq 1 24); do | |
| if curl -sf -m 10 http://localhost:8000/health/; then | |
| echo "" | |
| echo "Deployment verification successful (attempt $i)!" | |
| exit 0 | |
| fi | |
| echo "Attempt $i/24 failed, retrying in 5s..." | |
| sleep 5 | |
| done | |
| echo "ERROR: Health check failed" | |
| systemctl --user status "$SYSTEMD_UNIT" || true | |
| journalctl --user -u "$SYSTEMD_UNIT" --no-pager -n 80 || true | |
| exit 1 |