fix(security): close CGNAT + 192.0.0.0/24 HTTP-side SSRF gaps; drop d… #115
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Backend CI and Deploy | |
| on: | |
| push: | |
| branches: | |
| - main | |
| # Allowlist the backend image's actual inputs so unrelated changes (heartbeat, docs, | |
| # the other deploy workflows) can't rebuild/restart the backend. The image is built | |
| # solely from Main/backend/ (Dockerfile + that build context); the workflow file is | |
| # included so its own edits still ship. Previously a broad paths-ignore let any | |
| # non-heartbeat/docs change (e.g. a concierge workflow edit) bounce the backend. | |
| paths: | |
| - "Main/backend/**" | |
| - ".github/workflows/backend-deploy.yml" | |
| workflow_dispatch: | |
| env: | |
| BACKEND_DIR: Main/backend | |
| PYTHON_VERSION: "3.12" | |
| REGISTRY: ghcr.io | |
| # Shared droplet coordinates as repo-level Actions variables (see heartbeat/concierge), | |
| # so a host/user change is one repo setting, not an edit across three workflows. | |
| DROPLET_HOST: ${{ vars.DROPLET_HOST }} | |
| DROPLET_USER: ${{ vars.DROPLET_USER }} | |
| SYSTEMD_UNIT: fingpt-api | |
| jobs: | |
| build: | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| packages: write | |
| outputs: | |
| image: ${{ steps.meta.outputs.image }} | |
| sha_tag: ${{ steps.meta.outputs.sha_tag }} | |
| main_tag: ${{ steps.meta.outputs.main_tag }} | |
| latest_tag: ${{ steps.meta.outputs.latest_tag }} | |
| digest: ${{ steps.digest.outputs.digest }} | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 | |
| - name: Set up uv | |
| uses: astral-sh/setup-uv@caf0cab7a618c569241d31dcd442f54681755d39 # v3.2.4 | |
| - name: Install Python ${{ env.PYTHON_VERSION }} | |
| run: uv python install ${{ env.PYTHON_VERSION }} | |
| - name: Sync backend dependencies | |
| working-directory: ${{ env.BACKEND_DIR }} | |
| run: uv sync --frozen --python ${{ env.PYTHON_VERSION }} | |
| - name: Run Django system checks | |
| working-directory: ${{ env.BACKEND_DIR }} | |
| run: uv run python manage.py check | |
| - name: Verify deployment readiness | |
| working-directory: ${{ env.BACKEND_DIR }} | |
| run: uv run python verify_deployment.py | |
| - name: Compute image tags | |
| id: meta | |
| run: | | |
| REPO_SLUG=$(echo "${{ github.repository }}" | tr '[:upper:]' '[:lower:]') | |
| IMAGE_ID="${{ env.REGISTRY }}/${REPO_SLUG}-backend" | |
| echo "image=$IMAGE_ID" >> "$GITHUB_OUTPUT" | |
| echo "sha_tag=${IMAGE_ID}:${{ github.sha }}" >> "$GITHUB_OUTPUT" | |
| echo "main_tag=${IMAGE_ID}:main" >> "$GITHUB_OUTPUT" | |
| echo "latest_tag=${IMAGE_ID}:latest" >> "$GITHUB_OUTPUT" | |
| - name: Log in to GHCR | |
| uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0 | |
| with: | |
| registry: ${{ env.REGISTRY }} | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Build backend image | |
| run: | | |
| docker build \ | |
| --pull \ | |
| --file ${{ env.BACKEND_DIR }}/Dockerfile \ | |
| --tag ${{ steps.meta.outputs.sha_tag }} \ | |
| --tag ${{ steps.meta.outputs.main_tag }} \ | |
| --tag ${{ steps.meta.outputs.latest_tag }} \ | |
| ${{ env.BACKEND_DIR }} | |
| - name: Push backend image | |
| run: | | |
| docker push ${{ steps.meta.outputs.sha_tag }} | |
| docker push ${{ steps.meta.outputs.main_tag }} | |
| docker push ${{ steps.meta.outputs.latest_tag }} | |
| - name: Capture pushed image digest | |
| id: digest | |
| run: | | |
| DIGEST=$(docker inspect --format='{{index .RepoDigests 0}}' ${{ steps.meta.outputs.sha_tag }}) | |
| echo "digest=$DIGEST" >> "$GITHUB_OUTPUT" | |
| test: | |
| runs-on: ubuntu-latest | |
| needs: build | |
| permissions: | |
| contents: read | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 | |
| - name: Set up uv | |
| uses: astral-sh/setup-uv@caf0cab7a618c569241d31dcd442f54681755d39 # v3.2.4 | |
| - name: Install Python ${{ env.PYTHON_VERSION }} | |
| run: uv python install ${{ env.PYTHON_VERSION }} | |
| - name: Sync backend dependencies | |
| working-directory: ${{ env.BACKEND_DIR }} | |
| run: uv sync --frozen --python ${{ env.PYTHON_VERSION }} | |
| - name: Run backend tests | |
| working-directory: ${{ env.BACKEND_DIR }} | |
| run: uv run pytest tests -q | |
| deploy: | |
| runs-on: ubuntu-latest | |
| if: ${{ github.ref == 'refs/heads/main' }} | |
| needs: | |
| - build | |
| - test | |
| permissions: | |
| contents: read | |
| steps: | |
| - name: Determine deploy eligibility | |
| id: deploy-gate | |
| env: | |
| DEPLOY_SSH_KEY: ${{ secrets.DEPLOY_SSH_KEY }} | |
| GHCR_READ_TOKEN: ${{ secrets.GHCR_READ_TOKEN }} | |
| run: | | |
| set -euo pipefail | |
| if [[ -n "${DEPLOY_SSH_KEY:-}" && -n "${GHCR_READ_TOKEN:-}" ]]; then | |
| echo "enabled=true" >> "$GITHUB_OUTPUT" | |
| else | |
| echo "enabled=false" >> "$GITHUB_OUTPUT" | |
| fi | |
| - name: Deploy to Fedora droplet | |
| if: ${{ steps.deploy-gate.outputs.enabled == 'true' }} | |
| uses: appleboy/ssh-action@334f9259f2f8eb3376d33fa4c684fff373f2c2a6 # v0.1.10, SHA-pinned (holds the prod SSH key) | |
| env: | |
| REMOTE_IMAGE: ${{ needs.build.outputs.digest }} | |
| GHCR_USERNAME: ${{ github.repository_owner }} | |
| GHCR_TOKEN: ${{ secrets.GHCR_READ_TOKEN }} | |
| with: | |
| host: ${{ env.DROPLET_HOST }} | |
| username: ${{ env.DROPLET_USER }} | |
| key: ${{ secrets.DEPLOY_SSH_KEY }} | |
| port: 22 | |
| envs: REMOTE_IMAGE,GHCR_USERNAME,GHCR_TOKEN,SYSTEMD_UNIT | |
| script: | | |
| set -euo pipefail | |
| export XDG_RUNTIME_DIR=/run/user/$(id -u) | |
| podman login ${{ env.REGISTRY }} -u "$GHCR_USERNAME" -p "$GHCR_TOKEN" | |
| podman pull "$REMOTE_IMAGE" | |
| # Shared rootless network so the api container reaches redis by name. | |
| podman network exists fingpt-net || podman network create fingpt-net | |
| # Redis counter store (atomic incr/decr for agent budget + ratelimit). | |
| # Ephemeral cache: no persistence. Started idempotently (start-if-absent, not | |
| # --replace) BEFORE the pre-cutover gate so (a) the firewall self-test has redis | |
| # up on fingpt-net, and (b) an ABORTED deploy does not bounce the live redis / | |
| # wipe counters for the still-serving old container. | |
| # TRADE-OFF of start-if-absent: the running container stays FROZEN on whatever | |
| # redis:7-alpine image it was created from -- deploys never pick up redis image | |
| # updates (security patches included). To upgrade redis, run on the droplet: | |
| # podman pull docker.io/library/redis:7-alpine && podman rm -f fingpt-redis | |
| # then let the next deploy (or a re-run of this one) recreate it. Counters are | |
| # ephemeral by design, so the wipe is acceptable when done deliberately. | |
| if ! podman container exists fingpt-redis; then | |
| podman run -d --name fingpt-redis --restart=always \ | |
| --network fingpt-net \ | |
| --health-cmd 'redis-cli ping' --health-interval 10s --health-retries 5 \ | |
| docker.io/library/redis:7-alpine redis-server --save '' --appendonly no | |
| else | |
| podman start fingpt-redis >/dev/null 2>&1 || true | |
| fi | |
| # Pre-cutover firewall validation: run the NEW image's REAL entrypoint in | |
| # check-only mode on fingpt-net BEFORE the systemctl restart's --replace kills | |
| # the currently-serving container. --egress-check-only exercises the byte- | |
| # identical root-init PID1 runs at cutover (generate + sentinel checks + nft | |
| # load + transition-proof self-test + setpriv drop), then exits before the app | |
| # phase; an inline re-copy of that sequence (this step's previous shape) can | |
| # silently drift from the boot path it is supposed to prove. On failure, abort | |
| # so the OLD container keeps serving (there is no auto-rollback). This closes | |
| # the CI-invisible gap: the test job is pytest-only and never touches nft/netns. | |
| echo "Validating egress firewall on the new image (pre-cutover)..." | |
| podman run --rm --cap-add=NET_ADMIN --network fingpt-net \ | |
| --env REDIS_URL=redis://fingpt-redis:6379/0 \ | |
| "$REMOTE_IMAGE" --egress-check-only \ | |
| || { echo "ERROR: egress firewall pre-cutover validation FAILED; aborting deploy (old container left serving)"; exit 1; } | |
| echo "Egress firewall pre-cutover validation passed." | |
| # Record the currently-deployed image for manual rollback (no auto-rollback). | |
| PREV_IMAGE=$(podman inspect --format '{{.ImageName}}' "$SYSTEMD_UNIT" 2>/dev/null || echo "unknown") | |
| echo "Previous image (manual rollback target): $PREV_IMAGE" | |
| # Update systemd override to run the image we just pulled. | |
| # PID1 is now root-in-userns (no Dockerfile USER line); it loads the egress | |
| # firewall, chowns /app/runtime to fingpt, then setpriv-drops to uid1001 for | |
| # the app. The runtime bind mount still needs BOTH relabel suffixes on this | |
| # SELinux-enforcing Fedora host: | |
| # :U rootless podman recursively chowns the host dir to the in-container uid | |
| # (now root/0, since PID1 is root); entrypoint then re-chowns it to fingpt | |
| # so the dropped app user can write it (validated: spec Appendix A probe 3). | |
| # :Z relabels the (private) host dir to container_file_t (MAC/SELinux); | |
| # without it container_t is denied and the write is EACCES -- which | |
| # crash-looped the truth-layer store build the first time a deploy actually | |
| # wrote this mount (#322). Ownership alone is not enough; both are required. | |
| # --cap-add=NET_ADMIN is SECURITY/FUNCTIONALLY LOAD-BEARING: rootless podman's | |
| # default cap set excludes NET_ADMIN, so without it the root-in-userns init | |
| # cannot load the SSRF egress firewall (nft EPERM) -> entrypoint fail-closes -> | |
| # the container never serves. Do NOT drop it. (The app itself runs as uid1001 | |
| # with NET_ADMIN removed from its bounding set; see Main/backend/entrypoint.sh.) | |
| OVERRIDE_DIR="$HOME/.config/systemd/user/${SYSTEMD_UNIT}.service.d" | |
| mkdir -p "$OVERRIDE_DIR" | |
| cat > "$OVERRIDE_DIR/override.conf" <<EOF | |
| [Service] | |
| ExecStart= | |
| ExecStart=/usr/bin/podman run --name ${SYSTEMD_UNIT} --replace --rm --cap-add=NET_ADMIN --cgroups=split --sdnotify=conmon -d --memory=1.7g --memory-swap=2g --network fingpt-net -v /home/deploy/fingpt/runtime:/app/runtime:U,Z --publish 127.0.0.1:8000:8000 --env-file /home/deploy/fingpt/envs/.env.production --env REDIS_URL=redis://fingpt-redis:6379/0 ${REMOTE_IMAGE} | |
| EOF | |
| systemctl --user daemon-reload | |
| systemctl --user restart "$SYSTEMD_UNIT" | |
| podman image prune -f | |
| - name: Verify deployment health | |
| if: ${{ steps.deploy-gate.outputs.enabled == 'true' }} | |
| uses: appleboy/ssh-action@334f9259f2f8eb3376d33fa4c684fff373f2c2a6 # v0.1.10, SHA-pinned (holds the prod SSH key) | |
| with: | |
| host: ${{ env.DROPLET_HOST }} | |
| username: ${{ env.DROPLET_USER }} | |
| key: ${{ secrets.DEPLOY_SSH_KEY }} | |
| port: 22 | |
| envs: SYSTEMD_UNIT | |
| script: | | |
| set -euo pipefail | |
| export XDG_RUNTIME_DIR=/run/user/$(id -u) | |
| echo "Checking service status..." | |
| systemctl --user is-active "$SYSTEMD_UNIT" || { | |
| echo "ERROR: Service is not active" | |
| systemctl --user status "$SYSTEMD_UNIT" || true | |
| exit 1 | |
| } | |
| # Cold start is dominated by entrypoint.sh building the XBRL truth-layer | |
| # DuckDB store BEFORE gunicorn forks (~45s observed), on top of collectstatic | |
| # + Playwright verification. `systemctl is-active` only proves the container | |
| # launched, not that gunicorn is listening -- so poll /health/ across a window | |
| # comfortably above the measured cold start (~130s) instead of a fixed ~45s | |
| # wait, which raced the store build and failed the deploy even though the | |
| # container was healthy. Succeeds as soon as the endpoint answers. | |
| echo "Waiting 10s before first health probe..." | |
| sleep 10 | |
| echo "Checking health endpoint (up to ~130s for cold start)..." | |
| for i in $(seq 1 24); do | |
| if curl -sf -m 10 http://localhost:8000/health/; then | |
| echo "" | |
| echo "Deployment verification successful (attempt $i)!" | |
| exit 0 | |
| fi | |
| echo "Attempt $i/24 failed, retrying in 5s..." | |
| sleep 5 | |
| done | |
| echo "ERROR: Health check failed" | |
| systemctl --user status "$SYSTEMD_UNIT" || true | |
| journalctl --user -u "$SYSTEMD_UNIT" --no-pager -n 80 || true | |
| exit 1 |