ci: congestion control and qlog overrides for relay deploy

.github/workflows/deploy-relay.yml

@@ -1,7 +1,9 @@
 name: deploy relay
 
 # Ops controls for the CI relay (moqx-main.ci.openmoq.org + moqx-000 alias).
-# Restart, redeploy with a specific image tag, or change the log level.
+# Restart, redeploy with a specific image tag, or change the log level,
+# congestion control or qlog sampling. Overrides last until the next main-push
+# deploy.
 
 on:
   workflow_dispatch:
@@ -33,6 +35,50 @@ on:
           - "DBG2"
           - "DBG3"
           - "DBG4"
+      cc_mvfst:
+        description: "mvfst congestion control (default bbr)"
+        required: false
+        default: "default"
+        type: choice
+        options:
+          - "default"
+          - "bbr"
+          - "bbr2"
+          - "bbr2modular"
+          - "copa"
+          - "cubic"
+          - "newreno"
+      cc_pico:
+        description: "picoquic congestion control (default bbr)"
+        required: false
+        default: "default"
+        type: choice
+        options:
+          - "default"
+          - "bbr"
+          - "bbr1"
+          - "c4"
+          - "cubic"
+          - "dcubic"
+          - "fast"
+          - "newreno"
+          - "prague"
+          - "reno"
+      bbr_skip_probe_rtt:
+        description: "mvfst bbr: skip PROBE_RTT while app-limited"
+        required: false
+        type: boolean
+        default: false
+      qlog_sample_rate:
+        description: "mvfst qlog: fraction of new connections logged (1.0 is heavy)"
+        required: false
+        default: "off"
+        type: choice
+        options:
+          - "off"
+          - "0.01"
+          - "0.1"
+          - "1.0"
       enable_stats:
         description: "Enable stats dashboard (default enabled)"
         required: false
@@ -144,6 +190,10 @@ jobs:
           AWS_ACCESS_KEY_ID: ${{ secrets.OMOQ_CERTBOT_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.OMOQ_CERTBOT_SECRET_ACCESS_KEY }}
           ENABLE_STATS: ${{ inputs.enable_stats }}
+          MOQX_CC: ${{ inputs.cc_mvfst != 'default' && inputs.cc_mvfst || '' }}
+          MOQX_PICO_CC: ${{ inputs.cc_pico != 'default' && inputs.cc_pico || '' }}
+          MOQX_BBR_SKIP_PROBE_RTT: ${{ inputs.bbr_skip_probe_rtt }}
+          MOQX_QLOG_SAMPLE: ${{ inputs.qlog_sample_rate != 'off' && inputs.qlog_sample_rate || '' }}
           STATS_USER: ${{ secrets.STATS_USER }}
           STATS_PASSWORD: ${{ secrets.STATS_PASSWORD }}
           GRAFANA_ADMIN_PASSWORD: ${{ secrets.GRAFANA_ADMIN_PASSWORD }}
@@ -194,11 +244,21 @@ jobs:
           SLACK_WEBHOOK_URL: ${{ secrets.OMOQ_SLACK_WEBHOOK_URL }}
           DOMAIN: ${{ steps.target.outputs.domain }}
           IMAGE_TAG: ${{ steps.target.outputs.image_tag }}
+          CC_MVFST: ${{ inputs.cc_mvfst }}
+          CC_PICO: ${{ inputs.cc_pico }}
+          SKIP_PROBE_RTT: ${{ inputs.bbr_skip_probe_rtt }}
+          QLOG: ${{ inputs.qlog_sample_rate }}
         run: |
           SHORT="${GITHUB_SHA:0:7}"
           ACTION=${{ inputs.restart_only && '"restarted"' || '"deployed"' }}
           STATUS=${{ job.status == 'success' && '":rocket:"' || '":x:"' }}
           TEXT="${STATUS} *${{ github.repository }}* ${ACTION} \`${IMAGE_TAG}\` on \`${DOMAIN}:${RELAY_PORT}\`"
+          OV=""
+          [ "$CC_MVFST" != "default" ] && OV+=" cc_mvfst=${CC_MVFST}"
+          [ "$CC_PICO" != "default" ] && OV+=" cc_pico=${CC_PICO}"
+          [ "$SKIP_PROBE_RTT" = "true" ] && OV+=" bbr_skip_probe_rtt"
+          [ "$QLOG" != "off" ] && OV+=" qlog=${QLOG}"
+          [ -n "$OV" ] && [ "${{ inputs.restart_only }}" != "true" ] && TEXT+=" (overrides:${OV})"
           curl -sf -X POST "$SLACK_WEBHOOK_URL" \
             -H "Content-Type: application/json" \
             --data "{\"text\": \"${TEXT}\"}" || true

docker/config.docker.yaml

@@ -19,6 +19,8 @@ listener_defaults:
     max_server_recv_packets_per_loop: ${MOQX_RECV_PKTS}
     udp_socket_buffer_bytes: ${MOQX_UDP_BUFFER}
     ignore_path_mtu: ${MOQX_IGNORE_PATH_MTU}
+    bbr:
+      probe_rtt_disabled_if_app_limited: ${MOQX_BBR_SKIP_PROBE_RTT}
 
 # Two listeners: mvfst on MOQX_PORT and (when enabled) picoquic on
 # MOQX_PICO_PORT — same cert/versions/endpoint, different stack + port.
@@ -38,6 +40,8 @@ listeners:
       key_file: "${MOQX_KEY}"
       insecure: ${MOQX_INSECURE}
     endpoint: "${MOQX_ENDPOINT}"
+    quic:
+      cc_algo: ${MOQX_CC}
 ${MOQX_PICO_LISTENER}
 
 services:
@@ -62,3 +66,10 @@ admin:
   track_metrics_enabled: true
   address: "${MOQX_BIND_ADDR}"
   plaintext: true
+
+# The directory is always set so the admin API can capture on demand;
+# MOQX_QLOG_SAMPLE > 0 also qlogs that fraction of every new mvfst connection.
+logging:
+  qlog:
+    dir: "${MOQX_QLOG_DIR}"
+    sample_rate: ${MOQX_QLOG_SAMPLE}

docker/docker-compose.yml

@@ -46,7 +46,11 @@ services:
     volumes:
       - ${MOQX_CERTS_DIR:-/etc/letsencrypt}:/certs:ro
       - moqx-coredumps:/var/coredumps
+      - moqx-qlog:/var/log/moqx/qlog
       - ${MOQX_ENTRY:-./entrypoint.sh}:/usr/local/bin/entrypoint.sh:ro
+      # The template the entrypoint renders, from the same checkout, so settings
+      # added there take effect with any image tag.
+      - ./config.docker.yaml:/usr/local/share/moqx/config.docker.yaml:ro
     environment:
       MOQX_CERT: /certs/live/${DOMAIN}/fullchain.pem
       MOQX_KEY:  /certs/live/${DOMAIN}/privkey.pem
@@ -63,6 +67,12 @@ services:
       # IO worker threads — default matches the entrypoint (4); set to the core
       # count (e.g. 8) for sub-maximal load testing on the CI runner.
       MOQX_THREADS: ${MOQX_THREADS:-4}
+      # Congestion control per listener; empty = entrypoint default (bbr).
+      MOQX_CC: ${MOQX_CC:-}
+      MOQX_PICO_CC: ${MOQX_PICO_CC:-}
+      MOQX_BBR_SKIP_PROBE_RTT: ${MOQX_BBR_SKIP_PROBE_RTT:-}
+      # Fraction of new mvfst connections to qlog; empty = off.
+      MOQX_QLOG_SAMPLE: ${MOQX_QLOG_SAMPLE:-}
     ulimits:
       core: -1
       # QUIC multiplexes many connections over few UDP sockets, so fd pressure is
@@ -365,5 +375,6 @@ services:
 volumes:
   prometheus-targets:
   moqx-coredumps:
+  moqx-qlog:
   prometheus-data:
   grafana-data:

docker/entrypoint.sh

@@ -33,6 +33,12 @@
 #   MOQX_RECV_PKTS        — mvfst max_server_recv_packets_per_loop (default: 256)
 #   MOQX_UDP_BUFFER       — relay UDP socket buffer bytes (default: net.core.wmem_max)
 #   MOQX_IGNORE_PATH_MTU  — send full-size packets, skip PMTU (default: false)
+#   MOQX_CC               — mvfst listener congestion control (default: bbr;
+#                           bbr|bbr2|bbr2modular|copa|cubic|newreno|none)
+#   MOQX_PICO_CC          — picoquic listener congestion control (default: bbr;
+#                           bbr|bbr1|c4|cubic|dcubic|fast|newreno|prague|reno)
+#   MOQX_BBR_SKIP_PROBE_RTT — mvfst bbr: skip PROBE_RTT while app-limited
+#                           (default: false)
 #   MOQX_JEMALLOC         — LD_PRELOAD jemalloc (~10% speedup). "auto" (default)
 #                           probes the multiarch paths; off/false/0 uses the
 #                           system allocator; an explicit path forces that lib.
@@ -41,6 +47,12 @@
 #   MOQX_LOGGING  — folly XLOG config for the whole stack (empty = baseline INFO);
 #                e.g. DBG2 or "INFO,quic=WARN". moqx promotes it to folly's
 #                FOLLY_LOGGING env var internally. See docs/logging.md.
+#   MOQX_QLOG_SAMPLE — fraction of new mvfst connections to qlog, 0.0–1.0
+#                (default: 0 = on-demand captures only, via the admin
+#                /qlog/capture route). Fetch a file with the admin
+#                /logs?connection_id=<dcid>&type=qlog route.
+#   MOQX_QLOG_DIR — qlog directory (default: /var/log/moqx/qlog); files older
+#                than 3 days are deleted at startup.
 set -e
 
 # The whole stack logs via folly XLOG (configured by MOQX_LOGGING, handled in the
@@ -114,6 +126,9 @@ export MOQX_SEND_PKTS="${MOQX_SEND_PKTS:-16}"
 export MOQX_RECV_PKTS="${MOQX_RECV_PKTS:-256}"
 export MOQX_UDP_BUFFER="${MOQX_UDP_BUFFER:-$(cat /proc/sys/net/core/wmem_max 2>/dev/null || echo 1048576)}"
 export MOQX_IGNORE_PATH_MTU="${MOQX_IGNORE_PATH_MTU:-false}"
+export MOQX_CC="${MOQX_CC:-bbr}"
+export MOQX_PICO_CC="${MOQX_PICO_CC:-bbr}"
+export MOQX_BBR_SKIP_PROBE_RTT="${MOQX_BBR_SKIP_PROBE_RTT:-false}"
 
 # Second (picoquic) listener for dual-stack serving. Opt out with
 # MOQX_PICO_ENABLE=false. picoquic needs real TLS, so it is auto-disabled under
@@ -140,13 +155,22 @@ if [ "$MOQX_PICO_ENABLE" = "true" ]; then
       key_file: "${MOQX_KEY}"
       insecure: ${MOQX_INSECURE}
     endpoint: "${MOQX_ENDPOINT}"
+    quic:
+      cc_algo: ${MOQX_PICO_CC}
 PICO
 )
 else
   MOQX_PICO_LISTENER=""
 fi
 export MOQX_PICO_LISTENER
 
+case "${MOQX_QLOG_SAMPLE:-}" in off|"") MOQX_QLOG_SAMPLE=0 ;; esac
+export MOQX_QLOG_SAMPLE
+export MOQX_QLOG_DIR="${MOQX_QLOG_DIR:-/var/log/moqx/qlog}"
+if [ -d "$MOQX_QLOG_DIR" ]; then
+  find "$MOQX_QLOG_DIR" -name '*.qlog' -mtime +3 -delete 2>/dev/null || true
+fi
+
 CONFIG=/tmp/relay.yaml
 envsubst < /usr/local/share/moqx/config.docker.yaml > "$CONFIG"
 

docker/relay-deploy.sh

@@ -12,6 +12,11 @@
 #   RELAY_PORT        (default 4433)
 #   ADMIN_PORT        (default 8000)
 #   MOQX_LOGGING      (optional) folly XLOG config; empty = baseline INFO
+#   MOQX_CC, MOQX_PICO_CC, MOQX_BBR_SKIP_PROBE_RTT
+#                     (optional) congestion control overrides, see entrypoint.sh;
+#                     empty = entrypoint default
+#   MOQX_QLOG_SAMPLE  (optional) fraction of new mvfst connections to qlog;
+#                     empty = off
 #   PULL_IMAGE        (optional) full image ref to `docker pull` + retag :latest.
 #                     Empty → `docker compose pull` (compose's pinned :latest).
 #   ENABLE_STATS      "true" → stats stack + public dashboard
@@ -35,6 +40,10 @@ PUB_PORT="${STATS_PUBLIC_PORT:-4533}"
   echo "MOQX_PORT=${RELAY_PORT}"
   echo "MOQX_ADMIN_PORT=${ADMIN_PORT}"
   echo "MOQX_LOGGING=${MOQX_LOGGING:-}"
+  echo "MOQX_CC=${MOQX_CC:-}"
+  echo "MOQX_PICO_CC=${MOQX_PICO_CC:-}"
+  echo "MOQX_BBR_SKIP_PROBE_RTT=${MOQX_BBR_SKIP_PROBE_RTT:-}"
+  echo "MOQX_QLOG_SAMPLE=${MOQX_QLOG_SAMPLE:-}"
   echo "MOQX_CPUS=$(nproc)"
   echo "MOQX_THREADS=$(nproc)"
 } > .env
@@ -125,3 +134,37 @@ if [ "${ENABLE_STATS:-}" = "true" ]; then
   done
   ./grafana/publish-public-dashboard.sh publish || echo "::warning::public dashboard publish failed"
 fi
+
+# ── override check ───────────────────────────────────────────────────────────
+# Compare the relay's live config with the requested settings; fail on mismatch.
+curl -sf "http://127.0.0.1:${ADMIN_PORT}/config" | python3 -c '
+import json, os, sys
+cfg = json.load(sys.stdin)
+want = {
+    "mvfst": (os.environ.get("MOQX_CC") or "bbr",
+              os.environ.get("MOQX_BBR_SKIP_PROBE_RTT") == "true"),
+    "picoquic": (os.environ.get("MOQX_PICO_CC") or "bbr", None),
+}
+ok = True
+for l in cfg["listeners"]:
+    if l["quic_stack"] not in want:
+        continue
+    name = l["name"]
+    cc, skip = want[l["quic_stack"]]
+    got_cc = l["quic"]["cc_algo"]
+    got_skip = l["mvfst"]["bbr"]["probe_rtt_disabled_if_app_limited"]
+    extra = "" if skip is None else f" probe_rtt_disabled_if_app_limited={str(got_skip).lower()}"
+    print(f"==> {name}: cc_algo={got_cc}{extra}")
+    if got_cc != cc or (skip is not None and got_skip != skip):
+        print(f"::error::{name} is not running the requested congestion control")
+        ok = False
+want_qlog = float(os.environ.get("MOQX_QLOG_SAMPLE") or 0)
+qlog = (cfg.get("logging") or {}).get("qlog") or {}
+got_qlog = qlog.get("sample_rate", 0.0)
+qdir = qlog.get("dir", "-")
+print(f"==> qlog: sample_rate={got_qlog:g} dir={qdir}")
+if abs(got_qlog - want_qlog) > 1e-6:
+    print("::error::relay is not running the requested qlog sample rate")
+    ok = False
+sys.exit(0 if ok else 1)
+'