diff --git a/.github/workflows/deploy-relay.yml b/.github/workflows/deploy-relay.yml index 509ae574..25471d54 100644 --- a/.github/workflows/deploy-relay.yml +++ b/.github/workflows/deploy-relay.yml @@ -1,7 +1,9 @@ name: deploy relay # Ops controls for the CI relay (moqx-main.ci.openmoq.org + moqx-000 alias). -# Restart, redeploy with a specific image tag, or change the log level. +# Restart, redeploy with a specific image tag, or change the log level, +# congestion control or qlog sampling. Overrides last until the next main-push +# deploy. on: workflow_dispatch: @@ -33,6 +35,50 @@ on: - "DBG2" - "DBG3" - "DBG4" + cc_mvfst: + description: "mvfst congestion control (default bbr)" + required: false + default: "default" + type: choice + options: + - "default" + - "bbr" + - "bbr2" + - "bbr2modular" + - "copa" + - "cubic" + - "newreno" + cc_pico: + description: "picoquic congestion control (default bbr)" + required: false + default: "default" + type: choice + options: + - "default" + - "bbr" + - "bbr1" + - "c4" + - "cubic" + - "dcubic" + - "fast" + - "newreno" + - "prague" + - "reno" + bbr_skip_probe_rtt: + description: "mvfst bbr: skip PROBE_RTT while app-limited (default on)" + required: false + type: boolean + default: true + qlog_sample_rate: + description: "mvfst qlog: fraction of new connections logged (1.0 is heavy)" + required: false + default: "off" + type: choice + options: + - "off" + - "0.01" + - "0.1" + - "1.0" enable_stats: description: "Enable stats dashboard (default enabled)" required: false @@ -144,6 +190,10 @@ jobs: AWS_ACCESS_KEY_ID: ${{ secrets.OMOQ_CERTBOT_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.OMOQ_CERTBOT_SECRET_ACCESS_KEY }} ENABLE_STATS: ${{ inputs.enable_stats }} + MOQX_CC: ${{ inputs.cc_mvfst != 'default' && inputs.cc_mvfst || '' }} + MOQX_PICO_CC: ${{ inputs.cc_pico != 'default' && inputs.cc_pico || '' }} + MOQX_BBR_SKIP_PROBE_RTT: ${{ inputs.bbr_skip_probe_rtt }} + MOQX_QLOG_SAMPLE: ${{ inputs.qlog_sample_rate != 'off' && inputs.qlog_sample_rate || '' }} STATS_USER: ${{ secrets.STATS_USER }} STATS_PASSWORD: ${{ secrets.STATS_PASSWORD }} GRAFANA_ADMIN_PASSWORD: ${{ secrets.GRAFANA_ADMIN_PASSWORD }} @@ -194,11 +244,21 @@ jobs: SLACK_WEBHOOK_URL: ${{ secrets.OMOQ_SLACK_WEBHOOK_URL }} DOMAIN: ${{ steps.target.outputs.domain }} IMAGE_TAG: ${{ steps.target.outputs.image_tag }} + CC_MVFST: ${{ inputs.cc_mvfst }} + CC_PICO: ${{ inputs.cc_pico }} + SKIP_PROBE_RTT: ${{ inputs.bbr_skip_probe_rtt }} + QLOG: ${{ inputs.qlog_sample_rate }} run: | SHORT="${GITHUB_SHA:0:7}" ACTION=${{ inputs.restart_only && '"restarted"' || '"deployed"' }} STATUS=${{ job.status == 'success' && '":rocket:"' || '":x:"' }} TEXT="${STATUS} *${{ github.repository }}* ${ACTION} \`${IMAGE_TAG}\` on \`${DOMAIN}:${RELAY_PORT}\`" + OV="" + [ "$CC_MVFST" != "default" ] && OV+=" cc_mvfst=${CC_MVFST}" + [ "$CC_PICO" != "default" ] && OV+=" cc_pico=${CC_PICO}" + [ "$SKIP_PROBE_RTT" != "true" ] && OV+=" bbr_skip_probe_rtt=false" + [ "$QLOG" != "off" ] && OV+=" qlog=${QLOG}" + [ -n "$OV" ] && [ "${{ inputs.restart_only }}" != "true" ] && TEXT+=" (overrides:${OV})" curl -sf -X POST "$SLACK_WEBHOOK_URL" \ -H "Content-Type: application/json" \ --data "{\"text\": \"${TEXT}\"}" || true diff --git a/docker/config.docker.yaml b/docker/config.docker.yaml index 2b53cd89..7135e31d 100644 --- a/docker/config.docker.yaml +++ b/docker/config.docker.yaml @@ -19,6 +19,8 @@ listener_defaults: max_server_recv_packets_per_loop: ${MOQX_RECV_PKTS} udp_socket_buffer_bytes: ${MOQX_UDP_BUFFER} ignore_path_mtu: ${MOQX_IGNORE_PATH_MTU} + bbr: + probe_rtt_disabled_if_app_limited: ${MOQX_BBR_SKIP_PROBE_RTT} # Two listeners: mvfst on MOQX_PORT and (when enabled) picoquic on # MOQX_PICO_PORT — same cert/versions/endpoint, different stack + port. @@ -38,6 +40,8 @@ listeners: key_file: "${MOQX_KEY}" insecure: ${MOQX_INSECURE} endpoint: "${MOQX_ENDPOINT}" + quic: + cc_algo: ${MOQX_CC} ${MOQX_PICO_LISTENER} services: @@ -62,3 +66,10 @@ admin: track_metrics_enabled: true address: "${MOQX_BIND_ADDR}" plaintext: true + +# The directory is always set, for the admin log routes; MOQX_QLOG_SAMPLE > 0 +# qlogs that fraction of every new mvfst connection. +logging: + qlog: + dir: "${MOQX_QLOG_DIR}" + sample_rate: ${MOQX_QLOG_SAMPLE} diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 5d7a87ae..798a80d8 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -46,7 +46,11 @@ services: volumes: - ${MOQX_CERTS_DIR:-/etc/letsencrypt}:/certs:ro - moqx-coredumps:/var/coredumps + - moqx-qlog:/var/log/moqx/qlog - ${MOQX_ENTRY:-./entrypoint.sh}:/usr/local/bin/entrypoint.sh:ro + # The template the entrypoint renders, from the same checkout, so settings + # added there take effect with any image tag. + - ./config.docker.yaml:/usr/local/share/moqx/config.docker.yaml:ro environment: MOQX_CERT: /certs/live/${DOMAIN}/fullchain.pem MOQX_KEY: /certs/live/${DOMAIN}/privkey.pem @@ -63,6 +67,12 @@ services: # IO worker threads — default matches the entrypoint (4); set to the core # count (e.g. 8) for sub-maximal load testing on the CI runner. MOQX_THREADS: ${MOQX_THREADS:-4} + # Congestion control per listener; empty = entrypoint default (bbr). + MOQX_CC: ${MOQX_CC:-} + MOQX_PICO_CC: ${MOQX_PICO_CC:-} + MOQX_BBR_SKIP_PROBE_RTT: ${MOQX_BBR_SKIP_PROBE_RTT:-} + # Fraction of new mvfst connections to qlog; empty = off. + MOQX_QLOG_SAMPLE: ${MOQX_QLOG_SAMPLE:-} ulimits: core: -1 # QUIC multiplexes many connections over few UDP sockets, so fd pressure is @@ -365,5 +375,6 @@ services: volumes: prometheus-targets: moqx-coredumps: + moqx-qlog: prometheus-data: grafana-data: diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index b5549459..eaa39a7e 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -33,6 +33,12 @@ # MOQX_RECV_PKTS — mvfst max_server_recv_packets_per_loop (default: 256) # MOQX_UDP_BUFFER — relay UDP socket buffer bytes (default: net.core.wmem_max) # MOQX_IGNORE_PATH_MTU — send full-size packets, skip PMTU (default: false) +# MOQX_CC — mvfst listener congestion control (default: bbr; +# bbr|bbr2|bbr2modular|copa|cubic|newreno|none) +# MOQX_PICO_CC — picoquic listener congestion control (default: bbr; +# bbr|bbr1|c4|cubic|dcubic|fast|newreno|prague|reno) +# MOQX_BBR_SKIP_PROBE_RTT — mvfst bbr: skip PROBE_RTT while app-limited +# (default: false) # MOQX_JEMALLOC — LD_PRELOAD jemalloc (~10% speedup). "auto" (default) # probes the multiarch paths; off/false/0 uses the # system allocator; an explicit path forces that lib. @@ -41,6 +47,11 @@ # MOQX_LOGGING — folly XLOG config for the whole stack (empty = baseline INFO); # e.g. DBG2 or "INFO,quic=WARN". moqx promotes it to folly's # FOLLY_LOGGING env var internally. See docs/logging.md. +# MOQX_QLOG_SAMPLE — fraction of new mvfst connections to qlog, 0.0–1.0 +# (default: 0 = none). Fetch a file with the admin +# /logs?connection_id=&type=qlog route. +# MOQX_QLOG_DIR — qlog directory (default: /var/log/moqx/qlog); files older +# than 3 days are deleted at startup. set -e # The whole stack logs via folly XLOG (configured by MOQX_LOGGING, handled in the @@ -114,6 +125,9 @@ export MOQX_SEND_PKTS="${MOQX_SEND_PKTS:-16}" export MOQX_RECV_PKTS="${MOQX_RECV_PKTS:-256}" export MOQX_UDP_BUFFER="${MOQX_UDP_BUFFER:-$(cat /proc/sys/net/core/wmem_max 2>/dev/null || echo 1048576)}" export MOQX_IGNORE_PATH_MTU="${MOQX_IGNORE_PATH_MTU:-false}" +export MOQX_CC="${MOQX_CC:-bbr}" +export MOQX_PICO_CC="${MOQX_PICO_CC:-bbr}" +export MOQX_BBR_SKIP_PROBE_RTT="${MOQX_BBR_SKIP_PROBE_RTT:-false}" # Second (picoquic) listener for dual-stack serving. Opt out with # MOQX_PICO_ENABLE=false. picoquic needs real TLS, so it is auto-disabled under @@ -140,6 +154,8 @@ if [ "$MOQX_PICO_ENABLE" = "true" ]; then key_file: "${MOQX_KEY}" insecure: ${MOQX_INSECURE} endpoint: "${MOQX_ENDPOINT}" + quic: + cc_algo: ${MOQX_PICO_CC} PICO ) else @@ -147,6 +163,13 @@ else fi export MOQX_PICO_LISTENER +case "${MOQX_QLOG_SAMPLE:-}" in off|"") MOQX_QLOG_SAMPLE=0 ;; esac +export MOQX_QLOG_SAMPLE +export MOQX_QLOG_DIR="${MOQX_QLOG_DIR:-/var/log/moqx/qlog}" +if [ -d "$MOQX_QLOG_DIR" ]; then + find "$MOQX_QLOG_DIR" -name '*.qlog' -mmin +4320 -delete 2>/dev/null || true +fi + CONFIG=/tmp/relay.yaml envsubst < /usr/local/share/moqx/config.docker.yaml > "$CONFIG" diff --git a/docker/relay-deploy.sh b/docker/relay-deploy.sh index 42abbf53..354cbd69 100755 --- a/docker/relay-deploy.sh +++ b/docker/relay-deploy.sh @@ -12,6 +12,13 @@ # RELAY_PORT (default 4433) # ADMIN_PORT (default 8000) # MOQX_LOGGING (optional) folly XLOG config; empty = baseline INFO +# MOQX_CC, MOQX_PICO_CC +# (optional) congestion control overrides, see entrypoint.sh; +# empty = entrypoint default +# MOQX_BBR_SKIP_PROBE_RTT (optional) mvfst bbr: skip PROBE_RTT while app-limited; +# empty = true +# MOQX_QLOG_SAMPLE (optional) fraction of new mvfst connections to qlog; +# empty = off # PULL_IMAGE (optional) full image ref to `docker pull` + retag :latest. # Empty → `docker compose pull` (compose's pinned :latest). # ENABLE_STATS "true" → stats stack + public dashboard @@ -35,6 +42,10 @@ PUB_PORT="${STATS_PUBLIC_PORT:-4533}" echo "MOQX_PORT=${RELAY_PORT}" echo "MOQX_ADMIN_PORT=${ADMIN_PORT}" echo "MOQX_LOGGING=${MOQX_LOGGING:-}" + echo "MOQX_CC=${MOQX_CC:-}" + echo "MOQX_PICO_CC=${MOQX_PICO_CC:-}" + echo "MOQX_BBR_SKIP_PROBE_RTT=${MOQX_BBR_SKIP_PROBE_RTT:-true}" + echo "MOQX_QLOG_SAMPLE=${MOQX_QLOG_SAMPLE:-}" echo "MOQX_CPUS=$(nproc)" echo "MOQX_THREADS=$(nproc)" } > .env @@ -125,3 +136,37 @@ if [ "${ENABLE_STATS:-}" = "true" ]; then done ./grafana/publish-public-dashboard.sh publish || echo "::warning::public dashboard publish failed" fi + +# ── override check ─────────────────────────────────────────────────────────── +# Compare the relay's live config with the requested settings; fail on mismatch. +curl -sf "http://127.0.0.1:${ADMIN_PORT}/config" | python3 -c ' +import json, os, sys +cfg = json.load(sys.stdin) +want = { + "mvfst": (os.environ.get("MOQX_CC") or "bbr", + (os.environ.get("MOQX_BBR_SKIP_PROBE_RTT") or "true") == "true"), + "picoquic": (os.environ.get("MOQX_PICO_CC") or "bbr", None), +} +ok = True +for l in cfg["listeners"]: + if l["quic_stack"] not in want: + continue + name = l["name"] + cc, skip = want[l["quic_stack"]] + got_cc = l["quic"]["cc_algo"] + got_skip = l["mvfst"]["bbr"]["probe_rtt_disabled_if_app_limited"] + extra = "" if skip is None else f" probe_rtt_disabled_if_app_limited={str(got_skip).lower()}" + print(f"==> {name}: cc_algo={got_cc}{extra}") + if got_cc != cc or (skip is not None and got_skip != skip): + print(f"::error::{name} is not running the requested congestion control") + ok = False +want_qlog = float(os.environ.get("MOQX_QLOG_SAMPLE") or 0) +qlog = (cfg.get("logging") or {}).get("qlog") or {} +got_qlog = qlog.get("sample_rate", 0.0) +qdir = qlog.get("dir", "-") +print(f"==> qlog: sample_rate={got_qlog:g} dir={qdir}") +if abs(got_qlog - want_qlog) > 1e-6: + print("::error::relay is not running the requested qlog sample rate") + ok = False +sys.exit(0 if ok else 1) +'