From 4948543051c23b8604989e1d75d999bd50331cc5 Mon Sep 17 00:00:00 2001 From: Giovanni Marzot Date: Wed, 30 Sep 2026 18:57:29 -0400 Subject: [PATCH 1/6] ci: congestion control overrides for relay deploy deploy relay gains cc_mvfst, cc_pico and bbr_skip_probe_rtt inputs, passed through relay-deploy.sh, compose and the entrypoint into the config template. Defaults are unchanged (bbr, bbr, off). After deploy, relay-deploy.sh reads /config and fails if a listener is not running the requested setting, since an image older than the template ignores the mvfst values. --- .github/workflows/deploy-relay.yml | 48 +++++++++++++++++++++++++++++- docker/config.docker.yaml | 4 +++ docker/docker-compose.yml | 4 +++ docker/entrypoint.sh | 11 +++++++ docker/relay-deploy.sh | 33 ++++++++++++++++++++ 5 files changed, 99 insertions(+), 1 deletion(-) diff --git a/.github/workflows/deploy-relay.yml b/.github/workflows/deploy-relay.yml index 509ae5742..e49c4dcd7 100644 --- a/.github/workflows/deploy-relay.yml +++ b/.github/workflows/deploy-relay.yml @@ -1,7 +1,8 @@ name: deploy relay # Ops controls for the CI relay (moqx-main.ci.openmoq.org + moqx-000 alias). -# Restart, redeploy with a specific image tag, or change the log level. +# Restart, redeploy with a specific image tag, or change the log level or +# congestion control. Overrides last until the next main-push deploy. on: workflow_dispatch: @@ -33,6 +34,40 @@ on: - "DBG2" - "DBG3" - "DBG4" + cc_mvfst: + description: "mvfst congestion control (default bbr)" + required: false + default: "default" + type: choice + options: + - "default" + - "bbr" + - "bbr2" + - "bbr2modular" + - "copa" + - "cubic" + - "newreno" + cc_pico: + description: "picoquic congestion control (default bbr)" + required: false + default: "default" + type: choice + options: + - "default" + - "bbr" + - "bbr1" + - "c4" + - "cubic" + - "dcubic" + - "fast" + - "newreno" + - "prague" + - "reno" + bbr_skip_probe_rtt: + description: "mvfst bbr: skip PROBE_RTT while app-limited" + required: false + type: boolean + default: false enable_stats: description: "Enable stats dashboard (default enabled)" required: false @@ -144,6 +179,9 @@ jobs: AWS_ACCESS_KEY_ID: ${{ secrets.OMOQ_CERTBOT_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.OMOQ_CERTBOT_SECRET_ACCESS_KEY }} ENABLE_STATS: ${{ inputs.enable_stats }} + MOQX_CC: ${{ inputs.cc_mvfst != 'default' && inputs.cc_mvfst || '' }} + MOQX_PICO_CC: ${{ inputs.cc_pico != 'default' && inputs.cc_pico || '' }} + MOQX_BBR_SKIP_PROBE_RTT: ${{ inputs.bbr_skip_probe_rtt }} STATS_USER: ${{ secrets.STATS_USER }} STATS_PASSWORD: ${{ secrets.STATS_PASSWORD }} GRAFANA_ADMIN_PASSWORD: ${{ secrets.GRAFANA_ADMIN_PASSWORD }} @@ -194,11 +232,19 @@ jobs: SLACK_WEBHOOK_URL: ${{ secrets.OMOQ_SLACK_WEBHOOK_URL }} DOMAIN: ${{ steps.target.outputs.domain }} IMAGE_TAG: ${{ steps.target.outputs.image_tag }} + CC_MVFST: ${{ inputs.cc_mvfst }} + CC_PICO: ${{ inputs.cc_pico }} + SKIP_PROBE_RTT: ${{ inputs.bbr_skip_probe_rtt }} run: | SHORT="${GITHUB_SHA:0:7}" ACTION=${{ inputs.restart_only && '"restarted"' || '"deployed"' }} STATUS=${{ job.status == 'success' && '":rocket:"' || '":x:"' }} TEXT="${STATUS} *${{ github.repository }}* ${ACTION} \`${IMAGE_TAG}\` on \`${DOMAIN}:${RELAY_PORT}\`" + CC="" + [ "$CC_MVFST" != "default" ] && CC+=" mvfst=${CC_MVFST}" + [ "$CC_PICO" != "default" ] && CC+=" pico=${CC_PICO}" + [ "$SKIP_PROBE_RTT" = "true" ] && CC+=" bbr_skip_probe_rtt" + [ -n "$CC" ] && [ "${{ inputs.restart_only }}" != "true" ] && TEXT+=" (cc:${CC})" curl -sf -X POST "$SLACK_WEBHOOK_URL" \ -H "Content-Type: application/json" \ --data "{\"text\": \"${TEXT}\"}" || true diff --git a/docker/config.docker.yaml b/docker/config.docker.yaml index 2b53cd898..8dc452f60 100644 --- a/docker/config.docker.yaml +++ b/docker/config.docker.yaml @@ -19,6 +19,8 @@ listener_defaults: max_server_recv_packets_per_loop: ${MOQX_RECV_PKTS} udp_socket_buffer_bytes: ${MOQX_UDP_BUFFER} ignore_path_mtu: ${MOQX_IGNORE_PATH_MTU} + bbr: + probe_rtt_disabled_if_app_limited: ${MOQX_BBR_SKIP_PROBE_RTT} # Two listeners: mvfst on MOQX_PORT and (when enabled) picoquic on # MOQX_PICO_PORT — same cert/versions/endpoint, different stack + port. @@ -38,6 +40,8 @@ listeners: key_file: "${MOQX_KEY}" insecure: ${MOQX_INSECURE} endpoint: "${MOQX_ENDPOINT}" + quic: + cc_algo: ${MOQX_CC} ${MOQX_PICO_LISTENER} services: diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 5d7a87ae3..c203c3e9f 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -63,6 +63,10 @@ services: # IO worker threads — default matches the entrypoint (4); set to the core # count (e.g. 8) for sub-maximal load testing on the CI runner. MOQX_THREADS: ${MOQX_THREADS:-4} + # Congestion control per listener; empty = entrypoint default (bbr). + MOQX_CC: ${MOQX_CC:-} + MOQX_PICO_CC: ${MOQX_PICO_CC:-} + MOQX_BBR_SKIP_PROBE_RTT: ${MOQX_BBR_SKIP_PROBE_RTT:-} ulimits: core: -1 # QUIC multiplexes many connections over few UDP sockets, so fd pressure is diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index b55494594..d83b9f8c5 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -33,6 +33,12 @@ # MOQX_RECV_PKTS — mvfst max_server_recv_packets_per_loop (default: 256) # MOQX_UDP_BUFFER — relay UDP socket buffer bytes (default: net.core.wmem_max) # MOQX_IGNORE_PATH_MTU — send full-size packets, skip PMTU (default: false) +# MOQX_CC — mvfst listener congestion control (default: bbr; +# bbr|bbr2|bbr2modular|copa|cubic|newreno|none) +# MOQX_PICO_CC — picoquic listener congestion control (default: bbr; +# bbr|bbr1|c4|cubic|dcubic|fast|newreno|prague|reno) +# MOQX_BBR_SKIP_PROBE_RTT — mvfst bbr: skip PROBE_RTT while app-limited +# (default: false) # MOQX_JEMALLOC — LD_PRELOAD jemalloc (~10% speedup). "auto" (default) # probes the multiarch paths; off/false/0 uses the # system allocator; an explicit path forces that lib. @@ -114,6 +120,9 @@ export MOQX_SEND_PKTS="${MOQX_SEND_PKTS:-16}" export MOQX_RECV_PKTS="${MOQX_RECV_PKTS:-256}" export MOQX_UDP_BUFFER="${MOQX_UDP_BUFFER:-$(cat /proc/sys/net/core/wmem_max 2>/dev/null || echo 1048576)}" export MOQX_IGNORE_PATH_MTU="${MOQX_IGNORE_PATH_MTU:-false}" +export MOQX_CC="${MOQX_CC:-bbr}" +export MOQX_PICO_CC="${MOQX_PICO_CC:-bbr}" +export MOQX_BBR_SKIP_PROBE_RTT="${MOQX_BBR_SKIP_PROBE_RTT:-false}" # Second (picoquic) listener for dual-stack serving. Opt out with # MOQX_PICO_ENABLE=false. picoquic needs real TLS, so it is auto-disabled under @@ -140,6 +149,8 @@ if [ "$MOQX_PICO_ENABLE" = "true" ]; then key_file: "${MOQX_KEY}" insecure: ${MOQX_INSECURE} endpoint: "${MOQX_ENDPOINT}" + quic: + cc_algo: ${MOQX_PICO_CC} PICO ) else diff --git a/docker/relay-deploy.sh b/docker/relay-deploy.sh index 42abbf536..008bc231c 100755 --- a/docker/relay-deploy.sh +++ b/docker/relay-deploy.sh @@ -12,6 +12,9 @@ # RELAY_PORT (default 4433) # ADMIN_PORT (default 8000) # MOQX_LOGGING (optional) folly XLOG config; empty = baseline INFO +# MOQX_CC, MOQX_PICO_CC, MOQX_BBR_SKIP_PROBE_RTT +# (optional) congestion control overrides, see entrypoint.sh; +# empty = entrypoint default # PULL_IMAGE (optional) full image ref to `docker pull` + retag :latest. # Empty → `docker compose pull` (compose's pinned :latest). # ENABLE_STATS "true" → stats stack + public dashboard @@ -35,6 +38,9 @@ PUB_PORT="${STATS_PUBLIC_PORT:-4533}" echo "MOQX_PORT=${RELAY_PORT}" echo "MOQX_ADMIN_PORT=${ADMIN_PORT}" echo "MOQX_LOGGING=${MOQX_LOGGING:-}" + echo "MOQX_CC=${MOQX_CC:-}" + echo "MOQX_PICO_CC=${MOQX_PICO_CC:-}" + echo "MOQX_BBR_SKIP_PROBE_RTT=${MOQX_BBR_SKIP_PROBE_RTT:-}" echo "MOQX_CPUS=$(nproc)" echo "MOQX_THREADS=$(nproc)" } > .env @@ -125,3 +131,30 @@ if [ "${ENABLE_STATS:-}" = "true" ]; then done ./grafana/publish-public-dashboard.sh publish || echo "::warning::public dashboard publish failed" fi + +# ── congestion control check ───────────────────────────────────────────────── +# The mvfst settings live in the image's config template, so an older image +# silently ignores them. Compare against the live config and fail on mismatch. +curl -sf "http://127.0.0.1:${ADMIN_PORT}/config" | python3 -c ' +import json, os, sys +want = { + "mvfst": (os.environ.get("MOQX_CC") or "bbr", + os.environ.get("MOQX_BBR_SKIP_PROBE_RTT") == "true"), + "picoquic": (os.environ.get("MOQX_PICO_CC") or "bbr", None), +} +ok = True +for l in json.load(sys.stdin)["listeners"]: + if l["quic_stack"] not in want: + continue + name = l["name"] + cc, skip = want[l["quic_stack"]] + got_cc = l["quic"]["cc_algo"] + got_skip = l["mvfst"]["bbr"]["probe_rtt_disabled_if_app_limited"] + extra = "" if skip is None else f" probe_rtt_disabled_if_app_limited={str(got_skip).lower()}" + print(f"==> {name}: cc_algo={got_cc}{extra}") + if got_cc != cc or (skip is not None and got_skip != skip): + print(f"::error::{name} is not running the requested congestion control" + " (image predates the setting?)") + ok = False +sys.exit(0 if ok else 1) +' From 2a9925cbbccfb91e577b5cadae43d887e26edf79 Mon Sep 17 00:00:00 2001 From: Giovanni Marzot Date: Wed, 30 Sep 2026 19:21:38 -0400 Subject: [PATCH 2/6] ci: qlog sampling knob for relay deploy deploy relay gains a qlog_sample_rate input (off, 0.01, 0.1, 1.0) that sets logging.qlog for new mvfst connections. Off leaves the config unchanged. Files go to a moqx-qlog volume, are fetched with the admin /logs route, and are deleted after 3 days at container start. The post-deploy check also compares the qlog sample rate against /config. --- .github/workflows/deploy-relay.yml | 28 +++++++++++++++++++++------- docker/config.docker.yaml | 3 +++ docker/docker-compose.yml | 4 ++++ docker/entrypoint.sh | 22 ++++++++++++++++++++++ docker/relay-deploy.sh | 21 +++++++++++++++++---- 5 files changed, 67 insertions(+), 11 deletions(-) diff --git a/.github/workflows/deploy-relay.yml b/.github/workflows/deploy-relay.yml index e49c4dcd7..4eb36796d 100644 --- a/.github/workflows/deploy-relay.yml +++ b/.github/workflows/deploy-relay.yml @@ -1,8 +1,9 @@ name: deploy relay # Ops controls for the CI relay (moqx-main.ci.openmoq.org + moqx-000 alias). -# Restart, redeploy with a specific image tag, or change the log level or -# congestion control. Overrides last until the next main-push deploy. +# Restart, redeploy with a specific image tag, or change the log level, +# congestion control or qlog sampling. Overrides last until the next main-push +# deploy. on: workflow_dispatch: @@ -68,6 +69,16 @@ on: required: false type: boolean default: false + qlog_sample_rate: + description: "mvfst qlog: fraction of new connections logged (1.0 is heavy)" + required: false + default: "off" + type: choice + options: + - "off" + - "0.01" + - "0.1" + - "1.0" enable_stats: description: "Enable stats dashboard (default enabled)" required: false @@ -182,6 +193,7 @@ jobs: MOQX_CC: ${{ inputs.cc_mvfst != 'default' && inputs.cc_mvfst || '' }} MOQX_PICO_CC: ${{ inputs.cc_pico != 'default' && inputs.cc_pico || '' }} MOQX_BBR_SKIP_PROBE_RTT: ${{ inputs.bbr_skip_probe_rtt }} + MOQX_QLOG_SAMPLE: ${{ inputs.qlog_sample_rate != 'off' && inputs.qlog_sample_rate || '' }} STATS_USER: ${{ secrets.STATS_USER }} STATS_PASSWORD: ${{ secrets.STATS_PASSWORD }} GRAFANA_ADMIN_PASSWORD: ${{ secrets.GRAFANA_ADMIN_PASSWORD }} @@ -235,16 +247,18 @@ jobs: CC_MVFST: ${{ inputs.cc_mvfst }} CC_PICO: ${{ inputs.cc_pico }} SKIP_PROBE_RTT: ${{ inputs.bbr_skip_probe_rtt }} + QLOG: ${{ inputs.qlog_sample_rate }} run: | SHORT="${GITHUB_SHA:0:7}" ACTION=${{ inputs.restart_only && '"restarted"' || '"deployed"' }} STATUS=${{ job.status == 'success' && '":rocket:"' || '":x:"' }} TEXT="${STATUS} *${{ github.repository }}* ${ACTION} \`${IMAGE_TAG}\` on \`${DOMAIN}:${RELAY_PORT}\`" - CC="" - [ "$CC_MVFST" != "default" ] && CC+=" mvfst=${CC_MVFST}" - [ "$CC_PICO" != "default" ] && CC+=" pico=${CC_PICO}" - [ "$SKIP_PROBE_RTT" = "true" ] && CC+=" bbr_skip_probe_rtt" - [ -n "$CC" ] && [ "${{ inputs.restart_only }}" != "true" ] && TEXT+=" (cc:${CC})" + OV="" + [ "$CC_MVFST" != "default" ] && OV+=" cc_mvfst=${CC_MVFST}" + [ "$CC_PICO" != "default" ] && OV+=" cc_pico=${CC_PICO}" + [ "$SKIP_PROBE_RTT" = "true" ] && OV+=" bbr_skip_probe_rtt" + [ "$QLOG" != "off" ] && OV+=" qlog=${QLOG}" + [ -n "$OV" ] && [ "${{ inputs.restart_only }}" != "true" ] && TEXT+=" (overrides:${OV})" curl -sf -X POST "$SLACK_WEBHOOK_URL" \ -H "Content-Type: application/json" \ --data "{\"text\": \"${TEXT}\"}" || true diff --git a/docker/config.docker.yaml b/docker/config.docker.yaml index 8dc452f60..d39b91192 100644 --- a/docker/config.docker.yaml +++ b/docker/config.docker.yaml @@ -66,3 +66,6 @@ admin: track_metrics_enabled: true address: "${MOQX_BIND_ADDR}" plaintext: true + +# entrypoint.sh fills this with a logging.qlog block when MOQX_QLOG_SAMPLE > 0. +${MOQX_QLOG_BLOCK} diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index c203c3e9f..e9d060f01 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -46,6 +46,7 @@ services: volumes: - ${MOQX_CERTS_DIR:-/etc/letsencrypt}:/certs:ro - moqx-coredumps:/var/coredumps + - moqx-qlog:/var/log/moqx/qlog - ${MOQX_ENTRY:-./entrypoint.sh}:/usr/local/bin/entrypoint.sh:ro environment: MOQX_CERT: /certs/live/${DOMAIN}/fullchain.pem @@ -67,6 +68,8 @@ services: MOQX_CC: ${MOQX_CC:-} MOQX_PICO_CC: ${MOQX_PICO_CC:-} MOQX_BBR_SKIP_PROBE_RTT: ${MOQX_BBR_SKIP_PROBE_RTT:-} + # Fraction of new mvfst connections to qlog; empty = off. + MOQX_QLOG_SAMPLE: ${MOQX_QLOG_SAMPLE:-} ulimits: core: -1 # QUIC multiplexes many connections over few UDP sockets, so fd pressure is @@ -369,5 +372,6 @@ services: volumes: prometheus-targets: moqx-coredumps: + moqx-qlog: prometheus-data: grafana-data: diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index d83b9f8c5..c29384b24 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -47,6 +47,11 @@ # MOQX_LOGGING — folly XLOG config for the whole stack (empty = baseline INFO); # e.g. DBG2 or "INFO,quic=WARN". moqx promotes it to folly's # FOLLY_LOGGING env var internally. See docs/logging.md. +# MOQX_QLOG_SAMPLE — fraction of new mvfst connections to qlog, 0.0–1.0 +# (default: 0 = off). Fetch a file with the admin +# /logs?connection_id=&type=qlog route. +# MOQX_QLOG_DIR — qlog directory (default: /var/log/moqx/qlog); files older +# than 3 days are deleted at startup. set -e # The whole stack logs via folly XLOG (configured by MOQX_LOGGING, handled in the @@ -158,6 +163,23 @@ else fi export MOQX_PICO_LISTENER +export MOQX_QLOG_SAMPLE="${MOQX_QLOG_SAMPLE:-0}" +export MOQX_QLOG_DIR="${MOQX_QLOG_DIR:-/var/log/moqx/qlog}" +if [ -d "$MOQX_QLOG_DIR" ]; then + find "$MOQX_QLOG_DIR" -name '*.qlog' -mtime +3 -delete 2>/dev/null || true +fi +case "$MOQX_QLOG_SAMPLE" in + 0|0.0|off|"") MOQX_QLOG_BLOCK="" ;; + *) MOQX_QLOG_BLOCK=$(cat < "$CONFIG" diff --git a/docker/relay-deploy.sh b/docker/relay-deploy.sh index 008bc231c..288eb714b 100755 --- a/docker/relay-deploy.sh +++ b/docker/relay-deploy.sh @@ -15,6 +15,8 @@ # MOQX_CC, MOQX_PICO_CC, MOQX_BBR_SKIP_PROBE_RTT # (optional) congestion control overrides, see entrypoint.sh; # empty = entrypoint default +# MOQX_QLOG_SAMPLE (optional) fraction of new mvfst connections to qlog; +# empty = off # PULL_IMAGE (optional) full image ref to `docker pull` + retag :latest. # Empty → `docker compose pull` (compose's pinned :latest). # ENABLE_STATS "true" → stats stack + public dashboard @@ -41,6 +43,7 @@ PUB_PORT="${STATS_PUBLIC_PORT:-4533}" echo "MOQX_CC=${MOQX_CC:-}" echo "MOQX_PICO_CC=${MOQX_PICO_CC:-}" echo "MOQX_BBR_SKIP_PROBE_RTT=${MOQX_BBR_SKIP_PROBE_RTT:-}" + echo "MOQX_QLOG_SAMPLE=${MOQX_QLOG_SAMPLE:-}" echo "MOQX_CPUS=$(nproc)" echo "MOQX_THREADS=$(nproc)" } > .env @@ -132,18 +135,19 @@ if [ "${ENABLE_STATS:-}" = "true" ]; then ./grafana/publish-public-dashboard.sh publish || echo "::warning::public dashboard publish failed" fi -# ── congestion control check ───────────────────────────────────────────────── -# The mvfst settings live in the image's config template, so an older image -# silently ignores them. Compare against the live config and fail on mismatch. +# ── override check ─────────────────────────────────────────────────────────── +# The mvfst and qlog settings live in the image's config template, so an older +# image silently ignores them. Compare against the live config; fail on mismatch. curl -sf "http://127.0.0.1:${ADMIN_PORT}/config" | python3 -c ' import json, os, sys +cfg = json.load(sys.stdin) want = { "mvfst": (os.environ.get("MOQX_CC") or "bbr", os.environ.get("MOQX_BBR_SKIP_PROBE_RTT") == "true"), "picoquic": (os.environ.get("MOQX_PICO_CC") or "bbr", None), } ok = True -for l in json.load(sys.stdin)["listeners"]: +for l in cfg["listeners"]: if l["quic_stack"] not in want: continue name = l["name"] @@ -156,5 +160,14 @@ for l in json.load(sys.stdin)["listeners"]: print(f"::error::{name} is not running the requested congestion control" " (image predates the setting?)") ok = False +want_qlog = float(os.environ.get("MOQX_QLOG_SAMPLE") or 0) +qlog = (cfg.get("logging") or {}).get("qlog") or {} +got_qlog = qlog.get("sample_rate", 0.0) +qdir = qlog.get("dir", "-") +print(f"==> qlog: sample_rate={got_qlog:g} dir={qdir}") +if abs(got_qlog - want_qlog) > 1e-6: + print("::error::relay is not running the requested qlog sample rate" + " (image predates the setting?)") + ok = False sys.exit(0 if ok else 1) ' From 27cf640c6d71250980f42a10d7aadda2e308b258 Mon Sep 17 00:00:00 2001 From: Giovanni Marzot Date: Thu, 1 Oct 2026 11:21:48 -0400 Subject: [PATCH 3/6] ci: always set the relay qlog directory The template now always sets logging.qlog.dir, with sample_rate from MOQX_QLOG_SAMPLE (default 0), so the admin /qlog/capture route can capture on demand without sampling every connection. --- docker/config.docker.yaml | 8 ++++++-- docker/entrypoint.sh | 17 ++++------------- 2 files changed, 10 insertions(+), 15 deletions(-) diff --git a/docker/config.docker.yaml b/docker/config.docker.yaml index d39b91192..baa550c79 100644 --- a/docker/config.docker.yaml +++ b/docker/config.docker.yaml @@ -67,5 +67,9 @@ admin: address: "${MOQX_BIND_ADDR}" plaintext: true -# entrypoint.sh fills this with a logging.qlog block when MOQX_QLOG_SAMPLE > 0. -${MOQX_QLOG_BLOCK} +# The directory is always set so the admin API can capture on demand; +# MOQX_QLOG_SAMPLE > 0 also qlogs that fraction of every new mvfst connection. +logging: + qlog: + dir: "${MOQX_QLOG_DIR}" + sample_rate: ${MOQX_QLOG_SAMPLE} diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index c29384b24..e721c69de 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -48,7 +48,8 @@ # e.g. DBG2 or "INFO,quic=WARN". moqx promotes it to folly's # FOLLY_LOGGING env var internally. See docs/logging.md. # MOQX_QLOG_SAMPLE — fraction of new mvfst connections to qlog, 0.0–1.0 -# (default: 0 = off). Fetch a file with the admin +# (default: 0 = on-demand captures only, via the admin +# /qlog/capture route). Fetch a file with the admin # /logs?connection_id=&type=qlog route. # MOQX_QLOG_DIR — qlog directory (default: /var/log/moqx/qlog); files older # than 3 days are deleted at startup. @@ -163,22 +164,12 @@ else fi export MOQX_PICO_LISTENER -export MOQX_QLOG_SAMPLE="${MOQX_QLOG_SAMPLE:-0}" +case "${MOQX_QLOG_SAMPLE:-}" in off|"") MOQX_QLOG_SAMPLE=0 ;; esac +export MOQX_QLOG_SAMPLE export MOQX_QLOG_DIR="${MOQX_QLOG_DIR:-/var/log/moqx/qlog}" if [ -d "$MOQX_QLOG_DIR" ]; then find "$MOQX_QLOG_DIR" -name '*.qlog' -mtime +3 -delete 2>/dev/null || true fi -case "$MOQX_QLOG_SAMPLE" in - 0|0.0|off|"") MOQX_QLOG_BLOCK="" ;; - *) MOQX_QLOG_BLOCK=$(cat < "$CONFIG" From 5248277d424d5e82594f5a50f373319b9cfd3541 Mon Sep 17 00:00:00 2001 From: Giovanni Marzot Date: Sat, 3 Oct 2026 08:37:24 -0400 Subject: [PATCH 4/6] ci: mount the relay config template from the checkout The entrypoint already comes from the checkout; mounting the template it renders as well makes the deploy settings take effect with any image tag. --- docker/docker-compose.yml | 3 +++ docker/relay-deploy.sh | 9 +++------ 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index e9d060f01..798a80d8c 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -48,6 +48,9 @@ services: - moqx-coredumps:/var/coredumps - moqx-qlog:/var/log/moqx/qlog - ${MOQX_ENTRY:-./entrypoint.sh}:/usr/local/bin/entrypoint.sh:ro + # The template the entrypoint renders, from the same checkout, so settings + # added there take effect with any image tag. + - ./config.docker.yaml:/usr/local/share/moqx/config.docker.yaml:ro environment: MOQX_CERT: /certs/live/${DOMAIN}/fullchain.pem MOQX_KEY: /certs/live/${DOMAIN}/privkey.pem diff --git a/docker/relay-deploy.sh b/docker/relay-deploy.sh index 288eb714b..d5c3ce0c8 100755 --- a/docker/relay-deploy.sh +++ b/docker/relay-deploy.sh @@ -136,8 +136,7 @@ if [ "${ENABLE_STATS:-}" = "true" ]; then fi # ── override check ─────────────────────────────────────────────────────────── -# The mvfst and qlog settings live in the image's config template, so an older -# image silently ignores them. Compare against the live config; fail on mismatch. +# Compare the relay's live config with the requested settings; fail on mismatch. curl -sf "http://127.0.0.1:${ADMIN_PORT}/config" | python3 -c ' import json, os, sys cfg = json.load(sys.stdin) @@ -157,8 +156,7 @@ for l in cfg["listeners"]: extra = "" if skip is None else f" probe_rtt_disabled_if_app_limited={str(got_skip).lower()}" print(f"==> {name}: cc_algo={got_cc}{extra}") if got_cc != cc or (skip is not None and got_skip != skip): - print(f"::error::{name} is not running the requested congestion control" - " (image predates the setting?)") + print(f"::error::{name} is not running the requested congestion control") ok = False want_qlog = float(os.environ.get("MOQX_QLOG_SAMPLE") or 0) qlog = (cfg.get("logging") or {}).get("qlog") or {} @@ -166,8 +164,7 @@ got_qlog = qlog.get("sample_rate", 0.0) qdir = qlog.get("dir", "-") print(f"==> qlog: sample_rate={got_qlog:g} dir={qdir}") if abs(got_qlog - want_qlog) > 1e-6: - print("::error::relay is not running the requested qlog sample rate" - " (image predates the setting?)") + print("::error::relay is not running the requested qlog sample rate") ok = False sys.exit(0 if ok else 1) ' From 5b043cc580d394d6d039e035416d623f8586ca5d Mon Sep 17 00:00:00 2001 From: Giovanni Marzot Date: Sun, 4 Oct 2026 11:06:10 -0400 Subject: [PATCH 5/6] ci: qlog docs without the capture route; prune at exactly 72 hours --- docker/config.docker.yaml | 4 ++-- docker/entrypoint.sh | 5 ++--- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/docker/config.docker.yaml b/docker/config.docker.yaml index baa550c79..7135e31d3 100644 --- a/docker/config.docker.yaml +++ b/docker/config.docker.yaml @@ -67,8 +67,8 @@ admin: address: "${MOQX_BIND_ADDR}" plaintext: true -# The directory is always set so the admin API can capture on demand; -# MOQX_QLOG_SAMPLE > 0 also qlogs that fraction of every new mvfst connection. +# The directory is always set, for the admin log routes; MOQX_QLOG_SAMPLE > 0 +# qlogs that fraction of every new mvfst connection. logging: qlog: dir: "${MOQX_QLOG_DIR}" diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index e721c69de..eaa39a7e9 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -48,8 +48,7 @@ # e.g. DBG2 or "INFO,quic=WARN". moqx promotes it to folly's # FOLLY_LOGGING env var internally. See docs/logging.md. # MOQX_QLOG_SAMPLE — fraction of new mvfst connections to qlog, 0.0–1.0 -# (default: 0 = on-demand captures only, via the admin -# /qlog/capture route). Fetch a file with the admin +# (default: 0 = none). Fetch a file with the admin # /logs?connection_id=&type=qlog route. # MOQX_QLOG_DIR — qlog directory (default: /var/log/moqx/qlog); files older # than 3 days are deleted at startup. @@ -168,7 +167,7 @@ case "${MOQX_QLOG_SAMPLE:-}" in off|"") MOQX_QLOG_SAMPLE=0 ;; esac export MOQX_QLOG_SAMPLE export MOQX_QLOG_DIR="${MOQX_QLOG_DIR:-/var/log/moqx/qlog}" if [ -d "$MOQX_QLOG_DIR" ]; then - find "$MOQX_QLOG_DIR" -name '*.qlog' -mtime +3 -delete 2>/dev/null || true + find "$MOQX_QLOG_DIR" -name '*.qlog' -mmin +4320 -delete 2>/dev/null || true fi CONFIG=/tmp/relay.yaml From 4dba1498fdf07a613c9e1ec23cc92a1cc6ef79c0 Mon Sep 17 00:00:00 2001 From: Giovanni Marzot Date: Sun, 4 Oct 2026 13:45:36 -0400 Subject: [PATCH 6/6] ci: skip BBR PROBE_RTT while app-limited by default on relay deploys Both the main-push deploy and the manual deploy turn it on unless told otherwise; the image's own default stays off. --- .github/workflows/deploy-relay.yml | 6 +++--- docker/relay-deploy.sh | 8 +++++--- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/.github/workflows/deploy-relay.yml b/.github/workflows/deploy-relay.yml index 4eb36796d..25471d548 100644 --- a/.github/workflows/deploy-relay.yml +++ b/.github/workflows/deploy-relay.yml @@ -65,10 +65,10 @@ on: - "prague" - "reno" bbr_skip_probe_rtt: - description: "mvfst bbr: skip PROBE_RTT while app-limited" + description: "mvfst bbr: skip PROBE_RTT while app-limited (default on)" required: false type: boolean - default: false + default: true qlog_sample_rate: description: "mvfst qlog: fraction of new connections logged (1.0 is heavy)" required: false @@ -256,7 +256,7 @@ jobs: OV="" [ "$CC_MVFST" != "default" ] && OV+=" cc_mvfst=${CC_MVFST}" [ "$CC_PICO" != "default" ] && OV+=" cc_pico=${CC_PICO}" - [ "$SKIP_PROBE_RTT" = "true" ] && OV+=" bbr_skip_probe_rtt" + [ "$SKIP_PROBE_RTT" != "true" ] && OV+=" bbr_skip_probe_rtt=false" [ "$QLOG" != "off" ] && OV+=" qlog=${QLOG}" [ -n "$OV" ] && [ "${{ inputs.restart_only }}" != "true" ] && TEXT+=" (overrides:${OV})" curl -sf -X POST "$SLACK_WEBHOOK_URL" \ diff --git a/docker/relay-deploy.sh b/docker/relay-deploy.sh index d5c3ce0c8..354cbd694 100755 --- a/docker/relay-deploy.sh +++ b/docker/relay-deploy.sh @@ -12,9 +12,11 @@ # RELAY_PORT (default 4433) # ADMIN_PORT (default 8000) # MOQX_LOGGING (optional) folly XLOG config; empty = baseline INFO -# MOQX_CC, MOQX_PICO_CC, MOQX_BBR_SKIP_PROBE_RTT +# MOQX_CC, MOQX_PICO_CC # (optional) congestion control overrides, see entrypoint.sh; # empty = entrypoint default +# MOQX_BBR_SKIP_PROBE_RTT (optional) mvfst bbr: skip PROBE_RTT while app-limited; +# empty = true # MOQX_QLOG_SAMPLE (optional) fraction of new mvfst connections to qlog; # empty = off # PULL_IMAGE (optional) full image ref to `docker pull` + retag :latest. @@ -42,7 +44,7 @@ PUB_PORT="${STATS_PUBLIC_PORT:-4533}" echo "MOQX_LOGGING=${MOQX_LOGGING:-}" echo "MOQX_CC=${MOQX_CC:-}" echo "MOQX_PICO_CC=${MOQX_PICO_CC:-}" - echo "MOQX_BBR_SKIP_PROBE_RTT=${MOQX_BBR_SKIP_PROBE_RTT:-}" + echo "MOQX_BBR_SKIP_PROBE_RTT=${MOQX_BBR_SKIP_PROBE_RTT:-true}" echo "MOQX_QLOG_SAMPLE=${MOQX_QLOG_SAMPLE:-}" echo "MOQX_CPUS=$(nproc)" echo "MOQX_THREADS=$(nproc)" @@ -142,7 +144,7 @@ import json, os, sys cfg = json.load(sys.stdin) want = { "mvfst": (os.environ.get("MOQX_CC") or "bbr", - os.environ.get("MOQX_BBR_SKIP_PROBE_RTT") == "true"), + (os.environ.get("MOQX_BBR_SKIP_PROBE_RTT") or "true") == "true"), "picoquic": (os.environ.get("MOQX_PICO_CC") or "bbr", None), } ok = True