From 5bfe39a5035c52ff5884d23f1a5e4fb6306806d8 Mon Sep 17 00:00:00 2001 From: dmitrivasilyev Date: Wed, 13 May 2026 14:12:13 +0300 Subject: [PATCH 01/11] test(bdd): use single-quoted strings for JSON substring assertions in web-ui cucumber-rs `{string}` parameter parser does not unescape `\"` inside double-quoted Gherkin strings, so the nine `should contain` assertions on `/api/auth/config` and `/api/config` were searching the curl output for literal `\"key\":\"value\"` substrings with backslashes that never appear in the JSON body. The first three Web UI scenarios (`sso_enabled`, `current_user`, `role`/`source`, four bind-address keys, `changeable`) fail in BDD: Web UI of the post-merge run because of this. Swap the outer Gherkin quotes to single quotes so the embedded double quotes pass through verbatim and the substring search hits the JSON body. Scenario semantics are unchanged. --- tests/bdd/features/web-ui.feature | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/tests/bdd/features/web-ui.feature b/tests/bdd/features/web-ui.feature index c3b6bd6b3..3ae9f53d6 100644 --- a/tests/bdd/features/web-ui.feature +++ b/tests/bdd/features/web-ui.feature @@ -115,8 +115,8 @@ Feature: Web UI listener curl -s http://127.0.0.1:9127/api/auth/config """ Then the command should succeed - And the command output should contain "\"sso_enabled\":false" - And the command output should contain "\"current_user\":null" + And the command output should contain '"sso_enabled":false' + And the command output should contain '"current_user":null' Scenario: /api/auth/config carries admin identity for Basic auth When I run shell command: @@ -124,8 +124,8 @@ Feature: Web UI listener curl -s --user 'admin:webui_bdd' http://127.0.0.1:9127/api/auth/config """ Then the command should succeed - And the command output should contain "\"role\":\"admin\"" - And the command output should contain "\"source\":\"basic\"" + And the command output should contain '"role":"admin"' + And the command output should contain '"source":"basic"' Scenario: /api/admin/reload requires Admin and rejects anonymous with 401 When I run shell command: @@ -164,8 +164,8 @@ Feature: Web UI listener """ Then the command should succeed # Each bind-address key must appear with changeable=no. - And the command output should contain "\"key\":\"general.host\"" - And the command output should contain "\"key\":\"general.port\"" - And the command output should contain "\"key\":\"web.host\"" - And the command output should contain "\"key\":\"web.port\"" - And the command output should contain "\"changeable\":\"no\"" + And the command output should contain '"key":"general.host"' + And the command output should contain '"key":"general.port"' + And the command output should contain '"key":"web.host"' + And the command output should contain '"key":"web.port"' + And the command output should contain '"changeable":"no"' From 5ca7229cb1e8cb57ee57a95a6db5d0d14707b639 Mon Sep 17 00:00:00 2001 From: dmitrivasilyev Date: Wed, 13 May 2026 14:24:59 +0300 Subject: [PATCH 02/11] ci(bdd): prebuild BDD test binaries and reuse cargo cache across the matrix Each of the 23 entries in the `bdd-tests` matrix used to start from a cold workspace, run `cargo download` (1-2 min) and recompile `pg_doorman` plus the BDD test crate (4-9 min) inside the test-runner image before the actual tags ran. The fan-out wasted ~150 CPU-min of duplicate compile per CI run and stretched the wall clock to ~30 minutes even at 6 parallel suites. Add a `prebuild-bdd` job between `prepare-tests` and `bdd-tests`. It checks out the source, restores the cache, pulls the same test-runner image, and compiles `cargo test --test bdd --no-run`, `cargo test --test patroni_proxy_bdd --no-run`, and the `--features tls-migration` variant. `CARGO_HOME=/workspace/.cargo` keeps the registry and the build artefacts inside `github.workspace`, so `actions/cache@v4` stores `.cargo/` and `target/` without touching runner-level paths. Each matrix entry now `needs: prebuild-bdd`, restores the same cache, and reuses the prebuilt binaries. cargo resolves the test against the cached `target/`, so the per-suite cost drops to a link plus the scenario runtime. --- .github/workflows/bdd-tests.yml | 89 ++++++++++++++++++++++++++++++++- 1 file changed, 87 insertions(+), 2 deletions(-) diff --git a/.github/workflows/bdd-tests.yml b/.github/workflows/bdd-tests.yml index b650d646a..fa9a6097e 100644 --- a/.github/workflows/bdd-tests.yml +++ b/.github/workflows/bdd-tests.yml @@ -202,6 +202,76 @@ jobs: echo "✅ Using freshly built image: ${{ needs.check-image.outputs.image-tag }}" fi + # Compile the BDD test binaries once and stash cargo's registry + + # target dir in GHA cache so the 23-way matrix below does not pay a + # full `cargo download + compile` per suite. Default features cover + # all suites except `tls-migration`, which builds in its own target + # subdirectory because of the cargo features-hash. `CARGO_HOME` is + # pinned inside `/workspace/.cargo` so the cache stays under + # `github.workspace` and no host UID/permission tricks are needed. + prebuild-bdd: + name: Prebuild BDD test binaries + needs: prepare-tests + if: always() && needs.prepare-tests.result == 'success' + runs-on: ubuntu-latest + timeout-minutes: 30 + permissions: + contents: read + packages: read + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Restore cargo cache + uses: actions/cache@v4 + with: + path: | + .cargo + target + key: ${{ runner.os }}-bdd-cargo-${{ needs.prepare-tests.outputs.image-tag }}-${{ hashFiles('Cargo.lock', 'src/**', 'tests/**', 'build.rs') }} + restore-keys: | + ${{ runner.os }}-bdd-cargo-${{ needs.prepare-tests.outputs.image-tag }}- + ${{ runner.os }}-bdd-cargo- + + - name: Log in to Container Registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Pull test image from GHCR + uses: nick-fields/retry@v3 + with: + timeout_minutes: 5 + max_attempts: 3 + retry_wait_seconds: 10 + command: docker pull ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ needs.prepare-tests.outputs.image-tag }} + + - name: Compile bdd + patroni_proxy_bdd binaries (default features) + run: | + docker run --rm \ + --network=host \ + -e CARGO_NET_RETRY=10 \ + -e CARGO_HTTP_TIMEOUT=60 \ + -e CARGO_HOME=/workspace/.cargo \ + -v ${{ github.workspace }}:/workspace \ + -w /workspace \ + ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ needs.prepare-tests.outputs.image-tag }} \ + sh -c "cargo test --test bdd --no-run && cargo test --test patroni_proxy_bdd --no-run" + + - name: Compile bdd binary (tls-migration feature) + run: | + docker run --rm \ + --network=host \ + -e CARGO_NET_RETRY=10 \ + -e CARGO_HTTP_TIMEOUT=60 \ + -e CARGO_HOME=/workspace/.cargo \ + -v ${{ github.workspace }}:/workspace \ + -w /workspace \ + ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ needs.prepare-tests.outputs.image-tag }} \ + cargo test --features tls-migration --test bdd --no-run + # Single matrixed job replaces the 21 nearly-identical per-language / # per-suite jobs that used to live here. Each suite differs only in # the cargo command it runs; share everything else. See the cargo @@ -210,8 +280,8 @@ jobs: # sensitive lifecycle scenarios). bdd-tests: name: 'BDD: ${{ matrix.suite.name }}' - needs: prepare-tests - if: always() && needs.prepare-tests.result == 'success' + needs: [ prepare-tests, prebuild-bdd ] + if: always() && needs.prepare-tests.result == 'success' && needs.prebuild-bdd.result == 'success' runs-on: ubuntu-latest timeout-minutes: 30 permissions: @@ -255,6 +325,20 @@ jobs: - name: Checkout repository uses: actions/checkout@v4 + - name: Restore cargo cache from prebuild + # Same key shape as prebuild-bdd. Restore-keys fall back to any + # earlier prebuild for this image tag, then to any prebuild on + # this OS, so a matrix entry never starts from a cold cargo. + uses: actions/cache@v4 + with: + path: | + .cargo + target + key: ${{ runner.os }}-bdd-cargo-${{ needs.prepare-tests.outputs.image-tag }}-${{ hashFiles('Cargo.lock', 'src/**', 'tests/**', 'build.rs') }} + restore-keys: | + ${{ runner.os }}-bdd-cargo-${{ needs.prepare-tests.outputs.image-tag }}- + ${{ runner.os }}-bdd-cargo- + - name: Log in to Container Registry # Even when test-runner is currently a public package, the BDD # jobs depend on `docker pull` succeeding. Without explicit @@ -312,6 +396,7 @@ jobs: --network=host \ -e CARGO_NET_RETRY=10 \ -e CARGO_HTTP_TIMEOUT=60 \ + -e CARGO_HOME=/workspace/.cargo \ -v ${{ github.workspace }}:/workspace \ -w /workspace \ ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ needs.prepare-tests.outputs.image-tag }} \ From cb0a52921104ff8627c8d50ca8340bdfc339718e Mon Sep 17 00:00:00 2001 From: dmitrivasilyev Date: Wed, 13 May 2026 14:55:32 +0300 Subject: [PATCH 03/11] docs(binary-upgrade): replace cp with install and switch the systemd unit to Type=notify `cp pg_doorman_new /usr/bin/pg_doorman` rewrites the live inode the running pg_doorman is memory-mapped from. The process can take SIGBUS or SIGSEGV mid-upgrade because its text segment is being overwritten in place. `install -m 0755` writes to a temporary file and renames it onto the target, so the running inode survives and only new processes pick up the new file. The shipped `pg_doorman.service` already uses `Type=notify`, but the tutorial example showed `Type=simple`. With `Type=simple` systemd marks the unit `active` immediately after exec without waiting for `READY=1`, and the new process spawned by `SIGUSR2` cannot hand off `MainPID` to itself. `Type=notify` + `NotifyAccess=all` lines up the tutorial with the real readiness contract: pg_doorman sends `READY=1` once listeners are bound and `MAINPID=` during an upgrade, both via `sd_notify`. `ExecReload=SIGHUP` covers config reload; `kill -USR2` and `UPGRADE;` still drive the binary upgrade. --- .../en/src/tutorials/binary-upgrade.md | 23 +++++++++++++---- .../ru/src/tutorials/binary-upgrade.md | 25 +++++++++++++++---- 2 files changed, 38 insertions(+), 10 deletions(-) diff --git a/documentation/en/src/tutorials/binary-upgrade.md b/documentation/en/src/tutorials/binary-upgrade.md index eb96497c8..7cd8c3baa 100644 --- a/documentation/en/src/tutorials/binary-upgrade.md +++ b/documentation/en/src/tutorials/binary-upgrade.md @@ -21,8 +21,11 @@ cipher state with the `tls-migration` build (Linux, opt-in). ## Quick start ```bash -# 1. Drop the new binary in place. -cp pg_doorman_new /usr/bin/pg_doorman +# 1. Replace the binary atomically. `install` writes to a temporary +# file and renames it onto the target, so the inode the running +# process is mapped from is not overwritten in place. `cp` modifies +# the live inode and risks SIGBUS / SIGSEGV in the running pg_doorman. +install -m 0755 pg_doorman_new /usr/bin/pg_doorman # 2. Validate the new binary against the live config before triggering # the upgrade. SIGUSR2 also runs `-t` and aborts on failure, but @@ -321,15 +324,25 @@ SIGINT triggers binary upgrade in daemon mode or without a TTY (e.g. when spawne | Recommended for | systemd, containers, k8s | Legacy deployments | For zero-downtime upgrades with client migration, run in foreground -mode. systemd manages the process lifecycle: +mode. systemd manages the process lifecycle. Use `Type=notify` so the +unit reaches `active` only after pg_doorman signals readiness, and the +child process can update `MainPID` to itself during `SIGUSR2` upgrades: ```ini [Service] -Type=simple +Type=notify +NotifyAccess=all ExecStart=/usr/bin/pg_doorman /etc/pg_doorman.yaml -ExecReload=/bin/kill -SIGUSR2 $MAINPID +ExecReload=/bin/kill -SIGHUP $MAINPID ``` +`NotifyAccess=all` is required because the new process the upgrade +spawns sends `sd_notify MAINPID=` to systemd. Without it, +systemd silently rejects the MainPID handoff and keeps tracking the +dying old PID. `ExecReload=SIGHUP` covers config reload; the binary +upgrade is triggered by `kill -USR2 $MAINPID` or `UPGRADE;` from the +admin console. + ## Configuration ### `shutdown_timeout` diff --git a/documentation/ru/src/tutorials/binary-upgrade.md b/documentation/ru/src/tutorials/binary-upgrade.md index ad9682f84..9aae26b6a 100644 --- a/documentation/ru/src/tutorials/binary-upgrade.md +++ b/documentation/ru/src/tutorials/binary-upgrade.md @@ -25,8 +25,12 @@ OpenSSL. ## Быстрый старт ```bash -# 1. Положите новый бинарник на место. -cp pg_doorman_new /usr/bin/pg_doorman +# 1. Атомарно замените бинарник. `install` пишет во временный файл и +# переименовывает его на целевой путь, поэтому inode, из которого +# отображены сегменты работающего процесса, не перезаписывается. +# `cp` правит живой inode и может уронить старый pg_doorman через +# SIGBUS / SIGSEGV. +install -m 0755 pg_doorman_new /usr/bin/pg_doorman # 2. Проверьте, что новый бинарник принимает текущий конфиг, до того # как запускать обновление. SIGUSR2 тоже прогоняет `-t` и отменяет @@ -328,15 +332,26 @@ SHA-256 архива проверяется автоматически. | Рекомендуется для | systemd, контейнеры, Kubernetes | Старые установки | Для обновления без простоя и с миграцией клиентов запускайте PgDoorman -в foreground-режиме. systemd управляет жизненным циклом процесса: +в foreground-режиме. systemd управляет жизненным циклом процесса. +Используйте `Type=notify`, тогда unit становится `active` только после +сигнала готовности от pg_doorman, а новый процесс при `SIGUSR2` +обновляет `MainPID` на себя: ```ini [Service] -Type=simple +Type=notify +NotifyAccess=all ExecStart=/usr/bin/pg_doorman /etc/pg_doorman.yaml -ExecReload=/bin/kill -SIGUSR2 $MAINPID +ExecReload=/bin/kill -SIGHUP $MAINPID ``` +`NotifyAccess=all` обязателен: новый процесс, который порождает +upgrade, шлёт systemd `sd_notify MAINPID=`. Без `all` systemd +молча отбрасывает этот сигнал и продолжает считать MainPID-ом +умирающий старый процесс. `ExecReload=SIGHUP` отвечает за reload +конфига; обновление бинаря по-прежнему запускается через +`kill -USR2 $MAINPID` или `UPGRADE;` из админ-консоли. + ## Конфигурация ### `shutdown_timeout` From 03b1029591dcf929a93cd165d22c4f6ec170381e Mon Sep 17 00:00:00 2001 From: dmitrivasilyev Date: Wed, 13 May 2026 14:59:14 +0300 Subject: [PATCH 04/11] docs(binary-upgrade): tighten the devops surface for the upgrade tutorial Four operator-facing issues left over after the cp/Type=notify fix: - `kill -USR2 $(pgrep -f /usr/bin/pg_doorman)` matches command line and signals every pg_doorman process on the host. On a multi-tenant box or after a stuck previous upgrade the cleanup signal lands on neighbours. Switch the Quick start to `systemctl kill -s SIGUSR2 pg_doorman.service`, which targets the single tracked MainPID, and add a stand-alone tip for the PID-file path. - Verification step relied on `pgrep -f /usr/bin/pg_doorman`, which cannot tell the old PID from the new one once both are alive during the drain window. Replace it with `systemctl show -p MainPID --value`, the same field `Type=notify` updates from `MAINPID=`. - The shipped systemd example in "Daemon vs foreground" lacked the production knobs every operator ends up adding: `Restart=on-failure` + `RestartSec=5s` to absorb a single crash without a tight loop, `LimitNOFILE=1048576` to clear the default 1024 fd cap that connection-heavy poolers hit first, `User=pg_doorman`/`Group=pg_doorman` for a non-root listener, `KillMode=mixed` so SIGUSR2 reaches the parent and the children, and `TimeoutStopSec=120` aligned with `shutdown_timeout`. Each setting has a one-line comment explaining why it is there. - Operational checklist still asked operators to verify `ExecReload=/bin/kill -SIGUSR2 $MAINPID`. The new unit uses SIGHUP for reload; SIGUSR2 is the binary-upgrade trigger and lives outside `ExecReload`. Updated the checklist item to match. - Added a tip above Quick start naming `apt-get install --only-upgrade pg-doorman` / `dnf upgrade pg-doorman` as the preferred path when a package is in scope, so direct-binary `install -m 0755` is positioned as the fallback rather than the recommendation. EN and RU tutorials updated in lock-step. --- .../en/src/tutorials/binary-upgrade.md | 73 ++++++++++++++++--- .../ru/src/tutorials/binary-upgrade.md | 70 +++++++++++++++--- 2 files changed, 119 insertions(+), 24 deletions(-) diff --git a/documentation/en/src/tutorials/binary-upgrade.md b/documentation/en/src/tutorials/binary-upgrade.md index 7cd8c3baa..61478dc24 100644 --- a/documentation/en/src/tutorials/binary-upgrade.md +++ b/documentation/en/src/tutorials/binary-upgrade.md @@ -20,6 +20,15 @@ cipher state with the `tls-migration` build (Linux, opt-in). ## Quick start +```admonish tip title="Use the distro package whenever you can" +On hosts where pg_doorman comes from `apt install pg-doorman` / +`dnf install pg-doorman`, the package manager handles step 1 atomically +(`dpkg` stages into a temp path and renames into place; `rpm` does the +same). `apt-get install --only-upgrade pg-doorman` or `dnf upgrade +pg-doorman` is the idiomatic devops path. The manual `install` below is +for direct-binary deployments where no package manager is in scope. +``` + ```bash # 1. Replace the binary atomically. `install` writes to a temporary # file and renames it onto the target, so the inode the running @@ -33,12 +42,28 @@ install -m 0755 pg_doorman_new /usr/bin/pg_doorman # touching the running server. /usr/bin/pg_doorman -t /etc/pg_doorman.yaml -# 3. Trigger the upgrade. -kill -USR2 $(pgrep -f /usr/bin/pg_doorman) +# 3. Trigger the upgrade. Under systemd `Type=notify` the unit +# delivers SIGUSR2 to the tracked MainPID; this picks the single +# correct process even when other pg_doorman instances or stale +# child processes are running on the host. Direct +# `kill -USR2 $(pgrep -f /usr/bin/pg_doorman)` works but matches by +# command line and can target every instance on the host at once, +# which is why packaged installs go through systemctl. +sudo systemctl kill -s SIGUSR2 pg_doorman.service + +# 4. Verify: systemd tracks the new MainPID (Type=notify receives +# `MAINPID=` from the child during the handoff). Active +# state and the admin console confirm clients are still attached. +systemctl show -p MainPID --value pg_doorman.service +psql -h pgdoorman -p 6432 -c 'SHOW POOLS;' # served by the new process +``` -# 4. Verify: a new PID is running and clients are still connected. -pgrep -f /usr/bin/pg_doorman # new PID -psql -h pgdoorman -p 6432 -c 'SHOW POOLS;' # answers from new process +```admonish tip title="Standalone (no systemd) deployments" +If the unit is not running under systemd, read the PID file the daemon +writes (`daemon_pid_file`, default `/tmp/pg_doorman.pid`) instead of +parsing `pgrep`: `kill -USR2 "$(cat /var/run/pg_doorman/pg_doorman.pid)"`. +Foreground deployments not managed by systemd should keep the PID of +the supervising process and signal that one directly. ``` The same upgrade can be triggered from the admin console: @@ -331,17 +356,39 @@ child process can update `MainPID` to itself during `SIGUSR2` upgrades: ```ini [Service] Type=notify +# NotifyAccess=all lets the child process spawned by SIGUSR2 reach +# systemd's notify socket and hand off MainPID; without it, systemd +# silently keeps tracking the dying old PID. NotifyAccess=all ExecStart=/usr/bin/pg_doorman /etc/pg_doorman.yaml ExecReload=/bin/kill -SIGHUP $MAINPID +# Survive a crash, but back off so a restart loop does not amplify a +# bad release. on-failure ignores intentional `systemctl stop`. +Restart=on-failure +RestartSec=5s +# pg_doorman is connection-heavy: each client + each backend uses an +# fd, plus internal pipes. The default soft limit of 1024 is the first +# wall hit in production. +LimitNOFILE=1048576 +# Run as a non-privileged service account that owns +# /var/run/pg_doorman and the PID file. Avoid root if the listener +# uses a port above 1024. +User=pg_doorman +Group=pg_doorman +# kill-control: SIGUSR2 must reach both the parent and any children +# during an upgrade, and the final SIGTERM at stop must reach the +# leader directly. mixed delivers the stop signal to the main pid and +# the cleanup signal to the cgroup. +KillMode=mixed +TimeoutStopSec=120 ``` -`NotifyAccess=all` is required because the new process the upgrade -spawns sends `sd_notify MAINPID=` to systemd. Without it, -systemd silently rejects the MainPID handoff and keeps tracking the -dying old PID. `ExecReload=SIGHUP` covers config reload; the binary -upgrade is triggered by `kill -USR2 $MAINPID` or `UPGRADE;` from the -admin console. +`ExecReload=SIGHUP` covers config reload; the binary upgrade itself is +triggered separately by `systemctl kill -s SIGUSR2 pg_doorman.service` +or by `UPGRADE;` from the admin console. Adjust `TimeoutStopSec` to +match the value of `shutdown_timeout` in your `pg_doorman.yaml`, plus +a small margin, so systemd does not SIGKILL the process before the +drain finishes. ## Configuration @@ -514,7 +561,9 @@ Before rolling out binary upgrade to production: both processes use the same certificate and key files - [ ] Test the upgrade in staging: open a session, trigger SIGUSR2, verify the session continues working -- [ ] Verify systemd unit has `ExecReload=/bin/kill -SIGUSR2 $MAINPID` +- [ ] Verify the systemd unit is `Type=notify` with `NotifyAccess=all`, + `ExecReload=/bin/kill -SIGHUP $MAINPID` for config reload, and a + `TimeoutStopSec` that covers `shutdown_timeout` plus margin - [ ] Monitor logs for migration errors after the first production upgrade - [ ] Confirm old process exits (check PID file or `pgrep`) diff --git a/documentation/ru/src/tutorials/binary-upgrade.md b/documentation/ru/src/tutorials/binary-upgrade.md index 9aae26b6a..a07fabc11 100644 --- a/documentation/ru/src/tutorials/binary-upgrade.md +++ b/documentation/ru/src/tutorials/binary-upgrade.md @@ -24,6 +24,15 @@ OpenSSL. ## Быстрый старт +```admonish tip title="Используйте дистрибутивный пакет, если он есть" +Если pg_doorman поставлен через `apt install pg-doorman` или +`dnf install pg-doorman`, пакетный менеджер выполняет шаг 1 атомарно +(`dpkg` пишет во временный путь и переименовывает, `rpm` делает то +же). Идиоматичный путь — `apt-get install --only-upgrade pg-doorman` +или `dnf upgrade pg-doorman`. Ручной `install` ниже — для разворотов +прямо бинарём, без пакетного менеджера. +``` + ```bash # 1. Атомарно замените бинарник. `install` пишет во временный файл и # переименовывает его на целевой путь, поэтому inode, из которого @@ -38,12 +47,27 @@ install -m 0755 pg_doorman_new /usr/bin/pg_doorman # не трогая работающий сервер. /usr/bin/pg_doorman -t /etc/pg_doorman.yaml -# 3. Запустите обновление. -kill -USR2 $(pgrep -f /usr/bin/pg_doorman) +# 3. Запустите обновление. При systemd `Type=notify` команда шлёт +# SIGUSR2 в MainPID, который отслеживает systemd, — выбирается +# ровно один процесс, даже если на хосте есть другие инстансы +# pg_doorman или зомби-чайлды. Прямой `kill -USR2 $(pgrep -f +# /usr/bin/pg_doorman)` работает, но pgrep матчит по командной +# строке и может попасть во все инстансы сразу — поэтому +# пакетные установки используют systemctl. +sudo systemctl kill -s SIGUSR2 pg_doorman.service + +# 4. Проверьте: systemd подхватил новый MainPID (Type=notify получает +# `MAINPID=` от чайлда при handoff). Состояние active и +# админ-консоль подтверждают, что клиенты на месте. +systemctl show -p MainPID --value pg_doorman.service +psql -h pgdoorman -p 6432 -c 'SHOW POOLS;' # отвечает новый процесс +``` -# 4. Убедитесь: запущен новый PID, клиенты на месте. -pgrep -f /usr/bin/pg_doorman # новый PID -psql -h pgdoorman -p 6432 -c 'SHOW POOLS;' # отвечает уже новый процесс +```admonish tip title="Stand-alone установки без systemd" +Если процесс не управляется systemd, читайте PID-файл, который пишет +daemon (`daemon_pid_file`, по умолчанию `/tmp/pg_doorman.pid`), вместо +парсинга `pgrep`: `kill -USR2 "$(cat /var/run/pg_doorman/pg_doorman.pid)"`. +Foreground без systemd — шлите сигнал в PID супервизора процесса. ``` То же обновление можно запустить из консоли администратора: @@ -340,17 +364,37 @@ SHA-256 архива проверяется автоматически. ```ini [Service] Type=notify +# NotifyAccess=all нужен, чтобы новый процесс, который порождает +# SIGUSR2, мог достучаться до notify-сокета systemd и передать +# MainPID. Без него systemd молча продолжает следить за старым, +# умирающим PID. NotifyAccess=all ExecStart=/usr/bin/pg_doorman /etc/pg_doorman.yaml ExecReload=/bin/kill -SIGHUP $MAINPID +# Перезапускать после падения, но с паузой, чтобы плохой релиз не +# зациклился. on-failure не реагирует на штатный `systemctl stop`. +Restart=on-failure +RestartSec=5s +# pg_doorman тяжёл по дескрипторам: каждый клиент и каждый backend — +# отдельный fd, плюс служебные pipe. Дефолтный soft-лимит 1024 — это +# первая стена в продакшене. +LimitNOFILE=1048576 +# Сервисная учётка, которой принадлежат /var/run/pg_doorman и PID-файл. +# Использовать root не нужно, если listener слушает порт выше 1024. +User=pg_doorman +Group=pg_doorman +# Доставка сигналов: SIGUSR2 при upgrade должен дойти и до родителя, +# и до чайлдов; финальный SIGTERM при stop — в лидера группы. mixed +# шлёт stop-сигнал главному pid, а cleanup — в cgroup. +KillMode=mixed +TimeoutStopSec=120 ``` -`NotifyAccess=all` обязателен: новый процесс, который порождает -upgrade, шлёт systemd `sd_notify MAINPID=`. Без `all` systemd -молча отбрасывает этот сигнал и продолжает считать MainPID-ом -умирающий старый процесс. `ExecReload=SIGHUP` отвечает за reload -конфига; обновление бинаря по-прежнему запускается через -`kill -USR2 $MAINPID` или `UPGRADE;` из админ-консоли. +`ExecReload=SIGHUP` отвечает за reload конфига; обновление бинаря +запускается отдельно — `systemctl kill -s SIGUSR2 pg_doorman.service` +или `UPGRADE;` из админ-консоли. `TimeoutStopSec` подбирайте под +значение `shutdown_timeout` в `pg_doorman.yaml` плюс небольшой запас, +иначе systemd прибьёт процесс SIGKILL до окончания дренажа. ## Конфигурация @@ -527,7 +571,9 @@ Raw fd клиента недоступен или `dup()` не удался. В оба процесса используют одинаковые файлы сертификата и ключа - [ ] Протестировать обновление на staging: открыть сессию, отправить SIGUSR2, убедиться что сессия продолжает работать -- [ ] В systemd unit есть `ExecReload=/bin/kill -SIGUSR2 $MAINPID` +- [ ] В systemd unit указано `Type=notify` с `NotifyAccess=all`, + `ExecReload=/bin/kill -SIGHUP $MAINPID` для reload конфига и + `TimeoutStopSec`, покрывающий `shutdown_timeout` с запасом - [ ] Мониторинг логов на ошибки миграции после первого промышленного обновления - [ ] Подтвердить что старый процесс завершился (PID-файл или `pgrep`) From 2f978f3373082350d1a755fee7a8bfebb4cbd6ba Mon Sep 17 00:00:00 2001 From: dmitrivasilyev Date: Wed, 13 May 2026 15:10:54 +0300 Subject: [PATCH 05/11] fix(logger): auto-disable ANSI colours when stderr is not a TTY and honour NO_COLOR `TextLogger` writes `\x1b[31m`-style colour escapes to stderr whenever `args.no_color` is false. Under systemd that stderr is the journal pipe, not a terminal, so the escape bytes ended up inside the `MESSAGE` field and journalctl rendered every record as `[NNN blob data]`. Operators reading `journalctl -u pg-pooler` could not see the actual log text without remembering to pass `--no-color` or pin `Environment=NO_COLOR=...`. `clap`'s `env` attribute on the bool flag also made `NO_COLOR=1` (the canonical https://no-color.org/ value) crash startup because the parser only accepts `true`/`false`. Wire `should_use_color` between args and `TextLogger::new`. The pure `resolve_color(no_color_flag, env_no_color, stderr_is_tty)` predicate now gates the colour escapes on all three signals: explicit `--no-color`, any non-empty `NO_COLOR` value, and `stderr.is_terminal()`. The four cases of that predicate are covered by unit tests. Drop the `env` attribute on `args.no_color` so a `NO_COLOR=1` env var no longer fails clap parsing; the logger reads the standard's spelling directly via `var_os`. --- src/app/args.rs | 8 +++++-- src/app/logger.rs | 59 ++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 64 insertions(+), 3 deletions(-) diff --git a/src/app/args.rs b/src/app/args.rs index 4255d7433..d0d58beea 100644 --- a/src/app/args.rs +++ b/src/app/args.rs @@ -18,12 +18,16 @@ pub struct Args { #[clap(short='F', long, value_enum, default_value_t=LogFormat::Text, env)] pub log_format: LogFormat, + /// `env` is intentionally omitted: clap parses bool env vars as + /// `"true"`/`"false"` and would reject `NO_COLOR=1` at startup. + /// The logger respects the `NO_COLOR` env var per + /// independently of this flag and also + /// auto-disables colors when stderr is not a TTY. #[arg( short, long, default_value_t = false, - env, - help = "disable colors in the log output" + help = "force colors off in the log output" )] pub no_color: bool, diff --git a/src/app/logger.rs b/src/app/logger.rs index 153cc559b..bf1e211e6 100644 --- a/src/app/logger.rs +++ b/src/app/logger.rs @@ -40,13 +40,36 @@ fn init(args: &Args, syslog_name: Option) { } else { let inner: Box = match args.log_format { LogFormat::Structured => Box::new(JsonLogger::new()), - _ => Box::new(TextLogger::new(!args.no_color)), + _ => Box::new(TextLogger::new(should_use_color(args.no_color))), }; LogLevelController::new(inner, startup_level).register(); } } +/// Decide whether the text logger should emit ANSI colour escapes. +/// +/// The runtime answer threads three inputs through [`resolve_color`]: +/// the explicit `--no-color` flag, the standard `NO_COLOR` env var +/// (: any non-empty value disables colour), and +/// whether stderr is a TTY. Under systemd the unit's stderr is a pipe +/// to journald, not a terminal, so colour escapes used to land in the +/// journal as `[NNN blob data]` placeholders; auto-disabling for a +/// non-TTY stderr fixes that without requiring every operator to pass +/// `--no-color`. +fn should_use_color(no_color_flag: bool) -> bool { + use std::io::IsTerminal; + let env_no_color = std::env::var_os("NO_COLOR").is_some_and(|v| !v.is_empty()); + resolve_color(no_color_flag, env_no_color, std::io::stderr().is_terminal()) +} + +/// Pure decision used by [`should_use_color`]: colour is on only when +/// every gate is open. Split out so the matrix of inputs can be unit +/// tested without touching real env vars or a real terminal handle. +fn resolve_color(no_color_flag: bool, env_no_color: bool, stderr_is_tty: bool) -> bool { + !no_color_flag && !env_no_color && stderr_is_tty +} + /// Direct text logger — no tracing bridge. /// Format: `2024-01-07T19:19:38.080Z INFO pg_doorman::pool: message` struct TextLogger { @@ -169,3 +192,37 @@ impl Log for JsonLogger { let _ = std::io::stderr().flush(); } } + +#[cfg(test)] +mod tests { + use super::resolve_color; + + #[test] + fn color_on_only_when_every_gate_is_open() { + assert!(resolve_color( + /*no_color_flag=*/ false, /*env_no_color=*/ false, + /*stderr_is_tty=*/ true, + )); + } + + #[test] + fn color_off_when_no_color_flag_set() { + assert!(!resolve_color(true, false, true)); + } + + #[test] + fn color_off_when_no_color_env_set() { + // `NO_COLOR` is the open standard from https://no-color.org/: + // any non-empty value disables colour. The pure decision sees + // the boolean after the env lookup. + assert!(!resolve_color(false, true, true)); + } + + #[test] + fn color_off_when_stderr_is_not_a_tty() { + // The fix this guard exists for: under systemd the journal + // pipe is not a terminal, so default colour-on used to leak + // ANSI escapes into journalctl as `[NNN blob data]`. + assert!(!resolve_color(false, false, false)); + } +} From 41a68d7ea3d369734d883bd12d412acc7ca0b646 Mon Sep 17 00:00:00 2001 From: dmitrivasilyev Date: Wed, 13 May 2026 15:18:03 +0300 Subject: [PATCH 06/11] ci(bdd): retry docker login against ghcr.io so a single auth timeout no longer fails the whole job `docker/login-action@v3` calls `docker login` exactly once. When ghcr.io's auth endpoint times out mid-handshake (`Error response from daemon: Get \"https://ghcr.io/v2/\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)`) the entire BDD job, the prebuild job, or the image-check job dies before its real work starts. Outer retries on `docker pull` already handle the pull leg of the same flake, but the login step was a single point of failure. Replace every `Log in to Container Registry` step in `bdd-tests.yml` with `nick-fields/retry@v3` wrapping a plain `docker login ... --password-stdin`. Three attempts with a 10-second back-off match the policy we already use for `docker pull`, and the credential is passed via stdin so it never appears on the command line or in the process listing. --- .github/workflows/bdd-tests.yml | 90 +++++++++++++++++++++++++-------- 1 file changed, 69 insertions(+), 21 deletions(-) diff --git a/.github/workflows/bdd-tests.yml b/.github/workflows/bdd-tests.yml index fa9a6097e..e4347579a 100644 --- a/.github/workflows/bdd-tests.yml +++ b/.github/workflows/bdd-tests.yml @@ -40,11 +40,21 @@ jobs: echo "Computed image tag: flake-${FLAKE_HASH}" - name: Log in to Container Registry - uses: docker/login-action@v3 + # ghcr.io occasionally times out on the auth handshake under + # GHA load (Client.Timeout exceeded while awaiting headers). + # docker/login-action runs once; wrap docker login in + # nick-fields/retry so a transient handshake failure does not + # take out the whole BDD job. + uses: nick-fields/retry@v3 + env: + GHCR_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GHCR_ACTOR: ${{ github.actor }} with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} + timeout_minutes: 3 + max_attempts: 3 + retry_wait_seconds: 10 + command: | + echo "$GHCR_TOKEN" | docker login ${{ env.REGISTRY }} -u "$GHCR_ACTOR" --password-stdin - name: Check if image exists in registry id: check-exists @@ -125,11 +135,21 @@ jobs: echo "Image built successfully for ${{ matrix.arch }}" - name: Log in to Container Registry - uses: docker/login-action@v3 + # ghcr.io occasionally times out on the auth handshake under + # GHA load (Client.Timeout exceeded while awaiting headers). + # docker/login-action runs once; wrap docker login in + # nick-fields/retry so a transient handshake failure does not + # take out the whole BDD job. + uses: nick-fields/retry@v3 + env: + GHCR_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GHCR_ACTOR: ${{ github.actor }} with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} + timeout_minutes: 3 + max_attempts: 3 + retry_wait_seconds: 10 + command: | + echo "$GHCR_TOKEN" | docker login ${{ env.REGISTRY }} -u "$GHCR_ACTOR" --password-stdin - name: Load and push arch-specific image run: | @@ -150,11 +170,21 @@ jobs: packages: write steps: - name: Log in to Container Registry - uses: docker/login-action@v3 + # ghcr.io occasionally times out on the auth handshake under + # GHA load (Client.Timeout exceeded while awaiting headers). + # docker/login-action runs once; wrap docker login in + # nick-fields/retry so a transient handshake failure does not + # take out the whole BDD job. + uses: nick-fields/retry@v3 + env: + GHCR_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GHCR_ACTOR: ${{ github.actor }} with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} + timeout_minutes: 3 + max_attempts: 3 + retry_wait_seconds: 10 + command: | + echo "$GHCR_TOKEN" | docker login ${{ env.REGISTRY }} -u "$GHCR_ACTOR" --password-stdin - name: Create and push multi-arch manifest # Only the hash-tagged manifest is published. The image is @@ -234,11 +264,21 @@ jobs: ${{ runner.os }}-bdd-cargo- - name: Log in to Container Registry - uses: docker/login-action@v3 + # ghcr.io occasionally times out on the auth handshake under + # GHA load (Client.Timeout exceeded while awaiting headers). + # docker/login-action runs once; wrap docker login in + # nick-fields/retry so a transient handshake failure does not + # take out the whole BDD job. + uses: nick-fields/retry@v3 + env: + GHCR_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GHCR_ACTOR: ${{ github.actor }} with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} + timeout_minutes: 3 + max_attempts: 3 + retry_wait_seconds: 10 + command: | + echo "$GHCR_TOKEN" | docker login ${{ env.REGISTRY }} -u "$GHCR_ACTOR" --password-stdin - name: Pull test image from GHCR uses: nick-fields/retry@v3 @@ -343,12 +383,20 @@ jobs: # Even when test-runner is currently a public package, the BDD # jobs depend on `docker pull` succeeding. Without explicit # login they would silently break if the package visibility - # ever flips to private. - uses: docker/login-action@v3 + # ever flips to private. Wrapped in nick-fields/retry because + # ghcr.io's auth endpoint occasionally times out under GHA + # load (Client.Timeout exceeded while awaiting headers) and the + # base docker/login-action does not retry on its own. + uses: nick-fields/retry@v3 + env: + GHCR_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GHCR_ACTOR: ${{ github.actor }} with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} + timeout_minutes: 3 + max_attempts: 3 + retry_wait_seconds: 10 + command: | + echo "$GHCR_TOKEN" | docker login ${{ env.REGISTRY }} -u "$GHCR_ACTOR" --password-stdin - name: Pull test image from GHCR uses: nick-fields/retry@v3 From 73426a46c472257f6300f25fe0d43aa71267ddc0 Mon Sep 17 00:00:00 2001 From: dmitrivasilyev Date: Wed, 13 May 2026 15:18:30 +0300 Subject: [PATCH 07/11] docs(basic-usage): refresh --no-color help snapshot to match the new logger behaviour After the previous commit dropped the `env` attribute on `--no-color` and added auto-TTY detection plus `NO_COLOR` handling, the `--help` snapshots in `basic-usage.md` (EN and RU) still showed `disable colors in the log output [env: NO_COLOR=]`. Update the snapshot to the new help text and expand the option description to call out the auto-disable triggers (non-TTY stderr, non-empty `NO_COLOR`). --- documentation/en/src/tutorials/basic-usage.md | 4 ++-- documentation/ru/src/tutorials/basic-usage.md | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/documentation/en/src/tutorials/basic-usage.md b/documentation/en/src/tutorials/basic-usage.md index 98dc0d69b..bc3636187 100644 --- a/documentation/en/src/tutorials/basic-usage.md +++ b/documentation/en/src/tutorials/basic-usage.md @@ -21,7 +21,7 @@ Arguments: Options: -l, --log-level [env: LOG_LEVEL=] [default: INFO] -F, --log-format [env: LOG_FORMAT=] [default: text] [possible values: text, structured, debug] - -n, --no-color disable colors in the log output [env: NO_COLOR=] + -n, --no-color force colors off in the log output -d, --daemon run as daemon [env: DAEMON=] -h, --help Print help -V, --version Print version @@ -34,7 +34,7 @@ Options: | `-d`, `--daemon` | Run in the background. Without this option, the process will run in the foreground.

In daemon mode, setting `daemon_pid_file` and `syslog_prog_name` is required. No log messages will be written to stderr after going into the background. | | `-l`, `--log-level` | Set log level: `INFO`, `DEBUG`, or `WARN`. | | `-F`, `--log-format` | Set log format. Possible values: `text`, `structured`, `debug`. | -| `-n`, `--no-color` | Disable colors in the log output. | +| `-n`, `--no-color` | Force colors off in the log output. Colors are also auto-disabled when stderr is not a TTY (under systemd, the journal pipe is not a terminal) and when the `NO_COLOR` environment variable is set to any non-empty value. | | `-V`, `--version` | Show version information. | | `-h`, `--help` | Show help information. | diff --git a/documentation/ru/src/tutorials/basic-usage.md b/documentation/ru/src/tutorials/basic-usage.md index 3f2f9a7d4..88e6b8759 100644 --- a/documentation/ru/src/tutorials/basic-usage.md +++ b/documentation/ru/src/tutorials/basic-usage.md @@ -21,7 +21,7 @@ Arguments: Options: -l, --log-level [env: LOG_LEVEL=] [default: INFO] -F, --log-format [env: LOG_FORMAT=] [default: text] [possible values: text, structured, debug] - -n, --no-color disable colors in the log output [env: NO_COLOR=] + -n, --no-color force colors off in the log output -d, --daemon run as daemon [env: DAEMON=] -h, --help Print help -V, --version Print version @@ -34,7 +34,7 @@ Options: | `-d`, `--daemon` | Запуск в фоне. Без этого параметра процесс работает на переднем плане.

В daemon-режиме обязательны `daemon_pid_file` и `syslog_prog_name`. После ухода в фон сообщения в stderr больше не пишутся. | | `-l`, `--log-level` | Уровень логирования: `INFO`, `DEBUG` или `WARN`. | | `-F`, `--log-format` | Формат логов. Возможные значения: `text`, `structured`, `debug`. | -| `-n`, `--no-color` | Отключить цвета в выводе логов. | +| `-n`, `--no-color` | Принудительно отключить цвета в логах. Цвета также автоматически выключаются, если stderr не терминал (под systemd — это pipe в journald) и если переменная окружения `NO_COLOR` установлена в любое непустое значение. | | `-V`, `--version` | Показать информацию о версии. | | `-h`, `--help` | Показать справку. | From 4dd628b4fc26cea7e4ae9ffdf00bcb12794da947 Mon Sep 17 00:00:00 2001 From: dmitrivasilyev Date: Wed, 13 May 2026 15:22:48 +0300 Subject: [PATCH 08/11] ci(rpm): bundle perl-FindBin / perl-IPC-Cmd into the SRPM for EL7/EL8 COPR builds `pg-doorman.spec` listed `perl-FindBin` and `perl-IPC-Cmd` as `BuildRequires`. On RHEL/CentOS Stream 7 and 8 these modules ship only inside the modular `perl:5.30` stream, which COPR's mock filters out by default ("package perl-FindBin-... is filtered out by modular filtering"). COPR builders have no network access, so we cannot `dnf module enable perl` inside the chroot, and the rhel-8, centos-stream-8, epel-8, and epel-7 chroots fail at `dnf builddep` before pg_doorman compiles. AlmaLinux 9 / Rocky 9 / Fedora / Amazon Linux 2023 are unaffected because they expose the same modules outside modular streams. Apply the same offline-tarball pattern we already use for Rust: - The CI prepare-srpm step now installs `perl-FindBin` and `perl-IPC-Cmd` in the Fedora container, copies the `.pm` files from `@INC` into `perl_modules/`, and tarballs the directory as `perl_modules.tar.gz`. - The spec adds `Source3: perl_modules.tar.gz`, swaps the two missing `BuildRequires` for `perl-interpreter`, extracts the tarball into `_builddir/perl_modules`, and exports `PERL5LIB="_builddir/perl_modules:$PERL5LIB"` in `%build` so the openssl-sys Configure path resolves `use FindBin` and `use IPC::Cmd` from the bundle on every chroot, modular filtering or not. The pure-Perl module list covers FindBin plus the dependency closure IPC::Cmd actually walks at use-time (Module::Load::Conditional, Module::Load, Module::Metadata, Params::Check, Locale::Maketext::Simple, ExtUtils::MakeMaker, version, IPC::Open3). Modules without architecture-specific XS bits, so the tarball is portable across the EL7/EL8/EL9/Fedora/Amazon mix. --- .github/workflows/copr-publish.yaml | 36 ++++++++++++++++++++++++++++- pkg/rpm/pg-doorman.spec | 16 ++++++++++--- 2 files changed, 48 insertions(+), 4 deletions(-) diff --git a/.github/workflows/copr-publish.yaml b/.github/workflows/copr-publish.yaml index 7564653a6..bdf9bdcbf 100644 --- a/.github/workflows/copr-publish.yaml +++ b/.github/workflows/copr-publish.yaml @@ -98,6 +98,36 @@ jobs: echo "Downloaded Rust tarball:" ls -la rust-*.tar.gz + - name: Bundle Perl modules for offline build + run: | + # RHEL/CentOS Stream 7 and 8 ship FindBin and IPC::Cmd only + # inside modular Perl streams that COPR mock filters out + # by default (`package perl-FindBin-... is filtered out by + # modular filtering`). COPR builders also have no network + # access, so we cannot `dnf module enable perl` inside the + # chroot. Same pattern as the Rust tarball: ship the pure-Perl + # modules we need as a Source so the spec extracts them and + # the build sees them via PERL5LIB, regardless of distro. + dnf install -y perl perl-FindBin perl-IPC-Cmd + mkdir -p perl_modules + # Resolve actual on-disk path of each .pm via Perl itself so + # the tarball follows whichever vendor_perl / site_perl layout + # the prepare distro uses. + for module in FindBin IPC::Cmd IPC::Open3 Module::Load::Conditional Module::Load Module::Metadata Params::Check Locale::Maketext::Simple ExtUtils::MakeMaker version; do + path=$(perl -M"$module" -e "print \$INC{'${module//::/\/}.pm'}" 2>/dev/null || true) + if [ -n "$path" ] && [ -r "$path" ]; then + relative="${module//::/\/}.pm" + dest="perl_modules/$relative" + mkdir -p "$(dirname "$dest")" + cp -L "$path" "$dest" + echo "bundled: $relative <- $path" + else + echo "skipped: $module (not found)" + fi + done + tar czf perl_modules.tar.gz perl_modules/ + ls -la perl_modules.tar.gz + - name: Setup RPM build environment run: | rpmdev-setuptree @@ -154,9 +184,13 @@ jobs: # Copy vendor tarball cp "$SOURCE_DIR/vendor.tar.gz" ~/rpmbuild/SOURCES/ - + # Copy Rust tarball for offline COPR build cp "$SOURCE_DIR/rust-${{ env.RUST_VERSION }}-x86_64-unknown-linux-gnu.tar.gz" ~/rpmbuild/SOURCES/ + + # Copy bundled Perl modules for offline COPR build on EL7/EL8 + # (their modular filtering hides perl-FindBin / perl-IPC-Cmd). + cp "$SOURCE_DIR/perl_modules.tar.gz" ~/rpmbuild/SOURCES/ # Update spec file with correct version cd "$SOURCE_DIR" diff --git a/pkg/rpm/pg-doorman.spec b/pkg/rpm/pg-doorman.spec index f4608e5bd..bc7bd7d93 100644 --- a/pkg/rpm/pg-doorman.spec +++ b/pkg/rpm/pg-doorman.spec @@ -11,6 +11,13 @@ URL: https://github.com/ozontech/pg_doorman Source0: %{name}-%{version}.tar.gz Source1: vendor.tar.gz Source2: rust-%{rust_version}-x86_64-unknown-linux-gnu.tar.gz +# RHEL/CentOS Stream 7 and 8 ship perl-FindBin / perl-IPC-Cmd only +# inside modular Perl streams that COPR mock filters out by default, +# and COPR builders have no network access to `dnf module enable`. +# The CI workflow vendors the .pm files into this tarball; the build +# stanza extracts it and points PERL5LIB at the result so the build +# sees the modules regardless of distro. +Source3: perl_modules.tar.gz BuildRequires: gcc BuildRequires: gcc-c++ @@ -20,9 +27,7 @@ BuildRequires: cmake BuildRequires: clang BuildRequires: tar BuildRequires: patch -BuildRequires: perl -BuildRequires: perl-FindBin -BuildRequires: perl-IPC-Cmd +BuildRequires: perl-interpreter Requires: openssl @@ -39,6 +44,8 @@ This package includes: %setup -q -n %{name}-%{version} # Extract vendored dependencies tar xzf %{SOURCE1} +# Extract bundled Perl modules (FindBin, IPC::Cmd, ...) — see Source3 comment. +tar xzf %{SOURCE3} -C %{_builddir} %build # Install Rust toolchain from local tarball (COPR has no network access) @@ -48,6 +55,9 @@ tar xzf %{SOURCE2} -C %{_builddir} %{_builddir}/rust-%{rust_version}-x86_64-unknown-linux-gnu/install.sh --prefix="$RUST_INSTALL_DIR" --without=rust-docs export PATH="$RUST_INSTALL_DIR/bin:$PATH" +# Make the bundled pure-Perl modules visible to any build script that +# pulls them in (openssl-sys' Configure path uses FindBin and IPC::Cmd). +export PERL5LIB="%{_builddir}/perl_modules${PERL5LIB:+:$PERL5LIB}" # Configure cargo to use vendored dependencies mkdir -p .cargo From c4708d4dd2362789d84cfddaf68d0ab1a64fb1f3 Mon Sep 17 00:00:00 2001 From: dmitrivasilyev Date: Wed, 13 May 2026 15:36:11 +0300 Subject: [PATCH 09/11] ci(bdd): chown cargo cache after docker compile so actions/cache stops saving an empty tarball `prebuild-bdd` runs `cargo test --no-run` inside the test-runner container. The container runs as root, so the `target/` and `.cargo/` files it produces are root-owned on the host. `actions/cache@v4`'s post-step runs as the runner user (`uid 1001`) and silently fails to read the root-owned files; in the recent PR #251 runs it saved a roughly empty tarball in ~13 s and the next run's `Restore cargo cache` step finished in 0 s with nothing to restore. Result: every `prebuild-bdd` paid full 5-7 minutes of cargo build even though the cache key matched. Add a `Chown cargo cache so actions/cache can read it` step after the two docker compile steps. `sudo chown -R "$(id -u):$(id -g)" .cargo target` is enough on GitHub-hosted ubuntu-latest (sudo is available without password) to flip ownership back to the runner user before the cache post-step tars the directories. With the chown the save step takes its real time (single-digit minutes on a fully-built target), and the next run's restore actually populates `.cargo/` and `target/`. --- .github/workflows/bdd-tests.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.github/workflows/bdd-tests.yml b/.github/workflows/bdd-tests.yml index e4347579a..a0dd9f795 100644 --- a/.github/workflows/bdd-tests.yml +++ b/.github/workflows/bdd-tests.yml @@ -312,6 +312,17 @@ jobs: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ needs.prepare-tests.outputs.image-tag }} \ cargo test --features tls-migration --test bdd --no-run + - name: Chown cargo cache so actions/cache can read it + # The docker steps above ran as root, so `.cargo/` and `target/` + # are root-owned on the host. actions/cache@v4 saves files as + # the runner user (uid 1001); without this chown its post-step + # tar gets permission-denied on every file and silently writes + # an empty cache (~13s save, ~0s restore on the next run). + # `sudo` is available on GitHub-hosted ubuntu-latest by default. + run: | + sudo chown -R "$(id -u):$(id -g)" .cargo target + du -sh .cargo target + # Single matrixed job replaces the 21 nearly-identical per-language / # per-suite jobs that used to live here. Each suite differs only in # the cargo command it runs; share everything else. See the cargo From 8936939e2fd78ada067ccd7a93e287a97ba6cf0a Mon Sep 17 00:00:00 2001 From: dmitrivasilyev Date: Wed, 13 May 2026 15:37:00 +0300 Subject: [PATCH 10/11] ci(bdd): raise matrix max-parallel from 6 to 12 With prebuild-bdd populating the cache, each matrix entry is now bound by link time and the scenario itself (~1-2 min) rather than the full cargo build (~7-9 min). The CPU contention that drove the previous cap of 6 (sleep-heavy lifecycle and SCRAM passthrough reconnect losing their timing margin) does not apply to the link-only path, so we can safely double the fan-out. Wall clock on a green PR roughly halves; the timing-sensitive suites keep their `nick-fields/retry@v3` outer policy. --- .github/workflows/bdd-tests.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/bdd-tests.yml b/.github/workflows/bdd-tests.yml index a0dd9f795..26e073677 100644 --- a/.github/workflows/bdd-tests.yml +++ b/.github/workflows/bdd-tests.yml @@ -340,13 +340,13 @@ jobs: packages: read strategy: fail-fast: false - # Six concurrent suites is the cap this matrix was tuned for. - # GitHub-hosted runners share CPU under heavy fan-out; with 20+ - # BDD jobs in parallel, the sleep-based lifecycle tests and the - # SCRAM passthrough reconnect tests lost timing margin. This cap - # keeps those suites stable while reducing total wall time versus - # the previous limit of four. - max-parallel: 6 + # Twelve concurrent suites: the prebuild-bdd cache trims each + # suite to a link + scenario run, so the CPU pressure that drove + # the previous cap of 6 (sleep-based lifecycle tests and SCRAM + # passthrough reconnect losing timing margin) is gone. Doubling + # the fan-out halves the BDD wall clock on a green PR; the + # sleep-heavy suites still have their own retry policy. + max-parallel: 12 matrix: suite: - { name: "Go", cargo: "test --test bdd -- --tags @go" } From d8cdde9ded0bacc2dbcb21b43b662a757dc661c4 Mon Sep 17 00:00:00 2001 From: dmitrivasilyev Date: Wed, 13 May 2026 15:49:45 +0300 Subject: [PATCH 11/11] docs(binary-upgrade): align systemd example with the production unit pattern User flagged that their production unit differs from the tutorial. Adjust the example so it matches what is actually deployed, while keeping the comments load-bearing. - `NotifyAccess=exec` instead of `all`: the upgrade child is spawned via Command::spawn (fork + execve), so exec scope is enough; `all` was an over-broad surface. - `ExecReload=/bin/kill -SIGUSR2 $MAINPID` instead of SIGHUP: `pg_doorman -t` runs first under SIGUSR2 and aborts the reload on a bad config, so `systemctl reload` becomes the single command that drives a full safe binary upgrade. Quick start now uses `systemctl reload pg_doorman.service` accordingly. - `Restart=always` instead of `on-failure`: the production expectation is that the pool never stays down on the host, even after an explicit `systemctl stop`. - `LimitNOFILE=65536` instead of `1048576`: the 1M default was over-cautious; 65536 covers most OLTP pools and the comment now explains how to size it from `pool_size * num_pools` plus clients. - `User=postgres` / `Group=postgres` instead of a dedicated `pg_doorman` account: many deployments already have the postgres account, reusing it keeps file ownership aligned with PostgreSQL. - `SyslogIdentifier=pg-pooler`: matches the production journal identifier so `journalctl -u pg_doorman -t pg-pooler` is the documented lookup path. - `ExecStop=/bin/kill -SIGTERM $MAINPID`: redundant with the default but explicit makes the stop path obvious next to the reload path. - Drop `KillMode=mixed`: the production setup runs without it and works because Type=notify already updates MainPID during the handoff window; keeping it pulled out lines up with the deployed unit. Mention `MemoryMax/Nice/CPUAffinity` as orthogonal resource controls in a trailing paragraph. Operational checklist and step 3 of Quick start updated to match. EN and RU edited in lock-step. --- .../en/src/tutorials/binary-upgrade.md | 91 +++++++++++-------- .../ru/src/tutorials/binary-upgrade.md | 87 +++++++++++------- 2 files changed, 107 insertions(+), 71 deletions(-) diff --git a/documentation/en/src/tutorials/binary-upgrade.md b/documentation/en/src/tutorials/binary-upgrade.md index 61478dc24..dc49c652a 100644 --- a/documentation/en/src/tutorials/binary-upgrade.md +++ b/documentation/en/src/tutorials/binary-upgrade.md @@ -42,14 +42,16 @@ install -m 0755 pg_doorman_new /usr/bin/pg_doorman # touching the running server. /usr/bin/pg_doorman -t /etc/pg_doorman.yaml -# 3. Trigger the upgrade. Under systemd `Type=notify` the unit -# delivers SIGUSR2 to the tracked MainPID; this picks the single -# correct process even when other pg_doorman instances or stale -# child processes are running on the host. Direct +# 3. Trigger the upgrade. With `ExecReload=/bin/kill -SIGUSR2 $MAINPID` +# in the unit, `systemctl reload` runs the full binary upgrade +# (config validated by `pg_doorman -t` first, then state migrated, +# then old process drained). systemd delivers the signal to the +# tracked MainPID, so this targets the single correct process even +# when other pg_doorman instances are running on the host. Direct # `kill -USR2 $(pgrep -f /usr/bin/pg_doorman)` works but matches by -# command line and can target every instance on the host at once, -# which is why packaged installs go through systemctl. -sudo systemctl kill -s SIGUSR2 pg_doorman.service +# command line and can hit every instance, which is why packaged +# installs go through systemctl. +sudo systemctl reload pg_doorman.service # 4. Verify: systemd tracks the new MainPID (Type=notify receives # `MAINPID=` from the child during the handoff). Active @@ -356,39 +358,50 @@ child process can update `MainPID` to itself during `SIGUSR2` upgrades: ```ini [Service] Type=notify -# NotifyAccess=all lets the child process spawned by SIGUSR2 reach -# systemd's notify socket and hand off MainPID; without it, systemd -# silently keeps tracking the dying old PID. -NotifyAccess=all +# `exec` is enough: the child process that takes over on SIGUSR2 is +# spawned through Command::spawn (fork + execve), which puts it in +# scope for sd_notify. `all` works too, but exec narrows the notify +# surface to what pg_doorman actually uses. +NotifyAccess=exec ExecStart=/usr/bin/pg_doorman /etc/pg_doorman.yaml -ExecReload=/bin/kill -SIGHUP $MAINPID -# Survive a crash, but back off so a restart loop does not amplify a -# bad release. on-failure ignores intentional `systemctl stop`. -Restart=on-failure +# `systemctl reload` triggers a full binary upgrade (config validated, +# state migrated, old process drained). pg_doorman's SIGUSR2 path runs +# `pg_doorman -t` first and aborts the reload if the config is broken, +# so this is safe to wire as the reload signal. +ExecReload=/bin/kill -SIGUSR2 $MAINPID +# `systemctl stop` sends SIGTERM to the main pid for a graceful drain. +ExecStop=/bin/kill -SIGTERM $MAINPID +# Always restart: bring the pooler back even after a manual stop. +# Match the typical PostgreSQL-side HA expectation that the pool is +# never down on the host. +Restart=always RestartSec=5s # pg_doorman is connection-heavy: each client + each backend uses an -# fd, plus internal pipes. The default soft limit of 1024 is the first -# wall hit in production. -LimitNOFILE=1048576 -# Run as a non-privileged service account that owns -# /var/run/pg_doorman and the PID file. Avoid root if the listener -# uses a port above 1024. -User=pg_doorman -Group=pg_doorman -# kill-control: SIGUSR2 must reach both the parent and any children -# during an upgrade, and the final SIGTERM at stop must reach the -# leader directly. mixed delivers the stop signal to the main pid and -# the cleanup signal to the cgroup. -KillMode=mixed -TimeoutStopSec=120 +# fd, plus internal pipes. 65536 covers most OLTP pools; size it from +# `general.pool_size * num_pools` plus a few thousand for clients. +LimitNOFILE=65536 +# Run as a non-privileged service account that owns the PID file. On +# many deployments postgres already exists; reusing it keeps file +# ownership consistent with PostgreSQL itself. +User=postgres +Group=postgres +# Surface logs in journalctl under `pg-pooler` instead of the binary +# name; convenient for `journalctl -u pg_doorman -t pg-pooler`. +SyslogIdentifier=pg-pooler +TimeoutStopSec=30s ``` -`ExecReload=SIGHUP` covers config reload; the binary upgrade itself is -triggered separately by `systemctl kill -s SIGUSR2 pg_doorman.service` -or by `UPGRADE;` from the admin console. Adjust `TimeoutStopSec` to -match the value of `shutdown_timeout` in your `pg_doorman.yaml`, plus -a small margin, so systemd does not SIGKILL the process before the -drain finishes. +`systemctl reload pg_doorman` drives the binary upgrade: pg_doorman +runs `-t` on the new binary, aborts the reload if the config is bad, +otherwise spawns the new process and drains the old one. `UPGRADE;` +from the admin console reaches the same code path. Tune +`TimeoutStopSec` so it covers `shutdown_timeout` in +`pg_doorman.yaml` plus a small margin; otherwise systemd SIGKILLs the +old process before the drain finishes. + +Production deployments often layer resource controls on top of the +above: `MemoryMax=`, `Nice=-15`, `CPUAffinity=2,3,4,5,6,7,8,9`. These +are workload-specific and orthogonal to the upgrade contract. ## Configuration @@ -561,9 +574,11 @@ Before rolling out binary upgrade to production: both processes use the same certificate and key files - [ ] Test the upgrade in staging: open a session, trigger SIGUSR2, verify the session continues working -- [ ] Verify the systemd unit is `Type=notify` with `NotifyAccess=all`, - `ExecReload=/bin/kill -SIGHUP $MAINPID` for config reload, and a - `TimeoutStopSec` that covers `shutdown_timeout` plus margin +- [ ] Verify the systemd unit is `Type=notify` with + `NotifyAccess=exec`, `ExecReload=/bin/kill -SIGUSR2 $MAINPID` + (so `systemctl reload` runs a full binary upgrade with config + validation), and a `TimeoutStopSec` that covers + `shutdown_timeout` plus margin - [ ] Monitor logs for migration errors after the first production upgrade - [ ] Confirm old process exits (check PID file or `pgrep`) diff --git a/documentation/ru/src/tutorials/binary-upgrade.md b/documentation/ru/src/tutorials/binary-upgrade.md index a07fabc11..df0efb8f6 100644 --- a/documentation/ru/src/tutorials/binary-upgrade.md +++ b/documentation/ru/src/tutorials/binary-upgrade.md @@ -47,14 +47,16 @@ install -m 0755 pg_doorman_new /usr/bin/pg_doorman # не трогая работающий сервер. /usr/bin/pg_doorman -t /etc/pg_doorman.yaml -# 3. Запустите обновление. При systemd `Type=notify` команда шлёт -# SIGUSR2 в MainPID, который отслеживает systemd, — выбирается -# ровно один процесс, даже если на хосте есть другие инстансы -# pg_doorman или зомби-чайлды. Прямой `kill -USR2 $(pgrep -f +# 3. Запустите обновление. С `ExecReload=/bin/kill -SIGUSR2 $MAINPID` +# в юните `systemctl reload` запускает полный binary upgrade +# (сначала валидация `pg_doorman -t`, потом миграция состояния, +# потом drain старого процесса). systemd шлёт сигнал в MainPID, +# поэтому выбирается ровно один процесс, даже если на хосте есть +# другие инстансы pg_doorman. Прямой `kill -USR2 $(pgrep -f # /usr/bin/pg_doorman)` работает, но pgrep матчит по командной -# строке и может попасть во все инстансы сразу — поэтому -# пакетные установки используют systemctl. -sudo systemctl kill -s SIGUSR2 pg_doorman.service +# строке и может попасть во все инстансы сразу — пакетные +# установки используют systemctl. +sudo systemctl reload pg_doorman.service # 4. Проверьте: systemd подхватил новый MainPID (Type=notify получает # `MAINPID=` от чайлда при handoff). Состояние active и @@ -364,34 +366,52 @@ SHA-256 архива проверяется автоматически. ```ini [Service] Type=notify -# NotifyAccess=all нужен, чтобы новый процесс, который порождает -# SIGUSR2, мог достучаться до notify-сокета systemd и передать -# MainPID. Без него systemd молча продолжает следить за старым, -# умирающим PID. -NotifyAccess=all +# `exec` достаточно: новый процесс при SIGUSR2 порождается через +# Command::spawn (fork + execve), он попадает в exec-scope notify. +# `all` тоже работает, но `exec` сужает поверхность только до того, +# что действительно делает pg_doorman. +NotifyAccess=exec ExecStart=/usr/bin/pg_doorman /etc/pg_doorman.yaml -ExecReload=/bin/kill -SIGHUP $MAINPID -# Перезапускать после падения, но с паузой, чтобы плохой релиз не -# зациклился. on-failure не реагирует на штатный `systemctl stop`. -Restart=on-failure +# `systemctl reload` запускает полный binary upgrade (валидация +# конфига, миграция состояния, drain старого процесса). SIGUSR2 в +# pg_doorman сначала прогоняет `-t` и отменяет reload при битом +# конфиге, поэтому такой сигнал безопасно вешать на reload. +ExecReload=/bin/kill -SIGUSR2 $MAINPID +# `systemctl stop` шлёт SIGTERM в main pid для graceful drain. +ExecStop=/bin/kill -SIGTERM $MAINPID +# Перезапуск всегда: возвращаем пул на ноге даже после ручной +# остановки. Соответствует ожиданию HA на стороне PostgreSQL, что пул +# никогда не лежит на хосте. +Restart=always RestartSec=5s # pg_doorman тяжёл по дескрипторам: каждый клиент и каждый backend — -# отдельный fd, плюс служебные pipe. Дефолтный soft-лимит 1024 — это -# первая стена в продакшене. -LimitNOFILE=1048576 -# Сервисная учётка, которой принадлежат /var/run/pg_doorman и PID-файл. -# Использовать root не нужно, если listener слушает порт выше 1024. -User=pg_doorman -Group=pg_doorman -# Доставка сигналов: SIGUSR2 при upgrade должен дойти и до родителя, -# и до чайлдов; финальный SIGTERM при stop — в лидера группы. mixed -# шлёт stop-сигнал главному pid, а cleanup — в cgroup. -KillMode=mixed -TimeoutStopSec=120 +# отдельный fd, плюс служебные pipe. 65536 покрывает большинство +# OLTP-пулов; считается как `general.pool_size * num_pools` плюс +# несколько тысяч на клиентов. +LimitNOFILE=65536 +# Сервисная учётка, которой принадлежит PID-файл. На многих установках +# postgres уже существует — переиспользование оставляет владение +# файлов согласованным с самим PostgreSQL. +User=postgres +Group=postgres +# Имя в journalctl: `pg-pooler` вместо имени бинаря; удобно для +# `journalctl -u pg_doorman -t pg-pooler`. +SyslogIdentifier=pg-pooler +TimeoutStopSec=30s ``` -`ExecReload=SIGHUP` отвечает за reload конфига; обновление бинаря -запускается отдельно — `systemctl kill -s SIGUSR2 pg_doorman.service` +`systemctl reload pg_doorman` запускает binary upgrade: pg_doorman +прогоняет `-t` на новом бинаре, отменяет reload при битом конфиге, +иначе порождает новый процесс и дренирует старый. `UPGRADE;` из +админ-консоли идёт по тому же code path. Подбирайте `TimeoutStopSec` +с запасом над `shutdown_timeout` из `pg_doorman.yaml`, иначе systemd +прибьёт старый процесс SIGKILL-ом до завершения дренажа. + +Прод-установки часто добавляют сверху ограничения ресурсов: +`MemoryMax=`, `Nice=-15`, `CPUAffinity=2,3,4,5,6,7,8,9`. Это +workload-специфика и она ортогональна контракту upgrade. + + или `UPGRADE;` из админ-консоли. `TimeoutStopSec` подбирайте под значение `shutdown_timeout` в `pg_doorman.yaml` плюс небольшой запас, иначе systemd прибьёт процесс SIGKILL до окончания дренажа. @@ -571,9 +591,10 @@ Raw fd клиента недоступен или `dup()` не удался. В оба процесса используют одинаковые файлы сертификата и ключа - [ ] Протестировать обновление на staging: открыть сессию, отправить SIGUSR2, убедиться что сессия продолжает работать -- [ ] В systemd unit указано `Type=notify` с `NotifyAccess=all`, - `ExecReload=/bin/kill -SIGHUP $MAINPID` для reload конфига и - `TimeoutStopSec`, покрывающий `shutdown_timeout` с запасом +- [ ] В systemd unit указано `Type=notify` с `NotifyAccess=exec`, + `ExecReload=/bin/kill -SIGUSR2 $MAINPID` (тогда `systemctl + reload` запускает полный binary upgrade с валидацией конфига) + и `TimeoutStopSec`, покрывающий `shutdown_timeout` с запасом - [ ] Мониторинг логов на ошибки миграции после первого промышленного обновления - [ ] Подтвердить что старый процесс завершился (PID-файл или `pgrep`)