diff --git a/.github/workflows/bdd-tests.yml b/.github/workflows/bdd-tests.yml index b650d646a..26e073677 100644 --- a/.github/workflows/bdd-tests.yml +++ b/.github/workflows/bdd-tests.yml @@ -40,11 +40,21 @@ jobs: echo "Computed image tag: flake-${FLAKE_HASH}" - name: Log in to Container Registry - uses: docker/login-action@v3 + # ghcr.io occasionally times out on the auth handshake under + # GHA load (Client.Timeout exceeded while awaiting headers). + # docker/login-action runs once; wrap docker login in + # nick-fields/retry so a transient handshake failure does not + # take out the whole BDD job. + uses: nick-fields/retry@v3 + env: + GHCR_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GHCR_ACTOR: ${{ github.actor }} with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} + timeout_minutes: 3 + max_attempts: 3 + retry_wait_seconds: 10 + command: | + echo "$GHCR_TOKEN" | docker login ${{ env.REGISTRY }} -u "$GHCR_ACTOR" --password-stdin - name: Check if image exists in registry id: check-exists @@ -125,11 +135,21 @@ jobs: echo "Image built successfully for ${{ matrix.arch }}" - name: Log in to Container Registry - uses: docker/login-action@v3 + # ghcr.io occasionally times out on the auth handshake under + # GHA load (Client.Timeout exceeded while awaiting headers). + # docker/login-action runs once; wrap docker login in + # nick-fields/retry so a transient handshake failure does not + # take out the whole BDD job. + uses: nick-fields/retry@v3 + env: + GHCR_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GHCR_ACTOR: ${{ github.actor }} with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} + timeout_minutes: 3 + max_attempts: 3 + retry_wait_seconds: 10 + command: | + echo "$GHCR_TOKEN" | docker login ${{ env.REGISTRY }} -u "$GHCR_ACTOR" --password-stdin - name: Load and push arch-specific image run: | @@ -150,11 +170,21 @@ jobs: packages: write steps: - name: Log in to Container Registry - uses: docker/login-action@v3 + # ghcr.io occasionally times out on the auth handshake under + # GHA load (Client.Timeout exceeded while awaiting headers). + # docker/login-action runs once; wrap docker login in + # nick-fields/retry so a transient handshake failure does not + # take out the whole BDD job. + uses: nick-fields/retry@v3 + env: + GHCR_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GHCR_ACTOR: ${{ github.actor }} with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} + timeout_minutes: 3 + max_attempts: 3 + retry_wait_seconds: 10 + command: | + echo "$GHCR_TOKEN" | docker login ${{ env.REGISTRY }} -u "$GHCR_ACTOR" --password-stdin - name: Create and push multi-arch manifest # Only the hash-tagged manifest is published. The image is @@ -202,6 +232,97 @@ jobs: echo "✅ Using freshly built image: ${{ needs.check-image.outputs.image-tag }}" fi + # Compile the BDD test binaries once and stash cargo's registry + + # target dir in GHA cache so the 23-way matrix below does not pay a + # full `cargo download + compile` per suite. Default features cover + # all suites except `tls-migration`, which builds in its own target + # subdirectory because of the cargo features-hash. `CARGO_HOME` is + # pinned inside `/workspace/.cargo` so the cache stays under + # `github.workspace` and no host UID/permission tricks are needed. + prebuild-bdd: + name: Prebuild BDD test binaries + needs: prepare-tests + if: always() && needs.prepare-tests.result == 'success' + runs-on: ubuntu-latest + timeout-minutes: 30 + permissions: + contents: read + packages: read + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Restore cargo cache + uses: actions/cache@v4 + with: + path: | + .cargo + target + key: ${{ runner.os }}-bdd-cargo-${{ needs.prepare-tests.outputs.image-tag }}-${{ hashFiles('Cargo.lock', 'src/**', 'tests/**', 'build.rs') }} + restore-keys: | + ${{ runner.os }}-bdd-cargo-${{ needs.prepare-tests.outputs.image-tag }}- + ${{ runner.os }}-bdd-cargo- + + - name: Log in to Container Registry + # ghcr.io occasionally times out on the auth handshake under + # GHA load (Client.Timeout exceeded while awaiting headers). + # docker/login-action runs once; wrap docker login in + # nick-fields/retry so a transient handshake failure does not + # take out the whole BDD job. + uses: nick-fields/retry@v3 + env: + GHCR_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GHCR_ACTOR: ${{ github.actor }} + with: + timeout_minutes: 3 + max_attempts: 3 + retry_wait_seconds: 10 + command: | + echo "$GHCR_TOKEN" | docker login ${{ env.REGISTRY }} -u "$GHCR_ACTOR" --password-stdin + + - name: Pull test image from GHCR + uses: nick-fields/retry@v3 + with: + timeout_minutes: 5 + max_attempts: 3 + retry_wait_seconds: 10 + command: docker pull ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ needs.prepare-tests.outputs.image-tag }} + + - name: Compile bdd + patroni_proxy_bdd binaries (default features) + run: | + docker run --rm \ + --network=host \ + -e CARGO_NET_RETRY=10 \ + -e CARGO_HTTP_TIMEOUT=60 \ + -e CARGO_HOME=/workspace/.cargo \ + -v ${{ github.workspace }}:/workspace \ + -w /workspace \ + ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ needs.prepare-tests.outputs.image-tag }} \ + sh -c "cargo test --test bdd --no-run && cargo test --test patroni_proxy_bdd --no-run" + + - name: Compile bdd binary (tls-migration feature) + run: | + docker run --rm \ + --network=host \ + -e CARGO_NET_RETRY=10 \ + -e CARGO_HTTP_TIMEOUT=60 \ + -e CARGO_HOME=/workspace/.cargo \ + -v ${{ github.workspace }}:/workspace \ + -w /workspace \ + ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ needs.prepare-tests.outputs.image-tag }} \ + cargo test --features tls-migration --test bdd --no-run + + - name: Chown cargo cache so actions/cache can read it + # The docker steps above ran as root, so `.cargo/` and `target/` + # are root-owned on the host. actions/cache@v4 saves files as + # the runner user (uid 1001); without this chown its post-step + # tar gets permission-denied on every file and silently writes + # an empty cache (~13s save, ~0s restore on the next run). + # `sudo` is available on GitHub-hosted ubuntu-latest by default. + run: | + sudo chown -R "$(id -u):$(id -g)" .cargo target + du -sh .cargo target + # Single matrixed job replaces the 21 nearly-identical per-language / # per-suite jobs that used to live here. Each suite differs only in # the cargo command it runs; share everything else. See the cargo @@ -210,8 +331,8 @@ jobs: # sensitive lifecycle scenarios). bdd-tests: name: 'BDD: ${{ matrix.suite.name }}' - needs: prepare-tests - if: always() && needs.prepare-tests.result == 'success' + needs: [ prepare-tests, prebuild-bdd ] + if: always() && needs.prepare-tests.result == 'success' && needs.prebuild-bdd.result == 'success' runs-on: ubuntu-latest timeout-minutes: 30 permissions: @@ -219,13 +340,13 @@ jobs: packages: read strategy: fail-fast: false - # Six concurrent suites is the cap this matrix was tuned for. - # GitHub-hosted runners share CPU under heavy fan-out; with 20+ - # BDD jobs in parallel, the sleep-based lifecycle tests and the - # SCRAM passthrough reconnect tests lost timing margin. This cap - # keeps those suites stable while reducing total wall time versus - # the previous limit of four. - max-parallel: 6 + # Twelve concurrent suites: the prebuild-bdd cache trims each + # suite to a link + scenario run, so the CPU pressure that drove + # the previous cap of 6 (sleep-based lifecycle tests and SCRAM + # passthrough reconnect losing timing margin) is gone. Doubling + # the fan-out halves the BDD wall clock on a green PR; the + # sleep-heavy suites still have their own retry policy. + max-parallel: 12 matrix: suite: - { name: "Go", cargo: "test --test bdd -- --tags @go" } @@ -255,16 +376,38 @@ jobs: - name: Checkout repository uses: actions/checkout@v4 + - name: Restore cargo cache from prebuild + # Same key shape as prebuild-bdd. Restore-keys fall back to any + # earlier prebuild for this image tag, then to any prebuild on + # this OS, so a matrix entry never starts from a cold cargo. + uses: actions/cache@v4 + with: + path: | + .cargo + target + key: ${{ runner.os }}-bdd-cargo-${{ needs.prepare-tests.outputs.image-tag }}-${{ hashFiles('Cargo.lock', 'src/**', 'tests/**', 'build.rs') }} + restore-keys: | + ${{ runner.os }}-bdd-cargo-${{ needs.prepare-tests.outputs.image-tag }}- + ${{ runner.os }}-bdd-cargo- + - name: Log in to Container Registry # Even when test-runner is currently a public package, the BDD # jobs depend on `docker pull` succeeding. Without explicit # login they would silently break if the package visibility - # ever flips to private. - uses: docker/login-action@v3 + # ever flips to private. Wrapped in nick-fields/retry because + # ghcr.io's auth endpoint occasionally times out under GHA + # load (Client.Timeout exceeded while awaiting headers) and the + # base docker/login-action does not retry on its own. + uses: nick-fields/retry@v3 + env: + GHCR_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GHCR_ACTOR: ${{ github.actor }} with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} + timeout_minutes: 3 + max_attempts: 3 + retry_wait_seconds: 10 + command: | + echo "$GHCR_TOKEN" | docker login ${{ env.REGISTRY }} -u "$GHCR_ACTOR" --password-stdin - name: Pull test image from GHCR uses: nick-fields/retry@v3 @@ -312,6 +455,7 @@ jobs: --network=host \ -e CARGO_NET_RETRY=10 \ -e CARGO_HTTP_TIMEOUT=60 \ + -e CARGO_HOME=/workspace/.cargo \ -v ${{ github.workspace }}:/workspace \ -w /workspace \ ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ needs.prepare-tests.outputs.image-tag }} \ diff --git a/.github/workflows/copr-publish.yaml b/.github/workflows/copr-publish.yaml index 7564653a6..bdf9bdcbf 100644 --- a/.github/workflows/copr-publish.yaml +++ b/.github/workflows/copr-publish.yaml @@ -98,6 +98,36 @@ jobs: echo "Downloaded Rust tarball:" ls -la rust-*.tar.gz + - name: Bundle Perl modules for offline build + run: | + # RHEL/CentOS Stream 7 and 8 ship FindBin and IPC::Cmd only + # inside modular Perl streams that COPR mock filters out + # by default (`package perl-FindBin-... is filtered out by + # modular filtering`). COPR builders also have no network + # access, so we cannot `dnf module enable perl` inside the + # chroot. Same pattern as the Rust tarball: ship the pure-Perl + # modules we need as a Source so the spec extracts them and + # the build sees them via PERL5LIB, regardless of distro. + dnf install -y perl perl-FindBin perl-IPC-Cmd + mkdir -p perl_modules + # Resolve actual on-disk path of each .pm via Perl itself so + # the tarball follows whichever vendor_perl / site_perl layout + # the prepare distro uses. + for module in FindBin IPC::Cmd IPC::Open3 Module::Load::Conditional Module::Load Module::Metadata Params::Check Locale::Maketext::Simple ExtUtils::MakeMaker version; do + path=$(perl -M"$module" -e "print \$INC{'${module//::/\/}.pm'}" 2>/dev/null || true) + if [ -n "$path" ] && [ -r "$path" ]; then + relative="${module//::/\/}.pm" + dest="perl_modules/$relative" + mkdir -p "$(dirname "$dest")" + cp -L "$path" "$dest" + echo "bundled: $relative <- $path" + else + echo "skipped: $module (not found)" + fi + done + tar czf perl_modules.tar.gz perl_modules/ + ls -la perl_modules.tar.gz + - name: Setup RPM build environment run: | rpmdev-setuptree @@ -154,9 +184,13 @@ jobs: # Copy vendor tarball cp "$SOURCE_DIR/vendor.tar.gz" ~/rpmbuild/SOURCES/ - + # Copy Rust tarball for offline COPR build cp "$SOURCE_DIR/rust-${{ env.RUST_VERSION }}-x86_64-unknown-linux-gnu.tar.gz" ~/rpmbuild/SOURCES/ + + # Copy bundled Perl modules for offline COPR build on EL7/EL8 + # (their modular filtering hides perl-FindBin / perl-IPC-Cmd). + cp "$SOURCE_DIR/perl_modules.tar.gz" ~/rpmbuild/SOURCES/ # Update spec file with correct version cd "$SOURCE_DIR" diff --git a/documentation/en/src/tutorials/basic-usage.md b/documentation/en/src/tutorials/basic-usage.md index 98dc0d69b..bc3636187 100644 --- a/documentation/en/src/tutorials/basic-usage.md +++ b/documentation/en/src/tutorials/basic-usage.md @@ -21,7 +21,7 @@ Arguments: Options: -l, --log-level [env: LOG_LEVEL=] [default: INFO] -F, --log-format [env: LOG_FORMAT=] [default: text] [possible values: text, structured, debug] - -n, --no-color disable colors in the log output [env: NO_COLOR=] + -n, --no-color force colors off in the log output -d, --daemon run as daemon [env: DAEMON=] -h, --help Print help -V, --version Print version @@ -34,7 +34,7 @@ Options: | `-d`, `--daemon` | Run in the background. Without this option, the process will run in the foreground.

In daemon mode, setting `daemon_pid_file` and `syslog_prog_name` is required. No log messages will be written to stderr after going into the background. | | `-l`, `--log-level` | Set log level: `INFO`, `DEBUG`, or `WARN`. | | `-F`, `--log-format` | Set log format. Possible values: `text`, `structured`, `debug`. | -| `-n`, `--no-color` | Disable colors in the log output. | +| `-n`, `--no-color` | Force colors off in the log output. Colors are also auto-disabled when stderr is not a TTY (under systemd, the journal pipe is not a terminal) and when the `NO_COLOR` environment variable is set to any non-empty value. | | `-V`, `--version` | Show version information. | | `-h`, `--help` | Show help information. | diff --git a/documentation/en/src/tutorials/binary-upgrade.md b/documentation/en/src/tutorials/binary-upgrade.md index eb96497c8..dc49c652a 100644 --- a/documentation/en/src/tutorials/binary-upgrade.md +++ b/documentation/en/src/tutorials/binary-upgrade.md @@ -20,9 +20,21 @@ cipher state with the `tls-migration` build (Linux, opt-in). ## Quick start +```admonish tip title="Use the distro package whenever you can" +On hosts where pg_doorman comes from `apt install pg-doorman` / +`dnf install pg-doorman`, the package manager handles step 1 atomically +(`dpkg` stages into a temp path and renames into place; `rpm` does the +same). `apt-get install --only-upgrade pg-doorman` or `dnf upgrade +pg-doorman` is the idiomatic devops path. The manual `install` below is +for direct-binary deployments where no package manager is in scope. +``` + ```bash -# 1. Drop the new binary in place. -cp pg_doorman_new /usr/bin/pg_doorman +# 1. Replace the binary atomically. `install` writes to a temporary +# file and renames it onto the target, so the inode the running +# process is mapped from is not overwritten in place. `cp` modifies +# the live inode and risks SIGBUS / SIGSEGV in the running pg_doorman. +install -m 0755 pg_doorman_new /usr/bin/pg_doorman # 2. Validate the new binary against the live config before triggering # the upgrade. SIGUSR2 also runs `-t` and aborts on failure, but @@ -30,12 +42,30 @@ cp pg_doorman_new /usr/bin/pg_doorman # touching the running server. /usr/bin/pg_doorman -t /etc/pg_doorman.yaml -# 3. Trigger the upgrade. -kill -USR2 $(pgrep -f /usr/bin/pg_doorman) +# 3. Trigger the upgrade. With `ExecReload=/bin/kill -SIGUSR2 $MAINPID` +# in the unit, `systemctl reload` runs the full binary upgrade +# (config validated by `pg_doorman -t` first, then state migrated, +# then old process drained). systemd delivers the signal to the +# tracked MainPID, so this targets the single correct process even +# when other pg_doorman instances are running on the host. Direct +# `kill -USR2 $(pgrep -f /usr/bin/pg_doorman)` works but matches by +# command line and can hit every instance, which is why packaged +# installs go through systemctl. +sudo systemctl reload pg_doorman.service + +# 4. Verify: systemd tracks the new MainPID (Type=notify receives +# `MAINPID=` from the child during the handoff). Active +# state and the admin console confirm clients are still attached. +systemctl show -p MainPID --value pg_doorman.service +psql -h pgdoorman -p 6432 -c 'SHOW POOLS;' # served by the new process +``` -# 4. Verify: a new PID is running and clients are still connected. -pgrep -f /usr/bin/pg_doorman # new PID -psql -h pgdoorman -p 6432 -c 'SHOW POOLS;' # answers from new process +```admonish tip title="Standalone (no systemd) deployments" +If the unit is not running under systemd, read the PID file the daemon +writes (`daemon_pid_file`, default `/tmp/pg_doorman.pid`) instead of +parsing `pgrep`: `kill -USR2 "$(cat /var/run/pg_doorman/pg_doorman.pid)"`. +Foreground deployments not managed by systemd should keep the PID of +the supervising process and signal that one directly. ``` The same upgrade can be triggered from the admin console: @@ -321,15 +351,58 @@ SIGINT triggers binary upgrade in daemon mode or without a TTY (e.g. when spawne | Recommended for | systemd, containers, k8s | Legacy deployments | For zero-downtime upgrades with client migration, run in foreground -mode. systemd manages the process lifecycle: +mode. systemd manages the process lifecycle. Use `Type=notify` so the +unit reaches `active` only after pg_doorman signals readiness, and the +child process can update `MainPID` to itself during `SIGUSR2` upgrades: ```ini [Service] -Type=simple +Type=notify +# `exec` is enough: the child process that takes over on SIGUSR2 is +# spawned through Command::spawn (fork + execve), which puts it in +# scope for sd_notify. `all` works too, but exec narrows the notify +# surface to what pg_doorman actually uses. +NotifyAccess=exec ExecStart=/usr/bin/pg_doorman /etc/pg_doorman.yaml +# `systemctl reload` triggers a full binary upgrade (config validated, +# state migrated, old process drained). pg_doorman's SIGUSR2 path runs +# `pg_doorman -t` first and aborts the reload if the config is broken, +# so this is safe to wire as the reload signal. ExecReload=/bin/kill -SIGUSR2 $MAINPID +# `systemctl stop` sends SIGTERM to the main pid for a graceful drain. +ExecStop=/bin/kill -SIGTERM $MAINPID +# Always restart: bring the pooler back even after a manual stop. +# Match the typical PostgreSQL-side HA expectation that the pool is +# never down on the host. +Restart=always +RestartSec=5s +# pg_doorman is connection-heavy: each client + each backend uses an +# fd, plus internal pipes. 65536 covers most OLTP pools; size it from +# `general.pool_size * num_pools` plus a few thousand for clients. +LimitNOFILE=65536 +# Run as a non-privileged service account that owns the PID file. On +# many deployments postgres already exists; reusing it keeps file +# ownership consistent with PostgreSQL itself. +User=postgres +Group=postgres +# Surface logs in journalctl under `pg-pooler` instead of the binary +# name; convenient for `journalctl -u pg_doorman -t pg-pooler`. +SyslogIdentifier=pg-pooler +TimeoutStopSec=30s ``` +`systemctl reload pg_doorman` drives the binary upgrade: pg_doorman +runs `-t` on the new binary, aborts the reload if the config is bad, +otherwise spawns the new process and drains the old one. `UPGRADE;` +from the admin console reaches the same code path. Tune +`TimeoutStopSec` so it covers `shutdown_timeout` in +`pg_doorman.yaml` plus a small margin; otherwise systemd SIGKILLs the +old process before the drain finishes. + +Production deployments often layer resource controls on top of the +above: `MemoryMax=`, `Nice=-15`, `CPUAffinity=2,3,4,5,6,7,8,9`. These +are workload-specific and orthogonal to the upgrade contract. + ## Configuration ### `shutdown_timeout` @@ -501,7 +574,11 @@ Before rolling out binary upgrade to production: both processes use the same certificate and key files - [ ] Test the upgrade in staging: open a session, trigger SIGUSR2, verify the session continues working -- [ ] Verify systemd unit has `ExecReload=/bin/kill -SIGUSR2 $MAINPID` +- [ ] Verify the systemd unit is `Type=notify` with + `NotifyAccess=exec`, `ExecReload=/bin/kill -SIGUSR2 $MAINPID` + (so `systemctl reload` runs a full binary upgrade with config + validation), and a `TimeoutStopSec` that covers + `shutdown_timeout` plus margin - [ ] Monitor logs for migration errors after the first production upgrade - [ ] Confirm old process exits (check PID file or `pgrep`) diff --git a/documentation/ru/src/tutorials/basic-usage.md b/documentation/ru/src/tutorials/basic-usage.md index 3f2f9a7d4..88e6b8759 100644 --- a/documentation/ru/src/tutorials/basic-usage.md +++ b/documentation/ru/src/tutorials/basic-usage.md @@ -21,7 +21,7 @@ Arguments: Options: -l, --log-level [env: LOG_LEVEL=] [default: INFO] -F, --log-format [env: LOG_FORMAT=] [default: text] [possible values: text, structured, debug] - -n, --no-color disable colors in the log output [env: NO_COLOR=] + -n, --no-color force colors off in the log output -d, --daemon run as daemon [env: DAEMON=] -h, --help Print help -V, --version Print version @@ -34,7 +34,7 @@ Options: | `-d`, `--daemon` | Запуск в фоне. Без этого параметра процесс работает на переднем плане.

В daemon-режиме обязательны `daemon_pid_file` и `syslog_prog_name`. После ухода в фон сообщения в stderr больше не пишутся. | | `-l`, `--log-level` | Уровень логирования: `INFO`, `DEBUG` или `WARN`. | | `-F`, `--log-format` | Формат логов. Возможные значения: `text`, `structured`, `debug`. | -| `-n`, `--no-color` | Отключить цвета в выводе логов. | +| `-n`, `--no-color` | Принудительно отключить цвета в логах. Цвета также автоматически выключаются, если stderr не терминал (под systemd — это pipe в journald) и если переменная окружения `NO_COLOR` установлена в любое непустое значение. | | `-V`, `--version` | Показать информацию о версии. | | `-h`, `--help` | Показать справку. | diff --git a/documentation/ru/src/tutorials/binary-upgrade.md b/documentation/ru/src/tutorials/binary-upgrade.md index ad9682f84..df0efb8f6 100644 --- a/documentation/ru/src/tutorials/binary-upgrade.md +++ b/documentation/ru/src/tutorials/binary-upgrade.md @@ -24,9 +24,22 @@ OpenSSL. ## Быстрый старт +```admonish tip title="Используйте дистрибутивный пакет, если он есть" +Если pg_doorman поставлен через `apt install pg-doorman` или +`dnf install pg-doorman`, пакетный менеджер выполняет шаг 1 атомарно +(`dpkg` пишет во временный путь и переименовывает, `rpm` делает то +же). Идиоматичный путь — `apt-get install --only-upgrade pg-doorman` +или `dnf upgrade pg-doorman`. Ручной `install` ниже — для разворотов +прямо бинарём, без пакетного менеджера. +``` + ```bash -# 1. Положите новый бинарник на место. -cp pg_doorman_new /usr/bin/pg_doorman +# 1. Атомарно замените бинарник. `install` пишет во временный файл и +# переименовывает его на целевой путь, поэтому inode, из которого +# отображены сегменты работающего процесса, не перезаписывается. +# `cp` правит живой inode и может уронить старый pg_doorman через +# SIGBUS / SIGSEGV. +install -m 0755 pg_doorman_new /usr/bin/pg_doorman # 2. Проверьте, что новый бинарник принимает текущий конфиг, до того # как запускать обновление. SIGUSR2 тоже прогоняет `-t` и отменяет @@ -34,12 +47,29 @@ cp pg_doorman_new /usr/bin/pg_doorman # не трогая работающий сервер. /usr/bin/pg_doorman -t /etc/pg_doorman.yaml -# 3. Запустите обновление. -kill -USR2 $(pgrep -f /usr/bin/pg_doorman) +# 3. Запустите обновление. С `ExecReload=/bin/kill -SIGUSR2 $MAINPID` +# в юните `systemctl reload` запускает полный binary upgrade +# (сначала валидация `pg_doorman -t`, потом миграция состояния, +# потом drain старого процесса). systemd шлёт сигнал в MainPID, +# поэтому выбирается ровно один процесс, даже если на хосте есть +# другие инстансы pg_doorman. Прямой `kill -USR2 $(pgrep -f +# /usr/bin/pg_doorman)` работает, но pgrep матчит по командной +# строке и может попасть во все инстансы сразу — пакетные +# установки используют systemctl. +sudo systemctl reload pg_doorman.service + +# 4. Проверьте: systemd подхватил новый MainPID (Type=notify получает +# `MAINPID=` от чайлда при handoff). Состояние active и +# админ-консоль подтверждают, что клиенты на месте. +systemctl show -p MainPID --value pg_doorman.service +psql -h pgdoorman -p 6432 -c 'SHOW POOLS;' # отвечает новый процесс +``` -# 4. Убедитесь: запущен новый PID, клиенты на месте. -pgrep -f /usr/bin/pg_doorman # новый PID -psql -h pgdoorman -p 6432 -c 'SHOW POOLS;' # отвечает уже новый процесс +```admonish tip title="Stand-alone установки без systemd" +Если процесс не управляется systemd, читайте PID-файл, который пишет +daemon (`daemon_pid_file`, по умолчанию `/tmp/pg_doorman.pid`), вместо +парсинга `pgrep`: `kill -USR2 "$(cat /var/run/pg_doorman/pg_doorman.pid)"`. +Foreground без systemd — шлите сигнал в PID супервизора процесса. ``` То же обновление можно запустить из консоли администратора: @@ -328,15 +358,64 @@ SHA-256 архива проверяется автоматически. | Рекомендуется для | systemd, контейнеры, Kubernetes | Старые установки | Для обновления без простоя и с миграцией клиентов запускайте PgDoorman -в foreground-режиме. systemd управляет жизненным циклом процесса: +в foreground-режиме. systemd управляет жизненным циклом процесса. +Используйте `Type=notify`, тогда unit становится `active` только после +сигнала готовности от pg_doorman, а новый процесс при `SIGUSR2` +обновляет `MainPID` на себя: ```ini [Service] -Type=simple +Type=notify +# `exec` достаточно: новый процесс при SIGUSR2 порождается через +# Command::spawn (fork + execve), он попадает в exec-scope notify. +# `all` тоже работает, но `exec` сужает поверхность только до того, +# что действительно делает pg_doorman. +NotifyAccess=exec ExecStart=/usr/bin/pg_doorman /etc/pg_doorman.yaml +# `systemctl reload` запускает полный binary upgrade (валидация +# конфига, миграция состояния, drain старого процесса). SIGUSR2 в +# pg_doorman сначала прогоняет `-t` и отменяет reload при битом +# конфиге, поэтому такой сигнал безопасно вешать на reload. ExecReload=/bin/kill -SIGUSR2 $MAINPID +# `systemctl stop` шлёт SIGTERM в main pid для graceful drain. +ExecStop=/bin/kill -SIGTERM $MAINPID +# Перезапуск всегда: возвращаем пул на ноге даже после ручной +# остановки. Соответствует ожиданию HA на стороне PostgreSQL, что пул +# никогда не лежит на хосте. +Restart=always +RestartSec=5s +# pg_doorman тяжёл по дескрипторам: каждый клиент и каждый backend — +# отдельный fd, плюс служебные pipe. 65536 покрывает большинство +# OLTP-пулов; считается как `general.pool_size * num_pools` плюс +# несколько тысяч на клиентов. +LimitNOFILE=65536 +# Сервисная учётка, которой принадлежит PID-файл. На многих установках +# postgres уже существует — переиспользование оставляет владение +# файлов согласованным с самим PostgreSQL. +User=postgres +Group=postgres +# Имя в journalctl: `pg-pooler` вместо имени бинаря; удобно для +# `journalctl -u pg_doorman -t pg-pooler`. +SyslogIdentifier=pg-pooler +TimeoutStopSec=30s ``` +`systemctl reload pg_doorman` запускает binary upgrade: pg_doorman +прогоняет `-t` на новом бинаре, отменяет reload при битом конфиге, +иначе порождает новый процесс и дренирует старый. `UPGRADE;` из +админ-консоли идёт по тому же code path. Подбирайте `TimeoutStopSec` +с запасом над `shutdown_timeout` из `pg_doorman.yaml`, иначе systemd +прибьёт старый процесс SIGKILL-ом до завершения дренажа. + +Прод-установки часто добавляют сверху ограничения ресурсов: +`MemoryMax=`, `Nice=-15`, `CPUAffinity=2,3,4,5,6,7,8,9`. Это +workload-специфика и она ортогональна контракту upgrade. + + +или `UPGRADE;` из админ-консоли. `TimeoutStopSec` подбирайте под +значение `shutdown_timeout` в `pg_doorman.yaml` плюс небольшой запас, +иначе systemd прибьёт процесс SIGKILL до окончания дренажа. + ## Конфигурация ### `shutdown_timeout` @@ -512,7 +591,10 @@ Raw fd клиента недоступен или `dup()` не удался. В оба процесса используют одинаковые файлы сертификата и ключа - [ ] Протестировать обновление на staging: открыть сессию, отправить SIGUSR2, убедиться что сессия продолжает работать -- [ ] В systemd unit есть `ExecReload=/bin/kill -SIGUSR2 $MAINPID` +- [ ] В systemd unit указано `Type=notify` с `NotifyAccess=exec`, + `ExecReload=/bin/kill -SIGUSR2 $MAINPID` (тогда `systemctl + reload` запускает полный binary upgrade с валидацией конфига) + и `TimeoutStopSec`, покрывающий `shutdown_timeout` с запасом - [ ] Мониторинг логов на ошибки миграции после первого промышленного обновления - [ ] Подтвердить что старый процесс завершился (PID-файл или `pgrep`) diff --git a/pkg/rpm/pg-doorman.spec b/pkg/rpm/pg-doorman.spec index f4608e5bd..bc7bd7d93 100644 --- a/pkg/rpm/pg-doorman.spec +++ b/pkg/rpm/pg-doorman.spec @@ -11,6 +11,13 @@ URL: https://github.com/ozontech/pg_doorman Source0: %{name}-%{version}.tar.gz Source1: vendor.tar.gz Source2: rust-%{rust_version}-x86_64-unknown-linux-gnu.tar.gz +# RHEL/CentOS Stream 7 and 8 ship perl-FindBin / perl-IPC-Cmd only +# inside modular Perl streams that COPR mock filters out by default, +# and COPR builders have no network access to `dnf module enable`. +# The CI workflow vendors the .pm files into this tarball; the build +# stanza extracts it and points PERL5LIB at the result so the build +# sees the modules regardless of distro. +Source3: perl_modules.tar.gz BuildRequires: gcc BuildRequires: gcc-c++ @@ -20,9 +27,7 @@ BuildRequires: cmake BuildRequires: clang BuildRequires: tar BuildRequires: patch -BuildRequires: perl -BuildRequires: perl-FindBin -BuildRequires: perl-IPC-Cmd +BuildRequires: perl-interpreter Requires: openssl @@ -39,6 +44,8 @@ This package includes: %setup -q -n %{name}-%{version} # Extract vendored dependencies tar xzf %{SOURCE1} +# Extract bundled Perl modules (FindBin, IPC::Cmd, ...) — see Source3 comment. +tar xzf %{SOURCE3} -C %{_builddir} %build # Install Rust toolchain from local tarball (COPR has no network access) @@ -48,6 +55,9 @@ tar xzf %{SOURCE2} -C %{_builddir} %{_builddir}/rust-%{rust_version}-x86_64-unknown-linux-gnu/install.sh --prefix="$RUST_INSTALL_DIR" --without=rust-docs export PATH="$RUST_INSTALL_DIR/bin:$PATH" +# Make the bundled pure-Perl modules visible to any build script that +# pulls them in (openssl-sys' Configure path uses FindBin and IPC::Cmd). +export PERL5LIB="%{_builddir}/perl_modules${PERL5LIB:+:$PERL5LIB}" # Configure cargo to use vendored dependencies mkdir -p .cargo diff --git a/src/app/args.rs b/src/app/args.rs index 4255d7433..d0d58beea 100644 --- a/src/app/args.rs +++ b/src/app/args.rs @@ -18,12 +18,16 @@ pub struct Args { #[clap(short='F', long, value_enum, default_value_t=LogFormat::Text, env)] pub log_format: LogFormat, + /// `env` is intentionally omitted: clap parses bool env vars as + /// `"true"`/`"false"` and would reject `NO_COLOR=1` at startup. + /// The logger respects the `NO_COLOR` env var per + /// independently of this flag and also + /// auto-disables colors when stderr is not a TTY. #[arg( short, long, default_value_t = false, - env, - help = "disable colors in the log output" + help = "force colors off in the log output" )] pub no_color: bool, diff --git a/src/app/logger.rs b/src/app/logger.rs index 153cc559b..bf1e211e6 100644 --- a/src/app/logger.rs +++ b/src/app/logger.rs @@ -40,13 +40,36 @@ fn init(args: &Args, syslog_name: Option) { } else { let inner: Box = match args.log_format { LogFormat::Structured => Box::new(JsonLogger::new()), - _ => Box::new(TextLogger::new(!args.no_color)), + _ => Box::new(TextLogger::new(should_use_color(args.no_color))), }; LogLevelController::new(inner, startup_level).register(); } } +/// Decide whether the text logger should emit ANSI colour escapes. +/// +/// The runtime answer threads three inputs through [`resolve_color`]: +/// the explicit `--no-color` flag, the standard `NO_COLOR` env var +/// (: any non-empty value disables colour), and +/// whether stderr is a TTY. Under systemd the unit's stderr is a pipe +/// to journald, not a terminal, so colour escapes used to land in the +/// journal as `[NNN blob data]` placeholders; auto-disabling for a +/// non-TTY stderr fixes that without requiring every operator to pass +/// `--no-color`. +fn should_use_color(no_color_flag: bool) -> bool { + use std::io::IsTerminal; + let env_no_color = std::env::var_os("NO_COLOR").is_some_and(|v| !v.is_empty()); + resolve_color(no_color_flag, env_no_color, std::io::stderr().is_terminal()) +} + +/// Pure decision used by [`should_use_color`]: colour is on only when +/// every gate is open. Split out so the matrix of inputs can be unit +/// tested without touching real env vars or a real terminal handle. +fn resolve_color(no_color_flag: bool, env_no_color: bool, stderr_is_tty: bool) -> bool { + !no_color_flag && !env_no_color && stderr_is_tty +} + /// Direct text logger — no tracing bridge. /// Format: `2024-01-07T19:19:38.080Z INFO pg_doorman::pool: message` struct TextLogger { @@ -169,3 +192,37 @@ impl Log for JsonLogger { let _ = std::io::stderr().flush(); } } + +#[cfg(test)] +mod tests { + use super::resolve_color; + + #[test] + fn color_on_only_when_every_gate_is_open() { + assert!(resolve_color( + /*no_color_flag=*/ false, /*env_no_color=*/ false, + /*stderr_is_tty=*/ true, + )); + } + + #[test] + fn color_off_when_no_color_flag_set() { + assert!(!resolve_color(true, false, true)); + } + + #[test] + fn color_off_when_no_color_env_set() { + // `NO_COLOR` is the open standard from https://no-color.org/: + // any non-empty value disables colour. The pure decision sees + // the boolean after the env lookup. + assert!(!resolve_color(false, true, true)); + } + + #[test] + fn color_off_when_stderr_is_not_a_tty() { + // The fix this guard exists for: under systemd the journal + // pipe is not a terminal, so default colour-on used to leak + // ANSI escapes into journalctl as `[NNN blob data]`. + assert!(!resolve_color(false, false, false)); + } +} diff --git a/tests/bdd/features/web-ui.feature b/tests/bdd/features/web-ui.feature index c3b6bd6b3..3ae9f53d6 100644 --- a/tests/bdd/features/web-ui.feature +++ b/tests/bdd/features/web-ui.feature @@ -115,8 +115,8 @@ Feature: Web UI listener curl -s http://127.0.0.1:9127/api/auth/config """ Then the command should succeed - And the command output should contain "\"sso_enabled\":false" - And the command output should contain "\"current_user\":null" + And the command output should contain '"sso_enabled":false' + And the command output should contain '"current_user":null' Scenario: /api/auth/config carries admin identity for Basic auth When I run shell command: @@ -124,8 +124,8 @@ Feature: Web UI listener curl -s --user 'admin:webui_bdd' http://127.0.0.1:9127/api/auth/config """ Then the command should succeed - And the command output should contain "\"role\":\"admin\"" - And the command output should contain "\"source\":\"basic\"" + And the command output should contain '"role":"admin"' + And the command output should contain '"source":"basic"' Scenario: /api/admin/reload requires Admin and rejects anonymous with 401 When I run shell command: @@ -164,8 +164,8 @@ Feature: Web UI listener """ Then the command should succeed # Each bind-address key must appear with changeable=no. - And the command output should contain "\"key\":\"general.host\"" - And the command output should contain "\"key\":\"general.port\"" - And the command output should contain "\"key\":\"web.host\"" - And the command output should contain "\"key\":\"web.port\"" - And the command output should contain "\"changeable\":\"no\"" + And the command output should contain '"key":"general.host"' + And the command output should contain '"key":"general.port"' + And the command output should contain '"key":"web.host"' + And the command output should contain '"key":"web.port"' + And the command output should contain '"changeable":"no"'