diff --git a/nix/tools/tests.nix b/nix/tools/tests.nix index 27b9ad366a..2bbac03d39 100644 --- a/nix/tools/tests.nix +++ b/nix/tools/tests.nix @@ -9,6 +9,7 @@ , hpc-codecov , jq , lib +, nginx , postgrest , python3 , runtimeShell @@ -93,6 +94,7 @@ let args = [ "ARG_LEFTOVERS([pytest arguments])" ]; workingDir = "/"; withEnv = postgrest.env; + withPath = [ nginx ]; } '' ${cabal-install}/bin/cabal v2-build ${devCabalOptions} exe:postgrest @@ -155,6 +157,7 @@ let redirectTixFiles = false; withEnv = postgrest.env; withTmpDir = true; + withPath = [ nginx ]; } ( # required for `hpc markup` in CI; glibcLocales is not available e.g. on Darwin diff --git a/test/io/config.py b/test/io/config.py index 74d754d169..b31159e590 100644 --- a/test/io/config.py +++ b/test/io/config.py @@ -10,6 +10,7 @@ FIXTURES = yaml.load( (BASEDIR / "fixtures/fixtures.yaml").read_text(), Loader=yaml.Loader ) +NGINX_BIN = shutil.which("nginx") POSTGREST_BIN = shutil.which("postgrest") SECRET = "reallyreallyreallyreallyverysafe" diff --git a/test/io/nginx/nginx.conf b/test/io/nginx/nginx.conf new file mode 100644 index 0000000000..f29aa9e7ea --- /dev/null +++ b/test/io/nginx/nginx.conf @@ -0,0 +1,13 @@ +# the PG* variables are replaced by preprocessing, not done by nginx itself +daemon off; +pid ./nginx.pid; + +events {} + +stream { + server { + listen unix:$PGPROXYHOST/.s.PGSQL.5432; + proxy_timeout $PGPROXY_TIMEOUT; + proxy_pass unix:$PGHOST/.s.PGSQL.5432; + } +} diff --git a/test/io/postgrest.py b/test/io/postgrest.py index 8e23387e1f..d4c2aaefc2 100644 --- a/test/io/postgrest.py +++ b/test/io/postgrest.py @@ -8,12 +8,13 @@ import subprocess import tempfile import time +import string import urllib.parse import requests import requests_unixsocket -from config import POSTGREST_BIN, hpctixfile +from config import POSTGREST_BIN, NGINX_BIN, hpctixfile def sleep_until_postgrest_scache_reload(): @@ -170,6 +171,52 @@ def run( process.wait() +@contextlib.contextmanager +def run_pgproxy(env=None, proxy_timeout="1s"): + "Run nginx as a unix socket proxy for PostgreSQL and expose PGPROXYHOST." + env = dict(os.environ if env is None else env) + + with tempfile.TemporaryDirectory() as tmpdir: + # build a /conf/ so `nginx -p` picks the config automatically + tmpdir = pathlib.Path(tmpdir) + conf_dir = tmpdir / "conf" + conf_dir.mkdir(parents=True) + + nginx_env = dict(env) + nginx_env["PGPROXYHOST"] = str(tmpdir) + nginx_env["PGPROXY_TIMEOUT"] = proxy_timeout + + source_conf = pathlib.Path("test/io/nginx/nginx.conf") + out_conf = conf_dir / "nginx.conf" + out_conf.write_text( + string.Template(source_conf.read_text()).substitute(nginx_env) + ) + + process = subprocess.Popen( + [NGINX_BIN, "-p", str(tmpdir), "-e", "stderr"], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + env=nginx_env, + ) + + if process.poll() is not None: + (_, stderr_output) = process.communicate(timeout=1) + raise RuntimeError( + f"{NGINX_BIN} exited with {process.returncode}: {stderr_output}" + ) + + try: + yield str(tmpdir) + finally: + process.terminate() + try: + process.wait(timeout=1) + except subprocess.TimeoutExpired: + process.kill() + process.wait() + + def freeport(used_ports=None): "Find an unused free port on localhost." while True: diff --git a/test/io/test_io.py b/test/io/test_io.py index 4b8c9f9acf..56cdcba8a6 100644 --- a/test/io/test_io.py +++ b/test/io/test_io.py @@ -14,6 +14,7 @@ is_ipv6, reset_statement_timeout, run, + run_pgproxy, set_statement_timeout, sleep_until_postgrest_config_reload, sleep_until_postgrest_full_reload, @@ -2178,3 +2179,27 @@ def test_vary_default_header_set(defaultenv): response = postgrest.session.get("/projects") assert response.headers["Vary"] == "Accept, Prefer, Range" + + +@pytest.mark.xfail( + reason="pgrst_db_pool_available should not go negative on pg network failures", + strict=True, +) +def test_positive_pool_metric(defaultenv): + "When a network failure is caused on the pg connection, pgrst_db_pool_available stays positive" + + with run_pgproxy(defaultenv, proxy_timeout="10ms") as pgproxyhost: + env = {**defaultenv, "PGHOST": pgproxyhost} + + with run(env=env, wait_for_readiness=False) as postgrest: + time.sleep(2) + + response = postgrest.admin.get("/metrics", timeout=1) + assert response.status_code == 200 + + metrics = float( + re.search( + r"pgrst_db_pool_available (-?\d+(?:\.\d+)?)", response.text + ).group(1) + ) + assert metrics >= 0