Skip to content

Commit 0077aa2

Browse files
mkleczekmkleczek
committed
test(refactor): add withToxiproxy* in withTools
This commit introduces toxiproxy in withTools adding the following helpers: * withToxiproxyServer * withToxiproxyProxy * withToxiproxyPgProxy
1 parent 5f6f7dc commit 0077aa2

4 files changed

Lines changed: 122 additions & 11 deletions

File tree

‎nix/tools/withTools.nix‎

Lines changed: 115 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
, python3Packages
99
, writeText
1010
, writers
11+
, toxiproxy
1112
}:
1213
let
1314
withTmpDb =
@@ -54,16 +55,25 @@ let
5455
5556
export PGDATA="$tmpdir/db"
5657
export PGHOST="$tmpdir/socket"
58+
PGPORT=$(${randomPort})
59+
export PGPORT
5760
export PGUSER
5861
export PGDATABASE
5962
export PGRST_DB_SCHEMAS
6063
export PGTZ
6164
export PGOPTIONS
6265
6366
HBA_FILE="$tmpdir/pg_hba.conf"
64-
echo "local $PGDATABASE some_protected_user password" > "$HBA_FILE"
65-
echo "local $PGDATABASE all trust" >> "$HBA_FILE"
66-
echo "local replication all trust" >> "$HBA_FILE"
67+
{
68+
echo "local $PGDATABASE some_protected_user password"
69+
echo "local $PGDATABASE all trust"
70+
echo "local replication all trust"
71+
echo "host $PGDATABASE some_protected_user localhost password"
72+
echo "host $PGDATABASE all localhost trust"
73+
} >> "$HBA_FILE"
74+
75+
UNIX_PGHOST="$PGHOST"
76+
export TCP_PGHOST="localhost"
6777
6878
log "Initializing database cluster..."
6979
# We try to make the database cluster as independent as possible from the host
@@ -80,7 +90,7 @@ let
8090
# On MacOS, it's 104 chars
8191
# See: https://serverfault.com/questions/641347/check-if-a-path-exceeds-maximum-for-unix-domain-socket
8292
83-
pg_ctl -l "$tmpdir/db.log" -w start -o "-F -c listen_addresses=\"\" -c hba_file=$HBA_FILE -k $PGHOST -c log_statement=\"all\" " \
93+
pg_ctl -l "$tmpdir/db.log" -w start -o "-F -c listen_addresses=\"$TCP_PGHOST\" -c hba_file=$HBA_FILE -k $UNIX_PGHOST -c log_statement=\"all\" " \
8494
>> "$setuplog"
8595
8696
log "Creating a minimally privileged $PGUSER connection role..."
@@ -93,6 +103,7 @@ let
93103
replica_slot="replica_$RANDOM"
94104
replica_dir="$tmpdir/$replica_slot"
95105
replica_host="$tmpdir/socket_$replica_slot"
106+
replica_port=$(${randomPort})
96107
97108
mkdir -p "$replica_host"
98109
@@ -106,15 +117,16 @@ let
106117
log "Starting replica on $replica_host"
107118
108119
# We set a low max_standby_streaming_delay to make the replication conflict fail faster in tests (otherwise it waits for the default 30s)
109-
pg_ctl -D "$replica_dir" -l "$replica_dblog" -w start -o "-F -c listen_addresses=\"\" -c hba_file=$HBA_FILE -k $replica_host -c log_statement=\"all\" -c max_standby_streaming_delay=\"3s\" " \
120+
pg_ctl -D "$replica_dir" -l "$replica_dblog" -w start -o "-F -c listen_addresses=\"$TCP_PGHOST\" -c port=$replica_port -c hba_file=$HBA_FILE -k $replica_host -c log_statement=\"all\" -c max_standby_streaming_delay=\"3s\" " \
110121
>> "$setuplog"
111122
112123
>&2 echo "${commandName}: Replica enabled. You can connect to it with: psql 'postgres:///$PGDATABASE?host=$replica_host' -U postgres"
113124
>&2 echo "${commandName}: You can tail the replica logs with: tail -f $replica_dblog"
114125
115126
export PGREPLICAHOST="$replica_host"
127+
export PGREPLICAPORT="$replica_port"
116128
export PGREPLICASLOT="$replica_slot"
117-
export PGRST_DB_URI="postgres:///$PGDATABASE?host=$PGREPLICAHOST,$PGHOST"
129+
export PGRST_DB_URI="postgres:///$PGDATABASE?host=$PGREPLICAHOST,$PGHOST&port=$replica_port,$PGPORT"
118130
fi
119131
120132
# shellcheck disable=SC2329
@@ -372,6 +384,100 @@ let
372384
libraries = [ python3Packages.pandas python3Packages.tabulate python3Packages.psutil ];
373385
}
374386
(builtins.readFile ./monitor_pid.py);
387+
388+
randomPort =
389+
writers.writePython3 "postgrest-random-port"
390+
{ }
391+
''
392+
import socket
393+
s = socket.socket()
394+
s.bind(("127.0.0.1", 0))
395+
print(s.getsockname()[1])
396+
s.close()
397+
'';
398+
399+
withToxiproxyProxy =
400+
checkedShellScript
401+
{
402+
name = "postgrest-with-toxiproxy-proxy";
403+
docs = "Run <command> with Toxiproxy proxy created.";
404+
args =
405+
[
406+
"ARG_POSITIONAL_SINGLE([command], [Command to run])"
407+
"ARG_LEFTOVERS([command arguments])"
408+
"ARG_OPTIONAL_SINGLE([listen], [l], [Proxy will listen on this address])"
409+
"ARG_OPTIONAL_SINGLE([upstream], [u], [Proxy will forward to this address])"
410+
];
411+
positionalCompletion = "_command";
412+
workingDir = "/";
413+
withPath = [ toxiproxy ];
414+
}
415+
''
416+
proxyname="tp$RANDOM"
417+
toxiproxy-cli create -l "$_arg_listen" -u "$_arg_upstream" "$proxyname"
418+
419+
# shellcheck disable=SC2329
420+
stop () {
421+
toxiproxy-cli delete "$proxyname" || true
422+
}
423+
trap stop EXIT
424+
425+
(TOXI_PROXY_NAME="$proxyname" "$_arg_command" "''${_arg_leftovers[@]}")
426+
'';
427+
428+
withToxiproxyPg =
429+
checkedShellScript
430+
{
431+
name = "postgrest-with-toxiproxy-pg";
432+
docs = "Run <command> with a Toxiproxy proxy to PostgreSQL.";
433+
args =
434+
[
435+
"ARG_POSITIONAL_SINGLE([command], [Command to run])"
436+
"ARG_LEFTOVERS([command arguments])"
437+
"ARG_USE_ENV([TCP_PGHOST], [], [PG host name])"
438+
"ARG_USE_ENV([PGPORT], [], [PG port])"
439+
];
440+
positionalCompletion = "_command";
441+
workingDir = "/";
442+
}
443+
''
444+
proxy_port=''$(${randomPort})
445+
446+
${withToxiproxy} ${withToxiproxyProxy} -l "$TCP_PGHOST:$proxy_port" -u "$TCP_PGHOST:$PGPORT" \
447+
env "TOXI_PGPORT=$proxy_port" "$_arg_command" "''${_arg_leftovers[@]}"
448+
'';
449+
450+
withToxiproxy =
451+
checkedShellScript
452+
{
453+
name = "postgrest-with-toxiproxy";
454+
docs = "Run <command> with toxiproxy-server";
455+
args =
456+
[
457+
"ARG_POSITIONAL_SINGLE([command], [Command to run])"
458+
"ARG_LEFTOVERS([command arguments])"
459+
];
460+
positionalCompletion = "_command";
461+
workingDir = "/";
462+
withPath = [ toxiproxy ];
463+
}
464+
''
465+
if ! test -v TOXI_PROXY; then
466+
export TOXI_PROXY=""
467+
LOG_LEVEL=error toxiproxy-server&
468+
TOXIPROXY_PID=$!
469+
sleep 1 # give the server a moment to start
470+
471+
# shellcheck disable=SC2329
472+
stop () {
473+
kill "$TOXIPROXY_PID" || true
474+
wait "$TOXIPROXY_PID" || true
475+
}
476+
trap stop EXIT
477+
fi
478+
("$_arg_command" "''${_arg_leftovers[@]}")
479+
'';
480+
375481
in
376482
buildToolbox
377483
{
@@ -380,11 +486,12 @@ buildToolbox
380486
inherit
381487
withGit
382488
withPgAll
383-
withPgrst;
489+
withPgrst
490+
withToxiproxyPg;
384491
} // builtins.listToAttrs (
385492
# Create a `postgrest-with-pg-` for each PostgreSQL version
386493
builtins.map (pg: { inherit (pg) name; value = withTmpDb pg; }) postgresqlVersions
387494
);
388495
# make latest withPg available for other nix files
389-
extra = { inherit withPg; };
496+
extra = { inherit withPg withToxiproxyPg; };
390497
}

‎test/io/conftest.py‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,8 +9,9 @@ def dburi():
99
"Postgres database connection URI."
1010
dbname = os.environ["PGDATABASE"]
1111
host = os.environ["PGHOST"]
12+
port = os.environ["PGPORT"]
1213
user = os.environ["PGUSER"]
13-
return f"postgresql://?dbname={dbname}&host={host}&user={user}".encode()
14+
return f"postgresql://?dbname={dbname}&host={host}&port={port}&user={user}".encode()
1415

1516

1617
@pytest.fixture
@@ -19,6 +20,7 @@ def baseenv():
1920
return {
2021
"PGDATABASE": os.environ["PGDATABASE"],
2122
"PGHOST": os.environ["PGHOST"],
23+
"PGPORT": os.environ["PGPORT"],
2224
"PGUSER": os.environ["PGUSER"],
2325
}
2426

@@ -51,6 +53,7 @@ def replicaenv(defaultenv):
5153
**defaultenv,
5254
**conf,
5355
"PGHOST": os.environ["PGREPLICAHOST"] + "," + os.environ["PGHOST"],
56+
"PGPORT": os.environ["PGREPLICAPORT"] + "," + os.environ["PGPORT"],
5457
"PGREPLICASLOT": os.environ["PGREPLICASLOT"],
5558
},
5659
}
@@ -76,6 +79,7 @@ def metapostgrest():
7679
env = {
7780
"PGDATABASE": os.environ["PGDATABASE"],
7881
"PGHOST": os.environ["PGHOST"],
82+
"PGPORT": os.environ["PGPORT"],
7983
"PGUSER": role,
8084
"PGRST_DB_ANON_ROLE": role,
8185
"PGRST_DB_CONFIG": "true",

‎test/io/test_auth.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -165,7 +165,7 @@ def relativeSeconds(sec):
165165

166166
def test_fail_with_invalid_password(defaultenv):
167167
"Connecting with an invalid password should fail without retries."
168-
uri = f'postgresql://?dbname={defaultenv["PGDATABASE"]}&host={defaultenv["PGHOST"]}&user=some_protected_user&password=invalid_pass'
168+
uri = f'postgresql://?dbname={defaultenv["PGDATABASE"]}&host={defaultenv["PGHOST"]}&port={defaultenv["PGPORT"]}&user=some_protected_user&password=invalid_pass'
169169
env = {**defaultenv, "PGRST_DB_URI": uri}
170170
with run(env=env, wait_for_readiness=False) as postgrest:
171171
exitCode = wait_until_exit(postgrest)

‎test/io/test_io.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1849,7 +1849,7 @@ def test_log_listener_connection_start(defaultenv):
18491849
# Check for the listener start message containing host and port
18501850
# Do not check if pg version is displayed properly as it is tricky to test it
18511851
assert any(
1852-
f'"{defaultenv["PGHOST"]}:5432" and listening for database notifications on the "pgrst" channel'
1852+
f'"{defaultenv["PGHOST"]}:{defaultenv["PGPORT"]}" and listening for database notifications on the "pgrst" channel'
18531853
in line
18541854
for line in output
18551855
)

0 commit comments

Comments
 (0)