test: loadtest with varying reqs and batch_size

Checks performance and CPU/MEM usage of different batch_size and number of requests.

The results show on CI, and can be shown the same locally too.

Author: steve-chavez

.github/workflows/main.yml

@@ -55,6 +55,9 @@ jobs:
 
   loadtest:
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        params: [ {reqs: 10000, batch: 200}, {reqs: 20000, batch: 400}, {reqs: 40000, batch: 800} ]
     steps:
       - uses: actions/checkout@v4
 
@@ -72,8 +75,7 @@ jobs:
 
       - name: Run load test
         run: |
-          nix-shell --run "net-loadtest"
-          cat psrecord.md >> "$GITHUB_STEP_SUMMARY"
+          nix-shell --run "net-loadtest ${{ matrix.params.reqs }} ${{ matrix.params.batch }}" >> "$GITHUB_STEP_SUMMARY"
 
   coverage:
 

.gitignore

@@ -19,6 +19,5 @@ coverage_html
 nginx.pid
 tags
 net_worker.pid
-psrecord.*
 pg_net--*.sql
 !pg_net--*--*.sql

nix/loadtest.nix

@@ -1,7 +1,7 @@
 { writeShellScriptBin, psrecord, writers, python3Packages } :
 
 let
-  toMarkdown =
+  psrecordToMd =
     writers.writePython3 "psrecord-to-md"
       {
         libraries = [ python3Packages.pandas python3Packages.tabulate ];
@@ -35,22 +35,59 @@ let
 
         df.to_markdown(sys.stdout, index=False, tablefmt="github")
       '';
+
+  csvToMd =
+    writers.writePython3 "csv-to-md"
+      {
+        libraries = [ python3Packages.pandas python3Packages.tabulate ];
+      }
+      ''
+        import sys
+        import pandas as pd
+
+        pd.read_csv(sys.stdin) \
+          .fillna("") \
+          .convert_dtypes() \
+          .to_markdown(sys.stdout, index=False, floatfmt='.0f')
+      '';
+
 in
 
 writeShellScriptBin "net-loadtest" ''
   set -euo pipefail
 
-  net-with-nginx xpg psql -c "call wait_for_many_gets()" > /dev/null &
+  reqs=""
+  batch_size_opt=""
+
+  load_dir=test/load
+  mkdir -p $load_dir
+  echo "*" >> $load_dir/.gitignore
+
+  record_result=$load_dir/psrecord.md
+  query_result=$load_dir/query_out.md
+
+  query_csv=$load_dir/query.csv
+  record_log=$load_dir/psrecord.log
+
+  if [ -n "''${1:-}" ]; then
+    reqs="$1"
+  fi
+
+  if [ -n "''${2:-}" ]; then
+    batch_size_opt="-c pg_net.batch_size=$2"
+  fi
+
+  net-with-nginx xpg --options "-c log_min_messages=WARNING $batch_size_opt" \
+    psql -c "call wait_for_many_gets($reqs)" -c "\pset format csv" -c "\o $query_csv" -c "select * from pg_net_stats" > /dev/null &
 
   # wait for process to start so we can capture it with psrecord
   sleep 2
 
-  record_log=psrecord.log
-  record_result=psrecord.md
-
   ${psrecord}/bin/psrecord $(cat build-17/bgworker.pid) --interval 1 --log "$record_log" > /dev/null
 
-  cat $record_log | ${toMarkdown} > $record_result
+  echo -e "## Loadtest results\n"
+  cat $query_csv  | ${csvToMd}
 
-  echo "generated $record_result"
+  echo -e "\n\n## Loadtest elapsed seconds vs CPU/MEM\n"
+  cat $record_log | ${psrecordToMd}
 ''

nix/xpg.nix

@@ -3,8 +3,8 @@ let
   dep = fetchFromGitHub {
     owner  = "steve-chavez";
     repo   = "xpg";
-    rev    = "v1.4.1";
-    sha256 = "sha256-OI9g78KbguLh+ynOnRmnMM4lVOgNRAWkiI/YMmcMs+k=";
+    rev    = "v1.6.0";
+    sha256 = "sha256-NsdAmsYIRH/DWIZp93AHGYdPiJOztUIUSYcPikeebvw=";
   };
   xpg = import dep;
 in

test/utils/loadtest.sql

@@ -1,8 +1,11 @@
+create table time_taken (time_taken interval);
+
 create view pg_net_stats as
 select
   count(*) filter (where error_msg is null) as request_successes,
   count(*) filter (where error_msg is not null) as request_failures,
-  (select error_msg from net._http_response where error_msg is not null order by id desc limit 1) as last_failure_error
+  (select error_msg from net._http_response where error_msg is not null order by id desc limit 1) as last_failure_error,
+  (select time_taken from time_taken limit 1) as time_taken
 from net._http_response;
 
 -- loadtest using many gets, used to be called `repro_timeouts`
@@ -24,16 +27,14 @@ begin
 
   commit;
 
-  raise notice 'Waiting until % requests complete', number_of_requests;
+  raise notice 'Waiting until % requests complete, using a pg_net.batch_size of %', number_of_requests, current_setting('pg_net.batch_size')::text;
 
   perform net._await_response(last_id);
 
   select clock_timestamp() into second_time;
 
   select age(second_time, first_time) into time_taken;
 
-  raise notice 'Stats: %', (select to_json(x) from pg_net_stats x limit 1);
-
-  raise notice 'Time taken: %', time_taken;
+  insert into time_taken values (time_taken);
 end;
 $$ language plpgsql;