Merge pull request #19 from express-rate-limit/feat/improve-scalability

Feat/improve scalability

Author: adrianprelipcean

.github/workflows/cicd.yml

@@ -10,7 +10,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        node-version: [16.x, 18.x]
+        node-version: [16.x, lts/*, latest]
     steps:
       - uses: actions/checkout@v3
       - name: Use Node.js ${{ matrix.node-version }}

.mocharc.json

@@ -1,9 +1,6 @@
 {
 	"extension": ["ts"],
+	"loader": "ts-node/esm",
 	"spec": "test/**/*.spec.ts",
-	"node-option": [
-		"experimental-specifier-resolution=node",
-		"loader=ts-node/esm"
-	],
 	"recursive": true
 }

changelog.md

@@ -6,6 +6,13 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to
 [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [1.4.0](https://github.com/express-rate-limit/express-rate-limit-postgresql/releases/tag/v1.4.0)
+
+### Changed
+
+- Session handling is now done on the database layer to prevent concurrency
+  issues.
+
 ## [1.3.2](https://github.com/express-rate-limit/express-rate-limit-postgresql/releases/tag/v1.3.2)
 
 ### Added

contributing.md

@@ -23,7 +23,7 @@ Once you have installed the above, follow
 to
 [`fork`](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/working-with-forks)
 and [`clone`](https://github.com/git-guides/git-clone) the repository
-(`adrianprelipcean/express-rate-limit-postgresql`).
+(`express-rate-limit/rate-limit-postgresql`).
 
 Once you have forked and cloned the repository, you can
 [pick out an issue](https://github.com/express-rate-limit/express-rate-limit-postgresql/issues)
@@ -60,7 +60,6 @@ The library is written in
 of Node. The code is arranged as follows:
 
 ```sh
-rate-limit-postgresql
 ├── changelog.md
 ├── code_of_conduct.md
 ├── configs
@@ -71,6 +70,7 @@ rate-limit-postgresql
 ├── db
 │   ├── cli
 │   │   ├── apply_all_migrations.sh
+│   │   ├── create_migration.sh
 │   │   ├── init_db.sh
 │   │   └── run_tests.sh
 │   ├── linting
@@ -101,23 +101,21 @@ rate-limit-postgresql
 │   │   ├── 1-init.sql
 │   │   ├── 2-add-db-functions-agg.sql
 │   │   ├── 3-add-db-functions-ind.sql
-│   │   └── 4-add-db-functions-sessions.sql
-│   ├── models
-│   │   └── session.ts
+│   │   ├── 4-add-db-functions-sessions.sql
+│   │   ├── 5-hotfix-update-constraints.sql
+│   │   ├── 6-move-session-to-db-agg.sql
+│   │   └── 7-move-session-to-db-agg.sql
 │   ├── stores
 │   │   ├── aggregated_ip
 │   │   │   └── store_aggregated_ip.ts
 │   │   └── individual_ip
 │   │       └── store_individual_ip.ts
 │   └── util
-│       ├── migration_handler.ts
-│       └── session_handler.ts
+│       └── migration_handler.ts
 ├── test
-│   ├── stores
-│   │   ├── store_aggregated_ip.spec.ts
-│   │   └── store_individual_ip.spec.ts
-│   └── util
-│       └── session_handler.spec.ts
+│   └── stores
+│       ├── store_aggregated_ip.spec.ts
+│       └── store_individual_ip.spec.ts
 ├── third_party_licenses
 │   ├── dev_detailed.json
 │   ├── dev_summary.txt
@@ -222,20 +220,19 @@ changes to GitHub) your commits. To push your changes to your fork:
 ```
 
 If there are changes made to the `master` branch of the
-`adrianprelipcean/express-rate-limit-postgresql` repository, you may wish to
+`express-rate-limit/rate-limit-postgresql` repository, you may wish to
 [`rebase`](https://docs.github.com/en/get-started/using-git/about-git-rebase)
 your branch to include those changes. To rebase, or include the changes from the
-`master` branch of the `adrianprelipcean/express-rate-limit-postgresql`
-repository:
+`master` branch of the `express-rate-limit/rate-limit-postgresql` repository:
 
 ```
 > git fetch upstream master
 > git rebase upstream/master
 ```
 
 This will automatically add the changes from `master` branch of the
-`adrianprelipcean/express-rate-limit-postgresql` repository to the current
-branch. If you encounter any merge conflicts, follow
+`express-rate-limit/rate-limit-postgresql` repository to the current branch. If
+you encounter any merge conflicts, follow
 [this guide](https://docs.github.com/en/get-started/using-git/resolving-merge-conflicts-after-a-git-rebase)
 to resolve them.
 

db/tests/performance/aggregated_ip.test.sql

@@ -28,31 +28,31 @@ VALUES ('test-update', '00000000-0000-0000-0000-000000000000'::uuid, 20),
 
 SELECT performs_ok(
     $bd$
-    SELECT agg_increment as count FROM rate_limit.agg_increment('test-update', '00000000-0000-0000-0000-000000000000');
+    SELECT * from rate_limit.agg_increment('test-update', 'dedicated-test', 1000) AS (count int, expires_at timestamptz);
     $bd$,
     250,
     'inserting record should execute under 250ms'
 );
 
 SELECT performs_ok(
     $bd$
-    SELECT * FROM rate_limit.agg_decrement('test-decrement', '00000000-0000-0000-0000-000000000000');
+    SELECT * FROM rate_limit.agg_decrement('test-decrement', 'dedicated-test');
     $bd$,
     250,
     'decrementing query execute under 250ms'
 );
 
 SELECT performs_ok(
     $bd$
-    SELECT * FROM rate_limit.agg_reset_key('test-reset-key', '00000000-0000-0000-0000-000000000000')
+    SELECT * FROM rate_limit.agg_reset_key('test-reset-key', 'dedicated-test')
     $bd$,
     250,
     'resetting a key should execute under 250ms'
 );
 
 SELECT performs_ok(
     $bd$
-    SELECT * FROM rate_limit.agg_reset_session('00000000-0000-0000-0000-000000000000');
+    SELECT * FROM rate_limit.agg_reset_session('dedicated-test');
     $bd$,
     250,
     'resetting a session should execute under 250ms'

db/tests/performance/individual_ip.test.sql

@@ -28,31 +28,31 @@ VALUES ('test-count', '00000000-0000-0000-0000-000000000000'::uuid),
 
 SELECT performs_ok(
     $bd$
-    SELECT ind_increment as count FROM rate_limit.ind_increment('test-count', '00000000-0000-0000-0000-000000000000')
+    SELECT * FROM rate_limit.ind_increment('test-count', 'dedicated-test', 1000) AS (count int, expires_at timestamptz);
     $bd$,
     250,
     'inserting record should execute under 250ms'
 );
 
 SELECT performs_ok(
     $bd$
-    SELECT * FROM rate_limit.ind_decrement('test-decrement', '00000000-0000-0000-0000-000000000000');
+    SELECT * FROM rate_limit.ind_decrement('test-decrement', 'dedicated-test');
     $bd$,
     250,
     'decrementing query execute under 250ms'
 );
 
 SELECT performs_ok(
     $bd$
-    SELECT * FROM rate_limit.ind_reset_key('test-reset-key', '00000000-0000-0000-0000-000000000000');
+    SELECT * FROM rate_limit.ind_reset_key('test-reset-key', 'dedicated-test');
     $bd$,
     250,
     'resetting a key should execute under 250ms'
 );
 
 SELECT performs_ok(
     $bd$
-    SELECT * FROM rate_limit.ind_reset_session('00000000-0000-0000-0000-000000000000');
+    SELECT * FROM rate_limit.ind_reset_session('dedicated-test');
     $bd$,
     250,
     'resetting a session should execute under 250ms'

db/tests/unit/agg_decrement.test.sql

@@ -2,11 +2,12 @@ BEGIN;
 -- Plan the tests.
 SELECT plan(2);
 
-INSERT INTO rate_limit.sessions (id, name_, type_)
+INSERT INTO rate_limit.sessions (id, name_, type_, expires_at)
 SELECT
     '00000000-0000-0000-0000-000000000000'::uuid AS id,
     'dedicated-test'::text AS name_,
-    'aggregated'::text AS type_;
+    'aggregated'::text AS type_,
+    '2023-01-01 10:00:00+0' AS expires_at;
 
 INSERT INTO rate_limit.records_aggregated (key, session_id)
 SELECT
@@ -15,7 +16,7 @@ SELECT
 
 SELECT lives_ok(
     $have$
-    SELECT * FROM rate_limit.agg_decrement('existing-key', '00000000-0000-0000-0000-000000000000')
+    SELECT * FROM rate_limit.agg_decrement('existing-key', 'dedicated-test', '2023-01-01 09:00:00+0')
     $have$,
     'rate_limit.agg_decrement does not throw an error'
 );

db/tests/unit/agg_increment.test.sql

@@ -1,17 +1,25 @@
 BEGIN;
 -- Plan the tests.
-SELECT plan(3);
+SELECT plan(5);
 
-INSERT INTO rate_limit.sessions (id, name_, type_)
+INSERT INTO rate_limit.sessions (id, name_, type_, expires_at)
 SELECT
     '00000000-0000-0000-0000-000000000000'::uuid AS id,
     'dedicated-test'::text AS name_,
-    'aggregated'::text AS type_
+    'aggregated'::text AS type_,
+    '2023-01-01 10:00:00+0'::timestamptz AS expires_at
 UNION
 SELECT
     '00000000-1111-1111-1111-000000000000'::uuid AS id,
     'dedicated-test-2'::text AS name_,
-    'aggregated'::text AS type_;;
+    'aggregated'::text AS type_,
+    '2023-01-01 10:00:00+0' AS expires_at
+UNION
+SELECT
+    '00000000-2222-2222-2222-000000000000'::uuid AS id,
+    'dedicated-test-expired'::text AS name_,
+    'aggregated'::text AS type_,
+    '2023-01-01 08:00:00+0' AS expires_at;
 
 INSERT INTO rate_limit.records_aggregated (key, session_id)
 SELECT
@@ -20,34 +28,59 @@ SELECT
 
 SELECT results_eq(
     $have$
-    SELECT agg_increment AS count FROM rate_limit.agg_increment('new-key', '00000000-0000-0000-0000-000000000000')
+    SELECT * FROM rate_limit.agg_increment('new-key', 'dedicated-test', 1000, '2023-01-01 09:00:00+0') AS (count int, expires_at timestamptz);
     $have$,
     $want$
-    SELECT 1::int AS count;
+    SELECT 1::int AS count, '2023-01-01 10:00:00+0'::timestamptz as expires_at;
     $want$,
     'rate_limit.agg_increment returns correct count for new key'
 );
 
 SELECT results_eq(
     $have$
-    SELECT agg_increment AS count FROM rate_limit.agg_increment('new-key', '00000000-1111-1111-1111-000000000000')
+    SELECT * FROM rate_limit.agg_increment('new-key', 'dedicated-test-2', 1000, '2023-01-01 09:00:00+0') AS (count int, expires_at timestamptz);
     $have$,
     $want$
-    SELECT 1::int AS count;
+    SELECT 1::int AS count, '2023-01-01 10:00:00+0'::timestamptz as expires_at;
     $want$,
     'rate_limit.agg_increment returns correct count for new key on different session'
 );
 
 SELECT results_eq(
     $have$
-    SELECT agg_increment AS count FROM rate_limit.agg_increment('existing-key', '00000000-0000-0000-0000-000000000000')
+    SELECT * FROM rate_limit.agg_increment('existing-key', 'dedicated-test', 1000, '2023-01-01 09:00:00+0') AS (count int, expires_at timestamptz);
     $have$,
     $want$
-    SELECT 2::int AS count;
+    SELECT 2::int AS count, '2023-01-01 10:00:00+0'::timestamptz as expires_at;
     $want$,
     'rate_limit.agg_increment returns correct count for existing key'
 );
 
+SELECT results_eq(
+    $have$
+    SELECT * FROM rate_limit.agg_increment('existing-key', 'dedicated-test-expired', 1000, '2023-01-01 09:00:00+0') AS (count int, expires_at timestamptz);
+    $have$,
+    $want$
+    SELECT 1::int AS count, '2023-01-01 09:00:01+0'::timestamptz as expires_at;
+    $want$,
+    'rate_limit.agg_increment returns correct count for existing key for expired session'
+);
+
+SELECT bag_hasnt(
+    $have$
+    SELECT id, name_, type_, expires_at FROM rate_limit.sessions
+    $have$,
+    $miss$
+    SELECT
+    '00000000-2222-2222-2222-000000000000'::uuid AS id,
+    'dedicated-test-expired'::text AS name_,
+    'aggregated'::text AS type_,
+    '2023-01-01 08:00:00+0'::timestamptz as expires_at;
+    $miss$,
+    'expired session should not be reset after increment invoke'
+);
+
+
 -- Finish the tests and clean up.
 SELECT finish FROM finish();
 ROLLBACK;

db/tests/unit/agg_reset_key.test.sql

@@ -2,11 +2,12 @@ BEGIN;
 -- Plan the tests.
 SELECT plan(2);
 
-INSERT INTO rate_limit.sessions (id, name_, type_)
+INSERT INTO rate_limit.sessions (id, name_, type_, expires_at)
 SELECT
     '00000000-0000-0000-0000-000000000000'::uuid AS id,
     'dedicated-test'::text AS name_,
-    'aggregated'::text AS type_;
+    'aggregated'::text AS type_,
+    '2023-01-01 10:00+0'::timestamptz AS expires_at;
 
 INSERT INTO rate_limit.records_aggregated (key, session_id)
 SELECT
@@ -15,7 +16,7 @@ SELECT
 
 SELECT lives_ok(
     $have$
-    SELECT * FROM rate_limit.agg_reset_key('existing-key', '00000000-0000-0000-0000-000000000000')
+    SELECT * FROM rate_limit.agg_reset_key('existing-key', 'dedicated-test', '2023-01-01 09:00+0'::timestamptz )
     $have$,
     'rate_limit.agg_reset_key does not throw an error'
 );

db/tests/unit/agg_reset_session.test.sql

@@ -2,11 +2,12 @@ BEGIN;
 -- Plan the tests.
 SELECT plan(2);
 
-INSERT INTO rate_limit.sessions (id, name_, type_)
+INSERT INTO rate_limit.sessions (id, name_, type_, expires_at)
 SELECT
     '00000000-0000-0000-0000-000000000000'::uuid AS id,
     'dedicated-test'::text AS name_,
-    'aggregated'::text AS type_;
+    'aggregated'::text AS type_,
+    '2023-01-01 10:00+0'::timestamptz AS expires_at;
 
 INSERT INTO rate_limit.records_aggregated (key, session_id)
 SELECT
@@ -19,7 +20,7 @@ SELECT
 
 SELECT lives_ok(
     $have$
-    SELECT * FROM rate_limit.agg_reset_session('00000000-0000-0000-0000-000000000000')
+    SELECT * FROM rate_limit.agg_reset_session('dedicated-test', '2023-01-01 09:00+0')
     $have$,
     'rate_limit.agg_reset_session does not throw an error'
 );

db/tests/unit/ind_decrement.test.sql

@@ -2,11 +2,12 @@ BEGIN;
 -- Plan the tests.
 SELECT plan(2);
 
-INSERT INTO rate_limit.sessions (id, name_, type_)
+INSERT INTO rate_limit.sessions (id, name_, type_, expires_at)
 SELECT
     '00000000-0000-0000-0000-000000000000'::uuid AS id,
     'dedicated-test'::text AS name_,
-    'individual'::text AS type_;
+    'individual'::text AS type_,
+    '2023-01-01 10:00:00+0'::timestamptz AS expires_at;
 
 INSERT INTO rate_limit.individual_records (key, session_id)
 SELECT
@@ -23,7 +24,7 @@ SELECT
 
 SELECT lives_ok(
     $have$
-    SELECT * FROM rate_limit.ind_decrement('existing-key', '00000000-0000-0000-0000-000000000000')
+    SELECT * FROM rate_limit.ind_decrement('existing-key', 'dedicated-test', '2023-01-01 09:00:00+0'::timestamptz)
     $have$,
     'rate_limit.ind_decrement does not throw an error'
 );