diff --git a/.github/workflows/benchmark-smoke.yml b/.github/workflows/benchmark-smoke.yml new file mode 100644 index 000000000..58ec28793 --- /dev/null +++ b/.github/workflows/benchmark-smoke.yml @@ -0,0 +1,27 @@ +name: Benchmark smoke + +on: + pull_request: + paths: + - "apps/api/**" + - "benchmarks/**" + - "package.json" + - "package-lock.json" + - ".github/workflows/benchmark-smoke.yml" + workflow_dispatch: + +jobs: + benchmark-smoke: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-node@v4 + with: + node-version: 22 + cache: npm + + - run: npm ci + + - run: npm run benchmark:smoke diff --git a/apps/api/package.json b/apps/api/package.json index 25fa0e90e..16556de8f 100644 --- a/apps/api/package.json +++ b/apps/api/package.json @@ -5,7 +5,7 @@ "scripts": { "dev": "node src/server.js", "start": "node src/server.js", - "test": "node --test src/tests" + "test": "node --test src/tests/*.test.js" }, "dependencies": { "cors": "^2.8.5", diff --git a/benchmarks/.env.benchmark.example b/benchmarks/.env.benchmark.example new file mode 100644 index 000000000..fa71a137c --- /dev/null +++ b/benchmarks/.env.benchmark.example @@ -0,0 +1,10 @@ +# Optional. If omitted, the benchmark runner starts the local Express app on an ephemeral port. +BENCHMARK_TARGET_URL=http://127.0.0.1:4000 + +# Used when the benchmark runner starts the local app and when it requests a benchmark auth token. +JWT_SECRET=benchmark-secret + +# Tuning knobs. Defaults are intentionally small to avoid tripping the API rate limiter. +BENCHMARK_MODE=full +BENCHMARK_REQUESTS_PER_ENDPOINT=5 +BENCHMARK_CONCURRENCY=2 diff --git a/benchmarks/README.md b/benchmarks/README.md new file mode 100644 index 000000000..7117bd2c1 --- /dev/null +++ b/benchmarks/README.md @@ -0,0 +1,52 @@ +# API Benchmarks + +This suite benchmarks the current Express API surface with Node's built-in `fetch`. + +## Commands + +```bash +npm run benchmark +npm run benchmark:smoke +``` + +By default the runner starts the local API on an ephemeral loopback port. To target an existing local or staging API, copy `benchmarks/.env.benchmark.example` to `benchmarks/.env.benchmark` and set `BENCHMARK_TARGET_URL`. + +## Coverage + +The route manifest covers `/health` and every mounted `/api/*` route in `apps/api/src/app.js`: + +- `POST /api/auth/register` +- `POST /api/auth/login` +- `GET /api/auth/oauth/:provider/callback` +- `POST /api/auth/refresh` +- `GET /api/users` +- `POST /api/users` +- `GET /api/jobs` +- `POST /api/jobs` +- `GET /api/proposals` +- `POST /api/proposals` +- `POST /api/payments` +- `GET /api/reviews` +- `POST /api/reviews` +- `GET /api/messages` +- `POST /api/messages` +- `GET /api/notifications` +- `POST /api/notifications` +- `POST /api/uploads` +- `GET /api/search` +- `GET /api/admin/metrics` + +The protected admin route uses a benchmark token obtained from the local auth endpoint before the route suite starts. + +## Output + +Each run writes: + +- `benchmarks/results/benchmark-.json` +- `benchmarks/results/benchmark-.md` + +The JSON contains p50, p95, p99, sustained RPS, peak RPS, error rate, and TTFB metrics per endpoint. The Markdown report is intended for PR summaries. + +## Regression Gate + +`npm run benchmark:smoke` reads `benchmarks/thresholds.json` and exits non-zero if any endpoint exceeds its p99 latency threshold or error-rate threshold. diff --git a/benchmarks/results/benchmark-full.json b/benchmarks/results/benchmark-full.json new file mode 100644 index 000000000..c2fc6dbba --- /dev/null +++ b/benchmarks/results/benchmark-full.json @@ -0,0 +1,484 @@ +{ + "generatedAt": "2026-05-25T10:21:20.778Z", + "mode": "full", + "baseUrl": "local ephemeral Express server", + "requestsPerEndpoint": 5, + "concurrency": 2, + "environment": { + "platform": "win32", + "release": "10.0.26200", + "arch": "x64", + "cpuModel": "AMD Ryzen 7 9800X3D 8-Core Processor", + "logicalCores": 16, + "totalMemoryGB": 61.59, + "freeMemoryGB": 32.53, + "node": "v22.18.0" + }, + "routeCount": 21, + "apiRouteCount": 20, + "results": [ + { + "endpoint": "GET /health", + "description": "API health check", + "requests": 5, + "errors": 0, + "errorRatePercent": 0, + "latencyMs": { + "p50": 3.56, + "p95": 5.3, + "p99": 5.3 + }, + "ttfbMs": { + "p50": 3.38, + "p95": 5.12, + "p99": 5.12 + }, + "rps": { + "sustained": 495.31, + "peak": 20 + }, + "sampleError": null + }, + { + "endpoint": "POST /api/auth/register", + "description": "Client registration", + "requests": 5, + "errors": 0, + "errorRatePercent": 0, + "latencyMs": { + "p50": 4.99, + "p95": 5.26, + "p99": 5.26 + }, + "ttfbMs": { + "p50": 4.9, + "p95": 5.02, + "p99": 5.02 + }, + "rps": { + "sustained": 429.55, + "peak": 20 + }, + "sampleError": null + }, + { + "endpoint": "POST /api/auth/login", + "description": "Benchmark login token", + "requests": 5, + "errors": 0, + "errorRatePercent": 0, + "latencyMs": { + "p50": 3.1, + "p95": 3.61, + "p99": 3.61 + }, + "ttfbMs": { + "p50": 3.04, + "p95": 3.53, + "p99": 3.53 + }, + "rps": { + "sustained": 634.68, + "peak": 20 + }, + "sampleError": null + }, + { + "endpoint": "GET /api/auth/oauth/github/callback", + "description": "OAuth callback receipt", + "requests": 5, + "errors": 0, + "errorRatePercent": 0, + "latencyMs": { + "p50": 1.55, + "p95": 2.12, + "p99": 2.12 + }, + "ttfbMs": { + "p50": 1.48, + "p95": 2.05, + "p99": 2.05 + }, + "rps": { + "sustained": 1150.8, + "peak": 20 + }, + "sampleError": null + }, + { + "endpoint": "POST /api/auth/refresh", + "description": "Access token refresh", + "requests": 5, + "errors": 0, + "errorRatePercent": 0, + "latencyMs": { + "p50": 1.84, + "p95": 2.02, + "p99": 2.02 + }, + "ttfbMs": { + "p50": 1.75, + "p95": 1.96, + "p99": 1.96 + }, + "rps": { + "sustained": 1014.45, + "peak": 20 + }, + "sampleError": null + }, + { + "endpoint": "GET /api/users", + "description": "List users", + "requests": 5, + "errors": 0, + "errorRatePercent": 0, + "latencyMs": { + "p50": 1.04, + "p95": 1.31, + "p99": 1.31 + }, + "ttfbMs": { + "p50": 0.99, + "p95": 1.26, + "p99": 1.26 + }, + "rps": { + "sustained": 1690.79, + "peak": 20 + }, + "sampleError": null + }, + { + "endpoint": "POST /api/users", + "description": "Create user profile", + "requests": 5, + "errors": 0, + "errorRatePercent": 0, + "latencyMs": { + "p50": 1.6, + "p95": 1.61, + "p99": 1.61 + }, + "ttfbMs": { + "p50": 1.52, + "p95": 1.56, + "p99": 1.56 + }, + "rps": { + "sustained": 1277.11, + "peak": 20 + }, + "sampleError": null + }, + { + "endpoint": "GET /api/jobs", + "description": "List jobs", + "requests": 5, + "errors": 0, + "errorRatePercent": 0, + "latencyMs": { + "p50": 1.1, + "p95": 1.22, + "p99": 1.22 + }, + "ttfbMs": { + "p50": 1.04, + "p95": 1.16, + "p99": 1.16 + }, + "rps": { + "sustained": 1673.81, + "peak": 20 + }, + "sampleError": null + }, + { + "endpoint": "POST /api/jobs", + "description": "Create job", + "requests": 5, + "errors": 0, + "errorRatePercent": 0, + "latencyMs": { + "p50": 1.58, + "p95": 2.1, + "p99": 2.1 + }, + "ttfbMs": { + "p50": 1.53, + "p95": 2.04, + "p99": 2.04 + }, + "rps": { + "sustained": 1157.73, + "peak": 20 + }, + "sampleError": null + }, + { + "endpoint": "GET /api/proposals", + "description": "List proposals", + "requests": 5, + "errors": 0, + "errorRatePercent": 0, + "latencyMs": { + "p50": 1.21, + "p95": 1.35, + "p99": 1.35 + }, + "ttfbMs": { + "p50": 1.11, + "p95": 1.29, + "p99": 1.29 + }, + "rps": { + "sustained": 1477.93, + "peak": 20 + }, + "sampleError": null + }, + { + "endpoint": "POST /api/proposals", + "description": "Create proposal", + "requests": 5, + "errors": 0, + "errorRatePercent": 0, + "latencyMs": { + "p50": 1.39, + "p95": 1.64, + "p99": 1.64 + }, + "ttfbMs": { + "p50": 1.33, + "p95": 1.59, + "p99": 1.59 + }, + "rps": { + "sustained": 1155.19, + "peak": 20 + }, + "sampleError": null + }, + { + "endpoint": "POST /api/payments", + "description": "Create payment intent", + "requests": 5, + "errors": 0, + "errorRatePercent": 0, + "latencyMs": { + "p50": 1.53, + "p95": 1.63, + "p99": 1.63 + }, + "ttfbMs": { + "p50": 1.48, + "p95": 1.57, + "p99": 1.57 + }, + "rps": { + "sustained": 1260.05, + "peak": 20 + }, + "sampleError": null + }, + { + "endpoint": "GET /api/reviews", + "description": "List reviews", + "requests": 5, + "errors": 0, + "errorRatePercent": 0, + "latencyMs": { + "p50": 1.11, + "p95": 1.22, + "p99": 1.22 + }, + "ttfbMs": { + "p50": 1.06, + "p95": 1.15, + "p99": 1.15 + }, + "rps": { + "sustained": 1714.62, + "peak": 20 + }, + "sampleError": null + }, + { + "endpoint": "POST /api/reviews", + "description": "Create review", + "requests": 5, + "errors": 0, + "errorRatePercent": 0, + "latencyMs": { + "p50": 1.38, + "p95": 1.61, + "p99": 1.61 + }, + "ttfbMs": { + "p50": 1.33, + "p95": 1.55, + "p99": 1.55 + }, + "rps": { + "sustained": 1378.28, + "peak": 20 + }, + "sampleError": null + }, + { + "endpoint": "GET /api/messages", + "description": "List messages", + "requests": 5, + "errors": 0, + "errorRatePercent": 0, + "latencyMs": { + "p50": 1.04, + "p95": 1.15, + "p99": 1.15 + }, + "ttfbMs": { + "p50": 0.99, + "p95": 1.1, + "p99": 1.1 + }, + "rps": { + "sustained": 1821.23, + "peak": 20 + }, + "sampleError": null + }, + { + "endpoint": "POST /api/messages", + "description": "Send message", + "requests": 5, + "errors": 0, + "errorRatePercent": 0, + "latencyMs": { + "p50": 1.36, + "p95": 1.65, + "p99": 1.65 + }, + "ttfbMs": { + "p50": 1.31, + "p95": 1.59, + "p99": 1.59 + }, + "rps": { + "sustained": 1364.96, + "peak": 20 + }, + "sampleError": null + }, + { + "endpoint": "GET /api/notifications", + "description": "List notifications", + "requests": 5, + "errors": 0, + "errorRatePercent": 0, + "latencyMs": { + "p50": 1.03, + "p95": 1.14, + "p99": 1.14 + }, + "ttfbMs": { + "p50": 0.98, + "p95": 1.07, + "p99": 1.07 + }, + "rps": { + "sustained": 1741.61, + "peak": 20 + }, + "sampleError": null + }, + { + "endpoint": "POST /api/notifications", + "description": "Create notification", + "requests": 5, + "errors": 0, + "errorRatePercent": 0, + "latencyMs": { + "p50": 1.36, + "p95": 1.61, + "p99": 1.61 + }, + "ttfbMs": { + "p50": 1.3, + "p95": 1.56, + "p99": 1.56 + }, + "rps": { + "sustained": 1380.8, + "peak": 20 + }, + "sampleError": null + }, + { + "endpoint": "POST /api/uploads", + "description": "Upload portfolio file", + "requests": 5, + "errors": 0, + "errorRatePercent": 0, + "latencyMs": { + "p50": 2.21, + "p95": 6.08, + "p99": 6.08 + }, + "ttfbMs": { + "p50": 2.15, + "p95": 6.03, + "p99": 6.03 + }, + "rps": { + "sustained": 499.95, + "peak": 20 + }, + "sampleError": null + }, + { + "endpoint": "GET /api/search", + "description": "Global search", + "requests": 5, + "errors": 0, + "errorRatePercent": 0, + "latencyMs": { + "p50": 1.2, + "p95": 1.83, + "p99": 1.83 + }, + "ttfbMs": { + "p50": 1.15, + "p95": 1.78, + "p99": 1.78 + }, + "rps": { + "sustained": 1346.98, + "peak": 20 + }, + "sampleError": null + }, + { + "endpoint": "GET /api/admin/metrics", + "description": "Admin metrics", + "requests": 5, + "errors": 0, + "errorRatePercent": 0, + "latencyMs": { + "p50": 1.85, + "p95": 2.58, + "p99": 2.58 + }, + "ttfbMs": { + "p50": 1.79, + "p95": 2.53, + "p99": 2.53 + }, + "rps": { + "sustained": 941.6, + "peak": 20 + }, + "sampleError": null + } + ], + "thresholdFailures": [] +} diff --git a/benchmarks/results/benchmark-full.md b/benchmarks/results/benchmark-full.md new file mode 100644 index 000000000..a6c3a8512 --- /dev/null +++ b/benchmarks/results/benchmark-full.md @@ -0,0 +1,44 @@ +# API Benchmark Report (full) + +Generated: 2026-05-25T10:21:20.778Z +Target: local ephemeral Express server +Routes covered: 20 /api routes plus /health +Requests per endpoint: 5 +Concurrency: 2 + +## Environment + +- OS: win32 10.0.26200 x64 +- CPU: AMD Ryzen 7 9800X3D 8-Core Processor (16 logical cores) +- Memory: 61.59 GB total, 32.53 GB free at start +- Node: v22.18.0 + +## Results + +| Endpoint | Requests | p50 ms | p95 ms | p99 ms | p95 TTFB ms | Sustained RPS | Peak RPS | Error % | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | +| GET /health | 5 | 3.56 | 5.3 | 5.3 | 5.12 | 495.31 | 20 | 0 | +| POST /api/auth/register | 5 | 4.99 | 5.26 | 5.26 | 5.02 | 429.55 | 20 | 0 | +| POST /api/auth/login | 5 | 3.1 | 3.61 | 3.61 | 3.53 | 634.68 | 20 | 0 | +| GET /api/auth/oauth/github/callback | 5 | 1.55 | 2.12 | 2.12 | 2.05 | 1150.8 | 20 | 0 | +| POST /api/auth/refresh | 5 | 1.84 | 2.02 | 2.02 | 1.96 | 1014.45 | 20 | 0 | +| GET /api/users | 5 | 1.04 | 1.31 | 1.31 | 1.26 | 1690.79 | 20 | 0 | +| POST /api/users | 5 | 1.6 | 1.61 | 1.61 | 1.56 | 1277.11 | 20 | 0 | +| GET /api/jobs | 5 | 1.1 | 1.22 | 1.22 | 1.16 | 1673.81 | 20 | 0 | +| POST /api/jobs | 5 | 1.58 | 2.1 | 2.1 | 2.04 | 1157.73 | 20 | 0 | +| GET /api/proposals | 5 | 1.21 | 1.35 | 1.35 | 1.29 | 1477.93 | 20 | 0 | +| POST /api/proposals | 5 | 1.39 | 1.64 | 1.64 | 1.59 | 1155.19 | 20 | 0 | +| POST /api/payments | 5 | 1.53 | 1.63 | 1.63 | 1.57 | 1260.05 | 20 | 0 | +| GET /api/reviews | 5 | 1.11 | 1.22 | 1.22 | 1.15 | 1714.62 | 20 | 0 | +| POST /api/reviews | 5 | 1.38 | 1.61 | 1.61 | 1.55 | 1378.28 | 20 | 0 | +| GET /api/messages | 5 | 1.04 | 1.15 | 1.15 | 1.1 | 1821.23 | 20 | 0 | +| POST /api/messages | 5 | 1.36 | 1.65 | 1.65 | 1.59 | 1364.96 | 20 | 0 | +| GET /api/notifications | 5 | 1.03 | 1.14 | 1.14 | 1.07 | 1741.61 | 20 | 0 | +| POST /api/notifications | 5 | 1.36 | 1.61 | 1.61 | 1.56 | 1380.8 | 20 | 0 | +| POST /api/uploads | 5 | 2.21 | 6.08 | 6.08 | 6.03 | 499.95 | 20 | 0 | +| GET /api/search | 5 | 1.2 | 1.83 | 1.83 | 1.78 | 1346.98 | 20 | 0 | +| GET /api/admin/metrics | 5 | 1.85 | 2.58 | 2.58 | 2.53 | 941.6 | 20 | 0 | + +## Thresholds + +All configured benchmark thresholds passed. diff --git a/benchmarks/results/benchmark-smoke.json b/benchmarks/results/benchmark-smoke.json new file mode 100644 index 000000000..09b1eccd3 --- /dev/null +++ b/benchmarks/results/benchmark-smoke.json @@ -0,0 +1,484 @@ +{ + "generatedAt": "2026-05-25T10:21:16.004Z", + "mode": "smoke", + "baseUrl": "local ephemeral Express server", + "requestsPerEndpoint": 1, + "concurrency": 1, + "environment": { + "platform": "win32", + "release": "10.0.26200", + "arch": "x64", + "cpuModel": "AMD Ryzen 7 9800X3D 8-Core Processor", + "logicalCores": 16, + "totalMemoryGB": 61.59, + "freeMemoryGB": 32.58, + "node": "v22.18.0" + }, + "routeCount": 21, + "apiRouteCount": 20, + "results": [ + { + "endpoint": "GET /health", + "description": "API health check", + "requests": 1, + "errors": 0, + "errorRatePercent": 0, + "latencyMs": { + "p50": 2.98, + "p95": 2.98, + "p99": 2.98 + }, + "ttfbMs": { + "p50": 2.81, + "p95": 2.81, + "p99": 2.81 + }, + "rps": { + "sustained": 326.56, + "peak": 4 + }, + "sampleError": null + }, + { + "endpoint": "POST /api/auth/register", + "description": "Client registration", + "requests": 1, + "errors": 0, + "errorRatePercent": 0, + "latencyMs": { + "p50": 3.51, + "p95": 3.51, + "p99": 3.51 + }, + "ttfbMs": { + "p50": 3.38, + "p95": 3.38, + "p99": 3.38 + }, + "rps": { + "sustained": 282.41, + "peak": 4 + }, + "sampleError": null + }, + { + "endpoint": "POST /api/auth/login", + "description": "Benchmark login token", + "requests": 1, + "errors": 0, + "errorRatePercent": 0, + "latencyMs": { + "p50": 1.97, + "p95": 1.97, + "p99": 1.97 + }, + "ttfbMs": { + "p50": 1.88, + "p95": 1.88, + "p99": 1.88 + }, + "rps": { + "sustained": 504.72, + "peak": 4 + }, + "sampleError": null + }, + { + "endpoint": "GET /api/auth/oauth/github/callback", + "description": "OAuth callback receipt", + "requests": 1, + "errors": 0, + "errorRatePercent": 0, + "latencyMs": { + "p50": 1.41, + "p95": 1.41, + "p99": 1.41 + }, + "ttfbMs": { + "p50": 1.24, + "p95": 1.24, + "p99": 1.24 + }, + "rps": { + "sustained": 705.92, + "peak": 4 + }, + "sampleError": null + }, + { + "endpoint": "POST /api/auth/refresh", + "description": "Access token refresh", + "requests": 1, + "errors": 0, + "errorRatePercent": 0, + "latencyMs": { + "p50": 1.79, + "p95": 1.79, + "p99": 1.79 + }, + "ttfbMs": { + "p50": 1.64, + "p95": 1.64, + "p99": 1.64 + }, + "rps": { + "sustained": 556.14, + "peak": 4 + }, + "sampleError": null + }, + { + "endpoint": "GET /api/users", + "description": "List users", + "requests": 1, + "errors": 0, + "errorRatePercent": 0, + "latencyMs": { + "p50": 1.18, + "p95": 1.18, + "p99": 1.18 + }, + "ttfbMs": { + "p50": 1.02, + "p95": 1.02, + "p99": 1.02 + }, + "rps": { + "sustained": 842.53, + "peak": 4 + }, + "sampleError": null + }, + { + "endpoint": "POST /api/users", + "description": "Create user profile", + "requests": 1, + "errors": 0, + "errorRatePercent": 0, + "latencyMs": { + "p50": 1.14, + "p95": 1.14, + "p99": 1.14 + }, + "ttfbMs": { + "p50": 1.07, + "p95": 1.07, + "p99": 1.07 + }, + "rps": { + "sustained": 835.49, + "peak": 4 + }, + "sampleError": null + }, + { + "endpoint": "GET /api/jobs", + "description": "List jobs", + "requests": 1, + "errors": 0, + "errorRatePercent": 0, + "latencyMs": { + "p50": 2.21, + "p95": 2.21, + "p99": 2.21 + }, + "ttfbMs": { + "p50": 2.02, + "p95": 2.02, + "p99": 2.02 + }, + "rps": { + "sustained": 450.09, + "peak": 4 + }, + "sampleError": null + }, + { + "endpoint": "POST /api/jobs", + "description": "Create job", + "requests": 1, + "errors": 0, + "errorRatePercent": 0, + "latencyMs": { + "p50": 2.25, + "p95": 2.25, + "p99": 2.25 + }, + "ttfbMs": { + "p50": 2.17, + "p95": 2.17, + "p99": 2.17 + }, + "rps": { + "sustained": 434.57, + "peak": 4 + }, + "sampleError": null + }, + { + "endpoint": "GET /api/proposals", + "description": "List proposals", + "requests": 1, + "errors": 0, + "errorRatePercent": 0, + "latencyMs": { + "p50": 1.13, + "p95": 1.13, + "p99": 1.13 + }, + "ttfbMs": { + "p50": 1.01, + "p95": 1.01, + "p99": 1.01 + }, + "rps": { + "sustained": 870.32, + "peak": 4 + }, + "sampleError": null + }, + { + "endpoint": "POST /api/proposals", + "description": "Create proposal", + "requests": 1, + "errors": 0, + "errorRatePercent": 0, + "latencyMs": { + "p50": 1.32, + "p95": 1.32, + "p99": 1.32 + }, + "ttfbMs": { + "p50": 1.25, + "p95": 1.25, + "p99": 1.25 + }, + "rps": { + "sustained": 673.58, + "peak": 4 + }, + "sampleError": null + }, + { + "endpoint": "POST /api/payments", + "description": "Create payment intent", + "requests": 1, + "errors": 0, + "errorRatePercent": 0, + "latencyMs": { + "p50": 1.08, + "p95": 1.08, + "p99": 1.08 + }, + "ttfbMs": { + "p50": 1.02, + "p95": 1.02, + "p99": 1.02 + }, + "rps": { + "sustained": 887.63, + "peak": 4 + }, + "sampleError": null + }, + { + "endpoint": "GET /api/reviews", + "description": "List reviews", + "requests": 1, + "errors": 0, + "errorRatePercent": 0, + "latencyMs": { + "p50": 1.04, + "p95": 1.04, + "p99": 1.04 + }, + "ttfbMs": { + "p50": 0.93, + "p95": 0.93, + "p99": 0.93 + }, + "rps": { + "sustained": 957.49, + "peak": 4 + }, + "sampleError": null + }, + { + "endpoint": "POST /api/reviews", + "description": "Create review", + "requests": 1, + "errors": 0, + "errorRatePercent": 0, + "latencyMs": { + "p50": 1.14, + "p95": 1.14, + "p99": 1.14 + }, + "ttfbMs": { + "p50": 1.07, + "p95": 1.07, + "p99": 1.07 + }, + "rps": { + "sustained": 850.34, + "peak": 4 + }, + "sampleError": null + }, + { + "endpoint": "GET /api/messages", + "description": "List messages", + "requests": 1, + "errors": 0, + "errorRatePercent": 0, + "latencyMs": { + "p50": 0.77, + "p95": 0.77, + "p99": 0.77 + }, + "ttfbMs": { + "p50": 0.69, + "p95": 0.69, + "p99": 0.69 + }, + "rps": { + "sustained": 1296.51, + "peak": 4 + }, + "sampleError": null + }, + { + "endpoint": "POST /api/messages", + "description": "Send message", + "requests": 1, + "errors": 0, + "errorRatePercent": 0, + "latencyMs": { + "p50": 1.57, + "p95": 1.57, + "p99": 1.57 + }, + "ttfbMs": { + "p50": 1.51, + "p95": 1.51, + "p99": 1.51 + }, + "rps": { + "sustained": 622.94, + "peak": 4 + }, + "sampleError": null + }, + { + "endpoint": "GET /api/notifications", + "description": "List notifications", + "requests": 1, + "errors": 0, + "errorRatePercent": 0, + "latencyMs": { + "p50": 1.31, + "p95": 1.31, + "p99": 1.31 + }, + "ttfbMs": { + "p50": 1.25, + "p95": 1.25, + "p99": 1.25 + }, + "rps": { + "sustained": 761.27, + "peak": 4 + }, + "sampleError": null + }, + { + "endpoint": "POST /api/notifications", + "description": "Create notification", + "requests": 1, + "errors": 0, + "errorRatePercent": 0, + "latencyMs": { + "p50": 1.63, + "p95": 1.63, + "p99": 1.63 + }, + "ttfbMs": { + "p50": 1.56, + "p95": 1.56, + "p99": 1.56 + }, + "rps": { + "sustained": 599.02, + "peak": 4 + }, + "sampleError": null + }, + { + "endpoint": "POST /api/uploads", + "description": "Upload portfolio file", + "requests": 1, + "errors": 0, + "errorRatePercent": 0, + "latencyMs": { + "p50": 4.23, + "p95": 4.23, + "p99": 4.23 + }, + "ttfbMs": { + "p50": 4.16, + "p95": 4.16, + "p99": 4.16 + }, + "rps": { + "sustained": 217.91, + "peak": 4 + }, + "sampleError": null + }, + { + "endpoint": "GET /api/search", + "description": "Global search", + "requests": 1, + "errors": 0, + "errorRatePercent": 0, + "latencyMs": { + "p50": 1.4, + "p95": 1.4, + "p99": 1.4 + }, + "ttfbMs": { + "p50": 1.33, + "p95": 1.33, + "p99": 1.33 + }, + "rps": { + "sustained": 709.62, + "peak": 4 + }, + "sampleError": null + }, + { + "endpoint": "GET /api/admin/metrics", + "description": "Admin metrics", + "requests": 1, + "errors": 0, + "errorRatePercent": 0, + "latencyMs": { + "p50": 1.75, + "p95": 1.75, + "p99": 1.75 + }, + "ttfbMs": { + "p50": 1.67, + "p95": 1.67, + "p99": 1.67 + }, + "rps": { + "sustained": 567.99, + "peak": 4 + }, + "sampleError": null + } + ], + "thresholdFailures": [] +} diff --git a/benchmarks/results/benchmark-smoke.md b/benchmarks/results/benchmark-smoke.md new file mode 100644 index 000000000..c02e7b40b --- /dev/null +++ b/benchmarks/results/benchmark-smoke.md @@ -0,0 +1,44 @@ +# API Benchmark Report (smoke) + +Generated: 2026-05-25T10:21:16.004Z +Target: local ephemeral Express server +Routes covered: 20 /api routes plus /health +Requests per endpoint: 1 +Concurrency: 1 + +## Environment + +- OS: win32 10.0.26200 x64 +- CPU: AMD Ryzen 7 9800X3D 8-Core Processor (16 logical cores) +- Memory: 61.59 GB total, 32.58 GB free at start +- Node: v22.18.0 + +## Results + +| Endpoint | Requests | p50 ms | p95 ms | p99 ms | p95 TTFB ms | Sustained RPS | Peak RPS | Error % | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | +| GET /health | 1 | 2.98 | 2.98 | 2.98 | 2.81 | 326.56 | 4 | 0 | +| POST /api/auth/register | 1 | 3.51 | 3.51 | 3.51 | 3.38 | 282.41 | 4 | 0 | +| POST /api/auth/login | 1 | 1.97 | 1.97 | 1.97 | 1.88 | 504.72 | 4 | 0 | +| GET /api/auth/oauth/github/callback | 1 | 1.41 | 1.41 | 1.41 | 1.24 | 705.92 | 4 | 0 | +| POST /api/auth/refresh | 1 | 1.79 | 1.79 | 1.79 | 1.64 | 556.14 | 4 | 0 | +| GET /api/users | 1 | 1.18 | 1.18 | 1.18 | 1.02 | 842.53 | 4 | 0 | +| POST /api/users | 1 | 1.14 | 1.14 | 1.14 | 1.07 | 835.49 | 4 | 0 | +| GET /api/jobs | 1 | 2.21 | 2.21 | 2.21 | 2.02 | 450.09 | 4 | 0 | +| POST /api/jobs | 1 | 2.25 | 2.25 | 2.25 | 2.17 | 434.57 | 4 | 0 | +| GET /api/proposals | 1 | 1.13 | 1.13 | 1.13 | 1.01 | 870.32 | 4 | 0 | +| POST /api/proposals | 1 | 1.32 | 1.32 | 1.32 | 1.25 | 673.58 | 4 | 0 | +| POST /api/payments | 1 | 1.08 | 1.08 | 1.08 | 1.02 | 887.63 | 4 | 0 | +| GET /api/reviews | 1 | 1.04 | 1.04 | 1.04 | 0.93 | 957.49 | 4 | 0 | +| POST /api/reviews | 1 | 1.14 | 1.14 | 1.14 | 1.07 | 850.34 | 4 | 0 | +| GET /api/messages | 1 | 0.77 | 0.77 | 0.77 | 0.69 | 1296.51 | 4 | 0 | +| POST /api/messages | 1 | 1.57 | 1.57 | 1.57 | 1.51 | 622.94 | 4 | 0 | +| GET /api/notifications | 1 | 1.31 | 1.31 | 1.31 | 1.25 | 761.27 | 4 | 0 | +| POST /api/notifications | 1 | 1.63 | 1.63 | 1.63 | 1.56 | 599.02 | 4 | 0 | +| POST /api/uploads | 1 | 4.23 | 4.23 | 4.23 | 4.16 | 217.91 | 4 | 0 | +| GET /api/search | 1 | 1.4 | 1.4 | 1.4 | 1.33 | 709.62 | 4 | 0 | +| GET /api/admin/metrics | 1 | 1.75 | 1.75 | 1.75 | 1.67 | 567.99 | 4 | 0 | + +## Thresholds + +All configured benchmark thresholds passed. diff --git a/benchmarks/run-benchmarks.mjs b/benchmarks/run-benchmarks.mjs new file mode 100644 index 000000000..324c91678 --- /dev/null +++ b/benchmarks/run-benchmarks.mjs @@ -0,0 +1,500 @@ +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { fileURLToPath, pathToFileURL } from "node:url"; +import { performance } from "node:perf_hooks"; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); +const rootDir = path.resolve(__dirname, ".."); +const resultsDir = path.join(__dirname, "results"); +const thresholdsPath = path.join(__dirname, "thresholds.json"); +const envPath = path.join(__dirname, ".env.benchmark"); + +const routeManifest = [ + { method: "GET", path: "/health", description: "API health check" }, + { method: "POST", path: "/api/auth/register", description: "Client registration", json: registerPayload }, + { method: "POST", path: "/api/auth/login", description: "Benchmark login token", json: loginPayload }, + { method: "GET", path: "/api/auth/oauth/github/callback", description: "OAuth callback receipt" }, + { method: "POST", path: "/api/auth/refresh", description: "Access token refresh" }, + { method: "GET", path: "/api/users", description: "List users" }, + { method: "POST", path: "/api/users", description: "Create user profile", json: userPayload }, + { method: "GET", path: "/api/jobs", description: "List jobs" }, + { method: "POST", path: "/api/jobs", description: "Create job", json: jobPayload }, + { method: "GET", path: "/api/proposals", description: "List proposals" }, + { method: "POST", path: "/api/proposals", description: "Create proposal", json: proposalPayload }, + { method: "POST", path: "/api/payments", description: "Create payment intent", json: paymentPayload }, + { method: "GET", path: "/api/reviews", description: "List reviews" }, + { method: "POST", path: "/api/reviews", description: "Create review", json: reviewPayload }, + { method: "GET", path: "/api/messages", description: "List messages" }, + { method: "POST", path: "/api/messages", description: "Send message", json: messagePayload }, + { method: "GET", path: "/api/notifications", description: "List notifications" }, + { method: "POST", path: "/api/notifications", description: "Create notification", json: notificationPayload }, + { method: "POST", path: "/api/uploads", description: "Upload portfolio file", multipart: uploadPayload }, + { method: "GET", path: "/api/search?q=marketplace%20security%20review", description: "Global search" }, + { method: "GET", path: "/api/admin/metrics", description: "Admin metrics", auth: true } +]; + +const defaultModes = { + full: { requestsPerEndpoint: 5, concurrency: 2 }, + smoke: { requestsPerEndpoint: 1, concurrency: 1 } +}; + +main().catch((error) => { + console.error(error); + process.exitCode = 1; +}); + +async function main() { + loadEnvFile(envPath); + process.env.JWT_SECRET ??= "benchmark-secret"; + + const mode = getArgValue("--mode") ?? process.env.BENCHMARK_MODE ?? "full"; + if (!defaultModes[mode]) { + throw new Error(`Unsupported benchmark mode: ${mode}`); + } + + const modeConfig = defaultModes[mode]; + const requestsPerEndpoint = readPositiveInt("BENCHMARK_REQUESTS_PER_ENDPOINT", modeConfig.requestsPerEndpoint); + const concurrency = readPositiveInt("BENCHMARK_CONCURRENCY", modeConfig.concurrency); + const thresholds = JSON.parse(fs.readFileSync(thresholdsPath, "utf8")); + const target = await resolveTarget(); + + try { + const token = await getBenchmarkToken(target.baseUrl); + const startedAt = new Date().toISOString(); + const endpointResults = []; + + for (const endpoint of routeManifest) { + endpointResults.push(await runEndpoint({ + endpoint, + baseUrl: target.baseUrl, + token, + requestsPerEndpoint, + concurrency + })); + } + + const report = { + generatedAt: startedAt, + mode, + baseUrl: target.label, + requestsPerEndpoint, + concurrency, + environment: environmentSnapshot(), + routeCount: routeManifest.length, + apiRouteCount: routeManifest.filter((route) => route.path.startsWith("/api/")).length, + results: endpointResults + }; + + const failures = evaluateThresholds(endpointResults, thresholds); + report.thresholdFailures = failures; + + fs.mkdirSync(resultsDir, { recursive: true }); + fs.writeFileSync(path.join(resultsDir, `benchmark-${mode}.json`), `${JSON.stringify(report, null, 2)}\n`); + fs.writeFileSync(path.join(resultsDir, `benchmark-${mode}.md`), renderMarkdown(report)); + + console.log(renderConsoleSummary(report)); + if (failures.length > 0) { + throw new Error(`Benchmark threshold failures: ${failures.map((failure) => failure.endpoint).join(", ")}`); + } + } finally { + await target.close?.(); + } +} + +function loadEnvFile(filePath) { + if (!fs.existsSync(filePath)) { + return; + } + + const lines = fs.readFileSync(filePath, "utf8").split(/\r?\n/); + for (const line of lines) { + const trimmed = line.trim(); + if (!trimmed || trimmed.startsWith("#") || !trimmed.includes("=")) { + continue; + } + + const index = trimmed.indexOf("="); + const key = trimmed.slice(0, index).trim(); + const value = trimmed.slice(index + 1).trim().replace(/^["']|["']$/g, ""); + process.env[key] ??= value; + } +} + +async function resolveTarget() { + if (process.env.BENCHMARK_TARGET_URL) { + return { + baseUrl: process.env.BENCHMARK_TARGET_URL.replace(/\/$/, ""), + label: process.env.BENCHMARK_TARGET_URL.replace(/\/$/, "") + }; + } + + const { createApp } = await import(pathToFileURL(path.join(rootDir, "apps/api/src/app.js"))); + const app = createApp(); + const server = await new Promise((resolve) => { + const listener = app.listen(0, "127.0.0.1", () => resolve(listener)); + }); + const address = server.address(); + + return { + baseUrl: `http://127.0.0.1:${address.port}`, + label: "local ephemeral Express server", + close: () => new Promise((resolve) => server.close(resolve)) + }; +} + +async function getBenchmarkToken(baseUrl) { + const response = await fetch(`${baseUrl}/api/auth/login`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify(loginPayload(0)) + }); + const body = await response.json(); + if (!response.ok || !body?.data?.token) { + throw new Error("Unable to obtain benchmark auth token"); + } + return body.data.token; +} + +async function runEndpoint({ endpoint, baseUrl, token, requestsPerEndpoint, concurrency }) { + const timings = []; + const ttfbs = []; + const completions = []; + const errors = []; + let nextRequest = 0; + const started = performance.now(); + + async function worker() { + while (nextRequest < requestsPerEndpoint) { + const requestIndex = nextRequest; + nextRequest += 1; + const result = await performRequest({ endpoint, baseUrl, token, requestIndex }); + timings.push(result.latencyMs); + ttfbs.push(result.ttfbMs); + completions.push(result.completedAtMs - started); + if (result.error) { + errors.push(result.error); + } + } + } + + await Promise.all(Array.from({ length: Math.min(concurrency, requestsPerEndpoint) }, worker)); + + const elapsedMs = performance.now() - started; + const totalRequests = timings.length; + const errorCount = errors.length; + const errorRatePercent = totalRequests === 0 ? 100 : (errorCount / totalRequests) * 100; + + return { + endpoint: endpointName(endpoint), + description: endpoint.description, + requests: totalRequests, + errors: errorCount, + errorRatePercent: round(errorRatePercent), + latencyMs: { + p50: percentile(timings, 50), + p95: percentile(timings, 95), + p99: percentile(timings, 99) + }, + ttfbMs: { + p50: percentile(ttfbs, 50), + p95: percentile(ttfbs, 95), + p99: percentile(ttfbs, 99) + }, + rps: { + sustained: round(totalRequests / (elapsedMs / 1000)), + peak: peakRps(completions) + }, + sampleError: errors[0] ?? null + }; +} + +async function performRequest({ endpoint, baseUrl, token, requestIndex }) { + const request = buildRequest(endpoint, token, requestIndex); + const url = `${baseUrl}${endpoint.path}`; + const started = performance.now(); + + try { + const response = await fetch(url, request); + const ttfbMs = performance.now() - started; + await response.arrayBuffer(); + const completedAtMs = performance.now(); + const latencyMs = completedAtMs - started; + return { + latencyMs, + ttfbMs, + completedAtMs, + error: response.ok ? null : `HTTP ${response.status}` + }; + } catch (error) { + const completedAtMs = performance.now(); + return { + latencyMs: completedAtMs - started, + ttfbMs: completedAtMs - started, + completedAtMs, + error: error.message + }; + } +} + +function buildRequest(endpoint, token, requestIndex) { + const headers = {}; + const request = { method: endpoint.method, headers }; + + if (endpoint.auth) { + headers.authorization = `Bearer ${token}`; + } + + if (endpoint.json) { + headers["content-type"] = "application/json"; + request.body = JSON.stringify(endpoint.json(requestIndex)); + } + + if (endpoint.multipart) { + request.body = endpoint.multipart(requestIndex); + } + + return request; +} + +function registerPayload(index) { + return { + email: `benchmark.client.${Date.now()}.${index}@example.com`, + password: "benchmark-password", + role: "client" + }; +} + +function loginPayload() { + return { + email: "benchmark.client@example.com", + password: "benchmark-password" + }; +} + +function userPayload(index) { + return { + email: `benchmark.freelancer.${Date.now()}.${index}@example.com`, + name: "Benchmark Freelancer", + role: "freelancer", + skills: ["node", "api-benchmarking", "marketplace"], + hourlyRate: 85 + }; +} + +function jobPayload(index) { + return { + title: `Benchmark API integration ${index}`, + description: "Build and validate a marketplace integration using realistic benchmark payloads.", + budgetMin: 1200, + budgetMax: 2800, + categoryId: "cat_backend", + skills: ["node", "express", "performance"] + }; +} + +function proposalPayload(index) { + return { + jobId: `job_benchmark_${index}`, + freelancerId: "usr_benchmark_freelancer", + coverLetter: "I can deliver the API integration with measurable latency and reliability targets.", + bidAmount: 1800, + timelineDays: 10 + }; +} + +function paymentPayload(index) { + return { + amount: 25000 + index, + currency: "usd", + metadata: { + jobId: `job_benchmark_${index}`, + milestone: "benchmark-baseline" + } + }; +} + +function reviewPayload(index) { + return { + jobId: `job_benchmark_${index}`, + reviewerId: "usr_benchmark_client", + revieweeId: "usr_benchmark_freelancer", + rating: 5, + comment: "Clear delivery, fast responses, and measurable benchmark evidence." + }; +} + +function messagePayload(index) { + return { + conversationId: `conv_benchmark_${index % 2}`, + senderId: "usr_benchmark_client", + recipientId: "usr_benchmark_freelancer", + body: "Please confirm the milestone scope and attach the latest benchmark summary." + }; +} + +function notificationPayload(index) { + return { + userId: "usr_benchmark_freelancer", + type: "milestone_review", + message: `Benchmark milestone ${index} is ready for review.` + }; +} + +function uploadPayload(index) { + const form = new FormData(); + const content = JSON.stringify({ + filename: `portfolio-sample-${index}.json`, + summary: "Representative portfolio payload for upload benchmark", + skills: ["api", "security", "performance"], + samples: Array.from({ length: 8 }, (_, sample) => ({ sample, score: 90 + sample })) + }); + form.append("file", new Blob([content], { type: "application/json" }), `portfolio-sample-${index}.json`); + return form; +} + +function endpointName(endpoint) { + return `${endpoint.method} ${endpoint.path.split("?")[0]}`; +} + +function percentile(values, percentileValue) { + if (values.length === 0) { + return 0; + } + + const sorted = [...values].sort((a, b) => a - b); + const index = Math.min(sorted.length - 1, Math.ceil((percentileValue / 100) * sorted.length) - 1); + return round(sorted[index]); +} + +function peakRps(completions) { + if (completions.length === 0) { + return 0; + } + + const bucketMs = 250; + const buckets = new Map(); + for (const completedAt of completions) { + const bucket = Math.floor(completedAt / bucketMs); + buckets.set(bucket, (buckets.get(bucket) ?? 0) + 1); + } + return round((Math.max(...buckets.values()) * 1000) / bucketMs); +} + +function evaluateThresholds(results, thresholds) { + return results.flatMap((result) => { + const endpointThresholds = thresholds.endpoints?.[result.endpoint] ?? {}; + const p99Ms = endpointThresholds.p99Ms ?? thresholds.defaults.p99Ms; + const errorRatePercent = endpointThresholds.errorRatePercent ?? thresholds.defaults.errorRatePercent; + const failures = []; + + if (result.latencyMs.p99 > p99Ms) { + failures.push({ + endpoint: result.endpoint, + metric: "latencyMs.p99", + actual: result.latencyMs.p99, + threshold: p99Ms + }); + } + + if (result.errorRatePercent > errorRatePercent) { + failures.push({ + endpoint: result.endpoint, + metric: "errorRatePercent", + actual: result.errorRatePercent, + threshold: errorRatePercent + }); + } + + return failures; + }); +} + +function renderMarkdown(report) { + const rows = report.results.map((result) => [ + result.endpoint, + result.requests, + result.latencyMs.p50, + result.latencyMs.p95, + result.latencyMs.p99, + result.ttfbMs.p95, + result.rps.sustained, + result.rps.peak, + result.errorRatePercent + ]); + + return [ + `# API Benchmark Report (${report.mode})`, + "", + `Generated: ${report.generatedAt}`, + `Target: ${report.baseUrl}`, + `Routes covered: ${report.apiRouteCount} /api routes plus /health`, + `Requests per endpoint: ${report.requestsPerEndpoint}`, + `Concurrency: ${report.concurrency}`, + "", + "## Environment", + "", + `- OS: ${report.environment.platform} ${report.environment.release} ${report.environment.arch}`, + `- CPU: ${report.environment.cpuModel} (${report.environment.logicalCores} logical cores)`, + `- Memory: ${report.environment.totalMemoryGB} GB total, ${report.environment.freeMemoryGB} GB free at start`, + `- Node: ${report.environment.node}`, + "", + "## Results", + "", + "| Endpoint | Requests | p50 ms | p95 ms | p99 ms | p95 TTFB ms | Sustained RPS | Peak RPS | Error % |", + "| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |", + ...rows.map((row) => `| ${row.join(" | ")} |`), + "", + "## Thresholds", + "", + report.thresholdFailures.length === 0 + ? "All configured benchmark thresholds passed." + : report.thresholdFailures.map((failure) => `- ${failure.endpoint}: ${failure.metric} ${failure.actual} > ${failure.threshold}`).join("\n"), + "" + ].join("\n"); +} + +function renderConsoleSummary(report) { + return [ + `Benchmark mode: ${report.mode}`, + `Target: ${report.baseUrl}`, + `Routes covered: ${report.apiRouteCount} /api routes plus /health`, + `Results: benchmarks/results/benchmark-${report.mode}.json and .md`, + `Threshold failures: ${report.thresholdFailures.length}` + ].join("\n"); +} + +function environmentSnapshot() { + const cpu = os.cpus()[0] ?? {}; + return { + platform: os.platform(), + release: os.release(), + arch: os.arch(), + cpuModel: cpu.model?.trim() ?? "unknown", + logicalCores: os.cpus().length, + totalMemoryGB: round(os.totalmem() / 1024 / 1024 / 1024), + freeMemoryGB: round(os.freemem() / 1024 / 1024 / 1024), + node: process.version + }; +} + +function getArgValue(name) { + const prefix = `${name}=`; + const arg = process.argv.find((value) => value === name || value.startsWith(prefix)); + if (!arg) { + return null; + } + return arg === name ? "true" : arg.slice(prefix.length); +} + +function readPositiveInt(envName, fallback) { + const value = Number(process.env[envName] ?? fallback); + if (!Number.isInteger(value) || value <= 0) { + throw new Error(`${envName} must be a positive integer`); + } + return value; +} + +function round(value) { + return Math.round(value * 100) / 100; +} diff --git a/benchmarks/thresholds.json b/benchmarks/thresholds.json new file mode 100644 index 000000000..d21af4e6d --- /dev/null +++ b/benchmarks/thresholds.json @@ -0,0 +1,12 @@ +{ + "defaults": { + "p99Ms": 1000, + "errorRatePercent": 0 + }, + "endpoints": { + "POST /api/uploads": { + "p99Ms": 1500, + "errorRatePercent": 0 + } + } +} diff --git a/package.json b/package.json index 675e6e69d..47566fd3e 100644 --- a/package.json +++ b/package.json @@ -8,6 +8,8 @@ "scripts": { "build": "echo \"Run package-specific builds (e.g. npm run build -w apps/web)\"", "lint": "echo \"No root lint configured\"", + "benchmark": "node benchmarks/run-benchmarks.mjs", + "benchmark:smoke": "node benchmarks/run-benchmarks.mjs --mode=smoke", "test": "npm run test -w apps/api" } }