diff --git a/.github/workflows/k6-load-tests.yml b/.github/workflows/k6-load-tests.yml new file mode 100644 index 0000000..5e248a2 --- /dev/null +++ b/.github/workflows/k6-load-tests.yml @@ -0,0 +1,170 @@ +name: Load Tests + +on: [pull_request] + +jobs: + test-base: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + with: + ref: ${{ github.event.pull_request.base.sha }} + + - name: Set up Node.js + uses: actions/setup-node@v2 + with: + node-version: '20' + + - name: Install dependencies + run: npm ci + + - name: Build + run: npm run build + + - name: Start containers + run: docker-compose -f "docker-compose.yml" up -d --build + + - name: Start Node.js API + run: npm run start:devnet & + # TODO: export environment config variables when base config is implemented into the template + # env: + # SELF_URL: 'http://localhost:3000/assets-cdn' + # PUBLIC_API_PORT: 3000 + # PUBLIC_API_PREFIX: 'assets-cdn' + # PRIVATE_API_PORT: 4000 + + - name: Install k6 + run: | + sudo gpg -k + sudo gpg --no-default-keyring --keyring /usr/share/keyrings/k6-archive-keyring.gpg --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys C5AD17C747E3415A3642D57D77C6C491D6AC1D69 + echo "deb [signed-by=/usr/share/keyrings/k6-archive-keyring.gpg] https://dl.k6.io/deb stable main" | sudo tee /etc/apt/sources.list.d/k6.list + sudo apt-get update + sudo apt-get install k6 + + - name: Wait for API to be ready + run: | + until curl --output /dev/null --silent --fail http://localhost:4000/hello; do + echo 'Waiting for API...' + sleep 1 + done + + - name: Run k6 Load Test + run: k6 run ./k6/script.js + + - name: Upload result file for base branch + uses: actions/upload-artifact@v2 + with: + name: base-results + path: k6/output/summary.json + + test-head: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + with: + ref: ${{ github.event.pull_request.head.sha }} + + - name: Set up Node.js + uses: actions/setup-node@v2 + with: + node-version: '20' + + - name: Install dependencies + run: npm ci + + - name: Start containers + run: docker-compose -f "docker-compose.yml" up -d --build + + - name: Build + run: npm run build + + - name: Start Node.js API + run: npm run start:devnet & + # TODO: export environment config variables when base config is implemented into the template + # env: + # SELF_URL: 'http://localhost:3000/assets-cdn' + # PUBLIC_API_PORT: 3000 + # PUBLIC_API_PREFIX: 'assets-cdn' + # PRIVATE_API_PORT: 4000 + + - name: Install k6 + run: | + sudo gpg -k + sudo gpg --no-default-keyring --keyring /usr/share/keyrings/k6-archive-keyring.gpg --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys C5AD17C747E3415A3642D57D77C6C491D6AC1D69 + echo "deb [signed-by=/usr/share/keyrings/k6-archive-keyring.gpg] https://dl.k6.io/deb stable main" | sudo tee /etc/apt/sources.list.d/k6.list + sudo apt-get update + sudo apt-get install k6 + + - name: Wait for API to be ready + run: | + until curl --output /dev/null --silent --fail http://localhost:4000/hello; do + echo 'Waiting for API...' + sleep 1 + done + + - name: Run k6 Load Test + run: k6 run ./k6/script.js + + - name: Upload result file for head branch + uses: actions/upload-artifact@v2 + with: + name: head-results + path: k6/output/summary.json + + + compare-results: + runs-on: ubuntu-latest + needs: [test-base, test-head] + steps: + - uses: actions/checkout@v2 + + - name: Download all artifacts + uses: actions/download-artifact@v2 + with: + path: artifacts + + - name: Set up Node.js + uses: actions/setup-node@v2 + with: + node-version: '20' + + - name: Compare test results + run: | + node ./k6/compare-results.js ${{ github.event.pull_request.base.sha }} artifacts/base-results/summary.json ${{ github.event.pull_request.head.sha }} artifacts/head-results/summary.json report.md + + - name: Render the report from the template + id: template + uses: chuhlomin/render-template@v1 + if: github.event_name == 'pull_request' + with: + template: report.md + vars: | + base: ${{ github.event.pull_request.base.sha }} + head: ${{ github.event.pull_request.head.sha }} + + - name: Upload the report markdown + uses: actions/upload-artifact@v3 + if: github.event_name == 'pull_request' + with: + name: report-markdown + path: report.md + + - name: Find the comment containing the report + id: fc + uses: peter-evans/find-comment@v2 + if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository + with: + issue-number: ${{ github.event.pull_request.number }} + comment-author: 'github-actions[bot]' + body-includes: 'k6 load testing comparison' + + - name: Create or update the report comment + uses: peter-evans/create-or-update-comment@v2 + if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository + with: + comment-id: ${{ steps.fc.outputs.comment-id }} + issue-number: ${{ github.event.pull_request.number }} + body: ${{ steps.template.outputs.result }} + edit-mode: replace diff --git a/k6/.gitignore b/k6/.gitignore new file mode 100644 index 0000000..77d1aa0 --- /dev/null +++ b/k6/.gitignore @@ -0,0 +1,2 @@ +output/* +!output/.gitkeep \ No newline at end of file diff --git a/k6/compare-results.js b/k6/compare-results.js new file mode 100644 index 0000000..5e435c6 --- /dev/null +++ b/k6/compare-results.js @@ -0,0 +1,99 @@ +const fs = require('fs'); + +function generateComparisonTable(baseCommitHash, baseMetricsPath, targetCommitHash, targetMetricsPath, outputPath) { + // Load JSON outputs from k6 + const baseMetrics = JSON.parse(fs.readFileSync(baseMetricsPath, 'utf8')); + const targetMetrics = JSON.parse(fs.readFileSync(targetMetricsPath, 'utf8')); + + const baseData = extractMetrics(baseMetrics); + const targetData = extractMetrics(targetMetrics); + + const table = generateTable(baseCommitHash, baseData, targetCommitHash, targetData); + + fs.writeFileSync(outputPath, table); +} + +function extractMetrics(metrics) { + const extractedMetrics = {}; + const metricKeys = Object.keys(metrics.metrics); + + for (const key of metricKeys) { + if (key.endsWith('_http_req_duration')) { + const values = metrics.metrics[key].values; + const avgResponseTime = values.avg; + const maxResponseTime = values.max; + const p90 = values['p(90)']; + const p95 = values['p(95)']; + + const name = key.split('_')[0].charAt(0).toUpperCase() + key.split('_')[0].slice(1); + + if (!extractedMetrics[name]) { + extractedMetrics[name] = { avgResponseTime, maxResponseTime, p90, p95 }; + } else { + extractedMetrics[name].avgResponseTime = avgResponseTime; + extractedMetrics[name].maxResponseTime = maxResponseTime; + extractedMetrics[name].p90 = p90; + extractedMetrics[name].p95 = p95; + } + } + } + + extractedMetrics['Test Run Duration'] = metrics.state.testRunDurationMs; + + return extractedMetrics; +} + +function generateTable(baseCommitHash, baseData, targetCommitHash, targetData) { + let table = `k6 load testing comparison.\nBase Commit Hash: ${baseCommitHash}\nTarget Commit Hash: ${targetCommitHash}\n`; + table += `Test duration: ${baseData['Test Run Duration'].toFixed(2)} ms (base) | ${targetData['Test Run Duration'].toFixed(2)} ms (target) \n\n`; + table += '| Endpoint \ Metric | Average | Max | p(90) | p(95) |\n'; + table += '| ----------------- | ---- | ------ | ----- | ----- |\n'; + + for (const key of Object.keys(baseData)) { + if (key === 'Test Run Duration') { + continue; + } + const baseAvg = baseData[key].avgResponseTime; + const targetAvg = targetData[key].avgResponseTime; + const baseMax = baseData[key].maxResponseTime; + const targetMax = targetData[key].maxResponseTime; + const baseP90 = baseData[key].p90; + const targetP90 = targetData[key].p90; + const baseP95 = baseData[key].p95; + const targetP95 = targetData[key].p95; + + table += `| **${key}** | ${computeCell(baseAvg, targetAvg)} | ${computeCell(baseMax, targetMax)} | ${computeCell(baseP90, targetP90)} | ${computeCell(baseP95, targetP95)} | \n`; + } + + return table; +} + +function computeCell(baseDuration, targetDuration) { + return `${baseDuration.toFixed(2)} ms -> ${targetDuration.toFixed(2)} ms (${getDifferencePercentage(baseDuration, targetDuration)})`; +} + +function getDifferencePercentage(baseValue, targetValue) { + const difference = ((targetValue - baseValue) / baseValue) * 100; + const sign = difference >= 0 ? '+' : ''; + let output = `${sign}${difference.toFixed(2)}%`; + if (difference > 20.0) { + output += ' ⚠️'; + } else if (difference < -20) { + output += ' 🟢'; + } + + return `${output}`; +} + +if (process.argv.length !== 7) { + console.error('Usage: node compare-results.js baseCommitHash baseMetricsPath targetCommitHash targetMetricsPath outputFile'); + process.exit(1); +} + +const baseCommitHash = process.argv[2]; +const baseMetricsPath = process.argv[3]; +const targetCommitHash = process.argv[4]; +const targetMetricsPath = process.argv[5]; +const outputPath = process.argv[6]; + +generateComparisonTable(baseCommitHash, baseMetricsPath, targetCommitHash, targetMetricsPath, outputPath); diff --git a/k6/output/.gitkeep b/k6/output/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/k6/script.js b/k6/script.js new file mode 100644 index 0000000..d903d5d --- /dev/null +++ b/k6/script.js @@ -0,0 +1,56 @@ +import http from 'k6/http'; +import { Trend } from 'k6/metrics'; + +const BASE_URL = 'http://localhost:3000'; + +const usersApiCallTrend = new Trend('users_http_req_duration', true); +const examplesApiCallTrend = new Trend('examples_http_req_duration', true); +const tokensApiCallTrend = new Trend('tokens_http_req_duration', true); + +export const options = { + scenarios: { + users: { + executor: 'constant-vus', + vus: 10, + duration: '1m', + gracefulStop: '0s', + exec: 'users', + }, + tokens: { + executor: 'constant-vus', + vus: 10, + duration: '1m', + gracefulStop: '0s', + exec: 'tokens', + }, + examples: { + executor: 'constant-vus', + vus: 10, + duration: '1m', + gracefulStop: '0s', + exec: 'examples', + }, + }, + discardResponseBodies: true, +}; + +export function users() { + const response = http.get(`${BASE_URL}/users`); + usersApiCallTrend.add(response.timings.duration); +} + +export function examples() { + const response = http.get(`${BASE_URL}/examples`); + examplesApiCallTrend.add(response.timings.duration); +} + +export function tokens() { + const response = http.get(`${BASE_URL}/tokens`); + tokensApiCallTrend.add(response.timings.duration); +} + +export function handleSummary(data) { + return { + 'k6/output/summary.json': JSON.stringify(data), + }; +} diff --git a/package.json b/package.json index fe35009..62bcde4 100644 --- a/package.json +++ b/package.json @@ -72,7 +72,8 @@ "copy-devnet-config-cache-warmer": "cp ./apps/cache-warmer/config/config.devnet.yaml ./apps/cache-warmer/config/config.yaml", "copy-testnet-config-cache-warmer": "cp ./apps/cache-warmer/config/config.testnet.yaml ./apps/cache-warmer/config/config.yaml", "copy-mainnet-config-cache-warmer": "cp ./apps/cache-warmer/config/config.mainnet.yaml ./apps/cache-warmer/config/config.yaml", - "copy-custom-config-cache-warmer": "cp ./apps/cache-warmer/config/config.custom.yaml ./apps/cache-warmer/config/config.yaml" + "copy-custom-config-cache-warmer": "cp ./apps/cache-warmer/config/config.custom.yaml ./apps/cache-warmer/config/config.yaml", + "load-test": "k6 run ./k6/script.js" }, "dependencies": { "@multiversx/sdk-core": "^12.15.0",