fix: draft #100
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Performance Benchmark | |
| on: | |
| pull_request: | |
| push: | |
| branches: [main] | |
| workflow_dispatch: | |
| env: | |
| BCR_VERSION: "1.11.7" | |
| ANALYSIS_BCR_VERSION: "2.0.0-alpha.4" | |
| jobs: | |
| startup: | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| steps: | |
| - name: Checkout PR | |
| uses: actions/checkout@v6 | |
| with: | |
| fetch-depth: 0 | |
| path: rules_py_pr | |
| - name: Checkout HEAD main | |
| uses: actions/checkout@v6 | |
| with: | |
| ref: main | |
| path: rules_py_main | |
| - name: Install hyperfine | |
| run: | | |
| set -euo pipefail | |
| HYPERFINE_VERSION="1.18.0" | |
| DEB="hyperfine_${HYPERFINE_VERSION}_amd64.deb" | |
| URL="https://github.com/sharkdp/hyperfine/releases/download/v${HYPERFINE_VERSION}/${DEB}" | |
| wget -qO "$DEB" "$URL" | |
| sudo dpkg -i "$DEB" | |
| rm -f "$DEB" | |
| hyperfine --version | |
| # ── BCR baseline ──────────────────────────────────────────────────────── | |
| - name: Benchmark BCR ${{ env.BCR_VERSION }} | |
| run: | | |
| set -euo pipefail | |
| cd rules_py_pr/benchmark/startup | |
| python3 generate_module.py bcr --version "${BCR_VERSION}" | |
| OUT_BASE="/tmp/bazel-bcr" | |
| rm -rf "$OUT_BASE" | |
| START=$(date +%s%N) | |
| bazel --output_base="$OUT_BASE" --bazelrc=../../.github/workflows/ci.bazelrc build --disk_cache= //:bench //:bench_syspath | |
| END=$(date +%s%N) | |
| BUILD_MS=$(( (END - START) / 1000000 )) | |
| echo "{\"build_ms\": $BUILD_MS}" > "${GITHUB_WORKSPACE}/bcr-build.json" | |
| BIN=$(bazel --output_base="$OUT_BASE" cquery //:bench --disk_cache= --output=starlark --starlark:expr='target.files_to_run.executable.path' | tail -n1) | |
| test -x "$BIN" || { echo "ERROR: benchmark binary not executable: $BIN"; exit 1; } | |
| RUNFILES_DIR="$PWD/$BIN.runfiles" hyperfine --warmup 5 --runs 50 --export-json "${GITHUB_WORKSPACE}/bcr.json" "$BIN" | |
| BIN_SP=$(bazel --output_base="$OUT_BASE" cquery //:bench_syspath --disk_cache= --output=starlark --starlark:expr='target.files_to_run.executable.path' | tail -n1) | |
| test -x "$BIN_SP" || { echo "ERROR: bench_syspath binary not executable: $BIN_SP"; exit 1; } | |
| RUNFILES_DIR="$PWD/$BIN_SP.runfiles" "$BIN_SP" "$GITHUB_WORKSPACE/bcr-syspath.json" | |
| # ── HEAD main ─────────────────────────────────────────────────────────── | |
| - name: Benchmark HEAD main | |
| run: | | |
| set -euo pipefail | |
| cd rules_py_pr/benchmark/startup | |
| python3 generate_module.py local --path "$GITHUB_WORKSPACE/rules_py_main" | |
| OUT_BASE="/tmp/bazel-main" | |
| rm -rf "$OUT_BASE" | |
| START=$(date +%s%N) | |
| bazel --output_base="$OUT_BASE" --bazelrc=../../.github/workflows/ci.bazelrc build --disk_cache= //:bench //:bench_syspath | |
| END=$(date +%s%N) | |
| BUILD_MS=$(( (END - START) / 1000000 )) | |
| echo "{\"build_ms\": $BUILD_MS}" > "${GITHUB_WORKSPACE}/main-build.json" | |
| BIN=$(bazel --output_base="$OUT_BASE" cquery //:bench --disk_cache= --output=starlark --starlark:expr='target.files_to_run.executable.path' | tail -n1) | |
| test -x "$BIN" || { echo "ERROR: benchmark binary not executable: $BIN"; exit 1; } | |
| RUNFILES_DIR="$PWD/$BIN.runfiles" hyperfine --warmup 5 --runs 50 --export-json "${GITHUB_WORKSPACE}/main.json" "$BIN" | |
| BIN_SP=$(bazel --output_base="$OUT_BASE" cquery //:bench_syspath --disk_cache= --output=starlark --starlark:expr='target.files_to_run.executable.path' | tail -n1) | |
| test -x "$BIN_SP" || { echo "ERROR: bench_syspath binary not executable: $BIN_SP"; exit 1; } | |
| RUNFILES_DIR="$PWD/$BIN_SP.runfiles" "$BIN_SP" "$GITHUB_WORKSPACE/main-syspath.json" | |
| # ── Current commit (PR) ───────────────────────────────────────────────── | |
| - name: Benchmark current PR | |
| run: | | |
| set -euo pipefail | |
| cd rules_py_pr/benchmark/startup | |
| python3 generate_module.py local --path "$GITHUB_WORKSPACE/rules_py_pr" | |
| OUT_BASE="/tmp/bazel-pr" | |
| rm -rf "$OUT_BASE" | |
| START=$(date +%s%N) | |
| bazel --output_base="$OUT_BASE" --bazelrc=../../.github/workflows/ci.bazelrc build --disk_cache= //:bench //:bench_syspath | |
| END=$(date +%s%N) | |
| BUILD_MS=$(( (END - START) / 1000000 )) | |
| echo "{\"build_ms\": $BUILD_MS}" > "${GITHUB_WORKSPACE}/pr-build.json" | |
| BIN=$(bazel --output_base="$OUT_BASE" cquery //:bench --disk_cache= --output=starlark --starlark:expr='target.files_to_run.executable.path' | tail -n1) | |
| test -x "$BIN" || { echo "ERROR: benchmark binary not executable: $BIN"; exit 1; } | |
| RUNFILES_DIR="$PWD/$BIN.runfiles" hyperfine --warmup 5 --runs 50 --export-json "${GITHUB_WORKSPACE}/pr.json" "$BIN" | |
| BIN_SP=$(bazel --output_base="$OUT_BASE" cquery //:bench_syspath --disk_cache= --output=starlark --starlark:expr='target.files_to_run.executable.path' | tail -n1) | |
| test -x "$BIN_SP" || { echo "ERROR: bench_syspath binary not executable: $BIN_SP"; exit 1; } | |
| RUNFILES_DIR="$PWD/$BIN_SP.runfiles" "$BIN_SP" "$GITHUB_WORKSPACE/pr-syspath.json" | |
| # ── Compare ───────────────────────────────────────────────────────────── | |
| - name: Compare startup results | |
| id: compare | |
| continue-on-error: true | |
| run: | | |
| set -euo pipefail | |
| python3 rules_py_pr/benchmark/startup/compare.py \ | |
| --output-table startup-table.txt \ | |
| bcr.json main.json pr.json | |
| - name: Save PR number | |
| if: github.event_name == 'pull_request' | |
| run: echo "${{ github.event.pull_request.number }}" > pr-number.txt | |
| - name: Upload startup artifacts | |
| if: github.event_name == 'pull_request' | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: startup-benchmark-results | |
| path: | | |
| startup-table.txt | |
| pr-number.txt | |
| if-no-files-found: warn | |
| - name: Fail on regression | |
| if: steps.compare.outcome == 'failure' | |
| run: exit 1 | |
| analysis: | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| steps: | |
| - name: Checkout PR | |
| uses: actions/checkout@v6 | |
| with: | |
| fetch-depth: 0 | |
| path: rules_py_pr | |
| - name: Checkout HEAD main | |
| uses: actions/checkout@v6 | |
| with: | |
| ref: main | |
| path: rules_py_main | |
| - name: Install hyperfine | |
| run: | | |
| set -euo pipefail | |
| HYPERFINE_VERSION="1.18.0" | |
| DEB="hyperfine_${HYPERFINE_VERSION}_amd64.deb" | |
| URL="https://github.com/sharkdp/hyperfine/releases/download/v${HYPERFINE_VERSION}/${DEB}" | |
| wget -qO "$DEB" "$URL" | |
| sudo dpkg -i "$DEB" | |
| rm -f "$DEB" | |
| hyperfine --version | |
| - name: Generate workspace | |
| run: | | |
| set -euo pipefail | |
| cd rules_py_pr/benchmark/analysis | |
| python3 workspace/generate_workspace.py --root workspace --packages 50 | |
| # ── BCR baseline ──────────────────────────────────────────────────────── | |
| - name: Benchmark BCR ${{ env.ANALYSIS_BCR_VERSION }} | |
| run: | | |
| set -euo pipefail | |
| cd rules_py_pr/benchmark/analysis | |
| python3 generate_module.py bcr --version "${ANALYSIS_BCR_VERSION}" | |
| OUT_BASE="/tmp/bazel-bcr" | |
| rm -rf "$OUT_BASE" | |
| bazel --output_base="$OUT_BASE" --bazelrc=../../.github/workflows/ci.bazelrc fetch //workspace/... | |
| hyperfine --warmup 1 --runs 10 \ | |
| --prepare 'rm -rf /tmp/bazel-bcr-analysis' \ | |
| --export-json "${GITHUB_WORKSPACE}/bcr.json" \ | |
| 'bazel --output_base=/tmp/bazel-bcr-analysis --bazelrc=../../.github/workflows/ci.bazelrc build --disk_cache= --nobuild //workspace/...' | |
| bazel --output_base="$OUT_BASE" --bazelrc=../../.github/workflows/ci.bazelrc query //workspace/... --output=package > /tmp/bcr-packages.txt | |
| bazel --output_base="$OUT_BASE" --bazelrc=../../.github/workflows/ci.bazelrc query //workspace/... > /tmp/bcr-targets.txt | |
| echo "{\"packages\": $(wc -l < /tmp/bcr-packages.txt | tr -d ' '), \"targets\": $(wc -l < /tmp/bcr-targets.txt | tr -d ' ')}" > "${GITHUB_WORKSPACE}/bcr-aux.json" | |
| # ── HEAD main ─────────────────────────────────────────────────────────── | |
| - name: Benchmark HEAD main | |
| run: | | |
| set -euo pipefail | |
| cd rules_py_pr/benchmark/analysis | |
| python3 generate_module.py local --path "$GITHUB_WORKSPACE/rules_py_main" | |
| OUT_BASE="/tmp/bazel-main" | |
| rm -rf "$OUT_BASE" | |
| bazel --output_base="$OUT_BASE" --bazelrc=../../.github/workflows/ci.bazelrc fetch //workspace/... | |
| hyperfine --warmup 1 --runs 10 \ | |
| --prepare 'rm -rf /tmp/bazel-main-analysis' \ | |
| --export-json "${GITHUB_WORKSPACE}/main.json" \ | |
| 'bazel --output_base=/tmp/bazel-main-analysis --bazelrc=../../.github/workflows/ci.bazelrc build --disk_cache= --nobuild //workspace/...' | |
| bazel --output_base="$OUT_BASE" --bazelrc=../../.github/workflows/ci.bazelrc query //workspace/... --output=package > /tmp/main-packages.txt | |
| bazel --output_base="$OUT_BASE" --bazelrc=../../.github/workflows/ci.bazelrc query //workspace/... > /tmp/main-targets.txt | |
| echo "{\"packages\": $(wc -l < /tmp/main-packages.txt | tr -d ' '), \"targets\": $(wc -l < /tmp/main-targets.txt | tr -d ' ')}" > "${GITHUB_WORKSPACE}/main-aux.json" | |
| # ── Current commit (PR) ───────────────────────────────────────────────── | |
| - name: Benchmark current PR | |
| run: | | |
| set -euo pipefail | |
| cd rules_py_pr/benchmark/analysis | |
| python3 generate_module.py local --path "$GITHUB_WORKSPACE/rules_py_pr" | |
| OUT_BASE="/tmp/bazel-pr" | |
| rm -rf "$OUT_BASE" | |
| bazel --output_base="$OUT_BASE" --bazelrc=../../.github/workflows/ci.bazelrc fetch //workspace/... | |
| hyperfine --warmup 1 --runs 10 \ | |
| --prepare 'rm -rf /tmp/bazel-pr-analysis' \ | |
| --export-json "${GITHUB_WORKSPACE}/pr.json" \ | |
| 'bazel --output_base=/tmp/bazel-pr-analysis --bazelrc=../../.github/workflows/ci.bazelrc build --disk_cache= --nobuild //workspace/...' | |
| bazel --output_base="$OUT_BASE" --bazelrc=../../.github/workflows/ci.bazelrc query //workspace/... --output=package > /tmp/pr-packages.txt | |
| bazel --output_base="$OUT_BASE" --bazelrc=../../.github/workflows/ci.bazelrc query //workspace/... > /tmp/pr-targets.txt | |
| echo "{\"packages\": $(wc -l < /tmp/pr-packages.txt | tr -d ' '), \"targets\": $(wc -l < /tmp/pr-targets.txt | tr -d ' ')}" > "${GITHUB_WORKSPACE}/pr-aux.json" | |
| # ── Compare ───────────────────────────────────────────────────────────── | |
| - name: Compare analysis results | |
| id: compare | |
| continue-on-error: true | |
| run: | | |
| set -euo pipefail | |
| python3 rules_py_pr/benchmark/analysis/compare.py \ | |
| --output-table analysis-table.txt \ | |
| bcr.json main.json pr.json | |
| - name: Upload analysis artifacts | |
| if: github.event_name == 'pull_request' | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: analysis-benchmark-results | |
| path: | | |
| analysis-table.txt | |
| if-no-files-found: warn | |
| - name: Fail on regression | |
| if: steps.compare.outcome == 'failure' | |
| run: exit 1 | |
| comment: | |
| runs-on: ubuntu-latest | |
| needs: [startup, analysis] | |
| if: always() && github.event_name == 'pull_request' && (needs.startup.result == 'success' || needs.startup.result == 'failure') && (needs.analysis.result == 'success' || needs.analysis.result == 'failure') | |
| permissions: | |
| pull-requests: write | |
| steps: | |
| - name: Download startup artifacts | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: startup-benchmark-results | |
| - name: Download analysis artifacts | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: analysis-benchmark-results | |
| - name: Post combined benchmark comment | |
| uses: actions/github-script@v7 | |
| with: | |
| script: | | |
| const fs = require('fs'); | |
| const startup = fs.readFileSync('startup-table.txt', 'utf8').trim(); | |
| const analysis = fs.readFileSync('analysis-table.txt', 'utf8').trim(); | |
| const prNumber = parseInt(fs.readFileSync('pr-number.txt', 'utf8').trim()); | |
| const header = '<!-- performance-benchmark -->'; | |
| const body = `${header}\n\n${startup}\n\n${analysis}`; | |
| const { data: comments } = await github.rest.issues.listComments({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| issue_number: prNumber, | |
| }); | |
| const existing = comments.find(c => c.body && c.body.includes(header)); | |
| if (existing) { | |
| await github.rest.issues.updateComment({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| comment_id: existing.id, | |
| body: body, | |
| }); | |
| console.log(`Updated comment ${existing.id}`); | |
| } else { | |
| const { data: created } = await github.rest.issues.createComment({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| issue_number: prNumber, | |
| body: body, | |
| }); | |
| console.log(`Created comment ${created.id}`); | |
| } | |
| - name: Fail if any benchmark regressed | |
| if: needs.startup.result == 'failure' || needs.analysis.result == 'failure' | |
| run: exit 1 |