performance-benchmark.yml

name: Performance Benchmark

on:
  pull_request:
  push:
    branches: [main]
  workflow_dispatch:

env:
  BCR_VERSION: "1.11.7"
  ANALYSIS_BCR_VERSION: "2.0.0-alpha.4"

jobs:
  startup:
    runs-on: ubuntu-latest
    permissions:
      contents: read
    steps:
      - name: Checkout PR
        uses: actions/checkout@v6
        with:
          fetch-depth: 0
          path: rules_py_pr

      - name: Checkout HEAD main
        uses: actions/checkout@v6
        with:
          ref: main
          path: rules_py_main

      - name: Install hyperfine
        run: |
          set -euo pipefail
          HYPERFINE_VERSION="1.18.0"
          DEB="hyperfine_${HYPERFINE_VERSION}_amd64.deb"
          URL="https://github.com/sharkdp/hyperfine/releases/download/v${HYPERFINE_VERSION}/${DEB}"
          wget -qO "$DEB" "$URL"
          sudo dpkg -i "$DEB"
          rm -f "$DEB"
          hyperfine --version

      # ── BCR baseline ────────────────────────────────────────────────────────
      - name: Benchmark BCR ${{ env.BCR_VERSION }}
        run: |
          set -euo pipefail
          cd rules_py_pr/benchmark/startup
          python3 generate_module.py bcr --version "${BCR_VERSION}"

          OUT_BASE="/tmp/bazel-bcr"
          rm -rf "$OUT_BASE"

          START=$(date +%s%N)
          bazel --output_base="$OUT_BASE" --bazelrc=../../.github/workflows/ci.bazelrc build --disk_cache= //:bench //:bench_syspath
          END=$(date +%s%N)
          BUILD_MS=$(( (END - START) / 1000000 ))
          echo "{\"build_ms\": $BUILD_MS}" > "${GITHUB_WORKSPACE}/bcr-build.json"

          BIN=$(bazel --output_base="$OUT_BASE" cquery //:bench --disk_cache= --output=starlark --starlark:expr='target.files_to_run.executable.path' | tail -n1)
          test -x "$BIN" || { echo "ERROR: benchmark binary not executable: $BIN"; exit 1; }
          RUNFILES_DIR="$PWD/$BIN.runfiles" hyperfine --warmup 5 --runs 50 --export-json "${GITHUB_WORKSPACE}/bcr.json" "$BIN"

          BIN_SP=$(bazel --output_base="$OUT_BASE" cquery //:bench_syspath --disk_cache= --output=starlark --starlark:expr='target.files_to_run.executable.path' | tail -n1)
          test -x "$BIN_SP" || { echo "ERROR: bench_syspath binary not executable: $BIN_SP"; exit 1; }
          RUNFILES_DIR="$PWD/$BIN_SP.runfiles" "$BIN_SP" "$GITHUB_WORKSPACE/bcr-syspath.json"

      # ── HEAD main ───────────────────────────────────────────────────────────
      - name: Benchmark HEAD main
        run: |
          set -euo pipefail
          cd rules_py_pr/benchmark/startup
          python3 generate_module.py local --path "$GITHUB_WORKSPACE/rules_py_main"

          OUT_BASE="/tmp/bazel-main"
          rm -rf "$OUT_BASE"

          START=$(date +%s%N)
          bazel --output_base="$OUT_BASE" --bazelrc=../../.github/workflows/ci.bazelrc build --disk_cache= //:bench //:bench_syspath
          END=$(date +%s%N)
          BUILD_MS=$(( (END - START) / 1000000 ))
          echo "{\"build_ms\": $BUILD_MS}" > "${GITHUB_WORKSPACE}/main-build.json"

          BIN=$(bazel --output_base="$OUT_BASE" cquery //:bench --disk_cache= --output=starlark --starlark:expr='target.files_to_run.executable.path' | tail -n1)
          test -x "$BIN" || { echo "ERROR: benchmark binary not executable: $BIN"; exit 1; }
          RUNFILES_DIR="$PWD/$BIN.runfiles" hyperfine --warmup 5 --runs 50 --export-json "${GITHUB_WORKSPACE}/main.json" "$BIN"

          BIN_SP=$(bazel --output_base="$OUT_BASE" cquery //:bench_syspath --disk_cache= --output=starlark --starlark:expr='target.files_to_run.executable.path' | tail -n1)
          test -x "$BIN_SP" || { echo "ERROR: bench_syspath binary not executable: $BIN_SP"; exit 1; }
          RUNFILES_DIR="$PWD/$BIN_SP.runfiles" "$BIN_SP" "$GITHUB_WORKSPACE/main-syspath.json"

      # ── Current commit (PR) ─────────────────────────────────────────────────
      - name: Benchmark current PR
        run: |
          set -euo pipefail
          cd rules_py_pr/benchmark/startup
          python3 generate_module.py local --path "$GITHUB_WORKSPACE/rules_py_pr"

          OUT_BASE="/tmp/bazel-pr"
          rm -rf "$OUT_BASE"

          START=$(date +%s%N)
          bazel --output_base="$OUT_BASE" --bazelrc=../../.github/workflows/ci.bazelrc build --disk_cache= //:bench //:bench_syspath
          END=$(date +%s%N)
          BUILD_MS=$(( (END - START) / 1000000 ))
          echo "{\"build_ms\": $BUILD_MS}" > "${GITHUB_WORKSPACE}/pr-build.json"

          BIN=$(bazel --output_base="$OUT_BASE" cquery //:bench --disk_cache= --output=starlark --starlark:expr='target.files_to_run.executable.path' | tail -n1)
          test -x "$BIN" || { echo "ERROR: benchmark binary not executable: $BIN"; exit 1; }
          RUNFILES_DIR="$PWD/$BIN.runfiles" hyperfine --warmup 5 --runs 50 --export-json "${GITHUB_WORKSPACE}/pr.json" "$BIN"

          BIN_SP=$(bazel --output_base="$OUT_BASE" cquery //:bench_syspath --disk_cache= --output=starlark --starlark:expr='target.files_to_run.executable.path' | tail -n1)
          test -x "$BIN_SP" || { echo "ERROR: bench_syspath binary not executable: $BIN_SP"; exit 1; }
          RUNFILES_DIR="$PWD/$BIN_SP.runfiles" "$BIN_SP" "$GITHUB_WORKSPACE/pr-syspath.json"

      # ── Compare ─────────────────────────────────────────────────────────────
      - name: Compare startup results
        id: compare
        continue-on-error: true
        run: |
          set -euo pipefail
          python3 rules_py_pr/benchmark/startup/compare.py \
            --output-table startup-table.txt \
            bcr.json main.json pr.json

      - name: Save PR number
        if: github.event_name == 'pull_request'
        run: echo "${{ github.event.pull_request.number }}" > pr-number.txt

      - name: Upload startup artifacts
        if: github.event_name == 'pull_request'
        uses: actions/upload-artifact@v4
        with:
          name: startup-benchmark-results
          path: |
            startup-table.txt
            pr-number.txt
          if-no-files-found: warn

      - name: Fail on regression
        if: steps.compare.outcome == 'failure'
        run: exit 1

  analysis:
    runs-on: ubuntu-latest
    permissions:
      contents: read
    steps:
      - name: Checkout PR
        uses: actions/checkout@v6
        with:
          fetch-depth: 0
          path: rules_py_pr

      - name: Checkout HEAD main
        uses: actions/checkout@v6
        with:
          ref: main
          path: rules_py_main

      - name: Install hyperfine
        run: |
          set -euo pipefail
          HYPERFINE_VERSION="1.18.0"
          DEB="hyperfine_${HYPERFINE_VERSION}_amd64.deb"
          URL="https://github.com/sharkdp/hyperfine/releases/download/v${HYPERFINE_VERSION}/${DEB}"
          wget -qO "$DEB" "$URL"
          sudo dpkg -i "$DEB"
          rm -f "$DEB"
          hyperfine --version

      - name: Generate workspace
        run: |
          set -euo pipefail
          cd rules_py_pr/benchmark/analysis
          python3 workspace/generate_workspace.py --root workspace --packages 50

      # ── BCR baseline ────────────────────────────────────────────────────────
      - name: Benchmark BCR ${{ env.ANALYSIS_BCR_VERSION }}
        run: |
          set -euo pipefail
          cd rules_py_pr/benchmark/analysis
          python3 generate_module.py bcr --version "${ANALYSIS_BCR_VERSION}"

          OUT_BASE="/tmp/bazel-bcr"
          rm -rf "$OUT_BASE"

          bazel --output_base="$OUT_BASE" --bazelrc=../../.github/workflows/ci.bazelrc fetch //workspace/...

          hyperfine --warmup 1 --runs 10 \
            --prepare 'rm -rf /tmp/bazel-bcr-analysis' \
            --export-json "${GITHUB_WORKSPACE}/bcr.json" \
            'bazel --output_base=/tmp/bazel-bcr-analysis --bazelrc=../../.github/workflows/ci.bazelrc build --disk_cache= --nobuild //workspace/...'

          bazel --output_base="$OUT_BASE" --bazelrc=../../.github/workflows/ci.bazelrc query //workspace/... --output=package > /tmp/bcr-packages.txt
          bazel --output_base="$OUT_BASE" --bazelrc=../../.github/workflows/ci.bazelrc query //workspace/... > /tmp/bcr-targets.txt
          echo "{\"packages\": $(wc -l < /tmp/bcr-packages.txt | tr -d ' '), \"targets\": $(wc -l < /tmp/bcr-targets.txt | tr -d ' ')}" > "${GITHUB_WORKSPACE}/bcr-aux.json"

      # ── HEAD main ───────────────────────────────────────────────────────────
      - name: Benchmark HEAD main
        run: |
          set -euo pipefail
          cd rules_py_pr/benchmark/analysis
          python3 generate_module.py local --path "$GITHUB_WORKSPACE/rules_py_main"

          OUT_BASE="/tmp/bazel-main"
          rm -rf "$OUT_BASE"

          bazel --output_base="$OUT_BASE" --bazelrc=../../.github/workflows/ci.bazelrc fetch //workspace/...

          hyperfine --warmup 1 --runs 10 \
            --prepare 'rm -rf /tmp/bazel-main-analysis' \
            --export-json "${GITHUB_WORKSPACE}/main.json" \
            'bazel --output_base=/tmp/bazel-main-analysis --bazelrc=../../.github/workflows/ci.bazelrc build --disk_cache= --nobuild //workspace/...'

          bazel --output_base="$OUT_BASE" --bazelrc=../../.github/workflows/ci.bazelrc query //workspace/... --output=package > /tmp/main-packages.txt
          bazel --output_base="$OUT_BASE" --bazelrc=../../.github/workflows/ci.bazelrc query //workspace/... > /tmp/main-targets.txt
          echo "{\"packages\": $(wc -l < /tmp/main-packages.txt | tr -d ' '), \"targets\": $(wc -l < /tmp/main-targets.txt | tr -d ' ')}" > "${GITHUB_WORKSPACE}/main-aux.json"

      # ── Current commit (PR) ─────────────────────────────────────────────────
      - name: Benchmark current PR
        run: |
          set -euo pipefail
          cd rules_py_pr/benchmark/analysis
          python3 generate_module.py local --path "$GITHUB_WORKSPACE/rules_py_pr"

          OUT_BASE="/tmp/bazel-pr"
          rm -rf "$OUT_BASE"

          bazel --output_base="$OUT_BASE" --bazelrc=../../.github/workflows/ci.bazelrc fetch //workspace/...

          hyperfine --warmup 1 --runs 10 \
            --prepare 'rm -rf /tmp/bazel-pr-analysis' \
            --export-json "${GITHUB_WORKSPACE}/pr.json" \
            'bazel --output_base=/tmp/bazel-pr-analysis --bazelrc=../../.github/workflows/ci.bazelrc build --disk_cache= --nobuild //workspace/...'

          bazel --output_base="$OUT_BASE" --bazelrc=../../.github/workflows/ci.bazelrc query //workspace/... --output=package > /tmp/pr-packages.txt
          bazel --output_base="$OUT_BASE" --bazelrc=../../.github/workflows/ci.bazelrc query //workspace/... > /tmp/pr-targets.txt
          echo "{\"packages\": $(wc -l < /tmp/pr-packages.txt | tr -d ' '), \"targets\": $(wc -l < /tmp/pr-targets.txt | tr -d ' ')}" > "${GITHUB_WORKSPACE}/pr-aux.json"

      # ── Compare ─────────────────────────────────────────────────────────────
      - name: Compare analysis results
        id: compare
        continue-on-error: true
        run: |
          set -euo pipefail
          python3 rules_py_pr/benchmark/analysis/compare.py \
            --output-table analysis-table.txt \
            bcr.json main.json pr.json

      - name: Upload analysis artifacts
        if: github.event_name == 'pull_request'
        uses: actions/upload-artifact@v4
        with:
          name: analysis-benchmark-results
          path: |
            analysis-table.txt
          if-no-files-found: warn

      - name: Fail on regression
        if: steps.compare.outcome == 'failure'
        run: exit 1

  comment:
    runs-on: ubuntu-latest
    needs: [startup, analysis]
    if: always() && github.event_name == 'pull_request' && (needs.startup.result == 'success' || needs.startup.result == 'failure') && (needs.analysis.result == 'success' || needs.analysis.result == 'failure')
    permissions:
      pull-requests: write
    steps:
      - name: Download startup artifacts
        uses: actions/download-artifact@v4
        with:
          name: startup-benchmark-results

      - name: Download analysis artifacts
        uses: actions/download-artifact@v4
        with:
          name: analysis-benchmark-results

      - name: Post combined benchmark comment
        uses: actions/github-script@v7
        with:
          script: |
            const fs = require('fs');
            const startup = fs.readFileSync('startup-table.txt', 'utf8').trim();
            const analysis = fs.readFileSync('analysis-table.txt', 'utf8').trim();
            const prNumber = parseInt(fs.readFileSync('pr-number.txt', 'utf8').trim());
            const header = '<!-- performance-benchmark -->';
            const body = `${header}\n\n${startup}\n\n${analysis}`;
            const { data: comments } = await github.rest.issues.listComments({
              owner: context.repo.owner,
              repo: context.repo.repo,
              issue_number: prNumber,
            });
            const existing = comments.find(c => c.body && c.body.includes(header));
            if (existing) {
              await github.rest.issues.updateComment({
                owner: context.repo.owner,
                repo: context.repo.repo,
                comment_id: existing.id,
                body: body,
              });
              console.log(`Updated comment ${existing.id}`);
            } else {
              const { data: created } = await github.rest.issues.createComment({
                owner: context.repo.owner,
                repo: context.repo.repo,
                issue_number: prNumber,
                body: body,
              });
              console.log(`Created comment ${created.id}`);
            }

      - name: Fail if any benchmark regressed
        if: needs.startup.result == 'failure' || needs.analysis.result == 'failure'
        run: exit 1