Skip to content

Commit 95fb7a8

Browse files
xangcastlexangcastle
committed
preparing analysis benchmark
1 parent 025c748 commit 95fb7a8

11 files changed

Lines changed: 5297 additions & 0 deletions

File tree

‎.github/workflows/analysis-benchmark-comment.yml‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
name: Post Analysis Benchmark Comment
2+
3+
on:
4+
workflow_run:
5+
workflows: ["Analysis Benchmark"]
6+
types: [completed]
7+
8+
permissions:
9+
pull-requests: write
10+
11+
jobs:
12+
comment:
13+
runs-on: ubuntu-latest
14+
if: github.event.workflow_run.event == 'pull_request'
15+
steps:
16+
- name: Download benchmark artifacts
17+
uses: actions/download-artifact@v4
18+
with:
19+
name: analysis-benchmark-results
20+
run-id: ${{ github.event.workflow_run.id }}
21+
github-token: ${{ secrets.GITHUB_TOKEN }}
22+
23+
- name: Post PR comment
24+
uses: actions/github-script@v7
25+
with:
26+
script: |
27+
const fs = require('fs');
28+
const prNumber = parseInt(fs.readFileSync('pr-number.txt', 'utf8').trim());
29+
const table = fs.readFileSync('benchmark-table.txt', 'utf8').trim();
30+
const header = '<!-- analysis-benchmark -->';
31+
const body = `${header}\n${table}`;
32+
const { data: comments } = await github.rest.issues.listComments({
33+
owner: context.repo.owner,
34+
repo: context.repo.repo,
35+
issue_number: prNumber,
36+
});
37+
const existing = comments.find(c => c.body && c.body.includes(header));
38+
if (existing) {
39+
await github.rest.issues.updateComment({
40+
owner: context.repo.owner,
41+
repo: context.repo.repo,
42+
comment_id: existing.id,
43+
body: body,
44+
});
45+
console.log(`Updated comment ${existing.id}`);
46+
} else {
47+
const { data: created } = await github.rest.issues.createComment({
48+
owner: context.repo.owner,
49+
repo: context.repo.repo,
50+
issue_number: prNumber,
51+
body: body,
52+
});
53+
console.log(`Created comment ${created.id}`);
54+
}

‎.github/workflows/analysis-benchmark.yml‎

Lines changed: 145 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,145 @@
1+
name: Analysis Benchmark
2+
3+
on:
4+
pull_request:
5+
push:
6+
branches: [main]
7+
8+
env:
9+
BCR_VERSION: "2.0.0-alpha.4"
10+
11+
jobs:
12+
benchmark:
13+
runs-on: ubuntu-latest
14+
permissions:
15+
contents: read
16+
17+
steps:
18+
- name: Checkout PR
19+
uses: actions/checkout@v6
20+
with:
21+
fetch-depth: 0
22+
path: rules_py_pr
23+
24+
- name: Checkout HEAD main
25+
uses: actions/checkout@v6
26+
with:
27+
ref: main
28+
path: rules_py_main
29+
30+
- name: Install hyperfine
31+
run: |
32+
set -euo pipefail
33+
HYPERFINE_VERSION="1.18.0"
34+
DEB="hyperfine_${HYPERFINE_VERSION}_amd64.deb"
35+
URL="https://github.com/sharkdp/hyperfine/releases/download/v${HYPERFINE_VERSION}/${DEB}"
36+
wget -qO "$DEB" "$URL"
37+
sudo dpkg -i "$DEB"
38+
rm -f "$DEB"
39+
hyperfine --version
40+
41+
# ── BCR baseline ────────────────────────────────────────────────────────
42+
- name: Benchmark BCR ${{ env.BCR_VERSION }}
43+
run: |
44+
set -euo pipefail
45+
cd rules_py_pr/benchmark/analysis
46+
python3 generate_module.py bcr --version "${BCR_VERSION}"
47+
python3 workspace/generate_workspace.py --root workspace --packages 50
48+
49+
OUT_BASE="/tmp/bazel-bcr"
50+
rm -rf "$OUT_BASE"
51+
52+
# Populate repository cache so the measured runs are analysis-only.
53+
bazel --output_base="$OUT_BASE" --bazelrc=../../.github/workflows/ci.bazelrc fetch //workspace/...
54+
55+
hyperfine --warmup 1 --runs 10 \
56+
--prepare 'rm -rf /tmp/bazel-bcr-analysis' \
57+
--export-json "${GITHUB_WORKSPACE}/bcr.json" \
58+
'bazel --output_base=/tmp/bazel-bcr-analysis --bazelrc=../../.github/workflows/ci.bazelrc build --disk_cache= --nobuild //workspace/...'
59+
60+
# Auxiliary metrics: loaded packages and discovered targets.
61+
bazel --output_base="$OUT_BASE" --bazelrc=../../.github/workflows/ci.bazelrc query //workspace/... --output=package > /tmp/bcr-packages.txt
62+
bazel --output_base="$OUT_BASE" --bazelrc=../../.github/workflows/ci.bazelrc query //workspace/... > /tmp/bcr-targets.txt
63+
echo "{\"packages\": $(wc -l < /tmp/bcr-packages.txt | tr -d ' '), \"targets\": $(wc -l < /tmp/bcr-targets.txt | tr -d ' ')}" > "${GITHUB_WORKSPACE}/bcr-aux.json"
64+
65+
# ── HEAD main ───────────────────────────────────────────────────────────
66+
- name: Benchmark HEAD main
67+
run: |
68+
set -euo pipefail
69+
cd rules_py_pr/benchmark/analysis
70+
python3 generate_module.py local --path "$GITHUB_WORKSPACE/rules_py_main"
71+
python3 workspace/generate_workspace.py --root workspace --packages 50
72+
73+
OUT_BASE="/tmp/bazel-main"
74+
rm -rf "$OUT_BASE"
75+
76+
bazel --output_base="$OUT_BASE" --bazelrc=../../.github/workflows/ci.bazelrc fetch //workspace/...
77+
78+
hyperfine --warmup 1 --runs 10 \
79+
--prepare 'rm -rf /tmp/bazel-main-analysis' \
80+
--export-json "${GITHUB_WORKSPACE}/main.json" \
81+
'bazel --output_base=/tmp/bazel-main-analysis --bazelrc=../../.github/workflows/ci.bazelrc build --disk_cache= --nobuild //workspace/...'
82+
83+
bazel --output_base="$OUT_BASE" --bazelrc=../../.github/workflows/ci.bazelrc query //workspace/... --output=package > /tmp/main-packages.txt
84+
bazel --output_base="$OUT_BASE" --bazelrc=../../.github/workflows/ci.bazelrc query //workspace/... > /tmp/main-targets.txt
85+
echo "{\"packages\": $(wc -l < /tmp/main-packages.txt | tr -d ' '), \"targets\": $(wc -l < /tmp/main-targets.txt | tr -d ' ')}" > "${GITHUB_WORKSPACE}/main-aux.json"
86+
87+
# ── Current commit (PR) ─────────────────────────────────────────────────
88+
- name: Benchmark current PR
89+
run: |
90+
set -euo pipefail
91+
cd rules_py_pr/benchmark/analysis
92+
python3 generate_module.py local --path "$GITHUB_WORKSPACE/rules_py_pr"
93+
python3 workspace/generate_workspace.py --root workspace --packages 50
94+
95+
OUT_BASE="/tmp/bazel-pr"
96+
rm -rf "$OUT_BASE"
97+
98+
bazel --output_base="$OUT_BASE" --bazelrc=../../.github/workflows/ci.bazelrc fetch //workspace/...
99+
100+
hyperfine --warmup 1 --runs 10 \
101+
--prepare 'rm -rf /tmp/bazel-pr-analysis' \
102+
--export-json "${GITHUB_WORKSPACE}/pr.json" \
103+
'bazel --output_base=/tmp/bazel-pr-analysis --bazelrc=../../.github/workflows/ci.bazelrc build --disk_cache= --nobuild //workspace/...'
104+
105+
bazel --output_base="$OUT_BASE" --bazelrc=../../.github/workflows/ci.bazelrc query //workspace/... --output=package > /tmp/pr-packages.txt
106+
bazel --output_base="$OUT_BASE" --bazelrc=../../.github/workflows/ci.bazelrc query //workspace/... > /tmp/pr-targets.txt
107+
echo "{\"packages\": $(wc -l < /tmp/pr-packages.txt | tr -d ' '), \"targets\": $(wc -l < /tmp/pr-targets.txt | tr -d ' ')}" > "${GITHUB_WORKSPACE}/pr-aux.json"
108+
109+
# ── Compare & gate ──────────────────────────────────────────────────────
110+
- name: Compare results
111+
id: compare
112+
continue-on-error: true
113+
run: |
114+
set -euo pipefail
115+
for f in bcr.json main.json pr.json; do
116+
if [[ ! -f "$f" ]]; then
117+
echo "ERROR: missing result file: $f"
118+
exit 1
119+
fi
120+
done
121+
python3 rules_py_pr/benchmark/analysis/compare.py bcr.json main.json pr.json | tee benchmark-table.txt
122+
123+
- name: Save PR number
124+
if: github.event_name == 'pull_request'
125+
run: echo "${{ github.event.pull_request.number }}" > pr-number.txt
126+
127+
- name: Upload benchmark artifacts
128+
if: github.event_name == 'pull_request'
129+
uses: actions/upload-artifact@v4
130+
with:
131+
name: analysis-benchmark-results
132+
path: |
133+
bcr.json
134+
main.json
135+
pr.json
136+
bcr-aux.json
137+
main-aux.json
138+
pr-aux.json
139+
benchmark-table.txt
140+
pr-number.txt
141+
if-no-files-found: warn
142+
143+
- name: Fail on regression
144+
if: steps.compare.outcome == 'failure'
145+
run: exit 1

‎benchmark/analysis/.bazelrc‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
# Default dependency group for the analysis benchmark workload.
2+
common --@pypi//dep_group=default

‎benchmark/analysis/MODULE.bazel‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
"Benchmark workspace for Bazel analysis phase performance"
2+
3+
module(name = "analysis_benchmark")
4+
5+
bazel_dep(name = "aspect_rules_py")
6+
local_path_override(
7+
module_name = "aspect_rules_py",
8+
path = "../..",
9+
)
10+
11+
bazel_dep(name = "bazel_skylib", version = "1.9.0")
12+
bazel_dep(name = "rules_python", version = "1.9.0")
13+
14+
interpreters = use_extension("@aspect_rules_py//py:extensions.bzl", "python_interpreters")
15+
interpreters.toolchain(
16+
python_version = "3.11",
17+
)
18+
use_repo(interpreters, "python_interpreters")
19+
20+
register_toolchains("@python_interpreters//:all")
21+
22+
uv = use_extension("@aspect_rules_py//uv:extensions.bzl", "uv")
23+
uv.declare_hub(hub_name = "pypi")
24+
uv.project(
25+
hub_name = "pypi",
26+
lock = "//workspace:uv.lock",
27+
pyproject = "//workspace:pyproject.toml",
28+
)
29+
use_repo(uv, "pypi")

‎benchmark/analysis/MODULE.bazel.template‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
"Benchmark workspace for Bazel analysis phase performance"
2+
3+
module(name = "analysis_benchmark")
4+
5+
{{RULES_PY_DECLARATION}}
6+
7+
bazel_dep(name = "bazel_skylib", version = "1.9.0")
8+
bazel_dep(name = "rules_python", version = "1.9.0")
9+
10+
# Python interpreters provisioned from python-build-standalone via aspect_rules_py
11+
interpreters = use_extension("@aspect_rules_py//py:extensions.bzl", "python_interpreters")
12+
interpreters.toolchain(
13+
python_version = "3.11",
14+
)
15+
use_repo(interpreters, "python_interpreters")
16+
17+
register_toolchains("@python_interpreters//:all")
18+
19+
# UV-based PyPI hub used to stress the rules_py analysis extension.
20+
uv = use_extension("@aspect_rules_py//uv:extensions.bzl", "uv")
21+
uv.declare_hub(hub_name = "pypi")
22+
uv.project(
23+
hub_name = "pypi",
24+
lock = "//workspace:uv.lock",
25+
pyproject = "//workspace:pyproject.toml",
26+
)
27+
use_repo(uv, "pypi")

0 commit comments

Comments
 (0)