From d0a97df2d9775b42ffbdb6b6b22c816b5fc82be4 Mon Sep 17 00:00:00 2001 From: MilesCranmer Date: Mon, 29 Jul 2024 10:45:41 +0100 Subject: [PATCH 01/13] create simple benchmark suite --- benchmark/Project.toml | 2 ++ benchmark/benchmarks.jl | 49 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+) create mode 100644 benchmark/Project.toml create mode 100644 benchmark/benchmarks.jl diff --git a/benchmark/Project.toml b/benchmark/Project.toml new file mode 100644 index 00000000..05a4894b --- /dev/null +++ b/benchmark/Project.toml @@ -0,0 +1,2 @@ +[deps] +BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf" diff --git a/benchmark/benchmarks.jl b/benchmark/benchmarks.jl new file mode 100644 index 00000000..c057a885 --- /dev/null +++ b/benchmark/benchmarks.jl @@ -0,0 +1,49 @@ +using PythonCall, BenchmarkTools + +const SUITE = BenchmarkGroup() + +function test_pydict_init() + random = pyimport("random").random + x = pydict() + for i in pyrange(1000) + x[pystr(i)] = i + random() + end + return x +end + +SUITE["pydict"]["init"] = @benchmarkable test_pydict_init() + +function test_pydict_pydel() + random = pyimport("random").random + x = pydict() + for i in pyrange(1000) + k = pystr(i) + r = random() + v = i + r + x[k] = v + pydel!(k) + pydel!(r) + pydel!(v) + pydel!(i) + end + return x +end + +SUITE["pydict"]["pydel"] = @benchmarkable test_pydict_pydel() + +@generated function test_atpy(::Val{use_pydel}) where {use_pydel} + quote + @py begin + import random: random + x = {} + for i in range(1000) + x[str(i)] = i + random() + $(use_pydel ? :(@jl PythonCall.pydel!(i)) : :(nothing)) + end + x + end + end +end + +SUITE["@py"]["basic"] = @benchmarkable test_atpy(Val(false)) +SUITE["@py"]["pydel"] = @benchmarkable test_atpy(Val(true)) From 410e31cb8be95d336fa06372540c87f33198d421 Mon Sep 17 00:00:00 2001 From: MilesCranmer Date: Mon, 29 Jul 2024 10:47:16 +0100 Subject: [PATCH 02/13] more hierarchy in benchmark --- benchmark/benchmarks.jl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/benchmark/benchmarks.jl b/benchmark/benchmarks.jl index c057a885..48b7bd59 100644 --- a/benchmark/benchmarks.jl +++ b/benchmark/benchmarks.jl @@ -11,7 +11,7 @@ function test_pydict_init() return x end -SUITE["pydict"]["init"] = @benchmarkable test_pydict_init() +SUITE["julia"]["pydict"]["init"] = @benchmarkable test_pydict_init() function test_pydict_pydel() random = pyimport("random").random @@ -29,7 +29,7 @@ function test_pydict_pydel() return x end -SUITE["pydict"]["pydel"] = @benchmarkable test_pydict_pydel() +SUITE["julia"]["pydict"]["pydel"] = @benchmarkable test_pydict_pydel() @generated function test_atpy(::Val{use_pydel}) where {use_pydel} quote @@ -45,5 +45,5 @@ SUITE["pydict"]["pydel"] = @benchmarkable test_pydict_pydel() end end -SUITE["@py"]["basic"] = @benchmarkable test_atpy(Val(false)) -SUITE["@py"]["pydel"] = @benchmarkable test_atpy(Val(true)) +SUITE["@py"]["pydict"]["init"] = @benchmarkable test_atpy(Val(false)) +SUITE["@py"]["pydict"]["pydel"] = @benchmarkable test_atpy(Val(true)) From cd1948981f3d650e891c4d5f802ca7911470a69e Mon Sep 17 00:00:00 2001 From: MilesCranmer Date: Mon, 29 Jul 2024 10:48:57 +0100 Subject: [PATCH 03/13] create AirspeedVelocity github action for benchmarks --- .github/workflows/benchmark_pr.yml | 64 ++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 .github/workflows/benchmark_pr.yml diff --git a/.github/workflows/benchmark_pr.yml b/.github/workflows/benchmark_pr.yml new file mode 100644 index 00000000..4c695cda --- /dev/null +++ b/.github/workflows/benchmark_pr.yml @@ -0,0 +1,64 @@ +name: Benchmark a pull request + +on: + pull_request_target: + branches: + - master + +permissions: + pull-requests: write + +jobs: + generate_plots: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + - uses: julia-actions/setup-julia@v1 + with: + version: "1.9" + - uses: julia-actions/cache@v1 + - name: Extract Package Name from Project.toml + id: extract-package-name + run: | + PACKAGE_NAME=$(grep "^name" Project.toml | sed 's/^name = "\(.*\)"$/\1/') + echo "::set-output name=package_name::$PACKAGE_NAME" + - name: Build AirspeedVelocity + env: + JULIA_NUM_THREADS: 2 + run: | + # Lightweight build step, as sometimes the runner runs out of memory: + julia -e 'ENV["JULIA_PKG_PRECOMPILE_AUTO"]=0; import Pkg; Pkg.add(;url="https://github.com/MilesCranmer/AirspeedVelocity.jl.git")' + julia -e 'ENV["JULIA_PKG_PRECOMPILE_AUTO"]=0; import Pkg; Pkg.build("AirspeedVelocity")' + - name: Add ~/.julia/bin to PATH + run: | + echo "$HOME/.julia/bin" >> $GITHUB_PATH + - name: Run benchmarks + run: | + echo $PATH + ls -l ~/.julia/bin + mkdir results + benchpkg ${{ steps.extract-package-name.outputs.package_name }} --rev="${{github.event.repository.default_branch}},${{github.event.pull_request.head.sha}}" --url=${{ github.event.repository.clone_url }} --bench-on="${{github.event.pull_request.head.sha}}" --output-dir=results/ --tune --exeflags="-O3 --threads=auto" + - name: Create markdown table from benchmarks + run: | + benchpkgtable ${{ steps.extract-package-name.outputs.package_name }} --rev="${{github.event.repository.default_branch}},${{github.event.pull_request.head.sha}}" --input-dir=results/ --ratio > table.md + echo '### Benchmark Results' > body.md + echo '' >> body.md + echo '' >> body.md + cat table.md >> body.md + echo '' >> body.md + - name: Find Comment + uses: peter-evans/find-comment@v2 + id: fcbenchmark + with: + issue-number: ${{ github.event.pull_request.number }} + comment-author: 'github-actions[bot]' + body-includes: Benchmark Results + + - name: Comment on PR + uses: peter-evans/create-or-update-comment@v3 + with: + comment-id: ${{ steps.fcbenchmark.outputs.comment-id }} + issue-number: ${{ github.event.pull_request.number }} + body-path: body.md + edit-mode: replace From 072423da05c59459d621749880278d3c5dc3ea38 Mon Sep 17 00:00:00 2001 From: MilesCranmer Date: Mon, 29 Jul 2024 10:51:01 +0100 Subject: [PATCH 04/13] fix imports in benchmark --- benchmark/benchmarks.jl | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/benchmark/benchmarks.jl b/benchmark/benchmarks.jl index 48b7bd59..94414caa 100644 --- a/benchmark/benchmarks.jl +++ b/benchmark/benchmarks.jl @@ -1,4 +1,6 @@ -using PythonCall, BenchmarkTools +using BenchmarkTools +using PythonCall +using PythonCall: pydel!, pyimport, pydict, pystr, pyrange const SUITE = BenchmarkGroup() From 5f61a9c2f162976f2fea9c3658612247c0609608 Mon Sep 17 00:00:00 2001 From: Miles Cranmer Date: Mon, 29 Jul 2024 19:00:49 +0900 Subject: [PATCH 05/13] fix default branch name in CI --- .github/workflows/benchmark_pr.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/benchmark_pr.yml b/.github/workflows/benchmark_pr.yml index 4c695cda..4ad1b6a1 100644 --- a/.github/workflows/benchmark_pr.yml +++ b/.github/workflows/benchmark_pr.yml @@ -3,7 +3,7 @@ name: Benchmark a pull request on: pull_request_target: branches: - - master + - main permissions: pull-requests: write From fcc687abb7c7461b7eef983a55597c89496a72cb Mon Sep 17 00:00:00 2001 From: MilesCranmer Date: Fri, 2 Aug 2024 16:23:08 +0100 Subject: [PATCH 06/13] benchmarks: more hierarchy --- benchmark/benchmarks.jl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/benchmark/benchmarks.jl b/benchmark/benchmarks.jl index 94414caa..860844d4 100644 --- a/benchmark/benchmarks.jl +++ b/benchmark/benchmarks.jl @@ -13,7 +13,7 @@ function test_pydict_init() return x end -SUITE["julia"]["pydict"]["init"] = @benchmarkable test_pydict_init() +SUITE["basic"]["julia"]["pydict"]["init"] = @benchmarkable test_pydict_init() function test_pydict_pydel() random = pyimport("random").random @@ -31,7 +31,7 @@ function test_pydict_pydel() return x end -SUITE["julia"]["pydict"]["pydel"] = @benchmarkable test_pydict_pydel() +SUITE["basic"]["julia"]["pydict"]["pydel"] = @benchmarkable test_pydict_pydel() @generated function test_atpy(::Val{use_pydel}) where {use_pydel} quote @@ -47,5 +47,5 @@ SUITE["julia"]["pydict"]["pydel"] = @benchmarkable test_pydict_pydel() end end -SUITE["@py"]["pydict"]["init"] = @benchmarkable test_atpy(Val(false)) -SUITE["@py"]["pydict"]["pydel"] = @benchmarkable test_atpy(Val(true)) +SUITE["basic"]["@py"]["pydict"]["init"] = @benchmarkable test_atpy(Val(false)) +SUITE["basic"]["@py"]["pydict"]["pydel"] = @benchmarkable test_atpy(Val(true)) From 4f7858987626a1b04bb64570242b33d05a3bb4aa Mon Sep 17 00:00:00 2001 From: Eric Hanson <5846501+ericphanson@users.noreply.github.com> Date: Sat, 13 Jul 2024 18:14:32 +0200 Subject: [PATCH 07/13] timing + benchmark --- gcbench.jl | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 gcbench.jl diff --git a/gcbench.jl b/gcbench.jl new file mode 100644 index 00000000..a2e6ea1a --- /dev/null +++ b/gcbench.jl @@ -0,0 +1,30 @@ +using PythonCall, Test + +# https://github.com/JuliaCI/GCBenchmarks/blob/fc288c696381ebfdef8f002168addd0ec1b08e34/benches/serial/append/append.jl +macro gctime(ex) + quote + local prior = PythonCall.GC.SECONDS_SPENT_IN_GC[] + local ret = @timed $(esc(ex)) + Base.time_print(stdout, ret.time * 1e9, ret.gcstats.allocd, ret.gcstats.total_time, Base.gc_alloc_count(ret.gcstats); msg="Runtime") + local after = PythonCall.GC.SECONDS_SPENT_IN_GC[] + println(stdout) + Base.time_print(stdout, (after - prior) * 1e9; msg="Python GC time") + println(stdout) + ret.value + end +end + +function append_lots(iters=100 * 1024, size=1596) + v = pylist() + for i = 1:iters + v.append(pylist(rand(size))) + end + return v +end + +@time "Total" begin + @gctime append_lots() + @time "Next full GC" begin + GC.gc(true) + end +end From 397ef3f6476b95ec7e81b3c5ce710e254e6dd64c Mon Sep 17 00:00:00 2001 From: MilesCranmer Date: Fri, 2 Aug 2024 16:28:11 +0100 Subject: [PATCH 08/13] benchmarks: include gcbench in runs --- benchmark/benchmarks.jl | 4 ++++ gcbench.jl => benchmark/gcbench.jl | 7 ------- 2 files changed, 4 insertions(+), 7 deletions(-) rename gcbench.jl => benchmark/gcbench.jl (87%) diff --git a/benchmark/benchmarks.jl b/benchmark/benchmarks.jl index 860844d4..f4462883 100644 --- a/benchmark/benchmarks.jl +++ b/benchmark/benchmarks.jl @@ -49,3 +49,7 @@ end SUITE["basic"]["@py"]["pydict"]["init"] = @benchmarkable test_atpy(Val(false)) SUITE["basic"]["@py"]["pydict"]["pydel"] = @benchmarkable test_atpy(Val(true)) + + +include("gcbench.jl") +SUITE["gc"]["append_and_full_gc"] = @benchmarkable GC.gc(true) setup=(append_lots()) diff --git a/gcbench.jl b/benchmark/gcbench.jl similarity index 87% rename from gcbench.jl rename to benchmark/gcbench.jl index a2e6ea1a..45e1496c 100644 --- a/gcbench.jl +++ b/benchmark/gcbench.jl @@ -21,10 +21,3 @@ function append_lots(iters=100 * 1024, size=1596) end return v end - -@time "Total" begin - @gctime append_lots() - @time "Next full GC" begin - GC.gc(true) - end -end From 37c89d7546f0ef6749190e9b1e9a8438497073bb Mon Sep 17 00:00:00 2001 From: MilesCranmer Date: Fri, 2 Aug 2024 16:28:53 +0100 Subject: [PATCH 09/13] benchmarks: remove unused code --- benchmark/gcbench.jl | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/benchmark/gcbench.jl b/benchmark/gcbench.jl index 45e1496c..88b7bf81 100644 --- a/benchmark/gcbench.jl +++ b/benchmark/gcbench.jl @@ -1,19 +1,5 @@ using PythonCall, Test -# https://github.com/JuliaCI/GCBenchmarks/blob/fc288c696381ebfdef8f002168addd0ec1b08e34/benches/serial/append/append.jl -macro gctime(ex) - quote - local prior = PythonCall.GC.SECONDS_SPENT_IN_GC[] - local ret = @timed $(esc(ex)) - Base.time_print(stdout, ret.time * 1e9, ret.gcstats.allocd, ret.gcstats.total_time, Base.gc_alloc_count(ret.gcstats); msg="Runtime") - local after = PythonCall.GC.SECONDS_SPENT_IN_GC[] - println(stdout) - Base.time_print(stdout, (after - prior) * 1e9; msg="Python GC time") - println(stdout) - ret.value - end -end - function append_lots(iters=100 * 1024, size=1596) v = pylist() for i = 1:iters From 5295cc7c71625d5bac0245eca42d7cf6ee70d456 Mon Sep 17 00:00:00 2001 From: MilesCranmer Date: Fri, 2 Aug 2024 16:30:36 +0100 Subject: [PATCH 10/13] benchmarks: modularize --- benchmark/benchmarks.jl | 2 ++ benchmark/gcbench.jl | 6 +++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/benchmark/benchmarks.jl b/benchmark/benchmarks.jl index f4462883..70d2a48e 100644 --- a/benchmark/benchmarks.jl +++ b/benchmark/benchmarks.jl @@ -52,4 +52,6 @@ SUITE["basic"]["@py"]["pydict"]["pydel"] = @benchmarkable test_atpy(Val(true)) include("gcbench.jl") +using .GCBench: append_lots + SUITE["gc"]["append_and_full_gc"] = @benchmarkable GC.gc(true) setup=(append_lots()) diff --git a/benchmark/gcbench.jl b/benchmark/gcbench.jl index 88b7bf81..56f8fc05 100644 --- a/benchmark/gcbench.jl +++ b/benchmark/gcbench.jl @@ -1,4 +1,6 @@ -using PythonCall, Test +module GCBench + +using PythonCall function append_lots(iters=100 * 1024, size=1596) v = pylist() @@ -7,3 +9,5 @@ function append_lots(iters=100 * 1024, size=1596) end return v end + +end From 5d4e630a71ce43122c8c3d21db0b1b97c9ecea29 Mon Sep 17 00:00:00 2001 From: MilesCranmer Date: Fri, 2 Aug 2024 17:31:53 +0100 Subject: [PATCH 11/13] benchmarks: give more time to GC benchmark --- benchmark/benchmarks.jl | 6 +++++- benchmark/gcbench.jl | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/benchmark/benchmarks.jl b/benchmark/benchmarks.jl index 70d2a48e..db8b58cb 100644 --- a/benchmark/benchmarks.jl +++ b/benchmark/benchmarks.jl @@ -54,4 +54,8 @@ SUITE["basic"]["@py"]["pydict"]["pydel"] = @benchmarkable test_atpy(Val(true)) include("gcbench.jl") using .GCBench: append_lots -SUITE["gc"]["append_and_full_gc"] = @benchmarkable GC.gc(true) setup=(append_lots()) +SUITE["gc"]["append_and_full_gc"] = @benchmarkable( + GC.gc(true), + setup=(GC.gc(true); append_lots(size=159)), + seconds=30, +) diff --git a/benchmark/gcbench.jl b/benchmark/gcbench.jl index 56f8fc05..d9e83070 100644 --- a/benchmark/gcbench.jl +++ b/benchmark/gcbench.jl @@ -2,7 +2,7 @@ module GCBench using PythonCall -function append_lots(iters=100 * 1024, size=1596) +function append_lots(; iters=100 * 1024, size=1596) v = pylist() for i = 1:iters v.append(pylist(rand(size))) From ae73e43f1fae4a422f788d748a6532eadb3bb569 Mon Sep 17 00:00:00 2001 From: MilesCranmer Date: Fri, 2 Aug 2024 17:40:14 +0100 Subject: [PATCH 12/13] benchmarks: simplify naming scheme --- benchmark/benchmarks.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmark/benchmarks.jl b/benchmark/benchmarks.jl index db8b58cb..47ce71ad 100644 --- a/benchmark/benchmarks.jl +++ b/benchmark/benchmarks.jl @@ -54,7 +54,7 @@ SUITE["basic"]["@py"]["pydict"]["pydel"] = @benchmarkable test_atpy(Val(true)) include("gcbench.jl") using .GCBench: append_lots -SUITE["gc"]["append_and_full_gc"] = @benchmarkable( +SUITE["gc"]["full"] = @benchmarkable( GC.gc(true), setup=(GC.gc(true); append_lots(size=159)), seconds=30, From 83ddd096ba9c57e4cce0efc2042935ef6a6d08e8 Mon Sep 17 00:00:00 2001 From: Miles Cranmer Date: Sat, 3 Aug 2024 02:52:39 +0900 Subject: [PATCH 13/13] benchmarks: avoid issue of `tune`ing away from `evals=1` --- benchmark/benchmarks.jl | 1 + 1 file changed, 1 insertion(+) diff --git a/benchmark/benchmarks.jl b/benchmark/benchmarks.jl index 47ce71ad..38bbfd07 100644 --- a/benchmark/benchmarks.jl +++ b/benchmark/benchmarks.jl @@ -58,4 +58,5 @@ SUITE["gc"]["full"] = @benchmarkable( GC.gc(true), setup=(GC.gc(true); append_lots(size=159)), seconds=30, + evals=1, )