From d0a97df2d9775b42ffbdb6b6b22c816b5fc82be4 Mon Sep 17 00:00:00 2001
From: MilesCranmer <miles.cranmer@gmail.com>
Date: Mon, 29 Jul 2024 10:45:41 +0100
Subject: [PATCH 01/13] create simple benchmark suite
---
benchmark/Project.toml | 2 ++
benchmark/benchmarks.jl | 49 +++++++++++++++++++++++++++++++++++++++++
2 files changed, 51 insertions(+)
create mode 100644 benchmark/Project.toml
create mode 100644 benchmark/benchmarks.jl
diff --git a/benchmark/Project.toml b/benchmark/Project.toml
new file mode 100644
index 00000000..05a4894b
--- /dev/null
+++ b/benchmark/Project.toml
@@ -0,0 +1,2 @@
+[deps]
+BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf"
diff --git a/benchmark/benchmarks.jl b/benchmark/benchmarks.jl
new file mode 100644
index 00000000..c057a885
--- /dev/null
+++ b/benchmark/benchmarks.jl
@@ -0,0 +1,49 @@
+using PythonCall, BenchmarkTools
+
+const SUITE = BenchmarkGroup()
+
+function test_pydict_init()
+ random = pyimport("random").random
+ x = pydict()
+ for i in pyrange(1000)
+ x[pystr(i)] = i + random()
+ end
+ return x
+end
+
+SUITE["pydict"]["init"] = @benchmarkable test_pydict_init()
+
+function test_pydict_pydel()
+ random = pyimport("random").random
+ x = pydict()
+ for i in pyrange(1000)
+ k = pystr(i)
+ r = random()
+ v = i + r
+ x[k] = v
+ pydel!(k)
+ pydel!(r)
+ pydel!(v)
+ pydel!(i)
+ end
+ return x
+end
+
+SUITE["pydict"]["pydel"] = @benchmarkable test_pydict_pydel()
+
+@generated function test_atpy(::Val{use_pydel}) where {use_pydel}
+ quote
+ @py begin
+ import random: random
+ x = {}
+ for i in range(1000)
+ x[str(i)] = i + random()
+ $(use_pydel ? :(@jl PythonCall.pydel!(i)) : :(nothing))
+ end
+ x
+ end
+ end
+end
+
+SUITE["@py"]["basic"] = @benchmarkable test_atpy(Val(false))
+SUITE["@py"]["pydel"] = @benchmarkable test_atpy(Val(true))
From 410e31cb8be95d336fa06372540c87f33198d421 Mon Sep 17 00:00:00 2001
From: MilesCranmer <miles.cranmer@gmail.com>
Date: Mon, 29 Jul 2024 10:47:16 +0100
Subject: [PATCH 02/13] more hierarchy in benchmark
---
benchmark/benchmarks.jl | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/benchmark/benchmarks.jl b/benchmark/benchmarks.jl
index c057a885..48b7bd59 100644
--- a/benchmark/benchmarks.jl
+++ b/benchmark/benchmarks.jl
@@ -11,7 +11,7 @@ function test_pydict_init()
return x
end
-SUITE["pydict"]["init"] = @benchmarkable test_pydict_init()
+SUITE["julia"]["pydict"]["init"] = @benchmarkable test_pydict_init()
function test_pydict_pydel()
random = pyimport("random").random
@@ -29,7 +29,7 @@ function test_pydict_pydel()
return x
end
-SUITE["pydict"]["pydel"] = @benchmarkable test_pydict_pydel()
+SUITE["julia"]["pydict"]["pydel"] = @benchmarkable test_pydict_pydel()
@generated function test_atpy(::Val{use_pydel}) where {use_pydel}
quote
@@ -45,5 +45,5 @@ SUITE["pydict"]["pydel"] = @benchmarkable test_pydict_pydel()
end
end
-SUITE["@py"]["basic"] = @benchmarkable test_atpy(Val(false))
-SUITE["@py"]["pydel"] = @benchmarkable test_atpy(Val(true))
+SUITE["@py"]["pydict"]["init"] = @benchmarkable test_atpy(Val(false))
+SUITE["@py"]["pydict"]["pydel"] = @benchmarkable test_atpy(Val(true))
From cd1948981f3d650e891c4d5f802ca7911470a69e Mon Sep 17 00:00:00 2001
From: MilesCranmer <miles.cranmer@gmail.com>
Date: Mon, 29 Jul 2024 10:48:57 +0100
Subject: [PATCH 03/13] create AirspeedVelocity github action for benchmarks
---
.github/workflows/benchmark_pr.yml | 64 ++++++++++++++++++++++++++++++
1 file changed, 64 insertions(+)
create mode 100644 .github/workflows/benchmark_pr.yml
diff --git a/.github/workflows/benchmark_pr.yml b/.github/workflows/benchmark_pr.yml
new file mode 100644
index 00000000..4c695cda
--- /dev/null
+++ b/.github/workflows/benchmark_pr.yml
@@ -0,0 +1,64 @@
+name: Benchmark a pull request
+
+on:
+ pull_request_target:
+ branches:
+ - master
+
+permissions:
+ pull-requests: write
+
+jobs:
+ generate_plots:
+ runs-on: ubuntu-latest
+
+ steps:
+ - uses: actions/checkout@v2
+ - uses: julia-actions/setup-julia@v1
+ with:
+ version: "1.9"
+ - uses: julia-actions/cache@v1
+ - name: Extract Package Name from Project.toml
+ id: extract-package-name
+ run: |
+ PACKAGE_NAME=$(grep "^name" Project.toml | sed 's/^name = "\(.*\)"$/\1/')
+ echo "::set-output name=package_name::$PACKAGE_NAME"
+ - name: Build AirspeedVelocity
+ env:
+ JULIA_NUM_THREADS: 2
+ run: |
+ # Lightweight build step, as sometimes the runner runs out of memory:
+ julia -e 'ENV["JULIA_PKG_PRECOMPILE_AUTO"]=0; import Pkg; Pkg.add(;url="https://github.com/MilesCranmer/AirspeedVelocity.jl.git")'
+ julia -e 'ENV["JULIA_PKG_PRECOMPILE_AUTO"]=0; import Pkg; Pkg.build("AirspeedVelocity")'
+ - name: Add ~/.julia/bin to PATH
+ run: |
+ echo "$HOME/.julia/bin" >> $GITHUB_PATH
+ - name: Run benchmarks
+ run: |
+ echo $PATH
+ ls -l ~/.julia/bin
+ mkdir results
+ benchpkg ${{ steps.extract-package-name.outputs.package_name }} --rev="${{github.event.repository.default_branch}},${{github.event.pull_request.head.sha}}" --url=${{ github.event.repository.clone_url }} --bench-on="${{github.event.pull_request.head.sha}}" --output-dir=results/ --tune --exeflags="-O3 --threads=auto"
+ - name: Create markdown table from benchmarks
+ run: |
+ benchpkgtable ${{ steps.extract-package-name.outputs.package_name }} --rev="${{github.event.repository.default_branch}},${{github.event.pull_request.head.sha}}" --input-dir=results/ --ratio > table.md
+ echo '### Benchmark Results' > body.md
+ echo '' >> body.md
+ echo '' >> body.md
+ cat table.md >> body.md
+ echo '' >> body.md
+ - name: Find Comment
+ uses: peter-evans/find-comment@v2
+ id: fcbenchmark
+ with:
+ issue-number: ${{ github.event.pull_request.number }}
+ comment-author: 'github-actions[bot]'
+ body-includes: Benchmark Results
+
+ - name: Comment on PR
+ uses: peter-evans/create-or-update-comment@v3
+ with:
+ comment-id: ${{ steps.fcbenchmark.outputs.comment-id }}
+ issue-number: ${{ github.event.pull_request.number }}
+ body-path: body.md
+ edit-mode: replace
From 072423da05c59459d621749880278d3c5dc3ea38 Mon Sep 17 00:00:00 2001
From: MilesCranmer <miles.cranmer@gmail.com>
Date: Mon, 29 Jul 2024 10:51:01 +0100
Subject: [PATCH 04/13] fix imports in benchmark
---
benchmark/benchmarks.jl | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/benchmark/benchmarks.jl b/benchmark/benchmarks.jl
index 48b7bd59..94414caa 100644
--- a/benchmark/benchmarks.jl
+++ b/benchmark/benchmarks.jl
@@ -1,4 +1,6 @@
-using PythonCall, BenchmarkTools
+using BenchmarkTools
+using PythonCall
+using PythonCall: pydel!, pyimport, pydict, pystr, pyrange
const SUITE = BenchmarkGroup()
From 5f61a9c2f162976f2fea9c3658612247c0609608 Mon Sep 17 00:00:00 2001
From: Miles Cranmer <miles.cranmer@gmail.com>
Date: Mon, 29 Jul 2024 19:00:49 +0900
Subject: [PATCH 05/13] fix default branch name in CI
---
.github/workflows/benchmark_pr.yml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.github/workflows/benchmark_pr.yml b/.github/workflows/benchmark_pr.yml
index 4c695cda..4ad1b6a1 100644
--- a/.github/workflows/benchmark_pr.yml
+++ b/.github/workflows/benchmark_pr.yml
@@ -3,7 +3,7 @@ name: Benchmark a pull request
on:
pull_request_target:
branches:
- - master
+ - main
permissions:
pull-requests: write
From fcc687abb7c7461b7eef983a55597c89496a72cb Mon Sep 17 00:00:00 2001
From: MilesCranmer <miles.cranmer@gmail.com>
Date: Fri, 2 Aug 2024 16:23:08 +0100
Subject: [PATCH 06/13] benchmarks: more hierarchy
---
benchmark/benchmarks.jl | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/benchmark/benchmarks.jl b/benchmark/benchmarks.jl
index 94414caa..860844d4 100644
--- a/benchmark/benchmarks.jl
+++ b/benchmark/benchmarks.jl
@@ -13,7 +13,7 @@ function test_pydict_init()
return x
end
-SUITE["julia"]["pydict"]["init"] = @benchmarkable test_pydict_init()
+SUITE["basic"]["julia"]["pydict"]["init"] = @benchmarkable test_pydict_init()
function test_pydict_pydel()
random = pyimport("random").random
@@ -31,7 +31,7 @@ function test_pydict_pydel()
return x
end
-SUITE["julia"]["pydict"]["pydel"] = @benchmarkable test_pydict_pydel()
+SUITE["basic"]["julia"]["pydict"]["pydel"] = @benchmarkable test_pydict_pydel()
@generated function test_atpy(::Val{use_pydel}) where {use_pydel}
quote
@@ -47,5 +47,5 @@ SUITE["julia"]["pydict"]["pydel"] = @benchmarkable test_pydict_pydel()
end
end
-SUITE["@py"]["pydict"]["init"] = @benchmarkable test_atpy(Val(false))
-SUITE["@py"]["pydict"]["pydel"] = @benchmarkable test_atpy(Val(true))
+SUITE["basic"]["@py"]["pydict"]["init"] = @benchmarkable test_atpy(Val(false))
+SUITE["basic"]["@py"]["pydict"]["pydel"] = @benchmarkable test_atpy(Val(true))
From 4f7858987626a1b04bb64570242b33d05a3bb4aa Mon Sep 17 00:00:00 2001
From: Eric Hanson <5846501+ericphanson@users.noreply.github.com>
Date: Sat, 13 Jul 2024 18:14:32 +0200
Subject: [PATCH 07/13] timing + benchmark
---
gcbench.jl | 30 ++++++++++++++++++++++++++++++
1 file changed, 30 insertions(+)
create mode 100644 gcbench.jl
diff --git a/gcbench.jl b/gcbench.jl
new file mode 100644
index 00000000..a2e6ea1a
--- /dev/null
+++ b/gcbench.jl
@@ -0,0 +1,30 @@
+using PythonCall, Test
+
+# https://github.com/JuliaCI/GCBenchmarks/blob/fc288c696381ebfdef8f002168addd0ec1b08e34/benches/serial/append/append.jl
+macro gctime(ex)
+ quote
+ local prior = PythonCall.GC.SECONDS_SPENT_IN_GC[]
+ local ret = @timed $(esc(ex))
+ Base.time_print(stdout, ret.time * 1e9, ret.gcstats.allocd, ret.gcstats.total_time, Base.gc_alloc_count(ret.gcstats); msg="Runtime")
+ local after = PythonCall.GC.SECONDS_SPENT_IN_GC[]
+ println(stdout)
+ Base.time_print(stdout, (after - prior) * 1e9; msg="Python GC time")
+ println(stdout)
+ ret.value
+ end
+end
+
+function append_lots(iters=100 * 1024, size=1596)
+ v = pylist()
+ for i = 1:iters
+ v.append(pylist(rand(size)))
+ end
+ return v
+end
+
+@time "Total" begin
+ @gctime append_lots()
+ @time "Next full GC" begin
+ GC.gc(true)
+ end
+end
From 397ef3f6476b95ec7e81b3c5ce710e254e6dd64c Mon Sep 17 00:00:00 2001
From: MilesCranmer <miles.cranmer@gmail.com>
Date: Fri, 2 Aug 2024 16:28:11 +0100
Subject: [PATCH 08/13] benchmarks: include gcbench in runs
---
benchmark/benchmarks.jl | 4 ++++
gcbench.jl => benchmark/gcbench.jl | 7 -------
2 files changed, 4 insertions(+), 7 deletions(-)
rename gcbench.jl => benchmark/gcbench.jl (87%)
diff --git a/benchmark/benchmarks.jl b/benchmark/benchmarks.jl
index 860844d4..f4462883 100644
--- a/benchmark/benchmarks.jl
+++ b/benchmark/benchmarks.jl
@@ -49,3 +49,7 @@ end
SUITE["basic"]["@py"]["pydict"]["init"] = @benchmarkable test_atpy(Val(false))
SUITE["basic"]["@py"]["pydict"]["pydel"] = @benchmarkable test_atpy(Val(true))
+
+
+include("gcbench.jl")
+SUITE["gc"]["append_and_full_gc"] = @benchmarkable GC.gc(true) setup=(append_lots())
diff --git a/gcbench.jl b/benchmark/gcbench.jl
similarity index 87%
rename from gcbench.jl
rename to benchmark/gcbench.jl
index a2e6ea1a..45e1496c 100644
--- a/gcbench.jl
+++ b/benchmark/gcbench.jl
@@ -21,10 +21,3 @@ function append_lots(iters=100 * 1024, size=1596)
end
return v
end
-
-@time "Total" begin
- @gctime append_lots()
- @time "Next full GC" begin
- GC.gc(true)
- end
-end
From 37c89d7546f0ef6749190e9b1e9a8438497073bb Mon Sep 17 00:00:00 2001
From: MilesCranmer <miles.cranmer@gmail.com>
Date: Fri, 2 Aug 2024 16:28:53 +0100
Subject: [PATCH 09/13] benchmarks: remove unused code
---
benchmark/gcbench.jl | 14 --------------
1 file changed, 14 deletions(-)
diff --git a/benchmark/gcbench.jl b/benchmark/gcbench.jl
index 45e1496c..88b7bf81 100644
--- a/benchmark/gcbench.jl
+++ b/benchmark/gcbench.jl
@@ -1,19 +1,5 @@
using PythonCall, Test
-# https://github.com/JuliaCI/GCBenchmarks/blob/fc288c696381ebfdef8f002168addd0ec1b08e34/benches/serial/append/append.jl
-macro gctime(ex)
- quote
- local prior = PythonCall.GC.SECONDS_SPENT_IN_GC[]
- local ret = @timed $(esc(ex))
- Base.time_print(stdout, ret.time * 1e9, ret.gcstats.allocd, ret.gcstats.total_time, Base.gc_alloc_count(ret.gcstats); msg="Runtime")
- local after = PythonCall.GC.SECONDS_SPENT_IN_GC[]
- println(stdout)
- Base.time_print(stdout, (after - prior) * 1e9; msg="Python GC time")
- println(stdout)
- ret.value
- end
-end
-
function append_lots(iters=100 * 1024, size=1596)
v = pylist()
for i = 1:iters
From 5295cc7c71625d5bac0245eca42d7cf6ee70d456 Mon Sep 17 00:00:00 2001
From: MilesCranmer <miles.cranmer@gmail.com>
Date: Fri, 2 Aug 2024 16:30:36 +0100
Subject: [PATCH 10/13] benchmarks: modularize
---
benchmark/benchmarks.jl | 2 ++
benchmark/gcbench.jl | 6 +++++-
2 files changed, 7 insertions(+), 1 deletion(-)
diff --git a/benchmark/benchmarks.jl b/benchmark/benchmarks.jl
index f4462883..70d2a48e 100644
--- a/benchmark/benchmarks.jl
+++ b/benchmark/benchmarks.jl
@@ -52,4 +52,6 @@ SUITE["basic"]["@py"]["pydict"]["pydel"] = @benchmarkable test_atpy(Val(true))
include("gcbench.jl")
+using .GCBench: append_lots
+
SUITE["gc"]["append_and_full_gc"] = @benchmarkable GC.gc(true) setup=(append_lots())
diff --git a/benchmark/gcbench.jl b/benchmark/gcbench.jl
index 88b7bf81..56f8fc05 100644
--- a/benchmark/gcbench.jl
+++ b/benchmark/gcbench.jl
@@ -1,4 +1,6 @@
-using PythonCall, Test
+module GCBench
+
+using PythonCall
function append_lots(iters=100 * 1024, size=1596)
v = pylist()
@@ -7,3 +9,5 @@ function append_lots(iters=100 * 1024, size=1596)
end
return v
end
+
+end
From 5d4e630a71ce43122c8c3d21db0b1b97c9ecea29 Mon Sep 17 00:00:00 2001
From: MilesCranmer <miles.cranmer@gmail.com>
Date: Fri, 2 Aug 2024 17:31:53 +0100
Subject: [PATCH 11/13] benchmarks: give more time to GC benchmark
---
benchmark/benchmarks.jl | 6 +++++-
benchmark/gcbench.jl | 2 +-
2 files changed, 6 insertions(+), 2 deletions(-)
diff --git a/benchmark/benchmarks.jl b/benchmark/benchmarks.jl
index 70d2a48e..db8b58cb 100644
--- a/benchmark/benchmarks.jl
+++ b/benchmark/benchmarks.jl
@@ -54,4 +54,8 @@ SUITE["basic"]["@py"]["pydict"]["pydel"] = @benchmarkable test_atpy(Val(true))
include("gcbench.jl")
using .GCBench: append_lots
-SUITE["gc"]["append_and_full_gc"] = @benchmarkable GC.gc(true) setup=(append_lots())
+SUITE["gc"]["append_and_full_gc"] = @benchmarkable(
+ GC.gc(true),
+ setup=(GC.gc(true); append_lots(size=159)),
+ seconds=30,
+)
diff --git a/benchmark/gcbench.jl b/benchmark/gcbench.jl
index 56f8fc05..d9e83070 100644
--- a/benchmark/gcbench.jl
+++ b/benchmark/gcbench.jl
@@ -2,7 +2,7 @@ module GCBench
using PythonCall
-function append_lots(iters=100 * 1024, size=1596)
+function append_lots(; iters=100 * 1024, size=1596)
v = pylist()
for i = 1:iters
v.append(pylist(rand(size)))
From ae73e43f1fae4a422f788d748a6532eadb3bb569 Mon Sep 17 00:00:00 2001
From: MilesCranmer <miles.cranmer@gmail.com>
Date: Fri, 2 Aug 2024 17:40:14 +0100
Subject: [PATCH 12/13] benchmarks: simplify naming scheme
---
benchmark/benchmarks.jl | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/benchmark/benchmarks.jl b/benchmark/benchmarks.jl
index db8b58cb..47ce71ad 100644
--- a/benchmark/benchmarks.jl
+++ b/benchmark/benchmarks.jl
@@ -54,7 +54,7 @@ SUITE["basic"]["@py"]["pydict"]["pydel"] = @benchmarkable test_atpy(Val(true))
include("gcbench.jl")
using .GCBench: append_lots
-SUITE["gc"]["append_and_full_gc"] = @benchmarkable(
+SUITE["gc"]["full"] = @benchmarkable(
GC.gc(true),
setup=(GC.gc(true); append_lots(size=159)),
seconds=30,
From 83ddd096ba9c57e4cce0efc2042935ef6a6d08e8 Mon Sep 17 00:00:00 2001
From: Miles Cranmer <miles.cranmer@gmail.com>
Date: Sat, 3 Aug 2024 02:52:39 +0900
Subject: [PATCH 13/13] benchmarks: avoid issue of `tune`ing away from
`evals=1`
---
benchmark/benchmarks.jl | 1 +
1 file changed, 1 insertion(+)
diff --git a/benchmark/benchmarks.jl b/benchmark/benchmarks.jl
index 47ce71ad..38bbfd07 100644
--- a/benchmark/benchmarks.jl
+++ b/benchmark/benchmarks.jl
@@ -58,4 +58,5 @@ SUITE["gc"]["full"] = @benchmarkable(
GC.gc(true),
setup=(GC.gc(true); append_lots(size=159)),
seconds=30,
+ evals=1,
)