Skip to content

Commit 03368af

Browse files
authored
chore: Single-shot benchmarking + continuous benchmarking (#183)
Setups `iai-callgrind`-based benchmarks for single-shot benchmarking (by using `callgrind` internally). Cleans up the benchmark cases and adds some structure so they can be executed both with `iai-callgrind` and `criterion`, depending on whether we want to measure instruction count or time. They can be called separatedly, ```bash # Single-shot, requires some extra setup cargo bench --bench iai_benches # Time-based, takes longer to run cargo bench --bench criterion_benches ``` See DEVELOPMENT.md for instructions. --- The instruction count benchmarks are now uploaded to [bencher.dev](https://bencher.dev/perf/portgraph), so we get an historical comparison of the performance and a CI check can alert about regressions. I believe the "No thresholds found" error in this PR will go away once this gets run in `main`. The service choice was mainly between bencher.dev and codspeed.io . I choose the former since it supports single-shot benchmarks natively. See this issue in `ratatui` where the Bencher maintainer discusses some differences, [ratatui/ratatui#1092](https://www.github.com/ratatui/ratatui/issues/1092#issuecomment-2415565274).
1 parent cb11a97 commit 03368af

17 files changed

+781
-309
lines changed

.github/workflows/archive-bencher.yml

+20
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
name: Archive Bencher.dev PR benchmarks
2+
on:
3+
pull_request:
4+
types:
5+
- closed
6+
7+
jobs:
8+
archive_pr_branch:
9+
name: Archive closed PR branch with Bencher
10+
if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository
11+
runs-on: ubuntu-latest
12+
steps:
13+
- uses: actions/checkout@v4
14+
- uses: bencherdev/bencher@main
15+
- name: Archive closed PR branch with Bencher
16+
run: |
17+
bencher archive \
18+
--project portgraph \
19+
--token '${{ secrets.BENCHER_API_TOKEN }}' \
20+
--branch "$GITHUB_HEAD_REF"

.github/workflows/ci.yml

+54-5
Original file line numberDiff line numberDiff line change
@@ -54,20 +54,69 @@ jobs:
5454
run: cargo miri test
5555

5656
benches:
57+
name: continuous Benchmarking
5758
# Not required, we can ignore it for the merge queue check.
5859
if: github.event_name != 'merge_group'
5960
runs-on: ubuntu-latest
61+
permissions:
62+
checks: write
6063
steps:
61-
- uses: actions/checkout@v3
64+
- uses: actions/checkout@v4
6265
- name: Install stable toolchain
6366
uses: dtolnay/rust-toolchain@stable
6467
- uses: Swatinem/rust-cache@v2
6568
with:
6669
prefix-key: v0
67-
- name: Build benchmarks with no features
68-
run: cargo bench --verbose --no-run --no-default-features
69-
- name: Build benchmarks with all features
70-
run: cargo bench --verbose --no-run --all-features
70+
71+
# The installed iai-callgrind-runner version must match the
72+
# version of iai-callgrind in the Cargo.toml
73+
- uses: cargo-bins/cargo-binstall@main
74+
- name: Install iai-callgrind-runner
75+
run: |
76+
version=$(cargo metadata --format-version=1 |\
77+
jq '.packages[] | select(.name == "iai-callgrind").version' |\
78+
tr -d '"'
79+
)
80+
cargo binstall --no-confirm iai-callgrind-runner --version $version --force
81+
82+
- uses: bencherdev/bencher@main
83+
- name: Install valgrind
84+
run: sudo apt update && sudo apt install -y valgrind
85+
86+
- name: Track base branch IAI benchmarks
87+
if: github.event_name == 'push'
88+
run: |
89+
bencher run \
90+
--project portgraph \
91+
--token '${{ secrets.BENCHER_API_TOKEN }}' \
92+
--branch main \
93+
--testbed ubuntu-latest \
94+
--threshold-measure instructions \
95+
--threshold-test t_test \
96+
--threshold-max-sample-size 64 \
97+
--threshold-upper-boundary 0.99 \
98+
--thresholds-reset \
99+
--err \
100+
--github-actions '${{ secrets.HUGRBOT_PAT }}' \
101+
--adapter rust_iai_callgrind \
102+
"cargo bench --bench iai_benches"
103+
104+
- name: Track PR IAI benchmarks
105+
if: github.event_name == 'pull_request'
106+
run: |
107+
bencher run \
108+
--project portgraph \
109+
--token '${{ secrets.BENCHER_API_TOKEN }}' \
110+
--branch "${{ github.event.pull_request.head.ref }}" \
111+
--testbed ubuntu-latest \
112+
--start-point "${{ github.event.pull_request.base.ref }}" \
113+
--start-point-clone-thresholds \
114+
--start-point-reset \
115+
--err \
116+
--github-actions '${{ secrets.HUGRBOT_PAT }}' \
117+
--adapter rust_iai_callgrind \
118+
"cargo bench --bench iai_benches"
119+
# --start-point-hash '${{ github.event.pull_request.base.sha }}' \
71120

72121
tests:
73122
runs-on: ubuntu-latest

Cargo.toml

+9-1
Original file line numberDiff line numberDiff line change
@@ -38,11 +38,19 @@ petgraph = ["dep:petgraph"]
3838

3939
[dev-dependencies]
4040
criterion = { version = "0.5.1", features = ["html_reports"] }
41+
iai-callgrind = "0.14.0"
4142
rmp-serde = "1.1.1"
4243
rstest = "0.24.0"
4344
itertools = "0.14.0"
4445
insta = "1.39.0"
4546

4647
[[bench]]
47-
name = "bench_main"
48+
name = "criterion_benches"
4849
harness = false
50+
51+
[[bench]]
52+
name = "iai_benches"
53+
harness = false
54+
55+
[profile.bench]
56+
debug = true

DEVELOPMENT.md

+50-5
Original file line numberDiff line numberDiff line change
@@ -28,17 +28,62 @@ cargo build
2828
cargo test
2929
```
3030

31-
Run the benchmarks with:
31+
Finally, if you have rust nightly installed, you can run `miri` to detect
32+
undefined behaviour in the code.
3233

3334
```bash
34-
cargo bench
35+
cargo +nightly miri test
3536
```
3637

37-
Finally, if you have rust nightly installed, you can run `miri` to detect
38-
undefined behaviour in the code.
38+
## 🏋️ Benchmarking
39+
40+
We use two kinds of benchmarks in this project:
41+
42+
- A wall-clock time benchmark using `criterion`. This measures the time taken to
43+
run a function by running it multiple times.
44+
- A single-shot instruction count / memory hits benchmark using `iai-callgrind`.
45+
This measures the number of instructions executed and the number of cache hits
46+
and misses.
47+
48+
Both tools run the same set of test cases.
49+
50+
When profiling and debugging performance issues, you may also want to use
51+
[samply](https://github.com/mstange/samply) to visualize the see flame graphs of
52+
specific examples.
53+
54+
### Wall-clock time benchmarks
55+
56+
This is the simplest kind of benchmark. To run the, use:
3957

4058
```bash
41-
cargo +nightly miri test
59+
cargo bench --bench criterion_benches
60+
```
61+
62+
### Single-shot benchmarking
63+
64+
These benchmarks are useful when running in noisy environments, in addition to
65+
being faster than criterion. We run these on CI to track historical performance
66+
in [bencher.dev](https://bencher.dev/perf/portgraph).
67+
68+
To run these, you must have [`valgrind`](https://valgrind.org/) installed.
69+
Support for Apple Silicon (M1/M2/...) macs is
70+
[experimental](https://github.com/LouisBrunner/valgrind-macos/issues/56), so you
71+
will need to manually clone and compile the branch. See
72+
[`LouisBrunner/valgrind-macos`](https://github.com/LouisBrunner/valgrind-macos/blob/feature/m1/README)
73+
for instructions.
74+
75+
In addition to `valgrind`, you will need to install `iai-callgrind` runner. The
76+
pre-build binaries are available on
77+
[`cargo binstall`](https://github.com/cargo-bins/cargo-binstall).
78+
79+
```bash
80+
cargo binstall iai-callgrind-runner
81+
```
82+
83+
The benchmarks can then be run with:
84+
85+
```bash
86+
cargo bench --bench iai_benches
4287
```
4388

4489
## 💅 Coding Style

README.md

+3-1
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ portgraph
55
[![crates][]](https://crates.io/crates/portgraph)
66
[![msrv][]](https://github.com/CQCL/portgraph)
77
[![codecov][]](https://codecov.io/gh/CQCL/portgraph)
8+
[![bencher][]](https://bencher.dev/perf/portgraph)
89

910
Data structure library for directed graphs with first-level ports. Includes
1011
secondary data structures for node and port weights, and node hierarchies.
@@ -32,9 +33,10 @@ See [DEVELOPMENT.md](DEVELOPMENT.md) for instructions on setting up the developm
3233
This project is licensed under Apache License, Version 2.0 ([LICENSE][] or http://www.apache.org/licenses/LICENSE-2.0).
3334

3435
[API documentation here]: https://docs.rs/portgraph/
35-
[build_status]: https://github.com/CQCL/portgraph/workflows/Continuous%20integration/badge.svg?branch=main
36+
[build_status]: https://github.com/CQCL/portgraph/actions/workflows/ci.yml/badge.svg
3637
[crates]: https://img.shields.io/crates/v/portgraph
3738
[LICENSE]: LICENCE
3839
[msrv]: https://img.shields.io/badge/rust-1.75.0%2B-blue.svg?maxAge=3600
3940
[codecov]: https://img.shields.io/codecov/c/gh/CQCL/portgraph?logo=codecov
41+
[bencher]: https://img.shields.io/badge/bencher-.dev-blue.svg?logo=data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAMAAABEpIrGAAAAtFBMVEVHcEwEBAT+/v4AAADi4uKenp7u7/ABAQEAAAAAAAAAAAAwMDAAAAAAAAAAAACGh4e2tLJfX18AAAAAAAD///++wMEAAABoaGiRkZEAAABucXMAAAAAAAB1dXUxMTHMzMzh4eH/0rRJSUmsrKz29vb////+q3v8bQD9+PP/8+kZGhp9fn/8dAD/3cXk4OT/tIL9p2z8ZwD+vI+UlJTPmnn/5tT/xqH/2cD9k1D9kkL8ijj8fhszkVn/AAAAJXRSTlMA4fgz7ePq/o3zRv0hfcfb7dJoVubyDvvpqfjTt+Dl+tTz3uPjxce2cgAAAaJJREFUOI2lklmTqjAQhcMSOpFFQHB3nJl7O2wii46z/f//daOOJYkvU3XPA6Q4X53udEPIbxWv1+x2ZsF6ovsLd05n/PJ5wmd07jLVj7mDaFKwCPE4NREdrmbwEOu+MOerKQnmZtHXGBpDfwp28dV9YgKWxf/iZ/dV2OV0CLjO227X1cgnE451t9u9Oa43jChtCWQ9vsbxK/aZBOxS6cFI848sE7jx/Q2KLPvIU6UHwiJszsCYsfEZaDBS72nxJO/uQJcn3FIHEVBsuxvQtUgDbZIj1ym+BUZBEKH4Lhx3pM+aUzy1GBlGhO0JqaH7ZARy2Dc58BBAGDzfgWdguu+7gwAZoW/T4jYqsrUiK4qa1DanpQw4VObVM6uDjCiHy4pnJqYA1RWoAFI0Z/HwChW+HwEOV+AAcHzHiilA3TbH8FY/PDZtrQD+8kWIppBecXk0Qrws/QHglfZJ/oeIuRC5fNX9SW2SGBTPRo77/c9hrG7D4uml+l7qctNQ/x9GsEnuU0qWj9vyFvwp3NpS2z9PsPB0/1zGDziUJfCV/7js/9A/nA48HxVN/KwAAAAASUVORK5CYII=
4042
[CHANGELOG]: CHANGELOG.md

benches/bench_main.rs

-12
This file was deleted.

benches/benchmarks/convex.rs

+85-47
Original file line numberDiff line numberDiff line change
@@ -1,68 +1,106 @@
1-
use criterion::{black_box, criterion_group, AxisScale, BenchmarkId, Criterion, PlotConfiguration};
1+
use criterion::{criterion_group, Criterion};
22
use itertools::Itertools;
33
use portgraph::{algorithms::TopoConvexChecker, PortView};
4+
use portgraph::{NodeIndex, PortGraph};
45

5-
use super::generators::make_two_track_dag;
6-
7-
fn bench_convex_construction(c: &mut Criterion) {
8-
let mut g = c.benchmark_group("initialize convex checker object");
9-
g.plot_config(PlotConfiguration::default().summary_scale(AxisScale::Logarithmic));
10-
11-
for size in [100, 1_000, 10_000] {
12-
g.bench_with_input(
13-
BenchmarkId::new("initalize_convexity", size),
14-
&size,
15-
|b, size| {
16-
let graph = make_two_track_dag(*size);
17-
b.iter(|| black_box(TopoConvexChecker::new(&graph)))
18-
},
19-
);
6+
use crate::helpers::*;
7+
8+
// -----------------------------------------------------------------------------
9+
// Benchmark functions
10+
// -----------------------------------------------------------------------------
11+
12+
struct ConvexConstruction {
13+
graph: PortGraph,
14+
}
15+
impl SizedBenchmark for ConvexConstruction {
16+
fn name() -> &'static str {
17+
"initialize_convexity"
18+
}
19+
20+
fn setup(size: usize) -> Self {
21+
let graph = make_two_track_dag(size);
22+
Self { graph }
23+
}
24+
25+
fn run(&self) -> impl Sized {
26+
TopoConvexChecker::new(&self.graph)
2027
}
21-
g.finish();
2228
}
2329

2430
/// We benchmark the worst case scenario, where the "subgraph" is the
2531
/// entire graph itself.
26-
fn bench_convex_full(c: &mut Criterion) {
27-
let mut g = c.benchmark_group("Runtime convexity check. Full graph.");
28-
g.plot_config(PlotConfiguration::default().summary_scale(AxisScale::Logarithmic));
32+
struct ConvexFull {
33+
checker: TopoConvexChecker<PortGraph>,
34+
nodes: Vec<NodeIndex>,
35+
}
36+
impl SizedBenchmark for ConvexFull {
37+
fn name() -> &'static str {
38+
"check_convexity_full"
39+
}
2940

30-
for size in [100, 1_000, 10_000] {
41+
fn setup(size: usize) -> Self {
3142
let graph = make_two_track_dag(size);
32-
let checker = TopoConvexChecker::new(&graph);
33-
g.bench_with_input(
34-
BenchmarkId::new("check_convexity_full", size),
35-
&size,
36-
|b, _size| b.iter(|| black_box(checker.is_node_convex(graph.nodes_iter()))),
37-
);
43+
let nodes = graph.nodes_iter().collect_vec();
44+
let checker = TopoConvexChecker::new(graph);
45+
Self { checker, nodes }
46+
}
47+
48+
fn run(&self) -> impl Sized {
49+
self.checker.is_node_convex(self.nodes.iter().copied())
3850
}
39-
g.finish();
4051
}
4152

4253
/// We benchmark the an scenario where the size of the "subgraph" is sub-linear on the size of the graph.
43-
fn bench_convex_sparse(c: &mut Criterion) {
44-
let mut g = c.benchmark_group("Runtime convexity check. Sparse subgraph on an n^2 size graph.");
45-
g.plot_config(PlotConfiguration::default().summary_scale(AxisScale::Logarithmic));
46-
47-
for size in [100usize, 1_000, 5_000] {
48-
let graph_size = size.pow(2);
49-
let graph = make_two_track_dag(graph_size);
50-
let checker = TopoConvexChecker::new(&graph);
51-
let nodes = graph.nodes_iter().step_by(graph_size / size).collect_vec();
52-
g.bench_with_input(
53-
BenchmarkId::new("check_convexity_sparse", size),
54-
&size,
55-
|b, _size| b.iter(|| black_box(checker.is_node_convex(nodes.iter().copied()))),
56-
);
54+
struct ConvexSparse {
55+
checker: TopoConvexChecker<PortGraph>,
56+
nodes: Vec<NodeIndex>,
57+
}
58+
impl SizedBenchmark for ConvexSparse {
59+
fn name() -> &'static str {
60+
"check_convexity_sparse"
61+
}
62+
63+
fn setup(size: usize) -> Self {
64+
let graph = make_two_track_dag(size);
65+
let subgraph_size = (size as f64).sqrt().floor() as usize;
66+
let nodes = graph
67+
.nodes_iter()
68+
.step_by(size / subgraph_size)
69+
.collect_vec();
70+
let checker = TopoConvexChecker::new(graph);
71+
Self { checker, nodes }
72+
}
73+
74+
fn run(&self) -> impl Sized {
75+
self.checker.is_node_convex(self.nodes.iter().copied())
5776
}
58-
g.finish();
5977
}
6078

79+
// -----------------------------------------------------------------------------
80+
// iai_callgrind definitions
81+
// -----------------------------------------------------------------------------
82+
83+
sized_iai_benchmark!(callgrind_convex_construction, ConvexConstruction);
84+
sized_iai_benchmark!(callgrind_convex_full, ConvexFull);
85+
sized_iai_benchmark!(callgrind_convex_sparse, ConvexSparse);
86+
87+
iai_callgrind::library_benchmark_group!(
88+
name = callgrind_group;
89+
benchmarks =
90+
callgrind_convex_construction,
91+
callgrind_convex_full,
92+
callgrind_convex_sparse,
93+
);
94+
95+
// -----------------------------------------------------------------------------
96+
// Criterion definitions
97+
// -----------------------------------------------------------------------------
98+
6199
criterion_group! {
62-
name = benches;
100+
name = criterion_group;
63101
config = Criterion::default();
64102
targets =
65-
bench_convex_full,
66-
bench_convex_sparse,
67-
bench_convex_construction
103+
ConvexConstruction::criterion,
104+
ConvexFull::criterion,
105+
ConvexSparse::criterion,
68106
}

0 commit comments

Comments
 (0)