Skip to content

Commit 2a4cd4d

Browse files
ci: deserialize cargo bench result (#9624)
1 parent e43e319 commit 2a4cd4d

File tree

10 files changed

+262
-0
lines changed

10 files changed

+262
-0
lines changed

Cargo.lock

Lines changed: 6 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

crates/bench_tools/Cargo.toml

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,3 +8,16 @@ workspace = true
88

99
[dependencies]
1010
clap = { workspace = true, features = ["derive"] }
11+
criterion.workspace = true
12+
serde = { workspace = true, features = ["derive"] }
13+
14+
[dev-dependencies]
15+
apollo_infra_utils.workspace = true
16+
glob.workspace = true
17+
rstest.workspace = true
18+
serde_json.workspace = true
19+
20+
[[bench]]
21+
harness = false
22+
name = "dummy_bench"
23+
path = "src/benches/dummy_bench.rs"
Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
{
2+
"mean": {
3+
"confidence_interval": {
4+
"confidence_level": 0.95,
5+
"lower_bound": 0.4627450463077952,
6+
"upper_bound": 0.4672721018359347
7+
},
8+
"point_estimate": 0.464843138908376,
9+
"standard_error": 0.001159579008842323
10+
},
11+
"median": {
12+
"confidence_interval": {
13+
"confidence_level": 0.95,
14+
"lower_bound": 0.46018939259554725,
15+
"upper_bound": 0.4630457711597157
16+
},
17+
"point_estimate": 0.46110773204298744,
18+
"standard_error": 0.0007700792406655315
19+
},
20+
"median_abs_dev": {
21+
"confidence_interval": {
22+
"confidence_level": 0.95,
23+
"lower_bound": 0.004389190374631315,
24+
"upper_bound": 0.008068153741443406
25+
},
26+
"point_estimate": 0.0057180067999875826,
27+
"standard_error": 0.0008869433017250109
28+
},
29+
"slope": {
30+
"confidence_interval": {
31+
"confidence_level": 0.95,
32+
"lower_bound": 0.4613537171491014,
33+
"upper_bound": 0.4679905568783797
34+
},
35+
"point_estimate": 0.4641046136274776,
36+
"standard_error": 0.001730736547763549
37+
},
38+
"std_dev": {
39+
"confidence_interval": {
40+
"confidence_level": 0.95,
41+
"lower_bound": 0.007242933241527838,
42+
"upper_bound": 0.01557482754536567
43+
},
44+
"point_estimate": 0.01165848802634422,
45+
"standard_error": 0.00216291128423961
46+
}
47+
}
Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
{
2+
"mean": {
3+
"confidence_interval": {
4+
"confidence_level": 0.95,
5+
"lower_bound": 0.45957162353881537,
6+
"upper_bound": 0.47252808549400704
7+
},
8+
"point_estimate": 0.4656174818033215,
9+
"standard_error": 0.0033071130641006297
10+
},
11+
"median": {
12+
"confidence_interval": {
13+
"confidence_level": 0.95,
14+
"lower_bound": 0.45151445843970833,
15+
"upper_bound": 0.4538904649825635
16+
},
17+
"point_estimate": 0.4526323188039608,
18+
"standard_error": 0.0006992971024409143
19+
},
20+
"median_abs_dev": {
21+
"confidence_interval": {
22+
"confidence_level": 0.95,
23+
"lower_bound": 0.005240482428601258,
24+
"upper_bound": 0.010332193401481504
25+
},
26+
"point_estimate": 0.007727865411134973,
27+
"standard_error": 0.001301158952978215
28+
},
29+
"slope": {
30+
"confidence_interval": {
31+
"confidence_level": 0.95,
32+
"lower_bound": 0.455400693078925,
33+
"upper_bound": 0.46369875851255066
34+
},
35+
"point_estimate": 0.45933814542565166,
36+
"standard_error": 0.002119888457921763
37+
},
38+
"std_dev": {
39+
"confidence_interval": {
40+
"confidence_level": 0.95,
41+
"lower_bound": 0.02332892198500483,
42+
"upper_bound": 0.041110549300589966
43+
},
44+
"point_estimate": 0.03317521516535013,
45+
"standard_error": 0.004530135795169152
46+
}
47+
}

crates/bench_tools/src/benches.rs

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
#[cfg(test)]
2+
pub(crate) mod dummy_bench;
Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
use std::hint::black_box;
2+
3+
use criterion::{criterion_group, criterion_main, Criterion};
4+
5+
#[allow(dead_code)]
6+
fn dummy_function(n: u64) -> u64 {
7+
// Simple function that does some work
8+
(0..n).sum()
9+
}
10+
11+
/// Example benchmark function that demonstrates how to use Criterion for benchmarking.
12+
/// This is used to test the benchmarking infrastructure and generate sample benchmark results
13+
/// that can be parsed by the bench_tools framework.
14+
#[allow(dead_code)]
15+
fn dummy_benchmark(c: &mut Criterion) {
16+
// black_box prevents the compiler from optimizing away the function call during benchmarking
17+
c.bench_function("dummy_sum_100", |b| b.iter(|| black_box(dummy_function(100))));
18+
19+
c.bench_function("dummy_sum_1000", |b| b.iter(|| black_box(dummy_function(1000))));
20+
}
21+
22+
criterion_group!(benches, dummy_benchmark);
23+
criterion_main!(benches);

crates/bench_tools/src/lib.rs

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
#[cfg(test)]
2+
pub(crate) mod benches;
3+
pub mod types;

crates/bench_tools/src/types.rs

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
pub mod estimates;
2+
#[cfg(test)]
3+
mod estimates_test;
Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
use serde::Deserialize;
2+
3+
/// Criterion benchmark estimates.
4+
#[derive(Debug, Deserialize)]
5+
#[allow(dead_code)]
6+
pub struct Estimates {
7+
pub mean: Stat,
8+
pub median: Stat,
9+
pub std_dev: Stat,
10+
pub median_abs_dev: Stat,
11+
pub slope: Option<Stat>,
12+
}
13+
14+
/// Statistical estimate with confidence interval.
15+
#[derive(Debug, Deserialize)]
16+
#[allow(dead_code)]
17+
pub struct Stat {
18+
pub point_estimate: f64,
19+
pub standard_error: f64,
20+
pub confidence_interval: ConfidenceInterval,
21+
}
22+
23+
/// Confidence interval bounds.
24+
#[derive(Debug, Deserialize)]
25+
#[allow(dead_code)]
26+
pub struct ConfidenceInterval {
27+
pub confidence_level: f64,
28+
pub lower_bound: f64,
29+
pub upper_bound: f64,
30+
}
Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,88 @@
1+
use std::fs;
2+
use std::path::{Path, PathBuf};
3+
use std::process::Command;
4+
5+
use apollo_infra_utils::path::project_path;
6+
use rstest::{fixture, rstest};
7+
8+
use crate::types::estimates::Estimates;
9+
10+
/// Returns the bench_tools crate directory.
11+
#[fixture]
12+
fn bench_tools_crate_dir() -> PathBuf {
13+
std::env::var("CARGO_MANIFEST_DIR")
14+
.map(PathBuf::from)
15+
.unwrap_or_else(|_| std::env::current_dir().unwrap())
16+
}
17+
18+
/// Returns the directory where dummy benchmark estimate results are stored.
19+
#[fixture]
20+
fn dummy_bench_results_dir(bench_tools_crate_dir: PathBuf) -> PathBuf {
21+
bench_tools_crate_dir.join("data/dummy_benches_result")
22+
}
23+
24+
/// Returns the workspace root directory.
25+
#[fixture]
26+
fn workspace_root() -> PathBuf {
27+
project_path().expect("Failed to get project path")
28+
}
29+
30+
/// Returns the list of dummy benchmark names.
31+
#[fixture]
32+
fn dummy_bench_names() -> &'static [&'static str] {
33+
&["dummy_sum_100", "dummy_sum_1000"]
34+
}
35+
36+
/// Helper function to deserialize dummy bench estimates JSON files in a directory.
37+
fn assert_deserialize_dummy_bench_estimates(results_dir: &Path, bench_names: &[&str]) {
38+
for bench_name in bench_names {
39+
let path = results_dir.join(format!("{}_estimates.json", bench_name));
40+
let data = fs::read_to_string(&path)
41+
.unwrap_or_else(|e| panic!("Failed to read {}: {}", path.display(), e));
42+
43+
let _est: Estimates = serde_json::from_str(&data).unwrap_or_else(|e| {
44+
panic!("Failed to deserialize {}: {}\nContent: {}", path.display(), e, data)
45+
});
46+
}
47+
}
48+
49+
#[rstest]
50+
#[ignore]
51+
/// Run dummy benchmark and deserialize the results.
52+
fn run_dummy_bench_and_deserialize_estimates(
53+
workspace_root: PathBuf,
54+
dummy_bench_results_dir: PathBuf,
55+
dummy_bench_names: &[&str],
56+
) {
57+
// 1) Run dummy benchmark.
58+
let status = Command::new("cargo")
59+
.args(["bench", "-p", "bench_tools", "--bench", "dummy_bench"])
60+
.status()
61+
.expect("Failed to spawn `cargo bench`");
62+
assert!(status.success(), "`cargo bench` did not exit successfully");
63+
64+
// 2) Collect and save dummy_bench estimates.json files
65+
fs::create_dir_all(&dummy_bench_results_dir).expect("Failed to create results directory");
66+
67+
for bench_name in dummy_bench_names {
68+
let source_path =
69+
workspace_root.join("target/criterion").join(bench_name).join("new/estimates.json");
70+
let dest_path = dummy_bench_results_dir.join(format!("{}_estimates.json", bench_name));
71+
72+
// Read, parse, and write the result to the results directory.
73+
let data = fs::read_to_string(&source_path)
74+
.unwrap_or_else(|e| panic!("Failed to read {}: {}", source_path.display(), e));
75+
let json: serde_json::Value = serde_json::from_str(&data).expect("Failed to parse JSON");
76+
let pretty_json = serde_json::to_string_pretty(&json).expect("Failed to serialize JSON");
77+
fs::write(&dest_path, pretty_json).expect("Failed to write benchmark result");
78+
}
79+
80+
// 3) Deserialize and validate the saved results
81+
assert_deserialize_dummy_bench_estimates(&dummy_bench_results_dir, dummy_bench_names);
82+
}
83+
84+
#[rstest]
85+
/// Test that Estimates can be deserialized from the saved results.
86+
fn deserialize_dummy_bench_estimates(dummy_bench_results_dir: PathBuf, dummy_bench_names: &[&str]) {
87+
assert_deserialize_dummy_bench_estimates(&dummy_bench_results_dir, dummy_bench_names);
88+
}

0 commit comments

Comments
 (0)