ci: deserialize cargo bench result

Author: AvivYossef-starkware

Cargo.lock

@@ -8,3 +8,14 @@ workspace = true
 
 [dependencies]
 clap = { workspace = true, features = ["derive"] }
+criterion.workspace = true
+serde = { workspace = true, features = ["derive"] }
+
+[dev-dependencies]
+glob.workspace = true
+serde_json.workspace = true
+
+[[bench]]
+harness = false
+name = "dummy_bench"
+path = "src/benches/dummy_bench.rs"

bench_tools/Cargo.toml

@@ -0,0 +1,47 @@
+{
+  "mean": {
+    "confidence_interval": {
+      "confidence_level": 0.95,
+      "lower_bound": 0.9280511380687992,
+      "upper_bound": 0.9770326008358003
+    },
+    "point_estimate": 0.950567887590271,
+    "standard_error": 0.012543975607897394
+  },
+  "median": {
+    "confidence_interval": {
+      "confidence_level": 0.95,
+      "lower_bound": 0.8938102469877969,
+      "upper_bound": 0.9104509947667936
+    },
+    "point_estimate": 0.9035953146446528,
+    "standard_error": 0.004791521022015121
+  },
+  "median_abs_dev": {
+    "confidence_interval": {
+      "confidence_level": 0.95,
+      "lower_bound": 0.020115181011407187,
+      "upper_bound": 0.046529428364116145
+    },
+    "point_estimate": 0.035187421707573024,
+    "standard_error": 0.007212517355406062
+  },
+  "slope": {
+    "confidence_interval": {
+      "confidence_level": 0.95,
+      "lower_bound": 0.9570826102260792,
+      "upper_bound": 1.054776136887248
+    },
+    "point_estimate": 1.003565137518582,
+    "standard_error": 0.02512230790903042
+  },
+  "std_dev": {
+    "confidence_interval": {
+      "confidence_level": 0.95,
+      "lower_bound": 0.07004563729649199,
+      "upper_bound": 0.16991881743641407
+    },
+    "point_estimate": 0.12590223948952453,
+    "standard_error": 0.024908952526773723
+  }
+}

bench_tools/data/dummy_benches_result/dummy_sum_1000_estimates.json

@@ -0,0 +1,47 @@
+{
+  "mean": {
+    "confidence_interval": {
+      "confidence_level": 0.95,
+      "lower_bound": 0.9142287240580292,
+      "upper_bound": 0.9462306685839782
+    },
+    "point_estimate": 0.9285105039182452,
+    "standard_error": 0.008165206850548515
+  },
+  "median": {
+    "confidence_interval": {
+      "confidence_level": 0.95,
+      "lower_bound": 0.893664924087692,
+      "upper_bound": 0.9134582623157296
+    },
+    "point_estimate": 0.9007806648132172,
+    "standard_error": 0.005255178148246822
+  },
+  "median_abs_dev": {
+    "confidence_interval": {
+      "confidence_level": 0.95,
+      "lower_bound": 0.014930366068731485,
+      "upper_bound": 0.03712113624827432
+    },
+    "point_estimate": 0.02469204064276661,
+    "standard_error": 0.006102570427001133
+  },
+  "slope": {
+    "confidence_interval": {
+      "confidence_level": 0.95,
+      "lower_bound": 0.9148087731810076,
+      "upper_bound": 0.9545184104032152
+    },
+    "point_estimate": 0.9316599097782416,
+    "standard_error": 0.010133884941484783
+  },
+  "std_dev": {
+    "confidence_interval": {
+      "confidence_level": 0.95,
+      "lower_bound": 0.04309489165021185,
+      "upper_bound": 0.1199166738579459
+    },
+    "point_estimate": 0.08199702437151206,
+    "standard_error": 0.020739973590370026
+  }
+}

bench_tools/data/dummy_benches_result/dummy_sum_100_estimates.json

@@ -0,0 +1,2 @@
+#[cfg(test)]
+pub(crate) mod dummy_bench;

bench_tools/src/benches.rs

@@ -0,0 +1,17 @@
+use criterion::{black_box, criterion_group, criterion_main, Criterion};
+
+#[allow(dead_code)]
+fn dummy_function(n: u64) -> u64 {
+    // Simple function that does some work
+    (0..n).sum()
+}
+
+#[allow(dead_code)]
+fn dummy_benchmark(c: &mut Criterion) {
+    c.bench_function("dummy_sum_100", |b| b.iter(|| dummy_function(black_box(100))));
+
+    c.bench_function("dummy_sum_1000", |b| b.iter(|| dummy_function(black_box(1000))));
+}
+
+criterion_group!(benches, dummy_benchmark);
+criterion_main!(benches);

bench_tools/src/benches/dummy_bench.rs

@@ -0,0 +1,3 @@
+#[cfg(test)]
+pub(crate) mod benches;
+pub mod types;

bench_tools/src/lib.rs

@@ -0,0 +1,3 @@
+pub mod estimates;
+#[cfg(test)]
+mod estimates_test;

bench_tools/src/types.rs

@@ -0,0 +1,30 @@
+use serde::Deserialize;
+
+/// Criterion benchmark estimates.
+#[derive(Debug, Deserialize)]
+#[allow(dead_code)]
+pub struct Estimates {
+    pub mean: Stat,
+    pub median: Stat,
+    pub std_dev: Stat,
+    pub median_abs_dev: Stat,
+    pub slope: Option<Stat>,
+}
+
+/// Statistical estimate with confidence interval.
+#[derive(Debug, Deserialize)]
+#[allow(dead_code)]
+pub struct Stat {
+    pub point_estimate: f64,
+    pub standard_error: f64,
+    pub confidence_interval: ConfidenceInterval,
+}
+
+/// Confidence interval bounds.
+#[derive(Debug, Deserialize)]
+#[allow(dead_code)]
+pub struct ConfidenceInterval {
+    pub confidence_level: f64,
+    pub lower_bound: f64,
+    pub upper_bound: f64,
+}

bench_tools/src/types/estimates.rs

@@ -0,0 +1,100 @@
+use std::fs;
+use std::path::PathBuf;
+use std::process::Command;
+
+use glob::glob;
+
+use crate::types::estimates::Estimates;
+
+#[test]
+#[ignore]
+/// Run dummy benchmark and deserialize the results.
+fn run_dummy_bench_and_deserialize_all_estimates() {
+    let workspace_root = std::env::var("CARGO_MANIFEST_DIR")
+        .map(PathBuf::from)
+        .unwrap_or_else(|_| std::env::current_dir().unwrap())
+        .parent()
+        .unwrap()
+        .to_path_buf();
+
+    let data_dir = workspace_root.join("bench_tools/data/dummy_benches_result");
+
+    // 1) Run dummy benchmark.
+    let status = Command::new("cargo")
+        .args(["bench", "-p", "bench_tools", "--bench", "dummy_bench"])
+        .status()
+        .expect("Failed to spawn `cargo bench`");
+    assert!(status.success(), "`cargo bench` did not exit successfully");
+
+    // 2) Collect ALL estimates.json files under target/criterion/**/new/
+    let patterns =
+        vec!["target/criterion/**/new/estimates.json", "../target/criterion/**/new/estimates.json"];
+
+    let mut files: Vec<PathBuf> = Vec::new();
+    for pattern in &patterns {
+        for path in
+            glob(pattern).unwrap_or_else(|_| panic!("Invalid glob pattern: {}", pattern)).flatten()
+        {
+            files.push(path);
+        }
+    }
+    assert!(!files.is_empty(), "No Criterion results found; did any benches run?");
+
+    // 3) Save results to bench_tools/data.
+    fs::create_dir_all(&data_dir).expect("Failed to create data directory");
+    for path in &files {
+        if let Some(filename) = path.file_name() {
+            let bench_name = path
+                .parent()
+                .and_then(|p| p.parent())
+                .and_then(|p| p.file_name())
+                .and_then(|n| n.to_str())
+                .unwrap_or("unknown");
+            let dest = data_dir.join(format!("{}_{}", bench_name, filename.to_str().unwrap()));
+
+            // Read, parse, and write the result to the data directory.
+            let data = fs::read_to_string(path).expect("Failed to read benchmark result");
+            let json: serde_json::Value =
+                serde_json::from_str(&data).expect("Failed to parse JSON");
+            let pretty_json =
+                serde_json::to_string_pretty(&json).expect("Failed to serialize JSON");
+            fs::write(&dest, pretty_json).expect("Failed to write benchmark result");
+        }
+    }
+
+    // 4) Deserialize and validate the saved results
+    deserialize_estimates();
+}
+
+#[test]
+/// Test that Estimates can be deserialized from the saved results.
+fn deserialize_estimates() {
+    let workspace_root = std::env::var("CARGO_MANIFEST_DIR")
+        .map(PathBuf::from)
+        .unwrap_or_else(|_| std::env::current_dir().unwrap())
+        .parent()
+        .unwrap()
+        .to_path_buf();
+
+    let data_dir = workspace_root.join("bench_tools/data/dummy_benches_result");
+
+    // Collect all JSON files in the data directory
+    let pattern = data_dir.join("*.json");
+    let mut files: Vec<PathBuf> = Vec::new();
+
+    for path in glob(pattern.to_str().unwrap()).expect("Invalid glob pattern").flatten() {
+        files.push(path);
+    }
+
+    assert!(!files.is_empty(), "No JSON files found in {}", data_dir.display());
+
+    // Deserialize each file in the data directory.
+    for path in &files {
+        let data = fs::read_to_string(path)
+            .unwrap_or_else(|e| panic!("Failed to read {}: {}", path.display(), e));
+
+        let _est: Estimates = serde_json::from_str(&data).unwrap_or_else(|e| {
+            panic!("Failed to deserialize {}: {}\nContent: {}", path.display(), e, data)
+        });
+    }
+}