diff --git a/bench/runner/src/fairness.rs b/bench/runner/src/fairness.rs new file mode 100644 index 00000000..0aa24c1c --- /dev/null +++ b/bench/runner/src/fairness.rs @@ -0,0 +1,422 @@ +//! Two-axis fairness: identical *within* a family, declared *across* families. +//! +//! These are two different meanings of the word and blurring them produces a +//! table that looks comparable and is not: +//! +//! - **Within a database family** every target must declare the *same* harness — +//! same worker count, same pool size, same engine tuning. Only then is a +//! difference in the numbers attributable to the library rather than to the +//! configuration it was handed. Drift is a hard error, not a warning: a +//! silently unequal library comparison is worse than no comparison, and a +//! warning buried in the manifest is silent in every way that matters. +//! - **Across families** nothing is constrained. PostgreSQL over TCP with a +//! pool of 8 and an embedded SQLite with a pool of 1 *should* differ; the +//! difference is the stack comparison. What matters is that the configuration +//! is recorded per family so no reader mistakes it for a library difference, +//! which is what [`harness`] writes into the manifest. +//! +//! A "family" is a **comparison group**: the set of targets claiming to be +//! directly comparable. It usually maps onto the database engine, but it splits +//! where the harness cannot honestly be equalised. `sqlite-ts` is separate from +//! `sqlite` because `bun:sqlite` is a synchronous API on a single-threaded +//! runtime — a pool of 8 there is theatre, and forcing one to match the Rust +//! targets would cripple it in the name of fairness. drizzle-rs on rusqlite +//! versus drizzle-orm on Bun differs in language, runtime and concurrency model, +//! which makes it a *stack* comparison and therefore the across-family axis; +//! inside `sqlite-ts`, drizzle-orm versus bun:sqlite is a real library +//! comparison. Splitting a group changes enforcement and delta scoping only — +//! both groups still appear in one table. +//! +//! Families come from `fair.family`, which each target declares. It is not +//! inferred: `db.profile` separates configurations *inside* a group (prepared +//! vs unprepared) and `fair.db` names the SQL dialect shared by several engines, +//! so neither identifies the bracket a target competes in. It is also not taken +//! from the spec file a target arrived in — publish-class runs already execute +//! several PostgreSQL spec files back to back inside one job. + +use crate::code::{Code, Fail}; +use crate::model::{HarnessDoc, Target}; +use std::collections::BTreeMap; + +/// Enforce within-family harness identity and describe each family's harness. +/// +/// # Errors +/// +/// Fails when two targets in the same family declare different `fair.workers`, +/// `fair.pool`, or `fair.tuning`. The message names both targets and the field, +/// because the fix is always to change one of them. +pub fn harness(targets: &[Target]) -> Result, Fail> { + let mut families: BTreeMap<&str, Vec<&Target>> = BTreeMap::new(); + for target in targets { + families + .entry(target.fair.family.as_str()) + .or_default() + .push(target); + } + + families + .into_iter() + .map(|(family, members)| family_harness(family, &members)) + .collect() +} + +fn family_harness(family: &str, members: &[&Target]) -> Result { + let (exempt, compared): (Vec<&Target>, Vec<&Target>) = + members.iter().partition(|target| is_exempt(target)); + + let Some(reference) = compared.first() else { + // Every member opted out of the comparison, so there is no harness to + // enforce and none is claimed. + return Ok(HarnessDoc { + family: family.to_string(), + targets: Vec::new(), + workers: None, + pool: None, + tuning: None, + within_family_identical: false, + exempt: ids(&exempt), + }); + }; + + for target in compared.iter().skip(1) { + check(family, reference, target, "workers", |t| { + t.fair.workers.to_string() + })?; + check(family, reference, target, "pool", |t| { + t.fair.pool.to_string() + })?; + check(family, reference, target, "tuning", |t| { + t.fair.tuning.clone() + })?; + } + + Ok(HarnessDoc { + family: family.to_string(), + targets: ids(&compared), + workers: Some(reference.fair.workers), + pool: Some(reference.fair.pool), + tuning: Some(reference.fair.tuning.clone()), + within_family_identical: true, + exempt: ids(&exempt), + }) +} + +fn check( + family: &str, + reference: &Target, + target: &Target, + field: &str, + value: impl Fn(&Target) -> String, +) -> Result<(), Fail> { + let (found, expected) = (value(target), value(reference)); + if found == expected { + return Ok(()); + } + Err(Fail::new( + Code::InvalidInput, + format!( + "unfair {family} comparison: target {} declares fair.{field}={found} \ + but target {} declares fair.{field}={expected}. Targets in the same \ + database family must run an identical harness, otherwise the ranking \ + compares configurations instead of libraries. Change one of them, or \ + move it to its own family.", + target.id, reference.id + ), + )) +} + +/// A target serving from a replicated in-process cache has no connection pool +/// to equalise, so comparing its pool size against a SQL client's is meaningless. +/// It is listed in the manifest rather than dropped. +fn is_exempt(target: &Target) -> bool { + matches!(target.data_access.as_deref(), Some("in-process-cache")) +} + +fn ids(targets: &[&Target]) -> Vec { + targets.iter().map(|target| target.id.clone()).collect() +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::model::{ + Contract, Db, DisplayMeta, Driver, Exec, Fair, NameVer, Pool, Proc, Target, Wire, + }; + + fn target(id: &str, family: &str, workers: u32, pool: u32, tuning: &str) -> Target { + Target { + version: "v1".to_string(), + id: id.to_string(), + display: DisplayMeta { + name: id.to_string(), + description: None, + }, + lang: "rust".to_string(), + group: None, + data_access: None, + sql_variant: None, + runtime: NameVer { + name: "rust".to_string(), + ver: "1.95.0".to_string(), + }, + orm: NameVer { + name: "drizzle-rs".to_string(), + ver: "0.1.15".to_string(), + }, + driver: Driver { + name: "rusqlite".to_string(), + ver: "0.39.0".to_string(), + transport: None, + }, + proc: Proc { + mode: "single".to_string(), + workers, + }, + pool: Pool { + max: pool, + min: None, + acquire_ms: None, + }, + db: Db { + profile: "sqlite".to_string(), + hash: "sha256:0".to_string(), + prepared: None, + }, + wire: Wire { + format: "json".to_string(), + }, + fair: Fair { + family: family.to_string(), + workers, + pool, + db: "sqlite".to_string(), + schema: "sha256:0".to_string(), + contract: "v1".to_string(), + tuning: tuning.to_string(), + }, + contract: Contract { + ver: "v1".to_string(), + }, + parity: exec(), + warmup: None, + load: exec(), + server: None, + } + } + + fn exec() -> Exec { + Exec { + cmd: vec!["true".to_string()], + cwd: None, + env: Default::default(), + timeout_s: None, + } + } + + #[test] + fn a_family_running_one_harness_is_recorded_as_verified() { + let targets = [ + target("drizzle-rs-sqlite", "sqlite", 1, 8, "WAL"), + target("rusqlite-prepared", "sqlite", 1, 8, "WAL"), + ]; + let blocks = harness(&targets).expect("identical harness"); + + assert_eq!(blocks.len(), 1); + assert_eq!(blocks[0].family, "sqlite"); + assert_eq!(blocks[0].workers, Some(1)); + assert_eq!(blocks[0].pool, Some(8)); + assert_eq!(blocks[0].tuning.as_deref(), Some("WAL")); + assert!(blocks[0].within_family_identical); + assert_eq!( + blocks[0].targets, + ["drizzle-rs-sqlite", "rusqlite-prepared"] + ); + } + + #[test] + fn pool_drift_inside_a_family_fails_the_run() { + let targets = [ + target("drizzle-rs-sqlite", "sqlite", 1, 8, "WAL"), + target("bun-sqlite", "sqlite", 1, 1, "WAL"), + ]; + let err = harness(&targets).expect_err("pool drift must fail"); + + assert_eq!(err.code, Code::InvalidInput); + assert!(err.msg.contains("unfair sqlite comparison"), "{}", err.msg); + assert!(err.msg.contains("bun-sqlite"), "{}", err.msg); + assert!(err.msg.contains("drizzle-rs-sqlite"), "{}", err.msg); + assert!(err.msg.contains("fair.pool=1"), "{}", err.msg); + assert!(err.msg.contains("fair.pool=8"), "{}", err.msg); + } + + #[test] + fn worker_and_tuning_drift_fail_too() { + let workers = [ + target("a", "postgres", 1, 8, "stock"), + target("b", "postgres", 4, 8, "stock"), + ]; + let err = harness(&workers).expect_err("worker drift"); + assert!(err.msg.contains("fair.workers=4"), "{}", err.msg); + + let tuning = [ + target("a", "postgres", 1, 8, "stock postgres:18-alpine"), + target("b", "postgres", 1, 8, "shared_buffers=4GB"), + ]; + let err = harness(&tuning).expect_err("tuning drift"); + assert!( + err.msg.contains("fair.tuning=shared_buffers=4GB"), + "{}", + err.msg + ); + } + + #[test] + fn different_families_may_differ_freely() { + let targets = [ + target("drizzle-rs-sqlite", "sqlite", 1, 8, "WAL"), + target("drizzle-rs-turso", "turso", 1, 4, "WAL, turso"), + target( + "drizzle-rs-pg", + "postgres", + 1, + 8, + "stock postgres:18-alpine", + ), + ]; + let blocks = harness(&targets).expect("cross-family differences are allowed"); + + assert_eq!(blocks.len(), 3); + // Sorted by family so the manifest is stable across runs. + assert_eq!( + blocks.iter().map(|b| b.family.as_str()).collect::>(), + ["postgres", "sqlite", "turso"] + ); + assert_eq!(blocks[2].pool, Some(4)); + assert!(blocks.iter().all(|b| b.within_family_identical)); + } + + #[test] + fn an_in_process_cache_is_listed_as_exempt_not_dropped() { + let mut cache = target("spacetime-sdk-rs", "spacetimedb", 1, 1, "cache"); + cache.data_access = Some("in-process-cache".to_string()); + let targets = [ + target("spacetime-pgwire-rs", "spacetimedb", 1, 4, "stock"), + cache, + ]; + let blocks = harness(&targets).expect("exempt targets do not force drift"); + + assert_eq!(blocks[0].pool, Some(4)); + assert!(blocks[0].within_family_identical); + assert_eq!(blocks[0].targets, ["spacetime-pgwire-rs"]); + assert_eq!(blocks[0].exempt, ["spacetime-sdk-rs"]); + } + + /// Consumers join `harness[].family` against each target's declared + /// `fair.family`, and `harness[].targets` against the run's target list. + /// Both keys are echoed, never re-derived, so the join cannot go stale. + #[test] + fn every_emitted_key_traces_back_to_a_declared_target() { + let mut cache = target("spacetime-sdk-rs", "spacetimedb", 1, 1, "cache"); + cache.data_access = Some("in-process-cache".to_string()); + let targets = [ + target("drizzle-rs-sqlite", "sqlite", 1, 8, "WAL"), + target("drizzle-rs-pg", "postgres", 1, 8, "stock"), + target("spacetime-pgwire-rs", "spacetimedb", 1, 4, "stock"), + cache, + ]; + let blocks = harness(&targets).expect("harness"); + + for block in &blocks { + assert!( + targets.iter().any(|t| t.fair.family == block.family), + "family {} matches no target", + block.family + ); + for id in block.targets.iter().chain(&block.exempt) { + assert!( + targets.iter().any(|t| &t.id == id), + "harness names {id}, which is not in the run" + ); + } + } + // Every target is accounted for exactly once, compared or exempt. + let named: Vec<&String> = blocks + .iter() + .flat_map(|b| b.targets.iter().chain(&b.exempt)) + .collect(); + assert_eq!(named.len(), targets.len()); + } + + /// Within-family identity is enforced per *run*, but a family can span runs: + /// `targets.postgres.v1.json` and `targets.postgres-rust-orms.v1.json` both + /// declare `family: postgres`, and outside the publish topology they execute + /// as separate CI jobs producing separate artifacts. Each run would then + /// check only its own shard and pass, and the drift would not surface until + /// a consumer merged the two and marked the family unverified — after + /// publish, in someone else's UI. + /// + /// So check the union of every checked-in spec here, where a mismatch fails + /// CI at source instead. + #[test] + fn every_checked_in_spec_agrees_with_its_family_across_files() { + let spec_dir = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .parent() + .and_then(std::path::Path::parent) + .expect("workspace root") + .join("bench") + .join("spec"); + + let mut all: Vec = Vec::new(); + let mut files = 0; + for entry in std::fs::read_dir(&spec_dir).expect("read bench/spec") { + let path = entry.expect("dir entry").path(); + let name = path + .file_name() + .and_then(|n| n.to_str()) + .unwrap_or_default(); + if !name.starts_with("targets.") || !name.ends_with(".json") { + continue; + } + let body = std::fs::read_to_string(&path).expect("read target spec"); + all.extend( + serde_json::from_str::>(&body) + .unwrap_or_else(|err| panic!("{}: {err}", path.display())), + ); + files += 1; + } + assert!(files >= 8, "expected every targets.*.json, found {files}"); + + let blocks = harness(&all).unwrap_or_else(|err| { + panic!( + "checked-in specs declare an unfair comparison, which would only \ + have surfaced after publish in the parallel CI topology: {}", + err.msg + ) + }); + + // The case this exists for: one family, two spec files, one harness. + let postgres = blocks + .iter() + .find(|block| block.family == "postgres") + .expect("postgres family"); + assert!( + postgres.targets.len() > 5, + "postgres should span both Rust spec files, found {:?}", + postgres.targets + ); + assert!(postgres.within_family_identical); + } + + #[test] + fn a_family_of_only_exempt_targets_claims_no_harness() { + let mut cache = target("spacetime-sdk-rs", "spacetimedb", 1, 1, "cache"); + cache.data_access = Some("in-process-cache".to_string()); + let blocks = harness(&[cache]).expect("vacuous check"); + + assert!(!blocks[0].within_family_identical); + assert!(blocks[0].workers.is_none()); + assert!(blocks[0].pool.is_none()); + assert!(blocks[0].tuning.is_none()); + assert_eq!(blocks[0].exempt, ["spacetime-sdk-rs"]); + } +} diff --git a/bench/runner/src/load/mod.rs b/bench/runner/src/load/mod.rs index 55167a07..f72a2f7e 100644 --- a/bench/runner/src/load/mod.rs +++ b/bench/runner/src/load/mod.rs @@ -384,12 +384,15 @@ pub async fn run(args: Load) -> Result { shutdown?; jsonio::write(out, &measured.series, Code::RunFail)?; - // The trial aggregate is computed here, where the raw per-request samples - // still exist: exact hold-phase percentiles cannot be recovered from the - // per-bucket summaries alone. + // The trial aggregate and the per-step aggregates are computed here, where + // the raw per-request samples still exist: exact hold-phase percentiles + // cannot be recovered from the per-bucket summaries alone. if let Ok(point_out) = std::env::var("BENCH_POINT_OUT") { jsonio::write(PathBuf::from(point_out), &measured.aggregate, Code::RunFail)?; } + if let Ok(steps_out) = std::env::var("BENCH_STEPS_OUT") { + jsonio::write(PathBuf::from(steps_out), &measured.steps, Code::RunFail)?; + } Ok(Code::Success) } @@ -519,20 +522,186 @@ impl QueryBucket { } } - fn merge(&mut self, other: QueryBucket) { - self.latencies.extend(other.latencies); + fn merge(&mut self, other: &QueryBucket) { + self.latencies.extend_from_slice(&other.latencies); self.errors += other.errors; self.total += other.total; } } +/// One second (or `sampling.bucket_s` seconds) of completed requests. +struct Bucket { + time: String, + queries: Vec, + requests: u64, + errors: u64, + wall: f64, + cpu: Vec, + mem_mb: Option, +} + +impl Bucket { + fn point(&self, keys: &[QueryKey], latencies: &[f64], trial: u32, plan: SecondPlan) -> Point { + Point { + time: self.time.clone(), + rps: self.requests as f64 / self.wall, + err: error_rate(self.errors, self.requests), + latency: summarize_latency(latencies), + cpu: self.cpu.clone(), + mem_mb: self.mem_mb, + trial: Some(trial), + stage: Some(plan.stage), + phase: Some(plan.phase), + vus: Some(plan.vus), + requests: Some(self.requests), + queries: query_points(keys, &self.queries, self.wall), + } + } +} + +/// Running totals for one measurement window: either the whole trial, or a +/// single hold plateau of a concurrency ramp. +/// +/// Raw latency samples are retained until the window closes. A percentile +/// recombined from per-bucket percentiles is not the window's percentile, and +/// the step p99 that the saturation SLO is judged against has to be a real one. +struct Window { + stage: Option, + phase: Option, + vus: Option, + time: String, + queries: Vec, + requests: u64, + errors: u64, + wall: f64, + cpu: Vec, + cpu_samples: usize, + mem: Vec, +} + +impl Window { + fn new(queries: usize, stage: Option, phase: Option, vus: Option) -> Self { + Self { + stage, + phase, + vus, + time: String::new(), + queries: vec![QueryBucket::default(); queries], + requests: 0, + errors: 0, + wall: 0.0, + cpu: Vec::new(), + cpu_samples: 0, + mem: Vec::new(), + } + } + + fn absorb(&mut self, bucket: &Bucket) { + self.time.clone_from(&bucket.time); + for (slot, counted) in self.queries.iter_mut().zip(&bucket.queries) { + slot.merge(counted); + } + self.requests += bucket.requests; + self.errors += bucket.errors; + self.wall += bucket.wall; + if self.cpu.len() < bucket.cpu.len() { + self.cpu.resize(bucket.cpu.len(), 0.0); + } + for (slot, value) in self.cpu.iter_mut().zip(&bucket.cpu) { + *slot += value; + } + self.cpu_samples += 1; + if let Some(mem) = bucket.mem_mb { + self.mem.push(mem); + } + } + + fn is_empty(&self) -> bool { + self.cpu_samples == 0 + } + + /// Close the window. Percentiles come from every raw sample it collected. + fn finish(&self, keys: &[QueryKey], trial: u32) -> Point { + let mut merged: Vec = + Vec::with_capacity(self.queries.iter().map(|q| q.latencies.len()).sum()); + for bucket in &self.queries { + merged.extend_from_slice(&bucket.latencies); + } + + let wall = self.wall.max(0.001); + let mut cpu = self.cpu.clone(); + if self.cpu_samples > 0 { + for slot in &mut cpu { + *slot /= self.cpu_samples as f64; + } + } + if cpu.is_empty() { + cpu.push(0.0); + } + + Point { + time: if self.time.is_empty() { + now_rfc3339() + } else { + self.time.clone() + }, + rps: self.requests as f64 / wall, + err: error_rate(self.errors, self.requests), + latency: summarize_latency(&merged), + cpu, + mem_mb: (!self.mem.is_empty()).then(|| avg(&self.mem)), + trial: Some(trial), + stage: self.stage, + phase: self.phase, + vus: self.vus, + requests: Some(self.requests), + queries: query_points(keys, &self.queries, wall), + } + } +} + +fn error_rate(errors: u64, requests: u64) -> f64 { + if requests == 0 { + 0.0 + } else { + errors as f64 / requests as f64 + } +} + /// One second of the load profile, tagged with the stage it came from and the /// role that second plays in the measurement. #[derive(Clone, Copy, Debug, PartialEq, Eq)] -struct SecondPlan { - vus: u32, - stage: u32, - phase: Phase, +pub(crate) struct SecondPlan { + pub(crate) vus: u32, + pub(crate) stage: u32, + pub(crate) phase: Phase, +} + +/// The per-second plan a workload will actually run. +/// +/// Exposed so spec validation can check the schedule the load generator builds +/// rather than a second, drifting interpretation of the stage list. +pub(crate) fn schedule_for(workload: &Workload) -> Vec { + build_schedule( + &workload.stages, + workload.load.executor.starts_with("ramping"), + workload.warmup_seconds(), + ) +} + +/// The hold plateaus a workload will measure, ascending in time, as +/// `(stage index, concurrency)`. These are the steps of a saturation curve. +pub(crate) fn hold_steps(workload: &Workload) -> Vec<(u32, u32)> { + let mut steps: Vec<(u32, u32)> = Vec::new(); + for slot in schedule_for(workload) + .iter() + .filter(|slot| slot.phase == Phase::Hold) + { + if steps.last().is_none_or(|(stage, _)| *stage != slot.stage) { + steps.push((slot.stage, slot.vus)); + } + } + steps } /// One completed request, handed from a VU task to the bucket collector. @@ -542,11 +711,16 @@ struct Sample { latency_ms: f64, } -/// Everything one trial produces: the per-bucket series for charts, plus the -/// trial aggregate computed from the merged raw samples while they still exist. +/// Everything one trial produces: the per-bucket series for charts, the trial +/// aggregate, and one aggregate per hold plateau. The latter two are computed +/// from merged raw samples while they still exist, which is the only place an +/// exact percentile can come from. pub(crate) struct Measured { pub(crate) series: Vec, pub(crate) aggregate: Point, + /// One entry per hold plateau, ascending in time. Empty for a workload with + /// no steady state at all (a run shorter than its own warmup). + pub(crate) steps: Vec, } /// Requests are drawn `plans[(vu * PLAN_STRIDE + iter) % len]`. The pool is @@ -595,13 +769,10 @@ async fn measure_vus_async( let mut workers: Vec> = Vec::new(); let mut series: Vec = Vec::new(); - let mut agg_queries: Vec = vec![QueryBucket::default(); query_keys.len()]; - let mut agg_requests = 0_u64; - let mut agg_errors = 0_u64; - let mut agg_wall = 0.0_f64; - let mut agg_cpu: Vec = Vec::new(); - let mut agg_cpu_samples = 0_usize; - let mut agg_mem: Vec = Vec::new(); + let mut aggregate = Window::new(query_keys.len(), None, None, None); + // One window per hold plateau, in the order the ramp visits them. These are + // the steps of the saturation curve; the aggregate spans all of them. + let mut steps: Vec = Vec::new(); let mut idx = 0_usize; while idx < schedule.len() { @@ -705,44 +876,38 @@ async fn measure_vus_async( } let snapshot = sampler.latest(); - let point = Point { + let bucket = Bucket { time: now_rfc3339(), - rps: total as f64 / wall, - err: if total == 0 { - 0.0 - } else { - errors as f64 / total as f64 - }, - latency: summarize_latency(&latencies), - cpu: snapshot.cpu.clone(), + queries, + requests: total, + errors, + wall, + cpu: snapshot.cpu, mem_mb: snapshot.mem_mb, - trial: Some(trial), - stage: Some(plan.stage), - phase: Some(plan.phase), - requests: Some(total), - queries: query_points(&query_keys, &queries, wall), }; if counts_toward_aggregate(plan.phase) { - agg_requests += total; - agg_errors += errors; - agg_wall += wall; - for (slot, bucket) in agg_queries.iter_mut().zip(queries) { - slot.merge(bucket); - } - if agg_cpu.len() < snapshot.cpu.len() { - agg_cpu.resize(snapshot.cpu.len(), 0.0); - } - for (slot, value) in agg_cpu.iter_mut().zip(&snapshot.cpu) { - *slot += value; - } - agg_cpu_samples += 1; - if let Some(mem) = snapshot.mem_mb { - agg_mem.push(mem); - } + aggregate.absorb(&bucket); + } + // A step is one hold plateau. Ramp and warmup buckets are charted but + // never measured: they describe the transition, not the steady state. + if plan.phase == Phase::Hold { + let step = match steps.last_mut() { + Some(step) if step.stage == Some(plan.stage) => step, + _ => { + steps.push(Window::new( + query_keys.len(), + Some(plan.stage), + Some(Phase::Hold), + Some(plan.vus), + )); + steps.last_mut().expect("just pushed") + } + }; + step.absorb(&bucket); } - series.push(point); + series.push(bucket.point(&query_keys, &latencies, trial, plan)); idx = end; } @@ -753,7 +918,7 @@ async fn measure_vus_async( } sampler.stop(); - if agg_requests > 0 && agg_errors == agg_requests { + if aggregate.requests > 0 && aggregate.errors == aggregate.requests { let msg = first_err .lock() .ok() @@ -762,51 +927,17 @@ async fn measure_vus_async( return Err(Fail::new(Code::RunFail, msg)); } - // Exact percentiles over every steady-state sample of the trial. A median of - // per-second p95s is not the trial p95 — it systematically hides the tail. - let mut merged: Vec = Vec::with_capacity( - agg_queries + Ok(Measured { + series, + // Exact percentiles over every steady-state sample of the trial. A + // median of per-second p95s is not the trial p95 — it hides the tail. + aggregate: aggregate.finish(&query_keys, trial), + steps: steps .iter() - .map(|bucket| bucket.latencies.len()) - .sum(), - ); - for bucket in &agg_queries { - merged.extend_from_slice(&bucket.latencies); - } - - let agg_wall = agg_wall.max(0.001); - if agg_cpu_samples > 0 { - for slot in &mut agg_cpu { - *slot /= agg_cpu_samples as f64; - } - } - if agg_cpu.is_empty() { - agg_cpu.push(0.0); - } - - let aggregate = Point { - time: now_rfc3339(), - rps: agg_requests as f64 / agg_wall, - err: if agg_requests == 0 { - 0.0 - } else { - agg_errors as f64 / agg_requests as f64 - }, - latency: summarize_latency(&merged), - cpu: agg_cpu, - mem_mb: if agg_mem.is_empty() { - None - } else { - Some(avg(&agg_mem)) - }, - trial: Some(trial), - stage: None, - phase: None, - requests: Some(agg_requests), - queries: query_points(&query_keys, &agg_queries, agg_wall), - }; - - Ok(Measured { series, aggregate }) + .filter(|step| !step.is_empty()) + .map(|step| step.finish(&query_keys, trial)) + .collect(), + }) } fn plan_index(vu_id: u64, iter: u64, len: usize) -> usize { diff --git a/bench/runner/src/main.rs b/bench/runner/src/main.rs index b783719d..cc2a0bca 100644 --- a/bench/runner/src/main.rs +++ b/bench/runner/src/main.rs @@ -2,6 +2,7 @@ mod capture; mod cli; mod clock; mod code; +mod fairness; mod jsonio; mod load; mod model; @@ -9,6 +10,7 @@ mod parity; mod proc; mod publish; mod run; +mod saturation; mod schema; mod stats; mod workload_terms; diff --git a/bench/runner/src/model.rs b/bench/runner/src/model.rs index 52ae957e..3d9bb103 100644 --- a/bench/runner/src/model.rs +++ b/bench/runner/src/model.rs @@ -24,6 +24,11 @@ pub struct Workload { pub pacing: Pacing, pub sampling: Sampling, pub limits: Limits, + /// Declares this workload as a capacity measurement. Present only on the + /// stepped unpaced ramp; a paced workload cannot produce a capacity number + /// (see [`Pacing`]) and the runner refuses the combination. + #[serde(default)] + pub saturation: Option, } impl Workload { @@ -32,6 +37,54 @@ impl Workload { } } +/// A capacity measurement declared by the workload. +/// +/// The steps are not listed here: they *are* the hold stages of +/// [`Workload::stages`], so the ramp has exactly one definition and the spec +/// cannot drift from what the load generator runs. +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct SaturationSpec { + pub slo: Slo, +} + +/// The latency ceiling a step must stay under to be eligible as the peak. +#[derive(Debug, Clone, Copy, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub struct Slo { + pub metric: SloMetric, + pub ms: f64, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize, Serialize)] +#[serde(rename_all = "lowercase")] +pub enum SloMetric { + P50, + P90, + P95, + P99, +} + +impl SloMetric { + pub fn of(self, latency: &StepLatencyDoc) -> f64 { + match self { + Self::P50 => latency.p50, + Self::P90 => latency.p90, + Self::P95 => latency.p95, + Self::P99 => latency.p99, + } + } + + pub const fn as_str(self) -> &'static str { + match self { + Self::P50 => "p50", + Self::P90 => "p90", + Self::P95 => "p95", + Self::P99 => "p99", + } + } +} + #[derive(Debug, Deserialize)] #[serde(deny_unknown_fields)] pub struct Load { @@ -92,7 +145,7 @@ pub struct Sampling { pub bucket_s: u32, } -#[derive(Debug, Deserialize)] +#[derive(Debug, Clone, Copy, Deserialize)] #[serde(deny_unknown_fields)] pub struct Limits { pub err: f64, @@ -214,14 +267,32 @@ pub struct Wire { pub format: String, } +/// The harness knobs a target declares, and the family it competes in. +/// +/// Within a family these must be identical or the run fails: a ranked +/// library-vs-library table is only meaningful when every entry ran the same +/// harness. Across families they are free to differ — that difference *is* the +/// stack comparison — and the manifest records it per family so a reader cannot +/// mistake a stack difference for a library one. #[derive(Debug, Deserialize, Serialize, Clone)] #[serde(deny_unknown_fields)] pub struct Fair { + /// Comparison group: the targets this one claims to be directly comparable + /// to. Usually one per engine, but it splits where the harness cannot + /// honestly be equalised — `sqlite-ts` is separate from `sqlite` because + /// `bun:sqlite` is synchronous on a single-threaded runtime, so matching the + /// Rust pool of 8 would cripple it rather than make it fair. Declared rather + /// than inferred: `db.profile` distinguishes configurations *within* a group + /// and `fair.db` names the SQL dialect, so neither identifies the bracket. + pub family: String, pub workers: u32, pub pool: u32, pub db: String, pub schema: String, pub contract: String, + /// One-line summary of the engine tuning this target runs under (pragmas, + /// server image, cache settings). Compared for equality within a family. + pub tuning: String, } #[derive(Debug, Deserialize, Serialize, Clone)] @@ -300,7 +371,11 @@ pub struct SummaryDoc { pub group: Option, pub primary: PrimaryDoc, pub spread: SpreadDoc, - pub saturation: SaturationDoc, + /// Present only when the workload declared a capacity measurement. A paced + /// run has no capacity number, and absent is the honest answer — consumers + /// render it as "not measured", never as zero. + #[serde(skip_serializing_if = "Option::is_none")] + pub saturation: Option, } #[derive(Debug, Serialize, Deserialize, Clone)] @@ -384,10 +459,82 @@ pub struct RangeDoc { pub max: f64, } +/// Peak throughput under a declared latency SLO, plus the curve it was read off. +/// +/// Replaces the pre-contract `{knee_rps, knee_p95}` heuristic, which inferred a +/// knee from a p95-doubling rule over a *paced* run and fell back to the +/// highest-throughput bucket when no knee appeared. That fallback reported a +/// capacity-shaped number for a run whose throughput was capped by its own sleep +/// timer. The three outcomes below exist so "we could not measure it" is always +/// said out loud instead. #[derive(Debug, Serialize, Clone)] pub struct SaturationDoc { - pub knee_rps: f64, - pub knee_p95: f64, + pub slo: Slo, + pub outcome: Outcome, + /// The fastest qualifying step. Present only when `outcome == "saturated"`. + #[serde(skip_serializing_if = "Option::is_none")] + pub peak: Option, + /// Best throughput measured while holding the SLO. Present only when + /// `outcome == "did_not_saturate"`: capacity is at *least* this, and the + /// ramp ended before it found the knee. + #[serde(skip_serializing_if = "Option::is_none")] + pub lower_bound_rps: Option, + /// Every measured step, ascending by concurrency. The shape of + /// rps-vs-concurrency is what makes the headline checkable. + pub curve: Vec, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum Outcome { + /// A qualifying step was found and the ramp's last step failed to qualify, + /// so the ceiling is inside the measured range. + Saturated, + /// No step qualified as a peak: every one either breached the SLO or was + /// disqualified by its error rate. There is no peak, and none is reported. + SloNeverMet, + /// The ramp's last step still qualified, so the ceiling is somewhere above + /// the measured range. The best qualifying throughput is a lower bound, not + /// a peak — the ramp was too short. + DidNotSaturate, +} + +#[derive(Debug, Serialize, Clone)] +pub struct PeakDoc { + pub concurrency: u32, + pub rps: f64, + pub latency: StepLatencyDoc, + pub cpu: f64, + pub err: f64, +} + +#[derive(Debug, Serialize, Clone)] +pub struct CurveStepDoc { + pub concurrency: u32, + pub rps: f64, + pub latency: StepLatencyDoc, + pub err: f64, + pub cpu: f64, + pub slo_met: bool, + /// Why this step can never be the peak, or `null`. Always serialized so a + /// consumer can rely on the key being present. + pub disqualified: Option, +} + +impl CurveStepDoc { + /// A step is eligible to be the peak only when it met the SLO *and* stayed + /// within the error limit. + pub fn qualifies(&self) -> bool { + self.slo_met && self.disqualified.is_none() + } +} + +#[derive(Debug, Serialize, Clone, Copy)] +pub struct StepLatencyDoc { + pub p50: f64, + pub p90: f64, + pub p95: f64, + pub p99: f64, } #[derive(Debug, Serialize, Deserialize)] @@ -437,6 +584,11 @@ pub struct Point { pub stage: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub phase: Option, + /// Virtual users in flight. With `pacing.mode = none` there is no think + /// time, so this is also the request concurrency — the x-axis of the + /// saturation curve. Absent on aggregates, which span several VU counts. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub vus: Option, /// Completed requests counted in this bucket (the aggregation weight). #[serde(default, skip_serializing_if = "Option::is_none")] pub requests: Option, @@ -509,16 +661,32 @@ pub struct ManifestDoc { pub artifacts: Artifacts, pub runner: Runner, pub trials: TrialMeta, - /// Cross-target fairness findings for this run. Empty when every target - /// declares the same `fair` block (or is exempt). - #[serde(skip_serializing_if = "Vec::is_empty")] - pub fairness: Vec, + /// The harness each database family ran under, one entry per family present + /// in this run. Within-family drift is a hard error, so every block here + /// records a verified fact rather than an aspiration. + pub harness: Vec, } #[derive(Debug, Serialize)] -pub struct FairnessWarning { - pub kind: String, - pub msg: String, +pub struct HarnessDoc { + pub family: String, + /// Targets whose declared harness was compared for equality. + pub targets: Vec, + /// Absent only when every target in the family is exempt from the check. + #[serde(skip_serializing_if = "Option::is_none")] + pub workers: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub pool: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub tuning: Option, + /// True when at least one target was compared and they all agreed. False + /// only when there was nothing to compare, in which case `workers`, `pool` + /// and `tuning` are absent — never a claim that drift was tolerated. + pub within_family_identical: bool, + /// Targets excluded from the equality check because their configuration is + /// not comparable by design (an in-process cache has no connection pool). + #[serde(skip_serializing_if = "Vec::is_empty")] + pub exempt: Vec, } #[derive(Debug, Serialize)] diff --git a/bench/runner/src/parity.rs b/bench/runner/src/parity.rs index b93d4de5..f02b9478 100644 --- a/bench/runner/src/parity.rs +++ b/bench/runner/src/parity.rs @@ -894,11 +894,13 @@ mod tests { "db": { "profile": "sqlite", "hash": format!("sha256:{}", "1".repeat(64)) }, "wire": { "format": "json" }, "fair": { + "family": "sqlite", "workers": 1, "pool": 1, "db": "sqlite", "schema": format!("sha256:{}", "2".repeat(64)), - "contract": "v1" + "contract": "v1", + "tuning": "WAL journal, temp_store=MEMORY" }, "contract": { "ver": "v1" }, "parity": { "cmd": ["true"] }, diff --git a/bench/runner/src/run.rs b/bench/runner/src/run.rs index 073f2aef..4759aeaa 100644 --- a/bench/runner/src/run.rs +++ b/bench/runner/src/run.rs @@ -3,11 +3,11 @@ use crate::clock::now_rfc3339; use crate::code::{Code, Fail}; use crate::jsonio; use crate::model::{ - Artifacts, AvgPeakDoc, BoxMetricDoc, BoxPlotDoc, DatasetSummary, Event, Exec, FairnessWarning, - Gate, Gates, Headroom, LatencyDoc, Limits, LoadSummary, ManifestDoc, ManifestSummary, Point, + Artifacts, AvgPeakDoc, BoxMetricDoc, BoxPlotDoc, DatasetSummary, Event, Exec, Gate, Gates, + HarnessDoc, Headroom, LatencyDoc, Limits, LoadSummary, ManifestDoc, ManifestSummary, Point, PrimaryDoc, QueryDoc, QueryShapeDoc, RangeDoc, RequestDoc, ResultDoc, Runner, RunnerMetrics, - SaturationDoc, SpreadDoc, Status, SummaryDoc, Target, TargetMetaDoc, TimeseriesDoc, Topology, - TrialMeta, VarianceDoc, VarianceMetricDoc, Workload, + SaturationDoc, SaturationSpec, SpreadDoc, Status, SummaryDoc, Target, TargetMetaDoc, + TimeseriesDoc, Topology, TrialMeta, VarianceDoc, VarianceMetricDoc, Workload, }; use crate::stats::{avg, max, median, median_sorted, min, peak, sample_variance}; use crate::workload_terms::{CUSTOMER_SEARCH_TERMS, PRODUCT_SEARCH_TERMS}; @@ -68,8 +68,14 @@ fn run(args: Run) -> Result { let mut events = new_events_writer(&events_path)?; emit(&mut events, args.json, "info", "validate", "start")?; - for warning in &input.fairness { - emit(&mut events, args.json, "warn", "validate", &warning.msg)?; + for block in &input.harness { + emit( + &mut events, + args.json, + "info", + "validate", + harness_summary(block), + )?; } emit(&mut events, args.json, "info", "validate", "ok")?; @@ -221,7 +227,12 @@ struct RunInput { request_count_hint: usize, limits: Limits, bucket_s: u32, - fairness: Vec, + /// One entry per database family in this run. Built before any measurement + /// so an unfair comparison fails in seconds instead of after an hour. + harness: Vec, + /// Declared only by the stepped unpaced ramp; absent for the paced suite, + /// which cannot measure capacity. + saturation: Option, } struct TrialContext<'a> { @@ -266,13 +277,27 @@ struct TargetArtifacts<'a> { measurements: &'a [TrialMeasurement], summary: &'a PrimaryDoc, spread: &'a SpreadDoc, - saturation: &'a SaturationDoc, + saturation: Option<&'a SaturationDoc>, } #[derive(Debug, Clone)] struct TrialMeasurement { aggregate: Point, series: Vec, + /// One aggregate per hold plateau, ascending in time. Empty when the load + /// command emitted no per-step artifact. + steps: Vec, +} + +#[cfg(test)] +impl TrialMeasurement { + fn new(aggregate: Point, series: Vec) -> Self { + Self { + aggregate, + series, + steps: Vec::new(), + } + } } struct Baseline { @@ -347,7 +372,7 @@ fn load_input(args: &Run) -> Result { let trials = args.trials.unwrap_or_else(|| args.class.default_trials()); let bucket_s = workload.sampling.bucket_s.max(1); let limits = workload.limits; - let fairness = check_cross_target_fairness(&targets); + let harness = crate::fairness::harness(&targets)?; Ok(RunInput { suite, seed: workload.data.seed, @@ -361,54 +386,11 @@ fn load_input(args: &Run) -> Result { request_count_hint, limits, bucket_s, - fairness, + harness, + saturation: workload.saturation, }) } -/// Targets ranked against each other in one run must agree on their declared -/// fairness knobs, otherwise the leaderboard is comparing configurations rather -/// than implementations. Mismatches warn instead of failing: some are legitimate -/// (an in-process cache has no connection pool) and blocking the run would just -/// push people to delete the metadata. -fn check_cross_target_fairness(targets: &[Target]) -> Vec { - let mut warnings = Vec::new(); - let comparable: Vec<&Target> = targets - .iter() - .filter(|target| !is_fairness_exempt(target)) - .collect(); - let Some(reference) = comparable.first() else { - return warnings; - }; - - for target in comparable.iter().skip(1) { - if target.fair.pool != reference.fair.pool { - warnings.push(FairnessWarning { - kind: "fair.pool".to_string(), - msg: format!( - "target {} declares fair.pool={} but {} declares {}; \ - ranked comparisons assume an identical pool size", - target.id, target.fair.pool, reference.id, reference.fair.pool - ), - }); - } - if target.fair.workers != reference.fair.workers { - warnings.push(FairnessWarning { - kind: "fair.workers".to_string(), - msg: format!( - "target {} declares fair.workers={} but {} declares {}; \ - ranked comparisons assume an identical worker count", - target.id, target.fair.workers, reference.id, reference.fair.workers - ), - }); - } - } - warnings -} - -fn is_fairness_exempt(target: &Target) -> bool { - matches!(target.data_access.as_deref(), Some("in-process-cache")) -} - /// A requests-file entry: either a bare path string or an explicit request. #[derive(serde::Deserialize)] #[serde(untagged)] @@ -905,7 +887,17 @@ fn run_aggregate( .ok_or_else(|| Fail::new(Code::AggregateFail, "missing target trial points"))?; let summary = compute_primary(values); let spread = compute_spread(values, input.trials); - let saturation = compute_saturation(values); + let saturation = match &input.saturation { + Some(spec) => Some(measure_saturation( + &target.id, + spec, + &input.limits, + values, + events, + json, + )?), + None => None, + }; summary_map.insert(target.id.clone(), summary.clone()); write_target_artifacts( run_dir, @@ -918,7 +910,7 @@ fn run_aggregate( measurements: values, summary: &summary, spread: &spread, - saturation: &saturation, + saturation: saturation.as_ref(), }, )?; } @@ -949,7 +941,7 @@ fn write_target_artifacts(run_dir: &Path, doc: TargetArtifacts<'_>) -> Result<() group: doc.group.map(str::to_string), primary: doc.summary.clone(), spread: doc.spread.clone(), - saturation: doc.saturation.clone(), + saturation: doc.saturation.cloned(), }; jsonio::write( target_dir.join("summary.json"), @@ -1272,8 +1264,10 @@ fn run_target_load( "{}-{trial}-{idx}.series.json", sanitize(&target.id) )); + let steps_path = scratch.join(format!("{}-{trial}-{idx}.steps.json", sanitize(&target.id))); let _ = fs::remove_file(&point_path); let _ = fs::remove_file(&series_path); + let _ = fs::remove_file(&steps_path); let mut env = BTreeMap::new(); env.insert( @@ -1284,6 +1278,10 @@ fn run_target_load( "BENCH_TIMESERIES_OUT".to_string(), series_path.to_string_lossy().to_string(), ); + env.insert( + "BENCH_STEPS_OUT".to_string(), + steps_path.to_string_lossy().to_string(), + ); env.insert("BENCH_RUN_DIR".to_string(), run_dir.display().to_string()); env.insert("BENCH_SUITE".to_string(), suite.to_string()); env.insert("BENCH_TARGET_ID".to_string(), target.id.clone()); @@ -1317,6 +1315,22 @@ fn run_target_load( ) })?; + // Per-step aggregates carry exact percentiles for one hold plateau. They can + // only be produced where the raw samples live, so a missing file means "not + // measured" — never "derive it from the buckets", which would average + // percentiles together and misplace the knee. + let steps = if steps_path.exists() { + let mut steps = load_points(&steps_path)?; + for point in &mut steps { + validate_point(&target.id, point)?; + } + let trial_path = raw_dir.join(format!("{trial}.steps.json")); + let _ = fs::copy(&steps_path, &trial_path); + steps + } else { + Vec::new() + }; + let measurement = if series_path.exists() { let mut series = load_points(&series_path)?; if series.is_empty() { @@ -1349,7 +1363,11 @@ fn run_target_load( } else { point_from_series(&series) }; - TrialMeasurement { aggregate, series } + TrialMeasurement { + aggregate, + series, + steps, + } } else if point_path.exists() { let trial_path = raw_dir.join(format!("{trial}.point.json")); fs::copy(&point_path, &trial_path).map_err(|err| { @@ -1366,6 +1384,7 @@ fn run_target_load( TrialMeasurement { aggregate: point.clone(), series: vec![point], + steps, } } else { return Err(Fail::new( @@ -1378,6 +1397,7 @@ fn run_target_load( }; let _ = fs::remove_file(&point_path); let _ = fs::remove_file(&series_path); + let _ = fs::remove_file(&steps_path); let mut measurement = measurement; validate_point(&target.id, &mut measurement.aggregate)?; Ok(measurement) @@ -1649,6 +1669,7 @@ fn point_from_series(points: &[Point]) -> Point { trial: counted.first().and_then(|point| point.trial), stage: None, phase: None, + vus: None, requests: counted .iter() .filter_map(|point| point.requests) @@ -2245,12 +2266,17 @@ fn write_manifest( count: input.trials, aggregate: "median", }, - fairness: input - .fairness + harness: input + .harness .iter() - .map(|warning| FairnessWarning { - kind: warning.kind.clone(), - msg: warning.msg.clone(), + .map(|block| HarnessDoc { + family: block.family.clone(), + targets: block.targets.clone(), + workers: block.workers, + pool: block.pool, + tuning: block.tuning.clone(), + within_family_identical: block.within_family_identical, + exempt: block.exempt.clone(), }) .collect(), }; @@ -2454,78 +2480,78 @@ fn compute_spread(measurements: &[TrialMeasurement], trials: u32) -> SpreadDoc { } } -/// Consecutive degraded buckets required before calling a knee. One slow bucket -/// is noise; three in a row is a trend. -const SATURATION_RUN_LENGTH: usize = 3; -/// A bucket counts as degraded once its p95 exceeds this multiple of the -/// lowest-VU plateau's p95. -const SATURATION_P95_FACTOR: f64 = 2.0; - -/// Locate the saturation knee per trial, then report the median across trials. +/// Read one target's peak throughput off its concurrency ramp. /// -/// Scanning a series concatenated across trials made `windows(2)` compare the -/// last bucket of one trial with the first bucket of the next, and an absolute -/// `slope > 0.02` threshold has no meaning across targets with different -/// baseline latencies. This uses a relative criterion anchored on the trial's -/// own lowest-VU plateau. -fn compute_saturation(measurements: &[TrialMeasurement]) -> SaturationDoc { - let mut knee_rps = Vec::new(); - let mut knee_p95 = Vec::new(); - - for measurement in measurements { - let points: Vec<&Point> = measurement - .series - .iter() - .filter(|point| point.is_hold()) - .collect(); - if points.is_empty() { - continue; - } +/// This replaced a per-trial p95-doubling heuristic that reported +/// `{knee_rps, knee_p95}` for *every* workload, paced or not, and fell back to +/// the highest-throughput bucket whenever no knee appeared. Both halves were +/// wrong: a paced run's throughput is capped by its own sleep timer, so the +/// number it produced was not capacity, and the fallback meant "we did not find +/// a knee" was reported as a knee. There is now one saturation concept in the +/// artifact, it is only emitted when the workload declares the measurement, and +/// each way of failing to find a peak has its own name. +fn measure_saturation( + target_id: &str, + spec: &SaturationSpec, + limits: &Limits, + measurements: &[TrialMeasurement], + events: &mut BufWriter, + json: bool, +) -> Result { + let steps: Vec<&[Point]> = measurements + .iter() + .map(|measurement| measurement.steps.as_slice()) + .collect(); + if steps.iter().any(|trial| trial.is_empty()) { + return Err(Fail::new( + Code::AggregateFail, + format!( + "target {target_id} produced no per-step measurements, so its capacity \ + cannot be computed. A saturation workload requires a load command that \ + writes $BENCH_STEPS_OUT; per-step percentiles cannot be recovered from \ + the bucket series without averaging percentiles together." + ), + )); + } - let Some(baseline_stage) = points.iter().filter_map(|point| point.stage).min() else { - // No stage tags: fall back to the highest-throughput bucket. - if let Some(best) = points.iter().max_by(|a, b| a.rps.total_cmp(&b.rps)) { - knee_rps.push(best.rps); - knee_p95.push(best.latency.p95); - } - continue; - }; - let plateau: Vec = points - .iter() - .filter(|point| point.stage == Some(baseline_stage)) - .map(|point| point.latency.p95) - .collect(); - let threshold = median(&plateau) * SATURATION_P95_FACTOR; - - let mut streak = 0_usize; - let mut found = None; - for point in &points { - if threshold > 0.0 && point.latency.p95 > threshold { - streak += 1; - if streak >= SATURATION_RUN_LENGTH { - found = Some(*point); - break; - } - } else { - streak = 0; - } - } + let doc = crate::saturation::measure(target_id, spec, limits, &steps)?; + emit( + events, + json, + "info", + "aggregate", + saturation_summary(target_id, &doc), + )?; + Ok(doc) +} - let knee = found.or_else(|| { - points - .iter() - .max_by(|a, b| a.rps.total_cmp(&b.rps)) - .copied() - }); - if let Some(knee) = knee { - knee_rps.push(knee.rps); - knee_p95.push(knee.latency.p95); - } +/// Plain-language one-liner for the events stream, using the same words the +/// dashboard shows so the log and the UI cannot disagree. +fn saturation_summary(target_id: &str, doc: &SaturationDoc) -> String { + let slo = format!("{} < {} ms", doc.slo.metric.as_str(), doc.slo.ms); + match (&doc.outcome, &doc.peak, doc.lower_bound_rps) { + (_, Some(peak), _) => format!( + "{target_id} peak throughput {:.0} req/s at {slo} (concurrency {})", + peak.rps, peak.concurrency + ), + (_, _, Some(rps)) => format!( + "{target_id} at least {rps:.0} req/s at {slo} — knee not reached, extend the ramp" + ), + _ => format!("{target_id} never met the {slo} target at any concurrency"), } +} - SaturationDoc { - knee_rps: median(&knee_rps), - knee_p95: median(&knee_p95), +fn harness_summary(block: &HarnessDoc) -> String { + match (block.workers, block.pool, &block.tuning) { + (Some(workers), Some(pool), Some(tuning)) => format!( + "family {} harness verified identical across {} target(s): workers={workers} pool={pool} tuning={tuning}", + block.family, + block.targets.len() + ), + _ => format!( + "family {} declares no enforceable harness: every target is exempt from the check", + block.family + ), } } @@ -3018,6 +3044,90 @@ fn validate_workload(workload: &Workload) -> Result<(), Fail> { "workload.limits.p95 must be > 0 when provided", )); } + if workload.saturation.is_some() { + validate_saturation(workload)?; + } + Ok(()) +} + +/// The smallest ramp that can distinguish a knee from noise: two points to +/// establish the linear region and at least one beyond it. +const MIN_SATURATION_STEPS: usize = 3; + +/// A saturation workload has to be able to produce the number it claims. +/// +/// Every check here exists because the alternative is a plausible-looking +/// artifact built on a measurement that was never taken. +fn validate_saturation(workload: &Workload) -> Result<(), Fail> { + let Some(spec) = &workload.saturation else { + return Ok(()); + }; + let invalid = |msg: String| Fail::new(Code::InvalidInput, msg); + + if workload.pacing.mode != crate::model::PacingMode::None { + return Err(invalid(format!( + "workload.saturation requires pacing.mode=none, found {:?}. A paced run \ + sleeps a mean 187.5 ms per virtual user between requests, so its offered \ + load is capped near VUs / (think time + service time) no matter how fast \ + the target is — every healthy target converges on the same rps and the \ + number describes the sleep timer, not capacity.", + workload.pacing.mode + ))); + } + if workload.load.executor != "ramping-vus" { + return Err(invalid(format!( + "workload.saturation requires load.executor=ramping-vus, found {}. \ + Each step must ramp its concurrency in before holding, otherwise the \ + plateau is measured through its own thundering herd.", + workload.load.executor + ))); + } + if spec.slo.ms <= 0.0 { + return Err(invalid( + "workload.saturation.slo.ms must be > 0".to_string(), + )); + } + + // A stage split down the middle by the warmup window would contribute a + // step measured over only part of its plateau, silently shorter than its + // neighbours. + let schedule = crate::load::schedule_for(workload); + for (stage, _) in workload.stages.iter().enumerate() { + let stage = stage as u32; + let mut phases = schedule + .iter() + .filter(|slot| slot.stage == stage) + .map(|slot| slot.phase); + let Some(first) = phases.next() else { continue }; + if phases.any(|phase| phase != first) { + return Err(invalid(format!( + "workload.warmup_s={} ends inside stages[{stage}]. Set it to a \ + cumulative stage boundary so every measured step covers a whole \ + plateau.", + workload.warmup_seconds() + ))); + } + } + + let steps = crate::load::hold_steps(workload); + if steps.len() < MIN_SATURATION_STEPS { + return Err(invalid(format!( + "workload.saturation needs at least {MIN_SATURATION_STEPS} measured steps, \ + found {}. A step is a stage that holds its concurrency and is not inside \ + the warmup window; a shorter ramp cannot tell a knee from noise.", + steps.len() + ))); + } + for pair in steps.windows(2) { + let ((_, lower), (_, higher)) = (pair[0], pair[1]); + if higher <= lower { + return Err(invalid(format!( + "workload.saturation steps must climb in concurrency, but a step at \ + {higher} VUs follows one at {lower}. The curve is read left to right \ + and the peak is the highest qualifying step." + ))); + } + } Ok(()) } @@ -3273,9 +3383,9 @@ fn variance_metric(values: &[f64]) -> VarianceMetricDoc { mod tests { use super::{ Gate, MIX_CUSTOMER_BY_ID, MIX_SEARCH_CUSTOMER, MIX_SEARCH_PRODUCT, TrialMeasurement, - box_metric, combined_series, compute_headroom, compute_primary, compute_saturation, - compute_spread, headroom_gate, materialize_requests, point_from_series, - query_catalog_total_mix, request_path_skipped, resolve_seed, sample_variance, + box_metric, combined_series, compute_headroom, compute_primary, compute_spread, + headroom_gate, materialize_requests, point_from_series, query_catalog_total_mix, + request_path_skipped, resolve_seed, sample_variance, validate_workload, }; use crate::cli::Class; use crate::model::{Latency, Phase, Point}; @@ -3320,10 +3430,10 @@ mod tests { #[test] fn summary_uses_sample_peak_and_full_series() { - let measurements = vec![TrialMeasurement { - aggregate: point(150.0, 10.0), - series: vec![point(100.0, 20.0), point(250.0, 80.0)], - }]; + let measurements = vec![TrialMeasurement::new( + point(150.0, 10.0), + vec![point(100.0, 20.0), point(250.0, 80.0)], + )]; let primary = compute_primary(&measurements); assert_eq!(primary.rps.avg, 150.0); @@ -3335,15 +3445,15 @@ mod tests { #[test] fn primary_ignores_warmup_and_ramp_buckets_for_peaks() { // A cold-start bucket must not be able to set the published peak. - let measurements = vec![TrialMeasurement { - aggregate: point(100.0, 10.0), - series: vec![ + let measurements = vec![TrialMeasurement::new( + point(100.0, 10.0), + vec![ tagged(9_000.0, 95.0, Phase::Warmup, 0), tagged(8_000.0, 90.0, Phase::Ramp, 0), tagged(100.0, 20.0, Phase::Hold, 1), tagged(120.0, 25.0, Phase::Hold, 1), ], - }]; + )]; let primary = compute_primary(&measurements); assert_eq!(primary.rps.peak, 120.0); @@ -3361,10 +3471,10 @@ mod tests { p99: 12.0, p999: Some(20.0), }; - let measurements = vec![TrialMeasurement { + let measurements = vec![TrialMeasurement::new( aggregate, - series: vec![tagged(100.0, 10.0, Phase::Hold, 0)], - }]; + vec![tagged(100.0, 10.0, Phase::Hold, 0)], + )]; let primary = compute_primary(&measurements); assert_eq!(primary.latency.p50, Some(0.8)); @@ -3403,10 +3513,10 @@ mod tests { let mut measurements = BTreeMap::new(); measurements.insert( "t".to_string(), - vec![TrialMeasurement { - aggregate: point(100.0, 25.0), - series: vec![series_point], - }], + vec![TrialMeasurement::new( + point(100.0, 25.0), + vec![series_point], + )], ); let headroom = compute_headroom(&measurements); @@ -3414,27 +3524,88 @@ mod tests { assert_eq!(headroom.cpu_mean_peak, Some(25.0)); } + /// A saturation workload must be able to produce the number it claims, so + /// every way of asking for an unmeasurable one is refused up front rather + /// than after an hour of load. #[test] - fn saturation_knee_is_relative_and_per_trial() { - // Stage 0 plateaus at p95 = 2ms; the knee is the third consecutive - // bucket above 2x that. - let plateau = |rps: f64| latency_point(rps, 2.0, 0); - let degraded = |rps: f64| latency_point(rps, 9.0, 1); - let measurements = vec![TrialMeasurement { - aggregate: point(100.0, 10.0), - series: vec![ - plateau(100.0), - plateau(110.0), - degraded(200.0), - degraded(210.0), - degraded(220.0), - degraded(230.0), - ], - }]; + fn a_paced_saturation_workload_is_refused() { + let spec = saturation_workload(FOUR_STEP_RAMP, 6) + .replace(r#""mode": "none""#, r#""mode": "drizzle-benchmark""#); + let err = validate_workload(&parse_workload(&spec)).expect_err("paced ramp"); + assert!(err.msg.contains("requires pacing.mode=none"), "{}", err.msg); + // The reason, not just the rule. + assert!(err.msg.contains("sleep timer"), "{}", err.msg); + } + + #[test] + fn a_saturation_ramp_needs_enough_steps_to_show_a_knee() { + // Warmup at 8 (stages 0-1), then only two measured steps: 8 and 16. + let two_steps = r#" + { "sec": 2, "vus": 8 }, { "sec": 4, "vus": 8 }, + { "sec": 4, "vus": 8 }, + { "sec": 2, "vus": 16 }, { "sec": 4, "vus": 16 } + "#; + let err = validate_workload(&parse_workload(&saturation_workload(two_steps, 6))) + .expect_err("two steps"); + assert!(err.msg.contains("at least 3 measured steps"), "{}", err.msg); + } + + #[test] + fn a_saturation_ramp_must_climb() { + // Steps become 8, 16, 8, 64 — the third one goes backwards. + let spec = saturation_workload(FOUR_STEP_RAMP, 6).replace( + r#""vus": 32 }, { "sec": 4, "vus": 32 }"#, + r#""vus": 8 }, { "sec": 4, "vus": 8 }"#, + ); + let err = validate_workload(&parse_workload(&spec)).expect_err("ramp that dips"); + assert!(err.msg.contains("must climb in concurrency"), "{}", err.msg); + } + + #[test] + fn a_warmup_that_ends_mid_stage_is_refused() { + // 4 s of warmup cuts the 4 s stage 1 in half. + let err = validate_workload(&parse_workload(&saturation_workload(FOUR_STEP_RAMP, 4))) + .expect_err("split stage"); + assert!(err.msg.contains("ends inside stages["), "{}", err.msg); + } - let saturation = compute_saturation(&measurements); - assert_eq!(saturation.knee_rps, 220.0); - assert_eq!(saturation.knee_p95, 9.0); + #[test] + fn a_well_formed_saturation_ramp_validates() { + validate_workload(&parse_workload(&saturation_workload(FOUR_STEP_RAMP, 6))) + .expect("valid ramp"); + } + + fn parse_workload(body: &str) -> crate::model::Workload { + serde_json::from_str(body).expect("workload json") + } + + /// Warmup covers stages 0-1 (2 + 4 s); steps then hold at 8, 16, 32 and 64. + const FOUR_STEP_RAMP: &str = r#" + { "sec": 2, "vus": 8 }, { "sec": 4, "vus": 8 }, + { "sec": 4, "vus": 8 }, + { "sec": 2, "vus": 16 }, { "sec": 4, "vus": 16 }, + { "sec": 2, "vus": 32 }, { "sec": 4, "vus": 32 }, + { "sec": 2, "vus": 64 }, { "sec": 4, "vus": 64 } + "#; + + fn saturation_workload(stages: &str, warmup_s: u32) -> String { + format!( + r#"{{ + "version": "v1", + "suite": "throughput-http", + "name": "Ramp", + "load": {{ "kind": "closed", "executor": "ramping-vus", "unit": "1s", "concurrency": 64 }}, + "data": {{ "name": "b", "seed": 42, "schema": "s.sql" }}, + "shape": {{ "mode": "single", "endpoint": "/customer-by-id" }}, + "stages": [{stages}], + "warmup_s": {warmup_s}, + "requests": {{ "source": "generated", "file": "r.json", "skip": [] }}, + "pacing": {{ "mode": "none" }}, + "sampling": {{ "cpu_ms": 200, "bucket_s": 1 }}, + "limits": {{ "err": 0.01, "p95": null }}, + "saturation": {{ "slo": {{ "metric": "p99", "ms": 50 }} }} + }}"# + ) } fn tagged(rps: f64, cpu: f64, phase: Phase, stage: u32) -> Point { @@ -3445,27 +3616,12 @@ mod tests { point } - fn latency_point(rps: f64, p95: f64, stage: u32) -> Point { - let mut point = tagged(rps, 10.0, Phase::Hold, stage); - point.latency.p95 = p95; - point - } - #[test] fn spread_reports_sample_variance() { let measurements = vec![ - TrialMeasurement { - aggregate: point(100.0, 10.0), - series: vec![point(100.0, 10.0)], - }, - TrialMeasurement { - aggregate: point(200.0, 20.0), - series: vec![point(200.0, 20.0)], - }, - TrialMeasurement { - aggregate: point(300.0, 30.0), - series: vec![point(300.0, 30.0)], - }, + TrialMeasurement::new(point(100.0, 10.0), vec![point(100.0, 10.0)]), + TrialMeasurement::new(point(200.0, 20.0), vec![point(200.0, 20.0)]), + TrialMeasurement::new(point(300.0, 30.0), vec![point(300.0, 30.0)]), ]; let spread = compute_spread(&measurements, 3); @@ -3602,6 +3758,7 @@ mod tests { trial: None, stage: None, phase: None, + vus: None, requests: None, queries: Vec::new(), } diff --git a/bench/runner/src/saturation.rs b/bench/runner/src/saturation.rs new file mode 100644 index 00000000..3527becf --- /dev/null +++ b/bench/runner/src/saturation.rs @@ -0,0 +1,546 @@ +//! Peak throughput under a declared latency SLO. +//! +//! # The measurement +//! +//! A saturation workload is an unpaced (`pacing.mode = none`) closed-loop ramp: +//! each step ramps concurrency in, then *holds* it while the measurement is +//! taken. With no think time N virtual users are N requests in flight, so the +//! step's concurrency is the x-axis and `rps ~= N / service_time` until the +//! system runs out of capacity. Past that point rps flattens and latency climbs +//! linearly with N — the knee. +//! +//! # Why paced runs cannot produce this number +//! +//! Under `pacing.mode = drizzle-benchmark` each VU sleeps a mean 187.5 ms +//! between requests, so offered load is capped near `VUs / (think + service)` +//! regardless of how fast the target is. Every healthy target converges on the +//! same rps and a tenfold service-time difference barely moves it: the number +//! measures the sleep timer. That is why capacity lives in its own suite and the +//! paced suite keeps its own headline ("throughput at fixed load"). +//! +//! # Reading the result +//! +//! A step *qualifies* when it met the SLO and stayed inside `limits.err`. The +//! peak is the **fastest** qualifying step, not the widest one: a closed-loop +//! curve dips once the pool saturates, so the last step to survive the SLO is +//! frequently slower than an earlier one that also survived it, and reporting +//! that under the name "peak throughput" would understate the target and point +//! at a worse operating point on both axes. Ties go to the lower concurrency. +//! +//! Whether the ramp found the ceiling is a separate question from where the +//! maximum landed, and `outcome` answers it: a ramp whose last step still +//! qualified never reached the knee, so its best throughput is reported as a +//! lower bound rather than a peak. When nothing qualifies there is no peak at +//! all. No case is padded with a substitute number. + +use crate::code::{Code, Fail}; +use crate::model::{ + CurveStepDoc, Limits, Outcome, PeakDoc, Point, SaturationDoc, SaturationSpec, StepLatencyDoc, +}; +use crate::stats::{avg, median}; + +/// Build the saturation artifact from one step series per trial. +/// +/// Each `Point` is a whole hold plateau whose percentiles were computed from +/// that plateau's merged raw samples, so the per-step p99 is a real p99. Across +/// trials the reported value is the median, matching `summary.primary`. +/// +/// # Errors +/// +/// Fails when a trial produced no steps, when a step is missing its concurrency +/// tag or any percentile the curve reports, or when the trials disagree about +/// which steps ran. Each means the measurement is not the one the spec asked +/// for, and a patched-up curve would misrepresent where the knee is. +pub fn measure( + target_id: &str, + spec: &SaturationSpec, + limits: &Limits, + trials: &[&[Point]], +) -> Result { + let ladder = verified_ladder(target_id, trials)?; + + let curve = ladder + .iter() + .enumerate() + .map(|(idx, &concurrency)| step(spec, limits, concurrency, idx, trials)) + .collect::>(); + + // Peak throughput means the most throughput, not the most concurrency. A + // closed-loop curve can dip after the pool saturates and then flatten, so + // the highest-concurrency step that held the SLO is often slower than an + // earlier one that also held it — reporting the former under the name "peak + // throughput" would understate the target and point at a worse operating + // point on both axes. Ties go to the lower concurrency: the same throughput + // for fewer in-flight requests is strictly better. + let best = curve.iter().filter(|step| step.qualifies()).max_by(|a, b| { + a.rps + .total_cmp(&b.rps) + .then_with(|| b.concurrency.cmp(&a.concurrency)) + }); + + // Whether the ramp ever found the ceiling is a separate question from where + // the best step was: a ramp whose last step still held the SLO never reached + // the knee, however early the maximum happened to land. + let ramp_ended_inside_slo = curve.last().is_some_and(|step| step.qualifies()); + + let (outcome, peak, lower_bound_rps) = match best { + None => (Outcome::SloNeverMet, None, None), + // Known to reach at least this much while holding the SLO; the ceiling + // is somewhere above the ramp and was not measured. + Some(step) if ramp_ended_inside_slo => (Outcome::DidNotSaturate, None, Some(step.rps)), + Some(step) => { + let peak = PeakDoc { + concurrency: step.concurrency, + rps: step.rps, + latency: step.latency, + cpu: step.cpu, + err: step.err, + }; + (Outcome::Saturated, Some(peak), None) + } + }; + + Ok(SaturationDoc { + slo: spec.slo, + outcome, + peak, + lower_bound_rps, + curve, + }) +} + +/// The concurrency of every step, ascending, with every step verified complete +/// and the ladder verified identical across trials. +/// +/// Trials are separate processes running the same spec; if their ladders differ, +/// one of them dropped or mis-tagged a step and medianing them would silently +/// compare different plateaus. Percentiles are required rather than defaulted: +/// filling a missing `p50` from `p95` would publish one number under another's +/// name, and a step that did not record its own percentiles was not measured. +fn verified_ladder(target_id: &str, trials: &[&[Point]]) -> Result, Fail> { + let missing = |trial: usize, what: &str| { + Fail::new( + Code::AggregateFail, + format!("target {target_id} trial {trial} emitted a saturation step without {what}"), + ) + }; + + let mut ladder: Option> = None; + for (trial, steps) in trials.iter().enumerate() { + if steps.is_empty() { + return Err(Fail::new( + Code::AggregateFail, + format!( + "target {target_id} trial {trial} produced no saturation steps; \ + a saturation workload must hold at each concurrency long enough \ + to emit at least one hold bucket per step" + ), + )); + } + let found = steps + .iter() + .map(|point| { + if point.latency.p50.is_none() { + return Err(missing(trial, "a p50 latency")); + } + if point.latency.p90.is_none() { + return Err(missing(trial, "a p90 latency")); + } + point + .vus + .ok_or_else(|| missing(trial, "a concurrency (vus) tag")) + }) + .collect::, _>>()?; + + match &ladder { + None => ladder = Some(found), + Some(first) if *first != found => { + return Err(Fail::new( + Code::AggregateFail, + format!( + "target {target_id} trial {trial} ran concurrency ladder {found:?} \ + but trial 0 ran {first:?}; trials must measure the same steps" + ), + )); + } + Some(_) => {} + } + } + + ladder.ok_or_else(|| { + Fail::new( + Code::AggregateFail, + format!("target {target_id} has no trials to build a saturation curve from"), + ) + }) +} + +/// Combine one step across trials and judge it against the SLO and error limit. +fn step( + spec: &SaturationSpec, + limits: &Limits, + concurrency: u32, + idx: usize, + trials: &[&[Point]], +) -> CurveStepDoc { + let points: Vec<&Point> = trials.iter().map(|steps| &steps[idx]).collect(); + let per_trial = + |value: fn(&Point) -> f64| median(&points.iter().copied().map(value).collect::>()); + + // `expect` is discharged by `verified_ladder`, which rejects any step point + // missing a percentile. Substituting a neighbouring percentile would publish + // one number under another's name. + let latency = StepLatencyDoc { + p50: per_trial(|point| point.latency.p50.expect("p50 verified present")), + p90: per_trial(|point| point.latency.p90.expect("p90 verified present")), + p95: per_trial(|point| point.latency.p95), + p99: per_trial(|point| point.latency.p99), + }; + let err = per_trial(|point| point.err); + + CurveStepDoc { + concurrency, + rps: per_trial(|point| point.rps), + latency, + err, + cpu: per_trial(|point| avg(&point.cpu)), + slo_met: spec.slo.metric.of(&latency) <= spec.slo.ms, + disqualified: (err > limits.err).then(|| { + format!( + "error rate {:.2}% exceeds limit {:.2}%", + err * 100.0, + limits.err * 100.0 + ) + }), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::model::{Latency, Phase, Slo, SloMetric}; + + fn spec(ms: f64) -> SaturationSpec { + SaturationSpec { + slo: Slo { + metric: SloMetric::P99, + ms, + }, + } + } + + fn limits(err: f64) -> Limits { + Limits { + err, + p95: None, + cpu_mean_peak: None, + } + } + + /// One measured step: concurrency, throughput, p99, and error rate. + fn point(vus: u32, rps: f64, p99: f64, err: f64) -> Point { + Point { + time: "2026-01-01T00:00:00Z".to_string(), + rps, + err, + latency: Latency { + avg: p99 / 4.0, + p50: Some(p99 / 4.0), + p90: Some(p99 / 2.0), + p95: p99 * 0.8, + p99, + p999: Some(p99 * 1.2), + }, + cpu: vec![50.0, 50.0], + mem_mb: None, + trial: Some(0), + stage: None, + phase: Some(Phase::Hold), + vus: Some(vus), + requests: Some(rps as u64), + queries: Vec::new(), + } + } + + fn measured(steps: &[Point], slo_ms: f64, err_limit: f64) -> SaturationDoc { + measure("t", &spec(slo_ms), &limits(err_limit), &[steps]).expect("measure") + } + + /// The case that distinguishes "most throughput" from "most concurrency". + /// Throughput peaks at 16 VUs and sags afterwards while still holding the + /// SLO up to 64; the highest *qualifying concurrency* is 64, but the peak + /// *throughput* is at 16. Reporting 2 100 under the label "peak throughput" + /// when 3 100 was measured inside the same SLO would understate the target + /// and point at a worse operating point on both axes. + #[test] + fn peak_is_the_fastest_qualifying_step_not_the_widest() { + let steps = [ + point(8, 2_400.0, 4.0, 0.0), + point(16, 3_100.0, 9.0, 0.0), + point(32, 2_600.0, 18.0, 0.0), + point(64, 2_100.0, 40.0, 0.0), + point(128, 1_900.0, 120.0, 0.0), + ]; + let doc = measured(&steps, 50.0, 0.01); + + assert_eq!(doc.outcome, Outcome::Saturated); + let peak = doc.peak.expect("peak"); + assert_eq!(peak.rps, 3_100.0); + assert_eq!(peak.concurrency, 16); + assert_eq!(peak.latency.p99, 9.0); + // 64 held the SLO and is the widest qualifying step; it is not the peak. + assert!(doc.curve[3].qualifies()); + } + + /// Same throughput for fewer in-flight requests is strictly better. + #[test] + fn a_throughput_tie_breaks_toward_the_lower_concurrency() { + let steps = [ + point(8, 1_000.0, 4.0, 0.0), + point(16, 2_500.0, 9.0, 0.0), + point(32, 2_500.0, 19.0, 0.0), + point(64, 2_500.0, 120.0, 0.0), + ]; + let doc = measured(&steps, 50.0, 0.01); + + let peak = doc.peak.expect("peak"); + assert_eq!(peak.rps, 2_500.0); + assert_eq!(peak.concurrency, 16); + } + + /// A ramp that never breaches reports its best qualifying throughput as a + /// lower bound — the maximum is still the maximum, the ceiling is what was + /// not found. + #[test] + fn a_lower_bound_is_the_best_qualifying_throughput() { + let steps = [ + point(8, 1_000.0, 2.0, 0.0), + point(16, 3_000.0, 4.0, 0.0), + point(32, 2_400.0, 9.0, 0.0), + ]; + let doc = measured(&steps, 50.0, 0.01); + + assert_eq!(doc.outcome, Outcome::DidNotSaturate); + assert!(doc.peak.is_none()); + assert_eq!(doc.lower_bound_rps, Some(3_000.0)); + } + + /// On a monotone ramp the fastest qualifying step *is* the widest one, so + /// the two readings coincide. This is the shape the method assumes. + #[test] + fn a_monotone_ramp_peaks_at_its_last_qualifying_step() { + let steps = [ + point(8, 1_000.0, 9.0, 0.0), + point(16, 1_900.0, 18.0, 0.0), + point(32, 2_000.0, 45.0, 0.0), + point(64, 2_010.0, 120.0, 0.0), + point(128, 2_005.0, 260.0, 0.0), + ]; + let doc = measured(&steps, 50.0, 0.01); + + assert_eq!(doc.outcome, Outcome::Saturated); + assert!(doc.lower_bound_rps.is_none()); + let peak = doc.peak.expect("saturated runs carry a peak"); + assert_eq!(peak.concurrency, 32); + assert_eq!(peak.rps, 2_000.0); + assert_eq!(peak.latency.p99, 45.0); + // The whole ramp is reported, breaches included. + assert_eq!(doc.curve.len(), 5); + assert_eq!( + doc.curve.iter().map(|s| s.slo_met).collect::>(), + vec![true, true, true, false, false] + ); + } + + /// The peak is lifted out of the curve rather than recomputed, so its + /// concurrency is always one of the plotted steps. Consumers mark the peak + /// on the curve and deliberately withhold the marker rather than snapping it + /// to a neighbour, which would make any drift here visible. + #[test] + fn the_peak_is_always_one_of_the_plotted_steps() { + let steps = [ + point(8, 1_000.0, 9.0, 0.0), + point(16, 1_900.0, 18.0, 0.0), + point(32, 2_000.0, 45.0, 0.0), + point(64, 2_010.0, 120.0, 0.0), + ]; + let doc = measured(&steps, 50.0, 0.01); + let peak = doc.peak.expect("peak"); + + let plotted = doc + .curve + .iter() + .find(|step| step.concurrency == peak.concurrency) + .expect("peak concurrency must appear in the curve"); + assert_eq!(plotted.rps, peak.rps); + assert_eq!(plotted.latency.p99, peak.latency.p99); + assert_eq!(plotted.err, peak.err); + assert_eq!(plotted.cpu, peak.cpu); + } + + #[test] + fn no_qualifying_step_reports_no_peak_at_all() { + let steps = [ + point(8, 1_000.0, 80.0, 0.0), + point(16, 1_200.0, 190.0, 0.0), + point(32, 1_150.0, 400.0, 0.0), + ]; + let doc = measured(&steps, 50.0, 0.01); + + assert_eq!(doc.outcome, Outcome::SloNeverMet); + // The smallest step must never be promoted to a peak it did not earn. + assert!(doc.peak.is_none()); + assert!(doc.lower_bound_rps.is_none()); + assert_eq!(doc.curve.len(), 3); + assert!(doc.curve.iter().all(|step| !step.slo_met)); + } + + #[test] + fn a_ramp_that_never_breaks_reports_a_lower_bound_not_a_peak() { + let steps = [ + point(8, 1_000.0, 4.0, 0.0), + point(16, 2_000.0, 8.0, 0.0), + point(32, 4_000.0, 16.0, 0.0), + ]; + let doc = measured(&steps, 50.0, 0.01); + + assert_eq!(doc.outcome, Outcome::DidNotSaturate); + assert!(doc.peak.is_none()); + assert_eq!(doc.lower_bound_rps, Some(4_000.0)); + } + + /// The ramp reaching its end inside the SLO is what makes an outcome + /// `did_not_saturate` — not where the maximum happened to land. + #[test] + fn an_early_maximum_does_not_by_itself_mean_saturated() { + let steps = [ + point(8, 5_000.0, 2.0, 0.0), + point(16, 3_000.0, 4.0, 0.0), + point(32, 2_800.0, 9.0, 0.0), + ]; + let doc = measured(&steps, 50.0, 0.01); + + assert_eq!(doc.outcome, Outcome::DidNotSaturate); + assert_eq!(doc.lower_bound_rps, Some(5_000.0)); + } + + #[test] + fn an_over_error_step_is_disqualified_and_cannot_be_the_peak() { + let steps = [ + point(8, 1_000.0, 4.0, 0.0), + point(16, 2_000.0, 8.0, 0.0), + // Fast, well inside the SLO, but failing 3.2% of requests. + point(32, 3_800.0, 12.0, 0.032), + point(64, 3_900.0, 90.0, 0.05), + ]; + let doc = measured(&steps, 50.0, 0.01); + + assert_eq!(doc.outcome, Outcome::Saturated); + let peak = doc.peak.expect("peak"); + assert_eq!(peak.concurrency, 16, "a disqualified step cannot be peak"); + assert_eq!( + doc.curve[2].disqualified.as_deref(), + Some("error rate 3.20% exceeds limit 1.00%") + ); + // Disqualification is recorded, never silently skipped. + assert!(doc.curve[2].slo_met, "it met the SLO; it failed on errors"); + assert!(doc.curve[0].disqualified.is_none()); + } + + #[test] + fn every_step_disqualified_by_errors_yields_no_peak() { + let steps = [point(8, 1_000.0, 4.0, 0.5), point(16, 900.0, 8.0, 0.6)]; + let doc = measured(&steps, 50.0, 0.01); + + assert_eq!(doc.outcome, Outcome::SloNeverMet); + assert!(doc.peak.is_none()); + assert!(doc.curve.iter().all(|step| step.disqualified.is_some())); + } + + #[test] + fn steps_are_medianed_across_trials() { + let a = [point(8, 1_000.0, 10.0, 0.0), point(16, 1_500.0, 90.0, 0.0)]; + let b = [point(8, 1_400.0, 12.0, 0.0), point(16, 1_600.0, 95.0, 0.0)]; + let c = [point(8, 1_200.0, 11.0, 0.0), point(16, 1_550.0, 92.0, 0.0)]; + + let doc = measure( + "t", + &spec(50.0), + &limits(0.01), + &[a.as_slice(), b.as_slice(), c.as_slice()], + ) + .expect("measure"); + + assert_eq!(doc.curve[0].rps, 1_200.0); + assert_eq!(doc.curve[0].latency.p99, 11.0); + assert_eq!(doc.outcome, Outcome::Saturated); + assert_eq!(doc.peak.expect("peak").rps, 1_200.0); + } + + #[test] + fn trials_that_ran_different_ladders_are_refused() { + let a = [point(8, 1_000.0, 10.0, 0.0), point(16, 1_500.0, 20.0, 0.0)]; + let b = [point(8, 1_000.0, 10.0, 0.0), point(32, 1_500.0, 20.0, 0.0)]; + + let err = measure( + "drizzle-rs-sqlite", + &spec(50.0), + &limits(0.01), + &[a.as_slice(), b.as_slice()], + ) + .expect_err("mismatched ladders must fail"); + assert!( + err.msg.contains("must measure the same steps"), + "{}", + err.msg + ); + } + + #[test] + fn a_trial_with_no_steps_is_refused() { + let err = measure("t", &spec(50.0), &limits(0.01), &[&[]]) + .expect_err("an empty ladder must fail"); + assert!(err.msg.contains("no saturation steps"), "{}", err.msg); + } + + /// A missing percentile is a missing measurement. Filling it from a + /// neighbouring percentile would publish one number under another's name. + #[test] + fn a_step_missing_a_percentile_is_refused() { + let mut steps = [point(8, 1_000.0, 10.0, 0.0), point(16, 1_500.0, 20.0, 0.0)]; + steps[1].latency.p50 = None; + let err = measure("t", &spec(50.0), &limits(0.01), &[steps.as_slice()]) + .expect_err("missing p50 must fail"); + assert!(err.msg.contains("without a p50 latency"), "{}", err.msg); + + let mut steps = [point(8, 1_000.0, 10.0, 0.0), point(16, 1_500.0, 20.0, 0.0)]; + steps[0].vus = None; + let err = measure("t", &spec(50.0), &limits(0.01), &[steps.as_slice()]) + .expect_err("missing vus must fail"); + assert!(err.msg.contains("without a concurrency"), "{}", err.msg); + } + + #[test] + fn the_declared_metric_decides_the_slo() { + // p99 = 60 breaches a 50 ms p99 SLO, but p95 = 48 passes a 50 ms p95 one. + let steps = [point(8, 1_000.0, 60.0, 0.0), point(16, 1_100.0, 300.0, 0.0)]; + + let on_p99 = measured(&steps, 50.0, 0.01); + assert_eq!(on_p99.outcome, Outcome::SloNeverMet); + + let on_p95 = measure( + "t", + &SaturationSpec { + slo: Slo { + metric: SloMetric::P95, + ms: 50.0, + }, + }, + &limits(0.01), + &[steps.as_slice()], + ) + .expect("measure"); + assert_eq!(on_p95.outcome, Outcome::Saturated); + assert_eq!(on_p95.peak.expect("peak").concurrency, 8); + } +} diff --git a/bench/runner/tests/runner_smoke.rs b/bench/runner/tests/runner_smoke.rs index 9d11e306..7d1a2b49 100644 --- a/bench/runner/tests/runner_smoke.rs +++ b/bench/runner/tests/runner_smoke.rs @@ -19,6 +19,14 @@ fn checked_in_benchmark_specs_validate() { &root.join("bench/spec/workload.single-throughput.v1.json"), &root.join("docs/benchmark-spec/jsonschema/workload.v1.schema.json"), ); + validate_json_file( + &root.join("bench/spec/workload.saturation.v1.json"), + &root.join("docs/benchmark-spec/jsonschema/workload.v1.schema.json"), + ); + validate_json_file( + &root.join("bench/spec/workload.saturation-preview.v1.json"), + &root.join("docs/benchmark-spec/jsonschema/workload.v1.schema.json"), + ); for path in [ "bench/spec/targets.sqlite.v1.json", @@ -101,7 +109,8 @@ fn run_writes_contract_artifacts() { ) .expect("summary json"); assert!(summary.get("spread").is_some()); - assert!(summary.get("saturation").is_some()); + // This workload declares no capacity measurement, so it reports none. + assert!(summary.get("saturation").is_none()); // The n=5 bootstrap only ever measured its own resampling noise. assert!( summary["spread"].get("ci95").is_none(), @@ -582,6 +591,377 @@ fn run_publish_flag_updates_the_index() { assert_eq!(runs[0]["run_id"], run_id); } +/// `fair.family`, `target_meta[].fair.family` and `harness[].family` are one key +/// space. Consumers key harness lookup on it, so drift between the schemas would +/// silently render every affected row as "harness not declared" — a false +/// negative on exactly the disclosure the block exists to provide. +#[test] +fn the_family_vocabulary_is_one_key_space() { + let root = workspace_root(); + let target_schema = + read_json(&root.join("docs/benchmark-spec/jsonschema/target.v1.schema.json")); + let manifest_schema = + read_json(&root.join("docs/benchmark-spec/jsonschema/run-manifest.v1.schema.json")); + + let vocabulary = |schema: &Value| -> Vec { + schema["$defs"]["family"]["enum"] + .as_array() + .expect("family enum") + .iter() + .map(|value| value.as_str().expect("family id").to_string()) + .collect() + }; + let declared = vocabulary(&target_schema); + assert_eq!( + declared, + vocabulary(&manifest_schema), + "target.v1 and run-manifest.v1 family enums have drifted" + ); + + // Both manifest uses must resolve to that one definition rather than a + // parallel inline copy that could rot independently. + for pointer in [ + "/properties/harness/items/properties/family", + "/$defs/fair/properties/family", + ] { + assert_eq!( + manifest_schema.pointer(pointer).expect(pointer)["$ref"], + "#/$defs/family", + "{pointer} must reference the shared family definition" + ); + } + + // Ids only. A human-readable label frozen into a published artifact can + // never be reworded, so naming stays with the presentation layer. + for id in &declared { + assert!( + id.chars() + .all(|ch| ch.is_ascii_lowercase() || ch.is_ascii_digit() || ch == '-'), + "family id {id} looks like a display label" + ); + } + + // Every checked-in target declares a family from that vocabulary. + for path in [ + "bench/spec/targets.sqlite.v1.json", + "bench/spec/targets.sqlite-ts.v1.json", + "bench/spec/targets.libsql.v1.json", + "bench/spec/targets.turso.v1.json", + "bench/spec/targets.postgres.v1.json", + "bench/spec/targets.postgres-rust-orms.v1.json", + "bench/spec/targets.postgres-ts.v1.json", + "bench/spec/targets.spacetimedb.v1.json", + ] { + for target in read_json(&root.join(path)).as_array().expect("targets") { + let family = target["fair"]["family"].as_str().expect("fair.family"); + assert!( + declared.iter().any(|id| id == family), + "{path}: {family} is not in the family vocabulary" + ); + } + } +} + +/// A saturation workload must produce a schema-valid capacity artifact end to +/// end: a real curve tagged with concurrency, a named outcome, and a peak that +/// is one of the plotted steps. +#[test] +fn a_saturation_run_writes_a_capacity_artifact() { + let tmp = TempDir::new().expect("tmp"); + let root = tmp.path(); + let input = root.join("input"); + let out = root.join("out"); + fs::create_dir_all(&input).expect("mkdir input"); + fs::create_dir_all(&out).expect("mkdir out"); + + write_json(input.join("workload.json"), &saturation_workload_json()); + write_json(input.join("targets.json"), &targets_json()); + write_json( + input.join("requests.json"), + r#"[{"method":"GET","path":"/customer-by-id"}]"#, + ); + + let output = run_cmd( + &[ + "run", + "--suite", + "throughput-http", + "--workload", + input.join("workload.json").to_str().expect("workload path"), + "--targets", + input.join("targets.json").to_str().expect("targets path"), + "--requests", + input.join("requests.json").to_str().expect("requests path"), + "--out", + out.to_str().expect("out path"), + "--trials", + "2", + "--seed", + "42", + ], + true, + ); + let run_dir = out.join("runs").join(extract_run_id(&output)); + + let summary: Value = serde_json::from_str( + &fs::read_to_string( + run_dir + .join("targets") + .join("drizzle-rs-sqlite") + .join("summary.json"), + ) + .expect("summary read"), + ) + .expect("summary json"); + + let saturation = &summary["saturation"]; + assert_eq!(saturation["slo"]["metric"], "p99"); + let outcome = saturation["outcome"].as_str().expect("outcome is required"); + assert!( + ["saturated", "slo_never_met", "did_not_saturate"].contains(&outcome), + "unexpected outcome {outcome}" + ); + + // The curve is the whole ramp: one entry per declared step, ascending, each + // carrying the keys consumers rely on being present. + let curve = saturation["curve"].as_array().expect("curve array"); + assert_eq!( + curve + .iter() + .map(|step| step["concurrency"].as_u64().expect("concurrency")) + .collect::>(), + vec![2, 4, 8] + ); + for step in curve { + assert!(step["slo_met"].is_boolean(), "slo_met must be present"); + // `null`, never omitted — consumers key off the field existing. + assert!( + step.get("disqualified").is_some(), + "disqualified must be present on every step" + ); + assert!(step["latency"]["p99"].as_f64().is_some()); + assert!(step["rps"].as_f64().is_some()); + } + + // Exactly one of peak / lower_bound_rps, decided by the outcome. + match outcome { + "saturated" => { + let concurrency = saturation["peak"]["concurrency"] + .as_u64() + .expect("a saturated run carries a peak"); + assert!(saturation.get("lower_bound_rps").is_none()); + assert!( + curve + .iter() + .any(|step| step["concurrency"].as_u64() == Some(concurrency)), + "peak concurrency must be one of the plotted steps" + ); + } + "did_not_saturate" => { + assert!(saturation.get("peak").is_none()); + assert!(saturation["lower_bound_rps"].as_f64().is_some()); + } + _ => { + assert!(saturation.get("peak").is_none()); + assert!(saturation.get("lower_bound_rps").is_none()); + } + } + + // Per-step raw artifacts are kept: the curve has to be re-derivable. + assert!( + run_dir + .join("targets") + .join("drizzle-rs-sqlite") + .join("raw") + .join("trial") + .join("0.steps.json") + .exists() + ); + + let manifest: Value = serde_json::from_str( + &fs::read_to_string(run_dir.join("manifest.json")).expect("read manifest"), + ) + .expect("manifest json"); + let harness = manifest["harness"].as_array().expect("harness array"); + assert_eq!(harness.len(), 1); + assert_eq!(harness[0]["family"], "sqlite"); + assert_eq!(harness[0]["pool"], 1); + assert_eq!(harness[0]["within_family_identical"], true); + assert!(harness[0]["tuning"].as_str().is_some()); + // The warning list it replaced is gone, not emitted alongside it. + assert!(manifest.get("fairness").is_none()); + + // The emitted harness keys must be the same strings the targets declared: + // consumers join the two, and a drifted key reads as "harness not declared" + // on every affected row rather than failing loudly. + let declared: Vec<&str> = manifest["target_meta"] + .as_array() + .expect("target_meta") + .iter() + .map(|meta| meta["fair"]["family"].as_str().expect("fair.family")) + .collect(); + for block in harness { + let family = block["family"].as_str().expect("harness family"); + assert!( + declared.contains(&family), + "harness family {family} matches no target's fair.family {declared:?}" + ); + for id in block["targets"].as_array().expect("targets") { + let id = id.as_str().expect("target id"); + assert!( + manifest["targets"] + .as_array() + .expect("targets") + .iter() + .any(|t| t == id), + "harness names target {id}, which did not run" + ); + } + } + + let validate = run_cmd( + &["validate", "--run", run_dir.to_str().expect("run path")], + true, + ); + assert_eq!(validate.status.code(), Some(0)); +} + +/// A workload that declared no capacity measurement must not report one. The +/// removed heuristic emitted `saturation` unconditionally, which is how a +/// number bounded by the load generator's own sleep timer ended up published +/// under a capacity name. +#[test] +fn a_run_without_a_saturation_spec_emits_no_saturation_block() { + let tmp = TempDir::new().expect("tmp"); + let root = tmp.path(); + let input = root.join("input"); + let out = root.join("out"); + fs::create_dir_all(&input).expect("mkdir input"); + fs::create_dir_all(&out).expect("mkdir out"); + + write_json(input.join("workload.json"), &workload_json(17)); + write_json(input.join("targets.json"), &targets_json()); + write_json(input.join("requests.json"), r#"[]"#); + + let output = run_cmd( + &[ + "run", + "--suite", + "throughput-http", + "--workload", + input.join("workload.json").to_str().expect("workload path"), + "--targets", + input.join("targets.json").to_str().expect("targets path"), + "--requests", + input.join("requests.json").to_str().expect("requests path"), + "--out", + out.to_str().expect("out path"), + "--trials", + "1", + "--seed", + "42", + ], + true, + ); + + let summary: Value = serde_json::from_str( + &fs::read_to_string( + out.join("runs") + .join(extract_run_id(&output)) + .join("targets") + .join("drizzle-rs-sqlite") + .join("summary.json"), + ) + .expect("summary read"), + ) + .expect("summary json"); + assert!( + summary.get("saturation").is_none(), + "a workload that declared no capacity measurement must not report one" + ); +} + +/// Within a family the harness must be identical, and the failure has to name +/// both targets and the field so the fix is obvious. +#[test] +fn within_family_harness_drift_fails_the_run_before_any_load() { + let tmp = TempDir::new().expect("tmp"); + let root = tmp.path(); + let input = root.join("input"); + let out = root.join("out"); + fs::create_dir_all(&input).expect("mkdir input"); + fs::create_dir_all(&out).expect("mkdir out"); + + // Same family, different pool: exactly the comparison the check exists for. + let drifted = + targets_json().replace(r#""id": "drizzle-rs-sqlite","#, r#""id": "other-sqlite","#); + let drifted = drifted.replace(r#""pool": 1,"#, r#""pool": 4,"#); + let drifted = drifted.replace(r#""max": 1 }"#, r#""max": 4 }"#); + let mut both: Vec = serde_json::from_str(&targets_json()).expect("targets"); + both.extend(serde_json::from_str::>(&drifted).expect("drifted")); + write_json( + input.join("targets.json"), + &serde_json::to_string(&both).expect("targets json"), + ); + write_json(input.join("workload.json"), &workload_json(17)); + write_json(input.join("requests.json"), r#"[]"#); + + let output = run_cmd( + &[ + "run", + "--suite", + "throughput-http", + "--workload", + input.join("workload.json").to_str().expect("workload path"), + "--targets", + input.join("targets.json").to_str().expect("targets path"), + "--requests", + input.join("requests.json").to_str().expect("requests path"), + "--out", + out.to_str().expect("out path"), + ], + false, + ); + + assert_eq!(output.status.code(), Some(3), "invalid_input"); + let stderr = String::from_utf8(output.stderr).expect("utf8"); + assert!(stderr.contains("unfair sqlite comparison"), "{stderr}"); + assert!(stderr.contains("other-sqlite"), "{stderr}"); + assert!(stderr.contains("drizzle-rs-sqlite"), "{stderr}"); + assert!(stderr.contains("fair.pool=4"), "{stderr}"); + assert!(stderr.contains("fair.pool=1"), "{stderr}"); + // It fails before spending an hour producing an unusable comparison. + assert!(!out.join("runs").exists() || stderr.contains("fair.pool")); +} + +/// Warmup at 2 VUs (stages 0-1), then steps at 2, 4 and 8. +fn saturation_workload_json() -> String { + r#"{ + "version": "v1", + "suite": "throughput-http", + "name": "Saturation", + "load": { "kind": "closed", "executor": "ramping-vus", "unit": "1s", "concurrency": 8 }, + "data": { "name": "base", "seed": 42, "schema": "bench/schema.sql" }, + "shape": { "mode": "single", "endpoint": "/customer-by-id" }, + "stages": [ + { "sec": 1, "vus": 2 }, + { "sec": 1, "vus": 2 }, + { "sec": 2, "vus": 2 }, + { "sec": 1, "vus": 4 }, + { "sec": 2, "vus": 4 }, + { "sec": 1, "vus": 8 }, + { "sec": 2, "vus": 8 } + ], + "warmup_s": 2, + "requests": { "source": "generated", "file": "requests.json", "skip": [] }, + "pacing": { "mode": "none" }, + "sampling": { "cpu_ms": 100, "bucket_s": 1 }, + "limits": { "err": 0.01 }, + "saturation": { "slo": { "metric": "p99", "ms": 250 } } +}"# + .to_string() +} + fn run_cmd(args: &[&str], expect_success: bool) -> std::process::Output { let mut cmd = cargo_bin_cmd!("bench-runner"); if matches!(args.first(), Some(&"run")) && !args.contains(&"--class") { @@ -650,6 +1030,10 @@ fn validate_target_file(path: &Path, schema_path: &Path) { } } +fn read_json(path: &Path) -> Value { + serde_json::from_str(&fs::read_to_string(path).expect("read json")).expect("parse json") +} + fn workspace_root() -> PathBuf { let manifest = PathBuf::from(env!("CARGO_MANIFEST_DIR")); manifest @@ -1120,11 +1504,13 @@ fn targets_json() -> String { "db": { "profile": "sqlite", "hash": "sha256:1111111111111111111111111111111111111111111111111111111111111111" }, "wire": { "format": "json" }, "fair": { + "family": "sqlite", "workers": 1, "pool": 1, "db": "sqlite", "schema": "sha256:2222222222222222222222222222222222222222222222222222222222222222", - "contract": "v1" + "contract": "v1", + "tuning": "WAL journal, temp_store=MEMORY" }, "contract": { "ver": "v1" }, "parity": { diff --git a/bench/spec/targets.libsql.v1.json b/bench/spec/targets.libsql.v1.json index b6a91370..be590aab 100644 --- a/bench/spec/targets.libsql.v1.json +++ b/bench/spec/targets.libsql.v1.json @@ -39,11 +39,13 @@ "format": "json" }, "fair": { + "family": "libsql", "workers": 1, "pool": 8, "db": "sqlite", "schema": "sha256:67d569fc52dca6de6f2747867f2e7abe8f4cc736f2c2454f9da5f6e78901759d", - "contract": "v1" + "contract": "v1", + "tuning": "WAL journal, temp_store=MEMORY, read-only connections (query_only=ON)" }, "contract": { "ver": "v1" @@ -108,11 +110,13 @@ "format": "json" }, "fair": { + "family": "libsql", "workers": 1, "pool": 8, "db": "sqlite", "schema": "sha256:67d569fc52dca6de6f2747867f2e7abe8f4cc736f2c2454f9da5f6e78901759d", - "contract": "v1" + "contract": "v1", + "tuning": "WAL journal, temp_store=MEMORY, read-only connections (query_only=ON)" }, "contract": { "ver": "v1" @@ -177,11 +181,13 @@ "format": "json" }, "fair": { + "family": "libsql", "workers": 1, "pool": 8, "db": "sqlite", "schema": "sha256:67d569fc52dca6de6f2747867f2e7abe8f4cc736f2c2454f9da5f6e78901759d", - "contract": "v1" + "contract": "v1", + "tuning": "WAL journal, temp_store=MEMORY, read-only connections (query_only=ON)" }, "contract": { "ver": "v1" diff --git a/bench/spec/targets.postgres-rust-orms.v1.json b/bench/spec/targets.postgres-rust-orms.v1.json index 0a8d7bb4..70e4cc1e 100644 --- a/bench/spec/targets.postgres-rust-orms.v1.json +++ b/bench/spec/targets.postgres-rust-orms.v1.json @@ -39,11 +39,13 @@ "format": "binary" }, "fair": { + "family": "postgres", "workers": 1, "pool": 8, "db": "postgres", "schema": "sha256:67d569fc52dca6de6f2747867f2e7abe8f4cc736f2c2454f9da5f6e78901759d", - "contract": "v1" + "contract": "v1", + "tuning": "stock postgres:18-alpine, no server-side tuning applied" }, "contract": { "ver": "v1" @@ -98,11 +100,13 @@ "format": "binary" }, "fair": { + "family": "postgres", "workers": 1, "pool": 8, "db": "postgres", "schema": "sha256:67d569fc52dca6de6f2747867f2e7abe8f4cc736f2c2454f9da5f6e78901759d", - "contract": "v1" + "contract": "v1", + "tuning": "stock postgres:18-alpine, no server-side tuning applied" }, "contract": { "ver": "v1" @@ -158,11 +162,13 @@ "format": "binary" }, "fair": { + "family": "postgres", "workers": 1, "pool": 8, "db": "postgres", "schema": "sha256:67d569fc52dca6de6f2747867f2e7abe8f4cc736f2c2454f9da5f6e78901759d", - "contract": "v1" + "contract": "v1", + "tuning": "stock postgres:18-alpine, no server-side tuning applied" }, "contract": { "ver": "v1" @@ -217,11 +223,13 @@ "format": "binary" }, "fair": { + "family": "postgres", "workers": 1, "pool": 8, "db": "postgres", "schema": "sha256:67d569fc52dca6de6f2747867f2e7abe8f4cc736f2c2454f9da5f6e78901759d", - "contract": "v1" + "contract": "v1", + "tuning": "stock postgres:18-alpine, no server-side tuning applied" }, "contract": { "ver": "v1" diff --git a/bench/spec/targets.postgres-ts.v1.json b/bench/spec/targets.postgres-ts.v1.json index d55b0909..3e58cbf1 100644 --- a/bench/spec/targets.postgres-ts.v1.json +++ b/bench/spec/targets.postgres-ts.v1.json @@ -38,11 +38,13 @@ "format": "binary" }, "fair": { + "family": "postgres-ts", "workers": 1, "pool": 8, "db": "postgres", "schema": "sha256:67d569fc52dca6de6f2747867f2e7abe8f4cc736f2c2454f9da5f6e78901759d", - "contract": "v1" + "contract": "v1", + "tuning": "stock postgres:18-alpine, no server-side tuning applied" }, "contract": { "ver": "v1" @@ -100,11 +102,13 @@ "format": "binary" }, "fair": { + "family": "postgres-ts", "workers": 1, "pool": 8, "db": "postgres", "schema": "sha256:67d569fc52dca6de6f2747867f2e7abe8f4cc736f2c2454f9da5f6e78901759d", - "contract": "v1" + "contract": "v1", + "tuning": "stock postgres:18-alpine, no server-side tuning applied" }, "contract": { "ver": "v1" @@ -161,11 +165,13 @@ "format": "json" }, "fair": { + "family": "postgres-ts", "workers": 1, "pool": 8, "db": "postgres", "schema": "sha256:67d569fc52dca6de6f2747867f2e7abe8f4cc736f2c2454f9da5f6e78901759d", - "contract": "v1" + "contract": "v1", + "tuning": "stock postgres:18-alpine, no server-side tuning applied" }, "contract": { "ver": "v1" diff --git a/bench/spec/targets.postgres.v1.json b/bench/spec/targets.postgres.v1.json index 701d68f5..3ca1c152 100644 --- a/bench/spec/targets.postgres.v1.json +++ b/bench/spec/targets.postgres.v1.json @@ -38,11 +38,13 @@ "format": "json" }, "fair": { + "family": "postgres", "workers": 1, "pool": 8, "db": "postgres", "schema": "sha256:67d569fc52dca6de6f2747867f2e7abe8f4cc736f2c2454f9da5f6e78901759d", - "contract": "v1" + "contract": "v1", + "tuning": "stock postgres:18-alpine, no server-side tuning applied" }, "contract": { "ver": "v1" @@ -106,11 +108,13 @@ "format": "json" }, "fair": { + "family": "postgres", "workers": 1, "pool": 8, "db": "postgres", "schema": "sha256:67d569fc52dca6de6f2747867f2e7abe8f4cc736f2c2454f9da5f6e78901759d", - "contract": "v1" + "contract": "v1", + "tuning": "stock postgres:18-alpine, no server-side tuning applied" }, "contract": { "ver": "v1" @@ -174,11 +178,13 @@ "format": "json" }, "fair": { + "family": "postgres", "workers": 1, "pool": 8, "db": "postgres", "schema": "sha256:67d569fc52dca6de6f2747867f2e7abe8f4cc736f2c2454f9da5f6e78901759d", - "contract": "v1" + "contract": "v1", + "tuning": "stock postgres:18-alpine, no server-side tuning applied" }, "contract": { "ver": "v1" @@ -242,11 +248,13 @@ "format": "json" }, "fair": { + "family": "postgres", "workers": 1, "pool": 8, "db": "postgres", "schema": "sha256:67d569fc52dca6de6f2747867f2e7abe8f4cc736f2c2454f9da5f6e78901759d", - "contract": "v1" + "contract": "v1", + "tuning": "stock postgres:18-alpine, no server-side tuning applied" }, "contract": { "ver": "v1" @@ -309,11 +317,13 @@ "format": "json" }, "fair": { + "family": "postgres", "workers": 1, "pool": 8, "db": "postgres", "schema": "sha256:67d569fc52dca6de6f2747867f2e7abe8f4cc736f2c2454f9da5f6e78901759d", - "contract": "v1" + "contract": "v1", + "tuning": "stock postgres:18-alpine, no server-side tuning applied" }, "contract": { "ver": "v1" diff --git a/bench/spec/targets.spacetimedb.v1.json b/bench/spec/targets.spacetimedb.v1.json index cb354b77..b1cf88f3 100644 --- a/bench/spec/targets.spacetimedb.v1.json +++ b/bench/spec/targets.spacetimedb.v1.json @@ -38,11 +38,13 @@ "format": "text" }, "fair": { + "family": "spacetimedb", "workers": 1, "pool": 4, "db": "spacetimedb", "schema": "sha256:67d569fc52dca6de6f2747867f2e7abe8f4cc736f2c2454f9da5f6e78901759d", - "contract": "v1" + "contract": "v1", + "tuning": "stock spacetimedb standalone, default module storage" }, "contract": { "ver": "v1" @@ -98,11 +100,13 @@ "format": "bsatn" }, "fair": { + "family": "spacetimedb", "workers": 1, "pool": 1, "db": "spacetimedb", "schema": "sha256:67d569fc52dca6de6f2747867f2e7abe8f4cc736f2c2454f9da5f6e78901759d", - "contract": "v1" + "contract": "v1", + "tuning": "stock spacetimedb standalone, default module storage" }, "contract": { "ver": "v1" diff --git a/bench/spec/targets.sqlite-ts.v1.json b/bench/spec/targets.sqlite-ts.v1.json index 4de147be..3f01a417 100644 --- a/bench/spec/targets.sqlite-ts.v1.json +++ b/bench/spec/targets.sqlite-ts.v1.json @@ -39,11 +39,13 @@ "format": "json" }, "fair": { + "family": "sqlite-ts", "workers": 1, "pool": 1, "db": "sqlite", "schema": "sha256:67d569fc52dca6de6f2747867f2e7abe8f4cc736f2c2454f9da5f6e78901759d", - "contract": "v1" + "contract": "v1", + "tuning": "WAL journal, synchronous=NORMAL, temp_store=MEMORY, read-only connections (query_only=ON)" }, "contract": { "ver": "v1" @@ -112,11 +114,13 @@ "format": "json" }, "fair": { + "family": "sqlite-ts", "workers": 1, "pool": 1, "db": "sqlite", "schema": "sha256:67d569fc52dca6de6f2747867f2e7abe8f4cc736f2c2454f9da5f6e78901759d", - "contract": "v1" + "contract": "v1", + "tuning": "WAL journal, synchronous=NORMAL, temp_store=MEMORY, read-only connections (query_only=ON)" }, "contract": { "ver": "v1" diff --git a/bench/spec/targets.sqlite.v1.json b/bench/spec/targets.sqlite.v1.json index 3ff8e687..e69287e9 100644 --- a/bench/spec/targets.sqlite.v1.json +++ b/bench/spec/targets.sqlite.v1.json @@ -38,11 +38,13 @@ "format": "json" }, "fair": { + "family": "sqlite", "workers": 1, "pool": 8, "db": "sqlite", "schema": "sha256:67d569fc52dca6de6f2747867f2e7abe8f4cc736f2c2454f9da5f6e78901759d", - "contract": "v1" + "contract": "v1", + "tuning": "WAL journal, synchronous=NORMAL, temp_store=MEMORY, read-only connections (query_only=ON)" }, "contract": { "ver": "v1" @@ -106,11 +108,13 @@ "format": "json" }, "fair": { + "family": "sqlite", "workers": 1, "pool": 8, "db": "sqlite", "schema": "sha256:67d569fc52dca6de6f2747867f2e7abe8f4cc736f2c2454f9da5f6e78901759d", - "contract": "v1" + "contract": "v1", + "tuning": "WAL journal, synchronous=NORMAL, temp_store=MEMORY, read-only connections (query_only=ON)" }, "contract": { "ver": "v1" @@ -174,11 +178,13 @@ "format": "json" }, "fair": { + "family": "sqlite", "workers": 1, "pool": 8, "db": "sqlite", "schema": "sha256:67d569fc52dca6de6f2747867f2e7abe8f4cc736f2c2454f9da5f6e78901759d", - "contract": "v1" + "contract": "v1", + "tuning": "WAL journal, synchronous=NORMAL, temp_store=MEMORY, read-only connections (query_only=ON)" }, "contract": { "ver": "v1" @@ -241,11 +247,13 @@ "format": "json" }, "fair": { + "family": "sqlite", "workers": 1, "pool": 8, "db": "sqlite", "schema": "sha256:67d569fc52dca6de6f2747867f2e7abe8f4cc736f2c2454f9da5f6e78901759d", - "contract": "v1" + "contract": "v1", + "tuning": "WAL journal, synchronous=NORMAL, temp_store=MEMORY, read-only connections (query_only=ON)" }, "contract": { "ver": "v1" diff --git a/bench/spec/targets.turso.v1.json b/bench/spec/targets.turso.v1.json index 938d3e98..3c68ed94 100644 --- a/bench/spec/targets.turso.v1.json +++ b/bench/spec/targets.turso.v1.json @@ -38,11 +38,13 @@ "format": "json" }, "fair": { + "family": "turso", "workers": 1, "pool": 4, "db": "sqlite", "schema": "sha256:67d569fc52dca6de6f2747867f2e7abe8f4cc736f2c2454f9da5f6e78901759d", - "contract": "v1" + "contract": "v1", + "tuning": "MVCC journal (journal_mode=mvcc), temp_store=MEMORY" }, "contract": { "ver": "v1" @@ -106,11 +108,13 @@ "format": "json" }, "fair": { + "family": "turso", "workers": 1, "pool": 4, "db": "sqlite", "schema": "sha256:67d569fc52dca6de6f2747867f2e7abe8f4cc736f2c2454f9da5f6e78901759d", - "contract": "v1" + "contract": "v1", + "tuning": "MVCC journal (journal_mode=mvcc), temp_store=MEMORY" }, "contract": { "ver": "v1" @@ -174,11 +178,13 @@ "format": "json" }, "fair": { + "family": "turso", "workers": 1, "pool": 4, "db": "sqlite", "schema": "sha256:67d569fc52dca6de6f2747867f2e7abe8f4cc736f2c2454f9da5f6e78901759d", - "contract": "v1" + "contract": "v1", + "tuning": "MVCC journal (journal_mode=mvcc), temp_store=MEMORY" }, "contract": { "ver": "v1" @@ -243,11 +249,13 @@ "format": "json" }, "fair": { + "family": "turso", "workers": 1, "pool": 4, "db": "sqlite", "schema": "sha256:67d569fc52dca6de6f2747867f2e7abe8f4cc736f2c2454f9da5f6e78901759d", - "contract": "v1" + "contract": "v1", + "tuning": "MVCC journal (journal_mode=mvcc), temp_store=MEMORY" }, "contract": { "ver": "v1" diff --git a/bench/spec/workload.saturation-preview.v1.json b/bench/spec/workload.saturation-preview.v1.json new file mode 100644 index 00000000..a642914c --- /dev/null +++ b/bench/spec/workload.saturation-preview.v1.json @@ -0,0 +1,57 @@ +{ + "version": "v1", + "suite": "throughput-http", + "name": "Saturation Preview", + "load": { + "kind": "closed", + "executor": "ramping-vus", + "unit": "1s", + "concurrency": 1024 + }, + "data": { + "name": "bench", + "seed": 42, + "schema": "bench/schema.sql" + }, + "shape": { + "mode": "single", + "endpoint": "/customer-by-id" + }, + "stages": [ + { "sec": 2, "vus": 4 }, + { "sec": 8, "vus": 4 }, + + { "sec": 8, "vus": 4 }, + { "sec": 2, "vus": 16 }, + { "sec": 8, "vus": 16 }, + { "sec": 2, "vus": 64 }, + { "sec": 8, "vus": 64 }, + { "sec": 2, "vus": 256 }, + { "sec": 8, "vus": 256 }, + { "sec": 2, "vus": 1024 }, + { "sec": 8, "vus": 1024 } + ], + "warmup_s": 10, + "requests": { + "source": "generated", + "file": "bench/spec/requests.empty.v1.json", + "skip": [] + }, + "pacing": { + "mode": "none" + }, + "sampling": { + "cpu_ms": 200, + "bucket_s": 1 + }, + "limits": { + "err": 0.01, + "p95": null + }, + "saturation": { + "slo": { + "metric": "p99", + "ms": 25 + } + } +} diff --git a/bench/spec/workload.saturation.v1.json b/bench/spec/workload.saturation.v1.json new file mode 100644 index 00000000..2ca5a9ad --- /dev/null +++ b/bench/spec/workload.saturation.v1.json @@ -0,0 +1,65 @@ +{ + "version": "v1", + "suite": "throughput-http", + "name": "Saturation", + "load": { + "kind": "closed", + "executor": "ramping-vus", + "unit": "1s", + "concurrency": 1024 + }, + "data": { + "name": "bench", + "seed": 42, + "schema": "bench/schema.sql" + }, + "shape": { + "mode": "single", + "endpoint": "/customer-by-id" + }, + "stages": [ + { "sec": 5, "vus": 4 }, + { "sec": 15, "vus": 4 }, + + { "sec": 20, "vus": 4 }, + { "sec": 5, "vus": 8 }, + { "sec": 20, "vus": 8 }, + { "sec": 5, "vus": 16 }, + { "sec": 20, "vus": 16 }, + { "sec": 5, "vus": 32 }, + { "sec": 20, "vus": 32 }, + { "sec": 5, "vus": 64 }, + { "sec": 20, "vus": 64 }, + { "sec": 5, "vus": 128 }, + { "sec": 20, "vus": 128 }, + { "sec": 5, "vus": 256 }, + { "sec": 20, "vus": 256 }, + { "sec": 5, "vus": 512 }, + { "sec": 20, "vus": 512 }, + { "sec": 5, "vus": 1024 }, + { "sec": 20, "vus": 1024 } + ], + "warmup_s": 20, + "requests": { + "source": "generated", + "file": "bench/spec/requests.empty.v1.json", + "skip": [] + }, + "pacing": { + "mode": "none" + }, + "sampling": { + "cpu_ms": 200, + "bucket_s": 1 + }, + "limits": { + "err": 0.01, + "p95": null + }, + "saturation": { + "slo": { + "metric": "p99", + "ms": 25 + } + } +} diff --git a/docs/benchmark-spec/README.md b/docs/benchmark-spec/README.md index 45c7d9a3..16cae6cc 100644 --- a/docs/benchmark-spec/README.md +++ b/docs/benchmark-spec/README.md @@ -73,7 +73,32 @@ they belong with the contract: 8. **Timeseries are concatenated across trials.** `points` is every trial's buckets end to end; segment on `point.trial` rather than assuming one continuous timeline. -9. **A shared `cohort_id` is not a shared machine.** A cohort groups the runs +9. **Capacity is a separate suite with a separate headline.** `summary.saturation` + is written only by an unpaced stepped ramp that declares + `workload.saturation`, and it answers "how much load can this stack carry + while holding a latency SLO". The paced suite answers "what is the latency at + a fixed offered load" and, because of (7), *cannot* answer the first question: + its ceiling is the sleep timer, so every healthy target converges on the same + throughput. The two headlines — **peak throughput** and **throughput at fixed + load** — are never averaged together. A summary with no `saturation` key was + not measured for capacity; that is not zero. See `runner.v1.md` §6c. +10. **The saturation outcome is always named.** Exactly one of `saturated` + (peak found), `did_not_saturate` (ramp ended while still inside the SLO — the + top step is a lower bound, not a peak), or `slo_never_met` (no step + qualified, so there is no peak and none is reported). Steps over + `limits.err` are disqualified from being the peak, stay in the curve, and + carry the reason. +11. **Fairness means two different things.** Within a comparison group + (`fair.family`) the declared harness — workers, pool, tuning — must be + identical or the run fails, so a difference in the numbers is attributable to + the library. Across groups the configurations are free to differ, because + that difference is the stack comparison; `manifest.harness` records the + verified configuration per group so the two are never confused. A group is + usually one database engine but splits where the harness cannot honestly be + equalised (`sqlite-ts` runs a synchronous single-connection API, so it is not + ranked against the pooled Rust SQLite targets). Splitting affects enforcement + and delta scoping, not presentation. See `runner.v1.md` §5a. +12. **A shared `cohort_id` is not a shared machine.** A cohort groups the runs that belong to one logical comparison; when the families ran on separate CI VMs the host fields in `manifest.runner` differ and the numbers are only comparable within a family. Publish-class schedule and dispatch runs put the diff --git a/docs/benchmark-spec/jsonschema/run-manifest.v1.schema.json b/docs/benchmark-spec/jsonschema/run-manifest.v1.schema.json index 26027764..dcdc5901 100644 --- a/docs/benchmark-spec/jsonschema/run-manifest.v1.schema.json +++ b/docs/benchmark-spec/jsonschema/run-manifest.v1.schema.json @@ -381,9 +381,39 @@ } } }, + "harness": { + "type": "array", + "description": "The harness each database family ran under, one entry per family in this run, sorted by family. Within a family the declared harness must be identical or the run fails, so every entry records a verified fact. Across families the values are free to differ — that difference IS the stack comparison — and they are recorded here so a reader cannot mistake a stack difference for a library one.", + "items": { + "type": "object", + "additionalProperties": false, + "required": ["family", "within_family_identical"], + "properties": { + "family": { "$ref": "#/$defs/family" }, + "targets": { + "type": "array", + "description": "Targets whose declared harness was compared for equality. The runner always emits it; it is optional here because the contract mandates only `family` and `within_family_identical`.", + "items": { "type": "string", "pattern": "^[a-z0-9][a-z0-9-]*$" } + }, + "workers": { "type": "integer", "minimum": 1 }, + "pool": { "type": "integer", "minimum": 1 }, + "tuning": { "type": "string", "minLength": 1 }, + "within_family_identical": { + "type": "boolean", + "description": "True when at least one target was compared and they all agreed. False only when there was nothing to compare (every target in the family is exempt), in which case workers/pool/tuning are absent. It is never false because drift was tolerated — drift fails the run." + }, + "exempt": { + "type": "array", + "description": "Targets excluded from the equality check because their configuration is not comparable by design, e.g. an in-process cache with no connection pool. Listed rather than dropped.", + "items": { "type": "string", "pattern": "^[a-z0-9][a-z0-9-]*$" } + } + } + } + }, "fairness": { + "deprecated": true, + "description": "Removed. Recorded within-run fairness mismatches as non-fatal warnings, which is silent in every way that matters: a warning buried in the manifest still let an unequal library comparison ship. Within-family drift is now a hard error and `harness` records the verified configuration instead. Kept so runs recorded before that change still validate.", "type": "array", - "description": "Cross-target fairness findings. Non-fatal: targets ranked against each other should declare identical fair.pool / fair.workers, and each entry records a place where they did not.", "items": { "type": "object", "additionalProperties": false, @@ -418,6 +448,19 @@ } }, "$defs": { + "family": { + "type": "string", + "description": "Comparison-group vocabulary, ONE key space shared by `harness[].family`, `target_meta[].fair.family` and `fair.family` in target.v1. Consumers key harness lookup on it, so drift between the three would silently render every affected row as \"harness not declared\" — the runner has a test that fails if they diverge. Ids only: display labels belong to the presentation layer, because a name frozen into a published artifact can never be reworded.", + "enum": [ + "sqlite", + "sqlite-ts", + "libsql", + "turso", + "postgres", + "postgres-ts", + "spacetimedb" + ] + }, "name_ver": { "type": "object", "additionalProperties": false, @@ -480,13 +523,16 @@ "fair": { "type": "object", "additionalProperties": false, + "description": "The target's own declaration, copied verbatim from its spec. The per-family verified view is `harness`; this is what each target claimed.", "required": ["workers", "pool", "db", "schema", "contract"], "properties": { + "family": { "$ref": "#/$defs/family" }, "workers": { "type": "integer", "minimum": 1 }, "pool": { "type": "integer", "minimum": 1 }, "db": { "type": "string", "minLength": 1 }, "schema": { "type": "string", "pattern": "^sha256:[a-f0-9]{64}$" }, - "contract": { "type": "string", "minLength": 1 } + "contract": { "type": "string", "minLength": 1 }, + "tuning": { "type": "string", "minLength": 1 } } }, "contract": { diff --git a/docs/benchmark-spec/jsonschema/summary.v1.schema.json b/docs/benchmark-spec/jsonschema/summary.v1.schema.json index d8465c6d..a6a347bb 100644 --- a/docs/benchmark-spec/jsonschema/summary.v1.schema.json +++ b/docs/benchmark-spec/jsonschema/summary.v1.schema.json @@ -10,8 +10,7 @@ "suite", "target_id", "primary", - "spread", - "saturation" + "spread" ], "properties": { "version": { @@ -203,26 +202,170 @@ } }, "saturation": { + "$ref": "#/$defs/saturation", + "description": "Peak throughput under the workload's declared latency SLO. Emitted ONLY when the workload declared `saturation`; a paced run has no capacity number and the key is then absent, which consumers must render as \"not measured\" and never as zero. BREAKING: runs recorded before the saturation suite carry a `{knee_rps, knee_p95}` object here instead and no longer validate — that heuristic ran on paced workloads, where throughput is capped by the load generator's own sleep timer, so its \"knee\" described the sleep timer rather than the target. Those keys are gone rather than deprecated in place, so there is exactly one saturation concept in this artifact." + } + }, + "$defs": { + "saturation": { + "type": "object", + "additionalProperties": false, + "required": [ + "slo", + "outcome", + "curve" + ], + "properties": { + "slo": { + "type": "object", + "additionalProperties": false, + "description": "Copied from the workload. The headline is only meaningful quoted with it: \"peak throughput at p99 < 50 ms\".", + "required": [ + "metric", + "ms" + ], + "properties": { + "metric": { + "type": "string", + "enum": [ + "p50", + "p90", + "p95", + "p99" + ] + }, + "ms": { + "type": "number", + "exclusiveMinimum": 0 + } + } + }, + "outcome": { + "type": "string", + "description": "saturated: a qualifying step was found and the ramp's last step failed to qualify, so the ceiling was reached. slo_never_met: no step qualified — every one either breached the SLO or was disqualified by its error rate — so there is NO peak and none is reported. did_not_saturate: the last step still qualified, so the ramp ended before the knee and the best qualifying throughput is a lower bound, not a peak. The outcome depends only on whether the ramp ended inside the SLO, never on where the maximum landed.", + "enum": [ + "saturated", + "slo_never_met", + "did_not_saturate" + ] + }, + "peak": { + "type": "object", + "additionalProperties": false, + "description": "The qualifying step (SLO met AND not disqualified) with the HIGHEST THROUGHPUT; ties break toward the lower concurrency. Not the widest qualifying step: a closed-loop curve dips once the pool saturates, so the last step to survive the SLO is often slower than an earlier one, and reporting it under the name \"peak throughput\" would understate the target. `concurrency` is therefore where the maximum occurred and may sit mid-curve. Present if and only if outcome is `saturated`.", + "required": [ + "concurrency", + "rps", + "latency", + "cpu", + "err" + ], + "properties": { + "concurrency": { + "type": "integer", + "minimum": 1 + }, + "rps": { + "type": "number", + "minimum": 0 + }, + "latency": { + "$ref": "#/$defs/step_latency" + }, + "cpu": { + "type": "number", + "minimum": 0 + }, + "err": { + "type": "number", + "minimum": 0, + "maximum": 1 + } + } + }, + "lower_bound_rps": { + "type": "number", + "minimum": 0, + "description": "The best throughput measured while holding the SLO. Present if and only if outcome is `did_not_saturate`. Read as \"at least N req/s — knee not reached\"; presenting it as a peak would claim a limit that was never found." + }, + "curve": { + "type": "array", + "minItems": 1, + "description": "Every measured step, ascending by concurrency, breaches and disqualifications included. This is the throughput-vs-concurrency curve and it is what makes the headline checkable.", + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "concurrency", + "rps", + "latency", + "err", + "cpu", + "slo_met", + "disqualified" + ], + "properties": { + "concurrency": { + "type": "integer", + "minimum": 1 + }, + "rps": { + "type": "number", + "minimum": 0 + }, + "latency": { + "$ref": "#/$defs/step_latency" + }, + "err": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "cpu": { + "type": "number", + "minimum": 0 + }, + "slo_met": { + "type": "boolean" + }, + "disqualified": { + "type": ["string", "null"], + "description": "Why this step can never be the peak, e.g. \"error rate 3.20% exceeds limit 1.00%\", or null. Always present so consumers can rely on the key; a disqualified step is still charted, never silently dropped." + } + } + } + } + } + }, + "step_latency": { "type": "object", "additionalProperties": false, - "description": "Knee of the throughput/latency curve, computed per trial and medianed. A bucket counts as degraded once its p95 exceeds 2x the median p95 of that trial's lowest-VU hold plateau; the knee is the third consecutive degraded bucket, or the highest-throughput hold bucket if none qualifies.", + "description": "Percentiles over the merged raw samples of that step's hold plateau, medianed across trials. Not a percentile of per-bucket percentiles.", "required": [ - "knee_rps", - "knee_p95" + "p50", + "p90", + "p95", + "p99" ], "properties": { - "knee_rps": { + "p50": { "type": "number", "minimum": 0 }, - "knee_p95": { + "p90": { + "type": "number", + "minimum": 0 + }, + "p95": { + "type": "number", + "minimum": 0 + }, + "p99": { "type": "number", "minimum": 0 } } - } - }, - "$defs": { + }, "avg_peak": { "type": "object", "additionalProperties": false, diff --git a/docs/benchmark-spec/jsonschema/target.v1.schema.json b/docs/benchmark-spec/jsonschema/target.v1.schema.json index 741af4a2..9e48c3c9 100644 --- a/docs/benchmark-spec/jsonschema/target.v1.schema.json +++ b/docs/benchmark-spec/jsonschema/target.v1.schema.json @@ -197,14 +197,22 @@ "fair": { "type": "object", "additionalProperties": false, + "description": "Declared harness. Targets sharing a `family` must declare identical `workers`, `pool` and `tuning`, or the run fails: a ranked library-vs-library table is only meaningful when every entry ran the same harness. Across families the values are free to differ — that difference is the stack comparison — and the manifest records them per family.", "required": [ + "family", "workers", "pool", "db", "schema", - "contract" + "contract", + "tuning" ], "properties": { + "family": { + "type": "string", + "description": "Comparison group: the set of targets claiming to be directly comparable, within which the harness is enforced identical and within-group deltas are scoped. Usually one per database engine, but it splits when the harness genuinely cannot be equalised — `sqlite-ts` is separate from `sqlite`, and `postgres-ts` from `postgres`, because those runtimes are synchronous or single-threaded and matching the Rust pool there would be theatre rather than fairness. Conversely `postgres` deliberately spans two spec files: the builtin drizzle-rs/tokio-postgres targets and the sqlx/diesel/sea-orm/toasty file are all Rust on the same harness, and that is the most valuable library comparison in the suite. Grouping is declared per target precisely so it is not inferred from file layout, and `db.profile` (prepared vs unprepared) and `fair.db` (SQL dialect) identify neither. THIS ENUM IS ONE KEY SPACE with `harness[].family` and `target_meta[].fair.family` in run-manifest.v1; all three must be extended together, along with the dashboard's family vocabulary. Artifacts carry these ids only — display labels belong to the presentation layer, since a name frozen into a published artifact can never be reworded.", + "$ref": "#/$defs/family" + }, "workers": { "type": "integer", "minimum": 1 @@ -224,6 +232,11 @@ "contract": { "type": "string", "minLength": 1 + }, + "tuning": { + "type": "string", + "minLength": 1, + "description": "One-line summary of the engine tuning this target runs under (pragmas, server image, cache settings). Compared for equality within a family, and displayed per family so a reader cannot mistake a tuning difference for a library one." } } }, @@ -272,6 +285,19 @@ } }, "$defs": { + "family": { + "type": "string", + "description": "Comparison-group vocabulary. Kept byte-identical to `harness[].family` and `target_meta[].fair.family` in run-manifest.v1 — the runner has a test that fails if the three drift, because a mismatch would silently render every affected row as \"harness not declared\".", + "enum": [ + "sqlite", + "sqlite-ts", + "libsql", + "turso", + "postgres", + "postgres-ts", + "spacetimedb" + ] + }, "exec": { "type": "object", "additionalProperties": false, diff --git a/docs/benchmark-spec/jsonschema/timeseries.v1.schema.json b/docs/benchmark-spec/jsonschema/timeseries.v1.schema.json index 97db769e..ad5c798d 100644 --- a/docs/benchmark-spec/jsonschema/timeseries.v1.schema.json +++ b/docs/benchmark-spec/jsonschema/timeseries.v1.schema.json @@ -103,6 +103,11 @@ ], "description": "warmup = inside workload.warmup_s; ramp = VU count still interpolating; hold = sitting at the stage's declared VU count. Only hold buckets feed the primary aggregates." }, + "vus": { + "type": "integer", + "minimum": 0, + "description": "Virtual users in flight during this bucket. Under pacing.mode=none there is no think time, so this is also the request concurrency — the x-axis of the saturation curve. Absent on aggregate points, which span several VU counts, and on artifacts written before it was recorded." + }, "requests": { "type": "integer", "minimum": 0, diff --git a/docs/benchmark-spec/jsonschema/workload.v1.schema.json b/docs/benchmark-spec/jsonschema/workload.v1.schema.json index 794824cf..610f1f9a 100644 --- a/docs/benchmark-spec/jsonschema/workload.v1.schema.json +++ b/docs/benchmark-spec/jsonschema/workload.v1.schema.json @@ -279,6 +279,40 @@ "description": "Optional hard ceiling on peak mean-across-cores CPU. Absent: the headroom gate is informational (skip) — closed-loop ramps saturate colocated hosts by design. Set only where genuine headroom is expected (dedicated load/SUT hosts)." } } + }, + "saturation": { + "type": "object", + "additionalProperties": false, + "description": "Declares this workload as a capacity measurement, producing `summary.saturation`. The steps are not listed here: they ARE the hold stages of `stages`, so the ramp has one definition and the spec cannot drift from what runs. The runner refuses the block unless `pacing.mode=none` (a paced run's throughput is capped by its own sleep timer, so it cannot measure capacity), `load.executor=ramping-vus` (each step must ramp in before it holds), `warmup_s` lands on a stage boundary (so no step is measured over part of its plateau), and the ramp has at least three steps whose concurrency strictly increases.", + "required": [ + "slo" + ], + "properties": { + "slo": { + "type": "object", + "additionalProperties": false, + "description": "The latency ceiling a step must stay under to be eligible as the peak. The headline is always quoted with it attached — a throughput number without a latency bound is not a capacity claim.", + "required": [ + "metric", + "ms" + ], + "properties": { + "metric": { + "type": "string", + "enum": [ + "p50", + "p90", + "p95", + "p99" + ] + }, + "ms": { + "type": "number", + "exclusiveMinimum": 0 + } + } + } + } } } } diff --git a/docs/benchmark-spec/runner.v1.md b/docs/benchmark-spec/runner.v1.md index 8de80ada..8eee4cf3 100644 --- a/docs/benchmark-spec/runner.v1.md +++ b/docs/benchmark-spec/runner.v1.md @@ -133,6 +133,7 @@ Runner steps: - `BENCH_REQUESTS_FILE` - `BENCH_POINT_OUT` - `BENCH_TIMESERIES_OUT` + - `BENCH_STEPS_OUT` - `BENCH_POOL_SIZE` (from `target.pool.max`) - `BENCH_WORKERS` (from `target.proc.workers`) 6. aggregate results. @@ -154,10 +155,83 @@ Runtime parity: sizes its tokio runtime to exactly that (default `1`). Without it a Rust target takes every core while the Bun targets it is ranked against take one, and `fair.workers: 1` is false for half the table. -2. Targets ranked in the same run should agree on `fair.pool` and - `fair.workers`. Disagreements are recorded in `manifest.fairness` and emitted - as `warn` events rather than failing the run; targets declaring - `data_access: "in-process-cache"` are exempt, since they have no pool. +2. Harness fairness is enforced per database family — see §5a. + +## 5a. Two-axis fairness + +"Fair" means two different things and blurring them produces a table that looks +comparable and is not. + +**Within a database family — enforced identical.** Every target declares +`fair.family`, `fair.workers`, `fair.pool` and `fair.tuning`. Two targets in the +same family that disagree on workers, pool or tuning **fail the run** with exit +`3`, before any load is generated: + +```text +unfair sqlite comparison: target bun-sqlite declares fair.pool=1 but target +drizzle-rs-sqlite declares fair.pool=8. Targets in the same database family must +run an identical harness, otherwise the ranking compares configurations instead +of libraries. Change one of them, or move it to its own family. +``` + +This replaced a `warn` event plus a `manifest.fairness` entry. A warning buried +in an artifact is silent in every way that matters: it still let an unequal +library comparison ship, and a silently unequal comparison is worse than none. + +**A family can span runs, so the specs are also checked as a set.** `postgres` +covers both `targets.postgres.v1.json` and `targets.postgres-rust-orms.v1.json`, +and outside the publish topology (§13.4) those execute as separate CI jobs +producing separate artifacts. A per-run check would pass on each shard +independently and the drift would surface only when a consumer merged them — +after publish, in someone else's UI. A unit test therefore runs the same +enforcement over the union of every checked-in `targets.*.json`, so a pool +changed in one file fails CI at source. + +**Across families — declared and displayed.** Nothing is constrained. PostgreSQL +over TCP with a pool of 8 and an embedded SQLite with a pool of 1 *should* differ; +that difference is the stack comparison. `manifest.harness` records the verified +configuration per family so a reader cannot mistake a stack difference for a +library one. + +### `fair.family` is a comparison group, not an engine + +A family is **the set of targets claiming to be directly comparable**. It usually +maps one-to-one onto the database engine, but it splits when the harness cannot +honestly be equalised. + +`sqlite-ts` is the worked example. `bun-sqlite` and `drizzle-orm-sqlite` run +`bun:sqlite` — a synchronous API on a single-threaded runtime — so a pool of 8 +there is theatre. Raising their pool to match `targets.sqlite.v1.json` would +cripple them in the name of fairness, which is the opposite of what fairness is +for. drizzle-rs on rusqlite versus drizzle-orm on Bun differs in language, +runtime and concurrency model: that is a **stack** comparison, and stack +comparisons are the across-family axis. Inside `sqlite-ts`, `drizzle-orm-sqlite` +versus `bun-sqlite` is a real library comparison — same runtime, same pool of 1, +same pragmas — which is exactly what a family is for. + +Two consequences: + +1. **Delta scoping follows `fair.family`.** A target's within-family delta is + against the drizzle target *in its own group*, so `bun-sqlite` reads "vs + drizzle-orm on SQLite/Bun", not "vs drizzle-rs on SQLite/Rust". +2. **Presentation does not.** Both groups still appear in one global table with + `SQLite` in the database column. Only enforcement and delta scoping follow + `fair.family`; splitting a group does not hide a target. + +Family is **declared, not inferred**. `db.profile` separates configurations +*inside* a group (prepared vs unprepared) and `fair.db` names the SQL dialect +several engines share, so neither identifies the bracket a target competes in. +It is also not taken from the spec file a target arrived in — publish-class runs +already execute three PostgreSQL spec files back to back inside one job (§13.4). +The vocabulary is a closed enum in `target.v1.schema.json` +(`sqlite`, `sqlite-ts`, `libsql`, `turso`, `postgres`, `spacetimedb`) and must be +extended in lockstep with the dashboard's family vocabulary. + +Targets declaring `data_access: "in-process-cache"` are excluded from the +equality check — a replicated in-process cache has no connection pool to +equalise — and are listed in `harness[].exempt` rather than dropped. A family +whose members are all exempt reports `within_family_identical: false` with no +workers/pool/tuning, meaning "nothing to enforce", never "drift was tolerated". Target lifecycle: @@ -192,11 +266,17 @@ Load output rules: the runner derives an aggregate from it: request-count-weighted `rps`/`err` over `phase=hold` buckets, with percentiles approximated as the median of the per-bucket percentiles. -8. each emitted point should carry `trial`, `stage`, `phase`, and `requests`. - Points without a `phase` are treated as steady state for backwards - compatibility. -9. a series is persisted under `targets//raw/trial/.series.json`, - and the aggregate point under `targets//raw/trial/.point.json`. +8. each emitted point should carry `trial`, `stage`, `phase`, `vus`, and + `requests`. Points without a `phase` are treated as steady state for + backwards compatibility. +9. `load.cmd` may emit one aggregate per hold plateau to `BENCH_STEPS_OUT`: the + same `Point` shape, one entry per step, ascending in time, each tagged with + its `vus`. This is **required** for a workload declaring `saturation` — the + per-step percentiles it carries exist only where the raw samples do, and the + runner refuses to approximate them from the bucket series. +10. a series is persisted under `targets//raw/trial/.series.json`, + the aggregate point under `targets//raw/trial/.point.json`, + and the step list under `targets//raw/trial/.steps.json`. Implementation note: @@ -262,10 +342,164 @@ Output root: 5. `spread.ci95` has been removed. A 512-resample bootstrap over 3-5 trials describes the resampling, not the target. Use `spread.rps`, `spread.p95`, `spread.variance`, and `spread.boxplot`. -6. `saturation` is computed per trial and medianed. A bucket is degraded once - its p95 exceeds twice the median p95 of that trial's lowest-VU hold plateau; - the knee is the third consecutive degraded bucket, or the highest-throughput - hold bucket if none qualifies. It is never computed across a trial boundary. +6. `saturation` is emitted only when the workload declares it — see §6c. + +**Breaking artifact change.** `summary.saturation` previously always carried +`{knee_rps, knee_p95}`. Those keys are **removed**, not deprecated in place, and +runs recorded before this change no longer validate against `summary.v1`. The +heuristic ran on every workload including paced ones, where throughput is capped +by the load generator's own sleep timer — so its "knee" described the sleep timer +rather than the target — and it fell back to the highest-throughput bucket when +no knee appeared, reporting "no knee found" as a knee. The `did_not_saturate` +outcome is the honest expression of that situation. Keeping the old keys beside +the new block would leave two things called "saturation" in one artifact, so the +removal is deliberate: consumers discriminate on `saturation.outcome`, and an +archived run without it reads as "not measured". + +## 6c. Saturation: peak throughput under an SLO + +### Two suites, two headlines + +| | paced suite | saturation suite | +| --- | --- | --- | +| `pacing.mode` | `drizzle-benchmark` | `none` | +| what it answers | latency under a fixed offered load | how much load the stack can carry | +| headline | **throughput at fixed load** | **peak throughput** (always quoted "at p99 < N ms") | +| comparable to | drizzle-benchmarks' published TS numbers | nothing outside this suite | + +They are never averaged together. Each keeps its own number because they measure +different things. + +**Why the paced suite cannot produce a capacity number.** Under +`pacing.mode=drizzle-benchmark` every virtual user sleeps `(iteration % 6) * 75ms` +after each request — a mean of 187.5 ms. Offered load is therefore capped near + +```text +VUs / (mean think time + mean service time) +``` + +With think time two orders of magnitude larger than service time, that ceiling is +essentially `VUs / 0.1875 s` for *every* target. A tenfold difference in service +time moves the result by single-digit percent: every healthy target converges on +the same throughput because the number describes the sleep timer, not the +database. Capacity needs an unpaced measurement, which is why it lives in its own +suite and why the runner refuses `saturation` on a paced workload. + +### Method + +Unpaced closed-loop, stepped concurrency ramp. Each step ramps its VU count in +(`phase=ramp`, charted, not measured) and then holds it (`phase=hold`, measured). +With no think time, N virtual users are N requests in flight, so concurrency is +the x-axis and `rps ≈ N / service_time` until the stack runs out of capacity. + +The steps are not listed separately in the spec: they *are* the hold stages of +`workload.stages`, so there is one definition of the ramp and no way for a +declared ladder to drift from the one that runs. + +Each step's percentiles come from the merged raw samples of that plateau — the +load command emits them to `$BENCH_STEPS_OUT`, where the samples still exist. +A step aggregate is never derived from the bucket series: that would average +per-second percentiles together, which understates the tail and misplaces the +knee. A load command that does not write `$BENCH_STEPS_OUT` fails a saturation +run rather than getting an approximation. + +Across trials each step is the **median** of the per-trial step values, matching +`summary.primary`. + +### The headline and the three outcomes + +A step **qualifies** when its `slo.metric` percentile is at or under `slo.ms` +*and* its error rate is within `limits.err`. The peak is the qualifying step with +the **highest throughput**; ties break toward the lower concurrency, since the +same throughput for fewer in-flight requests is strictly better. Exactly one of +three outcomes is recorded, and all three are first class: + +| `outcome` | when | what the artifact carries | how to say it | +| --- | --- | --- | --- | +| `saturated` | a qualifying step exists and it is not the last step | `peak` | "peak throughput N req/s at p99 < 25 ms" | +| `did_not_saturate` | the last step still qualified | `lower_bound_rps` (the best qualifying throughput), no `peak` | "at least N req/s — knee not reached" | +| `slo_never_met` | no step qualified | neither | "never met the p99 target" | + +`did_not_saturate` is a finding about the *ramp*, not the target: the workload +ended before the knee and must be extended. The top step is a lower bound and is +never presented as a peak. `slo_never_met` covers both "even the smallest step +was too slow" and "every step was disqualified by errors"; the per-step +`disqualified` reason distinguishes them, and there is no peak to report either +way. + +A step over `limits.err` is **disqualified**: it can never be the peak, it stays +in the curve, and it carries the reason string +(`"error rate 3.20% exceeds limit 1.00%"`). It is never silently skipped. + +**Peak throughput means the most throughput, not the most concurrency.** A +closed-loop curve is often non-monotone: throughput dips once the pool saturates +and then flattens, so the *widest* step that held the SLO is frequently slower +than an earlier one that also held it. A measured drizzle-rs SQLite ramp does +31 457 rps at 16 VUs and 28 760 at 256, both inside a 25 ms p99 — reporting the +latter as "peak throughput at p99 < 25 ms" would understate the target by 9% and +point at a worse operating point on *both* axes. `peak.concurrency` is therefore +where the maximum occurred, not the last step to survive the SLO, and it can sit +mid-curve. Whether the ramp found the ceiling is a separate question, answered by +`outcome`: a ramp whose last step still qualified is `did_not_saturate` however +early its maximum landed. + +### The curve + +Every step records `concurrency`, `rps`, `latency` (p50/p90/p95/p99), `err`, +`cpu`, `slo_met` and `disqualified`. rps-vs-concurrency and +latency-vs-concurrency are the actual story; the headline is a single point read +off them, and publishing both is what makes it checkable. + +### Spec rules the runner enforces + +A workload carrying a `saturation` block is rejected unless: + +1. `pacing.mode = none` — see the ceiling above. +2. `load.executor = ramping-vus` — a step measured through its own thundering + herd is not a steady state. +3. `warmup_s` lands on a cumulative stage boundary — otherwise one step is + measured over part of its plateau and is silently shorter than its neighbours. +4. at least **3** measured steps — two points cannot distinguish a knee from + noise. +5. step concurrency strictly increases — the curve is read left to right. + +### Why the shipped ramps look the way they do + +`workload.saturation.v1.json` holds 20 s at each of 4, 8, 16, 32, 64, 128, 256, +512 and 1024 VUs, with a 5 s ramp into each and a 20 s warmup at the starting +concurrency. `workload.saturation-preview.v1.json` keeps the same span with 4x +spacing (4, 16, 64, 256, 1024) and 8 s holds, for PR-sized runs. + +- **Geometric spacing, not linear.** The knee's location is not known in advance + and moves by more than an order of magnitude between an in-process SQLite point + lookup and a PostgreSQL round trip. Doubling brackets the knee within a factor + of two anywhere across a 256x range in nine steps; linear spacing at the same + cost would cover a single octave and miss most of them. Because rps flattens at + the knee, coarse concurrency spacing costs little accuracy in the reported peak + rps. +- **Starts at 4.** Below the largest declared pool (8), so the first step has SLO + headroom even for the slowest stack in the table — `slo_never_met` should mean + something is wrong, not that the ramp started too high. +- **Ends at 1024.** A measured drizzle-rs SQLite ramp reaches p99 ≈ 27 ms at 512 + VUs and ≈ 70 ms at 1024, so the fastest stack in the suite breaches a 25 ms SLO + inside the ramp. `did_not_saturate` should mean the ramp needs extending, not + that it was never long enough for anyone. 1024 is also well inside the + precedent set by `workload.throughput.v1.json`, which runs to 3000 VUs. +- **20 s holds.** At 500 rps — a slow stack at low concurrency — that is 10 000 + samples per step, so the p99 tail has ~100 observations. The preview's 8 s holds + are deliberately thinner; a preview answers "does this pipeline work and is the + shape sane", not "publish this number". +- **`p99 < 25 ms`.** Loose enough that every stack has headroom at the smallest + step on a 4 vCPU runner, tight enough that the fastest stack breaches it before + 1024 VUs. It is also an ordinary web service level, which is the point: the + headline is a capacity claim only because a latency bound is attached to it. +- **Single endpoint (`/customer-by-id`).** A point lookup is where library + overhead is the largest share of service time, so the number is about the + library rather than the query planner. A mixed p99 SLO would in practice be a + threshold on whichever route is heaviest. + +The whole 9-step ramp is 240 s per trial per target, under the 300 s of the +existing paced `workload.throughput.v1.json`. The preview is 58 s. ## 6b. Host and Topology @@ -413,6 +647,15 @@ to every family job. Class resolution: | `full` | yes | `publish` | `workload.throughput.v1.json` | | `single` | no | `full` | `workload.single-throughput.v1.json` | | `single` | yes | `publish` | `workload.single-throughput.v1.json` | +| `saturation` | no | `small` | `workload.saturation-preview.v1.json` | +| `saturation` | yes | `publish` | `workload.saturation.v1.json` | + +The saturation rows are the intended wiring for the capacity suite; the specs and +runner support exist, and `runners.yml` picks them up when the `saturation` size +is added to the dispatch input. They are a second suite, not a replacement: they +produce `summary.saturation` (peak throughput under an SLO, §6c) while the paced +rows produce `summary.primary` (throughput at fixed load). A family needs both to +be described completely, and their headlines are reported separately. A run is publish-class on pushes to `main` and tags, on the weekly schedule, and on a manual dispatch from `main` with `publish_to_r2` set.