diff --git a/.github/actions/keynote-bench-setup/action.yml b/.github/actions/keynote-bench-setup/action.yml new file mode 100644 index 00000000000..95eb487e0f0 --- /dev/null +++ b/.github/actions/keynote-bench-setup/action.yml @@ -0,0 +1,49 @@ +name: Keynote Bench Setup +description: Shared setup for the keynote benchmark job + +inputs: + public_root: + description: Path to the public SpacetimeDB checkout in the caller workspace + required: false + default: . + rust_cache_workspaces: + description: Workspace paths passed to Swatinem/rust-cache + required: true + +runs: + using: composite + steps: + - uses: dsherret/rust-toolchain-file@v1 + + - name: Set default rust toolchain + shell: bash + run: rustup default $(rustup show active-toolchain | cut -d' ' -f1) + + - name: Cache Rust dependencies + uses: Swatinem/rust-cache@v2 + with: + workspaces: ${{ inputs.rust_cache_workspaces }} + shared-key: spacetimedb + save-if: false + prefix-key: v1 + + - name: Check v8 outputs + shell: bash + run: | + find "${CARGO_TARGET_DIR}"/ -type f | grep '[/_]v8' || true + if ! [ -f "${CARGO_TARGET_DIR}"/release/gn_out/obj/librusty_v8.a ]; then + echo "Could not find v8 output file librusty_v8.a; rebuilding manually." + cargo clean --release -p v8 || true + cargo build --release -p v8 + fi + working-directory: ${{ inputs.public_root }} + + - name: Build public keynote benchmark binaries + shell: bash + run: cargo build --release -p spacetimedb-cli -p spacetimedb-standalone + working-directory: ${{ inputs.public_root }} + + - name: Build TypeScript SDK + shell: bash + run: pnpm build + working-directory: ${{ inputs.public_root }}/crates/bindings-typescript diff --git a/.github/actions/setup-pnpm/action.yml b/.github/actions/setup-pnpm/action.yml index 1e438a8c745..2bf9ecdbdec 100644 --- a/.github/actions/setup-pnpm/action.yml +++ b/.github/actions/setup-pnpm/action.yml @@ -1,14 +1,24 @@ name: Set up pnpm description: Install pnpm and configure repository package-age policy before any install inputs: + package_json_file: + description: package.json file used to determine the pnpm version + required: false + default: package.json run_install: description: Run pnpm install after configuring pnpm required: false default: "false" + working_directory: + description: Directory to run pnpm install in + required: false + default: . runs: using: composite steps: - uses: pnpm/action-setup@v4 + with: + package_json_file: ${{ inputs.package_json_file }} - name: Configure pnpm package age policy shell: bash @@ -17,4 +27,5 @@ runs: - name: Install dependencies if: inputs.run_install == 'true' shell: bash + working-directory: ${{ inputs.working_directory }} run: pnpm install diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 040b2e80fa2..95ddb8c65b8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -377,30 +377,6 @@ jobs: with: ref: ${{ env.GIT_REF }} - - uses: dsherret/rust-toolchain-file@v1 - - name: Set default rust toolchain - run: rustup default $(rustup show active-toolchain | cut -d' ' -f1) - - - name: Cache Rust dependencies - uses: Swatinem/rust-cache@v2 - with: - workspaces: ${{ github.workspace }} - shared-key: spacetimedb - save-if: false - prefix-key: v1 - - - name: Check v8 outputs - run: | - find "${CARGO_TARGET_DIR}"/ -type f | grep '[/_]v8' || true - if ! [ -f "${CARGO_TARGET_DIR}"/release/gn_out/obj/librusty_v8.a ]; then - echo "Could not find v8 output file librusty_v8.a; rebuilding manually." - cargo clean --release -p v8 || true - cargo build --release -p v8 - fi - - - name: Build keynote benchmark binaries - run: cargo build --release -p spacetimedb-cli -p spacetimedb-standalone - # Node 24 is the current Active LTS line. - name: Set up Node.js uses: actions/setup-node@v4 @@ -409,11 +385,14 @@ jobs: - uses: ./.github/actions/setup-pnpm with: - run_install: true + package_json_file: package.json + working_directory: . + run_install: "true" - - name: Build TypeScript SDK - run: pnpm build - working-directory: crates/bindings-typescript + - uses: ./.github/actions/keynote-bench-setup + with: + public_root: . + rust_cache_workspaces: ${{ github.workspace }} - name: Run keynote-2 benchmark regression check run: cargo ci keynote-bench diff --git a/Cargo.lock b/Cargo.lock index 27c85d6a60f..792c12e2670 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -915,6 +915,7 @@ dependencies = [ "clap 4.5.50", "duct", "env_logger 0.10.2", + "keynote-bench-harness", "log", "regex", "reqwest", @@ -3819,6 +3820,16 @@ dependencies = [ "cpufeatures", ] +[[package]] +name = "keynote-bench-harness" +version = "0.1.0" +dependencies = [ + "anyhow", + "reqwest", + "serde_json", + "tempfile", +] + [[package]] name = "keynote-benchmarks" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index 10f64434fde..94b1025ecd1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -66,6 +66,7 @@ members = [ "sdks/rust/tests/procedural-view-pk-client", "sdks/rust/tests/event-table-client", "tools/ci", + "tools/keynote-bench-harness", "tools/license-check", "tools/replace-spacetimedb", "tools/generate-client-api", diff --git a/tools/ci/Cargo.toml b/tools/ci/Cargo.toml index deb9b7f4ee7..3374b66b34d 100644 --- a/tools/ci/Cargo.toml +++ b/tools/ci/Cargo.toml @@ -15,4 +15,5 @@ serde_json.workspace = true duct.workspace = true tempfile.workspace = true env_logger.workspace = true +keynote-bench-harness = { path = "../keynote-bench-harness" } spacetimedb-guard.workspace = true diff --git a/tools/ci/src/keynote_bench.rs b/tools/ci/src/keynote_bench.rs index 4aaff211d80..ed49bdd8185 100644 --- a/tools/ci/src/keynote_bench.rs +++ b/tools/ci/src/keynote_bench.rs @@ -1,318 +1,11 @@ -#![allow(clippy::disallowed_macros)] - -use anyhow::{bail, ensure, Context, Result}; -use serde_json::Value; +use anyhow::Result; +use keynote_bench_harness::KeynoteBenchConfig; use spacetimedb_guard::{ensure_binaries_built, SpacetimeDbGuard}; -use std::{ - fs, - path::{Path, PathBuf}, - process::Command, -}; - -const DATABASE_NAME: &str = "test-1"; -const KEYNOTE_DIR: &str = "templates/keynote-2"; -const KEYNOTE_BINDINGS_DIR: &str = "templates/keynote-2/module_bindings"; -const BENCH_SECONDS: &str = "60"; -const BENCH_CONCURRENCY: &str = "64"; -const MAX_INFLIGHT_PER_WORKER: &str = "96"; -const SEED_ACCOUNTS: &str = "100000"; -const SEED_INITIAL_BALANCE: &str = "1000000000000"; -const TRANSFER_REDUCER: &str = "transfer"; -const REDUCER_FUEL_METRIC: &str = "reducer_wasmtime_fuel_used"; -const REDUCER_FUEL_METRIC_TOTAL: &str = "reducer_wasmtime_fuel_used_total"; -const MAX_FUEL_RATIO: f64 = 2.0; - -struct BenchmarkModule { - label: &'static str, - module_dir: &'static str, - min_tps: f64, -} - -struct BenchmarkResult { - label: &'static str, - transfer_fuel_total: f64, -} - -const BENCHMARK_MODULES: &[BenchmarkModule] = &[ - BenchmarkModule { - label: "TypeScript", - module_dir: "templates/keynote-2/spacetimedb", - min_tps: 250_000.0, - }, - BenchmarkModule { - label: "Rust", - module_dir: "templates/keynote-2/rust_module", - min_tps: 250_000.0, - }, -]; pub fn run() -> Result<()> { let cli_path = ensure_binaries_built(); let server = SpacetimeDbGuard::spawn_in_temp_data_dir(); - let cli_config_dir = tempfile::tempdir().context("failed to create temporary CLI config directory")?; - let cli_config_path = cli_config_dir.path().join("config.toml"); - - let mut results = Vec::with_capacity(BENCHMARK_MODULES.len()); - for module in BENCHMARK_MODULES { - results.push(run_module_benchmark( - module, - &cli_path, - &cli_config_path, - &server.host_url, - )?); - } - check_transfer_fuel_ratio(&results)?; - - Ok(()) -} - -fn run_module_benchmark( - module: &BenchmarkModule, - cli_path: &Path, - config_path: &Path, - server_url: &str, -) -> Result { - eprintln!( - "Running keynote benchmark against {} module ({})...", - module.label, module.module_dir - ); - - publish_module(module, cli_path, config_path, server_url)?; - generate_module_bindings(module, cli_path, config_path)?; - seed_accounts(cli_path, config_path, server_url)?; - let runs_dir = tempfile::tempdir().context("failed to create temporary benchmark runs directory")?; - let transfer_fuel_before = transfer_fuel_total(server_url)?; - run_benchmark(module, server_url, runs_dir.path())?; - let transfer_fuel_after = transfer_fuel_total(server_url)?; - let transfer_fuel_total = transfer_fuel_after - transfer_fuel_before; - ensure!( - transfer_fuel_total > 0.0, - "{} keynote benchmark did not record any fuel for the {TRANSFER_REDUCER} reducer", - module.label - ); - - let result_path = find_result_json(runs_dir.path())?; - let result_json = fs::read_to_string(&result_path) - .with_context(|| format!("failed to read benchmark result {}", result_path.display()))?; - let tps = result_tps(&result_json)?; - - if tps < module.min_tps { - eprintln!( - "Keynote perf regression for {} module: throughput {tps:.0} TPS < {:.0} TPS\n\nResult JSON:\n{}", - module.label, module.min_tps, result_json - ); - bail!( - "keynote benchmark throughput for {} module is below threshold", - module.label - ); - } - - println!( - "Keynote perf check passed for {} module: throughput {tps:.0} TPS >= {:.0} TPS; \ - {TRANSFER_REDUCER} fuel total {transfer_fuel_total:.0} ({})", - module.label, - module.min_tps, - result_path.display() - ); - Ok(BenchmarkResult { - label: module.label, - transfer_fuel_total, - }) -} - -fn publish_module(module: &BenchmarkModule, cli_path: &Path, config_path: &Path, server_url: &str) -> Result<()> { - let label = format!("spacetime publish keynote {} module", module.label); - run_cli( - cli_path, - config_path, - &[ - "publish", - "--server", - server_url, - "--module-path", - module.module_dir, - "--yes", - "--clear-database", - DATABASE_NAME, - ], - &label, - ) -} - -fn generate_module_bindings(module: &BenchmarkModule, cli_path: &Path, config_path: &Path) -> Result<()> { - let label = format!("spacetime generate keynote {} TypeScript bindings", module.label); - run_cli( - cli_path, - config_path, - &[ - "generate", - "--lang", - "typescript", - "--out-dir", - KEYNOTE_BINDINGS_DIR, - "--module-path", - module.module_dir, - "--yes", - ], - &label, - ) -} - -fn seed_accounts(cli_path: &Path, config_path: &Path, server_url: &str) -> Result<()> { - run_cli( - cli_path, - config_path, - &[ - "call", - "--server", - server_url, - DATABASE_NAME, - "seed", - SEED_ACCOUNTS, - SEED_INITIAL_BALANCE, - ], - "spacetime call seed", - ) -} - -fn run_cli(cli_path: &Path, config_path: &Path, args: &[&str], label: &str) -> Result<()> { - let mut cmd = Command::new(cli_path); - cmd.arg("--config-path").arg(config_path).args(args); - run_command(&mut cmd, label) -} - -fn run_benchmark(module: &BenchmarkModule, server_url: &str, runs_dir: &Path) -> Result<()> { - let mut cmd = Command::new("pnpm"); - cmd.args([ - "run", - "bench", - DATABASE_NAME, - "--seconds", - BENCH_SECONDS, - "--concurrency", - BENCH_CONCURRENCY, - "--connectors", - "spacetimedb", - ]) - .current_dir(KEYNOTE_DIR) - .env("NODE_ENV", "production") - .env("BENCH_PIPELINED", "1") - .env("MAX_INFLIGHT_PER_WORKER", MAX_INFLIGHT_PER_WORKER) - .env("BENCH_RUNS_DIR", runs_dir) - .env("STDB_URL", server_url) - .env("STDB_MODULE", DATABASE_NAME) - .env("SEED_ACCOUNTS", SEED_ACCOUNTS) - .env("SEED_INITIAL_BALANCE", SEED_INITIAL_BALANCE); - let label = format!("keynote SpacetimeDB benchmark against {} module", module.label); - run_command(&mut cmd, &label) -} - -fn run_command(cmd: &mut Command, label: &str) -> Result<()> { - let status = cmd.status().with_context(|| format!("failed to spawn {label}"))?; - ensure!(status.success(), "{label} failed with status {status}"); - Ok(()) -} - -fn find_result_json(runs_dir: &Path) -> Result { - let mut matches = Vec::new(); - for entry in fs::read_dir(runs_dir).with_context(|| format!("failed to read {}", runs_dir.display()))? { - let entry = entry?; - let path = entry.path(); - let Some(name) = path.file_name().and_then(|name| name.to_str()) else { - continue; - }; - if name.starts_with("test-1-spacetimedb-") && name.ends_with(".json") { - matches.push(path); - } - } - - match matches.len() { - 0 => bail!( - "benchmark did not write a test-1-spacetimedb result JSON in {}", - runs_dir.display() - ), - 1 => Ok(matches.remove(0)), - _ => bail!( - "benchmark wrote multiple test-1-spacetimedb result JSON files in {}: {:?}", - runs_dir.display(), - matches - ), - } -} - -fn result_tps(result_json: &str) -> Result { - let value: Value = serde_json::from_str(result_json).context("failed to parse benchmark result JSON")?; - value - .pointer("/results/0/res/tps") - .and_then(Value::as_f64) - .context("benchmark result JSON is missing results[0].res.tps") -} - -fn transfer_fuel_total(server_url: &str) -> Result { - let metrics_url = format!("{}/v1/metrics", server_url.trim_end_matches('/')); - let metrics = reqwest::blocking::get(&metrics_url) - .with_context(|| format!("failed to fetch metrics from {metrics_url}"))? - .error_for_status() - .with_context(|| format!("metrics request to {metrics_url} failed"))? - .text() - .context("failed to read metrics response body")?; - - let transfer_label = format!(r#"reducer="{TRANSFER_REDUCER}""#); - let mut total = 0.0; - for line in metrics.lines() { - if !is_reducer_fuel_metric_line(line) || !line.contains(&transfer_label) { - continue; - } - let value = line - .split_whitespace() - .nth(1) - .with_context(|| format!("malformed {REDUCER_FUEL_METRIC} metric line: {line}"))? - .parse::() - .with_context(|| format!("invalid {REDUCER_FUEL_METRIC} metric value in line: {line}"))?; - total += value; - } - Ok(total) -} - -fn is_reducer_fuel_metric_line(line: &str) -> bool { - line.starts_with(REDUCER_FUEL_METRIC) || line.starts_with(REDUCER_FUEL_METRIC_TOTAL) -} - -fn check_transfer_fuel_ratio(results: &[BenchmarkResult]) -> Result<()> { - ensure!( - results.len() == 2, - "expected exactly two keynote benchmark results to compare fuel usage, got {}", - results.len() - ); - let [first, second] = results else { - unreachable!("length was checked above") - }; - - let higher = first.transfer_fuel_total.max(second.transfer_fuel_total); - let lower = first.transfer_fuel_total.min(second.transfer_fuel_total); - ensure!( - lower > 0.0, - "keynote benchmark fuel totals must be nonzero: {}={:.0}, {}={:.0}", - first.label, - first.transfer_fuel_total, - second.label, - second.transfer_fuel_total - ); - - let ratio = higher / lower; - println!( - "Keynote transfer fuel comparison: {}={:.0}, {}={:.0}, relative difference {ratio:.2}x", - first.label, first.transfer_fuel_total, second.label, second.transfer_fuel_total - ); - ensure!( - ratio < MAX_FUEL_RATIO, - "keynote benchmark transfer fuel totals differ by {ratio:.2}x, expected strictly less than {MAX_FUEL_RATIO}x: \ - {}={:.0}, {}={:.0}", - first.label, - first.transfer_fuel_total, - second.label, - second.transfer_fuel_total - ); + let server_url = server.host_url.clone(); - Ok(()) + keynote_bench_harness::run(KeynoteBenchConfig::standalone(".", cli_path, server_url)) } diff --git a/tools/keynote-bench-harness/Cargo.toml b/tools/keynote-bench-harness/Cargo.toml new file mode 100644 index 00000000000..175c3e19923 --- /dev/null +++ b/tools/keynote-bench-harness/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "keynote-bench-harness" +version = "0.1.0" +edition.workspace = true +publish = false + +[dependencies] +anyhow.workspace = true +reqwest = { workspace = true, features = ["blocking"] } +serde_json.workspace = true +tempfile.workspace = true diff --git a/tools/keynote-bench-harness/src/lib.rs b/tools/keynote-bench-harness/src/lib.rs new file mode 100644 index 00000000000..693c8c19880 --- /dev/null +++ b/tools/keynote-bench-harness/src/lib.rs @@ -0,0 +1,452 @@ +#![allow(clippy::disallowed_macros)] + +use anyhow::{bail, ensure, Context, Result}; +use serde_json::Value; +use std::{ + fs, + path::{Path, PathBuf}, + process::Command, +}; + +const DATABASE_NAME: &str = "test-1"; +const KEYNOTE_DIR: &str = "templates/keynote-2"; +const KEYNOTE_BINDINGS_DIR: &str = "templates/keynote-2/module_bindings"; +const BENCH_SECONDS: &str = "60"; +const BENCH_CONCURRENCY: &str = "64"; +const MAX_INFLIGHT_PER_WORKER: &str = "96"; +const SEED_ACCOUNTS: &str = "100000"; +const SEED_INITIAL_BALANCE: &str = "1000000000000"; +const TRANSFER_REDUCER: &str = "transfer"; +const REDUCER_FUEL_METRIC: &str = "reducer_wasmtime_fuel_used"; +const REDUCER_FUEL_METRIC_TOTAL: &str = "reducer_wasmtime_fuel_used_total"; +const MAX_FUEL_RATIO: f64 = 2.0; + +#[derive(Debug, Clone)] +pub struct KeynoteBenchConfig { + public_root: PathBuf, + cli_path: PathBuf, + server_url: String, + login_mode: LoginMode, + publish_confirmation: PublishConfirmation, +} + +impl KeynoteBenchConfig { + pub fn standalone( + public_root: impl Into, + cli_path: impl Into, + server_url: impl Into, + ) -> Self { + Self { + public_root: public_root.into(), + cli_path: cli_path.into(), + server_url: server_url.into(), + login_mode: LoginMode::None, + publish_confirmation: PublishConfirmation::Basic, + } + } + + pub fn remote_server( + public_root: impl Into, + cli_path: impl Into, + server_url: impl Into, + ) -> Self { + Self { + public_root: public_root.into(), + cli_path: cli_path.into(), + server_url: server_url.into(), + login_mode: LoginMode::ServerIssued, + publish_confirmation: PublishConfirmation::RemoteMigrateBreakClientsDeleteData, + } + } +} + +#[derive(Debug, Clone, Copy)] +enum LoginMode { + None, + ServerIssued, +} + +#[derive(Debug, Clone, Copy)] +enum PublishConfirmation { + Basic, + RemoteMigrateBreakClientsDeleteData, +} + +impl PublishConfirmation { + fn as_cli_arg(self) -> &'static str { + match self { + Self::Basic => "--yes", + Self::RemoteMigrateBreakClientsDeleteData => "--yes=remote,migrate,break-clients,delete-data", + } + } +} + +struct BenchmarkModule { + label: &'static str, + module_dir: &'static str, + min_tps: f64, +} + +struct BenchmarkResult { + label: &'static str, + transfer_fuel_total: f64, +} + +const BENCHMARK_MODULES: &[BenchmarkModule] = &[ + BenchmarkModule { + label: "TypeScript", + module_dir: "templates/keynote-2/spacetimedb", + min_tps: 250_000.0, + }, + BenchmarkModule { + label: "Rust", + module_dir: "templates/keynote-2/rust_module", + min_tps: 250_000.0, + }, +]; + +pub fn ensure_release_cli_built(root: &Path) -> Result { + let cli_name = if cfg!(windows) { + "spacetimedb-cli.exe" + } else { + "spacetimedb-cli" + }; + let target_dir = std::env::var("CARGO_TARGET_DIR") + .map(PathBuf::from) + .unwrap_or_else(|_| root.join("target")); + let cli_path = target_dir.join("release").join(cli_name); + ensure!( + cli_path.exists(), + "CLI binary not found at {}; build it with `cargo build --release -p spacetimedb-cli`", + cli_path.display() + ); + Ok(cli_path) +} + +pub fn run(config: KeynoteBenchConfig) -> Result<()> { + ensure!( + config.public_root.join(KEYNOTE_DIR).exists(), + "keynote benchmark template not found at {}", + config.public_root.join(KEYNOTE_DIR).display() + ); + + let cli_config_dir = tempfile::tempdir().context("failed to create temporary CLI config directory")?; + let cli_config_path = cli_config_dir.path().join("config.toml"); + + if let LoginMode::ServerIssued = config.login_mode { + login_with_server_issued_identity( + &config.public_root, + &config.cli_path, + &cli_config_path, + &config.server_url, + )?; + } + + let mut results = Vec::with_capacity(BENCHMARK_MODULES.len()); + for module in BENCHMARK_MODULES { + results.push(run_module_benchmark( + module, + &config.public_root, + &config.cli_path, + &cli_config_path, + &config.server_url, + config.publish_confirmation, + )?); + } + check_transfer_fuel_ratio(&results)?; + + Ok(()) +} + +fn run_module_benchmark( + module: &BenchmarkModule, + public_root: &Path, + cli_path: &Path, + config_path: &Path, + server_url: &str, + publish_confirmation: PublishConfirmation, +) -> Result { + eprintln!( + "Running keynote benchmark against {} module ({})...", + module.label, module.module_dir + ); + + publish_module( + module, + public_root, + cli_path, + config_path, + server_url, + publish_confirmation, + )?; + generate_module_bindings(module, public_root, cli_path, config_path)?; + seed_accounts(public_root, cli_path, config_path, server_url)?; + let runs_dir = tempfile::tempdir().context("failed to create temporary benchmark runs directory")?; + let transfer_fuel_before = transfer_fuel_total(server_url)?; + run_benchmark(module, public_root, server_url, runs_dir.path())?; + let transfer_fuel_after = transfer_fuel_total(server_url)?; + let transfer_fuel_total = transfer_fuel_after - transfer_fuel_before; + ensure!( + transfer_fuel_total > 0.0, + "{} keynote benchmark did not record any fuel for the {TRANSFER_REDUCER} reducer", + module.label + ); + + let result_path = find_result_json(runs_dir.path())?; + let result_json = fs::read_to_string(&result_path) + .with_context(|| format!("failed to read benchmark result {}", result_path.display()))?; + let tps = result_tps(&result_json)?; + + if tps < module.min_tps { + eprintln!( + "Keynote perf regression for {} module: throughput {tps:.0} TPS < {:.0} TPS\n\nResult JSON:\n{}", + module.label, module.min_tps, result_json + ); + bail!( + "keynote benchmark throughput for {} module is below threshold", + module.label + ); + } + + println!( + "Keynote perf check passed for {} module: throughput {tps:.0} TPS >= {:.0} TPS; \ + {TRANSFER_REDUCER} fuel total {transfer_fuel_total:.0} ({})", + module.label, + module.min_tps, + result_path.display() + ); + Ok(BenchmarkResult { + label: module.label, + transfer_fuel_total, + }) +} + +fn login_with_server_issued_identity( + public_root: &Path, + cli_path: &Path, + config_path: &Path, + server_url: &str, +) -> Result<()> { + run_cli( + public_root, + cli_path, + config_path, + &["login", "--server-issued-login", server_url], + "spacetime login", + ) +} + +fn publish_module( + module: &BenchmarkModule, + public_root: &Path, + cli_path: &Path, + config_path: &Path, + server_url: &str, + publish_confirmation: PublishConfirmation, +) -> Result<()> { + let label = format!("spacetime publish keynote {} module", module.label); + run_cli( + public_root, + cli_path, + config_path, + &[ + "publish", + "--server", + server_url, + "--module-path", + module.module_dir, + publish_confirmation.as_cli_arg(), + "--clear-database", + DATABASE_NAME, + ], + &label, + ) +} + +fn generate_module_bindings( + module: &BenchmarkModule, + public_root: &Path, + cli_path: &Path, + config_path: &Path, +) -> Result<()> { + let label = format!("spacetime generate keynote {} TypeScript bindings", module.label); + run_cli( + public_root, + cli_path, + config_path, + &[ + "generate", + "--lang", + "typescript", + "--out-dir", + KEYNOTE_BINDINGS_DIR, + "--module-path", + module.module_dir, + "--yes", + ], + &label, + ) +} + +fn seed_accounts(public_root: &Path, cli_path: &Path, config_path: &Path, server_url: &str) -> Result<()> { + run_cli( + public_root, + cli_path, + config_path, + &[ + "call", + "--server", + server_url, + DATABASE_NAME, + "seed", + SEED_ACCOUNTS, + SEED_INITIAL_BALANCE, + ], + "spacetime call seed", + ) +} + +fn run_cli(public_root: &Path, cli_path: &Path, config_path: &Path, args: &[&str], label: &str) -> Result<()> { + let mut cmd = Command::new(cli_path); + cmd.arg("--config-path") + .arg(config_path) + .args(args) + .current_dir(public_root); + run_command(&mut cmd, label) +} + +fn run_benchmark(module: &BenchmarkModule, public_root: &Path, server_url: &str, runs_dir: &Path) -> Result<()> { + let mut cmd = Command::new("pnpm"); + cmd.args([ + "run", + "bench", + DATABASE_NAME, + "--seconds", + BENCH_SECONDS, + "--concurrency", + BENCH_CONCURRENCY, + "--connectors", + "spacetimedb", + ]) + .current_dir(public_root.join(KEYNOTE_DIR)) + .env("NODE_ENV", "production") + .env("BENCH_PIPELINED", "1") + .env("MAX_INFLIGHT_PER_WORKER", MAX_INFLIGHT_PER_WORKER) + .env("BENCH_RUNS_DIR", runs_dir) + .env("STDB_URL", server_url) + .env("STDB_MODULE", DATABASE_NAME) + .env("SEED_ACCOUNTS", SEED_ACCOUNTS) + .env("SEED_INITIAL_BALANCE", SEED_INITIAL_BALANCE); + let label = format!("keynote SpacetimeDB benchmark against {} module", module.label); + run_command(&mut cmd, &label) +} + +fn run_command(cmd: &mut Command, label: &str) -> Result<()> { + let status = cmd.status().with_context(|| format!("failed to spawn {label}"))?; + ensure!(status.success(), "{label} failed with status {status}"); + Ok(()) +} + +fn find_result_json(runs_dir: &Path) -> Result { + let mut matches = Vec::new(); + for entry in fs::read_dir(runs_dir).with_context(|| format!("failed to read {}", runs_dir.display()))? { + let entry = entry?; + let path = entry.path(); + let Some(name) = path.file_name().and_then(|name| name.to_str()) else { + continue; + }; + if name.starts_with("test-1-spacetimedb-") && name.ends_with(".json") { + matches.push(path); + } + } + + match matches.len() { + 0 => bail!( + "benchmark did not write a test-1-spacetimedb result JSON in {}", + runs_dir.display() + ), + 1 => Ok(matches.remove(0)), + _ => bail!( + "benchmark wrote multiple test-1-spacetimedb result JSON files in {}: {:?}", + runs_dir.display(), + matches + ), + } +} + +fn result_tps(result_json: &str) -> Result { + let value: Value = serde_json::from_str(result_json).context("failed to parse benchmark result JSON")?; + value + .pointer("/results/0/res/tps") + .and_then(Value::as_f64) + .context("benchmark result JSON is missing results[0].res.tps") +} + +fn transfer_fuel_total(server_url: &str) -> Result { + let metrics_url = format!("{}/v1/metrics", server_url.trim_end_matches('/')); + let metrics = reqwest::blocking::get(&metrics_url) + .with_context(|| format!("failed to fetch metrics from {metrics_url}"))? + .error_for_status() + .with_context(|| format!("metrics request to {metrics_url} failed"))? + .text() + .context("failed to read metrics response body")?; + + let transfer_label = format!(r#"reducer="{TRANSFER_REDUCER}""#); + let mut total = 0.0; + for line in metrics.lines() { + if !is_reducer_fuel_metric_line(line) || !line.contains(&transfer_label) { + continue; + } + let value = line + .split_whitespace() + .nth(1) + .with_context(|| format!("malformed {REDUCER_FUEL_METRIC} metric line: {line}"))? + .parse::() + .with_context(|| format!("invalid {REDUCER_FUEL_METRIC} metric value in line: {line}"))?; + total += value; + } + Ok(total) +} + +fn is_reducer_fuel_metric_line(line: &str) -> bool { + line.starts_with(REDUCER_FUEL_METRIC) || line.starts_with(REDUCER_FUEL_METRIC_TOTAL) +} + +fn check_transfer_fuel_ratio(results: &[BenchmarkResult]) -> Result<()> { + ensure!( + results.len() == 2, + "expected exactly two keynote benchmark results to compare fuel usage, got {}", + results.len() + ); + let [first, second] = results else { + unreachable!("length was checked above") + }; + + let higher = first.transfer_fuel_total.max(second.transfer_fuel_total); + let lower = first.transfer_fuel_total.min(second.transfer_fuel_total); + ensure!( + lower > 0.0, + "keynote benchmark fuel totals must be nonzero: {}={:.0}, {}={:.0}", + first.label, + first.transfer_fuel_total, + second.label, + second.transfer_fuel_total + ); + + let ratio = higher / lower; + println!( + "Keynote transfer fuel comparison: {}={:.0}, {}={:.0}, relative difference {ratio:.2}x", + first.label, first.transfer_fuel_total, second.label, second.transfer_fuel_total + ); + ensure!( + ratio < MAX_FUEL_RATIO, + "keynote benchmark transfer fuel totals differ by {ratio:.2}x, expected strictly less than {MAX_FUEL_RATIO}x: \ + {}={:.0}, {}={:.0}", + first.label, + first.transfer_fuel_total, + second.label, + second.transfer_fuel_total + ); + + Ok(()) +}