diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e094a63..aa3a911 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,7 +5,10 @@ on: pull_request: branches: [main, develop] schedule: + # Nightly property tests + deploy dry-run - cron: '23 4 * * *' + # Weekly gas baseline rotation (Mondays 05:17 UTC) + - cron: '17 5 * * 1' jobs: changes: @@ -153,7 +156,7 @@ jobs: working-directory: . stellar-nightly: - if: github.event_name == 'schedule' + if: github.event_name == 'schedule' && github.event.schedule == '23 4 * * *' runs-on: ubuntu-latest defaults: run: @@ -168,7 +171,7 @@ jobs: - run: WRAITH_PROPTEST_CASES=16384 cargo test --workspace --test properties stellar-deploy-dryrun: - if: github.event_name == 'schedule' + if: github.event_name == 'schedule' && github.event.schedule == '23 4 * * *' runs-on: ubuntu-latest defaults: run: @@ -224,6 +227,145 @@ jobs: stellar/deploy-dryrun-output.txt stellar/contract-ids.txt + # Run benches on develop, upload results artifact, auto-update PERF.md. + stellar-bench-develop: + if: github.event_name == 'push' && github.ref == 'refs/heads/develop' + runs-on: ubuntu-latest + permissions: + contents: write + defaults: + run: + working-directory: stellar + steps: + - uses: actions/checkout@v4 + with: + token: ${{ secrets.GITHUB_TOKEN }} + + - uses: dtolnay/rust-toolchain@stable + + - name: Run gas bench + env: + GITHUB_SHA: ${{ github.sha }} + run: | + cargo bench -p wraith-stellar-bench --bench gas -- \ + --format json --out bench/results.json + + - name: Upload bench results artifact + uses: actions/upload-artifact@v4 + with: + name: stellar-bench-results + path: stellar/bench/results.json + retention-days: 90 + + - name: Update PERF.md current numbers + run: python3 bench/update_perf_md.py PERF.md bench/results.json + + - name: Commit PERF.md if changed + uses: stefanzweifel/git-auto-commit-action@v5 + with: + commit_message: 'chore(perf): auto-update Stellar PERF.md bench table' + file_pattern: stellar/PERF.md + commit_user_name: github-actions[bot] + commit_user_email: github-actions[bot]@users.noreply.github.com + + # PR gate: fail when any per-op instructions exceed baseline + 5%. + stellar-bench-pr: + needs: changes + if: github.event_name == 'pull_request' && needs.changes.outputs.stellar == 'true' + runs-on: ubuntu-latest + defaults: + run: + working-directory: stellar + steps: + - uses: actions/checkout@v4 + + - uses: dtolnay/rust-toolchain@stable + + - name: Download weekly baseline artifact + id: download-baseline + uses: dawidd6/action-download-artifact@v6 + continue-on-error: true + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + workflow: ci.yml + branch: develop + name: stellar-bench-baseline + path: stellar/bench/artifact-baseline + search_artifacts: true + if_no_artifact_found: warn + + - name: Resolve baseline path + id: baseline + run: | + if [ -f bench/artifact-baseline/baseline.json ]; then + echo "path=bench/artifact-baseline/baseline.json" >> "$GITHUB_OUTPUT" + echo "Using weekly-rotated artifact baseline" + else + echo "path=bench/baseline.json" >> "$GITHUB_OUTPUT" + echo "Using committed bench/baseline.json fallback" + fi + + - name: Run gas bench + env: + GITHUB_SHA: ${{ github.sha }} + run: | + cargo bench -p wraith-stellar-bench --bench gas -- \ + --format json --out bench/results.json + + - name: Compare vs baseline (+5% instructions gate) + run: | + python3 bench/compare.py \ + "${{ steps.baseline.outputs.path }}" \ + bench/results.json \ + --threshold-pct 5 + + - name: Upload PR bench results + if: always() + uses: actions/upload-artifact@v4 + with: + name: stellar-bench-pr-${{ github.event.pull_request.number }} + path: stellar/bench/results.json + retention-days: 30 + + # Weekly baseline rotation to catch slow drift under the 5% PR gate. + stellar-bench-baseline-rotate: + if: github.event_name == 'schedule' && github.event.schedule == '17 5 * * 1' + runs-on: ubuntu-latest + permissions: + contents: write + defaults: + run: + working-directory: stellar + steps: + - uses: actions/checkout@v4 + with: + ref: develop + token: ${{ secrets.GITHUB_TOKEN }} + + - uses: dtolnay/rust-toolchain@stable + + - name: Run gas bench (rotate baseline) + env: + GITHUB_SHA: ${{ github.sha }} + run: | + cargo bench -p wraith-stellar-bench --bench gas -- \ + --format json --out bench/baseline.json + + - name: Upload rotated baseline artifact + uses: actions/upload-artifact@v4 + with: + name: stellar-bench-baseline + path: stellar/bench/baseline.json + retention-days: 90 + + - name: Commit rotated baseline.json + uses: stefanzweifel/git-auto-commit-action@v5 + with: + commit_message: 'chore(perf): rotate Stellar gas bench baseline' + file_pattern: stellar/bench/baseline.json + commit_user_name: github-actions[bot] + commit_user_email: github-actions[bot]@users.noreply.github.com + solana: needs: changes if: needs.changes.outputs.solana == 'true' diff --git a/stellar/PERF.md b/stellar/PERF.md index d764f07..579f926 100644 --- a/stellar/PERF.md +++ b/stellar/PERF.md @@ -4,7 +4,8 @@ Measured on 2026-06-01 with `soroban-sdk = 22.0.0` resolved to `22.0.11`. The reusable harness is in `stellar/bench/` and can be re-run with: ```sh -cargo run -p wraith-stellar-bench +cargo bench -p wraith-stellar-bench --bench gas +# or: cargo run -p wraith-stellar-bench ``` ## How to Read the Units @@ -66,7 +67,8 @@ ownership checks. ## Current Numbers -These are the post-optimization harness results. + +These are the harness results auto-updated from `develop` (measured 2025-06-15, commit `seeded-from-`). | Contract | Function | Parameters | Instructions | Mem bytes | Read entries | Write entries | Read bytes | Write bytes | Event bytes | |---|---|---:|---:|---:|---:|---:|---:|---:|---:| @@ -89,6 +91,19 @@ These are the post-optimization harness results. | wraith-names | resolve | miss | 19766 | 1600 | 1 | 0 | 104 | 0 | 0 | | wraith-names | name_of | hit | 47042 | 5383 | 1 | 0 | 452 | 0 | 0 | | wraith-names | name_of | miss | 21581 | 1513 | 1 | 0 | 104 | 0 | 0 | + + +## Gas Regression Gate + +CI compares PR bench results against the weekly-rotated baseline in +`stellar/bench/baseline.json` (also published as the `stellar-bench-baseline` +workflow artifact). A PR fails when any per-op `instructions` exceeds +baseline + 5%. Re-run locally: + +```sh +cargo bench -p wraith-stellar-bench --bench gas -- --format json --out /tmp/bench.json +python3 bench/compare.py bench/baseline.json /tmp/bench.json +``` ## Batch vs Individual Crossover diff --git a/stellar/bench/Cargo.toml b/stellar/bench/Cargo.toml index a5991a8..f1d2c02 100644 --- a/stellar/bench/Cargo.toml +++ b/stellar/bench/Cargo.toml @@ -9,3 +9,7 @@ stealth-announcer = { path = "../stealth-announcer" } stealth-registry = { path = "../stealth-registry" } stealth-sender = { path = "../stealth-sender" } wraith-names = { path = "../wraith-names" } + +[[bench]] +name = "gas" +harness = false diff --git a/stellar/bench/baseline.json b/stellar/bench/baseline.json new file mode 100644 index 0000000..1359f34 --- /dev/null +++ b/stellar/bench/baseline.json @@ -0,0 +1,235 @@ +{ + "generated_at": "1750000000", + "commit": "seeded-from-perf-md", + "threshold_pct": 5.0, + "results": [ + { + "contract": "stealth-announcer", + "function": "announce", + "params": "metadata_len=0", + "instructions": 15458, + "mem_bytes": 1666, + "read_entries": 1, + "write_entries": 0, + "read_bytes": 104, + "write_bytes": 0, + "events_bytes": 216 + }, + { + "contract": "stealth-announcer", + "function": "announce", + "params": "metadata_len=32", + "instructions": 15458, + "mem_bytes": 1666, + "read_entries": 1, + "write_entries": 0, + "read_bytes": 104, + "write_bytes": 0, + "events_bytes": 248 + }, + { + "contract": "stealth-announcer", + "function": "announce", + "params": "metadata_len=256", + "instructions": 15458, + "mem_bytes": 1666, + "read_entries": 1, + "write_entries": 0, + "read_bytes": 104, + "write_bytes": 0, + "events_bytes": 472 + }, + { + "contract": "stealth-announcer", + "function": "announce", + "params": "metadata_len=1024", + "instructions": 15458, + "mem_bytes": 1666, + "read_entries": 1, + "write_entries": 0, + "read_bytes": 104, + "write_bytes": 0, + "events_bytes": 1240 + }, + { + "contract": "stealth-announcer", + "function": "announce", + "params": "metadata_len=4096", + "instructions": 15458, + "mem_bytes": 1666, + "read_entries": 1, + "write_entries": 0, + "read_bytes": 104, + "write_bytes": 0, + "events_bytes": 4312 + }, + { + "contract": "stealth-registry", + "function": "register_keys", + "params": "first_time", + "instructions": 33345, + "mem_bytes": 4461, + "read_entries": 1, + "write_entries": 2, + "read_bytes": 104, + "write_bytes": 332, + "events_bytes": 188 + }, + { + "contract": "stealth-registry", + "function": "register_keys", + "params": "replacement", + "instructions": 44880, + "mem_bytes": 6553, + "read_entries": 1, + "write_entries": 2, + "read_bytes": 260, + "write_bytes": 332, + "events_bytes": 188 + }, + { + "contract": "stealth-sender", + "function": "send", + "params": "asset=xlm", + "instructions": 182403, + "mem_bytes": 28137, + "read_entries": 5, + "write_entries": 3, + "read_bytes": 1068, + "write_bytes": 520, + "events_bytes": 484 + }, + { + "contract": "stealth-sender", + "function": "send", + "params": "asset=issued", + "instructions": 182355, + "mem_bytes": 28137, + "read_entries": 5, + "write_entries": 3, + "read_bytes": 1068, + "write_bytes": 520, + "events_bytes": 484 + }, + { + "contract": "stealth-sender", + "function": "batch_send", + "params": "batch_size=1", + "instructions": 184674, + "mem_bytes": 28137, + "read_entries": 5, + "write_entries": 3, + "read_bytes": 1068, + "write_bytes": 520, + "events_bytes": 484 + }, + { + "contract": "stealth-sender", + "function": "batch_send", + "params": "batch_size=5", + "instructions": 807519, + "mem_bytes": 120229, + "read_entries": 5, + "write_entries": 7, + "read_bytes": 1068, + "write_bytes": 1416, + "events_bytes": 2420 + }, + { + "contract": "stealth-sender", + "function": "batch_send", + "params": "batch_size=10", + "instructions": 1633634, + "mem_bytes": 245649, + "read_entries": 5, + "write_entries": 12, + "read_bytes": 1068, + "write_bytes": 2536, + "events_bytes": 4840 + }, + { + "contract": "stealth-sender", + "function": "batch_send", + "params": "batch_size=25", + "instructions": 4322337, + "mem_bytes": 690609, + "read_entries": 5, + "write_entries": 27, + "read_bytes": 1068, + "write_bytes": 5896, + "events_bytes": 12100 + }, + { + "contract": "wraith-names", + "function": "register", + "params": "name_len=3", + "instructions": 59792, + "mem_bytes": 6240, + "read_entries": 1, + "write_entries": 2, + "read_bytes": 104, + "write_bytes": 516, + "events_bytes": 204 + }, + { + "contract": "wraith-names", + "function": "register", + "params": "name_len=32", + "instructions": 61413, + "mem_bytes": 6327, + "read_entries": 1, + "write_entries": 2, + "read_bytes": 104, + "write_bytes": 572, + "events_bytes": 232 + }, + { + "contract": "wraith-names", + "function": "resolve", + "params": "hit", + "instructions": 46096, + "mem_bytes": 5456, + "read_entries": 1, + "write_entries": 0, + "read_bytes": 452, + "write_bytes": 0, + "events_bytes": 0 + }, + { + "contract": "wraith-names", + "function": "resolve", + "params": "miss", + "instructions": 19766, + "mem_bytes": 1600, + "read_entries": 1, + "write_entries": 0, + "read_bytes": 104, + "write_bytes": 0, + "events_bytes": 0 + }, + { + "contract": "wraith-names", + "function": "name_of", + "params": "hit", + "instructions": 47042, + "mem_bytes": 5383, + "read_entries": 1, + "write_entries": 0, + "read_bytes": 452, + "write_bytes": 0, + "events_bytes": 0 + }, + { + "contract": "wraith-names", + "function": "name_of", + "params": "miss", + "instructions": 21581, + "mem_bytes": 1513, + "read_entries": 1, + "write_entries": 0, + "read_bytes": 104, + "write_bytes": 0, + "events_bytes": 0 + } + ] +} diff --git a/stellar/bench/benches/gas.rs b/stellar/bench/benches/gas.rs new file mode 100644 index 0000000..a14bed9 --- /dev/null +++ b/stellar/bench/benches/gas.rs @@ -0,0 +1,85 @@ +//! Gas / resource budget bench harness. +//! +//! Run with: +//! ```sh +//! cargo bench -p wraith-stellar-bench --bench gas -- --format json --out results.json +//! ``` + +use std::env; +use std::fs; +use std::path::PathBuf; +use std::process; + +use wraith_stellar_bench::{collect_rows, print_markdown, to_json, BenchMeta}; + +fn main() { + let args: Vec = env::args().skip(1).collect(); + let mut format = "markdown".to_string(); + let mut out_path: Option = None; + + let mut i = 0; + while i < args.len() { + match args[i].as_str() { + "--format" => { + i += 1; + format = args + .get(i) + .cloned() + .unwrap_or_else(|| usage_exit("--format requires a value")); + } + "--out" => { + i += 1; + out_path = Some(PathBuf::from(args.get(i).cloned().unwrap_or_else(|| { + usage_exit("--out requires a path"); + }))); + } + // `cargo bench` may pass Criterion-style leftover args; ignore unknowns + // that start with `--bench` noise, but keep strictness for our flags. + "-h" | "--help" => usage_exit(""), + other if other.starts_with('-') => usage_exit(&format!("unknown argument: {other}")), + _ => {} + } + i += 1; + } + + let rows = collect_rows(); + let meta = BenchMeta::default(); + + let rendered = match format.as_str() { + "markdown" | "md" => wraith_stellar_bench::markdown_table(&rows), + "json" => to_json(&rows, &meta), + other => usage_exit(&format!("unsupported --format: {other}")), + }; + + if let Some(path) = out_path { + if let Some(parent) = path.parent() { + if !parent.as_os_str().is_empty() { + fs::create_dir_all(parent).unwrap_or_else(|e| { + eprintln!("failed to create {}: {e}", parent.display()); + process::exit(1); + }); + } + } + fs::write(&path, &rendered).unwrap_or_else(|e| { + eprintln!("failed to write {}: {e}", path.display()); + process::exit(1); + }); + eprintln!("wrote {}", path.display()); + } + + if format == "markdown" || format == "md" { + print_markdown(&rows); + } else { + print!("{rendered}"); + } +} + +fn usage_exit(msg: &str) -> ! { + if !msg.is_empty() { + eprintln!("error: {msg}"); + } + eprintln!( + "usage: cargo bench -p wraith-stellar-bench --bench gas -- [--format markdown|json] [--out path]" + ); + process::exit(if msg.is_empty() { 0 } else { 2 }); +} diff --git a/stellar/bench/compare.py b/stellar/bench/compare.py new file mode 100755 index 0000000..38ee645 --- /dev/null +++ b/stellar/bench/compare.py @@ -0,0 +1,108 @@ +#!/usr/bin/env python3 +"""Compare current Stellar gas bench results against a baseline. + +Fails (exit 1) when any per-op `instructions` exceeds baseline + threshold +(default 5%). Prints a clear regression diff for CI logs. +""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path +from typing import Any + + +def load_results(path: Path) -> dict[str, dict[str, Any]]: + data = json.loads(path.read_text()) + results = data.get("results") + if not isinstance(results, list): + raise SystemExit(f"{path}: missing 'results' array") + out: dict[str, dict[str, Any]] = {} + for row in results: + key = f"{row['contract']}::{row['function']}::{row['params']}" + out[key] = row + return out + + +def pct_delta(current: float, baseline: float) -> float: + if baseline == 0: + return 0.0 if current == 0 else float("inf") + return ((current - baseline) / baseline) * 100.0 + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("baseline", type=Path, help="baseline JSON from develop") + parser.add_argument("current", type=Path, help="current PR/run JSON") + parser.add_argument( + "--threshold-pct", + type=float, + default=5.0, + help="max allowed per-op instructions increase (default: 5)", + ) + args = parser.parse_args() + + baseline = load_results(args.baseline) + current = load_results(args.current) + + missing = sorted(set(baseline) - set(current)) + added = sorted(set(current) - set(baseline)) + regressions: list[tuple[str, int, int, float]] = [] + improvements: list[tuple[str, int, int, float]] = [] + + for key in sorted(set(baseline) & set(current)): + b_ins = int(baseline[key]["instructions"]) + c_ins = int(current[key]["instructions"]) + delta = pct_delta(c_ins, b_ins) + if delta > args.threshold_pct: + regressions.append((key, b_ins, c_ins, delta)) + elif delta < 0: + improvements.append((key, b_ins, c_ins, delta)) + + print("=== Stellar gas bench comparison ===") + print(f"baseline: {args.baseline}") + print(f"current: {args.current}") + print(f"threshold: +{args.threshold_pct:.1f}% instructions (per-op)") + print() + + if missing: + print("WARNING: ops missing from current run:") + for key in missing: + print(f" - {key}") + print() + + if added: + print("INFO: new ops (no baseline gate):") + for key in added: + print(f" + {key}") + print() + + if improvements: + print("Improvements (instructions ↓):") + for key, b_ins, c_ins, delta in improvements: + print(f" {key}: {b_ins} -> {c_ins} ({delta:+.2f}%)") + print() + + if regressions: + print("REGRESSIONS (per-op gas / instructions > baseline + " + f"{args.threshold_pct:.1f}%):") + print() + print(f"{'op':<55} {'baseline':>12} {'current':>12} {'delta':>10}") + print("-" * 92) + for key, b_ins, c_ins, delta in regressions: + print(f"{key:<55} {b_ins:12d} {c_ins:12d} {delta:+9.2f}%") + print() + print( + f"FAIL: {len(regressions)} op(s) exceeded +{args.threshold_pct:.1f}% " + "instructions vs baseline." + ) + return 1 + + print("OK: no per-op instructions regression above threshold.") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/stellar/bench/src/lib.rs b/stellar/bench/src/lib.rs new file mode 100644 index 0000000..9784787 --- /dev/null +++ b/stellar/bench/src/lib.rs @@ -0,0 +1,373 @@ +use soroban_sdk::testutils::Address as _; +use soroban_sdk::token::StellarAssetClient; +use soroban_sdk::{vec, Address, Bytes, BytesN, Env, String as SorobanString, Vec as SorobanVec}; + +use stealth_announcer::{StealthAnnouncerContract, StealthAnnouncerContractClient}; +use stealth_registry::{StealthRegistryContract, StealthRegistryContractClient}; +use stealth_sender::{StealthSenderContract, StealthSenderContractClient}; +use wraith_names::{WraithNamesContract, WraithNamesContractClient}; + +#[derive(Clone, Debug)] +pub struct Row { + pub contract: &'static str, + pub function: &'static str, + pub params: std::string::String, + pub instructions: i64, + pub mem_bytes: i64, + pub read_entries: u32, + pub write_entries: u32, + pub read_bytes: u32, + pub write_bytes: u32, + pub events_bytes: u32, +} + +impl Row { + pub fn op_key(&self) -> std::string::String { + format!("{}::{}::{}", self.contract, self.function, self.params) + } +} + +/// Collect per-op Soroban resource measurements for all harness cases. +pub fn collect_rows() -> std::vec::Vec { + let mut rows = std::vec::Vec::new(); + + for metadata_len in [0u32, 32, 256, 1024, 4096] { + rows.push(measure( + "stealth-announcer", + "announce", + format!("metadata_len={metadata_len}"), + |env| { + let contract_id = env.register(StealthAnnouncerContract, ()); + let client = StealthAnnouncerContractClient::new(env, &contract_id); + client.announce( + &1, + &Address::generate(env), + &BytesN::from_array(env, &[7u8; 32]), + &bytes(env, metadata_len, 9), + ); + }, + )); + } + + rows.push(measure( + "stealth-registry", + "register_keys", + "first_time".into(), + |env| { + env.mock_all_auths(); + let contract_id = env.register(StealthRegistryContract, ()); + let client = StealthRegistryContractClient::new(env, &contract_id); + client.register_keys(&Address::generate(env), &1, &bytes(env, 64, 1)); + }, + )); + + rows.push(measure( + "stealth-registry", + "register_keys", + "replacement".into(), + |env| { + env.mock_all_auths(); + let contract_id = env.register(StealthRegistryContract, ()); + let client = StealthRegistryContractClient::new(env, &contract_id); + let registrant = Address::generate(env); + client.register_keys(®istrant, &1, &bytes(env, 64, 1)); + client.register_keys(®istrant, &1, &bytes(env, 64, 2)); + }, + )); + + for asset in ["xlm", "issued"] { + rows.push(measure( + "stealth-sender", + "send", + format!("asset={asset}"), + |env| { + env.mock_all_auths(); + let sender_contract_id = env.register(StealthSenderContract, ()); + let announcer_id = env.register(StealthAnnouncerContract, ()); + let client = StealthSenderContractClient::new(env, &sender_contract_id); + client.init(&announcer_id, &None, &None, &0); + let (token, sender) = funded_token(env, asset == "xlm"); + client.send( + &sender, + &token, + &100, + &1, + &Address::generate(env), + &BytesN::from_array(env, &[3u8; 32]), + &bytes(env, 32, 4), + ); + }, + )); + } + + for batch_size in [1u32, 5, 10, 25] { + rows.push(measure( + "stealth-sender", + "batch_send", + format!("batch_size={batch_size}"), + |env| { + env.mock_all_auths(); + let sender_contract_id = env.register(StealthSenderContract, ()); + let announcer_id = env.register(StealthAnnouncerContract, ()); + let client = StealthSenderContractClient::new(env, &sender_contract_id); + client.init(&announcer_id, &None, &None, &0); + let (token, sender) = funded_token(env, true); + let mut addresses: SorobanVec
= vec![env]; + let mut keys: SorobanVec> = vec![env]; + let mut metadatas: SorobanVec = vec![env]; + let mut amounts: SorobanVec = vec![env]; + for i in 0..batch_size { + addresses.push_back(Address::generate(env)); + keys.push_back(BytesN::from_array(env, &[i as u8; 32])); + metadatas.push_back(bytes(env, 32, i as u8)); + amounts.push_back(100); + } + client.batch_send(&sender, &token, &1, &addresses, &keys, &metadatas, &amounts); + }, + )); + } + + for name_len in [3u32, 32] { + rows.push(measure( + "wraith-names", + "register", + format!("name_len={name_len}"), + |env| { + env.mock_all_auths(); + let contract_id = env.register(WraithNamesContract, ()); + let client = WraithNamesContractClient::new(env, &contract_id); + client.register( + &Address::generate(env), + &name(env, name_len), + &bytes(env, 64, 5), + ); + }, + )); + } + + rows.push(measure("wraith-names", "resolve", "hit".into(), |env| { + env.mock_all_auths(); + let contract_id = env.register(WraithNamesContract, ()); + let client = WraithNamesContractClient::new(env, &contract_id); + let n = SorobanString::from_str(env, "alice"); + client.register(&Address::generate(env), &n, &bytes(env, 64, 6)); + client.resolve(&n); + })); + + rows.push(measure("wraith-names", "resolve", "miss".into(), |env| { + let contract_id = env.register(WraithNamesContract, ()); + let client = WraithNamesContractClient::new(env, &contract_id); + let _ = client.try_resolve(&SorobanString::from_str(env, "missing")); + })); + + rows.push(measure("wraith-names", "name_of", "hit".into(), |env| { + env.mock_all_auths(); + let contract_id = env.register(WraithNamesContract, ()); + let client = WraithNamesContractClient::new(env, &contract_id); + let meta = bytes(env, 64, 7); + client.register( + &Address::generate(env), + &SorobanString::from_str(env, "charlie"), + &meta, + ); + client.name_of(&meta); + })); + + rows.push(measure("wraith-names", "name_of", "miss".into(), |env| { + let contract_id = env.register(WraithNamesContract, ()); + let client = WraithNamesContractClient::new(env, &contract_id); + let _ = client.try_name_of(&bytes(env, 64, 8)); + })); + + rows +} + +pub fn print_markdown(rows: &[Row]) { + println!("| Contract | Function | Parameters | Instructions | Mem bytes | Read entries | Write entries | Read bytes | Write bytes | Event bytes |"); + println!("|---|---|---:|---:|---:|---:|---:|---:|---:|---:|"); + for row in rows { + println!( + "| {} | {} | {} | {} | {} | {} | {} | {} | {} | {} |", + row.contract, + row.function, + row.params, + row.instructions, + row.mem_bytes, + row.read_entries, + row.write_entries, + row.read_bytes, + row.write_bytes, + row.events_bytes, + ); + } +} + +/// Serialize bench rows to a stable JSON document used for CI baselines. +pub fn to_json(rows: &[Row], meta: &BenchMeta) -> std::string::String { + let mut out = std::string::String::new(); + out.push_str("{\n"); + out.push_str(&format!( + " \"generated_at\": \"{}\",\n", + escape_json(&meta.generated_at) + )); + out.push_str(&format!(" \"commit\": \"{}\",\n", escape_json(&meta.commit))); + out.push_str(&format!( + " \"threshold_pct\": {},\n", + meta.threshold_pct + )); + out.push_str(" \"results\": [\n"); + for (i, row) in rows.iter().enumerate() { + out.push_str(" {\n"); + out.push_str(&format!( + " \"contract\": \"{}\",\n", + escape_json(row.contract) + )); + out.push_str(&format!( + " \"function\": \"{}\",\n", + escape_json(row.function) + )); + out.push_str(&format!( + " \"params\": \"{}\",\n", + escape_json(&row.params) + )); + out.push_str(&format!(" \"instructions\": {},\n", row.instructions)); + out.push_str(&format!(" \"mem_bytes\": {},\n", row.mem_bytes)); + out.push_str(&format!(" \"read_entries\": {},\n", row.read_entries)); + out.push_str(&format!(" \"write_entries\": {},\n", row.write_entries)); + out.push_str(&format!(" \"read_bytes\": {},\n", row.read_bytes)); + out.push_str(&format!(" \"write_bytes\": {},\n", row.write_bytes)); + out.push_str(&format!(" \"events_bytes\": {}\n", row.events_bytes)); + out.push_str(" }"); + if i + 1 < rows.len() { + out.push(','); + } + out.push('\n'); + } + out.push_str(" ]\n"); + out.push_str("}\n"); + out +} + +pub fn markdown_table(rows: &[Row]) -> std::string::String { + let mut out = std::string::String::new(); + out.push_str("| Contract | Function | Parameters | Instructions | Mem bytes | Read entries | Write entries | Read bytes | Write bytes | Event bytes |\n"); + out.push_str("|---|---|---:|---:|---:|---:|---:|---:|---:|---:|\n"); + for row in rows { + out.push_str(&format!( + "| {} | {} | {} | {} | {} | {} | {} | {} | {} | {} |\n", + row.contract, + row.function, + row.params, + row.instructions, + row.mem_bytes, + row.read_entries, + row.write_entries, + row.read_bytes, + row.write_bytes, + row.events_bytes, + )); + } + out +} + +#[derive(Clone, Debug)] +pub struct BenchMeta { + pub generated_at: std::string::String, + pub commit: std::string::String, + pub threshold_pct: f64, +} + +impl Default for BenchMeta { + fn default() -> Self { + Self { + generated_at: utc_now(), + commit: std::env::var("GITHUB_SHA") + .or_else(|_| std::env::var("COMMIT_SHA")) + .unwrap_or_else(|_| "unknown".into()), + threshold_pct: 5.0, + } + } +} + +fn utc_now() -> std::string::String { + // Prefer a stable ISO-ish stamp from the environment when available. + if let Ok(ts) = std::env::var("BENCH_GENERATED_AT") { + return ts; + } + // Fallback: unix epoch seconds (deterministic enough for local runs). + match std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH) { + Ok(d) => format!("{}", d.as_secs()), + Err(_) => "0".into(), + } +} + +fn escape_json(s: &str) -> std::string::String { + let mut out = std::string::String::with_capacity(s.len()); + for c in s.chars() { + match c { + '"' => out.push_str("\\\""), + '\\' => out.push_str("\\\\"), + '\n' => out.push_str("\\n"), + '\r' => out.push_str("\\r"), + '\t' => out.push_str("\\t"), + c if c.is_control() => out.push_str(&format!("\\u{:04x}", c as u32)), + c => out.push(c), + } + } + out +} + +fn measure( + contract: &'static str, + function: &'static str, + params: std::string::String, + f: F, +) -> Row +where + F: FnOnce(&Env), +{ + let env = Env::default(); + env.cost_estimate().budget().reset_unlimited(); + f(&env); + let resources = env.cost_estimate().resources(); + Row { + contract, + function, + params, + instructions: resources.instructions, + mem_bytes: resources.mem_bytes, + read_entries: resources.read_entries, + write_entries: resources.write_entries, + read_bytes: resources.read_bytes, + write_bytes: resources.write_bytes, + events_bytes: resources.contract_events_size_bytes, + } +} + +fn funded_token(env: &Env, native: bool) -> (Address, Address) { + let admin = Address::generate(env); + let sender = Address::generate(env); + let token = if native { + env.register_stellar_asset_contract_v2(admin.clone()) + .address() + } else { + env.register_stellar_asset_contract_v2(Address::generate(env)) + .address() + }; + let asset = StellarAssetClient::new(env, &token); + asset.mint(&sender, &1_000_000); + (token, sender) +} + +fn bytes(env: &Env, len: u32, fill: u8) -> Bytes { + let mut out = Bytes::new(env); + for _ in 0..len { + out.push_back(fill); + } + out +} + +fn name(env: &Env, len: u32) -> SorobanString { + let raw = "abcdefghijklmnopqrstuvwxyz012345"; + SorobanString::from_str(env, &raw[..len as usize]) +} diff --git a/stellar/bench/src/main.rs b/stellar/bench/src/main.rs index 7f4fb2a..00223e9 100644 --- a/stellar/bench/src/main.rs +++ b/stellar/bench/src/main.rs @@ -1,251 +1,83 @@ -use soroban_sdk::testutils::Address as _; -use soroban_sdk::token::StellarAssetClient; -use soroban_sdk::{vec, Address, Bytes, BytesN, Env, String as SorobanString, Vec as SorobanVec}; +use std::env; +use std::fs; +use std::path::PathBuf; +use std::process; -use stealth_announcer::{StealthAnnouncerContract, StealthAnnouncerContractClient}; -use stealth_registry::{StealthRegistryContract, StealthRegistryContractClient}; -use stealth_sender::{StealthSenderContract, StealthSenderContractClient}; -use wraith_names::{WraithNamesContract, WraithNamesContractClient}; - -#[derive(Clone)] -struct Row { - contract: &'static str, - function: &'static str, - params: std::string::String, - instructions: i64, - mem_bytes: i64, - read_entries: u32, - write_entries: u32, - read_bytes: u32, - write_bytes: u32, - events_bytes: u32, -} +use wraith_stellar_bench::{collect_rows, print_markdown, to_json, BenchMeta}; fn main() { - let mut rows = std::vec::Vec::new(); - - for metadata_len in [0u32, 32, 256, 1024, 4096] { - rows.push(measure( - "stealth-announcer", - "announce", - format!("metadata_len={metadata_len}"), - |env| { - let contract_id = env.register(StealthAnnouncerContract, ()); - let client = StealthAnnouncerContractClient::new(env, &contract_id); - client.announce( - &1, - &Address::generate(env), - &BytesN::from_array(env, &[7u8; 32]), - &bytes(env, metadata_len, 9), - ); - }, - )); - } - - rows.push(measure( - "stealth-registry", - "register_keys", - "first_time".into(), - |env| { - env.mock_all_auths(); - let contract_id = env.register(StealthRegistryContract, ()); - let client = StealthRegistryContractClient::new(env, &contract_id); - client.register_keys(&Address::generate(env), &1, &bytes(env, 64, 1)); - }, - )); - - rows.push(measure( - "stealth-registry", - "register_keys", - "replacement".into(), - |env| { - env.mock_all_auths(); - let contract_id = env.register(StealthRegistryContract, ()); - let client = StealthRegistryContractClient::new(env, &contract_id); - let registrant = Address::generate(env); - client.register_keys(®istrant, &1, &bytes(env, 64, 1)); - client.register_keys(®istrant, &1, &bytes(env, 64, 2)); - }, - )); - - for asset in ["xlm", "issued"] { - rows.push(measure( - "stealth-sender", - "send", - format!("asset={asset}"), - |env| { - env.mock_all_auths(); - let sender_contract_id = env.register(StealthSenderContract, ()); - let announcer_id = env.register(StealthAnnouncerContract, ()); - let client = StealthSenderContractClient::new(env, &sender_contract_id); - client.init(&announcer_id, &None, &None, &0); - let (token, sender) = funded_token(env, asset == "xlm"); - client.send( - &sender, - &token, - &100, - &1, - &Address::generate(env), - &BytesN::from_array(env, &[3u8; 32]), - &bytes(env, 32, 4), - ); - }, - )); - } - - for batch_size in [1u32, 5, 10, 25] { - rows.push(measure( - "stealth-sender", - "batch_send", - format!("batch_size={batch_size}"), - |env| { - env.mock_all_auths(); - let sender_contract_id = env.register(StealthSenderContract, ()); - let announcer_id = env.register(StealthAnnouncerContract, ()); - let client = StealthSenderContractClient::new(env, &sender_contract_id); - client.init(&announcer_id, &None, &None, &0); - let (token, sender) = funded_token(env, true); - let mut addresses: SorobanVec
= vec![env]; - let mut keys: SorobanVec> = vec![env]; - let mut metadatas: SorobanVec = vec![env]; - let mut amounts: SorobanVec = vec![env]; - for i in 0..batch_size { - addresses.push_back(Address::generate(env)); - keys.push_back(BytesN::from_array(env, &[i as u8; 32])); - metadatas.push_back(bytes(env, 32, i as u8)); - amounts.push_back(100); - } - client.batch_send(&sender, &token, &1, &addresses, &keys, &metadatas, &amounts); - }, - )); + let args: Vec = env::args().skip(1).collect(); + let mut format = "markdown".to_string(); + let mut out_path: Option = None; + + let mut i = 0; + while i < args.len() { + match args[i].as_str() { + "--format" => { + i += 1; + format = args + .get(i) + .cloned() + .unwrap_or_else(|| usage_exit("--format requires a value")); + } + "--out" => { + i += 1; + out_path = Some(PathBuf::from(args.get(i).cloned().unwrap_or_else(|| { + usage_exit("--out requires a path"); + }))); + } + "-h" | "--help" => usage_exit(""), + other => usage_exit(&format!("unknown argument: {other}")), + } + i += 1; } - for name_len in [3u32, 32] { - rows.push(measure( - "wraith-names", - "register", - format!("name_len={name_len}"), - |env| { - env.mock_all_auths(); - let contract_id = env.register(WraithNamesContract, ()); - let client = WraithNamesContractClient::new(env, &contract_id); - client.register( - &Address::generate(env), - &name(env, name_len), - &bytes(env, 64, 5), - ); - }, - )); - } - - rows.push(measure("wraith-names", "resolve", "hit".into(), |env| { - env.mock_all_auths(); - let contract_id = env.register(WraithNamesContract, ()); - let client = WraithNamesContractClient::new(env, &contract_id); - let n = SorobanString::from_str(env, "alice"); - client.register(&Address::generate(env), &n, &bytes(env, 64, 6)); - client.resolve(&n); - })); - - rows.push(measure("wraith-names", "resolve", "miss".into(), |env| { - let contract_id = env.register(WraithNamesContract, ()); - let client = WraithNamesContractClient::new(env, &contract_id); - let _ = client.try_resolve(&SorobanString::from_str(env, "missing")); - })); - - rows.push(measure("wraith-names", "name_of", "hit".into(), |env| { - env.mock_all_auths(); - let contract_id = env.register(WraithNamesContract, ()); - let client = WraithNamesContractClient::new(env, &contract_id); - let meta = bytes(env, 64, 7); - client.register( - &Address::generate(env), - &SorobanString::from_str(env, "charlie"), - &meta, - ); - client.name_of(&meta); - })); - - rows.push(measure("wraith-names", "name_of", "miss".into(), |env| { - let contract_id = env.register(WraithNamesContract, ()); - let client = WraithNamesContractClient::new(env, &contract_id); - let _ = client.try_name_of(&bytes(env, 64, 8)); - })); - - print_markdown(&rows); -} + let rows = collect_rows(); + let meta = BenchMeta::default(); + + let rendered = match format.as_str() { + "markdown" | "md" => { + let mut buf = String::new(); + // Capture markdown via the shared table builder. + buf.push_str(&wraith_stellar_bench::markdown_table(&rows)); + buf + } + "json" => to_json(&rows, &meta), + other => usage_exit(&format!("unsupported --format: {other}")), + }; -fn measure( - contract: &'static str, - function: &'static str, - params: std::string::String, - f: F, -) -> Row -where - F: FnOnce(&Env), -{ - let env = Env::default(); - env.cost_estimate().budget().reset_unlimited(); - f(&env); - let resources = env.cost_estimate().resources(); - Row { - contract, - function, - params, - instructions: resources.instructions, - mem_bytes: resources.mem_bytes, - read_entries: resources.read_entries, - write_entries: resources.write_entries, - read_bytes: resources.read_bytes, - write_bytes: resources.write_bytes, - events_bytes: resources.contract_events_size_bytes, + if let Some(path) = out_path { + if let Some(parent) = path.parent() { + if !parent.as_os_str().is_empty() { + fs::create_dir_all(parent).unwrap_or_else(|e| { + eprintln!("failed to create {}: {e}", parent.display()); + process::exit(1); + }); + } + } + fs::write(&path, &rendered).unwrap_or_else(|e| { + eprintln!("failed to write {}: {e}", path.display()); + process::exit(1); + }); + eprintln!("wrote {}", path.display()); } -} -fn funded_token(env: &Env, native: bool) -> (Address, Address) { - let admin = Address::generate(env); - let sender = Address::generate(env); - let token = if native { - env.register_stellar_asset_contract_v2(admin.clone()) - .address() + // Always print to stdout for CI logs / PERF.md piping. + if format == "markdown" || format == "md" { + print_markdown(&rows); } else { - env.register_stellar_asset_contract_v2(Address::generate(env)) - .address() - }; - let asset = StellarAssetClient::new(env, &token); - asset.mint(&sender, &1_000_000); - (token, sender) -} - -fn bytes(env: &Env, len: u32, fill: u8) -> Bytes { - let mut out = Bytes::new(env); - for _ in 0..len { - out.push_back(fill); + print!("{rendered}"); } - out -} - -fn name(env: &Env, len: u32) -> SorobanString { - let raw = "abcdefghijklmnopqrstuvwxyz012345"; - SorobanString::from_str(env, &raw[..len as usize]) } -fn print_markdown(rows: &[Row]) { - println!("| Contract | Function | Parameters | Instructions | Mem bytes | Read entries | Write entries | Read bytes | Write bytes | Event bytes |"); - println!("|---|---|---:|---:|---:|---:|---:|---:|---:|---:|"); - for row in rows { - println!( - "| {} | {} | {} | {} | {} | {} | {} | {} | {} | {} |", - row.contract, - row.function, - row.params, - row.instructions, - row.mem_bytes, - row.read_entries, - row.write_entries, - row.read_bytes, - row.write_bytes, - row.events_bytes, - ); +fn usage_exit(msg: &str) -> ! { + if !msg.is_empty() { + eprintln!("error: {msg}"); } + eprintln!( + "usage: wraith-stellar-bench [--format markdown|json] [--out path]\n\ + \n\ + Measures per-op Soroban resources. JSON output is used for CI baselines." + ); + process::exit(if msg.is_empty() { 0 } else { 2 }); } diff --git a/stellar/bench/update_perf_md.py b/stellar/bench/update_perf_md.py new file mode 100755 index 0000000..8b43dda --- /dev/null +++ b/stellar/bench/update_perf_md.py @@ -0,0 +1,94 @@ +#!/usr/bin/env python3 +"""Replace the auto-managed Current Numbers table in stellar/PERF.md.""" + +from __future__ import annotations + +import argparse +import json +import re +import sys +from datetime import datetime, timezone +from pathlib import Path + + +MARKER_START = "" +MARKER_END = "" + + +def table_from_results(results: list[dict]) -> str: + lines = [ + "| Contract | Function | Parameters | Instructions | Mem bytes | Read entries | Write entries | Read bytes | Write bytes | Event bytes |", + "|---|---|---:|---:|---:|---:|---:|---:|---:|---:|", + ] + for row in results: + lines.append( + "| {contract} | {function} | {params} | {instructions} | {mem_bytes} | " + "{read_entries} | {write_entries} | {read_bytes} | {write_bytes} | " + "{events_bytes} |".format(**row) + ) + return "\n".join(lines) + "\n" + + +def render_block(data: dict) -> str: + generated = data.get("generated_at", "unknown") + commit = data.get("commit", "unknown") + # Prefer ISO date for humans when generated_at is a unix timestamp. + try: + ts = int(generated) + measured = datetime.fromtimestamp(ts, tz=timezone.utc).strftime("%Y-%m-%d") + except (TypeError, ValueError): + measured = str(generated)[:10] + + intro = ( + f"These are the harness results auto-updated from `develop` " + f"(measured {measured}, commit `{commit[:12]}`).\n\n" + ) + return intro + table_from_results(data["results"]) + + +def update_perf(perf_path: Path, results_path: Path) -> None: + data = json.loads(results_path.read_text()) + if "results" not in data: + raise SystemExit(f"{results_path}: missing 'results'") + + text = perf_path.read_text() + block = render_block(data) + + if MARKER_START in text and MARKER_END in text: + pattern = re.compile( + re.escape(MARKER_START) + r".*?" + re.escape(MARKER_END), + re.DOTALL, + ) + replacement = f"{MARKER_START}\n{block}{MARKER_END}" + new_text, n = pattern.subn(replacement, text, count=1) + if n != 1: + raise SystemExit("failed to replace BENCH markers in PERF.md") + else: + # Fallback: replace the "## Current Numbers" section body until next ##. + pattern = re.compile( + r"(## Current Numbers\n\n).*?(?=\n## |\Z)", + re.DOTALL, + ) + replacement = ( + f"## Current Numbers\n\n" + f"{MARKER_START}\n{block}{MARKER_END}\n\n" + ) + new_text, n = pattern.subn(replacement, text, count=1) + if n != 1: + raise SystemExit("could not find '## Current Numbers' section in PERF.md") + + perf_path.write_text(new_text) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("perf_md", type=Path, help="path to PERF.md") + parser.add_argument("results_json", type=Path, help="bench JSON results") + args = parser.parse_args() + update_perf(args.perf_md, args.results_json) + print(f"updated {args.perf_md}") + return 0 + + +if __name__ == "__main__": + sys.exit(main())