From f7eaae0858b395dfde87843fa6533911751f2cd7 Mon Sep 17 00:00:00 2001 From: G4614 <92488762+G4614@users.noreply.github.com> Date: Wed, 17 Jun 2026 20:28:06 +0800 Subject: [PATCH 1/4] feat(bench): replace Rust bench harness with Python REST API benchmark suite MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace the local-only Rust benchmark harness (libkrun, Linux-only) with a Python REST API benchmark suite that runs against any remote BoxLite API. 8 scenarios covering latency, throughput, density, and stability: - latency-cold-start: POST /boxes + runner metrics + first exec - latency-exec: single exec on warm box - latency-lifecycle: full create → exec → delete cycle - throughput-exec-serial: 20 serial execs, execs/sec - throughput-exec-parallel: 10 concurrent execs, batch wall time - density-parallel-create: 5 concurrent box creates - stability-churn: 10 create+exec+delete cycles, drift tracking - stability-exec-loop: 50 execs on one box, degradation check CLI: python -m bench {list,run,compare} Auth: reuses e2e_auth.py (env vars or credentials.toml) Report: schema v1.0 JSON with p50/p90/p99/mean/stdev aggregates Includes baseline results from dev.boxlite.ai run (2026-06-17). --- bench/results/density-parallel-create.json | 145 ++++++++++++ bench/results/latency-cold-start.json | 127 +++++++++++ bench/results/latency-exec.json | 73 ++++++ bench/results/latency-lifecycle.json | 127 +++++++++++ bench/results/stability-churn.json | 163 ++++++++++++++ bench/results/stability-exec-loop.json | 163 ++++++++++++++ bench/results/throughput-exec-parallel.json | 127 +++++++++++ bench/results/throughput-exec-serial.json | 127 +++++++++++ scripts/test/e2e/bench/README.md | 76 +++++++ scripts/test/e2e/bench/__init__.py | 0 scripts/test/e2e/bench/__main__.py | 207 ++++++++++++++++++ scripts/test/e2e/bench/harness.py | 170 ++++++++++++++ scripts/test/e2e/bench/scenarios/__init__.py | 0 .../scenarios/density_parallel_create.py | 37 ++++ .../e2e/bench/scenarios/latency_cold_start.py | 39 ++++ .../test/e2e/bench/scenarios/latency_exec.py | 25 +++ .../e2e/bench/scenarios/latency_lifecycle.py | 34 +++ .../e2e/bench/scenarios/stability_churn.py | 29 +++ .../bench/scenarios/stability_exec_loop.py | 34 +++ .../scenarios/throughput_exec_parallel.py | 36 +++ .../bench/scenarios/throughput_exec_serial.py | 28 +++ scripts/test/e2e/lib/e2e_auth.py | 194 ++++++++++++++++ 22 files changed, 1961 insertions(+) create mode 100644 bench/results/density-parallel-create.json create mode 100644 bench/results/latency-cold-start.json create mode 100644 bench/results/latency-exec.json create mode 100644 bench/results/latency-lifecycle.json create mode 100644 bench/results/stability-churn.json create mode 100644 bench/results/stability-exec-loop.json create mode 100644 bench/results/throughput-exec-parallel.json create mode 100644 bench/results/throughput-exec-serial.json create mode 100644 scripts/test/e2e/bench/README.md create mode 100644 scripts/test/e2e/bench/__init__.py create mode 100644 scripts/test/e2e/bench/__main__.py create mode 100644 scripts/test/e2e/bench/harness.py create mode 100644 scripts/test/e2e/bench/scenarios/__init__.py create mode 100644 scripts/test/e2e/bench/scenarios/density_parallel_create.py create mode 100644 scripts/test/e2e/bench/scenarios/latency_cold_start.py create mode 100644 scripts/test/e2e/bench/scenarios/latency_exec.py create mode 100644 scripts/test/e2e/bench/scenarios/latency_lifecycle.py create mode 100644 scripts/test/e2e/bench/scenarios/stability_churn.py create mode 100644 scripts/test/e2e/bench/scenarios/stability_exec_loop.py create mode 100644 scripts/test/e2e/bench/scenarios/throughput_exec_parallel.py create mode 100644 scripts/test/e2e/bench/scenarios/throughput_exec_serial.py create mode 100644 scripts/test/e2e/lib/e2e_auth.py diff --git a/bench/results/density-parallel-create.json b/bench/results/density-parallel-create.json new file mode 100644 index 000000000..2300024ed --- /dev/null +++ b/bench/results/density-parallel-create.json @@ -0,0 +1,145 @@ +{ + "schema_version": "1.0", + "scenario": "density-parallel-create", + "metadata": { + "started_at": "2026-06-17T11:55:48.965970+00:00", + "label": "https://api.dev.boxlite.ai/api (20260605-p0-r3)", + "git_commit": "9e153f8b", + "host": { + "kernel": "macOS-26.4-arm64-arm-64bit", + "arch": "arm64", + "target": "https://api.dev.boxlite.ai/api" + } + }, + "sample_count": 4, + "warmup_count": 1, + "samples": [ + { + "iteration": 1, + "warmup": true, + "wall_ms": 3576.670417096466, + "metrics": { + "batch_wall_ms": 3576.670417096466, + "box_count": 5.0, + "per_box_mean_ms": 3039.4811334554106, + "per_box_max_ms": 3571.2176670785993, + "per_box_min_ms": 2684.674750082195 + } + }, + { + "iteration": 2, + "warmup": false, + "wall_ms": 2939.616624964401, + "metrics": { + "batch_wall_ms": 2939.616624964401, + "box_count": 5.0, + "per_box_mean_ms": 2402.3529085330665, + "per_box_max_ms": 2938.1887088529766, + "per_box_min_ms": 2239.59141690284 + } + }, + { + "iteration": 3, + "warmup": false, + "wall_ms": 4019.7949579451233, + "metrics": { + "batch_wall_ms": 4019.7949579451233, + "box_count": 5.0, + "per_box_mean_ms": 2653.2177915330976, + "per_box_max_ms": 4018.9031658228487, + "per_box_min_ms": 2084.5392090268433 + } + }, + { + "iteration": 4, + "warmup": false, + "wall_ms": 2297.7219170425087, + "metrics": { + "batch_wall_ms": 2297.7219170425087, + "box_count": 5.0, + "per_box_mean_ms": 2198.3412586618215, + "per_box_max_ms": 2296.7179170809686, + "per_box_min_ms": 2132.5164171867073 + } + }, + { + "iteration": 5, + "warmup": false, + "wall_ms": 3286.565958056599, + "metrics": { + "batch_wall_ms": 3286.565958056599, + "box_count": 5.0, + "per_box_mean_ms": 2292.9275666363537, + "per_box_max_ms": 3285.5944999027997, + "per_box_min_ms": 1889.6796249318868 + } + } + ], + "aggregates": [ + { + "name": "batch_wall_ms", + "unit": "ms", + "higher_is_better": false, + "min": 2297.7219170425087, + "p50": 3113.0912915105, + "p90": 3799.8262579785664, + "p99": 3997.7980879484676, + "max": 4019.7949579451233, + "mean": 3135.924864502158, + "stdev": 717.6443379071584, + "n": 4 + }, + { + "name": "box_count", + "unit": "ms", + "higher_is_better": false, + "min": 5.0, + "p50": 5.0, + "p90": 5.0, + "p99": 5.0, + "max": 5.0, + "mean": 5.0, + "stdev": 0.0, + "n": 4 + }, + { + "name": "per_box_max_ms", + "unit": "ms", + "higher_is_better": false, + "min": 2296.7179170809686, + "p50": 3111.891604377888, + "p90": 3798.9105660468344, + "p99": 3996.903905845247, + "max": 4018.9031658228487, + "mean": 3134.8510729148984, + "stdev": 717.7313694555344, + "n": 4 + }, + { + "name": "per_box_mean_ms", + "unit": "ms", + "higher_is_better": false, + "min": 2198.3412586618215, + "p50": 2347.64023758471, + "p90": 2577.9583266330883, + "p99": 2645.6918450430967, + "max": 2653.2177915330976, + "mean": 2386.709881341085, + "stdev": 196.2558118511848, + "n": 4 + }, + { + "name": "per_box_min_ms", + "unit": "ms", + "higher_is_better": false, + "min": 1889.6796249318868, + "p50": 2108.5278131067753, + "p90": 2207.4689169880003, + "p99": 2236.379166911356, + "max": 2239.59141690284, + "mean": 2086.5816670120694, + "stdev": 146.39737092950136, + "n": 4 + } + ] +} \ No newline at end of file diff --git a/bench/results/latency-cold-start.json b/bench/results/latency-cold-start.json new file mode 100644 index 000000000..9f164f387 --- /dev/null +++ b/bench/results/latency-cold-start.json @@ -0,0 +1,127 @@ +{ + "schema_version": "1.0", + "scenario": "latency-cold-start", + "metadata": { + "started_at": "2026-06-17T11:56:30.838525+00:00", + "label": "https://api.dev.boxlite.ai/api (20260605-p0-r3)", + "git_commit": "9e153f8b", + "host": { + "kernel": "macOS-26.4-arm64-arm-64bit", + "arch": "arm64", + "target": "https://api.dev.boxlite.ai/api" + } + }, + "sample_count": 4, + "warmup_count": 1, + "samples": [ + { + "iteration": 1, + "warmup": true, + "wall_ms": 1672.9062909726053, + "metrics": { + "api_create_wall_ms": 1672.9062909726053, + "runner_create_ms": 561.0, + "first_exec_wall_ms": 1898.1214999221265, + "total_ready_wall_ms": 5571.027790894732 + } + }, + { + "iteration": 2, + "warmup": false, + "wall_ms": 1746.9586660154164, + "metrics": { + "api_create_wall_ms": 1746.9586660154164, + "runner_create_ms": 559.0, + "first_exec_wall_ms": 1913.9461249578744, + "total_ready_wall_ms": 5660.904790973291 + } + }, + { + "iteration": 3, + "warmup": false, + "wall_ms": 1987.1243329253048, + "metrics": { + "api_create_wall_ms": 1987.1243329253048, + "runner_create_ms": 557.0, + "first_exec_wall_ms": 1277.7571249753237, + "total_ready_wall_ms": 5264.881457900628 + } + }, + { + "iteration": 4, + "warmup": false, + "wall_ms": 2309.0840000659227, + "metrics": { + "api_create_wall_ms": 2309.0840000659227, + "runner_create_ms": 549.0, + "first_exec_wall_ms": 992.5624160096049, + "total_ready_wall_ms": 5301.646416075528 + } + }, + { + "iteration": 5, + "warmup": false, + "wall_ms": 1629.7256250400096, + "metrics": { + "api_create_wall_ms": 1629.7256250400096, + "runner_create_ms": 565.0, + "first_exec_wall_ms": 1313.9789160341024, + "total_ready_wall_ms": 4943.704541074112 + } + } + ], + "aggregates": [ + { + "name": "api_create_wall_ms", + "unit": "ms", + "higher_is_better": false, + "min": 1629.7256250400096, + "p50": 1867.0414994703606, + "p90": 2212.4960999237373, + "p99": 2299.425210051704, + "max": 2309.0840000659227, + "mean": 1918.2231560116634, + "stdev": 300.04550905115923, + "n": 4 + }, + { + "name": "first_exec_wall_ms", + "unit": "ms", + "higher_is_better": false, + "min": 992.5624160096049, + "p50": 1295.868020504713, + "p90": 1733.9559622807428, + "p99": 1895.947108690161, + "max": 1913.9461249578744, + "mean": 1374.5611454942264, + "stdev": 387.2554756720032, + "n": 4 + }, + { + "name": "runner_create_ms", + "unit": "ms", + "higher_is_better": false, + "min": 549.0, + "p50": 558.0, + "p90": 563.2, + "p99": 564.8199999999999, + "max": 565.0, + "mean": 557.5, + "stdev": 6.60807586719967, + "n": 4 + }, + { + "name": "total_ready_wall_ms", + "unit": "ms", + "higher_is_better": false, + "min": 4943.704541074112, + "p50": 5283.263936988078, + "p90": 5553.127278503962, + "p99": 5650.127039726357, + "max": 5660.904790973291, + "mean": 5292.78430150589, + "stdev": 293.38625261176963, + "n": 4 + } + ] +} \ No newline at end of file diff --git a/bench/results/latency-exec.json b/bench/results/latency-exec.json new file mode 100644 index 000000000..73fdc013e --- /dev/null +++ b/bench/results/latency-exec.json @@ -0,0 +1,73 @@ +{ + "schema_version": "1.0", + "scenario": "latency-exec", + "metadata": { + "started_at": "2026-06-17T11:57:10.505744+00:00", + "label": "https://api.dev.boxlite.ai/api (20260605-p0-r3)", + "git_commit": "9e153f8b", + "host": { + "kernel": "macOS-26.4-arm64-arm-64bit", + "arch": "arm64", + "target": "https://api.dev.boxlite.ai/api" + } + }, + "sample_count": 4, + "warmup_count": 1, + "samples": [ + { + "iteration": 1, + "warmup": true, + "wall_ms": 1269.4566661957651, + "metrics": { + "exec_wall_ms": 1269.4566661957651 + } + }, + { + "iteration": 2, + "warmup": false, + "wall_ms": 1324.691542191431, + "metrics": { + "exec_wall_ms": 1324.691542191431 + } + }, + { + "iteration": 3, + "warmup": false, + "wall_ms": 1047.2902080509812, + "metrics": { + "exec_wall_ms": 1047.2902080509812 + } + }, + { + "iteration": 4, + "warmup": false, + "wall_ms": 1323.3436248265207, + "metrics": { + "exec_wall_ms": 1323.3436248265207 + } + }, + { + "iteration": 5, + "warmup": false, + "wall_ms": 972.9088328313082, + "metrics": { + "exec_wall_ms": 972.9088328313082 + } + } + ], + "aggregates": [ + { + "name": "exec_wall_ms", + "unit": "ms", + "higher_is_better": false, + "min": 972.9088328313082, + "p50": 1185.316916438751, + "p90": 1324.2871669819579, + "p99": 1324.6511046704836, + "max": 1324.691542191431, + "mean": 1167.0585519750603, + "stdev": 183.7677458556996, + "n": 4 + } + ] +} \ No newline at end of file diff --git a/bench/results/latency-lifecycle.json b/bench/results/latency-lifecycle.json new file mode 100644 index 000000000..f79767073 --- /dev/null +++ b/bench/results/latency-lifecycle.json @@ -0,0 +1,127 @@ +{ + "schema_version": "1.0", + "scenario": "latency-lifecycle", + "metadata": { + "started_at": "2026-06-17T11:57:46.145760+00:00", + "label": "https://api.dev.boxlite.ai/api (20260605-p0-r3)", + "git_commit": "9e153f8b", + "host": { + "kernel": "macOS-26.4-arm64-arm-64bit", + "arch": "arm64", + "target": "https://api.dev.boxlite.ai/api" + } + }, + "sample_count": 4, + "warmup_count": 1, + "samples": [ + { + "iteration": 1, + "warmup": true, + "wall_ms": 2031.3032909762114, + "metrics": { + "create_ms": 2031.3032909762114, + "exec_ms": 1841.7157079093158, + "delete_ms": 1029.8784158658236, + "total_lifecycle_ms": 6902.897414751351 + } + }, + { + "iteration": 2, + "warmup": false, + "wall_ms": 1694.4484158884734, + "metrics": { + "create_ms": 1694.4484158884734, + "exec_ms": 1037.8320410382003, + "delete_ms": 1910.7658751308918, + "total_lifecycle_ms": 6643.046332057565 + } + }, + { + "iteration": 3, + "warmup": false, + "wall_ms": 1565.035582985729, + "metrics": { + "create_ms": 1565.035582985729, + "exec_ms": 1060.217458056286, + "delete_ms": 1604.4299169443548, + "total_lifecycle_ms": 6229.68295798637 + } + }, + { + "iteration": 4, + "warmup": false, + "wall_ms": 1598.6193751450628, + "metrics": { + "create_ms": 1598.6193751450628, + "exec_ms": 1087.1619579847902, + "delete_ms": 1050.3291669301689, + "total_lifecycle_ms": 5736.110500060022 + } + }, + { + "iteration": 5, + "warmup": false, + "wall_ms": 1986.639792099595, + "metrics": { + "create_ms": 1986.639792099595, + "exec_ms": 2825.874208007008, + "delete_ms": 3280.008791014552, + "total_lifecycle_ms": 10092.522791121155 + } + } + ], + "aggregates": [ + { + "name": "create_ms", + "unit": "ms", + "higher_is_better": false, + "min": 1565.035582985729, + "p50": 1646.533895516768, + "p90": 1898.9823792362588, + "p99": 1977.8740508132614, + "max": 1986.639792099595, + "mean": 1711.185791529715, + "stdev": 191.64735647228133, + "n": 4 + }, + { + "name": "delete_ms", + "unit": "ms", + "higher_is_better": false, + "min": 1050.3291669301689, + "p50": 1757.5978960376233, + "p90": 2869.235916249454, + "p99": 3238.931503538042, + "max": 3280.008791014552, + "mean": 1961.383437504992, + "stdev": 948.4673004927765, + "n": 4 + }, + { + "name": "exec_ms", + "unit": "ms", + "higher_is_better": false, + "min": 1037.8320410382003, + "p50": 1073.689708020538, + "p90": 2304.260533000343, + "p99": 2773.712840506341, + "max": 2825.874208007008, + "mean": 1502.771416271571, + "stdev": 882.2990513217824, + "n": 4 + }, + { + "name": "total_lifecycle_ms", + "unit": "ms", + "higher_is_better": false, + "min": 5736.110500060022, + "p50": 6436.364645021968, + "p90": 9057.679853402078, + "p99": 9989.038497349247, + "max": 10092.522791121155, + "mean": 7175.340645306278, + "stdev": 1979.8098208624995, + "n": 4 + } + ] +} \ No newline at end of file diff --git a/bench/results/stability-churn.json b/bench/results/stability-churn.json new file mode 100644 index 000000000..df95bdf0c --- /dev/null +++ b/bench/results/stability-churn.json @@ -0,0 +1,163 @@ +{ + "schema_version": "1.0", + "scenario": "stability-churn", + "metadata": { + "started_at": "2026-06-17T12:04:13.728135+00:00", + "label": "https://api.dev.boxlite.ai/api (20260605-p0-r3)", + "git_commit": "9e153f8b", + "host": { + "kernel": "macOS-26.4-arm64-arm-64bit", + "arch": "arm64", + "target": "https://api.dev.boxlite.ai/api" + } + }, + "sample_count": 4, + "warmup_count": 1, + "samples": [ + { + "iteration": 1, + "warmup": true, + "wall_ms": 10.0, + "metrics": { + "cycles": 10.0, + "first_cycle_ms": 7182.926582871005, + "last_cycle_ms": 7340.350749902427, + "mean_cycle_ms": 7324.33072917629, + "max_cycle_ms": 9805.654415860772, + "drift_ms": 157.42416703142226 + } + }, + { + "iteration": 2, + "warmup": false, + "wall_ms": 10.0, + "metrics": { + "cycles": 10.0, + "first_cycle_ms": 7342.134666861966, + "last_cycle_ms": 7083.020499907434, + "mean_cycle_ms": 8021.569795743562, + "max_cycle_ms": 11236.863166792318, + "drift_ms": -259.11416695453227 + } + }, + { + "iteration": 3, + "warmup": false, + "wall_ms": 10.0, + "metrics": { + "cycles": 10.0, + "first_cycle_ms": 9219.60841701366, + "last_cycle_ms": 12760.74929209426, + "mean_cycle_ms": 8099.970362661406, + "max_cycle_ms": 12760.74929209426, + "drift_ms": 3541.1408750806004 + } + }, + { + "iteration": 4, + "warmup": false, + "wall_ms": 10.0, + "metrics": { + "cycles": 10.0, + "first_cycle_ms": 9233.286458067596, + "last_cycle_ms": 7973.614916903898, + "mean_cycle_ms": 7404.302483308129, + "max_cycle_ms": 11010.297999950126, + "drift_ms": -1259.6715411636978 + } + }, + { + "iteration": 5, + "warmup": false, + "wall_ms": 10.0, + "metrics": { + "cycles": 10.0, + "first_cycle_ms": 6702.772417105734, + "last_cycle_ms": 6296.033707913011, + "mean_cycle_ms": 7905.180679121986, + "max_cycle_ms": 12435.241499915719, + "drift_ms": -406.73870919272304 + } + } + ], + "aggregates": [ + { + "name": "cycles", + "unit": "ms", + "higher_is_better": false, + "min": 10.0, + "p50": 10.0, + "p90": 10.0, + "p99": 10.0, + "max": 10.0, + "mean": 10.0, + "stdev": 0.0, + "n": 4 + }, + { + "name": "drift_ms", + "unit": "ms", + "higher_is_better": false, + "min": -1259.6715411636978, + "p50": -332.92643807362765, + "p90": 2401.064362470061, + "p99": 3427.1332238195455, + "max": 3541.1408750806004, + "mean": 403.9041144424118, + "stdev": 2137.480875797691, + "n": 4 + }, + { + "name": "first_cycle_ms", + "unit": "ms", + "higher_is_better": false, + "min": 6702.772417105734, + "p50": 8280.871541937813, + "p90": 9229.183045751415, + "p99": 9232.876116835978, + "max": 9233.286458067596, + "mean": 8124.450489762239, + "stdev": 1298.983535854317, + "n": 4 + }, + { + "name": "last_cycle_ms", + "unit": "ms", + "higher_is_better": false, + "min": 6296.033707913011, + "p50": 7528.317708405666, + "p90": 11324.608979537152, + "p99": 12617.135260838548, + "max": 12760.74929209426, + "mean": 8528.354604204651, + "stdev": 2903.6269372291194, + "n": 4 + }, + { + "name": "max_cycle_ms", + "unit": "ms", + "higher_is_better": false, + "min": 11010.297999950126, + "p50": 11836.052333354019, + "p90": 12663.096954440698, + "p99": 12750.984058328904, + "max": 12760.74929209426, + "mean": 11860.787989688106, + "stdev": 866.5145870115629, + "n": 4 + }, + { + "name": "mean_cycle_ms", + "unit": "ms", + "higher_is_better": false, + "min": 7404.302483308129, + "p50": 7963.375237432774, + "p90": 8076.450192586053, + "p99": 8097.618345653871, + "max": 8099.970362661406, + "mean": 7857.755830208771, + "stdev": 312.7149656078294, + "n": 4 + } + ] +} \ No newline at end of file diff --git a/bench/results/stability-exec-loop.json b/bench/results/stability-exec-loop.json new file mode 100644 index 000000000..4bde90194 --- /dev/null +++ b/bench/results/stability-exec-loop.json @@ -0,0 +1,163 @@ +{ + "schema_version": "1.0", + "scenario": "stability-exec-loop", + "metadata": { + "started_at": "2026-06-17T12:11:04.262851+00:00", + "label": "https://api.dev.boxlite.ai/api (20260605-p0-r3)", + "git_commit": "9e153f8b", + "host": { + "kernel": "macOS-26.4-arm64-arm-64bit", + "arch": "arm64", + "target": "https://api.dev.boxlite.ai/api" + } + }, + "sample_count": 4, + "warmup_count": 1, + "samples": [ + { + "iteration": 1, + "warmup": true, + "wall_ms": 50.0, + "metrics": { + "exec_count": 50.0, + "mean_exec_ms": 1398.8010699907318, + "max_exec_ms": 3553.1241251155734, + "first_10_mean_ms": 1358.2221748773009, + "last_10_mean_ms": 1415.5296125914901, + "degradation_ms": 57.30743771418929 + } + }, + { + "iteration": 2, + "warmup": false, + "wall_ms": 50.0, + "metrics": { + "exec_count": 50.0, + "mean_exec_ms": 1354.3089783890173, + "max_exec_ms": 5588.053917046636, + "first_10_mean_ms": 1525.4153877263889, + "last_10_mean_ms": 1607.0998541312292, + "degradation_ms": 81.68446640484035 + } + }, + { + "iteration": 3, + "warmup": false, + "wall_ms": 50.0, + "metrics": { + "exec_count": 50.0, + "mean_exec_ms": 1392.742221658118, + "max_exec_ms": 3187.2187920380384, + "first_10_mean_ms": 1186.9040749035776, + "last_10_mean_ms": 1279.979266738519, + "degradation_ms": 93.0751918349415 + } + }, + { + "iteration": 4, + "warmup": false, + "wall_ms": 50.0, + "metrics": { + "exec_count": 50.0, + "mean_exec_ms": 1398.031120034866, + "max_exec_ms": 2866.791500011459, + "first_10_mean_ms": 1344.996629259549, + "last_10_mean_ms": 1473.0090293101966, + "degradation_ms": 128.01240005064756 + } + }, + { + "iteration": 5, + "warmup": false, + "wall_ms": 50.0, + "metrics": { + "exec_count": 50.0, + "mean_exec_ms": 1861.4900708571076, + "max_exec_ms": 10562.904916005209, + "first_10_mean_ms": 2072.206091834232, + "last_10_mean_ms": 1363.2030125940219, + "degradation_ms": -709.0030792402104 + } + } + ], + "aggregates": [ + { + "name": "degradation_ms", + "unit": "ms", + "higher_is_better": false, + "min": -709.0030792402104, + "p50": 87.37982911989093, + "p90": 117.53123758593576, + "p99": 126.96428380417638, + "max": 128.01240005064756, + "mean": -101.55775523744524, + "stdev": 405.44295683047136, + "n": 4 + }, + { + "name": "exec_count", + "unit": "ms", + "higher_is_better": false, + "min": 50.0, + "p50": 50.0, + "p90": 50.0, + "p99": 50.0, + "max": 50.0, + "mean": 50.0, + "stdev": 0.0, + "n": 4 + }, + { + "name": "first_10_mean_ms", + "unit": "ms", + "higher_is_better": false, + "min": 1186.9040749035776, + "p50": 1435.206008492969, + "p90": 1908.1688806018794, + "p99": 2055.802370710997, + "max": 2072.206091834232, + "mean": 1532.380545930937, + "stdev": 385.54154886308197, + "n": 4 + }, + { + "name": "last_10_mean_ms", + "unit": "ms", + "higher_is_better": false, + "min": 1279.979266738519, + "p50": 1418.1060209521092, + "p90": 1566.8726066849194, + "p99": 1603.0771293865982, + "max": 1607.0998541312292, + "mean": 1430.8227906934917, + "stdev": 141.6327233048371, + "n": 4 + }, + { + "name": "max_exec_ms", + "unit": "ms", + "higher_is_better": false, + "min": 2866.791500011459, + "p50": 4387.636354542337, + "p90": 9070.449616317637, + "p99": 10413.659386036452, + "max": 10562.904916005209, + "mean": 5551.242281275336, + "stdev": 3554.949625314432, + "n": 4 + }, + { + "name": "mean_exec_ms", + "unit": "ms", + "higher_is_better": false, + "min": 1354.3089783890173, + "p50": 1395.386670846492, + "p90": 1722.4523856104352, + "p99": 1847.5863023324403, + "max": 1861.4900708571076, + "mean": 1501.6430977347773, + "stdev": 240.68792347619257, + "n": 4 + } + ] +} \ No newline at end of file diff --git a/bench/results/throughput-exec-parallel.json b/bench/results/throughput-exec-parallel.json new file mode 100644 index 000000000..d580f2859 --- /dev/null +++ b/bench/results/throughput-exec-parallel.json @@ -0,0 +1,127 @@ +{ + "schema_version": "1.0", + "scenario": "throughput-exec-parallel", + "metadata": { + "started_at": "2026-06-17T12:11:53.657439+00:00", + "label": "https://api.dev.boxlite.ai/api (20260605-p0-r3)", + "git_commit": "9e153f8b", + "host": { + "kernel": "macOS-26.4-arm64-arm-64bit", + "arch": "arm64", + "target": "https://api.dev.boxlite.ai/api" + } + }, + "sample_count": 4, + "warmup_count": 1, + "samples": [ + { + "iteration": 1, + "warmup": true, + "wall_ms": 2679.127665935084, + "metrics": { + "batch_wall_ms": 2679.127665935084, + "concurrency": 10.0, + "per_exec_mean_ms": 1284.5329166390002, + "per_exec_max_ms": 2678.3877089619637 + } + }, + { + "iteration": 2, + "warmup": false, + "wall_ms": 3144.4284170866013, + "metrics": { + "batch_wall_ms": 3144.4284170866013, + "concurrency": 10.0, + "per_exec_mean_ms": 2266.0685039591044, + "per_exec_max_ms": 3141.7815419845283 + } + }, + { + "iteration": 3, + "warmup": false, + "wall_ms": 2346.654040971771, + "metrics": { + "batch_wall_ms": 2346.654040971771, + "concurrency": 10.0, + "per_exec_mean_ms": 1414.616237510927, + "per_exec_max_ms": 2344.272625166923 + } + }, + { + "iteration": 4, + "warmup": false, + "wall_ms": 2787.9763750825077, + "metrics": { + "batch_wall_ms": 2787.9763750825077, + "concurrency": 10.0, + "per_exec_mean_ms": 1527.1886666538194, + "per_exec_max_ms": 2785.0872080307454 + } + }, + { + "iteration": 5, + "warmup": false, + "wall_ms": 3430.6123331189156, + "metrics": { + "batch_wall_ms": 3430.6123331189156, + "concurrency": 10.0, + "per_exec_mean_ms": 1715.2613540645689, + "per_exec_max_ms": 3429.351749829948 + } + } + ], + "aggregates": [ + { + "name": "batch_wall_ms", + "unit": "ms", + "higher_is_better": false, + "min": 2346.654040971771, + "p50": 2966.2023960845545, + "p90": 3344.7571583092213, + "p99": 3422.026815637946, + "max": 3430.6123331189156, + "mean": 2927.417791564949, + "stdev": 467.98461544253433, + "n": 4 + }, + { + "name": "concurrency", + "unit": "ms", + "higher_is_better": false, + "min": 10.0, + "p50": 10.0, + "p90": 10.0, + "p99": 10.0, + "max": 10.0, + "mean": 10.0, + "stdev": 0.0, + "n": 4 + }, + { + "name": "per_exec_max_ms", + "unit": "ms", + "higher_is_better": false, + "min": 2344.272625166923, + "p50": 2963.434375007637, + "p90": 3343.080687476322, + "p99": 3420.7246435945854, + "max": 3429.351749829948, + "mean": 2925.123281253036, + "stdev": 468.39610621275096, + "n": 4 + }, + { + "name": "per_exec_mean_ms", + "unit": "ms", + "higher_is_better": false, + "min": 1414.616237510927, + "p50": 1621.2250103591941, + "p90": 2100.8263589907438, + "p99": 2249.5442894622684, + "max": 2266.0685039591044, + "mean": 1730.783690547105, + "stdev": 377.79340320378253, + "n": 4 + } + ] +} \ No newline at end of file diff --git a/bench/results/throughput-exec-serial.json b/bench/results/throughput-exec-serial.json new file mode 100644 index 000000000..48cf80e21 --- /dev/null +++ b/bench/results/throughput-exec-serial.json @@ -0,0 +1,127 @@ +{ + "schema_version": "1.0", + "scenario": "throughput-exec-serial", + "metadata": { + "started_at": "2026-06-17T12:14:52.320462+00:00", + "label": "https://api.dev.boxlite.ai/api (20260605-p0-r3)", + "git_commit": "9e153f8b", + "host": { + "kernel": "macOS-26.4-arm64-arm-64bit", + "arch": "arm64", + "target": "https://api.dev.boxlite.ai/api" + } + }, + "sample_count": 4, + "warmup_count": 1, + "samples": [ + { + "iteration": 1, + "warmup": true, + "wall_ms": 29440.824541961774, + "metrics": { + "total_wall_ms": 29440.824541961774, + "exec_count": 20.0, + "execs_per_sec": 0.6793287997587893, + "avg_exec_ms": 1472.0412270980887 + } + }, + { + "iteration": 2, + "warmup": false, + "wall_ms": 34346.95858298801, + "metrics": { + "total_wall_ms": 34346.95858298801, + "exec_count": 20.0, + "execs_per_sec": 0.5822931876683244, + "avg_exec_ms": 1717.3479291494004 + } + }, + { + "iteration": 3, + "warmup": false, + "wall_ms": 26724.6862500906, + "metrics": { + "total_wall_ms": 26724.6862500906, + "exec_count": 20.0, + "execs_per_sec": 0.7483717418733848, + "avg_exec_ms": 1336.23431250453 + } + }, + { + "iteration": 4, + "warmup": false, + "wall_ms": 27192.295416956767, + "metrics": { + "total_wall_ms": 27192.295416956767, + "exec_count": 20.0, + "execs_per_sec": 0.7355024536666461, + "avg_exec_ms": 1359.6147708478384 + } + }, + { + "iteration": 5, + "warmup": false, + "wall_ms": 26200.674292165786, + "metrics": { + "total_wall_ms": 26200.674292165786, + "exec_count": 20.0, + "execs_per_sec": 0.7633391330688066, + "avg_exec_ms": 1310.0337146082893 + } + } + ], + "aggregates": [ + { + "name": "avg_exec_ms", + "unit": "ms", + "higher_is_better": false, + "min": 1310.0337146082893, + "p50": 1347.9245416761842, + "p90": 1610.0279816589318, + "p99": 1706.6159344003536, + "max": 1717.3479291494004, + "mean": 1430.8076817775145, + "stdev": 192.09738611098606, + "n": 4 + }, + { + "name": "exec_count", + "unit": "ms", + "higher_is_better": false, + "min": 20.0, + "p50": 20.0, + "p90": 20.0, + "p99": 20.0, + "max": 20.0, + "mean": 20.0, + "stdev": 0.0, + "n": 4 + }, + { + "name": "execs_per_sec", + "unit": "ms", + "higher_is_better": true, + "min": 0.5822931876683244, + "p50": 0.7419370977700155, + "p90": 0.7588489157101801, + "p99": 0.762890111332944, + "max": 0.7633391330688066, + "mean": 0.7073766290692904, + "stdev": 0.08416121523541671, + "n": 4 + }, + { + "name": "total_wall_ms", + "unit": "ms", + "higher_is_better": false, + "min": 26200.674292165786, + "p50": 26958.490833523683, + "p90": 32200.55963317864, + "p99": 34132.31868800707, + "max": 34346.95858298801, + "mean": 28616.15363555029, + "stdev": 3841.947722219721, + "n": 4 + } + ] +} \ No newline at end of file diff --git a/scripts/test/e2e/bench/README.md b/scripts/test/e2e/bench/README.md new file mode 100644 index 000000000..12c496d89 --- /dev/null +++ b/scripts/test/e2e/bench/README.md @@ -0,0 +1,76 @@ +# REST API Benchmark Suite + +Performance harness for the BoxLite REST API. Measures end-to-end latency +through the full production path (`client → API → runner → VM`) — no local +libkrun or Linux required. + +## Quick start + +```bash +# Set credentials (or use ~/.boxlite/credentials.toml profile p1) +export BOXLITE_E2E_API_URL=https://api.dev.boxlite.ai/api +export BOXLITE_E2E_API_KEY=blk_live_... +export BOXLITE_E2E_AUTH=api-key + +# List scenarios +.venv/bin/python -m bench list + +# Run one scenario +.venv/bin/python -m bench run latency-cold-start --runs 10 --warmup 1 + +# Run all scenarios, save reports +.venv/bin/python -m bench run all --runs 5 --out-dir bench/results/ + +# Compare two runs for regression +.venv/bin/python -m bench compare bench/results/baseline.json bench/results/current.json +``` + +Run from the repo root (`scripts/test/e2e/` must be in the Python path). + +## Scenarios + +### Latency +| Scenario | What it measures | +|----------|------------------| +| `latency-cold-start` | POST /boxes wall clock + runner create_duration_ms + first exec | +| `latency-exec` | Single exec on a warm box | +| `latency-lifecycle` | Full create → exec → delete cycle | + +### Throughput +| Scenario | What it measures | +|----------|------------------| +| `throughput-exec-serial` | 20 serial execs on one box; execs/sec | +| `throughput-exec-parallel` | 10 concurrent execs on one box; batch wall + per-exec | + +### Density +| Scenario | What it measures | +|----------|------------------| +| `density-parallel-create` | 5 boxes created concurrently; per-box and batch wall | + +### Stability +| Scenario | What it measures | +|----------|------------------| +| `stability-churn` | 10 create+exec+delete cycles; latency drift | +| `stability-exec-loop` | 50 execs on one box; degradation over time | + +## Report schema + +Same v1.0 JSON format as the original bench harness: + +```json +{ + "schema_version": "1.0", + "scenario": "latency-cold-start", + "metadata": { "started_at": "...", "git_commit": "...", "host": {...} }, + "sample_count": 9, + "warmup_count": 1, + "samples": [{ "iteration": 1, "warmup": false, "wall_ms": 1948, "metrics": {...} }], + "aggregates": [{ "name": "api_create_wall_ms", "p50": 1948, "p90": 2893, ... }] +} +``` + +## Compare semantics + +`bench compare` joins aggregates by metric name, picks the percentile from +`--on` (default `p99`), and flags regressions exceeding `--threshold` (default 20%). +Direction-aware: lower-is-better for latency, higher-is-better for `*_per_sec`. diff --git a/scripts/test/e2e/bench/__init__.py b/scripts/test/e2e/bench/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/scripts/test/e2e/bench/__main__.py b/scripts/test/e2e/bench/__main__.py new file mode 100644 index 000000000..9f1784e70 --- /dev/null +++ b/scripts/test/e2e/bench/__main__.py @@ -0,0 +1,207 @@ +#!/usr/bin/env python3 +"""REST API benchmark runner. + +Usage: + python -m bench list + python -m bench run [--runs N] [--warmup M] [--out PATH] + python -m bench run all [--runs N] [--warmup M] [--out-dir DIR] + python -m bench compare [--on p99] [--threshold 0.20] + +Requires: BOXLITE_E2E_API_URL + BOXLITE_E2E_API_KEY env vars, +or a configured ~/.boxlite/credentials.toml profile. +""" +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + +from bench.harness import build_report, print_aggregates, ctx +from bench.scenarios import latency_cold_start +from bench.scenarios import latency_exec +from bench.scenarios import latency_lifecycle +from bench.scenarios import throughput_exec_serial +from bench.scenarios import throughput_exec_parallel +from bench.scenarios import density_parallel_create +from bench.scenarios import stability_churn +from bench.scenarios import stability_exec_loop + +SCENARIOS = { + latency_cold_start.SCENARIO: latency_cold_start, + latency_exec.SCENARIO: latency_exec, + latency_lifecycle.SCENARIO: latency_lifecycle, + throughput_exec_serial.SCENARIO: throughput_exec_serial, + throughput_exec_parallel.SCENARIO: throughput_exec_parallel, + density_parallel_create.SCENARIO: density_parallel_create, + stability_churn.SCENARIO: stability_churn, + stability_exec_loop.SCENARIO: stability_exec_loop, +} + + +def cmd_list(): + print("Available scenarios:\n") + for name in sorted(SCENARIOS): + mod = SCENARIOS[name] + doc = (mod.__doc__ or "").strip().split("\n")[0] + print(f" {name:35s} {doc}") + print() + + +def run_scenario(name: str, runs: int, warmup: int) -> dict: + mod = SCENARIOS[name] + print(f"Scenario: {name}") + print(f"Runs: {runs} (warmup: {warmup})") + print() + + samples = [] + for i in range(1, runs + 1): + is_warmup = i <= warmup + tag = " (warmup)" if is_warmup else "" + try: + m = mod.run_once(i) + headline = " ".join(f"{k}={v:.0f}" for k, v in list(m.items())[:4]) + print(f" [{i:2d}/{runs}]{tag} {headline}") + samples.append({ + "iteration": i, + "warmup": is_warmup, + "wall_ms": m.get(list(m.keys())[0], 0) if m else 0, + "metrics": m, + }) + except Exception as e: + print(f" [{i:2d}/{runs}]{tag} ERROR: {e}") + samples.append({ + "iteration": i, + "warmup": is_warmup, + "wall_ms": 0, + "metrics": {}, + "error": str(e), + }) + + return build_report(name, samples, warmup) + + +def cmd_run(args): + c = ctx() + print(f"Target: {c.url}\n") + + if args.scenario == "all": + reports = {} + for name in sorted(SCENARIOS): + report = run_scenario(name, args.runs, args.warmup) + print_aggregates(report) + print() + if args.out_dir: + out = Path(args.out_dir) / f"{name}.json" + out.parent.mkdir(parents=True, exist_ok=True) + out.write_text(json.dumps(report, indent=2)) + print(f" → {out}") + reports[name] = report + return reports + + if args.scenario not in SCENARIOS: + print(f"Unknown scenario: {args.scenario}") + print(f"Available: {', '.join(sorted(SCENARIOS))}") + sys.exit(1) + + report = run_scenario(args.scenario, args.runs, args.warmup) + print_aggregates(report) + + if args.out: + Path(args.out).write_text(json.dumps(report, indent=2)) + print(f"\nReport: {args.out}") + else: + print(f"\n{json.dumps(report, indent=2)}") + + return report + + +def cmd_compare(args): + baseline = json.loads(Path(args.baseline).read_text()) + current = json.loads(Path(args.current).read_text()) + + if baseline["schema_version"] != current["schema_version"]: + print(f"Schema mismatch: {baseline['schema_version']} vs {current['schema_version']}") + sys.exit(1) + if baseline["scenario"] != current["scenario"]: + print(f"Scenario mismatch: {baseline['scenario']} vs {current['scenario']}") + sys.exit(1) + + field = args.on + threshold = args.threshold + regressions = [] + + base_aggs = {a["name"]: a for a in baseline["aggregates"]} + curr_aggs = {a["name"]: a for a in current["aggregates"]} + + print(f"Comparing {baseline['scenario']}: {field}, threshold={threshold:.0%}") + print(f" Baseline: {baseline['metadata'].get('label', '?')} ({baseline['sample_count']} samples)") + print(f" Current: {current['metadata'].get('label', '?')} ({current['sample_count']} samples)") + print() + + all_names = sorted(set(base_aggs) | set(curr_aggs)) + for name in all_names: + ba = base_aggs.get(name) + ca = curr_aggs.get(name) + if not ba or not ca: + status = "NEW" if ca else "REMOVED" + print(f" {name:35s} {status}") + continue + + bv = ba.get(field, ba.get("p99", 0)) + cv = ca.get(field, ca.get("p99", 0)) + higher = ba.get("higher_is_better", False) + + if bv == 0: + ratio = 0.0 + elif higher: + ratio = (bv - cv) / bv + else: + ratio = (cv - bv) / bv + + regressed = ratio > threshold + marker = " ← REGRESSION" if regressed else "" + direction = "↑" if (cv > bv) != higher else "↓" + print(f" {name:35s} {bv:10.1f} → {cv:10.1f} {direction} {ratio:+.1%}{marker}") + if regressed: + regressions.append(name) + + if regressions: + print(f"\nFAIL: {len(regressions)} regression(s) exceed {threshold:.0%} threshold") + sys.exit(1) + else: + print("\nPASS: no regressions") + + +def main(): + parser = argparse.ArgumentParser(description="BoxLite REST API benchmark") + sub = parser.add_subparsers(dest="command") + + sub.add_parser("list", help="List available scenarios") + + run_p = sub.add_parser("run", help="Run a scenario") + run_p.add_argument("scenario", help="Scenario name, or 'all'") + run_p.add_argument("--runs", type=int, default=5) + run_p.add_argument("--warmup", type=int, default=1) + run_p.add_argument("--out", type=str, default=None) + run_p.add_argument("--out-dir", type=str, default=None) + + cmp_p = sub.add_parser("compare", help="Compare two reports") + cmp_p.add_argument("baseline") + cmp_p.add_argument("current") + cmp_p.add_argument("--on", default="p99") + cmp_p.add_argument("--threshold", type=float, default=0.20) + + args = parser.parse_args() + if args.command == "list": + cmd_list() + elif args.command == "run": + cmd_run(args) + elif args.command == "compare": + cmd_compare(args) + else: + parser.print_help() + + +if __name__ == "__main__": + main() diff --git a/scripts/test/e2e/bench/harness.py b/scripts/test/e2e/bench/harness.py new file mode 100644 index 000000000..0826e468c --- /dev/null +++ b/scripts/test/e2e/bench/harness.py @@ -0,0 +1,170 @@ +"""Shared bench harness: auth, HTTP helpers, stats, report schema. + +All scenarios import from here. Auth follows the same pattern as +the E2E test suite (e2e_auth.py) — env vars or ~/.boxlite/credentials.toml. +""" +from __future__ import annotations + +import json +import math +import os +import platform +import statistics +import subprocess +import sys +import time +import urllib.error +import urllib.request +from datetime import datetime, timezone +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "lib")) +from e2e_auth import auth_context, E2EAuthContext + +IMAGE = os.environ.get( + "BOXLITE_E2E_IMAGE", + "ghcr.io/boxlite-ai/boxlite-agent-base:20260605-p0-r3", +) + + +def ctx() -> E2EAuthContext: + return auth_context() + + +def api(method: str, path: str, body=None, *, timeout: int = 60) -> tuple[int, dict | None]: + c = ctx() + headers = c.auth_headers(content_type=body is not None) + data = json.dumps(body).encode() if body is not None else None + req = urllib.request.Request( + c.url_for(c.v1(path)), method=method, headers=headers, data=data, + ) + try: + with urllib.request.urlopen(req, timeout=timeout) as resp: + return resp.status, json.loads(resp.read() or "null") + except urllib.error.HTTPError as e: + raw = e.read() + try: + return e.code, json.loads(raw) if raw else None + except json.JSONDecodeError: + return e.code, {"_raw": raw.decode("utf-8", "replace")} + + +def create_box(image: str = IMAGE, **overrides) -> str: + opts = {"image": image, "cpus": 1, "memory_mib": 256, "disk_size_gb": 4} + opts.update(overrides) + status, body = api("POST", "boxes", opts) + if status not in (200, 201): + raise RuntimeError(f"create_box failed: {status} {body}") + return body["box_id"] + + +def delete_box(box_id: str): + try: + api("DELETE", f"boxes/{box_id}", timeout=15) + except Exception: + pass + + +def box_metrics(box_id: str) -> dict | None: + try: + status, body = api("GET", f"boxes/{box_id}/metrics", timeout=15) + return body if status == 200 else None + except Exception: + return None + + +def exec_command(box_id: str, command: str = "echo", args: list[str] | None = None, timeout: int = 30) -> tuple[int, dict | None]: + payload = {"command": command} + if args: + payload["args"] = args + return api("POST", f"boxes/{box_id}/exec", payload, timeout=timeout) + + +# ── Stats ────────────────────────────────────────────────────────── + +def percentile(data: list[float], p: float) -> float: + if not data: + return 0.0 + s = sorted(data) + k = (len(s) - 1) * (p / 100.0) + f, c = math.floor(k), math.ceil(k) + if f == c: + return s[int(k)] + return s[f] * (c - k) + s[c] * (k - f) + + +def aggregate(values: list[float], name: str, unit: str = "ms") -> dict: + higher = name.endswith("_per_sec") or name.endswith("_rps") + return { + "name": name, + "unit": unit, + "higher_is_better": higher, + "min": min(values), + "p50": percentile(values, 50), + "p90": percentile(values, 90), + "p99": percentile(values, 99), + "max": max(values), + "mean": statistics.mean(values), + "stdev": statistics.stdev(values) if len(values) > 1 else 0.0, + "n": len(values), + } + + +def git_commit() -> str: + try: + r = subprocess.run( + ["git", "rev-parse", "--short", "HEAD"], + capture_output=True, text=True, timeout=5, + ) + return r.stdout.strip() if r.returncode == 0 else "unknown" + except Exception: + return "unknown" + + +def build_report(scenario: str, samples: list[dict], warmup: int, label: str | None = None) -> dict: + active = [s for s in samples if not s.get("warmup", False)] + metric_names: set[str] = set() + for s in active: + metric_names.update(s.get("metrics", {}).keys()) + + aggregates = [] + for name in sorted(metric_names): + values = [s["metrics"][name] for s in active if name in s.get("metrics", {})] + if values: + aggregates.append(aggregate(values, name)) + + c = ctx() + return { + "schema_version": "1.0", + "scenario": scenario, + "metadata": { + "started_at": datetime.now(timezone.utc).isoformat(), + "label": label or f"{c.url} ({IMAGE.split(':')[-1]})", + "git_commit": git_commit(), + "host": { + "kernel": platform.platform(), + "arch": platform.machine(), + "target": c.url, + }, + }, + "sample_count": len(active), + "warmup_count": warmup, + "samples": samples, + "aggregates": aggregates, + } + + +def print_aggregates(report: dict): + print() + print("=" * 70) + print(f"Aggregates — {report['scenario']} ({report['sample_count']} samples)") + print("=" * 70) + for agg in report["aggregates"]: + print( + f" {agg['name']:35s} " + f"p50={agg['p50']:8.1f} " + f"p90={agg['p90']:8.1f} " + f"p99={agg['p99']:8.1f} " + f"mean={agg['mean']:8.1f} " + f"stdev={agg['stdev']:7.1f}" + ) diff --git a/scripts/test/e2e/bench/scenarios/__init__.py b/scripts/test/e2e/bench/scenarios/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/scripts/test/e2e/bench/scenarios/density_parallel_create.py b/scripts/test/e2e/bench/scenarios/density_parallel_create.py new file mode 100644 index 000000000..f8523267d --- /dev/null +++ b/scripts/test/e2e/bench/scenarios/density_parallel_create.py @@ -0,0 +1,37 @@ +"""Parallel box creation: fire N creates concurrently, measure per-box and batch wall.""" +from __future__ import annotations +import concurrent.futures +import time +from bench.harness import create_box, delete_box + +SCENARIO = "density-parallel-create" +BOX_COUNT = 5 + + +def _create_one(idx: int) -> tuple[str, float]: + t = time.monotonic() + bid = create_box() + return bid, (time.monotonic() - t) * 1000 + + +def run_once(iteration: int) -> dict[str, float]: + t0 = time.monotonic() + with concurrent.futures.ThreadPoolExecutor(max_workers=BOX_COUNT) as pool: + futs = [pool.submit(_create_one, i) for i in range(BOX_COUNT)] + results = [f.result() for f in concurrent.futures.as_completed(futs)] + batch_wall = (time.monotonic() - t0) * 1000 + + bids = [r[0] for r in results] + per_box = [r[1] for r in results] + + try: + return { + "batch_wall_ms": batch_wall, + "box_count": float(BOX_COUNT), + "per_box_mean_ms": sum(per_box) / len(per_box), + "per_box_max_ms": max(per_box), + "per_box_min_ms": min(per_box), + } + finally: + for bid in bids: + delete_box(bid) diff --git a/scripts/test/e2e/bench/scenarios/latency_cold_start.py b/scripts/test/e2e/bench/scenarios/latency_cold_start.py new file mode 100644 index 000000000..3a7280efd --- /dev/null +++ b/scripts/test/e2e/bench/scenarios/latency_cold_start.py @@ -0,0 +1,39 @@ +"""Cold-start latency: POST /boxes wall clock + runner-reported create_duration_ms. + +Per iteration: create box → fetch metrics → first exec → delete. +""" +from __future__ import annotations +import time +from bench.harness import api, create_box, delete_box, box_metrics, exec_command + +SCENARIO = "latency-cold-start" + + +def run_once(iteration: int) -> dict[str, float]: + metrics: dict[str, float] = {} + + t0 = time.monotonic() + bid = create_box() + metrics["api_create_wall_ms"] = (time.monotonic() - t0) * 1000 + + try: + time.sleep(2) + rm = box_metrics(bid) + if rm: + if rm.get("create_duration_ms"): + metrics["runner_create_ms"] = float(rm["create_duration_ms"]) + if rm.get("boot_duration_ms"): + metrics["runner_boot_ms"] = float(rm["boot_duration_ms"]) + + t1 = time.monotonic() + exec_command(bid, "echo", ["ready"]) + metrics["first_exec_wall_ms"] = (time.monotonic() - t1) * 1000 + + if "first_exec_wall_ms" in metrics: + metrics["total_ready_wall_ms"] = ( + metrics["api_create_wall_ms"] + 2000 + metrics["first_exec_wall_ms"] + ) + finally: + delete_box(bid) + + return metrics diff --git a/scripts/test/e2e/bench/scenarios/latency_exec.py b/scripts/test/e2e/bench/scenarios/latency_exec.py new file mode 100644 index 000000000..ca64db1a6 --- /dev/null +++ b/scripts/test/e2e/bench/scenarios/latency_exec.py @@ -0,0 +1,25 @@ +"""Exec latency on a warm box: N serial execs, measure per-exec wall time. + +Creates one box, warms up, then runs N execs and records each. +""" +from __future__ import annotations +import time +from bench.harness import create_box, delete_box, exec_command + +SCENARIO = "latency-exec" + + +def run_once(iteration: int) -> dict[str, float]: + bid = create_box() + try: + time.sleep(2) + # warm exec + exec_command(bid, "echo", ["warmup"]) + + t0 = time.monotonic() + exec_command(bid, "echo", ["bench"]) + wall = (time.monotonic() - t0) * 1000 + + return {"exec_wall_ms": wall} + finally: + delete_box(bid) diff --git a/scripts/test/e2e/bench/scenarios/latency_lifecycle.py b/scripts/test/e2e/bench/scenarios/latency_lifecycle.py new file mode 100644 index 000000000..507501ac9 --- /dev/null +++ b/scripts/test/e2e/bench/scenarios/latency_lifecycle.py @@ -0,0 +1,34 @@ +"""Full lifecycle latency: create → exec → delete, wall clock per stage.""" +from __future__ import annotations +import time +from bench.harness import api, create_box, delete_box, exec_command + +SCENARIO = "latency-lifecycle" + + +def run_once(iteration: int) -> dict[str, float]: + metrics: dict[str, float] = {} + + t0 = time.monotonic() + bid = create_box() + metrics["create_ms"] = (time.monotonic() - t0) * 1000 + + try: + time.sleep(2) + + t1 = time.monotonic() + exec_command(bid, "echo", ["lifecycle-ok"]) + metrics["exec_ms"] = (time.monotonic() - t1) * 1000 + + t2 = time.monotonic() + api("DELETE", f"boxes/{bid}", timeout=30) + metrics["delete_ms"] = (time.monotonic() - t2) * 1000 + + metrics["total_lifecycle_ms"] = ( + metrics["create_ms"] + 2000 + metrics["exec_ms"] + metrics["delete_ms"] + ) + except Exception: + delete_box(bid) + raise + + return metrics diff --git a/scripts/test/e2e/bench/scenarios/stability_churn.py b/scripts/test/e2e/bench/scenarios/stability_churn.py new file mode 100644 index 000000000..a86d94a23 --- /dev/null +++ b/scripts/test/e2e/bench/scenarios/stability_churn.py @@ -0,0 +1,29 @@ +"""Stability churn: N create+exec+delete cycles, track per-cycle latency drift.""" +from __future__ import annotations +import time +from bench.harness import create_box, delete_box, exec_command + +SCENARIO = "stability-churn" +CYCLES = 10 + + +def run_once(iteration: int) -> dict[str, float]: + cycle_times = [] + for i in range(CYCLES): + t0 = time.monotonic() + bid = create_box() + try: + time.sleep(2) + exec_command(bid, "echo", [f"churn-{i}"]) + finally: + delete_box(bid) + cycle_times.append((time.monotonic() - t0) * 1000) + + return { + "cycles": float(CYCLES), + "first_cycle_ms": cycle_times[0], + "last_cycle_ms": cycle_times[-1], + "mean_cycle_ms": sum(cycle_times) / len(cycle_times), + "max_cycle_ms": max(cycle_times), + "drift_ms": cycle_times[-1] - cycle_times[0], + } diff --git a/scripts/test/e2e/bench/scenarios/stability_exec_loop.py b/scripts/test/e2e/bench/scenarios/stability_exec_loop.py new file mode 100644 index 000000000..c1755ea46 --- /dev/null +++ b/scripts/test/e2e/bench/scenarios/stability_exec_loop.py @@ -0,0 +1,34 @@ +"""Stability exec loop: many execs on one long-lived box, check for degradation.""" +from __future__ import annotations +import time +from bench.harness import create_box, delete_box, exec_command + +SCENARIO = "stability-exec-loop" +EXEC_COUNT = 50 + + +def run_once(iteration: int) -> dict[str, float]: + bid = create_box() + try: + time.sleep(2) + exec_command(bid, "echo", ["warmup"]) + + times = [] + for i in range(EXEC_COUNT): + t = time.monotonic() + exec_command(bid, "echo", [f"loop-{i}"]) + times.append((time.monotonic() - t) * 1000) + + first_10 = sum(times[:10]) / 10 + last_10 = sum(times[-10:]) / 10 + + return { + "exec_count": float(EXEC_COUNT), + "mean_exec_ms": sum(times) / len(times), + "max_exec_ms": max(times), + "first_10_mean_ms": first_10, + "last_10_mean_ms": last_10, + "degradation_ms": last_10 - first_10, + } + finally: + delete_box(bid) diff --git a/scripts/test/e2e/bench/scenarios/throughput_exec_parallel.py b/scripts/test/e2e/bench/scenarios/throughput_exec_parallel.py new file mode 100644 index 000000000..15148ced1 --- /dev/null +++ b/scripts/test/e2e/bench/scenarios/throughput_exec_parallel.py @@ -0,0 +1,36 @@ +"""Parallel exec throughput: fire N execs concurrently on one box.""" +from __future__ import annotations +import concurrent.futures +import time +from bench.harness import create_box, delete_box, exec_command + +SCENARIO = "throughput-exec-parallel" +CONCURRENCY = 10 + + +def _do_exec(bid: str, idx: int) -> float: + t = time.monotonic() + exec_command(bid, "echo", [f"par-{idx}"]) + return (time.monotonic() - t) * 1000 + + +def run_once(iteration: int) -> dict[str, float]: + bid = create_box() + try: + time.sleep(2) + exec_command(bid, "echo", ["warmup"]) + + t0 = time.monotonic() + with concurrent.futures.ThreadPoolExecutor(max_workers=CONCURRENCY) as pool: + futs = [pool.submit(_do_exec, bid, i) for i in range(CONCURRENCY)] + per_exec = [f.result() for f in concurrent.futures.as_completed(futs)] + wall = (time.monotonic() - t0) * 1000 + + return { + "batch_wall_ms": wall, + "concurrency": float(CONCURRENCY), + "per_exec_mean_ms": sum(per_exec) / len(per_exec), + "per_exec_max_ms": max(per_exec), + } + finally: + delete_box(bid) diff --git a/scripts/test/e2e/bench/scenarios/throughput_exec_serial.py b/scripts/test/e2e/bench/scenarios/throughput_exec_serial.py new file mode 100644 index 000000000..0b94293d5 --- /dev/null +++ b/scripts/test/e2e/bench/scenarios/throughput_exec_serial.py @@ -0,0 +1,28 @@ +"""Serial exec throughput: run N execs on one box, report execs/sec.""" +from __future__ import annotations +import time +from bench.harness import create_box, delete_box, exec_command + +SCENARIO = "throughput-exec-serial" +EXEC_COUNT = 20 + + +def run_once(iteration: int) -> dict[str, float]: + bid = create_box() + try: + time.sleep(2) + exec_command(bid, "echo", ["warmup"]) + + t0 = time.monotonic() + for i in range(EXEC_COUNT): + exec_command(bid, "echo", [f"iter-{i}"]) + elapsed = time.monotonic() - t0 + + return { + "total_wall_ms": elapsed * 1000, + "exec_count": float(EXEC_COUNT), + "execs_per_sec": EXEC_COUNT / elapsed if elapsed > 0 else 0, + "avg_exec_ms": (elapsed * 1000) / EXEC_COUNT, + } + finally: + delete_box(bid) diff --git a/scripts/test/e2e/lib/e2e_auth.py b/scripts/test/e2e/lib/e2e_auth.py new file mode 100644 index 000000000..fe7e67807 --- /dev/null +++ b/scripts/test/e2e/lib/e2e_auth.py @@ -0,0 +1,194 @@ +"""Shared auth helpers for REST E2E tests. + +The suite can run against the same REST path with either API-key or OIDC +bearer tokens. Python SDK bindings still expose the generic bearer slot as +`ApiKeyCredential`; the API only sees `Authorization: Bearer `. +""" +from __future__ import annotations + +from dataclasses import dataclass +from functools import lru_cache +import json +import os +from pathlib import Path +import shutil +import subprocess +import tomllib +from typing import Any +import urllib.error +import urllib.request + + +DEFAULT_PROFILE = os.environ.get("BOXLITE_E2E_PROFILE", "p1") + + +def credentials_path() -> Path: + if os.environ.get("BOXLITE_HOME"): + return Path(os.environ["BOXLITE_HOME"]) / "credentials.toml" + return Path.home() / ".boxlite" / "credentials.toml" + + +def load_profile(name: str | None = None, *, required: bool = True) -> dict[str, Any]: + profile_name = name or DEFAULT_PROFILE + path = credentials_path() + if not path.exists(): + if required: + raise RuntimeError( + f"{path} missing; run scripts/test/e2e/fixture_setup.py first" + ) + return {} + data = tomllib.loads(path.read_text()) + profile = data.get("profiles", {}).get(profile_name) + if not profile: + if required: + raise RuntimeError(f"profile {profile_name!r} not in {path}; run fixture_setup.py") + return {} + return profile + + +@dataclass(frozen=True) +class E2EAuthContext: + auth: str + profile_name: str + url: str + token: str + path_prefix: str + + def auth_headers(self, *, content_type: bool = False) -> dict[str, str]: + headers = {"Authorization": f"Bearer {self.token}"} + if content_type: + headers["Content-Type"] = "application/json" + return headers + + def url_for(self, path: str) -> str: + if path.startswith("http://") or path.startswith("https://"): + return path + return f"{self.url.rstrip('/')}/{path.lstrip('/')}" + + def v1(self, path: str) -> str: + path = path.lstrip("/") + if self.path_prefix: + return f"/v1/{self.path_prefix}/{path}" + return f"/v1/{path}" + + def api_key_sdk_env(self) -> dict[str, str]: + if self.auth != "api-key": + raise RuntimeError("cross-language SDK E2E only supports AUTH=api-key today") + return { + "BOXLITE_E2E_URL": self.url, + "BOXLITE_E2E_API_KEY": self.token, + "BOXLITE_E2E_PREFIX": self.path_prefix, + } + + +@lru_cache(maxsize=1) +def auth_context() -> E2EAuthContext: + profile_name = os.environ.get("BOXLITE_E2E_PROFILE", DEFAULT_PROFILE) + profile = load_profile(profile_name, required=False) + auth = os.environ.get("BOXLITE_E2E_AUTH", profile.get("auth_method", "api_key")) + auth = auth.replace("_", "-").lower() + if auth not in ("api-key", "oidc"): + raise RuntimeError(f"BOXLITE_E2E_AUTH must be api-key or oidc, got {auth!r}") + + url = os.environ.get("BOXLITE_E2E_API_URL") or profile.get("url") + if not url: + raise RuntimeError(f"profile {profile_name!r} has no url") + + env_token = None + if auth == "api-key": + env_token = os.environ.get("BOXLITE_E2E_API_KEY") + token = env_token or profile.get("api_key") + missing = "BOXLITE_E2E_API_KEY or profile.api_key" + else: + env_token = os.environ.get("BOXLITE_E2E_OIDC_TOKEN") + if not env_token: + profile = refresh_stored_oidc_profile(profile_name, profile) + token = env_token or profile.get("access_token") + missing = "BOXLITE_E2E_OIDC_TOKEN or profile.access_token" + if not token: + raise RuntimeError(f"AUTH={auth} requires {missing}") + + explicit_prefix = os.environ.get("BOXLITE_E2E_PREFIX") + if explicit_prefix is not None: + path_prefix = explicit_prefix + elif os.environ.get("BOXLITE_E2E_DISCOVER_PREFIX", "1") != "0": + path_prefix = discover_path_prefix(url, token) + else: + path_prefix = profile.get("path_prefix") or "" + + return E2EAuthContext( + auth=auth, + profile_name=profile_name, + url=url, + token=token, + path_prefix=path_prefix, + ) + + +def refresh_stored_oidc_profile(profile_name: str, profile: dict[str, Any]) -> dict[str, Any]: + if not profile: + return profile + cli = os.environ.get("BOXLITE_E2E_CLI") or shutil.which("boxlite") + if not cli: + raise RuntimeError( + "AUTH=oidc with a stored profile requires the boxlite CLI so the " + "access token can be refreshed; set BOXLITE_E2E_OIDC_TOKEN to run " + "without the CLI" + ) + + result = subprocess.run( + [cli, "--profile", profile_name, "auth", "whoami"], + text=True, + capture_output=True, + timeout=60, + check=False, + ) + if result.returncode != 0: + raise RuntimeError( + "failed to refresh stored OIDC profile with `boxlite auth whoami`: " + f"{result.stderr or result.stdout}" + ) + return load_profile(profile_name) + + +def discover_path_prefix(url: str, token: str) -> str: + req = urllib.request.Request( + f"{url.rstrip('/')}/v1/me", + method="GET", + headers={"Authorization": f"Bearer {token}"}, + ) + try: + with urllib.request.urlopen(req, timeout=15) as response: + body = json.loads(response.read() or "null") + except urllib.error.HTTPError as exc: + raw = exc.read().decode("utf-8", "replace") + raise RuntimeError(f"GET /v1/me failed with HTTP {exc.code}: {raw}") from exc + return (body or {}).get("path_prefix") or "" + + +def request_json( + method: str, + path: str, + body: dict[str, Any] | None = None, + *, + timeout: int = 30, + authorized: bool = True, +) -> tuple[int, dict[str, Any] | None]: + ctx = auth_context() + headers = ctx.auth_headers(content_type=body is not None) if authorized else {} + req = urllib.request.Request( + ctx.url_for(path), + method=method, + headers=headers, + data=json.dumps(body).encode() if body is not None else None, + ) + try: + with urllib.request.urlopen(req, timeout=timeout) as response: + raw = response.read() + return response.status, json.loads(raw) if raw else None + except urllib.error.HTTPError as exc: + raw = exc.read() + try: + return exc.code, json.loads(raw) if raw else None + except json.JSONDecodeError: + return exc.code, {"_raw": raw.decode("utf-8", "replace")} From 2fb926f48d0cf4bd66a98122875a279b82ffbbdf Mon Sep 17 00:00:00 2001 From: G4614 <92488762+G4614@users.noreply.github.com> Date: Wed, 17 Jun 2026 20:53:54 +0800 Subject: [PATCH 2/4] =?UTF-8?q?fix(bench):=20address=20CodeRabbit=20review?= =?UTF-8?q?=20=E2=80=94=20exec=20validation,=20cleanup,=20compare?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - exec_command() now validates HTTP status centrally (fail-fast on non-2xx) - density_parallel_create: track created box IDs incrementally for cleanup - stability_churn/exec_loop: add total_wall_ms as first metric key - latency_cold_start: use `is not None` for zero-value runner metrics - __main__.py: mkdir parents for --out, strict --on field check, fix arrow - README.md: add blank lines around markdown tables --- scripts/test/e2e/bench/README.md | 4 +++ scripts/test/e2e/bench/__main__.py | 13 ++++++--- scripts/test/e2e/bench/harness.py | 5 +++- .../scenarios/density_parallel_create.py | 28 ++++++++++--------- .../e2e/bench/scenarios/latency_cold_start.py | 4 +-- .../e2e/bench/scenarios/stability_churn.py | 1 + .../bench/scenarios/stability_exec_loop.py | 1 + 7 files changed, 36 insertions(+), 20 deletions(-) diff --git a/scripts/test/e2e/bench/README.md b/scripts/test/e2e/bench/README.md index 12c496d89..c6af44f3d 100644 --- a/scripts/test/e2e/bench/README.md +++ b/scripts/test/e2e/bench/README.md @@ -30,6 +30,7 @@ Run from the repo root (`scripts/test/e2e/` must be in the Python path). ## Scenarios ### Latency + | Scenario | What it measures | |----------|------------------| | `latency-cold-start` | POST /boxes wall clock + runner create_duration_ms + first exec | @@ -37,17 +38,20 @@ Run from the repo root (`scripts/test/e2e/` must be in the Python path). | `latency-lifecycle` | Full create → exec → delete cycle | ### Throughput + | Scenario | What it measures | |----------|------------------| | `throughput-exec-serial` | 20 serial execs on one box; execs/sec | | `throughput-exec-parallel` | 10 concurrent execs on one box; batch wall + per-exec | ### Density + | Scenario | What it measures | |----------|------------------| | `density-parallel-create` | 5 boxes created concurrently; per-box and batch wall | ### Stability + | Scenario | What it measures | |----------|------------------| | `stability-churn` | 10 create+exec+delete cycles; latency drift | diff --git a/scripts/test/e2e/bench/__main__.py b/scripts/test/e2e/bench/__main__.py index 9f1784e70..b09af26a2 100644 --- a/scripts/test/e2e/bench/__main__.py +++ b/scripts/test/e2e/bench/__main__.py @@ -108,7 +108,9 @@ def cmd_run(args): print_aggregates(report) if args.out: - Path(args.out).write_text(json.dumps(report, indent=2)) + out = Path(args.out) + out.parent.mkdir(parents=True, exist_ok=True) + out.write_text(json.dumps(report, indent=2)) print(f"\nReport: {args.out}") else: print(f"\n{json.dumps(report, indent=2)}") @@ -148,8 +150,11 @@ def cmd_compare(args): print(f" {name:35s} {status}") continue - bv = ba.get(field, ba.get("p99", 0)) - cv = ca.get(field, ca.get("p99", 0)) + if field not in ba or field not in ca: + print(f" {name:35s} SKIP: missing '{field}'") + continue + bv = float(ba[field]) + cv = float(ca[field]) higher = ba.get("higher_is_better", False) if bv == 0: @@ -161,7 +166,7 @@ def cmd_compare(args): regressed = ratio > threshold marker = " ← REGRESSION" if regressed else "" - direction = "↑" if (cv > bv) != higher else "↓" + direction = "→" if cv == bv else ("↑" if cv > bv else "↓") print(f" {name:35s} {bv:10.1f} → {cv:10.1f} {direction} {ratio:+.1%}{marker}") if regressed: regressions.append(name) diff --git a/scripts/test/e2e/bench/harness.py b/scripts/test/e2e/bench/harness.py index 0826e468c..cb62e7738 100644 --- a/scripts/test/e2e/bench/harness.py +++ b/scripts/test/e2e/bench/harness.py @@ -77,7 +77,10 @@ def exec_command(box_id: str, command: str = "echo", args: list[str] | None = No payload = {"command": command} if args: payload["args"] = args - return api("POST", f"boxes/{box_id}/exec", payload, timeout=timeout) + status, body = api("POST", f"boxes/{box_id}/exec", payload, timeout=timeout) + if status not in (200, 201): + raise RuntimeError(f"exec failed: {status} {body}") + return status, body # ── Stats ────────────────────────────────────────────────────────── diff --git a/scripts/test/e2e/bench/scenarios/density_parallel_create.py b/scripts/test/e2e/bench/scenarios/density_parallel_create.py index f8523267d..e5395c93c 100644 --- a/scripts/test/e2e/bench/scenarios/density_parallel_create.py +++ b/scripts/test/e2e/bench/scenarios/density_parallel_create.py @@ -15,22 +15,24 @@ def _create_one(idx: int) -> tuple[str, float]: def run_once(iteration: int) -> dict[str, float]: - t0 = time.monotonic() - with concurrent.futures.ThreadPoolExecutor(max_workers=BOX_COUNT) as pool: - futs = [pool.submit(_create_one, i) for i in range(BOX_COUNT)] - results = [f.result() for f in concurrent.futures.as_completed(futs)] - batch_wall = (time.monotonic() - t0) * 1000 - - bids = [r[0] for r in results] - per_box = [r[1] for r in results] - + bids: list[str] = [] try: + t0 = time.monotonic() + with concurrent.futures.ThreadPoolExecutor(max_workers=BOX_COUNT) as pool: + futs = [pool.submit(_create_one, i) for i in range(BOX_COUNT)] + results = [] + for f in concurrent.futures.as_completed(futs): + bid, elapsed = f.result() + bids.append(bid) + results.append(elapsed) + batch_wall = (time.monotonic() - t0) * 1000 + return { "batch_wall_ms": batch_wall, - "box_count": float(BOX_COUNT), - "per_box_mean_ms": sum(per_box) / len(per_box), - "per_box_max_ms": max(per_box), - "per_box_min_ms": min(per_box), + "box_count": float(len(results)), + "per_box_mean_ms": sum(results) / len(results), + "per_box_max_ms": max(results), + "per_box_min_ms": min(results), } finally: for bid in bids: diff --git a/scripts/test/e2e/bench/scenarios/latency_cold_start.py b/scripts/test/e2e/bench/scenarios/latency_cold_start.py index 3a7280efd..ed1cfa91e 100644 --- a/scripts/test/e2e/bench/scenarios/latency_cold_start.py +++ b/scripts/test/e2e/bench/scenarios/latency_cold_start.py @@ -20,9 +20,9 @@ def run_once(iteration: int) -> dict[str, float]: time.sleep(2) rm = box_metrics(bid) if rm: - if rm.get("create_duration_ms"): + if rm.get("create_duration_ms") is not None: metrics["runner_create_ms"] = float(rm["create_duration_ms"]) - if rm.get("boot_duration_ms"): + if rm.get("boot_duration_ms") is not None: metrics["runner_boot_ms"] = float(rm["boot_duration_ms"]) t1 = time.monotonic() diff --git a/scripts/test/e2e/bench/scenarios/stability_churn.py b/scripts/test/e2e/bench/scenarios/stability_churn.py index a86d94a23..a61c185f0 100644 --- a/scripts/test/e2e/bench/scenarios/stability_churn.py +++ b/scripts/test/e2e/bench/scenarios/stability_churn.py @@ -20,6 +20,7 @@ def run_once(iteration: int) -> dict[str, float]: cycle_times.append((time.monotonic() - t0) * 1000) return { + "total_wall_ms": sum(cycle_times), "cycles": float(CYCLES), "first_cycle_ms": cycle_times[0], "last_cycle_ms": cycle_times[-1], diff --git a/scripts/test/e2e/bench/scenarios/stability_exec_loop.py b/scripts/test/e2e/bench/scenarios/stability_exec_loop.py index c1755ea46..eacdc38b1 100644 --- a/scripts/test/e2e/bench/scenarios/stability_exec_loop.py +++ b/scripts/test/e2e/bench/scenarios/stability_exec_loop.py @@ -23,6 +23,7 @@ def run_once(iteration: int) -> dict[str, float]: last_10 = sum(times[-10:]) / 10 return { + "total_wall_ms": sum(times), "exec_count": float(EXEC_COUNT), "mean_exec_ms": sum(times) / len(times), "max_exec_ms": max(times), From 47b74003da1ec40c2f2ab3883e6c803566444500 Mon Sep 17 00:00:00 2001 From: G4614 <92488762+G4614@users.noreply.github.com> Date: Wed, 17 Jun 2026 21:00:27 +0800 Subject: [PATCH 3/4] fix(bench): detect zero-compared metrics in compare to prevent false pass When --on field typo causes all metrics to SKIP, exit with error instead of reporting PASS with empty regressions list. --- scripts/test/e2e/bench/__main__.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/scripts/test/e2e/bench/__main__.py b/scripts/test/e2e/bench/__main__.py index b09af26a2..c9364cced 100644 --- a/scripts/test/e2e/bench/__main__.py +++ b/scripts/test/e2e/bench/__main__.py @@ -142,6 +142,7 @@ def cmd_compare(args): print() all_names = sorted(set(base_aggs) | set(curr_aggs)) + compared = 0 for name in all_names: ba = base_aggs.get(name) ca = curr_aggs.get(name) @@ -155,6 +156,7 @@ def cmd_compare(args): continue bv = float(ba[field]) cv = float(ca[field]) + compared += 1 higher = ba.get("higher_is_better", False) if bv == 0: @@ -171,6 +173,10 @@ def cmd_compare(args): if regressed: regressions.append(name) + if compared == 0: + print(f"\nERROR: no metrics compared — check that '{field}' exists in both reports") + sys.exit(1) + if regressions: print(f"\nFAIL: {len(regressions)} regression(s) exceed {threshold:.0%} threshold") sys.exit(1) From 8868534f2ebd63f0ab644fac2ef355e90a397749 Mon Sep 17 00:00:00 2001 From: G4614 <92488762+G4614@users.noreply.github.com> Date: Wed, 17 Jun 2026 21:37:11 +0800 Subject: [PATCH 4/4] =?UTF-8?q?feat(bench):=20add=20latency-restart=20scen?= =?UTF-8?q?ario=20=E2=80=94=20stop=20=E2=86=92=20start=20=E2=86=92=20exec?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Measures per-stage wall time for stopping a running box and restarting it: stop_wall_ms, start_wall_ms, first_exec_after_restart_ms, total_restart_ms. Also adds stop_box(), start_box(), wait_box_status() helpers to harness. --- scripts/test/e2e/bench/README.md | 1 + scripts/test/e2e/bench/__main__.py | 2 ++ scripts/test/e2e/bench/harness.py | 25 +++++++++++++ .../e2e/bench/scenarios/latency_restart.py | 36 +++++++++++++++++++ 4 files changed, 64 insertions(+) create mode 100644 scripts/test/e2e/bench/scenarios/latency_restart.py diff --git a/scripts/test/e2e/bench/README.md b/scripts/test/e2e/bench/README.md index c6af44f3d..c2d128c27 100644 --- a/scripts/test/e2e/bench/README.md +++ b/scripts/test/e2e/bench/README.md @@ -36,6 +36,7 @@ Run from the repo root (`scripts/test/e2e/` must be in the Python path). | `latency-cold-start` | POST /boxes wall clock + runner create_duration_ms + first exec | | `latency-exec` | Single exec on a warm box | | `latency-lifecycle` | Full create → exec → delete cycle | +| `latency-restart` | Stop → start → exec on a running box | ### Throughput diff --git a/scripts/test/e2e/bench/__main__.py b/scripts/test/e2e/bench/__main__.py index c9364cced..ecee28949 100644 --- a/scripts/test/e2e/bench/__main__.py +++ b/scripts/test/e2e/bench/__main__.py @@ -21,6 +21,7 @@ from bench.scenarios import latency_cold_start from bench.scenarios import latency_exec from bench.scenarios import latency_lifecycle +from bench.scenarios import latency_restart from bench.scenarios import throughput_exec_serial from bench.scenarios import throughput_exec_parallel from bench.scenarios import density_parallel_create @@ -31,6 +32,7 @@ latency_cold_start.SCENARIO: latency_cold_start, latency_exec.SCENARIO: latency_exec, latency_lifecycle.SCENARIO: latency_lifecycle, + latency_restart.SCENARIO: latency_restart, throughput_exec_serial.SCENARIO: throughput_exec_serial, throughput_exec_parallel.SCENARIO: throughput_exec_parallel, density_parallel_create.SCENARIO: density_parallel_create, diff --git a/scripts/test/e2e/bench/harness.py b/scripts/test/e2e/bench/harness.py index cb62e7738..f94fa2048 100644 --- a/scripts/test/e2e/bench/harness.py +++ b/scripts/test/e2e/bench/harness.py @@ -73,6 +73,31 @@ def box_metrics(box_id: str) -> dict | None: return None +def stop_box(box_id: str, timeout: int = 30): + status, body = api("POST", f"boxes/{box_id}/stop", timeout=timeout) + if status >= 300: + raise RuntimeError(f"stop_box failed: {status} {body}") + return status, body + + +def start_box(box_id: str, timeout: int = 30): + status, body = api("POST", f"boxes/{box_id}/start", timeout=timeout) + if status >= 300: + raise RuntimeError(f"start_box failed: {status} {body}") + return status, body + + +def wait_box_status(box_id: str, target: str, timeout: int = 60, poll_interval: float = 1.0): + """Poll until box reaches target status.""" + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + status, body = api("GET", f"boxes/{box_id}", timeout=10) + if status == 200 and body and body.get("status") == target: + return + time.sleep(poll_interval) + raise RuntimeError(f"box {box_id} not '{target}' after {timeout}s") + + def exec_command(box_id: str, command: str = "echo", args: list[str] | None = None, timeout: int = 30) -> tuple[int, dict | None]: payload = {"command": command} if args: diff --git a/scripts/test/e2e/bench/scenarios/latency_restart.py b/scripts/test/e2e/bench/scenarios/latency_restart.py new file mode 100644 index 000000000..fc3eed729 --- /dev/null +++ b/scripts/test/e2e/bench/scenarios/latency_restart.py @@ -0,0 +1,36 @@ +"""Restart latency: stop a running box and start it again, measure restart wall time.""" +from __future__ import annotations +import time +from bench.harness import create_box, delete_box, exec_command, stop_box, start_box, wait_box_status + +SCENARIO = "latency-restart" + + +def run_once(iteration: int) -> dict[str, float]: + bid = create_box() + try: + time.sleep(2) + exec_command(bid, "echo", ["pre-stop"]) + + t0 = time.monotonic() + stop_box(bid) + wait_box_status(bid, "stopped", timeout=30) + stop_wall = (time.monotonic() - t0) * 1000 + + t1 = time.monotonic() + start_box(bid) + wait_box_status(bid, "running", timeout=30) + start_wall = (time.monotonic() - t1) * 1000 + + t2 = time.monotonic() + exec_command(bid, "echo", ["post-restart"]) + first_exec_wall = (time.monotonic() - t2) * 1000 + + return { + "stop_wall_ms": stop_wall, + "start_wall_ms": start_wall, + "first_exec_after_restart_ms": first_exec_wall, + "total_restart_ms": stop_wall + start_wall + first_exec_wall, + } + finally: + delete_box(bid)