From ab78007f491126fcd4b6b9183d0e78da7cc39655 Mon Sep 17 00:00:00 2001 From: Alexandre Marques Date: Tue, 21 Jul 2026 17:47:39 -0400 Subject: [PATCH 1/5] feat(evaluate): add interactivity plot metric (tok/s/user vs tok/s/GPU) Plot goodput-style curves using 1000/ITL for interactivity and system throughput per GPU, with --num-gpus since GPU count is not in result files. Co-authored-by: Cursor --- scripts/evaluate/perf_utils.py | 171 ++++++++++++++++++++++++++++++++- scripts/evaluate/plot.py | 102 ++++++++++++++++++-- 2 files changed, 262 insertions(+), 11 deletions(-) diff --git a/scripts/evaluate/perf_utils.py b/scripts/evaluate/perf_utils.py index 27903d7fa..c558595ba 100644 --- a/scripts/evaluate/perf_utils.py +++ b/scripts/evaluate/perf_utils.py @@ -33,12 +33,15 @@ # Metric definitions (for plotting) # --------------------------------------------------------------------------- +DEFAULT_XLABEL = "Requests per second (RPS)" + METRICS: dict[str, dict[str, str | bool]] = { "latency": { "csv_col": "latency_median_s", "json_key": "request_latency", "stat": "median", "label": "Median Latency (s)", + "xlabel": DEFAULT_XLABEL, "increasing": True, }, "itl": { @@ -46,6 +49,7 @@ "json_key": "inter_token_latency_ms", "stat": "median", "label": "Median ITL (ms)", + "xlabel": DEFAULT_XLABEL, "increasing": True, }, "ttft": { @@ -53,6 +57,7 @@ "json_key": "time_to_first_token_ms", "stat": "median", "label": "Median TTFT (ms)", + "xlabel": DEFAULT_XLABEL, "increasing": True, }, "output_tps": { @@ -60,7 +65,20 @@ "json_key": "output_tokens_per_second", "stat": "median", "label": "Output Tokens/s", + "xlabel": DEFAULT_XLABEL, + "increasing": False, + }, + # x = 1000 / ITL_ms (tok/s/user); y = system_tps / num_gpus. + # System TPS is total_output_tokens / duration (falls back to guidellm mean), + # not the per-request median. Requires --num-gpus. + "interactivity": { + "csv_col": "output_tps_mean", + "json_key": "output_tokens_per_second", + "stat": "mean", + "label": "Token throughput per GPU (tok/s/GPU)", + "xlabel": "Interactivity (tok/s/user)", "increasing": False, + "requires_num_gpus": True, }, } @@ -95,6 +113,7 @@ "itl_median_ms", "ttft_median_ms", "output_tps_median", + "output_tps_mean", "total_output_tokens", ] @@ -123,10 +142,27 @@ class Vector(Metric): # --------------------------------------------------------------------------- +def _subset_from_sweep_json(data: dict) -> str: + """Extract subset name from guidellm 0.7+ or legacy 0.6 sweep JSON.""" + try: + return Path( + data["config"]["spec"]["data"][0]["load_kwargs"]["data_files"] + ).stem + except (KeyError, TypeError, IndexError): + pass + try: + return Path(data["args"]["data_args"][0]["data_files"]).stem + except (KeyError, TypeError, IndexError): + return "unknown" + + def _load_csv( filepath: Path, metric_name: str, ) -> dict[str, list[tuple[float, float]]]: + if metric_name == "interactivity": + return _load_interactivity_csv(filepath) + cfg = METRICS[metric_name] result: dict[str, list[tuple[float, float]]] = defaultdict(list) with filepath.open(newline="") as f: @@ -147,11 +183,14 @@ def _load_json( filepath: Path, metric_name: str, ) -> dict[str, list[tuple[float, float]]]: + if metric_name == "interactivity": + return _load_interactivity_json(filepath) + cfg = METRICS[metric_name] with filepath.open() as f: data = json.load(f) - subset = Path(data["config"]["spec"]["data"][0]["load_kwargs"]["data_files"]).stem + subset = _subset_from_sweep_json(data) points: list[tuple[float, float]] = [] for bench in data.get("benchmarks", []): if bench.get("config", {}).get("strategy", {}).get("type_") != "constant": @@ -167,6 +206,108 @@ def _load_json( return {subset: points} if points else {} +def _system_tps_from_bench(bench: dict) -> float | None: + """Aggregate output tokens/s for a benchmark (not per-request median).""" + metrics = bench.get("metrics", {}) + try: + out = metrics["output_token_count"]["successful"] + duration = float(bench["duration"]) + total = float(out["total_sum"]) + if duration > 0: + return total / duration + except (KeyError, TypeError, ValueError, ZeroDivisionError): + pass + try: + return float(metrics["output_tokens_per_second"]["successful"]["mean"]) + except (KeyError, TypeError, ValueError): + return None + + +def _interactivity_point_from_bench(bench: dict) -> tuple[float, float] | None: + """Return ``(itl_ms, system_tps)`` for a constant-rate benchmark.""" + if bench.get("config", {}).get("strategy", {}).get("type_") != "constant": + return None + try: + itl_ms = float( + bench["metrics"]["inter_token_latency_ms"]["successful"]["median"] + ) + except (KeyError, TypeError, ValueError): + return None + system_tps = _system_tps_from_bench(bench) + if system_tps is None or itl_ms <= 0: + return None + return (itl_ms, system_tps) + + +def _load_interactivity_json( + filepath: Path, +) -> dict[str, list[tuple[float, float]]]: + with filepath.open() as f: + data = json.load(f) + + subset = _subset_from_sweep_json(data) + points: list[tuple[float, float]] = [] + for bench in data.get("benchmarks", []): + pt = _interactivity_point_from_bench(bench) + if pt is not None: + points.append(pt) + points.sort(key=lambda p: p[0]) + return {subset: points} if points else {} + + +def _load_interactivity_from_artifact_jsons( + csv_path: Path, +) -> dict[str, list[tuple[float, float]]]: + """Fall back to sibling ``artifacts/run_*.json`` when CSV lacks system TPS.""" + artifacts = csv_path.parent / "artifacts" + if not artifacts.is_dir(): + return {} + combined: dict[str, list[tuple[float, float]]] = defaultdict(list) + for json_path in sorted(artifacts.glob("run_*.json")): + for subset, points in _load_interactivity_json(json_path).items(): + combined[subset].extend(points) + return dict(combined) + + +def _load_interactivity_csv( + filepath: Path, +) -> dict[str, list[tuple[float, float]]]: + """Load ``(itl_ms, system_tps)`` from CSV, or from sibling JSONs if needed.""" + result: dict[str, list[tuple[float, float]]] = defaultdict(list) + has_mean = False + with filepath.open(newline="") as f: + reader = csv.DictReader(f) + fieldnames = reader.fieldnames or [] + has_mean = "output_tps_mean" in fieldnames + if has_mean: + for row in reader: + if row.get("strategy") != "constant": + continue + try: + subset = re.sub(r"^run_", "", row.get("subset", "unknown")) + itl_ms = float(row["itl_median_ms"]) + system_tps = float(row["output_tps_mean"]) + if itl_ms > 0: + result[subset].append((itl_ms, system_tps)) + except (ValueError, KeyError): + continue + + if result: + return dict(result) + + fallback = _load_interactivity_from_artifact_jsons(filepath) + if fallback: + return fallback + + hint = ( + "CSV is missing output_tps_mean and no artifacts/run_*.json found. " + "Re-run evaluation or point --source at the sweep JSON." + if not has_mean + else "No constant-rate interactivity rows found." + ) + raise ValueError(f"Cannot load interactivity from {filepath}: {hint}") + + def load_data( filepath: Path, metric_name: str, @@ -180,6 +321,29 @@ def load_data( ) +def transform_interactivity( + data: dict[str, list[tuple[float, float]]], + num_gpus: int, +) -> dict[str, list[tuple[float, float]]]: + """Transform ``(itl_ms, system_tps)`` into ``(tok/s/user, tok/s/gpu)``. + + Interactivity (x) = ``1000 / ITL_ms``; throughput/GPU (y) = ``system_tps / num_gpus``. + """ + if num_gpus <= 0: + raise ValueError(f"num_gpus must be positive, got {num_gpus}") + + result: dict[str, list[tuple[float, float]]] = {} + for subset, points in data.items(): + transformed: list[tuple[float, float]] = [] + for itl_ms, system_tps in points: + if itl_ms <= 0: + continue + transformed.append((1000.0 / itl_ms, system_tps / num_gpus)) + if transformed: + result[subset] = transformed + return result + + def parse_source_args(source_args: list[str]) -> dict[str, list[Path]]: """Parse ``LABEL=PATH`` strings into ``{label: [path, ...]}``.""" result: dict[str, list[Path]] = defaultdict(list) @@ -386,6 +550,11 @@ def parse_sweep_file(filepath: Path) -> list[dict]: for metric_key, csv_key in METRICS_TO_EXTRACT: val = metrics.get(metric_key, {}) row[csv_key] = val.get("successful", {}).get("median", "") + row["output_tps_mean"] = ( + metrics.get("output_tokens_per_second", {}) + .get("successful", {}) + .get("mean", "") + ) out_tok = metrics.get("output_tokens", {}) row["total_output_tokens"] = out_tok.get("successful", {}).get("sum", "") rows.append(row) diff --git a/scripts/evaluate/plot.py b/scripts/evaluate/plot.py index 4ff918260..32558e669 100644 --- a/scripts/evaluate/plot.py +++ b/scripts/evaluate/plot.py @@ -9,7 +9,12 @@ python plot.py compare \\ --source "No Spec=nospec/results.csv" \\ --source "Eagle3=eagle3/results.csv" \\ - --metric latency --metric itl + --metric latency --metric itl --title "Qwen3-8B" + + python plot.py compare \\ + --source "No Spec=nospec/results.csv" \\ + --source "Eagle3=eagle3/results.csv" \\ + --metric interactivity --num-gpus 1 --title "Qwen3-8B" python plot.py speedup \\ --baseline "No Spec=nospec/results.csv" \\ @@ -34,6 +39,7 @@ parse_source_args, pretty_subset, smooth_curve, + transform_interactivity, ) COLOR_CYCLE = [ @@ -96,9 +102,35 @@ def _plot_compare_subset( ax.plot(x_smooth, y_smooth, color=color, linewidth=2.5, label=label, zorder=4) +def _require_num_gpus(metrics: list[str], num_gpus: int | None) -> None: + needs_gpus = any(METRICS[m].get("requires_num_gpus") for m in metrics) + if needs_gpus and num_gpus is None: + print( + "[ERROR] --num-gpus is required for metric 'interactivity' " + "(GPU count is not stored in result CSVs/JSONs).", + file=sys.stderr, + ) + sys.exit(1) + if num_gpus is not None and num_gpus <= 0: + print(f"[ERROR] --num-gpus must be positive, got {num_gpus}", file=sys.stderr) + sys.exit(1) + + +def _maybe_transform_interactivity( + data: dict[str, list[tuple[float, float]]], + metric_name: str, + num_gpus: int | None, +) -> dict[str, list[tuple[float, float]]]: + if not METRICS[metric_name].get("requires_num_gpus"): + return data + assert num_gpus is not None # validated by _require_num_gpus + return transform_interactivity(data, num_gpus) + + def run_compare(args: argparse.Namespace) -> None: metrics = args.metric or ["latency"] subset_filter = set(args.subsets.split(",")) if args.subsets else None + _require_num_gpus(metrics, args.num_gpus) try: sources = parse_source_args(args.source) @@ -112,6 +144,12 @@ def run_compare(args: argparse.Namespace) -> None: for metric_name in metrics: metric_cfg = METRICS[metric_name] all_data = _collect_all_data(sources, metric_name) + all_data = { + label: _maybe_transform_interactivity( + subset_data, metric_name, args.num_gpus + ) + for label, subset_data in all_data.items() + } all_subsets: set[str] = set() for label_data in all_data.values(): @@ -139,9 +177,13 @@ def run_compare(args: argparse.Namespace) -> None: source_labels, ) - ax.set_title(metric_cfg["label"], fontsize=14, fontweight="bold") - ax.set_xlabel("Requests per Second", fontsize=12) - ax.set_ylabel(metric_cfg["label"], fontsize=12) + title_parts = [] + if args.title: + title_parts.append(args.title) + title_parts.extend(pretty_subset(s) for s in sorted(all_subsets)) + ax.set_title(", ".join(title_parts) or str(metric_cfg["label"]), fontsize=14, fontweight="bold") + ax.set_xlabel(str(metric_cfg["xlabel"]), fontsize=12) + ax.set_ylabel(str(metric_cfg["label"]), fontsize=12) ax.legend(framealpha=0.9) ax.grid(True, alpha=0.3) fig.tight_layout() @@ -302,8 +344,8 @@ def _plot_speedup_subset( title_parts.append(title_prefix) title_parts.append(pretty_subset(subset)) ax.set_title(", ".join(title_parts), fontsize=14, fontweight="bold") - ax.set_xlabel("Requests per second (RPS)", fontsize=12) - ax.set_ylabel(metric_cfg["label"], fontsize=12) + ax.set_xlabel(str(metric_cfg["xlabel"]), fontsize=12) + ax.set_ylabel(str(metric_cfg["label"]), fontsize=12) ax.legend(framealpha=0.9) ax.grid(True, alpha=0.3) fig.tight_layout() @@ -313,11 +355,12 @@ def _plot_speedup_subset( def run_speedup(args: argparse.Namespace) -> None: metrics = args.metric or ["latency"] subset_filter = set(args.subsets.split(",")) if args.subsets else None + _require_num_gpus(metrics, args.num_gpus) args.output_dir.mkdir(parents=True, exist_ok=True) for metric_name in metrics: metric_cfg = METRICS[metric_name] - increasing = metric_cfg["increasing"] + increasing = bool(metric_cfg["increasing"]) try: baseline_label, baseline_data = _collect_points(args.baseline, metric_name) @@ -326,6 +369,13 @@ def run_speedup(args: argparse.Namespace) -> None: print(f"[ERROR] {e}", file=sys.stderr) sys.exit(1) + baseline_data = _maybe_transform_interactivity( + baseline_data, metric_name, args.num_gpus + ) + target_data = _maybe_transform_interactivity( + target_data, metric_name, args.num_gpus + ) + all_subsets = set(baseline_data.keys()) & set(target_data.keys()) if subset_filter: all_subsets &= subset_filter @@ -354,8 +404,10 @@ def run_speedup(args: argparse.Namespace) -> None: title_prefix=args.title, ) if not ok: + x_name = metric_cfg["xlabel"] print( - f"[WARN] No overlapping RPS range for subset '{subset}', skipping", + f"[WARN] No overlapping {x_name} range for subset '{subset}', " + "skipping", file=sys.stderr, ) plt.close(fig) @@ -380,9 +432,11 @@ def main() -> None: epilog=( "examples:\n" ' python plot.py compare --source "No Spec=nospec/results.csv" \\\n' - ' --source "Eagle3=eagle3/results.csv" --metric latency\n\n' + ' --source "Eagle3=eagle3/results.csv" --metric latency --title "Qwen3-8B"\n\n' + ' python plot.py compare --source "No Spec=nospec/results.csv" \\\n' + ' --source "Eagle3=eagle3/results.csv" --metric interactivity --num-gpus 1\n\n' ' python plot.py speedup --baseline "No Spec=nospec/results.csv" \\\n' - ' --target "Eagle3=eagle3/results.csv" --metric latency\n' + ' --target "Eagle3=eagle3/results.csv" --metric latency --title "Qwen3-8B"\n' ), ) sub = parser.add_subparsers(dest="command", title="commands") @@ -422,6 +476,23 @@ def main() -> None: default=None, help="Comma-separated subset filter (default: all found in data)", ) + cmp.add_argument( + "--title", + type=str, + default=None, + help="Optional title prefix for plots (e.g. model name)", + ) + cmp.add_argument( + "--num-gpus", + type=int, + default=None, + metavar="N", + help=( + "GPU count for metric 'interactivity' (tok/s/gpu = system_tps / N). " + "Required for interactivity; not stored in result CSVs/JSONs " + "(see vLLM logs: tensor_parallel_size / world_size)." + ), + ) cmp.set_defaults(func=run_compare) # --- speedup --- @@ -472,6 +543,17 @@ def main() -> None: default=None, help="Optional title prefix for plots (e.g. model name)", ) + spd.add_argument( + "--num-gpus", + type=int, + default=None, + metavar="N", + help=( + "GPU count for metric 'interactivity' (tok/s/gpu = system_tps / N). " + "Required for interactivity; not stored in result CSVs/JSONs " + "(see vLLM logs: tensor_parallel_size / world_size)." + ), + ) spd.set_defaults(func=run_speedup) args = parser.parse_args() From 0af154b8c2adbd227b0e4da1bd92a18fa5736271 Mon Sep 17 00:00:00 2001 From: Alexandre Marques Date: Tue, 21 Jul 2026 17:53:46 -0400 Subject: [PATCH 2/5] docs(evaluate): document interactivity plot metric Co-authored-by: Cursor --- .../tutorials/evaluating_performance.md | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/docs/user_guide/tutorials/evaluating_performance.md b/docs/user_guide/tutorials/evaluating_performance.md index 5f25e00e8..39b1fbbeb 100644 --- a/docs/user_guide/tutorials/evaluating_performance.md +++ b/docs/user_guide/tutorials/evaluating_performance.md @@ -106,13 +106,28 @@ Results are written to `acceptance.csv` in the output directory with per-categor python plot.py compare \ --source "No Spec=nospec/perf_results.csv" \ --source "DFlash=dflash/perf_results.csv" \ - --metric latency --output-dir ./plots + --metric latency --title "Qwen3-8B" --output-dir ./plots # Pairwise speedup (blue = faster, red = regression) python plot.py speedup \ --baseline "No Spec=nospec/perf_results.csv" \ --target "DFlash=dflash/perf_results.csv" \ --metric latency --title "Qwen3-8B" --output-dir ./plots + +# Interactivity: tok/s/user vs tok/s/GPU +python plot.py compare \ + --source "No Spec=nospec/perf_results.csv" \ + --source "DFlash=dflash/perf_results.csv" \ + --metric interactivity --num-gpus 1 --title "Qwen3-8B" --output-dir ./plots ``` -Both accept CSVs or raw GuideLLM sweep JSONs. Available metrics: `latency`, `itl`, `ttft`, `output_tps`. +Both accept CSVs or raw GuideLLM sweep JSONs. Available metrics: `latency`, `itl`, `ttft`, `output_tps`, `interactivity`. + +Most metrics plot the chosen y-value against requests per second (RPS). The `interactivity` metric instead plots: + +| Axis | Definition | +|------|------------| +| **x** — Interactivity (tok/s/user) | `1000 / median_ITL_ms` | +| **y** — Token throughput per GPU (tok/s/GPU) | `system_tps / num_gpus` | + +System throughput is aggregate output tokens per second (`total_output_tokens / duration`, falling back to GuideLLM's mean `output_tokens_per_second`), not the per-request median. `--num-gpus` is required because GPU count is not stored in result CSVs/JSONs (check vLLM logs for `tensor_parallel_size` / `world_size`). New sweeps write `output_tps_mean` into `perf_results.csv`; older CSVs without that column fall back to sibling `artifacts/run_*.json` files when present. From 5dc2d1832fee786a4a00284ca4eae3bec492a6b3 Mon Sep 17 00:00:00 2001 From: Alexandre Marques Date: Wed, 29 Jul 2026 17:44:23 -0400 Subject: [PATCH 3/5] Simplify interactivity metric: use direct output TPS, drop per-GPU normalization and guidellm 0.6 support - Use output_tokens_per_second mean directly from guidellm instead of estimating system TPS from total_output_tokens / duration - Remove --num-gpus flag and per-GPU throughput normalization; Y-axis is now total system output throughput (tok/s) - Drop legacy guidellm 0.6 JSON fallback path in _subset_from_sweep_json Co-authored-by: Cursor --- scripts/evaluate/perf_utils.py | 31 +++----------- scripts/evaluate/plot.py | 60 ++++----------------------- tests/unit/scripts/test_perf_utils.py | 10 +---- 3 files changed, 14 insertions(+), 87 deletions(-) diff --git a/scripts/evaluate/perf_utils.py b/scripts/evaluate/perf_utils.py index 9c144a02e..afb173f32 100644 --- a/scripts/evaluate/perf_utils.py +++ b/scripts/evaluate/perf_utils.py @@ -68,17 +68,14 @@ "xlabel": DEFAULT_XLABEL, "increasing": False, }, - # x = 1000 / ITL_ms (tok/s/user); y = system_tps / num_gpus. - # System TPS is total_output_tokens / duration (falls back to guidellm mean), - # not the per-request median. Requires --num-gpus. + # x = 1000 / ITL_ms (tok/s/user); y = system output TPS (mean). "interactivity": { "csv_col": "output_tps_mean", "json_key": "output_tokens_per_second", "stat": "mean", - "label": "Token throughput per GPU (tok/s/GPU)", + "label": "Output throughput (tok/s)", "xlabel": "Interactivity (tok/s/user)", "increasing": False, - "requires_num_gpus": True, }, } @@ -143,15 +140,11 @@ class Vector(Metric): def _subset_from_sweep_json(data: dict) -> str: - """Extract subset name from guidellm 0.7+ or legacy 0.6 sweep JSON.""" + """Extract subset name from a guidellm sweep JSON.""" try: return Path( data["config"]["spec"]["data"][0]["load_kwargs"]["data_files"] ).stem - except (KeyError, TypeError, IndexError): - pass - try: - return Path(data["args"]["data_args"][0]["data_files"]).stem except (KeyError, TypeError, IndexError): return "unknown" @@ -209,14 +202,6 @@ def _load_json( def _system_tps_from_bench(bench: dict) -> float | None: """Aggregate output tokens/s for a benchmark (not per-request median).""" metrics = bench.get("metrics", {}) - try: - out = metrics["output_token_count"]["successful"] - duration = float(bench["duration"]) - total = float(out["total_sum"]) - if duration > 0: - return total / duration - except (KeyError, TypeError, ValueError, ZeroDivisionError): - pass try: return float(metrics["output_tokens_per_second"]["successful"]["mean"]) except (KeyError, TypeError, ValueError): @@ -323,22 +308,18 @@ def load_data( def transform_interactivity( data: dict[str, list[tuple[float, float]]], - num_gpus: int, ) -> dict[str, list[tuple[float, float]]]: - """Transform ``(itl_ms, system_tps)`` into ``(tok/s/user, tok/s/gpu)``. + """Transform ``(itl_ms, system_tps)`` into ``(tok/s/user, system_tps)``. - Interactivity (x) = ``1000 / ITL_ms``; throughput/GPU (y) = ``system_tps / num_gpus``. + Interactivity (x) = ``1000 / ITL_ms``; throughput (y) = total system output TPS. """ - if num_gpus <= 0: - raise ValueError(f"num_gpus must be positive, got {num_gpus}") - result: dict[str, list[tuple[float, float]]] = {} for subset, points in data.items(): transformed: list[tuple[float, float]] = [] for itl_ms, system_tps in points: if itl_ms <= 0: continue - transformed.append((1000.0 / itl_ms, system_tps / num_gpus)) + transformed.append((1000.0 / itl_ms, system_tps)) if transformed: result[subset] = transformed return result diff --git a/scripts/evaluate/plot.py b/scripts/evaluate/plot.py index 32558e669..be7bbc4a2 100644 --- a/scripts/evaluate/plot.py +++ b/scripts/evaluate/plot.py @@ -14,7 +14,7 @@ python plot.py compare \\ --source "No Spec=nospec/results.csv" \\ --source "Eagle3=eagle3/results.csv" \\ - --metric interactivity --num-gpus 1 --title "Qwen3-8B" + --metric interactivity --title "Qwen3-8B" python plot.py speedup \\ --baseline "No Spec=nospec/results.csv" \\ @@ -102,35 +102,18 @@ def _plot_compare_subset( ax.plot(x_smooth, y_smooth, color=color, linewidth=2.5, label=label, zorder=4) -def _require_num_gpus(metrics: list[str], num_gpus: int | None) -> None: - needs_gpus = any(METRICS[m].get("requires_num_gpus") for m in metrics) - if needs_gpus and num_gpus is None: - print( - "[ERROR] --num-gpus is required for metric 'interactivity' " - "(GPU count is not stored in result CSVs/JSONs).", - file=sys.stderr, - ) - sys.exit(1) - if num_gpus is not None and num_gpus <= 0: - print(f"[ERROR] --num-gpus must be positive, got {num_gpus}", file=sys.stderr) - sys.exit(1) - - def _maybe_transform_interactivity( data: dict[str, list[tuple[float, float]]], metric_name: str, - num_gpus: int | None, ) -> dict[str, list[tuple[float, float]]]: - if not METRICS[metric_name].get("requires_num_gpus"): + if metric_name != "interactivity": return data - assert num_gpus is not None # validated by _require_num_gpus - return transform_interactivity(data, num_gpus) + return transform_interactivity(data) def run_compare(args: argparse.Namespace) -> None: metrics = args.metric or ["latency"] subset_filter = set(args.subsets.split(",")) if args.subsets else None - _require_num_gpus(metrics, args.num_gpus) try: sources = parse_source_args(args.source) @@ -145,9 +128,7 @@ def run_compare(args: argparse.Namespace) -> None: metric_cfg = METRICS[metric_name] all_data = _collect_all_data(sources, metric_name) all_data = { - label: _maybe_transform_interactivity( - subset_data, metric_name, args.num_gpus - ) + label: _maybe_transform_interactivity(subset_data, metric_name) for label, subset_data in all_data.items() } @@ -355,7 +336,6 @@ def _plot_speedup_subset( def run_speedup(args: argparse.Namespace) -> None: metrics = args.metric or ["latency"] subset_filter = set(args.subsets.split(",")) if args.subsets else None - _require_num_gpus(metrics, args.num_gpus) args.output_dir.mkdir(parents=True, exist_ok=True) for metric_name in metrics: @@ -369,12 +349,8 @@ def run_speedup(args: argparse.Namespace) -> None: print(f"[ERROR] {e}", file=sys.stderr) sys.exit(1) - baseline_data = _maybe_transform_interactivity( - baseline_data, metric_name, args.num_gpus - ) - target_data = _maybe_transform_interactivity( - target_data, metric_name, args.num_gpus - ) + baseline_data = _maybe_transform_interactivity(baseline_data, metric_name) + target_data = _maybe_transform_interactivity(target_data, metric_name) all_subsets = set(baseline_data.keys()) & set(target_data.keys()) if subset_filter: @@ -434,7 +410,7 @@ def main() -> None: ' python plot.py compare --source "No Spec=nospec/results.csv" \\\n' ' --source "Eagle3=eagle3/results.csv" --metric latency --title "Qwen3-8B"\n\n' ' python plot.py compare --source "No Spec=nospec/results.csv" \\\n' - ' --source "Eagle3=eagle3/results.csv" --metric interactivity --num-gpus 1\n\n' + ' --source "Eagle3=eagle3/results.csv" --metric interactivity\n\n' ' python plot.py speedup --baseline "No Spec=nospec/results.csv" \\\n' ' --target "Eagle3=eagle3/results.csv" --metric latency --title "Qwen3-8B"\n' ), @@ -482,17 +458,6 @@ def main() -> None: default=None, help="Optional title prefix for plots (e.g. model name)", ) - cmp.add_argument( - "--num-gpus", - type=int, - default=None, - metavar="N", - help=( - "GPU count for metric 'interactivity' (tok/s/gpu = system_tps / N). " - "Required for interactivity; not stored in result CSVs/JSONs " - "(see vLLM logs: tensor_parallel_size / world_size)." - ), - ) cmp.set_defaults(func=run_compare) # --- speedup --- @@ -543,17 +508,6 @@ def main() -> None: default=None, help="Optional title prefix for plots (e.g. model name)", ) - spd.add_argument( - "--num-gpus", - type=int, - default=None, - metavar="N", - help=( - "GPU count for metric 'interactivity' (tok/s/gpu = system_tps / N). " - "Required for interactivity; not stored in result CSVs/JSONs " - "(see vLLM logs: tensor_parallel_size / world_size)." - ), - ) spd.set_defaults(func=run_speedup) args = parser.parse_args() diff --git a/tests/unit/scripts/test_perf_utils.py b/tests/unit/scripts/test_perf_utils.py index 134375690..121717ec4 100644 --- a/tests/unit/scripts/test_perf_utils.py +++ b/tests/unit/scripts/test_perf_utils.py @@ -1,12 +1,4 @@ -"""Tests for scripts/evaluate/perf_utils.py. - -Covers the changed code paths from the guidellm 0.6→0.7 upgrade: - - parse_gen_kwargs (replaced build_backend_args) - - run_guidellm CLI command construction - - _load_json (new JSON output structure) - - parse_gen_len_file (new request stats structure) - - parse_sweep_file (unchanged, regression guard) -""" +"""Tests for scripts/evaluate/perf_utils.py.""" import importlib.util import json From b3176cf87d9521fbda1097e77ba47de3a2fcce56 Mon Sep 17 00:00:00 2001 From: Alexandre Marques Date: Wed, 29 Jul 2026 17:52:04 -0400 Subject: [PATCH 4/5] Restore test_perf_utils.py docstring Co-authored-by: Cursor --- tests/unit/scripts/test_perf_utils.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/tests/unit/scripts/test_perf_utils.py b/tests/unit/scripts/test_perf_utils.py index 121717ec4..134375690 100644 --- a/tests/unit/scripts/test_perf_utils.py +++ b/tests/unit/scripts/test_perf_utils.py @@ -1,4 +1,12 @@ -"""Tests for scripts/evaluate/perf_utils.py.""" +"""Tests for scripts/evaluate/perf_utils.py. + +Covers the changed code paths from the guidellm 0.6→0.7 upgrade: + - parse_gen_kwargs (replaced build_backend_args) + - run_guidellm CLI command construction + - _load_json (new JSON output structure) + - parse_gen_len_file (new request stats structure) + - parse_sweep_file (unchanged, regression guard) +""" import importlib.util import json From 2b0a4ba4e4fdb2819218d3158f8245b32a77ed60 Mon Sep 17 00:00:00 2001 From: Alexandre Marques Date: Wed, 29 Jul 2026 18:01:40 -0400 Subject: [PATCH 5/5] Fix E501 line-too-long lint errors in plot.py Co-authored-by: Cursor --- scripts/evaluate/plot.py | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/scripts/evaluate/plot.py b/scripts/evaluate/plot.py index be7bbc4a2..95ad84a9f 100644 --- a/scripts/evaluate/plot.py +++ b/scripts/evaluate/plot.py @@ -162,7 +162,8 @@ def run_compare(args: argparse.Namespace) -> None: if args.title: title_parts.append(args.title) title_parts.extend(pretty_subset(s) for s in sorted(all_subsets)) - ax.set_title(", ".join(title_parts) or str(metric_cfg["label"]), fontsize=14, fontweight="bold") + title = ", ".join(title_parts) or str(metric_cfg["label"]) + ax.set_title(title, fontsize=14, fontweight="bold") ax.set_xlabel(str(metric_cfg["xlabel"]), fontsize=12) ax.set_ylabel(str(metric_cfg["label"]), fontsize=12) ax.legend(framealpha=0.9) @@ -407,12 +408,18 @@ def main() -> None: formatter_class=argparse.RawDescriptionHelpFormatter, epilog=( "examples:\n" - ' python plot.py compare --source "No Spec=nospec/results.csv" \\\n' - ' --source "Eagle3=eagle3/results.csv" --metric latency --title "Qwen3-8B"\n\n' - ' python plot.py compare --source "No Spec=nospec/results.csv" \\\n' - ' --source "Eagle3=eagle3/results.csv" --metric interactivity\n\n' - ' python plot.py speedup --baseline "No Spec=nospec/results.csv" \\\n' - ' --target "Eagle3=eagle3/results.csv" --metric latency --title "Qwen3-8B"\n' + " python plot.py compare \\\n" + ' --source "No Spec=nospec/results.csv" \\\n' + ' --source "Eagle3=eagle3/results.csv" \\\n' + ' --metric latency --title "Qwen3-8B"\n\n' + " python plot.py compare \\\n" + ' --source "No Spec=nospec/results.csv" \\\n' + ' --source "Eagle3=eagle3/results.csv" \\\n' + " --metric interactivity\n\n" + " python plot.py speedup \\\n" + ' --baseline "No Spec=nospec/results.csv" \\\n' + ' --target "Eagle3=eagle3/results.csv" \\\n' + ' --metric latency --title "Qwen3-8B"\n' ), ) sub = parser.add_subparsers(dest="command", title="commands")