diff --git a/docs/user_guide/tutorials/evaluating_performance.md b/docs/user_guide/tutorials/evaluating_performance.md index 5f25e00e8..39b1fbbeb 100644 --- a/docs/user_guide/tutorials/evaluating_performance.md +++ b/docs/user_guide/tutorials/evaluating_performance.md @@ -106,13 +106,28 @@ Results are written to `acceptance.csv` in the output directory with per-categor python plot.py compare \ --source "No Spec=nospec/perf_results.csv" \ --source "DFlash=dflash/perf_results.csv" \ - --metric latency --output-dir ./plots + --metric latency --title "Qwen3-8B" --output-dir ./plots # Pairwise speedup (blue = faster, red = regression) python plot.py speedup \ --baseline "No Spec=nospec/perf_results.csv" \ --target "DFlash=dflash/perf_results.csv" \ --metric latency --title "Qwen3-8B" --output-dir ./plots + +# Interactivity: tok/s/user vs tok/s/GPU +python plot.py compare \ + --source "No Spec=nospec/perf_results.csv" \ + --source "DFlash=dflash/perf_results.csv" \ + --metric interactivity --num-gpus 1 --title "Qwen3-8B" --output-dir ./plots ``` -Both accept CSVs or raw GuideLLM sweep JSONs. Available metrics: `latency`, `itl`, `ttft`, `output_tps`. +Both accept CSVs or raw GuideLLM sweep JSONs. Available metrics: `latency`, `itl`, `ttft`, `output_tps`, `interactivity`. + +Most metrics plot the chosen y-value against requests per second (RPS). The `interactivity` metric instead plots: + +| Axis | Definition | +|------|------------| +| **x** — Interactivity (tok/s/user) | `1000 / median_ITL_ms` | +| **y** — Token throughput per GPU (tok/s/GPU) | `system_tps / num_gpus` | + +System throughput is aggregate output tokens per second (`total_output_tokens / duration`, falling back to GuideLLM's mean `output_tokens_per_second`), not the per-request median. `--num-gpus` is required because GPU count is not stored in result CSVs/JSONs (check vLLM logs for `tensor_parallel_size` / `world_size`). New sweeps write `output_tps_mean` into `perf_results.csv`; older CSVs without that column fall back to sibling `artifacts/run_*.json` files when present. diff --git a/scripts/evaluate/perf_utils.py b/scripts/evaluate/perf_utils.py index e237f7800..afb173f32 100644 --- a/scripts/evaluate/perf_utils.py +++ b/scripts/evaluate/perf_utils.py @@ -33,12 +33,15 @@ # Metric definitions (for plotting) # --------------------------------------------------------------------------- +DEFAULT_XLABEL = "Requests per second (RPS)" + METRICS: dict[str, dict[str, str | bool]] = { "latency": { "csv_col": "latency_median_s", "json_key": "request_latency", "stat": "median", "label": "Median Latency (s)", + "xlabel": DEFAULT_XLABEL, "increasing": True, }, "itl": { @@ -46,6 +49,7 @@ "json_key": "inter_token_latency_ms", "stat": "median", "label": "Median ITL (ms)", + "xlabel": DEFAULT_XLABEL, "increasing": True, }, "ttft": { @@ -53,6 +57,7 @@ "json_key": "time_to_first_token_ms", "stat": "median", "label": "Median TTFT (ms)", + "xlabel": DEFAULT_XLABEL, "increasing": True, }, "output_tps": { @@ -60,6 +65,16 @@ "json_key": "output_tokens_per_second", "stat": "median", "label": "Output Tokens/s", + "xlabel": DEFAULT_XLABEL, + "increasing": False, + }, + # x = 1000 / ITL_ms (tok/s/user); y = system output TPS (mean). + "interactivity": { + "csv_col": "output_tps_mean", + "json_key": "output_tokens_per_second", + "stat": "mean", + "label": "Output throughput (tok/s)", + "xlabel": "Interactivity (tok/s/user)", "increasing": False, }, } @@ -95,6 +110,7 @@ "itl_median_ms", "ttft_median_ms", "output_tps_median", + "output_tps_mean", "total_output_tokens", ] @@ -123,10 +139,23 @@ class Vector(Metric): # --------------------------------------------------------------------------- +def _subset_from_sweep_json(data: dict) -> str: + """Extract subset name from a guidellm sweep JSON.""" + try: + return Path( + data["config"]["spec"]["data"][0]["load_kwargs"]["data_files"] + ).stem + except (KeyError, TypeError, IndexError): + return "unknown" + + def _load_csv( filepath: Path, metric_name: str, ) -> dict[str, list[tuple[float, float]]]: + if metric_name == "interactivity": + return _load_interactivity_csv(filepath) + cfg = METRICS[metric_name] result: dict[str, list[tuple[float, float]]] = defaultdict(list) with filepath.open(newline="") as f: @@ -147,11 +176,14 @@ def _load_json( filepath: Path, metric_name: str, ) -> dict[str, list[tuple[float, float]]]: + if metric_name == "interactivity": + return _load_interactivity_json(filepath) + cfg = METRICS[metric_name] with filepath.open() as f: data = json.load(f) - subset = Path(data["config"]["spec"]["data"][0]["load_kwargs"]["data_files"]).stem + subset = _subset_from_sweep_json(data) points: list[tuple[float, float]] = [] for bench in data.get("benchmarks", []): if bench.get("config", {}).get("strategy", {}).get("type_") != "constant": @@ -167,6 +199,100 @@ def _load_json( return {subset: points} if points else {} +def _system_tps_from_bench(bench: dict) -> float | None: + """Aggregate output tokens/s for a benchmark (not per-request median).""" + metrics = bench.get("metrics", {}) + try: + return float(metrics["output_tokens_per_second"]["successful"]["mean"]) + except (KeyError, TypeError, ValueError): + return None + + +def _interactivity_point_from_bench(bench: dict) -> tuple[float, float] | None: + """Return ``(itl_ms, system_tps)`` for a constant-rate benchmark.""" + if bench.get("config", {}).get("strategy", {}).get("type_") != "constant": + return None + try: + itl_ms = float( + bench["metrics"]["inter_token_latency_ms"]["successful"]["median"] + ) + except (KeyError, TypeError, ValueError): + return None + system_tps = _system_tps_from_bench(bench) + if system_tps is None or itl_ms <= 0: + return None + return (itl_ms, system_tps) + + +def _load_interactivity_json( + filepath: Path, +) -> dict[str, list[tuple[float, float]]]: + with filepath.open() as f: + data = json.load(f) + + subset = _subset_from_sweep_json(data) + points: list[tuple[float, float]] = [] + for bench in data.get("benchmarks", []): + pt = _interactivity_point_from_bench(bench) + if pt is not None: + points.append(pt) + points.sort(key=lambda p: p[0]) + return {subset: points} if points else {} + + +def _load_interactivity_from_artifact_jsons( + csv_path: Path, +) -> dict[str, list[tuple[float, float]]]: + """Fall back to sibling ``artifacts/run_*.json`` when CSV lacks system TPS.""" + artifacts = csv_path.parent / "artifacts" + if not artifacts.is_dir(): + return {} + combined: dict[str, list[tuple[float, float]]] = defaultdict(list) + for json_path in sorted(artifacts.glob("run_*.json")): + for subset, points in _load_interactivity_json(json_path).items(): + combined[subset].extend(points) + return dict(combined) + + +def _load_interactivity_csv( + filepath: Path, +) -> dict[str, list[tuple[float, float]]]: + """Load ``(itl_ms, system_tps)`` from CSV, or from sibling JSONs if needed.""" + result: dict[str, list[tuple[float, float]]] = defaultdict(list) + has_mean = False + with filepath.open(newline="") as f: + reader = csv.DictReader(f) + fieldnames = reader.fieldnames or [] + has_mean = "output_tps_mean" in fieldnames + if has_mean: + for row in reader: + if row.get("strategy") != "constant": + continue + try: + subset = re.sub(r"^run_", "", row.get("subset", "unknown")) + itl_ms = float(row["itl_median_ms"]) + system_tps = float(row["output_tps_mean"]) + if itl_ms > 0: + result[subset].append((itl_ms, system_tps)) + except (ValueError, KeyError): + continue + + if result: + return dict(result) + + fallback = _load_interactivity_from_artifact_jsons(filepath) + if fallback: + return fallback + + hint = ( + "CSV is missing output_tps_mean and no artifacts/run_*.json found. " + "Re-run evaluation or point --source at the sweep JSON." + if not has_mean + else "No constant-rate interactivity rows found." + ) + raise ValueError(f"Cannot load interactivity from {filepath}: {hint}") + + def load_data( filepath: Path, metric_name: str, @@ -180,6 +306,25 @@ def load_data( ) +def transform_interactivity( + data: dict[str, list[tuple[float, float]]], +) -> dict[str, list[tuple[float, float]]]: + """Transform ``(itl_ms, system_tps)`` into ``(tok/s/user, system_tps)``. + + Interactivity (x) = ``1000 / ITL_ms``; throughput (y) = total system output TPS. + """ + result: dict[str, list[tuple[float, float]]] = {} + for subset, points in data.items(): + transformed: list[tuple[float, float]] = [] + for itl_ms, system_tps in points: + if itl_ms <= 0: + continue + transformed.append((1000.0 / itl_ms, system_tps)) + if transformed: + result[subset] = transformed + return result + + def parse_source_args(source_args: list[str]) -> dict[str, list[Path]]: """Parse ``LABEL=PATH`` strings into ``{label: [path, ...]}``.""" result: dict[str, list[Path]] = defaultdict(list) @@ -386,6 +531,11 @@ def parse_sweep_file(filepath: Path) -> list[dict]: for metric_key, csv_key in METRICS_TO_EXTRACT: val = metrics.get(metric_key, {}) row[csv_key] = val.get("successful", {}).get("median", "") + row["output_tps_mean"] = ( + metrics.get("output_tokens_per_second", {}) + .get("successful", {}) + .get("mean", "") + ) out_tok = metrics.get("output_tokens", {}) row["total_output_tokens"] = out_tok.get("successful", {}).get("sum", "") rows.append(row) diff --git a/scripts/evaluate/plot.py b/scripts/evaluate/plot.py index 4ff918260..95ad84a9f 100644 --- a/scripts/evaluate/plot.py +++ b/scripts/evaluate/plot.py @@ -9,7 +9,12 @@ python plot.py compare \\ --source "No Spec=nospec/results.csv" \\ --source "Eagle3=eagle3/results.csv" \\ - --metric latency --metric itl + --metric latency --metric itl --title "Qwen3-8B" + + python plot.py compare \\ + --source "No Spec=nospec/results.csv" \\ + --source "Eagle3=eagle3/results.csv" \\ + --metric interactivity --title "Qwen3-8B" python plot.py speedup \\ --baseline "No Spec=nospec/results.csv" \\ @@ -34,6 +39,7 @@ parse_source_args, pretty_subset, smooth_curve, + transform_interactivity, ) COLOR_CYCLE = [ @@ -96,6 +102,15 @@ def _plot_compare_subset( ax.plot(x_smooth, y_smooth, color=color, linewidth=2.5, label=label, zorder=4) +def _maybe_transform_interactivity( + data: dict[str, list[tuple[float, float]]], + metric_name: str, +) -> dict[str, list[tuple[float, float]]]: + if metric_name != "interactivity": + return data + return transform_interactivity(data) + + def run_compare(args: argparse.Namespace) -> None: metrics = args.metric or ["latency"] subset_filter = set(args.subsets.split(",")) if args.subsets else None @@ -112,6 +127,10 @@ def run_compare(args: argparse.Namespace) -> None: for metric_name in metrics: metric_cfg = METRICS[metric_name] all_data = _collect_all_data(sources, metric_name) + all_data = { + label: _maybe_transform_interactivity(subset_data, metric_name) + for label, subset_data in all_data.items() + } all_subsets: set[str] = set() for label_data in all_data.values(): @@ -139,9 +158,14 @@ def run_compare(args: argparse.Namespace) -> None: source_labels, ) - ax.set_title(metric_cfg["label"], fontsize=14, fontweight="bold") - ax.set_xlabel("Requests per Second", fontsize=12) - ax.set_ylabel(metric_cfg["label"], fontsize=12) + title_parts = [] + if args.title: + title_parts.append(args.title) + title_parts.extend(pretty_subset(s) for s in sorted(all_subsets)) + title = ", ".join(title_parts) or str(metric_cfg["label"]) + ax.set_title(title, fontsize=14, fontweight="bold") + ax.set_xlabel(str(metric_cfg["xlabel"]), fontsize=12) + ax.set_ylabel(str(metric_cfg["label"]), fontsize=12) ax.legend(framealpha=0.9) ax.grid(True, alpha=0.3) fig.tight_layout() @@ -302,8 +326,8 @@ def _plot_speedup_subset( title_parts.append(title_prefix) title_parts.append(pretty_subset(subset)) ax.set_title(", ".join(title_parts), fontsize=14, fontweight="bold") - ax.set_xlabel("Requests per second (RPS)", fontsize=12) - ax.set_ylabel(metric_cfg["label"], fontsize=12) + ax.set_xlabel(str(metric_cfg["xlabel"]), fontsize=12) + ax.set_ylabel(str(metric_cfg["label"]), fontsize=12) ax.legend(framealpha=0.9) ax.grid(True, alpha=0.3) fig.tight_layout() @@ -317,7 +341,7 @@ def run_speedup(args: argparse.Namespace) -> None: for metric_name in metrics: metric_cfg = METRICS[metric_name] - increasing = metric_cfg["increasing"] + increasing = bool(metric_cfg["increasing"]) try: baseline_label, baseline_data = _collect_points(args.baseline, metric_name) @@ -326,6 +350,9 @@ def run_speedup(args: argparse.Namespace) -> None: print(f"[ERROR] {e}", file=sys.stderr) sys.exit(1) + baseline_data = _maybe_transform_interactivity(baseline_data, metric_name) + target_data = _maybe_transform_interactivity(target_data, metric_name) + all_subsets = set(baseline_data.keys()) & set(target_data.keys()) if subset_filter: all_subsets &= subset_filter @@ -354,8 +381,10 @@ def run_speedup(args: argparse.Namespace) -> None: title_prefix=args.title, ) if not ok: + x_name = metric_cfg["xlabel"] print( - f"[WARN] No overlapping RPS range for subset '{subset}', skipping", + f"[WARN] No overlapping {x_name} range for subset '{subset}', " + "skipping", file=sys.stderr, ) plt.close(fig) @@ -379,10 +408,18 @@ def main() -> None: formatter_class=argparse.RawDescriptionHelpFormatter, epilog=( "examples:\n" - ' python plot.py compare --source "No Spec=nospec/results.csv" \\\n' - ' --source "Eagle3=eagle3/results.csv" --metric latency\n\n' - ' python plot.py speedup --baseline "No Spec=nospec/results.csv" \\\n' - ' --target "Eagle3=eagle3/results.csv" --metric latency\n' + " python plot.py compare \\\n" + ' --source "No Spec=nospec/results.csv" \\\n' + ' --source "Eagle3=eagle3/results.csv" \\\n' + ' --metric latency --title "Qwen3-8B"\n\n' + " python plot.py compare \\\n" + ' --source "No Spec=nospec/results.csv" \\\n' + ' --source "Eagle3=eagle3/results.csv" \\\n' + " --metric interactivity\n\n" + " python plot.py speedup \\\n" + ' --baseline "No Spec=nospec/results.csv" \\\n' + ' --target "Eagle3=eagle3/results.csv" \\\n' + ' --metric latency --title "Qwen3-8B"\n' ), ) sub = parser.add_subparsers(dest="command", title="commands") @@ -422,6 +459,12 @@ def main() -> None: default=None, help="Comma-separated subset filter (default: all found in data)", ) + cmp.add_argument( + "--title", + type=str, + default=None, + help="Optional title prefix for plots (e.g. model name)", + ) cmp.set_defaults(func=run_compare) # --- speedup ---