Add interactivity plot metric (tok/s/user vs tok/s/GPU)

docs/user_guide/tutorials/evaluating_performance.md

@@ -106,13 +106,28 @@ Results are written to `acceptance.csv` in the output directory with per-categor
 python plot.py compare \
     --source "No Spec=nospec/perf_results.csv" \
     --source "DFlash=dflash/perf_results.csv" \
-    --metric latency --output-dir ./plots
+    --metric latency --title "Qwen3-8B" --output-dir ./plots
 
 # Pairwise speedup (blue = faster, red = regression)
 python plot.py speedup \
     --baseline "No Spec=nospec/perf_results.csv" \
     --target "DFlash=dflash/perf_results.csv" \
     --metric latency --title "Qwen3-8B" --output-dir ./plots
+
+# Interactivity: tok/s/user vs tok/s/GPU
+python plot.py compare \
+    --source "No Spec=nospec/perf_results.csv" \
+    --source "DFlash=dflash/perf_results.csv" \
+    --metric interactivity --num-gpus 1 --title "Qwen3-8B" --output-dir ./plots
 ```
 
-Both accept CSVs or raw GuideLLM sweep JSONs. Available metrics: `latency`, `itl`, `ttft`, `output_tps`.
+Both accept CSVs or raw GuideLLM sweep JSONs. Available metrics: `latency`, `itl`, `ttft`, `output_tps`, `interactivity`.
+
+Most metrics plot the chosen y-value against requests per second (RPS). The `interactivity` metric instead plots:
+
+| Axis | Definition |
+|------|------------|
+| **x** — Interactivity (tok/s/user) | `1000 / median_ITL_ms` |
+| **y** — Token throughput per GPU (tok/s/GPU) | `system_tps / num_gpus` |
+
+System throughput is aggregate output tokens per second (`total_output_tokens / duration`, falling back to GuideLLM's mean `output_tokens_per_second`), not the per-request median. `--num-gpus` is required because GPU count is not stored in result CSVs/JSONs (check vLLM logs for `tensor_parallel_size` / `world_size`). New sweeps write `output_tps_mean` into `perf_results.csv`; older CSVs without that column fall back to sibling `artifacts/run_*.json` files when present.

scripts/evaluate/perf_utils.py

@@ -33,33 +33,48 @@
 # Metric definitions (for plotting)
 # ---------------------------------------------------------------------------
 
+DEFAULT_XLABEL = "Requests per second (RPS)"
+
 METRICS: dict[str, dict[str, str | bool]] = {
     "latency": {
         "csv_col": "latency_median_s",
         "json_key": "request_latency",
         "stat": "median",
         "label": "Median Latency (s)",
+        "xlabel": DEFAULT_XLABEL,
         "increasing": True,
     },
     "itl": {
         "csv_col": "itl_median_ms",
         "json_key": "inter_token_latency_ms",
         "stat": "median",
         "label": "Median ITL (ms)",
+        "xlabel": DEFAULT_XLABEL,
         "increasing": True,
     },
     "ttft": {
         "csv_col": "ttft_median_ms",
         "json_key": "time_to_first_token_ms",
         "stat": "median",
         "label": "Median TTFT (ms)",
+        "xlabel": DEFAULT_XLABEL,
         "increasing": True,
     },
     "output_tps": {
         "csv_col": "output_tps_median",
         "json_key": "output_tokens_per_second",
         "stat": "median",
         "label": "Output Tokens/s",
+        "xlabel": DEFAULT_XLABEL,
+        "increasing": False,
+    },
+    # x = 1000 / ITL_ms (tok/s/user); y = system output TPS (mean).
+    "interactivity": {
+        "csv_col": "output_tps_mean",
+        "json_key": "output_tokens_per_second",
+        "stat": "mean",
+        "label": "Output throughput (tok/s)",
+        "xlabel": "Interactivity (tok/s/user)",
         "increasing": False,
     },
 }
@@ -95,6 +110,7 @@
     "itl_median_ms",
     "ttft_median_ms",
     "output_tps_median",
+    "output_tps_mean",
     "total_output_tokens",
 ]
 
@@ -123,10 +139,23 @@ class Vector(Metric):
 # ---------------------------------------------------------------------------
 
 
+def _subset_from_sweep_json(data: dict) -> str:
+    """Extract subset name from a guidellm sweep JSON."""
+    try:
+        return Path(
+            data["config"]["spec"]["data"][0]["load_kwargs"]["data_files"]
+        ).stem
+    except (KeyError, TypeError, IndexError):
+        return "unknown"
+
+
 def _load_csv(
     filepath: Path,
     metric_name: str,
 ) -> dict[str, list[tuple[float, float]]]:
+    if metric_name == "interactivity":
+        return _load_interactivity_csv(filepath)
+
     cfg = METRICS[metric_name]
     result: dict[str, list[tuple[float, float]]] = defaultdict(list)
     with filepath.open(newline="") as f:
@@ -147,11 +176,14 @@ def _load_json(
     filepath: Path,
     metric_name: str,
 ) -> dict[str, list[tuple[float, float]]]:
+    if metric_name == "interactivity":
+        return _load_interactivity_json(filepath)
+
     cfg = METRICS[metric_name]
     with filepath.open() as f:
         data = json.load(f)
 
-    subset = Path(data["config"]["spec"]["data"][0]["load_kwargs"]["data_files"]).stem
+    subset = _subset_from_sweep_json(data)
     points: list[tuple[float, float]] = []
     for bench in data.get("benchmarks", []):
         if bench.get("config", {}).get("strategy", {}).get("type_") != "constant":
@@ -167,6 +199,100 @@ def _load_json(
     return {subset: points} if points else {}
 
 
+def _system_tps_from_bench(bench: dict) -> float | None:
+    """Aggregate output tokens/s for a benchmark (not per-request median)."""
+    metrics = bench.get("metrics", {})
+    try:
+        return float(metrics["output_tokens_per_second"]["successful"]["mean"])
+    except (KeyError, TypeError, ValueError):
+        return None
+
+
+def _interactivity_point_from_bench(bench: dict) -> tuple[float, float] | None:
+    """Return ``(itl_ms, system_tps)`` for a constant-rate benchmark."""
+    if bench.get("config", {}).get("strategy", {}).get("type_") != "constant":
+        return None
+    try:
+        itl_ms = float(
+            bench["metrics"]["inter_token_latency_ms"]["successful"]["median"]
+        )
+    except (KeyError, TypeError, ValueError):
+        return None
+    system_tps = _system_tps_from_bench(bench)
+    if system_tps is None or itl_ms <= 0:
+        return None
+    return (itl_ms, system_tps)
+
+
+def _load_interactivity_json(
+    filepath: Path,
+) -> dict[str, list[tuple[float, float]]]:
+    with filepath.open() as f:
+        data = json.load(f)
+
+    subset = _subset_from_sweep_json(data)
+    points: list[tuple[float, float]] = []
+    for bench in data.get("benchmarks", []):
+        pt = _interactivity_point_from_bench(bench)
+        if pt is not None:
+            points.append(pt)
+    points.sort(key=lambda p: p[0])
+    return {subset: points} if points else {}
+
+
+def _load_interactivity_from_artifact_jsons(
+    csv_path: Path,
+) -> dict[str, list[tuple[float, float]]]:
+    """Fall back to sibling ``artifacts/run_*.json`` when CSV lacks system TPS."""
+    artifacts = csv_path.parent / "artifacts"
+    if not artifacts.is_dir():
+        return {}
+    combined: dict[str, list[tuple[float, float]]] = defaultdict(list)
+    for json_path in sorted(artifacts.glob("run_*.json")):
+        for subset, points in _load_interactivity_json(json_path).items():
+            combined[subset].extend(points)
+    return dict(combined)
+
+
+def _load_interactivity_csv(
+    filepath: Path,
+) -> dict[str, list[tuple[float, float]]]:
+    """Load ``(itl_ms, system_tps)`` from CSV, or from sibling JSONs if needed."""
+    result: dict[str, list[tuple[float, float]]] = defaultdict(list)
+    has_mean = False
+    with filepath.open(newline="") as f:
+        reader = csv.DictReader(f)
+        fieldnames = reader.fieldnames or []
+        has_mean = "output_tps_mean" in fieldnames
+        if has_mean:
+            for row in reader:
+                if row.get("strategy") != "constant":
+                    continue
+                try:
+                    subset = re.sub(r"^run_", "", row.get("subset", "unknown"))
+                    itl_ms = float(row["itl_median_ms"])
+                    system_tps = float(row["output_tps_mean"])
+                    if itl_ms > 0:
+                        result[subset].append((itl_ms, system_tps))
+                except (ValueError, KeyError):
+                    continue
+
+    if result:
+        return dict(result)
+
+    fallback = _load_interactivity_from_artifact_jsons(filepath)
+    if fallback:
+        return fallback
+
+    hint = (
+        "CSV is missing output_tps_mean and no artifacts/run_*.json found. "
+        "Re-run evaluation or point --source at the sweep JSON."
+        if not has_mean
+        else "No constant-rate interactivity rows found."
+    )
+    raise ValueError(f"Cannot load interactivity from {filepath}: {hint}")
+
+
 def load_data(
     filepath: Path,
     metric_name: str,
@@ -180,6 +306,25 @@ def load_data(
     )
 
 
+def transform_interactivity(
+    data: dict[str, list[tuple[float, float]]],
+) -> dict[str, list[tuple[float, float]]]:
+    """Transform ``(itl_ms, system_tps)`` into ``(tok/s/user, system_tps)``.
+
+    Interactivity (x) = ``1000 / ITL_ms``; throughput (y) = total system output TPS.
+    """
+    result: dict[str, list[tuple[float, float]]] = {}
+    for subset, points in data.items():
+        transformed: list[tuple[float, float]] = []
+        for itl_ms, system_tps in points:
+            if itl_ms <= 0:
+                continue
+            transformed.append((1000.0 / itl_ms, system_tps))
+        if transformed:
+            result[subset] = transformed
+    return result
+
+
 def parse_source_args(source_args: list[str]) -> dict[str, list[Path]]:
     """Parse ``LABEL=PATH`` strings into ``{label: [path, ...]}``."""
     result: dict[str, list[Path]] = defaultdict(list)
@@ -386,6 +531,11 @@ def parse_sweep_file(filepath: Path) -> list[dict]:
         for metric_key, csv_key in METRICS_TO_EXTRACT:
             val = metrics.get(metric_key, {})
             row[csv_key] = val.get("successful", {}).get("median", "")
+        row["output_tps_mean"] = (
+            metrics.get("output_tokens_per_second", {})
+            .get("successful", {})
+            .get("mean", "")
+        )
         out_tok = metrics.get("output_tokens", {})
         row["total_output_tokens"] = out_tok.get("successful", {}).get("sum", "")
         rows.append(row)