From cd373f445e14465dde1f96a36fdd50a37fe7341e Mon Sep 17 00:00:00 2001 From: TanmayeeSharvani22 Date: Thu, 23 Jul 2026 11:38:15 +0530 Subject: [PATCH 1/2] Improve order-accuracy benchmark: stats, per-image breakdown, run metadata - Fix _collect_metrics calling nonexistent stream_density.calculate_total_fps, which raised AttributeError on every run for the image-based dine-in workflow (no GStreamer pipeline logs exist for it). Skipped via hasattr() check instead of emitting a misleading warning each time. - Add percentile/variance stats (mean, stddev, p50, p95, min, max) for latency and accuracy across all iterations, since a single mean hid significant run-to-run variance. - Add per-image breakdown (order_id -> avg accuracy/latency) so multi-image iteration runs don't collapse into one aggregate number. - Capture and stamp run_config (model name, target device, image max_size) into every results file by reading dine-in/.env and vlm_client.py, so results are self-describing without needing external tracking. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- benchmark-scripts/benchmark_order_accuracy.py | 146 +++++++++++++++--- 1 file changed, 126 insertions(+), 20 deletions(-) diff --git a/benchmark-scripts/benchmark_order_accuracy.py b/benchmark-scripts/benchmark_order_accuracy.py index e8d5e2e..9126628 100644 --- a/benchmark-scripts/benchmark_order_accuracy.py +++ b/benchmark-scripts/benchmark_order_accuracy.py @@ -74,7 +74,44 @@ def __init__( # YOLO_MODEL_PATH is set by the Makefile (YOLO_MODEL_PATH ?= ...) based on TARGET_DEVICE # and exported via the bare 'export' directive, so it arrives in os.environ already. # No fallback needed here — keep model selection as a single source of truth in the Makefile. - + + def _capture_run_metadata(self) -> Dict: + """ + Snapshot the active model/precision/resolution configuration so every + results file is self-describing (which model this run actually measured). + Reads dine-in/.env for OVMS_MODEL_NAME and the image max_size baked into + vlm_client.py, without requiring the app to expose a new endpoint. + """ + metadata = { + "model_name": None, + "target_device": self.target_device, + "image_max_size": None, + } + try: + compose_dir = os.path.dirname(self.compose_files[0]) if self.compose_files else "." + env_path = os.path.join(compose_dir, ".env") + if os.path.exists(env_path): + with open(env_path, "r") as f: + for line in f: + line = line.strip() + if line.startswith("OVMS_MODEL_NAME="): + metadata["model_name"] = line.split("=", 1)[1].strip() + + vlm_client_path = os.path.join(compose_dir, "src", "services", "vlm_client.py") + if os.path.exists(vlm_client_path): + with open(vlm_client_path, "r") as f: + for line in f: + if "max_size=" in line and line.strip().startswith("max_size"): + try: + metadata["image_max_size"] = int( + line.split("max_size=")[1].split(",")[0].strip() + ) + except (ValueError, IndexError): + pass + except Exception as e: + print(f"Warning: Could not capture run metadata: {e}") + return metadata + def docker_compose_cmd( self, action: str, @@ -173,6 +210,10 @@ def run_fixed_workers( # Collect metrics results = self._collect_metrics(workers) + + # Stamp this run with the model/precision/resolution that was actually + # under test, so results files are self-describing for later comparison. + results["run_config"] = self._capture_run_metadata() # Collect VLM metrics from vlm_metrics_logger results["vlm_metrics"] = self._collect_vlm_logger_metrics() @@ -339,8 +380,59 @@ def _collect_worker_results(self) -> Dict: pass if count > 0: worker_results["avg_latency_ms"] = total_latency / count - + + # Per-call latency/accuracy distribution across all iterations & images, + # since a single mean hides run-to-run variance and per-image differences. + latencies = [ + d["total_latency_ms"] for d in worker_results["details"] + if d.get("success") and d.get("total_latency_ms") is not None + ] + accuracies = [ + d["accuracy_score"] for d in worker_results["details"] + if d.get("accuracy_score") is not None + ] + worker_results["latency_stats"] = self._percentile_stats(latencies) + worker_results["accuracy_stats"] = self._percentile_stats(accuracies) + + # Per-image breakdown (order_id -> aggregated accuracy/latency), so multi-image + # runs (e.g. --iterations 8 cycling MCD-1001..1004) don't collapse into one number. + per_image: Dict[str, List[Dict]] = {} + for d in worker_results["details"]: + oid = d.get("order_id", "unknown") + per_image.setdefault(oid, []).append(d) + worker_results["per_image"] = { + oid: { + "count": len(items), + "avg_accuracy": sum(i.get("accuracy_score") or 0 for i in items) / len(items), + "avg_latency_ms": sum(i.get("total_latency_ms") or 0 for i in items) / len(items), + } + for oid, items in per_image.items() + } + return worker_results + + @staticmethod + def _percentile_stats(values: List[float]) -> Dict: + """Compute mean/stddev/p50/p95 for a list of numeric samples.""" + if not values: + return {"count": 0, "mean": 0.0, "stddev": 0.0, "p50": 0.0, "p95": 0.0, "min": 0.0, "max": 0.0} + n = len(values) + mean = sum(values) / n + variance = sum((v - mean) ** 2 for v in values) / n + stddev = variance ** 0.5 + ordered = sorted(values) + def _pct(p): + idx = min(n - 1, max(0, round(p / 100.0 * (n - 1)))) + return ordered[idx] + return { + "count": n, + "mean": round(mean, 2), + "stddev": round(stddev, 2), + "p50": round(_pct(50), 2), + "p95": round(_pct(95), 2), + "min": round(ordered[0], 2), + "max": round(ordered[-1], 2), + } def _clean_pipeline_logs(self): """Remove previous pipeline log and results files to prevent stale data.""" @@ -393,21 +485,27 @@ def _collect_metrics(self, num_pipelines: int) -> Dict: "vlm_inference": {} } - # Calculate FPS from logs - try: - with contextlib.redirect_stdout(io.StringIO()): - total_fps, fps_per_stream, fps_dict = stream_density.calculate_total_fps( - num_pipelines, - self.results_dir, - "order-accuracy" - ) - metrics["fps"] = { - "total": total_fps, - "per_stream": fps_per_stream, - "per_pipeline": fps_dict - } - except Exception as e: - print(f"Warning: Could not calculate FPS: {e}") + # Calculate FPS from logs. NOTE: stream_density.calculate_total_fps does not + # exist (GStreamer-pipeline-only API) -- it always raised AttributeError here + # for the image-based dine-in workflow, which has no GStreamer pipeline logs. + # A working FPS derivation for this workflow already runs later in + # run_fixed_workers() from worker_results/vlm_metrics, so this dead branch is + # skipped entirely instead of emitting a misleading warning on every run. + if hasattr(stream_density, "calculate_total_fps"): + try: + with contextlib.redirect_stdout(io.StringIO()): + total_fps, fps_per_stream, fps_dict = stream_density.calculate_total_fps( + num_pipelines, + self.results_dir, + "order-accuracy" + ) + metrics["fps"] = { + "total": total_fps, + "per_stream": fps_per_stream, + "per_pipeline": fps_dict + } + except Exception as e: + print(f"Warning: Could not calculate FPS: {e}") # Calculate latency from logs try: @@ -575,11 +673,19 @@ def _export_results(self, results: Dict, prefix: str): prefix: Filename prefix """ timestamp = time.strftime("%Y%m%d_%H%M%S") - + + # Tag filenames with the model under test so results files are + # identifiable at a glance without opening them (e.g. which model/precision + # a given run measured), instead of a bare timestamp. + model_tag = "" + model_name = results.get("run_config", {}).get("model_name") + if model_name: + model_tag = "_" + model_name.replace("/", "-").replace(" ", "-") + # Export JSON json_path = os.path.join( self.results_dir, - f"{prefix}_results_{timestamp}.json" + f"{prefix}{model_tag}_results_{timestamp}.json" ) with open(json_path, 'w') as f: json.dump(results, f, indent=2) @@ -588,7 +694,7 @@ def _export_results(self, results: Dict, prefix: str): # Export CSV summary csv_path = os.path.join( self.results_dir, - f"{prefix}_summary_{timestamp}.csv" + f"{prefix}{model_tag}_summary_{timestamp}.csv" ) self._write_csv_summary(results, csv_path) print(f"Summary exported to: {csv_path}") From 9081a2b10bfc73b596e05524f93bd5e0236a8817 Mon Sep 17 00:00:00 2001 From: TanmayeeSharvani22 Date: Tue, 28 Jul 2026 11:45:26 +0530 Subject: [PATCH 2/2] Address PR review: accurate metadata docstring, skip nulls in per-image averages - Rewrite _capture_run_metadata docstring to match the fields actually exported (model_name, target_device, image_max_size). - Average per-image accuracy/latency only over non-null samples so failed iterations no longer skew results toward 0. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- benchmark-scripts/benchmark_order_accuracy.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/benchmark-scripts/benchmark_order_accuracy.py b/benchmark-scripts/benchmark_order_accuracy.py index 9126628..cba2502 100644 --- a/benchmark-scripts/benchmark_order_accuracy.py +++ b/benchmark-scripts/benchmark_order_accuracy.py @@ -77,10 +77,10 @@ def __init__( def _capture_run_metadata(self) -> Dict: """ - Snapshot the active model/precision/resolution configuration so every - results file is self-describing (which model this run actually measured). - Reads dine-in/.env for OVMS_MODEL_NAME and the image max_size baked into - vlm_client.py, without requiring the app to expose a new endpoint. + Snapshot the active model configuration so every results file is + self-describing. Captures OVMS_MODEL_NAME (from a compose-adjacent .env + when present), the target device, and the VLM image max_size (parsed from + src/services/vlm_client.py when present). """ metadata = { "model_name": None, @@ -403,14 +403,20 @@ def _collect_worker_results(self) -> Dict: worker_results["per_image"] = { oid: { "count": len(items), - "avg_accuracy": sum(i.get("accuracy_score") or 0 for i in items) / len(items), - "avg_latency_ms": sum(i.get("total_latency_ms") or 0 for i in items) / len(items), + "avg_accuracy": self._mean_of_present(items, "accuracy_score"), + "avg_latency_ms": self._mean_of_present(items, "total_latency_ms"), } for oid, items in per_image.items() } return worker_results + @staticmethod + def _mean_of_present(items: List[Dict], key: str) -> float: + """Average `key` over items where it is present and non-null; 0.0 if none.""" + values = [i[key] for i in items if i.get(key) is not None] + return sum(values) / len(values) if values else 0.0 + @staticmethod def _percentile_stats(values: List[float]) -> Dict: """Compute mean/stddev/p50/p95 for a list of numeric samples."""