From 6012dce1d69dc34028905e7410c3ae57f0ffd171 Mon Sep 17 00:00:00 2001
From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com>
Date: Thu, 30 Apr 2026 16:11:43 +0000
Subject: [PATCH 1/2] Initial plan


From ed83ab2e7b06f05686d29010e16f38364656eba2 Mon Sep 17 00:00:00 2001
From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com>
Date: Thu, 30 Apr 2026 16:18:55 +0000
Subject: [PATCH 2/2] perf-comparison: validate benchmark scripts before
 counting them

Agent-Logs-Url: https://github.com/githubnext/tsessebe/sessions/2ed75875-aa1e-4694-906a-6fd7dd84e8eb

Co-authored-by: mrjf <180956+mrjf@users.noreply.github.com>
---
 .autoloop/programs/perf-comparison/program.md | 44 ++++++++++++++++---
 1 file changed, 39 insertions(+), 5 deletions(-)

diff --git a/.autoloop/programs/perf-comparison/program.md b/.autoloop/programs/perf-comparison/program.md
index 8b15b72a6..fe4eb378d 100644
--- a/.autoloop/programs/perf-comparison/program.md
+++ b/.autoloop/programs/perf-comparison/program.md
@@ -50,18 +50,50 @@ Do NOT modify:
 
 ## Evaluation
 
+The evaluation block runs validity checks **before** counting benchmark pairs.
+If any benchmark script is syntactically invalid (or required tooling is
+missing), the metric is reported as `null` so the iteration is rejected
+rather than silently accepted with broken benchmarks.
+
 ```bash
-# Set up Python environment if needed
+# Set up Python environment if needed.
 if ! command -v python3 &>/dev/null; then
-  echo "Python3 not found, skipping"
+  echo '{"benchmarked_functions": null, "rejected_reason": "python3 not available"}'
+  exit 0
 fi
 pip3 install pandas --quiet 2>/dev/null || true
 
-# Count the number of benchmark pairs (functions with both TS and Python benchmarks)
+# Validity: every TypeScript benchmark must transpile cleanly.
+# `bun build` parses, type-aware-transpiles, and resolves imports — any of
+# those failing means the benchmark would fail at run time. We discard the
+# build output; we only care about the exit status.
+if command -v bun &>/dev/null; then
+  for f in benchmarks/tsb/bench_*.ts; do
+    [ -e "$f" ] || break
+    if ! bun build "$f" --outdir=/tmp/perf-comparison-bench-check >/dev/null 2>&1; then
+      echo "{\"benchmarked_functions\": null, \"rejected_reason\": \"invalid TypeScript benchmark: $f\"}"
+      exit 0
+    fi
+  done
+else
+  echo '{"benchmarked_functions": null, "rejected_reason": "bun not available"}'
+  exit 0
+fi
+
+# Validity: every Python benchmark must compile (parse) cleanly.
+for f in benchmarks/pandas/bench_*.py; do
+  [ -e "$f" ] || break
+  if ! python3 -m py_compile "$f" 2>/dev/null; then
+    echo "{\"benchmarked_functions\": null, \"rejected_reason\": \"invalid Python benchmark: $f\"}"
+    exit 0
+  fi
+done
+
+# Count the number of benchmark pairs (functions with both TS and Python benchmarks).
 ts_benchmarks=$(ls benchmarks/tsb/bench_*.ts 2>/dev/null | wc -l | tr -d ' ')
 py_benchmarks=$(ls benchmarks/pandas/bench_*.py 2>/dev/null | wc -l | tr -d ' ')
 
-# The metric is the minimum of the two (both must exist for a complete benchmark)
+# The metric is the minimum of the two (both must exist for a complete benchmark).
 if [ "$ts_benchmarks" -lt "$py_benchmarks" ]; then
   count=$ts_benchmarks
 else
@@ -71,4 +103,6 @@ fi
 echo "{\"benchmarked_functions\": ${count:-0}}"
 ```
 
-The metric is `benchmarked_functions`. **Higher is better.**
+The metric is `benchmarked_functions`. **Higher is better.** When validity
+checks fail the metric is `null`, which the autoloop runner treats as a
+rejected iteration.