diff --git a/CLAUDE.md b/CLAUDE.md index 0387fd54..fb3a5338 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -282,6 +282,36 @@ Movement vs hand-audited: +`households_3, _7` (improvements); −`households_17` (lone regression). Movement vs pre-S1-S5 baseline: +`households_2, _7`, zero new regressions. +## Cascade stats per `(benchmark, agent_model, mode)` — use the script + +For "how is Opus doing on mini-interact in raw vs slayer right now?", the +canonical tool is `scripts/cascade_for_combo.py`. It walks +`runs////`, joins each annotation against its cloud +manifest (`results//cloud//manifest.json`, with a GCS +fallback that caches locally), filters to the requested `--mode` (raw or +slayer) and `--agent-model` substring (`opus` matches +`anthropic/claude-opus-4-7`), then picks the **latest non-`eval_failed`** +run per task — chronologically-latest can be a stale regrade whose grader +infrastructure choked, which would override a genuine earlier verdict. + +It prints BOTH the cumulative N1..N9 table (with `Δ vs prev` per tier; N1 +is the headline original-gold pass rate) AND the mutually-exclusive +partition L1..L11 (each task → exactly one tier). `--json` emits the +machine-readable payload. + +```bash +env -u SSH_AUTH_SOCK uv run python scripts/cascade_for_combo.py \ + --benchmark mini-interact --mode slayer --agent-model opus +env -u SSH_AUTH_SOCK uv run python scripts/cascade_for_combo.py \ + --benchmark mini-interact --mode raw --agent-model opus +``` + +Reach for this script, not ad-hoc `rglob` + cascade computation. The +"pick latest non-eval_failed" rule is non-obvious and the existing +`aggregate_cascading_latest` in `cascading_report.py` does NOT apply +either of these two filters. Contract is pinned by +`tests/scripts/test_cascade_for_combo.py`. + ## Debugging the cloud runner: pull live state, don't guess When a `bird-interact-cloud` run fails on GCE, **do not iterate by editing diff --git a/scripts/cascade_for_combo.py b/scripts/cascade_for_combo.py new file mode 100644 index 00000000..e29ef183 --- /dev/null +++ b/scripts/cascade_for_combo.py @@ -0,0 +1,383 @@ +"""Cascade-stat summary for one (benchmark, agent_model, mode) combination. + +For each ``(db, instance_id)`` under ``runs//``, picks the latest +submission annotation (by ``annotated_at``) produced by a cloud run whose +manifest matches the requested ``mode`` (raw|slayer) AND ``agent-model`` +substring (case-insensitive — ``opus`` matches +``anthropic/claude-opus-4-7``), SKIPPING runs whose +``evaluation.verdict == "eval_failed"``. Those are stale grader-infrastructure +failures that would otherwise wrongly override the genuine earlier verdict. + +Emits BOTH views, since both are useful: + +* Cumulative N1..N9 — each ``n_k`` is "task passes AT OR BELOW cascade tier + k" (monotone, ``n1 ⊆ n2 ⊆ ... ⊆ n9``). ``n1`` is the headline original-gold + pass rate. ``Δ vs prev`` shows the incremental tier contribution. +* Mutually-exclusive partition L1..L11 — each task → exactly one tier. This + is the "where did pass/fail land" view. + +Manifest lookup: reads ``results//cloud//manifest.json`` +when present; otherwise downloads from GCS (cached under that same path). +``--no-gcs`` disables the GCS fallback (run files whose manifest isn't local +get skipped, with a warning). + +Usage +----- + uv run python scripts/cascade_for_combo.py \\ + --benchmark mini-interact --mode slayer --agent-model opus +""" +from __future__ import annotations + +import argparse +import json +import logging +import re +import sys +from pathlib import Path +from typing import Optional + +from bird_interact_agents import paths +from bird_interact_agents.cloud import gcs +from bird_interact_agents.eval.annotation_io import read_submission_annotation +from bird_interact_agents.eval.cascading_report import ( + _annotation_cascade_bools, + _assign_partition_tier, + _PARTITION_IS_PASS, + _PARTITION_LABELS, +) + + +logger = logging.getLogger("cascade_for_combo") + + +_FILE_MODE_RE = re.compile(r"-(raw|slayer)-") + + +def mode_from_filename(name: str) -> Optional[str]: + """Return ``"raw"`` / ``"slayer"`` / ``None`` from the slot token in the + cloud-run filename (``---.json``).""" + m = _FILE_MODE_RE.search(name) + return m.group(1) if m else None + + +def load_manifest( + benchmark: str, run_id: str, *, gcs_client=None, allow_gcs: bool = True, +) -> Optional[dict]: + """Return the cloud manifest for ``run_id``, or ``None`` if unavailable. + + Reads from ``results//cloud//manifest.json`` when + present. Falls back to GCS (caching the result locally) unless + ``allow_gcs`` is False. + """ + local = ( + paths.results_root() / benchmark / "cloud" / run_id / "manifest.json" + ) + if local.exists(): + try: + return json.loads(local.read_text()) + except json.JSONDecodeError: + logger.warning("corrupt local manifest %s; re-fetching", local) + if not allow_gcs: + return None + try: + manifest = gcs.read_manifest(run_id, client=gcs_client) + except Exception as exc: # noqa: BLE001 — any GCS error is "skip this run" + logger.warning("GCS manifest for %s unavailable: %s", run_id, exc) + return None + local.parent.mkdir(parents=True, exist_ok=True) + local.write_text(json.dumps(manifest, indent=2, default=str)) + return manifest + + +def model_matches(manifest_model: Optional[str], requested: str) -> bool: + """Case-insensitive substring match. ``opus`` matches + ``anthropic/claude-opus-4-7``.""" + if not manifest_model: + return False + return requested.lower() in manifest_model.lower() + + +def _effective_verdict(data: dict) -> str: + """Mirror ``SubmissionAnnotation._migrate_invalid_verdict`` for raw-JSON + reads: legacy ``verdict="invalid"`` rows with + ``failure_classification.primary="other"`` are infra failures and must + be classified as ``"eval_failed"`` so they don't sneak past the + "skip eval_failed" filter.""" + ev = data.get("evaluation") or {} + fc = data.get("failure_classification") or {} + raw = ev.get("verdict") or "" + if raw == "invalid" and fc.get("primary") == "other": + return "eval_failed" + return raw + + +def collect_latest_per_task( + *, + benchmark: str, + mode: str, + agent_model: str, + runs_root: Optional[Path] = None, + allow_gcs: bool = True, + gcs_client=None, +) -> tuple[list[Path], dict[str, int]]: + """Pick the latest non-eval_failed run per ``(db, instance_id)`` that + matches ``(mode, agent_model)``. Returns the list of chosen file paths + plus a small bookkeeping dict (files seen / skipped / overrides used).""" + runs_root = runs_root or (paths.runs_root() / benchmark) + manifest_cache: dict[str, Optional[dict]] = {} + candidates: dict[tuple[str, str], list[tuple[str, Path, str]]] = {} + counters = { + "files_scanned": 0, + "matched_mode": 0, + "matched_model": 0, + "skipped_no_manifest": 0, + "stale_eval_failed_overridden": 0, + "skipped_eval_failed_only": 0, + } + if not runs_root.exists(): + return [], counters + + for path in runs_root.rglob("*.json"): + if path.name.endswith(".trajectory.json"): + continue + counters["files_scanned"] += 1 + file_mode = mode_from_filename(path.name) + if file_mode != mode: + continue + counters["matched_mode"] += 1 + try: + rel = path.relative_to(runs_root) + except ValueError: + continue + parts = rel.parts + if len(parts) != 3: + continue + db, iid, run_file = parts + run_id = run_file[:-5] + if run_id not in manifest_cache: + manifest_cache[run_id] = load_manifest( + benchmark, run_id, gcs_client=gcs_client, allow_gcs=allow_gcs, + ) + manifest = manifest_cache[run_id] + if manifest is None: + counters["skipped_no_manifest"] += 1 + continue + if manifest.get("query_mode") != mode: + continue + if not model_matches(manifest.get("agent_model"), agent_model): + continue + counters["matched_model"] += 1 + try: + data = json.loads(path.read_text()) + except json.JSONDecodeError as exc: + logger.warning("unreadable annotation %s: %s", path, exc) + continue + verdict = _effective_verdict(data) + annotated_at = data.get("annotated_at") or "" + candidates.setdefault((db, iid), []).append( + (annotated_at, path, verdict), + ) + + chosen: list[Path] = [] + for recs in candidates.values(): + recs.sort() + real = [r for r in recs if r[2] != "eval_failed"] + if not real: + counters["skipped_eval_failed_only"] += 1 + continue + if recs[-1][2] == "eval_failed": + counters["stale_eval_failed_overridden"] += 1 + chosen.append(real[-1][1]) + return chosen, counters + + +def aggregate(paths_in: list[Path]) -> dict: + n_counts = {f"n{i}": 0 for i in range(1, 10)} + p_counts = {k: 0 for k in _PARTITION_LABELS} + n = 0 + for p in paths_in: + ann = read_submission_annotation(p) + n += 1 + for k, v in _annotation_cascade_bools(ann).items(): + if v: + n_counts[k.split("_", 1)[0]] += 1 + p_counts[_assign_partition_tier(ann)] += 1 + return {"n": n, "n_counts": n_counts, "p_counts": p_counts} + + +_N_LABELS = { + "n1": "original gold", + "n2": "audited primary", + "n3": "any audited variant", + "n4": "correct up to tie order", + "n5": "LLM judge", + "n6": "numeric epsilon", + "n7": "trailing whitespace", + "n8": "column order", + "n9": "case fold", +} + + +def render( + *, + benchmark: str, + agent_model: str, + mode: str, + agg: dict, + counters: dict, +) -> str: + n = agg["n"] + lines: list[str] = [] + bar = "=" * 92 + lines.append(bar) + lines.append( + f"Cascade summary benchmark={benchmark} " + f"agent_model~={agent_model!r} mode={mode} " + f"n={n} (latest non-eval_failed run per task)" + ) + lines.append(bar) + if not n: + lines.append("") + lines.append( + "No matching tasks. Counters: " + json.dumps(counters, indent=2) + ) + return "\n".join(lines) + + lines.append("") + lines.append( + "Cumulative N-tier (each n_k = tasks passing AT OR BELOW cascade tier k)" + ) + lines.append("") + lines.append( + f" {'tier':<6}{'description':<30}" + f"{'cum_pass':>10}{'rate':>10}{'Δ vs prev':>12}" + ) + prev = 0 + for k in ("n1", "n2", "n3", "n4", "n5", "n6", "n7", "n8", "n9"): + c = agg["n_counts"][k] + delta = c - prev + lines.append( + f" {k:<6}{_N_LABELS[k]:<30}" + f"{c:>10}{c/n*100:>9.1f}%{delta:>+12d}" + ) + prev = c + + lines.append("") + lines.append( + "Mutually-exclusive partition (each task → exactly one tier)" + ) + lines.append("") + lines.append( + f" {'tier':<32}{'cnt':>5}{'rate':>9}{'cumsum':>9}{'cum_rate':>11}" + ) + cum = 0 + cum_pass = 0 + for label in _PARTITION_LABELS: + c = agg["p_counts"][label] + cum += c + is_pass = _PARTITION_IS_PASS[label] + if is_pass: + cum_pass += c + marker = "+" if is_pass else "-" + lines.append( + f" {marker} {label:<30}{c:>5}{c/n*100:>8.1f}%" + f"{cum:>9}{cum/n*100:>10.1f}%" + ) + lines.append("") + lines.append( + f"Cascade PASS = {cum_pass}/{n} ({cum_pass/n*100:.1f}%) " + f"L2 diagnostic = {agg['p_counts']['l2_wrong_original']} " + f"L11 hard fail = {agg['p_counts']['l11_fail']}" + ) + + lines.append("") + lines.append( + "Counters: " + f"files_scanned={counters['files_scanned']} " + f"matched_mode={counters['matched_mode']} " + f"matched_model={counters['matched_model']} " + f"skipped_no_manifest={counters['skipped_no_manifest']} " + f"stale_eval_failed_overridden={counters['stale_eval_failed_overridden']} " + f"skipped_eval_failed_only={counters['skipped_eval_failed_only']}" + ) + return "\n".join(lines) + + +def main(argv: Optional[list[str]] = None) -> int: + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + parser.add_argument( + "--benchmark", default="mini-interact", + help="Benchmark canonical name (default: mini-interact).", + ) + parser.add_argument( + "--mode", required=True, choices=("raw", "slayer"), + help="Query mode of the runs to include.", + ) + parser.add_argument( + "--agent-model", required=True, + help=( + "Case-insensitive substring match against the cloud manifest's " + "agent_model field (e.g. 'opus' matches " + "'anthropic/claude-opus-4-7')." + ), + ) + parser.add_argument( + "--no-gcs", action="store_true", + help=( + "Don't fall back to GCS for missing local manifests. Run files " + "whose manifest isn't already cached locally get skipped." + ), + ) + parser.add_argument( + "--json", action="store_true", + help="Emit machine-readable JSON instead of the human table.", + ) + args = parser.parse_args(argv) + + logging.basicConfig( + level=logging.WARNING, format="%(levelname)s %(name)s: %(message)s", + ) + + # GCS client is constructed lazily on the first cache miss inside + # ``gcs.read_manifest`` — eagerly constructing here would force every + # caller to have ADC even when the local cache is complete and no GCS + # fallback is actually needed. + chosen, counters = collect_latest_per_task( + benchmark=args.benchmark, + mode=args.mode, + agent_model=args.agent_model, + allow_gcs=not args.no_gcs, + ) + agg = aggregate(chosen) + + if args.json: + payload = { + "benchmark": args.benchmark, + "mode": args.mode, + "agent_model_filter": args.agent_model, + "n_tasks": agg["n"], + "cumulative_n_counts": agg["n_counts"], + "partition_l_counts": agg["p_counts"], + "counters": counters, + } + json.dump(payload, sys.stdout, indent=2) + sys.stdout.write("\n") + return 0 + + print( + render( + benchmark=args.benchmark, + agent_model=args.agent_model, + mode=args.mode, + agg=agg, + counters=counters, + ) + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/cloud/test_fetch_annotation_merge.py b/tests/cloud/test_fetch_annotation_merge.py index 4c9bfec5..f2d8466e 100644 --- a/tests/cloud/test_fetch_annotation_merge.py +++ b/tests/cloud/test_fetch_annotation_merge.py @@ -113,19 +113,24 @@ def test_merge_writes_annotation_to_main_checkout(tmp_path, monkeypatch): def test_merge_no_overwrite_if_present(tmp_path, monkeypatch): - from bird_interact_agents import paths as _paths from bird_interact_agents.cloud.post_run_merge import ( merge_submission_annotations, ) - main_checkout = tmp_path / "checkout" - dest_dir = main_checkout / "annotations" / "mini-interact" / "alien" - dest_dir.mkdir(parents=True) - fake_ann_root = main_checkout / "annotations" - monkeypatch.setattr(_paths, "annotations_root", lambda: fake_ann_root) + # Sandbox runs/ root via BIRD_RUNS_ROOT so the merge writes under tmp_path + # rather than the real checkout. Without this the test relies on whatever + # is currently in runs/mini-interact/alien/alien_1/r1.json — leftover from + # prior tests in the session. + runs_root = tmp_path / "runs" + monkeypatch.setenv("BIRD_RUNS_ROOT", str(runs_root)) + + dest = ( + runs_root / "mini-interact" / "alien" / "alien_1" / "r1.json" + ) + dest.parent.mkdir(parents=True) pre = _valid_submission_annotation_dict("alien_1") pre["annotated_by"] = "human-pre-existing" - (dest_dir / "alien_1.submission.r1.json").write_text(json.dumps(pre)) + dest.write_text(json.dumps(pre)) downloaded = tmp_path / "downloaded" rows = downloaded / "rows" / "alien_1" @@ -142,9 +147,7 @@ def test_merge_no_overwrite_if_present(tmp_path, monkeypatch): assert report.merged == 0 assert report.skipped_existing == 1 - surviving = json.loads( - (dest_dir / "alien_1.submission.r1.json").read_text() - ) + surviving = json.loads(dest.read_text()) assert surviving["annotated_by"] == "human-pre-existing" diff --git a/tests/scripts/test_cascade_for_combo.py b/tests/scripts/test_cascade_for_combo.py new file mode 100644 index 00000000..8a14095b --- /dev/null +++ b/tests/scripts/test_cascade_for_combo.py @@ -0,0 +1,398 @@ +"""Tests for ``scripts/cascade_for_combo.py``. + +Covers the contract that matters: pick the latest non-eval_failed run per +(db, instance_id) AFTER filtering by manifest's ``query_mode`` AND +``agent_model`` substring. The cumulative-N and partition-L aggregations +reuse the production helpers in +``bird_interact_agents.eval.cascading_report`` so are not re-tested here. +""" + +from __future__ import annotations + +import importlib.util +import json +import sys +from pathlib import Path + +import pytest + +SCRIPT = ( + Path(__file__).resolve().parents[2] / "scripts" / "cascade_for_combo.py" +) +_spec = importlib.util.spec_from_file_location("cascade_for_combo", SCRIPT) +cascade_for_combo = importlib.util.module_from_spec(_spec) +sys.modules["cascade_for_combo"] = cascade_for_combo +_spec.loader.exec_module(cascade_for_combo) + + +_BENCHMARK = "mini-interact" + + +def _write_annotation( + *, + runs_root: Path, + db: str, + iid: str, + run_id: str, + annotated_at: str, + n1: bool = False, + n2: bool = False, + verdict: str = "correct", +) -> Path: + """Write a minimal SubmissionAnnotation JSON. Mode is encoded into the + run_id (caller threads ``-raw-`` / ``-slayer-``).""" + dest = runs_root / _BENCHMARK / db / iid / f"{run_id}.json" + dest.parent.mkdir(parents=True, exist_ok=True) + dest.write_text( + json.dumps( + { + "schema_version": 1, + "kind": "submission_annotation", + "instance_id": iid, + "selected_database": db, + "task_annotation_ref": ( + f"annotations/{_BENCHMARK}/{db}/{iid}.task.json" + ), + "annotated_by": "test", + "annotated_at": annotated_at, + "submission": { + "cloud_run_id": run_id, + "trajectory_path": f"rows/{iid}/attempt-1.json", + "submitted_sql_path": None, + "predicted_row_count": 1, + "duration_s": 1.0, + "cost_usd_agent": 0.0, + "cost_usd_user_sim": 0.0, + "n_agent_turns": 1, + "n_ask_user_calls": 0, + }, + "evaluation": { + "phase1_against_original_gold": "pass" if n1 else "fail", + "phase1_against_audited_primary": "pass" if n2 else "fail", + "phase1_against_any_audited_variant": "pass" if n2 else "fail", + "phase1_against_variants": [], + "correct_up_to_tie_order": False, + "novel_reading_judgment": None, + "correct_under_numeric_epsilon": False, + "correct_under_trailing_whitespace": False, + "correct_under_column_order": False, + "correct_under_case_fold": False, + "numeric_epsilon": 1e-6, + "verdict": verdict, + "matched_variant_id": "primary" if n2 else None, + "rationale": "", + }, + "failure_classification": { + "primary": "no_fail" if n1 or n2 else "agent_miss", + "secondary": [], + "agent_at_fault": not (n1 or n2), + "remediation_target": "other", + "remediation_text": "", + "details": "", + }, + "decision_point": None, + "user_sim_interaction": { + "n_asks": 0, "key_responses": [], + "disclosed_resolutions": [], + "undisclosed_resolutions": [], + }, + "original_gold_annotated_correct": True, + } + ) + ) + return dest + + +def _write_manifest( + *, + results_root: Path, + run_id: str, + query_mode: str, + agent_model: str, +) -> None: + dest = ( + results_root / _BENCHMARK / "cloud" / run_id / "manifest.json" + ) + dest.parent.mkdir(parents=True, exist_ok=True) + dest.write_text( + json.dumps( + {"query_mode": query_mode, "agent_model": agent_model} + ) + ) + + +@pytest.fixture +def isolated_tree(tmp_path, monkeypatch): + """Point runs_root + results_root at tmp_path, and disable GCS lookup.""" + runs = tmp_path / "runs" + results = tmp_path / "results" + monkeypatch.setenv("BIRD_RUNS_ROOT", str(runs)) + monkeypatch.setenv("BIRD_RESULTS_ROOT", str(results)) + return runs, results + + +def test_filters_by_mode_and_model_substring(isolated_tree): + """Files outside the requested mode or model are excluded; substring + match is case-insensitive.""" + runs, results = isolated_tree + # opus / slayer — wanted, n1 pass + _write_annotation( + runs_root=runs, db="alien", iid="alien_1", + run_id="20260601t1000-claudes-slayer-aaaaaa", + annotated_at="2026-06-01T10:00:00+00:00", n1=True, n2=True, + ) + _write_manifest( + results_root=results, + run_id="20260601t1000-claudes-slayer-aaaaaa", + query_mode="slayer", + agent_model="anthropic/claude-OPUS-4-7", # uppercase — must still match + ) + # haiku / slayer — same task, must be filtered out by --agent-model opus + _write_annotation( + runs_root=runs, db="alien", iid="alien_2", + run_id="20260602t1000-claudes-slayer-bbbbbb", + annotated_at="2026-06-02T10:00:00+00:00", n1=True, + ) + _write_manifest( + results_root=results, + run_id="20260602t1000-claudes-slayer-bbbbbb", + query_mode="slayer", + agent_model="anthropic/claude-haiku-4-5", + ) + # opus / raw — same task, must be filtered out by --mode slayer + _write_annotation( + runs_root=runs, db="alien", iid="alien_3", + run_id="20260603t1000-claudesdk-raw-cccccc", + annotated_at="2026-06-03T10:00:00+00:00", n1=True, + ) + _write_manifest( + results_root=results, + run_id="20260603t1000-claudesdk-raw-cccccc", + query_mode="raw", + agent_model="anthropic/claude-opus-4-7", + ) + + chosen, counters = cascade_for_combo.collect_latest_per_task( + benchmark=_BENCHMARK, + mode="slayer", + agent_model="opus", + allow_gcs=False, + ) + assert [p.name for p in chosen] == [ + "20260601t1000-claudes-slayer-aaaaaa.json" + ], counters + assert counters["matched_mode"] == 2 # two slayer files seen + assert counters["matched_model"] == 1 # only the opus one passed + assert counters["skipped_no_manifest"] == 0 + + +def test_eval_failed_latest_falls_back_to_prior_real_verdict(isolated_tree): + """If chronologically-latest is eval_failed AND a prior real verdict + exists, the prior wins (and the counter records it).""" + runs, results = isolated_tree + iid = "alien_1" + # earlier real verdict + _write_annotation( + runs_root=runs, db="alien", iid=iid, + run_id="20260601t1000-claudes-slayer-aaaaaa", + annotated_at="2026-06-01T10:00:00+00:00", + n1=True, verdict="correct", + ) + _write_manifest( + results_root=results, + run_id="20260601t1000-claudes-slayer-aaaaaa", + query_mode="slayer", agent_model="anthropic/claude-opus-4-7", + ) + # later eval_failed regrade — must NOT be picked + _write_annotation( + runs_root=runs, db="alien", iid=iid, + run_id="20260612t1000-claudes-slayer-bbbbbb", + annotated_at="2026-06-12T10:00:00+00:00", + n1=False, verdict="eval_failed", + ) + _write_manifest( + results_root=results, + run_id="20260612t1000-claudes-slayer-bbbbbb", + query_mode="slayer", agent_model="anthropic/claude-opus-4-7", + ) + + chosen, counters = cascade_for_combo.collect_latest_per_task( + benchmark=_BENCHMARK, + mode="slayer", + agent_model="opus", + allow_gcs=False, + ) + assert len(chosen) == 1 + assert "20260601t1000" in chosen[0].name + assert counters["stale_eval_failed_overridden"] == 1 + + +def test_eval_failed_only_task_is_omitted_from_aggregate(isolated_tree): + """If every run for a task is ``eval_failed`` (no genuine verdict ever + came through), the task is omitted from ``chosen`` so it doesn't get + silently miscounted as an L11 hard fail in the cascade aggregate.""" + runs, results = isolated_tree + iid = "alien_1" + _write_annotation( + runs_root=runs, db="alien", iid=iid, + run_id="20260601t1000-claudes-slayer-aaaaaa", + annotated_at="2026-06-01T10:00:00+00:00", + n1=False, verdict="eval_failed", + ) + _write_manifest( + results_root=results, + run_id="20260601t1000-claudes-slayer-aaaaaa", + query_mode="slayer", agent_model="anthropic/claude-opus-4-7", + ) + _write_annotation( + runs_root=runs, db="alien", iid=iid, + run_id="20260602t1000-claudes-slayer-bbbbbb", + annotated_at="2026-06-02T10:00:00+00:00", + n1=False, verdict="eval_failed", + ) + _write_manifest( + results_root=results, + run_id="20260602t1000-claudes-slayer-bbbbbb", + query_mode="slayer", agent_model="anthropic/claude-opus-4-7", + ) + + chosen, counters = cascade_for_combo.collect_latest_per_task( + benchmark=_BENCHMARK, + mode="slayer", agent_model="opus", allow_gcs=False, + ) + assert chosen == [] + assert counters["skipped_eval_failed_only"] == 1 + assert counters["stale_eval_failed_overridden"] == 0 + + +def test_legacy_invalid_plus_other_is_treated_as_eval_failed(isolated_tree): + """Annotations from before the ``invalid`` → ``eval_failed`` migration + (``verdict="invalid"`` + ``failure_classification.primary="other"``) + must be classified as infra failures, mirroring + ``SubmissionAnnotation._migrate_invalid_verdict``.""" + runs, results = isolated_tree + iid = "alien_1" + # earlier real verdict + _write_annotation( + runs_root=runs, db="alien", iid=iid, + run_id="20260601t1000-claudes-slayer-aaaaaa", + annotated_at="2026-06-01T10:00:00+00:00", + n1=True, verdict="correct", + ) + _write_manifest( + results_root=results, + run_id="20260601t1000-claudes-slayer-aaaaaa", + query_mode="slayer", agent_model="anthropic/claude-opus-4-7", + ) + # later LEGACY infra failure (verdict=invalid + primary=other) — must + # be treated as eval_failed, NOT pick over the earlier correct verdict + later = _write_annotation( + runs_root=runs, db="alien", iid=iid, + run_id="20260612t1000-claudes-slayer-bbbbbb", + annotated_at="2026-06-12T10:00:00+00:00", + n1=False, verdict="invalid", + ) + raw = json.loads(later.read_text()) + raw["failure_classification"]["primary"] = "other" + later.write_text(json.dumps(raw)) + _write_manifest( + results_root=results, + run_id="20260612t1000-claudes-slayer-bbbbbb", + query_mode="slayer", agent_model="anthropic/claude-opus-4-7", + ) + + chosen, counters = cascade_for_combo.collect_latest_per_task( + benchmark=_BENCHMARK, + mode="slayer", agent_model="opus", allow_gcs=False, + ) + assert len(chosen) == 1 + assert "20260601t1000" in chosen[0].name + assert counters["stale_eval_failed_overridden"] == 1 + + +def test_no_manifest_with_no_gcs_skips_run(isolated_tree): + """When the local manifest is missing AND ``allow_gcs=False``, the run + is skipped and the counter records it.""" + runs, _results = isolated_tree + _write_annotation( + runs_root=runs, db="alien", iid="alien_1", + run_id="20260601t1000-claudes-slayer-aaaaaa", + annotated_at="2026-06-01T10:00:00+00:00", n1=True, + ) + # (no manifest written) + chosen, counters = cascade_for_combo.collect_latest_per_task( + benchmark=_BENCHMARK, + mode="slayer", + agent_model="opus", + allow_gcs=False, + ) + assert chosen == [] + assert counters["skipped_no_manifest"] == 1 + + +def test_main_does_not_require_gcs_when_local_cache_is_complete( + isolated_tree, monkeypatch, capsys, +): + """``main()`` must not construct a GCS client when every needed + manifest is already on disk — the canonical local-reporting invocation + has to work on machines without ADC.""" + runs, results = isolated_tree + _write_annotation( + runs_root=runs, db="alien", iid="alien_1", + run_id="20260601t1000-claudes-slayer-aaaaaa", + annotated_at="2026-06-01T10:00:00+00:00", n1=True, n2=True, + ) + _write_manifest( + results_root=results, + run_id="20260601t1000-claudes-slayer-aaaaaa", + query_mode="slayer", agent_model="anthropic/claude-opus-4-7", + ) + + def _no_adc(*_a, **_kw): + raise RuntimeError("ADC not available") + + monkeypatch.setattr(cascade_for_combo.gcs, "default_gcs_client", _no_adc) + + rc = cascade_for_combo.main([ + "--benchmark", _BENCHMARK, + "--mode", "slayer", "--agent-model", "opus", "--json", + ]) + assert rc == 0 + payload = json.loads(capsys.readouterr().out) + assert payload["n_tasks"] == 1 + assert payload["cumulative_n_counts"]["n1"] == 1 + + +def test_aggregate_partitions_correctly(isolated_tree): + """End-to-end: two n1-pass + one n2-only + one fail should produce + n1=2, n2=3, partition L1=2, L3=1, L11=1.""" + runs, results = isolated_tree + cases = [ + ("alien_1", True, True), # n1+n2 → L1 + ("alien_2", True, True), # n1+n2 → L1 + ("alien_3", False, True), # n2 only → L3 + ("alien_4", False, False), # nothing → L11 + ] + for i, (iid, n1, n2) in enumerate(cases): + run_id = f"20260601t100{i}-claudes-slayer-{'a'*6}" + _write_annotation( + runs_root=runs, db="alien", iid=iid, run_id=run_id, + annotated_at=f"2026-06-01T10:0{i}:00+00:00", + n1=n1, n2=n2, + verdict="correct" if (n1 or n2) else "agent_miss", + ) + _write_manifest( + results_root=results, run_id=run_id, + query_mode="slayer", agent_model="anthropic/claude-opus-4-7", + ) + chosen, _ = cascade_for_combo.collect_latest_per_task( + benchmark=_BENCHMARK, + mode="slayer", agent_model="opus", allow_gcs=False, + ) + agg = cascade_for_combo.aggregate(chosen) + assert agg["n"] == 4 + assert agg["n_counts"]["n1"] == 2 + assert agg["n_counts"]["n2"] == 3 + assert agg["p_counts"]["l1_correct_original"] == 2 + assert agg["p_counts"]["l3_audited_primary"] == 1 + assert agg["p_counts"]["l11_fail"] == 1