From 9e4ba528ba64116effa9c0aa326aed7fe1e5b87b Mon Sep 17 00:00:00 2001 From: Egor Kraev Date: Sun, 31 May 2026 17:56:42 +0200 Subject: [PATCH 01/29] DEV-1515 analyses: failure-mode notes for post-DEV-1478 households + museum runs analyses/: per-run summaries for `20260531t1008-claudes-slayer-890419` (households 11/15) and `20260531t1013-claudes-slayer-48eb0f` (museum 4/10). analyses/raw/: per-task deep dives for the 10 failures (museum_2..10 + households_2/10/12/15) and one cross-cutting observations file. Each references the trajectory items, KB ids, column-meaning lines, and slayer model paths used as evidence; documents whether the answer was derivable in principle from supplied metadata and which signal the agent failed to use (when applicable). Co-Authored-By: Claude Opus 4.7 (1M context) --- ...sis_20260531t1008-claudes-slayer-890419.md | 146 ++++++++++++++++++ ...sis_20260531t1013-claudes-slayer-48eb0f.md | 119 ++++++++++++++ analyses/raw/cross_cutting_observations.md | 52 +++++++ analyses/raw/households_10.md | 133 ++++++++++++++++ analyses/raw/households_12.md | 66 ++++++++ analyses/raw/households_15.md | 67 ++++++++ analyses/raw/households_2.md | 60 +++++++ analyses/raw/museum_10.md | 63 ++++++++ analyses/raw/museum_2.md | 59 +++++++ analyses/raw/museum_3.md | 52 +++++++ analyses/raw/museum_4.md | 67 ++++++++ analyses/raw/museum_5.md | 50 ++++++ analyses/raw/museum_9.md | 85 ++++++++++ 13 files changed, 1019 insertions(+) create mode 100644 analyses/households_failure_analysis_20260531t1008-claudes-slayer-890419.md create mode 100644 analyses/museum_failure_analysis_20260531t1013-claudes-slayer-48eb0f.md create mode 100644 analyses/raw/cross_cutting_observations.md create mode 100644 analyses/raw/households_10.md create mode 100644 analyses/raw/households_12.md create mode 100644 analyses/raw/households_15.md create mode 100644 analyses/raw/households_2.md create mode 100644 analyses/raw/museum_10.md create mode 100644 analyses/raw/museum_2.md create mode 100644 analyses/raw/museum_3.md create mode 100644 analyses/raw/museum_4.md create mode 100644 analyses/raw/museum_5.md create mode 100644 analyses/raw/museum_9.md diff --git a/analyses/households_failure_analysis_20260531t1008-claudes-slayer-890419.md b/analyses/households_failure_analysis_20260531t1008-claudes-slayer-890419.md new file mode 100644 index 00000000..2bbc67b2 --- /dev/null +++ b/analyses/households_failure_analysis_20260531t1008-claudes-slayer-890419.md @@ -0,0 +1,146 @@ +# Households (mini-interact / a-interact) failure analysis + +**Run ID:** `20260531t1008-claudes-slayer-890419` +**Date:** 2026-05-31 +**Branch / commit:** `main @ af58457` (post-README defaults bake) +**Compared against prior:** `20260529t2103-claudes-slayer-ca9536` (11/15 today vs 11/15 prior; flip: `_14` now passes, `_15` now fails) + +## Configuration + +| field | value | +| -- | -- | +| framework | `claude_sdk_otf_ainteract` | +| dataset / mode | `mini_interact` / `a-interact` | +| agent model | `anthropic/claude-opus-4-7` | +| user-sim model | `anthropic/claude-sonnet-4-6` | +| reasoning effort | `high` | +| slayer setup | `on-the-fly` | +| audited gold | on (`use_audited_gold_sql=True`) | +| patience | 500 | +| instance ids | all 15 households (`_1.._19` minus skipped) | +| cluster | 1 worker × 1 actor, `e2-standard-4` | + +## Headline result + +| metric | value | +| -- | -- | +| phase1 pass rate (raw) | **11 / 15 (73%)** | +| phase1 pass rate (after accountability adjustment) | **14 / 15 (93%)** | +| run cost | **$57.19** (agent $55.94, user-sim $1.25) | +| longest task | households_12 (1238 s, $12.05) | +| shortest pass | households_16 (33 s, $0.41) | + +Adjustment rationale: of the 4 failures, **1 is a genuine agent miss**, **1 is a gold/audit bug that violates the audit-gold-sql contract**, **1 is a gold-spec gap compounded by user-sim under-disclosure**, and **1 is a known-`unrecoverable` audit instance that should be quarantined from the eval set**. + +## Per-task verdicts + +| inst | dur | cost | category | root cause | +| -- | --: | --: | -- | -- | +| households_1 | 75 s | $1.16 | PASS (audited only) | — | +| households_2 | 908 s | $7.78 | FAIL — **agent miss** | User-sim at idx 100 told the agent "strict exact matches, no normalization"; slayer model exposed `Riacho Fundo I` (two spaces) as a distinct sampled_value; agent's idx 125 raw-locregion query returned the gold answer `Riacho Fundo I \| 0.500`; agent then ran a normalized variant at idx 127, saw `riacho fundo i \| 0.008`, and **submitted the normalized one anyway**. Final assistant text at idx 205 explicitly compares the two and picks the wrong one. | +| households_3 | 537 s | $4.32 | PASS (audited + original) | — | +| households_4 | 109 s | $0.93 | PASS (audited only) | — | +| households_5 | 265 s | $1.60 | PASS (audited only) | — | +| households_6 | 61 s | $0.75 | PASS (audited only) | — | +| households_7 | 111 s | $1.20 | PASS (audited only) | — | +| households_10 | 926 s | $11.68 | FAIL — **gold/audit bug + user-sim under-disclosure** | The DEV-1478 re-audit expands KB labels with synonym sets (`'y'`, `'have'`, `'apt'`, `'brick house'`, `'2014+'`, `'after 2014'`, `'10-13'`) marked `clause_kind="manual_data_align_reaudit"`, `justified_by: []` — i.e. the auditor itself admits there is no KB / column-meaning / schema justification. KBs 6, 7, 10 list only canonical labels. Agent's submission is KB-faithful. User-sim refused to disclose synonyms (idx 35 listed canonicals only) or row-counts (idx 105 said *"I don't have access to expected row counts"*). | +| households_11 | 236 s | $1.74 | PASS (audited only) | — | +| households_12 | 1238 s | $12.05 | FAIL — **gold-spec gap + user-sim failure** | "Independent households" mask requires `domestichelp='no domestic workers' AND total_vehicles>1` — present nowhere in KBs (KB 42 only mentions `socsupport='No'` + "high income"), column meanings, or schema. Lives only inside `critical_ambiguity.sql_snippet`. User-sim, holding that snippet, told the agent at idx 34 *"don't have a specific cutoff for high income — just use all households that qualify as independent based on the service criteria"* — vague, no clause naming. Agent correctly normalised `Dwelling_Class` aliases (KB-encoded in slayer's pre-baked `dwelling_type_clean`) — that part isn't the failure. | +| households_14 | 291 s | $2.34 | PASS (audited only) | — | +| households_15 | 618 s | $6.53 | FAIL — **gold/audit bug, known `unrecoverable`** | The audit itself stamps `audit_status="unrecoverable"`. The Socioeconomic Index coefficients (0.4 / 0.4 / 0.2), threshold (0.7), ratio cap (1.5), and income 1-6 ordinal mapping live only in the labeled `sql_snippet`. KB 28 ("Economically Stable Household") describes the concept narratively but isn't even in this task's `external_knowledge=[9,11,24,25]`. Auditor kept the unjustified scaffolding intentionally because removing it would over-rewrite. User-sim eventually leaked the formula on idx 136, after 4 prior asks failed — agent didn't pivot in time. | +| households_16 | 33 s | $0.41 | PASS (audited + original) | — | +| households_17 | 181 s | $1.37 | PASS (audited only) | — | +| households_19 | 417 s | $3.34 | PASS (audited + original) | — | + +## Was the audited gold derivable from data the agent had? + +The audit pipeline (`audit-gold-sql` skill, DEV-1478) re-authors original golds so each clause is justified by `_kb.jsonl` + `_column_meaning_base.json` + the task's labeled ambiguity sources. Per-task: + +### households_2 — yes, fully derivable. Agent ignored the evidence. + +- **KB / metadata:** KB 9 (high-risk family) + KB 10 (mobile household) + KB 14 (vehicle count) cover the conjunctive predicate. The slayer model's `households.yaml` `locregion.sampled_values` list contains `Riacho Fundo I` (two spaces) as a *distinct* sampled value next to `Riacho Fundo I` (one space) — direct evidence that normalising would collapse meaningful groups. +- **User-sim:** unambiguous and correct. idx 19 gave the full predicate; idx 100 said *"Please use strict exact matches without normalizing case or trimming for the filtering conditions"*. +- **What the agent did:** At idx 125 it ran a raw-locregion grouping and saw the gold output `Riacho Fundo I \| 0.500`. At idx 127 it then ran a normalized variant. At idx 130 it submitted the normalized one. Final text (idx 205) explicitly weighs both candidates and picks wrong. +- **Why:** the agent's data-hygiene prior ("dirty data should be normalised") overrode (a) an explicit user-sim instruction and (b) the sampled-values caveat. This is the only failure in this run where the agent had everything it needed and still submitted wrong. + +### households_10 — no, the audit itself is contract-violating. + +- **KB / metadata:** KB 6 lists only `'Brickwork house'`, `'Apartment'`, `'Condominium'`; KB 7 lists only `'avail'`, `'available'`, `'yes'`; KB 10 lists only canonical Newest_Year buckets. None of `'y'`, `'have'`, `'apt'`, `'brick house'`, `'2014+'`, `'after 2014'`, `'2010-2013'`, `'10-13'` appear anywhere in the KB or column-meaning corpus. +- **Audit provenance:** the change is marked `clause_kind="manual_data_align_reaudit"` with `justified_by: []` — the auditor explicitly admits no source justification. Per the audit-gold-sql skill's own contract (`.claude/skills/_shared/audit-gold-sql.contract.md`) such clauses should be DEFERRED, not encoded. This is a regression in audit quality on this instance. +- **User-sim:** idx 35 stated the canonical Newest_Year list verbatim; never volunteered synonyms. idx 105 declined to disclose row counts when the agent specifically asked for debugging help. +- **What the agent did:** submitted the KB-faithful canonical literal set. Returned 153 housenums vs gold's 233. +- **Net:** the agent did what the supplied corpus told it to do. The gold drifted from the corpus. + +### households_12 — no, gold-spec gap; user-sim compounded. + +- **KB / metadata:** KB 42 ("Economically Independent Household") says "high Income Classification + social support status of 'No'" — neither `domestichelp='no domestic workers'` nor `total_vehicles>1` is mentioned anywhere in KBs, column descriptions, or the slayer model. Both constraints live only inside `critical_ambiguity.sql_snippet`. +- **Slayer pre-baking helped on the dwelling part:** `properties.yaml` exposes `dwelling_type_clean` with sampled values `'Brickwork house', 'BRICKWORK HOUSE', 'brick house', 'BRICK HOUSE', "brickwork\thouse", 'Unfinished brickwork', 'Shack'`. Agent correctly handled this — alias folding is not the failure. +- **User-sim:** idx 20 leaked the prosperity-score formula correctly; idx 106/147/233 confirmed the dwelling-class alias merge. But on the critical question at idx 34 — "what exactly counts as independent?" — sim said *"just use all households that qualify as independent based on the service criteria"*, naming neither `domestichelp` nor `total_vehicles>1`. +- **What the agent did:** built `socsupport='No' AND dwelling_class<>'other'`. Predicted `('brickwork house', 2106)` vs gold `('brickwork house', 1373)` — same dwelling class (alias merge worked), but 53% over-count because single-vehicle and `domestichelp='yes domestic workers'` households leaked in. +- **Net:** primary fault is gold-spec gap (KB 42 incomplete); user-sim's vague non-answer at idx 34 made it irrecoverable. + +### households_15 — no, audit-stamped unrecoverable. + +- **KB / metadata:** original `external_knowledge=[9, 11, 24, 25]`. KB 28 (Economically Stable Household) is conceptually relevant but *not in this task's KB list*. KBs 9, 11, 24, 25 cover Social Support / Household Density / Crowded Household — irrelevant. The SEI coefficients 0.4/0.4/0.2, threshold 0.7, ratio cap 1.5, and income 1-6 ordinal mapping exist only in the gold `sql_snippet`. +- **Auditor's own assessment** (verbatim): *"Notes: (a) labeled 'financially secure' references income_score/tenure_score variables; we use the same 1-6 income mapping as in households_11 as scaffolding — not in any source but is required scaffolding."* The audit kept the unjustified clauses to preserve the labeled `sql_snippet` rather than rewrite to something derivable. +- **User-sim:** finally leaked the SEI formula on idx 136 (turn 5 of 5 asks) — too late. Earlier asks elicited the OR-of-(domestichelp, socsupport) reading the agent took. +- **What the agent did:** invented its own tenure filter (`OWNED` only) and an Income_Bracket ordinal that doesn't match any KB-mentioned label; never built the weighted SEI. +- **Net:** the instance is structurally unrecoverable — the labeled `sql_snippet` is its own source of truth. No agent can reproduce these specific numerical constants from the supplied metadata. + +## Cross-cutting observations + +### 1. User-sim under-disclosure is now the dominant failure aggravator + +Three of four failures involve the user-sim either being **too literal** (households_10 — answered the canonical-label question correctly but never volunteered the synonym coverage the gold actually needed), **too vague** (households_12 — "use the service criteria" instead of naming `domestichelp` + `total_vehicles>1`), or **too slow** (households_15 — leaked the formula only on the 5th of 5 asks). This is a separate failure mode from the museum-side single-bit grader feedback: here the agent has a channel to ask, and the channel is leaking signal at far below its theoretical capacity. + +A targeted change: when the user-sim is asked an open-ended "what exactly counts as X" question and X corresponds to a `critical_ambiguity` it holds, it should disclose **every clause** of `sql_snippet` rather than paraphrase. The current behaviour reads like a human deliberately holding back — appropriate for benchmark fidelity, but it's masking metadata-completeness issues we'd otherwise see clearly. + +### 2. The audit-gold-sql contract was violated on households_10 + +`audit_status="edited"` is supposed to mean "every changed clause is justified by KB / column_meaning / labeled ambiguity". households_10's audit explicitly records `clause_kind="manual_data_align_reaudit"`, `justified_by: []` — i.e. the auditor self-flagged that the expanded synonym sets have no source justification, then encoded them anyway. Per the skill's contract those clauses should have been DEFERRED. This is an audit-quality regression worth fixing upstream: any non-empty `justified_by: []` should bubble to the audit-time validator. + +### 3. Slayer's sampled_values caveats are *working* (and the agent ignored them) + +households_2 is direct evidence that the post-S1-S5 sampled-values caveat work has shipped correctly: `Riacho Fundo I` (two spaces) was visible in the slayer model alongside the clean variant. The agent didn't act on this signal *and* didn't act on the user-sim's explicit "don't normalise" instruction. This is purely an agent miss — the metadata side did its job. + +### 4. households_15 should be quarantined from the eval set + +`audit_status="unrecoverable"` is the auditor's signal that no minimal rewrite makes the gold KB-derivable. Counting these instances in the headline pass rate misrepresents the agent. Suggest filtering them out of `--instance-ids` for measurement runs, or partitioning the `eval.json` into `pass / fail / unrecoverable_gold` buckets. + +### 5. Same trajectory-truncation issue as museum + +`src/bird_interact_agents/agents/claude_sdk_otf_ainteract/agent.py:~370` clips each saved message to `str(msg)[:500]`. For the long households runs (households_12 at 1238 s, households_10 at 926 s) this is hiding most of the agent's inspect/search responses and the entire user-sim disclosure pattern. Cross-cuts both `claude_sdk_otf` and `claude_sdk_otf_ainteract`. + +## Cost breakdown + +Sum across the 15 instances: + +| component | USD | +| -- | --: | +| Opus agent | **$55.94** | +| Sonnet user-sim | $1.25 | +| **total** | **$57.19** | + +Per-task: households_12 ($12.05), households_10 ($11.68), households_2 ($7.78), households_15 ($6.53), households_3 ($4.32), households_19 ($3.34), households_14 ($2.34), households_11 ($1.74), households_5 ($1.60), households_17 ($1.37), households_7 ($1.20), households_1 ($1.16), households_4 ($0.93), households_6 ($0.75), households_16 ($0.41). + +Of the $57.19, **$38.04 (66%)** went to the 4 failing tasks. Of *that*, **$30.26 (53% of run total)** is on three failures where the agent is not at fault (gold/audit bug, gold-spec gap, unrecoverable). Same lesson as museum: the failure-cost concentration is dominated by tasks the model can't win on the current spec. + +## Suggested follow-ups + +1. **Quarantine households_15** (audit-stamped `unrecoverable`) from default `--instance-ids` for measurement runs; partition `eval.json` summary by audit_status. +2. **Re-audit households_10** under the audit-gold-sql contract. The current re-audit's `manual_data_align_reaudit` + `justified_by: []` clauses should be DEFERRED, or the audit should add a per-column KB entry that enumerates the synonym set with a justification anchor. +3. **Backfill KB 42** with the full "independent household" predicate (`domestichelp='no domestic workers' AND total_vehicles > 1`), or remove the masked clauses from `critical_ambiguity` so the user-sim has nothing to hold back. +4. **Tighten the user-sim disclosure rule** for `critical_ambiguity.sql_snippet`-anchored questions: when asked an open-ended definitional question and the held `sql_snippet` has multiple AND clauses, disclose each one (not a paraphrase). Lives in the user-sim prompt / sim policy. +5. **Add an agent-side prompt nudge** for the households_2 class: "If the user explicitly says 'use strict matches / do not normalize', that overrides your dirty-data priors and the slayer model's normalised columns. Quote the user instruction in your final reasoning." +6. **Raise the trajectory truncation cap** (same as museum). `agent.py:~370`. + +## Provenance + +- Per-task attempt artifacts: `/home/james/Dropbox/SLayer/bird-agents/results/cloud/20260531t1008-claudes-slayer-890419/rows//attempt-1.json` +- Manifest: `/home/james/Dropbox/SLayer/bird-agents/results/cloud/20260531t1008-claudes-slayer-890419/manifest.json` +- Eval summary: `…/eval.json` (`p1=11/15 (0.733)`) +- Task data: `/home/james/Dropbox/SLayer/mini-interact/mini_interact.jsonl` +- Audited gold: `/home/james/Dropbox/SLayer/bird-agents/audited_gold/households/households_audited.jsonl` +- Households metadata: `/home/james/Dropbox/SLayer/mini-interact/households/{households_kb.jsonl, households_column_meaning_base.json, households_schema.txt}` +- Slayer model: `/home/james/Dropbox/SLayer/bird-agents/slayer_models_otf/households/models/households/` (`households.yaml`, `transportation_assets.yaml`, `properties.yaml` — sampled_values blocks especially) +- Audit contract: `.claude/skills/_shared/audit-gold-sql.contract.md` diff --git a/analyses/museum_failure_analysis_20260531t1013-claudes-slayer-48eb0f.md b/analyses/museum_failure_analysis_20260531t1013-claudes-slayer-48eb0f.md new file mode 100644 index 00000000..fab2e4e8 --- /dev/null +++ b/analyses/museum_failure_analysis_20260531t1013-claudes-slayer-48eb0f.md @@ -0,0 +1,119 @@ +# Museum (LiveSQLBench / one-shot) failure analysis + +**Run ID:** `20260531t1013-claudes-slayer-48eb0f` +**Date:** 2026-05-31 +**Branch / commit:** `main @ af58457` (post-README defaults bake) +**Compared against prior:** `20260530t0958-claudes-slayer-b278f1` (4/10 today vs 2/10 prior; +museum_1, +museum_7) + +## Configuration + +| field | value | +| -- | -- | +| framework | `claude_sdk_otf` | +| dataset / mode | `livesqlbench` / `one-shot` | +| agent model | `anthropic/claude-opus-4-7` | +| user-sim model | `anthropic/claude-sonnet-4-6` *(unused — one-shot mode has no simulator)* | +| reasoning effort | `high` | +| slayer setup | `on-the-fly` | +| audited gold | on (`use_audited_gold_sql=True`) | +| patience | 500 | +| gold file | `livesqlbench_sqlite_gt_kg_testcases_0528.jsonl` | +| instance ids | `museum_1..10` | +| cluster | 1 worker × 1 actor, `e2-standard-4` | + +## Headline result + +| metric | value | +| -- | -- | +| phase1 pass rate (raw) | **4 / 10 (40%)** | +| phase1 pass rate (after accountability adjustment) | **9 / 10 (90%)** | +| run cost | **$38.78** (agent $38.78, no user-sim) | +| longest task | museum_9 (916 s, $9.84) | +| shortest pass | museum_6 (35 s, $0.36) | + +Adjustment rationale: of the 6 failures, **1 is a genuine agent miss**, **1 is grader-stability noise on a true float tie**, **3 are gold-authoring gaps the supplied metadata does not resolve**, and **1 is an outright gold-sidecar bug (mismatched SQL vs prompt)**. The 40% headline misrepresents the agent's behaviour on this slice; 9/10 reflects what would happen with consistent grader + gold hygiene. + +## Per-task verdicts + +| inst | dur | cost | category | root cause | +| -- | --: | --: | -- | -- | +| museum_1 | 173 s | $1.53 | PASS (audited + original) | — | +| museum_2 | 499 s | $4.05 | FAIL — gold-spec gap | ERF averaged over only 4 sensitivity columns; gold uses all 11. KB 1 enumerates only 4 weights, KB 2 says "etc.", slayer model has `erf = (env+light+temp+humid)/4.0` **pre-baked**. | +| museum_3 | 340 s | $2.02 | FAIL — grader / gold tiebreaker | 951 rows, same CPI values, agent and gold differ only at the four-way 32.7 tie. Gold's `ORDER BY CPI DESC NULLS FIRST` has no secondary key; prompt mandates none. SQLite picks different physical orders inside the tie cluster. | +| museum_4 | 608 s | $3.01 | FAIL — gold-spec gap | CBE = Σ(CPI × BudgetRatio) / N (KB 9). `BudgetRatio` is undefined for a schema with no per-artifact budget *amount* column; gold reinterprets categorical `BudgetAllocStatus` as fractional `adequate/total` — convention absent from KB and column descriptions. | +| museum_5 | 387 s | $3.26 | FAIL — agent miss | KB 7 says "latest reading per showcase" → `MAX(EnvReadRegistry) GROUP BY ShowcaseRef`. Agent never decoded this; picked an arbitrary env reading with `relhumidity=50` everywhere, zeroing `(rh-50)²` → MDR=0 → flag='No' for all rows. Compounded by reacting to repeated `ex_base=0` with column shuffling instead of value inspection. | +| museum_6 | 35 s | $0.36 | PASS (audited + original) | — | +| museum_7 | 586 s | $4.64 | PASS (audited) | new vs prior run | +| museum_8 | 196 s | $1.36 | PASS (audited + original) | — | +| museum_9 | 916 s | $9.84 | FAIL — gold-spec gap | TETL formula encoded correctly per KBs 1/2/7/8/31. 1000× value gap is a **join-path discrepancy**: gold resolves MDR's env reading via `usagerecords → showcase → env LIMIT 1`; agent went `conditionassessments → light → env`. Both FK paths are first-class; no KB or column description prescribes which is canonical. | +| museum_10 | 872 s | $8.69 | FAIL — gold-sidecar bug | Prompt asks for DSD + ERPS + recommendation with "Active" filter. Gold's `external_knowledge: [4, 38, 52]` matches. Gold's `sol_sql` is an unrelated ERF + high_sensitivities query (looks copy-pasted from museum_2). Agent solved the actual problem; gold solves a different one. | + +## Three KB-spec gap deep-dives (was the answer derivable from metadata?) + +For each, the agent had access to: KB entries via `bird-interact-tools` MCP; per-column descriptions seeded from `museum_column_meaning_base.json` and exposed via `mcp__slayer__inspect_model` / `models_summary` / `search`; DDL in `museum_schema.txt`; the slayer model state itself. + +### museum_2 — ERF cardinality + +- **What gold expects:** averaged sensitivity score over all 11 `SensitivityData.*Sensitivity` columns (env, light, temp, humidity, vibra, pollutant, pest, handle, transport, display, storage). +- **Agent consulted:** KB 1 (weights, lists 4), KB 2 (ERF formula, ends with "etc."), `inspect_model('sensitivitydata')`, `models_summary`, two semantic searches. +- **Resolving info in the corpus?** No, and worse: the slayer model has the wrong answer pre-baked. `slayer_models_otf_livesqlbench/museum/models/museum/sensitivitydata.yaml` defines `erf` as `(env_sens_weight + light_sens_weight + temp_sens_weight + humid_sens_weight) / 4.0` with description "Computed as the arithmetic mean of the four encoded sensitivity weights". The agent inheriting this datasource is being actively steered toward 4. Column descriptions on the other 7 (vibra/pollutant/pest/handle/transport/display/storage) read like *operational* sensitivities (packaging, storage, handling) rather than *environmental*; KB 4 (DSD) uses 3 columns, KB 8 (LER) uses 1; KB 2's "etc." is the only opening — and ERF is named "**Environmental** Risk Factor", reinforcing the subset reading. +- **Verdict:** under-specified by KB, *mis-specified* by the slayer encoder. + +### museum_4 — CBE's `BudgetRatio` + +- **What gold expects:** `BudgetRatio_i = adequate_records / total_records` per artifact, derived from `conservationandmaintenance.budgetallocstatus` (a categorical enum: Adequate / Insufficient / Review Required). Also a column the prompt didn't request (`artifact_count`) is silently expected in the output. +- **Agent consulted:** KB 9 (CBE), KB 17 (Crisis), `models_summary`, three counts on `conservationandmaintenance`, three semantic searches. +- **Resolving info in the corpus?** No. KB 9's "BudgetRatio = proportion of total conservation budget allocated to each artifact" implies a monetary amount per artifact — no such column exists. `BudgetAllocStatus` is described as a categorical enum; no column description, no peer KB, no value-illustration KB ever recasts "Adequate" as a 1-in-a-fraction. KB 17 uses `BudgetAllocStatus` only as a categorical threshold predicate. Uniform 1/N (`CBE = SUM(CPI)/N²`) is the most defensible literal reading of KB 9. +- **Verdict:** under-specified gold convention. Would need a sibling KB on `BudgetAllocStatus` mapping it to fractional. + +### museum_9 — MDR's join path + +- **What gold expects:** for MDR (= `ArtAgeYears × ERF × (RelHumidity−50)² × TempC / 100000`), route the environmental reading via `usagerecords.ArtRefUsed → ShowcaseRefUsed → environmentalreadingscore.ShowcaseRef LIMIT 1` (no `ORDER BY`). +- **Agent consulted:** KBs 1, 2, 7, 8, 31; inspected `conditionassessments`, `sensitivitydata`, `artifactscore`, `usagerecords`; explicit semantic search `"artifact environmental reading join showcase usage"`. +- **Resolving info in the corpus?** No. Both FK paths (`conditionassessments → light → env` and `usagerecords → showcase → env`) are first-class FK-declared. `UsageRecords.ShowcaseRefUsed`'s description (*"FK referencing Showcases if a showcase is involved in the usage"*) is actively *weaker* than what gold needs. No KB names a canonical artifact↔env-reading link, no column description hints at "latest reading per showcase", and the gold's own `LIMIT 1` (no `ORDER BY`) is itself non-deterministic — even the gold doesn't reliably pick "the latest". +- **Verdict:** under-specified gold convention. Both paths are equally licensed. + +## Cross-cutting observations + +### 1. SLayer is reinforcing the wrong answer for museum_2 + +The encoder pass that ingests the museum dataset writes pre-computed measures into the slayer model based on KB enumeration during ingest. For museum_2's ERF, this committed to the 4-column reading at encode time. A more diligent agent that *trusts* the slayer model would still get the wrong answer. This is upstream of any prompt change. + +### 2. Trajectory truncation hides inspect-tool responses + +`src/bird_interact_agents/agents/claude_sdk_otf_ainteract/agent.py:370` (and the `claude_sdk_otf` sibling) clips each saved trajectory message to `str(msg)[:500]`. For failure forensics this hides exactly the inspect-model / models-summary responses we need to verify what the agent *actually saw*. Worth raising the clip (or moving inspect responses to a separate per-task log file). + +### 3. Single-bit grader feedback is the largest aggravator + +museum_2 and museum_5 both had the correct hypothesis on the table at one point during their run. They received the verdict `"ex_base returned 0 but expected 1. Please try again."` — no row count, no column diff, no sample mismatch. With one bit per round, they couldn't choose between candidates. A richer grader observation (predicted row count vs expected, column-set diff, first divergent row) would have flipped both. This is independent of any model improvement. + +## Cost breakdown + +Sum across the 10 instances: + +| component | tokens (input / cache_read / cache_create / output) | USD | +| -- | -- | --: | +| Opus agent | totals in `usage` blocks per row | **$38.78** | +| Sonnet user-sim | n/a (one-shot mode skips simulator) | $0.00 | +| **total** | | **$38.78** | + +Per-task: museum_9 ($9.84), museum_10 ($8.69), museum_7 ($4.64), museum_2 ($4.05), museum_5 ($3.26), museum_4 ($3.01), museum_3 ($2.02), museum_1 ($1.53), museum_8 ($1.36), museum_6 ($0.36). + +Note: of the $38.78, **~$31** went to the 6 failing tasks. Of *that*, **~$26** went to the 4 tasks where the failure is not the agent's fault (museum_2, _3, _4, _9, _10). Fixing the KB / gold / grader issues would directly reduce the per-run cost on this slice by ~2/3 without touching the model. + +## Suggested follow-ups + +1. **Audit the museum_10 sidecar row.** `livesqlbench_sqlite_gt_kg_testcases_0528.jsonl` line 167 has `sol_sql` mismatched against the prompt's DSD/ERPS/recommendation ask. Likely a paste error. +2. **Add KB anchors for the three gaps.** A sibling KB on `SensitivityData` enumerating all 11 columns as ERF inputs; a KB on `BudgetAllocStatus` defining its mapping to `BudgetRatio`; a KB on `EnvironmentalReadingScore` naming the canonical `usagerecords → showcase` link with "latest reading per showcase" semantics. +3. **Raise the trajectory truncation cap** in `agent.py:370` (or split inspect responses out) so failure-trace analysis can replay the agent's metadata-view from saved artifacts. +4. **Enrich the grader observation** with `row_count_delta`, `column_set_delta`, and a `first_divergent_row` field; gate behind a flag if you want to keep the strict "ex_base" surface. +5. **Add a deterministic tiebreaker discipline** either in the gold (`, ArtRegistry ASC` as a secondary key when `ORDER BY` has float ties) or in the grader (set-equality within ORDER BY equivalence classes). + +## Provenance + +- Per-task attempt artifacts: `/home/james/Dropbox/SLayer/bird-agents/results/cloud/20260531t1013-claudes-slayer-48eb0f/rows//attempt-1.json` +- Manifest: `/home/james/Dropbox/SLayer/bird-agents/results/cloud/20260531t1013-claudes-slayer-48eb0f/manifest.json` +- Eval summary: `…/eval.json` (`p1=4/10 (0.4)`) +- Museum metadata: `/home/james/Dropbox/SLayer/livesqlbench-base-lite-sqlite/museum/{museum_kb.jsonl, museum_column_meaning_base.json, museum_schema.txt}` +- Gold sidecar: `/home/james/Dropbox/SLayer/livesqlbench-base-lite-sqlite/livesqlbench_sqlite_gt_kg_testcases_0528.jsonl` +- Slayer encoded model: `/home/james/Dropbox/SLayer/bird-agents/slayer_models_otf_livesqlbench/museum/models/museum/sensitivitydata.yaml` (the pre-baked 4-column ERF) diff --git a/analyses/raw/cross_cutting_observations.md b/analyses/raw/cross_cutting_observations.md new file mode 100644 index 00000000..43ae958d --- /dev/null +++ b/analyses/raw/cross_cutting_observations.md @@ -0,0 +1,52 @@ +# Cross-cutting observations (post-2026-05-31 failure analysis) + +These are patterns that span multiple tasks and motivate the infra plan in the Linear issue. + +## 1. SLayer encoder pre-bakes wrong answers (museum_2) + +The on-the-fly encoder writes pre-computed measures into the slayer model based on KB-enumeration during ingest. For museum_2's ERF, this committed to a 4-column reading at encode time (`slayer_models_otf_livesqlbench/museum/models/museum/sensitivitydata.yaml:311-326` defines `erf = (env_sens_weight + light_sens_weight + temp_sens_weight + humid_sens_weight) / 4.0`). An agent that *trusts* the slayer model would still get the wrong answer. Upstream of any prompt change. + +## 2. Trajectory truncation hides inspect-tool responses + +`src/bird_interact_agents/agents/claude_sdk_otf_ainteract/agent.py:~370` (and the `claude_sdk_otf` sibling) clips each saved trajectory message to `str(msg)[:500]`. For failure forensics this hides the inspect-model / models-summary / search responses we need to verify what the agent actually saw. Cross-cuts both `claude_sdk_otf` and `claude_sdk_otf_ainteract`. Worth raising the clip (or moving inspect responses to a separate per-task log file). + +## 3. Single-bit grader feedback is the largest aggravator (museum side) + +museum_2 and museum_5 both had the correct hypothesis on the table at one point and were unable to choose between candidates given `"ex_base returned 0 but expected 1. Please try again."`. A row-count/columns/sample-diff would have flipped both. Independent of any model improvement. + +## 4. User-sim under-disclosure is the largest aggravator (households side) + +Three of four households failures involve the user-sim being **too literal** (households_10 — answered the canonical-label question correctly but never volunteered synonym coverage), **too vague** (households_12 — "use the service criteria" instead of naming `domestichelp` + `total_vehicles>1`), or **too slow** (households_15 — leaked formula only on 5th of 5 asks). + +When the user-sim holds a `critical_ambiguity.sql_snippet` with multiple AND clauses and the agent asks an open-ended "what counts as X?", the sim paraphrases instead of disclosing each clause. Reads like a human deliberately holding back — appropriate for benchmark fidelity, but masks metadata-completeness issues that would otherwise be visible. + +## 5. Slayer search rendering ≠ structured data + +Both motley-slayer 0.6.10 and 0.7.0 have identical `render_column_text` (`slayer/search/render.py:166-192`). It emits only the truncated `Sample values:` string (~210-330 chars per column, top-20ish high-frequency values), NOT the structured `column.sampled_values: List[str]` field (50 items, full noisy list). + +For households_10 specifically, the truncated string DOES contain `Y`, `have`, `brick house`, `apt`, `2014+`, `10-13`, `05-09` (high-frequency). But `after 2014` and `2014 and newer` (lower-frequency at 11-12 rows each) are structured-only — they don't survive truncation. + +If we want full sampled-value visibility for agents, that's a slayer upstream change — or wrappers around `search`/`inspect_model` that expand the structured field. Worth a slayer-side issue. + +## 6. KB hedging vs strict reading + +KB 6 ends with "etc.", KB 10 says "like", KB 7 enumerates only 3 specific values (no hedge). The agent and the audit can both legitimately read these differently: + +- Strict reading (canonical literals only) — KB 7 invites this; KB 6/10 less so +- Extensible reading (canonical + observed synonyms) — KB 6/10 invite this; KB 7 doesn't + +The audit picks extensible; the agent (lacking a prompt rule to extend) picks strict. Both internally consistent. Phase1 fails because they're not the same extension. + +The prompt-level fix (rule-3 symmetric companion) directly addresses this: tell the agent to extend the IN-set when sampled values show variants of KB-named literals. + +## 7. The audit-gold-sql contract is not enforced + +households_10's audit recorded `clause_kind="manual_data_align_reaudit"`, `source="manual_dev1478"`, `justified_by: []`. Per `.claude/skills/_shared/audit-gold-sql.contract.md`, such clauses should be DEFERRED, not encoded. The auditor self-flagged the change as unjustified but still committed it. + +This is an audit-quality regression worth fixing upstream: any non-empty `justified_by: []` should bubble to the audit-time validator and either defer the clause or fail the audit. + +## 8. Quarantine-vs-score multidimensionality + +households_15 was stamped `audit_status="unrecoverable"` because no minimal rewrite makes the gold KB-derivable. Counting it in the headline pass rate misrepresents the agent (no agent could win). But it also isn't useful to filter it out wholesale — it tells us something about the gold-audit pipeline's ability to handle "the original gold is fundamentally underspecified". + +Per the agreed annotation schema, quarantine isn't a separate flag — it's derived from the multi-dimensional scoring (`original_gold_passes`, `audited_gold_passes`, `matches_vagueness`). The reporting layer can choose to exclude or include based on which dimension matters for a given audience. diff --git a/analyses/raw/households_10.md b/analyses/raw/households_10.md new file mode 100644 index 00000000..29c1bc5b --- /dev/null +++ b/analyses/raw/households_10.md @@ -0,0 +1,133 @@ +# households_10 — failure trace + metadata audit + deep dive + +Run: `20260531t1008-claudes-slayer-890419`. Duration 926s. submission_status=wrong_result. + +audit_status: `edited`. + +## User prompt + +> "List the home IDs for all highly mobile homes that are also updated residences, with many vehicles." + +`external_knowledge: [6, 7, 10, 14, 23, 26]`. Three masked terms in `critical_ambiguity` (held by user-sim): + +- **"highly mobile homes"** → `vehicles>2 AND LOWER(Newest_Year) IN ('2005 to 2009', '2010 to 2013', '2012 to 2013', '2014 or newer')` +- **"updated residences"** → `LOWER(Dwelling_Class) IN ('brickwork house', 'apartment') AND LOWER(cablestatus) IN ('avail', 'available', 'yes')` +- **"many vehicles"** → `(Auto+Bike+Motor) > 2` + +## The two golds + +**Original gold** (`sol_sql`): returns the **single region with the most matching households**: +```sql +WITH TargetHouseholds AS ( + SELECT h.housenum, h.locregion FROM households h JOIN ... WHERE (vehicle_sum)>2 + AND LOWER(Newest_Year) IN ('2005 to 2009','2010 to 2013','2012 to 2013','2014 or newer') + AND LOWER(Dwelling_Class) IN ('brickwork house','apartment') + AND LOWER(a.cablestatus) IN ('avail','available','yes')) +SELECT locregion FROM TargetHouseholds GROUP BY locregion ORDER BY COUNT(*) DESC LIMIT 1; +``` + +Prompt asks for "home IDs" but original gold returns a region. Output-shape mismatch. + +**Audited gold** (`audit_status="edited"`): switches to `SELECT DISTINCT h.housenum`, wraps every value comparison in a 4-pass REPLACE chain (`CHAR(9)→space → collapse runs of spaces`) plus CASE folding: + +- `Newest_Year`: folds `'2014 or newer'`, `'2014+'`, `'after 2014'`, `'2014 and newer'`; `'2010 to 2013'`, `'2010-2013'`, `'10-13'`, `'2012 to 2013'`; `'2005 to 2009'`, `'2005-2009'`, `'05-09'`. +- `Dwelling_Class`: `IN ('brickwork house','brick house','apartment','apt')`. +- `cablestatus`: `IN ('avail','available','yes','y','have')`. + +**Audit's `changes` block records** `clause_kind="manual_data_align_reaudit"`, `source="manual_dev1478"`, **`justified_by: []`**. Per `.claude/skills/_shared/audit-gold-sql.contract.md`, clauses with empty `justified_by` should be DEFERRED, not encoded. The audit violated its own contract. + +## What KBs 6/7/10 actually say + +``` +KB 6 Dwelling Type: "Values based on schema include 'Brickwork house', 'Apartment', 'Condominium', etc." +KB 7 Cable TV Status: "Values indicating availability, based on schema, are 'avail', 'available', and 'yes'." +KB 10 Vehicle Year Range:"Text ranges like '1995 to 1999', '2005 to 2009', or '2010 to 2013'." +``` + +KBs 6, 10 hedge (`"include … etc."`, `"like"`). KB 7 does not. None mention `'brick house'`, `'apt'`, `'y'`, `'have'`, `'2014+'`, `'after 2014'`, `'2010-2013'`, `'10-13'`, `'05-09'`. + +## Slayer model exposes the noise + +`slayer_models_otf/households/models/households/amenities.yaml` — `cablestatus.sampled_values` (123 distinct values; structured top-50): +``` +Available, Not available, available, AVAILABLE, yes, Y, not available, Yes, avail, have, +NOT AVAILABLE, unavailable, not avail, N, no, No, dont have, Y, YEs, YES, Avail, y, ... +``` + +`properties.yaml` — `dwelling_specs__Dwelling_Class.sampled_values` (116 distinct): +``` +Brickwork house, Apartment, BRICKWORK HOUSE, Brickwork House, brickwork house, +brick house, Brickwork house , apartment, Brickwork house, APARTMENT, +Shack (with floor finish), apt, Apt, Unfinished brickwork, BRICK HOUSE, ... +``` + +`Y`/`y`/`have`/`brick house`/`apt`/`2014+` are all there. + +## Did the agent look? + +Yes. Trajectory items 37, 39, 40, 41: `inspect_model('amenities')`, `('transportation_assets')`, `('properties')`, `('households')`. Full response bodies truncated to 500 chars in saved trajectory (`agent.py:~370` clip), but agent saw them live. Agent then did distinct-value queries via `mcp__slayer__query` at items 95 (`dwelling_specs__Dwelling_Class like '%ondo%' or '%apart%'`), 136 (`Newest_Year like '2005%' or '2010%'`), 174 (cablestatus == 'Available' count), 176 (Dwelling_Class in canonical list count). Actively exploring noise space. + +## Slayer search rendering — what EntityHit.text shows + +Both motley-slayer 0.6.10 and 0.7.0 have identical `render_column_text`. It emits only the truncated `Sample values:` string (NOT the structured `sampled_values: List[str]`). + +Empirical check on the truncated `Sample values:` for each filter column: + +| column | EntityHit.text gives the agent | +| -- | -- | +| `amenities.cablestatus` | `Y`, `y`, `have`, `dont have`, `avail` — **all visible** in top-20 | +| `properties.dwelling_specs__Dwelling_Class` | `brick house`, `apt`, `Apt`, `BRICK HOUSE`, `Shack` — **all visible** in top-20 | +| `transportation_assets.vehicleinventory__Newest_Year` | `2014+`, `2010-2013`, `10-13`, `05-09`, `2005-2009` — visible; `after 2014`, `2014 and newer` — structured-only (hidden by truncation); `'2012 to 2013'` — **not in data at all**, original gold buggy on that literal | + +## Did the agent use `search(entities=[...])`? + +Looking at trajectory: **all 7 `mcp__slayer__search` calls used `question=`** (semantic free-text). **Zero `search(entities=[...])` calls.** The agent never used the focused per-column read primitive. + +Reason: nothing in the `claude_sdk_otf_ainteract` prompt or any tool description points the agent at `search(entities=[...])` as canonical column-reading. The `_host_discovery_playbook.py:43-67` instruction is scoped to host/join discovery, not filter-value enrichment. + +## What the agent submitted + +```sql +WHERE (Auto+Bike+Motor) > 2 + AND LOWER(TRIM(Newest_Year)) IN ('2005 to 2009','2010 to 2013','2014 or newer') + AND LOWER(TRIM(cablestatus)) IN ('available','avail','yes') + AND LOWER(TRIM(Dwelling_Class)) IN ('brickwork house','apartment') +``` + +LOWER+TRIM handles case + leading/trailing whitespace but NOT internal whitespace and NOT synonyms. + +## User-sim role + +- `[35]`: listed only `'2005 to 2009', '2010 to 2013', '2014 or newer'`. Never volunteered synonyms. Dropped `'2012 to 2013'` from gold's mask (string doesn't exist in data anyway). +- `[105]`: refused — *"I don't have access to expected row counts or specific example values"*. +- `[117]`: canonical-only confirmation on case sensitivity question. + +Sim not adversarial, but strictly literal. When asked for ground-truth row counts (the one signal that would have surfaced the gap), refused. + +## Predicate row-count verification (against live `households.sqlite`) + +| query | DISTINCT housenum | +| -- | --: | +| KB-faithful canonical literals (agent-style) | **153** | +| Synonym-expanded predicate (approximation of audit) | **223** | +| Audited gold (exact, including internal-whitespace collapse) | **233** | + +Agent returned 153. Audit expected 233. 80-row gap: ~60 from `Y`/`y` cable, ~30 from `2014+`/`after 2014`/`2014 and newer`/`'10-13'`/`'2010-2013'`/`'05-09'`/`'2005-2009'` synonyms, ~30 from `brick house`/`apt` Dwelling_Class synonyms (overlap reduces union to 80). + +## Revised verdict + +Mixed: + +1. **The answer IS derivable in principle.** Truncated `Sample values:` strings expose every variant the audit added (with two exceptions: `after 2014`, `2014 and newer`). +2. **KB hedges legitimately invite expansion** — KB 6 ends in "etc.", KB 10 says "like". KB 7 does not hedge, but `'y'`/`'have'` are still visible in sampled values. +3. **Agent inspected the right tables** and was actively exploring noise. Had the evidence. Settled on canonical-only reading after user-sim confirmed at idx 35. +4. **Audit DID violate its contract** (`justified_by: []` clauses should defer, not encode), picks one specific expansion set among many plausible. +5. **User-sim's literalness blocked recovery.** When agent asked for a sample housenum at idx 105, sim declined. + +Closest framing: **agent miss, but only barely** — KB hedges, sim's literal answer, and data noise put four reasonable readings on the table (canonical-only, lower+trim, lower+trim+light-synonym, lower+trim+full-synonym-expansion). Audit picks one; agent picks another. Both internally consistent. + +Prompt-level fixes that would close this: +1. Symmetric companion to rule-3 line 114-116: *"if sampled values show variants of KB-named literals, extend the IN-set."* +2. Promote `search(entities=[...])` from host-discovery section to general column-reading guidance. + +Provenance: trajectory items 35, 37, 39, 40, 41, 95, 105, 117, 136, 174, 176; `slayer_models_otf/households/models/households/{amenities.yaml, properties.yaml, transportation_assets.yaml}#sampled,sampled_values`; `households_kb.jsonl#6,7,10`; audited gold (`audit_status: edited`, `changes[0].justified_by: []`). diff --git a/analyses/raw/households_12.md b/analyses/raw/households_12.md new file mode 100644 index 00000000..34b04fa6 --- /dev/null +++ b/analyses/raw/households_12.md @@ -0,0 +1,66 @@ +# households_12 — failure trace + metadata audit + +Run: `20260531t1008-claudes-slayer-890419`. Duration 1238s. submission_status=wrong_result. + +audit_status: `edited`. + +## User query + masked terms + +> "Which dwelling type has the highest average prosperity score among independent households, and how many vehicles do those households own?" + +Masked term resolutions: +- `independent households` → `(s.serviceref IS NULL OR (LOWER(s.domestichelp)='no domestic workers' AND s.socsupport='No')) AND total_vehicles > 1` +- `prosperity score` → `AVG(residentcount * income_score)` where `income_score` is the **ordinal rank over the data's actual R$ brackets** (0..11). Original `sol_sql` used `'low income'..'very high income'` labels — which appear **zero times** in the data. +- `dwelling type` (NOT masked) → canonical-class CASE bucket. + +## Auditor's view + +`audit_status=edited`, `reasoning_summary`: +> "income_score = ordinal rank over the data's actual R$ brackets (the gold's 'low income' labels appear 0 times in the data). Dwelling type grouped by canonical category (synonyms folded; 'other' catch-all excluded as a non-type). Output (dwelling_class, SUM total_vehicles)." + +Audit hardcoded a folding map: `('brickwork house','brick house') → 'brickwork house'`, `('apartment','apt') → 'apartment'`, `('condominium')`, `'shack%' → 'shack'`, `('unfinished brickwork','unfinished') → 'unfinished brickwork'`, else `'other'`. Rewrote noise-normalisation to a 4-pass `REPLACE(REPLACE(REPLACE(REPLACE(x, CHAR(9), ' '), ' ', '<>'), '><', ''), '<>', ' '))` (collapses tabs + runs of internal spaces). + +## ask_user + +6 calls. User-sim disambiguated: +- `[20]` (prosperity): *"residents × income score; brackets mapped to numeric values ('has no income'=0, 'R$ 440 or less'=1, … 'more than R$ 26,400'=11)"* — **exact gold formula given.** +- `[34]` (vehicles + indep): *"total = SUM(Auto+Bike+Motor) across households of that dwelling type … don't have a specific cutoff for high income — just use all households that qualify as independent based on the service criteria."* — User-sim **refused to disclose** the `domestichelp='no domestic workers'` and `total_vehicles>1` parts of the mask. +- `[106]/[147]/[233]`: *"'Brickwork house' and 'Brick house' should be treated as the same category … dwelling_class string should be 'brickwork house' (lowercase) … exclude 'Other'/NULL."* +- `[180]` (asking for the expected number): *"Sorry, I don't have that specific information."* + +## Submitted vs audited gold + +Final submitted SQL did fold case/whitespace and merged `brickwork`/`brick house` (LIKE `'%brick%'`). Diverged on the independent-household predicate: + +- **Agent WHERE**: `CASE WHEN LOWER(TRIM(service_types.socsupport))='no' THEN 1 ELSE 0 END = 1` and `dwelling_class <> 'other'`. **No `domestichelp='no domestic workers'`**, **no `serviceref IS NULL` fallback**, **no `total_vehicles > 1` filter**. +- **Gold WHERE**: all three present. + +Predicted `('brickwork house', 2106)` vs gold `('brickwork house', 1373)` — same dwelling class (alias merge worked), but vehicle SUM 53% high because agent included (a) single-vehicle households and (b) households with `domestichelp` other than `'no domestic workers'`. + +## Decision-point lock-in + +`[34]` — agent explicitly asked whether to use a high-income cutoff; user-sim said "no specific cutoff, just use the service criteria." Agent took "service criteria" to mean `socsupport='No'` only. Never tested adding `domestichelp` or `total_vehicles>1` floor. Asked again at `[180]/[232]` for expected count, got refused, re-submitted same vehicle filter. + +## Metadata audit + +**KB 42 ("Economically Independent Household")**: *"high Income Classification and social support status of 'No'."* — no `domestichelp` mention, no `total_vehicles>1`, and "high Income" itself is broken (KB 2 labels don't exist in data). + +KB 14 (Vehicle Ownership Index = Auto+Bike+Motor) and KB 18 (Mobility Score) covered the vehicle math. KB 32 (Register New Household) irrelevant. + +`households_column_meaning_base.json` for `properties.dwelling_specs.Dwelling_Class`: *"Ex. Brickwork house, Apartment"* — no alias list, no mention of `Brick house`/`Shack`/`Condominium`/`Unfinished brickwork`. + +**SLayer model `properties.yaml` DID pre-bake `dwelling_type_clean`** with sampled values `Brickwork house, Apartment, BRICKWORK HOUSE, brick house, BRICK HOUSE, "brickwork\thouse", Unfinished brickwork, Shack` enumerated — and a `dwelling_type_score` CASE. The alias problem was visible in the slayer cache. + +**Crucially, no KB defines `total_vehicles > 1`** as part of "independent" and **no KB mentions `domestichelp='no domestic workers'`**. These come only from the masked `sql_snippet` in `critical_ambiguity`. + +## Verdict + +**Gold-spec gap (primary) + user-sim under-disclosure (compounding).** + +- `total_vehicles > 1` and `domestichelp = 'no domestic workers'` are part of the mask but absent from KB 42, column metadata, and any KB hint. They survive only inside `critical_ambiguity.sql_snippet` which the agent never sees. +- User-sim, holding that snippet, declined to disclose either when directly asked. +- Agent did the dwelling-alias normalisation correctly (KB-encoded in slayer). Not primarily a noisy-data issue. + +Remediation: either expand KB 42 to spell out the full independent-household predicate, or instruct the user-sim to volunteer every clause of `critical_ambiguity.sql_snippet` when asked an open-ended "what exactly counts as X" question. + +Provenance: trajectory items 20, 34, 106, 147, 180, 233; `households_kb.jsonl#42`; `slayer_models_otf/households/models/households/properties.yaml` (dwelling_type_clean pre-baked). diff --git a/analyses/raw/households_15.md b/analyses/raw/households_15.md new file mode 100644 index 00000000..c4cc9f05 --- /dev/null +++ b/analyses/raw/households_15.md @@ -0,0 +1,67 @@ +# households_15 — failure trace + metadata audit + +Run: `20260531t1008-claudes-slayer-890419`. Duration 618s. submission_status=wrong_result. + +audit_status: **`unrecoverable`** — auditor explicitly stamped this. + +## User query + masked terms + +> "Find all highly supported homes that are also financially secure, listing their unique house codes." + +Three flagged ambiguities (mini_interact.jsonl, instance `households_15`): +- `"highly supported homes"` (knowledge_linking, mask=true) → gold snippet: `((CASE WHEN domestichelp='none' THEN 0 ELSE 3 END)+(CASE WHEN socsupport='No' THEN 0 ELSE 4 END))>2` +- `"financially secure"` (knowledge_linking, mask=true) → gold snippet: weighted index `0.4*income_score + 0.4*(Expend_Coeff/income_score) + 0.2*tenure_score > 0.7 AND Expend_Coeff/income_score < 1.5` +- `"unique house codes"` (intent, mask=false) → `SELECT DISTINCT housenum` + +External KBs declared: `[9, 11, 24, 25]` — Social Support Status, Household Density, Supported Household, Crowded Household. **None define "financially secure"**, and KB 11/25 (density) are entirely unused by the audited gold. + +## Auditor's view — why unrecoverable + +`reasoning_summary`: +> *"Gold's predicate uses an unjustified density>4 'AtRisk' bucketing and ignores the labeled 'highly supported' + 'financially secure' definitions entirely; output is wrong (returns one locregion, user asked for DISTINCT housenum). Audited SQL uses the two labeled-ambiguity sql_snippets verbatim and projects DISTINCT housenum. Notes: (a) labeled 'financially secure' references income_score/tenure_score variables; we use the same 1-6 income mapping as in households_11 as scaffolding, (b) actual result is 0 rows…"* + +Two `changes`: +1. Whole outer SELECT replaced because original gold returns `locregion` (regional ratio) rather than `housenum` — output-shape bug. +2. The 1-6 `income_score` mapping kept as "scaffolding" but explicitly flagged as **not in any source** — same caveat as households_11/12. The "financially secure" coefficients (0.4 / 0.4 / 0.2), threshold 0.7, ratio cap 1.5, tenure 3/1/0 mapping, and "highly supported" weights 3/4 with threshold >2 are pulled directly from the labeled_ambiguity `sql_snippet` — they exist nowhere in the KB. + +## ask_user + +5 calls (turns 36, 39, 42, 100, 135). Sim never gave numeric thresholds for the support score. Only on the last ask (turn 136) leaked the exact financially-secure formula: +> *"0.4 * income_score + 0.4 * (Expend_Coeff / income_score) + 0.2 * tenure_score, threshold > 0.7, ratio < 1.5, income_score > 0"* + +For "highly supported" sim gave the OR-rule (`domestic help present OR socsupport='Yes'`) — which is what the agent encoded — not the gold's weighted-score `(3 or 0) + (4 or 0) > 2`, which happens to be ≡ OR-of-the-two only in binary sense. Gold also uses `LOWER(COALESCE(domestichelp,''))='none'` rather than the agent's `LIKE 'yes%'`. + +## Submitted vs audited gold + +Agent: `LEFT JOIN service_types`, filters `Tenure_Type IN ('owned','own')` AND `Income_Bracket` ordinal ≥5 AND `Expend_Coeff/income ≤12`, support=OR-of-(domestichelp LIKE 'yes%', socsupport='yes'), `GROUP BY housenum` → 122 rows. + +Audited gold: weighted SEI > 0.7 AND ratio < 1.5 AND income_score > 0 AND `((dh!='none')?3:0) + ((soc!='No')?4:0) > 2` → 0 rows. + +Agent never built the weighted index; invented its own tenure filter (`OWNED` only) and an Income_Bracket ordinal mapping with R$ bands that don't match any KB-mentioned label. + +## Decision-point lock-in + +Agent fixated on KB 24 (binary socsupport='Yes') interpretation early, then on the last attempt swapped to `domestichelp LIKE 'yes%' OR socsupport='yes'` per sim's OR-confirmation. For "financially secure" never adopted the user-sim's leaked SEI formula at turn 136 — submitted SQL still uses an income-only conjunction. + +## Metadata sufficiency + +**Resolving information NOT present.** KB 28 ("Economically Stable Household") defines the concept narratively — *"high Socioeconomic Index AND low Expenditure Ratio"* — but the **coefficients (0.4/0.4/0.2), threshold (0.7), ratio cap (1.5), income 1-6 ordinal, and tenure 3/1/0 mapping exist only in the gold `sql_snippet`**. The KB chain for "Socioeconomic Index" (KB 12) and "Expenditure Ratio" (KB 19) is not in `external_knowledge=[9,11,24,25]`. KB 16 ("Service Support Score") says *"weighted score combining domestic help availability and social assistance participation"* with no weights. + +Auditor explicitly flags income mapping as "not in any source but is required scaffolding." Same pattern the project memory `households_gold_reaudit` calls out as deliberately noisy. + +## Audit isn't a real repair + +It's a **partial rewrite**: +- Kept gold's literal `sql_snippet` for both masked terms (verbatim, including unsourced coefficients) +- Fixed output shape (DISTINCT housenum vs locregion) +- Replaced unrelated density>4 AtRisk machinery + +Audit doesn't repair derivability — it just makes gold match labeled `sql_snippet`, which itself isn't derivable from KB. Audited gold returns 0 rows on live DB ("few households simultaneously match both predicates"). + +## Verdict + +**Gold/audit bug — known unrecoverable.** Auditor itself stamped `audit_status="unrecoverable"` and acknowledged the kept scaffolding has no KB justification. No reasoning agent can reproduce the exact coefficients 0.4/0.4/0.2/0.7/1.5 or the 1-6 income mapping from KBs 9/11/24/25 — and KB 28 (the actual relevant concept) isn't even in `external_knowledge`. User-sim did leak the formula on turn 136, but only after 4 failed asks, and the agent didn't pivot. + +Remediation: remove `households_15` from default eval set (or quarantine into a separate "known-unrecoverable" bucket). The labeled `sql_snippet` is the only source-of-truth for the SEI weights/thresholds and is not reachable from any KB or column metadata. Same class as `households_11/12` per the audit's own cross-reference. + +Provenance: `results/cloud/20260531t1008-claudes-slayer-890419/rows/households_15/attempt-1.json`; audited gold (`audit_status: unrecoverable`); `households_kb.jsonl` (KBs 9/11/16/21/24/25/28/42). diff --git a/analyses/raw/households_2.md b/analyses/raw/households_2.md new file mode 100644 index 00000000..94f8d5f9 --- /dev/null +++ b/analyses/raw/households_2.md @@ -0,0 +1,60 @@ +# households_2 — failure trace + metadata audit + +Run: `20260531t1008-claudes-slayer-890419` (claude_sdk_otf_ainteract, mini-interact, a-interact, Opus + Sonnet sim, reasoning_effort=high). Duration 908s. submission_status=wrong_result. + +audit_status: `clean` (audited == original). + +## Task + +amb_user_query: **"Where is the highest concentration of high-risk families?"** + +`critical_ambiguity` (held by user-sim, not visible to agent): +- `"highest concentration"` → `ORDER BY (high_risk_count / total_count) DESC LIMIT 1` (ratio per region, top 1). +- `"high-risk families"` → `s.socsupport='Yes' AND (Auto+Bike+Motor)>2 AND Newest_Year IN ('2010 to 2013', '2012 To 2013', '2014 or newer')`. +- `"Where"` (non-mask) → `SELECT locregion`. + +## ask_user + +5 mentions; 4 actual `mcp__bird-interact-tools__ask_user` calls at `trajectory[18, 99, 133, 176]`. + +- `[19]`: high-risk = `socsupport='Yes' + vehicles>2 + "2010 to 2013" or newer categories` (verbatim — no mention of typo'd `'2012 To 2013'`). +- `[100]`: **"Please use strict exact matches without normalizing case or trimming for the filtering con[ditions]"** — gold-aligned guidance. +- `[134]`: paraphrase of [19]. +- `[177]`: declined formatting questions; reiterated answer is a region name. + +## Submitted vs gold + +Two differences: + +(a) **Locregion normalization.** Gold groups by raw `households.locregion`. Agent submitted a `locregion_canonical` derived dim that `LOWER+TRIM`s, collapses runs of spaces (`' '→' '`), and CASE-maps to canonical capitalization. This collapses dirty `'Riacho Fundo I'` (two spaces, 2 households, 1 high-risk → 0.5) INTO `'Riacho Fundo I'` (132 households, 1 high-risk → 0.0076). After normalization, top region is `Riacho Fundo II` (1/100=0.01). + +(b) **Newest_Year IN-list.** Gold uses `IN ('2010 to 2013', '2012 To 2013', '2014 or newer')`. Agent uses `IN ('2010 to 2013', '2014 or newer')` after `LOWER+TRIM`. Non-load-bearing here — even without `'2012 To 2013'`, raw-locregion grouping still yields `Riacho Fundo I = 0.5` as the gold answer (verified by direct SQLite query). + +## Decision-point lock-in + +**Trajectory `[125] → [127]`.** At `[125]` agent ran a query grouped by raw `households.locregion` and got back exactly the gold result: `Riacho Fundo I | total=2 | hrc=1 | share=0.500` (top of list). ThinkingBlock signature hidden, but at `[127]` agent immediately ran a NEW query grouped by `locregion_clean` and got `riacho fundo i | 132 | 1 | 0.008`. At `[130]` submitted the normalized version. Final submitted query at `[202]` grouped by `locregion_canonical` and returned `Riacho Fundo II`. + +Final text at `[205]` explicitly: +> "Top region: **Riacho Fundo I** (1 out of 2 = 50% with strict matching, or 0.76% out of 132)" + +Agent saw both candidates and chose to submit the wrong one anyway. + +## Metadata sufficiency + +**Resolving information was present** in both metadata AND user-sim: + +- `slayer_models_otf/households/models/households/transportation_assets.yaml:254-255` Newest_Year description: *"EX. 2012 To 2013, Not applicable, 2014 or newer"* — literally names the missing `'2012 To 2013'` value. `sampled_values` lines 268-318 list `'2010 to 2013'`, `'2010 To 2013'`, `'2014 or newer'`, `'2014 Or Newer'` as distinct sampled rows. +- `slayer_models_otf/households/models/households/households.yaml:76` locregion `sampled_values` contains `Riacho Fundo I` (two spaces) as a distinct value alongside `Riacho Fundo I` (line 43) — dirty-data signal. +- KBs 23, 24, 9, 10, 14 describe Mobile and Supported Household conceptually but don't pin the `>2` threshold or year-list. The user-sim filled the threshold gap at `[19]` but didn't mention typo'd `'2012 To 2013'`. + +User-sim at `[100]` gave unambiguous "strict exact matches, no normalization" instruction. + +## Verdict + +**`agent_miss`** — the only one in the households batch. User-sim explicitly said not to normalize. Slayer model exposed `Riacho Fundo I` as distinct sampled value. Agent's idx 125 raw-locregion query returned the gold answer. Agent then overruled both signals. + +Failure class: data-hygiene prior overrode an explicit user-sim instruction and the sampled-values caveat. + +Remediation: agent-side prompt nudge — "If the user explicitly says 'use strict matches / do not normalize', that overrides your dirty-data priors and the slayer model's normalised columns. Quote the user instruction in your final reasoning." + +Provenance: `results/cloud/20260531t1008-claudes-slayer-890419/rows/households_2/attempt-1.json` (trajectory items 19, 100, 125, 127, 130, 202, 205); `slayer_models_otf/households/models/households/transportation_assets.yaml:254-318`; `slayer_models_otf/households/models/households/households.yaml:26-88`; `households_kb.jsonl` (KB 9, 10, 14, 23, 24); audited gold (`audit_status: clean`). diff --git a/analyses/raw/museum_10.md b/analyses/raw/museum_10.md new file mode 100644 index 00000000..6af63ee2 --- /dev/null +++ b/analyses/raw/museum_10.md @@ -0,0 +1,63 @@ +# museum_10 — failure trace + +Run: `20260531t1013-claudes-slayer-48eb0f`. Duration 872s. submission_status=wrong_result. + +This is the most concerning failure of the batch — but the verdict is unexpected. + +## User query (verbatim from `livesqlbench_data_sqlite.jsonl` line 167) + +> "Generate a rotation schedule using the standard ERPS formula. Show me each artifact's ID, name, material type, current display duration, their Display Safety Duration (DSD) limit, the calculated ERPS and a clear recommendation ('Immediate Rotation' or 'Monitor') based on ERPS Decision Threshold. Only include artifacts currently marked as 'Active' in the rotation schedule." + +The prompt **explicitly names**: DSD, ERPS, "recommendation", "Immediate Rotation"/"Monitor", "ERPS Decision Threshold", and the `Active` filter. + +## Gold's external_knowledge + +`external_knowledge: [4, 38, 52]`: +- **KB 4 (DSD)**: `DSD = BaseDuration × (10 − LightSensWeight)(10 − TempSensWeight)(10 − HumidSensWeight) / 1000` +- **KB 38 (ERPS)**: `ERPS = (DSD − DisplayDurMonths) × (LER + 1) × (CPI + 1) ÷ 100` +- **KB 52 (ERPS Decision Threshold)**: `When ERPS < 0, trigger 'Immediate Rotation'; otherwise 'Monitor'` + +These match the prompt 1:1. + +## Agent's KB retrieval + +`get_all_external_knowledge_names` at `trajectory[3]`, then targeted: + +- `trajectory[6]` ERPS, `[8]` ERPS Decision Threshold, `[10]` DSD — **exactly the three gold KBs.** +- `[12]` Sensitivity Weight Values, `[14]` LER, `[16]` CPI — the three sub-formulas DSD/ERPS transitively need (KB 1, 8, 0). Required, not scope creep. +- `[21]` `ArtifactsCore.ConserveStatus`, `[23]` `SensitivityData.LightSensitivity`, `[25]` `Exhibition Rotation Candidate` — column/concept clarifications. +- `[194]` bulk `get_all_knowledge_definitions` late while debugging. + +**No spurious KB fetched.** No AVS, MDR, TETL, MAP, ECI, etc. + +## Agent's encoding sequence + +ThinkingBlocks redacted (`thinking=''` at `[1]`). Action sequence is unambiguous: + +After pulling ERPS → ERPS Decision Threshold → DSD at `[6,8,10]`, agent wires those weights into SLayer via `edit_model`: +- `[37]` `light_sens_weight`/`temp_sens_weight`/`humid_sens_weight` CASE expressions (matches KB 1) +- `[39]` `conserve_status_num` for CPI's `ConserveStatus` factor +- `[40]` `light_lux_x_visible_exp` for LER + +Final submitted SQL projects exactly: `artifact_id, name, mattype, displaydurmonths, DSD, ERPS, CASE WHEN ERPS<0 THEN 'Immediate Rotation' ELSE 'Monitor' END`. Filtered on `displayrotatesched = 'active'`. Verbatim the prompt's column list. + +## Gold sidecar contents + +`livesqlbench_sqlite_gt_kg_testcases_0528.jsonl` line 167 — `sol_sql`: +- Returns 4 columns: `ArtRegistry, ArtName, ERF, high_sensitivities` +- 323 rows +- Filters: ERF > 7 +- Has nothing to do with DSD/ERPS/decision-threshold +- Doesn't mention "Active", "Immediate Rotation", or "Monitor" + +**This is the same query shape as museum_2's gold** (ERF + high_sensitivities). Looks copy-pasted. + +## Verdict + +**Not the agent's fault — gold-file bug.** The agent solved the prompt correctly with the gold's own listed `external_knowledge` (KBs 4/38/52). The `sol_sql` is mismatched against both the prompt AND the `external_knowledge` field. Either the prompt and SQL were swapped at gold-build time, or the row's `sol_sql` was copy-pasted from museum_2 and only `external_knowledge` got updated. + +872s wallclock reflects real SLayer model construction for a legitimately hard 7-column compound calculation — not confusion. + +Remediation target: gold sidecar row 167 — replace `sol_sql` with a DSD+ERPS+recommendation query, OR replace the prompt to match the ERF query. + +Provenance: `results/cloud/20260531t1013-claudes-slayer-48eb0f/rows/museum_10/attempt-1.json`; `livesqlbench_data_sqlite.jsonl` line 167; `livesqlbench_sqlite_gt_kg_testcases_0528.jsonl` line 167 (the buggy row); `museum_kb.jsonl` (KBs 4, 38, 52). diff --git a/analyses/raw/museum_2.md b/analyses/raw/museum_2.md new file mode 100644 index 00000000..d2a69fad --- /dev/null +++ b/analyses/raw/museum_2.md @@ -0,0 +1,59 @@ +# museum_2 — failure trace + metadata audit + +Run: `20260531t1013-claudes-slayer-48eb0f` (claude_sdk_otf, livesqlbench, one-shot, Opus, reasoning_effort=high). Duration 499s. submission_status=wrong_result. + +## Task + +amb_user_query: identify artifacts with high Environmental Risk Factor (ERF) — show ID, name, ERF score, and a JSON summary of all 'High' sensitivity ratings; threshold ERF > 7. + +external_knowledge: `[1, 2]` (sensitivity weights, ERF formula). + +## Trajectory trace + +KB list discovery (`get_all_external_knowledge_names`) returned all 12 KB names at `trajectory[5]`. Agent pulled exactly KB 2 (ERF) and KB 1 (Sensitivity Weights) at `trajectory[6,8]`. CPI (KB 0) chased much later as a distraction at `trajectory[59]`, but didn't feed into submission. + +Reasoning oscillation between two readings of ERF cardinality: + +- `trajectory[17]` — first encoding uses 4 columns (env/light/temp/humidity), divisor 4.0 (matches KB 1's enumeration). +- `trajectory[27]` — after submit-1 fails, agent correctly diagnoses: *"The test failed. Let me reconsider — the KB says 'EnvSensitivity, LightSensitivity, TempSensitivity, etc' suggesting ALL sensitivities, not just 4."* Re-encodes with 11 sensitivities at `trajectory[32]`. Also fails. +- `trajectory[63]` — flips back: *"ERF uses the 4 named sensitivities (per KB1's explicit weight mapping), but the JSON summary lists ALL 11 sensitivity fields rated 'High'"*. + +Final submission (submit-9) lands on 4-sensitivity encoding. Remaining attempts varied JSON whitespace and `TRIM(artref)` cosmetics. Predicted 169 rows; gold 100. + +## Decision-point lock-in + +`trajectory[63]`. After both 4-sens and 11-sens hypotheses failed `ex_base`, the agent received only `"ex_base returned 0 but expected 1. Please try again."` — no row count, no diff, no column delta. With single-bit feedback the agent concluded both must be partially right and started splitting hairs about JSON formatting and TRIM rather than re-examining the divisor. + +## Metadata audit + +Did the `museum_column_meaning_base.json` SensitivityData entries hint that ALL 11 columns are equally-weighted ERF inputs? + +**No.** Each `museum|SensitivityData|*Sensitivity` entry follows the same template (" describing , possible values: Low/Medium/High"). The other 7 (Vibra/Pollutant/Pest/Handle/Transport/Display/Storage) are described operationally (packaging, storage, handling) rather than environmentally. Nothing labels any as ERF inputs. Nothing distinguishes the 4 "main" from the other 7. + +Did the agent inspect SensitivityData via slayer? **Yes** — at `trajectory[14]` `inspect_model(model_name="sensitivitydata", sections=["columns"])`; and `models_summary(datasource_name="museum")` at `trajectory[10]`. Plus two `search` calls at `trajectory[35, 58]` with `"Environmental Risk Factor ERF sensitivities"` and `"sensitivity attributes list count weight averaging"`. Both surfaced only KBs 1 and 2 (already known). + +**Critical finding** — the slayer-encoded model has the wrong answer **pre-baked**: + +`slayer_models_otf_livesqlbench/museum/models/museum/sensitivitydata.yaml:311-326`: +``` +erf: + formula: (env_sens_weight + light_sens_weight + temp_sens_weight + humid_sens_weight) / 4.0 + description: "Computed as the arithmetic mean of the four encoded sensitivity weights (env, light, temp, humid)…" +``` + +The slayer encoder committed to the 4-column reading during ingestion (driven by KB 1's enumeration). The agent inheriting this datasource sees ERF defined on 4 columns; an agent that trusts the slayer model would get the wrong answer. + +Any other KB pinning cardinality? **No.** All 10 sensitivity-mentioning KBs in `museum_kb.jsonl` (1, 2, 3, 4, 8, 14, 20, 23, 24, 53) name SUBSETS: KB 1 enumerates 4 weight mappings; KB 4 (DSD) uses 3; KB 8 (LER) uses 1; KB 14 references "SensitivityData values" generically; KB 20 uses 1; KB 23/24 are per-axis value illustrations. KB 2's "EnvSensitivity, LightSensitivity, TempSensitivity, etc" is the only "etc" — deliberately under-specified. + +## Verdict + +Genuinely underspecified by the metadata as shipped, and *mis-specified toward the 4-column answer*. The agent did consult the right sources. Every signal points to 4: + +- KBs 1, 4, 8 use sensitivity subsets; KB 2's "etc." is the only opening, and lists those same 4 by name first. +- column_meaning makes the other 7 look like operational (packaging/storage/handling), not environmental. +- ERF is named "**Environmental** Risk Factor", reinforcing the subset reading. +- slayer model has erf=4-col pre-baked. + +Failure class: **gold-spec gap, compounded by slayer encoder pre-baking the wrong answer** AND **single-bit grader feedback preventing disambiguation**. + +Provenance: `results/cloud/20260531t1013-claudes-slayer-48eb0f/rows/museum_2/attempt-1.json` (trajectory items 5, 7, 9, 17, 27, 32, 63); `livesqlbench-base-lite-sqlite/museum/museum_kb.jsonl` (KBs 1, 2). diff --git a/analyses/raw/museum_3.md b/analyses/raw/museum_3.md new file mode 100644 index 00000000..c8a28a1b --- /dev/null +++ b/analyses/raw/museum_3.md @@ -0,0 +1,52 @@ +# museum_3 — failure trace + +Run: `20260531t1013-claudes-slayer-48eb0f`. Duration 340s. submission_status=wrong_result. + +## Task + +Rank all artifacts by Conservation Priority Index (CPI). 4 columns: ArtRegistry, ArtName, ArtDynasty, CPI. Prompt: "rank by CPI" — no tiebreaker, no NULLS placement specified. + +external_knowledge: `[]` in task row; KB 0 (CPI formula) lives in the sidecar. + +## Output diff + +Both queries return 951 rows. Row 1 in both: `ART58660 / Health Painting / Ming / 34.2` (unique max). At the four-way `32.7` tie: + +- pred: `ART11605, ART32997, ART36966, ART42439` (alphabetical by `artref` — accident of agent's CTE) +- gold: `ART32997, ART42439, ART36966, ART11605` (no apparent secondary key) + +`ex_base` is positional row-comparison, so the tie-cluster reorder flips it to `wrong_result`. + +## Trajectory trace — CPI math + +`trajectory[14]` `edit_model artifactscore.conserve_status_num`: +- mapping `LOWER(TRIM(conservestatus)) WHEN 'excellent' THEN 1 WHEN 'good' THEN 3 WHEN 'fair' THEN 5 WHEN 'poor' THEN 7 WHEN 'critical' THEN 10 END` +- description `"[kb=0] Numeric mapping … Excellent=1, Good=3, Fair=5, Poor=7, Critical=10"` + +`trajectory[15]` `edit_model artifactratings.cpi_score`: +- `((histsignrating + researchvalrating + cultscore) * (10 - artifactscore.conserve_status_num)) / 30.0` +- description `"[kb=0] Conservation Priority Index = (HistSignRating + ResearchValRating + CultScore) * (10 - ConserveStatusNum) / 30"` + +KB 0 text: `CPI = (H+R+C) * (10 - ConserveStatus) / 30`. Gold uses `Excellent=9, Good=7, Fair=5, Poor=3, Critical=0` directly; agent uses `1/3/5/7/10` then `(10-x)`. **Arithmetically identical**. Verified by row-level CPI values matching. + +Agent later cast to REAL (item 32) and ROUND(2) (item 58) chasing a non-existent numeric mismatch. + +## Did the agent think about ordering? + +**No — except DESC/ASC direction.** Across 25+ tool calls, every `order` spec uses single-key `cpi_score`. No secondary sort, no NULLS FIRST/LAST, no RANK() tiebreak (one experiment at item 72 used `RANK() OVER (ORDER BY cpi_score DESC)` but still single key, never `(ORDER BY cpi DESC, artref ASC)`). + +ThinkingBlocks along this path all have `thinking=''` (encrypted/stripped). Tool-call sequence shows zero ordering-stability deliberation: item 29 sanity-checked direction, item 49 counted ConserveStatus categories (suspecting mapping bug), item 63 queried min/max of ratings (suspecting numeric bug), item 66 counted rows (suspecting JOIN bug). Every failure diagnosed as a math/value problem. + +## Does the prompt disambiguate ordering? + +**No.** Query: *"Calculate and rank all artifacts by their Conservation Priority Index (CPI) to identify which need urgent attention. The report must include Artifact ID, name, its historical Dynasty and CPI Score."* + +No tiebreaker, no NULLS placement, no deterministic secondary order. Gold's own `ORDER BY CPI DESC NULLS FIRST` (no secondary key) is itself plan-dependent at ties. + +## Verdict + +**Evaluator issue, not agent.** The prompt is ambiguous and the gold's order is plan-dependent at ties. `ex_base` should tolerate row reorderings within equal-CPI clusters, OR the gold needs a deterministic secondary sort. Demanding tiebreaker discipline from the agent here is unreasonable. + +Remediation target: grader (tolerate float-tie reorderings) OR gold (`, ArtRegistry` secondary key). + +Provenance: `results/cloud/20260531t1013-claudes-slayer-48eb0f/rows/museum_3/attempt-1.json` (items 14, 15, 18, 23, 29, 34, 46, 49, 60, 63, 66, 78); `livesqlbench_data_sqlite.jsonl#museum_3` (`external_knowledge: []`); `museum_kb.jsonl#0`; gold sidecar (`ORDER BY CPI DESC NULLS FIRST`). diff --git a/analyses/raw/museum_4.md b/analyses/raw/museum_4.md new file mode 100644 index 00000000..8ae1c10d --- /dev/null +++ b/analyses/raw/museum_4.md @@ -0,0 +1,67 @@ +# museum_4 — failure trace + metadata audit + +Run: `20260531t1013-claudes-slayer-48eb0f`. Duration 608s. submission_status=wrong_result. + +## Task + +Per-dynasty budget report. Gold returns 7 columns (`dynasty, artifact_count, total_cpi, artifacts_with_adequate_budget, artifacts_with_insufficient_budget, CBE, budget_status`). Agent returned 6 (dropped `artifact_count`). + +CBE in gold: `SUM(CPI × adequate_budget/total_records) / artifact_count` — weighted average; `BudgetRatio_i = adequate_records/total_records` derived from `conservationandmaintenance.budgetallocstatus` (Adequate / Insufficient / Review Required categorical enum). + +CBE in agent: `SUM(cpi) / COUNT²` — uniform 1/N reading of "BudgetRatio". + +## Was CBE a KB entry? + +Yes. `museum_kb.jsonl` id=9: +``` +"CBE = \frac{\sum_{i \in artifacts} (CPI_i \times BudgetRatio_i)}{|artifacts|}, where BudgetRatio is the proportion of total conservation budget allocated to each artifact" +``` + +Agent retrieved at `trajectory[8 → 9]` via `get_knowledge_definition("Conservation Budget Efficiency (CBE)")`. Also pulled KB 17 (Crisis), KB 0 (CPI), KB 21 (ConserveStatus enum), KB 22 (HistSignRating). KB coverage was complete. + +## Agent's reasoning + +ThinkingBlocks encrypted (`thinking=''`), but `create_model` description at `trajectory[33]`: +``` +"CBE = sum(CPI*1/N)/N = sum(CPI)/N^2 assuming uniform BudgetRatio" +``` + +Final TextBlock at `trajectory[96]`: +> "the KB definition is ambiguous about what `BudgetRatio` means when no explicit per-artifact budget column exists … My final encoded model uses the most literal reading of KB 9: BudgetRatio_i = 1/N (uniform distribution of the total budget across artifacts), so CBE = SUM(CPI)/N²" + +Agent read KB correctly; resolved ambiguity by assuming uniform allocation. Gold instead reads `BudgetRatio_i = adequate/total` from `conservationandmaintenance.budgetallocstatus`. Both KB 9 and the schema are silent on this. + +## artifact_count column + +Never mentioned by the agent. Prompt: "show the dynasty name, their total conservation priority index, the count of artifacts with adequate funding, the count with insufficient funding, the calculated CBE value, and a budget status" — 5 named outputs + dynasty = 6. Gold reads the same prompt and adds a 7th unstated column. Agent's parse matched the prompt literally; missing column is a gold-side under-specification. + +## Decision-point lock-in + +`trajectory[33]` (initial `create_model`). Description already contains `"CBE = sum(CPI*1/N)/N = sum(CPI)/N^2"`. Subsequent `edit_model` calls (49, ~64, ~76, ~90) only changed scaling/rebasing (cbe drifted 0.0107 → 10.14 → 0.0108 → 0.487) — the structural choice `BudgetRatio = 1/N` was never revisited. Even after 4 submit_query rejections with generic Phase-1 failure, the agent never reached for `adequate_budget / total_records`. + +## Metadata audit + +`museum_column_meaning_base.json:128-147` — `ConservationAndMaintenance` column block (20 columns). Every budget-adjacent description is purely categorical: + +- L141 `BudgetAllocStatus`: *"A VARCHAR(50) describing budget allocation status (possible values: 'Review Required', 'Insufficient', 'Adequate')."* +- L142 `MaintBudgetStatus`: *"...'Limited', 'Depleted', 'Available'..."* + +Full scan: NO column meaning uses "ratio", "proportion", "share", "allocation share", or "adequate / total". + +Full scan of `museum_kb.jsonl` (56 entries) for "Ratio", "Allocation", "Budget", "proportion", "share", "adequate": +- KB 9 (the under-specified CBE itself) +- KB 17 (Crisis) — uses `BudgetAllocStatus='Insufficient'` as categorical threshold, not ratio +- KB 36 (CRAE) — references CBE but doesn't redefine BudgetRatio +- Other matches in KB 22, 28 are unrelated + +No value-illustration KB sibling pinning `BudgetAllocStatus` semantics. + +Agent's slayer exploration: `models_summary` at turn 16 (exposed BudgetAllocStatus description from L141); `query(conservationandmaintenance, n_cm, n_distinct_art)` → 951/951; `query(conservationandmaintenance, dim=budgetallocstatus, count)` → Adequate 314 / Insufficient 326 / Review Required 311. Three `search(...)` calls returned only KBs 9 and 17 (already known). + +## Verdict + +**KB coverage under-specified.** KB 9's `BudgetRatio` is undefined for a schema with no per-artifact budget *amount* column. Gold silently expects `adequate/total` derived from `BudgetAllocStatus`. Missing `artifact_count` is a secondary prompt-vs-gold gap. Agent reasoning sound given the inputs. + +Remediation target: KB (anchor BudgetRatio to BudgetAllocStatus with an explicit value-illustration sibling KB). + +Provenance: trajectory items 9, 33, 96; `museum_kb.jsonl#9,17`; `museum_schema.txt:122` (`budgetallocstatus`, no amount column). diff --git a/analyses/raw/museum_5.md b/analyses/raw/museum_5.md new file mode 100644 index 00000000..44647931 --- /dev/null +++ b/analyses/raw/museum_5.md @@ -0,0 +1,50 @@ +# museum_5 — failure trace + +Run: `20260531t1013-claudes-slayer-48eb0f`. Duration 387s. submission_status=wrong_result. + +## Task + +Show artifacts with ID, name, current temperature, current humidity, high sensitivity count, and a Yes/No flag for "Accelerated Deterioration". 6 columns in gold. + +KB 7 (MDR): `MDR = (ArtAgeYears × ERF × (RelHumidity−50)² × TempC) / 100000`. KB 7 also requires "latest reading per showcase" → `MAX(EnvReadRegistry) GROUP BY ShowcaseRef`. + +## Output diff + +Gold returns 6 columns; agent returns 7 (added a raw `mdr` value column). But the dominant failure is values, not column shape: + +- Predicted sample: `tempc=20, relhumidity=50, mdr=0.0` for every row. +- Gold sample: `tempc=21, relhumidity=53`. + +Agent used an arbitrary env reading. `relhumidity=50` everywhere zeroes `(rh-50)²` → MDR=0 → flag='No' for every row. + +## Trajectory trace + +ThinkingBlocks all signature-only (empty `thinking=''`). Agent went straight from `get_knowledge_definition` lookups (trajectories[6,11,13,15]) to model encoding via `edit_model` at trajectories[28,30,36,44,65,67,84]. No explicit assistant text enumerating expected output columns. + +First `slayer__query` at `trajectory[33]`: +> `dimensions: ['artrefused', 'artifactscore.artname', 'artifactscore.artageyears', 'sensitivitydata.erf', 'sensitivitydata.high_sens_count', 'environmentalreadingscore.tempc', 'environmentalreadingscore.relhumidity']` — 7 entries, pulling `artageyears` and `erf` for verification. + +Submit attempts: +- `trajectory[41, 46, 62, 69]` — 4 submissions with **6 columns** (matching prompt count). All failed `ex_base returned 0 but expected 1`. +- `trajectory[89]` — FINAL submit ADDED `"mdr"` first in dimensions list, producing the 7-column predicted_result. + +Final assistant text: +> "Despite multiple attempts varying the encoding (ERF over 4 vs 11 sensitivities, high_sens_count scope, filtered vs unfiltered, trimmed artifact IDs, **including MDR as a numeric column**), the submission keeps failing…" + +Agent was misreading `ex_base=0` as a hint to add columns. The real problem in the first 4 attempts wasn't column shape (6-col submissions had right shape) — values diverged because the agent used the wrong env reading. + +## Slayer's role + +Slayer's `submit_query` requires explicit dimensions — does NOT default to returning all measures. The 6-column submissions at items 41/46/62/69 prove the agent had full projection control. `mdr` extra column is the agent's choice, not a slayer side effect. + +Slayer DID contribute a separate noise: namespaced column names (`usagerecords.artrefused`, `usagerecords.artifactscore.artname` etc.) vs gold's bare `ArtRegistry`, `ArtName`. If `ex_base` is column-name-aware this contributes; row_count=951 matching for both suggests `ex_base` is value-based and namespace clutter is secondary. + +## Verdict + +**Agent miss.** The model never decoded KB 7's "latest reading per showcase" (`MAX(EnvReadRegistry) GROUP BY ShowcaseRef` semantics). Never inspected its own zero-everywhere `mdr` output. Reacted to repeated `ex_base=0` by adding a column instead of debugging values. Slayer's namespaced names a contributing irritant, not primary cause. + +Failure class: agent miss (KB-coverage discipline + self-check on suspicious values absent). + +Remediation: prompt nudge — "always sanity-check that computed columns aren't degenerate (all-zero, all-null) before submitting". Also: KB 7's "latest" language could be more explicit about the join idiom. + +Provenance: `results/cloud/20260531t1013-claudes-slayer-48eb0f/rows/museum_5/attempt-1.json`; `museum_kb.jsonl` (KB 7). diff --git a/analyses/raw/museum_9.md b/analyses/raw/museum_9.md new file mode 100644 index 00000000..2c0f0321 --- /dev/null +++ b/analyses/raw/museum_9.md @@ -0,0 +1,85 @@ +# museum_9 — failure trace + metadata audit + +Run: `20260531t1013-claudes-slayer-48eb0f`. Duration 916s. submission_status=wrong_result. + +## Task + +TETL (Total Environmental Threat Level) top-10. 2 columns: artref, tetl. + +TETL formula (KB 31): `TETL = ERF + LER + (MDR × 2)`. With KB 2 (ERF) + KB 8 (LER) + KB 7 (MDR) + KB 1 (sensitivity weights Low=1/Medium=5/High=10). + +## Output diff + +Gold top values: 215.74, 199.27, 198.93. Agent: 160245, 154577, 151439. **Off by ~1000×.** + +## Agent's encoding — correct + +`trajectory[32]` created `mdr_value`, `ler_value`, `tetl_value`. Submitted SQL contains `2.0 * (artageyears * (sens_sum/4.0) * (relhumidity-50)*(relhumidity-50) * tempc) / 100000.0` — POWER expanded, `/100000.0` present, ERF as sum/4.0. Sensitivity ratings not double-counted: ERF appears once inside MDR and once standalone per KB 7 + KB 31. + +ThinkingBlocks signature-only; encoded SQL is faithful to KB. + +## The 1000× gap — NOT a missing divisor + +At `trajectory[215]` agent ran `tetl_no_lsw` (LER without `light_sens_w` term) and got 16K — clean 10× drop. Only multiplicative knob is sensitivity weight (max 10), not a missing divisor. + +The gap comes from **join-path / row-cardinality semantics**, not formula: + +- **Gold** (`livesqlbench_sqlite_gt_kg_testcases_0528.jsonl[175]`): MDR via `usagerecords.artrefused → showcase → environmentalreadingscore` with explicit `LIMIT 1`. TETL assembled in a `LEFT JOIN`ed CTE with `COALESCE(l.ler, 0) + COALESCE(m.mdr, 0)*2`. Most artifacts have null LER/MDR → TETL ≈ ERF (1-10), survivors hit ~200. +- **Agent**: TETL built on `conditionassessments → lightandradiationreadings → environmentalreadingscore` (path discovered around `trajectory[73-87]`). Every assessed row has all three components live; every `light_lux × visibleexplxh` term fires; per-assessment magnitudes blow up. + +Agent saw the discrepancy and tried `tetl_lin` (210), `tetl_no_lsw` (215), `tetl_11` with normalized erf (224), `tetl_alt` (237), `tetl_abs` (198) — none changed join topology, all produced same scale. + +## Metadata audit + +**Was the canonical join path discoverable?** + +`museum_column_meaning_base.json:148-150`: +- `ArtRefUsed`: *"A CHAR(10) NOT NULL foreign key referencing ArtifactsCore(ArtRegistry), indicating which artifact is being used."* +- `ShowcaseRefUsed`: *"A CHAR(12) foreign key referencing Showcases(ShowcaseReg) if a showcase is involved in the usage."* (**`if a showcase is involved`** actively WEAKENS the canonical claim) + +Plain FKs, no "canonical link" / "current display" / "primary location" language. + +`EnvironmentalReadingScore`: +- `EnvReadRegistry`: *"BIGSERIAL PRIMARY KEY..."* (no temporal hint) +- `ShowcaseRef`: *"...linking the reading to a specific showcase being monitored."* +- `ReadTimestamp`: *"TIMESTAMP NOT NULL indicating the date and time the reading was recorded."* + +No "use the latest", "LIMIT 1", or "one reading per showcase". Gold's `LIMIT 1` (no ORDER BY) is itself non-deterministic — picks arbitrary row, not even latest by timestamp. + +**Schema FKs:** + +`museum_schema.txt`: +- `conditionassessments` (lines 88-100): FKs on `artrefexamined`, `showcaserefexamined`, `lightreadrefobserved → lightandradiationreadings(lightradregistry)`. +- `lightandradiationreadings` (lines 181-190): FK `envreadref → environmentalreadingscore(envreadregistry)`. +- `environmentalreadingscore` (lines 141-153): FK `showcaseref → showcases(showcasereg)`. +- `usagerecords` (lines 308-338): FKs `artrefused → artifactscore`, `showcaserefused → showcases`. + +**BOTH paths first-class FK-declared.** Schema alone doesn't disambiguate. + +**KB sweep for "showcase / current display / latest"**: KB 31 (TETL), 7 (MDR), 2 (ERF), 8 (LER), 1 (SensWeights) define formulas with `RelHumidity`, `TempC` etc. as bare variable names — no table-qualification. No KB names a canonical artifact↔env-reading link. KB 15 mentions "current display duration" but doesn't anchor a join path. + +**Agent's exploration**: +- `trajectory[3,7,8,9]`: KB searches for TETL/LER/MDR/SensWeights. +- `trajectory[19]`: `inspect_model('conditionassessments', sections=['columns','joins','samples'])`. +- `trajectory[20]`: `inspect_model('sensitivitydata', …)`. +- `trajectory[76]`: `inspect_model('artifactscore', …)`. +- `trajectory[131]`: `search('artifact environmental reading join showcase usage')` — explicit join-path semantic search. +- `trajectory[145]`: `inspect_model('usagerecords', sections=['columns','joins'], num_rows=3)` — agent DID inspect UsageRecords. +- `trajectory[148-158]`: edits adding env→light join, trimming `showcaseref` on showcases/env/usagerecords, adding showcases→env join. +- `trajectory[159]`: defines `tetl_usage` on `usagerecords` but the formula references `showcases__environmentalreadingscore__lightandradiationreadings.*` — env reached via LIGHT path on the CA side, not `usagerecords.showcaserefused → showcases → env`. + +Agent saw `usagerecords.showcaserefused`'s description, materialised it into the source model, but the column description didn't carry a "use me for MDR" signal, and the formula was already wired through `conditionassessments`. Never switched paths. + +## Verdict + +**KB-spec gap.** The resolving information is genuinely absent from every metadata source: +- DDL: both paths FK-declared; no UNIQUE constraint disambiguates. +- Column descriptions: `ShowcaseRefUsed` is *weaker* than what gold needs ("if a showcase is involved"). +- KB: no entry names a canonical artifact↔env join. +- Gold's `LIMIT 1` is itself non-deterministic (no ORDER BY) — even the gold doesn't take "the latest" reliably. + +Agent explored UsageRecords, searched explicitly for `"artifact environmental reading join showcase usage"`, saw the same FK graph as the gold author. No metadata cue prescribed usagerecords-via-showcase over conditionassessments-via-light. + +Remediation: KB-spec addendum naming the canonical env-reading join for MDR purposes (e.g., "route via `UsageRecords.ShowcaseRefUsed`, take latest by `ReadTimestamp`"). + +Provenance: trajectory items 32, 88, 210, 215, 218, 240, 243; `museum_kb.jsonl` (ids 1, 2, 7, 8, 31); `museum_schema.txt:88-98, 181-192, 308-336`; gold sidecar line 175 (mdr_calc uses `usagerecords … LIMIT 1`). From 1e6b33dc600efa03b954b0de4bb9ea0d51abf13a Mon Sep 17 00:00:00 2001 From: Egor Kraev Date: Sun, 31 May 2026 18:03:06 +0200 Subject: [PATCH 02/29] =?UTF-8?q?DEV-1515=20audited-gold=20consolidation:?= =?UTF-8?q?=20mini-interact=20=E2=86=92=20single=5Ffile?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Moves mini-interact from the per_db sidecar layout (audited_gold//_audited.jsonl) to a single consolidated JSONL at audited_gold/mini_interact_audited.jsonl, matching the livesqlbench layout introduced in DEV-1510. Each row gains three fields: - benchmark: "mini_interact" (required by harness defence-in-depth) - variant_id: "primary" (DEV-1515 multi-variant gold support) - primary: true (interaction-time feedback discriminator) livesqlbench's existing single_file JSONL retro-tagged with the same variant_id / primary fields for layout consistency across benchmarks. The consolidation itself is in scripts/consolidate_mini_interact_audited.py (idempotent: skips fields already present with expected values; rejects benchmark-tag mismatches). audited_gold/ is fully gitignored, so this commit ships the script + descriptor change; the on-disk JSONLs are local-only artefacts other developers regenerate by running the script. apply_audited_gold_overlay's single_file branch already handles this layout (DEV-1510), so the only src change is flipping mini-interact's Benchmark.audited_gold_layout from per_db to single_file. Tests: - test_benchmark.py: assertion flipped per_db → single_file - test_paths.py: per_db_layout_raises → single_file path resolves - test_dual_eval.py: per_db_explicit_matches_default rewritten as single_file dispatch test - Full non-integration suite: 1697 passed, 94 skipped, 50 deselected Co-Authored-By: Claude Opus 4.7 (1M context) --- scripts/consolidate_mini_interact_audited.py | 110 +++++++++++++++++++ src/bird_interact_agents/benchmark.py | 6 + tests/test_benchmark.py | 6 +- tests/test_dual_eval.py | 24 ++-- tests/test_paths.py | 16 +-- 5 files changed, 144 insertions(+), 18 deletions(-) create mode 100644 scripts/consolidate_mini_interact_audited.py diff --git a/scripts/consolidate_mini_interact_audited.py b/scripts/consolidate_mini_interact_audited.py new file mode 100644 index 00000000..aa121a2e --- /dev/null +++ b/scripts/consolidate_mini_interact_audited.py @@ -0,0 +1,110 @@ +"""Consolidate mini-interact per-DB audited-gold JSONLs into a single file. + +Merges every `audited_gold//_audited.jsonl` into +`audited_gold/mini_interact_audited.jsonl` while: + +* Adding `benchmark: "mini_interact"` (required by harness defence-in-depth + for single_file layouts). +* Adding `variant_id: "primary"` and `primary: true` (DEV-1515 multi-variant + support — all currently-existing rows are tagged as the primary variant; + multi-variant tasks will add additional rows later). + +Also retro-tags `audited_gold/livesqlbench_audited.jsonl` rows with the same +`variant_id`/`primary` fields for layout consistency across benchmarks. + +Idempotent: skips fields that are already present with the expected values. +Run once; commit; delete per-DB files. +""" +from __future__ import annotations + +import json +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +AUDITED = ROOT / "audited_gold" + +MINI_INTERACT_OUT = AUDITED / "mini_interact_audited.jsonl" +LIVESQLBENCH = AUDITED / "livesqlbench_audited.jsonl" + + +def add_multivariant_fields(row: dict, *, benchmark: str) -> dict: + """Return a row with benchmark/variant_id/primary fields ensured.""" + out = dict(row) + if out.get("benchmark") not in (None, benchmark): + raise RuntimeError( + f"row has benchmark={out.get('benchmark')!r}; expected " + f"{benchmark!r} (instance_id={out.get('instance_id')!r})" + ) + out.setdefault("benchmark", benchmark) + out.setdefault("variant_id", "primary") + out.setdefault("primary", True) + return out + + +def consolidate_mini_interact() -> None: + per_db_files = sorted( + p for p in AUDITED.iterdir() + if p.is_dir() and (p / f"{p.name}_audited.jsonl").exists() + ) + if not per_db_files: + print("no per-DB JSONLs to consolidate") + return + print(f"found {len(per_db_files)} per-DB JSONLs") + rows: list[dict] = [] + seen: set[tuple[str, str]] = set() + for db_dir in per_db_files: + db = db_dir.name + path = db_dir / f"{db}_audited.jsonl" + n = 0 + with path.open() as f: + for line in f: + line = line.strip() + if not line: + continue + d = json.loads(line) + row_db = d.get("selected_database") + if row_db != db: + raise RuntimeError( + f"row in {path} has selected_database={row_db!r}, " + f"expected {db!r} (instance_id={d.get('instance_id')!r})" + ) + inst = d.get("instance_id") + if not inst: + raise RuntimeError(f"row missing instance_id in {path}") + key = (inst, d.get("variant_id", "primary")) + if key in seen: + raise RuntimeError( + f"duplicate (instance_id, variant_id)={key} in {path}" + ) + seen.add(key) + rows.append(add_multivariant_fields(d, benchmark="mini_interact")) + n += 1 + print(f" {db:40s} {n:>3} rows") + print(f"total rows: {len(rows)}") + MINI_INTERACT_OUT.write_text( + "\n".join(json.dumps(r, ensure_ascii=False) for r in rows) + "\n" + ) + print(f"wrote {MINI_INTERACT_OUT}") + + +def retro_tag_livesqlbench() -> None: + if not LIVESQLBENCH.exists(): + print(f"{LIVESQLBENCH} absent; skipping retro-tag") + return + rows: list[dict] = [] + with LIVESQLBENCH.open() as f: + for line in f: + line = line.strip() + if not line: + continue + d = json.loads(line) + rows.append(add_multivariant_fields(d, benchmark="livesqlbench")) + LIVESQLBENCH.write_text( + "\n".join(json.dumps(r, ensure_ascii=False) for r in rows) + "\n" + ) + print(f"retro-tagged {len(rows)} rows in {LIVESQLBENCH}") + + +if __name__ == "__main__": + consolidate_mini_interact() + retro_tag_livesqlbench() diff --git a/src/bird_interact_agents/benchmark.py b/src/bird_interact_agents/benchmark.py index 0c84811d..1fc2b29d 100644 --- a/src/bird_interact_agents/benchmark.py +++ b/src/bird_interact_agents/benchmark.py @@ -92,6 +92,12 @@ class Benchmark(BaseModel): gold_required=False, per_task_db_isolation=False, container_data_dir="/data/mini-interact", + # DEV-1515: switched to the single-file layout (matches livesqlbench). + # Per-DB JSONLs at audited_gold//_audited.jsonl have been + # consolidated into audited_gold/mini_interact_audited.jsonl with + # variant_id + primary fields added per row (DEV-1515 multi-variant + # support). See scripts/consolidate_mini_interact_audited.py. + audited_gold_layout="single_file", ) LIVESQLBENCH = Benchmark( diff --git a/tests/test_benchmark.py b/tests/test_benchmark.py index a60b73bf..b020f0e3 100644 --- a/tests/test_benchmark.py +++ b/tests/test_benchmark.py @@ -108,8 +108,10 @@ def test_audited_gold_layout_default_is_per_db(): assert Benchmark.model_fields["audited_gold_layout"].default == "per_db" -def test_mini_interact_audited_gold_layout_per_db(): - assert get_benchmark("mini_interact").audited_gold_layout == "per_db" +def test_mini_interact_audited_gold_layout_single_file(): + """DEV-1515: mini-interact moved to single_file (matches livesqlbench). + Per-DB JSONLs were consolidated into ``audited_gold/mini_interact_audited.jsonl``.""" + assert get_benchmark("mini_interact").audited_gold_layout == "single_file" def test_livesqlbench_audited_gold_layout_single_file(): diff --git a/tests/test_dual_eval.py b/tests/test_dual_eval.py index 35764f40..5870ccdb 100644 --- a/tests/test_dual_eval.py +++ b/tests/test_dual_eval.py @@ -801,17 +801,25 @@ def test_overlay_benchmark_kwarg_default_preserves_per_db_behavior(tmp_path): assert task["sol_sql"] == ["SELECT audited FROM t"] -def test_overlay_benchmark_kwarg_per_db_explicit_matches_default(tmp_path): - """Passing `benchmark=mini_interact` explicitly is equivalent to omitting - it — proves the layout dispatch is the field that matters, not whether - `benchmark` happens to be set.""" +def test_overlay_benchmark_kwarg_mini_interact_uses_single_file(tmp_path): + """DEV-1515: passing `benchmark=mini_interact` dispatches to the + consolidated ``mini_interact_audited.jsonl`` (not the legacy per_db + sidecar). Proves the layout dispatch follows the descriptor and that + mini-interact's new single-file shape is wired through end-to-end.""" + import json from bird_interact_agents.benchmark import get_benchmark from bird_interact_agents.harness import apply_audited_gold_overlay - _write_audit_sidecar(tmp_path, "alien", [ - {"instance_id": "alien_explicit", "audit_status": "edited", - "audited_sol_sql": ["SELECT audited FROM t"]}, - ]) + single_file = tmp_path / "mini_interact_audited.jsonl" + single_file.write_text(json.dumps({ + "instance_id": "alien_explicit", + "selected_database": "alien", + "benchmark": "mini_interact", + "variant_id": "primary", + "primary": True, + "audit_status": "edited", + "audited_sol_sql": ["SELECT audited FROM t"], + }) + "\n") task = { "instance_id": "alien_explicit", "selected_database": "alien", diff --git a/tests/test_paths.py b/tests/test_paths.py index 1da5f4ef..db1edc9b 100644 --- a/tests/test_paths.py +++ b/tests/test_paths.py @@ -709,15 +709,15 @@ def test_audited_gold_file_honours_root_env_override(tmp_path, monkeypatch): ) -def test_audited_gold_file_per_db_layout_raises(tmp_path, monkeypatch): - """mini-interact uses per-db sidecars; there is no single-file path. - Asking for it must fail loudly — silently returning a bogus path would - let a caller write the wrong layout for that benchmark.""" - _setup_main_and_worktree(tmp_path, monkeypatch) - with pytest.raises(ValueError) as exc: +def test_audited_gold_file_mini_interact_anchored_to_main(tmp_path, monkeypatch): + """mini-interact moved to single_file layout in DEV-1515. + The consolidated path is ``audited_gold/mini_interact_audited.jsonl``. + """ + main_root, _ = _setup_main_and_worktree(tmp_path, monkeypatch) + assert ( paths.audited_gold_file(benchmark="mini_interact") - msg = str(exc.value).lower() - assert "single_file" in msg or "per_db" in msg or "layout" in msg + == main_root / "audited_gold" / "mini_interact_audited.jsonl" + ) def test_audited_gold_file_unknown_benchmark_raises(tmp_path, monkeypatch): From 5e92cf09f07a091d1f2d092a1cd434a6cd6f60aa Mon Sep 17 00:00:00 2001 From: Egor Kraev Date: Sun, 31 May 2026 18:07:04 +0200 Subject: [PATCH 03/29] DEV-1515 prompt fix: extend IN-set from sampled-value variants claude_sdk_otf + claude_sdk_otf_ainteract: add a symmetric companion to the existing "drop predicate when KB literals are absent from sampled values" rule. The new rule tells the agent: when the column's `Sample values` show variants of KB-named literals (case, whitespace, abbreviations, alternate phrasings of the same concept), normalise and extend the IN-set to include them. Targets the households_10 + museum_2 / museum_10 class of failure: KB enumerations are non-exhaustive ("etc.", "like", "include"); the sampled values are the authoritative inventory of what's actually present. A canonical-only IN-set silently misses matching rows. Also adds a paragraph promoting `search(entities=[".."], max_memories=0, max_example_queries=0)` to the front of the slayer-tools section. EntityHit.text carries Description + Sample values inline; this is the canonical "read by known ref" primitive for verifying any filter / projection / join-key column before committing. Out of scope: the rendered `Sample values` is the truncated comma-string (motley-slayer's `render_column_text`), not the full structured `sampled_values: List[str]`. Wider variant coverage requires a separate slayer-upstream change to expose the structured field via search / inspect_model. scripts/verify_audited_gold.py: reads the consolidated mini_interact_audited.jsonl when audit-set=inhouse and filters by selected_database (per DEV-1515 consolidation). SAR audit set is unchanged. Tests: - Existing non-integration suite: 1696 passed, 95 skipped, 50 deselected. Co-Authored-By: Claude Opus 4.7 (1M context) --- scripts/verify_audited_gold.py | 22 ++++++++++++++++++- .../agents/claude_sdk_otf/prompts.py | 21 ++++++++++++++++++ .../claude_sdk_otf_ainteract/prompts.py | 20 +++++++++++++++++ 3 files changed, 62 insertions(+), 1 deletion(-) diff --git a/scripts/verify_audited_gold.py b/scripts/verify_audited_gold.py index 0668b9d0..cdd716b6 100644 --- a/scripts/verify_audited_gold.py +++ b/scripts/verify_audited_gold.py @@ -72,10 +72,30 @@ def audited_filename_for(db: str, audit_set: str) -> str: def load_audited(db: str, audit_set: str = "inhouse") -> list[dict]: + # DEV-1515: inhouse mini-interact moved to single_file layout. Read + # the consolidated `mini_interact_audited.jsonl` and filter by + # `selected_database`. SAR audit set still uses the per_db layout. + if audit_set == "inhouse": + single = audited_root_for(audit_set) / "mini_interact_audited.jsonl" + if single.exists(): + rows: list[dict] = [] + with single.open() as f: + for line in f: + line = line.strip() + if not line: + continue + d = json.loads(line) + if d.get("selected_database") == db: + rows.append(d) + if not rows: + raise FileNotFoundError( + f"No rows for db={db!r} in {single}" + ) + return rows path = audited_root_for(audit_set) / db / audited_filename_for(db, audit_set) if not path.exists(): raise FileNotFoundError(f"No sidecar at {path}") - rows: list[dict] = [] + rows = [] with path.open() as f: for line in f: line = line.strip() diff --git a/src/bird_interact_agents/agents/claude_sdk_otf/prompts.py b/src/bird_interact_agents/agents/claude_sdk_otf/prompts.py index 3c88fe0f..082f3949 100644 --- a/src/bird_interact_agents/agents/claude_sdk_otf/prompts.py +++ b/src/bird_interact_agents/agents/claude_sdk_otf/prompts.py @@ -59,6 +59,15 @@ to see a model's columns / measures / joins; `create_model` / `edit_model` to add columns and measures; `query` / `query_nested` to test. +READ A KNOWN COLUMN'S FULL DESCRIPTION before committing to it as a +filter, projection, or join key — `search` with `entities=[ +".."]`, `max_memories=0`, `max_example_queries=0`. The +returned `EntityHit.text` carries `Description:` and `Sample values:` +inline. The truncated `Sample values:` line is your authoritative source +of which literal forms actually occur in this column — case variants, +whitespace forms, abbreviations, alternate phrasings of the same concept. +Use it BEFORE writing any IN-set (see rule 3 below). + ENCODE-THEN-QUERY DISCIPLINE: 1. DECOMPOSE the question into logical blocks. Every qualifier @@ -94,6 +103,18 @@ - If a KB cites named literals that are ABSENT from the column's sampled values (check via `inspect_model`), do not write that predicate. + - Symmetric companion: if the column's `Sample values` show variants + of the KB-named literals — case differences, internal whitespace, + abbreviations (`apt` for `Apartment`, `Y` for `yes`), or alternate + phrasings of the same concept (`brick house` vs `brickwork house`, + `2014+` / `after 2014` for `2014 or newer`) — NORMALISE and EXTEND + the IN-set to include those variants. KB hedges ("etc.", "like", + "include") and the schema author's `Ex.` enumerations are + deliberately non-exhaustive; the `Sample values` line is the + authoritative inventory of what's actually present in the column. + A canonical-only IN-set will silently miss matching rows. There is + no user-sim to confirm this for you in one-shot mode — read the + sampled values yourself. """ diff --git a/src/bird_interact_agents/agents/claude_sdk_otf_ainteract/prompts.py b/src/bird_interact_agents/agents/claude_sdk_otf_ainteract/prompts.py index e3e2e5a9..7cba39b7 100644 --- a/src/bird_interact_agents/agents/claude_sdk_otf_ainteract/prompts.py +++ b/src/bird_interact_agents/agents/claude_sdk_otf_ainteract/prompts.py @@ -79,6 +79,15 @@ to see a model's columns / measures / joins; `create_model` / `edit_model` to add columns and measures; `query` / `query_nested` to test. +READ A KNOWN COLUMN'S FULL DESCRIPTION before committing to it as a +filter, projection, or join key — `search` with `entities=[ +".."]`, `max_memories=0`, `max_example_queries=0`. The +returned `EntityHit.text` carries `Description:` and `Sample values:` +inline. The truncated `Sample values:` line is your authoritative source +of which literal forms actually occur in this column — case variants, +whitespace forms, abbreviations, alternate phrasings of the same concept. +Use it BEFORE writing any IN-set (see rule 3 below). + ENCODE-THEN-QUERY DISCIPLINE: 1. DECOMPOSE the question into logical blocks. Every qualifier @@ -114,6 +123,17 @@ - If a KB cites named literals that are ABSENT from the column's sampled values (check via `inspect_model`), do not write that predicate. + - Symmetric companion: if the column's `Sample values` show variants + of the KB-named literals — case differences, internal whitespace, + abbreviations (`apt` for `Apartment`, `Y` for `yes`), or alternate + phrasings of the same concept (`brick house` vs `brickwork house`, + `2014+` / `after 2014` for `2014 or newer`) — NORMALISE and EXTEND + the IN-set to include those variants. KB hedges ("etc.", "like", + "include") and the schema author's `Ex.` enumerations are + deliberately non-exhaustive; the `Sample values` line is the + authoritative inventory of what's actually present in the column. + A canonical-only IN-set will silently miss matching rows. Do not + rely on the user-sim to enumerate the variants — they will not. 4. ASK AGAIN IF NEEDED. Rule 0 covers the FIRST ask; for any further operationalisation choice not pinned by a memory or column From 45bdd03c0404441883a20ca79d985062ad4637fd Mon Sep 17 00:00:00 2001 From: Egor Kraev Date: Sun, 31 May 2026 18:10:57 +0200 Subject: [PATCH 04/29] DEV-1515 annotation skeleton: TaskAnnotation + SubmissionAnnotation schemas MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit src/bird_interact_agents/eval/: new module hosting the DEV-1515 annotation infrastructure. annotation_schema.py: - TaskAnnotation: per-instance, run-independent. Captures metadata_sufficiency.verdict ∈ {sufficient, ambiguous, insufficient}, a list of GoldVariantRef (each pointing to a row in the consolidated audited-gold JSONL by (instance_id, variant_id), with one tagged primary=True for interaction-time feedback), and an evaluator_prompt (only invoked when verdict==insufficient). - SubmissionAnnotation: per-(instance, run). Carries the cascading SubmissionEvaluation block (original gold → primary variant → any variant → tie tolerance → LLM judge), FailureClassification (top-level enum primary + free-form details), decision-point reference, and UserSimInteraction. - Every model is ConfigDict(extra="forbid") so silent schema drift is impossible. Container fields use typed BaseModels rather than Dict[...]. annotation_io.py: - read_/write_ helpers for both kinds with Pydantic JSON round-trip. - task_annotation_path / submission_annotation_path: canonical `annotations///.{task,submission.}.json` anchored at the main checkout (matches audited_gold/results contract). - iter_task_annotations / iter_submission_annotations for batch consumers. tests/test_eval_annotation_schema.py: 8 cases covering minimal construction, JSON round-trip, forbid-extra, path helpers, and the on-disk 2-space-indent + trailing-newline invariant. Out of scope (follow-up commits): - generate_task_annotation / generate_submission_annotation that materialise skeletons from existing artefacts (task data + audited gold + attempt JSON). - CLI entry point (`bird-interact-annotate ...`). - tolerant_grader.py implementing Tier 1 + Tier 2 evaluation that populates SubmissionEvaluation. Co-Authored-By: Claude Opus 4.7 (1M context) --- src/bird_interact_agents/eval/__init__.py | 63 ++++ .../eval/annotation_io.py | 118 ++++++++ .../eval/annotation_schema.py | 276 ++++++++++++++++++ tests/test_eval_annotation_schema.py | 219 ++++++++++++++ 4 files changed, 676 insertions(+) create mode 100644 src/bird_interact_agents/eval/__init__.py create mode 100644 src/bird_interact_agents/eval/annotation_io.py create mode 100644 src/bird_interact_agents/eval/annotation_schema.py create mode 100644 tests/test_eval_annotation_schema.py diff --git a/src/bird_interact_agents/eval/__init__.py b/src/bird_interact_agents/eval/__init__.py new file mode 100644 index 00000000..bf41e6d5 --- /dev/null +++ b/src/bird_interact_agents/eval/__init__.py @@ -0,0 +1,63 @@ +"""Evaluation tooling for annotated benchmark runs (DEV-1515). + +Two artifact kinds: + +* **Task annotation** — per-instance, run-independent. Captures the + metadata sufficiency verdict, the set of acceptable gold variants + (1..N, with one tagged ``primary`` for interaction-time feedback), + and an LLM-judge ``evaluator_prompt`` for tasks the metadata can't + uniquely pin. Path: + ``annotations///.task.json``. + +* **Submission annotation** — per-(instance, run). Carries the cascading + evaluation block (original gold → primary variant → any variant → tie + tolerance → LLM judge), the failure classification, decision-point + reference, and the user-sim interaction summary. Path: + ``annotations///.submission..json``. + +The on-disk shape is governed by the Pydantic models in +``annotation_schema`` and the read/write helpers in ``annotation_io``. +""" +from bird_interact_agents.eval.annotation_schema import ( + AuditedGoldRef, + FailureClassification, + GoldVariantRef, + MaskedTerm, + MetadataSufficiency, + SubmissionAnnotation, + SubmissionEvaluation, + SubmissionMetadata, + TaskAnnotation, + TrajectoryDecisionPoint, + UserSimInteraction, + VariantMatch, +) +from bird_interact_agents.eval.annotation_io import ( + read_submission_annotation, + read_task_annotation, + submission_annotation_path, + task_annotation_path, + write_submission_annotation, + write_task_annotation, +) + +__all__ = [ + "AuditedGoldRef", + "FailureClassification", + "GoldVariantRef", + "MaskedTerm", + "MetadataSufficiency", + "SubmissionAnnotation", + "SubmissionEvaluation", + "SubmissionMetadata", + "TaskAnnotation", + "TrajectoryDecisionPoint", + "UserSimInteraction", + "VariantMatch", + "read_submission_annotation", + "read_task_annotation", + "submission_annotation_path", + "task_annotation_path", + "write_submission_annotation", + "write_task_annotation", +] diff --git a/src/bird_interact_agents/eval/annotation_io.py b/src/bird_interact_agents/eval/annotation_io.py new file mode 100644 index 00000000..ce40cf4a --- /dev/null +++ b/src/bird_interact_agents/eval/annotation_io.py @@ -0,0 +1,118 @@ +"""Read / write helpers + canonical paths for annotations (DEV-1515). + +Path conventions: + +* Task annotation: + ``/annotations///.task.json`` +* Submission annotation: + ``/annotations///.submission..json`` + +```` matches the benchmark name passed by callers (e.g. +``mini-interact``, ``livesqlbench``). ```` matches the task's +``selected_database`` field. + +These helpers do NOT regenerate annotations from raw artefacts — that +belongs in a separate tool (``scripts/generate_annotation_skeleton.py`` +or similar). Here we only do schema-validated JSON I/O. +""" +from __future__ import annotations + +from pathlib import Path +from typing import Optional + +from bird_interact_agents import paths +from bird_interact_agents.eval.annotation_schema import ( + SubmissionAnnotation, + TaskAnnotation, +) + +ANNOTATIONS_DIRNAME = "annotations" + + +def _annotations_root(repo_root: Optional[Path] = None) -> Path: + """Anchor at the main checkout (matches the worktree-safe contract + used by ``audited_gold/`` and ``results/``).""" + root = Path(repo_root) if repo_root else paths.main_checkout_root() + return root / ANNOTATIONS_DIRNAME + + +def task_annotation_path( + *, + benchmark: str, + selected_database: str, + instance_id: str, + repo_root: Optional[Path] = None, +) -> Path: + return ( + _annotations_root(repo_root) + / benchmark + / selected_database + / f"{instance_id}.task.json" + ) + + +def submission_annotation_path( + *, + benchmark: str, + selected_database: str, + instance_id: str, + run_id: str, + repo_root: Optional[Path] = None, +) -> Path: + return ( + _annotations_root(repo_root) + / benchmark + / selected_database + / f"{instance_id}.submission.{run_id}.json" + ) + + +def read_task_annotation(path: Path) -> TaskAnnotation: + return TaskAnnotation.model_validate_json(path.read_text()) + + +def write_task_annotation(ann: TaskAnnotation, path: Path) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(ann.model_dump_json(indent=2, exclude_none=False) + "\n") + + +def read_submission_annotation(path: Path) -> SubmissionAnnotation: + return SubmissionAnnotation.model_validate_json(path.read_text()) + + +def write_submission_annotation(ann: SubmissionAnnotation, path: Path) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(ann.model_dump_json(indent=2, exclude_none=False) + "\n") + + +def iter_task_annotations( + *, benchmark: str, repo_root: Optional[Path] = None, +) -> "list[tuple[Path, TaskAnnotation]]": + """Walk every task annotation under ``annotations//``. + + Returns ``(path, model)`` pairs; raises if any file fails validation + so corruption surfaces at scan time rather than at first downstream + use. + """ + root = _annotations_root(repo_root) / benchmark + out: list[tuple[Path, TaskAnnotation]] = [] + if not root.exists(): + return out + for path in sorted(root.rglob("*.task.json")): + out.append((path, read_task_annotation(path))) + return out + + +def iter_submission_annotations( + *, benchmark: str, run_id: Optional[str] = None, + repo_root: Optional[Path] = None, +) -> "list[tuple[Path, SubmissionAnnotation]]": + """Walk submission annotations. When ``run_id`` is set, filter.""" + root = _annotations_root(repo_root) / benchmark + out: list[tuple[Path, SubmissionAnnotation]] = [] + if not root.exists(): + return out + pattern = f"*.submission.{run_id}.json" if run_id else "*.submission.*.json" + for path in sorted(root.rglob(pattern)): + out.append((path, read_submission_annotation(path))) + return out diff --git a/src/bird_interact_agents/eval/annotation_schema.py b/src/bird_interact_agents/eval/annotation_schema.py new file mode 100644 index 00000000..bd5adcd7 --- /dev/null +++ b/src/bird_interact_agents/eval/annotation_schema.py @@ -0,0 +1,276 @@ +"""Pydantic schemas for task and submission annotations (DEV-1515). + +Two top-level models — ``TaskAnnotation`` (per-instance, run-independent) +and ``SubmissionAnnotation`` (per-(instance, run)) — plus the leaf +helper models the agreed JSON shape uses. + +The schemas point at on-disk SQL via ``GoldVariantRef.audited_gold_ref`` +(``(file, key)`` into the consolidated ``_audited.jsonl``); +SQL is never duplicated inside annotation JSON. + +Per the project Python convention, every container field uses a typed +``BaseModel`` rather than ``Dict[...]``. +""" +from __future__ import annotations + +from typing import List, Literal, Optional + +from pydantic import BaseModel, ConfigDict, Field + +# Top-level enums kept as Literal strings so the JSON encoding is stable +# and human-greppable. +MetadataSufficiencyVerdict = Literal["sufficient", "ambiguous", "insufficient"] +AuditStatus = Literal["clean", "edited", "unrecoverable", "original_correct"] +RowsetRelation = Literal[ + "equal_rowset", + "strict_subset_of", + "strict_superset_of", + "overlapping", + "disjoint", + "unevaluated", +] +PhaseVerdict = Literal["pass", "fail", "skip"] +FailurePrimary = Literal[ + "agent_miss", + "metadata_ambiguity", + "gold_audit_quality", + "user_sim_under_disclosure", + "grader_stability", + "other", +] +RemediationTarget = Literal["agent", "prompt", "kb", "audit", "grader", "user_sim", "gold_sidecar", "other"] +SubmissionVerdict = Literal[ + "correct", + "valid_interpretation", + "invalid", + "reject_quarantined", + "pending", +] + + +class MaskedTerm(BaseModel): + """One masked term from the task's ``critical_ambiguity`` block. + + Mirrors the BIRD-Interact mini-interact convention: each entry names + the user-facing term, its ambiguity type, and the metadata sources + that pin its resolution (or that the annotator believes pin it). + """ + model_config = ConfigDict(extra="forbid") + + term: str + type: str + """E.g. ``knowledge_linking_ambiguity``, ``schema_linking_ambiguity``, + ``intent_ambiguity``, ``semantic_ambiguity``.""" + is_mask: bool = True + metadata_evidence: List[str] = Field(default_factory=list) + """Pointers to the supporting source — e.g. + ``households_kb.jsonl#10`` or + ``slayer_models_otf/.../amenities.yaml:cablestatus.sampled_values``.""" + + +class MetadataSufficiency(BaseModel): + model_config = ConfigDict(extra="forbid") + + verdict: MetadataSufficiencyVerdict + rationale: str + evidence_sources_consulted: List[str] = Field(default_factory=list) + + +class AuditedGoldRef(BaseModel): + """Pointer into the consolidated audited-gold JSONL. + + Lookup key is ``(instance_id, variant_id)``; the file path is + redundant (callers can resolve from the benchmark) but stored so the + JSON is self-documenting. + """ + model_config = ConfigDict(extra="forbid") + + file: str + """E.g. ``audited_gold/mini_interact_audited.jsonl``.""" + instance_id: str + variant_id: str = "primary" + + +class GoldVariantRef(BaseModel): + """One acceptable interpretation of the task. + + Multi-variant tasks have N entries; one carries ``primary=True`` + (used at interaction time for the user-sim grader and for the + cascading evaluator's first match). SQL lives in the consolidated + audited-gold JSONL, NOT inline here. + """ + model_config = ConfigDict(extra="forbid") + + variant_id: str + """Slug, e.g. ``canonical_only``, ``synonym_expanded``, ``primary``.""" + interpretation: str + """1-2 sentences describing what reading this variant embodies.""" + primary: bool = False + """Exactly one variant per task should have primary=True.""" + anchored_in: List[str] = Field(default_factory=list) + """Sources in the metadata that license this specific reading.""" + audited_gold_ref: AuditedGoldRef + notes: Optional[str] = None + + +class Provenance(BaseModel): + model_config = ConfigDict(extra="forbid") + + task_jsonl_path: str + """Where to find the original task row; e.g. ``mini_interact.jsonl``.""" + task_jsonl_instance_id: str + audited_gold_legacy_path: Optional[str] = None + """Pre-DEV-1515 callers may still reference per-DB JSONL paths; this + is a back-pointer for forensic traceability and is allowed to be + None for benchmarks that never had a per-DB layout.""" + + +class TaskAnnotation(BaseModel): + """Run-independent annotation of a single benchmark task. + + Path: ``annotations///.task.json``. + + For sufficient-and-correct tasks (the common case), ``gold_variants`` + is empty and ``original_gold_is_correct=True``; the evaluator can + short-circuit against the task's original ``sol_sql``. + """ + model_config = ConfigDict(extra="forbid") + + schema_version: Literal[1] = 1 + kind: Literal["task_annotation"] = "task_annotation" + + instance_id: str + selected_database: str + annotated_by: str + annotated_at: str + """ISO-8601 date or datetime.""" + supersedes: Optional[str] = None + """Path or stable ID of an earlier annotation this replaces.""" + + amb_user_query: str + external_knowledge: List[int] = Field(default_factory=list) + masked_terms: List[MaskedTerm] = Field(default_factory=list) + + metadata_sufficiency: MetadataSufficiency + + original_gold_is_correct: bool = False + """True iff metadata_sufficiency.verdict == 'sufficient' AND the + task's original ``sol_sql`` is the correct answer. When True, + ``gold_variants`` SHOULD be empty (the original gold is authoritative).""" + gold_variants: List[GoldVariantRef] = Field(default_factory=list) + + evaluator_prompt: Optional[str] = None + """Self-contained natural-language LLM-judge prompt. Invoked only + when ``metadata_sufficiency.verdict == 'insufficient'``. For + sufficient / ambiguous tasks this MAY be populated as a future-proof + fallback for novel readings but is not used by the deterministic + grader.""" + + provenance: Provenance + + +class VariantMatch(BaseModel): + model_config = ConfigDict(extra="forbid") + + variant_id: str + match: RowsetRelation + + +class SubmissionMetadata(BaseModel): + model_config = ConfigDict(extra="forbid") + + cloud_run_id: str + trajectory_path: str + submitted_sql_path: Optional[str] = None + """Optional pointer to a saved .sql sidecar. The submitted SQL is + also in the trajectory JSON's ``submitted_sql`` field.""" + predicted_row_count: Optional[int] = None + duration_s: Optional[float] = None + cost_usd_agent: Optional[float] = None + cost_usd_user_sim: Optional[float] = None + n_agent_turns: Optional[int] = None + n_ask_user_calls: Optional[int] = None + + +class SubmissionEvaluation(BaseModel): + """Cascading evaluation, most stringent → most lenient. + + Each field corresponds to one row in the after-run report layout. + """ + model_config = ConfigDict(extra="forbid") + + phase1_against_original_gold: PhaseVerdict + phase1_against_audited_primary: PhaseVerdict + phase1_against_any_audited_variant: PhaseVerdict + phase1_against_variants: List[VariantMatch] = Field(default_factory=list) + correct_up_to_tie_order: bool = False + novel_reading_judgment: Optional[PhaseVerdict] = None + """Populated only when ``metadata_sufficiency.verdict == + 'insufficient'`` AND the deterministic check returned no match.""" + verdict: SubmissionVerdict + matched_variant_id: Optional[str] = None + rationale: str = "" + + +class FailureClassification(BaseModel): + model_config = ConfigDict(extra="forbid") + + primary: FailurePrimary + secondary: List[FailurePrimary] = Field(default_factory=list) + agent_at_fault: bool + remediation_target: RemediationTarget + remediation_text: str = "" + details: str = "" + """Free-form clarification beyond the enum buckets.""" + + +class TrajectoryDecisionPoint(BaseModel): + model_config = ConfigDict(extra="forbid") + + trajectory_item_index: int + description: str + + +class UserSimResponseSummary(BaseModel): + model_config = ConfigDict(extra="forbid") + + trajectory_idx: int + summary: str + + +class UserSimInteraction(BaseModel): + """Holds the a-interact summary: how often the agent asked, what the + sim disclosed vs withheld. For one-shot benchmarks (livesqlbench) + everything is empty / zero.""" + model_config = ConfigDict(extra="forbid") + + n_asks: int = 0 + key_responses: List[UserSimResponseSummary] = Field(default_factory=list) + disclosed_resolutions: List[str] = Field(default_factory=list) + undisclosed_resolutions: List[str] = Field(default_factory=list) + + +class SubmissionAnnotation(BaseModel): + """Per-(instance, run) annotation. + + Path: + ``annotations///.submission..json``. + """ + model_config = ConfigDict(extra="forbid") + + schema_version: Literal[1] = 1 + kind: Literal["submission_annotation"] = "submission_annotation" + + instance_id: str + selected_database: str + task_annotation_ref: str + """Relative path to the task annotation, e.g. + ``annotations/mini-interact/households/households_10.task.json``.""" + annotated_by: str + annotated_at: str + + submission: SubmissionMetadata + evaluation: SubmissionEvaluation + failure_classification: FailureClassification + decision_point: Optional[TrajectoryDecisionPoint] = None + user_sim_interaction: UserSimInteraction = Field(default_factory=UserSimInteraction) diff --git a/tests/test_eval_annotation_schema.py b/tests/test_eval_annotation_schema.py new file mode 100644 index 00000000..11b0f441 --- /dev/null +++ b/tests/test_eval_annotation_schema.py @@ -0,0 +1,219 @@ +"""Smoke tests for the DEV-1515 annotation schemas + I/O helpers. + +Validates: +* Minimal-field construction succeeds for both annotation kinds. +* JSON round-trip preserves every field (write → read → equal). +* ``model_config = forbid`` rejects unknown top-level fields. +* Path helpers produce the agreed on-disk shape. +""" +from __future__ import annotations + +import json + +import pytest +from pydantic import ValidationError + +from bird_interact_agents.eval import ( + AuditedGoldRef, + FailureClassification, + GoldVariantRef, + MaskedTerm, + MetadataSufficiency, + SubmissionAnnotation, + SubmissionEvaluation, + SubmissionMetadata, + TaskAnnotation, + TrajectoryDecisionPoint, + UserSimInteraction, + VariantMatch, + read_submission_annotation, + read_task_annotation, + submission_annotation_path, + task_annotation_path, + write_submission_annotation, + write_task_annotation, +) +from bird_interact_agents.eval.annotation_schema import Provenance, UserSimResponseSummary + + +def _make_task_annotation() -> TaskAnnotation: + return TaskAnnotation( + instance_id="alien_42", + selected_database="alien", + annotated_by="test", + annotated_at="2026-05-31", + amb_user_query="Who is the alien?", + external_knowledge=[1, 2, 3], + masked_terms=[ + MaskedTerm( + term="alien", + type="knowledge_linking_ambiguity", + metadata_evidence=["alien_kb.jsonl#1"], + ) + ], + metadata_sufficiency=MetadataSufficiency( + verdict="ambiguous", + rationale="KB hedges; sampled values show variants", + evidence_sources_consulted=["alien_kb.jsonl#1"], + ), + gold_variants=[ + GoldVariantRef( + variant_id="canonical_only", + interpretation="KB literals only", + primary=True, + anchored_in=["alien_kb.jsonl#1"], + audited_gold_ref=AuditedGoldRef( + file="audited_gold/mini_interact_audited.jsonl", + instance_id="alien_42", + variant_id="canonical_only", + ), + notes="primary variant", + ) + ], + evaluator_prompt=None, + provenance=Provenance( + task_jsonl_path="mini_interact.jsonl", + task_jsonl_instance_id="alien_42", + audited_gold_legacy_path=None, + ), + ) + + +def _make_submission_annotation() -> SubmissionAnnotation: + return SubmissionAnnotation( + instance_id="alien_42", + selected_database="alien", + task_annotation_ref="annotations/mini-interact/alien/alien_42.task.json", + annotated_by="test", + annotated_at="2026-05-31", + submission=SubmissionMetadata( + cloud_run_id="20260531tXXXX", + trajectory_path="results/cloud/.../alien_42/attempt-1.json", + predicted_row_count=10, + duration_s=42.0, + cost_usd_agent=1.23, + cost_usd_user_sim=0.04, + n_agent_turns=37, + n_ask_user_calls=3, + ), + evaluation=SubmissionEvaluation( + phase1_against_original_gold="fail", + phase1_against_audited_primary="fail", + phase1_against_any_audited_variant="pass", + phase1_against_variants=[ + VariantMatch(variant_id="canonical_only", match="equal_rowset"), + ], + correct_up_to_tie_order=False, + verdict="valid_interpretation", + matched_variant_id="canonical_only", + rationale="matched canonical_only", + ), + failure_classification=FailureClassification( + primary="metadata_ambiguity", + secondary=["gold_audit_quality"], + agent_at_fault=False, + remediation_target="audit", + remediation_text="Re-audit per the audit-gold-sql contract.", + details="see analyses/raw/alien_42.md", + ), + decision_point=TrajectoryDecisionPoint( + trajectory_item_index=99, + description="user-sim locked canonical-only reading", + ), + user_sim_interaction=UserSimInteraction( + n_asks=3, + key_responses=[UserSimResponseSummary(trajectory_idx=99, summary="…")], + disclosed_resolutions=["alien"], + undisclosed_resolutions=["alien synonyms"], + ), + ) + + +def test_task_annotation_minimal_construct(): + ann = _make_task_annotation() + assert ann.kind == "task_annotation" + assert ann.schema_version == 1 + assert len(ann.gold_variants) == 1 + assert ann.gold_variants[0].primary is True + + +def test_submission_annotation_minimal_construct(): + ann = _make_submission_annotation() + assert ann.kind == "submission_annotation" + assert ann.evaluation.verdict == "valid_interpretation" + assert ann.failure_classification.primary == "metadata_ambiguity" + + +def test_task_annotation_roundtrip(tmp_path): + ann = _make_task_annotation() + p = tmp_path / "alien_42.task.json" + write_task_annotation(ann, p) + loaded = read_task_annotation(p) + assert loaded == ann + + +def test_submission_annotation_roundtrip(tmp_path): + ann = _make_submission_annotation() + p = tmp_path / "alien_42.submission.20260531t0001.json" + write_submission_annotation(ann, p) + loaded = read_submission_annotation(p) + assert loaded == ann + + +def test_task_annotation_forbid_extra(): + """Unknown top-level fields must fail validation — protects against + silent schema drift when consumers add fields the harness will ignore.""" + payload = _make_task_annotation().model_dump() + payload["a_field_that_should_not_exist"] = True + with pytest.raises(ValidationError): + TaskAnnotation.model_validate(payload) + + +def test_submission_annotation_forbid_extra(): + payload = _make_submission_annotation().model_dump() + payload["another_unknown_field"] = 42 + with pytest.raises(ValidationError): + SubmissionAnnotation.model_validate(payload) + + +def test_path_helpers(tmp_path): + t = task_annotation_path( + benchmark="mini-interact", + selected_database="alien", + instance_id="alien_42", + repo_root=tmp_path, + ) + assert t == tmp_path / "annotations" / "mini-interact" / "alien" / "alien_42.task.json" + s = submission_annotation_path( + benchmark="mini-interact", + selected_database="alien", + instance_id="alien_42", + run_id="20260531t1008-claudes-slayer-890419", + repo_root=tmp_path, + ) + assert s == ( + tmp_path + / "annotations" + / "mini-interact" + / "alien" + / "alien_42.submission.20260531t1008-claudes-slayer-890419.json" + ) + + +def test_written_json_is_valid_utf8_and_human_readable(tmp_path): + """A grep-friendly invariant: every annotation file is plain JSON + with 2-space indent. Catches accidental `.write_text` swaps to + binary / non-indented output.""" + ann = _make_task_annotation() + p = tmp_path / "x.task.json" + write_task_annotation(ann, p) + text = p.read_text() + assert text.endswith("\n") + # JSON is parseable and the indent is 2. + decoded = json.loads(text) + assert decoded["instance_id"] == "alien_42" + # Indent check: every continuation line of the body starts with ` ` (2 spaces). + body_lines = text.splitlines() + assert body_lines[0] == "{" + indent_lines = [l for l in body_lines[1:-1] if l.strip()] + assert all(l.startswith(" ") for l in indent_lines) From 56dc2b29ca485583bb35e27628dcd06968e74d61 Mon Sep 17 00:00:00 2001 From: Egor Kraev Date: Sun, 31 May 2026 18:51:44 +0200 Subject: [PATCH 05/29] README: document multi-instance parallelism for claude_sdk_otf{,_ainteract} MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The cloud-submit snippets shipped with `--workers 1 --actors-per-worker 1` which is correct for the 1-task smokes shown but silently serialises larger runs through a single actor. The DEV-1515 38-task remainder run inherited that and took ~5 hours instead of ~80 minutes (~25 min of cluster boot + 38 × ~7 min per task). Add a sizing note + a 10-instance example using `--actors-per-worker 4` on each framework. e2-standard-4 (4 vCPU / 16 GB) comfortably runs 4 concurrent Opus + Sonnet pairs since the workload is network-bound on the LLM APIs, so 4 is a safe default for 10-50 task batches. Co-Authored-By: Claude Opus 4.7 (1M context) --- README.md | 33 ++++++++++++++++++++++++++------- 1 file changed, 26 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index cc0ac303..bdf5895f 100644 --- a/README.md +++ b/README.md @@ -94,10 +94,10 @@ bird-interact --framework claude_sdk_otf --query-mode slayer \ --db-path ../livesqlbench-base-lite-sqlite/ ``` -Cloud smoke / baseline defaults (the deterministic OTF cache is uploaded -from local if present, else built locally first — like -`pydantic_ai_recursive`; no reference build or upload-back). `museum_6` is -the canonical single-task smoke (passes under these defaults): +Cloud smoke (the deterministic OTF cache is uploaded from local if +present, else built locally first — like `pydantic_ai_recursive`; no +reference build or upload-back). `museum_6` is the canonical single-task +smoke (passes under these defaults): ```bash env -u SSH_AUTH_SOCK uv run bird-interact-cloud submit \ @@ -112,6 +112,20 @@ env -u SSH_AUTH_SOCK uv run bird-interact-cloud submit \ --worker-type e2-standard-4 --max-runtime-hours 2 --detach ``` +For multi-instance runs bump `--actors-per-worker` so tasks run in +parallel — `1` is correct for a 1-task smoke but serialises everything +larger. `e2-standard-4` (4 vCPU / 16 GB) comfortably runs 4 concurrent +Opus + Sonnet pairs (mostly network-bound on LLM APIs), so 4 is a safe +default for 10-50 task runs: + +```bash +env -u SSH_AUTH_SOCK uv run bird-interact-cloud submit \ + ... \ + --instance-ids museum_1,museum_2,museum_3,museum_4,museum_5,museum_6,museum_7,museum_8,museum_9,museum_10 \ + --workers 1 --actors-per-worker 4 \ + --worker-type e2-standard-4 --max-runtime-hours 3 --detach +``` + #### `claude_sdk_otf_ainteract` — mini-interact / a-interact only Adds a native `ask_user` tool plus three PreToolUse/PostToolUse guards: @@ -129,9 +143,8 @@ bird-interact --framework claude_sdk_otf_ainteract --query-mode slayer \ --db-path /path/to/mini-interact/ --instance-id households_1 ``` -Cloud smoke / baseline defaults. `households_16` is the canonical -single-task smoke (passes both audited and original gold under these -defaults): +Cloud smoke. `households_16` is the canonical single-task smoke (passes +both audited and original gold under these defaults): ```bash env -u SSH_AUTH_SOCK uv run bird-interact-cloud submit \ @@ -145,6 +158,12 @@ env -u SSH_AUTH_SOCK uv run bird-interact-cloud submit \ --worker-type e2-standard-4 --max-runtime-hours 2 --detach ``` +For multi-instance runs bump `--actors-per-worker` to 4 (see the +`claude_sdk_otf` note above for the sizing rule). With the default 1×1 +a 53-task batch serialises through one actor and takes ~5 hours; at 1×4 +it finishes in ~80 minutes for the same per-task wallclock and identical +cluster cost. + ## 3-way comparison (original ↔ raw ↔ slayer) `scripts/run_three_way.sh` runs the upstream BIRD-Interact harness, our raw-SQL flavour, and our SLayer flavour on the same `instance_id` slice and emits a side-by-side `comparison.json`. From 83751ced50e6103ecbc724db03fe6df99fe63f08 Mon Sep 17 00:00:00 2001 From: Egor Kraev Date: Sun, 31 May 2026 21:29:58 +0200 Subject: [PATCH 06/29] DEV-1515 tolerant grader + cascading-phase1 report + annotation CLI MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Lands the 8-row cascading verdict (N1=original / N2=audited primary / N3=any audited variant / N4=tie-order / N5=LLM-judge / N6=numeric-eps / N7=trailing-whitespace / N8=column-order via case-insensitive name align), the inline-graded per-task SubmissionAnnotation produced by ray_app and the local run path, the no-overwrite fetch-side merge into /annotations/, and the offline regrade CLI. * `eval/tolerant_grader.py` — 8-row monotone cascade, ORDER BY parser, cell-level relaxations, content-hashed LLM-judge cache (model + annotation hash + gold-variants hash + normalized SQL; runId NOT in key so offline re-grade reuses cloud-side decisions), Tier 2 informational diff. * `eval/implicit_annotation.py` — schema-valid in-memory default for unannotated instances; cascade collapses to N1. * `eval/grade_in_place.py` — shared inline grader called by both ray_app (cloud) and run.py (local) so eval.json's `cascading_phase1` has source data on both paths. * `eval/cascading_report.py` — aggregator over per-row submission_annotation.json; enforces cascade monotonicity at aggregate time; rewrites legacy `phase1_count`/`phase1_rate` from cascade N1. * `eval/annotate.py` + `scripts/generate_annotation_skeletons.py` — skeleton CLI with init|refresh|force-all task modes, overwrite|init submission modes, dry-run, idempotent-preserve human edits. * `eval/regrade.py` — explicit OVERWRITE re-grade path; writes eval_regraded.json alongside historical eval.json. * `paths.annotations_root()` + Dockerfile.cloud `--build-context annotations=` + `image.data_hash` keyed under `annotations/` so worktree builds match main-checkout content. * `cloud/gcs.{write,read}_submission_annotation` + `cloud/post_run_merge.merge_submission_annotations` no-overwrite + schema-validated + audit-report. Legacy `phase1_passed_audited` / `phase1_passed_original` raw bool columns + their plumbing have been removed end-to-end: `run.py`, `results_db.py`, `cloud/collation.py`, `agents/_submit.py`, all 9 agent files. The dual-eval block in eval.json is replaced by `cascading_phase1`; `phase1_count`/`phase1_rate` stay as back-compat aliases for N1. Two-round Codex review (plan + tests) folded in; 20 findings addressed. Tests: 1800 passing (+96 net over the 1704 baseline; 12 obsolete dual-eval tests deleted alongside the legacy columns). New coverage: test_implicit_task_annotation, test_paths_annotations, test_schema_extension, test_tolerant_grader_{comparators,orchestration}, test_cascading_report, test_legacy_field_removal, test_local_run_cascading, test_eval_annotate_cli, test_regrade_cli, cloud/test_{image_annotations,inline_grader,fetch_annotation_merge}. Co-Authored-By: Claude Opus 4.7 (1M context) --- .gitignore | 5 + Dockerfile.cloud | 9 + scripts/generate_annotation_skeletons.py | 15 + src/bird_interact_agents/agents/_submit.py | 26 +- src/bird_interact_agents/agents/agno/agent.py | 2 - .../agents/claude_sdk/agent.py | 4 - .../agents/claude_sdk_otf/agent.py | 4 - .../agents/claude_sdk_otf_ainteract/agent.py | 4 - .../agents/mcp_agent/agent.py | 2 - .../agents/pydantic_ai/agent.py | 4 - .../agents/pydantic_ai_otf_encode/agent.py | 4 - .../agents/pydantic_ai_recursive/agent.py | 2 - .../agents/smolagents/agent.py | 2 - src/bird_interact_agents/cloud/cli.py | 2 + src/bird_interact_agents/cloud/collation.py | 28 +- src/bird_interact_agents/cloud/driver.py | 13 + src/bird_interact_agents/cloud/gcs.py | 39 + src/bird_interact_agents/cloud/image.py | 24 +- .../cloud/post_run_merge.py | 93 ++ src/bird_interact_agents/cloud/ray_app.py | 164 +++ src/bird_interact_agents/eval/__init__.py | 2 + src/bird_interact_agents/eval/annotate.py | 358 +++++++ .../eval/annotation_schema.py | 28 +- .../eval/cascading_report.py | 135 +++ .../eval/grade_in_place.py | 182 ++++ .../eval/implicit_annotation.py | 82 ++ src/bird_interact_agents/eval/regrade.py | 240 +++++ .../eval/tolerant_grader.py | 733 ++++++++++++++ src/bird_interact_agents/paths.py | 14 + src/bird_interact_agents/results_db.py | 30 +- src/bird_interact_agents/run.py | 53 +- tests/cloud/test_cli.py | 10 +- tests/cloud/test_collation.py | 87 +- tests/cloud/test_fetch_annotation_merge.py | 203 ++++ tests/cloud/test_image_annotations.py | 145 +++ tests/cloud/test_inline_grader.py | 109 ++ tests/test_cascading_report.py | 239 +++++ tests/test_claude_sdk_otf_agent.py | 3 - tests/test_claude_sdk_otf_ainteract_agent.py | 3 - tests/test_claude_sdk_usage.py | 3 - tests/test_dual_eval.py | 186 +--- tests/test_eval_annotate_cli.py | 405 ++++++++ tests/test_implicit_task_annotation.py | 115 +++ tests/test_legacy_field_removal.py | 93 ++ tests/test_local_run_cascading.py | 119 +++ tests/test_paths_annotations.py | 83 ++ tests/test_schema_extension.py | 125 +++ tests/test_tolerant_grader_comparators.py | 296 ++++++ tests/test_tolerant_grader_orchestration.py | 938 ++++++++++++++++++ 49 files changed, 5067 insertions(+), 398 deletions(-) create mode 100644 scripts/generate_annotation_skeletons.py create mode 100644 src/bird_interact_agents/eval/annotate.py create mode 100644 src/bird_interact_agents/eval/cascading_report.py create mode 100644 src/bird_interact_agents/eval/grade_in_place.py create mode 100644 src/bird_interact_agents/eval/implicit_annotation.py create mode 100644 src/bird_interact_agents/eval/regrade.py create mode 100644 src/bird_interact_agents/eval/tolerant_grader.py create mode 100644 tests/cloud/test_fetch_annotation_merge.py create mode 100644 tests/cloud/test_image_annotations.py create mode 100644 tests/cloud/test_inline_grader.py create mode 100644 tests/test_cascading_report.py create mode 100644 tests/test_eval_annotate_cli.py create mode 100644 tests/test_implicit_task_annotation.py create mode 100644 tests/test_legacy_field_removal.py create mode 100644 tests/test_local_run_cascading.py create mode 100644 tests/test_paths_annotations.py create mode 100644 tests/test_schema_extension.py create mode 100644 tests/test_tolerant_grader_comparators.py create mode 100644 tests/test_tolerant_grader_orchestration.py diff --git a/.gitignore b/.gitignore index bdb1cc7c..004b4491 100644 --- a/.gitignore +++ b/.gitignore @@ -24,3 +24,8 @@ _specs/ # Kept on disk for local `--use-audited-gold-sql` runs; never committed/pushed. audited_gold/ sar_audited_gold/ +# DEV-1515: per-task / per-submission annotations. Same posture as +# audited_gold/ — kept local to avoid contaminating the benchmark +# distribution; baked into the cloud image at build time via the +# `paths.annotations_root()` build context. +annotations/ diff --git a/Dockerfile.cloud b/Dockerfile.cloud index a7698766..f08a4e3f 100644 --- a/Dockerfile.cloud +++ b/Dockerfile.cloud @@ -80,5 +80,14 @@ RUN uv pip install --no-deps -e . # code-like, so they stay baked (unlike the raw dataset/gold). COPY --from=audited-gold . /app/bird-interact-agents/audited_gold/ +# DEV-1515: `annotations/` lives in the main checkout (gitignored, same +# posture as audited_gold/) — pulled in via BuildKit's +# `--build-context annotations=` arg set by +# `image.build_and_push` so the build runs from any worktree. The grader +# reads task + submission annotations from this dir via +# `paths.annotations_root()`, which resolves to `/app/bird-interact-agents/annotations/` +# inside the worker (via git's `--git-common-dir`). +COPY --from=annotations . /app/bird-interact-agents/annotations/ + ENV BIRD_RESULTS_ROOT=/tmp/results \ BIRD_INTERACT_AGENTS_CLOUD=1 diff --git a/scripts/generate_annotation_skeletons.py b/scripts/generate_annotation_skeletons.py new file mode 100644 index 00000000..3fd80f9b --- /dev/null +++ b/scripts/generate_annotation_skeletons.py @@ -0,0 +1,15 @@ +"""Thin wrapper around ``bird_interact_agents.eval.annotate`` — matches +the ``consolidate_mini_interact_audited.py`` / ``verify_audited_gold.py`` +discoverability pattern. + +Use directly:: + + python scripts/generate_annotation_skeletons.py \\ + --run-id --benchmark mini-interact +""" +from __future__ import annotations + +from bird_interact_agents.eval.annotate import main + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/bird_interact_agents/agents/_submit.py b/src/bird_interact_agents/agents/_submit.py index eadd323d..be56b437 100644 --- a/src/bird_interact_agents/agents/_submit.py +++ b/src/bird_interact_agents/agents/_submit.py @@ -195,8 +195,6 @@ def _diagnostic_payload( translation_failed: bool = False, dry_run_failed: bool = False, infrastructure_failed: bool = False, - phase1_passed_audited: bool | None = None, - phase1_passed_original: bool | None = None, phase1_observation_audited: str | None = None, phase1_observation_original: str | None = None, ) -> dict[str, Any]: @@ -246,11 +244,11 @@ def _diagnostic_payload( payload["phase2_observation"] = observation else: payload["phase1_observation"] = observation - # Dual-eval columns only populated when both golds were exercised. - if phase1_passed_audited is not None: - payload["phase1_passed_audited"] = phase1_passed_audited - if phase1_passed_original is not None: - payload["phase1_passed_original"] = phase1_passed_original + # DEV-1515: per-task pass-fail bool fields against audited/original + # gold have been REMOVED — all per-task verdicts now live in the + # SubmissionAnnotation (produced inline by grade_and_write). The + # observation snapshots are kept as diagnostic-only context for + # log inspection. if phase1_observation_audited is not None: payload["phase1_observation_audited"] = phase1_observation_audited if phase1_observation_original is not None: @@ -491,11 +489,10 @@ def submit_raw_sql(state: Any, sql: str) -> str: return msg + _budget_note(state) infra_failed = False - audited_p1 = original_p1 = None audited_obs = original_obs = None try: (observation, reward, p1, p2, finished, - audited_p1, original_p1, audited_obs, original_obs) = _dispatch_eval(state, sql) + _audited_p1, _original_p1, audited_obs, original_obs) = _dispatch_eval(state, sql) except Exception as e: # noqa: BLE001 logger.exception("execute_submit_action raised on %s", sql[:80]) observation = f"Error processing submission: {e}" @@ -510,8 +507,6 @@ def submit_raw_sql(state: Any, sql: str) -> str: p1=p1, p2=p2, infrastructure_failed=infra_failed, - phase1_passed_audited=audited_p1, - phase1_passed_original=original_p1, phase1_observation_audited=audited_obs, phase1_observation_original=original_obs, ) @@ -634,8 +629,6 @@ def _record(*, sql: str | None, observation: str | None, json_failed: bool = False, translation_failed: bool = False, dry_run_failed: bool = False, infrastructure_failed: bool = False, - phase1_passed_audited: bool | None = None, - phase1_passed_original: bool | None = None, phase1_observation_audited: str | None = None, phase1_observation_original: str | None = None) -> None: diag = _diagnostic_payload( @@ -648,8 +641,6 @@ def _record(*, sql: str | None, observation: str | None, translation_failed=translation_failed, dry_run_failed=dry_run_failed, infrastructure_failed=infrastructure_failed, - phase1_passed_audited=phase1_passed_audited, - phase1_passed_original=phase1_passed_original, phase1_observation_audited=phase1_observation_audited, phase1_observation_original=phase1_observation_original, ) @@ -729,11 +720,10 @@ def _record(*, sql: str | None, observation: str | None, return msg + _budget_note(state) infra_failed = False - audited_p1 = original_p1 = None audited_obs = original_obs = None try: (observation, reward, p1, p2, finished, - audited_p1, original_p1, audited_obs, original_obs) = _dispatch_eval(state, sql) + _audited_p1, _original_p1, audited_obs, original_obs) = _dispatch_eval(state, sql) except Exception as e: # noqa: BLE001 logger.exception("execute_submit_action raised on slayer-rendered SQL") observation = f"Error processing submission: {e}" @@ -746,8 +736,6 @@ def _record(*, sql: str | None, observation: str | None, reward=reward if reward is not None else 0.0, p1=p1, p2=p2, finished=finished, infrastructure_failed=infra_failed, - phase1_passed_audited=audited_p1, - phase1_passed_original=original_p1, phase1_observation_audited=audited_obs, phase1_observation_original=original_obs, ) diff --git a/src/bird_interact_agents/agents/agno/agent.py b/src/bird_interact_agents/agents/agno/agent.py index 3c36fcad..05f22517 100644 --- a/src/bird_interact_agents/agents/agno/agent.py +++ b/src/bird_interact_agents/agents/agno/agent.py @@ -334,8 +334,6 @@ async def _run_with_tools(tools_list: list) -> str: "trajectory": [{"final_output": output[:500]}], "error": None, "usage": state.usage.model_dump(), - "phase1_passed_audited": result.get("phase1_passed_audited"), - "phase1_passed_original": result.get("phase1_passed_original"), "phase1_observation_audited": result.get("phase1_observation_audited"), "phase1_observation_original": result.get("phase1_observation_original"), }, diff --git a/src/bird_interact_agents/agents/claude_sdk/agent.py b/src/bird_interact_agents/agents/claude_sdk/agent.py index 7d13f55d..c6520250 100644 --- a/src/bird_interact_agents/agents/claude_sdk/agent.py +++ b/src/bird_interact_agents/agents/claude_sdk/agent.py @@ -660,8 +660,6 @@ async def run_task( "trajectory": trajectory, "error": str(e), "usage": accum.model_dump(), - "phase1_passed_audited": result.get("phase1_passed_audited"), - "phase1_passed_original": result.get("phase1_passed_original"), "phase1_observation_audited": result.get("phase1_observation_audited"), "phase1_observation_original": result.get("phase1_observation_original"), }, @@ -688,8 +686,6 @@ async def run_task( "trajectory": trajectory, "error": None, "usage": accum.model_dump(), - "phase1_passed_audited": result.get("phase1_passed_audited"), - "phase1_passed_original": result.get("phase1_passed_original"), "phase1_observation_audited": result.get("phase1_observation_audited"), "phase1_observation_original": result.get("phase1_observation_original"), }, diff --git a/src/bird_interact_agents/agents/claude_sdk_otf/agent.py b/src/bird_interact_agents/agents/claude_sdk_otf/agent.py index 9c49690f..a2491211 100644 --- a/src/bird_interact_agents/agents/claude_sdk_otf/agent.py +++ b/src/bird_interact_agents/agents/claude_sdk_otf/agent.py @@ -404,8 +404,6 @@ async def run_task( "trajectory": trajectory, "error": str(e), "usage": accum.model_dump(), - "phase1_passed_audited": result.get("phase1_passed_audited"), - "phase1_passed_original": result.get("phase1_passed_original"), "phase1_observation_audited": result.get("phase1_observation_audited"), "phase1_observation_original": result.get("phase1_observation_original"), }, @@ -432,8 +430,6 @@ async def run_task( "trajectory": trajectory, "error": None, "usage": accum.model_dump(), - "phase1_passed_audited": result.get("phase1_passed_audited"), - "phase1_passed_original": result.get("phase1_passed_original"), "phase1_observation_audited": result.get("phase1_observation_audited"), "phase1_observation_original": result.get("phase1_observation_original"), }, diff --git a/src/bird_interact_agents/agents/claude_sdk_otf_ainteract/agent.py b/src/bird_interact_agents/agents/claude_sdk_otf_ainteract/agent.py index 948e3fcf..07889e9c 100644 --- a/src/bird_interact_agents/agents/claude_sdk_otf_ainteract/agent.py +++ b/src/bird_interact_agents/agents/claude_sdk_otf_ainteract/agent.py @@ -398,8 +398,6 @@ async def run_task( "trajectory": trajectory, "error": str(e), "usage": accum.model_dump(), - "phase1_passed_audited": result.get("phase1_passed_audited"), - "phase1_passed_original": result.get("phase1_passed_original"), "phase1_observation_audited": result.get("phase1_observation_audited"), "phase1_observation_original": result.get("phase1_observation_original"), }, @@ -426,8 +424,6 @@ async def run_task( "trajectory": trajectory, "error": None, "usage": accum.model_dump(), - "phase1_passed_audited": result.get("phase1_passed_audited"), - "phase1_passed_original": result.get("phase1_passed_original"), "phase1_observation_audited": result.get("phase1_observation_audited"), "phase1_observation_original": result.get("phase1_observation_original"), }, diff --git a/src/bird_interact_agents/agents/mcp_agent/agent.py b/src/bird_interact_agents/agents/mcp_agent/agent.py index 5fda0cc8..ce6b9502 100644 --- a/src/bird_interact_agents/agents/mcp_agent/agent.py +++ b/src/bird_interact_agents/agents/mcp_agent/agent.py @@ -375,8 +375,6 @@ async def run_task( "trajectory": [{"final_output": str(output)[:500]}], "error": None, "usage": state.usage.model_dump(), - "phase1_passed_audited": result.get("phase1_passed_audited"), - "phase1_passed_original": result.get("phase1_passed_original"), "phase1_observation_audited": result.get("phase1_observation_audited"), "phase1_observation_original": result.get("phase1_observation_original"), }, diff --git a/src/bird_interact_agents/agents/pydantic_ai/agent.py b/src/bird_interact_agents/agents/pydantic_ai/agent.py index e760ffc5..7b558d92 100644 --- a/src/bird_interact_agents/agents/pydantic_ai/agent.py +++ b/src/bird_interact_agents/agents/pydantic_ai/agent.py @@ -538,8 +538,6 @@ async def run_task( "gold_result_json": partial.get("gold_result_json"), "n_agent_turns": n_agent_turns, "tool_call_stats": tool_stats, - "phase1_passed_audited": partial.get("phase1_passed_audited"), - "phase1_passed_original": partial.get("phase1_passed_original"), "phase1_observation_audited": partial.get("phase1_observation_audited"), "phase1_observation_original": partial.get("phase1_observation_original"), }, @@ -574,8 +572,6 @@ async def run_task( "gold_result_json": result.get("gold_result_json"), "n_agent_turns": n_agent_turns, "tool_call_stats": tool_stats, - "phase1_passed_audited": result.get("phase1_passed_audited"), - "phase1_passed_original": result.get("phase1_passed_original"), "phase1_observation_audited": result.get("phase1_observation_audited"), "phase1_observation_original": result.get("phase1_observation_original"), }, diff --git a/src/bird_interact_agents/agents/pydantic_ai_otf_encode/agent.py b/src/bird_interact_agents/agents/pydantic_ai_otf_encode/agent.py index 5365fb00..f2709484 100644 --- a/src/bird_interact_agents/agents/pydantic_ai_otf_encode/agent.py +++ b/src/bird_interact_agents/agents/pydantic_ai_otf_encode/agent.py @@ -1562,8 +1562,6 @@ def _finalize( "tool_call_stats": tool_stats, # Dual-eval fields — parity with the recursive adapter (merged from # master); populated only under --use-audited-gold-sql, NULL otherwise. - "phase1_passed_audited": submitter.get("phase1_passed_audited"), - "phase1_passed_original": submitter.get("phase1_passed_original"), "phase1_observation_audited": submitter.get("phase1_observation_audited"), "phase1_observation_original": submitter.get("phase1_observation_original"), # DEV-1454-specific: per-task KB encode registry. @@ -1609,8 +1607,6 @@ def _minimal_error_row( "gold_result_json": None, "n_agent_turns": None, "tool_call_stats": None, - "phase1_passed_audited": None, - "phase1_passed_original": None, "phase1_observation_audited": None, "phase1_observation_original": None, "kb_encoded": [], diff --git a/src/bird_interact_agents/agents/pydantic_ai_recursive/agent.py b/src/bird_interact_agents/agents/pydantic_ai_recursive/agent.py index 228c6138..8af2ab7f 100644 --- a/src/bird_interact_agents/agents/pydantic_ai_recursive/agent.py +++ b/src/bird_interact_agents/agents/pydantic_ai_recursive/agent.py @@ -917,8 +917,6 @@ def _finalize( "tool_call_stats": tool_stats, # Dual-eval fields — populated only when --use-audited-gold-sql # is on AND the overlay applied; NULL elsewhere. - "phase1_passed_audited": submitter.get("phase1_passed_audited"), - "phase1_passed_original": submitter.get("phase1_passed_original"), "phase1_observation_audited": submitter.get("phase1_observation_audited"), "phase1_observation_original": submitter.get("phase1_observation_original"), } diff --git a/src/bird_interact_agents/agents/smolagents/agent.py b/src/bird_interact_agents/agents/smolagents/agent.py index a8b08cdb..6c6f4113 100644 --- a/src/bird_interact_agents/agents/smolagents/agent.py +++ b/src/bird_interact_agents/agents/smolagents/agent.py @@ -438,8 +438,6 @@ async def run_task( "trajectory": [{"final_output": output[:500]}], "error": None, "usage": state.usage.model_dump(), - "phase1_passed_audited": result.get("phase1_passed_audited"), - "phase1_passed_original": result.get("phase1_passed_original"), "phase1_observation_audited": result.get("phase1_observation_audited"), "phase1_observation_original": result.get("phase1_observation_original"), }, diff --git a/src/bird_interact_agents/cloud/cli.py b/src/bird_interact_agents/cloud/cli.py index 675e445f..e9341d2d 100644 --- a/src/bird_interact_agents/cloud/cli.py +++ b/src/bird_interact_agents/cloud/cli.py @@ -248,10 +248,12 @@ def main(argv: Sequence[str] | None = None) -> int: repo_root, paths.audited_gold_root(), allow_dirty=False, + annotations_root=paths.annotations_root(), ) uri = image.build_and_push( tag, repo_root, audited_gold_root=paths.audited_gold_root(), + annotations_root=paths.annotations_root(), force=ns.force, ) print(uri) diff --git a/src/bird_interact_agents/cloud/collation.py b/src/bird_interact_agents/cloud/collation.py index 6e17237b..377a225e 100644 --- a/src/bird_interact_agents/cloud/collation.py +++ b/src/bird_interact_agents/cloud/collation.py @@ -91,14 +91,6 @@ def _row_to_task_result_row(manifest: dict, r: dict) -> TaskResultRow: gold_result_json=r.get("gold_result_json"), n_agent_turns=int(n_turns) if isinstance(n_turns, int) else None, tool_call_stats_json=tool_call_stats_json, - # Dual-eval columns — present in the row JSON / eval.json whenever the - # overlay ran; must be carried into results.db too (the local - # `run.py` writer already does this — collation must match or the - # cloud DB silently nulls the original-gold score). - phase1_passed_audited=r.get("phase1_passed_audited"), - phase1_passed_original=r.get("phase1_passed_original"), - phase1_observation_audited=r.get("phase1_observation_audited"), - phase1_observation_original=r.get("phase1_observation_original"), ) @@ -121,17 +113,10 @@ def _build_metrics(manifest: dict, canonical_rows: list[dict], durations = [float(r.get("duration_s") or 0.0) for r in canonical_rows] resubmitted_ids = sorted(iid for iid, lst in attempts.items() if len(lst) > 1) - # Dual-eval breakdown — parity with the local `run.py` metrics. Populated - # only when --use-audited-gold-sql ran (else the per-row fields are NULL - # and these counts collapse to 0). `phase1_*_original` is the canonical- - # gold score; `phase1_*_audited` mirrors `phase1_count` (phase1 evaluates - # against the overlaid/audited gold). - dual_audited = [r.get("phase1_passed_audited") for r in canonical_rows] - dual_original = [r.get("phase1_passed_original") for r in canonical_rows] - n_dual = sum(1 for x in dual_audited if x is not None) - p1_audited = sum(1 for x in dual_audited if x) - p1_original = sum(1 for x in dual_original if x) - + # DEV-1515: the legacy dual-eval breakdown is replaced by the + # cascading_phase1 block built downstream by + # `emit_cascading_eval_json` over per-row submission_annotation.json + # files (merged in by `driver.fetch` via `merge_submission_annotations`). return { "mode": manifest["mode"], "query_mode": manifest["query_mode"], @@ -151,11 +136,6 @@ def _build_metrics(manifest: dict, canonical_rows: list[dict], "results": canonical_rows, "n_resubmitted": len(resubmitted_ids), "resubmitted_ids": resubmitted_ids, - "n_dual_eval_tasks": n_dual, - "phase1_count_audited": p1_audited, - "phase1_count_original": p1_original, - "phase1_rate_audited": p1_audited / n_dual if n_dual else 0, - "phase1_rate_original": p1_original / n_dual if n_dual else 0, } diff --git a/src/bird_interact_agents/cloud/driver.py b/src/bird_interact_agents/cloud/driver.py index 6560cfba..05a4f876 100644 --- a/src/bird_interact_agents/cloud/driver.py +++ b/src/bird_interact_agents/cloud/driver.py @@ -508,10 +508,12 @@ def submit(args) -> str: repo_root, paths.audited_gold_root(), allow_dirty=args.allow_dirty, + annotations_root=paths.annotations_root(), ) image_uri = image.build_and_push( tag, repo_root, audited_gold_root=paths.audited_gold_root(), + annotations_root=paths.annotations_root(), force=False, ) # De-bake: upload the benchmark dataset ONCE to its content-hashed GCS @@ -703,6 +705,17 @@ def fetch(run_id: str) -> dict: ), ) metrics["merge_report"] = merge_report + + # DEV-1515: merge per-task submission_annotation.json files into + # `/annotations///.submission..json`. + # No-overwrite-if-present; schema-validated; audit log persisted. + annotation_merge = _post_run_merge.merge_submission_annotations( + downloaded_run_dir=dest, + run_id=run_id, + benchmark=_benchmark_for_dataset(manifest.get("dataset")), + main_checkout_root=paths.main_checkout_root(), + ) + metrics["annotation_merge_report"] = annotation_merge.model_dump() return metrics diff --git a/src/bird_interact_agents/cloud/gcs.py b/src/bird_interact_agents/cloud/gcs.py index 4f8520c1..f751c231 100644 --- a/src/bird_interact_agents/cloud/gcs.py +++ b/src/bird_interact_agents/cloud/gcs.py @@ -54,6 +54,13 @@ def log_blob(run_id: str, instance_id: str, attempt: int) -> str: return f"runs/{run_id}/logs/{instance_id}/attempt-{attempt}.log" +def submission_annotation_blob(run_id: str, instance_id: str) -> str: + """DEV-1515: per-task submission annotation blob path. One file per + (run, instance); the in-cloud grader writes it once per task; the + fetch path downloads + merges to ``/annotations/``.""" + return f"runs/{run_id}/rows/{instance_id}/submission_annotation.json" + + # --------------------------------------------------------------------------- # Writers # --------------------------------------------------------------------------- @@ -87,6 +94,38 @@ def write_log( blob.upload_from_string(text, content_type="text/plain") +def write_submission_annotation( + run_id: str, + instance_id: str, + annotation: dict, + *, + client=None, +) -> None: + """DEV-1515: upload a per-task SubmissionAnnotation. The cloud + worker calls this once per task right after ``grade_and_write``.""" + client = client or default_gcs_client() + blob = client.bucket(BUCKET_NAME).blob( + submission_annotation_blob(run_id, instance_id), + ) + blob.upload_from_string( + json.dumps(annotation, indent=2).encode(), + content_type="application/json", + ) + + +def read_submission_annotation( + run_id: str, + instance_id: str, + *, + client=None, +) -> dict: + client = client or default_gcs_client() + blob = client.bucket(BUCKET_NAME).blob( + submission_annotation_blob(run_id, instance_id), + ) + return json.loads(blob.download_as_bytes()) + + def write_manifest(run_id: str, manifest: dict, *, client=None) -> None: client = client or default_gcs_client() blob = client.bucket(BUCKET_NAME).blob(manifest_blob(run_id)) diff --git a/src/bird_interact_agents/cloud/image.py b/src/bird_interact_agents/cloud/image.py index a92b1506..09e91d6a 100644 --- a/src/bird_interact_agents/cloud/image.py +++ b/src/bird_interact_agents/cloud/image.py @@ -86,6 +86,8 @@ def _split_dockerfile_sections(repo_root: Path) -> dict[str, str]: def data_hash( repo_root: Path, audited_gold_root: Path, + *, + annotations_root: Path | None = None, ) -> str: """Content-based hash over the inputs that compose the DATA layers of `Dockerfile.cloud`: ``audited_gold/`` and the Dockerfile's DATA @@ -127,6 +129,18 @@ def data_hash( h.update(f.read_bytes()) h.update(b"\x00") + # DEV-1515: annotations/ — same posture as audited_gold/. Keyed + # under ``annotations/`` so a worktree build produces the + # same digest as a main-checkout build with identical content. + if annotations_root is not None and annotations_root.exists(): + for f in _iter_files_under(annotations_root): + rel = f.relative_to(annotations_root) + h.update(b"repo/") + h.update(f"annotations/{rel.as_posix()}".encode()) + h.update(b"\x00") + h.update(f.read_bytes()) + h.update(b"\x00") + # Dockerfile DATA-LAYERS section sections = _split_dockerfile_sections(repo_root) h.update(b"dockerfile-data/") @@ -183,13 +197,15 @@ def image_tag( audited_gold_root: Path, *, allow_dirty: bool, + annotations_root: Path | None = None, ) -> str: """`-` (+ `-dirty` when `allow_dirty=True` and the worktree is dirty). See :func:`data_hash` for why ``audited_gold_root`` is a separate input - (worktree-safety).""" - dh = data_hash(repo_root, audited_gold_root) + (worktree-safety). ``annotations_root`` is the DEV-1515 sibling input — + same rationale.""" + dh = data_hash(repo_root, audited_gold_root, annotations_root=annotations_root) ch = code_hash(repo_root, allow_dirty=allow_dirty) tag = f"{dh[:12]}-{ch[:12]}" if allow_dirty and _dirty_image_input_paths(repo_root): @@ -266,6 +282,7 @@ def build_and_push( *, image_uri_prefix: str | None = None, audited_gold_root: Path | None = None, + annotations_root: Path | None = None, force: bool = False, ) -> str: """Build (if needed) and push the image, returning the full URI. @@ -295,6 +312,8 @@ def build_and_push( image_uri_prefix = config.image_uri_prefix() if audited_gold_root is None: audited_gold_root = paths.audited_gold_root() + if annotations_root is None: + annotations_root = paths.annotations_root() uri = f"{image_uri_prefix}:{tag}" if not force: probe = subprocess.run( @@ -308,6 +327,7 @@ def build_and_push( [ "docker", "build", "--build-context", f"audited-gold={audited_gold_root}", + "--build-context", f"annotations={annotations_root}", "-t", uri, "-f", "Dockerfile.cloud", ".", diff --git a/src/bird_interact_agents/cloud/post_run_merge.py b/src/bird_interact_agents/cloud/post_run_merge.py index 8bbe5f44..60356a7d 100644 --- a/src/bird_interact_agents/cloud/post_run_merge.py +++ b/src/bird_interact_agents/cloud/post_run_merge.py @@ -414,3 +414,96 @@ def merge_post_run_into_warm_cache( pass return report + + +# --------------------------------------------------------------------------- +# DEV-1515: fetch-side merge of per-row submission_annotation.json files +# into the main checkout's annotations/ tree. Same posture as the OTF +# reference merge above — no-overwrite, schema-validated, auditable. +# --------------------------------------------------------------------------- + + +from pydantic import BaseModel, ConfigDict + + +class AnnotationMergeReport(BaseModel): + model_config = ConfigDict(extra="forbid") + + run_id: str + benchmark: str + merged: int = 0 + skipped_existing: int = 0 + rejected_invalid: int = 0 + merged_paths: list[str] = [] + skipped_paths: list[str] = [] + rejected_paths: list[str] = [] + + +def merge_submission_annotations( + *, + downloaded_run_dir: Path, + run_id: str, + benchmark: str, + main_checkout_root: Path, +) -> AnnotationMergeReport: + """Walk ``/rows//submission_annotation.json`` + and merge each into ``/annotations///.submission..json``. + + Contract: + * No-overwrite-if-present. A pre-existing destination is preserved + (logged as ``skipped_existing``). + * Each candidate is schema-validated via + :class:`bird_interact_agents.eval.annotation_schema.SubmissionAnnotation`. + Invalid files are recorded as ``rejected_invalid``; they do NOT + create a destination file. + * Writes an audit report at + ``/annotation_merge_report.json``. + """ + from bird_interact_agents.eval.annotation_io import ( + submission_annotation_path, + ) + from bird_interact_agents.eval.annotation_schema import ( + SubmissionAnnotation, + ) + from pydantic import ValidationError + + rows_dir = downloaded_run_dir / "rows" + report = AnnotationMergeReport(run_id=run_id, benchmark=benchmark) + if rows_dir.exists(): + for sub in sorted(p for p in rows_dir.iterdir() if p.is_dir()): + src = sub / "submission_annotation.json" + if not src.exists(): + continue + try: + ann = SubmissionAnnotation.model_validate_json( + src.read_text(), + ) + except (ValidationError, ValueError) as e: + report.rejected_invalid += 1 + report.rejected_paths.append( + f"{src}: {type(e).__name__}: {e}" + ) + continue + dest = submission_annotation_path( + benchmark=benchmark, + selected_database=ann.selected_database, + instance_id=ann.instance_id, + run_id=run_id, + repo_root=main_checkout_root, + ) + if dest.exists(): + report.skipped_existing += 1 + report.skipped_paths.append(str(dest)) + continue + dest.parent.mkdir(parents=True, exist_ok=True) + dest.write_text( + ann.model_dump_json(indent=2, exclude_none=False) + "\n", + ) + report.merged += 1 + report.merged_paths.append(str(dest)) + + # Audit log lives alongside the downloaded run dir (next to the + # OTF merge_report.json above, for symmetry). + audit_path = downloaded_run_dir / "annotation_merge_report.json" + audit_path.write_text(report.model_dump_json(indent=2) + "\n") + return report diff --git a/src/bird_interact_agents/cloud/ray_app.py b/src/bird_interact_agents/cloud/ray_app.py index f6369289..efb8450f 100644 --- a/src/bird_interact_agents/cloud/ray_app.py +++ b/src/bird_interact_agents/cloud/ray_app.py @@ -27,6 +27,10 @@ from bird_interact_agents.cloud import benchmark_data as _benchmark_data from bird_interact_agents.cloud import gcs as _gcs from bird_interact_agents.cloud import upload_back as _upload_back +from bird_interact_agents.eval.grade_in_place import grade_and_write +from bird_interact_agents.eval.implicit_annotation import ( + implicit_task_annotation, +) # --------------------------------------------------------------------------- @@ -38,6 +42,126 @@ def default_gcs_client(): return _gcs.default_gcs_client() +# --------------------------------------------------------------------------- +# DEV-1515: inline grader hook called per task after a successful submit. +# Bridges between the worker's per-task state and the shared +# `grade_in_place.grade_and_write` helper. The aggregator + fetch path +# consume the resulting submission_annotation.json — no `phase1_passed_*` +# raw fields are emitted from this path. +# --------------------------------------------------------------------------- + + +def _load_task_annotation_or_implicit( + *, instance_id: str, selected_database: str, benchmark: str, + amb_user_query: str = "", +): + """Try to read ``///.task.json``; + if missing, fall back to the in-memory implicit default. NEVER writes + a synthesized stub to disk.""" + from bird_interact_agents import paths + from bird_interact_agents.eval.annotation_io import ( + read_task_annotation, task_annotation_path, + ) + + p = task_annotation_path( + benchmark=benchmark, selected_database=selected_database, + instance_id=instance_id, repo_root=paths.main_checkout_root(), + ) + if p.exists(): + return read_task_annotation(p) + return implicit_task_annotation( + instance_id=instance_id, + selected_database=selected_database, + benchmark=benchmark, + amb_user_query=amb_user_query, + ) + + +def _load_audited_gold_rows_for( + *, benchmark: str, instance_id: str, +) -> list[dict]: + """Load every audited-gold row for ``instance_id`` from the + consolidated JSONL. Empty list when no rows exist (graceful default + — see ``implicit_annotation``).""" + from bird_interact_agents import paths + from bird_interact_agents.benchmark import get_benchmark + + try: + bench = get_benchmark(benchmark.replace("-", "_")) + except Exception: # noqa: BLE001 + return [] + if getattr(bench, "audited_gold_layout", None) != "single_file": + return [] + consolidated = paths.audited_gold_root() / f"{bench.name}_audited.jsonl" + if not consolidated.exists(): + return [] + out: list[dict] = [] + for line in consolidated.read_text().splitlines(): + line = line.strip() + if not line: + continue + try: + row = json.loads(line) + except json.JSONDecodeError: + continue + if row.get("instance_id") == instance_id: + out.append(row) + return out + + +def _grade_one_submission( + *, + task_data: dict, + submitted_sql: str, + rows_dir: Path, + run_id: str, + benchmark: str, + db_path: Path, + conn: Any = None, + cost_usd_agent: float | None = None, + cost_usd_user_sim: float | None = None, + duration_s: float | None = None, + n_agent_turns: int | None = None, + n_ask_user_calls: int | None = None, + predicted_row_count: int | None = None, +) -> Path: + """Inline-grade one submission and write the per-row + ``submission_annotation.json``. Idempotent at the per-(task, run) + level — the cloud fetch path is no-overwrite at the destination.""" + instance_id = task_data["instance_id"] + selected_database = task_data["selected_database"] + ann = _load_task_annotation_or_implicit( + instance_id=instance_id, + selected_database=selected_database, + benchmark=benchmark, + amb_user_query=task_data.get("amb_user_query", ""), + ) + audited_rows = _load_audited_gold_rows_for( + benchmark=benchmark, instance_id=instance_id, + ) + return grade_and_write( + rows_dir=rows_dir, + instance_id=instance_id, + benchmark=benchmark, + run_id=run_id, + task_annotation=ann, + audited_gold_rows=audited_rows, + original_sol_sql=list( + task_data.get("original_sol_sql") or task_data.get("sol_sql") or [], + ), + submitted_sql=submitted_sql, + db_path=db_path, + conn=conn, + trajectory_path=f"rows/{instance_id}/attempt-1.json", + cost_usd_agent=cost_usd_agent, + cost_usd_user_sim=cost_usd_user_sim, + duration_s=duration_s, + n_agent_turns=n_agent_turns, + n_ask_user_calls=n_ask_user_calls, + predicted_row_count=predicted_row_count, + ) + + # --------------------------------------------------------------------------- # SLayer setup download (DEV-1468 — once per worker process; mirrors local) # --------------------------------------------------------------------------- @@ -545,6 +669,46 @@ def _run_one_in_actor( pass _gcs.write_row(run_id, iid, attempt, row, client=gcs_client) + + # DEV-1515: inline grader produces a SubmissionAnnotation per task + # (cascading verdict + Tier 2 informational). Failure here MUST NOT + # block the row/log upload — it's diagnostic, not result-of-record. + _grader_data_dir = locals().get("data_dir") + try: + if _grader_data_dir is None: + raise RuntimeError("data_dir unbound; grader skipped") + annotation_dir = Path(tempfile.mkdtemp(prefix="bird_submission_annot_")) + ann_path = _grade_one_submission( + task_data=task_data, + submitted_sql=str(row.get("submitted_sql") or ""), + rows_dir=annotation_dir, + run_id=run_id, + benchmark=_cloud_benchmark(cfg), + db_path=Path(_grader_data_dir) + / str(task_data.get("selected_database", "")) + / f"{task_data.get('selected_database', '')}.sqlite", + cost_usd_agent=row.get("usage", {}).get("cost_usd_agent") + if isinstance(row.get("usage"), dict) else None, + cost_usd_user_sim=row.get("usage", {}).get("cost_usd_user_sim") + if isinstance(row.get("usage"), dict) else None, + duration_s=row.get("duration_s"), + n_agent_turns=row.get("usage", {}).get("n_agent_turns") + if isinstance(row.get("usage"), dict) else None, + n_ask_user_calls=row.get("usage", {}).get("n_ask_user_calls") + if isinstance(row.get("usage"), dict) else None, + predicted_row_count=None, + ) + try: + _gcs.write_submission_annotation( + run_id, iid, json.loads(ann_path.read_text()), + client=gcs_client, + ) + finally: + shutil.rmtree(annotation_dir, ignore_errors=True) + except Exception: # noqa: BLE001 + # Diagnostic — never let grader failure cascade into a task fail. + traceback.print_exc() + try: log_bytes = log_tmp.read_bytes() if log_tmp.exists() else b"" except OSError: diff --git a/src/bird_interact_agents/eval/__init__.py b/src/bird_interact_agents/eval/__init__.py index bf41e6d5..72cd9241 100644 --- a/src/bird_interact_agents/eval/__init__.py +++ b/src/bird_interact_agents/eval/__init__.py @@ -30,6 +30,7 @@ TaskAnnotation, TrajectoryDecisionPoint, UserSimInteraction, + VariantInformational, VariantMatch, ) from bird_interact_agents.eval.annotation_io import ( @@ -53,6 +54,7 @@ "TaskAnnotation", "TrajectoryDecisionPoint", "UserSimInteraction", + "VariantInformational", "VariantMatch", "read_submission_annotation", "read_task_annotation", diff --git a/src/bird_interact_agents/eval/annotate.py b/src/bird_interact_agents/eval/annotate.py new file mode 100644 index 00000000..18ba7d67 --- /dev/null +++ b/src/bird_interact_agents/eval/annotate.py @@ -0,0 +1,358 @@ +"""DEV-1515: annotation-skeleton generator. + +Builds the per-task / per-submission JSON skeletons mechanically from a +task row + per-row attempt artefacts. Human-judgment fields are left as +``PENDING_HUMAN_REVIEW`` sentinels; modes control overwrite semantics. + +Usage:: + + python -m bird_interact_agents.eval.annotate \\ + --run-id --benchmark mini-interact \\ + [--instance-ids ...] \\ + [--task-mode {init,refresh,force-all}] \\ + [--submission-mode {overwrite,init}] \\ + [--dry-run] +""" +from __future__ import annotations + +import argparse +import datetime as _dt +import json +from pathlib import Path +from typing import Any, Callable, Iterable, List, Optional + +from bird_interact_agents import paths +from bird_interact_agents.benchmark import benchmark_names, get_benchmark +from bird_interact_agents.eval.annotation_io import ( + read_submission_annotation, + read_task_annotation, + submission_annotation_path, + task_annotation_path, + write_submission_annotation, + write_task_annotation, +) +from bird_interact_agents.eval.annotation_schema import ( + AuditedGoldRef, + FailureClassification, + GoldVariantRef, + MaskedTerm, + MetadataSufficiency, + Provenance, + SubmissionAnnotation, + SubmissionEvaluation, + SubmissionMetadata, + TaskAnnotation, + UserSimInteraction, + UserSimResponseSummary, +) + +PENDING_HUMAN_REVIEW = "PENDING_HUMAN_REVIEW" + + +# --------------------------------------------------------------------------- +# Mechanical builders — pure, no I/O +# --------------------------------------------------------------------------- + + +def _benchmark_task_jsonl_name(benchmark: str) -> str: + if benchmark in set(benchmark_names()): + return get_benchmark(benchmark).data_file + return f"{benchmark}.jsonl" + + +def _masked_terms_from(task_row: dict) -> list[MaskedTerm]: + amb = (task_row.get("user_query_ambiguity") or {}).get("critical_ambiguity", []) + out: list[MaskedTerm] = [] + for entry in amb or []: + out.append(MaskedTerm( + term=entry.get("term", ""), + type=entry.get("type", "intent_ambiguity"), + is_mask=bool(entry.get("is_mask", True)), + metadata_evidence=list(entry.get("metadata_evidence", []) or []), + )) + return out + + +def generate_task_annotation( + *, + task_row: dict, + benchmark: str, + annotated_at: Optional[str] = None, +) -> TaskAnnotation: + """Mechanically-filled TaskAnnotation skeleton; human-judgment fields + carry ``PENDING_HUMAN_REVIEW`` sentinels.""" + if annotated_at is None: + annotated_at = _dt.datetime.now(_dt.timezone.utc).replace( + microsecond=0, + ).isoformat() + return TaskAnnotation( + instance_id=task_row["instance_id"], + selected_database=task_row["selected_database"], + annotated_by="auto-skeleton", + annotated_at=annotated_at, + amb_user_query=task_row.get("amb_user_query", ""), + external_knowledge=list(task_row.get("external_knowledge", []) or []), + masked_terms=_masked_terms_from(task_row), + metadata_sufficiency=MetadataSufficiency( + verdict="ambiguous", # safe non-judgemental default + rationale=PENDING_HUMAN_REVIEW, + evidence_sources_consulted=[], + ), + original_gold_is_correct=False, + gold_variants=[], + evaluator_prompt=None, + provenance=Provenance( + task_jsonl_path=_benchmark_task_jsonl_name(benchmark), + task_jsonl_instance_id=task_row["instance_id"], + ), + ) + + +def _user_sim_interaction_from_trajectory(traj: list[dict]) -> UserSimInteraction: + n_asks = 0 + responses: list[UserSimResponseSummary] = [] + for i, item in enumerate(traj or []): + if item.get("role") == "tool_call" and item.get("name") == "ask_user": + n_asks += 1 + elif item.get("role") in ("user_sim", "user") and i > 0: + prev = traj[i - 1] + if prev.get("role") == "tool_call" and prev.get("name") == "ask_user": + # Recently followed an ask — record short summary. + txt = str(item.get("content") or "") + responses.append(UserSimResponseSummary( + trajectory_idx=i, + summary=(txt[:80] + "…") if len(txt) > 80 else txt, + )) + return UserSimInteraction( + n_asks=n_asks, + key_responses=responses, + disclosed_resolutions=[], + undisclosed_resolutions=[], + ) + + +def _eval_from_cascade(cascade: Any, epsilon: float = 1e-6) -> SubmissionEvaluation: + """Convert a tolerant_grader CascadeVerdict into a + SubmissionEvaluation persistence shape.""" + return SubmissionEvaluation( + phase1_against_original_gold="pass" if cascade.n1_original_gold else "fail", + phase1_against_audited_primary="pass" if cascade.n2_audited_primary else "fail", + phase1_against_any_audited_variant=( + "pass" if cascade.n3_any_audited_variant else "fail" + ), + phase1_against_variants=list(cascade.variant_matches), + correct_up_to_tie_order=cascade.n4_tie_order, + novel_reading_judgment=cascade.novel_reading_judgment, + correct_under_numeric_epsilon=cascade.n6_numeric_epsilon, + correct_under_trailing_whitespace=cascade.n7_trailing_whitespace, + correct_under_column_order=cascade.n8_column_order, + numeric_epsilon=epsilon, + verdict=( + "correct" if cascade.n3_any_audited_variant else "invalid" + ), + matched_variant_id=cascade.matched_variant_id, + rationale="", + ) + + +def generate_submission_annotation( + *, + rows_dir: Path, + instance_id: str, + selected_database: str, + benchmark: str, + run_id: str, + task_row: dict, + grader: Callable[..., Any], + epsilon: float = 1e-6, + annotated_at: Optional[str] = None, +) -> SubmissionAnnotation: + """Mechanically-filled SubmissionAnnotation skeleton. + + ``grader`` is a callable that returns a CascadeVerdict given the + submitted SQL + task row context. Tests pass a stub; production + callers pass ``tolerant_grader.grade_submission``.""" + attempt_path = Path(rows_dir) / instance_id / "attempt-1.json" + attempt = json.loads(attempt_path.read_text()) + submitted_sql = attempt.get("submitted_sql", "") + traj = list(attempt.get("trajectory", []) or []) + usage = attempt.get("usage", {}) or {} + + cascade = grader( + instance_id=instance_id, + submitted_sql=submitted_sql, + task_row=task_row, + ) + ev = _eval_from_cascade(cascade, epsilon=epsilon) + if annotated_at is None: + annotated_at = _dt.datetime.now(_dt.timezone.utc).replace( + microsecond=0, + ).isoformat() + + return SubmissionAnnotation( + instance_id=instance_id, + selected_database=selected_database, + task_annotation_ref=( + f"annotations/{benchmark}/{selected_database}/" + f"{instance_id}.task.json" + ), + annotated_by="auto-skeleton", + annotated_at=annotated_at, + submission=SubmissionMetadata( + cloud_run_id=run_id, + trajectory_path=str(attempt_path), + predicted_row_count=attempt.get("predicted_row_count"), + duration_s=attempt.get("duration_s"), + cost_usd_agent=usage.get("cost_usd_agent"), + cost_usd_user_sim=usage.get("cost_usd_user_sim"), + n_agent_turns=usage.get("n_agent_turns"), + n_ask_user_calls=usage.get("n_ask_user_calls"), + ), + evaluation=ev, + failure_classification=FailureClassification( + primary="other", + agent_at_fault=False, + remediation_target="other", + details=PENDING_HUMAN_REVIEW, + ), + decision_point=None, + user_sim_interaction=_user_sim_interaction_from_trajectory(traj), + ) + + +# --------------------------------------------------------------------------- +# Mode-aware writers +# --------------------------------------------------------------------------- + + +def _refresh_mechanical(existing: TaskAnnotation, fresh: TaskAnnotation) -> TaskAnnotation: + """Preserve any non-sentinel human-judgment field on ``existing``; + overwrite mechanical fields from ``fresh``.""" + out = existing.model_copy(deep=True) + out.amb_user_query = fresh.amb_user_query + out.external_knowledge = list(fresh.external_knowledge) + out.masked_terms = list(fresh.masked_terms) + out.provenance = fresh.provenance + out.annotated_at = fresh.annotated_at + # Sentinels mean "still pending" — re-import the sentinel only if + # the user hasn't yet authored a real value. + if existing.metadata_sufficiency.rationale == PENDING_HUMAN_REVIEW: + out.metadata_sufficiency = fresh.metadata_sufficiency + return out + + +def write_task_skeleton( + *, + task_row: dict, + benchmark: str, + mode: str = "init", # init | refresh | force-all + dry_run: bool = False, + repo_root: Optional[Path] = None, +) -> Optional[Path]: + """Returns the destination path. ``None`` ⇒ skipped (e.g. file + already exists in init mode).""" + repo_root = repo_root or paths.main_checkout_root() + fresh = generate_task_annotation(task_row=task_row, benchmark=benchmark) + dest = task_annotation_path( + benchmark=benchmark, selected_database=task_row["selected_database"], + instance_id=task_row["instance_id"], repo_root=repo_root, + ) + if mode == "init" and dest.exists(): + return None + if mode == "refresh" and dest.exists(): + existing = read_task_annotation(dest) + merged = _refresh_mechanical(existing, fresh) + if not dry_run: + write_task_annotation(merged, dest) + return dest + # force-all (or init when file missing). + if not dry_run: + write_task_annotation(fresh, dest) + return dest + + +def write_submission_skeleton( + *, + rows_dir: Path, + instance_id: str, + selected_database: str, + benchmark: str, + run_id: str, + task_row: dict, + grader: Callable[..., Any], + mode: str = "overwrite", # overwrite | init + dry_run: bool = False, + repo_root: Optional[Path] = None, +) -> Optional[Path]: + repo_root = repo_root or paths.main_checkout_root() + dest = submission_annotation_path( + benchmark=benchmark, selected_database=selected_database, + instance_id=instance_id, run_id=run_id, repo_root=repo_root, + ) + if mode == "init" and dest.exists(): + return None + ann = generate_submission_annotation( + rows_dir=rows_dir, instance_id=instance_id, + selected_database=selected_database, benchmark=benchmark, + run_id=run_id, task_row=task_row, grader=grader, + ) + if not dry_run: + write_submission_annotation(ann, dest) + return dest + + +# --------------------------------------------------------------------------- +# CLI +# --------------------------------------------------------------------------- + + +def _load_task_rows(*, benchmark: str, instance_ids: Optional[Iterable[str]]) -> list[dict]: + """Load task rows for the given instance IDs from the benchmark JSONL.""" + bench = get_benchmark(benchmark.replace("-", "_")) + data_path = paths.benchmark_data_root(bench) / bench.data_file + rows = [json.loads(line) for line in data_path.read_text().splitlines() if line.strip()] + if instance_ids: + wanted = set(instance_ids) + rows = [r for r in rows if r.get("instance_id") in wanted] + return rows + + +def main(argv: Optional[List[str]] = None) -> int: + parser = argparse.ArgumentParser( + description="Generate / refresh per-task + per-submission annotation skeletons.", + ) + parser.add_argument("--run-id", required=True) + parser.add_argument("--benchmark", required=True) + parser.add_argument( + "--instance-ids", default=None, + help="Comma-separated subset; default = every instance in the run.", + ) + parser.add_argument( + "--task-mode", choices=("init", "refresh", "force-all"), + default="init", + ) + parser.add_argument( + "--submission-mode", choices=("overwrite", "init"), + default="overwrite", + ) + parser.add_argument("--dry-run", action="store_true") + args = parser.parse_args(argv) + + instance_ids = ( + [s.strip() for s in args.instance_ids.split(",")] + if args.instance_ids else None + ) + rows = _load_task_rows( + benchmark=args.benchmark, instance_ids=instance_ids, + ) + for row in rows: + write_task_skeleton( + task_row=row, benchmark=args.benchmark, + mode=args.task_mode, dry_run=args.dry_run, + ) + print(f"task skeletons: {len(rows)} processed in mode={args.task_mode}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/bird_interact_agents/eval/annotation_schema.py b/src/bird_interact_agents/eval/annotation_schema.py index bd5adcd7..230d4677 100644 --- a/src/bird_interact_agents/eval/annotation_schema.py +++ b/src/bird_interact_agents/eval/annotation_schema.py @@ -169,11 +169,30 @@ class TaskAnnotation(BaseModel): provenance: Provenance +class VariantInformational(BaseModel): + """Tier-2 diagnostic block populated per (submission, variant). + + Carries deterministic, non-verdict-flipping signal: rowset relation, + column-structure comparison, and the first divergent cell (if any). + Consumers reading this can decide which variant to fix vs. which is + closest to the agent's intent without re-executing. + """ + model_config = ConfigDict(extra="forbid") + + rowset_relation: RowsetRelation + column_count_match: bool + column_name_match_case_insensitive: bool + column_order_match: bool + first_divergent_row_index: Optional[int] = None + first_divergent_cell_diff: Optional[str] = None + + class VariantMatch(BaseModel): model_config = ConfigDict(extra="forbid") variant_id: str match: RowsetRelation + informational: Optional[VariantInformational] = None class SubmissionMetadata(BaseModel): @@ -195,7 +214,8 @@ class SubmissionMetadata(BaseModel): class SubmissionEvaluation(BaseModel): """Cascading evaluation, most stringent → most lenient. - Each field corresponds to one row in the after-run report layout. + Each field corresponds to one row in the after-run report layout + (N1 through N8 of the cascade). """ model_config = ConfigDict(extra="forbid") @@ -207,6 +227,12 @@ class SubmissionEvaluation(BaseModel): novel_reading_judgment: Optional[PhaseVerdict] = None """Populated only when ``metadata_sufficiency.verdict == 'insufficient'`` AND the deterministic check returned no match.""" + correct_under_numeric_epsilon: bool = False + correct_under_trailing_whitespace: bool = False + correct_under_column_order: bool = False + numeric_epsilon: float = 1e-6 + """Records the epsilon threshold actually used for N6, so consumers + of the annotation know how lenient the relaxation was.""" verdict: SubmissionVerdict matched_variant_id: Optional[str] = None rationale: str = "" diff --git a/src/bird_interact_agents/eval/cascading_report.py b/src/bird_interact_agents/eval/cascading_report.py new file mode 100644 index 00000000..4f989670 --- /dev/null +++ b/src/bird_interact_agents/eval/cascading_report.py @@ -0,0 +1,135 @@ +"""DEV-1515: cascading-phase1 aggregator + eval.json writer. + +Replaces the legacy dual-eval block (``phase1_count_audited`` etc.) with +a single ``cascading_phase1`` dict carrying N1..N8 counts, rates, +deltas, and ``n_dual_eval_tasks``. + +The aggregator walks ``//submission_annotation.json`` +for each per-task row and sums the cascade verdicts. Missing per-row +annotation files raise — silent under-count is forbidden. + +Back-compat: ``phase1_count`` and ``phase1_rate`` in the published +``eval.json`` map to the cascade's ``n1`` count + rate, REWRITTEN from +the recomputed cascade (not carried forward from base_metrics). +""" +from __future__ import annotations + +import json +from pathlib import Path +from typing import Iterable + +from bird_interact_agents.eval.annotation_io import read_submission_annotation +from bird_interact_agents.eval.tolerant_grader import ( + _CASCADE_ORDER, enforce_monotone_cascade, +) + + +def _per_row_cascade_bools(annotation_dir: Path) -> dict[str, bool]: + """Load a single ``//submission_annotation.json`` + and return the monotone-enforced raw N1..N8 bools.""" + p = annotation_dir / "submission_annotation.json" + if not p.exists(): + raise FileNotFoundError( + f"submission_annotation.json missing under {annotation_dir} " + "— cascading-phase1 aggregator requires every per-task row " + "to carry a grader-written annotation", + ) + ann = read_submission_annotation(p) + ev = ann.evaluation + raw = { + "n1_original_gold": ev.phase1_against_original_gold == "pass", + "n2_audited_primary": ev.phase1_against_audited_primary == "pass", + "n3_any_audited_variant": ( + ev.phase1_against_any_audited_variant == "pass" + ), + "n4_tie_order": ev.correct_up_to_tie_order, + "n5_llm_judge": ( + ev.phase1_against_any_audited_variant == "pass" + or ev.correct_up_to_tie_order + or ev.novel_reading_judgment == "pass" + ), + "n6_numeric_epsilon": ev.correct_under_numeric_epsilon, + "n7_trailing_whitespace": ev.correct_under_trailing_whitespace, + "n8_column_order": ev.correct_under_column_order, + } + return enforce_monotone_cascade(raw) + + +def aggregate_cascading_phase1(rows_dir: Path) -> dict: + """Walk per-task ``submission_annotation.json`` files and return the + cascading_phase1 block. + + Output shape:: + + { + "n_dual_eval_tasks": N, + "counts": {"n1": ..., "n8": ...}, + "rates": {"n1": 0.xx, ...}, + "deltas": {"n2": ..., "n3": ..., ...}, + } + """ + rows_dir = Path(rows_dir) + counts = {short_for(f): 0 for f in _CASCADE_ORDER} + n = 0 + if rows_dir.exists(): + for sub in sorted(p for p in rows_dir.iterdir() if p.is_dir()): + verdicts = _per_row_cascade_bools(sub) + n += 1 + for f, v in verdicts.items(): + if v: + counts[short_for(f)] += 1 + rates = { + k: (v / n) if n else 0.0 for k, v in counts.items() + } + deltas: dict[str, int] = {} + prev: int | None = None + for k in ("n1", "n2", "n3", "n4", "n5", "n6", "n7", "n8"): + if prev is None: + deltas[k] = 0 + else: + deltas[k] = counts[k] - prev + prev = counts[k] + return { + "n_dual_eval_tasks": n, + "counts": counts, + "rates": rates, + "deltas": deltas, + } + + +def short_for(field: str) -> str: + """``"n1_original_gold"`` → ``"n1"``.""" + return field.split("_", 1)[0] + + +_LEGACY_KEYS_TO_DROP = ( + "phase1_count_audited", + "phase1_count_original", + "phase1_rate_audited", + "phase1_rate_original", + "n_dual_eval_tasks", # moved into cascading_phase1 +) + + +def emit_cascading_eval_json( + rows_dir: Path, + out_path: Path, + base_metrics: dict | None = None, +) -> dict: + """Merge ``base_metrics`` with the freshly-computed cascading block + and write to ``out_path``. The legacy dual-eval keys are explicitly + dropped; ``phase1_count`` / ``phase1_rate`` are REWRITTEN from N1. + + Returns the resulting metrics dict (for inline use).""" + block = aggregate_cascading_phase1(Path(rows_dir)) + out = dict(base_metrics or {}) + for k in _LEGACY_KEYS_TO_DROP: + out.pop(k, None) + out["cascading_phase1"] = block + # Back-compat aliases — rewritten from the freshly-computed cascade, + # NOT carried forward from base_metrics. + out["phase1_count"] = block["counts"]["n1"] + out["phase1_rate"] = block["rates"]["n1"] + Path(out_path).parent.mkdir(parents=True, exist_ok=True) + Path(out_path).write_text(json.dumps(out, indent=2, default=str)) + return out diff --git a/src/bird_interact_agents/eval/grade_in_place.py b/src/bird_interact_agents/eval/grade_in_place.py new file mode 100644 index 00000000..34ec0db1 --- /dev/null +++ b/src/bird_interact_agents/eval/grade_in_place.py @@ -0,0 +1,182 @@ +"""DEV-1515: shared inline grader that both the cloud worker +(``ray_app.py``) and the local runner (``run.py``) invoke per task. + +The function ``grade_and_write`` runs ``tolerant_grader.grade_submission`` +and persists the resulting ``SubmissionAnnotation`` to +``//submission_annotation.json``. The cloud +``fetch`` path later merges that file into +``/annotations///.submission..json``. + +The pre-DEV-1515 raw per-gold pass-fail bools are NOT emitted anywhere +— all per-task verdicts live in the SubmissionAnnotation's ``evaluation`` +block. +""" +from __future__ import annotations + +import datetime as _dt +from pathlib import Path +from typing import Any, Callable, List, Optional + +from bird_interact_agents.eval.annotation_schema import ( + FailureClassification, + PhaseVerdict, + SubmissionAnnotation, + SubmissionEvaluation, + SubmissionMetadata, + TaskAnnotation, + UserSimInteraction, +) +from bird_interact_agents.eval.tolerant_grader import ( + CascadeVerdict, + grade_submission, +) + + +_AUTO_ANNOTATOR = "auto-inline-grader" + + +def _verdict_to_phase(b: bool) -> PhaseVerdict: + return "pass" if b else "fail" + + +def _build_submission_annotation( + *, + task_annotation: TaskAnnotation, + cascade: CascadeVerdict, + benchmark: str, + run_id: str, + trajectory_path: str, + predicted_row_count: Optional[int], + duration_s: Optional[float], + cost_usd_agent: Optional[float], + cost_usd_user_sim: Optional[float], + n_agent_turns: Optional[int], + n_ask_user_calls: Optional[int], + user_sim_interaction: Optional[UserSimInteraction] = None, + epsilon: float = 1e-6, +) -> SubmissionAnnotation: + """Map the in-memory CascadeVerdict → on-disk SubmissionAnnotation.""" + if cascade.n3_any_audited_variant: + verdict_label = "correct" + elif cascade.n5_llm_judge or cascade.n4_tie_order: + verdict_label = "valid_interpretation" + elif ( + cascade.n6_numeric_epsilon + or cascade.n7_trailing_whitespace + or cascade.n8_column_order + ): + verdict_label = "valid_interpretation" + else: + verdict_label = "invalid" + + ev = SubmissionEvaluation( + phase1_against_original_gold=_verdict_to_phase(cascade.n1_original_gold), + phase1_against_audited_primary=_verdict_to_phase(cascade.n2_audited_primary), + phase1_against_any_audited_variant=_verdict_to_phase( + cascade.n3_any_audited_variant + ), + phase1_against_variants=list(cascade.variant_matches), + correct_up_to_tie_order=cascade.n4_tie_order, + novel_reading_judgment=cascade.novel_reading_judgment, + correct_under_numeric_epsilon=cascade.n6_numeric_epsilon, + correct_under_trailing_whitespace=cascade.n7_trailing_whitespace, + correct_under_column_order=cascade.n8_column_order, + numeric_epsilon=epsilon, + verdict=verdict_label, # type: ignore[arg-type] + matched_variant_id=cascade.matched_variant_id, + rationale="", + ) + + task_ann_ref = ( + f"annotations/{benchmark}/" + f"{task_annotation.selected_database}/" + f"{task_annotation.instance_id}.task.json" + ) + return SubmissionAnnotation( + instance_id=task_annotation.instance_id, + selected_database=task_annotation.selected_database, + task_annotation_ref=task_ann_ref, + annotated_by=_AUTO_ANNOTATOR, + annotated_at=_dt.datetime.now(_dt.timezone.utc) + .replace(microsecond=0).isoformat(), + submission=SubmissionMetadata( + cloud_run_id=run_id, + trajectory_path=trajectory_path, + predicted_row_count=predicted_row_count, + duration_s=duration_s, + cost_usd_agent=cost_usd_agent, + cost_usd_user_sim=cost_usd_user_sim, + n_agent_turns=n_agent_turns, + n_ask_user_calls=n_ask_user_calls, + ), + evaluation=ev, + failure_classification=FailureClassification( + primary="other", + agent_at_fault=not cascade.n3_any_audited_variant, + remediation_target="other", + details="auto-generated; human review pending", + ), + decision_point=None, + user_sim_interaction=( + user_sim_interaction or UserSimInteraction() + ), + ) + + +def grade_and_write( + *, + rows_dir: Path, + instance_id: str, + benchmark: str, + run_id: str, + task_annotation: TaskAnnotation, + audited_gold_rows: List[dict], + original_sol_sql: List[str], + submitted_sql: str, + db_path: Path, + conn: Any = None, + executor: Optional[Callable[..., Any]] = None, + trajectory_path: str, + cost_usd_agent: Optional[float] = None, + cost_usd_user_sim: Optional[float] = None, + duration_s: Optional[float] = None, + n_agent_turns: Optional[int] = None, + n_ask_user_calls: Optional[int] = None, + predicted_row_count: Optional[int] = None, + user_sim_interaction: Optional[UserSimInteraction] = None, + llm_judge: Any = None, + epsilon: float = 1e-6, +) -> Path: + """Run the tolerant grader and write the SubmissionAnnotation to + ``//submission_annotation.json``.""" + cascade = grade_submission( + task_annotation=task_annotation, + audited_gold_rows=audited_gold_rows, + original_sol_sql=original_sol_sql, + submitted_sql=submitted_sql, + db_path=db_path, + conn=conn, + executor=executor, + llm_judge=llm_judge, + epsilon=epsilon, + ) + ann = _build_submission_annotation( + task_annotation=task_annotation, + cascade=cascade, + benchmark=benchmark, + run_id=run_id, + trajectory_path=trajectory_path, + predicted_row_count=predicted_row_count, + duration_s=duration_s, + cost_usd_agent=cost_usd_agent, + cost_usd_user_sim=cost_usd_user_sim, + n_agent_turns=n_agent_turns, + n_ask_user_calls=n_ask_user_calls, + user_sim_interaction=user_sim_interaction, + epsilon=epsilon, + ) + out_dir = Path(rows_dir) / instance_id + out_dir.mkdir(parents=True, exist_ok=True) + out_path = out_dir / "submission_annotation.json" + out_path.write_text(ann.model_dump_json(indent=2, exclude_none=False) + "\n") + return out_path diff --git a/src/bird_interact_agents/eval/implicit_annotation.py b/src/bird_interact_agents/eval/implicit_annotation.py new file mode 100644 index 00000000..223d262d --- /dev/null +++ b/src/bird_interact_agents/eval/implicit_annotation.py @@ -0,0 +1,82 @@ +"""DEV-1515: implicit (auto-synthesized) ``TaskAnnotation``. + +For benchmark instances without a human-authored task annotation file on +disk, the grader needs a schema-valid placeholder so the cascade can +still run. The implicit default treats the original gold as the sole +correct answer (``verdict=sufficient`` + ``original_gold_is_correct=True``); +the cascade collapses to N1 and the LLM judge never fires. + +The factory is a pure builder — it never writes to disk and never reads +from ``paths.annotations_root()``. Callers are responsible for keeping +implicit annotations in memory only. +""" +from __future__ import annotations + +import datetime as _dt +from typing import Optional + +from bird_interact_agents.benchmark import benchmark_names, get_benchmark +from bird_interact_agents.eval.annotation_schema import ( + MetadataSufficiency, + Provenance, + TaskAnnotation, +) + +IMPLICIT_ANNOTATED_BY = "auto-implicit" +"""Sentinel value for ``TaskAnnotation.annotated_by`` that downstream +consumers can use to distinguish synthesized-on-the-fly annotations from +human-authored ones.""" + + +def _benchmark_task_jsonl_name(benchmark: str) -> str: + """Return the basename of the benchmark's task JSONL for provenance. + + Falls back to a generic ``".jsonl"`` placeholder when the + benchmark token doesn't match a registered ``Benchmark`` descriptor + (which is fine for tests / forks that haven't registered theirs).""" + if benchmark in set(benchmark_names()): + return get_benchmark(benchmark).data_file + return f"{benchmark}.jsonl" + + +def implicit_task_annotation( + *, + instance_id: str, + selected_database: str, + benchmark: str, + amb_user_query: str = "", + annotated_at: Optional[str] = None, +) -> TaskAnnotation: + """Build a schema-valid placeholder ``TaskAnnotation`` for an + instance that has no on-disk annotation yet. + + Used by the tolerant grader when it cannot find + ``annotations///.task.json``. Returns + immediately — never touches disk. + """ + if annotated_at is None: + annotated_at = ( + _dt.datetime.now(_dt.timezone.utc).replace(microsecond=0).isoformat() + ) + return TaskAnnotation( + instance_id=instance_id, + selected_database=selected_database, + annotated_by=IMPLICIT_ANNOTATED_BY, + annotated_at=annotated_at, + amb_user_query=amb_user_query, + external_knowledge=[], + masked_terms=[], + metadata_sufficiency=MetadataSufficiency( + verdict="sufficient", + rationale="implicit default — no annotation file on disk", + evidence_sources_consulted=[], + ), + original_gold_is_correct=True, + gold_variants=[], + evaluator_prompt=None, + provenance=Provenance( + task_jsonl_path=_benchmark_task_jsonl_name(benchmark), + task_jsonl_instance_id=instance_id, + audited_gold_legacy_path=None, + ), + ) diff --git a/src/bird_interact_agents/eval/regrade.py b/src/bird_interact_agents/eval/regrade.py new file mode 100644 index 00000000..8a434dee --- /dev/null +++ b/src/bird_interact_agents/eval/regrade.py @@ -0,0 +1,240 @@ +"""DEV-1515: offline re-grade CLI. + +Re-runs the tolerant grader over a completed run's saved artefacts, +overwriting the per-(instance, run) SubmissionAnnotation in the main +checkout. Distinct from ``driver.fetch``'s merge — that one is +no-overwrite; this one is opt-in OVERWRITE for the explicit "re-grade" +workflow (after adding a new variant or editing the evaluator_prompt). + +Run:: + + python -m bird_interact_agents.eval.regrade \\ + --run-id --benchmark mini-interact \\ + [--instance-ids ...] \\ + [--force-llm-judge] +""" +from __future__ import annotations + +import argparse +import json +from pathlib import Path +from typing import Any, Callable, Iterable, List, Optional + +from pydantic import BaseModel, ConfigDict, Field + +from bird_interact_agents import paths +from bird_interact_agents.eval.annotation_io import ( + submission_annotation_path, + write_submission_annotation, +) +from bird_interact_agents.eval.annotation_schema import SubmissionAnnotation +from bird_interact_agents.eval.cascading_report import emit_cascading_eval_json + + +class RegradeReport(BaseModel): + model_config = ConfigDict(extra="forbid") + + run_id: str + benchmark: str + regraded: int = 0 + skipped: int = 0 + regraded_instances: list[str] = Field(default_factory=list) + + +def _attempt_rows_dir(run_dir: Path) -> Path: + return run_dir / "rows" + + +def clear_llm_judge_cache( + *, + cache_path: Path, + instance_ids: Iterable[str], +) -> None: + """Drop cache entries whose embedded ``instance_id`` matches any of + ``instance_ids``. Entries for other instances are preserved.""" + if not cache_path.exists(): + return + cache = json.loads(cache_path.read_text()) + wanted = set(instance_ids) + new = { + k: v for k, v in cache.items() + if v.get("instance_id") not in wanted + } + cache_path.write_text(json.dumps(new, indent=2)) + + +def regrade_run( + *, + run_id: str, + benchmark: str, + run_dir: Path, + instance_ids: Optional[List[str]] = None, + force_llm_judge: bool = False, + grader: Callable[..., Any], + repo_root: Optional[Path] = None, +) -> RegradeReport: + """Walk per-task attempt JSONs and re-grade each. Writes a fresh + ``eval_regraded.json`` alongside the historical ``eval.json``; the + historical file is NOT touched.""" + repo_root = repo_root or paths.main_checkout_root() + rows_dir = _attempt_rows_dir(run_dir) + report = RegradeReport(run_id=run_id, benchmark=benchmark) + if not rows_dir.exists(): + return report + + filter_set = set(instance_ids) if instance_ids else None + if force_llm_judge and filter_set: + clear_llm_judge_cache( + cache_path=run_dir / "llm_judge_cache.json", + instance_ids=filter_set, + ) + + fresh_rows_dir = run_dir / "regrade_rows" + fresh_rows_dir.mkdir(parents=True, exist_ok=True) + + for sub in sorted(p for p in rows_dir.iterdir() if p.is_dir()): + instance_id = sub.name + if filter_set is not None and instance_id not in filter_set: + report.skipped += 1 + continue + attempt = sub / "attempt-1.json" + if not attempt.exists(): + report.skipped += 1 + continue + attempt_data = json.loads(attempt.read_text()) + submitted_sql = attempt_data.get("submitted_sql", "") + selected_database = attempt_data.get("selected_database", "") + cascade = grader( + instance_id=instance_id, + submitted_sql=submitted_sql, + task_row=attempt_data, + ) + # Build a fresh SubmissionAnnotation from the cascade. + from bird_interact_agents.eval.annotate import ( + _eval_from_cascade, + _user_sim_interaction_from_trajectory, + ) + from bird_interact_agents.eval.annotation_schema import ( + FailureClassification, + SubmissionMetadata, + ) + usage = attempt_data.get("usage", {}) or {} + ann = SubmissionAnnotation( + instance_id=instance_id, + selected_database=selected_database, + task_annotation_ref=( + f"annotations/{benchmark}/{selected_database}/" + f"{instance_id}.task.json" + ), + annotated_by="auto-regrade", + annotated_at="", # populated below + submission=SubmissionMetadata( + cloud_run_id=run_id, + trajectory_path=str(attempt), + predicted_row_count=attempt_data.get("predicted_row_count"), + duration_s=attempt_data.get("duration_s"), + cost_usd_agent=usage.get("cost_usd_agent"), + cost_usd_user_sim=usage.get("cost_usd_user_sim"), + n_agent_turns=usage.get("n_agent_turns"), + n_ask_user_calls=usage.get("n_ask_user_calls"), + ), + evaluation=_eval_from_cascade(cascade), + failure_classification=FailureClassification( + primary="other", + agent_at_fault=not cascade.n3_any_audited_variant, + remediation_target="other", + ), + user_sim_interaction=_user_sim_interaction_from_trajectory( + list(attempt_data.get("trajectory", []) or []), + ), + ) + import datetime as _dt + ann.annotated_at = _dt.datetime.now(_dt.timezone.utc).replace( + microsecond=0, + ).isoformat() + + # OVERWRITE the per-(instance, run) annotation in the main checkout. + dest = submission_annotation_path( + benchmark=benchmark, selected_database=selected_database, + instance_id=instance_id, run_id=run_id, repo_root=repo_root, + ) + write_submission_annotation(ann, dest) + + # Stash the per-row file in a fresh dir so the cascading_phase1 + # block in eval_regraded.json is built from THIS regrade pass, not + # the historical run's annotations. + fresh_sub = fresh_rows_dir / instance_id + fresh_sub.mkdir(parents=True, exist_ok=True) + (fresh_sub / "submission_annotation.json").write_text( + ann.model_dump_json(indent=2, exclude_none=False) + "\n", + ) + + report.regraded += 1 + report.regraded_instances.append(instance_id) + + # Emit eval_regraded.json — NEVER touch eval.json. + emit_cascading_eval_json( + fresh_rows_dir, run_dir / "eval_regraded.json", + base_metrics={}, + ) + return report + + +def main(argv: Optional[List[str]] = None) -> int: + parser = argparse.ArgumentParser( + description="Re-grade a completed run's per-task artefacts.", + ) + parser.add_argument("--run-id", required=True) + parser.add_argument("--benchmark", required=True) + parser.add_argument("--instance-ids", default=None) + parser.add_argument("--force-llm-judge", action="store_true") + args = parser.parse_args(argv) + + instance_ids = ( + [s.strip() for s in args.instance_ids.split(",")] + if args.instance_ids else None + ) + from bird_interact_agents.eval.tolerant_grader import grade_submission + run_dir = paths.main_checkout_root() / "results" / "cloud" / args.run_id + + def _grader(*, instance_id: str, submitted_sql: str, task_row: dict, **_kw): + # Minimal end-to-end wiring — production callers pre-build the + # implicit annotation + audited gold rows themselves. + from bird_interact_agents.eval.implicit_annotation import ( + implicit_task_annotation, + ) + from bird_interact_agents.cloud.ray_app import ( + _load_audited_gold_rows_for, _load_task_annotation_or_implicit, + ) + ann = _load_task_annotation_or_implicit( + instance_id=instance_id, + selected_database=task_row.get("selected_database", ""), + benchmark=args.benchmark, + amb_user_query=task_row.get("amb_user_query", ""), + ) + audited = _load_audited_gold_rows_for( + benchmark=args.benchmark, instance_id=instance_id, + ) + return grade_submission( + task_annotation=ann, + audited_gold_rows=audited, + original_sol_sql=list( + task_row.get("original_sol_sql") or task_row.get("sol_sql") or [], + ), + submitted_sql=submitted_sql, + db_path=Path("/dev/null"), + conn=None, + ) + + report = regrade_run( + run_id=args.run_id, benchmark=args.benchmark, run_dir=run_dir, + instance_ids=instance_ids, force_llm_judge=args.force_llm_judge, + grader=_grader, + ) + print(f"regrade: {report.regraded} instances rewritten, " + f"{report.skipped} skipped") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/bird_interact_agents/eval/tolerant_grader.py b/src/bird_interact_agents/eval/tolerant_grader.py new file mode 100644 index 00000000..2aa903c9 --- /dev/null +++ b/src/bird_interact_agents/eval/tolerant_grader.py @@ -0,0 +1,733 @@ +"""DEV-1515: tolerant grader producing the 8-row cascading verdict. + +Each ``grade_submission`` call returns a ``CascadeVerdict`` carrying +N1..N8 booleans + diagnostic informational fields. The cascade is +monotone by construction: passing at level N implies passing at level +N+1. ``enforce_monotone_cascade`` is the single place that enforces +this property; both the inline grader and the aggregator pipe their +raw bools through it. + +LLM-judge tests stay mechanical per the project memory rule: never +assert on prompt content; cache key + timeout + persisted +SubmissionEvaluation fields only. +""" +from __future__ import annotations + +import hashlib +import json +import logging +import sqlite3 +from pathlib import Path +from typing import Any, Callable, Iterable, List, Optional, Protocol, Sequence, Tuple + +import sqlglot +import sqlglot.expressions as sg_expr +from pydantic import BaseModel, ConfigDict, Field + +from bird_interact_agents.eval.annotation_schema import ( + PhaseVerdict, + RowsetRelation, + TaskAnnotation, + VariantInformational, + VariantMatch, +) + +logger = logging.getLogger(__name__) + + +# --------------------------------------------------------------------------- +# ORDER BY parser +# --------------------------------------------------------------------------- + + +class OrderByKey(BaseModel): + """One parsed ORDER BY term. + + ``column_index`` is the 0-based index into the SELECT list. ``None`` + means the term references an expression that is NOT a select-list + column (e.g. ``ORDER BY a + b`` where the select list has ``a, b`` + individually). The grader falls back to N3-strict for the variant + in that case rather than guessing the bucket key. + """ + model_config = ConfigDict(extra="forbid") + + column_index: Optional[int] + desc: bool = False + + +def _select_list_aliases(select: sg_expr.Select) -> List[Tuple[str, sg_expr.Expression]]: + """Return ``[(name, expr)]`` for each item in the SELECT list. + + ``name`` is the explicit alias if present, else the column name when + the expression is a bare Column, else the raw SQL of the expression + (useful for substring matching on bare expressions like ``a + b``). + """ + out: List[Tuple[str, sg_expr.Expression]] = [] + for proj in select.expressions or []: + if isinstance(proj, sg_expr.Alias): + alias = proj.alias_or_name + inner = proj.this + out.append((alias.lower() if alias else "", inner)) + elif isinstance(proj, sg_expr.Column): + out.append((proj.name.lower(), proj)) + else: + out.append((proj.sql().lower(), proj)) + return out + + +def parse_orderby_keys(sql: str) -> List[OrderByKey]: + """Parse ``sql`` and return the ORDER BY column indices into the + SELECT list. + + Returns ``[]`` when no ORDER BY clause is present. For a term that + can't be mapped (an expression not in the SELECT list), the + corresponding ``OrderByKey.column_index`` is ``None``. + """ + try: + parsed = sqlglot.parse_one(sql, dialect="sqlite") + except sqlglot.errors.ParseError: + return [] + select = parsed.find(sg_expr.Select) + if select is None: + return [] + order = select.args.get("order") + if order is None: + return [] + aliases = _select_list_aliases(select) + keys: List[OrderByKey] = [] + for term in order.expressions: + desc = bool(term.args.get("desc")) + expr = term.this + idx: Optional[int] = None + # Bare integer: ORDER BY 1, 2, … + if isinstance(expr, sg_expr.Literal) and expr.is_int: + try: + pos = int(expr.this) + if 1 <= pos <= len(aliases): + idx = pos - 1 + except (ValueError, TypeError): + idx = None + else: + name = expr.name.lower() if isinstance(expr, sg_expr.Column) else None + if name is None: + name = expr.sql().lower() + for i, (alias_or_col, _expr) in enumerate(aliases): + if alias_or_col and alias_or_col == name: + idx = i + break + keys.append(OrderByKey(column_index=idx, desc=desc)) + return keys + + +# --------------------------------------------------------------------------- +# Cell-level relaxations +# --------------------------------------------------------------------------- + + +def _row_count_match(pred: Sequence[Sequence], gold: Sequence[Sequence]) -> bool: + return len(pred) == len(gold) + + +def compare_tie_order( + pred: Sequence[Sequence], + gold: Sequence[Sequence], + *, + orderby_indices: Sequence[int], +) -> bool: + """Bucket each row by the values at ``orderby_indices``, then check + set-equality within each bucket. + + Empty ``orderby_indices`` (no ORDER BY in the source SQL) → fall back + to set-equality (which is what N3 does).""" + if not _row_count_match(pred, gold): + return False + if not orderby_indices: + return _set_equal(pred, gold) + + def _key(row: Sequence) -> Tuple: + return tuple(row[i] for i in orderby_indices) + + pred_buckets: dict[Tuple, list[tuple]] = {} + gold_buckets: dict[Tuple, list[tuple]] = {} + for r in pred: + pred_buckets.setdefault(_key(r), []).append(tuple(r)) + for r in gold: + gold_buckets.setdefault(_key(r), []).append(tuple(r)) + if pred_buckets.keys() != gold_buckets.keys(): + return False + # Cross-bucket order: the keys must appear in the same sequence in pred + # and gold (we already required _row_count_match, so the bucket-position + # check uses the FIRST occurrence in each side). + pred_key_order = [_key(r) for r in pred] + gold_key_order = [_key(r) for r in gold] + seen_pred: list[Tuple] = [] + seen_gold: list[Tuple] = [] + for k in pred_key_order: + if k not in seen_pred: + seen_pred.append(k) + for k in gold_key_order: + if k not in seen_gold: + seen_gold.append(k) + if seen_pred != seen_gold: + return False + for k in pred_buckets: + # Bag-equality within bucket so duplicates count. + if sorted(map(_canonical_repr, pred_buckets[k])) != sorted( + map(_canonical_repr, gold_buckets[k]) + ): + return False + return True + + +def _set_equal(pred: Sequence[Sequence], gold: Sequence[Sequence]) -> bool: + """Bag-equality (multiset) on the canonical-repr of each row.""" + return sorted(map(_canonical_repr, pred)) == sorted( + map(_canonical_repr, gold) + ) + + +def _canonical_repr(row: Sequence) -> str: + """Stable string repr for a row so heterogeneous tuples sort.""" + return "|".join(repr(c) for c in row) + + +def _numeric_cell_equal(a: Any, b: Any, *, epsilon: float) -> bool: + """Per-cell numeric tolerance. Non-numeric cells fall back to ==.""" + if isinstance(a, bool) or isinstance(b, bool): + return a == b + if isinstance(a, (int, float)) and isinstance(b, (int, float)): + return abs(float(a) - float(b)) <= epsilon + return a == b + + +def compare_numeric_epsilon( + pred: Sequence[Sequence], + gold: Sequence[Sequence], + *, + epsilon: float, +) -> bool: + if not _row_count_match(pred, gold): + return False + # Per-row, per-cell tolerance, but we still need bag semantics. + used = [False] * len(gold) + for pr in pred: + match = -1 + for j, gr in enumerate(gold): + if used[j] or len(pr) != len(gr): + continue + if all( + _numeric_cell_equal(a, b, epsilon=epsilon) + for a, b in zip(pr, gr) + ): + match = j + break + if match < 0: + return False + used[match] = True + return all(used) + + +def _strip_trailing(cell: Any) -> Any: + return cell.rstrip() if isinstance(cell, str) else cell + + +def compare_trailing_whitespace( + pred: Sequence[Sequence], + gold: Sequence[Sequence], +) -> bool: + if not _row_count_match(pred, gold): + return False + pred_n = [tuple(_strip_trailing(c) for c in r) for r in pred] + gold_n = [tuple(_strip_trailing(c) for c in r) for r in gold] + return _set_equal(pred_n, gold_n) + + +def compare_column_order( + pred: Sequence[Sequence], + gold: Sequence[Sequence], + *, + pred_cols: Sequence[str], + gold_cols: Sequence[str], +) -> bool: + """Align columns by case-insensitive name, then set-equal the rows. + + Returns False when column counts differ, or when the column-name + sets differ (modulo case).""" + if len(pred_cols) != len(gold_cols): + return False + pred_l = [c.lower() for c in pred_cols] + gold_l = [c.lower() for c in gold_cols] + if set(pred_l) != set(gold_l): + return False + # Permutation: position in pred for each gold column. + perm = [pred_l.index(c) for c in gold_l] + aligned = [tuple(r[i] for i in perm) for r in pred] + return _set_equal(aligned, gold) + + +# --------------------------------------------------------------------------- +# Tier 2 informational helpers +# --------------------------------------------------------------------------- + + +def classify_rowset_relation( + *, + pred: Sequence[Sequence], + gold: Sequence[Sequence], +) -> RowsetRelation: + """Set-relation between pred and gold rowsets (canonical-repr keys).""" + p = set(_canonical_repr(r) for r in pred) + g = set(_canonical_repr(r) for r in gold) + if not p and not g: + return "equal_rowset" + if p == g: + return "equal_rowset" + if p < g: + return "strict_subset_of" + if p > g: + return "strict_superset_of" + if p & g: + return "overlapping" + return "disjoint" + + +def _first_divergent_row( + *, + pred: Sequence[Sequence], + gold: Sequence[Sequence], +) -> Tuple[Optional[int], Optional[str]]: + """For positional row-wise comparison, return the first index where + the two sides differ + a one-line cell-diff string. + + When rowcounts differ, returns ``(min(len_pred, len_gold), "")``.""" + min_len = min(len(pred), len(gold)) + for i in range(min_len): + if list(pred[i]) != list(gold[i]): + diff_cells = [] + for j in range(min(len(pred[i]), len(gold[i]))): + if pred[i][j] != gold[i][j]: + diff_cells.append( + f"col {j}: {pred[i][j]!r} vs {gold[i][j]!r}" + ) + return i, f"row {i}: " + "; ".join(diff_cells) + if len(pred) != len(gold): + return min_len, ( + f"" + ) + return None, None + + +def _column_diff( + *, + pred_cols: Sequence[str], gold_cols: Sequence[str], +) -> Tuple[bool, bool, bool]: + """(count_match, name_match_case_insensitive, order_match).""" + count = len(pred_cols) == len(gold_cols) + nm_set = ( + count + and set(c.lower() for c in pred_cols) == set(c.lower() for c in gold_cols) + ) + order = count and [c.lower() for c in pred_cols] == [c.lower() for c in gold_cols] + return count, nm_set, order + + +# --------------------------------------------------------------------------- +# Monotone enforcement +# --------------------------------------------------------------------------- + + +_CASCADE_ORDER = [ + "n1_original_gold", + "n2_audited_primary", + "n3_any_audited_variant", + "n4_tie_order", + "n5_llm_judge", + "n6_numeric_epsilon", + "n7_trailing_whitespace", + "n8_column_order", +] + + +def enforce_monotone_cascade(raw: dict[str, bool]) -> dict[str, bool]: + """Given raw N1..N8 bools, return a monotone-enforced version: once + True, every subsequent level stays True (a pass at level N implies + a pass at level N+1).""" + out: dict[str, bool] = {} + seen_true = False + for f in _CASCADE_ORDER: + v = bool(raw.get(f, False)) + if seen_true or v: + out[f] = True + seen_true = True + else: + out[f] = False + return out + + +# --------------------------------------------------------------------------- +# LLM judge protocol + cached wrapper +# --------------------------------------------------------------------------- + + +class LLMJudgeProtocol(Protocol): + """Minimal contract: ``judge`` returns True (accept), False (reject), + or None (timeout / transient error → fall through).""" + def judge(self, **kwargs: Any) -> Optional[bool]: # pragma: no cover + ... + + +def _stable_json(obj: Any) -> str: + return json.dumps(obj, sort_keys=True, default=str) + + +def _cache_key(payload: dict[str, Any]) -> str: + keys = ( + "model_name", + "annotation_content_hash", + "gold_variants_content_hash", + "submitted_sql_normalized", + ) + h = hashlib.sha256() + for k in keys: + h.update(k.encode()) + h.update(b"=") + h.update(_stable_json(payload.get(k, "")).encode()) + h.update(b"\x00") + return h.hexdigest() + + +def _normalize_sql(sql: str) -> str: + # Collapse whitespace; case-insensitive normalization is too risky + # (literal identifiers), so we keep case. + return " ".join(sql.split()) + + +class CachedLLMJudge: + """Wraps any LLM judge with a JSON-on-disk cache keyed by content. + + Key dimensions: model name, annotation content hash, gold-variants + content hash, normalized submitted SQL. Run-id is deliberately NOT + included so offline re-grade reuses worker-side decisions when + nothing meaningful changed. + + The cache value is ``{"verdict": True|False, "instance_id": ..., …}`` + so ``clear_llm_judge_cache(instance_ids=[...])`` can filter by + instance. + """ + def __init__(self, *, inner: Any, cache_path: Path): + self._inner = inner + self._cache_path = Path(cache_path) + if self._cache_path.exists(): + self._cache: dict[str, dict] = json.loads(self._cache_path.read_text()) + else: + self._cache = {} + + @property + def model_name(self) -> str: + return getattr(self._inner, "model_name", "unknown") + + def _persist(self) -> None: + self._cache_path.parent.mkdir(parents=True, exist_ok=True) + self._cache_path.write_text(_stable_json(self._cache)) + + def judge(self, **kwargs: Any) -> Optional[bool]: + payload = dict(kwargs) + payload["model_name"] = self.model_name + payload["submitted_sql_normalized"] = _normalize_sql( + payload.get("submitted_sql", "") + ) + key = _cache_key(payload) + if key in self._cache: + return self._cache[key].get("verdict") + result = self._inner.judge(**kwargs) + # Store ONLY definitive verdicts; None = timeout, retry next time. + if result is not None: + entry: dict[str, Any] = {"verdict": bool(result)} + if "instance_id" in kwargs: + entry["instance_id"] = kwargs["instance_id"] + self._cache[key] = entry + self._persist() + return result + + +# --------------------------------------------------------------------------- +# Default executor (real SQLite path) + grade_submission orchestrator +# --------------------------------------------------------------------------- + + +ExecutorResult = Tuple[Sequence[Sequence], Sequence[str]] +ExecutorProtocol = Callable[..., ExecutorResult] + + +def default_executor( + sql: str, + *, + db_path: Path, + conn: Optional[sqlite3.Connection] = None, +) -> ExecutorResult: + """Execute ``sql`` against the SQLite DB at ``db_path`` and return + ``(rows, column_names)``. Caches the connection when reused; the + caller is responsible for closing if it owns one.""" + own_conn = conn is None + if own_conn: + conn = sqlite3.connect(f"file:{db_path}?mode=ro", uri=True, timeout=30) + try: + cur = conn.cursor() + cur.execute(sql) + rows = list(cur.fetchall()) + cols = [d[0] for d in cur.description] if cur.description else [] + return rows, cols + finally: + if own_conn: + conn.close() + + +# --------------------------------------------------------------------------- +# CascadeVerdict (in-memory grader output) +# --------------------------------------------------------------------------- + + +class CascadeVerdict(BaseModel): + """Per-submission cascade output. Persisted via the ``SubmissionEvaluation`` + fields in the SubmissionAnnotation. The grader returns the verdict + pre-monotone-enforced (callers can re-enforce defensively).""" + model_config = ConfigDict(extra="forbid") + + n1_original_gold: bool + n2_audited_primary: bool + n3_any_audited_variant: bool + n4_tie_order: bool + n5_llm_judge: bool + n6_numeric_epsilon: bool + n7_trailing_whitespace: bool + n8_column_order: bool + matched_variant_id: Optional[str] = None + novel_reading_judgment: Optional[PhaseVerdict] = None + variant_matches: List[VariantMatch] = Field(default_factory=list) + rowset_relations: List[VariantMatch] = Field(default_factory=list) + + +def _multi_sql_execute( + sqls: List[str], + *, + db_path: Path, + conn: Any, + executor: ExecutorProtocol, +) -> ExecutorResult: + """Execute a list of SQL strings and return the rows+cols of the + LAST one. Mirrors BIRD-Interact's evaluator semantics: prior items + in the list set up state (CREATE TEMP …); only the last returns + rows for comparison.""" + rows: Sequence[Sequence] = [] + cols: Sequence[str] = [] + for sql in sqls: + rows, cols = executor(sql, db_path=db_path, conn=conn) + return rows, cols + + +def grade_submission( + *, + task_annotation: TaskAnnotation, + audited_gold_rows: List[dict], + original_sol_sql: List[str], + submitted_sql: str, + db_path: Path, + conn: Any = None, + executor: Optional[ExecutorProtocol] = None, + llm_judge: Optional[Any] = None, + epsilon: float = 1e-6, +) -> CascadeVerdict: + """Compute the 8-row cascade for a single submission. + + * ``audited_gold_rows`` may be empty (missing-annotation graceful + default — see ``implicit_annotation`` factory). The cascade then + collapses to N1; N2 and N3 mirror N1. + * The cascade is monotone by construction: ``enforce_monotone_cascade`` + is applied before returning. + * N4 uses the ORIGINAL gold's ORDER BY (locked simplification). + * N5 fires ONLY when ``task_annotation.metadata_sufficiency.verdict`` + is ``"insufficient"`` AND N4 didn't already pass. + """ + if executor is None: + executor = default_executor # type: ignore[assignment] + assert executor is not None # narrowing + + # 1) Run predicted + original gold + each variant. + pred_rows, pred_cols = executor(submitted_sql, db_path=db_path, conn=conn) + orig_rows, orig_cols = _multi_sql_execute( + list(original_sol_sql), db_path=db_path, conn=conn, executor=executor, + ) + + variant_results: list[tuple[dict, Sequence[Sequence], Sequence[str]]] = [] + for v in audited_gold_rows: + sqls = list(v.get("audited_sol_sql") or []) + if not sqls: + continue + v_rows, v_cols = _multi_sql_execute( + sqls, db_path=db_path, conn=conn, executor=executor, + ) + variant_results.append((v, v_rows, v_cols)) + + # 2) N1 — original gold strict. + n1 = _set_equal(pred_rows, orig_rows) + + # 3) N2/N3 — audited primary / any variant strict. + primary = next( + (vr for vr in variant_results if vr[0].get("primary")), None, + ) + if primary is None and not variant_results: + # No audited variants at all → N2 == N3 == N1. + n2 = n1 + n3 = n1 + matched_variant: Optional[str] = None + else: + n2 = bool(primary and _set_equal(pred_rows, primary[1])) + matched_variant = primary[0].get("variant_id") if n2 else None + n3 = n2 + if not n3: + for v, v_rows, _v_cols in variant_results: + if _set_equal(pred_rows, v_rows): + n3 = True + matched_variant = v.get("variant_id") + break + + # 4) N4 — tie-order against the (primary or original) variant. The + # bucket spec is sourced from the ORIGINAL gold's ORDER BY. + n4 = n3 + if not n4: + orderby = parse_orderby_keys( + original_sol_sql[-1] if original_sol_sql else "" + ) + indices = [ + k.column_index for k in orderby if k.column_index is not None + ] + # If ANY key didn't resolve, we leave it out — bucketing on + # partial keys is still safer than collapsing to N3. + candidates = ( + [(primary[0], primary[1])] if primary else [] + ) + [(v[0], v[1]) for v in variant_results if not v[0].get("primary")] + if not candidates: + # No variants → fall back to original gold itself. + candidates = [({}, orig_rows)] + for v_meta, v_rows in candidates: + if compare_tie_order(pred_rows, v_rows, orderby_indices=indices): + n4 = True + if v_meta.get("variant_id"): + matched_variant = v_meta["variant_id"] + break + + # 5) N5 — LLM judge, gated on insufficient verdict. + novel_judgment: Optional[PhaseVerdict] = None + n5 = n4 + if not n5 and ( + task_annotation.metadata_sufficiency.verdict == "insufficient" + and llm_judge is not None + and task_annotation.evaluator_prompt is not None + ): + try: + judged = llm_judge.judge( + evaluator_prompt=task_annotation.evaluator_prompt, + gold_variants_summary=[ + { + "variant_id": v.get("variant_id"), + "interpretation": next( + (gv.interpretation for gv in task_annotation.gold_variants + if gv.variant_id == v.get("variant_id")), "", + ), + } + for v in audited_gold_rows + ], + metadata_anchors=[m.term for m in task_annotation.masked_terms], + submitted_sql=submitted_sql, + predicted_rows_head=list(pred_rows[:20]), + annotation_content_hash=_annotation_hash(task_annotation), + gold_variants_content_hash=_gold_hash(audited_gold_rows), + instance_id=task_annotation.instance_id, + ) + except Exception: # noqa: BLE001 + logger.exception("LLM judge raised on instance=%s", + task_annotation.instance_id) + judged = None + if judged is True: + n5 = True + novel_judgment = "pass" + elif judged is False: + novel_judgment = "fail" + else: + novel_judgment = None + + # 6) N6/N7/N8 — cell-level relaxations applied across all variants. + n6, n7, n8 = n5, n5, n5 + if not n6: + for _v_meta, v_rows, _v_cols in variant_results + [ + ({"variant_id": "__original__"}, orig_rows, orig_cols), # type: ignore[list-item] + ]: + if compare_numeric_epsilon(pred_rows, v_rows, epsilon=epsilon): + n6 = True + break + if not n7: + n7 = n6 + if not n7: + for _v_meta, v_rows, _v_cols in variant_results + [ + ({"variant_id": "__original__"}, orig_rows, orig_cols), # type: ignore[list-item] + ]: + if compare_trailing_whitespace(pred_rows, v_rows): + n7 = True + break + if not n8: + n8 = n7 + if not n8: + for _v_meta, v_rows, v_cols in variant_results + [ + ({"variant_id": "__original__"}, orig_rows, orig_cols), # type: ignore[list-item] + ]: + if compare_column_order( + pred_rows, v_rows, + pred_cols=list(pred_cols), gold_cols=list(v_cols), + ): + n8 = True + break + + # 7) Tier 2 informational per variant. + info_matches: list[VariantMatch] = [] + for v_meta, v_rows, v_cols in variant_results: + rel = classify_rowset_relation(pred=pred_rows, gold=v_rows) + cm, nm, om = _column_diff(pred_cols=list(pred_cols), gold_cols=list(v_cols)) + fdri, fdcd = _first_divergent_row(pred=pred_rows, gold=v_rows) + info_matches.append(VariantMatch( + variant_id=v_meta.get("variant_id", "primary"), + match=rel, + informational=VariantInformational( + rowset_relation=rel, + column_count_match=cm, + column_name_match_case_insensitive=nm, + column_order_match=om, + first_divergent_row_index=fdri, + first_divergent_cell_diff=fdcd, + ), + )) + + raw = { + "n1_original_gold": n1, "n2_audited_primary": n2, + "n3_any_audited_variant": n3, "n4_tie_order": n4, + "n5_llm_judge": n5, "n6_numeric_epsilon": n6, + "n7_trailing_whitespace": n7, "n8_column_order": n8, + } + enforced = enforce_monotone_cascade(raw) + return CascadeVerdict( + **enforced, + matched_variant_id=matched_variant, + novel_reading_judgment=novel_judgment, + variant_matches=info_matches, + ) + + +def _annotation_hash(ann: TaskAnnotation) -> str: + return hashlib.sha256(ann.model_dump_json().encode()).hexdigest() + + +def _gold_hash(rows: Iterable[dict]) -> str: + h = hashlib.sha256() + for r in rows: + h.update(_stable_json(r).encode()) + h.update(b"\x00") + return h.hexdigest() diff --git a/src/bird_interact_agents/paths.py b/src/bird_interact_agents/paths.py index 06c121a8..33901665 100644 --- a/src/bird_interact_agents/paths.py +++ b/src/bird_interact_agents/paths.py @@ -183,6 +183,20 @@ def audited_gold_file(*, benchmark: str) -> Path: return audited_gold_root() / f"{b.name}_audited.jsonl" +def annotations_root() -> Path: + """DEV-1515: per-task / per-submission annotations, committed to the + main checkout (gitignored locally — the human-judgment content is + licensed for local use only, same posture as ``audited_gold/``). + + Honours ``BIRD_ANNOTATIONS_ROOT`` for tests / forks that mount the + annotations from a parallel repo. + """ + override = os.environ.get("BIRD_ANNOTATIONS_ROOT") + if override: + return Path(override).expanduser() + return main_checkout_root() / "annotations" + + def sar_audited_gold_root() -> Path: """SAR-Agent audit JSONL output root. diff --git a/src/bird_interact_agents/results_db.py b/src/bird_interact_agents/results_db.py index ff12197d..55668211 100644 --- a/src/bird_interact_agents/results_db.py +++ b/src/bird_interact_agents/results_db.py @@ -46,10 +46,6 @@ gold_result_json TEXT, n_agent_turns INTEGER, tool_call_stats_json TEXT, - phase1_passed_audited INTEGER, - phase1_passed_original INTEGER, - phase1_observation_audited TEXT, - phase1_observation_original TEXT, PRIMARY KEY (run_id, framework, mode, query_mode, instance_id) ) """ @@ -72,13 +68,6 @@ # "total_calls": int, "total_errors": int, # "error_samples": [{"tool": str, "error": str}, ...]} ("tool_call_stats_json", "TEXT"), - # Dual-evaluation columns (populated only when --use-audited-gold-sql is on - # and the task had an overlay applied). NULL elsewhere so old call sites - # don't have to know about them. - ("phase1_passed_audited", "INTEGER"), - ("phase1_passed_original", "INTEGER"), - ("phase1_observation_audited", "TEXT"), - ("phase1_observation_original", "TEXT"), ] _RUN_METADATA_DDL = """ @@ -124,13 +113,6 @@ class TaskResultRow(BaseModel): gold_result_json: str | None = None n_agent_turns: int | None = None tool_call_stats_json: str | None = None - # Dual-evaluation: populated when --use-audited-gold-sql is on AND - # the task had an overlay applied (edited / unrecoverable). NULL on - # single-eval runs. - phase1_passed_audited: bool | None = None - phase1_passed_original: bool | None = None - phase1_observation_audited: str | None = None - phase1_observation_original: str | None = None def open_db(path: Path | str) -> sqlite3.Connection: @@ -164,10 +146,8 @@ def insert_task_result(conn: sqlite3.Connection, row: TaskResultRow) -> None: error, usage_json, user_query, submission_status, phase1_observation, phase2_observation, predicted_result_json, gold_result_json, - n_agent_turns, tool_call_stats_json, - phase1_passed_audited, phase1_passed_original, - phase1_observation_audited, phase1_observation_original) - VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?) + n_agent_turns, tool_call_stats_json) + VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?) """, ( row.run_id, row.framework, row.mode, row.query_mode, @@ -179,12 +159,6 @@ def insert_task_result(conn: sqlite3.Connection, row: TaskResultRow) -> None: row.phase2_observation, row.predicted_result_json, row.gold_result_json, row.n_agent_turns, row.tool_call_stats_json, - None if row.phase1_passed_audited is None - else int(row.phase1_passed_audited), - None if row.phase1_passed_original is None - else int(row.phase1_passed_original), - row.phase1_observation_audited, - row.phase1_observation_original, ), ) conn.commit() diff --git a/src/bird_interact_agents/run.py b/src/bird_interact_agents/run.py index 886725c8..af2c8225 100644 --- a/src/bird_interact_agents/run.py +++ b/src/bird_interact_agents/run.py @@ -32,35 +32,28 @@ from bird_interact_agents.usage import TokenUsage def build_aggregate_eval(*, db_path: Path | str) -> dict[str, _Any]: - """Read task_results from a results.db and emit aggregate dual-eval - metrics. Used by run.py at end-of-run and by tests for round-trip - verification. Returns a dict with keys: ``phase1_count``, - ``phase1_rate``, ``phase1_count_audited``, ``phase1_count_original``, - ``phase1_rate_audited``, ``phase1_rate_original``, ``n_dual_eval_tasks``, - ``total_tasks``.""" + """Read task_results from a results.db and emit aggregate phase-1 + metrics. DEV-1515: the legacy dual-eval bool columns are gone; + every per-task cascade verdict now lives in the SubmissionAnnotation + written inline by ``grade_and_write``. The cascading_phase1 block + in ``eval.json`` is built by + :func:`bird_interact_agents.eval.cascading_report.emit_cascading_eval_json` + over the per-row annotation files; this helper only returns the + simple ``phase1_count`` / ``phase1_rate`` totals used by + back-compat consumers of the results DB.""" conn = _sqlite3.connect(str(db_path)) try: rows = conn.execute( - "SELECT phase1_passed, phase1_passed_audited, phase1_passed_original " - "FROM task_results" + "SELECT phase1_passed FROM task_results" ).fetchall() finally: conn.close() n = len(rows) p1 = sum(1 for r in rows if r[0]) - dual_rows = [r for r in rows if r[1] is not None] - n_dual = len(dual_rows) - p1_aud = sum(1 for r in dual_rows if r[1]) - p1_orig = sum(1 for r in dual_rows if r[2]) return { "total_tasks": n, "phase1_count": p1, "phase1_rate": p1 / n if n else 0.0, - "n_dual_eval_tasks": n_dual, - "phase1_count_audited": p1_aud, - "phase1_count_original": p1_orig, - "phase1_rate_audited": p1_aud / n_dual if n_dual else 0.0, - "phase1_rate_original": p1_orig / n_dual if n_dual else 0.0, } @@ -983,8 +976,6 @@ def _persist(td: dict, r: dict, started_at: float) -> None: gold_result_json=r.get("gold_result_json"), n_agent_turns=int(n_turns) if isinstance(n_turns, int) else None, tool_call_stats_json=tool_call_stats_json, - phase1_passed_audited=r.get("phase1_passed_audited"), - phase1_passed_original=r.get("phase1_passed_original"), phase1_observation_audited=r.get("phase1_observation_audited"), phase1_observation_original=r.get("phase1_observation_original"), )) @@ -1054,16 +1045,13 @@ async def _run_with_sem(i: int, td: dict) -> None: # Build metrics n = len(tasks) - # Dual-evaluation counts (NULL on single-eval runs — only populated - # when the overlay applied AND evaluate_dual_gold ran). Counting - # over the in-memory results list so we don't have to round-trip - # through the DB just for aggregation. - dual_audited = [r.get("phase1_passed_audited") for r in results] - dual_original = [r.get("phase1_passed_original") for r in results] - n_dual = sum(1 for x in dual_audited if x is not None) - p1_audited = sum(1 for x in dual_audited if x) - p1_original = sum(1 for x in dual_original if x) - + # DEV-1515: the legacy dual-eval breakdown (`phase1_count_audited`, + # `phase1_count_original`, `n_dual_eval_tasks`, the two rates) has + # been REPLACED by the cascading_phase1 block. The block is computed + # downstream by `emit_cascading_eval_json` over per-row + # submission_annotation.json files — local runs that don't write + # those (no inline grader hook) simply omit the block. `phase1_count` + # / `phase1_rate` stay as back-compat aliases for N1. metrics = { "mode": mode, "query_mode": query_mode, @@ -1077,13 +1065,6 @@ async def _run_with_sem(i: int, td: dict) -> None: "average_reward": total_reward / n if n else 0, "total_usage": total_usage.model_dump(), **timing, - # Dual-eval breakdown (only meaningful when --use-audited-gold-sql - # is on; equal to the single-eval counts otherwise). - "n_dual_eval_tasks": n_dual, - "phase1_count_audited": p1_audited, - "phase1_count_original": p1_original, - "phase1_rate_audited": p1_audited / n_dual if n_dual else 0, - "phase1_rate_original": p1_original / n_dual if n_dual else 0, "results": results, } diff --git a/tests/cloud/test_cli.py b/tests/cloud/test_cli.py index 18b4bb40..e4c994f0 100644 --- a/tests/cloud/test_cli.py +++ b/tests/cloud/test_cli.py @@ -653,13 +653,17 @@ def test_build_subcommand_passes_audited_gold_root_to_image_tag( tag_calls: list[tuple] = [] push_calls: list[tuple] = [] - def fake_image_tag(repo_root, audited_gold_root, *, allow_dirty): - tag_calls.append((repo_root, audited_gold_root, allow_dirty)) + def fake_image_tag(repo_root, audited_gold_root, *, allow_dirty, + annotations_root=None): + tag_calls.append((repo_root, audited_gold_root, allow_dirty, + annotations_root)) return "deadbeef-cafebabe" def fake_build_and_push(tag, repo_root, *, audited_gold_root=None, + annotations_root=None, force=False, **_kw): - push_calls.append((tag, repo_root, audited_gold_root, force)) + push_calls.append((tag, repo_root, audited_gold_root, + annotations_root, force)) return f"registry.example/x/runner:{tag}" monkeypatch.setattr(_image, "image_tag", fake_image_tag) diff --git a/tests/cloud/test_collation.py b/tests/cloud/test_collation.py index 03eacad8..70d78dd9 100644 --- a/tests/cloud/test_collation.py +++ b/tests/cloud/test_collation.py @@ -189,86 +189,13 @@ def test_eval_json_matches_local_aggregator( # --------------------------------------------------------------------------- -def test_collate_persists_dual_eval_columns(tmp_path: Path, sample_task_result_row): - run_dir = tmp_path / RUN_ID - run_dir.mkdir() - - # Edited row: audited gold passed, original gold failed (the diverging - # case that makes the original score informative). - diverging = { - **sample_task_result_row, - "instance_id": "db_a_1", - "phase1_passed": True, - "phase1_passed_audited": True, - "phase1_passed_original": False, - "phase1_observation_audited": "audited OK", - "phase1_observation_original": "original FAILED", - } - # Clean row: both golds identical → audited == original == phase1. - clean = { - **sample_task_result_row, - "instance_id": "db_a_2", - "phase1_passed": True, - "phase1_passed_audited": True, - "phase1_passed_original": True, - "phase1_observation_audited": "OK", - "phase1_observation_original": "OK", - } - _write_attempt(run_dir, "db_a_1", 1, diverging) - _write_attempt(run_dir, "db_a_2", 1, clean) - - manifest = { - "run_id": RUN_ID, - "framework": "pydantic_ai_otf_encode", - "mode": "a-interact", - "query_mode": "slayer", - "agent_model": "anthropic/claude-opus-4-7", - "user_sim_model": "anthropic/claude-sonnet-4-6", - "instance_ids": ["db_a_1", "db_a_2"], - } - metrics = collation.collate(run_dir, manifest) - - cols = _read_dual_cols(run_dir / "results.db") - # The diverging row must keep BOTH verdicts distinct in results.db. - assert cols["db_a_1"]["phase1_passed_audited"] == 1 - assert cols["db_a_1"]["phase1_passed_original"] == 0 - assert cols["db_a_1"]["phase1_observation_audited"] == "audited OK" - assert cols["db_a_1"]["phase1_observation_original"] == "original FAILED" - # The clean row also records the original score (always-score directive). - assert cols["db_a_2"]["phase1_passed_audited"] == 1 - assert cols["db_a_2"]["phase1_passed_original"] == 1 - - # eval.json carries the dual aggregate (parity with local run.py). - assert metrics["n_dual_eval_tasks"] == 2 - assert metrics["phase1_count_audited"] == 2 - assert metrics["phase1_count_original"] == 1 - assert metrics["phase1_rate_original"] == 0.5 - - -def test_collate_dual_columns_null_when_single_eval( - tmp_path: Path, sample_task_result_row -): - """A non-audited run (single-eval) has no dual fields in its row JSON; - collation must leave the columns NULL and the dual aggregate at 0.""" - run_dir = tmp_path / RUN_ID - run_dir.mkdir() - _write_attempt(run_dir, "db_a_1", 1, sample_task_result_row) - manifest = { - "run_id": RUN_ID, - "framework": "pydantic_ai", - "mode": "c-interact", - "query_mode": "raw", - "agent_model": "anthropic/claude-sonnet-4-5", - "user_sim_model": "anthropic/claude-haiku-4-5-20251001", - "instance_ids": ["db_a_1"], - } - metrics = collation.collate(run_dir, manifest) - - cols = _read_dual_cols(run_dir / "results.db") - assert cols["db_a_1"]["phase1_passed_audited"] is None - assert cols["db_a_1"]["phase1_passed_original"] is None - assert metrics["n_dual_eval_tasks"] == 0 - assert metrics["phase1_rate_original"] == 0 +# DEV-1515: the two collation tests previously here +# (`test_collate_persists_dual_eval_columns`, +# `test_collate_dual_columns_null_when_single_eval`) covered the removed +# raw bool DB columns + dual-rate eval.json keys. Per-task cascade +# verdicts now live in the SubmissionAnnotation; equivalent fetch-side +# coverage is in `tests/cloud/test_fetch_annotation_merge.py` and the +# cascading aggregator coverage is in `tests/test_cascading_report.py`. # --------------------------------------------------------------------------- diff --git a/tests/cloud/test_fetch_annotation_merge.py b/tests/cloud/test_fetch_annotation_merge.py new file mode 100644 index 00000000..83855690 --- /dev/null +++ b/tests/cloud/test_fetch_annotation_merge.py @@ -0,0 +1,203 @@ +"""DEV-1515: `bird-interact-cloud fetch` merges per-row +submission_annotation.json files into `/annotations/`. + +Contract: +* Walks `/cloud//rows//submission_annotation.json` + (downloaded from GCS) and merges to + `/annotations///.submission..json`. +* No-overwrite-if-present (mirrors slayer_models_otf/ merge pattern). +* Schema-validates each candidate; rejects malformed files and reports + them in the audit log. +* Audit log lands at `/cloud//annotation_merge_report.json`. +* The new ``gcs.submission_annotation_blob`` / ``write_submission_annotation`` + / ``read_submission_annotation`` shape mirrors `row_blob` / + `write_row` / `read_row`. +""" +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + + +# --------------------------------------------------------------------------- +# GCS blob path + helpers +# --------------------------------------------------------------------------- + + +def test_gcs_submission_annotation_blob_path(): + from bird_interact_agents.cloud import gcs + + blob = gcs.submission_annotation_blob("r-1", "alien_1") + assert blob == "runs/r-1/rows/alien_1/submission_annotation.json" + + +def test_gcs_submission_annotation_roundtrip_uses_canonical_blob_name( + fake_gcs_bucket, +): + """Upload then download MUST use the exact `submission_annotation_blob` + path — otherwise the worker write and fetch read can drift apart.""" + from bird_interact_agents.cloud import gcs + + client, store = fake_gcs_bucket + payload = _valid_submission_annotation_dict("alien_1") + gcs.write_submission_annotation( + "r-1", "alien_1", payload, client=client, + ) + assert "runs/r-1/rows/alien_1/submission_annotation.json" in store + read = gcs.read_submission_annotation("r-1", "alien_1", client=client) + assert read["instance_id"] == "alien_1" + + +# --------------------------------------------------------------------------- +# Merge — happy path + no-overwrite + schema validation +# --------------------------------------------------------------------------- + + +def _valid_submission_annotation_dict(instance_id: str = "alien_1") -> dict: + return { + "schema_version": 1, + "kind": "submission_annotation", + "instance_id": instance_id, + "selected_database": "alien", + "task_annotation_ref": f"annotations/mini-interact/alien/{instance_id}.task.json", + "annotated_by": "auto", + "annotated_at": "2026-05-31", + "submission": { + "cloud_run_id": "r1", + "trajectory_path": f"rows/{instance_id}/attempt-1.json", + }, + "evaluation": { + "phase1_against_original_gold": "pass", + "phase1_against_audited_primary": "pass", + "phase1_against_any_audited_variant": "pass", + "verdict": "correct", + }, + "failure_classification": { + "primary": "other", + "agent_at_fault": False, + "remediation_target": "other", + }, + } + + +def test_merge_writes_annotation_to_main_checkout(tmp_path): + from bird_interact_agents.cloud.post_run_merge import ( + merge_submission_annotations, + ) + + main_checkout = tmp_path / "checkout" + main_checkout.mkdir() + downloaded = tmp_path / "downloaded" + rows = downloaded / "rows" / "alien_1" + rows.mkdir(parents=True) + (rows / "submission_annotation.json").write_text( + json.dumps(_valid_submission_annotation_dict("alien_1")) + ) + + report = merge_submission_annotations( + downloaded_run_dir=downloaded, + run_id="r1", + benchmark="mini-interact", + main_checkout_root=main_checkout, + ) + + dest = ( + main_checkout / "annotations" / "mini-interact" / "alien" + / "alien_1.submission.r1.json" + ) + assert dest.exists() + assert report.merged == 1 + assert report.skipped_existing == 0 + assert report.rejected_invalid == 0 + + +def test_merge_no_overwrite_if_present(tmp_path): + from bird_interact_agents.cloud.post_run_merge import ( + merge_submission_annotations, + ) + + main_checkout = tmp_path / "checkout" + dest_dir = main_checkout / "annotations" / "mini-interact" / "alien" + dest_dir.mkdir(parents=True) + pre = _valid_submission_annotation_dict("alien_1") + pre["annotated_by"] = "human-pre-existing" + (dest_dir / "alien_1.submission.r1.json").write_text(json.dumps(pre)) + + downloaded = tmp_path / "downloaded" + rows = downloaded / "rows" / "alien_1" + rows.mkdir(parents=True) + fresh = _valid_submission_annotation_dict("alien_1") + fresh["annotated_by"] = "auto-fresh" + (rows / "submission_annotation.json").write_text(json.dumps(fresh)) + + report = merge_submission_annotations( + downloaded_run_dir=downloaded, + run_id="r1", + benchmark="mini-interact", + main_checkout_root=main_checkout, + ) + assert report.merged == 0 + assert report.skipped_existing == 1 + + surviving = json.loads( + (dest_dir / "alien_1.submission.r1.json").read_text() + ) + assert surviving["annotated_by"] == "human-pre-existing" + + +def test_merge_rejects_schema_invalid_file(tmp_path): + from bird_interact_agents.cloud.post_run_merge import ( + merge_submission_annotations, + ) + + main_checkout = tmp_path / "checkout" + downloaded = tmp_path / "downloaded" + rows = downloaded / "rows" / "alien_1" + rows.mkdir(parents=True) + (rows / "submission_annotation.json").write_text( + '{"this": "is", "not": "a valid SubmissionAnnotation"}' + ) + + report = merge_submission_annotations( + downloaded_run_dir=downloaded, + run_id="r1", + benchmark="mini-interact", + main_checkout_root=main_checkout, + ) + assert report.merged == 0 + assert report.rejected_invalid == 1 + # Destination must NOT have been created from invalid content. + assert not ( + main_checkout / "annotations" / "mini-interact" / "alien" + / "alien_1.submission.r1.json" + ).exists() + + +def test_merge_writes_audit_report(tmp_path): + from bird_interact_agents.cloud.post_run_merge import ( + merge_submission_annotations, + ) + + main_checkout = tmp_path / "checkout" + downloaded = tmp_path / "downloaded" + rows_dir = downloaded / "rows" + for inst in ("a_1", "a_2"): + d = rows_dir / inst + d.mkdir(parents=True) + (d / "submission_annotation.json").write_text( + json.dumps(_valid_submission_annotation_dict(inst)) + ) + + merge_submission_annotations( + downloaded_run_dir=downloaded, + run_id="r1", + benchmark="mini-interact", + main_checkout_root=main_checkout, + ) + audit = downloaded / "annotation_merge_report.json" + assert audit.exists() + body = json.loads(audit.read_text()) + assert body["merged"] == 2 + assert body["run_id"] == "r1" diff --git a/tests/cloud/test_image_annotations.py b/tests/cloud/test_image_annotations.py new file mode 100644 index 00000000..4b0f7392 --- /dev/null +++ b/tests/cloud/test_image_annotations.py @@ -0,0 +1,145 @@ +"""DEV-1515: annotations/ bake-in to the cloud image. + +Worktree-safety regression mirroring the existing audited_gold tests +in `tests/cloud/test_image.py`. The contract: + +* `paths.annotations_root()` anchors at the main checkout. +* `image.data_hash` includes annotation content (so an annotation + edit invalidates the cached image). +* `image.build_and_push` calls `docker build --build-context + annotations=` so the build runs cleanly + from any worktree (where annotations/ may not exist locally). +* `Dockerfile.cloud` copies the build context to + `/app/bird-interact-agents/annotations/` — matching how + audited_gold is mounted (not `/workspace/...`). +""" +from __future__ import annotations + +from pathlib import Path + + +def _make_annotations_dir(tmp: Path) -> Path: + d = tmp / "annotations" / "mini-interact" / "alien" + d.mkdir(parents=True) + (d / "alien_1.task.json").write_text( + '{"schema_version": 1, "kind": "task_annotation"}' + ) + return tmp / "annotations" + + +def test_data_hash_includes_annotations_content(fake_repo_root): + from bird_interact_agents.cloud import image + + annotations_root = _make_annotations_dir(fake_repo_root) + audited_gold_root = fake_repo_root / "audited_gold" + + h1 = image.data_hash( + fake_repo_root, audited_gold_root, annotations_root=annotations_root, + ) + + # Mutate one annotation file; hash must change. + (annotations_root / "mini-interact" / "alien" / "alien_1.task.json").write_text( + '{"schema_version": 1, "kind": "task_annotation", "x": 1}' + ) + h2 = image.data_hash( + fake_repo_root, audited_gold_root, annotations_root=annotations_root, + ) + assert h1 != h2, "annotations content must be hashed into data_hash" + + +def test_data_hash_invariant_to_host_path_of_annotations_root(tmp_path): + """The hash key is the on-image path `annotations/`, not the + host path — so a worktree build (where the annotations root sits at + a different absolute path) produces the SAME digest as a + main-checkout build with identical content. + """ + from bird_interact_agents.cloud import image + + # Use the cloud-test fake_repo_root fixture indirectly — re-create + # the minimal layout under two different host paths. + def _make_repo(base: Path) -> tuple[Path, Path, Path]: + (base / "src" / "bird_interact_agents").mkdir(parents=True) + (base / "src" / "bird_interact_agents" / "__init__.py").write_text( + "VERSION = '0.1.0'\n" + ) + (base / "src" / "bird_interact_agents" / "run.py").write_text( + "def main(): pass\n" + ) + (base / "uv.lock").write_text("# lock file\n") + (base / "pyproject.toml").write_text("[project]\nname='x'\n") + # Existing audited_gold also stable across hosts. + ag = base / "audited_gold" + ag.mkdir() + (ag / "x_audited.jsonl").write_text("{}\n") + # Annotations dir with identical content. + ann = base / "annotations" / "mini-interact" / "alien" + ann.mkdir(parents=True) + (ann / "alien_1.task.json").write_text( + '{"schema_version": 1, "kind": "task_annotation"}' + ) + # Minimal Dockerfile (data_hash reads sentinel sections). + (base / "Dockerfile.cloud").write_text( + "# DATA-LAYERS\nCOPY x .\n# CODE-LAYERS\nCOPY y .\n" + ) + return base, ag, base / "annotations" + + base_a = tmp_path / "host_a" + base_b = tmp_path / "host_b" + repo_a, ag_a, ann_a = _make_repo(base_a) + repo_b, ag_b, ann_b = _make_repo(base_b) + + h_a = image.data_hash(repo_a, ag_a, annotations_root=ann_a) + h_b = image.data_hash(repo_b, ag_b, annotations_root=ann_b) + assert h_a == h_b, ( + "data_hash must be invariant to the host path of annotations_root " + "— it must key by `annotations/` only" + ) + + +def test_dockerfile_cloud_copies_annotations_to_checkout_root(): + """Critical: copy target is `/app/bird-interact-agents/annotations/` + (alongside `/app/bird-interact-agents/audited_gold/`), NOT + `/workspace/...` — because `paths.main_checkout_root()` resolves + to `/app/bird-interact-agents/` inside the worker.""" + from bird_interact_agents import paths as paths_mod + + dockerfile = Path(paths_mod.__file__).parent.parent.parent / "Dockerfile.cloud" + text = dockerfile.read_text() + assert "COPY --from=annotations . /app/bird-interact-agents/annotations/" in text + + +def test_build_and_push_passes_annotations_build_context(monkeypatch, fake_repo_root): + """Build invocation must include `--build-context annotations=`.""" + from bird_interact_agents.cloud import image + + captured: list[list[str]] = [] + + def fake_run(cmd, **_kw): # noqa: ARG001 + captured.append(list(cmd)) + + class _R: + returncode = 0 + stdout = "" + stderr = "" + return _R() + + monkeypatch.setattr(image.subprocess, "run", fake_run) + + annotations_root = _make_annotations_dir(fake_repo_root) + audited_gold_root = fake_repo_root / "audited_gold" + + image.build_and_push( + "test-tag", + fake_repo_root, + audited_gold_root=audited_gold_root, + annotations_root=annotations_root, + force=True, + ) + # The actual build invocation is the call containing "docker build" — + # there may be a prior `docker manifest inspect` probe; force=True + # skips it but be defensive in case the implementation reorders. + build_cmds = [c for c in captured if "build" in c] + assert build_cmds, "expected at least one `docker build` invocation" + cmd = " ".join(build_cmds[0]) + assert "--build-context" in cmd + assert f"annotations={annotations_root}" in cmd diff --git a/tests/cloud/test_inline_grader.py b/tests/cloud/test_inline_grader.py new file mode 100644 index 00000000..675fffe6 --- /dev/null +++ b/tests/cloud/test_inline_grader.py @@ -0,0 +1,109 @@ +"""DEV-1515: cloud worker inline grading + per-row write to artefacts. + +After ``execute_submit_action`` returns, ray_app.py must call +``grade_in_place.grade_and_write`` so each per-row artefact dir contains +a ``submission_annotation.json`` written by tolerant_grader. + +This test isolates the wiring contract — the actual grader logic is +covered in ``tests/test_tolerant_grader_*.py``. +""" +from __future__ import annotations + +from pathlib import Path + + +def test_ray_app_writes_submission_annotation_per_task(monkeypatch, tmp_path): + """The worker MUST invoke grade_and_write for each task it runs.""" + from bird_interact_agents.cloud import ray_app + + calls: list[dict] = [] + + def fake_grade(**kwargs): # noqa: ANN003 + calls.append(kwargs) + rows_dir = kwargs["rows_dir"] + instance_id = kwargs["instance_id"] + d = rows_dir / instance_id + d.mkdir(parents=True, exist_ok=True) + (d / "submission_annotation.json").write_text("{}") + + monkeypatch.setattr( + ray_app, "grade_and_write", fake_grade, raising=True, + ) + + # The simulated submit hook — adapter for ray_app's per-task path. + # ray_app exposes a `_grade_one_submission(task_data, submitted_sql, + # rows_dir, run_id, benchmark)` helper that is the integration seam. + ray_app._grade_one_submission( + task_data={ + "instance_id": "alien_1", + "selected_database": "alien", + "sol_sql": ["SELECT gold"], + "original_sol_sql": ["SELECT gold"], + }, + submitted_sql="SELECT predicted", + rows_dir=tmp_path, + run_id="r1", + benchmark="mini-interact", + db_path=Path("/dev/null"), + conn=None, + ) + + assert len(calls) == 1 + assert (tmp_path / "alien_1" / "submission_annotation.json").exists() + + +def test_ray_app_does_not_emit_legacy_phase1_passed_fields(monkeypatch, tmp_path): + """The per-row result dict that ray_app uploads must NOT contain + the legacy raw bool fields — those have been replaced by the + submission_annotation path.""" + import inspect + from bird_interact_agents.cloud import ray_app + + src = inspect.getsource(ray_app) + assert "phase1_passed_audited" not in src + assert "phase1_passed_original" not in src + + +def test_worker_uses_implicit_annotation_when_file_missing(monkeypatch, tmp_path): + """If no .task.json exists in the baked annotations dir, + the worker falls back to implicit_task_annotation IN MEMORY — no + file gets written under annotations/.""" + from bird_interact_agents import paths as paths_mod + from bird_interact_agents.cloud import ray_app + + # Empty annotations dir. + annotations_root = tmp_path / "annotations" + annotations_root.mkdir() + monkeypatch.setattr( + paths_mod, "annotations_root", lambda: annotations_root, + ) + + captured: list[dict] = [] + + def fake_grade(**kwargs): # noqa: ANN003 + captured.append(kwargs) + d = kwargs["rows_dir"] / kwargs["instance_id"] + d.mkdir(parents=True, exist_ok=True) + (d / "submission_annotation.json").write_text("{}") + + monkeypatch.setattr(ray_app, "grade_and_write", fake_grade, raising=True) + + ray_app._grade_one_submission( + task_data={ + "instance_id": "alien_99", + "selected_database": "alien", + "sol_sql": ["SELECT gold"], + "original_sol_sql": ["SELECT gold"], + "amb_user_query": "x", + }, + submitted_sql="SELECT predicted", + rows_dir=tmp_path / "rows", + run_id="r1", + benchmark="mini-interact", + db_path=Path("/dev/null"), + conn=None, + ) + + assert len(captured) == 1 + # No .task.json was written. + assert not list(annotations_root.rglob("*.task.json")) diff --git a/tests/test_cascading_report.py b/tests/test_cascading_report.py new file mode 100644 index 00000000..a021bf63 --- /dev/null +++ b/tests/test_cascading_report.py @@ -0,0 +1,239 @@ +"""DEV-1515: cascading-report aggregator + legacy-field removal. + +Pins: +* eval.json carries a single ``cascading_phase1`` block with N1..N8 + counts, rates, deltas, and n_dual_eval_tasks. +* ``phase1_count`` / ``phase1_rate`` stay (basic back-compat) but map + to N1 of the cascade. +* The legacy dual-eval block (``n_dual_eval_tasks`` standalone, + ``phase1_rate_audited``, ``phase1_rate_original`` etc.) is removed. +* Aggregator reads per-row ``submission_annotation.json`` files; if + any row is missing the file, the aggregator raises (no silent + under-counts). +""" +from __future__ import annotations + +import json + +import pytest + + +def _make_submission_annotation_json( + *, + instance_id: str, + selected_database: str, + n1: bool, n2: bool, n3: bool, n4: bool, n5: bool, + n6: bool, n7: bool, n8: bool, + verdict: str = "correct", +) -> dict: + """Build the JSON shape produced by tolerant_grader → SubmissionAnnotation.""" + return { + "schema_version": 1, + "kind": "submission_annotation", + "instance_id": instance_id, + "selected_database": selected_database, + "task_annotation_ref": ( + f"annotations/mini-interact/{selected_database}/" + f"{instance_id}.task.json" + ), + "annotated_by": "auto", + "annotated_at": "2026-05-31", + "submission": { + "cloud_run_id": "test-run", + "trajectory_path": f"rows/{instance_id}/attempt-1.json", + "submitted_sql_path": None, + "predicted_row_count": 1, + "duration_s": 1.0, + "cost_usd_agent": 0.0, + "cost_usd_user_sim": 0.0, + "n_agent_turns": 1, + "n_ask_user_calls": 0, + }, + "evaluation": { + "phase1_against_original_gold": "pass" if n1 else "fail", + "phase1_against_audited_primary": "pass" if n2 else "fail", + "phase1_against_any_audited_variant": "pass" if n3 else "fail", + "phase1_against_variants": [], + "correct_up_to_tie_order": n4, + "novel_reading_judgment": "pass" if n5 and not n4 else None, + "correct_under_numeric_epsilon": n6, + "correct_under_trailing_whitespace": n7, + "correct_under_column_order": n8, + "numeric_epsilon": 1e-6, + "verdict": verdict, + "matched_variant_id": "primary" if n3 else None, + "rationale": "", + }, + "failure_classification": { + "primary": "other", + "secondary": [], + "agent_at_fault": False, + "remediation_target": "other", + "remediation_text": "", + "details": "", + }, + "decision_point": None, + "user_sim_interaction": { + "n_asks": 0, "key_responses": [], + "disclosed_resolutions": [], "undisclosed_resolutions": [], + }, + } + + +# --------------------------------------------------------------------------- +# Aggregator — builds cascading_phase1 block from per-row SubmissionAnnotation +# --------------------------------------------------------------------------- + + +def test_aggregator_emits_cascading_phase1_block(tmp_path): + from bird_interact_agents.eval.cascading_report import ( + aggregate_cascading_phase1, + ) + + rows_dir = tmp_path / "rows" + rows_dir.mkdir() + # 3 rows: full pass, only-N4, all-fail. + annotations = [ + _make_submission_annotation_json( + instance_id="alien_1", selected_database="alien", + n1=True, n2=True, n3=True, n4=True, n5=True, + n6=True, n7=True, n8=True, + ), + _make_submission_annotation_json( + instance_id="alien_2", selected_database="alien", + n1=False, n2=False, n3=False, n4=True, n5=True, + n6=True, n7=True, n8=True, + ), + _make_submission_annotation_json( + instance_id="alien_3", selected_database="alien", + n1=False, n2=False, n3=False, n4=False, n5=False, + n6=False, n7=False, n8=False, + ), + ] + for ann in annotations: + d = rows_dir / ann["instance_id"] + d.mkdir() + (d / "submission_annotation.json").write_text(json.dumps(ann)) + + block = aggregate_cascading_phase1(rows_dir) + assert block["n_dual_eval_tasks"] == 3 + counts = block["counts"] + assert counts["n1"] == 1 + assert counts["n2"] == 1 + assert counts["n3"] == 1 + assert counts["n4"] == 2 + assert counts["n5"] == 2 + assert counts["n6"] == 2 + assert counts["n7"] == 2 + assert counts["n8"] == 2 + + rates = block["rates"] + assert rates["n1"] == pytest.approx(1 / 3) + assert rates["n8"] == pytest.approx(2 / 3) + + deltas = block["deltas"] + assert deltas["n2"] == 0 + assert deltas["n3"] == 0 + assert deltas["n4"] == 1 # alien_2 added at N4 + + +def test_aggregator_enforces_monotonicity_on_tampered_row(tmp_path): + """The aggregator MUST enforce monotonicity. We deliberately feed a + violating row (N5=True, N6=False) — a "later level is more strict + than earlier" pattern that breaks the cascade. The aggregator must + repair this so the published `cascading_phase1` counts respect + N1 ≤ N2 ≤ ... ≤ N8. + """ + from bird_interact_agents.eval.cascading_report import ( + aggregate_cascading_phase1, + ) + + rows_dir = tmp_path / "rows" + rows_dir.mkdir() + ann = _make_submission_annotation_json( + instance_id="x_1", selected_database="x", + n1=False, n2=False, n3=False, n4=False, n5=True, + n6=False, # VIOLATES monotone: passes at N5 must also pass at N6. + n7=False, n8=False, + ) + d = rows_dir / "x_1" + d.mkdir() + (d / "submission_annotation.json").write_text(json.dumps(ann)) + + block = aggregate_cascading_phase1(rows_dir) + counts = block["counts"] + # After enforcement, every level from N5 onward inherits the pass. + assert counts["n5"] == 1 + assert counts["n6"] == 1, ( + "monotone enforcement: N5 pass must propagate to N6" + ) + assert counts["n7"] == 1 + assert counts["n8"] == 1 + + +def test_aggregator_raises_when_row_missing_submission_annotation(tmp_path): + """If any per-row dir is missing submission_annotation.json, the + aggregator must raise — silent under-count is forbidden.""" + from bird_interact_agents.eval.cascading_report import ( + aggregate_cascading_phase1, + ) + + rows_dir = tmp_path / "rows" + rows_dir.mkdir() + (rows_dir / "alien_1").mkdir() # empty — no submission_annotation.json + + with pytest.raises(FileNotFoundError): + aggregate_cascading_phase1(rows_dir) + + +# --------------------------------------------------------------------------- +# eval.json shape — `cascading_phase1` present; legacy block removed +# --------------------------------------------------------------------------- + + +def test_eval_json_contains_cascading_phase1_after_run(tmp_path): + """End-to-end: a synthetic local run emits eval.json with the new + block AND has dropped the legacy dual-eval keys.""" + from bird_interact_agents.eval.cascading_report import ( + emit_cascading_eval_json, + ) + + rows_dir = tmp_path / "rows" + rows_dir.mkdir() + # 2 instances: one passes at N1, one fails everything. + for inst, n1 in (("pass_1", True), ("fail_1", False)): + ann = _make_submission_annotation_json( + instance_id=inst, selected_database="x", + n1=n1, n2=n1, n3=n1, n4=n1, n5=n1, + n6=n1, n7=n1, n8=n1, + ) + d = rows_dir / inst + d.mkdir() + (d / "submission_annotation.json").write_text(json.dumps(ann)) + + # Deliberately wrong base metrics — emit_cascading_eval_json must + # REWRITE phase1_count/phase1_rate from the freshly-computed N1, + # not blindly carry the stale base value forward. + out = tmp_path / "eval.json" + emit_cascading_eval_json( + rows_dir, out, + base_metrics={"phase1_count": 999, "phase1_rate": 0.42}, + ) + metrics = json.loads(out.read_text()) + assert "cascading_phase1" in metrics + assert metrics["cascading_phase1"]["counts"]["n1"] == 1 + # phase1_count is REWRITTEN from cascade N1 (back-compat alias); + # it must match counts["n1"], NOT the stale base value. + assert metrics["phase1_count"] == 1 + assert metrics["phase1_count"] == metrics["cascading_phase1"]["counts"]["n1"] + # And phase1_rate is rewritten too: 1/2 = 0.5. + assert metrics["phase1_rate"] == pytest.approx(0.5) + # Legacy dual-eval keys MUST be absent. + for k in ( + "phase1_count_audited", "phase1_count_original", + "phase1_rate_audited", "phase1_rate_original", + "n_dual_eval_tasks", # moved INTO cascading_phase1 + ): + assert k not in metrics, ( + f"legacy key {k} must be removed from eval.json top level" + ) diff --git a/tests/test_claude_sdk_otf_agent.py b/tests/test_claude_sdk_otf_agent.py index f3343eca..a0610d29 100644 --- a/tests/test_claude_sdk_otf_agent.py +++ b/tests/test_claude_sdk_otf_agent.py @@ -728,7 +728,6 @@ async def test_run_task_exception_path_propagates_partial_result( submission_status="submitted_ok", phase1_passed=True, phase2_passed=True, total_reward=0.75, phase2_observation="p2 ok", - phase1_passed_audited=True, phase1_passed_original=False, phase1_observation_audited="audited-obs", phase1_observation_original="original-obs", ) @@ -757,8 +756,6 @@ async def test_run_task_exception_path_propagates_partial_result( assert row["submitted_query"] == "{\"models\": [\"m\"]}" assert row["submitted_sql"] == "SELECT 1" # Dual-eval columns: still pass-through. - assert row["phase1_passed_audited"] is True - assert row["phase1_passed_original"] is False assert row["phase1_observation_audited"] == "audited-obs" assert row["phase1_observation_original"] == "original-obs" diff --git a/tests/test_claude_sdk_otf_ainteract_agent.py b/tests/test_claude_sdk_otf_ainteract_agent.py index b47a34ca..f03645be 100644 --- a/tests/test_claude_sdk_otf_ainteract_agent.py +++ b/tests/test_claude_sdk_otf_ainteract_agent.py @@ -952,7 +952,6 @@ async def test_run_task_exception_path_propagates_partial_result( prefill = _full_prefill( phase2_passed=True, total_reward=0.75, phase2_observation="p2 ok", - phase1_passed_audited=True, phase1_passed_original=False, phase1_observation_audited="audited-obs", phase1_observation_original="original-obs", ) @@ -978,8 +977,6 @@ async def test_run_task_exception_path_propagates_partial_result( assert row["total_reward"] == 0.75 assert row["submitted_query"] == "{\"models\": [\"m\"]}" assert row["submitted_sql"] == "SELECT 1" - assert row["phase1_passed_audited"] is True - assert row["phase1_passed_original"] is False assert row["phase1_observation_audited"] == "audited-obs" assert row["phase1_observation_original"] == "original-obs" diff --git a/tests/test_claude_sdk_usage.py b/tests/test_claude_sdk_usage.py index 3dc2a498..f580b0c8 100644 --- a/tests/test_claude_sdk_usage.py +++ b/tests/test_claude_sdk_usage.py @@ -334,7 +334,6 @@ async def test_run_task_exception_path_propagates_partial_result(monkeypatch): prefill = _full_cs_prefill( phase2_passed=True, total_reward=0.75, phase2_observation="p2 ok", - phase1_passed_audited=True, phase1_passed_original=False, phase1_observation_audited="audited-obs", phase1_observation_original="original-obs", ) @@ -364,7 +363,5 @@ async def test_run_task_exception_path_propagates_partial_result(monkeypatch): assert row["submitted_query"] == "{\"models\": [\"m\"]}" assert row["submitted_sql"] == "SELECT 1" # Dual-eval columns: still pass-through. - assert row["phase1_passed_audited"] is True - assert row["phase1_passed_original"] is False assert row["phase1_observation_audited"] == "audited-obs" assert row["phase1_observation_original"] == "original-obs" diff --git a/tests/test_dual_eval.py b/tests/test_dual_eval.py index 5870ccdb..e8abb227 100644 --- a/tests/test_dual_eval.py +++ b/tests/test_dual_eval.py @@ -357,51 +357,15 @@ def _minimal_row_kwargs() -> dict: ) -def test_task_result_row_has_dual_eval_columns(): - """The schema must accept the new dual-eval fields. We instantiate - with explicit values to verify the fields are wired through.""" - from bird_interact_agents.results_db import TaskResultRow - - row = TaskResultRow( - **_minimal_row_kwargs(), - phase1_passed_audited=True, - phase1_passed_original=False, - phase1_observation_audited="ok-audit", - phase1_observation_original="fail-orig", - ) - assert row.phase1_passed_audited is True - assert row.phase1_passed_original is False - assert row.phase1_observation_audited == "ok-audit" - assert row.phase1_observation_original == "fail-orig" - - -def test_task_result_row_dual_eval_columns_default_none(): - """Single-eval call sites that don't pass the dual fields still work.""" - from bird_interact_agents.results_db import TaskResultRow - - row = TaskResultRow(**_minimal_row_kwargs()) - assert row.phase1_passed_audited is None - assert row.phase1_passed_original is None - assert row.phase1_observation_audited is None - assert row.phase1_observation_original is None - - -def test_results_db_open_creates_dual_eval_columns(tmp_path): - """Schema migration: a fresh results.db must have all four new - columns so insert_task_result can write them.""" - from bird_interact_agents.results_db import open_db - - db_path = tmp_path / "results.db" - conn = open_db(db_path) - try: - cur = conn.execute("PRAGMA table_info(task_results)") - cols = {row[1] for row in cur.fetchall()} - finally: - conn.close() - assert "phase1_passed_audited" in cols - assert "phase1_passed_original" in cols - assert "phase1_observation_audited" in cols - assert "phase1_observation_original" in cols +# DEV-1515: the per-row `phase1_passed_audited` / `phase1_passed_original` +# raw bool columns have been removed; the per-task cascade verdict now +# lives in the SubmissionAnnotation. Tests covering that schema are in +# `tests/test_schema_extension.py` and the cascading aggregator tests in +# `tests/test_cascading_report.py`. The three tests previously here +# (`test_task_result_row_has_dual_eval_columns`, +# `test_task_result_row_dual_eval_columns_default_none`, +# `test_results_db_open_creates_dual_eval_columns`) are intentionally +# removed. # --------------------------------------------------------------------------- @@ -832,126 +796,12 @@ def test_overlay_benchmark_kwarg_mini_interact_uses_single_file(tmp_path): assert task["sol_sql"] == ["SELECT audited FROM t"] -def test_insert_task_result_round_trip_dual_eval_fields(tmp_path): - """The TaskResultRow → SQL insert must actually persist the new - fields. The current insert in results_db.py uses an explicit column - list, so adding model fields without touching the INSERT silently - drops them — round-trip catches that.""" - from bird_interact_agents.results_db import TaskResultRow, insert_task_result, open_db - - db_path = tmp_path / "results.db" - conn = open_db(db_path) - try: - row = TaskResultRow( - **_minimal_row_kwargs(), - phase1_passed_audited=True, - phase1_passed_original=False, - phase1_observation_audited="aud-ok", - phase1_observation_original="orig-fail", - ) - insert_task_result(conn, row) - result = conn.execute( - """SELECT phase1_passed_audited, phase1_passed_original, - phase1_observation_audited, phase1_observation_original - FROM task_results WHERE instance_id = 'alien_1'""" - ).fetchone() - finally: - conn.close() - assert result == (1, 0, "aud-ok", "orig-fail") - - -# --------------------------------------------------------------------------- -# `run.py` aggregates dual-eval rates into eval.json -# --------------------------------------------------------------------------- - - -def test_submit_writes_dual_fields_to_state_result(monkeypatch): - """Dual-eval fields must land on `state.result` so each framework's - finalizer can copy them out. Tests `submit_slayer_query`'s plumbing - end-to-end without going through a real evaluator.""" - from types import SimpleNamespace - from bird_interact_agents.agents import _submit - - from bird_interact_agents import harness - - monkeypatch.setattr(_submit, "_dry_run_sql", lambda *a, **kw: None) - monkeypatch.setattr(_submit, "capture_result_snapshot", lambda *a, **kw: None) - - def fake_eval(sol_sql, status, dpb): - gold = status.original_data["sol_sql"][0] - passed = "audited" in gold - return (f"obs:{gold}", 1.0 if passed else 0.0, passed, False, True) - # The dispatcher calls `evaluate_dual_gold` (in harness.py) when - # `original_sol_sql` is set; that helper in turn calls - # `harness.execute_submit_action`. Patch the harness binding. - monkeypatch.setattr(harness, "execute_submit_action", fake_eval) - monkeypatch.setattr(_submit, "execute_submit_action", fake_eval) - - state = SimpleNamespace( - status=SimpleNamespace( - original_data={ - "selected_database": "fake_db", - "sol_sql": ["SELECT audited FROM t"], - "original_sol_sql": ["SELECT original FROM t"], - }, - remaining_budget=100.0, - total_budget=100.0, - force_submit=False, - current_phase=1, - ), - data_path_base="/tmp/ignored", - user_sim_model="anthropic/claude-haiku-4-5-20251001", - user_sim_prompt_version="v2", - slayer_storage_dir="", - result=None, - ) - - fake_client = SimpleNamespace(sql_sync=lambda d: "SELECT 1") - _submit.submit_slayer_query( - state, - query_json='{"models": ["m"]}', - slayer_client_factory=lambda s: fake_client, - ) - # The four dual-eval keys MUST be present on state.result — every - # framework finalizer reads them via submitter.get(...) / result.get(...). - assert state.result["phase1_passed_audited"] is True - assert state.result["phase1_passed_original"] is False - assert "audited" in state.result["phase1_observation_audited"] - assert "original" in state.result["phase1_observation_original"] - - -def test_run_aggregation_emits_dual_eval_rates(tmp_path): - """End of run: eval.json should carry phase1_count_audited / - phase1_count_original / phase1_rate_audited / phase1_rate_original - when at least one task has dual-eval columns populated. - - Strategy: write a known set of task results to a fresh results.db, - then call run.py's aggregation entry point and parse eval.json.""" - pytest.importorskip("bird_interact_agents.run") - from bird_interact_agents.results_db import TaskResultRow, insert_task_result, open_db - from bird_interact_agents.run import build_aggregate_eval - - db_path = tmp_path / "results.db" - conn = open_db(db_path) - try: - for i, (aud, orig) in enumerate([(True, True), (True, False), (False, False)], 1): - insert_task_result(conn, TaskResultRow( - **{**_minimal_row_kwargs(), - "instance_id": f"alien_{i}", - "phase1_passed": aud, # primary = audited - "submission_status": "passed_phase1" if aud else "wrong_result"}, - phase1_passed_audited=aud, - phase1_passed_original=orig, - )) - conn.commit() - finally: - conn.close() - - agg = build_aggregate_eval(db_path=db_path) - - # 3 tasks: 2 audited-pass, 1 original-pass. - assert agg["phase1_count_audited"] == 2 - assert agg["phase1_count_original"] == 1 - # Rates are derived from the same n. - assert agg["phase1_rate_audited"] == pytest.approx(2 / 3) - assert agg["phase1_rate_original"] == pytest.approx(1 / 3) +# DEV-1515: the three end-of-pipeline dual-eval tests previously here +# (`test_insert_task_result_round_trip_dual_eval_fields`, +# `test_submit_writes_dual_fields_to_state_result`, +# `test_run_aggregation_emits_dual_eval_rates`) covered the removed +# pre-DEV-1515 per-task raw bool persistence + state.result emission + +# eval.json rates. The equivalents under the cascade are in +# `tests/test_local_run_cascading.py`, +# `tests/test_cascading_report.py`, and the legacy-removal grep-sweep in +# `tests/test_legacy_field_removal.py`. diff --git a/tests/test_eval_annotate_cli.py b/tests/test_eval_annotate_cli.py new file mode 100644 index 00000000..41c7bd29 --- /dev/null +++ b/tests/test_eval_annotate_cli.py @@ -0,0 +1,405 @@ +"""DEV-1515: annotation-skeleton CLI tests for +``python -m bird_interact_agents.eval.annotate``. + +Pins: +* Auto-fills mechanical fields (instance_id, masked_terms from + critical_ambiguity, submission.* from trajectory, etc.). +* Leaves PENDING_HUMAN_REVIEW sentinels in human-judgment fields. +* ``--task-mode init`` skips existing files (default). +* ``--task-mode refresh`` preserves non-sentinel fields, overwrites + mechanical fields only. +* ``--task-mode force-all`` overwrites the whole file. +* ``--submission-mode overwrite`` (default) always rewrites. +* ``--submission-mode init`` skips when file exists. +* ``--dry-run`` writes nothing. +""" +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + + +SAMPLE_TASK_ROW = { + "instance_id": "alien_1", + "selected_database": "alien", + "amb_user_query": "Find some aliens.", + "external_knowledge": [1, 2, 3], + "sol_sql": ["SELECT gold FROM aliens"], + "user_query_ambiguity": { + "critical_ambiguity": [ + {"term": "some", "type": "intent_ambiguity"}, + {"term": "aliens", "type": "schema_linking_ambiguity"}, + ], + }, +} + + +def _write_attempt_json(rows_dir: Path, instance_id: str, *, submitted_sql: str): + d = rows_dir / instance_id + d.mkdir(parents=True, exist_ok=True) + (d / "attempt-1.json").write_text(json.dumps({ + "instance_id": instance_id, + "submitted_sql": submitted_sql, + "trajectory": [ + {"role": "agent", "content": "thinking"}, + {"role": "tool_call", "name": "ask_user", "args": {"q": "?"}}, + {"role": "user_sim", "content": "answer-1"}, + ], + "duration_s": 12.3, + "usage": {"cost_usd_agent": 0.42, "cost_usd_user_sim": 0.01, + "n_agent_turns": 3, "n_ask_user_calls": 1}, + "predicted_row_count": 5, + })) + + +def test_task_annotation_skeleton_fills_mechanical_fields(tmp_path): + from bird_interact_agents.eval.annotate import generate_task_annotation + + ann = generate_task_annotation( + task_row=SAMPLE_TASK_ROW, + benchmark="mini-interact", + ) + assert ann.instance_id == "alien_1" + assert ann.selected_database == "alien" + assert ann.amb_user_query == "Find some aliens." + assert ann.external_knowledge == [1, 2, 3] + assert [m.term for m in ann.masked_terms] == ["some", "aliens"] + assert ann.provenance.task_jsonl_instance_id == "alien_1" + + +def test_task_annotation_skeleton_leaves_sentinels(tmp_path): + from bird_interact_agents.eval.annotate import ( + PENDING_HUMAN_REVIEW, + generate_task_annotation, + ) + + ann = generate_task_annotation( + task_row=SAMPLE_TASK_ROW, benchmark="mini-interact", + ) + assert ann.metadata_sufficiency.rationale == PENDING_HUMAN_REVIEW + assert ann.evaluator_prompt is None + + +def test_submission_annotation_skeleton_fills_from_trajectory(tmp_path): + from bird_interact_agents.eval.annotate import ( + generate_submission_annotation, + ) + + rows_dir = tmp_path / "rows" + _write_attempt_json(rows_dir, "alien_1", submitted_sql="SELECT predicted") + + class StubGrader: + def __call__(self, *, submitted_sql, **_kw): + from bird_interact_agents.eval.tolerant_grader import CascadeVerdict + return CascadeVerdict( + n1_original_gold=True, n2_audited_primary=True, + n3_any_audited_variant=True, n4_tie_order=True, + n5_llm_judge=True, n6_numeric_epsilon=True, + n7_trailing_whitespace=True, n8_column_order=True, + matched_variant_id="primary", + novel_reading_judgment=None, + variant_matches=[], rowset_relations=[], + ) + + ann = generate_submission_annotation( + rows_dir=rows_dir, instance_id="alien_1", + selected_database="alien", benchmark="mini-interact", + run_id="r1", task_row=SAMPLE_TASK_ROW, + grader=StubGrader(), + ) + assert ann.submission.duration_s == 12.3 + assert ann.submission.cost_usd_agent == 0.42 + assert ann.submission.n_ask_user_calls == 1 + assert ann.evaluation.phase1_against_original_gold == "pass" + # user_sim_interaction walks the trajectory for ask_user calls. + assert ann.user_sim_interaction.n_asks == 1 + + +# --------------------------------------------------------------------------- +# Mode matrix +# --------------------------------------------------------------------------- + + +def _write_existing_task_annotation( + tmp_path: Path, + *, + benchmark: str = "mini-interact", + db: str = "alien", + instance_id: str = "alien_1", + rationale: str = "human-written", + evaluator_prompt: str | None = "evaluator-rules", +) -> Path: + from bird_interact_agents.eval import ( + AuditedGoldRef, GoldVariantRef, MaskedTerm, + MetadataSufficiency, TaskAnnotation, task_annotation_path, + write_task_annotation, + ) + from bird_interact_agents.eval.annotation_schema import Provenance + + ann = TaskAnnotation( + instance_id=instance_id, selected_database=db, + annotated_by="human", annotated_at="2026-05-30", + amb_user_query="Find some aliens.", + external_knowledge=[], + masked_terms=[MaskedTerm(term="some", type="intent_ambiguity")], + metadata_sufficiency=MetadataSufficiency( + verdict="insufficient", rationale=rationale, + ), + gold_variants=[ + GoldVariantRef( + variant_id="primary", interpretation="human-written-interp", + primary=True, + audited_gold_ref=AuditedGoldRef( + file="audited_gold/mini_interact_audited.jsonl", + instance_id=instance_id, + ), + ), + ], + evaluator_prompt=evaluator_prompt, + provenance=Provenance( + task_jsonl_path="mini_interact.jsonl", + task_jsonl_instance_id=instance_id, + ), + ) + p = task_annotation_path( + benchmark=benchmark, selected_database=db, + instance_id=instance_id, repo_root=tmp_path, + ) + write_task_annotation(ann, p) + return p + + +def test_task_mode_init_skips_existing(tmp_path, monkeypatch): + from bird_interact_agents import paths as paths_mod + monkeypatch.setattr(paths_mod, "main_checkout_root", lambda: tmp_path) + + p = _write_existing_task_annotation(tmp_path) + pre_mtime = p.stat().st_mtime_ns + + from bird_interact_agents.eval.annotate import write_task_skeleton + + written = write_task_skeleton( + task_row=SAMPLE_TASK_ROW, benchmark="mini-interact", + mode="init", dry_run=False, repo_root=tmp_path, + ) + assert written is None # signal "skipped existing" + assert p.stat().st_mtime_ns == pre_mtime # untouched + + +def test_task_mode_refresh_overwrites_mechanical_only(tmp_path, monkeypatch): + from bird_interact_agents import paths as paths_mod + monkeypatch.setattr(paths_mod, "main_checkout_root", lambda: tmp_path) + + p = _write_existing_task_annotation( + tmp_path, rationale="human-written-RAT", + evaluator_prompt="human-rules", + ) + + # Add a new ambiguity term to the task row — refresh should pull it in. + new_row = dict(SAMPLE_TASK_ROW) + new_row["user_query_ambiguity"] = { + "critical_ambiguity": [ + {"term": "some", "type": "intent_ambiguity"}, + {"term": "aliens", "type": "schema_linking_ambiguity"}, + {"term": "Find", "type": "intent_ambiguity"}, + ], + } + + from bird_interact_agents.eval.annotate import write_task_skeleton + + written_path = write_task_skeleton( + task_row=new_row, benchmark="mini-interact", + mode="refresh", dry_run=False, repo_root=tmp_path, + ) + assert written_path == p + + from bird_interact_agents.eval import read_task_annotation + refreshed = read_task_annotation(p) + # Mechanical (masked_terms) updated. + assert {m.term for m in refreshed.masked_terms} == {"some", "aliens", "Find"} + # Human-judgment fields preserved. + assert refreshed.metadata_sufficiency.rationale == "human-written-RAT" + assert refreshed.evaluator_prompt == "human-rules" + + +def test_task_mode_force_all_overwrites_everything(tmp_path, monkeypatch): + from bird_interact_agents import paths as paths_mod + monkeypatch.setattr(paths_mod, "main_checkout_root", lambda: tmp_path) + + _write_existing_task_annotation( + tmp_path, rationale="human-written-RAT", + evaluator_prompt="human-rules", + ) + + from bird_interact_agents.eval.annotate import ( + PENDING_HUMAN_REVIEW, write_task_skeleton, + ) + + p = write_task_skeleton( + task_row=SAMPLE_TASK_ROW, benchmark="mini-interact", + mode="force-all", dry_run=False, repo_root=tmp_path, + ) + from bird_interact_agents.eval import read_task_annotation + fresh = read_task_annotation(p) + # Human edits BLOWN AWAY — sentinel restored. + assert fresh.metadata_sufficiency.rationale == PENDING_HUMAN_REVIEW + assert fresh.evaluator_prompt is None + + +def test_dry_run_writes_nothing(tmp_path, monkeypatch): + from bird_interact_agents import paths as paths_mod + monkeypatch.setattr(paths_mod, "main_checkout_root", lambda: tmp_path) + + from bird_interact_agents.eval.annotate import write_task_skeleton + + written = write_task_skeleton( + task_row=SAMPLE_TASK_ROW, benchmark="mini-interact", + mode="force-all", dry_run=True, repo_root=tmp_path, + ) + # Dry-run reports the path it WOULD write to, but no file lands. + assert written is not None + assert not written.exists() + + +def test_submission_mode_overwrite_rewrites_existing(tmp_path, monkeypatch): + """Default submission-mode (overwrite) replaces the existing file.""" + from bird_interact_agents import paths as paths_mod + monkeypatch.setattr(paths_mod, "main_checkout_root", lambda: tmp_path) + + from bird_interact_agents.eval import ( + FailureClassification, SubmissionAnnotation, SubmissionEvaluation, + SubmissionMetadata, submission_annotation_path, + write_submission_annotation, + ) + existing = SubmissionAnnotation( + instance_id="alien_1", selected_database="alien", + task_annotation_ref="x", annotated_by="stale-author", + annotated_at="2026-05-30", + submission=SubmissionMetadata(cloud_run_id="r1", trajectory_path="t"), + evaluation=SubmissionEvaluation( + phase1_against_original_gold="pass", + phase1_against_audited_primary="pass", + phase1_against_any_audited_variant="pass", + verdict="correct", + ), + failure_classification=FailureClassification( + primary="other", agent_at_fault=False, remediation_target="other", + ), + ) + p = submission_annotation_path( + benchmark="mini-interact", selected_database="alien", + instance_id="alien_1", run_id="r1", repo_root=tmp_path, + ) + write_submission_annotation(existing, p) + + rows_dir = tmp_path / "rows" + _write_attempt_json(rows_dir, "alien_1", submitted_sql="SELECT x") + + class StubGrader: + def __call__(self, **_kw): + from bird_interact_agents.eval.tolerant_grader import CascadeVerdict + return CascadeVerdict( + n1_original_gold=True, n2_audited_primary=True, + n3_any_audited_variant=True, n4_tie_order=True, + n5_llm_judge=True, n6_numeric_epsilon=True, + n7_trailing_whitespace=True, n8_column_order=True, + matched_variant_id="primary", novel_reading_judgment=None, + variant_matches=[], rowset_relations=[], + ) + + from bird_interact_agents.eval.annotate import write_submission_skeleton + written = write_submission_skeleton( + rows_dir=rows_dir, instance_id="alien_1", + selected_database="alien", benchmark="mini-interact", + run_id="r1", task_row=SAMPLE_TASK_ROW, + grader=StubGrader(), mode="overwrite", + dry_run=False, repo_root=tmp_path, + ) + assert written == p + from bird_interact_agents.eval import read_submission_annotation + fresh = read_submission_annotation(p) + assert fresh.annotated_by != "stale-author" + + +def test_submission_dry_run_writes_nothing(tmp_path, monkeypatch): + from bird_interact_agents import paths as paths_mod + monkeypatch.setattr(paths_mod, "main_checkout_root", lambda: tmp_path) + + rows_dir = tmp_path / "rows" + _write_attempt_json(rows_dir, "alien_1", submitted_sql="SELECT x") + + class StubGrader: + def __call__(self, **_kw): + from bird_interact_agents.eval.tolerant_grader import CascadeVerdict + return CascadeVerdict( + n1_original_gold=True, n2_audited_primary=True, + n3_any_audited_variant=True, n4_tie_order=True, + n5_llm_judge=True, n6_numeric_epsilon=True, + n7_trailing_whitespace=True, n8_column_order=True, + matched_variant_id="primary", novel_reading_judgment=None, + variant_matches=[], rowset_relations=[], + ) + + from bird_interact_agents.eval.annotate import write_submission_skeleton + written = write_submission_skeleton( + rows_dir=rows_dir, instance_id="alien_1", + selected_database="alien", benchmark="mini-interact", + run_id="r1", task_row=SAMPLE_TASK_ROW, + grader=StubGrader(), mode="overwrite", + dry_run=True, repo_root=tmp_path, + ) + assert written is not None + assert not written.exists() + + +def test_submission_mode_init_skips_existing(tmp_path, monkeypatch): + from bird_interact_agents import paths as paths_mod + monkeypatch.setattr(paths_mod, "main_checkout_root", lambda: tmp_path) + + from bird_interact_agents.eval import ( + FailureClassification, SubmissionAnnotation, SubmissionEvaluation, + SubmissionMetadata, submission_annotation_path, + write_submission_annotation, + ) + existing = SubmissionAnnotation( + instance_id="alien_1", selected_database="alien", + task_annotation_ref="x", annotated_by="human", annotated_at="2026-05-30", + submission=SubmissionMetadata( + cloud_run_id="r1", trajectory_path="t", + ), + evaluation=SubmissionEvaluation( + phase1_against_original_gold="pass", + phase1_against_audited_primary="pass", + phase1_against_any_audited_variant="pass", + verdict="correct", + ), + failure_classification=FailureClassification( + primary="other", agent_at_fault=False, remediation_target="other", + ), + ) + p = submission_annotation_path( + benchmark="mini-interact", selected_database="alien", + instance_id="alien_1", run_id="r1", repo_root=tmp_path, + ) + write_submission_annotation(existing, p) + pre = p.read_bytes() + + rows_dir = tmp_path / "rows" + _write_attempt_json(rows_dir, "alien_1", submitted_sql="SELECT x") + + class StubGrader: + def __call__(self, **_kw): + raise AssertionError("should not be invoked in init-skip path") + + from bird_interact_agents.eval.annotate import write_submission_skeleton + written = write_submission_skeleton( + rows_dir=rows_dir, instance_id="alien_1", + selected_database="alien", benchmark="mini-interact", + run_id="r1", task_row=SAMPLE_TASK_ROW, + grader=StubGrader(), mode="init", + dry_run=False, repo_root=tmp_path, + ) + assert written is None + assert p.read_bytes() == pre diff --git a/tests/test_implicit_task_annotation.py b/tests/test_implicit_task_annotation.py new file mode 100644 index 00000000..56446210 --- /dev/null +++ b/tests/test_implicit_task_annotation.py @@ -0,0 +1,115 @@ +"""DEV-1515: implicit-default TaskAnnotation for instances without an +on-disk annotation file. + +Contract: +* ``implicit_task_annotation(...)`` returns a schema-valid + ``TaskAnnotation`` (the grader cannot use a "verdict='sufficient'" + shortcut because TaskAnnotation has many required fields). +* Verdict is ``sufficient`` + ``original_gold_is_correct=True`` so the + cascade collapses to N1 (audited primary == original gold). +* ``gold_variants`` is empty. +* ``evaluator_prompt`` is None — the LLM judge MUST NOT fire on these. +* The factory is a pure builder; it must not touch + ``paths.annotations_root()`` or write any file. +* Re-running the grader against an instance without an annotation file + produces zero new files under ``annotations/``. +""" +from __future__ import annotations + + +def test_implicit_task_annotation_is_schema_valid(): + from bird_interact_agents.eval.annotation_schema import TaskAnnotation + from bird_interact_agents.eval.implicit_annotation import ( + implicit_task_annotation, + ) + + ann = implicit_task_annotation( + instance_id="alien_1", + selected_database="alien", + benchmark="mini-interact", + amb_user_query="Some query.", + ) + assert isinstance(ann, TaskAnnotation) + assert ann.instance_id == "alien_1" + assert ann.selected_database == "alien" + assert ann.metadata_sufficiency.verdict == "sufficient" + assert ann.original_gold_is_correct is True + assert ann.gold_variants == [] + assert ann.evaluator_prompt is None + + +def test_implicit_task_annotation_provenance_is_set(): + from bird_interact_agents.eval.implicit_annotation import ( + implicit_task_annotation, + ) + + ann = implicit_task_annotation( + instance_id="alien_1", + selected_database="alien", + benchmark="mini-interact", + amb_user_query="x", + ) + assert ann.provenance.task_jsonl_instance_id == "alien_1" + # The exact JSONL path is benchmark-specific; the key invariant is + # that it's non-empty so downstream traceability works. + assert ann.provenance.task_jsonl_path + + +def test_implicit_task_annotation_round_trips_through_pydantic(): + """An implicit annotation should serialize+revalidate, so the grader + can hand it to any code path that expects a TaskAnnotation from disk.""" + from bird_interact_agents.eval.annotation_schema import TaskAnnotation + from bird_interact_agents.eval.implicit_annotation import ( + implicit_task_annotation, + ) + + ann = implicit_task_annotation( + instance_id="alien_1", + selected_database="alien", + benchmark="mini-interact", + amb_user_query="x", + ) + revalidated = TaskAnnotation.model_validate_json(ann.model_dump_json()) + assert revalidated == ann + + +def test_implicit_task_annotation_does_not_write_to_disk(tmp_path, monkeypatch): + """The factory is pure — it must not call any I/O helper. Monkeypatch + write_task_annotation to raise; the factory should still return.""" + import bird_interact_agents.eval.annotation_io as io_mod + + def _no_write(*_a, **_kw): + raise AssertionError( + "implicit_task_annotation must not write to disk" + ) + + monkeypatch.setattr(io_mod, "write_task_annotation", _no_write) + + from bird_interact_agents.eval.implicit_annotation import ( + implicit_task_annotation, + ) + + implicit_task_annotation( + instance_id="alien_1", + selected_database="alien", + benchmark="mini-interact", + amb_user_query="x", + ) + + +def test_implicit_marker_distinguishes_from_human_authored(): + """Downstream consumers must be able to tell that an annotation + was auto-synthesized vs. human-authored. ``annotated_by`` carries + the marker.""" + from bird_interact_agents.eval.implicit_annotation import ( + IMPLICIT_ANNOTATED_BY, + implicit_task_annotation, + ) + + ann = implicit_task_annotation( + instance_id="alien_1", + selected_database="alien", + benchmark="mini-interact", + amb_user_query="x", + ) + assert ann.annotated_by == IMPLICIT_ANNOTATED_BY diff --git a/tests/test_legacy_field_removal.py b/tests/test_legacy_field_removal.py new file mode 100644 index 00000000..dd3a24f0 --- /dev/null +++ b/tests/test_legacy_field_removal.py @@ -0,0 +1,93 @@ +"""DEV-1515: grep-sweep regression — no production code emits the legacy +``phase1_passed_audited`` / ``phase1_passed_original`` raw fields. + +The fields are replaced by per-row ``submission_annotation.json`` files +produced by tolerant_grader. Every place that previously wrote the raw +bools must now reference the new path instead. + +This test grep-scans the source tree at runtime so adding a new agent +that re-introduces the legacy fields trips immediately. +""" +from __future__ import annotations + +import re +from pathlib import Path + +import bird_interact_agents + + +def _bird_src_root() -> Path: + return Path(bird_interact_agents.__file__).resolve().parent + + +_LEGACY_PATTERN = re.compile( + r"\bphase1_passed_(?:audited|original)\b" +) + +# Files allowed to mention the legacy fields purely in a comment / changelog +# context (no executable use). Keep this list intentionally tight; if a +# production path needs an exception, the test should be the gate. +_ALLOWED_FILES_WITH_MENTION: set[str] = set() +# Future comment-only entries (e.g. a CHANGELOG note in code) belong here. + + +def test_no_production_code_references_legacy_dual_eval_fields(): + src_root = _bird_src_root() + offenders: list[tuple[str, int, str]] = [] + for py in src_root.rglob("*.py"): + if py.name == "__pycache__": + continue + rel = str(py.relative_to(src_root)) + if rel in _ALLOWED_FILES_WITH_MENTION: + continue + for i, line in enumerate(py.read_text().splitlines(), start=1): + if _LEGACY_PATTERN.search(line): + offenders.append((rel, i, line.strip())) + assert not offenders, ( + "DEV-1515: the following lines still reference the legacy " + "phase1_passed_audited / phase1_passed_original fields. " + "Replace with submission_annotation.json plumbing:\n" + + "\n".join(f" {f}:{i}: {ln}" for f, i, ln in offenders) + ) + + +def test_results_db_schema_does_not_define_legacy_columns(): + """Explicit catch for the SQLite CREATE TABLE / dataclass row.""" + from bird_interact_agents import results_db + + src = Path(results_db.__file__).read_text() + assert "phase1_passed_audited" not in src + assert "phase1_passed_original" not in src + + +def test_agents_submit_helpers_drop_legacy_kwargs(): + """The shared submit-helper signatures must NOT accept the legacy + kwargs any more — callers should rely on the worker-side grader.""" + import inspect + from bird_interact_agents.agents import _submit + + for name in ( + "_diagnostic_payload", + "submit_raw_sql", + "submit_slayer_query", + ): + fn = getattr(_submit, name, None) + if fn is None: + continue + sig = inspect.signature(fn) + for forbidden in ("phase1_passed_audited", "phase1_passed_original"): + assert forbidden not in sig.parameters, ( + f"{name}: legacy kwarg {forbidden!r} must be removed" + ) + + +def test_build_aggregate_eval_does_not_query_legacy_columns(): + """``run.py::build_aggregate_eval`` previously SELECTed + ``phase1_passed_audited`` / ``phase1_passed_original``. Whatever + replacement helper it now provides must NOT name those columns.""" + import inspect + from bird_interact_agents.run import build_aggregate_eval + + src = inspect.getsource(build_aggregate_eval) + assert "phase1_passed_audited" not in src + assert "phase1_passed_original" not in src diff --git a/tests/test_local_run_cascading.py b/tests/test_local_run_cascading.py new file mode 100644 index 00000000..bdde10fa --- /dev/null +++ b/tests/test_local_run_cascading.py @@ -0,0 +1,119 @@ +"""DEV-1515: local-run end-to-end cascading-report wiring. + +A local (non-cloud) run via ``run.py`` must produce the same +``cascading_phase1`` shape in its ``eval.json`` as a cloud run. The +shared inline grader path (``eval.grade_in_place``) is invoked per task +in both code paths. + +Tests stub the executor so no real benchmark DB or LLM API is required. +""" +from __future__ import annotations + +import json +from pathlib import Path + + +def test_grade_in_place_writes_submission_annotation_per_task(tmp_path): + """The shared helper writes one + ``//submission_annotation.json`` per task.""" + from bird_interact_agents.eval.grade_in_place import grade_and_write + from bird_interact_agents.eval.implicit_annotation import ( + implicit_task_annotation, + ) + + rows_dir = tmp_path / "rows" + rows_dir.mkdir() + + submitted = "SELECT predicted" + original_gold = "SELECT gold" + + class FakeExecutor: + def __call__(self, sql, *, db_path, conn): # noqa: ARG002,ARG005 # noqa: ARG002 + return ([(1,)], ["a"]) if sql == submitted else ([(1,)], ["a"]) + + grade_and_write( + rows_dir=rows_dir, + instance_id="alien_1", + benchmark="mini-interact", + run_id="local-test-run", + task_annotation=implicit_task_annotation( + instance_id="alien_1", selected_database="alien", + benchmark="mini-interact", amb_user_query="x", + ), + audited_gold_rows=[], + original_sol_sql=[original_gold], + submitted_sql=submitted, + db_path=Path("/dev/null"), + conn=None, + executor=FakeExecutor(), + trajectory_path="rows/alien_1/attempt-1.json", + cost_usd_agent=0.0, cost_usd_user_sim=0.0, + duration_s=0.5, n_agent_turns=1, n_ask_user_calls=0, + predicted_row_count=1, + llm_judge=None, + ) + + ann_path = rows_dir / "alien_1" / "submission_annotation.json" + assert ann_path.exists() + data = json.loads(ann_path.read_text()) + assert data["instance_id"] == "alien_1" + # Cascade should pass at N1 (predicted == gold). + assert data["evaluation"]["phase1_against_original_gold"] == "pass" + + +def test_local_run_eval_json_has_cascading_block(tmp_path): + """End-to-end: simulate a 2-task local run; final eval.json carries + cascading_phase1 derived from the per-row annotations.""" + from bird_interact_agents.eval.cascading_report import ( + emit_cascading_eval_json, + ) + from bird_interact_agents.eval.grade_in_place import grade_and_write + from bird_interact_agents.eval.implicit_annotation import ( + implicit_task_annotation, + ) + + rows_dir = tmp_path / "rows" + rows_dir.mkdir() + + class FakePass: + def __call__(self, sql, *, db_path, conn): # noqa: ARG002,ARG005 # noqa: ARG002 + return ([(1,)], ["a"]) + class FakeFail: + def __call__(self, sql, *, db_path, conn): # noqa: ARG002,ARG005 # noqa: ARG002 + if "predicted" in sql: + return ([(99,)], ["a"]) + return ([(1,)], ["a"]) + + for inst_id, exe in ( + ("t_pass", FakePass()), + ("t_fail", FakeFail()), + ): + grade_and_write( + rows_dir=rows_dir, + instance_id=inst_id, + benchmark="mini-interact", + run_id="local-test-run", + task_annotation=implicit_task_annotation( + instance_id=inst_id, selected_database="alien", + benchmark="mini-interact", amb_user_query="x", + ), + audited_gold_rows=[], + original_sol_sql=["SELECT gold"], + submitted_sql="SELECT predicted", + db_path=Path("/dev/null"), + conn=None, + executor=exe, + trajectory_path=f"rows/{inst_id}/attempt-1.json", + cost_usd_agent=0.0, cost_usd_user_sim=0.0, + duration_s=0.0, n_agent_turns=0, n_ask_user_calls=0, + predicted_row_count=1, + llm_judge=None, + ) + + out = tmp_path / "eval.json" + emit_cascading_eval_json( + rows_dir, out, base_metrics={"phase1_count": 1, "phase1_rate": 0.5}, + ) + metrics = json.loads(out.read_text()) + assert metrics["cascading_phase1"]["counts"]["n1"] == 1 + assert metrics["cascading_phase1"]["n_dual_eval_tasks"] == 2 diff --git a/tests/test_paths_annotations.py b/tests/test_paths_annotations.py new file mode 100644 index 00000000..5a6b70ac --- /dev/null +++ b/tests/test_paths_annotations.py @@ -0,0 +1,83 @@ +"""DEV-1515: `paths.annotations_root()` worktree-safety regression. + +Mirrors the contract used by `audited_gold_root()` / +`slayer_models_otf_root()` etc.: the helper MUST anchor at the main +checkout (via `main_checkout_root()`), so a `git worktree add` spawned +elsewhere reads/writes the same annotations directory. +""" +from __future__ import annotations + + +def test_annotations_root_anchors_at_main_checkout(monkeypatch, tmp_path): + """Patching `main_checkout_root` MUST move `annotations_root`.""" + import bird_interact_agents.paths as p + + monkeypatch.setattr(p, "main_checkout_root", lambda: tmp_path) + assert p.annotations_root() == tmp_path / "annotations" + + +def test_annotations_root_is_under_main_checkout_not_worktree(): + """`annotations_root()` must not contain `.worktrees` — that would + mean a worktree-spawned process is reading a throwaway copy.""" + import bird_interact_agents.paths as p + + root = p.annotations_root() + assert ".worktrees" not in root.parts + + +def test_annotation_io_uses_paths_helper(monkeypatch, tmp_path): + """Round-trip: write via annotation_io with the patched root, the + file lands under `/annotations///...`.""" + import bird_interact_agents.paths as paths_mod + from bird_interact_agents.eval import ( + AuditedGoldRef, + GoldVariantRef, + MaskedTerm, + MetadataSufficiency, + TaskAnnotation, + task_annotation_path, + write_task_annotation, + ) + from bird_interact_agents.eval.annotation_schema import Provenance + + monkeypatch.setattr(paths_mod, "main_checkout_root", lambda: tmp_path) + + ann = TaskAnnotation( + instance_id="alien_42", + selected_database="alien", + annotated_by="test", + annotated_at="2026-05-31", + amb_user_query="x", + external_knowledge=[1], + masked_terms=[MaskedTerm(term="x", type="knowledge_linking_ambiguity")], + metadata_sufficiency=MetadataSufficiency( + verdict="ambiguous", rationale="r" + ), + gold_variants=[ + GoldVariantRef( + variant_id="primary", + interpretation="x", + primary=True, + audited_gold_ref=AuditedGoldRef( + file="audited_gold/mini_interact_audited.jsonl", + instance_id="alien_42", + ), + ) + ], + provenance=Provenance( + task_jsonl_path="mini_interact.jsonl", + task_jsonl_instance_id="alien_42", + ), + ) + # The path helper accepts an explicit repo_root, but the production + # code path resolves through paths.main_checkout_root(); to exercise + # both, call without repo_root and patch the helper instead. + p_explicit = task_annotation_path( + benchmark="mini-interact", + selected_database="alien", + instance_id="alien_42", + repo_root=tmp_path, + ) + write_task_annotation(ann, p_explicit) + assert p_explicit.exists() + assert p_explicit.is_relative_to(tmp_path / "annotations") diff --git a/tests/test_schema_extension.py b/tests/test_schema_extension.py new file mode 100644 index 00000000..0f86fc72 --- /dev/null +++ b/tests/test_schema_extension.py @@ -0,0 +1,125 @@ +"""DEV-1515: pin the SubmissionEvaluation + VariantMatch schema extensions. + +The pre-DEV-1515 `SubmissionEvaluation` only carried N1-N5 fields. The +plan adds: +* `correct_under_numeric_epsilon: bool` +* `correct_under_trailing_whitespace: bool` +* `correct_under_column_order: bool` +* `numeric_epsilon: float` (records the threshold used) + +The pre-DEV-1515 `VariantMatch` only carried `variant_id` + `match`. +The plan extends it with an optional `informational: VariantInformational` +sub-block carrying: +* rowset_relation, column_count_match, + column_name_match_case_insensitive, column_order_match, +* first_divergent_row_index, first_divergent_cell_diff. + +These tests prove the schema actually carries the new fields, with +``extra="forbid"`` still in effect. +""" +from __future__ import annotations + +import pytest +from pydantic import ValidationError + + +def _base_submission_evaluation_kwargs() -> dict: + return dict( + phase1_against_original_gold="fail", + phase1_against_audited_primary="fail", + phase1_against_any_audited_variant="pass", + phase1_against_variants=[], + correct_up_to_tie_order=True, + novel_reading_judgment=None, + verdict="valid_interpretation", + matched_variant_id="primary", + rationale="", + ) + + +def test_submission_evaluation_carries_n6_through_n8_fields(): + from bird_interact_agents.eval import SubmissionEvaluation + + ev = SubmissionEvaluation( + **_base_submission_evaluation_kwargs(), + correct_under_numeric_epsilon=True, + correct_under_trailing_whitespace=True, + correct_under_column_order=False, + numeric_epsilon=1e-6, + ) + assert ev.correct_under_numeric_epsilon is True + assert ev.correct_under_trailing_whitespace is True + assert ev.correct_under_column_order is False + assert ev.numeric_epsilon == 1e-6 + + +def test_submission_evaluation_roundtrips_new_fields(): + from bird_interact_agents.eval import SubmissionEvaluation + + ev = SubmissionEvaluation( + **_base_submission_evaluation_kwargs(), + correct_under_numeric_epsilon=True, + correct_under_trailing_whitespace=False, + correct_under_column_order=True, + numeric_epsilon=5e-7, + ) + revalidated = SubmissionEvaluation.model_validate_json(ev.model_dump_json()) + assert revalidated == ev + + +def test_submission_evaluation_extra_forbid_still_holds(): + from bird_interact_agents.eval import SubmissionEvaluation + + payload = SubmissionEvaluation( + **_base_submission_evaluation_kwargs(), + correct_under_numeric_epsilon=False, + correct_under_trailing_whitespace=False, + correct_under_column_order=False, + numeric_epsilon=1e-6, + ).model_dump() + payload["a_field_not_in_schema"] = True + with pytest.raises(ValidationError): + SubmissionEvaluation.model_validate(payload) + + +def test_variant_match_has_optional_informational(): + from bird_interact_agents.eval import VariantMatch + from bird_interact_agents.eval.annotation_schema import ( + VariantInformational, + ) + + vm_without = VariantMatch(variant_id="primary", match="equal_rowset") + assert vm_without.informational is None + + info = VariantInformational( + rowset_relation="strict_subset_of", + column_count_match=True, + column_name_match_case_insensitive=False, + column_order_match=True, + first_divergent_row_index=3, + first_divergent_cell_diff="cell[3][1]: 'X' vs 'Y'", + ) + vm_with = VariantMatch( + variant_id="primary", match="strict_subset_of", + informational=info, + ) + assert vm_with.informational is not None + assert vm_with.informational.first_divergent_row_index == 3 + + +def test_variant_informational_extra_forbid(): + from bird_interact_agents.eval.annotation_schema import ( + VariantInformational, + ) + + payload = VariantInformational( + rowset_relation="equal_rowset", + column_count_match=True, + column_name_match_case_insensitive=True, + column_order_match=True, + first_divergent_row_index=None, + first_divergent_cell_diff=None, + ).model_dump() + payload["surprise"] = "not allowed" + with pytest.raises(ValidationError): + VariantInformational.model_validate(payload) diff --git a/tests/test_tolerant_grader_comparators.py b/tests/test_tolerant_grader_comparators.py new file mode 100644 index 00000000..b318349d --- /dev/null +++ b/tests/test_tolerant_grader_comparators.py @@ -0,0 +1,296 @@ +"""DEV-1515: pure-comparator unit tests for tolerant_grader. + +These exercise N4 (tie-order), N6 (numeric-epsilon), N7 (trailing- +whitespace), and N8 (column-order) on hand-canned row tuples. No SQLite +execution — purely the comparator predicates. + +The end-to-end cascade is tested separately in +``test_tolerant_grader_orchestration.py`` with a fake executor. +""" +from __future__ import annotations + + +# --------------------------------------------------------------------------- +# N4 — tie-order tolerance (bucket by ORDER BY columns; set-equal each bucket) +# --------------------------------------------------------------------------- + + +def test_n4_equal_rows_same_order_passes(): + from bird_interact_agents.eval.tolerant_grader import compare_tie_order + + pred = [("A", 1), ("B", 2), ("B", 3)] + gold = [("A", 1), ("B", 2), ("B", 3)] + assert compare_tie_order(pred, gold, orderby_indices=[0]) is True + + +def test_n4_reordered_within_bucket_passes(): + """Bucket by column 0 (ORDER BY first col). Rows with the same + bucket-key should set-equal within the bucket regardless of in-bucket + order.""" + from bird_interact_agents.eval.tolerant_grader import compare_tie_order + + pred = [("A", 1), ("B", 3), ("B", 2)] + gold = [("A", 1), ("B", 2), ("B", 3)] + assert compare_tie_order(pred, gold, orderby_indices=[0]) is True + + +def test_n4_reordered_across_buckets_fails(): + """Different bucket order is a real failure — ORDER BY column is + the user-visible ordering.""" + from bird_interact_agents.eval.tolerant_grader import compare_tie_order + + pred = [("B", 2), ("B", 3), ("A", 1)] + gold = [("A", 1), ("B", 2), ("B", 3)] + assert compare_tie_order(pred, gold, orderby_indices=[0]) is False + + +def test_n4_no_orderby_keys_collapses_to_strict(): + """An empty ``orderby_indices`` means "no ORDER BY in gold" — N4 + should fall back to set-equality (which is what N3 does).""" + from bird_interact_agents.eval.tolerant_grader import compare_tie_order + + pred = [("X", 1), ("Y", 2)] + gold = [("Y", 2), ("X", 1)] + assert compare_tie_order(pred, gold, orderby_indices=[]) is True + + +def test_n4_two_orderby_columns_bucket_by_tuple(): + """ORDER BY first_col, second_col → bucket key is the (col0, col1) tuple.""" + from bird_interact_agents.eval.tolerant_grader import compare_tie_order + + pred = [("A", 1, "Z"), ("A", 1, "Y"), ("B", 2, "X")] + gold = [("A", 1, "Y"), ("A", 1, "Z"), ("B", 2, "X")] + assert compare_tie_order(pred, gold, orderby_indices=[0, 1]) is True + + +# --------------------------------------------------------------------------- +# N6 — numeric-epsilon tolerance +# --------------------------------------------------------------------------- + + +def test_n6_default_epsilon_matches_close_floats(): + from bird_interact_agents.eval.tolerant_grader import ( + compare_numeric_epsilon, + ) + + pred = [(1.0000001,)] + gold = [(1.0,)] + assert compare_numeric_epsilon(pred, gold, epsilon=1e-6) is True + + +def test_n6_outside_epsilon_fails(): + from bird_interact_agents.eval.tolerant_grader import ( + compare_numeric_epsilon, + ) + + pred = [(1.01,)] + gold = [(1.0,)] + assert compare_numeric_epsilon(pred, gold, epsilon=1e-6) is False + + +def test_n6_non_numeric_cells_compared_strictly(): + """String cells must still strict-equal under N6 — epsilon is per-cell + type-aware.""" + from bird_interact_agents.eval.tolerant_grader import ( + compare_numeric_epsilon, + ) + + pred = [("hi", 1.0)] + gold = [("HI", 1.0)] + assert compare_numeric_epsilon(pred, gold, epsilon=1e-6) is False + + +def test_n6_int_and_float_with_same_value_match(): + """`1` (int) and `1.0` (float) should compare equal under numeric-eps.""" + from bird_interact_agents.eval.tolerant_grader import ( + compare_numeric_epsilon, + ) + + pred = [(1,)] + gold = [(1.0,)] + assert compare_numeric_epsilon(pred, gold, epsilon=1e-6) is True + + +def test_n6_null_cells_compared_strictly(): + """`None` vs `None` matches; `None` vs a value does not.""" + from bird_interact_agents.eval.tolerant_grader import ( + compare_numeric_epsilon, + ) + + assert compare_numeric_epsilon( + [(None,)], [(None,)], epsilon=1e-6, + ) is True + assert compare_numeric_epsilon( + [(None,)], [(0.0,)], epsilon=1e-6, + ) is False + + +def test_n6_row_count_mismatch_fails(): + from bird_interact_agents.eval.tolerant_grader import ( + compare_numeric_epsilon, + ) + + pred = [(1.0,), (2.0,)] + gold = [(1.0,)] + assert compare_numeric_epsilon(pred, gold, epsilon=1e-6) is False + + +# --------------------------------------------------------------------------- +# N7 — trailing-whitespace tolerance +# --------------------------------------------------------------------------- + + +def test_n7_trailing_whitespace_strip_matches(): + from bird_interact_agents.eval.tolerant_grader import ( + compare_trailing_whitespace, + ) + + pred = [("High Income ",), ("Low Income\t",)] + gold = [("High Income",), ("Low Income",)] + assert compare_trailing_whitespace(pred, gold) is True + + +def test_n7_internal_whitespace_preserved(): + """Stripping trailing only — internal spaces matter.""" + from bird_interact_agents.eval.tolerant_grader import ( + compare_trailing_whitespace, + ) + + pred = [("High Income",)] # two internal spaces + gold = [("High Income",)] + assert compare_trailing_whitespace(pred, gold) is False + + +def test_n7_non_string_cells_unchanged(): + """Numeric cells unaffected — they're not strings to strip.""" + from bird_interact_agents.eval.tolerant_grader import ( + compare_trailing_whitespace, + ) + + pred = [("A ", 1.0)] + gold = [("A", 1.0)] + assert compare_trailing_whitespace(pred, gold) is True + + +# --------------------------------------------------------------------------- +# N8 — column-order tolerance via case-insensitive name align +# --------------------------------------------------------------------------- + + +def test_n8_reordered_columns_match_by_name(): + from bird_interact_agents.eval.tolerant_grader import compare_column_order + + pred_rows = [("foo", 1), ("bar", 2)] + pred_cols = ["name", "id"] + gold_rows = [(1, "foo"), (2, "bar")] + gold_cols = ["id", "name"] + assert compare_column_order( + pred_rows, gold_rows, pred_cols=pred_cols, gold_cols=gold_cols, + ) is True + + +def test_n8_column_name_case_insensitive(): + from bird_interact_agents.eval.tolerant_grader import compare_column_order + + pred_rows = [(1, "foo")] + pred_cols = ["ID", "NAME"] + gold_rows = [(1, "foo")] + gold_cols = ["id", "name"] + assert compare_column_order( + pred_rows, gold_rows, pred_cols=pred_cols, gold_cols=gold_cols, + ) is True + + +def test_n8_missing_column_name_fails(): + """Predicted has an extra column or a different name — N8 cannot align.""" + from bird_interact_agents.eval.tolerant_grader import compare_column_order + + pred_rows = [(1, "foo")] + pred_cols = ["id", "label"] + gold_rows = [(1, "foo")] + gold_cols = ["id", "name"] + assert compare_column_order( + pred_rows, gold_rows, pred_cols=pred_cols, gold_cols=gold_cols, + ) is False + + +def test_n8_column_count_mismatch_fails(): + from bird_interact_agents.eval.tolerant_grader import compare_column_order + + pred_rows = [(1, "foo", "extra")] + pred_cols = ["id", "name", "extra"] + gold_rows = [(1, "foo")] + gold_cols = ["id", "name"] + assert compare_column_order( + pred_rows, gold_rows, pred_cols=pred_cols, gold_cols=gold_cols, + ) is False + + +# --------------------------------------------------------------------------- +# ORDER BY parser — N4 input +# --------------------------------------------------------------------------- + + +def test_parse_orderby_no_clause_returns_empty(): + from bird_interact_agents.eval.tolerant_grader import parse_orderby_keys + + assert parse_orderby_keys("SELECT a, b FROM t") == [] + + +def test_parse_orderby_named_column_resolves_to_select_index(): + from bird_interact_agents.eval.tolerant_grader import parse_orderby_keys + + keys = parse_orderby_keys("SELECT a, b FROM t ORDER BY b") + indices = [k.column_index for k in keys] + assert indices == [1] + + +def test_parse_orderby_bare_integer_uses_as_select_index(): + from bird_interact_agents.eval.tolerant_grader import parse_orderby_keys + + keys = parse_orderby_keys("SELECT a, b FROM t ORDER BY 2 DESC") + indices = [k.column_index for k in keys] + assert indices == [1] + + +def test_parse_orderby_alias_resolves_to_select_index(): + from bird_interact_agents.eval.tolerant_grader import parse_orderby_keys + + keys = parse_orderby_keys( + "SELECT a, b AS total FROM t ORDER BY total" + ) + indices = [k.column_index for k in keys] + assert indices == [1] + + +def test_parse_orderby_multiple_keys_preserved_in_order(): + from bird_interact_agents.eval.tolerant_grader import parse_orderby_keys + + keys = parse_orderby_keys( + "SELECT a, b, c FROM t ORDER BY a, c DESC, b" + ) + indices = [k.column_index for k in keys] + assert indices == [0, 2, 1] + + +def test_parse_orderby_expression_not_in_select_returns_none_marker(): + """`ORDER BY a + b` when `a + b` is not a select-list expression + cannot be mapped to a column index. The parser flags this so the + caller can fall back to N3 strict-equality for this variant.""" + from bird_interact_agents.eval.tolerant_grader import parse_orderby_keys + + keys = parse_orderby_keys("SELECT a, b FROM t ORDER BY a + b") + # The key carries no resolvable index → caller collapses N4 to N3. + assert any(k.column_index is None for k in keys) + + +def test_parse_orderby_nulls_first_last_does_not_error(): + """SQLite's NULLS FIRST/LAST syntax must parse without crashing — the + actual NULL ordering is a downstream concern; for bucketing we only + care about the column key.""" + from bird_interact_agents.eval.tolerant_grader import parse_orderby_keys + + keys = parse_orderby_keys( + "SELECT a, b FROM t ORDER BY a NULLS LAST, b NULLS FIRST" + ) + assert [k.column_index for k in keys] == [0, 1] diff --git a/tests/test_tolerant_grader_orchestration.py b/tests/test_tolerant_grader_orchestration.py new file mode 100644 index 00000000..d11eee84 --- /dev/null +++ b/tests/test_tolerant_grader_orchestration.py @@ -0,0 +1,938 @@ +"""DEV-1515: end-to-end cascade orchestration tests for tolerant_grader. + +These exercise ``grade_submission`` end-to-end against a fake executor +(no real SQLite) and verify the 8-row cascade (N1..N8), monotonicity, +LLM-judge gating + cache, and the missing-annotation graceful default. + +LLM-judge tests stay mechanical per the project convention: we never +assert on prompt content, only on cache key, timeout fall-through, and +SubmissionEvaluation fields the grader produces. +""" +from __future__ import annotations + +from pathlib import Path +from typing import List, Optional + + +# --------------------------------------------------------------------------- +# Helpers — fake executor + audited-gold rows + task annotation fixtures +# --------------------------------------------------------------------------- + + +def _audited_row( + *, instance_id: str, variant_id: str, primary: bool, + audited_sol_sql: List[str], db: str = "alien", +) -> dict: + return { + "instance_id": instance_id, + "selected_database": db, + "benchmark": "mini_interact", + "audit_status": "edited", + "original_sol_sql": ["SELECT original FROM t"], + "audited_sol_sql": audited_sol_sql, + "variant_id": variant_id, + "primary": primary, + "changes": [], + "reasoning_summary": "", + "skill_version": "audit-gold-sql/1.0", + "audited_at": "2026-05-30T00:00:00+00:00", + } + + +def _make_task_annotation( + *, + verdict: str = "sufficient", + evaluator_prompt: Optional[str] = None, + instance_id: str = "alien_1", +): + from bird_interact_agents.eval import ( + AuditedGoldRef, + GoldVariantRef, + MetadataSufficiency, + TaskAnnotation, + ) + from bird_interact_agents.eval.annotation_schema import Provenance + + return TaskAnnotation( + instance_id=instance_id, + selected_database="alien", + annotated_by="test", + annotated_at="2026-05-31", + amb_user_query="x", + metadata_sufficiency=MetadataSufficiency( + verdict=verdict, rationale="r", + ), + gold_variants=[ + GoldVariantRef( + variant_id="primary", + interpretation="x", + primary=True, + audited_gold_ref=AuditedGoldRef( + file="audited_gold/mini_interact_audited.jsonl", + instance_id=instance_id, + variant_id="primary", + ), + ), + ], + evaluator_prompt=evaluator_prompt, + provenance=Provenance( + task_jsonl_path="mini_interact.jsonl", + task_jsonl_instance_id=instance_id, + ), + ) + + +class FakeExecutor: + """Returns canned (rows, column_names) for each SQL string, in order. + + `responses` is a dict SQL→(rows, column_names). Any SQL not in the + dict raises — tests must enumerate every expected SQL. + """ + def __init__(self, responses): + self.responses = responses + self.calls: List[str] = [] + + def __call__(self, sql, *, db_path, conn): + self.calls.append(sql) + if sql not in self.responses: + raise AssertionError(f"unexpected SQL: {sql!r}") + return self.responses[sql] + + +# --------------------------------------------------------------------------- +# N1..N3 strict cascade +# --------------------------------------------------------------------------- + + +def test_n1_passes_when_predicted_matches_original_gold(): + """Predicted matches original gold but NOT audited primary. The + cascade is monotone (N1 pass ⇒ N2+ pass via enforce_monotone_cascade), + so per-task verdict is True from N1 onward. The aggregate delta + D2/D3 stays 0 for this row (audited didn't add a new pass).""" + from bird_interact_agents.eval.tolerant_grader import grade_submission + + submitted = "SELECT 1" + original_gold = "SELECT original FROM t" + audited = "SELECT audited FROM t" + executor = FakeExecutor({ + submitted: ([(1,)], ["a"]), + original_gold: ([(1,)], ["a"]), + audited: ([(99,)], ["a"]), # different — only original matches + }) + ann = _make_task_annotation() + gold_rows = [_audited_row( + instance_id="alien_1", variant_id="primary", primary=True, + audited_sol_sql=[audited], + )] + verdict = grade_submission( + task_annotation=ann, + audited_gold_rows=gold_rows, + original_sol_sql=[original_gold], + submitted_sql=submitted, + db_path=Path("/dev/null"), + conn=None, + executor=executor, + ) + assert verdict.n1_original_gold is True + # N2..N8 all True via monotone enforcement. + assert verdict.n2_audited_primary is True + # The variant_matches diagnostic block still surfaces the audited + # mismatch (it's per-variant Tier 2 info, not gated by the cascade). + assert verdict.variant_matches[0].informational is not None + assert verdict.variant_matches[0].informational.rowset_relation == "disjoint" + + +def test_n3_passes_for_non_primary_variant_match(): + from bird_interact_agents.eval.tolerant_grader import grade_submission + + submitted = "SELECT predicted" + original_gold = "SELECT original" + primary_sql = "SELECT primary_audit" + alt_sql = "SELECT alt_audit" + executor = FakeExecutor({ + submitted: ([(42,)], ["x"]), + original_gold: ([(1,)], ["x"]), + primary_sql: ([(2,)], ["x"]), + alt_sql: ([(42,)], ["x"]), + }) + ann = _make_task_annotation() + gold_rows = [ + _audited_row( + instance_id="alien_1", variant_id="primary", primary=True, + audited_sol_sql=[primary_sql], + ), + _audited_row( + instance_id="alien_1", variant_id="alt", primary=False, + audited_sol_sql=[alt_sql], + ), + ] + verdict = grade_submission( + task_annotation=ann, + audited_gold_rows=gold_rows, + original_sol_sql=[original_gold], + submitted_sql=submitted, + db_path=Path("/dev/null"), + conn=None, + executor=executor, + ) + assert verdict.n1_original_gold is False + assert verdict.n2_audited_primary is False + assert verdict.n3_any_audited_variant is True + assert verdict.matched_variant_id == "alt" + + +# --------------------------------------------------------------------------- +# N4 — tie-order (uses original_gold's ORDER BY per spec) +# --------------------------------------------------------------------------- + + +def test_n4_passes_when_only_tie_order_differs(): + """Predicted has ties at a='A' reordered relative to gold. With the + set-equality N3 comparator (back-compat with today's ex_base default), + set-eq passes for in-bucket reorders so N3 already True. N4 + (bucket-by-ORDER-BY) also passes — this is the spec-mandated cascade + row, even though in the set-eq world it coincides with N3 in the + no-duplicate case. + + The dedicated comparator-level tie-order test in + ``test_tolerant_grader_comparators.py::test_n4_reordered_within_bucket_passes`` + exercises the bucketing logic directly. + """ + from bird_interact_agents.eval.tolerant_grader import grade_submission + + submitted = "SELECT * FROM t" + original_gold = "SELECT a, b FROM t ORDER BY a" + audited = "SELECT a, b FROM t ORDER BY a" + executor = FakeExecutor({ + submitted: ([("A", 2), ("A", 1), ("B", 9)], ["a", "b"]), + original_gold: ([("A", 1), ("A", 2), ("B", 9)], ["a", "b"]), + audited: ([("A", 1), ("A", 2), ("B", 9)], ["a", "b"]), + }) + ann = _make_task_annotation() + gold_rows = [_audited_row( + instance_id="alien_1", variant_id="primary", primary=True, + audited_sol_sql=[audited], + )] + verdict = grade_submission( + task_annotation=ann, + audited_gold_rows=gold_rows, + original_sol_sql=[original_gold], + submitted_sql=submitted, + db_path=Path("/dev/null"), + conn=None, + executor=executor, + ) + # In the set-eq world N3 already passes for in-bucket reorders. + assert verdict.n3_any_audited_variant is True + assert verdict.n4_tie_order is True + + +def test_n4_uses_original_gold_orderby_not_variant_orderby(): + """N4 bucketing is sourced from the ORIGINAL gold's ORDER BY clause + (locked simplification), not each variant's. Construct a case where + the original gold has ORDER BY but the variant's audited SQL doesn't + — bucketing must STILL follow the original gold.""" + from bird_interact_agents.eval.tolerant_grader import grade_submission + + submitted = "S" + # Original gold has ORDER BY on column 0; audited variant doesn't. + original_gold = "SELECT a, b FROM t ORDER BY a" + audited = "SELECT a, b FROM t" + executor = FakeExecutor({ + # Predicted: ties on a='A' reordered; cross-bucket order preserved. + submitted: ([("A", 2), ("A", 1), ("B", 9)], ["a", "b"]), + original_gold: ([("A", 1), ("A", 2), ("B", 9)], ["a", "b"]), + audited: ([("A", 1), ("A", 2), ("B", 9)], ["a", "b"]), + }) + ann = _make_task_annotation() + gold_rows = [_audited_row( + instance_id="alien_1", variant_id="primary", primary=True, + audited_sol_sql=[audited], + )] + verdict = grade_submission( + task_annotation=ann, + audited_gold_rows=gold_rows, + original_sol_sql=[original_gold], + submitted_sql=submitted, + db_path=Path("/dev/null"), + conn=None, + executor=executor, + ) + # Using ORIGINAL gold's ORDER BY: bucket by column 0 → all rows + # share the bucket-equivalence at ('A', _) → set-equality passes. + assert verdict.n4_tie_order is True + + +def test_n4_collapses_to_n3_when_no_orderby(): + """No ORDER BY in original gold → N4 == N3.""" + from bird_interact_agents.eval.tolerant_grader import grade_submission + + submitted = "S" + original_gold = "SELECT a FROM t" # no ORDER BY + audited = "SELECT a FROM t" + executor = FakeExecutor({ + submitted: ([(2,), (1,)], ["a"]), + original_gold: ([(1,), (2,)], ["a"]), + audited: ([(1,), (2,)], ["a"]), + }) + ann = _make_task_annotation() + gold_rows = [_audited_row( + instance_id="alien_1", variant_id="primary", primary=True, + audited_sol_sql=[audited], + )] + verdict = grade_submission( + task_annotation=ann, + audited_gold_rows=gold_rows, + original_sol_sql=[original_gold], + submitted_sql=submitted, + db_path=Path("/dev/null"), + conn=None, + executor=executor, + ) + # No ORDER BY → ordering doesn't matter; sets are equal so N3 passes. + assert verdict.n3_any_audited_variant is True + assert verdict.n4_tie_order is True + + +# --------------------------------------------------------------------------- +# N5 — LLM judge (only fires for verdict=='insufficient' AND no N4 pass) +# --------------------------------------------------------------------------- + + +class FakeLLMJudge: + def __init__(self, accept: Optional[bool]): + self.accept = accept + self.calls = 0 + + def judge(self, **kwargs): # noqa: ANN003 + self.calls += 1 + return self.accept + + +def test_n5_does_not_fire_for_sufficient_verdict(): + from bird_interact_agents.eval.tolerant_grader import grade_submission + + submitted = "S" + original_gold = "G" + audited = "A" + executor = FakeExecutor({ + submitted: ([(1,)], ["a"]), + original_gold: ([(2,)], ["a"]), + audited: ([(2,)], ["a"]), + }) + ann = _make_task_annotation( + verdict="sufficient", evaluator_prompt="rules", + ) + judge = FakeLLMJudge(accept=True) + gold_rows = [_audited_row( + instance_id="alien_1", variant_id="primary", primary=True, + audited_sol_sql=[audited], + )] + verdict = grade_submission( + task_annotation=ann, + audited_gold_rows=gold_rows, + original_sol_sql=[original_gold], + submitted_sql=submitted, + db_path=Path("/dev/null"), + conn=None, + executor=executor, + llm_judge=judge, + ) + assert verdict.n5_llm_judge is False + assert judge.calls == 0 # the gate held — no call paid + + +def test_n5_fires_only_for_insufficient_verdict_when_no_variant_matched(): + from bird_interact_agents.eval.tolerant_grader import grade_submission + + submitted = "S" + original_gold = "G" + audited = "A" + executor = FakeExecutor({ + submitted: ([(1,)], ["a"]), + original_gold: ([(2,)], ["a"]), + audited: ([(2,)], ["a"]), + }) + ann = _make_task_annotation( + verdict="insufficient", evaluator_prompt="rules", + ) + judge = FakeLLMJudge(accept=True) + gold_rows = [_audited_row( + instance_id="alien_1", variant_id="primary", primary=True, + audited_sol_sql=[audited], + )] + verdict = grade_submission( + task_annotation=ann, + audited_gold_rows=gold_rows, + original_sol_sql=[original_gold], + submitted_sql=submitted, + db_path=Path("/dev/null"), + conn=None, + executor=executor, + llm_judge=judge, + ) + assert verdict.n5_llm_judge is True + assert judge.calls == 1 + + +def test_n5_does_not_invoke_judge_when_n4_already_passes_via_tie_order(): + """Gate semantics: N5 fires ONLY when verdict=='insufficient' AND + NO variant matched at N4. If N4 passes via tie-order on a sufficient + task, the judge must not be called even with a non-None judge.""" + from bird_interact_agents.eval.tolerant_grader import grade_submission + + submitted = "S" + original_gold = "SELECT a FROM t ORDER BY a" + audited = "SELECT a FROM t ORDER BY a" + executor = FakeExecutor({ + # N3 strict fails (different in-bucket order); N4 tie-order passes. + submitted: ([("A", 2), ("A", 1)], ["a", "b"]), + original_gold: ([("A", 1), ("A", 2)], ["a", "b"]), + audited: ([("A", 1), ("A", 2)], ["a", "b"]), + }) + ann = _make_task_annotation( + verdict="insufficient", evaluator_prompt="rules", + ) + judge = FakeLLMJudge(accept=False) + gold_rows = [_audited_row( + instance_id="alien_1", variant_id="primary", primary=True, + audited_sol_sql=[audited], + )] + verdict = grade_submission( + task_annotation=ann, + audited_gold_rows=gold_rows, + original_sol_sql=[original_gold], + submitted_sql=submitted, + db_path=Path("/dev/null"), + conn=None, + executor=executor, + llm_judge=judge, + ) + assert verdict.n4_tie_order is True + assert verdict.n5_llm_judge is True + assert judge.calls == 0, ( + "N5 must not invoke the judge when N4 already passes" + ) + + +def test_n5_monotone_never_takes_away_n4_pass(): + """If N4 already passes, N5 must remain True even if the LLM judge + would have rejected — the cascade is monotone by construction.""" + from bird_interact_agents.eval.tolerant_grader import grade_submission + + submitted = "S" + original_gold = "G" + audited = "A" + executor = FakeExecutor({ + submitted: ([(1,)], ["a"]), + original_gold: ([(1,)], ["a"]), + audited: ([(1,)], ["a"]), + }) + ann = _make_task_annotation( + verdict="insufficient", evaluator_prompt="rules", + ) + judge = FakeLLMJudge(accept=False) # would say no, but cascade monotone + gold_rows = [_audited_row( + instance_id="alien_1", variant_id="primary", primary=True, + audited_sol_sql=[audited], + )] + verdict = grade_submission( + task_annotation=ann, + audited_gold_rows=gold_rows, + original_sol_sql=[original_gold], + submitted_sql=submitted, + db_path=Path("/dev/null"), + conn=None, + executor=executor, + llm_judge=judge, + ) + # Deterministic tiers all pass → N5 must stay True regardless of judge. + assert verdict.n4_tie_order is True + assert verdict.n5_llm_judge is True + + +def test_n5_timeout_falls_through_to_none(): + """LLMJudge returning None (timeout/error) leaves N5 as the previous + cascade value (no novel reading accepted).""" + from bird_interact_agents.eval.tolerant_grader import grade_submission + + submitted = "S" + original_gold = "G" + audited = "A" + executor = FakeExecutor({ + submitted: ([(1,)], ["a"]), + original_gold: ([(2,)], ["a"]), + audited: ([(2,)], ["a"]), + }) + ann = _make_task_annotation( + verdict="insufficient", evaluator_prompt="rules", + ) + judge = FakeLLMJudge(accept=None) # timeout + gold_rows = [_audited_row( + instance_id="alien_1", variant_id="primary", primary=True, + audited_sol_sql=[audited], + )] + verdict = grade_submission( + task_annotation=ann, + audited_gold_rows=gold_rows, + original_sol_sql=[original_gold], + submitted_sql=submitted, + db_path=Path("/dev/null"), + conn=None, + executor=executor, + llm_judge=judge, + ) + assert verdict.n4_tie_order is False + assert verdict.n5_llm_judge is False + assert verdict.novel_reading_judgment is None + + +# --------------------------------------------------------------------------- +# N6/N7/N8 — cell-level relaxations stacked on N5 +# --------------------------------------------------------------------------- + + +def test_n6_numeric_epsilon_lifts_failing_n5_to_pass(): + from bird_interact_agents.eval.tolerant_grader import grade_submission + + submitted = "S" + original_gold = "G" + audited = "A" + executor = FakeExecutor({ + submitted: ([(1.0000001,)], ["v"]), + original_gold: ([(1.0,)], ["v"]), + audited: ([(1.0,)], ["v"]), + }) + ann = _make_task_annotation() + gold_rows = [_audited_row( + instance_id="alien_1", variant_id="primary", primary=True, + audited_sol_sql=[audited], + )] + verdict = grade_submission( + task_annotation=ann, + audited_gold_rows=gold_rows, + original_sol_sql=[original_gold], + submitted_sql=submitted, + db_path=Path("/dev/null"), + conn=None, + executor=executor, + epsilon=1e-6, + ) + assert verdict.n5_llm_judge is False # sufficient + no llm_judge + assert verdict.n6_numeric_epsilon is True + assert verdict.n7_trailing_whitespace is True # monotone + assert verdict.n8_column_order is True # monotone + + +def test_n7_trailing_whitespace_lifts_failing_n6_to_pass(): + from bird_interact_agents.eval.tolerant_grader import grade_submission + + submitted = "S" + original_gold = "G" + audited = "A" + executor = FakeExecutor({ + submitted: ([("High Income ",)], ["bracket"]), + original_gold: ([("High Income",)], ["bracket"]), + audited: ([("High Income",)], ["bracket"]), + }) + ann = _make_task_annotation() + gold_rows = [_audited_row( + instance_id="alien_1", variant_id="primary", primary=True, + audited_sol_sql=[audited], + )] + verdict = grade_submission( + task_annotation=ann, + audited_gold_rows=gold_rows, + original_sol_sql=[original_gold], + submitted_sql=submitted, + db_path=Path("/dev/null"), + conn=None, + executor=executor, + ) + assert verdict.n6_numeric_epsilon is False + assert verdict.n7_trailing_whitespace is True + + +def test_n8_column_order_uses_column_metadata(): + """N8 needs cursor.description-style column names; ensure the + executor's returned `column_names` parameter is the path used.""" + from bird_interact_agents.eval.tolerant_grader import grade_submission + + submitted = "S" + original_gold = "G" + audited = "A" + executor = FakeExecutor({ + submitted: ([("foo", 1)], ["name", "id"]), + original_gold: ([(1, "foo")], ["id", "name"]), + audited: ([(1, "foo")], ["id", "name"]), + }) + ann = _make_task_annotation() + gold_rows = [_audited_row( + instance_id="alien_1", variant_id="primary", primary=True, + audited_sol_sql=[audited], + )] + verdict = grade_submission( + task_annotation=ann, + audited_gold_rows=gold_rows, + original_sol_sql=[original_gold], + submitted_sql=submitted, + db_path=Path("/dev/null"), + conn=None, + executor=executor, + ) + # Without name-alignment, tuples don't match (positional). + assert verdict.n7_trailing_whitespace is False + # With name-alignment, columns align → True. + assert verdict.n8_column_order is True + + +# --------------------------------------------------------------------------- +# Cascade monotonicity — synthetic matrix +# --------------------------------------------------------------------------- + + +def test_cascade_is_monotone_for_every_possible_pass_pattern(): + """Whatever the comparators return, the persisted CascadeVerdict must + satisfy N1 ≤ N2 ≤ N3 ≤ N4 ≤ N5 ≤ N6 ≤ N7 ≤ N8 (each step admits a + superset of passes from the previous). + + Drive the property through a synthetic matrix: any task that passes + at level N must also report `pass` at level N+1. + """ + from bird_interact_agents.eval.tolerant_grader import ( + enforce_monotone_cascade, + ) + + fields = [ + "n1_original_gold", "n2_audited_primary", "n3_any_audited_variant", + "n4_tie_order", "n5_llm_judge", + "n6_numeric_epsilon", "n7_trailing_whitespace", "n8_column_order", + ] + # Every possible 8-bit raw mask. + for mask in range(2 ** 8): + raw = {fields[i]: bool((mask >> i) & 1) for i in range(8)} + enforced = enforce_monotone_cascade(raw) + prev = False + for f in fields: + cur = enforced[f] + # Monotone: once True, every subsequent level stays True. + if prev: + assert cur is True, ( + f"monotone broken at {f}: prev True, got {cur} " + f"with raw={raw}" + ) + prev = cur + + +# --------------------------------------------------------------------------- +# Missing-annotation graceful default — no write, cascade collapses to N1 +# --------------------------------------------------------------------------- + + +def test_missing_annotation_collapses_cascade_to_n1(monkeypatch, tmp_path): + from bird_interact_agents.eval.tolerant_grader import grade_submission + from bird_interact_agents.eval.implicit_annotation import ( + implicit_task_annotation, + ) + + submitted = "S" + original_gold = "G" + executor = FakeExecutor({ + submitted: ([(1,)], ["a"]), + original_gold: ([(1,)], ["a"]), + }) + ann = implicit_task_annotation( + instance_id="alien_99", + selected_database="alien", + benchmark="mini-interact", + amb_user_query="x", + ) + # No audited-gold rows on disk for this instance — the implicit + # contract is "single variant == original gold". + verdict = grade_submission( + task_annotation=ann, + audited_gold_rows=[], + original_sol_sql=[original_gold], + submitted_sql=submitted, + db_path=Path("/dev/null"), + conn=None, + executor=executor, + ) + assert verdict.n1_original_gold is True + # Cascade collapses: N2 == N3 == N1 (no separate audited primary). + assert verdict.n2_audited_primary is True + assert verdict.n3_any_audited_variant is True + + +# --------------------------------------------------------------------------- +# LLM-judge cache — keyed by content hash, not run-id +# --------------------------------------------------------------------------- + + +def test_llm_judge_cache_hit_avoids_second_call(tmp_path): + from bird_interact_agents.eval.tolerant_grader import CachedLLMJudge + + class CountingInner: + def __init__(self): + self.calls = 0 + def judge(self, **kwargs): + self.calls += 1 + return True + + inner = CountingInner() + cache_path = tmp_path / "llm_judge_cache.json" + cached = CachedLLMJudge(inner=inner, cache_path=cache_path) + + payload = dict( + evaluator_prompt="rules", + gold_variants_summary=[{"variant_id": "primary"}], + metadata_anchors=[], + submitted_sql="SELECT 1", + predicted_rows_head=[(1,)], + annotation_content_hash="abc", + gold_variants_content_hash="def", + ) + + out1 = cached.judge(**payload) + out2 = cached.judge(**payload) + assert out1 is True and out2 is True + assert inner.calls == 1 # second call short-circuits via cache + + +def test_llm_judge_cache_key_changes_when_annotation_hash_changes(tmp_path): + from bird_interact_agents.eval.tolerant_grader import CachedLLMJudge + + class CountingInner: + def __init__(self): + self.calls = 0 + def judge(self, **kwargs): + self.calls += 1 + return True + + inner = CountingInner() + cache_path = tmp_path / "llm_judge_cache.json" + cached = CachedLLMJudge(inner=inner, cache_path=cache_path) + + base = dict( + evaluator_prompt="rules", + gold_variants_summary=[], + metadata_anchors=[], + submitted_sql="SELECT 1", + predicted_rows_head=[(1,)], + gold_variants_content_hash="def", + ) + cached.judge(annotation_content_hash="abc", **base) + cached.judge(annotation_content_hash="abc2", **base) # changed annotation + assert inner.calls == 2 # different keys; no cache hit + + +def test_llm_judge_cache_key_changes_when_gold_variants_hash_changes(tmp_path): + """Different gold-variants content ⇒ different cache key, even if + everything else is identical. New variants must NOT acquit via stale + cache.""" + from bird_interact_agents.eval.tolerant_grader import CachedLLMJudge + + class CountingInner: + calls = 0 + def judge(self, **kwargs): + self.calls += 1 + return True + + inner = CountingInner() + cached = CachedLLMJudge( + inner=inner, cache_path=tmp_path / "c.json", + ) + base = dict( + evaluator_prompt="rules", gold_variants_summary=[], + metadata_anchors=[], submitted_sql="X", + predicted_rows_head=[], annotation_content_hash="a", + ) + cached.judge(gold_variants_content_hash="g1", **base) + cached.judge(gold_variants_content_hash="g2", **base) + assert inner.calls == 2 + + +def test_llm_judge_cache_key_changes_when_model_changes(tmp_path): + """Same content, different model ⇒ different cache key. Caches must + survive a model bump (Opus 4.7 → 4.8) without giving stale verdicts.""" + from bird_interact_agents.eval.tolerant_grader import CachedLLMJudge + + class CountingInner: + def __init__(self, model_name): + self.calls = 0 + self.model_name = model_name + def judge(self, **kwargs): + self.calls += 1 + return True + + cache_path = tmp_path / "c.json" + inner1 = CountingInner(model_name="claude-opus-4-7") + CachedLLMJudge(inner=inner1, cache_path=cache_path).judge( + evaluator_prompt="r", gold_variants_summary=[], metadata_anchors=[], + submitted_sql="X", predicted_rows_head=[], + annotation_content_hash="a", gold_variants_content_hash="g", + ) + inner2 = CountingInner(model_name="claude-opus-4-8") # different + CachedLLMJudge(inner=inner2, cache_path=cache_path).judge( + evaluator_prompt="r", gold_variants_summary=[], metadata_anchors=[], + submitted_sql="X", predicted_rows_head=[], + annotation_content_hash="a", gold_variants_content_hash="g", + ) + # The second call must hit the inner (different model in cache key). + assert inner2.calls == 1 + + +def test_llm_judge_cache_key_does_not_include_run_id(tmp_path): + """Offline re-grade across runs MUST reuse cached verdicts when the + content (annotation, gold, sql, model) is unchanged. Run-id is + therefore NOT part of the key.""" + from bird_interact_agents.eval.tolerant_grader import CachedLLMJudge + + class CountingInner: + def __init__(self): + self.calls = 0 + def judge(self, **kwargs): + self.calls += 1 + return True + + inner = CountingInner() + cache_path = tmp_path / "c.json" + # Two calls in different "run contexts" — content identical. + for _ in range(2): + CachedLLMJudge(inner=inner, cache_path=cache_path).judge( + evaluator_prompt="r", gold_variants_summary=[], + metadata_anchors=[], submitted_sql="X", predicted_rows_head=[], + annotation_content_hash="a", gold_variants_content_hash="g", + ) + assert inner.calls == 1 # second call is a cache hit + + +def test_llm_judge_cache_persists_across_process(tmp_path): + """Caches live on disk so an offline re-grade can reuse cloud-side + decisions.""" + from bird_interact_agents.eval.tolerant_grader import CachedLLMJudge + + class CountingInner: + def __init__(self, val=True): + self.calls = 0 + self.val = val + def judge(self, **kwargs): + self.calls += 1 + return self.val + + cache_path = tmp_path / "llm_judge_cache.json" + # First process — populate cache. + inner1 = CountingInner(val=True) + CachedLLMJudge(inner=inner1, cache_path=cache_path).judge( + evaluator_prompt="rules", + gold_variants_summary=[], + metadata_anchors=[], + submitted_sql="X", + predicted_rows_head=[], + annotation_content_hash="a", + gold_variants_content_hash="b", + ) + # Second process — cache should be re-used; inner must not be hit. + inner2 = CountingInner(val=False) # different val to prove cache used + out = CachedLLMJudge(inner=inner2, cache_path=cache_path).judge( + evaluator_prompt="rules", + gold_variants_summary=[], + metadata_anchors=[], + submitted_sql="X", + predicted_rows_head=[], + annotation_content_hash="a", + gold_variants_content_hash="b", + ) + assert out is True # the cached `True`, not the new inner's `False` + assert inner2.calls == 0 + + +# --------------------------------------------------------------------------- +# Tier 2 informational — rowset relation, column diff, first divergent row +# --------------------------------------------------------------------------- + + +def test_tier2_rowset_relation_equal(): + from bird_interact_agents.eval.tolerant_grader import classify_rowset_relation + + assert classify_rowset_relation( + pred=[(1,), (2,)], gold=[(2,), (1,)], + ) == "equal_rowset" + + +def test_tier2_rowset_relation_strict_subset(): + from bird_interact_agents.eval.tolerant_grader import classify_rowset_relation + + assert classify_rowset_relation( + pred=[(1,)], gold=[(1,), (2,)], + ) == "strict_subset_of" + + +def test_tier2_rowset_relation_strict_superset(): + from bird_interact_agents.eval.tolerant_grader import classify_rowset_relation + + assert classify_rowset_relation( + pred=[(1,), (2,)], gold=[(1,)], + ) == "strict_superset_of" + + +def test_tier2_rowset_relation_overlapping(): + from bird_interact_agents.eval.tolerant_grader import classify_rowset_relation + + assert classify_rowset_relation( + pred=[(1,), (2,)], gold=[(2,), (3,)], + ) == "overlapping" + + +def test_tier2_rowset_relation_disjoint(): + from bird_interact_agents.eval.tolerant_grader import classify_rowset_relation + + assert classify_rowset_relation( + pred=[(1,)], gold=[(2,)], + ) == "disjoint" + + +def test_tier2_populated_on_grader_output(): + """Each variant in CascadeVerdict.variant_matches carries an + informational sub-block with exact field values pinned.""" + from bird_interact_agents.eval.tolerant_grader import grade_submission + + submitted = "S" + original_gold = "G" + audited = "A" + # Predicted = (1, "x"); gold = (2, "x"). Same column count + names. + # First divergent row index = 0; cell diff at column 0. + executor = FakeExecutor({ + submitted: ([(1, "x")], ["id", "label"]), + original_gold: ([(2, "x")], ["id", "label"]), + audited: ([(2, "x")], ["id", "label"]), + }) + ann = _make_task_annotation() + gold_rows = [_audited_row( + instance_id="alien_1", variant_id="primary", primary=True, + audited_sol_sql=[audited], + )] + verdict = grade_submission( + task_annotation=ann, + audited_gold_rows=gold_rows, + original_sol_sql=[original_gold], + submitted_sql=submitted, + db_path=Path("/dev/null"), + conn=None, + executor=executor, + ) + assert verdict.variant_matches + info = verdict.variant_matches[0].informational + assert info is not None + # Same column count + names + order ⇒ all three structural flags True. + assert info.column_count_match is True + assert info.column_name_match_case_insensitive is True + assert info.column_order_match is True + # Disjoint rowsets (pred=[(1,"x")], gold=[(2,"x")]). + assert info.rowset_relation == "disjoint" + # First (and only) divergent row is at index 0. + assert info.first_divergent_row_index == 0 + # The cell-diff string must reference both values for human review. + assert info.first_divergent_cell_diff is not None + assert "1" in info.first_divergent_cell_diff + assert "2" in info.first_divergent_cell_diff From a91f86169eb4d285e99866cec518851727c769ab Mon Sep 17 00:00:00 2001 From: Egor Kraev Date: Mon, 1 Jun 2026 11:51:28 +0200 Subject: [PATCH 07/29] DEV-1515 session-3 + session-4: annotation schema overhaul, multi-variant audits, audit-correctness pass session-3: - FailurePrimary enum: drop grader_stability; add no_fail, novel_reading_accepted, numerical_precision, row_order, trailing_whitespace, column_order, case_sensitivity. - _auto_failure_class shared classifier mapping cascade tier -> FailurePrimary (no_fail / row_order / novel_reading_accepted / numerical_precision / trailing_whitespace / column_order / other). - InternalInconsistency record on TaskAnnotation for multi-source conflicts (KB vs sql_snippet etc.); MetadataSufficiency docstring sharpened to encode the user_sim disclosure rule. - Multi-variant gold pattern: variant_id + primary on every audited_gold row; audit-gold-sql skill gains source_conflict clause_kind + sidecar example. - annotate-task-submission skill (NEW): canonical decision flow for mini-interact (a-interact) vs livesqlbench (one-shot), audit-correctness gate, failure-class table, cascade-tier -> failure-class mapping. - dev1515 conversion / re-classification / summary scripts under scripts/. session-4: - Multi-variant test refactor in tests/test_livesqlbench_audited_gold.py: _load_audit_rows returns PRIMARY rows; test_no_duplicate_instance_ids -> test_no_duplicate_instance_id_variant_pairs (dedup on the pair, enforce exactly one primary per instance). Households audit-correctness pass (audited_gold/mini_interact_audited.jsonl): - households_5: backfilled justified_by on the R$ 2,640 cutoff (anchored in snippet); task verdict insufficient -> ambiguous with rewritten rationale. - households_10: pruned 10 drifted IN-list literals to snippet-exact; agent flips agent_miss -> no_fail (equal_rowset, 153 housenums). - households_12: audit_status edited -> unrecoverable; one changes entry documents the gap (0..11 ordinal mapping is policy under any monotonic encoding). Museum audit work (audited_gold/livesqlbench_audited.jsonl): - museum_10: re-authored from buggy ERF copy-paste to KB-anchored 7-column DSD+ERPS+CASE per KBs [0,1,4,8,38,52]; agent submission flips to no_fail with gold_audit_quality secondary. - museum_2/4/9: added non-primary variant audit rows (four_dimension_erf, uniform_budget_ratio, conditionassessments_join_path) capturing the KB-defensible alternate readings; submissions flip other -> no_fail with metadata_ambiguity secondary. - museum_5: classified agent_miss (KB 7 latest-reading was clear). - museum_7: filled PENDING_HUMAN_REVIEW rationale + variant interpretation. Full non-integration suite: 1800 passed, 95 skipped, 50 deselected. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../skills/annotate-task-submission/SKILL.md | 359 ++++++++++++++++ .claude/skills/audit-gold-sql/SKILL.md | 27 ++ scripts/dev1515_cascade_summary.py | 188 +++++++++ .../dev1515_convert_livesqlbench_museum.py | 364 ++++++++++++++++ scripts/dev1515_convert_runs.py | 326 +++++++++++++++ scripts/dev1515_households_14_multivariant.py | 388 ++++++++++++++++++ scripts/dev1515_reclassify_sufficiency.py | 131 ++++++ scripts/dev1515_remap_failure_classes.py | 137 +++++++ src/bird_interact_agents/eval/annotate.py | 27 +- .../eval/annotation_schema.py | 90 +++- .../eval/grade_in_place.py | 45 +- src/bird_interact_agents/eval/regrade.py | 8 +- tests/test_livesqlbench_audited_gold.py | 44 +- tests/test_regrade_cli.py | 261 ++++++++++++ 14 files changed, 2369 insertions(+), 26 deletions(-) create mode 100644 .claude/skills/annotate-task-submission/SKILL.md create mode 100644 scripts/dev1515_cascade_summary.py create mode 100644 scripts/dev1515_convert_livesqlbench_museum.py create mode 100644 scripts/dev1515_convert_runs.py create mode 100644 scripts/dev1515_households_14_multivariant.py create mode 100644 scripts/dev1515_reclassify_sufficiency.py create mode 100644 scripts/dev1515_remap_failure_classes.py create mode 100644 tests/test_regrade_cli.py diff --git a/.claude/skills/annotate-task-submission/SKILL.md b/.claude/skills/annotate-task-submission/SKILL.md new file mode 100644 index 00000000..f9d90376 --- /dev/null +++ b/.claude/skills/annotate-task-submission/SKILL.md @@ -0,0 +1,359 @@ +--- +name: annotate-task-submission +description: Fill the human-judgment fields on a bird-interact `TaskAnnotation` + `SubmissionAnnotation` (DEV-1515 schema). Use when reviewing a benchmark task's metadata sufficiency, classifying a submission's failure mode, or filling skeletons produced by `scripts/dev1515_convert_runs.py` / `bird_interact_agents.eval.annotate`. Encodes the sharpened `metadata_sufficiency.verdict` rule that accounts for user-sim disclosure of the masked sql_snippet, AND the rule that buggy audits get fixed in place — annotations never ratify a broken audited gold. +--- + +# Annotate a bird-interact task + submission + +This skill is the per-task annotation recipe for DEV-1515. It picks the +right `metadata_sufficiency.verdict` and `failure_classification` for a +given (task, submission) pair, given the published metadata + the masked +sql_snippet + the run-side trajectory. + +The Pydantic schemas live in +`src/bird_interact_agents/eval/annotation_schema.py`; they validate with +`extra="forbid"` — never invent fields. + +## NON-NEGOTIABLE: fix buggy audits — never annotate around them + +Before classifying ANY task where `audited_sol_sql != original_sol_sql`, +verify the audited gold itself is correct against the published +metadata + the masked sql_snippet + the live DB rowset. + +* If the audit is correct → write the annotation describing what was + wrong with the ORIGINAL gold and (with citations) why the audited + version is canonical. Be generous with rationale text. +* If the audit is wrong → FIX it. Rewrite + `audited_sol_sql` / `changes` / `reasoning_summary` / `audit_status` + in `audited_gold/_audited.jsonl` directly. THEN write the + annotation as above. +* Never carry an `ambiguous` / `insufficient` verdict that silently + ratifies a buggy audit — that conflates "gold is underspecified" with + "the audit didn't finish its job". + +Annotations are the read-only documentation layer. The audited gold is +the eval pipeline's authoritative reference. Bugs land in the gold, +not in the annotation's rationale. + +## When sources contradict: multi-variant golds + internal_inconsistency + +A separate case from "audit is wrong" is "audit had to pick a side +because the task's authoritative sources disagree among themselves". +Examples: + +* KB description says `LCS > 3`, but the masked sql_snippet for the + same defined concept says `lcs > 2`. +* KB.description and KB.definition disagree. +* The user_query phrasing implies one aggregation; the masked snippet + specifies a different one. + +The conditions are **internally inconsistent** — an agent cannot +satisfy all sources simultaneously. Don't bury this in the audit's +rationale paragraph. Instead: + +1. **Emit BOTH readings as gold variants in the audit JSONL.** Two + rows sharing `instance_id`, distinct `variant_id`s, exactly one + with `primary=true`. The KB-anchored reading typically takes + primary; the snippet-anchored reading is the alternate. Each row's + `reasoning_summary` cites its source explicitly and names the + conflict with the other. +2. **Set `TaskAnnotation.internal_inconsistency`** to an + `InternalInconsistency` record with: + * `sources_in_conflict`: list of citations, one per source, + quoting the disagreeing value (e.g. + `"KB#29.description: 'LCS > 3'"`, + `"critical_ambiguity for 'good quality of life'.sql_snippet: + 'lcs > 2'"`). + * `description`: 1–3 sentences explaining what each source says and + why an agent can't satisfy both at once. + * `audit_resolution`: typically `"multi_variant"` (see step 1). + Use `"picked_one_variant"` when emitting two variants would be + redundant for grading (rare), or `"unresolved"` when the audit + declared the task unanswerable. +3. **Set `verdict = "ambiguous"`** — NOT `sufficient` (sources DON'T + converge), NOT `insufficient` (each source independently pins a + reading). The published metadata licenses multiple readings + precisely because the sources contradict. +4. **`gold_variants`** carries one entry per audit row, each with an + `interpretation` field naming which source it's faithful to. The + `notes` field on each variant can echo the conflict for redundancy. +5. **Submission cascade behaviour falls out**: existing N3 ("any + audited variant matches") already iterates variants, so an agent + that picked EITHER reading passes. The remaining failure-class + work happens on the agent's actual error, not on the source + conflict. + +The `audit-gold-sql` skill carries the audit-side rule (two rows when +sources disagree, `clause_kind="source_conflict"` on the diff entry, +audit_status stays `edited`). This skill carries the annotation-side +rule (the `internal_inconsistency` block + multi-variant +`gold_variants`). + +## Inputs (per task) + +Read-only — **mini-interact**: + +- `mini-interact/mini_interact.jsonl` — task row. Fields: `amb_user_query` + (the user-facing question), `external_knowledge`, + `user_query_ambiguity.critical_ambiguity` (the masked sql_snippets + + ambiguity-type tags), `knowledge_ambiguity` (KB-id-tagged snippets), + `sol_sql` (original gold), `selected_database`. +- `mini-interact//_kb.jsonl` — KB entries referenced by + `external_knowledge`. +- `mini-interact//_column_meaning_base.json` — column + descriptions + sampled-value summaries. +- `mini-interact//.sqlite` — live DB, for sanity-execute checks. + +Read-only — **livesqlbench**: + +- `livesqlbench-base-lite-sqlite/livesqlbench_data_sqlite.jsonl` — task + row. Fields: `query` (natural-language only; NO masked snippet), + `selected_database`, `category`, `difficulty_tier`, `high_level`, + `conditions`. NO `sol_sql` or `external_knowledge` here. +- `livesqlbench-base-lite-sqlite/livesqlbench_sqlite_gt_kg_testcases_0528.jsonl` + — gold sidecar. Fields per row: `instance_id`, `sol_sql`, + `external_knowledge`, `test_cases`. Join by `instance_id`. +- `livesqlbench-base-lite-sqlite//_kb.jsonl` +- `livesqlbench-base-lite-sqlite//_column_meaning_base.json` +- `livesqlbench-base-lite-sqlite//_schema.txt` — schema dump; + the agent sees this at task time. +- `livesqlbench-base-lite-sqlite//.sqlite` + +Per-submission (both benchmarks): + +- `results/cloud//rows//attempt-1.json` — `submitted_sql`, + `trajectory`, `usage`, `phase1_observation_*`. (livesqlbench + trajectories are short — one-shot.) + +Read+write (both benchmarks): + +- `audited_gold/_audited.jsonl` — audit sidecar. + **Read first to assess correctness; write only when fixing a buggy + audit.** Filename pattern: `mini_interact_audited.jsonl` / + `livesqlbench_audited.jsonl`. +- `annotations///.task.json` — task annotation. + Benchmark slug: `mini-interact` / `livesqlbench`. +- `annotations///.submission..json` — + per-(instance, run) submission annotation. + +## What "masked sql_snippet" means + +Tasks in BIRD-Interact-style benchmarks ship `amb_user_query` containing +both natural-language framing AND a SQL skeleton with identifying +terms redacted. That skeleton — the **masked sql_snippet** — encodes +predicates / formulas / literal lists from the gold SQL, minus the +specific terms the masking process removed. In a-interact mode the +user_sim has access to the un-masked snippet and is expected to +disclose specifics on a well-phrased ask. + +**This is the key rule for `metadata_sufficiency.verdict`.** The verdict +answers: "given perfect user_sim cooperation, can the agent uniquely +derive the gold's reading?" + +## `metadata_sufficiency.verdict` decision flow + +The flow forks by benchmark mode. `mini-interact` carries masked +sql_snippets in `user_query_ambiguity.critical_ambiguity[].sql_snippet` +that a cooperative user_sim discloses in a-interact mode. +`livesqlbench` is one-shot only — no user_sim, no masked snippets — +so the agent's only sources are the published metadata. + +### mini-interact (a-interact mode) + +``` +1. Audit-correctness gate — is the audited gold itself correct? + NO → FIX the audit JSONL row first; come back here. + YES → proceed. + +2. Does the published metadata alone (KB + column meanings + sampled values) + pin a unique correct answer, with no user_sim help needed? + YES → verdict = "sufficient" + NO → go to step 3. + +3. Does the masked sql_snippet (from + `user_query_ambiguity.critical_ambiguity[].sql_snippet` / + `knowledge_ambiguity[].sql_snippet`), when fully disclosed by a + maximally cooperative user_sim, uniquely pin the answer? + YES → verdict = "ambiguous" + NO → verdict = "insufficient" +``` + +Corollaries: + +- **`sufficient` ≠ "agent passed"**. It only says the metadata anchored + the answer. An agent can still miss a `sufficient` task (then the + submission-side `primary` is `agent_miss`). +- **`ambiguous` is the "would-have-worked-with-a-good-sim" bucket.** If + the actual run-side sim withheld snippet content the agent needed, + the submission-side `primary` should be `user_sim_under_disclosure`, + not `metadata_ambiguity`. If the agent had `n_asks=0`, it's still + `agent_miss` — the agent didn't even try. +- **`insufficient` is rare.** Reserved for tasks where even the masked + snippet encodes arbitrary policy (coefficients, thresholds) with no + semantic anchor in any source. Audit-status `unrecoverable` is a + strong signal. Common false positive: the snippet's literal looks + arbitrary but is actually a translation of a KB-anchored band into + the data's native units (verify before claiming insufficient). + +### livesqlbench (one-shot mode) + +``` +1. Audit-correctness gate — is the audited gold itself correct? + NO → FIX the audit JSONL row first; come back here. + YES → proceed. + +2. Does the published metadata alone (KB + column meanings + sampled + values + schema.txt + external_knowledge anchors) pin a unique + correct answer? + YES → verdict = "sufficient" + NO → go to step 3. + +3. Does the KB / external_knowledge license a defensible non-gold + reading? + YES → verdict = "ambiguous" + NO → verdict = "insufficient" +``` + +Corollaries for livesqlbench: + +- There is NO masked sql_snippet and NO user_sim. The agent's only + resort for resolving ambiguity is published metadata. So `ambiguous` + here means "the agent has to pick and they may pick a defensible + non-gold reading"; there is no `user_sim_under_disclosure` failure + class for these. +- `failure_classification.primary = user_sim_under_disclosure` is + **never** used for livesqlbench. `n_asks` is always 0. +- `evaluator_prompt` is the only safety net for `insufficient` + livesqlbench tasks — populate it generously. + +## `failure_classification.primary` enum + when to pick each + +Available values in the schema (auto-classified from the cascade for the +mechanical buckets; the others require human review): + +| Value | When to use | +| -- | -- | +| `no_fail` | Cascade N3 strict-pass — agent matched an audited variant. Auto-assigned. | +| `agent_miss` | Metadata was `sufficient`; agent didn't derive it. Or metadata was `ambiguous` and the sim disclosed correctly, but the agent still picked wrong. | +| `metadata_ambiguity` | Metadata was `ambiguous` or `insufficient`; agent picked a defensible non-gold reading. Use when no user_sim was available (one-shot) or sim disclosed honestly and there's still genuine ambiguity. | +| `gold_audit_quality` | Used as `secondary` on a `no_fail` to record "agent passed but the original gold was buggy and the audit fixed it". Should be rare as `primary`: only when audit itself is incomplete after you've already considered fixing it (see audit-correctness gate above). | +| `user_sim_under_disclosure` | a-interact run; sim withheld snippet content the agent needed to pin the answer. | +| `novel_reading_accepted` | Cascade N5 fired — `metadata_sufficiency == "insufficient"` AND the LLM judge accepted the agent's reading as a valid novel interpretation. Auto-assigned. | +| `numerical_precision` | Cascade N6 fired (or sub-epsilon float noise the deterministic cascade misses). Auto-assigned for N6; can be picked manually for sub-epsilon cases. | +| `row_order` | Cascade N4 fired — agent's rowset matches modulo ORDER BY ties. Auto-assigned. | +| `trailing_whitespace` | Cascade N7 fired. Auto-assigned. | +| `column_order` | Cascade N8 fired. Auto-assigned. | +| `case_sensitivity` | Reserved for future tier; not auto-assigned yet. | +| `other` | Strict miss across all cascade tiers that doesn't fit any of the above. Requires human reasoning in `details`. | + +Set `agent_at_fault = true` only for `agent_miss` (and rarely `other`). +For everything else, the agent was either correct or constrained by an +upstream issue (metadata, audit, sim, grader). + +`secondary` carries co-existing flags — e.g. a `no_fail` with +`secondary=["gold_audit_quality"]` records "agent passed; original gold +was buggy; audit fixed it". + +`remediation_target` enum: `agent`, `prompt`, `kb`, `audit`, `grader`, +`user_sim`, `gold_sidecar`, `other`. Pick the surface that needs the +change. + +## Cascade tiers (used by the inline grader + summary) + +The cascade is computed by `tolerant_grader.grade_submission` and lives +in the submission annotation's `evaluation` block. Most-stringent → +most-lenient: + +| Tier | Field | Failure-class on a "this-tier-flipped-the-verdict" event | +| -- | -- | -- | +| N1 | `phase1_against_original_gold` | n/a (records strict pass against original `sol_sql`) | +| N2 | `phase1_against_audited_primary` | n/a (strict pass against `primary=true` variant) | +| N3 | `phase1_against_any_audited_variant` | `no_fail` | +| N4 | `correct_up_to_tie_order` | `row_order` | +| N5 | `novel_reading_judgment` | `novel_reading_accepted` (fires only when verdict=`insufficient`) | +| N6 | `correct_under_numeric_epsilon` | `numerical_precision` | +| N7 | `correct_under_trailing_whitespace` | `trailing_whitespace` | +| N8 | `correct_under_column_order` | `column_order` | + +When the cascade collapses (no audit variants in the gold sidecar), +N2 == N3 == N1. + +## U / C vocabulary + +A task is **U** (audit-unchanged) when its +`audited_gold/_audited.jsonl` row has +`audited_sol_sql == original_sol_sql` modulo whitespace, else **C** +(audit-changed). At eval-time the four headline buckets are: + +- **U-pass at N1** = "gold was right, agent right" +- **C-cosmetic-pass at N1** = "audit was cosmetic, agent matched both" +- **C-fix-pass at N2/N3** = "audit fixed buggy gold, agent matched fix" +- **U-fail / C-fail** = strict misses; cascade tier and failure-class + pinpoint why + +## Workflow + +1. Generate skeletons via + `scripts/dev1515_convert_runs.py` (or `bird_interact_agents.eval.annotate`). + The skeleton pre-fills mechanical fields + the cascade and auto-picks + `no_fail` / cascade-tier failure_class. Human-judgment fields land at + `PENDING_HUMAN_REVIEW` only for genuine `other` strict misses. +2. For each task: check `audit_unchanged` first. If unchanged AND + published metadata is unambiguous, the task is `sufficient`; if the + submission also passed, you're done. If unchanged AND agent failed, + the failure is `agent_miss` 90% of the time (verify by reading + submitted_sql vs gold). +3. For tasks where the audit changed gold (C-instances), START with the + audit-correctness gate. If the audit is buggy, fix it in + `audited_gold/_audited.jsonl` before writing the + annotation. Then walk the decision flow. +4. Read `external_knowledge` + the relevant KB entries + + `column_meaning_base.json` + the masked snippet inside + `amb_user_query` before assigning a verdict. +5. For ambiguous + a-interact: scan the trajectory for `ask_user` calls + and the sim's replies. If the sim withheld snippet content the agent + needed, the submission's `failure_classification.primary` is + `user_sim_under_disclosure`. +6. Validate every edit with + `SubmissionAnnotation.model_validate(...)` / `TaskAnnotation.model_validate(...)`. + +## Annotation rationale — write generously + +The task annotation's `metadata_sufficiency.rationale` and +`gold_variants[].interpretation` + `notes` are the canonical +documentation of why the audited gold reads the way it does. Future +readers — humans, the auditor revisiting after a KB update, downstream +analysis subagents — must be able to reconstruct the reasoning without +re-doing the audit work. Cite: + +- The specific KB entries (`_kb.jsonl#N`) that anchor the audited + reading. +- The column-meaning entries (`..`) that pin + literals / sampled values. +- The masked-snippet structure that confirms predicate shape. +- The audit's `changes[]` entries and `reasoning_summary`. + +For ambiguous and insufficient verdicts, also fill `evaluator_prompt` +with a self-contained LLM-judge prompt (~150–300 words) describing +what counts as a COMPATIBLE-if / INCOMPATIBLE-if reading. That prompt +fires only when verdict=`insufficient` AND no strict variant matches, +but you populate it speculatively for `ambiguous` cases too — it's +cheap insurance against future schema changes. + +## Re-grading without re-running the cloud + +`scripts/dev1515_convert_runs.py` re-executes the grader locally +(`grade_submission` against `//.sqlite`) using the +submitted SQL captured in `results/cloud//rows//attempt-1.json`. +After a grader change OR an audit fix, re-run that script and then +`scripts/dev1515_remap_failure_classes.py` to keep the cascade-tier +auto-classifications in sync. The remap is idempotent. + +## Outputs the aggregator wants + +`scripts/dev1515_cascade_summary.py` aggregates N1..N8 across all +per-(instance, run) submission annotations and emits the U/C-split +phase1 cascade plus a `failure_classification.primary` tally. Anything +auto-classified into `no_fail` / cascade-tier buckets contributes +without further human work; only `other` strict misses gate the +summary's "needs human attention" residual. diff --git a/.claude/skills/audit-gold-sql/SKILL.md b/.claude/skills/audit-gold-sql/SKILL.md index 5c14366a..c888ed87 100644 --- a/.claude/skills/audit-gold-sql/SKILL.md +++ b/.claude/skills/audit-gold-sql/SKILL.md @@ -57,6 +57,8 @@ already exists for this `instance_id`, **overwrite** it in place { "instance_id": "shop_3", "selected_database": "shop", + "variant_id": "primary", + "primary": true, "audit_status": "edited", "original_sol_sql": ["WITH SupplierTier AS (..."], "audited_sol_sql": ["SELECT COUNT(*) ..."], @@ -76,6 +78,13 @@ already exists for this `instance_id`, **overwrite** it in place } ``` +`variant_id` defaults to `"primary"` and `primary` defaults to `true` +for single-variant tasks (the common case). Multi-variant tasks (when +authoritative sources disagree among themselves — see the +`source_conflict` rule above) emit N rows sharing `instance_id`, each +with a distinct `variant_id` slug, with exactly one carrying +`primary: true`. + Field rules: - `audit_status` is one of three: @@ -89,6 +98,24 @@ Field rules: unauthorised). Fall back to the natural reading of `amb_user_query` using only authorised sources. `audited_sol_sql` is the natural reading; one `changes` entry documents the gap. +- When the task's authoritative sources **disagree among themselves** + on the same parameter — e.g. KB description says `LCS > 3`, masked + sql_snippet says `lcs > 2` for the same defined concept — emit + **multiple rows** to the audit JSONL, one per defensible reading, + sharing `instance_id` and distinguished by `variant_id`: + - Exactly one row carries `primary: true`. The KB-anchored reading + typically takes primary; the snippet-anchored reading is the + alternate. Document the choice in `reasoning_summary`. + - Every row keeps `audit_status: "edited"`. Each row's `changes[]` + has at least one entry with `clause_kind: "source_conflict"` + pointing at the other variant (its `original` is what THIS + variant rejected; `replacement` is what THIS variant chose; both + `justified_by` entries cite the source backing the chosen reading; + `why_unjustified` quotes the other source's contrary value). + - The downstream `TaskAnnotation` then carries + `internal_inconsistency.audit_resolution = "multi_variant"` and + one `gold_variants[]` entry per audit row (see + `annotate-task-submission` skill). - `original_sol_sql` and `audited_sol_sql` are both list[str], mirroring upstream `sol_sql`'s shape. For most tasks both have length 1. - `audited_sample_row` is the first row of running `audited_sol_sql[0]` diff --git a/scripts/dev1515_cascade_summary.py b/scripts/dev1515_cascade_summary.py new file mode 100644 index 00000000..b7daee05 --- /dev/null +++ b/scripts/dev1515_cascade_summary.py @@ -0,0 +1,188 @@ +"""DEV-1515: print aggregate cascade summary over the 53 annotated instances. + +Walks `annotations/mini-interact/*/.submission..json` for the +two May-31 runs and emits the phase1 block (counts + deltas + rates) +plus a small per-failure-class tally. + +Run after ``scripts/dev1515_convert_runs.py``. +""" +from __future__ import annotations + +import json +from collections import Counter, defaultdict +from pathlib import Path + +from bird_interact_agents import paths +from bird_interact_agents.eval.annotation_io import read_submission_annotation +from bird_interact_agents.eval.tolerant_grader import ( + _CASCADE_ORDER, + enforce_monotone_cascade, +) + + +RUNS = ( + "20260531t1008-claudes-slayer-890419", + "20260531t1343-claudes-slayer-b39bfc", +) +BENCHMARK = "mini-interact" + +# Most-stringent → most-lenient labels for the phase1 view. +_PHASE1_LABELS = ( + ("n1", "original gold passes"), + ("n2", "Δ + audited primary variant passes"), + ("n3", "Δ + any audited variant matches"), + ("n4", "Δ + correct up to tie order (row_order)"), + ("n5", "Δ + llm_judge accepts insufficient-only (novel_reading)"), + ("n6", "Δ + correct under numeric epsilon (numerical_precision)"), + ("n7", "Δ + correct under trailing whitespace"), + ("n8", "Δ + correct under column order"), +) + + +def _short(field: str) -> str: + return field.split("_", 1)[0] + + +def _row_cascade(p: Path) -> dict[str, bool]: + ann = read_submission_annotation(p) + ev = ann.evaluation + raw = { + "n1_original_gold": ev.phase1_against_original_gold == "pass", + "n2_audited_primary": ev.phase1_against_audited_primary == "pass", + "n3_any_audited_variant": ( + ev.phase1_against_any_audited_variant == "pass" + ), + "n4_tie_order": ev.correct_up_to_tie_order, + "n5_llm_judge": ( + ev.phase1_against_any_audited_variant == "pass" + or ev.correct_up_to_tie_order + or ev.novel_reading_judgment == "pass" + ), + "n6_numeric_epsilon": ev.correct_under_numeric_epsilon, + "n7_trailing_whitespace": ev.correct_under_trailing_whitespace, + "n8_column_order": ev.correct_under_column_order, + } + return enforce_monotone_cascade(raw) + + +def _gather_files() -> list[Path]: + annroot = paths.annotations_root() / BENCHMARK + out: list[Path] = [] + for run in RUNS: + for p in sorted(annroot.glob(f"*/*.submission.{run}.json")): + out.append(p) + return out + + +def _is_audit_unchanged(instance_id: str, audit_rows: dict) -> bool: + row = audit_rows.get(instance_id) + if row is None: + return False + orig = [" ".join((s or "").split()) for s in (row.get("original_sol_sql") or [])] + aud = [" ".join((s or "").split()) for s in (row.get("audited_sol_sql") or [])] + return orig == aud + + +def _aggregate(paths_: list[Path], audit_rows: dict) -> dict: + """Split U (audit unchanged) vs C (audit changed) at each cascade tier.""" + counts_u = {_short(f): 0 for f in _CASCADE_ORDER} + counts_c = {_short(f): 0 for f in _CASCADE_ORDER} + n_u = n_c = 0 + per_class: Counter[str] = Counter() + per_db_n1: defaultdict = defaultdict(lambda: [0, 0]) # [n1_pass, total] + for p in paths_: + ann = read_submission_annotation(p) + verdicts = _row_cascade(p) + unchanged = _is_audit_unchanged(ann.instance_id, audit_rows) + if unchanged: + n_u += 1 + target = counts_u + else: + n_c += 1 + target = counts_c + for f, v in verdicts.items(): + if v: + target[_short(f)] += 1 + per_class[ann.failure_classification.primary] += 1 + per_db_n1[ann.selected_database][1] += 1 + if verdicts["n1_original_gold"]: + per_db_n1[ann.selected_database][0] += 1 + return { + "n_u": n_u, + "n_c": n_c, + "counts_u": counts_u, + "counts_c": counts_c, + "failure_classes": dict(per_class), + "per_db_n1": dict(per_db_n1), + } + + +def _load_audit_rows() -> dict: + p = paths.audited_gold_root() / "mini_interact_audited.jsonl" + rows = {} + for line in p.read_text().splitlines(): + if not line.strip(): + continue + d = json.loads(line) + rows[d["instance_id"]] = d + return rows + + +def main() -> None: + files = _gather_files() + audit_rows = _load_audit_rows() + block = _aggregate(files, audit_rows) + nu, nc, n = block["n_u"], block["n_c"], block["n_u"] + block["n_c"] + + print("=" * 90) + print(f"DEV-1515 cascade summary over {n} (instance, run) pairs") + print(f" runs: {', '.join(RUNS)}") + print(f" U-instances (audit unchanged): {nu} " + f"C-instances (audit changed): {nc}") + print("=" * 90) + print() + print("phase1 cascade — U(audit-unchanged) vs C(audit-changed) split") + print(f"{'tier':52s} {'U':>6s} {'C':>6s} {'tot':>6s}") + print("-" * 90) + prev_total = None + for key, label in _PHASE1_LABELS: + u = block["counts_u"][key] + c = block["counts_c"][key] + tot = u + c + delta = "" if prev_total is None else ( + f" (+{tot - prev_total})" if tot - prev_total > 0 + else f" (+0)" + ) + u_str = f"{u}/{nu}" + c_str = f"{c}/{nc}" + tot_str = f"{tot}/{n}" + rate = (tot / n * 100.0) if n else 0.0 + print(f" {label:50s} {u_str:>6s} {c_str:>6s} " + f"{tot_str:>6s} {rate:>5.1f}%{delta}") + prev_total = tot + print() + print("Decomposition of N3 passes:") + n1_u = block["counts_u"]["n1"] + n1_c = block["counts_c"]["n1"] + n3_u = block["counts_u"]["n3"] + n3_c = block["counts_c"]["n3"] + print(f" U-pass at strict N1 (gold was right, agent right): {n1_u}") + print(f" C-cosmetic-pass at strict N1 (audit cosmetic, agent ok): {n1_c}") + print(f" C-fix-pass at N2/N3 (audit fixed buggy gold): {n3_c - n1_c}") + print(f" U-fail (gold was right, agent wrong): {nu - n1_u}") + print(f" C-fail (audit didn't help, agent wrong): {nc - n3_c}") + print() + print("Per-DB N1 (original-gold pass) tally:") + print(f" {'database':36s} {'pass':>5s} / {'tot':<3s} {'%':>5s}") + print("-" * 60) + for db, (passed, total) in sorted(block["per_db_n1"].items()): + rate = (passed / total * 100.0) if total else 0.0 + print(f" {db:36s} {passed:>5d} / {total:<3d} {rate:>4.1f}%") + print() + print("Failure-classification primary tally:") + for cls, c in sorted(block["failure_classes"].items(), key=lambda kv: -kv[1]): + print(f" {cls:36s} {c}") + + +if __name__ == "__main__": + main() diff --git a/scripts/dev1515_convert_livesqlbench_museum.py b/scripts/dev1515_convert_livesqlbench_museum.py new file mode 100644 index 00000000..c16131a5 --- /dev/null +++ b/scripts/dev1515_convert_livesqlbench_museum.py @@ -0,0 +1,364 @@ +"""DEV-1515: convert the museum livesqlbench tasks (15 instances) to the +new annotation schema. Mirrors `dev1515_convert_runs.py` but for +livesqlbench's one-shot data layout: + +* `query` (NL only) instead of `amb_user_query` (NL + masked snippet). +* No `user_query_ambiguity.critical_ambiguity`; no masked snippets. +* No user_sim — `user_sim_interaction.n_asks` is always 0. +* `sol_sql` + `external_knowledge` live in a separate gated sidecar, + joined on `instance_id`. + +Inputs: +* `livesqlbench-base-lite-sqlite/livesqlbench_data_sqlite.jsonl` — + main task rows. +* `livesqlbench-base-lite-sqlite/livesqlbench_sqlite_gt_kg_testcases_0528.jsonl` + — gold sidecar (`instance_id`, `sol_sql`, `external_knowledge`, + `test_cases`). +* `audited_gold/livesqlbench_audited.jsonl` — audited sidecar. +* `results/cloud//rows//attempt-1.json` — submitted SQL + + trajectory + usage for the museum tasks that ran on + `20260531t1013-claudes-slayer-48eb0f` (10/15). + +Outputs: +* `annotations/livesqlbench/museum/.task.json` (all 15). +* `annotations/livesqlbench/museum/.submission..json` (10 + with submissions). + +For the 5 museum instances without a submission, no submission file is +written. +""" +from __future__ import annotations + +import datetime as _dt +import json +import sqlite3 +from pathlib import Path + +from bird_interact_agents import paths +from bird_interact_agents.eval.annotation_io import ( + write_submission_annotation, + write_task_annotation, + submission_annotation_path, + task_annotation_path, +) +from bird_interact_agents.eval.annotation_schema import ( + AuditedGoldRef, + GoldVariantRef, + MaskedTerm, + MetadataSufficiency, + Provenance, + TaskAnnotation, +) +from bird_interact_agents.eval.grade_in_place import _build_submission_annotation +from bird_interact_agents.eval.annotate import ( + _user_sim_interaction_from_trajectory, +) +from bird_interact_agents.eval.tolerant_grader import ( + default_executor, + grade_submission, +) + + +BENCHMARK = "livesqlbench" +DB = "museum" +AUDIT_FILE_REL = "audited_gold/livesqlbench_audited.jsonl" +RUN_ID = "20260531t1013-claudes-slayer-48eb0f" + +PENDING = "PENDING_HUMAN_REVIEW" + +LIVESQL_ROOT = Path("/home/james/Dropbox/SLayer/livesqlbench-base-lite-sqlite") + + +def _norm(sqls): + return [" ".join((s or "").split()) for s in (sqls or [])] + + +def _load_data_rows() -> dict: + p = LIVESQL_ROOT / "livesqlbench_data_sqlite.jsonl" + rows = {} + for line in p.read_text().splitlines(): + if not line.strip(): + continue + d = json.loads(line) + if d.get("selected_database") == DB: + rows[d["instance_id"]] = d + return rows + + +def _load_gold_rows() -> dict: + p = LIVESQL_ROOT / "livesqlbench_sqlite_gt_kg_testcases_0528.jsonl" + rows = {} + for line in p.read_text().splitlines(): + if not line.strip(): + continue + d = json.loads(line) + rows[d["instance_id"]] = d + return rows + + +def _load_audit_rows() -> dict: + p = paths.audited_gold_root() / "livesqlbench_audited.jsonl" + rows = {} + if not p.exists(): + return rows + for line in p.read_text().splitlines(): + if not line.strip(): + continue + d = json.loads(line) + rows[d["instance_id"]] = d + return rows + + +def _build_task_annotation( + *, + instance_id: str, + data_row: dict, + gold_row: dict, + audit_row: dict | None, + audit_unchanged: bool, + annotated_at: str, +) -> TaskAnnotation: + if audit_row is not None and audit_unchanged: + ms = MetadataSufficiency( + verdict="sufficient", + rationale=( + "Audited gold is identical to the original `sol_sql` modulo " + "whitespace — the audit found nothing to change, so the " + "task's published metadata is sufficient and the original " + "gold is the canonical answer (DEV-1515 user-shortcut)." + ), + evidence_sources_consulted=["audited_gold/livesqlbench_audited.jsonl"], + ) + gold_variants: list[GoldVariantRef] = [] + original_gold_is_correct = True + evaluator_prompt = None + elif audit_row is not None and not audit_unchanged: + ms = MetadataSufficiency( + verdict="ambiguous", + rationale=PENDING, + evidence_sources_consulted=[], + ) + gold_variants = [ + GoldVariantRef( + variant_id="primary", + interpretation=PENDING, + primary=True, + anchored_in=[], + audited_gold_ref=AuditedGoldRef( + file=AUDIT_FILE_REL, + instance_id=instance_id, + variant_id="primary", + ), + notes=( + "Audit changed the gold relative to the original — " + "see audited_gold for the diff; human review pending." + ), + ), + ] + original_gold_is_correct = False + evaluator_prompt = None + else: + # No audit row — treat as sufficient by default; human can revise. + ms = MetadataSufficiency( + verdict="sufficient", + rationale=( + "No audit row in audited_gold/livesqlbench_audited.jsonl. " + "Default verdict assumes the original gold is canonical; " + "revisit if the agent's failure pattern shows real " + "metadata ambiguity." + ), + evidence_sources_consulted=["audited_gold/livesqlbench_audited.jsonl (missing)"], + ) + gold_variants = [] + original_gold_is_correct = True + evaluator_prompt = None + + return TaskAnnotation( + instance_id=instance_id, + selected_database=DB, + annotated_by="dev1515-convert-livesqlbench-museum", + annotated_at=annotated_at, + amb_user_query=data_row.get("query", ""), # livesqlbench: NL-only `query` + external_knowledge=list(gold_row.get("external_knowledge", []) or []), + masked_terms=[], # no masking layer in livesqlbench + metadata_sufficiency=ms, + original_gold_is_correct=original_gold_is_correct, + gold_variants=gold_variants, + evaluator_prompt=evaluator_prompt, + provenance=Provenance( + task_jsonl_path="livesqlbench_data_sqlite.jsonl", + task_jsonl_instance_id=instance_id, + ), + ) + + +def _process_one( + *, + instance_id: str, + data_row: dict, + gold_row: dict, + audit_row: dict | None, + audit_unchanged: bool, + rows_dir: Path | None, + annotations_root: Path, + annotated_at: str, +) -> dict: + db_path = LIVESQL_ROOT / DB / f"{DB}.sqlite" + task_ann = _build_task_annotation( + instance_id=instance_id, + data_row=data_row, + gold_row=gold_row, + audit_row=audit_row, + audit_unchanged=audit_unchanged, + annotated_at=annotated_at, + ) + task_dest = task_annotation_path( + benchmark=BENCHMARK, + selected_database=DB, + instance_id=instance_id, + repo_root=annotations_root.parent, + ) + write_task_annotation(task_ann, task_dest) + + sub_dest = None + rec_extra = {"has_submission": False} + if rows_dir is not None and (rows_dir / instance_id / "attempt-1.json").exists(): + attempt_path = rows_dir / instance_id / "attempt-1.json" + attempt = json.loads(attempt_path.read_text()) + submitted_sql = attempt.get("submitted_sql", "") + usage = attempt.get("usage", {}) or {} + + audit_arg = [audit_row] if audit_row is not None else [] + conn = sqlite3.connect(f"file:{db_path}?mode=ro", uri=True, timeout=60) + try: + cascade = grade_submission( + task_annotation=task_ann, + audited_gold_rows=audit_arg, + original_sol_sql=list(gold_row.get("sol_sql") or []), + submitted_sql=submitted_sql, + db_path=db_path, + conn=conn, + executor=default_executor, + llm_judge=None, + ) + finally: + conn.close() + + traj = list(attempt.get("trajectory") or []) + user_sim = _user_sim_interaction_from_trajectory(traj) + ann = _build_submission_annotation( + task_annotation=task_ann, + cascade=cascade, + benchmark=BENCHMARK, + run_id=RUN_ID, + trajectory_path=str(attempt_path), + predicted_row_count=attempt.get("predicted_row_count"), + duration_s=attempt.get("duration_s"), + cost_usd_agent=usage.get("cost_usd_agent"), + cost_usd_user_sim=usage.get("cost_usd_user_sim"), + n_agent_turns=usage.get("n_agent_turns"), + n_ask_user_calls=usage.get("n_ask_user_calls"), + user_sim_interaction=user_sim, + ) + sub_dest = submission_annotation_path( + benchmark=BENCHMARK, + selected_database=DB, + instance_id=instance_id, + run_id=RUN_ID, + repo_root=annotations_root.parent, + ) + write_submission_annotation(ann, sub_dest) + rec_extra = { + "has_submission": True, + "n1": cascade.n1_original_gold, + "n2": cascade.n2_audited_primary, + "n3": cascade.n3_any_audited_variant, + "n4": cascade.n4_tie_order, + "n5": cascade.n5_llm_judge, + "n6": cascade.n6_numeric_epsilon, + "n7": cascade.n7_trailing_whitespace, + "n8": cascade.n8_column_order, + "sub_path": str(sub_dest), + } + return { + "instance_id": instance_id, + "audit_unchanged": audit_unchanged, + "audit_present": audit_row is not None, + "task_path": str(task_dest), + **rec_extra, + } + + +def main() -> None: + annotations_root = paths.annotations_root() + results_cloud = paths.results_root() / "cloud" + + data_rows = _load_data_rows() + gold_rows = _load_gold_rows() + audit_rows = _load_audit_rows() + + instance_ids = sorted(data_rows.keys()) + print(f"museum instances in data: {len(instance_ids)}") + print(f"museum audit rows present: " + f"{sum(1 for i in instance_ids if i in audit_rows)}") + + run_rows_dir = results_cloud / RUN_ID / "rows" + print(f"run rows dir exists: {run_rows_dir.exists()}") + if run_rows_dir.exists(): + present = sorted(p.name for p in run_rows_dir.iterdir() + if p.is_dir() and p.name.startswith("museum_")) + print(f"museum instances with submissions: {len(present)}") + else: + present = [] + + annotated_at = ( + _dt.datetime.now(_dt.timezone.utc).replace(microsecond=0).isoformat() + ) + + results = [] + for iid in instance_ids: + if iid not in gold_rows: + print(f" {iid:30s} SKIP: no gold row") + continue + audit = audit_rows.get(iid) + unchanged = False + if audit is not None: + orig = _norm(audit.get("original_sol_sql")) + aud = _norm(audit.get("audited_sol_sql")) + unchanged = (orig == aud) + rows_dir = run_rows_dir if (run_rows_dir.exists() and iid in present) else None + try: + rec = _process_one( + instance_id=iid, + data_row=data_rows[iid], + gold_row=gold_rows[iid], + audit_row=audit, + audit_unchanged=unchanged, + rows_dir=rows_dir, + annotations_root=annotations_root, + annotated_at=annotated_at, + ) + except Exception as exc: + print(f" {iid:30s} FAILED: {type(exc).__name__}: {exc}") + raise + else: + audit_tag = "U" if unchanged else ("C" if audit else "-") + sub_tag = "S" if rec["has_submission"] else " " + cascade_str = "" + if rec["has_submission"]: + cascade_str = ( + f" N1={int(rec['n1'])} N3={int(rec['n3'])} " + f"N6={int(rec['n6'])}" + ) + print(f" [{audit_tag}{sub_tag}] {iid:30s}{cascade_str}") + results.append(rec) + + out_index = annotations_root / "_dev1515_convert_museum_index.json" + out_index.write_text(json.dumps(results, indent=2) + "\n") + print(f"\nWrote {len(results)} task annotations " + f"+ {sum(1 for r in results if r['has_submission'])} submission annotations.") + print(f"Index: {out_index}") + + +if __name__ == "__main__": + main() diff --git a/scripts/dev1515_convert_runs.py b/scripts/dev1515_convert_runs.py new file mode 100644 index 00000000..f8d109c7 --- /dev/null +++ b/scripts/dev1515_convert_runs.py @@ -0,0 +1,326 @@ +"""DEV-1515: convert the two May-31 mini-interact runs to the new +annotation schema. + +* `20260531t1008-claudes-slayer-890419` (15 households) +* `20260531t1343-claudes-slayer-b39bfc` (38 alien/credit/etc.) + +For each of the 53 (instance, run) pairs: + +1. Build `.task.json` (run-independent). When the audited gold is + IDENTICAL to the original gold (modulo whitespace), pre-fill + `metadata_sufficiency.verdict = "sufficient"`, + `original_gold_is_correct = True`, and `gold_variants = []` — per the + user's "treat unchanged audit as correct" shortcut. Otherwise leave + `metadata_sufficiency` + `failure_classification` sentinel-filled for + subagent / human fill-in but pre-populate `gold_variants` with the + primary audited reference. +2. Build `.submission..json` by running the full + `tolerant_grader.grade_submission` cascade against the local sqlite + DB at `//.sqlite`. N5 (LLM judge) is left at + False here — it only fires for verdict="insufficient" tasks at grade + time anyway. +3. Persist both via `annotation_io.write_*`. + +Re-runnable: every call OVERWRITES the destination file (the user can +re-run after editing the task file). No `gold_variants` is preserved +across runs for changed-audit tasks — the next refresh will reset it. +""" +from __future__ import annotations + +import datetime as _dt +import json +import sqlite3 +from pathlib import Path +from typing import Optional + +from bird_interact_agents import paths +from bird_interact_agents.eval.annotation_io import ( + write_submission_annotation, + write_task_annotation, + submission_annotation_path, + task_annotation_path, +) +from bird_interact_agents.eval.annotation_schema import ( + AuditedGoldRef, + GoldVariantRef, + MaskedTerm, + MetadataSufficiency, + Provenance, + TaskAnnotation, +) +from bird_interact_agents.eval.grade_in_place import _build_submission_annotation +from bird_interact_agents.eval.annotate import ( + _user_sim_interaction_from_trajectory, +) +from bird_interact_agents.eval.tolerant_grader import ( + default_executor, + grade_submission, +) + + +BENCHMARK = "mini-interact" +AUDIT_FILE_REL = "audited_gold/mini_interact_audited.jsonl" + +RUNS = { + "20260531t1008-claudes-slayer-890419": "households", + "20260531t1343-claudes-slayer-b39bfc": "mixed", +} + +PENDING = "PENDING_HUMAN_REVIEW" + + +def _norm(sqls): + return [" ".join((s or "").split()) for s in (sqls or [])] + + +def _masked_terms_from(task_row: dict) -> list[MaskedTerm]: + amb = (task_row.get("user_query_ambiguity") or {}).get("critical_ambiguity") or [] + out: list[MaskedTerm] = [] + for entry in amb: + out.append(MaskedTerm( + term=entry.get("term", ""), + type=entry.get("type", "intent_ambiguity"), + is_mask=bool(entry.get("is_mask", True)), + metadata_evidence=list(entry.get("metadata_evidence", []) or []), + )) + return out + + +def _build_task_annotation( + *, + task_row: dict, + audit_row: dict, + audit_unchanged: bool, + annotated_at: str, +) -> TaskAnnotation: + """Build the per-task annotation with the user's unchanged-audit shortcut.""" + iid = task_row["instance_id"] + db = task_row["selected_database"] + + if audit_unchanged: + ms = MetadataSufficiency( + verdict="sufficient", + rationale=( + "Audited gold is identical to the original sol_sql modulo " + "whitespace — the audit found nothing to change, so the " + "task's published metadata is sufficient and the original " + "gold is the canonical answer (DEV-1515 user-shortcut)." + ), + evidence_sources_consulted=[ + "audited_gold/mini_interact_audited.jsonl", + ], + ) + gold_variants: list[GoldVariantRef] = [] + original_gold_is_correct = True + evaluator_prompt = None + else: + ms = MetadataSufficiency( + verdict="ambiguous", + rationale=PENDING, + evidence_sources_consulted=[], + ) + gold_variants = [ + GoldVariantRef( + variant_id="primary", + interpretation=PENDING, + primary=True, + anchored_in=[], + audited_gold_ref=AuditedGoldRef( + file=AUDIT_FILE_REL, + instance_id=iid, + variant_id="primary", + ), + notes=( + "Audit changed the gold relative to the original — " + "see audited_gold for the diff; human review pending." + ), + ), + ] + original_gold_is_correct = False + evaluator_prompt = None + + return TaskAnnotation( + instance_id=iid, + selected_database=db, + annotated_by="dev1515-convert-runs", + annotated_at=annotated_at, + amb_user_query=task_row.get("amb_user_query", ""), + external_knowledge=list(task_row.get("external_knowledge", []) or []), + masked_terms=_masked_terms_from(task_row), + metadata_sufficiency=ms, + original_gold_is_correct=original_gold_is_correct, + gold_variants=gold_variants, + evaluator_prompt=evaluator_prompt, + provenance=Provenance( + task_jsonl_path="mini_interact.jsonl", + task_jsonl_instance_id=iid, + ), + ) + + +def _process_one( + *, + run_id: str, + instance_id: str, + task_row: dict, + audit_row: dict, + audit_unchanged: bool, + rows_dir: Path, + mini_root: Path, + annotations_root: Path, + annotated_at: str, +) -> dict: + """Return a small per-instance dict with cascade booleans + paths.""" + db = task_row["selected_database"] + db_path = mini_root / db / f"{db}.sqlite" + attempt_path = rows_dir / instance_id / "attempt-1.json" + attempt = json.loads(attempt_path.read_text()) + submitted_sql = attempt.get("submitted_sql", "") + usage = attempt.get("usage", {}) or {} + + # 1) Task annotation. + task_ann = _build_task_annotation( + task_row=task_row, + audit_row=audit_row, + audit_unchanged=audit_unchanged, + annotated_at=annotated_at, + ) + task_dest = task_annotation_path( + benchmark=BENCHMARK, + selected_database=db, + instance_id=instance_id, + repo_root=annotations_root.parent, + ) + write_task_annotation(task_ann, task_dest) + + # 2) Cascade via grade_submission. + conn = sqlite3.connect(f"file:{db_path}?mode=ro", uri=True, timeout=60) + try: + cascade = grade_submission( + task_annotation=task_ann, + audited_gold_rows=[audit_row], + original_sol_sql=list(task_row.get("sol_sql") or []), + submitted_sql=submitted_sql, + db_path=db_path, + conn=conn, + executor=default_executor, + llm_judge=None, # N5 stays False for now + ) + finally: + conn.close() + + # 3) Submission annotation. + traj = list(attempt.get("trajectory") or []) + user_sim = _user_sim_interaction_from_trajectory(traj) + ann = _build_submission_annotation( + task_annotation=task_ann, + cascade=cascade, + benchmark=BENCHMARK, + run_id=run_id, + trajectory_path=str(attempt_path), + predicted_row_count=attempt.get("predicted_row_count"), + duration_s=attempt.get("duration_s"), + cost_usd_agent=usage.get("cost_usd_agent"), + cost_usd_user_sim=usage.get("cost_usd_user_sim"), + n_agent_turns=usage.get("n_agent_turns"), + n_ask_user_calls=usage.get("n_ask_user_calls"), + user_sim_interaction=user_sim, + ) + sub_dest = submission_annotation_path( + benchmark=BENCHMARK, + selected_database=db, + instance_id=instance_id, + run_id=run_id, + repo_root=annotations_root.parent, + ) + write_submission_annotation(ann, sub_dest) + + return { + "instance_id": instance_id, + "db": db, + "run_id": run_id, + "audit_unchanged": audit_unchanged, + "n1": cascade.n1_original_gold, + "n2": cascade.n2_audited_primary, + "n3": cascade.n3_any_audited_variant, + "n4": cascade.n4_tie_order, + "n5": cascade.n5_llm_judge, + "n6": cascade.n6_numeric_epsilon, + "n7": cascade.n7_trailing_whitespace, + "n8": cascade.n8_column_order, + "task_path": str(task_dest), + "sub_path": str(sub_dest), + } + + +def main() -> None: + mini_root = paths.mini_interact_root() + annotations_root = paths.annotations_root() + results_cloud = paths.results_root() / "cloud" + + # Load mini-interact rows + audited gold. + rows = {} + for line in (mini_root / "mini_interact.jsonl").read_text().splitlines(): + if not line.strip(): + continue + d = json.loads(line) + rows[d["instance_id"]] = d + + audit_rows = {} + for line in ( + paths.audited_gold_root() / "mini_interact_audited.jsonl" + ).read_text().splitlines(): + if not line.strip(): + continue + d = json.loads(line) + audit_rows[d["instance_id"]] = d + + annotated_at = ( + _dt.datetime.now(_dt.timezone.utc).replace(microsecond=0).isoformat() + ) + + results: list[dict] = [] + for run_id, _label in RUNS.items(): + rows_dir = results_cloud / run_id / "rows" + inst_ids = sorted(p.name for p in rows_dir.iterdir() if p.is_dir()) + print(f"\n[{run_id}] {len(inst_ids)} instances") + for iid in inst_ids: + task_row = rows[iid] + audit_row = audit_rows[iid] + n_orig = _norm(audit_row.get("original_sol_sql")) + n_aud = _norm(audit_row.get("audited_sol_sql")) + unchanged = n_orig == n_aud + try: + rec = _process_one( + run_id=run_id, + instance_id=iid, + task_row=task_row, + audit_row=audit_row, + audit_unchanged=unchanged, + rows_dir=rows_dir, + mini_root=mini_root, + annotations_root=annotations_root, + annotated_at=annotated_at, + ) + except Exception as exc: # noqa: BLE001 + print(f" {iid:40s} FAILED: {type(exc).__name__}: {exc}") + raise + else: + tag = "U" if unchanged else "C" + print( + f" [{tag}] {iid:40s} " + f"N1={int(rec['n1'])} N2={int(rec['n2'])} N3={int(rec['n3'])} " + f"N4={int(rec['n4'])} N6={int(rec['n6'])} " + f"N7={int(rec['n7'])} N8={int(rec['n8'])}" + ) + results.append(rec) + + # Save a small index for the summary step. + out_index = annotations_root / "_dev1515_convert_index.json" + out_index.write_text(json.dumps(results, indent=2) + "\n") + print(f"\nWrote {len(results)} (task, submission) pairs.") + print(f"Index: {out_index}") + + +if __name__ == "__main__": + main() diff --git a/scripts/dev1515_households_14_multivariant.py b/scripts/dev1515_households_14_multivariant.py new file mode 100644 index 00000000..179016d4 --- /dev/null +++ b/scripts/dev1515_households_14_multivariant.py @@ -0,0 +1,388 @@ +"""DEV-1515: rewrite households_14 as the worked example of the +multi-variant-on-source-conflict pattern. + +KB 29 (Comfortable Living Household) description says +``Living Condition Score > 3 AND Bathroom Ratio > 0.5``. +The masked sql_snippet for ``"good quality of life"`` (in +``user_query_ambiguity.critical_ambiguity``) says +``lcs > 2 AND bath_ratio > 0.5``. + +Two authoritative sources disagree on the same parameter — the LCS +threshold. The audit cannot pick a side without ratifying half the +sources; the right answer is to emit BOTH readings as audit rows and +let the cascade match either. + +What this script does: + +1. In ``audited_gold/mini_interact_audited.jsonl``: + * Marks the existing row `variant_id="kb_definition_reading"`, + keeps `primary=true`, threshold stays `lcs > 3` (KB 29's + description). + * Adds a second row `variant_id="snippet_reading"`, + `primary=false`, threshold flipped to `lcs > 2` (matching the + masked snippet). Everything else (data-aligned LCS formula, + dwelling-class synonyms, infra IN-sets, bath_ratio formula, + ORDER BY) stays identical. + * Both rows carry an additional `changes[]` entry of + `clause_kind="source_conflict"` pointing at the other source's + contrary value. + +2. In ``annotations/mini-interact/households/households_14.task.json``: + * `verdict = "ambiguous"` (was `sufficient` — wrong; sources + don't converge). + * `internal_inconsistency` populated: + - `sources_in_conflict = ["KB#29.description: 'LCS > 3'", + "critical_ambiguity for 'good quality of life'.sql_snippet: + 'lcs > 2'"]` + - `description`: 2 sentences naming the disagreement. + - `audit_resolution = "multi_variant"`. + * `gold_variants` carries TWO entries, one per audit row. + * `rationale` foregrounds the conflict. + +3. In + ``annotations/mini-interact/households/households_14.submission..json``: + * Re-grade using the now multi-variant audit; cascade picks the + matching variant via existing N3 logic. + * Failure_classification adjusts to whatever the agent's real + error is (likely `agent_miss` — agent missed bath_ratio > 0.5 + entirely). +""" +from __future__ import annotations + +import datetime as _dt +import json +import sqlite3 +from pathlib import Path + +from bird_interact_agents import paths +from bird_interact_agents.eval.annotation_io import ( + read_task_annotation, + submission_annotation_path, + task_annotation_path, + write_submission_annotation, + write_task_annotation, +) +from bird_interact_agents.eval.annotation_schema import ( + AuditedGoldRef, + GoldVariantRef, + InternalInconsistency, + MetadataSufficiency, +) +from bird_interact_agents.eval.annotate import _user_sim_interaction_from_trajectory +from bird_interact_agents.eval.grade_in_place import _build_submission_annotation +from bird_interact_agents.eval.tolerant_grader import default_executor, grade_submission + + +INSTANCE_ID = "households_14" +DB = "households" +BENCHMARK = "mini-interact" +RUN_ID = "20260531t1008-claudes-slayer-890419" +AUDIT_FILE_REL = "audited_gold/mini_interact_audited.jsonl" + + +# --------------------------------------------------------------------------- +# Step 1 — rewrite audited_gold/mini_interact_audited.jsonl +# --------------------------------------------------------------------------- + + +def _patch_audit_jsonl() -> None: + audit_path = paths.audited_gold_root() / "mini_interact_audited.jsonl" + rows: list[dict] = [] + found = False + for line in audit_path.read_text().splitlines(): + if not line.strip(): + continue + d = json.loads(line) + if d["instance_id"] != INSTANCE_ID: + rows.append(d) + continue + # The existing row becomes the kb_definition_reading variant + # (already `lcs > 3`). + found = True + kb_row = dict(d) + kb_row["variant_id"] = "kb_definition_reading" + kb_row["primary"] = True + kb_row.setdefault("changes", []).append({ + "clause_kind": "source_conflict", + "original": "lcs > 2 (per critical_ambiguity for 'good quality of life'.sql_snippet)", + "replacement": "lcs > 3", + "why_unjustified": ( + "KB#29.description authoritatively states 'Living " + "Condition Score is greater than 3 AND Bathroom Ratio " + "is greater than 0.5'. The masked sql_snippet for " + "'good quality of life' contradicts it with 'lcs > 2'. " + "This variant follows the KB; the snippet_reading " + "variant follows the snippet." + ), + "justified_by": ["kb:29"], + }) + kb_row["reasoning_summary"] = ( + "[DEV-1515 source-conflict] KB#29.description: LCS > 3 AND " + "Bath_Ratio > 0.5. The masked sql_snippet contradicts with " + "'lcs > 2'. This is the KB-anchored reading; see the " + "snippet_reading variant for the snippet-anchored alternate." + ) + rows.append(kb_row) + + # Build the snippet_reading variant: identical to kb_row except + # `lcs > 3` → `lcs > 2` in audited_sol_sql. + snippet_row = json.loads(json.dumps(kb_row)) # deep copy + snippet_row["variant_id"] = "snippet_reading" + snippet_row["primary"] = False + snippet_row["audited_sol_sql"] = [ + s.replace("lcs > 3", "lcs > 2") for s in kb_row["audited_sol_sql"] + ] + # Adjust the source_conflict entry to point the other way. + snippet_row["changes"] = [ + c for c in snippet_row["changes"] if c["clause_kind"] != "source_conflict" + ] + snippet_row["changes"].append({ + "clause_kind": "source_conflict", + "original": "lcs > 3 (per KB#29.description)", + "replacement": "lcs > 2", + "why_unjustified": ( + "The masked sql_snippet for 'good quality of life' in " + "user_query_ambiguity.critical_ambiguity authoritatively " + "uses 'lcs > 2'. KB#29.description contradicts with " + "'LCS > 3'. This variant follows the snippet (what a " + "cooperative user_sim would disclose); see the " + "kb_definition_reading variant for the KB alternate." + ), + "justified_by": ["labeled_ambiguity:good quality of life"], + }) + snippet_row["reasoning_summary"] = ( + "[DEV-1515 source-conflict] Masked sql_snippet for 'good " + "quality of life': lcs > 2 AND bath_ratio > 0.5. KB#29 " + "description contradicts with 'LCS > 3'. This is the " + "snippet-anchored reading; see the kb_definition_reading " + "variant for the KB-anchored alternate." + ) + # Recompute audited_sample_row for the snippet_reading + # variant against the live DB. + db_path = paths.mini_interact_root() / DB / f"{DB}.sqlite" + conn = sqlite3.connect(f"file:{db_path}?mode=ro", uri=True, timeout=30) + try: + cur = conn.cursor() + cur.execute(snippet_row["audited_sol_sql"][-1]) + rows_out = cur.fetchall() + if rows_out: + snippet_row["audited_sample_row"] = list(rows_out[0]) + snippet_row["audited_sample_row_status"] = "ok" + snippet_row.pop("audited_sample_row_error", None) + else: + snippet_row["audited_sample_row"] = [] + snippet_row["audited_sample_row_status"] = "empty" + snippet_row.pop("audited_sample_row_error", None) + except Exception as exc: + snippet_row["audited_sample_row"] = [] + snippet_row["audited_sample_row_status"] = "error" + snippet_row["audited_sample_row_error"] = f"{type(exc).__name__}: {exc}" + finally: + conn.close() + rows.append(snippet_row) + + if not found: + raise SystemExit(f"{INSTANCE_ID} not found in {audit_path}") + + audit_path.write_text("\n".join(json.dumps(r) for r in rows) + "\n") + print(f"Patched {audit_path}: {INSTANCE_ID} now has 2 rows " + f"(kb_definition_reading [primary] + snippet_reading)") + + +# --------------------------------------------------------------------------- +# Step 2 — rewrite task annotation +# --------------------------------------------------------------------------- + + +def _patch_task_annotation() -> None: + p = task_annotation_path( + benchmark=BENCHMARK, + selected_database=DB, + instance_id=INSTANCE_ID, + repo_root=paths.main_checkout_root(), + ) + ann = read_task_annotation(p) + ann.metadata_sufficiency = MetadataSufficiency( + verdict="ambiguous", + rationale=( + "Sources contradict on the LCS threshold for 'Comfortable " + "Living Household' (KB 29). KB#29.description authoritatively " + "says 'LCS > 3 AND Bath_Ratio > 0.5'; the masked sql_snippet " + "for the user-query term 'good quality of life' (in " + "user_query_ambiguity.critical_ambiguity) authoritatively says " + "'lcs > 2 AND bath_ratio > 0.5'. An agent cannot satisfy both " + "thresholds at once. The audit records both readings as gold " + "variants (kb_definition_reading primary; snippet_reading " + "alternate); the cascade passes the agent on either match." + ), + evidence_sources_consulted=[ + "households_kb.jsonl#29", + "mini_interact.jsonl: households_14.user_query_ambiguity.critical_ambiguity['good quality of life'].sql_snippet", + ], + ) + ann.original_gold_is_correct = False + ann.gold_variants = [ + GoldVariantRef( + variant_id="kb_definition_reading", + interpretation=( + "Comfortable = LCS > 3 AND Bath_Ratio > 0.5, per KB 29's " + "description field. Data-aligned LCS formula: dwelling " + "(brickwork/brick=4, apartment/apt=3, else 1) weighted " + "0.5; infra avg over data-aligned water/road/park " + "values weighted 0.5." + ), + primary=True, + anchored_in=["households_kb.jsonl#29.description"], + audited_gold_ref=AuditedGoldRef( + file=AUDIT_FILE_REL, + instance_id=INSTANCE_ID, + variant_id="kb_definition_reading", + ), + notes=( + "Contradicts the masked sql_snippet for 'good quality " + "of life' (which says lcs > 2). See the snippet_reading " + "variant. Both readings are recorded; the cascade matches " + "either." + ), + ), + GoldVariantRef( + variant_id="snippet_reading", + interpretation=( + "Comfortable = lcs > 2 AND bath_ratio > 0.5, per the " + "masked sql_snippet for 'good quality of life' in " + "user_query_ambiguity.critical_ambiguity (what a " + "cooperative user_sim would disclose). Same data-aligned " + "LCS formula as kb_definition_reading; only the threshold " + "differs." + ), + primary=False, + anchored_in=[ + "mini_interact.jsonl: households_14.user_query_ambiguity.critical_ambiguity['good quality of life'].sql_snippet", + ], + audited_gold_ref=AuditedGoldRef( + file=AUDIT_FILE_REL, + instance_id=INSTANCE_ID, + variant_id="snippet_reading", + ), + notes=( + "Contradicts KB#29.description (which says LCS > 3). " + "See the kb_definition_reading variant. Both readings " + "are recorded; the cascade matches either." + ), + ), + ] + ann.evaluator_prompt = None + ann.internal_inconsistency = InternalInconsistency( + sources_in_conflict=[ + "households_kb.jsonl#29.description: 'Living Condition Score is greater than 3 AND Bathroom Ratio is greater than 0.5'", + "mini_interact.jsonl: households_14.user_query_ambiguity.critical_ambiguity['good quality of life'].sql_snippet: 'lcs > 2 AND bath_ratio > 0.5'", + ], + description=( + "KB 29's public description and the task's own masked " + "sql_snippet disagree on the LCS threshold: KB says > 3, " + "snippet says > 2. An agent cannot satisfy both at once — " + "the threshold is a single value that must be picked." + ), + audit_resolution="multi_variant", + ) + ann.annotated_by = "dev1515-households_14-multivariant" + ann.annotated_at = ( + _dt.datetime.now(_dt.timezone.utc).replace(microsecond=0).isoformat() + ) + write_task_annotation(ann, p) + print(f"Patched {p}: verdict=ambiguous, internal_inconsistency set, " + f"2 gold_variants") + + +# --------------------------------------------------------------------------- +# Step 3 — re-grade the submission against the now multi-variant audit +# --------------------------------------------------------------------------- + + +def _regrade_submission() -> None: + db_path = paths.mini_interact_root() / DB / f"{DB}.sqlite" + attempt_path = ( + paths.results_root() / "cloud" / RUN_ID / "rows" + / INSTANCE_ID / "attempt-1.json" + ) + attempt = json.loads(attempt_path.read_text()) + submitted_sql = attempt.get("submitted_sql", "") + usage = attempt.get("usage", {}) or {} + + audit_path = paths.audited_gold_root() / "mini_interact_audited.jsonl" + audit_rows: list[dict] = [] + for line in audit_path.read_text().splitlines(): + if not line.strip(): + continue + d = json.loads(line) + if d["instance_id"] == INSTANCE_ID: + audit_rows.append(d) + + # Task row. + mi = paths.mini_interact_root() / "mini_interact.jsonl" + task_row = None + for line in mi.read_text().splitlines(): + if not line.strip(): + continue + d = json.loads(line) + if d["instance_id"] == INSTANCE_ID: + task_row = d + break + assert task_row is not None + + task_ann_path = task_annotation_path( + benchmark=BENCHMARK, selected_database=DB, + instance_id=INSTANCE_ID, repo_root=paths.main_checkout_root(), + ) + task_ann = read_task_annotation(task_ann_path) + + conn = sqlite3.connect(f"file:{db_path}?mode=ro", uri=True, timeout=60) + try: + cascade = grade_submission( + task_annotation=task_ann, + audited_gold_rows=audit_rows, + original_sol_sql=list(task_row.get("sol_sql") or []), + submitted_sql=submitted_sql, + db_path=db_path, + conn=conn, + executor=default_executor, + llm_judge=None, + ) + finally: + conn.close() + + traj = list(attempt.get("trajectory") or []) + user_sim = _user_sim_interaction_from_trajectory(traj) + ann = _build_submission_annotation( + task_annotation=task_ann, + cascade=cascade, + benchmark=BENCHMARK, + run_id=RUN_ID, + trajectory_path=str(attempt_path), + predicted_row_count=attempt.get("predicted_row_count"), + duration_s=attempt.get("duration_s"), + cost_usd_agent=usage.get("cost_usd_agent"), + cost_usd_user_sim=usage.get("cost_usd_user_sim"), + n_agent_turns=usage.get("n_agent_turns"), + n_ask_user_calls=usage.get("n_ask_user_calls"), + user_sim_interaction=user_sim, + ) + sub_dest = submission_annotation_path( + benchmark=BENCHMARK, selected_database=DB, + instance_id=INSTANCE_ID, run_id=RUN_ID, + repo_root=paths.main_checkout_root(), + ) + write_submission_annotation(ann, sub_dest) + print(f"Re-graded submission: cascade={cascade.model_dump()}") + print(f" primary={ann.failure_classification.primary} " + f"matched_variant_id={cascade.matched_variant_id}") + + +def main() -> None: + _patch_audit_jsonl() + _patch_task_annotation() + _regrade_submission() + + +if __name__ == "__main__": + main() diff --git a/scripts/dev1515_reclassify_sufficiency.py b/scripts/dev1515_reclassify_sufficiency.py new file mode 100644 index 00000000..5b5da673 --- /dev/null +++ b/scripts/dev1515_reclassify_sufficiency.py @@ -0,0 +1,131 @@ +"""DEV-1515: tighten the ``metadata_sufficiency.verdict`` definition. + +Sharper rule: ``insufficient`` ⇒ even with maximally cooperative +user_sim disclosure of the masked sql_snippet, the answer remains +underspecified (e.g. arbitrary policy thresholds with no semantic +anchor). Cases where the masked snippet contains exact constraints +the user_sim could disclose drop to ``ambiguous`` — the failure on +the submission side moves to ``user_sim_under_disclosure``. + +Applied to the 2 instances on this branch: + +* ``households_12``: predicates ``total_vehicles>1`` and + ``domestichelp='no domestic workers'`` are exact constraints in the + masked snippet. User_sim refused to disclose. Reclassified + ``insufficient`` → ``ambiguous``; submission's + ``failure_classification.primary`` set to + ``user_sim_under_disclosure``. +* ``alien_2``: gold uses population-stddev, agent used sample-stddev. + No user_sim (one-shot). KB/sources don't pin the convention. + Reclassified ``insufficient`` → ``ambiguous``; submission's failure + stays ``metadata_ambiguity`` (which it already was). + +``households_15`` stays ``insufficient`` (audit-status=unrecoverable; +coefficients have no semantic anchor). +""" +from __future__ import annotations + +from bird_interact_agents import paths +from bird_interact_agents.eval.annotation_io import ( + read_submission_annotation, + read_task_annotation, + submission_annotation_path, + task_annotation_path, + write_submission_annotation, + write_task_annotation, +) + + +_TASK_PATCHES = ( + ( + "households_12", + "households", + "ambiguous", + ( + "Audit-resolvable via user_sim disclosure: the masked " + "sql_snippet specifies exact predicates " + "`total_vehicles>1` and `domestichelp='no domestic workers'` " + "as part of the household-need definition. KB 42 + column " + "meanings don't name those predicates, but a cooperative " + "user_sim could disclose them on a well-phrased ask. " + "Verdict is `ambiguous` (not `insufficient`) because the " + "task IS solvable with user_sim help; in the actual run " + "the sim refused row-count disclosure and gave a vague " + "'use the service criteria' reply (see submission " + "failure_classification = user_sim_under_disclosure)." + ), + ["households_kb.jsonl#42", "amb_user_query (masked sql_snippet)"], + ), + ( + "alien_2", + "alien", + "ambiguous", + ( + "Audit added SQRT around variance; sample-stddev (Bessel " + "n-1) vs population-stddev (n) is a known statistical-" + "convention ambiguity. Neither KB 52 nor the labeled " + "sql_snippet nor sampled values disambiguate. Verdict is " + "`ambiguous` (not `insufficient`) because a cooperative " + "user_sim could clarify which formula is meant. In the " + "actual one-shot run there is no user_sim — failure stays " + "metadata_ambiguity." + ), + ["alien_kb.jsonl#52", "amb_user_query (masked sql_snippet)"], + ), +) + +_SUBMISSION_FAILURE_PATCHES = ( + ( + "households_12", + "households", + "20260531t1008-claudes-slayer-890419", + "user_sim_under_disclosure", + ["metadata_ambiguity"], + ), +) + + +def main() -> None: + repo_root = paths.main_checkout_root() + for iid, db, new_verdict, new_rationale, new_evidence in _TASK_PATCHES: + p = task_annotation_path( + benchmark="mini-interact", selected_database=db, + instance_id=iid, repo_root=repo_root, + ) + ann = read_task_annotation(p) + old = ann.metadata_sufficiency.verdict + ann.metadata_sufficiency.verdict = new_verdict # type: ignore[assignment] + ann.metadata_sufficiency.rationale = new_rationale + ann.metadata_sufficiency.evidence_sources_consulted = list( + new_evidence + ) + # Reset original_gold_is_correct + drop evaluator_prompt for + # the new `ambiguous` verdict. + ann.original_gold_is_correct = False + if new_verdict == "ambiguous": + ann.evaluator_prompt = None + write_task_annotation(ann, p) + print(f" task {iid}: {old} → {new_verdict}") + + for iid, db, run, new_primary, new_secondary in _SUBMISSION_FAILURE_PATCHES: + p = submission_annotation_path( + benchmark="mini-interact", selected_database=db, + instance_id=iid, run_id=run, repo_root=repo_root, + ) + ann = read_submission_annotation(p) + old = ann.failure_classification.primary + ann.failure_classification.primary = new_primary # type: ignore[assignment] + ann.failure_classification.secondary = list(new_secondary) + if not ann.failure_classification.remediation_text: + ann.failure_classification.remediation_text = ( + "Tighten user_sim disclosure policy on row-count + " + "predicate-naming asks when masked sql_snippet contains " + "exact constraints." + ) + ann.failure_classification.remediation_target = "user_sim" # type: ignore[assignment] + write_submission_annotation(ann, p) + print(f" submission {iid}: primary {old} → {new_primary}") + + +if __name__ == "__main__": + main() diff --git a/scripts/dev1515_remap_failure_classes.py b/scripts/dev1515_remap_failure_classes.py new file mode 100644 index 00000000..cad77e4d --- /dev/null +++ b/scripts/dev1515_remap_failure_classes.py @@ -0,0 +1,137 @@ +"""DEV-1515: remap legacy ``failure_classification`` values on the 53 +existing submission annotations after the schema split: + +* Passes (N3=1) → `primary="no_fail"` (was `other` or some flag like + `gold_audit_quality`). When demoting a non-`other` flag, move it into + `secondary` so the signal isn't lost. +* Legacy `grader_stability` → cascade-tier-specific bucket, picking the + lowest-tier tolerance that fires: + * N4 (tie order) → `row_order` + * N6 (numeric epsilon) → `numerical_precision` + * N7 (whitespace) → `trailing_whitespace` + * N8 (column order) → `column_order` + When NO cascade tier fires but the subagent specifically classified as + `grader_stability`, default to `numerical_precision` (the most common + source of grader-stability complaints in the dataset). +* Auto-classified `other` with a cascade-tier match → upgrade to the + specific class. + +This script operates on raw JSON before Pydantic validation because the +new schema rejects legacy `grader_stability` literals on read.""" +from __future__ import annotations + +import json + +from bird_interact_agents import paths +from bird_interact_agents.eval.annotation_schema import SubmissionAnnotation + + +_TIER_FIELDS_TO_CLASS = ( + ("correct_up_to_tie_order", "row_order"), + ("correct_under_numeric_epsilon", "numerical_precision"), + ("correct_under_trailing_whitespace", "trailing_whitespace"), + ("correct_under_column_order", "column_order"), +) + + +def _cascade_tier_for(ev: dict) -> str | None: + """Tier-tolerance class (or ``None``). + + Order matches the cascade: N4 (row_order) > N5 (novel_reading_accepted) + > N6 (numerical_precision) > N7 > N8. N5 only counts when the LLM + judge actually returned a pass — the field can be present with a + null/None value when the judge wasn't invoked.""" + if ev.get("phase1_against_any_audited_variant") == "pass": + return None # N3-strict; no tolerance needed + if ev.get("correct_up_to_tie_order"): + return "row_order" + if ev.get("novel_reading_judgment") == "pass": + return "novel_reading_accepted" + for field, cls in _TIER_FIELDS_TO_CLASS[1:]: + if ev.get(field): + return cls + return None + + +def _remap_raw(raw: dict) -> tuple[bool, str]: + """Mutate ``raw`` in-place; return ``(changed, reason)``.""" + ev = raw.get("evaluation") or {} + fc = raw.setdefault("failure_classification", {}) + primary = fc.get("primary") + secondary = list(fc.get("secondary") or []) + reasons: list[str] = [] + + # Step 1: N3 pass → no_fail. + if ev.get("phase1_against_any_audited_variant") == "pass": + if primary != "no_fail": + displaced = primary + fc["primary"] = "no_fail" + fc["agent_at_fault"] = False + if displaced and displaced != "other" and displaced not in secondary: + secondary.append(displaced) + reasons.append(f"{displaced}→no_fail; +secondary={displaced}") + else: + reasons.append(f"{displaced or ''}→no_fail") + if not fc.get("remediation_target") or fc["remediation_target"] == "other": + fc["remediation_target"] = "other" + # Also rewrite legacy grader_stability in secondary into a + # cascade-tier bucket while we're here. + new_sec: list[str] = [] + for s in secondary: + if s == "grader_stability": + tier = _cascade_tier_for(ev) or "numerical_precision" + new_sec.append(tier) + reasons.append(f"sec grader_stability→{tier}") + else: + new_sec.append(s) + fc["secondary"] = new_sec + return (bool(reasons), "; ".join(reasons)) + + # Step 2: not an N3 pass. Resolve legacy grader_stability. + tier = _cascade_tier_for(ev) + if primary == "grader_stability": + new = tier or "numerical_precision" # default for sub-epsilon float noise + fc["primary"] = new + fc["agent_at_fault"] = False + fc["remediation_target"] = "grader" + reasons.append(f"grader_stability→{new}") + elif tier and primary == "other": + # Auto-classified `other` upgrade. + fc["primary"] = tier + fc["agent_at_fault"] = False + fc["remediation_target"] = "grader" + reasons.append(f"other→{tier}") + + # Step 3: legacy grader_stability in secondary → split. + new_sec = [] + for s in secondary: + if s == "grader_stability": + sec_tier = tier or "numerical_precision" + new_sec.append(sec_tier) + reasons.append(f"sec grader_stability→{sec_tier}") + else: + new_sec.append(s) + fc["secondary"] = new_sec + + return (bool(reasons), "; ".join(reasons)) + + +def main() -> None: + annroot = paths.annotations_root() / "mini-interact" + n_seen = n_changed = 0 + print(f"Walking {annroot}") + for p in sorted(annroot.glob("*/*.submission.*.json")): + n_seen += 1 + raw = json.loads(p.read_text()) + changed, reason = _remap_raw(raw) + # Always re-validate to guarantee schema conformance. + SubmissionAnnotation.model_validate(raw) + if changed: + n_changed += 1 + p.write_text(json.dumps(raw, indent=2) + "\n") + print(f" {raw['instance_id']:40s} {reason}") + print(f"\nVisited {n_seen} submission annotations; rewrote {n_changed}.") + + +if __name__ == "__main__": + main() diff --git a/src/bird_interact_agents/eval/annotate.py b/src/bird_interact_agents/eval/annotate.py index 18ba7d67..c8643e01 100644 --- a/src/bird_interact_agents/eval/annotate.py +++ b/src/bird_interact_agents/eval/annotate.py @@ -45,6 +45,7 @@ UserSimInteraction, UserSimResponseSummary, ) +from bird_interact_agents.eval.grade_in_place import _auto_failure_class PENDING_HUMAN_REVIEW = "PENDING_HUMAN_REVIEW" @@ -131,6 +132,25 @@ def _user_sim_interaction_from_trajectory(traj: list[dict]) -> UserSimInteractio ) +def _skeleton_failure_classification(cascade: Any) -> FailureClassification: + """Auto-classify from the cascade; ``other`` for strict misses still + needing human review.""" + primary, at_fault, remediation = _auto_failure_class(cascade) + if primary == "other": + details = PENDING_HUMAN_REVIEW + else: + details = ( + "Auto-classified from cascade verdict; no human review " + "needed for no_fail / cascade-tier categories." + ) + return FailureClassification( + primary=primary, # type: ignore[arg-type] + agent_at_fault=at_fault, + remediation_target=remediation, # type: ignore[arg-type] + details=details, + ) + + def _eval_from_cascade(cascade: Any, epsilon: float = 1e-6) -> SubmissionEvaluation: """Convert a tolerant_grader CascadeVerdict into a SubmissionEvaluation persistence shape.""" @@ -209,12 +229,7 @@ def generate_submission_annotation( n_ask_user_calls=usage.get("n_ask_user_calls"), ), evaluation=ev, - failure_classification=FailureClassification( - primary="other", - agent_at_fault=False, - remediation_target="other", - details=PENDING_HUMAN_REVIEW, - ), + failure_classification=_skeleton_failure_classification(cascade), decision_point=None, user_sim_interaction=_user_sim_interaction_from_trajectory(traj), ) diff --git a/src/bird_interact_agents/eval/annotation_schema.py b/src/bird_interact_agents/eval/annotation_schema.py index 230d4677..897d5402 100644 --- a/src/bird_interact_agents/eval/annotation_schema.py +++ b/src/bird_interact_agents/eval/annotation_schema.py @@ -31,13 +31,31 @@ ] PhaseVerdict = Literal["pass", "fail", "skip"] FailurePrimary = Literal[ + "no_fail", "agent_miss", "metadata_ambiguity", "gold_audit_quality", "user_sim_under_disclosure", - "grader_stability", + "novel_reading_accepted", + "numerical_precision", + "row_order", + "trailing_whitespace", + "column_order", + "case_sensitivity", "other", ] +"""``no_fail`` ⇒ agent passed N3-strict; nothing to attribute. +``other`` ⇒ failure happened but doesn't fit any listed category. + +Cascade-tier relaxation buckets (``row_order``, ``numerical_precision``, +``trailing_whitespace``, ``column_order``, ``case_sensitivity``) each +flag exactly which grader-tolerance dimension flipped the verdict. + +``novel_reading_accepted`` ⇒ strict cascade (N1..N4, N6..N8) failed AND +``task.metadata_sufficiency.verdict == "insufficient"`` AND the LLM +judge accepted the agent's reading as a valid novel interpretation +(N5). The underlying issue is the task's metadata — the judge merely +ratified the agent's plausible reading.""" RemediationTarget = Literal["agent", "prompt", "kb", "audit", "grader", "user_sim", "gold_sidecar", "other"] SubmissionVerdict = Literal[ "correct", @@ -69,6 +87,22 @@ class MaskedTerm(BaseModel): class MetadataSufficiency(BaseModel): + """Verdict semantics (sharpened DEV-1515): + + * ``sufficient`` — published metadata (KB + column meanings + + sampled values) alone pins the answer; no user_sim help needed. + * ``ambiguous`` — published metadata licenses multiple readings, + BUT a maximally cooperative user_sim disclosing the masked + sql_snippet on a well-phrased ask would resolve them. If the + run-side sim refused / under-disclosed, the submission's + ``failure_classification.primary`` should be + ``user_sim_under_disclosure`` (not ``metadata_ambiguity``). + * ``insufficient`` — even with maximally cooperative user_sim + disclosure of the masked sql_snippet, the answer remains + underspecified (e.g. arbitrary policy thresholds / coefficients + with no semantic anchor). The companion ``evaluator_prompt`` then + describes what counts as a valid LLM-judge-accepted reading. + """ model_config = ConfigDict(extra="forbid") verdict: MetadataSufficiencyVerdict @@ -125,6 +159,53 @@ class Provenance(BaseModel): None for benchmarks that never had a per-DB layout.""" +InconsistencyResolution = Literal[ + "multi_variant", + "picked_one_variant", + "unresolved", +] + + +class InternalInconsistency(BaseModel): + """Flag set when two or more authoritative sources within the task + disagree on the SAME parameter — an agent cannot satisfy all sources + simultaneously. + + Examples + -------- + * KB description says ``LCS > 3``; masked sql_snippet says + ``lcs > 2`` for the same defined concept. + * KB definition field contradicts the KB description field. + * The user_query phrasing implies one aggregation; the masked + snippet specifies another. + + This is task-level: it describes the source landscape, not the + agent's submission. Most tasks have ``internal_inconsistency = + None`` (sources converge). + """ + model_config = ConfigDict(extra="forbid") + + sources_in_conflict: List[str] + """Citations naming each conflicting source. E.g. + ``['KB#29.description: "LCS > 3"', 'critical_ambiguity for "good + quality of life".sql_snippet: "lcs > 2"']``.""" + + description: str + """1-3 sentences explaining what each source says and why an agent + cannot satisfy them all at once. Quote the disagreeing values.""" + + audit_resolution: InconsistencyResolution + """How the audit handled the inconsistency: + + * ``multi_variant`` — audited_gold JSONL has N rows, one per + reading; the task's ``gold_variants`` mirrors them. + * ``picked_one_variant`` — audit chose one reading; the alternate + is documented here but is NOT in the audited_gold JSONL. + * ``unresolved`` — audit declared the task unanswerable; + ``audit_status`` is typically ``unrecoverable``. + """ + + class TaskAnnotation(BaseModel): """Run-independent annotation of a single benchmark task. @@ -166,6 +247,13 @@ class TaskAnnotation(BaseModel): fallback for novel readings but is not used by the deterministic grader.""" + internal_inconsistency: Optional[InternalInconsistency] = None + """Set when the task's authoritative sources disagree on the same + parameter (KB vs sql_snippet, KB.description vs KB.definition, + etc.). ``None`` for tasks where all sources converge — the common + case. When set, ``audit_resolution=multi_variant`` should also be + reflected in ``gold_variants`` carrying one entry per reading.""" + provenance: Provenance diff --git a/src/bird_interact_agents/eval/grade_in_place.py b/src/bird_interact_agents/eval/grade_in_place.py index 34ec0db1..51a0bb08 100644 --- a/src/bird_interact_agents/eval/grade_in_place.py +++ b/src/bird_interact_agents/eval/grade_in_place.py @@ -39,6 +39,35 @@ def _verdict_to_phase(b: bool) -> PhaseVerdict: return "pass" if b else "fail" +def _auto_failure_class(cascade: CascadeVerdict) -> tuple[str, bool, str]: + """Pick the (primary, agent_at_fault, remediation_target) triple + purely from the cascade. ``no_fail`` for N3 passes; each lower + cascade tier maps to its own bucket so consumers know exactly which + grader-tolerance dimension flipped the verdict. ``other`` is the + only catch-all that requires human review. + + N5 (LLM judge) is treated as its own outcome (``novel_reading_accepted``) + because the judge fires ONLY when ``metadata_sufficiency.verdict`` + is ``"insufficient"`` — so the agent is being accepted under a + valid-novel-reading exception, not under a tolerance-of-the-gold.""" + if cascade.n3_any_audited_variant: + return ("no_fail", False, "other") + if cascade.n4_tie_order: + return ("row_order", False, "grader") + if cascade.n5_llm_judge and cascade.novel_reading_judgment == "pass": + # Strict cascade missed; the judge accepted a novel reading + # because the metadata couldn't pin a single answer. + return ("novel_reading_accepted", False, "kb") + if cascade.n6_numeric_epsilon: + return ("numerical_precision", False, "grader") + if cascade.n7_trailing_whitespace: + return ("trailing_whitespace", False, "grader") + if cascade.n8_column_order: + return ("column_order", False, "grader") + # Genuine strict miss — let the human classify. + return ("other", True, "other") + + def _build_submission_annotation( *, task_annotation: TaskAnnotation, @@ -69,6 +98,8 @@ def _build_submission_annotation( else: verdict_label = "invalid" + auto_primary, auto_at_fault, auto_remediation = _auto_failure_class(cascade) + ev = SubmissionEvaluation( phase1_against_original_gold=_verdict_to_phase(cascade.n1_original_gold), phase1_against_audited_primary=_verdict_to_phase(cascade.n2_audited_primary), @@ -111,10 +142,16 @@ def _build_submission_annotation( ), evaluation=ev, failure_classification=FailureClassification( - primary="other", - agent_at_fault=not cascade.n3_any_audited_variant, - remediation_target="other", - details="auto-generated; human review pending", + primary=auto_primary, # type: ignore[arg-type] + agent_at_fault=auto_at_fault, + remediation_target=auto_remediation, # type: ignore[arg-type] + details=( + "Auto-classified from cascade verdict; no human review " + "needed for no_fail / cascade-tier categories." + if auto_primary != "other" + else "Strict miss across all cascade tiers; human " + "review pending — pick the specific failure class." + ), ), decision_point=None, user_sim_interaction=( diff --git a/src/bird_interact_agents/eval/regrade.py b/src/bird_interact_agents/eval/regrade.py index 8a434dee..2e2c4fe5 100644 --- a/src/bird_interact_agents/eval/regrade.py +++ b/src/bird_interact_agents/eval/regrade.py @@ -112,10 +112,10 @@ def regrade_run( # Build a fresh SubmissionAnnotation from the cascade. from bird_interact_agents.eval.annotate import ( _eval_from_cascade, + _skeleton_failure_classification, _user_sim_interaction_from_trajectory, ) from bird_interact_agents.eval.annotation_schema import ( - FailureClassification, SubmissionMetadata, ) usage = attempt_data.get("usage", {}) or {} @@ -139,11 +139,7 @@ def regrade_run( n_ask_user_calls=usage.get("n_ask_user_calls"), ), evaluation=_eval_from_cascade(cascade), - failure_classification=FailureClassification( - primary="other", - agent_at_fault=not cascade.n3_any_audited_variant, - remediation_target="other", - ), + failure_classification=_skeleton_failure_classification(cascade), user_sim_interaction=_user_sim_interaction_from_trajectory( list(attempt_data.get("trajectory", []) or []), ), diff --git a/tests/test_livesqlbench_audited_gold.py b/tests/test_livesqlbench_audited_gold.py index 4da27577..65906d8e 100644 --- a/tests/test_livesqlbench_audited_gold.py +++ b/tests/test_livesqlbench_audited_gold.py @@ -113,7 +113,19 @@ def _iter_audit_rows() -> Iterator[dict]: def _load_audit_rows() -> dict[str, dict]: - return {r["instance_id"]: r for r in _iter_audit_rows()} + """Return PRIMARY audit rows keyed by instance_id. + + Multi-variant pattern (DEV-1515): a task may carry one primary row plus + N non-primary alternates sharing the same instance_id. Tests that pin a + specific reading (e.g. museum_7 edited, museum_9 clean) should reach the + primary; iterating tests (citation resolvability, status consistency) + should use ``_iter_audit_rows`` to see every variant. + """ + out: dict[str, dict] = {} + for r in _iter_audit_rows(): + if r.get("primary", True): + out[r["instance_id"]] = r + return out def _livesqlbench_data_or_skip() -> Path: @@ -334,15 +346,29 @@ def test_audit_rows_audited_at_is_iso8601(): ) -def test_no_duplicate_instance_ids(): - """`latest-wins` is the dedup contract for the overlay; the file MUST - not ship with duplicates because there's no canonical order.""" - seen = [] +def test_no_duplicate_instance_id_variant_pairs(): + """The dedup contract is on the (instance_id, variant_id) pair, not on + instance_id alone — DEV-1515 multi-variant audits ship N rows per task + (one primary + alternates). Also: each instance_id MUST have exactly one + primary row.""" + seen_pairs: list[tuple[str, str]] = [] + primaries_per_iid: dict[str, int] = {} for row in _iter_audit_rows(): - seen.append(row["instance_id"]) - assert len(seen) == len(set(seen)), ( - f"duplicate instance_ids in audit file: " - f"{[i for i in seen if seen.count(i) > 1]}" + iid = row["instance_id"] + vid = row.get("variant_id", "primary") + seen_pairs.append((iid, vid)) + if row.get("primary", True): + primaries_per_iid[iid] = primaries_per_iid.get(iid, 0) + 1 + dupes = [p for p in seen_pairs if seen_pairs.count(p) > 1] + assert not dupes, ( + f"duplicate (instance_id, variant_id) pairs in audit file: {dupes}" + ) + over_primaries = { + iid: n for iid, n in primaries_per_iid.items() if n != 1 + } + assert not over_primaries, ( + f"each instance_id must have exactly one primary row; " + f"got counts: {over_primaries}" ) diff --git a/tests/test_regrade_cli.py b/tests/test_regrade_cli.py new file mode 100644 index 00000000..2a84f868 --- /dev/null +++ b/tests/test_regrade_cli.py @@ -0,0 +1,261 @@ +"""DEV-1515: offline re-grade CLI for already-completed runs. + +`python -m bird_interact_agents.eval.regrade --run-id + [--instance-ids ...] [--benchmark ...] [--force-llm-judge]` + +Contract: +* Walks `/cloud//rows//attempt-1.json` for each + instance. +* Re-runs `grade_submission` with the locally-loaded LLM-judge cache + at `/cloud//llm_judge_cache.json`. +* OVERWRITES `/annotations///.submission..json` + (this is the explicit re-grade path; distinct from `fetch`'s + no-overwrite merge). +* Writes a fresh `/cloud//eval_regraded.json` — + the historical `eval.json` is NOT mutated. +* `--instance-ids` filters which rows get re-graded. +* `--force-llm-judge` invalidates cache entries for affected rows. +""" +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + + +def _write_attempt(run_dir: Path, instance_id: str, *, submitted_sql: str = "S"): + d = run_dir / "rows" / instance_id + d.mkdir(parents=True, exist_ok=True) + (d / "attempt-1.json").write_text(json.dumps({ + "instance_id": instance_id, + "selected_database": "alien", + "submitted_sql": submitted_sql, + "trajectory": [], + "usage": {"cost_usd_agent": 0.0, "cost_usd_user_sim": 0.0, + "n_agent_turns": 0, "n_ask_user_calls": 0}, + "duration_s": 0.0, + "predicted_row_count": 0, + "sol_sql": ["SELECT gold"], + "original_sol_sql": ["SELECT gold"], + })) + + +def test_regrade_walks_run_artefacts(tmp_path, monkeypatch): + from bird_interact_agents import paths as paths_mod + monkeypatch.setattr(paths_mod, "main_checkout_root", lambda: tmp_path) + + run_dir = tmp_path / "results" / "cloud" / "r1" + for inst in ("alien_1", "alien_2"): + _write_attempt(run_dir, inst) + + from bird_interact_agents.eval.regrade import regrade_run + + class StubGrader: + def __call__(self, **kw): + from bird_interact_agents.eval.tolerant_grader import CascadeVerdict + return CascadeVerdict( + n1_original_gold=True, n2_audited_primary=True, + n3_any_audited_variant=True, n4_tie_order=True, + n5_llm_judge=True, n6_numeric_epsilon=True, + n7_trailing_whitespace=True, n8_column_order=True, + matched_variant_id="primary", + novel_reading_judgment=None, + variant_matches=[], rowset_relations=[], + ) + + report = regrade_run( + run_id="r1", + benchmark="mini-interact", + run_dir=run_dir, + instance_ids=None, + force_llm_judge=False, + grader=StubGrader(), + repo_root=tmp_path, + ) + assert report.regraded == 2 + + +def test_regrade_overwrites_existing_submission_annotation(tmp_path, monkeypatch): + from bird_interact_agents import paths as paths_mod + monkeypatch.setattr(paths_mod, "main_checkout_root", lambda: tmp_path) + + # Pre-existing submission annotation at the destination. + dest_dir = tmp_path / "annotations" / "mini-interact" / "alien" + dest_dir.mkdir(parents=True) + dest = dest_dir / "alien_1.submission.r1.json" + dest.write_text('{"annotated_by": "stale", "kind": "submission_annotation"}') + + run_dir = tmp_path / "results" / "cloud" / "r1" + _write_attempt(run_dir, "alien_1") + + from bird_interact_agents.eval.regrade import regrade_run + + class StubGrader: + def __call__(self, **kw): + from bird_interact_agents.eval.tolerant_grader import CascadeVerdict + return CascadeVerdict( + n1_original_gold=True, n2_audited_primary=True, + n3_any_audited_variant=True, n4_tie_order=True, + n5_llm_judge=True, n6_numeric_epsilon=True, + n7_trailing_whitespace=True, n8_column_order=True, + matched_variant_id="primary", + novel_reading_judgment=None, + variant_matches=[], rowset_relations=[], + ) + + regrade_run( + run_id="r1", benchmark="mini-interact", run_dir=run_dir, + instance_ids=None, force_llm_judge=False, + grader=StubGrader(), repo_root=tmp_path, + ) + refreshed = json.loads(dest.read_text()) + assert refreshed["annotated_by"] != "stale" + + +def test_regrade_respects_instance_id_filter(tmp_path, monkeypatch): + from bird_interact_agents import paths as paths_mod + monkeypatch.setattr(paths_mod, "main_checkout_root", lambda: tmp_path) + + run_dir = tmp_path / "results" / "cloud" / "r1" + for inst in ("alien_1", "alien_2", "alien_3"): + _write_attempt(run_dir, inst) + + seen: list[str] = [] + + class StubGrader: + def __call__(self, **kw): + seen.append(kw.get("submitted_sql", "?")) + from bird_interact_agents.eval.tolerant_grader import CascadeVerdict + return CascadeVerdict( + n1_original_gold=True, n2_audited_primary=True, + n3_any_audited_variant=True, n4_tie_order=True, + n5_llm_judge=True, n6_numeric_epsilon=True, + n7_trailing_whitespace=True, n8_column_order=True, + matched_variant_id="primary", + novel_reading_judgment=None, + variant_matches=[], rowset_relations=[], + ) + + from bird_interact_agents.eval.regrade import regrade_run + + regrade_run( + run_id="r1", benchmark="mini-interact", run_dir=run_dir, + instance_ids=["alien_2"], force_llm_judge=False, + grader=StubGrader(), repo_root=tmp_path, + ) + assert len(seen) == 1 + + +def test_regrade_writes_eval_regraded_not_eval_json(tmp_path, monkeypatch): + from bird_interact_agents import paths as paths_mod + monkeypatch.setattr(paths_mod, "main_checkout_root", lambda: tmp_path) + + run_dir = tmp_path / "results" / "cloud" / "r1" + _write_attempt(run_dir, "alien_1") + # Historical eval.json — must NOT be overwritten by regrade. + eval_json = run_dir / "eval.json" + eval_json.write_text('{"phase1_count": 999}') + + class StubGrader: + def __call__(self, **kw): + from bird_interact_agents.eval.tolerant_grader import CascadeVerdict + return CascadeVerdict( + n1_original_gold=True, n2_audited_primary=True, + n3_any_audited_variant=True, n4_tie_order=True, + n5_llm_judge=True, n6_numeric_epsilon=True, + n7_trailing_whitespace=True, n8_column_order=True, + matched_variant_id="primary", + novel_reading_judgment=None, + variant_matches=[], rowset_relations=[], + ) + + from bird_interact_agents.eval.regrade import regrade_run + + regrade_run( + run_id="r1", benchmark="mini-interact", run_dir=run_dir, + instance_ids=None, force_llm_judge=False, + grader=StubGrader(), repo_root=tmp_path, + ) + + eval_regraded = run_dir / "eval_regraded.json" + assert eval_regraded.exists() + # Historical preserved. + assert json.loads(eval_json.read_text())["phase1_count"] == 999 + + +def test_regrade_force_llm_judge_clears_cache_entries(tmp_path, monkeypatch): + """`--force-llm-judge` MUST drop matching keys from + `/cloud//llm_judge_cache.json` before re-grading. + + Cache entries embed `instance_id` so the clearer can filter. The + contract: after `clear_llm_judge_cache(..., instance_ids=["alien_1"])` + NO cached entry whose key/value references `alien_1` remains; entries + for OTHER instances are preserved. + """ + from bird_interact_agents import paths as paths_mod + monkeypatch.setattr(paths_mod, "main_checkout_root", lambda: tmp_path) + + run_dir = tmp_path / "results" / "cloud" / "r1" + _write_attempt(run_dir, "alien_1") + + cache_path = run_dir / "llm_judge_cache.json" + cache_path.parent.mkdir(parents=True, exist_ok=True) + cache_path.write_text(json.dumps({ + "k_for_alien_1": {"instance_id": "alien_1", "verdict": True}, + "k_for_alien_2": {"instance_id": "alien_2", "verdict": False}, + })) + + from bird_interact_agents.eval.regrade import clear_llm_judge_cache + + clear_llm_judge_cache( + cache_path=cache_path, instance_ids=["alien_1"], + ) + remaining = json.loads(cache_path.read_text()) + assert "k_for_alien_1" not in remaining, ( + "force_llm_judge must drop the alien_1 cache entry" + ) + assert "k_for_alien_2" in remaining, ( + "entries for other instances must be preserved" + ) + + +def test_regrade_run_force_llm_judge_reinvokes_judge(tmp_path, monkeypatch): + """End-to-end: `regrade_run(..., force_llm_judge=True)` calls the + grader exactly once and the cache is empty afterward for the + filtered instances.""" + from bird_interact_agents import paths as paths_mod + monkeypatch.setattr(paths_mod, "main_checkout_root", lambda: tmp_path) + + run_dir = tmp_path / "results" / "cloud" / "r1" + _write_attempt(run_dir, "alien_1") + cache_path = run_dir / "llm_judge_cache.json" + cache_path.write_text(json.dumps({ + "k_for_alien_1": {"instance_id": "alien_1", "verdict": True}, + })) + + calls: list[dict] = [] + + class StubGrader: + def __call__(self, **kw): + calls.append(kw) + from bird_interact_agents.eval.tolerant_grader import CascadeVerdict + return CascadeVerdict( + n1_original_gold=False, n2_audited_primary=False, + n3_any_audited_variant=False, n4_tie_order=False, + n5_llm_judge=False, n6_numeric_epsilon=False, + n7_trailing_whitespace=False, n8_column_order=False, + matched_variant_id=None, novel_reading_judgment=None, + variant_matches=[], rowset_relations=[], + ) + + from bird_interact_agents.eval.regrade import regrade_run + + regrade_run( + run_id="r1", benchmark="mini-interact", run_dir=run_dir, + instance_ids=["alien_1"], force_llm_judge=True, + grader=StubGrader(), repo_root=tmp_path, + ) + assert len(calls) == 1 + remaining = json.loads(cache_path.read_text()) + assert "k_for_alien_1" not in remaining From 5f0d374d97ccee408845cbbbd22a9cc516cab58d Mon Sep 17 00:00:00 2001 From: Egor Kraev Date: Mon, 1 Jun 2026 12:31:24 +0200 Subject: [PATCH 08/29] =?UTF-8?q?DEV-1515=20review=20fixes=20=E2=80=94=20G?= =?UTF-8?q?roups=201-4=20from=20/process-reviews=20triage=20on=20PR=20#16?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Group 1 — Regrade CLI is operational + multi-statement SQL preserves TEMP state (CodeRabbit r3330838210/r3330838211 + Codex): * eval/tolerant_grader.py: _multi_sql_execute opens ONE shared sqlite connection when conn=None and len(sqls)>1, so CREATE TEMP setup statements share state with the final SELECT (closed in a finally so it doesn't leak across grade_submission calls). New regression test in tests/test_tolerant_grader_multi_sql_conn.py exercises the path with conn=None + a real two-statement gold list against a fresh sqlite DB. * eval/regrade.py: stop passing db_path=Path("/dev/null"); resolve the real per-DB sqlite via paths.benchmark_data_root(args.benchmark)// .sqlite using the attempt JSON's database key. Wrap the grader call in try/except so a sqlite/SQL error skips the instance rather than crashing the whole CLI. Group 2 — Cascade verdict-label drift (Codex): * eval/grade_in_place.py: extract verdict_label_from_cascade() as the single source of truth for cascade → SubmissionEvaluation.verdict. Both _build_submission_annotation (inline grader) and eval/annotate._eval_from_cascade (skeleton + regrade CLI) now go through it. Without this, N4/N5/N6/N7/N8 cascade-tier passes landed in annotate-side annotations with verdict="invalid" while the inline grader emitted "valid_interpretation". New tests/test_verdict_label_shared.py pins each cascade-tier mapping AND asserts the two persistence paths produce the same label. Group 3 — Cloud eval.json gains the cascading_phase1 block (Codex): * cloud/driver.py: after merge_submission_annotations, walk the downloaded run's //submission_annotation.json files and call emit_cascading_eval_json to rewrite eval.json with the cascading_phase1 block + back-compat phase1_count/phase1_rate aliases. Extracted to _emit_cascading_phase1_on_fetch so it's unit-testable without round-tripping through GCS download + collation. Older runs pre-dating the DEV-1515 worker hook (no per-row annotations) skip cleanly. New tests/cloud/test_fetch_cascading_phase1.py pins the happy path + both no-op edges. Group 4 — Markdown lint + dead code + fresh-checkout build (Codex + CodeRabbit r3330838206/r3330838207/r3330838208 + summary-nitpick): * analyses/raw/households_10.md + museum_2.md + museum_4.md: add language tags to fenced code blocks (text / yaml) to clear MD040. * tests/cloud/test_collation.py: delete dead _read_dual_cols helper (queried phase1_passed_audited / phase1_passed_original columns that were removed in the DEV-1515 schema overhaul; would fail if called). * paths.py: annotations_root() and audited_gold_root() now mkdir(parents=True, exist_ok=True) so a fresh checkout (where both dirs are gitignored) doesn't fail cloud/image.build_and_push's BuildKit --build-context resolve before the data is populated. Full non-integration suite: 1811 passed, 95 skipped, 50 deselected (was 1800; net +11 from the three new test files). Co-Authored-By: Claude Opus 4.7 (1M context) --- analyses/raw/households_10.md | 6 +- analyses/raw/museum_2.md | 2 +- analyses/raw/museum_4.md | 4 +- ...eum.py => dev1515_convert_livesqlbench.py} | 0 src/bird_interact_agents/cloud/driver.py | 36 +++++ src/bird_interact_agents/eval/annotate.py | 15 +- .../eval/grade_in_place.py | 42 +++-- src/bird_interact_agents/eval/regrade.py | 48 ++++-- .../eval/tolerant_grader.py | 37 ++++- src/bird_interact_agents/paths.py | 19 ++- tests/cloud/test_collation.py | 15 -- tests/cloud/test_fetch_cascading_phase1.py | 144 ++++++++++++++++++ tests/test_tolerant_grader_multi_sql_conn.py | 120 +++++++++++++++ tests/test_verdict_label_shared.py | 91 +++++++++++ 14 files changed, 524 insertions(+), 55 deletions(-) rename scripts/{dev1515_convert_livesqlbench_museum.py => dev1515_convert_livesqlbench.py} (100%) create mode 100644 tests/cloud/test_fetch_cascading_phase1.py create mode 100644 tests/test_tolerant_grader_multi_sql_conn.py create mode 100644 tests/test_verdict_label_shared.py diff --git a/analyses/raw/households_10.md b/analyses/raw/households_10.md index 29c1bc5b..7104802d 100644 --- a/analyses/raw/households_10.md +++ b/analyses/raw/households_10.md @@ -38,7 +38,7 @@ Prompt asks for "home IDs" but original gold returns a region. Output-shape mism ## What KBs 6/7/10 actually say -``` +```text KB 6 Dwelling Type: "Values based on schema include 'Brickwork house', 'Apartment', 'Condominium', etc." KB 7 Cable TV Status: "Values indicating availability, based on schema, are 'avail', 'available', and 'yes'." KB 10 Vehicle Year Range:"Text ranges like '1995 to 1999', '2005 to 2009', or '2010 to 2013'." @@ -49,13 +49,13 @@ KBs 6, 10 hedge (`"include … etc."`, `"like"`). KB 7 does not. None mention `' ## Slayer model exposes the noise `slayer_models_otf/households/models/households/amenities.yaml` — `cablestatus.sampled_values` (123 distinct values; structured top-50): -``` +```text Available, Not available, available, AVAILABLE, yes, Y, not available, Yes, avail, have, NOT AVAILABLE, unavailable, not avail, N, no, No, dont have, Y, YEs, YES, Avail, y, ... ``` `properties.yaml` — `dwelling_specs__Dwelling_Class.sampled_values` (116 distinct): -``` +```text Brickwork house, Apartment, BRICKWORK HOUSE, Brickwork House, brickwork house, brick house, Brickwork house , apartment, Brickwork house, APARTMENT, Shack (with floor finish), apt, Apt, Unfinished brickwork, BRICK HOUSE, ... diff --git a/analyses/raw/museum_2.md b/analyses/raw/museum_2.md index d2a69fad..7d7b3cc4 100644 --- a/analyses/raw/museum_2.md +++ b/analyses/raw/museum_2.md @@ -35,7 +35,7 @@ Did the agent inspect SensitivityData via slayer? **Yes** — at `trajectory[14] **Critical finding** — the slayer-encoded model has the wrong answer **pre-baked**: `slayer_models_otf_livesqlbench/museum/models/museum/sensitivitydata.yaml:311-326`: -``` +```yaml erf: formula: (env_sens_weight + light_sens_weight + temp_sens_weight + humid_sens_weight) / 4.0 description: "Computed as the arithmetic mean of the four encoded sensitivity weights (env, light, temp, humid)…" diff --git a/analyses/raw/museum_4.md b/analyses/raw/museum_4.md index 8ae1c10d..a82b7975 100644 --- a/analyses/raw/museum_4.md +++ b/analyses/raw/museum_4.md @@ -13,7 +13,7 @@ CBE in agent: `SUM(cpi) / COUNT²` — uniform 1/N reading of "BudgetRatio". ## Was CBE a KB entry? Yes. `museum_kb.jsonl` id=9: -``` +```text "CBE = \frac{\sum_{i \in artifacts} (CPI_i \times BudgetRatio_i)}{|artifacts|}, where BudgetRatio is the proportion of total conservation budget allocated to each artifact" ``` @@ -22,7 +22,7 @@ Agent retrieved at `trajectory[8 → 9]` via `get_knowledge_definition("Conserva ## Agent's reasoning ThinkingBlocks encrypted (`thinking=''`), but `create_model` description at `trajectory[33]`: -``` +```text "CBE = sum(CPI*1/N)/N = sum(CPI)/N^2 assuming uniform BudgetRatio" ``` diff --git a/scripts/dev1515_convert_livesqlbench_museum.py b/scripts/dev1515_convert_livesqlbench.py similarity index 100% rename from scripts/dev1515_convert_livesqlbench_museum.py rename to scripts/dev1515_convert_livesqlbench.py diff --git a/src/bird_interact_agents/cloud/driver.py b/src/bird_interact_agents/cloud/driver.py index 05a4f876..13ccc1fd 100644 --- a/src/bird_interact_agents/cloud/driver.py +++ b/src/bird_interact_agents/cloud/driver.py @@ -16,6 +16,7 @@ from bird_interact_agents.cloud import benchmark_data, cluster, config, gcs, image, prereqs from bird_interact_agents.cloud import collation as _collation from bird_interact_agents.cloud import post_run_merge as _post_run_merge +from bird_interact_agents.eval import cascading_report as _cascading_report # Imported by NAME (not via the `gcs` module attr) so tests that mock # `driver.gcs` still get the real pure mapping — only the I/O helpers # (`gcs.upload_dir_prefix` etc.) need to be mockable. @@ -716,9 +717,44 @@ def fetch(run_id: str) -> dict: main_checkout_root=paths.main_checkout_root(), ) metrics["annotation_merge_report"] = annotation_merge.model_dump() + + metrics = _emit_cascading_phase1_on_fetch(dest=dest, metrics=metrics) return metrics +def _emit_cascading_phase1_on_fetch(*, dest: Path, metrics: dict) -> dict: + """Aggregate per-row submission_annotation.json files into the + ``cascading_phase1`` block on the fetched run's eval.json. + + Extracted so the post-fetch behaviour can be exercised in unit tests + without round-tripping through GCS download + collation. Older runs + (pre-DEV-1515 worker code) won't have per-row annotation files; skip + rather than fail. + """ + rows_dir = dest / "rows" + eval_path = dest / "eval.json" + if not rows_dir.exists() or not eval_path.exists(): + return metrics + has_per_row_anns = any( + (row_dir / "submission_annotation.json").exists() + for row_dir in rows_dir.iterdir() if row_dir.is_dir() + ) + if not has_per_row_anns: + return metrics + try: + return _cascading_report.emit_cascading_eval_json( + rows_dir=rows_dir, + out_path=eval_path, + base_metrics=metrics, + ) + except FileNotFoundError as exc: + # Aggregator is strict — a missing per-row file raises so we + # never silently under-count. Surface it as a side-channel entry + # and leave the in-memory metrics writeable. + metrics["cascading_phase1_error"] = str(exc) + return metrics + + def kill(run_id: str) -> None: cache = yaml_cache_dir() yaml_path = cache / f"{run_id}.yaml" diff --git a/src/bird_interact_agents/eval/annotate.py b/src/bird_interact_agents/eval/annotate.py index c8643e01..23801374 100644 --- a/src/bird_interact_agents/eval/annotate.py +++ b/src/bird_interact_agents/eval/annotate.py @@ -153,7 +153,16 @@ def _skeleton_failure_classification(cascade: Any) -> FailureClassification: def _eval_from_cascade(cascade: Any, epsilon: float = 1e-6) -> SubmissionEvaluation: """Convert a tolerant_grader CascadeVerdict into a - SubmissionEvaluation persistence shape.""" + SubmissionEvaluation persistence shape. Uses + :func:`grade_in_place.verdict_label_from_cascade` so this helper and + the cloud/local-runner builder cannot drift apart — without the + shared mapping, N4/N5/N6/N7/N8 cascade-tier passes would land here + with ``verdict="invalid"`` while the inline grader's annotations + carried ``verdict="valid_interpretation"``.""" + from bird_interact_agents.eval.grade_in_place import ( + verdict_label_from_cascade, + ) + return SubmissionEvaluation( phase1_against_original_gold="pass" if cascade.n1_original_gold else "fail", phase1_against_audited_primary="pass" if cascade.n2_audited_primary else "fail", @@ -167,9 +176,7 @@ def _eval_from_cascade(cascade: Any, epsilon: float = 1e-6) -> SubmissionEvaluat correct_under_trailing_whitespace=cascade.n7_trailing_whitespace, correct_under_column_order=cascade.n8_column_order, numeric_epsilon=epsilon, - verdict=( - "correct" if cascade.n3_any_audited_variant else "invalid" - ), + verdict=verdict_label_from_cascade(cascade), # type: ignore[arg-type] matched_variant_id=cascade.matched_variant_id, rationale="", ) diff --git a/src/bird_interact_agents/eval/grade_in_place.py b/src/bird_interact_agents/eval/grade_in_place.py index 51a0bb08..ef3536dd 100644 --- a/src/bird_interact_agents/eval/grade_in_place.py +++ b/src/bird_interact_agents/eval/grade_in_place.py @@ -39,6 +39,35 @@ def _verdict_to_phase(b: bool) -> PhaseVerdict: return "pass" if b else "fail" +def verdict_label_from_cascade(cascade: CascadeVerdict) -> str: + """Map a cascade verdict → the ``SubmissionEvaluation.verdict`` label. + + Shared by ``_build_submission_annotation`` (cloud + local runners) and + ``annotate._eval_from_cascade`` (skeleton CLI + regrade CLI). Keep + them in sync — every cascade tier that flips the headline verdict + should appear here: + + * ``n3_any_audited_variant`` → ``"correct"`` (strict set-equal pass) + * ``n4_tie_order`` / ``n5_llm_judge`` / ``n6_numeric_epsilon`` / + ``n7_trailing_whitespace`` / ``n8_column_order`` → + ``"valid_interpretation"`` (cascade-tier acceptance — the row is + not strictly identical to the gold but matches under a named + tolerance / under the LLM judge for an ``insufficient`` task) + * otherwise → ``"invalid"`` + """ + if cascade.n3_any_audited_variant: + return "correct" + if ( + cascade.n4_tie_order + or cascade.n5_llm_judge + or cascade.n6_numeric_epsilon + or cascade.n7_trailing_whitespace + or cascade.n8_column_order + ): + return "valid_interpretation" + return "invalid" + + def _auto_failure_class(cascade: CascadeVerdict) -> tuple[str, bool, str]: """Pick the (primary, agent_at_fault, remediation_target) triple purely from the cascade. ``no_fail`` for N3 passes; each lower @@ -85,18 +114,7 @@ def _build_submission_annotation( epsilon: float = 1e-6, ) -> SubmissionAnnotation: """Map the in-memory CascadeVerdict → on-disk SubmissionAnnotation.""" - if cascade.n3_any_audited_variant: - verdict_label = "correct" - elif cascade.n5_llm_judge or cascade.n4_tie_order: - verdict_label = "valid_interpretation" - elif ( - cascade.n6_numeric_epsilon - or cascade.n7_trailing_whitespace - or cascade.n8_column_order - ): - verdict_label = "valid_interpretation" - else: - verdict_label = "invalid" + verdict_label = verdict_label_from_cascade(cascade) auto_primary, auto_at_fault, auto_remediation = _auto_failure_class(cascade) diff --git a/src/bird_interact_agents/eval/regrade.py b/src/bird_interact_agents/eval/regrade.py index 2e2c4fe5..21e6406b 100644 --- a/src/bird_interact_agents/eval/regrade.py +++ b/src/bird_interact_agents/eval/regrade.py @@ -103,12 +103,28 @@ def regrade_run( continue attempt_data = json.loads(attempt.read_text()) submitted_sql = attempt_data.get("submitted_sql", "") - selected_database = attempt_data.get("selected_database", "") - cascade = grader( - instance_id=instance_id, - submitted_sql=submitted_sql, - task_row=attempt_data, + # The cloud worker writes the per-DB token to ``database``; + # ``selected_database`` is only present on the source data row, + # so fall back if the attempt file is missing it. + selected_database = ( + attempt_data.get("selected_database") + or attempt_data.get("database") + or "" ) + attempt_data["selected_database"] = selected_database + try: + cascade = grader( + instance_id=instance_id, + submitted_sql=submitted_sql, + task_row=attempt_data, + ) + except Exception as exc: # noqa: BLE001 — operational CLI + print( + f" skip {instance_id}: grader raised " + f"{type(exc).__name__}: {exc}" + ) + report.skipped += 1 + continue # Build a fresh SubmissionAnnotation from the cascade. from bird_interact_agents.eval.annotate import ( _eval_from_cascade, @@ -196,15 +212,27 @@ def main(argv: Optional[List[str]] = None) -> int: def _grader(*, instance_id: str, submitted_sql: str, task_row: dict, **_kw): # Minimal end-to-end wiring — production callers pre-build the # implicit annotation + audited gold rows themselves. - from bird_interact_agents.eval.implicit_annotation import ( - implicit_task_annotation, - ) from bird_interact_agents.cloud.ray_app import ( _load_audited_gold_rows_for, _load_task_annotation_or_implicit, ) + selected_database = task_row.get("selected_database", "") + if not selected_database: + raise ValueError( + f"{instance_id}: attempt JSON missing selected_database / " + f"database key — cannot resolve sqlite path." + ) + # Real per-DB sqlite for this benchmark. ``benchmark_data_root`` + # accepts canonical names ("mini_interact"/"livesqlbench") and + # the hyphenated CLI alias ("mini-interact"); the registry rejects + # unknown tokens, so a typo'd ``--benchmark`` fails loudly here. + db_path = ( + paths.benchmark_data_root(args.benchmark) + / selected_database + / f"{selected_database}.sqlite" + ) ann = _load_task_annotation_or_implicit( instance_id=instance_id, - selected_database=task_row.get("selected_database", ""), + selected_database=selected_database, benchmark=args.benchmark, amb_user_query=task_row.get("amb_user_query", ""), ) @@ -218,7 +246,7 @@ def _grader(*, instance_id: str, submitted_sql: str, task_row: dict, **_kw): task_row.get("original_sol_sql") or task_row.get("sol_sql") or [], ), submitted_sql=submitted_sql, - db_path=Path("/dev/null"), + db_path=db_path, conn=None, ) diff --git a/src/bird_interact_agents/eval/tolerant_grader.py b/src/bird_interact_agents/eval/tolerant_grader.py index 2aa903c9..1f86bdf0 100644 --- a/src/bird_interact_agents/eval/tolerant_grader.py +++ b/src/bird_interact_agents/eval/tolerant_grader.py @@ -517,12 +517,37 @@ def _multi_sql_execute( """Execute a list of SQL strings and return the rows+cols of the LAST one. Mirrors BIRD-Interact's evaluator semantics: prior items in the list set up state (CREATE TEMP …); only the last returns - rows for comparison.""" - rows: Sequence[Sequence] = [] - cols: Sequence[str] = [] - for sql in sqls: - rows, cols = executor(sql, db_path=db_path, conn=conn) - return rows, cols + rows for comparison. + + Connection-scoped state (TEMP tables, PRAGMAs, attached DBs) only + survives across statements inside the SAME sqlite connection. When + the caller passes ``conn=None`` and lets the default SQLite executor + open one, the executor would otherwise open+close a fresh connection + per statement and the setup statements would silently lose their + TEMP state before the final comparison statement runs. Open one + shared connection here (closed in a finally) so the whole list runs + against the same connection. + """ + own_conn: Optional[sqlite3.Connection] = None + if conn is None and executor is default_executor and len(sqls) > 1: + try: + conn = sqlite3.connect( + f"file:{db_path}?mode=ro", uri=True, timeout=30, + ) + own_conn = conn + except sqlite3.Error: + # db_path unreachable / not a sqlite file — let the + # downstream executor call raise with the original error. + conn = None + try: + rows: Sequence[Sequence] = [] + cols: Sequence[str] = [] + for sql in sqls: + rows, cols = executor(sql, db_path=db_path, conn=conn) + return rows, cols + finally: + if own_conn is not None: + own_conn.close() def grade_submission( diff --git a/src/bird_interact_agents/paths.py b/src/bird_interact_agents/paths.py index 33901665..dcadd553 100644 --- a/src/bird_interact_agents/paths.py +++ b/src/bird_interact_agents/paths.py @@ -148,11 +148,18 @@ def audited_gold_root() -> Path: """Audited-gold SQL sidecars — committed, must live in the main checkout. Honours `BIRD_AUDITED_GOLD_ROOT` override (used by SAR-audit tests). + + Always creates the directory if missing — the dir is gitignored so a + fresh checkout doesn't have one, and downstream callers (notably + ``cloud/image.build_and_push``'s BuildKit ``--build-context``) need + at least an empty dir on disk before they can mount it. """ override = os.environ.get("BIRD_AUDITED_GOLD_ROOT") if override: return Path(override).expanduser() - return main_checkout_root() / "audited_gold" + path = main_checkout_root() / "audited_gold" + path.mkdir(parents=True, exist_ok=True) + return path def audited_gold_file(*, benchmark: str) -> Path: @@ -190,11 +197,19 @@ def annotations_root() -> Path: Honours ``BIRD_ANNOTATIONS_ROOT`` for tests / forks that mount the annotations from a parallel repo. + + Always creates the directory if missing — same posture as + ``audited_gold_root``: the dir is gitignored so a fresh checkout + doesn't have one, and ``cloud/image.build_and_push`` mounts it via + BuildKit ``--build-context``, which needs at least an empty dir on + disk before docker build can resolve it. """ override = os.environ.get("BIRD_ANNOTATIONS_ROOT") if override: return Path(override).expanduser() - return main_checkout_root() / "annotations" + path = main_checkout_root() / "annotations" + path.mkdir(parents=True, exist_ok=True) + return path def sar_audited_gold_root() -> Path: diff --git a/tests/cloud/test_collation.py b/tests/cloud/test_collation.py index 70d78dd9..ee7bde80 100644 --- a/tests/cloud/test_collation.py +++ b/tests/cloud/test_collation.py @@ -28,21 +28,6 @@ def _read_rows(db_path: Path) -> list[dict]: return rows -def _read_dual_cols(db_path: Path) -> dict[str, dict]: - conn = sqlite3.connect(str(db_path)) - conn.row_factory = sqlite3.Row - out = { - r["instance_id"]: dict(r) - for r in conn.execute( - "SELECT instance_id, phase1_passed_audited, phase1_passed_original, " - "phase1_observation_audited, phase1_observation_original " - "FROM task_results" - ) - } - conn.close() - return out - - # --------------------------------------------------------------------------- # T20 — multiple attempts per iid: latest wins in results.db; older stays on disk. # --------------------------------------------------------------------------- diff --git a/tests/cloud/test_fetch_cascading_phase1.py b/tests/cloud/test_fetch_cascading_phase1.py new file mode 100644 index 00000000..7ca1a25d --- /dev/null +++ b/tests/cloud/test_fetch_cascading_phase1.py @@ -0,0 +1,144 @@ +"""DEV-1515 review-followup: ``driver.fetch`` must aggregate the per-row +submission_annotation.json files into the ``cascading_phase1`` block on +the downloaded eval.json. Without this the cloud's eval.json carries +neither the legacy dual-eval breakdown (removed in DEV-1515) nor the new +cascade block — published results lose their headline metric. + +The actual aggregation logic is tested in +``test_cascading_report.py``; these tests pin the fetch-side wiring (no +op for legacy runs without per-row annotations; happy path rewrites +eval.json; missing annotations surfaced as a side-channel error rather +than swallowed). +""" +from __future__ import annotations + +import json +from pathlib import Path + + +def _valid_submission_annotation_dict(instance_id: str = "alien_1") -> dict: + return { + "schema_version": 1, + "kind": "submission_annotation", + "instance_id": instance_id, + "selected_database": "alien", + "task_annotation_ref": f"annotations/mini-interact/alien/{instance_id}.task.json", + "annotated_by": "auto", + "annotated_at": "2026-05-31", + "submission": { + "cloud_run_id": "r1", + "trajectory_path": f"rows/{instance_id}/attempt-1.json", + }, + "evaluation": { + "phase1_against_original_gold": "pass", + "phase1_against_audited_primary": "pass", + "phase1_against_any_audited_variant": "pass", + "verdict": "correct", + }, + "failure_classification": { + "primary": "no_fail", + "agent_at_fault": False, + "remediation_target": "other", + }, + } + + +def _seed_run_dir( + tmp_path: Path, + *, + instances: list[str], + base_metrics: dict, +) -> Path: + """Build a `/` shaped like what `fetch` finishes with: a base + eval.json from collation + per-row submission_annotation.json files + from the worker hook.""" + dest = tmp_path / "run_dest" + dest.mkdir() + (dest / "eval.json").write_text(json.dumps(base_metrics)) + for iid in instances: + d = dest / "rows" / iid + d.mkdir(parents=True) + (d / "submission_annotation.json").write_text( + json.dumps(_valid_submission_annotation_dict(iid)), + ) + return dest + + +def test_fetch_emit_writes_cascading_phase1_into_eval_json(tmp_path: Path): + """Happy path: per-row submission_annotation files exist and the + downloaded eval.json gets a ``cascading_phase1`` block + back-compat + ``phase1_count`` / ``phase1_rate`` aliases rewritten from N1.""" + from bird_interact_agents.cloud.driver import ( + _emit_cascading_phase1_on_fetch, + ) + + dest = _seed_run_dir( + tmp_path, + instances=["alien_1", "alien_2"], + base_metrics={ + "framework": "claude_sdk", + "phase1_count": 999, # stale value the helper should rewrite + "phase1_rate": 0.99, + }, + ) + new_metrics = _emit_cascading_phase1_on_fetch( + dest=dest, metrics={"framework": "claude_sdk"}, + ) + assert "cascading_phase1" in new_metrics, ( + "fetch must emit the cascading_phase1 block when per-row " + "submission_annotation.json files exist" + ) + on_disk = json.loads((dest / "eval.json").read_text()) + assert "cascading_phase1" in on_disk + # phase1_count is the rewritten alias for N1; both rows have N1 pass. + assert on_disk["cascading_phase1"]["counts"]["n1"] == 2 + assert on_disk["phase1_count"] == 2 + + +def test_fetch_emit_noop_when_no_per_row_annotations(tmp_path: Path): + """Older runs that pre-date the DEV-1515 worker code won't have + `//submission_annotation.json` files. The helper must + skip cleanly rather than fail — those runs publish eval.json + without the cascading block, which is correct since the per-row + data isn't available.""" + from bird_interact_agents.cloud.driver import ( + _emit_cascading_phase1_on_fetch, + ) + + dest = tmp_path / "run_dest" + dest.mkdir() + (dest / "eval.json").write_text(json.dumps({"framework": "claude_sdk"})) + # rows/ dir exists but contains an attempt-1.json (legacy worker) + (dest / "rows" / "alien_1").mkdir(parents=True) + (dest / "rows" / "alien_1" / "attempt-1.json").write_text("{}") + + new_metrics = _emit_cascading_phase1_on_fetch( + dest=dest, metrics={"framework": "claude_sdk"}, + ) + assert "cascading_phase1" not in new_metrics + on_disk = json.loads((dest / "eval.json").read_text()) + assert "cascading_phase1" not in on_disk + + +def test_fetch_emit_noop_when_eval_json_absent(tmp_path: Path): + """Defensive: if eval.json was never written (collation failed?) + we don't materialise one with only the cascading block — the + headline metrics would be misleading. Skip and let the caller + surface the collation failure.""" + from bird_interact_agents.cloud.driver import ( + _emit_cascading_phase1_on_fetch, + ) + + dest = tmp_path / "run_dest" + dest.mkdir() + d = dest / "rows" / "alien_1" + d.mkdir(parents=True) + (d / "submission_annotation.json").write_text( + json.dumps(_valid_submission_annotation_dict("alien_1")), + ) + + new_metrics = _emit_cascading_phase1_on_fetch( + dest=dest, metrics={"framework": "claude_sdk"}, + ) + assert "cascading_phase1" not in new_metrics + assert not (dest / "eval.json").exists() diff --git a/tests/test_tolerant_grader_multi_sql_conn.py b/tests/test_tolerant_grader_multi_sql_conn.py new file mode 100644 index 00000000..a1611892 --- /dev/null +++ b/tests/test_tolerant_grader_multi_sql_conn.py @@ -0,0 +1,120 @@ +"""DEV-1515 review-followup: a multi-statement gold list (`CREATE TEMP …` ++ final `SELECT`) must share its SQLite connection so the setup state +survives to the final comparison statement. + +Before the fix in `_multi_sql_execute`, `default_executor` opened and +closed a fresh `sqlite3.Connection` per call when `conn=None`, so the +TEMP table created by the first statement vanished before the second +statement ran, and the final `SELECT` raised +`sqlite3.OperationalError: no such table: temp_setup`. +""" +from __future__ import annotations + +import sqlite3 +from pathlib import Path + + +def _make_db(tmp_path: Path) -> Path: + db = tmp_path / "scratch.sqlite" + con = sqlite3.connect(str(db)) + try: + con.execute("CREATE TABLE base (id INTEGER, label TEXT)") + con.executemany( + "INSERT INTO base (id, label) VALUES (?, ?)", + [(1, "a"), (2, "b"), (3, "c")], + ) + con.commit() + finally: + con.close() + return db + + +def _audited_row(*, audited_sol_sql: list[str]) -> dict: + return { + "instance_id": "alien_1", + "selected_database": "alien", + "benchmark": "mini_interact", + "audit_status": "edited", + "original_sol_sql": ["SELECT id FROM base WHERE id = 1"], + "audited_sol_sql": audited_sol_sql, + "variant_id": "primary", + "primary": True, + "changes": [], + "reasoning_summary": "", + "skill_version": "audit-gold-sql/1.0", + "audited_at": "2026-05-30T00:00:00+00:00", + } + + +def _task_annotation(): + from bird_interact_agents.eval import ( + AuditedGoldRef, + GoldVariantRef, + MetadataSufficiency, + TaskAnnotation, + ) + from bird_interact_agents.eval.annotation_schema import Provenance + + return TaskAnnotation( + instance_id="alien_1", + selected_database="alien", + annotated_by="test", + annotated_at="2026-05-31", + amb_user_query="x", + metadata_sufficiency=MetadataSufficiency( + verdict="sufficient", rationale="r", + ), + gold_variants=[ + GoldVariantRef( + variant_id="primary", + interpretation="x", + primary=True, + audited_gold_ref=AuditedGoldRef( + file="audited_gold/mini_interact_audited.jsonl", + instance_id="alien_1", + variant_id="primary", + ), + ), + ], + evaluator_prompt=None, + provenance=Provenance( + task_jsonl_path="mini_interact.jsonl", + task_jsonl_instance_id="alien_1", + ), + ) + + +def test_multi_sql_gold_shares_temp_state_with_conn_none(tmp_path: Path): + """Audited gold = [CREATE TEMP setup, SELECT from temp]. The two + statements MUST run against the same SQLite connection or the SELECT + will fail with `no such table`. Caller passes ``conn=None`` so the + grader's default executor is exercised end-to-end.""" + from bird_interact_agents.eval.tolerant_grader import grade_submission + + db = _make_db(tmp_path) + audited = _audited_row( + audited_sol_sql=[ + "CREATE TEMP TABLE temp_setup AS SELECT id FROM base WHERE id < 3", + "SELECT id FROM temp_setup ORDER BY id", + ], + ) + cascade = grade_submission( + task_annotation=_task_annotation(), + audited_gold_rows=[audited], + original_sol_sql=["SELECT id FROM base WHERE id < 3 ORDER BY id"], + submitted_sql="SELECT id FROM base WHERE id IN (1, 2) ORDER BY id", + db_path=db, + conn=None, + ) + # The audited primary's two-statement list produces rows (1,) (2,); + # the submission produces the same rowset. n2/n3 must pass — they + # can only pass if the TEMP table created by the first statement + # survived to the second. + assert cascade.n2_audited_primary is True, ( + "n2 should pass — audited primary's CREATE TEMP + SELECT yields " + "{1,2}, matching the submission. If TEMP state was lost between " + "statements, the SELECT raised and this would be False." + ) + assert cascade.n3_any_audited_variant is True + # n1 (original gold) also runs against the same DB and should match. + assert cascade.n1_original_gold is True diff --git a/tests/test_verdict_label_shared.py b/tests/test_verdict_label_shared.py new file mode 100644 index 00000000..04318d17 --- /dev/null +++ b/tests/test_verdict_label_shared.py @@ -0,0 +1,91 @@ +"""DEV-1515 review-followup: cascade → SubmissionEvaluation.verdict +labelling MUST be consistent across the two persistence paths. + +Two helpers map a ``CascadeVerdict`` to a ``SubmissionEvaluation`` +fixture: ``grade_in_place._build_submission_annotation`` (the cloud ++ local inline grader) and ``annotate._eval_from_cascade`` (the +skeleton CLI + the regrade CLI). Before extracting +``verdict_label_from_cascade`` the annotate path returned +``verdict="invalid"`` for every cascade tier below N3, while the +inline grader returned ``"valid_interpretation"`` for N4/N5/N6/N7/N8. + +Pin each cascade-tier → verdict mapping AND assert both helpers emit +the same value. +""" +from __future__ import annotations + + +def _cascade(**flags): + from bird_interact_agents.eval.tolerant_grader import CascadeVerdict + + return CascadeVerdict( + n1_original_gold=flags.get("n1", False), + n2_audited_primary=flags.get("n2", False), + n3_any_audited_variant=flags.get("n3", False), + n4_tie_order=flags.get("n4", False), + n5_llm_judge=flags.get("n5", False), + n6_numeric_epsilon=flags.get("n6", False), + n7_trailing_whitespace=flags.get("n7", False), + n8_column_order=flags.get("n8", False), + matched_variant_id=flags.get("matched_variant_id"), + novel_reading_judgment=flags.get("novel"), + ) + + +def _both_verdicts(cascade) -> tuple[str, str]: + """Return (inline_verdict, annotate_verdict) for the given cascade.""" + from bird_interact_agents.eval.grade_in_place import ( + verdict_label_from_cascade, + ) + from bird_interact_agents.eval.annotate import _eval_from_cascade + + inline = verdict_label_from_cascade(cascade) + ev = _eval_from_cascade(cascade) + return inline, ev.verdict + + +def test_n3_strict_yields_correct_on_both_paths(): + inline, annotate = _both_verdicts(_cascade(n3=True)) + assert inline == "correct" + assert annotate == "correct" + + +def test_n4_only_yields_valid_interpretation_on_both_paths(): + inline, annotate = _both_verdicts(_cascade(n4=True)) + assert inline == "valid_interpretation" + assert annotate == "valid_interpretation", ( + "regression: annotate._eval_from_cascade used to return 'invalid' " + "for N4 tie-order passes — the helper must reuse " + "grade_in_place.verdict_label_from_cascade so the two persistence " + "paths agree." + ) + + +def test_n5_only_yields_valid_interpretation_on_both_paths(): + inline, annotate = _both_verdicts(_cascade(n5=True, novel="pass")) + assert inline == "valid_interpretation" + assert annotate == "valid_interpretation" + + +def test_n6_only_yields_valid_interpretation_on_both_paths(): + inline, annotate = _both_verdicts(_cascade(n6=True)) + assert inline == "valid_interpretation" + assert annotate == "valid_interpretation" + + +def test_n7_only_yields_valid_interpretation_on_both_paths(): + inline, annotate = _both_verdicts(_cascade(n7=True)) + assert inline == "valid_interpretation" + assert annotate == "valid_interpretation" + + +def test_n8_only_yields_valid_interpretation_on_both_paths(): + inline, annotate = _both_verdicts(_cascade(n8=True)) + assert inline == "valid_interpretation" + assert annotate == "valid_interpretation" + + +def test_no_tier_yields_invalid_on_both_paths(): + inline, annotate = _both_verdicts(_cascade()) + assert inline == "invalid" + assert annotate == "invalid" From 627ad1427c2d34c23607e59f8c29827fd9117ba4 Mon Sep 17 00:00:00 2001 From: Egor Kraev Date: Mon, 1 Jun 2026 13:15:34 +0200 Subject: [PATCH 09/29] DEV-1515 follow-up: N9 case-fold tier + multi-variant audit on source contradiction + pre-submit mutation guard MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three independent fixes surfaced by the 53-task post-DEV-1478 run audit (opus-agent / sonnet-sim; 15 households + 38 other tasks; 9 cases tagged agent_miss + 1 alien_6 secondary). 1. N9 cascade tier — `correct_under_case_fold` (case_sensitivity) ---------------------------------------------------------------- lca_17 returned the correct percentages but `'I/II/III/IV'` wage-level labels where the gold has `'i/ii/iii/iv'` (LOWER(TRIM(JSON_EXTRACT(...))) on the dim). The existing trailing-whitespace tier strips at the cell edge, not inside a string; column-order tier doesn't touch cell content. The `case_sensitivity` value was already reserved in `FailurePrimary` but had no comparator wired up. * eval/tolerant_grader.py: new `compare_case_fold` mirroring the `compare_trailing_whitespace` shape (per-cell `.lower()` on strings, non-strings unchanged); `n9_case_fold` appended to `_CASCADE_ORDER` and `CascadeVerdict`; `grade_submission` adds the N9 block iterating variants + `__original__` after N8. * eval/annotation_schema.py: `SubmissionEvaluation` adds `correct_under_case_fold: bool = False`. * eval/grade_in_place.py: `verdict_label_from_cascade` adds n9 to the `valid_interpretation` disjunction; `_auto_failure_class` finally wires `case_sensitivity` (was dead enum value). * eval/annotate.py: mirror n9 in `_eval_from_cascade`. * scripts/dev1515_convert_runs.py + dev1515_cascade_summary.py: serialize and label the new tier. * Tests: 4 new comparator tests, 1 orchestration "n9 lifts failing n8 to pass", 1 verdict-label parity test, 2 schema round-trip tests + extends the 2^n cascade-monotonicity loop to 2^9. 2. Multi-variant audit on source contradiction (mandatory) -------------------------------------------------------- `planets_data_2` exposed the rule's enforcement gap: KB 31 ("Retrograde Orbit") defines retrograde as strict `inclination > 90`; the `critical_ambiguity` sql_snippet for "spinning backwards" uses inclusive `inclination >= 90`. The original auditor noticed the conflict in `reasoning_summary` and collapsed to single primary `>= 90` anyway. The agent followed KB 31 verbatim, returned 0 rows, and got tagged `agent_miss` for a contradiction the audit should have surfaced as multi-variant. * _shared/audit-gold-sql.contract.md: new top-level "Multi-variant audit on source contradiction (MANDATORY)" section with the contradiction-shapes table, the mechanics, and a synthetic BAD-vs-GOOD example. Picking one and noting the loser in reasoning_summary is explicitly forbidden — there is no "labeled wins" or "KB-anchored is primary" tie-breaker. * audit-gold-sql/SKILL.md: trim the inline multi-variant rule to point back at the shared contract; add Step 3.5 "Contradiction check (MUST precede any single-variant commit)" listing the mini-interact source set to scan. * audit-gold-sql-livesqlbench/SKILL.md: parallel Step 4 in the livesqlbench procedure, listing the livesqlbench source set (KB items, column meanings, schema FKs — no labeled-ambiguity blocks). 3. Pre-submit mutation check — prompt guard ----------------------------------------- Across the 10 agent-miss cases, 5 had the same root cause: the agent applied a TRIM / LOWER / ROUND / CAST / CASE-canonicalize / output-shape choice on the final-assembly step that wasn't named in the question, in an `ask_user` reply, or in an encoded KB — silently corrupting otherwise-correct rowsets. Households_2, alien_5, alien_6 (defensive normalizations); credit_7, organ_transplant_4 (ignored a NAMED transformation in an `ask_user` reply). * claude_sdk_otf_ainteract/prompts.py: new step 6 PRE-SUBMIT MUTATION CHECK before SUBMIT. Each mutation MUST be (a) named in the user question, (b) named OR authorized in an `ask_user` reply this session, or (c) required by an encoded KB; otherwise drop it. Explicit clause for the inverse: when a reply DID name a transformation, the reply IS the authorization — apply it. * claude_sdk_otf/prompts.py: parallel step 5 for the one-shot framework (no `ask_user`, so only (a) and (c) are valid authorization sources). Cascade summary post-changes (53-instance run): agent_miss 9 → 8, no_fail 34 → 35 (planets_data_2 reclassified to no_fail since the agent's submission now matches the new `kb_strict` variant via n3_any_audited_variant). N9 row appears at the bottom with (+0) delta — existing annotations were written before the tier existed; a fresh re-grade is out of scope here. Untouched but worth noting: the on-disk `audited_gold/mini_interact_audited.jsonl` row for `planets_data_2` was split into two variants (`labeled_snippet` primary `>= 90`, `kb_strict` alternate `> 90`), and the corresponding `annotations/mini-interact/planets_data/planets_data_2.task.json` + `.submission.20260531t1343-claudes-slayer-b39bfc.json` were updated with `internal_inconsistency.audit_resolution="multi_variant"` + two `gold_variants` + `failure_classification.primary="no_fail"`. Both trees are gitignored (per the existing policy), so these data edits ride into the cloud image via the build-context, not via this commit. Test suite: 1819 passed, 95 skipped (60s), no new failures. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../skills/_shared/audit-gold-sql.contract.md | 88 +++++++++++++++ .../audit-gold-sql-livesqlbench/SKILL.md | 20 +++- .claude/skills/audit-gold-sql/SKILL.md | 54 ++++++--- scripts/dev1515_cascade_summary.py | 104 +++++------------- scripts/dev1515_convert_runs.py | 1 + .../agents/claude_sdk_otf/prompts.py | 12 +- .../claude_sdk_otf_ainteract/prompts.py | 17 ++- src/bird_interact_agents/eval/annotate.py | 1 + .../eval/annotation_schema.py | 1 + .../eval/grade_in_place.py | 6 +- .../eval/tolerant_grader.py | 29 +++++ tests/test_eval_annotate_cli.py | 3 + tests/test_eval_annotation_schema.py | 17 +++ tests/test_regrade_cli.py | 5 + tests/test_tolerant_grader_comparators.py | 38 +++++++ tests/test_tolerant_grader_orchestration.py | 38 ++++++- tests/test_verdict_label_shared.py | 7 ++ 17 files changed, 337 insertions(+), 104 deletions(-) diff --git a/.claude/skills/_shared/audit-gold-sql.contract.md b/.claude/skills/_shared/audit-gold-sql.contract.md index 8e90e5e5..5d8faa7f 100644 --- a/.claude/skills/_shared/audit-gold-sql.contract.md +++ b/.claude/skills/_shared/audit-gold-sql.contract.md @@ -92,6 +92,94 @@ Use one of: `outer_select`, `projection`, `from`, `join`, `where_predicate`, 6. **Persist** the JSONL line per the benchmark's on-disk layout (see each SKILL.md). +## Multi-variant audit on source contradiction (MANDATORY) + +If two of the task's authoritative sources DIRECTLY contradict on the +SAME parameter, the audit MUST emit **multi-variant** — one row per +defensible reading. Collapsing to a single variant with a note is +**forbidden**. The agent under test cannot satisfy mutually-exclusive +sources; forcing one reading tags genuine ambiguity as `agent_miss`. + +What counts as a contradiction: + +| Same parameter | Source A | Source B | Verdict | +| --- | --- | --- | --- | +| Boundary operator on a threshold | KB definition: "greater than 7" | labeled sql_snippet: `score >= 7` | **contradiction** (strict vs inclusive) | +| Numeric threshold value | KB: "> 3" | labeled sql_snippet: `> 2` | **contradiction** | +| Bucket inclusion | column_meaning enum: `{A,B,C}` | KB cites bucket `D` | **contradiction** | +| Aggregation choice | KB: "average of …" | labeled sql_snippet: `SUM(…)` | **contradiction** | +| KB-internal | KB.description: one cutoff | KB.definition: another | **contradiction** | + +Not a contradiction: one source silent, the other specifies; one source +operationalises in SQL what the other describes in prose; different +parameters that happen to live near each other in the metadata. + +**The recipe addendum.** Before classifying the audit as single-variant +in step 4 of the per-clause recipe, scan ALL loaded authoritative +sources (KB items, column meanings, every labeled/anchor citation +shape your benchmark exposes — see the per-benchmark SKILL.md for the +full source list) for pairwise direct contradictions. ANY contradicting +pair triggers multi-variant. No "labeled wins" tie-breaker. No "KB- +anchored is primary" tie-breaker. If you find yourself writing a +regretful note in `reasoning_summary` that names two sources and +explains why you went with one, STOP — that's the multi-variant +trigger and you skipped it. Go back and emit two rows. + +**Mechanics when emitting multi-variant.** + +- Each row shares `instance_id`; rows are distinguished by `variant_id` + (a short kebab-case slug naming the reading, e.g. `labeled_snippet`, + `kb_strict`). +- Exactly one row carries `primary: true`. The choice is arbitrary — + it only controls which row the grader's strict-N2 path targets by + default; `n3_any_audited_variant` accepts either. Document the + choice in `reasoning_summary` as bookkeeping, not as authority. +- Every row keeps `audit_status: "edited"`. +- Each row's `changes[]` carries AT LEAST one entry with + `clause_kind: "source_conflict"` pointing at the other variant: + - `original`: what THIS variant rejected (paraphrase the other + variant's reading, in parens cite the source). + - `replacement`: what THIS variant chose (paraphrase, in parens + cite the source). + - `why_unjustified`: quote the rejected source's contrary value. + - `justified_by`: the citation tokens backing THIS variant's choice. +- The downstream TaskAnnotation carries + `internal_inconsistency.audit_resolution = "multi_variant"` with + `sources_in_conflict[]` quoting both sides verbatim, and one + `gold_variants[]` entry per audit row. + +**Minimal worked example (synthetic).** A task asks "list high-score +items"; KB X.definition says "score must be greater than 7" (strict); +the labeled sql_snippet for "high score" says `score >= 7` (inclusive). +The audit emits two rows: + +```jsonl +{"instance_id": "demo_5", "variant_id": "labeled_snippet", "primary": true, + "audit_status": "edited", + "audited_sol_sql": ["… WHERE foo.score >= 7 …"], + "changes": [{"clause_kind": "source_conflict", + "original": "foo.score > 7 (KB X strict reading)", + "replacement": "foo.score >= 7 (labeled-snippet inclusive reading)", + "why_unjustified": "KB X.definition: 'score must be greater than 7' is strict; this variant rejects KB X in favor of the labeled sql_snippet's inclusive boundary.", + "justified_by": ["labeled_ambiguity:high score"]}], + "reasoning_summary": "Task is internally inconsistent on the score threshold. This variant follows the labeled sql_snippet."} +{"instance_id": "demo_5", "variant_id": "kb_strict", "primary": false, + "audit_status": "edited", + "audited_sol_sql": ["… WHERE foo.score > 7 …"], + "changes": [{"clause_kind": "source_conflict", + "original": "foo.score >= 7 (labeled-snippet inclusive reading)", + "replacement": "foo.score > 7 (KB X strict reading)", + "why_unjustified": "labeled_ambiguity:high score uses inclusive >=; this variant rejects the labeled snippet in favor of KB X.definition's explicit 'greater than' wording.", + "justified_by": ["kb:foo_kb#X"]}], + "reasoning_summary": "Task is internally inconsistent on the score threshold. This variant follows KB X's strict reading."} +``` + +(For LiveSQLBench tasks the citation token for the snippet-anchored +variant would be `external_knowledge:` or `column_meaning:…` +instead of `labeled_ambiguity:` — LiveSQLBench has no labeled-ambiguity +blocks. See the per-benchmark SKILL.md for which sources can take part +in a contradiction.) + ## Anti-patterns (call out in the audit) - **Set arithmetic on counts** with no natural-language analogue in the diff --git a/.claude/skills/audit-gold-sql-livesqlbench/SKILL.md b/.claude/skills/audit-gold-sql-livesqlbench/SKILL.md index 3047752d..ace30fac 100644 --- a/.claude/skills/audit-gold-sql-livesqlbench/SKILL.md +++ b/.claude/skills/audit-gold-sql-livesqlbench/SKILL.md @@ -99,10 +99,24 @@ substituting livesqlbench's inputs / citations / output path: join chain isn't obviously pinned by KB+column-meaning text. The `_schema.txt` is small (one CREATE TABLE per table) — grep the relevant table names + `FOREIGN KEY` lines. -4. **Classify** per shared contract (clean / edited / unrecoverable). -5. **Sanity-execute** `audited_sol_sql[0]` against `.sqlite` (or +4. **Contradiction check** (MUST precede single-variant commit). + Run the shared contract's "Multi-variant audit on source + contradiction (MANDATORY)" check. For livesqlbench, the sources + that can participate in a contradiction are: KB items + (`_kb.jsonl` — both `description` and `definition` fields), + column meanings (`_column_meaning_base.json`), and the + relationships implied by `_schema.txt` FKs. (livesqlbench has + NO labeled-ambiguity blocks; the contradiction-side citations are + always `kb:`, `external_knowledge:`, or `column_meaning:…`.) + ANY pair pinning different values for the SAME operational choice + triggers multi-variant per the shared contract's mechanics. +5. **Classify** per shared contract (clean / edited / unrecoverable, + or — if step 4 fired — multi-variant with N edited rows). +6. **Sanity-execute** `audited_sol_sql[0]` against `.sqlite` (or `_template.sqlite`). Record first row in `audited_sample_row`. -6. **Persist**: append/overwrite the row in + For multi-variant, execute EACH variant's SQL and record its own + sample row. +7. **Persist**: append/overwrite the row(s) in `audited_gold/livesqlbench_audited.jsonl`. Each row carries `benchmark: "livesqlbench"` and `skill_version: "audit-gold-sql-livesqlbench/1.0"`. diff --git a/.claude/skills/audit-gold-sql/SKILL.md b/.claude/skills/audit-gold-sql/SKILL.md index c888ed87..2bb1944e 100644 --- a/.claude/skills/audit-gold-sql/SKILL.md +++ b/.claude/skills/audit-gold-sql/SKILL.md @@ -98,24 +98,17 @@ Field rules: unauthorised). Fall back to the natural reading of `amb_user_query` using only authorised sources. `audited_sol_sql` is the natural reading; one `changes` entry documents the gap. -- When the task's authoritative sources **disagree among themselves** - on the same parameter — e.g. KB description says `LCS > 3`, masked - sql_snippet says `lcs > 2` for the same defined concept — emit - **multiple rows** to the audit JSONL, one per defensible reading, - sharing `instance_id` and distinguished by `variant_id`: - - Exactly one row carries `primary: true`. The KB-anchored reading - typically takes primary; the snippet-anchored reading is the - alternate. Document the choice in `reasoning_summary`. - - Every row keeps `audit_status: "edited"`. Each row's `changes[]` - has at least one entry with `clause_kind: "source_conflict"` - pointing at the other variant (its `original` is what THIS - variant rejected; `replacement` is what THIS variant chose; both - `justified_by` entries cite the source backing the chosen reading; - `why_unjustified` quotes the other source's contrary value). - - The downstream `TaskAnnotation` then carries - `internal_inconsistency.audit_resolution = "multi_variant"` and - one `gold_variants[]` entry per audit row (see - `annotate-task-submission` skill). +- **Source contradiction → multi-variant is mandatory** — see the + shared contract's "Multi-variant audit on source contradiction + (MANDATORY)" section. For mini-interact, the sources that can + participate in a contradiction are: KB items (`_kb.jsonl` — + both `description` and `definition` fields), column meanings + (`_column_meaning_base.json`), labeled-ambiguity sql_snippets + (`user_query_ambiguity.critical_ambiguity[]` and + `non_critical_ambiguity[]`), and knowledge-ambiguity sql_snippets + (`knowledge_ambiguity[]`). The recurring shape is `kb:` vs + `labeled_ambiguity:` disagreeing on a boundary / threshold / + bucket. Step 3.5 below is the mechanical contradiction-check pass. - `original_sol_sql` and `audited_sol_sql` are both list[str], mirroring upstream `sol_sql`'s shape. For most tasks both have length 1. - `audited_sample_row` is the first row of running `audited_sol_sql[0]` @@ -271,6 +264,31 @@ For `shop_3`: | critical_ambiguity[1] | "premium suppliers" | `LOWER(tier) IN ('gold','platinum') AND LOWER(ship_class) IN ('express','priority')` — defines the premium predicate | | knowledge_ambiguity[0] | "Premium Supplier" (deleted_knowledge=22) | Same `LOWER+IN` premium predicate | +### Step 3.5 — Contradiction check (MUST precede any single-variant commit) + +Before committing to `audit_status="clean"` or `"edited"` with a SINGLE +audit row, run the contradiction check from the shared contract's +"Multi-variant audit on source contradiction (MANDATORY)" section. +Pairwise-scan the sources you loaded in Steps 1-3: + +- every KB item touched by this task (both `description` and + `definition` fields, in case they disagree internally) +- every column meaning referenced +- every `critical_ambiguity` / `non_critical_ambiguity` sql_snippet +- every `knowledge_ambiguity` sql_snippet + +ANY pair that pins different values for the SAME operational choice +(threshold value, boundary operator, bucket inclusion, aggregation +choice, etc.) triggers multi-variant. Emit one audit row per +defensible reading per the mechanics in the shared contract; reflect +the contradiction on the TaskAnnotation via +`internal_inconsistency.audit_resolution = "multi_variant"`. + +If NO contradictions are found, proceed to Step 4 (single-variant +audit). If you wrote a regretful note in `reasoning_summary` that +names two sources and explains why you went with one, you SKIPPED +this step — go back. + ### Step 4 — Decompose the gold SQL Parse the gold SQL into clauses manually. List every: diff --git a/scripts/dev1515_cascade_summary.py b/scripts/dev1515_cascade_summary.py index b7daee05..b0658511 100644 --- a/scripts/dev1515_cascade_summary.py +++ b/scripts/dev1515_cascade_summary.py @@ -1,14 +1,14 @@ """DEV-1515: print aggregate cascade summary over the 53 annotated instances. Walks `annotations/mini-interact/*/.submission..json` for the -two May-31 runs and emits the phase1 block (counts + deltas + rates) -plus a small per-failure-class tally. +two May-31 runs and emits the phase1 monotone cascade (each tier is the +cumulative pass count up to and including that tier), plus a small +per-failure-class tally. Run after ``scripts/dev1515_convert_runs.py``. """ from __future__ import annotations -import json from collections import Counter, defaultdict from pathlib import Path @@ -29,13 +29,14 @@ # Most-stringent → most-lenient labels for the phase1 view. _PHASE1_LABELS = ( ("n1", "original gold passes"), - ("n2", "Δ + audited primary variant passes"), - ("n3", "Δ + any audited variant matches"), - ("n4", "Δ + correct up to tie order (row_order)"), - ("n5", "Δ + llm_judge accepts insufficient-only (novel_reading)"), - ("n6", "Δ + correct under numeric epsilon (numerical_precision)"), - ("n7", "Δ + correct under trailing whitespace"), - ("n8", "Δ + correct under column order"), + ("n2", "audited primary variant passes"), + ("n3", "any audited variant matches"), + ("n4", "correct up to tie order (row_order)"), + ("n5", "llm_judge accepts insufficient-only (novel_reading)"), + ("n6", "correct under numeric epsilon (numerical_precision)"), + ("n7", "correct under trailing whitespace"), + ("n8", "correct under column order"), + ("n9", "correct under case fold (case_sensitivity)"), ) @@ -61,6 +62,7 @@ def _row_cascade(p: Path) -> dict[str, bool]: "n6_numeric_epsilon": ev.correct_under_numeric_epsilon, "n7_trailing_whitespace": ev.correct_under_trailing_whitespace, "n8_column_order": ev.correct_under_column_order, + "n9_case_fold": ev.correct_under_case_fold, } return enforce_monotone_cascade(raw) @@ -74,104 +76,52 @@ def _gather_files() -> list[Path]: return out -def _is_audit_unchanged(instance_id: str, audit_rows: dict) -> bool: - row = audit_rows.get(instance_id) - if row is None: - return False - orig = [" ".join((s or "").split()) for s in (row.get("original_sol_sql") or [])] - aud = [" ".join((s or "").split()) for s in (row.get("audited_sol_sql") or [])] - return orig == aud - - -def _aggregate(paths_: list[Path], audit_rows: dict) -> dict: - """Split U (audit unchanged) vs C (audit changed) at each cascade tier.""" - counts_u = {_short(f): 0 for f in _CASCADE_ORDER} - counts_c = {_short(f): 0 for f in _CASCADE_ORDER} - n_u = n_c = 0 +def _aggregate(paths_: list[Path]) -> dict: + counts = {_short(f): 0 for f in _CASCADE_ORDER} + n = 0 per_class: Counter[str] = Counter() per_db_n1: defaultdict = defaultdict(lambda: [0, 0]) # [n1_pass, total] for p in paths_: ann = read_submission_annotation(p) verdicts = _row_cascade(p) - unchanged = _is_audit_unchanged(ann.instance_id, audit_rows) - if unchanged: - n_u += 1 - target = counts_u - else: - n_c += 1 - target = counts_c + n += 1 for f, v in verdicts.items(): if v: - target[_short(f)] += 1 + counts[_short(f)] += 1 per_class[ann.failure_classification.primary] += 1 per_db_n1[ann.selected_database][1] += 1 if verdicts["n1_original_gold"]: per_db_n1[ann.selected_database][0] += 1 return { - "n_u": n_u, - "n_c": n_c, - "counts_u": counts_u, - "counts_c": counts_c, + "n": n, + "counts": counts, "failure_classes": dict(per_class), "per_db_n1": dict(per_db_n1), } -def _load_audit_rows() -> dict: - p = paths.audited_gold_root() / "mini_interact_audited.jsonl" - rows = {} - for line in p.read_text().splitlines(): - if not line.strip(): - continue - d = json.loads(line) - rows[d["instance_id"]] = d - return rows - - def main() -> None: files = _gather_files() - audit_rows = _load_audit_rows() - block = _aggregate(files, audit_rows) - nu, nc, n = block["n_u"], block["n_c"], block["n_u"] + block["n_c"] + block = _aggregate(files) + n = block["n"] print("=" * 90) print(f"DEV-1515 cascade summary over {n} (instance, run) pairs") print(f" runs: {', '.join(RUNS)}") - print(f" U-instances (audit unchanged): {nu} " - f"C-instances (audit changed): {nc}") print("=" * 90) print() - print("phase1 cascade — U(audit-unchanged) vs C(audit-changed) split") - print(f"{'tier':52s} {'U':>6s} {'C':>6s} {'tot':>6s}") + print("phase1 monotone cascade (cumulative passes up to and including each tier)") + print(f"{'tier':56s} {'pass':>8s} {'rate':>6s} delta") print("-" * 90) prev_total = None for key, label in _PHASE1_LABELS: - u = block["counts_u"][key] - c = block["counts_c"][key] - tot = u + c - delta = "" if prev_total is None else ( - f" (+{tot - prev_total})" if tot - prev_total > 0 - else f" (+0)" - ) - u_str = f"{u}/{nu}" - c_str = f"{c}/{nc}" - tot_str = f"{tot}/{n}" + tot = block["counts"][key] rate = (tot / n * 100.0) if n else 0.0 - print(f" {label:50s} {u_str:>6s} {c_str:>6s} " - f"{tot_str:>6s} {rate:>5.1f}%{delta}") + delta = "" if prev_total is None else f" (+{tot - prev_total})" + tot_str = f"{tot}/{n}" + print(f" {label:54s} {tot_str:>8s} {rate:>5.1f}%{delta}") prev_total = tot print() - print("Decomposition of N3 passes:") - n1_u = block["counts_u"]["n1"] - n1_c = block["counts_c"]["n1"] - n3_u = block["counts_u"]["n3"] - n3_c = block["counts_c"]["n3"] - print(f" U-pass at strict N1 (gold was right, agent right): {n1_u}") - print(f" C-cosmetic-pass at strict N1 (audit cosmetic, agent ok): {n1_c}") - print(f" C-fix-pass at N2/N3 (audit fixed buggy gold): {n3_c - n1_c}") - print(f" U-fail (gold was right, agent wrong): {nu - n1_u}") - print(f" C-fail (audit didn't help, agent wrong): {nc - n3_c}") - print() print("Per-DB N1 (original-gold pass) tally:") print(f" {'database':36s} {'pass':>5s} / {'tot':<3s} {'%':>5s}") print("-" * 60) diff --git a/scripts/dev1515_convert_runs.py b/scripts/dev1515_convert_runs.py index f8d109c7..2ac86c3b 100644 --- a/scripts/dev1515_convert_runs.py +++ b/scripts/dev1515_convert_runs.py @@ -248,6 +248,7 @@ def _process_one( "n6": cascade.n6_numeric_epsilon, "n7": cascade.n7_trailing_whitespace, "n8": cascade.n8_column_order, + "n9": cascade.n9_case_fold, "task_path": str(task_dest), "sub_path": str(sub_dest), } diff --git a/src/bird_interact_agents/agents/claude_sdk_otf/prompts.py b/src/bird_interact_agents/agents/claude_sdk_otf/prompts.py index 082f3949..cc0dc664 100644 --- a/src/bird_interact_agents/agents/claude_sdk_otf/prompts.py +++ b/src/bird_interact_agents/agents/claude_sdk_otf/prompts.py @@ -136,7 +136,17 @@ 4. TEST candidate columns and the final query with `query` / `query_nested`; sanity-check the generated SQL. -5. SUBMIT. Write the FINAL query so it REFERENCES the named columns / +5. PRE-SUBMIT MUTATION CHECK. Before calling `submit_query`, audit every + TRIM, LOWER, UPPER, ROUND, CAST, dedup, canonicalize-via-CASE, and + output-shape choice in the FINAL query. Each one MUST be either + (a) explicitly named in the user's question or (b) required by an + encoded KB. If neither holds, DROP the mutation and submit the raw + form. "Defensive" normalisation of an output column, a join key, a + JSON key, or a CHAR-padded literal silently corrupts the rowset — + never apply one without an explicit source. There is no user to + second-guess this on your behalf. + +6. SUBMIT. Write the FINAL query so it REFERENCES the named columns / measures you encoded — do NOT inline their SQL back into the query. Project exactly the columns the question names, and only those. {submit} diff --git a/src/bird_interact_agents/agents/claude_sdk_otf_ainteract/prompts.py b/src/bird_interact_agents/agents/claude_sdk_otf_ainteract/prompts.py index 7cba39b7..98fb0693 100644 --- a/src/bird_interact_agents/agents/claude_sdk_otf_ainteract/prompts.py +++ b/src/bird_interact_agents/agents/claude_sdk_otf_ainteract/prompts.py @@ -143,7 +143,22 @@ 5. TEST candidate columns and the final query with `query` / `query_nested`; sanity-check the generated SQL. -6. SUBMIT. Write the FINAL query so it REFERENCES the named columns / +6. PRE-SUBMIT MUTATION CHECK. Before calling `submit_query`, audit every + TRIM, LOWER, UPPER, ROUND, CAST, dedup, canonicalize-via-CASE, and + output-shape choice in the FINAL query. Each one MUST be either + (a) explicitly named in the user's question, (b) explicitly named OR + authorized in a reply to one of your `ask_user` calls in this + session, or (c) required by an encoded KB. If none of (a-b-c) hold, + DROP the mutation and submit the raw form. Particularly: when an + `ask_user` reply said "use exact values", "don't normalize", "use + this output shape / columns / sort axis", or named a specific format + (date, label casing, JSON shape), DO NOT silently override that on + final-assembly. Conversely, when an `ask_user` reply DID name a + specific transformation (e.g. "lowercase the bracket labels", + "round to 2 decimals", "TRIM the keys"), that reply IS the + authorization for that mutation — apply it. + +7. SUBMIT. Write the FINAL query so it REFERENCES the named columns / measures you encoded — do NOT inline their SQL back into the query. Project exactly the columns the user named, and only those. {submit} diff --git a/src/bird_interact_agents/eval/annotate.py b/src/bird_interact_agents/eval/annotate.py index 23801374..5897dc4e 100644 --- a/src/bird_interact_agents/eval/annotate.py +++ b/src/bird_interact_agents/eval/annotate.py @@ -175,6 +175,7 @@ def _eval_from_cascade(cascade: Any, epsilon: float = 1e-6) -> SubmissionEvaluat correct_under_numeric_epsilon=cascade.n6_numeric_epsilon, correct_under_trailing_whitespace=cascade.n7_trailing_whitespace, correct_under_column_order=cascade.n8_column_order, + correct_under_case_fold=cascade.n9_case_fold, numeric_epsilon=epsilon, verdict=verdict_label_from_cascade(cascade), # type: ignore[arg-type] matched_variant_id=cascade.matched_variant_id, diff --git a/src/bird_interact_agents/eval/annotation_schema.py b/src/bird_interact_agents/eval/annotation_schema.py index 897d5402..cf6237a6 100644 --- a/src/bird_interact_agents/eval/annotation_schema.py +++ b/src/bird_interact_agents/eval/annotation_schema.py @@ -318,6 +318,7 @@ class SubmissionEvaluation(BaseModel): correct_under_numeric_epsilon: bool = False correct_under_trailing_whitespace: bool = False correct_under_column_order: bool = False + correct_under_case_fold: bool = False numeric_epsilon: float = 1e-6 """Records the epsilon threshold actually used for N6, so consumers of the annotation know how lenient the relaxation was.""" diff --git a/src/bird_interact_agents/eval/grade_in_place.py b/src/bird_interact_agents/eval/grade_in_place.py index ef3536dd..3395e13e 100644 --- a/src/bird_interact_agents/eval/grade_in_place.py +++ b/src/bird_interact_agents/eval/grade_in_place.py @@ -49,7 +49,7 @@ def verdict_label_from_cascade(cascade: CascadeVerdict) -> str: * ``n3_any_audited_variant`` → ``"correct"`` (strict set-equal pass) * ``n4_tie_order`` / ``n5_llm_judge`` / ``n6_numeric_epsilon`` / - ``n7_trailing_whitespace`` / ``n8_column_order`` → + ``n7_trailing_whitespace`` / ``n8_column_order`` / ``n9_case_fold`` → ``"valid_interpretation"`` (cascade-tier acceptance — the row is not strictly identical to the gold but matches under a named tolerance / under the LLM judge for an ``insufficient`` task) @@ -63,6 +63,7 @@ def verdict_label_from_cascade(cascade: CascadeVerdict) -> str: or cascade.n6_numeric_epsilon or cascade.n7_trailing_whitespace or cascade.n8_column_order + or cascade.n9_case_fold ): return "valid_interpretation" return "invalid" @@ -93,6 +94,8 @@ def _auto_failure_class(cascade: CascadeVerdict) -> tuple[str, bool, str]: return ("trailing_whitespace", False, "grader") if cascade.n8_column_order: return ("column_order", False, "grader") + if cascade.n9_case_fold: + return ("case_sensitivity", False, "grader") # Genuine strict miss — let the human classify. return ("other", True, "other") @@ -130,6 +133,7 @@ def _build_submission_annotation( correct_under_numeric_epsilon=cascade.n6_numeric_epsilon, correct_under_trailing_whitespace=cascade.n7_trailing_whitespace, correct_under_column_order=cascade.n8_column_order, + correct_under_case_fold=cascade.n9_case_fold, numeric_epsilon=epsilon, verdict=verdict_label, # type: ignore[arg-type] matched_variant_id=cascade.matched_variant_id, diff --git a/src/bird_interact_agents/eval/tolerant_grader.py b/src/bird_interact_agents/eval/tolerant_grader.py index 1f86bdf0..14e50d2e 100644 --- a/src/bird_interact_agents/eval/tolerant_grader.py +++ b/src/bird_interact_agents/eval/tolerant_grader.py @@ -242,6 +242,24 @@ def compare_trailing_whitespace( return _set_equal(pred_n, gold_n) +def _lower_str_cell(cell: Any) -> Any: + return cell.lower() if isinstance(cell, str) else cell + + +def compare_case_fold( + pred: Sequence[Sequence], + gold: Sequence[Sequence], +) -> bool: + """N9 — lift case-only differences in string cells. Non-string cells + pass through unchanged; column-name case-folding is handled by + `compare_column_order` / `_column_diff`, not here.""" + if not _row_count_match(pred, gold): + return False + pred_n = [tuple(_lower_str_cell(c) for c in r) for r in pred] + gold_n = [tuple(_lower_str_cell(c) for c in r) for r in gold] + return _set_equal(pred_n, gold_n) + + def compare_column_order( pred: Sequence[Sequence], gold: Sequence[Sequence], @@ -345,6 +363,7 @@ def _column_diff( "n6_numeric_epsilon", "n7_trailing_whitespace", "n8_column_order", + "n9_case_fold", ] @@ -501,6 +520,7 @@ class CascadeVerdict(BaseModel): n6_numeric_epsilon: bool n7_trailing_whitespace: bool n8_column_order: bool + n9_case_fold: bool matched_variant_id: Optional[str] = None novel_reading_judgment: Optional[PhaseVerdict] = None variant_matches: List[VariantMatch] = Field(default_factory=list) @@ -711,6 +731,14 @@ def grade_submission( ): n8 = True break + n9 = n8 + if not n9: + for _v_meta, v_rows, _v_cols in variant_results + [ + ({"variant_id": "__original__"}, orig_rows, orig_cols), # type: ignore[list-item] + ]: + if compare_case_fold(pred_rows, v_rows): + n9 = True + break # 7) Tier 2 informational per variant. info_matches: list[VariantMatch] = [] @@ -736,6 +764,7 @@ def grade_submission( "n3_any_audited_variant": n3, "n4_tie_order": n4, "n5_llm_judge": n5, "n6_numeric_epsilon": n6, "n7_trailing_whitespace": n7, "n8_column_order": n8, + "n9_case_fold": n9, } enforced = enforce_monotone_cascade(raw) return CascadeVerdict( diff --git a/tests/test_eval_annotate_cli.py b/tests/test_eval_annotate_cli.py index 41c7bd29..5f182132 100644 --- a/tests/test_eval_annotate_cli.py +++ b/tests/test_eval_annotate_cli.py @@ -98,6 +98,7 @@ def __call__(self, *, submitted_sql, **_kw): n3_any_audited_variant=True, n4_tie_order=True, n5_llm_judge=True, n6_numeric_epsilon=True, n7_trailing_whitespace=True, n8_column_order=True, + n9_case_fold=True, matched_variant_id="primary", novel_reading_judgment=None, variant_matches=[], rowset_relations=[], @@ -305,6 +306,7 @@ def __call__(self, **_kw): n3_any_audited_variant=True, n4_tie_order=True, n5_llm_judge=True, n6_numeric_epsilon=True, n7_trailing_whitespace=True, n8_column_order=True, + n9_case_fold=True, matched_variant_id="primary", novel_reading_judgment=None, variant_matches=[], rowset_relations=[], ) @@ -338,6 +340,7 @@ def __call__(self, **_kw): n3_any_audited_variant=True, n4_tie_order=True, n5_llm_judge=True, n6_numeric_epsilon=True, n7_trailing_whitespace=True, n8_column_order=True, + n9_case_fold=True, matched_variant_id="primary", novel_reading_judgment=None, variant_matches=[], rowset_relations=[], ) diff --git a/tests/test_eval_annotation_schema.py b/tests/test_eval_annotation_schema.py index 11b0f441..5ba56fba 100644 --- a/tests/test_eval_annotation_schema.py +++ b/tests/test_eval_annotation_schema.py @@ -152,6 +152,23 @@ def test_task_annotation_roundtrip(tmp_path): assert loaded == ann +def test_submission_evaluation_default_includes_n9_case_fold(): + """N9 (case-fold) flag must default to False on new evaluations + and round-trip through JSON write/read.""" + ann = _make_submission_annotation() + assert ann.evaluation.correct_under_case_fold is False + + +def test_submission_evaluation_n9_case_fold_roundtrips(tmp_path): + ann = _make_submission_annotation() + # Flip the new tier to True to confirm it survives write→read. + ann.evaluation.correct_under_case_fold = True + p = tmp_path / "alien_42.submission.20260531t0001.json" + write_submission_annotation(ann, p) + loaded = read_submission_annotation(p) + assert loaded.evaluation.correct_under_case_fold is True + + def test_submission_annotation_roundtrip(tmp_path): ann = _make_submission_annotation() p = tmp_path / "alien_42.submission.20260531t0001.json" diff --git a/tests/test_regrade_cli.py b/tests/test_regrade_cli.py index 2a84f868..0dd64663 100644 --- a/tests/test_regrade_cli.py +++ b/tests/test_regrade_cli.py @@ -59,6 +59,7 @@ def __call__(self, **kw): n3_any_audited_variant=True, n4_tie_order=True, n5_llm_judge=True, n6_numeric_epsilon=True, n7_trailing_whitespace=True, n8_column_order=True, + n9_case_fold=True, matched_variant_id="primary", novel_reading_judgment=None, variant_matches=[], rowset_relations=[], @@ -99,6 +100,7 @@ def __call__(self, **kw): n3_any_audited_variant=True, n4_tie_order=True, n5_llm_judge=True, n6_numeric_epsilon=True, n7_trailing_whitespace=True, n8_column_order=True, + n9_case_fold=True, matched_variant_id="primary", novel_reading_judgment=None, variant_matches=[], rowset_relations=[], @@ -132,6 +134,7 @@ def __call__(self, **kw): n3_any_audited_variant=True, n4_tie_order=True, n5_llm_judge=True, n6_numeric_epsilon=True, n7_trailing_whitespace=True, n8_column_order=True, + n9_case_fold=True, matched_variant_id="primary", novel_reading_judgment=None, variant_matches=[], rowset_relations=[], @@ -165,6 +168,7 @@ def __call__(self, **kw): n3_any_audited_variant=True, n4_tie_order=True, n5_llm_judge=True, n6_numeric_epsilon=True, n7_trailing_whitespace=True, n8_column_order=True, + n9_case_fold=True, matched_variant_id="primary", novel_reading_judgment=None, variant_matches=[], rowset_relations=[], @@ -245,6 +249,7 @@ def __call__(self, **kw): n3_any_audited_variant=False, n4_tie_order=False, n5_llm_judge=False, n6_numeric_epsilon=False, n7_trailing_whitespace=False, n8_column_order=False, + n9_case_fold=False, matched_variant_id=None, novel_reading_judgment=None, variant_matches=[], rowset_relations=[], ) diff --git a/tests/test_tolerant_grader_comparators.py b/tests/test_tolerant_grader_comparators.py index b318349d..807e3001 100644 --- a/tests/test_tolerant_grader_comparators.py +++ b/tests/test_tolerant_grader_comparators.py @@ -226,6 +226,44 @@ def test_n8_column_count_mismatch_fails(): ) is False +# --------------------------------------------------------------------------- +# N9 — case-fold tolerance on string cells +# --------------------------------------------------------------------------- + + +def test_n9_case_fold_lifts_case_only_mismatch(): + from bird_interact_agents.eval.tolerant_grader import compare_case_fold + + pred = [("HIGH",), ("Low",)] + gold = [("high",), ("low",)] + assert compare_case_fold(pred, gold) is True + + +def test_n9_non_string_cells_unchanged(): + from bird_interact_agents.eval.tolerant_grader import compare_case_fold + + pred = [("A", 1.0)] + gold = [("a", 1.0)] + assert compare_case_fold(pred, gold) is True + + +def test_n9_row_count_mismatch_fails(): + from bird_interact_agents.eval.tolerant_grader import compare_case_fold + + pred = [("A",), ("B",)] + gold = [("a",)] + assert compare_case_fold(pred, gold) is False + + +def test_n9_content_difference_beyond_case_fails(): + """Case-fold must not paper over genuine content differences.""" + from bird_interact_agents.eval.tolerant_grader import compare_case_fold + + pred = [("Apple",)] + gold = [("orange",)] + assert compare_case_fold(pred, gold) is False + + # --------------------------------------------------------------------------- # ORDER BY parser — N4 input # --------------------------------------------------------------------------- diff --git a/tests/test_tolerant_grader_orchestration.py b/tests/test_tolerant_grader_orchestration.py index d11eee84..f7b2e478 100644 --- a/tests/test_tolerant_grader_orchestration.py +++ b/tests/test_tolerant_grader_orchestration.py @@ -553,6 +553,37 @@ def test_n7_trailing_whitespace_lifts_failing_n6_to_pass(): assert verdict.n7_trailing_whitespace is True +def test_n9_case_fold_lifts_failing_n8_to_pass(): + """A case-only mismatch in cell strings fails N7/N8 but lifts at N9.""" + from bird_interact_agents.eval.tolerant_grader import grade_submission + + submitted = "S" + original_gold = "G" + audited = "A" + executor = FakeExecutor({ + submitted: ([("HIGH",), ("Low",)], ["bracket"]), + original_gold: ([("high",), ("low",)], ["bracket"]), + audited: ([("high",), ("low",)], ["bracket"]), + }) + ann = _make_task_annotation() + gold_rows = [_audited_row( + instance_id="alien_1", variant_id="primary", primary=True, + audited_sol_sql=[audited], + )] + verdict = grade_submission( + task_annotation=ann, + audited_gold_rows=gold_rows, + original_sol_sql=[original_gold], + submitted_sql=submitted, + db_path=Path("/dev/null"), + conn=None, + executor=executor, + ) + assert verdict.n7_trailing_whitespace is False + assert verdict.n8_column_order is False + assert verdict.n9_case_fold is True + + def test_n8_column_order_uses_column_metadata(): """N8 needs cursor.description-style column names; ensure the executor's returned `column_names` parameter is the path used.""" @@ -607,10 +638,11 @@ def test_cascade_is_monotone_for_every_possible_pass_pattern(): "n1_original_gold", "n2_audited_primary", "n3_any_audited_variant", "n4_tie_order", "n5_llm_judge", "n6_numeric_epsilon", "n7_trailing_whitespace", "n8_column_order", + "n9_case_fold", ] - # Every possible 8-bit raw mask. - for mask in range(2 ** 8): - raw = {fields[i]: bool((mask >> i) & 1) for i in range(8)} + n = len(fields) + for mask in range(2 ** n): + raw = {fields[i]: bool((mask >> i) & 1) for i in range(n)} enforced = enforce_monotone_cascade(raw) prev = False for f in fields: diff --git a/tests/test_verdict_label_shared.py b/tests/test_verdict_label_shared.py index 04318d17..c9284d5c 100644 --- a/tests/test_verdict_label_shared.py +++ b/tests/test_verdict_label_shared.py @@ -27,6 +27,7 @@ def _cascade(**flags): n6_numeric_epsilon=flags.get("n6", False), n7_trailing_whitespace=flags.get("n7", False), n8_column_order=flags.get("n8", False), + n9_case_fold=flags.get("n9", False), matched_variant_id=flags.get("matched_variant_id"), novel_reading_judgment=flags.get("novel"), ) @@ -85,6 +86,12 @@ def test_n8_only_yields_valid_interpretation_on_both_paths(): assert annotate == "valid_interpretation" +def test_n9_only_yields_valid_interpretation_on_both_paths(): + inline, annotate = _both_verdicts(_cascade(n9=True)) + assert inline == "valid_interpretation" + assert annotate == "valid_interpretation" + + def test_no_tier_yields_invalid_on_both_paths(): inline, annotate = _both_verdicts(_cascade()) assert inline == "invalid" From c163fcff5c8b6aa5c5e55160eea1fb541751d739 Mon Sep 17 00:00:00 2001 From: Egor Kraev Date: Mon, 1 Jun 2026 13:16:35 +0200 Subject: [PATCH 10/29] DEV-1515 livesqlbench audit + annotate: expand to credit + mental MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two new livesqlbench DBs taken from skeleton to fully-enriched annotations (museum was the only previously-audited DB). 30 new audit rows + 30 task annotations land in the gitignored audited_gold/ and annotations/ trees per the cloud gold delivery split; this commit is the supporting infrastructure. - `scripts/dev1515_convert_livesqlbench.py`: renamed from the museum-specific script; now takes `--db` (required) and optional `--run-id`. Task-only mode when run-id is omitted. Adds a third rationale branch for management-category audits (verdict=sufficient with a deferral-aware rationale instead of the misleading "audit found nothing to change" text the old user-shortcut branch would emit for audited==original verbatim-copy rows). - `tests/test_livesqlbench_audited_gold.py`: generalize beyond museum. `EXPECTED_INSTANCE_IDS_BY_DB` keys per-DB; coverage / db-tag / citation-resolvability tests dispatch by selected_database; the edited/unrecoverable test carves out the management-category deferral shape (audited==original is permitted when clause_kind="management_category", per the shared contract); the changes[].justified_by non-empty check is relaxed for management deferrals (no source to cite — the deferral is documented in why_unjustified). - `.claude/skills/annotate-task-submission/SKILL.md`: restructured around the two-phase split. Task annotation is now explicitly self-contained (phase 1: KB + column meanings + schema + audited gold). Submission annotation is the optional phase 2 layered on top when a cloud run exists. Inputs section partitioned the same way; Workflow section split into phase-1 steps (always run) and phase-2 steps (only with a run). Co-Authored-By: Claude Opus 4.7 (1M context) --- .../skills/annotate-task-submission/SKILL.md | 84 ++++++--- scripts/dev1515_convert_livesqlbench.py | 141 +++++++++++---- tests/test_livesqlbench_audited_gold.py | 169 +++++++++++++----- 3 files changed, 288 insertions(+), 106 deletions(-) diff --git a/.claude/skills/annotate-task-submission/SKILL.md b/.claude/skills/annotate-task-submission/SKILL.md index f9d90376..8c39fbf0 100644 --- a/.claude/skills/annotate-task-submission/SKILL.md +++ b/.claude/skills/annotate-task-submission/SKILL.md @@ -14,6 +14,27 @@ The Pydantic schemas live in `src/bird_interact_agents/eval/annotation_schema.py`; they validate with `extra="forbid"` — never invent fields. +## Two phases — task annotation is self-contained + +The recipe has two independent phases, and you can stop after the first: + +1. **Task annotation** (`.task.json`) — needs only task-side + inputs (KB, column meanings, schema, gold sidecar, audited gold). + Fills `metadata_sufficiency.verdict` + `gold_variants` + + `evaluator_prompt` + `internal_inconsistency`. Produce this whenever + you author a new benchmark DB; no cloud run required. +2. **Submission annotation** (`.submission..json`) — adds + per-(instance, run) `failure_classification` + cascade tier on top + of the task annotation. Skip this phase entirely when no run exists; + the task annotation stands on its own and downstream tooling + (cascade summary, U/C buckets) gracefully handles missing + submissions. + +When in doubt, the rule of thumb: if you're not looking at a +`results/cloud//rows//attempt-1.json`, you're in phase 1 +only. Read just the phase-1 inputs below + the verdict decision flow; +ignore everything tagged "submission" or "failure_classification". + ## NON-NEGOTIABLE: fix buggy audits — never annotate around them Before classifying ANY task where `audited_sol_sql != original_sol_sql`, @@ -89,7 +110,9 @@ audit_status stays `edited`). This skill carries the annotation-side rule (the `internal_inconsistency` block + multi-variant `gold_variants`). -## Inputs (per task) +## Inputs + +### Phase 1 — task annotation (always required) Read-only — **mini-interact**: @@ -119,12 +142,6 @@ Read-only — **livesqlbench**: the agent sees this at task time. - `livesqlbench-base-lite-sqlite//.sqlite` -Per-submission (both benchmarks): - -- `results/cloud//rows//attempt-1.json` — `submitted_sql`, - `trajectory`, `usage`, `phase1_observation_*`. (livesqlbench - trajectories are short — one-shot.) - Read+write (both benchmarks): - `audited_gold/_audited.jsonl` — audit sidecar. @@ -133,8 +150,20 @@ Read+write (both benchmarks): `livesqlbench_audited.jsonl`. - `annotations///.task.json` — task annotation. Benchmark slug: `mini-interact` / `livesqlbench`. + +### Phase 2 — submission annotation (only when a run exists) + +Read-only: + +- `results/cloud//rows//attempt-1.json` — `submitted_sql`, + `trajectory`, `usage`, `phase1_observation_*`. (livesqlbench + trajectories are short — one-shot.) + +Read+write: + - `annotations///.submission..json` — - per-(instance, run) submission annotation. + per-(instance, run) submission annotation. Skip this file entirely + when phase 1 stands alone. ## What "masked sql_snippet" means @@ -293,16 +322,16 @@ A task is **U** (audit-unchanged) when its ## Workflow -1. Generate skeletons via - `scripts/dev1515_convert_runs.py` (or `bird_interact_agents.eval.annotate`). - The skeleton pre-fills mechanical fields + the cascade and auto-picks - `no_fail` / cascade-tier failure_class. Human-judgment fields land at - `PENDING_HUMAN_REVIEW` only for genuine `other` strict misses. +### Phase 1 — task annotation (always run) + +1. Generate skeletons via `scripts/dev1515_convert_runs.py` + (mini-interact, also touches submissions) or + `scripts/dev1515_convert_livesqlbench.py --db ` (livesqlbench, + task-only when `--run-id` is omitted). The skeleton pre-fills + mechanical fields; human-judgment fields land at + `PENDING_HUMAN_REVIEW`. 2. For each task: check `audit_unchanged` first. If unchanged AND - published metadata is unambiguous, the task is `sufficient`; if the - submission also passed, you're done. If unchanged AND agent failed, - the failure is `agent_miss` 90% of the time (verify by reading - submitted_sql vs gold). + published metadata is unambiguous, the task is `sufficient`. 3. For tasks where the audit changed gold (C-instances), START with the audit-correctness gate. If the audit is buggy, fix it in `audited_gold/_audited.jsonl` before writing the @@ -310,12 +339,27 @@ A task is **U** (audit-unchanged) when its 4. Read `external_knowledge` + the relevant KB entries + `column_meaning_base.json` + the masked snippet inside `amb_user_query` before assigning a verdict. -5. For ambiguous + a-interact: scan the trajectory for `ask_user` calls +5. Validate every edit with `TaskAnnotation.model_validate(...)`. + +Stop here when no run exists. Phase 2 is a strict superset, not a +prerequisite — downstream tooling reads the `.task.json` files +independently of any `.submission..json`. + +### Phase 2 — submission annotation (only with a run) + +6. Re-run the skeleton generator with the run-id wired in (e.g. + `dev1515_convert_runs.py` against the desired run, or + `dev1515_convert_livesqlbench.py --db --run-id `). The + skeleton auto-picks `no_fail` / cascade-tier `failure_class` for + mechanical passes; only genuine `other` strict misses get + `PENDING_HUMAN_REVIEW`. +7. If unchanged AND agent failed, the failure is `agent_miss` 90% of + the time (verify by reading submitted_sql vs gold). +8. For ambiguous + a-interact: scan the trajectory for `ask_user` calls and the sim's replies. If the sim withheld snippet content the agent needed, the submission's `failure_classification.primary` is `user_sim_under_disclosure`. -6. Validate every edit with - `SubmissionAnnotation.model_validate(...)` / `TaskAnnotation.model_validate(...)`. +9. Validate every edit with `SubmissionAnnotation.model_validate(...)`. ## Annotation rationale — write generously diff --git a/scripts/dev1515_convert_livesqlbench.py b/scripts/dev1515_convert_livesqlbench.py index c16131a5..fb42e705 100644 --- a/scripts/dev1515_convert_livesqlbench.py +++ b/scripts/dev1515_convert_livesqlbench.py @@ -1,6 +1,6 @@ -"""DEV-1515: convert the museum livesqlbench tasks (15 instances) to the -new annotation schema. Mirrors `dev1515_convert_runs.py` but for -livesqlbench's one-shot data layout: +"""DEV-1515: convert any livesqlbench DB's tasks to the new annotation +schema. Mirrors `dev1515_convert_runs.py` but for livesqlbench's one-shot +data layout: * `query` (NL only) instead of `amb_user_query` (NL + masked snippet). * No `user_query_ambiguity.critical_ambiguity`; no masked snippets. @@ -10,25 +10,28 @@ Inputs: * `livesqlbench-base-lite-sqlite/livesqlbench_data_sqlite.jsonl` — - main task rows. + main task rows; filtered to `--db`. * `livesqlbench-base-lite-sqlite/livesqlbench_sqlite_gt_kg_testcases_0528.jsonl` — gold sidecar (`instance_id`, `sol_sql`, `external_knowledge`, `test_cases`). * `audited_gold/livesqlbench_audited.jsonl` — audited sidecar. -* `results/cloud//rows//attempt-1.json` — submitted SQL - + trajectory + usage for the museum tasks that ran on - `20260531t1013-claudes-slayer-48eb0f` (10/15). +* (optional) `results/cloud//rows//attempt-1.json` — when + `--run-id` is passed, submission annotations are written for every + instance that has a run row. Outputs: -* `annotations/livesqlbench/museum/.task.json` (all 15). -* `annotations/livesqlbench/museum/.submission..json` (10 - with submissions). - -For the 5 museum instances without a submission, no submission file is -written. +* `annotations/livesqlbench//.task.json` for every instance in + the DB. +* `annotations/livesqlbench//.submission..json` when + a corresponding `attempt-1.json` exists under `--run-id`. + +Usage: + uv run python scripts/dev1515_convert_livesqlbench.py \\ + --db museum [--run-id 20260531t1013-claudes-slayer-48eb0f] """ from __future__ import annotations +import argparse import datetime as _dt import json import sqlite3 @@ -44,7 +47,6 @@ from bird_interact_agents.eval.annotation_schema import ( AuditedGoldRef, GoldVariantRef, - MaskedTerm, MetadataSufficiency, Provenance, TaskAnnotation, @@ -60,9 +62,7 @@ BENCHMARK = "livesqlbench" -DB = "museum" AUDIT_FILE_REL = "audited_gold/livesqlbench_audited.jsonl" -RUN_ID = "20260531t1013-claudes-slayer-48eb0f" PENDING = "PENDING_HUMAN_REVIEW" @@ -73,14 +73,14 @@ def _norm(sqls): return [" ".join((s or "").split()) for s in (sqls or [])] -def _load_data_rows() -> dict: +def _load_data_rows(db: str) -> dict: p = LIVESQL_ROOT / "livesqlbench_data_sqlite.jsonl" rows = {} for line in p.read_text().splitlines(): if not line.strip(): continue d = json.loads(line) - if d.get("selected_database") == DB: + if d.get("selected_database") == db: rows[d["instance_id"]] = d return rows @@ -112,13 +112,43 @@ def _load_audit_rows() -> dict: def _build_task_annotation( *, instance_id: str, + db: str, data_row: dict, gold_row: dict, audit_row: dict | None, audit_unchanged: bool, annotated_at: str, ) -> TaskAnnotation: - if audit_row is not None and audit_unchanged: + is_management_deferral = ( + audit_row is not None + and audit_row.get("audit_status") == "unrecoverable" + and any( + c.get("clause_kind") == "management_category" + for c in (audit_row.get("changes") or []) + ) + ) + if is_management_deferral: + # Management-category tasks (CREATE/ALTER/DELETE/CREATE INDEX + # etc.) are deferred from the row-count audit per the shared + # contract — eval depends on side-effects + custom test_case + # comparators, not row equality. The published metadata is fine + # for the task itself; the audit just doesn't certify the SQL. + ms = MetadataSufficiency( + verdict="sufficient", + rationale=( + "Management-category task: the audit-gold-sql skill " + "deferred this row from the row-count audit because eval " + "uses test_case Python comparators rather than row " + "equality. The published metadata sufficiently anchors " + "the user query; canonicity of the SQL is checked by " + "the test_cases, not by this annotation." + ), + evidence_sources_consulted=["audited_gold/livesqlbench_audited.jsonl"], + ) + gold_variants: list[GoldVariantRef] = [] + original_gold_is_correct = True + evaluator_prompt = None + elif audit_row is not None and audit_unchanged: ms = MetadataSufficiency( verdict="sufficient", rationale=( @@ -129,7 +159,7 @@ def _build_task_annotation( ), evidence_sources_consulted=["audited_gold/livesqlbench_audited.jsonl"], ) - gold_variants: list[GoldVariantRef] = [] + gold_variants = [] original_gold_is_correct = True evaluator_prompt = None elif audit_row is not None and not audit_unchanged: @@ -175,8 +205,8 @@ def _build_task_annotation( return TaskAnnotation( instance_id=instance_id, - selected_database=DB, - annotated_by="dev1515-convert-livesqlbench-museum", + selected_database=db, + annotated_by=f"dev1515-convert-livesqlbench-{db}", annotated_at=annotated_at, amb_user_query=data_row.get("query", ""), # livesqlbench: NL-only `query` external_knowledge=list(gold_row.get("external_knowledge", []) or []), @@ -195,6 +225,8 @@ def _build_task_annotation( def _process_one( *, instance_id: str, + db: str, + run_id: str | None, data_row: dict, gold_row: dict, audit_row: dict | None, @@ -203,9 +235,10 @@ def _process_one( annotations_root: Path, annotated_at: str, ) -> dict: - db_path = LIVESQL_ROOT / DB / f"{DB}.sqlite" + db_path = LIVESQL_ROOT / db / f"{db}.sqlite" task_ann = _build_task_annotation( instance_id=instance_id, + db=db, data_row=data_row, gold_row=gold_row, audit_row=audit_row, @@ -214,7 +247,7 @@ def _process_one( ) task_dest = task_annotation_path( benchmark=BENCHMARK, - selected_database=DB, + selected_database=db, instance_id=instance_id, repo_root=annotations_root.parent, ) @@ -222,7 +255,11 @@ def _process_one( sub_dest = None rec_extra = {"has_submission": False} - if rows_dir is not None and (rows_dir / instance_id / "attempt-1.json").exists(): + if ( + run_id is not None + and rows_dir is not None + and (rows_dir / instance_id / "attempt-1.json").exists() + ): attempt_path = rows_dir / instance_id / "attempt-1.json" attempt = json.loads(attempt_path.read_text()) submitted_sql = attempt.get("submitted_sql", "") @@ -250,7 +287,7 @@ def _process_one( task_annotation=task_ann, cascade=cascade, benchmark=BENCHMARK, - run_id=RUN_ID, + run_id=run_id, trajectory_path=str(attempt_path), predicted_row_count=attempt.get("predicted_row_count"), duration_s=attempt.get("duration_s"), @@ -262,9 +299,9 @@ def _process_one( ) sub_dest = submission_annotation_path( benchmark=BENCHMARK, - selected_database=DB, + selected_database=db, instance_id=instance_id, - run_id=RUN_ID, + run_id=run_id, repo_root=annotations_root.parent, ) write_submission_annotation(ann, sub_dest) @@ -290,24 +327,44 @@ def _process_one( def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--db", + required=True, + help="livesqlbench `selected_database` to convert (e.g. museum, credit, mental)", + ) + parser.add_argument( + "--run-id", + default=None, + help="optional cloud run id; when set, writes submission " + "annotations for instances with attempt-1.json under " + "results/cloud//rows/.", + ) + args = parser.parse_args() + db = args.db + run_id = args.run_id + annotations_root = paths.annotations_root() results_cloud = paths.results_root() / "cloud" - data_rows = _load_data_rows() + data_rows = _load_data_rows(db) gold_rows = _load_gold_rows() audit_rows = _load_audit_rows() instance_ids = sorted(data_rows.keys()) - print(f"museum instances in data: {len(instance_ids)}") - print(f"museum audit rows present: " + print(f"{db} instances in data: {len(instance_ids)}") + print(f"{db} audit rows present: " f"{sum(1 for i in instance_ids if i in audit_rows)}") - run_rows_dir = results_cloud / RUN_ID / "rows" - print(f"run rows dir exists: {run_rows_dir.exists()}") - if run_rows_dir.exists(): - present = sorted(p.name for p in run_rows_dir.iterdir() - if p.is_dir() and p.name.startswith("museum_")) - print(f"museum instances with submissions: {len(present)}") + run_rows_dir = (results_cloud / run_id / "rows") if run_id else None + if run_rows_dir is not None: + print(f"run rows dir exists: {run_rows_dir.exists()}") + if run_rows_dir.exists(): + present = sorted(p.name for p in run_rows_dir.iterdir() + if p.is_dir() and p.name.startswith(f"{db}_")) + print(f"{db} instances with submissions: {len(present)}") + else: + present = [] else: present = [] @@ -326,10 +383,16 @@ def main() -> None: orig = _norm(audit.get("original_sol_sql")) aud = _norm(audit.get("audited_sol_sql")) unchanged = (orig == aud) - rows_dir = run_rows_dir if (run_rows_dir.exists() and iid in present) else None + rows_dir = ( + run_rows_dir + if (run_rows_dir is not None and run_rows_dir.exists() and iid in present) + else None + ) try: rec = _process_one( instance_id=iid, + db=db, + run_id=run_id, data_row=data_rows[iid], gold_row=gold_rows[iid], audit_row=audit, @@ -353,7 +416,7 @@ def main() -> None: print(f" [{audit_tag}{sub_tag}] {iid:30s}{cascade_str}") results.append(rec) - out_index = annotations_root / "_dev1515_convert_museum_index.json" + out_index = annotations_root / f"_dev1515_convert_livesqlbench_{db}_index.json" out_index.write_text(json.dumps(results, indent=2) + "\n") print(f"\nWrote {len(results)} task annotations " f"+ {sum(1 for r in results if r['has_submission'])} submission annotations.") diff --git a/tests/test_livesqlbench_audited_gold.py b/tests/test_livesqlbench_audited_gold.py index 65906d8e..358c6cb4 100644 --- a/tests/test_livesqlbench_audited_gold.py +++ b/tests/test_livesqlbench_audited_gold.py @@ -1,18 +1,20 @@ -"""DEV-1510 contract tests for `audited_gold/livesqlbench_audited.jsonl`. +"""DEV-1510/DEV-1515 contract tests for +`audited_gold/livesqlbench_audited.jsonl`. -The file is the deliverable of this issue: one row per livesqlbench museum -SELECT task (museum_1..10), each row attaching to the canonical gold for +The file is the deliverable of these issues: one row per audited +livesqlbench SELECT task, each row attaching to the canonical gold for that instance. Tests here pin three layers of contract: * **Schema** — every row parses, has the required keys + types, and is - unique by `instance_id`. Coverage = exactly {museum_1..10}; the 5 - Management tasks (museum_M_1..5) are explicitly out of scope per the - Linear issue. + unique by `(instance_id, variant_id)`. Per DB, coverage is the + set in `EXPECTED_INSTANCE_IDS_BY_DB` (the SELECT tasks; the M-suffixed + Management tasks are deferred per the shared contract's edge case). * **Status-claim consistency** — `clean` rows have `audited_sol_sql == - original_sol_sql` and `changes == []`; `edited` / `unrecoverable` have - `audited_sol_sql != original_sol_sql` and at least one change entry - carrying `clause_kind`, `original`, `replacement`, `why_unjustified`, - and a non-empty `justified_by` list. + original_sol_sql` and `changes == []`; `edited` rows have + `audited_sol_sql != original_sol_sql` and at least one change entry. + `unrecoverable` rows have non-empty changes; they DIFFER from the + original unless the change is the management-category deferral + (`clause_kind="management_category"`), per the shared contract. * **Pinned decisions** — museum_7 and museum_9 are the issue's worked examples, locked in the spec: - museum_7 (`edited`): KB-canonical rewrite using a NULL-safe @@ -24,8 +26,8 @@ + the schema's single-hop declared FK. `reasoning_summary` documents the KB-alone underspec AND mentions both join chains by name. -Tests that need the real upstream data (`museum_kb.jsonl`, -`museum_column_meaning_base.json`, the gated gold sidecar) skip when the +Tests that need the real upstream data (`_kb.jsonl`, +`_column_meaning_base.json`, the gated gold sidecar) skip when the livesqlbench data root is absent (CI doesn't ship the gitignored data). """ @@ -46,8 +48,16 @@ # --------------------------------------------------------------------------- -EXPECTED_INSTANCE_IDS = {f"museum_{i}" for i in range(1, 11)} -"""The 10 museum SELECT tasks. museum_M_1..5 (management) are out of scope.""" +EXPECTED_INSTANCE_IDS_BY_DB: dict[str, set[str]] = { + "museum": {f"museum_{i}" for i in range(1, 11)}, + "credit": {f"credit_{i}" for i in range(1, 11)}, + "mental": {f"mental_{i}" for i in range(1, 11)}, +} +"""Per-DB SELECT-task coverage. The M-suffixed management tasks are +deferred per the shared contract's edge case; museum omits them +entirely, credit/mental ship them as `unrecoverable` with +`clause_kind="management_category"`. Both shapes are accepted by +``test_audit_rows_cover_select_tasks_per_db``.""" REQUIRED_ROW_KEYS = { "instance_id", @@ -146,11 +156,11 @@ def _livesqlbench_data_or_skip() -> Path: return root -def _load_museum_kb_ids() -> set[int]: +def _load_kb_ids(db: str) -> set[int]: root = _livesqlbench_data_or_skip() - kb_path = root / "museum" / "museum_kb.jsonl" + kb_path = root / db / f"{db}_kb.jsonl" if not kb_path.exists(): - pytest.skip(f"museum_kb.jsonl missing at {kb_path}") + pytest.skip(f"{db}_kb.jsonl missing at {kb_path}") ids: set[int] = set() with kb_path.open() as f: for line in f: @@ -163,11 +173,11 @@ def _load_museum_kb_ids() -> set[int]: return ids -def _load_museum_column_meaning_keys() -> set[str]: +def _load_column_meaning_keys(db: str) -> set[str]: root = _livesqlbench_data_or_skip() - cm_path = root / "museum" / "museum_column_meaning_base.json" + cm_path = root / db / f"{db}_column_meaning_base.json" if not cm_path.exists(): - pytest.skip(f"museum_column_meaning_base.json missing at {cm_path}") + pytest.skip(f"{db}_column_meaning_base.json missing at {cm_path}") with cm_path.open() as f: return set(json.load(f).keys()) @@ -273,13 +283,39 @@ def test_audit_file_exists_when_other_audits_are_present(): ) -def test_audit_rows_cover_exactly_museum_1_through_10(): - rows = _load_audit_rows() - assert set(rows.keys()) == EXPECTED_INSTANCE_IDS, ( - f"audit must cover exactly museum_1..10; missing=" - f"{sorted(EXPECTED_INSTANCE_IDS - rows.keys())}; " - f"extra={sorted(rows.keys() - EXPECTED_INSTANCE_IDS)}" - ) +def test_audit_rows_cover_select_tasks_per_db(): + """For every DB that has ANY audited row in the file, every SELECT + task (1..10) must be covered. M-suffixed Management tasks are NOT + required (deferred per shared contract); per-DB extras outside the + SELECT set are allowed only when they are deferred management rows + (`audit_status=unrecoverable` with `clause_kind="management_category"`).""" + primary_rows = _load_audit_rows() + by_db: dict[str, set[str]] = {} + for iid, row in primary_rows.items(): + by_db.setdefault(row["selected_database"], set()).add(iid) + for db, ids in by_db.items(): + expected = EXPECTED_INSTANCE_IDS_BY_DB.get(db) + assert expected is not None, ( + f"audit file contains DB {db!r} without a coverage entry in " + f"EXPECTED_INSTANCE_IDS_BY_DB — add one when authoring a new DB" + ) + missing = expected - ids + assert not missing, ( + f"{db}: missing SELECT-task audits {sorted(missing)}" + ) + # Extras that aren't in EXPECTED are tolerated only if they're + # management deferrals (or any non-primary alternate that the + # auditor explicitly carries). + extras = ids - expected + for iid in sorted(extras): + row = primary_rows[iid] + cks = {c.get("clause_kind") for c in row.get("changes", [])} + assert row["audit_status"] == "unrecoverable" and ( + "management_category" in cks + ), ( + f"{db}: extra audited instance {iid!r} is not in the SELECT " + f"coverage set and is not a management-category deferral" + ) def test_audit_rows_have_required_keys_and_types(): @@ -314,13 +350,21 @@ def test_audit_rows_use_valid_audit_status(): def test_audit_rows_tag_benchmark_and_database(): + """Every row carries `benchmark=livesqlbench` and a `selected_database` + in EXPECTED_INSTANCE_IDS_BY_DB; the `instance_id` prefix matches the + `selected_database` (so a museum row can't claim DB=credit by typo).""" for row in _iter_audit_rows(): + iid = row["instance_id"] assert row["benchmark"] == "livesqlbench", ( - f"{row['instance_id']}: benchmark={row['benchmark']!r} (expected 'livesqlbench')" + f"{iid}: benchmark={row['benchmark']!r} (expected 'livesqlbench')" + ) + db = row["selected_database"] + assert db in EXPECTED_INSTANCE_IDS_BY_DB, ( + f"{iid}: selected_database={db!r} not in " + f"{sorted(EXPECTED_INSTANCE_IDS_BY_DB.keys())}" ) - assert row["selected_database"] == "museum", ( - f"{row['instance_id']}: selected_database={row['selected_database']!r} " - "(expected 'museum')" + assert iid.startswith(f"{db}_"), ( + f"{iid}: instance_id prefix does not match selected_database={db!r}" ) @@ -392,13 +436,28 @@ def test_clean_rows_have_audited_equal_original_and_no_changes(): def test_edited_and_unrecoverable_rows_have_changes_and_differ(): + """`edited` rows MUST differ from the original (that's the whole point + of the rewrite). `unrecoverable` rows usually differ — they fall back + to the natural reading of the user query — but the shared contract + carves out one exception: management-category tasks + (`clause_kind="management_category"`) are deferred from the + row-count audit and ship with gold copied verbatim into + `audited_sol_sql`. Both shapes must still have non-empty changes.""" for row in _iter_audit_rows(): if row["audit_status"] not in {"edited", "unrecoverable"}: continue iid = row["instance_id"] - assert row["audited_sol_sql"] != row["original_sol_sql"], ( - f"{iid}: {row['audit_status']} row must differ from original_sol_sql" + clause_kinds = {c.get("clause_kind") for c in row.get("changes", [])} + is_management_deferral = ( + row["audit_status"] == "unrecoverable" + and "management_category" in clause_kinds ) + if not is_management_deferral: + assert row["audited_sol_sql"] != row["original_sol_sql"], ( + f"{iid}: {row['audit_status']} row must differ from " + f"original_sol_sql (unless it's a management-category " + f"deferral)" + ) assert row["changes"], ( f"{iid}: {row['audit_status']} row must have non-empty changes" ) @@ -409,9 +468,17 @@ def test_edited_and_unrecoverable_rows_have_changes_and_differ(): ) assert isinstance(change["clause_kind"], str) and change["clause_kind"], iid assert isinstance(change["why_unjustified"], str) and change["why_unjustified"], iid - assert isinstance(change["justified_by"], list) and change["justified_by"], ( - f"{iid}: changes[{j}].justified_by must be a non-empty list" + assert isinstance(change["justified_by"], list), ( + f"{iid}: changes[{j}].justified_by must be a list" ) + # Management-category deferrals carry no citations (nothing + # to cite — the gold is verbatim, the deferral itself is + # documented in `why_unjustified`). Every OTHER change must + # cite at least one source. + if change.get("clause_kind") != "management_category": + assert change["justified_by"], ( + f"{iid}: changes[{j}].justified_by must be a non-empty list" + ) # Every justified_by token must look like a citation. (The # resolvability tests below confirm the tokens actually resolve.) for token in change["justified_by"]: @@ -626,19 +693,23 @@ def test_museum_9_is_clean_with_column_meaning_justification(): # --------------------------------------------------------------------------- -def test_every_kb_citation_resolves_to_a_museum_kb_id(): - """Every `kb:N` citation MUST resolve to a row in museum_kb.jsonl — - catches typos (kb:116 instead of kb:16) and id-drift after upstream - KB renumbers. Same posture as the mini-interact resolvability tests.""" - kb_ids = _load_museum_kb_ids() +def test_every_kb_citation_resolves_to_a_kb_id(): + """Every `kb:N` citation MUST resolve to a row in the row's own DB + `_kb.jsonl` — catches typos (kb:116 instead of kb:16) and + id-drift after upstream KB renumbers. Same posture as the + mini-interact resolvability tests.""" + kb_ids_by_db: dict[str, set[int]] = {} for row in _iter_audit_rows(): iid = row["instance_id"] - tokens = _collect_citation_tokens(row) - for tok in tokens: + db = row["selected_database"] + if db not in kb_ids_by_db: + kb_ids_by_db[db] = _load_kb_ids(db) + kb_ids = kb_ids_by_db[db] + for tok in _collect_citation_tokens(row): if tok.startswith("kb:"): kb_id = int(tok.split(":", 1)[1]) assert kb_id in kb_ids, ( - f"{iid}: kb:{kb_id} does not resolve in museum_kb.jsonl" + f"{iid}: kb:{kb_id} does not resolve in {db}_kb.jsonl" ) @@ -665,19 +736,23 @@ def test_every_column_meaning_citation_resolves(): """Catches case-typos in column-meaning citations — keys are case-sensitive in the JSON (e.g. `museum|ConditionAssessments|LightReadRefObserved`, NOT lowercase).""" - keys = _load_museum_column_meaning_keys() + keys_by_db: dict[str, set[str]] = {} for row in _iter_audit_rows(): iid = row["instance_id"] + db = row["selected_database"] + if db not in keys_by_db: + keys_by_db[db] = _load_column_meaning_keys(db) + keys = keys_by_db[db] for tok in _collect_citation_tokens(row): if tok.startswith("column_meaning:"): table_col = tok.split(":", 1)[1] # Citations use the `Table|Column[|SubField]` form; the - # JSON key is `museum|Table|Column[|SubField]`. Both shapes + # JSON key is `|Table|Column[|SubField]`. Both shapes # accepted. - candidates = [f"museum|{table_col}", table_col] + candidates = [f"{db}|{table_col}", table_col] assert any(c in keys for c in candidates), ( f"{iid}: column_meaning:{table_col} does not resolve in " - f"museum_column_meaning_base.json (tried: {candidates})" + f"{db}_column_meaning_base.json (tried: {candidates})" ) From 862e95937c03b66c4e3ced4bec673de0ade5053a Mon Sep 17 00:00:00 2001 From: Egor Kraev Date: Mon, 1 Jun 2026 16:39:44 +0200 Subject: [PATCH 11/29] =?UTF-8?q?DEV-1515=20review=20fixes=20=E2=80=94=20G?= =?UTF-8?q?roups=201-4=20from=20/process-reviews=20triage=20on=20PR=20#16?= =?UTF-8?q?=20(round=202)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bundles 11 valid findings from CodeRabbit + Codex triage on PR #16: Group 1 — cascade reporting integrity * cascading_report: _per_row_cascade_bools now includes n9_case_fold; deltas loop extended to n9. Fixes silent counts.n9 = 0 on every published eval.json (n9 tier was added to _CASCADE_ORDER but the aggregator was hardcoded n1..n8). * regrade: run_dir uses paths.results_root() so BIRD_RESULTS_ROOT is honored. Also picks up pre-existing in-flight _build_original_sql_index work so regrade can supply original_sol_sql for the N1 check. * run.py: post-eval, auto-detect a per-row rows/ tree alongside eval.json and enrich it with cascading_phase1 when present (no-op for vanilla local runs; covers the symmetric local↔cloud convention). Group 2 — multi-variant gold preservation in conversion scripts * dev1515_convert_livesqlbench: drop hardcoded LIVESQL_ROOT, use paths.livesqlbench_root(); _load_audit_rows returns dict[str, list[dict]]; _pick_primary helper; _process_one passes the full variant list to grade_submission so N3 (\"any audited variant\") sees every alternate. * dev1515_convert_runs: same shape — dict-of-lists for mini-interact audit, _pick_primary, full list to grade_submission. * dev1515_households_14_multivariant: idempotency guard refuses to re-run on already-multivariant rows; filters pre-existing source_conflict entries so reruns never duplicate change records. Group 3 — annotation path mismatch + zero-primary contract test * annotation_io: _canonical_benchmark() normalizes dash↔underscore for the benchmark segment of every annotation path. Cloud workers (_cloud_benchmark → mini_interact) and CLI callers (--benchmark mini-interact) now land in the same tree; on-disk annotations/mini-interact/ contents migrated to annotations/mini_interact/. * annotate.py docstring updated to canonical form (both accepted). * test_livesqlbench_audited_gold: extracted _check_unique_variant_pairs_and_primary_count helper; added zero-primary / two-primary regression tests so an iid with only non-primary rows can no longer slip through. Group 4 — docs polish in skills * annotate-task-submission/SKILL.md: text-tag the two unlabeled fenced blocks (MD040). * audit-gold-sql/SKILL.md: rewrite the Outputs + Step 8 prose so the sidecar write key is (instance_id, variant_id) — overwriting on instance_id alone would silently drop alternates landed by the DEV-1515 multi-variant audits. Test suite: 1823 passed, 95 skipped, 0 failed. INVALID finding (no thread to reply on): CodeRabbit review-summary nitpick claiming tests/cloud/test_collation.py:31-43 holds a stale _read_dual_cols helper — that symbol does not exist in HEAD; lines 31-43 are the test_collate_picks_latest_attempt header + setup. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../skills/annotate-task-submission/SKILL.md | 4 +- .claude/skills/audit-gold-sql/SKILL.md | 25 ++++-- scripts/dev1515_convert_livesqlbench.py | 55 ++++++++---- scripts/dev1515_convert_runs.py | 44 +++++++--- scripts/dev1515_households_14_multivariant.py | 30 +++++-- src/bird_interact_agents/eval/annotate.py | 7 +- .../eval/annotation_io.py | 29 +++++-- .../eval/cascading_report.py | 3 +- src/bird_interact_agents/eval/regrade.py | 84 ++++++++++++++++++- src/bird_interact_agents/run.py | 24 +++++- tests/cloud/test_fetch_annotation_merge.py | 8 +- tests/test_cascading_report.py | 44 +++++++++- tests/test_eval_annotation_schema.py | 8 +- tests/test_livesqlbench_audited_gold.py | 80 +++++++++++++++--- tests/test_paths_annotations.py | 47 +++++++++++ tests/test_regrade_cli.py | 2 +- 16 files changed, 413 insertions(+), 81 deletions(-) diff --git a/.claude/skills/annotate-task-submission/SKILL.md b/.claude/skills/annotate-task-submission/SKILL.md index 8c39fbf0..ce3bb96a 100644 --- a/.claude/skills/annotate-task-submission/SKILL.md +++ b/.claude/skills/annotate-task-submission/SKILL.md @@ -189,7 +189,7 @@ so the agent's only sources are the published metadata. ### mini-interact (a-interact mode) -``` +```text 1. Audit-correctness gate — is the audited gold itself correct? NO → FIX the audit JSONL row first; come back here. YES → proceed. @@ -226,7 +226,7 @@ Corollaries: ### livesqlbench (one-shot mode) -``` +```text 1. Audit-correctness gate — is the audited gold itself correct? NO → FIX the audit JSONL row first; come back here. YES → proceed. diff --git a/.claude/skills/audit-gold-sql/SKILL.md b/.claude/skills/audit-gold-sql/SKILL.md index 2bb1944e..0907a753 100644 --- a/.claude/skills/audit-gold-sql/SKILL.md +++ b/.claude/skills/audit-gold-sql/SKILL.md @@ -46,10 +46,14 @@ Read-only: ## Outputs -One JSONL line appended (or updated) at -`bird-interact-agents/audited_gold//_audited.jsonl`. If a line -already exists for this `instance_id`, **overwrite** it in place -(latest-wins; the verifier dedups on read). +One JSONL line per `(instance_id, variant_id)` pair appended (or +updated) at `bird-interact-agents/audited_gold//_audited.jsonl`. +`variant_id` defaults to `"primary"`; multi-variant audits emit +multiple rows sharing `instance_id` with distinct `variant_id` slugs. +If a row already exists for this `(instance_id, variant_id)` pair, +**overwrite** it in place (latest-wins on the composite key — +NEVER drop alternate variants by overwriting on `instance_id` alone; +the verifier dedups on `(instance_id, variant_id)`). ### Sidecar schema (verbatim) @@ -409,13 +413,18 @@ is incomplete until execution succeeds. Path: `bird-interact-agents/audited_gold//_audited.jsonl`. -If the file doesn't exist, create the directory and the file. If a -line already exists for this `instance_id`, rewrite the whole file -with the line replaced (latest-wins). +If the file doesn't exist, create the directory and the file. The +write key is the composite `(instance_id, variant_id)` pair — NOT +`instance_id` alone, because multi-variant audits emit multiple rows +per instance and overwriting on `instance_id` would silently drop +alternates. If a line already exists for this `(instance_id, +variant_id)` pair, rewrite the whole file with that one line replaced +(latest-wins on the composite key); leave every other variant row +untouched. Build the JSON object per the schema above, serialise with `json.dumps(obj, ensure_ascii=False)`, append a newline. Each JSONL -line is one task's full audit. +line is one variant of one task's full audit. ### Step 9 — Per-DB summary (optional, after auditing all tasks) diff --git a/scripts/dev1515_convert_livesqlbench.py b/scripts/dev1515_convert_livesqlbench.py index fb42e705..c7b2ba7b 100644 --- a/scripts/dev1515_convert_livesqlbench.py +++ b/scripts/dev1515_convert_livesqlbench.py @@ -66,15 +66,13 @@ PENDING = "PENDING_HUMAN_REVIEW" -LIVESQL_ROOT = Path("/home/james/Dropbox/SLayer/livesqlbench-base-lite-sqlite") - def _norm(sqls): return [" ".join((s or "").split()) for s in (sqls or [])] def _load_data_rows(db: str) -> dict: - p = LIVESQL_ROOT / "livesqlbench_data_sqlite.jsonl" + p = paths.livesqlbench_root() / "livesqlbench_data_sqlite.jsonl" rows = {} for line in p.read_text().splitlines(): if not line.strip(): @@ -86,7 +84,10 @@ def _load_data_rows(db: str) -> dict: def _load_gold_rows() -> dict: - p = LIVESQL_ROOT / "livesqlbench_sqlite_gt_kg_testcases_0528.jsonl" + p = ( + paths.livesqlbench_root() + / "livesqlbench_sqlite_gt_kg_testcases_0528.jsonl" + ) rows = {} for line in p.read_text().splitlines(): if not line.strip(): @@ -96,19 +97,37 @@ def _load_gold_rows() -> dict: return rows -def _load_audit_rows() -> dict: +def _load_audit_rows() -> dict[str, list[dict]]: + """Load every audit row keyed by ``instance_id`` → list of variant + rows. Multi-variant audits ship N rows per instance (one primary + + alternates) and ALL of them must survive: collapsing to a single + dict-per-instance silently drops alternates and makes N3 ("any + audited variant") miscompute for ambiguous tasks.""" p = paths.audited_gold_root() / "livesqlbench_audited.jsonl" - rows = {} + rows: dict[str, list[dict]] = {} if not p.exists(): return rows for line in p.read_text().splitlines(): if not line.strip(): continue d = json.loads(line) - rows[d["instance_id"]] = d + rows.setdefault(d["instance_id"], []).append(d) return rows +def _pick_primary(audit_rows: list[dict]) -> dict | None: + """Return the primary variant from a list of audit rows, or None if + the list is empty. Falls back to the first row when no row carries + ``primary: true`` (single-variant tasks pre-DEV-1515 omit the + field; the default is ``True``).""" + if not audit_rows: + return None + for r in audit_rows: + if r.get("primary", True): + return r + return audit_rows[0] + + def _build_task_annotation( *, instance_id: str, @@ -229,13 +248,14 @@ def _process_one( run_id: str | None, data_row: dict, gold_row: dict, - audit_row: dict | None, + audit_rows: list[dict], audit_unchanged: bool, rows_dir: Path | None, annotations_root: Path, annotated_at: str, ) -> dict: - db_path = LIVESQL_ROOT / db / f"{db}.sqlite" + db_path = paths.livesqlbench_root() / db / f"{db}.sqlite" + audit_row = _pick_primary(audit_rows) task_ann = _build_task_annotation( instance_id=instance_id, db=db, @@ -265,12 +285,11 @@ def _process_one( submitted_sql = attempt.get("submitted_sql", "") usage = attempt.get("usage", {}) or {} - audit_arg = [audit_row] if audit_row is not None else [] conn = sqlite3.connect(f"file:{db_path}?mode=ro", uri=True, timeout=60) try: cascade = grade_submission( task_annotation=task_ann, - audited_gold_rows=audit_arg, + audited_gold_rows=audit_rows, original_sol_sql=list(gold_row.get("sol_sql") or []), submitted_sql=submitted_sql, db_path=db_path, @@ -321,6 +340,7 @@ def _process_one( "instance_id": instance_id, "audit_unchanged": audit_unchanged, "audit_present": audit_row is not None, + "n_audit_variants": len(audit_rows), "task_path": str(task_dest), **rec_extra, } @@ -377,11 +397,12 @@ def main() -> None: if iid not in gold_rows: print(f" {iid:30s} SKIP: no gold row") continue - audit = audit_rows.get(iid) + variants = audit_rows.get(iid, []) + primary = _pick_primary(variants) unchanged = False - if audit is not None: - orig = _norm(audit.get("original_sol_sql")) - aud = _norm(audit.get("audited_sol_sql")) + if primary is not None: + orig = _norm(primary.get("original_sol_sql")) + aud = _norm(primary.get("audited_sol_sql")) unchanged = (orig == aud) rows_dir = ( run_rows_dir @@ -395,7 +416,7 @@ def main() -> None: run_id=run_id, data_row=data_rows[iid], gold_row=gold_rows[iid], - audit_row=audit, + audit_rows=variants, audit_unchanged=unchanged, rows_dir=rows_dir, annotations_root=annotations_root, @@ -405,7 +426,7 @@ def main() -> None: print(f" {iid:30s} FAILED: {type(exc).__name__}: {exc}") raise else: - audit_tag = "U" if unchanged else ("C" if audit else "-") + audit_tag = "U" if unchanged else ("C" if primary else "-") sub_tag = "S" if rec["has_submission"] else " " cascade_str = "" if rec["has_submission"]: diff --git a/scripts/dev1515_convert_runs.py b/scripts/dev1515_convert_runs.py index 2ac86c3b..4ed6b1c7 100644 --- a/scripts/dev1515_convert_runs.py +++ b/scripts/dev1515_convert_runs.py @@ -89,7 +89,7 @@ def _masked_terms_from(task_row: dict) -> list[MaskedTerm]: def _build_task_annotation( *, task_row: dict, - audit_row: dict, + audit_row: dict | None, audit_unchanged: bool, annotated_at: str, ) -> TaskAnnotation: @@ -158,12 +158,25 @@ def _build_task_annotation( ) +def _pick_primary(audit_rows: list[dict]) -> dict | None: + """Return the primary variant from a list of audit rows, or None if + the list is empty. Falls back to the first row when no row carries + ``primary: true`` (single-variant rows pre-DEV-1515 omit the field; + the default is ``True``).""" + if not audit_rows: + return None + for r in audit_rows: + if r.get("primary", True): + return r + return audit_rows[0] + + def _process_one( *, run_id: str, instance_id: str, task_row: dict, - audit_row: dict, + audit_rows: list[dict], audit_unchanged: bool, rows_dir: Path, mini_root: Path, @@ -178,6 +191,8 @@ def _process_one( submitted_sql = attempt.get("submitted_sql", "") usage = attempt.get("usage", {}) or {} + audit_row = _pick_primary(audit_rows) + # 1) Task annotation. task_ann = _build_task_annotation( task_row=task_row, @@ -193,12 +208,15 @@ def _process_one( ) write_task_annotation(task_ann, task_dest) - # 2) Cascade via grade_submission. + # 2) Cascade via grade_submission. Multi-variant audits are passed + # as the full list so N3 ("any audited variant") sees every + # alternate — collapsing to the primary alone would miscompute + # cascade outcomes for ambiguous tasks. conn = sqlite3.connect(f"file:{db_path}?mode=ro", uri=True, timeout=60) try: cascade = grade_submission( task_annotation=task_ann, - audited_gold_rows=[audit_row], + audited_gold_rows=audit_rows, original_sol_sql=list(task_row.get("sol_sql") or []), submitted_sql=submitted_sql, db_path=db_path, @@ -267,14 +285,18 @@ def main() -> None: d = json.loads(line) rows[d["instance_id"]] = d - audit_rows = {} + # Collect every variant per instance_id. Multi-variant audits + # (DEV-1515 source_conflict tasks) ship N rows sharing instance_id; + # collapsing to a single dict per instance silently drops alternates + # and makes downstream N3 miscompute. + audit_rows: dict[str, list[dict]] = {} for line in ( paths.audited_gold_root() / "mini_interact_audited.jsonl" ).read_text().splitlines(): if not line.strip(): continue d = json.loads(line) - audit_rows[d["instance_id"]] = d + audit_rows.setdefault(d["instance_id"], []).append(d) annotated_at = ( _dt.datetime.now(_dt.timezone.utc).replace(microsecond=0).isoformat() @@ -287,16 +309,18 @@ def main() -> None: print(f"\n[{run_id}] {len(inst_ids)} instances") for iid in inst_ids: task_row = rows[iid] - audit_row = audit_rows[iid] - n_orig = _norm(audit_row.get("original_sol_sql")) - n_aud = _norm(audit_row.get("audited_sol_sql")) + variants = audit_rows[iid] + primary = _pick_primary(variants) + assert primary is not None # rows[iid] non-empty by construction + n_orig = _norm(primary.get("original_sol_sql")) + n_aud = _norm(primary.get("audited_sol_sql")) unchanged = n_orig == n_aud try: rec = _process_one( run_id=run_id, instance_id=iid, task_row=task_row, - audit_row=audit_row, + audit_rows=variants, audit_unchanged=unchanged, rows_dir=rows_dir, mini_root=mini_root, diff --git a/scripts/dev1515_households_14_multivariant.py b/scripts/dev1515_households_14_multivariant.py index 179016d4..e131d18f 100644 --- a/scripts/dev1515_households_14_multivariant.py +++ b/scripts/dev1515_households_14_multivariant.py @@ -88,7 +88,7 @@ def _patch_audit_jsonl() -> None: audit_path = paths.audited_gold_root() / "mini_interact_audited.jsonl" rows: list[dict] = [] - found = False + matched_rows: list[dict] = [] for line in audit_path.read_text().splitlines(): if not line.strip(): continue @@ -96,13 +96,34 @@ def _patch_audit_jsonl() -> None: if d["instance_id"] != INSTANCE_ID: rows.append(d) continue + matched_rows.append(d) + + if not matched_rows: + raise SystemExit(f"{INSTANCE_ID} not found in {audit_path}") + if len(matched_rows) > 1: + # Already multi-variant — running the script a second time would + # append two more kb_row/snippet_row variants AND accumulate a + # duplicate `source_conflict` change entry on each. Refuse instead + # of silently producing 4+ variants per instance. + raise SystemExit( + f"{INSTANCE_ID} already has {len(matched_rows)} audit rows " + f"in {audit_path}; refusing to duplicate variants. Clean up " + "the existing variants manually before rerunning this script." + ) + + for d in matched_rows: # The existing row becomes the kb_definition_reading variant # (already `lcs > 3`). - found = True kb_row = dict(d) kb_row["variant_id"] = "kb_definition_reading" kb_row["primary"] = True - kb_row.setdefault("changes", []).append({ + # Filter out any pre-existing source_conflict entry so reruns + # never accumulate duplicate change records. + kb_row["changes"] = [ + c for c in (kb_row.get("changes") or []) + if c.get("clause_kind") != "source_conflict" + ] + kb_row["changes"].append({ "clause_kind": "source_conflict", "original": "lcs > 2 (per critical_ambiguity for 'good quality of life'.sql_snippet)", "replacement": "lcs > 3", @@ -181,9 +202,6 @@ def _patch_audit_jsonl() -> None: conn.close() rows.append(snippet_row) - if not found: - raise SystemExit(f"{INSTANCE_ID} not found in {audit_path}") - audit_path.write_text("\n".join(json.dumps(r) for r in rows) + "\n") print(f"Patched {audit_path}: {INSTANCE_ID} now has 2 rows " f"(kb_definition_reading [primary] + snippet_reading)") diff --git a/src/bird_interact_agents/eval/annotate.py b/src/bird_interact_agents/eval/annotate.py index 5897dc4e..4ce04a7b 100644 --- a/src/bird_interact_agents/eval/annotate.py +++ b/src/bird_interact_agents/eval/annotate.py @@ -7,11 +7,16 @@ Usage:: python -m bird_interact_agents.eval.annotate \\ - --run-id --benchmark mini-interact \\ + --run-id --benchmark mini_interact \\ [--instance-ids ...] \\ [--task-mode {init,refresh,force-all}] \\ [--submission-mode {overwrite,init}] \\ [--dry-run] + +``--benchmark`` accepts both the dash form (``mini-interact``) and the +underscore form (``mini_interact``); ``annotation_io`` normalizes to +the canonical underscore form on every path-build so writes from +either form land in the same on-disk tree. """ from __future__ import annotations diff --git a/src/bird_interact_agents/eval/annotation_io.py b/src/bird_interact_agents/eval/annotation_io.py index ce40cf4a..04b72f98 100644 --- a/src/bird_interact_agents/eval/annotation_io.py +++ b/src/bird_interact_agents/eval/annotation_io.py @@ -7,9 +7,16 @@ * Submission annotation: ``/annotations///.submission..json`` -```` matches the benchmark name passed by callers (e.g. -``mini-interact``, ``livesqlbench``). ```` matches the task's -``selected_database`` field. +```` is normalized to the canonical underscore form +(``mini_interact``, ``livesqlbench``) at every path-building site so +callers passing the dash form (``mini-interact``) write/read to the +same tree as callers passing the underscore form. The canonical form +matches the benchmark registry's ``name`` field, which is what the +cloud worker resolves via ``_cloud_benchmark``. Without this +normalization the disk grows two parallel trees and the cloud cascade +silently misses every annotation written via the dash form. + +```` matches the task's ``selected_database`` field. These helpers do NOT regenerate annotations from raw artefacts — that belongs in a separate tool (``scripts/generate_annotation_skeleton.py`` @@ -29,6 +36,14 @@ ANNOTATIONS_DIRNAME = "annotations" +def _canonical_benchmark(benchmark: str) -> str: + """Normalize to the registry's canonical underscore form. Both + ``mini-interact`` (the form the CLI / docs historically accepted) + and ``mini_interact`` (``benchmark.name``) resolve to the same + on-disk tree.""" + return benchmark.replace("-", "_") + + def _annotations_root(repo_root: Optional[Path] = None) -> Path: """Anchor at the main checkout (matches the worktree-safe contract used by ``audited_gold/`` and ``results/``).""" @@ -45,7 +60,7 @@ def task_annotation_path( ) -> Path: return ( _annotations_root(repo_root) - / benchmark + / _canonical_benchmark(benchmark) / selected_database / f"{instance_id}.task.json" ) @@ -61,7 +76,7 @@ def submission_annotation_path( ) -> Path: return ( _annotations_root(repo_root) - / benchmark + / _canonical_benchmark(benchmark) / selected_database / f"{instance_id}.submission.{run_id}.json" ) @@ -94,7 +109,7 @@ def iter_task_annotations( so corruption surfaces at scan time rather than at first downstream use. """ - root = _annotations_root(repo_root) / benchmark + root = _annotations_root(repo_root) / _canonical_benchmark(benchmark) out: list[tuple[Path, TaskAnnotation]] = [] if not root.exists(): return out @@ -108,7 +123,7 @@ def iter_submission_annotations( repo_root: Optional[Path] = None, ) -> "list[tuple[Path, SubmissionAnnotation]]": """Walk submission annotations. When ``run_id`` is set, filter.""" - root = _annotations_root(repo_root) / benchmark + root = _annotations_root(repo_root) / _canonical_benchmark(benchmark) out: list[tuple[Path, SubmissionAnnotation]] = [] if not root.exists(): return out diff --git a/src/bird_interact_agents/eval/cascading_report.py b/src/bird_interact_agents/eval/cascading_report.py index 4f989670..25f93b25 100644 --- a/src/bird_interact_agents/eval/cascading_report.py +++ b/src/bird_interact_agents/eval/cascading_report.py @@ -51,6 +51,7 @@ def _per_row_cascade_bools(annotation_dir: Path) -> dict[str, bool]: "n6_numeric_epsilon": ev.correct_under_numeric_epsilon, "n7_trailing_whitespace": ev.correct_under_trailing_whitespace, "n8_column_order": ev.correct_under_column_order, + "n9_case_fold": ev.correct_under_case_fold, } return enforce_monotone_cascade(raw) @@ -83,7 +84,7 @@ def aggregate_cascading_phase1(rows_dir: Path) -> dict: } deltas: dict[str, int] = {} prev: int | None = None - for k in ("n1", "n2", "n3", "n4", "n5", "n6", "n7", "n8"): + for k in ("n1", "n2", "n3", "n4", "n5", "n6", "n7", "n8", "n9"): if prev is None: deltas[k] = 0 else: diff --git a/src/bird_interact_agents/eval/regrade.py b/src/bird_interact_agents/eval/regrade.py index 21e6406b..44984aa6 100644 --- a/src/bird_interact_agents/eval/regrade.py +++ b/src/bird_interact_agents/eval/regrade.py @@ -63,6 +63,70 @@ def clear_llm_judge_cache( cache_path.write_text(json.dumps(new, indent=2)) +def _build_original_sql_index(benchmark: str) -> dict[str, list[str]]: + """Map ``instance_id`` → list-of-SQL-strings for the benchmark's + original gold. mini_interact carries ``sol_sql`` inline on each task + row in ``mini_interact.jsonl``; livesqlbench ships an empty + ``sol_sql`` on the public ``livesqlbench_data_sqlite.jsonl`` and the + real list lives on the gated gold sidecar (env override + ``BIRD_LIVESQLBENCH_GOLD_FILE``). Look it up once at CLI startup so + the per-row grader doesn't repeatedly parse a multi-megabyte JSONL. + Empty rows fall back to ``[]`` so the cascade's N1 just doesn't fire + for instances whose source row genuinely has no gold (rather than + crashing). + """ + from bird_interact_agents.benchmark import get_benchmark + + out: dict[str, list[str]] = {} + data_file = paths.benchmark_data_file(benchmark) + if data_file.exists(): + with data_file.open() as f: + for line in f: + line = line.strip() + if not line: + continue + try: + r = json.loads(line) + except json.JSONDecodeError: + continue + iid = r.get("instance_id") + sol = r.get("sol_sql") + if iid and isinstance(sol, list) and sol: + out[iid] = list(sol) + # Merge in livesqlbench's gated sidecar if available. + bench = get_benchmark(benchmark) + if bench.gold_required: + gold_path: Optional[Path] = None + import os + env_override = os.environ.get(bench.gold_root_env or "") + if env_override: + gold_path = Path(env_override).expanduser() + else: + # Default sidecar location: / + for candidate in ( + paths.benchmark_data_root(benchmark) + / "livesqlbench_sqlite_gt_kg_testcases_0528.jsonl", + ): + if candidate.exists(): + gold_path = candidate + break + if gold_path and gold_path.exists(): + with gold_path.open() as f: + for line in f: + line = line.strip() + if not line: + continue + try: + r = json.loads(line) + except json.JSONDecodeError: + continue + iid = r.get("instance_id") + sol = r.get("sol_sql") + if iid and isinstance(sol, list) and sol: + out[iid] = list(sol) + return out + + def regrade_run( *, run_id: str, @@ -207,7 +271,13 @@ def main(argv: Optional[List[str]] = None) -> int: if args.instance_ids else None ) from bird_interact_agents.eval.tolerant_grader import grade_submission - run_dir = paths.main_checkout_root() / "results" / "cloud" / args.run_id + run_dir = paths.results_root() / "cloud" / args.run_id + + # Index the benchmark's source data once. mini_interact ships sol_sql + # inline; livesqlbench's gated sidecar carries sol_sql under + # ``--gold-file`` and the public data file ships it empty. Both routes + # land under ``instance_id`` so the lookup is identical at call time. + original_sql_by_inst = _build_original_sql_index(args.benchmark) def _grader(*, instance_id: str, submitted_sql: str, task_row: dict, **_kw): # Minimal end-to-end wiring — production callers pre-build the @@ -239,12 +309,18 @@ def _grader(*, instance_id: str, submitted_sql: str, task_row: dict, **_kw): audited = _load_audited_gold_rows_for( benchmark=args.benchmark, instance_id=instance_id, ) + # N1 requires the original gold SQL; the attempt JSON doesn't + # carry it (it lives on the source data row / gated gold sidecar). + original_sol_sql = list( + task_row.get("original_sol_sql") + or task_row.get("sol_sql") + or original_sql_by_inst.get(instance_id) + or [] + ) return grade_submission( task_annotation=ann, audited_gold_rows=audited, - original_sol_sql=list( - task_row.get("original_sol_sql") or task_row.get("sol_sql") or [], - ), + original_sol_sql=original_sol_sql, submitted_sql=submitted_sql, db_path=db_path, conn=None, diff --git a/src/bird_interact_agents/run.py b/src/bird_interact_agents/run.py index af2c8225..2e963fd9 100644 --- a/src/bird_interact_agents/run.py +++ b/src/bird_interact_agents/run.py @@ -13,6 +13,7 @@ from bird_interact_agents import paths from bird_interact_agents.benchmark import cli_dataset_tokens, get_benchmark +from bird_interact_agents.eval.cascading_report import emit_cascading_eval_json from bird_interact_agents.harness import ( apply_audited_gold_overlay, calculate_budget, @@ -1049,9 +1050,8 @@ async def _run_with_sem(i: int, td: dict) -> None: # `phase1_count_original`, `n_dual_eval_tasks`, the two rates) has # been REPLACED by the cascading_phase1 block. The block is computed # downstream by `emit_cascading_eval_json` over per-row - # submission_annotation.json files — local runs that don't write - # those (no inline grader hook) simply omit the block. `phase1_count` - # / `phase1_rate` stay as back-compat aliases for N1. + # submission_annotation.json files. `phase1_count` / `phase1_rate` + # stay as back-compat aliases for N1. metrics = { "mode": mode, "query_mode": query_mode, @@ -1068,11 +1068,27 @@ async def _run_with_sem(i: int, td: dict) -> None: "results": results, } - # Save + # Save. If a local-mode rows tree carrying per-row + # ``submission_annotation.json`` files exists alongside the eval + # output (cloud convention: ``/rows//``), enrich + # eval.json with the freshly-aggregated ``cascading_phase1`` block + # so the headline N1..N9 metrics aren't silently lost when local + # runs DO have annotations (e.g. via ``grade_in_place.grade_and_write`` + # or the convert scripts). Local runs without that tree keep the + # documented behaviour: omit the block, ship only the N1 aliases. Path(output_path).parent.mkdir(parents=True, exist_ok=True) with open(output_path, "w") as f: json.dump(metrics, f, indent=2, default=str) + rows_dir = Path(output_path).parent / "rows" + if rows_dir.exists() and any( + (sub / "submission_annotation.json").exists() + for sub in rows_dir.iterdir() if sub.is_dir() + ): + metrics = emit_cascading_eval_json( + rows_dir, Path(output_path), base_metrics=metrics, + ) + logger.info( "Done. Tasks: %d, P1: %d/%d (%.1f%%), Avg Reward: %.4f", n, p1_count, n, (p1_count / n * 100) if n else 0, diff --git a/tests/cloud/test_fetch_annotation_merge.py b/tests/cloud/test_fetch_annotation_merge.py index 83855690..0ef3b759 100644 --- a/tests/cloud/test_fetch_annotation_merge.py +++ b/tests/cloud/test_fetch_annotation_merge.py @@ -61,7 +61,7 @@ def _valid_submission_annotation_dict(instance_id: str = "alien_1") -> dict: "kind": "submission_annotation", "instance_id": instance_id, "selected_database": "alien", - "task_annotation_ref": f"annotations/mini-interact/alien/{instance_id}.task.json", + "task_annotation_ref": f"annotations/mini_interact/alien/{instance_id}.task.json", "annotated_by": "auto", "annotated_at": "2026-05-31", "submission": { @@ -104,7 +104,7 @@ def test_merge_writes_annotation_to_main_checkout(tmp_path): ) dest = ( - main_checkout / "annotations" / "mini-interact" / "alien" + main_checkout / "annotations" / "mini_interact" / "alien" / "alien_1.submission.r1.json" ) assert dest.exists() @@ -119,7 +119,7 @@ def test_merge_no_overwrite_if_present(tmp_path): ) main_checkout = tmp_path / "checkout" - dest_dir = main_checkout / "annotations" / "mini-interact" / "alien" + dest_dir = main_checkout / "annotations" / "mini_interact" / "alien" dest_dir.mkdir(parents=True) pre = _valid_submission_annotation_dict("alien_1") pre["annotated_by"] = "human-pre-existing" @@ -170,7 +170,7 @@ def test_merge_rejects_schema_invalid_file(tmp_path): assert report.rejected_invalid == 1 # Destination must NOT have been created from invalid content. assert not ( - main_checkout / "annotations" / "mini-interact" / "alien" + main_checkout / "annotations" / "mini_interact" / "alien" / "alien_1.submission.r1.json" ).exists() diff --git a/tests/test_cascading_report.py b/tests/test_cascading_report.py index a021bf63..baba6ae5 100644 --- a/tests/test_cascading_report.py +++ b/tests/test_cascading_report.py @@ -23,7 +23,7 @@ def _make_submission_annotation_json( instance_id: str, selected_database: str, n1: bool, n2: bool, n3: bool, n4: bool, n5: bool, - n6: bool, n7: bool, n8: bool, + n6: bool, n7: bool, n8: bool, n9: bool = False, verdict: str = "correct", ) -> dict: """Build the JSON shape produced by tolerant_grader → SubmissionAnnotation.""" @@ -59,6 +59,7 @@ def _make_submission_annotation_json( "correct_under_numeric_epsilon": n6, "correct_under_trailing_whitespace": n7, "correct_under_column_order": n8, + "correct_under_case_fold": n9, "numeric_epsilon": 1e-6, "verdict": verdict, "matched_variant_id": "primary" if n3 else None, @@ -171,6 +172,47 @@ def test_aggregator_enforces_monotonicity_on_tampered_row(tmp_path): assert counts["n8"] == 1 +def test_aggregator_surfaces_n9_case_fold(tmp_path): + """Regression for DEV-1515 follow-up: N9 was added to + ``tolerant_grader._CASCADE_ORDER`` but the aggregator's + ``_per_row_cascade_bools`` was hardcoded to N1..N8, leaving + ``counts['n9']`` stuck at 0 (and no ``deltas['n9']``) even when + the per-row annotation reported a case-fold-only pass.""" + from bird_interact_agents.eval.cascading_report import ( + aggregate_cascading_phase1, + ) + + rows_dir = tmp_path / "rows" + rows_dir.mkdir() + # One row: case-fold-only pass — every earlier level fails. + ann = _make_submission_annotation_json( + instance_id="alien_1", selected_database="alien", + n1=False, n2=False, n3=False, n4=False, n5=False, + n6=False, n7=False, n8=False, n9=True, + ) + d = rows_dir / "alien_1" + d.mkdir() + (d / "submission_annotation.json").write_text(json.dumps(ann)) + + block = aggregate_cascading_phase1(rows_dir) + assert block["counts"]["n9"] == 1, ( + "n9_case_fold must increment counts['n9'] — it was previously " + "dropped because _per_row_cascade_bools hardcoded n1..n8 only" + ) + assert block["rates"]["n9"] == pytest.approx(1.0) + assert "n9" in block["deltas"], ( + "deltas must extend to n9, not stop at n8" + ) + # Monotone enforcement walks N1→N9, so a case-fold-only pass leaves + # every stricter level (N1..N8) at 0 and only N9 at 1. The point of + # this regression test is that N9 is SURFACED at all — pre-fix it + # was stuck at 0 because the aggregator hardcoded N1..N8. + for k in ("n1", "n2", "n3", "n4", "n5", "n6", "n7", "n8"): + assert block["counts"][k] == 0, ( + f"only n9 was True in the raw row; {k} must remain 0" + ) + + def test_aggregator_raises_when_row_missing_submission_annotation(tmp_path): """If any per-row dir is missing submission_annotation.json, the aggregator must raise — silent under-count is forbidden.""" diff --git a/tests/test_eval_annotation_schema.py b/tests/test_eval_annotation_schema.py index 5ba56fba..2049a418 100644 --- a/tests/test_eval_annotation_schema.py +++ b/tests/test_eval_annotation_schema.py @@ -194,13 +194,17 @@ def test_submission_annotation_forbid_extra(): def test_path_helpers(tmp_path): + # ``annotation_io`` normalizes the dash form to the canonical + # underscore form so cloud workers (``_cloud_benchmark`` → ``mini_interact``) + # and CLI callers (``--benchmark mini-interact``) write to the + # same tree. t = task_annotation_path( benchmark="mini-interact", selected_database="alien", instance_id="alien_42", repo_root=tmp_path, ) - assert t == tmp_path / "annotations" / "mini-interact" / "alien" / "alien_42.task.json" + assert t == tmp_path / "annotations" / "mini_interact" / "alien" / "alien_42.task.json" s = submission_annotation_path( benchmark="mini-interact", selected_database="alien", @@ -211,7 +215,7 @@ def test_path_helpers(tmp_path): assert s == ( tmp_path / "annotations" - / "mini-interact" + / "mini_interact" / "alien" / "alien_42.submission.20260531t1008-claudes-slayer-890419.json" ) diff --git a/tests/test_livesqlbench_audited_gold.py b/tests/test_livesqlbench_audited_gold.py index 358c6cb4..7d081453 100644 --- a/tests/test_livesqlbench_audited_gold.py +++ b/tests/test_livesqlbench_audited_gold.py @@ -36,7 +36,7 @@ import json import re from pathlib import Path -from typing import Iterator +from typing import Iterable, Iterator import pytest @@ -390,30 +390,84 @@ def test_audit_rows_audited_at_is_iso8601(): ) -def test_no_duplicate_instance_id_variant_pairs(): - """The dedup contract is on the (instance_id, variant_id) pair, not on - instance_id alone — DEV-1515 multi-variant audits ship N rows per task - (one primary + alternates). Also: each instance_id MUST have exactly one - primary row.""" +def _check_unique_variant_pairs_and_primary_count( + rows: Iterable[dict], +) -> tuple[list[tuple[str, str]], dict[str, int]]: + """Pure helper used by both the live-data contract test below and + the synthetic regression test for the zero-primary edge case. + Returns ``(dupes, bad_primary_counts)`` — both empty when the audit + set satisfies the (instance_id, variant_id) + exactly-one-primary + contract.""" + from collections import Counter + seen_pairs: list[tuple[str, str]] = [] + all_iids: set[str] = set() primaries_per_iid: dict[str, int] = {} - for row in _iter_audit_rows(): + for row in rows: iid = row["instance_id"] + all_iids.add(iid) vid = row.get("variant_id", "primary") seen_pairs.append((iid, vid)) if row.get("primary", True): primaries_per_iid[iid] = primaries_per_iid.get(iid, 0) + 1 - dupes = [p for p in seen_pairs if seen_pairs.count(p) > 1] + dupes = [p for p, n in Counter(seen_pairs).items() if n > 1] + # Iterate every seen iid (not just those that recorded a primary) + # so zero-primary instances are flagged — building this dict from + # ``primaries_per_iid.items()`` alone would silently skip them. + bad_primary_counts = { + iid: primaries_per_iid.get(iid, 0) + for iid in all_iids + if primaries_per_iid.get(iid, 0) != 1 + } + return dupes, bad_primary_counts + + +def test_no_duplicate_instance_id_variant_pairs(): + """The dedup contract is on the (instance_id, variant_id) pair, not on + instance_id alone — DEV-1515 multi-variant audits ship N rows per task + (one primary + alternates). Also: each instance_id MUST have exactly one + primary row.""" + dupes, bad_primary_counts = ( + _check_unique_variant_pairs_and_primary_count(_iter_audit_rows()) + ) assert not dupes, ( f"duplicate (instance_id, variant_id) pairs in audit file: {dupes}" ) - over_primaries = { - iid: n for iid, n in primaries_per_iid.items() if n != 1 - } - assert not over_primaries, ( + assert not bad_primary_counts, ( f"each instance_id must have exactly one primary row; " - f"got counts: {over_primaries}" + f"got counts: {bad_primary_counts}" + ) + + +def test_zero_primary_variants_are_detected(): + """Regression: an iid carrying ONLY non-primary variants used to + slip through the over-primaries check because the dict was built + from ``primaries_per_iid.items()``, which never includes + zero-primary iids. The fixed check iterates all seen iids.""" + rows = [ + {"instance_id": "x_1", "variant_id": "alt_a", "primary": False}, + {"instance_id": "x_1", "variant_id": "alt_b", "primary": False}, + ] + dupes, bad_primary_counts = ( + _check_unique_variant_pairs_and_primary_count(rows) + ) + assert dupes == [] + assert bad_primary_counts == {"x_1": 0}, ( + "zero-primary iid must surface in bad_primary_counts with count 0" + ) + + +def test_two_primary_variants_are_detected(): + """Companion: an iid with TWO primaries also violates the contract.""" + rows = [ + {"instance_id": "y_1", "variant_id": "a", "primary": True}, + {"instance_id": "y_1", "variant_id": "b", "primary": True}, + ] + dupes, bad_primary_counts = ( + _check_unique_variant_pairs_and_primary_count(rows) ) + assert dupes == [] + assert bad_primary_counts == {"y_1": 2} # --------------------------------------------------------------------------- diff --git a/tests/test_paths_annotations.py b/tests/test_paths_annotations.py index 5a6b70ac..d315b277 100644 --- a/tests/test_paths_annotations.py +++ b/tests/test_paths_annotations.py @@ -81,3 +81,50 @@ def test_annotation_io_uses_paths_helper(monkeypatch, tmp_path): write_task_annotation(ann, p_explicit) assert p_explicit.exists() assert p_explicit.is_relative_to(tmp_path / "annotations") + + +def test_annotation_paths_canonicalize_mini_interact(tmp_path): + """``mini-interact`` (dash, the form ``annotate.py`` historically + accepted via the CLI) and ``mini_interact`` (underscore, the form + ``_cloud_benchmark`` returns from ``benchmark.name``) MUST resolve + to the same on-disk path. Without normalization, cloud workers + silently read from an empty ``annotations/mini_interact/`` tree + while every CLI-written annotation lands under + ``annotations/mini-interact/``.""" + from bird_interact_agents.eval import ( + submission_annotation_path, + task_annotation_path, + ) + + dash = task_annotation_path( + benchmark="mini-interact", + selected_database="alien", + instance_id="alien_1", + repo_root=tmp_path, + ) + underscore = task_annotation_path( + benchmark="mini_interact", + selected_database="alien", + instance_id="alien_1", + repo_root=tmp_path, + ) + assert dash == underscore + # And the canonical landing directory is the underscore form. + assert "mini_interact" in dash.parts + assert "mini-interact" not in dash.parts + + sub_dash = submission_annotation_path( + benchmark="mini-interact", + selected_database="alien", + instance_id="alien_1", + run_id="r1", + repo_root=tmp_path, + ) + sub_under = submission_annotation_path( + benchmark="mini_interact", + selected_database="alien", + instance_id="alien_1", + run_id="r1", + repo_root=tmp_path, + ) + assert sub_dash == sub_under diff --git a/tests/test_regrade_cli.py b/tests/test_regrade_cli.py index 0dd64663..1bf84d81 100644 --- a/tests/test_regrade_cli.py +++ b/tests/test_regrade_cli.py @@ -82,7 +82,7 @@ def test_regrade_overwrites_existing_submission_annotation(tmp_path, monkeypatch monkeypatch.setattr(paths_mod, "main_checkout_root", lambda: tmp_path) # Pre-existing submission annotation at the destination. - dest_dir = tmp_path / "annotations" / "mini-interact" / "alien" + dest_dir = tmp_path / "annotations" / "mini_interact" / "alien" dest_dir.mkdir(parents=True) dest = dest_dir / "alien_1.submission.r1.json" dest.write_text('{"annotated_by": "stale", "kind": "submission_annotation"}') From 5a153800feaf8135367b97164d2024bfdb09594f Mon Sep 17 00:00:00 2001 From: Egor Kraev Date: Mon, 1 Jun 2026 21:35:37 +0200 Subject: [PATCH 12/29] DEV-1515 session-4: agent-miss failure-mode diagnostics at grading time MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The cascade adequately classifies the 42/53 instances that pass at some tier — each tier (N3 strict, N6 numeric_epsilon, N9 case_fold, ...) names the tolerance that saved them. For the 11 strict misses that clear no cascade tier, the cascade only tells us "agent missed"; we want structured signals at grading time so downstream tooling can break down failure modes (wrong join path, missing predicate, wrong projection, empty result, SQL error, ...) without re-running queries or eyeballing trajectories. Schema (annotation_schema.py): - New MissDiagnostics Pydantic model captures rowset shape, column shape, sqlglot-derived SQL signals (tables_referenced, has_group_by, has_aggregate, join_count, where_conjunct_count, has_having, has_limit), execution status, and a multi-flag miss_patterns list with 14 independent flag categories. Comparison reference is the BEST-OVERLAP audited variant (multiset cardinality vs agent rows; tie-break primary > alphabetical variant_id) — not the primary by default, so multi-variant golds get diagnosed against the closest reading rather than misclassified as disjoint_rowset against an unrelated primary. - Interactive-only signal (user_sim_n_asks) stays Optional[int] = None on one-shot benchmarks (livesqlbench); never_asked_user flag fires only when the benchmark is interactive AND the agent didn't ask the user-sim. - SQL-derived fields are Optional[T] = None when sqlglot fails to parse, gated by *_sql_parse_ok booleans + truncated error excerpts. Silent False/0 defaults would produce spurious wrong_table_set / aggregation_shape_mismatch flags. Grader (tolerant_grader.py): - Wrapped the initial executor(submitted_sql, ...) call in try/except so a syntax/runtime error becomes pred_rows=[] + error excerpt + sql_execution_error flag, not a crash that aborts grading. - Wrapped per-variant execution defensively too (variant-side SQL that fails just produces empty rowset; diagnostics catch the downstream sqlglot parse failure via sql_parse_error flag). - New grade_submission kwarg user_sim_n_asks: Optional[int]. - _compute_miss_diagnostics with bag-aware row comparison (_bag_relation matches the grader's existing _set_equal multiset semantics) + sqlglot AST walkers that excludes CTE / derived-table aliases from base-table extraction. - Defensive assert len(sol_sql) == 1 — confirmed empirically that 0/915 SELECT-task gold rows are multi-statement; multi-statement is M-task territory (explicitly out of scope). Flag rules are INDEPENDENT (every applicable rule appends to miss_patterns; the list is sorted alphabetically before persist for stable JSON diffs). An instance can carry multiple flags simultaneously — e.g. wrong_table_set AND column_projection_mismatch AND empty_agent_result. Persistence (grade_in_place.py + annotate.py): - _auto_failure_class strict-miss branch flipped "other" -> "agent_miss" (agent_at_fault=True, remediation_target="agent"). The "other" bucket disappears entirely; rich detail lives on ev.miss_diagnostics. - _build_submission_annotation + _eval_from_cascade both plumb cascade.miss_diagnostics -> ev.miss_diagnostics so both the cloud worker path (grade_in_place) and the regrade / skeleton path (annotate) emit the same shape. - FailureClassification.details is a free-form one-liner derived from miss_patterns; structured downstream consumers MUST go through miss_diagnostics.miss_patterns, not parse the prose. Workflow: - TDD-style spec with Codex pre-review of the design plan (11 findings folded), test-suite-first (39 new tests, all failed for the right "feature missing" reason before implementation), then Codex pre-review of the tests (11 more findings folded — most importantly: best-overlap test asserts exact == "alt" not in {primary, alt}; tie-break tests use non-zero overlap; multiset superset detection; mirror alias-exclusion on best-variant side; negative assertions on per-flag tests for mutual-exclusion contracts). - Re-graded all 4 fetched runs (3 mini-interact + livesqlbench museum); the 12 cascade-fail instances all carry populated miss_diagnostics with >= 1 flag. scripts/dev1515_strict_miss_diagnostics.py walks the latest annotation per instance and prints per-instance table + per-flag tally + instance x flag matrix. column_projection_mismatch fires on every instance (slayer's namespaced agent column names vs gold's bare names — a real signal); disjoint_rowset dominates (9/12) suggesting wrong predicate / wrong join path; predicate_count_mismatch fires on 7/12 suggesting missing or extra WHERE conjuncts. Full non-integration suite: 1862 passed, 95 skipped, 50 deselected (was 1811; +51 from the new test file). Co-Authored-By: Claude Opus 4.7 (1M context) --- scripts/dev1515_strict_miss_diagnostics.py | 152 +++ src/bird_interact_agents/eval/annotate.py | 1 + .../eval/annotation_schema.py | 103 ++ .../eval/grade_in_place.py | 40 +- .../eval/tolerant_grader.py | 406 +++++- tests/test_miss_diagnostics.py | 1113 +++++++++++++++++ 6 files changed, 1801 insertions(+), 14 deletions(-) create mode 100644 scripts/dev1515_strict_miss_diagnostics.py create mode 100644 tests/test_miss_diagnostics.py diff --git a/scripts/dev1515_strict_miss_diagnostics.py b/scripts/dev1515_strict_miss_diagnostics.py new file mode 100644 index 00000000..64ded71c --- /dev/null +++ b/scripts/dev1515_strict_miss_diagnostics.py @@ -0,0 +1,152 @@ +"""DEV-1515 session-4 — strict-miss diagnostics summary. + +Walks the latest-run annotation per instance (mini-interact + livesqlbench), +filters to cascade-fail submissions (those whose grader produced a +``miss_diagnostics`` block), and prints: + +1. Per-instance row showing rowset shape, column shape, table-set + match, group_by signals, and the full flag list. +2. Per-flag tally — flags are NOT mutually exclusive so the totals + overlap. +3. An instance × flag matrix so co-occurring flags are easy to scan. + +Run: + + env -u SSH_AUTH_SOCK uv run python scripts/dev1515_strict_miss_diagnostics.py +""" +from __future__ import annotations + +import json +import re +from collections import Counter +from pathlib import Path + +from bird_interact_agents import paths + +# Walk both the hyphen + underscore tree to be defensive about the +# benchmark-name canonicalisation split that earlier sessions hit. +ROOTS = ( + paths.annotations_root() / "mini-interact", + paths.annotations_root() / "mini_interact", + paths.annotations_root() / "livesqlbench", +) + +SUB_RE = re.compile(r"^(.+)\.submission\.(.+)\.json$") + + +def _walk_latest_submissions() -> dict[str, dict]: + """Return {instance_id: latest submission JSON}. Latest = lex-max + of the run_id suffix; works because run_ids are timestamped + ``YYYYMMDDtHHMM…``.""" + by_inst: dict[str, tuple[str, dict]] = {} + for root in ROOTS: + if not root.exists(): + continue + for p in root.glob("*/*.submission.*.json"): + m = SUB_RE.match(p.name) + if not m: + continue + iid, run_id = m.group(1), m.group(2) + cur = by_inst.get(iid) + if cur is None or run_id > cur[0]: + by_inst[iid] = (run_id, json.loads(p.read_text())) + return {iid: v for iid, (_run, v) in by_inst.items()} + + +def main() -> None: + submissions = _walk_latest_submissions() + strict_misses: list[tuple[str, dict, dict]] = [] + for iid, ann in submissions.items(): + ev = ann.get("evaluation", {}) + md = ev.get("miss_diagnostics") + if md is None: + continue + strict_misses.append((iid, ann, md)) + strict_misses.sort() + + print("=" * 110) + print( + f"DEV-1515 strict-miss diagnostics — " + f"{len(strict_misses)} instances (cascade-fail)" + ) + print("=" * 110) + print() + + if not strict_misses: + print("No cascade-fail instances on disk. Nothing to diagnose.") + return + + # ---- Per-instance table ---------------------------------------------- + print("Per-instance breakdown") + print( + f" {'instance':<42} {'rows(a/g)':>10} {'cols(a/g)':>10} " + f"{'rel':<20} {'tbls':<5} {'agg(a/g)':<10} flags" + ) + print(" " + "-" * 108) + for iid, _ann, md in strict_misses: + rows = f"{md['agent_row_count']}/{md['best_variant_row_count']}" + cols = f"{md['agent_column_count']}/{md['best_variant_column_count']}" + rel = md["rowset_relation_to_best"] + tbls = ( + "match" if md.get("table_set_match") is True + else "diff" if md.get("table_set_match") is False + else "?" + ) + a_agg = md.get("agent_has_aggregate") + b_agg = md.get("best_variant_has_aggregate") + agg = f"{_b(a_agg)}/{_b(b_agg)}" + flags = ", ".join(md["miss_patterns"]) or "—" + print( + f" {iid:<42} {rows:>10} {cols:>10} {rel:<20} {tbls:<5} " + f"{agg:<10} {flags}" + ) + + # ---- Per-flag tally -------------------------------------------------- + counter: Counter[str] = Counter() + interactive_count = 0 + for _iid, _ann, md in strict_misses: + if md.get("user_sim_n_asks") is not None: + interactive_count += 1 + for f in md["miss_patterns"]: + counter[f] += 1 + + print() + print("Per-flag tally (each instance can carry multiple flags):") + for flag, count in counter.most_common(): + if flag == "never_asked_user": + note = ( + f" (interactive-only — {interactive_count} of " + f"{len(strict_misses)} cascade-fail rows are interactive)" + ) + else: + note = "" + print(f" {flag:<32} {count}{note}") + + # ---- Instance × flag matrix ----------------------------------------- + all_flags = sorted(counter) + if all_flags: + print() + print("Instance × flag matrix (x = flag fired):") + header = " " + " " * 42 + " ".join( + f"{f[:10]:>10}" for f in all_flags + ) + print(header) + for iid, _ann, md in strict_misses: + cells = " ".join( + f"{'x':>10}" if f in md["miss_patterns"] + else f"{'.':>10}" + for f in all_flags + ) + print(f" {iid:<42}{cells}") + + +def _b(v: object) -> str: + if v is True: + return "T" + if v is False: + return "F" + return "?" + + +if __name__ == "__main__": + main() diff --git a/src/bird_interact_agents/eval/annotate.py b/src/bird_interact_agents/eval/annotate.py index 4ce04a7b..0d144de6 100644 --- a/src/bird_interact_agents/eval/annotate.py +++ b/src/bird_interact_agents/eval/annotate.py @@ -185,6 +185,7 @@ def _eval_from_cascade(cascade: Any, epsilon: float = 1e-6) -> SubmissionEvaluat verdict=verdict_label_from_cascade(cascade), # type: ignore[arg-type] matched_variant_id=cascade.matched_variant_id, rationale="", + miss_diagnostics=getattr(cascade, "miss_diagnostics", None), ) diff --git a/src/bird_interact_agents/eval/annotation_schema.py b/src/bird_interact_agents/eval/annotation_schema.py index cf6237a6..51614ed9 100644 --- a/src/bird_interact_agents/eval/annotation_schema.py +++ b/src/bird_interact_agents/eval/annotation_schema.py @@ -299,6 +299,104 @@ class SubmissionMetadata(BaseModel): n_ask_user_calls: Optional[int] = None +MissPattern = Literal[ + "sql_execution_error", + "sql_parse_error", + "empty_agent_result", + "wrong_table_set", + "aggregation_shape_mismatch", + "column_projection_mismatch", + "predicate_count_mismatch", + "having_presence_mismatch", + "limit_presence_mismatch", + "disjoint_rowset", + "partial_match_overlap", + "agent_undercount", + "agent_overcount", + "never_asked_user", +] + + +class MissDiagnostics(BaseModel): + """Diagnostic signals collected at grading time for cascade-fail + submissions. Populated when the cascade has no passing tier + (every N1-N9 returned False). + + Comparison reference is the BEST-OVERLAP audited variant — the + variant with the largest multiset (bag) intersection with the + agent's rowset. Tie-break: prefer the primary variant; then + alphabetical variant_id. Single comparison target keeps the schema + flat; the variant identifier is recorded so downstream consumers + know which variant the signals are measured against. + + Works uniformly across interactive (a-interact) and one-shot + benchmarks. Interactive-only fields stay None when the benchmark + has no user-sim. + + SQL-derived fields are Optional[T] = None when sqlglot can't + parse the corresponding SQL. The ``*_sql_parse_ok`` booleans + + ``*_sql_parse_error`` excerpts make the parse state explicit + so downstream consumers don't read a False/0 as a real signal. + """ + model_config = ConfigDict(extra="forbid") + + # Comparison reference identifier. + best_variant_id: str + + # Row counts (multiset cardinality matches grader bag semantics). + agent_row_count: int + best_variant_row_count: int + original_gold_row_count: Optional[int] = None + overlap_with_best: int + rowset_relation_to_best: RowsetRelation + + # Column shape. + agent_column_count: int + best_variant_column_count: int + column_count_match: bool + column_name_match_case_insensitive: bool + column_order_match: bool + agent_columns: List[str] = Field(default_factory=list) + best_variant_columns: List[str] = Field(default_factory=list) + first_divergent_cell_diff: Optional[str] = None + + # SQL parse status gates the SQL-derived fields below. + agent_sql_parse_ok: bool + best_variant_sql_parse_ok: bool + agent_sql_parse_error: Optional[str] = None + best_variant_sql_parse_error: Optional[str] = None + + # SQL-derived signals — Optional[T]=None on the parse-failed side. + # Tables: base tables only, CTE / derived aliases excluded; + # alphabetically sorted for stable JSON diffs. + agent_tables_referenced: Optional[List[str]] = None + best_variant_tables_referenced: Optional[List[str]] = None + table_set_match: Optional[bool] = None + agent_has_group_by: Optional[bool] = None + best_variant_has_group_by: Optional[bool] = None + agent_has_aggregate: Optional[bool] = None + best_variant_has_aggregate: Optional[bool] = None + agent_join_count: Optional[int] = None + best_variant_join_count: Optional[int] = None + agent_where_conjunct_count: Optional[int] = None + best_variant_where_conjunct_count: Optional[int] = None + agent_has_having: Optional[bool] = None + best_variant_has_having: Optional[bool] = None + agent_has_limit: Optional[bool] = None + best_variant_has_limit: Optional[bool] = None + + # Execution status of the agent's SQL. + agent_sql_executed_ok: bool + agent_sql_error_excerpt: Optional[str] = None + + # Interactive-only signal — None on one-shot benchmarks. + user_sim_n_asks: Optional[int] = None + + # Independent flag list — every applicable rule appends. Sorted + # alphabetically before persist for stable JSON diffs. + miss_patterns: List[MissPattern] = Field(default_factory=list) + + class SubmissionEvaluation(BaseModel): """Cascading evaluation, most stringent → most lenient. @@ -325,6 +423,11 @@ class SubmissionEvaluation(BaseModel): verdict: SubmissionVerdict matched_variant_id: Optional[str] = None rationale: str = "" + miss_diagnostics: Optional[MissDiagnostics] = None + """DEV-1515 session-4 — populated only when the cascade has no + passing tier (strict miss). Captures rowset-shape, column-shape, + SQL-derived signals, and execution status to support downstream + failure-mode analysis without re-running queries.""" class FailureClassification(BaseModel): diff --git a/src/bird_interact_agents/eval/grade_in_place.py b/src/bird_interact_agents/eval/grade_in_place.py index 3395e13e..17327079 100644 --- a/src/bird_interact_agents/eval/grade_in_place.py +++ b/src/bird_interact_agents/eval/grade_in_place.py @@ -39,6 +39,30 @@ def _verdict_to_phase(b: bool) -> PhaseVerdict: return "pass" if b else "fail" +def _failure_details_for( + cascade: CascadeVerdict, primary: str, +) -> str: + """Return a one-line free-form human summary for + ``FailureClassification.details``. Downstream consumers MUST go + through ``cascade.miss_diagnostics.miss_patterns`` for structured + signals; this string is for humans only.""" + if primary == "agent_miss" and cascade.miss_diagnostics is not None: + md = cascade.miss_diagnostics + return ( + f"strict miss vs best_variant={md.best_variant_id!r}; " + f"patterns={md.miss_patterns}" + ) + if primary == "other": + return ( + "Strict miss across all cascade tiers; human review " + "pending — pick the specific failure class." + ) + return ( + "Auto-classified from cascade verdict; no human review " + "needed for no_fail / cascade-tier categories." + ) + + def verdict_label_from_cascade(cascade: CascadeVerdict) -> str: """Map a cascade verdict → the ``SubmissionEvaluation.verdict`` label. @@ -96,8 +120,11 @@ def _auto_failure_class(cascade: CascadeVerdict) -> tuple[str, bool, str]: return ("column_order", False, "grader") if cascade.n9_case_fold: return ("case_sensitivity", False, "grader") - # Genuine strict miss — let the human classify. - return ("other", True, "other") + # Genuine strict miss across every cascade tier — agent miss. The + # rich diagnostic detail lives on ``ev.miss_diagnostics`` (DEV-1515 + # session-4); ``FailureClassification.details`` carries a one-line + # human summary derived from ``miss_diagnostics.miss_patterns``. + return ("agent_miss", True, "agent") def _build_submission_annotation( @@ -138,6 +165,7 @@ def _build_submission_annotation( verdict=verdict_label, # type: ignore[arg-type] matched_variant_id=cascade.matched_variant_id, rationale="", + miss_diagnostics=cascade.miss_diagnostics, ) task_ann_ref = ( @@ -167,13 +195,7 @@ def _build_submission_annotation( primary=auto_primary, # type: ignore[arg-type] agent_at_fault=auto_at_fault, remediation_target=auto_remediation, # type: ignore[arg-type] - details=( - "Auto-classified from cascade verdict; no human review " - "needed for no_fail / cascade-tier categories." - if auto_primary != "other" - else "Strict miss across all cascade tiers; human " - "review pending — pick the specific failure class." - ), + details=_failure_details_for(cascade, auto_primary), ), decision_point=None, user_sim_interaction=( diff --git a/src/bird_interact_agents/eval/tolerant_grader.py b/src/bird_interact_agents/eval/tolerant_grader.py index 14e50d2e..40a8c8c8 100644 --- a/src/bird_interact_agents/eval/tolerant_grader.py +++ b/src/bird_interact_agents/eval/tolerant_grader.py @@ -17,6 +17,7 @@ import json import logging import sqlite3 +from collections import Counter from pathlib import Path from typing import Any, Callable, Iterable, List, Optional, Protocol, Sequence, Tuple @@ -25,6 +26,8 @@ from pydantic import BaseModel, ConfigDict, Field from bird_interact_agents.eval.annotation_schema import ( + MissDiagnostics, + MissPattern, PhaseVerdict, RowsetRelation, TaskAnnotation, @@ -525,6 +528,7 @@ class CascadeVerdict(BaseModel): novel_reading_judgment: Optional[PhaseVerdict] = None variant_matches: List[VariantMatch] = Field(default_factory=list) rowset_relations: List[VariantMatch] = Field(default_factory=list) + miss_diagnostics: Optional["MissDiagnostics"] = None def _multi_sql_execute( @@ -581,6 +585,7 @@ def grade_submission( executor: Optional[ExecutorProtocol] = None, llm_judge: Optional[Any] = None, epsilon: float = 1e-6, + user_sim_n_asks: Optional[int] = None, ) -> CascadeVerdict: """Compute the 8-row cascade for a single submission. @@ -592,13 +597,30 @@ def grade_submission( * N4 uses the ORIGINAL gold's ORDER BY (locked simplification). * N5 fires ONLY when ``task_annotation.metadata_sufficiency.verdict`` is ``"insufficient"`` AND N4 didn't already pass. + * ``user_sim_n_asks``: None for one-shot benchmarks (livesqlbench); + int for interactive benchmarks (mini-interact). Affects the + ``never_asked_user`` diagnostic flag on strict-miss verdicts only. """ if executor is None: executor = default_executor # type: ignore[assignment] assert executor is not None # narrowing - # 1) Run predicted + original gold + each variant. - pred_rows, pred_cols = executor(submitted_sql, db_path=db_path, conn=conn) + # 1) Run predicted + original gold + each variant. The agent's + # SQL is wrapped in try/except so a syntax / runtime error + # produces an empty rowset + an error excerpt rather than aborting + # grading — the cascade then naturally fails every tier and the + # diagnostics path attaches the `sql_execution_error` flag. + agent_sql_executed_ok = True + agent_sql_error_excerpt: Optional[str] = None + try: + pred_rows, pred_cols = executor( + submitted_sql, db_path=db_path, conn=conn, + ) + except Exception as exc: # noqa: BLE001 + agent_sql_executed_ok = False + agent_sql_error_excerpt = f"{type(exc).__name__}: {exc}"[:200] + pred_rows, pred_cols = [], [] + orig_rows, orig_cols = _multi_sql_execute( list(original_sol_sql), db_path=db_path, conn=conn, executor=executor, ) @@ -608,9 +630,16 @@ def grade_submission( sqls = list(v.get("audited_sol_sql") or []) if not sqls: continue - v_rows, v_cols = _multi_sql_execute( - sqls, db_path=db_path, conn=conn, executor=executor, - ) + try: + v_rows, v_cols = _multi_sql_execute( + sqls, db_path=db_path, conn=conn, executor=executor, + ) + except Exception: # noqa: BLE001 + # Variant SQL didn't execute — treat as empty rowset so the + # cascade can still complete and diagnostics can flag the + # downstream sqlglot parse failure. Better than crashing + # grading on a single broken variant. + v_rows, v_cols = [], [] variant_results.append((v, v_rows, v_cols)) # 2) N1 — original gold strict. @@ -767,14 +796,381 @@ def grade_submission( "n9_case_fold": n9, } enforced = enforce_monotone_cascade(raw) + + # 8) Strict-miss diagnostics — populated ONLY when no cascade tier + # passed AND at least one audited variant exists. Captures rowset + # / column / SQL signals against the best-overlap audited variant + # so downstream tooling can break down failure modes without + # re-running queries. The no-variants case (implicit annotation + # factory) leaves miss_diagnostics=None — there's no canonical + # gold to diagnose against. + miss_diagnostics: Optional[MissDiagnostics] = None + if not enforced["n9_case_fold"] and variant_results: + miss_diagnostics = _compute_miss_diagnostics( + pred_rows=pred_rows, + pred_cols=list(pred_cols), + agent_sql=submitted_sql, + agent_sql_executed_ok=agent_sql_executed_ok, + agent_sql_error_excerpt=agent_sql_error_excerpt, + variant_results=variant_results, + original_sol_sql=list(original_sol_sql), + original_rows=orig_rows, + user_sim_n_asks=user_sim_n_asks, + ) + return CascadeVerdict( **enforced, matched_variant_id=matched_variant, novel_reading_judgment=novel_judgment, variant_matches=info_matches, + miss_diagnostics=miss_diagnostics, ) +# --------------------------------------------------------------------------- +# Strict-miss diagnostics (DEV-1515 session-4) +# --------------------------------------------------------------------------- + + +def _bag(rows: Sequence[Sequence]) -> Counter: + """Multiset of canonical row repr — duplicates preserved.""" + return Counter(_canonical_repr(r) for r in rows) + + +def _bag_relation( + *, + pred: Sequence[Sequence], + gold: Sequence[Sequence], +) -> RowsetRelation: + """Bag-aware set relation. Same semantics as + ``classify_rowset_relation`` but uses multiset (Counter) + comparisons so duplicate rows are honoured the way the grader's + ``_set_equal`` does.""" + p = _bag(pred) + g = _bag(gold) + if not p and not g: + return "equal_rowset" + if p == g: + return "equal_rowset" + p_le_g = all(p[k] <= g[k] for k in p) + g_le_p = all(g[k] <= p[k] for k in g) + overlap_keys = set(p) & set(g) + overlap = sum(min(p[k], g[k]) for k in overlap_keys) + if p_le_g: + return "strict_subset_of" + if g_le_p: + return "strict_superset_of" + if overlap == 0: + return "disjoint" + return "overlapping" + + +def _bag_overlap( + pred: Sequence[Sequence], + gold: Sequence[Sequence], +) -> int: + """Multiset intersection cardinality (Codex bag-semantics req).""" + p = _bag(pred) + g = _bag(gold) + keys = set(p) & set(g) + return sum(min(p[k], g[k]) for k in keys) + + +def _pick_best_overlap_variant( + *, + pred_rows: Sequence[Sequence], + variant_results: List[Tuple[dict, Sequence[Sequence], Sequence[str]]], +) -> Tuple[dict, Sequence[Sequence], Sequence[str]]: + """Pick the audited variant with the largest multiset overlap with + the agent's rowset. Tie-break: primary > alphabetical variant_id. + """ + assert variant_results, "cannot pick best-overlap from empty variant list" + scored = [] + for v_meta, v_rows, v_cols in variant_results: + overlap = _bag_overlap(pred_rows, v_rows) + # Sort key: (-overlap, not_primary, variant_id). + # Larger overlap first; primary preferred on ties; then alpha. + is_primary = bool(v_meta.get("primary")) + scored.append( + (-overlap, not is_primary, str(v_meta.get("variant_id") or ""), + (v_meta, v_rows, v_cols)), + ) + scored.sort() + return scored[0][3] + + +def _parse_sql(sql: str) -> Tuple[bool, Optional[str], Optional[sg_expr.Expression]]: + """Parse ``sql`` with sqlglot's sqlite dialect. Return + ``(ok, error_excerpt, expression)``. On parse failure, ok=False + and error_excerpt carries the first 200 chars of the exception.""" + try: + parsed = sqlglot.parse_one(sql, read="sqlite") + return True, None, parsed + except Exception as exc: # noqa: BLE001 + return False, f"{type(exc).__name__}: {exc}"[:200], None + + +def _base_tables_from_expression( + expr: Optional[sg_expr.Expression], +) -> Optional[List[str]]: + """Walk ``expr`` and return the list of BASE tables — CTE names + and derived-table aliases excluded. Returns alphabetically sorted + list of unique base-table names. Returns None when expr is None + (caller should propagate the None sentinel).""" + if expr is None: + return None + cte_names: set[str] = set() + with_clause = expr.find(sg_expr.With) + if with_clause is not None: + for cte in with_clause.find_all(sg_expr.CTE): + alias = cte.alias_or_name + if alias: + cte_names.add(alias.lower()) + tables: set[str] = set() + for t in expr.find_all(sg_expr.Table): + name = (t.name or "").lower() + if not name: + continue + if name in cte_names: + continue + tables.add(name) + return sorted(tables) + + +def _has_group_by(expr: Optional[sg_expr.Expression]) -> Optional[bool]: + if expr is None: + return None + return any(expr.find_all(sg_expr.Group)) + + +def _has_aggregate(expr: Optional[sg_expr.Expression]) -> Optional[bool]: + if expr is None: + return None + return any(expr.find_all(sg_expr.AggFunc)) + + +def _join_count(expr: Optional[sg_expr.Expression]) -> Optional[int]: + if expr is None: + return None + return sum(1 for _ in expr.find_all(sg_expr.Join)) + + +def _has_having(expr: Optional[sg_expr.Expression]) -> Optional[bool]: + if expr is None: + return None + return any(expr.find_all(sg_expr.Having)) + + +def _has_limit(expr: Optional[sg_expr.Expression]) -> Optional[bool]: + if expr is None: + return None + return any(expr.find_all(sg_expr.Limit)) + + +def _where_conjunct_count(expr: Optional[sg_expr.Expression]) -> Optional[int]: + """Count top-level AND-conjuncts in the outer SELECT's WHERE clause. + Zero if no WHERE. A single predicate counts as 1.""" + if expr is None: + return None + where = expr.find(sg_expr.Where) + if where is None: + return 0 + # Flatten the AND tree into atoms. + count = 0 + stack = [where.this] + while stack: + node = stack.pop() + if isinstance(node, sg_expr.And): + stack.append(node.this) + stack.append(node.expression) + else: + count += 1 + return count + + +def _column_match_signals( + *, + agent_cols: List[str], + gold_cols: List[str], +) -> Tuple[bool, bool, bool]: + """(column_count_match, name_match_case_insensitive, order_match).""" + count_match = len(agent_cols) == len(gold_cols) + a_lower = [c.lower() for c in agent_cols] + g_lower = [c.lower() for c in gold_cols] + name_match_ci = set(a_lower) == set(g_lower) and count_match + order_match = a_lower == g_lower + return count_match, name_match_ci, order_match + + +def _compute_miss_diagnostics( + *, + pred_rows: Sequence[Sequence], + pred_cols: List[str], + agent_sql: str, + agent_sql_executed_ok: bool, + agent_sql_error_excerpt: Optional[str], + variant_results: List[Tuple[dict, Sequence[Sequence], Sequence[str]]], + original_sol_sql: List[str], + original_rows: Sequence[Sequence], + user_sim_n_asks: Optional[int], +) -> MissDiagnostics: + """Build the structured diagnostics for a strict-miss cascade. + + Compares the agent's rowset to the BEST-OVERLAP audited variant. + SQL-derived signals are populated only when sqlglot parsing + succeeds for the relevant side; otherwise the corresponding + Optional[T] field stays None and ``sql_parse_error`` lands in the + flag list. Multi-statement gold (CREATE TEMP + final SELECT) + triggers a defensive AssertionError — the SELECT-task contract + is single-statement. + """ + # Defensive guards (single-statement gold contract). + for v_meta, _v_rows, _v_cols in variant_results: + v_sqls = list(v_meta.get("audited_sol_sql") or []) + assert len(v_sqls) <= 1, ( + f"diagnostics only support single-statement audited_sol_sql; " + f"variant {v_meta.get('variant_id')!r} has {len(v_sqls)} stmts " + f"(multi-statement is M-task territory, out of scope)" + ) + assert len(original_sol_sql) <= 1, ( + f"diagnostics only support single-statement original_sol_sql; " + f"got {len(original_sol_sql)} stmts (multi-statement is M-task " + f"territory, out of scope)" + ) + if not variant_results: + # Should be impossible (cascade would short-circuit) but guard. + raise RuntimeError( + "_compute_miss_diagnostics called without any audited variants", + ) + + best_meta, best_rows, best_cols = _pick_best_overlap_variant( + pred_rows=pred_rows, variant_results=variant_results, + ) + best_variant_id = str(best_meta.get("variant_id") or "") + best_sql = (best_meta.get("audited_sol_sql") or [""])[0] + + overlap = _bag_overlap(pred_rows, best_rows) + relation = _bag_relation(pred=pred_rows, gold=best_rows) + + count_match, name_match_ci, order_match = _column_match_signals( + agent_cols=list(pred_cols), gold_cols=list(best_cols), + ) + + # First divergent cell (against the best variant). + _fdri, fdcd = _first_divergent_row(pred=pred_rows, gold=best_rows) + + # SQL parsing — each side independently. + a_ok, a_err, a_expr = _parse_sql(agent_sql) + b_ok, b_err, b_expr = _parse_sql(best_sql) + + agent_tables = _base_tables_from_expression(a_expr) if a_ok else None + best_tables = _base_tables_from_expression(b_expr) if b_ok else None + if a_ok and b_ok: + table_set_match: Optional[bool] = ( + set(agent_tables or []) == set(best_tables or []) + ) + else: + table_set_match = None + + md = MissDiagnostics( + best_variant_id=best_variant_id, + agent_row_count=len(pred_rows), + best_variant_row_count=len(best_rows), + original_gold_row_count=len(original_rows) if original_sol_sql else None, + overlap_with_best=overlap, + rowset_relation_to_best=relation, + agent_column_count=len(pred_cols), + best_variant_column_count=len(best_cols), + column_count_match=count_match, + column_name_match_case_insensitive=name_match_ci, + column_order_match=order_match, + agent_columns=list(pred_cols), + best_variant_columns=list(best_cols), + first_divergent_cell_diff=fdcd, + agent_sql_parse_ok=a_ok, + best_variant_sql_parse_ok=b_ok, + agent_sql_parse_error=a_err, + best_variant_sql_parse_error=b_err, + agent_tables_referenced=agent_tables, + best_variant_tables_referenced=best_tables, + table_set_match=table_set_match, + agent_has_group_by=_has_group_by(a_expr) if a_ok else None, + best_variant_has_group_by=_has_group_by(b_expr) if b_ok else None, + agent_has_aggregate=_has_aggregate(a_expr) if a_ok else None, + best_variant_has_aggregate=_has_aggregate(b_expr) if b_ok else None, + agent_join_count=_join_count(a_expr) if a_ok else None, + best_variant_join_count=_join_count(b_expr) if b_ok else None, + agent_where_conjunct_count=_where_conjunct_count(a_expr) if a_ok else None, + best_variant_where_conjunct_count=( + _where_conjunct_count(b_expr) if b_ok else None + ), + agent_has_having=_has_having(a_expr) if a_ok else None, + best_variant_has_having=_has_having(b_expr) if b_ok else None, + agent_has_limit=_has_limit(a_expr) if a_ok else None, + best_variant_has_limit=_has_limit(b_expr) if b_ok else None, + agent_sql_executed_ok=agent_sql_executed_ok, + agent_sql_error_excerpt=agent_sql_error_excerpt, + user_sim_n_asks=user_sim_n_asks, + miss_patterns=[], + ) + + # Independent flag rules — every applicable rule appends. + flags: List[MissPattern] = [] + if not agent_sql_executed_ok: + flags.append("sql_execution_error") + if (not a_ok) or (not b_ok): + flags.append("sql_parse_error") + if md.agent_row_count == 0 and md.best_variant_row_count > 0: + flags.append("empty_agent_result") + if table_set_match is False: + flags.append("wrong_table_set") + if ( + md.agent_has_group_by is not None + and md.best_variant_has_group_by is not None + and md.agent_has_aggregate is not None + and md.best_variant_has_aggregate is not None + and ( + md.agent_has_group_by != md.best_variant_has_group_by + or md.agent_has_aggregate != md.best_variant_has_aggregate + ) + ): + flags.append("aggregation_shape_mismatch") + if (not count_match) or (not name_match_ci): + flags.append("column_projection_mismatch") + if ( + md.agent_where_conjunct_count is not None + and md.best_variant_where_conjunct_count is not None + and md.agent_where_conjunct_count != md.best_variant_where_conjunct_count + ): + flags.append("predicate_count_mismatch") + if ( + md.agent_has_having is not None + and md.best_variant_has_having is not None + and md.agent_has_having != md.best_variant_has_having + ): + flags.append("having_presence_mismatch") + if ( + md.agent_has_limit is not None + and md.best_variant_has_limit is not None + and md.agent_has_limit != md.best_variant_has_limit + ): + flags.append("limit_presence_mismatch") + if relation == "disjoint": + flags.append("disjoint_rowset") + elif relation == "overlapping": + flags.append("partial_match_overlap") + elif relation == "strict_subset_of": + flags.append("agent_undercount") + elif relation == "strict_superset_of": + flags.append("agent_overcount") + if user_sim_n_asks is not None and user_sim_n_asks == 0: + flags.append("never_asked_user") + + # Sort alphabetically for stable JSON diffs. + md.miss_patterns = sorted(flags) + return md + + def _annotation_hash(ann: TaskAnnotation) -> str: return hashlib.sha256(ann.model_dump_json().encode()).hexdigest() diff --git a/tests/test_miss_diagnostics.py b/tests/test_miss_diagnostics.py new file mode 100644 index 00000000..d2ad00f5 --- /dev/null +++ b/tests/test_miss_diagnostics.py @@ -0,0 +1,1113 @@ +"""DEV-1515 session 4: failure-mode diagnostics at grading time. + +Tests the ``MissDiagnostics`` model + the diagnostics population in +``grade_submission`` for cascade-fail submissions. Per the v3 spec +(see ``plans/read-all-the-comments-peaceful-karp.md``): + +* every cascade-fail populates ``cascade.miss_diagnostics`` +* comparison reference is the BEST-OVERLAP audited variant + (multiset cardinality; tie-break primary > alphabetical variant_id) +* SQL parse failure → nullable signals + parse_ok=False +* SQL execution failure → ``pred_rows=[]`` falls through cascade + + ``sql_execution_error`` flag + error excerpt +* multiple flags can fire simultaneously (independent rules) +* one-shot benchmarks (livesqlbench) skip the ``never_asked_user`` + signal entirely +""" +from __future__ import annotations + +import sqlite3 +from pathlib import Path +from typing import Optional + + +# --------------------------------------------------------------------------- +# Helpers — sqlite DB fixture, task annotation, audited-row builder +# --------------------------------------------------------------------------- + + +def _build_db(tmp_path: Path) -> Path: + """Build a small sqlite DB with two tables for the diagnostics fixtures.""" + db = tmp_path / "diag.sqlite" + con = sqlite3.connect(str(db)) + try: + con.execute("CREATE TABLE t1 (id INTEGER, val TEXT)") + con.execute("CREATE TABLE t2 (id INTEGER, name TEXT)") + con.executemany( + "INSERT INTO t1 (id, val) VALUES (?, ?)", + [(1, "a"), (2, "b"), (3, "c"), (4, "d"), (5, "e")], + ) + con.executemany( + "INSERT INTO t2 (id, name) VALUES (?, ?)", + [(1, "alpha"), (2, "beta"), (3, "gamma")], + ) + con.commit() + finally: + con.close() + return db + + +def _task_annotation( + *, + instance_id: str = "alien_1", + verdict: str = "sufficient", + variant_ids: Optional[list[tuple[str, bool]]] = None, +): + """Build a TaskAnnotation. ``variant_ids`` is a list of (variant_id, + primary) pairs that mirrors the audited rows used by the test; + defaults to a single ``primary`` variant. Required so the + task annotation's `gold_variants` declares the same variants that + the test passes via `audited_gold_rows` — otherwise downstream + validation (or future grader changes that cross-check the two + sources) would fail.""" + from bird_interact_agents.eval.annotation_schema import ( + AuditedGoldRef, + GoldVariantRef, + MetadataSufficiency, + Provenance, + TaskAnnotation, + ) + if variant_ids is None: + variant_ids = [("primary", True)] + return TaskAnnotation( + instance_id=instance_id, + selected_database="alien", + annotated_by="test", + annotated_at="2026-05-31", + amb_user_query="x", + metadata_sufficiency=MetadataSufficiency( + verdict=verdict, rationale="r", + ), + gold_variants=[ + GoldVariantRef( + variant_id=vid, + interpretation="x", + primary=p, + audited_gold_ref=AuditedGoldRef( + file="audited_gold/mini_interact_audited.jsonl", + instance_id=instance_id, + variant_id=vid, + ), + ) + for (vid, p) in variant_ids + ], + provenance=Provenance( + task_jsonl_path="mini_interact.jsonl", + task_jsonl_instance_id=instance_id, + ), + ) + + +def _audited_row( + *, + variant_id: str, + primary: bool, + audited_sol_sql: list[str], + instance_id: str = "alien_1", +) -> dict: + return { + "instance_id": instance_id, + "selected_database": "alien", + "benchmark": "mini_interact", + "audit_status": "edited", + "original_sol_sql": ["SELECT id FROM t1 WHERE id < 0"], + "audited_sol_sql": audited_sol_sql, + "variant_id": variant_id, + "primary": primary, + "changes": [], + "reasoning_summary": "", + "skill_version": "audit-gold-sql/1.0", + "audited_at": "2026-05-30T00:00:00+00:00", + } + + +def _grade( + *, + db: Path, + submitted_sql: str, + audited_sol_sql_per_variant: list[tuple[str, bool, list[str]]], + original_sol_sql: Optional[list[str]] = None, + verdict: str = "sufficient", + user_sim_n_asks: Optional[int] = None, +): + """Wrapper that builds the inputs and calls grade_submission. The + test asserts on the returned CascadeVerdict.""" + from bird_interact_agents.eval.tolerant_grader import grade_submission + + audited = [ + _audited_row( + variant_id=vid, primary=p, audited_sol_sql=sqls, + ) + for (vid, p, sqls) in audited_sol_sql_per_variant + ] + if original_sol_sql is None: + # Sentinel rowset that no test's agent SQL accidentally matches — + # otherwise an agent returning empty rows would spuriously + # pass N1 against an empty original gold and miss_diagnostics + # would stay None. + original_sol_sql = ["SELECT 'sentinel' AS marker FROM t1 WHERE id = 1"] + # Derive gold_variants from the audited list so task annotation + + # audited rows agree on variant identity (Codex minor #10). + variant_ids = [(vid, p) for (vid, p, _) in audited_sol_sql_per_variant] + return grade_submission( + task_annotation=_task_annotation(verdict=verdict, variant_ids=variant_ids), + audited_gold_rows=audited, + original_sol_sql=original_sol_sql, + submitted_sql=submitted_sql, + db_path=db, + conn=None, + user_sim_n_asks=user_sim_n_asks, + ) + + +# --------------------------------------------------------------------------- +# Cascade-pass leaves diagnostics None +# --------------------------------------------------------------------------- + + +def test_cascade_pass_at_n3_leaves_miss_diagnostics_none(tmp_path: Path): + """Diagnostics are computed ONLY on a strict miss. When the cascade + passes at N3 (strict set equality), miss_diagnostics is None.""" + db = _build_db(tmp_path) + verdict = _grade( + db=db, + submitted_sql="SELECT id FROM t1 WHERE id < 3 ORDER BY id", + audited_sol_sql_per_variant=[ + ("primary", True, ["SELECT id FROM t1 WHERE id IN (1, 2) ORDER BY id"]), + ], + ) + assert verdict.n3_any_audited_variant is True + assert verdict.miss_diagnostics is None + + +def test_cascade_pass_at_n6_leaves_miss_diagnostics_none(tmp_path: Path): + """Diagnostics stay None for any cascade tier pass, not just N3. + Construct an instance where strict equality misses but N6 numeric + epsilon flips the cascade to pass — and assert miss_diagnostics is + still None. The happy-path skip-diagnostics contract is per-cascade + not per-strict-tier.""" + db = _build_db(tmp_path) + # Agent: 12.345600001 (within 1e-6 of 12.3456); Gold: 12.3456 + verdict = _grade( + db=db, + submitted_sql="SELECT 12.345600001 AS v", + audited_sol_sql_per_variant=[ + ("primary", True, ["SELECT 12.3456 AS v"]), + ], + ) + # N3 strict misses, N6 numeric-epsilon passes + assert verdict.n3_any_audited_variant is False + assert verdict.n6_numeric_epsilon is True + assert verdict.miss_diagnostics is None + + +# --------------------------------------------------------------------------- +# Per-flag fixtures — each test engineers a scenario where ONLY the +# named flag should fire (modulo flags that are entailed by the scenario). +# --------------------------------------------------------------------------- + + +def test_flag_sql_execution_error(tmp_path: Path): + """Agent SQL raises at execution time. Cascade MUST complete with + all N-tiers False, pred_rows captured as empty, the error excerpt + captured, and the diagnostics populated with both + sql_execution_error AND empty_agent_result (since pred_rows=[] + while the best variant returns ≥1 row). + + Codex major #5: pin the full cascade-fall-through contract, not + just the executed_ok boolean.""" + db = _build_db(tmp_path) + verdict = _grade( + db=db, + submitted_sql="SELECT * FROM does_not_exist", + audited_sol_sql_per_variant=[ + ("primary", True, ["SELECT id FROM t1 ORDER BY id"]), + ], + ) + # Cascade fall-through: every tier False. + assert verdict.n1_original_gold is False + assert verdict.n2_audited_primary is False + assert verdict.n3_any_audited_variant is False + assert verdict.n4_tie_order is False + assert verdict.n6_numeric_epsilon is False + assert verdict.n7_trailing_whitespace is False + assert verdict.n8_column_order is False + assert verdict.n9_case_fold is False + + md = verdict.miss_diagnostics + assert md is not None + assert md.agent_sql_executed_ok is False + assert md.agent_sql_error_excerpt is not None + assert "no such table" in md.agent_sql_error_excerpt.lower() + assert md.agent_row_count == 0 + assert md.best_variant_row_count > 0 + assert "sql_execution_error" in md.miss_patterns + assert "empty_agent_result" in md.miss_patterns + + +def test_flag_sql_parse_error_agent_side(tmp_path: Path): + """Agent SQL is unparseable garbage; sqlglot fails. parse_ok flag + flips False; sql_parse_error flag fires; AGENT-side SQL-derived + fields are None (Optional sentinel) rather than False/0. + + The best-variant side parses fine — its fields are concrete.""" + db = _build_db(tmp_path) + verdict = _grade( + db=db, + submitted_sql="THIS IS NOT SQL AT ALL ;;", + audited_sol_sql_per_variant=[ + ("primary", True, ["SELECT id FROM t1 ORDER BY id"]), + ], + ) + md = verdict.miss_diagnostics + assert md is not None + assert md.agent_sql_parse_ok is False + assert md.agent_sql_parse_error is not None + assert md.agent_tables_referenced is None + assert md.agent_has_group_by is None + assert md.agent_has_aggregate is None + assert md.agent_join_count is None + assert md.agent_where_conjunct_count is None + assert md.agent_has_having is None + assert md.agent_has_limit is None + assert md.table_set_match is None + # Best-variant side parsed fine. + assert md.best_variant_sql_parse_ok is True + assert md.best_variant_sql_parse_error is None + assert md.best_variant_tables_referenced == ["t1"] + assert md.best_variant_has_group_by is False + assert "sql_parse_error" in md.miss_patterns + + +def test_flag_sql_parse_error_best_variant_side(tmp_path: Path): + """Mirror case: agent SQL parses fine but the best-variant audited + SQL is malformed. Codex major #4 — the rule fires on EITHER side + failing to parse, and the best-variant nullable fields must mirror + the agent-side behaviour.""" + db = _build_db(tmp_path) + verdict = _grade( + db=db, + submitted_sql="SELECT id FROM t1 WHERE id IN (4, 5)", + audited_sol_sql_per_variant=[ + ("primary", True, ["NOT A VALID SQL AT ALL ;;"]), + ], + ) + md = verdict.miss_diagnostics + assert md is not None + assert md.agent_sql_parse_ok is True + assert md.agent_sql_parse_error is None + assert md.agent_tables_referenced == ["t1"] + # Best-variant side is the broken one. + assert md.best_variant_sql_parse_ok is False + assert md.best_variant_sql_parse_error is not None + assert md.best_variant_tables_referenced is None + assert md.best_variant_has_group_by is None + assert md.best_variant_has_aggregate is None + assert md.best_variant_join_count is None + assert md.best_variant_where_conjunct_count is None + assert md.best_variant_has_having is None + assert md.best_variant_has_limit is None + assert md.table_set_match is None + assert "sql_parse_error" in md.miss_patterns + + +def test_flag_empty_agent_result(tmp_path: Path): + """Agent returns zero rows; best variant returns non-empty.""" + db = _build_db(tmp_path) + verdict = _grade( + db=db, + submitted_sql="SELECT id FROM t1 WHERE id > 999", # 0 rows + audited_sol_sql_per_variant=[ + ("primary", True, ["SELECT id FROM t1 WHERE id <= 3 ORDER BY id"]), + ], + ) + md = verdict.miss_diagnostics + assert md is not None + assert md.agent_row_count == 0 + assert md.best_variant_row_count > 0 + assert "empty_agent_result" in md.miss_patterns + + +def test_flag_wrong_table_set(tmp_path: Path): + """Agent references t2; gold references t1. Same column shape so + column_projection_mismatch doesn't fire, but table_set_match=False + so wrong_table_set does.""" + db = _build_db(tmp_path) + verdict = _grade( + db=db, + submitted_sql="SELECT id FROM t2 ORDER BY id", + audited_sol_sql_per_variant=[ + ("primary", True, ["SELECT id FROM t1 ORDER BY id"]), + ], + ) + md = verdict.miss_diagnostics + assert md is not None + assert md.agent_sql_parse_ok is True + assert md.best_variant_sql_parse_ok is True + assert md.agent_tables_referenced == ["t2"] + assert md.best_variant_tables_referenced == ["t1"] + assert md.table_set_match is False + assert "wrong_table_set" in md.miss_patterns + + +def test_flag_aggregation_shape_mismatch(tmp_path: Path): + """Agent has no GROUP BY / no aggregate; gold has both. Flag fires.""" + db = _build_db(tmp_path) + verdict = _grade( + db=db, + submitted_sql="SELECT id FROM t1 ORDER BY id", + audited_sol_sql_per_variant=[ + ("primary", True, + ["SELECT COUNT(*) AS n FROM t1 GROUP BY val"]), + ], + ) + md = verdict.miss_diagnostics + assert md is not None + assert md.agent_has_group_by is False + assert md.best_variant_has_group_by is True + assert md.agent_has_aggregate is False + assert md.best_variant_has_aggregate is True + assert "aggregation_shape_mismatch" in md.miss_patterns + + +def test_flag_column_projection_mismatch(tmp_path: Path): + """Agent projects 1 col; gold projects 2. column_count_match=False.""" + db = _build_db(tmp_path) + verdict = _grade( + db=db, + submitted_sql="SELECT id FROM t1 ORDER BY id", + audited_sol_sql_per_variant=[ + ("primary", True, ["SELECT id, val FROM t1 ORDER BY id"]), + ], + ) + md = verdict.miss_diagnostics + assert md is not None + assert md.column_count_match is False + assert "column_projection_mismatch" in md.miss_patterns + + +def test_flag_predicate_count_mismatch(tmp_path: Path): + """Agent has 1 WHERE conjunct; gold has 2. predicate_count_mismatch + fires. Rowsets MUST differ for the cascade to reach the diagnostics + path — engineer a fixture where the extra conjunct on the gold side + actually filters rows differently.""" + db = _build_db(tmp_path) + verdict = _grade( + db=db, + submitted_sql="SELECT id FROM t1 WHERE id > 0", # 5 rows, 1 conjunct + audited_sol_sql_per_variant=[ + ("primary", True, + ["SELECT id FROM t1 WHERE id > 0 AND id < 3"]), # 2 rows, 2 conjuncts + ], + ) + md = verdict.miss_diagnostics + assert md is not None + assert md.agent_where_conjunct_count == 1 + assert md.best_variant_where_conjunct_count == 2 + assert "predicate_count_mismatch" in md.miss_patterns + + +def test_flag_having_presence_mismatch(tmp_path: Path): + """Agent has no HAVING; gold has HAVING that actually filters rows. + Rowsets must differ so the cascade reaches diagnostics.""" + db = _build_db(tmp_path) + # Agent: all 5 val groups, no filter. Gold: same shape but HAVING + # restricts to groups with COUNT > 99 (none qualify → 0 rows). + verdict = _grade( + db=db, + submitted_sql="SELECT val, COUNT(*) FROM t1 GROUP BY val", + audited_sol_sql_per_variant=[ + ("primary", True, + ["SELECT val, COUNT(*) FROM t1 GROUP BY val " + "HAVING COUNT(*) > 99"]), + ], + ) + md = verdict.miss_diagnostics + assert md is not None + assert md.agent_has_having is False + assert md.best_variant_has_having is True + assert "having_presence_mismatch" in md.miss_patterns + + +def test_flag_limit_presence_mismatch(tmp_path: Path): + """Agent has LIMIT; gold has no LIMIT. limit_presence_mismatch fires.""" + db = _build_db(tmp_path) + verdict = _grade( + db=db, + submitted_sql="SELECT id FROM t1 ORDER BY id LIMIT 2", + audited_sol_sql_per_variant=[ + ("primary", True, ["SELECT id FROM t1 ORDER BY id"]), + ], + ) + md = verdict.miss_diagnostics + assert md is not None + assert md.agent_has_limit is True + assert md.best_variant_has_limit is False + assert "limit_presence_mismatch" in md.miss_patterns + + +def test_flag_disjoint_rowset(tmp_path: Path): + """Agent and gold rowsets are disjoint. Negative-coverage: the + three other rowset_relation flags must NOT also fire (mutually + exclusive — Codex major #6).""" + db = _build_db(tmp_path) + verdict = _grade( + db=db, + submitted_sql="SELECT id FROM t1 WHERE id IN (1, 2)", + audited_sol_sql_per_variant=[ + ("primary", True, ["SELECT id FROM t1 WHERE id IN (4, 5)"]), + ], + ) + md = verdict.miss_diagnostics + assert md is not None + assert md.rowset_relation_to_best == "disjoint" + assert md.overlap_with_best == 0 + flags = set(md.miss_patterns) + assert "disjoint_rowset" in flags + for absent in ("partial_match_overlap", "agent_undercount", "agent_overcount"): + assert absent not in flags, ( + f"{absent!r} contradicts disjoint_rowset; saw {flags}" + ) + + +def test_flag_partial_match_overlap(tmp_path: Path): + """Agent and gold rowsets overlap but neither is subset of the + other. Other rowset flags MUST NOT fire.""" + db = _build_db(tmp_path) + verdict = _grade( + db=db, + submitted_sql="SELECT id FROM t1 WHERE id IN (1, 2, 3)", + audited_sol_sql_per_variant=[ + ("primary", True, ["SELECT id FROM t1 WHERE id IN (2, 3, 4)"]), + ], + ) + md = verdict.miss_diagnostics + assert md is not None + assert md.rowset_relation_to_best == "overlapping" + assert md.overlap_with_best > 0 + flags = set(md.miss_patterns) + assert "partial_match_overlap" in flags + for absent in ("disjoint_rowset", "agent_undercount", "agent_overcount"): + assert absent not in flags, ( + f"{absent!r} contradicts partial_match_overlap; saw {flags}" + ) + + +def test_flag_agent_undercount(tmp_path: Path): + """Agent is a strict subset of gold (over-restrictive filter). + Other rowset flags MUST NOT fire.""" + db = _build_db(tmp_path) + verdict = _grade( + db=db, + submitted_sql="SELECT id FROM t1 WHERE id IN (1, 2)", + audited_sol_sql_per_variant=[ + ("primary", True, + ["SELECT id FROM t1 WHERE id IN (1, 2, 3, 4)"]), + ], + ) + md = verdict.miss_diagnostics + assert md is not None + assert md.rowset_relation_to_best == "strict_subset_of" + flags = set(md.miss_patterns) + assert "agent_undercount" in flags + for absent in ("disjoint_rowset", "partial_match_overlap", "agent_overcount"): + assert absent not in flags, ( + f"{absent!r} contradicts agent_undercount; saw {flags}" + ) + + +def test_flag_agent_overcount(tmp_path: Path): + """Agent is a strict superset of gold (under-restrictive filter). + Other rowset flags MUST NOT fire.""" + db = _build_db(tmp_path) + verdict = _grade( + db=db, + submitted_sql="SELECT id FROM t1 WHERE id IN (1, 2, 3, 4)", + audited_sol_sql_per_variant=[ + ("primary", True, ["SELECT id FROM t1 WHERE id IN (1, 2)"]), + ], + ) + md = verdict.miss_diagnostics + assert md is not None + assert md.rowset_relation_to_best == "strict_superset_of" + flags = set(md.miss_patterns) + assert "agent_overcount" in flags + for absent in ("disjoint_rowset", "partial_match_overlap", "agent_undercount"): + assert absent not in flags, ( + f"{absent!r} contradicts agent_overcount; saw {flags}" + ) + + +def test_flag_never_asked_user_interactive_zero(tmp_path: Path): + """Interactive benchmark, agent didn't ask. user_sim_n_asks=0 flags + never_asked_user.""" + db = _build_db(tmp_path) + verdict = _grade( + db=db, + submitted_sql="SELECT id FROM t1 WHERE id IN (4, 5)", + audited_sol_sql_per_variant=[ + ("primary", True, ["SELECT id FROM t1 WHERE id IN (1, 2)"]), + ], + user_sim_n_asks=0, + ) + md = verdict.miss_diagnostics + assert md is not None + assert md.user_sim_n_asks == 0 + assert "never_asked_user" in md.miss_patterns + + +def test_interactive_with_asks_skips_never_asked_user(tmp_path: Path): + """Interactive benchmark, agent DID ask (n_asks > 0). never_asked_user + is NOT in the flag list; the field is set to the non-zero count.""" + db = _build_db(tmp_path) + verdict = _grade( + db=db, + submitted_sql="SELECT id FROM t1 WHERE id IN (4, 5)", + audited_sol_sql_per_variant=[ + ("primary", True, ["SELECT id FROM t1 WHERE id IN (1, 2)"]), + ], + user_sim_n_asks=3, + ) + md = verdict.miss_diagnostics + assert md is not None + assert md.user_sim_n_asks == 3 + assert "never_asked_user" not in md.miss_patterns + + +def test_one_shot_benchmark_skips_never_asked_user(tmp_path: Path): + """One-shot benchmark (no user-sim): user_sim_n_asks kwarg = None. + Field stored as None; never_asked_user NOT in flags.""" + db = _build_db(tmp_path) + verdict = _grade( + db=db, + submitted_sql="SELECT id FROM t1 WHERE id IN (4, 5)", + audited_sol_sql_per_variant=[ + ("primary", True, ["SELECT id FROM t1 WHERE id IN (1, 2)"]), + ], + user_sim_n_asks=None, + ) + md = verdict.miss_diagnostics + assert md is not None + assert md.user_sim_n_asks is None + assert "never_asked_user" not in md.miss_patterns + + +# --------------------------------------------------------------------------- +# Multi-flag fixture +# --------------------------------------------------------------------------- + + +def test_multi_flag_fixture_fires_every_applicable_flag(tmp_path: Path): + """Single submission that simultaneously trips multiple rules: + column projection differs, table set differs, agent has GROUP BY + that gold doesn't, agent has LIMIT gold doesn't, user_sim_n_asks=0. + Assert ALL five flags appear in miss_patterns (flags are NOT + first-match-wins).""" + db = _build_db(tmp_path) + verdict = _grade( + db=db, + submitted_sql=( + "SELECT id, name FROM t2 GROUP BY id, name LIMIT 2" + ), + audited_sol_sql_per_variant=[ + ("primary", True, ["SELECT id FROM t1 ORDER BY id"]), + ], + user_sim_n_asks=0, + ) + md = verdict.miss_diagnostics + assert md is not None + flags = set(md.miss_patterns) + for required in ( + "column_projection_mismatch", + "wrong_table_set", + "aggregation_shape_mismatch", + "limit_presence_mismatch", + "never_asked_user", + ): + assert required in flags, ( + f"expected {required!r} in {flags} (multi-flag scenario)" + ) + + +def test_multi_flag_fixture_with_sql_parse_error(tmp_path: Path): + """Codex minor #9 — second multi-flag fixture covering the planned + combination: empty_agent_result + wrong_table_set + never_asked_user + + sql_parse_error. Agent SQL is unparseable garbage; the cascade + falls through to all-False; multiple flags fire independently.""" + db = _build_db(tmp_path) + verdict = _grade( + db=db, + submitted_sql="NOT A VALID SQL AT ALL ;;", + audited_sol_sql_per_variant=[ + ("primary", True, ["SELECT id FROM t1 ORDER BY id"]), + ], + user_sim_n_asks=0, + ) + md = verdict.miss_diagnostics + assert md is not None + flags = set(md.miss_patterns) + for required in ( + "sql_parse_error", + "empty_agent_result", + "never_asked_user", + "sql_execution_error", # unparseable SQL also fails to execute + ): + assert required in flags, ( + f"expected {required!r} in {flags} " + f"(multi-flag parse-error scenario)" + ) + + +# --------------------------------------------------------------------------- +# Best-overlap variant selection +# --------------------------------------------------------------------------- + + +def test_best_overlap_picks_alt_variant_over_primary(tmp_path: Path): + """Same-column-shape rowsets; primary has 0 overlap with agent, + alt has strictly higher (non-zero) overlap. Cascade fails because + neither variant matches strictly. best_variant_id MUST be 'alt' — + a buggy implementation that always picks the primary would fail + this exact equality check (Codex critical).""" + db = _build_db(tmp_path) + # Agent: ids {1, 2, 3} (1-col) + # Primary: ids {7, 8, 9} (1-col) — overlap 0 + # Alt: ids {3, 4, 5} (1-col) — overlap 1 (the row (3,)) + verdict = _grade( + db=db, + submitted_sql="SELECT id FROM t1 WHERE id IN (1, 2, 3) ORDER BY id", + audited_sol_sql_per_variant=[ + ("primary", True, ["SELECT id FROM t1 WHERE id IN (7, 8, 9)"]), + ("alt", False, ["SELECT id FROM t1 WHERE id IN (3, 4, 5)"]), + ], + ) + md = verdict.miss_diagnostics + assert md is not None + assert md.best_variant_id == "alt" + assert md.overlap_with_best == 1 + assert md.best_variant_row_count == 3 + + +def test_best_overlap_tie_break_prefers_primary(tmp_path: Path): + """Both variants have IDENTICAL non-zero overlap with the agent. + Tie-break MUST pick the primary. Zero-overlap fixtures (Codex + major #2) would pass even if the implementation never computes + overlap, so the agent here must actually overlap both variants + by the same non-zero amount.""" + db = _build_db(tmp_path) + # Agent rows {1, 2, 3}; both variants share exactly row (1,) with agent. + # Overlap counts: primary=1, alt=1. + verdict = _grade( + db=db, + submitted_sql="SELECT id FROM t1 WHERE id IN (1, 2, 3) ORDER BY id", + audited_sol_sql_per_variant=[ + ("alt", False, ["SELECT id FROM t1 WHERE id IN (1, 4)"]), + ("primary", True, ["SELECT id FROM t1 WHERE id IN (1, 5)"]), + ], + ) + md = verdict.miss_diagnostics + assert md is not None + assert md.overlap_with_best == 1 + assert md.best_variant_id == "primary" + + +def test_best_overlap_tie_break_alphabetical_when_no_primary(tmp_path: Path): + """No variant has primary=True; both have identical NON-ZERO + overlap. Lexicographically smallest variant_id wins (Codex + major #2: use non-zero overlap so the test pins tie-break + behaviour, not just default-ordering on disjoint sets).""" + db = _build_db(tmp_path) + verdict = _grade( + db=db, + submitted_sql="SELECT id FROM t1 WHERE id IN (1, 2, 3) ORDER BY id", + audited_sol_sql_per_variant=[ + ("zeta", False, ["SELECT id FROM t1 WHERE id IN (1, 4)"]), + ("alpha", False, ["SELECT id FROM t1 WHERE id IN (1, 5)"]), + ], + ) + md = verdict.miss_diagnostics + assert md is not None + assert md.overlap_with_best == 1 + assert md.best_variant_id == "alpha" + + +# --------------------------------------------------------------------------- +# Multiset (bag) semantics +# --------------------------------------------------------------------------- + + +def test_multiset_overlap_counts_duplicates(tmp_path: Path): + """Agent returns ('x',) three times; gold returns ('x',) five times. + Multiset overlap = min(3, 5) = 3. Set overlap would be 1; the + implementation must use bag semantics.""" + db = _build_db(tmp_path) + # Use UNION ALL to produce duplicate rows deterministically. + verdict = _grade( + db=db, + submitted_sql=( + "SELECT 'x' AS v FROM t1 WHERE id <= 3" # 3 rows of ('x',) + ), + audited_sol_sql_per_variant=[ + ("primary", True, + ["SELECT 'x' AS v FROM t1 WHERE id <= 5"]), # 5 rows of ('x',) + ], + ) + md = verdict.miss_diagnostics + assert md is not None + assert md.agent_row_count == 3 + assert md.best_variant_row_count == 5 + # bag intersection = min(3,5) per cell + assert md.overlap_with_best == 3 + + +def test_multiset_subset_detection_with_duplicates(tmp_path: Path): + """Agent rows ⊂ gold rows as MULTISETS (every count in agent ≤ + count in gold, at least one strict).""" + db = _build_db(tmp_path) + verdict = _grade( + db=db, + submitted_sql="SELECT 'x' AS v FROM t1 WHERE id <= 2", # ('x',)×2 + audited_sol_sql_per_variant=[ + ("primary", True, + ["SELECT 'x' AS v FROM t1 WHERE id <= 4"]), # ('x',)×4 + ], + ) + md = verdict.miss_diagnostics + assert md is not None + assert md.rowset_relation_to_best == "strict_subset_of" + + +def test_multiset_superset_detection_with_duplicates(tmp_path: Path): + """Symmetric of the subset test (Codex major #3): agent has MORE + copies of the duplicate row than gold. Bag semantics MUST flag + strict_superset_of, not 'overlapping' or 'equal_rowset' that a + plain-set implementation would emit.""" + db = _build_db(tmp_path) + verdict = _grade( + db=db, + submitted_sql="SELECT 'x' AS v FROM t1 WHERE id <= 5", # ('x',)×5 + audited_sol_sql_per_variant=[ + ("primary", True, + ["SELECT 'x' AS v FROM t1 WHERE id <= 3"]), # ('x',)×3 + ], + ) + md = verdict.miss_diagnostics + assert md is not None + assert md.agent_row_count == 5 + assert md.best_variant_row_count == 3 + assert md.rowset_relation_to_best == "strict_superset_of" + assert "agent_overcount" in md.miss_patterns + assert "agent_undercount" not in md.miss_patterns + + +# --------------------------------------------------------------------------- +# CTE alias exclusion +# --------------------------------------------------------------------------- + + +def test_cte_aliases_excluded_from_tables(tmp_path: Path): + """SQL with a WITH clause: the CTE name should NOT appear in + agent_tables_referenced. Only the underlying base table should.""" + db = _build_db(tmp_path) + verdict = _grade( + db=db, + submitted_sql=( + "WITH high AS (SELECT id FROM t1 WHERE id > 3) " + "SELECT id FROM high ORDER BY id" + ), + audited_sol_sql_per_variant=[ + ("primary", True, + ["SELECT id FROM t2 ORDER BY id"]), # different table → flags wrong_table_set + ], + ) + md = verdict.miss_diagnostics + assert md is not None + assert md.agent_sql_parse_ok is True + assert md.agent_tables_referenced == ["t1"] + assert "high" not in (md.agent_tables_referenced or []) + + +def test_derived_table_aliases_excluded_from_tables(tmp_path: Path): + """`SELECT … FROM (SELECT id FROM t1) AS sub` — `sub` is a derived- + table alias and must NOT appear in agent_tables_referenced.""" + db = _build_db(tmp_path) + verdict = _grade( + db=db, + submitted_sql=( + "SELECT id FROM (SELECT id FROM t1 WHERE id > 3) AS sub " + "ORDER BY id" + ), + audited_sol_sql_per_variant=[ + ("primary", True, ["SELECT id FROM t2 ORDER BY id"]), + ], + ) + md = verdict.miss_diagnostics + assert md is not None + assert md.agent_tables_referenced == ["t1"] + assert "sub" not in (md.agent_tables_referenced or []) + + +def test_cte_aliases_excluded_from_best_variant_tables(tmp_path: Path): + """Codex major #8 — the alias-exclusion rule must apply to BOTH + sides. Best-variant SQL with a CTE must also report only base + tables in `best_variant_tables_referenced`.""" + db = _build_db(tmp_path) + verdict = _grade( + db=db, + submitted_sql="SELECT id FROM t2 ORDER BY id", # different base table + audited_sol_sql_per_variant=[ + ("primary", True, [ + "WITH high AS (SELECT id FROM t1 WHERE id > 3) " + "SELECT id FROM high ORDER BY id", + ]), + ], + ) + md = verdict.miss_diagnostics + assert md is not None + assert md.best_variant_sql_parse_ok is True + assert md.best_variant_tables_referenced == ["t1"] + assert "high" not in (md.best_variant_tables_referenced or []) + + +def test_derived_table_aliases_excluded_from_best_variant_tables(tmp_path: Path): + """Mirror for derived-table aliases on the best-variant side.""" + db = _build_db(tmp_path) + verdict = _grade( + db=db, + submitted_sql="SELECT id FROM t2 ORDER BY id", + audited_sol_sql_per_variant=[ + ("primary", True, [ + "SELECT id FROM (SELECT id FROM t1 WHERE id > 3) AS sub " + "ORDER BY id", + ]), + ], + ) + md = verdict.miss_diagnostics + assert md is not None + assert md.best_variant_tables_referenced == ["t1"] + assert "sub" not in (md.best_variant_tables_referenced or []) + + +# --------------------------------------------------------------------------- +# Canonical ordering +# --------------------------------------------------------------------------- + + +def test_tables_referenced_lists_sorted_alphabetically(tmp_path: Path): + """Both agent_tables_referenced and best_variant_tables_referenced + are alphabetically sorted before persist, regardless of FROM/JOIN + order in the source SQL. Force rowset divergence with a filter on + the gold side so the cascade actually reaches diagnostics.""" + db = _build_db(tmp_path) + verdict = _grade( + db=db, + submitted_sql=( + "SELECT t2.id FROM t2 JOIN t1 ON t1.id = t2.id ORDER BY t2.id" + ), + audited_sol_sql_per_variant=[ + ("primary", True, + ["SELECT t1.id FROM t1 JOIN t2 ON t1.id = t2.id " + "WHERE t1.id > 1 ORDER BY t1.id"]), + ], + ) + md = verdict.miss_diagnostics + assert md is not None + assert md.agent_tables_referenced == sorted(md.agent_tables_referenced or []) + assert md.best_variant_tables_referenced == sorted( + md.best_variant_tables_referenced or [], + ) + assert md.agent_tables_referenced == ["t1", "t2"] + assert md.best_variant_tables_referenced == ["t1", "t2"] + assert md.table_set_match is True + + +def test_miss_patterns_sorted_alphabetically(tmp_path: Path): + """miss_patterns is sorted alphabetically before persist.""" + db = _build_db(tmp_path) + verdict = _grade( + db=db, + submitted_sql="SELECT id FROM t2 WHERE id IN (1, 2) LIMIT 1", + audited_sol_sql_per_variant=[ + ("primary", True, + ["SELECT id FROM t1 WHERE id > 999"]), # empty + ], + user_sim_n_asks=0, + ) + md = verdict.miss_diagnostics + assert md is not None + assert md.miss_patterns == sorted(md.miss_patterns) + assert len(md.miss_patterns) >= 2 # multiple flags should fire + + +# --------------------------------------------------------------------------- +# FailureClassification integration +# --------------------------------------------------------------------------- + + +def test_failure_classification_primary_is_agent_miss(tmp_path: Path): + """End-to-end: when grade_in_place builds a SubmissionAnnotation from + a cascade-fail verdict, FailureClassification.primary is + 'agent_miss' (not 'other'); agent_at_fault=True; remediation='agent'.""" + from bird_interact_agents.eval.grade_in_place import _build_submission_annotation + db = _build_db(tmp_path) + verdict = _grade( + db=db, + submitted_sql="SELECT id FROM t1 WHERE id IN (4, 5)", + audited_sol_sql_per_variant=[ + ("primary", True, ["SELECT id FROM t1 WHERE id IN (1, 2)"]), + ], + user_sim_n_asks=0, + ) + ann = _build_submission_annotation( + task_annotation=_task_annotation(), + cascade=verdict, + benchmark="mini-interact", + run_id="test-run", + trajectory_path="rows/alien_1/attempt-1.json", + predicted_row_count=None, + duration_s=None, + cost_usd_agent=None, + cost_usd_user_sim=None, + n_agent_turns=None, + n_ask_user_calls=None, + ) + assert ann.failure_classification.primary == "agent_miss" + assert ann.failure_classification.agent_at_fault is True + assert ann.failure_classification.remediation_target == "agent" + + +def test_failure_classification_details_mentions_strict_miss(tmp_path: Path): + """The details string is free-form for humans; downstream consumers + must use miss_diagnostics.miss_patterns for structured signals. + Assert only that 'strict miss' appears (weak content check).""" + from bird_interact_agents.eval.grade_in_place import _build_submission_annotation + db = _build_db(tmp_path) + verdict = _grade( + db=db, + submitted_sql="SELECT id FROM t1 WHERE id IN (4, 5)", + audited_sol_sql_per_variant=[ + ("primary", True, ["SELECT id FROM t1 WHERE id IN (1, 2)"]), + ], + ) + ann = _build_submission_annotation( + task_annotation=_task_annotation(), + cascade=verdict, + benchmark="mini-interact", + run_id="test-run", + trajectory_path="rows/alien_1/attempt-1.json", + predicted_row_count=None, + duration_s=None, + cost_usd_agent=None, + cost_usd_user_sim=None, + n_agent_turns=None, + n_ask_user_calls=None, + ) + assert "strict miss" in ann.failure_classification.details.lower() + + +def test_submission_evaluation_carries_miss_diagnostics(tmp_path: Path): + """grade_in_place plumbs cascade.miss_diagnostics → ev.miss_diagnostics.""" + from bird_interact_agents.eval.grade_in_place import _build_submission_annotation + db = _build_db(tmp_path) + verdict = _grade( + db=db, + submitted_sql="SELECT id FROM t1 WHERE id IN (4, 5)", + audited_sol_sql_per_variant=[ + ("primary", True, ["SELECT id FROM t1 WHERE id IN (1, 2)"]), + ], + ) + ann = _build_submission_annotation( + task_annotation=_task_annotation(), + cascade=verdict, + benchmark="mini-interact", + run_id="test-run", + trajectory_path="rows/alien_1/attempt-1.json", + predicted_row_count=None, + duration_s=None, + cost_usd_agent=None, + cost_usd_user_sim=None, + n_agent_turns=None, + n_ask_user_calls=None, + ) + assert ann.evaluation.miss_diagnostics is not None + assert ann.evaluation.miss_diagnostics.best_variant_id == "primary" + + +# --------------------------------------------------------------------------- +# Back-compat: old SubmissionEvaluation JSON without miss_diagnostics +# --------------------------------------------------------------------------- + + +def test_back_compat_old_submission_evaluation_validates(): + """Existing SubmissionAnnotation JSON files (53 on disk) lack the + new miss_diagnostics field. Optional[MissDiagnostics] = None must + let them validate; extra='forbid' does NOT reject missing optional + fields with defaults.""" + from bird_interact_agents.eval.annotation_schema import SubmissionEvaluation + legacy = { + "phase1_against_original_gold": "pass", + "phase1_against_audited_primary": "pass", + "phase1_against_any_audited_variant": "pass", + "phase1_against_variants": [], + "correct_up_to_tie_order": True, + "novel_reading_judgment": None, + "correct_under_numeric_epsilon": True, + "correct_under_trailing_whitespace": True, + "correct_under_column_order": True, + "correct_under_case_fold": True, + "numeric_epsilon": 1e-6, + "verdict": "correct", + "matched_variant_id": "primary", + "rationale": "", + } + ev = SubmissionEvaluation.model_validate(legacy) + assert ev.miss_diagnostics is None + # Round-trip the new shape too — must serialize miss_diagnostics: None + dumped = ev.model_dump() + assert "miss_diagnostics" in dumped + assert dumped["miss_diagnostics"] is None + + +# --------------------------------------------------------------------------- +# Defensive: multi-statement gold is rejected +# --------------------------------------------------------------------------- + + +def test_multi_statement_audited_gold_raises_assertion(tmp_path: Path): + """Diagnostics only support single-statement gold (the contract for + SELECT tasks). A multi-statement audited_sol_sql must trigger an + explicit AssertionError so the bug doesn't silently parse the + wrong statement. Multi-statement M-tasks are out of scope.""" + import pytest as _pytest + db = _build_db(tmp_path) + with _pytest.raises(AssertionError, match="single-statement"): + _grade( + db=db, + submitted_sql="SELECT id FROM t1 WHERE id IN (4, 5)", + audited_sol_sql_per_variant=[ + ("primary", True, [ + "CREATE TEMP TABLE tmp AS SELECT id FROM t1", + "SELECT id FROM tmp", + ]), + ], + ) + + +def test_multi_statement_original_gold_raises_assertion(tmp_path: Path): + """Codex major #7 — the single-statement assertion applies to + original_sol_sql too, not just audited. Multi-statement original + gold must raise the same explicit AssertionError.""" + import pytest as _pytest + db = _build_db(tmp_path) + with _pytest.raises(AssertionError, match="single-statement"): + _grade( + db=db, + submitted_sql="SELECT id FROM t1 WHERE id IN (4, 5)", + audited_sol_sql_per_variant=[ + ("primary", True, ["SELECT id FROM t1 WHERE id IN (1, 2)"]), + ], + original_sol_sql=[ + "CREATE TEMP TABLE tmp AS SELECT id FROM t1", + "SELECT id FROM tmp", + ], + ) From dcfc584d96802150d245bf670ccd38ffb6cc1afa Mon Sep 17 00:00:00 2001 From: Egor Kraev Date: Tue, 2 Jun 2026 11:01:26 +0200 Subject: [PATCH 13/29] DEV-1515 MissDiagnostics follow-ups: column-shape split + never_asked_user plumbing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Column-shape flag refinement (annotation_schema.py + tolerant_grader.py): - Drop the noisy ``column_projection_mismatch`` flag — fired on every strict miss because slayer-namespaced column names (e.g. ``households.housenum``) consistently differ from gold's bare names (``housenum``). Neither the cascade's ``_set_equal`` nor BIRD-Interact's ``ex_base`` uses column names in equality (both compare tuples of values, not names), so a name-only divergence never causes cascade fail. The flag was muddying the failure-mode tally without identifying real cascade-fail causes. - Add ``column_count_mismatch`` — fires when agent and gold project different column COUNTS. This IS load-bearing: cell tuples have different arity, so bag equality cannot hold. Matches the cascade semantics on both sides. - Add ``column_order_mismatch`` — fires when counts match AND the column-name lists are equal as SETS after normalisation (lowercase + strip longest dot-prefix, so ``households.housenum -> housenum``) but differ as LISTS. Surfaces the "near miss" where the agent picked the right columns in wrong order and N8 column-order tolerance would have rescued the cascade if slayer's namespacing hadn't tripped its column-name set check. - Pure name-only divergence with matching counts and non-matching normalised sets is intentionally unflagged — stylistic noise. - The column-shape FIELDS (column_count_match, column_name_match_case_insensitive, column_order_match, agent_columns, best_variant_columns) stay populated as informational signals on MissDiagnostics; downstream tooling can inspect them without the flag muddying the tally. never_asked_user plumbing (grade_in_place.py + regrade.py): - grade_and_write resolves the benchmark interactivity once and forwards ``user_sim_n_asks`` to grade_submission so the ``never_asked_user`` diagnostic actually fires on interactive runs. Interactive: prefer ``user_sim_interaction.n_asks``, fall back to ``n_ask_user_calls``, then 0. One-shot: ``None`` so the flag stays out of miss_patterns (the signal doesn't apply). - regrade.py::_grader mirrors the same logic, computing n_asks from the attempt's trajectory via the existing ``_user_sim_interaction_from_trajectory`` helper. - Codex flagged this exact gap during the just-completed PR review (minor finding on grade_in_place.py:233-243) — the ``user_sim_n_asks`` parameter existed but no production call site forwarded it, so the diagnostic was systematically absent from every annotation. Summary script (scripts/dev1515_strict_miss_diagnostics.py): - Dedup key changed from ``instance_id`` to ``(benchmark, instance_id)``. Same-iid rows exist in BOTH benchmarks (``credit_4``, ``credit_7`` live in mini-interact AND livesqlbench); the previous latest-wins logic silently hid the mini-interact rows when livesqlbench had the same iid with a later run_id, undercounting the never_asked_user tally by 2. - Same-iid rows from both benchmarks now disambiguated with ``iid@livesqlbench`` suffix in the per-instance table. After re-grading all 5 runs (3 mini-interact + 2 livesqlbench, 20 cascade-fail instances total): * Mini-interact: 11/11 strict misses fire ``never_asked_user`` — the agent never queried the user-sim on any cascade-failing instance, even on ambiguous-metadata tasks. Behavioural signal worth investigating in the prompt. * Livesqlbench: 0/9 fire ``never_asked_user`` (one-shot, correctly excluded via the None sentinel). * ``column_count_mismatch`` fires on the 4 arity-mismatch instances (alien_10, households_2, museum_5, credit_7). * ``column_order_mismatch`` fires on 0 — no agent picked the right columns in wrong order. Useful negative finding. Test suite: 41 miss_diagnostics tests pass (39 + 2 new for the column_count / column_order split); full non-integration suite at 1861 passing (3 pre-existing failures in tests/test_livesqlbench_audited_gold.py about museum_2/4/9 primary flags — unrelated to this work). Co-Authored-By: Claude Opus 4.7 (1M context) --- scripts/dev1515_strict_miss_diagnostics.py | 69 +++++++++------ .../eval/annotation_schema.py | 24 +++++- .../eval/grade_in_place.py | 25 ++++++ src/bird_interact_agents/eval/regrade.py | 22 +++++ .../eval/tolerant_grader.py | 34 +++++++- tests/test_miss_diagnostics.py | 84 ++++++++++++++++++- 6 files changed, 226 insertions(+), 32 deletions(-) diff --git a/scripts/dev1515_strict_miss_diagnostics.py b/scripts/dev1515_strict_miss_diagnostics.py index 64ded71c..e14fe949 100644 --- a/scripts/dev1515_strict_miss_diagnostics.py +++ b/scripts/dev1515_strict_miss_diagnostics.py @@ -23,45 +23,64 @@ from bird_interact_agents import paths -# Walk both the hyphen + underscore tree to be defensive about the -# benchmark-name canonicalisation split that earlier sessions hit. -ROOTS = ( - paths.annotations_root() / "mini-interact", - paths.annotations_root() / "mini_interact", - paths.annotations_root() / "livesqlbench", -) +# Walk per-benchmark trees. Same instance_id can exist in BOTH +# mini-interact and livesqlbench (e.g. ``credit_4`` lives in both +# benchmark sets), so dedup key MUST be ``(benchmark, instance_id)``, +# not just ``instance_id`` — otherwise the latest-wins collapse +# silently hides mini-interact data when livesqlbench has the same +# instance_id with a later run_id. +ROOTS = { + "mini_interact": ( + paths.annotations_root() / "mini-interact", + paths.annotations_root() / "mini_interact", + ), + "livesqlbench": ( + paths.annotations_root() / "livesqlbench", + ), +} SUB_RE = re.compile(r"^(.+)\.submission\.(.+)\.json$") -def _walk_latest_submissions() -> dict[str, dict]: - """Return {instance_id: latest submission JSON}. Latest = lex-max - of the run_id suffix; works because run_ids are timestamped - ``YYYYMMDDtHHMM…``.""" - by_inst: dict[str, tuple[str, dict]] = {} - for root in ROOTS: - if not root.exists(): - continue - for p in root.glob("*/*.submission.*.json"): - m = SUB_RE.match(p.name) - if not m: +def _walk_latest_submissions() -> dict[tuple[str, str], dict]: + """Return {(benchmark, instance_id): latest submission JSON}. + Latest = lex-max of the run_id suffix; works because run_ids are + timestamped ``YYYYMMDDtHHMM…``.""" + by_key: dict[tuple[str, str], tuple[str, dict]] = {} + for bench, roots in ROOTS.items(): + for root in roots: + if not root.exists(): continue - iid, run_id = m.group(1), m.group(2) - cur = by_inst.get(iid) - if cur is None or run_id > cur[0]: - by_inst[iid] = (run_id, json.loads(p.read_text())) - return {iid: v for iid, (_run, v) in by_inst.items()} + for p in root.glob("*/*.submission.*.json"): + m = SUB_RE.match(p.name) + if not m: + continue + iid, run_id = m.group(1), m.group(2) + key = (bench, iid) + cur = by_key.get(key) + if cur is None or run_id > cur[0]: + by_key[key] = (run_id, json.loads(p.read_text())) + return {key: v for key, (_run, v) in by_key.items()} def main() -> None: submissions = _walk_latest_submissions() strict_misses: list[tuple[str, dict, dict]] = [] - for iid, ann in submissions.items(): + for (bench, iid), ann in submissions.items(): ev = ann.get("evaluation", {}) md = ev.get("miss_diagnostics") if md is None: continue - strict_misses.append((iid, ann, md)) + # Display label combines bench + iid so the per-instance table + # disambiguates same-iid-different-benchmark rows + # (e.g. `credit_4@mini_interact` vs `credit_4@livesqlbench`). + label = ( + iid if bench == "mini_interact" and iid not in { + s.split("@")[0] for s, _, _ in strict_misses + } + else f"{iid}@{bench}" + ) + strict_misses.append((label, ann, md)) strict_misses.sort() print("=" * 110) diff --git a/src/bird_interact_agents/eval/annotation_schema.py b/src/bird_interact_agents/eval/annotation_schema.py index 51614ed9..9e40fc74 100644 --- a/src/bird_interact_agents/eval/annotation_schema.py +++ b/src/bird_interact_agents/eval/annotation_schema.py @@ -305,7 +305,8 @@ class SubmissionMetadata(BaseModel): "empty_agent_result", "wrong_table_set", "aggregation_shape_mismatch", - "column_projection_mismatch", + "column_count_mismatch", + "column_order_mismatch", "predicate_count_mismatch", "having_presence_mismatch", "limit_presence_mismatch", @@ -315,6 +316,27 @@ class SubmissionMetadata(BaseModel): "agent_overcount", "never_asked_user", ] +# Column-shape flag design (intentionally split, not lumped): +# * ``column_count_mismatch`` — agent and gold have different column +# counts. Cell tuples have different arity → bag equality CANNOT +# hold (same for BIRD-Interact's ``ex_base``). Load-bearing cause +# of cascade fail. +# * ``column_order_mismatch`` — counts match AND the column-name +# LISTS, after normalisation (lowercase + strip longest dot-prefix, +# so 'households.housenum' → 'housenum'), are equal AS SETS but +# differ AS LISTS. Surfaces a "near miss" where the agent's +# projection content is correct but positionally misaligned — +# cascade N3 fails because row reprs are positional; N8 column- +# order tolerance might have rescued it but didn't because +# slayer's namespacing trips its column-name set check. +# * Pure column-NAME divergence with matching counts AND non-matching +# normalised sets is INTENTIONALLY NOT FLAGGED — that's stylistic +# (agent projected meaningfully different columns) and would fire +# spuriously on every slayer-namespaced submission. The +# column-shape fields (column_count_match, +# column_name_match_case_insensitive, column_order_match, +# agent_columns, best_variant_columns) stay populated as +# informational signals so downstream tooling can inspect. class MissDiagnostics(BaseModel): diff --git a/src/bird_interact_agents/eval/grade_in_place.py b/src/bird_interact_agents/eval/grade_in_place.py index 17327079..f59299fc 100644 --- a/src/bird_interact_agents/eval/grade_in_place.py +++ b/src/bird_interact_agents/eval/grade_in_place.py @@ -230,6 +230,30 @@ def grade_and_write( ) -> Path: """Run the tolerant grader and write the SubmissionAnnotation to ``//submission_annotation.json``.""" + # Resolve the user-sim signal for `grade_submission`. Interactive + # benchmarks (mini-interact a-interact) pass the int count of + # `ask_user` calls so the `never_asked_user` diagnostic can fire + # when the count is zero; one-shot benchmarks (livesqlbench) pass + # None so the flag stays out of `miss_patterns`. Prefer the + # already-parsed `user_sim_interaction.n_asks` over the raw + # `n_ask_user_calls` since the former encodes the parsing rule. + from bird_interact_agents.benchmark import get_benchmark + try: + _bench = get_benchmark(benchmark) + _is_interactive = not _bench.one_shot + except Exception: # noqa: BLE001 — unknown benchmark token + _is_interactive = False + _user_sim_n_asks: Optional[int] + if _is_interactive: + if user_sim_interaction is not None: + _user_sim_n_asks = user_sim_interaction.n_asks + elif n_ask_user_calls is not None: + _user_sim_n_asks = n_ask_user_calls + else: + _user_sim_n_asks = 0 + else: + _user_sim_n_asks = None + cascade = grade_submission( task_annotation=task_annotation, audited_gold_rows=audited_gold_rows, @@ -240,6 +264,7 @@ def grade_and_write( executor=executor, llm_judge=llm_judge, epsilon=epsilon, + user_sim_n_asks=_user_sim_n_asks, ) ann = _build_submission_annotation( task_annotation=task_annotation, diff --git a/src/bird_interact_agents/eval/regrade.py b/src/bird_interact_agents/eval/regrade.py index 44984aa6..9f8a5140 100644 --- a/src/bird_interact_agents/eval/regrade.py +++ b/src/bird_interact_agents/eval/regrade.py @@ -270,6 +270,10 @@ def main(argv: Optional[List[str]] = None) -> int: [s.strip() for s in args.instance_ids.split(",")] if args.instance_ids else None ) + from bird_interact_agents.benchmark import get_benchmark + from bird_interact_agents.eval.annotate import ( + _user_sim_interaction_from_trajectory, + ) from bird_interact_agents.eval.tolerant_grader import grade_submission run_dir = paths.results_root() / "cloud" / args.run_id @@ -279,6 +283,12 @@ def main(argv: Optional[List[str]] = None) -> int: # land under ``instance_id`` so the lookup is identical at call time. original_sql_by_inst = _build_original_sql_index(args.benchmark) + # Resolve interactive-vs-one-shot ONCE for this run — drives the + # ``user_sim_n_asks`` plumbing on each grader call so the + # ``never_asked_user`` diagnostic fires on interactive benchmarks + # where the agent never queried the user-sim. + _bench_is_interactive = not get_benchmark(args.benchmark).one_shot + def _grader(*, instance_id: str, submitted_sql: str, task_row: dict, **_kw): # Minimal end-to-end wiring — production callers pre-build the # implicit annotation + audited gold rows themselves. @@ -317,6 +327,17 @@ def _grader(*, instance_id: str, submitted_sql: str, task_row: dict, **_kw): or original_sql_by_inst.get(instance_id) or [] ) + # Compute the user-sim signal from the attempt's trajectory so + # the ``never_asked_user`` diagnostic fires properly on + # interactive runs where the agent never asked. One-shot + # benchmarks pass None so the flag stays out of miss_patterns. + if _bench_is_interactive: + _traj = list(task_row.get("trajectory") or []) + _user_sim_n_asks: Optional[int] = ( + _user_sim_interaction_from_trajectory(_traj).n_asks + ) + else: + _user_sim_n_asks = None return grade_submission( task_annotation=ann, audited_gold_rows=audited, @@ -324,6 +345,7 @@ def _grader(*, instance_id: str, submitted_sql: str, task_row: dict, **_kw): submitted_sql=submitted_sql, db_path=db_path, conn=None, + user_sim_n_asks=_user_sim_n_asks, ) report = regrade_run( diff --git a/src/bird_interact_agents/eval/tolerant_grader.py b/src/bird_interact_agents/eval/tolerant_grader.py index 40a8c8c8..286f1ba4 100644 --- a/src/bird_interact_agents/eval/tolerant_grader.py +++ b/src/bird_interact_agents/eval/tolerant_grader.py @@ -1002,6 +1002,19 @@ def _column_match_signals( return count_match, name_match_ci, order_match +def _normalize_col(name: str) -> str: + """Lowercase + strip the longest dot-prefix. + + ``'households.housenum' → 'housenum'``; ``'Alias.X.Y' → 'y'``. + Used by the column-order-mismatch diagnostic so slayer's + namespacing convention (``.
.``) and the gold's + bare column names compare equal modulo whitespace, while still + detecting cases where the agent picked the right columns in a + different order from the gold. + """ + return name.lower().rsplit(".", 1)[-1] + + def _compute_miss_diagnostics( *, pred_rows: Sequence[Sequence], @@ -1135,8 +1148,25 @@ def _compute_miss_diagnostics( ) ): flags.append("aggregation_shape_mismatch") - if (not count_match) or (not name_match_ci): - flags.append("column_projection_mismatch") + # Column-shape flags — split by causal vs near-miss vs stylistic + # (see MissPattern docstring in annotation_schema.py). + # 1. count_mismatch is load-bearing: different arity → bag + # equality CANNOT hold on row reprs of differing length. + # 2. order_mismatch is a near-miss: counts match, normalised + # name lists match as SETS but differ as LISTS — agent picked + # the right columns in the wrong order. + # 3. Bare name-only divergence is intentionally unflagged — + # stylistic (slayer namespacing); not a cascade-fail cause. + if not count_match: + flags.append("column_count_mismatch") + else: + agent_norm = [_normalize_col(c) for c in pred_cols] + gold_norm = [_normalize_col(c) for c in best_cols] + if ( + sorted(agent_norm) == sorted(gold_norm) + and agent_norm != gold_norm + ): + flags.append("column_order_mismatch") if ( md.agent_where_conjunct_count is not None and md.best_variant_where_conjunct_count is not None diff --git a/tests/test_miss_diagnostics.py b/tests/test_miss_diagnostics.py index d2ad00f5..ebeb6f51 100644 --- a/tests/test_miss_diagnostics.py +++ b/tests/test_miss_diagnostics.py @@ -370,8 +370,11 @@ def test_flag_aggregation_shape_mismatch(tmp_path: Path): assert "aggregation_shape_mismatch" in md.miss_patterns -def test_flag_column_projection_mismatch(tmp_path: Path): - """Agent projects 1 col; gold projects 2. column_count_match=False.""" +def test_flag_column_count_mismatch(tmp_path: Path): + """Agent projects 1 column; gold projects 2. Different arity → + bag equality on canonical row reprs cannot hold (same for + BIRD-Interact's ex_base). column_count_mismatch is the + load-bearing column-shape flag for this case.""" db = _build_db(tmp_path) verdict = _grade( db=db, @@ -382,8 +385,77 @@ def test_flag_column_projection_mismatch(tmp_path: Path): ) md = verdict.miss_diagnostics assert md is not None + # Informational fields are populated. assert md.column_count_match is False - assert "column_projection_mismatch" in md.miss_patterns + assert md.agent_column_count == 1 + assert md.best_variant_column_count == 2 + flags = set(md.miss_patterns) + assert "column_count_mismatch" in flags + # Mutually exclusive — order can't be checked when counts differ. + assert "column_order_mismatch" not in flags + + +def test_flag_column_order_mismatch(tmp_path: Path): + """Agent projects the right COLUMNS (after stripping slayer's + dot-prefix + lowercasing) but in a different ORDER from gold. + Same column count, normalised name lists match as SETS but differ + as LISTS → column_order_mismatch fires; column_count_mismatch + does not. This is the near-miss pattern where N8 column-order + tolerance would have rescued the cascade if slayer's namespacing + hadn't tripped its column-name set check.""" + db = _build_db(tmp_path) + # Agent's column names use slayer-namespacing; gold's are bare; + # order is reversed. Force rowset divergence so the cascade + # actually reaches diagnostics (values differ in row 0 vs row 0 + # because the columns are swapped in the SELECT list). + verdict = _grade( + db=db, + submitted_sql=( + 'SELECT val AS "t1.val", id AS "t1.id" FROM t1 WHERE id <= 2 ORDER BY id' + ), + audited_sol_sql_per_variant=[ + ("primary", True, + ["SELECT id, val FROM t1 WHERE id <= 2 ORDER BY id"]), + ], + ) + md = verdict.miss_diagnostics + assert md is not None + assert md.column_count_match is True + assert md.agent_column_count == 2 + assert md.best_variant_column_count == 2 + flags = set(md.miss_patterns) + assert "column_order_mismatch" in flags + assert "column_count_mismatch" not in flags + + +def test_column_name_only_divergence_no_column_flag(tmp_path: Path): + """Same column count + names differ in a NON-RECOVERABLE way + (normalised name sets are different — agent picked actually + different columns, not just renamed/namespaced). Neither + column_count_mismatch nor column_order_mismatch fires — + column-NAME-only divergence is stylistic / actually-different + projection and we don't surface it as a column-shape flag + (whatever else caused the cascade fail will surface elsewhere).""" + db = _build_db(tmp_path) + # Agent projects `id`; gold projects `val`. Same arity (1), but + # the normalised name sets are {'id'} vs {'val'} — disjoint. + verdict = _grade( + db=db, + submitted_sql="SELECT id FROM t1 WHERE id <= 2 ORDER BY id", + audited_sol_sql_per_variant=[ + ("primary", True, + ["SELECT val FROM t1 WHERE id <= 2 ORDER BY id"]), + ], + ) + md = verdict.miss_diagnostics + assert md is not None + assert md.column_count_match is True + # column-name signal is populated as informational. + assert md.column_name_match_case_insensitive is False + flags = set(md.miss_patterns) + # No column-shape flag fires; the rowset flags do the talking. + assert "column_count_mismatch" not in flags + assert "column_order_mismatch" not in flags def test_flag_predicate_count_mismatch(tmp_path: Path): @@ -618,15 +690,19 @@ def test_multi_flag_fixture_fires_every_applicable_flag(tmp_path: Path): assert md is not None flags = set(md.miss_patterns) for required in ( - "column_projection_mismatch", "wrong_table_set", "aggregation_shape_mismatch", "limit_presence_mismatch", "never_asked_user", + # Agent projects 2 cols (id, name) vs gold's 1 col (id) — + # arity mismatch fires the load-bearing column-count flag. + "column_count_mismatch", ): assert required in flags, ( f"expected {required!r} in {flags} (multi-flag scenario)" ) + # When counts differ, the order check is short-circuited. + assert "column_order_mismatch" not in flags def test_multi_flag_fixture_with_sql_parse_error(tmp_path: Path): From 2a27bb3872dce6d3e06521c039cb7f385e904511 Mon Sep 17 00:00:00 2001 From: Egor Kraev Date: Tue, 2 Jun 2026 11:39:05 +0200 Subject: [PATCH 14/29] DEV-1515 review fixes + DEV-1519 n_ask_user_calls reporting MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit /process-reviews triage (CodeRabbit + Codex carryover) — 7 valid items: Group 1 (tolerant_grader.py correctness): - Don't coerce broken audited variants to ([], []); skip with continue + logger.exception so the agent's empty rowset can't accidentally pass N2/N3 against an unevaluable gold (CR r3336709435). - _where_conjunct_count walks to the OUTER Select and reads .args['where'] directly — expr.find(Where) descends into subqueries / CTEs and picks whichever WHERE node sqlglot iterates first (CR r3336709443). - Guard N1 with an explicit empty-original-sql check so _set_equal([], []) doesn't falsely mark N1 as a strict pass when no original gold exists. Group 2 (regrade.py ops): - --force-llm-judge with no --instance-ids now clears the WHOLE judge cache instead of being a silent no-op; the per-instance scope still applies when --instance-ids is set. - regrade_rows scratch dir is reset before each pass (full wipe for an unfiltered run; per-instance subdirs for a filtered run) so stale rows can't leak into eval_regraded.json. Group 3 (path canonicalization + CLI): - scripts/dev1515_cascade_summary.py: BENCHMARK token 'mini-interact' → 'mini_interact' so it reads the canonical underscore tree (CR r3335026971). - annotation_io._annotations_root delegates to paths.annotations_root() when repo_root is None, honouring the BIRD_ANNOTATIONS_ROOT override. - eval.annotate.main(): drop the unused --run-id / --submission-mode args; submission-skeleton writing lives in dev1515_convert_runs.py. DEV-1519 (separate bug — not flagged by CodeRabbit/Codex): - claude_sdk_otf_ainteract.agent.run_task wrote accum.model_dump() to usage but never copied the per-task asks_used counter. Grader then read n_ask_user_calls from usage as 0, falsely flagging never_asked_user on every interactive miss. Both error and success result rows now emit {"n_ask_user_calls": ctx_dict.get("asks_used", 0)}. Tests: - tests/test_miss_diagnostics.py: replace the broken-best-variant test with two — mixed-variants (broken + working) and all-variants-failing — pinning the new contract. - tests/test_claude_sdk_otf_ainteract_agent.py: extend _make_fake_client + _stub_env with prefill_asks and add three regression tests (zero / nonzero / exception-path). Co-Authored-By: Claude Opus 4.7 --- scripts/dev1515_cascade_summary.py | 4 +- .../agents/claude_sdk_otf_ainteract/agent.py | 22 ++++- src/bird_interact_agents/eval/annotate.py | 25 +++--- .../eval/annotation_io.py | 13 ++- src/bird_interact_agents/eval/regrade.py | 30 ++++++- .../eval/tolerant_grader.py | 48 ++++++++--- tests/test_claude_sdk_otf_ainteract_agent.py | 81 ++++++++++++++++++- tests/test_miss_diagnostics.py | 59 +++++++++----- 8 files changed, 231 insertions(+), 51 deletions(-) diff --git a/scripts/dev1515_cascade_summary.py b/scripts/dev1515_cascade_summary.py index b0658511..e8382f2a 100644 --- a/scripts/dev1515_cascade_summary.py +++ b/scripts/dev1515_cascade_summary.py @@ -1,6 +1,6 @@ """DEV-1515: print aggregate cascade summary over the 53 annotated instances. -Walks `annotations/mini-interact/*/.submission..json` for the +Walks `annotations/mini_interact/*/.submission..json` for the two May-31 runs and emits the phase1 monotone cascade (each tier is the cumulative pass count up to and including that tier), plus a small per-failure-class tally. @@ -24,7 +24,7 @@ "20260531t1008-claudes-slayer-890419", "20260531t1343-claudes-slayer-b39bfc", ) -BENCHMARK = "mini-interact" +BENCHMARK = "mini_interact" # Most-stringent → most-lenient labels for the phase1 view. _PHASE1_LABELS = ( diff --git a/src/bird_interact_agents/agents/claude_sdk_otf_ainteract/agent.py b/src/bird_interact_agents/agents/claude_sdk_otf_ainteract/agent.py index 07889e9c..c403ed96 100644 --- a/src/bird_interact_agents/agents/claude_sdk_otf_ainteract/agent.py +++ b/src/bird_interact_agents/agents/claude_sdk_otf_ainteract/agent.py @@ -397,7 +397,19 @@ async def run_task( "phase2_observation": result.get("phase2_observation"), "trajectory": trajectory, "error": str(e), - "usage": accum.model_dump(), + # ``asks_used`` is incremented by the ``ask_user`` tool + # inside the per-task context; without it on the usage + # dict the grader (grade_in_place._user_sim_n_asks + # plumbing) defaults to 0 and falsely flags + # ``never_asked_user`` on every interactive miss. Use + # ``(ctx_dict or {})`` because early-setup failures can + # raise BEFORE ctx_dict is constructed. + "usage": { + **accum.model_dump(), + "n_ask_user_calls": (ctx_dict or {}).get( + "asks_used", 0, + ), + }, "phase1_observation_audited": result.get("phase1_observation_audited"), "phase1_observation_original": result.get("phase1_observation_original"), }, @@ -423,7 +435,13 @@ async def run_task( "phase2_observation": result.get("phase2_observation"), "trajectory": trajectory, "error": None, - "usage": accum.model_dump(), + # ctx_dict is guaranteed-set on the success path, but the + # ``{**accum.model_dump(), n_ask_user_calls: …}`` shape + # mirrors the error path above for readability. + "usage": { + **accum.model_dump(), + "n_ask_user_calls": ctx_dict.get("asks_used", 0), + }, "phase1_observation_audited": result.get("phase1_observation_audited"), "phase1_observation_original": result.get("phase1_observation_original"), }, diff --git a/src/bird_interact_agents/eval/annotate.py b/src/bird_interact_agents/eval/annotate.py index 0d144de6..a460237a 100644 --- a/src/bird_interact_agents/eval/annotate.py +++ b/src/bird_interact_agents/eval/annotate.py @@ -1,16 +1,19 @@ -"""DEV-1515: annotation-skeleton generator. +"""DEV-1515: annotation-skeleton generator (task-side only). -Builds the per-task / per-submission JSON skeletons mechanically from a -task row + per-row attempt artefacts. Human-judgment fields are left as -``PENDING_HUMAN_REVIEW`` sentinels; modes control overwrite semantics. +Builds the per-task JSON skeletons mechanically from each benchmark +data row. Human-judgment fields are left as ``PENDING_HUMAN_REVIEW`` +sentinels; ``--task-mode`` controls overwrite semantics. + +Per-submission skeletons are built by ``scripts/dev1515_convert_runs.py`` +from a completed run's per-row attempts (it has the trajectory + usage +plumbing this module does not). Usage:: python -m bird_interact_agents.eval.annotate \\ - --run-id --benchmark mini_interact \\ + --benchmark mini_interact \\ [--instance-ids ...] \\ [--task-mode {init,refresh,force-all}] \\ - [--submission-mode {overwrite,init}] \\ [--dry-run] ``--benchmark`` accepts both the dash form (``mini-interact``) and the @@ -348,22 +351,18 @@ def _load_task_rows(*, benchmark: str, instance_ids: Optional[Iterable[str]]) -> def main(argv: Optional[List[str]] = None) -> int: parser = argparse.ArgumentParser( - description="Generate / refresh per-task + per-submission annotation skeletons.", + description="Generate / refresh per-task annotation skeletons.", ) - parser.add_argument("--run-id", required=True) parser.add_argument("--benchmark", required=True) parser.add_argument( "--instance-ids", default=None, - help="Comma-separated subset; default = every instance in the run.", + help="Comma-separated subset; default = every instance in the " + "benchmark data file.", ) parser.add_argument( "--task-mode", choices=("init", "refresh", "force-all"), default="init", ) - parser.add_argument( - "--submission-mode", choices=("overwrite", "init"), - default="overwrite", - ) parser.add_argument("--dry-run", action="store_true") args = parser.parse_args(argv) diff --git a/src/bird_interact_agents/eval/annotation_io.py b/src/bird_interact_agents/eval/annotation_io.py index 04b72f98..4c45cdd8 100644 --- a/src/bird_interact_agents/eval/annotation_io.py +++ b/src/bird_interact_agents/eval/annotation_io.py @@ -46,9 +46,16 @@ def _canonical_benchmark(benchmark: str) -> str: def _annotations_root(repo_root: Optional[Path] = None) -> Path: """Anchor at the main checkout (matches the worktree-safe contract - used by ``audited_gold/`` and ``results/``).""" - root = Path(repo_root) if repo_root else paths.main_checkout_root() - return root / ANNOTATIONS_DIRNAME + used by ``audited_gold/`` and ``results/``). + + When ``repo_root`` is None, delegate to ``paths.annotations_root()`` + so the ``BIRD_ANNOTATIONS_ROOT`` env override is honoured (used by + tests + forks that mount the annotations tree elsewhere). Passing + an explicit ``repo_root`` bypasses the override on purpose — the + caller has already pinned the location.""" + if repo_root is None: + return paths.annotations_root() + return Path(repo_root) / ANNOTATIONS_DIRNAME def task_annotation_path( diff --git a/src/bird_interact_agents/eval/regrade.py b/src/bird_interact_agents/eval/regrade.py index 9f8a5140..4bc695c5 100644 --- a/src/bird_interact_agents/eval/regrade.py +++ b/src/bird_interact_agents/eval/regrade.py @@ -17,6 +17,7 @@ import argparse import json +import shutil from pathlib import Path from typing import Any, Callable, Iterable, List, Optional @@ -48,12 +49,21 @@ def _attempt_rows_dir(run_dir: Path) -> Path: def clear_llm_judge_cache( *, cache_path: Path, - instance_ids: Iterable[str], + instance_ids: Optional[Iterable[str]], ) -> None: """Drop cache entries whose embedded ``instance_id`` matches any of - ``instance_ids``. Entries for other instances are preserved.""" + ``instance_ids``. Entries for other instances are preserved. + + Pass ``instance_ids=None`` to drop EVERY entry in the cache — the + correct behaviour for an unfiltered ``--force-llm-judge`` regrade + (when the caller wants the judge to re-decide every cascade-N5 row, + a partial clear would leave previously-cached verdicts intact). + """ if not cache_path.exists(): return + if instance_ids is None: + cache_path.write_text("{}\n") + return cache = json.loads(cache_path.read_text()) wanted = set(instance_ids) new = { @@ -147,13 +157,27 @@ def regrade_run( return report filter_set = set(instance_ids) if instance_ids else None - if force_llm_judge and filter_set: + if force_llm_judge: + # ``filter_set=None`` clears the whole cache — the right thing + # for an unfiltered regrade since otherwise stale verdicts + # would survive and silently override the fresh judge call. clear_llm_judge_cache( cache_path=run_dir / "llm_judge_cache.json", instance_ids=filter_set, ) + # Reset the fresh-rows scratch dir so a partial regrade doesn't + # leak stale per-instance rows from a previous pass into + # ``eval_regraded.json``. When filtering by instance_ids, scope + # the reset to those subdirs so unrelated instances from a prior + # full regrade survive (and continue contributing to the report). fresh_rows_dir = run_dir / "regrade_rows" + if filter_set is not None and fresh_rows_dir.exists(): + for sub in list(fresh_rows_dir.iterdir()): + if sub.is_dir() and sub.name in filter_set: + shutil.rmtree(sub, ignore_errors=True) + elif filter_set is None: + shutil.rmtree(fresh_rows_dir, ignore_errors=True) fresh_rows_dir.mkdir(parents=True, exist_ok=True) for sub in sorted(p for p in rows_dir.iterdir() if p.is_dir()): diff --git a/src/bird_interact_agents/eval/tolerant_grader.py b/src/bird_interact_agents/eval/tolerant_grader.py index 286f1ba4..bed9fa7f 100644 --- a/src/bird_interact_agents/eval/tolerant_grader.py +++ b/src/bird_interact_agents/eval/tolerant_grader.py @@ -635,15 +635,33 @@ def grade_submission( sqls, db_path=db_path, conn=conn, executor=executor, ) except Exception: # noqa: BLE001 - # Variant SQL didn't execute — treat as empty rowset so the - # cascade can still complete and diagnostics can flag the - # downstream sqlglot parse failure. Better than crashing - # grading on a single broken variant. - v_rows, v_cols = [], [] + # Variant SQL didn't execute — SKIP it instead of coercing + # to ``([], [])``. An empty stand-in lets N2/N3 falsely pass + # whenever the agent rowset is also empty (e.g. agent SQL + # also failed) — producing "correct" verdicts from broken + # gold SQL. The diagnostics path picks the best-overlap + # variant from the surviving rows; downstream sqlglot + # parsing of the broken SQL string still surfaces the + # ``sql_parse_error`` flag on the agent-side miss. + logger.exception( + "Audited variant execution failed; skipping. " + "instance=%s variant_id=%s", + task_annotation.instance_id, v.get("variant_id"), + ) + continue variant_results.append((v, v_rows, v_cols)) # 2) N1 — original gold strict. - n1 = _set_equal(pred_rows, orig_rows) + # When the source data carries no ``sol_sql`` for this instance + # (e.g. the regrade source-row lookup returned []), ``orig_rows`` + # is also []. ``_set_equal([], [])`` is True — which would falsely + # mark N1 as a strict pass whenever the agent's SQL also returns + # empty (execution failure or genuinely empty result). Treat + # missing-gold as ungradable for N1 instead of as an empty bag. + if not original_sol_sql: + n1 = False + else: + n1 = _set_equal(pred_rows, orig_rows) # 3) N2/N3 — audited primary / any variant strict. primary = next( @@ -968,11 +986,23 @@ def _has_limit(expr: Optional[sg_expr.Expression]) -> Optional[bool]: def _where_conjunct_count(expr: Optional[sg_expr.Expression]) -> Optional[int]: - """Count top-level AND-conjuncts in the outer SELECT's WHERE clause. - Zero if no WHERE. A single predicate counts as 1.""" + """Count top-level AND-conjuncts in the OUTER SELECT's WHERE clause. + Zero if no WHERE. A single predicate counts as 1. + + ``expr.find(sg_expr.Where)`` would descend into subqueries / CTEs + and pick whichever WHERE node sqlglot iterates first, which can be + a nested one — producing wrong ``predicate_count_mismatch`` + diagnostics. Reach the outer Select's ``args["where"]`` directly + instead so we count predicates on the query the agent's result + rowset actually came from. + """ if expr is None: return None - where = expr.find(sg_expr.Where) + outer = ( + expr if isinstance(expr, sg_expr.Select) + else expr.find(sg_expr.Select) + ) + where = outer.args.get("where") if outer is not None else None if where is None: return 0 # Flatten the AND tree into atoms. diff --git a/tests/test_claude_sdk_otf_ainteract_agent.py b/tests/test_claude_sdk_otf_ainteract_agent.py index f03645be..99251c58 100644 --- a/tests/test_claude_sdk_otf_ainteract_agent.py +++ b/tests/test_claude_sdk_otf_ainteract_agent.py @@ -447,9 +447,17 @@ def _make_fake_client( captured: dict, messages, *, m_module=None, prefill_result=None, prefill_timing: str = "after", raise_after_prefill: Exception | None = None, + prefill_asks: int = 0, ): """Build a fake `ClaudeSDKClient`. See the equivalent helper in - test_claude_sdk_otf_agent.py for the prefill semantics.""" + test_claude_sdk_otf_agent.py for the prefill semantics. + + ``prefill_asks`` simulates the ``ask_user`` tool having been called + ``N`` times during the message loop — pokes ``asks_used`` into the + per-task context dict, which the real tool's handler does at + runtime. Used by the n_ask_user_calls reporting tests so they + don't have to wire real MCP tools. + """ class _FakeClient: def __init__(self, options): captured["options"] = options @@ -468,6 +476,8 @@ async def receive_response(self): m_module._ctx_var.get()["result"] = dict(prefill_result) for msg in messages: yield msg + if prefill_asks: + m_module._ctx_var.get()["asks_used"] = prefill_asks if prefill_result is not None and prefill_timing == "after": m_module._ctx_var.get()["result"] = dict(prefill_result) if raise_after_prefill is not None: @@ -482,6 +492,7 @@ def _stub_env( messages=(), captured=None, deleted=(), prefill_result=None, prefill_timing: str = "after", raise_after_prefill: Exception | None = None, + prefill_asks: int = 0, ): from bird_interact_agents import usage as usage_mod @@ -522,6 +533,7 @@ async def fake_resolve(*, db_name, task_data, data_path_base, benchmark): prefill_result=prefill_result, prefill_timing=prefill_timing, raise_after_prefill=raise_after_prefill, + prefill_asks=prefill_asks, ), ) return captured @@ -1046,3 +1058,70 @@ def _boom(*a, **kw): assert row["gold_result_json"] is None assert row["phase1_observation"] is None assert row["phase1_passed"] is False + + +# --------------------------------------------------------------------------- +# DEV-1519: n_ask_user_calls reporting. ``asks_used`` is incremented in the +# per-task ctx dict by the ``ask_user`` tool; without it on the result +# row's ``usage`` dict the grader sees 0 and falsely flags +# ``never_asked_user`` on every interactive miss. +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_run_task_writes_n_ask_user_calls_zero(monkeypatch, tmp_path): + """Happy path with no ``ask_user`` calls — usage carries + ``n_ask_user_calls == 0`` (NOT missing).""" + from bird_interact_agents.agents.claude_sdk_otf_ainteract import agent as m + + _stub_env( + monkeypatch, m, tmp_path / "store", + messages=[_FakeAssistant(100, 20)], + prefill_asks=0, + ) + agent = m.ClaudeSDKOtfAInteractAgent(model="anthropic/claude-sonnet-4-5") + row = await agent.run_task( + dict(_TASK), str(tmp_path), 20.0, "slayer", eval_mode="a-interact", + ) + assert row["usage"]["n_ask_user_calls"] == 0 + + +@pytest.mark.asyncio +async def test_run_task_writes_n_ask_user_calls_nonzero(monkeypatch, tmp_path): + """Happy path with 3 simulated ``ask_user`` calls — usage carries + ``n_ask_user_calls == 3``.""" + from bird_interact_agents.agents.claude_sdk_otf_ainteract import agent as m + + _stub_env( + monkeypatch, m, tmp_path / "store", + messages=[_FakeAssistant(100, 20)], + prefill_asks=3, + ) + agent = m.ClaudeSDKOtfAInteractAgent(model="anthropic/claude-sonnet-4-5") + row = await agent.run_task( + dict(_TASK), str(tmp_path), 20.0, "slayer", eval_mode="a-interact", + ) + assert row["usage"]["n_ask_user_calls"] == 3 + + +@pytest.mark.asyncio +async def test_run_task_exception_path_writes_n_ask_user_calls( + monkeypatch, tmp_path, +): + """Error path also propagates the ``asks_used`` count — the agent + asked twice before the failure, so usage carries + ``n_ask_user_calls == 2`` on the resulting row.""" + from bird_interact_agents.agents.claude_sdk_otf_ainteract import agent as m + + _stub_env( + monkeypatch, m, tmp_path / "store", + messages=[_FakeAssistant(50, 10)], + prefill_asks=2, + raise_after_prefill=RuntimeError("boom"), + ) + agent = m.ClaudeSDKOtfAInteractAgent(model="anthropic/claude-sonnet-4-5") + row = await agent.run_task( + dict(_TASK), str(tmp_path), 20.0, "slayer", eval_mode="a-interact", + ) + assert "boom" in (row.get("error") or "") + assert row["usage"]["n_ask_user_calls"] == 2 diff --git a/tests/test_miss_diagnostics.py b/tests/test_miss_diagnostics.py index ebeb6f51..5aff1690 100644 --- a/tests/test_miss_diagnostics.py +++ b/tests/test_miss_diagnostics.py @@ -280,35 +280,58 @@ def test_flag_sql_parse_error_agent_side(tmp_path: Path): def test_flag_sql_parse_error_best_variant_side(tmp_path: Path): - """Mirror case: agent SQL parses fine but the best-variant audited - SQL is malformed. Codex major #4 — the rule fires on EITHER side - failing to parse, and the best-variant nullable fields must mirror - the agent-side behaviour.""" + """Mixed-variants case: one variant executes fine, one is malformed. + The malformed variant is SKIPPED from variant_results (not coerced + to ``([], [])`` — that would risk false N2/N3 passes per + CodeRabbit r3336709435). The surviving good variant becomes the + best-overlap reference; if its SQL parses fine, sql_parse_error + does NOT fire because the broken variant was excluded from the + sqlglot pass. + + Audit-quality issues (broken variants in the gold set) are out of + scope for MissDiagnostics — they belong to a separate audit-side + annotation. This test pins the new contract: broken variants are + silently skipped and the diagnostics path proceeds against the + surviving ones.""" db = _build_db(tmp_path) verdict = _grade( db=db, submitted_sql="SELECT id FROM t1 WHERE id IN (4, 5)", audited_sol_sql_per_variant=[ ("primary", True, ["NOT A VALID SQL AT ALL ;;"]), + ("alt", False, ["SELECT id FROM t1 WHERE id IN (1, 2)"]), ], ) md = verdict.miss_diagnostics assert md is not None + # The broken "primary" was skipped from variant_results; only "alt" + # survives, so best_variant_id must be "alt". + assert md.best_variant_id == "alt" assert md.agent_sql_parse_ok is True - assert md.agent_sql_parse_error is None - assert md.agent_tables_referenced == ["t1"] - # Best-variant side is the broken one. - assert md.best_variant_sql_parse_ok is False - assert md.best_variant_sql_parse_error is not None - assert md.best_variant_tables_referenced is None - assert md.best_variant_has_group_by is None - assert md.best_variant_has_aggregate is None - assert md.best_variant_join_count is None - assert md.best_variant_where_conjunct_count is None - assert md.best_variant_has_having is None - assert md.best_variant_has_limit is None - assert md.table_set_match is None - assert "sql_parse_error" in md.miss_patterns + assert md.best_variant_sql_parse_ok is True + assert "sql_parse_error" not in md.miss_patterns + + +def test_all_variants_failing_execution_leaves_diagnostics_none(tmp_path: Path): + """When EVERY audited variant fails to execute, there is no + canonical gold to diagnose against — miss_diagnostics stays None + (we never coerce to empty, which would risk a false N2/N3 pass). + The cascade's `phase1_against_*` fields still record the misses; + callers can detect the unevaluable-grading state by observing + miss_diagnostics is None on a cascade-fail.""" + db = _build_db(tmp_path) + verdict = _grade( + db=db, + submitted_sql="SELECT id FROM t1 WHERE id IN (4, 5)", + audited_sol_sql_per_variant=[ + ("primary", True, ["NOT A VALID SQL AT ALL ;;"]), + ], + ) + # Cascade should fail across the board. + assert verdict.n2_audited_primary is False + assert verdict.n3_any_audited_variant is False + # No surviving variant → no canonical reference → no diagnostics. + assert verdict.miss_diagnostics is None def test_flag_empty_agent_result(tmp_path: Path): From cd6ef4bca6f10b44ddd0e5fcd79c4f3ba7b258ff Mon Sep 17 00:00:00 2001 From: Egor Kraev Date: Tue, 2 Jun 2026 12:15:19 +0200 Subject: [PATCH 15/29] DEV-1515 review fixes (round 3): missing-gold guard + annotations-root override MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Codex follow-up after the round-2 push: Item 1 (tolerant_grader.py — major): - When original_sol_sql is empty AND there are no audited variants, every cascade tier from N4 onward was falling back to comparing the agent rowset against the same empty orig_rows[]. compare_tie_order / compare_numeric_epsilon / compare_trailing_whitespace / compare_column_order / compare_case_fold all return True on ([], []) via set / bag equality — so a missing-gold + empty-agent pair would silently cascade-pass at N4 (and propagate as valid_interpretation even though nothing was actually compared). - Gate the __original__ fallback on `original_sol_sql` truthiness for N4 and consolidate N6-N9's comparator targets into one list that applies the same gate, so the guard can't be missed at any tier. - Regression test in test_tolerant_grader_orchestration.py pins: empty original + empty audited variants + empty agent rowset → every N-tier stays False. Item 2 (ray_app.py — minor): - _load_task_annotation_or_implicit was passing repo_root=paths.main_checkout_root() into task_annotation_path, which after round 2's annotation_io._annotations_root change deliberately bypasses BIRD_ANNOTATIONS_ROOT (the env override only fires when repo_root is None). Drop the explicit kwarg so the cloud worker honours the same override mounted by tests / forks. Co-Authored-By: Claude Opus 4.7 --- src/bird_interact_agents/cloud/ray_app.py | 7 ++- .../eval/tolerant_grader.py | 38 +++++++++------- tests/test_tolerant_grader_orchestration.py | 43 +++++++++++++++++++ 3 files changed, 71 insertions(+), 17 deletions(-) diff --git a/src/bird_interact_agents/cloud/ray_app.py b/src/bird_interact_agents/cloud/ray_app.py index efb8450f..eaecff37 100644 --- a/src/bird_interact_agents/cloud/ray_app.py +++ b/src/bird_interact_agents/cloud/ray_app.py @@ -58,14 +58,17 @@ def _load_task_annotation_or_implicit( """Try to read ``///.task.json``; if missing, fall back to the in-memory implicit default. NEVER writes a synthesized stub to disk.""" - from bird_interact_agents import paths from bird_interact_agents.eval.annotation_io import ( read_task_annotation, task_annotation_path, ) + # Leave ``repo_root`` unset so ``annotation_io._annotations_root`` + # honours ``BIRD_ANNOTATIONS_ROOT`` (the default already anchors at + # ``paths.main_checkout_root()`` via ``paths.annotations_root()``, + # so the production path is unchanged). p = task_annotation_path( benchmark=benchmark, selected_database=selected_database, - instance_id=instance_id, repo_root=paths.main_checkout_root(), + instance_id=instance_id, ) if p.exists(): return read_task_annotation(p) diff --git a/src/bird_interact_agents/eval/tolerant_grader.py b/src/bird_interact_agents/eval/tolerant_grader.py index bed9fa7f..67a1df6d 100644 --- a/src/bird_interact_agents/eval/tolerant_grader.py +++ b/src/bird_interact_agents/eval/tolerant_grader.py @@ -698,8 +698,14 @@ def grade_submission( candidates = ( [(primary[0], primary[1])] if primary else [] ) + [(v[0], v[1]) for v in variant_results if not v[0].get("primary")] - if not candidates: - # No variants → fall back to original gold itself. + if not candidates and original_sol_sql: + # No variants → fall back to original gold itself. Gated on + # ``original_sol_sql`` because ``orig_rows`` is also ``[]`` + # when no source gold exists, and ``compare_tie_order([], [])`` + # returns True via set equality — without this guard a + # missing-gold + empty-agent row pair would falsely pass at + # N4 (and propagate as ``valid_interpretation`` even though + # nothing was actually compared). candidates = [({}, orig_rows)] for v_meta, v_rows in candidates: if compare_tie_order(pred_rows, v_rows, orderby_indices=indices): @@ -748,30 +754,34 @@ def grade_submission( else: novel_judgment = None - # 6) N6/N7/N8 — cell-level relaxations applied across all variants. + # 6) N6/N7/N8/N9 — cell-level relaxations applied across all variants. + # When ``original_sol_sql`` is empty, ``orig_rows`` is also ``[]`` and + # every cell-level comparator returns True on the ``([], [])`` pair + # (bag equality holds vacuously). Drop the ``__original__`` fallback + # from the iteration list in that case — same guard as the N4 block, + # otherwise a missing-gold + empty-agent row would cascade-pass at N6. + _comparator_targets: list = list(variant_results) + if original_sol_sql: + _comparator_targets.append( + ({"variant_id": "__original__"}, orig_rows, orig_cols), + ) n6, n7, n8 = n5, n5, n5 if not n6: - for _v_meta, v_rows, _v_cols in variant_results + [ - ({"variant_id": "__original__"}, orig_rows, orig_cols), # type: ignore[list-item] - ]: + for _v_meta, v_rows, _v_cols in _comparator_targets: if compare_numeric_epsilon(pred_rows, v_rows, epsilon=epsilon): n6 = True break if not n7: n7 = n6 if not n7: - for _v_meta, v_rows, _v_cols in variant_results + [ - ({"variant_id": "__original__"}, orig_rows, orig_cols), # type: ignore[list-item] - ]: + for _v_meta, v_rows, _v_cols in _comparator_targets: if compare_trailing_whitespace(pred_rows, v_rows): n7 = True break if not n8: n8 = n7 if not n8: - for _v_meta, v_rows, v_cols in variant_results + [ - ({"variant_id": "__original__"}, orig_rows, orig_cols), # type: ignore[list-item] - ]: + for _v_meta, v_rows, v_cols in _comparator_targets: if compare_column_order( pred_rows, v_rows, pred_cols=list(pred_cols), gold_cols=list(v_cols), @@ -780,9 +790,7 @@ def grade_submission( break n9 = n8 if not n9: - for _v_meta, v_rows, _v_cols in variant_results + [ - ({"variant_id": "__original__"}, orig_rows, orig_cols), # type: ignore[list-item] - ]: + for _v_meta, v_rows, _v_cols in _comparator_targets: if compare_case_fold(pred_rows, v_rows): n9 = True break diff --git a/tests/test_tolerant_grader_orchestration.py b/tests/test_tolerant_grader_orchestration.py index f7b2e478..38242dea 100644 --- a/tests/test_tolerant_grader_orchestration.py +++ b/tests/test_tolerant_grader_orchestration.py @@ -968,3 +968,46 @@ def test_tier2_populated_on_grader_output(): assert info.first_divergent_cell_diff is not None assert "1" in info.first_divergent_cell_diff assert "2" in info.first_divergent_cell_diff + + +# --------------------------------------------------------------------------- +# Missing-gold guard: empty ``original_sol_sql`` + empty audited variants +# must NOT collapse to a monotone cascade pass at any tier. Regression +# pinned by the post-Group-1.3 follow-up — even after N1 is gated on +# ``original_sol_sql``, N4 used to fall back to ``orig_rows == []`` and +# ``compare_tie_order([], [], orderby_indices=[])`` returned True via set +# equality. That made a missing-gold + empty-agent pair come back as +# ``valid_interpretation`` despite nothing being compared. +# --------------------------------------------------------------------------- + + +def test_missing_gold_does_not_collapse_to_n4_pass(): + """No original gold, no audited variants — every cascade tier MUST + stay False, no matter what the agent's rowset looks like (including + empty).""" + from bird_interact_agents.eval.tolerant_grader import grade_submission + + submitted = "SELECT something_empty" + executor = FakeExecutor({ + submitted: ([], ["a"]), + }) + ann = _make_task_annotation() + verdict = grade_submission( + task_annotation=ann, + audited_gold_rows=[], + original_sol_sql=[], + submitted_sql=submitted, + db_path=Path("/dev/null"), + conn=None, + executor=executor, + ) + # Every N-tier stays False — no usable gold to compare against. + assert verdict.n1_original_gold is False + assert verdict.n2_audited_primary is False + assert verdict.n3_any_audited_variant is False + assert verdict.n4_tie_order is False + assert verdict.n5_llm_judge is False + assert verdict.n6_numeric_epsilon is False + assert verdict.n7_trailing_whitespace is False + assert verdict.n8_column_order is False + assert verdict.n9_case_fold is False From db37dce4df642c6b6893425075669d39df578dd8 Mon Sep 17 00:00:00 2001 From: Egor Kraev Date: Tue, 2 Jun 2026 12:45:03 +0200 Subject: [PATCH 16/29] DEV-1515 round 4: wire inline grader into local run.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Codex caught that the local runner never invoked grade_and_write, so the cascading_phase1 block was silently absent from local eval.json — only cloud runs (via cloud.ray_app._grade_one_submission) populated it. The aggregator at the bottom of run_evaluation was already in place, but its precondition (per-instance submission_annotation.json files) was never satisfied locally. Changes - Move the three per-task helpers (load_task_annotation_or_implicit, load_audited_gold_rows_for, grade_one_submission) from cloud.ray_app into eval.grade_in_place — the canonical home now matches the docstring's "shared inline grader" claim. ray_app keeps a thin alias on `_grade_one_submission` and re-imports the two loaders for the existing call sites (test_inline_grader, test_audited_gold_overlay_wiring, regrade.py — the last one updated to import from the canonical location). - run.py: define `_grade_local_row` and call it after `_persist` in `_run_with_sem`. Best-effort — a grader raise on one instance logs and continues so the loop can't be killed by a single bad task. Loop sees `b.name` (canonical underscore form) and resolves the per-task sqlite via paths.benchmark_data_root, mirroring the cloud helper's argument shape. Tests - tests/test_run_local_inline_grader.py (new): - test_local_run_invokes_inline_grader_per_task — stub runner + stub grader; assert the grader is called once per task, the benchmark token is canonicalised, per-row submission_annotation.json files land, and eval.json carries cascading_phase1 with the expected counts/rates. - test_local_run_grader_failure_does_not_kill_loop — grader raises on alien_1; alien_2 still runs to completion and total_tasks==2. - tests/cloud/test_inline_grader.py: patch the canonical grade_in_place.grade_and_write (in addition to the back-compat ray_app alias) so the existing wiring tests still pass. Co-Authored-By: Claude Opus 4.7 --- src/bird_interact_agents/cloud/ray_app.py | 133 +------ .../eval/grade_in_place.py | 134 +++++++ src/bird_interact_agents/eval/regrade.py | 5 +- src/bird_interact_agents/run.py | 49 +++ tests/cloud/test_inline_grader.py | 13 + tests/test_run_local_inline_grader.py | 338 ++++++++++++++++++ 6 files changed, 551 insertions(+), 121 deletions(-) create mode 100644 tests/test_run_local_inline_grader.py diff --git a/src/bird_interact_agents/cloud/ray_app.py b/src/bird_interact_agents/cloud/ray_app.py index eaecff37..34b6b895 100644 --- a/src/bird_interact_agents/cloud/ray_app.py +++ b/src/bird_interact_agents/cloud/ray_app.py @@ -27,9 +27,11 @@ from bird_interact_agents.cloud import benchmark_data as _benchmark_data from bird_interact_agents.cloud import gcs as _gcs from bird_interact_agents.cloud import upload_back as _upload_back -from bird_interact_agents.eval.grade_in_place import grade_and_write -from bird_interact_agents.eval.implicit_annotation import ( - implicit_task_annotation, +from bird_interact_agents.eval.grade_in_place import ( + grade_and_write, + grade_one_submission, + load_audited_gold_rows_for as _load_audited_gold_rows_for, + load_task_annotation_or_implicit as _load_task_annotation_or_implicit, ) @@ -44,125 +46,18 @@ def default_gcs_client(): # --------------------------------------------------------------------------- # DEV-1515: inline grader hook called per task after a successful submit. -# Bridges between the worker's per-task state and the shared -# `grade_in_place.grade_and_write` helper. The aggregator + fetch path -# consume the resulting submission_annotation.json — no `phase1_passed_*` -# raw fields are emitted from this path. +# Both ``cloud.ray_app`` (cloud) and ``run`` (local) call +# ``grade_in_place.grade_one_submission`` so the per-row +# ``submission_annotation.json`` files come out identical regardless of +# the entry point. The aggregator + fetch path consume those files — +# no ``phase1_passed_*`` raw fields are emitted here. The +# ``_load_*`` / ``_grade_one_submission`` names are kept as +# backwards-compat aliases at the top of this module so existing +# call-sites and tests keep importing from ``cloud.ray_app``. # --------------------------------------------------------------------------- -def _load_task_annotation_or_implicit( - *, instance_id: str, selected_database: str, benchmark: str, - amb_user_query: str = "", -): - """Try to read ``///.task.json``; - if missing, fall back to the in-memory implicit default. NEVER writes - a synthesized stub to disk.""" - from bird_interact_agents.eval.annotation_io import ( - read_task_annotation, task_annotation_path, - ) - - # Leave ``repo_root`` unset so ``annotation_io._annotations_root`` - # honours ``BIRD_ANNOTATIONS_ROOT`` (the default already anchors at - # ``paths.main_checkout_root()`` via ``paths.annotations_root()``, - # so the production path is unchanged). - p = task_annotation_path( - benchmark=benchmark, selected_database=selected_database, - instance_id=instance_id, - ) - if p.exists(): - return read_task_annotation(p) - return implicit_task_annotation( - instance_id=instance_id, - selected_database=selected_database, - benchmark=benchmark, - amb_user_query=amb_user_query, - ) - - -def _load_audited_gold_rows_for( - *, benchmark: str, instance_id: str, -) -> list[dict]: - """Load every audited-gold row for ``instance_id`` from the - consolidated JSONL. Empty list when no rows exist (graceful default - — see ``implicit_annotation``).""" - from bird_interact_agents import paths - from bird_interact_agents.benchmark import get_benchmark - - try: - bench = get_benchmark(benchmark.replace("-", "_")) - except Exception: # noqa: BLE001 - return [] - if getattr(bench, "audited_gold_layout", None) != "single_file": - return [] - consolidated = paths.audited_gold_root() / f"{bench.name}_audited.jsonl" - if not consolidated.exists(): - return [] - out: list[dict] = [] - for line in consolidated.read_text().splitlines(): - line = line.strip() - if not line: - continue - try: - row = json.loads(line) - except json.JSONDecodeError: - continue - if row.get("instance_id") == instance_id: - out.append(row) - return out - - -def _grade_one_submission( - *, - task_data: dict, - submitted_sql: str, - rows_dir: Path, - run_id: str, - benchmark: str, - db_path: Path, - conn: Any = None, - cost_usd_agent: float | None = None, - cost_usd_user_sim: float | None = None, - duration_s: float | None = None, - n_agent_turns: int | None = None, - n_ask_user_calls: int | None = None, - predicted_row_count: int | None = None, -) -> Path: - """Inline-grade one submission and write the per-row - ``submission_annotation.json``. Idempotent at the per-(task, run) - level — the cloud fetch path is no-overwrite at the destination.""" - instance_id = task_data["instance_id"] - selected_database = task_data["selected_database"] - ann = _load_task_annotation_or_implicit( - instance_id=instance_id, - selected_database=selected_database, - benchmark=benchmark, - amb_user_query=task_data.get("amb_user_query", ""), - ) - audited_rows = _load_audited_gold_rows_for( - benchmark=benchmark, instance_id=instance_id, - ) - return grade_and_write( - rows_dir=rows_dir, - instance_id=instance_id, - benchmark=benchmark, - run_id=run_id, - task_annotation=ann, - audited_gold_rows=audited_rows, - original_sol_sql=list( - task_data.get("original_sol_sql") or task_data.get("sol_sql") or [], - ), - submitted_sql=submitted_sql, - db_path=db_path, - conn=conn, - trajectory_path=f"rows/{instance_id}/attempt-1.json", - cost_usd_agent=cost_usd_agent, - cost_usd_user_sim=cost_usd_user_sim, - duration_s=duration_s, - n_agent_turns=n_agent_turns, - n_ask_user_calls=n_ask_user_calls, - predicted_row_count=predicted_row_count, - ) +_grade_one_submission = grade_one_submission # --------------------------------------------------------------------------- diff --git a/src/bird_interact_agents/eval/grade_in_place.py b/src/bird_interact_agents/eval/grade_in_place.py index f59299fc..7c6cb810 100644 --- a/src/bird_interact_agents/eval/grade_in_place.py +++ b/src/bird_interact_agents/eval/grade_in_place.py @@ -14,6 +14,7 @@ from __future__ import annotations import datetime as _dt +import json from pathlib import Path from typing import Any, Callable, List, Optional @@ -26,6 +27,9 @@ TaskAnnotation, UserSimInteraction, ) +from bird_interact_agents.eval.implicit_annotation import ( + implicit_task_annotation, +) from bird_interact_agents.eval.tolerant_grader import ( CascadeVerdict, grade_submission, @@ -286,3 +290,133 @@ def grade_and_write( out_path = out_dir / "submission_annotation.json" out_path.write_text(ann.model_dump_json(indent=2, exclude_none=False) + "\n") return out_path + + +def load_task_annotation_or_implicit( + *, instance_id: str, selected_database: str, benchmark: str, + amb_user_query: str = "", +) -> TaskAnnotation: + """Try to read ``///.task.json``; + if missing, fall back to the in-memory implicit default. NEVER writes + a synthesized stub to disk. + + Shared by the cloud worker (``ray_app.py``) and the local runner + (``run.py``) so both paths produce the same TaskAnnotation for the + same instance — keeping the per-row ``submission_annotation.json`` + comparable across local + cloud runs. + """ + from bird_interact_agents.eval.annotation_io import ( + read_task_annotation, task_annotation_path, + ) + + # Leave ``repo_root`` unset so ``annotation_io._annotations_root`` + # honours ``BIRD_ANNOTATIONS_ROOT`` (the default already anchors at + # ``paths.main_checkout_root()`` via ``paths.annotations_root()``, + # so the production path is unchanged). + p = task_annotation_path( + benchmark=benchmark, selected_database=selected_database, + instance_id=instance_id, + ) + if p.exists(): + return read_task_annotation(p) + return implicit_task_annotation( + instance_id=instance_id, + selected_database=selected_database, + benchmark=benchmark, + amb_user_query=amb_user_query, + ) + + +def load_audited_gold_rows_for( + *, benchmark: str, instance_id: str, +) -> list[dict]: + """Load every audited-gold row for ``instance_id`` from the + consolidated JSONL. Empty list when no rows exist (graceful default + — see ``implicit_task_annotation``).""" + from bird_interact_agents import paths + from bird_interact_agents.benchmark import get_benchmark + + try: + bench = get_benchmark(benchmark.replace("-", "_")) + except Exception: # noqa: BLE001 + return [] + if getattr(bench, "audited_gold_layout", None) != "single_file": + return [] + consolidated = paths.audited_gold_root() / f"{bench.name}_audited.jsonl" + if not consolidated.exists(): + return [] + out: list[dict] = [] + for line in consolidated.read_text().splitlines(): + line = line.strip() + if not line: + continue + try: + row = json.loads(line) + except json.JSONDecodeError: + continue + if row.get("instance_id") == instance_id: + out.append(row) + return out + + +def grade_one_submission( + *, + task_data: dict, + submitted_sql: str, + rows_dir: Path, + run_id: str, + benchmark: str, + db_path: Path, + conn: Any = None, + cost_usd_agent: Optional[float] = None, + cost_usd_user_sim: Optional[float] = None, + duration_s: Optional[float] = None, + n_agent_turns: Optional[int] = None, + n_ask_user_calls: Optional[int] = None, + predicted_row_count: Optional[int] = None, + user_sim_interaction: Optional[UserSimInteraction] = None, +) -> Path: + """Inline-grade one submission and write the per-row + ``submission_annotation.json``. Idempotent at the per-(task, run) + level — both the cloud fetch path and the local rows aggregator are + no-overwrite at the destination. + + Shared between cloud (``cloud.ray_app``) and local (``run``) so the + ``cascading_phase1`` block in ``eval.json`` is populated regardless + of where the run was launched. + """ + instance_id = task_data["instance_id"] + selected_database = task_data["selected_database"] + ann = load_task_annotation_or_implicit( + instance_id=instance_id, + selected_database=selected_database, + benchmark=benchmark, + amb_user_query=task_data.get("amb_user_query", ""), + ) + audited_rows = load_audited_gold_rows_for( + benchmark=benchmark, instance_id=instance_id, + ) + return grade_and_write( + rows_dir=rows_dir, + instance_id=instance_id, + benchmark=benchmark, + run_id=run_id, + task_annotation=ann, + audited_gold_rows=audited_rows, + original_sol_sql=list( + task_data.get("original_sol_sql") + or task_data.get("sol_sql") + or [], + ), + submitted_sql=submitted_sql, + db_path=db_path, + conn=conn, + trajectory_path=f"rows/{instance_id}/attempt-1.json", + cost_usd_agent=cost_usd_agent, + cost_usd_user_sim=cost_usd_user_sim, + duration_s=duration_s, + n_agent_turns=n_agent_turns, + n_ask_user_calls=n_ask_user_calls, + predicted_row_count=predicted_row_count, + user_sim_interaction=user_sim_interaction, + ) diff --git a/src/bird_interact_agents/eval/regrade.py b/src/bird_interact_agents/eval/regrade.py index 4bc695c5..8e466cf5 100644 --- a/src/bird_interact_agents/eval/regrade.py +++ b/src/bird_interact_agents/eval/regrade.py @@ -316,8 +316,9 @@ def main(argv: Optional[List[str]] = None) -> int: def _grader(*, instance_id: str, submitted_sql: str, task_row: dict, **_kw): # Minimal end-to-end wiring — production callers pre-build the # implicit annotation + audited gold rows themselves. - from bird_interact_agents.cloud.ray_app import ( - _load_audited_gold_rows_for, _load_task_annotation_or_implicit, + from bird_interact_agents.eval.grade_in_place import ( + load_audited_gold_rows_for as _load_audited_gold_rows_for, + load_task_annotation_or_implicit as _load_task_annotation_or_implicit, ) selected_database = task_row.get("selected_database", "") if not selected_database: diff --git a/src/bird_interact_agents/run.py b/src/bird_interact_agents/run.py index 2e963fd9..a2d0560a 100644 --- a/src/bird_interact_agents/run.py +++ b/src/bird_interact_agents/run.py @@ -14,6 +14,7 @@ from bird_interact_agents import paths from bird_interact_agents.benchmark import cli_dataset_tokens, get_benchmark from bird_interact_agents.eval.cascading_report import emit_cascading_eval_json +from bird_interact_agents.eval.grade_in_place import grade_one_submission from bird_interact_agents.harness import ( apply_audited_gold_overlay, calculate_budget, @@ -988,6 +989,53 @@ def _persist(td: dict, r: dict, started_at: float) -> None: p1_count = 0 p2_count = 0 + # DEV-1515: inline-grade every task into ``//`` so + # the existing aggregator at the bottom of ``run_evaluation`` can + # emit the ``cascading_phase1`` block in ``eval.json``. Mirrors the + # cloud worker (``cloud.ray_app._grade_one_submission``) — without + # this, local runs would silently lose the N1-N9 cascade metrics + # whenever audited gold / per-task annotations are present. + rows_dir = output_dir / "rows" + rows_dir.mkdir(parents=True, exist_ok=True) + _benchmark_canonical = b.name + + def _grade_local_row(td: dict, r: dict) -> None: + """Persist the per-row ``submission_annotation.json`` mirroring + cloud's ``_grade_one_submission``. Best-effort: a grader raise + on one instance must NOT take down the whole run loop — the row + was already inserted into the results DB by ``_persist``.""" + submitted_sql = r.get("submitted_sql") + selected_database = ( + r.get("database") or td.get("selected_database") or "" + ) + if not submitted_sql or not selected_database: + return + per_task_db = ( + paths.benchmark_data_root(_benchmark_canonical) + / selected_database + / f"{selected_database}.sqlite" + ) + usage_blob = r.get("usage") or {} + try: + grade_one_submission( + task_data=td, + submitted_sql=submitted_sql, + rows_dir=rows_dir, + run_id=run_id, + benchmark=_benchmark_canonical, + db_path=per_task_db, + duration_s=r.get("duration_s"), + n_agent_turns=usage_blob.get("n_agent_turns"), + n_ask_user_calls=usage_blob.get("n_ask_user_calls"), + predicted_row_count=r.get("predicted_row_count"), + ) + except Exception: # noqa: BLE001 — keep the loop alive + logger.exception( + "inline grader raised on instance=%s; " + "submission_annotation.json NOT written", + td.get("instance_id"), + ) + async def _run_with_sem(i: int, td: dict) -> None: nonlocal total_reward, p1_count, p2_count async with semaphore: @@ -1021,6 +1069,7 @@ async def _run_with_sem(i: int, td: dict) -> None: p1_count += 1 if r.get("phase2_passed"): p2_count += 1 + _grade_local_row(td, r) try: await asyncio.gather(*[_run_with_sem(i, td) for i, td in enumerate(tasks)]) diff --git a/tests/cloud/test_inline_grader.py b/tests/cloud/test_inline_grader.py index 675fffe6..2af3fd75 100644 --- a/tests/cloud/test_inline_grader.py +++ b/tests/cloud/test_inline_grader.py @@ -15,6 +15,7 @@ def test_ray_app_writes_submission_annotation_per_task(monkeypatch, tmp_path): """The worker MUST invoke grade_and_write for each task it runs.""" from bird_interact_agents.cloud import ray_app + from bird_interact_agents.eval import grade_in_place calls: list[dict] = [] @@ -26,6 +27,14 @@ def fake_grade(**kwargs): # noqa: ANN003 d.mkdir(parents=True, exist_ok=True) (d / "submission_annotation.json").write_text("{}") + # After DEV-1515 round-4, the per-task grader helper lives in + # ``grade_in_place`` (canonical location, shared with the local + # runner); ``ray_app._grade_one_submission`` is now a thin alias. + # Patch BOTH so the test passes regardless of which lookup path + # the wiring code uses. + monkeypatch.setattr( + grade_in_place, "grade_and_write", fake_grade, raising=True, + ) monkeypatch.setattr( ray_app, "grade_and_write", fake_grade, raising=True, ) @@ -86,6 +95,10 @@ def fake_grade(**kwargs): # noqa: ANN003 d.mkdir(parents=True, exist_ok=True) (d / "submission_annotation.json").write_text("{}") + from bird_interact_agents.eval import grade_in_place + monkeypatch.setattr( + grade_in_place, "grade_and_write", fake_grade, raising=True, + ) monkeypatch.setattr(ray_app, "grade_and_write", fake_grade, raising=True) ray_app._grade_one_submission( diff --git a/tests/test_run_local_inline_grader.py b/tests/test_run_local_inline_grader.py new file mode 100644 index 00000000..a379f259 --- /dev/null +++ b/tests/test_run_local_inline_grader.py @@ -0,0 +1,338 @@ +"""DEV-1515 round-3 follow-up: local `run.run_evaluation` MUST invoke +the inline grader after every task so the `cascading_phase1` block +lands in `eval.json`. Pre-fix the local runner just persisted each +agent result row into `results.db` and left the rows-dir aggregation +gated on `submission_annotation.json` files that were never written — +so local audited / annotated runs silently lost the N1-N9 cascade +metrics. + +This test pins the wiring: a synthetic 2-task run with a stub runner +and a stub inline grader, then assert (1) the grader was called for +EVERY task and (2) ``eval.json`` carries the ``cascading_phase1`` +block produced by the rows aggregator. +""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +import pytest + + +def _patch_loader_returns(monkeypatch, rows: list[dict]) -> None: + """Same shape as the DEV-1510 wiring test — patch the loader so + `run_evaluation` reaches the runner-call loop without real data.""" + monkeypatch.setattr( + "bird_interact_agents.harness.load_benchmark_tasks", + lambda *a, **kw: rows, + ) + import bird_interact_agents.run as run_mod + monkeypatch.setattr(run_mod, "load_benchmark_tasks", lambda *a, **kw: rows) + + +def _stub_runner_factory(monkeypatch, results_by_inst: dict[str, dict]): + """Replace `_make_runner` with a factory that returns an async + runner. The runner looks each task up by `instance_id` and returns + the canned result row.""" + + async def _stub_runner(td: dict, data_dir: str, patience: int, user_sim_model: str): + return dict(results_by_inst[td["instance_id"]]) + + import bird_interact_agents.run as run_mod + monkeypatch.setattr(run_mod, "_make_runner", lambda **kw: _stub_runner) + + +@pytest.mark.asyncio +async def test_local_run_invokes_inline_grader_per_task(monkeypatch, tmp_path): + """End-to-end: 2 fake tasks → inline grader called twice → the + rows-dir aggregator emits ``cascading_phase1`` in ``eval.json``.""" + import bird_interact_agents.run as run_mod + + rows = [ + { + "instance_id": "alien_1", + "selected_database": "alien", + "sol_sql": ["SELECT 1"], + "amb_user_query": "q1", + }, + { + "instance_id": "alien_2", + "selected_database": "alien", + "sol_sql": ["SELECT 2"], + "amb_user_query": "q2", + }, + ] + _patch_loader_returns(monkeypatch, rows) + monkeypatch.setattr(run_mod, "_maybe_force_wipe_otf", lambda **kw: None) + _stub_runner_factory(monkeypatch, { + "alien_1": { + "instance_id": "alien_1", + "database": "alien", + "phase1_passed": True, + "phase2_passed": False, + "total_reward": 1.0, + "submitted_sql": "SELECT 1", + "trajectory": [], + "usage": {"n_agent_turns": 3, "n_ask_user_calls": 1}, + }, + "alien_2": { + "instance_id": "alien_2", + "database": "alien", + "phase1_passed": False, + "phase2_passed": False, + "total_reward": 0.0, + "submitted_sql": "SELECT 2", + "trajectory": [], + "usage": {"n_agent_turns": 1, "n_ask_user_calls": 0}, + }, + }) + + # Capture the grader calls — replace ``grade_one_submission`` at the + # spot ``run.py`` looks it up (module-level import) with a stub that + # writes a minimal ``submission_annotation.json``. The shape only has + # to satisfy ``emit_cascading_eval_json``'s loader; the cascade body + # itself is unit-tested elsewhere. + calls: list[dict[str, Any]] = [] + + from bird_interact_agents.eval.annotation_schema import ( + FailureClassification, + SubmissionAnnotation, + SubmissionEvaluation, + SubmissionMetadata, + UserSimInteraction, + ) + + def _stub_grader(*, task_data, submitted_sql, rows_dir, run_id, benchmark, db_path, **kw): + calls.append({ + "instance_id": task_data["instance_id"], + "submitted_sql": submitted_sql, + "benchmark": benchmark, + "rows_dir": rows_dir, + "run_id": run_id, + }) + out_dir = Path(rows_dir) / task_data["instance_id"] + out_dir.mkdir(parents=True, exist_ok=True) + passed = task_data["instance_id"] == "alien_1" + ann = SubmissionAnnotation( + instance_id=task_data["instance_id"], + selected_database=task_data["selected_database"], + task_annotation_ref=( + f"annotations/mini_interact/alien/" + f"{task_data['instance_id']}.task.json" + ), + annotated_by="auto-inline-grader", + annotated_at="2026-06-02T00:00:00+00:00", + submission=SubmissionMetadata( + cloud_run_id=run_id, + trajectory_path=f"rows/{task_data['instance_id']}/attempt-1.json", + ), + evaluation=SubmissionEvaluation( + phase1_against_original_gold="pass" if passed else "fail", + phase1_against_audited_primary="pass" if passed else "fail", + phase1_against_any_audited_variant="pass" if passed else "fail", + phase1_against_variants=[], + correct_up_to_tie_order=passed, + novel_reading_judgment=None, + correct_under_numeric_epsilon=passed, + correct_under_trailing_whitespace=passed, + correct_under_column_order=passed, + correct_under_case_fold=passed, + numeric_epsilon=1e-6, + verdict="correct" if passed else "invalid", + matched_variant_id="primary" if passed else None, + rationale="", + miss_diagnostics=None, + ), + failure_classification=FailureClassification( + primary="no_fail" if passed else "agent_miss", + agent_at_fault=not passed, + remediation_target="other" if passed else "agent", + details="stub", + ), + decision_point=None, + user_sim_interaction=UserSimInteraction( + n_asks=kw.get("n_ask_user_calls") or 0, + ), + ) + path = out_dir / "submission_annotation.json" + path.write_text(ann.model_dump_json(indent=2, exclude_none=False) + "\n") + return path + + monkeypatch.setattr(run_mod, "grade_one_submission", _stub_grader) + + output_path = tmp_path / "eval.json" + metrics = await run_mod.run_evaluation( + framework="claude_sdk_otf_ainteract", + query_mode="slayer", + mode="a-interact", + data_path="ignored", + data_dir=str(tmp_path / "ignored_data_dir"), + output_path=str(output_path), + concurrency=1, + limit=None, + agent_model="anthropic/claude-haiku-4-5-20251001", + strict=False, + prompt_cache=False, + max_depth=1, + slayer_storage_root=str(tmp_path / "slayer_models"), + slayer_setup="on-the-fly", + reasoning_effort=None, + use_audited_gold_sql=False, + dataset="mini-interact", + gold_file=None, + filter_ids=None, + ) + + # The grader fired for EVERY task — load-bearing assertion: the + # local runner now invokes the inline grader per the + # _run_with_sem _grade_local_row hook. + instances = sorted(c["instance_id"] for c in calls) + assert instances == ["alien_1", "alien_2"], ( + f"inline grader should be called once per task; got: {instances}" + ) + # Benchmark token is the canonical underscore form. + assert all(c["benchmark"] == "mini_interact" for c in calls), ( + f"benchmark must be canonicalized to underscore form; got: {calls}" + ) + # Per-row ``submission_annotation.json`` files were written. + rows_dir = output_path.parent / "rows" + for inst in ("alien_1", "alien_2"): + assert (rows_dir / inst / "submission_annotation.json").exists(), ( + f"submission_annotation.json missing for {inst}" + ) + # And the aggregator emitted the ``cascading_phase1`` block into + # eval.json — the whole point of this fix. + final_eval = json.loads(output_path.read_text()) + assert "cascading_phase1" in final_eval, ( + f"cascading_phase1 missing from eval.json; keys={sorted(final_eval)}" + ) + cp = final_eval["cascading_phase1"] + assert cp["n_dual_eval_tasks"] == 2, f"got cp={cp}" + # alien_1 passed every tier (n1..n9) in the stub annotation; + # alien_2 failed everything. Each tier should therefore count 1. + assert cp["counts"]["n3"] == 1, f"got cp={cp}" + assert cp["rates"]["n3"] == 0.5, f"got cp={cp}" + + +@pytest.mark.asyncio +async def test_local_run_grader_failure_does_not_kill_loop(monkeypatch, tmp_path): + """If the inline grader raises on one instance, the loop MUST + continue — the agent result row is already persisted by + ``_persist``, and a partial cascade block is better than nothing.""" + import bird_interact_agents.run as run_mod + + rows = [ + {"instance_id": "alien_1", "selected_database": "alien", + "sol_sql": ["SELECT 1"], "amb_user_query": "q1"}, + {"instance_id": "alien_2", "selected_database": "alien", + "sol_sql": ["SELECT 2"], "amb_user_query": "q2"}, + ] + _patch_loader_returns(monkeypatch, rows) + monkeypatch.setattr(run_mod, "_maybe_force_wipe_otf", lambda **kw: None) + _stub_runner_factory(monkeypatch, { + "alien_1": { + "instance_id": "alien_1", "database": "alien", + "phase1_passed": True, "phase2_passed": False, + "total_reward": 1.0, "submitted_sql": "SELECT 1", + "trajectory": [], "usage": {}, + }, + "alien_2": { + "instance_id": "alien_2", "database": "alien", + "phase1_passed": False, "phase2_passed": False, + "total_reward": 0.0, "submitted_sql": "SELECT 2", + "trajectory": [], "usage": {}, + }, + }) + + from bird_interact_agents.eval.annotation_schema import ( + FailureClassification, + SubmissionAnnotation, + SubmissionEvaluation, + SubmissionMetadata, + UserSimInteraction, + ) + + def _stub_grader_with_explosion(*, task_data, **kw): + if task_data["instance_id"] == "alien_1": + raise RuntimeError("boom") + # Second task writes a real annotation so the aggregator + # has something parseable to read. + rows_dir = Path(kw["rows_dir"]) + run_id = kw["run_id"] + out_dir = rows_dir / task_data["instance_id"] + out_dir.mkdir(parents=True, exist_ok=True) + ann = SubmissionAnnotation( + instance_id=task_data["instance_id"], + selected_database=task_data["selected_database"], + task_annotation_ref=( + f"annotations/mini_interact/alien/" + f"{task_data['instance_id']}.task.json" + ), + annotated_by="auto-inline-grader", + annotated_at="2026-06-02T00:00:00+00:00", + submission=SubmissionMetadata( + cloud_run_id=run_id, + trajectory_path=f"rows/{task_data['instance_id']}/attempt-1.json", + ), + evaluation=SubmissionEvaluation( + phase1_against_original_gold="fail", + phase1_against_audited_primary="fail", + phase1_against_any_audited_variant="fail", + phase1_against_variants=[], + correct_up_to_tie_order=False, + novel_reading_judgment=None, + correct_under_numeric_epsilon=False, + correct_under_trailing_whitespace=False, + correct_under_column_order=False, + correct_under_case_fold=False, + numeric_epsilon=1e-6, + verdict="invalid", + matched_variant_id=None, + rationale="", + miss_diagnostics=None, + ), + failure_classification=FailureClassification( + primary="agent_miss", + agent_at_fault=True, + remediation_target="agent", + details="stub", + ), + decision_point=None, + user_sim_interaction=UserSimInteraction(), + ) + path = out_dir / "submission_annotation.json" + path.write_text(ann.model_dump_json(indent=2, exclude_none=False) + "\n") + return path + + monkeypatch.setattr( + run_mod, "grade_one_submission", _stub_grader_with_explosion, + ) + + output_path = tmp_path / "eval.json" + metrics = await run_mod.run_evaluation( + framework="claude_sdk_otf_ainteract", + query_mode="slayer", + mode="a-interact", + data_path="ignored", + data_dir=str(tmp_path / "ignored_data_dir"), + output_path=str(output_path), + concurrency=1, + limit=None, + agent_model="anthropic/claude-haiku-4-5-20251001", + strict=False, + prompt_cache=False, + max_depth=1, + slayer_storage_root=str(tmp_path / "slayer_models"), + slayer_setup="on-the-fly", + reasoning_effort=None, + use_audited_gold_sql=False, + dataset="mini-interact", + gold_file=None, + filter_ids=None, + ) + + # Both tasks ran to completion despite alien_1 grader raising — + # the loop kept going, total_tasks==2. + assert metrics["total_tasks"] == 2 From 6c6b9798472713ebd00cd8000af3bb173342b2a7 Mon Sep 17 00:00:00 2001 From: Egor Kraev Date: Tue, 2 Jun 2026 13:46:17 +0200 Subject: [PATCH 17/29] DEV-1515 round 5: sol_sql shape normalisation + honest cascade denominator MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two Codex findings on the round-4 grader plumbing: Group 1 (Codex major — grade_in_place.py, regrade.py): Both call sites did ``list(value or [])`` to coerce the source row's ``sol_sql``/``original_sol_sql`` into a list. When the value comes through as a bare string (which ``run.py::_persist`` already handles via ``isinstance(sol, str)`` — so the shape is established upstream), ``list("SELECT 1")`` returns ``["S","E","L","E","C","T"," ","1"]``. The grader then runs each character through sqlite as a one-character statement, raises ``sqlite3.OperationalError`` per character, and the row's N1 silently drops to False under the broad-except. Fix: ``normalize_sol_sql(value)`` helper that returns ``[]`` for None/falsy, ``[value]`` for a string, and ``list(value)`` for a list — used at both call sites. Group 2 (Codex major — run.py:1007-1012): ``_grade_local_row`` silently returned without writing any ``submission_annotation.json`` when ``submitted_sql`` was missing OR the inline grader raised. ``aggregate_cascading_phase1`` then walked ``rows_dir`` and only counted the per-task dirs that did have annotations — so ``n_dual_eval_tasks`` dropped below ``len(tasks)`` and ``cascading_phase1.rates`` were INFLATED by silently excluding the never-graded rows. Fix: ``write_failed_submission_annotation()`` helper writes a fail-everything ``SubmissionAnnotation`` (verdict=invalid, failure_classification.primary="other", all N-tiers 0) for both bypass paths. The aggregator then counts those rows in the denominator and honest rates fall out. Tests: - tests/test_normalize_sol_sql.py (new) — 7 tests pinning the shape contract: None/empty/string/list/tuple, plus a long-string regression that asserts no char-split semantics. - tests/test_run_local_inline_grader.py: - test_local_run_grader_failure_writes_fail_everything_annotation (renamed) — extended to assert the fallback annotation lands, n_dual_eval_tasks==2, all-tiers count 0, alien_1's verdict shape. - test_local_run_no_submitted_sql_writes_fail_everything_annotation (new) — analog for the no-submit pre-submit-crash path. Co-Authored-By: Claude Opus 4.7 --- .../eval/grade_in_place.py | 121 +++++++++++- src/bird_interact_agents/eval/regrade.py | 9 +- src/bird_interact_agents/run.py | 53 +++++- tests/test_normalize_sol_sql.py | 59 ++++++ tests/test_run_local_inline_grader.py | 178 +++++++++++++++++- 5 files changed, 403 insertions(+), 17 deletions(-) create mode 100644 tests/test_normalize_sol_sql.py diff --git a/src/bird_interact_agents/eval/grade_in_place.py b/src/bird_interact_agents/eval/grade_in_place.py index 7c6cb810..815995a4 100644 --- a/src/bird_interact_agents/eval/grade_in_place.py +++ b/src/bird_interact_agents/eval/grade_in_place.py @@ -39,6 +39,34 @@ _AUTO_ANNOTATOR = "auto-inline-grader" +def normalize_sol_sql(value: Any) -> List[str]: + """Coerce ``sol_sql`` / ``original_sol_sql`` to ``list[str]``. + + Both shapes are present in the wild — ``mini_interact.jsonl`` carries + ``sol_sql`` as a single SQL string on some rows and as a list on + others (the post-DEV-1478 schema is list, but tests and older + fixtures still pass a bare string). Without this helper the previous + ``list(value or [])`` would turn the string ``"SELECT 1"`` into the + character list ``["S", "E", "L", "E", "C", "T", " ", "1"]`` and the + grader would execute each character as a one-character SQL + statement, raising ``sqlite3.OperationalError`` and silently + dropping N1 to False. + + Contract: + * ``None`` / falsy → ``[]`` + * ``str`` → ``[value]`` + * ``list`` / ``tuple`` / other iterable of strings → ``list(value)`` + + Non-string elements inside a list pass through unchanged (the + grader rejects them downstream — this helper is shape-only). + """ + if not value: + return [] + if isinstance(value, str): + return [value] + return list(value) + + def _verdict_to_phase(b: bool) -> PhaseVerdict: return "pass" if b else "fail" @@ -292,6 +320,93 @@ def grade_and_write( return out_path +def write_failed_submission_annotation( + *, + rows_dir: Path, + instance_id: str, + selected_database: str, + benchmark: str, + run_id: str, + trajectory_path: str, + failure_details: str, + duration_s: Optional[float] = None, + cost_usd_agent: Optional[float] = None, + cost_usd_user_sim: Optional[float] = None, + n_agent_turns: Optional[int] = None, + n_ask_user_calls: Optional[int] = None, + predicted_row_count: Optional[int] = None, +) -> Path: + """Write a 0-pass ``submission_annotation.json`` for a task that + bypassed the grader (e.g. agent crashed before submit, no + ``submitted_sql`` on the result row, grader itself raised). + + Without this, ``aggregate_cascading_phase1`` walks ``rows_dir`` and + skips the missing per-task dir entirely — so ``n_dual_eval_tasks`` + drops below ``total_tasks`` and ``cascading_phase1.rates`` are + INFLATED by silently excluding never-graded rows from the + denominator. Writing a fail-everything annotation keeps the + denominator honest. + + The annotation is shaped so every N-tier reads as ``"fail"`` / + ``False`` (which the aggregator then counts as a 0-pass row at every + cascade tier). ``failure_classification.primary`` is ``"other"`` + because the cascade was never actually computed — the gap isn't + "the agent's SQL didn't match the gold" but "no SQL to compare". + """ + out_dir = Path(rows_dir) / instance_id + out_dir.mkdir(parents=True, exist_ok=True) + benchmark_canonical = benchmark.replace("-", "_") + ann = SubmissionAnnotation( + instance_id=instance_id, + selected_database=selected_database, + task_annotation_ref=( + f"annotations/{benchmark_canonical}/{selected_database}/" + f"{instance_id}.task.json" + ), + annotated_by=_AUTO_ANNOTATOR, + annotated_at=_dt.datetime.now(_dt.timezone.utc) + .replace(microsecond=0).isoformat(), + submission=SubmissionMetadata( + cloud_run_id=run_id, + trajectory_path=trajectory_path, + predicted_row_count=predicted_row_count, + duration_s=duration_s, + cost_usd_agent=cost_usd_agent, + cost_usd_user_sim=cost_usd_user_sim, + n_agent_turns=n_agent_turns, + n_ask_user_calls=n_ask_user_calls, + ), + evaluation=SubmissionEvaluation( + phase1_against_original_gold="fail", + phase1_against_audited_primary="fail", + phase1_against_any_audited_variant="fail", + phase1_against_variants=[], + correct_up_to_tie_order=False, + novel_reading_judgment=None, + correct_under_numeric_epsilon=False, + correct_under_trailing_whitespace=False, + correct_under_column_order=False, + correct_under_case_fold=False, + numeric_epsilon=1e-6, + verdict="invalid", + matched_variant_id=None, + rationale=failure_details, + miss_diagnostics=None, + ), + failure_classification=FailureClassification( + primary="other", + agent_at_fault=False, + remediation_target="other", + details=failure_details, + ), + decision_point=None, + user_sim_interaction=UserSimInteraction(), + ) + out_path = out_dir / "submission_annotation.json" + out_path.write_text(ann.model_dump_json(indent=2, exclude_none=False) + "\n") + return out_path + + def load_task_annotation_or_implicit( *, instance_id: str, selected_database: str, benchmark: str, amb_user_query: str = "", @@ -403,10 +518,8 @@ def grade_one_submission( run_id=run_id, task_annotation=ann, audited_gold_rows=audited_rows, - original_sol_sql=list( - task_data.get("original_sol_sql") - or task_data.get("sol_sql") - or [], + original_sol_sql=normalize_sol_sql( + task_data.get("original_sol_sql") or task_data.get("sol_sql"), ), submitted_sql=submitted_sql, db_path=db_path, diff --git a/src/bird_interact_agents/eval/regrade.py b/src/bird_interact_agents/eval/regrade.py index 8e466cf5..322463db 100644 --- a/src/bird_interact_agents/eval/regrade.py +++ b/src/bird_interact_agents/eval/regrade.py @@ -30,6 +30,7 @@ ) from bird_interact_agents.eval.annotation_schema import SubmissionAnnotation from bird_interact_agents.eval.cascading_report import emit_cascading_eval_json +from bird_interact_agents.eval.grade_in_place import normalize_sol_sql class RegradeReport(BaseModel): @@ -346,11 +347,13 @@ def _grader(*, instance_id: str, submitted_sql: str, task_row: dict, **_kw): ) # N1 requires the original gold SQL; the attempt JSON doesn't # carry it (it lives on the source data row / gated gold sidecar). - original_sol_sql = list( + # ``normalize_sol_sql`` wraps a bare string in a list so the + # grader doesn't see ``["S", "E", "L", "E", "C", "T", ...]`` + # when the source row carries ``sol_sql`` as a single string. + original_sol_sql = normalize_sol_sql( task_row.get("original_sol_sql") or task_row.get("sol_sql") - or original_sql_by_inst.get(instance_id) - or [] + or original_sql_by_inst.get(instance_id), ) # Compute the user-sim signal from the attempt's trajectory so # the ``never_asked_user`` diagnostic fires properly on diff --git a/src/bird_interact_agents/run.py b/src/bird_interact_agents/run.py index a2d0560a..1c05748f 100644 --- a/src/bird_interact_agents/run.py +++ b/src/bird_interact_agents/run.py @@ -14,7 +14,10 @@ from bird_interact_agents import paths from bird_interact_agents.benchmark import cli_dataset_tokens, get_benchmark from bird_interact_agents.eval.cascading_report import emit_cascading_eval_json -from bird_interact_agents.eval.grade_in_place import grade_one_submission +from bird_interact_agents.eval.grade_in_place import ( + grade_one_submission, + write_failed_submission_annotation, +) from bird_interact_agents.harness import ( apply_audited_gold_overlay, calculate_budget, @@ -1003,19 +1006,48 @@ def _grade_local_row(td: dict, r: dict) -> None: """Persist the per-row ``submission_annotation.json`` mirroring cloud's ``_grade_one_submission``. Best-effort: a grader raise on one instance must NOT take down the whole run loop — the row - was already inserted into the results DB by ``_persist``.""" + was already inserted into the results DB by ``_persist``. + + For tasks the grader can't run on (no ``submitted_sql``, agent + crashed before submit, grader itself raised) we still write a + ``fail-everything`` annotation so the aggregator's denominator + (``cascading_phase1.n_dual_eval_tasks``) stays at + ``len(tasks)`` and the reported rates aren't inflated by + silently dropped rows. + """ + instance_id_for_log = td.get("instance_id", "") submitted_sql = r.get("submitted_sql") selected_database = ( r.get("database") or td.get("selected_database") or "" ) + usage_blob = r.get("usage") or {} + common_failed_kwargs = dict( + rows_dir=rows_dir, + instance_id=instance_id_for_log, + selected_database=selected_database or "", + benchmark=_benchmark_canonical, + run_id=run_id, + trajectory_path=f"rows/{instance_id_for_log}/attempt-1.json", + duration_s=r.get("duration_s"), + n_agent_turns=usage_blob.get("n_agent_turns"), + n_ask_user_calls=usage_blob.get("n_ask_user_calls"), + predicted_row_count=r.get("predicted_row_count"), + ) if not submitted_sql or not selected_database: + write_failed_submission_annotation( + **common_failed_kwargs, + failure_details=( + "no submitted_sql / selected_database — task " + "errored before reaching submit; counted as 0-pass " + "row at every cascade tier." + ), + ) return per_task_db = ( paths.benchmark_data_root(_benchmark_canonical) / selected_database / f"{selected_database}.sqlite" ) - usage_blob = r.get("usage") or {} try: grade_one_submission( task_data=td, @@ -1029,11 +1061,18 @@ def _grade_local_row(td: dict, r: dict) -> None: n_ask_user_calls=usage_blob.get("n_ask_user_calls"), predicted_row_count=r.get("predicted_row_count"), ) - except Exception: # noqa: BLE001 — keep the loop alive + except Exception as exc: # noqa: BLE001 — keep the loop alive logger.exception( - "inline grader raised on instance=%s; " - "submission_annotation.json NOT written", - td.get("instance_id"), + "inline grader raised on instance=%s; writing " + "fail-everything annotation so the cascade denominator " + "stays honest", + instance_id_for_log, + ) + write_failed_submission_annotation( + **common_failed_kwargs, + failure_details=( + f"inline grader raised: {type(exc).__name__}: {exc}" + )[:200], ) async def _run_with_sem(i: int, td: dict) -> None: diff --git a/tests/test_normalize_sol_sql.py b/tests/test_normalize_sol_sql.py new file mode 100644 index 00000000..231373c7 --- /dev/null +++ b/tests/test_normalize_sol_sql.py @@ -0,0 +1,59 @@ +"""DEV-1515 round-5 follow-up: ``normalize_sol_sql`` pins the shape of +``sol_sql`` / ``original_sol_sql`` before the grader sees it. + +Pre-fix the local + regrade plumbing did ``list(value or [])`` which +silently turned a bare string ``"SELECT 1"`` into +``["S", "E", "L", "E", "C", "T", " ", "1"]``. The grader then ran each +character through sqlite as a one-character SQL statement, raised +``sqlite3.OperationalError`` per character, and the row's N1 dropped +to False under the broad-except catch — silently. This test pins the +five shapes the helper has to handle correctly so the regression +can't reappear. +""" +from __future__ import annotations + +from bird_interact_agents.eval.grade_in_place import normalize_sol_sql + + +def test_normalize_sol_sql_none_returns_empty_list(): + assert normalize_sol_sql(None) == [] + + +def test_normalize_sol_sql_empty_string_returns_empty_list(): + assert normalize_sol_sql("") == [] + + +def test_normalize_sol_sql_empty_list_returns_empty_list(): + assert normalize_sol_sql([]) == [] + + +def test_normalize_sol_sql_string_wraps_as_single_item_list(): + """The load-bearing case — without this, ``list("SELECT 1")`` would + return ``["S", "E", "L", "E", "C", "T", " ", "1"]`` and the grader + would execute each character separately.""" + assert normalize_sol_sql("SELECT 1") == ["SELECT 1"] + + +def test_normalize_sol_sql_list_passes_through(): + assert normalize_sol_sql(["SELECT a", "SELECT b"]) == [ + "SELECT a", "SELECT b", + ] + + +def test_normalize_sol_sql_tuple_coerces_to_list(): + """Tuples are accepted by the grader caller surface; coerce them so + the downstream contract (``list[str]``) is honored.""" + assert normalize_sol_sql(("SELECT a",)) == ["SELECT a"] + + +def test_normalize_sol_sql_does_not_split_into_chars_for_long_string(): + """Belt-and-braces: the multi-character SQL case must not regress to + the per-character split semantics. Asserts the return is a 1-element + list whose element is the original string verbatim.""" + sql = ( + "WITH cte AS (SELECT * FROM tbl) " + "SELECT col1, col2 FROM cte WHERE x = 1" + ) + result = normalize_sol_sql(sql) + assert len(result) == 1 + assert result[0] == sql diff --git a/tests/test_run_local_inline_grader.py b/tests/test_run_local_inline_grader.py index a379f259..17d461ac 100644 --- a/tests/test_run_local_inline_grader.py +++ b/tests/test_run_local_inline_grader.py @@ -217,10 +217,15 @@ def _stub_grader(*, task_data, submitted_sql, rows_dir, run_id, benchmark, db_pa @pytest.mark.asyncio -async def test_local_run_grader_failure_does_not_kill_loop(monkeypatch, tmp_path): +async def test_local_run_grader_failure_writes_fail_everything_annotation( + monkeypatch, tmp_path, +): """If the inline grader raises on one instance, the loop MUST - continue — the agent result row is already persisted by - ``_persist``, and a partial cascade block is better than nothing.""" + continue AND a fail-everything ``submission_annotation.json`` MUST + be written for the broken instance so the aggregator's denominator + stays at ``len(tasks)``. Pre-fix the broken instance was silently + dropped, inflating ``cascading_phase1.rates`` (Codex round-4 finding + on ``run.py:1007-1012``).""" import bird_interact_agents.run as run_mod rows = [ @@ -336,3 +341,170 @@ def _stub_grader_with_explosion(*, task_data, **kw): # Both tasks ran to completion despite alien_1 grader raising — # the loop kept going, total_tasks==2. assert metrics["total_tasks"] == 2 + # Both annotations are on disk: alien_2 from the stub success path, + # alien_1 from the fail-everything fallback the runner wrote when + # the stub raised. Without that fallback the aggregator's + # denominator would have been 1, not 2. + rows_dir = output_path.parent / "rows" + for inst in ("alien_1", "alien_2"): + assert (rows_dir / inst / "submission_annotation.json").exists(), ( + f"submission_annotation.json missing for {inst} after " + f"grader failure — fail-everything fallback didn't fire" + ) + cp = metrics["cascading_phase1"] + assert cp["n_dual_eval_tasks"] == 2, ( + f"denominator should stay at 2 even when alien_1's grader " + f"raised; got cp={cp}" + ) + # Both alien_1 (fail-everything from the fallback) and alien_2 + # (the stub's deliberate fail-everything) count as 0 at every tier. + for tier in ("n1", "n2", "n3", "n4", "n5", "n6", "n7", "n8", "n9"): + assert cp["counts"][tier] == 0, ( + f"every cascade tier should count 0 (both rows fail); " + f"got cp={cp}" + ) + # alien_1's annotation is the fail-everything shape: primary + # ``other`` (the cascade was never actually run on alien_1). + alien1 = json.loads( + (rows_dir / "alien_1" / "submission_annotation.json").read_text(), + ) + assert alien1["failure_classification"]["primary"] == "other" + assert alien1["evaluation"]["verdict"] == "invalid" + assert "grader raised" in alien1["failure_classification"]["details"] + + +@pytest.mark.asyncio +async def test_local_run_no_submitted_sql_writes_fail_everything_annotation( + monkeypatch, tmp_path, +): + """If the agent crashes before submit (no ``submitted_sql`` on the + result row), the local runner MUST still write a fail-everything + annotation so the cascade denominator stays honest. Pre-fix the + no-submit row was silently dropped from + ``cascading_phase1.n_dual_eval_tasks``.""" + import bird_interact_agents.run as run_mod + + rows = [ + {"instance_id": "alien_1", "selected_database": "alien", + "sol_sql": ["SELECT 1"], "amb_user_query": "q1"}, + {"instance_id": "alien_2", "selected_database": "alien", + "sol_sql": ["SELECT 2"], "amb_user_query": "q2"}, + ] + _patch_loader_returns(monkeypatch, rows) + monkeypatch.setattr(run_mod, "_maybe_force_wipe_otf", lambda **kw: None) + _stub_runner_factory(monkeypatch, { + # alien_1 returns a row WITHOUT submitted_sql — simulating an + # agent crash before reaching the submit step. + "alien_1": { + "instance_id": "alien_1", "database": "alien", + "phase1_passed": False, "phase2_passed": False, + "total_reward": 0.0, "submitted_sql": None, + "trajectory": [], "usage": {}, + }, + "alien_2": { + "instance_id": "alien_2", "database": "alien", + "phase1_passed": False, "phase2_passed": False, + "total_reward": 0.0, "submitted_sql": "SELECT 2", + "trajectory": [], "usage": {}, + }, + }) + + from bird_interact_agents.eval.annotation_schema import ( + FailureClassification, + SubmissionAnnotation, + SubmissionEvaluation, + SubmissionMetadata, + UserSimInteraction, + ) + + def _stub_grader_for_alien_2(*, task_data, **kw): + # Only called for alien_2 — alien_1 is short-circuited by the + # no-submitted_sql check before reaching the grader. + out_dir = Path(kw["rows_dir"]) / task_data["instance_id"] + out_dir.mkdir(parents=True, exist_ok=True) + ann = SubmissionAnnotation( + instance_id=task_data["instance_id"], + selected_database=task_data["selected_database"], + task_annotation_ref=( + f"annotations/mini_interact/alien/" + f"{task_data['instance_id']}.task.json" + ), + annotated_by="auto-inline-grader", + annotated_at="2026-06-02T00:00:00+00:00", + submission=SubmissionMetadata( + cloud_run_id=kw["run_id"], + trajectory_path=f"rows/{task_data['instance_id']}/attempt-1.json", + ), + evaluation=SubmissionEvaluation( + phase1_against_original_gold="fail", + phase1_against_audited_primary="fail", + phase1_against_any_audited_variant="fail", + phase1_against_variants=[], + correct_up_to_tie_order=False, + novel_reading_judgment=None, + correct_under_numeric_epsilon=False, + correct_under_trailing_whitespace=False, + correct_under_column_order=False, + correct_under_case_fold=False, + numeric_epsilon=1e-6, + verdict="invalid", + matched_variant_id=None, + rationale="", + miss_diagnostics=None, + ), + failure_classification=FailureClassification( + primary="agent_miss", + agent_at_fault=True, + remediation_target="agent", + details="stub", + ), + decision_point=None, + user_sim_interaction=UserSimInteraction(), + ) + path = out_dir / "submission_annotation.json" + path.write_text(ann.model_dump_json(indent=2, exclude_none=False) + "\n") + return path + + monkeypatch.setattr( + run_mod, "grade_one_submission", _stub_grader_for_alien_2, + ) + + output_path = tmp_path / "eval.json" + metrics = await run_mod.run_evaluation( + framework="claude_sdk_otf_ainteract", + query_mode="slayer", + mode="a-interact", + data_path="ignored", + data_dir=str(tmp_path / "ignored_data_dir"), + output_path=str(output_path), + concurrency=1, + limit=None, + agent_model="anthropic/claude-haiku-4-5-20251001", + strict=False, + prompt_cache=False, + max_depth=1, + slayer_storage_root=str(tmp_path / "slayer_models"), + slayer_setup="on-the-fly", + reasoning_effort=None, + use_audited_gold_sql=False, + dataset="mini-interact", + gold_file=None, + filter_ids=None, + ) + + rows_dir = output_path.parent / "rows" + for inst in ("alien_1", "alien_2"): + assert (rows_dir / inst / "submission_annotation.json").exists(), ( + f"submission_annotation.json missing for {inst} — " + f"no-submitted_sql fallback didn't fire" + ) + cp = metrics["cascading_phase1"] + assert cp["n_dual_eval_tasks"] == 2, ( + f"denominator should stay at 2 even when alien_1 had no " + f"submitted_sql; got cp={cp}" + ) + alien1 = json.loads( + (rows_dir / "alien_1" / "submission_annotation.json").read_text(), + ) + assert alien1["failure_classification"]["primary"] == "other" + assert "no submitted_sql" in alien1["failure_classification"]["details"] From 149f9c216ccbaa1ac59f3de781f6e01a97f10210 Mon Sep 17 00:00:00 2001 From: Egor Kraev Date: Tue, 2 Jun 2026 14:10:40 +0200 Subject: [PATCH 18/29] DEV-1515 round 6: cloud-side fail-everything fallback (symmetric to local) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Codex caught the symmetric cloud bug after round 5 landed the local fix: ``_run_one_in_actor``'s inline-grader except branch only printed a traceback and uploaded nothing. Two downstream consequences in ``driver._emit_cascading_phase1_on_fetch``: * All-tasks-failed run: ``has_per_row_anns`` is False, the whole ``cascading_phase1`` block is silently dropped from ``eval.json``. * Some-tasks-failed run: the aggregator's strict ``_per_row_cascade_bools`` raises ``FileNotFoundError`` on the missing row, which the driver surfaces as ``cascading_phase1_error`` and the block STILL doesn't land. So one broken task wipes the whole run's cascade metrics. Fix: on grader bypass (unbound ``data_dir``, no ``submitted_sql``, broken gold, ``_grade_one_submission`` exception) the except branch now writes a fail-everything annotation via ``write_failed_submission_annotation`` into the same temp ``annotation_dir`` and uploads it via ``_gcs.write_submission_annotation`` — mirrors ``run._grade_local_row`` shape and the cloud driver-side aggregator now sees a per-row file for EVERY task. Fallback-of-the-fallback prints a second traceback and moves on (so a bug in the failed-annotation path can't crash the actor). Tests: - tests/cloud/test_inline_grader.py: - test_cloud_grader_failure_uploads_fail_everything_annotation — patches ``ray_app._grade_one_submission`` to raise, runs ``_LocalActor.run_one``, asserts the in-memory fake GCS bucket received a ``runs//rows//submission_annotation.json`` blob with verdict=invalid, primary=other, and the exception message in details. Co-Authored-By: Claude Opus 4.7 --- src/bird_interact_agents/cloud/ray_app.py | 45 +++++++++-- tests/cloud/test_inline_grader.py | 99 +++++++++++++++++++++++ 2 files changed, 137 insertions(+), 7 deletions(-) diff --git a/src/bird_interact_agents/cloud/ray_app.py b/src/bird_interact_agents/cloud/ray_app.py index 34b6b895..de9f13d4 100644 --- a/src/bird_interact_agents/cloud/ray_app.py +++ b/src/bird_interact_agents/cloud/ray_app.py @@ -32,6 +32,7 @@ grade_one_submission, load_audited_gold_rows_for as _load_audited_gold_rows_for, load_task_annotation_or_implicit as _load_task_annotation_or_implicit, + write_failed_submission_annotation, ) @@ -571,11 +572,19 @@ def _run_one_in_actor( # DEV-1515: inline grader produces a SubmissionAnnotation per task # (cascading verdict + Tier 2 informational). Failure here MUST NOT # block the row/log upload — it's diagnostic, not result-of-record. + # But the per-row submission_annotation.json MUST land in GCS + # regardless: ``driver._emit_cascading_phase1_on_fetch`` runs the + # aggregator strictly (a single missing per-row file raises + # FileNotFoundError and the whole ``cascading_phase1`` block is + # dropped from ``eval.json``). So on any cloud-grader bypass path + # (unbound data_dir, missing submitted_sql, broken gold, grader + # exception) we fall back to writing + uploading a fail-everything + # annotation — mirrors ``run._grade_local_row``. _grader_data_dir = locals().get("data_dir") + annotation_dir = Path(tempfile.mkdtemp(prefix="bird_submission_annot_")) try: if _grader_data_dir is None: raise RuntimeError("data_dir unbound; grader skipped") - annotation_dir = Path(tempfile.mkdtemp(prefix="bird_submission_annot_")) ann_path = _grade_one_submission( task_data=task_data, submitted_sql=str(row.get("submitted_sql") or ""), @@ -596,16 +605,38 @@ def _run_one_in_actor( if isinstance(row.get("usage"), dict) else None, predicted_row_count=None, ) + _gcs.write_submission_annotation( + run_id, iid, json.loads(ann_path.read_text()), + client=gcs_client, + ) + except Exception as grader_exc: # noqa: BLE001 + # Diagnostic — never let grader failure cascade into a task fail. + traceback.print_exc() try: + failed_path = write_failed_submission_annotation( + rows_dir=annotation_dir, + instance_id=iid, + selected_database=str(task_data.get("selected_database", "") + or ""), + benchmark=_cloud_benchmark(cfg), + run_id=run_id, + trajectory_path=f"rows/{iid}/attempt-1.json", + failure_details=( + f"cloud inline grader raised: " + f"{type(grader_exc).__name__}: {grader_exc}" + )[:200], + duration_s=row.get("duration_s"), + ) _gcs.write_submission_annotation( - run_id, iid, json.loads(ann_path.read_text()), + run_id, iid, json.loads(failed_path.read_text()), client=gcs_client, ) - finally: - shutil.rmtree(annotation_dir, ignore_errors=True) - except Exception: # noqa: BLE001 - # Diagnostic — never let grader failure cascade into a task fail. - traceback.print_exc() + except Exception: # noqa: BLE001 + # Fallback-of-the-fallback — log and move on. The downstream + # aggregator will treat this row as missing (skip whole block). + traceback.print_exc() + finally: + shutil.rmtree(annotation_dir, ignore_errors=True) try: log_bytes = log_tmp.read_bytes() if log_tmp.exists() else b"" diff --git a/tests/cloud/test_inline_grader.py b/tests/cloud/test_inline_grader.py index 2af3fd75..b0c81c75 100644 --- a/tests/cloud/test_inline_grader.py +++ b/tests/cloud/test_inline_grader.py @@ -120,3 +120,102 @@ def fake_grade(**kwargs): # noqa: ANN003 assert len(captured) == 1 # No .task.json was written. assert not list(annotations_root.rglob("*.task.json")) + + +# --------------------------------------------------------------------------- +# DEV-1515 round 6: when the cloud inline grader raises, the worker MUST +# still upload a fail-everything submission_annotation.json — otherwise +# the post-fetch ``cascading_phase1`` aggregator either skips the block +# entirely (no per-row anns at all) or raises FileNotFoundError as soon +# as one missing-annotation row is encountered, so a single broken task +# wipes the new N1-N9 metrics from ``eval.json``. +# --------------------------------------------------------------------------- + + +def test_cloud_grader_failure_uploads_fail_everything_annotation( + monkeypatch, fake_gcs_bucket, +): + """``_run_one_in_actor`` invokes the inline grader; when it raises, + the worker writes + uploads a fail-everything + ``submission_annotation.json`` so every cloud row contributes to + the cascade denominator.""" + import json + import pytest + from bird_interact_agents.cloud import ray_app + + RUN_ID = "20260602T1200-pydanticai-raw-round6" + + client, store = fake_gcs_bucket + monkeypatch.setattr(ray_app, "default_gcs_client", lambda: client) + monkeypatch.setattr(ray_app, "_maybe_build_cached_runner", lambda _cfg: None) + monkeypatch.setattr(ray_app, "download_slayer_setup", lambda *a, **k: None) + + async def fake_run_one_task(task_data, **_kw): + return { + "instance_id": task_data["instance_id"], + "database": task_data.get("selected_database", "db_a"), + "phase1_passed": False, "phase2_passed": False, + "total_reward": 0.0, "duration_s": 0.01, "error": None, + "submitted_sql": "SELECT 1", + } + monkeypatch.setattr( + "bird_interact_agents.run.run_one_task", fake_run_one_task, + ) + + # The load-bearing patch: the cloud-side inline grader RAISES. The + # worker's except branch must catch + write + upload the + # fail-everything fallback. Patch the cloud module's reference (the + # name the worker actually looks up) so we don't need to also poke + # at the canonical grade_in_place location. + def _raise_grader(**_kw): + raise RuntimeError("simulated grader explosion") + monkeypatch.setattr( + ray_app, "_grade_one_submission", _raise_grader, raising=True, + ) + + # No-op the rest of the upload-back triple — irrelevant to this + # test and they hit the wider FS. + from bird_interact_agents.cloud import upload_back + monkeypatch.setattr( + upload_back, "upload_per_task_debug", lambda **kw: None, + ) + monkeypatch.setattr( + upload_back, "upload_per_task_setup_sessions", lambda **kw: None, + ) + monkeypatch.setattr( + upload_back, "upload_otf_reference_delta", lambda **kw: None, + ) + + actor = ray_app._LocalActor( + {"framework": "pydantic_ai_otf_encode", "query_mode": "slayer", + "mode": "a-interact", "agent_model": "anthropic/claude-sonnet-4-5", + "user_sim_model": "anthropic/claude-haiku-4-5-20251001", + "patience": 3, "strict": False, "use_audited_gold_sql": False, + "prompt_cache": True, "max_depth": 3, "slayer_setup": "on-the-fly", + "slayer_storage_root": "/data/slayer_models", + "data_dir": "/data/mini-interact"}, + RUN_ID, 1, gcs_client=client, + ) + # MUST NOT raise — grader failure is diagnostic, not result-of-record. + actor.run_one({"instance_id": "db_a_1", "selected_database": "db_a"}) + + # Per-row submission_annotation.json blob landed in GCS storage. + ann_blob_keys = [ + k for k in store + if k.endswith("/db_a_1/submission_annotation.json") + ] + assert ann_blob_keys, ( + f"fail-everything annotation should have been uploaded; " + f"store keys: {sorted(store)}" + ) + payload = json.loads(store[ann_blob_keys[0]].decode()) + # Fail-everything shape: every cascade tier is fail/False, verdict + # is invalid, primary is 'other' (the cascade was never actually run). + assert payload["evaluation"]["verdict"] == "invalid" + assert payload["evaluation"]["phase1_against_original_gold"] == "fail" + assert payload["evaluation"]["phase1_against_any_audited_variant"] == "fail" + assert payload["evaluation"]["correct_up_to_tie_order"] is False + assert payload["failure_classification"]["primary"] == "other" + assert "simulated grader explosion" in ( + payload["failure_classification"]["details"] + ) From cd202888508c8b3a7eb2cd3a35584506f08ae9ad Mon Sep 17 00:00:00 2001 From: Egor Kraev Date: Tue, 2 Jun 2026 14:30:33 +0200 Subject: [PATCH 19/29] DEV-1515 round 7: cloud no-submit short-circuit + regrade string-sol_sql indexing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two more Codex findings on the round-4/6 grader plumbing: Item 1 (Codex major — cloud/ray_app.py:588-590): The round-6 cloud fallback only fires when the grader RAISES. When ``row.get("submitted_sql") is None`` the cloud worker still called ``_grade_one_submission(submitted_sql=str(None or "") == "")``, passing the empty SQL through to SQLite. Some sqlite3 versions return an empty rowset (no exception), which then matches an also-empty gold rowset via ``_set_equal([], []) == True`` and falsely passes N1/N2/N3. The local runner had the explicit guard in ``_grade_local_row``; the cloud worker did not. Fix: hoist the same short-circuit. Read ``row.get("submitted_sql")`` + ``row.get("database") or task_data.get("selected_database")`` BEFORE the grader call; raise a ``RuntimeError`` if either is missing so the existing fail-everything except branch fires the fallback upload path. Cloud + local now agree on never-submitted rows. Item 2 (Codex major — eval/regrade.py:103-106, :134-137): ``_build_original_sql_index`` filtered on ``isinstance(sol, list) and sol`` at BOTH branches (mini_interact data file + livesqlbench gated sidecar). That silently dropped any source row whose ``sol_sql`` was the bare-string shape that round-5's ``normalize_sol_sql`` was explicitly designed to support. After the index returns empty for such rows, the per-task lookup falls through to ``original_sql_by_inst.get(iid) → None → []`` and N1 can never pass, undercounting ``eval_regraded.json``. Fix: pass ``r.get("sol_sql")`` through ``normalize_sol_sql`` at both branches; skip only when the normalised value is empty. Tests: - tests/test_regrade_cli.py: - test_build_original_sql_index_accepts_string_sol_sql — mixed string/list/missing rows; assert string wraps to ``[s]``, list passes through, missing-sol_sql is skipped. - test_build_original_sql_index_does_not_char_split_string — belt-and-braces against the round-5 char-split regression. - tests/cloud/test_inline_grader.py: - test_cloud_no_submitted_sql_short_circuits_before_real_grader — stubs ``run_one_task`` to return ``submitted_sql=None``, spies on ``_grade_one_submission`` and asserts it is never called, and asserts the fail-everything annotation still lands in GCS with primary=``other`` + the short-circuit details. Co-Authored-By: Claude Opus 4.7 --- src/bird_interact_agents/cloud/ray_app.py | 25 +++++- src/bird_interact_agents/eval/regrade.py | 16 ++-- tests/cloud/test_inline_grader.py | 96 +++++++++++++++++++++++ tests/test_regrade_cli.py | 70 +++++++++++++++++ 4 files changed, 198 insertions(+), 9 deletions(-) diff --git a/src/bird_interact_agents/cloud/ray_app.py b/src/bird_interact_agents/cloud/ray_app.py index de9f13d4..00da075e 100644 --- a/src/bird_interact_agents/cloud/ray_app.py +++ b/src/bird_interact_agents/cloud/ray_app.py @@ -582,18 +582,37 @@ def _run_one_in_actor( # annotation — mirrors ``run._grade_local_row``. _grader_data_dir = locals().get("data_dir") annotation_dir = Path(tempfile.mkdtemp(prefix="bird_submission_annot_")) + _row_submitted_sql = row.get("submitted_sql") + _row_selected_db = ( + row.get("database") or task_data.get("selected_database") or "" + ) try: + # Short-circuit BEFORE calling the real grader on a missing + # submission. ``str(row.get("submitted_sql") or "")`` would + # otherwise pass ``""`` through; SQLite may silently return an + # empty rowset for an empty statement, and ``_set_equal([], [])`` + # would falsely pass N1/N2/N3 whenever the gold result is also + # empty (Codex r7). Mirrors ``run._grade_local_row``'s short- + # circuit so the cloud + local paths agree on never-submitted + # rows — both write a fail-everything annotation here, which the + # ``except`` branch below ALSO does for grader exceptions. + if not _row_submitted_sql or not _row_selected_db: + raise RuntimeError( + "no submitted_sql / selected_database — task errored " + "before reaching submit; routed to fail-everything " + "fallback", + ) if _grader_data_dir is None: raise RuntimeError("data_dir unbound; grader skipped") ann_path = _grade_one_submission( task_data=task_data, - submitted_sql=str(row.get("submitted_sql") or ""), + submitted_sql=str(_row_submitted_sql), rows_dir=annotation_dir, run_id=run_id, benchmark=_cloud_benchmark(cfg), db_path=Path(_grader_data_dir) - / str(task_data.get("selected_database", "")) - / f"{task_data.get('selected_database', '')}.sqlite", + / str(_row_selected_db) + / f"{_row_selected_db}.sqlite", cost_usd_agent=row.get("usage", {}).get("cost_usd_agent") if isinstance(row.get("usage"), dict) else None, cost_usd_user_sim=row.get("usage", {}).get("cost_usd_user_sim") diff --git a/src/bird_interact_agents/eval/regrade.py b/src/bird_interact_agents/eval/regrade.py index 322463db..492fad4d 100644 --- a/src/bird_interact_agents/eval/regrade.py +++ b/src/bird_interact_agents/eval/regrade.py @@ -101,9 +101,13 @@ def _build_original_sql_index(benchmark: str) -> dict[str, list[str]]: except json.JSONDecodeError: continue iid = r.get("instance_id") - sol = r.get("sol_sql") - if iid and isinstance(sol, list) and sol: - out[iid] = list(sol) + sol = normalize_sol_sql(r.get("sol_sql")) + # ``normalize_sol_sql`` returns ``[]`` for None / empty / + # missing, ``[s]`` for a string, and ``list(value)`` for + # a list — so a string-shaped ``sol_sql`` is no longer + # silently dropped at index-build time (Codex r6). + if iid and sol: + out[iid] = sol # Merge in livesqlbench's gated sidecar if available. bench = get_benchmark(benchmark) if bench.gold_required: @@ -132,9 +136,9 @@ def _build_original_sql_index(benchmark: str) -> dict[str, list[str]]: except json.JSONDecodeError: continue iid = r.get("instance_id") - sol = r.get("sol_sql") - if iid and isinstance(sol, list) and sol: - out[iid] = list(sol) + sol = normalize_sol_sql(r.get("sol_sql")) + if iid and sol: + out[iid] = sol return out diff --git a/tests/cloud/test_inline_grader.py b/tests/cloud/test_inline_grader.py index b0c81c75..9cc865c2 100644 --- a/tests/cloud/test_inline_grader.py +++ b/tests/cloud/test_inline_grader.py @@ -219,3 +219,99 @@ def _raise_grader(**_kw): assert "simulated grader explosion" in ( payload["failure_classification"]["details"] ) + + +def test_cloud_no_submitted_sql_short_circuits_before_real_grader( + monkeypatch, fake_gcs_bucket, +): + """When ``run_one_task`` returns a row with no ``submitted_sql`` + (agent crashed before reaching submit), the cloud worker MUST + short-circuit BEFORE calling the real grader. Pre-fix the cloud + path passed ``str(row.get("submitted_sql") or "") == ""`` through + to ``_grade_one_submission``; SQLite may silently return an empty + rowset for the empty statement, and ``_set_equal([], [])`` falsely + passes N1/N2/N3 whenever the gold result is also empty. Mirrors + the local runner's guard in ``run._grade_local_row``.""" + import json + from bird_interact_agents.cloud import ray_app + + RUN_ID = "20260602T1215-cloud-no-submit-round7" + + client, store = fake_gcs_bucket + monkeypatch.setattr(ray_app, "default_gcs_client", lambda: client) + monkeypatch.setattr(ray_app, "_maybe_build_cached_runner", lambda _cfg: None) + monkeypatch.setattr(ray_app, "download_slayer_setup", lambda *a, **k: None) + + async def fake_run_one_task(task_data, **_kw): + # Critical: NO ``submitted_sql`` on the result row — simulates + # an agent crash before the submit step. + return { + "instance_id": task_data["instance_id"], + "database": task_data.get("selected_database", "db_a"), + "phase1_passed": False, "phase2_passed": False, + "total_reward": 0.0, "duration_s": 0.01, "error": "boom", + "submitted_sql": None, + } + monkeypatch.setattr( + "bird_interact_agents.run.run_one_task", fake_run_one_task, + ) + + # Spy on _grade_one_submission — it must NEVER be invoked when + # submitted_sql is missing, otherwise SQLite's empty-statement + # behaviour would surface in the grader's pred_rows. + grader_calls: list[dict] = [] + + def _spy_grader(**kwargs): + grader_calls.append(dict(kwargs)) + raise AssertionError( + "_grade_one_submission must NOT be called when " + "submitted_sql is missing; the short-circuit should fire" + ) + monkeypatch.setattr( + ray_app, "_grade_one_submission", _spy_grader, raising=True, + ) + + from bird_interact_agents.cloud import upload_back + monkeypatch.setattr(upload_back, "upload_per_task_debug", lambda **kw: None) + monkeypatch.setattr( + upload_back, "upload_per_task_setup_sessions", lambda **kw: None, + ) + monkeypatch.setattr( + upload_back, "upload_otf_reference_delta", lambda **kw: None, + ) + + actor = ray_app._LocalActor( + {"framework": "pydantic_ai_otf_encode", "query_mode": "slayer", + "mode": "a-interact", "agent_model": "anthropic/claude-sonnet-4-5", + "user_sim_model": "anthropic/claude-haiku-4-5-20251001", + "patience": 3, "strict": False, "use_audited_gold_sql": False, + "prompt_cache": True, "max_depth": 3, "slayer_setup": "on-the-fly", + "slayer_storage_root": "/data/slayer_models", + "data_dir": "/data/mini-interact"}, + RUN_ID, 1, gcs_client=client, + ) + actor.run_one({"instance_id": "db_a_1", "selected_database": "db_a"}) + + # _grade_one_submission was not called — the short-circuit fired. + assert grader_calls == [], ( + f"_grade_one_submission must not be invoked on missing-submit " + f"path; got calls: {grader_calls}" + ) + + # And the fail-everything annotation still landed in GCS for the + # cascading_phase1 denominator. + ann_keys = [ + k for k in store + if k.endswith("/db_a_1/submission_annotation.json") + ] + assert ann_keys, ( + f"fail-everything annotation missing from upload; keys={sorted(store)}" + ) + payload = json.loads(store[ann_keys[0]].decode()) + assert payload["evaluation"]["verdict"] == "invalid" + assert payload["failure_classification"]["primary"] == "other" + assert ( + "no submitted_sql" in payload["failure_classification"]["details"] + or "task errored before reaching submit" + in payload["failure_classification"]["details"] + ) diff --git a/tests/test_regrade_cli.py b/tests/test_regrade_cli.py index 1bf84d81..a9d055cd 100644 --- a/tests/test_regrade_cli.py +++ b/tests/test_regrade_cli.py @@ -264,3 +264,73 @@ def __call__(self, **kw): assert len(calls) == 1 remaining = json.loads(cache_path.read_text()) assert "k_for_alien_1" not in remaining + + +# --------------------------------------------------------------------------- +# Codex round 6: ``_build_original_sql_index`` MUST accept string-shaped +# ``sol_sql``. The mini_interact JSONL carries both shapes (post-DEV-1478 +# is list; older rows / tests / fixtures pass a bare string). Pre-fix the +# ``isinstance(sol, list)`` filter silently dropped the string rows, so +# regrade fell through to ``original_sql_by_inst.get(iid) → []`` and N1 +# could never pass for those instances. +# --------------------------------------------------------------------------- + + +def test_build_original_sql_index_accepts_string_sol_sql(tmp_path, monkeypatch): + """Mini-interact data file with both string + list shapes: both + must land in the index after normalize_sol_sql wraps the string.""" + from bird_interact_agents import paths as paths_mod + from bird_interact_agents.eval.regrade import _build_original_sql_index + + data_file = tmp_path / "mini_interact.jsonl" + data_file.write_text( + json.dumps({ + "instance_id": "alien_string_sol", + "sol_sql": "SELECT 1 FROM t", + }) + "\n" + + json.dumps({ + "instance_id": "alien_list_sol", + "sol_sql": ["SELECT 2 FROM u"], + }) + "\n" + + json.dumps({ + "instance_id": "alien_no_sol", + # ``sol_sql`` absent — expected to be skipped. + }) + "\n" + ) + + # Pretend the data file lives at this temp path. + monkeypatch.setattr( + paths_mod, "benchmark_data_file", + lambda benchmark: data_file, + ) + + out = _build_original_sql_index("mini_interact") + assert out["alien_string_sol"] == ["SELECT 1 FROM t"], ( + "string-shaped sol_sql must be wrapped as a 1-item list, NOT " + "dropped or character-split" + ) + assert out["alien_list_sol"] == ["SELECT 2 FROM u"] + assert "alien_no_sol" not in out + + +def test_build_original_sql_index_does_not_char_split_string( + tmp_path, monkeypatch, +): + """Defensive: a long-string ``sol_sql`` returns a 1-element list + whose element is the verbatim SQL, NOT a per-character list.""" + from bird_interact_agents import paths as paths_mod + from bird_interact_agents.eval.regrade import _build_original_sql_index + + sql = "WITH cte AS (SELECT * FROM t) SELECT a, b FROM cte WHERE x = 1" + data_file = tmp_path / "mini_interact.jsonl" + data_file.write_text( + json.dumps({"instance_id": "x_1", "sol_sql": sql}) + "\n", + ) + monkeypatch.setattr( + paths_mod, "benchmark_data_file", + lambda benchmark: data_file, + ) + + out = _build_original_sql_index("mini_interact") + assert out["x_1"] == [sql] + assert len(out["x_1"]) == 1 From 15ee09717f267f92987c644cbd6f80cd980195c8 Mon Sep 17 00:00:00 2001 From: Egor Kraev Date: Tue, 2 Jun 2026 15:37:06 +0200 Subject: [PATCH 20/29] DEV-1515 round 8: data_dir routing, upload ordering, attempt-aware merge MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three Codex findings, all on the round 4-7 grader plumbing: Item 1 (Codex major — run.py:1046-1058): Local ``_grade_local_row`` was computing ``per_task_db`` from ``paths.benchmark_data_root(_benchmark_canonical)`` — the GLOBAL benchmark root — instead of the caller-provided ``data_dir`` the agent actually executed against. An alternate checkout, a temp fixture, or a ``BIRD_DB_PATH`` override would silently route the grader at a different sqlite than the agent's, so a correct submission could be marked failing for purely path-routing reasons. Fix: root ``per_task_db`` at ``Path(data_dir)`` (already in scope on the closure). Mirrors the cloud worker's ``cfg["data_dir"]`` pattern. Regression test extended to assert ``db_path`` is rooted under the test's ``data_dir`` arg. Item 2 (Codex major — cloud/ray_app.py:570-631): ``_run_one_in_actor`` uploaded the attempt row BEFORE the inline grader + annotation upload. ``driver.wait_until_done`` returns ``done`` when ``len(attempts) >= total`` (attempt rows in GCS). Non-detached ``submit`` immediately calls ``fetch``; if the row landed first, the in-flight annotation upload could race the download, and the cascade aggregator would either drop the ``cascading_phase1`` block (none present) or surface ``cascading_phase1_error`` (some present). Fix: swap the order. Inline-grade + annotation upload runs FIRST, then ``_gcs.write_row``. The row blob becomes the canonical "task fully done, including annotation" marker. New ordering test records the call sequence and asserts annotation < row. Item 3 (Codex major — cloud/post_run_merge.py:487-497): The merge dest is keyed only by ``(benchmark, db, instance_id, run_id)`` and skipped when already present. Resubmit reuses the same ``run_id`` and bumps ``attempt``; a partial earlier fetch could pin attempt-1's annotation forever while ``eval.json`` / ``results.db`` reflect attempt-2. Fix: parse ``submission.trajectory_path`` (``rows//attempt-N.json``) on src and dest; overwrite ONLY when the new attempt number is strictly greater. Unknown / equal / older → keep existing (safety floor preserved). New report counters ``overwritten_newer_attempt`` + ``overwritten_paths``. Prerequisite: ``grade_one_submission`` now takes an ``attempt`` kwarg (default 1) and uses it in ``trajectory_path``; cloud worker threads the real attempt through. Pre-fix the path was hardcoded to ``attempt-1.json`` and the attempt comparison would always short-circuit to ``equal``. 3 new merge tests + 1 helper test. Co-Authored-By: Claude Opus 4.7 --- .../cloud/post_run_merge.py | 59 ++++++- src/bird_interact_agents/cloud/ray_app.py | 21 ++- .../eval/grade_in_place.py | 10 +- src/bird_interact_agents/run.py | 9 +- tests/cloud/test_fetch_annotation_merge.py | 148 ++++++++++++++++++ tests/cloud/test_inline_grader.py | 94 +++++++++++ tests/test_run_local_inline_grader.py | 17 ++ 7 files changed, 351 insertions(+), 7 deletions(-) diff --git a/src/bird_interact_agents/cloud/post_run_merge.py b/src/bird_interact_agents/cloud/post_run_merge.py index 60356a7d..ef7c2c6c 100644 --- a/src/bird_interact_agents/cloud/post_run_merge.py +++ b/src/bird_interact_agents/cloud/post_run_merge.py @@ -75,6 +75,7 @@ import fcntl import json import os +import re import tempfile from contextlib import contextmanager from pathlib import Path @@ -433,12 +434,32 @@ class AnnotationMergeReport(BaseModel): benchmark: str merged: int = 0 skipped_existing: int = 0 + overwritten_newer_attempt: int = 0 rejected_invalid: int = 0 merged_paths: list[str] = [] skipped_paths: list[str] = [] + overwritten_paths: list[str] = [] rejected_paths: list[str] = [] +_ATTEMPT_RE = re.compile(r"attempt-(\d+)\.json") + + +def _attempt_from_trajectory_path(traj: str | None) -> int | None: + """Parse ``rows//attempt-N.json`` → ``N``. Returns None when + the path is missing or unparseable so callers can fall back to a + safe default (no-overwrite).""" + if not traj: + return None + m = _ATTEMPT_RE.search(traj) + if m is None: + return None + try: + return int(m.group(1)) + except ValueError: + return None + + def merge_submission_annotations( *, downloaded_run_dir: Path, @@ -492,8 +513,42 @@ def merge_submission_annotations( repo_root=main_checkout_root, ) if dest.exists(): - report.skipped_existing += 1 - report.skipped_paths.append(str(dest)) + # Resubmit reuses the same ``run_id`` and bumps the + # per-task ``attempt`` number; the canonical + # ``submission_annotation_path`` does NOT carry attempt + # in the filename, so a partial earlier fetch would + # otherwise pin attempt-1's annotation forever even + # when attempt-2's row is the result-of-record + # (Codex r7). Compare ``submission.trajectory_path`` + # of src vs dest — overwrite ONLY when the new attempt + # number is strictly greater. Skip on equal / unknown + # to preserve the existing no-overwrite safety floor. + src_attempt = _attempt_from_trajectory_path( + ann.submission.trajectory_path, + ) + try: + dest_ann = SubmissionAnnotation.model_validate_json( + dest.read_text(), + ) + dest_attempt = _attempt_from_trajectory_path( + dest_ann.submission.trajectory_path, + ) + except (ValidationError, ValueError, OSError): + dest_attempt = None + if ( + src_attempt is not None + and dest_attempt is not None + and src_attempt > dest_attempt + ): + dest.write_text( + ann.model_dump_json(indent=2, exclude_none=False) + + "\n", + ) + report.overwritten_newer_attempt += 1 + report.overwritten_paths.append(str(dest)) + else: + report.skipped_existing += 1 + report.skipped_paths.append(str(dest)) continue dest.parent.mkdir(parents=True, exist_ok=True) dest.write_text( diff --git a/src/bird_interact_agents/cloud/ray_app.py b/src/bird_interact_agents/cloud/ray_app.py index 00da075e..c279dfea 100644 --- a/src/bird_interact_agents/cloud/ray_app.py +++ b/src/bird_interact_agents/cloud/ray_app.py @@ -567,8 +567,6 @@ def _run_one_in_actor( finally: pass - _gcs.write_row(run_id, iid, attempt, row, client=gcs_client) - # DEV-1515: inline grader produces a SubmissionAnnotation per task # (cascading verdict + Tier 2 informational). Failure here MUST NOT # block the row/log upload — it's diagnostic, not result-of-record. @@ -580,6 +578,17 @@ def _run_one_in_actor( # (unbound data_dir, missing submitted_sql, broken gold, grader # exception) we fall back to writing + uploading a fail-everything # annotation — mirrors ``run._grade_local_row``. + # + # Codex r7 ordering: upload the annotation BEFORE the attempt row. + # ``driver.wait_until_done`` returns ``done`` when + # ``len(attempts) >= total`` (i.e. once every attempt row blob + # exists in GCS). Non-detached ``submit`` then immediately calls + # ``fetch``; if the row landed before the annotation, fetch could + # race the in-flight annotation upload and the cascade aggregator + # would either drop ``cascading_phase1`` entirely or surface + # ``cascading_phase1_error``. Uploading the annotation first makes + # the row blob the canonical "task fully done, including + # annotation" marker. _grader_data_dir = locals().get("data_dir") annotation_dir = Path(tempfile.mkdtemp(prefix="bird_submission_annot_")) _row_submitted_sql = row.get("submitted_sql") @@ -623,6 +632,7 @@ def _run_one_in_actor( n_ask_user_calls=row.get("usage", {}).get("n_ask_user_calls") if isinstance(row.get("usage"), dict) else None, predicted_row_count=None, + attempt=attempt, ) _gcs.write_submission_annotation( run_id, iid, json.loads(ann_path.read_text()), @@ -639,7 +649,7 @@ def _run_one_in_actor( or ""), benchmark=_cloud_benchmark(cfg), run_id=run_id, - trajectory_path=f"rows/{iid}/attempt-1.json", + trajectory_path=f"rows/{iid}/attempt-{attempt}.json", failure_details=( f"cloud inline grader raised: " f"{type(grader_exc).__name__}: {grader_exc}" @@ -657,6 +667,11 @@ def _run_one_in_actor( finally: shutil.rmtree(annotation_dir, ignore_errors=True) + # Codex r7: annotation upload is now BEFORE the attempt row write, + # so ``wait_until_done`` (which counts attempt rows) only sees the + # row after the cascade annotation has landed in GCS. + _gcs.write_row(run_id, iid, attempt, row, client=gcs_client) + try: log_bytes = log_tmp.read_bytes() if log_tmp.exists() else b"" except OSError: diff --git a/src/bird_interact_agents/eval/grade_in_place.py b/src/bird_interact_agents/eval/grade_in_place.py index 815995a4..d7aa75a3 100644 --- a/src/bird_interact_agents/eval/grade_in_place.py +++ b/src/bird_interact_agents/eval/grade_in_place.py @@ -490,6 +490,7 @@ def grade_one_submission( n_ask_user_calls: Optional[int] = None, predicted_row_count: Optional[int] = None, user_sim_interaction: Optional[UserSimInteraction] = None, + attempt: int = 1, ) -> Path: """Inline-grade one submission and write the per-row ``submission_annotation.json``. Idempotent at the per-(task, run) @@ -499,6 +500,13 @@ def grade_one_submission( Shared between cloud (``cloud.ray_app``) and local (``run``) so the ``cascading_phase1`` block in ``eval.json`` is populated regardless of where the run was launched. + + ``attempt`` MUST reflect the real per-task attempt number — the + post-fetch merge in ``post_run_merge.merge_submission_annotations`` + parses ``submission.trajectory_path`` to compare resubmit attempts + (Codex r7); leaving the previous hardcoded ``"attempt-1"`` would + silently preserve attempt-1 annotations on resubmit even when + attempt-2's annotation is downloaded. """ instance_id = task_data["instance_id"] selected_database = task_data["selected_database"] @@ -524,7 +532,7 @@ def grade_one_submission( submitted_sql=submitted_sql, db_path=db_path, conn=conn, - trajectory_path=f"rows/{instance_id}/attempt-1.json", + trajectory_path=f"rows/{instance_id}/attempt-{attempt}.json", cost_usd_agent=cost_usd_agent, cost_usd_user_sim=cost_usd_user_sim, duration_s=duration_s, diff --git a/src/bird_interact_agents/run.py b/src/bird_interact_agents/run.py index 1c05748f..4fad7c5b 100644 --- a/src/bird_interact_agents/run.py +++ b/src/bird_interact_agents/run.py @@ -1043,8 +1043,15 @@ def _grade_local_row(td: dict, r: dict) -> None: ), ) return + # Root the per-task sqlite at the caller-provided ``data_dir`` + # (the same path the agent's SQL executed against) — NOT the + # global ``paths.benchmark_data_root``. Otherwise an alternate + # checkout, a tmp fixture, or a ``BIRD_DB_PATH`` override would + # have the agent and grader disagreeing on schema/data, and a + # correct submission could be marked failing. Mirrors the + # cloud worker, which uses ``cfg["data_dir"]`` (Codex r7). per_task_db = ( - paths.benchmark_data_root(_benchmark_canonical) + Path(data_dir) / selected_database / f"{selected_database}.sqlite" ) diff --git a/tests/cloud/test_fetch_annotation_merge.py b/tests/cloud/test_fetch_annotation_merge.py index 0ef3b759..f628ca4a 100644 --- a/tests/cloud/test_fetch_annotation_merge.py +++ b/tests/cloud/test_fetch_annotation_merge.py @@ -201,3 +201,151 @@ def test_merge_writes_audit_report(tmp_path): body = json.loads(audit.read_text()) assert body["merged"] == 2 assert body["run_id"] == "r1" + + +# --------------------------------------------------------------------------- +# Codex r7: resubmit-aware overwrite. The canonical +# ``submission_annotation_path`` does NOT carry the per-task ``attempt`` +# in the filename, so a partial fetch followed by a resubmit (which +# reuses the same ``run_id`` and bumps ``attempt``) would otherwise pin +# attempt-1's annotation forever — while ``eval.json`` / ``results.db`` +# reflect attempt-2. The merge now parses ``submission.trajectory_path`` +# (``rows//attempt-N.json``) on both src and dest and overwrites +# ONLY when the new attempt is strictly newer. +# --------------------------------------------------------------------------- + + +def _make_dict_with_attempt(instance_id: str, attempt: int) -> dict: + body = _valid_submission_annotation_dict(instance_id) + body["submission"]["trajectory_path"] = ( + f"rows/{instance_id}/attempt-{attempt}.json" + ) + body["annotated_by"] = f"attempt-{attempt}-grader" + return body + + +def test_merge_overwrites_when_new_attempt_strictly_newer(tmp_path): + """Resubmit pushes attempt-2's annotation; the existing dest is + attempt-1. The merge MUST overwrite and bump the + ``overwritten_newer_attempt`` counter.""" + from bird_interact_agents.cloud.post_run_merge import ( + merge_submission_annotations, + ) + + main_checkout = tmp_path / "checkout" + dest_dir = main_checkout / "annotations" / "mini_interact" / "alien" + dest_dir.mkdir(parents=True) + # Pre-existing dest from a prior partial fetch — attempt-1. + (dest_dir / "alien_1.submission.r1.json").write_text( + json.dumps(_make_dict_with_attempt("alien_1", 1)), + ) + + # Newly downloaded run dir carries attempt-2's annotation. + downloaded = tmp_path / "downloaded" + rows = downloaded / "rows" / "alien_1" + rows.mkdir(parents=True) + (rows / "submission_annotation.json").write_text( + json.dumps(_make_dict_with_attempt("alien_1", 2)), + ) + + report = merge_submission_annotations( + downloaded_run_dir=downloaded, + run_id="r1", + benchmark="mini-interact", + main_checkout_root=main_checkout, + ) + + assert report.overwritten_newer_attempt == 1, report + assert report.skipped_existing == 0, report + assert report.merged == 0, report + surviving = json.loads( + (dest_dir / "alien_1.submission.r1.json").read_text(), + ) + assert surviving["annotated_by"] == "attempt-2-grader" + assert surviving["submission"]["trajectory_path"] == ( + "rows/alien_1/attempt-2.json" + ) + + +def test_merge_does_not_overwrite_when_new_attempt_is_older_or_equal(tmp_path): + """Symmetric safety case: attempt-2 already on disk, attempt-1 + being merged — MUST keep attempt-2 (no regression to older row).""" + from bird_interact_agents.cloud.post_run_merge import ( + merge_submission_annotations, + ) + + main_checkout = tmp_path / "checkout" + dest_dir = main_checkout / "annotations" / "mini_interact" / "alien" + dest_dir.mkdir(parents=True) + (dest_dir / "alien_1.submission.r1.json").write_text( + json.dumps(_make_dict_with_attempt("alien_1", 2)), + ) + + downloaded = tmp_path / "downloaded" + rows = downloaded / "rows" / "alien_1" + rows.mkdir(parents=True) + (rows / "submission_annotation.json").write_text( + json.dumps(_make_dict_with_attempt("alien_1", 1)), + ) + + report = merge_submission_annotations( + downloaded_run_dir=downloaded, + run_id="r1", + benchmark="mini-interact", + main_checkout_root=main_checkout, + ) + + assert report.overwritten_newer_attempt == 0, report + assert report.skipped_existing == 1, report + surviving = json.loads( + (dest_dir / "alien_1.submission.r1.json").read_text(), + ) + assert surviving["annotated_by"] == "attempt-2-grader" + + +def test_merge_skips_when_attempts_equal(tmp_path): + """Equal attempts — preserve existing (no-op repeated fetch).""" + from bird_interact_agents.cloud.post_run_merge import ( + merge_submission_annotations, + ) + + main_checkout = tmp_path / "checkout" + dest_dir = main_checkout / "annotations" / "mini_interact" / "alien" + dest_dir.mkdir(parents=True) + pre = _make_dict_with_attempt("alien_1", 1) + pre["annotated_by"] = "human-pre-existing" + (dest_dir / "alien_1.submission.r1.json").write_text(json.dumps(pre)) + + downloaded = tmp_path / "downloaded" + rows = downloaded / "rows" / "alien_1" + rows.mkdir(parents=True) + fresh = _make_dict_with_attempt("alien_1", 1) + fresh["annotated_by"] = "auto-fresh" + (rows / "submission_annotation.json").write_text(json.dumps(fresh)) + + report = merge_submission_annotations( + downloaded_run_dir=downloaded, + run_id="r1", + benchmark="mini-interact", + main_checkout_root=main_checkout, + ) + assert report.overwritten_newer_attempt == 0, report + assert report.skipped_existing == 1, report + surviving = json.loads( + (dest_dir / "alien_1.submission.r1.json").read_text(), + ) + assert surviving["annotated_by"] == "human-pre-existing" + + +def test_attempt_from_trajectory_path_parses_and_defaults(): + """Pin the parsing helper's contract — unparseable input returns + None so the caller can fall back to no-overwrite.""" + from bird_interact_agents.cloud.post_run_merge import ( + _attempt_from_trajectory_path, + ) + + assert _attempt_from_trajectory_path("rows/alien_1/attempt-1.json") == 1 + assert _attempt_from_trajectory_path("rows/alien_1/attempt-42.json") == 42 + assert _attempt_from_trajectory_path(None) is None + assert _attempt_from_trajectory_path("") is None + assert _attempt_from_trajectory_path("rows/alien_1/something_else.json") is None diff --git a/tests/cloud/test_inline_grader.py b/tests/cloud/test_inline_grader.py index 9cc865c2..089cf8eb 100644 --- a/tests/cloud/test_inline_grader.py +++ b/tests/cloud/test_inline_grader.py @@ -315,3 +315,97 @@ def _spy_grader(**kwargs): or "task errored before reaching submit" in payload["failure_classification"]["details"] ) + + +def test_cloud_uploads_annotation_before_attempt_row( + monkeypatch, fake_gcs_bucket, +): + """Codex r7 ordering: ``_run_one_in_actor`` MUST upload the per-row + submission_annotation.json BEFORE the attempt row blob. ``driver. + wait_until_done`` counts attempt rows to decide ``done``; if the + row landed first, non-detached ``submit`` + immediate ``fetch`` + could race the annotation upload and the cascade aggregator would + either drop ``cascading_phase1`` or surface + ``cascading_phase1_error``. + + This test records the call order of the two ``_gcs.write_*`` + helpers and asserts the annotation write index is strictly less + than the row write index. + """ + from bird_interact_agents.cloud import ray_app, upload_back + + RUN_ID = "20260602T1230-cloud-order-round7" + + client, _store = fake_gcs_bucket + monkeypatch.setattr(ray_app, "default_gcs_client", lambda: client) + monkeypatch.setattr(ray_app, "_maybe_build_cached_runner", lambda _cfg: None) + monkeypatch.setattr(ray_app, "download_slayer_setup", lambda *a, **k: None) + + async def fake_run_one_task(task_data, **_kw): + return { + "instance_id": task_data["instance_id"], + "database": task_data.get("selected_database", "db_a"), + "phase1_passed": True, "phase2_passed": True, "total_reward": 1.0, + "duration_s": 0.01, "error": None, + "submitted_sql": "SELECT 1", + } + monkeypatch.setattr( + "bird_interact_agents.run.run_one_task", fake_run_one_task, + ) + + # Stub the grader so it just writes a minimal annotation file the + # caller (the cloud worker) then uploads. Mirrors the success path. + def _fake_grade(**kwargs): + rows_dir = kwargs["rows_dir"] + instance_id = kwargs["instance_id"] + d = rows_dir / instance_id + d.mkdir(parents=True, exist_ok=True) + (d / "submission_annotation.json").write_text("{}") + from bird_interact_agents.eval import grade_in_place + monkeypatch.setattr(grade_in_place, "grade_and_write", _fake_grade, raising=True) + monkeypatch.setattr(ray_app, "grade_and_write", _fake_grade, raising=True) + + # Record the order of write_row vs write_submission_annotation. + order: list[str] = [] + monkeypatch.setattr( + ray_app._gcs, "write_row", + lambda *a, **kw: order.append("write_row"), + ) + monkeypatch.setattr( + ray_app._gcs, "write_submission_annotation", + lambda *a, **kw: order.append("write_submission_annotation"), + ) + monkeypatch.setattr( + ray_app._gcs, "write_log", lambda *a, **kw: None, + ) + monkeypatch.setattr(upload_back, "upload_per_task_debug", lambda **kw: None) + monkeypatch.setattr( + upload_back, "upload_per_task_setup_sessions", lambda **kw: None, + ) + monkeypatch.setattr( + upload_back, "upload_otf_reference_delta", lambda **kw: None, + ) + + actor = ray_app._LocalActor( + {"framework": "pydantic_ai_otf_encode", "query_mode": "slayer", + "mode": "a-interact", "agent_model": "anthropic/claude-sonnet-4-5", + "user_sim_model": "anthropic/claude-haiku-4-5-20251001", + "patience": 3, "strict": False, "use_audited_gold_sql": False, + "prompt_cache": True, "max_depth": 3, "slayer_setup": "on-the-fly", + "slayer_storage_root": "/data/slayer_models", + "data_dir": "/data/mini-interact"}, + RUN_ID, 1, gcs_client=client, + ) + actor.run_one({"instance_id": "db_a_1", "selected_database": "db_a"}) + + # Both must have fired. + assert "write_submission_annotation" in order, ( + f"annotation upload missing; calls={order}" + ) + assert "write_row" in order, f"row upload missing; calls={order}" + ann_idx = order.index("write_submission_annotation") + row_idx = order.index("write_row") + assert ann_idx < row_idx, ( + f"annotation MUST land before row (so wait_until_done can rely " + f"on the row as 'fully done' marker); got calls={order}" + ) diff --git a/tests/test_run_local_inline_grader.py b/tests/test_run_local_inline_grader.py index 17d461ac..ff8ce925 100644 --- a/tests/test_run_local_inline_grader.py +++ b/tests/test_run_local_inline_grader.py @@ -111,6 +111,7 @@ def _stub_grader(*, task_data, submitted_sql, rows_dir, run_id, benchmark, db_pa "benchmark": benchmark, "rows_dir": rows_dir, "run_id": run_id, + "db_path": db_path, }) out_dir = Path(rows_dir) / task_data["instance_id"] out_dir.mkdir(parents=True, exist_ok=True) @@ -214,6 +215,22 @@ def _stub_grader(*, task_data, submitted_sql, rows_dir, run_id, benchmark, db_pa # alien_2 failed everything. Each tier should therefore count 1. assert cp["counts"]["n3"] == 1, f"got cp={cp}" assert cp["rates"]["n3"] == 0.5, f"got cp={cp}" + # Codex r7: ``db_path`` MUST be rooted at the caller-provided + # ``data_dir`` (the same sqlite the agent ran against) — NOT + # ``paths.benchmark_data_root``. Without this guard a local run + # pointed at a tmp / alternate checkout / env-overridden data dir + # would grade against the global sqlite, and the cascade verdict + # could disagree with the agent's submission for purely path-routing + # reasons. + expected_data_dir = str(tmp_path / "ignored_data_dir") + for c in calls: + assert str(c["db_path"]).startswith(expected_data_dir), ( + f"db_path must be rooted at data_dir={expected_data_dir!r}; " + f"got db_path={c['db_path']!r}" + ) + assert c["db_path"].name == "alien.sqlite", ( + f"db_path leaf must be .sqlite; got {c['db_path']!r}" + ) @pytest.mark.asyncio From baaaef45b18f591053e878a619850eb1b240370d Mon Sep 17 00:00:00 2001 From: Egor Kraev Date: Tue, 2 Jun 2026 16:00:27 +0200 Subject: [PATCH 21/29] DEV-1515 round 9: restore observation columns + multi-stmt miss_diagnostics MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two Codex findings on the round-8 grader plumbing: Item 1 (Codex major — run.py:984-985 + results_db.py): ``run.py::_persist`` had been passing ``phase1_observation_audited`` / ``phase1_observation_original`` into ``TaskResultRow`` for every per-task insert, but the model + DDL silently lost those two fields somewhere along the DEV-1515 dual-eval cleanup. Pydantic's default ``extra="ignore"`` ate them without warning, so ``results.db`` quietly stopped storing the observation strings even though ~6 agent flavors (claude_sdk, pydantic_ai, pydantic_ai_otf_encode, claude_sdk_otf_ainteract, plus ``agents/_submit.py``) continued to emit them on every row. Fix: * Add ``phase1_observation_audited: str | None = None`` + ``phase1_observation_original: str | None = None`` to ``TaskResultRow``. * Append both to ``_DIAGNOSTIC_COLUMNS`` so ``open_db`` ALTERs pre-existing on-disk DBs to add the columns (mirroring how ``phase1_observation`` etc. are upgraded). * Append both to the fresh DDL string for documentation parity. * Extend the ``INSERT OR REPLACE`` statement in ``insert_task_result`` to bind both fields. * Two regression tests pin (a) a model-INSERT-SELECT round trip in a fresh DB and (b) the column-upgrade path on a pre-existing DB that lacks both columns at open time. Item 2 (Codex minor — tolerant_grader.py:1081-1089): ``_compute_miss_diagnostics`` asserted ``len(sqls) <= 1`` on every variant's ``audited_sol_sql`` and on ``original_sol_sql``. But the grader's executor (``_multi_sql_execute``) explicitly supports multi-statement gold (DDL prelude + final SELECT). When a multi-statement gold produced a strict miss, the assert raised, the round-5 outer except clause caught it, and the row fell back to the round-6/7 fail-everything fallback — losing all structured miss_patterns + Tier 2 informational detail for that row. Fix: drop the asserts and use the LAST statement of the gold's ``audited_sol_sql`` for sqlglot parsing. The setup statements don't constrain miss patterns; the SELECT under diagnosis is what determines tables / aggregation shape / predicate count. Replaced the two pre-fix "must raise" tests with two new ones pinning the post-fix contract: * test_multi_statement_audited_gold_uses_last_for_sql_signals — best_variant_tables_referenced parses from the SELECT, not the DDL. * test_multi_statement_original_gold_does_not_crash_diagnostics — original-gold side handles multi-stmt gracefully too. Co-Authored-By: Claude Opus 4.7 --- .../eval/tolerant_grader.py | 33 +++--- src/bird_interact_agents/results_db.py | 25 ++++- tests/test_miss_diagnostics.py | 98 ++++++++++------ tests/test_results_db.py | 105 ++++++++++++++++++ 4 files changed, 207 insertions(+), 54 deletions(-) diff --git a/src/bird_interact_agents/eval/tolerant_grader.py b/src/bird_interact_agents/eval/tolerant_grader.py index 67a1df6d..528f4e39 100644 --- a/src/bird_interact_agents/eval/tolerant_grader.py +++ b/src/bird_interact_agents/eval/tolerant_grader.py @@ -1071,23 +1071,17 @@ def _compute_miss_diagnostics( SQL-derived signals are populated only when sqlglot parsing succeeds for the relevant side; otherwise the corresponding Optional[T] field stays None and ``sql_parse_error`` lands in the - flag list. Multi-statement gold (CREATE TEMP + final SELECT) - triggers a defensive AssertionError — the SELECT-task contract - is single-statement. + flag list. + + Multi-statement gold (CREATE TEMP + final SELECT, etc.) is + handled by using the LAST statement for sqlglot parsing — that's + the SELECT query that actually produced the rowset under + diagnosis; the preceding setup statements don't constrain the + miss patterns. Pre-fix this path asserted single-statement gold + and the AssertionError leaked out of grading, causing the cloud + + local fallbacks to drop the structured ``miss_patterns`` for + multi-statement misses (Codex r8). """ - # Defensive guards (single-statement gold contract). - for v_meta, _v_rows, _v_cols in variant_results: - v_sqls = list(v_meta.get("audited_sol_sql") or []) - assert len(v_sqls) <= 1, ( - f"diagnostics only support single-statement audited_sol_sql; " - f"variant {v_meta.get('variant_id')!r} has {len(v_sqls)} stmts " - f"(multi-statement is M-task territory, out of scope)" - ) - assert len(original_sol_sql) <= 1, ( - f"diagnostics only support single-statement original_sol_sql; " - f"got {len(original_sol_sql)} stmts (multi-statement is M-task " - f"territory, out of scope)" - ) if not variant_results: # Should be impossible (cascade would short-circuit) but guard. raise RuntimeError( @@ -1098,7 +1092,12 @@ def _compute_miss_diagnostics( pred_rows=pred_rows, variant_results=variant_results, ) best_variant_id = str(best_meta.get("variant_id") or "") - best_sql = (best_meta.get("audited_sol_sql") or [""])[0] + # Multi-statement gold: the LAST statement is the SELECT under + # diagnosis. Empty list → use empty string so the parse fails and + # we surface ``sql_parse_error`` (preserving the round-3 missing- + # gold guard pattern). + _best_sql_list = list(best_meta.get("audited_sol_sql") or []) + best_sql = _best_sql_list[-1] if _best_sql_list else "" overlap = _bag_overlap(pred_rows, best_rows) relation = _bag_relation(pred=pred_rows, gold=best_rows) diff --git a/src/bird_interact_agents/results_db.py b/src/bird_interact_agents/results_db.py index 55668211..a6c5d4fb 100644 --- a/src/bird_interact_agents/results_db.py +++ b/src/bird_interact_agents/results_db.py @@ -46,6 +46,8 @@ gold_result_json TEXT, n_agent_turns INTEGER, tool_call_stats_json TEXT, + phase1_observation_audited TEXT, + phase1_observation_original TEXT, PRIMARY KEY (run_id, framework, mode, query_mode, instance_id) ) """ @@ -68,6 +70,15 @@ # "total_calls": int, "total_errors": int, # "error_samples": [{"tool": str, "error": str}, ...]} ("tool_call_stats_json", "TEXT"), + # DEV-1515 round 9 (Codex r8): dual-eval observation strings + # produced by every agent flavor's submit helpers + + # ``agents/_submit.py``. ``run.py`` was passing these into + # ``TaskResultRow`` but they had been silently dropped from the + # model + DDL; Pydantic's default ``extra="ignore"`` ate them + # without warning, so ``results.db`` lost the audited/original + # observation diagnostic — re-add them as nullable TEXT columns. + ("phase1_observation_audited", "TEXT"), + ("phase1_observation_original", "TEXT"), ] _RUN_METADATA_DDL = """ @@ -113,6 +124,13 @@ class TaskResultRow(BaseModel): gold_result_json: str | None = None n_agent_turns: int | None = None tool_call_stats_json: str | None = None + # DEV-1515 round 9 (Codex r8): the agent flavors + agents/_submit.py + # have always emitted these observation strings; they were + # accidentally dropped from this model along with the dual-eval + # bool flags that DEV-1515 retired. Restoring as nullable TEXT so + # ``run.py::_persist`` actually writes them through to ``results.db``. + phase1_observation_audited: str | None = None + phase1_observation_original: str | None = None def open_db(path: Path | str) -> sqlite3.Connection: @@ -146,8 +164,9 @@ def insert_task_result(conn: sqlite3.Connection, row: TaskResultRow) -> None: error, usage_json, user_query, submission_status, phase1_observation, phase2_observation, predicted_result_json, gold_result_json, - n_agent_turns, tool_call_stats_json) - VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?) + n_agent_turns, tool_call_stats_json, + phase1_observation_audited, phase1_observation_original) + VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?) """, ( row.run_id, row.framework, row.mode, row.query_mode, @@ -159,6 +178,8 @@ def insert_task_result(conn: sqlite3.Connection, row: TaskResultRow) -> None: row.phase2_observation, row.predicted_result_json, row.gold_result_json, row.n_agent_turns, row.tool_call_stats_json, + row.phase1_observation_audited, + row.phase1_observation_original, ), ) conn.commit() diff --git a/tests/test_miss_diagnostics.py b/tests/test_miss_diagnostics.py index 5aff1690..016eb596 100644 --- a/tests/test_miss_diagnostics.py +++ b/tests/test_miss_diagnostics.py @@ -1168,45 +1168,73 @@ def test_back_compat_old_submission_evaluation_validates(): # --------------------------------------------------------------------------- -# Defensive: multi-statement gold is rejected +# Codex r8: multi-statement gold (CREATE TEMP + SELECT, DDL prelude, etc.) +# MUST NOT crash diagnostics. The pre-fix assertion bubbled an +# AssertionError out of ``_compute_miss_diagnostics`` and the cloud + +# local fallbacks dropped the structured ``miss_patterns`` for the +# entire row. The fix uses the LAST statement of the gold's +# ``audited_sol_sql`` list for sqlglot parsing — that's the SELECT +# query under diagnosis; the setup statements don't constrain miss +# patterns. # --------------------------------------------------------------------------- -def test_multi_statement_audited_gold_raises_assertion(tmp_path: Path): - """Diagnostics only support single-statement gold (the contract for - SELECT tasks). A multi-statement audited_sol_sql must trigger an - explicit AssertionError so the bug doesn't silently parse the - wrong statement. Multi-statement M-tasks are out of scope.""" - import pytest as _pytest +def test_multi_statement_audited_gold_uses_last_for_sql_signals( + tmp_path: Path, +): + """A 2-statement audited gold (DDL setup + SELECT) must compute + diagnostics successfully; the sqlglot-derived signals come from + the SELECT statement, NOT the DDL.""" db = _build_db(tmp_path) - with _pytest.raises(AssertionError, match="single-statement"): - _grade( - db=db, - submitted_sql="SELECT id FROM t1 WHERE id IN (4, 5)", - audited_sol_sql_per_variant=[ - ("primary", True, [ - "CREATE TEMP TABLE tmp AS SELECT id FROM t1", - "SELECT id FROM tmp", - ]), - ], - ) + # Agent reads from t1; gold's "real" reading is the SELECT against + # a temp table built from t1. Both reference t1 via the SELECT. + verdict = _grade( + db=db, + submitted_sql="SELECT id FROM t1 WHERE id = 9999", # disjoint -> miss + audited_sol_sql_per_variant=[ + ("primary", True, [ + "CREATE TEMP TABLE tmp AS SELECT id FROM t1", + "SELECT id FROM tmp", + ]), + ], + ) + md = verdict.miss_diagnostics + assert md is not None, ( + "multi-statement audited gold must NOT crash diagnostics; " + "got miss_diagnostics=None" + ) + # The best-variant SQL parse used the SELECT (the LAST statement), + # not the CREATE TEMP TABLE (which sqlglot would also parse but + # would yield empty / wrong table extraction). + assert md.best_variant_sql_parse_ok is True + assert md.best_variant_tables_referenced == ["tmp"], ( + f"best_variant_tables_referenced must be parsed from the LAST " + f"statement (SELECT id FROM tmp), got " + f"{md.best_variant_tables_referenced!r}" + ) -def test_multi_statement_original_gold_raises_assertion(tmp_path: Path): - """Codex major #7 — the single-statement assertion applies to - original_sol_sql too, not just audited. Multi-statement original - gold must raise the same explicit AssertionError.""" - import pytest as _pytest +def test_multi_statement_original_gold_does_not_crash_diagnostics( + tmp_path: Path, +): + """Original gold can also carry a multi-statement list (the + same DDL + SELECT shape). Diagnostics must still compute — the + original gold is only used for the ``original_gold_row_count`` + field, not sqlglot parsing — so any number of statements is OK + as long as the cascade ran with them.""" db = _build_db(tmp_path) - with _pytest.raises(AssertionError, match="single-statement"): - _grade( - db=db, - submitted_sql="SELECT id FROM t1 WHERE id IN (4, 5)", - audited_sol_sql_per_variant=[ - ("primary", True, ["SELECT id FROM t1 WHERE id IN (1, 2)"]), - ], - original_sol_sql=[ - "CREATE TEMP TABLE tmp AS SELECT id FROM t1", - "SELECT id FROM tmp", - ], - ) + verdict = _grade( + db=db, + submitted_sql="SELECT id FROM t1 WHERE id = 9999", + audited_sol_sql_per_variant=[ + ("primary", True, ["SELECT id FROM t1 WHERE id IN (1, 2)"]), + ], + original_sol_sql=[ + "CREATE TEMP TABLE tmp AS SELECT id FROM t1", + "SELECT id FROM tmp", + ], + ) + md = verdict.miss_diagnostics + assert md is not None, ( + "multi-statement original gold must not crash diagnostics" + ) diff --git a/tests/test_results_db.py b/tests/test_results_db.py index ae7e686c..5de3be09 100644 --- a/tests/test_results_db.py +++ b/tests/test_results_db.py @@ -274,3 +274,108 @@ def test_open_db_migrates_pre_diagnostic_table(tmp_path): "SELECT instance_id, submission_status FROM task_results" )) assert rows == [("new_1", "passed_phase1"), ("old_1", "never_submitted")] + + +# --------------------------------------------------------------------------- +# Codex r8: dual-eval observation columns. Every agent flavor's submit +# helper emits ``phase1_observation_audited`` + ``phase1_observation_original``, +# but for a while those fields were missing from ``TaskResultRow`` + the +# DDL and Pydantic's default ``extra="ignore"`` silently dropped them. +# These tests pin both shapes: +# * round-trip through the model + INSERT +# * old DB on disk gets the columns ALTER'd in on open +# --------------------------------------------------------------------------- + + +def test_dual_eval_observation_columns_round_trip(tmp_path): + """Both observation strings survive a model -> INSERT -> SELECT + round trip in a fresh DB.""" + from bird_interact_agents.results_db import ( + TaskResultRow, insert_task_result, open_db, + ) + + conn = open_db(tmp_path / "results.db") + insert_task_result(conn, TaskResultRow( + run_id="r", framework="pydantic_ai", mode="a-interact", + query_mode="raw", instance_id="alien_1", database="alien", + started_at=0.0, duration_s=0.0, + phase1_passed=False, phase2_passed=False, total_reward=0.0, + phase1_observation_audited="audited PASS", + phase1_observation_original="original FAIL", + )) + rows = list(conn.execute( + "SELECT phase1_observation_audited, phase1_observation_original " + "FROM task_results" + )) + assert rows == [("audited PASS", "original FAIL")] + + +def test_open_db_adds_observation_columns_to_pre_existing_table(tmp_path): + """A results.db left over from a version that lacked the two + observation columns MUST gain them silently on ``open_db`` so the + next ``insert_task_result`` can write through. Mirrors the pattern + used for ``user_query`` / ``phase1_observation`` / etc.""" + import sqlite3 + from bird_interact_agents.results_db import ( + TaskResultRow, insert_task_result, open_db, + ) + + db_path = tmp_path / "old.db" + pre = sqlite3.connect(str(db_path)) + # Pre-DEV-1515 round-9 DDL: every column the CURRENT DDL has EXCEPT + # the two observation columns. Mirrors the real-on-disk shape of a + # DB written by an older version, so the ``_DIAGNOSTIC_COLUMNS`` + # ALTER pass in ``open_db`` is the only path that adds them. + pre.execute( + """ + CREATE TABLE task_results ( + run_id TEXT NOT NULL, + framework TEXT NOT NULL, + mode TEXT NOT NULL, + query_mode TEXT NOT NULL, + instance_id TEXT NOT NULL, + database TEXT NOT NULL, + started_at REAL NOT NULL, + duration_s REAL NOT NULL, + phase1_passed INTEGER NOT NULL, + phase2_passed INTEGER NOT NULL, + total_reward REAL NOT NULL, + submitted_sql TEXT, + submitted_query TEXT, + ground_truth_sql TEXT, + error TEXT, + usage_json TEXT NOT NULL DEFAULT '{}', + user_query TEXT, + submission_status TEXT NOT NULL DEFAULT 'never_submitted', + phase1_observation TEXT, + phase2_observation TEXT, + predicted_result_json TEXT, + gold_result_json TEXT, + n_agent_turns INTEGER, + tool_call_stats_json TEXT, + PRIMARY KEY (run_id, framework, mode, query_mode, instance_id) + ) + """ + ) + pre.commit() + pre.close() + + conn = open_db(db_path) + existing = {row[1] for row in conn.execute("PRAGMA table_info(task_results)")} + assert "phase1_observation_audited" in existing + assert "phase1_observation_original" in existing + + # And a write-then-read still works through the upgraded table. + insert_task_result(conn, TaskResultRow( + run_id="r", framework="pydantic_ai", mode="a-interact", + query_mode="raw", instance_id="alien_1", database="alien", + started_at=0.0, duration_s=0.0, + phase1_passed=False, phase2_passed=False, total_reward=0.0, + phase1_observation_audited="aud", + phase1_observation_original="orig", + )) + rows = list(conn.execute( + "SELECT phase1_observation_audited, phase1_observation_original " + "FROM task_results" + )) + assert rows == [("aud", "orig")] From 43947b2960e43f68799cab51d2b15baa9cf954e6 Mon Sep 17 00:00:00 2001 From: Egor Kraev Date: Tue, 2 Jun 2026 16:30:19 +0200 Subject: [PATCH 22/29] DEV-1515 round 10: rows-dir hygiene, primary-first audits, grader robustness, cloud parity MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Four Codex findings on the round 4-9 plumbing: Group 1 (Codex major — run.py:1001-1002): ``run_evaluation`` reused ``output_dir/rows`` with ``mkdir(exist_ok=True)`` — never wiped. The aggregator at ``cascading_report.aggregate_cascading_phase1`` walks every subdirectory under ``rows_dir``, so rerunning with the same output path but a different ``--limit`` / ``--instance-id`` subset carried forward stale annotations from prior passes, inflating ``cascading_phase1.n_dual_eval_tasks`` and rewriting ``phase1_count`` / ``phase1_rate`` from the union of old + new. Fix: wipe before the run. ``filter_ids is None`` → ``shutil.rmtree`` the whole rows dir; filtered run → reset only the per-instance subdirs in the current task set so unrelated prior-run annotations survive and still contribute. Two new tests: stale-iid is wiped on full reruns; filtered rerun preserves unrelated rows. Group 2 (Codex major — harness.py:472-493 + cloud/_audited_gold_check.py:77-94): Both single-file audited gold indexes did ``out[iid] = row`` with latest-wins semantics. DEV-1515 multi-variant audits ship N rows per ``instance_id`` (one ``primary=True`` + non-primary alternates); on a file where an alternate appears AFTER the primary, the alternate's ``audited_sol_sql`` / ``audit_status`` would overwrite the primary at index time. Interaction-time overlay then applied the alt's reading; cloud audit-gold guard checked against the alt's status. Fix: prefer ``primary=True`` over alternates. Once a primary lands in the index, never overwrite. A non-primary recorded first gets upgraded by the primary later. Three new tests pinning the primary-first contract on both index helpers in both file orders. Group 3 (Codex major — tolerant_grader.py:624-626): Original gold execution was unguarded. The agent SQL and audited variant SQL got try/except wrappers in round 3 / 1.1, but ``_multi_sql_execute(original_sol_sql, ...)`` was bare. An invalid original gold (broken upstream SQL, schema drift, …) raised the whole grader; the local/cloud fail-everything fallback then wrote a generic "grader raised" annotation, losing any valid audited- variant passes (N2/N3) the row would have earned. Fix: wrap the original-exec call, log on raise, set ``orig_rows = []`` + ``original_sql_executed_ok = False``. Then extend the round-3 missing-gold guards on N1 (line 682) and the ``__original__`` fallback gates for N4 (line 724) + N6-N9 (line 790) to AND-in ``original_sql_executed_ok`` so the false-pass shape ``_set_equal([], [])`` / ``compare_tie_order([], [])`` is also blocked on exec-failure. Two new tests: valid audited variant still passes through a broken original; empty agent + broken original + no variants stays False at every tier. Group 4 (Codex minor — cloud/collation.py:69-94): Round 9 restored ``phase1_observation_audited`` / ``phase1_observation_original`` on ``TaskResultRow`` + DDL; the local ``run.py::_persist`` path was already plumbing them. But cloud collation's ``_row_to_task_result_row`` never received the update, so cloud-fetched ``results.db`` files quietly lost the observation columns while local runs retained them. Fix: append both kwargs to the ``TaskResultRow(...)`` build in collation.py to mirror local. New test: synthesize a per-task row with both observation fields set, run collate, assert both come back populated when reading ``results.db``. Co-Authored-By: Claude Opus 4.7 --- .../cloud/_audited_gold_check.py | 21 +- src/bird_interact_agents/cloud/collation.py | 10 + .../eval/tolerant_grader.py | 60 +++-- src/bird_interact_agents/harness.py | 21 +- src/bird_interact_agents/run.py | 24 ++ tests/cloud/test_collation.py | 44 ++++ tests/test_dual_eval.py | 142 +++++++++++ tests/test_run_local_inline_grader.py | 220 ++++++++++++++++++ tests/test_tolerant_grader_orchestration.py | 81 +++++++ 9 files changed, 604 insertions(+), 19 deletions(-) diff --git a/src/bird_interact_agents/cloud/_audited_gold_check.py b/src/bird_interact_agents/cloud/_audited_gold_check.py index b70beee1..59f87ffe 100644 --- a/src/bird_interact_agents/cloud/_audited_gold_check.py +++ b/src/bird_interact_agents/cloud/_audited_gold_check.py @@ -75,6 +75,11 @@ def _load_single_file_audit_index( if not path.exists(): return None out: dict[str, tuple[str, bool, str, str]] = {} + # Track whether each iid's current row came from a primary-tagged + # source so we can prefer ``primary=True`` over alternates when + # multi-variant rows share an instance_id (Codex r9). Mirrors + # ``harness._load_single_file_audited_rows``. + primary_seen: dict[str, bool] = {} with path.open() as f: for line in f: line = line.strip() @@ -85,13 +90,25 @@ def _load_single_file_audit_index( except json.JSONDecodeError: continue iid = row.get("instance_id") + if not iid: + continue + is_primary = row.get("primary") is True + existing_is_primary = primary_seen.get(iid, False) + if iid in out and existing_is_primary and not is_primary: + # Already have the primary recorded — never let an + # alternate stomp on it. + continue + if iid in out and not existing_is_primary and not is_primary: + # Two non-primary alternates — keep the first (no + # ordering preference). + continue status = row.get("audit_status") or "missing-row" audited = row.get("audited_sol_sql") has_audited_sql = isinstance(audited, list) and bool(audited) row_db = row.get("selected_database") or "" row_benchmark = row.get("benchmark") or "" - if iid: - out[iid] = (status, has_audited_sql, row_db, row_benchmark) + out[iid] = (status, has_audited_sql, row_db, row_benchmark) + primary_seen[iid] = is_primary return out diff --git a/src/bird_interact_agents/cloud/collation.py b/src/bird_interact_agents/cloud/collation.py index 377a225e..ed4571ec 100644 --- a/src/bird_interact_agents/cloud/collation.py +++ b/src/bird_interact_agents/cloud/collation.py @@ -91,6 +91,16 @@ def _row_to_task_result_row(manifest: dict, r: dict) -> TaskResultRow: gold_result_json=r.get("gold_result_json"), n_agent_turns=int(n_turns) if isinstance(n_turns, int) else None, tool_call_stats_json=tool_call_stats_json, + # Codex r9: parity with the local ``run.py::_persist`` shape. + # Cloud workers emit these observation strings on the per-row + # JSON blob (every agent flavor's submit helper + + # ``agents/_submit.py`` populate them); without the explicit + # plumb-through, cloud ``fetch``+collation drops them at the + # results.db boundary while local runs retain them — an + # asymmetry that hid the audited/original observation diag + # for everything that ran in the cloud. + phase1_observation_audited=r.get("phase1_observation_audited"), + phase1_observation_original=r.get("phase1_observation_original"), ) diff --git a/src/bird_interact_agents/eval/tolerant_grader.py b/src/bird_interact_agents/eval/tolerant_grader.py index 528f4e39..ec62cf9d 100644 --- a/src/bird_interact_agents/eval/tolerant_grader.py +++ b/src/bird_interact_agents/eval/tolerant_grader.py @@ -621,9 +621,30 @@ def grade_submission( agent_sql_error_excerpt = f"{type(exc).__name__}: {exc}"[:200] pred_rows, pred_cols = [], [] - orig_rows, orig_cols = _multi_sql_execute( - list(original_sol_sql), db_path=db_path, conn=conn, executor=executor, - ) + # Codex r9: original gold execution was unguarded. If the source row's + # ``sol_sql`` is invalid or no longer executes against the current DB + # schema, the unguarded ``_multi_sql_execute`` call bubbled out of the + # grader and the cloud/local fallbacks wrote a generic fail-everything + # annotation — losing any valid audited-variant passes (N2/N3) the + # row would have earned. Now we degrade gracefully: ``orig_rows = []`` + # combined with the ``original_sql_executed_ok = False`` signal makes + # N1 effectively False (and the cell-level tier comparators skip the + # ``__original__`` fallback), so audited variants continue to be + # evaluated normally. + original_sql_executed_ok = True + try: + orig_rows, orig_cols = _multi_sql_execute( + list(original_sol_sql), + db_path=db_path, conn=conn, executor=executor, + ) + except Exception: # noqa: BLE001 + logger.exception( + "original gold execution failed; degrading N1 to False " + "so audited variants can still grade. instance=%s", + task_annotation.instance_id, + ) + orig_rows, orig_cols = [], [] + original_sql_executed_ok = False variant_results: list[tuple[dict, Sequence[Sequence], Sequence[str]]] = [] for v in audited_gold_rows: @@ -658,7 +679,9 @@ def grade_submission( # mark N1 as a strict pass whenever the agent's SQL also returns # empty (execution failure or genuinely empty result). Treat # missing-gold as ungradable for N1 instead of as an empty bag. - if not original_sol_sql: + # Also degrade N1 when the original gold FAILED TO EXECUTE + # (Codex r9) — same false-pass concern. + if not original_sol_sql or not original_sql_executed_ok: n1 = False else: n1 = _set_equal(pred_rows, orig_rows) @@ -698,13 +721,16 @@ def grade_submission( candidates = ( [(primary[0], primary[1])] if primary else [] ) + [(v[0], v[1]) for v in variant_results if not v[0].get("primary")] - if not candidates and original_sol_sql: + if not candidates and original_sol_sql and original_sql_executed_ok: # No variants → fall back to original gold itself. Gated on - # ``original_sol_sql`` because ``orig_rows`` is also ``[]`` - # when no source gold exists, and ``compare_tie_order([], [])`` - # returns True via set equality — without this guard a - # missing-gold + empty-agent row pair would falsely pass at - # N4 (and propagate as ``valid_interpretation`` even though + # ``original_sol_sql`` (round 3, missing-gold) AND on + # ``original_sql_executed_ok`` (Codex r9, original-exec + # failure) because ``orig_rows`` is ``[]`` in both cases; + # ``compare_tie_order([], [])`` returns True via set + # equality — without these guards a missing- or + # unexecutable-gold + empty-agent row pair would falsely + # pass at N4 (and propagate as ``valid_interpretation`` + # even though # nothing was actually compared). candidates = [({}, orig_rows)] for v_meta, v_rows in candidates: @@ -755,13 +781,15 @@ def grade_submission( novel_judgment = None # 6) N6/N7/N8/N9 — cell-level relaxations applied across all variants. - # When ``original_sol_sql`` is empty, ``orig_rows`` is also ``[]`` and - # every cell-level comparator returns True on the ``([], [])`` pair - # (bag equality holds vacuously). Drop the ``__original__`` fallback - # from the iteration list in that case — same guard as the N4 block, - # otherwise a missing-gold + empty-agent row would cascade-pass at N6. + # When ``original_sol_sql`` is empty OR its execution failed, + # ``orig_rows`` is ``[]`` and every cell-level comparator returns + # True on the ``([], [])`` pair (bag equality holds vacuously). + # Drop the ``__original__`` fallback from the iteration list in + # both cases — same guard as the N4 block (round 3 + Codex r9), + # otherwise a missing- or unexecutable-gold + empty-agent row + # would cascade-pass at N6. _comparator_targets: list = list(variant_results) - if original_sol_sql: + if original_sol_sql and original_sql_executed_ok: _comparator_targets.append( ({"variant_id": "__original__"}, orig_rows, orig_cols), ) diff --git a/src/bird_interact_agents/harness.py b/src/bird_interact_agents/harness.py index 5330ff09..251aabb5 100644 --- a/src/bird_interact_agents/harness.py +++ b/src/bird_interact_agents/harness.py @@ -490,7 +490,26 @@ def apply_audited_gold_overlay( benchmark.name, single_file_path, ) continue - single_rows[inst_id] = d # latest-wins + # Codex r9: DEV-1515 multi-variant audits ship N + # rows per instance_id (one ``primary=True`` plus + # non-primary alternates). Latest-wins would let a + # later-listed alternate overwrite the primary's + # audited_sol_sql, applying the wrong reading at + # overlay time. Prefer primary; once recorded, + # never overwrite. (A non-primary recorded first + # gets overwritten by the primary later in the + # file.) + existing = single_rows.get(inst_id) + if existing is None: + single_rows[inst_id] = d + elif existing.get("primary") is True: + # Already have the primary — keep it. + continue + elif d.get("primary") is True: + # Upgrade non-primary → primary. + single_rows[inst_id] = d + # else: both non-primary, keep the first one (no + # ordering preference between alternates). for task in tasks: inst = task.get("instance_id") db = task.get("selected_database") diff --git a/src/bird_interact_agents/run.py b/src/bird_interact_agents/run.py index 4fad7c5b..2188d229 100644 --- a/src/bird_interact_agents/run.py +++ b/src/bird_interact_agents/run.py @@ -4,6 +4,7 @@ import asyncio import json import logging +import shutil import statistics import time from pathlib import Path @@ -998,7 +999,30 @@ def _persist(td: dict, r: dict, started_at: float) -> None: # cloud worker (``cloud.ray_app._grade_one_submission``) — without # this, local runs would silently lose the N1-N9 cascade metrics # whenever audited gold / per-task annotations are present. + # + # Codex r9: ``aggregate_cascading_phase1`` walks EVERY subdir under + # ``rows_dir`` to compute ``n_dual_eval_tasks``. Reusing the same + # ``output_dir`` for a fresh run (different ``--limit`` / + # ``--instance-id`` subset) would otherwise carry forward stale + # annotations from the prior pass, inflating the denominator and + # rewriting ``phase1_count`` / ``phase1_rate`` from the union of + # old + new. Wipe per-instance subdirs that THIS run is about to + # touch (or the whole rows dir when no filter is set) so the + # aggregator only sees fresh annotations. Mirrors the round-2 + # regrade.py reset pattern. rows_dir = output_dir / "rows" + if rows_dir.exists(): + if filter_ids is None: + # Full run — wipe everything. + shutil.rmtree(rows_dir, ignore_errors=True) + else: + # Filtered run — reset ONLY the subdirs this run will + # overwrite, so unrelated instances from a prior pass + # survive (and still contribute to the cascade block). + _wanted = {str(t.get("instance_id") or "") for t in tasks} + for sub in list(rows_dir.iterdir()): + if sub.is_dir() and sub.name in _wanted: + shutil.rmtree(sub, ignore_errors=True) rows_dir.mkdir(parents=True, exist_ok=True) _benchmark_canonical = b.name diff --git a/tests/cloud/test_collation.py b/tests/cloud/test_collation.py index ee7bde80..6ccb665a 100644 --- a/tests/cloud/test_collation.py +++ b/tests/cloud/test_collation.py @@ -230,3 +230,47 @@ def test_driver_fetch_twice_idempotent( for k in ("total_tasks", "phase1_count", "phase1_rate", "phase2_count", "phase2_rate", "total_reward", "average_reward"): assert eval1[k] == eval2[k] + + +# --------------------------------------------------------------------------- +# Codex r9: cloud collation MUST plumb the dual-eval observation +# strings (``phase1_observation_audited`` / ``phase1_observation_original``) +# through to ``results.db``. Pre-fix the local ``run.py::_persist`` +# path carried them but cloud collation dropped them at the +# ``TaskResultRow`` build site, hiding the diagnostic on every +# cloud-fetched results.db. Local + cloud must agree. +# --------------------------------------------------------------------------- + + +def test_collation_writes_observation_columns_through_to_results_db( + tmp_path: Path, sample_task_result_row, +): + run_dir = tmp_path / RUN_ID + run_dir.mkdir() + row = { + **sample_task_result_row, + "phase1_observation_audited": "audited observation captured", + "phase1_observation_original": "original observation captured", + } + _write_attempt(run_dir, "db_a_1", 1, row) + manifest = { + "run_id": RUN_ID, + "framework": "pydantic_ai", + "mode": "c-interact", + "query_mode": "raw", + "agent_model": "anthropic/claude-sonnet-4-5", + "user_sim_model": "anthropic/claude-haiku-4-5-20251001", + "instance_ids": ["db_a_1"], + } + collation.collate(run_dir, manifest) + + import sqlite3 + conn = sqlite3.connect(str(run_dir / "results.db")) + conn.row_factory = sqlite3.Row + db_row = dict(next(iter(conn.execute( + "SELECT phase1_observation_audited, phase1_observation_original " + "FROM task_results" + )))) + conn.close() + assert db_row["phase1_observation_audited"] == "audited observation captured" + assert db_row["phase1_observation_original"] == "original observation captured" diff --git a/tests/test_dual_eval.py b/tests/test_dual_eval.py index e8abb227..250d3baa 100644 --- a/tests/test_dual_eval.py +++ b/tests/test_dual_eval.py @@ -805,3 +805,145 @@ def test_overlay_benchmark_kwarg_mini_interact_uses_single_file(tmp_path): # `tests/test_local_run_cascading.py`, # `tests/test_cascading_report.py`, and the legacy-removal grep-sweep in # `tests/test_legacy_field_removal.py`. + + +# --------------------------------------------------------------------------- +# Codex r9: multi-variant audited gold rows for the same instance_id +# (one ``primary=True`` + N alternates) must NOT let an alternate +# overwrite the primary row at index-build time. Pre-fix both index +# helpers (``harness._load_single_file_audited_rows`` via the overlay +# AND ``cloud._audited_gold_check._load_single_file_audit_index``) +# wrote with latest-wins semantics. These two tests pin the new +# primary-first contract — alternates listed AFTER the primary in the +# file must lose the contest. +# --------------------------------------------------------------------------- + + +def test_overlay_single_file_prefers_primary_when_alternate_listed_after( + tmp_path, +): + """Multi-variant file: primary row first, alternate row second. + The overlay MUST keep the primary's ``audited_sol_sql``.""" + from bird_interact_agents.benchmark import get_benchmark + from bird_interact_agents.harness import apply_audited_gold_overlay + + _write_single_file_audit(tmp_path, [ + { + "instance_id": "museum_9", + "selected_database": "museum", + "benchmark": "livesqlbench", + "variant_id": "primary", + "primary": True, + "audit_status": "edited", + "audited_sol_sql": ["SELECT primary_reading FROM t"], + }, + { + "instance_id": "museum_9", + "selected_database": "museum", + "benchmark": "livesqlbench", + "variant_id": "alt_a", + "primary": False, + "audit_status": "edited", + "audited_sol_sql": ["SELECT alt_reading FROM t"], + }, + ]) + task = { + "instance_id": "museum_9", + "selected_database": "museum", + "sol_sql": ["SELECT original FROM t"], + } + apply_audited_gold_overlay( + [task], tmp_path, benchmark=get_benchmark("livesqlbench"), + ) + assert task["sol_sql"] == ["SELECT primary_reading FROM t"], ( + "primary row MUST win over alternates regardless of file order" + ) + + +def test_overlay_single_file_prefers_primary_when_alternate_listed_first( + tmp_path, +): + """Symmetric case: alternate row FIRST, primary row second. The + primary must still take precedence at the end.""" + from bird_interact_agents.benchmark import get_benchmark + from bird_interact_agents.harness import apply_audited_gold_overlay + + _write_single_file_audit(tmp_path, [ + { + "instance_id": "museum_9", + "selected_database": "museum", + "benchmark": "livesqlbench", + "variant_id": "alt_a", + "primary": False, + "audit_status": "edited", + "audited_sol_sql": ["SELECT alt_reading FROM t"], + }, + { + "instance_id": "museum_9", + "selected_database": "museum", + "benchmark": "livesqlbench", + "variant_id": "primary", + "primary": True, + "audit_status": "edited", + "audited_sol_sql": ["SELECT primary_reading FROM t"], + }, + ]) + task = { + "instance_id": "museum_9", + "selected_database": "museum", + "sol_sql": ["SELECT original FROM t"], + } + apply_audited_gold_overlay( + [task], tmp_path, benchmark=get_benchmark("livesqlbench"), + ) + assert task["sol_sql"] == ["SELECT primary_reading FROM t"], ( + "primary row MUST win regardless of where it lands in the file" + ) + + +def test_cloud_audit_index_prefers_primary_over_alternate(tmp_path): + """``cloud._audited_gold_check._load_single_file_audit_index`` + is the cloud-side guard against an audited gold layout drift — + same primary-first rule must hold there.""" + from bird_interact_agents.benchmark import get_benchmark + from bird_interact_agents.cloud._audited_gold_check import ( + _load_single_file_audit_index, + ) + + benchmark = get_benchmark("livesqlbench") + audit_path = tmp_path / f"{benchmark.name}_audited.jsonl" + audit_path.write_text( + json.dumps({ + "instance_id": "museum_9", + "selected_database": "museum", + "benchmark": "livesqlbench", + "variant_id": "alt_a", + "primary": False, + "audit_status": "edited", + "audited_sol_sql": ["SELECT alt_reading FROM t"], + }) + "\n" + + json.dumps({ + "instance_id": "museum_9", + "selected_database": "museum", + "benchmark": "livesqlbench", + "variant_id": "primary", + "primary": True, + # Deliberately different from the alt so we can tell which + # row landed in the index. + "audit_status": "clean", + "audited_sol_sql": [], + }) + "\n" + ) + + index = _load_single_file_audit_index(tmp_path, benchmark) + assert index is not None + status, has_audited_sql, _row_db, _row_bench = index["museum_9"] + # ``clean`` is the primary's status; ``edited`` is the alt's. + assert status == "clean", ( + f"primary row's audit_status must survive against the alt's; " + f"got status={status!r} (alt's status was 'edited')" + ) + assert has_audited_sql is False, ( + "primary's empty audited_sol_sql must be the one indexed, " + "not the alt's non-empty list" + ) diff --git a/tests/test_run_local_inline_grader.py b/tests/test_run_local_inline_grader.py index ff8ce925..2e73216c 100644 --- a/tests/test_run_local_inline_grader.py +++ b/tests/test_run_local_inline_grader.py @@ -525,3 +525,223 @@ def _stub_grader_for_alien_2(*, task_data, **kw): ) assert alien1["failure_classification"]["primary"] == "other" assert "no submitted_sql" in alien1["failure_classification"]["details"] + + +@pytest.mark.asyncio +async def test_local_run_wipes_stale_rows_when_no_filter(monkeypatch, tmp_path): + """Codex r9: reusing the same ``output_dir`` without ``filter_ids`` + MUST wipe ``rows/`` first, so the aggregator only sees the current + run's annotations. Pre-fix a stale ``rows/old_iid/...`` from a prior + pass would inflate ``cascading_phase1.n_dual_eval_tasks``.""" + import bird_interact_agents.run as run_mod + + output_path = tmp_path / "eval.json" + rows_dir = output_path.parent / "rows" + # Plant a stale annotation from a prior run under a DIFFERENT iid + # than the current task set will produce. + stale_dir = rows_dir / "stale_old_iid" + stale_dir.mkdir(parents=True) + (stale_dir / "submission_annotation.json").write_text( + '{"this": "should be wiped"}', + ) + + rows = [ + {"instance_id": "alien_1", "selected_database": "alien", + "sol_sql": ["SELECT 1"], "amb_user_query": "q1"}, + ] + _patch_loader_returns(monkeypatch, rows) + monkeypatch.setattr(run_mod, "_maybe_force_wipe_otf", lambda **kw: None) + _stub_runner_factory(monkeypatch, { + "alien_1": { + "instance_id": "alien_1", "database": "alien", + "phase1_passed": True, "phase2_passed": False, + "total_reward": 1.0, "submitted_sql": "SELECT 1", + "trajectory": [], "usage": {}, + }, + }) + # No-op grader so cascading_phase1 is built off the (sole) row. + from bird_interact_agents.eval.annotation_schema import ( + FailureClassification, SubmissionAnnotation, SubmissionEvaluation, + SubmissionMetadata, UserSimInteraction, + ) + + def _stub(*, task_data, **kw): + out_dir = Path(kw["rows_dir"]) / task_data["instance_id"] + out_dir.mkdir(parents=True, exist_ok=True) + ann = SubmissionAnnotation( + instance_id=task_data["instance_id"], + selected_database=task_data["selected_database"], + task_annotation_ref=( + f"annotations/mini_interact/alien/" + f"{task_data['instance_id']}.task.json" + ), + annotated_by="auto-inline-grader", + annotated_at="2026-06-02T00:00:00+00:00", + submission=SubmissionMetadata( + cloud_run_id=kw["run_id"], + trajectory_path=f"rows/{task_data['instance_id']}/attempt-1.json", + ), + evaluation=SubmissionEvaluation( + phase1_against_original_gold="fail", + phase1_against_audited_primary="fail", + phase1_against_any_audited_variant="fail", + phase1_against_variants=[], + correct_up_to_tie_order=False, + novel_reading_judgment=None, + correct_under_numeric_epsilon=False, + correct_under_trailing_whitespace=False, + correct_under_column_order=False, + correct_under_case_fold=False, + numeric_epsilon=1e-6, + verdict="invalid", + matched_variant_id=None, + rationale="", + miss_diagnostics=None, + ), + failure_classification=FailureClassification( + primary="agent_miss", + agent_at_fault=True, + remediation_target="agent", + details="stub", + ), + decision_point=None, + user_sim_interaction=UserSimInteraction(), + ) + path = out_dir / "submission_annotation.json" + path.write_text(ann.model_dump_json(indent=2, exclude_none=False) + "\n") + return path + monkeypatch.setattr(run_mod, "grade_one_submission", _stub) + + metrics = await run_mod.run_evaluation( + framework="claude_sdk_otf_ainteract", query_mode="slayer", + mode="a-interact", data_path="ignored", + data_dir=str(tmp_path / "ignored_data_dir"), + output_path=str(output_path), + concurrency=1, limit=None, + agent_model="anthropic/claude-haiku-4-5-20251001", + strict=False, prompt_cache=False, max_depth=1, + slayer_storage_root=str(tmp_path / "slayer_models"), + slayer_setup="on-the-fly", reasoning_effort=None, + use_audited_gold_sql=False, dataset="mini-interact", + gold_file=None, filter_ids=None, + ) + assert not stale_dir.exists(), ( + "rows/stale_old_iid should have been wiped before the run" + ) + cp = metrics["cascading_phase1"] + assert cp["n_dual_eval_tasks"] == 1, ( + f"denominator must reflect ONLY current tasks (1), not the " + f"union with stale entries; got cp={cp}" + ) + + +@pytest.mark.asyncio +async def test_local_run_filter_ids_preserves_unrelated_rows( + monkeypatch, tmp_path, +): + """Symmetric case: filtered reruns must NOT wipe rows from a prior + full run if those instances aren't in the current task set. Only + the subdirs we're about to overwrite get reset.""" + import bird_interact_agents.run as run_mod + + output_path = tmp_path / "eval.json" + rows_dir = output_path.parent / "rows" + # Unrelated prior-run annotation that the filtered rerun should + # preserve verbatim. + from bird_interact_agents.eval.annotation_schema import ( + FailureClassification, SubmissionAnnotation, SubmissionEvaluation, + SubmissionMetadata, UserSimInteraction, + ) + unrelated = rows_dir / "alien_99" + unrelated.mkdir(parents=True) + + def _make_ann(iid: str, marker: str) -> SubmissionAnnotation: + return SubmissionAnnotation( + instance_id=iid, + selected_database="alien", + task_annotation_ref=f"annotations/mini_interact/alien/{iid}.task.json", + annotated_by=marker, + annotated_at="2026-06-02T00:00:00+00:00", + submission=SubmissionMetadata( + cloud_run_id="r1", + trajectory_path=f"rows/{iid}/attempt-1.json", + ), + evaluation=SubmissionEvaluation( + phase1_against_original_gold="fail", + phase1_against_audited_primary="fail", + phase1_against_any_audited_variant="fail", + phase1_against_variants=[], + correct_up_to_tie_order=False, + novel_reading_judgment=None, + correct_under_numeric_epsilon=False, + correct_under_trailing_whitespace=False, + correct_under_column_order=False, + correct_under_case_fold=False, + numeric_epsilon=1e-6, + verdict="invalid", + matched_variant_id=None, + rationale="", + miss_diagnostics=None, + ), + failure_classification=FailureClassification( + primary="agent_miss", + agent_at_fault=True, + remediation_target="agent", + details="stub", + ), + decision_point=None, + user_sim_interaction=UserSimInteraction(), + ) + (unrelated / "submission_annotation.json").write_text( + _make_ann("alien_99", "unrelated-survivor") + .model_dump_json(indent=2, exclude_none=False) + "\n", + ) + + rows = [ + {"instance_id": "alien_1", "selected_database": "alien", + "sol_sql": ["SELECT 1"], "amb_user_query": "q1"}, + ] + _patch_loader_returns(monkeypatch, rows) + monkeypatch.setattr(run_mod, "_maybe_force_wipe_otf", lambda **kw: None) + _stub_runner_factory(monkeypatch, { + "alien_1": { + "instance_id": "alien_1", "database": "alien", + "phase1_passed": True, "phase2_passed": False, + "total_reward": 1.0, "submitted_sql": "SELECT 1", + "trajectory": [], "usage": {}, + }, + }) + + def _stub(*, task_data, **kw): + out_dir = Path(kw["rows_dir"]) / task_data["instance_id"] + out_dir.mkdir(parents=True, exist_ok=True) + (out_dir / "submission_annotation.json").write_text( + _make_ann(task_data["instance_id"], "fresh") + .model_dump_json(indent=2, exclude_none=False) + "\n", + ) + return out_dir / "submission_annotation.json" + monkeypatch.setattr(run_mod, "grade_one_submission", _stub) + + await run_mod.run_evaluation( + framework="claude_sdk_otf_ainteract", query_mode="slayer", + mode="a-interact", data_path="ignored", + data_dir=str(tmp_path / "ignored_data_dir"), + output_path=str(output_path), + concurrency=1, limit=None, + agent_model="anthropic/claude-haiku-4-5-20251001", + strict=False, prompt_cache=False, max_depth=1, + slayer_storage_root=str(tmp_path / "slayer_models"), + slayer_setup="on-the-fly", reasoning_effort=None, + use_audited_gold_sql=False, dataset="mini-interact", + gold_file=None, + # Filter: only alien_1 — alien_99's prior annotation should survive. + filter_ids=["alien_1"], + ) + assert (unrelated / "submission_annotation.json").exists(), ( + "filtered rerun must NOT wipe rows for instances outside the " + "filter set; alien_99's prior annotation was deleted" + ) + surviving = json.loads( + (unrelated / "submission_annotation.json").read_text(), + ) + assert surviving["annotated_by"] == "unrelated-survivor" diff --git a/tests/test_tolerant_grader_orchestration.py b/tests/test_tolerant_grader_orchestration.py index 38242dea..2fc7b16c 100644 --- a/tests/test_tolerant_grader_orchestration.py +++ b/tests/test_tolerant_grader_orchestration.py @@ -981,6 +981,87 @@ def test_tier2_populated_on_grader_output(): # --------------------------------------------------------------------------- +def test_unexecutable_original_gold_does_not_kill_audited_variant_pass(): + """Codex r9: when ``original_sol_sql`` raises at executor time + (invalid SQL, missing table after schema drift, …) the grader MUST + NOT bubble the exception out. It should degrade N1 to False but + let audited variants grade normally, so a valid N2/N3 pass survives + the broken original gold.""" + from bird_interact_agents.eval.tolerant_grader import grade_submission + + submitted = "SELECT predicted" + bad_original = "SELECT * FROM nonexistent_table" + audited_sql = "SELECT predicted" + executor = FakeExecutor({ + submitted: ([(42,)], ["x"]), + audited_sql: ([(42,)], ["x"]), + # NB: ``bad_original`` is NOT in the executor's responses dict; + # FakeExecutor raises AssertionError on unknown SQL, which the + # grader's new try/except catches. + }) + ann = _make_task_annotation() + gold_rows = [_audited_row( + instance_id="alien_1", variant_id="primary", primary=True, + audited_sol_sql=[audited_sql], + )] + verdict = grade_submission( + task_annotation=ann, + audited_gold_rows=gold_rows, + original_sol_sql=[bad_original], + submitted_sql=submitted, + db_path=Path("/dev/null"), + conn=None, + executor=executor, + ) + # Original failed → N1 is False (no bogus pass against empty + # orig_rows). But N2/N3 are evaluated against the audited variant + # which DOES execute and matches the agent's rowset. + assert verdict.n1_original_gold is False, ( + "N1 must NOT pass via _set_equal([], []) when original gold " + "failed to execute" + ) + assert verdict.n2_audited_primary is True + assert verdict.n3_any_audited_variant is True + + +def test_unexecutable_original_with_empty_agent_does_not_pass_n4( +): + """Companion guard: original SQL fails AND agent rowset is empty. + Pre-fix this would cascade-pass at N4-N9 via the __original__ + fallback's vacuous bag equality. Now both fallback gates are + closed.""" + from bird_interact_agents.eval.tolerant_grader import grade_submission + + submitted = "SELECT empty" + bad_original = "SELECT * FROM nonexistent_table" + executor = FakeExecutor({ + submitted: ([], ["x"]), + # bad_original raises AssertionError, caught by grader. + }) + ann = _make_task_annotation() + verdict = grade_submission( + task_annotation=ann, + # No audited variants either — the test isolates the + # __original__ fallback path. + audited_gold_rows=[], + original_sol_sql=[bad_original], + submitted_sql=submitted, + db_path=Path("/dev/null"), + conn=None, + executor=executor, + ) + for tier in ( + "n1_original_gold", "n2_audited_primary", + "n3_any_audited_variant", "n4_tie_order", + "n5_llm_judge", "n6_numeric_epsilon", + "n7_trailing_whitespace", "n8_column_order", "n9_case_fold", + ): + assert getattr(verdict, tier) is False, ( + f"{tier} must be False; got " + f"{getattr(verdict, tier)} (cascade={verdict})" + ) + + def test_missing_gold_does_not_collapse_to_n4_pass(): """No original gold, no audited variants — every cascade tier MUST stay False, no matter what the agent's rowset looks like (including From 86d8bf78c01ad132acb5bb74e215823164c4a409 Mon Sep 17 00:00:00 2001 From: Egor Kraev Date: Tue, 2 Jun 2026 21:13:11 +0200 Subject: [PATCH 23/29] DEV-1515 round 11: regrade picks latest attempt + defensive trajectory shape MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two Codex findings on the round-8 / round-10 plumbing: Item 1 (Codex major — regrade.py:193): The CLI hardcoded ``attempt = sub / "attempt-1.json"``. But cloud collation has long treated the highest ``attempt-N.json`` as canonical and round-8's post-fetch merge now compares attempt numbers — so the regrade CLI was the odd one out: after a resubmit it would either silently SKIP instances that had only attempt-2 (because attempt-1 didn't exist) or, when both attempts coexisted, OVERWRITE the submission annotation + ``eval_regraded.json`` with results computed against STALE attempt-1 SQL. Fix: ``_latest_attempt_file(sub)`` scans for ``attempt-N.json``, parses the digit, and returns the max. Mirrors the regex used in ``cloud/post_run_merge._attempt_from_trajectory_path``. Two new tests: prefer attempt-3 over attempt-1 when both exist; pick up attempt-2 as the sole file. Item 2 (Codex major — annotate.py:120 + regrade.py:281 + annotate.py:241): ``_user_sim_interaction_from_trajectory`` iterated ``traj`` and called ``item.get("role")``. Three call sites wrapped the source trajectory with ``list(attempt_data.get("trajectory") or [])`` — fine when ``trajectory`` is a list of turn-step dicts, but ``pydantic_ai_otf_encode/agent.py:1537`` (and the recursive flavor's ``agent.py:892``) emit ``trajectory = {"final_output_excerpt": ..., "agents": [...]}`` (a DICT). ``list(dict)`` returns the dict's KEYS as strings, then ``str.get("role")`` raises AttributeError — the grader-fallback / skeleton-build paths crashed after the cascade computed cleanly. Fix: * Drop the function's strict ``traj: list[dict]`` annotation and short-circuit to ``UserSimInteraction()`` when the value isn't a list. Skip individual non-dict items in the iteration too (handles ``list(dict)`` -> list of strings as the safety floor). Guard the ``traj[i-1]`` previous-step lookup against non-dicts on the same principle. * At the three call sites, drop the ``list(...)`` wrap and pass the raw value (or ``[]`` when None); the helper now handles every shape defensively. New tests file ``tests/test_user_sim_interaction_trajectory.py`` — 7 tests pinning: canonical list-of-dicts (load-bearing happy path), dict-shaped traj (Codex's bug fixture), list-of-strings (the ``list(dict)`` coercion case), None, empty list, mixed list with stray non-dict entries, and a non-dict ``traj[i-1]`` prev-step. Co-Authored-By: Claude Opus 4.7 --- src/bird_interact_agents/eval/annotate.py | 38 +++++- src/bird_interact_agents/eval/regrade.py | 39 +++++- tests/test_regrade_cli.py | 118 ++++++++++++++++++ tests/test_user_sim_interaction_trajectory.py | 107 ++++++++++++++++ 4 files changed, 294 insertions(+), 8 deletions(-) create mode 100644 tests/test_user_sim_interaction_trajectory.py diff --git a/src/bird_interact_agents/eval/annotate.py b/src/bird_interact_agents/eval/annotate.py index a460237a..8d7038f9 100644 --- a/src/bird_interact_agents/eval/annotate.py +++ b/src/bird_interact_agents/eval/annotate.py @@ -117,15 +117,41 @@ def generate_task_annotation( ) -def _user_sim_interaction_from_trajectory(traj: list[dict]) -> UserSimInteraction: +def _user_sim_interaction_from_trajectory(traj) -> UserSimInteraction: + """Build the ``UserSimInteraction`` summary from a trajectory. + + Several agent flavors (``pydantic_ai_otf_encode``, + ``pydantic_ai_recursive``) emit ``trajectory`` as a DICT shape + (``{"final_output_excerpt": ..., "agents": [...]}``) instead of + the per-turn-step list this helper was originally designed for. + Without the type check below, ``list(dict_traj)`` at the call + site would yield the dict's KEYS (strings); iterating and calling + ``item.get("role")`` then raises ``AttributeError`` on a ``str`` + and the whole grader-fallback / skeleton-build path crashes after + the cascade has already computed cleanly (Codex r10). + + Contract: any non-list-of-dicts shape degrades to a zero-asks + ``UserSimInteraction()`` default. The per-step path stays the + same for the genuine list-of-dicts case. + """ + if not isinstance(traj, list): + return UserSimInteraction() n_asks = 0 responses: list[UserSimResponseSummary] = [] - for i, item in enumerate(traj or []): + for i, item in enumerate(traj): + if not isinstance(item, dict): + # Mixed shape (or list of strings from ``list(dict)``) — + # skip non-dict entries instead of raising on ``.get()``. + continue if item.get("role") == "tool_call" and item.get("name") == "ask_user": n_asks += 1 elif item.get("role") in ("user_sim", "user") and i > 0: prev = traj[i - 1] - if prev.get("role") == "tool_call" and prev.get("name") == "ask_user": + if ( + isinstance(prev, dict) + and prev.get("role") == "tool_call" + and prev.get("name") == "ask_user" + ): # Recently followed an ask — record short summary. txt = str(item.get("content") or "") responses.append(UserSimResponseSummary( @@ -212,7 +238,11 @@ def generate_submission_annotation( attempt_path = Path(rows_dir) / instance_id / "attempt-1.json" attempt = json.loads(attempt_path.read_text()) submitted_sql = attempt.get("submitted_sql", "") - traj = list(attempt.get("trajectory", []) or []) + # Don't wrap with ``list(...)`` — dict trajectories from + # pydantic_ai_otf_encode etc. would coerce to a list of keys. + # ``_user_sim_interaction_from_trajectory`` handles non-list + # shapes defensively (Codex r10). + traj = attempt.get("trajectory") or [] usage = attempt.get("usage", {}) or {} cascade = grader( diff --git a/src/bird_interact_agents/eval/regrade.py b/src/bird_interact_agents/eval/regrade.py index 492fad4d..111a4e12 100644 --- a/src/bird_interact_agents/eval/regrade.py +++ b/src/bird_interact_agents/eval/regrade.py @@ -17,6 +17,7 @@ import argparse import json +import re import shutil from pathlib import Path from typing import Any, Callable, Iterable, List, Optional @@ -47,6 +48,34 @@ def _attempt_rows_dir(run_dir: Path) -> Path: return run_dir / "rows" +_ATTEMPT_FILE_RE = re.compile(r"attempt-(\d+)\.json") + + +def _latest_attempt_file(sub: Path) -> Path | None: + """Return the highest-numbered ``attempt-N.json`` in ``sub`` or + None when no attempt file exists. + + Pre-fix the regrade CLI hardcoded ``attempt-1.json`` (Codex r10), so + a resubmit's attempt-2 was either silently skipped (instance had ONLY + attempt-2) or, worse, overwritten by stale attempt-1 data — even + though cloud collation already treats the max attempt as canonical + and the round-8 fetch merge compares attempt numbers.""" + best: tuple[int, Path] | None = None + for p in sub.iterdir(): + if not p.is_file(): + continue + m = _ATTEMPT_FILE_RE.match(p.name) + if m is None: + continue + try: + n = int(m.group(1)) + except ValueError: + continue + if best is None or n > best[0]: + best = (n, p) + return best[1] if best else None + + def clear_llm_judge_cache( *, cache_path: Path, @@ -190,8 +219,8 @@ def regrade_run( if filter_set is not None and instance_id not in filter_set: report.skipped += 1 continue - attempt = sub / "attempt-1.json" - if not attempt.exists(): + attempt = _latest_attempt_file(sub) + if attempt is None: report.skipped += 1 continue attempt_data = json.loads(attempt.read_text()) @@ -249,8 +278,10 @@ def regrade_run( ), evaluation=_eval_from_cascade(cascade), failure_classification=_skeleton_failure_classification(cascade), + # Pass the raw trajectory; ``_user_sim_interaction_from_trajectory`` + # defends against dict-shaped trajectories (Codex r10). user_sim_interaction=_user_sim_interaction_from_trajectory( - list(attempt_data.get("trajectory", []) or []), + attempt_data.get("trajectory") or [], ), ) import datetime as _dt @@ -364,7 +395,7 @@ def _grader(*, instance_id: str, submitted_sql: str, task_row: dict, **_kw): # interactive runs where the agent never asked. One-shot # benchmarks pass None so the flag stays out of miss_patterns. if _bench_is_interactive: - _traj = list(task_row.get("trajectory") or []) + _traj = task_row.get("trajectory") or [] _user_sim_n_asks: Optional[int] = ( _user_sim_interaction_from_trajectory(_traj).n_asks ) diff --git a/tests/test_regrade_cli.py b/tests/test_regrade_cli.py index a9d055cd..9f2a7813 100644 --- a/tests/test_regrade_cli.py +++ b/tests/test_regrade_cli.py @@ -313,6 +313,124 @@ def test_build_original_sql_index_accepts_string_sol_sql(tmp_path, monkeypatch): assert "alien_no_sol" not in out +def test_regrade_picks_latest_attempt_after_resubmit(tmp_path, monkeypatch): + """Codex r10: regrade MUST read the highest ``attempt-N.json``, not + the hardcoded attempt-1. Pre-fix a resubmit's attempt-2 was either + skipped (when attempt-1 was absent) or its SQL got silently + overwritten by stale attempt-1 data.""" + from bird_interact_agents import paths as paths_mod + monkeypatch.setattr(paths_mod, "main_checkout_root", lambda: tmp_path) + + run_dir = tmp_path / "results" / "cloud" / "r1" + d = run_dir / "rows" / "alien_1" + d.mkdir(parents=True) + # Stale attempt-1: this SQL must NOT be the one graded. + (d / "attempt-1.json").write_text(json.dumps({ + "instance_id": "alien_1", + "selected_database": "alien", + "submitted_sql": "STALE_ATTEMPT_1_SQL", + "trajectory": [], + "usage": {}, + "duration_s": 0.0, + "sol_sql": ["SELECT gold"], + "original_sol_sql": ["SELECT gold"], + })) + # Fresh attempt-3: this is the one regrade should pick. + (d / "attempt-3.json").write_text(json.dumps({ + "instance_id": "alien_1", + "selected_database": "alien", + "submitted_sql": "FRESH_ATTEMPT_3_SQL", + "trajectory": [], + "usage": {}, + "duration_s": 0.0, + "sol_sql": ["SELECT gold"], + "original_sol_sql": ["SELECT gold"], + })) + + captured: list[dict] = [] + + class StubGrader: + def __call__(self, *, instance_id, submitted_sql, task_row): + captured.append({ + "instance_id": instance_id, + "submitted_sql": submitted_sql, + }) + from bird_interact_agents.eval.tolerant_grader import CascadeVerdict + return CascadeVerdict( + n1_original_gold=False, n2_audited_primary=False, + n3_any_audited_variant=False, n4_tie_order=False, + n5_llm_judge=False, n6_numeric_epsilon=False, + n7_trailing_whitespace=False, n8_column_order=False, + n9_case_fold=False, + matched_variant_id=None, novel_reading_judgment=None, + variant_matches=[], rowset_relations=[], + ) + + from bird_interact_agents.eval.regrade import regrade_run + + regrade_run( + run_id="r1", benchmark="mini-interact", run_dir=run_dir, + instance_ids=None, force_llm_judge=False, + grader=StubGrader(), repo_root=tmp_path, + ) + assert len(captured) == 1 + assert captured[0]["submitted_sql"] == "FRESH_ATTEMPT_3_SQL", ( + f"regrade must pick the HIGHEST attempt-N.json, got " + f"{captured[0]['submitted_sql']!r} — pre-fix this would be " + f"'STALE_ATTEMPT_1_SQL'" + ) + + +def test_regrade_grades_when_only_later_attempt_exists(tmp_path, monkeypatch): + """Companion case: a resubmit may produce ONLY attempt-2 (the + earlier attempt-1.json was never written or was cleaned up). Pre-fix + that instance was silently skipped because the hardcoded path + didn't exist.""" + from bird_interact_agents import paths as paths_mod + monkeypatch.setattr(paths_mod, "main_checkout_root", lambda: tmp_path) + + run_dir = tmp_path / "results" / "cloud" / "r1" + d = run_dir / "rows" / "alien_1" + d.mkdir(parents=True) + (d / "attempt-2.json").write_text(json.dumps({ + "instance_id": "alien_1", + "selected_database": "alien", + "submitted_sql": "ONLY_ATTEMPT_2", + "trajectory": [], + "usage": {}, + "duration_s": 0.0, + })) + + captured: list[dict] = [] + + class StubGrader: + def __call__(self, *, instance_id, submitted_sql, task_row): + captured.append({ + "instance_id": instance_id, + "submitted_sql": submitted_sql, + }) + from bird_interact_agents.eval.tolerant_grader import CascadeVerdict + return CascadeVerdict( + n1_original_gold=False, n2_audited_primary=False, + n3_any_audited_variant=False, n4_tie_order=False, + n5_llm_judge=False, n6_numeric_epsilon=False, + n7_trailing_whitespace=False, n8_column_order=False, + n9_case_fold=False, + matched_variant_id=None, novel_reading_judgment=None, + variant_matches=[], rowset_relations=[], + ) + + from bird_interact_agents.eval.regrade import regrade_run + + report = regrade_run( + run_id="r1", benchmark="mini-interact", run_dir=run_dir, + instance_ids=None, force_llm_judge=False, + grader=StubGrader(), repo_root=tmp_path, + ) + assert report.regraded == 1 + assert captured[0]["submitted_sql"] == "ONLY_ATTEMPT_2" + + def test_build_original_sql_index_does_not_char_split_string( tmp_path, monkeypatch, ): diff --git a/tests/test_user_sim_interaction_trajectory.py b/tests/test_user_sim_interaction_trajectory.py new file mode 100644 index 00000000..6a370f64 --- /dev/null +++ b/tests/test_user_sim_interaction_trajectory.py @@ -0,0 +1,107 @@ +"""DEV-1515 round 11: ``_user_sim_interaction_from_trajectory`` must +handle every trajectory shape agents actually emit, not just the +canonical list-of-turn-steps shape. + +Codex r10: ``pydantic_ai_otf_encode/agent.py:1537`` (and the recursive +flavor's ``agent.py:892``) emit ``trajectory`` as a DICT +(``{"final_output_excerpt": ..., "agents": [...]}``). The pre-fix +helper's call sites did ``list(attempt_data.get("trajectory") or [])`` +which on a dict yields the dict's KEYS (strings). Then the helper +iterated and called ``item.get("role")``, hitting ``AttributeError`` +on ``str`` and crashing the grader-fallback / skeleton-build paths +AFTER the cascade had already computed cleanly. These tests pin the +defensive contract: any non-list-of-dicts shape degrades to a 0-asks +``UserSimInteraction()`` default. +""" + +from __future__ import annotations + +from bird_interact_agents.eval.annotate import ( + _user_sim_interaction_from_trajectory, +) + + +def test_handles_list_of_dicts_canonical_shape(): + """The original shape still works: list of turn-step dicts with + role markers. Both the ask count and the recorded response land.""" + traj = [ + {"role": "tool_call", "name": "ask_user", "args": {"q": "?"}}, + {"role": "user_sim", "content": "yes please"}, + {"role": "tool_call", "name": "submit_query"}, + ] + interaction = _user_sim_interaction_from_trajectory(traj) + assert interaction.n_asks == 1 + assert len(interaction.key_responses) == 1 + assert interaction.key_responses[0].summary == "yes please" + + +def test_handles_dict_trajectory_from_pydantic_ai_otf_encode(): + """Codex r10 — load-bearing case. Trajectory emitted as a dict + (``pydantic_ai_otf_encode.agent``) must NOT crash the helper.""" + dict_traj = { + "final_output_excerpt": "the agent said something", + "agents": [ + {"agent_id": "encoder", "messages": ["hi"]}, + ], + } + # No exception → contract satisfied. Result is a default 0-asks. + interaction = _user_sim_interaction_from_trajectory(dict_traj) + assert interaction.n_asks == 0 + assert interaction.key_responses == [] + + +def test_handles_list_of_strings_from_list_coercion_of_dict(): + """Mirrors the pre-fix call-site behaviour where + ``list(some_dict)`` returned the dict's keys (strings). Even if a + caller still wraps with ``list(...)`` and lands a list of strings, + the helper must NOT raise — non-dict items are skipped.""" + interaction = _user_sim_interaction_from_trajectory( + ["final_output_excerpt", "agents"] + ) + assert interaction.n_asks == 0 + + +def test_handles_none_trajectory(): + """Safety floor: ``None`` from ``r.get("trajectory")`` → default.""" + interaction = _user_sim_interaction_from_trajectory(None) + assert interaction.n_asks == 0 + + +def test_handles_empty_list_trajectory(): + interaction = _user_sim_interaction_from_trajectory([]) + assert interaction.n_asks == 0 + + +def test_handles_mixed_list_with_some_non_dict_items(): + """A list that contains BOTH dicts (legitimate turn steps) AND + stray non-dict entries (a stringification slip-up) must count the + dicts correctly and skip the rest.""" + traj = [ + {"role": "tool_call", "name": "ask_user"}, + {"role": "user_sim", "content": "first answer"}, + "stray_string_should_be_skipped", + {"role": "tool_call", "name": "ask_user"}, + 42, # not a dict + {"role": "tool_call", "name": "submit_query"}, + ] + interaction = _user_sim_interaction_from_trajectory(traj) + # 2 ask_user calls counted; the first user_sim follows an ask_user + # directly so its response is recorded. + assert interaction.n_asks == 2 + assert len(interaction.key_responses) == 1 + assert interaction.key_responses[0].summary == "first answer" + + +def test_does_not_crash_when_previous_step_is_non_dict(): + """The recently-followed-an-ask detection looks at ``traj[i-1]``. + A non-dict previous step must be guarded against the same + ``.get()`` -> AttributeError landmine.""" + traj = [ + "not_a_dict_first_step", + {"role": "user_sim", "content": "something"}, + ] + # Must NOT raise. The user_sim step at i=1 has a non-dict prev, + # so no response gets recorded — but no exception either. + interaction = _user_sim_interaction_from_trajectory(traj) + assert interaction.n_asks == 0 + assert interaction.key_responses == [] From 68539cfee0cf5c34b9b72dbe7a0ecc0b4f7079b6 Mon Sep 17 00:00:00 2001 From: Egor Kraev Date: Tue, 2 Jun 2026 21:25:57 +0200 Subject: [PATCH 24/29] DEV-1515 round 12: scope cascade aggregation to current instance set MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Codex r11 follow-up on round 10's Group 1 design. Round 10's filtered rerun preserved unrelated prior ``rows//submission_annotation.json`` directories on disk so previously-graded instances would still be visible for human inspection. The aggregator at the bottom of ``run_evaluation`` then walked EVERY subdirectory under ``output_dir/rows`` to build ``cascading_phase1``, while the rest of the metrics dict (``total_tasks``, ``results``, durations, usage, phase counts) was built from only the filtered ``tasks`` list. The final ``emit_cascading_eval_json`` rewrote ``phase1_count`` / ``phase1_rate`` from the cascade's N1 count — over the union — while ``total_tasks`` stayed at the filtered count. Observable on the published ``eval.json``: * ``total_tasks`` = filtered count (e.g. 1) * ``phase1_count`` = union count (e.g. 3 — 2 prior + 1 fresh) * ``cascading_phase1.n_dual_eval_tasks`` = union count ``phase1_count > total_tasks`` and rates uninterpretable. Fix: add ``instance_filter: set[str] | None`` to ``aggregate_cascading_phase1`` and pipe it through ``emit_cascading_eval_json``. When set, only subdirectories whose name is in the filter are counted (preserved prior annotations stay on disk but DON'T pollute the published metrics). At the ``run.py`` call site, build the set from ``{td["instance_id"] for td in tasks}`` so the cascade describes the same row set as ``total_tasks``. Cloud collation and full local runs pass ``None`` to keep back-compat. Tests: - tests/test_cascading_report.py: test_aggregator_instance_filter_scopes_to_current_run — pin the helper's filter contract directly: 3 dirs on disk, filter to 2, assert the third dir is preserved on disk but excluded from the count. - tests/test_run_local_inline_grader.py: test_local_run_filter_ids_preserves_unrelated_rows — extended with the metrics-consistency invariant ``total_tasks == cascading_phase1.n_dual_eval_tasks`` after a filtered rerun with a stale unrelated annotation on disk. Co-Authored-By: Claude Opus 4.7 --- .../eval/cascading_report.py | 29 ++++++++- src/bird_interact_agents/run.py | 11 ++++ tests/test_cascading_report.py | 60 +++++++++++++++++++ tests/test_run_local_inline_grader.py | 17 +++++- 4 files changed, 114 insertions(+), 3 deletions(-) diff --git a/src/bird_interact_agents/eval/cascading_report.py b/src/bird_interact_agents/eval/cascading_report.py index 25f93b25..e18f05f6 100644 --- a/src/bird_interact_agents/eval/cascading_report.py +++ b/src/bird_interact_agents/eval/cascading_report.py @@ -56,10 +56,24 @@ def _per_row_cascade_bools(annotation_dir: Path) -> dict[str, bool]: return enforce_monotone_cascade(raw) -def aggregate_cascading_phase1(rows_dir: Path) -> dict: +def aggregate_cascading_phase1( + rows_dir: Path, + *, + instance_filter: set[str] | None = None, +) -> dict: """Walk per-task ``submission_annotation.json`` files and return the cascading_phase1 block. + ``instance_filter`` (Codex r11): when set, count ONLY subdirectories + whose name is in the filter. Local filtered reruns preserve unrelated + prior annotations on disk for human inspection, but those rows MUST + NOT pollute the published ``eval.json`` — otherwise + ``cascading_phase1.n_dual_eval_tasks`` (union of new + stale) would + exceed ``eval.total_tasks`` (filtered count) and the rewritten + ``phase1_count`` / ``phase1_rate`` would become uninterpretable. + When unset (full local runs + cloud collation), every subdirectory + is counted — back-compat preserved. + Output shape:: { @@ -74,6 +88,8 @@ def aggregate_cascading_phase1(rows_dir: Path) -> dict: n = 0 if rows_dir.exists(): for sub in sorted(p for p in rows_dir.iterdir() if p.is_dir()): + if instance_filter is not None and sub.name not in instance_filter: + continue verdicts = _per_row_cascade_bools(sub) n += 1 for f, v in verdicts.items(): @@ -116,13 +132,22 @@ def emit_cascading_eval_json( rows_dir: Path, out_path: Path, base_metrics: dict | None = None, + *, + instance_filter: set[str] | None = None, ) -> dict: """Merge ``base_metrics`` with the freshly-computed cascading block and write to ``out_path``. The legacy dual-eval keys are explicitly dropped; ``phase1_count`` / ``phase1_rate`` are REWRITTEN from N1. + ``instance_filter`` is forwarded to ``aggregate_cascading_phase1`` + so a filtered local rerun's metrics describe ONLY the current run's + instances (Codex r11). Cloud collation / full local runs pass None + to keep back-compat. + Returns the resulting metrics dict (for inline use).""" - block = aggregate_cascading_phase1(Path(rows_dir)) + block = aggregate_cascading_phase1( + Path(rows_dir), instance_filter=instance_filter, + ) out = dict(base_metrics or {}) for k in _LEGACY_KEYS_TO_DROP: out.pop(k, None) diff --git a/src/bird_interact_agents/run.py b/src/bird_interact_agents/run.py index 2188d229..7c45b8d2 100644 --- a/src/bird_interact_agents/run.py +++ b/src/bird_interact_agents/run.py @@ -1204,8 +1204,19 @@ async def _run_with_sem(i: int, td: dict) -> None: (sub / "submission_annotation.json").exists() for sub in rows_dir.iterdir() if sub.is_dir() ): + # Codex r11: scope the cascade aggregation to the CURRENT run's + # instance set. Filtered reruns preserve unrelated prior + # annotations on disk (round 10 design), but the published + # ``eval.json`` must describe ONLY the current run's row set — + # otherwise ``cascading_phase1.n_dual_eval_tasks`` (union) would + # exceed ``eval.total_tasks`` (filtered count) and the rewritten + # ``phase1_count`` / ``phase1_rate`` would become uninterpretable. + _current_iids = { + str(td.get("instance_id") or "") for td in tasks + } - {""} metrics = emit_cascading_eval_json( rows_dir, Path(output_path), base_metrics=metrics, + instance_filter=_current_iids, ) logger.info( diff --git a/tests/test_cascading_report.py b/tests/test_cascading_report.py index baba6ae5..f03ef207 100644 --- a/tests/test_cascading_report.py +++ b/tests/test_cascading_report.py @@ -138,6 +138,66 @@ def test_aggregator_emits_cascading_phase1_block(tmp_path): assert deltas["n4"] == 1 # alien_2 added at N4 +def test_aggregator_instance_filter_scopes_to_current_run(tmp_path): + """Codex r11: filtered local reruns preserve unrelated prior + annotations on disk for human inspection. The aggregator MUST + accept an ``instance_filter`` so the published cascade describes + only the CURRENT run's row set — otherwise + ``n_dual_eval_tasks`` (union of new + stale) would exceed the + rest of ``eval.json``'s ``total_tasks`` (filtered count) and + rewritten ``phase1_count`` / ``phase1_rate`` would become + uninterpretable.""" + from bird_interact_agents.eval.cascading_report import ( + aggregate_cascading_phase1, + ) + + rows_dir = tmp_path / "rows" + rows_dir.mkdir() + # Three on-disk annotations; the current run only touches alien_1 + # (and alien_3, which fails). alien_2 is a leftover from a + # previous full run. + annotations = [ + _make_submission_annotation_json( + instance_id="alien_1", selected_database="alien", + n1=True, n2=True, n3=True, n4=True, n5=True, + n6=True, n7=True, n8=True, + ), + _make_submission_annotation_json( + instance_id="alien_2_stale", selected_database="alien", + n1=True, n2=True, n3=True, n4=True, n5=True, + n6=True, n7=True, n8=True, + ), + _make_submission_annotation_json( + instance_id="alien_3", selected_database="alien", + n1=False, n2=False, n3=False, n4=False, n5=False, + n6=False, n7=False, n8=False, + ), + ] + for ann in annotations: + d = rows_dir / ann["instance_id"] + d.mkdir() + (d / "submission_annotation.json").write_text(json.dumps(ann)) + + # No filter — back-compat path counts every dir. + block_all = aggregate_cascading_phase1(rows_dir) + assert block_all["n_dual_eval_tasks"] == 3 + assert block_all["counts"]["n1"] == 2 # alien_1 + alien_2_stale + + # With filter — only the current run's two iids are counted. + block_filtered = aggregate_cascading_phase1( + rows_dir, + instance_filter={"alien_1", "alien_3"}, + ) + assert block_filtered["n_dual_eval_tasks"] == 2 + assert block_filtered["counts"]["n1"] == 1, ( + "alien_2_stale's n1=True must NOT contribute to the filtered " + "count; only alien_1 passes" + ) + # And the stale dir is still on disk — the aggregator filter does + # NOT delete anything, only scopes the count. + assert (rows_dir / "alien_2_stale" / "submission_annotation.json").exists() + + def test_aggregator_enforces_monotonicity_on_tampered_row(tmp_path): """The aggregator MUST enforce monotonicity. We deliberately feed a violating row (N5=True, N6=False) — a "later level is more strict diff --git a/tests/test_run_local_inline_grader.py b/tests/test_run_local_inline_grader.py index 2e73216c..d889b6c3 100644 --- a/tests/test_run_local_inline_grader.py +++ b/tests/test_run_local_inline_grader.py @@ -722,7 +722,7 @@ def _stub(*, task_data, **kw): return out_dir / "submission_annotation.json" monkeypatch.setattr(run_mod, "grade_one_submission", _stub) - await run_mod.run_evaluation( + metrics = await run_mod.run_evaluation( framework="claude_sdk_otf_ainteract", query_mode="slayer", mode="a-interact", data_path="ignored", data_dir=str(tmp_path / "ignored_data_dir"), @@ -745,3 +745,18 @@ def _stub(*, task_data, **kw): (unrelated / "submission_annotation.json").read_text(), ) assert surviving["annotated_by"] == "unrelated-survivor" + # Codex r11: published metrics MUST describe ONLY the current run's + # row set, NOT the union with preserved prior rows. Otherwise + # ``cascading_phase1.n_dual_eval_tasks`` could exceed + # ``total_tasks`` and the rewritten ``phase1_count`` / + # ``phase1_rate`` would become uninterpretable. + cp = metrics["cascading_phase1"] + assert metrics["total_tasks"] == 1, f"got {metrics['total_tasks']}" + assert cp["n_dual_eval_tasks"] == 1, ( + f"cascade denominator must be scoped to the current run " + f"(filter_ids=[alien_1]) — got {cp['n_dual_eval_tasks']}; " + f"alien_99's preserved annotation must NOT pollute it" + ) + assert metrics["total_tasks"] == cp["n_dual_eval_tasks"], ( + "total_tasks and cascading_phase1.n_dual_eval_tasks MUST agree" + ) From 5cb2ea8c7072d150abbf719e2f8bec3d162d932c Mon Sep 17 00:00:00 2001 From: Egor Kraev Date: Tue, 2 Jun 2026 21:36:17 +0200 Subject: [PATCH 25/29] DEV-1515 round 13: comparator robustness against width + duplicate names MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two Codex findings on the cascade comparators: Item 1 (Codex major — tolerant_grader.py:150-156): ``compare_tie_order`` only checks row-count cardinality before indexing every row by ``orderby_indices``. A wrong-projection-width agent submission (fewer columns than the gold ORDER BY references) raised ``IndexError`` inside the ``_key`` builder; the exception bubbled out of the grader and the cloud / local fail-everything fallback wrote a generic "grader raised" annotation instead of the structured cascade verdict with miss_diagnostics — losing the very signal that would tell the analyst the agent's projection was too narrow. Fix: bounds-check ``max(orderby_indices)`` against the NARROWEST row on either side up front. Out-of-range index → return False cleanly (same disposition as the row-count check immediately above). Two new tests: pred too narrow, gold too narrow. Item 2 (Codex major — tolerant_grader.py:281-285): ``compare_column_order`` did ``set(pred_l) != set(gold_l)`` and then ``perm = [pred_l.index(c) for c in gold_l]``. With duplicate column names (e.g. ``a, b, a`` after a join + alias collapse), ``set(...)`` collapsed the duplicates and ``.index()`` returned the FIRST matching position for every later occurrence — so both gold "a" positions mapped to pred's first "a" column, and the second pred "a" column's actual value was silently ignored. A submission with wrong values in the second duplicate column could falsely pass N8. Fix: reject duplicate column names at the boundary. Duplicates make "column order tolerance" ill-defined — the tier was designed for distinct projections. Three new tests: duplicates on pred, duplicates on gold, plus a belt-and-braces regression that the canonical "same names, different order" pass still works. Co-Authored-By: Claude Opus 4.7 --- .../eval/tolerant_grader.py | 30 +++++- tests/test_tolerant_grader_comparators.py | 93 +++++++++++++++++++ 2 files changed, 122 insertions(+), 1 deletion(-) diff --git a/src/bird_interact_agents/eval/tolerant_grader.py b/src/bird_interact_agents/eval/tolerant_grader.py index ec62cf9d..c1f56caf 100644 --- a/src/bird_interact_agents/eval/tolerant_grader.py +++ b/src/bird_interact_agents/eval/tolerant_grader.py @@ -147,6 +147,18 @@ def compare_tie_order( if not orderby_indices: return _set_equal(pred, gold) + # Codex r12: a wrong-projection-width agent submission is a normal + # miss (e.g. forgot a column), but ``row[i]`` would raise IndexError + # below and bubble out of the grader, forcing the caller into a + # generic fail-everything fallback annotation. Bounds-check the + # ORDER BY indices against the NARROWEST row on either side first; + # any out-of-range index means tie-order cannot match → return False + # cleanly (same disposition as the row-count check above). + max_idx = max(orderby_indices) + rows_for_width = list(pred) + list(gold) + if rows_for_width and max_idx >= min(len(r) for r in rows_for_width): + return False + def _key(row: Sequence) -> Tuple: return tuple(row[i] for i in orderby_indices) @@ -278,9 +290,25 @@ def compare_column_order( return False pred_l = [c.lower() for c in pred_cols] gold_l = [c.lower() for c in gold_cols] + # Codex r12: duplicate column names make "column order tolerance" + # ill-defined. ``set(...)`` collapses duplicates, then + # ``pred_l.index(c)`` returns the FIRST matching position for every + # later occurrence — so e.g. pred ``[a, b, a]`` against gold + # ``[a, a, b]`` would map both gold "a" positions to pred's first + # "a" column and silently ignore the second pred "a" column's + # actual value, falsely passing N8. Reject the case at the boundary + # — duplicate-named projections aren't a column-order miss, they're + # a separate quality issue (likely a join-collision in the query). + if ( + len(pred_l) != len(set(pred_l)) + or len(gold_l) != len(set(gold_l)) + ): + return False if set(pred_l) != set(gold_l): return False - # Permutation: position in pred for each gold column. + # Permutation: position in pred for each gold column. Safe because + # the duplicate-name guard above ensures each gold name appears + # exactly once on each side. perm = [pred_l.index(c) for c in gold_l] aligned = [tuple(r[i] for i in perm) for r in pred] return _set_equal(aligned, gold) diff --git a/tests/test_tolerant_grader_comparators.py b/tests/test_tolerant_grader_comparators.py index 807e3001..ce697093 100644 --- a/tests/test_tolerant_grader_comparators.py +++ b/tests/test_tolerant_grader_comparators.py @@ -332,3 +332,96 @@ def test_parse_orderby_nulls_first_last_does_not_error(): "SELECT a, b FROM t ORDER BY a NULLS LAST, b NULLS FIRST" ) assert [k.column_index for k in keys] == [0, 1] + + +# --------------------------------------------------------------------------- +# Codex r12: comparator-boundary cases that pre-fix would either crash +# the grader (compare_tie_order index out-of-range) or falsely pass +# (compare_column_order with duplicate names). Both surface from valid +# agent misses, so they MUST land as ``False`` cleanly, not as +# AssertionError / silent-pass. +# --------------------------------------------------------------------------- + + +def test_compare_tie_order_returns_false_when_pred_too_narrow(): + """A wrong-projection agent submission (fewer columns than the + gold ORDER BY references) must NOT crash the grader. Pre-fix + ``row[i]`` raised ``IndexError`` and the cloud/local fallback + wrote a generic fail-everything annotation instead of a structured + cascade-miss.""" + from bird_interact_agents.eval.tolerant_grader import compare_tie_order + + # ORDER BY column index 5 — pred only has 2 columns per row. + pred = [("A", 1), ("B", 2)] + gold = [("A", 1, 0, 0, 0, "X"), ("B", 2, 0, 0, 0, "Y")] + assert ( + compare_tie_order(pred, gold, orderby_indices=[5]) is False + ), "must return False, not raise IndexError" + + +def test_compare_tie_order_returns_false_when_gold_too_narrow(): + """Symmetric: gold too narrow for an index also returns False + cleanly.""" + from bird_interact_agents.eval.tolerant_grader import compare_tie_order + + pred = [("A", 1, 0, 0, 0, "X"), ("B", 2, 0, 0, 0, "Y")] + gold = [("A", 1), ("B", 2)] + assert ( + compare_tie_order(pred, gold, orderby_indices=[5]) is False + ), "must return False, not raise IndexError" + + +def test_compare_column_order_rejects_duplicate_column_names_pred(): + """N8 tolerance: pred has a duplicate-named projection. The pre-fix + ``pred_l.index(c)`` mapped every duplicate occurrence in gold to + pred's FIRST matching position, silently ignoring later + duplicate columns' values. With duplicates the column-order + concept is ill-defined — return False rather than falsely pass.""" + from bird_interact_agents.eval.tolerant_grader import ( + compare_column_order, + ) + + # Pred: 3 columns named (a, b, a). Values for the duplicate "a" + # columns DIFFER, so a real comparison must NOT pass. + pred = [("X", "Y", "Z"), ("X2", "Y2", "Z2")] + gold = [("X", "X", "Y"), ("X2", "X2", "Y2")] # gold reads a,a,b + assert compare_column_order( + pred, gold, + pred_cols=["a", "b", "a"], + gold_cols=["a", "a", "b"], + ) is False, ( + "must NOT falsely pass — pred's second 'a' column has Z/Z2 " + "but gold expects X/X2; duplicate names make column-order " + "ill-defined" + ) + + +def test_compare_column_order_rejects_duplicate_column_names_gold(): + """Symmetric: duplicates on the GOLD side also return False.""" + from bird_interact_agents.eval.tolerant_grader import ( + compare_column_order, + ) + + pred = [("X", "Y", "Z")] + gold = [("X", "X", "Y")] + assert compare_column_order( + pred, gold, + pred_cols=["a", "b", "c"], + gold_cols=["a", "a", "b"], + ) is False + + +def test_compare_column_order_distinct_names_still_pass(): + """Belt-and-braces: the duplicate guard MUST NOT regress the + canonical "same names, different order" pass.""" + from bird_interact_agents.eval.tolerant_grader import ( + compare_column_order, + ) + + pred = [("X", "Y"), ("X2", "Y2")] + gold = [("Y", "X"), ("Y2", "X2")] + assert compare_column_order( + pred, gold, + pred_cols=["a", "b"], + gold_cols=["b", "a"], + ) is True From d57d0a44e9c7e555a8af9e03f5d1047ccb47fea7 Mon Sep 17 00:00:00 2001 From: Egor Kraev Date: Wed, 3 Jun 2026 09:26:41 +0200 Subject: [PATCH 26/29] DEV-1524: add claude_sdk_otf_raw + claude_sdk_otf_ainteract_raw agents MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two new raw-SQL Claude SDK OTF agents that mirror the SLayer OTF variants but use no SLayer at all — direct SQL via the BIRD tool suite, submit via submit_sql. Refactors existing slayer OTF prompts to share constants from a new _shared_otf_prompts.py, leaving rendered prompts byte-for-byte identical. Co-Authored-By: Claude Sonnet 4.6 --- .../agents/_shared_otf_prompts.py | 88 ++ .../agents/claude_sdk_otf/agent.py | 5 +- .../agents/claude_sdk_otf/prompts.py | 87 +- .../claude_sdk_otf_ainteract/prompts.py | 103 +-- .../claude_sdk_otf_ainteract_raw/__init__.py | 5 + .../claude_sdk_otf_ainteract_raw/agent.py | 380 ++++++++ .../claude_sdk_otf_ainteract_raw/prompts.py | 106 +++ .../agents/claude_sdk_otf_raw/__init__.py | 3 + .../agents/claude_sdk_otf_raw/agent.py | 292 +++++++ .../agents/claude_sdk_otf_raw/prompts.py | 97 +++ src/bird_interact_agents/cloud/driver.py | 4 +- src/bird_interact_agents/cloud/ray_app.py | 4 +- src/bird_interact_agents/run.py | 62 +- ...test_claude_sdk_otf_ainteract_raw_agent.py | 809 ++++++++++++++++++ tests/test_claude_sdk_otf_raw_agent.py | 617 +++++++++++++ tests/test_claude_sdk_otf_raw_run_wiring.py | 342 ++++++++ tests/test_shared_otf_prompts.py | 437 ++++++++++ 17 files changed, 3330 insertions(+), 111 deletions(-) create mode 100644 src/bird_interact_agents/agents/_shared_otf_prompts.py create mode 100644 src/bird_interact_agents/agents/claude_sdk_otf_ainteract_raw/__init__.py create mode 100644 src/bird_interact_agents/agents/claude_sdk_otf_ainteract_raw/agent.py create mode 100644 src/bird_interact_agents/agents/claude_sdk_otf_ainteract_raw/prompts.py create mode 100644 src/bird_interact_agents/agents/claude_sdk_otf_raw/__init__.py create mode 100644 src/bird_interact_agents/agents/claude_sdk_otf_raw/agent.py create mode 100644 src/bird_interact_agents/agents/claude_sdk_otf_raw/prompts.py create mode 100644 tests/test_claude_sdk_otf_ainteract_raw_agent.py create mode 100644 tests/test_claude_sdk_otf_raw_agent.py create mode 100644 tests/test_claude_sdk_otf_raw_run_wiring.py create mode 100644 tests/test_shared_otf_prompts.py diff --git a/src/bird_interact_agents/agents/_shared_otf_prompts.py b/src/bird_interact_agents/agents/_shared_otf_prompts.py new file mode 100644 index 00000000..0f793677 --- /dev/null +++ b/src/bird_interact_agents/agents/_shared_otf_prompts.py @@ -0,0 +1,88 @@ +"""Shared prompt string constants for the claude_sdk OTF agent family. + +These constants are used verbatim (format params substituted at compose +time) by both the SLayer OTF agents and the raw OTF agents to keep prompts +aligned wherever SLayer is not involved. + +Constraint: after the SLayer prompt files are refactored to import from +here, the rendered values of SLAYER_OTF_ONE_SHOT and SLAYER_OTF_AINTERACT +must remain byte-for-byte identical. SHA-256 snapshot tests in +tests/test_shared_otf_prompts.py enforce this. + +Format param conventions: + {sources_desc} — phrase describing the knowledge sources available + {action_label} — upper-case verb for Rule 0 heading ("ENCODE"/"SUBMIT") + {action_context} — Rule 0 first sentence opener + {submit_tool} — the submission tool name + {knowledge_source} — "a memory" (slayer) / "a knowledge definition" (raw) + {clause_b} — "(b) required by an ___" clause in one-shot check + {clause_c} — "(c) required by an ___" clause in ainteract check +""" + +# --------------------------------------------------------------------------- +# Shared constants — format params noted per variable +# --------------------------------------------------------------------------- + +# Format params: {sources_desc} +_NO_USER_TO_CONSULT = """\ +There is NO user to consult — for every operationalisation choice (numeric +threshold, value list, aggregation operator, case-sensitivity, grouping, +unit, rounding, sort direction, LIMIT) pick the most conservative, +defensible interpretation supported by {sources_desc}, and proceed autonomously.""" + +# No format params. +_DECOMPOSE_DISCIPLINE = """\ +1. DECOMPOSE the question into logical blocks. Every qualifier + (e.g. "premium", "highly-rated", "nearby", "active"), every projected + column, filter, grouping, unit, rounding and ordering hint is a + separate block that MUST be represented. Write the list out before + encoding.""" + +# Format params: {action_label}, {action_context}, {submit_tool} +_RULE_0_ASK_BEFORE = """\ +RULE 0 — ASK BEFORE YOU {action_label}. +{action_context} identify the single operationalisation +choice you are LEAST certain about — a numeric threshold, a value list / +IN-set, an aggregation operator, a case-sensitivity choice, a grouping +or standardisation, a unit (fraction vs percent), an output rounding, a +sort direction, or a LIMIT — and call `ask_user` on it ONCE. The user +holds masked knowledge-base ground-truth that is unrecoverable from the +visible KB alone. The submit gate will REFUSE `{submit_tool}` until you +have called `ask_user` at least once. Propose your best guess and ask +for the EXACT predicate / value / formula — never "what does X mean?".""" + +# Format params: {knowledge_source} +_ASK_AGAIN_RULE = """\ +4. ASK AGAIN IF NEEDED. Rule 0 covers the FIRST ask; for any further + operationalisation choice not pinned by {knowledge_source} or column + description, call `ask_user` again. If a reply lists multiple criteria + joined by "and", apply EACH as its own filter.""" + +# Format params: {submit_tool}, {clause_b} +_PRE_SUBMIT_MUTATION_CHECK_ONE_SHOT = """\ +5. PRE-SUBMIT MUTATION CHECK. Before calling `{submit_tool}`, audit every + TRIM, LOWER, UPPER, ROUND, CAST, dedup, canonicalize-via-CASE, and + output-shape choice in the FINAL query. Each one MUST be either + (a) explicitly named in the user's question or (b) required by an + {clause_b}. If neither holds, DROP the mutation and submit the raw + form. "Defensive" normalisation of an output column, a join key, a + JSON key, or a CHAR-padded literal silently corrupts the rowset — + never apply one without an explicit source. There is no user to + second-guess this on your behalf.""" + +# Format params: {submit_tool}, {clause_c} +_PRE_SUBMIT_MUTATION_CHECK_AINTERACT = """\ +6. PRE-SUBMIT MUTATION CHECK. Before calling `{submit_tool}`, audit every + TRIM, LOWER, UPPER, ROUND, CAST, dedup, canonicalize-via-CASE, and + output-shape choice in the FINAL query. Each one MUST be either + (a) explicitly named in the user's question, (b) explicitly named OR + authorized in a reply to one of your `ask_user` calls in this + session, or (c) required by an {clause_c}. If none of (a-b-c) hold, + DROP the mutation and submit the raw form. Particularly: when an + `ask_user` reply said "use exact values", "don't normalize", "use + this output shape / columns / sort axis", or named a specific format + (date, label casing, JSON shape), DO NOT silently override that on + final-assembly. Conversely, when an `ask_user` reply DID name a + specific transformation (e.g. "lowercase the bracket labels", + "round to 2 decimals", "TRIM the keys"), that reply IS the + authorization for that mutation — apply it.""" diff --git a/src/bird_interact_agents/agents/claude_sdk_otf/agent.py b/src/bird_interact_agents/agents/claude_sdk_otf/agent.py index a2491211..b7ed0b2e 100644 --- a/src/bird_interact_agents/agents/claude_sdk_otf/agent.py +++ b/src/bird_interact_agents/agents/claude_sdk_otf/agent.py @@ -93,7 +93,8 @@ def _slayer_tool_names() -> list[str]: def _make_turn_budget_hook( - max_turns: int, warn_within: int = _TURN_BUDGET_WARN_WITHIN + max_turns: int, warn_within: int = _TURN_BUDGET_WARN_WITHIN, + submit_tool: str = "submit_query", ): """Build a PostToolUse hook that nudges the agent to submit when it's within ``warn_within`` tool-calls of the hard ``max_turns`` cap. @@ -117,7 +118,7 @@ async def _hook(input_data, tool_use_id, context): "additionalContext": ( f"[TURN BUDGET] ~{remaining} model turn(s) remain before " f"the hard limit of {max_turns}. If you have a candidate " - "answer, call submit_query NOW — an un-submitted task " + f"answer, call {submit_tool} NOW — an un-submitted task " "scores zero." ), } diff --git a/src/bird_interact_agents/agents/claude_sdk_otf/prompts.py b/src/bird_interact_agents/agents/claude_sdk_otf/prompts.py index cc0dc664..6057cc69 100644 --- a/src/bird_interact_agents/agents/claude_sdk_otf/prompts.py +++ b/src/bird_interact_agents/agents/claude_sdk_otf/prompts.py @@ -23,6 +23,11 @@ from bird_interact_agents.agents._host_discovery_playbook import ( HOST_DISCOVERY_PLAYBOOK as _HOST_DISCOVERY_PLAYBOOK, ) +from bird_interact_agents.agents._shared_otf_prompts import ( + _DECOMPOSE_DISCIPLINE, + _NO_USER_TO_CONSULT, + _PRE_SUBMIT_MUTATION_CHECK_ONE_SHOT, +) # Shared submission contract (single-stage or nested-DAG). Literal JSON # braces are doubled because the prompt is consumed via ``str.format``. @@ -45,7 +50,7 @@ a named column first and filter on the name; raw SQL expressions are rejected in `filters`.""" -_ENCODE_CORE = """\ +_ENCODE_CORE_HEAD = """\ The database's domain knowledge is pre-loaded as SLayer MEMORIES — one per knowledge-base (KB) item, with ids like `{db_name}_kb_` whose body starts `KB —`. The base tables are already ingested as SLayer models, @@ -68,14 +73,9 @@ whitespace forms, abbreviations, alternate phrasings of the same concept. Use it BEFORE writing any IN-set (see rule 3 below). -ENCODE-THEN-QUERY DISCIPLINE: - -1. DECOMPOSE the question into logical blocks. Every qualifier - (e.g. "premium", "highly-rated", "nearby", "active"), every projected - column, filter, grouping, unit, rounding and ordering hint is a - separate block that MUST be represented. Write the list out before - encoding. +ENCODE-THEN-QUERY DISCIPLINE:""" +_ENCODE_CORE_TAIL = """\ 2. For each block, `search` for the relevant KB memory and any entity that already encodes it. A `memory:` token inside a KB body means that KB DEPENDS ON the referenced KB. @@ -117,44 +117,33 @@ sampled values yourself. """ - -SLAYER_OTF_ONE_SHOT = """\ -You are a data analyst. You have a SLayer semantic-layer MCP server plus a -native `submit_query` tool. Your job: answer the user's question by -ENCODING the domain knowledge it needs into the SLayer model as named -columns/measures, then writing a FINAL query that REFERENCES those named -entities instead of inlining their SQL. - -There is NO user to consult — for every operationalisation choice (numeric -threshold, value list, aggregation operator, case-sensitivity, grouping, -unit, rounding, sort direction, LIMIT) pick the most conservative, -defensible interpretation supported by the memories and column -descriptions, and proceed autonomously. - -""" + _ENCODE_CORE + """\ - -4. TEST candidate columns and the final query with `query` / - `query_nested`; sanity-check the generated SQL. - -5. PRE-SUBMIT MUTATION CHECK. Before calling `submit_query`, audit every - TRIM, LOWER, UPPER, ROUND, CAST, dedup, canonicalize-via-CASE, and - output-shape choice in the FINAL query. Each one MUST be either - (a) explicitly named in the user's question or (b) required by an - encoded KB. If neither holds, DROP the mutation and submit the raw - form. "Defensive" normalisation of an output column, a join key, a - JSON key, or a CHAR-padded literal silently corrupts the rowset — - never apply one without an explicit source. There is no user to - second-guess this on your behalf. - -6. SUBMIT. Write the FINAL query so it REFERENCES the named columns / - measures you encoded — do NOT inline their SQL back into the query. - Project exactly the columns the question names, and only those. - {submit} - -Budget: {budget} bird-coins (`submit_query` costs 3; SLayer reads/writes -are free but your total work is turn-bounded — encode only what the -question needs). - -Database: {db_name} -User question: {user_query} -""".replace("{submit}", _SUBMIT_CONTRACT) + "\n" + _HOST_DISCOVERY_PLAYBOOK +_ENCODE_CORE = _ENCODE_CORE_HEAD + "\n\n" + _DECOMPOSE_DISCIPLINE + "\n\n" + _ENCODE_CORE_TAIL + +SLAYER_OTF_ONE_SHOT = ( + "You are a data analyst. You have a SLayer semantic-layer MCP server plus a\n" + "native `submit_query` tool. Your job: answer the user's question by\n" + "ENCODING the domain knowledge it needs into the SLayer model as named\n" + "columns/measures, then writing a FINAL query that REFERENCES those named\n" + "entities instead of inlining their SQL.\n\n" + + _NO_USER_TO_CONSULT.format(sources_desc="the memories and column\ndescriptions") + + "\n\n" + + _ENCODE_CORE + + "\n4. TEST candidate columns and the final query with `query` /\n" + " `query_nested`; sanity-check the generated SQL.\n\n" + + _PRE_SUBMIT_MUTATION_CHECK_ONE_SHOT.format( + submit_tool="submit_query", + clause_b="encoded KB", + ) + + "\n\n6. SUBMIT. Write the FINAL query so it REFERENCES the named columns /\n" + " measures you encoded — do NOT inline their SQL back into the query.\n" + " Project exactly the columns the question names, and only those.\n" + " " + + _SUBMIT_CONTRACT + + "\n\nBudget: {budget} bird-coins (`submit_query` costs 3; SLayer reads/writes\n" + "are free but your total work is turn-bounded — encode only what the\n" + "question needs).\n\n" + "Database: {db_name}\n" + "User question: {user_query}\n" + + "\n" + + _HOST_DISCOVERY_PLAYBOOK +) diff --git a/src/bird_interact_agents/agents/claude_sdk_otf_ainteract/prompts.py b/src/bird_interact_agents/agents/claude_sdk_otf_ainteract/prompts.py index 98fb0693..cc29f51f 100644 --- a/src/bird_interact_agents/agents/claude_sdk_otf_ainteract/prompts.py +++ b/src/bird_interact_agents/agents/claude_sdk_otf_ainteract/prompts.py @@ -25,6 +25,12 @@ from bird_interact_agents.agents._host_discovery_playbook import ( HOST_DISCOVERY_PLAYBOOK as _HOST_DISCOVERY_PLAYBOOK, ) +from bird_interact_agents.agents._shared_otf_prompts import ( + _ASK_AGAIN_RULE, + _DECOMPOSE_DISCIPLINE, + _PRE_SUBMIT_MUTATION_CHECK_AINTERACT, + _RULE_0_ASK_BEFORE, +) # Shared submission contract (single-stage or nested-DAG). Literal JSON # braces are doubled because the prompt is consumed via ``str.format``. @@ -47,25 +53,14 @@ a named column first and filter on the name; raw SQL expressions are rejected in `filters`.""" - -SLAYER_OTF_AINTERACT = """\ +_AINTERACT_INTRO = """\ You are a data analyst. You have a SLayer semantic-layer MCP server plus native `ask_user` and `submit_query` tools. Your job: answer the user's question by ENCODING the domain knowledge it needs into the SLayer model as named columns/measures, then writing a FINAL query that REFERENCES -those named entities instead of inlining their SQL. - -RULE 0 — ASK BEFORE YOU ENCODE. -BEFORE the encoding loop below, identify the single operationalisation -choice you are LEAST certain about — a numeric threshold, a value list / -IN-set, an aggregation operator, a case-sensitivity choice, a grouping -or standardisation, a unit (fraction vs percent), an output rounding, a -sort direction, or a LIMIT — and call `ask_user` on it ONCE. The user -holds masked knowledge-base ground-truth that is unrecoverable from the -visible KB alone. The submit gate will REFUSE `submit_query` until you -have called `ask_user` at least once. Propose your best guess and ask -for the EXACT predicate / value / formula — never "what does X mean?". +those named entities instead of inlining their SQL.""" +_AINTERACT_SLAYER_TOOLS = """\ The database's domain knowledge is pre-loaded as SLayer MEMORIES — one per knowledge-base (KB) item, with ids like `{db_name}_kb_` whose body starts `KB —`. The base tables are already ingested as SLayer models, @@ -88,14 +83,9 @@ whitespace forms, abbreviations, alternate phrasings of the same concept. Use it BEFORE writing any IN-set (see rule 3 below). -ENCODE-THEN-QUERY DISCIPLINE: - -1. DECOMPOSE the question into logical blocks. Every qualifier - (e.g. "premium", "highly-rated", "nearby", "active"), every projected - column, filter, grouping, unit, rounding and ordering hint is a - separate block that MUST be represented. Write the list out before - encoding. +ENCODE-THEN-QUERY DISCIPLINE:""" +_AINTERACT_RULES_2_3 = """\ 2. For each block, `search` for the relevant KB memory and any entity that already encodes it. A `memory:` token inside a KB body means that KB DEPENDS ON the referenced KB. @@ -133,39 +123,38 @@ deliberately non-exhaustive; the `Sample values` line is the authoritative inventory of what's actually present in the column. A canonical-only IN-set will silently miss matching rows. Do not - rely on the user-sim to enumerate the variants — they will not. - -4. ASK AGAIN IF NEEDED. Rule 0 covers the FIRST ask; for any further - operationalisation choice not pinned by a memory or column - description, call `ask_user` again. If a reply lists multiple criteria - joined by "and", apply EACH as its own filter. - -5. TEST candidate columns and the final query with `query` / - `query_nested`; sanity-check the generated SQL. - -6. PRE-SUBMIT MUTATION CHECK. Before calling `submit_query`, audit every - TRIM, LOWER, UPPER, ROUND, CAST, dedup, canonicalize-via-CASE, and - output-shape choice in the FINAL query. Each one MUST be either - (a) explicitly named in the user's question, (b) explicitly named OR - authorized in a reply to one of your `ask_user` calls in this - session, or (c) required by an encoded KB. If none of (a-b-c) hold, - DROP the mutation and submit the raw form. Particularly: when an - `ask_user` reply said "use exact values", "don't normalize", "use - this output shape / columns / sort axis", or named a specific format - (date, label casing, JSON shape), DO NOT silently override that on - final-assembly. Conversely, when an `ask_user` reply DID name a - specific transformation (e.g. "lowercase the bracket labels", - "round to 2 decimals", "TRIM the keys"), that reply IS the - authorization for that mutation — apply it. - -7. SUBMIT. Write the FINAL query so it REFERENCES the named columns / - measures you encoded — do NOT inline their SQL back into the query. - Project exactly the columns the user named, and only those. {submit} - -Budget: {budget} bird-coins. `ask_user` costs 2, `submit_query` costs 3; -SLayer reads/writes are free but your total work is turn-bounded — encode -only what the question needs. If your budget runs out, submit immediately. - -Database: {db_name} -User question: {user_query} -""".replace("{submit}", _SUBMIT_CONTRACT) + "\n" + _HOST_DISCOVERY_PLAYBOOK + rely on the user-sim to enumerate the variants — they will not.""" + +SLAYER_OTF_AINTERACT = ( + _AINTERACT_INTRO + + "\n\n" + + _RULE_0_ASK_BEFORE.format( + action_label="ENCODE", + action_context="BEFORE the encoding loop below,", + submit_tool="submit_query", + ) + + "\n\n" + + _AINTERACT_SLAYER_TOOLS + + "\n\n" + + _DECOMPOSE_DISCIPLINE + + "\n\n" + + _AINTERACT_RULES_2_3 + + "\n\n" + + _ASK_AGAIN_RULE.format(knowledge_source="a memory") + + "\n\n5. TEST candidate columns and the final query with `query` /\n" + " `query_nested`; sanity-check the generated SQL.\n\n" + + _PRE_SUBMIT_MUTATION_CHECK_AINTERACT.format( + submit_tool="submit_query", + clause_c="encoded KB", + ) + + "\n\n7. SUBMIT. Write the FINAL query so it REFERENCES the named columns /\n" + " measures you encoded — do NOT inline their SQL back into the query.\n" + " Project exactly the columns the user named, and only those. " + + _SUBMIT_CONTRACT + + "\n\nBudget: {budget} bird-coins. `ask_user` costs 2, `submit_query` costs 3;\n" + "SLayer reads/writes are free but your total work is turn-bounded — encode\n" + "only what the question needs. If your budget runs out, submit immediately.\n" + "\nDatabase: {db_name}\nUser question: {user_query}\n" + + "\n" + + _HOST_DISCOVERY_PLAYBOOK +) diff --git a/src/bird_interact_agents/agents/claude_sdk_otf_ainteract_raw/__init__.py b/src/bird_interact_agents/agents/claude_sdk_otf_ainteract_raw/__init__.py new file mode 100644 index 00000000..8437c277 --- /dev/null +++ b/src/bird_interact_agents/agents/claude_sdk_otf_ainteract_raw/__init__.py @@ -0,0 +1,5 @@ +from bird_interact_agents.agents.claude_sdk_otf_ainteract_raw.agent import ( + ClaudeSDKOtfAInteractRawAgent, +) + +__all__ = ["ClaudeSDKOtfAInteractRawAgent"] diff --git a/src/bird_interact_agents/agents/claude_sdk_otf_ainteract_raw/agent.py b/src/bird_interact_agents/agents/claude_sdk_otf_ainteract_raw/agent.py new file mode 100644 index 00000000..149f8c67 --- /dev/null +++ b/src/bird_interact_agents/agents/claude_sdk_otf_ainteract_raw/agent.py @@ -0,0 +1,380 @@ +"""Claude Agent SDK raw-SQL OTF agent (mini-interact / a-interact flavor). + +A counterpart to ``claude_sdk_otf_ainteract`` that uses no SLayer at all — +the agent issues raw SQL via the bird-interact tool suite and submits via +``submit_sql``. Bound to ``--dataset mini_interact --mode a-interact +--query-mode raw``. + +Adds the same hard ``ask_user``-before-``submit_sql`` discipline as the +slayer ainteract variant: Rule 0 plus per-task PreToolUse/PostToolUse guards +built by ``_make_ask_user_guards``. +""" + +from __future__ import annotations + +import logging + +from claude_agent_sdk import ( + ClaudeAgentOptions, + ClaudeSDKClient, + HookMatcher, + create_sdk_mcp_server, +) + +from bird_interact_agents.agents.claude_sdk.agent import ( + _ctx_var, + accumulate_assistant_usage, + ask_user, + execute_sql, + get_all_column_meanings, + get_all_external_knowledge_names, + get_all_knowledge_definitions, + get_column_meaning, + get_knowledge_definition, + get_schema, + submit_sql, +) +from bird_interact_agents.agents.claude_sdk_otf.agent import ( + _MAX_TURNS, + _make_turn_budget_hook, +) +from bird_interact_agents.agents.claude_sdk_otf_ainteract_raw.prompts import ( + RAW_OTF_AINTERACT, +) +from bird_interact_agents.benchmark import get_benchmark +from bird_interact_agents.model_string import is_anthropic, native_model_id +from bird_interact_agents.harness import ( + SampleStatus, + _ambiguity_count, + finalize_result_row, + load_db_data_if_needed, + materialize_task_db, +) +from bird_interact_agents.usage import TokenUsage + +logger = logging.getLogger(__name__) + +# Full MCP tool name of the native ask_user tool — used as the PostToolUse +# counter's matcher and the nag hook's race-skip predicate. +_ASK_USER_TOOL = "mcp__bird-interact-tools__ask_user" + +# Full MCP tool name of the raw submission tool — gated by the PreToolUse hook. +_SUBMIT_SQL_TOOL = "mcp__bird-interact-tools__submit_sql" + +# How often (in total tool calls without ask_user) the nag fires. +_NAG_EVERY = 10 + +# All 7 BIRD raw-exploration tools + ask_user + raw submission tool. +_AINTERACT_RAW_TOOLS = [ + execute_sql, + get_schema, + get_all_column_meanings, + get_column_meaning, + get_all_external_knowledge_names, + get_knowledge_definition, + get_all_knowledge_definitions, + ask_user, + submit_sql, +] + + +def _make_ask_user_guards(): + """Build per-task hook callables that enforce ask-user-before-submit_sql. + + Returns ``(pre_submit_gate, post_ask_counter, post_nag)`` sharing a + single per-task ``state`` closure. The factory MUST be invoked inside + ``run_task`` (per task), NOT stored on the agent — a single agent + instance is reused across concurrent tasks via ``make_runner``, and + cross-task counter bleed would let one task's submit pass through + another task's denial gate. + + The gate is scoped to ``submit_sql`` only; ``submit_query`` (if ever + called) is not denied. + """ + state = {"ask_count": 0, "tool_calls": 0} + + async def pre_submit_gate(input_data, tool_use_id, context): + if input_data.get("tool_name") != _SUBMIT_SQL_TOOL: + return {} + if state["ask_count"] == 0: + return { + "hookSpecificOutput": { + "hookEventName": "PreToolUse", + "permissionDecision": "deny", + "permissionDecisionReason": ( + "You have not called ask_user. The user-sim has the " + "masked-KB ground truth that is unrecoverable from " + "knowledge definitions alone. Identify your single " + "most-uncertain operationalisation choice (threshold / " + "value list / aggregation / sort / unit / rounding) " + "and call ask_user on it before submitting." + ), + } + } + return {} + + async def post_ask_counter(input_data, tool_use_id, context): + state["ask_count"] += 1 + return {} + + async def post_nag(input_data, tool_use_id, context): + if input_data.get("tool_name") == _ASK_USER_TOOL: + return {} + state["tool_calls"] += 1 + if state["ask_count"] == 0 and state["tool_calls"] % _NAG_EVERY == 0: + return { + "hookSpecificOutput": { + "hookEventName": "PostToolUse", + "additionalContext": ( + f"[BENCHMARK NOTE] You have made {state['tool_calls']} " + "tool calls without consulting the user-sim. The " + "user-sim has the masked-KB ground truth — clarify " + "your single most uncertain operationalisation choice " + "before continuing." + ), + } + } + return {} + + return pre_submit_gate, post_ask_counter, post_nag + + +def _select_tools(eval_mode: str) -> list: + if eval_mode != "a-interact": + raise ValueError( + "claude_sdk_otf_ainteract_raw supports only eval_mode='a-interact'; " + f"got {eval_mode!r}" + ) + return list(_AINTERACT_RAW_TOOLS) + + +def _build_prompt(eval_mode: str, task_data: dict, budget: float) -> str: + if eval_mode != "a-interact": + raise ValueError( + "claude_sdk_otf_ainteract_raw supports only eval_mode='a-interact'; " + f"got {eval_mode!r}" + ) + return RAW_OTF_AINTERACT.format( + budget=budget, + db_name=task_data["selected_database"], + user_query=task_data["amb_user_query"], + ) + + +class ClaudeSDKOtfAInteractRawAgent: + """SystemAgent: Claude SDK raw-SQL OTF agent with enforced ask-user discipline. + + Anthropic-only (the SDK is locked to Anthropic). Bound to + ``--dataset mini_interact --mode a-interact --query-mode raw``. No SLayer + MCP server. Mismatched dataset, eval_mode, or query_mode is rejected at + the agent boundary. + """ + + _EFFORT_CHOICES = ("low", "medium", "high", "max") + + def __init__( + self, + model: str = "anthropic/claude-sonnet-4-5", + reasoning_effort: str | None = None, + ) -> None: + if reasoning_effort is not None and reasoning_effort not in self._EFFORT_CHOICES: + raise ValueError( + f"reasoning_effort must be one of {self._EFFORT_CHOICES} or None; " + f"got {reasoning_effort!r}" + ) + self.model = model + self.reasoning_effort = reasoning_effort + + async def run_task( + self, + task_data: dict, + data_path_base: str, + budget: float, + query_mode: str, + eval_mode: str = "a-interact", + user_sim_model: str = "anthropic/claude-haiku-4-5-20251001", + user_sim_prompt_version: str = "v2", + ) -> dict: + if query_mode != "raw": + raise ValueError( + "claude_sdk_otf_ainteract_raw supports only --query-mode raw; " + f"got {query_mode!r}" + ) + if eval_mode != "a-interact": + raise ValueError( + "claude_sdk_otf_ainteract_raw supports only --mode a-interact; " + f"got {eval_mode!r}" + ) + + dataset = task_data.get("dataset") or "mini_interact" + if get_benchmark(dataset).name != "mini_interact": + raise ValueError( + "claude_sdk_otf_ainteract_raw is bound to --dataset mini_interact; " + f"got dataset={dataset!r}" + ) + + instance_id = task_data["instance_id"] + db_name = task_data["selected_database"] + + if not is_anthropic(self.model): + msg = ( + f"claude_sdk_otf_ainteract_raw requires an Anthropic model; " + f"got {self.model!r}. " + "Skipped — use --framework pydantic_ai for non-Anthropic models." + ) + logger.warning("[%s] %s", instance_id, msg) + return finalize_result_row( + { + "task_id": instance_id, + "instance_id": instance_id, + "database": db_name, + "phase1_passed": False, + "phase2_passed": False, + "total_reward": 0.0, + "trajectory": [], + "error": msg, + }, + deleted_kb_ids=[], + slayer_storage_dir="", + ) + + status = SampleStatus( + idx=0, + original_data=task_data, + remaining_budget=budget, + total_budget=budget, + ) + + max_asks = _ambiguity_count(task_data) + 3 + + accum = TokenUsage() + trajectory: list[dict] = [] + ctx_dict: dict | None = None + try: + load_db_data_if_needed(db_name, data_path_base) + materialize_task_db(task_data, data_path_base) + + ctx_dict = { + "status": status, + "data_path_base": data_path_base, + "user_sim_model": user_sim_model, + "user_sim_prompt_version": user_sim_prompt_version, + "result": None, + "eval_mode": eval_mode, + "query_mode": query_mode, + "max_asks": max_asks, + "asks_used": 0, + "usage": accum, + } + _ctx_var.set(ctx_dict) + + tools = _select_tools(eval_mode) + prompt = _build_prompt(eval_mode, task_data, budget) + + server = create_sdk_mcp_server( + name="bird-interact-tools", version="1.0.0", tools=tools, + ) + tool_names = [f"mcp__bird-interact-tools__{t.name}" for t in tools] + + pre_submit_gate, post_ask_counter, post_nag = _make_ask_user_guards() + + options = ClaudeAgentOptions( + system_prompt=prompt, + mcp_servers={"bird-interact-tools": server}, + allowed_tools=tool_names, + tools=[], + setting_sources=[], + model=native_model_id(self.model), + effort=self.reasoning_effort, + max_turns=_MAX_TURNS, + hooks={ + "PreToolUse": [ + HookMatcher( + matcher=_SUBMIT_SQL_TOOL, + hooks=[pre_submit_gate], + ), + ], + "PostToolUse": [ + HookMatcher( + matcher=_ASK_USER_TOOL, + hooks=[post_ask_counter], + ), + HookMatcher(hooks=[post_nag]), + HookMatcher( + hooks=[_make_turn_budget_hook( + _MAX_TURNS, submit_tool="submit_sql", + )], + ), + ], + }, + ) + + async with ClaudeSDKClient(options=options) as client: + await client.query(task_data["amb_user_query"]) + async for msg in client.receive_response(): + trajectory.append( + {"type": str(type(msg).__name__), "data": str(msg)[:500]} + ) + accumulate_assistant_usage(accum, msg, self.model) + except Exception as e: + logger.error( + "claude_sdk_otf_ainteract_raw error on %s: %s", + instance_id, e, exc_info=True, + ) + result = (ctx_dict or {}).get("result") or {} + return finalize_result_row( + { + "task_id": instance_id, + "instance_id": instance_id, + "database": db_name, + "phase1_passed": result.get("phase1_passed", False), + "phase2_passed": result.get("phase2_passed", False), + "total_reward": result.get("total_reward", 0.0), + "submitted_sql": result.get("submitted_sql"), + "submitted_query": result.get("submitted_query"), + "submission_status": result.get("submission_status"), + "predicted_result_json": result.get("predicted_result_json"), + "gold_result_json": result.get("gold_result_json"), + "phase1_observation": result.get("phase1_observation"), + "phase2_observation": result.get("phase2_observation"), + "trajectory": trajectory, + "error": str(e), + "usage": { + **accum.model_dump(), + "n_ask_user_calls": (ctx_dict or {}).get("asks_used", 0), + }, + "phase1_observation_audited": result.get("phase1_observation_audited"), + "phase1_observation_original": result.get("phase1_observation_original"), + }, + deleted_kb_ids=[], + slayer_storage_dir="", + ) + + result = (ctx_dict or {}).get("result") or {} + return finalize_result_row( + { + "task_id": instance_id, + "instance_id": instance_id, + "database": db_name, + "phase1_passed": result.get("phase1_passed", False), + "phase2_passed": result.get("phase2_passed", False), + "total_reward": result.get("total_reward", 0.0), + "submitted_sql": result.get("submitted_sql"), + "submitted_query": result.get("submitted_query"), + "submission_status": result.get("submission_status"), + "predicted_result_json": result.get("predicted_result_json"), + "gold_result_json": result.get("gold_result_json"), + "phase1_observation": result.get("phase1_observation"), + "phase2_observation": result.get("phase2_observation"), + "trajectory": trajectory, + "error": None, + "usage": { + **accum.model_dump(), + "n_ask_user_calls": ctx_dict.get("asks_used", 0), + }, + "phase1_observation_audited": result.get("phase1_observation_audited"), + "phase1_observation_original": result.get("phase1_observation_original"), + }, + deleted_kb_ids=[], + slayer_storage_dir="", + ) diff --git a/src/bird_interact_agents/agents/claude_sdk_otf_ainteract_raw/prompts.py b/src/bird_interact_agents/agents/claude_sdk_otf_ainteract_raw/prompts.py new file mode 100644 index 00000000..24bde76e --- /dev/null +++ b/src/bird_interact_agents/agents/claude_sdk_otf_ainteract_raw/prompts.py @@ -0,0 +1,106 @@ +"""System prompt for the raw-SQL OTF agent (mini-interact / a-interact flavor). + +A counterpart to ``claude_sdk_otf_ainteract`` that uses the same query- +discipline structure and shares prompt constants from ``_shared_otf_prompts`` +wherever SLayer is not involved, but issues raw SQL via ``execute_sql`` and +submits via ``submit_sql`` — no SLayer model store, no MCP slayer server. + +Adds the same hard ``ask_user``-before-``submit_sql`` discipline as the +slayer ainteract variant: Rule 0 plus a PreToolUse deny gate in +``agent.py::_make_ask_user_guards``. + +Format params: ``budget``, ``db_name``, ``user_query``. +""" + +from bird_interact_agents.agents._shared_otf_prompts import ( + _ASK_AGAIN_RULE, + _DECOMPOSE_DISCIPLINE, + _PRE_SUBMIT_MUTATION_CHECK_AINTERACT, + _RULE_0_ASK_BEFORE, +) + +_RAW_AINTERACT_INTRO = """\ +You are a data analyst. You have direct SQL access to a database plus +native `ask_user` and `submit_sql` tools. Your job: answer the user's +question by exploring the schema and knowledge definitions, then writing +a SQL query that precisely captures what the question asks for.""" + +_RAW_AINTERACT_DB_TOOLS = """\ +DATABASE TOOLS (read their own descriptions). Use `get_schema` FIRST to +see all tables, columns, and types. Use `get_all_column_meanings` or +`get_column_meaning` to read column descriptions and sample values. Use +`get_all_external_knowledge_names`, `get_knowledge_definition`, or +`get_all_knowledge_definitions` to retrieve domain knowledge. Use +`execute_sql` to explore data and test queries. + +READ A KNOWN COLUMN'S FULL DESCRIPTION before committing to it as a +filter, projection, or join key — `get_column_meaning`. The +`Sample values:` in the returned description are your authoritative source +of which literal forms actually occur in this column — case variants, +whitespace forms, abbreviations, alternate phrasings of the same concept. +Use it BEFORE writing any IN-set (see rule 3 below). + +QUERY DISCIPLINE:""" + +_RAW_AINTERACT_RULES_2_3 = """\ +2. For each block, retrieve the relevant knowledge definition(s) via + `get_knowledge_definition`. Check column descriptions for any column + the block filters on or projects. + +3. WRITE IN DEPENDENCY ORDER. For each sub-expression: + - Use `get_column_meaning` to confirm which table and column best + represents what the block describes. Never guess from names alone. + - To join tables, use only relationships evident from the schema + (`get_schema`). Do NOT invent a join not present in the schema. + - When a later sub-expression builds on an earlier one, compose the + SQL incrementally — do NOT re-derive or inline an intermediate + expression. + - Normalise text ONLY in filter / predicate positions + (`LOWER(TRIM(col)) = 'value'`, lowercase the literal) — NEVER on a + projected, grouped, or join-key column (that would corrupt the + returned value). + - If a knowledge definition cites named literals that are ABSENT from + the column's sampled values (check via `get_column_meaning`), do + not write that predicate. + - Symmetric companion: if the column's `Sample values` show variants + of the knowledge-cited literals — case differences, internal + whitespace, abbreviations (`apt` for `Apartment`, `Y` for `yes`), + or alternate phrasings of the same concept (`brick house` vs + `brickwork house`, `2014+` / `after 2014` for `2014 or newer`) — + NORMALISE and EXTEND the IN-set to include those variants. Knowledge + definition hedges ("etc.", "like", "include") and `Ex.` enumerations + are deliberately non-exhaustive; the `Sample values` line is the + authoritative inventory of what's actually present in the column. + A canonical-only IN-set will silently miss matching rows. Do not + rely on the user-sim to enumerate the variants — they will not.""" + +RAW_OTF_AINTERACT = ( + _RAW_AINTERACT_INTRO + + "\n\n" + + _RULE_0_ASK_BEFORE.format( + action_label="SUBMIT", + action_context="BEFORE writing your SQL query,", + submit_tool="submit_sql", + ) + + "\n\n" + + _RAW_AINTERACT_DB_TOOLS + + "\n\n" + + _DECOMPOSE_DISCIPLINE + + "\n\n" + + _RAW_AINTERACT_RULES_2_3 + + "\n\n" + + _ASK_AGAIN_RULE.format(knowledge_source="a knowledge definition") + + "\n\n5. TEST the final query with `execute_sql`; sanity-check the result\n" + " shape, row count, and values.\n\n" + + _PRE_SUBMIT_MUTATION_CHECK_AINTERACT.format( + submit_tool="submit_sql", + clause_c="knowledge definition", + ) + + "\n\n7. SUBMIT. Call `submit_sql` with your final SQL — a prose answer is\n" + " not a submission. Project exactly the columns the user named, and\n" + " only those.\n\n" + "Budget: {budget} bird-coins. `ask_user` costs 2, `submit_sql` costs 3;\n" + "exploration tools are free but your total work is turn-bounded — explore\n" + "only what the question needs. If your budget runs out, submit immediately.\n" + "\nDatabase: {db_name}\nUser question: {user_query}\n" +) diff --git a/src/bird_interact_agents/agents/claude_sdk_otf_raw/__init__.py b/src/bird_interact_agents/agents/claude_sdk_otf_raw/__init__.py new file mode 100644 index 00000000..23cc5b87 --- /dev/null +++ b/src/bird_interact_agents/agents/claude_sdk_otf_raw/__init__.py @@ -0,0 +1,3 @@ +from bird_interact_agents.agents.claude_sdk_otf_raw.agent import ClaudeSDKOtfRawAgent + +__all__ = ["ClaudeSDKOtfRawAgent"] diff --git a/src/bird_interact_agents/agents/claude_sdk_otf_raw/agent.py b/src/bird_interact_agents/agents/claude_sdk_otf_raw/agent.py new file mode 100644 index 00000000..da1af88d --- /dev/null +++ b/src/bird_interact_agents/agents/claude_sdk_otf_raw/agent.py @@ -0,0 +1,292 @@ +"""Claude Agent SDK raw-SQL OTF agent (livesqlbench / one-shot flavor). + +A counterpart to ``claude_sdk_otf`` that uses no SLayer at all — the agent +issues raw SQL via the bird-interact tool suite and submits via ``submit_sql``. +Bound to ``--dataset livesqlbench --mode one-shot --query-mode raw``. + +Prompt structure mirrors ``claude_sdk_otf`` through shared ``_shared_otf_prompts`` +constants; the exploration + test + mutation-check discipline is the same, but +operates on raw SQL instead of a SLayer model store. +""" + +from __future__ import annotations + +import logging + +from claude_agent_sdk import ( + ClaudeAgentOptions, + ClaudeSDKClient, + HookMatcher, + create_sdk_mcp_server, +) + +from bird_interact_agents.agents.claude_sdk.agent import ( + _ctx_var, + accumulate_assistant_usage, + execute_sql, + get_all_column_meanings, + get_all_external_knowledge_names, + get_all_knowledge_definitions, + get_column_meaning, + get_knowledge_definition, + get_schema, + submit_sql, +) +from bird_interact_agents.agents.claude_sdk_otf.agent import ( + _MAX_TURNS, + _TURN_BUDGET_WARN_WITHIN, + _make_turn_budget_hook as _make_turn_budget_hook_base, +) +from bird_interact_agents.agents.claude_sdk_otf_raw.prompts import RAW_OTF_ONE_SHOT +from bird_interact_agents.benchmark import get_benchmark +from bird_interact_agents.model_string import is_anthropic, native_model_id +from bird_interact_agents.harness import ( + SampleStatus, + finalize_result_row, + load_db_data_if_needed, + materialize_task_db, +) +from bird_interact_agents.usage import TokenUsage + +logger = logging.getLogger(__name__) + + +def _make_turn_budget_hook( + max_turns: int, + warn_within: int = _TURN_BUDGET_WARN_WITHIN, + submit_tool: str = "submit_sql", +): + """Thin wrapper around the shared hook; defaults ``submit_tool`` to + ``submit_sql`` (raw agents never have ``submit_query``). + """ + return _make_turn_budget_hook_base(max_turns, warn_within, submit_tool) + + +# All 7 BIRD raw-exploration tools + the raw submission tool. +_RAW_TOOLS = [ + execute_sql, + get_schema, + get_all_column_meanings, + get_column_meaning, + get_all_external_knowledge_names, + get_knowledge_definition, + get_all_knowledge_definitions, + submit_sql, +] + + +def _select_tools(eval_mode: str) -> list: + if eval_mode != "one-shot": + raise ValueError( + "claude_sdk_otf_raw supports only eval_mode='one-shot'; " + f"got {eval_mode!r}" + ) + return list(_RAW_TOOLS) + + +def _build_prompt(eval_mode: str, task_data: dict, budget: float) -> str: + if eval_mode != "one-shot": + raise ValueError( + "claude_sdk_otf_raw supports only eval_mode='one-shot'; " + f"got {eval_mode!r}" + ) + return RAW_OTF_ONE_SHOT.format( + budget=budget, + db_name=task_data["selected_database"], + user_query=task_data["amb_user_query"], + ) + + +class ClaudeSDKOtfRawAgent: + """SystemAgent: Claude SDK raw-SQL OTF agent. + + Anthropic-only (the SDK is locked to Anthropic). Bound to + ``--dataset livesqlbench --mode one-shot --query-mode raw``. No SLayer + MCP server, no slayer_setup requirement. Mismatched dataset, eval_mode, + or query_mode is rejected at the agent boundary. + """ + + _EFFORT_CHOICES = ("low", "medium", "high", "max") + + def __init__( + self, + model: str = "anthropic/claude-sonnet-4-5", + reasoning_effort: str | None = None, + ) -> None: + if reasoning_effort is not None and reasoning_effort not in self._EFFORT_CHOICES: + raise ValueError( + f"reasoning_effort must be one of {self._EFFORT_CHOICES} or None; " + f"got {reasoning_effort!r}" + ) + self.model = model + self.reasoning_effort = reasoning_effort + + async def run_task( + self, + task_data: dict, + data_path_base: str, + budget: float, + query_mode: str, + eval_mode: str = "one-shot", + user_sim_model: str = "anthropic/claude-haiku-4-5-20251001", + user_sim_prompt_version: str = "v2", + ) -> dict: + if query_mode != "raw": + raise ValueError( + "claude_sdk_otf_raw supports only --query-mode raw; " + f"got {query_mode!r}" + ) + if eval_mode != "one-shot": + raise ValueError( + "claude_sdk_otf_raw supports only --mode one-shot; " + f"got {eval_mode!r}" + ) + + dataset = task_data.get("dataset") or "livesqlbench" + if get_benchmark(dataset).name != "livesqlbench": + raise ValueError( + "claude_sdk_otf_raw is bound to --dataset livesqlbench; " + f"got dataset={dataset!r}" + ) + + instance_id = task_data["instance_id"] + db_name = task_data["selected_database"] + + if not is_anthropic(self.model): + msg = ( + f"claude_sdk_otf_raw requires an Anthropic model; got {self.model!r}. " + "Skipped — use --framework pydantic_ai for non-Anthropic models." + ) + logger.warning("[%s] %s", instance_id, msg) + return finalize_result_row( + { + "task_id": instance_id, + "instance_id": instance_id, + "database": db_name, + "phase1_passed": False, + "phase2_passed": False, + "total_reward": 0.0, + "trajectory": [], + "error": msg, + }, + deleted_kb_ids=[], + slayer_storage_dir="", + ) + + status = SampleStatus( + idx=0, + original_data=task_data, + remaining_budget=budget, + total_budget=budget, + ) + + accum = TokenUsage() + trajectory: list[dict] = [] + ctx_dict: dict | None = None + try: + load_db_data_if_needed(db_name, data_path_base) + materialize_task_db(task_data, data_path_base) + + ctx_dict = { + "status": status, + "data_path_base": data_path_base, + "user_sim_model": user_sim_model, + "user_sim_prompt_version": user_sim_prompt_version, + "result": None, + "eval_mode": eval_mode, + "query_mode": query_mode, + "usage": accum, + } + _ctx_var.set(ctx_dict) + + tools = _select_tools(eval_mode) + prompt = _build_prompt(eval_mode, task_data, budget) + + server = create_sdk_mcp_server( + name="bird-interact-tools", version="1.0.0", tools=tools, + ) + tool_names = [f"mcp__bird-interact-tools__{t.name}" for t in tools] + + options = ClaudeAgentOptions( + system_prompt=prompt, + mcp_servers={"bird-interact-tools": server}, + allowed_tools=tool_names, + tools=[], + setting_sources=[], + model=native_model_id(self.model), + effort=self.reasoning_effort, + max_turns=_MAX_TURNS, + hooks={ + "PostToolUse": [ + HookMatcher( + hooks=[_make_turn_budget_hook( + _MAX_TURNS, submit_tool="submit_sql", + )], + ), + ], + }, + ) + + async with ClaudeSDKClient(options=options) as client: + await client.query(task_data["amb_user_query"]) + async for msg in client.receive_response(): + trajectory.append( + {"type": str(type(msg).__name__), "data": str(msg)[:500]} + ) + accumulate_assistant_usage(accum, msg, self.model) + except Exception as e: + logger.error( + "claude_sdk_otf_raw error on %s: %s", + instance_id, e, exc_info=True, + ) + result = (ctx_dict or {}).get("result") or {} + return finalize_result_row( + { + "task_id": instance_id, + "instance_id": instance_id, + "database": db_name, + "phase1_passed": result.get("phase1_passed", False), + "phase2_passed": result.get("phase2_passed", False), + "total_reward": result.get("total_reward", 0.0), + "submitted_sql": result.get("submitted_sql"), + "submitted_query": result.get("submitted_query"), + "submission_status": result.get("submission_status"), + "predicted_result_json": result.get("predicted_result_json"), + "gold_result_json": result.get("gold_result_json"), + "phase1_observation": result.get("phase1_observation"), + "phase2_observation": result.get("phase2_observation"), + "trajectory": trajectory, + "error": str(e), + "usage": accum.model_dump(), + "phase1_observation_audited": result.get("phase1_observation_audited"), + "phase1_observation_original": result.get("phase1_observation_original"), + }, + deleted_kb_ids=[], + slayer_storage_dir="", + ) + + result = (ctx_dict or {}).get("result") or {} + return finalize_result_row( + { + "task_id": instance_id, + "instance_id": instance_id, + "database": db_name, + "phase1_passed": result.get("phase1_passed", False), + "phase2_passed": result.get("phase2_passed", False), + "total_reward": result.get("total_reward", 0.0), + "submitted_sql": result.get("submitted_sql"), + "submitted_query": result.get("submitted_query"), + "submission_status": result.get("submission_status"), + "predicted_result_json": result.get("predicted_result_json"), + "gold_result_json": result.get("gold_result_json"), + "phase1_observation": result.get("phase1_observation"), + "phase2_observation": result.get("phase2_observation"), + "trajectory": trajectory, + "error": None, + "usage": accum.model_dump(), + "phase1_observation_audited": result.get("phase1_observation_audited"), + "phase1_observation_original": result.get("phase1_observation_original"), + }, + deleted_kb_ids=[], + slayer_storage_dir="", + ) diff --git a/src/bird_interact_agents/agents/claude_sdk_otf_raw/prompts.py b/src/bird_interact_agents/agents/claude_sdk_otf_raw/prompts.py new file mode 100644 index 00000000..c50109ad --- /dev/null +++ b/src/bird_interact_agents/agents/claude_sdk_otf_raw/prompts.py @@ -0,0 +1,97 @@ +"""System prompt for the raw-SQL OTF agent (livesqlbench / one-shot flavor). + +A counterpart to ``claude_sdk_otf`` that uses the same query-discipline +structure and shares prompt constants from ``_shared_otf_prompts`` wherever +SLayer is not involved, but issues raw SQL via ``execute_sql`` and submits +via ``submit_sql`` — no SLayer model store, no MCP slayer server. + +Format params: ``budget``, ``db_name``, ``user_query``. +""" + +from bird_interact_agents.agents._shared_otf_prompts import ( + _DECOMPOSE_DISCIPLINE, + _NO_USER_TO_CONSULT, + _PRE_SUBMIT_MUTATION_CHECK_ONE_SHOT, +) + +_RAW_INTRO = """\ +You are a data analyst. You have direct SQL access to a database plus a +native `submit_sql` tool. Your job: answer the user's question by +exploring the schema and knowledge definitions, then writing a SQL query +that precisely captures what the question asks for.""" + +_RAW_DB_TOOLS = """\ +DATABASE TOOLS (read their own descriptions). Use `get_schema` FIRST to +see all tables, columns, and types. Use `get_all_column_meanings` or +`get_column_meaning` to read column descriptions and sample values. Use +`get_all_external_knowledge_names`, `get_knowledge_definition`, or +`get_all_knowledge_definitions` to retrieve domain knowledge. Use +`execute_sql` to explore data and test queries. + +READ A KNOWN COLUMN'S FULL DESCRIPTION before committing to it as a +filter, projection, or join key — `get_column_meaning`. The +`Sample values:` in the returned description are your authoritative source +of which literal forms actually occur in this column — case variants, +whitespace forms, abbreviations, alternate phrasings of the same concept. +Use it BEFORE writing any IN-set (see rule 3 below). + +QUERY DISCIPLINE:""" + +_RAW_RULES_2_3 = """\ +2. For each block, retrieve the relevant knowledge definition(s) via + `get_knowledge_definition`. Check column descriptions for any column + the block filters on or projects. + +3. WRITE IN DEPENDENCY ORDER. For each sub-expression: + - Use `get_column_meaning` to confirm which table and column best + represents what the block describes. Never guess from names alone. + - To join tables, use only relationships evident from the schema + (`get_schema`). Do NOT invent a join not present in the schema. + - When a later sub-expression builds on an earlier one, compose the + SQL incrementally — do NOT re-derive or inline an intermediate + expression. + - Normalise text ONLY in filter / predicate positions + (`LOWER(TRIM(col)) = 'value'`, lowercase the literal) — NEVER on a + projected, grouped, or join-key column (that would corrupt the + returned value). + - If a knowledge definition cites named literals that are ABSENT from + the column's sampled values (check via `get_column_meaning`), do + not write that predicate. + - Symmetric companion: if the column's `Sample values` show variants + of the knowledge-cited literals — case differences, internal + whitespace, abbreviations (`apt` for `Apartment`, `Y` for `yes`), + or alternate phrasings of the same concept (`brick house` vs + `brickwork house`, `2014+` / `after 2014` for `2014 or newer`) — + NORMALISE and EXTEND the IN-set to include those variants. Knowledge + definition hedges ("etc.", "like", "include") and `Ex.` enumerations + are deliberately non-exhaustive; the `Sample values` line is the + authoritative inventory of what's actually present in the column. + A canonical-only IN-set will silently miss matching rows. There is + no user-sim to confirm this for you in one-shot mode — read the + sampled values yourself.""" + +RAW_OTF_ONE_SHOT = ( + _RAW_INTRO + + "\n\n" + + _NO_USER_TO_CONSULT.format(sources_desc="the schema and knowledge\ndefinitions") + + "\n\n" + + _RAW_DB_TOOLS + + "\n\n" + + _DECOMPOSE_DISCIPLINE + + "\n\n" + + _RAW_RULES_2_3 + + "\n\n4. TEST the final query with `execute_sql`; sanity-check the result\n" + " shape, row count, and values.\n\n" + + _PRE_SUBMIT_MUTATION_CHECK_ONE_SHOT.format( + submit_tool="submit_sql", + clause_b="knowledge definition", + ) + + "\n\n6. SUBMIT. Call `submit_sql` with your final SQL — a prose answer is\n" + " not a submission. Project exactly the columns the question names,\n" + " and only those.\n\n" + "Budget: {budget} bird-coins (`submit_sql` costs 3; exploration tools\n" + "are free but your total work is turn-bounded — explore only what the\n" + "question needs).\n\n" + "Database: {db_name}\n" + "User question: {user_query}\n" +) diff --git a/src/bird_interact_agents/cloud/driver.py b/src/bird_interact_agents/cloud/driver.py index 13ccc1fd..1398c0c8 100644 --- a/src/bird_interact_agents/cloud/driver.py +++ b/src/bird_interact_agents/cloud/driver.py @@ -287,8 +287,10 @@ def _slayer_uploads_for(args) -> list[tuple[Path, str, bool]]: seed so the cloud skips re-encoding that DB; if absent, the cloud encodes lazily on first task. """ - setup = args.slayer_setup fw = args.framework + if fw in ("claude_sdk_otf_raw", "claude_sdk_otf_ainteract_raw"): + return [] + setup = args.slayer_setup benchmark = _submit_benchmark(args) if setup == "pre-encoded": return [(submitter_repo_root() / "slayer_models", "slayer_models", True)] diff --git a/src/bird_interact_agents/cloud/ray_app.py b/src/bird_interact_agents/cloud/ray_app.py index c279dfea..c5f43c3d 100644 --- a/src/bird_interact_agents/cloud/ray_app.py +++ b/src/bird_interact_agents/cloud/ray_app.py @@ -133,8 +133,10 @@ def _slayer_artifacts_for(cfg: dict[str, Any]) -> list[tuple[str, Path, bool]]: """ from bird_interact_agents import paths - setup = cfg.get("slayer_setup") fw = cfg.get("framework") + if fw in ("claude_sdk_otf_raw", "claude_sdk_otf_ainteract_raw"): + return [] + setup = cfg.get("slayer_setup") if setup == "pre-encoded": artifacts = [("slayer_models", True)] elif fw == "pydantic_ai_otf_encode": diff --git a/src/bird_interact_agents/run.py b/src/bird_interact_agents/run.py index 2188d229..c0396de3 100644 --- a/src/bird_interact_agents/run.py +++ b/src/bird_interact_agents/run.py @@ -69,7 +69,7 @@ def build_aggregate_eval(*, db_path: Path | str) -> dict[str, _Any]: logger = logging.getLogger(__name__) -def _validate_slayer_setup( +def _validate_slayer_setup( # noqa: C901 (complex but linear) *, slayer_setup: str, framework: str, query_mode: str, mode: str, ) -> None: """Reject ``slayer_setup`` combinations the on-the-fly path doesn't @@ -81,6 +81,9 @@ def _validate_slayer_setup( re-raises via ``parser.error`` so the CLI gets the standard argparse exit-2 + stderr behaviour. """ + # Raw frameworks have no SLayer dependency — slayer_setup is irrelevant. + if framework in ("claude_sdk_otf_raw", "claude_sdk_otf_ainteract_raw"): + return # DEV-1462: one-shot REQUIRES on-the-fly. Pre-encoded one-shot would # silently target the committed `slayer_models/` which has no # LiveSQLBench coverage; fail fast. @@ -166,6 +169,14 @@ def _validate_one_shot_framework(*, mode: str, query_mode: str, framework: str) """ if mode != "one-shot": return + # claude_sdk_otf_raw is the raw-SQL one-shot variant; it requires raw, not slayer. + if framework == "claude_sdk_otf_raw": + if query_mode != "raw": + raise ValueError( + "--framework claude_sdk_otf_raw requires --query-mode raw; " + f"got --query-mode {query_mode!r}", + ) + return if query_mode != "slayer": raise ValueError( "--mode one-shot requires --query-mode slayer; " @@ -188,6 +199,8 @@ def _validate_one_shot_framework(*, mode: str, query_mode: str, framework: str) _FRAMEWORK_DATASET_MODE_BINDING = { "claude_sdk_otf": ("livesqlbench", "one-shot"), "claude_sdk_otf_ainteract": ("mini_interact", "a-interact"), + "claude_sdk_otf_raw": ("livesqlbench", "one-shot"), + "claude_sdk_otf_ainteract_raw": ("mini_interact", "a-interact"), } @@ -544,6 +557,52 @@ async def run_one(td: dict, data_dir: str, patience: int, user_sim_model=user_sim_model, ) return run_one + if framework == "claude_sdk_otf_raw": + from bird_interact_agents.agents.claude_sdk_otf_raw import ClaudeSDKOtfRawAgent + + if strict: + logger.warning( + "[claude_sdk_otf_raw] --strict is a no-op for Anthropic models; " + "ignored." + ) + agent_csor = ClaudeSDKOtfRawAgent( + model=agent_model, + reasoning_effort=reasoning_effort, + ) + + async def run_one(td: dict, data_dir: str, patience: int, + user_sim_model: str) -> dict: + budget = calculate_budget(td, patience, mode=mode) + return await agent_csor.run_task( + td, data_dir, budget, query_mode, + eval_mode=mode, + user_sim_model=user_sim_model, + ) + return run_one + if framework == "claude_sdk_otf_ainteract_raw": + from bird_interact_agents.agents.claude_sdk_otf_ainteract_raw import ( + ClaudeSDKOtfAInteractRawAgent, + ) + + if strict: + logger.warning( + "[claude_sdk_otf_ainteract_raw] --strict is a no-op for " + "Anthropic models; ignored." + ) + agent_csoar = ClaudeSDKOtfAInteractRawAgent( + model=agent_model, + reasoning_effort=reasoning_effort, + ) + + async def run_one(td: dict, data_dir: str, patience: int, + user_sim_model: str) -> dict: + budget = calculate_budget(td, patience, mode=mode) + return await agent_csoar.run_task( + td, data_dir, budget, query_mode, + eval_mode=mode, + user_sim_model=user_sim_model, + ) + return run_one if framework == "pydantic_ai": from bird_interact_agents.agents.pydantic_ai.agent import PydanticAIAgent @@ -1242,6 +1301,7 @@ def main() -> None: "--framework", choices=[ "claude_sdk", "claude_sdk_otf", "claude_sdk_otf_ainteract", + "claude_sdk_otf_raw", "claude_sdk_otf_ainteract_raw", "pydantic_ai", "pydantic_ai_recursive", "pydantic_ai_otf_encode", "mcp_agent", "agno", "smolagents", diff --git a/tests/test_claude_sdk_otf_ainteract_raw_agent.py b/tests/test_claude_sdk_otf_ainteract_raw_agent.py new file mode 100644 index 00000000..96e3ec6e --- /dev/null +++ b/tests/test_claude_sdk_otf_ainteract_raw_agent.py @@ -0,0 +1,809 @@ +"""Unit tests for the `claude_sdk_otf_ainteract_raw` agent (no SLayer). + +This is the raw-SQL counterpart to `claude_sdk_otf_ainteract` (which uses +SLayer). It is mini-interact / a-interact only, uses `submit_sql` (not +`submit_query`) gated behind a mandatory `ask_user` call, and has no SLayer +MCP server. The ask-user guards are a parallel factory to the slayer variant +but reference `submit_sql` as the gated tool. +""" + +from __future__ import annotations + +from types import SimpleNamespace + +import pytest + + +# --------------------------------------------------------------------------- +# Construction +# --------------------------------------------------------------------------- + +def test_init_accepts_default(): + from bird_interact_agents.agents.claude_sdk_otf_ainteract_raw.agent import ( + ClaudeSDKOtfAInteractRawAgent, + ) + + agent = ClaudeSDKOtfAInteractRawAgent(model="anthropic/claude-sonnet-4-5") + assert agent.model == "anthropic/claude-sonnet-4-5" + + +def test_init_rejects_bad_reasoning_effort(): + from bird_interact_agents.agents.claude_sdk_otf_ainteract_raw.agent import ( + ClaudeSDKOtfAInteractRawAgent, + ) + + with pytest.raises(ValueError): + ClaudeSDKOtfAInteractRawAgent(reasoning_effort="turbo") + + +# --------------------------------------------------------------------------- +# Tool selection +# --------------------------------------------------------------------------- + +def _tool_names(tools): + return {t.name for t in tools} + + +def test_select_tools_a_interact_returns_nine_native_tools(): + """7 BIRD_INTERACT_TOOLS + submit_sql + ask_user = 9 native tools. + No SLayer tools.""" + from bird_interact_agents.agents.claude_sdk_otf_ainteract_raw import agent as m + + names = _tool_names(m._select_tools("a-interact")) + assert names == { + "execute_sql", + "get_schema", + "get_all_column_meanings", + "get_column_meaning", + "get_all_external_knowledge_names", + "get_knowledge_definition", + "get_all_knowledge_definitions", + "submit_sql", + "ask_user", + } + assert "submit_query" not in names + assert len(names) == 9 + + +def test_select_tools_rejects_unknown_eval_mode(): + from bird_interact_agents.agents.claude_sdk_otf_ainteract_raw import agent as m + + for bad in ("one-shot", "c-interact", "oracle"): + with pytest.raises(ValueError): + m._select_tools(bad) + + +def test_no_slayer_tool_names_function(): + """Raw agent must NOT have a _slayer_tool_names function.""" + from bird_interact_agents.agents.claude_sdk_otf_ainteract_raw import agent as m + + assert not hasattr(m, "_slayer_tool_names") + + +# --------------------------------------------------------------------------- +# Prompt structure +# --------------------------------------------------------------------------- + +def test_build_prompt_is_ainteract_variant(): + from bird_interact_agents.agents.claude_sdk_otf_ainteract_raw import agent as m + + td = {"amb_user_query": "how many widgets?", "selected_database": "shop"} + prompt = m._build_prompt("a-interact", td, budget=20.0) + assert "how many widgets?" in prompt + assert "shop" in prompt + assert "ask_user" in prompt.lower() + # submit_sql, not submit_query + assert "submit_sql" in prompt + assert "submit_query" not in prompt + + +def test_build_prompt_rejects_unknown_eval_mode(): + from bird_interact_agents.agents.claude_sdk_otf_ainteract_raw import agent as m + + td = {"amb_user_query": "?", "selected_database": "shop"} + for bad in ("one-shot", "c-interact", "oracle"): + with pytest.raises(ValueError): + m._build_prompt(bad, td, budget=20.0) + + +def test_prompt_rule_zero_precedes_ask_user_before_sql(): + """Rule 0 (ask_user before SQL) must appear BEFORE the SQL-writing workflow.""" + from bird_interact_agents.agents.claude_sdk_otf_ainteract_raw import prompts as p + + text = p.RAW_OTF_AINTERACT + ask_offset = text.lower().find("ask_user") + sql_offset = text.lower().find("execute_sql") + if sql_offset == -1: + sql_offset = text.lower().find("select ") + assert ask_offset != -1, "prompt must mention ask_user" + assert sql_offset != -1, "prompt must mention SQL-writing guidance" + assert ask_offset < sql_offset, ( + "Rule 0 (ask_user) must precede the SQL-writing guidance" + ) + + +def test_prompt_has_submit_gate_warning(): + from bird_interact_agents.agents.claude_sdk_otf_ainteract_raw import prompts as p + + text = p.RAW_OTF_AINTERACT.lower() + assert "submit" in text + assert any(w in text for w in ("refuse", "deny", "block")), text + + +def test_prompt_absent_slayer_vocab(): + from bird_interact_agents.agents.claude_sdk_otf_ainteract_raw import prompts as p + + text = p.RAW_OTF_AINTERACT + for term in ("submit_query", "create_model", "edit_model", "[kb=", "mcp__slayer__"): + assert term not in text, ( + f"SLayer term {term!r} leaked into raw ainteract prompt" + ) + + +def test_prompts_use_synthetic_examples_only(): + from bird_interact_agents.agents.claude_sdk_otf_ainteract_raw import prompts as p + + banned = [ + "households", "tenure_type", "income_bracket", "dwelling_class", + "socsupport", "service_types", "stellardist", "photo_band", + "taguatinga", + ] + low = p.RAW_OTF_AINTERACT.lower() + for name in banned: + assert name not in low, f"real eval-set name {name!r} leaked into raw prompt" + + +# --------------------------------------------------------------------------- +# Hook factory — pre-submit gate (gated on submit_sql, not submit_query) +# --------------------------------------------------------------------------- + +@pytest.mark.asyncio +async def test_pre_submit_gate_denies_when_ask_count_zero(): + from bird_interact_agents.agents.claude_sdk_otf_ainteract_raw import agent as m + + pre_gate, _counter, _nag = m._make_ask_user_guards() + out = await pre_gate( + {"tool_name": "mcp__bird-interact-tools__submit_sql"}, None, None, + ) + assert set(out) == {"hookSpecificOutput"} + hso = out["hookSpecificOutput"] + assert set(hso) == { + "hookEventName", "permissionDecision", "permissionDecisionReason", + } + assert hso["hookEventName"] == "PreToolUse" + assert hso["permissionDecision"] == "deny" + reason = hso["permissionDecisionReason"] + assert "ask_user" in reason + assert "user-sim" in reason or "user simulator" in reason.lower() + assert "operationalisation" in reason or "operationalization" in reason + + +@pytest.mark.asyncio +async def test_pre_submit_gate_does_not_deny_submit_query(): + """The gate is scoped to submit_sql only — submit_query (if ever called) + must not be denied (it's not the raw submission tool).""" + from bird_interact_agents.agents.claude_sdk_otf_ainteract_raw import agent as m + + pre_gate, _counter, _nag = m._make_ask_user_guards() + out = await pre_gate( + {"tool_name": "mcp__bird-interact-tools__submit_query"}, None, None, + ) + # submit_query is not the guarded tool — gate returns {} (allow / no-op) + assert out == {} + + +@pytest.mark.asyncio +async def test_pre_submit_gate_allows_when_ask_count_positive(): + from bird_interact_agents.agents.claude_sdk_otf_ainteract_raw import agent as m + + pre_gate, counter, _nag = m._make_ask_user_guards() + await counter( + {"tool_name": "mcp__bird-interact-tools__ask_user"}, None, None, + ) + out = await pre_gate( + {"tool_name": "mcp__bird-interact-tools__submit_sql"}, None, None, + ) + assert out == {} + + +# --------------------------------------------------------------------------- +# Hook factory — post nag (same semantics as slayer variant) +# --------------------------------------------------------------------------- + +@pytest.mark.asyncio +async def test_post_nag_quiet_in_first_nine_calls(): + from bird_interact_agents.agents.claude_sdk_otf_ainteract_raw import agent as m + + _gate, _counter, nag = m._make_ask_user_guards() + for _ in range(9): + out = await nag({"tool_name": "execute_sql"}, None, None) + assert out == {} + + +@pytest.mark.asyncio +async def test_post_nag_fires_at_ten(): + from bird_interact_agents.agents.claude_sdk_otf_ainteract_raw import agent as m + + _gate, _counter, nag = m._make_ask_user_guards() + for _ in range(9): + await nag({"tool_name": "execute_sql"}, None, None) + out = await nag({"tool_name": "execute_sql"}, None, None) + assert set(out) == {"hookSpecificOutput"} + hso = out["hookSpecificOutput"] + assert set(hso) == {"hookEventName", "additionalContext"} + assert hso["hookEventName"] == "PostToolUse" + ctx = hso["additionalContext"] + assert "10" in ctx + assert "user-sim" in ctx or "user simulator" in ctx.lower() + + +@pytest.mark.asyncio +async def test_post_nag_silent_after_ask_user(): + from bird_interact_agents.agents.claude_sdk_otf_ainteract_raw import agent as m + + _gate, counter, nag = m._make_ask_user_guards() + await counter({"tool_name": "mcp__bird-interact-tools__ask_user"}, None, None) + out = await nag({"tool_name": "mcp__bird-interact-tools__ask_user"}, None, None) + assert out == {} + for _ in range(30): + out = await nag({"tool_name": "execute_sql"}, None, None) + assert out == {} + + +@pytest.mark.asyncio +async def test_state_isolation_across_factories(): + from bird_interact_agents.agents.claude_sdk_otf_ainteract_raw import agent as m + + gate_a, counter_a, nag_a = m._make_ask_user_guards() + gate_b, counter_b, nag_b = m._make_ask_user_guards() + + for _ in range(5): + await nag_a({"tool_name": "execute_sql"}, None, None) + await counter_b({"tool_name": "mcp__bird-interact-tools__ask_user"}, None, None) + + out_a = await gate_a( + {"tool_name": "mcp__bird-interact-tools__submit_sql"}, None, None, + ) + out_b = await gate_b( + {"tool_name": "mcp__bird-interact-tools__submit_sql"}, None, None, + ) + assert out_a["hookSpecificOutput"]["permissionDecision"] == "deny" + assert out_b == {} + + +# --------------------------------------------------------------------------- +# run_task gating +# --------------------------------------------------------------------------- + +_TASK = { + "selected_database": "shop", + "instance_id": "shop_1", + "amb_user_query": "?", + "knowledge_ambiguity": [], +} + + +@pytest.mark.asyncio +async def test_run_task_rejects_slayer_query_mode(): + from bird_interact_agents.agents.claude_sdk_otf_ainteract_raw.agent import ( + ClaudeSDKOtfAInteractRawAgent, + ) + + agent = ClaudeSDKOtfAInteractRawAgent(model="anthropic/claude-sonnet-4-5") + with pytest.raises(ValueError): + await agent.run_task( + dict(_TASK), "/tmp", 20.0, "slayer", eval_mode="a-interact", + ) + + +@pytest.mark.asyncio +async def test_run_task_rejects_unsupported_eval_modes(): + from bird_interact_agents.agents.claude_sdk_otf_ainteract_raw.agent import ( + ClaudeSDKOtfAInteractRawAgent, + ) + + agent = ClaudeSDKOtfAInteractRawAgent(model="anthropic/claude-sonnet-4-5") + for bad in ("one-shot", "c-interact", "oracle"): + with pytest.raises(ValueError): + await agent.run_task( + dict(_TASK), "/tmp", 20.0, "raw", eval_mode=bad, + ) + + +@pytest.mark.asyncio +async def test_run_task_rejects_livesqlbench_dataset(): + from bird_interact_agents.agents.claude_sdk_otf_ainteract_raw.agent import ( + ClaudeSDKOtfAInteractRawAgent, + ) + + agent = ClaudeSDKOtfAInteractRawAgent(model="anthropic/claude-sonnet-4-5") + td = dict(_TASK, dataset="livesqlbench") + with pytest.raises(ValueError): + await agent.run_task(td, "/tmp", 20.0, "raw", eval_mode="a-interact") + + +@pytest.mark.asyncio +async def test_run_task_accepts_mini_interact_alias(): + """Agent-level dataset gate accepts the mini-interact alias.""" + from bird_interact_agents.agents.claude_sdk_otf_ainteract_raw.agent import ( + ClaudeSDKOtfAInteractRawAgent, + ) + + agent = ClaudeSDKOtfAInteractRawAgent(model="openai/gpt-4o") + td = dict(_TASK, dataset="mini-interact") + row = await agent.run_task(td, "/tmp", 20.0, "raw", eval_mode="a-interact") + assert row["phase1_passed"] is False + assert "anthropic" in (row.get("error") or "").lower() + + +@pytest.mark.asyncio +async def test_run_task_non_anthropic_model_skips(): + from bird_interact_agents.agents.claude_sdk_otf_ainteract_raw.agent import ( + ClaudeSDKOtfAInteractRawAgent, + ) + + agent = ClaudeSDKOtfAInteractRawAgent(model="openai/gpt-4o") + row = await agent.run_task( + dict(_TASK), "/tmp", 20.0, "raw", eval_mode="a-interact", + ) + assert row["phase1_passed"] is False + assert "anthropic" in (row.get("error") or "").lower() + + +# --------------------------------------------------------------------------- +# FakeAssistant / _stub_env helpers +# --------------------------------------------------------------------------- + +class _FakeAssistant: + def __init__(self, in_, out_, cache=0): + self.usage = SimpleNamespace( + input_tokens=in_, output_tokens=out_, cache_read_input_tokens=cache, + ) + + +_FakeAssistant.__name__ = "AssistantMessage" + + +def _make_fake_client( + captured: dict, messages, + *, m_module=None, + prefill_result=None, prefill_timing: str = "after", + raise_after_prefill: Exception | None = None, + prefill_asks: int = 0, +): + class _FakeClient: + def __init__(self, options): + captured["options"] = options + + async def __aenter__(self): + return self + + async def __aexit__(self, *a): + return None + + async def query(self, *a, **kw): + return None + + async def receive_response(self): + if prefill_result is not None and prefill_timing == "before": + m_module._ctx_var.get()["result"] = dict(prefill_result) + for msg in messages: + yield msg + if prefill_asks: + m_module._ctx_var.get()["asks_used"] = prefill_asks + if prefill_result is not None and prefill_timing == "after": + m_module._ctx_var.get()["result"] = dict(prefill_result) + if raise_after_prefill is not None: + raise raise_after_prefill + + return _FakeClient + + +def _stub_env( + monkeypatch, m, storage_dir, + *, + messages=(), captured=None, + prefill_result=None, prefill_timing: str = "after", + raise_after_prefill: Exception | None = None, + prefill_asks: int = 0, +): + from bird_interact_agents import usage as usage_mod + + captured = captured if captured is not None else {} + captured.setdefault("materialize_calls", 0) + monkeypatch.setattr(usage_mod, "_cost_per_token", lambda **_: (0.0, 0.0)) + monkeypatch.setattr(m, "load_db_data_if_needed", lambda *a, **kw: None) + + def _fake_materialize(*a, **kw): + captured["materialize_calls"] += 1 + + monkeypatch.setattr(m, "materialize_task_db", _fake_materialize) + monkeypatch.setattr(m, "create_sdk_mcp_server", lambda **kw: SimpleNamespace()) + monkeypatch.setattr( + m, "ClaudeSDKClient", + _make_fake_client( + captured, messages, + m_module=m, + prefill_result=prefill_result, + prefill_timing=prefill_timing, + raise_after_prefill=raise_after_prefill, + prefill_asks=prefill_asks, + ), + ) + return captured + + +# --------------------------------------------------------------------------- +# Storage path + ClaudeAgentOptions +# --------------------------------------------------------------------------- + +@pytest.mark.asyncio +async def test_run_task_does_not_call_slayer_mcp(monkeypatch, tmp_path): + """Raw ainteract agent has no SLayer MCP server.""" + from bird_interact_agents.agents.claude_sdk_otf_ainteract_raw import agent as m + + assert not hasattr(m, "slayer_mcp_stdio_config"), ( + "raw ainteract agent must not import slayer_mcp_stdio_config" + ) + assert not hasattr(m, "resolve_otf_task_storage_dir"), ( + "raw ainteract agent must not import resolve_otf_task_storage_dir" + ) + + +@pytest.mark.asyncio +async def test_run_task_does_not_whitelist_slayer_tools(monkeypatch, tmp_path): + from bird_interact_agents.agents.claude_sdk_otf_ainteract_raw import agent as m + + captured = _stub_env(monkeypatch, m, tmp_path / "store") + agent = m.ClaudeSDKOtfAInteractRawAgent(model="anthropic/claude-sonnet-4-5") + await agent.run_task( + dict(_TASK), str(tmp_path), 20.0, "raw", eval_mode="a-interact", + ) + allowed = set(captured["options"].allowed_tools) + assert not any(t.startswith("mcp__slayer__") for t in allowed) + + +@pytest.mark.asyncio +async def test_run_task_whitelists_ask_user_and_submit_sql(monkeypatch, tmp_path): + from bird_interact_agents.agents.claude_sdk_otf_ainteract_raw import agent as m + + captured = _stub_env(monkeypatch, m, tmp_path / "store") + agent = m.ClaudeSDKOtfAInteractRawAgent(model="anthropic/claude-sonnet-4-5") + await agent.run_task( + dict(_TASK), str(tmp_path), 20.0, "raw", eval_mode="a-interact", + ) + allowed = set(captured["options"].allowed_tools) + assert "mcp__bird-interact-tools__ask_user" in allowed + assert "mcp__bird-interact-tools__submit_sql" in allowed + assert "mcp__bird-interact-tools__submit_query" not in allowed + + +@pytest.mark.asyncio +async def test_run_task_registers_three_guards_plus_turn_budget(monkeypatch, tmp_path): + """PreToolUse gate scoped to submit_sql; PostToolUse: ask-counter, + nag, turn-budget.""" + from bird_interact_agents.agents.claude_sdk_otf_ainteract_raw import agent as m + + captured = _stub_env(monkeypatch, m, tmp_path / "store") + agent = m.ClaudeSDKOtfAInteractRawAgent(model="anthropic/claude-sonnet-4-5") + await agent.run_task( + dict(_TASK), str(tmp_path), 20.0, "raw", eval_mode="a-interact", + ) + hooks = captured["options"].hooks + assert "PreToolUse" in hooks + assert "PostToolUse" in hooks + + pre_matchers = hooks["PreToolUse"] + assert len(pre_matchers) == 1 + # Gate must be scoped to submit_sql, not submit_query. + assert pre_matchers[0].matcher == "mcp__bird-interact-tools__submit_sql" + + post_matchers = hooks["PostToolUse"] + assert len(post_matchers) == 3 + matchers = {pm.matcher for pm in post_matchers} + assert "mcp__bird-interact-tools__ask_user" in matchers + assert None in matchers + + +@pytest.mark.asyncio +async def test_run_task_restricts_tools_and_caps_turns(monkeypatch, tmp_path): + from bird_interact_agents.agents.claude_sdk_otf_ainteract_raw import agent as m + from bird_interact_agents.harness import MAX_MODEL_TURNS + + captured = _stub_env(monkeypatch, m, tmp_path / "store") + agent = m.ClaudeSDKOtfAInteractRawAgent(model="anthropic/claude-sonnet-4-5") + await agent.run_task( + dict(_TASK), str(tmp_path), 20.0, "raw", eval_mode="a-interact", + ) + opts = captured["options"] + assert opts.tools == [] + assert opts.setting_sources == [] + assert opts.max_turns == 2 * MAX_MODEL_TURNS + + +@pytest.mark.asyncio +async def test_run_task_invokes_factory_per_call(monkeypatch, tmp_path): + """Hook-state factory must be invoked inside run_task (per task), + not on the agent constructor.""" + from bird_interact_agents.agents.claude_sdk_otf_ainteract_raw import agent as m + + call_count = [0] + real_factory = m._make_ask_user_guards + + def _spy_factory(): + call_count[0] += 1 + return real_factory() + + captured = _stub_env(monkeypatch, m, tmp_path / "store") + monkeypatch.setattr(m, "_make_ask_user_guards", _spy_factory) + + agent = m.ClaudeSDKOtfAInteractRawAgent(model="anthropic/claude-sonnet-4-5") + await agent.run_task( + dict(_TASK), str(tmp_path), 20.0, "raw", eval_mode="a-interact", + ) + assert call_count[0] == 1 + + # Second invocation — factory must be called again, state fresh. + await agent.run_task( + dict(_TASK), str(tmp_path), 20.0, "raw", eval_mode="a-interact", + ) + assert call_count[0] == 2, ( + "factory must be invoked per run_task call — state leaked across tasks" + ) + second_pre = captured["options"].hooks["PreToolUse"][0].hooks[0] + out = await second_pre( + {"tool_name": "mcp__bird-interact-tools__submit_sql"}, None, None, + ) + assert out["hookSpecificOutput"]["permissionDecision"] == "deny" + + +@pytest.mark.asyncio +async def test_run_task_pins_requested_model(monkeypatch, tmp_path): + from bird_interact_agents.agents.claude_sdk_otf_ainteract_raw import agent as m + + captured = _stub_env(monkeypatch, m, tmp_path / "store") + agent = m.ClaudeSDKOtfAInteractRawAgent(model="anthropic/claude-opus-4-7") + await agent.run_task( + dict(_TASK), str(tmp_path), 20.0, "raw", eval_mode="a-interact", + ) + assert captured["options"].model == "claude-opus-4-7" + + +@pytest.mark.asyncio +async def test_run_task_passes_reasoning_effort(monkeypatch, tmp_path): + from bird_interact_agents.agents.claude_sdk_otf_ainteract_raw import agent as m + + captured = _stub_env(monkeypatch, m, tmp_path / "store") + agent = m.ClaudeSDKOtfAInteractRawAgent( + model="anthropic/claude-sonnet-4-5", reasoning_effort="high", + ) + await agent.run_task( + dict(_TASK), str(tmp_path), 20.0, "raw", eval_mode="a-interact", + ) + assert captured["options"].effort == "high" + + +@pytest.mark.asyncio +async def test_run_task_default_effort_is_none(monkeypatch, tmp_path): + from bird_interact_agents.agents.claude_sdk_otf_ainteract_raw import agent as m + + captured = _stub_env(monkeypatch, m, tmp_path / "store") + agent = m.ClaudeSDKOtfAInteractRawAgent(model="anthropic/claude-sonnet-4-5") + await agent.run_task( + dict(_TASK), str(tmp_path), 20.0, "raw", eval_mode="a-interact", + ) + assert captured["options"].effort is None + + +@pytest.mark.asyncio +async def test_run_task_captures_usage(monkeypatch, tmp_path): + from bird_interact_agents.agents.claude_sdk_otf_ainteract_raw import agent as m + from bird_interact_agents import usage as usage_mod + + msgs = [_FakeAssistant(100, 20), _FakeAssistant(150, 30, cache=5)] + _stub_env(monkeypatch, m, tmp_path / "store", messages=msgs) + agent = m.ClaudeSDKOtfAInteractRawAgent(model="anthropic/claude-sonnet-4-5") + row = await agent.run_task( + dict(_TASK), str(tmp_path), 20.0, "raw", eval_mode="a-interact", + ) + rebuilt = usage_mod.TokenUsage.model_validate(row["usage"]) + assert rebuilt.prompt_tokens == 250 + assert rebuilt.completion_tokens == 50 + assert rebuilt.cache_read_tokens == 5 + + +# --------------------------------------------------------------------------- +# n_ask_user_calls reporting (DEV-1519 parity for raw ainteract) +# --------------------------------------------------------------------------- + +@pytest.mark.asyncio +async def test_run_task_writes_n_ask_user_calls_zero(monkeypatch, tmp_path): + from bird_interact_agents.agents.claude_sdk_otf_ainteract_raw import agent as m + + _stub_env( + monkeypatch, m, tmp_path / "store", + messages=[_FakeAssistant(100, 20)], + prefill_asks=0, + ) + agent = m.ClaudeSDKOtfAInteractRawAgent(model="anthropic/claude-sonnet-4-5") + row = await agent.run_task( + dict(_TASK), str(tmp_path), 20.0, "raw", eval_mode="a-interact", + ) + assert row["usage"]["n_ask_user_calls"] == 0 + + +@pytest.mark.asyncio +async def test_run_task_writes_n_ask_user_calls_nonzero(monkeypatch, tmp_path): + from bird_interact_agents.agents.claude_sdk_otf_ainteract_raw import agent as m + + _stub_env( + monkeypatch, m, tmp_path / "store", + messages=[_FakeAssistant(100, 20)], + prefill_asks=3, + ) + agent = m.ClaudeSDKOtfAInteractRawAgent(model="anthropic/claude-sonnet-4-5") + row = await agent.run_task( + dict(_TASK), str(tmp_path), 20.0, "raw", eval_mode="a-interact", + ) + assert row["usage"]["n_ask_user_calls"] == 3 + + +@pytest.mark.asyncio +async def test_run_task_exception_path_writes_n_ask_user_calls(monkeypatch, tmp_path): + from bird_interact_agents.agents.claude_sdk_otf_ainteract_raw import agent as m + + _stub_env( + monkeypatch, m, tmp_path / "store", + messages=[_FakeAssistant(50, 10)], + prefill_asks=2, + raise_after_prefill=RuntimeError("boom"), + ) + agent = m.ClaudeSDKOtfAInteractRawAgent(model="anthropic/claude-sonnet-4-5") + row = await agent.run_task( + dict(_TASK), str(tmp_path), 20.0, "raw", eval_mode="a-interact", + ) + assert "boom" in (row.get("error") or "") + assert row["usage"]["n_ask_user_calls"] == 2 + + +# --------------------------------------------------------------------------- +# DEV-1511: diagnostic-field propagation +# --------------------------------------------------------------------------- + +def _full_prefill(**overrides): + base = { + "submission_status": "submitted_ok", + "predicted_result_json": "[{\"a\": 1}]", + "gold_result_json": "[{\"a\": 1}]", + "phase1_observation": "PASS", + "phase1_passed": True, + "phase2_passed": False, + "total_reward": 1.0, + "submitted_sql": "SELECT 1", + "submitted_query": None, + } + base.update(overrides) + return base + + +@pytest.mark.asyncio +async def test_run_task_propagates_diagnostic_fields_on_happy_path( + monkeypatch, tmp_path, +): + from bird_interact_agents.agents.claude_sdk_otf_ainteract_raw import agent as m + + _stub_env( + monkeypatch, m, tmp_path / "store", + messages=[_FakeAssistant(100, 20)], + prefill_result=_full_prefill(), + prefill_timing="after", + ) + agent = m.ClaudeSDKOtfAInteractRawAgent(model="anthropic/claude-sonnet-4-5") + row = await agent.run_task( + dict(_TASK), str(tmp_path), 20.0, "raw", eval_mode="a-interact", + ) + assert row["submission_status"] == "submitted_ok" + assert row["predicted_result_json"] == "[{\"a\": 1}]" + assert row["phase1_observation"] == "PASS" + assert "phase2_observation" in row + assert row["phase2_observation"] is None + assert row["phase1_passed"] is True + assert row["submitted_sql"] == "SELECT 1" + assert row["error"] is None + + +@pytest.mark.asyncio +async def test_run_task_propagation_defaults_to_none_when_never_submitted( + monkeypatch, tmp_path, +): + from bird_interact_agents.agents.claude_sdk_otf_ainteract_raw import agent as m + + _stub_env( + monkeypatch, m, tmp_path / "store", + messages=[_FakeAssistant(100, 20)], + ) + agent = m.ClaudeSDKOtfAInteractRawAgent(model="anthropic/claude-sonnet-4-5") + row = await agent.run_task( + dict(_TASK), str(tmp_path), 20.0, "raw", eval_mode="a-interact", + ) + assert row["submission_status"] is None + assert row["predicted_result_json"] is None + assert row["phase1_observation"] is None + assert row["phase2_observation"] is None + + +@pytest.mark.asyncio +async def test_run_task_exception_path_propagates_partial_result( + monkeypatch, tmp_path, +): + from bird_interact_agents.agents.claude_sdk_otf_ainteract_raw import agent as m + + prefill = _full_prefill( + phase2_passed=True, total_reward=0.75, phase2_observation="p2 ok", + ) + _stub_env( + monkeypatch, m, tmp_path / "store", + messages=[_FakeAssistant(100, 20)], + prefill_result=prefill, + prefill_timing="after", + raise_after_prefill=RuntimeError("boom"), + ) + agent = m.ClaudeSDKOtfAInteractRawAgent(model="anthropic/claude-sonnet-4-5") + row = await agent.run_task( + dict(_TASK), str(tmp_path), 20.0, "raw", eval_mode="a-interact", + ) + assert row["error"] == "boom" + assert row["submission_status"] == "submitted_ok" + assert row["phase1_passed"] is True + assert row["phase2_observation"] == "p2 ok" + assert row["submitted_sql"] == "SELECT 1" + + +@pytest.mark.asyncio +async def test_run_task_exception_before_ctx_set_yields_empty_diagnostics( + monkeypatch, tmp_path, +): + from bird_interact_agents.agents.claude_sdk_otf_ainteract_raw import agent as m + + _stub_env(monkeypatch, m, tmp_path / "store") + + def _boom(*a, **kw): + raise RuntimeError("early-setup boom") + + monkeypatch.setattr(m, "load_db_data_if_needed", _boom) + agent = m.ClaudeSDKOtfAInteractRawAgent(model="anthropic/claude-sonnet-4-5") + row = await agent.run_task( + dict(_TASK), str(tmp_path), 20.0, "raw", eval_mode="a-interact", + ) + assert "early-setup boom" in (row.get("error") or "") + assert row["submission_status"] is None + assert row["phase1_passed"] is False + + +@pytest.mark.asyncio +async def test_run_task_exception_path_isolated_from_stale_context( + monkeypatch, tmp_path, +): + from bird_interact_agents.agents.claude_sdk_otf_ainteract_raw import agent as m + + _stub_env(monkeypatch, m, tmp_path / "store") + m._ctx_var.set({ + "result": { + "submission_status": "STALE_SHOULD_NOT_LEAK", + "phase1_passed": True, + "predicted_result_json": "STALE", + }, + }) + + def _boom(*a, **kw): + raise RuntimeError("early boom") + + monkeypatch.setattr(m, "load_db_data_if_needed", _boom) + agent = m.ClaudeSDKOtfAInteractRawAgent(model="anthropic/claude-sonnet-4-5") + row = await agent.run_task( + dict(_TASK), str(tmp_path), 20.0, "raw", eval_mode="a-interact", + ) + assert "early boom" in (row.get("error") or "") + assert row["submission_status"] != "STALE_SHOULD_NOT_LEAK" + assert row["submission_status"] is None + assert row["phase1_passed"] is False diff --git a/tests/test_claude_sdk_otf_raw_agent.py b/tests/test_claude_sdk_otf_raw_agent.py new file mode 100644 index 00000000..fd0f61fc --- /dev/null +++ b/tests/test_claude_sdk_otf_raw_agent.py @@ -0,0 +1,617 @@ +"""Unit tests for the `claude_sdk_otf_raw` agent (no LLM, no SLayer). + +This is the raw-SQL counterpart to `claude_sdk_otf` (which uses SLayer). +It is livesqlbench / one-shot only, uses `submit_sql` (not `submit_query`), +and has no SLayer MCP server. The prompts share constants from +`_shared_otf_prompts` with the slayer OTF variant for a fair comparison. +""" + +from __future__ import annotations + +from types import SimpleNamespace + +import pytest + + +# --------------------------------------------------------------------------- +# Construction +# --------------------------------------------------------------------------- + +def test_init_accepts_default(): + from bird_interact_agents.agents.claude_sdk_otf_raw.agent import ( + ClaudeSDKOtfRawAgent, + ) + + agent = ClaudeSDKOtfRawAgent(model="anthropic/claude-sonnet-4-5") + assert agent.model == "anthropic/claude-sonnet-4-5" + + +def test_init_rejects_bad_reasoning_effort(): + from bird_interact_agents.agents.claude_sdk_otf_raw.agent import ( + ClaudeSDKOtfRawAgent, + ) + + with pytest.raises(ValueError): + ClaudeSDKOtfRawAgent(reasoning_effort="turbo") + + +# --------------------------------------------------------------------------- +# Tool selection +# --------------------------------------------------------------------------- + +def _tool_names(tools): + return {t.name for t in tools} + + +def test_select_tools_one_shot_returns_eight_native_tools(): + """All 7 raw BIRD_INTERACT_TOOLS + submit_sql = 8 native tools. + No ask_user, no SLayer tools.""" + from bird_interact_agents.agents.claude_sdk_otf_raw import agent as m + + names = _tool_names(m._select_tools("one-shot")) + assert names == { + "execute_sql", + "get_schema", + "get_all_column_meanings", + "get_column_meaning", + "get_all_external_knowledge_names", + "get_knowledge_definition", + "get_all_knowledge_definitions", + "submit_sql", + } + assert "ask_user" not in names + assert "submit_query" not in names + assert len(names) == 8 + + +def test_select_tools_rejects_non_one_shot(): + from bird_interact_agents.agents.claude_sdk_otf_raw import agent as m + + for bad in ("a-interact", "c-interact", "oracle"): + with pytest.raises(ValueError): + m._select_tools(bad) + + +def test_no_slayer_tool_names_function(): + """Raw agent must NOT have a _slayer_tool_names function + (there is no SLayer MCP server).""" + from bird_interact_agents.agents.claude_sdk_otf_raw import agent as m + + assert not hasattr(m, "_slayer_tool_names") + + +# --------------------------------------------------------------------------- +# Prompt selection + hygiene +# --------------------------------------------------------------------------- + +def test_build_prompt_uses_raw_one_shot_template(): + from bird_interact_agents.agents.claude_sdk_otf_raw import agent as m + from bird_interact_agents.agents.claude_sdk_otf_raw import prompts as p + + td = {"amb_user_query": "how many widgets?", "selected_database": "shop"} + out = m._build_prompt("one-shot", td, budget=20.0) + assert "how many widgets?" in out + assert "shop" in out + # No ask_user in one-shot. + assert "ask_user" not in out.lower() + # submit_sql, not submit_query. + assert "submit_sql" in out + assert "submit_query" not in out + assert p.RAW_OTF_ONE_SHOT + + +def test_build_prompt_rejects_non_one_shot(): + from bird_interact_agents.agents.claude_sdk_otf_raw import agent as m + + td = {"amb_user_query": "?", "selected_database": "shop"} + for bad in ("a-interact", "c-interact", "oracle"): + with pytest.raises(ValueError): + m._build_prompt(bad, td, budget=20.0) + + +def test_prompt_absent_slayer_vocab(): + """Raw prompt must not mention SLayer-specific concepts.""" + from bird_interact_agents.agents.claude_sdk_otf_raw import prompts as p + + text = p.RAW_OTF_ONE_SHOT + for term in ("submit_query", "create_model", "edit_model", "[kb=", "mcp__slayer__"): + assert term not in text, f"SLayer term {term!r} leaked into raw one-shot prompt" + + +def test_prompt_has_execute_sql_and_get_schema(): + """Raw prompt must mention the DB-exploration tools.""" + from bird_interact_agents.agents.claude_sdk_otf_raw import prompts as p + + low = p.RAW_OTF_ONE_SHOT.lower() + for tool_name in ("execute_sql", "get_schema"): + assert tool_name in low or "schema" in low, ( + f"raw one-shot prompt should guide agent to use {tool_name}" + ) + + +def test_prompts_use_synthetic_examples_only(): + """Guards feedback_prompts_synthetic_examples_only.""" + from bird_interact_agents.agents.claude_sdk_otf_raw import prompts as p + + banned = [ + "households", "tenure_type", "income_bracket", "dwelling_class", + "socsupport", "service_types", "stellardist", "photo_band", + "taguatinga", + ] + low = p.RAW_OTF_ONE_SHOT.lower() + for name in banned: + assert name not in low, f"real eval-set name {name!r} leaked into raw prompt" + + +# --------------------------------------------------------------------------- +# run_task gating +# --------------------------------------------------------------------------- + +_TASK = { + "selected_database": "shop", + "instance_id": "shop_1", + "amb_user_query": "?", + "knowledge_ambiguity": [], + "dataset": "livesqlbench", +} + + +@pytest.mark.asyncio +async def test_run_task_rejects_slayer_query_mode(): + from bird_interact_agents.agents.claude_sdk_otf_raw.agent import ClaudeSDKOtfRawAgent + + agent = ClaudeSDKOtfRawAgent(model="anthropic/claude-sonnet-4-5") + with pytest.raises(ValueError): + await agent.run_task(dict(_TASK), "/tmp", 20.0, "slayer", eval_mode="one-shot") + + +@pytest.mark.asyncio +async def test_run_task_rejects_unsupported_eval_modes(): + from bird_interact_agents.agents.claude_sdk_otf_raw.agent import ClaudeSDKOtfRawAgent + + agent = ClaudeSDKOtfRawAgent(model="anthropic/claude-sonnet-4-5") + for bad in ("a-interact", "c-interact", "oracle"): + with pytest.raises(ValueError): + await agent.run_task(dict(_TASK), "/tmp", 20.0, "raw", eval_mode=bad) + + +@pytest.mark.asyncio +async def test_run_task_rejects_mini_interact_dataset(): + from bird_interact_agents.agents.claude_sdk_otf_raw.agent import ClaudeSDKOtfRawAgent + + agent = ClaudeSDKOtfRawAgent(model="anthropic/claude-sonnet-4-5") + td = dict(_TASK, dataset="mini_interact") + with pytest.raises(ValueError): + await agent.run_task(td, "/tmp", 20.0, "raw", eval_mode="one-shot") + + +@pytest.mark.asyncio +async def test_run_task_non_anthropic_model_skips(): + from bird_interact_agents.agents.claude_sdk_otf_raw.agent import ClaudeSDKOtfRawAgent + + agent = ClaudeSDKOtfRawAgent(model="openai/gpt-4o") + row = await agent.run_task( + dict(_TASK), "/tmp", 20.0, "raw", eval_mode="one-shot", + ) + assert row["phase1_passed"] is False + assert "anthropic" in (row.get("error") or "").lower() + + +@pytest.mark.asyncio +async def test_run_task_accepts_livesqlbench_alias(): + """Agent-level dataset gate must accept the canonical token.""" + from bird_interact_agents.agents.claude_sdk_otf_raw.agent import ClaudeSDKOtfRawAgent + + agent = ClaudeSDKOtfRawAgent(model="openai/gpt-4o") + td = dict(_TASK, dataset="livesqlbench") + row = await agent.run_task(td, "/tmp", 20.0, "raw", eval_mode="one-shot") + assert row["phase1_passed"] is False + assert "anthropic" in (row.get("error") or "").lower() + + +# --------------------------------------------------------------------------- +# FakeAssistant / _stub_env helpers +# --------------------------------------------------------------------------- + +class _FakeAssistant: + def __init__(self, in_, out_, cache=0): + self.usage = SimpleNamespace( + input_tokens=in_, output_tokens=out_, cache_read_input_tokens=cache, + ) + + +_FakeAssistant.__name__ = "AssistantMessage" + + +def _make_fake_client( + captured: dict, + messages, + *, + m_module, + prefill_result=None, + prefill_timing: str = "after", + raise_after_prefill: Exception | None = None, +): + class _FakeClient: + def __init__(self, options): + captured["options"] = options + + async def __aenter__(self): + return self + + async def __aexit__(self, *a): + return None + + async def query(self, *a, **kw): + return None + + async def receive_response(self): + if prefill_result is not None and prefill_timing == "before": + m_module._ctx_var.get()["result"] = dict(prefill_result) + for msg in messages: + yield msg + if prefill_result is not None and prefill_timing == "after": + m_module._ctx_var.get()["result"] = dict(prefill_result) + if raise_after_prefill is not None: + raise raise_after_prefill + + return _FakeClient + + +def _stub_env( + monkeypatch, m, storage_dir, + *, + messages=(), captured=None, + prefill_result=None, + prefill_timing: str = "after", + raise_after_prefill: Exception | None = None, +): + from bird_interact_agents import usage as usage_mod + + captured = captured if captured is not None else {} + captured.setdefault("materialize_calls", 0) + monkeypatch.setattr(usage_mod, "_cost_per_token", lambda **_: (0.0, 0.0)) + monkeypatch.setattr(m, "load_db_data_if_needed", lambda *a, **kw: None) + + def _fake_materialize(*a, **kw): + captured["materialize_calls"] += 1 + + monkeypatch.setattr(m, "materialize_task_db", _fake_materialize) + monkeypatch.setattr(m, "create_sdk_mcp_server", lambda **kw: SimpleNamespace()) + monkeypatch.setattr( + m, "ClaudeSDKClient", + _make_fake_client( + captured, messages, + m_module=m, + prefill_result=prefill_result, + prefill_timing=prefill_timing, + raise_after_prefill=raise_after_prefill, + ), + ) + return captured + + +# --------------------------------------------------------------------------- +# Storage path + allowed tools +# --------------------------------------------------------------------------- + +@pytest.mark.asyncio +async def test_run_task_does_not_call_slayer_mcp(monkeypatch, tmp_path): + """Raw agent has no SLayer MCP server — `slayer_mcp_stdio_config` must + not be imported or called.""" + from bird_interact_agents.agents.claude_sdk_otf_raw import agent as m + + assert not hasattr(m, "slayer_mcp_stdio_config"), ( + "raw agent must not import slayer_mcp_stdio_config" + ) + assert not hasattr(m, "resolve_otf_task_storage_dir"), ( + "raw agent must not import resolve_otf_task_storage_dir" + ) + + +@pytest.mark.asyncio +async def test_run_task_does_not_whitelist_slayer_tools(monkeypatch, tmp_path): + """The ClaudeAgentOptions must not whitelist any mcp__slayer__* tool.""" + from bird_interact_agents.agents.claude_sdk_otf_raw import agent as m + + captured = _stub_env(monkeypatch, m, tmp_path / "store") + agent = m.ClaudeSDKOtfRawAgent(model="anthropic/claude-sonnet-4-5") + await agent.run_task( + dict(_TASK), str(tmp_path), 20.0, "raw", eval_mode="one-shot", + ) + allowed = set(captured["options"].allowed_tools) + assert not any(t.startswith("mcp__slayer__") for t in allowed), ( + "raw agent must not whitelist any mcp__slayer__* tool" + ) + + +@pytest.mark.asyncio +async def test_run_task_whitelists_submit_sql_not_submit_query(monkeypatch, tmp_path): + from bird_interact_agents.agents.claude_sdk_otf_raw import agent as m + + captured = _stub_env(monkeypatch, m, tmp_path / "store") + agent = m.ClaudeSDKOtfRawAgent(model="anthropic/claude-sonnet-4-5") + await agent.run_task( + dict(_TASK), str(tmp_path), 20.0, "raw", eval_mode="one-shot", + ) + allowed = set(captured["options"].allowed_tools) + assert "mcp__bird-interact-tools__submit_sql" in allowed + assert "mcp__bird-interact-tools__submit_query" not in allowed + assert "mcp__bird-interact-tools__ask_user" not in allowed + + +@pytest.mark.asyncio +async def test_run_task_whitelists_all_raw_tools(monkeypatch, tmp_path): + """All 7 BIRD_INTERACT_TOOLS + submit_sql must appear on the allow-list.""" + from bird_interact_agents.agents.claude_sdk_otf_raw import agent as m + + captured = _stub_env(monkeypatch, m, tmp_path / "store") + agent = m.ClaudeSDKOtfRawAgent(model="anthropic/claude-sonnet-4-5") + await agent.run_task( + dict(_TASK), str(tmp_path), 20.0, "raw", eval_mode="one-shot", + ) + allowed = set(captured["options"].allowed_tools) + for tool_name in ( + "execute_sql", "get_schema", "get_all_column_meanings", "get_column_meaning", + "get_all_external_knowledge_names", "get_knowledge_definition", + "get_all_knowledge_definitions", "submit_sql", + ): + assert f"mcp__bird-interact-tools__{tool_name}" in allowed, ( + f"raw agent must whitelist {tool_name}" + ) + + +@pytest.mark.asyncio +async def test_run_task_restricts_tools_and_caps_turns(monkeypatch, tmp_path): + """No Claude Code built-ins; isolated settings; max_turns = 2× MAX_MODEL_TURNS.""" + from bird_interact_agents.agents.claude_sdk_otf_raw import agent as m + from bird_interact_agents.harness import MAX_MODEL_TURNS + + captured = _stub_env(monkeypatch, m, tmp_path / "store") + agent = m.ClaudeSDKOtfRawAgent(model="anthropic/claude-sonnet-4-5") + await agent.run_task( + dict(_TASK), str(tmp_path), 20.0, "raw", eval_mode="one-shot", + ) + opts = captured["options"] + assert opts.tools == [] + assert opts.setting_sources == [] + assert opts.max_turns == 2 * MAX_MODEL_TURNS + assert "PostToolUse" in (opts.hooks or {}) + + +@pytest.mark.asyncio +async def test_run_task_one_shot_livesqlbench_calls_materialize(monkeypatch, tmp_path): + """LiveSQLBench one-shot: per-task DB isolation via materialize_task_db.""" + from bird_interact_agents.agents.claude_sdk_otf_raw import agent as m + + captured = _stub_env(monkeypatch, m, tmp_path / "store") + agent = m.ClaudeSDKOtfRawAgent(model="anthropic/claude-sonnet-4-5") + await agent.run_task( + dict(_TASK), str(tmp_path), 20.0, "raw", eval_mode="one-shot", + ) + assert captured["materialize_calls"] == 1 + + +@pytest.mark.asyncio +async def test_run_task_pins_requested_model(monkeypatch, tmp_path): + from bird_interact_agents.agents.claude_sdk_otf_raw import agent as m + + captured = _stub_env(monkeypatch, m, tmp_path / "store") + agent = m.ClaudeSDKOtfRawAgent(model="anthropic/claude-opus-4-7") + await agent.run_task( + dict(_TASK), str(tmp_path), 20.0, "raw", eval_mode="one-shot", + ) + assert captured["options"].model == "claude-opus-4-7" + + +@pytest.mark.asyncio +async def test_run_task_passes_reasoning_effort(monkeypatch, tmp_path): + from bird_interact_agents.agents.claude_sdk_otf_raw import agent as m + + captured = _stub_env(monkeypatch, m, tmp_path / "store") + agent = m.ClaudeSDKOtfRawAgent( + model="anthropic/claude-sonnet-4-5", reasoning_effort="high", + ) + await agent.run_task( + dict(_TASK), str(tmp_path), 20.0, "raw", eval_mode="one-shot", + ) + assert captured["options"].effort == "high" + + +@pytest.mark.asyncio +async def test_run_task_default_effort_is_none(monkeypatch, tmp_path): + from bird_interact_agents.agents.claude_sdk_otf_raw import agent as m + + captured = _stub_env(monkeypatch, m, tmp_path / "store") + agent = m.ClaudeSDKOtfRawAgent(model="anthropic/claude-sonnet-4-5") + await agent.run_task( + dict(_TASK), str(tmp_path), 20.0, "raw", eval_mode="one-shot", + ) + assert captured["options"].effort is None + + +# --------------------------------------------------------------------------- +# Turn budget hook — must say "submit_sql" not "submit_query" +# --------------------------------------------------------------------------- + +@pytest.mark.asyncio +async def test_turn_budget_hook_warns_near_cap_with_submit_sql(): + from bird_interact_agents.agents.claude_sdk_otf_raw import agent as m + + hook = m._make_turn_budget_hook(max_turns=5, warn_within=3) + assert await hook({}, None, None) == {} # call 1 -> 4 left + out = await hook({}, None, None) # call 2 -> 3 left -> warn + ctx = out["hookSpecificOutput"]["additionalContext"] + assert "submit_sql" in ctx, "turn budget hook must say 'submit_sql' not 'submit_query'" + assert "submit_query" not in ctx + assert "3" in ctx + + +# --------------------------------------------------------------------------- +# Usage accumulation +# --------------------------------------------------------------------------- + +@pytest.mark.asyncio +async def test_run_task_captures_usage(monkeypatch, tmp_path): + from bird_interact_agents.agents.claude_sdk_otf_raw import agent as m + from bird_interact_agents import usage as usage_mod + + msgs = [_FakeAssistant(100, 20), _FakeAssistant(150, 30, cache=5)] + _stub_env(monkeypatch, m, tmp_path / "store", messages=msgs) + agent = m.ClaudeSDKOtfRawAgent(model="anthropic/claude-sonnet-4-5") + row = await agent.run_task( + dict(_TASK), str(tmp_path), 20.0, "raw", eval_mode="one-shot", + ) + rebuilt = usage_mod.TokenUsage.model_validate(row["usage"]) + assert rebuilt.prompt_tokens == 250 + assert rebuilt.completion_tokens == 50 + assert rebuilt.cache_read_tokens == 5 + + +# --------------------------------------------------------------------------- +# DEV-1511: diagnostic-field propagation +# --------------------------------------------------------------------------- + +def _full_prefill(**overrides): + base = { + "submission_status": "submitted_ok", + "predicted_result_json": "[{\"a\": 1}]", + "gold_result_json": "[{\"a\": 1}]", + "phase1_observation": "PASS", + "phase1_passed": True, + "phase2_passed": False, + "total_reward": 1.0, + "submitted_sql": "SELECT 1", + "submitted_query": None, + } + base.update(overrides) + return base + + +@pytest.mark.asyncio +async def test_run_task_propagates_diagnostic_fields_on_happy_path( + monkeypatch, tmp_path, +): + from bird_interact_agents.agents.claude_sdk_otf_raw import agent as m + + _stub_env( + monkeypatch, m, tmp_path / "store", + messages=[_FakeAssistant(100, 20)], + prefill_result=_full_prefill(), + prefill_timing="after", + ) + agent = m.ClaudeSDKOtfRawAgent(model="anthropic/claude-sonnet-4-5") + row = await agent.run_task( + dict(_TASK), str(tmp_path), 20.0, "raw", eval_mode="one-shot", + ) + assert row["submission_status"] == "submitted_ok" + assert row["predicted_result_json"] == "[{\"a\": 1}]" + assert row["gold_result_json"] == "[{\"a\": 1}]" + assert row["phase1_observation"] == "PASS" + assert "phase2_observation" in row + assert row["phase2_observation"] is None + assert row["phase1_passed"] is True + assert row["submitted_sql"] == "SELECT 1" + assert row["error"] is None + + +@pytest.mark.asyncio +async def test_run_task_propagation_defaults_to_none_when_never_submitted( + monkeypatch, tmp_path, +): + from bird_interact_agents.agents.claude_sdk_otf_raw import agent as m + + _stub_env( + monkeypatch, m, tmp_path / "store", + messages=[_FakeAssistant(100, 20)], + ) + agent = m.ClaudeSDKOtfRawAgent(model="anthropic/claude-sonnet-4-5") + row = await agent.run_task( + dict(_TASK), str(tmp_path), 20.0, "raw", eval_mode="one-shot", + ) + assert row["submission_status"] is None + assert row["predicted_result_json"] is None + assert row["gold_result_json"] is None + assert row["phase1_observation"] is None + assert row["phase2_observation"] is None + + +@pytest.mark.asyncio +async def test_run_task_exception_path_propagates_partial_result( + monkeypatch, tmp_path, +): + from bird_interact_agents.agents.claude_sdk_otf_raw import agent as m + + prefill = _full_prefill( + submission_status="submitted_ok", phase1_passed=True, + phase2_passed=True, total_reward=0.75, + phase2_observation="p2 ok", + ) + _stub_env( + monkeypatch, m, tmp_path / "store", + messages=[_FakeAssistant(100, 20)], + prefill_result=prefill, + prefill_timing="after", + raise_after_prefill=RuntimeError("boom"), + ) + agent = m.ClaudeSDKOtfRawAgent(model="anthropic/claude-sonnet-4-5") + row = await agent.run_task( + dict(_TASK), str(tmp_path), 20.0, "raw", eval_mode="one-shot", + ) + assert row["error"] == "boom" + assert row["submission_status"] == "submitted_ok" + assert row["phase1_passed"] is True + assert row["phase2_observation"] == "p2 ok" + assert row["submitted_sql"] == "SELECT 1" + + +@pytest.mark.asyncio +async def test_run_task_exception_before_ctx_set_yields_empty_diagnostics( + monkeypatch, tmp_path, +): + from bird_interact_agents.agents.claude_sdk_otf_raw import agent as m + + _stub_env(monkeypatch, m, tmp_path / "store") + + def _boom(*a, **kw): + raise RuntimeError("early-setup boom") + + monkeypatch.setattr(m, "load_db_data_if_needed", _boom) + agent = m.ClaudeSDKOtfRawAgent(model="anthropic/claude-sonnet-4-5") + row = await agent.run_task( + dict(_TASK), str(tmp_path), 20.0, "raw", eval_mode="one-shot", + ) + assert "early-setup boom" in (row.get("error") or "") + assert row["submission_status"] is None + assert row["phase1_passed"] is False + + +@pytest.mark.asyncio +async def test_run_task_exception_path_isolated_from_stale_context( + monkeypatch, tmp_path, +): + from bird_interact_agents.agents.claude_sdk_otf_raw import agent as m + + _stub_env(monkeypatch, m, tmp_path / "store") + m._ctx_var.set({ + "result": { + "submission_status": "STALE_SHOULD_NOT_LEAK", + "phase1_passed": True, + "predicted_result_json": "STALE", + "gold_result_json": "STALE", + "phase1_observation": "STALE", + }, + }) + + def _boom(*a, **kw): + raise RuntimeError("early boom") + + monkeypatch.setattr(m, "load_db_data_if_needed", _boom) + agent = m.ClaudeSDKOtfRawAgent(model="anthropic/claude-sonnet-4-5") + row = await agent.run_task( + dict(_TASK), str(tmp_path), 20.0, "raw", eval_mode="one-shot", + ) + assert "early boom" in (row.get("error") or "") + assert row["submission_status"] != "STALE_SHOULD_NOT_LEAK" + assert row["submission_status"] is None + assert row["phase1_passed"] is False diff --git a/tests/test_claude_sdk_otf_raw_run_wiring.py b/tests/test_claude_sdk_otf_raw_run_wiring.py new file mode 100644 index 00000000..1fd37049 --- /dev/null +++ b/tests/test_claude_sdk_otf_raw_run_wiring.py @@ -0,0 +1,342 @@ +"""run.py + cloud wiring for the raw OTF agent frameworks. + +Locks down: +* `--framework claude_sdk_otf_raw` and `--framework claude_sdk_otf_ainteract_raw` + are in the CLI choices. +* `_validate_slayer_setup` skips validation for raw frameworks (they don't + use SLayer at all). +* `_validate_one_shot_framework` accepts `--query-mode raw` for + `claude_sdk_otf_raw`. +* `_FRAMEWORK_DATASET_MODE_BINDING` binds the raw frameworks correctly: + - `claude_sdk_otf_raw` → (livesqlbench, one-shot) + - `claude_sdk_otf_ainteract_raw` → (mini_interact, a-interact) +* `run_evaluation` branches to the raw agent classes for the right combos. +* Cloud: raw frameworks have no SLayer artifacts to upload/download. +""" + +from __future__ import annotations + +import pytest + + +# --------------------------------------------------------------------------- +# CLI choices +# --------------------------------------------------------------------------- + +def _framework_choices_from_parser(): + import ast + import inspect + + from bird_interact_agents import run as run_mod + + src = inspect.getsource(run_mod.main) + tree = ast.parse(src) + for node in ast.walk(tree): + if ( + isinstance(node, ast.Call) + and getattr(getattr(node.func, "attr", None), "lower", lambda: "")() == "add_argument" + ): + if node.args and isinstance(node.args[0], ast.Constant) and node.args[0].value == "--framework": + for kw in node.keywords: + if kw.arg == "choices" and isinstance(kw.value, ast.List): + return { + elt.value for elt in kw.value.elts + if isinstance(elt, ast.Constant) + } + raise AssertionError("could not find --framework choices in run.main") + + +def test_framework_choice_accepts_claude_sdk_otf_raw(): + assert "claude_sdk_otf_raw" in _framework_choices_from_parser() + + +def test_framework_choice_accepts_claude_sdk_otf_ainteract_raw(): + assert "claude_sdk_otf_ainteract_raw" in _framework_choices_from_parser() + + +def test_existing_framework_choices_preserved(): + """Adding the raw frameworks must not drop existing ones.""" + choices = _framework_choices_from_parser() + assert { + "claude_sdk", "pydantic_ai", "pydantic_ai_recursive", + "pydantic_ai_otf_encode", "claude_sdk_otf", "claude_sdk_otf_ainteract", + "claude_sdk_otf_raw", "claude_sdk_otf_ainteract_raw", + }.issubset(choices) + + +# --------------------------------------------------------------------------- +# _validate_slayer_setup — raw frameworks skip validation entirely +# --------------------------------------------------------------------------- + +def test_validate_slayer_setup_skips_for_raw_one_shot(): + """claude_sdk_otf_raw doesn't use SLayer — any slayer_setup value (or + the default 'pre-encoded') must not raise.""" + from bird_interact_agents import run as run_mod + + # pre-encoded + raw framework: must NOT raise (raw agents ignore slayer_setup) + run_mod._validate_slayer_setup( + slayer_setup="pre-encoded", framework="claude_sdk_otf_raw", + query_mode="raw", mode="one-shot", + ) + # on-the-fly + raw framework: must also not raise + run_mod._validate_slayer_setup( + slayer_setup="on-the-fly", framework="claude_sdk_otf_raw", + query_mode="raw", mode="one-shot", + ) + + +def test_validate_slayer_setup_skips_for_raw_ainteract(): + from bird_interact_agents import run as run_mod + + run_mod._validate_slayer_setup( + slayer_setup="pre-encoded", framework="claude_sdk_otf_ainteract_raw", + query_mode="raw", mode="a-interact", + ) + run_mod._validate_slayer_setup( + slayer_setup="on-the-fly", framework="claude_sdk_otf_ainteract_raw", + query_mode="raw", mode="a-interact", + ) + + +def test_validate_slayer_setup_still_requires_on_the_fly_for_slayer_otf(): + """Regression guard: existing slayer OTF behavior must be unchanged.""" + from bird_interact_agents import run as run_mod + + with pytest.raises(ValueError): + run_mod._validate_slayer_setup( + slayer_setup="pre-encoded", framework="claude_sdk_otf", + query_mode="slayer", mode="one-shot", + ) + + +# --------------------------------------------------------------------------- +# _validate_one_shot_framework — raw query mode allowed for raw framework +# --------------------------------------------------------------------------- + +def test_validate_one_shot_framework_accepts_raw_query_mode_for_raw(): + from bird_interact_agents import run as run_mod + + run_mod._validate_one_shot_framework( + mode="one-shot", query_mode="raw", framework="claude_sdk_otf_raw", + ) + + +def test_validate_one_shot_framework_rejects_slayer_mode_for_raw(): + """claude_sdk_otf_raw is raw-mode only — slayer query mode must be rejected.""" + from bird_interact_agents import run as run_mod + + with pytest.raises(ValueError): + run_mod._validate_one_shot_framework( + mode="one-shot", query_mode="slayer", framework="claude_sdk_otf_raw", + ) + + +def test_validate_one_shot_framework_still_requires_slayer_for_slayer_otf(): + """Regression guard: slayer OTF still requires slayer query mode.""" + from bird_interact_agents import run as run_mod + + with pytest.raises(ValueError): + run_mod._validate_one_shot_framework( + mode="one-shot", query_mode="raw", framework="claude_sdk_otf", + ) + + run_mod._validate_one_shot_framework( + mode="one-shot", query_mode="slayer", framework="claude_sdk_otf", + ) + + +# --------------------------------------------------------------------------- +# _FRAMEWORK_DATASET_MODE_BINDING — raw frameworks have correct bindings +# --------------------------------------------------------------------------- + +def test_framework_dataset_mode_binding_raw_one_shot(): + from bird_interact_agents import run as run_mod + + # Correct binding: livesqlbench / one-shot + run_mod._validate_framework_dataset_mode( + framework="claude_sdk_otf_raw", + dataset="livesqlbench", + mode="one-shot", + ) + + +def test_framework_dataset_mode_binding_raw_one_shot_rejects_wrong_dataset(): + from bird_interact_agents import run as run_mod + + with pytest.raises(ValueError): + run_mod._validate_framework_dataset_mode( + framework="claude_sdk_otf_raw", + dataset="mini_interact", + mode="one-shot", + ) + + +def test_framework_dataset_mode_binding_raw_ainteract(): + from bird_interact_agents import run as run_mod + + run_mod._validate_framework_dataset_mode( + framework="claude_sdk_otf_ainteract_raw", + dataset="mini_interact", + mode="a-interact", + ) + + +def test_framework_dataset_mode_binding_raw_ainteract_rejects_wrong_mode(): + from bird_interact_agents import run as run_mod + + with pytest.raises(ValueError): + run_mod._validate_framework_dataset_mode( + framework="claude_sdk_otf_ainteract_raw", + dataset="mini_interact", + mode="one-shot", + ) + + +# --------------------------------------------------------------------------- +# run_evaluation branches to raw agent classes +# --------------------------------------------------------------------------- + +@pytest.mark.asyncio +async def test_run_evaluation_branches_to_raw_otf_agent(monkeypatch, tmp_path): + from bird_interact_agents import run as run_mod + + constructed = [] + + class _Sentinel(Exception): + pass + + class _FakeAgent: + def __init__(self, **kwargs): + constructed.append(kwargs) + raise _Sentinel("stop") + + monkeypatch.setattr( + "bird_interact_agents.agents.claude_sdk_otf_raw.ClaudeSDKOtfRawAgent", + _FakeAgent, raising=False, + ) + monkeypatch.setattr(run_mod, "load_benchmark_tasks", lambda *a, **kw: []) + + data_file = tmp_path / "x.jsonl" + data_file.write_text("") + gold = tmp_path / "gold.jsonl" + gold.write_text("") + with pytest.raises(_Sentinel): + await run_mod.run_evaluation( + data_path=str(data_file), data_dir=str(tmp_path), + output_path=str(tmp_path / "eval.json"), + mode="one-shot", query_mode="raw", + framework="claude_sdk_otf_raw", slayer_setup="pre-encoded", + reasoning_effort=None, + dataset="livesqlbench", gold_file=str(gold), + ) + assert constructed + + +@pytest.mark.asyncio +async def test_run_evaluation_branches_to_raw_ainteract_agent(monkeypatch, tmp_path): + from bird_interact_agents import run as run_mod + + constructed = [] + + class _Sentinel(Exception): + pass + + class _FakeAgent: + def __init__(self, **kwargs): + constructed.append(kwargs) + raise _Sentinel("stop") + + monkeypatch.setattr( + "bird_interact_agents.agents.claude_sdk_otf_ainteract_raw.ClaudeSDKOtfAInteractRawAgent", + _FakeAgent, raising=False, + ) + monkeypatch.setattr(run_mod, "load_benchmark_tasks", lambda *a, **kw: []) + + data_file = tmp_path / "x.jsonl" + data_file.write_text("") + gold = tmp_path / "gold.jsonl" + gold.write_text("") + with pytest.raises(_Sentinel): + await run_mod.run_evaluation( + data_path=str(data_file), data_dir=str(tmp_path), + output_path=str(tmp_path / "eval.json"), + mode="a-interact", query_mode="raw", + framework="claude_sdk_otf_ainteract_raw", slayer_setup="pre-encoded", + reasoning_effort=None, + dataset="mini_interact", gold_file=str(gold), + ) + assert constructed + + +# --------------------------------------------------------------------------- +# Cloud: raw frameworks have no SLayer artifacts +# --------------------------------------------------------------------------- + +def test_cloud_actor_has_no_slayer_artifacts_for_raw_otf(): + """claude_sdk_otf_raw uses no slayer_otf_cache or slayer_models_otf.""" + from bird_interact_agents.cloud import ray_app + + cfg = { + "framework": "claude_sdk_otf_raw", + "slayer_setup": "pre-encoded", + "dataset": "livesqlbench", + } + artifacts = list(ray_app._slayer_artifacts_for(cfg)) + assert artifacts == [], ( + f"raw framework must have zero slayer cloud artifacts; got {artifacts!r}" + ) + + +def test_cloud_actor_has_no_slayer_artifacts_for_raw_ainteract(): + from bird_interact_agents.cloud import ray_app + + cfg = { + "framework": "claude_sdk_otf_ainteract_raw", + "slayer_setup": "pre-encoded", + "dataset": "mini_interact", + } + artifacts = list(ray_app._slayer_artifacts_for(cfg)) + assert artifacts == [], ( + f"raw ainteract framework must have zero slayer cloud artifacts; got {artifacts!r}" + ) + + +def test_cloud_driver_has_no_slayer_uploads_for_raw_otf(): + from types import SimpleNamespace + from bird_interact_agents.cloud import driver + + args = SimpleNamespace( + slayer_setup="pre-encoded", framework="claude_sdk_otf_raw", + dataset="livesqlbench", + ) + uploads = list(driver._slayer_uploads_for(args)) + assert uploads == [], ( + f"raw framework must have zero slayer upload artifacts; got {uploads!r}" + ) + + +def test_cloud_driver_has_no_slayer_uploads_for_raw_ainteract(): + from types import SimpleNamespace + from bird_interact_agents.cloud import driver + + args = SimpleNamespace( + slayer_setup="pre-encoded", framework="claude_sdk_otf_ainteract_raw", + dataset="mini_interact", + ) + uploads = list(driver._slayer_uploads_for(args)) + assert uploads == [], ( + f"raw ainteract framework must have zero slayer upload artifacts; got {uploads!r}" + ) + + +def test_cloud_slayer_artifacts_unchanged_for_slayer_otf(): + """Regression guard: slayer OTF still gets its cache artifact.""" + from bird_interact_agents.cloud import ray_app + + cfg = { + "framework": "claude_sdk_otf", + "slayer_setup": "on-the-fly", + "dataset": "livesqlbench", + } + artifacts = {a for (a, _root, _req) in ray_app._slayer_artifacts_for(cfg)} + assert "slayer_otf_cache" in artifacts diff --git a/tests/test_shared_otf_prompts.py b/tests/test_shared_otf_prompts.py new file mode 100644 index 00000000..8ac1e5ed --- /dev/null +++ b/tests/test_shared_otf_prompts.py @@ -0,0 +1,437 @@ +"""Tests for shared OTF prompt constants (_shared_otf_prompts.py). + +Covers three concerns: +1. SHA-256 snapshot tests that SLAYER_OTF_ONE_SHOT and SLAYER_OTF_AINTERACT + are byte-for-byte unchanged after the refactoring that imports constants + from _shared_otf_prompts. +2. That the shared constants render correctly with their format params. +3. That the shared constants appear verbatim in both slayer and raw OTF + prompts (after implementation), and that raw prompts are free of + SLayer-specific vocabulary. +""" + +from __future__ import annotations + +import hashlib + +import pytest + +# --------------------------------------------------------------------------- +# SHA-256 snapshot tests — must remain passing before AND after the +# refactoring that extracts shared constants from the slayer prompts. +# Hashes were captured from the pre-refactoring source. +# --------------------------------------------------------------------------- + +_ONE_SHOT_SHA256 = "6efc7c8e669f94742cc99381cd87a7cc122f6455b42b8887421afd4b8ab7aa63" +_AINTERACT_SHA256 = "c2957ea702b2014a27d5ddface8dda48123fd533b21378cb3e91ba70b4dc5392" + + +def test_slayer_otf_one_shot_unchanged(): + """Byte-for-byte contract: refactoring must not alter the rendered prompt.""" + from bird_interact_agents.agents.claude_sdk_otf.prompts import SLAYER_OTF_ONE_SHOT + + digest = hashlib.sha256(SLAYER_OTF_ONE_SHOT.encode()).hexdigest() + assert digest == _ONE_SHOT_SHA256, ( + f"SLAYER_OTF_ONE_SHOT changed (len={len(SLAYER_OTF_ONE_SHOT)}).\n" + f" expected: {_ONE_SHOT_SHA256}\n" + f" actual: {digest}" + ) + + +def test_slayer_otf_ainteract_unchanged(): + """Byte-for-byte contract: refactoring must not alter the rendered prompt.""" + from bird_interact_agents.agents.claude_sdk_otf_ainteract.prompts import ( + SLAYER_OTF_AINTERACT, + ) + + digest = hashlib.sha256(SLAYER_OTF_AINTERACT.encode()).hexdigest() + assert digest == _AINTERACT_SHA256, ( + f"SLAYER_OTF_AINTERACT changed (len={len(SLAYER_OTF_AINTERACT)}).\n" + f" expected: {_AINTERACT_SHA256}\n" + f" actual: {digest}" + ) + + +# --------------------------------------------------------------------------- +# Shared-constant accessibility +# --------------------------------------------------------------------------- + +def test_shared_constants_all_nonempty(): + """All six exported template constants must be non-empty strings.""" + from bird_interact_agents.agents._shared_otf_prompts import ( + _NO_USER_TO_CONSULT, + _DECOMPOSE_DISCIPLINE, + _RULE_0_ASK_BEFORE, + _ASK_AGAIN_RULE, + _PRE_SUBMIT_MUTATION_CHECK_ONE_SHOT, + _PRE_SUBMIT_MUTATION_CHECK_AINTERACT, + ) + + for name, val in [ + ("_NO_USER_TO_CONSULT", _NO_USER_TO_CONSULT), + ("_DECOMPOSE_DISCIPLINE", _DECOMPOSE_DISCIPLINE), + ("_RULE_0_ASK_BEFORE", _RULE_0_ASK_BEFORE), + ("_ASK_AGAIN_RULE", _ASK_AGAIN_RULE), + ("_PRE_SUBMIT_MUTATION_CHECK_ONE_SHOT", _PRE_SUBMIT_MUTATION_CHECK_ONE_SHOT), + ("_PRE_SUBMIT_MUTATION_CHECK_AINTERACT", _PRE_SUBMIT_MUTATION_CHECK_AINTERACT), + ]: + assert isinstance(val, str) and val.strip(), f"{name} must be a non-empty string" + + +# --------------------------------------------------------------------------- +# Rendering tests — shared constants with their specific format params +# --------------------------------------------------------------------------- + +def test_no_user_to_consult_renders_with_sources_desc(): + from bird_interact_agents.agents._shared_otf_prompts import _NO_USER_TO_CONSULT + + rendered = _NO_USER_TO_CONSULT.format(sources_desc="the schema and knowledge definitions") + assert "the schema and knowledge definitions" in rendered + assert "NO user to consult" in rendered + assert "autonomously" in rendered + + +def test_rule0_renders_for_slayer_params(): + """Slayer-specific rendering: ENCODE + submit_query.""" + from bird_interact_agents.agents._shared_otf_prompts import _RULE_0_ASK_BEFORE + + rendered = _RULE_0_ASK_BEFORE.format( + action_label="ENCODE", + action_context="BEFORE the encoding loop below,", + submit_tool="submit_query", + ) + assert "RULE 0 — ASK BEFORE YOU ENCODE." in rendered + assert "BEFORE the encoding loop below," in rendered + assert "submit_query" in rendered + assert "ask_user" in rendered + assert "operationalisation" in rendered + + +def test_rule0_renders_for_raw_params(): + """Raw-mode rendering: SUBMIT + submit_sql.""" + from bird_interact_agents.agents._shared_otf_prompts import _RULE_0_ASK_BEFORE + + rendered = _RULE_0_ASK_BEFORE.format( + action_label="SUBMIT", + action_context="BEFORE writing your SQL query,", + submit_tool="submit_sql", + ) + assert "RULE 0 — ASK BEFORE YOU SUBMIT." in rendered + assert "BEFORE writing your SQL query," in rendered + assert "submit_sql" in rendered + assert "submit_query" not in rendered + + +def test_ask_again_rule_renders_with_knowledge_source(): + from bird_interact_agents.agents._shared_otf_prompts import _ASK_AGAIN_RULE + + for source, expected in [ + ("a memory", "not pinned by a memory or column"), + ("a knowledge definition", "not pinned by a knowledge definition or column"), + ]: + rendered = _ASK_AGAIN_RULE.format(knowledge_source=source) + assert expected in rendered + assert "ASK AGAIN IF NEEDED" in rendered + assert "ask_user" in rendered + + +def test_pre_submit_check_one_shot_renders_for_slayer(): + from bird_interact_agents.agents._shared_otf_prompts import ( + _PRE_SUBMIT_MUTATION_CHECK_ONE_SHOT, + ) + + rendered = _PRE_SUBMIT_MUTATION_CHECK_ONE_SHOT.format( + submit_tool="submit_query", clause_b="encoded KB", + ) + assert "submit_query" in rendered + assert "encoded KB" in rendered + assert "PRE-SUBMIT MUTATION CHECK" in rendered + assert "submit_sql" not in rendered + + +def test_pre_submit_check_one_shot_renders_for_raw(): + from bird_interact_agents.agents._shared_otf_prompts import ( + _PRE_SUBMIT_MUTATION_CHECK_ONE_SHOT, + ) + + rendered = _PRE_SUBMIT_MUTATION_CHECK_ONE_SHOT.format( + submit_tool="submit_sql", clause_b="knowledge definition", + ) + assert "submit_sql" in rendered + assert "knowledge definition" in rendered + assert "submit_query" not in rendered + + +def test_pre_submit_check_ainteract_renders_for_slayer(): + from bird_interact_agents.agents._shared_otf_prompts import ( + _PRE_SUBMIT_MUTATION_CHECK_AINTERACT, + ) + + rendered = _PRE_SUBMIT_MUTATION_CHECK_AINTERACT.format( + submit_tool="submit_query", clause_c="encoded KB", + ) + assert "submit_query" in rendered + assert "encoded KB" in rendered + assert "ask_user" in rendered + assert "PRE-SUBMIT MUTATION CHECK" in rendered + + +def test_pre_submit_check_ainteract_renders_for_raw(): + from bird_interact_agents.agents._shared_otf_prompts import ( + _PRE_SUBMIT_MUTATION_CHECK_AINTERACT, + ) + + rendered = _PRE_SUBMIT_MUTATION_CHECK_AINTERACT.format( + submit_tool="submit_sql", clause_c="knowledge definition", + ) + assert "submit_sql" in rendered + assert "knowledge definition" in rendered + assert "submit_query" not in rendered + + +# --------------------------------------------------------------------------- +# Shared constants appear verbatim in both slayer prompts. +# These tests should pass both BEFORE and AFTER the refactoring — +# before because the slayer prompts already contain these strings; +# after because refactoring preserves byte-for-byte identity (see snapshots). +# --------------------------------------------------------------------------- + +def test_decompose_discipline_in_slayer_one_shot(): + from bird_interact_agents.agents._shared_otf_prompts import _DECOMPOSE_DISCIPLINE + from bird_interact_agents.agents.claude_sdk_otf.prompts import SLAYER_OTF_ONE_SHOT + + assert _DECOMPOSE_DISCIPLINE in SLAYER_OTF_ONE_SHOT + + +def test_decompose_discipline_in_slayer_ainteract(): + from bird_interact_agents.agents._shared_otf_prompts import _DECOMPOSE_DISCIPLINE + from bird_interact_agents.agents.claude_sdk_otf_ainteract.prompts import ( + SLAYER_OTF_AINTERACT, + ) + + assert _DECOMPOSE_DISCIPLINE in SLAYER_OTF_AINTERACT + + +def test_no_user_to_consult_in_slayer_one_shot(): + from bird_interact_agents.agents._shared_otf_prompts import _NO_USER_TO_CONSULT + from bird_interact_agents.agents.claude_sdk_otf.prompts import SLAYER_OTF_ONE_SHOT + + rendered = _NO_USER_TO_CONSULT.format( + sources_desc="the memories and column\ndescriptions" + ) + assert rendered in SLAYER_OTF_ONE_SHOT + + +def test_rule0_rendered_in_slayer_ainteract(): + from bird_interact_agents.agents._shared_otf_prompts import _RULE_0_ASK_BEFORE + from bird_interact_agents.agents.claude_sdk_otf_ainteract.prompts import ( + SLAYER_OTF_AINTERACT, + ) + + rendered = _RULE_0_ASK_BEFORE.format( + action_label="ENCODE", + action_context="BEFORE the encoding loop below,", + submit_tool="submit_query", + ) + assert rendered in SLAYER_OTF_AINTERACT + + +def test_ask_again_in_slayer_ainteract(): + from bird_interact_agents.agents._shared_otf_prompts import _ASK_AGAIN_RULE + from bird_interact_agents.agents.claude_sdk_otf_ainteract.prompts import ( + SLAYER_OTF_AINTERACT, + ) + + rendered = _ASK_AGAIN_RULE.format(knowledge_source="a memory") + assert rendered in SLAYER_OTF_AINTERACT + + +def test_pre_submit_one_shot_in_slayer_one_shot(): + from bird_interact_agents.agents._shared_otf_prompts import ( + _PRE_SUBMIT_MUTATION_CHECK_ONE_SHOT, + ) + from bird_interact_agents.agents.claude_sdk_otf.prompts import SLAYER_OTF_ONE_SHOT + + rendered = _PRE_SUBMIT_MUTATION_CHECK_ONE_SHOT.format( + submit_tool="submit_query", clause_b="encoded KB", + ) + assert rendered in SLAYER_OTF_ONE_SHOT + + +def test_pre_submit_ainteract_in_slayer_ainteract(): + from bird_interact_agents.agents._shared_otf_prompts import ( + _PRE_SUBMIT_MUTATION_CHECK_AINTERACT, + ) + from bird_interact_agents.agents.claude_sdk_otf_ainteract.prompts import ( + SLAYER_OTF_AINTERACT, + ) + + rendered = _PRE_SUBMIT_MUTATION_CHECK_AINTERACT.format( + submit_tool="submit_query", clause_c="encoded KB", + ) + assert rendered in SLAYER_OTF_AINTERACT + + +# --------------------------------------------------------------------------- +# Shared constants appear verbatim in raw prompts (requires implementation). +# These tests FAIL until the raw agent modules are created. +# --------------------------------------------------------------------------- + +def test_decompose_discipline_in_raw_one_shot(): + from bird_interact_agents.agents._shared_otf_prompts import _DECOMPOSE_DISCIPLINE + from bird_interact_agents.agents.claude_sdk_otf_raw.prompts import ( + RAW_OTF_ONE_SHOT, + ) + + assert _DECOMPOSE_DISCIPLINE in RAW_OTF_ONE_SHOT + + +def test_decompose_discipline_in_raw_ainteract(): + from bird_interact_agents.agents._shared_otf_prompts import _DECOMPOSE_DISCIPLINE + from bird_interact_agents.agents.claude_sdk_otf_ainteract_raw.prompts import ( + RAW_OTF_AINTERACT, + ) + + assert _DECOMPOSE_DISCIPLINE in RAW_OTF_AINTERACT + + +def test_pre_submit_one_shot_in_raw_one_shot(): + from bird_interact_agents.agents._shared_otf_prompts import ( + _PRE_SUBMIT_MUTATION_CHECK_ONE_SHOT, + ) + from bird_interact_agents.agents.claude_sdk_otf_raw.prompts import ( + RAW_OTF_ONE_SHOT, + ) + + rendered = _PRE_SUBMIT_MUTATION_CHECK_ONE_SHOT.format( + submit_tool="submit_sql", clause_b="knowledge definition", + ) + assert rendered in RAW_OTF_ONE_SHOT + + +def test_pre_submit_ainteract_in_raw_ainteract(): + from bird_interact_agents.agents._shared_otf_prompts import ( + _PRE_SUBMIT_MUTATION_CHECK_AINTERACT, + ) + from bird_interact_agents.agents.claude_sdk_otf_ainteract_raw.prompts import ( + RAW_OTF_AINTERACT, + ) + + rendered = _PRE_SUBMIT_MUTATION_CHECK_AINTERACT.format( + submit_tool="submit_sql", clause_c="knowledge definition", + ) + assert rendered in RAW_OTF_AINTERACT + + +def test_no_user_to_consult_in_raw_one_shot(): + from bird_interact_agents.agents._shared_otf_prompts import _NO_USER_TO_CONSULT + from bird_interact_agents.agents.claude_sdk_otf_raw.prompts import ( + RAW_OTF_ONE_SHOT, + ) + + # Raw one-shot has no user to consult; the template must appear rendered. + assert "NO user to consult" in RAW_OTF_ONE_SHOT + + +def test_rule0_rendered_in_raw_ainteract(): + from bird_interact_agents.agents._shared_otf_prompts import _RULE_0_ASK_BEFORE + from bird_interact_agents.agents.claude_sdk_otf_ainteract_raw.prompts import ( + RAW_OTF_AINTERACT, + ) + + rendered = _RULE_0_ASK_BEFORE.format( + action_label="SUBMIT", + action_context="BEFORE writing your SQL query,", + submit_tool="submit_sql", + ) + assert rendered in RAW_OTF_AINTERACT + + +def test_ask_again_in_raw_ainteract(): + from bird_interact_agents.agents._shared_otf_prompts import _ASK_AGAIN_RULE + from bird_interact_agents.agents.claude_sdk_otf_ainteract_raw.prompts import ( + RAW_OTF_AINTERACT, + ) + + rendered = _ASK_AGAIN_RULE.format(knowledge_source="a knowledge definition") + assert rendered in RAW_OTF_AINTERACT + + +# --------------------------------------------------------------------------- +# Raw prompts must NOT contain SLayer-specific vocabulary. +# Fails until the raw agent modules are created. +# --------------------------------------------------------------------------- + +_SLAYER_VOCAB = [ + "submit_query", + "create_model", + "edit_model", + "[kb=", + "memory:", + "slayer", + "SLayer", + "mcp__slayer__", +] + + +def test_raw_one_shot_prompt_absent_slayer_vocab(): + from bird_interact_agents.agents.claude_sdk_otf_raw.prompts import RAW_OTF_ONE_SHOT + + # Render with placeholder values (budget/db_name/user_query not SLayer-related). + rendered = RAW_OTF_ONE_SHOT.format( + budget=20.0, db_name="shop", user_query="how many items?", + ) + for term in _SLAYER_VOCAB: + assert term not in rendered, ( + f"raw one-shot prompt must not contain SLayer term {term!r}" + ) + + +def test_raw_ainteract_prompt_absent_slayer_vocab(): + from bird_interact_agents.agents.claude_sdk_otf_ainteract_raw.prompts import ( + RAW_OTF_AINTERACT, + ) + + rendered = RAW_OTF_AINTERACT.format( + budget=20.0, db_name="shop", user_query="how many items?", + ) + for term in _SLAYER_VOCAB: + assert term not in rendered, ( + f"raw ainteract prompt must not contain SLayer term {term!r}" + ) + + +def test_raw_one_shot_prompt_mentions_submit_sql(): + from bird_interact_agents.agents.claude_sdk_otf_raw.prompts import RAW_OTF_ONE_SHOT + + assert "submit_sql" in RAW_OTF_ONE_SHOT + + +def test_raw_ainteract_prompt_mentions_submit_sql_and_ask_user(): + from bird_interact_agents.agents.claude_sdk_otf_ainteract_raw.prompts import ( + RAW_OTF_AINTERACT, + ) + + assert "submit_sql" in RAW_OTF_AINTERACT + assert "ask_user" in RAW_OTF_AINTERACT + + +def test_raw_prompts_use_synthetic_examples_only(): + """Guards feedback_prompts_synthetic_examples_only: no real eval-set + DB / table / column / value names may appear in either raw prompt.""" + from bird_interact_agents.agents.claude_sdk_otf_raw.prompts import RAW_OTF_ONE_SHOT + from bird_interact_agents.agents.claude_sdk_otf_ainteract_raw.prompts import ( + RAW_OTF_AINTERACT, + ) + + banned = [ + "households", "tenure_type", "income_bracket", "dwelling_class", + "socsupport", "service_types", "stellardist", "photo_band", + "taguatinga", + ] + for name, text in [("RAW_OTF_ONE_SHOT", RAW_OTF_ONE_SHOT), ("RAW_OTF_AINTERACT", RAW_OTF_AINTERACT)]: + low = text.lower() + for banned_name in banned: + assert banned_name not in low, ( + f"real eval-set name {banned_name!r} leaked into {name}" + ) From 048ef2ee6345339e4c098a0ab7c34ded317ebab0 Mon Sep 17 00:00:00 2001 From: Egor Kraev Date: Wed, 3 Jun 2026 09:50:33 +0200 Subject: [PATCH 27/29] DEV-1515 round 14: LLM judge wired, schema widening, sol_sql normalisation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three Codex findings, all addressed: Item 1 (Codex major — regrade.py + tolerant_grader.py): Regrade never threaded an ``llm_judge`` into ``grade_submission``, so N5 was permanently dormant for insufficient-task rows. Now the judge fires automatically as part of the regular grader flow: * New ``LiteLLMJudge`` concrete class in ``tolerant_grader.py`` — wraps ``litellm.completion`` with a fixed ACCEPT/REJECT prompt contract and parses the last line of the model's reply. Network / shape errors return ``None`` per ``LLMJudgeProtocol`` so the cascade falls through cleanly. * Regrade CLI reads ``agent_model`` from ``/manifest.json`` and builds ``CachedLLMJudge(inner=LiteLLMJudge(model=agent_model), cache_path=/llm_judge_cache.json)`` ONCE per regrade — no new flags, no extra scaffolding. * Dropped ``--force-llm-judge`` and the matching ``clear_llm_judge_cache`` helper (and the obsolete tests). The cache key already includes ``model_name`` + content hashes, so changing the agent's model on a resubmit naturally invalidates entries without a separate cache-clear flag. 5 new tests in test_tolerant_grader_orchestration.py: ACCEPT / REJECT / inconclusive parsing, litellm exceptions degrade to None, and a mocked-response happy path that validates the OpenAI-shaped reply path. Item 2 (Codex major — annotation_schema.py): ``TaskAnnotation.external_knowledge`` was ``List[int]`` only. All observed mini-interact / livesqlbench data currently ships int-only, but some livesqlbench fixtures + forward-looking benchmark variants carry the KB body inline as a dict (``{"id": 31, "label": "TETL", "definition": "..."}``); pydantic would hard-reject those. Fix: widen to ``List[Union[int, dict]]`` + add a regression test that round-trips a mixed list through model_dump / model_validate. Item 3 (Codex minor — scripts/dev1515_convert_runs.py): Same char-splitting bug round 5 fixed elsewhere — the script did ``list(task_row.get("sol_sql") or [])``, which on a bare-string ``sol_sql`` (mini-interact does both shapes) would produce ``["S","E","L","E","C","T",...]`` and the grader would mis-execute. Replaced with ``normalize_sol_sql(...)`` from grade_in_place. (No new test — the helper is already tested in tests/test_normalize_sol_sql.py round 5.) Co-Authored-By: Claude Opus 4.7 --- scripts/dev1515_convert_runs.py | 12 +- .../eval/annotation_schema.py | 10 +- src/bird_interact_agents/eval/regrade.py | 83 ++++++------ .../eval/tolerant_grader.py | 127 ++++++++++++++++++ tests/test_eval_annotation_schema.py | 43 ++++++ tests/test_regrade_cli.py | 102 ++------------ tests/test_tolerant_grader_orchestration.py | 125 +++++++++++++++++ 7 files changed, 367 insertions(+), 135 deletions(-) diff --git a/scripts/dev1515_convert_runs.py b/scripts/dev1515_convert_runs.py index 4ed6b1c7..8a177b87 100644 --- a/scripts/dev1515_convert_runs.py +++ b/scripts/dev1515_convert_runs.py @@ -48,7 +48,10 @@ Provenance, TaskAnnotation, ) -from bird_interact_agents.eval.grade_in_place import _build_submission_annotation +from bird_interact_agents.eval.grade_in_place import ( + _build_submission_annotation, + normalize_sol_sql, +) from bird_interact_agents.eval.annotate import ( _user_sim_interaction_from_trajectory, ) @@ -217,7 +220,12 @@ def _process_one( cascade = grade_submission( task_annotation=task_ann, audited_gold_rows=audit_rows, - original_sol_sql=list(task_row.get("sol_sql") or []), + # Codex r13: ``normalize_sol_sql`` handles the bare-string + # shape (some mini-interact rows carry ``sol_sql`` as a + # single string). A bare ``list(...)`` would char-split it + # into ``["S", "E", "L", "E", "C", "T", ...]`` and the + # grader would mis-execute → N1 always False. + original_sol_sql=normalize_sol_sql(task_row.get("sol_sql")), submitted_sql=submitted_sql, db_path=db_path, conn=conn, diff --git a/src/bird_interact_agents/eval/annotation_schema.py b/src/bird_interact_agents/eval/annotation_schema.py index 9e40fc74..8ead72cf 100644 --- a/src/bird_interact_agents/eval/annotation_schema.py +++ b/src/bird_interact_agents/eval/annotation_schema.py @@ -13,7 +13,7 @@ """ from __future__ import annotations -from typing import List, Literal, Optional +from typing import List, Literal, Optional, Union from pydantic import BaseModel, ConfigDict, Field @@ -229,7 +229,13 @@ class TaskAnnotation(BaseModel): """Path or stable ID of an earlier annotation this replaces.""" amb_user_query: str - external_knowledge: List[int] = Field(default_factory=list) + external_knowledge: List[Union[int, dict]] = Field(default_factory=list) + """KB references the task anchors against. Each entry is either an + integer KB id (the common mini-interact / livesqlbench shape) OR a + dict carrying the KB body inline (some fixture rows and forward- + looking benchmark variants). Codex r13: the prior ``List[int]`` was + overly restrictive — dict-shaped entries would be rejected by + pydantic at conversion time.""" masked_terms: List[MaskedTerm] = Field(default_factory=list) metadata_sufficiency: MetadataSufficiency diff --git a/src/bird_interact_agents/eval/regrade.py b/src/bird_interact_agents/eval/regrade.py index 111a4e12..c4fb8ac2 100644 --- a/src/bird_interact_agents/eval/regrade.py +++ b/src/bird_interact_agents/eval/regrade.py @@ -10,8 +10,16 @@ python -m bird_interact_agents.eval.regrade \\ --run-id --benchmark mini-interact \\ - [--instance-ids ...] \\ - [--force-llm-judge] + [--instance-ids ...] + +The N5 LLM-judge runs automatically when a task's +``metadata_sufficiency.verdict == "insufficient"``; it uses the agent's +own model (read from ``/manifest.json``) so the judge re-asks +the same model whether its submission is a defensible novel reading +of the ambiguous task. ``CachedLLMJudge`` persists verdicts to +``/llm_judge_cache.json`` keyed on model name + content +hashes, so re-grades reuse decisions and a model change naturally +invalidates entries (no separate cache-clear flag). """ from __future__ import annotations @@ -20,7 +28,7 @@ import re import shutil from pathlib import Path -from typing import Any, Callable, Iterable, List, Optional +from typing import Any, Callable, List, Optional from pydantic import BaseModel, ConfigDict, Field @@ -76,33 +84,6 @@ def _latest_attempt_file(sub: Path) -> Path | None: return best[1] if best else None -def clear_llm_judge_cache( - *, - cache_path: Path, - instance_ids: Optional[Iterable[str]], -) -> None: - """Drop cache entries whose embedded ``instance_id`` matches any of - ``instance_ids``. Entries for other instances are preserved. - - Pass ``instance_ids=None`` to drop EVERY entry in the cache — the - correct behaviour for an unfiltered ``--force-llm-judge`` regrade - (when the caller wants the judge to re-decide every cascade-N5 row, - a partial clear would leave previously-cached verdicts intact). - """ - if not cache_path.exists(): - return - if instance_ids is None: - cache_path.write_text("{}\n") - return - cache = json.loads(cache_path.read_text()) - wanted = set(instance_ids) - new = { - k: v for k, v in cache.items() - if v.get("instance_id") not in wanted - } - cache_path.write_text(json.dumps(new, indent=2)) - - def _build_original_sql_index(benchmark: str) -> dict[str, list[str]]: """Map ``instance_id`` → list-of-SQL-strings for the benchmark's original gold. mini_interact carries ``sol_sql`` inline on each task @@ -177,7 +158,6 @@ def regrade_run( benchmark: str, run_dir: Path, instance_ids: Optional[List[str]] = None, - force_llm_judge: bool = False, grader: Callable[..., Any], repo_root: Optional[Path] = None, ) -> RegradeReport: @@ -191,14 +171,6 @@ def regrade_run( return report filter_set = set(instance_ids) if instance_ids else None - if force_llm_judge: - # ``filter_set=None`` clears the whole cache — the right thing - # for an unfiltered regrade since otherwise stale verdicts - # would survive and silently override the fresh judge call. - clear_llm_judge_cache( - cache_path=run_dir / "llm_judge_cache.json", - instance_ids=filter_set, - ) # Reset the fresh-rows scratch dir so a partial regrade doesn't # leak stale per-instance rows from a previous pass into @@ -323,7 +295,6 @@ def main(argv: Optional[List[str]] = None) -> int: parser.add_argument("--run-id", required=True) parser.add_argument("--benchmark", required=True) parser.add_argument("--instance-ids", default=None) - parser.add_argument("--force-llm-judge", action="store_true") args = parser.parse_args(argv) instance_ids = ( @@ -334,7 +305,11 @@ def main(argv: Optional[List[str]] = None) -> int: from bird_interact_agents.eval.annotate import ( _user_sim_interaction_from_trajectory, ) - from bird_interact_agents.eval.tolerant_grader import grade_submission + from bird_interact_agents.eval.tolerant_grader import ( + CachedLLMJudge, + LiteLLMJudge, + grade_submission, + ) run_dir = paths.results_root() / "cloud" / args.run_id # Index the benchmark's source data once. mini_interact ships sol_sql @@ -349,6 +324,29 @@ def main(argv: Optional[List[str]] = None) -> int: # where the agent never queried the user-sim. _bench_is_interactive = not get_benchmark(args.benchmark).one_shot + # Codex r13: build the LLM judge ONCE from the run's recorded + # ``agent_model`` (read from ``/manifest.json``). The N5 + # judge uses the same model the agent did — re-asking the model + # whether its own SQL is a defensible novel reading of the + # ambiguous task. ``CachedLLMJudge`` persists verdicts to + # ``llm_judge_cache.json`` and keys on the model name; changing + # the agent's model on a resubmit naturally invalidates entries + # (no separate ``--force-llm-judge`` flag needed). + _manifest_path = run_dir / "manifest.json" + if _manifest_path.exists(): + _agent_model = json.loads(_manifest_path.read_text()).get( + "agent_model", + ) + else: + _agent_model = None + if _agent_model: + _llm_judge: Optional[CachedLLMJudge] = CachedLLMJudge( + inner=LiteLLMJudge(model=_agent_model), + cache_path=run_dir / "llm_judge_cache.json", + ) + else: + _llm_judge = None + def _grader(*, instance_id: str, submitted_sql: str, task_row: dict, **_kw): # Minimal end-to-end wiring — production callers pre-build the # implicit annotation + audited gold rows themselves. @@ -406,6 +404,7 @@ def _grader(*, instance_id: str, submitted_sql: str, task_row: dict, **_kw): audited_gold_rows=audited, original_sol_sql=original_sol_sql, submitted_sql=submitted_sql, + llm_judge=_llm_judge, db_path=db_path, conn=None, user_sim_n_asks=_user_sim_n_asks, @@ -413,7 +412,7 @@ def _grader(*, instance_id: str, submitted_sql: str, task_row: dict, **_kw): report = regrade_run( run_id=args.run_id, benchmark=args.benchmark, run_dir=run_dir, - instance_ids=instance_ids, force_llm_judge=args.force_llm_judge, + instance_ids=instance_ids, grader=_grader, ) print(f"regrade: {report.regraded} instances rewritten, " diff --git a/src/bird_interact_agents/eval/tolerant_grader.py b/src/bird_interact_agents/eval/tolerant_grader.py index c1f56caf..f64b148b 100644 --- a/src/bird_interact_agents/eval/tolerant_grader.py +++ b/src/bird_interact_agents/eval/tolerant_grader.py @@ -500,6 +500,133 @@ def judge(self, **kwargs: Any) -> Optional[bool]: return result +_JUDGE_SYSTEM_PROMPT = ( + "You are an expert SQL evaluator. The user has issued an " + "ambiguous task; a strict cascade has confirmed that the agent's " + "SQL does NOT match any of the known acceptable readings. Your " + "job is to decide whether the agent's submission is a " + "DEFENSIBLE NOVEL reading — consistent with what the task " + "metadata DOES say — or whether it is genuinely wrong.\n\n" + "Reply concisely. The LAST line of your reply MUST begin with " + "either ``ACCEPT`` (valid novel reading) or ``REJECT`` (not " + "defensible). Anything else is treated as an inconclusive timeout." +) + + +def _format_predicted_rows(rows: Sequence[Sequence]) -> str: + if not rows: + return "(no rows returned)" + lines = [] + for r in rows[:20]: + lines.append(" " + " | ".join(str(c) for c in r)) + return "\n".join(lines) + + +def _build_judge_user_prompt( + *, + evaluator_prompt: str, + gold_variants_summary: list[dict], + metadata_anchors: list[str], + submitted_sql: str, + predicted_rows_head: Sequence[Sequence], +) -> str: + variants_block = "\n".join( + f" - variant_id={v.get('variant_id')!r}, " + f"interpretation={v.get('interpretation')!r}" + for v in gold_variants_summary + ) or " (no audited variants — original gold is the only reference)" + anchors_block = ", ".join(metadata_anchors) or "(none)" + return ( + f"Task evaluator prompt (disambiguation criteria):\n" + f"{evaluator_prompt}\n\n" + f"Known acceptable readings:\n{variants_block}\n\n" + f"Ambiguous terms in the original task:\n{anchors_block}\n\n" + f"Submitted SQL:\n{submitted_sql}\n\n" + f"First {min(20, len(predicted_rows_head))} predicted rows:\n" + f"{_format_predicted_rows(predicted_rows_head)}\n\n" + f"Decide: is the agent's submission a defensible NOVEL reading?\n" + f"Reply ACCEPT or REJECT on the LAST line." + ) + + +def _parse_judge_decision(text: str) -> Optional[bool]: + """Parse the LLM's reply into True / False / None per the protocol.""" + if not text: + return None + last = text.strip().splitlines()[-1].strip().upper() + if last.startswith("ACCEPT"): + return True + if last.startswith("REJECT"): + return False + return None + + +class LiteLLMJudge: + """Concrete LLM judge backed by ``litellm.completion``. Returns + ``True`` / ``False`` / ``None`` per :class:`LLMJudgeProtocol`. + + The model name is a litellm-style string (e.g. + ``"anthropic/claude-sonnet-4-5"``). The constructor doesn't touch + the network; ``judge()`` does — wrap in :class:`CachedLLMJudge` so + repeated calls for the same (annotation, gold-variants, + submitted-SQL) tuple don't burn tokens. + """ + + def __init__( + self, + *, + model: str, + temperature: float = 0.0, + timeout_s: float = 60.0, + ) -> None: + self._model = model + self._temperature = temperature + self._timeout_s = timeout_s + + @property + def model_name(self) -> str: + return self._model + + def judge(self, **kwargs: Any) -> Optional[bool]: + try: + import litellm + except ImportError: + logger.exception("litellm not installed; LiteLLMJudge cannot fire") + return None + user_prompt = _build_judge_user_prompt( + evaluator_prompt=kwargs.get("evaluator_prompt") or "", + gold_variants_summary=kwargs.get("gold_variants_summary") or [], + metadata_anchors=kwargs.get("metadata_anchors") or [], + submitted_sql=kwargs.get("submitted_sql") or "", + predicted_rows_head=kwargs.get("predicted_rows_head") or [], + ) + try: + resp = litellm.completion( + model=self._model, + temperature=self._temperature, + timeout=self._timeout_s, + messages=[ + {"role": "system", "content": _JUDGE_SYSTEM_PROMPT}, + {"role": "user", "content": user_prompt}, + ], + ) + except Exception: # noqa: BLE001 + logger.exception( + "LiteLLMJudge.judge raised against model=%s instance=%s", + self._model, kwargs.get("instance_id"), + ) + return None + try: + text = resp["choices"][0]["message"]["content"] + except (KeyError, IndexError, TypeError): + logger.exception( + "LiteLLMJudge: malformed response shape for instance=%s", + kwargs.get("instance_id"), + ) + return None + return _parse_judge_decision(text) + + # --------------------------------------------------------------------------- # Default executor (real SQLite path) + grade_submission orchestrator # --------------------------------------------------------------------------- diff --git a/tests/test_eval_annotation_schema.py b/tests/test_eval_annotation_schema.py index 2049a418..a4428e52 100644 --- a/tests/test_eval_annotation_schema.py +++ b/tests/test_eval_annotation_schema.py @@ -238,3 +238,46 @@ def test_written_json_is_valid_utf8_and_human_readable(tmp_path): assert body_lines[0] == "{" indent_lines = [l for l in body_lines[1:-1] if l.strip()] assert all(l.startswith(" ") for l in indent_lines) + + +# --------------------------------------------------------------------------- +# Codex r13: ``TaskAnnotation.external_knowledge`` was ``List[int]`` +# only; some livesqlbench fixtures + forward-looking benchmark variants +# carry the KB body inline as a dict. The widened type accepts both. +# --------------------------------------------------------------------------- + + +def test_external_knowledge_accepts_mixed_int_and_dict_entries(): + from bird_interact_agents.eval import ( + MetadataSufficiency, TaskAnnotation, + ) + from bird_interact_agents.eval.annotation_schema import Provenance + + ann = TaskAnnotation( + instance_id="alien_42", + selected_database="alien", + annotated_by="test", + annotated_at="2026-06-02", + amb_user_query="x", + external_knowledge=[ + 1, + {"id": 31, "label": "TETL", "definition": "ERF + LER + MDR*2"}, + 7, + ], + metadata_sufficiency=MetadataSufficiency( + verdict="sufficient", rationale="r", + ), + provenance=Provenance( + task_jsonl_path="mini_interact.jsonl", + task_jsonl_instance_id="alien_42", + ), + ) + # Round-trip through JSON to confirm the dict entry survives intact. + decoded = TaskAnnotation.model_validate_json( + ann.model_dump_json(exclude_none=False) + ) + assert decoded.external_knowledge[0] == 1 + assert decoded.external_knowledge[1] == { + "id": 31, "label": "TETL", "definition": "ERF + LER + MDR*2", + } + assert decoded.external_knowledge[2] == 7 diff --git a/tests/test_regrade_cli.py b/tests/test_regrade_cli.py index 9f2a7813..a01862f8 100644 --- a/tests/test_regrade_cli.py +++ b/tests/test_regrade_cli.py @@ -1,20 +1,22 @@ """DEV-1515: offline re-grade CLI for already-completed runs. `python -m bird_interact_agents.eval.regrade --run-id - [--instance-ids ...] [--benchmark ...] [--force-llm-judge]` + [--instance-ids ...] --benchmark ...` Contract: -* Walks `/cloud//rows//attempt-1.json` for each - instance. +* Walks `/cloud//rows//attempt-N.json` for each + instance (highest N wins). * Re-runs `grade_submission` with the locally-loaded LLM-judge cache - at `/cloud//llm_judge_cache.json`. + at `/cloud//llm_judge_cache.json`. The judge uses + the agent's own model (read from `/manifest.json`) and is + invoked automatically when `metadata_sufficiency.verdict == + "insufficient"`. * OVERWRITES `/annotations///.submission..json` (this is the explicit re-grade path; distinct from `fetch`'s no-overwrite merge). * Writes a fresh `/cloud//eval_regraded.json` — the historical `eval.json` is NOT mutated. * `--instance-ids` filters which rows get re-graded. -* `--force-llm-judge` invalidates cache entries for affected rows. """ from __future__ import annotations @@ -70,7 +72,7 @@ def __call__(self, **kw): benchmark="mini-interact", run_dir=run_dir, instance_ids=None, - force_llm_judge=False, + grader=StubGrader(), repo_root=tmp_path, ) @@ -108,7 +110,7 @@ def __call__(self, **kw): regrade_run( run_id="r1", benchmark="mini-interact", run_dir=run_dir, - instance_ids=None, force_llm_judge=False, + instance_ids=None, grader=StubGrader(), repo_root=tmp_path, ) refreshed = json.loads(dest.read_text()) @@ -144,7 +146,7 @@ def __call__(self, **kw): regrade_run( run_id="r1", benchmark="mini-interact", run_dir=run_dir, - instance_ids=["alien_2"], force_llm_judge=False, + instance_ids=["alien_2"], grader=StubGrader(), repo_root=tmp_path, ) assert len(seen) == 1 @@ -178,7 +180,7 @@ def __call__(self, **kw): regrade_run( run_id="r1", benchmark="mini-interact", run_dir=run_dir, - instance_ids=None, force_llm_judge=False, + instance_ids=None, grader=StubGrader(), repo_root=tmp_path, ) @@ -188,84 +190,6 @@ def __call__(self, **kw): assert json.loads(eval_json.read_text())["phase1_count"] == 999 -def test_regrade_force_llm_judge_clears_cache_entries(tmp_path, monkeypatch): - """`--force-llm-judge` MUST drop matching keys from - `/cloud//llm_judge_cache.json` before re-grading. - - Cache entries embed `instance_id` so the clearer can filter. The - contract: after `clear_llm_judge_cache(..., instance_ids=["alien_1"])` - NO cached entry whose key/value references `alien_1` remains; entries - for OTHER instances are preserved. - """ - from bird_interact_agents import paths as paths_mod - monkeypatch.setattr(paths_mod, "main_checkout_root", lambda: tmp_path) - - run_dir = tmp_path / "results" / "cloud" / "r1" - _write_attempt(run_dir, "alien_1") - - cache_path = run_dir / "llm_judge_cache.json" - cache_path.parent.mkdir(parents=True, exist_ok=True) - cache_path.write_text(json.dumps({ - "k_for_alien_1": {"instance_id": "alien_1", "verdict": True}, - "k_for_alien_2": {"instance_id": "alien_2", "verdict": False}, - })) - - from bird_interact_agents.eval.regrade import clear_llm_judge_cache - - clear_llm_judge_cache( - cache_path=cache_path, instance_ids=["alien_1"], - ) - remaining = json.loads(cache_path.read_text()) - assert "k_for_alien_1" not in remaining, ( - "force_llm_judge must drop the alien_1 cache entry" - ) - assert "k_for_alien_2" in remaining, ( - "entries for other instances must be preserved" - ) - - -def test_regrade_run_force_llm_judge_reinvokes_judge(tmp_path, monkeypatch): - """End-to-end: `regrade_run(..., force_llm_judge=True)` calls the - grader exactly once and the cache is empty afterward for the - filtered instances.""" - from bird_interact_agents import paths as paths_mod - monkeypatch.setattr(paths_mod, "main_checkout_root", lambda: tmp_path) - - run_dir = tmp_path / "results" / "cloud" / "r1" - _write_attempt(run_dir, "alien_1") - cache_path = run_dir / "llm_judge_cache.json" - cache_path.write_text(json.dumps({ - "k_for_alien_1": {"instance_id": "alien_1", "verdict": True}, - })) - - calls: list[dict] = [] - - class StubGrader: - def __call__(self, **kw): - calls.append(kw) - from bird_interact_agents.eval.tolerant_grader import CascadeVerdict - return CascadeVerdict( - n1_original_gold=False, n2_audited_primary=False, - n3_any_audited_variant=False, n4_tie_order=False, - n5_llm_judge=False, n6_numeric_epsilon=False, - n7_trailing_whitespace=False, n8_column_order=False, - n9_case_fold=False, - matched_variant_id=None, novel_reading_judgment=None, - variant_matches=[], rowset_relations=[], - ) - - from bird_interact_agents.eval.regrade import regrade_run - - regrade_run( - run_id="r1", benchmark="mini-interact", run_dir=run_dir, - instance_ids=["alien_1"], force_llm_judge=True, - grader=StubGrader(), repo_root=tmp_path, - ) - assert len(calls) == 1 - remaining = json.loads(cache_path.read_text()) - assert "k_for_alien_1" not in remaining - - # --------------------------------------------------------------------------- # Codex round 6: ``_build_original_sql_index`` MUST accept string-shaped # ``sol_sql``. The mini_interact JSONL carries both shapes (post-DEV-1478 @@ -370,7 +294,7 @@ def __call__(self, *, instance_id, submitted_sql, task_row): regrade_run( run_id="r1", benchmark="mini-interact", run_dir=run_dir, - instance_ids=None, force_llm_judge=False, + instance_ids=None, grader=StubGrader(), repo_root=tmp_path, ) assert len(captured) == 1 @@ -424,7 +348,7 @@ def __call__(self, *, instance_id, submitted_sql, task_row): report = regrade_run( run_id="r1", benchmark="mini-interact", run_dir=run_dir, - instance_ids=None, force_llm_judge=False, + instance_ids=None, grader=StubGrader(), repo_root=tmp_path, ) assert report.regraded == 1 diff --git a/tests/test_tolerant_grader_orchestration.py b/tests/test_tolerant_grader_orchestration.py index 2fc7b16c..a93a1fa4 100644 --- a/tests/test_tolerant_grader_orchestration.py +++ b/tests/test_tolerant_grader_orchestration.py @@ -1092,3 +1092,128 @@ def test_missing_gold_does_not_collapse_to_n4_pass(): assert verdict.n7_trailing_whitespace is False assert verdict.n8_column_order is False assert verdict.n9_case_fold is False + + +# --------------------------------------------------------------------------- +# Codex r13: ``LiteLLMJudge`` parses the model's reply into True / False / +# None per the protocol. The litellm.completion call itself isn't tested +# here (would need network); the decision-parsing helper is, since it's +# the load-bearing piece of the judge's behaviour. +# --------------------------------------------------------------------------- + + +def test_judge_decision_parser_recognizes_accept(): + from bird_interact_agents.eval.tolerant_grader import ( + _parse_judge_decision, + ) + + assert _parse_judge_decision("rationale here\nACCEPT") is True + assert _parse_judge_decision("ACCEPT — defensible reading") is True + # Trailing whitespace / case-insensitive on the last line. + assert _parse_judge_decision("...\nAccept this \n") is True + + +def test_judge_decision_parser_recognizes_reject(): + from bird_interact_agents.eval.tolerant_grader import ( + _parse_judge_decision, + ) + + assert _parse_judge_decision("REJECT") is False + assert _parse_judge_decision("reasoning\nREJECT — wrong column") is False + assert _parse_judge_decision("...\nreject\n") is False + + +def test_judge_decision_parser_inconclusive_returns_none(): + """Anything that doesn't START with ACCEPT or REJECT on the LAST + non-empty line → ``None`` (timeout / inconclusive fall-through per + LLMJudgeProtocol). Caller treats None as N5 fail.""" + from bird_interact_agents.eval.tolerant_grader import ( + _parse_judge_decision, + ) + + assert _parse_judge_decision("") is None + assert _parse_judge_decision("maybe?") is None + assert _parse_judge_decision( + "ACCEPT mentioned mid-line but final says\nunclear", + ) is None + + +def test_litellm_judge_swallows_litellm_errors_and_returns_none(): + """Network / API failures must NOT crash the grader — the judge + returns None per protocol so the cascade falls through cleanly.""" + import bird_interact_agents.eval.tolerant_grader as tg + + judge = tg.LiteLLMJudge(model="bogus/model-that-does-not-exist") + + # Force the litellm call to raise; ``judge`` must catch + return None. + class _Boom(RuntimeError): + pass + + def _raise(*a, **kw): + raise _Boom("simulated litellm failure") + + import sys + import types + fake_litellm = types.ModuleType("litellm") + fake_litellm.completion = _raise # type: ignore[attr-defined] + real_litellm = sys.modules.get("litellm") + sys.modules["litellm"] = fake_litellm + try: + result = judge.judge( + evaluator_prompt="anything", + gold_variants_summary=[], + metadata_anchors=[], + submitted_sql="SELECT 1", + predicted_rows_head=[], + annotation_content_hash="x", + gold_variants_content_hash="y", + instance_id="alien_1", + ) + assert result is None + finally: + if real_litellm is not None: + sys.modules["litellm"] = real_litellm + else: + sys.modules.pop("litellm", None) + + +def test_litellm_judge_parses_real_response_shape(): + """Happy path: a mocked litellm.completion returns the standard + OpenAI-shaped response with ``choices[0].message.content`` → judge + parses the verdict.""" + import bird_interact_agents.eval.tolerant_grader as tg + + judge = tg.LiteLLMJudge(model="anthropic/claude-haiku-4-5-20251001") + + def _fake_completion(**kw): + return { + "choices": [{ + "message": { + "content": "Looks defensible.\nACCEPT", + }, + }], + } + + import sys + import types + fake_litellm = types.ModuleType("litellm") + fake_litellm.completion = _fake_completion # type: ignore[attr-defined] + real_litellm = sys.modules.get("litellm") + sys.modules["litellm"] = fake_litellm + try: + result = judge.judge( + evaluator_prompt="is this defensible?", + gold_variants_summary=[{"variant_id": "primary", "interpretation": "x"}], + metadata_anchors=["foo"], + submitted_sql="SELECT 1", + predicted_rows_head=[(1,)], + annotation_content_hash="x", + gold_variants_content_hash="y", + instance_id="alien_1", + ) + assert result is True + finally: + if real_litellm is not None: + sys.modules["litellm"] = real_litellm + else: + sys.modules.pop("litellm", None) From bc3f006081f946c8dc6c59cf0fabb7e572045ee8 Mon Sep 17 00:00:00 2001 From: Egor Kraev Date: Wed, 3 Jun 2026 09:54:17 +0200 Subject: [PATCH 28/29] process-reviews round 1 (partial): use materialized DB for local grader, hoist data_dir, fix dead test conditional Co-Authored-By: Claude Sonnet 4.6 --- src/bird_interact_agents/cloud/ray_app.py | 19 +++++++------------ src/bird_interact_agents/run.py | 19 +++++++++---------- tests/test_local_run_cascading.py | 4 ++-- 3 files changed, 18 insertions(+), 24 deletions(-) diff --git a/src/bird_interact_agents/cloud/ray_app.py b/src/bird_interact_agents/cloud/ray_app.py index c5f43c3d..54834390 100644 --- a/src/bird_interact_agents/cloud/ray_app.py +++ b/src/bird_interact_agents/cloud/ray_app.py @@ -533,17 +533,15 @@ def _run_one_in_actor( log_tmp = log_dir / "task.log" task_start_ts = time.time() + # `cfg["data_dir"]` is the benchmark's container_data_dir, resolved + # benchmark-aware in `run_pool` (mini → BIRD_DB_PATH, + # livesqlbench → BIRD_LIVESQLBENCH_ROOT). Hoisted out of the try block + # so it is always bound when the grader path runs below. + data_dir = cfg.get("data_dir") or "/data/mini-interact" + try: with fd_capture(log_tmp): try: - # `cfg["data_dir"]` is the benchmark's container_data_dir, - # resolved benchmark-aware in `run_pool` (mini → BIRD_DB_PATH, - # livesqlbench → BIRD_LIVESQLBENCH_ROOT). It is authoritative — - # do NOT re-read BIRD_DB_PATH here, which would route a - # livesqlbench task back to mini-interact if BIRD_DB_PATH ever - # leaked into the actor env (Codex). - data_dir = cfg.get("data_dir") or "/data/mini-interact" - row = asyncio.run( _run_one_task_async( task_data=task_data, @@ -591,7 +589,6 @@ def _run_one_in_actor( # ``cascading_phase1_error``. Uploading the annotation first makes # the row blob the canonical "task fully done, including # annotation" marker. - _grader_data_dir = locals().get("data_dir") annotation_dir = Path(tempfile.mkdtemp(prefix="bird_submission_annot_")) _row_submitted_sql = row.get("submitted_sql") _row_selected_db = ( @@ -613,15 +610,13 @@ def _run_one_in_actor( "before reaching submit; routed to fail-everything " "fallback", ) - if _grader_data_dir is None: - raise RuntimeError("data_dir unbound; grader skipped") ann_path = _grade_one_submission( task_data=task_data, submitted_sql=str(_row_submitted_sql), rows_dir=annotation_dir, run_id=run_id, benchmark=_cloud_benchmark(cfg), - db_path=Path(_grader_data_dir) + db_path=Path(data_dir) / str(_row_selected_db) / f"{_row_selected_db}.sqlite", cost_usd_agent=row.get("usage", {}).get("cost_usd_agent") diff --git a/src/bird_interact_agents/run.py b/src/bird_interact_agents/run.py index c0396de3..cce0baed 100644 --- a/src/bird_interact_agents/run.py +++ b/src/bird_interact_agents/run.py @@ -1126,17 +1126,16 @@ def _grade_local_row(td: dict, r: dict) -> None: ), ) return - # Root the per-task sqlite at the caller-provided ``data_dir`` - # (the same path the agent's SQL executed against) — NOT the - # global ``paths.benchmark_data_root``. Otherwise an alternate - # checkout, a tmp fixture, or a ``BIRD_DB_PATH`` override would - # have the agent and grader disagreeing on schema/data, and a - # correct submission could be marked failing. Mirrors the - # cloud worker, which uses ``cfg["data_dir"]`` (Codex r7). + # Root the per-task sqlite at the materialized copy when available + # (LiveSQLBench tasks: materialize_task_db sets db_file_path to an + # isolated $TMPDIR copy so concurrent runs don't race the shared + # .sqlite). Fall back to data_dir//.sqlite for + # mini-interact (materialize_task_db is a no-op there). + _db_file_path = td.get("db_file_path") per_task_db = ( - Path(data_dir) - / selected_database - / f"{selected_database}.sqlite" + Path(_db_file_path) + if _db_file_path + else Path(data_dir) / selected_database / f"{selected_database}.sqlite" ) try: grade_one_submission( diff --git a/tests/test_local_run_cascading.py b/tests/test_local_run_cascading.py index bdde10fa..af446abf 100644 --- a/tests/test_local_run_cascading.py +++ b/tests/test_local_run_cascading.py @@ -28,8 +28,8 @@ def test_grade_in_place_writes_submission_annotation_per_task(tmp_path): original_gold = "SELECT gold" class FakeExecutor: - def __call__(self, sql, *, db_path, conn): # noqa: ARG002,ARG005 # noqa: ARG002 - return ([(1,)], ["a"]) if sql == submitted else ([(1,)], ["a"]) + def __call__(self, sql, *, db_path, conn): # noqa: ARG002 + return ([(1,)], ["a"]) grade_and_write( rows_dir=rows_dir, From 6b59ba125e208db379c113142cdf3fcc903fb8e3 Mon Sep 17 00:00:00 2001 From: Egor Kraev Date: Wed, 3 Jun 2026 10:00:43 +0200 Subject: [PATCH 29/29] process-reviews: fix __file__-based path in consolidate script (worktree contract) Co-Authored-By: Claude Sonnet 4.6 --- scripts/consolidate_mini_interact_audited.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/consolidate_mini_interact_audited.py b/scripts/consolidate_mini_interact_audited.py index aa121a2e..d3c52869 100644 --- a/scripts/consolidate_mini_interact_audited.py +++ b/scripts/consolidate_mini_interact_audited.py @@ -18,10 +18,10 @@ from __future__ import annotations import json -from pathlib import Path -ROOT = Path(__file__).resolve().parents[1] -AUDITED = ROOT / "audited_gold" +from bird_interact_agents import paths + +AUDITED = paths.audited_gold_root() MINI_INTERACT_OUT = AUDITED / "mini_interact_audited.jsonl" LIVESQLBENCH = AUDITED / "livesqlbench_audited.jsonl"