feat(reports): BIRD-INTERACT-1.0 a-Interact submission generator (DEV-1553)

src/bird_interact_agents/cloud/cli.py

@@ -151,6 +151,15 @@ def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace:
     sp_build = sub.add_parser("build")
     sp_build.add_argument("--force", action="store_true")
 
+    # DEV-1553: `bird-interact-cloud submission` — generate a
+    # BIRD-INTERACT-1.0 a-Interact submission directory from existing
+    # cloud runs. Subparser-registration lives in reports.cli so the
+    # cloud CLI module doesn't drag in the reports package at import time
+    # for non-submission subcommands.
+    from bird_interact_agents.reports import cli as _reports_cli
+
+    _reports_cli.add_subparser(sub)
+
     ns = p.parse_args(argv)
 
     # `--subscription-auth` / `--no-subscription-auth` is required on submit
@@ -359,6 +368,10 @@ def main(argv: Sequence[str] | None = None) -> int:
                 f"{row['mode']}  {row['status']}  {row['done']}/{row['total']}"
             )
         return 0
+    if ns.subcommand == "submission":
+        from bird_interact_agents.reports import cli as _reports_cli
+
+        return _reports_cli.run_submission(ns)
     if ns.subcommand == "build":
         from bird_interact_agents import paths
         from bird_interact_agents.cloud import image

src/bird_interact_agents/paths.py

@@ -294,3 +294,20 @@ def results_root() -> Path:
 def benchmarks_root() -> Path:
     """pytest-benchmark output (committed)."""
     return main_checkout_root() / ".benchmarks"
+
+
+def reports_root() -> Path:
+    """BIRD-INTERACT-1.0 submission reports (DEV-1553). Worktree-safe.
+
+    Honors ``BIRD_REPORTS_ROOT``; otherwise the main checkout's
+    ``reports/`` so all worktree-generated submissions aggregate in one
+    place. Creates the directory lazily (mirrors ``runs_root``).
+    """
+    override = os.environ.get("BIRD_REPORTS_ROOT")
+    if override:
+        p = Path(override).expanduser()
+        p.mkdir(parents=True, exist_ok=True)
+        return p
+    path = main_checkout_root() / "reports"
+    path.mkdir(parents=True, exist_ok=True)
+    return path

src/bird_interact_agents/reports/__init__.py

@@ -0,0 +1,13 @@
+"""BIRD-INTERACT-1.0 submission report generator (DEV-1553).
+
+Converts an existing cloud run (or per-instance selection across runs)
+into the JSONL + manifest layout that the leaderboard at
+``bird.bench25@gmail.com`` expects. Reconstructs ``prompt_flow`` from
+existing ``runs/<bench>/<db>/<id>/<run-id>.trajectory.json`` and
+``results/<bench>/cloud/<run-id>/results.db`` — no harness-runtime
+changes required.
+
+a-Interact custom-agent setting only. The Section VI Universal Cost
+Scheme is THE contract: fixed ``ask=2 / submit=3 / execute=1``;
+everything else token-aware via the ``in<250 AND out<1000`` rule.
+"""

src/bird_interact_agents/reports/action_canonicalize.py

@@ -0,0 +1,110 @@
+"""MCP-tool-name → upstream-canonical action-string mapping.
+
+The bird-interact-tools MCP wrappers (and the harness's top-level
+``ask_user`` tool) map to the same short action names upstream's
+``eval_react_bird_interact.py`` emits: ``submit``, ``execute``, ``ask``,
+``get_schema``, ``get_all_column_meanings``, ``get_column_meaning``,
+``get_all_external_knowledge_names``, ``get_knowledge_definition``,
+``get_all_knowledge_definitions``. Anything else falls through to
+``<tool_name>(<json_args>)`` so the trajectory is preserved verbatim.
+
+Section VI prose names (``ask_user`` / ``submit_sql`` / ``execute_sql``)
+are exposed via ``SECTION_VI_NAME_TO_CANONICAL`` for documentation.
+"""
+
+from __future__ import annotations
+
+import json
+from typing import Any
+
+
+# (Tool name, canonical name, has-args). Order: most-specific first.
+# The bird-interact-tools MCP server exposes `ask_user` AND
+# `submit_query` AND `execute_sql` under the `mcp__bird-interact-tools__`
+# prefix; the bare `ask_user` entry covers harnesses (e.g. the claude_sdk
+# top-level tool) that register the tool directly. Both must canonicalise
+# to the Section VI fixed-cost names.
+_KNOWN: list[tuple[str, str]] = [
+    ("mcp__bird-interact-tools__submit_query", "submit"),
+    ("mcp__bird-interact-tools__execute_sql", "execute"),
+    ("mcp__bird-interact-tools__ask_user", "ask"),
+    ("ask_user", "ask"),
+    ("mcp__bird-interact-tools__get_schema", "get_schema"),
+    ("mcp__bird-interact-tools__get_all_column_meanings", "get_all_column_meanings"),
+    ("mcp__bird-interact-tools__get_column_meaning", "get_column_meaning"),
+    (
+        "mcp__bird-interact-tools__get_all_external_knowledge_names",
+        "get_all_external_knowledge_names",
+    ),
+    (
+        "mcp__bird-interact-tools__get_knowledge_definition",
+        "get_knowledge_definition",
+    ),
+    (
+        "mcp__bird-interact-tools__get_all_knowledge_definitions",
+        "get_all_knowledge_definitions",
+    ),
+]
+
+CANONICAL_NAME_MAP: dict[str, str] = dict(_KNOWN)
+
+# Section VI prose ↔ upstream short form. Documented here for clarity.
+SECTION_VI_NAME_TO_CANONICAL: dict[str, str] = {
+    "ask_user": "ask",
+    "submit_sql": "submit",
+    "execute_sql": "execute",
+}
+
+
+def _compact(obj: Any) -> str:
+    return json.dumps(obj, separators=(",", ":"), sort_keys=False)
+
+
+def _sql_from_input(tool_input: dict[str, Any]) -> str:
+    """submit_query carries SQL in ``query_json`` (slayer mode) or
+    ``query`` (raw mode); execute_sql uses ``query``."""
+    for key in ("query_json", "query", "sql"):
+        if key in tool_input:
+            return str(tool_input[key])
+    return _compact(tool_input)
+
+
+def canonicalize_action(tool_name: str, tool_input: dict[str, Any]) -> str:
+    """Return the canonical action string for one tool call.
+
+    * ``submit_query`` / ``execute_sql`` → ``submit(<sql>)`` / ``execute(<sql>)``
+    * ``ask_user`` → ``ask(<question>)``
+    * Zero-arg helpers → ``<canonical>()``.
+    * Arg-bearing helpers (``get_column_meaning`` /
+      ``get_knowledge_definition``) → ``<canonical>(<compact_json>)``.
+    * Unknown tools → ``<tool_name>(<compact_json>)``.
+    """
+    if tool_name == "mcp__bird-interact-tools__submit_query":
+        return f"submit({_sql_from_input(tool_input)})"
+    if tool_name == "mcp__bird-interact-tools__execute_sql":
+        return f"execute({_sql_from_input(tool_input)})"
+    if tool_name in ("ask_user", "mcp__bird-interact-tools__ask_user"):
+        question = tool_input.get("question", _compact(tool_input))
+        return f"ask({question})"
+
+    canonical = CANONICAL_NAME_MAP.get(tool_name)
+    if canonical is None:
+        return f"{tool_name}({_compact(tool_input)})"
+    if not tool_input:
+        return f"{canonical}()"
+    return f"{canonical}({_compact(tool_input)})"
+
+
+def action_args_string(tool_name: str, tool_input: dict[str, Any]) -> str:
+    """The string that ``count_tokens`` measures for ``action_input_tokens``.
+
+    Per the spec: SQL string for submit/execute; question string for
+    ask; ``compact json.dumps`` for everything else.
+    """
+    if tool_name == "mcp__bird-interact-tools__submit_query":
+        return _sql_from_input(tool_input)
+    if tool_name == "mcp__bird-interact-tools__execute_sql":
+        return _sql_from_input(tool_input)
+    if tool_name in ("ask_user", "mcp__bird-interact-tools__ask_user"):
+        return str(tool_input.get("question", _compact(tool_input)))
+    return _compact(tool_input)

src/bird_interact_agents/reports/adapters/__init__.py

@@ -0,0 +1,67 @@
+"""Per-framework trajectory→Turn adapters.
+
+Real cloud runs persist ``framework="claude_sdk"`` in ``results.db.
+run_metadata`` (that's the CLI flag of ``bird-interact-cloud submit``);
+the SLayer-vs-raw / one-shot-vs-a-interact distinction lives in the
+``query_mode`` and ``mode`` columns. The submission report is supported
+only on the ``(framework="claude_sdk", query_mode="slayer")`` combo:
+
+* ``query_mode="slayer"``: Anthropic SDK messages are persisted as
+  nested dicts (``data`` is a dict with ``content``, ``model``, …) and
+  the shared dict-walker handles them.
+* ``query_mode="raw"``: trajectory entries' ``data`` field is a
+  Python-repr STRING; needs a separate string-repr parser that isn't
+  written yet.
+
+We also accept ``framework="claude_sdk_otf"`` /
+``"claude_sdk_otf_ainteract"`` for forward-compatibility with any future
+run-metadata schema that promotes the internal agent name to a
+persisted field — both share the same dict walker.
+
+Other frameworks (pydantic_ai*, smolagents, agno, mcp_agent) are out of
+scope for DEV-1553.
+"""
+
+from __future__ import annotations
+
+from typing import Callable, Iterable
+
+from bird_interact_agents.reports.adapters.base import Turn
+from bird_interact_agents.reports.adapters.claude_sdk_otf import (
+    walk_trajectory as _claude_sdk_otf_walk,
+)
+
+
+# Keyed by (framework, query_mode). All entries point at the dict-walker;
+# the only thing that varies is which combinations we accept.
+_REGISTRY: dict[tuple[str, str], Callable[[list[dict]], Iterable[Turn]]] = {
+    ("claude_sdk", "slayer"): _claude_sdk_otf_walk,
+    ("claude_sdk_otf", "slayer"): _claude_sdk_otf_walk,
+    ("claude_sdk_otf_ainteract", "slayer"): _claude_sdk_otf_walk,
+}
+
+
+class UnknownFrameworkError(ValueError):
+    pass
+
+
+def get_adapter(
+    framework: str, *, query_mode: str = "slayer"
+) -> Callable[[list[dict]], Iterable[Turn]]:
+    """Resolve ``(framework, query_mode)`` to its trajectory walker.
+
+    Raises ``UnknownFrameworkError`` for any unsupported combo — the
+    error message spells out the supported list so the failure surfaces
+    with actionable guidance, not a mid-walk AttributeError.
+    """
+    try:
+        return _REGISTRY[(framework, query_mode)]
+    except KeyError:
+        raise UnknownFrameworkError(
+            f"no submission-report adapter registered for "
+            f"(framework={framework!r}, query_mode={query_mode!r}); "
+            f"supported: {sorted(_REGISTRY)}"
+        )
+
+
+__all__ = ["Turn", "get_adapter", "UnknownFrameworkError"]

src/bird_interact_agents/reports/adapters/base.py

@@ -0,0 +1,38 @@
+"""Shared ``Turn`` dataclass for trajectory adapters."""
+
+from __future__ import annotations
+
+from typing import Any
+
+from pydantic import BaseModel
+
+
+class Turn(BaseModel):
+    """One agent assistant-message + its emitted action + the resulting
+    observation. Framework-agnostic intermediate representation.
+    """
+
+    # Agent model used for this turn (e.g. ``claude-opus-4-7``). Most
+    # runs hold this constant across turns, but we carry it per-turn so
+    # multi-model trajectories work the day someone files one.
+    model: str
+
+    # The exact text the agent saw to produce this turn. For turn 0 this
+    # is the initial task statement; for later turns it's the
+    # concatenation of every tool_result + free UserMessage text seen
+    # between the previous tool_use and this one.
+    prompt: str
+
+    # Rendered raw assistant message. Carries thinking + text + tool_use
+    # JSON when include_thinking=True; thinking stripped otherwise.
+    response_raw: str
+
+    # Tool call.
+    tool_name: str
+    tool_input: dict[str, Any]
+    tool_use_id: str
+
+    # Tool result text (collapsed into a single string).
+    observation: str
+
+    model_config = {"arbitrary_types_allowed": True}