diff --git a/benchmarks/harbor-buzz-orchestra/manifests/tb-cobol-sonnet-haiku-criteria.yaml b/benchmarks/harbor-buzz-orchestra/manifests/tb-cobol-sonnet-haiku-criteria.yaml new file mode 100644 index 0000000000..aa5ec28417 --- /dev/null +++ b/benchmarks/harbor-buzz-orchestra/manifests/tb-cobol-sonnet-haiku-criteria.yaml @@ -0,0 +1,49 @@ +# Treatment cell paired with tb-cobol-sonnet-haiku.yaml. Identical staffing, +# endpoints, prices, and budget; the only two differences are the intervention: +# 1. the -criteria personas, which add the acceptance-criteria rules (write +# the task's stated conditions into `todo`, then show the command and its +# literal output for each before finishing), and +# 2. todo_enforcement: true, which turns on buzz-dev-mcp's _Stop hook so an +# agent cannot end its turn while a todo item is still open. +# Run this against the control on the same task set to read the effect. Keep +# every other field byte-identical to the control or the comparison is +# confounded. +schema_version: "1" +condition: tb-cobol-sonnet46-2haiku45-criteria +todo_enforcement: true +roster: + - id: orch + kind: orchestrator + role: lead + count: 1 + endpoint: claude-sonnet-4-6 + model_revision: claude-sonnet-4-6 + prompt: + path: personas/orchestrator-tb-criteria.md + sha256: b208b077d9758f1343d14c6a4a2fae8bfad5625905d43f085d209f3a55d463ea + generation: + max_output_tokens: 4096 + context_window_tokens: 200000 + - id: worker + kind: worker + role: implementer + count: 2 + endpoint: claude-haiku-4-5 + model_revision: claude-haiku-4-5 + prompt: + path: personas/worker-tb-criteria.md + sha256: 2df5fad04298ad85dd3e661e69668b7e660d95fa1a30fb33d62282d784746d97 + generation: + max_output_tokens: 4096 + context_window_tokens: 200000 +prices: + claude-sonnet-4-6: + input_per_million_usd: 3 + cached_input_per_million_usd: 0.3 + output_per_million_usd: 15 + claude-haiku-4-5: + input_per_million_usd: 1 + cached_input_per_million_usd: 0.1 + output_per_million_usd: 5 +trial_budget: + timeout_seconds: 900 diff --git a/benchmarks/harbor-buzz-orchestra/personas/orchestrator-tb-criteria.md b/benchmarks/harbor-buzz-orchestra/personas/orchestrator-tb-criteria.md new file mode 100644 index 0000000000..c9f3dfa3ec --- /dev/null +++ b/benchmarks/harbor-buzz-orchestra/personas/orchestrator-tb-criteria.md @@ -0,0 +1,54 @@ +# Orchestrator — Terminal-Bench team + +You are the orchestrator of a small team solving a terminal task. You do not +run commands yourself; your workers do. You coordinate over a Buzz channel. +Your team, your channel id, and the user you report to are listed in the +"Your team" section below. + +Your `shell` tool has the `buzz` CLI on PATH, already authenticated as +you. Nothing you write is visible to anyone unless you publish it: every +message — step assignments, verification requests, the final `DONE:` +report — must be sent with +`buzz messages send --channel --content `. Your turn is +not complete until you have published your message. Do not use the shell +for task work — that is your workers' job. + +Tasks arrive as a channel message from the user @mentioning you. Address +each assignment to a specific worker by @mention, exactly one worker per +step. You may assign independent steps to different workers, but never give +two workers overlapping or conflicting work — they share one task +environment and one filesystem. + +Rules: +1. Read the task instruction. Break it into the smallest concrete steps. +2. Assign each step to a worker with an @mention. One step per message. + State the exact goal and the success check, not just the command to run. + Relay the task's requirements verbatim — use the paths the task states, + and do not add constraints the task does not state (paths, encodings, + byte-level rules). Where the task is silent, let standard tool defaults + apply. +3. Wait for the worker's report before assigning the next dependent step. +4. When a worker reports output, verify it against the task's success + criteria before moving on: assign a verification step that runs the + task's own success check and shows real output. Assign each verification + step to a different worker than the one whose work is being verified — + independent verification, never self-review. Do not report completion on + a worker's claim alone. +5. When the task is complete and verified, report back to the user: publish + a final message starting with `DONE:` that @mentions the user and + summarizes what was produced and how it was verified. The task is not + finished until this message is published — never conclude silently. +6. Before you assign the first step, write the task's stated success + conditions into your `todo` list — one item per condition, quoted from the + task. Every path, every filename, every count, every threshold, and every + "all", "each", and "both". These items are the definition of done; a step + is not a substitute for a condition. +7. Before you publish `DONE:`, put next to each condition the command a worker + ran and its literal output. If an output contradicts a condition, you are + not done: assign the work to close it. Do not explain the contradiction + away, and never accept a check that is easier to pass than the one the task + stated — "it imports" is not "it works", and "no errors reported" is not + "no leaks reported". + +Keep messages short. Never fabricate command output. If a worker's report is +ambiguous, ask them to re-run with the exact verification command. diff --git a/benchmarks/harbor-buzz-orchestra/personas/worker-tb-criteria.md b/benchmarks/harbor-buzz-orchestra/personas/worker-tb-criteria.md new file mode 100644 index 0000000000..8481e966c5 --- /dev/null +++ b/benchmarks/harbor-buzz-orchestra/personas/worker-tb-criteria.md @@ -0,0 +1,41 @@ +# Worker — Terminal-Bench team + +You are a worker agent with terminal access, coordinating over a Buzz +channel. Your team, your channel id, and your orchestrator are listed in +the "Your team" section below. + +You work directly in the task environment: your `shell` tool runs +commands in it, and your file tools read and edit its files. The same +shell has the `buzz` CLI on PATH, already authenticated as you; reports go +to the channel with +`buzz messages send --channel --content `. Your turn is +not complete until you have published your report. + +The orchestrator only wakes for messages that @mention it. Every report +you publish must start with an @mention of the agent that assigned you the +step (use their name exactly as it appears in their message). If the +assignment's event id is visible in your context, also pass +`--reply-to ` to thread the report. A report that mentions +nobody is invisible and the task will stall. + +Rules: +1. Act only on steps assigned to you by the orchestrator's @mention. If a + message assigns a step to a different worker, ignore it. +2. Execute the requested step in the terminal BEFORE writing any report. + Never describe output you have not yet produced. Prefer the smallest + command that achieves the stated goal. Use the paths the task or the + assignment states; do not invent paths. +3. Report back in one message: the command you ran, its exit code, and the + relevant output (trimmed, never invented). +4. If a command fails, report the failure verbatim and stop — do not + improvise a different approach without the orchestrator's direction. +5. Never claim success without showing the verifying output. +6. Before you start, write the step's stated success conditions into your + `todo` list — one item per condition, quoted from the assignment. Every + path, every filename, every count, every threshold, and every "all", + "each", and "both". +7. Before you report, put next to each condition the command you ran and its + literal output. If an output contradicts a condition, you are not done: + keep working. Do not explain the contradiction away, and never substitute a + check that is easier to pass than the one stated — "it imports" is not "it + works", and "no errors reported" is not "no leaks reported". diff --git a/benchmarks/harbor-buzz-orchestra/src/harbor_buzz_orchestra/container_runtime.py b/benchmarks/harbor-buzz-orchestra/src/harbor_buzz_orchestra/container_runtime.py index 149a5295a7..4d25d4341b 100644 --- a/benchmarks/harbor-buzz-orchestra/src/harbor_buzz_orchestra/container_runtime.py +++ b/benchmarks/harbor-buzz-orchestra/src/harbor_buzz_orchestra/container_runtime.py @@ -38,6 +38,11 @@ FORWARDER_LOG = f"{REMOTE_LOGS}/relay-forwarder.log" # How many done-poll iterations between in-container liveness probes. LIVENESS_EVERY = 10 +# How many times _Stop may refuse end_turn before the agent is allowed to +# finish anyway, when todo_enforcement is on. buzz-agent's own default is 3; +# trials run against a hard wall clock and 13% of observed failures were +# AgentTimeoutError, so the benchmark buys fewer retries than production. +TODO_STOP_MAX_REJECTIONS = 2 class RuntimeLaunchError(RuntimeError): @@ -150,6 +155,7 @@ async def run( credential=credential, agent_class=classes[credential.agent_id], trial_dir=trial_dir, + todo_enforcement=manifest.todo_enforcement, ) ) await self._wait_for_agents_ready( @@ -176,6 +182,7 @@ async def run( "completion_message": final_message["content"], "agent_runtime": "in-container", "agent_hints_enabled": False, + "agent_todo_enforcement": manifest.todo_enforcement, "task_seed": "user-identity-prompt", "agent_max_rounds": { credential.agent_id: ( @@ -296,6 +303,7 @@ async def _launch_agent( credential: AgentCredential, agent_class: AgentClass, trial_dir: Path, + todo_enforcement: bool = False, ) -> _Agent: if not credential.llm_endpoint: raise RuntimeLaunchError("credential llm_endpoint must not be empty") @@ -324,6 +332,7 @@ async def _launch_agent( agent_class=agent_class, endpoint=endpoint, remote_prompt=remote_prompt, + todo_enforcement=todo_enforcement, ) command = ( f"{shlex.quote(f'{REMOTE_BIN}/buzz-acp')} dict[str, str]: """The desktop-launch environment: real acp/agent/dev-mcp wiring.""" - return { + env = { **endpoint.env, "BUZZ_RELAY_URL": trial.relay_ws_url, "BUZZ_PRIVATE_KEY": credential.nostr_secret_key, @@ -381,6 +391,15 @@ def _agent_env( "BUZZ_AGENT_NO_HINTS": "1", endpoint.api_key_env: credential.llm_api_key, } + if todo_enforcement: + # buzz-dev-mcp already exposes the `todo` tool in every cell, but + # its _Stop/_PostCompact hooks are never called unless a server is + # named here, which leaves the todo list advisory: an agent can + # open items and still end its turn. "*" enables hooks for every + # attached MCP server. + env["MCP_HOOK_SERVERS"] = "*" + env["BUZZ_AGENT_STOP_MAX_REJECTIONS"] = str(TODO_STOP_MAX_REJECTIONS) + return env # -- lifecycle ------------------------------------------------------------- diff --git a/benchmarks/harbor-buzz-orchestra/src/harbor_buzz_orchestra/manifest.py b/benchmarks/harbor-buzz-orchestra/src/harbor_buzz_orchestra/manifest.py index 309c0a5770..22f99ff219 100644 --- a/benchmarks/harbor-buzz-orchestra/src/harbor_buzz_orchestra/manifest.py +++ b/benchmarks/harbor-buzz-orchestra/src/harbor_buzz_orchestra/manifest.py @@ -92,6 +92,13 @@ class ExperimentManifest(StrictModel): roster: tuple[AgentClass, ...] = Field(min_length=1) prices: dict[str, Price] trial_budget: TrialBudget + # Enable buzz-dev-mcp's _Stop/_PostCompact lifecycle hooks, which make the + # `todo` tool binding instead of advisory: _Stop refuses end_turn while any + # item is open, and _PostCompact re-injects the list after a context + # handoff. Off by default, matching buzz-agent's own default (hook_servers + # is HookServers::None when MCP_HOOK_SERVERS is unset), so this stays an + # opt-in experimental condition rather than a silent change to every cell. + todo_enforcement: bool = False metadata: dict[str, Any] = Field(default_factory=dict) @model_validator(mode="after") diff --git a/benchmarks/harbor-buzz-orchestra/tests/test_container_runtime.py b/benchmarks/harbor-buzz-orchestra/tests/test_container_runtime.py index ebf0eb4b5d..475d2c0cbb 100644 --- a/benchmarks/harbor-buzz-orchestra/tests/test_container_runtime.py +++ b/benchmarks/harbor-buzz-orchestra/tests/test_container_runtime.py @@ -290,6 +290,37 @@ async def test_launch_wires_the_desktop_environment(tmp_path, configured, expect ) +@pytest.mark.parametrize( + ("todo_enforcement", "expected_hooks"), [(False, None), (True, "*")] +) +async def test_todo_enforcement_gates_the_mcp_lifecycle_hooks( + tmp_path, todo_enforcement, expected_hooks +): + manifest = write_manifest(tmp_path) + orch = credential("orch-1", "orchestrator", "orch-model") + trial = trial_handle((orch,)) + environment = Environment( + responses={"buzz-acp": ExecResult(stdout="4242\n", stderr="", return_code=0)} + ) + await runtime(tmp_path)._launch_agent( + environment=environment, + trial=trial, + credential=orch, + agent_class=manifest.roster[0], + trial_dir=tmp_path, + todo_enforcement=todo_enforcement, + ) + _, env = environment.commands[-1] + # buzz-dev-mcp is attached either way, so the `todo` tool is callable in + # both arms; only the _Stop/_PostCompact enforcement is gated. + assert env["BUZZ_ACP_MCP_COMMAND"] == f"{REMOTE_BIN}/buzz-dev-mcp" + assert env.get("MCP_HOOK_SERVERS") == expected_hooks + if todo_enforcement: + assert env["BUZZ_AGENT_STOP_MAX_REJECTIONS"] == "2" + else: + assert "BUZZ_AGENT_STOP_MAX_REJECTIONS" not in env + + def test_runtime_rejects_unbounded_agent_rounds(tmp_path): with pytest.raises(ValueError, match="positive"): runtime(tmp_path, max_agent_rounds=0) diff --git a/benchmarks/harbor-buzz-orchestra/tests/test_manifest.py b/benchmarks/harbor-buzz-orchestra/tests/test_manifest.py index f8230036b3..ac22b6028b 100644 --- a/benchmarks/harbor-buzz-orchestra/tests/test_manifest.py +++ b/benchmarks/harbor-buzz-orchestra/tests/test_manifest.py @@ -27,6 +27,15 @@ def test_hash_changes_when_staffing_changes(manifest_data): assert ExperimentManifest.load(changed).sha256 != first.sha256 +def test_todo_enforcement_defaults_off_and_changes_condition_identity(manifest_data): + control = ExperimentManifest.load(manifest_data) + assert control.todo_enforcement is False + treatment = ExperimentManifest.load({**manifest_data, "todo_enforcement": True}) + assert treatment.todo_enforcement is True + # The flag is part of the condition, so the two arms hash differently. + assert treatment.sha256 != control.sha256 + + @pytest.mark.parametrize( ("mutation", "match"), [