From 771be9978ad5416b63acbd83b280eeeec6f82672 Mon Sep 17 00:00:00 2001 From: kywch Date: Wed, 12 Aug 2026 10:17:28 -0700 Subject: [PATCH 1/3] fix(gemini): strip Google model prefix --- src/benchflow/acp/runtime.py | 2 ++ tests/test_acp.py | 20 +++++++++++--------- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/src/benchflow/acp/runtime.py b/src/benchflow/acp/runtime.py index 0bf808155..052df2875 100644 --- a/src/benchflow/acp/runtime.py +++ b/src/benchflow/acp/runtime.py @@ -267,6 +267,8 @@ def _format_acp_model(model: str, agent: str) -> str: return f"litellm/{model}" return model if find_provider(model) else bare if not agent_cfg or agent_cfg.acp_model_format != "provider/model": + if agent == "gemini" and bare.startswith("google/gemini-"): + return bare.removeprefix("google/") return bare # Already has a slash — assume it's provider/model already if "/" in bare: diff --git a/tests/test_acp.py b/tests/test_acp.py index 783838a4a..8c7c9cc8c 100644 --- a/tests/test_acp.py +++ b/tests/test_acp.py @@ -1110,22 +1110,24 @@ def _make_mocks(): @pytest.mark.asyncio @pytest.mark.parametrize( - "model_in, expected_model", + "agent, model_in, expected_model", [ # Registered vllm/ prefix stripped; HF org/model intact — this is # what pi-acp and other ACP agents need for downstream routing. - ("vllm/Qwen/Qwen3.5-35B-A3B", "Qwen/Qwen3.5-35B-A3B"), - ("zai/glm-5", "glm-5"), + ("test-agent", "vllm/Qwen/Qwen3.5-35B-A3B", "Qwen/Qwen3.5-35B-A3B"), + ("test-agent", "zai/glm-5", "glm-5"), # Bare HF ID (no registered prefix) passes through unchanged. - ("Qwen/Qwen3-Coder", "Qwen/Qwen3-Coder"), + ("test-agent", "Qwen/Qwen3-Coder", "Qwen/Qwen3-Coder"), # Vertex ADC provider — prefix stripped like any other registered one. - ("anthropic-vertex/claude-sonnet-4-6", "claude-sonnet-4-6"), + ("test-agent", "anthropic-vertex/claude-sonnet-4-6", "claude-sonnet-4-6"), # No prefix at all — unchanged. - ("claude-sonnet-4-6", "claude-sonnet-4-6"), + ("test-agent", "claude-sonnet-4-6", "claude-sonnet-4-6"), + # Gemini CLI expects a bare model ID, unlike models.dev agents. + ("gemini", "google/gemini-3.1-flash-lite-preview", "gemini-3.1-flash-lite-preview"), ], - ids=["vllm-hf", "zai", "bare-hf", "vertex", "no-prefix"], + ids=["vllm-hf", "zai", "bare-hf", "vertex", "no-prefix", "gemini-google"], ) - async def test_model_id_selection(self, model_in, expected_model, tmp_path): + async def test_model_id_selection(self, agent, model_in, expected_model, tmp_path): from benchflow.acp.runtime import connect_acp mock_acp = self._make_mocks() @@ -1137,7 +1139,7 @@ async def test_model_id_selection(self, model_in, expected_model, tmp_path): ): await connect_acp( env=mock_env, - agent="test-agent", + agent=agent, agent_launch="test-agent", agent_env={}, sandbox_user=None, From 22152b2e07c1a19feeb9e87cfd15f991b6af2790 Mon Sep 17 00:00:00 2001 From: kywch Date: Wed, 12 Aug 2026 10:24:08 -0700 Subject: [PATCH 2/3] style: format Gemini ACP test --- tests/test_acp.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/test_acp.py b/tests/test_acp.py index 8c7c9cc8c..a76a30264 100644 --- a/tests/test_acp.py +++ b/tests/test_acp.py @@ -1123,7 +1123,11 @@ def _make_mocks(): # No prefix at all — unchanged. ("test-agent", "claude-sonnet-4-6", "claude-sonnet-4-6"), # Gemini CLI expects a bare model ID, unlike models.dev agents. - ("gemini", "google/gemini-3.1-flash-lite-preview", "gemini-3.1-flash-lite-preview"), + ( + "gemini", + "google/gemini-3.1-flash-lite-preview", + "gemini-3.1-flash-lite-preview", + ), ], ids=["vllm-hf", "zai", "bare-hf", "vertex", "no-prefix", "gemini-google"], ) From bc9a1c362dd076bba8a03019d261033ab7ec4685 Mon Sep 17 00:00:00 2001 From: kywch Date: Wed, 12 Aug 2026 10:30:32 -0700 Subject: [PATCH 3/3] fix(judge): allow scratch edits in native titles --- tests/integration/agent_judge.py | 4 +++- tests/test_judge_robustness.py | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/integration/agent_judge.py b/tests/integration/agent_judge.py index 04abe3c7b..652f2831b 100644 --- a/tests/integration/agent_judge.py +++ b/tests/integration/agent_judge.py @@ -130,6 +130,7 @@ def _mask_scratch_paths(command: str) -> str: # tampering. Only the command after ``: $ `` is the agent's actual action, so the # execute scan must strip the description first. _ACP_EXECUTE_PREFIX_RE = re.compile(r".*?: \$ ", re.DOTALL) +_ACP_EDIT_PATH_RE = re.compile(r"\bEditing\s+(?P\S+)\s*$") def _acp_execute_command(title: str) -> str: @@ -155,7 +156,8 @@ def _acp_write_target(title: str) -> str: stripped = title.strip() prefix = "file_editor:" if not stripped.startswith(prefix): - return title + match = _ACP_EDIT_PATH_RE.search(stripped) + return match.group("path") if match else title # Titles carry trailing prose after the JSON payload (observed live: # ``file_editor: {...}: Editing /tmp/test_rnn.py``), so parse the LEADING # object with raw_decode instead of json.loads — a whole-title fallback diff --git a/tests/test_judge_robustness.py b/tests/test_judge_robustness.py index 14a7573bd..95336c337 100644 --- a/tests/test_judge_robustness.py +++ b/tests/test_judge_robustness.py @@ -126,6 +126,7 @@ def test_scan_verifier_tamper(event, should_flag): ), False, ), + (_native("edit", "Create test script: Editing /tmp/test_rnn.py"), False), ( _native( "edit",