From 1fb7e21328f19a67b6fe8b74e132d03009242a7d Mon Sep 17 00:00:00 2001 From: Haoqian Li Date: Fri, 26 Jun 2026 15:58:03 +0800 Subject: [PATCH] fix(models): bound provider-facing observation output --- src/minisweagent/models/utils/actions_text.py | 4 +- .../models/utils/actions_toolcall.py | 4 +- .../models/utils/actions_toolcall_response.py | 4 +- src/minisweagent/models/utils/observations.py | 39 ++++++++ tests/models/test_observation_truncation.py | 89 +++++++++++++++++++ 5 files changed, 137 insertions(+), 3 deletions(-) create mode 100644 src/minisweagent/models/utils/observations.py create mode 100644 tests/models/test_observation_truncation.py diff --git a/src/minisweagent/models/utils/actions_text.py b/src/minisweagent/models/utils/actions_text.py index 3129a1db4..8a5fbdf2f 100644 --- a/src/minisweagent/models/utils/actions_text.py +++ b/src/minisweagent/models/utils/actions_text.py @@ -9,6 +9,7 @@ from jinja2 import StrictUndefined, Template from minisweagent.exceptions import FormatError +from minisweagent.models.utils.observations import bounded_observation_output from minisweagent.models.utils.openai_multimodal import expand_multimodal_content @@ -50,8 +51,9 @@ def format_observation_messages( """Format execution outputs into user observation messages.""" results = [] for output in outputs: + bounded_output = bounded_observation_output(output) content = Template(observation_template, undefined=StrictUndefined).render( - output=output, **(template_vars or {}) + output=bounded_output, **(template_vars or {}) ) msg: dict = { "role": "user", diff --git a/src/minisweagent/models/utils/actions_toolcall.py b/src/minisweagent/models/utils/actions_toolcall.py index 98812e9e0..24afdb236 100644 --- a/src/minisweagent/models/utils/actions_toolcall.py +++ b/src/minisweagent/models/utils/actions_toolcall.py @@ -6,6 +6,7 @@ from jinja2 import StrictUndefined, Template from minisweagent.exceptions import FormatError +from minisweagent.models.utils.observations import bounded_observation_output from minisweagent.models.utils.openai_multimodal import expand_multimodal_content BASH_TOOL = { @@ -88,8 +89,9 @@ def format_toolcall_observation_messages( padded_outputs = outputs + [not_executed] * (len(actions) - len(outputs)) results = [] for action, output in zip(actions, padded_outputs): + bounded_output = bounded_observation_output(output) content = Template(observation_template, undefined=StrictUndefined).render( - output=output, **(template_vars or {}) + output=bounded_output, **(template_vars or {}) ) msg = { "content": content, diff --git a/src/minisweagent/models/utils/actions_toolcall_response.py b/src/minisweagent/models/utils/actions_toolcall_response.py index 79e52c9fc..349abfef1 100644 --- a/src/minisweagent/models/utils/actions_toolcall_response.py +++ b/src/minisweagent/models/utils/actions_toolcall_response.py @@ -6,6 +6,7 @@ from jinja2 import StrictUndefined, Template from minisweagent.exceptions import FormatError +from minisweagent.models.utils.observations import bounded_observation_output # OpenRouter/OpenAI Responses API uses a flat structure (no nested "function" key) BASH_TOOL_RESPONSE_API = { @@ -116,8 +117,9 @@ def format_toolcall_observation_messages( padded_outputs = outputs + [not_executed] * (len(actions) - len(outputs)) results = [] for action, output in zip(actions, padded_outputs): + bounded_output = bounded_observation_output(output) content = Template(observation_template, undefined=StrictUndefined).render( - output=output, **(template_vars or {}) + output=bounded_output, **(template_vars or {}) ) msg: dict = { "extra": { diff --git a/src/minisweagent/models/utils/observations.py b/src/minisweagent/models/utils/observations.py new file mode 100644 index 000000000..0d0763255 --- /dev/null +++ b/src/minisweagent/models/utils/observations.py @@ -0,0 +1,39 @@ +"""Utilities for preparing execution observations for model providers.""" + +MAX_OBSERVATION_OUTPUT_BYTES = 2000 + + +def truncate_observation_text(text: object, *, max_bytes: int = MAX_OBSERVATION_OUTPUT_BYTES) -> str: + """Bound provider-facing command output while preserving both ends.""" + text = "" if text is None else str(text) + encoded = text.encode("utf-8") + if len(encoded) <= max_bytes: + return text + + omitted = len(encoded) - max_bytes + while True: + marker = f"\n...[output truncated, {omitted} bytes omitted]...\n" + marker_bytes = len(marker.encode("utf-8")) + remaining = max(max_bytes - marker_bytes, 0) + new_omitted = len(encoded) - remaining + if new_omitted == omitted: + break + omitted = new_omitted + + if marker_bytes > max_bytes: + return marker.encode("utf-8")[:max_bytes].decode("utf-8", errors="ignore") + + remaining = max(max_bytes - marker_bytes, 0) + head_bytes = remaining // 2 + tail_bytes = remaining - head_bytes + + head = encoded[:head_bytes].decode("utf-8", errors="ignore") + tail = encoded[-tail_bytes:].decode("utf-8", errors="ignore") if tail_bytes else "" + return f"{head}{marker}{tail}" + + +def bounded_observation_output(output: dict) -> dict: + """Return a shallow copy with only the provider-facing output truncated.""" + bounded = dict(output) + bounded["output"] = truncate_observation_text(bounded.get("output", "")) + return bounded diff --git a/tests/models/test_observation_truncation.py b/tests/models/test_observation_truncation.py new file mode 100644 index 000000000..2be34951c --- /dev/null +++ b/tests/models/test_observation_truncation.py @@ -0,0 +1,89 @@ +import json +from pathlib import Path + +import yaml + +from minisweagent.models.litellm_model import LitellmModel +from minisweagent.models.utils.actions_text import format_observation_messages +from minisweagent.models.utils.actions_toolcall import format_toolcall_observation_messages +from minisweagent.models.utils.actions_toolcall_response import ( + format_toolcall_observation_messages as format_response_observation_messages, +) +from minisweagent.models.utils.observations import MAX_OBSERVATION_OUTPUT_BYTES + + +def _large_output() -> str: + return "A" * 10000 + "B" * 10000 + + +def test_toolcall_observation_bounds_provider_facing_output(): + output = _large_output() + messages = format_toolcall_observation_messages( + actions=[{"command": "generate output", "tool_call_id": "call_1"}], + outputs=[{"output": output, "returncode": 0}], + observation_template="{{ output.output }}", + ) + + content = messages[0]["content"] + assert len(content.encode("utf-8")) <= MAX_OBSERVATION_OUTPUT_BYTES + assert "output truncated" in content + assert content.startswith("A") + assert content.endswith("B") + assert messages[0]["extra"]["raw_output"] == output + + +def test_toolcall_observation_stays_bounded_across_format_error_retries(): + output = _large_output() + messages = format_toolcall_observation_messages( + actions=[{"command": "generate output", "tool_call_id": "call_1"}], + outputs=[{"output": output, "returncode": 0}], + observation_template="{{ output.output }}", + ) + + assert len(messages[0]["content"].encode("utf-8")) * 3 < 8192 + + +def test_mini_template_observation_stays_bounded_for_provider_retries(): + output = _large_output() + config_path = Path(__file__).parents[2] / "src" / "minisweagent" / "config" / "mini.yaml" + template = yaml.safe_load(config_path.read_text())["model"]["observation_template"] + + model = LitellmModel(model_name="openai/gpt-4o", observation_template=template) + messages = model.format_observation_messages( + {"extra": {"actions": [{"command": "generate output", "tool_call_id": "call_1"}]}}, + [{"output": output, "returncode": 0, "exception_info": ""}], + ) + provider_messages = model._prepare_messages_for_api(messages) + + content = provider_messages[0]["content"] + assert len(content.encode("utf-8")) * 3 < 8192 + assert "output truncated" in content + assert "raw_output" not in json.dumps(provider_messages) + assert output not in json.dumps(provider_messages) + + +def test_response_api_observation_bounds_provider_facing_output(): + output = _large_output() + messages = format_response_observation_messages( + actions=[{"command": "generate output", "tool_call_id": "call_1"}], + outputs=[{"output": output, "returncode": 0}], + observation_template="{{ output.output }}", + ) + + content = messages[0]["output"] + assert len(content.encode("utf-8")) <= MAX_OBSERVATION_OUTPUT_BYTES + assert "output truncated" in content + assert messages[0]["extra"]["raw_output"] == output + + +def test_text_observation_bounds_provider_facing_output(): + output = _large_output() + messages = format_observation_messages( + [{"output": output, "returncode": 0}], + observation_template="{{ output.output }}", + ) + + content = messages[0]["content"] + assert len(content.encode("utf-8")) <= MAX_OBSERVATION_OUTPUT_BYTES + assert "output truncated" in content + assert messages[0]["extra"]["raw_output"] == output