diff --git a/src/benchflow/skill_eval/_core.py b/src/benchflow/skill_eval/_core.py index e9849b17d..4c006131d 100644 --- a/src/benchflow/skill_eval/_core.py +++ b/src/benchflow/skill_eval/_core.py @@ -19,7 +19,6 @@ from typing import Any, Literal import tomli_w -import yaml from benchflow._paths import assert_within, safe_path_segment from benchflow.skill_policy import ( @@ -27,7 +26,7 @@ SKILL_MODE_WITH_SKILL, validate_container_mount_path, ) -from benchflow.task.document import render_task_md +from benchflow.task.document import dump_frontmatter_yaml, render_task_md from .schema import DEFAULT_SKILL_MOUNT_DIR, validate_evals_json @@ -314,7 +313,7 @@ def _build_verifier_md(dataset: EvalDataset, case: EvalCase) -> str: }, }, } - rendered_frontmatter = yaml.safe_dump(frontmatter, sort_keys=False) + rendered_frontmatter = dump_frontmatter_yaml(frontmatter) return ( f"---\n{rendered_frontmatter}---\n\n## role:reviewer\n\n" "Judge whether the agent trajectory satisfies the case-specific " diff --git a/src/benchflow/task/_document_parse.py b/src/benchflow/task/_document_parse.py index 934379524..db4e42c69 100644 --- a/src/benchflow/task/_document_parse.py +++ b/src/benchflow/task/_document_parse.py @@ -8,8 +8,10 @@ from __future__ import annotations +import datetime import re import tomllib +from collections.abc import Sequence from copy import deepcopy from dataclasses import dataclass from pathlib import Path @@ -29,12 +31,79 @@ TASK_DOCUMENT_FILENAME = "task.md" _DOCUMENT_ONLY_FRONTMATTER_KEYS = {"agents", "benchflow", "scenes", "user"} +_FLOW_SEQUENCE_MAX_WIDTH = 80 +# Flow lists render after a key prefix and indentation the representer cannot +# see (PyYAML exposes no emit-time column), so charge a fixed allowance to +# keep typical in-context lines within the width cap instead of measuring +# the list standalone and overflowing by the prefix length. +_FLOW_SEQUENCE_CONTEXT_ALLOWANCE = 8 +_FLOW_SEQUENCE_SCALAR_TYPES = (str, int, float, bool, datetime.date) _SECTION_RE = re.compile( r"^##\s+(prompt|role:[A-Za-z0-9_.-]+|scene:[A-Za-z0-9_.-]+|user-persona)\s*$", re.IGNORECASE | re.MULTILINE, ) +class _CompactDumper(yaml.SafeDumper): + """SafeDumper that keeps short scalar-only lists in flow style. + + Hand-written frontmatter like ``tags: [parsing, nlp]`` stays compact + across migrate/normalize round-trips instead of exploding into block + bullets. Long lists, lists with nested collections, and lists with + multiline strings keep PyYAML's block layout. + """ + + +def _sequence_flow_style(data: list[Any]) -> bool: + """Decide whether ``data`` renders as a single-line flow sequence. + + Flow style requires every item to be a short scalar (``str``/``int``/ + ``float``/``bool``/``date``/``None``, no embedded newlines) and the + standalone flow rendering — with PyYAML's own quoting applied — to fit + within :data:`_FLOW_SEQUENCE_MAX_WIDTH` minus + :data:`_FLOW_SEQUENCE_CONTEXT_ALLOWANCE` characters (the allowance stands + in for the key prefix and indentation the representer cannot see). The + check is a pure function of ``data``, so output stays deterministic. + """ + + for item in data: + if item is not None and not isinstance(item, _FLOW_SEQUENCE_SCALAR_TYPES): + return False + if isinstance(item, str) and "\n" in item: + return False + rendered = yaml.dump( + data, + Dumper=yaml.SafeDumper, + default_flow_style=True, + width=float("inf"), + ).strip() + return len(rendered) <= _FLOW_SEQUENCE_MAX_WIDTH - _FLOW_SEQUENCE_CONTEXT_ALLOWANCE + + +def _represent_compact_sequence( + dumper: yaml.SafeDumper, data: Sequence[Any] +) -> yaml.SequenceNode: + items = list(data) + return dumper.represent_sequence( + "tag:yaml.org,2002:seq", items, flow_style=_sequence_flow_style(items) + ) + + +_CompactDumper.add_representer(list, _represent_compact_sequence) +_CompactDumper.add_representer(tuple, _represent_compact_sequence) + + +def dump_frontmatter_yaml(data: dict[str, Any]) -> str: + """Serialize ``task.md`` frontmatter with compact flow-style arrays. + + Canonical emitter for every frontmatter dump site: identical to + ``yaml.safe_dump(data, sort_keys=False)`` except that short scalar-only + lists render in flow style (see :class:`_CompactDumper`). + """ + + return yaml.dump(data, Dumper=_CompactDumper, sort_keys=False) + + @dataclass(frozen=True) class TaskDocument: """Parsed ``task.md`` document. @@ -143,7 +212,7 @@ def render_task_md(frontmatter: dict[str, Any] | str, instruction: str) -> str: ("oracle" if key == "solution" else key): value for key, value in data.items() } - rendered_frontmatter = yaml.safe_dump(data, sort_keys=False) + rendered_frontmatter = dump_frontmatter_yaml(data) body = _escape_reserved_section_headings(instruction.strip()) return f"---\n{rendered_frontmatter}---\n\n## prompt\n\n{body}\n" @@ -190,7 +259,7 @@ def render_normalized_task_md(text: str, *, path: str | Path | None = None) -> s ) _config_from_frontmatter(normalized) _parse_roles(normalized) - rendered_frontmatter = yaml.safe_dump(normalized, sort_keys=False) + rendered_frontmatter = dump_frontmatter_yaml(normalized) rendered_body = body.strip() suffix = f"\n\n{rendered_body}\n" if rendered_body else "\n" return f"---\n{rendered_frontmatter}---{suffix}" diff --git a/src/benchflow/task/document.py b/src/benchflow/task/document.py index 7ab3a3426..722bf07d7 100644 --- a/src/benchflow/task/document.py +++ b/src/benchflow/task/document.py @@ -62,6 +62,7 @@ _string_dict, # noqa: F401 _string_list, # noqa: F401 _unescape_reserved_section_headings, # noqa: F401 + dump_frontmatter_yaml, # noqa: F401 render_normalized_task_md, render_task_md, # noqa: F401 render_task_md_from_legacy, diff --git a/tests/test_task_document.py b/tests/test_task_document.py index b9f5ed1fc..5b52d4ac4 100644 --- a/tests/test_task_document.py +++ b/tests/test_task_document.py @@ -19,6 +19,7 @@ VerifierEnvironmentMode, ) from benchflow.task.document import ( + dump_frontmatter_yaml, render_normalized_task_md, render_task_md, render_task_md_from_legacy, @@ -156,6 +157,97 @@ def test_task_document_profile_normalization_is_stable() -> None: assert set(document.roles) == {"architect", "implementer", "reviewer"} +def test_frontmatter_short_scalar_list_renders_flow_style() -> None: + """Short scalar-only lists stay compact instead of exploding into bullets.""" + rendered = render_task_md( + {"schema_version": "1.3", "metadata": {"tags": ["a", "b", "c"]}}, + "Do it.", + ) + + assert "tags: [a, b, c]" in rendered + document = TaskDocument.from_text(rendered) + assert document.config.metadata["tags"] == ["a", "b", "c"] + + +def test_frontmatter_long_list_stays_block_style() -> None: + """Lists whose flow rendering exceeds the width cap keep block bullets.""" + tags = [f"item-{index:02d}" for index in range(20)] + rendered = dump_frontmatter_yaml({"metadata": {"tags": tags}}) + + assert "tags:\n" in rendered + assert "- item-00\n" in rendered + assert "[" not in rendered + assert yaml.safe_load(rendered) == {"metadata": {"tags": tags}} + + +def test_frontmatter_single_long_item_stays_block_style() -> None: + """The width cap applies to rendered length, not item count.""" + tags = ["x" * 100] + rendered = dump_frontmatter_yaml({"metadata": {"tags": tags}}) + + assert "[" not in rendered + assert yaml.safe_load(rendered) == {"metadata": {"tags": tags}} + + +def test_frontmatter_nested_collection_list_stays_block_style() -> None: + """Lists holding mappings or lists never collapse to flow style.""" + data = {"scenes": [{"name": "s1"}, {"name": "s2"}], "grid": [[1, 2], [3, 4]]} + rendered = dump_frontmatter_yaml(data) + + assert "scenes:\n- name: s1\n- name: s2\n" in rendered + assert "grid:\n-" in rendered + assert yaml.safe_load(rendered) == data + + +def test_frontmatter_multiline_string_item_stays_block_style() -> None: + data = {"metadata": {"notes": ["line1\nline2", "x"]}} + rendered = dump_frontmatter_yaml(data) + + assert "notes:\n" in rendered + assert "[" not in rendered + assert yaml.safe_load(rendered) == data + + +def test_frontmatter_special_character_items_round_trip_in_flow_style() -> None: + """PyYAML quoting keeps flow-style items with YAML metacharacters lossless.""" + tags = ["a, b", "c: d", "[x]", " lead", "trail ", "unié中"] + data = {"metadata": {"tags": tags}} + dumped = dump_frontmatter_yaml(data) + + assert "tags: [" in dumped + assert yaml.safe_load(dumped) == data + assert dump_frontmatter_yaml(yaml.safe_load(dumped)) == dumped + + +def test_frontmatter_empty_list_renders_flow_style() -> None: + rendered = dump_frontmatter_yaml({"metadata": {"tags": []}}) + + assert "tags: []" in rendered + assert yaml.safe_load(rendered) == {"metadata": {"tags": []}} + + +def test_normalize_preserves_hand_written_flow_arrays() -> None: + """Normalization keeps a compact hand-written array and stays a fixed point.""" + source = """--- +schema_version: '1.3' +metadata: + tags: [parsing, nlp] +--- +## prompt + +Keep my tags compact. +""" + + normalized = render_normalized_task_md(source) + + assert "tags: [parsing, nlp]" in normalized + assert render_normalized_task_md(normalized) == normalized + assert TaskDocument.from_text(normalized).config.metadata["tags"] == [ + "parsing", + "nlp", + ] + + def test_task_document_unknown_profile_fails_closed() -> None: """Guards commit 00b32e2a's handoff goal against silent profile fallback.""" with pytest.raises(TaskDocumentParseError, match=r"unknown task\.md profile"):