From a2160132edc8ab79368a9d1df0167b23a08bbd84 Mon Sep 17 00:00:00 2001 From: Xiangyi Li Date: Sun, 9 Aug 2026 12:24:59 -0700 Subject: [PATCH 1/2] feat(task): emit compact flow-style YAML arrays in task.md frontmatter MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Frontmatter emitters used yaml.safe_dump, which renders every sequence as multiline block bullets — a hand-written 'tags: [parsing, nlp]' exploded into bullets on any migrate/normalize round-trip. Add _CompactDumper (SafeDumper subclass) in task/_document_parse.py whose sequence representer emits flow style for lists of short scalars (str/int/float/ bool/None, no embedded newlines) whose standalone flow rendering fits in 80 chars, and block style otherwise. One canonical dump_frontmatter_yaml helper is shared by all three frontmatter dump sites (render_task_md, render_normalized_task_md, skill_eval verifier.md); the machine-consumed litellm config dumps are intentionally untouched. Output remains deterministic and a dump/load/dump fixed point; PyYAML handles quoting for items with YAML metacharacters inside flow style. --- src/benchflow/skill_eval/_core.py | 5 +- src/benchflow/task/_document_parse.py | 65 ++++++++++++++++++- src/benchflow/task/document.py | 1 + tests/test_task_document.py | 92 +++++++++++++++++++++++++++ 4 files changed, 158 insertions(+), 5 deletions(-) diff --git a/src/benchflow/skill_eval/_core.py b/src/benchflow/skill_eval/_core.py index e9849b17d..4c006131d 100644 --- a/src/benchflow/skill_eval/_core.py +++ b/src/benchflow/skill_eval/_core.py @@ -19,7 +19,6 @@ from typing import Any, Literal import tomli_w -import yaml from benchflow._paths import assert_within, safe_path_segment from benchflow.skill_policy import ( @@ -27,7 +26,7 @@ SKILL_MODE_WITH_SKILL, validate_container_mount_path, ) -from benchflow.task.document import render_task_md +from benchflow.task.document import dump_frontmatter_yaml, render_task_md from .schema import DEFAULT_SKILL_MOUNT_DIR, validate_evals_json @@ -314,7 +313,7 @@ def _build_verifier_md(dataset: EvalDataset, case: EvalCase) -> str: }, }, } - rendered_frontmatter = yaml.safe_dump(frontmatter, sort_keys=False) + rendered_frontmatter = dump_frontmatter_yaml(frontmatter) return ( f"---\n{rendered_frontmatter}---\n\n## role:reviewer\n\n" "Judge whether the agent trajectory satisfies the case-specific " diff --git a/src/benchflow/task/_document_parse.py b/src/benchflow/task/_document_parse.py index 934379524..2988324a1 100644 --- a/src/benchflow/task/_document_parse.py +++ b/src/benchflow/task/_document_parse.py @@ -10,6 +10,7 @@ import re import tomllib +from collections.abc import Sequence from copy import deepcopy from dataclasses import dataclass from pathlib import Path @@ -29,12 +30,72 @@ TASK_DOCUMENT_FILENAME = "task.md" _DOCUMENT_ONLY_FRONTMATTER_KEYS = {"agents", "benchflow", "scenes", "user"} +_FLOW_SEQUENCE_MAX_WIDTH = 80 +_FLOW_SEQUENCE_SCALAR_TYPES = (str, int, float, bool) _SECTION_RE = re.compile( r"^##\s+(prompt|role:[A-Za-z0-9_.-]+|scene:[A-Za-z0-9_.-]+|user-persona)\s*$", re.IGNORECASE | re.MULTILINE, ) +class _CompactDumper(yaml.SafeDumper): + """SafeDumper that keeps short scalar-only lists in flow style. + + Hand-written frontmatter like ``tags: [parsing, nlp]`` stays compact + across migrate/normalize round-trips instead of exploding into block + bullets. Long lists, lists with nested collections, and lists with + multiline strings keep PyYAML's block layout. + """ + + +def _sequence_flow_style(data: list[Any]) -> bool: + """Decide whether ``data`` renders as a single-line flow sequence. + + Flow style requires every item to be a short scalar (``str``/``int``/ + ``float``/``bool``/``None``, no embedded newlines) and the standalone flow + rendering — with PyYAML's own quoting applied — to fit within + :data:`_FLOW_SEQUENCE_MAX_WIDTH` characters. The check is a pure function + of ``data``, so output stays deterministic. + """ + + for item in data: + if item is not None and not isinstance(item, _FLOW_SEQUENCE_SCALAR_TYPES): + return False + if isinstance(item, str) and "\n" in item: + return False + rendered = yaml.dump( + data, + Dumper=yaml.SafeDumper, + default_flow_style=True, + width=float("inf"), + ).strip() + return len(rendered) <= _FLOW_SEQUENCE_MAX_WIDTH + + +def _represent_compact_sequence( + dumper: yaml.SafeDumper, data: Sequence[Any] +) -> yaml.SequenceNode: + items = list(data) + return dumper.represent_sequence( + "tag:yaml.org,2002:seq", items, flow_style=_sequence_flow_style(items) + ) + + +_CompactDumper.add_representer(list, _represent_compact_sequence) +_CompactDumper.add_representer(tuple, _represent_compact_sequence) + + +def dump_frontmatter_yaml(data: dict[str, Any]) -> str: + """Serialize ``task.md`` frontmatter with compact flow-style arrays. + + Canonical emitter for every frontmatter dump site: identical to + ``yaml.safe_dump(data, sort_keys=False)`` except that short scalar-only + lists render in flow style (see :class:`_CompactDumper`). + """ + + return yaml.dump(data, Dumper=_CompactDumper, sort_keys=False) + + @dataclass(frozen=True) class TaskDocument: """Parsed ``task.md`` document. @@ -143,7 +204,7 @@ def render_task_md(frontmatter: dict[str, Any] | str, instruction: str) -> str: ("oracle" if key == "solution" else key): value for key, value in data.items() } - rendered_frontmatter = yaml.safe_dump(data, sort_keys=False) + rendered_frontmatter = dump_frontmatter_yaml(data) body = _escape_reserved_section_headings(instruction.strip()) return f"---\n{rendered_frontmatter}---\n\n## prompt\n\n{body}\n" @@ -190,7 +251,7 @@ def render_normalized_task_md(text: str, *, path: str | Path | None = None) -> s ) _config_from_frontmatter(normalized) _parse_roles(normalized) - rendered_frontmatter = yaml.safe_dump(normalized, sort_keys=False) + rendered_frontmatter = dump_frontmatter_yaml(normalized) rendered_body = body.strip() suffix = f"\n\n{rendered_body}\n" if rendered_body else "\n" return f"---\n{rendered_frontmatter}---{suffix}" diff --git a/src/benchflow/task/document.py b/src/benchflow/task/document.py index 7ab3a3426..722bf07d7 100644 --- a/src/benchflow/task/document.py +++ b/src/benchflow/task/document.py @@ -62,6 +62,7 @@ _string_dict, # noqa: F401 _string_list, # noqa: F401 _unescape_reserved_section_headings, # noqa: F401 + dump_frontmatter_yaml, # noqa: F401 render_normalized_task_md, render_task_md, # noqa: F401 render_task_md_from_legacy, diff --git a/tests/test_task_document.py b/tests/test_task_document.py index b9f5ed1fc..5b52d4ac4 100644 --- a/tests/test_task_document.py +++ b/tests/test_task_document.py @@ -19,6 +19,7 @@ VerifierEnvironmentMode, ) from benchflow.task.document import ( + dump_frontmatter_yaml, render_normalized_task_md, render_task_md, render_task_md_from_legacy, @@ -156,6 +157,97 @@ def test_task_document_profile_normalization_is_stable() -> None: assert set(document.roles) == {"architect", "implementer", "reviewer"} +def test_frontmatter_short_scalar_list_renders_flow_style() -> None: + """Short scalar-only lists stay compact instead of exploding into bullets.""" + rendered = render_task_md( + {"schema_version": "1.3", "metadata": {"tags": ["a", "b", "c"]}}, + "Do it.", + ) + + assert "tags: [a, b, c]" in rendered + document = TaskDocument.from_text(rendered) + assert document.config.metadata["tags"] == ["a", "b", "c"] + + +def test_frontmatter_long_list_stays_block_style() -> None: + """Lists whose flow rendering exceeds the width cap keep block bullets.""" + tags = [f"item-{index:02d}" for index in range(20)] + rendered = dump_frontmatter_yaml({"metadata": {"tags": tags}}) + + assert "tags:\n" in rendered + assert "- item-00\n" in rendered + assert "[" not in rendered + assert yaml.safe_load(rendered) == {"metadata": {"tags": tags}} + + +def test_frontmatter_single_long_item_stays_block_style() -> None: + """The width cap applies to rendered length, not item count.""" + tags = ["x" * 100] + rendered = dump_frontmatter_yaml({"metadata": {"tags": tags}}) + + assert "[" not in rendered + assert yaml.safe_load(rendered) == {"metadata": {"tags": tags}} + + +def test_frontmatter_nested_collection_list_stays_block_style() -> None: + """Lists holding mappings or lists never collapse to flow style.""" + data = {"scenes": [{"name": "s1"}, {"name": "s2"}], "grid": [[1, 2], [3, 4]]} + rendered = dump_frontmatter_yaml(data) + + assert "scenes:\n- name: s1\n- name: s2\n" in rendered + assert "grid:\n-" in rendered + assert yaml.safe_load(rendered) == data + + +def test_frontmatter_multiline_string_item_stays_block_style() -> None: + data = {"metadata": {"notes": ["line1\nline2", "x"]}} + rendered = dump_frontmatter_yaml(data) + + assert "notes:\n" in rendered + assert "[" not in rendered + assert yaml.safe_load(rendered) == data + + +def test_frontmatter_special_character_items_round_trip_in_flow_style() -> None: + """PyYAML quoting keeps flow-style items with YAML metacharacters lossless.""" + tags = ["a, b", "c: d", "[x]", " lead", "trail ", "unié中"] + data = {"metadata": {"tags": tags}} + dumped = dump_frontmatter_yaml(data) + + assert "tags: [" in dumped + assert yaml.safe_load(dumped) == data + assert dump_frontmatter_yaml(yaml.safe_load(dumped)) == dumped + + +def test_frontmatter_empty_list_renders_flow_style() -> None: + rendered = dump_frontmatter_yaml({"metadata": {"tags": []}}) + + assert "tags: []" in rendered + assert yaml.safe_load(rendered) == {"metadata": {"tags": []}} + + +def test_normalize_preserves_hand_written_flow_arrays() -> None: + """Normalization keeps a compact hand-written array and stays a fixed point.""" + source = """--- +schema_version: '1.3' +metadata: + tags: [parsing, nlp] +--- +## prompt + +Keep my tags compact. +""" + + normalized = render_normalized_task_md(source) + + assert "tags: [parsing, nlp]" in normalized + assert render_normalized_task_md(normalized) == normalized + assert TaskDocument.from_text(normalized).config.metadata["tags"] == [ + "parsing", + "nlp", + ] + + def test_task_document_unknown_profile_fails_closed() -> None: """Guards commit 00b32e2a's handoff goal against silent profile fallback.""" with pytest.raises(TaskDocumentParseError, match=r"unknown task\.md profile"): From 85817e742153d46ace23974e8d86950eb0bd44ba Mon Sep 17 00:00:00 2001 From: Xiangyi Li Date: Sun, 9 Aug 2026 12:33:34 -0700 Subject: [PATCH 2/2] refactor(task): charge context allowance in flow-width check; accept date scalars MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review follow-ups: the 80-char cap measured the list standalone, so a list under a key prefix could render past the cap or wrap mid-flow at deep nesting (uglier than the block bullets it replaces) — charge a fixed 8-char allowance since PyYAML exposes no emit-time column. Also admit datetime.date/datetime items (SafeDumper renders them fine; date lists previously stayed block for no reason). --- src/benchflow/task/_document_parse.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/src/benchflow/task/_document_parse.py b/src/benchflow/task/_document_parse.py index 2988324a1..db4e42c69 100644 --- a/src/benchflow/task/_document_parse.py +++ b/src/benchflow/task/_document_parse.py @@ -8,6 +8,7 @@ from __future__ import annotations +import datetime import re import tomllib from collections.abc import Sequence @@ -31,7 +32,12 @@ _DOCUMENT_ONLY_FRONTMATTER_KEYS = {"agents", "benchflow", "scenes", "user"} _FLOW_SEQUENCE_MAX_WIDTH = 80 -_FLOW_SEQUENCE_SCALAR_TYPES = (str, int, float, bool) +# Flow lists render after a key prefix and indentation the representer cannot +# see (PyYAML exposes no emit-time column), so charge a fixed allowance to +# keep typical in-context lines within the width cap instead of measuring +# the list standalone and overflowing by the prefix length. +_FLOW_SEQUENCE_CONTEXT_ALLOWANCE = 8 +_FLOW_SEQUENCE_SCALAR_TYPES = (str, int, float, bool, datetime.date) _SECTION_RE = re.compile( r"^##\s+(prompt|role:[A-Za-z0-9_.-]+|scene:[A-Za-z0-9_.-]+|user-persona)\s*$", re.IGNORECASE | re.MULTILINE, @@ -52,10 +58,12 @@ def _sequence_flow_style(data: list[Any]) -> bool: """Decide whether ``data`` renders as a single-line flow sequence. Flow style requires every item to be a short scalar (``str``/``int``/ - ``float``/``bool``/``None``, no embedded newlines) and the standalone flow - rendering — with PyYAML's own quoting applied — to fit within - :data:`_FLOW_SEQUENCE_MAX_WIDTH` characters. The check is a pure function - of ``data``, so output stays deterministic. + ``float``/``bool``/``date``/``None``, no embedded newlines) and the + standalone flow rendering — with PyYAML's own quoting applied — to fit + within :data:`_FLOW_SEQUENCE_MAX_WIDTH` minus + :data:`_FLOW_SEQUENCE_CONTEXT_ALLOWANCE` characters (the allowance stands + in for the key prefix and indentation the representer cannot see). The + check is a pure function of ``data``, so output stays deterministic. """ for item in data: @@ -69,7 +77,7 @@ def _sequence_flow_style(data: list[Any]) -> bool: default_flow_style=True, width=float("inf"), ).strip() - return len(rendered) <= _FLOW_SEQUENCE_MAX_WIDTH + return len(rendered) <= _FLOW_SEQUENCE_MAX_WIDTH - _FLOW_SEQUENCE_CONTEXT_ALLOWANCE def _represent_compact_sequence(