From e4d6ecbc833cf1cda0b608820a609516afe7cd82 Mon Sep 17 00:00:00 2001 From: Mark Lavercombe Date: Wed, 15 Jul 2026 08:43:50 +1000 Subject: [PATCH 1/2] Add run --baseline: execute every check against the unmodified tree MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two of twelve lanes in a real fix swarm failed on verify assertions that were wrong about the PRE-change tree — one demanded a status its own lane spec forbade, one was unsatisfiable on a pristine repo. An honest worker burned ~100k tokens against the latter before anyone knew the check was the bug. run --baseline makes that question answerable in one command, before any worker spawns: every task's check runs against a fresh scratch taskdir (a detached worktree of the repo when the manifest uses worktrees), and the report shows pass/FAIL with the check's own output excerpt. Reading it is the orchestrator's judgment call, which the report spells out: a FAIL is expected for assertions that demand the new behavior workers will build; a FAIL on an assertion about unchanged behavior means the check itself is broken and will burn worker attempts against something no model can satisfy. Design notes: - spawns nothing: runs before preflight_engine_bins on purpose, so a missing engine binary cannot block baseline - writes no model-log rows and touches neither the manifest workdir nor the repo (scratch worktrees are removed; the scratch root is deleted) - checks run for real, including any exports they perform — same semantics as at run time - exits 0 regardless of failures: baseline reports, the orchestrator judges Test: end-to-end manifest with a deliberately nonexistent engine binary and two checks — one asserting current tree state (must pass baseline), one demanding a worker-built file (expected FAIL) — asserting the report lines, the excerpt, the guidance, exit 0, an absent model log, and zero leaked taskdirs or worktrees. Co-Authored-By: Claude Fable 5 --- ringer.py | 105 ++++++++++++++++++ tests/test_baseline_mode.py | 208 ++++++++++++++++++++++++++++++++++++ 2 files changed, 313 insertions(+) create mode 100644 tests/test_baseline_mode.py diff --git a/ringer.py b/ringer.py index aa75f84..60cd897 100755 --- a/ringer.py +++ b/ringer.py @@ -8498,6 +8498,98 @@ def shorten(value: str, limit: int) -> str: return clean[: max(0, limit - 3)] + "..." +async def run_baseline(manifest: Manifest, *, config: AppConfig) -> int: + """Execute every task's CHECK against the unmodified tree. Spawn nothing. + + The point: a check assertion that encodes NEW behavior is *expected* to + fail here, but an assertion that encodes UNCHANGED behavior and fails + here is a bug in the check itself — and at run time it will burn a + worker's attempts against something no model can satisfy. Running the + checks once, before any worker spawns, makes that question answerable in + one command. The harness only reports; deciding which failures are + expected is the orchestrator's judgment. + + Checks run for real — including any exports or side effects they perform + (e.g. a fix-swarm check writing its patch file). Each task gets a fresh + scratch taskdir (a detached worktree when the manifest uses worktrees), + removed afterwards, so no state leaks between checks or into a later run. + """ + del config # engines are irrelevant: baseline spawns no workers + verifier = Verifier() + worktrees = manifest.worktrees and manifest.repo is not None + baseline_root = Path(tempfile.mkdtemp(prefix="ringer-baseline-")) + total = len(manifest.tasks) + print(f"Baseline: executing {total} check(s) with no workers spawned.") + failures = 0 + errors = 0 + try: + for task in manifest.tasks: + taskdir = baseline_root / task.key + if worktrees: + proc = await asyncio.create_subprocess_exec( + "git", + "-C", + str(manifest.repo), + "worktree", + "add", + "--detach", + str(taskdir), + "HEAD", + stdin=asyncio.subprocess.DEVNULL, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.STDOUT, + ) + stdout, _ = await proc.communicate() + if proc.returncode != 0: + errors += 1 + print(f"{task.key:<24} baseline: ERROR (git worktree add failed)") + message = stdout.decode("utf-8", errors="replace").strip() + for line in message.splitlines()[:4]: + print(f" {line}") + continue + else: + taskdir.mkdir(parents=True, exist_ok=True) + try: + verify = await verifier.verify(task, taskdir) + status = "pass" if verify.ok else "FAIL" + timed_out = ", timed out" if verify.check_timed_out else "" + print( + f"{task.key:<24} baseline: {status} " + f"(rc={verify.check_returncode}{timed_out})" + ) + if not verify.ok: + failures += 1 + excerpt = verify.raw_output_excerpt.strip() + for line in excerpt.splitlines()[:6]: + print(f" {line}") + finally: + if worktrees: + proc = await asyncio.create_subprocess_exec( + "git", + "-C", + str(manifest.repo), + "worktree", + "remove", + "--force", + str(taskdir), + stdin=asyncio.subprocess.DEVNULL, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.STDOUT, + ) + await proc.communicate() + finally: + shutil.rmtree(baseline_root, ignore_errors=True) + passed = total - failures - errors + print(f"\nbaseline: {passed} pass, {failures} fail, {errors} error of {total} check(s).") + print( + "Reading the results: a FAIL is EXPECTED for assertions that demand the\n" + "NEW behavior workers will build. A FAIL on an assertion about UNCHANGED\n" + "behavior means the check itself is broken and will burn worker attempts\n" + "against something no model can satisfy — fix the check before spawning." + ) + return 0 + + def append_text(path: Path, text: str) -> None: path.parent.mkdir(parents=True, exist_ok=True) with path.open("a", encoding="utf-8") as fh: @@ -8967,6 +9059,15 @@ def build_parser() -> argparse.ArgumentParser: help="disable zero-LLM HTML status/report artifacts (see [artifact] in config.toml)", ) run_parser.add_argument("--dry-run", action="store_true", help="print the plan without spawning codex") + run_parser.add_argument( + "--baseline", + action="store_true", + help=( + "execute every task's CHECK against the unmodified tree and report, " + "spawning no workers — assertions about unchanged behavior that fail " + "baseline are bugs in the check, not work for a model" + ), + ) lint_parser = subparsers.add_parser("lint", help="lint a ringer manifest") lint_parser.add_argument("manifest", type=Path, help="path to ringer.json") @@ -9093,6 +9194,10 @@ def main(argv: list[str] | None = None) -> int: force_browser=args.browser, ) return 0 + if getattr(args, "baseline", False): + # Deliberately before preflight_engine_bins: baseline spawns no + # workers, so a missing engine binary must not block it. + return asyncio.run(run_baseline(manifest, config=config)) preflight_engine_bins(manifest, config) if args.command == "run": start_catalog_auto_refresh() diff --git a/tests/test_baseline_mode.py b/tests/test_baseline_mode.py new file mode 100644 index 0000000..11ef5fe --- /dev/null +++ b/tests/test_baseline_mode.py @@ -0,0 +1,208 @@ +#!/usr/bin/env python3 +"""`run --baseline` executes the checks against the unmodified tree, no workers. + +Observed in the field (2026-07-14): two of twelve fix-swarm lanes failed on +verify assertions that were wrong about the PRE-change tree — one of them +unsatisfiable on a pristine repo. An honest worker burned ~100k tokens +against it. Running every check once, before any worker spawns, makes +"which of my assertions are already wrong?" a one-command question. + +Pinned here: + * checks run against fresh scratch taskdirs (detached worktrees when the + manifest uses worktrees) and report pass/FAIL with output excerpts + * no workers spawn — a deliberately broken engine binary must not matter, + and the engine preflight must not block baseline + * no model-log rows are written + * no scratch worktrees or taskdirs leak into the manifest workdir or repo + * exit code 0 — baseline reports; the orchestrator judges +""" +from __future__ import annotations + +import json +import os +import re +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] + + +def toml_string(value: object) -> str: + return json.dumps(str(value)) + + +def init_git_repo(path: Path) -> None: + path.mkdir(parents=True, exist_ok=True) + (path / "README.md").write_text("hello baseline\n", encoding="utf-8") + env = os.environ.copy() + env.update( + { + "GIT_AUTHOR_NAME": "test", + "GIT_AUTHOR_EMAIL": "test@example.com", + "GIT_COMMITTER_NAME": "test", + "GIT_COMMITTER_EMAIL": "test@example.com", + } + ) + for args in ( + ["git", "-C", str(path), "init", "--quiet"], + ["git", "-C", str(path), "add", "README.md"], + ["git", "-C", str(path), "commit", "--quiet", "-m", "init"], + ): + subprocess.run(args, check=True, env=env, capture_output=True) + + +class BaselineModeTests(unittest.TestCase): + def test_baseline_runs_checks_without_spawning_workers(self) -> None: + with tempfile.TemporaryDirectory() as temp_root: + root = Path(temp_root) + home = root / "home" + ringer_home = root / "ringer-home" + state_dir = root / "state" + workdir = root / "work" + repo = root / "repo" + config_path = root / "config.toml" + manifest_path = root / "manifest.json" + model_log = root / "runs.jsonl" + + home.mkdir() + ringer_home.mkdir() + init_git_repo(repo) + + # The engine binary does not exist. Baseline must neither spawn + # it nor be blocked by the startup engine preflight. + config_path.write_text( + "\n".join( + [ + f"state_dir = {toml_string(state_dir)}", + "", + "[eval]", + 'backend = "jsonl"', + f"jsonl_path = {toml_string(model_log)}", + "", + "[artifact]", + "enabled = false", + "", + "[engines.missing]", + 'bin = "/nonexistent/engine-binary"', + "args_template = [", + ' "{spec}",', + "]", + "sandbox_args = []", + "full_access_args = []", + "", + ] + ), + encoding="utf-8", + ) + + manifest_path.write_text( + json.dumps( + { + "run_name": "baseline-test", + "workdir": str(workdir), + "max_parallel": 2, + "worktrees": True, + "repo": str(repo), + "tasks": [ + { + "key": "unchanged-behavior", + "engine": "missing", + "spec": "Placeholder spec; baseline never spawns a worker.", + # Asserts something already true of the tree — + # the kind of assertion that MUST pass baseline. + "check": ( + "grep -q 'hello baseline' README.md || " + "{ echo FAIL: README.md lost its content; exit 1; }" + ), + }, + { + "key": "new-behavior", + "engine": "missing", + "spec": "Placeholder spec; baseline never spawns a worker.", + # Demands a file a worker would create — the + # kind of assertion EXPECTED to fail baseline. + "check": ( + "test -f built-by-worker.txt || " + "{ echo FAIL: built-by-worker.txt not present; exit 1; }" + ), + }, + ], + }, + indent=2, + ), + encoding="utf-8", + ) + + env = os.environ.copy() + env["HOME"] = str(home) + env["RINGER_HOME"] = str(ringer_home) + env["XDG_CONFIG_HOME"] = str(root / "xdg-config") + + proc = subprocess.run( + [ + sys.executable, + "ringer.py", + "run", + str(manifest_path), + "--config", + str(config_path), + "--no-dashboard", + "--baseline", + ], + cwd=ROOT, + env=env, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + timeout=30, + ) + + combined_output = proc.stdout + proc.stderr + + # Baseline reports; it does not judge. Exit 0 even with failures. + self.assertEqual(0, proc.returncode, combined_output) + + self.assertRegex( + combined_output, + re.compile( + r"^unchanged-behavior\s+baseline: pass \(rc=0\)", re.MULTILINE + ), + combined_output, + ) + self.assertRegex( + combined_output, + re.compile(r"^new-behavior\s+baseline: FAIL \(rc=1\)", re.MULTILINE), + combined_output, + ) + # The failing check's WHY is excerpted. + self.assertIn("built-by-worker.txt not present", combined_output) + # The guidance that makes the report readable. + self.assertIn("fix the check before spawning", combined_output) + self.assertIn("1 pass, 1 fail, 0 error of 2 check(s)", combined_output) + + # No workers, no model rows, no leaked scratch state. + self.assertFalse(model_log.exists(), "baseline wrote model-log rows") + self.assertFalse( + (workdir / "unchanged-behavior").exists() + or (workdir / "new-behavior").exists(), + "baseline leaked taskdirs into the manifest workdir", + ) + worktree_list = subprocess.run( + ["git", "-C", str(repo), "worktree", "list"], + capture_output=True, + text=True, + check=True, + ).stdout.strip() + self.assertEqual( + 1, + len(worktree_list.splitlines()), + f"baseline leaked worktrees:\n{worktree_list}", + ) + + +if __name__ == "__main__": + unittest.main(verbosity=2) From 5f1a5175ed449666f0779da948fa7c9fef72cafc Mon Sep 17 00:00:00 2001 From: Jonathan Edwards Date: Thu, 16 Jul 2026 12:09:07 -0400 Subject: [PATCH 2/2] Baseline hardening: key containment + leaked-worktree reporting Resolve each baseline taskdir and refuse keys that escape the scratch root (same rule as the real run path), and surface failed worktree removals per task and in the summary instead of letting a clean report hide leaked state. Merge with current main keeps --baseline alongside --allow-noncanonical-route. Co-Authored-By: Claude Fable 5 --- ringer.py | 24 ++++++++++++++++++++++-- tests/test_baseline_mode.py | 37 +++++++++++++++++++++++++++++++++++++ 2 files changed, 59 insertions(+), 2 deletions(-) diff --git a/ringer.py b/ringer.py index a05d8ec..0dca41b 100755 --- a/ringer.py +++ b/ringer.py @@ -9212,9 +9212,16 @@ async def run_baseline(manifest: Manifest, *, config: AppConfig) -> int: print(f"Baseline: executing {total} check(s) with no workers spawned.") failures = 0 errors = 0 + leaked_worktrees: list[str] = [] try: for task in manifest.tasks: - taskdir = baseline_root / task.key + taskdir = (baseline_root / task.key).resolve() + # Same containment rule as the real run path: a key must not + # escape its scratch root. + if not taskdir.is_relative_to(baseline_root.resolve()) or taskdir == baseline_root.resolve(): + errors += 1 + print(f"{task.key:<24} baseline: ERROR (task key escapes the baseline scratch root)") + continue if worktrees: proc = await asyncio.create_subprocess_exec( "git", @@ -9266,11 +9273,24 @@ async def run_baseline(manifest: Manifest, *, config: AppConfig) -> int: stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.STDOUT, ) - await proc.communicate() + stdout, _ = await proc.communicate() + if proc.returncode != 0: + # A clean summary must not hide leaked worktree state. + leaked_worktrees.append(str(taskdir)) + message = stdout.decode("utf-8", errors="replace").strip() + print(f"{task.key:<24} baseline: WARNING (worktree remove failed, leaked {taskdir})") + for line in message.splitlines()[:2]: + print(f" {line}") finally: shutil.rmtree(baseline_root, ignore_errors=True) passed = total - failures - errors print(f"\nbaseline: {passed} pass, {failures} fail, {errors} error of {total} check(s).") + if leaked_worktrees: + print( + f"WARNING: {len(leaked_worktrees)} baseline worktree(s) could not be removed; " + f"clean up with `git -C {shlex.quote(str(manifest.repo))} worktree prune` after " + "removing the directories above." + ) print( "Reading the results: a FAIL is EXPECTED for assertions that demand the\n" "NEW behavior workers will build. A FAIL on an assertion about UNCHANGED\n" diff --git a/tests/test_baseline_mode.py b/tests/test_baseline_mode.py index 11ef5fe..cacd856 100644 --- a/tests/test_baseline_mode.py +++ b/tests/test_baseline_mode.py @@ -204,5 +204,42 @@ def test_baseline_runs_checks_without_spawning_workers(self) -> None: ) +class BaselineContainmentTests(unittest.TestCase): + def test_task_key_cannot_escape_baseline_scratch_root(self) -> None: + import asyncio + import contextlib + import importlib.util + import io + + spec = importlib.util.spec_from_file_location("ringer_baseline_test", ROOT / "ringer.py") + assert spec is not None and spec.loader is not None + ringer = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = ringer + spec.loader.exec_module(ringer) + + with tempfile.TemporaryDirectory() as temp_root: + root = Path(temp_root) + manifest = ringer.Manifest.from_obj( + { + "run_name": "baseline-escape-test", + "workdir": str(root / "work"), + "tasks": [ + { + "key": "../escape", + "spec": "Placeholder; baseline spawns nothing.", + "check": "touch escaped.txt", + } + ], + } + ) + buffer = io.StringIO() + with contextlib.redirect_stdout(buffer): + rc = asyncio.run(ringer.run_baseline(manifest, config=None)) + output = buffer.getvalue() + self.assertEqual(0, rc, output) + self.assertIn("task key escapes the baseline scratch root", output) + self.assertIn("0 pass, 0 fail, 1 error of 1 check(s)", output) + + if __name__ == "__main__": unittest.main(verbosity=2)