Add run --baseline: execute every check against the unmodified tree

ringer.py

@@ -9188,6 +9188,118 @@ def shorten(value: str, limit: int) -> str:
     return clean[: max(0, limit - 3)] + "..."
 
 
+async def run_baseline(manifest: Manifest, *, config: AppConfig) -> int:
+    """Execute every task's CHECK against the unmodified tree. Spawn nothing.
+
+    The point: a check assertion that encodes NEW behavior is *expected* to
+    fail here, but an assertion that encodes UNCHANGED behavior and fails
+    here is a bug in the check itself — and at run time it will burn a
+    worker's attempts against something no model can satisfy. Running the
+    checks once, before any worker spawns, makes that question answerable in
+    one command. The harness only reports; deciding which failures are
+    expected is the orchestrator's judgment.
+
+    Checks run for real — including any exports or side effects they perform
+    (e.g. a fix-swarm check writing its patch file). Each task gets a fresh
+    scratch taskdir (a detached worktree when the manifest uses worktrees),
+    removed afterwards, so no state leaks between checks or into a later run.
+    """
+    del config  # engines are irrelevant: baseline spawns no workers
+    verifier = Verifier()
+    worktrees = manifest.worktrees and manifest.repo is not None
+    baseline_root = Path(tempfile.mkdtemp(prefix="ringer-baseline-"))
+    total = len(manifest.tasks)
+    print(f"Baseline: executing {total} check(s) with no workers spawned.")
+    failures = 0
+    errors = 0
+    leaked_worktrees: list[str] = []
+    try:
+        for task in manifest.tasks:
+            taskdir = (baseline_root / task.key).resolve()
+            # Same containment rule as the real run path: a key must not
+            # escape its scratch root.
+            if not taskdir.is_relative_to(baseline_root.resolve()) or taskdir == baseline_root.resolve():
+                errors += 1
+                print(f"{task.key:<24} baseline: ERROR (task key escapes the baseline scratch root)")
+                continue
+            if worktrees:
+                proc = await asyncio.create_subprocess_exec(
+                    "git",
+                    "-C",
+                    str(manifest.repo),
+                    "worktree",
+                    "add",
+                    "--detach",
+                    str(taskdir),
+                    "HEAD",
+                    stdin=asyncio.subprocess.DEVNULL,
+                    stdout=asyncio.subprocess.PIPE,
+                    stderr=asyncio.subprocess.STDOUT,
+                )
+                stdout, _ = await proc.communicate()
+                if proc.returncode != 0:
+                    errors += 1
+                    print(f"{task.key:<24} baseline: ERROR (git worktree add failed)")
+                    message = stdout.decode("utf-8", errors="replace").strip()
+                    for line in message.splitlines()[:4]:
+                        print(f"    {line}")
+                    continue
+            else:
+                taskdir.mkdir(parents=True, exist_ok=True)
+            try:
+                verify = await verifier.verify(task, taskdir)
+                status = "pass" if verify.ok else "FAIL"
+                timed_out = ", timed out" if verify.check_timed_out else ""
+                print(
+                    f"{task.key:<24} baseline: {status} "
+                    f"(rc={verify.check_returncode}{timed_out})"
+                )
+                if not verify.ok:
+                    failures += 1
+                    excerpt = verify.raw_output_excerpt.strip()
+                    for line in excerpt.splitlines()[:6]:
+                        print(f"    {line}")
+            finally:
+                if worktrees:
+                    proc = await asyncio.create_subprocess_exec(
+                        "git",
+                        "-C",
+                        str(manifest.repo),
+                        "worktree",
+                        "remove",
+                        "--force",
+                        str(taskdir),
+                        stdin=asyncio.subprocess.DEVNULL,
+                        stdout=asyncio.subprocess.PIPE,
+                        stderr=asyncio.subprocess.STDOUT,
+                    )
+                    stdout, _ = await proc.communicate()
+                    if proc.returncode != 0:
+                        # A clean summary must not hide leaked worktree state.
+                        leaked_worktrees.append(str(taskdir))
+                        message = stdout.decode("utf-8", errors="replace").strip()
+                        print(f"{task.key:<24} baseline: WARNING (worktree remove failed, leaked {taskdir})")
+                        for line in message.splitlines()[:2]:
+                            print(f"    {line}")
+    finally:
+        shutil.rmtree(baseline_root, ignore_errors=True)
+    passed = total - failures - errors
+    print(f"\nbaseline: {passed} pass, {failures} fail, {errors} error of {total} check(s).")
+    if leaked_worktrees:
+        print(
+            f"WARNING: {len(leaked_worktrees)} baseline worktree(s) could not be removed; "
+            f"clean up with `git -C {shlex.quote(str(manifest.repo))} worktree prune` after "
+            "removing the directories above."
+        )
+    print(
+        "Reading the results: a FAIL is EXPECTED for assertions that demand the\n"
+        "NEW behavior workers will build. A FAIL on an assertion about UNCHANGED\n"
+        "behavior means the check itself is broken and will burn worker attempts\n"
+        "against something no model can satisfy — fix the check before spawning."
+    )
+    return 0
+
+
 def append_text(path: Path, text: str) -> None:
     path.parent.mkdir(parents=True, exist_ok=True)
     with path.open("a", encoding="utf-8") as fh:
@@ -9810,6 +9922,15 @@ def build_parser() -> argparse.ArgumentParser:
         help="disable zero-LLM HTML status/report artifacts (see [artifact] in config.toml)",
     )
     run_parser.add_argument("--dry-run", action="store_true", help="print the plan without spawning codex")
+    run_parser.add_argument(
+        "--baseline",
+        action="store_true",
+        help=(
+            "execute every task's CHECK against the unmodified tree and report, "
+            "spawning no workers — assertions about unchanged behavior that fail "
+            "baseline are bugs in the check, not work for a model"
+        ),
+    )
     run_parser.add_argument(
         "--allow-noncanonical-route",
         action="store_true",
@@ -9993,6 +10114,10 @@ def main(argv: list[str] | None = None) -> int:
                 force_browser=args.browser,
             )
             return 0
+        if getattr(args, "baseline", False):
+            # Deliberately before preflight_engine_bins: baseline spawns no
+            # workers, so a missing engine binary must not block it.
+            return asyncio.run(run_baseline(manifest, config=config))
         preflight_engine_bins(manifest, config)
         if args.command == "run":
             start_catalog_auto_refresh()

tests/test_baseline_mode.py

@@ -0,0 +1,245 @@
+#!/usr/bin/env python3
+"""`run --baseline` executes the checks against the unmodified tree, no workers.
+
+Observed in the field (2026-07-14): two of twelve fix-swarm lanes failed on
+verify assertions that were wrong about the PRE-change tree — one of them
+unsatisfiable on a pristine repo. An honest worker burned ~100k tokens
+against it. Running every check once, before any worker spawns, makes
+"which of my assertions are already wrong?" a one-command question.
+
+Pinned here:
+  * checks run against fresh scratch taskdirs (detached worktrees when the
+    manifest uses worktrees) and report pass/FAIL with output excerpts
+  * no workers spawn — a deliberately broken engine binary must not matter,
+    and the engine preflight must not block baseline
+  * no model-log rows are written
+  * no scratch worktrees or taskdirs leak into the manifest workdir or repo
+  * exit code 0 — baseline reports; the orchestrator judges
+"""
+from __future__ import annotations
+
+import json
+import os
+import re
+import subprocess
+import sys
+import tempfile
+import unittest
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[1]
+
+
+def toml_string(value: object) -> str:
+    return json.dumps(str(value))
+
+
+def init_git_repo(path: Path) -> None:
+    path.mkdir(parents=True, exist_ok=True)
+    (path / "README.md").write_text("hello baseline\n", encoding="utf-8")
+    env = os.environ.copy()
+    env.update(
+        {
+            "GIT_AUTHOR_NAME": "test",
+            "GIT_AUTHOR_EMAIL": "test@example.com",
+            "GIT_COMMITTER_NAME": "test",
+            "GIT_COMMITTER_EMAIL": "test@example.com",
+        }
+    )
+    for args in (
+        ["git", "-C", str(path), "init", "--quiet"],
+        ["git", "-C", str(path), "add", "README.md"],
+        ["git", "-C", str(path), "commit", "--quiet", "-m", "init"],
+    ):
+        subprocess.run(args, check=True, env=env, capture_output=True)
+
+
+class BaselineModeTests(unittest.TestCase):
+    def test_baseline_runs_checks_without_spawning_workers(self) -> None:
+        with tempfile.TemporaryDirectory() as temp_root:
+            root = Path(temp_root)
+            home = root / "home"
+            ringer_home = root / "ringer-home"
+            state_dir = root / "state"
+            workdir = root / "work"
+            repo = root / "repo"
+            config_path = root / "config.toml"
+            manifest_path = root / "manifest.json"
+            model_log = root / "runs.jsonl"
+
+            home.mkdir()
+            ringer_home.mkdir()
+            init_git_repo(repo)
+
+            # The engine binary does not exist. Baseline must neither spawn
+            # it nor be blocked by the startup engine preflight.
+            config_path.write_text(
+                "\n".join(
+                    [
+                        f"state_dir = {toml_string(state_dir)}",
+                        "",
+                        "[eval]",
+                        'backend = "jsonl"',
+                        f"jsonl_path = {toml_string(model_log)}",
+                        "",
+                        "[artifact]",
+                        "enabled = false",
+                        "",
+                        "[engines.missing]",
+                        'bin = "/nonexistent/engine-binary"',
+                        "args_template = [",
+                        '  "{spec}",',
+                        "]",
+                        "sandbox_args = []",
+                        "full_access_args = []",
+                        "",
+                    ]
+                ),
+                encoding="utf-8",
+            )
+
+            manifest_path.write_text(
+                json.dumps(
+                    {
+                        "run_name": "baseline-test",
+                        "workdir": str(workdir),
+                        "max_parallel": 2,
+                        "worktrees": True,
+                        "repo": str(repo),
+                        "tasks": [
+                            {
+                                "key": "unchanged-behavior",
+                                "engine": "missing",
+                                "spec": "Placeholder spec; baseline never spawns a worker.",
+                                # Asserts something already true of the tree —
+                                # the kind of assertion that MUST pass baseline.
+                                "check": (
+                                    "grep -q 'hello baseline' README.md || "
+                                    "{ echo FAIL: README.md lost its content; exit 1; }"
+                                ),
+                            },
+                            {
+                                "key": "new-behavior",
+                                "engine": "missing",
+                                "spec": "Placeholder spec; baseline never spawns a worker.",
+                                # Demands a file a worker would create — the
+                                # kind of assertion EXPECTED to fail baseline.
+                                "check": (
+                                    "test -f built-by-worker.txt || "
+                                    "{ echo FAIL: built-by-worker.txt not present; exit 1; }"
+                                ),
+                            },
+                        ],
+                    },
+                    indent=2,
+                ),
+                encoding="utf-8",
+            )
+
+            env = os.environ.copy()
+            env["HOME"] = str(home)
+            env["RINGER_HOME"] = str(ringer_home)
+            env["XDG_CONFIG_HOME"] = str(root / "xdg-config")
+
+            proc = subprocess.run(
+                [
+                    sys.executable,
+                    "ringer.py",
+                    "run",
+                    str(manifest_path),
+                    "--config",
+                    str(config_path),
+                    "--no-dashboard",
+                    "--baseline",
+                ],
+                cwd=ROOT,
+                env=env,
+                text=True,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE,
+                check=False,
+                timeout=30,
+            )
+
+            combined_output = proc.stdout + proc.stderr
+
+            # Baseline reports; it does not judge. Exit 0 even with failures.
+            self.assertEqual(0, proc.returncode, combined_output)
+
+            self.assertRegex(
+                combined_output,
+                re.compile(
+                    r"^unchanged-behavior\s+baseline: pass \(rc=0\)", re.MULTILINE
+                ),
+                combined_output,
+            )
+            self.assertRegex(
+                combined_output,
+                re.compile(r"^new-behavior\s+baseline: FAIL \(rc=1\)", re.MULTILINE),
+                combined_output,
+            )
+            # The failing check's WHY is excerpted.
+            self.assertIn("built-by-worker.txt not present", combined_output)
+            # The guidance that makes the report readable.
+            self.assertIn("fix the check before spawning", combined_output)
+            self.assertIn("1 pass, 1 fail, 0 error of 2 check(s)", combined_output)
+
+            # No workers, no model rows, no leaked scratch state.
+            self.assertFalse(model_log.exists(), "baseline wrote model-log rows")
+            self.assertFalse(
+                (workdir / "unchanged-behavior").exists()
+                or (workdir / "new-behavior").exists(),
+                "baseline leaked taskdirs into the manifest workdir",
+            )
+            worktree_list = subprocess.run(
+                ["git", "-C", str(repo), "worktree", "list"],
+                capture_output=True,
+                text=True,
+                check=True,
+            ).stdout.strip()
+            self.assertEqual(
+                1,
+                len(worktree_list.splitlines()),
+                f"baseline leaked worktrees:\n{worktree_list}",
+            )
+
+
+class BaselineContainmentTests(unittest.TestCase):
+    def test_task_key_cannot_escape_baseline_scratch_root(self) -> None:
+        import asyncio
+        import contextlib
+        import importlib.util
+        import io
+
+        spec = importlib.util.spec_from_file_location("ringer_baseline_test", ROOT / "ringer.py")
+        assert spec is not None and spec.loader is not None
+        ringer = importlib.util.module_from_spec(spec)
+        sys.modules[spec.name] = ringer
+        spec.loader.exec_module(ringer)
+
+        with tempfile.TemporaryDirectory() as temp_root:
+            root = Path(temp_root)
+            manifest = ringer.Manifest.from_obj(
+                {
+                    "run_name": "baseline-escape-test",
+                    "workdir": str(root / "work"),
+                    "tasks": [
+                        {
+                            "key": "../escape",
+                            "spec": "Placeholder; baseline spawns nothing.",
+                            "check": "touch escaped.txt",
+                        }
+                    ],
+                }
+            )
+            buffer = io.StringIO()
+            with contextlib.redirect_stdout(buffer):
+                rc = asyncio.run(ringer.run_baseline(manifest, config=None))
+            output = buffer.getvalue()
+            self.assertEqual(0, rc, output)
+            self.assertIn("task key escapes the baseline scratch root", output)
+            self.assertIn("0 pass, 0 fail, 1 error of 1 check(s)", output)
+
+
+if __name__ == "__main__":
+    unittest.main(verbosity=2)