Codex/check ponyrace subagents

docs/architecture.md

@@ -32,7 +32,8 @@ src/
 - create project onboarding files
 - run the requirements brainstorm gate
 - draft a goal contract from a raw request
-- run the 4-bot / 3-approval requirement court plus a non-voting Judge
+- run each requirement-court pony through a `RequirementPonyRunner`
+- tally the requirement court with the manifest decision rule plus a non-voting Judge
 - read Pony Trail snapshot history and plan file-level reverts
 - print visible role-bot discussion before worker execution is allowed
 
@@ -46,6 +47,8 @@ The CLI should call this runtime through its exported interface instead of knowi
 
 The runtime treats `provider` and `name` as configuration values. This keeps goal discussion model selection editable in `.ponytrail/manifest.json` without coupling the core runtime to a specific vendor SDK or CLI flag shape.
 
+`requirement-court.ts` exposes a `RequirementPonyRunner` seam. The deterministic default keeps the CLI offline-friendly, while tests and future plugins can inject process-backed or model-backed ponies. The court invokes the runner for each configured voter in each round until the manifest decision rule approves the direction or the maximum round count is reached.
+
 ## Plugins
 
 `src/plugins` is the seam for things that vary by environment or integration:
@@ -107,9 +110,9 @@ Human request
   -> ponytrail runtime
   -> requirements brainstorm
   -> ask human for details when unclear
-  -> Product Manager, Project Manager, Engineer, and Testing ponies discuss
-  -> visible role-pony discussion is printed
-  -> 3 of 4 voting ponies approve the direction
+  -> Product Manager, Project Manager, Engineer, and Testing ponies discuss by round
+  -> visible round discussion is printed
+  -> manifest-defined voting ponies approve the direction
   -> Requirement Judge summarizes and merges one detailed requirement
   -> human confirms the direction
   -> worker adapter execution remains gated

src/cli.ts

@@ -18,6 +18,8 @@ import {
   prepareGoalDiscussion,
   type RecordedSnapshotCommit,
   type RequirementCourtResult,
+  type RequirementPonyRunner,
+  type RunRequirementCourtInput,
   readSnapshotHistory,
   recordSnapshotPost,
   recordSnapshotPre,
@@ -110,6 +112,7 @@ export interface BuildProgramOptions {
   clarificationPrompter?: GoalClarificationPrompter;
   projectNamePrompter?: ProjectNamePrompter;
   setupPrompter?: SetupPrompter;
+  ponyRunner?: RequirementPonyRunner | undefined;
   revertApprovalPrompter?: RevertApprovalPrompter;
 }
 
@@ -127,6 +130,7 @@ export function buildProgram(options: BuildProgramOptions = {}): Command {
   const clarificationPrompter = options.clarificationPrompter ?? promptForGoalClarifications;
   const projectNamePrompter = options.projectNamePrompter ?? promptForProjectName;
   const setupPrompter = options.setupPrompter ?? promptForSetup;
+  const ponyRunner = options.ponyRunner;
   const revertApprovalPrompter = options.revertApprovalPrompter ?? promptForRevertApproval;
   const program = new Command();
 
@@ -274,7 +278,8 @@ export function buildProgram(options: BuildProgramOptions = {}): Command {
           rootDir,
           clarificationPrompter,
           manifestPath: commandOptions.manifest,
-          printJson: commandOptions.json,
+          ponyRunner,
+          jsonOutput: commandOptions.json ? "contract" : false,
         });
       },
     );
@@ -295,7 +300,8 @@ export function buildProgram(options: BuildProgramOptions = {}): Command {
           rootDir,
           clarificationPrompter,
           manifestPath: commandOptions.manifest,
-          printJson: commandOptions.json,
+          ponyRunner,
+          jsonOutput: commandOptions.json ? "court" : false,
           discussionHeading: "Pony race",
           printVisibleThinking: true,
         });
@@ -341,7 +347,8 @@ export function buildProgram(options: BuildProgramOptions = {}): Command {
           rootDir,
           clarificationPrompter,
           manifestPath: commandOptions.manifest,
-          printJson: false,
+          ponyRunner,
+          jsonOutput: false,
         });
       },
     );
@@ -625,7 +632,8 @@ interface RunGoalFlowInput {
   rootDir: string;
   clarificationPrompter: GoalClarificationPrompter;
   manifestPath: string;
-  printJson: boolean;
+  ponyRunner?: RequirementPonyRunner | undefined;
+  jsonOutput: "contract" | "court" | false;
   discussionHeading?: string;
   printVisibleThinking?: boolean;
 }
@@ -636,7 +644,7 @@ async function runGoalFlow(requestParts: string[], input: RunGoalFlowInput): Pro
   const preparedDiscussion = prepareGoalDiscussion(request, { manifest });
 
   if (preparedDiscussion.status === "needs_clarification") {
-    if (input.printJson) {
+    if (input.jsonOutput) {
       console.log(JSON.stringify(preparedDiscussion, null, 2));
       return;
     }
@@ -667,28 +675,50 @@ async function runGoalFlow(requestParts: string[], input: RunGoalFlowInput): Pro
       return;
     }
 
-    printRequirementCourtResult(
-      await runRequirementCourt(clarifiedDiscussion.contract, { manifest }),
-      {
-        discussionHeading: input.discussionHeading,
-        printVisibleThinking: input.printVisibleThinking,
-      },
+    const result = await runRequirementCourt(
+      clarifiedDiscussion.contract,
+      createRunRequirementCourtInput(manifest, input.ponyRunner),
     );
+
+    printRequirementCourtResult(result, {
+      discussionHeading: input.discussionHeading,
+      printVisibleThinking: input.printVisibleThinking,
+    });
     return;
   }
 
-  if (input.printJson) {
+  if (input.jsonOutput === "contract") {
     console.log(JSON.stringify(preparedDiscussion.contract, null, 2));
     return;
   }
 
-  printRequirementCourtResult(
-    await runRequirementCourt(preparedDiscussion.contract, { manifest }),
-    {
-      discussionHeading: input.discussionHeading,
-      printVisibleThinking: input.printVisibleThinking,
-    },
+  const result = await runRequirementCourt(
+    preparedDiscussion.contract,
+    createRunRequirementCourtInput(manifest, input.ponyRunner),
   );
+
+  if (input.jsonOutput === "court") {
+    console.log(JSON.stringify(result, null, 2));
+    return;
+  }
+
+  printRequirementCourtResult(result, {
+    discussionHeading: input.discussionHeading,
+    printVisibleThinking: input.printVisibleThinking,
+  });
+}
+
+function createRunRequirementCourtInput(
+  manifest: RunRequirementCourtInput["manifest"],
+  ponyRunner: RequirementPonyRunner | undefined,
+): RunRequirementCourtInput {
+  const input: RunRequirementCourtInput = { manifest };
+
+  if (ponyRunner) {
+    input.ponyRunner = ponyRunner;
+  }
+
+  return input;
 }
 
 interface RequirementCourtOutputOptions {
@@ -701,14 +731,24 @@ function printRequirementCourtResult(
   options: RequirementCourtOutputOptions = {},
 ): void {
   console.log(pc.cyan(options.discussionHeading ?? "Requirement discussion"));
-  for (const entry of result.discussion) {
-    console.log(entry.line);
+  for (const round of result.rounds) {
+    console.log("");
+    console.log(pc.cyan(`Round ${round.round}`));
+    for (const entry of round.discussion) {
+      console.log(entry.line);
+    }
   }
 
   if (options.printVisibleThinking) {
     printVisibleThinkingTranscript(result);
   }
 
+  console.log("");
+  console.log(pc.cyan("Final votes"));
+  for (const vote of result.votes) {
+    console.log(`${vote.botId}: ${vote.vote} (${vote.confidence})`);
+  }
+
   console.log("");
   console.log(pc.cyan("Judge summary"));
   console.log(result.judge.summary);

src/runtimes/ponytrail/requirement-court.ts

@@ -194,6 +194,10 @@ function createDiscussionEntry(
   if (!message) {
     throw new Error(`Requirement pony ${bot.id} returned an empty discussion message.`);
   }
+  if (response.confidence < 0 || response.confidence > 1) {
+    throw new Error(`Requirement pony ${bot.id} returned confidence outside 0..1.`);
+  }
+
   const visibleThinking = response.visibleThinking ?? createDefaultVisibleThinking(bot, message);
 
   return {

tests/cli.test.ts

@@ -9,6 +9,7 @@ import {
   type SetupPrompter,
   type SetupPromptIo,
 } from "../src/cli";
+import type { RequirementPonyRunner } from "../src/runtimes/ponytrail";
 import { createDefaultSetupReviewBots } from "../src/runtimes/ponytrail/manifest";
 
 describe("cli", () => {
@@ -658,6 +659,109 @@ describe("cli", () => {
     }
   });
 
+  test("ponyrace runs the configured pony runner and prints round output", async () => {
+    const rootDir = await mkdtemp(join(tmpdir(), "ponytrail-cli-"));
+    const logs: string[] = [];
+    const ponyCalls: string[] = [];
+    const originalLog = console.log;
+    const ponyRunner: RequirementPonyRunner = async ({ bot, contract }) => {
+      ponyCalls.push(bot.id);
+
+      return {
+        message: `${bot.id} accepted ${contract.title}`,
+        visibleThinking: {
+          focus: `Evaluate as ${bot.displayName}.`,
+          concern: "Keep role-specific risks visible.",
+          recommendation: "Approve the direction.",
+        },
+        vote: "approve",
+        confidence: 0.91,
+        requiredChanges: [],
+      };
+    };
+
+    console.log = (...values: unknown[]) => {
+      logs.push(values.join(" "));
+    };
+
+    try {
+      await buildProgram({ cwd: rootDir, ponyRunner }).parseAsync(
+        ["onboard", "--dir", ".", "--name", "CLI Court", "--home", rootDir],
+        { from: "user" },
+      );
+
+      await buildProgram({ cwd: rootDir, ponyRunner }).parseAsync(
+        ["ponyrace", "Add", "CSV", "import", "to", "admin", "dashboard"],
+        { from: "user" },
+      );
+
+      expect(ponyCalls).toEqual([
+        "product_manager_bot",
+        "project_manager_bot",
+        "engineer_bot",
+        "testing_bot",
+      ]);
+      expect(stripAnsiLines(logs)).toContain("Pony race");
+      expect(stripAnsiLines(logs)).toContain("Round 1");
+      expect(logs.some((line) => line.includes("product_manager_bot accepted"))).toBe(true);
+      expect(stripAnsiLines(logs)).toContain("Visible thinking transcript");
+      expect(stripAnsiLines(logs)).toContain("Final votes");
+      expect(logs.some((line) => line.includes("product_manager_bot: approve (0.91)"))).toBe(true);
+      expect(logs.some((line) => line.includes("Human confirmation: pending"))).toBe(true);
+    } finally {
+      console.log = originalLog;
+      await rm(rootDir, { recursive: true, force: true });
+    }
+  });
+
+  test("ponyrace JSON output includes court discussion results", async () => {
+    const rootDir = await mkdtemp(join(tmpdir(), "ponytrail-cli-"));
+    const logs: string[] = [];
+    const ponyCalls: string[] = [];
+    const originalLog = console.log;
+    const ponyRunner: RequirementPonyRunner = async ({ bot }) => {
+      ponyCalls.push(bot.id);
+
+      return {
+        message: `${bot.id} json review`,
+        vote: "approve",
+        confidence: 0.88,
+        requiredChanges: [],
+      };
+    };
+
+    console.log = (...values: unknown[]) => {
+      logs.push(values.join(" "));
+    };
+
+    try {
+      await buildProgram({ cwd: rootDir, ponyRunner }).parseAsync(
+        ["onboard", "--dir", ".", "--name", "CLI Court", "--home", rootDir],
+        { from: "user" },
+      );
+
+      await buildProgram({ cwd: rootDir, ponyRunner }).parseAsync(
+        ["ponyrace", "--json", "Add", "CSV", "import", "to", "admin", "dashboard"],
+        { from: "user" },
+      );
+
+      const output = JSON.parse(logs.at(-1) ?? "{}");
+      expect(ponyCalls).toEqual([
+        "product_manager_bot",
+        "project_manager_bot",
+        "engineer_bot",
+        "testing_bot",
+      ]);
+      expect(output.rounds.map((round: { round: number }) => round.round)).toEqual([1]);
+      expect(output.discussion[0].message).toBe("product_manager_bot json review");
+      expect(output.votes[0].confidence).toBe(0.88);
+      expect(output.humanConfirmation).toBe("pending");
+    } finally {
+      console.log = originalLog;
+      await rm(rootDir, { recursive: true, force: true });
+    }
+  });
+
   test("stream-goal remains a compatibility alias for requirement discussion", async () => {
     const rootDir = await mkdtemp(join(tmpdir(), "ponytrail-cli-"));
     const logs: string[] = [];