diff --git a/docs/architecture.md b/docs/architecture.md index 196225c4..3ebf3668 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -32,7 +32,8 @@ src/ - create project onboarding files - run the requirements brainstorm gate - draft a goal contract from a raw request -- run the 4-bot / 3-approval requirement court plus a non-voting Judge +- run each requirement-court pony through a `RequirementPonyRunner` +- tally the requirement court with the manifest decision rule plus a non-voting Judge - read Pony Trail snapshot history and plan file-level reverts - print visible role-bot discussion before worker execution is allowed @@ -46,6 +47,8 @@ The CLI should call this runtime through its exported interface instead of knowi The runtime treats `provider` and `name` as configuration values. This keeps goal discussion model selection editable in `.ponytrail/manifest.json` without coupling the core runtime to a specific vendor SDK or CLI flag shape. +`requirement-court.ts` exposes a `RequirementPonyRunner` seam. The deterministic default keeps the CLI offline-friendly, while tests and future plugins can inject process-backed or model-backed ponies. The court invokes the runner for each configured voter in each round until the manifest decision rule approves the direction or the maximum round count is reached. + ## Plugins `src/plugins` is the seam for things that vary by environment or integration: @@ -107,9 +110,9 @@ Human request -> ponytrail runtime -> requirements brainstorm -> ask human for details when unclear - -> Product Manager, Project Manager, Engineer, and Testing ponies discuss - -> visible role-pony discussion is printed - -> 3 of 4 voting ponies approve the direction + -> Product Manager, Project Manager, Engineer, and Testing ponies discuss by round + -> visible round discussion is printed + -> manifest-defined voting ponies approve the direction -> Requirement Judge summarizes and merges one detailed requirement -> human confirms the direction -> worker adapter execution remains gated diff --git a/src/cli.ts b/src/cli.ts index f66f3472..3962da9c 100755 --- a/src/cli.ts +++ b/src/cli.ts @@ -18,6 +18,8 @@ import { prepareGoalDiscussion, type RecordedSnapshotCommit, type RequirementCourtResult, + type RequirementPonyRunner, + type RunRequirementCourtInput, readSnapshotHistory, recordSnapshotPost, recordSnapshotPre, @@ -110,6 +112,7 @@ export interface BuildProgramOptions { clarificationPrompter?: GoalClarificationPrompter; projectNamePrompter?: ProjectNamePrompter; setupPrompter?: SetupPrompter; + ponyRunner?: RequirementPonyRunner | undefined; revertApprovalPrompter?: RevertApprovalPrompter; } @@ -127,6 +130,7 @@ export function buildProgram(options: BuildProgramOptions = {}): Command { const clarificationPrompter = options.clarificationPrompter ?? promptForGoalClarifications; const projectNamePrompter = options.projectNamePrompter ?? promptForProjectName; const setupPrompter = options.setupPrompter ?? promptForSetup; + const ponyRunner = options.ponyRunner; const revertApprovalPrompter = options.revertApprovalPrompter ?? promptForRevertApproval; const program = new Command(); @@ -274,7 +278,8 @@ export function buildProgram(options: BuildProgramOptions = {}): Command { rootDir, clarificationPrompter, manifestPath: commandOptions.manifest, - printJson: commandOptions.json, + ponyRunner, + jsonOutput: commandOptions.json ? "contract" : false, }); }, ); @@ -295,7 +300,8 @@ export function buildProgram(options: BuildProgramOptions = {}): Command { rootDir, clarificationPrompter, manifestPath: commandOptions.manifest, - printJson: commandOptions.json, + ponyRunner, + jsonOutput: commandOptions.json ? "court" : false, discussionHeading: "Pony race", printVisibleThinking: true, }); @@ -341,7 +347,8 @@ export function buildProgram(options: BuildProgramOptions = {}): Command { rootDir, clarificationPrompter, manifestPath: commandOptions.manifest, - printJson: false, + ponyRunner, + jsonOutput: false, }); }, ); @@ -625,7 +632,8 @@ interface RunGoalFlowInput { rootDir: string; clarificationPrompter: GoalClarificationPrompter; manifestPath: string; - printJson: boolean; + ponyRunner?: RequirementPonyRunner | undefined; + jsonOutput: "contract" | "court" | false; discussionHeading?: string; printVisibleThinking?: boolean; } @@ -636,7 +644,7 @@ async function runGoalFlow(requestParts: string[], input: RunGoalFlowInput): Pro const preparedDiscussion = prepareGoalDiscussion(request, { manifest }); if (preparedDiscussion.status === "needs_clarification") { - if (input.printJson) { + if (input.jsonOutput) { console.log(JSON.stringify(preparedDiscussion, null, 2)); return; } @@ -667,28 +675,50 @@ async function runGoalFlow(requestParts: string[], input: RunGoalFlowInput): Pro return; } - printRequirementCourtResult( - await runRequirementCourt(clarifiedDiscussion.contract, { manifest }), - { - discussionHeading: input.discussionHeading, - printVisibleThinking: input.printVisibleThinking, - }, + const result = await runRequirementCourt( + clarifiedDiscussion.contract, + createRunRequirementCourtInput(manifest, input.ponyRunner), ); + + printRequirementCourtResult(result, { + discussionHeading: input.discussionHeading, + printVisibleThinking: input.printVisibleThinking, + }); return; } - if (input.printJson) { + if (input.jsonOutput === "contract") { console.log(JSON.stringify(preparedDiscussion.contract, null, 2)); return; } - printRequirementCourtResult( - await runRequirementCourt(preparedDiscussion.contract, { manifest }), - { - discussionHeading: input.discussionHeading, - printVisibleThinking: input.printVisibleThinking, - }, + const result = await runRequirementCourt( + preparedDiscussion.contract, + createRunRequirementCourtInput(manifest, input.ponyRunner), ); + + if (input.jsonOutput === "court") { + console.log(JSON.stringify(result, null, 2)); + return; + } + + printRequirementCourtResult(result, { + discussionHeading: input.discussionHeading, + printVisibleThinking: input.printVisibleThinking, + }); +} + +function createRunRequirementCourtInput( + manifest: RunRequirementCourtInput["manifest"], + ponyRunner: RequirementPonyRunner | undefined, +): RunRequirementCourtInput { + const input: RunRequirementCourtInput = { manifest }; + + if (ponyRunner) { + input.ponyRunner = ponyRunner; + } + + return input; } interface RequirementCourtOutputOptions { @@ -701,14 +731,24 @@ function printRequirementCourtResult( options: RequirementCourtOutputOptions = {}, ): void { console.log(pc.cyan(options.discussionHeading ?? "Requirement discussion")); - for (const entry of result.discussion) { - console.log(entry.line); + for (const round of result.rounds) { + console.log(""); + console.log(pc.cyan(`Round ${round.round}`)); + for (const entry of round.discussion) { + console.log(entry.line); + } } if (options.printVisibleThinking) { printVisibleThinkingTranscript(result); } + console.log(""); + console.log(pc.cyan("Final votes")); + for (const vote of result.votes) { + console.log(`${vote.botId}: ${vote.vote} (${vote.confidence})`); + } + console.log(""); console.log(pc.cyan("Judge summary")); console.log(result.judge.summary); diff --git a/src/runtimes/ponytrail/requirement-court.ts b/src/runtimes/ponytrail/requirement-court.ts index 63599efa..c8624e98 100644 --- a/src/runtimes/ponytrail/requirement-court.ts +++ b/src/runtimes/ponytrail/requirement-court.ts @@ -194,6 +194,10 @@ function createDiscussionEntry( if (!message) { throw new Error(`Requirement pony ${bot.id} returned an empty discussion message.`); } + if (response.confidence < 0 || response.confidence > 1) { + throw new Error(`Requirement pony ${bot.id} returned confidence outside 0..1.`); + } + const visibleThinking = response.visibleThinking ?? createDefaultVisibleThinking(bot, message); return { diff --git a/tests/cli.test.ts b/tests/cli.test.ts index 15af9309..f8585fd4 100644 --- a/tests/cli.test.ts +++ b/tests/cli.test.ts @@ -9,6 +9,7 @@ import { type SetupPrompter, type SetupPromptIo, } from "../src/cli"; +import type { RequirementPonyRunner } from "../src/runtimes/ponytrail"; import { createDefaultSetupReviewBots } from "../src/runtimes/ponytrail/manifest"; describe("cli", () => { @@ -658,6 +659,109 @@ describe("cli", () => { } }); + test("ponyrace runs the configured pony runner and prints round output", async () => { + const rootDir = await mkdtemp(join(tmpdir(), "ponytrail-cli-")); + const logs: string[] = []; + const ponyCalls: string[] = []; + const originalLog = console.log; + const ponyRunner: RequirementPonyRunner = async ({ bot, contract }) => { + ponyCalls.push(bot.id); + + return { + message: `${bot.id} accepted ${contract.title}`, + visibleThinking: { + focus: `Evaluate as ${bot.displayName}.`, + concern: "Keep role-specific risks visible.", + recommendation: "Approve the direction.", + }, + vote: "approve", + confidence: 0.91, + requiredChanges: [], + }; + }; + + console.log = (...values: unknown[]) => { + logs.push(values.join(" ")); + }; + + try { + await buildProgram({ cwd: rootDir, ponyRunner }).parseAsync( + ["onboard", "--dir", ".", "--name", "CLI Court", "--home", rootDir], + { from: "user" }, + ); + + await buildProgram({ cwd: rootDir, ponyRunner }).parseAsync( + ["ponyrace", "Add", "CSV", "import", "to", "admin", "dashboard"], + { from: "user" }, + ); + + expect(ponyCalls).toEqual([ + "product_manager_bot", + "project_manager_bot", + "engineer_bot", + "testing_bot", + ]); + expect(stripAnsiLines(logs)).toContain("Pony race"); + expect(stripAnsiLines(logs)).toContain("Round 1"); + expect(logs.some((line) => line.includes("product_manager_bot accepted"))).toBe(true); + expect(stripAnsiLines(logs)).toContain("Visible thinking transcript"); + expect(stripAnsiLines(logs)).toContain("Final votes"); + expect(logs.some((line) => line.includes("product_manager_bot: approve (0.91)"))).toBe(true); + expect(logs.some((line) => line.includes("Human confirmation: pending"))).toBe(true); + } finally { + console.log = originalLog; + await rm(rootDir, { recursive: true, force: true }); + } + }); + + test("ponyrace JSON output includes court discussion results", async () => { + const rootDir = await mkdtemp(join(tmpdir(), "ponytrail-cli-")); + const logs: string[] = []; + const ponyCalls: string[] = []; + const originalLog = console.log; + const ponyRunner: RequirementPonyRunner = async ({ bot }) => { + ponyCalls.push(bot.id); + + return { + message: `${bot.id} json review`, + vote: "approve", + confidence: 0.88, + requiredChanges: [], + }; + }; + + console.log = (...values: unknown[]) => { + logs.push(values.join(" ")); + }; + + try { + await buildProgram({ cwd: rootDir, ponyRunner }).parseAsync( + ["onboard", "--dir", ".", "--name", "CLI Court", "--home", rootDir], + { from: "user" }, + ); + + await buildProgram({ cwd: rootDir, ponyRunner }).parseAsync( + ["ponyrace", "--json", "Add", "CSV", "import", "to", "admin", "dashboard"], + { from: "user" }, + ); + + const output = JSON.parse(logs.at(-1) ?? "{}"); + expect(ponyCalls).toEqual([ + "product_manager_bot", + "project_manager_bot", + "engineer_bot", + "testing_bot", + ]); + expect(output.rounds.map((round: { round: number }) => round.round)).toEqual([1]); + expect(output.discussion[0].message).toBe("product_manager_bot json review"); + expect(output.votes[0].confidence).toBe(0.88); + expect(output.humanConfirmation).toBe("pending"); + } finally { + console.log = originalLog; + await rm(rootDir, { recursive: true, force: true }); + } + }); + test("stream-goal remains a compatibility alias for requirement discussion", async () => { const rootDir = await mkdtemp(join(tmpdir(), "ponytrail-cli-")); const logs: string[] = [];