From 9410d33899b768e0f20d0f80a28377807f50a203 Mon Sep 17 00:00:00 2001 From: hshum Date: Fri, 17 Jul 2026 13:17:39 -0700 Subject: [PATCH] feat(chat): honor JSON and table response shapes end to end MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The last shape boundary left model-side falls. "as JSON" / "in a markdown table" are detected (anchored on request verbs so incidental mentions stay memos), instructed via the exhaustive switch, and deterministically enforced from the RAW model output — parseMemo is heading-driven and lossy, so the policy gains an optional rawText param. JSON: first balanced block via string-aware brace scan, fence-stripped, 20k-char bound, parse-validated, re-serialized deterministically. Table: longest consecutive |...| line run, header+separator minimum. Both fail closed with an honest retry message on non-compliance, and unsupported-run honesty surfaces as a risks row since a limitation cannot ride inside JSON without corrupting it. Both renderers show structured bodies in a bounded monospace block with its own horizontal scroll, via a shared isStructuredAnswer helper. Co-Authored-By: Claude Fable 5 --- CHANGELOG/pages/redesign-chat.md | 28 ++++ .../redesign/chatRuns.responseShape.test.ts | 72 +++++++++ convex/domains/redesign/chatRuns.ts | 146 +++++++++++++++++- shared/redesign/answerFormat.ts | 15 ++ src/features/redesign/agent-workspace.css | 17 ++ .../redesign/pages/ReproducibleChatPage.tsx | 29 ++-- .../redesign/surfaces/ChatSurface.tsx | 19 ++- 7 files changed, 306 insertions(+), 20 deletions(-) create mode 100644 shared/redesign/answerFormat.ts diff --git a/CHANGELOG/pages/redesign-chat.md b/CHANGELOG/pages/redesign-chat.md index 4dd19adca..0b0742aab 100644 --- a/CHANGELOG/pages/redesign-chat.md +++ b/CHANGELOG/pages/redesign-chat.md @@ -3,6 +3,34 @@ Append-only lane for the public redesign chat, reproducible answer receipts, and their transition into an authenticated live conversation. Newest entries first. +## 2026-07-17 - Honor JSON and table response shapes end to end + +The last declared shape boundary falls: "as JSON" / "in a markdown table" are now +detected, instructed, and deterministically enforced. The policy reads the RAW model +output (parseMemo is lossy for structured blocks), extracts the first balanced JSON +block (string-aware brace scan, fence-stripped, 20k-char bound) or the longest +`| ... |` line run, validates, and fails closed with an honest retry message when the +model did not comply. Honesty on unsupported runs surfaces as a risks row - a +limitation cannot ride inside JSON without corrupting it. Both renderers now show +structured bodies in a bounded monospace block (`.rd-answer-structured`) with its own +horizontal scroll, via a shared `isStructuredAnswer` so they cannot disagree. + +**PR / canonical main commit**: `PENDING #NNN MAIN SHA / FINAL QA`. + +**Evidence state**: +- Source: `pending` +- Checks: `npx tsc --noEmit` -> 0 errors. `npx vitest run chatRuns.responseShape + + ChatResponseShape.guard + ChatRunLifecycle.guard` -> 30 passed (4 new: detector + incl. incidental-mention negatives, fenced-JSON passthrough with brace-in-string, + invalid-JSON fail-closed + risks-row honesty, table extraction + missing-table + fail-closed). +- Visual proof: `not recorded` at entry time - mono block verified in the design pass. +- Preview: Tier B one-flow-regression runs on the PR. +- Production live: `not recorded` at entry time. + +**Author**: Homen Shum + Claude Fable 5. +**Touches**: chat runtime + both answer renderers + agent-workspace.css. + ## 2026-07-17 - Resolve the four hardening residuals from the production audit The 2026-07-16 audit cycle left four filed residuals (#567-#570); all four land here. diff --git a/convex/domains/redesign/chatRuns.responseShape.test.ts b/convex/domains/redesign/chatRuns.responseShape.test.ts index edb24f3a2..1122c9852 100644 --- a/convex/domains/redesign/chatRuns.responseShape.test.ts +++ b/convex/domains/redesign/chatRuns.responseShape.test.ts @@ -148,6 +148,78 @@ describe("redesign chat runtime response policy", () => { expect(unsupported.shortAnswer.split(/\s+/).length).toBeLessThanOrEqual(8); }); + it("detects explicit JSON and table format requests, not incidental mentions", () => { + expect(detectRequestedResponseShape("Return the funding rounds as JSON.")).toEqual({ kind: "json" }); + expect(detectRequestedResponseShape("Respond with valid JSON only.")).toEqual({ kind: "json" }); + expect(detectRequestedResponseShape("Output a JSON array of competitors.")).toEqual({ kind: "json" }); + expect(detectRequestedResponseShape("Compare the vendors in a markdown table.")).toEqual({ kind: "table" }); + expect(detectRequestedResponseShape("Format the pricing tiers as a table.")).toEqual({ kind: "table" }); + // Incidental prose stays a memo. + expect(detectRequestedResponseShape("Why did the JSON parser fail last run?")).toEqual({ kind: "memo" }); + expect(detectRequestedResponseShape("Check the table below for the raw numbers.")).toEqual({ kind: "memo" }); + }); + + it("passes through valid JSON from raw output, normalized and fence-stripped", () => { + const raw = 'Here is the data:\n```json\n{"company":"Acme","round":"Series B","note":"brace } inside string"}\n```\nHope that helps!'; + const shaped = applyDeterministicResponsePolicy( + "Return the round as JSON.", + memo, + [{ idx: 1, source: "https://example.com/acme", quote: "Acme raised." }], + raw, + ); + expect(JSON.parse(shaped.shortAnswer)).toEqual({ + company: "Acme", + round: "Series B", + note: "brace } inside string", + }); + expect(shaped.whyItMatters).toBe(""); + expect(shaped.risks).toEqual([]); + }); + + it("fails closed with an honest message when JSON is invalid, and surfaces the source-needed limitation as a risks row on unsupported runs", () => { + const invalid = applyDeterministicResponsePolicy( + "Return the round as JSON.", + memo, + [{ idx: 1, source: "https://example.com/acme" }], + "The model rambled with {broken json", + ); + expect(invalid.shortAnswer).toContain("did not produce valid JSON"); + + const unsupported = applyDeterministicResponsePolicy( + "Return the round as JSON.", + memo, + [{ source: "Setup" }], + '{"company":"Acme"}', + ); + // A limitation cannot ride inside JSON without corrupting it — it must + // surface as a risks row instead, which un-compacts the render. + expect(JSON.parse(unsupported.shortAnswer)).toEqual({ company: "Acme" }); + expect(unsupported.risks).toContain( + "Source needed: no supported URL is available, so source-strength and claim-strength comparisons are unverified.", + ); + }); + + it("extracts the markdown table block and drops surrounding prose", () => { + const raw = "Sure!\n| Vendor | Price |\n| --- | --- |\n| Acme | $10 |\n| Bolt | $12 |\nLet me know if you need more."; + const shaped = applyDeterministicResponsePolicy( + "Compare vendors in a markdown table.", + memo, + [{ idx: 1, source: "https://example.com/acme" }], + raw, + ); + expect(shaped.shortAnswer.split("\n")).toHaveLength(4); + expect(shaped.shortAnswer.startsWith("| Vendor |")).toBe(true); + expect(shaped.shortAnswer).not.toContain("Sure!"); + + const missing = applyDeterministicResponsePolicy( + "Compare vendors in a markdown table.", + memo, + [{ idx: 1, source: "https://example.com/acme" }], + "No table here, just prose.", + ); + expect(missing.shortAnswer).toContain("did not produce a Markdown table"); + }); + it("does not mistake incidental prose about titles for a shape request", () => { expect( detectRequestedResponseShape("Summarize the report and explain why its title is misleading."), diff --git a/convex/domains/redesign/chatRuns.ts b/convex/domains/redesign/chatRuns.ts index fe713f106..3024e4892 100644 --- a/convex/domains/redesign/chatRuns.ts +++ b/convex/domains/redesign/chatRuns.ts @@ -820,7 +820,9 @@ export type RequestedResponseShape = | { kind: "bullets"; count: number } | { kind: "sentence" } | { kind: "paragraph" } - | { kind: "word_limit"; limit: number }; + | { kind: "word_limit"; limit: number } + | { kind: "json" } + | { kind: "table" }; const RESPONSE_COUNT_WORDS: Record = { one: 1, @@ -898,6 +900,19 @@ function detectWordLimit(normalized: string): number | null { return null; } +// Explicit machine-format asks. Anchored on format keywords next to a request +// verb or preposition so incidental prose ("the JSON parser failed", "the +// table below") stays a memo. +const JSON_SHAPE_PATTERNS: RegExp[] = [ + /\b(?:as|in|return|output|respond (?:in|with))\s+(?:valid\s+|raw\s+|pure\s+)?json\b/, + /\bjson\s+(?:only|format|object|array)\b/, + /\bformat(?:ted)?\s+as\s+json\b/, +]; +const TABLE_SHAPE_PATTERNS: RegExp[] = [ + /\b(?:as|in|return|output|format(?:ted)? as)\s+a?\s*(?:markdown\s+)?table\b/, + /\b(?:markdown\s+)?table\s+(?:only|format)\b/, +]; + export function detectRequestedResponseShape(prompt: string): RequestedResponseShape { const normalized = prompt.replace(/\s+/g, " ").trim().toLowerCase(); if (TITLE_ONLY_PATTERNS.some((pattern) => pattern.test(normalized))) { @@ -910,6 +925,15 @@ export function detectRequestedResponseShape(prompt: string): RequestedResponseS const count = bulletMatch ? parseRequestedCount(bulletMatch[1]) : null; if (count) return { kind: "bullets", count }; + // Machine formats outrank prose shapes: "one row per company, as JSON" + // is a JSON ask, not a sentence ask. + if (JSON_SHAPE_PATTERNS.some((pattern) => pattern.test(normalized))) { + return { kind: "json" }; + } + if (TABLE_SHAPE_PATTERNS.some((pattern) => pattern.test(normalized))) { + return { kind: "table" }; + } + if (SINGLE_SENTENCE_PATTERNS.some((pattern) => pattern.test(normalized))) { return { kind: "sentence" }; } @@ -972,6 +996,22 @@ export function responseShapeSystemInstructions(shape: RequestedResponseShape): citationInstruction: inlineCitations, calculationInstruction: compactCalculation, }; + case "json": + return { + shapeInstruction: + "The user requested JSON. Output ONLY one valid JSON object or array — no code fences, no prose before or after, no comments.", + citationInstruction: + "Do not embed [N] citation markers inside JSON values; supporting sources appear in the evidence panel.", + calculationInstruction: compactCalculation, + }; + case "table": + return { + shapeInstruction: + "The user requested a table. Output ONLY one GitHub-flavored Markdown table (header row, separator row, data rows) — no code fences and no prose before or after.", + citationInstruction: + "Do not embed [N] citation markers inside table cells; supporting sources appear in the evidence panel.", + calculationInstruction: compactCalculation, + }; case "memo": return { shapeInstruction: `Produce a banker-style memo. Do not include To, From, Date, or Subject headers. Use exactly these markdown section headings: @@ -1030,6 +1070,69 @@ function citationIndices(text: string): number[] { // Shared by the superlative gate and the sentence/word-limit shape policies. const SENTENCE_SPLITTER = /(?<=[.!?])\s+(?!\[\d+\])/; +function stripCodeFences(text: string): string { + return text.replace(/```[a-z]*\r?\n?/gi, "").replace(/```/g, "").trim(); +} + +// Upper bound on a structured block we will validate and return verbatim. +// Truncating JSON breaks validity, so oversized blocks fail honestly instead. +const MAX_STRUCTURED_BLOCK_CHARS = 20_000; + +/** + * First balanced top-level JSON object/array in the text, string-aware so + * braces inside quoted values don't unbalance the scan. Null when nothing + * balanced is found. + */ +function extractJsonBlock(raw: string): string | null { + const text = stripCodeFences(raw); + const start = text.search(/[[{]/); + if (start === -1) return null; + const open = text[start]; + const close = open === "{" ? "}" : "]"; + let depth = 0; + let inString = false; + let escaped = false; + for (let i = start; i < text.length; i++) { + const ch = text[i]; + if (escaped) { + escaped = false; + continue; + } + if (ch === "\\") { + escaped = inString; + continue; + } + if (ch === '"') { + inString = !inString; + continue; + } + if (inString) continue; + if (ch === open) depth++; + else if (ch === close) { + depth--; + if (depth === 0) return text.slice(start, i + 1); + } + } + return null; +} + +/** Longest run of consecutive `| ... |` lines — header + separator minimum. */ +function extractMarkdownTable(raw: string): string | null { + const lines = stripCodeFences(raw).split("\n"); + let best: string[] = []; + let current: string[] = []; + for (const line of lines) { + if (/^\s*\|.*\|\s*$/.test(line)) { + current.push(line.trim()); + } else { + if (current.length > best.length) best = current; + current = []; + } + } + if (current.length > best.length) best = current; + return best.length >= 2 ? best.join("\n") : null; +} + // Flatten memo prose into plain sentences: markdown bullet/number prefixes // removed per line, whitespace collapsed. function plainProse(parts: string[]): string { @@ -1127,6 +1230,10 @@ export function applyDeterministicResponsePolicy( blocking?: boolean; verificationState?: EvidenceVerificationState; }>, + // Unparsed model output. JSON/table shapes must read this: parseMemo is + // heading-driven and lossy, so structured blocks cannot be reconstructed + // from memo fields. Optional so prose-shape callers/tests stay unchanged. + rawText?: string, ): ParsedMemo { const supportedRows = evidence.filter( (row) => @@ -1197,6 +1304,41 @@ export function applyDeterministicResponsePolicy( }; } + if (shape.kind === "json") { + const block = extractJsonBlock(rawText ?? honest.shortAnswer); + let rendered: string | null = null; + if (block && block.length <= MAX_STRUCTURED_BLOCK_CHARS) { + try { + rendered = JSON.stringify(JSON.parse(block), null, 2); + } catch { + rendered = null; + } + } + return { + shortAnswer: + rendered + ?? "The run did not produce valid JSON. Retry the run, or drop the JSON constraint for a prose answer.", + whyItMatters: "", + // The limitation cannot ride inside the JSON without corrupting it, so + // it surfaces as a risks row (the compact gate shows risks when present). + risks: rendered && !hasSupportedUrl ? [SOURCE_NEEDED_LIMITATION] : [], + nextAction: "", + }; + } + + if (shape.kind === "table") { + const table = extractMarkdownTable(rawText ?? honest.shortAnswer); + const bounded = table && table.length <= MAX_STRUCTURED_BLOCK_CHARS ? table : null; + return { + shortAnswer: + bounded + ?? "The run did not produce a Markdown table. Retry the run, or drop the table constraint for a prose answer.", + whyItMatters: "", + risks: bounded && !hasSupportedUrl ? [SOURCE_NEEDED_LIMITATION] : [], + nextAction: "", + }; + } + if (shape.kind === "sentence") { const prose = plainProse([honest.shortAnswer]) || plainProse([honest.whyItMatters]) || "Evidence review."; const sentence = (prose.split(SENTENCE_SPLITTER)[0] ?? prose).trim().slice(0, 400); @@ -2011,7 +2153,7 @@ Context: ${JSON.stringify(contextBundle)}${conversationSection}${pinnedSection}$ if (evidence.length > 0 && !/\[\d+\]/.test(parsed.whyItMatters)) { parsed.whyItMatters = `${parsed.whyItMatters} [1]`; } - parsed = applyDeterministicResponsePolicy(args.prompt, parsed, evidence); + parsed = applyDeterministicResponsePolicy(args.prompt, parsed, evidence, rawText); const tr4 = { step: "Bind evidence", detail: `${evidence.length} citations from ${groundingChunks.length} Gemini chunks + ${fallbackSources.length} fallback sources + ${memoryEvidence.length} cached sources`, diff --git a/shared/redesign/answerFormat.ts b/shared/redesign/answerFormat.ts new file mode 100644 index 000000000..6f8debcf8 --- /dev/null +++ b/shared/redesign/answerFormat.ts @@ -0,0 +1,15 @@ +/** + * Shared detection for machine-formatted answer bodies (JSON / Markdown + * table) produced by the json/table response shapes in + * convex/domains/redesign/chatRuns.ts. Both the live ChatSurface and the + * reproducible replay page consult this so the two renderers cannot disagree + * about when an answer needs monospace block treatment instead of prose. + */ +export function isStructuredAnswer(text: string): boolean { + const trimmed = text.trimStart(); + if (/^[[{]/.test(trimmed)) return true; + const pipeLines = text + .split("\n") + .filter((line) => /^\s*\|.*\|\s*$/.test(line)); + return pipeLines.length >= 2; +} diff --git a/src/features/redesign/agent-workspace.css b/src/features/redesign/agent-workspace.css index 3a56b4210..e47ed9140 100644 --- a/src/features/redesign/agent-workspace.css +++ b/src/features/redesign/agent-workspace.css @@ -85,6 +85,23 @@ gap: 18px; } +/* JSON / Markdown-table answer bodies (json/table response shapes). Mono + block with its own horizontal scroll so wide tables never stretch the + workspace column. */ +[data-redesign] .rd-answer-structured { + margin: 0; + padding: 12px 14px; + color: var(--rd-ink-strong); + font-family: var(--rd-font-mono, ui-monospace, monospace); + font-size: 12.5px; + line-height: 1.55; + white-space: pre; + overflow-x: auto; + background: var(--rd-surface-sunken, rgba(127, 127, 127, 0.06)); + border: 1px solid var(--rd-line-faint); + border-radius: 8px; +} + [data-redesign] .rd-answer-copy { margin: 0; color: var(--rd-ink-strong); diff --git a/src/features/redesign/pages/ReproducibleChatPage.tsx b/src/features/redesign/pages/ReproducibleChatPage.tsx index 3ee400945..da853fa2c 100644 --- a/src/features/redesign/pages/ReproducibleChatPage.tsx +++ b/src/features/redesign/pages/ReproducibleChatPage.tsx @@ -17,6 +17,7 @@ import { useEffect, useMemo } from "react"; import { useNavigate } from "react-router-dom"; import { useRedesignChatByHash, type ChatAnswer } from "../hooks/useRedesignChatRun"; +import { isStructuredAnswer } from "../../../../shared/redesign/answerFormat"; interface ReproducibleChatPageProps { hash: string; @@ -220,18 +221,22 @@ export function ReproducibleChatPage({ hash }: ReproducibleChatPageProps) {
Short answer
-

- {packet.shortAnswer} -

+ {isStructuredAnswer(packet.shortAnswer) ? ( + 
{packet.shortAnswer}
+ ) : ( +

+ {packet.shortAnswer} +

+ )}
{packet.whyItMatters?.trim() &&
diff --git a/src/features/redesign/surfaces/ChatSurface.tsx b/src/features/redesign/surfaces/ChatSurface.tsx index 4137b26e7..890f7244b 100644 --- a/src/features/redesign/surfaces/ChatSurface.tsx +++ b/src/features/redesign/surfaces/ChatSurface.tsx @@ -28,6 +28,7 @@ import { } from "../hooks/useRedesignChatRun"; import { buildGraphContextBridgePacket } from "../lib/graphContextBridge"; import { buildConversationContext } from "../lib/chatContinuation"; +import { isStructuredAnswer } from "../../../../shared/redesign/answerFormat"; import { useMutation, useQuery, useConvexAuth } from "convex/react"; import { api } from "../../../../convex/_generated/api"; import { LiveResearchChecklist, type ResearchStage, type ResearchStageId } from "../../research/LiveResearchChecklist"; @@ -1875,14 +1876,20 @@ function AnswerPacket({ )} - {/* Short answer — citations clickable + hover-linked to evidence list */} + {/* Short answer — citations clickable + hover-linked to evidence list. + JSON/table shapes render as a monospace block: citations never live + inside structured bodies (the shape instruction forbids them). */}
Short answer
- -

- {renderInlineWithCites(packet.shortAnswer, packet.evidence, handleCiteEnter, handleCiteLeave, onProbeRunWithoutSource ? probeWithoutSource : undefined, maskedIdx)} -

- + {isStructuredAnswer(packet.shortAnswer) ? ( + 
{packet.shortAnswer}
+ ) : ( + +

+ {renderInlineWithCites(packet.shortAnswer, packet.evidence, handleCiteEnter, handleCiteLeave, onProbeRunWithoutSource ? probeWithoutSource : undefined, maskedIdx)} +

+ + )}
{/* Compact response shapes intentionally omit memo-only sections. */}