feat(ai): retrieve over CopilotKit's source, not only its docs

packages/outpost/ai/src/confidence.test.ts

@@ -1,6 +1,6 @@
 import { describe, it, expect, beforeAll, afterAll, beforeEach } from 'vitest';
 import { LLMock } from '@copilotkit/aimock';
-import { ConfidenceScorer } from './confidence.js';
+import { ConfidenceScorer, CONFIDENCE_SYSTEM_PROMPT } from './confidence.js';
 import { ConfidenceLevel } from './types.js';
 import type { SearchResult } from './types.js';
 
@@ -171,3 +171,24 @@ describe('ConfidenceScorer', () => {
         });
     });
 });
+
+// The generator was taught that retrieved source code is fair to cite. This
+// prompt is the other half: it used to tell the scorer the assistant "could not
+// read CopilotKit's source", so a correct code-grounded answer was exactly the
+// shape it was instructed to mark down — and the pipeline takes min(generator,
+// scorer), so the code-search win got clawed back at scoring time.
+describe('CONFIDENCE_SYSTEM_PROMPT', () => {
+    it('no longer tells the scorer the assistant could not read the source', () => {
+        expect(CONFIDENCE_SYSTEM_PROMPT).not.toContain("could not read CopilotKit's source");
+    });
+
+    it('tells the scorer that citing retrieved code is correct, not a markdown', () => {
+        expect(CONFIDENCE_SYSTEM_PROMPT).toContain('SOURCE CODE');
+        expect(CONFIDENCE_SYSTEM_PROMPT).toContain('do NOT mark a response down for citing');
+    });
+
+    it('still holds the line on what the assistant genuinely cannot do', () => {
+        expect(CONFIDENCE_SYSTEM_PROMPT).toContain('reproduce the user');
+        expect(CONFIDENCE_SYSTEM_PROMPT).toContain('run any test');
+    });
+});

packages/outpost/ai/src/confidence.ts

@@ -11,14 +11,21 @@ export interface ConfidenceAssessment {
     degraded: boolean;
 }
 
-const CONFIDENCE_SYSTEM_PROMPT = `You are a confidence scoring system for an AI support assistant. Your job is to assess whether a generated response adequately answers the user's question based on the provided search results.
+/**
+ * Exported for testing, like GROUNDING_RULES in generator.ts. A prompt that
+ * contradicts the generator's is invisible at runtime — the pipeline takes
+ * min(generator, scorer), so the scorer quietly claws back what the generator
+ * was allowed to do — and the only way to pin the two together is to assert on
+ * the text.
+ */
+export const CONFIDENCE_SYSTEM_PROMPT = `You are a confidence scoring system for an AI support assistant. Your job is to assess whether a generated response adequately answers the user's question based on the provided search results.
 
 Evaluate these factors:
 1. **Relevance**: Do the search results actually cover the topic the user asked about?
 2. **Coverage**: Does the response address all parts of the question?
 3. **Specificity**: Is the response specific and actionable, or vague and generic?
 4. **Accuracy indicators**: Does the response cite specific features, APIs, or code patterns that exist in CopilotKit?
-5. **Groundedness**: Is every specific claim traceable to the search results above? The assistant that wrote this response could not read CopilotKit's source, reproduce the user's problem, or run any test — it only had these search results. Score LOW when the response:
+5. **Groundedness**: Is every specific claim traceable to the search results above? The search results may include CopilotKit SOURCE CODE as well as documentation pages, and naming a file that appears in them is correct and expected — do NOT mark a response down for citing retrieved code. What the assistant could not do is reproduce the user's problem or run any test, and it had nothing beyond these search results. Score LOW when the response:
    - confirms a bug, asserts a root cause, or claims to have reproduced or tested anything
    - names a file, CSS class, component, prop, hook, or version that does not appear in the search results
    - hedges ("likely", "may vary") and then states the same claim as fact

packages/outpost/ai/src/generator.test.ts

@@ -272,13 +272,86 @@ describe('ResponseGenerator', () => {
 
     // Regression for CopilotKit/CopilotKit#6167: the bot posted "Bug Confirmed"
     // with invented CSS class names for a report it never reproduced. The
-    // generator is a single stateless call over docs search — it has no repo
-    // access and runs no tests — so the prompt has to forbid those claims.
+    // The generator is a single stateless call over retrieval results. It now
+    // receives SOURCE CODE as well as docs, so the boundary the prompt has to
+    // draw moved: retrieved code is fair to cite, un-retrieved files are not, and
+    // a repro or a test run is still something it cannot do.
+    // GROUNDING_RULES tells the model that code entries are shown with their file
+    // path and that the code wins a disagreement with the docs. Both instructions
+    // are unusable if a docs page and a code hit render identically, so the
+    // rendering is part of the contract, not cosmetic.
+    describe('buildSystemPrompt source labelling', () => {
+        const generator = new ResponseGenerator({ apiKey: 'test-key' });
+        const build = (sources: SearchResult[]) =>
+            (
+                generator as unknown as {
+                    buildSystemPrompt: (s: SearchResult[], src?: undefined) => string;
+                }
+            ).buildSystemPrompt(sources, undefined);
+
+        it('labels a code source distinctly from a docs source', () => {
+            const prompt = build([
+                {
+                    title: 'packages/react-core/src/index.ts',
+                    content: 'export const x = 1;',
+                    score: 0.9,
+                    kind: 'code',
+                },
+                {
+                    title: 'api-reference/components/CopilotKit',
+                    content: 'The CopilotKit provider.',
+                    score: 0.8,
+                    kind: 'docs',
+                },
+            ]);
+
+            expect(prompt).toContain('[SOURCE CODE Source 1: packages/react-core/src/index.ts');
+            expect(prompt).toContain('[DOCS Source 2: api-reference/components/CopilotKit');
+        });
+
+        // The JSON result format and the plain-text fallback carry no marker, so an
+        // unlabelled source must not be asserted as either kind.
+        it('leaves a source of unknown kind unlabelled', () => {
+            const prompt = build([
+                { title: 'Untitled', content: 'something', score: 0.5 },
+            ]);
+
+            expect(prompt).toContain('[Source 1: Untitled');
+            expect(prompt).not.toContain('DOCS Source 1');
+            expect(prompt).not.toContain('SOURCE CODE Source 1');
+        });
+    });
+
     describe('GROUNDING_RULES', () => {
-        it('states the model has not read source, reproduced, or tested', () => {
-            expect(GROUNDING_RULES).toContain('have NOT read');
+        it('states the model has not reproduced or tested, and has read only what was retrieved', () => {
             expect(GROUNDING_RULES).toContain('reproduced');
             expect(GROUNDING_RULES).toContain('run any test');
+            expect(GROUNDING_RULES).toContain('not read any file that is not in the Documentation Context');
+        });
+
+        // The regression guard that matters. This exact instruction was in the
+        // prompt while retrieval was docs-only, and it is what told the model to
+        // disclaim the best evidence it had once code search landed — the reason a
+        // reporter was told a shipped feature had no timeline. If it comes back,
+        // code search is silently neutered again.
+        it('no longer claims the model cannot read the source at all', () => {
+            expect(GROUNDING_RULES).not.toContain("have NOT read CopilotKit's source code");
+        });
+
+        it('allows citing retrieved code, since that is now in the context', () => {
+            expect(GROUNDING_RULES).toContain('SOURCE CODE');
+            expect(GROUNDING_RULES).toContain('cite the file');
+        });
+
+        // "Docs silence != feature missing" — the rule that stops the case-A
+        // failure, where the docs not mentioning subagents became "not supported".
+        it('forbids reading documentation silence as absence', () => {
+            expect(GROUNDING_RULES).toContain('Documentation silence is not evidence');
+            expect(GROUNDING_RULES).toContain('not supported');
+        });
+
+        it('resolves a code/docs conflict in favour of the code', () => {
+            expect(GROUNDING_RULES).toContain('the code is what ships');
         });
 
         it('forbids confirming a bug or asserting a root cause', () => {

packages/outpost/ai/src/generator.ts

@@ -8,17 +8,38 @@ import { config } from './config.js';
 
 /**
  * Epistemic guardrails. The generator is a SINGLE stateless model call over
- * documentation search results — it cannot read CopilotKit's source, cannot run
- * a repro, and cannot execute tests. Without these rules it will happily assert
- * a confirmed root cause built from generic framework priors (see
- * CopilotKit/CopilotKit#6167, where the bot posted "Bug Confirmed" plus invented
- * CSS class names for a cursor-jump report it never reproduced).
+ * retrieval results — it cannot run a repro and cannot execute tests. Without
+ * these rules it will happily assert a confirmed root cause built from generic
+ * framework priors (see CopilotKit/CopilotKit#6167, where the bot posted "Bug
+ * Confirmed" plus invented CSS class names for a cursor-jump report it never
+ * reproduced).
  *
  * Every rule here exists to keep the response's claims inside what the provided
  * Documentation Context actually supports.
+ *
+ * ## What changed when code search landed
+ *
+ * These rules previously opened with "You have NOT read CopilotKit's source
+ * code. Never write or imply otherwise." That was true while the pipeline only
+ * called `search-docs`, and it is now false: `searchCode` results are in the
+ * Documentation Context, so the old line instructed the model to disclaim the
+ * best evidence it had. It is the reason a reporter asking whether Deep Agents
+ * supports subagents was told there was no timeline for a feature that already
+ * shipped — the docs did not cover it, and the model was forbidden from having
+ * looked anywhere else.
+ *
+ * The honest boundary is narrower than the old one and still real: retrieved
+ * code is fair to cite, files that were NOT retrieved are not, and a repro or a
+ * test run remains something the model cannot do. Note also that documentation
+ * silence stopped being evidence of absence the moment code became searchable,
+ * which is why "never say not supported on the strength of the docs alone" sits
+ * alongside the capability rather than after it.
  */
 export const GROUNDING_RULES = `Grounding rules (these override the personality and formatting rules above when they conflict):
-- You have NOT read CopilotKit's source code, reproduced the user's problem, or run any test. Never write or imply otherwise.
+- The Documentation Context may include CopilotKit SOURCE CODE as well as documentation pages. Code entries are shown with their file path. You may state what that code does, and cite the file.
+- You have NOT reproduced the user's problem or run any test, and you have not read any file that is not in the Documentation Context. Never write or imply otherwise.
+- Documentation silence is not evidence a feature is missing. If the docs do not cover something but the code shows it working, say it works and that the docs do not cover it yet. Never say "not supported" on the strength of the docs alone.
+- Where code and docs disagree, the code is what ships. Say so plainly rather than reporting both.
 - Never confirm a bug. Do not write "bug confirmed", "this is a real bug", "known issue", "root cause is", or "the fix is" about behavior you cannot see. Acknowledge the report and say engineering will verify.
 - Only name identifiers — file paths, CSS class names, component names, props, hooks, config keys, version numbers — that appear verbatim in the Documentation Context. If it is not there, describe the concept in prose instead of guessing a name.
 - Mark any causal explanation as a hypothesis exactly once ("one possibility is…"), and never restate it as established fact later in the same response. If you hedge a claim, do not close by asserting it.
@@ -213,7 +234,13 @@ export class ResponseGenerator {
         const sourceContext = sources
             .map((s, i) => {
                 const urlLine = s.sourceUrl ? `\nURL: ${s.sourceUrl}` : '';
-                return `[Source ${i + 1}: ${s.title} (relevance: ${s.score.toFixed(2)})]${urlLine}\n${s.content}`;
+                // Labelled by kind, because GROUNDING_RULES now tells the model
+                // that code entries are shown with their file path and that the
+                // code wins a disagreement with the docs. Rendering both as an
+                // identical `[Source N: title]` left that instruction resolvable
+                // only by guessing at the title's shape.
+                const kindLabel = s.kind === 'code' ? 'SOURCE CODE ' : s.kind === 'docs' ? 'DOCS ' : '';
+                return `[${kindLabel}Source ${i + 1}: ${s.title} (relevance: ${s.score.toFixed(2)})]${urlLine}\n${s.content}`;
             })
             .join('\n\n');