feat(llm): Prism-style model router + wire the /ask path through it

convex/events.ts

@@ -30,6 +30,7 @@ import { v } from "convex/values";
 import { internal } from "./_generated/api";
 import { action, query, mutation, internalMutation, internalQuery } from "./_generated/server";
 import { enforceRateLimit } from "./scratchnodeRateLimit";
+import { routeLLM, askAnswerSignals } from "../shared/llm/router";
 import { rerankWithGemini, condenseQuery, type TriCandidate } from "../shared/search/triSearch";
 
 class ConvexError<T extends Record<string, unknown>> extends Error {
@@ -80,7 +81,10 @@ const MAX_OWNER_KEY_LEN = 120;
 const MAX_AGENT_CONTEXT_SOURCES = 6;
 const MAX_AGENT_CONTEXT_CHARS = 9_000;
 const MAX_LINKUP_SOURCES = 4;
-const SCRATCHNODE_DEFAULT_ASK_MODEL = "claude-haiku-4-5-20251001";
+// /ask model selection lives in shared/llm/router.ts — the ask_answer pool
+// (Haiku floor; escalates to Sonnet on long / analytical / multi-entity Qs).
+// Legacy SCRATCHNODE_ASK_MODEL still force-pins for ops; the pool floor defaults
+// to the exact prior Haiku id so an unset env is behavior-preserving.
 
 // Phase 4 follow-up Item 1: optional email channel for claim codes.
 // Validation kept basic on purpose — Resend re-validates server-side, and
@@ -640,12 +644,13 @@ async function generateProviderAnswer(args: {
   eventName: string;
   question: string;
   sources: Array<{ title: string; excerpt: string; body: string; uri?: string }>;
+  model: string; // chosen by the LLM router (shared/llm/router.ts) at the call site
 }) {
   const apiKey = process.env.ANTHROPIC_API_KEY;
   if (!apiKey) {
     return { ok: false as const, detail: "ANTHROPIC_API_KEY not configured." };
   }
-  const model = process.env.SCRATCHNODE_ASK_MODEL || SCRATCHNODE_DEFAULT_ASK_MODEL;
+  const model = args.model;
   const system = [
     "You are ScratchNode's event answer agent.",
     "Answer only from the provided public event sources.",
@@ -2059,10 +2064,25 @@ export const askAgent = action({
       });
     }
 
+    // Prism-style routing (shared/llm/router.ts): a deterministic planner picks
+    // the model for THIS question from the ask_answer pool (Haiku floor ->
+    // Sonnet on long / analytical / multi-entity Qs). Single-shot path with no
+    // prompt-cache to evict, so we route per request at zero added latency.
+    const askRoute = routeLLM("ask_answer", askAnswerSignals(question, topSources.length));
+    const askModel = process.env.SCRATCHNODE_ASK_MODEL || askRoute.model; // legacy env force-pins
+    trace.push({
+      step: "model_route",
+      status: "ok",
+      detail: process.env.SCRATCHNODE_ASK_MODEL
+        ? `env-pinned ${askModel}`
+        : `${askRoute.reason} -> ${askRoute.model}${askRoute.escalated ? " (escalated)" : ""}`,
+      durationMs: 0,
+    });
     const provider = await generateProviderAnswer({
       eventName: prepared.event.name,
       question,
       sources: topSources,
+      model: askModel,
     });
     let body: string;
     let agentMode: "provider" | "provider_fallback";

docs/architecture/LLM_ROUTER.md

@@ -0,0 +1,83 @@
+# NodeBench LLM Router (Prism-style planner-on-a-pool)
+
+One deterministic entry point — `routeLLM(taskClass, signals)` in
+`shared/llm/router.ts` — for "which model serves THIS turn?". Replaces the
+~7 scattered, uncoordinated model-selection mechanisms across the codebase
+with a single planner-on-a-pool, so every served LLM feature routes the same
+way and emits the same telemetry.
+
+## Prior art
+
+- **Augment "Prism"** (2026) — *Introducing Augment Prism: model routing to
+  reduce cost and maintain quality.*
+  <https://www.augmentcode.com/blog/augment-prism-model-routing-to-reduce-cost-and-maintain-quality>
+  - **Borrowed:** a small fast *planner* picks, per user turn, which model from
+    a **fixed pool tuned to a quality target** handles the request ("you pick
+    Prism, Prism picks the model"). Their measured result: **20–30% cheaper at
+    negligible quality loss**, driven by the finding that *the top 10% of turns
+    consume ~57% of LLM rounds* — most turns are light work billed at frontier
+    rates because users pin the big model once and never switch.
+  - **Their hard part we deliberately scope out of V1:** *switching evicts the
+    prompt cache (~10× hit)*, so Prism only switches mid-session when the
+    expected win beats the eviction cost, and keeps the route sticky across
+    tool-call follow-ups. That cache-stickiness matters for long agent sessions.
+- **"Kilo Code" complexity routing** — `assessComplexity() → model tier`,
+  already present in `server/agentHarness.ts`. The router generalizes it.
+
+## NodeBench adaptation (why it's *easier* here)
+
+Most of NodeBench's served surface is **single-shot** — event `/ask`, query
+classification, extraction, QA judging — independent requests with **no prompt
+cache to evict between them**. Prism's ~10× switch penalty therefore does **not
+apply** to those paths, so we route **per-request with a pure heuristic planner**
+(zero added latency, no planner LLM call). Cache-aware stickiness — Prism's
+genuinely hard part — is reserved for the **multi-turn agent loops** (FastAgent,
+Convex agents) and layered on top of this core later.
+
+## Routing direction
+
+| Direction | When | Quality safety |
+|---|---|---|
+| **Escalate-up** (floor → climb on complexity) | default path is already cheap (e.g. `/ask` floors at Haiku) | always safe — going up never lowers quality. **V1 ships this.** |
+| **Demote-down** (strong default → drop to cheaper on light turns) | path over-provisions (e.g. persona router pinning Opus for every banker turn) | **eval-GATED** — only demote once the cheaper model's measured agreement with the target stays above threshold (fed by `agentRunJudge` / dogfood scores). *Next layer.* |
+
+## Reliability invariants (`.claude/rules/agentic_reliability.md`)
+
+- **DETERMINISTIC** — `routeLLM` is a pure function of `(taskClass, signals, env)`.
+  No `Date.now()` / `Math.random()`. Same turn → same route (replay-safe).
+- **HONEST** — `forceTarget` and any uncertainty resolve **up** to the quality
+  target, never silently down.
+- **BOUND** — pools are fixed finite literals.
+
+## Status
+
+| Task class | Pool (floor → heavy) | Wired call site |
+|---|---|---|
+| `ask_answer` | Haiku `claude-haiku-4-5-20251001` → Sonnet `claude-sonnet-4-6` | ✅ `convex/events.ts` `generateProviderAnswer` (PR: this one) |
+| `classify` | gemini-3.1-flash-lite (single) | ⬜ `server/routes/search.ts` query classify |
+| `extract` | flash-lite → flash | ⬜ `server/routes/search.ts` signal extraction |
+| `synthesize` | flash-lite → flash | ⬜ `server/routes/search.ts` answer synthesis |
+| `agent_reason` | Sonnet → Opus | ⬜ FastAgent loop (needs cache-sticky wrapper) |
+| `judge` | flash → haiku | ⬜ `agentRunJudge` / dogfood QA |
+
+Floors are pinned to **current production behavior**, so wiring a call site
+through the router is a no-op until signals say otherwise. Ops can pin/retune
+any model via env (`SCRATCHNODE_ASK_MODEL_LIGHT` / `_HEAVY`;
+`SCRATCHNODE_ASK_MODEL` still force-pins the whole `/ask` path).
+
+## Observability
+
+The `/ask` path emits a `model_route` trace step (visible in the "Show trace"
+UI) and stores `modelId` + `estimatedCostCents` on `liveEventAnswers`, so
+escalations and their cost are auditable per answer. `getAskTelemetry`
+aggregates them.
+
+## Roadmap (next PRs)
+
+1. Wire `classify` / `extract` / `synthesize` in `server/routes/search.ts`.
+2. Cache-sticky wrapper for `agent_reason` (FastAgent + Convex agents) — cache
+   the route per conversation, reuse on tool-result turns, switch only when the
+   expected win beats the cache-eviction cost.
+3. Eval-gated **demote-down**: per `(taskClass, model)` rolling agreement from
+   `agentRunJudge` / dogfood scores; demote only above threshold.
+4. Routing dashboard: cost + escalation rate per task class.

shared/llm/router.test.ts

@@ -0,0 +1,132 @@
+/**
+ * Scenario-based tests for the NodeBench LLM Router (shared/llm/router.ts).
+ *
+ * Per .claude/rules/scenario_testing.md each test starts from a real persona/goal.
+ * The router is the single model-selection point for served LLM features, so the
+ * risks are: a regression that escalates *simple* questions (cost blowup), a
+ * regression that *fails to* escalate genuinely hard ones (quality loss), and
+ * non-determinism (same turn routing differently across calls — breaks replay).
+ */
+import { afterEach, describe, expect, it } from "vitest";
+import {
+  routeLLM,
+  computeComplexityScore,
+  askAnswerSignals,
+  getPools,
+  ESCALATE_THRESHOLD,
+  HEAVY_THRESHOLD,
+} from "./router";
+
+const FLOOR = "claude-haiku-4-5-20251001";
+const HEAVY = "claude-sonnet-4-6";
+
+describe("LLM router — ask_answer escalation (event attendee)", () => {
+  /**
+   * Persona: attendee asks a quick factual lookup. Goal: fast, cheap answer.
+   * Expected: stays on the Haiku floor — no cost escalation for simple Qs.
+   */
+  it("keeps a short factual question on the Haiku floor", () => {
+    const r = routeLLM("ask_answer", askAnswerSignals("What time is the keynote?", 2));
+    expect(r.model).toBe(FLOOR);
+    expect(r.tier).toBe("light");
+    expect(r.escalated).toBe(false);
+  });
+
+  /**
+   * Persona: attendee asks a comparative/strategic question over many sources.
+   * Goal: a well-reasoned synthesis. Expected: escalate to Sonnet.
+   */
+  it("escalates a long analytical multi-entity question to Sonnet", () => {
+    const q =
+      "Compare the MCP auth approach versus the legacy RBAC model and explain why teams should pick one — what are the tradeoffs and risks for a multi-tenant deployment at scale?";
+    const r = routeLLM("ask_answer", askAnswerSignals(q, 7));
+    expect(r.model).toBe(HEAVY);
+    expect(r.tier).toBe("heavy");
+    expect(r.escalated).toBe(true);
+    expect(r.score).toBeGreaterThanOrEqual(HEAVY_THRESHOLD);
+  });
+
+  /**
+   * Many sources alone (heavy synthesis load) should lift complexity even for a
+   * plainly-worded question.
+   */
+  it("escalates when there are many sources to synthesize", () => {
+    const r = routeLLM("ask_answer", { inputChars: 50, sourceCount: 12, complexityHint: "medium", multiEntity: true });
+    expect(r.escalated).toBe(true);
+  });
+
+  /**
+   * Adversarial: a caller marks the turn high-stakes. Expected: always the
+   * quality target, regardless of how trivial the signals look.
+   */
+  it("forceTarget always routes to the quality target", () => {
+    const r = routeLLM("ask_answer", { inputChars: 5, sourceCount: 0, forceTarget: true });
+    expect(r.model).toBe(HEAVY);
+    expect(r.tier).toBe("heavy");
+    expect(r.escalated).toBe(true);
+    expect(r.reason).toMatch(/forced/i);
+  });
+});
+
+describe("LLM router — determinism + monotonicity (replay safety)", () => {
+  it("is deterministic: identical signals route identically across calls", () => {
+    const sig = askAnswerSignals("How should we evaluate voice agents for latency and barge-in?", 5);
+    const a = routeLLM("ask_answer", sig);
+    const b = routeLLM("ask_answer", sig);
+    expect(a).toEqual(b);
+  });
+
+  it("complexity score is monotonic — adding signal weight never lowers it", () => {
+    const base = computeComplexityScore({ inputChars: 100 });
+    const more = computeComplexityScore({ inputChars: 100, sourceCount: 10 });
+    const most = computeComplexityScore({ inputChars: 700, sourceCount: 10, multiEntity: true, complexityHint: "high" });
+    expect(more).toBeGreaterThanOrEqual(base);
+    expect(most).toBeGreaterThanOrEqual(more);
+    expect(most).toBeLessThanOrEqual(1); // bounded
+  });
+
+  it("score sits below the escalate threshold for trivial input", () => {
+    expect(computeComplexityScore({ inputChars: 20, sourceCount: 1 })).toBeLessThan(ESCALATE_THRESHOLD);
+  });
+});
+
+describe("LLM router — signal extraction", () => {
+  it("detects multi-entity comparison questions", () => {
+    expect(askAnswerSignals("Anthropic vs OpenAI for tool use", 3).multiEntity).toBe(true);
+    expect(askAnswerSignals("compare Brave and Serper", 3).multiEntity).toBe(true);
+    expect(askAnswerSignals("What is MCP?", 3).multiEntity).toBe(false);
+  });
+
+  it("flags analytical intent as high complexity", () => {
+    expect(askAnswerSignals("Why did the panel recommend scoped credentials?", 3).complexityHint).toBe("high");
+    expect(askAnswerSignals("Who is speaking next?", 1).complexityHint).toBe("low");
+  });
+});
+
+describe("LLM router — single-candidate pools never escalate", () => {
+  it("classify pool stays on its only (light) model even under high complexity", () => {
+    const r = routeLLM("classify", { complexityHint: "high", inputChars: 999, multiEntity: true });
+    expect(r.tier).toBe("light");
+    expect(r.escalated).toBe(false);
+    expect(getPools().classify.candidates).toHaveLength(1);
+  });
+});
+
+describe("LLM router — ops env override", () => {
+  afterEach(() => {
+    delete process.env.SCRATCHNODE_ASK_MODEL_HEAVY;
+    delete process.env.SCRATCHNODE_ASK_MODEL_LIGHT;
+  });
+
+  it("respects SCRATCHNODE_ASK_MODEL_HEAVY for the escalation target", () => {
+    process.env.SCRATCHNODE_ASK_MODEL_HEAVY = "claude-opus-4-7";
+    const r = routeLLM("ask_answer", { forceTarget: true });
+    expect(r.model).toBe("claude-opus-4-7");
+  });
+
+  it("respects SCRATCHNODE_ASK_MODEL_LIGHT for the floor", () => {
+    process.env.SCRATCHNODE_ASK_MODEL_LIGHT = "claude-haiku-pinned";
+    const r = routeLLM("ask_answer", askAnswerSignals("hi", 1));
+    expect(r.model).toBe("claude-haiku-pinned");
+  });
+});