diff --git a/convex/events.ts b/convex/events.ts index 81f0b8d5d..919bbb5cb 100644 --- a/convex/events.ts +++ b/convex/events.ts @@ -30,6 +30,7 @@ import { v } from "convex/values"; import { internal } from "./_generated/api"; import { action, query, mutation, internalMutation, internalQuery } from "./_generated/server"; import { enforceRateLimit } from "./scratchnodeRateLimit"; +import { routeLLM, askAnswerSignals } from "../shared/llm/router"; import { rerankWithGemini, condenseQuery, type TriCandidate } from "../shared/search/triSearch"; class ConvexError> extends Error { @@ -80,7 +81,10 @@ const MAX_OWNER_KEY_LEN = 120; const MAX_AGENT_CONTEXT_SOURCES = 6; const MAX_AGENT_CONTEXT_CHARS = 9_000; const MAX_LINKUP_SOURCES = 4; -const SCRATCHNODE_DEFAULT_ASK_MODEL = "claude-haiku-4-5-20251001"; +// /ask model selection lives in shared/llm/router.ts — the ask_answer pool +// (Haiku floor; escalates to Sonnet on long / analytical / multi-entity Qs). +// Legacy SCRATCHNODE_ASK_MODEL still force-pins for ops; the pool floor defaults +// to the exact prior Haiku id so an unset env is behavior-preserving. // Phase 4 follow-up Item 1: optional email channel for claim codes. // Validation kept basic on purpose — Resend re-validates server-side, and @@ -640,12 +644,13 @@ async function generateProviderAnswer(args: { eventName: string; question: string; sources: Array<{ title: string; excerpt: string; body: string; uri?: string }>; + model: string; // chosen by the LLM router (shared/llm/router.ts) at the call site }) { const apiKey = process.env.ANTHROPIC_API_KEY; if (!apiKey) { return { ok: false as const, detail: "ANTHROPIC_API_KEY not configured." }; } - const model = process.env.SCRATCHNODE_ASK_MODEL || SCRATCHNODE_DEFAULT_ASK_MODEL; + const model = args.model; const system = [ "You are ScratchNode's event answer agent.", "Answer only from the provided public event sources.", @@ -2059,10 +2064,25 @@ export const askAgent = action({ }); } + // Prism-style routing (shared/llm/router.ts): a deterministic planner picks + // the model for THIS question from the ask_answer pool (Haiku floor -> + // Sonnet on long / analytical / multi-entity Qs). Single-shot path with no + // prompt-cache to evict, so we route per request at zero added latency. + const askRoute = routeLLM("ask_answer", askAnswerSignals(question, topSources.length)); + const askModel = process.env.SCRATCHNODE_ASK_MODEL || askRoute.model; // legacy env force-pins + trace.push({ + step: "model_route", + status: "ok", + detail: process.env.SCRATCHNODE_ASK_MODEL + ? `env-pinned ${askModel}` + : `${askRoute.reason} -> ${askRoute.model}${askRoute.escalated ? " (escalated)" : ""}`, + durationMs: 0, + }); const provider = await generateProviderAnswer({ eventName: prepared.event.name, question, sources: topSources, + model: askModel, }); let body: string; let agentMode: "provider" | "provider_fallback"; diff --git a/docs/architecture/LLM_ROUTER.md b/docs/architecture/LLM_ROUTER.md new file mode 100644 index 000000000..3f7de15de --- /dev/null +++ b/docs/architecture/LLM_ROUTER.md @@ -0,0 +1,83 @@ +# NodeBench LLM Router (Prism-style planner-on-a-pool) + +One deterministic entry point — `routeLLM(taskClass, signals)` in +`shared/llm/router.ts` — for "which model serves THIS turn?". Replaces the +~7 scattered, uncoordinated model-selection mechanisms across the codebase +with a single planner-on-a-pool, so every served LLM feature routes the same +way and emits the same telemetry. + +## Prior art + +- **Augment "Prism"** (2026) — *Introducing Augment Prism: model routing to + reduce cost and maintain quality.* + + - **Borrowed:** a small fast *planner* picks, per user turn, which model from + a **fixed pool tuned to a quality target** handles the request ("you pick + Prism, Prism picks the model"). Their measured result: **20–30% cheaper at + negligible quality loss**, driven by the finding that *the top 10% of turns + consume ~57% of LLM rounds* — most turns are light work billed at frontier + rates because users pin the big model once and never switch. + - **Their hard part we deliberately scope out of V1:** *switching evicts the + prompt cache (~10× hit)*, so Prism only switches mid-session when the + expected win beats the eviction cost, and keeps the route sticky across + tool-call follow-ups. That cache-stickiness matters for long agent sessions. +- **"Kilo Code" complexity routing** — `assessComplexity() → model tier`, + already present in `server/agentHarness.ts`. The router generalizes it. + +## NodeBench adaptation (why it's *easier* here) + +Most of NodeBench's served surface is **single-shot** — event `/ask`, query +classification, extraction, QA judging — independent requests with **no prompt +cache to evict between them**. Prism's ~10× switch penalty therefore does **not +apply** to those paths, so we route **per-request with a pure heuristic planner** +(zero added latency, no planner LLM call). Cache-aware stickiness — Prism's +genuinely hard part — is reserved for the **multi-turn agent loops** (FastAgent, +Convex agents) and layered on top of this core later. + +## Routing direction + +| Direction | When | Quality safety | +|---|---|---| +| **Escalate-up** (floor → climb on complexity) | default path is already cheap (e.g. `/ask` floors at Haiku) | always safe — going up never lowers quality. **V1 ships this.** | +| **Demote-down** (strong default → drop to cheaper on light turns) | path over-provisions (e.g. persona router pinning Opus for every banker turn) | **eval-GATED** — only demote once the cheaper model's measured agreement with the target stays above threshold (fed by `agentRunJudge` / dogfood scores). *Next layer.* | + +## Reliability invariants (`.claude/rules/agentic_reliability.md`) + +- **DETERMINISTIC** — `routeLLM` is a pure function of `(taskClass, signals, env)`. + No `Date.now()` / `Math.random()`. Same turn → same route (replay-safe). +- **HONEST** — `forceTarget` and any uncertainty resolve **up** to the quality + target, never silently down. +- **BOUND** — pools are fixed finite literals. + +## Status + +| Task class | Pool (floor → heavy) | Wired call site | +|---|---|---| +| `ask_answer` | Haiku `claude-haiku-4-5-20251001` → Sonnet `claude-sonnet-4-6` | ✅ `convex/events.ts` `generateProviderAnswer` (PR: this one) | +| `classify` | gemini-3.1-flash-lite (single) | ⬜ `server/routes/search.ts` query classify | +| `extract` | flash-lite → flash | ⬜ `server/routes/search.ts` signal extraction | +| `synthesize` | flash-lite → flash | ⬜ `server/routes/search.ts` answer synthesis | +| `agent_reason` | Sonnet → Opus | ⬜ FastAgent loop (needs cache-sticky wrapper) | +| `judge` | flash → haiku | ⬜ `agentRunJudge` / dogfood QA | + +Floors are pinned to **current production behavior**, so wiring a call site +through the router is a no-op until signals say otherwise. Ops can pin/retune +any model via env (`SCRATCHNODE_ASK_MODEL_LIGHT` / `_HEAVY`; +`SCRATCHNODE_ASK_MODEL` still force-pins the whole `/ask` path). + +## Observability + +The `/ask` path emits a `model_route` trace step (visible in the "Show trace" +UI) and stores `modelId` + `estimatedCostCents` on `liveEventAnswers`, so +escalations and their cost are auditable per answer. `getAskTelemetry` +aggregates them. + +## Roadmap (next PRs) + +1. Wire `classify` / `extract` / `synthesize` in `server/routes/search.ts`. +2. Cache-sticky wrapper for `agent_reason` (FastAgent + Convex agents) — cache + the route per conversation, reuse on tool-result turns, switch only when the + expected win beats the cache-eviction cost. +3. Eval-gated **demote-down**: per `(taskClass, model)` rolling agreement from + `agentRunJudge` / dogfood scores; demote only above threshold. +4. Routing dashboard: cost + escalation rate per task class. diff --git a/shared/llm/router.test.ts b/shared/llm/router.test.ts new file mode 100644 index 000000000..b74e07088 --- /dev/null +++ b/shared/llm/router.test.ts @@ -0,0 +1,132 @@ +/** + * Scenario-based tests for the NodeBench LLM Router (shared/llm/router.ts). + * + * Per .claude/rules/scenario_testing.md each test starts from a real persona/goal. + * The router is the single model-selection point for served LLM features, so the + * risks are: a regression that escalates *simple* questions (cost blowup), a + * regression that *fails to* escalate genuinely hard ones (quality loss), and + * non-determinism (same turn routing differently across calls — breaks replay). + */ +import { afterEach, describe, expect, it } from "vitest"; +import { + routeLLM, + computeComplexityScore, + askAnswerSignals, + getPools, + ESCALATE_THRESHOLD, + HEAVY_THRESHOLD, +} from "./router"; + +const FLOOR = "claude-haiku-4-5-20251001"; +const HEAVY = "claude-sonnet-4-6"; + +describe("LLM router — ask_answer escalation (event attendee)", () => { + /** + * Persona: attendee asks a quick factual lookup. Goal: fast, cheap answer. + * Expected: stays on the Haiku floor — no cost escalation for simple Qs. + */ + it("keeps a short factual question on the Haiku floor", () => { + const r = routeLLM("ask_answer", askAnswerSignals("What time is the keynote?", 2)); + expect(r.model).toBe(FLOOR); + expect(r.tier).toBe("light"); + expect(r.escalated).toBe(false); + }); + + /** + * Persona: attendee asks a comparative/strategic question over many sources. + * Goal: a well-reasoned synthesis. Expected: escalate to Sonnet. + */ + it("escalates a long analytical multi-entity question to Sonnet", () => { + const q = + "Compare the MCP auth approach versus the legacy RBAC model and explain why teams should pick one — what are the tradeoffs and risks for a multi-tenant deployment at scale?"; + const r = routeLLM("ask_answer", askAnswerSignals(q, 7)); + expect(r.model).toBe(HEAVY); + expect(r.tier).toBe("heavy"); + expect(r.escalated).toBe(true); + expect(r.score).toBeGreaterThanOrEqual(HEAVY_THRESHOLD); + }); + + /** + * Many sources alone (heavy synthesis load) should lift complexity even for a + * plainly-worded question. + */ + it("escalates when there are many sources to synthesize", () => { + const r = routeLLM("ask_answer", { inputChars: 50, sourceCount: 12, complexityHint: "medium", multiEntity: true }); + expect(r.escalated).toBe(true); + }); + + /** + * Adversarial: a caller marks the turn high-stakes. Expected: always the + * quality target, regardless of how trivial the signals look. + */ + it("forceTarget always routes to the quality target", () => { + const r = routeLLM("ask_answer", { inputChars: 5, sourceCount: 0, forceTarget: true }); + expect(r.model).toBe(HEAVY); + expect(r.tier).toBe("heavy"); + expect(r.escalated).toBe(true); + expect(r.reason).toMatch(/forced/i); + }); +}); + +describe("LLM router — determinism + monotonicity (replay safety)", () => { + it("is deterministic: identical signals route identically across calls", () => { + const sig = askAnswerSignals("How should we evaluate voice agents for latency and barge-in?", 5); + const a = routeLLM("ask_answer", sig); + const b = routeLLM("ask_answer", sig); + expect(a).toEqual(b); + }); + + it("complexity score is monotonic — adding signal weight never lowers it", () => { + const base = computeComplexityScore({ inputChars: 100 }); + const more = computeComplexityScore({ inputChars: 100, sourceCount: 10 }); + const most = computeComplexityScore({ inputChars: 700, sourceCount: 10, multiEntity: true, complexityHint: "high" }); + expect(more).toBeGreaterThanOrEqual(base); + expect(most).toBeGreaterThanOrEqual(more); + expect(most).toBeLessThanOrEqual(1); // bounded + }); + + it("score sits below the escalate threshold for trivial input", () => { + expect(computeComplexityScore({ inputChars: 20, sourceCount: 1 })).toBeLessThan(ESCALATE_THRESHOLD); + }); +}); + +describe("LLM router — signal extraction", () => { + it("detects multi-entity comparison questions", () => { + expect(askAnswerSignals("Anthropic vs OpenAI for tool use", 3).multiEntity).toBe(true); + expect(askAnswerSignals("compare Brave and Serper", 3).multiEntity).toBe(true); + expect(askAnswerSignals("What is MCP?", 3).multiEntity).toBe(false); + }); + + it("flags analytical intent as high complexity", () => { + expect(askAnswerSignals("Why did the panel recommend scoped credentials?", 3).complexityHint).toBe("high"); + expect(askAnswerSignals("Who is speaking next?", 1).complexityHint).toBe("low"); + }); +}); + +describe("LLM router — single-candidate pools never escalate", () => { + it("classify pool stays on its only (light) model even under high complexity", () => { + const r = routeLLM("classify", { complexityHint: "high", inputChars: 999, multiEntity: true }); + expect(r.tier).toBe("light"); + expect(r.escalated).toBe(false); + expect(getPools().classify.candidates).toHaveLength(1); + }); +}); + +describe("LLM router — ops env override", () => { + afterEach(() => { + delete process.env.SCRATCHNODE_ASK_MODEL_HEAVY; + delete process.env.SCRATCHNODE_ASK_MODEL_LIGHT; + }); + + it("respects SCRATCHNODE_ASK_MODEL_HEAVY for the escalation target", () => { + process.env.SCRATCHNODE_ASK_MODEL_HEAVY = "claude-opus-4-7"; + const r = routeLLM("ask_answer", { forceTarget: true }); + expect(r.model).toBe("claude-opus-4-7"); + }); + + it("respects SCRATCHNODE_ASK_MODEL_LIGHT for the floor", () => { + process.env.SCRATCHNODE_ASK_MODEL_LIGHT = "claude-haiku-pinned"; + const r = routeLLM("ask_answer", askAnswerSignals("hi", 1)); + expect(r.model).toBe("claude-haiku-pinned"); + }); +}); diff --git a/shared/llm/router.ts b/shared/llm/router.ts new file mode 100644 index 000000000..108918ee0 --- /dev/null +++ b/shared/llm/router.ts @@ -0,0 +1,261 @@ +/** + * shared/llm/router.ts — NodeBench LLM Router (Prism-style planner-on-a-pool). + * + * One place every served LLM feature asks "which model for THIS turn?". A small + * DETERMINISTIC planner reads a task class + cheap signals and picks from a + * fixed, quality-targeted pool — matching Augment Prism's "you pick the router, + * the router picks the model" shape, without paying an extra LLM call on the + * latency-critical single-shot paths. + * + * Pattern: planner-on-a-pool model routing. + * Prior art: + * - Augment "Prism" (2026) — a small planner routes each user turn across a + * pool tuned to a frontier quality target; cache-aware sticky switching is + * the hard part on long agent sessions. + * https://www.augmentcode.com/blog/augment-prism-model-routing-to-reduce-cost-and-maintain-quality + * - "Kilo Code" complexity routing — assessComplexity() -> model tier, already + * present in server/agentHarness.ts; this generalizes it into one entry point. + * + * NodeBench adaptation (what's DIFFERENT from Prism, and why it's easier here): + * Most of NodeBench's surface is SINGLE-SHOT (event /ask, query classify, + * extraction, QA judge) — independent requests with NO prompt cache to evict + * between them. Prism's ~10x "switching evicts the cache" penalty therefore + * does NOT apply to those paths, so we route per-request with a pure heuristic + * planner (zero added latency, no planner LLM call). Cache-aware stickiness — + * Prism's genuinely hard part — is reserved for the multi-turn agent loops + * (FastAgent, Convex agents) and layered ON TOP of this core in a later step. + * + * Routing direction: + * - ESCALATE-UP (default → floor, climb on complexity) is ALWAYS quality-safe + * and is what V1 ships (e.g. ask_answer floors at Haiku, climbs to Sonnet on + * hard/analytical/multi-entity questions). No eval gate needed to go up. + * - DEMOTE-DOWN (default → strong, drop to cheaper on light turns) is the cost + * lever for over-provisioned paths (e.g. persona router pinning Opus). It is + * intentionally eval-GATED — only demote a task class to a cheaper model once + * that model's measured agreement with the target stays above threshold. That + * gate (fed by agentRunJudge / dogfood scores) is the next layer; not in V1. + * + * Reliability (.claude/rules/agentic_reliability.md): + * - DETERMINISTIC: routeLLM is a pure function of (taskClass, signals, env). + * No Date.now(), no Math.random() — same inputs always yield the same route. + * - HONEST: forceTarget and any uncertainty resolve UP to the quality target, + * never silently down. + * - BOUND: pools are fixed, finite literals. + * + * See docs/architecture/LLM_ROUTER.md. + */ + +export type RouteProvider = "anthropic" | "google" | "openai" | "openrouter"; +export type RouteTier = "light" | "balanced" | "heavy"; + +export type TaskClass = + | "ask_answer" // event /ask synthesis — Haiku floor, escalate to Sonnet on hard Qs + | "classify" // query intent classification — cheapest, latency-critical + | "extract" // structured signal extraction from text + | "synthesize" // final answer synthesis from a tool/retrieval trace + | "agent_reason" // multi-turn agent reasoning — strong target, demote on light turns + | "judge"; // eval / QA scoring — cheap but calibrated + +export interface RouteCandidate { + /** The EXACT provider model id sent on the wire (not an internal alias). */ + model: string; + provider: RouteProvider; + tier: RouteTier; + /** Relative $ weight within this pool (floor candidate = 1). Telemetry only. */ + relCost: number; +} + +export interface TaskPool { + /** Ordered lightest -> heaviest. candidates[0] is the cost floor. */ + candidates: RouteCandidate[]; +} + +export interface RouteSignals { + /** Length of the prompt / question in characters. */ + inputChars?: number; + /** Number of retrieval sources to synthesize over. */ + sourceCount?: number; + /** Upstream complexity assessment, if the caller already has one. */ + complexityHint?: "low" | "medium" | "high"; + /** Comparison / multi-subject query ("X vs Y", "compare ..."). */ + multiEntity?: boolean; + /** The answer needs fresh/external data. */ + freshnessRequired?: boolean; + /** Caller demands top quality regardless of signals (high-stakes). */ + forceTarget?: boolean; +} + +export interface RouteDecision { + taskClass: TaskClass; + model: string; + provider: RouteProvider; + tier: RouteTier; + /** 0..1 complexity score that drove the decision. */ + score: number; + /** Did we route above the pool floor? */ + escalated: boolean; + reason: string; +} + +// Climb above the floor once complexity crosses this; reach the heaviest tier at +// HEAVY_THRESHOLD. Tuned conservatively: only genuinely hard turns escalate. +export const ESCALATE_THRESHOLD = 0.5; +export const HEAVY_THRESHOLD = 0.8; + +/** Pure env read (no Date/random). Lets ops pin or retune without a deploy. */ +function envModel(name: string, fallback: string): string { + const raw = + typeof process !== "undefined" && process && process.env ? process.env[name] : undefined; + return (raw || "").trim() || fallback; +} + +/** + * The pools. IDs are the EXACT strings each provider's API expects. Floors are + * pinned to CURRENT production behavior, so routing a call site through here is + * a no-op until signals say otherwise. + * + * Only `ask_answer` is wired into a live call site in V1. The rest are the + * documented target-state pools the next PRs wire their call sites into. + */ +export function getPools(): Record { + return { + ask_answer: { + candidates: [ + { + model: envModel("SCRATCHNODE_ASK_MODEL_LIGHT", "claude-haiku-4-5-20251001"), + provider: "anthropic", + tier: "light", + relCost: 1, + }, + { + model: envModel("SCRATCHNODE_ASK_MODEL_HEAVY", "claude-sonnet-4-6"), + provider: "anthropic", + tier: "heavy", + relCost: 3, + }, + ], + }, + classify: { + candidates: [ + { model: "gemini-3.1-flash-lite-preview", provider: "google", tier: "light", relCost: 1 }, + ], + }, + extract: { + candidates: [ + { model: "gemini-3.1-flash-lite-preview", provider: "google", tier: "light", relCost: 1 }, + { model: "gemini-3-flash-preview", provider: "google", tier: "balanced", relCost: 2 }, + ], + }, + synthesize: { + candidates: [ + { model: "gemini-3.1-flash-lite-preview", provider: "google", tier: "light", relCost: 1 }, + { model: "gemini-3-flash-preview", provider: "google", tier: "balanced", relCost: 2 }, + ], + }, + agent_reason: { + candidates: [ + { model: "claude-sonnet-4-6", provider: "anthropic", tier: "balanced", relCost: 1 }, + { model: "claude-opus-4-7", provider: "anthropic", tier: "heavy", relCost: 5 }, + ], + }, + judge: { + candidates: [ + { model: "gemini-3-flash-preview", provider: "google", tier: "light", relCost: 1 }, + { model: "claude-haiku-4-5-20251001", provider: "anthropic", tier: "balanced", relCost: 2 }, + ], + }, + }; +} + +/** + * Deterministic complexity score in [0,1] from cheap signals. No model call. + * Additive + capped so it's monotonic: more/heavier signals never lower the score. + */ +export function computeComplexityScore(signals: RouteSignals = {}): number { + let n = 0; + if (signals.complexityHint === "high") n += 0.5; + else if (signals.complexityHint === "medium") n += 0.25; + const chars = signals.inputChars ?? 0; + if (chars > 600) n += 0.3; + else if (chars > 240) n += 0.15; + const sources = signals.sourceCount ?? 0; + if (sources > 8) n += 0.25; + else if (sources > 4) n += 0.12; + if (signals.multiEntity) n += 0.3; + if (signals.freshnessRequired) n += 0.1; + return Math.min(1, Number(n.toFixed(4))); +} + +function decide( + taskClass: TaskClass, + c: RouteCandidate, + floor: RouteCandidate, + score: number, + reason: string, +): RouteDecision { + return { + taskClass, + model: c.model, + provider: c.provider, + tier: c.tier, + score, + escalated: c.model !== floor.model || c.tier !== floor.tier, + reason, + }; +} + +/** + * Pick the model for a task class given cheap signals. Pure + deterministic. + * + * - forceTarget -> the heaviest candidate (quality target, fail-safe). + * - else climb from the floor by complexity score: + * score >= HEAVY_THRESHOLD -> heaviest + * score >= ESCALATE_THRESHOLD -> next tier up from floor (if any) + * otherwise -> floor + */ +export function routeLLM(taskClass: TaskClass, signals: RouteSignals = {}): RouteDecision { + const pool = getPools()[taskClass]; + const candidates = pool.candidates; + const floor = candidates[0]; + const heaviest = candidates[candidates.length - 1]; + + if (signals.forceTarget) { + return decide(taskClass, heaviest, floor, 1, "forced quality target (high-stakes / caller override)"); + } + + const score = computeComplexityScore(signals); + let chosen = floor; + if (score >= HEAVY_THRESHOLD) { + chosen = heaviest; + } else if (score >= ESCALATE_THRESHOLD && candidates.length > 1) { + chosen = candidates[1]; + } + + const reason = + chosen.model === floor.model && chosen.tier === floor.tier + ? `floor ${floor.tier} (complexity ${score.toFixed(2)})` + : `escalated to ${chosen.tier} (complexity ${score.toFixed(2)})`; + return decide(taskClass, chosen, floor, score, reason); +} + +// ── Per-task-class signal helpers ──────────────────────────────────────────── + +const ANALYTICAL_RE = + /\b(compare|comparison|versus|vs\.?|trade-?offs?|why|how should|strateg(y|ic)|implications?|pros and cons|better or worse|risks?)\b/i; +const MULTI_ENTITY_RE = /\bvs\.?\b|\bversus\b|\bcompare\b|\bbetween\b .+ and /i; + +/** + * Derive ask_answer routing signals from the raw question + retrieved source + * count. Short factual lookups stay on the Haiku floor; long, analytical, or + * multi-entity questions escalate to Sonnet. + */ +export function askAnswerSignals(question: string, sourceCount: number): RouteSignals { + const q = (question || "").trim(); + const analytical = ANALYTICAL_RE.test(q); + return { + inputChars: q.length, + sourceCount, + multiEntity: MULTI_ENTITY_RE.test(q), + complexityHint: analytical ? "high" : q.length > 160 ? "medium" : "low", + }; +}