From abc1e719a34a3f54d935bd1fef811f2968b786d2 Mon Sep 17 00:00:00 2001 From: Dutch Dim Date: Wed, 15 Jul 2026 20:52:57 +0200 Subject: [PATCH] feat: governance-aware LLM routing from OpenMythos benchmark scores MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit RoutingRequest gains riskCategory (an OpenMythos category like 'injection'). When set, the router prefers the healthy candidate whose model has the best latest benchmark score for that category — but only when the score clears GOVERNANCE_ROUTER_FLOOR (default 3/5). No score, no discriminating data, or below-floor: static task routing unchanged. Decisions cite their evidence ("scored 4.20/5 on 'injection'"). Scores come from openmythos_eval_runs metadata (latest completed run per subject model), so the nightly scheduler keeps the routing table current with zero extra state. Verified live: with a seeded 4.2 injection score the REST route endpoint picked that model with the governance reason; a 1.7 overthinking score fell back to static routing. No API changes needed — /apex/llm/route already passes the request body through. Co-Authored-By: Claude Fable 5 --- .../src/__tests__/governance-router.test.ts | 101 ++++++++++++++++++ .../server/src/services/llm-router-service.ts | 71 ++++++++++++ 2 files changed, 172 insertions(+) create mode 100644 packages/server/src/__tests__/governance-router.test.ts diff --git a/packages/server/src/__tests__/governance-router.test.ts b/packages/server/src/__tests__/governance-router.test.ts new file mode 100644 index 00000000..2c26f338 --- /dev/null +++ b/packages/server/src/__tests__/governance-router.test.ts @@ -0,0 +1,101 @@ +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import type { Database } from 'better-sqlite3'; +import { createTestDb } from './helpers/test-db'; +import { LlmRouterService } from '../services/llm-router-service'; + +const OLLAMA_MODEL = 'qwen2.5:14b-instruct-q4_K_M'; // the router's ollama provider model + +function insertEvalRun(db: Database, run: { + id: string; + subjectModel: string; + categoryScores: Record; + finishedAt?: string; +}) { + db.prepare(` + INSERT INTO openmythos_eval_runs (id, agent_id, status, total_cases, completed_cases, overall_score, started_at, finished_at, metadata) + VALUES (?, ?, 'completed', 78, 78, 3.0, ?, ?, ?) + `).run( + run.id, + `nightly:${run.subjectModel}`, + run.finishedAt ?? '2026-07-15T10:00:00.000Z', + run.finishedAt ?? '2026-07-15T10:05:00.000Z', + JSON.stringify({ subject_model: run.subjectModel, category_scores: run.categoryScores }), + ); +} + +describe('LlmRouterService governance routing', () => { + let db: Database; + let service: LlmRouterService; + const previousFloor = process.env.GOVERNANCE_ROUTER_FLOOR; + + beforeEach(() => { + delete process.env.GOVERNANCE_ROUTER_FLOOR; + db = createTestDb(); + service = new LlmRouterService(db); + // ollama needs no API key; a successful call marks it active + service.recordPerformance({ provider: 'ollama', taskType: 'chat', latencyMs: 100, success: true }); + }); + + afterEach(() => { + db.close(); + if (previousFloor === undefined) delete process.env.GOVERNANCE_ROUTER_FLOOR; + else process.env.GOVERNANCE_ROUTER_FLOOR = previousFloor; + }); + + it('prefers the provider whose model has the best benchmark score for the risk category', () => { + insertEvalRun(db, { id: 'r1', subjectModel: OLLAMA_MODEL, categoryScores: { injection: 4.2 } }); + + const decision = service.route({ taskType: 'chat', prompt: 'handle untrusted input', riskCategory: 'injection' }); + + expect(decision.provider).toBe('ollama'); + expect(decision.model).toBe(OLLAMA_MODEL); + expect(decision.reason).toContain('Governance'); + expect(decision.reason).toContain('injection'); + expect(decision.reason).toContain('4.20'); + }); + + it('falls back to static routing when the only scored model is below the floor', () => { + insertEvalRun(db, { id: 'r1', subjectModel: OLLAMA_MODEL, categoryScores: { overthinking: 1.7 } }); + + const decision = service.route({ taskType: 'chat', prompt: 'x', riskCategory: 'overthinking' }); + + // ollama is still the first healthy static candidate, but via the static path + expect(decision.reason).not.toContain('Governance'); + }); + + it('the floor is configurable via GOVERNANCE_ROUTER_FLOOR', () => { + process.env.GOVERNANCE_ROUTER_FLOOR = '1.5'; + insertEvalRun(db, { id: 'r1', subjectModel: OLLAMA_MODEL, categoryScores: { overthinking: 1.7 } }); + + const decision = service.route({ taskType: 'chat', prompt: 'x', riskCategory: 'overthinking' }); + + expect(decision.reason).toContain('Governance'); + }); + + it('the latest eval run wins when a model was re-benchmarked', () => { + insertEvalRun(db, { id: 'r1', subjectModel: OLLAMA_MODEL, categoryScores: { injection: 4.5 }, finishedAt: '2026-07-14T10:00:00.000Z' }); + insertEvalRun(db, { id: 'r2', subjectModel: OLLAMA_MODEL, categoryScores: { injection: 2.0 }, finishedAt: '2026-07-15T10:00:00.000Z' }); + + const decision = service.route({ taskType: 'chat', prompt: 'x', riskCategory: 'injection' }); + + // dropped below the default floor of 3 → static routing + expect(decision.reason).not.toContain('Governance'); + }); + + it('behaves exactly as before when no riskCategory is given', () => { + insertEvalRun(db, { id: 'r1', subjectModel: OLLAMA_MODEL, categoryScores: { injection: 4.2 } }); + + const decision = service.route({ taskType: 'chat', prompt: 'x' }); + + expect(decision.reason).not.toContain('Governance'); + expect(decision.provider).toBe('ollama'); + }); + + it('ignores governance data for categories with no scores', () => { + insertEvalRun(db, { id: 'r1', subjectModel: OLLAMA_MODEL, categoryScores: { injection: 4.2 } }); + + const decision = service.route({ taskType: 'chat', prompt: 'x', riskCategory: 'cross-lingual' }); + + expect(decision.reason).not.toContain('Governance'); + }); +}); diff --git a/packages/server/src/services/llm-router-service.ts b/packages/server/src/services/llm-router-service.ts index d0efe46d..5aeef303 100644 --- a/packages/server/src/services/llm-router-service.ts +++ b/packages/server/src/services/llm-router-service.ts @@ -32,6 +32,13 @@ interface RoutingRequest { maxTokens?: number; temperature?: number; preferredProvider?: LlmProvider; + /** + * OpenMythos risk category (e.g. 'injection', 'hallucination'). When set, + * routing prefers the healthy provider whose model has the best benchmark + * score for that category (from openmythos_eval_runs), if any scores + * >= GOVERNANCE_ROUTER_FLOOR. Otherwise static task routing applies. + */ + riskCategory?: string; } interface RoutingDecision { @@ -91,6 +98,12 @@ export class LlmRouterService { ? [request.preferredProvider] : this.taskRouting[request.taskType] || ['litellm']; + // Governance override: benchmark evidence beats static preference order. + if (request.riskCategory) { + const governed = this.pickByGovernance(request, candidates); + if (governed) return governed; + } + // Find first healthy candidate for (const providerName of candidates) { const provider = this.providers.get(providerName); @@ -212,6 +225,64 @@ export class LlmRouterService { }; } + /** + * Pick the healthy candidate whose model has the best OpenMythos score for + * the requested risk category, if any healthy candidate is scored at or + * above the floor. Returns null when governance data doesn't discriminate, + * so static routing decides. + */ + private pickByGovernance(request: RoutingRequest, candidates: LlmProvider[]): RoutingDecision | null { + const scores = this.governanceScores(request.riskCategory!); + if (scores.size === 0) return null; + + const floor = Number(process.env.GOVERNANCE_ROUTER_FLOOR ?? '3'); + let best: { provider: ProviderConfig; score: number } | null = null; + for (const providerName of candidates) { + const provider = this.providers.get(providerName); + if (!provider || provider.status !== 'active') continue; + if (provider.apiKeyEnv && !process.env[provider.apiKeyEnv]) continue; + const score = scores.get(provider.model); + if (score === undefined || score < floor) continue; + if (!best || score > best.score) best = { provider, score }; + } + if (!best) return null; + + return { + provider: best.provider.name, + model: best.provider.model, + reason: `Governance: ${best.provider.model} scored ${best.score.toFixed(2)}/5 on '${request.riskCategory}' (OpenMythos benchmark)`, + estimatedCost: (request.maxTokens || 4096) / 1_000_000 * best.provider.costPerMtok, + estimatedLatencyMs: best.provider.avgLatencyMs, + }; + } + + /** + * Latest OpenMythos category score per subject model, from completed eval runs. + */ + private governanceScores(category: string): Map { + const scores = new Map(); + let rows: Array<{ metadata: string }>; + try { + rows = this.db.prepare(` + SELECT metadata FROM openmythos_eval_runs + WHERE status = 'completed' + ORDER BY finished_at ASC + `).all() as Array<{ metadata: string }>; + } catch { + return scores; // isolated consumers without the eval table + } + for (const row of rows) { + try { + const metadata = JSON.parse(row.metadata || '{}') as { subject_model?: string; category_scores?: Record }; + const score = metadata.category_scores?.[category]; + if (metadata.subject_model && typeof score === 'number') { + scores.set(metadata.subject_model, score); // ascending order → latest run wins + } + } catch { /* skip malformed rows */ } + } + return scores; + } + private findProviderByModelId(modelId: string): ProviderConfig | undefined { for (const provider of this.providers.values()) { if (provider.model === modelId || provider.name === modelId) return provider;