diff --git a/packages/dashboard/src/App.tsx b/packages/dashboard/src/App.tsx index 1c8e53e4..7b2ff4b1 100644 --- a/packages/dashboard/src/App.tsx +++ b/packages/dashboard/src/App.tsx @@ -39,6 +39,7 @@ const WorkstationUrlsPage = lazy(() => import('./pages/WorkstationUrlsPage').the const EconomyPage = lazy(() => import('./pages/EconomyPage').then((module) => ({ default: module.EconomyPage }))); const PipelineBuilderPage = lazy(() => import('./pages/PipelineBuilderPage').then((module) => ({ default: module.PipelineBuilderPage }))); const FederationPage = lazy(() => import('./pages/FederationPage').then((module) => ({ default: module.FederationPage }))); +const GovernanceScorecardPage = lazy(() => import('./pages/GovernanceScorecardPage').then((module) => ({ default: module.GovernanceScorecardPage }))); function DataLoader({ children }: { children: React.ReactNode }) { const { setTasks, setAgents } = useStore(); @@ -92,6 +93,7 @@ export function App() { } /> } /> } /> + } /> } /> } /> } /> diff --git a/packages/dashboard/src/components/Layout.tsx b/packages/dashboard/src/components/Layout.tsx index 38bebd2d..4e5c79b3 100644 --- a/packages/dashboard/src/components/Layout.tsx +++ b/packages/dashboard/src/components/Layout.tsx @@ -1,6 +1,6 @@ import { useEffect, useState, type ReactNode } from 'react'; import { Outlet, Link, useLocation } from 'react-router-dom'; -import { Activity, ListTodo, Users, Shield, CheckSquare, PlugZap, BarChart3, ScrollText, FolderGit, LogOut, DollarSign, Network, Cpu, Workflow, BrainCircuit, Gauge, BookUser, Brain, Menu, X } from 'lucide-react'; +import { Activity, ListTodo, Users, Shield, ShieldCheck, CheckSquare, PlugZap, BarChart3, ScrollText, FolderGit, LogOut, DollarSign, Network, Cpu, Workflow, BrainCircuit, Gauge, BookUser, Brain, Menu, X } from 'lucide-react'; import { useAuthStore } from '../lib/auth-store'; export function Layout() { @@ -86,6 +86,12 @@ export function Layout() { label="Policies" active={isActive('/policies')} /> + } + label="Governance" + active={isActive('/governance')} + /> } diff --git a/packages/dashboard/src/lib/api.ts b/packages/dashboard/src/lib/api.ts index 60eef931..872f41e7 100644 --- a/packages/dashboard/src/lib/api.ts +++ b/packages/dashboard/src/lib/api.ts @@ -33,6 +33,29 @@ export const API_BASE = import.meta.env.PROD ? '/api' : import.meta.env.VITE_API const AUTH_SESSION_KEY = 'djimitflo_auth_session'; +export type AgentGovernanceScore = { + agentId: string; + overallScore: number; + categoryScores: Record; + totalCases: number; + lastEvalAt: string; + trend: 'improving' | 'stable' | 'declining'; +}; + +export type OpenMythosRun = { + id: string; + agentId: string; + status: string; + totalCases: number; + completedCases: number; + overallScore: number; + subjectModel: string | null; + oracleCases: number | null; + judgeCases: number | null; + startedAt: string | null; + finishedAt: string | null; +}; + type UsageQuota = { provider: string; tier: string; @@ -974,6 +997,20 @@ class ApiClient { return this.request(`/evidence/review/${taskId}`); } + // OpenMythos governance + async getOpenMythosLeaderboard(): Promise<{ leaderboard: AgentGovernanceScore[] }> { + return this.request('/openmythos/leaderboard'); + } + + async getOpenMythosRuns(limit?: number): Promise<{ runs: OpenMythosRun[] }> { + const query = limit ? `?limit=${limit}` : ''; + return this.request(`/openmythos/runs${query}`); + } + + async getOpenMythosTrend(agentId: string): Promise<{ agentId: string; trend: Array<{ date: string; score: number }> }> { + return this.request(`/openmythos/trend/${encodeURIComponent(agentId)}`); + } + // Observability async getObservabilityMetrics(): Promise { return this.request('/observability/metrics'); diff --git a/packages/dashboard/src/pages/GovernanceScorecardPage.test.tsx b/packages/dashboard/src/pages/GovernanceScorecardPage.test.tsx new file mode 100644 index 00000000..159b6096 --- /dev/null +++ b/packages/dashboard/src/pages/GovernanceScorecardPage.test.tsx @@ -0,0 +1,58 @@ +import { describe, it, expect, vi } from 'vitest'; +import { render, screen } from '@testing-library/react'; +import { GovernanceScorecardPage } from './GovernanceScorecardPage'; + +vi.mock('../lib/api', () => ({ + api: { + getOpenMythosLeaderboard: vi.fn().mockResolvedValue({ + leaderboard: [ + { + agentId: 'agent-a', + overallScore: 3.5, + categoryScores: { injection: 4.0, hallucination: 2.1 }, + totalCases: 78, + lastEvalAt: '2026-07-15T10:05:00.000Z', + trend: 'improving', + }, + ], + }), + getOpenMythosRuns: vi.fn().mockResolvedValue({ + runs: [ + { + id: 'run-12345678', + agentId: 'agent-a', + status: 'completed', + totalCases: 78, + completedCases: 78, + overallScore: 3.5, + subjectModel: 'llama3.1:8b', + oracleCases: 78, + judgeCases: 0, + startedAt: '2026-07-15T10:00:00.000Z', + finishedAt: '2026-07-15T10:05:00.000Z', + }, + ], + }), + }, +})); + +describe('GovernanceScorecardPage', () => { + it('renders the leaderboard with scores, trend, and category chips', async () => { + render(); + + expect(screen.getByText('Governance Scorecard')).toBeTruthy(); + expect((await screen.findAllByText('agent-a')).length).toBeGreaterThan(0); + expect(screen.getAllByText('3.50').length).toBeGreaterThan(0); + expect(screen.getByLabelText('improving')).toBeTruthy(); + expect(screen.getByText('injection 4.0')).toBeTruthy(); + expect(screen.getByText('hallucination 2.1')).toBeTruthy(); + }); + + it('renders the recent runs with model and oracle provenance', async () => { + render(); + + expect(await screen.findByText('llama3.1:8b')).toBeTruthy(); + expect(screen.getByText('completed')).toBeTruthy(); + expect(screen.getByText(/78 oracle/)).toBeTruthy(); + }); +}); diff --git a/packages/dashboard/src/pages/GovernanceScorecardPage.tsx b/packages/dashboard/src/pages/GovernanceScorecardPage.tsx new file mode 100644 index 00000000..026f799b --- /dev/null +++ b/packages/dashboard/src/pages/GovernanceScorecardPage.tsx @@ -0,0 +1,141 @@ +import { useCallback, useEffect, useState } from 'react'; +import { ShieldCheck, TrendingUp, TrendingDown, Minus, RefreshCw, FlaskConical } from 'lucide-react'; +import { api, type AgentGovernanceScore, type OpenMythosRun } from '../lib/api'; + +function scoreColor(score: number): string { + if (score >= 4) return 'text-status-completed'; + if (score >= 3) return 'text-risk-medium'; + return 'text-status-error'; +} + +function TrendIcon({ trend }: { trend: AgentGovernanceScore['trend'] }) { + if (trend === 'improving') return ; + if (trend === 'declining') return ; + return ; +} + +export function GovernanceScorecardPage() { + const [leaderboard, setLeaderboard] = useState([]); + const [runs, setRuns] = useState([]); + const [loading, setLoading] = useState(true); + const [error, setError] = useState(null); + + const load = useCallback(async () => { + setLoading(true); + setError(null); + try { + const [board, recent] = await Promise.all([ + api.getOpenMythosLeaderboard(), + api.getOpenMythosRuns(20), + ]); + setLeaderboard(board.leaderboard); + setRuns(recent.runs); + } catch (err) { + setError(err instanceof Error ? err.message : 'Failed to load governance data'); + } finally { + setLoading(false); + } + }, []); + + useEffect(() => { + void load(); + }, [load]); + + return ( +
+
+
+

Governance Scorecard

+

+ OpenMythos Governance Benchmark results per agent — latest score, category breakdown, and trend (0–5 scale). +

+
+ +
+ + {error && ( +
{error}
+ )} + + {loading ? ( +
Loading governance data...
+ ) : leaderboard.length === 0 ? ( +
+ +

No governance evaluations recorded yet.

+

POST /api/openmythos/eval/:agentId to run the benchmark.

+
+ ) : ( +
+

Leaderboard

+
+ {leaderboard.map((agent, rank) => ( +
+
+ #{rank + 1} + {agent.agentId} + {agent.overallScore.toFixed(2)} + + + {agent.totalCases} cases · {agent.lastEvalAt ? new Date(agent.lastEvalAt).toLocaleString() : 'n/a'} + +
+ {Object.keys(agent.categoryScores).length > 0 && ( +
+ {Object.entries(agent.categoryScores) + .sort(([, a], [, b]) => a - b) + .map(([category, score]) => ( + + {category} {score.toFixed(1)} + + ))} +
+ )} +
+ ))} +
+
+ )} + + {!loading && runs.length > 0 && ( +
+

Recent Eval Runs

+
+ {runs.map((run) => ( +
+ + {(run.id ?? '').slice(0, 8)} + {run.agentId} + {run.subjectModel && ( + {run.subjectModel} + )} + {run.status} + {run.overallScore.toFixed(2)} + + {run.completedCases}/{run.totalCases} cases + {run.oracleCases !== null && ` · ${run.oracleCases} oracle`} + {run.finishedAt && ` · ${new Date(run.finishedAt).toLocaleString()}`} + +
+ ))} +
+
+ )} +
+ ); +} diff --git a/packages/mcp-server/src/__tests__/mcp-server.test.ts b/packages/mcp-server/src/__tests__/mcp-server.test.ts index 2a651645..386079c5 100644 --- a/packages/mcp-server/src/__tests__/mcp-server.test.ts +++ b/packages/mcp-server/src/__tests__/mcp-server.test.ts @@ -6,6 +6,7 @@ import { registerLoopTools } from '../tools/loops.js'; import { registerGoalTools } from '../tools/goals.js'; import { registerAgentTools } from '../tools/agents.js'; import { registerMissionControlTools } from '../tools/mission-control.js'; +import { registerOpenMythosTools } from '../tools/openmythos.js'; function createTestDb(): DbHandle { const db = new Database(':memory:'); @@ -43,6 +44,14 @@ function createTestDb(): DbHandle { metadata_json TEXT DEFAULT '{}', level TEXT DEFAULT 'info', created_at TEXT NOT NULL ); + CREATE TABLE openmythos_eval_runs ( + id TEXT PRIMARY KEY, agent_id TEXT NOT NULL, + started_at TEXT, finished_at TEXT, + total_cases INTEGER DEFAULT 0, completed_cases INTEGER DEFAULT 0, + overall_score REAL DEFAULT 0, status TEXT DEFAULT 'pending', + categories_json TEXT DEFAULT '[]', metadata TEXT DEFAULT '{}', + created_at TEXT NOT NULL DEFAULT (datetime('now')) + ); `); return { db, close: () => db.close() }; } @@ -53,6 +62,7 @@ function createTestServer(dbHandle: DbHandle): McpServer { registerGoalTools(server, dbHandle); registerAgentTools(server, dbHandle); registerMissionControlTools(server, dbHandle); + registerOpenMythosTools(server, dbHandle); return server; } @@ -110,4 +120,48 @@ describe('MCP Server Tools', () => { expect(parsed.summary.activeLoans).toBe(0); expect(parsed.summary.pendingGoals).toBe(0); }); + + it('registers the openmythos tools', () => { + const toolNames = Object.keys((server as any)._registeredTools || {}); + expect(toolNames).toContain('djimitflo_openmythos_leaderboard'); + expect(toolNames).toContain('djimitflo_openmythos_score'); + }); + + it('openmythos_leaderboard ranks latest completed run per agent', async () => { + const insert = dbHandle.db.prepare(` + INSERT INTO openmythos_eval_runs (id, agent_id, status, completed_cases, overall_score, finished_at, metadata, created_at) + VALUES (?, ?, ?, ?, ?, ?, ?, datetime('now')) + `); + insert.run('r1', 'agent-a', 'completed', 78, 2.0, '2026-07-14T10:00:00Z', '{"category_scores":{"injection":3.0},"subject_model":"llama3.1:8b"}'); + insert.run('r2', 'agent-a', 'completed', 78, 3.5, '2026-07-15T10:00:00Z', '{"category_scores":{"injection":4.0},"subject_model":"llama3.1:8b"}'); + insert.run('r3', 'agent-b', 'completed', 78, 2.5, '2026-07-15T10:00:00Z', '{}'); + insert.run('r4', 'agent-c', 'failed', 0, 0, null, '{}'); + + const tool = (server as any)._registeredTools['djimitflo_openmythos_leaderboard']; + const result = await tool.handler({}); + const parsed = JSON.parse(result.content[0].text); + + expect(parsed.map((row: { agentId: string }) => row.agentId)).toEqual(['agent-a', 'agent-b']); + expect(parsed[0].overallScore).toBe(3.5); + expect(parsed[0].categoryScores).toEqual({ injection: 4.0 }); + expect(parsed[0].subjectModel).toBe('llama3.1:8b'); + }); + + it('openmythos_score returns latest score with trend, and errors on unknown agent', async () => { + const insert = dbHandle.db.prepare(` + INSERT INTO openmythos_eval_runs (id, agent_id, status, completed_cases, overall_score, finished_at, metadata, created_at) + VALUES (?, ?, 'completed', 78, ?, ?, '{}', datetime('now')) + `); + insert.run('r1', 'agent-a', 2.0, '2026-07-14T10:00:00Z'); + insert.run('r2', 'agent-a', 3.0, '2026-07-15T10:00:00Z'); + + const tool = (server as any)._registeredTools['djimitflo_openmythos_score']; + const result = await tool.handler({ agentId: 'agent-a' }); + const parsed = JSON.parse(result.content[0].text); + expect(parsed.overallScore).toBe(3.0); + expect(parsed.trend.map((t: { score: number }) => t.score)).toEqual([2.0, 3.0]); + + const missing = await tool.handler({ agentId: 'nope' }); + expect(missing.isError).toBe(true); + }); }); diff --git a/packages/mcp-server/src/index.ts b/packages/mcp-server/src/index.ts index 3c60621f..c28a1fa4 100644 --- a/packages/mcp-server/src/index.ts +++ b/packages/mcp-server/src/index.ts @@ -21,6 +21,7 @@ import { registerAgentTools } from './tools/agents.js'; import { registerMissionControlTools } from './tools/mission-control.js'; import { registerOrchestrationTools } from './tools/orchestration.js'; import { registerOkfTools } from './tools/okf.js'; +import { registerOpenMythosTools } from './tools/openmythos.js'; interface ServerOptions { transport: 'stdio' | 'http'; @@ -52,6 +53,7 @@ async function main() { registerMissionControlTools(server, db); registerOrchestrationTools(server, db); registerOkfTools(server); + registerOpenMythosTools(server, db); if (opts.transport === 'stdio') { const transport = new StdioServerTransport(); diff --git a/packages/mcp-server/src/tools/openmythos.ts b/packages/mcp-server/src/tools/openmythos.ts new file mode 100644 index 00000000..5e4eb0a3 --- /dev/null +++ b/packages/mcp-server/src/tools/openmythos.ts @@ -0,0 +1,99 @@ +/** + * OpenMythos governance MCP tools. + * Exposes: openmythos_leaderboard, openmythos_score + * + * Read-only views over openmythos_eval_runs — evals themselves are started + * via the REST API (POST /api/openmythos/eval/:agentId), not from MCP. + */ + +import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; +import { z } from 'zod'; +import type { DbHandle } from '../db.js'; + +interface RunRow { + id: string; + agent_id: string; + status: string; + total_cases: number; + completed_cases: number; + overall_score: number; + started_at: string | null; + finished_at: string | null; + metadata: string; +} + +function parseMetadata(raw: string): { subject_model?: string; category_scores?: Record; oracle_cases?: number; judge_cases?: number } { + try { return JSON.parse(raw || '{}'); } catch { return {}; } +} + +export function registerOpenMythosTools(server: McpServer, dbHandle: DbHandle) { + const { db } = dbHandle; + + server.registerTool( + 'djimitflo_openmythos_leaderboard', + { + description: 'Governance leaderboard: latest OpenMythos benchmark score per agent (0-5 scale), best first, with per-category scores', + inputSchema: {}, + }, + async () => { + // SQLite bare-column semantics: with MAX() in a GROUP BY, the other + // selected columns come from the max row — one pass, no correlated subquery. + const rows = db.prepare(` + SELECT id, agent_id, status, total_cases, completed_cases, overall_score, started_at, MAX(finished_at) AS finished_at, metadata + FROM openmythos_eval_runs + WHERE status = 'completed' + GROUP BY agent_id + ORDER BY overall_score DESC + `).all() as RunRow[]; + + const leaderboard = rows.map((r) => { + const metadata = parseMetadata(r.metadata); + return { + agentId: r.agent_id, + overallScore: r.overall_score, + categoryScores: metadata.category_scores ?? {}, + subjectModel: metadata.subject_model ?? null, + completedCases: r.completed_cases, + lastEvalAt: r.finished_at, + }; + }); + return { content: [{ type: 'text' as const, text: JSON.stringify(leaderboard, null, 2) }] }; + } + ); + + server.registerTool( + 'djimitflo_openmythos_score', + { + description: 'Latest OpenMythos governance score for one agent, with per-category scores and recent score trend', + inputSchema: { + agentId: z.string().describe('The agent ID that was evaluated'), + }, + }, + async ({ agentId }: { agentId: string }) => { + const runs = db.prepare(` + SELECT id, agent_id, status, total_cases, completed_cases, overall_score, started_at, finished_at, metadata + FROM openmythos_eval_runs + WHERE agent_id = ? AND status = 'completed' + ORDER BY finished_at DESC + LIMIT 10 + `).all(agentId) as RunRow[]; + + if (runs.length === 0) { + return { content: [{ type: 'text' as const, text: `No completed OpenMythos evaluations for agent: ${agentId}` }], isError: true }; + } + + const latest = runs[0]; + const metadata = parseMetadata(latest.metadata); + const score = { + agentId, + overallScore: latest.overall_score, + categoryScores: metadata.category_scores ?? {}, + subjectModel: metadata.subject_model ?? null, + completedCases: latest.completed_cases, + lastEvalAt: latest.finished_at, + trend: runs.slice().reverse().map((r) => ({ date: r.finished_at, score: r.overall_score })), + }; + return { content: [{ type: 'text' as const, text: JSON.stringify(score, null, 2) }] }; + } + ); +} diff --git a/packages/server/src/__tests__/openmythos-scorecard.test.ts b/packages/server/src/__tests__/openmythos-scorecard.test.ts new file mode 100644 index 00000000..78ee9363 --- /dev/null +++ b/packages/server/src/__tests__/openmythos-scorecard.test.ts @@ -0,0 +1,123 @@ +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import type { Database } from 'better-sqlite3'; +import { createTestDb } from './helpers/test-db'; +import { OpenMythosEvalService } from '../services/openmythos-eval-service'; + +function insertRun(db: Database, run: { + id: string; + agentId: string; + status?: string; + overallScore?: number; + completedCases?: number; + startedAt?: string; + finishedAt?: string; + metadata?: Record; +}) { + db.prepare(` + INSERT INTO openmythos_eval_runs (id, agent_id, status, total_cases, completed_cases, overall_score, started_at, finished_at, metadata) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) + `).run( + run.id, + run.agentId, + run.status ?? 'completed', + run.completedCases ?? 78, + run.completedCases ?? 78, + run.overallScore ?? 3.0, + run.startedAt ?? '2026-07-15T10:00:00.000Z', + run.finishedAt ?? '2026-07-15T10:05:00.000Z', + JSON.stringify(run.metadata ?? {}), + ); +} + +describe('OpenMythos scorecard queries', () => { + let db: Database; + let service: OpenMythosEvalService; + + beforeEach(() => { + db = createTestDb(); + service = new OpenMythosEvalService(db); + }); + + afterEach(() => { + db.close(); + }); + + describe('listRuns', () => { + it('returns runs newest first with parsed metadata', () => { + insertRun(db, { + id: 'run-old', agentId: 'agent-a', startedAt: '2026-07-14T10:00:00.000Z', + metadata: { subject_model: 'llama3.1:8b', oracle_cases: 78, judge_cases: 0 }, + }); + insertRun(db, { id: 'run-new', agentId: 'agent-b', startedAt: '2026-07-15T10:00:00.000Z' }); + + const runs = service.listRuns(); + expect(runs.map((r) => r.id)).toEqual(['run-new', 'run-old']); + expect(runs[1]).toMatchObject({ + agentId: 'agent-a', + subjectModel: 'llama3.1:8b', + oracleCases: 78, + judgeCases: 0, + }); + expect(runs[0].subjectModel).toBeNull(); + }); + + it('respects the limit and survives malformed metadata', () => { + for (let i = 0; i < 5; i++) { + insertRun(db, { id: `run-${i}`, agentId: 'agent-a', startedAt: `2026-07-1${i}T10:00:00.000Z` }); + } + db.prepare("UPDATE openmythos_eval_runs SET metadata = 'not json' WHERE id = 'run-4'").run(); + + const runs = service.listRuns(2); + expect(runs).toHaveLength(2); + expect(runs[0].id).toBe('run-4'); + expect(runs[0].subjectModel).toBeNull(); + }); + + it('returns empty array when there are no runs', () => { + expect(service.listRuns()).toEqual([]); + }); + }); + + describe('getLeaderboard', () => { + it('ranks agents by latest completed score, best first', () => { + insertRun(db, { + id: 'a1', agentId: 'agent-a', overallScore: 2.0, finishedAt: '2026-07-14T10:00:00.000Z', + metadata: { category_scores: { injection: 3.5 } }, + }); + insertRun(db, { + id: 'a2', agentId: 'agent-a', overallScore: 3.5, finishedAt: '2026-07-15T10:00:00.000Z', + metadata: { category_scores: { injection: 4.0 } }, + }); + insertRun(db, { id: 'b1', agentId: 'agent-b', overallScore: 2.5 }); + + const board = service.getLeaderboard(); + expect(board.map((s) => s.agentId)).toEqual(['agent-a', 'agent-b']); + // latest run wins, not the best run + expect(board[0].overallScore).toBe(3.5); + expect(board[0].categoryScores).toEqual({ injection: 4.0 }); + expect(board[0].trend).toBe('improving'); + }); + + it('excludes agents with only failed runs', () => { + insertRun(db, { id: 'f1', agentId: 'agent-failed', status: 'failed' }); + insertRun(db, { id: 'c1', agentId: 'agent-ok' }); + + const board = service.getLeaderboard(); + expect(board.map((s) => s.agentId)).toEqual(['agent-ok']); + }); + + it('returns empty array with no eval data', () => { + expect(service.getLeaderboard()).toEqual([]); + }); + + it('one agent with malformed metadata does not sink the leaderboard', () => { + insertRun(db, { id: 'good-1', agentId: 'agent-good', overallScore: 3.0 }); + insertRun(db, { id: 'bad-1', agentId: 'agent-bad', overallScore: 2.0 }); + db.prepare("UPDATE openmythos_eval_runs SET metadata = '{broken' WHERE id = 'bad-1'").run(); + + const board = service.getLeaderboard(); + expect(board.map((s) => s.agentId)).toEqual(['agent-good', 'agent-bad']); + expect(board[1].categoryScores).toEqual({}); + }); + }); +}); diff --git a/packages/server/src/routes/openmythos.ts b/packages/server/src/routes/openmythos.ts index 4958e7c7..f56b9701 100644 --- a/packages/server/src/routes/openmythos.ts +++ b/packages/server/src/routes/openmythos.ts @@ -31,6 +31,25 @@ export function createOpenMythosRoutes(db: Database, auth?: AuthMiddleware): Rou } }); + // GET /api/openmythos/runs — recent eval runs across all agents + router.get('/runs', requirePermission('read:evidence'), (req, res, next) => { + try { + const limit = req.query.limit ? Math.min(Math.max(Number(req.query.limit) || 20, 1), 100) : 20; + res.json({ runs: evalService.listRuns(limit) }); + } catch (error) { + next(error); + } + }); + + // GET /api/openmythos/leaderboard — latest score per agent, best first + router.get('/leaderboard', requirePermission('read:evidence'), (_req, res, next) => { + try { + res.json({ leaderboard: evalService.getLeaderboard() }); + } catch (error) { + next(error); + } + }); + // GET /api/openmythos/score/:agentId — get latest scores router.get('/score/:agentId', requirePermission('read:evidence'), (req, res, next) => { try { diff --git a/packages/server/src/services/openmythos-eval-service.ts b/packages/server/src/services/openmythos-eval-service.ts index 29104ada..2c4e5a96 100644 --- a/packages/server/src/services/openmythos-eval-service.ts +++ b/packages/server/src/services/openmythos-eval-service.ts @@ -506,7 +506,8 @@ Respond with JSON: {"score": , "rationale": ""}`; else if (diff < -0.1) trend = 'declining'; } - const metadata = JSON.parse(run.metadata || '{}') as { category_scores?: Record }; + let metadata: { category_scores?: Record } = {}; + try { metadata = JSON.parse(run.metadata || '{}'); } catch { /* malformed metadata must not sink the score */ } return { agentId, overallScore: run.overall_score, @@ -576,6 +577,63 @@ Respond with JSON: {"score": , "rationale": ""}`; } + /** + * List recent eval runs across all agents (newest first). + */ + listRuns(limit = 20): Array<{ + id: string; + agentId: string; + status: string; + totalCases: number; + completedCases: number; + overallScore: number; + subjectModel: string | null; + oracleCases: number | null; + judgeCases: number | null; + startedAt: string | null; + finishedAt: string | null; + }> { + const rows = this.db.prepare(` + SELECT id, agent_id, status, total_cases, completed_cases, overall_score, started_at, finished_at, metadata + FROM openmythos_eval_runs + ORDER BY started_at DESC + LIMIT ? + `).all(limit) as Array>; + + return rows.map((r) => { + let metadata: { subject_model?: string; oracle_cases?: number; judge_cases?: number } = {}; + try { metadata = JSON.parse((r.metadata as string) || '{}'); } catch { /* keep empty */ } + return { + id: r.id as string, + agentId: r.agent_id as string, + status: r.status as string, + totalCases: r.total_cases as number, + completedCases: r.completed_cases as number, + overallScore: r.overall_score as number, + subjectModel: metadata.subject_model ?? null, + oracleCases: metadata.oracle_cases ?? null, + judgeCases: metadata.judge_cases ?? null, + startedAt: (r.started_at as string) ?? null, + finishedAt: (r.finished_at as string) ?? null, + }; + }); + } + + /** + * Governance leaderboard: latest score per agent, best first. + */ + getLeaderboard(): AgentScore[] { + const agents = this.db.prepare(` + SELECT DISTINCT agent_id FROM openmythos_eval_runs WHERE status = 'completed' + `).all() as Array<{ agent_id: string }>; + + // ponytail: one getAgentScore query per agent — fine at fleet sizes, batch when agents > ~1k + return agents + .map((a) => this.getAgentScore(a.agent_id)) + .filter((s): s is AgentScore => s !== null) + .sort((a, b) => b.overallScore - a.overallScore); + } + /** * Filter cases based on discrimination power. * Excludes cases where all models got the same score (spread=0) over last N runs.