Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions packages/dashboard/src/App.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ const WorkstationUrlsPage = lazy(() => import('./pages/WorkstationUrlsPage').the
const EconomyPage = lazy(() => import('./pages/EconomyPage').then((module) => ({ default: module.EconomyPage })));
const PipelineBuilderPage = lazy(() => import('./pages/PipelineBuilderPage').then((module) => ({ default: module.PipelineBuilderPage })));
const FederationPage = lazy(() => import('./pages/FederationPage').then((module) => ({ default: module.FederationPage })));
const GovernanceScorecardPage = lazy(() => import('./pages/GovernanceScorecardPage').then((module) => ({ default: module.GovernanceScorecardPage })));

function DataLoader({ children }: { children: React.ReactNode }) {
const { setTasks, setAgents } = useStore();
Expand Down Expand Up @@ -92,6 +93,7 @@ export function App() {
<Route path="swarm" element={<SwarmOverviewPage />} />
<Route path="approvals" element={<ApprovalQueuePage />} />
<Route path="policies" element={<PolicyCenterPage />} />
<Route path="governance" element={<GovernanceScorecardPage />} />
<Route path="mcp-permissions" element={<MCPPermissionsPage />} />
<Route path="observability" element={<ObservabilityPage />} />
<Route path="tasks/:taskId/review" element={<ReviewPage />} />
Expand Down
8 changes: 7 additions & 1 deletion packages/dashboard/src/components/Layout.tsx
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import { useEffect, useState, type ReactNode } from 'react';
import { Outlet, Link, useLocation } from 'react-router-dom';
import { Activity, ListTodo, Users, Shield, CheckSquare, PlugZap, BarChart3, ScrollText, FolderGit, LogOut, DollarSign, Network, Cpu, Workflow, BrainCircuit, Gauge, BookUser, Brain, Menu, X } from 'lucide-react';
import { Activity, ListTodo, Users, Shield, ShieldCheck, CheckSquare, PlugZap, BarChart3, ScrollText, FolderGit, LogOut, DollarSign, Network, Cpu, Workflow, BrainCircuit, Gauge, BookUser, Brain, Menu, X } from 'lucide-react';
import { useAuthStore } from '../lib/auth-store';

export function Layout() {
Expand Down Expand Up @@ -86,6 +86,12 @@ export function Layout() {
label="Policies"
active={isActive('/policies')}
/>
<NavLink
to="/governance"
icon={<ShieldCheck className="w-5 h-5" />}
label="Governance"
active={isActive('/governance')}
/>
<NavLink
to="/mcp-permissions"
icon={<PlugZap className="w-5 h-5" />}
Expand Down
37 changes: 37 additions & 0 deletions packages/dashboard/src/lib/api.ts
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,29 @@ export const API_BASE = import.meta.env.PROD ? '/api' : import.meta.env.VITE_API

const AUTH_SESSION_KEY = 'djimitflo_auth_session';

export type AgentGovernanceScore = {
agentId: string;
overallScore: number;
categoryScores: Record<string, number>;
totalCases: number;
lastEvalAt: string;
trend: 'improving' | 'stable' | 'declining';
};

export type OpenMythosRun = {
id: string;
agentId: string;
status: string;
totalCases: number;
completedCases: number;
overallScore: number;
subjectModel: string | null;
oracleCases: number | null;
judgeCases: number | null;
startedAt: string | null;
finishedAt: string | null;
};

type UsageQuota = {
provider: string;
tier: string;
Expand Down Expand Up @@ -974,6 +997,20 @@ class ApiClient {
return this.request(`/evidence/review/${taskId}`);
}

// OpenMythos governance
async getOpenMythosLeaderboard(): Promise<{ leaderboard: AgentGovernanceScore[] }> {
return this.request('/openmythos/leaderboard');
}

async getOpenMythosRuns(limit?: number): Promise<{ runs: OpenMythosRun[] }> {
const query = limit ? `?limit=${limit}` : '';
return this.request(`/openmythos/runs${query}`);
}

async getOpenMythosTrend(agentId: string): Promise<{ agentId: string; trend: Array<{ date: string; score: number }> }> {
return this.request(`/openmythos/trend/${encodeURIComponent(agentId)}`);
}

// Observability
async getObservabilityMetrics(): Promise<ObservabilityMetrics> {
return this.request('/observability/metrics');
Expand Down
58 changes: 58 additions & 0 deletions packages/dashboard/src/pages/GovernanceScorecardPage.test.tsx
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
import { describe, it, expect, vi } from 'vitest';
import { render, screen } from '@testing-library/react';
import { GovernanceScorecardPage } from './GovernanceScorecardPage';

vi.mock('../lib/api', () => ({
api: {
getOpenMythosLeaderboard: vi.fn().mockResolvedValue({
leaderboard: [
{
agentId: 'agent-a',
overallScore: 3.5,
categoryScores: { injection: 4.0, hallucination: 2.1 },
totalCases: 78,
lastEvalAt: '2026-07-15T10:05:00.000Z',
trend: 'improving',
},
],
}),
getOpenMythosRuns: vi.fn().mockResolvedValue({
runs: [
{
id: 'run-12345678',
agentId: 'agent-a',
status: 'completed',
totalCases: 78,
completedCases: 78,
overallScore: 3.5,
subjectModel: 'llama3.1:8b',
oracleCases: 78,
judgeCases: 0,
startedAt: '2026-07-15T10:00:00.000Z',
finishedAt: '2026-07-15T10:05:00.000Z',
},
],
}),
},
}));

describe('GovernanceScorecardPage', () => {
it('renders the leaderboard with scores, trend, and category chips', async () => {
render(<GovernanceScorecardPage />);

expect(screen.getByText('Governance Scorecard')).toBeTruthy();
expect((await screen.findAllByText('agent-a')).length).toBeGreaterThan(0);
expect(screen.getAllByText('3.50').length).toBeGreaterThan(0);
expect(screen.getByLabelText('improving')).toBeTruthy();
expect(screen.getByText('injection 4.0')).toBeTruthy();
expect(screen.getByText('hallucination 2.1')).toBeTruthy();
});

it('renders the recent runs with model and oracle provenance', async () => {
render(<GovernanceScorecardPage />);

expect(await screen.findByText('llama3.1:8b')).toBeTruthy();
expect(screen.getByText('completed')).toBeTruthy();
expect(screen.getByText(/78 oracle/)).toBeTruthy();
});
});
141 changes: 141 additions & 0 deletions packages/dashboard/src/pages/GovernanceScorecardPage.tsx
Original file line number Diff line number Diff line change
@@ -0,0 +1,141 @@
import { useCallback, useEffect, useState } from 'react';
import { ShieldCheck, TrendingUp, TrendingDown, Minus, RefreshCw, FlaskConical } from 'lucide-react';
import { api, type AgentGovernanceScore, type OpenMythosRun } from '../lib/api';

function scoreColor(score: number): string {
if (score >= 4) return 'text-status-completed';
if (score >= 3) return 'text-risk-medium';
return 'text-status-error';
}

function TrendIcon({ trend }: { trend: AgentGovernanceScore['trend'] }) {
if (trend === 'improving') return <TrendingUp className="w-4 h-4 text-status-completed" aria-label="improving" />;
if (trend === 'declining') return <TrendingDown className="w-4 h-4 text-status-error" aria-label="declining" />;
return <Minus className="w-4 h-4 text-foreground-tertiary" aria-label="stable" />;
}

export function GovernanceScorecardPage() {
const [leaderboard, setLeaderboard] = useState<AgentGovernanceScore[]>([]);
const [runs, setRuns] = useState<OpenMythosRun[]>([]);
const [loading, setLoading] = useState(true);
const [error, setError] = useState<string | null>(null);

const load = useCallback(async () => {
setLoading(true);
setError(null);
try {
const [board, recent] = await Promise.all([
api.getOpenMythosLeaderboard(),
api.getOpenMythosRuns(20),
]);
setLeaderboard(board.leaderboard);
setRuns(recent.runs);
} catch (err) {
setError(err instanceof Error ? err.message : 'Failed to load governance data');
} finally {
setLoading(false);
}
}, []);

useEffect(() => {
void load();
}, [load]);

return (
<div className="p-8 space-y-6">
<div className="flex items-start justify-between">
<div>
<h1 className="text-3xl font-bold text-foreground">Governance Scorecard</h1>
<p className="text-foreground-secondary mt-2">
OpenMythos Governance Benchmark results per agent — latest score, category breakdown, and trend (0–5 scale).
</p>
</div>
<button
onClick={() => void load()}
disabled={loading}
className="flex items-center gap-2 bg-background-secondary border border-border rounded-lg px-3 py-1.5 text-sm text-foreground hover:bg-background-elevated disabled:opacity-50"
>
<RefreshCw className={`w-4 h-4 ${loading ? 'animate-spin' : ''}`} />
Refresh
</button>
</div>

{error && (
<div className="bg-status-error/10 border border-status-error/30 rounded-lg p-4 text-sm text-status-error">{error}</div>
)}

{loading ? (
<div className="bg-background-secondary border border-border rounded-lg p-8 text-foreground-secondary">Loading governance data...</div>
) : leaderboard.length === 0 ? (
<div className="bg-background-secondary border border-border rounded-lg p-12 text-center">
<ShieldCheck className="w-12 h-12 text-foreground-muted mx-auto mb-4" />
<p className="text-foreground-secondary">No governance evaluations recorded yet.</p>
<p className="text-xs text-foreground-tertiary mt-2 font-mono">POST /api/openmythos/eval/:agentId to run the benchmark.</p>
</div>
) : (
<div className="bg-background-secondary border border-border rounded-lg p-6">
<h2 className="text-lg font-semibold text-foreground mb-4">Leaderboard</h2>
<div className="space-y-4">
{leaderboard.map((agent, rank) => (
<div key={agent.agentId} className="border-l-2 border-border pl-4 py-2">
<div className="flex items-center gap-3 flex-wrap">
<span className="text-xs font-mono text-foreground-tertiary">#{rank + 1}</span>
<span className="text-sm font-medium text-foreground">{agent.agentId}</span>
<span className={`text-lg font-bold ${scoreColor(agent.overallScore)}`}>{agent.overallScore.toFixed(2)}</span>
<TrendIcon trend={agent.trend} />
<span className="text-xs text-foreground-tertiary">
{agent.totalCases} cases · {agent.lastEvalAt ? new Date(agent.lastEvalAt).toLocaleString() : 'n/a'}
</span>
</div>
{Object.keys(agent.categoryScores).length > 0 && (
<div className="flex flex-wrap gap-1.5 mt-2">
{Object.entries(agent.categoryScores)
.sort(([, a], [, b]) => a - b)
.map(([category, score]) => (
<span
key={category}
className={`text-xs px-1.5 py-0.5 rounded bg-background-elevated ${scoreColor(score)}`}
title={`${category}: ${score.toFixed(2)}/5`}
>
{category} {score.toFixed(1)}
</span>
))}
</div>
)}
</div>
))}
</div>
</div>
)}

{!loading && runs.length > 0 && (
<div className="bg-background-secondary border border-border rounded-lg p-6">
<h2 className="text-lg font-semibold text-foreground mb-4">Recent Eval Runs</h2>
<div className="space-y-2">
{runs.map((run) => (
<div key={run.id} className="flex items-center gap-3 flex-wrap border-l-2 border-border pl-3 py-1">
<FlaskConical className="w-3.5 h-3.5 text-foreground-tertiary" />
<span className="text-xs font-mono text-foreground-tertiary">{(run.id ?? '').slice(0, 8)}</span>
<span className="text-sm text-foreground">{run.agentId}</span>
{run.subjectModel && (
<span className="text-xs px-1.5 py-0.5 rounded bg-background-elevated text-foreground-secondary font-mono">{run.subjectModel}</span>
)}
<span className={`text-xs px-1.5 py-0.5 rounded ${
run.status === 'completed' ? 'bg-status-completed/10 text-status-completed' :
run.status === 'failed' ? 'bg-status-error/10 text-status-error' :
'bg-background-elevated text-foreground-secondary'
}`}>{run.status}</span>
<span className={`text-sm font-semibold ${scoreColor(run.overallScore)}`}>{run.overallScore.toFixed(2)}</span>
<span className="text-xs text-foreground-tertiary">
{run.completedCases}/{run.totalCases} cases
{run.oracleCases !== null && ` · ${run.oracleCases} oracle`}
{run.finishedAt && ` · ${new Date(run.finishedAt).toLocaleString()}`}
</span>
</div>
))}
</div>
</div>
)}
</div>
);
}
54 changes: 54 additions & 0 deletions packages/mcp-server/src/__tests__/mcp-server.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import { registerLoopTools } from '../tools/loops.js';
import { registerGoalTools } from '../tools/goals.js';
import { registerAgentTools } from '../tools/agents.js';
import { registerMissionControlTools } from '../tools/mission-control.js';
import { registerOpenMythosTools } from '../tools/openmythos.js';

function createTestDb(): DbHandle {
const db = new Database(':memory:');
Expand Down Expand Up @@ -43,6 +44,14 @@ function createTestDb(): DbHandle {
metadata_json TEXT DEFAULT '{}', level TEXT DEFAULT 'info',
created_at TEXT NOT NULL
);
CREATE TABLE openmythos_eval_runs (
id TEXT PRIMARY KEY, agent_id TEXT NOT NULL,
started_at TEXT, finished_at TEXT,
total_cases INTEGER DEFAULT 0, completed_cases INTEGER DEFAULT 0,
overall_score REAL DEFAULT 0, status TEXT DEFAULT 'pending',
categories_json TEXT DEFAULT '[]', metadata TEXT DEFAULT '{}',
created_at TEXT NOT NULL DEFAULT (datetime('now'))
);
`);
return { db, close: () => db.close() };
}
Expand All @@ -53,6 +62,7 @@ function createTestServer(dbHandle: DbHandle): McpServer {
registerGoalTools(server, dbHandle);
registerAgentTools(server, dbHandle);
registerMissionControlTools(server, dbHandle);
registerOpenMythosTools(server, dbHandle);
return server;
}

Expand Down Expand Up @@ -110,4 +120,48 @@ describe('MCP Server Tools', () => {
expect(parsed.summary.activeLoans).toBe(0);
expect(parsed.summary.pendingGoals).toBe(0);
});

it('registers the openmythos tools', () => {
const toolNames = Object.keys((server as any)._registeredTools || {});
expect(toolNames).toContain('djimitflo_openmythos_leaderboard');
expect(toolNames).toContain('djimitflo_openmythos_score');
});

it('openmythos_leaderboard ranks latest completed run per agent', async () => {
const insert = dbHandle.db.prepare(`
INSERT INTO openmythos_eval_runs (id, agent_id, status, completed_cases, overall_score, finished_at, metadata, created_at)
VALUES (?, ?, ?, ?, ?, ?, ?, datetime('now'))
`);
insert.run('r1', 'agent-a', 'completed', 78, 2.0, '2026-07-14T10:00:00Z', '{"category_scores":{"injection":3.0},"subject_model":"llama3.1:8b"}');
insert.run('r2', 'agent-a', 'completed', 78, 3.5, '2026-07-15T10:00:00Z', '{"category_scores":{"injection":4.0},"subject_model":"llama3.1:8b"}');
insert.run('r3', 'agent-b', 'completed', 78, 2.5, '2026-07-15T10:00:00Z', '{}');
insert.run('r4', 'agent-c', 'failed', 0, 0, null, '{}');

const tool = (server as any)._registeredTools['djimitflo_openmythos_leaderboard'];
const result = await tool.handler({});
const parsed = JSON.parse(result.content[0].text);

expect(parsed.map((row: { agentId: string }) => row.agentId)).toEqual(['agent-a', 'agent-b']);
expect(parsed[0].overallScore).toBe(3.5);
expect(parsed[0].categoryScores).toEqual({ injection: 4.0 });
expect(parsed[0].subjectModel).toBe('llama3.1:8b');
});

it('openmythos_score returns latest score with trend, and errors on unknown agent', async () => {
const insert = dbHandle.db.prepare(`
INSERT INTO openmythos_eval_runs (id, agent_id, status, completed_cases, overall_score, finished_at, metadata, created_at)
VALUES (?, ?, 'completed', 78, ?, ?, '{}', datetime('now'))
`);
insert.run('r1', 'agent-a', 2.0, '2026-07-14T10:00:00Z');
insert.run('r2', 'agent-a', 3.0, '2026-07-15T10:00:00Z');

const tool = (server as any)._registeredTools['djimitflo_openmythos_score'];
const result = await tool.handler({ agentId: 'agent-a' });
const parsed = JSON.parse(result.content[0].text);
expect(parsed.overallScore).toBe(3.0);
expect(parsed.trend.map((t: { score: number }) => t.score)).toEqual([2.0, 3.0]);

const missing = await tool.handler({ agentId: 'nope' });
expect(missing.isError).toBe(true);
});
});
2 changes: 2 additions & 0 deletions packages/mcp-server/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ import { registerAgentTools } from './tools/agents.js';
import { registerMissionControlTools } from './tools/mission-control.js';
import { registerOrchestrationTools } from './tools/orchestration.js';
import { registerOkfTools } from './tools/okf.js';
import { registerOpenMythosTools } from './tools/openmythos.js';

interface ServerOptions {
transport: 'stdio' | 'http';
Expand Down Expand Up @@ -52,6 +53,7 @@ async function main() {
registerMissionControlTools(server, db);
registerOrchestrationTools(server, db);
registerOkfTools(server);
registerOpenMythosTools(server, db);

if (opts.transport === 'stdio') {
const transport = new StdioServerTransport();
Expand Down
Loading
Loading