feat: Prometheus metrics endpoint

packages/server/package.json

@@ -26,6 +26,7 @@
     "better-sqlite3": "^12.11.1",
     "cors": "^2.8.5",
     "express": "^5.2.1",
+    "express-rate-limit": "^8.5.2",
     "jsonwebtoken": "^9.0.3",
     "tar-stream": "^2.2.0",
     "ws": "^8.20.1",

packages/server/src/__tests__/metrics.test.ts

@@ -0,0 +1,100 @@
+import { afterEach, beforeEach, describe, expect, it } from 'vitest';
+import type { Database } from 'better-sqlite3';
+import type { Request, Response } from 'express';
+import { createTestDb } from './helpers/test-db';
+import { createMetricsHandler, metricsRateLimiter } from '../routes/metrics';
+
+function call(handler: ReturnType<typeof createMetricsHandler>, authorization?: string) {
+  const req = { headers: authorization ? { authorization } : {} } as Request;
+  const out = { status: 0, contentType: '', body: '' };
+  const res = {
+    status(code: number) { out.status = code; return this; },
+    end() { return this; },
+    set(_key: string, value: string) { out.contentType = value; return this; },
+    send(body: string) { out.status = out.status || 200; out.body = body; return this; },
+  } as unknown as Response;
+  handler(req, res);
+  return out;
+}
+
+describe('GET /metrics', () => {
+  let db: Database;
+  const previousToken = process.env.METRICS_TOKEN;
+
+  beforeEach(() => {
+    db = createTestDb();
+    process.env.METRICS_TOKEN = 'scrape-secret';
+  });
+
+  afterEach(() => {
+    db.close();
+    if (previousToken === undefined) delete process.env.METRICS_TOKEN;
+    else process.env.METRICS_TOKEN = previousToken;
+  });
+
+  it('is 404 (invisible) when METRICS_TOKEN is unset', () => {
+    delete process.env.METRICS_TOKEN;
+    const out = call(createMetricsHandler(db), 'Bearer anything');
+    expect(out.status).toBe(404);
+  });
+
+  it('rejects missing or wrong bearer tokens', () => {
+    const handler = createMetricsHandler(db);
+    expect(call(handler).status).toBe(401);
+    expect(call(handler, 'Bearer wrong').status).toBe(401);
+  });
+
+  it('emits status gauges, openmythos scores, and process metrics in prometheus text format', () => {
+    db.prepare(`
+      INSERT INTO tasks (id, title, description, status, priority, risk_level, execution_mode)
+      VALUES ('t1', 'a', 'b', 'pending', 'medium', 'low', 'local'), ('t2', 'c', 'd', 'pending', 'medium', 'low', 'local')
+    `).run();
+    db.prepare(`
+      INSERT INTO openmythos_eval_runs (id, agent_id, status, total_cases, completed_cases, overall_score, started_at, finished_at, metadata)
+      VALUES ('r1', 'nightly:llama3.1:8b', 'completed', 78, 78, 2.692, '2026-07-15T10:00:00Z', '2026-07-15T10:02:00Z', '{}')
+    `).run();
+
+    const out = call(createMetricsHandler(db, () => 3), 'Bearer scrape-secret');
+
+    expect(out.status).toBe(200);
+    expect(out.contentType).toContain('text/plain');
+    expect(out.body).toContain('djimitflo_tasks{status="pending"} 2');
+    expect(out.body).toContain('djimitflo_openmythos_score{agent="nightly:llama3.1:8b"} 2.692');
+    expect(out.body).toContain('djimitflo_ws_clients 3');
+    expect(out.body).toMatch(/djimitflo_process_uptime_seconds \d+/);
+    expect(out.body).toMatch(/djimitflo_process_memory_rss_bytes \d+/);
+  });
+
+  it('rate-limits scrapes per IP', async () => {
+    const invoke = async () => {
+      const req = { ip: '203.0.113.7', headers: {}, method: 'GET', path: '/metrics', app: { get: () => false } } as unknown as Request;
+      const out = { status: 0, nextCalled: false };
+      const res = {
+        status(code: number) { out.status = code; return this; },
+        send() { return this; },
+        end() { return this; },
+        setHeader() { return this; },
+        getHeader() { return undefined; },
+      } as unknown as Response;
+      await metricsRateLimiter(req, res, () => { out.nextCalled = true; });
+      return out;
+    };
+
+    for (let i = 0; i < 300; i++) {
+      expect((await invoke()).nextCalled).toBe(true);
+    }
+    const blocked = await invoke();
+    expect(blocked.nextCalled).toBe(false);
+    expect(blocked.status).toBe(429);
+  });
+
+  it('escapes label values', () => {
+    db.prepare(`
+      INSERT INTO openmythos_eval_runs (id, agent_id, status, total_cases, completed_cases, overall_score, started_at, finished_at, metadata)
+      VALUES ('r1', 'agent"with\\quotes', 'completed', 1, 1, 3.0, '2026-07-15T10:00:00Z', '2026-07-15T10:02:00Z', '{}')
+    `).run();
+
+    const out = call(createMetricsHandler(db), 'Bearer scrape-secret');
+    expect(out.body).toContain('djimitflo_openmythos_score{agent="agent\\"with\\\\quotes"} 3');
+  });
+});

packages/server/src/index.ts

@@ -16,6 +16,7 @@ import { requestLogger } from './middleware/request-logger';
 import { createAuthMiddleware } from './middleware/auth';
 import { AuthService } from './services/auth-service';
 import { createRoutes } from './routes';
+import { createMetricsHandler, metricsRateLimiter } from './routes/metrics';
 import { WebSocketService } from './services/websocket-service';
 import { ExecutionEngine } from './execution/execution-engine';
 import { MemorySyncService } from './services/memory-sync-service';
@@ -197,6 +198,9 @@ async function main() {
   const wsService = new WebSocketService(wss, authService, db);
   console.log('🔌 WebSocket server initialized (authenticated)');
 
+  // Prometheus exposition — default-off, armed by METRICS_TOKEN (see routes/metrics.ts)
+  app.get('/metrics', metricsRateLimiter, createMetricsHandler(db, () => wsService.getClientCount()));
+
   // Create execution engine
   const executionEngine = new ExecutionEngine(db, wsService);
   console.log('⚙️  Execution engine initialized');

packages/server/src/routes/metrics.ts

@@ -0,0 +1,101 @@
+/**
+ * Prometheus exposition endpoint (text format 0.0.4), mounted at GET /metrics.
+ *
+ * Default-off: responds 404 unless METRICS_TOKEN is set, and requires
+ * `Authorization: Bearer <METRICS_TOKEN>` (JWT auth doesn't fit scrapers).
+ * All gauges are computed per scrape from SQLite — no counters kept in
+ * process, so restarts and multi-node deploys need no extra state.
+ */
+
+import { timingSafeEqual } from 'crypto';
+import type { Request, Response } from 'express';
+import type { Database } from 'better-sqlite3';
+import rateLimit from 'express-rate-limit';
+
+/**
+ * 300 scrapes / 15 min per IP — generous for any Prometheus interval, tight
+ * enough to blunt token brute-forcing. express-rate-limit (rather than the
+ * in-repo RateLimiter) so CodeQL's js/missing-rate-limiting recognizes it.
+ */
+export const metricsRateLimiter = rateLimit({
+  windowMs: 15 * 60 * 1000,
+  limit: 300,
+  standardHeaders: false,
+  legacyHeaders: false,
+});
+
+function escapeLabel(value: string): string {
+  return value.replace(/\\/g, '\\\\').replace(/"/g, '\\"').replace(/\n/g, '\\n');
+}
+
+function authorized(req: Request): boolean {
+  const token = process.env.METRICS_TOKEN;
+  if (!token) return false;
+  const expected = Buffer.from(`Bearer ${token}`);
+  const provided = Buffer.from(req.headers.authorization || '');
+  return provided.length === expected.length && timingSafeEqual(provided, expected);
+}
+
+export function createMetricsHandler(db: Database, getWsClients?: () => number) {
+  const statusGauge = (name: string, help: string, table: string): string[] => {
+    const lines = [`# HELP ${name} ${help}`, `# TYPE ${name} gauge`];
+    try {
+      const rows = db.prepare(`SELECT status, COUNT(*) AS n FROM ${table} GROUP BY status`).all() as Array<{ status: string; n: number }>;
+      for (const row of rows) lines.push(`${name}{status="${escapeLabel(String(row.status))}"} ${row.n}`);
+    } catch { /* table absent on this deploy — emit the header only */ }
+    return lines;
+  };
+
+  return (req: Request, res: Response): void => {
+    if (!process.env.METRICS_TOKEN) {
+      res.status(404).end();
+      return;
+    }
+    if (!authorized(req)) {
+      res.status(401).end();
+      return;
+    }
+
+    const lines: string[] = [
+      ...statusGauge('djimitflo_tasks', 'Tasks by status', 'tasks'),
+      ...statusGauge('djimitflo_agents', 'Agents by status', 'agents'),
+      ...statusGauge('djimitflo_loop_runs', 'Loop runs by status', 'loop_runs'),
+      ...statusGauge('djimitflo_worker_leases', 'Worker leases by status', 'worker_leases'),
+      ...statusGauge('djimitflo_approvals', 'Approvals by status', 'approvals'),
+      ...statusGauge('djimitflo_work_items', 'Work items by status', 'work_items'),
+    ];
+
+    lines.push(
+      '# HELP djimitflo_openmythos_score Latest OpenMythos governance score per agent (0-5)',
+      '# TYPE djimitflo_openmythos_score gauge',
+    );
+    try {
+      const rows = db.prepare(`
+        SELECT agent_id, overall_score, MAX(finished_at) AS finished_at
+        FROM openmythos_eval_runs WHERE status = 'completed' GROUP BY agent_id
+      `).all() as Array<{ agent_id: string; overall_score: number }>;
+      for (const row of rows) {
+        lines.push(`djimitflo_openmythos_score{agent="${escapeLabel(row.agent_id)}"} ${row.overall_score}`);
+      }
+    } catch { /* eval table absent */ }
+
+    if (getWsClients) {
+      lines.push(
+        '# HELP djimitflo_ws_clients Connected WebSocket clients',
+        '# TYPE djimitflo_ws_clients gauge',
+        `djimitflo_ws_clients ${getWsClients()}`,
+      );
+    }
+
+    lines.push(
+      '# HELP djimitflo_process_uptime_seconds Server process uptime',
+      '# TYPE djimitflo_process_uptime_seconds gauge',
+      `djimitflo_process_uptime_seconds ${Math.round(process.uptime())}`,
+      '# HELP djimitflo_process_memory_rss_bytes Resident set size',
+      '# TYPE djimitflo_process_memory_rss_bytes gauge',
+      `djimitflo_process_memory_rss_bytes ${process.memoryUsage().rss}`,
+    );
+
+    res.set('Content-Type', 'text/plain; version=0.0.4; charset=utf-8').send(lines.join('\n') + '\n');
+  };
+}