Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
77 changes: 71 additions & 6 deletions packages/server/src/__tests__/execution-engine.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -95,14 +95,23 @@ describe('ExecutionEngine', () => {
CREATE TABLE IF NOT EXISTS audit_events (
id TEXT PRIMARY KEY,
event_type TEXT NOT NULL,
timestamp TEXT NOT NULL DEFAULT (datetime('now')),
user_id TEXT,
agent_id TEXT,
task_id TEXT,
execution_event_id TEXT,
action TEXT NOT NULL,
resource_type TEXT,
resource_type TEXT NOT NULL,
resource_id TEXT,
task_id TEXT,
actor TEXT NOT NULL DEFAULT 'system',
risk_level TEXT,
metadata TEXT NOT NULL DEFAULT '{}',
created_at TEXT NOT NULL DEFAULT (datetime('now'))
actor TEXT,
risk_level TEXT NOT NULL DEFAULT 'low' CHECK(risk_level IN ('low', 'medium', 'high', 'critical')),
before TEXT,
after TEXT,
ip_address TEXT,
user_agent TEXT,
metadata TEXT,
created_at TEXT NOT NULL DEFAULT (datetime('now')),
updated_at TEXT NOT NULL DEFAULT (datetime('now'))
);
CREATE TABLE IF NOT EXISTS evidence (
id TEXT PRIMARY KEY,
Expand Down Expand Up @@ -202,4 +211,60 @@ describe('ExecutionEngine', () => {
const row = db.prepare('SELECT status FROM tasks WHERE id = ?').get(task.id) as any;
expect(['running', 'completed']).toContain(row.status);
});

describe('governance gate integration', () => {
const GATE_KEYS = ['GOVERNANCE_GATE_ENABLED', 'GOVERNANCE_GATE_FLOOR', 'GOVERNANCE_GATE_MODEL_MAP'];
const previousEnv: Record<string, string | undefined> = {};

beforeEach(() => {
for (const key of GATE_KEYS) { previousEnv[key] = process.env[key]; delete process.env[key]; }
});

afterEach(() => {
for (const key of GATE_KEYS) {
if (previousEnv[key] === undefined) delete process.env[key];
else process.env[key] = previousEnv[key];
}
});

function insertLowScoreRun(agentId: string, score: number) {
db.prepare(`
INSERT INTO openmythos_eval_runs (id, agent_id, status, total_cases, completed_cases, overall_score, started_at, finished_at, metadata)
VALUES (?, ?, 'completed', 78, 78, ?, '2026-07-15T10:00:00Z', '2026-07-15T10:05:00Z', '{}')
`).run(`run-${Math.random().toString(36).slice(2)}`, agentId, score);
}

it('tightens allow to awaiting_approval when the benchmarked model is below the floor', async () => {
process.env.GOVERNANCE_GATE_ENABLED = 'true';
process.env.GOVERNANCE_GATE_MODEL_MAP = 'mock=weak-model';
insertLowScoreRun('nightly:weak-model', 1.9);

const task = createTask();
db.prepare('INSERT INTO tasks (id, title, description, status, priority, risk_level, execution_mode) VALUES (?, ?, ?, ?, ?, ?, ?)').run(
task.id, task.title, task.description, 'pending', 'medium', 'low', 'local',
);

const result = await engine.executeTask(task.id, 'mock');

expect(result.status).toBe('awaiting_approval');
expect(result.reason).toContain('Governance gate');

const evidence = db.prepare("SELECT * FROM execution_evidence WHERE task_id = ? AND source = 'governance-gate'").all(task.id);
expect(evidence.length).toBe(1);
});

it('does not fire when the benchmarked model clears the floor', async () => {
process.env.GOVERNANCE_GATE_ENABLED = 'true';
process.env.GOVERNANCE_GATE_MODEL_MAP = 'mock=strong-model';
insertLowScoreRun('nightly:strong-model', 4.2);

const task = createTask();
db.prepare('INSERT INTO tasks (id, title, description, status, priority, risk_level, execution_mode) VALUES (?, ?, ?, ?, ?, ?, ?)').run(
task.id, task.title, task.description, 'pending', 'medium', 'low', 'local',
);

const result = await engine.executeTask(task.id, 'mock');
expect(result.status).toBe('started');
});
});
});
110 changes: 110 additions & 0 deletions packages/server/src/__tests__/governance-gate.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,110 @@
import { afterEach, beforeEach, describe, expect, it } from 'vitest';
import type { Database } from 'better-sqlite3';
import { createTestDb } from './helpers/test-db';
import { GovernanceGateService } from '../services/governance-gate-service';

const GATE_ENV_KEYS = ['GOVERNANCE_GATE_ENABLED', 'GOVERNANCE_GATE_FLOOR', 'GOVERNANCE_GATE_MODEL_MAP'];

function insertRun(db: Database, run: {
id: string;
agentId: string;
score: number;
finishedAt: string;
subjectModel?: string;
}) {
db.prepare(`
INSERT INTO openmythos_eval_runs (id, agent_id, status, total_cases, completed_cases, overall_score, started_at, finished_at, metadata)
VALUES (?, ?, 'completed', 78, 78, ?, ?, ?, ?)
`).run(run.id, run.agentId, run.score, run.finishedAt, run.finishedAt,
JSON.stringify(run.subjectModel ? { subject_model: run.subjectModel } : {}));
}

describe('GovernanceGateService', () => {
let db: Database;
let gate: GovernanceGateService;
const previousEnv = { ...process.env };

beforeEach(() => {
for (const key of GATE_ENV_KEYS) delete process.env[key];
process.env.GOVERNANCE_GATE_ENABLED = 'true';
db = createTestDb();
gate = new GovernanceGateService(db);
});

afterEach(() => {
db.close();
for (const key of Object.keys(process.env)) {
if (!(key in previousEnv)) delete process.env[key];
}
Object.assign(process.env, previousEnv);
});

it('allows everything when disabled', () => {
delete process.env.GOVERNANCE_GATE_ENABLED;
insertRun(db, { id: 'r1', agentId: 'agent-a', score: 1.0, finishedAt: '2026-07-15T10:00:00Z' });

const verdict = gate.assess({ agent_id: 'agent-a' }, 'mock');
expect(verdict.action).toBe('allow');
expect(verdict.reason).toContain('disabled');
});

it('allows when there is no governance evidence', () => {
const verdict = gate.assess({ agent_id: 'agent-unknown' }, 'mock');
expect(verdict.action).toBe('allow');
expect(verdict.reason).toContain('No governance evidence');
});

it('allows when the latest score clears the floor', () => {
insertRun(db, { id: 'r1', agentId: 'agent-a', score: 3.4, finishedAt: '2026-07-15T10:00:00Z' });

const verdict = gate.assess({ agent_id: 'agent-a' }, 'mock');
expect(verdict.action).toBe('allow');
expect(verdict.score).toBe(3.4);
});

it('requires approval when the latest score is below the floor', () => {
insertRun(db, { id: 'r1', agentId: 'agent-a', score: 2.1, finishedAt: '2026-07-15T10:00:00Z' });

const verdict = gate.assess({ agent_id: 'agent-a' }, 'mock');
expect(verdict.action).toBe('require_approval');
expect(verdict.reason).toContain('2.10');
expect(verdict.reason).toContain('approval required');
expect(verdict.flagRetirement).toBe(false);
});

it('respects a configured floor', () => {
process.env.GOVERNANCE_GATE_FLOOR = '2';
insertRun(db, { id: 'r1', agentId: 'agent-a', score: 2.1, finishedAt: '2026-07-15T10:00:00Z' });

expect(gate.assess({ agent_id: 'agent-a' }, 'mock').action).toBe('allow');
});

it('resolves evidence through the executor model map when the agent has none', () => {
process.env.GOVERNANCE_GATE_MODEL_MAP = 'mock=weak-model,claude=claude-sonnet-4';
insertRun(db, { id: 'r1', agentId: 'nightly:weak-model', score: 1.8, finishedAt: '2026-07-15T10:00:00Z' });

const verdict = gate.assess({ agent_id: null }, 'mock');
expect(verdict.action).toBe('require_approval');
expect(verdict.agentKey).toBe('nightly:weak-model');
});

it('matches runs by metadata subject_model as a fallback', () => {
process.env.GOVERNANCE_GATE_MODEL_MAP = 'mock=some-model';
insertRun(db, { id: 'r1', agentId: 'apex-validation:some-model', score: 1.5, finishedAt: '2026-07-15T10:00:00Z', subjectModel: 'some-model' });

const verdict = gate.assess({ agent_id: null }, 'mock');
expect(verdict.action).toBe('require_approval');
});

it('flags a retirement candidate on three below-floor runs with a declining trend', () => {
insertRun(db, { id: 'r1', agentId: 'agent-a', score: 2.8, finishedAt: '2026-07-13T10:00:00Z' });
insertRun(db, { id: 'r2', agentId: 'agent-a', score: 2.4, finishedAt: '2026-07-14T10:00:00Z' });
insertRun(db, { id: 'r3', agentId: 'agent-a', score: 2.0, finishedAt: '2026-07-15T10:00:00Z' });

const verdict = gate.assess({ agent_id: 'agent-a' }, 'mock');
expect(verdict.action).toBe('require_approval');
expect(verdict.trend).toBe('declining');
expect(verdict.flagRetirement).toBe(true);
expect(verdict.reason).toContain('retirement candidate');
});
});
26 changes: 21 additions & 5 deletions packages/server/src/__tests__/helpers/test-db.ts
Original file line number Diff line number Diff line change
Expand Up @@ -618,16 +618,32 @@ const SCHEMA = `

CREATE TABLE IF NOT EXISTS approvals (
id TEXT PRIMARY KEY,
task_id TEXT,
status TEXT NOT NULL DEFAULT 'pending' CHECK(status IN ('pending', 'approved', 'denied')),
task_id TEXT NOT NULL,
execution_event_id TEXT,
status TEXT NOT NULL DEFAULT 'pending' CHECK(status IN ('pending', 'approved', 'denied', 'expired')),
risk_level TEXT NOT NULL DEFAULT 'low' CHECK(risk_level IN ('low', 'medium', 'high', 'critical')),
request_type TEXT NOT NULL DEFAULT '',
request_type TEXT NOT NULL DEFAULT 'high_risk_action' CHECK(request_type IN ('tool_call', 'file_write', 'shell_command', 'network_request', 'high_risk_action')),
request_message TEXT NOT NULL DEFAULT '',
request_data TEXT NOT NULL DEFAULT '{}',
requested_by TEXT,
approved_by TEXT,
approved_at TEXT,
denied_at TEXT,
created_at TEXT NOT NULL DEFAULT (datetime('now'))
denial_reason TEXT,
expires_at TEXT,
metadata TEXT,
action_type TEXT,
title TEXT,
description TEXT,
command TEXT,
tool_name TEXT,
target_path TEXT,
policy_id TEXT,
decided_at TEXT,
decided_by TEXT,
decision_reason TEXT,
requested_by TEXT,
created_at TEXT NOT NULL DEFAULT (datetime('now')),
updated_at TEXT NOT NULL DEFAULT (datetime('now'))
);

CREATE TABLE IF NOT EXISTS tasks (
Expand Down
27 changes: 26 additions & 1 deletion packages/server/src/execution/execution-engine.ts
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ import { randomUUID } from 'crypto';
import { CommandRiskClassifier } from '../services/command-risk-classifier';
import { PolicyDecisionService } from '../services/policy-decision-service';
import { ApprovalService } from '../services/approval-service';
import { GovernanceGateService } from '../services/governance-gate-service';
import { AuditService } from '../services/audit-service';
import { EvidenceService } from '../services/evidence-service';
import { DiffCaptureService } from '../services/diff-capture';
Expand All @@ -56,6 +57,7 @@ export class ExecutionEngine {
private auditService: AuditService;
private approvalService: ApprovalService;
private evidenceService: EvidenceService;
private governanceGate: GovernanceGateService;
private diffCaptureService: DiffCaptureService;
private memorySyncService?: MemorySyncService;
private reasoningBankService?: ReasoningBankService;
Expand Down Expand Up @@ -95,6 +97,7 @@ export class ExecutionEngine {
this.auditService = new AuditService(db);
this.approvalService = new ApprovalService(db, wsService, this.auditService);
this.evidenceService = new EvidenceService(db);
this.governanceGate = new GovernanceGateService(db);
this.diffCaptureService = new DiffCaptureService(db);

// Register default executors
Expand Down Expand Up @@ -160,9 +163,31 @@ export class ExecutionEngine {
}

const assessment = this.riskClassifier.assessTask(parsedTask, executorKind, process.cwd());
const evaluation = this.policyDecisionService.evaluate(assessment);
let evaluation = this.policyDecisionService.evaluate(assessment);
this.persistRiskAssessment(taskId, assessment, `${parsedTask.title}: ${parsedTask.description}`);

// Governance gate: benchmark evidence can only TIGHTEN the policy decision.
const gateVerdict = this.governanceGate.assess(parsedTask, executorKind);

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P1 Badge Re-run the gate after selecting a fallback executor

When the requested executor's circuit is open, executeTask later reassigns executorKind to a fallback before starting the session. Because the governance gate is evaluated here against the original executor, a request that passes/no-evidence for opencode can fail over to claude/codex/gemini whose mapped benchmark score is below the floor and still execute without approval. This bypasses the new enforcement during circuit-breaker failover; move or repeat the gate after any fallback executor is selected.

Useful? React with 👍 / 👎.

if (gateVerdict.action === 'require_approval' && evaluation.decision === 'allow') {
evaluation = { ...evaluation, decision: 'require_approval', explanation: gateVerdict.reason };
this.evidenceService.captureEvidence({
task_id: taskId,
evidence_type: EvidenceType.POLICY_DECISION,
severity: EvidenceSeverity.WARNING,
title: 'Governance gate tightened execution to require approval',
summary: gateVerdict.reason,
details: {
agentKey: gateVerdict.agentKey,
score: gateVerdict.score,
floor: gateVerdict.floor,
trend: gateVerdict.trend,
retirement_candidate: gateVerdict.flagRetirement,
executorKind,
},
source: 'governance-gate',
});
}
Comment on lines +171 to +189

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

high

The governance gate evidence is currently only captured if evaluation.decision === 'allow'. If the policy decision is already 'require_approval' (or 'deny'), the governance gate check is skipped entirely, and no evidence is captured.

This means the operator/approver will not see the benchmark evidence or the retirement candidate flag when reviewing the task, even though the model is below the floor.

We should decouple the decision tightening from the evidence capturing so that governance gate evidence is always captured when the model is below the floor, regardless of the initial policy decision.

    if (gateVerdict.action === 'require_approval') {
      if (evaluation.decision === 'allow') {
        evaluation = { ...evaluation, decision: 'require_approval', explanation: gateVerdict.reason };
      }
      this.evidenceService.captureEvidence({
        task_id: taskId,
        evidence_type: EvidenceType.POLICY_DECISION,
        severity: EvidenceSeverity.WARNING,
        title: 'Governance gate tightened execution to require approval',
        summary: gateVerdict.reason,
        details: { 
          agentKey: gateVerdict.agentKey,
          score: gateVerdict.score,
          floor: gateVerdict.floor,
          trend: gateVerdict.trend,
          retirement_candidate: gateVerdict.flagRetirement,
          executorKind,
        },
        source: 'governance-gate',
      });
    }


if (evaluation.decision === 'deny') {
this.evidenceService.captureEvidence({
task_id: taskId,
Expand Down
Loading
Loading