diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fdecfdc..96060c1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -59,3 +59,14 @@ jobs: exit 1 fi echo "breaking drift correctly detected (exit 1)" + - name: covenant check catches behavioral drift (schema identical) + run: | + set +e + COVENANT_BEHAVIOR_DRIFT=1 covenant check + code=$? + set -e + if [ "$code" -ne 1 ]; then + echo "::error::expected exit 1 (behavioral drift), got $code" + exit 1 + fi + echo "behavioral drift correctly detected (exit 1)" diff --git a/CLAUDE.md b/CLAUDE.md index 22f2977..34053fc 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -10,6 +10,7 @@ pytest # Postgres tests skip unless COVENANT_TEST_DB is ruff check . && mypy covenant # both must pass; mypy is strict covenant check # lint the example server against the committed baseline COVENANT_DRIFT=1 covenant check # inject a real breaking change; must exit 1 +COVENANT_BEHAVIOR_DRIFT=1 covenant check # body-only drift (schema identical); probes must catch it, exit 1 ``` - `covenant` spawns the stdio server with `python` from PATH — the venv must be active (or its Scripts dir on PATH) or introspection fails with `ModuleNotFoundError: mcp`. @@ -22,6 +23,7 @@ COVENANT_DRIFT=1 covenant check # inject a real breaking change; must exit 1 | 0 contract core | `covenant/*.py` | Depends only on `mcp`, `typer`, `rich`. `diff.py` is pure (no I/O). | | 1 proxy + quarantine | `covenant/proxy/` | `fastapi`/`httpx`/`uvicorn` are the `[proxy]` extra, lazily imported in `cli.py`. | | 2 store | `covenant/store/` | `asyncpg` is the `[store]` extra. Store writes are best-effort: log and swallow, never fail the request path. | +| 3 probes + judge | `covenant/fingerprint.py`, `covenant/judge/` | Probes *execute* tools at snapshot/check time (list read-only tools only). `anthropic`/`google-genai` are the `[judge]` extra, imported on use; the model-name prefix picks the provider. | Design specs (rationale, rule tables, named decisions) live in `docs/superpowers/specs/` — read the relevant spec before changing classifier or proxy behavior. @@ -31,5 +33,6 @@ Design specs (rationale, rule tables, named decisions) live in `docs/superpowers - `covenant.lock.json` is deterministic: sorted keys, no timestamp. Re-snapshotting an unchanged server must be byte-identical. - `schema_hash` covers schemas only, never `description` (a typo fix must not read as an identity change). - Drift detection is Covenant-owned (`POST /covenant/refresh` re-lists the upstream itself). Never make enforcement depend on the client's `tools/list` timing — the SDK can list *after* the call it should have protected. +- Judge verdicts are advisory: DEGRADED, never BREAKING — a probabilistic detector must not trigger quarantine (Layer 3 spec). - CLI errors are typed `CovenantError` → one clean line, exit 2. Never a stack trace, never swallowed. - Exit codes are load-bearing (CI contract): 0 clean, 1 breaking (or degraded under `--strict`), 2 config/connection error. diff --git a/README.md b/README.md index cedeee5..350a59f 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,7 @@ Covenant makes the contract explicit, versioned, and enforced: - **`covenant snapshot`** — introspect an MCP server (stdio or streamable-HTTP) and commit its tool contracts to a deterministic `covenant.lock.json` - **`covenant check`** — diff the live server against the baseline, classify every change **BREAKING / DEGRADED / COMPATIBLE**, and exit non-zero in CI when the contract breaks - **`covenant proxy`** — a transparent reverse-proxy that **quarantines** drifted tools at runtime, so downstream agents get a clean "tool unavailable" instead of silently hallucinating +- **`[[probes]]` + `--judge`** — behavioral fingerprints of what tools **actually return** (works even when a server declares no `outputSchema`), plus an optional LLM judge for semantic drift ## Quickstart @@ -26,21 +27,25 @@ Snapshot the bundled example server, then break it for real — `COVENANT_DRIFT= ```bash $ covenant check -OK no schema drift - contract matches the baseline. +OK no drift - contract matches the baseline. $ COVENANT_DRIFT=1 covenant check - Covenant - contract drift -+-------------------------------------------------------------------------+ -| tier | location | change | -|------------+----------+-------------------------------------------------| -| BREAKING | output | output field 'balance_usd' removed | -| COMPATIBLE | output | output required field 'available_balance' added | -+-------------------------------------------------------------------------+ -x 1 breaking change(s) - downstream agents would fail silently. + Covenant - contract drift ++--------------------------------------------------------------------------------------------+ +| tier | location | change | +|------------+----------+--------------------------------------------------------------------| +| BREAKING | behavior | probe get_account: output field 'balance_usd' removed | +| BREAKING | output | output field 'balance_usd' removed | +| COMPATIBLE | behavior | probe get_account: output optional field 'available_balance' added | +| COMPATIBLE | output | output required field 'available_balance' added | ++--------------------------------------------------------------------------------------------+ +x 2 breaking change(s) - downstream agents would fail silently. Fix or quarantine. $ echo $? 1 ``` +The lie is caught twice: in the declared schema (`output` rows) and — because the committed config probes the tool — in the actual response body (`behavior` rows). + Point it at your own server via [covenant.toml](covenant.toml) (a stdio launch command **or** an HTTP URL), or override inline with `--server`: ```bash @@ -79,6 +84,43 @@ Commit `covenant.toml` + `covenant.lock.json`, then: This repo runs exactly that against its own example server on every push — including a job that *injects* the breaking change and asserts Covenant catches it. See [ci.yml](.github/workflows/ci.yml). +## Behavioral drift: probes + judge + +A schema check can't see a server that *lies* — schema unchanged, response different. And most real MCP tools declare no `outputSchema` at all, so there is nothing to diff. Probes cover both. Commit safe, **read-only** example calls in `covenant.toml`: + +```toml +[[probes]] +tool = "get_transactions" +args = { account_id = "acct-001" } +``` + +`covenant snapshot` runs each probe and stores its response **fingerprint** — the type shape of what actually came back, never the values, which legitimately change — in the lock. `covenant check` re-runs the probes and classifies shape drift with the same severity model, at location `behavior`: + +```bash +$ COVENANT_BEHAVIOR_DRIFT=1 covenant check # response body renames a field; schema untouched + Covenant - contract drift ++-------------------------------------------------------------------------------------------------+ +| tier | location | change | +|------------+----------+-------------------------------------------------------------------------| +| BREAKING | behavior | probe get_transactions: output field 'transactions[].amount_usd' | +| | | removed | +| COMPATIBLE | behavior | probe get_transactions: output optional field | +| | | 'transactions[].amount_cents' added | ++-------------------------------------------------------------------------------------------------+ +x 1 breaking change(s) - downstream agents would fail silently. Fix or quarantine. +``` + +For drift a fingerprint can't see — same shape, changed *meaning*, like a balance quietly rescaled from dollars to cents — add the LLM judge: + +```bash +pip install -e ".[judge]" +covenant check --judge # try it: COVENANT_SEMANTIC_DRIFT=1 covenant check --judge --strict +``` + +The judge model is set with `[judge] model` in `covenant.toml`; the name picks the provider — `claude-*` models use `ANTHROPIC_API_KEY`, `gemini-*` models use `GOOGLE_API_KEY`. + +Judge verdicts are **advisory by design**: they render DEGRADED (fail only under `--strict`), never BREAKING — a probabilistic detector must not trigger quarantine. Full rationale: [Layer 3 design spec](docs/superpowers/specs/2026-07-03-covenant-layer3-behavioral-probes-design.md). + ## Runtime guard: the proxy The linter catches drift at ship time; the proxy contains it at runtime. It forwards every JSON-RPC exchange byte-for-byte (SSE passthrough included) so the client can't tell it's there — but a `tools/call` to a quarantined tool is short-circuited with a clean MCP `isError` result and never forwarded. @@ -117,7 +159,7 @@ Covenant is built in dependency-ordered layers; each ships alone and each higher | 0 | Contract core — introspection, committed baseline, severity classifier, CLI | ✅ shipped | | 1 | Transparent proxy + quarantine | ✅ shipped | | 2 | Postgres contract store (call log, drift events, durable quarantine) | ✅ shipped | -| 3 | Probe agent + RAG — behavioral fingerprints, LLM-judge for semantic drift | roadmap | +| 3 | Behavioral probes — response fingerprints + LLM judge for semantic drift | ✅ shipped | | 4 | Observability — OTel spans, Prometheus, dashboard | roadmap | | 5 | K8s operator + Helm — `MCPContract` CRD, probes as Jobs | roadmap | @@ -127,11 +169,11 @@ Design specs for the shipped layers live in [docs/superpowers/specs](docs/superp ```bash pip install -e ".[dev]" -pytest # 86 tests; Postgres-backed tests skip without a DB +pytest # 112 tests; Postgres-backed tests skip without a DB ruff check . && mypy covenant ``` -Layer boundaries are enforced by imports: the core (`covenant/*.py`) depends only on `mcp`, `typer`, `rich`; the proxy extras (`fastapi`, `httpx`, `uvicorn`) and store extras (`asyncpg`) are optional and lazily imported. +Layer boundaries are enforced by imports: the core (`covenant/*.py`) depends only on `mcp`, `typer`, `rich`; the proxy extras (`fastapi`, `httpx`, `uvicorn`), store extra (`asyncpg`), and judge extra (`anthropic`) are optional and imported on use. ## License diff --git a/covenant.lock.json b/covenant.lock.json index 3bd1c96..bccdb06 100644 --- a/covenant.lock.json +++ b/covenant.lock.json @@ -1,5 +1,82 @@ { "covenant_version": "0.1.0", + "probes": [ + { + "args": { + "account_id": "acct-001" + }, + "fingerprint": { + "properties": { + "account_id": { + "type": "string" + }, + "balance_usd": { + "type": "number" + }, + "currency": { + "type": "string" + }, + "holder": { + "type": "string" + } + }, + "type": "object" + }, + "sample": { + "account_id": "acct-001", + "balance_usd": 4210.0, + "currency": "USD", + "holder": "Ada Lovelace" + }, + "tool": "get_account" + }, + { + "args": { + "account_id": "acct-001" + }, + "fingerprint": { + "properties": { + "account_id": { + "type": "string" + }, + "transactions": { + "items": { + "properties": { + "amount_usd": { + "type": "number" + }, + "merchant": { + "type": "string" + }, + "txn_id": { + "type": "string" + } + }, + "type": "object" + }, + "type": "array" + } + }, + "type": "object" + }, + "sample": { + "account_id": "acct-001", + "transactions": [ + { + "amount_usd": -42.5, + "merchant": "Grocer & Co", + "txn_id": "t-1001" + }, + { + "amount_usd": 1800.0, + "merchant": "Payroll Inc", + "txn_id": "t-1002" + } + ] + }, + "tool": "get_transactions" + } + ], "server": "python examples/mcp_server.py", "tools": { "convert_currency": { @@ -92,6 +169,24 @@ }, "schema_hash": "sha256:43abb25bb6afff16632fdcf65f2e92ecf5f64ab3e1ccf367b6c5b3d25b3288df" }, + "get_transactions": { + "description": "List recent transactions for an account.", + "inputSchema": { + "properties": { + "account_id": { + "title": "Account Id", + "type": "string" + } + }, + "required": [ + "account_id" + ], + "title": "get_transactionsArguments", + "type": "object" + }, + "outputSchema": null, + "schema_hash": "sha256:554a5ad1c8a17f1bcfd7eb11c48a7080d6b5bea1dfff99688b887268305e310b" + }, "get_weather": { "description": "Return current weather for a city.", "inputSchema": { diff --git a/covenant.toml b/covenant.toml index cd82e12..21c78a2 100644 --- a/covenant.toml +++ b/covenant.toml @@ -8,3 +8,18 @@ command = "python examples/mcp_server.py" [baseline] path = "covenant.lock.json" + +[judge] +# The model name picks the provider: claude-* -> ANTHROPIC_API_KEY, gemini-* -> GOOGLE_API_KEY. +model = "gemini-2.5-flash" + +# Layer 3: behavioral probes — example calls executed at snapshot/check time to +# fingerprint what tools *actually return*. Only list read-only tools. + +[[probes]] +tool = "get_account" +args = { account_id = "acct-001" } + +[[probes]] +tool = "get_transactions" +args = { account_id = "acct-001" } diff --git a/covenant/cli.py b/covenant/cli.py index d288c52..dfd1664 100644 --- a/covenant/cli.py +++ b/covenant/cli.py @@ -12,11 +12,13 @@ from rich.console import Console from . import report -from .config import load_config +from ._types import JsonDict +from .config import Config, load_config from .contract import contract_from_tool, read_baseline, write_baseline -from .diff import diff_tools +from .diff import Change, diff_probes, diff_tools from .errors import CovenantError -from .introspect import introspect +from .fingerprint import fingerprint, probe_key +from .introspect import introspect, run_probes app = typer.Typer(add_completion=False, help="A contract linter for MCP servers.") console = Console() @@ -27,6 +29,57 @@ ) +def _snapshot_probes(cfg: Config) -> list[JsonDict]: + """Run the configured probes and build their baseline records.""" + records: list[JsonDict] = [] + for r in run_probes(cfg, cfg.probes): + if r["is_error"]: + raise CovenantError(f"probe {r['tool']} failed at snapshot: {r['error']}") + records.append({ + "tool": r["tool"], + "args": r["args"], + "fingerprint": fingerprint(r["response"]), + "sample": r["response"], + }) + return records + + +def _check_probes( + cfg: Config, base_tools: list[JsonDict], base_probes: list[JsonDict], judge: bool +) -> list[Change]: + """Re-run the probes, diff fingerprints, and (optionally) judge semantics.""" + base_by = {probe_key(p["tool"], p.get("args")): p for p in base_probes} + missing = [p.tool for p in cfg.probes if probe_key(p.tool, p.args) not in base_by] + if missing: + raise CovenantError( + f"probe(s) not in baseline: {', '.join(missing)} - " + "re-run `covenant snapshot --force`" + ) + live = run_probes(cfg, cfg.probes) + changes = diff_probes(base_probes, live) + if not judge: + return changes + from .judge import judge_probe # the [judge] extra is optional; import on use + + descriptions = {t["name"]: t.get("description") for t in base_tools} + shape_drifted = {c.tool for c in changes} + for r in live: + if r["is_error"] or r["tool"] in shape_drifted: + continue # errors and shape drift are already reported; judge only clean shapes + sample = base_by[probe_key(r["tool"], r["args"])].get("sample") + verdict = judge_probe( + r["tool"], descriptions.get(r["tool"]), r["args"], + sample, r["response"], model=cfg.judge_model, + ) + if verdict.drift: + changes.append(Change( + r["tool"], "behavior", None, "semantic_drift", "degraded", + f"probe {r['tool']}: semantic drift suspected - {verdict.reason}", + note="LLM-judge verdict - review manually", + )) + return changes + + @app.command() def snapshot( server: str | None = _server_opt, @@ -41,8 +94,9 @@ def snapshot( tools = introspect(cfg) contracts = [contract_from_tool(t) for t in tools] + probes = _snapshot_probes(cfg) if cfg.probes else None target = cfg.server_url or cfg.server_command or "" - write_baseline(path, contracts, server=target) + write_baseline(path, contracts, server=target, probes=probes) except CovenantError as e: err.print(f"[red]error:[/red] {e}") raise typer.Exit(2) from e @@ -50,6 +104,8 @@ def snapshot( console.print(f"[green]OK snapshotted {len(contracts)} tool(s)[/green] -> {path}") for c in contracts: console.print(f" [cyan]{c.name}[/cyan] [dim]{c.schema_hash[:19]}...[/dim]") + if probes: + console.print(f" [dim]+ {len(probes)} behavioral probe(s) fingerprinted[/dim]") @app.command() @@ -57,17 +113,24 @@ def check( server: str | None = _server_opt, strict: bool = typer.Option(False, "--strict", help="Fail on degraded changes too."), json_out: bool = typer.Option(False, "--json", help="Emit changes as JSON."), + judge: bool = typer.Option( + False, "--judge", + help="Judge probe responses for semantic drift with an LLM (needs ANTHROPIC_API_KEY).", + ), ) -> None: """Diff the live server against the baseline and classify every change.""" try: cfg = load_config(server_override=server) - _, base_tools = read_baseline(cfg.baseline_path) + _, base_tools, base_probes = read_baseline(cfg.baseline_path) current = introspect(cfg) + changes = diff_tools(base_tools, current) + if cfg.probes: + changes += _check_probes(cfg, base_tools, base_probes, judge) + elif judge: + raise CovenantError("--judge needs [[probes]] in covenant.toml to judge") except CovenantError as e: err.print(f"[red]error:[/red] {e}") raise typer.Exit(2) from e - - changes = diff_tools(base_tools, current) if json_out: console.print_json(report.to_json(changes)) else: @@ -104,7 +167,7 @@ def proxy( raise CovenantError('persistence needs: pip install "covenant-mcp[store]"') from e store = PostgresStore(database_url) - _, base_tools = read_baseline(baseline) + _, base_tools, _ = read_baseline(baseline) except CovenantError as e: err.print(f"[red]error:[/red] {e}") raise typer.Exit(2) from e diff --git a/covenant/config.py b/covenant/config.py index 9127379..8c1cc94 100644 --- a/covenant/config.py +++ b/covenant/config.py @@ -1,14 +1,17 @@ -"""Load connection + baseline settings from covenant.toml, with CLI overrides. +"""Load connection + baseline + probe settings from covenant.toml, with CLI overrides. A server is reached either over stdio (a ``command`` launched as a subprocess) or over HTTP (a ``url``). Exactly one must be resolved. A ``--server`` CLI override wins over the file and is treated as a URL if it looks like one, else a command. + +``[[probes]]`` entries are Layer 3's behavioral probes: example calls (tool + args) +that snapshot/check will *execute* against the server — only list read-only tools. """ from __future__ import annotations import tomllib -from dataclasses import dataclass +from dataclasses import dataclass, field from pathlib import Path from ._types import JsonDict @@ -17,17 +20,39 @@ DEFAULT_BASELINE = "covenant.lock.json" +@dataclass +class Probe: + tool: str + args: JsonDict + + @dataclass class Config: server_command: str | None server_url: str | None baseline_path: str + probes: list[Probe] = field(default_factory=list) + judge_model: str | None = None def _looks_like_url(value: str) -> bool: return value.startswith(("http://", "https://")) +def _parse_probes(data: JsonDict) -> list[Probe]: + probes: list[Probe] = [] + for i, entry in enumerate(data.get("probes") or []): + tool = entry.get("tool") if isinstance(entry, dict) else None + args = entry.get("args", {}) if isinstance(entry, dict) else None + if not isinstance(tool, str) or not tool or not isinstance(args, dict): + raise ConfigError( + f'probe #{i + 1} is invalid: each [[probes]] needs tool = "name" ' + "and an optional args table" + ) + probes.append(Probe(tool=tool, args=args)) + return probes + + def load_config(path: str | Path = "covenant.toml", server_override: str | None = None) -> Config: p = Path(path) data: JsonDict = {} @@ -52,5 +77,15 @@ def load_config(path: str | Path = "covenant.toml", server_override: str | None if not command and not url: raise ConfigError("no server configured: set [server].command or .url, or pass --server") + judge_model = (data.get("judge") or {}).get("model") + if judge_model is not None and not isinstance(judge_model, str): + raise ConfigError("[judge].model must be a string") + baseline_path = (data.get("baseline", {}) or {}).get("path", DEFAULT_BASELINE) - return Config(server_command=command, server_url=url, baseline_path=baseline_path) + return Config( + server_command=command, + server_url=url, + baseline_path=baseline_path, + probes=_parse_probes(data), + judge_model=judge_model, + ) diff --git a/covenant/contract.py b/covenant/contract.py index bb77aab..6a5a316 100644 --- a/covenant/contract.py +++ b/covenant/contract.py @@ -53,8 +53,10 @@ def contract_from_tool(tool: JsonDict) -> ToolContract: ) -def to_baseline(contracts: list[ToolContract], server: str) -> JsonDict: - return { +def to_baseline( + contracts: list[ToolContract], server: str, probes: list[JsonDict] | None = None +) -> JsonDict: + data: JsonDict = { "covenant_version": BASELINE_VERSION, "server": server, "tools": { @@ -67,16 +69,26 @@ def to_baseline(contracts: list[ToolContract], server: str) -> JsonDict: for c in contracts }, } - - -def write_baseline(path: str | Path, contracts: list[ToolContract], server: str) -> None: - data = to_baseline(contracts, server) + if probes: + # Sorted so the lock stays deterministic; each record carries the response + # fingerprint plus the raw sample the judge compares against. + data["probes"] = sorted(probes, key=lambda p: (p["tool"], _canonical(p.get("args")))) + return data + + +def write_baseline( + path: str | Path, + contracts: list[ToolContract], + server: str, + probes: list[JsonDict] | None = None, +) -> None: + data = to_baseline(contracts, server, probes) text = json.dumps(data, indent=2, sort_keys=True) + "\n" Path(path).write_text(text, encoding="utf-8") -def read_baseline(path: str | Path) -> tuple[str, list[JsonDict]]: - """Read a baseline file; return (server, tools) where tools are wire-shape dicts.""" +def read_baseline(path: str | Path) -> tuple[str, list[JsonDict], list[JsonDict]]: + """Read a baseline file; return (server, wire-shape tool dicts, probe records).""" p = Path(path) if not p.exists(): raise BaselineError(f"baseline not found: {p} (run `covenant snapshot` first)") @@ -94,4 +106,4 @@ def read_baseline(path: str | Path) -> tuple[str, list[JsonDict]]: } for name, t in (data.get("tools") or {}).items() ] - return data.get("server", ""), tools + return data.get("server", ""), tools, data.get("probes") or [] diff --git a/covenant/diff.py b/covenant/diff.py index a00b007..1cde192 100644 --- a/covenant/diff.py +++ b/covenant/diff.py @@ -9,9 +9,10 @@ from __future__ import annotations import json -from dataclasses import dataclass +from dataclasses import dataclass, replace from ._types import JsonDict +from .fingerprint import fingerprint, probe_key _COMPOSED = ("$ref", "allOf", "anyOf", "oneOf") @@ -231,3 +232,36 @@ def diff_tools(baseline: list[JsonDict], current: list[JsonDict]) -> list[Change )) return changes + + +def diff_probes(baseline: list[JsonDict], live: list[JsonDict]) -> list[Change]: + """Diff live probe responses against baselined fingerprints (Layer 3). + + Responses are output-side by definition, so the classifier's output rules apply + unchanged; results are relabeled ``behavior`` so a report distinguishes "the + schema changed" from "the actual response changed". Live probes without a + baselined counterpart are skipped — the CLI refuses to run in that state. + """ + base_by = {probe_key(p["tool"], p.get("args")): p for p in baseline} + changes: list[Change] = [] + for lp in live: + bp = base_by.get(probe_key(lp["tool"], lp.get("args"))) + if bp is None: + continue + tool = str(lp["tool"]) + if lp.get("is_error"): + changes.append(Change( + tool, "behavior", None, "probe_errored", "degraded", + f"probe {tool}: live call returned an error - {lp.get('error')}", + )) + continue + base_fp, live_fp = bp["fingerprint"], fingerprint(lp["response"]) + if _type_set(base_fp) == {"object"} and _type_set(live_fp) == {"object"}: + raw = _diff_object(tool, "output", base_fp, live_fp) + else: + raw = _diff_field(tool, "output", "response", base_fp, live_fp) + changes += [ + replace(c, location="behavior", message=f"probe {tool}: {c.message}") + for c in raw + ] + return changes diff --git a/covenant/fingerprint.py b/covenant/fingerprint.py new file mode 100644 index 0000000..0e965f9 --- /dev/null +++ b/covenant/fingerprint.py @@ -0,0 +1,41 @@ +"""Infer a minimal JSON-Schema-shaped fingerprint from a live probe response. + +A fingerprint captures the *type shape* of what a tool actually returned — never its +values, which legitimately change between runs. Shapes are what agents rely on, so +fingerprint diffs feed the same output-side severity rules as declared schemas. + +Locked inference rules (Layer 3 design spec): int/float collapse to ``number`` so a +value that happens to be whole never flaps the shape; objects carry no ``required`` +(a missing key already reports as field-removed); arrays keep ``items`` only when +every element fingerprints identically, so ordering can't flap the shape either. +""" + +from __future__ import annotations + +import json + +from ._types import JsonDict + + +def fingerprint(value: object) -> JsonDict: + if isinstance(value, bool): + return {"type": "boolean"} + if isinstance(value, int | float): + return {"type": "number"} + if isinstance(value, str): + return {"type": "string"} + if value is None: + return {"type": "null"} + if isinstance(value, dict): + return {"type": "object", "properties": {k: fingerprint(v) for k, v in value.items()}} + if isinstance(value, list): + shapes = [fingerprint(v) for v in value] + if shapes and all(s == shapes[0] for s in shapes): + return {"type": "array", "items": shapes[0]} + return {"type": "array"} + raise ValueError(f"cannot fingerprint non-JSON value of type {type(value).__name__}") + + +def probe_key(tool: str, args: JsonDict | None) -> str: + """Identity of a probe: tool + canonical args. Changing args means a new baseline.""" + return f"{tool}:{json.dumps(args or {}, sort_keys=True, separators=(',', ':'))}" diff --git a/covenant/introspect.py b/covenant/introspect.py index 126e1fb..66f2da0 100644 --- a/covenant/introspect.py +++ b/covenant/introspect.py @@ -1,15 +1,20 @@ -"""Connect to an MCP server and list its tools — transport-agnostic. +"""Connect to an MCP server — list its tools, and call Layer 3 probes. Supports stdio (a ``command`` launched as a subprocess) and streamable-HTTP (a -``url``). Returns tools in MCP wire shape (``name``/``description``/``inputSchema``/ -``outputSchema``) so the differ and the contract model consume them directly. +``url``). Tools come back in MCP wire shape (``name``/``description``/``inputSchema``/ +``outputSchema``) so the differ and the contract model consume them directly. Probe +calls resolve to comparable JSON: ``structuredContent`` when the server provides it, +else the first text block (parsed as JSON when possible). """ from __future__ import annotations import asyncio +import json import os import shlex +from collections.abc import AsyncIterator +from contextlib import asynccontextmanager from typing import Any from mcp import ClientSession, StdioServerParameters @@ -17,7 +22,7 @@ from mcp.client.streamable_http import streamablehttp_client from ._types import JsonDict -from .config import Config +from .config import Config, Probe from .errors import ConnectionError, CovenantError @@ -27,23 +32,18 @@ def _split_command(command: str) -> list[str]: return shlex.split(command) -def _tool_to_dict(tool: Any) -> JsonDict: - return { - "name": tool.name, - "description": tool.description, - "inputSchema": tool.inputSchema, - "outputSchema": getattr(tool, "outputSchema", None), - } - - -async def _list_tools(session: ClientSession) -> list[JsonDict]: - await session.initialize() - result = await session.list_tools() - return [_tool_to_dict(t) for t in result.tools] - - -async def _introspect_stdio(command: str) -> list[JsonDict]: - parts = _split_command(command) +@asynccontextmanager +async def _session(config: Config) -> AsyncIterator[ClientSession]: + """One initialized client session over whichever transport is configured.""" + if config.server_url: + async with ( + streamablehttp_client(config.server_url) as (read, write, _), + ClientSession(read, write) as session, + ): + await session.initialize() + yield session + return + parts = _split_command(config.server_command or "") # Inherit the full environment: a linter must launch the user's server the way # they run it (the SDK default is a minimal safe env that would hide config vars). params = StdioServerParameters(command=parts[0], args=parts[1:], env=dict(os.environ)) @@ -51,21 +51,57 @@ async def _introspect_stdio(command: str) -> list[JsonDict]: stdio_client(params) as (read, write), ClientSession(read, write) as session, ): - return await _list_tools(session) + await session.initialize() + yield session -async def _introspect_http(url: str) -> list[JsonDict]: - async with ( - streamablehttp_client(url) as (read, write, _), - ClientSession(read, write) as session, - ): - return await _list_tools(session) +def _tool_to_dict(tool: Any) -> JsonDict: + return { + "name": tool.name, + "description": tool.description, + "inputSchema": tool.inputSchema, + "outputSchema": getattr(tool, "outputSchema", None), + } + + +def _resolve_result(result: Any) -> tuple[object, bool, str | None]: + """Resolve a CallToolResult to ``(response, is_error, error_text)``.""" + content = result.content or [] + if result.isError: + texts = [b.text for b in content if hasattr(b, "text")] + return None, True, "; ".join(texts) or "tool returned an error" + if result.structuredContent is not None: + return result.structuredContent, False, None + for block in content: + text = getattr(block, "text", None) + if isinstance(text, str): + try: + return json.loads(text), False, None + except json.JSONDecodeError: + return text, False, None + return None, False, None async def _introspect(config: Config) -> list[JsonDict]: - if config.server_url: - return await _introspect_http(config.server_url) - return await _introspect_stdio(config.server_command or "") + async with _session(config) as session: + result = await session.list_tools() + return [_tool_to_dict(t) for t in result.tools] + + +async def _run_probes(config: Config, probes: list[Probe]) -> list[JsonDict]: + records: list[JsonDict] = [] + async with _session(config) as session: + for probe in probes: + result = await session.call_tool(probe.tool, probe.args) + response, is_error, error = _resolve_result(result) + records.append({ + "tool": probe.tool, + "args": probe.args, + "response": response, + "is_error": is_error, + "error": error, + }) + return records def introspect(config: Config) -> list[JsonDict]: @@ -77,3 +113,14 @@ def introspect(config: Config) -> list[JsonDict]: except Exception as e: # noqa: BLE001 - surface any transport failure as one clean error target = config.server_url or config.server_command raise ConnectionError(f"could not introspect MCP server ({target}): {e}") from e + + +def run_probes(config: Config, probes: list[Probe]) -> list[JsonDict]: + """Call each probe against the live server; return resolved response records.""" + try: + return asyncio.run(_run_probes(config, probes)) + except CovenantError: + raise + except Exception as e: # noqa: BLE001 - surface any transport failure as one clean error + target = config.server_url or config.server_command + raise ConnectionError(f"could not probe MCP server ({target}): {e}") from e diff --git a/covenant/judge/__init__.py b/covenant/judge/__init__.py new file mode 100644 index 0000000..b92f17a --- /dev/null +++ b/covenant/judge/__init__.py @@ -0,0 +1,115 @@ +"""Layer 3's semantic judge — an LLM verdict on probe responses whose *shape* is +unchanged but whose *meaning* may have drifted (units, scale, encoding, repurposed +fields). Fingerprints can't see these; schema diffs never could. + +A verdict is advisory by design: DEGRADED, never BREAKING — a probabilistic +detector must not trigger quarantine (a false positive is a self-inflicted outage). +Judge failures are loud CovenantErrors: the user explicitly opted in with --judge. +""" + +from __future__ import annotations + +import json +from dataclasses import dataclass + +from .._types import JsonDict +from ..errors import CovenantError + +DEFAULT_MODEL = "claude-haiku-4-5-20251001" # cheap and fast; override via [judge].model + +_SYSTEM = ( + "You are Covenant's semantic-drift judge for MCP tool contracts. Compare a " + "baseline response with a live response from the same tool called with the same " + "arguments. The consumer is an LLM agent that trusts the live response to mean " + "the same thing as the baseline: same units, scale, encoding, and per-field " + "semantics. Fresh values are fine; flag only changes of meaning (unit or scale " + "shifts, format changes, repurposed fields). Reply with ONLY this JSON: " + '{"drift": true|false, "reason": ""}' +) + + +@dataclass(frozen=True) +class Verdict: + drift: bool + reason: str + + +def _complete(model: str, system: str, user: str) -> str: + """Route by model name: gemini-* uses GOOGLE_API_KEY, else Anthropic + ANTHROPIC_API_KEY.""" + if model.startswith("gemini"): + return _complete_google(model, system, user) + return _complete_anthropic(model, system, user) + + +def _complete_anthropic(model: str, system: str, user: str) -> str: + try: + import anthropic + except ImportError as e: + raise CovenantError('--judge needs extras: pip install "covenant-mcp[judge]"') from e + try: + # Hold the client in a local: a chained temporary can be garbage-collected + # mid-call (seen on Python 3.14), and the SDK's __del__ closes its transport. + client = anthropic.Anthropic() + msg = client.messages.create( + model=model, + max_tokens=300, + system=system, + messages=[{"role": "user", "content": user}], + ) + except Exception as e: # noqa: BLE001 - missing key / API failure: one clean error + raise CovenantError(f"judge call failed: {e}") from e + return "".join(getattr(b, "text", "") for b in msg.content) + + +def _complete_google(model: str, system: str, user: str) -> str: + try: + from google import genai + from google.genai import types + except ImportError as e: + raise CovenantError('--judge needs extras: pip install "covenant-mcp[judge]"') from e + try: + # Hold the client in a local: a chained temporary is garbage-collected + # mid-call on Python 3.14, and SyncHttpxClient.__del__ closes the transport. + client = genai.Client() + resp = client.models.generate_content( + model=model, + contents=user, + config=types.GenerateContentConfig( + system_instruction=system, + max_output_tokens=300, + # a one-line verdict needs no thinking budget eating the output cap + thinking_config=types.ThinkingConfig(thinking_budget=0), + ), + ) + except Exception as e: # noqa: BLE001 - missing key / API failure: one clean error + raise CovenantError(f"judge call failed: {e}") from e + return resp.text or "" + + +def judge_probe( + tool: str, + description: str | None, + args: JsonDict, + baseline_sample: object, + live_response: object, + model: str | None = None, +) -> Verdict: + """Ask the judge whether the live response semantically drifted from the baseline.""" + payload = json.dumps( + { + "tool": tool, + "description": description, + "arguments": args, + "baseline_response": baseline_sample, + "live_response": live_response, + }, + sort_keys=True, + ) + raw = _complete(model or DEFAULT_MODEL, _SYSTEM, payload).strip() + if raw.startswith("```"): + raw = raw.strip("`").removeprefix("json").strip() + try: + data = json.loads(raw) + return Verdict(drift=bool(data["drift"]), reason=str(data["reason"])) + except (json.JSONDecodeError, KeyError, TypeError) as e: + raise CovenantError(f"judge returned an unparseable verdict: {raw!r}") from e diff --git a/covenant/report.py b/covenant/report.py index 6451921..b3c57ee 100644 --- a/covenant/report.py +++ b/covenant/report.py @@ -34,9 +34,9 @@ def to_json(changes: list[Change]) -> str: def render(changes: list[Change], strict: bool, console: Console | None = None) -> None: console = console or Console() if not changes: - console.print("[green]OK no schema drift[/green] - contract matches the baseline.") - console.print("[dim]note: a clean check means no *schema* drift, not 'contract safe' " - "(behavioral / material-description drift is not checked here).[/dim]") + console.print("[green]OK no drift[/green] - contract matches the baseline.") + console.print("[dim]note: schemas and configured probes only - unprobed behavior " + "and description materiality are not checked.[/dim]") return table = Table(title="Covenant - contract drift", show_lines=False) diff --git a/docs/BUILD_LOG.md b/docs/BUILD_LOG.md index 083fb6e..bd6c90f 100644 --- a/docs/BUILD_LOG.md +++ b/docs/BUILD_LOG.md @@ -56,3 +56,9 @@ CodeRabbit auto-reviewed PR #1 and raised 9 inline findings. Each was verified a | Spec map | Layer 0 spec's module map "missing" `proxy` | **Skipped** — proxy is Layer 1 scope, documented in the Layer 1 spec | After the fixes: `pytest` 87 passed / 3 skipped, `ruff` + `mypy --strict` clean, drift demo still 0 (clean) / 1 (breaking). This is the loop working as designed — an external reviewer found a real classifier bug, and the contract-check dogfood job plus the rule-table tests caught the fix landing correctly. + +## Layer 3 — behavioral probes + semantic judge (same day, second branch) + +`feat/layer3-behavioral-probes`. A clean schema check can't see a lying server, and most real MCP tools declare no `outputSchema` at all. Layer 3 adds `[[probes]]` (safe example calls in `covenant.toml`): `snapshot` fingerprints each response's *type shape* into the lock, `check` re-runs the probes and classifies drift **with the unchanged Layer 0 classifier** (responses are output-side by definition), rendered at location `behavior`. `covenant check --judge` (optional `[judge]` extra) additionally sends baseline sample + live response to an LLM for semantic drift (dollars→cents) — verdicts are advisory DEGRADED, never BREAKING, so a probabilistic detector cannot cause a quarantine outage. + +Verified end-to-end: 112 tests (22 new), ruff + strict mypy clean, lock still byte-deterministic after re-snapshot, `COVENANT_BEHAVIOR_DRIFT=1` (schema untouched, body renames a field) exits 1 via the probe path, and the classic `COVENANT_DRIFT=1` is now caught twice — declared schema and actual body. CI dogfood gained the behavioral lever. Design: `docs/superpowers/specs/2026-07-03-covenant-layer3-behavioral-probes-design.md`. diff --git a/docs/superpowers/specs/2026-07-03-covenant-layer3-behavioral-probes-design.md b/docs/superpowers/specs/2026-07-03-covenant-layer3-behavioral-probes-design.md new file mode 100644 index 0000000..0c4a316 --- /dev/null +++ b/docs/superpowers/specs/2026-07-03-covenant-layer3-behavioral-probes-design.md @@ -0,0 +1,69 @@ +# Covenant Layer 3 — behavioral probes + semantic judge + +Date: 2026-07-03 · Status: shipped · Depends on: Layer 0 classifier (reused unchanged) + +## Problem + +Layer 0 diffs *declared* schemas. Two blind spots: + +1. Most real MCP tools declare no (or loose) `outputSchema` — there is nothing to diff. +2. A server can lie: schema unchanged while the actual response shape or meaning drifts. + A clean `check` deliberately says "no *schema* drift", not "contract safe" — this layer + is the answer to that caveat. + +## Mechanism + +A **probe** is a user-defined safe example call — `[[probes]]` in `covenant.toml` with a +`tool` and `args`. Only list read-only tools: probes are *called* at snapshot and check time. + +- `covenant snapshot` runs each probe and stores in the lock: a **fingerprint** (the type + shape inferred from the actual response) plus the raw **sample** response. +- `covenant check` re-runs the probes and diffs live vs baseline fingerprint **with the + Layer 0 classifier**. Responses are output-side by definition, so the direction principle + applies unchanged: lost field / structural retype / gains-null = BREAKING, scalar retype = + DEGRADED, additions = COMPATIBLE. Changes render with location `behavior`. +- `covenant check --judge` additionally sends (tool description, args, baseline sample, + live response) to an LLM judge for **semantic** drift that shape can't see (dollars→cents). + +## Fingerprint rules (locked) + +- `bool`→boolean · `int`/`float`→number (collapsed: a value that happens to be whole must + not flap integer↔number between runs) · `str`→string · `None`→null +- `dict` → object with per-key fingerprints. No `required` list — a key missing at check + time already reports as field-removed. +- `list` → array; `items` kept only when every element fingerprints identically, else bare + array. Accepted limitation: a uniform array going heterogeneous fingerprints as bare and + is not flagged — probe tools that return uniform collections. +- Response source order: `structuredContent` → first text block parsed as JSON → raw text + as a string. + +## Severity decisions + +| Case | Tier | Why | +|---|---|---| +| Live response loses a field / structural retype / gains null | BREAKING | Same silent-lie class as schema output changes — classifier reused verbatim | +| Probe now returns `isError` | DEGRADED | Loud: the agent sees the error. Direction principle. | +| Judge suspects semantic drift | DEGRADED | The detector is probabilistic; a false BREAKING is a self-inflicted quarantine outage. Same precedent as the composition punt: flag for review, never auto-break. | +| Probe in covenant.toml but not in the lock | error, exit 2 | A baseline mismatch is a config state, not drift — re-snapshot. | + +Judge failures (missing key, API error, unparseable verdict) are loud `CovenantError`s → +exit 2, never silently skipped: the user explicitly opted in with `--judge`. + +## Determinism + +The lock stays deterministic for a server whose probe responses are stable (sorted keys; +probes sorted by tool + canonical args). A server returning volatile values (timestamps) +legitimately changes `sample` between snapshots — probe stable read-only tools. + +## Demo levers (examples/mcp_server.py) + +- `COVENANT_BEHAVIOR_DRIFT=1` — `get_transactions` (loose `dict` output, invisible to + Layer 0) renames `amount_usd`→`amount_cents` in the response body only. Schema check + stays clean; the probe catches BREAKING. CI-runnable, no API key. +- `COVENANT_SEMANTIC_DRIFT=1` — `get_account` returns the balance ×100. Schema and shape + identical; only `--judge` catches it. + +## Out of scope + +Automatic probe generation, value-level assertions (snapshot testing), judging live +traffic at the proxy (cost — Layer 3 judges probes on demand). diff --git a/examples/mcp_server.py b/examples/mcp_server.py index b65fa4b..8b0beb5 100644 --- a/examples/mcp_server.py +++ b/examples/mcp_server.py @@ -8,6 +8,12 @@ COVENANT_DRIFT=1 covenant check # Covenant catches the breaking diff reproduces a real breaking change end-to-end, nothing simulated. + +Layer 3 levers (declared schemas stay identical — only behavior changes): +COVENANT_BEHAVIOR_DRIFT=1 renames `amount_usd`->`amount_cents` inside +`get_transactions` response bodies (probe fingerprints catch it); +COVENANT_SEMANTIC_DRIFT=1 makes `get_account` return the balance in cents — +same schema, same shape, changed meaning (only the LLM judge sees it). """ import os @@ -26,6 +32,8 @@ ) DRIFT = os.environ.get("COVENANT_DRIFT") == "1" +BEHAVIOR_DRIFT = os.environ.get("COVENANT_BEHAVIOR_DRIFT") == "1" +SEMANTIC_DRIFT = os.environ.get("COVENANT_SEMANTIC_DRIFT") == "1" _ACCOUNTS = { "acct-001": ("Ada Lovelace", 4210.00, "USD"), @@ -51,6 +59,8 @@ class Account(BaseModel): def get_account(account_id: str) -> Account: """Look up a bank account and its current balance.""" holder, balance, currency = _ACCOUNTS.get(account_id, ("Unknown", 0.0, "USD")) + if SEMANTIC_DRIFT: + balance = balance * 100 # cents: schema and shape identical, meaning changed fields = {"account_id": account_id, "holder": holder, "currency": currency} fields["available_balance" if DRIFT else "balance_usd"] = balance return Account(**fields) @@ -81,6 +91,27 @@ def convert_currency(amount: float, to_currency: str = "EUR") -> Conversion: return Conversion(amount=amount, rate=rate, converted=round(amount * rate, 2)) +_TRANSACTIONS = { + "acct-001": [ + {"txn_id": "t-1001", "amount_usd": -42.5, "merchant": "Grocer & Co"}, + {"txn_id": "t-1002", "amount_usd": 1800.0, "merchant": "Payroll Inc"}, + ], +} + + +@mcp.tool() +def get_transactions(account_id: str) -> dict: + """List recent transactions for an account.""" + txns = _TRANSACTIONS.get(account_id, []) + if BEHAVIOR_DRIFT: # the response body changes; the declared (loose) schema does not + txns = [ + {"txn_id": t["txn_id"], "amount_cents": int(t["amount_usd"] * 100), + "merchant": t["merchant"]} + for t in txns + ] + return {"account_id": account_id, "transactions": txns} + + if __name__ == "__main__": if os.environ.get("COVENANT_HTTP") == "1": mcp.run(transport="streamable-http") # for the Layer 1 proxy demo diff --git a/pyproject.toml b/pyproject.toml index dd65d33..46c643f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,6 +45,10 @@ proxy = [ store = [ "asyncpg>=0.29", ] +judge = [ + "anthropic>=0.40", + "google-genai>=1.0", +] dev = [ "pytest>=8", "anyio>=4", @@ -54,6 +58,8 @@ dev = [ "uvicorn>=0.30", "httpx>=0.27", "asyncpg>=0.29", + "anthropic>=0.40", + "google-genai>=1.0", ] [tool.hatch.build.targets.wheel] diff --git a/tests/test_cli.py b/tests/test_cli.py index 82d6a17..e48665c 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -56,3 +56,43 @@ def test_check_without_baseline_errors(tmp_path, monkeypatch): monkeypatch.chdir(tmp_path) r = runner.invoke(app, ["check", "--server", SERVER]) assert r.exit_code == 2 + + +def _write_toml(tmp_path, extra=""): + (tmp_path / "covenant.toml").write_text( + f"[server]\ncommand = '{SERVER}'\n{extra}", encoding="utf-8" + ) + + +def test_probes_catch_behavioral_drift(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + _write_toml(tmp_path, '\n[[probes]]\ntool = "get_transactions"\n' + 'args = { account_id = "acct-001" }\n') + r = runner.invoke(app, ["snapshot"]) + assert r.exit_code == 0, r.output + r = runner.invoke(app, ["check"]) + assert r.exit_code == 0, r.output + + monkeypatch.setenv("COVENANT_BEHAVIOR_DRIFT", "1") # schema identical; response body lies + r = runner.invoke(app, ["check"]) + assert r.exit_code == 1, r.output + assert "amount_usd" in r.output + assert "BREAKING" in r.output + assert "behavior" in r.output + + +def test_probe_missing_from_baseline_errors(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + _write_toml(tmp_path) + runner.invoke(app, ["snapshot"]) + _write_toml(tmp_path, '\n[[probes]]\ntool = "get_weather"\nargs = { city = "Haifa" }\n') + r = runner.invoke(app, ["check"]) + assert r.exit_code == 2 + assert "snapshot" in r.output + + +def test_judge_without_probes_errors(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + runner.invoke(app, ["snapshot", "--server", SERVER]) + r = runner.invoke(app, ["check", "--server", SERVER, "--judge"]) + assert r.exit_code == 2 diff --git a/tests/test_contract.py b/tests/test_contract.py index bc77480..bf1fef1 100644 --- a/tests/test_contract.py +++ b/tests/test_contract.py @@ -46,7 +46,7 @@ def test_baseline_round_trips(tmp_path): path = tmp_path / "covenant.lock.json" write_baseline(path, contracts, server="python examples/mcp_server.py") - server, tools = read_baseline(path) + server, tools, _ = read_baseline(path) assert server == "python examples/mcp_server.py" assert tools[0]["name"] == "get_account" assert tools[0]["inputSchema"]["properties"]["id"]["type"] == "string" diff --git a/tests/test_fingerprint.py b/tests/test_fingerprint.py new file mode 100644 index 0000000..b475adf --- /dev/null +++ b/tests/test_fingerprint.py @@ -0,0 +1,40 @@ +"""Fingerprint inference: the locked rules from the Layer 3 spec.""" + +import pytest + +from covenant.fingerprint import fingerprint, probe_key + + +def test_scalars(): + assert fingerprint(True) == {"type": "boolean"} + assert fingerprint(3) == {"type": "number"} + assert fingerprint(3.5) == {"type": "number"} # collapsed: no int/float flapping + assert fingerprint("x") == {"type": "string"} + assert fingerprint(None) == {"type": "null"} + + +def test_nested_object(): + assert fingerprint({"balance": 42.5, "meta": {"currency": "USD"}}) == { + "type": "object", + "properties": { + "balance": {"type": "number"}, + "meta": {"type": "object", "properties": {"currency": {"type": "string"}}}, + }, + } + + +def test_arrays_keep_items_only_when_uniform(): + assert fingerprint([1, 2.5]) == {"type": "array", "items": {"type": "number"}} + assert fingerprint([1, "a"]) == {"type": "array"} + assert fingerprint([]) == {"type": "array"} + + +def test_non_json_value_is_loud(): + with pytest.raises(ValueError): + fingerprint({1, 2}) + + +def test_probe_key_canonicalizes_args(): + assert probe_key("t", {"a": 1, "b": 2}) == probe_key("t", {"b": 2, "a": 1}) + assert probe_key("t", None) == probe_key("t", {}) + assert probe_key("t", {"a": 1}) != probe_key("t", {"a": 2}) diff --git a/tests/test_introspect.py b/tests/test_introspect.py index eb8b935..b7b6ce6 100644 --- a/tests/test_introspect.py +++ b/tests/test_introspect.py @@ -2,8 +2,8 @@ import sys -from covenant.config import Config -from covenant.introspect import introspect +from covenant.config import Config, Probe +from covenant.introspect import introspect, run_probes def _example_config(): @@ -22,3 +22,10 @@ def test_get_account_output_schema_has_balance_usd(): acct = next(t for t in tools if t["name"] == "get_account") assert acct["outputSchema"] is not None assert "balance_usd" in acct["outputSchema"]["properties"] + + +def test_run_probes_resolves_a_real_tool_response(): + (rec,) = run_probes(_example_config(), [Probe(tool="get_weather", args={"city": "Haifa"})]) + assert rec["is_error"] is False + assert rec["response"]["city"] == "Haifa" + assert rec["response"]["temp_c"] == 21.5 diff --git a/tests/test_judge.py b/tests/test_judge.py new file mode 100644 index 0000000..cbbd0ee --- /dev/null +++ b/tests/test_judge.py @@ -0,0 +1,62 @@ +"""The semantic judge: prompt payload, verdict parsing, loud failures.""" + +import pytest + +import covenant.judge as judge_mod +from covenant.errors import CovenantError +from covenant.judge import Verdict, judge_probe + + +def _fake_complete(reply): + calls = {} + + def fake(model, system, user): + calls["model"], calls["system"], calls["user"] = model, system, user + return reply + + return fake, calls + + +def test_drift_verdict_parsed_and_payload_carries_both_responses(monkeypatch): + fake, calls = _fake_complete('{"drift": true, "reason": "balance rescaled to cents"}') + monkeypatch.setattr(judge_mod, "_complete", fake) + v = judge_probe("get_account", "desc", {"id": "a"}, {"balance_usd": 42.5}, + {"balance_usd": 4250.0}) + assert v == Verdict(drift=True, reason="balance rescaled to cents") + assert "42.5" in calls["user"] + assert "4250.0" in calls["user"] + assert calls["model"] == judge_mod.DEFAULT_MODEL + + +def test_fenced_json_is_tolerated(monkeypatch): + fake, _ = _fake_complete('```json\n{"drift": false, "reason": "same meaning"}\n```') + monkeypatch.setattr(judge_mod, "_complete", fake) + assert judge_probe("t", None, {}, {}, {}).drift is False + + +def test_unparseable_verdict_is_loud(monkeypatch): + fake, _ = _fake_complete("cannot judge, sorry") + monkeypatch.setattr(judge_mod, "_complete", fake) + with pytest.raises(CovenantError): + judge_probe("t", None, {}, {}, {}) + + +def test_model_override_wins(monkeypatch): + fake, calls = _fake_complete('{"drift": false, "reason": "ok"}') + monkeypatch.setattr(judge_mod, "_complete", fake) + judge_probe("t", None, {}, {}, {}, model="custom-model") + assert calls["model"] == "custom-model" + + +def test_gemini_models_route_to_google(monkeypatch): + fake, calls = _fake_complete('{"drift": false, "reason": "ok"}') + monkeypatch.setattr(judge_mod, "_complete_google", fake) + judge_probe("t", None, {}, {}, {}, model="gemini-2.5-flash") + assert calls["model"] == "gemini-2.5-flash" + + +def test_default_model_routes_to_anthropic(monkeypatch): + fake, calls = _fake_complete('{"drift": false, "reason": "ok"}') + monkeypatch.setattr(judge_mod, "_complete_anthropic", fake) + judge_probe("t", None, {}, {}, {}) + assert calls["model"] == judge_mod.DEFAULT_MODEL diff --git a/tests/test_probes.py b/tests/test_probes.py new file mode 100644 index 0000000..a254cd5 --- /dev/null +++ b/tests/test_probes.py @@ -0,0 +1,101 @@ +"""Layer 3 probe pipeline: config parsing, lock round-trip, and fingerprint diffs.""" + +import pytest + +from covenant.config import load_config +from covenant.contract import read_baseline, write_baseline +from covenant.diff import diff_probes +from covenant.errors import ConfigError +from covenant.fingerprint import fingerprint + + +def _probe(tool, response, args=None): + return { + "tool": tool, "args": args or {}, + "fingerprint": fingerprint(response), "sample": response, + } + + +def _live(tool, response, args=None, is_error=False, error=None): + return { + "tool": tool, "args": args or {}, + "response": response, "is_error": is_error, "error": error, + } + + +def test_lost_response_field_is_breaking(): + base = [_probe("get_txns", {"amount_usd": 1.0, "merchant": "x"})] + live = [_live("get_txns", {"amount_cents": 100, "merchant": "x"})] + changes = diff_probes(base, live) + kinds = {(c.kind, c.tier) for c in changes} + assert ("removed", "breaking") in kinds # amount_usd gone: silent lie + assert ("added", "compatible") in kinds # amount_cents new: additive + assert all(c.location == "behavior" for c in changes) + + +def test_nested_array_field_loss_is_breaking(): + base = [_probe("t", {"txns": [{"amount_usd": 1.0}, {"amount_usd": 2.0}]})] + live = [_live("t", {"txns": [{"amount_cents": 100}, {"amount_cents": 200}]})] + fields = {(c.field, c.tier) for c in diff_probes(base, live)} + assert ("txns[].amount_usd", "breaking") in fields + + +def test_scalar_retype_is_degraded(): + base = [_probe("t", {"balance": 42.5})] + live = [_live("t", {"balance": "42.50"})] + (c,) = diff_probes(base, live) + assert (c.kind, c.tier) == ("type_changed_scalar", "degraded") + + +def test_value_change_same_shape_is_clean(): + base = [_probe("t", {"balance": 42.5})] + live = [_live("t", {"balance": 4250.0})] # cents pun: invisible to shape, the judge's job + assert diff_probes(base, live) == [] + + +def test_probe_error_is_degraded_and_loud(): + base = [_probe("t", {"ok": 1})] + live = [_live("t", None, is_error=True, error="boom")] + (c,) = diff_probes(base, live) + assert c.tier == "degraded" + assert c.kind == "probe_errored" + assert "boom" in c.message + + +def test_probes_matched_by_args_identity(): + base = [_probe("t", {"a": 1}, args={"id": "x"})] + live = [_live("t", {"a": 1}, args={"id": "x"}), _live("t", {}, args={"id": "y"})] + assert diff_probes(base, live) == [] # unmatched live probe skipped; CLI gates that state + + +def test_lock_round_trips_probes_sorted(tmp_path): + path = tmp_path / "covenant.lock.json" + probes = [_probe("b", {"x": 1}), _probe("a", {"y": "z"})] + write_baseline(path, [], server="cmd", probes=probes) + _, _, loaded = read_baseline(path) + assert [p["tool"] for p in loaded] == ["a", "b"] # sorted: lock stays deterministic + assert loaded[1]["fingerprint"] == fingerprint({"x": 1}) + assert loaded[1]["sample"] == {"x": 1} + + +def test_config_parses_probes_and_judge_model(tmp_path): + cfg_file = tmp_path / "covenant.toml" + cfg_file.write_text( + '[server]\ncommand = "python x.py"\n' + '[judge]\nmodel = "my-model"\n' + '[[probes]]\ntool = "get_account"\nargs = { account_id = "a-1" }\n' + '[[probes]]\ntool = "ping"\n', + encoding="utf-8", + ) + cfg = load_config(cfg_file) + assert [(p.tool, p.args) for p in cfg.probes] == [ + ("get_account", {"account_id": "a-1"}), ("ping", {}), + ] + assert cfg.judge_model == "my-model" + + +def test_config_rejects_probe_without_tool(tmp_path): + cfg_file = tmp_path / "covenant.toml" + cfg_file.write_text('[server]\ncommand = "x"\n[[probes]]\nargs = { a = 1 }\n', encoding="utf-8") + with pytest.raises(ConfigError): + load_config(cfg_file)