diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 96060c1..f523d51 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -4,6 +4,8 @@ on: push: branches: [main] pull_request: + schedule: + - cron: "17 6 3 * *" # monthly freshness alarm: catch dependency/runner rot before a visitor does jobs: test: @@ -25,8 +27,8 @@ jobs: --health-timeout 3s --health-retries 10 steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 + - uses: actions/checkout@v5 + - uses: actions/setup-python@v6 with: python-version: ${{ matrix.python-version }} - run: pip install -e ".[dev]" @@ -41,8 +43,8 @@ jobs: contract-check: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 + - uses: actions/checkout@v5 + - uses: actions/setup-python@v6 with: python-version: "3.12" - run: pip install -e . @@ -70,3 +72,14 @@ jobs: exit 1 fi echo "behavioral drift correctly detected (exit 1)" + - name: covenant check catches value drift (schema and shape identical) + run: | + set +e + COVENANT_SEMANTIC_DRIFT=1 covenant check + code=$? + set -e + if [ "$code" -ne 1 ]; then + echo "::error::expected exit 1 (value drift), got $code" + exit 1 + fi + echo "value drift correctly detected by expect pins (exit 1)" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index c9b74e4..5489b08 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -8,8 +8,8 @@ jobs: build: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 + - uses: actions/checkout@v5 + - uses: actions/setup-python@v6 with: python-version: "3.12" - run: python -m pip install build twine diff --git a/CLAUDE.md b/CLAUDE.md index 9b65a5f..e91458e 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -11,6 +11,7 @@ ruff check . && mypy covenant # both must pass; mypy is strict covenant check # lint the example server against the committed baseline COVENANT_DRIFT=1 covenant check # inject a real breaking change; must exit 1 COVENANT_BEHAVIOR_DRIFT=1 covenant check # body-only drift (schema identical); probes must catch it, exit 1 +COVENANT_SEMANTIC_DRIFT=1 covenant check # value-only drift (schema AND shape identical); expect pins must catch it, exit 1 ``` - `covenant` spawns the stdio server with `python` from PATH — the venv must be active (or its Scripts dir on PATH) or introspection fails with `ModuleNotFoundError: mcp`. @@ -36,5 +37,6 @@ Design specs (rationale, rule tables, named decisions) live in `docs/specs/` — - `schema_hash` covers schemas only, never `description` (a typo fix must not read as an identity change). - Drift detection is Covenant-owned (`POST /covenant/refresh` re-lists the upstream itself). Never make enforcement depend on the client's `tools/list` timing — the SDK can list *after* the call it should have protected. - Judge verdicts are advisory: DEGRADED, never BREAKING — a probabilistic detector must not trigger quarantine (Layer 3 spec). +- Value pins (`expect` on a probe) are deterministic and exact: a mismatch is BREAKING, non-configurable. No tolerance, no regex, no auto-pinning. - CLI errors are typed `CovenantError` → one clean line, exit 2. Never a stack trace, never swallowed. - Exit codes are load-bearing (CI contract): 0 clean, 1 breaking (or degraded under `--strict`), 2 config/connection error. diff --git a/README.md b/README.md index 6477cc9..0c234b3 100644 --- a/README.md +++ b/README.md @@ -39,6 +39,7 @@ The lie is caught twice: in the declared schema (`output` rows) and in the actua Point it at your own server via [covenant.toml](https://github.com/Mhemd139/Covenant/blob/main/covenant.toml), or inline: ```bash +pip install covenant-mcp covenant snapshot --server http://localhost:8000/mcp # or a stdio launch command covenant check --server http://localhost:8000/mcp --json ``` @@ -52,7 +53,7 @@ The consumer of an MCP tool is an LLM agent that re-reads tool definitions on ev | Change | Tier | | --- | --- | -| Output field removed · output required→optional · output gains `null` · structural output retype · tool removed | **BREAKING** | +| Output field removed · output required→optional · output gains `null` · structural output retype · tool removed · pinned value changed | **BREAKING** | | Input retyped/narrowed · new required input · scalar output retype · risky enum changes · description changed | DEGRADED | | Optional input added · output field added · input enum widened | COMPATIBLE | @@ -65,7 +66,7 @@ Commit `covenant.toml` + `covenant.lock.json`, then: ```yaml - name: Contract check run: | - pip install "covenant-mcp @ git+https://github.com/Mhemd139/Covenant" + pip install covenant-mcp covenant check --json # exit 1 on breaking drift, 2 on config/connection error ``` @@ -81,7 +82,20 @@ tool = "get_transactions" args = { account_id = "acct-001" } ``` -`snapshot` stores each response's **fingerprint** (the type shape of what actually came back); `check` re-runs the probes and classifies shape drift with the same severity model. For drift a fingerprint can't see — same shape, changed *meaning*, like a balance quietly rescaled from dollars to cents — add the LLM judge: +`snapshot` stores each response's **fingerprint** (the type shape of what actually came back); `check` re-runs the probes and classifies shape drift with the same severity model. + +A fingerprint remembers that *a number lives there* — not which number. When the exact value is part of the contract — a reference balance, a currency code, a unit — **pin it**: + +```toml +[[probes]] +tool = "get_account" +args = { account_id = "acct-001" } +expect = { balance_usd = 4210.0, currency = "USD" } +``` + +`check` compares every pinned field against the live response with exact equality — no tolerance, no patterns. A mismatch is **BREAKING**: schema and shape still match while the value lies (a balance rescaled to cents, dollars quietly converted to another currency) — exactly the silent failure the direction principle exists to catch. Pins are opt-in and deterministic, like `pip --require-hashes`: nothing is pinned unless you type it. Try it on the example server — `COVENANT_SEMANTIC_DRIFT=1 covenant check` rescales the live balance ×100 and exits 1. + +For drift you *didn't* pin — fields too volatile to pin, meaning shifts across the whole response — add the LLM judge: ```bash pip install -e ".[judge]" diff --git a/covenant.toml b/covenant.toml index 21c78a2..6ff2f96 100644 --- a/covenant.toml +++ b/covenant.toml @@ -19,6 +19,10 @@ model = "gemini-2.5-flash" [[probes]] tool = "get_account" args = { account_id = "acct-001" } +# Value pins: exact values that are part of the contract. Schema and shape can't +# see a rescale or a currency swap; a pinned mismatch is BREAKING. Pin only fields +# that are deterministic for these args — volatile fields are the judge's job. +expect = { balance_usd = 4210.0, currency = "USD" } [[probes]] tool = "get_transactions" diff --git a/covenant/cli.py b/covenant/cli.py index 5cd200f..143d4e8 100644 --- a/covenant/cli.py +++ b/covenant/cli.py @@ -15,7 +15,7 @@ from ._types import JsonDict from .config import Config, load_config from .contract import contract_from_tool, read_baseline, write_baseline -from .diff import Change, diff_probes, diff_tools +from .diff import Change, diff_expect, diff_probes, diff_tools from .errors import CovenantError from .fingerprint import fingerprint, probe_key from .introspect import introspect, run_probes @@ -47,7 +47,7 @@ def _snapshot_probes(cfg: Config) -> list[JsonDict]: def _check_probes( cfg: Config, base_tools: list[JsonDict], base_probes: list[JsonDict], judge: bool ) -> list[Change]: - """Re-run the probes, diff fingerprints, and (optionally) judge semantics.""" + """Re-run the probes, diff fingerprints + value pins, and (optionally) judge.""" base_by = {probe_key(p["tool"], p.get("args")): p for p in base_probes} missing = [p.tool for p in cfg.probes if probe_key(p.tool, p.args) not in base_by] if missing: @@ -57,6 +57,11 @@ def _check_probes( ) live = run_probes(cfg, cfg.probes) changes = diff_probes(base_probes, live) + live_by = {probe_key(r["tool"], r["args"]): r for r in live} + for p in cfg.probes: + r = live_by[probe_key(p.tool, p.args)] # run_probes returns one record per probe + if p.expect and not r["is_error"]: # an errored probe already reports probe_errored + changes += diff_expect(p.tool, p.expect, r["response"]) if not judge: return changes from .judge import judge_probe # the [judge] extra is optional; import on use diff --git a/covenant/config.py b/covenant/config.py index 2eefcdc..b9c7ffe 100644 --- a/covenant/config.py +++ b/covenant/config.py @@ -6,6 +6,8 @@ ``[[probes]]`` entries are Layer 3's behavioral probes: example calls (tool + args) that snapshot/check will *execute* against the server — only list read-only tools. +An optional ``expect`` table pins exact output values; a pinned field that comes +back missing or unequal at check time is BREAKING (see diff.diff_expect). """ from __future__ import annotations @@ -24,6 +26,7 @@ class Probe: tool: str args: JsonDict + expect: JsonDict = field(default_factory=dict) @dataclass @@ -44,12 +47,14 @@ def _parse_probes(data: JsonDict) -> list[Probe]: for i, entry in enumerate(data.get("probes") or []): tool = entry.get("tool") if isinstance(entry, dict) else None args = entry.get("args", {}) if isinstance(entry, dict) else None - if not isinstance(tool, str) or not tool or not isinstance(args, dict): + expect = entry.get("expect", {}) if isinstance(entry, dict) else None + if (not isinstance(tool, str) or not tool + or not isinstance(args, dict) or not isinstance(expect, dict)): raise ConfigError( f'probe #{i + 1} is invalid: each [[probes]] needs tool = "name" ' - "and an optional args table" + "and optional args / expect tables" ) - probes.append(Probe(tool=tool, args=args)) + probes.append(Probe(tool=tool, args=args, expect=expect)) return probes diff --git a/covenant/diff.py b/covenant/diff.py index 1cde192..0de4a6e 100644 --- a/covenant/diff.py +++ b/covenant/diff.py @@ -265,3 +265,29 @@ def diff_probes(baseline: list[JsonDict], live: list[JsonDict]) -> list[Change]: for c in raw ] return changes + + +def diff_expect(tool: str, expect: JsonDict, response: object) -> list[Change]: + """Check declared value pins (``expect`` on a probe) against a live response. + + Shapes can't see a value lie — dollars rescaled to cents, USD quietly converted — + so a pin makes the exact value part of the contract. Comparison is exact equality, + no tolerance. A pinned field that is missing or unequal is an output-side *silent* + failure (the agent reads the wrong value confidently), so every mismatch is + BREAKING — deterministic, unlike the advisory judge. + """ + resp: JsonDict = response if isinstance(response, dict) else {} + changes: list[Change] = [] + for name in sorted(expect): + if name not in resp: + changes.append(Change( + tool, "behavior", name, "value_pin_missing", "breaking", + f"probe {tool}: pinned field '{name}' missing from response", + )) + elif resp[name] != expect[name]: + changes.append(Change( + tool, "behavior", name, "value_pin_mismatch", "breaking", + f"probe {tool}: pinned field '{name}' expected {expect[name]!r}, " + f"got {resp[name]!r}", + )) + return changes diff --git a/docs/specs/2026-07-03-covenant-layer3-behavioral-probes-design.md b/docs/specs/2026-07-03-covenant-layer3-behavioral-probes-design.md index 0c4a316..768cb8e 100644 --- a/docs/specs/2026-07-03-covenant-layer3-behavioral-probes-design.md +++ b/docs/specs/2026-07-03-covenant-layer3-behavioral-probes-design.md @@ -22,8 +22,12 @@ A **probe** is a user-defined safe example call — `[[probes]]` in `covenant.to Layer 0 classifier**. Responses are output-side by definition, so the direction principle applies unchanged: lost field / structural retype / gains-null = BREAKING, scalar retype = DEGRADED, additions = COMPATIBLE. Changes render with location `behavior`. +- A probe may declare **value pins** — `expect = { field = value }` (v0.1.1): exact output + values that are part of the contract, compared with exact equality on every check. Pins + live in `covenant.toml` only, never in the lock — declared truth, not observed state, so + adding a pin never requires re-snapshotting. - `covenant check --judge` additionally sends (tool description, args, baseline sample, - live response) to an LLM judge for **semantic** drift that shape can't see (dollars→cents). + live response) to an LLM judge for **semantic** drift in fields too volatile to pin. ## Fingerprint rules (locked) @@ -43,6 +47,7 @@ A **probe** is a user-defined safe example call — `[[probes]]` in `covenant.to |---|---|---| | Live response loses a field / structural retype / gains null | BREAKING | Same silent-lie class as schema output changes — classifier reused verbatim | | Probe now returns `isError` | DEGRADED | Loud: the agent sees the error. Direction principle. | +| Pinned field missing or unequal | BREAKING | Deterministic and user-declared — no false-positive class, unlike the judge. Schema and shape still match while the value lies: the exact silent failure quarantine exists for. | | Judge suspects semantic drift | DEGRADED | The detector is probabilistic; a false BREAKING is a self-inflicted quarantine outage. Same precedent as the composition punt: flag for review, never auto-break. | | Probe in covenant.toml but not in the lock | error, exit 2 | A baseline mismatch is a config state, not drift — re-snapshot. | @@ -61,9 +66,11 @@ legitimately changes `sample` between snapshots — probe stable read-only tools Layer 0) renames `amount_usd`→`amount_cents` in the response body only. Schema check stays clean; the probe catches BREAKING. CI-runnable, no API key. - `COVENANT_SEMANTIC_DRIFT=1` — `get_account` returns the balance ×100. Schema and shape - identical; only `--judge` catches it. + identical; the committed `expect` pin catches it deterministically (BREAKING, exit 1, + CI-runnable, no API key). `--judge` also flags it, and covers the unpinned fields. ## Out of scope -Automatic probe generation, value-level assertions (snapshot testing), judging live -traffic at the proxy (cost — Layer 3 judges probes on demand). +Automatic probe generation; *auto-generated* value pins (snapshot-testing style — a pin +the user didn't type is a pin they won't trust) and tolerance/regex matchers on pins; +judging live traffic at the proxy (cost — Layer 3 judges probes on demand). diff --git a/pyproject.toml b/pyproject.toml index b8ab881..262dbbe 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "covenant-mcp" -version = "0.1.0" +version = "0.1.1" description = "A contract linter for MCP servers — catch breaking tool-schema changes before they ship." readme = "README.md" license = "MIT" diff --git a/tests/test_cli.py b/tests/test_cli.py index e48665c..3d2315f 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -81,6 +81,23 @@ def test_probes_catch_behavioral_drift(tmp_path, monkeypatch): assert "behavior" in r.output +def test_pins_catch_value_drift(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + _write_toml(tmp_path, '\n[[probes]]\ntool = "get_account"\n' + 'args = { account_id = "acct-001" }\n' + 'expect = { balance_usd = 4210.0, currency = "USD" }\n') + r = runner.invoke(app, ["snapshot"]) + assert r.exit_code == 0, r.output + r = runner.invoke(app, ["check"]) + assert r.exit_code == 0, r.output + + monkeypatch.setenv("COVENANT_SEMANTIC_DRIFT", "1") # same schema, same shape, value x100 + r = runner.invoke(app, ["check"]) + assert r.exit_code == 1, r.output + assert "balance_usd" in r.output + assert "BREAKING" in r.output + + def test_probe_missing_from_baseline_errors(tmp_path, monkeypatch): monkeypatch.chdir(tmp_path) _write_toml(tmp_path) diff --git a/tests/test_probes.py b/tests/test_probes.py index a254cd5..a994842 100644 --- a/tests/test_probes.py +++ b/tests/test_probes.py @@ -4,7 +4,7 @@ from covenant.config import load_config from covenant.contract import read_baseline, write_baseline -from covenant.diff import diff_probes +from covenant.diff import diff_expect, diff_probes from covenant.errors import ConfigError from covenant.fingerprint import fingerprint @@ -49,10 +49,32 @@ def test_scalar_retype_is_degraded(): def test_value_change_same_shape_is_clean(): base = [_probe("t", {"balance": 42.5})] - live = [_live("t", {"balance": 4250.0})] # cents pun: invisible to shape, the judge's job + live = [_live("t", {"balance": 4250.0})] # cents pun: invisible to shape — pin it, or judge it assert diff_probes(base, live) == [] +def test_pin_match_is_clean(): + # exact equality, but TOML ints must match float responses (1350 == 1350.0) + assert diff_expect("t", {"balance": 1350, "currency": "USD"}, + {"balance": 1350.0, "currency": "USD", "extra": "x"}) == [] + + +def test_pin_value_mismatch_is_breaking(): + (c,) = diff_expect("t", {"balance": 42.5}, {"balance": 4250.0}) + assert (c.kind, c.tier, c.location) == ("value_pin_mismatch", "breaking", "behavior") + assert "42.5" in c.message and "4250.0" in c.message + + +def test_pin_missing_field_is_breaking(): + (c,) = diff_expect("t", {"balance": 42.5}, {"amount": 42.5}) + assert (c.kind, c.tier) == ("value_pin_missing", "breaking") + + +def test_pin_on_non_dict_response_is_breaking(): + (c,) = diff_expect("t", {"balance": 42.5}, [42.5]) + assert c.kind == "value_pin_missing" + + def test_probe_error_is_degraded_and_loud(): base = [_probe("t", {"ok": 1})] live = [_live("t", None, is_error=True, error="boom")] @@ -99,3 +121,25 @@ def test_config_rejects_probe_without_tool(tmp_path): cfg_file.write_text('[server]\ncommand = "x"\n[[probes]]\nargs = { a = 1 }\n', encoding="utf-8") with pytest.raises(ConfigError): load_config(cfg_file) + + +def test_config_parses_expect_and_defaults_empty(tmp_path): + cfg_file = tmp_path / "covenant.toml" + cfg_file.write_text( + '[server]\ncommand = "x"\n' + '[[probes]]\ntool = "t"\nexpect = { balance = 4210.0 }\n' + '[[probes]]\ntool = "u"\n', + encoding="utf-8", + ) + cfg = load_config(cfg_file) + assert cfg.probes[0].expect == {"balance": 4210.0} + assert cfg.probes[1].expect == {} # opt-in: no pins unless the user types them + + +def test_config_rejects_non_table_expect(tmp_path): + cfg_file = tmp_path / "covenant.toml" + cfg_file.write_text( + '[server]\ncommand = "x"\n[[probes]]\ntool = "t"\nexpect = 42\n', encoding="utf-8" + ) + with pytest.raises(ConfigError): + load_config(cfg_file)