diff --git a/README.md b/README.md index 350a59f..63cf32b 100644 --- a/README.md +++ b/README.md @@ -94,7 +94,7 @@ tool = "get_transactions" args = { account_id = "acct-001" } ``` -`covenant snapshot` runs each probe and stores its response **fingerprint** — the type shape of what actually came back, never the values, which legitimately change — in the lock. `covenant check` re-runs the probes and classifies shape drift with the same severity model, at location `behavior`: +`covenant snapshot` runs each probe and stores its response **fingerprint** — the type shape of what actually came back — in the lock, alongside one **sample** response so the optional `--judge` pass has a baseline to compare meaning against. Shape drift is judged on the fingerprint alone; values legitimately change between runs. `covenant check` re-runs the probes and classifies shape drift with the same severity model, at location `behavior`: ```bash $ COVENANT_BEHAVIOR_DRIFT=1 covenant check # response body renames a field; schema untouched diff --git a/covenant/cli.py b/covenant/cli.py index dfd1664..bcc4466 100644 --- a/covenant/cli.py +++ b/covenant/cli.py @@ -62,14 +62,16 @@ def _check_probes( from .judge import judge_probe # the [judge] extra is optional; import on use descriptions = {t["name"]: t.get("description") for t in base_tools} - shape_drifted = {c.tool for c in changes} + # A probe is identified by tool + args, so judge each live probe on its own + # merits: skip only the probes that themselves errored or shape-drifted, never + # a clean probe that happens to share a tool name with a drifted sibling. for r in live: - if r["is_error"] or r["tool"] in shape_drifted: + base = base_by[probe_key(r["tool"], r["args"])] # every live key is baselined (see above) + if r["is_error"] or diff_probes([base], [r]): continue # errors and shape drift are already reported; judge only clean shapes - sample = base_by[probe_key(r["tool"], r["args"])].get("sample") verdict = judge_probe( r["tool"], descriptions.get(r["tool"]), r["args"], - sample, r["response"], model=cfg.judge_model, + base.get("sample"), r["response"], model=cfg.judge_model, ) if verdict.drift: changes.append(Change( diff --git a/covenant/config.py b/covenant/config.py index 8c1cc94..2eefcdc 100644 --- a/covenant/config.py +++ b/covenant/config.py @@ -77,7 +77,10 @@ def load_config(path: str | Path = "covenant.toml", server_override: str | None if not command and not url: raise ConfigError("no server configured: set [server].command or .url, or pass --server") - judge_model = (data.get("judge") or {}).get("model") + judge = data.get("judge") or {} + if not isinstance(judge, dict): + raise ConfigError("[judge] must be a table") + judge_model = judge.get("model") if judge_model is not None and not isinstance(judge_model, str): raise ConfigError("[judge].model must be a string") diff --git a/covenant/fingerprint.py b/covenant/fingerprint.py index 0e965f9..67b93b0 100644 --- a/covenant/fingerprint.py +++ b/covenant/fingerprint.py @@ -15,6 +15,7 @@ import json from ._types import JsonDict +from .errors import CovenantError def fingerprint(value: object) -> JsonDict: @@ -33,7 +34,7 @@ def fingerprint(value: object) -> JsonDict: if shapes and all(s == shapes[0] for s in shapes): return {"type": "array", "items": shapes[0]} return {"type": "array"} - raise ValueError(f"cannot fingerprint non-JSON value of type {type(value).__name__}") + raise CovenantError(f"cannot fingerprint non-JSON value of type {type(value).__name__}") def probe_key(tool: str, args: JsonDict | None) -> str: diff --git a/covenant/introspect.py b/covenant/introspect.py index 66f2da0..41fe81b 100644 --- a/covenant/introspect.py +++ b/covenant/introspect.py @@ -92,7 +92,10 @@ async def _run_probes(config: Config, probes: list[Probe]) -> list[JsonDict]: records: list[JsonDict] = [] async with _session(config) as session: for probe in probes: - result = await session.call_tool(probe.tool, probe.args) + try: + result = await session.call_tool(probe.tool, probe.args) + except Exception as e: # noqa: BLE001 - name the probe so a config typo isn't a "connection" error + raise CovenantError(f"probe {probe.tool} failed: {e}") from e response, is_error, error = _resolve_result(result) records.append({ "tool": probe.tool, diff --git a/covenant/judge/__init__.py b/covenant/judge/__init__.py index b92f17a..a45aab9 100644 --- a/covenant/judge/__init__.py +++ b/covenant/judge/__init__.py @@ -110,6 +110,9 @@ def judge_probe( raw = raw.strip("`").removeprefix("json").strip() try: data = json.loads(raw) - return Verdict(drift=bool(data["drift"]), reason=str(data["reason"])) + drift = data["drift"] + if not isinstance(drift, bool): + raise TypeError(f"drift must be a boolean, got {type(drift).__name__}") + return Verdict(drift=drift, reason=str(data["reason"])) except (json.JSONDecodeError, KeyError, TypeError) as e: raise CovenantError(f"judge returned an unparseable verdict: {raw!r}") from e diff --git a/tests/test_fingerprint.py b/tests/test_fingerprint.py index b475adf..3688f29 100644 --- a/tests/test_fingerprint.py +++ b/tests/test_fingerprint.py @@ -2,6 +2,7 @@ import pytest +from covenant.errors import CovenantError from covenant.fingerprint import fingerprint, probe_key @@ -30,7 +31,7 @@ def test_arrays_keep_items_only_when_uniform(): def test_non_json_value_is_loud(): - with pytest.raises(ValueError): + with pytest.raises(CovenantError): fingerprint({1, 2})