diff --git a/ACCEPTANCE_GATES.md b/ACCEPTANCE_GATES.md index 80bf71b..807a5b7 100644 --- a/ACCEPTANCE_GATES.md +++ b/ACCEPTANCE_GATES.md @@ -95,24 +95,27 @@ jobs: - name: Generate evidence bundle run: python scripts/evidence_bundle.py - name: Verify gates - run: python scripts/verify_gates.py + run: python scripts/verify_gates.py --evidence-dir evidence_bundle/ --json ``` ## Verification +The checked-in validator enforces the machine-readable report contract before +any release decision is made: + ```bash -# Full verification -python scripts/verify_gates.py --evidence-dir evidence_bundle/ - -# Quick check -python -c " -import json -with open('evidence_bundle/gates_status.json') as f: - gates = json.load(f) -failed = [k for k, v in gates.items() if v['status'] != 'PASS'] -if failed: - print(f'FAILED: {failed}') - exit(1) -print('ALL GATES PASS') -" +python scripts/verify_gates.py --evidence-dir evidence_bundle/ --json ``` + +`evidence_bundle/gates_status.json` must contain schema version `1`, the +audited repository and commit, and exactly one entry for every `G1.1` through +`G7.4` criterion. Each entry has a `PASS`, `FAIL`, or `BLOCKED` status and at +least one evidence-root-relative regular-file evidence path. Missing, unknown, +duplicate, absolute, traversal, symlinked, or malformed report/evidence causes +verification to fail. A report containing any `FAIL` or `BLOCKED` gate also +exits nonzero. The validator is intentionally evidence-neutral: it checks the +report and referenced artifacts, but never invents evidence or upgrades a +missing criterion. + +See `scripts/verify_gates.py` and `tests/test_verify_gates.py` for the +versioned contract and fail-closed cases. diff --git a/scripts/verify_gates.py b/scripts/verify_gates.py new file mode 100644 index 0000000..b78cf83 --- /dev/null +++ b/scripts/verify_gates.py @@ -0,0 +1,211 @@ +#!/usr/bin/env python3 +"""Validate a complete, local release acceptance-gate evidence report. + +The verifier is intentionally stdlib-only and fail-closed. It validates the +report structure and confirms that every referenced evidence file exists below +the caller-supplied evidence directory. It never turns a missing artifact or +an unknown gate into a pass. +""" + +from __future__ import annotations + +import argparse +import json +import stat +import sys +from pathlib import Path, PurePosixPath +from typing import Any + +SCHEMA_VERSION = 1 +REPOSITORY = "mrnicholasbcarter-code/verdict-core" +REPORT_NAME = "gates_status.json" +STATUSES = frozenset({"PASS", "FAIL", "BLOCKED"}) + +# Keep this list aligned with the G1-G7 IDs in ACCEPTANCE_GATES.md. The +# verifier deliberately requires the complete set rather than accepting a +# partial report that could be mistaken for a release decision. +GATE_IDS = tuple( + gate_id + for category, count in ((1, 5), (2, 4), (3, 4), (4, 4), (5, 4), (6, 4), (7, 4)) + for gate_id in (f"G{category}.{criterion}" for criterion in range(1, count + 1)) +) + + +class GateValidationError(ValueError): + """Raised when a report or its evidence is unsafe or incomplete.""" + + +def _canonical_json(value: object) -> bytes: + return ( + json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":")) + "\n" + ).encode("utf-8") + + +def _reject_duplicate_keys(pairs: list[tuple[str, Any]]) -> dict[str, Any]: + result: dict[str, Any] = {} + for key, value in pairs: + if key in result: + raise GateValidationError(f"duplicate JSON key: {key}") + result[key] = value + return result + + +def _reject_constant(value: str) -> None: + raise GateValidationError(f"non-standard JSON constant: {value}") + + +def _load_canonical_json(path: Path) -> dict[str, Any]: + try: + raw = path.read_bytes() + except OSError as exc: + raise GateValidationError(f"cannot read {REPORT_NAME}") from exc + if path.is_symlink(): + raise GateValidationError(f"{REPORT_NAME} must not be a symlink") + try: + value = json.loads( + raw.decode("utf-8"), + object_pairs_hook=_reject_duplicate_keys, + parse_constant=_reject_constant, + ) + except (UnicodeDecodeError, json.JSONDecodeError, GateValidationError) as exc: + raise GateValidationError(f"{REPORT_NAME} is malformed JSON") from exc + if not isinstance(value, dict): + raise GateValidationError(f"{REPORT_NAME} must contain a JSON object") + if raw != _canonical_json(value): + raise GateValidationError(f"{REPORT_NAME} is not canonical JSON") + return value + + +def _validate_evidence_root(value: Path) -> Path: + root = Path(value) + if root.is_symlink() or not root.is_dir(): + raise GateValidationError("evidence directory must be a real directory") + return root + + +def _validate_evidence_path(root: Path, value: object) -> str: + if not isinstance(value, str) or not value or "\\" in value: + raise GateValidationError("evidence paths must be repository-relative POSIX paths") + relative = PurePosixPath(value) + if ( + relative.is_absolute() + or relative.as_posix() != value + or any(part in {"", ".", ".."} for part in relative.parts) + or value == REPORT_NAME + ): + raise GateValidationError(f"unsafe evidence path: {value!r}") + + current = root + for part in relative.parts: + current /= part + try: + mode = current.lstat().st_mode + except OSError as exc: + raise GateValidationError(f"evidence file is missing: {value}") from exc + if stat.S_ISLNK(mode): + raise GateValidationError(f"evidence path contains a symlink: {value}") + if not stat.S_ISREG(mode): + raise GateValidationError(f"evidence path is not a regular file: {value}") + return value + + +def _validate_report(root: Path, report: dict[str, Any]) -> dict[str, Any]: + expected_fields = {"schema_version", "repository", "commit", "gates"} + if set(report) != expected_fields: + raise GateValidationError( + "report fields must be exactly schema_version, repository, commit, gates" + ) + if report.get("schema_version") != SCHEMA_VERSION: + raise GateValidationError("unsupported report schema_version") + if report.get("repository") != REPOSITORY: + raise GateValidationError("report repository does not match this project") + commit = report.get("commit") + if ( + not isinstance(commit, str) + or len(commit) != 40 + or any(character not in "0123456789abcdef" for character in commit) + ): + raise GateValidationError("report commit must be a full lowercase git SHA") + + gates = report.get("gates") + if not isinstance(gates, list) or [ + item.get("id") for item in gates if isinstance(item, dict) + ] != list(GATE_IDS): + raise GateValidationError("report must contain every gate ID in documented order") + + counts = {status: 0 for status in STATUSES} + for index, item in enumerate(gates): + if not isinstance(item, dict) or set(item) != {"id", "status", "evidence"}: + raise GateValidationError(f"gate {index} has unexpected or missing fields") + gate_id = item["id"] + status = item["status"] + evidence = item["evidence"] + if not isinstance(gate_id, str) or gate_id != GATE_IDS[index]: + raise GateValidationError(f"gate {index} has an invalid ID") + if not isinstance(status, str) or status not in STATUSES: + raise GateValidationError(f"{gate_id} has an invalid status") + if not isinstance(evidence, list) or not evidence: + raise GateValidationError(f"{gate_id} must cite at least one evidence file") + seen: set[str] = set() + for evidence_path in evidence: + normalized = _validate_evidence_path(root, evidence_path) + if normalized in seen: + raise GateValidationError(f"{gate_id} cites duplicate evidence: {normalized}") + seen.add(normalized) + counts[status] += 1 + + return { + "valid": True, + "schema_version": SCHEMA_VERSION, + "repository": REPOSITORY, + "commit": commit, + "gate_count": len(gates), + "passed": counts["PASS"], + "failed": counts["FAIL"], + "blocked": counts["BLOCKED"], + "all_passed": counts["PASS"] == len(GATE_IDS), + } + + +def validate_gates(evidence_dir: Path) -> dict[str, Any]: + """Validate ``gates_status.json`` and all referenced evidence files.""" + + root = _validate_evidence_root(evidence_dir) + report = _load_canonical_json(root / REPORT_NAME) + return _validate_report(root, report) + + +def _emit(summary: dict[str, Any], *, as_json: bool) -> None: + if as_json: + print(json.dumps(summary, sort_keys=True, separators=(",", ":"))) + else: + state = "PASS" + if summary["failed"]: + state = "FAIL" + elif summary["blocked"]: + state = "BLOCKED" + print( + f"{state}: validated {summary['gate_count']} gates " + f"({summary['passed']} pass, {summary['failed']} fail, {summary['blocked']} blocked)" + ) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--evidence-dir", type=Path, required=True) + parser.add_argument("--json", action="store_true", dest="as_json") + args = parser.parse_args(argv) + try: + summary = validate_gates(args.evidence_dir) + except GateValidationError as exc: + if args.as_json: + print(json.dumps({"valid": False, "error": str(exc)}, sort_keys=True)) + else: + print(f"INVALID: {exc}", file=sys.stderr) + return 1 + _emit(summary, as_json=args.as_json) + return 0 if summary["all_passed"] else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_verify_gates.py b/tests/test_verify_gates.py new file mode 100644 index 0000000..98e3c0f --- /dev/null +++ b/tests/test_verify_gates.py @@ -0,0 +1,154 @@ +"""Tests for the fail-closed release acceptance-gate verifier.""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from scripts.verify_gates import GATE_IDS, REPORT_NAME, GateValidationError, main, validate_gates + + +def _write_report(root: Path, *, statuses: dict[str, str] | None = None) -> None: + evidence: dict[str, list[str]] = {} + for gate_id in GATE_IDS: + path = f"checks/{gate_id}.txt" + evidence[gate_id] = [path] + target = root / path + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(f"evidence for {gate_id}\n", encoding="utf-8") + report = { + "schema_version": 1, + "repository": "mrnicholasbcarter-code/verdict-core", + "commit": "0" * 40, + "gates": [ + { + "id": gate_id, + "status": (statuses or {}).get(gate_id, "PASS"), + "evidence": evidence[gate_id], + } + for gate_id in GATE_IDS + ], + } + (root / REPORT_NAME).write_bytes( + (json.dumps(report, sort_keys=True, separators=(",", ":")) + "\n").encode("utf-8") + ) + + +def test_complete_report_and_evidence_pass(tmp_path: Path) -> None: + _write_report(tmp_path) + + summary = validate_gates(tmp_path) + + assert summary["gate_count"] == len(GATE_IDS) == 29 + assert summary["passed"] == 29 + assert summary["all_passed"] is True + + +def test_json_cli_is_machine_readable(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: + _write_report(tmp_path) + + assert main(["--evidence-dir", str(tmp_path), "--json"]) == 0 + output = json.loads(capsys.readouterr().out) + + assert output["all_passed"] is True + assert output["valid"] is True + assert output["gate_count"] == 29 + + +def test_non_pass_gate_is_valid_but_blocks_release(tmp_path: Path) -> None: + _write_report(tmp_path, statuses={"G3.2": "BLOCKED"}) + + assert main(["--evidence-dir", str(tmp_path)]) == 1 + assert validate_gates(tmp_path)["all_passed"] is False + + +def test_json_cli_reports_invalid_input_without_claiming_a_pass( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + (tmp_path / REPORT_NAME).write_text("{not-json", encoding="utf-8") + + assert main(["--evidence-dir", str(tmp_path), "--json"]) == 1 + output = json.loads(capsys.readouterr().out) + + assert output["valid"] is False + assert "malformed" in output["error"] + + +@pytest.mark.parametrize( + ("mutation", "message"), + [ + (lambda report: report["gates"].pop(), "every gate ID"), + ( + lambda report: report["gates"].__setitem__( + 0, {"id": "G9.1", "status": "PASS", "evidence": ["checks/G1.1.txt"]} + ), + "every gate ID", + ), + ( + lambda report: report["gates"].__setitem__( + 0, {"id": "G1.1", "status": "PASS", "evidence": ["../outside.txt"]} + ), + "unsafe evidence path", + ), + ( + lambda report: report["gates"].__setitem__( + 0, {"id": "G1.1", "status": "PASS", "evidence": ["checks/missing.txt"]} + ), + "missing", + ), + ( + lambda report: report["gates"].__setitem__( + 0, {"id": "G1.1", "status": "PASS", "evidence": []} + ), + "at least one", + ), + ( + lambda report: report["gates"].__setitem__( + 0, + { + "id": "G1.1", + "status": "PASS", + "evidence": ["checks/G1.1.txt", "checks/G1.1.txt"], + }, + ), + "duplicate evidence", + ), + ( + lambda report: report["gates"].__setitem__( + 0, {"id": "G1.1", "status": "UNKNOWN", "evidence": ["checks/G1.1.txt"]} + ), + "invalid status", + ), + ], +) +def test_invalid_report_fails_closed(tmp_path: Path, mutation, message: str) -> None: + _write_report(tmp_path) + report_path = tmp_path / REPORT_NAME + report = json.loads(report_path.read_text(encoding="utf-8")) + mutation(report) + report_path.write_text( + json.dumps(report, sort_keys=True, separators=(",", ":")) + "\n", encoding="utf-8" + ) + + with pytest.raises(GateValidationError, match=message): + validate_gates(tmp_path) + + +def test_noncanonical_report_and_symlink_evidence_are_rejected(tmp_path: Path) -> None: + _write_report(tmp_path) + report_path = tmp_path / REPORT_NAME + report = json.loads(report_path.read_text(encoding="utf-8")) + report_path.write_text(json.dumps(report, indent=2), encoding="utf-8") + with pytest.raises(GateValidationError, match="canonical"): + validate_gates(tmp_path) + + _write_report(tmp_path) + link = tmp_path / "checks" / "G1.1.txt" + real = link.with_name("real.txt") + link.unlink() + real.write_text("outside link target\n", encoding="utf-8") + link.symlink_to(real) + with pytest.raises(GateValidationError, match="symlink"): + validate_gates(tmp_path)