diff --git a/scripts/milestone1_conformance_report.py b/scripts/milestone1_conformance_report.py new file mode 100644 index 0000000..c78e21b --- /dev/null +++ b/scripts/milestone1_conformance_report.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 +"""Check that docs/MILESTONE1.md matches the trinity.m1 code. Zero API cost. + +MILESTONE1.md is the miner-facing contract — the ``< 1,000,000`` param budget, +``1``/day rate, ``≥ 0.02`` win margin, pack layout, and ``0.7·domain + +0.3·difficulty`` composite. Those numbers also live in ``trinity.m1.constants`` +and the pack/metrics code. Nothing checked the two agree; a doc edit or a +constant change could split them, and a miner following the document would then +build a pack the validator rejects. + + python scripts/milestone1_conformance_report.py + python scripts/milestone1_conformance_report.py --doc docs/MILESTONE1.md --json + +Exits 1 on any mismatch, 0 when doc and code agree, 2 when the document cannot +be read or parsed. +""" +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + +_REPO = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(_REPO / "src")) + +from trinity.m1.conformance import check, default_doc_path, render # noqa: E402 + + +def main(argv: list[str] | None = None) -> int: + """Print the conformance report; exit non-zero on mismatch.""" + ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + ap.add_argument("--doc", type=Path, default=None, + help="path to MILESTONE1.md (default: the repo's docs/MILESTONE1.md)") + ap.add_argument("--json", action="store_true", dest="as_json", help="emit JSON") + args = ap.parse_args(argv) + + doc_path = args.doc if args.doc is not None else default_doc_path() + try: + report = check(doc_path=doc_path) + except FileNotFoundError: + print(f"no such file: {doc_path}", file=sys.stderr) + return 2 + except ValueError as exc: + print(f"could not parse MILESTONE1.md: {exc}", file=sys.stderr) + return 2 + + print(json.dumps(report.to_dict(), indent=2) if args.as_json else render(report)) + return 0 if report.ok else 1 + + +if __name__ == "__main__": # pragma: no cover + raise SystemExit(main()) diff --git a/src/trinity/m1/conformance.py b/src/trinity/m1/conformance.py new file mode 100644 index 0000000..ae18bc6 --- /dev/null +++ b/src/trinity/m1/conformance.py @@ -0,0 +1,268 @@ +"""Verify the Milestone-1 code matches the rules published in ``docs/MILESTONE1.md``. + +MILESTONE1.md is the miner-facing contract: it tabulates the submission limits +(``< 1,000,000`` params, ``1`` per day, win margin ``≥ 0.02``), the pack layout +(which files land under ``submissions//m1/``), and the composite formula +(``0.7·domain + 0.3·difficulty``). Those numbers also live independently in +``trinity.m1.constants`` and the pack/metrics code. Nothing checks that the two +agree — a doc edit or a constant change could silently split them, and a miner +following the document would then build a pack the validator rejects (or vice +versa). + +This parses the document and asserts it against the code: + +* **rules** — the published limits equal ``constants.MAX_HEAD_PARAMS`` / + ``RATE_LIMIT_*`` / ``WIN_MARGIN``; +* **scoring** — the published composite weights equal ``constants.DOMAIN_WEIGHT`` + (and the ``TriageMetrics.composite`` docstring); +* **pack layout** — every file the document lists is one the packer actually + writes, and every file the packer can write is documented (a file the code + emits but the doc omits is reported, not silently accepted). + +Pure stdlib. No torch, no network — ``pack.save_milestone1_pack`` is numpy-only, +and this only needs the *set* of files it can write, encoded in +:data:`CODE_PACK_FILES`, cross-checked against it by the tests. +""" +from __future__ import annotations + +import re +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +from trinity.m1.constants import ( + DOMAIN_WEIGHT, + MAX_HEAD_PARAMS, + RATE_LIMIT_MAX_SUBMISSIONS, + RATE_LIMIT_WINDOW_DAYS, + WIN_MARGIN, +) + +__all__ = [ + "CODE_PACK_FILES", + "ConformanceReport", + "Finding", + "check", + "default_doc_path", + "parse_pack_layout", + "parse_rules", + "parse_scoring_weights", +] + +#: Files ``pack.save_milestone1_pack`` can write, and whether each is required. +#: ``tests/test_m1_conformance.py`` saves a real pack and asserts this set is +#: exactly what lands on disk, so it cannot drift from the packer. +CODE_PACK_FILES: dict[str, str] = { + "config.json": "required", + "W_domain.npy": "required", + "W_diff.npy": "required", + "attention_query.npy": "optional (pool=attentive)", + "W_k.npy": "optional (attentive + key projection)", +} + +_REQUIRED_PACK_FILES = tuple(f for f, kind in CODE_PACK_FILES.items() if kind == "required") + + +def default_doc_path() -> Path: + """``docs/MILESTONE1.md`` relative to the installed package.""" + return Path(__file__).resolve().parents[3] / "docs" / "MILESTONE1.md" + + +def _section(md: str, heading: str) -> str: + """Return the body between ``## heading`` and the next ``## `` heading.""" + m = re.search( + rf"^##\s+{re.escape(heading)}\s*$(.*?)(?=^##\s|\Z)", md, re.S | re.M + ) + if m is None: + raise ValueError(f"MILESTONE1.md section '## {heading}' not found") + return m.group(1) + + +def _int(text: str) -> int: + """First integer in ``text``, commas/thin-spaces allowed (``1,000,000``).""" + m = re.search(r"\d[\d,\s]*", text) + if m is None: + raise ValueError(f"no integer in {text!r}") + return int(re.sub(r"[,\s]", "", m.group(0))) + + +def parse_rules(md: str) -> dict[str, Any]: + """Extract the documented submission limits from the ``## Rules`` table. + + Returns keys ``max_params`` (int), ``rate_per_day`` (int) and ``win_margin`` + (float). + + Raises: + ValueError: if the section or any of the three rows is missing. + """ + body = _section(md, "Rules") + rows = { + c[0].strip().strip("*").strip().lower(): c[1].strip() + for line in body.splitlines() + if line.strip().startswith("|") and not line.strip().startswith("|--") + for c in [ [x.strip() for x in line.strip().strip("|").split("|")] ] + if len(c) == 2 + } + try: + size = rows["size"] + rate = rows["rate"] + margin = rows["win margin"] + except KeyError as e: + raise ValueError(f"MILESTONE1.md Rules table missing row: {e}") from None + + win = re.search(r"[\d.]+", margin) + if win is None: + raise ValueError(f"no win-margin number in {margin!r}") + per_day = "day" in rate.lower() + return { + "max_params": _int(size), + "rate_per_day": _int(rate) if per_day else None, + "win_margin": float(win.group(0)), + } + + +def parse_scoring_weights(md: str) -> tuple[float, float]: + """Return ``(domain_weight, difficulty_weight)`` from the ``## Scoring`` block.""" + body = _section(md, "Scoring") + nums = re.findall(r"([\d.]+)\s*[×x*]\s*(?:domain|difficulty)", body) + if len(nums) < 2: + raise ValueError(f"could not parse composite weights from: {body!r}") + return float(nums[0]), float(nums[1]) + + +def parse_pack_layout(md: str) -> list[str]: + """Filenames listed in the ``## Pack layout`` fenced block, in order.""" + body = _section(md, "Pack layout") + fence = re.search(r"```(.*?)```", body, re.S) + if fence is None: + raise ValueError("MILESTONE1.md Pack layout has no fenced block") + files: list[str] = [] + for line in fence.group(1).splitlines(): + tok = line.strip().split() + if tok and re.fullmatch(r"[\w.\-]+\.(npy|json)", tok[0]): + files.append(tok[0]) + if not files: + raise ValueError("no pack files found in the Pack layout block") + return files + + +@dataclass(frozen=True) +class Finding: + """One conformance check outcome.""" + + check: str + ok: bool + detail: str + + def to_dict(self) -> dict[str, Any]: + return {"check": self.check, "ok": self.ok, "detail": self.detail} + + +@dataclass(frozen=True) +class ConformanceReport: + findings: list[Finding] = field(default_factory=list) + notes: list[str] = field(default_factory=list) + + @property + def ok(self) -> bool: + return all(f.ok for f in self.findings) + + @property + def failures(self) -> list[Finding]: + return [f for f in self.findings if not f.ok] + + def to_dict(self) -> dict[str, Any]: + return { + "ok": self.ok, + "findings": [f.to_dict() for f in self.findings], + "notes": list(self.notes), + } + + +def check(md: str | None = None, *, doc_path: Path | str | None = None) -> ConformanceReport: + """Compare ``docs/MILESTONE1.md`` against ``trinity.m1`` code. + + Args: + md: document contents; when ``None`` read from ``doc_path`` or + :func:`default_doc_path`. + doc_path: where to read the document from. + + Raises: + ValueError: if a documented section cannot be parsed. + FileNotFoundError: if the document file is missing. + """ + if md is None: + path = Path(doc_path) if doc_path is not None else default_doc_path() + if not path.exists(): + raise FileNotFoundError(str(path)) + md = path.read_text() + + rules = parse_rules(md) + dom_w, diff_w = parse_scoring_weights(md) + doc_files = parse_pack_layout(md) + + findings: list[Finding] = [] + notes: list[str] = [] + + findings.append(Finding( + "param_budget", + rules["max_params"] == MAX_HEAD_PARAMS, + f"doc says < {rules['max_params']:,}; constants.MAX_HEAD_PARAMS = {MAX_HEAD_PARAMS:,}", + )) + findings.append(Finding( + "rate_limit", + rules["rate_per_day"] == RATE_LIMIT_MAX_SUBMISSIONS and RATE_LIMIT_WINDOW_DAYS == 1, + f"doc says {rules['rate_per_day']}/day; constants = " + f"{RATE_LIMIT_MAX_SUBMISSIONS} per {RATE_LIMIT_WINDOW_DAYS} day(s)", + )) + findings.append(Finding( + "win_margin", + abs(rules["win_margin"] - WIN_MARGIN) < 1e-12, + f"doc says >= {rules['win_margin']}; constants.WIN_MARGIN = {WIN_MARGIN}", + )) + findings.append(Finding( + "composite_weights", + abs(dom_w - DOMAIN_WEIGHT) < 1e-12 and abs(diff_w - (1.0 - DOMAIN_WEIGHT)) < 1e-12, + f"doc says {dom_w}·domain + {diff_w}·difficulty; " + f"constants.DOMAIN_WEIGHT = {DOMAIN_WEIGHT}", + )) + + # Pack layout: every documented file must be one the packer writes. + phantom = [f for f in doc_files if f not in CODE_PACK_FILES] + findings.append(Finding( + "pack_layout_no_phantom_files", + not phantom, + "documented files not produced by the packer: " + (", ".join(phantom) or "none"), + )) + missing_required = [f for f in _REQUIRED_PACK_FILES if f not in doc_files] + findings.append(Finding( + "pack_layout_documents_required_files", + not missing_required, + "required pack files absent from the doc: " + (", ".join(missing_required) or "none"), + )) + + # A file the code can write but the doc omits is a documentation gap, not a + # code defect — reported as a note rather than a failure. + undocumented = [f for f in CODE_PACK_FILES if f not in doc_files] + if undocumented: + notes.append( + "packer can also write " + ", ".join(undocumented) + + " (attentive packs); the Pack layout block does not list " + + ("it" if len(undocumented) == 1 else "them") + "." + ) + + return ConformanceReport(findings=findings, notes=notes) + + +def render(report: ConformanceReport) -> str: + lines = ["MILESTONE1.md ↔ trinity.m1 conformance"] + for f in report.findings: + lines.append(f" [{'ok' if f.ok else 'MISMATCH'}] {f.check}: {f.detail}") + for note in report.notes: + lines.append(f" note: {note}") + lines.append( + " verdict: doc and code agree" + if report.ok + else f" verdict: {len(report.failures)} MISMATCH(es)" + ) + return "\n".join(lines) diff --git a/tests/test_m1_conformance.py b/tests/test_m1_conformance.py new file mode 100644 index 0000000..cacca58 --- /dev/null +++ b/tests/test_m1_conformance.py @@ -0,0 +1,297 @@ +"""MILESTONE1.md ↔ trinity.m1 conformance + the offline miner workflow. + +Two things nothing else guards: + +* the miner-facing rules in ``docs/MILESTONE1.md`` (param budget, rate, win + margin, composite weights, pack layout) match ``trinity.m1``'s code; +* the offline ``pack → preflight`` path the document prescribes actually round- + trips, at the documented boundaries. + +``CODE_PACK_FILES`` in the conformance module is a hand-maintained list; the +``test_code_pack_files_matches_what_the_packer_writes`` test saves a real pack +and asserts the list is exactly what lands on disk, so it cannot drift. +""" +from __future__ import annotations + +import json +import os +import subprocess +import sys +from pathlib import Path + +import numpy as np +import pytest + +from trinity.m1.conformance import ( + CODE_PACK_FILES, + check, + default_doc_path, + parse_pack_layout, + parse_rules, + parse_scoring_weights, + render, +) +from trinity.m1.constants import ( + DOMAIN_WEIGHT, + MAX_HEAD_PARAMS, + RATE_LIMIT_MAX_SUBMISSIONS, + WIN_MARGIN, +) +from trinity.m1.gates import run_m1_gates +from trinity.m1.leaderboard import decide_m1_winner +from trinity.m1.pack import Milestone1Pack, load_milestone1_pack, save_milestone1_pack +from trinity.m1.domains import DOMAINS_5 + +_REPO = Path(__file__).resolve().parents[1] +_DOC = _REPO / "docs" / "MILESTONE1.md" +_SCRIPT = _REPO / "scripts" / "milestone1_conformance_report.py" +_SRC = str(_REPO / "src") + +D_H = 1024 + + +def _pack(config="5-domain", d_h=D_H, scale=0.01): + rng = np.random.default_rng(0) + n_dom = len(DOMAINS_5) if config == "5-domain" else 20 + return Milestone1Pack( + config=config, + pool="penultimate", + d_h=d_h, + W_domain=(rng.normal(size=(n_dom, d_h)) * scale).astype(np.float32), + W_diff=(rng.normal(size=(5, d_h)) * scale).astype(np.float32), + ) + + +# -------------------------------------------------------------------------- +# the document parses, and matches the code +# -------------------------------------------------------------------------- + + +def test_conformance_holds_on_the_shipped_doc(): + report = check() + assert report.ok, render(report) + + +def test_default_doc_path_points_at_the_repo_doc(): + assert default_doc_path() == _DOC + assert _DOC.exists() + + +def test_parse_rules_reads_the_three_limits(): + rules = parse_rules(_DOC.read_text()) + assert rules["max_params"] == MAX_HEAD_PARAMS == 1_000_000 + assert rules["rate_per_day"] == RATE_LIMIT_MAX_SUBMISSIONS == 1 + assert rules["win_margin"] == WIN_MARGIN == 0.02 + + +def test_parse_scoring_weights_matches_the_constant(): + dom, diff = parse_scoring_weights(_DOC.read_text()) + assert dom == DOMAIN_WEIGHT + assert dom + diff == pytest.approx(1.0) + + +def test_parse_pack_layout_lists_the_core_files(): + files = parse_pack_layout(_DOC.read_text()) + assert "config.json" in files + assert "W_domain.npy" in files + assert "W_diff.npy" in files + + +# -------------------------------------------------------------------------- +# drift is detected +# -------------------------------------------------------------------------- + + +def _doc_with(**subs): + md = _DOC.read_text() + for old, new in subs.items(): + md = md.replace(old, new) + return md + + +def test_a_changed_param_budget_in_the_doc_is_caught(): + report = check(md=_doc_with(**{"< 1,000,000": "< 2,000,000"})) + assert report.ok is False + assert any(f.check == "param_budget" for f in report.failures) + + +def test_a_changed_win_margin_in_the_doc_is_caught(): + report = check(md=_doc_with(**{"≥ 0.02": "≥ 0.05"})) + assert report.ok is False + assert any(f.check == "win_margin" for f in report.failures) + + +def test_a_changed_composite_weight_in_the_doc_is_caught(): + report = check(md=_doc_with(**{"0.7 × domain_accuracy + 0.3": "0.6 × domain_accuracy + 0.4"})) + assert report.ok is False + assert any(f.check == "composite_weights" for f in report.failures) + + +def test_a_phantom_pack_file_in_the_doc_is_caught(): + report = check(md=_doc_with(**{"W_diff.npy": "W_phantom.npy"})) + assert report.ok is False + assert any(f.check == "pack_layout_no_phantom_files" for f in report.failures) + + +def test_a_missing_section_is_a_parse_error(): + with pytest.raises(ValueError, match="Rules"): + parse_rules("# doc with no rules section\n") + + +# -------------------------------------------------------------------------- +# CODE_PACK_FILES cannot drift from the packer +# -------------------------------------------------------------------------- + + +def test_code_pack_files_matches_what_the_packer_writes(tmp_path): + """Save required + all-optional packs; the union of files must equal the list.""" + # penultimate pack -> the required files only + save_milestone1_pack(tmp_path / "req", _pack()) + req = {p.name for p in (tmp_path / "req").iterdir()} + assert req == set(f for f, k in CODE_PACK_FILES.items() if k == "required") + + # attentive pack with key projection -> every optional file too. Use a small + # d_h so W_k (d_h × d_h) stays well under the 1M param budget. + small = 100 + rng = np.random.default_rng(1) + full = Milestone1Pack( + config="5-domain", pool="attentive", d_h=small, + W_domain=(rng.normal(size=(5, small)) * 0.01).astype(np.float32), + W_diff=(rng.normal(size=(5, small)) * 0.01).astype(np.float32), + attention_query=(rng.normal(size=(small,)) * 0.01).astype(np.float32), + W_k=(rng.normal(size=(small, small)) * 0.001).astype(np.float32), + ) + save_milestone1_pack(tmp_path / "full", full) + allf = {p.name for p in (tmp_path / "full").iterdir()} + assert allf == set(CODE_PACK_FILES) + + +def test_the_wk_omission_is_reported_as_a_note(): + """W_k.npy is writable but absent from the doc's layout block.""" + notes = check().notes + assert any("W_k.npy" in n for n in notes) + + +# -------------------------------------------------------------------------- +# the offline pack -> preflight workflow (MILESTONE1.md steps 2-3) +# -------------------------------------------------------------------------- + + +def test_pack_round_trips_through_disk(tmp_path): + save_milestone1_pack(tmp_path / "m1", _pack()) + reloaded = load_milestone1_pack(tmp_path / "m1") + assert reloaded.config == "5-domain" + assert reloaded.n_params == 10_240 # doc: "≈ 10,240 params" + + +def test_documented_default_pack_size(): + """The doc states the default 5-domain penultimate pack ≈ 10,240 params.""" + assert _pack().n_params == 5 * D_H + 5 * D_H == 10_240 + + +def test_default_pack_passes_all_offline_gates(tmp_path): + save_milestone1_pack(tmp_path / "m1", _pack()) + pack = load_milestone1_pack(tmp_path / "m1") + results = run_m1_gates(pack, miner="alice", repo_root=tmp_path, skip_rate_limit=True) + assert all(r.ok for r in results), [r.reason for r in results if not r.ok] + + +def test_param_budget_boundary_is_exactly_the_documented_limit(): + """Doc says '< 1,000,000': 999,999 ok, 1,000,000 rejected.""" + # d_h chosen so 2 rows (domain rows == 5, diff == 5) won't hit 1M easily; + # instead assert the pack.validate boundary directly via a fabricated count. + from trinity.m1.pack import count_pack_params + + # A 5-domain penultimate pack has (n_dom + 5) * d_h params. Pick d_h so the + # count straddles the limit: (5+5)*d_h. At d_h=100_000 -> 1,000,000 exactly. + just_over = _pack(d_h=100_000) + assert count_pack_params(just_over) == 1_000_000 + with pytest.raises(ValueError, match="limit"): + just_over.validate() + + just_under = _pack(d_h=99_999) + assert count_pack_params(just_under) == 999_990 # < 1,000,000 + just_under.validate() # must not raise + + +def test_win_margin_decision_matches_the_documented_rule(): + # No king -> first challenger wins outright. + wins, _ = decide_m1_winner(king_composite=None, challenger_composite=0.10) + assert wins is True + # Exactly at king + margin -> wins (>= is documented). + wins, _ = decide_m1_winner(king_composite=0.80, challenger_composite=0.80 + WIN_MARGIN) + assert wins is True + # A hair under the margin -> loses. + wins, _ = decide_m1_winner(king_composite=0.80, challenger_composite=0.80 + WIN_MARGIN - 1e-6) + assert wins is False + + +# -------------------------------------------------------------------------- +# report + CLI +# -------------------------------------------------------------------------- + + +def test_report_serializes(): + payload = check().to_dict() + json.dumps(payload) + assert payload["ok"] is True + assert len(payload["findings"]) >= 6 + + +def test_render_reads_cleanly(): + assert "doc and code agree" in render(check()) + + +def _run(*args): + env = {**os.environ, "PYTHONPATH": _SRC + os.pathsep + os.environ.get("PYTHONPATH", "")} + return subprocess.run( + [sys.executable, str(_SCRIPT), *args], capture_output=True, text=True, env=env + ) + + +def test_cli_passes_on_the_real_doc(): + r = _run() + assert r.returncode == 0, r.stdout + r.stderr + assert "conformance" in r.stdout + + +def test_cli_json(): + r = _run("--json") + assert r.returncode == 0 + assert json.loads(r.stdout)["ok"] is True + + +def test_cli_flags_a_mismatched_doc(tmp_path): + doc = tmp_path / "MILESTONE1.md" + doc.write_text(_doc_with(**{"≥ 0.02": "≥ 0.09"})) + r = _run("--doc", str(doc)) + assert r.returncode == 1 + assert "MISMATCH" in r.stdout + + +def test_cli_missing_doc_is_graceful(tmp_path): + r = _run("--doc", str(tmp_path / "nope.md")) + assert r.returncode == 2 + assert "no such file" in r.stderr + + +def test_cli_unparseable_doc_is_graceful(tmp_path): + doc = tmp_path / "MILESTONE1.md" + doc.write_text("# nothing here\n") + r = _run("--doc", str(doc)) + assert r.returncode == 2 + assert "could not parse" in r.stderr + + +# -------------------------------------------------------------------------- +# import cost +# -------------------------------------------------------------------------- + + +def test_module_imports_without_torch(): + env = {**os.environ, "PYTHONPATH": _SRC + os.pathsep + os.environ.get("PYTHONPATH", "")} + code = ("import sys; import trinity.m1.conformance; " + "print('torch' in sys.modules)") + out = subprocess.run([sys.executable, "-c", code], capture_output=True, + text=True, check=True, env=env) + assert out.stdout.strip() == "False", out.stdout