diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ace46173..f5a9d6db 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -358,6 +358,12 @@ jobs: - name: Enforce Markdown links and heading anchors run: uv run python scripts/check_markdown_links.py + - name: Enforce renderable math and non-hollow sections + # correlation.md rendered broken math for three weeks because no gate + # read the delimiters, and pointer stubs satisfied the section-link + # check by resolving to a heading that carried nothing. + run: uv run python scripts/check_doc_rot.py + - name: Enforce generated synthetic terminal demo # README shows the real default renderer over a reserved no-network # fixture. Block stale screenshots when panel behavior changes. diff --git a/CHANGELOG.md b/CHANGELOG.md index 8e14f06b..09aeff03 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,6 +28,12 @@ operator, corporate group, ownership, or control. ### Added +- `scripts/check_doc_rot.py` is a new blocking gate stage rejecting Markdown + that renders silently wrong on github.com: LaTeX `\(` and `\[` delimiters the + renderer treats as escaped literal brackets, unclosed `$$` display blocks, and + sections hollowed out into pointers that redirect the reader without carrying + content. It runs in `scripts/check.py` and CI, and it found one live defect on + its first run. - `scripts/quality_scorecard.py` emits the Phase 1 product-quality baseline as one dated, revision-bound, aggregate-safe artifact. It is network-free and corpus-free, and it is a diagnostic rather than a gate. It measures the one @@ -49,6 +55,11 @@ operator, corporate group, ownership, or control. ### Fixed +- `docs/statistical-assurance.md` rendered its signed marginal entropy formula + as literal bracketed LaTeX rather than math, so github.com printed the raw + expression between plain brackets. Same defect class as the + `docs/correlation.md` repair, in a file that repair did not touch, found by + the new doc-rot gate on its first run. - The scheduled provider-drift gate no longer asserts that a reserved domain carries a third-party Microsoft 365 tenant. `example.org` lost its tenant registration, GetUserRealm answered `NameSpaceType: Unknown`, and the gate diff --git a/docs/statistical-assurance.md b/docs/statistical-assurance.md index fe53dc1f..d51ce283 100644 --- a/docs/statistical-assurance.md +++ b/docs/statistical-assurance.md @@ -131,9 +131,9 @@ level and does not guarantee that the posterior is near 0.5. `entropy_reduction_nats` is -\[ +$$ H(P_m(X))-H(P_m(X\mid e)). -\] +$$ It can be negative. It is a signed marginal entropy change, not realized pointwise information gain. Summing it across dependent nodes can double count diff --git a/scripts/check.py b/scripts/check.py index f29a572d..83b593c4 100644 --- a/scripts/check.py +++ b/scripts/check.py @@ -69,6 +69,7 @@ (_CORE, "cost-surface", [_PY, "scripts/check_cost_surface.py"]), (_CORE, "text-hygiene", [_PY, "scripts/check_text_hygiene.py"]), (_CORE, "markdown-links", [_PY, "scripts/check_markdown_links.py"]), + (_CORE, "doc-rot", [_PY, "scripts/check_doc_rot.py"]), (_CORE, "terminal-demo", [_PY, "scripts/generate_terminal_demo.py", "--check"]), (_CORE, "clusterfuzzlite-requirements", [_PY, "scripts/check_clusterfuzzlite_requirements.py"]), (_CORE, "schema-sources", [_PY, "scripts/check_schema_sources.py"]), diff --git a/scripts/check_doc_rot.py b/scripts/check_doc_rot.py new file mode 100644 index 00000000..dbc7826b --- /dev/null +++ b/scripts/check_doc_rot.py @@ -0,0 +1,218 @@ +#!/usr/bin/env python3 +"""Fail when Markdown rots in ways that render silently wrong on github.com. + +Two failure classes, both drawn from a real incident. On 2026-07-11 a single +commit converted 285 lines of working ``$``-delimited math in +``docs/correlation.md`` into ``\\( ... \\)`` and ``\\[ ... \\]``, and replaced +five sections with one-paragraph pointer stubs that existed only to preserve an +anchor. Both survived every gate for three weeks, because no checker looked at +either thing. + +1. **Math delimiters.** GitHub's Markdown renderer does not support LaTeX + ``\\(`` or ``\\[`` delimiters. It treats them as escaped literal brackets, + drops the backslash, and prints the raw LaTeX in running prose. The + ``$``-delimited forms are the ones that render, so the others are rejected. + +2. **Pointer stubs.** A section whose body only redirects the reader is not + content. ``check_section_links.py`` verifies that a referenced section number + resolves, which a stub satisfies perfectly, so a hollowed-out section reads + as a live cross-reference target while carrying nothing. + +Both checks skip fenced code blocks and inline code spans, because documenting +either pattern is legitimate and this file itself does it. + +Run with:: + + uv run python scripts/check_doc_rot.py +""" + +from __future__ import annotations + +import argparse +import re +from dataclasses import dataclass +from pathlib import Path + +_ROOT = Path(__file__).resolve().parents[1] + +# Scanned surfaces: the docs tree plus the two root documents readers reach +# first. CHANGELOG.md is excluded because its historical entries preserve the +# wording used when each version shipped and must not be rewritten. +_SCAN_DIRECTORIES = ("docs",) +_SCAN_FILES = ("README.md", "ROADMAP.md") + +_FENCE = re.compile(r"^\s*(```|~~~)") +_INLINE_CODE = re.compile(r"`[^`]*`") +_HEADING = re.compile(r"^(#{2,6})\s+(.*)$") + +# GitHub renders neither of these as math. Matching the opening delimiters is +# enough; a document never has one without meaning to open a math span. +_BROKEN_MATH = re.compile(r"\\\(|\\\[") + +# A body under this many words that also redirects the reader is a pointer, not +# a section. Real short sections exist in this repository (an eight-word "Setup", +# an eleven-word "Reporting vulnerabilities"), so length alone is not evidence; +# the redirect vocabulary is what separates a terse section from a hollow one. +_STUB_MAX_WORDS = 60 + +_REDIRECT = re.compile( + r"\b(?:former section|formerly section|see section|is now section|are now section" + r"|moved to section|superseded by section|replaced by section" + r"|retained (?:only )?as an anchor|legacy .{0,30}anchor)\b", + re.IGNORECASE, +) + + +@dataclass(frozen=True, slots=True) +class Finding: + """One rejected location, reported as an editor-navigable reference.""" + + path: Path + line: int + check: str + detail: str + + def render(self) -> str: + """Return the finding as ``path:line: check: detail``. + + Repository paths are reported relative to the root so an editor can jump + to them. A path outside the root still reports, absolutely, rather than + raising: ``check_paths`` accepts arbitrary files. + """ + try: + location = self.path.relative_to(_ROOT).as_posix() + except ValueError: + location = self.path.as_posix() + return f"{location}:{self.line}: {self.check}: {self.detail}" + + +def _strip_inline_code(line: str) -> str: + """Blank out inline code spans so documented patterns are not flagged.""" + return _INLINE_CODE.sub("``", line) + + +def _code_fence_mask(lines: list[str]) -> list[bool]: + """Return, per line, whether that line sits inside a fenced code block.""" + inside = False + mask: list[bool] = [] + for line in lines: + if _FENCE.match(line): + # The fence markers themselves count as code so an opening fence + # carrying an info string is never scanned as prose. + mask.append(True) + inside = not inside + continue + mask.append(inside) + return mask + + +def _check_math_delimiters(path: Path, lines: list[str], mask: list[bool]) -> list[Finding]: + """Reject LaTeX delimiters that github.com does not render as math.""" + findings: list[Finding] = [] + for index, line in enumerate(lines, start=1): + if mask[index - 1]: + continue + match = _BROKEN_MATH.search(_strip_inline_code(line)) + if match is None: + continue + found = match.group(0) + wanted = "$...$" if found == "\\(" else "$$...$$" + findings.append( + Finding( + path, + index, + "math-delimiter", + f"{found} does not render as math on github.com; use {wanted}", + ) + ) + return findings + + +def _check_unbalanced_display_math(path: Path, lines: list[str], mask: list[bool]) -> list[Finding]: + """Reject an odd number of ``$$`` fences, which swallows the text after it.""" + opens: list[int] = [] + for index, line in enumerate(lines, start=1): + if mask[index - 1]: + continue + for _ in range(_strip_inline_code(line).count("$$")): + if opens: + opens.pop() + else: + opens.append(index) + return [Finding(path, line, "display-math", "unclosed $$ display-math block") for line in opens] + + +def _check_pointer_stubs(path: Path, lines: list[str], mask: list[bool]) -> list[Finding]: + """Reject sections whose body only redirects the reader elsewhere.""" + headings: list[tuple[int, int, str]] = [] + for index, line in enumerate(lines): + if mask[index]: + continue + match = _HEADING.match(line) + if match is not None: + headings.append((index, len(match.group(1)), match.group(2).strip())) + + findings: list[Finding] = [] + for position, (index, level, title) in enumerate(headings): + following = headings[position + 1] if position + 1 < len(headings) else None + # A heading that introduces subsections is a container. Its own body is + # allowed to be an intro sentence or nothing at all. + if following is not None and following[1] > level: + continue + end = following[0] if following is not None else len(lines) + body = "\n".join(lines[index + 1 : end]).strip() + if len(body.split()) >= _STUB_MAX_WORDS: + continue + if _REDIRECT.search(body) is None and _REDIRECT.search(title) is None: + continue + findings.append( + Finding( + path, + index + 1, + "pointer-stub", + f'"{title}" redirects the reader without carrying content; restore it or remove the heading', + ) + ) + return findings + + +def _scanned_paths() -> list[Path]: + """Return every Markdown file in scope, in a stable order.""" + paths: list[Path] = [] + for directory in _SCAN_DIRECTORIES: + paths.extend(sorted((_ROOT / directory).rglob("*.md"))) + paths.extend(_ROOT / name for name in _SCAN_FILES if (_ROOT / name).is_file()) + return paths + + +def check_paths(paths: list[Path]) -> list[Finding]: + """Return every finding across the supplied Markdown files.""" + findings: list[Finding] = [] + for path in paths: + lines = path.read_text(encoding="utf-8").splitlines() + mask = _code_fence_mask(lines) + findings.extend(_check_math_delimiters(path, lines, mask)) + findings.extend(_check_unbalanced_display_math(path, lines, mask)) + findings.extend(_check_pointer_stubs(path, lines, mask)) + return findings + + +def main(argv: list[str] | None = None) -> int: + """Report Markdown rot and fail closed when any is found.""" + parser = argparse.ArgumentParser(description="Reject Markdown that renders silently wrong on github.com.") + parser.parse_args(argv) + + paths = _scanned_paths() + findings = check_paths(paths) + if findings: + print(f"FAIL: {len(findings)} Markdown rendering or content defect(s):") + for finding in findings: + print(f" {finding.render()}") + return 1 + + print(f"OK: {len(paths)} Markdown files carry renderable math and no pointer-stub sections.") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_doc_rot.py b/tests/test_doc_rot.py new file mode 100644 index 00000000..99245b18 --- /dev/null +++ b/tests/test_doc_rot.py @@ -0,0 +1,143 @@ +"""Doc-rot gate: Markdown that renders wrong on github.com fails the build. + +``scripts/check_doc_rot.py`` rejects two silent failure classes taken from a +real incident: LaTeX delimiters GitHub does not render as math, and sections +hollowed out into pointers that still satisfy the section-link check. The first +test IS the CI gate, so a regression in the shipped docs fails the build. +""" + +from __future__ import annotations + +import subprocess +import sys +from pathlib import Path + +from scripts.check_doc_rot import Finding, check_paths + +REPO_ROOT = Path(__file__).resolve().parent.parent +_CHECKER = REPO_ROOT / "scripts" / "check_doc_rot.py" + + +def _run(*args: str) -> subprocess.CompletedProcess[str]: + return subprocess.run( # noqa: S603 - fixed interpreter + repo-local script, no untrusted input. + [sys.executable, str(_CHECKER), *args], + capture_output=True, + text=True, + cwd=REPO_ROOT, + check=False, + ) + + +def _write(tmp_path: Path, body: str) -> Path: + doc = tmp_path / "doc.md" + doc.write_text(body, encoding="utf-8") + return doc + + +def test_shipped_documentation_is_free_of_rot() -> None: + result = _run() + assert result.returncode == 0, ( + f"Markdown rendering or content defects:\n{result.stdout}\n" + "Use $...$ or $$...$$ for math, and restore any section reduced to a pointer." + ) + assert "OK:" in result.stdout + + +def test_inline_math_delimiter_is_rejected(tmp_path: Path) -> None: + # Assembled at runtime so this test's own source stays clean for the gate. + opener = "\\" + "(" + findings = check_paths([_write(tmp_path, f"## Heading\n\nThe value {opener}x{opener[0]})$ is set.\n")]) + + assert [finding.check for finding in findings] == ["math-delimiter"] + assert "$...$" in findings[0].detail + + +def test_display_math_delimiter_is_rejected(tmp_path: Path) -> None: + opener = "\\" + "[" + findings = check_paths([_write(tmp_path, f"## Heading\n\n{opener}\nx = 1\n\\]\n")]) + + assert [finding.check for finding in findings] == ["math-delimiter"] + assert "$$...$$" in findings[0].detail + + +def test_dollar_math_is_accepted(tmp_path: Path) -> None: + doc = _write(tmp_path, "## Heading\n\nInline $x$ and display:\n\n$$\ny = 2x\n$$\n\nDone.\n") + + assert check_paths([doc]) == [] + + +def test_math_delimiters_inside_a_code_fence_are_ignored(tmp_path: Path) -> None: + opener = "\\" + "[" + doc = _write(tmp_path, f"## Heading\n\n```latex\n{opener}\nx = 1\n\\]\n```\n\nProse.\n") + + assert check_paths([doc]) == [] + + +def test_math_delimiters_inside_inline_code_are_ignored(tmp_path: Path) -> None: + opener = "\\" + "(" + doc = _write(tmp_path, f"## Heading\n\nGitHub does not render `{opener} ... \\)` as math.\n") + + assert check_paths([doc]) == [] + + +def test_unclosed_display_math_is_rejected(tmp_path: Path) -> None: + findings = check_paths([_write(tmp_path, "## Heading\n\n$$\ny = 2x\n\nStranded prose.\n")]) + + assert [finding.check for finding in findings] == ["display-math"] + + +def test_pointer_stub_section_is_rejected(tmp_path: Path) -> None: + # The shape that survived three weeks: a heading kept only so historical + # cross-references still resolve, carrying no content of its own. + body = ( + "## Real section\n\n" + ("Substantive content. " * 40) + "\n\n### 4.4 Legacy validation-strategy anchor\n\n" + "Historical changelog and validation records cite the former section 4.4. The\n" + "current validation contract is section 8.\n" + ) + findings = check_paths([_write(tmp_path, body)]) + + assert [finding.check for finding in findings] == ["pointer-stub"] + assert "4.4" in findings[0].detail + + +def test_short_section_without_redirect_language_is_accepted(tmp_path: Path) -> None: + # Genuinely terse sections exist in this repository and must stay legal. + doc = _write(tmp_path, "## Reporting vulnerabilities\n\nEmail the maintainer. Do not open a public issue.\n") + + assert check_paths([doc]) == [] + + +def test_container_heading_with_subsections_is_accepted(tmp_path: Path) -> None: + # A parent heading whose body is empty because subsections follow it is + # ordinary structure, not a hollowed-out section. + doc = _write(tmp_path, "## 3. Bayesian evidence semantics\n\n### 3.1 Units\n\n" + ("Content. " * 40) + "\n") + + assert check_paths([doc]) == [] + + +def test_long_section_mentioning_another_section_is_accepted(tmp_path: Path) -> None: + # Cross-referencing is normal writing. Only a section that is *nothing but* + # a redirect is rejected, so length is what separates the two. + doc = _write( + tmp_path, + "## Analysis\n\n" + ("Real analytical content. " * 40) + "\nFor the derivation see section 8.\n", + ) + + assert check_paths([doc]) == [] + + +def test_findings_render_as_navigable_references(tmp_path: Path) -> None: + opener = "\\" + "[" + findings = check_paths([_write(tmp_path, f"## Heading\n\n{opener}\nx\n\\]\n")]) + rendered = findings[0].render() + + # A path outside the repository must still report rather than raise, since + # check_paths accepts arbitrary files. + assert rendered.startswith(tmp_path.as_posix()) + assert f":{findings[0].line}: math-delimiter:" in rendered + + +def test_repository_findings_render_relative_to_the_root() -> None: + finding = Finding(REPO_ROOT / "docs" / "correlation.md", 42, "math-delimiter", "detail") + + assert finding.render() == "docs/correlation.md:42: math-delimiter: detail" diff --git a/tests/test_quality_scorecard.py b/tests/test_quality_scorecard.py index 7f201f7e..860de24d 100644 --- a/tests/test_quality_scorecard.py +++ b/tests/test_quality_scorecard.py @@ -13,7 +13,6 @@ _byte_length, _catalog_surface, _duplicate_definition_bytes, - _mcp_context_cost, _render_markdown, build_scorecard, ) @@ -21,10 +20,22 @@ @pytest.fixture(scope="module") def scorecard() -> dict[str, Any]: - """Build the scorecard once; it registers the MCP surface and reads the catalog.""" + """Build the scorecard once; it registers the MCP surface and reads the catalog. + + Module-scoped on purpose. Measuring the MCP surface spins up an asyncio event + loop, and on Windows every new loop allocates a socket pair for its self-pipe. + Rebuilding per test multiplies that churn across parallel workers for no gain, + since the measurement is deterministic for a given revision. + """ return build_scorecard() +@pytest.fixture(scope="module") +def mcp_cost(scorecard: dict[str, Any]) -> dict[str, Any]: + """Reuse the scorecard's MCP measurement rather than recomputing it.""" + return scorecard["measurements"]["mcp_context_cost"] + + def test_byte_length_measures_the_compact_wire_form() -> None: # Padded JSON must not inflate the reported cost: a client receives the # compact form, so the measurement has to ignore incidental whitespace. @@ -77,28 +88,25 @@ def test_duplicate_definition_accounting_tolerates_a_bare_surface() -> None: assert result["most_redundant"] == [] -def test_mcp_context_cost_reports_the_registered_surface() -> None: - cost = _mcp_context_cost() - - assert cost["tool_count"] == len(cost["per_tool"]) - assert cost["tool_count"] > 0 - assert cost["session_context_bytes"] == cost["discovery_bytes"] + cost["instruction_preamble_bytes"] - assert all(entry["total_bytes"] > 0 for entry in cost["per_tool"]) +def test_mcp_context_cost_reports_the_registered_surface(mcp_cost: dict[str, Any]) -> None: + assert mcp_cost["tool_count"] == len(mcp_cost["per_tool"]) + assert mcp_cost["tool_count"] > 0 + assert mcp_cost["session_context_bytes"] == mcp_cost["discovery_bytes"] + mcp_cost["instruction_preamble_bytes"] + assert all(entry["total_bytes"] > 0 for entry in mcp_cost["per_tool"]) -def test_mcp_per_tool_costs_are_ranked_largest_first() -> None: - sizes = [entry["total_bytes"] for entry in _mcp_context_cost()["per_tool"]] +def test_mcp_per_tool_costs_are_ranked_largest_first(mcp_cost: dict[str, Any]) -> None: + sizes = [entry["total_bytes"] for entry in mcp_cost["per_tool"]] assert sizes == sorted(sizes, reverse=True) -def test_mcp_headroom_bound_is_not_larger_than_the_payload() -> None: - cost = _mcp_context_cost() - headroom = cost["headroom"] +def test_mcp_headroom_bound_is_not_larger_than_the_payload(mcp_cost: dict[str, Any]) -> None: + headroom = mcp_cost["headroom"] # Dropping a field can only shrink the payload. A trim that appears to grow # it would mean the measurement is not reading the same serialization. - assert headroom["discovery_without_output_schema_bytes"] <= cost["discovery_bytes"] + assert headroom["discovery_without_output_schema_bytes"] <= mcp_cost["discovery_bytes"] assert 0.0 <= headroom["output_schema_share_of_discovery"] <= 1.0