diff --git a/.gitignore b/.gitignore index 5bee3478..875c9b59 100644 --- a/.gitignore +++ b/.gitignore @@ -150,6 +150,11 @@ docs/accuracy.md # Typed DNS catalog baseline (2026-07-17): aggregate counts and generic # provider patterns only. No target names, target records, tenant IDs, or rows. !/validation/2026-07-17-typed-catalog-baseline.md +# Phase 1 product-quality baseline (2026-08-05): generated by +# scripts/quality_scorecard.py from the registered MCP surface and the shipped +# catalog. Network-free and corpus-free, so it contains protocol byte counts and +# catalog counts only. No target names, records, tenant IDs, or rows. +!/validation/2026-08-05-quality-baseline-scorecard.md # Per-pattern base-rate reducer and its aggregate memo (2026-07-25): # provider-owned patterns and aggregate counts only; the reducer never # reads target-identifying fields and its output stays in ignored paths. diff --git a/scripts/quality_scorecard.py b/scripts/quality_scorecard.py index 9544d32e..b8a555b2 100644 --- a/scripts/quality_scorecard.py +++ b/scripts/quality_scorecard.py @@ -419,7 +419,9 @@ def _render_markdown(scorecard: dict[str, Any]) -> str: lines = [ "# Product-quality baseline scorecard", "", - f"Commit `{commit[:12]}`, catalog `{(revisions['catalog_digest_sha256'] or 'unknown')[:12]}`, " + # The catalog digest carries its algorithm prefix so neither a reader nor + # the backticked-commit-receipt gate mistakes a content hash for a commit. + f"Commit `{commit[:12]}`, catalog `sha256:{(revisions['catalog_digest_sha256'] or 'unknown')[:12]}`, " f"Python {scorecard['environment']['python']}.", "", "Network-free, corpus-free, aggregate-safe. Diagnostic artifact, not a gate.", diff --git a/tests/test_quality_scorecard.py b/tests/test_quality_scorecard.py index 2acd1937..7f201f7e 100644 --- a/tests/test_quality_scorecard.py +++ b/tests/test_quality_scorecard.py @@ -3,6 +3,7 @@ from __future__ import annotations import json +import re from typing import Any import pytest @@ -159,6 +160,18 @@ def test_markdown_memo_reports_the_measured_numbers(scorecard: dict[str, Any]) - assert entry["metric"] in memo +def test_markdown_memo_never_prints_a_non_commit_hash_as_a_commit_receipt(scorecard: dict[str, Any]) -> None: + # test_documentation_integrity resolves every backticked hex string in + # validation/*.md as a git commit. The catalog digest is a content hash, so + # it must not be emitted bare or a committed memo breaks that gate. + memo = _render_markdown(scorecard) + receipts = set(re.findall(r"`([0-9a-f]{7,40})`", memo)) + digest = scorecard["revisions"]["catalog_digest_sha256"] or "" + + assert digest[:12] not in receipts + assert f"`sha256:{digest[:12]}`" in memo + + def test_markdown_memo_tolerates_an_absent_git_revision(scorecard: dict[str, Any]) -> None: # The harness must still render inside a source archive with no git metadata. detached = {**scorecard, "revisions": {**scorecard["revisions"], "commit": None}} diff --git a/validation/2026-08-05-quality-baseline-scorecard.md b/validation/2026-08-05-quality-baseline-scorecard.md new file mode 100644 index 00000000..9c20ebfa --- /dev/null +++ b/validation/2026-08-05-quality-baseline-scorecard.md @@ -0,0 +1,67 @@ +# Product-quality baseline scorecard + +Commit `8a0536b34caa`, catalog `sha256:004afd17fe96`, Python 3.14.4. + +Network-free, corpus-free, aggregate-safe. Diagnostic artifact, not a gate. + +## MCP context cost + +What an agent pays before it does any work. + +| Measure | Value | +|---|---:| +| Registered tools | 22 | +| Discovery payload | 81,880 bytes | +| Instruction preamble | 9,996 bytes | +| **Session context before first call** | **91,876 bytes** | +| Order-of-magnitude tokens | ~22,969 | + +Where the discovery payload goes: + +| Component | Bytes | Share | +|---|---:|---:| +| `outputSchema` | 51,358 | 63.9% | +| `description` | 21,867 | 27.2% | +| `inputSchema` | 4,696 | 5.9% | +| `annotations` | 1,945 | 2.4% | +| `name` | 441 | 0.5% | + +Output schemas are 63.1% of discovery. Removing them entirely would leave 30,170 bytes, which is an upper bound on headroom rather than a proposal: structured output is part of the contract clients validate against. + +Cross-tool definition duplication accounts for only 6,424 bytes across 5 shared definition bodies, so a shared-definition scheme is not the lever either. + +Largest and smallest registered tools: + +- `reevaluate_domain` at 14,729 bytes +- `clear_ephemeral_fingerprints` at 583 bytes +- top 5 tools carry 47.4% of discovery + +## Catalog surface + +| Measure | Value | +|---|---:| +| Entries | 860 | +| Detection rules | 1,070 | +| Rules carrying a verification date | 26 (2.4%) | +| Undated rules | 1,044 | + +## Measured elsewhere + +| Metric | Owner | +|---|---| +| component latency and peak allocation | `scripts/characterize_performance.py` | +| claim lineage and provenance completeness | `scripts/check_default_claim_audit.py` | +| typed catalog coverage from real results | `validation/catalog_baseline.py` | +| claim-family precision, benefit, and safety | `docs/quality-baseline-preregistration.md` | + +## Unmeasured + +Named rather than omitted, because an absent metric reads as a passing one. + +| Metric | Blocked by | +|---|---| +| classified versus unclassified observable surface | private corpus | +| CT marginal signal gain relative to latency cost | network and private corpus | +| end-to-end cold and warm p50/p95 for single, batch, graph, and MCP workflows | network | +| degraded-source rate and partial-result rate | network and private corpus | +| MCP result payload bytes under real lookups | network |