op1: reject fabricated compute + config-vs-proof mismatch (compute-gaming)

tests/test_integrity_trained.py

@@ -8,13 +8,63 @@
 
 from validator.integrity import (
     check_checkpoint_trained,
+    check_compute_plausibility,
+    check_recipe_config_matches_proof,
     nats_per_token_from_bpb,
 )
 
 VOCAB = 50257
 RANDOM_NATS = math.log(VOCAB)  # ~10.82
 
 
+# --- compute-plausibility: anti compute-gaming -------------------------------
+H100 = {"gpu_name": "NVIDIA H100 80GB HBM3"}
+
+
+def test_rejects_fabricated_compute_the_5ctaoqf1_king():
+    # 5.557B tokens in 6788s on ONE H100 => 818k tok/s => ~126% MFU = impossible.
+    fs = {"tokens_seen": 5_557_452_800, "wall_clock_s": 6787.88, "n_params": 253_874_184}
+    ok, reason = check_compute_plausibility(fs, H100)
+    assert not ok and "fabricated compute" in reason and "MFU" in reason
+
+
+def test_accepts_a_real_30h_run():
+    fs = {"tokens_seen": 5_557_452_800, "wall_clock_s": 109_000, "n_params": 253_874_184}  # ~51k tok/s
+    assert check_compute_plausibility(fs, H100)[0]
+
+
+def test_accepts_an_optimized_run_under_the_ceiling():
+    fs = {"tokens_seen": 5_557_452_800, "wall_clock_s": 22_000, "n_params": 253_874_184}  # ~250k tok/s, ~39% MFU
+    assert check_compute_plausibility(fs, H100)[0]
+
+
+def test_incomplete_training_summary_is_skipped_not_rejected():
+    assert check_compute_plausibility({"tokens_seen": 0, "wall_clock_s": 0}, {})[0]
+    assert check_compute_plausibility({}, None)[0]
+
+
+def test_unknown_gpu_uses_fastest_peak_to_avoid_false_reject():
+    fs = {"tokens_seen": 5_557_452_800, "wall_clock_s": 22_000, "n_params": 253_874_184}
+    assert check_compute_plausibility(fs, {"gpu_name": "Some Future GPU"})[0]
+
+
+# --- declared-recipe-matches-proof -------------------------------------------
+def test_rejects_config_step_mismatch_the_5ctaoqf1_king():
+    patch = '+++ b/configs/muon_wsd_qknorm_b20593.json\n+{\n+  "total_steps": 40000,\n+  "qk_norm": true\n+}\n'
+    ok, reason = check_recipe_config_matches_proof(patch, {"steps": 10600})
+    assert not ok and "mismatch" in reason
+
+
+def test_accepts_matching_config_steps():
+    patch = '+++ b/configs/run.json\n+{\n+  "total_steps": 10600\n+}\n'
+    assert check_recipe_config_matches_proof(patch, {"steps": 10600})[0]
+
+
+def test_config_match_skips_when_no_config_or_no_steps():
+    assert check_recipe_config_matches_proof("+++ b/model/x.py\n+x = 1\n", {"steps": 10600})[0]
+    assert check_recipe_config_matches_proof('+++ b/configs/c.json\n+{"total_steps": 5}\n', {})[0]
+
+
 def test_rejects_the_uid155_random_king():
     # Measured in the incident: ~11.0 nats/token, log claimed final_loss 3.05.
     ok, reason = check_checkpoint_trained(11.0, VOCAB, claimed_final_loss=3.0496)

validator/integrity.py

@@ -85,3 +85,120 @@ def check_checkpoint_trained(
             )
 
     return True, "ok"
+
+
+# --- Compute-plausibility (anti compute-gaming) -------------------------------
+#
+# `wall_clock_s` is MINER-DECLARED (and not in bundle_hash), so a miner can
+# under-claim it to look efficient and win the compute-weighted crown — train a
+# real model over ~30 H100h but report ~2h. The give-away is physics: the implied
+# model-FLOP rate (~6*N*tok/s) cannot exceed the GPU's bf16 matmul peak, and real
+# sustained TRAINING MFU is ~30-55%. An implied MFU above the ceiling means the
+# wall_clock_s (hence the compute cost) is fabricated.
+MAX_PLAUSIBLE_MFU = 0.7
+# bf16 dense matmul peak (TFLOP/s) per GPU family — the hard physical ceiling.
+_GPU_BF16_PEAK_TFLOPS = {
+    "a100": 312.0, "a800": 312.0, "l4": 121.0, "l40": 362.0, "4090": 165.0,
+    "h100": 989.0, "h200": 989.0, "h800": 989.0,
+    "b100": 1800.0, "b200": 2250.0, "gb200": 2500.0,
+}
+# Unknown GPU -> assume the fastest known part, so we NEVER false-reject; the gate
+# only fires when even the fastest plausible GPU cannot explain the throughput.
+_DEFAULT_PEAK_TFLOPS = 2500.0
+
+
+def _gpu_bf16_peak_flops(gpu_name: str | None) -> float:
+    g = (gpu_name or "").lower()
+    for key, tflops in _GPU_BF16_PEAK_TFLOPS.items():
+        if key in g:
+            return tflops * 1e12
+    return _DEFAULT_PEAK_TFLOPS * 1e12
+
+
+def check_compute_plausibility(
+    final_state: dict,
+    calibration: dict | None = None,
+    *,
+    max_mfu: float = MAX_PLAUSIBLE_MFU,
+) -> tuple[bool, str]:
+    """Reject a bundle whose declared training throughput is physically impossible.
+
+    tokens_seen / wall_clock_s implies ~6*N FLOPs/token; over the declared GPU's
+    bf16 peak that is the achieved MFU. An implied MFU > `max_mfu` means the
+    wall_clock_s (and the efficiency-gate compute cost it drives) is fabricated.
+    Best-effort: a missing/incomplete training_summary is skipped (deferred to the
+    other gates), not rejected. Returns (ok, reason); ok=False -> reject.
+    """
+    fs = final_state or {}
+    try:
+        tokens = float(fs.get("tokens_seen", 0) or 0)
+        wall = float(fs.get("wall_clock_s", 0) or 0)
+        n = float(fs.get("n_params", 0) or 0)
+    except (TypeError, ValueError):
+        return True, "compute-plausibility: non-numeric training_summary (skipped)"
+    if tokens <= 0 or wall <= 0 or n <= 0:
+        return True, "compute-plausibility: incomplete training_summary (skipped)"
+    gpu = (calibration or {}).get("gpu_name") or fs.get("gpu_name") or fs.get("device") or ""
+    flops_per_s = 6.0 * n * tokens / wall  # 6N FLOPs/token (fwd+bwd)
+    mfu = flops_per_s / _gpu_bf16_peak_flops(gpu)
+    if mfu > max_mfu:
+        return False, (
+            f"fabricated compute: {tokens / wall:,.0f} tok/s for a {n / 1e6:.0f}M model on "
+            f"'{gpu or 'unknown'}' => {mfu * 100:.0f}% MFU (> {max_mfu * 100:.0f}% physical max); "
+            f"wall_clock_s={wall:.0f}s for {tokens:,.0f} tokens is not achievable"
+        )
+    return True, f"compute plausible: {tokens / wall:,.0f} tok/s, {mfu * 100:.0f}% MFU"
+
+
+def _added_config_jsons(patch_text: str) -> list[dict]:
+    """Parse every NEW/whole configs/*.json the patch adds (best-effort)."""
+    import json
+
+    out: list[dict] = []
+    path: str | None = None
+    buf: list[str] = []
+
+    def _flush() -> None:
+        if path and path.endswith(".json") and "config" in path and buf:
+            try:
+                out.append(json.loads("\n".join(buf)))
+            except Exception:  # noqa: BLE001 — partial/edited config, skip
+                pass
+
+    for ln in (patch_text or "").splitlines():
+        if ln.startswith("+++ b/"):
+            _flush()
+            path, buf = ln[6:], []
+        elif ln.startswith("+") and not ln.startswith("+++"):
+            buf.append(ln[1:])
+    _flush()
+    return out
+
+
+def check_recipe_config_matches_proof(patch_text: str, final_state: dict) -> tuple[bool, str]:
+    """A submitted training config (configs/*.json) must match what the proof ran.
+
+    If the patch declares `total_steps` that differs from the steps the proof
+    recorded, the crowned checkpoint was NOT produced by the declared recipe (the
+    submitted config is a decoy). Best-effort: skipped when no config is added or no
+    proof step count exists. Returns (ok, reason); ok=False -> reject.
+    """
+    fs = final_state or {}
+    proof_steps = fs.get("steps")
+    if proof_steps is None:
+        proof_steps = (fs.get("config") or {}).get("total_steps")
+    if proof_steps is None:
+        return True, "config-match: no proof step count (skipped)"
+    for cfg in _added_config_jsons(patch_text):
+        declared = cfg.get("total_steps")
+        if declared is None:
+            continue
+        try:
+            if int(declared) != int(proof_steps):
+                return False, (
+                    f"declared recipe mismatch: submitted config total_steps={declared} but the "
+                    f"proof ran {proof_steps} steps — crowned checkpoint not from the submitted recipe"
+                )
+        except (TypeError, ValueError):
+            continue
+    return True, "config matches proof"

validator/validator.py

@@ -41,6 +41,7 @@
 )
 from proof.runner import _load_restricted_paths, scan_diff_for_exploit_patterns, scan_diff_for_restricted
 from proof.sources import compute_container_measurement
+from validator.integrity import check_compute_plausibility, check_recipe_config_matches_proof
 
 # Hard-coded sanity bounds for the miner-submitted model config. The validator
 # loads checkpoint['config'] from an attacker-controlled file; without bounds
@@ -297,6 +298,33 @@ def op1_diff_and_integrity(
         if exploit_hits:
             return False, f"patch injects off-protocol inputs: {exploit_hits[0][0]} :: {exploit_hits[0][1]}"
 
+    # Compute-plausibility + declared-recipe-matches-proof (anti compute-gaming):
+    # wall_clock_s is miner-declared (not in bundle_hash), so it can be under-claimed
+    # to win the compute-weighted crown. Reject physically-impossible training
+    # throughput + a submitted config whose step count the proof never ran.
+    fs_path = proof_dir / "training" / "final_state.json"
+    if fs_path.exists():
+        try:
+            final_state = json.loads(fs_path.read_text(encoding="utf-8", errors="replace"))
+        except (ValueError, OSError):
+            final_state = {}
+        calibration: dict = {}
+        cal_path = proof_dir / "calibration.json"
+        if cal_path.exists():
+            try:
+                calibration = json.loads(cal_path.read_text(encoding="utf-8", errors="replace"))
+            except (ValueError, OSError):
+                calibration = {}
+        ok_c, detail_c = check_compute_plausibility(final_state, calibration)
+        if not ok_c:
+            return False, detail_c
+        if patch_path.exists():
+            ok_m, detail_m = check_recipe_config_matches_proof(
+                patch_path.read_text(encoding="utf-8", errors="replace"), final_state
+            )
+            if not ok_m:
+                return False, detail_m
+
     return True, "ok"