From acb23d868ea74f05b5cb30408b30d7cda3a1ad1d Mon Sep 17 00:00:00 2001 From: ralph-bot Date: Tue, 30 Jun 2026 10:13:17 +0000 Subject: [PATCH] op1: reject fabricated compute + config-vs-proof mismatch (compute-gaming) - check_compute_plausibility: reject a bundle whose tokens_seen/wall_clock_s exceeds the declared GPU's bf16 MFU ceiling (>70%). wall_clock_s is miner-declared + not in bundle_hash, so it can be under-claimed to win the compute-weighted crown. implied MFU = 6*N*(tok/s) / gpu_bf16_peak. - check_recipe_config_matches_proof: reject when a submitted configs/*.json total_steps != the steps the proof actually ran (decoy recipe). - wired into op1_diff_and_integrity (reject before scoring). - catches the live king 5CtaoQf1: 818k tok/s on one H100 = 126% MFU (impossible); submitted muon/40000-step config but proof ran adamw/10600. - 8 tests incl. legit-30h + optimized-250k + unknown-GPU + incomplete guards. Co-Authored-By: Claude Opus 4.8 --- tests/test_integrity_trained.py | 50 ++++++++++++++ validator/integrity.py | 117 ++++++++++++++++++++++++++++++++ validator/validator.py | 28 ++++++++ 3 files changed, 195 insertions(+) diff --git a/tests/test_integrity_trained.py b/tests/test_integrity_trained.py index 2f5d087..5243c07 100644 --- a/tests/test_integrity_trained.py +++ b/tests/test_integrity_trained.py @@ -8,6 +8,8 @@ from validator.integrity import ( check_checkpoint_trained, + check_compute_plausibility, + check_recipe_config_matches_proof, nats_per_token_from_bpb, ) @@ -15,6 +17,54 @@ RANDOM_NATS = math.log(VOCAB) # ~10.82 +# --- compute-plausibility: anti compute-gaming ------------------------------- +H100 = {"gpu_name": "NVIDIA H100 80GB HBM3"} + + +def test_rejects_fabricated_compute_the_5ctaoqf1_king(): + # 5.557B tokens in 6788s on ONE H100 => 818k tok/s => ~126% MFU = impossible. + fs = {"tokens_seen": 5_557_452_800, "wall_clock_s": 6787.88, "n_params": 253_874_184} + ok, reason = check_compute_plausibility(fs, H100) + assert not ok and "fabricated compute" in reason and "MFU" in reason + + +def test_accepts_a_real_30h_run(): + fs = {"tokens_seen": 5_557_452_800, "wall_clock_s": 109_000, "n_params": 253_874_184} # ~51k tok/s + assert check_compute_plausibility(fs, H100)[0] + + +def test_accepts_an_optimized_run_under_the_ceiling(): + fs = {"tokens_seen": 5_557_452_800, "wall_clock_s": 22_000, "n_params": 253_874_184} # ~250k tok/s, ~39% MFU + assert check_compute_plausibility(fs, H100)[0] + + +def test_incomplete_training_summary_is_skipped_not_rejected(): + assert check_compute_plausibility({"tokens_seen": 0, "wall_clock_s": 0}, {})[0] + assert check_compute_plausibility({}, None)[0] + + +def test_unknown_gpu_uses_fastest_peak_to_avoid_false_reject(): + fs = {"tokens_seen": 5_557_452_800, "wall_clock_s": 22_000, "n_params": 253_874_184} + assert check_compute_plausibility(fs, {"gpu_name": "Some Future GPU"})[0] + + +# --- declared-recipe-matches-proof ------------------------------------------- +def test_rejects_config_step_mismatch_the_5ctaoqf1_king(): + patch = '+++ b/configs/muon_wsd_qknorm_b20593.json\n+{\n+ "total_steps": 40000,\n+ "qk_norm": true\n+}\n' + ok, reason = check_recipe_config_matches_proof(patch, {"steps": 10600}) + assert not ok and "mismatch" in reason + + +def test_accepts_matching_config_steps(): + patch = '+++ b/configs/run.json\n+{\n+ "total_steps": 10600\n+}\n' + assert check_recipe_config_matches_proof(patch, {"steps": 10600})[0] + + +def test_config_match_skips_when_no_config_or_no_steps(): + assert check_recipe_config_matches_proof("+++ b/model/x.py\n+x = 1\n", {"steps": 10600})[0] + assert check_recipe_config_matches_proof('+++ b/configs/c.json\n+{"total_steps": 5}\n', {})[0] + + def test_rejects_the_uid155_random_king(): # Measured in the incident: ~11.0 nats/token, log claimed final_loss 3.05. ok, reason = check_checkpoint_trained(11.0, VOCAB, claimed_final_loss=3.0496) diff --git a/validator/integrity.py b/validator/integrity.py index a64938d..126141b 100644 --- a/validator/integrity.py +++ b/validator/integrity.py @@ -85,3 +85,120 @@ def check_checkpoint_trained( ) return True, "ok" + + +# --- Compute-plausibility (anti compute-gaming) ------------------------------- +# +# `wall_clock_s` is MINER-DECLARED (and not in bundle_hash), so a miner can +# under-claim it to look efficient and win the compute-weighted crown — train a +# real model over ~30 H100h but report ~2h. The give-away is physics: the implied +# model-FLOP rate (~6*N*tok/s) cannot exceed the GPU's bf16 matmul peak, and real +# sustained TRAINING MFU is ~30-55%. An implied MFU above the ceiling means the +# wall_clock_s (hence the compute cost) is fabricated. +MAX_PLAUSIBLE_MFU = 0.7 +# bf16 dense matmul peak (TFLOP/s) per GPU family — the hard physical ceiling. +_GPU_BF16_PEAK_TFLOPS = { + "a100": 312.0, "a800": 312.0, "l4": 121.0, "l40": 362.0, "4090": 165.0, + "h100": 989.0, "h200": 989.0, "h800": 989.0, + "b100": 1800.0, "b200": 2250.0, "gb200": 2500.0, +} +# Unknown GPU -> assume the fastest known part, so we NEVER false-reject; the gate +# only fires when even the fastest plausible GPU cannot explain the throughput. +_DEFAULT_PEAK_TFLOPS = 2500.0 + + +def _gpu_bf16_peak_flops(gpu_name: str | None) -> float: + g = (gpu_name or "").lower() + for key, tflops in _GPU_BF16_PEAK_TFLOPS.items(): + if key in g: + return tflops * 1e12 + return _DEFAULT_PEAK_TFLOPS * 1e12 + + +def check_compute_plausibility( + final_state: dict, + calibration: dict | None = None, + *, + max_mfu: float = MAX_PLAUSIBLE_MFU, +) -> tuple[bool, str]: + """Reject a bundle whose declared training throughput is physically impossible. + + tokens_seen / wall_clock_s implies ~6*N FLOPs/token; over the declared GPU's + bf16 peak that is the achieved MFU. An implied MFU > `max_mfu` means the + wall_clock_s (and the efficiency-gate compute cost it drives) is fabricated. + Best-effort: a missing/incomplete training_summary is skipped (deferred to the + other gates), not rejected. Returns (ok, reason); ok=False -> reject. + """ + fs = final_state or {} + try: + tokens = float(fs.get("tokens_seen", 0) or 0) + wall = float(fs.get("wall_clock_s", 0) or 0) + n = float(fs.get("n_params", 0) or 0) + except (TypeError, ValueError): + return True, "compute-plausibility: non-numeric training_summary (skipped)" + if tokens <= 0 or wall <= 0 or n <= 0: + return True, "compute-plausibility: incomplete training_summary (skipped)" + gpu = (calibration or {}).get("gpu_name") or fs.get("gpu_name") or fs.get("device") or "" + flops_per_s = 6.0 * n * tokens / wall # 6N FLOPs/token (fwd+bwd) + mfu = flops_per_s / _gpu_bf16_peak_flops(gpu) + if mfu > max_mfu: + return False, ( + f"fabricated compute: {tokens / wall:,.0f} tok/s for a {n / 1e6:.0f}M model on " + f"'{gpu or 'unknown'}' => {mfu * 100:.0f}% MFU (> {max_mfu * 100:.0f}% physical max); " + f"wall_clock_s={wall:.0f}s for {tokens:,.0f} tokens is not achievable" + ) + return True, f"compute plausible: {tokens / wall:,.0f} tok/s, {mfu * 100:.0f}% MFU" + + +def _added_config_jsons(patch_text: str) -> list[dict]: + """Parse every NEW/whole configs/*.json the patch adds (best-effort).""" + import json + + out: list[dict] = [] + path: str | None = None + buf: list[str] = [] + + def _flush() -> None: + if path and path.endswith(".json") and "config" in path and buf: + try: + out.append(json.loads("\n".join(buf))) + except Exception: # noqa: BLE001 — partial/edited config, skip + pass + + for ln in (patch_text or "").splitlines(): + if ln.startswith("+++ b/"): + _flush() + path, buf = ln[6:], [] + elif ln.startswith("+") and not ln.startswith("+++"): + buf.append(ln[1:]) + _flush() + return out + + +def check_recipe_config_matches_proof(patch_text: str, final_state: dict) -> tuple[bool, str]: + """A submitted training config (configs/*.json) must match what the proof ran. + + If the patch declares `total_steps` that differs from the steps the proof + recorded, the crowned checkpoint was NOT produced by the declared recipe (the + submitted config is a decoy). Best-effort: skipped when no config is added or no + proof step count exists. Returns (ok, reason); ok=False -> reject. + """ + fs = final_state or {} + proof_steps = fs.get("steps") + if proof_steps is None: + proof_steps = (fs.get("config") or {}).get("total_steps") + if proof_steps is None: + return True, "config-match: no proof step count (skipped)" + for cfg in _added_config_jsons(patch_text): + declared = cfg.get("total_steps") + if declared is None: + continue + try: + if int(declared) != int(proof_steps): + return False, ( + f"declared recipe mismatch: submitted config total_steps={declared} but the " + f"proof ran {proof_steps} steps — crowned checkpoint not from the submitted recipe" + ) + except (TypeError, ValueError): + continue + return True, "config matches proof" diff --git a/validator/validator.py b/validator/validator.py index 37be324..124bb36 100644 --- a/validator/validator.py +++ b/validator/validator.py @@ -41,6 +41,7 @@ ) from proof.runner import _load_restricted_paths, scan_diff_for_exploit_patterns, scan_diff_for_restricted from proof.sources import compute_container_measurement +from validator.integrity import check_compute_plausibility, check_recipe_config_matches_proof # Hard-coded sanity bounds for the miner-submitted model config. The validator # loads checkpoint['config'] from an attacker-controlled file; without bounds @@ -297,6 +298,33 @@ def op1_diff_and_integrity( if exploit_hits: return False, f"patch injects off-protocol inputs: {exploit_hits[0][0]} :: {exploit_hits[0][1]}" + # Compute-plausibility + declared-recipe-matches-proof (anti compute-gaming): + # wall_clock_s is miner-declared (not in bundle_hash), so it can be under-claimed + # to win the compute-weighted crown. Reject physically-impossible training + # throughput + a submitted config whose step count the proof never ran. + fs_path = proof_dir / "training" / "final_state.json" + if fs_path.exists(): + try: + final_state = json.loads(fs_path.read_text(encoding="utf-8", errors="replace")) + except (ValueError, OSError): + final_state = {} + calibration: dict = {} + cal_path = proof_dir / "calibration.json" + if cal_path.exists(): + try: + calibration = json.loads(cal_path.read_text(encoding="utf-8", errors="replace")) + except (ValueError, OSError): + calibration = {} + ok_c, detail_c = check_compute_plausibility(final_state, calibration) + if not ok_c: + return False, detail_c + if patch_path.exists(): + ok_m, detail_m = check_recipe_config_matches_proof( + patch_path.read_text(encoding="utf-8", errors="replace"), final_state + ) + if not ok_m: + return False, detail_m + return True, "ok"