diff --git a/tests/test_integrity_trained.py b/tests/test_integrity_trained.py index 5243c07..ec44b1f 100644 --- a/tests/test_integrity_trained.py +++ b/tests/test_integrity_trained.py @@ -7,6 +7,7 @@ import pytest from validator.integrity import ( + check_canonical_data_source, check_checkpoint_trained, check_compute_plausibility, check_recipe_config_matches_proof, @@ -65,6 +66,27 @@ def test_config_match_skips_when_no_config_or_no_steps(): assert check_recipe_config_matches_proof('+++ b/configs/c.json\n+{"total_steps": 5}\n', {})[0] +# --- canonical data source (anti data-lock-bypass) --------------------------- +def test_rejects_noncanonical_host_manifest_ea576b0a(): + fs = {"config": {"manifest_path": "/home/root/diony/recipe/data/data_manifest.json", "data_base_dir": "data"}} + ok, reason = check_canonical_data_source(fs) + assert not ok and "non-canonical data source" in reason + + +def test_rejects_mnt_data_base_dir(): + assert not check_canonical_data_source({"config": {"data_base_dir": "/mnt/scratch/SN40/data_50b"}})[0] + + +def test_accepts_canonical_relative_data_path_7fd43cef(): + fs = {"config": {"manifest_path": "data/data_manifest.json", "data_base_dir": "data"}} + assert check_canonical_data_source(fs)[0] + + +def test_data_source_skips_when_no_config(): + assert check_canonical_data_source({})[0] + assert check_canonical_data_source({"config": {}})[0] + + def test_rejects_the_uid155_random_king(): # Measured in the incident: ~11.0 nats/token, log claimed final_loss 3.05. ok, reason = check_checkpoint_trained(11.0, VOCAB, claimed_final_loss=3.0496) diff --git a/validator/integrity.py b/validator/integrity.py index 126141b..21ae09f 100644 --- a/validator/integrity.py +++ b/validator/integrity.py @@ -23,6 +23,7 @@ from __future__ import annotations import math +import re # Reject if held-out loss >= this fraction of the random baseline ln(vocab). # A real ~254M model sits at ~3-4.5 nats/token; random is ~10.8 for vocab 50257. @@ -202,3 +203,28 @@ def check_recipe_config_matches_proof(patch_text: str, final_state: dict) -> tup except (TypeError, ValueError): continue return True, "config matches proof" + + +# Miner-host data paths a canonical run must never reference. The locked +# canonical data_manifest is relative (e.g. "data/data_manifest.json"); a config +# that points manifest_path/data_base_dir at /home, /mnt, … is a data-lock bypass +# (the run trained on the miner's own data, possibly contaminated with the +# held-out, then claimed the canonical recipe). The in-the-wild case: +# manifest_path="/home/root/diony/recipe/data/data_manifest.json". +_HOST_DATA_PATH_RE = re.compile(r"^\s*(?:~|\.\.)?/(?:home|root|mnt|media|srv|scratch|Users)\b") + + +def check_canonical_data_source(final_state: dict) -> tuple[bool, str]: + """Reject a bundle whose training config points the data manifest/dir at a + miner-host path. NOTE: this lives in `final_state.config`, NOT the patch diff, + so the restricted/exploit patch scanners miss it — op1 must check it here. + Best-effort: skipped when there is no config. Returns (ok, reason).""" + cfg = (final_state or {}).get("config") or {} + for key in ("manifest_path", "data_base_dir", "data_dir", "data_path"): + v = cfg.get(key) + if isinstance(v, str) and _HOST_DATA_PATH_RE.match(v): + return False, ( + f"non-canonical data source: config.{key}={v!r} is a miner-host path — " + f"the run bypassed the locked canonical data_manifest" + ) + return True, "canonical data source" diff --git a/validator/validator.py b/validator/validator.py index 124bb36..4a86da6 100644 --- a/validator/validator.py +++ b/validator/validator.py @@ -41,7 +41,11 @@ ) from proof.runner import _load_restricted_paths, scan_diff_for_exploit_patterns, scan_diff_for_restricted from proof.sources import compute_container_measurement -from validator.integrity import check_compute_plausibility, check_recipe_config_matches_proof +from validator.integrity import ( + check_canonical_data_source, + check_compute_plausibility, + check_recipe_config_matches_proof, +) # Hard-coded sanity bounds for the miner-submitted model config. The validator # loads checkpoint['config'] from an attacker-controlled file; without bounds @@ -318,6 +322,9 @@ def op1_diff_and_integrity( ok_c, detail_c = check_compute_plausibility(final_state, calibration) if not ok_c: return False, detail_c + ok_d, detail_d = check_canonical_data_source(final_state) + if not ok_d: + return False, detail_d if patch_path.exists(): ok_m, detail_m = check_recipe_config_matches_proof( patch_path.read_text(encoding="utf-8", errors="replace"), final_state