diff --git a/eval/val_bpb.py b/eval/val_bpb.py index 365c8b5..93ebd50 100644 --- a/eval/val_bpb.py +++ b/eval/val_bpb.py @@ -39,6 +39,28 @@ # and for Phase 0 smoke tests that don't construct a sealed pool. DEFAULT_BYTES_PER_TOKEN = 4.0 +# Validator-pinned hidden-eval window. The eval MUST NOT derive its sequence +# length from the miner's checkpoint config (the old `cfg.max_seq_len // 2`): a +# miner could enlarge max_seq_len to be scored against an easier, longer-context +# eval than the king faced, and every model evaluated on a different window is +# not comparable. Pinning to a FIXED window makes all models comparable and +# removes a miner lever. This is a CANONICAL (image-baked / installed) constant — +# in the sandbox it is read from the trusted eval package, never miner-supplied. +EVAL_SEQ_LEN = 512 + + +def pinned_eval_seq_len(model_max_seq_len: int) -> int: + """The validator-pinned hidden-eval window: ``min(EVAL_SEQ_LEN, max_seq_len)``, + floored at 2 (compute_val_bpb needs seq_len+1 tokens per window). + + Single source of truth so EVERY eval path (in-process, subprocess, sandbox, + audit) pins seq_len identically and the host can independently re-derive and + verify the value a container echoes. The cap at the model's own max_seq_len + lets small-context models still load; the floor at EVAL_SEQ_LEN means no + miner can choose a window LARGER than the validator's. + """ + return max(2, min(EVAL_SEQ_LEN, int(model_max_seq_len))) + def compute_val_bpb( model: torch.nn.Module, diff --git a/tests/test_eval_seq_len_pin.py b/tests/test_eval_seq_len_pin.py new file mode 100644 index 0000000..647b023 --- /dev/null +++ b/tests/test_eval_seq_len_pin.py @@ -0,0 +1,93 @@ +"""Validator-pinned hidden-eval window (EVAL_SEQ_LEN). + +The eval seq_len must be the VALIDATOR's, not the miner's. Previously every path +derived it from the miner's checkpoint (cfg.max_seq_len // 2), letting a miner +enlarge max_seq_len to be scored on an easier, longer-context eval than the king +faced — and making different models non-comparable. `pinned_eval_seq_len` is the +single source of truth; the sandbox host independently re-derives it and REJECTS +a container that echoes anything else (verified, not trusted). +""" +from __future__ import annotations + +import dataclasses +import json +import sys +from pathlib import Path + +import numpy as np +import pytest +import torch + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) + +import ralph_bootstrap # noqa: F401 +from eval.val_bpb import EVAL_SEQ_LEN, pinned_eval_seq_len + +RECIPE_DIR = str(ralph_bootstrap.RECIPE_DIR) +if RECIPE_DIR not in sys.path: + sys.path.insert(0, RECIPE_DIR) +try: + from model import RalphBase, RalphConfig # noqa: E402 + _HAVE_MODEL = True +except Exception: # noqa: BLE001 + _HAVE_MODEL = False + + +def test_pinned_seq_len_caps_floors_and_never_miner_widened(): + assert pinned_eval_seq_len(2048) == EVAL_SEQ_LEN # large context capped DOWN to the pin + assert pinned_eval_seq_len(EVAL_SEQ_LEN) == EVAL_SEQ_LEN + assert pinned_eval_seq_len(256) == 256 # small-context model: its own max + assert pinned_eval_seq_len(1) == 2 # floored at 2 (needs seq_len+1 tokens) + # The old miner-controlled value for a long-context model is NOT what we pin. + assert pinned_eval_seq_len(4096) != 4096 // 2 + + +@pytest.mark.skipif(not _HAVE_MODEL, reason="canonical model package not importable") +def test_sandbox_host_rejects_tampered_seq_len(tmp_path, monkeypatch): + """A container that echoes seq_len != the host-pinned value is REJECTED — the + host re-derives the window itself and never trusts the manifest.""" + import validator.sandbox as sbx + import validator.validator as vv + from validator.sandbox import SandboxResult + + cfg = RalphConfig( + vocab_size=64, dim=32, n_layers=2, n_heads=2, head_dim=16, + ffn_mult=2.0, max_seq_len=16, + ) + model = RalphBase(cfg) + proof = tmp_path / "proof" + (proof / "training").mkdir(parents=True) + torch.save( + {"model": model.state_dict(), "config": dataclasses.asdict(cfg)}, + proof / "training" / "checkpoint.pt", + ) + (proof / "patch.diff").write_text("") + ralph_root = tmp_path / "root" + evdir = ralph_root / "eval" / "private" + evdir.mkdir(parents=True) + np.random.default_rng(1).integers(0, cfg.vocab_size, size=300, dtype=np.uint16).tofile( + evdir / "active_tokens.bin" + ) + + monkeypatch.setenv("RALPH_SANDBOX", "1") + monkeypatch.setenv("RALPH_SANDBOX_IMAGE", "ralph-eval-sandbox@sha256:" + "a" * 64) + + bad = cfg.max_seq_len // 2 # 8, the OLD miner-controlled value != pinned 16 + + def tampering_sandbox(cfg_, *, container_argv, mounts, out_dir, timeout_s, **kw): + out = Path(out_dir) + out.mkdir(parents=True, exist_ok=True) + n_windows = (300 - 1) // bad + np.save(out / "nlls.npy", np.ones(n_windows * bad, dtype=np.float32)) + (out / "manifest.json").write_text(json.dumps({ + "status": "ok", "seq_len": bad, "tokens_emitted": n_windows * bad, + "benchmark_accuracy": 0.0, "benchmark_examples": 0, "model_config": {}, + })) + return SandboxResult(returncode=0, stdout="ok", stderr="", timed_out=False) + + monkeypatch.setattr(sbx, "run_in_sandbox", tampering_sandbox) + + assert pinned_eval_seq_len(cfg.max_seq_len) == 16 and bad == 8 # the gap under test + ok, detail, result = vv.op4_hidden_eval(ralph_root, proof) + assert not ok and result is None + assert "seq_len mismatch" in detail diff --git a/tests/test_sandbox_eval.py b/tests/test_sandbox_eval.py index a07a876..1cbd55b 100644 --- a/tests/test_sandbox_eval.py +++ b/tests/test_sandbox_eval.py @@ -59,7 +59,9 @@ def test_sandbox_eval_reduction_matches_in_process_val_bpb(tmp_path): saved = np.load(out_dir / "nlls.npy") manifest = json.loads((out_dir / "manifest.json").read_text()) assert manifest["status"] == "ok" - seq_len = cfg.max_seq_len // 2 + from eval.val_bpb import pinned_eval_seq_len + seq_len = pinned_eval_seq_len(cfg.max_seq_len) # host-pinned, not max_seq_len//2 + assert manifest["seq_len"] == seq_len assert saved.shape[0] == expected_token_count(len(tokens), seq_len) assert manifest["tokens_emitted"] == saved.shape[0] @@ -122,9 +124,11 @@ def fake_run_in_sandbox(cfg_, *, container_argv, mounts, out_dir, timeout_s, **k assert "sandboxed" in detail # /out host scratch is removed on exit — no per-submission /tmp leak. assert set(glob.glob(pat)) == before, "sandbox /out scratch leaked" - ref = compute_val_bpb(model, tokens, cfg.max_seq_len // 2, bytes_per_token=4.0) + from eval.val_bpb import pinned_eval_seq_len + ref = compute_val_bpb(model, tokens, pinned_eval_seq_len(cfg.max_seq_len), bytes_per_token=4.0) assert result.val_bpb == pytest.approx(ref["val_bpb"], rel=1e-4) assert result.tokens_evaluated == ref["tokens_evaluated"] + assert result.val_seq_len == pinned_eval_seq_len(cfg.max_seq_len) def test_op4_sandbox_fails_closed_when_runtime_unavailable(tmp_path, monkeypatch): diff --git a/validator/audit.py b/validator/audit.py index 6d3d9fe..8bdc7a0 100644 --- a/validator/audit.py +++ b/validator/audit.py @@ -27,6 +27,7 @@ from model import RalphBase, RalphConfig from eval import run_hidden_eval +from eval.val_bpb import pinned_eval_seq_len from proof.runner import run_proof_test @@ -116,7 +117,7 @@ def run_audit( model = model.cuda() eval_result = run_hidden_eval( model, ralph_root / "eval" / "private", - seq_len=cfg.max_seq_len // 2, + seq_len=pinned_eval_seq_len(cfg.max_seq_len), ) # Hidden-eval on the miner's checkpoint for comparison — same SAFE loader. @@ -128,7 +129,7 @@ def run_audit( miner_model = miner_model.cuda() miner_eval = run_hidden_eval( miner_model, ralph_root / "eval" / "private", - seq_len=cfg.max_seq_len // 2, + seq_len=pinned_eval_seq_len(cfg.max_seq_len), ) miner_bpb = miner_eval.val_bpb diff --git a/validator/audit_report.py b/validator/audit_report.py index cf0420c..3023ebd 100644 --- a/validator/audit_report.py +++ b/validator/audit_report.py @@ -127,7 +127,8 @@ def _eval_input_for(scored: dict, seed: int) -> dict: # lets an auditor confirm it re-runs over the identical held-out data. "sealed_stream_manifest_hash": scored.get("sealed_stream_manifest_hash"), "seed": seed, - # The context length the hidden-eval used (RalphConfig.max_seq_len//2). + # The context length the hidden-eval used: validator-pinned + # min(EVAL_SEQ_LEN, max_seq_len), so an auditor re-runs the same window. "val_seq_len": scored.get("val_seq_len"), "ladder_rungs": list(_STANDARD_LADDER_RUNGS_LABELED), } diff --git a/validator/eval_in_workdir.py b/validator/eval_in_workdir.py index 894dabc..5cd8598 100644 --- a/validator/eval_in_workdir.py +++ b/validator/eval_in_workdir.py @@ -56,6 +56,7 @@ def main() -> int: from model import RalphBase, RalphConfig from eval import run_hidden_eval + from eval.val_bpb import pinned_eval_seq_len except Exception as e: print(f"ERROR: import failed: {e}", file=sys.stderr) return 1 @@ -103,7 +104,11 @@ def main() -> int: try: eval_root = workdir if (workdir / "eval" / "private").is_dir() else ralph_root - result = run_hidden_eval(model, eval_root / "eval" / "private", seq_len=cfg.max_seq_len // 2) + # Validator-pinned eval window — NOT the miner-controlled cfg.max_seq_len//2. + result = run_hidden_eval( + model, eval_root / "eval" / "private", + seq_len=pinned_eval_seq_len(cfg.max_seq_len), + ) except Exception as e: print(f"ERROR: hidden_eval crashed: {e}", file=sys.stderr) return 2 diff --git a/validator/sandbox_eval.py b/validator/sandbox_eval.py index 18db98d..2893449 100644 --- a/validator/sandbox_eval.py +++ b/validator/sandbox_eval.py @@ -76,7 +76,7 @@ def run_sandbox_eval( # Trusted helpers FIRST (canonical/installed), before the workdir goes on the # path — miner code in workdir must not be able to shadow the reducer. from eval.benchmark import compute_benchmark_score - from eval.val_bpb import load_eval_tokens, per_position_nlls + from eval.val_bpb import load_eval_tokens, per_position_nlls, pinned_eval_seq_len sys.path.insert(0, str(Path(workdir).resolve())) import torch @@ -99,7 +99,10 @@ def run_sandbox_eval( if torch.cuda.is_available(): model = model.cuda() - seq_len = cfg.max_seq_len // 2 + # Validator-pinned window from the TRUSTED (image-baked) eval package — NOT + # miner-controlled. The host re-derives the same value and rejects the + # manifest if the container echoes anything else. + seq_len = pinned_eval_seq_len(cfg.max_seq_len) eval_dir = Path(eval_dir) tokens = np.asarray(load_eval_tokens(eval_dir / "active_tokens.bin")) nlls = per_position_nlls(model, tokens, seq_len, batch_size) diff --git a/validator/validator.py b/validator/validator.py index e7a6a76..72a726e 100644 --- a/validator/validator.py +++ b/validator/validator.py @@ -658,7 +658,7 @@ def _sandboxed_hidden_eval( hash_target_stream, reduce_token_nlls, ) - from eval.val_bpb import DEFAULT_BYTES_PER_TOKEN, load_eval_tokens + from eval.val_bpb import DEFAULT_BYTES_PER_TOKEN, load_eval_tokens, pinned_eval_seq_len from ralph_bootstrap import RECIPE_DIR from validator.sandbox import Mount, SandboxConfig, SandboxUnavailable, is_pinned_image, run_in_sandbox @@ -709,7 +709,22 @@ def _sandboxed_hidden_eval( return False, "op4 sandbox produced no nlls/manifest output", None manifest = json.loads(man_path.read_text()) + # Host-PIN the eval window: re-derive seq_len from the checkpoint config + # ourselves and REJECT if the container echoed anything else. The manifest + # value is verified, never trusted — a miner cannot widen (or otherwise + # pick) the eval window from inside the container. + try: + expected_seq_len = pinned_eval_seq_len( + _safe_load_checkpoint_config(ckpt_path)["max_seq_len"] + ) + except (KeyError, ValueError, RuntimeError, OSError) as e: + return False, f"op4 sandbox could not derive host seq_len: {e}", None seq_len = int(manifest["seq_len"]) + if seq_len != expected_seq_len: + return False, ( + f"op4 sandbox seq_len mismatch: container echoed {seq_len}, host " + f"pins {expected_seq_len} (miner cannot choose the eval window)" + ), None tokens = np.asarray(load_eval_tokens(eval_dir / "active_tokens.bin")) eval_set_hash = hash_target_stream(tokens) # HOST-computed, not miner-supplied try: