op4: pin hidden-eval seq_len validator-side (not miner)

eval/val_bpb.py

@@ -39,6 +39,28 @@
 # and for Phase 0 smoke tests that don't construct a sealed pool.
 DEFAULT_BYTES_PER_TOKEN = 4.0
 
+# Validator-pinned hidden-eval window. The eval MUST NOT derive its sequence
+# length from the miner's checkpoint config (the old `cfg.max_seq_len // 2`): a
+# miner could enlarge max_seq_len to be scored against an easier, longer-context
+# eval than the king faced, and every model evaluated on a different window is
+# not comparable. Pinning to a FIXED window makes all models comparable and
+# removes a miner lever. This is a CANONICAL (image-baked / installed) constant —
+# in the sandbox it is read from the trusted eval package, never miner-supplied.
+EVAL_SEQ_LEN = 512
+
+
+def pinned_eval_seq_len(model_max_seq_len: int) -> int:
+    """The validator-pinned hidden-eval window: ``min(EVAL_SEQ_LEN, max_seq_len)``,
+    floored at 2 (compute_val_bpb needs seq_len+1 tokens per window).
+
+    Single source of truth so EVERY eval path (in-process, subprocess, sandbox,
+    audit) pins seq_len identically and the host can independently re-derive and
+    verify the value a container echoes. The cap at the model's own max_seq_len
+    lets small-context models still load; the floor at EVAL_SEQ_LEN means no
+    miner can choose a window LARGER than the validator's.
+    """
+    return max(2, min(EVAL_SEQ_LEN, int(model_max_seq_len)))
+
 
 def compute_val_bpb(
     model: torch.nn.Module,

tests/test_eval_seq_len_pin.py

@@ -0,0 +1,93 @@
+"""Validator-pinned hidden-eval window (EVAL_SEQ_LEN).
+
+The eval seq_len must be the VALIDATOR's, not the miner's. Previously every path
+derived it from the miner's checkpoint (cfg.max_seq_len // 2), letting a miner
+enlarge max_seq_len to be scored on an easier, longer-context eval than the king
+faced — and making different models non-comparable. `pinned_eval_seq_len` is the
+single source of truth; the sandbox host independently re-derives it and REJECTS
+a container that echoes anything else (verified, not trusted).
+"""
+from __future__ import annotations
+
+import dataclasses
+import json
+import sys
+from pathlib import Path
+
+import numpy as np
+import pytest
+import torch
+
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
+
+import ralph_bootstrap  # noqa: F401
+from eval.val_bpb import EVAL_SEQ_LEN, pinned_eval_seq_len
+
+RECIPE_DIR = str(ralph_bootstrap.RECIPE_DIR)
+if RECIPE_DIR not in sys.path:
+    sys.path.insert(0, RECIPE_DIR)
+try:
+    from model import RalphBase, RalphConfig  # noqa: E402
+    _HAVE_MODEL = True
+except Exception:  # noqa: BLE001
+    _HAVE_MODEL = False
+
+
+def test_pinned_seq_len_caps_floors_and_never_miner_widened():
+    assert pinned_eval_seq_len(2048) == EVAL_SEQ_LEN     # large context capped DOWN to the pin
+    assert pinned_eval_seq_len(EVAL_SEQ_LEN) == EVAL_SEQ_LEN
+    assert pinned_eval_seq_len(256) == 256               # small-context model: its own max
+    assert pinned_eval_seq_len(1) == 2                   # floored at 2 (needs seq_len+1 tokens)
+    # The old miner-controlled value for a long-context model is NOT what we pin.
+    assert pinned_eval_seq_len(4096) != 4096 // 2
+
+
+@pytest.mark.skipif(not _HAVE_MODEL, reason="canonical model package not importable")
+def test_sandbox_host_rejects_tampered_seq_len(tmp_path, monkeypatch):
+    """A container that echoes seq_len != the host-pinned value is REJECTED — the
+    host re-derives the window itself and never trusts the manifest."""
+    import validator.sandbox as sbx
+    import validator.validator as vv
+    from validator.sandbox import SandboxResult
+
+    cfg = RalphConfig(
+        vocab_size=64, dim=32, n_layers=2, n_heads=2, head_dim=16,
+        ffn_mult=2.0, max_seq_len=16,
+    )
+    model = RalphBase(cfg)
+    proof = tmp_path / "proof"
+    (proof / "training").mkdir(parents=True)
+    torch.save(
+        {"model": model.state_dict(), "config": dataclasses.asdict(cfg)},
+        proof / "training" / "checkpoint.pt",
+    )
+    (proof / "patch.diff").write_text("")
+    ralph_root = tmp_path / "root"
+    evdir = ralph_root / "eval" / "private"
+    evdir.mkdir(parents=True)
+    np.random.default_rng(1).integers(0, cfg.vocab_size, size=300, dtype=np.uint16).tofile(
+        evdir / "active_tokens.bin"
+    )
+
+    monkeypatch.setenv("RALPH_SANDBOX", "1")
+    monkeypatch.setenv("RALPH_SANDBOX_IMAGE", "ralph-eval-sandbox@sha256:" + "a" * 64)
+
+    bad = cfg.max_seq_len // 2  # 8, the OLD miner-controlled value != pinned 16
+
+    def tampering_sandbox(cfg_, *, container_argv, mounts, out_dir, timeout_s, **kw):
+        out = Path(out_dir)
+        out.mkdir(parents=True, exist_ok=True)
+        n_windows = (300 - 1) // bad
+        np.save(out / "nlls.npy", np.ones(n_windows * bad, dtype=np.float32))
+        (out / "manifest.json").write_text(json.dumps({
+            "status": "ok", "seq_len": bad, "tokens_emitted": n_windows * bad,
+            "benchmark_accuracy": 0.0, "benchmark_examples": 0, "model_config": {},
+        }))
+        return SandboxResult(returncode=0, stdout="ok", stderr="", timed_out=False)
+
+    monkeypatch.setattr(sbx, "run_in_sandbox", tampering_sandbox)
+
+    assert pinned_eval_seq_len(cfg.max_seq_len) == 16 and bad == 8  # the gap under test
+    ok, detail, result = vv.op4_hidden_eval(ralph_root, proof)
+    assert not ok and result is None
+    assert "seq_len mismatch" in detail

tests/test_sandbox_eval.py

@@ -59,7 +59,9 @@ def test_sandbox_eval_reduction_matches_in_process_val_bpb(tmp_path):
     saved = np.load(out_dir / "nlls.npy")
     manifest = json.loads((out_dir / "manifest.json").read_text())
     assert manifest["status"] == "ok"
-    seq_len = cfg.max_seq_len // 2
+    from eval.val_bpb import pinned_eval_seq_len
+    seq_len = pinned_eval_seq_len(cfg.max_seq_len)  # host-pinned, not max_seq_len//2
+    assert manifest["seq_len"] == seq_len
     assert saved.shape[0] == expected_token_count(len(tokens), seq_len)
     assert manifest["tokens_emitted"] == saved.shape[0]
 
@@ -122,9 +124,11 @@ def fake_run_in_sandbox(cfg_, *, container_argv, mounts, out_dir, timeout_s, **k
     assert "sandboxed" in detail
     # /out host scratch is removed on exit — no per-submission /tmp leak.
     assert set(glob.glob(pat)) == before, "sandbox /out scratch leaked"
-    ref = compute_val_bpb(model, tokens, cfg.max_seq_len // 2, bytes_per_token=4.0)
+    from eval.val_bpb import pinned_eval_seq_len
+    ref = compute_val_bpb(model, tokens, pinned_eval_seq_len(cfg.max_seq_len), bytes_per_token=4.0)
     assert result.val_bpb == pytest.approx(ref["val_bpb"], rel=1e-4)
     assert result.tokens_evaluated == ref["tokens_evaluated"]
+    assert result.val_seq_len == pinned_eval_seq_len(cfg.max_seq_len)
 
 
 def test_op4_sandbox_fails_closed_when_runtime_unavailable(tmp_path, monkeypatch):

validator/audit.py

@@ -27,6 +27,7 @@
 from model import RalphBase, RalphConfig
 
 from eval import run_hidden_eval
+from eval.val_bpb import pinned_eval_seq_len
 from proof.runner import run_proof_test
 
 
@@ -116,7 +117,7 @@ def run_audit(
         model = model.cuda()
     eval_result = run_hidden_eval(
         model, ralph_root / "eval" / "private",
-        seq_len=cfg.max_seq_len // 2,
+        seq_len=pinned_eval_seq_len(cfg.max_seq_len),
     )
 
     # Hidden-eval on the miner's checkpoint for comparison — same SAFE loader.
@@ -128,7 +129,7 @@ def run_audit(
         miner_model = miner_model.cuda()
     miner_eval = run_hidden_eval(
         miner_model, ralph_root / "eval" / "private",
-        seq_len=cfg.max_seq_len // 2,
+        seq_len=pinned_eval_seq_len(cfg.max_seq_len),
     )
 
     miner_bpb = miner_eval.val_bpb

validator/audit_report.py

@@ -127,7 +127,8 @@ def _eval_input_for(scored: dict, seed: int) -> dict:
         # lets an auditor confirm it re-runs over the identical held-out data.
         "sealed_stream_manifest_hash": scored.get("sealed_stream_manifest_hash"),
         "seed": seed,
-        # The context length the hidden-eval used (RalphConfig.max_seq_len//2).
+        # The context length the hidden-eval used: validator-pinned
+        # min(EVAL_SEQ_LEN, max_seq_len), so an auditor re-runs the same window.
         "val_seq_len": scored.get("val_seq_len"),
         "ladder_rungs": list(_STANDARD_LADDER_RUNGS_LABELED),
     }

validator/eval_in_workdir.py

@@ -56,6 +56,7 @@ def main() -> int:
         from model import RalphBase, RalphConfig
 
         from eval import run_hidden_eval
+        from eval.val_bpb import pinned_eval_seq_len
     except Exception as e:
         print(f"ERROR: import failed: {e}", file=sys.stderr)
         return 1
@@ -103,7 +104,11 @@ def main() -> int:
 
     try:
         eval_root = workdir if (workdir / "eval" / "private").is_dir() else ralph_root
-        result = run_hidden_eval(model, eval_root / "eval" / "private", seq_len=cfg.max_seq_len // 2)
+        # Validator-pinned eval window — NOT the miner-controlled cfg.max_seq_len//2.
+        result = run_hidden_eval(
+            model, eval_root / "eval" / "private",
+            seq_len=pinned_eval_seq_len(cfg.max_seq_len),
+        )
     except Exception as e:
         print(f"ERROR: hidden_eval crashed: {e}", file=sys.stderr)
         return 2

validator/sandbox_eval.py

@@ -76,7 +76,7 @@ def run_sandbox_eval(
     # Trusted helpers FIRST (canonical/installed), before the workdir goes on the
     # path — miner code in workdir must not be able to shadow the reducer.
     from eval.benchmark import compute_benchmark_score
-    from eval.val_bpb import load_eval_tokens, per_position_nlls
+    from eval.val_bpb import load_eval_tokens, per_position_nlls, pinned_eval_seq_len
 
     sys.path.insert(0, str(Path(workdir).resolve()))
     import torch
@@ -99,7 +99,10 @@ def run_sandbox_eval(
     if torch.cuda.is_available():
         model = model.cuda()
 
-    seq_len = cfg.max_seq_len // 2
+    # Validator-pinned window from the TRUSTED (image-baked) eval package — NOT
+    # miner-controlled. The host re-derives the same value and rejects the
+    # manifest if the container echoes anything else.
+    seq_len = pinned_eval_seq_len(cfg.max_seq_len)
     eval_dir = Path(eval_dir)
     tokens = np.asarray(load_eval_tokens(eval_dir / "active_tokens.bin"))
     nlls = per_position_nlls(model, tokens, seq_len, batch_size)

validator/validator.py

@@ -658,7 +658,7 @@ def _sandboxed_hidden_eval(
         hash_target_stream,
         reduce_token_nlls,
     )
-    from eval.val_bpb import DEFAULT_BYTES_PER_TOKEN, load_eval_tokens
+    from eval.val_bpb import DEFAULT_BYTES_PER_TOKEN, load_eval_tokens, pinned_eval_seq_len
     from ralph_bootstrap import RECIPE_DIR
     from validator.sandbox import Mount, SandboxConfig, SandboxUnavailable, is_pinned_image, run_in_sandbox
 
@@ -709,7 +709,22 @@ def _sandboxed_hidden_eval(
             return False, "op4 sandbox produced no nlls/manifest output", None
 
         manifest = json.loads(man_path.read_text())
+        # Host-PIN the eval window: re-derive seq_len from the checkpoint config
+        # ourselves and REJECT if the container echoed anything else. The manifest
+        # value is verified, never trusted — a miner cannot widen (or otherwise
+        # pick) the eval window from inside the container.
+        try:
+            expected_seq_len = pinned_eval_seq_len(
+                _safe_load_checkpoint_config(ckpt_path)["max_seq_len"]
+            )
+        except (KeyError, ValueError, RuntimeError, OSError) as e:
+            return False, f"op4 sandbox could not derive host seq_len: {e}", None
         seq_len = int(manifest["seq_len"])
+        if seq_len != expected_seq_len:
+            return False, (
+                f"op4 sandbox seq_len mismatch: container echoed {seq_len}, host "
+                f"pins {expected_seq_len} (miner cannot choose the eval window)"
+            ), None
         tokens = np.asarray(load_eval_tokens(eval_dir / "active_tokens.bin"))
         eval_set_hash = hash_target_stream(tokens)  # HOST-computed, not miner-supplied
         try: