eval: reject non-causal forward() so a miner can't control its own val_bpb

eval/__init__.py

@@ -17,10 +17,20 @@
     select_active_streams,
     write_manifest,
 )
-from .val_bpb import DEFAULT_BYTES_PER_TOKEN, compute_val_bpb, compute_val_bpb_on_stream
+from .val_bpb import (
+    DEFAULT_BYTES_PER_TOKEN,
+    EVAL_SEQ_LEN,
+    NonCausalModelError,
+    assert_causal,
+    compute_val_bpb,
+    compute_val_bpb_on_stream,
+)
 
 __all__ = [
     "DEFAULT_BYTES_PER_TOKEN",
+    "EVAL_SEQ_LEN",
+    "NonCausalModelError",
+    "assert_causal",
     "HiddenEvalResult",
     "MANIFEST_VERSION",
     "SealedStreamBatch",

eval/hidden_eval.py

@@ -21,7 +21,7 @@
 
 from .benchmark import compute_benchmark_score, make_placeholder_examples
 from .downstream.types import DownstreamReport
-from .val_bpb import compute_val_bpb, load_eval_tokens
+from .val_bpb import assert_causal, compute_val_bpb, load_eval_tokens
 
 
 @dataclass
@@ -133,6 +133,11 @@ def run_hidden_eval(
         rng = np.random.default_rng(424242)
         eval_tokens = rng.integers(0, 50257, size=4096, dtype=np.uint16)
 
+    # The validator scores the miner's OWN forward() to compute val_bpb. Reject a
+    # non-causal forward that peeks at the next token (the answer for position t is
+    # input[t+1]) before trusting any score — otherwise val_bpb can be driven to ~0.
+    assert_causal(model, np.asarray(eval_tokens), seq_len=seq_len)
+
     bpb_result = compute_val_bpb(
         model,
         np.asarray(eval_tokens),

eval/val_bpb.py

@@ -39,6 +39,82 @@
 # and for Phase 0 smoke tests that don't construct a sealed pool.
 DEFAULT_BYTES_PER_TOKEN = 4.0
 
+# Validator-pinned eval window. The hidden eval must NOT use a sequence length
+# derived from the miner's checkpoint config (cfg.max_seq_len // 2): a miner can
+# enlarge it to score against an easier, longer-context eval than the king used.
+# Callers cap with min(EVAL_SEQ_LEN, cfg.max_seq_len) so small-context models
+# still load, but no miner can choose a window larger than the validator's.
+EVAL_SEQ_LEN = 512
+
+
+class NonCausalModelError(RuntimeError):
+    """A model's logits at position t depend on tokens AFTER t.
+
+    The validator runs the miner's own forward() to compute val_bpb. compute_val_bpb
+    feeds the model the whole window in one call, and the target for position t is
+    just input[t+1] — so a NON-causal forward can read the answer and emit a perfect
+    prediction, collapsing val_bpb to ~0 (an unbeatable, fraudulent king). Honest
+    causal LMs are invariant to future tokens; this error rejects ones that aren't.
+    """
+
+
+def assert_causal(
+    model: torch.nn.Module,
+    eval_tokens: np.ndarray,
+    seq_len: int,
+    device: torch.device | None = None,
+    n_probes: int = 4,
+    atol: float = 1e-3,
+) -> None:
+    """Reject a model whose logits at position t depend on tokens after t.
+
+    Defeats the look-ahead exploit (the validator scores the miner's own
+    forward(), and the answer for position t — input[t+1] — sits inside the
+    model's input). For several interior split points k, we copy a real eval
+    window and overwrite the FUTURE positions [k+1:] with a DIFFERENT real
+    held-out slice, then check the logits at positions [:k+1] are unchanged.
+
+    Using a *real, different* future (not uniform-random noise) is deliberate:
+    a random future is trivially distinguishable from real text, which would let
+    an adaptive cheat behave causally during the probe and look ahead only on the
+    real eval. A realistic-but-different future closes that evasion — a causal
+    model is still invariant, a look-ahead one is not.
+
+    No-op when the stream is too short to build a base + decoy window (the caller
+    deploys the real held-out shard in production). Raises NonCausalModelError on
+    failure; the caller treats that as a rejected submission.
+
+    `atol` is generous on purpose: an honest causal model's prefix logits are
+    invariant to future tokens by construction (so the difference is ~0), while a
+    look-ahead cheat must move logits by a large margin to drive val_bpb toward 0.
+    The wide gap means a loose tolerance keeps detection certain yet leaves no room
+    for a false positive from GPU/bf16 attention-kernel tiling noise.
+    """
+    eval_tokens = np.asarray(eval_tokens)
+    if seq_len < 2 or len(eval_tokens) < 2 * (seq_len + 1):
+        return
+    if device is None:
+        device = next(model.parameters()).device
+    model.eval()
+    base = torch.from_numpy(eval_tokens[:seq_len].astype(np.int64))[None].to(device)
+    decoy = torch.from_numpy(eval_tokens[seq_len : 2 * seq_len].astype(np.int64))[None].to(device)
+    with torch.no_grad():
+        base_logits, _ = model(base)
+        for i in range(1, n_probes + 1):
+            k = (seq_len * i) // (n_probes + 1)  # interior split points
+            if k < 1 or k >= seq_len - 1:
+                continue
+            if torch.equal(base[:, k + 1 :], decoy[:, k + 1 :]):
+                continue  # identical tail (degenerate stream) — this k proves nothing
+            alt = base.clone()
+            alt[:, k + 1 :] = decoy[:, k + 1 :]  # realistic, different future
+            alt_logits, _ = model(alt)
+            if not torch.allclose(base_logits[:, : k + 1], alt_logits[:, : k + 1], atol=atol):
+                raise NonCausalModelError(
+                    f"logits at positions <= {k} changed when future tokens were "
+                    f"altered — non-causal forward(); val_bpb is untrustworthy, rejected"
+                )
+
 
 def compute_val_bpb(
     model: torch.nn.Module,

tests/test_eval_forward_trust.py

@@ -0,0 +1,96 @@
+"""Eval-integrity tests: the validator must not trust a val_bpb produced by a
+non-causal forward().
+
+The validator scores the miner's OWN forward() to compute val_bpb, feeding the
+whole window in one call. The target for position t is input[t+1] — so a
+non-causal forward can read the answer and emit a perfect prediction, collapsing
+val_bpb to ~0 and crowning an unbeatable, fraudulent king. `assert_causal`
+rejects such models. These tests pin:
+
+  1. an honest causal model passes the probe (no false positive);
+  2. a look-ahead forward (peeks at input[t+1]) is rejected;
+  3. run_hidden_eval surfaces the rejection end-to-end;
+  4. the probe is a no-op on a stream too short to build a base+decoy window;
+  5. the eval window EVAL_SEQ_LEN is a fixed validator constant.
+"""
+
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+
+import numpy as np
+import pytest
+import torch
+import torch.nn as nn
+
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
+
+import ralph_bootstrap  # noqa: F401
+from eval import EVAL_SEQ_LEN, run_hidden_eval
+from eval.val_bpb import NonCausalModelError, assert_causal
+
+VOCAB = 50257  # matches the GPT-2 BPE vocab the eval stream uses
+
+
+class CausalModel(nn.Module):
+    """Genuinely causal: logits[t] depend only on input[t] (a per-position map),
+    a strict subset of [0..t]. Stands in for any honest LM for the probe."""
+
+    def __init__(self, vocab: int = VOCAB, dim: int = 16) -> None:
+        super().__init__()
+        self.emb = nn.Embedding(vocab, dim)
+        self.head = nn.Linear(dim, vocab)
+
+    def forward(self, idx, targets=None):
+        return self.head(self.emb(idx)), None
+
+
+class LookAheadModel(nn.Module):
+    """Malicious forward(): one-hots input[t+1] (== target[t]). The dummy
+    parameter stands in for the trivial structural patch a miner adds to route
+    op4 into the patched-eval path where their forward() runs."""
+
+    def __init__(self, vocab: int = VOCAB) -> None:
+        super().__init__()
+        self.vocab = vocab
+        self.p = nn.Parameter(torch.zeros(1))
+
+    def forward(self, idx, targets=None):
+        B, T = idx.shape
+        logits = torch.full((B, T, self.vocab), -30.0)
+        nxt = idx[:, 1:]
+        ar = torch.arange(T - 1)
+        for b in range(B):
+            logits[b, ar, nxt[b]] = 30.0
+        return logits + self.p * 0, None
+
+
+def _stream(n: int = 4096, seed: int = 0) -> np.ndarray:
+    return np.random.default_rng(seed).integers(0, VOCAB, size=n, dtype=np.uint16)
+
+
+def test_causal_model_passes_probe():
+    # No raise: an honest causal model is invariant to future tokens.
+    assert_causal(CausalModel(), _stream(), seq_len=64)
+
+
+def test_lookahead_model_is_rejected():
+    with pytest.raises(NonCausalModelError):
+        assert_causal(LookAheadModel(), _stream(), seq_len=64)
+
+
+def test_run_hidden_eval_rejects_lookahead(tmp_path: Path):
+    # Empty eval dir -> run_hidden_eval uses its synthetic stream; the probe runs
+    # before compute_val_bpb and rejects the cheat end-to-end.
+    with pytest.raises(NonCausalModelError):
+        run_hidden_eval(LookAheadModel(), tmp_path, seq_len=64)
+
+
+def test_short_stream_is_noop():
+    # Too little data to build base+decoy -> probe skips rather than false-reject.
+    assert_causal(LookAheadModel(), _stream(n=50), seq_len=64)
+
+
+def test_eval_seq_len_is_a_pinned_constant():
+    assert isinstance(EVAL_SEQ_LEN, int) and EVAL_SEQ_LEN > 0

validator/eval_in_workdir.py

@@ -102,8 +102,12 @@ def main() -> int:
         model = model.cuda()
 
     try:
+        from eval import EVAL_SEQ_LEN
         eval_root = workdir if (workdir / "eval" / "private").is_dir() else ralph_root
-        result = run_hidden_eval(model, eval_root / "eval" / "private", seq_len=cfg.max_seq_len // 2)
+        # Pin the eval window validator-side (see op4_hidden_eval) — not the
+        # miner-controlled cfg.max_seq_len.
+        eval_seq_len = min(EVAL_SEQ_LEN, cfg.max_seq_len)
+        result = run_hidden_eval(model, eval_root / "eval" / "private", seq_len=eval_seq_len)
     except Exception as e:
         print(f"ERROR: hidden_eval crashed: {e}", file=sys.stderr)
         return 2

validator/validator.py

@@ -27,7 +27,7 @@
 
 from model import RalphBase, RalphConfig
 
-from eval import HiddenEvalResult, run_hidden_eval
+from eval import EVAL_SEQ_LEN, HiddenEvalResult, run_hidden_eval
 from miner.submit import lookup_handshake, verify_signature
 from proof.mock_attest import (
     MockAttestation,
@@ -587,7 +587,11 @@ def op4_hidden_eval(
         raise
     if torch.cuda.is_available():
         model = model.cuda()
-    result = run_hidden_eval(model, ralph_root / "eval" / "private", seq_len=cfg.max_seq_len // 2)
+    # Pin the eval window validator-side. Using cfg.max_seq_len (miner-controlled)
+    # would let a miner enlarge the eval context to score an easier eval than the
+    # king. Cap at the model's max_seq_len so small-context models still load.
+    eval_seq_len = min(EVAL_SEQ_LEN, cfg.max_seq_len)
+    result = run_hidden_eval(model, ralph_root / "eval" / "private", seq_len=eval_seq_len)
     return True, f"val_bpb={result.val_bpb:.4f} bench={result.benchmark_accuracy:.3f}", result