eval: fail closed when held-out shard/benchmark missing

eval/hidden_eval.py

@@ -13,6 +13,8 @@
 from __future__ import annotations
 
 import json
+import os
+import sys
 from dataclasses import asdict, dataclass
 from pathlib import Path
 
@@ -117,21 +119,50 @@ def _sealed_stream_manifest_hash(
     })
 
 
+def _synthetic_eval_allowed() -> bool:
+    """True iff the operator has explicitly opted into the synthetic-eval fallback.
+
+    OFF by default so a validator whose held-out shard / benchmark is missing
+    FAILS CLOSED instead of silently scoring every checkpoint against random
+    tokens + placeholder questions — which crowns the LEAST-trained model
+    (val_bpb ~3.9, benchmark_accuracy ~1.0 for anything). Mirrors the
+    RALPH_ALLOW_MOCK_ATTESTATION / RALPH_ALLOW_REAL_ATTEST_STUB pattern: set
+    RALPH_ALLOW_SYNTHETIC_EVAL=1 for CPU smoke / testnet only, never on mainnet.
+    """
+    return os.environ.get("RALPH_ALLOW_SYNTHETIC_EVAL") == "1"
+
+
 def run_hidden_eval(
     model: torch.nn.Module,
     eval_dir: Path | str,
     seq_len: int = 256,
     bpb_batch_size: int = 8,
 ) -> HiddenEvalResult:
     eval_dir = Path(eval_dir)
+    allow_synthetic = _synthetic_eval_allowed()
+
     tokens_path = eval_dir / "active_tokens.bin"
     if tokens_path.exists():
         eval_tokens = load_eval_tokens(tokens_path)
-    else:
-        # Phase 0 fallback: synthesize a small reproducible eval token stream
-        # so the smoke test runs without a pre-built eval shard.
+    elif allow_synthetic:
+        # Opt-in reproducible synthetic stream (CPU smoke / testnet only) — never
+        # a silent default. See _synthetic_eval_allowed.
+        print(
+            f"[eval] WARNING: RALPH_ALLOW_SYNTHETIC_EVAL=1 — scoring val_bpb "
+            f"against a SYNTHETIC random token stream (no {tokens_path}). "
+            f"Testnet/CPU only; this must never run on mainnet.",
+            file=sys.stderr,
+        )
         rng = np.random.default_rng(424242)
         eval_tokens = rng.integers(0, 50257, size=4096, dtype=np.uint16)
+    else:
+        raise FileNotFoundError(
+            f"held-out eval stream not found: {tokens_path}. The validator will "
+            f"NOT fall back to random tokens — that silently scores every "
+            f"checkpoint against noise and crowns the least-trained model. "
+            f"Deploy the real held-out shard, or set RALPH_ALLOW_SYNTHETIC_EVAL=1 "
+            f"to opt into the reproducible synthetic stream (CPU smoke / testnet)."
+        )
 
     bpb_result = compute_val_bpb(
         model,
@@ -143,8 +174,21 @@ def run_hidden_eval(
     benchmark_path = eval_dir / "active_benchmark.json"
     if benchmark_path.exists():
         examples = json.loads(benchmark_path.read_text())
-    else:
+    elif allow_synthetic:
+        print(
+            f"[eval] WARNING: RALPH_ALLOW_SYNTHETIC_EVAL=1 — scoring benchmark "
+            f"against PLACEHOLDER examples (no {benchmark_path}); any model "
+            f"scores ~1.0. Testnet/CPU only.",
+            file=sys.stderr,
+        )
         examples = make_placeholder_examples(n=50)
+    else:
+        raise FileNotFoundError(
+            f"held-out benchmark not found: {benchmark_path}. The validator will "
+            f"NOT fall back to placeholder questions (any model scores ~1.0). "
+            f"Deploy the real benchmark mix, or set RALPH_ALLOW_SYNTHETIC_EVAL=1 "
+            f"(CPU smoke / testnet)."
+        )
 
     bench_result = compute_benchmark_score(model, examples)
 

scripts/smoke_test.py

@@ -34,6 +34,13 @@
 # attestation. The mainnet gate (single attested-execution tier) rejects mock;
 # the sim/testnet accepts it behind this flag. Set before importing the stack.
 os.environ.setdefault("RALPH_ALLOW_MOCK_ATTESTATION", "1")
+# This CPU box ships only a tiny dev token shard and no benchmark mix, so the
+# hidden-eval would otherwise fail closed. Opt into the synthetic fallback for
+# the smoke test (testnet relaxation, same spirit as the mock attestation).
+os.environ.setdefault("RALPH_ALLOW_SYNTHETIC_EVAL", "1")
+# The require-GitHub-PR gate (PR #56) rejects local submissions that carry no
+# recipe PR URL; the CPU smoke test submits directly, so relax it here.
+os.environ.setdefault("RALPH_REQUIRE_GH_PR", "0")
 
 sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
 

tests/conftest.py

@@ -0,0 +1,27 @@
+"""Shared pytest fixtures for the Ralph test suite."""
+
+from __future__ import annotations
+
+import os
+
+import pytest
+
+
+@pytest.fixture(autouse=True)
+def _allow_synthetic_eval():
+    """Default the CPU test suite into the synthetic hidden-eval fallback.
+
+    No real held-out shard / benchmark mix ships in-repo, so the hidden-eval
+    would otherwise fail closed (the production default — see
+    eval.hidden_eval._synthetic_eval_allowed). The test suite runs on CPU
+    against the reproducible synthetic stream, same spirit as the mock
+    attestation relaxation. Tests that assert the fail-closed behavior opt out
+    with `monkeypatch.delenv("RALPH_ALLOW_SYNTHETIC_EVAL", raising=False)`.
+    """
+    prev = os.environ.get("RALPH_ALLOW_SYNTHETIC_EVAL")
+    os.environ["RALPH_ALLOW_SYNTHETIC_EVAL"] = "1"
+    yield
+    if prev is None:
+        os.environ.pop("RALPH_ALLOW_SYNTHETIC_EVAL", None)
+    else:
+        os.environ["RALPH_ALLOW_SYNTHETIC_EVAL"] = prev

tests/test_eval_fail_closed.py

@@ -0,0 +1,58 @@
+"""Hidden-eval fail-closed guard.
+
+A validator whose held-out shard / benchmark is missing must NOT silently fall
+back to random tokens + placeholder questions — that scores every checkpoint
+against noise (val_bpb ~3.9, benchmark_accuracy ~1.0) and crowns the
+least-trained model. By default run_hidden_eval raises; the synthetic fallback
+is reachable only behind RALPH_ALLOW_SYNTHETIC_EVAL=1 (CPU smoke / testnet).
+"""
+
+from __future__ import annotations
+
+import numpy as np
+import pytest
+import torch
+
+from eval.hidden_eval import run_hidden_eval
+
+
+class _Tiny(torch.nn.Module):
+    def __init__(self, vocab=50257, dim=8):
+        super().__init__()
+        self.embed = torch.nn.Embedding(vocab, dim)
+        self.out = torch.nn.Linear(dim, vocab)
+
+    def forward(self, x):
+        return self.out(self.embed(x)), None
+
+
+def _model():
+    torch.manual_seed(0)
+    return _Tiny()
+
+
+def _write_tokens(path, n=4096):
+    np.arange(n, dtype=np.uint16).tofile(path)
+
+
+def test_missing_tokens_fails_closed(tmp_path, monkeypatch):
+    """No active_tokens.bin + flag unset → raise, naming the missing shard."""
+    monkeypatch.delenv("RALPH_ALLOW_SYNTHETIC_EVAL", raising=False)
+    with pytest.raises(FileNotFoundError, match="held-out eval stream not found"):
+        run_hidden_eval(_model(), tmp_path, seq_len=32)
+
+
+def test_missing_benchmark_fails_closed(tmp_path, monkeypatch):
+    """Tokens present but no active_benchmark.json + flag unset → still raise."""
+    monkeypatch.delenv("RALPH_ALLOW_SYNTHETIC_EVAL", raising=False)
+    _write_tokens(tmp_path / "active_tokens.bin")
+    with pytest.raises(FileNotFoundError, match="held-out benchmark not found"):
+        run_hidden_eval(_model(), tmp_path, seq_len=32)
+
+
+def test_synthetic_opt_in_runs(tmp_path, monkeypatch):
+    """With RALPH_ALLOW_SYNTHETIC_EVAL=1, the fallback runs (no shard needed)."""
+    monkeypatch.setenv("RALPH_ALLOW_SYNTHETIC_EVAL", "1")
+    res = run_hidden_eval(_model(), tmp_path, seq_len=32)
+    assert res.val_bpb > 0
+    assert res.val_seq_len == 32

tests/test_hidden_eval_schema_versioning.py

@@ -370,14 +370,17 @@ def test_old_dict_without_new_keys_deserializes(self):
         assert r.sealed_stream_manifest_hash is None
         assert r.tail_val_bpb is None
 
-    def test_run_hidden_eval_surfaces_fields(self, tmp_path):
+    def test_run_hidden_eval_surfaces_fields(self, tmp_path, monkeypatch):
         """End-to-end: run_hidden_eval on a tiny model populates val_seq_len,
-        sealed_stream_manifest_hash, and tail_val_bpb (Phase-0 fallback path,
+        sealed_stream_manifest_hash, and tail_val_bpb (synthetic fallback path,
         no on-disk shard needed)."""
         import torch
 
         from eval.hidden_eval import run_hidden_eval
 
+        # tmp_path has no shard → opt into the synthetic fallback explicitly.
+        monkeypatch.setenv("RALPH_ALLOW_SYNTHETIC_EVAL", "1")
+
         class _Tiny(torch.nn.Module):
             def __init__(self, vocab=50257, dim=8):
                 super().__init__()