From 17674d6029ae08f3861826cacf53ce19f8c8efad Mon Sep 17 00:00:00 2001 From: Bitzy Date: Thu, 25 Jun 2026 12:15:19 +0000 Subject: [PATCH] eval: fail closed when held-out shard/benchmark missing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - run_hidden_eval silently fell back to random tokens (seed 424242) + placeholder benchmark when active_tokens.bin / active_benchmark.json were absent — scoring every checkpoint against noise - that crowned the LEAST-trained model (val_bpb ~3.9, bench 1.0); the signature is live on sn40 (king gain ~0.0586) - now fail closed by default; gate the synthetic fallback behind RALPH_ALLOW_SYNTHETIC_EVAL=1 (mirrors RALPH_ALLOW_MOCK_ATTESTATION) - tests/conftest.py defaults the CPU suite to synthetic; fail-closed tests opt out via monkeypatch.delenv - smoke_test: add synthetic + RALPH_REQUIRE_GH_PR=0 testnet relaxations - new tests/test_eval_fail_closed.py; full suite 901 passed, 7 skipped Co-Authored-By: Claude Opus 4.8 (1M context) --- eval/hidden_eval.py | 52 ++++++++++++++++-- scripts/smoke_test.py | 7 +++ tests/conftest.py | 27 ++++++++++ tests/test_eval_fail_closed.py | 58 +++++++++++++++++++++ tests/test_hidden_eval_schema_versioning.py | 7 ++- 5 files changed, 145 insertions(+), 6 deletions(-) create mode 100644 tests/conftest.py create mode 100644 tests/test_eval_fail_closed.py diff --git a/eval/hidden_eval.py b/eval/hidden_eval.py index 8ef8f1b..e285db8 100644 --- a/eval/hidden_eval.py +++ b/eval/hidden_eval.py @@ -13,6 +13,8 @@ from __future__ import annotations import json +import os +import sys from dataclasses import asdict, dataclass from pathlib import Path @@ -117,6 +119,19 @@ def _sealed_stream_manifest_hash( }) +def _synthetic_eval_allowed() -> bool: + """True iff the operator has explicitly opted into the synthetic-eval fallback. + + OFF by default so a validator whose held-out shard / benchmark is missing + FAILS CLOSED instead of silently scoring every checkpoint against random + tokens + placeholder questions — which crowns the LEAST-trained model + (val_bpb ~3.9, benchmark_accuracy ~1.0 for anything). Mirrors the + RALPH_ALLOW_MOCK_ATTESTATION / RALPH_ALLOW_REAL_ATTEST_STUB pattern: set + RALPH_ALLOW_SYNTHETIC_EVAL=1 for CPU smoke / testnet only, never on mainnet. + """ + return os.environ.get("RALPH_ALLOW_SYNTHETIC_EVAL") == "1" + + def run_hidden_eval( model: torch.nn.Module, eval_dir: Path | str, @@ -124,14 +139,30 @@ def run_hidden_eval( bpb_batch_size: int = 8, ) -> HiddenEvalResult: eval_dir = Path(eval_dir) + allow_synthetic = _synthetic_eval_allowed() + tokens_path = eval_dir / "active_tokens.bin" if tokens_path.exists(): eval_tokens = load_eval_tokens(tokens_path) - else: - # Phase 0 fallback: synthesize a small reproducible eval token stream - # so the smoke test runs without a pre-built eval shard. + elif allow_synthetic: + # Opt-in reproducible synthetic stream (CPU smoke / testnet only) — never + # a silent default. See _synthetic_eval_allowed. + print( + f"[eval] WARNING: RALPH_ALLOW_SYNTHETIC_EVAL=1 — scoring val_bpb " + f"against a SYNTHETIC random token stream (no {tokens_path}). " + f"Testnet/CPU only; this must never run on mainnet.", + file=sys.stderr, + ) rng = np.random.default_rng(424242) eval_tokens = rng.integers(0, 50257, size=4096, dtype=np.uint16) + else: + raise FileNotFoundError( + f"held-out eval stream not found: {tokens_path}. The validator will " + f"NOT fall back to random tokens — that silently scores every " + f"checkpoint against noise and crowns the least-trained model. " + f"Deploy the real held-out shard, or set RALPH_ALLOW_SYNTHETIC_EVAL=1 " + f"to opt into the reproducible synthetic stream (CPU smoke / testnet)." + ) bpb_result = compute_val_bpb( model, @@ -143,8 +174,21 @@ def run_hidden_eval( benchmark_path = eval_dir / "active_benchmark.json" if benchmark_path.exists(): examples = json.loads(benchmark_path.read_text()) - else: + elif allow_synthetic: + print( + f"[eval] WARNING: RALPH_ALLOW_SYNTHETIC_EVAL=1 — scoring benchmark " + f"against PLACEHOLDER examples (no {benchmark_path}); any model " + f"scores ~1.0. Testnet/CPU only.", + file=sys.stderr, + ) examples = make_placeholder_examples(n=50) + else: + raise FileNotFoundError( + f"held-out benchmark not found: {benchmark_path}. The validator will " + f"NOT fall back to placeholder questions (any model scores ~1.0). " + f"Deploy the real benchmark mix, or set RALPH_ALLOW_SYNTHETIC_EVAL=1 " + f"(CPU smoke / testnet)." + ) bench_result = compute_benchmark_score(model, examples) diff --git a/scripts/smoke_test.py b/scripts/smoke_test.py index 0801080..6263c5e 100644 --- a/scripts/smoke_test.py +++ b/scripts/smoke_test.py @@ -34,6 +34,13 @@ # attestation. The mainnet gate (single attested-execution tier) rejects mock; # the sim/testnet accepts it behind this flag. Set before importing the stack. os.environ.setdefault("RALPH_ALLOW_MOCK_ATTESTATION", "1") +# This CPU box ships only a tiny dev token shard and no benchmark mix, so the +# hidden-eval would otherwise fail closed. Opt into the synthetic fallback for +# the smoke test (testnet relaxation, same spirit as the mock attestation). +os.environ.setdefault("RALPH_ALLOW_SYNTHETIC_EVAL", "1") +# The require-GitHub-PR gate (PR #56) rejects local submissions that carry no +# recipe PR URL; the CPU smoke test submits directly, so relax it here. +os.environ.setdefault("RALPH_REQUIRE_GH_PR", "0") sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..b50f1ea --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,27 @@ +"""Shared pytest fixtures for the Ralph test suite.""" + +from __future__ import annotations + +import os + +import pytest + + +@pytest.fixture(autouse=True) +def _allow_synthetic_eval(): + """Default the CPU test suite into the synthetic hidden-eval fallback. + + No real held-out shard / benchmark mix ships in-repo, so the hidden-eval + would otherwise fail closed (the production default — see + eval.hidden_eval._synthetic_eval_allowed). The test suite runs on CPU + against the reproducible synthetic stream, same spirit as the mock + attestation relaxation. Tests that assert the fail-closed behavior opt out + with `monkeypatch.delenv("RALPH_ALLOW_SYNTHETIC_EVAL", raising=False)`. + """ + prev = os.environ.get("RALPH_ALLOW_SYNTHETIC_EVAL") + os.environ["RALPH_ALLOW_SYNTHETIC_EVAL"] = "1" + yield + if prev is None: + os.environ.pop("RALPH_ALLOW_SYNTHETIC_EVAL", None) + else: + os.environ["RALPH_ALLOW_SYNTHETIC_EVAL"] = prev diff --git a/tests/test_eval_fail_closed.py b/tests/test_eval_fail_closed.py new file mode 100644 index 0000000..e9f1842 --- /dev/null +++ b/tests/test_eval_fail_closed.py @@ -0,0 +1,58 @@ +"""Hidden-eval fail-closed guard. + +A validator whose held-out shard / benchmark is missing must NOT silently fall +back to random tokens + placeholder questions — that scores every checkpoint +against noise (val_bpb ~3.9, benchmark_accuracy ~1.0) and crowns the +least-trained model. By default run_hidden_eval raises; the synthetic fallback +is reachable only behind RALPH_ALLOW_SYNTHETIC_EVAL=1 (CPU smoke / testnet). +""" + +from __future__ import annotations + +import numpy as np +import pytest +import torch + +from eval.hidden_eval import run_hidden_eval + + +class _Tiny(torch.nn.Module): + def __init__(self, vocab=50257, dim=8): + super().__init__() + self.embed = torch.nn.Embedding(vocab, dim) + self.out = torch.nn.Linear(dim, vocab) + + def forward(self, x): + return self.out(self.embed(x)), None + + +def _model(): + torch.manual_seed(0) + return _Tiny() + + +def _write_tokens(path, n=4096): + np.arange(n, dtype=np.uint16).tofile(path) + + +def test_missing_tokens_fails_closed(tmp_path, monkeypatch): + """No active_tokens.bin + flag unset → raise, naming the missing shard.""" + monkeypatch.delenv("RALPH_ALLOW_SYNTHETIC_EVAL", raising=False) + with pytest.raises(FileNotFoundError, match="held-out eval stream not found"): + run_hidden_eval(_model(), tmp_path, seq_len=32) + + +def test_missing_benchmark_fails_closed(tmp_path, monkeypatch): + """Tokens present but no active_benchmark.json + flag unset → still raise.""" + monkeypatch.delenv("RALPH_ALLOW_SYNTHETIC_EVAL", raising=False) + _write_tokens(tmp_path / "active_tokens.bin") + with pytest.raises(FileNotFoundError, match="held-out benchmark not found"): + run_hidden_eval(_model(), tmp_path, seq_len=32) + + +def test_synthetic_opt_in_runs(tmp_path, monkeypatch): + """With RALPH_ALLOW_SYNTHETIC_EVAL=1, the fallback runs (no shard needed).""" + monkeypatch.setenv("RALPH_ALLOW_SYNTHETIC_EVAL", "1") + res = run_hidden_eval(_model(), tmp_path, seq_len=32) + assert res.val_bpb > 0 + assert res.val_seq_len == 32 diff --git a/tests/test_hidden_eval_schema_versioning.py b/tests/test_hidden_eval_schema_versioning.py index d223274..c819e19 100644 --- a/tests/test_hidden_eval_schema_versioning.py +++ b/tests/test_hidden_eval_schema_versioning.py @@ -370,14 +370,17 @@ def test_old_dict_without_new_keys_deserializes(self): assert r.sealed_stream_manifest_hash is None assert r.tail_val_bpb is None - def test_run_hidden_eval_surfaces_fields(self, tmp_path): + def test_run_hidden_eval_surfaces_fields(self, tmp_path, monkeypatch): """End-to-end: run_hidden_eval on a tiny model populates val_seq_len, - sealed_stream_manifest_hash, and tail_val_bpb (Phase-0 fallback path, + sealed_stream_manifest_hash, and tail_val_bpb (synthetic fallback path, no on-disk shard needed).""" import torch from eval.hidden_eval import run_hidden_eval + # tmp_path has no shard → opt into the synthetic fallback explicitly. + monkeypatch.setenv("RALPH_ALLOW_SYNTHETIC_EVAL", "1") + class _Tiny(torch.nn.Module): def __init__(self, vocab=50257, dim=8): super().__init__()