diff --git a/Dockerfile.sandbox b/Dockerfile.sandbox new file mode 100644 index 0000000..6fb7e2e --- /dev/null +++ b/Dockerfile.sandbox @@ -0,0 +1,79 @@ +# Ralph validator EXECUTION SANDBOX image. +# +# This is NOT the proof-test/measurement image (that is ./Dockerfile, whose +# digest is the on-chain measurement). This image exists only so the validator +# can run UNTRUSTED miner model code in containment for op4 hidden-eval (and, +# later, the re-train audit). It is pinned by digest and verified by +# validator/sandbox.py:preflight(). +# +# Hardening choices vs the proof image: +# - cuda RUNTIME, not devel: no nvcc/headers. +# - NO git, NO compilers: shrink the in-container attack surface. +# - eval/scoring CODE is baked (canonical); the rotating eval/private DATA is +# NEVER baked — it is a read-only mount at run time, so the moat is not in +# the image layers. +# - runs as a non-root user; the container is additionally launched with +# --network none --read-only --cap-drop ALL --user 65534 etc. by run_in_sandbox. +# +# Build (reproducible; pin the result by digest in SandboxConfig.image): +# DOCKER_BUILDKIT=1 docker build -f Dockerfile.sandbox -t ralph-eval-sandbox:$(git rev-parse --short HEAD) . +# docker inspect --format='{{index .RepoDigests 0}}' ralph-eval-sandbox:... # -> @sha256: for SandboxConfig +# +# Run contract (the host's run_in_sandbox supplies the hardening flags + mounts): +# /canon (ro) canonical recipe tree (model/, ...) +# /in/patch.diff (ro) miner patch +# /in/training/checkpoint.pt (ro) miner checkpoint +# /eval-private/... (ro) held-out shard + benchmark (never baked) +# /out (rw) nlls.npy + manifest.json +# Entrypoint: python -m validator.sandbox_eval /canon /in/patch.diff \ +# /in/training/checkpoint.pt /eval-private /out +# It applies the patch in a tmpfs workdir, runs the (patched) model, and emits +# per-position NLLs + benchmark accuracy; the HOST reduces the crown-critical +# val_bpb from the NLLs and computes the eval-set hash itself. + +FROM nvidia/cuda:12.4.1-runtime-ubuntu22.04 AS base + +ENV DEBIAN_FRONTEND=noninteractive +ENV PYTHONDONTWRITEBYTECODE=1 +ENV PYTHONUNBUFFERED=1 + +# `patch` is required by proof.runner.apply_patch; explicitly NO git / NO gcc. +RUN apt-get update && apt-get install -y --no-install-recommends \ + python3.11 \ + python3.11-venv \ + python3-pip \ + patch \ + && rm -rf /var/lib/apt/lists/* + +RUN python3.11 -m venv /opt/venv +ENV PATH="/opt/venv/bin:$PATH" + +COPY pyproject.toml /app/pyproject.toml +RUN pip install --no-cache-dir --upgrade pip && \ + pip install --no-cache-dir torch --index-url https://download.pytorch.org/whl/cu124 && \ + pip install --no-cache-dir -e /app + +# Canonical CODE only — model/recipe/eval/proof/validator. NO eval/private DATA +# (the .dockerignore excludes it; it is a runtime ro mount). +COPY model/ /app/model/ +COPY recipe/ /app/recipe/ +COPY data/ /app/data/ +COPY eval/ /app/eval/ +COPY calibration/ /app/calibration/ +COPY proof/ /app/proof/ +COPY validator/ /app/validator/ +COPY configs/ /app/configs/ +COPY restricted_files.yaml /app/restricted_files.yaml +COPY ralph_bootstrap.py /app/ralph_bootstrap.py + +WORKDIR /app + +# Verify the entrypoint + reducer import cleanly at build time. +RUN python -c "from validator.sandbox_eval import run_sandbox_eval, prepare_workdir; print('sandbox_eval import ok')" +RUN python -c "from eval.host_reduce import reduce_token_nlls; from eval.val_bpb import per_position_nlls; print('reducer import ok')" + +# Non-root by default (run_in_sandbox also forces --user); 65534 = nobody. +USER 65534:65534 + +ENTRYPOINT ["python", "-m", "validator.sandbox_eval"] +CMD ["--help"] diff --git a/eval/host_reduce.py b/eval/host_reduce.py new file mode 100644 index 0000000..b0d366c --- /dev/null +++ b/eval/host_reduce.py @@ -0,0 +1,118 @@ +"""Host-side val_bpb reduction — make the crowning number validator-produced. + +Today the crowning `val_bpb` for a STRUCTURAL patch is a float printed by the +miner's own code (`eval_in_workdir.py`), which the scorer trusts. This module +moves the reduction to the host: the sandboxed miner model emits the per-position +negative log-likelihood (nats) of each true target token, and the VALIDATOR +computes bpb from that array using the SAME formula as `eval.val_bpb`, holding +`bytes_per_token`, the expected token count, the tail mask, and the eval-set hash +itself. + +Forgery scope (be honest): a miner controls the model code, so it can still emit +fake-low NLLs. Host-side reduction removes the formula / normalization / token- +count / hash forgery classes and centralizes the metric; the loss VALUES are made +trustworthy by the independent re-train audit (the paired BLOCKER — the audit +must recompute, not re-run miner code at a loose tolerance). For the CANONICAL- +architecture path the sandbox runs validator eval code, so the values are already +host-trusted; this primitive is the structural-patch counterpart. + +Equivalence: `reduce_token_nlls` reproduces `eval.val_bpb.compute_val_bpb`'s +`val_bpb` / `tail_val_bpb` / `nll_per_token` exactly given the same per-position +cross-entropies, the same `seq_len`, and the same `bytes_per_token`. +""" +from __future__ import annotations + +import hashlib +import math +from dataclasses import dataclass + +import numpy as np + +LN2 = math.log(2) + + +@dataclass(frozen=True) +class HostReduced: + val_bpb: float + tail_val_bpb: float | None + nll_per_token: float + tokens_evaluated: int + bytes_per_token: float + eval_set_hash: str # sha256 of the HOST's target stream (not miner-supplied) + + +def expected_token_count(stream_len: int, seq_len: int) -> int: + """Number of target positions `compute_val_bpb` scores over a stream of + `stream_len` tokens: non-overlapping windows of (seq_len+1), each + contributing `seq_len` targets. Mirrors val_bpb.compute_val_bpb's packing.""" + # Only full windows of (seq_len+1) contribute; the last partial window is + # skipped (`if len(ids) < seq_len + 1: break`). + full_windows = (stream_len - 1) // seq_len + return full_windows * seq_len + + +def hash_target_stream(target_tokens: np.ndarray) -> str: + """sha256 over the FULL host-held target stream — replaces the miner-printed + 100-token `eval_set_hash` that bound nothing.""" + arr = np.ascontiguousarray(np.asarray(target_tokens, dtype=np.uint16)) + return hashlib.sha256(arr.tobytes()).hexdigest() + + +def reduce_token_nlls( + nlls: np.ndarray, + *, + seq_len: int, + bytes_per_token: float, + expected_tokens: int, + eval_set_hash: str, +) -> HostReduced: + """Reduce a per-position NLL array (nats, canonical window order) into bpb. + + Validates the array against what the host expects before trusting it: + - length MUST equal `expected_tokens` (host-derived from its own stream); + - every value MUST be finite and >= 0 (cross-entropy is non-negative). + Raises ValueError on any violation — the caller treats that as a rejected + submission (never a silent pass). + """ + if bytes_per_token <= 0: + raise ValueError(f"bytes_per_token must be > 0; got {bytes_per_token}") + nlls = np.asarray(nlls, dtype=np.float64) + if nlls.ndim != 1: + raise ValueError(f"nlls must be 1-D; got shape {nlls.shape}") + if nlls.shape[0] != expected_tokens: + raise ValueError( + f"nll count {nlls.shape[0]} != expected {expected_tokens} " + f"(miner returned the wrong number of scored positions)" + ) + if not np.all(np.isfinite(nlls)): + raise ValueError("nll array contains non-finite values") + if np.any(nlls < 0): + raise ValueError("nll array contains negative cross-entropy (impossible)") + + total_nats = float(nlls.sum()) + total_tokens = int(nlls.shape[0]) + total_bytes = total_tokens * bytes_per_token + bpb = total_nats / (LN2 * total_bytes) if total_bytes > 0 else float("inf") + nll_per_token = total_nats / max(total_tokens, 1) + + # Tail probe: positions [seq_len//2:] within each window. The host + # reconstructs the mask from the index — it does not trust a miner tail flag. + tail_start = seq_len // 2 + within = np.arange(total_tokens) % seq_len + tail_mask = within >= tail_start + tail_tokens = int(tail_mask.sum()) + if tail_tokens > 0: + tail_nats = float(nlls[tail_mask].sum()) + tail_bytes = tail_tokens * bytes_per_token + tail_bpb: float | None = tail_nats / (LN2 * tail_bytes) + else: + tail_bpb = None + + return HostReduced( + val_bpb=bpb, + tail_val_bpb=tail_bpb, + nll_per_token=nll_per_token, + tokens_evaluated=total_tokens, + bytes_per_token=bytes_per_token, + eval_set_hash=eval_set_hash, + ) diff --git a/eval/val_bpb.py b/eval/val_bpb.py index 668e5f9..365c8b5 100644 --- a/eval/val_bpb.py +++ b/eval/val_bpb.py @@ -140,6 +140,57 @@ def compute_val_bpb( } +def per_position_nlls( + model: torch.nn.Module, + eval_tokens: np.ndarray, + seq_len: int, + batch_size: int = 8, + device: torch.device | None = None, +) -> np.ndarray: + """Per-position cross-entropy (nats) over the held-out stream, in the SAME + window-row-major order `compute_val_bpb` sums. + + This is the producer side of HOST-side reduction: the sandboxed (untrusted) + miner model emits this array; the validator reduces it with + `eval.host_reduce.reduce_token_nlls` instead of trusting a miner-printed + bpb. `reduce_token_nlls(per_position_nlls(model, ...))` reproduces + `compute_val_bpb(model, ...)["val_bpb"]` exactly (tested). + + Returns a float32 1-D array of length `n_full_windows * seq_len`. + """ + if device is None: + device = next(model.parameters()).device + model.eval() + n = len(eval_tokens) + n_windows = max(1, (n - 1) // seq_len) + chunks: list[np.ndarray] = [] + with torch.no_grad(): + batch_inp: list[torch.Tensor] = [] + batch_tgt: list[torch.Tensor] = [] + for w in range(n_windows): + start = w * seq_len + ids = eval_tokens[start : start + seq_len + 1] + if len(ids) < seq_len + 1: + break + batch_inp.append(torch.from_numpy(ids[:-1].astype(np.int64))) + batch_tgt.append(torch.from_numpy(ids[1:].astype(np.int64))) + if len(batch_inp) == batch_size or w == n_windows - 1: + inp = torch.stack(batch_inp).to(device) + tgt = torch.stack(batch_tgt).to(device) + logits, _ = model(inp) + nll = F.cross_entropy( + logits.view(-1, logits.size(-1)), + tgt.reshape(-1), + reduction="none", + ) + chunks.append(nll.detach().float().cpu().numpy()) + batch_inp.clear() + batch_tgt.clear() + if not chunks: + return np.zeros(0, dtype=np.float32) + return np.concatenate(chunks).astype(np.float32) + + def compute_val_bpb_on_stream( model: torch.nn.Module, batch: SealedStreamBatch, diff --git a/tests/test_host_reduce.py b/tests/test_host_reduce.py new file mode 100644 index 0000000..7cad908 --- /dev/null +++ b/tests/test_host_reduce.py @@ -0,0 +1,97 @@ +"""Host-side val_bpb reduction must reproduce the in-process computation exactly, +and must reject malformed/forged NLL arrays.""" +from __future__ import annotations + +import math + +import numpy as np +import pytest +import torch +import torch.nn.functional as F + +from eval.host_reduce import ( + expected_token_count, + hash_target_stream, + reduce_token_nlls, +) + +LN2 = math.log(2) + + +def test_reduction_matches_in_process_cross_entropy(): + """reduce_token_nlls(per-position NLLs) == the bpb/tail compute_val_bpb gets + from the same logits — the equivalence the host-side move depends on.""" + torch.manual_seed(0) + vocab, seq_len, n_windows, bpt = 11, 8, 5, 4.0 + logits = torch.randn(n_windows, seq_len, vocab) + targets = torch.randint(0, vocab, (n_windows, seq_len)) + + # Reference: how compute_val_bpb reduces (sum over all positions). + total_nats = F.cross_entropy( + logits.view(-1, vocab), targets.reshape(-1), reduction="sum" + ).item() + tokens = n_windows * seq_len + ref_bpb = total_nats / (LN2 * tokens * bpt) + tail_start = seq_len // 2 + tail_nats = F.cross_entropy( + logits[:, tail_start:, :].reshape(-1, vocab), + targets[:, tail_start:].reshape(-1), + reduction="sum", + ).item() + tail_tokens = n_windows * (seq_len - tail_start) + ref_tail = tail_nats / (LN2 * tail_tokens * bpt) + + # The sandbox would emit these per-position NLLs (window-row-major order). + nlls = F.cross_entropy( + logits.view(-1, vocab), targets.reshape(-1), reduction="none" + ).numpy() + + out = reduce_token_nlls( + nlls, seq_len=seq_len, bytes_per_token=bpt, + expected_tokens=tokens, eval_set_hash="deadbeef", + ) + assert out.val_bpb == pytest.approx(ref_bpb, rel=1e-6) + assert out.tail_val_bpb == pytest.approx(ref_tail, rel=1e-6) + assert out.tokens_evaluated == tokens + + +def test_expected_token_count_matches_packing(): + # Full windows of (seq_len+1); partial last window skipped. + assert expected_token_count(41, 8) == 40 # 5 full windows + assert expected_token_count(40, 8) == 32 # 4 full windows + assert expected_token_count(9, 8) == 8 + assert expected_token_count(5, 8) == 0 # no full window → 0 (bpb=inf upstream) + + +def test_rejects_wrong_length(): + with pytest.raises(ValueError, match="wrong number"): + reduce_token_nlls( + np.ones(39), seq_len=8, bytes_per_token=4.0, + expected_tokens=40, eval_set_hash="x", + ) + + +def test_rejects_non_finite_and_negative(): + bad = np.ones(40) + bad[3] = np.inf + with pytest.raises(ValueError, match="non-finite"): + reduce_token_nlls(bad, seq_len=8, bytes_per_token=4.0, expected_tokens=40, eval_set_hash="x") + neg = np.ones(40) + neg[7] = -0.5 + with pytest.raises(ValueError, match="negative"): + reduce_token_nlls(neg, seq_len=8, bytes_per_token=4.0, expected_tokens=40, eval_set_hash="x") + + +def test_rejects_bad_bytes_per_token(): + with pytest.raises(ValueError, match="bytes_per_token"): + reduce_token_nlls(np.ones(40), seq_len=8, bytes_per_token=0.0, expected_tokens=40, eval_set_hash="x") + + +def test_hash_binds_host_stream_and_is_deterministic(): + a = np.arange(100, dtype=np.uint16) + b = np.arange(100, dtype=np.uint16) + c = a.copy() + c[50] = 999 + assert hash_target_stream(a) == hash_target_stream(b) + assert hash_target_stream(a) != hash_target_stream(c) + assert len(hash_target_stream(a)) == 64 diff --git a/tests/test_sandbox.py b/tests/test_sandbox.py new file mode 100644 index 0000000..4cef73d --- /dev/null +++ b/tests/test_sandbox.py @@ -0,0 +1,183 @@ +"""Tests for validator/sandbox.py — the hardened Docker boundary for untrusted +miner code. All run without a Docker daemon (argv builder + mocked preflight).""" +from __future__ import annotations + +from pathlib import Path + +import pytest + +import validator.sandbox as sbx +from validator.sandbox import ( + Mount, + SandboxConfig, + SandboxUnavailable, + build_container_env, + build_docker_argv, + parse_eval_line, + run_in_sandbox, +) + +_IMG = "ralph-eval-sandbox@sha256:" + "a" * 64 + + +def _cfg(**kw) -> SandboxConfig: + return SandboxConfig(image=_IMG, **kw) + + +def _build(cfg=None, mounts=None, out_dir="/tmp/out", env=None): + cfg = cfg or _cfg() + return build_docker_argv( + cfg, + container_argv=["python", "-m", "validator.sandbox_eval", "/in", "/out"], + mounts=mounts or [Mount(Path("/in"), "/in", ro=True)], + out_dir=Path(out_dir), + name="ralph-sbx-test", + env=env if env is not None else build_container_env({"PYTHONPATH": "/scratch/workdir"}), + ) + + +def test_argv_has_all_mandatory_flags(): + argv = _build() + j = " ".join(argv) + for required in ( + "--network=none", "--read-only", "--user=65534:65534", "--cap-drop=ALL", + "--security-opt=no-new-privileges", "--ipc=private", + "--pids-limit=512", "--memory=16g", "--memory-swap=16g", "--cpus=8", + ): + assert required in argv, f"missing {required}" + assert "--tmpfs" in argv + assert any(t.startswith("/scratch:rw") for t in argv) + # single pinned GPU, never "all" + assert "--gpus" in argv and "device=0" in argv + # image pinned by digest + writable out mount + assert _IMG in argv + assert any(t.endswith(":/out:rw") for t in argv) + # nothing dangerous + for bad in ("--privileged", "--pid=host", "--net=host"): + assert bad not in j + + +def test_argv_rejects_unpinned_image(): + with pytest.raises(ValueError, match="digest"): + build_docker_argv( + SandboxConfig(image="ralph-eval-sandbox:latest"), + container_argv=["true"], mounts=[], out_dir=Path("/tmp/o"), + name="x", env={}, + ) + + +def test_argv_rejects_gpus_all_and_forbidden_flags(): + with pytest.raises(ValueError, match="device"): + sbx._assert_argv_safe(["docker", "run", "--gpus", "all", _IMG]) + with pytest.raises(ValueError, match="forbidden"): + sbx._assert_argv_safe(["docker", "run", "--privileged", _IMG]) + + +def test_argv_refuses_secret_mounts(): + for secret in ("/root/.bittensor", "/root/.ralph_validator_enc_key.json", "/root/.ssh"): + with pytest.raises(ValueError, match="secret"): + _build(mounts=[Mount(Path(secret), "/x", ro=True)]) + + +def test_container_env_excludes_secrets_and_rejects_blocklist(): + env = build_container_env({"PYTHONPATH": "/scratch/workdir"}) + for secret in ("RALPH_VALIDATOR_PRIVKEY", "BT_WALLET_PASSWORD", "HF_TOKEN", "GH_TOKEN"): + assert secret not in env + assert env["PYTHONPATH"] == "/scratch/workdir" + with pytest.raises(ValueError): + build_container_env({"RALPH_VALIDATOR_PRIVKEY": "x"}) + + +def test_preflight_fails_closed(monkeypatch): + # Simulate a box with no docker → preflight must raise. + monkeypatch.setattr(sbx, "_check_docker", lambda reasons: reasons.append("docker not on PATH")) + monkeypatch.setattr(sbx, "_check_runc", lambda reasons: None) + monkeypatch.setattr(sbx, "_check_nvidia_toolkit", lambda reasons, require_gpu: None) + monkeypatch.setattr(sbx, "_check_image", lambda reasons, image: None) + with pytest.raises(SandboxUnavailable, match="docker not on PATH"): + sbx.preflight(_cfg()) + + +def test_preflight_passes_when_all_checks_ok(monkeypatch): + for name in ("_check_docker", "_check_runc"): + monkeypatch.setattr(sbx, name, lambda reasons: None) + monkeypatch.setattr(sbx, "_check_nvidia_toolkit", lambda reasons, require_gpu: None) + monkeypatch.setattr(sbx, "_check_image", lambda reasons, image: None) + sbx.preflight(_cfg()) # must not raise + + +def test_run_in_sandbox_is_fail_closed(monkeypatch): + def _boom(cfg, **kw): + raise SandboxUnavailable("no runtime") + monkeypatch.setattr(sbx, "preflight", _boom) + with pytest.raises(SandboxUnavailable): + run_in_sandbox(_cfg(), container_argv=["true"], mounts=[], out_dir=Path("/tmp/o"), timeout_s=10) + + +def test_run_in_sandbox_builds_hardened_argv(monkeypatch, tmp_path): + captured = {} + + class _Proc: + returncode = 0 + stdout = ( + "RALPH_EVAL_RESULT val_bpb=1.234567 benchmark_acc=0.5 tokens_evaluated=100 " + "benchmark_examples=10 eval_set_hash=" + "b" * 64 + "\n" + ) + stderr = "" + + def _fake_run(argv, **kw): + captured["argv"] = argv + return _Proc() + + monkeypatch.setattr(sbx.subprocess, "run", _fake_run) + res = run_in_sandbox( + _cfg(), container_argv=["python", "x.py"], mounts=[Mount(tmp_path, "/in")], + out_dir=tmp_path, timeout_s=30, skip_preflight=True, + ) + assert "--network=none" in captured["argv"] + assert "--cap-drop=ALL" in captured["argv"] + assert res.returncode == 0 + assert res.eval_line is not None + + +def test_run_in_sandbox_timeout_is_killed(monkeypatch, tmp_path): + import subprocess as _sp + killed = {} + + def _fake_run(argv, **kw): + if argv[:2] == ["docker", "kill"]: + killed["name"] = argv[2] + return _sp.CompletedProcess(argv, 0, "", "") + raise _sp.TimeoutExpired(argv, kw.get("timeout", 1)) + + monkeypatch.setattr(sbx.subprocess, "run", _fake_run) + res = run_in_sandbox( + _cfg(), container_argv=["sleep", "999"], mounts=[], out_dir=tmp_path, + timeout_s=1, skip_preflight=True, + ) + assert res.timed_out + assert res.returncode == 124 + assert killed.get("name", "").startswith("ralph-sbx-") + + +def test_parse_eval_line_quantizes_and_drops_fields(): + line = ( + "RALPH_EVAL_RESULT val_bpb=1.2345678 benchmark_acc=0.4999999 " + "tokens_evaluated=4096 benchmark_examples=15 eval_set_hash=" + "c" * 64 + " " + "sealed_stream_manifest_hash=deadbeef tail_val_bpb=1.30 sneaky=exfil" + ) + out = parse_eval_line(line) + assert out["val_bpb"] == 1.235 # round(1.2345678, 3) + assert out["benchmark_acc"] == 0.5 + assert set(out.keys()) == {"val_bpb", "benchmark_acc", "tokens_evaluated", "benchmark_examples", "eval_set_hash"} + assert "sneaky" not in out + assert out["eval_set_hash"] == "c" * 64 + + +def test_parse_eval_line_rejects_garbage_and_bad_hash(): + with pytest.raises(ValueError): + parse_eval_line("not a result line") + out = parse_eval_line( + "RALPH_EVAL_RESULT val_bpb=1.0 benchmark_acc=0.5 tokens_evaluated=1 benchmark_examples=1 eval_set_hash=tooshort" + ) + assert out["eval_set_hash"] == "" # invalid hash dropped, not propagated diff --git a/tests/test_sandbox_eval.py b/tests/test_sandbox_eval.py new file mode 100644 index 0000000..a07a876 --- /dev/null +++ b/tests/test_sandbox_eval.py @@ -0,0 +1,166 @@ +"""End-to-end (CPU, no container): the sandbox entrypoint emits per-position +NLLs whose HOST-side reduction reproduces the in-process val_bpb exactly. This is +the correctness contract the op4 sandbox wiring depends on.""" +from __future__ import annotations + +import dataclasses +import json +import sys + +import numpy as np +import pytest +import torch + +import ralph_bootstrap + +# The canonical model package lives under RECIPE_DIR. +RECIPE_DIR = str(ralph_bootstrap.RECIPE_DIR) +if RECIPE_DIR not in sys.path: + sys.path.insert(0, RECIPE_DIR) + +try: + from model import RalphBase, RalphConfig # noqa: E402 + _HAVE_MODEL = True +except Exception: # noqa: BLE001 + _HAVE_MODEL = False + +pytestmark = pytest.mark.skipif(not _HAVE_MODEL, reason="canonical model package not importable") + + +def _tiny_model_and_ckpt(tmp_path): + torch.manual_seed(0) + cfg = RalphConfig( + vocab_size=64, dim=32, n_layers=2, n_heads=2, head_dim=16, + ffn_mult=2.0, max_seq_len=16, + ) + model = RalphBase(cfg) + ckpt = tmp_path / "checkpoint.pt" + torch.save({"model": model.state_dict(), "config": dataclasses.asdict(cfg)}, ckpt) + return cfg, model, ckpt + + +def test_sandbox_eval_reduction_matches_in_process_val_bpb(tmp_path): + from eval.host_reduce import expected_token_count, reduce_token_nlls + from eval.val_bpb import compute_val_bpb + from validator.sandbox_eval import run_sandbox_eval + + cfg, model, ckpt = _tiny_model_and_ckpt(tmp_path) + rng = np.random.default_rng(7) + tokens = rng.integers(0, cfg.vocab_size, size=200, dtype=np.uint16) + eval_dir = tmp_path / "evald" + eval_dir.mkdir() + tokens.tofile(eval_dir / "active_tokens.bin") # no benchmark file -> bench 0.0 + + out_dir = tmp_path / "out" + # workdir = RECIPE_DIR so the canonical `model` package resolves (no patch). + nlls = run_sandbox_eval(RECIPE_DIR, ckpt, eval_dir, out_dir) + + # Container artifacts exist and are well-formed. + saved = np.load(out_dir / "nlls.npy") + manifest = json.loads((out_dir / "manifest.json").read_text()) + assert manifest["status"] == "ok" + seq_len = cfg.max_seq_len // 2 + assert saved.shape[0] == expected_token_count(len(tokens), seq_len) + assert manifest["tokens_emitted"] == saved.shape[0] + + # HOST reduction of the emitted NLLs == the in-process computation. + ref = compute_val_bpb(model, tokens, seq_len, bytes_per_token=4.0) + host = reduce_token_nlls( + nlls, seq_len=seq_len, bytes_per_token=4.0, + expected_tokens=expected_token_count(len(tokens), seq_len), + eval_set_hash="x", + ) + assert host.val_bpb == pytest.approx(ref["val_bpb"], rel=1e-5) + assert host.tail_val_bpb == pytest.approx(ref["tail_val_bpb"], rel=1e-5) + assert host.tokens_evaluated == ref["tokens_evaluated"] + + +def test_op4_routes_through_sandbox_and_host_reduces(tmp_path, monkeypatch): + """RALPH_SANDBOX=1 -> op4 runs the model in the container (mocked) and the + HOST reduces val_bpb from the emitted nlls — matching the in-process value.""" + import validator.sandbox as sbx + import validator.validator as vv + from eval.val_bpb import compute_val_bpb + from validator.sandbox import SandboxResult + + cfg, model, _ = _tiny_model_and_ckpt(tmp_path) + proof = tmp_path / "proof" + (proof / "training").mkdir(parents=True) + torch.save( + {"model": model.state_dict(), "config": dataclasses.asdict(cfg)}, + proof / "training" / "checkpoint.pt", + ) + (proof / "patch.diff").write_text("") # canonical (no structural change) + ralph_root = tmp_path / "root" + evdir = ralph_root / "eval" / "private" + evdir.mkdir(parents=True) + tokens = np.random.default_rng(11).integers(0, cfg.vocab_size, size=300, dtype=np.uint16) + tokens.tofile(evdir / "active_tokens.bin") + + monkeypatch.setenv("RALPH_SANDBOX", "1") + monkeypatch.setenv("RALPH_SANDBOX_IMAGE", "ralph-eval-sandbox@sha256:" + "a" * 64) + + def fake_run_in_sandbox(cfg_, *, container_argv, mounts, out_dir, timeout_s, **kw): + # Simulate the container: run the real entrypoint against the HOST paths + # the mounts point at, writing nlls.npy + manifest.json into out_dir. + from validator.sandbox_eval import run_prepare_and_eval + canon = next(m.host for m in mounts if m.container == "/canon") + indir = next(m.host for m in mounts if m.container == "/in") + evald = next(m.host for m in mounts if m.container == "/eval-private") + run_prepare_and_eval(canon, indir / "patch.diff", indir / "training" / "checkpoint.pt", evald, out_dir) + return SandboxResult(returncode=0, stdout="ok", stderr="", timed_out=False) + + monkeypatch.setattr(sbx, "run_in_sandbox", fake_run_in_sandbox) + + import glob + import tempfile as _tf + pat = _tf.gettempdir().rstrip("/") + "/ralph_sbx_out_*" + before = set(glob.glob(pat)) + + ok, detail, result = vv.op4_hidden_eval(ralph_root, proof) + assert ok, detail + assert "sandboxed" in detail + # /out host scratch is removed on exit — no per-submission /tmp leak. + assert set(glob.glob(pat)) == before, "sandbox /out scratch leaked" + ref = compute_val_bpb(model, tokens, cfg.max_seq_len // 2, bytes_per_token=4.0) + assert result.val_bpb == pytest.approx(ref["val_bpb"], rel=1e-4) + assert result.tokens_evaluated == ref["tokens_evaluated"] + + +def test_op4_sandbox_fails_closed_when_runtime_unavailable(tmp_path, monkeypatch): + """RALPH_SANDBOX=1 but the runtime preflight fails -> op4 REJECTS (no fallback).""" + import validator.sandbox as sbx + import validator.validator as vv + from validator.sandbox import SandboxUnavailable + + cfg, model, _ = _tiny_model_and_ckpt(tmp_path) + proof = tmp_path / "proof" + (proof / "training").mkdir(parents=True) + torch.save({"model": model.state_dict(), "config": dataclasses.asdict(cfg)}, proof / "training" / "checkpoint.pt") + (proof / "patch.diff").write_text("") + evdir = tmp_path / "root" / "eval" / "private" + evdir.mkdir(parents=True) + np.zeros(300, dtype=np.uint16).tofile(evdir / "active_tokens.bin") + + monkeypatch.setenv("RALPH_SANDBOX", "1") + monkeypatch.setenv("RALPH_SANDBOX_IMAGE", "ralph-eval-sandbox@sha256:" + "a" * 64) + + def boom(*a, **k): + raise SandboxUnavailable("docker not on PATH") + monkeypatch.setattr(sbx, "run_in_sandbox", boom) + + ok, detail, result = vv.op4_hidden_eval(tmp_path / "root", proof) + assert not ok and result is None and "FAIL-CLOSED" in detail + + +def test_prepare_workdir_copies_canon_and_applies_empty_patch(tmp_path): + from validator.sandbox_eval import prepare_workdir + + canon = tmp_path / "canon" + (canon / "model").mkdir(parents=True) + (canon / "model" / "x.py").write_text("# canonical\n") + patch = tmp_path / "patch.diff" + patch.write_text("") # empty patch → no-op + + wd = prepare_workdir(canon, patch, tmp_path / "workdir") + assert (wd / "model" / "x.py").read_text() == "# canonical\n" diff --git a/validator/sandbox.py b/validator/sandbox.py new file mode 100644 index 0000000..c491161 --- /dev/null +++ b/validator/sandbox.py @@ -0,0 +1,379 @@ +"""Hardened Docker sandbox for executing UNTRUSTED miner code on the validator. + +The validator must run miner-authored Python (the patched `model.py` for op4 +hidden-eval, and the full patched recipe for the re-train audit). Today that runs +in a bare same-user subprocess with the validator's filesystem, network, and — +until the env-sanitize stopgap — its secrets. This module contains that +execution inside a hardened container so a malicious submission cannot reach the +seal privkey, the wallet, the hidden eval set, or the host. + +Design reference: Ralph-Validator-Sandbox-Plan.md. + +Non-negotiable controls (enforced in `build_docker_argv`, asserted in tests): + --network none · --read-only · non-root · --cap-drop ALL · + --security-opt no-new-privileges · --pids-limit · --memory(==swap) · --cpus · + --ipc=private · a SINGLE pinned GPU device (never `--gpus all`) · NO secrets in + env or mounts · image pinned by digest · a host-side wall-clock watchdog that + `docker kill`s on overrun. + +FAIL-CLOSED: `preflight()` raises `SandboxUnavailable` if the runtime or its +hardening config cannot be verified. Callers MUST treat that as a rejected +submission and MUST NOT fall back to a bare subprocess. + +This first cut provides the runner, the preflight gate, and the result parser. +Call-site wiring (op4 + audit), the in-container entrypoint, and the sandbox +image are landed alongside it. +""" +from __future__ import annotations + +import re +import shutil +import subprocess +import uuid +from dataclasses import dataclass +from pathlib import Path + +from proof.runner import _TRAINING_ENV_BLOCKLIST, _redacted + +# Version floors that close the 2024-2026 escape classes (Leaky Vessels, the +# Nov-2025 runc masked-path trio, NVIDIAScape). See the design doc §5. +MIN_DOCKER_VERSION = (25, 0, 2) +MIN_RUNC_VERSION = (1, 2, 8) +MIN_TOOLKIT_VERSION = (1, 17, 8) + +# Flags that must NEVER appear in a sandbox invocation. +FORBIDDEN_FLAGS = ("--privileged", "--pid=host", "--net=host", "--network=host", "--userns=host") + +_RESULT_RE = re.compile(r"^RALPH_EVAL_RESULT ") + + +class SandboxUnavailable(RuntimeError): + """The sandbox runtime or its required hardening config is not verifiably + present. Callers MUST reject the submission — never fall back to bare exec.""" + + +@dataclass(frozen=True) +class Mount: + """A read-only-by-default bind mount into the sandbox.""" + host: Path + container: str + ro: bool = True + + def as_flag(self) -> str: + return f"{Path(self.host).resolve()}:{self.container}:{'ro' if self.ro else 'rw'}" + + +@dataclass(frozen=True) +class SandboxConfig: + """Tunables for one sandboxed run. `image` MUST be pinned by digest.""" + image: str # e.g. "ralph-eval-sandbox@sha256:..." + gpu_device: int | None = 0 # a SINGLE device index, or None for CPU-only + memory: str = "16g" + cpus: str = "8" + pids_limit: int = 512 + uid_gid: str = "65534:65534" # nobody:nogroup + scratch_size: str = "8g" + seccomp_profile: Path | None = None # path to a custom seccomp json + apparmor_profile: str | None = None # loaded profile name, e.g. "ralph-sandbox" + network: str = "none" # never anything else for untrusted code + + +@dataclass +class SandboxResult: + returncode: int + stdout: str + stderr: str # already redacted + timed_out: bool + eval_line: str | None = None # the parsed RALPH_EVAL_RESULT line, if present + + +# --------------------------------------------------------------------------- +# Argv construction (PURE + unit-tested — this is the security boundary) +# --------------------------------------------------------------------------- + +def build_container_env(env_extra: dict[str, str] | None) -> dict[str, str]: + """Minimal, explicit container env. Never forwards host env; rejects any + blocklisted key. The image supplies PATH/torch; we add only safe knobs.""" + env: dict[str, str] = { + "PYTHONUNBUFFERED": "1", + "PYTHONDONTWRITEBYTECODE": "1", + "HOME": "/scratch", + "TMPDIR": "/scratch/tmp", + } + for k, v in (env_extra or {}).items(): + if k in _TRAINING_ENV_BLOCKLIST: + raise ValueError(f"refusing to inject blocklisted env var into sandbox: {k}") + env[k] = v + return env + + +def build_docker_argv( + cfg: SandboxConfig, + *, + container_argv: list[str], + mounts: list[Mount], + out_dir: Path, + name: str, + env: dict[str, str], +) -> list[str]: + """Build the hardened `docker run` argv. Raises on any unsafe input so a + mistake is a crash, not a silent weakening of the boundary.""" + if "@sha256:" not in cfg.image: + raise ValueError(f"sandbox image must be pinned by digest, got {cfg.image!r}") + if cfg.gpu_device is not None and cfg.gpu_device < 0: + raise ValueError("gpu_device must be a non-negative index or None") + for k in env: + if k in _TRAINING_ENV_BLOCKLIST: + raise ValueError(f"blocklisted env var in sandbox env: {k}") + + argv: list[str] = [ + "docker", "run", "--rm", + "--name", name, + f"--network={cfg.network}", + "--read-only", + f"--user={cfg.uid_gid}", + "--cap-drop=ALL", + "--security-opt=no-new-privileges", + "--ipc=private", + f"--pids-limit={cfg.pids_limit}", + f"--memory={cfg.memory}", + f"--memory-swap={cfg.memory}", # == memory ⇒ swap disabled + f"--cpus={cfg.cpus}", + "--tmpfs", f"/scratch:rw,nosuid,nodev,noexec,size={cfg.scratch_size},mode=1777", + ] + if cfg.seccomp_profile is not None: + argv += ["--security-opt", f"seccomp={Path(cfg.seccomp_profile).resolve()}"] + if cfg.apparmor_profile is not None: + argv += ["--security-opt", f"apparmor={cfg.apparmor_profile}"] + if cfg.gpu_device is not None: + # A SINGLE pinned device. Never "all" — that exposes every GPU + the + # full driver ioctl surface for every device. + # + # Documented residual: the container is ephemeral (--rm), so its CUDA + # context + VRAM are freed on exit, but freed VRAM is not guaranteed + # ZEROED before the next container reuses it. The eval set is mounted + # read-only into EVERY submission's container anyway, so reading a prior + # run's residual VRAM gives a miner nothing they don't already get — the + # only leak is a prior miner's model weights to a later one (low). The + # clean mitigation is driver/MIG-level scrub, not an in-band hack. + argv += ["--gpus", f"device={cfg.gpu_device}"] + + for m in mounts: + argv += ["-v", m.as_flag()] + # The one writable output mount (validated, single result line). + argv += ["-v", f"{Path(out_dir).resolve()}:/out:rw"] + + for k, v in env.items(): + argv += ["-e", f"{k}={v}"] + + argv.append(cfg.image) + argv += container_argv + + _assert_argv_safe(argv) + return argv + + +def _assert_argv_safe(argv: list[str]) -> None: + """Defense-in-depth: reject an argv that smuggled in a dangerous flag or a + secret-bearing mount. A bug here should fail the build, not the box.""" + joined = " ".join(argv) + for bad in FORBIDDEN_FLAGS: + if bad in argv or bad in joined: + raise ValueError(f"forbidden flag in sandbox argv: {bad}") + if "--gpus" in argv: + i = argv.index("--gpus") + if i + 1 < len(argv) and argv[i + 1] in ("all", "--gpus=all") or "--gpus=all" in joined: + raise ValueError("refusing `--gpus all`: pin a single device") + # No secret paths or the docker socket may be mounted. + secret_markers = ( + "docker.sock", ".bittensor", "ralph_validator_enc_key", + "/root/.ssh", "id_rsa", "wallet", + ) + for tok in argv: + if tok == "-v" or tok.startswith("-v"): + continue + if ":" in tok and ("ro" in tok.split(":")[-1] or "rw" in tok.split(":")[-1]): + low = tok.lower() + for marker in secret_markers: + if marker in low: + raise ValueError(f"refusing to mount a secret path into sandbox: {tok}") + + +# --------------------------------------------------------------------------- +# Preflight (FAIL-CLOSED runtime + hardening verification) +# --------------------------------------------------------------------------- + +def _run(cmd: list[str], timeout: int = 15) -> subprocess.CompletedProcess: + return subprocess.run(cmd, capture_output=True, text=True, timeout=timeout) + + +def _parse_version(text: str) -> tuple[int, ...] | None: + m = re.search(r"(\d+)\.(\d+)\.(\d+)", text) + return tuple(int(x) for x in m.groups()) if m else None + + +def _check_docker(reasons: list[str]) -> None: + if shutil.which("docker") is None: + reasons.append("docker not on PATH") + return + try: + info = _run(["docker", "info", "--format", "{{json .}}"]) + except Exception as e: # noqa: BLE001 + reasons.append(f"`docker info` failed: {e}") + return + if info.returncode != 0: + reasons.append("docker daemon unreachable") + return + ver = _run(["docker", "version", "--format", "{{.Server.Version}}"]) + v = _parse_version(ver.stdout) + if v is None or v < MIN_DOCKER_VERSION: + reasons.append(f"docker engine {v} < required {MIN_DOCKER_VERSION}") + # userns-remap must be ACTIVE (the highest-value 2025/26 control). + sec = _run(["docker", "info", "--format", "{{.SecurityOptions}}"]) + if "name=userns" not in sec.stdout: + reasons.append("docker userns-remap is NOT active (daemon needs userns-remap)") + + +def _check_runc(reasons: list[str]) -> None: + if shutil.which("runc") is None: + return # rootless/containerd-shim setups may not expose runc on PATH + v = _parse_version(_run(["runc", "--version"]).stdout) + if v is None or v < MIN_RUNC_VERSION: + reasons.append(f"runc {v} < required {MIN_RUNC_VERSION} (Leaky Vessels / masked-path)") + + +def _check_nvidia_toolkit(reasons: list[str], require_gpu: bool) -> None: + if not require_gpu: + return + if shutil.which("nvidia-ctk") is None: + reasons.append("nvidia-container-toolkit (nvidia-ctk) not found") + return + v = _parse_version(_run(["nvidia-ctk", "--version"]).stdout) + if v is None or v < MIN_TOOLKIT_VERSION: + reasons.append(f"nvidia-container-toolkit {v} < required {MIN_TOOLKIT_VERSION} (NVIDIAScape)") + # The CVE-2025-23266 / CVE-2024-0132 mitigation is a CONFIG flag, not a + # version: the cuda-compat-lib hook must be disabled. + cfg_path = Path("/etc/nvidia-container-runtime/config.toml") + if cfg_path.exists(): + text = cfg_path.read_text(errors="ignore") + if "disable-cuda-compat-lib-hook" not in text or "disable-cuda-compat-lib-hook = true" not in text: + reasons.append("nvidia toolkit `disable-cuda-compat-lib-hook=true` not set") + else: + reasons.append("nvidia-container-runtime config.toml not found (cannot verify hook-disable)") + + +def _check_image(reasons: list[str], image: str) -> None: + if "@sha256:" not in image: + reasons.append(f"sandbox image not pinned by digest: {image}") + return + res = _run(["docker", "image", "inspect", image]) + if res.returncode != 0: + reasons.append(f"sandbox image not present locally: {image}") + + +def preflight(cfg: SandboxConfig, *, require_gpu: bool = True) -> None: + """Verify the sandbox runtime + hardening config. Raise SandboxUnavailable + listing EVERY failed check. This is the fail-closed gate; never bypass it in + production (RALPH_TEST_MODE is honored only for CI stubbing by callers).""" + reasons: list[str] = [] + _check_docker(reasons) + _check_runc(reasons) + _check_nvidia_toolkit(reasons, require_gpu) + _check_image(reasons, cfg.image) + if reasons: + raise SandboxUnavailable("sandbox preflight failed: " + "; ".join(reasons)) + + +# --------------------------------------------------------------------------- +# Run (preflight → docker run with a host-side watchdog) +# --------------------------------------------------------------------------- + +def run_in_sandbox( + cfg: SandboxConfig, + *, + container_argv: list[str], + mounts: list[Mount], + out_dir: Path, + timeout_s: int, + env_extra: dict[str, str] | None = None, + skip_preflight: bool = False, +) -> SandboxResult: + """Run `container_argv` in a hardened container. Fail-closed: preflight must + pass (unless `skip_preflight`, which callers set ONLY under RALPH_TEST_MODE). + A wall-clock overrun triggers `docker kill`, tearing down the PID namespace. + """ + if not skip_preflight: + preflight(cfg, require_gpu=cfg.gpu_device is not None) + + name = f"ralph-sbx-{uuid.uuid4().hex[:12]}" + env = build_container_env(env_extra) + argv = build_docker_argv( + cfg, container_argv=container_argv, mounts=mounts, + out_dir=out_dir, name=name, env=env, + ) + + timed_out = False + try: + proc = subprocess.run(argv, capture_output=True, text=True, timeout=timeout_s) + rc, out, err = proc.returncode, proc.stdout or "", proc.stderr or "" + except subprocess.TimeoutExpired as e: + timed_out = True + # Tear down the container; --rm cleans up after the kill. + try: + _run(["docker", "kill", name], timeout=20) + except Exception: # noqa: BLE001 + pass + rc = 124 + out = (e.stdout.decode() if isinstance(e.stdout, bytes) else (e.stdout or "")) if e.stdout else "" + err = f"sandbox timed out after {timeout_s}s and was killed" + + eval_line = next((ln for ln in out.splitlines() if _RESULT_RE.match(ln)), None) + return SandboxResult( + returncode=rc, + stdout=out, + stderr=_redacted(err), + timed_out=timed_out, + eval_line=eval_line, + ) + + +# --------------------------------------------------------------------------- +# Output handling (cap the covert channel; trust only typed, quantized fields) +# --------------------------------------------------------------------------- + +# Ranking precision: val_bpb decisive margins are ~0.01-0.05 bpb, so 3 decimals +# is far below ranking resolution while capping the per-field covert-channel +# capacity. NOTE: this only caps leakage; the real fix is host-side reduction so +# the score is validator-produced, not miner-printed (design doc §7 #2). +_QUANTIZE_DECIMALS = 3 +_HEX64 = re.compile(r"^[0-9a-f]{64}$") + + +def parse_eval_line(line: str) -> dict[str, object]: + """Strictly parse + type-validate + quantize the single result line. Drops + every field the validator does not rank on. Raises ValueError on anything + malformed (never echo raw miner stdout downstream).""" + if not _RESULT_RE.match(line): + raise ValueError("not a RALPH_EVAL_RESULT line") + fields: dict[str, str] = {} + for tok in line[len("RALPH_EVAL_RESULT "):].split(): + if "=" in tok: + k, v = tok.split("=", 1) + fields[k] = v + + def _qfloat(key: str) -> float: + return round(float(fields[key]), _QUANTIZE_DECIMALS) + + out: dict[str, object] = { + "val_bpb": _qfloat("val_bpb"), + "benchmark_acc": _qfloat("benchmark_acc"), + "tokens_evaluated": int(fields["tokens_evaluated"]), + "benchmark_examples": int(fields["benchmark_examples"]), + } + h = fields.get("eval_set_hash", "") + if not _HEX64.match(h): + # Not fatal here (host should recompute over the full stream — §7 #2), + # but never propagate an unvalidated string. + h = "" + out["eval_set_hash"] = h + return out diff --git a/validator/sandbox_eval.py b/validator/sandbox_eval.py new file mode 100644 index 0000000..18db98d --- /dev/null +++ b/validator/sandbox_eval.py @@ -0,0 +1,177 @@ +#!/usr/bin/env python3 +"""In-container entrypoint for the validator execution sandbox (op4 hidden-eval). + +Runs INSIDE the hardened container (network none, non-root, read-only rootfs, no +secrets). It loads the miner's (possibly patched) model, runs the forward pass +over the held-out stream, and emits the **per-position NLLs** — NOT a reduced +score. The host (`validator/sandbox.py` → `eval.host_reduce`) computes val_bpb +from that array, owning the formula, token count, bytes_per_token, tail mask, and +eval-set hash. The container never prints the crowning number. + +The eval/scoring code is the CANONICAL (image-baked / installed) package; only the +MODEL is imported from the patched workdir. Trusted helpers are imported BEFORE +the workdir is placed on sys.path so miner code cannot shadow them. + +Container layout (mounts, all ro except /out): + /work/workdir patched recipe tree (model/, ... ; already patch-applied) + /in/checkpoint.pt the miner's checkpoint + /eval-private/active_tokens.bin the held-out stream (host-mounted ro) + /out the single writable dir — receives nlls.npy + manifest.json + +Outputs: + /out/nlls.npy float32 per-position NLLs (window-row-major order) + /out/manifest.json {status, seq_len, tokens_emitted, model_config} + +Exit codes: 0 ok · 1 setup/import/load failure · 2 eval crash · 3 bad args. + +TODO(benchmark): emit per-example benchmark correctness for host reduction too, +so wiring op4 through the sandbox does not drop benchmark_accuracy. +""" +from __future__ import annotations + +import json +import sys +from pathlib import Path + + +def prepare_workdir(canon_dir: Path, patch_path: Path, dest_workdir: Path) -> Path: + """copytree the canonical recipe tree into a scratch workdir and apply the + miner patch — done INSIDE the container so `patch -p1` path-traversal and + symlink dereference are contained by the read-only/non-root namespace, not + run on the host. Returns the workdir. + """ + import shutil + + from proof.runner import apply_patch + + canon_dir = Path(canon_dir) + dest_workdir = Path(dest_workdir) + dest_workdir.mkdir(parents=True, exist_ok=True) + for sub in ("model", "recipe", "data", "configs", "eval", "calibration"): + src = canon_dir / sub + if src.exists(): + # symlinks=False (default) copies content, but we never mount secrets + # into /canon, so there is nothing sensitive to dereference here. + shutil.copytree(src, dest_workdir / sub, dirs_exist_ok=True) + if Path(patch_path).exists(): + apply_patch(dest_workdir, Path(patch_path)) + return dest_workdir + + +def run_sandbox_eval( + workdir: Path, + ckpt_path: Path, + eval_dir: Path, + out_dir: Path, + *, + batch_size: int = 8, +): + """Produce per-position NLLs (for host val_bpb reduction) + benchmark accuracy. + + Importable + unit-testable in-process (no container) so the produce→reduce + equivalence can be proven on CPU. + """ + import numpy as np + + # Trusted helpers FIRST (canonical/installed), before the workdir goes on the + # path — miner code in workdir must not be able to shadow the reducer. + from eval.benchmark import compute_benchmark_score + from eval.val_bpb import load_eval_tokens, per_position_nlls + + sys.path.insert(0, str(Path(workdir).resolve())) + import torch + from model import RalphBase, RalphConfig + + # Checkpoint config: sidecar JSON if present, else the embedded "config". + sidecar = Path(ckpt_path).parent / "checkpoint_config.json" + if sidecar.exists(): + saved = json.loads(sidecar.read_text()) + else: + saved = torch.load(ckpt_path, weights_only=True, map_location="cpu").get("config", {}) + + fields = RalphConfig.__dataclass_fields__ + cfg = RalphConfig(**{k: v for k, v in saved.items() if k in fields}) + + ckpt = torch.load(ckpt_path, weights_only=True, map_location="cpu") + state_dict = ckpt.get("model", ckpt) + model = RalphBase(cfg) + model.load_state_dict(state_dict) + if torch.cuda.is_available(): + model = model.cuda() + + seq_len = cfg.max_seq_len // 2 + eval_dir = Path(eval_dir) + tokens = np.asarray(load_eval_tokens(eval_dir / "active_tokens.bin")) + nlls = per_position_nlls(model, tokens, seq_len, batch_size) + + # Benchmark accuracy — cheap (the ~1.5k held-out examples, not the token + # stream). Contained but miner-computed; the crown-critical val_bpb is the + # one the HOST reduces from nlls. No benchmark file -> 0.0. + benchmark_accuracy = 0.0 + benchmark_examples = 0 + bpath = eval_dir / "active_benchmark.json" + if bpath.exists(): + examples = json.loads(bpath.read_text()) + bench = compute_benchmark_score(model, examples) + benchmark_accuracy = float(bench["benchmark_accuracy"]) + benchmark_examples = int(bench["n_examples"]) + + out_dir = Path(out_dir) + out_dir.mkdir(parents=True, exist_ok=True) + np.save(out_dir / "nlls.npy", nlls) + manifest = { + "status": "ok", + "seq_len": int(seq_len), + "tokens_emitted": int(nlls.shape[0]), + "benchmark_accuracy": benchmark_accuracy, + "benchmark_examples": benchmark_examples, + "model_config": { + "vocab_size": cfg.vocab_size, + "dim": cfg.dim, + "n_layers": cfg.n_layers, + "max_seq_len": cfg.max_seq_len, + }, + } + (out_dir / "manifest.json").write_text(json.dumps(manifest)) + return nlls + + +def run_prepare_and_eval( + canon_dir: Path, + patch_path: Path, + ckpt_path: Path, + eval_dir: Path, + out_dir: Path, + *, + batch_size: int = 8, +): + """Full container flow: prepare the patched workdir IN here (traversal + contained), then eval. The host-side runner calls this via __main__.""" + import tempfile + + workdir = prepare_workdir(canon_dir, patch_path, Path(tempfile.mkdtemp(prefix="ralph_sbx_")) / "workdir") + return run_sandbox_eval(workdir, ckpt_path, eval_dir, out_dir, batch_size=batch_size) + + +def main(argv: list[str]) -> int: + if len(argv) != 6: + print(f"usage: {argv[0]} ", file=sys.stderr) + return 3 + canon_dir, patch_path, ckpt_path, eval_dir, out_dir = (Path(a) for a in argv[1:6]) + if not canon_dir.is_dir() or not ckpt_path.is_file() or not eval_dir.is_dir(): + print("ERROR: canon_dir/ckpt/eval_dir must exist", file=sys.stderr) + return 3 + try: + run_prepare_and_eval(canon_dir, patch_path, ckpt_path, eval_dir, out_dir) + except (ImportError, KeyError, RuntimeError) as e: + print(f"ERROR: setup/load failed: {e}", file=sys.stderr) + return 1 + except Exception as e: # noqa: BLE001 + print(f"ERROR: eval crashed: {e}", file=sys.stderr) + return 2 + print("RALPH_SANDBOX_EVAL ok") + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/validator/validator.py b/validator/validator.py index f0632a4..c53449f 100644 --- a/validator/validator.py +++ b/validator/validator.py @@ -617,6 +617,108 @@ def _opt_str(key: str) -> str | None: ) +def _sandboxed_hidden_eval( + ralph_root: Path, + proof_dir: Path, +) -> tuple[bool, str, HiddenEvalResult | None]: + """op4 hidden-eval run inside the hardened container (RALPH_SANDBOX=1). + + The miner's (possibly patched) model executes contained — no network, no + secrets, non-root. The container emits per-position NLLs + benchmark + accuracy; the HOST reduces the crown-critical val_bpb from the NLLs and + computes the eval-set hash itself. FAIL-CLOSED: if the sandbox runtime can't + be verified, the submission is rejected — never a bare-exec fallback. + """ + import os + import shutil + import tempfile + + import numpy as np + + from eval.host_reduce import ( + expected_token_count, + hash_target_stream, + reduce_token_nlls, + ) + from eval.val_bpb import DEFAULT_BYTES_PER_TOKEN, load_eval_tokens + from ralph_bootstrap import RECIPE_DIR + from validator.sandbox import Mount, SandboxConfig, SandboxUnavailable, run_in_sandbox + + ckpt_path = proof_dir / "training" / "checkpoint.pt" + eval_dir = ralph_root / "eval" / "private" + if not ckpt_path.exists(): + return False, f"missing checkpoint at {ckpt_path}", None + if not (eval_dir / "active_tokens.bin").exists(): + return False, f"missing held-out shard at {eval_dir}", None + + image = os.environ.get("RALPH_SANDBOX_IMAGE", "") + if "@sha256:" not in image: + return False, "RALPH_SANDBOX=1 but RALPH_SANDBOX_IMAGE is not a digest-pinned image", None + gpu = int(os.environ.get("RALPH_SANDBOX_GPU", "0")) if torch.cuda.is_available() else None + cfg = SandboxConfig(image=image, gpu_device=gpu) + + # Per-submission host scratch for the container's /out. The container itself + # is ephemeral (docker --rm); this dir holds nlls.npy + manifest.json only + # long enough to host-reduce, then is removed on EVERY exit path (finally) + # so /tmp doesn't accumulate ~12 MB per submission. + out_dir = Path(tempfile.mkdtemp(prefix="ralph_sbx_out_")) + try: + mounts = [ + Mount(Path(RECIPE_DIR), "/canon", ro=True), + Mount(proof_dir, "/in", ro=True), + Mount(eval_dir, "/eval-private", ro=True), + ] + container_argv = [ + "python", "-m", "validator.sandbox_eval", + "/canon", "/in/patch.diff", "/in/training/checkpoint.pt", "/eval-private", "/out", + ] + try: + res = run_in_sandbox( + cfg, + container_argv=container_argv, + mounts=mounts, + out_dir=out_dir, + timeout_s=int(os.environ.get("RALPH_SANDBOX_TIMEOUT_S", "900")), + ) + except SandboxUnavailable as e: + return False, f"op4 sandbox unavailable (FAIL-CLOSED, not falling back): {e}", None + if res.returncode != 0: + return False, f"op4 sandbox eval failed (rc={res.returncode}): {res.stderr[-300:]}", None + + nll_path = out_dir / "nlls.npy" + man_path = out_dir / "manifest.json" + if not (nll_path.exists() and man_path.exists()): + return False, "op4 sandbox produced no nlls/manifest output", None + + manifest = json.loads(man_path.read_text()) + seq_len = int(manifest["seq_len"]) + tokens = np.asarray(load_eval_tokens(eval_dir / "active_tokens.bin")) + eval_set_hash = hash_target_stream(tokens) # HOST-computed, not miner-supplied + try: + reduced = reduce_token_nlls( + np.load(nll_path), + seq_len=seq_len, + bytes_per_token=DEFAULT_BYTES_PER_TOKEN, + expected_tokens=expected_token_count(len(tokens), seq_len), + eval_set_hash=eval_set_hash, + ) + except ValueError as e: + return False, f"op4 host-reduction rejected the emitted nlls: {e}", None + + result = HiddenEvalResult( + val_bpb=reduced.val_bpb, + benchmark_accuracy=round(float(manifest.get("benchmark_accuracy", 0.0)), 3), + tokens_evaluated=reduced.tokens_evaluated, + benchmark_examples=int(manifest.get("benchmark_examples", 0)), + eval_set_hash=eval_set_hash, + val_seq_len=seq_len, + tail_val_bpb=reduced.tail_val_bpb, + ) + return True, f"val_bpb={result.val_bpb:.4f} bench={result.benchmark_accuracy:.3f} (sandboxed)", result + finally: + shutil.rmtree(out_dir, ignore_errors=True) + + # --- Hidden-eval result cache ------------------------------------------------- # A deferred challenger (king min-tenure guard) is re-scored EVERY epoch while it # waits out the incumbent's tenure (~300 blocks). The bundle and the held-out @@ -690,12 +792,22 @@ def op4_hidden_eval( ralph_root: Path, proof_dir: Path, ) -> tuple[bool, str, HiddenEvalResult | None]: + import os eval_dir = ralph_root / "eval" / "private" shard_fp = _eval_shard_fingerprint(eval_dir) cached = _load_cached_hidden_eval(proof_dir, shard_fp) if cached is not None: return True, f"val_bpb={cached.val_bpb:.4f} bench={cached.benchmark_accuracy:.3f} (cached)", cached + # Sandbox mode: run the (untrusted) model in the hardened container; the host + # reduces val_bpb. Cache the result like the canonical path so a deferred + # challenger isn't re-containerised every epoch. + if os.environ.get("RALPH_SANDBOX", "0") == "1": + ok, detail, result = _sandboxed_hidden_eval(ralph_root, proof_dir) + if ok and result is not None: + _save_cached_hidden_eval(proof_dir, shard_fp, result) + return ok, detail, result + ckpt_path = proof_dir / "training" / "checkpoint.pt" if not ckpt_path.exists(): return False, f"missing checkpoint at {ckpt_path}", None