From 57bb6a3667d1c81e5d82c37e6a9cd701ff92c153 Mon Sep 17 00:00:00 2001 From: Bitzy Date: Sat, 27 Jun 2026 15:17:29 +0000 Subject: [PATCH] validator: stop op4 eval subprocess inheriting secrets - op4 _patched_hidden_eval imports + executes the miner's patched model.py; it was spawned with no env=, so miner code inherited the validator's full os.environ - pass allowlist-only env via proof.runner._sanitized_env (mirrors the miner-side training subprocess) - extend the env blocklist: RALPH_VALIDATOR_PRIVKEY[_FILE] (the seal privkey) + the enforcement escape hatches (TEST_MODE, ALLOW_SYNTHETIC _EVAL, ALLOW_MOCK_ATTESTATION, SKIP_HANDSHAKE) - redact subprocess stderr before re-raising - stopgap ahead of the full execution sandbox (op4 + audit) Co-Authored-By: Claude Opus 4.8 (1M context) --- proof/runner.py | 7 +++ tests/test_op4_env_sanitize.py | 104 +++++++++++++++++++++++++++++++++ validator/validator.py | 11 +++- 3 files changed, 120 insertions(+), 2 deletions(-) create mode 100644 tests/test_op4_env_sanitize.py diff --git a/proof/runner.py b/proof/runner.py index efc03a0..fcc5957 100644 --- a/proof/runner.py +++ b/proof/runner.py @@ -71,6 +71,13 @@ "AWS_SECRET_ACCESS_KEY", "AWS_ACCESS_KEY_ID", "AWS_SESSION_TOKEN", "GCP_SERVICE_ACCOUNT_KEY", "GOOGLE_APPLICATION_CREDENTIALS", "GH_TOKEN", "GITHUB_TOKEN", + # Ralph validator secrets + unsafe escape hatches. The op4 patched-eval + # subprocess imports and runs the miner's model.py; it must never see the + # libsodium seal privkey (decrypts every bundle), nor be able to read/flip + # the enforcement toggles (e.g. fall back to a synthetic eval stream). + "RALPH_VALIDATOR_PRIVKEY", "RALPH_VALIDATOR_PRIVKEY_FILE", + "RALPH_TEST_MODE", "RALPH_ALLOW_SYNTHETIC_EVAL", + "RALPH_ALLOW_MOCK_ATTESTATION", "RALPH_SKIP_HANDSHAKE", ) diff --git a/tests/test_op4_env_sanitize.py b/tests/test_op4_env_sanitize.py new file mode 100644 index 0000000..aff5e16 --- /dev/null +++ b/tests/test_op4_env_sanitize.py @@ -0,0 +1,104 @@ +"""Stopgap: the op4 patched-eval subprocess must NOT inherit validator secrets. + +`validator/eval_in_workdir.py` imports and executes the miner's patched +`model.py`. Before this fix the subprocess was spawned with no `env=` kwarg, so +the miner's code inherited the validator's full `os.environ` — including the +libsodium seal privkey (`RALPH_VALIDATOR_PRIVKEY`, decrypts every bundle), the +Bittensor wallet, and cloud tokens. These tests pin the allowlist-only env and +the extended blocklist. +""" +from __future__ import annotations + +import subprocess as _subprocess +from pathlib import Path + +import pytest + +from proof.runner import _TRAINING_ENV_BLOCKLIST, _sanitized_env + +# Secrets / escape hatches the eval subprocess must never see. +_MUST_BLOCK = ( + "RALPH_VALIDATOR_PRIVKEY", + "RALPH_VALIDATOR_PRIVKEY_FILE", + "RALPH_TEST_MODE", + "RALPH_ALLOW_SYNTHETIC_EVAL", + "RALPH_ALLOW_MOCK_ATTESTATION", + "RALPH_SKIP_HANDSHAKE", +) + + +def test_blocklist_covers_validator_secrets(): + for name in _MUST_BLOCK: + assert name in _TRAINING_ENV_BLOCKLIST, f"{name} missing from blocklist" + + +def test_sanitized_env_scrubs_secrets(monkeypatch): + secret_val = "supersecret_privkey_value_0123456789" + monkeypatch.setenv("RALPH_VALIDATOR_PRIVKEY", secret_val) + monkeypatch.setenv("RALPH_VALIDATOR_PRIVKEY_FILE", "/root/.ralph_validator_enc_key.json") + monkeypatch.setenv("BT_WALLET_PASSWORD", "wallet_password_value") + monkeypatch.setenv("HF_TOKEN", "hf_tokenvalue_abcdefgh") + # RALPH_ALLOW_SYNTHETIC_EVAL is set by the autouse conftest fixture. + + env = _sanitized_env(extra={"PYTHONPATH": "/tmp/workdir"}) + + for name in (*_MUST_BLOCK, "BT_WALLET_PASSWORD", "HF_TOKEN"): + assert name not in env, f"{name} leaked into sanitized env" + assert secret_val not in env.values() + # The allowlist + extra still pass through. + assert env.get("PYTHONPATH") == "/tmp/workdir" + + +def test_sanitized_env_rejects_blocklisted_extra(): + with pytest.raises(ValueError): + _sanitized_env(extra={"RALPH_VALIDATOR_PRIVKEY": "x"}) + + +def test_patched_eval_subprocess_uses_sanitized_env(monkeypatch, tmp_path): + """End-to-end: the env handed to the eval subprocess excludes every secret.""" + import shutil + + from validator.validator import _patched_hidden_eval + + # Plant secrets in the parent environment. + monkeypatch.setenv("RALPH_VALIDATOR_PRIVKEY", "seal_privkey_should_not_leak_123") + monkeypatch.setenv("BT_WALLET_PASSWORD", "wallet_pw_should_not_leak") + monkeypatch.setenv("HF_TOKEN", "hf_should_not_leak_abcdefgh") + + proof_dir = tmp_path / "proof" + (proof_dir / "training").mkdir(parents=True) + (proof_dir / "patch.diff").write_text("") # empty patch → apply_patch no-ops + ckpt = proof_dir / "training" / "checkpoint.pt" + ckpt.write_bytes(b"") + + # Neutralize the heavy/real steps; we only care about the subprocess env. + monkeypatch.setattr(shutil, "copytree", lambda *a, **k: None) + + captured: dict = {} + + class _FakeProc: + returncode = 0 + stdout = ( + "RALPH_EVAL_RESULT val_bpb=1.500000 benchmark_acc=0.500000 " + "tokens_evaluated=100 benchmark_examples=10 eval_set_hash=deadbeef\n" + ) + stderr = "" + + def _fake_run(argv, **kwargs): + captured["env"] = kwargs.get("env") + return _FakeProc() + + monkeypatch.setattr(_subprocess, "run", _fake_run) + + ok, _detail, _result = _patched_hidden_eval(tmp_path, proof_dir, ckpt) + assert ok, _detail + + env = captured["env"] + assert env is not None, "subprocess was spawned without an explicit env" + for name in (*_MUST_BLOCK, "BT_WALLET_PASSWORD", "HF_TOKEN"): + assert name not in env, f"{name} leaked to the miner-code subprocess" + leaked = {"seal_privkey_should_not_leak_123", "wallet_pw_should_not_leak", "hf_should_not_leak_abcdefgh"} + assert not (leaked & set(env.values())), "a secret value leaked into the subprocess env" + # PYTHONPATH is the per-bundle patched workdir (a TemporaryDirectory created + # inside the function), so we can only assert the suffix. + assert Path(env.get("PYTHONPATH", "")).name == "workdir" diff --git a/validator/validator.py b/validator/validator.py index af2ad66..15d8143 100644 --- a/validator/validator.py +++ b/validator/validator.py @@ -446,7 +446,7 @@ def _patched_hidden_eval( import subprocess import tempfile - from proof.runner import apply_patch + from proof.runner import _redacted, _sanitized_env, apply_patch patch_path = proof_dir / "patch.diff" if not patch_path.exists(): @@ -486,11 +486,18 @@ def _patched_hidden_eval( capture_output=True, text=True, timeout=240, + # SECURITY: eval_in_workdir.py imports and EXECUTES the miner's + # patched model.py. Never hand it the validator's environment — + # the seal privkey (RALPH_VALIDATOR_PRIVKEY), wallet, and cloud + # tokens would leak straight to attacker-controlled code. Pass an + # allowlist-only env (mirrors the miner-side training subprocess). + # PYTHONPATH points at the patched workdir so its model package wins. + env=_sanitized_env(extra={"PYTHONPATH": str(workdir)}), ) except subprocess.TimeoutExpired: return False, "patched-eval subprocess timed out (>240s)", None if res.returncode != 0: - tail = (res.stderr or "")[-300:] + tail = _redacted(res.stderr or "")[-300:] return False, f"patched-eval subprocess exit={res.returncode}: {tail}", None marker = "RALPH_EVAL_RESULT "