diff --git a/Dockerfile.sandbox b/Dockerfile.sandbox index 6fb7e2e..10df4dd 100644 --- a/Dockerfile.sandbox +++ b/Dockerfile.sandbox @@ -62,6 +62,7 @@ COPY eval/ /app/eval/ COPY calibration/ /app/calibration/ COPY proof/ /app/proof/ COPY validator/ /app/validator/ +COPY miner/ /app/miner/ COPY configs/ /app/configs/ COPY restricted_files.yaml /app/restricted_files.yaml COPY ralph_bootstrap.py /app/ralph_bootstrap.py diff --git a/validator/sandbox.py b/validator/sandbox.py index b86d8b3..d094bc1 100644 --- a/validator/sandbox.py +++ b/validator/sandbox.py @@ -108,7 +108,7 @@ def build_container_env(env_extra: dict[str, str] | None) -> dict[str, str]: "PYTHONUNBUFFERED": "1", "PYTHONDONTWRITEBYTECODE": "1", "HOME": "/scratch", - "TMPDIR": "/scratch/tmp", + "TMPDIR": "/scratch", } for k, v in (env_extra or {}).items(): if k in _TRAINING_ENV_BLOCKLIST: diff --git a/validator/service.py b/validator/service.py index 969f433..96fe2bb 100644 --- a/validator/service.py +++ b/validator/service.py @@ -143,12 +143,29 @@ def _safe_current_block(chain) -> int: def archive_bundle(bundle_dir: Path, queue_dir: Path, dest: str) -> None: - """Move a processed bundle to scored/ or rejected/.""" + """Move a processed bundle to scored/ or rejected/. + + Rejected bundles are terminal — their hash is recorded in hf_state.json so + they are never re-downloaded or re-evaluated — so the heavy training/ + checkpoint (~0.5 GB each) is GC'd on archive to keep queue/rejected/ from + filling the disk. The lightweight manifests are kept as an audit trail. + Opt out with RALPH_QUEUE_GC_REJECTED=0. + """ target = queue_dir / dest / bundle_dir.name target.parent.mkdir(parents=True, exist_ok=True) if target.exists(): shutil.rmtree(target) shutil.move(str(bundle_dir), str(target)) + if dest == "rejected" and os.environ.get( + "RALPH_QUEUE_GC_REJECTED", "1" + ).strip().lower() not in {"0", "false", "no", "off"}: + ckpt_dir = target / "training" + if ckpt_dir.exists(): + try: + shutil.rmtree(ckpt_dir) + log_info(f" queue GC: removed rejected checkpoint {target.name}/training") + except OSError as e: + log_warn(f"queue GC: could not remove {ckpt_dir}: {e}") def _close_losing_prs(bundle_dir: Path, reason: str) -> None: diff --git a/validator/validator.py b/validator/validator.py index 83a2ed4..42bc593 100644 --- a/validator/validator.py +++ b/validator/validator.py @@ -686,8 +686,8 @@ def _sandboxed_hidden_eval( Mount(proof_dir, "/in", ro=True), Mount(eval_dir, "/eval-private", ro=True), ] + # ENTRYPOINT already runs `python -m validator.sandbox_eval`; pass args only. container_argv = [ - "python", "-m", "validator.sandbox_eval", "/canon", "/in/patch.diff", "/in/training/checkpoint.pt", "/eval-private", "/out", ] try: @@ -945,6 +945,11 @@ def _hosb_sandbox_nlls(ralph_root: Path, proof_dir: Path, idx_grid, tgt_grid, se grid_dir = Path(tempfile.mkdtemp(prefix="ralph_hosb_grid_")) out_dir = Path(tempfile.mkdtemp(prefix="ralph_hosb_out_")) + # userns-remap maps the container --user to an unprivileged host uid that + # cannot touch root-owned 0700 mkdtemp dirs; open the dir bits so the (ro) + # grid is readable and the (rw) out is writable from the remapped namespace. + os.chmod(grid_dir, 0o755) + os.chmod(out_dir, 0o777) try: np.save(grid_dir / "idx_grid.npy", idx_grid) np.save(grid_dir / "scored_idx.npy", scored_idx) # positions, NOT the answer @@ -966,8 +971,9 @@ def _hosb_sandbox_nlls(ralph_root: Path, proof_dir: Path, idx_grid, tgt_grid, se Mount(proof_dir, "/in", ro=True), Mount(grid_dir, "/grid", ro=True), # idx_grid + scored_idx — NO targets, NO raw shard ] + # ENTRYPOINT already runs `python -m validator.sandbox_eval`; pass args only. container_argv = [ - "python", "-m", "validator.sandbox_eval", "--grid", + "--grid", "/canon", "/in/patch.diff", "/in/training/checkpoint.pt", "/grid", "/out", ] res = run_in_sandbox(