From 8a1ea81e11b622658b80f4acc52c6f861bdc59d3 Mon Sep 17 00:00:00 2001 From: Benjamin Feuer Date: Thu, 13 Aug 2026 11:20:58 -0700 Subject: [PATCH] [rl] Reject trace uploads in offline Hub runtimes --- docs/debug-log-trace-upload-offline.md | 29 +++++++++++++++++++ hpc/rl_launch_utils.py | 24 +++++++++++++-- .../jupiter/24GPU_qwen3_30b_a3b_thinking.yaml | 4 ++- tests/hpc/test_rl_trace_upload.py | 26 +++++++++++++++++ 4 files changed, 79 insertions(+), 4 deletions(-) create mode 100644 docs/debug-log-trace-upload-offline.md create mode 100644 tests/hpc/test_rl_trace_upload.py diff --git a/docs/debug-log-trace-upload-offline.md b/docs/debug-log-trace-upload-offline.md new file mode 100644 index 000000000..df0a82001 --- /dev/null +++ b/docs/debug-log-trace-upload-offline.md @@ -0,0 +1,29 @@ +# Debugging log for offline trace upload + +Stop Jupiter links from promising a Hugging Face trace upload while Hub access is disabled. + +## Initial status + +The 30B Jupiter recipe enabled post-run trace uploads and exported `HF_HUB_OFFLINE=1`. Every link retained its +local traces but the uploader failed while creating the Hugging Face dataset repository. + +## Hypothesis 1 + +The conflict is fully knowable at launch validation: both settings live in the same parsed YAML. Rejecting the +combination before submission prevents a delayed failure after hours of training. + +## Changes to make + +Validate `terminal_bench.trace_upload.enabled` against the host and Apptainer Hub-offline variables before any +dataset or model staging. Disable trace upload in the affected Jupiter recipe and retain `trials_dir` for +archive-based collection. + +## Results + +The focused trace-upload and durable-path suite passes (16 tests). Ruff passes over the complete `tests/` tree. +The validator rejects host or Apptainer `HF_HUB_OFFLINE` values before launch and permits the same environment +when trace upload is disabled. + +## Future work + +- [ ] Add an archive uploader when the campaign needs automatic publication from a networked login node. diff --git a/hpc/rl_launch_utils.py b/hpc/rl_launch_utils.py index 2ce432570..bdd20a458 100644 --- a/hpc/rl_launch_utils.py +++ b/hpc/rl_launch_utils.py @@ -936,6 +936,24 @@ def _build_rl_container_env(container: Mapping[str, Any], exp_args: dict) -> str return "\n".join(lines) +def validate_trace_upload_environment( + terminal_bench: Mapping[str, Any], container: Mapping[str, Any] +) -> None: + """Reject trace uploads from a runtime configured for offline Hub access.""" + trace_upload = terminal_bench.get("trace_upload") or {} + if not trace_upload.get("enabled"): + return + + extra_env = container.get("extra_env") or {} + offline_keys = ("HF_HUB_OFFLINE", "APPTAINERENV_HF_HUB_OFFLINE") + enabled = [key for key in offline_keys if str(extra_env.get(key, "")).lower() in {"1", "true", "yes"}] + if enabled: + raise ValueError( + "terminal_bench.trace_upload.enabled=true conflicts with " + f"container.extra_env {', '.join(enabled)}; disable trace upload or remove offline Hub mode" + ) + + def construct_rl_sbatch_script(exp_args: dict, hpc) -> RLLaunchArtifacts: """Construct RL sbatch script using the universal template system. @@ -970,6 +988,8 @@ def construct_rl_sbatch_script(exp_args: dict, hpc) -> RLLaunchArtifacts: parsed = parse_rl_config(rl_config_path, model_override=exp_args.get("model_path")) print(f"Loaded RL config from: {parsed.config_path}") + container = parsed.raw.get("container") or {} + validate_trace_upload_environment(parsed.terminal_bench or {}, container) # --- RL container section (Apptainer SIF + overlays + pydeps + extra env) --- # Optional top-level `container:` block in the RL yaml. When present it lets a @@ -988,9 +1008,7 @@ def construct_rl_sbatch_script(exp_args: dict, hpc) -> RLLaunchArtifacts: # # An explicit --rl_container_sif CLI flag still wins (only fills if unset), so # nothing changes for configs without a `container:` section. - rl_container_env_block = _build_rl_container_env( - parsed.raw.get("container") or {}, exp_args - ) + rl_container_env_block = _build_rl_container_env(container, exp_args) # Extract agent name and harbor_env from terminal_bench config yaml_agent_name, yaml_harbor_env = extract_terminal_bench_agent_env(parsed) diff --git a/hpc/skyrl_yaml/jupiter/24GPU_qwen3_30b_a3b_thinking.yaml b/hpc/skyrl_yaml/jupiter/24GPU_qwen3_30b_a3b_thinking.yaml index dad99fd46..a0bd715b9 100644 --- a/hpc/skyrl_yaml/jupiter/24GPU_qwen3_30b_a3b_thinking.yaml +++ b/hpc/skyrl_yaml/jupiter/24GPU_qwen3_30b_a3b_thinking.yaml @@ -77,7 +77,9 @@ terminal_bench: enabled: false trace_upload: - enabled: true + # Jupiter model loading is deliberately offline. Traces remain in trials_dir + # for archive-based collection instead of a post-run Hub upload. + enabled: false repo_org: DCAgent episodes: last dataset_type: SFT diff --git a/tests/hpc/test_rl_trace_upload.py b/tests/hpc/test_rl_trace_upload.py new file mode 100644 index 000000000..c4156fa79 --- /dev/null +++ b/tests/hpc/test_rl_trace_upload.py @@ -0,0 +1,26 @@ +import pytest + +from hpc.rl_launch_utils import validate_trace_upload_environment + + +def test_trace_upload_rejects_offline_hub_environment() -> None: + terminal_bench = {"trace_upload": {"enabled": True}} + container = { + "extra_env": { + "HF_HUB_OFFLINE": 1, + "APPTAINERENV_HF_HUB_OFFLINE": 1, + } + } + + with pytest.raises( + ValueError, + match=r"trace_upload\.enabled=true conflicts with .*HF_HUB_OFFLINE", + ): + validate_trace_upload_environment(terminal_bench, container) + + +def test_disabled_trace_upload_allows_offline_hub_environment() -> None: + terminal_bench = {"trace_upload": {"enabled": False}} + container = {"extra_env": {"HF_HUB_OFFLINE": 1}} + + validate_trace_upload_environment(terminal_bench, container)