[rl] Reject trace uploads in offline Hub runtimes

docs/debug-log-trace-upload-offline.md

@@ -0,0 +1,29 @@
+# Debugging log for offline trace upload
+
+Stop Jupiter links from promising a Hugging Face trace upload while Hub access is disabled.
+
+## Initial status
+
+The 30B Jupiter recipe enabled post-run trace uploads and exported `HF_HUB_OFFLINE=1`. Every link retained its
+local traces but the uploader failed while creating the Hugging Face dataset repository.
+
+## Hypothesis 1
+
+The conflict is fully knowable at launch validation: both settings live in the same parsed YAML. Rejecting the
+combination before submission prevents a delayed failure after hours of training.
+
+## Changes to make
+
+Validate `terminal_bench.trace_upload.enabled` against the host and Apptainer Hub-offline variables before any
+dataset or model staging. Disable trace upload in the affected Jupiter recipe and retain `trials_dir` for
+archive-based collection.
+
+## Results
+
+The focused trace-upload and durable-path suite passes (16 tests). Ruff passes over the complete `tests/` tree.
+The validator rejects host or Apptainer `HF_HUB_OFFLINE` values before launch and permits the same environment
+when trace upload is disabled.
+
+## Future work
+
+- [ ] Add an archive uploader when the campaign needs automatic publication from a networked login node.

hpc/rl_launch_utils.py

@@ -938,6 +938,24 @@ def _build_rl_container_env(container: Mapping[str, Any], exp_args: dict) -> str
     return "\n".join(lines)
 
 
+def validate_trace_upload_environment(
+    terminal_bench: Mapping[str, Any], container: Mapping[str, Any]
+) -> None:
+    """Reject trace uploads from a runtime configured for offline Hub access."""
+    trace_upload = terminal_bench.get("trace_upload") or {}
+    if not trace_upload.get("enabled"):
+        return
+
+    extra_env = container.get("extra_env") or {}
+    offline_keys = ("HF_HUB_OFFLINE", "APPTAINERENV_HF_HUB_OFFLINE")
+    enabled = [key for key in offline_keys if str(extra_env.get(key, "")).lower() in {"1", "true", "yes"}]
+    if enabled:
+        raise ValueError(
+            "terminal_bench.trace_upload.enabled=true conflicts with "
+            f"container.extra_env {', '.join(enabled)}; disable trace upload or remove offline Hub mode"
+        )
+
+
 def construct_rl_sbatch_script(exp_args: dict, hpc) -> RLLaunchArtifacts:
     """Construct RL sbatch script using the universal template system.
 
@@ -972,6 +990,8 @@ def construct_rl_sbatch_script(exp_args: dict, hpc) -> RLLaunchArtifacts:
 
     parsed = parse_rl_config(rl_config_path, model_override=exp_args.get("model_path"))
     print(f"Loaded RL config from: {parsed.config_path}")
+    container = parsed.raw.get("container") or {}
+    validate_trace_upload_environment(parsed.terminal_bench or {}, container)
 
     # --- RL container section (Apptainer SIF + overlays + pydeps + extra env) ---
     # Optional top-level `container:` block in the RL yaml. When present it lets a
@@ -990,9 +1010,7 @@ def construct_rl_sbatch_script(exp_args: dict, hpc) -> RLLaunchArtifacts:
     #
     # An explicit --rl_container_sif CLI flag still wins (only fills if unset), so
     # nothing changes for configs without a `container:` section.
-    rl_container_env_block = _build_rl_container_env(
-        parsed.raw.get("container") or {}, exp_args
-    )
+    rl_container_env_block = _build_rl_container_env(container, exp_args)
 
     # Extract agent name and harbor_env from terminal_bench config
     yaml_agent_name, yaml_harbor_env = extract_terminal_bench_agent_env(parsed)

hpc/skyrl_yaml/jupiter/24GPU_qwen3_30b_a3b_thinking.yaml

@@ -77,7 +77,9 @@ terminal_bench:
     enabled: false
 
   trace_upload:
-    enabled: true
+    # Jupiter model loading is deliberately offline. Traces remain in trials_dir
+    # for archive-based collection instead of a post-run Hub upload.
+    enabled: false
     repo_org: DCAgent
     episodes: last
     dataset_type: SFT

tests/hpc/test_rl_trace_upload.py

@@ -0,0 +1,26 @@
+import pytest
+
+from hpc.rl_launch_utils import validate_trace_upload_environment
+
+
+def test_trace_upload_rejects_offline_hub_environment() -> None:
+    terminal_bench = {"trace_upload": {"enabled": True}}
+    container = {
+        "extra_env": {
+            "HF_HUB_OFFLINE": 1,
+            "APPTAINERENV_HF_HUB_OFFLINE": 1,
+        }
+    }
+
+    with pytest.raises(
+        ValueError,
+        match=r"trace_upload\.enabled=true conflicts with .*HF_HUB_OFFLINE",
+    ):
+        validate_trace_upload_environment(terminal_bench, container)
+
+
+def test_disabled_trace_upload_allows_offline_hub_environment() -> None:
+    terminal_bench = {"trace_upload": {"enabled": False}}
+    container = {"extra_env": {"HF_HUB_OFFLINE": 1}}
+
+    validate_trace_upload_environment(terminal_bench, container)