[hpc] Bound Ray plasma startup

docs/debug-log-ray-plasma-startup.md

@@ -0,0 +1,47 @@
+# Debugging log for Ray plasma startup
+
+Keep large-object-store Ray clusters from failing nondeterministically while the local plasma store initializes.
+
+## Initial status
+
+Jupiter job 1262271 started GCS and registered all five worker nodes, but its head raylet exited before the
+cluster became usable. The preserved `raylet.out` records ten failed attempts to connect to the local
+`plasma_store` socket, followed by a fatal `No such file or directory`. The launcher then reached its bounded
+cluster-startup deadline and preserved the Ray logs.
+
+## Hypothesis 1
+
+The existing `RAY_raylet_start_wait_time_s=120` setting also governs the local plasma-socket connection.
+
+## Results
+
+Refuted by Ray 2.51.1 source. GCS registration reads `raylet_start_wait_time_s`; the plasma client instead reads
+`raylet_client_num_connect_attempts`, whose default is ten attempts at one-second intervals.
+
+The immediately preceding job 1262020 used the same 40 GiB object-store size and runtime. Its plasma store became
+ready after about eight seconds. In job 1262271, initialization exceeded ten seconds and the raylet aborted. This
+paired observation supports a startup race rather than an unreachable head node.
+
+## Hypothesis 2
+
+Passing `RAY_raylet_client_num_connect_attempts=120` through the shared Apptainer prefix will bound the same
+plasma-socket startup phase without weakening the existing GCS-registration deadline.
+
+## Changes to make
+
+- Preserve `RAY_raylet_start_wait_time_s=120` for GCS registration.
+- Add the independent plasma-socket attempt limit to every in-container Ray invocation.
+- Test both defaults and explicit operator overrides through `build_apptainer_prefix`.
+
+## Results
+
+The focused regression test failed before the change because the plasma-socket control was absent and passes
+after both independent controls are emitted.
+
+Jupiter job 1264144 validated the runtime contract against Ray 2.51.1 in the production r5 SIF. It enabled
+plasma page preallocation, created a 40 GiB object store, connected a driver, observed all four GH200 GPUs, and
+shut Ray down cleanly. Slurm completed the job with exit code 0 in 42 seconds.
+
+## Future work
+
+- [ ] Confirm the next six-node TaskTrove resume forms the complete Ray cluster before trainer initialization.

hpc/rl_launch_utils.py

@@ -183,22 +183,15 @@ def build_apptainer_prefix(
     # later. Defaults to offline but honors an explicit host override.
     wandb_mode = os.environ.get("WANDB_MODE", "offline")
     prefix.extend(["--env", f"WANDB_MODE={wandb_mode}"])
-    # Raise Ray's raylet-startup grace window. On busy 6-node GH200 allocations
-    # the raylet (head AND worker) intermittently fails to finish registering
-    # with the local GCS inside Ray's default 30s (`RAY_raylet_start_wait_time_s`),
-    # so `ray start` aborts with "The current node timed out during startup ...
-    # the GCS has become overloaded" → the driver then loops on
-    # "Failed to connect to GCS ... within 5 seconds" until the 600s wait window
-    # expires (job 930367, #232 cp2). This is slow-to-form, NOT unreachable — the
-    # driver runs ON the head node and still can't reach its own 6379. Ray's own
-    # error message recommends raising this config. apptainer passes it via --env
-    # so it reaches the in-SIF `ray start`; host `env KEY=val` prefixes do NOT
-    # cross the container boundary (and the proxychains path drops ray_env_vars
-    # entirely), making this --env injection the only point that reaches every
-    # ray invocation (head, worker, wait/poll scripts) uniformly. Honors an
-    # explicit host override.
+    # Ray has independent startup deadlines for GCS registration and for the
+    # local plasma-store socket. A large object-store mmap can exceed the
+    # plasma client's default ten one-second connection attempts even while GCS
+    # is healthy. Pass both controls through Apptainer so every head, worker,
+    # and wait process observes them; each still honors an explicit override.
     raylet_wait = os.environ.get("RAY_raylet_start_wait_time_s", "120")
     prefix.extend(["--env", f"RAY_raylet_start_wait_time_s={raylet_wait}"])
+    raylet_connect_attempts = os.environ.get("RAY_raylet_client_num_connect_attempts", "120")
+    prefix.extend(["--env", f"RAY_raylet_client_num_connect_attempts={raylet_connect_attempts}"])
     prefix.append(sif)
     return prefix
 

tests/analysis/test_watch_iris_harbor.py

@@ -662,7 +662,7 @@ def test_glm52_capacity_floor_keeps_normal_single_node_progress_healthy():
 def test_harbor_job_kind_classifies_evalchemy_and_keeps_existing_kinds():
     assert (
         watcher.harbor_job_kind(
-            'bash -c \'exec /opt/eval/evalchemy/.venv/bin/python '
+            "bash -c 'exec /opt/eval/evalchemy/.venv/bin/python "
             '"$IRIS_WORKDIR/experiments/evals/evalchemy/run_evalchemy_client.py"\'',
             "/owner/eval-eval-abcd1234",
         )
@@ -702,7 +702,9 @@ def test_read_gcs_progress_falls_back_to_per_trial_when_aggregate_missing(
     monkeypatch, tmp_path
 ):
     # A running datagen job whose Harbor aggregate result.json is not on GCS yet.
-    job = _job(jobs_dir="gs://marin-us-central1/ot-agent/run-xyz", harbor_job_name="run-xyz")
+    job = _job(
+        jobs_dir="gs://marin-us-central1/ot-agent/run-xyz", harbor_job_name="run-xyz"
+    )
 
     def fake_cat(*_args, **_kwargs):
         return SimpleNamespace(
@@ -770,10 +772,18 @@ def test_read_evalchemy_progress_counts_non_empty_results(monkeypatch, tmp_path)
         watcher,
         "iter_objects",
         lambda *_a, **_k: [
-            {"Key": "iris/marinbase-eval/qwen-qwen3-30b-a3b/tier2-run1/MATH500_0shot/qwen/results_1.json"},
-            {"Key": "iris/marinbase-eval/qwen-qwen3-30b-a3b/tier2-run1/HumanEvalPlus_0shot/qwen/results_2.json"},
-            {"Key": "iris/marinbase-eval/qwen-qwen3-30b-a3b/tier2-run1/MBPPPlus_0shot/qwen/empty_results.json"},
-            {"Key": "iris/marinbase-eval/qwen-qwen3-30b-a3b/tier2-run1/MATH500_0shot/qwen/samples_MATH500.jsonl"},
+            {
+                "Key": "iris/marinbase-eval/qwen-qwen3-30b-a3b/tier2-run1/MATH500_0shot/qwen/results_1.json"
+            },
+            {
+                "Key": "iris/marinbase-eval/qwen-qwen3-30b-a3b/tier2-run1/HumanEvalPlus_0shot/qwen/results_2.json"
+            },
+            {
+                "Key": "iris/marinbase-eval/qwen-qwen3-30b-a3b/tier2-run1/MBPPPlus_0shot/qwen/empty_results.json"
+            },
+            {
+                "Key": "iris/marinbase-eval/qwen-qwen3-30b-a3b/tier2-run1/MATH500_0shot/qwen/samples_MATH500.jsonl"
+            },
         ],
     )
 

tests/hpc/test_apptainer_ray_startup.py

@@ -0,0 +1,35 @@
+from hpc.rl_launch_utils import build_apptainer_prefix
+
+
+def _container_environment(prefix: list[str]) -> dict[str, str]:
+    return {
+        value.split("=", 1)[0]: value.split("=", 1)[1]
+        for index, value in enumerate(prefix)
+        if index > 0 and prefix[index - 1] == "--env"
+    }
+
+
+def test_apptainer_runtime_bounds_both_raylet_startup_phases(monkeypatch) -> None:
+    monkeypatch.delenv("RAY_raylet_start_wait_time_s", raising=False)
+    monkeypatch.delenv("RAY_raylet_client_num_connect_attempts", raising=False)
+
+    environment = _container_environment(
+        build_apptainer_prefix("runtime.sif", binds=[])
+    )
+
+    assert environment["RAY_raylet_start_wait_time_s"] == "120"
+    assert environment["RAY_raylet_client_num_connect_attempts"] == "120"
+
+
+def test_apptainer_runtime_honors_explicit_raylet_startup_overrides(
+    monkeypatch,
+) -> None:
+    monkeypatch.setenv("RAY_raylet_start_wait_time_s", "181")
+    monkeypatch.setenv("RAY_raylet_client_num_connect_attempts", "182")
+
+    environment = _container_environment(
+        build_apptainer_prefix("runtime.sif", binds=[])
+    )
+
+    assert environment["RAY_raylet_start_wait_time_s"] == "181"
+    assert environment["RAY_raylet_client_num_connect_attempts"] == "182"