From d8e68e97f57072a555481249e7abef87a1c569a2 Mon Sep 17 00:00:00 2001 From: Benjamin Feuer Date: Thu, 6 Aug 2026 10:22:39 -0400 Subject: [PATCH 1/3] [hpc] Bound Ray plasma startup MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pass Ray’s plasma-socket connection-attempt limit through the shared Apptainer prefix. Large object-store mappings can exceed Ray’s ten-second default even when GCS is healthy, causing otherwise identical cluster starts to fail nondeterministically. --- docs/debug-log-ray-plasma-startup.md | 44 +++++++++++++++++++++++++ hpc/rl_launch_utils.py | 21 ++++-------- tests/hpc/test_apptainer_ray_startup.py | 35 ++++++++++++++++++++ 3 files changed, 86 insertions(+), 14 deletions(-) create mode 100644 docs/debug-log-ray-plasma-startup.md create mode 100644 tests/hpc/test_apptainer_ray_startup.py diff --git a/docs/debug-log-ray-plasma-startup.md b/docs/debug-log-ray-plasma-startup.md new file mode 100644 index 000000000..38c8fa2cc --- /dev/null +++ b/docs/debug-log-ray-plasma-startup.md @@ -0,0 +1,44 @@ +# Debugging log for Ray plasma startup + +Keep large-object-store Ray clusters from failing nondeterministically while the local plasma store initializes. + +## Initial status + +Jupiter job 1262271 started GCS and registered all five worker nodes, but its head raylet exited before the +cluster became usable. The preserved `raylet.out` records ten failed attempts to connect to the local +`plasma_store` socket, followed by a fatal `No such file or directory`. The launcher then reached its bounded +cluster-startup deadline and preserved the Ray logs. + +## Hypothesis 1 + +The existing `RAY_raylet_start_wait_time_s=120` setting also governs the local plasma-socket connection. + +## Results + +Refuted by Ray 2.51.1 source. GCS registration reads `raylet_start_wait_time_s`; the plasma client instead reads +`raylet_client_num_connect_attempts`, whose default is ten attempts at one-second intervals. + +The immediately preceding job 1262020 used the same 40 GiB object-store size and runtime. Its plasma store became +ready after about eight seconds. In job 1262271, initialization exceeded ten seconds and the raylet aborted. This +paired observation supports a startup race rather than an unreachable head node. + +## Hypothesis 2 + +Passing `RAY_raylet_client_num_connect_attempts=120` through the shared Apptainer prefix will bound the same +plasma-socket startup phase without weakening the existing GCS-registration deadline. + +## Changes to make + +- Preserve `RAY_raylet_start_wait_time_s=120` for GCS registration. +- Add the independent plasma-socket attempt limit to every in-container Ray invocation. +- Test both defaults and explicit operator overrides through `build_apptainer_prefix`. + +## Results + +The focused regression test failed before the change because the plasma-socket control was absent and passes +after both independent controls are emitted. + +## Future work + +- [ ] Re-run a six-node Jupiter startup with the production 40 GiB object store and verify the head raylet remains + alive after a deliberately slow plasma mapping. diff --git a/hpc/rl_launch_utils.py b/hpc/rl_launch_utils.py index 858e8e1fe..c4aed7b9b 100644 --- a/hpc/rl_launch_utils.py +++ b/hpc/rl_launch_utils.py @@ -183,22 +183,15 @@ def build_apptainer_prefix( # later. Defaults to offline but honors an explicit host override. wandb_mode = os.environ.get("WANDB_MODE", "offline") prefix.extend(["--env", f"WANDB_MODE={wandb_mode}"]) - # Raise Ray's raylet-startup grace window. On busy 6-node GH200 allocations - # the raylet (head AND worker) intermittently fails to finish registering - # with the local GCS inside Ray's default 30s (`RAY_raylet_start_wait_time_s`), - # so `ray start` aborts with "The current node timed out during startup ... - # the GCS has become overloaded" → the driver then loops on - # "Failed to connect to GCS ... within 5 seconds" until the 600s wait window - # expires (job 930367, #232 cp2). This is slow-to-form, NOT unreachable — the - # driver runs ON the head node and still can't reach its own 6379. Ray's own - # error message recommends raising this config. apptainer passes it via --env - # so it reaches the in-SIF `ray start`; host `env KEY=val` prefixes do NOT - # cross the container boundary (and the proxychains path drops ray_env_vars - # entirely), making this --env injection the only point that reaches every - # ray invocation (head, worker, wait/poll scripts) uniformly. Honors an - # explicit host override. + # Ray has independent startup deadlines for GCS registration and for the + # local plasma-store socket. A large object-store mmap can exceed the + # plasma client's default ten one-second connection attempts even while GCS + # is healthy. Pass both controls through Apptainer so every head, worker, + # and wait process observes them; each still honors an explicit override. raylet_wait = os.environ.get("RAY_raylet_start_wait_time_s", "120") prefix.extend(["--env", f"RAY_raylet_start_wait_time_s={raylet_wait}"]) + raylet_connect_attempts = os.environ.get("RAY_raylet_client_num_connect_attempts", "120") + prefix.extend(["--env", f"RAY_raylet_client_num_connect_attempts={raylet_connect_attempts}"]) prefix.append(sif) return prefix diff --git a/tests/hpc/test_apptainer_ray_startup.py b/tests/hpc/test_apptainer_ray_startup.py new file mode 100644 index 000000000..6c4b06d6d --- /dev/null +++ b/tests/hpc/test_apptainer_ray_startup.py @@ -0,0 +1,35 @@ +from hpc.rl_launch_utils import build_apptainer_prefix + + +def _container_environment(prefix: list[str]) -> dict[str, str]: + return { + value.split("=", 1)[0]: value.split("=", 1)[1] + for index, value in enumerate(prefix) + if index > 0 and prefix[index - 1] == "--env" + } + + +def test_apptainer_runtime_bounds_both_raylet_startup_phases(monkeypatch) -> None: + monkeypatch.delenv("RAY_raylet_start_wait_time_s", raising=False) + monkeypatch.delenv("RAY_raylet_client_num_connect_attempts", raising=False) + + environment = _container_environment( + build_apptainer_prefix("runtime.sif", binds=[]) + ) + + assert environment["RAY_raylet_start_wait_time_s"] == "120" + assert environment["RAY_raylet_client_num_connect_attempts"] == "120" + + +def test_apptainer_runtime_honors_explicit_raylet_startup_overrides( + monkeypatch, +) -> None: + monkeypatch.setenv("RAY_raylet_start_wait_time_s", "181") + monkeypatch.setenv("RAY_raylet_client_num_connect_attempts", "182") + + environment = _container_environment( + build_apptainer_prefix("runtime.sif", binds=[]) + ) + + assert environment["RAY_raylet_start_wait_time_s"] == "181" + assert environment["RAY_raylet_client_num_connect_attempts"] == "182" From a3e93cddc3fc4b77c5e24fbfecb5ae8e3d96ebd3 Mon Sep 17 00:00:00 2001 From: Benjamin Feuer Date: Thu, 6 Aug 2026 10:28:17 -0400 Subject: [PATCH 2/3] [hpc] Record Ray startup validation --- docs/debug-log-ray-plasma-startup.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/docs/debug-log-ray-plasma-startup.md b/docs/debug-log-ray-plasma-startup.md index 38c8fa2cc..dd95914ac 100644 --- a/docs/debug-log-ray-plasma-startup.md +++ b/docs/debug-log-ray-plasma-startup.md @@ -38,7 +38,10 @@ plasma-socket startup phase without weakening the existing GCS-registration dead The focused regression test failed before the change because the plasma-socket control was absent and passes after both independent controls are emitted. +Jupiter job 1264144 validated the runtime contract against Ray 2.51.1 in the production r5 SIF. It enabled +plasma page preallocation, created a 40 GiB object store, connected a driver, observed all four GH200 GPUs, and +shut Ray down cleanly. Slurm completed the job with exit code 0 in 42 seconds. + ## Future work -- [ ] Re-run a six-node Jupiter startup with the production 40 GiB object store and verify the head raylet remains - alive after a deliberately slow plasma mapping. +- [ ] Confirm the next six-node TaskTrove resume forms the complete Ray cluster before trainer initialization. From 2f36f4327cc4d98f06833f9823a61482799ce728 Mon Sep 17 00:00:00 2001 From: Benjamin Feuer Date: Thu, 6 Aug 2026 10:28:17 -0400 Subject: [PATCH 3/3] [tests] Format watcher regression tests --- tests/analysis/test_watch_iris_harbor.py | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/tests/analysis/test_watch_iris_harbor.py b/tests/analysis/test_watch_iris_harbor.py index b33137d02..9542c7791 100644 --- a/tests/analysis/test_watch_iris_harbor.py +++ b/tests/analysis/test_watch_iris_harbor.py @@ -662,7 +662,7 @@ def test_glm52_capacity_floor_keeps_normal_single_node_progress_healthy(): def test_harbor_job_kind_classifies_evalchemy_and_keeps_existing_kinds(): assert ( watcher.harbor_job_kind( - 'bash -c \'exec /opt/eval/evalchemy/.venv/bin/python ' + "bash -c 'exec /opt/eval/evalchemy/.venv/bin/python " '"$IRIS_WORKDIR/experiments/evals/evalchemy/run_evalchemy_client.py"\'', "/owner/eval-eval-abcd1234", ) @@ -702,7 +702,9 @@ def test_read_gcs_progress_falls_back_to_per_trial_when_aggregate_missing( monkeypatch, tmp_path ): # A running datagen job whose Harbor aggregate result.json is not on GCS yet. - job = _job(jobs_dir="gs://marin-us-central1/ot-agent/run-xyz", harbor_job_name="run-xyz") + job = _job( + jobs_dir="gs://marin-us-central1/ot-agent/run-xyz", harbor_job_name="run-xyz" + ) def fake_cat(*_args, **_kwargs): return SimpleNamespace( @@ -770,10 +772,18 @@ def test_read_evalchemy_progress_counts_non_empty_results(monkeypatch, tmp_path) watcher, "iter_objects", lambda *_a, **_k: [ - {"Key": "iris/marinbase-eval/qwen-qwen3-30b-a3b/tier2-run1/MATH500_0shot/qwen/results_1.json"}, - {"Key": "iris/marinbase-eval/qwen-qwen3-30b-a3b/tier2-run1/HumanEvalPlus_0shot/qwen/results_2.json"}, - {"Key": "iris/marinbase-eval/qwen-qwen3-30b-a3b/tier2-run1/MBPPPlus_0shot/qwen/empty_results.json"}, - {"Key": "iris/marinbase-eval/qwen-qwen3-30b-a3b/tier2-run1/MATH500_0shot/qwen/samples_MATH500.jsonl"}, + { + "Key": "iris/marinbase-eval/qwen-qwen3-30b-a3b/tier2-run1/MATH500_0shot/qwen/results_1.json" + }, + { + "Key": "iris/marinbase-eval/qwen-qwen3-30b-a3b/tier2-run1/HumanEvalPlus_0shot/qwen/results_2.json" + }, + { + "Key": "iris/marinbase-eval/qwen-qwen3-30b-a3b/tier2-run1/MBPPPlus_0shot/qwen/empty_results.json" + }, + { + "Key": "iris/marinbase-eval/qwen-qwen3-30b-a3b/tier2-run1/MATH500_0shot/qwen/samples_MATH500.jsonl" + }, ], )