diff --git a/docs/debug-log-ray-plasma-startup.md b/docs/debug-log-ray-plasma-startup.md new file mode 100644 index 000000000..dd95914ac --- /dev/null +++ b/docs/debug-log-ray-plasma-startup.md @@ -0,0 +1,47 @@ +# Debugging log for Ray plasma startup + +Keep large-object-store Ray clusters from failing nondeterministically while the local plasma store initializes. + +## Initial status + +Jupiter job 1262271 started GCS and registered all five worker nodes, but its head raylet exited before the +cluster became usable. The preserved `raylet.out` records ten failed attempts to connect to the local +`plasma_store` socket, followed by a fatal `No such file or directory`. The launcher then reached its bounded +cluster-startup deadline and preserved the Ray logs. + +## Hypothesis 1 + +The existing `RAY_raylet_start_wait_time_s=120` setting also governs the local plasma-socket connection. + +## Results + +Refuted by Ray 2.51.1 source. GCS registration reads `raylet_start_wait_time_s`; the plasma client instead reads +`raylet_client_num_connect_attempts`, whose default is ten attempts at one-second intervals. + +The immediately preceding job 1262020 used the same 40 GiB object-store size and runtime. Its plasma store became +ready after about eight seconds. In job 1262271, initialization exceeded ten seconds and the raylet aborted. This +paired observation supports a startup race rather than an unreachable head node. + +## Hypothesis 2 + +Passing `RAY_raylet_client_num_connect_attempts=120` through the shared Apptainer prefix will bound the same +plasma-socket startup phase without weakening the existing GCS-registration deadline. + +## Changes to make + +- Preserve `RAY_raylet_start_wait_time_s=120` for GCS registration. +- Add the independent plasma-socket attempt limit to every in-container Ray invocation. +- Test both defaults and explicit operator overrides through `build_apptainer_prefix`. + +## Results + +The focused regression test failed before the change because the plasma-socket control was absent and passes +after both independent controls are emitted. + +Jupiter job 1264144 validated the runtime contract against Ray 2.51.1 in the production r5 SIF. It enabled +plasma page preallocation, created a 40 GiB object store, connected a driver, observed all four GH200 GPUs, and +shut Ray down cleanly. Slurm completed the job with exit code 0 in 42 seconds. + +## Future work + +- [ ] Confirm the next six-node TaskTrove resume forms the complete Ray cluster before trainer initialization. diff --git a/hpc/rl_launch_utils.py b/hpc/rl_launch_utils.py index 858e8e1fe..c4aed7b9b 100644 --- a/hpc/rl_launch_utils.py +++ b/hpc/rl_launch_utils.py @@ -183,22 +183,15 @@ def build_apptainer_prefix( # later. Defaults to offline but honors an explicit host override. wandb_mode = os.environ.get("WANDB_MODE", "offline") prefix.extend(["--env", f"WANDB_MODE={wandb_mode}"]) - # Raise Ray's raylet-startup grace window. On busy 6-node GH200 allocations - # the raylet (head AND worker) intermittently fails to finish registering - # with the local GCS inside Ray's default 30s (`RAY_raylet_start_wait_time_s`), - # so `ray start` aborts with "The current node timed out during startup ... - # the GCS has become overloaded" → the driver then loops on - # "Failed to connect to GCS ... within 5 seconds" until the 600s wait window - # expires (job 930367, #232 cp2). This is slow-to-form, NOT unreachable — the - # driver runs ON the head node and still can't reach its own 6379. Ray's own - # error message recommends raising this config. apptainer passes it via --env - # so it reaches the in-SIF `ray start`; host `env KEY=val` prefixes do NOT - # cross the container boundary (and the proxychains path drops ray_env_vars - # entirely), making this --env injection the only point that reaches every - # ray invocation (head, worker, wait/poll scripts) uniformly. Honors an - # explicit host override. + # Ray has independent startup deadlines for GCS registration and for the + # local plasma-store socket. A large object-store mmap can exceed the + # plasma client's default ten one-second connection attempts even while GCS + # is healthy. Pass both controls through Apptainer so every head, worker, + # and wait process observes them; each still honors an explicit override. raylet_wait = os.environ.get("RAY_raylet_start_wait_time_s", "120") prefix.extend(["--env", f"RAY_raylet_start_wait_time_s={raylet_wait}"]) + raylet_connect_attempts = os.environ.get("RAY_raylet_client_num_connect_attempts", "120") + prefix.extend(["--env", f"RAY_raylet_client_num_connect_attempts={raylet_connect_attempts}"]) prefix.append(sif) return prefix diff --git a/tests/analysis/test_watch_iris_harbor.py b/tests/analysis/test_watch_iris_harbor.py index b33137d02..9542c7791 100644 --- a/tests/analysis/test_watch_iris_harbor.py +++ b/tests/analysis/test_watch_iris_harbor.py @@ -662,7 +662,7 @@ def test_glm52_capacity_floor_keeps_normal_single_node_progress_healthy(): def test_harbor_job_kind_classifies_evalchemy_and_keeps_existing_kinds(): assert ( watcher.harbor_job_kind( - 'bash -c \'exec /opt/eval/evalchemy/.venv/bin/python ' + "bash -c 'exec /opt/eval/evalchemy/.venv/bin/python " '"$IRIS_WORKDIR/experiments/evals/evalchemy/run_evalchemy_client.py"\'', "/owner/eval-eval-abcd1234", ) @@ -702,7 +702,9 @@ def test_read_gcs_progress_falls_back_to_per_trial_when_aggregate_missing( monkeypatch, tmp_path ): # A running datagen job whose Harbor aggregate result.json is not on GCS yet. - job = _job(jobs_dir="gs://marin-us-central1/ot-agent/run-xyz", harbor_job_name="run-xyz") + job = _job( + jobs_dir="gs://marin-us-central1/ot-agent/run-xyz", harbor_job_name="run-xyz" + ) def fake_cat(*_args, **_kwargs): return SimpleNamespace( @@ -770,10 +772,18 @@ def test_read_evalchemy_progress_counts_non_empty_results(monkeypatch, tmp_path) watcher, "iter_objects", lambda *_a, **_k: [ - {"Key": "iris/marinbase-eval/qwen-qwen3-30b-a3b/tier2-run1/MATH500_0shot/qwen/results_1.json"}, - {"Key": "iris/marinbase-eval/qwen-qwen3-30b-a3b/tier2-run1/HumanEvalPlus_0shot/qwen/results_2.json"}, - {"Key": "iris/marinbase-eval/qwen-qwen3-30b-a3b/tier2-run1/MBPPPlus_0shot/qwen/empty_results.json"}, - {"Key": "iris/marinbase-eval/qwen-qwen3-30b-a3b/tier2-run1/MATH500_0shot/qwen/samples_MATH500.jsonl"}, + { + "Key": "iris/marinbase-eval/qwen-qwen3-30b-a3b/tier2-run1/MATH500_0shot/qwen/results_1.json" + }, + { + "Key": "iris/marinbase-eval/qwen-qwen3-30b-a3b/tier2-run1/HumanEvalPlus_0shot/qwen/results_2.json" + }, + { + "Key": "iris/marinbase-eval/qwen-qwen3-30b-a3b/tier2-run1/MBPPPlus_0shot/qwen/empty_results.json" + }, + { + "Key": "iris/marinbase-eval/qwen-qwen3-30b-a3b/tier2-run1/MATH500_0shot/qwen/samples_MATH500.jsonl" + }, ], ) diff --git a/tests/hpc/test_apptainer_ray_startup.py b/tests/hpc/test_apptainer_ray_startup.py new file mode 100644 index 000000000..6c4b06d6d --- /dev/null +++ b/tests/hpc/test_apptainer_ray_startup.py @@ -0,0 +1,35 @@ +from hpc.rl_launch_utils import build_apptainer_prefix + + +def _container_environment(prefix: list[str]) -> dict[str, str]: + return { + value.split("=", 1)[0]: value.split("=", 1)[1] + for index, value in enumerate(prefix) + if index > 0 and prefix[index - 1] == "--env" + } + + +def test_apptainer_runtime_bounds_both_raylet_startup_phases(monkeypatch) -> None: + monkeypatch.delenv("RAY_raylet_start_wait_time_s", raising=False) + monkeypatch.delenv("RAY_raylet_client_num_connect_attempts", raising=False) + + environment = _container_environment( + build_apptainer_prefix("runtime.sif", binds=[]) + ) + + assert environment["RAY_raylet_start_wait_time_s"] == "120" + assert environment["RAY_raylet_client_num_connect_attempts"] == "120" + + +def test_apptainer_runtime_honors_explicit_raylet_startup_overrides( + monkeypatch, +) -> None: + monkeypatch.setenv("RAY_raylet_start_wait_time_s", "181") + monkeypatch.setenv("RAY_raylet_client_num_connect_attempts", "182") + + environment = _container_environment( + build_apptainer_prefix("runtime.sif", binds=[]) + ) + + assert environment["RAY_raylet_start_wait_time_s"] == "181" + assert environment["RAY_raylet_client_num_connect_attempts"] == "182"