From fba04f6f8b43009ae8481c1fa359abf1db8f4675 Mon Sep 17 00:00:00 2001 From: Jacob Ioffe Date: Tue, 14 Jul 2026 17:45:58 +0000 Subject: [PATCH 01/11] ops(harness): add portable latest-main nightlies Signed-off-by: Jacob Ioffe --- .../src/nemo_retriever/harness/artifacts.py | 8 +- .../src/nemo_retriever/harness/cli.py | 44 ++- .../src/nemo_retriever/harness/runsets.py | 97 +++++- .../nemo_retriever/harness/vidore_access.py | 151 +++++++++ .../tests/test_harness_artifacts.py | 14 + .../test_harness_latest_main_launcher.py | 209 ++++++++++++ .../tests/test_harness_nightly_launcher.py | 270 +++++++++++++++ nemo_retriever/tests/test_harness_runfiles.py | 78 +++++ .../tests/test_harness_vidore_access.py | 141 ++++++++ ops/retriever-nightly/README.md | 307 ++++++++++++++++++ .../SECOND_HOST_VALIDATION.md | 144 ++++++++ .../dataset_paths.datasets.yaml | 33 ++ ops/retriever-nightly/nightly.env.example | 16 + ops/retriever-nightly/run-latest-main.sh | 200 ++++++++++++ ops/retriever-nightly/run-nightly.sh | 291 +++++++++++++++++ .../systemd/nrl-harness-batch-hf.service | 19 ++ .../systemd/nrl-harness-batch-hf.timer | 14 + 17 files changed, 2024 insertions(+), 12 deletions(-) create mode 100644 nemo_retriever/src/nemo_retriever/harness/vidore_access.py create mode 100644 nemo_retriever/tests/test_harness_latest_main_launcher.py create mode 100644 nemo_retriever/tests/test_harness_nightly_launcher.py create mode 100644 nemo_retriever/tests/test_harness_vidore_access.py create mode 100644 ops/retriever-nightly/README.md create mode 100644 ops/retriever-nightly/SECOND_HOST_VALIDATION.md create mode 100644 ops/retriever-nightly/dataset_paths.datasets.yaml create mode 100644 ops/retriever-nightly/nightly.env.example create mode 100755 ops/retriever-nightly/run-latest-main.sh create mode 100755 ops/retriever-nightly/run-nightly.sh create mode 100644 ops/retriever-nightly/systemd/nrl-harness-batch-hf.service create mode 100644 ops/retriever-nightly/systemd/nrl-harness-batch-hf.timer diff --git a/nemo_retriever/src/nemo_retriever/harness/artifacts.py b/nemo_retriever/src/nemo_retriever/harness/artifacts.py index 0f41956bf2..5416f6e7f5 100644 --- a/nemo_retriever/src/nemo_retriever/harness/artifacts.py +++ b/nemo_retriever/src/nemo_retriever/harness/artifacts.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2024-25, NVIDIA CORPORATION & AFFILIATES. +# SPDX-FileCopyrightText: Copyright (c) 2024-26, NVIDIA CORPORATION & AFFILIATES. # All rights reserved. # SPDX-License-Identifier: Apache-2.0 @@ -14,7 +14,7 @@ NEMO_RETRIEVER_ROOT = Path(__file__).resolve().parents[3] DEFAULT_ARTIFACTS_ROOT = NEMO_RETRIEVER_ROOT / "artifacts" -_COMMIT_RE = re.compile(r"^[0-9a-fA-F]{7,40}$") +_COMMIT_RE = re.compile(r"^[0-9a-fA-F]{7,64}$") def now_timestr() -> str: @@ -25,7 +25,7 @@ def _normalize_commit(value: str | None) -> str | None: text = (value or "").strip() if not _COMMIT_RE.match(text): return None - return text[:7] + return text.lower() def _resolve_git_dir(repo_root: Path) -> Path | None: @@ -98,7 +98,7 @@ def last_commit() -> str: repo_root = NEMO_RETRIEVER_ROOT.parent try: result = subprocess.run( - ["git", "rev-parse", "--short", "HEAD"], + ["git", "rev-parse", "HEAD"], cwd=str(repo_root), check=False, capture_output=True, diff --git a/nemo_retriever/src/nemo_retriever/harness/cli.py b/nemo_retriever/src/nemo_retriever/harness/cli.py index 94f2890ee9..6541c49241 100644 --- a/nemo_retriever/src/nemo_retriever/harness/cli.py +++ b/nemo_retriever/src/nemo_retriever/harness/cli.py @@ -15,7 +15,7 @@ list_benchmarks, list_runsets, ) -from nemo_retriever.harness.contracts import EXIT_INVALID, FailurePayload +from nemo_retriever.harness.contracts import EXIT_INVALID, EXIT_MISSING_INPUT, FailurePayload from nemo_retriever.harness.revamp_runner import ( HarnessRunError, run_benchmark, @@ -32,6 +32,7 @@ post_slack_payload, resolve_slack_webhook_url, ) +from nemo_retriever.harness.vidore_access import VidoreAccessError, check_vidore_access app = typer.Typer( help=( @@ -45,6 +46,42 @@ def _echo_json(payload: object) -> None: typer.echo(json.dumps(payload, indent=2, sort_keys=False)) +@app.command("check-vidore-access") +def check_vidore_access_command( + dataset_names: Annotated[ + list[str] | None, + typer.Option("--dataset", help="Check one ViDoRe dataset name. Repeat to check multiple datasets."), + ] = None, + timeout_seconds: Annotated[ + float, + typer.Option("--timeout-seconds", min=1.0, help="HTTP timeout for each streamed range request."), + ] = 20.0, + require_token: Annotated[ + bool, + typer.Option("--require-token/--allow-anonymous", help="Require HF_TOKEN before checking public data."), + ] = True, + json_output: Annotated[bool, typer.Option("--json", help="Emit machine-readable JSON.")] = False, +) -> None: + """Verify ViDoRe queries, qrels, and corpus objects without downloading them.""" + + try: + results = check_vidore_access( + dataset_names=dataset_names, + timeout_seconds=timeout_seconds, + require_token=require_token, + ) + except VidoreAccessError as exc: + typer.echo(str(exc), err=True) + raise typer.Exit(code=EXIT_MISSING_INPUT) from exc + + payload = {"success": True, "datasets": [result.to_dict() for result in results]} + if json_output: + _echo_json(payload) + return + for result in results: + typer.echo(f"{result.dataset}: access ok ({', '.join(result.checked_partitions)})") + + @app.command("list") def list_command( json_output: Annotated[bool, typer.Option("--json", help="Emit machine-readable JSON.")] = False, @@ -263,6 +300,10 @@ def run_files_command( bool, typer.Option("--dry-run", help="Resolve configuration and write plans without executing ingest or query."), ] = False, + isolate_runs: Annotated[ + bool, + typer.Option("--isolate-runs", help="Run each child in a fresh process to release batch memory."), + ] = False, json_output: Annotated[bool, typer.Option("--json", help="Emit session summary JSON to stdout.")] = False, ) -> None: """Run one or more runfiles, optionally with machine-local dataset paths.""" @@ -277,6 +318,7 @@ def run_files_command( overrides=set_values or (), requirements=requirements or (), dry_run=dry_run, + isolate_runs=isolate_runs, ) except HarnessRunError as exc: typer.echo(exc.failure.message, err=True) diff --git a/nemo_retriever/src/nemo_retriever/harness/runsets.py b/nemo_retriever/src/nemo_retriever/harness/runsets.py index ef43e8bc36..a05c3370c2 100644 --- a/nemo_retriever/src/nemo_retriever/harness/runsets.py +++ b/nemo_retriever/src/nemo_retriever/harness/runsets.py @@ -6,6 +6,7 @@ from dataclasses import dataclass from difflib import get_close_matches +import multiprocessing from pathlib import Path import re from typing import Any, Mapping, Sequence @@ -218,6 +219,75 @@ def _failed_child_outcome( ) +def _isolated_run_worker(connection: Any, run: PreparedRun, *, output_dir: str, run_id: str) -> None: + """Execute one prepared benchmark in a fresh spawned process.""" + + try: + outcome = run_prepared_benchmark( + run.prepared, + output_dir=output_dir, + run_id=run_id, + runfile_payload=run.runfile_payload, + runfile_path=run.runfile_path, + ) + connection.send(("outcome", outcome)) + except Exception as exc: + connection.send(("error", f"{type(exc).__name__}: {exc}")) + finally: + connection.close() + + +def _run_prepared_benchmark_isolated(run: PreparedRun, *, output_dir: str, run_id: str) -> RunOutcome: + """Run one child with a process boundary that releases Ray and dataframe memory.""" + + context = multiprocessing.get_context("spawn") + receive_connection, send_connection = context.Pipe(duplex=False) + process = context.Process( + target=_isolated_run_worker, + kwargs={ + "connection": send_connection, + "run": run, + "output_dir": output_dir, + "run_id": run_id, + }, + name=f"retriever-harness-{run.artifact_name}", + ) + try: + process.start() + except BaseException: + receive_connection.close() + send_connection.close() + raise + send_connection.close() + + message: tuple[str, Any] | None = None + try: + try: + message = receive_connection.recv() + except EOFError: + pass + except BaseException: + if process.is_alive(): + process.terminate() + process.join() + raise + finally: + receive_connection.close() + + process.join() + if process.exitcode != 0: + raise RuntimeError(f"isolated benchmark process exited with code {process.exitcode}") + if message is None: + raise RuntimeError("isolated benchmark process returned no outcome") + + message_type, payload = message + if message_type == "error": + raise RuntimeError(str(payload)) + if message_type != "outcome" or not isinstance(payload, RunOutcome): + raise RuntimeError("isolated benchmark process returned an invalid outcome") + return payload + + def _session_summary( *, session_type: str, @@ -252,6 +322,7 @@ def _run_session( run_commit: str, dry_run: bool, summary_extra: Mapping[str, Any], + isolate_runs: bool = False, ) -> RunOutcome: try: session_dir.mkdir(parents=True, exist_ok=True) @@ -264,14 +335,22 @@ def _run_session( exit_code = EXIT_SUCCESS for run in runs: artifact_dir = session_dir / run.artifact_name + run_id = f"{session_name}_{run.artifact_name}" try: - outcome = run_prepared_benchmark( - run.prepared, - output_dir=str(artifact_dir), - run_id=f"{session_name}_{run.artifact_name}", - runfile_payload=run.runfile_payload, - runfile_path=run.runfile_path, - ) + if isolate_runs: + outcome = _run_prepared_benchmark_isolated( + run, + output_dir=str(artifact_dir), + run_id=run_id, + ) + else: + outcome = run_prepared_benchmark( + run.prepared, + output_dir=str(artifact_dir), + run_id=run_id, + runfile_payload=run.runfile_payload, + runfile_path=run.runfile_path, + ) except Exception as exc: outcome = _failed_child_outcome( benchmark=run.prepared.benchmark, @@ -372,6 +451,7 @@ def run_runfiles( overrides: Sequence[str] = (), requirements: Sequence[str] = (), dry_run: bool = False, + isolate_runs: bool = False, ) -> RunOutcome: if not runfiles: raise _invalid_runfile("At least one runfile path is required.") @@ -441,6 +521,7 @@ def run_runfiles( "session_name": session_name, "run_commit": run_commit, "dataset_paths_file": dataset_paths_value, + "isolate_runs": bool(isolate_runs), "runfiles": expanded_runs, }, run_commit=run_commit, @@ -448,5 +529,7 @@ def run_runfiles( summary_extra={ "session_name": session_name, "dataset_paths_file": dataset_paths_value, + "isolate_runs": bool(isolate_runs), }, + isolate_runs=isolate_runs, ) diff --git a/nemo_retriever/src/nemo_retriever/harness/vidore_access.py b/nemo_retriever/src/nemo_retriever/harness/vidore_access.py new file mode 100644 index 0000000000..526321bb40 --- /dev/null +++ b/nemo_retriever/src/nemo_retriever/harness/vidore_access.py @@ -0,0 +1,151 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# SPDX-License-Identifier: Apache-2.0 + +"""Fast authenticated access checks for remote ViDoRe evaluation data.""" + +from __future__ import annotations + +from collections.abc import Callable, Sequence +from dataclasses import asdict, dataclass +import os +from typing import Any +from urllib.parse import urlparse + +from nemo_retriever.harness.benchmark_registry import get_benchmark, get_runset + +_EVALUATION_PARTITIONS = ("queries", "qrels", "corpus") +_SUCCESS_STATUSES = {200, 206} + + +class VidoreAccessError(RuntimeError): + """Raised when a ViDoRe repository or one of its data objects is inaccessible.""" + + +@dataclass(frozen=True) +class VidoreAccessResult: + dataset: str + revision: str + checked_partitions: tuple[str, ...] + authenticated: bool + + def to_dict(self) -> dict[str, Any]: + return asdict(self) + + +def default_vidore_dataset_names() -> tuple[str, ...]: + """Return the registry-owned ViDoRe v3 dataset names used by the nightly.""" + + runset = get_runset("vidore_v3_all") + return tuple(get_benchmark(benchmark).dataset for benchmark in runset.runs) + + +def _exception_status(exc: BaseException) -> int | None: + response = getattr(exc, "response", None) + return getattr(response, "status_code", None) + + +def _safe_api_error(*, operation: str, dataset: str, exc: BaseException) -> VidoreAccessError: + status = _exception_status(exc) + status_suffix = f" (HTTP {status})" if status is not None else "" + return VidoreAccessError(f"{operation} failed for {dataset}{status_suffix}: {type(exc).__name__}") + + +def _partition_files(info: Any, *, dataset: str) -> dict[str, str]: + files = [str(sibling.rfilename) for sibling in getattr(info, "siblings", ())] + selected: dict[str, str] = {} + for partition in _EVALUATION_PARTITIONS: + matches = sorted(path for path in files if path.startswith(f"{partition}/") and path.endswith(".parquet")) + if matches: + selected[partition] = matches[0] + missing = [partition for partition in _EVALUATION_PARTITIONS if partition not in selected] + if missing: + raise VidoreAccessError(f"{dataset} is missing evaluation partitions: {', '.join(missing)}") + return selected + + +def check_vidore_access( + *, + dataset_names: Sequence[str] | None = None, + token: str | None = None, + timeout_seconds: float = 20.0, + require_token: bool = True, + api: Any | None = None, + get: Callable[..., Any] | None = None, +) -> list[VidoreAccessResult]: + """Probe one byte from each ViDoRe queries/qrels/corpus partition. + + The range requests follow Hugging Face redirects but stream the response, + so a successful check does not download the underlying parquet objects. + Error messages intentionally exclude tokens and signed URLs. + """ + + effective_token = token or os.environ.get("HF_TOKEN") or os.environ.get("HUGGING_FACE_HUB_TOKEN") + if require_token and not effective_token: + raise VidoreAccessError( + "HF_TOKEN is not set; add it to the mode-600 nightly configuration before checking ViDoRe access" + ) + + if api is None: + from huggingface_hub import HfApi + + api = HfApi(token=effective_token) + if get is None: + import requests + + get = requests.get + + authenticated = bool(effective_token) + if authenticated: + try: + api.whoami() + except Exception as exc: + raise _safe_api_error(operation="HF_TOKEN validation", dataset="Hugging Face", exc=exc) from exc + + from huggingface_hub import hf_hub_url + + results: list[VidoreAccessResult] = [] + for dataset in tuple(dataset_names or default_vidore_dataset_names()): + repo_id = f"vidore/{dataset}" + try: + info = api.dataset_info(repo_id, files_metadata=True) + except Exception as exc: + raise _safe_api_error(operation="dataset metadata request", dataset=dataset, exc=exc) from exc + + revision = str(getattr(info, "sha", None) or "main") + partition_files = _partition_files(info, dataset=dataset) + for partition, filename in partition_files.items(): + url = hf_hub_url(repo_id=repo_id, filename=filename, repo_type="dataset", revision=revision) + headers = {"Range": "bytes=0-0"} + if effective_token: + headers["Authorization"] = f"Bearer {effective_token}" + try: + with get( + url, + headers=headers, + allow_redirects=True, + stream=True, + timeout=timeout_seconds, + ) as response: + status_chain = [item.status_code for item in response.history] + [response.status_code] + if response.status_code not in _SUCCESS_STATUSES: + chain = " -> ".join(str(status) for status in status_chain) + final_host = urlparse(response.url).hostname or "unknown host" + raise VidoreAccessError( + f"{dataset} {filename} failed with status chain {chain} at {final_host}" + ) + if not next(response.iter_content(chunk_size=1), b""): + raise VidoreAccessError(f"{dataset} {filename} returned an empty response") + except VidoreAccessError: + raise + except Exception as exc: + raise VidoreAccessError(f"{dataset} {partition} range request failed: {type(exc).__name__}") from exc + + results.append( + VidoreAccessResult( + dataset=dataset, + revision=revision, + checked_partitions=_EVALUATION_PARTITIONS, + authenticated=authenticated, + ) + ) + return results diff --git a/nemo_retriever/tests/test_harness_artifacts.py b/nemo_retriever/tests/test_harness_artifacts.py index 8e22ebc0a7..acd5e97c80 100644 --- a/nemo_retriever/tests/test_harness_artifacts.py +++ b/nemo_retriever/tests/test_harness_artifacts.py @@ -4,6 +4,7 @@ from contextlib import contextmanager import json +import subprocess import pytest @@ -31,6 +32,19 @@ def _write_json(path, payload): path.write_text(json.dumps(payload), encoding="utf-8") +def test_last_commit_preserves_the_full_git_sha(monkeypatch): + import nemo_retriever.harness.artifacts as artifacts + + full_sha = "abcdef0123456789abcdef0123456789abcdef01" + monkeypatch.setattr( + artifacts.subprocess, + "run", + lambda *args, **kwargs: subprocess.CompletedProcess(args[0], 0, stdout=f"{full_sha}\n", stderr=""), + ) + + assert artifacts.last_commit() == full_sha + + def test_artifact_manifest_uses_relative_paths_and_includes_lancedb(tmp_path): writer = ArtifactWriter(artifact_dir=tmp_path, run_id="run-1", benchmark="jp20_beir") writer.status(status="running", phase="ingest") diff --git a/nemo_retriever/tests/test_harness_latest_main_launcher.py b/nemo_retriever/tests/test_harness_latest_main_launcher.py new file mode 100644 index 0000000000..f769802afd --- /dev/null +++ b/nemo_retriever/tests/test_harness_latest_main_launcher.py @@ -0,0 +1,209 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import os +from pathlib import Path +import subprocess + +import pytest + + +REPO_ROOT = Path(__file__).resolve().parents[2] +LATEST_MAIN_LAUNCHER = REPO_ROOT / "ops" / "retriever-nightly" / "run-latest-main.sh" + + +def _git(repository: Path, *args: str) -> subprocess.CompletedProcess[str]: + return subprocess.run( + ["git", "-C", str(repository), *args], + check=True, + capture_output=True, + text=True, + ) + + +def _commit(repository: Path, message: str) -> str: + _git(repository, "add", ".") + _git(repository, "commit", "-qm", message) + return _git(repository, "rev-parse", "HEAD").stdout.strip() + + +@pytest.fixture +def latest_main_fixture(tmp_path: Path): + source = tmp_path / "source" + source.mkdir() + subprocess.run(["git", "init", "-q", "-b", "main", str(source)], check=True) + _git(source, "config", "user.email", "nightly-test@example.com") + _git(source, "config", "user.name", "Nightly Test") + + launcher = source / "ops" / "retriever-nightly" / "run-nightly.sh" + launcher.parent.mkdir(parents=True) + launcher.write_text( + "\n".join( + ( + "#!/usr/bin/env bash", + "set -uo pipefail", + 'if [[ -n "${EXPECT_DEEP_GEMM_WARMUP+x}" && ' + '"${VLLM_DEEP_GEMM_WARMUP:-}" != "$EXPECT_DEEP_GEMM_WARMUP" ]]; then', + " exit 98", + "fi", + 'checkout="$(git -C "$(dirname -- "$0")" rev-parse --show-toplevel)"', + 'commit="$(git -C "$checkout" rev-parse HEAD)"', + 'printf "%s|%s|%s\\n" "$commit" "$*" "${UV_PROJECT_ENVIRONMENT:-}" >>"$FAKE_LATEST_CALLS"', + 'for arg in "$@"; do', + ' if [[ "$arg" == "--check-vidore-access" ]]; then', + ' exit "${FAKE_ACCESS_RC:-0}"', + " fi", + "done", + 'exit "${FAKE_RUN_RC:-0}"', + "", + ) + ), + encoding="utf-8", + ) + launcher.chmod(0o755) + (source / "version.txt").write_text("initial\n", encoding="utf-8") + initial_commit = _commit(source, "initial") + + controller = tmp_path / "controller" + subprocess.run(["git", "clone", "-q", str(source), str(controller)], check=True) + + (source / "version.txt").write_text("latest\n", encoding="utf-8") + latest_commit = _commit(source, "latest") + + config_dir = tmp_path / "config" + config_dir.mkdir() + config_file = config_dir / "nightly.env" + config_file.write_text("HF_TOKEN=test-token\n", encoding="utf-8") + config_file.chmod(0o600) + + checkout_root = tmp_path / "latest-checkouts" + calls_path = tmp_path / "latest-calls.txt" + env = os.environ.copy() + env.update( + { + "HOME": str(tmp_path / "home"), + "RETRIEVER_CONFIG_FILE": str(config_file), + "RETRIEVER_UPDATE_REPOSITORY": str(controller), + "RETRIEVER_LATEST_SOURCE": str(source), + "RETRIEVER_LATEST_REF": "main", + "RETRIEVER_LATEST_CHECKOUT_ROOT": str(checkout_root), + "RETRIEVER_LATEST_KEEP_CHECKOUTS": "2", + "FAKE_LATEST_CALLS": str(calls_path), + } + ) + + def run(*args: str, extra_env: dict[str, str] | None = None) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [str(LATEST_MAIN_LAUNCHER), *args], + cwd=tmp_path, + env=env | (extra_env or {}), + check=False, + capture_output=True, + text=True, + ) + + def calls() -> list[tuple[str, str, str]]: + if not calls_path.exists(): + return [] + return [tuple(line.split("|", 2)) for line in calls_path.read_text(encoding="utf-8").splitlines()] + + return run, calls, source, controller, checkout_root, initial_commit, latest_commit + + +def test_scheduled_launcher_fetches_latest_main_into_immutable_worktree(latest_main_fixture) -> None: + run, calls, _source, controller, checkout_root, initial_commit, latest_commit = latest_main_fixture + + result = run() + + assert result.returncode == 0, result.stderr + assert _git(controller, "rev-parse", "HEAD").stdout.strip() == initial_commit + assert calls() == [ + (latest_commit, "--check-vidore-access", str(checkout_root / ".venv")), + (latest_commit, "", str(checkout_root / ".venv")), + ] + selected_checkout = checkout_root / f"main-{latest_commit}" + assert selected_checkout.is_dir() + assert _git(selected_checkout, "rev-parse", "HEAD").stdout.strip() == latest_commit + + +def test_help_does_not_require_configuration_or_fetch(tmp_path: Path) -> None: + env = os.environ.copy() + env.update({"HOME": str(tmp_path / "missing-home")}) + env.pop("RETRIEVER_CONFIG_FILE", None) + + result = subprocess.run( + [str(LATEST_MAIN_LAUNCHER), "--help"], + env=env, + check=False, + capture_output=True, + text=True, + ) + + assert result.returncode == 0, result.stderr + assert "Fetch upstream/main" in result.stdout + + +@pytest.mark.parametrize("configured, expected", [(None, "skip"), ("full", "full")]) +def test_scheduled_run_has_safe_warmup_default_and_allows_override( + latest_main_fixture, configured: str | None, expected: str +) -> None: + run, _calls, *_rest = latest_main_fixture + extra_env = {"EXPECT_DEEP_GEMM_WARMUP": expected} + if configured is not None: + extra_env["VLLM_DEEP_GEMM_WARMUP"] = configured + + result = run("--dry-run", extra_env=extra_env) + + assert result.returncode == 0, result.stderr + + +def test_scheduled_launcher_fails_closed_when_access_preflight_fails(latest_main_fixture) -> None: + run, calls, _source, _controller, _checkout_root, _initial_commit, latest_commit = latest_main_fixture + + result = run(extra_env={"FAKE_ACCESS_RC": "3"}) + + assert result.returncode == 3 + assert calls() == [(latest_commit, "--check-vidore-access", str(_checkout_root / ".venv"))] + + +def test_scheduled_launcher_does_not_run_stale_commit_when_fetch_fails(latest_main_fixture) -> None: + run, calls, _source, _controller, _checkout_root, _initial_commit, _latest_commit = latest_main_fixture + + result = run(extra_env={"RETRIEVER_LATEST_SOURCE": str(_checkout_root / "missing-source")}) + + assert result.returncode != 0 + assert calls() == [] + + +def test_scheduled_launcher_rejects_modified_controller(latest_main_fixture) -> None: + run, calls, _source, controller, _checkout_root, _initial_commit, _latest_commit = latest_main_fixture + (controller / "version.txt").write_text("modified\n", encoding="utf-8") + + result = run() + + assert result.returncode == 64 + assert calls() == [] + + +def test_scheduled_launcher_prunes_only_old_managed_worktrees(latest_main_fixture) -> None: + run, _calls, source, _controller, checkout_root, _initial_commit, latest_commit = latest_main_fixture + assert run(extra_env={"RETRIEVER_LATEST_KEEP_CHECKOUTS": "1"}).returncode == 0 + + (source / "version.txt").write_text("newer\n", encoding="utf-8") + newer_commit = _commit(source, "newer") + result = run(extra_env={"RETRIEVER_LATEST_KEEP_CHECKOUTS": "1"}) + + assert result.returncode == 0, result.stderr + assert not (checkout_root / f"main-{latest_commit}").exists() + assert (checkout_root / f"main-{newer_commit}").is_dir() + + +def test_systemd_service_uses_latest_main_controller() -> None: + service = (REPO_ROOT / "ops" / "retriever-nightly" / "systemd" / "nrl-harness-batch-hf.service").read_text( + encoding="utf-8" + ) + + assert "ExecStart=" in service + assert "/run-latest-main.sh" in service diff --git a/nemo_retriever/tests/test_harness_nightly_launcher.py b/nemo_retriever/tests/test_harness_nightly_launcher.py new file mode 100644 index 0000000000..d2a3323e2a --- /dev/null +++ b/nemo_retriever/tests/test_harness_nightly_launcher.py @@ -0,0 +1,270 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import json +import os +from pathlib import Path +import subprocess +import sys + +import pytest + + +REPO_ROOT = Path(__file__).resolve().parents[2] +LAUNCHER = REPO_ROOT / "ops" / "retriever-nightly" / "run-nightly.sh" +DEFAULT_RUNFILES = ( + "nemo_retriever/harness/runfiles/jp20_beir.json", + "nemo_retriever/harness/runfiles/bo767_beir.json", + "nemo_retriever/harness/runfiles/earnings_beir.json", + "nemo_retriever/harness/runfiles/financebench_beir.json", + "nemo_retriever/harness/runfiles/vidore_v3_computer_science_beir.json", + "nemo_retriever/harness/runfiles/vidore_v3_energy_beir.json", + "nemo_retriever/harness/runfiles/vidore_v3_finance_en_beir.json", + "nemo_retriever/harness/runfiles/vidore_v3_finance_fr_beir.json", + "nemo_retriever/harness/runfiles/vidore_v3_hr_beir.json", + "nemo_retriever/harness/runfiles/vidore_v3_industrial_beir.json", + "nemo_retriever/harness/runfiles/vidore_v3_pharmaceuticals_beir.json", + "nemo_retriever/harness/runfiles/vidore_v3_physics_beir.json", +) +SLACK_WEBHOOK_URL = "https://hooks.slack.com/services/test/webhook/value" + + +@pytest.fixture +def nightly_launcher(tmp_path: Path): + checkout = tmp_path / "checkout" + runfiles_dir = checkout / "nemo_retriever" / "harness" / "runfiles" + runfiles_dir.mkdir(parents=True) + for relative_path in DEFAULT_RUNFILES: + (checkout / relative_path).write_text("{}\n", encoding="utf-8") + default_dataset_paths = checkout / "ops" / "retriever-nightly" / "dataset_paths.datasets.yaml" + default_dataset_paths.parent.mkdir(parents=True) + default_dataset_paths.write_text("schema_version: 1\ndatasets: {}\n", encoding="utf-8") + + subprocess.run(["git", "init", "-q", str(checkout)], check=True) + subprocess.run(["git", "-C", str(checkout), "config", "user.email", "harness-test@example.com"], check=True) + subprocess.run(["git", "-C", str(checkout), "config", "user.name", "Harness Test"], check=True) + subprocess.run(["git", "-C", str(checkout), "add", "."], check=True) + subprocess.run(["git", "-C", str(checkout), "commit", "-qm", "test fixture"], check=True) + + calls_path = tmp_path / "uv-calls.jsonl" + fake_uv = tmp_path / "uv" + fake_uv.write_text( + "\n".join( + ( + f"#!{sys.executable}", + "import json", + "import os", + "from pathlib import Path", + "import sys", + "args = sys.argv[1:]", + "expected_warmup = os.environ.get('EXPECT_DEEP_GEMM_WARMUP')", + "if expected_warmup is not None and os.environ.get('VLLM_DEEP_GEMM_WARMUP') != expected_warmup:", + " raise SystemExit(98)", + "with Path(os.environ['FAKE_UV_CALLS']).open('a', encoding='utf-8') as stream:", + " stream.write(json.dumps(args) + '\\n')", + "if 'run-files' in args:", + " if os.environ.get('SLACK_WEBHOOK_URL'):", + " raise SystemExit(97)", + " output_dir = Path(args[args.index('--output-dir') + 1])", + " output_dir.mkdir(parents=True, exist_ok=True)", + " (output_dir / 'session_summary.json').write_text('{}\\n', encoding='utf-8')", + " raise SystemExit(int(os.environ.get('FAKE_RUN_RC', '0')))", + "if 'check-vidore-access' in args:", + " raise SystemExit(int(os.environ.get('FAKE_ACCESS_RC', '0')))", + "if 'post-slack' in args:", + " if not os.environ.get('SLACK_WEBHOOK_URL'):", + " raise SystemExit(96)", + " raise SystemExit(int(os.environ.get('FAKE_POST_RC', '0')))", + "raise SystemExit(99)", + "", + ) + ), + encoding="utf-8", + ) + fake_uv.chmod(0o755) + + env = os.environ.copy() + env.update( + { + "HOME": str(tmp_path / "home"), + "RETRIEVER_CHECKOUT": str(checkout), + "RETRIEVER_NIGHTLY_ROOT": str(tmp_path / "nightly-root"), + "RETRIEVER_UV_BIN": str(fake_uv), + "FAKE_UV_CALLS": str(calls_path), + } + ) + env.pop("VLLM_DEEP_GEMM_WARMUP", None) + env.pop("SLACK_WEBHOOK_URL", None) + + def run(*args: str, extra_env: dict[str, str] | None = None) -> subprocess.CompletedProcess[str]: + command_env = env | (extra_env or {}) + return subprocess.run( + [str(LAUNCHER), *args], + cwd=tmp_path, + env=command_env, + check=False, + capture_output=True, + text=True, + ) + + def calls() -> list[list[str]]: + if not calls_path.exists(): + return [] + return [json.loads(line) for line in calls_path.read_text(encoding="utf-8").splitlines()] + + return run, calls + + +def test_normal_run_uses_root_and_checked_in_dataset_defaults(nightly_launcher, tmp_path: Path) -> None: + run, calls = nightly_launcher + + result = run("--dry-run") + + assert result.returncode == 0, result.stderr + invocation = calls()[0] + assert invocation[invocation.index("--dataset-paths") + 1].endswith( + "/ops/retriever-nightly/dataset_paths.datasets.yaml" + ) + assert Path(invocation[invocation.index("--output-dir") + 1]).parent == ( + tmp_path / "nightly-root" / "retriever-nightly-artifacts" + ) + + +def test_launcher_rejects_untracked_checkout_files(nightly_launcher, tmp_path: Path) -> None: + run, calls = nightly_launcher + (tmp_path / "checkout" / "untracked.py").write_text("print('changed')\n", encoding="utf-8") + + result = run("--dry-run") + + assert result.returncode == 64 + assert calls() == [] + assert "untracked changes" in result.stderr + + +@pytest.mark.parametrize("configured, expected", [(None, "skip"), ("full", "full")]) +def test_deep_gemm_warmup_has_safe_default_and_allows_override( + nightly_launcher, configured: str | None, expected: str +) -> None: + run, _calls = nightly_launcher + extra_env = {"EXPECT_DEEP_GEMM_WARMUP": expected} + if configured is not None: + extra_env["VLLM_DEEP_GEMM_WARMUP"] = configured + + result = run("--check-vidore-access", extra_env=extra_env) + + assert result.returncode == 0, result.stderr + + +def test_manual_dry_run_launches_full_batch_suite_without_slack(nightly_launcher, tmp_path: Path) -> None: + run, calls = nightly_launcher + cli_dataset_paths = tmp_path / "cli-dataset-paths.yaml" + cli_dataset_paths.write_text("schema_version: 1\ndatasets: {}\n", encoding="utf-8") + + result = run( + "--dataset-paths", + cli_dataset_paths.name, + "--artifact-root", + "cli-artifacts", + "--dry-run", + ) + + assert result.returncode == 0, result.stderr + assert len(calls()) == 1 + invocation = calls()[0] + assert invocation[invocation.index("--dataset-paths") + 1] == str(cli_dataset_paths) + assert Path(invocation[invocation.index("--output-dir") + 1]).parent == tmp_path / "cli-artifacts" + assert invocation[invocation.index("--mode") + 1] == "batch" + assert "--dry-run" in invocation + assert "--isolate-runs" not in invocation + assert tuple(invocation[-len(DEFAULT_RUNFILES) :]) == DEFAULT_RUNFILES + + +def test_access_preflight_checks_vidore_without_starting_a_session(nightly_launcher) -> None: + run, calls = nightly_launcher + + result = run("--check-vidore-access") + + assert result.returncode == 0, result.stderr + assert len(calls()) == 1 + invocation = calls()[0] + assert invocation[-3:] == ["retriever", "harness", "check-vidore-access"] + assert "run-files" not in invocation + + +def test_access_preflight_propagates_failure(nightly_launcher) -> None: + run, calls = nightly_launcher + + result = run("--check-vidore-access", extra_env={"FAKE_ACCESS_RC": "3"}) + + assert result.returncode == 3 + assert len(calls()) == 1 + + +def test_configured_webhook_posts_terminal_session_to_slack(nightly_launcher, tmp_path: Path) -> None: + run, calls = nightly_launcher + config_dir = tmp_path / "nightly-root" / ".config" / "nemo-retriever" / "nightly" + config_dir.mkdir(parents=True) + config_file = config_dir / "nightly.env" + config_file.write_text(f"SLACK_WEBHOOK_URL={SLACK_WEBHOOK_URL}\n", encoding="utf-8") + config_file.chmod(0o600) + + result = run() + + assert result.returncode == 0, result.stderr + assert len(calls()) == 2 + run_invocation, post_invocation = calls() + session_dir = Path(run_invocation[run_invocation.index("--output-dir") + 1]) + assert "--isolate-runs" in run_invocation + assert "post-slack" in post_invocation + assert post_invocation[-1] == str(session_dir) + assert (session_dir / ".slack_post_attempted").exists() + + +def test_missing_webhook_completes_without_slack(nightly_launcher) -> None: + run, calls = nightly_launcher + + result = run() + + assert result.returncode == 0, result.stderr + assert len(calls()) == 1 + assert "run-files" in calls()[0] + + +def test_no_slack_suppresses_configured_webhook(nightly_launcher) -> None: + run, calls = nightly_launcher + + result = run("--no-slack", extra_env={"SLACK_WEBHOOK_URL": SLACK_WEBHOOK_URL}) + + assert result.returncode == 0, result.stderr + assert len(calls()) == 1 + invocation = calls()[0] + session_dir = Path(invocation[invocation.index("--output-dir") + 1]) + assert "run-files" in invocation + assert not (session_dir / ".slack_post_attempted").exists() + + +def test_dry_run_never_posts_when_webhook_is_present(nightly_launcher) -> None: + run, calls = nightly_launcher + + result = run("--dry-run", extra_env={"SLACK_WEBHOOK_URL": SLACK_WEBHOOK_URL}) + + assert result.returncode == 0, result.stderr + assert len(calls()) == 1 + assert "--dry-run" in calls()[0] + + +def test_invalid_webhook_fails_before_work_unless_slack_is_suppressed(nightly_launcher) -> None: + run, calls = nightly_launcher + + invalid_result = run(extra_env={"SLACK_WEBHOOK_URL": "not-a-webhook"}) + + assert invalid_result.returncode == 64 + assert calls() == [] + + suppressed_result = run("--no-slack", extra_env={"SLACK_WEBHOOK_URL": "not-a-webhook"}) + + assert suppressed_result.returncode == 0, suppressed_result.stderr + assert len(calls()) == 1 diff --git a/nemo_retriever/tests/test_harness_runfiles.py b/nemo_retriever/tests/test_harness_runfiles.py index e554f15947..d8fe85fb74 100644 --- a/nemo_retriever/tests/test_harness_runfiles.py +++ b/nemo_retriever/tests/test_harness_runfiles.py @@ -192,6 +192,84 @@ def fake_run_benchmark(prepared, **kwargs): assert [run["exit_code"] for run in outcome.results["runs"]] == [10, 0] +def test_run_files_can_isolate_each_child_process(monkeypatch, tmp_path): + runfiles = [] + for name in ("jp20_beir", "bo767_beir"): + path = tmp_path / f"{name}.json" + _write_json(path, {"schema_version": 1, "name": name, "benchmark": name}) + runfiles.append(path) + + isolated_calls = [] + + def fake_isolated_run(run, *, output_dir, run_id): + isolated_calls.append((run.prepared.benchmark, run_id)) + return _successful_outcome(run.prepared.benchmark, output_dir) + + monkeypatch.setattr( + "nemo_retriever.harness.runsets._run_prepared_benchmark_isolated", + fake_isolated_run, + ) + monkeypatch.setattr( + "nemo_retriever.harness.runsets.run_prepared_benchmark", + lambda *args, **kwargs: pytest.fail("in-process runner should not be used"), + ) + + outcome = run_runfiles( + runfiles, + output_dir=str(tmp_path / "session"), + session_name="isolated", + overrides=(f'dataset.path="{tmp_path}"',), + dry_run=True, + isolate_runs=True, + ) + + assert outcome.exit_code == 0 + assert outcome.results["isolate_runs"] is True + assert isolated_calls == [ + ("jp20_beir", "isolated_001_jp20_beir"), + ("bo767_beir", "isolated_002_bo767_beir"), + ] + assert len(outcome.results["runs"]) == 2 + + +def test_run_files_spawned_child_writes_terminal_summary(tmp_path): + documents = tmp_path / "documents" + documents.mkdir() + (documents / "sample.pdf").write_bytes(b"%PDF-1.4\n%%EOF\n") + query_file = tmp_path / "queries.csv" + query_file.write_text("query_id,query\n1,test\n", encoding="utf-8") + dataset_paths = tmp_path / "dataset_paths.yaml" + dataset_paths.write_text( + "\n".join( + ( + "schema_version: 1", + "datasets:", + " jp20:", + f" path: {documents}", + f" query_file: {query_file}", + ) + ), + encoding="utf-8", + ) + runfile = tmp_path / "jp20_beir.json" + _write_json(runfile, {"schema_version": 1, "name": "jp20_beir", "benchmark": "jp20_beir"}) + + outcome = run_runfiles( + [runfile], + output_dir=str(tmp_path / "session"), + dataset_paths_file=dataset_paths, + mode="batch", + dry_run=True, + isolate_runs=True, + ) + + assert outcome.exit_code == 0 + assert outcome.results["isolate_runs"] is True + assert outcome.results["runs"][0]["success"] is True + assert (outcome.artifact_dir / "001_jp20_beir" / "results.json").exists() + assert outcome.results_path == outcome.artifact_dir / "session_summary.json" + + def test_run_files_removes_stale_summary_before_child_execution(monkeypatch, tmp_path): runfile = tmp_path / "jp20_beir.json" _write_json(runfile, {"schema_version": 1, "name": "jp20_beir", "benchmark": "jp20_beir"}) diff --git a/nemo_retriever/tests/test_harness_vidore_access.py b/nemo_retriever/tests/test_harness_vidore_access.py new file mode 100644 index 0000000000..22e9610928 --- /dev/null +++ b/nemo_retriever/tests/test_harness_vidore_access.py @@ -0,0 +1,141 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +from types import SimpleNamespace + +import pytest + +from nemo_retriever.harness.vidore_access import VidoreAccessError, check_vidore_access + + +class _Response: + def __init__( + self, + status_code: int, + *, + final_host: str = "cas-bridge.xethub.hf.co", + content: bytes = b"x", + ) -> None: + self.status_code = status_code + self.history = [SimpleNamespace(status_code=302)] + self.url = f"https://{final_host}/signed-object" + self.headers = {"content-range": "bytes 0-0/1024"} if status_code == 206 else {} + self.content = content + self.iter_content_calls: list[int] = [] + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc, tb): + return False + + def iter_content(self, chunk_size: int): + self.iter_content_calls.append(chunk_size) + if self.content: + yield self.content[:chunk_size] + + +class _Api: + def __init__(self) -> None: + self.whoami_calls = 0 + + def whoami(self): + self.whoami_calls += 1 + return {"auth": {"accessToken": {"role": "read"}}} + + def dataset_info(self, repo_id: str, *, files_metadata: bool): + assert repo_id == "vidore/vidore_v3_hr" + assert files_metadata is True + return SimpleNamespace( + sha="abc123", + private=False, + gated=False, + siblings=[ + SimpleNamespace(rfilename="queries/test-00000-of-00001.parquet"), + SimpleNamespace(rfilename="qrels/test-00000-of-00001.parquet"), + SimpleNamespace(rfilename="corpus/test-00000-of-00001.parquet"), + ], + ) + + +def test_check_vidore_access_probes_one_byte_from_each_evaluation_partition() -> None: + api = _Api() + requests = [] + + def get(url, **kwargs): + response = _Response(206) + requests.append((url, kwargs, response)) + return response + + results = check_vidore_access( + dataset_names=("vidore_v3_hr",), + token="hf-secret", + timeout_seconds=7, + api=api, + get=get, + ) + + assert api.whoami_calls == 1 + assert len(results) == 1 + assert results[0].dataset == "vidore_v3_hr" + assert results[0].revision == "abc123" + assert results[0].checked_partitions == ("queries", "qrels", "corpus") + assert results[0].authenticated is True + assert len(requests) == 3 + assert all(call[1]["headers"]["Range"] == "bytes=0-0" for call in requests) + assert all(call[1]["headers"]["Authorization"] == "Bearer hf-secret" for call in requests) + assert all(call[1]["timeout"] == 7 for call in requests) + assert all(call[1]["stream"] is True for call in requests) + assert all(call[2].iter_content_calls == [1] for call in requests) + + +def test_check_vidore_access_reports_cas_redirect_failure_without_leaking_token() -> None: + def get(url, **kwargs): + return _Response(403) + + with pytest.raises(VidoreAccessError) as exc_info: + check_vidore_access( + dataset_names=("vidore_v3_hr",), + token="hf-secret", + api=_Api(), + get=get, + ) + + message = str(exc_info.value) + assert "vidore_v3_hr" in message + assert "queries/test-00000-of-00001.parquet" in message + assert "status chain 302 -> 403" in message + assert "cas-bridge.xethub.hf.co" in message + assert "hf-secret" not in message + + +def test_check_vidore_access_requires_queries_qrels_and_corpus() -> None: + api = _Api() + + def incomplete_dataset_info(repo_id: str, *, files_metadata: bool): + return SimpleNamespace( + sha="abc123", + private=False, + gated=False, + siblings=[SimpleNamespace(rfilename="queries/test-00000-of-00001.parquet")], + ) + + api.dataset_info = incomplete_dataset_info + + with pytest.raises(VidoreAccessError, match="missing evaluation partitions: qrels, corpus"): + check_vidore_access(dataset_names=("vidore_v3_hr",), token="hf-secret", api=api, get=lambda *a, **k: None) + + +def test_check_vidore_access_rejects_an_empty_success_response() -> None: + def get(url, **kwargs): + return _Response(206, content=b"") + + with pytest.raises(VidoreAccessError, match="returned an empty response"): + check_vidore_access( + dataset_names=("vidore_v3_hr",), + token="hf-secret", + api=_Api(), + get=get, + ) diff --git a/ops/retriever-nightly/README.md b/ops/retriever-nightly/README.md new file mode 100644 index 0000000000..ccbd40c290 --- /dev/null +++ b/ops/retriever-nightly/README.md @@ -0,0 +1,307 @@ + + + +# Retriever Nightly Launcher + +This directory provides a manual launcher and a latest-main controller for the +existing systemd deployment. Both run the library and ViDoRe v3 benchmark suite +through the portable harness interface. The manual launcher never changes Git +state; the controller fetches and manages immutable detached worktrees. Neither +installs a scheduler or distributes datasets. + +## Manual Teammate Kickoff + +### Prerequisites + +The supported v1 host is a Linux NVIDIA workstation with: + +- a clean NeMo Retriever Git checkout; +- `git`, Bash, `uv`, `flock`, `realpath`, and NVIDIA drivers available; +- access to the twelve benchmark datasets through `/datasets` or other local + paths; +- a Hugging Face read token in `HF_TOKEN` plus outbound HTTPS access to `huggingface.co`, + `cas-server.xethub.hf.co`, and `cas-bridge.xethub.hf.co` for ViDoRe + queries, qrels, and corpus metadata; and +- enough system RAM, local model cache, and artifact storage for the selected + runfiles. The complete batch suite is not validated on 128 GiB hosts. + +`uv` may be set with `RETRIEVER_UV_BIN`, discovered from `PATH`, or installed at +`$HOME/.local/bin/uv`. The locked `nemo_retriever` project selects Python 3.12 +and the repository dependencies. + +Batch mode starts its models locally, so it needs no model-provider API keys. +On a host with the standard `/datasets/nv-ingest` layout, `nightly.env` accepts +only two secrets and one optional path override: + +| Setting | Required | Purpose | +| --- | --- | --- | +| `HF_TOKEN` | yes | Read-only access to ViDoRe evaluation data. | +| `SLACK_WEBHOOK_URL` | no | Enables one terminal Slack post for real runs. | +| `RETRIEVER_DATASET_PATHS` | nonstandard hosts only | Replaces the checked-in `/datasets/nv-ingest` map. | + +On hosts with a writable `/raid/$USER`, the launcher automatically keeps its +private configuration, artifacts, and managed latest-main checkouts there. +Other hosts use `$HOME`. + +Create the private configuration and populate `HF_TOKEN`: + +```bash +if [[ -d /raid/$USER && -w /raid/$USER ]]; then + RETRIEVER_NIGHTLY_ROOT=/raid/$USER +else + RETRIEVER_NIGHTLY_ROOT=$HOME +fi +RETRIEVER_NIGHTLY_CONFIG_DIR="$RETRIEVER_NIGHTLY_ROOT/.config/nemo-retriever/nightly" +mkdir -p "$RETRIEVER_NIGHTLY_CONFIG_DIR" +chmod 700 "$RETRIEVER_NIGHTLY_CONFIG_DIR" +test -e "$RETRIEVER_NIGHTLY_CONFIG_DIR/nightly.env" || \ + cp ops/retriever-nightly/nightly.env.example "$RETRIEVER_NIGHTLY_CONFIG_DIR/nightly.env" +chmod 600 "$RETRIEVER_NIGHTLY_CONFIG_DIR/nightly.env" +${EDITOR:-vi} "$RETRIEVER_NIGHTLY_CONFIG_DIR/nightly.env" +``` + +The checked-in `dataset_paths.datasets.yaml` already describes the standard +`/datasets/nv-ingest` layout. Only hosts with a different layout need to copy +and edit `nemo_retriever/harness/dataset_paths.example.yaml`, then set +`RETRIEVER_DATASET_PATHS` in `nightly.env`. + +The launcher sources only the detected mode-`600` configuration; it does not +discover a repository `.env` file. `RETRIEVER_CONFIG_FILE` remains an advanced +override for deployments that deliberately store the file elsewhere. + +Verify the token and read one byte from one remote parquet object in every +ViDoRe evaluation partition before starting GPU work: + +```bash +./ops/retriever-nightly/run-nightly.sh --check-vidore-access +``` + +The access check does not download full parquet objects. A redirect failure +such as `302 -> 403 at cas-bridge.xethub.hf.co` is a Hugging Face/CAS delivery +failure; do not start the full suite until the check exits zero. + +Then preflight the complete twelve-benchmark suite without starting ingest or +query: + +```bash +./ops/retriever-nightly/run-nightly.sh --dry-run +``` + +Inspect the resulting `session_summary.json` and child plans. Dry-runs never +post to Slack. A real run posts when `SLACK_WEBHOOK_URL` is configured; use +`--no-slack` for a real functional test that must not post. The launcher prints +the timestamped session directory on success or terminal harness failure. + +Use one positional runfile for a smaller real canary before the full run: + +```bash +./ops/retriever-nightly/run-nightly.sh \ + --no-slack \ + nemo_retriever/harness/runfiles/jp20_beir.json +``` + +Run the complete suite from the current checkout with no positional runfiles: + +```bash +./ops/retriever-nightly/run-nightly.sh +``` + +If `SLACK_WEBHOOK_URL` is configured, that real run posts its terminal summary. +Add `--no-slack` only when the full run is itself a functional test that must +not post. + +### Select the Code Under Test + +The launcher runs the current clean checkout and records its commit in the +session artifacts. It never changes that checkout. To run the latest fetched +`upstream/main` without disturbing another worktree: + +```bash +git fetch upstream main +git worktree add --detach ../NeMo-Retriever-benchmark-main upstream/main +cd ../NeMo-Retriever-benchmark-main +``` + +To run an exact fetched commit, replace `upstream/main` with its SHA and choose +a distinct worktree directory: + +```bash +git worktree add --detach ../NeMo-Retriever-benchmark-abc1234 abc1234 +cd ../NeMo-Retriever-benchmark-abc1234 +``` + +Run the same launcher command from that worktree. Tracked, staged, and untracked +changes are rejected so a result is attributable to the recorded commit. +Ignored cache files do not make the checkout dirty; datasets, configuration, +and artifacts remain outside the checkout. + +### Scheduled Latest-Main Selection + +Manual `run-nightly.sh` invocations deliberately run the current clean +checkout. The scheduled service instead calls `run-latest-main.sh`, which: + +1. fetches `main` from the `upstream` remote; +2. resolves the fetched commit before doing any GPU work; +3. creates or reuses an immutable detached worktree named `main-`; +4. runs the ViDoRe access check from that selected commit; and +5. invokes that commit's `run-nightly.sh` only when fetch and access preflight + both succeed. + +A fetch failure is fail-closed: the controller does not fall back to yesterday's +commit. It never runs `git pull`, merges into the controller checkout, or moves +the reviewed controller branch. Session artifacts still record the exact +selected SHA in `run_commit`. The fetched source is also recorded locally at +`refs/retriever-nightly/latest-main` for inspection. + +Immutable worktrees and one shared `uv` project environment live under the +detected nightly root at `retriever-nightly-checkouts`; on `/raid` hosts this is +`/raid/$USER/retriever-nightly-checkouts`. The seven most recently used SHA +worktrees are retained. Modified managed worktrees are never deleted +automatically. + +The one-time deployment setup must provide an `upstream` remote. Scheduled +users do not choose a branch or commit after that: + +```bash +git remote get-url upstream >/dev/null 2>&1 || \ + git remote add upstream https://github.com/NVIDIA/NeMo-Retriever.git +./ops/retriever-nightly/run-latest-main.sh --dry-run +``` + +The dry-run fetches and selects the latest commit but skips remote access and +GPU execution. Use `run-latest-main.sh --check-vidore-access` to validate the +selected latest-main commit and its machine credentials without starting a +session. + +### Slack Report + +To enable Slack for real runs, add the incoming-webhook URL to the same private +mode-`600` `nightly.env` used for `HF_TOKEN`: + +```bash +SLACK_WEBHOOK_URL=https://hooks.slack.com/services/... +``` + +The URL itself is the Slack switch. If it is unset or empty, real runs complete +without posting. If it is set, the launcher validates it before expensive work +and posts once after `session_summary.json` exists. An invalid configured URL +fails preflight. Pass `--no-slack` for canaries or other real functional tests; +that flag suppresses webhook validation and posting. Dry-runs and access checks +never post. The launcher removes the URL from the benchmark child environment +and exposes it only to the final Slack command. + +Command-line flags override values loaded from `RETRIEVER_CONFIG_FILE`; those +values override the launcher defaults. Run either launcher with `--help` for +its supported interface. + +## Runtime Contract + +The launcher takes a nonblocking host-local lock, forces batch mode, and runs +12 checked-in runfiles as one session: JP20, BO767, Earnings, FinanceBench, and +all eight public ViDoRe v3 domains. Real sessions execute each child in a fresh +spawned process so Ray and materialized dataframe memory are released before +the next benchmark; the parent still writes one terminal session summary. +Dry-runs stay in the parent process because they do not materialize datasets. A +configured Slack report runs once after a terminal session summary exists. A +`.slack_post_attempted` marker prevents a second attempt for the same session. +Incoming webhooks do not provide an idempotency key, so ambiguous transport +failures require human inspection. + +The Slack report keeps the library benchmarks detailed and collapses the full +ViDoRe v3 suite into total ingest time, aggregate pages/sec, macro-average +Recall@5 and nDCG@10 for the English and complete suites, and one accuracy row +per domain. Per-domain throughput and timing remain in the session artifacts. + +If one runtime child fails, `run-files` continues the remaining datasets and +writes a failed session summary. When Slack is configured, the launcher still +attempts one report and returns the harness status. If the harness succeeds but +Slack fails, it returns the Slack command's nonzero status. + +The launcher defaults `VLLM_DEEP_GEMM_WARMUP=skip` unless the caller explicitly +sets another vLLM-supported mode. This skips the optional compatibility-sensitive +warmup without disabling DeepGEMM kernels. It intentionally does not set +`VLLM_USE_DEEP_GEMM=0` or `VLLM_MOE_USE_DEEP_GEMM=0`. Set the warmup variable +explicitly, for example to `full`, only when validating another mode. This +matches the reliability direction under discussion in +[NVIDIA/NeMo-Retriever PR #2292](https://github.com/NVIDIA/NeMo-Retriever/pull/2292). + +## Troubleshooting Preflight And Host Memory + +`--check-vidore-access` validates the configured token, reads repository +metadata, and follows the same Hugging Face redirects used by `datasets` while +reading one byte from one parquet object in each of the queries, qrels, and +corpus partitions. If the token is valid but the final CAS host returns `403`, compare +the same check +from another network before rotating credentials. Success elsewhere points to +host proxy, firewall, or egress policy; failure from multiple networks should +be escalated with the named dataset object to Hugging Face or ViDoRe. + +Batch ingest currently materializes each terminal Ray dataset in Python. +High-resolution page payloads can therefore consume substantially more system +RAM than the final LanceDB table. The nightly's per-run process boundary +prevents that memory from accumulating across the twelve children, but an +individual large benchmark must still fit on the host. If Ray reports the +dataset and VDB write complete while a child remains idle at high RSS, capture +`run.log`, `status.json`, process RSS, and the Ray task summary. Retry only that +runfile as a focused reproduction; do not classify the symptom as GPU OOM +unless the GPU process or kernel logs show an actual allocation failure. + +## System Service And Timer + +Manual kickoff does not install recurrence. The existing root-managed service +is a separate adapter for one deployment host. For its `local-jioffe` service +user, the detected nightly root is `/raid/local-jioffe` when that directory is +writable and `/localhome/local-jioffe` otherwise. Its layout is: + +- clean deployment worktree: + `/localhome/local-jioffe/NeMo-Retriever-deploy/nightly` (the reviewed, + pinned controller checkout); +- immutable latest-main worktrees and shared `uv` environment: + `/retriever-nightly-checkouts`; +- machine configuration: + `/.config/nemo-retriever/nightly/nightly.env`; +- full local artifacts: + `/retriever-nightly-artifacts`. + +The configuration file must be owned by `local-jioffe` with mode `600`. It must +contain `HF_TOKEN`; an optional nonempty `SLACK_WEBHOOK_URL` enables Slack for +real scheduled runs. Before enabling the timer, run +`run-latest-main.sh --dry-run` and +`run-latest-main.sh --check-vidore-access` as the service user with the same +`HOME` and configuration file used by systemd. Use `--no-slack` on any real +deployment canary that must not post. The reviewed controller checkout remains +pinned; each scheduled invocation independently selects the latest fetched +`upstream/main` commit. + +The root-managed service runs as `local-jioffe`. The timer starts it every day +at midnight in `America/New_York`, preserving local midnight across EDT and EST +changes. `Persistent=false` deliberately avoids an unscheduled catch-up run if +the host was unavailable at midnight. + +Install or refresh the units from the deployment checkout: + +```bash +sudo install -m 0644 \ + ops/retriever-nightly/systemd/nrl-harness-batch-hf.service \ + /etc/systemd/system/nrl-harness-batch-hf.service +sudo install -m 0644 \ + ops/retriever-nightly/systemd/nrl-harness-batch-hf.timer \ + /etc/systemd/system/nrl-harness-batch-hf.timer +sudo systemctl daemon-reload +sudo systemctl enable --now nrl-harness-batch-hf.timer +``` + +Inspect the schedule and the most recent run: + +```bash +systemctl list-timers nrl-harness-batch-hf.timer --all +systemctl show nrl-harness-batch-hf.service \ + -p Result -p ExecMainCode -p ExecMainStatus +journalctl -u nrl-harness-batch-hf.service --since today --no-pager +``` + +Disable future recurrence without changing completed artifacts: + +```bash +sudo systemctl disable --now nrl-harness-batch-hf.timer +``` diff --git a/ops/retriever-nightly/SECOND_HOST_VALIDATION.md b/ops/retriever-nightly/SECOND_HOST_VALIDATION.md new file mode 100644 index 0000000000..8d8fbb8439 --- /dev/null +++ b/ops/retriever-nightly/SECOND_HOST_VALIDATION.md @@ -0,0 +1,144 @@ + + + +# Portable Nightly Second-Host Validation + +Use this checklist on a separate Linux NVIDIA workstation before handing the +launcher to additional teammates. The review uses the pushed feature branch, +keeps datasets and artifacts outside the repository, and does not install a +timer or post to Slack. + +## 1. Create a Local Review Branch + +In an existing clone whose `origin` points to the contributor fork: + +```bash +git fetch origin jioffe502/retriever-nightly-vidore-v3 +git switch --create review/portable-nightly \ + --track origin/jioffe502/retriever-nightly-vidore-v3 +git status --short +git rev-parse HEAD +``` + +`git status --short` must be empty. If this is a new clone, add the NVIDIA +repository as `upstream` for later comparisons: + +```bash +git remote add upstream https://github.com/NVIDIA/NeMo-Retriever.git +``` + +The launcher runs exactly the checked-out commit. It does not fetch, pull, or +switch refs. + +## 2. Prepare the Host + +Confirm that `uv` and the NVIDIA driver are available: + +```bash +uv --version +nvidia-smi +``` + +This host has the standard `/datasets/nv-ingest` layout and `/raid/$USER`, so +the checked-in dataset map and launcher path defaults apply. Create one private +configuration file and populate `HF_TOKEN`. A read token is sufficient: + +```bash +RETRIEVER_NIGHTLY_CONFIG_DIR="/raid/$USER/.config/nemo-retriever/nightly" +mkdir -p "$RETRIEVER_NIGHTLY_CONFIG_DIR" +chmod 700 "$RETRIEVER_NIGHTLY_CONFIG_DIR" +test -e "$RETRIEVER_NIGHTLY_CONFIG_DIR/nightly.env" || \ + cp ops/retriever-nightly/nightly.env.example "$RETRIEVER_NIGHTLY_CONFIG_DIR/nightly.env" +chmod 600 "$RETRIEVER_NIGHTLY_CONFIG_DIR/nightly.env" +${EDITOR:-vi} "$RETRIEVER_NIGHTLY_CONFIG_DIR/nightly.env" +``` + +The launcher discovers that file and writes artifacts under +`/raid/$USER/retriever-nightly-artifacts`; no shell exports are required. It +does not read a repository `.env`. On a host without the standard dataset +layout, copy and edit `nemo_retriever/harness/dataset_paths.example.yaml` and +set only `RETRIEVER_DATASET_PATHS` in `nightly.env`. + +## 3. Verify ViDoRe Evaluation Access + +Before starting GPU work, validate the configured token and read one byte from +one remote parquet object in each of the queries, qrels, and corpus partitions: + +```bash +./ops/retriever-nightly/run-nightly.sh --check-vidore-access +``` + +The command should exit zero and report access for all eight ViDoRe v3 +datasets. It does not download the full objects. Do not start the complete +suite if this check reports a Hugging Face or CAS redirect failure. + +## 4. Preflight All Twelve Benchmarks + +```bash +./ops/retriever-nightly/run-nightly.sh --dry-run +``` + +The command should exit zero, report a new timestamped session directory, and +write a `session_summary.json` with `dry_run: true`, twelve runs, and a +`run_commit` matching `git rev-parse HEAD`. `isolate_runs` is `false` because +the dry-run does not materialize batch data. + +## 5. Run the JP20 Canary + +```bash +./ops/retriever-nightly/run-nightly.sh \ + --no-slack \ + nemo_retriever/harness/runfiles/jp20_beir.json +``` + +Confirm that the command exits zero and the session summary contains one +successful run with `isolate_runs: true`. The launcher defaults the optional +DeepGEMM warmup to `skip`; no host setting is needed. + +## 6. Capture the Handoff Evidence + +For the dry-run and JP20 sessions, record: + +- the launcher exit code and printed session directory; +- `success`, `exit_code`, `dry_run`, `isolate_runs`, `run_commit`, and the + number of `runs` in `session_summary.json`; +- any failed child name and its artifact directory; and +- the GPU model and driver from `nvidia-smi`. + +The pre-handoff validation is complete when the ViDoRe access check, twelve-run +dry-run, and real JP20 canary succeed with terminal summaries attributed to the +review branch commit. A second full suite is not required before handoff. Do +not enable the timer or post to Slack as part of this functional review. + +## 7. Validate Latest-Main Selection After Merge + +Do not use the latest-main controller while validating the feature branch; its +purpose is to select `upstream/main`. After the PR merges, validate the +scheduled selection path without GPU work: + +```bash +git remote get-url upstream +./ops/retriever-nightly/run-latest-main.sh --dry-run +``` + +Confirm that the printed selected SHA matches +`git rev-parse refs/retriever-nightly/latest-main`, the controller checkout did +not move, and the dry-run summary has that SHA as `run_commit`. The installed +systemd service uses this controller automatically, so scheduled users do not +fetch or select a commit themselves. + +## 8. Run the Lead's First Complete Nightly + +After the latest-main dry-run succeeds, run the controller without positional +runfiles or `--no-slack`: + +```bash +./ops/retriever-nightly/run-latest-main.sh +``` + +The controller checks ViDoRe access, then runs the four library benchmarks and +all eight ViDoRe v3 domains. Each child runs in a fresh process; failures do not +prevent later children from running, and the parent writes one terminal +`session_summary.json`. If `SLACK_WEBHOOK_URL` is configured, that terminal +summary posts once. Confirm the full `run_commit`, twelve child results, final +exit code, and Slack message before enabling the timer described in `README.md`. diff --git a/ops/retriever-nightly/dataset_paths.datasets.yaml b/ops/retriever-nightly/dataset_paths.datasets.yaml new file mode 100644 index 0000000000..f62fce312a --- /dev/null +++ b/ops/retriever-nightly/dataset_paths.datasets.yaml @@ -0,0 +1,33 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: 1 +datasets: + jp20: + path: /datasets/nv-ingest/jp20 + query_file: /datasets/nv-ingest/ground_truths/jp20_query_gt.csv + bo767: + path: /datasets/nv-ingest/bo767 + query_file: /datasets/nv-ingest/ground_truths/bo767_query_gt.csv + earnings_consulting: + path: /datasets/nv-ingest/earnings_consulting_flattened + query_file: /datasets/nv-ingest/ground_truths/earnings_consulting_multimodal.csv + financebench: + path: /datasets/nv-ingest/foundation_rag/financebench + query_file: /datasets/nv-ingest/ground_truths/financebench_train.json + vidore_v3_computer_science: + path: /datasets/nv-ingest/vidore_v3/vidore_v3_computer_science + vidore_v3_energy: + path: /datasets/nv-ingest/vidore_v3/vidore_v3_energy + vidore_v3_finance_en: + path: /datasets/nv-ingest/vidore_v3/vidore_v3_finance_en + vidore_v3_finance_fr: + path: /datasets/nv-ingest/vidore_v3/vidore_v3_finance_fr + vidore_v3_hr: + path: /datasets/nv-ingest/vidore_v3/vidore_v3_hr + vidore_v3_industrial: + path: /datasets/nv-ingest/vidore_v3/vidore_v3_industrial + vidore_v3_pharmaceuticals: + path: /datasets/nv-ingest/vidore_v3/vidore_v3_pharmaceuticals + vidore_v3_physics: + path: /datasets/nv-ingest/vidore_v3/vidore_v3_physics diff --git a/ops/retriever-nightly/nightly.env.example b/ops/retriever-nightly/nightly.env.example new file mode 100644 index 0000000000..4048526810 --- /dev/null +++ b/ops/retriever-nightly/nightly.env.example @@ -0,0 +1,16 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# SPDX-License-Identifier: Apache-2.0 + +# Copy to the detected private configuration path and set mode 600. On hosts +# with /raid/$USER, the launcher keeps configuration, artifacts, and managed +# latest-main checkouts there. Otherwise it uses $HOME. +# +# Required operator input. A read token is sufficient: +HF_TOKEN= + +# Optional: uncomment to post terminal real-run summaries to Slack. +# SLACK_WEBHOOK_URL=https://hooks.slack.com/services/... + +# Override only when this host does not use the checked-in +# /datasets/nv-ingest path map: +# RETRIEVER_DATASET_PATHS=/path/to/dataset_paths.yaml diff --git a/ops/retriever-nightly/run-latest-main.sh b/ops/retriever-nightly/run-latest-main.sh new file mode 100755 index 0000000000..36cce3d462 --- /dev/null +++ b/ops/retriever-nightly/run-latest-main.sh @@ -0,0 +1,200 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# SPDX-License-Identifier: Apache-2.0 + +set -uo pipefail + +umask 077 + +readonly EXIT_CONFIG=64 +readonly EXIT_FETCH=69 +readonly EXIT_ALREADY_RUNNING=75 + +log() { + printf 'retriever-nightly-latest: %s\n' "$*" >&2 +} + +usage() { + cat <<'EOF' +Usage: run-latest-main.sh [RUN-NIGHTLY-OPTION ...] + +Fetch upstream/main into an immutable worktree, preflight ViDoRe access, and +run that commit's nightly launcher. All non-help options are forwarded to +run-nightly.sh. + +Options: + --help Show this help text without fetching or requiring configuration. +EOF +} + +prune_managed_worktrees() { + local repository="$1" + local checkout_root="$2" + local selected_checkout="$3" + local keep_count="$4" + local retained=0 + local candidate base + local -a candidates=() + + mapfile -t candidates < <( + find "$checkout_root" -mindepth 1 -maxdepth 1 -type d -name 'main-*' -printf '%T@ %p\n' \ + | sort -nr \ + | sed -E 's/^[^ ]+ //' + ) + for candidate in "${candidates[@]}"; do + base="$(basename -- "$candidate")" + if [[ ! "$base" =~ ^main-[0-9a-f]{40,64}$ ]]; then + continue + fi + if [[ "$candidate" == "$selected_checkout" || $retained -lt $keep_count ]]; then + ((retained += 1)) + continue + fi + if [[ -n "$(git -C "$candidate" status --porcelain --untracked-files=normal 2>/dev/null)" ]]; then + log "retaining modified managed worktree: $candidate" + continue + fi + if ! git -C "$repository" worktree remove --force "$candidate"; then + log "could not prune managed worktree: $candidate" + fi + done + git -C "$repository" worktree prune || log "could not prune stale Git worktree metadata" +} + +main() { + local arg + for arg in "$@"; do + if [[ "$arg" == "--help" ]]; then + usage + return 0 + fi + done + + local default_nightly_root="$HOME" + local raid_nightly_root="/raid/$(id -un)" + if [[ -d "$raid_nightly_root" && -w "$raid_nightly_root" ]]; then + default_nightly_root="$raid_nightly_root" + fi + local nightly_root="${RETRIEVER_NIGHTLY_ROOT:-$default_nightly_root}" + local config_file="${RETRIEVER_CONFIG_FILE:-$nightly_root/.config/nemo-retriever/nightly/nightly.env}" + if [[ ! -f "$config_file" ]]; then + log "nightly configuration is missing: $config_file" + return "$EXIT_CONFIG" + fi + if [[ "$(stat -c '%a' "$config_file")" != "600" || \ + "$(stat -c '%u' "$config_file")" != "$(id -u)" ]]; then + log "nightly configuration must be owned by the service user with mode 600" + return "$EXIT_CONFIG" + fi + set -a + # shellcheck disable=SC1090 + source "$config_file" + set +a + unset SLACK_WEBHOOK_URL + + nightly_root="${RETRIEVER_NIGHTLY_ROOT:-$nightly_root}" + export RETRIEVER_CONFIG_FILE="$config_file" + export RETRIEVER_NIGHTLY_ROOT="$nightly_root" + if [[ -z "${VLLM_DEEP_GEMM_WARMUP+x}" ]]; then + export VLLM_DEEP_GEMM_WARMUP=skip + fi + + local script_dir repository source_ref source_name remote_ref fetched_ref checkout_root keep_count uv_environment + script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" + repository="${RETRIEVER_UPDATE_REPOSITORY:-$(git -C "$script_dir" rev-parse --show-toplevel)}" + source_name="${RETRIEVER_LATEST_SOURCE:-upstream}" + source_ref="${RETRIEVER_LATEST_REF:-main}" + remote_ref="$source_ref" + if [[ "$remote_ref" != refs/* ]]; then + remote_ref="refs/heads/$remote_ref" + fi + fetched_ref="refs/retriever-nightly/latest-main" + checkout_root="${RETRIEVER_LATEST_CHECKOUT_ROOT:-$nightly_root/retriever-nightly-checkouts}" + keep_count="${RETRIEVER_LATEST_KEEP_CHECKOUTS:-7}" + uv_environment="${UV_PROJECT_ENVIRONMENT:-$checkout_root/.venv}" + + if [[ ! "$keep_count" =~ ^[1-9][0-9]*$ ]]; then + log "RETRIEVER_LATEST_KEEP_CHECKOUTS must be a positive integer" + return "$EXIT_CONFIG" + fi + if ! git check-ref-format "$remote_ref"; then + log "RETRIEVER_LATEST_REF is not a valid Git ref: $source_ref" + return "$EXIT_CONFIG" + fi + if [[ ! -d "$repository/.git" && ! -f "$repository/.git" ]]; then + log "update repository is not a Git worktree: $repository" + return "$EXIT_CONFIG" + fi + if [[ -n "$(git -C "$repository" status --porcelain --untracked-files=normal)" ]]; then + log "reviewed controller checkout has local changes: $repository" + return "$EXIT_CONFIG" + fi + if ! mkdir -p "$checkout_root"; then + log "could not create latest-main checkout root: $checkout_root" + return "$EXIT_CONFIG" + fi + chmod 700 "$checkout_root" + + exec 8>"$checkout_root/.latest-main.lock" || return "$EXIT_CONFIG" + if ! flock -n 8; then + log "another latest-main selection is already running" + return "$EXIT_ALREADY_RUNNING" + fi + + log "fetching $source_name $source_ref" + if ! git -C "$repository" fetch --no-tags "$source_name" "+$remote_ref:$fetched_ref"; then + log "fetch failed; refusing to run a stale commit" + return "$EXIT_FETCH" + fi + + local target_commit selected_checkout selected_head target_launcher preflight_access run_rc + target_commit="$(git -C "$repository" rev-parse --verify "$fetched_ref^{commit}")" || return "$EXIT_FETCH" + selected_checkout="$checkout_root/main-$target_commit" + if [[ -e "$selected_checkout" ]]; then + selected_head="$(git -C "$selected_checkout" rev-parse --verify HEAD 2>/dev/null || true)" + if [[ "$selected_head" != "$target_commit" ]]; then + log "managed checkout has an unexpected commit: $selected_checkout" + return "$EXIT_CONFIG" + fi + if [[ -n "$(git -C "$selected_checkout" status --porcelain --untracked-files=normal)" ]]; then + log "managed checkout has local changes: $selected_checkout" + return "$EXIT_CONFIG" + fi + elif ! git -C "$repository" worktree add --detach "$selected_checkout" "$target_commit"; then + log "could not create detached worktree for $target_commit" + return "$EXIT_CONFIG" + fi + touch "$selected_checkout" + + target_launcher="$selected_checkout/ops/retriever-nightly/run-nightly.sh" + if [[ ! -x "$target_launcher" ]]; then + log "selected commit does not contain an executable nightly launcher: $target_launcher" + return "$EXIT_CONFIG" + fi + + export RETRIEVER_SELECTED_CHECKOUT="$selected_checkout" + export UV_PROJECT_ENVIRONMENT="$uv_environment" + log "selected $source_name/$source_ref commit $target_commit in $selected_checkout" + + preflight_access=1 + for arg in "$@"; do + if [[ "$arg" == "--dry-run" || "$arg" == "--check-vidore-access" || "$arg" == "--help" ]]; then + preflight_access=0 + fi + done + + run_rc=0 + if [[ "$preflight_access" == "1" ]]; then + "$target_launcher" --check-vidore-access || run_rc=$? + fi + if ((run_rc == 0)); then + "$target_launcher" "$@" || run_rc=$? + else + log "ViDoRe access preflight failed; skipping GPU work" + fi + + prune_managed_worktrees "$repository" "$checkout_root" "$selected_checkout" "$keep_count" + return "$run_rc" +} + +main "$@" diff --git a/ops/retriever-nightly/run-nightly.sh b/ops/retriever-nightly/run-nightly.sh new file mode 100755 index 0000000000..4891a22ad3 --- /dev/null +++ b/ops/retriever-nightly/run-nightly.sh @@ -0,0 +1,291 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# SPDX-License-Identifier: Apache-2.0 + +set -uo pipefail + +umask 077 + +readonly EXIT_CONFIG=64 +readonly EXIT_INTERNAL=70 +readonly EXIT_CANNOT_CREATE=73 +readonly EXIT_ALREADY_RUNNING=75 + +log() { + printf 'retriever-nightly: %s\n' "$*" >&2 +} + +usage() { + cat <<'EOF' +Usage: run-nightly.sh [OPTIONS] [--] [RUNFILE ...] + +Run the library and ViDoRe v3 harness suite from the current clean checkout. +With no RUNFILE arguments, all 12 checked-in nightly runfiles are executed. + +Options: + --dataset-paths PATH Machine-local dataset path map. + --artifact-root PATH Parent directory for timestamped session artifacts. + --check-vidore-access Validate authenticated ViDoRe evaluation-data access and exit. + --dry-run Resolve and validate the suite without executing it. + --no-slack Do not post, even when SLACK_WEBHOOK_URL is configured. + --help Show this help text. +EOF +} + +absolute_path() { + realpath -m -- "$1" +} + +default_nightly_root="$HOME" +raid_nightly_root="/raid/$(id -un)" +if [[ -d "$raid_nightly_root" && -w "$raid_nightly_root" ]]; then + default_nightly_root="$raid_nightly_root" +fi +nightly_root="${RETRIEVER_NIGHTLY_ROOT:-$default_nightly_root}" +readonly config_file="${RETRIEVER_CONFIG_FILE:-$nightly_root/.config/nemo-retriever/nightly/nightly.env}" +if [[ -f "$config_file" ]]; then + if [[ "$(stat -c '%a' "$config_file")" != "600" || "$(stat -c '%u' "$config_file")" != "$(id -u)" ]]; then + log "nightly configuration must be owned by the service user with mode 600" + exit "$EXIT_CONFIG" + fi + set -a + # shellcheck disable=SC1090 + source "$config_file" + set +a +fi +slack_webhook_url="${SLACK_WEBHOOK_URL:-}" +unset SLACK_WEBHOOK_URL + +readonly script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +checkout="${RETRIEVER_SELECTED_CHECKOUT:-${RETRIEVER_CHECKOUT:-$(git -C "$script_dir" rev-parse --show-toplevel)}}" +nightly_root="${RETRIEVER_NIGHTLY_ROOT:-$nightly_root}" +artifact_root="${RETRIEVER_ARTIFACT_ROOT:-$nightly_root/retriever-nightly-artifacts}" +dataset_paths="${RETRIEVER_DATASET_PATHS:-$checkout/ops/retriever-nightly/dataset_paths.datasets.yaml}" +session_name="${RETRIEVER_SESSION_NAME:-library_vidore_v3_batch_hf}" +slack_title="${RETRIEVER_SLACK_TITLE:-nemo-retriever library + ViDoRe v3 nightly}" +run_mode="${RETRIEVER_MODE:-batch}" +dry_run=0 +check_vidore_access=0 +skip_slack=0 +runfiles=() + +while (($#)); do + case "$1" in + --dataset-paths) + if (($# < 2)); then + log "--dataset-paths requires a path" + exit "$EXIT_CONFIG" + fi + dataset_paths="$2" + shift 2 + ;; + --artifact-root) + if (($# < 2)); then + log "--artifact-root requires a path" + exit "$EXIT_CONFIG" + fi + artifact_root="$2" + shift 2 + ;; + --dry-run) + dry_run=1 + shift + ;; + --check-vidore-access) + check_vidore_access=1 + shift + ;; + --no-slack) + skip_slack=1 + shift + ;; + --help) + usage + exit 0 + ;; + --) + shift + runfiles+=("$@") + break + ;; + -*) + log "unknown option: $1" + exit "$EXIT_CONFIG" + ;; + *) + runfiles+=("$1") + shift + ;; + esac +done + +if ((check_vidore_access && (dry_run || ${#runfiles[@]} > 0))); then + log "--check-vidore-access cannot be combined with --dry-run or runfiles" + exit "$EXIT_CONFIG" +fi + +if [[ -z "${VLLM_DEEP_GEMM_WARMUP+x}" ]]; then + export VLLM_DEEP_GEMM_WARMUP=skip +fi + +checkout="$(absolute_path "$checkout")" +artifact_root="$(absolute_path "$artifact_root")" +dataset_paths="$(absolute_path "$dataset_paths")" + +if [[ -n "${RETRIEVER_UV_BIN:-}" ]]; then + uv_bin="$RETRIEVER_UV_BIN" +elif uv_bin="$(command -v uv 2>/dev/null)"; then + : +else + uv_bin="$HOME/.local/bin/uv" +fi +if [[ "$uv_bin" != /* ]]; then + uv_bin="$(command -v -- "$uv_bin" 2>/dev/null || absolute_path "$uv_bin")" +fi + +if [[ ! -x "$uv_bin" ]]; then + log "uv executable is missing or not executable: $uv_bin" + exit "$EXIT_CONFIG" +fi +if [[ ! -d "$checkout/.git" && ! -f "$checkout/.git" ]]; then + log "deployment checkout is not a Git worktree: $checkout" + exit "$EXIT_CONFIG" +fi +if [[ -n "$(git -C "$checkout" status --porcelain --untracked-files=normal --ignore-submodules)" ]]; then + log "deployment checkout has tracked, staged, or untracked changes; refusing to run" + exit "$EXIT_CONFIG" +fi +if ((check_vidore_access)); then + cd "$checkout" || exit "$EXIT_CONFIG" + "$uv_bin" run --frozen --project nemo_retriever retriever harness check-vidore-access + exit $? +fi +if [[ ! -f "$dataset_paths" ]]; then + log "dataset path map is missing: $dataset_paths" + exit "$EXIT_CONFIG" +fi +post_slack=0 +if ((!dry_run && !skip_slack)) && [[ -n "$slack_webhook_url" ]]; then + post_slack=1 + if [[ "$slack_webhook_url" == *$'\n'* || \ + "$slack_webhook_url" != https://hooks.slack.com/services/* ]]; then + log "SLACK_WEBHOOK_URL must contain one Slack incoming-webhook URL" + exit "$EXIT_CONFIG" + fi +fi +if ((${#runfiles[@]} == 0)); then + runfiles=( + nemo_retriever/harness/runfiles/jp20_beir.json + nemo_retriever/harness/runfiles/bo767_beir.json + nemo_retriever/harness/runfiles/earnings_beir.json + nemo_retriever/harness/runfiles/financebench_beir.json + nemo_retriever/harness/runfiles/vidore_v3_computer_science_beir.json + nemo_retriever/harness/runfiles/vidore_v3_energy_beir.json + nemo_retriever/harness/runfiles/vidore_v3_finance_en_beir.json + nemo_retriever/harness/runfiles/vidore_v3_finance_fr_beir.json + nemo_retriever/harness/runfiles/vidore_v3_hr_beir.json + nemo_retriever/harness/runfiles/vidore_v3_industrial_beir.json + nemo_retriever/harness/runfiles/vidore_v3_pharmaceuticals_beir.json + nemo_retriever/harness/runfiles/vidore_v3_physics_beir.json + ) +fi +readonly -a runfiles +for runfile in "${runfiles[@]}"; do + resolved_runfile="$runfile" + if [[ "$resolved_runfile" != /* ]]; then + resolved_runfile="$checkout/$resolved_runfile" + fi + if [[ ! -f "$resolved_runfile" ]]; then + log "required runfile is missing: $runfile" + exit "$EXIT_CONFIG" + fi +done + +if ! mkdir -p "$artifact_root"; then + log "could not create artifact root: $artifact_root" + exit "$EXIT_CANNOT_CREATE" +fi + +exec 9>"$artifact_root/.retriever-nightly.lock" || exit "$EXIT_CANNOT_CREATE" +if ! flock -n 9; then + log "another nightly invocation holds the lock" + exit "$EXIT_ALREADY_RUNNING" +fi + +readonly timestamp="$(date -u +%Y%m%d_%H%M%S_UTC)" +readonly session_dir="$artifact_root/${session_name}-${timestamp}" +readonly session_summary="$session_dir/session_summary.json" +readonly slack_attempt_marker="$session_dir/.slack_post_attempted" + +if [[ -e "$session_dir" ]]; then + log "refusing to reuse an existing session directory: $session_dir" + exit "$EXIT_CANNOT_CREATE" +fi + +cd "$checkout" || exit "$EXIT_CONFIG" + +run_rc=0 +dry_run_args=() +isolation_args=(--isolate-runs) +if ((dry_run)); then + dry_run_args=(--dry-run) + isolation_args=() +fi +"$uv_bin" run --frozen --project nemo_retriever retriever harness run-files \ + --session-name "$session_name" \ + --output-dir "$session_dir" \ + --dataset-paths "$dataset_paths" \ + --mode "$run_mode" \ + "${isolation_args[@]}" \ + "${dry_run_args[@]}" \ + "${runfiles[@]}" || run_rc=$? + +if [[ ! -f "$session_summary" ]]; then + log "run-files did not produce a terminal session summary: $session_summary" + if ((run_rc != 0)); then + exit "$run_rc" + fi + exit "$EXIT_INTERNAL" +fi + +if ((post_slack == 0)); then + if ((run_rc != 0)); then + log "session completed without Slack; returning harness exit code $run_rc: $session_dir" + exit "$run_rc" + fi + log "session completed without Slack: $session_dir" + exit 0 +fi + +export SLACK_WEBHOOK_URL="$slack_webhook_url" + +if ! (set -o noclobber; : >"$slack_attempt_marker") 2>/dev/null; then + unset SLACK_WEBHOOK_URL + log "Slack post was already attempted for this session; refusing a duplicate" + if ((run_rc != 0)); then + exit "$run_rc" + fi + exit "$EXIT_CONFIG" +fi + +post_rc=0 +"$uv_bin" run --frozen --project nemo_retriever retriever harness post-slack \ + --title "$slack_title" \ + "$session_dir" || post_rc=$? +unset SLACK_WEBHOOK_URL +unset slack_webhook_url + +if ((run_rc != 0)); then + if ((post_rc != 0)); then + log "harness and Slack post both failed; returning harness exit code $run_rc" + else + log "session report posted; returning harness exit code $run_rc" + fi + exit "$run_rc" +fi +if ((post_rc != 0)); then + log "harness passed but Slack post failed with exit code $post_rc" + exit "$post_rc" +fi + +log "session completed and one Slack post succeeded: $session_dir" diff --git a/ops/retriever-nightly/systemd/nrl-harness-batch-hf.service b/ops/retriever-nightly/systemd/nrl-harness-batch-hf.service new file mode 100644 index 0000000000..6c5f0be923 --- /dev/null +++ b/ops/retriever-nightly/systemd/nrl-harness-batch-hf.service @@ -0,0 +1,19 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# SPDX-License-Identifier: Apache-2.0 + +[Unit] +Description=Run latest upstream/main NeMo Retriever and ViDoRe v3 nightly session +Documentation=file:/localhome/local-jioffe/NeMo-Retriever-deploy/nightly/ops/retriever-nightly/README.md +Wants=network-online.target +After=network-online.target + +[Service] +Type=oneshot +User=local-jioffe +Group=local-jioffe +WorkingDirectory=/localhome/local-jioffe/NeMo-Retriever-deploy/nightly +Environment=HOME=/localhome/local-jioffe +ExecStart=/usr/bin/bash /localhome/local-jioffe/NeMo-Retriever-deploy/nightly/ops/retriever-nightly/run-latest-main.sh +UMask=0077 +TimeoutStartSec=infinity +Restart=no diff --git a/ops/retriever-nightly/systemd/nrl-harness-batch-hf.timer b/ops/retriever-nightly/systemd/nrl-harness-batch-hf.timer new file mode 100644 index 0000000000..e088847c60 --- /dev/null +++ b/ops/retriever-nightly/systemd/nrl-harness-batch-hf.timer @@ -0,0 +1,14 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# SPDX-License-Identifier: Apache-2.0 + +[Unit] +Description=Schedule the NeMo Retriever batch-HF nightly at midnight Eastern + +[Timer] +OnCalendar=*-*-* 00:00:00 America/New_York +Persistent=false +AccuracySec=1m +Unit=nrl-harness-batch-hf.service + +[Install] +WantedBy=timers.target From ed8a6f0a6472201e2e24cd4b5d79fa2ed56a7b85 Mon Sep 17 00:00:00 2001 From: Jacob Ioffe Date: Tue, 14 Jul 2026 21:29:17 +0000 Subject: [PATCH 02/11] fix(ops): make nightly timer portable Signed-off-by: Jacob Ioffe --- .../test_harness_latest_main_launcher.py | 51 +++- ops/retriever-nightly/README.md | 94 +++---- .../SECOND_HOST_VALIDATION.md | 65 +++-- ops/retriever-nightly/install-systemd.sh | 248 ++++++++++++++++++ .../systemd/nrl-harness-batch-hf.service | 19 -- 5 files changed, 384 insertions(+), 93 deletions(-) create mode 100755 ops/retriever-nightly/install-systemd.sh delete mode 100644 ops/retriever-nightly/systemd/nrl-harness-batch-hf.service diff --git a/nemo_retriever/tests/test_harness_latest_main_launcher.py b/nemo_retriever/tests/test_harness_latest_main_launcher.py index f769802afd..a9347ab743 100644 --- a/nemo_retriever/tests/test_harness_latest_main_launcher.py +++ b/nemo_retriever/tests/test_harness_latest_main_launcher.py @@ -5,6 +5,7 @@ import os from pathlib import Path +import shutil import subprocess import pytest @@ -12,6 +13,7 @@ REPO_ROOT = Path(__file__).resolve().parents[2] LATEST_MAIN_LAUNCHER = REPO_ROOT / "ops" / "retriever-nightly" / "run-latest-main.sh" +SYSTEMD_INSTALLER = REPO_ROOT / "ops" / "retriever-nightly" / "install-systemd.sh" def _git(repository: Path, *args: str) -> subprocess.CompletedProcess[str]: @@ -200,10 +202,49 @@ def test_scheduled_launcher_prunes_only_old_managed_worktrees(latest_main_fixtur assert (checkout_root / f"main-{newer_commit}").is_dir() -def test_systemd_service_uses_latest_main_controller() -> None: - service = (REPO_ROOT / "ops" / "retriever-nightly" / "systemd" / "nrl-harness-batch-hf.service").read_text( - encoding="utf-8" +def test_systemd_installer_renders_portable_latest_main_service(tmp_path: Path) -> None: + result = subprocess.run( + [str(SYSTEMD_INSTALLER), "--render", str(tmp_path)], + cwd=REPO_ROOT, + check=False, + capture_output=True, + text=True, ) - assert "ExecStart=" in service - assert "/run-latest-main.sh" in service + assert result.returncode == 0, result.stderr + service = (tmp_path / "nrl-harness-batch-hf.service").read_text(encoding="utf-8") + timer = (tmp_path / "nrl-harness-batch-hf.timer").read_text(encoding="utf-8") + assert f"WorkingDirectory={REPO_ROOT}" in service + assert f'ExecStart=/usr/bin/bash "{LATEST_MAIN_LAUNCHER}"' in service + assert "User=" in service + assert "Group=" in service + assert "local-jioffe" not in service + assert "/localhome/" not in service + assert "OnCalendar=*-*-* 00:00:00 America/New_York" in timer + assert "Persistent=false" in timer + + +def test_systemd_installer_can_schedule_current_tracking_branch_for_draft(tmp_path: Path) -> None: + controller = tmp_path / "controller with space" + nightly_ops = controller / "ops" / "retriever-nightly" + shutil.copytree(REPO_ROOT / "ops" / "retriever-nightly", nightly_ops) + subprocess.run(["git", "init", "-q", "-b", "review", str(controller)], check=True) + _git(controller, "config", "branch.review.remote", "fork") + _git(controller, "config", "branch.review.merge", "refs/heads/feature/nightly") + output_dir = tmp_path / "units" + + result = subprocess.run( + [str(nightly_ops / "install-systemd.sh"), "--render", str(output_dir), "--test-current-branch"], + cwd=controller, + check=False, + capture_output=True, + text=True, + ) + + assert result.returncode == 0, result.stderr + service = (output_dir / "nrl-harness-batch-hf.service").read_text(encoding="utf-8") + escaped_controller = str(controller).replace(" ", r"\x20") + assert 'Environment="RETRIEVER_LATEST_SOURCE=fork"' in service + assert 'Environment="RETRIEVER_LATEST_REF=feature/nightly"' in service + assert f"WorkingDirectory={escaped_controller}" in service + assert f'ExecStart=/usr/bin/bash "{controller}/ops/retriever-nightly/run-latest-main.sh"' in service diff --git a/ops/retriever-nightly/README.md b/ops/retriever-nightly/README.md index ccbd40c290..d713dacaa6 100644 --- a/ops/retriever-nightly/README.md +++ b/ops/retriever-nightly/README.md @@ -3,11 +3,12 @@ # Retriever Nightly Launcher -This directory provides a manual launcher and a latest-main controller for the -existing systemd deployment. Both run the library and ViDoRe v3 benchmark suite +This directory provides a manual launcher, a latest-main controller, and a +portable systemd installer. They run the library and ViDoRe v3 benchmark suite through the portable harness interface. The manual launcher never changes Git -state; the controller fetches and manages immutable detached worktrees. Neither -installs a scheduler or distributes datasets. +state; the controller fetches and manages immutable detached worktrees; the +installer renders a daily service for the current user and checkout. The tools +do not distribute datasets. ## Manual Teammate Kickoff @@ -246,49 +247,48 @@ dataset and VDB write complete while a child remains idle at high RSS, capture runfile as a focused reproduction; do not classify the symptom as GPU OOM unless the GPU process or kernel logs show an actual allocation failure. -## System Service And Timer - -Manual kickoff does not install recurrence. The existing root-managed service -is a separate adapter for one deployment host. For its `local-jioffe` service -user, the detected nightly root is `/raid/local-jioffe` when that directory is -writable and `/localhome/local-jioffe` otherwise. Its layout is: - -- clean deployment worktree: - `/localhome/local-jioffe/NeMo-Retriever-deploy/nightly` (the reviewed, - pinned controller checkout); -- immutable latest-main worktrees and shared `uv` environment: - `/retriever-nightly-checkouts`; -- machine configuration: - `/.config/nemo-retriever/nightly/nightly.env`; -- full local artifacts: - `/retriever-nightly-artifacts`. - -The configuration file must be owned by `local-jioffe` with mode `600`. It must -contain `HF_TOKEN`; an optional nonempty `SLACK_WEBHOOK_URL` enables Slack for -real scheduled runs. Before enabling the timer, run -`run-latest-main.sh --dry-run` and -`run-latest-main.sh --check-vidore-access` as the service user with the same -`HOME` and configuration file used by systemd. Use `--no-slack` on any real -deployment canary that must not post. The reviewed controller checkout remains -pinned; each scheduled invocation independently selects the latest fetched -`upstream/main` commit. - -The root-managed service runs as `local-jioffe`. The timer starts it every day -at midnight in `America/New_York`, preserving local midnight across EDT and EST -changes. `Persistent=false` deliberately avoids an unscheduled catch-up run if -the host was unavailable at midnight. - -Install or refresh the units from the deployment checkout: +## Portable System Service And Timer + +Manual kickoff does not install recurrence. Run the installer as the intended +service user from a clean, reviewed controller checkout: + +```bash +./ops/retriever-nightly/install-systemd.sh +``` + +The installer derives the service user, group, home directory, and absolute +controller path; renders both units; installs them through `sudo`; and enables +the timer. No checked-in username or checkout path is used. The launcher still +selects writable `/raid/$USER` automatically for configuration, managed +latest-main worktrees, the shared `uv` environment, and artifacts. Other hosts +use the service user's home directory. + +The private `nightly.env` must already be owned by the service user with mode +`600`. It must contain `HF_TOKEN`; a nonempty `SLACK_WEBHOOK_URL` enables one +terminal Slack post for every real scheduled run. Before installing the timer, +run `run-latest-main.sh --dry-run` and +`run-latest-main.sh --check-vidore-access` as that user. The production service +fetches `upstream/main` on every invocation and fails closed rather than running +a stale commit. + +For pre-merge validation only, install a timer that discovers and fetches the +current branch's configured tracking remote and branch: + +```bash +./ops/retriever-nightly/install-systemd.sh --test-current-branch +``` + +This mode lets a second host exercise the draft branch through the real service +and timer. After the PR merges, rerun the installer without +`--test-current-branch`; that removes the test override and restores the +production `upstream/main` selection. The timer starts the service every day at +midnight in `America/New_York`. `Persistent=false` deliberately avoids an +unscheduled catch-up run if the host was unavailable at midnight. + +Kick off one complete service run immediately after installation: ```bash -sudo install -m 0644 \ - ops/retriever-nightly/systemd/nrl-harness-batch-hf.service \ - /etc/systemd/system/nrl-harness-batch-hf.service -sudo install -m 0644 \ - ops/retriever-nightly/systemd/nrl-harness-batch-hf.timer \ - /etc/systemd/system/nrl-harness-batch-hf.timer -sudo systemctl daemon-reload -sudo systemctl enable --now nrl-harness-batch-hf.timer +sudo systemctl start --no-block nrl-harness-batch-hf.service ``` Inspect the schedule and the most recent run: @@ -300,8 +300,8 @@ systemctl show nrl-harness-batch-hf.service \ journalctl -u nrl-harness-batch-hf.service --since today --no-pager ``` -Disable future recurrence without changing completed artifacts: +Disable and remove the service and timer without changing completed artifacts: ```bash -sudo systemctl disable --now nrl-harness-batch-hf.timer +./ops/retriever-nightly/install-systemd.sh --uninstall ``` diff --git a/ops/retriever-nightly/SECOND_HOST_VALIDATION.md b/ops/retriever-nightly/SECOND_HOST_VALIDATION.md index 8d8fbb8439..e398a6c2ff 100644 --- a/ops/retriever-nightly/SECOND_HOST_VALIDATION.md +++ b/ops/retriever-nightly/SECOND_HOST_VALIDATION.md @@ -5,8 +5,9 @@ Use this checklist on a separate Linux NVIDIA workstation before handing the launcher to additional teammates. The review uses the pushed feature branch, -keeps datasets and artifacts outside the repository, and does not install a -timer or post to Slack. +keeps datasets and artifacts outside the repository, validates the functional +path before installing recurrence, and finishes with one complete service run +and terminal Slack post. ## 1. Create a Local Review Branch @@ -41,7 +42,8 @@ nvidia-smi This host has the standard `/datasets/nv-ingest` layout and `/raid/$USER`, so the checked-in dataset map and launcher path defaults apply. Create one private -configuration file and populate `HF_TOKEN`. A read token is sufficient: +configuration file and populate `HF_TOKEN` and `SLACK_WEBHOOK_URL`. A read +Hugging Face token is sufficient: ```bash RETRIEVER_NIGHTLY_CONFIG_DIR="/raid/$USER/.config/nemo-retriever/nightly" @@ -105,35 +107,34 @@ For the dry-run and JP20 sessions, record: - any failed child name and its artifact directory; and - the GPU model and driver from `nvidia-smi`. -The pre-handoff validation is complete when the ViDoRe access check, twelve-run +The functional validation is complete when the ViDoRe access check, twelve-run dry-run, and real JP20 canary succeed with terminal summaries attributed to the -review branch commit. A second full suite is not required before handoff. Do -not enable the timer or post to Slack as part of this functional review. +review branch commit. These steps do not post to Slack. -## 7. Validate Latest-Main Selection After Merge +## 7. Install the Portable Daily Timer -Do not use the latest-main controller while validating the feature branch; its -purpose is to select `upstream/main`. After the PR merges, validate the -scheduled selection path without GPU work: +Install the root-managed service and daily timer for the current service user, +absolute controller checkout, and current branch's tracked remote branch: ```bash -git remote get-url upstream -./ops/retriever-nightly/run-latest-main.sh --dry-run +./ops/retriever-nightly/install-systemd.sh --test-current-branch +systemctl list-timers nrl-harness-batch-hf.timer --all +systemctl cat nrl-harness-batch-hf.service ``` -Confirm that the printed selected SHA matches -`git rev-parse refs/retriever-nightly/latest-main`, the controller checkout did -not move, and the dry-run summary has that SHA as `run_commit`. The installed -systemd service uses this controller automatically, so scheduled users do not -fetch or select a commit themselves. +The rendered service must name the current user and checkout, not a checked-in +host identity. Test mode embeds the tracking remote and branch in the service, +so the scheduled controller exercises the unmerged PR instead of selecting an +older `upstream/main`. The timer runs daily at midnight in +`America/New_York`; it does not run immediately when enabled. -## 8. Run the Lead's First Complete Nightly +## 8. Start the Complete Nightly and Slack Report -After the latest-main dry-run succeeds, run the controller without positional -runfiles or `--no-slack`: +Start the installed service once without waiting for the next timer event: ```bash -./ops/retriever-nightly/run-latest-main.sh +sudo systemctl start --no-block nrl-harness-batch-hf.service +journalctl -fu nrl-harness-batch-hf.service ``` The controller checks ViDoRe access, then runs the four library benchmarks and @@ -141,4 +142,24 @@ all eight ViDoRe v3 domains. Each child runs in a fresh process; failures do not prevent later children from running, and the parent writes one terminal `session_summary.json`. If `SLACK_WEBHOOK_URL` is configured, that terminal summary posts once. Confirm the full `run_commit`, twelve child results, final -exit code, and Slack message before enabling the timer described in `README.md`. +service exit status, and Slack message. The already-enabled timer will launch +the next run at the following scheduled midnight. + +## 9. Switch the Timer to Production After Merge + +After the PR merges, validate production latest-main selection and reinstall +without the test override: + +```bash +git remote get-url upstream +./ops/retriever-nightly/run-latest-main.sh --dry-run +./ops/retriever-nightly/install-systemd.sh +systemctl cat nrl-harness-batch-hf.service +``` + +Confirm that the selected SHA matches +`git rev-parse refs/retriever-nightly/latest-main`, the summary records that SHA +as `run_commit`, and the reinstalled service no longer contains +`RETRIEVER_LATEST_SOURCE` or `RETRIEVER_LATEST_REF`. Future timer events now +fetch and run the latest `upstream/main` automatically. A second complete suite +is not required solely to make this production switch. diff --git a/ops/retriever-nightly/install-systemd.sh b/ops/retriever-nightly/install-systemd.sh new file mode 100755 index 0000000000..6f8d66dfa8 --- /dev/null +++ b/ops/retriever-nightly/install-systemd.sh @@ -0,0 +1,248 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +readonly EXIT_CONFIG=64 +readonly SERVICE_NAME="nrl-harness-batch-hf.service" +readonly TIMER_NAME="nrl-harness-batch-hf.timer" +temporary_directory="" + +cleanup() { + if [[ -n "$temporary_directory" && -d "$temporary_directory" ]]; then + rm -rf -- "$temporary_directory" + fi +} + +trap cleanup EXIT + +log() { + printf 'retriever-nightly-install: %s\n' "$*" >&2 +} + +usage() { + cat <<'EOF' +Usage: install-systemd.sh [--test-current-branch] + install-systemd.sh --render DIRECTORY [--test-current-branch] + install-systemd.sh --uninstall + +Render and install a root-managed daily systemd service for the current user +and this clean controller checkout. The production default runs the latest +fetched upstream/main commit. + +Options: + --render DIRECTORY Render both units without installing or using sudo. + --test-current-branch Temporarily schedule the current branch's tracked + remote branch for pre-merge deployment validation. + --uninstall Disable and remove the installed service and timer. + --help Show this help text. +EOF +} + +escape_systemd_value() { + local value="$1" + if [[ "$value" == *$'\n'* || "$value" == *$'\r'* ]]; then + log "systemd values cannot contain newlines" + return "$EXIT_CONFIG" + fi + value="${value//%/%%}" + value="${value//\\/\\\\}" + value="${value//\"/\\\"}" + printf '%s' "$value" +} + +escape_systemd_path() { + local value="$1" + if [[ "$value" == *$'\n'* || "$value" == *$'\r'* ]]; then + log "systemd paths cannot contain newlines" + return "$EXIT_CONFIG" + fi + value="${value//%/%%}" + value="${value//\\/\\x5c}" + value="${value//\"/\\x22}" + value="${value//$'\t'/\\x09}" + value="${value// /\\x20}" + printf '%s' "$value" +} + +detect_nightly_root() { + local service_user="$1" + local service_home="$2" + local raid_root="/raid/$service_user" + if [[ -d "$raid_root" && -w "$raid_root" ]]; then + printf '%s' "$raid_root" + else + printf '%s' "$service_home" + fi +} + +render_units() { + local output_dir="$1" + local repository="$2" + local service_user="$3" + local service_group="$4" + local service_home="$5" + local source_name="$6" + local source_ref="$7" + local script_dir="$8" + local escaped_repository escaped_repository_path escaped_home escaped_source escaped_ref + escaped_repository="$(escape_systemd_value "$repository")" + escaped_repository_path="$(escape_systemd_path "$repository")" + escaped_home="$(escape_systemd_value "$service_home")" + escaped_source="$(escape_systemd_value "$source_name")" + escaped_ref="$(escape_systemd_value "$source_ref")" + + mkdir -p "$output_dir" + { + printf '%s\n' \ + '# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES.' \ + '# SPDX-License-Identifier: Apache-2.0' \ + '' \ + '[Unit]' \ + 'Description=Run latest selected NeMo Retriever and ViDoRe v3 nightly session' + printf 'Documentation="file:%s/ops/retriever-nightly/README.md"\n' "$escaped_repository" + printf '%s\n' \ + 'Wants=network-online.target' \ + 'After=network-online.target' \ + '' \ + '[Service]' \ + 'Type=oneshot' + printf 'User=%s\n' "$service_user" + printf 'Group=%s\n' "$service_group" + printf 'WorkingDirectory=%s\n' "$escaped_repository_path" + printf 'Environment="HOME=%s"\n' "$escaped_home" + if [[ -n "$source_name" ]]; then + printf 'Environment="RETRIEVER_LATEST_SOURCE=%s"\n' "$escaped_source" + printf 'Environment="RETRIEVER_LATEST_REF=%s"\n' "$escaped_ref" + fi + printf 'ExecStart=/usr/bin/bash "%s/ops/retriever-nightly/run-latest-main.sh"\n' "$escaped_repository" + printf '%s\n' \ + 'UMask=0077' \ + 'TimeoutStartSec=infinity' \ + 'Restart=no' + } >"$output_dir/$SERVICE_NAME" + install -m 0644 "$script_dir/systemd/$TIMER_NAME" "$output_dir/$TIMER_NAME" + chmod 0644 "$output_dir/$SERVICE_NAME" +} + +main() { + local mode="install" + local action_selected=0 + local render_dir="" + local test_current_branch=0 + while (($#)); do + case "$1" in + --render) + [[ $# -ge 2 ]] || { usage >&2; return "$EXIT_CONFIG"; } + ((action_selected == 0)) || { usage >&2; return "$EXIT_CONFIG"; } + action_selected=1 + mode="render" + render_dir="$2" + shift 2 + ;; + --test-current-branch) + test_current_branch=1 + shift + ;; + --uninstall) + ((action_selected == 0)) || { usage >&2; return "$EXIT_CONFIG"; } + action_selected=1 + mode="uninstall" + shift + ;; + --help) + usage + return 0 + ;; + *) + usage >&2 + return "$EXIT_CONFIG" + ;; + esac + done + if [[ "$mode" == "uninstall" && $test_current_branch == 1 ]]; then + log "--uninstall and --test-current-branch cannot be combined" + return "$EXIT_CONFIG" + fi + if [[ "$mode" == "uninstall" ]]; then + sudo systemctl disable --now "$TIMER_NAME" || true + sudo rm -f "/etc/systemd/system/$SERVICE_NAME" "/etc/systemd/system/$TIMER_NAME" + sudo systemctl daemon-reload + log "removed $SERVICE_NAME and $TIMER_NAME" + return 0 + fi + + local script_dir repository service_user service_group service_home nightly_root config_file + script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" + repository="$(git -C "$script_dir" rev-parse --show-toplevel)" + service_user="$(id -un)" + service_group="$(id -gn)" + service_home="${HOME:?HOME must be set}" + nightly_root="$(detect_nightly_root "$service_user" "$service_home")" + config_file="$nightly_root/.config/nemo-retriever/nightly/nightly.env" + + if [[ ! "$service_user" =~ ^[a-zA-Z_][a-zA-Z0-9_-]*[$]?$ || \ + ! "$service_group" =~ ^[a-zA-Z_][a-zA-Z0-9_-]*[$]?$ ]]; then + log "service user and group names must be valid systemd identifiers" + return "$EXIT_CONFIG" + fi + + local source_name="" + local source_ref="" + if ((test_current_branch)); then + local branch merge_ref + branch="$(git -C "$repository" symbolic-ref --quiet --short HEAD)" || { + log "--test-current-branch requires a checked-out branch" + return "$EXIT_CONFIG" + } + source_name="$(git -C "$repository" config --get "branch.$branch.remote" || true)" + merge_ref="$(git -C "$repository" config --get "branch.$branch.merge" || true)" + if [[ -z "$source_name" || "$merge_ref" != refs/heads/* ]]; then + log "current branch must track a remote branch" + return "$EXIT_CONFIG" + fi + source_ref="${merge_ref#refs/heads/}" + log "test mode will fetch $source_name/$source_ref instead of upstream/main" + elif [[ "$mode" != "render" ]] && ! git -C "$repository" remote get-url upstream >/dev/null 2>&1; then + log "production installation requires an upstream remote" + return "$EXIT_CONFIG" + fi + + if [[ "$mode" == "render" ]]; then + render_units "$render_dir" "$repository" "$service_user" "$service_group" "$service_home" \ + "$source_name" "$source_ref" "$script_dir" + log "rendered portable units in $render_dir" + return 0 + fi + + if ((EUID == 0)); then + log "run this installer as the intended service user, without sudo" + return "$EXIT_CONFIG" + fi + if [[ -n "$(git -C "$repository" status --porcelain --untracked-files=normal)" ]]; then + log "controller checkout has local changes: $repository" + return "$EXIT_CONFIG" + fi + if [[ ! -f "$config_file" ]]; then + log "nightly configuration is missing: $config_file" + return "$EXIT_CONFIG" + fi + if [[ "$(stat -c '%a' "$config_file")" != "600" || \ + "$(stat -c '%u' "$config_file")" != "$(id -u)" ]]; then + log "nightly configuration must be owned by $service_user with mode 600" + return "$EXIT_CONFIG" + fi + + temporary_directory="$(mktemp -d)" + render_units "$temporary_directory" "$repository" "$service_user" "$service_group" "$service_home" \ + "$source_name" "$source_ref" "$script_dir" + sudo install -m 0644 "$temporary_directory/$SERVICE_NAME" "/etc/systemd/system/$SERVICE_NAME" + sudo install -m 0644 "$temporary_directory/$TIMER_NAME" "/etc/systemd/system/$TIMER_NAME" + sudo systemctl daemon-reload + sudo systemctl enable --now "$TIMER_NAME" + log "installed daily timer for $service_user using controller $repository" + log "start one full run now with: sudo systemctl start --no-block $SERVICE_NAME" +} + +main "$@" diff --git a/ops/retriever-nightly/systemd/nrl-harness-batch-hf.service b/ops/retriever-nightly/systemd/nrl-harness-batch-hf.service deleted file mode 100644 index 6c5f0be923..0000000000 --- a/ops/retriever-nightly/systemd/nrl-harness-batch-hf.service +++ /dev/null @@ -1,19 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. -# SPDX-License-Identifier: Apache-2.0 - -[Unit] -Description=Run latest upstream/main NeMo Retriever and ViDoRe v3 nightly session -Documentation=file:/localhome/local-jioffe/NeMo-Retriever-deploy/nightly/ops/retriever-nightly/README.md -Wants=network-online.target -After=network-online.target - -[Service] -Type=oneshot -User=local-jioffe -Group=local-jioffe -WorkingDirectory=/localhome/local-jioffe/NeMo-Retriever-deploy/nightly -Environment=HOME=/localhome/local-jioffe -ExecStart=/usr/bin/bash /localhome/local-jioffe/NeMo-Retriever-deploy/nightly/ops/retriever-nightly/run-latest-main.sh -UMask=0077 -TimeoutStartSec=infinity -Restart=no From 882ba2a0442eee430aeee0686a31818eb51988a0 Mon Sep 17 00:00:00 2001 From: Jacob Ioffe Date: Tue, 14 Jul 2026 21:40:04 +0000 Subject: [PATCH 03/11] fix(ops): simplify nightly launch configuration Signed-off-by: Jacob Ioffe --- .../test_harness_latest_main_launcher.py | 76 ++---- .../tests/test_harness_nightly_launcher.py | 20 ++ ops/retriever-nightly/README.md | 123 ++++----- .../SECOND_HOST_VALIDATION.md | 84 ++---- ops/retriever-nightly/install-systemd.sh | 248 ------------------ ops/retriever-nightly/nightly.env.example | 3 +- ops/retriever-nightly/run-latest-main.sh | 27 +- ops/retriever-nightly/run-nightly.sh | 2 +- .../systemd/nrl-harness-batch-hf.timer | 14 - 9 files changed, 128 insertions(+), 469 deletions(-) delete mode 100755 ops/retriever-nightly/install-systemd.sh delete mode 100644 ops/retriever-nightly/systemd/nrl-harness-batch-hf.timer diff --git a/nemo_retriever/tests/test_harness_latest_main_launcher.py b/nemo_retriever/tests/test_harness_latest_main_launcher.py index a9347ab743..6060640954 100644 --- a/nemo_retriever/tests/test_harness_latest_main_launcher.py +++ b/nemo_retriever/tests/test_harness_latest_main_launcher.py @@ -5,7 +5,6 @@ import os from pathlib import Path -import shutil import subprocess import pytest @@ -13,7 +12,7 @@ REPO_ROOT = Path(__file__).resolve().parents[2] LATEST_MAIN_LAUNCHER = REPO_ROOT / "ops" / "retriever-nightly" / "run-latest-main.sh" -SYSTEMD_INSTALLER = REPO_ROOT / "ops" / "retriever-nightly" / "install-systemd.sh" +SLACK_WEBHOOK_URL = "https://hooks.slack.com/services/test/webhook/value" def _git(repository: Path, *args: str) -> subprocess.CompletedProcess[str]: @@ -50,6 +49,13 @@ def latest_main_fixture(tmp_path: Path): '"${VLLM_DEEP_GEMM_WARMUP:-}" != "$EXPECT_DEEP_GEMM_WARMUP" ]]; then', " exit 98", "fi", + 'if [[ -n "${EXPECT_HF_TOKEN+x}" && "${HF_TOKEN:-}" != "$EXPECT_HF_TOKEN" ]]; then', + " exit 97", + "fi", + 'if [[ -n "${EXPECT_SLACK_WEBHOOK_URL+x}" && ' + '"${SLACK_WEBHOOK_URL:-}" != "$EXPECT_SLACK_WEBHOOK_URL" ]]; then', + " exit 96", + "fi", 'checkout="$(git -C "$(dirname -- "$0")" rev-parse --show-toplevel)"', 'commit="$(git -C "$checkout" rev-parse HEAD)"', 'printf "%s|%s|%s\\n" "$commit" "$*" "${UV_PROJECT_ENVIRONMENT:-}" >>"$FAKE_LATEST_CALLS"', @@ -147,6 +153,24 @@ def test_help_does_not_require_configuration_or_fetch(tmp_path: Path) -> None: assert "Fetch upstream/main" in result.stdout +def test_scheduled_launcher_uses_exported_secrets_without_config_file(latest_main_fixture, tmp_path: Path) -> None: + run, calls, _source, _controller, _checkout_root, _initial_commit, latest_commit = latest_main_fixture + + result = run( + "--dry-run", + extra_env={ + "RETRIEVER_CONFIG_FILE": str(tmp_path / "missing-nightly.env"), + "HF_TOKEN": "exported-read-token", + "SLACK_WEBHOOK_URL": SLACK_WEBHOOK_URL, + "EXPECT_HF_TOKEN": "exported-read-token", + "EXPECT_SLACK_WEBHOOK_URL": SLACK_WEBHOOK_URL, + }, + ) + + assert result.returncode == 0, result.stderr + assert calls() == [(latest_commit, "--dry-run", str(_checkout_root / ".venv"))] + + @pytest.mark.parametrize("configured, expected", [(None, "skip"), ("full", "full")]) def test_scheduled_run_has_safe_warmup_default_and_allows_override( latest_main_fixture, configured: str | None, expected: str @@ -200,51 +224,3 @@ def test_scheduled_launcher_prunes_only_old_managed_worktrees(latest_main_fixtur assert result.returncode == 0, result.stderr assert not (checkout_root / f"main-{latest_commit}").exists() assert (checkout_root / f"main-{newer_commit}").is_dir() - - -def test_systemd_installer_renders_portable_latest_main_service(tmp_path: Path) -> None: - result = subprocess.run( - [str(SYSTEMD_INSTALLER), "--render", str(tmp_path)], - cwd=REPO_ROOT, - check=False, - capture_output=True, - text=True, - ) - - assert result.returncode == 0, result.stderr - service = (tmp_path / "nrl-harness-batch-hf.service").read_text(encoding="utf-8") - timer = (tmp_path / "nrl-harness-batch-hf.timer").read_text(encoding="utf-8") - assert f"WorkingDirectory={REPO_ROOT}" in service - assert f'ExecStart=/usr/bin/bash "{LATEST_MAIN_LAUNCHER}"' in service - assert "User=" in service - assert "Group=" in service - assert "local-jioffe" not in service - assert "/localhome/" not in service - assert "OnCalendar=*-*-* 00:00:00 America/New_York" in timer - assert "Persistent=false" in timer - - -def test_systemd_installer_can_schedule_current_tracking_branch_for_draft(tmp_path: Path) -> None: - controller = tmp_path / "controller with space" - nightly_ops = controller / "ops" / "retriever-nightly" - shutil.copytree(REPO_ROOT / "ops" / "retriever-nightly", nightly_ops) - subprocess.run(["git", "init", "-q", "-b", "review", str(controller)], check=True) - _git(controller, "config", "branch.review.remote", "fork") - _git(controller, "config", "branch.review.merge", "refs/heads/feature/nightly") - output_dir = tmp_path / "units" - - result = subprocess.run( - [str(nightly_ops / "install-systemd.sh"), "--render", str(output_dir), "--test-current-branch"], - cwd=controller, - check=False, - capture_output=True, - text=True, - ) - - assert result.returncode == 0, result.stderr - service = (output_dir / "nrl-harness-batch-hf.service").read_text(encoding="utf-8") - escaped_controller = str(controller).replace(" ", r"\x20") - assert 'Environment="RETRIEVER_LATEST_SOURCE=fork"' in service - assert 'Environment="RETRIEVER_LATEST_REF=feature/nightly"' in service - assert f"WorkingDirectory={escaped_controller}" in service - assert f'ExecStart=/usr/bin/bash "{controller}/ops/retriever-nightly/run-latest-main.sh"' in service diff --git a/nemo_retriever/tests/test_harness_nightly_launcher.py b/nemo_retriever/tests/test_harness_nightly_launcher.py index d2a3323e2a..a6033f456d 100644 --- a/nemo_retriever/tests/test_harness_nightly_launcher.py +++ b/nemo_retriever/tests/test_harness_nightly_launcher.py @@ -63,6 +63,9 @@ def nightly_launcher(tmp_path: Path): "expected_warmup = os.environ.get('EXPECT_DEEP_GEMM_WARMUP')", "if expected_warmup is not None and os.environ.get('VLLM_DEEP_GEMM_WARMUP') != expected_warmup:", " raise SystemExit(98)", + "expected_hf_token = os.environ.get('EXPECT_HF_TOKEN')", + "if expected_hf_token is not None and os.environ.get('HF_TOKEN') != expected_hf_token:", + " raise SystemExit(95)", "with Path(os.environ['FAKE_UV_CALLS']).open('a', encoding='utf-8') as stream:", " stream.write(json.dumps(args) + '\\n')", "if 'run-files' in args:", @@ -223,6 +226,23 @@ def test_configured_webhook_posts_terminal_session_to_slack(nightly_launcher, tm assert (session_dir / ".slack_post_attempted").exists() +def test_exported_secrets_post_without_config_file(nightly_launcher) -> None: + run, calls = nightly_launcher + + result = run( + extra_env={ + "HF_TOKEN": "exported-read-token", + "EXPECT_HF_TOKEN": "exported-read-token", + "SLACK_WEBHOOK_URL": SLACK_WEBHOOK_URL, + } + ) + + assert result.returncode == 0, result.stderr + assert len(calls()) == 2 + assert "run-files" in calls()[0] + assert "post-slack" in calls()[1] + + def test_missing_webhook_completes_without_slack(nightly_launcher) -> None: run, calls = nightly_launcher diff --git a/ops/retriever-nightly/README.md b/ops/retriever-nightly/README.md index d713dacaa6..ee6c7781b8 100644 --- a/ops/retriever-nightly/README.md +++ b/ops/retriever-nightly/README.md @@ -3,12 +3,27 @@ # Retriever Nightly Launcher -This directory provides a manual launcher, a latest-main controller, and a -portable systemd installer. They run the library and ViDoRe v3 benchmark suite -through the portable harness interface. The manual launcher never changes Git -state; the controller fetches and manages immutable detached worktrees; the -installer renders a daily service for the current user and checkout. The tools -do not distribute datasets. +This directory provides a manual launcher and a latest-main controller. Both +run the library and ViDoRe v3 benchmark suite through the portable harness +interface. The manual launcher never changes Git state; the controller fetches +and manages immutable detached worktrees. Neither tool installs a scheduler or +distributes datasets. + +## One-Command Full Run + +Export the two supported secrets and invoke the launcher from a clean checkout: + +```bash +export HF_TOKEN=... +export SLACK_WEBHOOK_URL=https://hooks.slack.com/services/... +./ops/retriever-nightly/run-nightly.sh +``` + +That foreground command runs all twelve benchmarks and posts one terminal +Slack summary. `SLACK_WEBHOOK_URL` is optional; omit it to run without posting. +The launcher writes artifacts outside the checkout and prints the terminal +session directory. No `sudo`, systemd service, or configuration file is +required. ## Manual Teammate Kickoff @@ -31,8 +46,8 @@ The supported v1 host is a Linux NVIDIA workstation with: and the repository dependencies. Batch mode starts its models locally, so it needs no model-provider API keys. -On a host with the standard `/datasets/nv-ingest` layout, `nightly.env` accepts -only two secrets and one optional path override: +On a host with the standard `/datasets/nv-ingest` layout, the environment or +optional `nightly.env` accepts only two secrets and one optional path override: | Setting | Required | Purpose | | --- | --- | --- | @@ -44,7 +59,9 @@ On hosts with a writable `/raid/$USER`, the launcher automatically keeps its private configuration, artifacts, and managed latest-main checkouts there. Other hosts use `$HOME`. -Create the private configuration and populate `HF_TOKEN`: +Direct exports are the smallest configuration interface. A private file is +optional for operators who do not want to export the same values in every +shell. To create it: ```bash if [[ -d /raid/$USER && -w /raid/$USER ]]; then @@ -66,9 +83,10 @@ The checked-in `dataset_paths.datasets.yaml` already describes the standard and edit `nemo_retriever/harness/dataset_paths.example.yaml`, then set `RETRIEVER_DATASET_PATHS` in `nightly.env`. -The launcher sources only the detected mode-`600` configuration; it does not -discover a repository `.env` file. `RETRIEVER_CONFIG_FILE` remains an advanced -override for deployments that deliberately store the file elsewhere. +Both launchers source the detected file only when it exists; otherwise they use +the current environment. An existing secrets file must be owned by the +invoking user with mode `600`. The launchers do not discover a repository +`.env` file. `RETRIEVER_CONFIG_FILE` remains an optional advanced path override. Verify the token and read one byte from one remote parquet object in every ViDoRe evaluation partition before starting GPU work: @@ -136,10 +154,11 @@ changes are rejected so a result is attributable to the recorded commit. Ignored cache files do not make the checkout dirty; datasets, configuration, and artifacts remain outside the checkout. -### Scheduled Latest-Main Selection +### Latest-Main Selection Manual `run-nightly.sh` invocations deliberately run the current clean -checkout. The scheduled service instead calls `run-latest-main.sh`, which: +checkout. `run-latest-main.sh` is the corresponding one-command controller for +a checkout where this feature is already present; it: 1. fetches `main` from the `upstream` remote; 2. resolves the fetched commit before doing any GPU work; @@ -160,8 +179,8 @@ detected nightly root at `retriever-nightly-checkouts`; on `/raid` hosts this is worktrees are retained. Modified managed worktrees are never deleted automatically. -The one-time deployment setup must provide an `upstream` remote. Scheduled -users do not choose a branch or commit after that: +The one-time checkout setup must provide an `upstream` remote. Operators do not +choose a branch or commit after that: ```bash git remote get-url upstream >/dev/null 2>&1 || \ @@ -172,15 +191,20 @@ git remote get-url upstream >/dev/null 2>&1 || \ The dry-run fetches and selects the latest commit but skips remote access and GPU execution. Use `run-latest-main.sh --check-vidore-access` to validate the selected latest-main commit and its machine credentials without starting a -session. +session. Once that selected main commit contains this launcher, the complete +latest-main suite is also one command: + +```bash +./ops/retriever-nightly/run-latest-main.sh +``` ### Slack Report -To enable Slack for real runs, add the incoming-webhook URL to the same private -mode-`600` `nightly.env` used for `HF_TOKEN`: +To enable Slack for real runs, export the incoming-webhook URL or place it in +the optional mode-`600` `nightly.env`: ```bash -SLACK_WEBHOOK_URL=https://hooks.slack.com/services/... +export SLACK_WEBHOOK_URL=https://hooks.slack.com/services/... ``` The URL itself is the Slack switch. If it is unset or empty, real runs complete @@ -246,62 +270,3 @@ dataset and VDB write complete while a child remains idle at high RSS, capture `run.log`, `status.json`, process RSS, and the Ray task summary. Retry only that runfile as a focused reproduction; do not classify the symptom as GPU OOM unless the GPU process or kernel logs show an actual allocation failure. - -## Portable System Service And Timer - -Manual kickoff does not install recurrence. Run the installer as the intended -service user from a clean, reviewed controller checkout: - -```bash -./ops/retriever-nightly/install-systemd.sh -``` - -The installer derives the service user, group, home directory, and absolute -controller path; renders both units; installs them through `sudo`; and enables -the timer. No checked-in username or checkout path is used. The launcher still -selects writable `/raid/$USER` automatically for configuration, managed -latest-main worktrees, the shared `uv` environment, and artifacts. Other hosts -use the service user's home directory. - -The private `nightly.env` must already be owned by the service user with mode -`600`. It must contain `HF_TOKEN`; a nonempty `SLACK_WEBHOOK_URL` enables one -terminal Slack post for every real scheduled run. Before installing the timer, -run `run-latest-main.sh --dry-run` and -`run-latest-main.sh --check-vidore-access` as that user. The production service -fetches `upstream/main` on every invocation and fails closed rather than running -a stale commit. - -For pre-merge validation only, install a timer that discovers and fetches the -current branch's configured tracking remote and branch: - -```bash -./ops/retriever-nightly/install-systemd.sh --test-current-branch -``` - -This mode lets a second host exercise the draft branch through the real service -and timer. After the PR merges, rerun the installer without -`--test-current-branch`; that removes the test override and restores the -production `upstream/main` selection. The timer starts the service every day at -midnight in `America/New_York`. `Persistent=false` deliberately avoids an -unscheduled catch-up run if the host was unavailable at midnight. - -Kick off one complete service run immediately after installation: - -```bash -sudo systemctl start --no-block nrl-harness-batch-hf.service -``` - -Inspect the schedule and the most recent run: - -```bash -systemctl list-timers nrl-harness-batch-hf.timer --all -systemctl show nrl-harness-batch-hf.service \ - -p Result -p ExecMainCode -p ExecMainStatus -journalctl -u nrl-harness-batch-hf.service --since today --no-pager -``` - -Disable and remove the service and timer without changing completed artifacts: - -```bash -./ops/retriever-nightly/install-systemd.sh --uninstall -``` diff --git a/ops/retriever-nightly/SECOND_HOST_VALIDATION.md b/ops/retriever-nightly/SECOND_HOST_VALIDATION.md index e398a6c2ff..698ec8ff0e 100644 --- a/ops/retriever-nightly/SECOND_HOST_VALIDATION.md +++ b/ops/retriever-nightly/SECOND_HOST_VALIDATION.md @@ -6,8 +6,8 @@ Use this checklist on a separate Linux NVIDIA workstation before handing the launcher to additional teammates. The review uses the pushed feature branch, keeps datasets and artifacts outside the repository, validates the functional -path before installing recurrence, and finishes with one complete service run -and terminal Slack post. +path first, and finishes with one foreground full-suite command and terminal +Slack post. It does not install or start an operating-system service. ## 1. Create a Local Review Branch @@ -41,25 +41,18 @@ nvidia-smi ``` This host has the standard `/datasets/nv-ingest` layout and `/raid/$USER`, so -the checked-in dataset map and launcher path defaults apply. Create one private -configuration file and populate `HF_TOKEN` and `SLACK_WEBHOOK_URL`. A read -Hugging Face token is sufficient: +the checked-in dataset map and launcher path defaults apply. Export the two +secrets; a read Hugging Face token is sufficient: ```bash -RETRIEVER_NIGHTLY_CONFIG_DIR="/raid/$USER/.config/nemo-retriever/nightly" -mkdir -p "$RETRIEVER_NIGHTLY_CONFIG_DIR" -chmod 700 "$RETRIEVER_NIGHTLY_CONFIG_DIR" -test -e "$RETRIEVER_NIGHTLY_CONFIG_DIR/nightly.env" || \ - cp ops/retriever-nightly/nightly.env.example "$RETRIEVER_NIGHTLY_CONFIG_DIR/nightly.env" -chmod 600 "$RETRIEVER_NIGHTLY_CONFIG_DIR/nightly.env" -${EDITOR:-vi} "$RETRIEVER_NIGHTLY_CONFIG_DIR/nightly.env" +export HF_TOKEN=... +export SLACK_WEBHOOK_URL=https://hooks.slack.com/services/... ``` -The launcher discovers that file and writes artifacts under -`/raid/$USER/retriever-nightly-artifacts`; no shell exports are required. It -does not read a repository `.env`. On a host without the standard dataset -layout, copy and edit `nemo_retriever/harness/dataset_paths.example.yaml` and -set only `RETRIEVER_DATASET_PATHS` in `nightly.env`. +The launcher writes artifacts under `/raid/$USER/retriever-nightly-artifacts`. +No configuration file is required. On a host without the standard dataset +layout, copy and edit `nemo_retriever/harness/dataset_paths.example.yaml`, then +export `RETRIEVER_DATASET_PATHS` with that path. ## 3. Verify ViDoRe Evaluation Access @@ -111,55 +104,18 @@ The functional validation is complete when the ViDoRe access check, twelve-run dry-run, and real JP20 canary succeed with terminal summaries attributed to the review branch commit. These steps do not post to Slack. -## 7. Install the Portable Daily Timer +## 7. Run the Complete Nightly and Slack Report -Install the root-managed service and daily timer for the current service user, -absolute controller checkout, and current branch's tracked remote branch: +From the clean draft checkout, start all twelve benchmarks with one command: ```bash -./ops/retriever-nightly/install-systemd.sh --test-current-branch -systemctl list-timers nrl-harness-batch-hf.timer --all -systemctl cat nrl-harness-batch-hf.service +./ops/retriever-nightly/run-nightly.sh ``` -The rendered service must name the current user and checkout, not a checked-in -host identity. Test mode embeds the tracking remote and branch in the service, -so the scheduled controller exercises the unmerged PR instead of selecting an -older `upstream/main`. The timer runs daily at midnight in -`America/New_York`; it does not run immediately when enabled. - -## 8. Start the Complete Nightly and Slack Report - -Start the installed service once without waiting for the next timer event: - -```bash -sudo systemctl start --no-block nrl-harness-batch-hf.service -journalctl -fu nrl-harness-batch-hf.service -``` - -The controller checks ViDoRe access, then runs the four library benchmarks and -all eight ViDoRe v3 domains. Each child runs in a fresh process; failures do not -prevent later children from running, and the parent writes one terminal -`session_summary.json`. If `SLACK_WEBHOOK_URL` is configured, that terminal -summary posts once. Confirm the full `run_commit`, twelve child results, final -service exit status, and Slack message. The already-enabled timer will launch -the next run at the following scheduled midnight. - -## 9. Switch the Timer to Production After Merge - -After the PR merges, validate production latest-main selection and reinstall -without the test override: - -```bash -git remote get-url upstream -./ops/retriever-nightly/run-latest-main.sh --dry-run -./ops/retriever-nightly/install-systemd.sh -systemctl cat nrl-harness-batch-hf.service -``` - -Confirm that the selected SHA matches -`git rev-parse refs/retriever-nightly/latest-main`, the summary records that SHA -as `run_commit`, and the reinstalled service no longer contains -`RETRIEVER_LATEST_SOURCE` or `RETRIEVER_LATEST_REF`. Future timer events now -fetch and run the latest `upstream/main` automatically. A second complete suite -is not required solely to make this production switch. +The launcher runs the four library benchmarks and all eight ViDoRe v3 domains. +Each child runs in a fresh process; failures do not prevent later children from +running, and the parent writes one terminal `session_summary.json`. Because +`SLACK_WEBHOOK_URL` is exported, that terminal summary posts once. Confirm the +full `run_commit`, twelve child results, final command exit status, and Slack +message. The process remains attached to the invoking shell; use the host's +normal session manager if it must survive a disconnected terminal. diff --git a/ops/retriever-nightly/install-systemd.sh b/ops/retriever-nightly/install-systemd.sh deleted file mode 100755 index 6f8d66dfa8..0000000000 --- a/ops/retriever-nightly/install-systemd.sh +++ /dev/null @@ -1,248 +0,0 @@ -#!/usr/bin/env bash -# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. -# SPDX-License-Identifier: Apache-2.0 - -set -euo pipefail - -readonly EXIT_CONFIG=64 -readonly SERVICE_NAME="nrl-harness-batch-hf.service" -readonly TIMER_NAME="nrl-harness-batch-hf.timer" -temporary_directory="" - -cleanup() { - if [[ -n "$temporary_directory" && -d "$temporary_directory" ]]; then - rm -rf -- "$temporary_directory" - fi -} - -trap cleanup EXIT - -log() { - printf 'retriever-nightly-install: %s\n' "$*" >&2 -} - -usage() { - cat <<'EOF' -Usage: install-systemd.sh [--test-current-branch] - install-systemd.sh --render DIRECTORY [--test-current-branch] - install-systemd.sh --uninstall - -Render and install a root-managed daily systemd service for the current user -and this clean controller checkout. The production default runs the latest -fetched upstream/main commit. - -Options: - --render DIRECTORY Render both units without installing or using sudo. - --test-current-branch Temporarily schedule the current branch's tracked - remote branch for pre-merge deployment validation. - --uninstall Disable and remove the installed service and timer. - --help Show this help text. -EOF -} - -escape_systemd_value() { - local value="$1" - if [[ "$value" == *$'\n'* || "$value" == *$'\r'* ]]; then - log "systemd values cannot contain newlines" - return "$EXIT_CONFIG" - fi - value="${value//%/%%}" - value="${value//\\/\\\\}" - value="${value//\"/\\\"}" - printf '%s' "$value" -} - -escape_systemd_path() { - local value="$1" - if [[ "$value" == *$'\n'* || "$value" == *$'\r'* ]]; then - log "systemd paths cannot contain newlines" - return "$EXIT_CONFIG" - fi - value="${value//%/%%}" - value="${value//\\/\\x5c}" - value="${value//\"/\\x22}" - value="${value//$'\t'/\\x09}" - value="${value// /\\x20}" - printf '%s' "$value" -} - -detect_nightly_root() { - local service_user="$1" - local service_home="$2" - local raid_root="/raid/$service_user" - if [[ -d "$raid_root" && -w "$raid_root" ]]; then - printf '%s' "$raid_root" - else - printf '%s' "$service_home" - fi -} - -render_units() { - local output_dir="$1" - local repository="$2" - local service_user="$3" - local service_group="$4" - local service_home="$5" - local source_name="$6" - local source_ref="$7" - local script_dir="$8" - local escaped_repository escaped_repository_path escaped_home escaped_source escaped_ref - escaped_repository="$(escape_systemd_value "$repository")" - escaped_repository_path="$(escape_systemd_path "$repository")" - escaped_home="$(escape_systemd_value "$service_home")" - escaped_source="$(escape_systemd_value "$source_name")" - escaped_ref="$(escape_systemd_value "$source_ref")" - - mkdir -p "$output_dir" - { - printf '%s\n' \ - '# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES.' \ - '# SPDX-License-Identifier: Apache-2.0' \ - '' \ - '[Unit]' \ - 'Description=Run latest selected NeMo Retriever and ViDoRe v3 nightly session' - printf 'Documentation="file:%s/ops/retriever-nightly/README.md"\n' "$escaped_repository" - printf '%s\n' \ - 'Wants=network-online.target' \ - 'After=network-online.target' \ - '' \ - '[Service]' \ - 'Type=oneshot' - printf 'User=%s\n' "$service_user" - printf 'Group=%s\n' "$service_group" - printf 'WorkingDirectory=%s\n' "$escaped_repository_path" - printf 'Environment="HOME=%s"\n' "$escaped_home" - if [[ -n "$source_name" ]]; then - printf 'Environment="RETRIEVER_LATEST_SOURCE=%s"\n' "$escaped_source" - printf 'Environment="RETRIEVER_LATEST_REF=%s"\n' "$escaped_ref" - fi - printf 'ExecStart=/usr/bin/bash "%s/ops/retriever-nightly/run-latest-main.sh"\n' "$escaped_repository" - printf '%s\n' \ - 'UMask=0077' \ - 'TimeoutStartSec=infinity' \ - 'Restart=no' - } >"$output_dir/$SERVICE_NAME" - install -m 0644 "$script_dir/systemd/$TIMER_NAME" "$output_dir/$TIMER_NAME" - chmod 0644 "$output_dir/$SERVICE_NAME" -} - -main() { - local mode="install" - local action_selected=0 - local render_dir="" - local test_current_branch=0 - while (($#)); do - case "$1" in - --render) - [[ $# -ge 2 ]] || { usage >&2; return "$EXIT_CONFIG"; } - ((action_selected == 0)) || { usage >&2; return "$EXIT_CONFIG"; } - action_selected=1 - mode="render" - render_dir="$2" - shift 2 - ;; - --test-current-branch) - test_current_branch=1 - shift - ;; - --uninstall) - ((action_selected == 0)) || { usage >&2; return "$EXIT_CONFIG"; } - action_selected=1 - mode="uninstall" - shift - ;; - --help) - usage - return 0 - ;; - *) - usage >&2 - return "$EXIT_CONFIG" - ;; - esac - done - if [[ "$mode" == "uninstall" && $test_current_branch == 1 ]]; then - log "--uninstall and --test-current-branch cannot be combined" - return "$EXIT_CONFIG" - fi - if [[ "$mode" == "uninstall" ]]; then - sudo systemctl disable --now "$TIMER_NAME" || true - sudo rm -f "/etc/systemd/system/$SERVICE_NAME" "/etc/systemd/system/$TIMER_NAME" - sudo systemctl daemon-reload - log "removed $SERVICE_NAME and $TIMER_NAME" - return 0 - fi - - local script_dir repository service_user service_group service_home nightly_root config_file - script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" - repository="$(git -C "$script_dir" rev-parse --show-toplevel)" - service_user="$(id -un)" - service_group="$(id -gn)" - service_home="${HOME:?HOME must be set}" - nightly_root="$(detect_nightly_root "$service_user" "$service_home")" - config_file="$nightly_root/.config/nemo-retriever/nightly/nightly.env" - - if [[ ! "$service_user" =~ ^[a-zA-Z_][a-zA-Z0-9_-]*[$]?$ || \ - ! "$service_group" =~ ^[a-zA-Z_][a-zA-Z0-9_-]*[$]?$ ]]; then - log "service user and group names must be valid systemd identifiers" - return "$EXIT_CONFIG" - fi - - local source_name="" - local source_ref="" - if ((test_current_branch)); then - local branch merge_ref - branch="$(git -C "$repository" symbolic-ref --quiet --short HEAD)" || { - log "--test-current-branch requires a checked-out branch" - return "$EXIT_CONFIG" - } - source_name="$(git -C "$repository" config --get "branch.$branch.remote" || true)" - merge_ref="$(git -C "$repository" config --get "branch.$branch.merge" || true)" - if [[ -z "$source_name" || "$merge_ref" != refs/heads/* ]]; then - log "current branch must track a remote branch" - return "$EXIT_CONFIG" - fi - source_ref="${merge_ref#refs/heads/}" - log "test mode will fetch $source_name/$source_ref instead of upstream/main" - elif [[ "$mode" != "render" ]] && ! git -C "$repository" remote get-url upstream >/dev/null 2>&1; then - log "production installation requires an upstream remote" - return "$EXIT_CONFIG" - fi - - if [[ "$mode" == "render" ]]; then - render_units "$render_dir" "$repository" "$service_user" "$service_group" "$service_home" \ - "$source_name" "$source_ref" "$script_dir" - log "rendered portable units in $render_dir" - return 0 - fi - - if ((EUID == 0)); then - log "run this installer as the intended service user, without sudo" - return "$EXIT_CONFIG" - fi - if [[ -n "$(git -C "$repository" status --porcelain --untracked-files=normal)" ]]; then - log "controller checkout has local changes: $repository" - return "$EXIT_CONFIG" - fi - if [[ ! -f "$config_file" ]]; then - log "nightly configuration is missing: $config_file" - return "$EXIT_CONFIG" - fi - if [[ "$(stat -c '%a' "$config_file")" != "600" || \ - "$(stat -c '%u' "$config_file")" != "$(id -u)" ]]; then - log "nightly configuration must be owned by $service_user with mode 600" - return "$EXIT_CONFIG" - fi - - temporary_directory="$(mktemp -d)" - render_units "$temporary_directory" "$repository" "$service_user" "$service_group" "$service_home" \ - "$source_name" "$source_ref" "$script_dir" - sudo install -m 0644 "$temporary_directory/$SERVICE_NAME" "/etc/systemd/system/$SERVICE_NAME" - sudo install -m 0644 "$temporary_directory/$TIMER_NAME" "/etc/systemd/system/$TIMER_NAME" - sudo systemctl daemon-reload - sudo systemctl enable --now "$TIMER_NAME" - log "installed daily timer for $service_user using controller $repository" - log "start one full run now with: sudo systemctl start --no-block $SERVICE_NAME" -} - -main "$@" diff --git a/ops/retriever-nightly/nightly.env.example b/ops/retriever-nightly/nightly.env.example index 4048526810..712a576dfb 100644 --- a/ops/retriever-nightly/nightly.env.example +++ b/ops/retriever-nightly/nightly.env.example @@ -5,7 +5,8 @@ # with /raid/$USER, the launcher keeps configuration, artifacts, and managed # latest-main checkouts there. Otherwise it uses $HOME. # -# Required operator input. A read token is sufficient: +# Required for authenticated ViDoRe runs when HF_TOKEN is not already exported. +# A read token is sufficient: HF_TOKEN= # Optional: uncomment to post terminal real-run summaries to Slack. diff --git a/ops/retriever-nightly/run-latest-main.sh b/ops/retriever-nightly/run-latest-main.sh index 36cce3d462..b0538f0af3 100755 --- a/ops/retriever-nightly/run-latest-main.sh +++ b/ops/retriever-nightly/run-latest-main.sh @@ -77,19 +77,18 @@ main() { fi local nightly_root="${RETRIEVER_NIGHTLY_ROOT:-$default_nightly_root}" local config_file="${RETRIEVER_CONFIG_FILE:-$nightly_root/.config/nemo-retriever/nightly/nightly.env}" - if [[ ! -f "$config_file" ]]; then - log "nightly configuration is missing: $config_file" - return "$EXIT_CONFIG" - fi - if [[ "$(stat -c '%a' "$config_file")" != "600" || \ - "$(stat -c '%u' "$config_file")" != "$(id -u)" ]]; then - log "nightly configuration must be owned by the service user with mode 600" - return "$EXIT_CONFIG" + if [[ -f "$config_file" ]]; then + if [[ "$(stat -c '%a' "$config_file")" != "600" || \ + "$(stat -c '%u' "$config_file")" != "$(id -u)" ]]; then + log "nightly configuration must be owned by the invoking user with mode 600" + return "$EXIT_CONFIG" + fi + set -a + # shellcheck disable=SC1090 + source "$config_file" + set +a fi - set -a - # shellcheck disable=SC1090 - source "$config_file" - set +a + local slack_webhook_url="${SLACK_WEBHOOK_URL:-}" unset SLACK_WEBHOOK_URL nightly_root="${RETRIEVER_NIGHTLY_ROOT:-$nightly_root}" @@ -188,7 +187,11 @@ main() { "$target_launcher" --check-vidore-access || run_rc=$? fi if ((run_rc == 0)); then + if [[ -n "$slack_webhook_url" ]]; then + export SLACK_WEBHOOK_URL="$slack_webhook_url" + fi "$target_launcher" "$@" || run_rc=$? + unset SLACK_WEBHOOK_URL else log "ViDoRe access preflight failed; skipping GPU work" fi diff --git a/ops/retriever-nightly/run-nightly.sh b/ops/retriever-nightly/run-nightly.sh index 4891a22ad3..bb86756ff2 100755 --- a/ops/retriever-nightly/run-nightly.sh +++ b/ops/retriever-nightly/run-nightly.sh @@ -45,7 +45,7 @@ nightly_root="${RETRIEVER_NIGHTLY_ROOT:-$default_nightly_root}" readonly config_file="${RETRIEVER_CONFIG_FILE:-$nightly_root/.config/nemo-retriever/nightly/nightly.env}" if [[ -f "$config_file" ]]; then if [[ "$(stat -c '%a' "$config_file")" != "600" || "$(stat -c '%u' "$config_file")" != "$(id -u)" ]]; then - log "nightly configuration must be owned by the service user with mode 600" + log "nightly configuration must be owned by the invoking user with mode 600" exit "$EXIT_CONFIG" fi set -a diff --git a/ops/retriever-nightly/systemd/nrl-harness-batch-hf.timer b/ops/retriever-nightly/systemd/nrl-harness-batch-hf.timer deleted file mode 100644 index e088847c60..0000000000 --- a/ops/retriever-nightly/systemd/nrl-harness-batch-hf.timer +++ /dev/null @@ -1,14 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. -# SPDX-License-Identifier: Apache-2.0 - -[Unit] -Description=Schedule the NeMo Retriever batch-HF nightly at midnight Eastern - -[Timer] -OnCalendar=*-*-* 00:00:00 America/New_York -Persistent=false -AccuracySec=1m -Unit=nrl-harness-batch-hf.service - -[Install] -WantedBy=timers.target From 8cadf6693943926d6a98e7e55ca105564e0edcb0 Mon Sep 17 00:00:00 2001 From: Jacob Ioffe Date: Wed, 15 Jul 2026 13:46:33 +0000 Subject: [PATCH 04/11] refactor(ops): unify nightly launcher entry point Signed-off-by: Jacob Ioffe --- ... => test_harness_nightly_git_selection.py} | 43 ++-- .../tests/test_harness_nightly_launcher.py | 1 + ops/retriever-nightly/README.md | 158 ++++++++----- .../SECOND_HOST_VALIDATION.md | 43 +++- ops/retriever-nightly/nightly.env.example | 8 +- ops/retriever-nightly/run-latest-main.sh | 203 ----------------- ops/retriever-nightly/run-nightly.sh | 208 +++++++++++++++++- 7 files changed, 378 insertions(+), 286 deletions(-) rename nemo_retriever/tests/{test_harness_latest_main_launcher.py => test_harness_nightly_git_selection.py} (83%) delete mode 100755 ops/retriever-nightly/run-latest-main.sh diff --git a/nemo_retriever/tests/test_harness_latest_main_launcher.py b/nemo_retriever/tests/test_harness_nightly_git_selection.py similarity index 83% rename from nemo_retriever/tests/test_harness_latest_main_launcher.py rename to nemo_retriever/tests/test_harness_nightly_git_selection.py index 6060640954..b3d1fba447 100644 --- a/nemo_retriever/tests/test_harness_latest_main_launcher.py +++ b/nemo_retriever/tests/test_harness_nightly_git_selection.py @@ -11,7 +11,7 @@ REPO_ROOT = Path(__file__).resolve().parents[2] -LATEST_MAIN_LAUNCHER = REPO_ROOT / "ops" / "retriever-nightly" / "run-latest-main.sh" +NIGHTLY_LAUNCHER = REPO_ROOT / "ops" / "retriever-nightly" / "run-nightly.sh" SLACK_WEBHOOK_URL = "https://hooks.slack.com/services/test/webhook/value" @@ -104,7 +104,7 @@ def latest_main_fixture(tmp_path: Path): def run(*args: str, extra_env: dict[str, str] | None = None) -> subprocess.CompletedProcess[str]: return subprocess.run( - [str(LATEST_MAIN_LAUNCHER), *args], + [str(NIGHTLY_LAUNCHER), *args], cwd=tmp_path, env=env | (extra_env or {}), check=False, @@ -120,7 +120,7 @@ def calls() -> list[tuple[str, str, str]]: return run, calls, source, controller, checkout_root, initial_commit, latest_commit -def test_scheduled_launcher_fetches_latest_main_into_immutable_worktree(latest_main_fixture) -> None: +def test_launcher_fetches_latest_main_into_immutable_worktree(latest_main_fixture) -> None: run, calls, _source, controller, checkout_root, initial_commit, latest_commit = latest_main_fixture result = run() @@ -131,7 +131,7 @@ def test_scheduled_launcher_fetches_latest_main_into_immutable_worktree(latest_m (latest_commit, "--check-vidore-access", str(checkout_root / ".venv")), (latest_commit, "", str(checkout_root / ".venv")), ] - selected_checkout = checkout_root / f"main-{latest_commit}" + selected_checkout = checkout_root / f"commit-{latest_commit}" assert selected_checkout.is_dir() assert _git(selected_checkout, "rev-parse", "HEAD").stdout.strip() == latest_commit @@ -142,7 +142,7 @@ def test_help_does_not_require_configuration_or_fetch(tmp_path: Path) -> None: env.pop("RETRIEVER_CONFIG_FILE", None) result = subprocess.run( - [str(LATEST_MAIN_LAUNCHER), "--help"], + [str(NIGHTLY_LAUNCHER), "--help"], env=env, check=False, capture_output=True, @@ -150,10 +150,25 @@ def test_help_does_not_require_configuration_or_fetch(tmp_path: Path) -> None: ) assert result.returncode == 0, result.stderr - assert "Fetch upstream/main" in result.stdout + assert "latest" in result.stdout + assert "--ref REF" in result.stdout -def test_scheduled_launcher_uses_exported_secrets_without_config_file(latest_main_fixture, tmp_path: Path) -> None: +def test_explicit_ref_runs_local_commit_without_fetching(latest_main_fixture) -> None: + run, calls, _source, _controller, checkout_root, initial_commit, _latest_commit = latest_main_fixture + + result = run( + "--ref", + "HEAD", + "--dry-run", + extra_env={"RETRIEVER_LATEST_SOURCE": str(checkout_root / "missing-source")}, + ) + + assert result.returncode == 0, result.stderr + assert calls() == [(initial_commit, "--dry-run", str(checkout_root / ".venv"))] + + +def test_launcher_selection_uses_exported_secrets_without_config_file(latest_main_fixture, tmp_path: Path) -> None: run, calls, _source, _controller, _checkout_root, _initial_commit, latest_commit = latest_main_fixture result = run( @@ -172,7 +187,7 @@ def test_scheduled_launcher_uses_exported_secrets_without_config_file(latest_mai @pytest.mark.parametrize("configured, expected", [(None, "skip"), ("full", "full")]) -def test_scheduled_run_has_safe_warmup_default_and_allows_override( +def test_selected_run_has_safe_warmup_default_and_allows_override( latest_main_fixture, configured: str | None, expected: str ) -> None: run, _calls, *_rest = latest_main_fixture @@ -185,7 +200,7 @@ def test_scheduled_run_has_safe_warmup_default_and_allows_override( assert result.returncode == 0, result.stderr -def test_scheduled_launcher_fails_closed_when_access_preflight_fails(latest_main_fixture) -> None: +def test_launcher_fails_closed_when_access_preflight_fails(latest_main_fixture) -> None: run, calls, _source, _controller, _checkout_root, _initial_commit, latest_commit = latest_main_fixture result = run(extra_env={"FAKE_ACCESS_RC": "3"}) @@ -194,7 +209,7 @@ def test_scheduled_launcher_fails_closed_when_access_preflight_fails(latest_main assert calls() == [(latest_commit, "--check-vidore-access", str(_checkout_root / ".venv"))] -def test_scheduled_launcher_does_not_run_stale_commit_when_fetch_fails(latest_main_fixture) -> None: +def test_launcher_does_not_run_stale_commit_when_fetch_fails(latest_main_fixture) -> None: run, calls, _source, _controller, _checkout_root, _initial_commit, _latest_commit = latest_main_fixture result = run(extra_env={"RETRIEVER_LATEST_SOURCE": str(_checkout_root / "missing-source")}) @@ -203,7 +218,7 @@ def test_scheduled_launcher_does_not_run_stale_commit_when_fetch_fails(latest_ma assert calls() == [] -def test_scheduled_launcher_rejects_modified_controller(latest_main_fixture) -> None: +def test_launcher_rejects_modified_controller(latest_main_fixture) -> None: run, calls, _source, controller, _checkout_root, _initial_commit, _latest_commit = latest_main_fixture (controller / "version.txt").write_text("modified\n", encoding="utf-8") @@ -213,7 +228,7 @@ def test_scheduled_launcher_rejects_modified_controller(latest_main_fixture) -> assert calls() == [] -def test_scheduled_launcher_prunes_only_old_managed_worktrees(latest_main_fixture) -> None: +def test_launcher_prunes_only_old_managed_worktrees(latest_main_fixture) -> None: run, _calls, source, _controller, checkout_root, _initial_commit, latest_commit = latest_main_fixture assert run(extra_env={"RETRIEVER_LATEST_KEEP_CHECKOUTS": "1"}).returncode == 0 @@ -222,5 +237,5 @@ def test_scheduled_launcher_prunes_only_old_managed_worktrees(latest_main_fixtur result = run(extra_env={"RETRIEVER_LATEST_KEEP_CHECKOUTS": "1"}) assert result.returncode == 0, result.stderr - assert not (checkout_root / f"main-{latest_commit}").exists() - assert (checkout_root / f"main-{newer_commit}").is_dir() + assert not (checkout_root / f"commit-{latest_commit}").exists() + assert (checkout_root / f"commit-{newer_commit}").is_dir() diff --git a/nemo_retriever/tests/test_harness_nightly_launcher.py b/nemo_retriever/tests/test_harness_nightly_launcher.py index a6033f456d..3efe16e75b 100644 --- a/nemo_retriever/tests/test_harness_nightly_launcher.py +++ b/nemo_retriever/tests/test_harness_nightly_launcher.py @@ -94,6 +94,7 @@ def nightly_launcher(tmp_path: Path): { "HOME": str(tmp_path / "home"), "RETRIEVER_CHECKOUT": str(checkout), + "RETRIEVER_SELECTED_CHECKOUT": str(checkout), "RETRIEVER_NIGHTLY_ROOT": str(tmp_path / "nightly-root"), "RETRIEVER_UV_BIN": str(fake_uv), "FAKE_UV_CALLS": str(calls_path), diff --git a/ops/retriever-nightly/README.md b/ops/retriever-nightly/README.md index ee6c7781b8..3c6190303d 100644 --- a/ops/retriever-nightly/README.md +++ b/ops/retriever-nightly/README.md @@ -3,15 +3,27 @@ # Retriever Nightly Launcher -This directory provides a manual launcher and a latest-main controller. Both -run the library and ViDoRe v3 benchmark suite through the portable harness -interface. The manual launcher never changes Git state; the controller fetches -and manages immutable detached worktrees. Neither tool installs a scheduler or -distributes datasets. +This directory provides one public launcher for the library and ViDoRe v3 +benchmark suite: -## One-Command Full Run +| Workflow | Command | Code that runs | +| --- | --- | --- | +| Regular nightly | `run-nightly.sh` | Freshly fetched `upstream/main`. | +| Review the checked-out PR | `run-nightly.sh --ref HEAD` | The current checkout's committed `HEAD`. | +| Reproduce an exact revision | `run-nightly.sh --ref ` | An already available local Git commit. | + +The launcher resolves one commit, creates or reuses an immutable detached +worktree, and exits after one terminal session summary. Recurrence is +deliberately kept outside its interface; the [daily `tmux` +workflow](#daily-runs-with-tmux) is a small shell loop rather than an installed +scheduler. The launcher never merges into or moves the controller checkout, +and it does not distribute datasets. -Export the two supported secrets and invoke the launcher from a clean checkout: +## Quick Start On A Standard Host + +After this launcher is present on `upstream/main`, a workstation with +`/datasets/nv-ingest` and writable `/raid/$USER` needs only a Hugging Face token +and, optionally, a Slack webhook: ```bash export HF_TOKEN=... @@ -21,11 +33,20 @@ export SLACK_WEBHOOK_URL=https://hooks.slack.com/services/... That foreground command runs all twelve benchmarks and posts one terminal Slack summary. `SLACK_WEBHOOK_URL` is optional; omit it to run without posting. -The launcher writes artifacts outside the checkout and prints the terminal -session directory. No `sudo`, systemd service, or configuration file is -required. +Before GPU work, it fetches `upstream/main`, selects its exact commit, and +checks ViDoRe access. It then uses the checked-in `/datasets/nv-ingest` map, +writes artifacts under `/raid/$USER/retriever-nightly-artifacts`, and prints the +terminal session directory. No model-provider API key, `nightly.env`, `sudo`, +systemd service, or timer is required. + +The only required operator input on that host is `HF_TOKEN`. Set +`SLACK_WEBHOOK_URL` when the terminal summary should post to Slack. A custom +dataset map or artifact root is needed only when the host does not have the +standard paths. While reviewing this unmerged PR, add `--ref HEAD` to every +launcher command so the review commit runs instead of the current upstream +`main`. -## Manual Teammate Kickoff +## Validate The Current PR Checkout ### Prerequisites @@ -56,7 +77,7 @@ optional `nightly.env` accepts only two secrets and one optional path override: | `RETRIEVER_DATASET_PATHS` | nonstandard hosts only | Replaces the checked-in `/datasets/nv-ingest` map. | On hosts with a writable `/raid/$USER`, the launcher automatically keeps its -private configuration, artifacts, and managed latest-main checkouts there. +private configuration, artifacts, and managed Git checkouts there. Other hosts use `$HOME`. Direct exports are the smallest configuration interface. A private file is @@ -83,16 +104,16 @@ The checked-in `dataset_paths.datasets.yaml` already describes the standard and edit `nemo_retriever/harness/dataset_paths.example.yaml`, then set `RETRIEVER_DATASET_PATHS` in `nightly.env`. -Both launchers source the detected file only when it exists; otherwise they use +The launcher sources the detected file only when it exists; otherwise it uses the current environment. An existing secrets file must be owned by the -invoking user with mode `600`. The launchers do not discover a repository +invoking user with mode `600`. The launcher does not discover a repository `.env` file. `RETRIEVER_CONFIG_FILE` remains an optional advanced path override. Verify the token and read one byte from one remote parquet object in every ViDoRe evaluation partition before starting GPU work: ```bash -./ops/retriever-nightly/run-nightly.sh --check-vidore-access +./ops/retriever-nightly/run-nightly.sh --ref HEAD --check-vidore-access ``` The access check does not download full parquet objects. A redirect failure @@ -103,7 +124,7 @@ Then preflight the complete twelve-benchmark suite without starting ingest or query: ```bash -./ops/retriever-nightly/run-nightly.sh --dry-run +./ops/retriever-nightly/run-nightly.sh --ref HEAD --dry-run ``` Inspect the resulting `session_summary.json` and child plans. Dry-runs never @@ -115,57 +136,31 @@ Use one positional runfile for a smaller real canary before the full run: ```bash ./ops/retriever-nightly/run-nightly.sh \ + --ref HEAD \ --no-slack \ nemo_retriever/harness/runfiles/jp20_beir.json ``` -Run the complete suite from the current checkout with no positional runfiles: +Run the complete suite from the review commit with no positional runfiles: ```bash -./ops/retriever-nightly/run-nightly.sh +./ops/retriever-nightly/run-nightly.sh --ref HEAD ``` If `SLACK_WEBHOOK_URL` is configured, that real run posts its terminal summary. Add `--no-slack` only when the full run is itself a functional test that must not post. -### Select the Code Under Test - -The launcher runs the current clean checkout and records its commit in the -session artifacts. It never changes that checkout. To run the latest fetched -`upstream/main` without disturbing another worktree: - -```bash -git fetch upstream main -git worktree add --detach ../NeMo-Retriever-benchmark-main upstream/main -cd ../NeMo-Retriever-benchmark-main -``` - -To run an exact fetched commit, replace `upstream/main` with its SHA and choose -a distinct worktree directory: - -```bash -git worktree add --detach ../NeMo-Retriever-benchmark-abc1234 abc1234 -cd ../NeMo-Retriever-benchmark-abc1234 -``` - -Run the same launcher command from that worktree. Tracked, staged, and untracked -changes are rejected so a result is attributable to the recorded commit. -Ignored cache files do not make the checkout dirty; datasets, configuration, -and artifacts remain outside the checkout. - -### Latest-Main Selection +## Git Selection -Manual `run-nightly.sh` invocations deliberately run the current clean -checkout. `run-latest-main.sh` is the corresponding one-command controller for -a checkout where this feature is already present; it: +With no `--ref`, `run-nightly.sh`: 1. fetches `main` from the `upstream` remote; 2. resolves the fetched commit before doing any GPU work; -3. creates or reuses an immutable detached worktree named `main-`; +3. creates or reuses an immutable detached worktree named `commit-`; 4. runs the ViDoRe access check from that selected commit; and -5. invokes that commit's `run-nightly.sh` only when fetch and access preflight - both succeed. +5. invokes that commit's nightly execution path only when fetch and access + preflight both succeed. A fetch failure is fail-closed: the controller does not fall back to yesterday's commit. It never runs `git pull`, merges into the controller checkout, or moves @@ -179,26 +174,71 @@ detected nightly root at `retriever-nightly-checkouts`; on `/raid` hosts this is worktrees are retained. Modified managed worktrees are never deleted automatically. -The one-time checkout setup must provide an `upstream` remote. Operators do not -choose a branch or commit after that: +The one-time controller checkout setup must provide an `upstream` remote. +Operators do not choose a branch or commit for normal nightlies after that: ```bash git remote get-url upstream >/dev/null 2>&1 || \ git remote add upstream https://github.com/NVIDIA/NeMo-Retriever.git -./ops/retriever-nightly/run-latest-main.sh --dry-run +./ops/retriever-nightly/run-nightly.sh --dry-run ``` The dry-run fetches and selects the latest commit but skips remote access and -GPU execution. Use `run-latest-main.sh --check-vidore-access` to validate the +GPU execution. Use `run-nightly.sh --check-vidore-access` to validate the selected latest-main commit and its machine credentials without starting a session. Once that selected main commit contains this launcher, the complete -latest-main suite is also one command: +latest-main suite remains the same one command: ```bash -./ops/retriever-nightly/run-latest-main.sh +./ops/retriever-nightly/run-nightly.sh ``` -### Slack Report +`--ref REF` skips the fetch and resolves an existing local Git ref or commit. +The selected commit still runs from a managed detached worktree, so current +branch state is never moved. Use `--ref HEAD` for this draft PR and `--ref +` to reproduce an earlier run. The controller checkout must be clean; +tracked, staged, and untracked changes are rejected so every result is +attributable to its recorded commit. Ignored cache files do not make the +checkout dirty. + +## Daily Runs With `tmux` + +The launcher remains a one-shot developer tool. Use a transparent shell loop +inside `tmux` when a workstation should start the latest `upstream/main` +nightly approximately every 24 hours: + +```bash +tmux new -s retriever-nightly + +export HF_TOKEN=... +export SLACK_WEBHOOK_URL=https://hooks.slack.com/services/... + +interval=86400 +while true; do + started="$(date +%s)" + ./ops/retriever-nightly/run-nightly.sh + elapsed=$(( $(date +%s) - started )) + if (( elapsed < interval )); then + sleep "$(( interval - elapsed ))" + fi +done +``` + +Enter the exports inside the new `tmux` session so they do not depend on an +older tmux server's saved environment. Detach with `Ctrl-b d`, inspect it with +`tmux attach -t retriever-nightly`, and stop it with `tmux kill-session -t +retriever-nightly`. + +The loop is serial: runs never overlap. It targets a 24-hour start-to-start +interval; if one run exceeds 24 hours, the next begins only after it finishes. +The loop survives an SSH disconnect but not a workstation reboot. This is an +operator-owned development workflow, not an installed service or timer. + +While the launcher PR is unmerged, use `run-nightly.sh --ref HEAD` in the loop +to exercise the review commit. After merge, remove `--ref HEAD`; the default +fetches and runs the latest `upstream/main` on every iteration. + +## Slack Report To enable Slack for real runs, export the incoming-webhook URL or place it in the optional mode-`600` `nightly.env`: @@ -216,8 +256,8 @@ never post. The launcher removes the URL from the benchmark child environment and exposes it only to the final Slack command. Command-line flags override values loaded from `RETRIEVER_CONFIG_FILE`; those -values override the launcher defaults. Run either launcher with `--help` for -its supported interface. +values override the launcher defaults. Run the launcher with `--help` for its +supported interface. ## Runtime Contract diff --git a/ops/retriever-nightly/SECOND_HOST_VALIDATION.md b/ops/retriever-nightly/SECOND_HOST_VALIDATION.md index 698ec8ff0e..a302196f8f 100644 --- a/ops/retriever-nightly/SECOND_HOST_VALIDATION.md +++ b/ops/retriever-nightly/SECOND_HOST_VALIDATION.md @@ -28,8 +28,10 @@ repository as `upstream` for later comparisons: git remote add upstream https://github.com/NVIDIA/NeMo-Retriever.git ``` -The launcher runs exactly the checked-out commit. It does not fetch, pull, or -switch refs. +Every validation command below passes `--ref HEAD`, so the launcher runs +exactly the checked-out review commit from an immutable detached worktree. It +does not fetch or move the review branch. The no-`--ref` production workflow +fetches `upstream/main` only after this feature is merged. ## 2. Prepare the Host @@ -60,7 +62,7 @@ Before starting GPU work, validate the configured token and read one byte from one remote parquet object in each of the queries, qrels, and corpus partitions: ```bash -./ops/retriever-nightly/run-nightly.sh --check-vidore-access +./ops/retriever-nightly/run-nightly.sh --ref HEAD --check-vidore-access ``` The command should exit zero and report access for all eight ViDoRe v3 @@ -70,7 +72,7 @@ suite if this check reports a Hugging Face or CAS redirect failure. ## 4. Preflight All Twelve Benchmarks ```bash -./ops/retriever-nightly/run-nightly.sh --dry-run +./ops/retriever-nightly/run-nightly.sh --ref HEAD --dry-run ``` The command should exit zero, report a new timestamped session directory, and @@ -82,6 +84,7 @@ the dry-run does not materialize batch data. ```bash ./ops/retriever-nightly/run-nightly.sh \ + --ref HEAD \ --no-slack \ nemo_retriever/harness/runfiles/jp20_beir.json ``` @@ -109,7 +112,7 @@ review branch commit. These steps do not post to Slack. From the clean draft checkout, start all twelve benchmarks with one command: ```bash -./ops/retriever-nightly/run-nightly.sh +./ops/retriever-nightly/run-nightly.sh --ref HEAD ``` The launcher runs the four library benchmarks and all eight ViDoRe v3 domains. @@ -119,3 +122,33 @@ running, and the parent writes one terminal `session_summary.json`. Because full `run_commit`, twelve child results, final command exit status, and Slack message. The process remains attached to the invoking shell; use the host's normal session manager if it must survive a disconnected terminal. + +## 8. Start The Post-Merge Daily Workflow + +After the launcher is merged to `upstream/main`, keep the one-shot launcher in +a transparent 24-hour loop inside `tmux`: + +```bash +tmux new -s retriever-nightly + +export HF_TOKEN=... +export SLACK_WEBHOOK_URL=https://hooks.slack.com/services/... + +interval=86400 +while true; do + started="$(date +%s)" + ./ops/retriever-nightly/run-nightly.sh + elapsed=$(( $(date +%s) - started )) + if (( elapsed < interval )); then + sleep "$(( interval - elapsed ))" + fi +done +``` + +The default command fetches the newest `upstream/main` on each iteration, +preflights ViDoRe access, runs the suite, and posts once when +`SLACK_WEBHOOK_URL` is set. Enter the exports inside the new tmux session, +detach with `Ctrl-b d`, and inspect it later with `tmux attach -t +retriever-nightly`. The serial loop does not overlap runs. It survives SSH +disconnects but must be restarted after a workstation reboot; no service or +timer is installed. diff --git a/ops/retriever-nightly/nightly.env.example b/ops/retriever-nightly/nightly.env.example index 712a576dfb..5aa5c41a70 100644 --- a/ops/retriever-nightly/nightly.env.example +++ b/ops/retriever-nightly/nightly.env.example @@ -1,9 +1,11 @@ # SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. # SPDX-License-Identifier: Apache-2.0 -# Copy to the detected private configuration path and set mode 600. On hosts -# with /raid/$USER, the launcher keeps configuration, artifacts, and managed -# latest-main checkouts there. Otherwise it uses $HOME. +# This file is optional; direct environment exports are the smallest setup. +# Operators who want persistent values may copy it to the detected private +# configuration path and set mode 600. On hosts with /raid/$USER, the launcher +# keeps configuration, artifacts, and managed Git checkouts there. Otherwise +# it uses $HOME. # # Required for authenticated ViDoRe runs when HF_TOKEN is not already exported. # A read token is sufficient: diff --git a/ops/retriever-nightly/run-latest-main.sh b/ops/retriever-nightly/run-latest-main.sh deleted file mode 100755 index b0538f0af3..0000000000 --- a/ops/retriever-nightly/run-latest-main.sh +++ /dev/null @@ -1,203 +0,0 @@ -#!/usr/bin/env bash -# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. -# SPDX-License-Identifier: Apache-2.0 - -set -uo pipefail - -umask 077 - -readonly EXIT_CONFIG=64 -readonly EXIT_FETCH=69 -readonly EXIT_ALREADY_RUNNING=75 - -log() { - printf 'retriever-nightly-latest: %s\n' "$*" >&2 -} - -usage() { - cat <<'EOF' -Usage: run-latest-main.sh [RUN-NIGHTLY-OPTION ...] - -Fetch upstream/main into an immutable worktree, preflight ViDoRe access, and -run that commit's nightly launcher. All non-help options are forwarded to -run-nightly.sh. - -Options: - --help Show this help text without fetching or requiring configuration. -EOF -} - -prune_managed_worktrees() { - local repository="$1" - local checkout_root="$2" - local selected_checkout="$3" - local keep_count="$4" - local retained=0 - local candidate base - local -a candidates=() - - mapfile -t candidates < <( - find "$checkout_root" -mindepth 1 -maxdepth 1 -type d -name 'main-*' -printf '%T@ %p\n' \ - | sort -nr \ - | sed -E 's/^[^ ]+ //' - ) - for candidate in "${candidates[@]}"; do - base="$(basename -- "$candidate")" - if [[ ! "$base" =~ ^main-[0-9a-f]{40,64}$ ]]; then - continue - fi - if [[ "$candidate" == "$selected_checkout" || $retained -lt $keep_count ]]; then - ((retained += 1)) - continue - fi - if [[ -n "$(git -C "$candidate" status --porcelain --untracked-files=normal 2>/dev/null)" ]]; then - log "retaining modified managed worktree: $candidate" - continue - fi - if ! git -C "$repository" worktree remove --force "$candidate"; then - log "could not prune managed worktree: $candidate" - fi - done - git -C "$repository" worktree prune || log "could not prune stale Git worktree metadata" -} - -main() { - local arg - for arg in "$@"; do - if [[ "$arg" == "--help" ]]; then - usage - return 0 - fi - done - - local default_nightly_root="$HOME" - local raid_nightly_root="/raid/$(id -un)" - if [[ -d "$raid_nightly_root" && -w "$raid_nightly_root" ]]; then - default_nightly_root="$raid_nightly_root" - fi - local nightly_root="${RETRIEVER_NIGHTLY_ROOT:-$default_nightly_root}" - local config_file="${RETRIEVER_CONFIG_FILE:-$nightly_root/.config/nemo-retriever/nightly/nightly.env}" - if [[ -f "$config_file" ]]; then - if [[ "$(stat -c '%a' "$config_file")" != "600" || \ - "$(stat -c '%u' "$config_file")" != "$(id -u)" ]]; then - log "nightly configuration must be owned by the invoking user with mode 600" - return "$EXIT_CONFIG" - fi - set -a - # shellcheck disable=SC1090 - source "$config_file" - set +a - fi - local slack_webhook_url="${SLACK_WEBHOOK_URL:-}" - unset SLACK_WEBHOOK_URL - - nightly_root="${RETRIEVER_NIGHTLY_ROOT:-$nightly_root}" - export RETRIEVER_CONFIG_FILE="$config_file" - export RETRIEVER_NIGHTLY_ROOT="$nightly_root" - if [[ -z "${VLLM_DEEP_GEMM_WARMUP+x}" ]]; then - export VLLM_DEEP_GEMM_WARMUP=skip - fi - - local script_dir repository source_ref source_name remote_ref fetched_ref checkout_root keep_count uv_environment - script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" - repository="${RETRIEVER_UPDATE_REPOSITORY:-$(git -C "$script_dir" rev-parse --show-toplevel)}" - source_name="${RETRIEVER_LATEST_SOURCE:-upstream}" - source_ref="${RETRIEVER_LATEST_REF:-main}" - remote_ref="$source_ref" - if [[ "$remote_ref" != refs/* ]]; then - remote_ref="refs/heads/$remote_ref" - fi - fetched_ref="refs/retriever-nightly/latest-main" - checkout_root="${RETRIEVER_LATEST_CHECKOUT_ROOT:-$nightly_root/retriever-nightly-checkouts}" - keep_count="${RETRIEVER_LATEST_KEEP_CHECKOUTS:-7}" - uv_environment="${UV_PROJECT_ENVIRONMENT:-$checkout_root/.venv}" - - if [[ ! "$keep_count" =~ ^[1-9][0-9]*$ ]]; then - log "RETRIEVER_LATEST_KEEP_CHECKOUTS must be a positive integer" - return "$EXIT_CONFIG" - fi - if ! git check-ref-format "$remote_ref"; then - log "RETRIEVER_LATEST_REF is not a valid Git ref: $source_ref" - return "$EXIT_CONFIG" - fi - if [[ ! -d "$repository/.git" && ! -f "$repository/.git" ]]; then - log "update repository is not a Git worktree: $repository" - return "$EXIT_CONFIG" - fi - if [[ -n "$(git -C "$repository" status --porcelain --untracked-files=normal)" ]]; then - log "reviewed controller checkout has local changes: $repository" - return "$EXIT_CONFIG" - fi - if ! mkdir -p "$checkout_root"; then - log "could not create latest-main checkout root: $checkout_root" - return "$EXIT_CONFIG" - fi - chmod 700 "$checkout_root" - - exec 8>"$checkout_root/.latest-main.lock" || return "$EXIT_CONFIG" - if ! flock -n 8; then - log "another latest-main selection is already running" - return "$EXIT_ALREADY_RUNNING" - fi - - log "fetching $source_name $source_ref" - if ! git -C "$repository" fetch --no-tags "$source_name" "+$remote_ref:$fetched_ref"; then - log "fetch failed; refusing to run a stale commit" - return "$EXIT_FETCH" - fi - - local target_commit selected_checkout selected_head target_launcher preflight_access run_rc - target_commit="$(git -C "$repository" rev-parse --verify "$fetched_ref^{commit}")" || return "$EXIT_FETCH" - selected_checkout="$checkout_root/main-$target_commit" - if [[ -e "$selected_checkout" ]]; then - selected_head="$(git -C "$selected_checkout" rev-parse --verify HEAD 2>/dev/null || true)" - if [[ "$selected_head" != "$target_commit" ]]; then - log "managed checkout has an unexpected commit: $selected_checkout" - return "$EXIT_CONFIG" - fi - if [[ -n "$(git -C "$selected_checkout" status --porcelain --untracked-files=normal)" ]]; then - log "managed checkout has local changes: $selected_checkout" - return "$EXIT_CONFIG" - fi - elif ! git -C "$repository" worktree add --detach "$selected_checkout" "$target_commit"; then - log "could not create detached worktree for $target_commit" - return "$EXIT_CONFIG" - fi - touch "$selected_checkout" - - target_launcher="$selected_checkout/ops/retriever-nightly/run-nightly.sh" - if [[ ! -x "$target_launcher" ]]; then - log "selected commit does not contain an executable nightly launcher: $target_launcher" - return "$EXIT_CONFIG" - fi - - export RETRIEVER_SELECTED_CHECKOUT="$selected_checkout" - export UV_PROJECT_ENVIRONMENT="$uv_environment" - log "selected $source_name/$source_ref commit $target_commit in $selected_checkout" - - preflight_access=1 - for arg in "$@"; do - if [[ "$arg" == "--dry-run" || "$arg" == "--check-vidore-access" || "$arg" == "--help" ]]; then - preflight_access=0 - fi - done - - run_rc=0 - if [[ "$preflight_access" == "1" ]]; then - "$target_launcher" --check-vidore-access || run_rc=$? - fi - if ((run_rc == 0)); then - if [[ -n "$slack_webhook_url" ]]; then - export SLACK_WEBHOOK_URL="$slack_webhook_url" - fi - "$target_launcher" "$@" || run_rc=$? - unset SLACK_WEBHOOK_URL - else - log "ViDoRe access preflight failed; skipping GPU work" - fi - - prune_managed_worktrees "$repository" "$checkout_root" "$selected_checkout" "$keep_count" - return "$run_rc" -} - -main "$@" diff --git a/ops/retriever-nightly/run-nightly.sh b/ops/retriever-nightly/run-nightly.sh index bb86756ff2..3b2a9b3c07 100755 --- a/ops/retriever-nightly/run-nightly.sh +++ b/ops/retriever-nightly/run-nightly.sh @@ -7,6 +7,7 @@ set -uo pipefail umask 077 readonly EXIT_CONFIG=64 +readonly EXIT_FETCH=69 readonly EXIT_INTERNAL=70 readonly EXIT_CANNOT_CREATE=73 readonly EXIT_ALREADY_RUNNING=75 @@ -19,10 +20,12 @@ usage() { cat <<'EOF' Usage: run-nightly.sh [OPTIONS] [--] [RUNFILE ...] -Run the library and ViDoRe v3 harness suite from the current clean checkout. -With no RUNFILE arguments, all 12 checked-in nightly runfiles are executed. +Fetch and run the library and ViDoRe v3 harness suite from the latest +upstream/main commit. With no RUNFILE arguments, all 12 checked-in nightly +runfiles are executed. Options: + --ref REF Run an existing local Git ref instead of fetching main. --dataset-paths PATH Machine-local dataset path map. --artifact-root PATH Parent directory for timestamped session artifacts. --check-vidore-access Validate authenticated ViDoRe evaluation-data access and exit. @@ -36,6 +39,171 @@ absolute_path() { realpath -m -- "$1" } +prune_managed_worktrees() { + local repository="$1" + local checkout_root="$2" + local selected_checkout="$3" + local keep_count="$4" + local retained=0 + local candidate base + local -a candidates=() + + mapfile -t candidates < <( + find "$checkout_root" -mindepth 1 -maxdepth 1 -type d -name 'commit-*' -printf '%T@ %p\n' \ + | sort -nr \ + | sed -E 's/^[^ ]+ //' + ) + for candidate in "${candidates[@]}"; do + base="$(basename -- "$candidate")" + if [[ ! "$base" =~ ^commit-[0-9a-f]{40,64}$ ]]; then + continue + fi + if [[ "$candidate" == "$selected_checkout" || $retained -lt $keep_count ]]; then + ((retained += 1)) + continue + fi + if [[ -n "$(git -C "$candidate" status --porcelain --untracked-files=normal 2>/dev/null)" ]]; then + log "retaining modified managed worktree: $candidate" + continue + fi + if ! git -C "$repository" worktree remove --force "$candidate"; then + log "could not prune managed worktree: $candidate" + fi + done + git -C "$repository" worktree prune || log "could not prune stale Git worktree metadata" +} + +select_checkout_and_run() { + local requested_ref="$1" + shift + + local repository source_name source_ref remote_ref fetched_ref checkout_root keep_count uv_environment + repository="${RETRIEVER_UPDATE_REPOSITORY:-$(git -C "$script_dir" rev-parse --show-toplevel)}" + source_name="${RETRIEVER_LATEST_SOURCE:-upstream}" + source_ref="${RETRIEVER_LATEST_REF:-main}" + remote_ref="$source_ref" + if [[ "$remote_ref" != refs/* ]]; then + remote_ref="refs/heads/$remote_ref" + fi + fetched_ref="refs/retriever-nightly/latest-main" + checkout_root="${RETRIEVER_LATEST_CHECKOUT_ROOT:-$nightly_root/retriever-nightly-checkouts}" + keep_count="${RETRIEVER_LATEST_KEEP_CHECKOUTS:-7}" + uv_environment="${UV_PROJECT_ENVIRONMENT:-$checkout_root/.venv}" + + if [[ ! "$keep_count" =~ ^[1-9][0-9]*$ ]]; then + log "RETRIEVER_LATEST_KEEP_CHECKOUTS must be a positive integer" + return "$EXIT_CONFIG" + fi + if [[ ! -d "$repository/.git" && ! -f "$repository/.git" ]]; then + log "controller checkout is not a Git worktree: $repository" + return "$EXIT_CONFIG" + fi + if [[ -n "$(git -C "$repository" status --porcelain --untracked-files=normal)" ]]; then + log "controller checkout has local changes: $repository" + return "$EXIT_CONFIG" + fi + if ! mkdir -p "$checkout_root"; then + log "could not create managed checkout root: $checkout_root" + return "$EXIT_CONFIG" + fi + chmod 700 "$checkout_root" + + exec 8>"$checkout_root/.selection.lock" || return "$EXIT_CONFIG" + if ! flock -n 8; then + log "another nightly Git selection is already running" + return "$EXIT_ALREADY_RUNNING" + fi + + local target_commit selection_label + if [[ -z "$requested_ref" ]]; then + if ! git check-ref-format "$remote_ref"; then + log "RETRIEVER_LATEST_REF is not a valid Git ref: $source_ref" + return "$EXIT_CONFIG" + fi + log "fetching $source_name $source_ref" + if ! git -C "$repository" fetch --no-tags "$source_name" "+$remote_ref:$fetched_ref"; then + log "fetch failed; refusing to run a stale commit" + return "$EXIT_FETCH" + fi + target_commit="$(git -C "$repository" rev-parse --verify "$fetched_ref^{commit}")" || \ + return "$EXIT_FETCH" + selection_label="$source_name/$source_ref" + else + if [[ "$requested_ref" == -* ]]; then + log "--ref must name a Git ref or commit" + return "$EXIT_CONFIG" + fi + target_commit="$(git -C "$repository" rev-parse --verify "$requested_ref^{commit}" 2>/dev/null)" || { + log "could not resolve --ref $requested_ref to a local commit" + return "$EXIT_CONFIG" + } + selection_label="$requested_ref" + fi + + local selected_checkout selected_head target_launcher preflight_access run_rc + selected_checkout="$checkout_root/commit-$target_commit" + if [[ -e "$selected_checkout" ]]; then + selected_head="$(git -C "$selected_checkout" rev-parse --verify HEAD 2>/dev/null || true)" + if [[ "$selected_head" != "$target_commit" ]]; then + log "managed checkout has an unexpected commit: $selected_checkout" + return "$EXIT_CONFIG" + fi + if [[ -n "$(git -C "$selected_checkout" status --porcelain --untracked-files=normal)" ]]; then + log "managed checkout has local changes: $selected_checkout" + return "$EXIT_CONFIG" + fi + elif ! git -C "$repository" worktree add --detach "$selected_checkout" "$target_commit"; then + log "could not create detached worktree for $target_commit" + return "$EXIT_CONFIG" + fi + touch "$selected_checkout" + + target_launcher="$selected_checkout/ops/retriever-nightly/run-nightly.sh" + if [[ ! -x "$target_launcher" ]]; then + log "selected commit does not contain an executable nightly launcher: $target_launcher" + return "$EXIT_CONFIG" + fi + + export RETRIEVER_SELECTED_CHECKOUT="$selected_checkout" + export UV_PROJECT_ENVIRONMENT="$uv_environment" + log "selected $selection_label commit $target_commit in $selected_checkout" + + preflight_access=1 + local arg + for arg in "$@"; do + if [[ "$arg" == "--dry-run" || "$arg" == "--check-vidore-access" ]]; then + preflight_access=0 + fi + done + + run_rc=0 + if [[ "$preflight_access" == "1" ]]; then + "$target_launcher" --check-vidore-access || run_rc=$? + fi + if ((run_rc == 0)); then + if [[ -n "$slack_webhook_url" ]]; then + export SLACK_WEBHOOK_URL="$slack_webhook_url" + fi + "$target_launcher" "$@" || run_rc=$? + unset SLACK_WEBHOOK_URL + else + log "ViDoRe access preflight failed; skipping GPU work" + fi + + prune_managed_worktrees "$repository" "$checkout_root" "$selected_checkout" "$keep_count" + return "$run_rc" +} + +for argument in "$@"; do + if [[ "$argument" == "--" ]]; then + break + fi + if [[ "$argument" == "--help" ]]; then + usage + exit 0 + fi +done + default_nightly_root="$HOME" raid_nightly_root="/raid/$(id -un)" if [[ -d "$raid_nightly_root" && -w "$raid_nightly_root" ]]; then @@ -57,6 +225,42 @@ slack_webhook_url="${SLACK_WEBHOOK_URL:-}" unset SLACK_WEBHOOK_URL readonly script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +if [[ -z "${RETRIEVER_SELECTED_CHECKOUT:-}" ]]; then + requested_ref="" + forwarded_args=() + while (($#)); do + case "$1" in + --ref) + if (($# < 2)); then + log "--ref requires a Git ref or commit" + exit "$EXIT_CONFIG" + fi + if [[ -n "$requested_ref" ]]; then + log "--ref may be specified only once" + exit "$EXIT_CONFIG" + fi + requested_ref="$2" + shift 2 + ;; + --) + forwarded_args+=("$@") + break + ;; + *) + forwarded_args+=("$1") + shift + ;; + esac + done + export RETRIEVER_CONFIG_FILE="$config_file" + export RETRIEVER_NIGHTLY_ROOT="$nightly_root" + if [[ -z "${VLLM_DEEP_GEMM_WARMUP+x}" ]]; then + export VLLM_DEEP_GEMM_WARMUP=skip + fi + select_checkout_and_run "$requested_ref" "${forwarded_args[@]}" + exit $? +fi + checkout="${RETRIEVER_SELECTED_CHECKOUT:-${RETRIEVER_CHECKOUT:-$(git -C "$script_dir" rev-parse --show-toplevel)}}" nightly_root="${RETRIEVER_NIGHTLY_ROOT:-$nightly_root}" artifact_root="${RETRIEVER_ARTIFACT_ROOT:-$nightly_root/retriever-nightly-artifacts}" From 98804c46652f16418a0b1cca8097f8eecc3c2647 Mon Sep 17 00:00:00 2001 From: Jacob Ioffe Date: Thu, 16 Jul 2026 14:56:34 +0000 Subject: [PATCH 05/11] test(ops): scope nightly launcher tests to source checkouts Signed-off-by: Jacob Ioffe --- nemo_retriever/tests/test_harness_nightly_git_selection.py | 4 ++++ nemo_retriever/tests/test_harness_nightly_launcher.py | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/nemo_retriever/tests/test_harness_nightly_git_selection.py b/nemo_retriever/tests/test_harness_nightly_git_selection.py index b3d1fba447..6647b9705d 100644 --- a/nemo_retriever/tests/test_harness_nightly_git_selection.py +++ b/nemo_retriever/tests/test_harness_nightly_git_selection.py @@ -12,6 +12,10 @@ REPO_ROOT = Path(__file__).resolve().parents[2] NIGHTLY_LAUNCHER = REPO_ROOT / "ops" / "retriever-nightly" / "run-nightly.sh" +pytestmark = pytest.mark.skipif( + not (REPO_ROOT / ".git").exists(), + reason="nightly launcher tests require a full source checkout", +) SLACK_WEBHOOK_URL = "https://hooks.slack.com/services/test/webhook/value" diff --git a/nemo_retriever/tests/test_harness_nightly_launcher.py b/nemo_retriever/tests/test_harness_nightly_launcher.py index 3efe16e75b..5b60640786 100644 --- a/nemo_retriever/tests/test_harness_nightly_launcher.py +++ b/nemo_retriever/tests/test_harness_nightly_launcher.py @@ -15,6 +15,10 @@ REPO_ROOT = Path(__file__).resolve().parents[2] LAUNCHER = REPO_ROOT / "ops" / "retriever-nightly" / "run-nightly.sh" +pytestmark = pytest.mark.skipif( + not (REPO_ROOT / ".git").exists(), + reason="nightly launcher tests require a full source checkout", +) DEFAULT_RUNFILES = ( "nemo_retriever/harness/runfiles/jp20_beir.json", "nemo_retriever/harness/runfiles/bo767_beir.json", From c866357427810ad4d815309457cd40d243017c79 Mon Sep 17 00:00:00 2001 From: Jacob Ioffe Date: Thu, 16 Jul 2026 21:51:43 +0000 Subject: [PATCH 06/11] fix(ops): run nightly from current checkout by default Signed-off-by: Jacob Ioffe --- .../src/nemo_retriever/harness/artifacts.py | 19 +++ .../src/nemo_retriever/harness/runsets.py | 7 +- .../tests/test_harness_artifacts.py | 16 ++ .../test_harness_nightly_git_selection.py | 50 ++++-- .../tests/test_harness_nightly_launcher.py | 21 +++ nemo_retriever/tests/test_harness_runfiles.py | 3 + ops/retriever-nightly/README.md | 121 +++++++------- .../SECOND_HOST_VALIDATION.md | 19 ++- ops/retriever-nightly/run-nightly.sh | 152 +++++++++++------- 9 files changed, 263 insertions(+), 145 deletions(-) diff --git a/nemo_retriever/src/nemo_retriever/harness/artifacts.py b/nemo_retriever/src/nemo_retriever/harness/artifacts.py index 5416f6e7f5..d4712a2235 100644 --- a/nemo_retriever/src/nemo_retriever/harness/artifacts.py +++ b/nemo_retriever/src/nemo_retriever/harness/artifacts.py @@ -118,6 +118,25 @@ def last_commit() -> str: return "unknown" +def working_tree_dirty() -> bool | None: + """Return whether the source checkout is modified, or ``None`` outside Git.""" + + repo_root = NEMO_RETRIEVER_ROOT.parent + try: + result = subprocess.run( + ["git", "status", "--porcelain", "--untracked-files=normal", "--ignore-submodules"], + cwd=str(repo_root), + check=False, + capture_output=True, + text=True, + ) + except Exception: + return None + if result.returncode != 0: + return None + return bool(result.stdout.strip()) + + def get_artifacts_root(base_dir: str | None = None) -> Path: if base_dir: return Path(base_dir).expanduser().resolve() diff --git a/nemo_retriever/src/nemo_retriever/harness/runsets.py b/nemo_retriever/src/nemo_retriever/harness/runsets.py index a05c3370c2..0ee7b51095 100644 --- a/nemo_retriever/src/nemo_retriever/harness/runsets.py +++ b/nemo_retriever/src/nemo_retriever/harness/runsets.py @@ -12,7 +12,7 @@ from typing import Any, Mapping, Sequence from nemo_retriever.harness.artifact_writer import redact -from nemo_retriever.harness.artifacts import get_artifacts_root, last_commit, now_timestr +from nemo_retriever.harness.artifacts import get_artifacts_root, last_commit, now_timestr, working_tree_dirty from nemo_retriever.harness.benchmark_registry import get_benchmark, get_runset, runset_names from nemo_retriever.harness.contracts import ( EXIT_ARTIFACT_WRITE_FAILURE, @@ -436,7 +436,7 @@ def run_runset( expanded_payload={"runset": spec.to_dict(), "runs": expanded_runs}, run_commit=last_commit(), dry_run=dry_run, - summary_extra={"runset": spec.name}, + summary_extra={"runset": spec.name, "working_tree_dirty": working_tree_dirty()}, ) @@ -458,6 +458,7 @@ def run_runfiles( session_name = _validate_session_label(session_name, field="--session-name") run_commit = last_commit() + source_worktree_dirty = working_tree_dirty() local_dataset_paths = load_dataset_paths(dataset_paths_file) requests = [load_runfile(path) for path in runfiles] runs: list[PreparedRun] = [] @@ -520,6 +521,7 @@ def run_runfiles( expanded_payload={ "session_name": session_name, "run_commit": run_commit, + "working_tree_dirty": source_worktree_dirty, "dataset_paths_file": dataset_paths_value, "isolate_runs": bool(isolate_runs), "runfiles": expanded_runs, @@ -530,6 +532,7 @@ def run_runfiles( "session_name": session_name, "dataset_paths_file": dataset_paths_value, "isolate_runs": bool(isolate_runs), + "working_tree_dirty": source_worktree_dirty, }, isolate_runs=isolate_runs, ) diff --git a/nemo_retriever/tests/test_harness_artifacts.py b/nemo_retriever/tests/test_harness_artifacts.py index acd5e97c80..6641bfd02c 100644 --- a/nemo_retriever/tests/test_harness_artifacts.py +++ b/nemo_retriever/tests/test_harness_artifacts.py @@ -45,6 +45,22 @@ def test_last_commit_preserves_the_full_git_sha(monkeypatch): assert artifacts.last_commit() == full_sha +@pytest.mark.parametrize( + ("returncode", "stdout", "expected"), + [(0, "", False), (0, " M local.py\n", True), (128, "", None)], +) +def test_working_tree_dirty_reports_git_state(monkeypatch, returncode, stdout, expected): + import nemo_retriever.harness.artifacts as artifacts + + monkeypatch.setattr( + artifacts.subprocess, + "run", + lambda *args, **kwargs: subprocess.CompletedProcess(args[0], returncode, stdout=stdout, stderr=""), + ) + + assert artifacts.working_tree_dirty() is expected + + def test_artifact_manifest_uses_relative_paths_and_includes_lancedb(tmp_path): writer = ArtifactWriter(artifact_dir=tmp_path, run_id="run-1", benchmark="jp20_beir") writer.status(status="running", phase="ingest") diff --git a/nemo_retriever/tests/test_harness_nightly_git_selection.py b/nemo_retriever/tests/test_harness_nightly_git_selection.py index 6647b9705d..2b3da91700 100644 --- a/nemo_retriever/tests/test_harness_nightly_git_selection.py +++ b/nemo_retriever/tests/test_harness_nightly_git_selection.py @@ -80,6 +80,7 @@ def latest_main_fixture(tmp_path: Path): controller = tmp_path / "controller" subprocess.run(["git", "clone", "-q", str(source), str(controller)], check=True) + _git(controller, "remote", "add", "upstream", str(source)) (source / "version.txt").write_text("latest\n", encoding="utf-8") latest_commit = _commit(source, "latest") @@ -98,8 +99,6 @@ def latest_main_fixture(tmp_path: Path): "HOME": str(tmp_path / "home"), "RETRIEVER_CONFIG_FILE": str(config_file), "RETRIEVER_UPDATE_REPOSITORY": str(controller), - "RETRIEVER_LATEST_SOURCE": str(source), - "RETRIEVER_LATEST_REF": "main", "RETRIEVER_LATEST_CHECKOUT_ROOT": str(checkout_root), "RETRIEVER_LATEST_KEEP_CHECKOUTS": "2", "FAKE_LATEST_CALLS": str(calls_path), @@ -124,11 +123,24 @@ def calls() -> list[tuple[str, str, str]]: return run, calls, source, controller, checkout_root, initial_commit, latest_commit -def test_launcher_fetches_latest_main_into_immutable_worktree(latest_main_fixture) -> None: - run, calls, _source, controller, checkout_root, initial_commit, latest_commit = latest_main_fixture +def test_launcher_runs_current_checkout_without_fetching(latest_main_fixture) -> None: + run, calls, _source, controller, _checkout_root, initial_commit, _latest_commit = latest_main_fixture result = run() + assert result.returncode == 0, result.stderr + assert _git(controller, "rev-parse", "HEAD").stdout.strip() == initial_commit + assert calls() == [ + (initial_commit, "--check-vidore-access", ""), + (initial_commit, "", ""), + ] + + +def test_remote_ref_fetches_latest_commit_into_immutable_worktree(latest_main_fixture) -> None: + run, calls, _source, controller, checkout_root, initial_commit, latest_commit = latest_main_fixture + + result = run("--ref", "upstream/main") + assert result.returncode == 0, result.stderr assert _git(controller, "rev-parse", "HEAD").stdout.strip() == initial_commit assert calls() == [ @@ -154,7 +166,7 @@ def test_help_does_not_require_configuration_or_fetch(tmp_path: Path) -> None: ) assert result.returncode == 0, result.stderr - assert "latest" in result.stdout + assert "current checkout" in result.stdout assert "--ref REF" in result.stdout @@ -173,7 +185,7 @@ def test_explicit_ref_runs_local_commit_without_fetching(latest_main_fixture) -> def test_launcher_selection_uses_exported_secrets_without_config_file(latest_main_fixture, tmp_path: Path) -> None: - run, calls, _source, _controller, _checkout_root, _initial_commit, latest_commit = latest_main_fixture + run, calls, _source, _controller, _checkout_root, initial_commit, _latest_commit = latest_main_fixture result = run( "--dry-run", @@ -187,7 +199,7 @@ def test_launcher_selection_uses_exported_secrets_without_config_file(latest_mai ) assert result.returncode == 0, result.stderr - assert calls() == [(latest_commit, "--dry-run", str(_checkout_root / ".venv"))] + assert calls() == [(initial_commit, "--dry-run", "")] @pytest.mark.parametrize("configured, expected", [(None, "skip"), ("full", "full")]) @@ -205,40 +217,44 @@ def test_selected_run_has_safe_warmup_default_and_allows_override( def test_launcher_fails_closed_when_access_preflight_fails(latest_main_fixture) -> None: - run, calls, _source, _controller, _checkout_root, _initial_commit, latest_commit = latest_main_fixture + run, calls, _source, _controller, _checkout_root, initial_commit, _latest_commit = latest_main_fixture result = run(extra_env={"FAKE_ACCESS_RC": "3"}) assert result.returncode == 3 - assert calls() == [(latest_commit, "--check-vidore-access", str(_checkout_root / ".venv"))] + assert calls() == [(initial_commit, "--check-vidore-access", "")] def test_launcher_does_not_run_stale_commit_when_fetch_fails(latest_main_fixture) -> None: - run, calls, _source, _controller, _checkout_root, _initial_commit, _latest_commit = latest_main_fixture + run, calls, _source, controller, checkout_root, _initial_commit, _latest_commit = latest_main_fixture + _git(controller, "remote", "set-url", "upstream", str(checkout_root / "missing-source")) - result = run(extra_env={"RETRIEVER_LATEST_SOURCE": str(_checkout_root / "missing-source")}) + result = run("--ref", "upstream/main") assert result.returncode != 0 assert calls() == [] -def test_launcher_rejects_modified_controller(latest_main_fixture) -> None: - run, calls, _source, controller, _checkout_root, _initial_commit, _latest_commit = latest_main_fixture +def test_launcher_runs_modified_current_checkout(latest_main_fixture) -> None: + run, calls, _source, controller, _checkout_root, initial_commit, _latest_commit = latest_main_fixture (controller / "version.txt").write_text("modified\n", encoding="utf-8") result = run() - assert result.returncode == 64 - assert calls() == [] + assert result.returncode == 0, result.stderr + assert calls() == [ + (initial_commit, "--check-vidore-access", ""), + (initial_commit, "", ""), + ] def test_launcher_prunes_only_old_managed_worktrees(latest_main_fixture) -> None: run, _calls, source, _controller, checkout_root, _initial_commit, latest_commit = latest_main_fixture - assert run(extra_env={"RETRIEVER_LATEST_KEEP_CHECKOUTS": "1"}).returncode == 0 + assert run("--ref", "upstream/main", extra_env={"RETRIEVER_LATEST_KEEP_CHECKOUTS": "1"}).returncode == 0 (source / "version.txt").write_text("newer\n", encoding="utf-8") newer_commit = _commit(source, "newer") - result = run(extra_env={"RETRIEVER_LATEST_KEEP_CHECKOUTS": "1"}) + result = run("--ref", "upstream/main", extra_env={"RETRIEVER_LATEST_KEEP_CHECKOUTS": "1"}) assert result.returncode == 0, result.stderr assert not (checkout_root / f"commit-{latest_commit}").exists() diff --git a/nemo_retriever/tests/test_harness_nightly_launcher.py b/nemo_retriever/tests/test_harness_nightly_launcher.py index 5b60640786..8ae19b4b84 100644 --- a/nemo_retriever/tests/test_harness_nightly_launcher.py +++ b/nemo_retriever/tests/test_harness_nightly_launcher.py @@ -152,6 +152,27 @@ def test_launcher_rejects_untracked_checkout_files(nightly_launcher, tmp_path: P assert "untracked changes" in result.stderr +def test_current_checkout_allows_and_labels_local_changes(nightly_launcher, tmp_path: Path) -> None: + run, calls = nightly_launcher + (tmp_path / "checkout" / "untracked.py").write_text("print('changed')\n", encoding="utf-8") + + result = run( + extra_env={ + "RETRIEVER_ALLOW_DIRTY_CHECKOUT": "1", + "SLACK_WEBHOOK_URL": SLACK_WEBHOOK_URL, + } + ) + + assert result.returncode == 0, result.stderr + assert "WARNING: running the current checkout with local changes" in result.stderr + run_invocation, post_invocation = calls() + session_dir = Path(run_invocation[run_invocation.index("--output-dir") + 1]) + provenance = (session_dir / "source_worktree_status.txt").read_text(encoding="utf-8") + assert "working_tree_dirty=true" in provenance + assert "?? untracked.py" in provenance + assert post_invocation[post_invocation.index("--title") + 1].startswith("[LOCAL CHANGES]") + + @pytest.mark.parametrize("configured, expected", [(None, "skip"), ("full", "full")]) def test_deep_gemm_warmup_has_safe_default_and_allows_override( nightly_launcher, configured: str | None, expected: str diff --git a/nemo_retriever/tests/test_harness_runfiles.py b/nemo_retriever/tests/test_harness_runfiles.py index d8fe85fb74..ddfc9ae453 100644 --- a/nemo_retriever/tests/test_harness_runfiles.py +++ b/nemo_retriever/tests/test_harness_runfiles.py @@ -74,6 +74,7 @@ def fake_run_benchmark(prepared, **kwargs): return _successful_outcome(prepared.benchmark, kwargs["output_dir"]) monkeypatch.setattr("nemo_retriever.harness.runsets.run_prepared_benchmark", fake_run_benchmark) + monkeypatch.setattr("nemo_retriever.harness.runsets.working_tree_dirty", lambda: True) outcome = run_runfiles( [runfile], @@ -96,12 +97,14 @@ def fake_run_benchmark(prepared, **kwargs): "query.top_k=20", ) assert outcome.results["session_name"] == "library_beir" + assert outcome.results["working_tree_dirty"] is True assert outcome.results["runs"][0]["dataset"] == "jp20" assert outcome.results["runs"][0]["artifact_dir"] == "001_jp20_beir" assert outcome.results["runs"][0]["results_path"] == "001_jp20_beir/results.json" assert "results" not in outcome.results expanded = json.loads((outcome.artifact_dir / "expanded_runs.json").read_text(encoding="utf-8")) + assert expanded["working_tree_dirty"] is True assert expanded["runfiles"][0]["dataset_paths"] == { "path": str(documents), "query_file": str(query_file), diff --git a/ops/retriever-nightly/README.md b/ops/retriever-nightly/README.md index 3c6190303d..f83999175a 100644 --- a/ops/retriever-nightly/README.md +++ b/ops/retriever-nightly/README.md @@ -8,22 +8,23 @@ benchmark suite: | Workflow | Command | Code that runs | | --- | --- | --- | -| Regular nightly | `run-nightly.sh` | Freshly fetched `upstream/main`. | -| Review the checked-out PR | `run-nightly.sh --ref HEAD` | The current checkout's committed `HEAD`. | -| Reproduce an exact revision | `run-nightly.sh --ref ` | An already available local Git commit. | - -The launcher resolves one commit, creates or reuses an immutable detached -worktree, and exits after one terminal session summary. Recurrence is -deliberately kept outside its interface; the [daily `tmux` -workflow](#daily-runs-with-tmux) is a small shell loop rather than an installed -scheduler. The launcher never merges into or moves the controller checkout, -and it does not distribute datasets. +| Test the current checkout | `run-nightly.sh` | The current branch, including local changes. | +| Regular latest-main nightly | `run-nightly.sh --ref upstream/main` | Freshly fetched `upstream/main` in a clean worktree. | +| Reproduce an exact revision | `run-nightly.sh --ref ` | A clean worktree at an available local commit. | + +With no `--ref`, the launcher runs the checkout that contains the script and +does not fetch, switch branches, or reject local changes. With `--ref`, it +resolves one commit and creates or reuses an immutable detached worktree. It +exits after one terminal session summary. Recurrence is deliberately kept +outside its interface; the [daily `tmux` workflow](#daily-runs-with-tmux) is a +small shell loop rather than an installed scheduler. The launcher never merges +into or moves the invoking checkout, and it does not distribute datasets. ## Quick Start On A Standard Host -After this launcher is present on `upstream/main`, a workstation with -`/datasets/nv-ingest` and writable `/raid/$USER` needs only a Hugging Face token -and, optionally, a Slack webhook: +A workstation with `/datasets/nv-ingest` and writable `/raid/$USER` needs only +a Hugging Face token and, optionally, a Slack webhook to run its current +checkout: ```bash export HF_TOKEN=... @@ -31,20 +32,19 @@ export SLACK_WEBHOOK_URL=https://hooks.slack.com/services/... ./ops/retriever-nightly/run-nightly.sh ``` -That foreground command runs all twelve benchmarks and posts one terminal -Slack summary. `SLACK_WEBHOOK_URL` is optional; omit it to run without posting. -Before GPU work, it fetches `upstream/main`, selects its exact commit, and -checks ViDoRe access. It then uses the checked-in `/datasets/nv-ingest` map, -writes artifacts under `/raid/$USER/retriever-nightly-artifacts`, and prints the -terminal session directory. No model-provider API key, `nightly.env`, `sudo`, -systemd service, or timer is required. +That foreground command runs the current branch exactly as it exists, all +twelve benchmarks, and one terminal Slack summary. `SLACK_WEBHOOK_URL` is +optional; omit it to run without posting. Before GPU work, it checks ViDoRe +access. It then uses the checked-in `/datasets/nv-ingest` map, writes artifacts +under `/raid/$USER/retriever-nightly-artifacts`, and prints the terminal session +directory. No model-provider API key, `nightly.env`, `sudo`, systemd service, +or timer is required. The only required operator input on that host is `HF_TOKEN`. Set `SLACK_WEBHOOK_URL` when the terminal summary should post to Slack. A custom dataset map or artifact root is needed only when the host does not have the -standard paths. While reviewing this unmerged PR, add `--ref HEAD` to every -launcher command so the review commit runs instead of the current upstream -`main`. +standard paths. Use `--ref upstream/main` only when the operator deliberately +wants the newest clean upstream commit instead of the checked-out code. ## Validate The Current PR Checkout @@ -52,7 +52,7 @@ launcher command so the review commit runs instead of the current upstream The supported v1 host is a Linux NVIDIA workstation with: -- a clean NeMo Retriever Git checkout; +- a NeMo Retriever Git checkout; - `git`, Bash, `uv`, `flock`, `realpath`, and NVIDIA drivers available; - access to the twelve benchmark datasets through `/datasets` or other local paths; @@ -113,7 +113,7 @@ Verify the token and read one byte from one remote parquet object in every ViDoRe evaluation partition before starting GPU work: ```bash -./ops/retriever-nightly/run-nightly.sh --ref HEAD --check-vidore-access +./ops/retriever-nightly/run-nightly.sh --check-vidore-access ``` The access check does not download full parquet objects. A redirect failure @@ -124,7 +124,7 @@ Then preflight the complete twelve-benchmark suite without starting ingest or query: ```bash -./ops/retriever-nightly/run-nightly.sh --ref HEAD --dry-run +./ops/retriever-nightly/run-nightly.sh --dry-run ``` Inspect the resulting `session_summary.json` and child plans. Dry-runs never @@ -136,15 +136,14 @@ Use one positional runfile for a smaller real canary before the full run: ```bash ./ops/retriever-nightly/run-nightly.sh \ - --ref HEAD \ --no-slack \ nemo_retriever/harness/runfiles/jp20_beir.json ``` -Run the complete suite from the review commit with no positional runfiles: +Run the complete suite from the current checkout with no positional runfiles: ```bash -./ops/retriever-nightly/run-nightly.sh --ref HEAD +./ops/retriever-nightly/run-nightly.sh ``` If `SLACK_WEBHOOK_URL` is configured, that real run posts its terminal summary. @@ -155,18 +154,24 @@ not post. With no `--ref`, `run-nightly.sh`: -1. fetches `main` from the `upstream` remote; -2. resolves the fetched commit before doing any GPU work; -3. creates or reuses an immutable detached worktree named `commit-`; -4. runs the ViDoRe access check from that selected commit; and -5. invokes that commit's nightly execution path only when fetch and access - preflight both succeed. - -A fetch failure is fail-closed: the controller does not fall back to yesterday's -commit. It never runs `git pull`, merges into the controller checkout, or moves -the reviewed controller branch. Session artifacts still record the exact -selected SHA in `run_commit`. The fetched source is also recorded locally at -`refs/retriever-nightly/latest-main` for inspection. +1. selects the Git checkout that contains the launcher; +2. runs its current branch and working tree without fetching or switching; +3. permits tracked, staged, and untracked changes; and +4. runs the ViDoRe access check before real GPU work. + +Every session records the checkout's HEAD in `run_commit` and whether it had +local changes in `working_tree_dirty`. Dirty runs also write +`source_worktree_status.txt` in the session directory and prefix the Slack +title with `[LOCAL CHANGES]`. The status artifact records paths and Git state, +not file contents, so a dirty run is intentionally identifiable but not fully +reproducible. + +`--ref REF` requests a clean committed run. A local branch, tag, or SHA is +resolved without fetching. A remote branch such as `upstream/main` is fetched +first, then resolved fail-closed; a fetch failure never falls back to a stale +remote-tracking commit. The selected commit runs from an immutable detached +worktree named `commit-`. The launcher never runs `git pull`, merges +into the invoking checkout, or moves its current branch. Immutable worktrees and one shared `uv` project environment live under the detected nightly root at `retriever-nightly-checkouts`; on `/raid` hosts this is @@ -174,32 +179,28 @@ detected nightly root at `retriever-nightly-checkouts`; on `/raid` hosts this is worktrees are retained. Modified managed worktrees are never deleted automatically. -The one-time controller checkout setup must provide an `upstream` remote. -Operators do not choose a branch or commit for normal nightlies after that: +The one-time latest-main setup must provide an `upstream` remote. Request a +clean latest-main preflight explicitly: ```bash git remote get-url upstream >/dev/null 2>&1 || \ git remote add upstream https://github.com/NVIDIA/NeMo-Retriever.git -./ops/retriever-nightly/run-nightly.sh --dry-run +./ops/retriever-nightly/run-nightly.sh --ref upstream/main --dry-run ``` The dry-run fetches and selects the latest commit but skips remote access and -GPU execution. Use `run-nightly.sh --check-vidore-access` to validate the -selected latest-main commit and its machine credentials without starting a -session. Once that selected main commit contains this launcher, the complete -latest-main suite remains the same one command: +GPU execution. Use `run-nightly.sh --ref upstream/main --check-vidore-access` +to validate that commit and the machine credentials without starting a +session. The complete latest-main suite is: ```bash -./ops/retriever-nightly/run-nightly.sh +./ops/retriever-nightly/run-nightly.sh --ref upstream/main ``` -`--ref REF` skips the fetch and resolves an existing local Git ref or commit. -The selected commit still runs from a managed detached worktree, so current -branch state is never moved. Use `--ref HEAD` for this draft PR and `--ref -` to reproduce an earlier run. The controller checkout must be clean; -tracked, staged, and untracked changes are rejected so every result is -attributable to its recorded commit. Ignored cache files do not make the -checkout dirty. +Use `--ref HEAD` when local changes should be ignored and only the current +commit should run, or `--ref ` to reproduce an earlier run. The +invoking checkout may itself be dirty because the selected ref always runs in +a separate clean worktree. Ignored cache files do not mark a run dirty. ## Daily Runs With `tmux` @@ -216,7 +217,7 @@ export SLACK_WEBHOOK_URL=https://hooks.slack.com/services/... interval=86400 while true; do started="$(date +%s)" - ./ops/retriever-nightly/run-nightly.sh + ./ops/retriever-nightly/run-nightly.sh --ref upstream/main elapsed=$(( $(date +%s) - started )) if (( elapsed < interval )); then sleep "$(( interval - elapsed ))" @@ -234,9 +235,9 @@ interval; if one run exceeds 24 hours, the next begins only after it finishes. The loop survives an SSH disconnect but not a workstation reboot. This is an operator-owned development workflow, not an installed service or timer. -While the launcher PR is unmerged, use `run-nightly.sh --ref HEAD` in the loop -to exercise the review commit. After merge, remove `--ref HEAD`; the default -fetches and runs the latest `upstream/main` on every iteration. +While testing an unmerged branch, omit `--ref upstream/main` to run that +checkout on every iteration. Keep `--ref upstream/main` for the production +loop so every iteration fetches and runs the newest clean upstream commit. ## Slack Report diff --git a/ops/retriever-nightly/SECOND_HOST_VALIDATION.md b/ops/retriever-nightly/SECOND_HOST_VALIDATION.md index a302196f8f..ef9657879b 100644 --- a/ops/retriever-nightly/SECOND_HOST_VALIDATION.md +++ b/ops/retriever-nightly/SECOND_HOST_VALIDATION.md @@ -28,10 +28,10 @@ repository as `upstream` for later comparisons: git remote add upstream https://github.com/NVIDIA/NeMo-Retriever.git ``` -Every validation command below passes `--ref HEAD`, so the launcher runs -exactly the checked-out review commit from an immutable detached worktree. It -does not fetch or move the review branch. The no-`--ref` production workflow -fetches `upstream/main` only after this feature is merged. +Every validation command below omits `--ref`, so the launcher runs exactly the +checked-out review branch without fetching or moving it. Keeping the review +checkout clean makes these validation results attributable to its HEAD. The +production latest-main workflow explicitly passes `--ref upstream/main`. ## 2. Prepare the Host @@ -62,7 +62,7 @@ Before starting GPU work, validate the configured token and read one byte from one remote parquet object in each of the queries, qrels, and corpus partitions: ```bash -./ops/retriever-nightly/run-nightly.sh --ref HEAD --check-vidore-access +./ops/retriever-nightly/run-nightly.sh --check-vidore-access ``` The command should exit zero and report access for all eight ViDoRe v3 @@ -72,7 +72,7 @@ suite if this check reports a Hugging Face or CAS redirect failure. ## 4. Preflight All Twelve Benchmarks ```bash -./ops/retriever-nightly/run-nightly.sh --ref HEAD --dry-run +./ops/retriever-nightly/run-nightly.sh --dry-run ``` The command should exit zero, report a new timestamped session directory, and @@ -84,7 +84,6 @@ the dry-run does not materialize batch data. ```bash ./ops/retriever-nightly/run-nightly.sh \ - --ref HEAD \ --no-slack \ nemo_retriever/harness/runfiles/jp20_beir.json ``` @@ -112,7 +111,7 @@ review branch commit. These steps do not post to Slack. From the clean draft checkout, start all twelve benchmarks with one command: ```bash -./ops/retriever-nightly/run-nightly.sh --ref HEAD +./ops/retriever-nightly/run-nightly.sh ``` The launcher runs the four library benchmarks and all eight ViDoRe v3 domains. @@ -137,7 +136,7 @@ export SLACK_WEBHOOK_URL=https://hooks.slack.com/services/... interval=86400 while true; do started="$(date +%s)" - ./ops/retriever-nightly/run-nightly.sh + ./ops/retriever-nightly/run-nightly.sh --ref upstream/main elapsed=$(( $(date +%s) - started )) if (( elapsed < interval )); then sleep "$(( interval - elapsed ))" @@ -145,7 +144,7 @@ while true; do done ``` -The default command fetches the newest `upstream/main` on each iteration, +The explicit remote ref fetches the newest `upstream/main` on each iteration, preflights ViDoRe access, runs the suite, and posts once when `SLACK_WEBHOOK_URL` is set. Enter the exports inside the new tmux session, detach with `Ctrl-b d`, and inspect it later with `tmux attach -t diff --git a/ops/retriever-nightly/run-nightly.sh b/ops/retriever-nightly/run-nightly.sh index 3b2a9b3c07..c7d4e2ddcd 100755 --- a/ops/retriever-nightly/run-nightly.sh +++ b/ops/retriever-nightly/run-nightly.sh @@ -20,12 +20,11 @@ usage() { cat <<'EOF' Usage: run-nightly.sh [OPTIONS] [--] [RUNFILE ...] -Fetch and run the library and ViDoRe v3 harness suite from the latest -upstream/main commit. With no RUNFILE arguments, all 12 checked-in nightly -runfiles are executed. +Run the library and ViDoRe v3 harness suite from the current checkout. With no +RUNFILE arguments, all 12 checked-in nightly runfiles are executed. Options: - --ref REF Run an existing local Git ref instead of fetching main. + --ref REF Run a clean checkout of a Git ref. Remote refs are fetched. --dataset-paths PATH Machine-local dataset path map. --artifact-root PATH Parent directory for timestamped session artifacts. --check-vidore-access Validate authenticated ViDoRe evaluation-data access and exit. @@ -35,6 +34,56 @@ Options: EOF } +run_checkout() { + local selected_checkout="$1" + local selection_label="$2" + local uv_environment="$3" + local allow_dirty="$4" + shift 4 + + local target_commit target_launcher preflight_access run_rc arg + target_commit="$(git -C "$selected_checkout" rev-parse --verify HEAD^{commit})" || \ + return "$EXIT_CONFIG" + target_launcher="$selected_checkout/ops/retriever-nightly/run-nightly.sh" + if [[ ! -x "$target_launcher" ]]; then + log "selected checkout does not contain an executable nightly launcher: $target_launcher" + return "$EXIT_CONFIG" + fi + + export RETRIEVER_SELECTED_CHECKOUT="$selected_checkout" + if [[ -n "$uv_environment" ]]; then + export UV_PROJECT_ENVIRONMENT="$uv_environment" + fi + if [[ "$allow_dirty" == "1" ]]; then + export RETRIEVER_ALLOW_DIRTY_CHECKOUT=1 + else + unset RETRIEVER_ALLOW_DIRTY_CHECKOUT + fi + log "selected $selection_label commit $target_commit in $selected_checkout" + + preflight_access=1 + for arg in "$@"; do + if [[ "$arg" == "--dry-run" || "$arg" == "--check-vidore-access" ]]; then + preflight_access=0 + fi + done + + run_rc=0 + if [[ "$preflight_access" == "1" ]]; then + "$target_launcher" --check-vidore-access || run_rc=$? + fi + if ((run_rc == 0)); then + if [[ -n "$slack_webhook_url" ]]; then + export SLACK_WEBHOOK_URL="$slack_webhook_url" + fi + "$target_launcher" "$@" || run_rc=$? + unset SLACK_WEBHOOK_URL + else + log "ViDoRe access preflight failed; skipping GPU work" + fi + return "$run_rc" +} + absolute_path() { realpath -m -- "$1" } @@ -77,15 +126,8 @@ select_checkout_and_run() { local requested_ref="$1" shift - local repository source_name source_ref remote_ref fetched_ref checkout_root keep_count uv_environment + local repository checkout_root keep_count uv_environment repository="${RETRIEVER_UPDATE_REPOSITORY:-$(git -C "$script_dir" rev-parse --show-toplevel)}" - source_name="${RETRIEVER_LATEST_SOURCE:-upstream}" - source_ref="${RETRIEVER_LATEST_REF:-main}" - remote_ref="$source_ref" - if [[ "$remote_ref" != refs/* ]]; then - remote_ref="refs/heads/$remote_ref" - fi - fetched_ref="refs/retriever-nightly/latest-main" checkout_root="${RETRIEVER_LATEST_CHECKOUT_ROOT:-$nightly_root/retriever-nightly-checkouts}" keep_count="${RETRIEVER_LATEST_KEEP_CHECKOUTS:-7}" uv_environment="${UV_PROJECT_ENVIRONMENT:-$checkout_root/.venv}" @@ -98,10 +140,12 @@ select_checkout_and_run() { log "controller checkout is not a Git worktree: $repository" return "$EXIT_CONFIG" fi - if [[ -n "$(git -C "$repository" status --porcelain --untracked-files=normal)" ]]; then - log "controller checkout has local changes: $repository" - return "$EXIT_CONFIG" + + if [[ -z "$requested_ref" ]]; then + run_checkout "$repository" "current checkout" "${UV_PROJECT_ENVIRONMENT:-}" 1 "$@" + return $? fi + if ! mkdir -p "$checkout_root"; then log "could not create managed checkout root: $checkout_root" return "$EXIT_CONFIG" @@ -114,20 +158,23 @@ select_checkout_and_run() { return "$EXIT_ALREADY_RUNNING" fi - local target_commit selection_label - if [[ -z "$requested_ref" ]]; then + local target_commit selection_label remote_name remote_branch remote_ref fetched_ref + remote_name="${requested_ref%%/*}" + remote_branch="${requested_ref#*/}" + if [[ "$requested_ref" == */* ]] && git -C "$repository" remote get-url "$remote_name" >/dev/null 2>&1; then + remote_ref="refs/heads/$remote_branch" + fetched_ref="refs/retriever-nightly/selected-remote" if ! git check-ref-format "$remote_ref"; then - log "RETRIEVER_LATEST_REF is not a valid Git ref: $source_ref" + log "--ref is not a valid remote branch: $requested_ref" return "$EXIT_CONFIG" fi - log "fetching $source_name $source_ref" - if ! git -C "$repository" fetch --no-tags "$source_name" "+$remote_ref:$fetched_ref"; then + log "fetching $remote_name $remote_branch" + if ! git -C "$repository" fetch --no-tags "$remote_name" "+$remote_ref:$fetched_ref"; then log "fetch failed; refusing to run a stale commit" return "$EXIT_FETCH" fi target_commit="$(git -C "$repository" rev-parse --verify "$fetched_ref^{commit}")" || \ return "$EXIT_FETCH" - selection_label="$source_name/$source_ref" else if [[ "$requested_ref" == -* ]]; then log "--ref must name a Git ref or commit" @@ -137,10 +184,10 @@ select_checkout_and_run() { log "could not resolve --ref $requested_ref to a local commit" return "$EXIT_CONFIG" } - selection_label="$requested_ref" fi + selection_label="$requested_ref" - local selected_checkout selected_head target_launcher preflight_access run_rc + local selected_checkout selected_head run_rc selected_checkout="$checkout_root/commit-$target_commit" if [[ -e "$selected_checkout" ]]; then selected_head="$(git -C "$selected_checkout" rev-parse --verify HEAD 2>/dev/null || true)" @@ -158,37 +205,8 @@ select_checkout_and_run() { fi touch "$selected_checkout" - target_launcher="$selected_checkout/ops/retriever-nightly/run-nightly.sh" - if [[ ! -x "$target_launcher" ]]; then - log "selected commit does not contain an executable nightly launcher: $target_launcher" - return "$EXIT_CONFIG" - fi - - export RETRIEVER_SELECTED_CHECKOUT="$selected_checkout" - export UV_PROJECT_ENVIRONMENT="$uv_environment" - log "selected $selection_label commit $target_commit in $selected_checkout" - - preflight_access=1 - local arg - for arg in "$@"; do - if [[ "$arg" == "--dry-run" || "$arg" == "--check-vidore-access" ]]; then - preflight_access=0 - fi - done - run_rc=0 - if [[ "$preflight_access" == "1" ]]; then - "$target_launcher" --check-vidore-access || run_rc=$? - fi - if ((run_rc == 0)); then - if [[ -n "$slack_webhook_url" ]]; then - export SLACK_WEBHOOK_URL="$slack_webhook_url" - fi - "$target_launcher" "$@" || run_rc=$? - unset SLACK_WEBHOOK_URL - else - log "ViDoRe access preflight failed; skipping GPU work" - fi + run_checkout "$selected_checkout" "$selection_label" "$uv_environment" 0 "$@" || run_rc=$? prune_managed_worktrees "$repository" "$checkout_root" "$selected_checkout" "$keep_count" return "$run_rc" @@ -355,9 +373,16 @@ if [[ ! -d "$checkout/.git" && ! -f "$checkout/.git" ]]; then log "deployment checkout is not a Git worktree: $checkout" exit "$EXIT_CONFIG" fi -if [[ -n "$(git -C "$checkout" status --porcelain --untracked-files=normal --ignore-submodules)" ]]; then - log "deployment checkout has tracked, staged, or untracked changes; refusing to run" - exit "$EXIT_CONFIG" +checkout_status="$(git -C "$checkout" status --porcelain --untracked-files=normal --ignore-submodules)" +checkout_dirty=0 +if [[ -n "$checkout_status" ]]; then + checkout_dirty=1 + if [[ "${RETRIEVER_ALLOW_DIRTY_CHECKOUT:-}" != "1" ]]; then + log "deployment checkout has tracked, staged, or untracked changes; refusing to run" + exit "$EXIT_CONFIG" + fi + log "WARNING: running the current checkout with local changes" + slack_title="[LOCAL CHANGES] $slack_title" fi if ((check_vidore_access)); then cd "$checkout" || exit "$EXIT_CONFIG" @@ -426,6 +451,21 @@ if [[ -e "$session_dir" ]]; then exit "$EXIT_CANNOT_CREATE" fi +if ((checkout_dirty)); then + if ! mkdir -p "$session_dir"; then + log "could not create session directory for source provenance: $session_dir" + exit "$EXIT_CANNOT_CREATE" + fi + { + printf 'run_commit=%s\n' "$(git -C "$checkout" rev-parse --verify HEAD^{commit})" + printf 'working_tree_dirty=true\n' + git -C "$checkout" status --short --branch --untracked-files=normal --ignore-submodules + } >"$session_dir/source_worktree_status.txt" || { + log "could not record dirty-worktree provenance: $session_dir/source_worktree_status.txt" + exit "$EXIT_CANNOT_CREATE" + } +fi + cd "$checkout" || exit "$EXIT_CONFIG" run_rc=0 From 098e993f06db6ad1f0c0396dbe5c202e194d22af Mon Sep 17 00:00:00 2001 From: Jacob Ioffe Date: Mon, 20 Jul 2026 21:46:23 +0000 Subject: [PATCH 07/11] fix(ops): clarify nightly dataset map input Signed-off-by: Jacob Ioffe --- .../harness/dataset_paths.example.yaml | 2 + .../test_harness_nightly_git_selection.py | 2 + .../tests/test_harness_nightly_launcher.py | 13 +++++ ops/retriever-nightly/README.md | 49 +++++++++++++++++-- .../SECOND_HOST_VALIDATION.md | 12 ++++- ops/retriever-nightly/nightly.env.example | 3 +- ops/retriever-nightly/run-nightly.sh | 9 +++- 7 files changed, 81 insertions(+), 9 deletions(-) diff --git a/nemo_retriever/harness/dataset_paths.example.yaml b/nemo_retriever/harness/dataset_paths.example.yaml index cc9b5ec10c..81e22363b1 100644 --- a/nemo_retriever/harness/dataset_paths.example.yaml +++ b/nemo_retriever/harness/dataset_paths.example.yaml @@ -1,4 +1,6 @@ schema_version: 1 +# Copy this file outside the checkout, replace the paths for the host, and pass +# the YAML file itself with --dataset-paths. Do not pass a dataset directory. datasets: jp20: path: /path/to/datasets/nv-ingest/jp20 diff --git a/nemo_retriever/tests/test_harness_nightly_git_selection.py b/nemo_retriever/tests/test_harness_nightly_git_selection.py index 2b3da91700..041b6eaaca 100644 --- a/nemo_retriever/tests/test_harness_nightly_git_selection.py +++ b/nemo_retriever/tests/test_harness_nightly_git_selection.py @@ -168,6 +168,8 @@ def test_help_does_not_require_configuration_or_fetch(tmp_path: Path) -> None: assert result.returncode == 0, result.stderr assert "current checkout" in result.stdout assert "--ref REF" in result.stdout + assert "--dataset-paths YAML_FILE" in result.stdout + assert "YAML file" in result.stdout def test_explicit_ref_runs_local_commit_without_fetching(latest_main_fixture) -> None: diff --git a/nemo_retriever/tests/test_harness_nightly_launcher.py b/nemo_retriever/tests/test_harness_nightly_launcher.py index 8ae19b4b84..b94339f93f 100644 --- a/nemo_retriever/tests/test_harness_nightly_launcher.py +++ b/nemo_retriever/tests/test_harness_nightly_launcher.py @@ -211,6 +211,19 @@ def test_manual_dry_run_launches_full_batch_suite_without_slack(nightly_launcher assert tuple(invocation[-len(DEFAULT_RUNFILES) :]) == DEFAULT_RUNFILES +def test_dataset_paths_rejects_a_directory_with_actionable_error(nightly_launcher, tmp_path: Path) -> None: + run, calls = nightly_launcher + dataset_directory = tmp_path / "jp20" + dataset_directory.mkdir() + + result = run("--dataset-paths", str(dataset_directory), "--dry-run") + + assert result.returncode == 64 + assert calls() == [] + assert "--dataset-paths expects a YAML file, not a directory" in result.stderr + assert str(dataset_directory) in result.stderr + + def test_access_preflight_checks_vidore_without_starting_a_session(nightly_launcher) -> None: run, calls = nightly_launcher diff --git a/ops/retriever-nightly/README.md b/ops/retriever-nightly/README.md index f83999175a..3604706cdd 100644 --- a/ops/retriever-nightly/README.md +++ b/ops/retriever-nightly/README.md @@ -74,7 +74,7 @@ optional `nightly.env` accepts only two secrets and one optional path override: | --- | --- | --- | | `HF_TOKEN` | yes | Read-only access to ViDoRe evaluation data. | | `SLACK_WEBHOOK_URL` | no | Enables one terminal Slack post for real runs. | -| `RETRIEVER_DATASET_PATHS` | nonstandard hosts only | Replaces the checked-in `/datasets/nv-ingest` map. | +| `RETRIEVER_DATASET_PATHS` | nonstandard hosts only | Path to a YAML file that replaces the checked-in `/datasets/nv-ingest` map. | On hosts with a writable `/raid/$USER`, the launcher automatically keeps its private configuration, artifacts, and managed Git checkouts there. @@ -99,10 +99,49 @@ chmod 600 "$RETRIEVER_NIGHTLY_CONFIG_DIR/nightly.env" ${EDITOR:-vi} "$RETRIEVER_NIGHTLY_CONFIG_DIR/nightly.env" ``` -The checked-in `dataset_paths.datasets.yaml` already describes the standard -`/datasets/nv-ingest` layout. Only hosts with a different layout need to copy -and edit `nemo_retriever/harness/dataset_paths.example.yaml`, then set -`RETRIEVER_DATASET_PATHS` in `nightly.env`. +### Dataset Paths On A Nonstandard Host + +The checked-in `dataset_paths.datasets.yaml` describes the standard +`/datasets/nv-ingest` layout. Do not pass `--dataset-paths` on a host with that +layout. + +On any other host, `--dataset-paths` takes the path to a YAML configuration +file, not a dataset directory. Copy the complete twelve-dataset template +outside the checkout, replace its paths, and pass the resulting file: + +```bash +cp nemo_retriever/harness/dataset_paths.example.yaml \ + /raid/$USER/retriever-dataset-paths.yaml +${EDITOR:-vi} /raid/$USER/retriever-dataset-paths.yaml +./ops/retriever-nightly/run-nightly.sh \ + --dataset-paths /raid/$USER/retriever-dataset-paths.yaml \ + --dry-run +``` + +For a JP20-only canary, the YAML file may contain only its local dataset and +ground-truth query paths: + +```yaml +schema_version: 1 +datasets: + jp20: + path: /raid/data/jp20 + query_file: /raid/data/jp20_query_gt.csv +``` + +Run that one benchmark by also supplying its runfile: + +```bash +./ops/retriever-nightly/run-nightly.sh \ + --dataset-paths /raid/data/retriever-dataset-paths.yaml \ + --no-slack \ + nemo_retriever/harness/runfiles/jp20_beir.json +``` + +Keep the machine-local YAML outside the repository. For repeated runs, export +`RETRIEVER_DATASET_PATHS=/raid/$USER/retriever-dataset-paths.yaml` or set the +same value in the optional `nightly.env`; the command-line flag is simplest for +a one-off run. The launcher sources the detected file only when it exists; otherwise it uses the current environment. An existing secrets file must be owned by the diff --git a/ops/retriever-nightly/SECOND_HOST_VALIDATION.md b/ops/retriever-nightly/SECOND_HOST_VALIDATION.md index ef9657879b..01e6fe298d 100644 --- a/ops/retriever-nightly/SECOND_HOST_VALIDATION.md +++ b/ops/retriever-nightly/SECOND_HOST_VALIDATION.md @@ -54,7 +54,17 @@ export SLACK_WEBHOOK_URL=https://hooks.slack.com/services/... The launcher writes artifacts under `/raid/$USER/retriever-nightly-artifacts`. No configuration file is required. On a host without the standard dataset layout, copy and edit `nemo_retriever/harness/dataset_paths.example.yaml`, then -export `RETRIEVER_DATASET_PATHS` with that path. +pass the resulting YAML file to `--dataset-paths`. The option takes the YAML +file itself, not a dataset directory: + +```bash +cp nemo_retriever/harness/dataset_paths.example.yaml \ + /raid/$USER/retriever-dataset-paths.yaml +${EDITOR:-vi} /raid/$USER/retriever-dataset-paths.yaml +./ops/retriever-nightly/run-nightly.sh \ + --dataset-paths /raid/$USER/retriever-dataset-paths.yaml \ + --dry-run +``` ## 3. Verify ViDoRe Evaluation Access diff --git a/ops/retriever-nightly/nightly.env.example b/ops/retriever-nightly/nightly.env.example index 5aa5c41a70..89ee943623 100644 --- a/ops/retriever-nightly/nightly.env.example +++ b/ops/retriever-nightly/nightly.env.example @@ -15,5 +15,6 @@ HF_TOKEN= # SLACK_WEBHOOK_URL=https://hooks.slack.com/services/... # Override only when this host does not use the checked-in -# /datasets/nv-ingest path map: +# /datasets/nv-ingest path map. This value must name a YAML file, not a dataset +# directory: # RETRIEVER_DATASET_PATHS=/path/to/dataset_paths.yaml diff --git a/ops/retriever-nightly/run-nightly.sh b/ops/retriever-nightly/run-nightly.sh index c7d4e2ddcd..409ea056fc 100755 --- a/ops/retriever-nightly/run-nightly.sh +++ b/ops/retriever-nightly/run-nightly.sh @@ -25,7 +25,8 @@ RUNFILE arguments, all 12 checked-in nightly runfiles are executed. Options: --ref REF Run a clean checkout of a Git ref. Remote refs are fetched. - --dataset-paths PATH Machine-local dataset path map. + --dataset-paths YAML_FILE + YAML file mapping benchmark names to local dataset paths. --artifact-root PATH Parent directory for timestamped session artifacts. --check-vidore-access Validate authenticated ViDoRe evaluation-data access and exit. --dry-run Resolve and validate the suite without executing it. @@ -389,8 +390,12 @@ if ((check_vidore_access)); then "$uv_bin" run --frozen --project nemo_retriever retriever harness check-vidore-access exit $? fi +if [[ -d "$dataset_paths" ]]; then + log "--dataset-paths expects a YAML file, not a directory: $dataset_paths" + exit "$EXIT_CONFIG" +fi if [[ ! -f "$dataset_paths" ]]; then - log "dataset path map is missing: $dataset_paths" + log "dataset paths YAML file is missing: $dataset_paths" exit "$EXIT_CONFIG" fi post_slack=0 From 51a8013b5cf259b963803a72153eef2b9dd1e882 Mon Sep 17 00:00:00 2001 From: Jacob Ioffe Date: Tue, 21 Jul 2026 18:53:57 +0000 Subject: [PATCH 08/11] fix(ops): align nightly configuration docs Signed-off-by: Jacob Ioffe --- .../harness_retriever_ingest_query_prd.md | 18 ++++--- nemo_retriever/harness/README.md | 33 +++++++++++-- .../nemo_retriever/harness/vidore_access.py | 5 +- .../test_harness_nightly_git_selection.py | 36 ++++++++++++++ .../tests/test_harness_vidore_access.py | 9 ++++ ops/retriever-nightly/README.md | 24 ++++++---- ops/retriever-nightly/nightly.env.example | 4 +- ops/retriever-nightly/run-nightly.sh | 48 +++++++++++++++++-- 8 files changed, 149 insertions(+), 28 deletions(-) diff --git a/nemo_retriever/developer_docs/harness_retriever_ingest_query_prd.md b/nemo_retriever/developer_docs/harness_retriever_ingest_query_prd.md index d8b7fd5c58..a61f001159 100644 --- a/nemo_retriever/developer_docs/harness_retriever_ingest_query_prd.md +++ b/nemo_retriever/developer_docs/harness_retriever_ingest_query_prd.md @@ -1,15 +1,17 @@ # Retriever Harness PRD: End-to-End Ingest/Query Benchmarks -Last updated: 2026-07-01 +Last updated: 2026-07-21 ## Implementation Status -The current implementation includes the core runner described here plus two +The current implementation includes the core runner described here plus four orchestration-neutral extensions: `run-files` applies a machine-local dataset -path map to one or more checked-in runfiles, and `post-slack` renders or posts -completed artifacts. It does not include recurring scheduling, deployment, -locking, retry policy, or secret distribution. The harness README is the -normative user and agent guide; this PRD records the design rationale. +path map to one or more checked-in runfiles; `--isolate-runs` gives each child a +fresh process; `check-vidore-access` validates remote ViDoRe evaluation data; +and `post-slack` renders or posts completed artifacts. It does not include +recurring scheduling, deployment, locking, retry policy, or secret +distribution. The harness README is the normative user and agent guide; this +PRD records the design rationale. ## Summary @@ -737,6 +739,10 @@ validation through the CLI and artifact contract. `expanded_runs.json`. - `retriever harness run-files` runs one or more checked-in requests with an optional machine-local dataset path map. +- `retriever harness run-files --isolate-runs` executes each child in a fresh + process while retaining one terminal session summary. +- `retriever harness check-vidore-access` validates authenticated access to the + ViDoRe queries, qrels, and corpus partitions without downloading them. - `retriever harness post-slack --preview` reads artifacts without requiring a webhook or contacting Slack. - Every run writes `status.json`, `events.jsonl`, and `results.json`. diff --git a/nemo_retriever/harness/README.md b/nemo_retriever/harness/README.md index 73ed9bb412..5491bac711 100644 --- a/nemo_retriever/harness/README.md +++ b/nemo_retriever/harness/README.md @@ -76,10 +76,17 @@ uses the product service APIs for ingest and query while preserving the same overrides. - `run-set`: expand a code-owned benchmark group using registry paths. - `run-files`: execute one or more runfiles with an optional machine-local - dataset path map. This is the portable entry point for the checked-in suite. + dataset path map and optional per-child process isolation. This is the + portable entry point for the checked-in suite. +- `check-vidore-access`: validate authenticated access to the queries, qrels, + and corpus objects for all eight ViDoRe v3 datasets without downloading them. - `post-slack`: preview or post existing artifacts; it never executes a run. - `diff`: compare two run artifact directories by `results.json` summary metrics. +For the opinionated twelve-benchmark workstation workflow, including Git +selection, dataset defaults, Slack, and daily recurrence, use the +[Retriever nightly launcher](../../ops/retriever-nightly/README.md). + Legacy graph-pipeline, sweep, recurring-job, runner, reporting-UI, and portal commands are not part of this CLI surface. Scheduling and deployment belong to separate infrastructure, not the benchmark harness. @@ -154,9 +161,18 @@ uv run --project nemo_retriever retriever harness run-files \ nemo_retriever/harness/runfiles/financebench_beir.json ``` -The ViDoRe v3 library follows the same runfile-first contract. Each benchmark -uses the VL embed model with `text_image` page embeddings and page-level BEIR -scoring. Run one domain while validating a machine or configuration: +The ViDoRe v3 library follows the same runfile-first contract. Before GPU work, +export a Hugging Face read token and validate all remote evaluation partitions: + +```bash +export HF_TOKEN=... +uv run --project nemo_retriever retriever harness check-vidore-access +``` + +The check streams one byte from each queries, qrels, and corpus partition; it +does not download the complete parquet objects. Each benchmark uses the VL +embed model with `text_image` page embeddings and page-level BEIR scoring. Run +one domain while validating a machine or configuration: ```bash uv run --project nemo_retriever retriever harness run-files \ @@ -191,6 +207,11 @@ default dataset paths are mounted. Prefer the checked-in runfiles for nightly or other orchestrated sessions because they carry per-dataset integrity gates and accept a machine-local path map. +Add `--isolate-runs` to a real multi-run `run-files` session when each child +should execute in a fresh spawned process. The process boundary releases Ray +and materialized dataframe memory before the next benchmark while preserving +one parent-owned `session_summary.json`. + `run-files` owns the session layout and execution mode. Runfiles passed to this command cannot set their own `output_dir`, `run_id`, or `dry_run`; use the session-level `--dry-run` flag instead. The session uses the following paths and @@ -418,8 +439,12 @@ results: { "session_type": "runfiles", "session_name": "library_beir", + "run_commit": "0123456789abcdef0123456789abcdef01234567", + "working_tree_dirty": false, "success": true, "exit_code": 0, + "dry_run": false, + "isolate_runs": true, "runs": [ { "benchmark": "jp20_beir", diff --git a/nemo_retriever/src/nemo_retriever/harness/vidore_access.py b/nemo_retriever/src/nemo_retriever/harness/vidore_access.py index 526321bb40..bfb2698be1 100644 --- a/nemo_retriever/src/nemo_retriever/harness/vidore_access.py +++ b/nemo_retriever/src/nemo_retriever/harness/vidore_access.py @@ -1,4 +1,5 @@ # SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: Apache-2.0 """Fast authenticated access checks for remote ViDoRe evaluation data.""" @@ -81,9 +82,7 @@ def check_vidore_access( effective_token = token or os.environ.get("HF_TOKEN") or os.environ.get("HUGGING_FACE_HUB_TOKEN") if require_token and not effective_token: - raise VidoreAccessError( - "HF_TOKEN is not set; add it to the mode-600 nightly configuration before checking ViDoRe access" - ) + raise VidoreAccessError("HF_TOKEN is not set; export HF_TOKEN before checking ViDoRe access") if api is None: from huggingface_hub import HfApi diff --git a/nemo_retriever/tests/test_harness_nightly_git_selection.py b/nemo_retriever/tests/test_harness_nightly_git_selection.py index 041b6eaaca..081a72b028 100644 --- a/nemo_retriever/tests/test_harness_nightly_git_selection.py +++ b/nemo_retriever/tests/test_harness_nightly_git_selection.py @@ -1,4 +1,5 @@ # SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: Apache-2.0 from __future__ import annotations @@ -60,6 +61,10 @@ def latest_main_fixture(tmp_path: Path): '"${SLACK_WEBHOOK_URL:-}" != "$EXPECT_SLACK_WEBHOOK_URL" ]]; then', " exit 96", "fi", + 'if [[ -n "${EXPECT_DATASET_PATHS+x}" && ' + '"${RETRIEVER_DATASET_PATHS:-}" != "$EXPECT_DATASET_PATHS" ]]; then', + " exit 95", + "fi", 'checkout="$(git -C "$(dirname -- "$0")" rev-parse --show-toplevel)"', 'commit="$(git -C "$checkout" rev-parse HEAD)"', 'printf "%s|%s|%s\\n" "$commit" "$*" "${UV_PROJECT_ENVIRONMENT:-}" >>"$FAKE_LATEST_CALLS"', @@ -204,6 +209,37 @@ def test_launcher_selection_uses_exported_secrets_without_config_file(latest_mai assert calls() == [(initial_commit, "--dry-run", "")] +def test_exported_settings_override_optional_config_file(latest_main_fixture, tmp_path: Path) -> None: + run, calls, _source, _controller, _checkout_root, initial_commit, _latest_commit = latest_main_fixture + config_file = tmp_path / "config" / "nightly.env" + config_file.write_text( + "\n".join( + ( + "HF_TOKEN=file-read-token", + "SLACK_WEBHOOK_URL=https://hooks.slack.com/services/file/webhook/value", + "RETRIEVER_DATASET_PATHS=/file/dataset-paths.yaml", + "", + ) + ), + encoding="utf-8", + ) + + result = run( + "--dry-run", + extra_env={ + "HF_TOKEN": "exported-read-token", + "EXPECT_HF_TOKEN": "exported-read-token", + "SLACK_WEBHOOK_URL": SLACK_WEBHOOK_URL, + "EXPECT_SLACK_WEBHOOK_URL": SLACK_WEBHOOK_URL, + "RETRIEVER_DATASET_PATHS": "/exported/dataset-paths.yaml", + "EXPECT_DATASET_PATHS": "/exported/dataset-paths.yaml", + }, + ) + + assert result.returncode == 0, result.stderr + assert calls() == [(initial_commit, "--dry-run", "")] + + @pytest.mark.parametrize("configured, expected", [(None, "skip"), ("full", "full")]) def test_selected_run_has_safe_warmup_default_and_allows_override( latest_main_fixture, configured: str | None, expected: str diff --git a/nemo_retriever/tests/test_harness_vidore_access.py b/nemo_retriever/tests/test_harness_vidore_access.py index 22e9610928..6795a6771f 100644 --- a/nemo_retriever/tests/test_harness_vidore_access.py +++ b/nemo_retriever/tests/test_harness_vidore_access.py @@ -1,4 +1,5 @@ # SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: Apache-2.0 from __future__ import annotations @@ -128,6 +129,14 @@ def incomplete_dataset_info(repo_id: str, *, files_metadata: bool): check_vidore_access(dataset_names=("vidore_v3_hr",), token="hf-secret", api=api, get=lambda *a, **k: None) +def test_check_vidore_access_missing_token_recommends_direct_export(monkeypatch) -> None: + monkeypatch.delenv("HF_TOKEN", raising=False) + monkeypatch.delenv("HUGGING_FACE_HUB_TOKEN", raising=False) + + with pytest.raises(VidoreAccessError, match="export HF_TOKEN"): + check_vidore_access(dataset_names=("vidore_v3_hr",)) + + def test_check_vidore_access_rejects_an_empty_success_response() -> None: def get(url, **kwargs): return _Response(206, content=b"") diff --git a/ops/retriever-nightly/README.md b/ops/retriever-nightly/README.md index 3604706cdd..baef1ead3b 100644 --- a/ops/retriever-nightly/README.md +++ b/ops/retriever-nightly/README.md @@ -72,7 +72,7 @@ optional `nightly.env` accepts only two secrets and one optional path override: | Setting | Required | Purpose | | --- | --- | --- | -| `HF_TOKEN` | yes | Read-only access to ViDoRe evaluation data. | +| `HF_TOKEN` | every real launcher run | Read-only access for the automatic ViDoRe preflight. | | `SLACK_WEBHOOK_URL` | no | Enables one terminal Slack post for real runs. | | `RETRIEVER_DATASET_PATHS` | nonstandard hosts only | Path to a YAML file that replaces the checked-in `/datasets/nv-ingest` map. | @@ -82,7 +82,8 @@ Other hosts use `$HOME`. Direct exports are the smallest configuration interface. A private file is optional for operators who do not want to export the same values in every -shell. To create it: +shell. Already-exported supported settings take precedence over values in that +file. To create it: ```bash if [[ -d /raid/$USER && -w /raid/$USER ]]; then @@ -143,10 +144,11 @@ Keep the machine-local YAML outside the repository. For repeated runs, export same value in the optional `nightly.env`; the command-line flag is simplest for a one-off run. -The launcher sources the detected file only when it exists; otherwise it uses -the current environment. An existing secrets file must be owned by the -invoking user with mode `600`. The launcher does not discover a repository -`.env` file. `RETRIEVER_CONFIG_FILE` remains an optional advanced path override. +The launcher loads the detected file only when it exists and uses its values as +defaults for settings that were not already exported. An existing secrets file +must be owned by the invoking user with mode `600`. The launcher does not +discover a repository `.env` file. `RETRIEVER_CONFIG_FILE` remains an optional +advanced path override. Verify the token and read one byte from one remote parquet object in every ViDoRe evaluation partition before starting GPU work: @@ -179,6 +181,9 @@ Use one positional runfile for a smaller real canary before the full run: nemo_retriever/harness/runfiles/jp20_beir.json ``` +The launcher performs the ViDoRe access preflight before every real invocation, +including a JP20-only canary, so `HF_TOKEN` is still required for this command. + Run the complete suite from the current checkout with no positional runfiles: ```bash @@ -295,9 +300,10 @@ that flag suppresses webhook validation and posting. Dry-runs and access checks never post. The launcher removes the URL from the benchmark child environment and exposes it only to the final Slack command. -Command-line flags override values loaded from `RETRIEVER_CONFIG_FILE`; those -values override the launcher defaults. Run the launcher with `--help` for its -supported interface. +Configuration precedence is, from highest to lowest: command-line flags, +supported variables already exported when the launcher starts, values loaded +from `RETRIEVER_CONFIG_FILE`, and launcher defaults. Run the launcher with +`--help` for its supported interface. ## Runtime Contract diff --git a/ops/retriever-nightly/nightly.env.example b/ops/retriever-nightly/nightly.env.example index 89ee943623..8194123fcf 100644 --- a/ops/retriever-nightly/nightly.env.example +++ b/ops/retriever-nightly/nightly.env.example @@ -8,8 +8,8 @@ # it uses $HOME. # # Required for authenticated ViDoRe runs when HF_TOKEN is not already exported. -# A read token is sufficient: -HF_TOKEN= +# A read token is sufficient. Uncomment only when storing it in this file: +# HF_TOKEN=hf_... # Optional: uncomment to post terminal real-run summaries to Slack. # SLACK_WEBHOOK_URL=https://hooks.slack.com/services/... diff --git a/ops/retriever-nightly/run-nightly.sh b/ops/retriever-nightly/run-nightly.sh index 409ea056fc..df1dfdc3c6 100755 --- a/ops/retriever-nightly/run-nightly.sh +++ b/ops/retriever-nightly/run-nightly.sh @@ -35,6 +35,49 @@ Options: EOF } +load_config_defaults() { + local path="$1" + local name + local -a supported_variables=( + HF_TOKEN + HUGGING_FACE_HUB_TOKEN + SLACK_WEBHOOK_URL + RETRIEVER_ARTIFACT_ROOT + RETRIEVER_CHECKOUT + RETRIEVER_DATASET_PATHS + RETRIEVER_LATEST_CHECKOUT_ROOT + RETRIEVER_LATEST_KEEP_CHECKOUTS + RETRIEVER_MODE + RETRIEVER_SESSION_NAME + RETRIEVER_SLACK_TITLE + RETRIEVER_UPDATE_REPOSITORY + RETRIEVER_UV_BIN + UV_PROJECT_ENVIRONMENT + VLLM_DEEP_GEMM_WARMUP + ) + local -A inherited=() + local -A inherited_values=() + + for name in "${supported_variables[@]}"; do + if [[ -v "$name" ]]; then + inherited["$name"]=1 + inherited_values["$name"]="${!name}" + fi + done + + set -a + # shellcheck disable=SC1090 + source "$path" + set +a + + for name in "${supported_variables[@]}"; do + if [[ "${inherited[$name]:-}" == "1" ]]; then + printf -v "$name" '%s' "${inherited_values[$name]}" + export "$name" + fi + done +} + run_checkout() { local selected_checkout="$1" local selection_label="$2" @@ -235,10 +278,7 @@ if [[ -f "$config_file" ]]; then log "nightly configuration must be owned by the invoking user with mode 600" exit "$EXIT_CONFIG" fi - set -a - # shellcheck disable=SC1090 - source "$config_file" - set +a + load_config_defaults "$config_file" fi slack_webhook_url="${SLACK_WEBHOOK_URL:-}" unset SLACK_WEBHOOK_URL From 29d8ba887f0f718261b401a575115f7e744341f5 Mon Sep 17 00:00:00 2001 From: Jacob Ioffe Date: Tue, 21 Jul 2026 22:28:36 +0000 Subject: [PATCH 09/11] fix(harness): bound isolated child execution Signed-off-by: Jacob Ioffe --- nemo_retriever/harness/README.md | 4 +- .../src/nemo_retriever/harness/cli.py | 11 +- .../src/nemo_retriever/harness/runsets.py | 43 +++++- nemo_retriever/tests/test_harness_runfiles.py | 128 +++++++++++++++++- ops/retriever-nightly/README.md | 4 +- 5 files changed, 177 insertions(+), 13 deletions(-) diff --git a/nemo_retriever/harness/README.md b/nemo_retriever/harness/README.md index 5491bac711..9cef6533e0 100644 --- a/nemo_retriever/harness/README.md +++ b/nemo_retriever/harness/README.md @@ -210,7 +210,9 @@ and accept a machine-local path map. Add `--isolate-runs` to a real multi-run `run-files` session when each child should execute in a fresh spawned process. The process boundary releases Ray and materialized dataframe memory before the next benchmark while preserving -one parent-owned `session_summary.json`. +one parent-owned `session_summary.json`. Isolated children have a six-hour +wall-time limit by default so one hung child cannot block the session forever; +use `--child-timeout-seconds` to adjust that limit for an unusually long run. `run-files` owns the session layout and execution mode. Runfiles passed to this command cannot set their own `output_dir`, `run_id`, or `dry_run`; use the diff --git a/nemo_retriever/src/nemo_retriever/harness/cli.py b/nemo_retriever/src/nemo_retriever/harness/cli.py index 6541c49241..acafc9f79a 100644 --- a/nemo_retriever/src/nemo_retriever/harness/cli.py +++ b/nemo_retriever/src/nemo_retriever/harness/cli.py @@ -24,7 +24,7 @@ from nemo_retriever.harness.diff import diff_artifact_dirs from nemo_retriever.harness.resolution import make_run_id from nemo_retriever.harness.runfile import load_runfile -from nemo_retriever.harness.runsets import run_runfiles, run_runset +from nemo_retriever.harness.runsets import DEFAULT_ISOLATED_CHILD_TIMEOUT_SECONDS, run_runfiles, run_runset from nemo_retriever.harness.slack import ( DEFAULT_SLACK_METRIC_KEYS, build_slack_payload, @@ -304,6 +304,14 @@ def run_files_command( bool, typer.Option("--isolate-runs", help="Run each child in a fresh process to release batch memory."), ] = False, + child_timeout_seconds: Annotated[ + float, + typer.Option( + "--child-timeout-seconds", + min=1.0, + help="Maximum wall time for each isolated child before it is terminated.", + ), + ] = DEFAULT_ISOLATED_CHILD_TIMEOUT_SECONDS, json_output: Annotated[bool, typer.Option("--json", help="Emit session summary JSON to stdout.")] = False, ) -> None: """Run one or more runfiles, optionally with machine-local dataset paths.""" @@ -319,6 +327,7 @@ def run_files_command( requirements=requirements or (), dry_run=dry_run, isolate_runs=isolate_runs, + isolated_child_timeout_seconds=child_timeout_seconds, ) except HarnessRunError as exc: typer.echo(exc.failure.message, err=True) diff --git a/nemo_retriever/src/nemo_retriever/harness/runsets.py b/nemo_retriever/src/nemo_retriever/harness/runsets.py index 0ee7b51095..82b60aca25 100644 --- a/nemo_retriever/src/nemo_retriever/harness/runsets.py +++ b/nemo_retriever/src/nemo_retriever/harness/runsets.py @@ -29,6 +29,8 @@ from nemo_retriever.harness.runfile import load_runfile _SESSION_LABEL_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.-]*$") +DEFAULT_ISOLATED_CHILD_TIMEOUT_SECONDS = 6 * 60 * 60 +_ISOLATED_PROCESS_STOP_TIMEOUT_SECONDS = 30 @dataclass(frozen=True) @@ -237,7 +239,24 @@ def _isolated_run_worker(connection: Any, run: PreparedRun, *, output_dir: str, connection.close() -def _run_prepared_benchmark_isolated(run: PreparedRun, *, output_dir: str, run_id: str) -> RunOutcome: +def _stop_isolated_process(process: multiprocessing.Process) -> None: + """Best-effort bounded cleanup for an isolated benchmark process.""" + + if process.is_alive(): + process.terminate() + process.join(timeout=_ISOLATED_PROCESS_STOP_TIMEOUT_SECONDS) + if process.is_alive(): + process.kill() + process.join(timeout=_ISOLATED_PROCESS_STOP_TIMEOUT_SECONDS) + + +def _run_prepared_benchmark_isolated( + run: PreparedRun, + *, + output_dir: str, + run_id: str, + timeout_seconds: float = DEFAULT_ISOLATED_CHILD_TIMEOUT_SECONDS, +) -> RunOutcome: """Run one child with a process boundary that releases Ray and dataframe memory.""" context = multiprocessing.get_context("spawn") @@ -262,19 +281,25 @@ def _run_prepared_benchmark_isolated(run: PreparedRun, *, output_dir: str, run_i message: tuple[str, Any] | None = None try: + if not receive_connection.poll(timeout_seconds): + raise TimeoutError(f"isolated benchmark {run.name!r} exceeded the {timeout_seconds:g}-second child timeout") try: message = receive_connection.recv() except EOFError: pass except BaseException: - if process.is_alive(): - process.terminate() - process.join() + _stop_isolated_process(process) raise finally: receive_connection.close() - process.join() + process.join(timeout=_ISOLATED_PROCESS_STOP_TIMEOUT_SECONDS) + if process.is_alive(): + _stop_isolated_process(process) + raise RuntimeError( + f"isolated benchmark {run.name!r} returned an outcome but did not exit within " + f"{_ISOLATED_PROCESS_STOP_TIMEOUT_SECONDS} seconds" + ) if process.exitcode != 0: raise RuntimeError(f"isolated benchmark process exited with code {process.exitcode}") if message is None: @@ -323,6 +348,7 @@ def _run_session( dry_run: bool, summary_extra: Mapping[str, Any], isolate_runs: bool = False, + isolated_child_timeout_seconds: float = DEFAULT_ISOLATED_CHILD_TIMEOUT_SECONDS, ) -> RunOutcome: try: session_dir.mkdir(parents=True, exist_ok=True) @@ -342,6 +368,7 @@ def _run_session( run, output_dir=str(artifact_dir), run_id=run_id, + timeout_seconds=isolated_child_timeout_seconds, ) else: outcome = run_prepared_benchmark( @@ -452,9 +479,12 @@ def run_runfiles( requirements: Sequence[str] = (), dry_run: bool = False, isolate_runs: bool = False, + isolated_child_timeout_seconds: float = DEFAULT_ISOLATED_CHILD_TIMEOUT_SECONDS, ) -> RunOutcome: if not runfiles: raise _invalid_runfile("At least one runfile path is required.") + if isolated_child_timeout_seconds <= 0: + raise _invalid_runfile("--child-timeout-seconds must be greater than zero.") session_name = _validate_session_label(session_name, field="--session-name") run_commit = last_commit() @@ -524,6 +554,7 @@ def run_runfiles( "working_tree_dirty": source_worktree_dirty, "dataset_paths_file": dataset_paths_value, "isolate_runs": bool(isolate_runs), + "isolated_child_timeout_seconds": isolated_child_timeout_seconds if isolate_runs else None, "runfiles": expanded_runs, }, run_commit=run_commit, @@ -532,7 +563,9 @@ def run_runfiles( "session_name": session_name, "dataset_paths_file": dataset_paths_value, "isolate_runs": bool(isolate_runs), + "isolated_child_timeout_seconds": isolated_child_timeout_seconds if isolate_runs else None, "working_tree_dirty": source_worktree_dirty, }, isolate_runs=isolate_runs, + isolated_child_timeout_seconds=isolated_child_timeout_seconds, ) diff --git a/nemo_retriever/tests/test_harness_runfiles.py b/nemo_retriever/tests/test_harness_runfiles.py index ddfc9ae453..c360420f9d 100644 --- a/nemo_retriever/tests/test_harness_runfiles.py +++ b/nemo_retriever/tests/test_harness_runfiles.py @@ -4,17 +4,19 @@ import json from pathlib import Path +from types import SimpleNamespace import pytest from nemo_retriever.harness.contracts import ( EXIT_ARTIFACT_WRITE_FAILURE, + EXIT_INTERNAL_ERROR, EXIT_MISSING_INPUT, FailurePayload, HarnessRunError, RunOutcome, ) -from nemo_retriever.harness.runsets import run_runfiles, run_runset +from nemo_retriever.harness.runsets import _run_prepared_benchmark_isolated, run_runfiles, run_runset def _write_json(path: Path, payload: dict) -> None: @@ -204,8 +206,8 @@ def test_run_files_can_isolate_each_child_process(monkeypatch, tmp_path): isolated_calls = [] - def fake_isolated_run(run, *, output_dir, run_id): - isolated_calls.append((run.prepared.benchmark, run_id)) + def fake_isolated_run(run, *, output_dir, run_id, timeout_seconds): + isolated_calls.append((run.prepared.benchmark, run_id, timeout_seconds)) return _successful_outcome(run.prepared.benchmark, output_dir) monkeypatch.setattr( @@ -229,12 +231,128 @@ def fake_isolated_run(run, *, output_dir, run_id): assert outcome.exit_code == 0 assert outcome.results["isolate_runs"] is True assert isolated_calls == [ - ("jp20_beir", "isolated_001_jp20_beir"), - ("bo767_beir", "isolated_002_bo767_beir"), + ("jp20_beir", "isolated_001_jp20_beir", 6 * 60 * 60), + ("bo767_beir", "isolated_002_bo767_beir", 6 * 60 * 60), ] assert len(outcome.results["runs"]) == 2 +def test_run_files_isolated_timeout_records_failure_and_continues(monkeypatch, tmp_path): + runfiles = [] + for name in ("jp20_beir", "bo767_beir"): + path = tmp_path / f"{name}.json" + _write_json(path, {"schema_version": 1, "name": name, "benchmark": name}) + runfiles.append(path) + + isolated_calls = [] + + def fake_isolated_run(run, *, output_dir, run_id, timeout_seconds): + isolated_calls.append((run.prepared.benchmark, timeout_seconds)) + if run.prepared.benchmark == "jp20_beir": + raise TimeoutError("isolated benchmark exceeded the child timeout") + return _successful_outcome(run.prepared.benchmark, output_dir) + + monkeypatch.setattr( + "nemo_retriever.harness.runsets._run_prepared_benchmark_isolated", + fake_isolated_run, + ) + + outcome = run_runfiles( + runfiles, + output_dir=str(tmp_path / "session"), + overrides=(f'dataset.path="{tmp_path}"',), + dry_run=True, + isolate_runs=True, + isolated_child_timeout_seconds=17, + ) + + assert outcome.exit_code == EXIT_INTERNAL_ERROR + assert isolated_calls == [("jp20_beir", 17), ("bo767_beir", 17)] + assert [run["success"] for run in outcome.results["runs"]] == [False, True] + assert "TimeoutError" in outcome.results["runs"][0]["failure_reason"] + assert outcome.results["isolated_child_timeout_seconds"] == 17 + + +def test_isolated_child_timeout_uses_bounded_terminate_and_kill(monkeypatch): + class FakeConnection: + def __init__(self, *, readable): + self.readable = readable + self.closed = False + self.poll_timeout = None + + def poll(self, timeout): + self.poll_timeout = timeout + return self.readable + + def recv(self): + raise AssertionError("recv should not be called when poll times out") + + def close(self): + self.closed = True + + class FakeProcess: + def __init__(self): + self.alive = True + self.exitcode = None + self.started = False + self.terminated = False + self.killed = False + self.join_timeouts = [] + + def start(self): + self.started = True + + def is_alive(self): + return self.alive + + def terminate(self): + self.terminated = True + + def kill(self): + self.killed = True + self.alive = False + self.exitcode = -9 + + def join(self, timeout): + self.join_timeouts.append(timeout) + + receive_connection = FakeConnection(readable=False) + send_connection = FakeConnection(readable=False) + process = FakeProcess() + + class FakeContext: + @staticmethod + def Pipe(*, duplex): + assert duplex is False + return receive_connection, send_connection + + @staticmethod + def Process(**_kwargs): + return process + + monkeypatch.setattr( + "nemo_retriever.harness.runsets.multiprocessing.get_context", + lambda method: FakeContext(), + ) + + run = SimpleNamespace(name="hung", artifact_name="hung") + with pytest.raises(TimeoutError, match="exceeded the 1-second child timeout"): + _run_prepared_benchmark_isolated( + run, + output_dir="/tmp/unused", + run_id="hung", + timeout_seconds=1, + ) + + assert process.started is True + assert process.terminated is True + assert process.killed is True + assert process.join_timeouts == [30, 30] + assert receive_connection.poll_timeout == 1 + assert receive_connection.closed is True + assert send_connection.closed is True + + def test_run_files_spawned_child_writes_terminal_summary(tmp_path): documents = tmp_path / "documents" documents.mkdir() diff --git a/ops/retriever-nightly/README.md b/ops/retriever-nightly/README.md index baef1ead3b..f2e7649d02 100644 --- a/ops/retriever-nightly/README.md +++ b/ops/retriever-nightly/README.md @@ -326,7 +326,9 @@ per domain. Per-domain throughput and timing remain in the session artifacts. If one runtime child fails, `run-files` continues the remaining datasets and writes a failed session summary. When Slack is configured, the launcher still attempts one report and returns the harness status. If the harness succeeds but -Slack fails, it returns the Slack command's nonzero status. +Slack fails, it returns the Slack command's nonzero status. Process-isolated +children also have a six-hour wall-time limit; a child that exceeds it is +terminated, recorded as failed, and does not prevent later datasets from running. The launcher defaults `VLLM_DEEP_GEMM_WARMUP=skip` unless the caller explicitly sets another vLLM-supported mode. This skips the optional compatibility-sensitive From 38e1a36bff5d46866195591b0e7a77e7ac63499f Mon Sep 17 00:00:00 2001 From: Jacob Ioffe Date: Tue, 21 Jul 2026 23:37:02 +0000 Subject: [PATCH 10/11] refactor(harness): make run isolation automatic Signed-off-by: Jacob Ioffe --- .../harness_retriever_ingest_query_prd.md | 14 ++--- nemo_retriever/harness/README.md | 12 ++-- .../src/nemo_retriever/harness/cli.py | 16 +---- .../src/nemo_retriever/harness/runsets.py | 20 +++---- .../tests/test_harness_nightly_launcher.py | 4 +- nemo_retriever/tests/test_harness_runfiles.py | 58 +++++++++++++------ .../SECOND_HOST_VALIDATION.md | 3 +- ops/retriever-nightly/run-nightly.sh | 3 - 8 files changed, 66 insertions(+), 64 deletions(-) diff --git a/nemo_retriever/developer_docs/harness_retriever_ingest_query_prd.md b/nemo_retriever/developer_docs/harness_retriever_ingest_query_prd.md index a61f001159..fc4daba6a9 100644 --- a/nemo_retriever/developer_docs/harness_retriever_ingest_query_prd.md +++ b/nemo_retriever/developer_docs/harness_retriever_ingest_query_prd.md @@ -4,11 +4,11 @@ Last updated: 2026-07-21 ## Implementation Status -The current implementation includes the core runner described here plus four +The current implementation includes the core runner described here plus three orchestration-neutral extensions: `run-files` applies a machine-local dataset -path map to one or more checked-in runfiles; `--isolate-runs` gives each child a -fresh process; `check-vidore-access` validates remote ViDoRe evaluation data; -and `post-slack` renders or posts completed artifacts. It does not include +path map to one or more checked-in runfiles and gives each real child a fresh +process; `check-vidore-access` validates remote ViDoRe evaluation data; and +`post-slack` renders or posts completed artifacts. It does not include recurring scheduling, deployment, locking, retry policy, or secret distribution. The harness README is the normative user and agent guide; this PRD records the design rationale. @@ -738,9 +738,9 @@ validation through the CLI and artifact contract. - `retriever harness run-set ` expands a code-owned ablation and writes `expanded_runs.json`. - `retriever harness run-files` runs one or more checked-in requests with an - optional machine-local dataset path map. -- `retriever harness run-files --isolate-runs` executes each child in a fresh - process while retaining one terminal session summary. + optional machine-local dataset path map. Real children execute sequentially + in fresh processes while retaining one terminal session summary; dry-runs + stay in the parent process. - `retriever harness check-vidore-access` validates authenticated access to the ViDoRe queries, qrels, and corpus partitions without downloading them. - `retriever harness post-slack --preview` reads artifacts without requiring a diff --git a/nemo_retriever/harness/README.md b/nemo_retriever/harness/README.md index 9cef6533e0..718d42f2b9 100644 --- a/nemo_retriever/harness/README.md +++ b/nemo_retriever/harness/README.md @@ -207,12 +207,12 @@ default dataset paths are mounted. Prefer the checked-in runfiles for nightly or other orchestrated sessions because they carry per-dataset integrity gates and accept a machine-local path map. -Add `--isolate-runs` to a real multi-run `run-files` session when each child -should execute in a fresh spawned process. The process boundary releases Ray -and materialized dataframe memory before the next benchmark while preserving -one parent-owned `session_summary.json`. Isolated children have a six-hour -wall-time limit by default so one hung child cannot block the session forever; -use `--child-timeout-seconds` to adjust that limit for an unusually long run. +Real `run-files` sessions execute children sequentially, each in a fresh spawned +process. The process boundary releases Ray and materialized dataframe memory +before the next benchmark while preserving one parent-owned +`session_summary.json`. A code-owned six-hour child deadline prevents one hung +benchmark from blocking the session forever. Dry-runs stay in the parent +process because they do not materialize batch data. `run-files` owns the session layout and execution mode. Runfiles passed to this command cannot set their own `output_dir`, `run_id`, or `dry_run`; use the diff --git a/nemo_retriever/src/nemo_retriever/harness/cli.py b/nemo_retriever/src/nemo_retriever/harness/cli.py index acafc9f79a..36fce36b58 100644 --- a/nemo_retriever/src/nemo_retriever/harness/cli.py +++ b/nemo_retriever/src/nemo_retriever/harness/cli.py @@ -24,7 +24,7 @@ from nemo_retriever.harness.diff import diff_artifact_dirs from nemo_retriever.harness.resolution import make_run_id from nemo_retriever.harness.runfile import load_runfile -from nemo_retriever.harness.runsets import DEFAULT_ISOLATED_CHILD_TIMEOUT_SECONDS, run_runfiles, run_runset +from nemo_retriever.harness.runsets import run_runfiles, run_runset from nemo_retriever.harness.slack import ( DEFAULT_SLACK_METRIC_KEYS, build_slack_payload, @@ -300,18 +300,6 @@ def run_files_command( bool, typer.Option("--dry-run", help="Resolve configuration and write plans without executing ingest or query."), ] = False, - isolate_runs: Annotated[ - bool, - typer.Option("--isolate-runs", help="Run each child in a fresh process to release batch memory."), - ] = False, - child_timeout_seconds: Annotated[ - float, - typer.Option( - "--child-timeout-seconds", - min=1.0, - help="Maximum wall time for each isolated child before it is terminated.", - ), - ] = DEFAULT_ISOLATED_CHILD_TIMEOUT_SECONDS, json_output: Annotated[bool, typer.Option("--json", help="Emit session summary JSON to stdout.")] = False, ) -> None: """Run one or more runfiles, optionally with machine-local dataset paths.""" @@ -326,8 +314,6 @@ def run_files_command( overrides=set_values or (), requirements=requirements or (), dry_run=dry_run, - isolate_runs=isolate_runs, - isolated_child_timeout_seconds=child_timeout_seconds, ) except HarnessRunError as exc: typer.echo(exc.failure.message, err=True) diff --git a/nemo_retriever/src/nemo_retriever/harness/runsets.py b/nemo_retriever/src/nemo_retriever/harness/runsets.py index 82b60aca25..e46a4149ea 100644 --- a/nemo_retriever/src/nemo_retriever/harness/runsets.py +++ b/nemo_retriever/src/nemo_retriever/harness/runsets.py @@ -29,7 +29,7 @@ from nemo_retriever.harness.runfile import load_runfile _SESSION_LABEL_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.-]*$") -DEFAULT_ISOLATED_CHILD_TIMEOUT_SECONDS = 6 * 60 * 60 +_ISOLATED_CHILD_TIMEOUT_SECONDS = 6 * 60 * 60 _ISOLATED_PROCESS_STOP_TIMEOUT_SECONDS = 30 @@ -255,7 +255,6 @@ def _run_prepared_benchmark_isolated( *, output_dir: str, run_id: str, - timeout_seconds: float = DEFAULT_ISOLATED_CHILD_TIMEOUT_SECONDS, ) -> RunOutcome: """Run one child with a process boundary that releases Ray and dataframe memory.""" @@ -281,8 +280,11 @@ def _run_prepared_benchmark_isolated( message: tuple[str, Any] | None = None try: - if not receive_connection.poll(timeout_seconds): - raise TimeoutError(f"isolated benchmark {run.name!r} exceeded the {timeout_seconds:g}-second child timeout") + if not receive_connection.poll(_ISOLATED_CHILD_TIMEOUT_SECONDS): + raise TimeoutError( + f"isolated benchmark {run.name!r} exceeded the " + f"{_ISOLATED_CHILD_TIMEOUT_SECONDS:g}-second child timeout" + ) try: message = receive_connection.recv() except EOFError: @@ -348,7 +350,6 @@ def _run_session( dry_run: bool, summary_extra: Mapping[str, Any], isolate_runs: bool = False, - isolated_child_timeout_seconds: float = DEFAULT_ISOLATED_CHILD_TIMEOUT_SECONDS, ) -> RunOutcome: try: session_dir.mkdir(parents=True, exist_ok=True) @@ -368,7 +369,6 @@ def _run_session( run, output_dir=str(artifact_dir), run_id=run_id, - timeout_seconds=isolated_child_timeout_seconds, ) else: outcome = run_prepared_benchmark( @@ -478,13 +478,9 @@ def run_runfiles( overrides: Sequence[str] = (), requirements: Sequence[str] = (), dry_run: bool = False, - isolate_runs: bool = False, - isolated_child_timeout_seconds: float = DEFAULT_ISOLATED_CHILD_TIMEOUT_SECONDS, ) -> RunOutcome: if not runfiles: raise _invalid_runfile("At least one runfile path is required.") - if isolated_child_timeout_seconds <= 0: - raise _invalid_runfile("--child-timeout-seconds must be greater than zero.") session_name = _validate_session_label(session_name, field="--session-name") run_commit = last_commit() @@ -543,6 +539,7 @@ def run_runfiles( ) dataset_paths_value = str(dataset_paths_file.expanduser().resolve()) if dataset_paths_file else None + isolate_runs = not dry_run return _run_session( session_type="runfiles", session_name=session_name, @@ -554,7 +551,6 @@ def run_runfiles( "working_tree_dirty": source_worktree_dirty, "dataset_paths_file": dataset_paths_value, "isolate_runs": bool(isolate_runs), - "isolated_child_timeout_seconds": isolated_child_timeout_seconds if isolate_runs else None, "runfiles": expanded_runs, }, run_commit=run_commit, @@ -563,9 +559,7 @@ def run_runfiles( "session_name": session_name, "dataset_paths_file": dataset_paths_value, "isolate_runs": bool(isolate_runs), - "isolated_child_timeout_seconds": isolated_child_timeout_seconds if isolate_runs else None, "working_tree_dirty": source_worktree_dirty, }, isolate_runs=isolate_runs, - isolated_child_timeout_seconds=isolated_child_timeout_seconds, ) diff --git a/nemo_retriever/tests/test_harness_nightly_launcher.py b/nemo_retriever/tests/test_harness_nightly_launcher.py index b94339f93f..52d672751c 100644 --- a/nemo_retriever/tests/test_harness_nightly_launcher.py +++ b/nemo_retriever/tests/test_harness_nightly_launcher.py @@ -208,6 +208,7 @@ def test_manual_dry_run_launches_full_batch_suite_without_slack(nightly_launcher assert invocation[invocation.index("--mode") + 1] == "batch" assert "--dry-run" in invocation assert "--isolate-runs" not in invocation + assert "--child-timeout-seconds" not in invocation assert tuple(invocation[-len(DEFAULT_RUNFILES) :]) == DEFAULT_RUNFILES @@ -259,7 +260,8 @@ def test_configured_webhook_posts_terminal_session_to_slack(nightly_launcher, tm assert len(calls()) == 2 run_invocation, post_invocation = calls() session_dir = Path(run_invocation[run_invocation.index("--output-dir") + 1]) - assert "--isolate-runs" in run_invocation + assert "--isolate-runs" not in run_invocation + assert "--child-timeout-seconds" not in run_invocation assert "post-slack" in post_invocation assert post_invocation[-1] == str(session_dir) assert (session_dir / ".slack_post_attempted").exists() diff --git a/nemo_retriever/tests/test_harness_runfiles.py b/nemo_retriever/tests/test_harness_runfiles.py index c360420f9d..6c8a4878a5 100644 --- a/nemo_retriever/tests/test_harness_runfiles.py +++ b/nemo_retriever/tests/test_harness_runfiles.py @@ -85,6 +85,7 @@ def fake_run_benchmark(prepared, **kwargs): dataset_paths_file=dataset_paths, overrides=("query.top_k=20",), requirements=("pages==1940",), + dry_run=True, ) prepared, kwargs = calls[0] @@ -197,7 +198,9 @@ def fake_run_benchmark(prepared, **kwargs): assert [run["exit_code"] for run in outcome.results["runs"]] == [10, 0] -def test_run_files_can_isolate_each_child_process(monkeypatch, tmp_path): +def test_real_run_files_automatically_isolate_each_child_process(monkeypatch, tmp_path): + from nemo_retriever.harness.execution import PreparedBenchmark + runfiles = [] for name in ("jp20_beir", "bo767_beir"): path = tmp_path / f"{name}.json" @@ -206,10 +209,22 @@ def test_run_files_can_isolate_each_child_process(monkeypatch, tmp_path): isolated_calls = [] - def fake_isolated_run(run, *, output_dir, run_id, timeout_seconds): - isolated_calls.append((run.prepared.benchmark, run_id, timeout_seconds)) + def fake_preflight(benchmark, **kwargs): + return PreparedBenchmark( + benchmark=benchmark, + mode=kwargs["mode"], + overrides=tuple(kwargs["overrides"]), + requirements=tuple(kwargs["requirements"]), + dry_run=kwargs["dry_run"], + resolved={"dataset": {"name": benchmark}, "ingest": {}}, + dataset_path=tmp_path, + ) + + def fake_isolated_run(run, *, output_dir, run_id): + isolated_calls.append((run.prepared.benchmark, run_id)) return _successful_outcome(run.prepared.benchmark, output_dir) + monkeypatch.setattr("nemo_retriever.harness.runsets.preflight_benchmark", fake_preflight) monkeypatch.setattr( "nemo_retriever.harness.runsets._run_prepared_benchmark_isolated", fake_isolated_run, @@ -224,20 +239,20 @@ def fake_isolated_run(run, *, output_dir, run_id, timeout_seconds): output_dir=str(tmp_path / "session"), session_name="isolated", overrides=(f'dataset.path="{tmp_path}"',), - dry_run=True, - isolate_runs=True, ) assert outcome.exit_code == 0 assert outcome.results["isolate_runs"] is True assert isolated_calls == [ - ("jp20_beir", "isolated_001_jp20_beir", 6 * 60 * 60), - ("bo767_beir", "isolated_002_bo767_beir", 6 * 60 * 60), + ("jp20_beir", "isolated_001_jp20_beir"), + ("bo767_beir", "isolated_002_bo767_beir"), ] assert len(outcome.results["runs"]) == 2 def test_run_files_isolated_timeout_records_failure_and_continues(monkeypatch, tmp_path): + from nemo_retriever.harness.execution import PreparedBenchmark + runfiles = [] for name in ("jp20_beir", "bo767_beir"): path = tmp_path / f"{name}.json" @@ -246,12 +261,24 @@ def test_run_files_isolated_timeout_records_failure_and_continues(monkeypatch, t isolated_calls = [] - def fake_isolated_run(run, *, output_dir, run_id, timeout_seconds): - isolated_calls.append((run.prepared.benchmark, timeout_seconds)) + def fake_preflight(benchmark, **kwargs): + return PreparedBenchmark( + benchmark=benchmark, + mode=kwargs["mode"], + overrides=tuple(kwargs["overrides"]), + requirements=tuple(kwargs["requirements"]), + dry_run=kwargs["dry_run"], + resolved={"dataset": {"name": benchmark}, "ingest": {}}, + dataset_path=tmp_path, + ) + + def fake_isolated_run(run, *, output_dir, run_id): + isolated_calls.append(run.prepared.benchmark) if run.prepared.benchmark == "jp20_beir": raise TimeoutError("isolated benchmark exceeded the child timeout") return _successful_outcome(run.prepared.benchmark, output_dir) + monkeypatch.setattr("nemo_retriever.harness.runsets.preflight_benchmark", fake_preflight) monkeypatch.setattr( "nemo_retriever.harness.runsets._run_prepared_benchmark_isolated", fake_isolated_run, @@ -261,16 +288,12 @@ def fake_isolated_run(run, *, output_dir, run_id, timeout_seconds): runfiles, output_dir=str(tmp_path / "session"), overrides=(f'dataset.path="{tmp_path}"',), - dry_run=True, - isolate_runs=True, - isolated_child_timeout_seconds=17, ) assert outcome.exit_code == EXIT_INTERNAL_ERROR - assert isolated_calls == [("jp20_beir", 17), ("bo767_beir", 17)] + assert isolated_calls == ["jp20_beir", "bo767_beir"] assert [run["success"] for run in outcome.results["runs"]] == [False, True] assert "TimeoutError" in outcome.results["runs"][0]["failure_reason"] - assert outcome.results["isolated_child_timeout_seconds"] == 17 def test_isolated_child_timeout_uses_bounded_terminate_and_kill(monkeypatch): @@ -334,6 +357,7 @@ def Process(**_kwargs): "nemo_retriever.harness.runsets.multiprocessing.get_context", lambda method: FakeContext(), ) + monkeypatch.setattr("nemo_retriever.harness.runsets._ISOLATED_CHILD_TIMEOUT_SECONDS", 1) run = SimpleNamespace(name="hung", artifact_name="hung") with pytest.raises(TimeoutError, match="exceeded the 1-second child timeout"): @@ -341,7 +365,6 @@ def Process(**_kwargs): run, output_dir="/tmp/unused", run_id="hung", - timeout_seconds=1, ) assert process.started is True @@ -353,7 +376,7 @@ def Process(**_kwargs): assert send_connection.closed is True -def test_run_files_spawned_child_writes_terminal_summary(tmp_path): +def test_run_files_dry_run_stays_in_process_and_writes_terminal_summary(tmp_path): documents = tmp_path / "documents" documents.mkdir() (documents / "sample.pdf").write_bytes(b"%PDF-1.4\n%%EOF\n") @@ -381,11 +404,10 @@ def test_run_files_spawned_child_writes_terminal_summary(tmp_path): dataset_paths_file=dataset_paths, mode="batch", dry_run=True, - isolate_runs=True, ) assert outcome.exit_code == 0 - assert outcome.results["isolate_runs"] is True + assert outcome.results["isolate_runs"] is False assert outcome.results["runs"][0]["success"] is True assert (outcome.artifact_dir / "001_jp20_beir" / "results.json").exists() assert outcome.results_path == outcome.artifact_dir / "session_summary.json" diff --git a/ops/retriever-nightly/SECOND_HOST_VALIDATION.md b/ops/retriever-nightly/SECOND_HOST_VALIDATION.md index 01e6fe298d..c841dc4a32 100644 --- a/ops/retriever-nightly/SECOND_HOST_VALIDATION.md +++ b/ops/retriever-nightly/SECOND_HOST_VALIDATION.md @@ -99,7 +99,8 @@ the dry-run does not materialize batch data. ``` Confirm that the command exits zero and the session summary contains one -successful run with `isolate_runs: true`. The launcher defaults the optional +successful run with `isolate_runs: true`. Real `run-files` sessions isolate +each sequential child automatically. The launcher defaults the optional DeepGEMM warmup to `skip`; no host setting is needed. ## 6. Capture the Handoff Evidence diff --git a/ops/retriever-nightly/run-nightly.sh b/ops/retriever-nightly/run-nightly.sh index df1dfdc3c6..017c924c3f 100755 --- a/ops/retriever-nightly/run-nightly.sh +++ b/ops/retriever-nightly/run-nightly.sh @@ -515,17 +515,14 @@ cd "$checkout" || exit "$EXIT_CONFIG" run_rc=0 dry_run_args=() -isolation_args=(--isolate-runs) if ((dry_run)); then dry_run_args=(--dry-run) - isolation_args=() fi "$uv_bin" run --frozen --project nemo_retriever retriever harness run-files \ --session-name "$session_name" \ --output-dir "$session_dir" \ --dataset-paths "$dataset_paths" \ --mode "$run_mode" \ - "${isolation_args[@]}" \ "${dry_run_args[@]}" \ "${runfiles[@]}" || run_rc=$? From bd06d60a8846784dab22231ee4c4ffabffb97347 Mon Sep 17 00:00:00 2001 From: Jacob Ioffe Date: Wed, 22 Jul 2026 16:46:47 +0000 Subject: [PATCH 11/11] fix(harness): preserve received isolated outcomes Signed-off-by: Jacob Ioffe --- .../src/nemo_retriever/harness/runsets.py | 4 +- nemo_retriever/tests/test_harness_runfiles.py | 65 +++++++++++++++++++ 2 files changed, 67 insertions(+), 2 deletions(-) diff --git a/nemo_retriever/src/nemo_retriever/harness/runsets.py b/nemo_retriever/src/nemo_retriever/harness/runsets.py index e46a4149ea..ffb85da1f3 100644 --- a/nemo_retriever/src/nemo_retriever/harness/runsets.py +++ b/nemo_retriever/src/nemo_retriever/harness/runsets.py @@ -302,9 +302,9 @@ def _run_prepared_benchmark_isolated( f"isolated benchmark {run.name!r} returned an outcome but did not exit within " f"{_ISOLATED_PROCESS_STOP_TIMEOUT_SECONDS} seconds" ) - if process.exitcode != 0: - raise RuntimeError(f"isolated benchmark process exited with code {process.exitcode}") if message is None: + if process.exitcode != 0: + raise RuntimeError(f"isolated benchmark process exited with code {process.exitcode}") raise RuntimeError("isolated benchmark process returned no outcome") message_type, payload = message diff --git a/nemo_retriever/tests/test_harness_runfiles.py b/nemo_retriever/tests/test_harness_runfiles.py index 6c8a4878a5..9f2d454543 100644 --- a/nemo_retriever/tests/test_harness_runfiles.py +++ b/nemo_retriever/tests/test_harness_runfiles.py @@ -5,6 +5,7 @@ import json from pathlib import Path from types import SimpleNamespace +from unittest.mock import Mock import pytest @@ -296,6 +297,70 @@ def fake_isolated_run(run, *, output_dir, run_id): assert "TimeoutError" in outcome.results["runs"][0]["failure_reason"] +def _fake_exited_isolated_child(monkeypatch, *, message, exitcode): + receive_connection = Mock() + receive_connection.poll.return_value = True + if message is None: + receive_connection.recv.side_effect = EOFError + else: + receive_connection.recv.return_value = message + send_connection = Mock() + process = Mock(exitcode=exitcode) + process.is_alive.return_value = False + context = Mock() + context.Pipe.return_value = receive_connection, send_connection + context.Process.return_value = process + + monkeypatch.setattr( + "nemo_retriever.harness.runsets.multiprocessing.get_context", + lambda method: context, + ) + return process, receive_connection, send_connection + + +def test_isolated_child_returns_valid_outcome_after_nonzero_process_exit(monkeypatch, tmp_path): + expected = _successful_outcome("jp20_beir", str(tmp_path / "jp20")) + process, receive_connection, send_connection = _fake_exited_isolated_child( + monkeypatch, + message=("outcome", expected), + exitcode=1, + ) + + run = SimpleNamespace(name="jp20_beir", artifact_name="jp20_beir") + outcome = _run_prepared_benchmark_isolated( + run, + output_dir=str(tmp_path), + run_id="jp20_beir", + ) + + assert outcome is expected + process.start.assert_called_once_with() + process.join.assert_called_once_with(timeout=30) + receive_connection.close.assert_called_once_with() + send_connection.close.assert_called_once_with() + + +def test_isolated_child_reports_nonzero_process_exit_without_outcome(monkeypatch, tmp_path): + process, receive_connection, send_connection = _fake_exited_isolated_child( + monkeypatch, + message=None, + exitcode=1, + ) + + run = SimpleNamespace(name="jp20_beir", artifact_name="jp20_beir") + with pytest.raises(RuntimeError, match="isolated benchmark process exited with code 1"): + _run_prepared_benchmark_isolated( + run, + output_dir=str(tmp_path), + run_id="jp20_beir", + ) + + process.start.assert_called_once_with() + process.join.assert_called_once_with(timeout=30) + receive_connection.close.assert_called_once_with() + send_connection.close.assert_called_once_with() + + def test_isolated_child_timeout_uses_bounded_terminate_and_kill(monkeypatch): class FakeConnection: def __init__(self, *, readable):