From d7e73f493b4293d38f5521fdf689a934e8d87367 Mon Sep 17 00:00:00 2001 From: Jiwen Cai Date: Fri, 17 Jul 2026 04:46:33 +0000 Subject: [PATCH 1/2] Add isaacteleop.rig: YAML-driven tmux teleop-rig launcher MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace scripts/run_se3_demo.sh with a generic, wheel-shipped launcher module that brings up a teleop rig — the configured set of CloudXR runtime + producer plugins + consumer apps — in a tmux session from a declarative YAML rig file. The same rig files serve demos, production teleop, and data collection. The rig file is the single source of configuration: params live in its params: block and are substituted into every command that references them; there are deliberately no CLI override knobs (edit the file and relaunch to change them). - isaacteleop.rig (src/core/rig/python/): config.py (schema/loader: unknown top-level and entry keys are hard errors, {key} substitution from params with {python} -> sys.executable, tmux-safe name validation, warn-only lint for Python consumers that would self-launch a second CloudXR runtime), launcher.py (pane orchestration behind a single injectable run_tmux seam), __main__.py (positional rig YAML, --no-runtime, --kill; bare invocation prints usage plus an example, never a stack trace). Public API: RigConfig, load_rig_config, launch_rig, kill_rig. - Pane orchestration: the runtime pane starts immediately; producer/consumer panes wait for the runtime, load its env, then auto-run their command; pane-id addressing; idempotent session reuse with a kill hint; $TMUX switch-client vs attach; PYTHONPATH forwarding; per-pane titles with session-scoped pane-border-status; pre-attach instructions. - Layout: tmux main-horizontal with a session-scoped main-pane-height of 25% (RUNTIME_PANE_HEIGHT) — the runtime is pane 0 and becomes a slim full-width strip on top (it only prints status lines and the web-client URL); worker panes tile in the remaining 75%. Panes are re-laid out tiled after every split so chained splits never hit tmux's "pane too small" limit at any rig size; under --no-runtime all panes are peer workers and simply stay tiled. - Worker panes auto-load the CloudXR env, then auto-run: each pane's spawn command is a POSIX wrapper (never typed input; stty echo bracketing keeps machinery invisible) that polls for the runtime_started sentinel (recreated only once the runtime actually serves), sources /cloudxr.env, prints a status line and an auto-run banner, and runs the rig command via sh -c with a scoped 'trap : INT' (Ctrl+C kills the app, not the pane; a command syntax error cannot kill the wrapper). On command exit it prints the exit status and drops to the user's shell with the same command pre-typed (buffered via send-keys -l into its own tty while echo is off) — so an app that exited early, e.g. started before the headset connected, reruns with one Enter. On sentinel timeout (120 s) the command is not run: the wrapper prints the manual-source remedy and pre-types the command instead. The run dir follows the runtime command's --cloudxr-install-dir when present, else $CXR_INSTALL_DIR, else ~/.cloudxr, mirroring the runtime's own EnvConfig resolution. Workers behave identically under --no-runtime. - --kill tears down a rig's session by rig file (kill_rig() also exported for programmatic use); idempotent, rejects --no-runtime; the session-reuse hint and pre-attach instructions point at --kill first with the raw tmux one-liner as the alternative. - rigs/se3_tracker.yaml: first rig file, annotated as the schema exemplar (collection_id defined once under params and referenced by both sides — a mismatch is silent no-data; NOTE about --no-launch-cloudxr-runtime for Python TeleopSession consumers). Rig files are checkout artifacts and are not shipped in the wheel. - docs: Rig Launcher reference page (prerequisites, running a rig, --no-runtime/--kill, auto-run + exit-status + Enter-to-rerun behavior, rig YAML schema key reference, collection_id single-sourcing rule, self-launching-runtime warning, troubleshooting table), registered in the References toctree after the CloudXR runtime page. - Packaging: rig_python copy target wired into python_package and the pyproject packages list; pyyaml>=6.0.3 added to the base requirements. - Tests: src/core/rig_tests/python — pytest cases against a recording FakeTmux and the shipped rig file (including regression tests that pane machinery is never typed and never echoes, that commands run only when the runtime is ready, and that Ctrl+C kills the app not the pane), registered as ctest entries; no tmux or headset needed. - Delete scripts/run_se3_demo.sh (the runbook in the meta-repo docs/ is updated to the new invocation separately). pytest 58/58, ctest rig-filtered 5/5, pre-commit clean, Sphinx -W green; pane bring-up smoke-tested in real tmux under bash and zsh (auto-run, exit-and-rerun, and timeout captures) and end to end against a live session. Signed-off-by: Jiwen Cai --- docs/source/index.rst | 1 + docs/source/references/rig.rst | 213 +++++++ rigs/se3_tracker.yaml | 30 + src/core/CMakeLists.txt | 8 + src/core/python/CMakeLists.txt | 3 +- src/core/python/pyproject.toml.in | 1 + src/core/python/requirements.txt | 1 + src/core/rig/CMakeLists.txt | 6 + src/core/rig/python/CMakeLists.txt | 36 ++ src/core/rig/python/__init__.py | 41 ++ src/core/rig/python/__main__.py | 89 +++ src/core/rig/python/config.py | 293 ++++++++++ src/core/rig/python/launcher.py | 561 ++++++++++++++++++ src/core/rig_tests/CMakeLists.txt | 9 + src/core/rig_tests/python/CMakeLists.txt | 29 + src/core/rig_tests/python/conftest.py | 48 ++ src/core/rig_tests/python/pyproject.toml | 22 + src/core/rig_tests/python/test_config.py | 245 ++++++++ src/core/rig_tests/python/test_launcher.py | 624 +++++++++++++++++++++ src/core/rig_tests/python/test_main.py | 39 ++ 20 files changed, 2298 insertions(+), 1 deletion(-) create mode 100644 docs/source/references/rig.rst create mode 100644 rigs/se3_tracker.yaml create mode 100644 src/core/rig/CMakeLists.txt create mode 100644 src/core/rig/python/CMakeLists.txt create mode 100644 src/core/rig/python/__init__.py create mode 100644 src/core/rig/python/__main__.py create mode 100644 src/core/rig/python/config.py create mode 100644 src/core/rig/python/launcher.py create mode 100644 src/core/rig_tests/CMakeLists.txt create mode 100644 src/core/rig_tests/python/CMakeLists.txt create mode 100644 src/core/rig_tests/python/conftest.py create mode 100644 src/core/rig_tests/python/pyproject.toml create mode 100644 src/core/rig_tests/python/test_config.py create mode 100644 src/core/rig_tests/python/test_launcher.py create mode 100644 src/core/rig_tests/python/test_main.py diff --git a/docs/source/index.rst b/docs/source/index.rst index 4c6c12994..761eb8096 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -67,6 +67,7 @@ Table of Contents references/camera_streaming references/mcap_record_replay references/cloudxr + references/rig references/oob_teleop_control references/egocentric_hand_reconstruction references/license diff --git a/docs/source/references/rig.rst b/docs/source/references/rig.rst new file mode 100644 index 000000000..f77143fd6 --- /dev/null +++ b/docs/source/references/rig.rst @@ -0,0 +1,213 @@ +.. SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +.. SPDX-License-Identifier: Apache-2.0 + +.. _rig-launcher: + +Rig Launcher +============ + +A typical Isaac Teleop setup is three processes on one machine: the CloudXR +runtime, one or more **producer** plugins that publish device data, and one or +more **consumer** apps (a Python ``TeleopSession`` script or a C++ example +binary) that read the streams. Such a configured set is a *rig* — the same +shape serves demos, production teleop, and data collection. +``isaacteleop.rig`` starts a rig in a single tmux session from a small YAML +file, instead of you juggling three terminals by hand: + +.. code-block:: bash + + # from the Teleop repository root + python -m isaacteleop.rig rigs/se3_tracker.yaml + +The module ships in the ``isaacteleop`` wheel; the rig files are part of the +source checkout under ``rigs/`` (they reference ``install/`` binaries, so they +only make sense next to a built tree). + +.. contents:: Sections + :local: + :depth: 1 + :backlinks: none + +Prerequisites +------------- + +- ``tmux`` installed (``sudo apt install tmux``). +- A built and installed Teleop tree (see :ref:`install-isaacteleop-pip-package` + and the build reference) — the rig references binaries under + ``install/plugins/`` and ``install/examples/``. +- The ``isaacteleop`` wheel installed in the **current** Python environment. + tmux panes do not inherit your venv; the launcher bakes the absolute path of + its own interpreter (and your ``PYTHONPATH``, if set) into the pane commands, + so whatever environment you launch from is the one the rig runs in. + +Run a rig +--------- + +.. code-block:: bash + + # from the Teleop repository root + python -m isaacteleop.rig rigs/se3_tracker.yaml + + # a CloudXR runtime is already running elsewhere (skip the runtime pane): + python -m isaacteleop.rig rigs/se3_tracker.yaml --no-runtime + +What happens: + +1. **Preflight** — the launcher verifies tmux is available, the referenced + binaries exist and are executable (with the exact ``cmake`` remedy if not), + and the interpreter can import ``isaacteleop.cloudxr``. Nothing is created + until preflight passes. +2. **Runtime pane starts immediately** (a slim full-width strip on top — + 25% of the window; it only prints status lines — with the worker panes + tiled below). It runs ``python -m isaacteleop.cloudxr --accept-eula`` by + default and prints the web-client URL. The runtime is a **host + singleton** — one per machine. +3. **Worker panes load the CloudXR environment automatically.** Each + producer/consumer pane waits (up to two minutes) for the runtime's + ``runtime_started`` sentinel, then runs + ``source /run/cloudxr.env`` so ``XR_RUNTIME_JSON`` and + friends point at the runtime — you never source it by hand. The install + dir follows the runtime command's ``--cloudxr-install-dir`` (default + ``~/.cloudxr``). +4. **Producer/consumer panes then run their commands automatically.** As + soon as the CloudXR environment is loaded, each pane prints a banner + (``[producer: ...] running: ``) and runs its command — no + :kbd:`Enter` needed. When the command exits, the pane reports + ``[rig] command exited with status N — press Enter to rerun`` and drops + to an interactive shell with the same command pre-typed at the prompt. + Anything that calls ``xrGetSystem`` before a headset connects exits with + ``Failed to get OpenXR system`` — so if a pane's app started before you + connected the headset to the printed URL, connect it and press + :kbd:`Enter` in that pane to rerun. If the runtime never comes up, the + pane does *not* run the command; it prints a remedy and leaves the + command pre-typed instead. +5. The launcher then attaches (from a plain shell) or switches your current + client (from inside tmux — no nesting). + +Re-running the same rig just switches to the existing session; it does +**not** re-apply ``--no-runtime`` or pick up edits to the rig file. Start +over with: + +.. code-block:: bash + + python -m isaacteleop.rig rigs/se3_tracker.yaml --kill + +which kills the rig's tmux session and every process in it (equivalent to +``tmux kill-session -t se3_tracker``, without needing to know the session +name). Killing a rig that is not running is a no-op. + +The rig YAML +------------ + +``rigs/se3_tracker.yaml`` is the annotated exemplar — copy it to write your +own: + +.. code-block:: yaml + + name: se3_tracker # rig id AND tmux session name (letters/digits/-/_) + description: CloudXR runtime + SE3 controller tracker plugin + pose printer + cwd: .. # pane working dir, relative to this file + params: # shared values, substituted into the commands below + hand: right + collection_id: se3_tracker # defined ONCE, referenced by both sides below + # runtime: optional full-command override for the runtime pane; default: + # {python} -m isaacteleop.cloudxr --accept-eula + producers: # publish device data into the runtime + - name: se3 tracker plugin (requires headset + controller) + command: "install/plugins/controller_se3_tracker/controller_se3_tracker_plugin {hand} {collection_id}" + consumers: # read the streams — a TeleopSession script or a C++ binary + - name: se3 printer (requires headset) + command: "install/examples/schemaio/se3_printer {collection_id}" + +Top-level keys: + +.. list-table:: + :header-rows: 1 + :widths: 18 12 70 + + * - Key + - Required + - Meaning + * - ``name`` + - yes + - Rig id **and** tmux session name. Letters, digits, ``-``, ``_`` only. + * - ``description`` + - no + - Free text, printed when the session is created. + * - ``cwd`` + - no + - Working directory for every pane and the base for relative command + paths, resolved relative to the YAML file's directory (default: the + YAML's directory). + * - ``params`` + - no + - Flat mapping of ``{placeholder}`` values shared by the commands + below — the rig file is the single source of configuration; edit it + (and relaunch with ``--kill``) to change them. + * - ``runtime`` + - no + - Full-command override for the runtime pane (e.g. to add + ``--host-client`` or a different device profile). Default: + ``{python} -m isaacteleop.cloudxr --accept-eula``. + * - ``producers`` / ``consumers`` + - at least one entry total + - Lists of ``{name, command}`` entries. ``name`` is free text shown in + the pane title (a good place for hardware prerequisites); ``command`` + is a shell string run verbatim in the pane. + +Commands are plain shell strings. ``{python}`` always expands to the absolute +path of the launching interpreter; every other ``{placeholder}`` must be +declared under ``params`` (literal braces are written ``{{`` / ``}}``). +Unknown top-level keys, unknown entry keys, and unknown placeholders are hard +errors — a typo fails loudly at load time instead of misbehaving in a pane. + +.. important:: + + Producers and consumers rendezvous on a shared ``collection_id``, and a + mismatch is **silent no-data** by design. Define it once under ``params`` + and reference it as ``{collection_id}`` in every command — then one edit + in one place changes both sides together. + +.. warning:: + + Python ``TeleopSession`` example scripts launch their **own** CloudXR + runtime by default (``--launch-cloudxr-runtime`` defaults to true). The + runtime is a host singleton, so auto-running such a consumer kills the + runtime pane — the headset drops and every producer stalls. Always add + ``--no-launch-cloudxr-runtime`` to Python consumer commands in a rig; the + launcher prints a warning (and repeats it in the pane banner) when a + command looks like it is missing the flag. + + See :ref:`dedicated-cloudxr-runtime` for more on standalone runtime + workflows. + +Troubleshooting +--------------- + +.. list-table:: + :header-rows: 1 + :widths: 45 55 + + * - Symptom + - Likely cause + * - ``... not found or not executable — build and install first`` + - The rig references ``install/`` binaries that are not built yet; run + the printed ``cmake`` commands. + * - ``cannot import 'isaacteleop.cloudxr'`` + - You launched from an environment without the ``isaacteleop`` wheel; + activate the right venv (or ``pip install install/wheels/isaacteleop-*.whl``). + * - A worker pane sits idle without running its command + - The pane is still waiting for the runtime's ``runtime_started`` + sentinel; check the runtime pane. After the (two-minute) wait times + out, the pane prints a remedy and leaves the command pre-typed. + * - ``[rig] command exited with status ...`` right after launch + - The app auto-ran before the headset connected (classic: + ``Failed to get OpenXR system``); connect the headset to the + runtime's URL, then press :kbd:`Enter` in the pane — the command is + already pre-typed at the prompt. + * - Runtime pane dies when a consumer starts + - The consumer self-launched a second runtime; add + ``--no-launch-cloudxr-runtime`` to its command in the rig. + * - Edits to the rig file seem ignored + - The session already existed; relaunch after + ``python -m isaacteleop.rig --kill``. diff --git a/rigs/se3_tracker.yaml b/rigs/se3_tracker.yaml new file mode 100644 index 000000000..ec1811cde --- /dev/null +++ b/rigs/se3_tracker.yaml @@ -0,0 +1,30 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Run with: python -m isaacteleop.rig rigs/se3_tracker.yaml +# +# The runtime pane starts immediately; producer/consumer panes wait for the +# runtime, load its CloudXR env, then run their commands automatically. An +# app that started before the headset connected exits with 'Failed to get +# OpenXR system' — press Enter in its pane to rerun it once the headset is in. +# Relative command paths and the pane working dir resolve against `cwd` +# (relative to this file; default: this file's directory). +name: se3_tracker # rig id AND tmux session name (letters/digits/-/_ only) +description: CloudXR runtime + SE3 controller tracker plugin + pose printer +cwd: .. # -> Teleop repo root +params: # shared values, substituted into the commands below + hand: right + collection_id: se3_tracker # producers and consumers rendezvous on this, defined ONCE + # here — a mismatch is SILENT no-data, so always reference + # it as {collection_id} instead of repeating the literal +# runtime: optional full-command override for the runtime pane; default: +# {python} -m isaacteleop.cloudxr --accept-eula +producers: # publish device data into the runtime + - name: se3 tracker plugin (requires headset + controller) + command: "install/plugins/controller_se3_tracker/controller_se3_tracker_plugin {hand} {collection_id}" +consumers: # read the streams — a TeleopSession script or a C++ binary + - name: se3 printer (requires headset) + command: "install/examples/schemaio/se3_printer {collection_id}" +# NOTE: Python TeleopSession consumers self-launch a CloudXR runtime by default. +# Add --no-launch-cloudxr-runtime to their command or they will KILL the runtime +# pane (the runtime is a host singleton on WSS port 48322). diff --git a/src/core/CMakeLists.txt b/src/core/CMakeLists.txt index 0f1e21f15..ad044a70a 100644 --- a/src/core/CMakeLists.txt +++ b/src/core/CMakeLists.txt @@ -47,6 +47,9 @@ add_subdirectory(teleop_session_manager) # CloudXR runtime helper (pure Python, docker-less) add_subdirectory(cloudxr) +# Rig launcher (pure Python, YAML-driven tmux teleop rigs) +add_subdirectory(rig) + # Python wheel packaging (combines both modules) if(BUILD_PYTHON_BINDINGS) add_subdirectory(python) @@ -71,6 +74,11 @@ if(BUILD_TESTING) add_subdirectory(cloudxr_tests) endif() + # Rig launcher tests (Python) + if(BUILD_PYTHON_BINDINGS) + add_subdirectory(rig_tests) + endif() + # MCAP tests (C++) add_subdirectory(mcap_tests/cpp) diff --git a/src/core/python/CMakeLists.txt b/src/core/python/CMakeLists.txt index a977ec3a8..694e2162b 100644 --- a/src/core/python/CMakeLists.txt +++ b/src/core/python/CMakeLists.txt @@ -59,6 +59,7 @@ add_custom_target(python_package ALL COMMAND ${CMAKE_COMMAND} -E make_directory "${CMAKE_BINARY_DIR}/python_package/$/isaacteleop/schema" COMMAND ${CMAKE_COMMAND} -E make_directory "${CMAKE_BINARY_DIR}/python_package/$/isaacteleop/teleop_session_manager" COMMAND ${CMAKE_COMMAND} -E make_directory "${CMAKE_BINARY_DIR}/python_package/$/isaacteleop/cloudxr" + COMMAND ${CMAKE_COMMAND} -E make_directory "${CMAKE_BINARY_DIR}/python_package/$/isaacteleop/rig" COMMAND ${CMAKE_COMMAND} -E make_directory "${CMAKE_BINARY_DIR}/python_package/$/isaacteleop/deviceio_trackers" COMMAND ${CMAKE_COMMAND} -E make_directory "${CMAKE_BINARY_DIR}/python_package/$/isaacteleop/deviceio_session" COMMAND ${CMAKE_COMMAND} -E make_directory "${CMAKE_BINARY_DIR}/python_package/$/isaacteleop/viz" @@ -82,7 +83,7 @@ add_custom_target(python_package ALL COMMAND ${CMAKE_COMMAND} -E copy "${CMAKE_CURRENT_SOURCE_DIR}/requirements-retargeters.txt" "${CMAKE_BINARY_DIR}/python_package/$/" COMMAND ${CMAKE_COMMAND} -E copy "${CMAKE_CURRENT_SOURCE_DIR}/requirements-retargeters-lite.txt" "${CMAKE_BINARY_DIR}/python_package/$/" COMMAND ${CMAKE_COMMAND} -E copy "${CMAKE_CURRENT_SOURCE_DIR}/requirements-grounding.txt" "${CMAKE_BINARY_DIR}/python_package/$/" - DEPENDS deviceio_trackers_py deviceio_session_py oxr_py plugin_manager_py schema_py retargeting_engine_python retargeters_python haptic_devices_python retargeting_engine_ui_python teleop_session_manager_python cloudxr_python + DEPENDS deviceio_trackers_py deviceio_session_py oxr_py plugin_manager_py schema_py retargeting_engine_python retargeters_python haptic_devices_python retargeting_engine_ui_python teleop_session_manager_python cloudxr_python rig_python COMMENT "Preparing Python package structure" ) diff --git a/src/core/python/pyproject.toml.in b/src/core/python/pyproject.toml.in index 1946d2808..c9baa4e01 100644 --- a/src/core/python/pyproject.toml.in +++ b/src/core/python/pyproject.toml.in @@ -55,6 +55,7 @@ packages = [ "isaacteleop.retargeting_engine_ui", "isaacteleop.teleop_session_manager", "isaacteleop.cloudxr", + "isaacteleop.rig", "isaacteleop.haptic_devices", @VIZ_PACKAGES_BLOCK@@ROBOTIC_GROUNDING_PACKAGES_BLOCK@] include-package-data = false diff --git a/src/core/python/requirements.txt b/src/core/python/requirements.txt index 3d6b003b9..a57b2934c 100644 --- a/src/core/python/requirements.txt +++ b/src/core/python/requirements.txt @@ -1 +1,2 @@ numpy>=1.23.0 +pyyaml>=6.0.3 diff --git a/src/core/rig/CMakeLists.txt b/src/core/rig/CMakeLists.txt new file mode 100644 index 000000000..c92466210 --- /dev/null +++ b/src/core/rig/CMakeLists.txt @@ -0,0 +1,6 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +if(BUILD_PYTHON_BINDINGS) + add_subdirectory(python) +endif() diff --git a/src/core/rig/python/CMakeLists.txt b/src/core/rig/python/CMakeLists.txt new file mode 100644 index 000000000..ea310e1f5 --- /dev/null +++ b/src/core/rig/python/CMakeLists.txt @@ -0,0 +1,36 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Rig Launcher Python Module (pure Python) + +# Determine output directory for wheel packaging +set(RIG_OUTPUT_DIR "${CMAKE_BINARY_DIR}/python_package/$/isaacteleop/rig") + +# List of Python files to copy +set(RIG_PYTHON_FILES + "${CMAKE_CURRENT_SOURCE_DIR}/__init__.py" + "${CMAKE_CURRENT_SOURCE_DIR}/__main__.py" + "${CMAKE_CURRENT_SOURCE_DIR}/config.py" + "${CMAKE_CURRENT_SOURCE_DIR}/launcher.py" +) + +# Copy Python files to output directory during build +foreach(PYTHON_FILE ${RIG_PYTHON_FILES}) + get_filename_component(FILE_NAME ${PYTHON_FILE} NAME) + set(OUTPUT_PATH "${RIG_OUTPUT_DIR}/${FILE_NAME}") + + add_custom_command( + OUTPUT "${OUTPUT_PATH}" + COMMAND ${CMAKE_COMMAND} -E copy "${PYTHON_FILE}" "${OUTPUT_PATH}" + DEPENDS "${PYTHON_FILE}" + COMMENT "Copying ${FILE_NAME} to rig package" + ) + + list(APPEND RIG_OUTPUT_FILES "${OUTPUT_PATH}") +endforeach() + +# Create custom target to ensure all files are copied +add_custom_target(rig_python ALL + DEPENDS ${RIG_OUTPUT_FILES} + COMMENT "Building rig Python module" +) diff --git a/src/core/rig/python/__init__.py b/src/core/rig/python/__init__.py new file mode 100644 index 000000000..af99cbf84 --- /dev/null +++ b/src/core/rig/python/__init__.py @@ -0,0 +1,41 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""YAML-driven tmux launcher for Isaac Teleop rigs. + +A *rig* is one teleop setup: the CloudXR runtime, one or more producer +plugins that publish device data, and one or more consumer apps (a Python +``TeleopSession`` script or a C++ binary) that read the streams. Rigs are +described by small YAML files (see ``rigs/se3_tracker.yaml`` in the Teleop +repository, and :mod:`~isaacteleop.rig.config` for the schema). + +Usage:: + + python -m isaacteleop.rig rigs/se3_tracker.yaml + +The runtime pane starts immediately; producer/consumer panes are pre-typed +but NOT executed — press Enter in each pane after the headset connects. + +The YAML schema is the stable contract; the Python API below is exported +for tests and power users on a best-effort basis. +""" + +from .config import ( + ProcessConfig, + RigConfig, + RigConfigError, + RigError, + load_rig_config, +) +from .launcher import PreflightError, kill_rig, launch_rig + +__all__ = [ + "PreflightError", + "ProcessConfig", + "RigConfig", + "RigConfigError", + "RigError", + "kill_rig", + "launch_rig", + "load_rig_config", +] diff --git a/src/core/rig/python/__main__.py b/src/core/rig/python/__main__.py new file mode 100644 index 000000000..329b79005 --- /dev/null +++ b/src/core/rig/python/__main__.py @@ -0,0 +1,89 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Entry point for ``python -m isaacteleop.rig``. + +Launches a teleop rig (CloudXR runtime + producer plugins + consumer apps) +in a tmux session from a YAML rig file. +""" + +from __future__ import annotations + +import argparse +import subprocess +import sys + +from .config import RigError, load_rig_config +from .launcher import kill_rig, launch_rig + +_EXAMPLE = ( + "example (from the Teleop repository root):\n" + " python -m isaacteleop.rig rigs/se3_tracker.yaml\n" + "rig files live in the Teleop repository under rigs/" +) + + +def _build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + prog="python -m isaacteleop.rig", + description="Launch a teleop rig (CloudXR runtime + producers + consumers) in tmux", + epilog=_EXAMPLE, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + parser.add_argument( + "rig", + nargs="?", + metavar="RIG_YAML", + help="Path to a rig file (e.g. rigs/se3_tracker.yaml).", + ) + parser.add_argument( + "--no-runtime", + action="store_true", + help=( + "Skip the runtime pane when a CloudXR runtime is already running " + "elsewhere. That external runtime is still a host singleton: " + "consumers must connect to it, not self-launch a runtime beside it." + ), + ) + parser.add_argument( + "--kill", + action="store_true", + help=( + "Kill the rig's tmux session (and every process in it) instead " + "of launching, e.g. to relaunch after editing the rig file." + ), + ) + return parser + + +def main(argv: list[str] | None = None) -> int: + """CLI entry point. Returns a process exit code (never a stack trace).""" + parser = _build_parser() + args = parser.parse_args(argv) + + if args.rig is None: + parser.print_usage() + print(_EXAMPLE) + return 0 + + try: + config = load_rig_config(args.rig) + if args.kill: + if args.no_runtime: + parser.error("--kill cannot be combined with --no-runtime") + kill_rig(config) + else: + launch_rig(config, no_runtime=args.no_runtime) + except RigError as exc: + print(f"error: {exc}", file=sys.stderr) + return 1 + except subprocess.CalledProcessError as exc: + stderr = (exc.stderr or "").strip() if isinstance(exc.stderr, str) else "" + detail = f": {stderr}" if stderr else "" + print(f"error: tmux command failed ({exc}){detail}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/core/rig/python/config.py b/src/core/rig/python/config.py new file mode 100644 index 000000000..4b861d5d0 --- /dev/null +++ b/src/core/rig/python/config.py @@ -0,0 +1,293 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Schema and loader for teleop-rig YAML files. + +A *rig* file describes one tmux teleop rig: the CloudXR runtime pane plus +producer plugins and consumer apps. See ``rigs/se3_tracker.yaml`` in the +Teleop repository for an annotated exemplar. Top-level keys:: + + name: rig id AND tmux session name (required) + description: free text (optional) + cwd: working dir for every pane, relative to the YAML file's + directory (optional; default: the YAML's directory) + params: flat str->scalar dict of {placeholder} values shared by + the commands below; edit the file to change them (optional) + runtime: full-command override for the runtime pane (optional; + default: "{python} -m isaacteleop.cloudxr --accept-eula") + producers: list of {name, command} — publish device data (optional) + consumers: list of {name, command} — read the streams (optional) + +At least one producer or consumer is required. Unknown keys are a hard +error. Commands are shell strings typed verbatim into panes; ``{key}`` +placeholders are substituted from ``params`` (literal braces must be +escaped as ``{{`` / ``}}``). The reserved ``{python}`` placeholder expands +to the launching interpreter (:data:`sys.executable`). +""" + +from __future__ import annotations + +import dataclasses +import re +import shlex +import sys +from pathlib import Path +from typing import Any, Mapping, Sequence + +import yaml + +#: Reserved placeholder expanding to the launching interpreter. +PYTHON_PLACEHOLDER = "python" + +#: Default runtime pane command (used when the rig has no ``runtime:`` key). +DEFAULT_RUNTIME_COMMAND = "{python} -m isaacteleop.cloudxr --accept-eula" + +#: Flag a Python TeleopSession script needs so it does not start a second +#: (host-singleton) CloudXR runtime next to the managed runtime pane. +NO_LAUNCH_RUNTIME_FLAG = "--no-launch-cloudxr-runtime" + +_TOP_LEVEL_KEYS = frozenset( + {"name", "description", "cwd", "params", "runtime", "producers", "consumers"} +) +_ENTRY_KEYS = frozenset({"name", "command"}) + +#: ``name:`` doubles as the tmux session name, so keep it shell/tmux-safe +#: (tmux targets treat ``.`` and ``:`` specially; spaces need quoting). +_NAME_PATTERN = re.compile(r"^[A-Za-z0-9_-]+$") + + +class RigError(Exception): + """Base class for user-facing rig-launcher errors (no stack trace).""" + + +class RigConfigError(RigError): + """A rig YAML file is invalid.""" + + +@dataclasses.dataclass(frozen=True) +class ProcessConfig: + """One producer or consumer pane: display name + raw command string.""" + + name: str + command: str + + +@dataclasses.dataclass(frozen=True) +class RigConfig: + """A parsed teleop rig (see the module docstring for the YAML schema).""" + + name: str + description: str + cwd: Path + params: dict[str, str] + runtime: str | None + producers: tuple[ProcessConfig, ...] + consumers: tuple[ProcessConfig, ...] + source: Path + + @property + def runtime_command(self) -> str: + """The raw (unsubstituted) runtime pane command.""" + return self.runtime if self.runtime is not None else DEFAULT_RUNTIME_COMMAND + + +def _require_str(value: Any, what: str, source: Path) -> str: + """Return *value* if it is a non-empty string; hard error otherwise.""" + if not isinstance(value, str) or not value.strip(): + raise RigConfigError(f"{source}: {what} must be a non-empty string") + return value + + +def _parse_entries(value: Any, role: str, source: Path) -> tuple[ProcessConfig, ...]: + """Parse a ``producers:`` / ``consumers:`` list of {name, command} dicts.""" + if value is None: + return () + if not isinstance(value, list): + raise RigConfigError( + f"{source}: '{role}' must be a list of {{name, command}} entries" + ) + entries = [] + for i, entry in enumerate(value): + if not isinstance(entry, dict): + raise RigConfigError( + f"{source}: {role}[{i}] must be a mapping with 'name' and 'command' keys" + ) + unknown = sorted(set(entry) - _ENTRY_KEYS) + if unknown: + raise RigConfigError( + f"{source}: unknown key(s) {unknown} in {role}[{i}] " + f"(each entry takes exactly: name, command)" + ) + missing = sorted(_ENTRY_KEYS - set(entry)) + if missing: + raise RigConfigError( + f"{source}: {role}[{i}] is missing required key(s) {missing}" + ) + entries.append( + ProcessConfig( + name=_require_str(entry["name"], f"{role}[{i}].name", source), + command=_require_str(entry["command"], f"{role}[{i}].command", source), + ) + ) + return tuple(entries) + + +def _parse_params(value: Any, source: Path) -> dict[str, str]: + """Parse the ``params:`` mapping; scalars are coerced to strings.""" + if value is None: + return {} + if not isinstance(value, dict): + raise RigConfigError(f"{source}: 'params' must be a mapping of KEY: value") + params: dict[str, str] = {} + for key, val in value.items(): + if not isinstance(key, str): + raise RigConfigError( + f"{source}: 'params' keys must be strings (got {key!r})" + ) + if key == PYTHON_PLACEHOLDER: + raise RigConfigError( + f"{source}: param '{PYTHON_PLACEHOLDER}' is reserved " + "(it always expands to the launching interpreter)" + ) + if isinstance(val, (dict, list)): + raise RigConfigError(f"{source}: param '{key}' must be a scalar") + params[key] = str(val) + return params + + +def load_rig_config(path: str | Path) -> RigConfig: + """Load and validate a rig YAML file. + + Args: + path: Path to the rig file (e.g. ``rigs/se3_tracker.yaml``). + + Returns: + The validated :class:`RigConfig`. ``cwd`` is resolved to an + absolute path against the YAML file's directory. + + Raises: + RigConfigError: On a missing file, YAML parse error, unknown or + missing keys, or invalid values. + """ + source = Path(path) + if not source.is_file(): + raise RigConfigError( + f"rig file not found: {source} — see rigs/se3_tracker.yaml in the " + "Teleop repository for an example" + ) + try: + data = yaml.safe_load(source.read_text()) + except yaml.YAMLError as exc: + raise RigConfigError( + f"{source}: invalid YAML: {exc} — see rigs/se3_tracker.yaml for a valid example" + ) from exc + if not isinstance(data, dict): + raise RigConfigError(f"{source}: top level must be a mapping of rig keys") + + unknown = sorted(set(data) - _TOP_LEVEL_KEYS) + if unknown: + raise RigConfigError( + f"{source}: unknown top-level key(s) {unknown} " + f"(known keys: {', '.join(sorted(_TOP_LEVEL_KEYS))})" + ) + if "name" not in data: + raise RigConfigError(f"{source}: missing required key 'name'") + + name = _require_str(data["name"], "'name'", source) + if not _NAME_PATTERN.match(name): + raise RigConfigError( + f"{source}: name '{name}' is used as the tmux session name — " + "use letters/digits/-/_ only" + ) + description = "" + if data.get("description") is not None: + description = _require_str(data["description"], "'description'", source) + runtime = None + if data.get("runtime") is not None: + runtime = _require_str(data["runtime"], "'runtime'", source) + + yaml_dir = source.resolve().parent + cwd = yaml_dir + if data.get("cwd") is not None: + cwd = (yaml_dir / _require_str(data["cwd"], "'cwd'", source)).resolve() + + producers = _parse_entries(data.get("producers"), "producers", source) + consumers = _parse_entries(data.get("consumers"), "consumers", source) + if not producers and not consumers: + raise RigConfigError( + f"{source}: rig must define at least one producer or consumer" + ) + + return RigConfig( + name=name, + description=description, + cwd=cwd, + params=_parse_params(data.get("params"), source), + runtime=runtime, + producers=producers, + consumers=consumers, + source=source, + ) + + +def substitute_command(command: str, params: Mapping[str, str], source: Path) -> str: + """Expand ``{key}`` placeholders in a command string. + + ``{python}`` expands to the quoted launching interpreter; every other + placeholder must be declared in *params*. + + Raises: + RigConfigError: On an unknown placeholder or malformed braces + (literal braces must be escaped as ``{{`` / ``}}``). + """ + # A plain dict raises KeyError from format_map on unknown placeholders. + table = dict(params) + table[PYTHON_PLACEHOLDER] = shlex.quote(sys.executable) + try: + return command.format_map(table) + except KeyError as exc: + raise RigConfigError( + f"{source}: unknown placeholder {{{exc.args[0]}}} in command {command!r} " + f"— declare it under 'params:' (literal braces must be escaped as {{{{ / }}}})" + ) from exc + except (ValueError, IndexError) as exc: + raise RigConfigError( + f"{source}: malformed placeholder in command {command!r}: {exc} " + f"(literal braces must be escaped as {{{{ / }}}})" + ) from exc + + +def find_runtime_footguns( + processes: Sequence[ProcessConfig], runtime_managed: bool +) -> list[str]: + """Warn-only lint: pre-typed Python commands that would self-launch a runtime. + + Python TeleopSession scripts launch their own CloudXR runtime by + default. The runtime is a host singleton (fixed WSS port), so a second + runtime KILLS the managed runtime pane — the headset drops and every + producer stalls. Heuristic (narrow, never a gate): the managed runtime + pane is enabled AND a pre-typed command contains ``{python}`` and a + ``.py`` token or ``-m isaacteleop`` AND lacks ``--no-launch-cloudxr-runtime``. + + Returns: + Warning strings to print; empty when nothing looks risky. + """ + if not runtime_managed: + return [] + warnings = [] + for proc in processes: + cmd = proc.command + if "{python}" not in cmd or NO_LAUNCH_RUNTIME_FLAG in cmd: + continue + looks_python_app = "-m isaacteleop" in cmd or any( + token.endswith(".py") for token in cmd.split() + ) + if looks_python_app: + warnings.append( + f"warning: '{proc.name}' looks like a Python TeleopSession script " + f"without {NO_LAUNCH_RUNTIME_FLAG}. Such scripts start their own " + "CloudXR runtime by default; the runtime is a host singleton, so a " + "second runtime kills the runtime pane — the headset drops and " + f"producers stall. Add {NO_LAUNCH_RUNTIME_FLAG} to its command." + ) + return warnings diff --git a/src/core/rig/python/launcher.py b/src/core/rig/python/launcher.py new file mode 100644 index 000000000..aafba8fd3 --- /dev/null +++ b/src/core/rig/python/launcher.py @@ -0,0 +1,561 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""tmux orchestration for teleop rigs. + +Launches one tmux session per rig: the CloudXR runtime pane starts +immediately; each producer/consumer pane waits for the runtime to come up, +sources the CloudXR env it writes, and then RUNS its command automatically. +When the command exits (the classic early exit: started before a headset +connected, so 'Failed to get OpenXR system'), the pane prints the exit +status and drops to an interactive shell with the same command pre-typed — +recovery is one Enter. + +Each pane is spawned RUNNING a small POSIX wrapper (its tmux shell-command) +instead of having setup lines typed into a starting shell — typed setup +races shell startup and gets echoed twice (raw by the tty, then again by +the line editor). The wrapper turns off tty echo, does its setup (worker +panes: wait for the runtime, source the CloudXR env, print a banner, run +the command), pre-types the rig command into its own pane while echo is +off (the keystrokes buffer invisibly in the tty), restores echo, and execs +the user's shell — whose line editor then displays the buffered command +once, at a real prompt, editable and awaiting Enter. If the runtime never +comes up the wrapper does NOT run the command (it would fail confusingly +without the env); it prints a remedy and pre-types the command instead. + +All tmux interaction goes through a single injectable ``run_tmux`` seam so +the pane plan is unit-testable without tmux or a headset. +""" + +from __future__ import annotations + +import os +import shlex +import shutil +import subprocess +import sys +from pathlib import Path +from typing import Callable, Mapping, Sequence + +from .config import ( + RigConfig, + RigError, + find_runtime_footguns, + substitute_command, +) + +#: Signature of the tmux seam: run one tmux subcommand, return its stdout. +RunTmux = Callable[[Sequence[str]], str] + +#: One planned pane: (role, name, raw command, substituted command, footgun). +#: ``footgun`` marks a Python app missing --no-launch-cloudxr-runtime. +_Pane = tuple[str, str, str, str, bool] + +_BUILD_REMEDY = ( + "build and install first:\n" + " cmake -B build -DBUILD_EXAMPLES=ON -DBUILD_PYTHON_BINDINGS=ON\n" + " cmake --build build --parallel && cmake --install build" +) + +#: Maximum time [s] a worker pane waits for the runtime before giving up on +#: auto-loading the CloudXR env (matches the runtime's own startup timeout +#: order of magnitude; the pane prints a manual remedy on expiry). +CLOUDXR_ENV_WAIT_TIMEOUT_SEC = 120 + +#: Height of the runtime pane in the ``main-horizontal`` layout. The runtime +#: only prints status lines and the web-client URL, so it gets a slim strip +#: on top and the worker panes share the rest (tmux accepts a percentage). +RUNTIME_PANE_HEIGHT = "25%" + + +class PreflightError(RigError): + """A launch precondition failed (each message names one cause + one remedy).""" + + +def _run_tmux(args: Sequence[str]) -> str: + """Run ``tmux `` and return its stripped stdout. + + ``attach-session`` must inherit the terminal (it takes over the tty), + so it alone runs uncaptured; everything else is captured so pane ids + from ``-PF '#{pane_id}'`` can be returned. + """ + if args and args[0] == "attach-session": + subprocess.run(["tmux", *args], check=True) + return "" + result = subprocess.run(["tmux", *args], check=True, capture_output=True, text=True) + return result.stdout.strip() + + +def _check_tmux_installed() -> None: + """Fail early with an install hint when tmux is not on PATH.""" + if shutil.which("tmux") is None: + raise PreflightError( + "tmux not found on PATH — install it (e.g. `sudo apt install tmux`) and rerun" + ) + + +def _check_python_can_import_cloudxr(env: Mapping[str, str]) -> None: + """Verify the pane interpreter can import isaacteleop.cloudxr. + + tmux panes spawn fresh shells that do not inherit the caller's venv, so + the absolute ``sys.executable`` (plus a forwarded PYTHONPATH) must be + able to import the package on its own. + """ + probe = subprocess.run( + [sys.executable, "-c", "import isaacteleop.cloudxr"], + env=dict(env), + capture_output=True, + ) + if probe.returncode != 0: + raise PreflightError( + f"{sys.executable} cannot import 'isaacteleop.cloudxr' — run from the " + "environment where the isaacteleop wheel is installed " + "(pip install install/wheels/isaacteleop-*.whl)" + ) + + +def _check_commands_exist(panes: Sequence[tuple[str, str]], cwd: Path) -> None: + """Require path-like first command tokens to exist and be executable. + + Mirrors the old run_se3_demo.sh preflight. *panes* is a sequence of + (name, substituted command) pairs. + """ + for name, command in panes: + try: + tokens = shlex.split(command) + except ValueError: + continue # unbalanced quotes: let the user's shell report it + if not tokens: + continue + first = tokens[0] + # Rule: only a first token containing a path separator is checked + # here — a bare command name is resolved via PATH by the pane shell, + # which we cannot (and should not) second-guess. + if os.sep not in first: + continue + path = Path(first) if os.path.isabs(first) else cwd / first + if not (path.is_file() and os.access(path, os.X_OK)): + raise PreflightError( + f"'{name}': {path} not found or not executable — {_BUILD_REMEDY}" + ) + + +def _cloudxr_run_dir(runtime_command: str | None, env: Mapping[str, str]) -> Path: + """Return the CloudXR run dir (holds ``cloudxr.env`` and ``runtime_started``). + + Resolution mirrors the runtime's own ``EnvConfig``: an explicit + ``--cloudxr-install-dir`` in the runtime command wins, else + ``$CXR_INSTALL_DIR``, else ``~/.cloudxr``. + """ + install: str | None = None + if runtime_command: + try: + tokens = shlex.split(runtime_command) + except ValueError: + tokens = [] + for i, token in enumerate(tokens): + if token == "--cloudxr-install-dir" and i + 1 < len(tokens): + install = tokens[i + 1] + elif token.startswith("--cloudxr-install-dir="): + install = token.partition("=")[2] + if install is None: + install = env.get("CXR_INSTALL_DIR") or "~/.cloudxr" + return Path(install).expanduser() / "run" + + +def _cloudxr_env_wait_command(run_dir: Path) -> str: + """Build the shell line each worker pane RUNS to load the CloudXR env. + + OpenXR producers/consumers need the env the runtime writes to + ``/cloudxr.env`` (``XR_RUNTIME_JSON`` etc.). The runtime deletes + any stale ``runtime_started`` sentinel at startup and recreates it only + once it is actually serving, so the sentinel is the "runtime successfully + started" signal to wait on. The wait is bounded: a runtime that never + comes up leaves an actionable message in the pane, not a stuck loop. + """ + sentinel = shlex.quote(str(run_dir / "runtime_started")) + env_file = str(run_dir / "cloudxr.env") + ok_msg = shlex.quote("[cloudxr] runtime is up — env loaded") + fail_msg = shlex.quote( + f"[cloudxr] runtime not ready after {CLOUDXR_ENV_WAIT_TIMEOUT_SEC}s — " + f"check the runtime pane, then run: source {env_file}" + ) + return ( + f"i=0; until [ -e {sentinel} ] || [ $i -ge {CLOUDXR_ENV_WAIT_TIMEOUT_SEC} ]; " + f"do sleep 1; i=$((i+1)); done; " + f"if [ -e {sentinel} ]; then . {shlex.quote(env_file)} && echo {ok_msg}; " + f"else echo {fail_msg}; fi" + ) + + +def _pythonpath_prefix(command: str, raw_command: str, env: Mapping[str, str]) -> str: + """Forward the caller's PYTHONPATH into commands that run our interpreter. + + Launched via PYTHONPATH instead of an installed wheel? The fresh pane + shell won't have it, so prefix it onto any command that used the + ``{python}`` placeholder. + """ + pythonpath = env.get("PYTHONPATH") + if pythonpath and "{python}" in raw_command: + return f"PYTHONPATH={shlex.quote(pythonpath)} {command}" + return command + + +def launch_rig( + config: RigConfig, + *, + no_runtime: bool = False, + run_tmux: RunTmux | None = None, +) -> None: + """Launch (or re-attach to) the tmux session for a teleop rig. + + The rig file is the single source of configuration: params live in its + ``params:`` block and are substituted into every command that + references them (edit the file to change them). + + Args: + config: The parsed rig (see :func:`~.config.load_rig_config`). + no_runtime: Skip the runtime pane (a CloudXR runtime is already + running elsewhere, e.g. after ``python -m isaacteleop.cloudxr``). + run_tmux: Injectable tmux seam for tests. When ``None`` the real + tmux is used and environment preflight checks (tmux on PATH, + interpreter can import isaacteleop.cloudxr) run first. + + Raises: + PreflightError: When a launch precondition fails. + RigConfigError: On bad placeholders. + """ + env = os.environ + params = config.params + + # Resolve the pane plan. Runtime first (starts immediately); producers + # and consumers auto-run once the runtime env is ready. + plan: list[_Pane] = [] + if not no_runtime: + raw = config.runtime_command + # Honest title: only claim isaacteleop.cloudxr when it IS the default. + runtime_name = ( + "isaacteleop.cloudxr" if config.runtime is None else "custom runtime" + ) + plan.append( + ( + "runtime", + runtime_name, + raw, + substitute_command(raw, params, config.source), + False, + ) + ) + for role, procs in (("producer", config.producers), ("consumer", config.consumers)): + for proc in procs: + plan.append( + ( + role, + proc.name, + proc.command, + substitute_command(proc.command, params, config.source), + bool(find_runtime_footguns([proc], runtime_managed=not no_runtime)), + ) + ) + + if not config.cwd.is_dir(): + raise PreflightError( + f"working directory {config.cwd} (from 'cwd:' in {config.source}) does not exist" + ) + _check_commands_exist( + [(name, resolved) for role, name, _, resolved, _ in plan if role != "runtime"], + config.cwd, + ) + for warning in find_runtime_footguns( + [*config.producers, *config.consumers], runtime_managed=not no_runtime + ): + print(warning, file=sys.stderr) + + if run_tmux is None: + _check_tmux_installed() + if any("{python}" in raw for _, _, raw, _, _ in plan): + _check_python_can_import_cloudxr(env) + run_tmux = _run_tmux + + session = config.name + if _session_exists(run_tmux, session): + message = ( + f"session '{session}' already exists — switching to it " + f"(kill with: python -m isaacteleop.rig {config.source} --kill)" + ) + if no_runtime: + message += ( + "\nnote: --no-runtime ignored for an existing session; " + "kill it first to relaunch with new settings" + ) + print(message) + _goto_session(run_tmux, session, env) + return + + runtime_resolved = plan[0][3] if not no_runtime else None + cloudxr_run_dir = _cloudxr_run_dir(runtime_resolved, env) + _create_session(run_tmux, session, config.cwd, plan, env, cloudxr_run_dir) + _print_instructions(session, config.description, plan) + _goto_session(run_tmux, session, env) + + +def kill_rig(config: RigConfig, *, run_tmux: RunTmux | None = None) -> None: + """Kill the rig's tmux session (and every process running in it). + + Idempotent: killing a rig whose session does not exist just reports + that there is nothing to do. + + Args: + config: The parsed rig; its ``name`` is the tmux session name. + run_tmux: Injectable tmux seam for tests (real tmux when ``None``). + """ + if run_tmux is None: + _check_tmux_installed() + run_tmux = _run_tmux + session = config.name + if not _session_exists(run_tmux, session): + print(f"no session '{session}' to kill") + return + run_tmux(["kill-session", "-t", session]) + print(f"killed session '{session}'") + + +def _session_exists(run_tmux: RunTmux, session: str) -> bool: + """Return whether the tmux session already exists (has-session probe).""" + try: + run_tmux(["has-session", "-t", session]) + except subprocess.CalledProcessError: + return False + return True + + +def _goto_session(run_tmux: RunTmux, session: str, env: Mapping[str, str]) -> None: + """Attach from a plain shell; switch the client when already inside tmux.""" + if env.get("TMUX"): + run_tmux(["switch-client", "-t", session]) + else: + run_tmux(["attach-session", "-t", session]) + + +def _autorun_banner(role: str, name: str, command: str, footgun: bool) -> str: + """Build the echo command a worker pane runs right before its command. + + The message is shlex-quoted as a whole so pane names (or commands) + containing quotes or shell metacharacters cannot break out of the echo. + Foot-gun panes get an in-pane WARNING in addition to the stderr print at + launch time. + """ + message = f"[{role}: {name}] running: {command}" + if footgun: + message += ( + " — WARNING: this looks like a Python TeleopSession script without " + "--no-launch-cloudxr-runtime; it will start a second " + "runtime and kill the runtime pane" + ) + return "echo " + shlex.quote(message) + + +def _self_type_command(command: str) -> str: + """Build the wrapper line that pre-types *command* into the pane's own tty. + + Runs inside a pane wrapper while tty echo is OFF: the keystrokes buffer + invisibly in the pty and the interactive shell's line editor (readline/ + ZLE reads pending input on startup) displays them once, at the prompt, + editable. tmux writes the pane input before replying to the client, so + the keys are buffered before the wrapper's next line runs. + """ + return f'tmux send-keys -t "$TMUX_PANE" -l -- {shlex.quote(command)}' + + +def _runtime_pane_command(command: str) -> str: + """Build the tmux shell-command a runtime pane is spawned running. + + Pre-types *command* plus Enter with tty echo off, then execs the user's + shell: the shell reads the buffered line at its first prompt, echoes it + exactly once, and runs it immediately (the runtime needs no headset + gate). The trailing shell keeps the pane alive and interactive after + the runtime exits. + """ + return "\n".join( + [ + "stty -echo", + _self_type_command(command), + 'tmux send-keys -t "$TMUX_PANE" C-m', + "stty echo", + 'exec "${SHELL:-sh}" -l', + ] + ) + + +def _worker_pane_command( + command: str, run_dir: Path, role: str, name: str, footgun: bool +) -> str: + """Build the tmux shell-command a producer/consumer pane is spawned running. + + With tty echo off (so none of this machinery ever appears as typed + input): wait for the runtime and source the CloudXR env (see + :func:`_cloudxr_env_wait_command`). Runtime up? Print the auto-run + banner and RUN the command (echo restored so the app's tty behaves + normally; via ``sh -c`` so a syntax error in the command cannot kill + the wrapper, and never ``exec`` — the pane must survive the exit); when + it exits, print its exit status with a rerun hint. Either way, finish + by pre-typing the command WITHOUT Enter (with echo off again), restore + echo, and exec the user's shell — which inherits the sourced env + (``cloudxr.env`` uses ``export``) and displays the pre-typed command + once, at its prompt: one Enter reruns. On sentinel timeout the command + is NOT run (it would fail confusingly without the env); the wait line + already printed the remedy and the pre-typed command awaits. + + ``trap : INT`` while the command runs: Ctrl+C must kill the app, not + the wrapper (a non-interactive sh whose foreground child dies of + SIGINT exits too — taking the pane with it — unless INT is trapped). + """ + sentinel = shlex.quote(str(run_dir / "runtime_started")) + return "\n".join( + [ + "stty -echo", + _cloudxr_env_wait_command(run_dir), + f"if [ -e {sentinel} ]; then", + _autorun_banner(role, name, command, footgun), + "stty echo", + "trap : INT", + f"sh -c {shlex.quote(command)}", + "s=$?", + "trap - INT", + 'echo "[rig] command exited with status $s — press Enter to rerun"', + "stty -echo", + "fi", + _self_type_command(command), + "stty echo", + 'exec "${SHELL:-sh}" -l', + ] + ) + + +def _create_session( + run_tmux: RunTmux, + session: str, + cwd: Path, + plan: Sequence[_Pane], + env: Mapping[str, str], + cloudxr_run_dir: Path, +) -> None: + """Create the session, each pane spawned running its wrapper command. + + Pane ids are captured via ``-PF '#{pane_id}'`` (never indices) so the + layout is immune to base-index / pane-base-index user settings. Layout: + ``main-horizontal`` with the runtime as the main pane — a full-width + strip of :data:`RUNTIME_PANE_HEIGHT` on top (it only prints status and + the web-client URL) — and the workers tiled below. Under ``--no-runtime`` + all panes are peers and stay ``tiled``. + + Every pane's setup runs as its SPAWN COMMAND (the trailing tmux + shell-command), never as keystrokes typed by the launcher: keystrokes + racing shell startup are echoed raw by the tty and then re-echoed by + the line editor, showing everything twice. See + :func:`_runtime_pane_command` / :func:`_worker_pane_command`. + + The embedded ``send-keys`` payloads use ``-l`` (literal: no key-name + lookup) behind a ``--`` terminator (a substituted command starting + with ``-`` must not parse as a tmux option). + """ + pane_commands = [] + for role, name, raw, resolved, footgun in plan: + command = _pythonpath_prefix(resolved, raw, env) + if role == "runtime": + # The runtime is a host-level singleton and must be up before + # anything else: its wrapper presses Enter itself. + pane_commands.append(_runtime_pane_command(command)) + else: + # Producers/consumers wait for the runtime env, then auto-run; + # an early exit (headset not connected yet) reruns on Enter. + pane_commands.append( + _worker_pane_command(command, cloudxr_run_dir, role, name, footgun) + ) + + first_pane = run_tmux( + [ + "new-session", + "-d", + "-P", + "-F", + "#{pane_id}", + "-s", + session, + "-n", + "rig", + "-c", + str(cwd), + pane_commands[0], + ] + ) + # Session-scoped only (-t , no -g): never touch global tmux config. + run_tmux(["set-option", "-t", session, "pane-border-status", "top"]) + + pane_ids = [first_pane] + for i in range(1, len(plan)): + pane_ids.append( + run_tmux( + [ + "split-window", + "-v", + "-P", + "-F", + "#{pane_id}", + "-t", + pane_ids[-1], + "-c", + str(cwd), + pane_commands[i], + ] + ) + ) + # Redistribute after every split so chained splits never hit tmux's + # "pane too small" limit, no matter how many panes the rig has. + run_tmux(["select-layout", "-t", session, "tiled"]) + if len(plan) > 1 and plan[0][0] == "runtime": + # Final layout: the runtime (pane 0 → the main pane) as a slim + # full-width strip on top, workers tiled in the space below. + run_tmux(["set-option", "-t", session, "main-pane-height", RUNTIME_PANE_HEIGHT]) + run_tmux(["select-layout", "-t", session, "main-horizontal"]) + # else (--no-runtime): all panes are peer workers — stay tiled. + + for pane_id, (role, name, _, _, _) in zip(pane_ids, plan): + title = ( + f"runtime: {name} (running)" + if role == "runtime" + else f"{role}: {name} — auto-runs once the runtime is up" + ) + run_tmux(["select-pane", "-t", pane_id, "-T", title]) + + run_tmux(["select-pane", "-t", pane_ids[0]]) + run_tmux( + [ + "display-message", + "-t", + pane_ids[0], + "Connect the headset to the printed URL — if a pane's app exited " + "before the headset was in, press Enter in it to rerun", + ] + ) + + +def _print_instructions(session: str, description: str, plan: Sequence[_Pane]) -> None: + """Print the pane rundown BEFORE attaching (it survives detach).""" + header = f"Rig session '{session}' created" + if description: + header += f": {description}" + print(header) + for role, name, _, _, _ in plan: + if role == "runtime": + print( + f" - {role}: {name} — running; connect the headset to the URL it prints" + ) + else: + print(f" - {role}: {name} — runs automatically once the runtime is up") + print( + "Worker panes load the CloudXR env (source .../run/cloudxr.env) and run " + "their command automatically once the runtime is up; if a command exited " + "before the headset connected, press Enter in its pane to rerun." + ) + print(f"Kill the session with --kill (or: tmux kill-session -t {session})") diff --git a/src/core/rig_tests/CMakeLists.txt b/src/core/rig_tests/CMakeLists.txt new file mode 100644 index 000000000..cbe9f2948 --- /dev/null +++ b/src/core/rig_tests/CMakeLists.txt @@ -0,0 +1,9 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# ============================================================================== +# Rig Launcher Tests +# ============================================================================== + +# Python tests +add_subdirectory(python) diff --git a/src/core/rig_tests/python/CMakeLists.txt b/src/core/rig_tests/python/CMakeLists.txt new file mode 100644 index 000000000..94dbf6f0e --- /dev/null +++ b/src/core/rig_tests/python/CMakeLists.txt @@ -0,0 +1,29 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# ============================================================================== +# Rig Launcher Python Tests +# ============================================================================== + +# Discover and add individual Python tests +# This makes each test show up separately in CTest output + +# First, find all test files (only in this directory, not recursively) +file(GLOB TEST_FILES + RELATIVE "${CMAKE_CURRENT_SOURCE_DIR}" + "${CMAKE_CURRENT_SOURCE_DIR}/test_*.py" +) + +# Add each test file as a separate CTest test +foreach(test_file ${TEST_FILES}) + # Get test name from filename (remove .py and path) + get_filename_component(test_name "${test_file}" NAME_WE) + + # Add the test. No PYTHONPATH to the staged package is needed: + # conftest.py imports the rig SOURCE files directly. + add_test( + NAME "rig_${test_name}" + COMMAND uv run --python ${ISAAC_TELEOP_PYTHON_VERSION} --extra dev pytest -v --tb=short "${test_file}" + WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" + ) +endforeach() diff --git a/src/core/rig_tests/python/conftest.py b/src/core/rig_tests/python/conftest.py new file mode 100644 index 000000000..31c262234 --- /dev/null +++ b/src/core/rig_tests/python/conftest.py @@ -0,0 +1,48 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Make rig python sources importable without installing ``isaacteleop``. + +Synthetic package ``rig_py_test_ns``: the modules use sibling +relative imports (``from .config import …``), so they are loaded as +submodules of a synthetic package whose ``__path__`` points at the source +directory. Tests therefore always exercise the SOURCE files, not a stale +build tree; wheel packaging is covered by the wheel-content check in the +build, not by these tests. +""" + +from __future__ import annotations + +import importlib.util +import sys +import types +from pathlib import Path + +_RIG_PY = Path(__file__).resolve().parents[2] / "rig" / "python" + +RIG_TEST_PKG = "rig_py_test_ns" + + +def _ensure_rig_package() -> None: + if RIG_TEST_PKG in sys.modules: + return + pkg = types.ModuleType(RIG_TEST_PKG) + pkg.__path__ = [str(_RIG_PY)] + sys.modules[RIG_TEST_PKG] = pkg + + def load(mod: str) -> None: + full = f"{RIG_TEST_PKG}.{mod}" + path = _RIG_PY / f"{mod}.py" + spec = importlib.util.spec_from_file_location(full, path) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + sys.modules[full] = module + spec.loader.exec_module(module) + setattr(sys.modules[RIG_TEST_PKG], mod, module) + + load("config") + load("launcher") + load("__main__") + + +_ensure_rig_package() diff --git a/src/core/rig_tests/python/pyproject.toml b/src/core/rig_tests/python/pyproject.toml new file mode 100644 index 000000000..e2cd981e1 --- /dev/null +++ b/src/core/rig_tests/python/pyproject.toml @@ -0,0 +1,22 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +[project] +name = "teleopcore-rig-tests" +version = "0.0.0" # Internal tests - not versioned +description = "Rig launcher unit tests for TeleopCore" +requires-python = ">=3.10,<3.14" + +[project.optional-dependencies] +dev = [ + "pytest", + "pyyaml>=6.0.3", +] + +[tool.pytest.ini_options] +pythonpath = ["."] +python_files = ["test_*.py"] +python_classes = ["Test*"] +python_functions = ["test_*"] +# Prevent pytest from recursing into parent directories +norecursedirs = [".git", ".venv", "build", "dist", "*.egg", "__pycache__"] diff --git a/src/core/rig_tests/python/test_config.py b/src/core/rig_tests/python/test_config.py new file mode 100644 index 000000000..cf78d59ca --- /dev/null +++ b/src/core/rig_tests/python/test_config.py @@ -0,0 +1,245 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Unit tests for the teleop-rig YAML schema and loader.""" + +from __future__ import annotations + +import shlex +import sys +from pathlib import Path + +import pytest +from rig_py_test_ns.config import ( + DEFAULT_RUNTIME_COMMAND, + RigConfigError, + ProcessConfig, + find_runtime_footguns, + load_rig_config, + substitute_command, +) + +REPO_ROOT = Path(__file__).resolve().parents[4] +SE3_RIG = REPO_ROOT / "rigs" / "se3_tracker.yaml" + + +def write_rig(tmp_path: Path, body: str) -> Path: + path = tmp_path / "rig.yaml" + path.write_text(body) + return path + + +MINIMAL = """ +name: mini +consumers: + - name: printer + command: "echo hi" +""" + + +# --------------------------------------------------------------------------- +# Loading the shipped exemplar +# --------------------------------------------------------------------------- + + +def test_load_se3_rig(): + config = load_rig_config(SE3_RIG) + assert config.name == "se3_tracker" + assert config.description + assert config.cwd == REPO_ROOT # cwd: .. resolves against rigs/ + assert config.runtime is None + assert config.runtime_command == DEFAULT_RUNTIME_COMMAND + assert config.params == {"hand": "right", "collection_id": "se3_tracker"} + assert len(config.producers) == 1 + assert len(config.consumers) == 1 + # collection_id is single-sourced: both sides reference the placeholder. + assert "{collection_id}" in config.producers[0].command + assert "{collection_id}" in config.consumers[0].command + + +def test_se3_rig_has_no_footguns(): + config = load_rig_config(SE3_RIG) + assert ( + find_runtime_footguns( + [*config.producers, *config.consumers], runtime_managed=True + ) + == [] + ) + + +# --------------------------------------------------------------------------- +# Validation errors (each a hard error naming the file) +# --------------------------------------------------------------------------- + + +def test_missing_file_is_config_error(tmp_path): + with pytest.raises(RigConfigError, match="rig file not found"): + load_rig_config(tmp_path / "nope.yaml") + + +def test_invalid_yaml_is_config_error(tmp_path): + path = write_rig(tmp_path, "name: [unclosed") + with pytest.raises(RigConfigError, match="invalid YAML"): + load_rig_config(path) + + +def test_unknown_top_level_key_is_hard_error(tmp_path): + path = write_rig(tmp_path, MINIMAL + "streams: {}\n") + with pytest.raises(RigConfigError, match=r"unknown top-level key.*streams"): + load_rig_config(path) + + +def test_missing_name_is_hard_error(tmp_path): + path = write_rig(tmp_path, "consumers:\n - name: x\n command: y\n") + with pytest.raises(RigConfigError, match="missing required key 'name'"): + load_rig_config(path) + + +def test_rig_without_processes_is_hard_error(tmp_path): + path = write_rig(tmp_path, "name: empty\n") + with pytest.raises(RigConfigError, match="at least one producer or consumer"): + load_rig_config(path) + + +def test_unknown_entry_key_is_hard_error(tmp_path): + path = write_rig( + tmp_path, + "name: x\nconsumers:\n - name: y\n command: z\n autostart: true\n", + ) + with pytest.raises(RigConfigError, match=r"unknown key.*autostart"): + load_rig_config(path) + + +def test_entry_missing_command_is_hard_error(tmp_path): + path = write_rig(tmp_path, "name: x\nproducers:\n - name: y\n") + with pytest.raises(RigConfigError, match=r"missing required key.*command"): + load_rig_config(path) + + +def test_tmux_safe_name_is_accepted(tmp_path): + path = write_rig(tmp_path, MINIMAL.replace("name: mini", "name: Se3-Rig_2")) + assert load_rig_config(path).name == "Se3-Rig_2" + + +@pytest.mark.parametrize("bad_name", ["se3 rig", "se3:rig", "se3.rig", "r'ig"]) +def test_tmux_unsafe_name_is_hard_error(tmp_path, bad_name): + path = write_rig(tmp_path, MINIMAL.replace("name: mini", f'name: "{bad_name}"')) + with pytest.raises(RigConfigError, match="used as the tmux session name"): + load_rig_config(path) + + +def test_param_named_python_is_reserved(tmp_path): + path = write_rig(tmp_path, MINIMAL + "params:\n python: /usr/bin/python\n") + with pytest.raises(RigConfigError, match="reserved"): + load_rig_config(path) + + +# --------------------------------------------------------------------------- +# cwd resolution +# --------------------------------------------------------------------------- + + +def test_cwd_defaults_to_yaml_directory(tmp_path): + path = write_rig(tmp_path, MINIMAL) + assert load_rig_config(path).cwd == tmp_path.resolve() + + +def test_cwd_resolves_relative_to_yaml_directory(tmp_path): + (tmp_path / "sub").mkdir() + path = tmp_path / "sub" / "rig.yaml" + path.write_text(MINIMAL + "cwd: ..\n") + assert load_rig_config(path).cwd == tmp_path.resolve() + + +# --------------------------------------------------------------------------- +# Placeholder substitution +# --------------------------------------------------------------------------- + + +def test_python_placeholder_expands_to_sys_executable(tmp_path): + result = substitute_command( + "{python} -m isaacteleop.cloudxr", {}, tmp_path / "c.yaml" + ) + assert result == f"{shlex.quote(sys.executable)} -m isaacteleop.cloudxr" + + +def test_params_substituted(tmp_path): + result = substitute_command( + "./plugin {hand} {collection_id}", + {"hand": "left", "collection_id": "abc"}, + tmp_path / "c.yaml", + ) + assert result == "./plugin left abc" + + +def test_unknown_placeholder_is_hard_error(tmp_path): + with pytest.raises(RigConfigError, match=r"unknown placeholder \{hand\}") as exc: + substitute_command("./plugin {hand}", {}, tmp_path / "c.yaml") + # The remedy mentions brace escaping. + assert "{{" in str(exc.value) + + +def test_malformed_brace_is_hard_error_mentioning_escaping(tmp_path): + with pytest.raises(RigConfigError, match=r"\{\{"): + substitute_command("echo ${VAR:-x}", {}, tmp_path / "c.yaml") + + +def test_escaped_braces_pass_through(tmp_path): + assert ( + substitute_command("echo {{literal}}", {}, tmp_path / "c.yaml") + == "echo {literal}" + ) + + +# --------------------------------------------------------------------------- +# Foot-gun lint (warn-only, narrow) +# --------------------------------------------------------------------------- + + +def _proc(command: str) -> ProcessConfig: + return ProcessConfig(name="app", command=command) + + +def test_footgun_fires_on_python_script_without_flag(): + warnings = find_runtime_footguns( + [_proc("{python} examples/teleop/python/example.py {collection_id}")], + runtime_managed=True, + ) + assert len(warnings) == 1 + assert "--no-launch-cloudxr-runtime" in warnings[0] + # Names the symptom, not just the flag. + assert "kills the runtime pane" in warnings[0] + + +def test_footgun_fires_on_module_invocation(): + warnings = find_runtime_footguns( + [_proc("{python} -m isaacteleop.examples.foo")], runtime_managed=True + ) + assert len(warnings) == 1 + + +def test_footgun_quiet_when_flag_present(): + assert ( + find_runtime_footguns( + [_proc("{python} example.py --no-launch-cloudxr-runtime")], + runtime_managed=True, + ) + == [] + ) + + +def test_footgun_quiet_on_cpp_binary(): + assert ( + find_runtime_footguns( + [_proc("install/examples/schemaio/se3_printer {collection_id}")], + runtime_managed=True, + ) + == [] + ) + + +def test_footgun_quiet_when_runtime_not_managed(): + assert ( + find_runtime_footguns([_proc("{python} example.py")], runtime_managed=False) + == [] + ) diff --git a/src/core/rig_tests/python/test_launcher.py b/src/core/rig_tests/python/test_launcher.py new file mode 100644 index 000000000..3dc01a116 --- /dev/null +++ b/src/core/rig_tests/python/test_launcher.py @@ -0,0 +1,624 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Unit tests for the tmux pane plan (fake tmux — no tmux/headset needed).""" + +from __future__ import annotations + +import shlex +import subprocess +import sys +from pathlib import Path + +import pytest +from rig_py_test_ns.config import RigConfig, ProcessConfig +from rig_py_test_ns.launcher import PreflightError, kill_rig, launch_rig + + +class FakeTmux: + """Recording fake for the ``run_tmux`` seam; hands out canned pane ids.""" + + def __init__(self, existing_sessions: set[str] | None = None): + self.calls: list[list[str]] = [] + self.sessions = existing_sessions or set() + self._pane_counter = 0 + + def __call__(self, args): + args = list(args) + self.calls.append(args) + if args[0] == "has-session": + session = args[args.index("-t") + 1] + if session not in self.sessions: + raise subprocess.CalledProcessError(1, ["tmux", *args]) + return "" + if args[0] in ("new-session", "split-window"): + self._pane_counter += 1 + return f"%{self._pane_counter}" + return "" + + def named(self, name: str) -> list[list[str]]: + return [c for c in self.calls if c[0] == name] + + def pane_commands(self) -> list[str]: + """The wrapper shell-command each pane was spawned running, plan order.""" + return [c[-1] for c in self.calls if c[0] in ("new-session", "split-window")] + + +def pretype_line(wrapper: str) -> str: + """The single line of *wrapper* that pre-types the rig command.""" + (line,) = [ + line + for line in wrapper.splitlines() + if line.startswith("tmux send-keys") and " -l " in line + ] + return line + + +def pretyped_command(wrapper: str) -> str: + """Extract the literal rig command a pane wrapper pre-types into itself. + + Also asserts the send-keys shape: targeted at the pane's own tty, in + literal mode (``-l``: no tmux key-name lookup) behind a ``--`` + terminator (a payload starting with ``-`` must not parse as an option). + """ + tokens = shlex.split(pretype_line(wrapper)) + assert tokens[:4] == ["tmux", "send-keys", "-t", "$TMUX_PANE"] + assert tokens[-3:-1] == ["-l", "--"] + return tokens[-1] + + +def make_exe(tmp_path: Path, rel: str) -> str: + """Create an executable at ``tmp_path/rel``; return the relative path.""" + path = tmp_path / rel + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("#!/bin/sh\n") + path.chmod(0o755) + return rel + + +def make_config(tmp_path: Path, **kwargs) -> RigConfig: + producer = make_exe(tmp_path, "install/plugins/foo/foo_plugin") + consumer = make_exe(tmp_path, "install/examples/bar/bar_printer") + defaults = dict( + name="test_rig", + description="test rig", + cwd=tmp_path, + params={"hand": "right", "collection_id": "cid"}, + runtime=None, + producers=( + ProcessConfig("foo plugin", f"{producer} {{hand}} {{collection_id}}"), + ), + consumers=(ProcessConfig("bar printer", f"{consumer} {{collection_id}}"),), + source=tmp_path / "rig.yaml", + ) + defaults.update(kwargs) + return RigConfig(**defaults) + + +@pytest.fixture(autouse=True) +def clean_env(monkeypatch): + monkeypatch.delenv("TMUX", raising=False) + monkeypatch.delenv("PYTHONPATH", raising=False) + + +# --------------------------------------------------------------------------- +# Fresh-launch pane plan +# --------------------------------------------------------------------------- + + +def test_fresh_launch_call_plan(tmp_path): + tmux = FakeTmux() + launch_rig(make_config(tmp_path), run_tmux=tmux) + + # Session created detached, addressed by pane id, in the rig cwd; the + # first pane is SPAWNED RUNNING the runtime wrapper (trailing + # shell-command), never a bare shell the launcher types into. + (new_session,) = tmux.named("new-session") + assert new_session[:-1] == [ + "new-session", + "-d", + "-P", + "-F", + "#{pane_id}", + "-s", + "test_rig", + "-n", + "rig", + "-c", + str(tmp_path), + ] + runtime_wrapper = new_session[-1] + + # Session-scoped options only (-t ), never global (-g): + # pane-border-status plus the runtime strip height for main-horizontal. + set_options = tmux.named("set-option") + assert ["set-option", "-t", "test_rig", "pane-border-status", "top"] in set_options + assert [ + "set-option", + "-t", + "test_rig", + "main-pane-height", + "25%", + ] in set_options + assert all("-g" not in c for c in set_options) + + # Panes split -v (direction is irrelevant — re-laid out below); each + # split pane spawns running its own worker wrapper. + splits = tmux.named("split-window") + assert [c[1] for c in splits] == ["-v", "-v"] + assert splits[0][splits[0].index("-t") + 1] == "%1" + assert splits[1][splits[1].index("-t") + 1] == "%2" + + # Interleaved tiled re-layouts keep splits from running out of space; + # the final layout is main-horizontal: runtime = slim top strip, + # workers tiled below. + layouts = [c[-1] for c in tmux.named("select-layout")] + assert layouts == ["tiled", "tiled", "main-horizontal"] + assert all(c[c.index("-t") + 1] == "test_rig" for c in tmux.named("select-layout")) + + # The launcher never send-keys into a pane: keystrokes racing shell + # startup are echoed raw by the tty and again by the line editor. + assert not tmux.named("send-keys") + + # Runtime wrapper: echo off, command pre-typed into its own tty and + # executed (C-m) while echo is off, echo restored, exec user shell. + lines = runtime_wrapper.splitlines() + assert lines[0] == "stty -echo" + assert ( + pretyped_command(runtime_wrapper) + == f"{shlex.quote(sys.executable)} -m isaacteleop.cloudxr --accept-eula" + ) + enter = 'tmux send-keys -t "$TMUX_PANE" C-m' + assert lines.index(enter) < lines.index("stty echo") + assert lines[-1] == 'exec "${SHELL:-sh}" -l' + + # Worker wrappers: echo off, env wait+source RUNS, then — runtime up — + # banner RUNS, the substituted command RUNS inside the wrapper, its exit + # status is reported with a rerun hint, and the SAME command is + # pre-typed WITHOUT Enter for the trailing interactive shell. + for wrapper, expected in zip( + [c[-1] for c in splits], + ( + "install/plugins/foo/foo_plugin right cid", + "install/examples/bar/bar_printer cid", + ), + ): + lines = wrapper.splitlines() + assert lines[0] == "stty -echo" + assert "runtime_started" in lines[1] # env wait+source line + assert lines[2].startswith("if [ -e ") # auto-run gated on the sentinel + assert lines[3].startswith("echo ") # banner announces the auto-run + assert f"running: {expected}" in lines[3] + assert f"sh -c {shlex.quote(expected)}" in lines # command RUNS in wrapper + assert ( + 'echo "[rig] command exited with status $s — press Enter to rerun"' in lines + ) + assert pretyped_command(wrapper) == expected + # The rerun pre-type happens AFTER the command ran (outside the if). + assert lines.index(pretype_line(wrapper)) > lines.index("fi") + assert "C-m" not in wrapper # the rerun awaits a real Enter + assert lines[-2:] == ["stty echo", 'exec "${SHELL:-sh}" -l'] + + # Pane titles carry role, name, and the auto-run behavior. + titles = [c for c in tmux.named("select-pane") if "-T" in c] + title_texts = [c[c.index("-T") + 1] for c in titles] + assert title_texts[0] == "runtime: isaacteleop.cloudxr (running)" + assert "producer: foo plugin" in title_texts[1] + assert "auto-runs once the runtime is up" in title_texts[1] + assert "consumer: bar printer" in title_texts[2] + + # Runtime pane focused; garnish message; attach last (TMUX unset). + assert tmux.named("select-pane")[-1] == ["select-pane", "-t", "%1"] + assert tmux.named("display-message") + assert tmux.calls[-1][0] == "attach-session" + assert tmux.calls[-1][-1] == "test_rig" + + +def test_instructions_printed_before_attach(tmp_path, capsys): + tmux = FakeTmux() + launch_rig(make_config(tmp_path), run_tmux=tmux) + out = capsys.readouterr().out + assert "runs automatically once the runtime is up" in out + assert "press Enter in its pane to rerun" in out + assert "tmux kill-session -t test_rig" in out + assert "test rig" in out # the rig description is shown + + +def test_layout_scales_to_rigs_with_many_panes(tmp_path): + producer = "install/plugins/foo/foo_plugin" + config = make_config( + tmp_path, + producers=tuple( + ProcessConfig(f"plugin {i}", f"{producer} {{hand}} {{collection_id}}") + for i in range(3) + ), + ) + tmux = FakeTmux() + launch_rig(config, run_tmux=tmux) # runtime + 3 producers + 1 consumer + # A tiled re-layout after every split keeps chained splits from hitting + # tmux's "pane too small" limit; the final layout is main-horizontal. + layouts = [c[-1] for c in tmux.named("select-layout")] + assert layouts == ["tiled"] * 4 + ["main-horizontal"] + + +def test_no_runtime_stays_tiled_without_main_pane(tmp_path): + tmux = FakeTmux() + launch_rig(make_config(tmp_path), no_runtime=True, run_tmux=tmux) + # All panes are peer workers: no runtime strip, no main-horizontal. + layouts = [c[-1] for c in tmux.named("select-layout")] + assert layouts == ["tiled"] # one split between the two workers + assert not any("main-pane-height" in c for c in tmux.named("set-option")) + + +def test_switch_client_when_inside_tmux(tmp_path, monkeypatch): + monkeypatch.setenv("TMUX", "/tmp/tmux-1000/default,1234,0") + tmux = FakeTmux() + launch_rig(make_config(tmp_path), run_tmux=tmux) + assert tmux.calls[-1] == ["switch-client", "-t", "test_rig"] + assert not tmux.named("attach-session") + + +# --------------------------------------------------------------------------- +# Idempotent session reuse +# --------------------------------------------------------------------------- + + +def test_existing_session_is_reused(tmp_path, capsys): + tmux = FakeTmux(existing_sessions={"test_rig"}) + launch_rig(make_config(tmp_path), run_tmux=tmux) + assert [c[0] for c in tmux.calls] == ["has-session", "attach-session"] + out = capsys.readouterr().out + assert "--kill" in out # points at the built-in kill, not raw tmux + assert "ignored for an existing session" not in out # nothing was ignored + + +def test_existing_session_notes_ignored_settings(tmp_path, capsys): + tmux = FakeTmux(existing_sessions={"test_rig"}) + launch_rig(make_config(tmp_path), no_runtime=True, run_tmux=tmux) + out = capsys.readouterr().out + assert "--no-runtime ignored for an existing session" in out + assert "kill it first" in out + + +# --------------------------------------------------------------------------- +# kill_rig +# --------------------------------------------------------------------------- + + +def test_kill_rig_kills_the_existing_session(tmp_path, capsys): + tmux = FakeTmux(existing_sessions={"test_rig"}) + kill_rig(make_config(tmp_path), run_tmux=tmux) + assert tmux.calls == [ + ["has-session", "-t", "test_rig"], + ["kill-session", "-t", "test_rig"], + ] + assert "killed session 'test_rig'" in capsys.readouterr().out + + +def test_kill_rig_is_idempotent_when_no_session(tmp_path, capsys): + tmux = FakeTmux() # no sessions + kill_rig(make_config(tmp_path), run_tmux=tmux) + assert not tmux.named("kill-session") + assert "no session 'test_rig' to kill" in capsys.readouterr().out + + +# --------------------------------------------------------------------------- +# --no-runtime +# --------------------------------------------------------------------------- + + +def test_no_runtime_skips_runtime_pane(tmp_path): + tmux = FakeTmux() + launch_rig(make_config(tmp_path), no_runtime=True, run_tmux=tmux) + assert len(tmux.named("split-window")) == 1 # two worker panes only + wrappers = tmux.pane_commands() + assert not any("isaacteleop.cloudxr" in w for w in wrappers) + # Both panes are workers and behave exactly as with a managed runtime: + # wait-and-source, auto-run, pre-typed rerun, no synthetic Enter. + for wrapper, expected in zip( + wrappers, + ( + "install/plugins/foo/foo_plugin right cid", + "install/examples/bar/bar_printer cid", + ), + ): + assert f"sh -c {shlex.quote(expected)}" in wrapper.splitlines() + assert pretyped_command(wrapper) == expected + assert "C-m" not in wrapper + + +# --------------------------------------------------------------------------- +# Worker auto-run: gated on the runtime sentinel, rerunnable via pre-type +# --------------------------------------------------------------------------- + + +def test_command_runs_only_when_runtime_is_ready(tmp_path): + """Auto-run is gated on the sentinel: on wait timeout the command must + NOT run (it would fail confusingly without the CloudXR env). + + The run line lives inside the ``if [ -e ]`` block; the rerun + pre-type lives after ``fi`` so BOTH paths leave the command at the + prompt (timeout: pre-typed but never run; happy path: rerun on Enter). + """ + tmux = FakeTmux() + launch_rig(make_config(tmp_path), run_tmux=tmux) + wrapper = tmux.pane_commands()[1] + lines = wrapper.splitlines() + run_at = lines.index( + f"sh -c {shlex.quote('install/plugins/foo/foo_plugin right cid')}" + ) + if_at = next(i for i, line in enumerate(lines) if line.startswith("if [ -e ")) + fi_at = lines.index("fi") + assert if_at < run_at < fi_at + # Exit status + rerun hint prints right after the command, inside the if. + exit_at = lines.index( + 'echo "[rig] command exited with status $s — press Enter to rerun"' + ) + assert run_at < exit_at < fi_at + assert lines.index(pretype_line(wrapper)) > fi_at + + +def test_ctrl_c_kills_the_app_not_the_pane(tmp_path): + """INT is trapped (no-op, so children still get default SIGINT) only + around the command run: Ctrl+C stops the app while the wrapper survives + to offer the pre-typed rerun. Without the trap a non-interactive sh + whose foreground child dies of SIGINT exits too — closing the pane. + """ + tmux = FakeTmux() + launch_rig(make_config(tmp_path), run_tmux=tmux) + for wrapper in tmux.pane_commands()[1:]: + lines = wrapper.splitlines() + run_at = next(i for i, line in enumerate(lines) if line.startswith("sh -c ")) + assert lines[run_at - 1] == "trap : INT" + assert lines[run_at + 1 : run_at + 3] == ["s=$?", "trap - INT"] + + +# --------------------------------------------------------------------------- +# CloudXR env auto-load in worker panes +# --------------------------------------------------------------------------- + + +def _env_wait_lines(tmux: FakeTmux) -> list[str]: + # The bounded-wait line, not the `if [ -e ]` auto-run gate + # (both mention the runtime_started sentinel). + return [ + line + for wrapper in tmux.pane_commands() + for line in wrapper.splitlines() + if "runtime_started" in line and "until" in line + ] + + +def test_worker_panes_wait_for_runtime_then_source_env(tmp_path): + tmux = FakeTmux() + launch_rig(make_config(tmp_path), run_tmux=tmux) + lines = _env_wait_lines(tmux) + assert len(lines) == 2 # one per worker pane, none in the runtime wrapper + assert "runtime_started" not in tmux.pane_commands()[0] # runtime pane + wait = lines[0] + # Waits on the runtime_started sentinel (recreated by the runtime once it + # is actually serving), then sources the env file it writes. + home = str(Path("~/.cloudxr/run").expanduser()) + assert f"{home}/runtime_started" in wait + assert f"{home}/cloudxr.env" in wait + # Bounded wait with an actionable fallback, never a stuck loop. + assert "runtime not ready" in wait + assert "source" in wait + # The wait line RUNS as part of the pane wrapper, before the banner and + # the pre-typed command, while tty echo is suppressed. + wrapper = tmux.pane_commands()[1] + wlines = wrapper.splitlines() + assert wlines[0] == "stty -echo" + assert wlines[1] == wait + assert wlines.index(wait) < wlines.index(pretype_line(wrapper)) + + +def test_env_wait_still_runs_with_no_runtime(tmp_path): + # An external runtime also writes the sentinel + env file: workers still + # wait-and-source (the wait returns immediately when it is already up). + tmux = FakeTmux() + launch_rig(make_config(tmp_path), no_runtime=True, run_tmux=tmux) + assert len(_env_wait_lines(tmux)) == 2 + + +def test_env_wait_honors_cloudxr_install_dir_from_runtime_override(tmp_path): + config = make_config( + tmp_path, + runtime="{python} -m isaacteleop.cloudxr --cloudxr-install-dir /opt/cxr", + ) + tmux = FakeTmux() + launch_rig(config, run_tmux=tmux) + wait = _env_wait_lines(tmux)[0] + assert "/opt/cxr/run/runtime_started" in wait + assert "/opt/cxr/run/cloudxr.env" in wait + + +def test_env_wait_honors_cxr_install_dir_env_var(tmp_path, monkeypatch): + monkeypatch.setenv("CXR_INSTALL_DIR", "/srv/cloudxr") + tmux = FakeTmux() + launch_rig(make_config(tmp_path), no_runtime=True, run_tmux=tmux) + wait = _env_wait_lines(tmux)[0] + assert "/srv/cloudxr/run/cloudxr.env" in wait + + +# --------------------------------------------------------------------------- +# Interpreter and PYTHONPATH propagation +# --------------------------------------------------------------------------- + + +def test_pythonpath_forwarded_only_to_python_commands(tmp_path, monkeypatch): + monkeypatch.setenv("PYTHONPATH", "/some/build/python_package") + config = make_config( + tmp_path, + consumers=( + ProcessConfig("py consumer", "{python} app.py --no-launch-cloudxr-runtime"), + ), + ) + tmux = FakeTmux() + launch_rig(config, run_tmux=tmux) + typed = [pretyped_command(w) for w in tmux.pane_commands()] + runtime_cmd, producer_cmd, consumer_cmd = typed + assert runtime_cmd.startswith("PYTHONPATH=/some/build/python_package ") + assert not producer_cmd.startswith("PYTHONPATH=") # C++ binary: untouched + assert consumer_cmd.startswith("PYTHONPATH=/some/build/python_package ") + + +def test_runtime_override_from_yaml(tmp_path): + config = make_config( + tmp_path, runtime="{python} -m isaacteleop.cloudxr --host-client" + ) + tmux = FakeTmux() + launch_rig(config, run_tmux=tmux) + assert pretyped_command(tmux.pane_commands()[0]).endswith("--host-client") + # Honest title: an overridden runtime is not labelled isaacteleop.cloudxr. + titles = [c[c.index("-T") + 1] for c in tmux.named("select-pane") if "-T" in c] + assert titles[0] == "runtime: custom runtime (running)" + + +# --------------------------------------------------------------------------- +# Wrapper quoting: hostile characters arrive literally +# --------------------------------------------------------------------------- + + +def test_hostile_characters_are_sent_literally(tmp_path): + hostile = "install/plugins/foo/foo_plugin 'a;b' \"$(rm -rf x)\" `date` {hand}" + config = make_config( + tmp_path, + producers=(ProcessConfig("hostile", hostile),), + ) + tmux = FakeTmux() + launch_rig(config, run_tmux=tmux) + expected = hostile.replace("{hand}", "right") + # The wrapper's embedded send-keys pre-types the exact command: quotes, + # command substitution, and backticks survive the wrapper shell verbatim + # (pretyped_command also asserts the -l / -- send-keys shape). + typed = [pretyped_command(w) for w in tmux.pane_commands()] + assert typed.count(expected) == 1 + + +def test_apostrophe_in_name_cannot_break_the_banner(tmp_path): + producer = make_exe(tmp_path, "install/plugins/foo/foo_plugin") + config = make_config( + tmp_path, + producers=( + ProcessConfig("foo's plugin (needs 'headset')", f"{producer} {{hand}}"), + ), + consumers=(), + ) + tmux = FakeTmux() + launch_rig(config, run_tmux=tmux) + banners = [ + line + for wrapper in tmux.pane_commands() + for line in wrapper.splitlines() + if line.startswith("echo ") and "running:" in line + ] + assert len(banners) == 1 + # The whole message is one shlex-quoted word: the name's quotes cannot + # terminate the echo argument or inject shell syntax into the wrapper. + words = shlex.split(banners[0]) + assert words[0] == "echo" + assert len(words) == 2 + assert "foo's plugin (needs 'headset')" in words[1] + + +def test_pane_machinery_is_never_typed_and_never_echoes(tmp_path): + """Regression: machinery typed into a starting shell displays twice. + + Keystrokes sent while the shell is still starting are echoed raw by the + tty line discipline, then re-echoed by the line editor at the prompt. + The machinery must run as the pane's spawn command, with tty echo off + around everything up to (and including) the pre-typing. + """ + tmux = FakeTmux() + launch_rig(make_config(tmp_path), run_tmux=tmux) + assert not tmux.named("send-keys") # launcher never types into a pane + for wrapper in tmux.pane_commands(): + lines = wrapper.splitlines() + assert lines[0] == "stty -echo" # echo off before anything else + # Pre-typing happens while echo is off: the last stty toggle before + # the pre-type line must be -echo (worker wrappers legitimately + # restore echo around RUNNING the command, then turn it off again). + stty_before = [ + line + for line in lines[: lines.index(pretype_line(wrapper))] + if line.startswith("stty ") + ] + assert stty_before[-1] == "stty -echo" + # Echo is handed back only at the end, before the user's shell. + assert lines[-2:] == ["stty echo", 'exec "${SHELL:-sh}" -l'] + + +# --------------------------------------------------------------------------- +# Preflight +# --------------------------------------------------------------------------- + + +def test_missing_binary_preflight_error(tmp_path): + config = make_config( + tmp_path, + producers=( + ProcessConfig("ghost", "install/plugins/ghost/ghost_plugin {hand}"), + ), + ) + with pytest.raises(PreflightError, match="not found or not executable") as exc: + launch_rig(config, run_tmux=FakeTmux()) + assert "cmake" in str(exc.value) # remedy included + + +def test_missing_cwd_preflight_error(tmp_path): + config = make_config(tmp_path, cwd=tmp_path / "nope") + with pytest.raises(PreflightError, match="does not exist"): + launch_rig(config, run_tmux=FakeTmux()) + + +def test_bare_command_names_skip_binary_check(tmp_path): + config = make_config( + tmp_path, + producers=(ProcessConfig("on path", "echo {hand}"),), + consumers=(), + ) + launch_rig(config, run_tmux=FakeTmux()) # must not raise + + +def test_footgun_warning_printed(tmp_path, capsys): + config = make_config( + tmp_path, + consumers=( + ProcessConfig("py consumer", "{python} session.py {collection_id}"), + ), + ) + launch_rig(config, run_tmux=FakeTmux()) + assert "--no-launch-cloudxr-runtime" in capsys.readouterr().err + + +def test_footgun_warning_shown_inside_the_offending_pane(tmp_path): + config = make_config( + tmp_path, + consumers=( + ProcessConfig("py consumer", "{python} session.py {collection_id}"), + ), + ) + tmux = FakeTmux() + launch_rig(config, run_tmux=tmux) + banners = [ + line + for wrapper in tmux.pane_commands() + for line in wrapper.splitlines() + if line.startswith("echo ") + ] + flagged = [b for b in banners if "--no-launch-cloudxr-runtime" in b] + assert len(flagged) == 1 # only the offending pane's banner warns + assert "WARNING" in flagged[0] + assert "kill the runtime pane" in flagged[0] + + +def test_footgun_warning_suppressed_with_no_runtime(tmp_path, capsys): + config = make_config( + tmp_path, + consumers=( + ProcessConfig("py consumer", "{python} session.py {collection_id}"), + ), + ) + launch_rig(config, no_runtime=True, run_tmux=FakeTmux()) + assert "--no-launch-cloudxr-runtime" not in capsys.readouterr().err diff --git a/src/core/rig_tests/python/test_main.py b/src/core/rig_tests/python/test_main.py new file mode 100644 index 000000000..7515874b0 --- /dev/null +++ b/src/core/rig_tests/python/test_main.py @@ -0,0 +1,39 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Unit tests for the python -m isaacteleop.rig CLI surface.""" + +from __future__ import annotations + +import pytest +from rig_py_test_ns.__main__ import main + + +def test_bare_invocation_prints_usage_not_a_stack_trace(capsys): + assert main([]) == 0 + out = capsys.readouterr().out + assert "usage:" in out + assert "rigs/se3_tracker.yaml" in out # one example command + pointer to rigs/ + + +def test_missing_rig_file_is_a_friendly_error(tmp_path, capsys): + assert main([str(tmp_path / "nope.yaml")]) == 1 + err = capsys.readouterr().err + assert err.startswith("error:") + assert "rig file not found" in err + + +def test_invalid_rig_is_a_friendly_error(tmp_path, capsys): + path = tmp_path / "bad.yaml" + path.write_text("name: x\nunknown_key: 1\n") + assert main([str(path)]) == 1 + assert "unknown top-level key" in capsys.readouterr().err + + +def test_kill_rejects_no_runtime(tmp_path, capsys): + path = tmp_path / "rig.yaml" + path.write_text("name: x\nconsumers:\n - name: y\n command: echo hi\n") + with pytest.raises(SystemExit) as exc: + main([str(path), "--kill", "--no-runtime"]) + assert exc.value.code == 2 + assert "--kill cannot be combined" in capsys.readouterr().err From 5ccd10854b1355e37d626466e797ed52ddc8f0b2 Mon Sep 17 00:00:00 2001 From: Jiwen Cai Date: Mon, 20 Jul 2026 00:14:10 +0000 Subject: [PATCH 2/2] Address review feedback: env-ready gating, reattach ordering, stale sentinel cleanup MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - rig/python/CMakeLists: create RIG_OUTPUT_DIR before copying so clean builds work on CMake < 3.26 (the repo floor is 3.20) - launcher: gate worker auto-run on cloudxr.env sourcing SUCCESS via a rig_env_ready flag, not on sentinel existence; [ -r ] guard prevents dash's fatal '.' on a missing file from killing the pane wrapper; distinct source-failure message (timeout message unchanged) - launcher: probe for an existing session before launch-only preflight — edits to a running rig's YAML never block reattachment (reattach needs only a loadable rig file and tmux) - launcher: clear a stale runtime_started sentinel before starting a managed runtime (workers probe within milliseconds; the runtime needs seconds to delete it itself); never under --no-runtime, where the external runtime owns the sentinel; an undeletable sentinel surfaces as a PreflightError with a remedy - rig/__init__: fix the stale 'pre-typed but NOT executed' docstring — panes auto-run; Enter is the rerun mechanism - rig_tests/CMakeLists: CONFIGURE_DEPENDS so new test files register with CTest without a manual reconfigure - config: read rig files as UTF-8; a non-UTF-8 file is a RigConfigError naming the real cause, never a traceback - tests: pin HOME in the autouse fixture so the suite can never unlink a real ~/.cloudxr sentinel; cover the reattach-despite-broken-plan path, the full sentinel ownership matrix, and the new messages - docs: extend step 4 wording and add a troubleshooting row for the source-failure message Tests: 66 passed (rig_tests via uv/pytest). Lint: pre-commit clean (SKIP=check-copyright-year). Clean-configure + rig_python target build verified from an empty build tree. Signed-off-by: Jiwen Cai --- docs/source/references/rig.rst | 11 +- src/core/rig/python/CMakeLists.txt | 1 + src/core/rig/python/__init__.py | 9 +- src/core/rig/python/config.py | 6 +- src/core/rig/python/launcher.py | 130 ++++++++++++------ src/core/rig_tests/python/CMakeLists.txt | 5 +- src/core/rig_tests/python/test_config.py | 9 ++ src/core/rig_tests/python/test_launcher.py | 151 +++++++++++++++++++-- 8 files changed, 260 insertions(+), 62 deletions(-) diff --git a/docs/source/references/rig.rst b/docs/source/references/rig.rst index f77143fd6..80a1eb470 100644 --- a/docs/source/references/rig.rst +++ b/docs/source/references/rig.rst @@ -78,9 +78,9 @@ What happens: Anything that calls ``xrGetSystem`` before a headset connects exits with ``Failed to get OpenXR system`` — so if a pane's app started before you connected the headset to the printed URL, connect it and press - :kbd:`Enter` in that pane to rerun. If the runtime never comes up, the - pane does *not* run the command; it prints a remedy and leaves the - command pre-typed instead. + :kbd:`Enter` in that pane to rerun. If the runtime never comes up (or + its environment fails to load), the pane does *not* run the command; it + prints a remedy and leaves the command pre-typed instead. 5. The launcher then attaches (from a plain shell) or switches your current client (from inside tmux — no nesting). @@ -200,6 +200,11 @@ Troubleshooting - The pane is still waiting for the runtime's ``runtime_started`` sentinel; check the runtime pane. After the (two-minute) wait times out, the pane prints a remedy and leaves the command pre-typed. + * - ``[cloudxr] runtime is up but loading ... failed`` + - The runtime came up but its ``cloudxr.env`` could not be sourced; + the pane does not run its command. Check the error printed above + the message, ``source`` the file as the message says, then press + :kbd:`Enter` — the command is already pre-typed. * - ``[rig] command exited with status ...`` right after launch - The app auto-ran before the headset connected (classic: ``Failed to get OpenXR system``); connect the headset to the diff --git a/src/core/rig/python/CMakeLists.txt b/src/core/rig/python/CMakeLists.txt index ea310e1f5..5a19083f8 100644 --- a/src/core/rig/python/CMakeLists.txt +++ b/src/core/rig/python/CMakeLists.txt @@ -21,6 +21,7 @@ foreach(PYTHON_FILE ${RIG_PYTHON_FILES}) add_custom_command( OUTPUT "${OUTPUT_PATH}" + COMMAND ${CMAKE_COMMAND} -E make_directory "${RIG_OUTPUT_DIR}" COMMAND ${CMAKE_COMMAND} -E copy "${PYTHON_FILE}" "${OUTPUT_PATH}" DEPENDS "${PYTHON_FILE}" COMMENT "Copying ${FILE_NAME} to rig package" diff --git a/src/core/rig/python/__init__.py b/src/core/rig/python/__init__.py index af99cbf84..87867ab08 100644 --- a/src/core/rig/python/__init__.py +++ b/src/core/rig/python/__init__.py @@ -13,8 +13,13 @@ python -m isaacteleop.rig rigs/se3_tracker.yaml -The runtime pane starts immediately; producer/consumer panes are pre-typed -but NOT executed — press Enter in each pane after the headset connects. +The runtime pane starts immediately. Each producer/consumer pane waits +(up to two minutes) for the runtime to come up, sources the CloudXR env, +and then RUNS its command automatically. A command that exits early (the +classic: started before the headset connected) is left pre-typed at an +interactive prompt — one Enter reruns it. If the runtime never comes up +(or its env fails to load) the command is NOT run; the pane prints what +to do and leaves the command pre-typed. The YAML schema is the stable contract; the Python API below is exported for tests and power users on a best-effort basis. diff --git a/src/core/rig/python/config.py b/src/core/rig/python/config.py index 4b861d5d0..83498b2b5 100644 --- a/src/core/rig/python/config.py +++ b/src/core/rig/python/config.py @@ -176,7 +176,11 @@ def load_rig_config(path: str | Path) -> RigConfig: "Teleop repository for an example" ) try: - data = yaml.safe_load(source.read_text()) + data = yaml.safe_load(source.read_text(encoding="utf-8")) + except UnicodeDecodeError as exc: + raise RigConfigError( + f"{source}: not valid UTF-8 (rig files must be UTF-8 text): {exc}" + ) from exc except yaml.YAMLError as exc: raise RigConfigError( f"{source}: invalid YAML: {exc} — see rigs/se3_tracker.yaml for a valid example" diff --git a/src/core/rig/python/launcher.py b/src/core/rig/python/launcher.py index aafba8fd3..0881f84f2 100644 --- a/src/core/rig/python/launcher.py +++ b/src/core/rig/python/launcher.py @@ -20,8 +20,9 @@ off (the keystrokes buffer invisibly in the tty), restores echo, and execs the user's shell — whose line editor then displays the buffered command once, at a real prompt, editable and awaiting Enter. If the runtime never -comes up the wrapper does NOT run the command (it would fail confusingly -without the env); it prints a remedy and pre-types the command instead. +comes up (or its env fails to load) the wrapper does NOT run the command +(it would fail confusingly without the env); it prints a remedy and +pre-types the command instead. All tmux interaction goes through a single injectable ``run_tmux`` seam so the pane plan is unit-testable without tmux or a headset. @@ -62,6 +63,11 @@ #: order of magnitude; the pane prints a manual remedy on expiry). CLOUDXR_ENV_WAIT_TIMEOUT_SEC = 120 +#: Sentinel file the runtime creates in its run dir once it is actually +#: serving. Worker panes wait on it; the launcher deletes a stale one +#: before starting a managed runtime (see :func:`launch_rig`). +_RUNTIME_STARTED_SENTINEL = "runtime_started" + #: Height of the runtime pane in the ``main-horizontal`` layout. The runtime #: only prints status lines and the web-client URL, so it gets a slim strip #: on top and the worker panes share the rest (tmux accepts a percentage). @@ -167,24 +173,43 @@ def _cloudxr_env_wait_command(run_dir: Path) -> str: """Build the shell line each worker pane RUNS to load the CloudXR env. OpenXR producers/consumers need the env the runtime writes to - ``/cloudxr.env`` (``XR_RUNTIME_JSON`` etc.). The runtime deletes - any stale ``runtime_started`` sentinel at startup and recreates it only - once it is actually serving, so the sentinel is the "runtime successfully - started" signal to wait on. The wait is bounded: a runtime that never - comes up leaves an actionable message in the pane, not a stuck loop. + ``/cloudxr.env`` (``XR_RUNTIME_JSON`` etc.). The launcher + deletes any stale ``runtime_started`` sentinel before starting a managed + runtime, and the runtime recreates it only once it is actually serving, + so the sentinel is the "runtime successfully started" signal to wait on. + The wait is bounded: a runtime that never comes up leaves an actionable + message in the pane, not a stuck loop. + + Sets ``rig_env_ready=1`` only after ``cloudxr.env`` was successfully + sourced; :func:`_worker_pane_command` gates the auto-run on that shell + variable. Both the ``.`` and the flag assignment run in the wrapper's + top-level shell — never a subshell — so the variable AND the sourced + exports survive into the rest of the wrapper (and the final exec'd + shell). The ``[ -r ... ]`` guard is load-bearing: ``.`` is a POSIX + special built-in that aborts a non-interactive shell when the file is + missing or unreadable, which would kill the pane wrapper. """ - sentinel = shlex.quote(str(run_dir / "runtime_started")) + sentinel = shlex.quote(str(run_dir / _RUNTIME_STARTED_SENTINEL)) env_file = str(run_dir / "cloudxr.env") + quoted_env = shlex.quote(env_file) ok_msg = shlex.quote("[cloudxr] runtime is up — env loaded") - fail_msg = shlex.quote( + source_fail_msg = shlex.quote( + f"[cloudxr] runtime is up but loading {env_file} failed — see any " + f"errors above, then run: source {env_file}" + ) + timeout_msg = shlex.quote( f"[cloudxr] runtime not ready after {CLOUDXR_ENV_WAIT_TIMEOUT_SEC}s — " f"check the runtime pane, then run: source {env_file}" ) return ( + f"rig_env_ready=0; " f"i=0; until [ -e {sentinel} ] || [ $i -ge {CLOUDXR_ENV_WAIT_TIMEOUT_SEC} ]; " f"do sleep 1; i=$((i+1)); done; " - f"if [ -e {sentinel} ]; then . {shlex.quote(env_file)} && echo {ok_msg}; " - f"else echo {fail_msg}; fi" + f"if [ -e {sentinel} ]; then " + f"if [ -r {quoted_env} ] && . {quoted_env}; then " + f"rig_env_ready=1; echo {ok_msg}; " + f"else echo {source_fail_msg}; fi; " + f"else echo {timeout_msg}; fi" ) @@ -213,19 +238,47 @@ def launch_rig( ``params:`` block and are substituted into every command that references them (edit the file to change them). + An existing session is re-attached BEFORE any launch-only preflight + (plan substitution, cwd/command checks): edits to a running rig's YAML + can never block getting back into the live session. + Args: config: The parsed rig (see :func:`~.config.load_rig_config`). no_runtime: Skip the runtime pane (a CloudXR runtime is already running elsewhere, e.g. after ``python -m isaacteleop.cloudxr``). run_tmux: Injectable tmux seam for tests. When ``None`` the real - tmux is used and environment preflight checks (tmux on PATH, - interpreter can import isaacteleop.cloudxr) run first. + tmux is used: tmux-on-PATH is checked immediately (the session + probe needs it) and the interpreter-can-import- + isaacteleop.cloudxr probe runs with the launch-only preflight. Raises: PreflightError: When a launch precondition fails. RigConfigError: On bad placeholders. """ env = os.environ + + using_real_tmux = run_tmux is None + if using_real_tmux: + _check_tmux_installed() + run_tmux = _run_tmux + + # Reattach must win before any launch-only preflight below: a broken + # edit to a running rig's YAML must never block reattachment. + session = config.name + if _session_exists(run_tmux, session): + message = ( + f"session '{session}' already exists — switching to it " + f"(kill with: python -m isaacteleop.rig {config.source} --kill)" + ) + if no_runtime: + message += ( + "\nnote: --no-runtime ignored for an existing session; " + "kill it first to relaunch with new settings" + ) + print(message) + _goto_session(run_tmux, session, env) + return + params = config.params # Resolve the pane plan. Runtime first (starts immediately); producers @@ -270,30 +323,25 @@ def launch_rig( [*config.producers, *config.consumers], runtime_managed=not no_runtime ): print(warning, file=sys.stderr) - - if run_tmux is None: - _check_tmux_installed() - if any("{python}" in raw for _, _, raw, _, _ in plan): - _check_python_can_import_cloudxr(env) - run_tmux = _run_tmux - - session = config.name - if _session_exists(run_tmux, session): - message = ( - f"session '{session}' already exists — switching to it " - f"(kill with: python -m isaacteleop.rig {config.source} --kill)" - ) - if no_runtime: - message += ( - "\nnote: --no-runtime ignored for an existing session; " - "kill it first to relaunch with new settings" - ) - print(message) - _goto_session(run_tmux, session, env) - return + if using_real_tmux and any("{python}" in raw for _, _, raw, _, _ in plan): + _check_python_can_import_cloudxr(env) runtime_resolved = plan[0][3] if not no_runtime else None cloudxr_run_dir = _cloudxr_run_dir(runtime_resolved, env) + if not no_runtime: + # A stale sentinel from a prior run must not gate workers open before + # THIS runtime is serving: workers spawn (and test the sentinel) + # within milliseconds, while the runtime needs seconds to boot and + # delete it itself. Under --no-runtime the sentinel belongs to the + # external runtime — never touch it. + stale = cloudxr_run_dir / _RUNTIME_STARTED_SENTINEL + try: + stale.unlink(missing_ok=True) + except OSError as exc: + raise PreflightError( + f"cannot remove stale {stale}: {exc} — fix permissions on " + f"{cloudxr_run_dir} (was the runtime run with sudo?)" + ) from exc _create_session(run_tmux, session, config.cwd, plan, env, cloudxr_run_dir) _print_instructions(session, config.description, plan) _goto_session(run_tmux, session, env) @@ -394,7 +442,7 @@ def _worker_pane_command( With tty echo off (so none of this machinery ever appears as typed input): wait for the runtime and source the CloudXR env (see - :func:`_cloudxr_env_wait_command`). Runtime up? Print the auto-run + :func:`_cloudxr_env_wait_command`). Env sourced? Print the auto-run banner and RUN the command (echo restored so the app's tty behaves normally; via ``sh -c`` so a syntax error in the command cannot kill the wrapper, and never ``exec`` — the pane must survive the exit); when @@ -402,20 +450,22 @@ def _worker_pane_command( by pre-typing the command WITHOUT Enter (with echo off again), restore echo, and exec the user's shell — which inherits the sourced env (``cloudxr.env`` uses ``export``) and displays the pre-typed command - once, at its prompt: one Enter reruns. On sentinel timeout the command - is NOT run (it would fail confusingly without the env); the wait line - already printed the remedy and the pre-typed command awaits. + once, at its prompt: one Enter reruns. On wait timeout OR env-load + failure the command is NOT run (it would fail confusingly without the + env): the auto-run is gated on the ``rig_env_ready`` shell variable the + wait line sets only after a successful source — never on the sentinel, + which can exist while the env failed to load. The wait line already + printed the remedy and the pre-typed command awaits. ``trap : INT`` while the command runs: Ctrl+C must kill the app, not the wrapper (a non-interactive sh whose foreground child dies of SIGINT exits too — taking the pane with it — unless INT is trapped). """ - sentinel = shlex.quote(str(run_dir / "runtime_started")) return "\n".join( [ "stty -echo", _cloudxr_env_wait_command(run_dir), - f"if [ -e {sentinel} ]; then", + 'if [ "$rig_env_ready" -eq 1 ]; then', _autorun_banner(role, name, command, footgun), "stty echo", "trap : INT", diff --git a/src/core/rig_tests/python/CMakeLists.txt b/src/core/rig_tests/python/CMakeLists.txt index 94dbf6f0e..faa0208e2 100644 --- a/src/core/rig_tests/python/CMakeLists.txt +++ b/src/core/rig_tests/python/CMakeLists.txt @@ -8,9 +8,12 @@ # Discover and add individual Python tests # This makes each test show up separately in CTest output -# First, find all test files (only in this directory, not recursively) +# First, find all test files (only in this directory, not recursively). +# CONFIGURE_DEPENDS: a new test_*.py re-triggers cmake at build time, so +# it registers with CTest without a manual reconfigure. file(GLOB TEST_FILES RELATIVE "${CMAKE_CURRENT_SOURCE_DIR}" + CONFIGURE_DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/test_*.py" ) diff --git a/src/core/rig_tests/python/test_config.py b/src/core/rig_tests/python/test_config.py index cf78d59ca..6618ddcd6 100644 --- a/src/core/rig_tests/python/test_config.py +++ b/src/core/rig_tests/python/test_config.py @@ -83,6 +83,15 @@ def test_invalid_yaml_is_config_error(tmp_path): load_rig_config(path) +def test_non_utf8_rig_file_is_config_error(tmp_path): + # Rig files are read as UTF-8 on every platform; a non-UTF-8 file must + # fail as a config error (never a stack trace), naming the real cause. + path = tmp_path / "rig.yaml" + path.write_bytes("name: café\n".encode("latin-1")) + with pytest.raises(RigConfigError, match="not valid UTF-8"): + load_rig_config(path) + + def test_unknown_top_level_key_is_hard_error(tmp_path): path = write_rig(tmp_path, MINIMAL + "streams: {}\n") with pytest.raises(RigConfigError, match=r"unknown top-level key.*streams"): diff --git a/src/core/rig_tests/python/test_launcher.py b/src/core/rig_tests/python/test_launcher.py index 3dc01a116..163296829 100644 --- a/src/core/rig_tests/python/test_launcher.py +++ b/src/core/rig_tests/python/test_launcher.py @@ -96,9 +96,14 @@ def make_config(tmp_path: Path, **kwargs) -> RigConfig: @pytest.fixture(autouse=True) -def clean_env(monkeypatch): +def clean_env(monkeypatch, tmp_path): monkeypatch.delenv("TMUX", raising=False) monkeypatch.delenv("PYTHONPATH", raising=False) + # Hermetic run-dir resolution: launch_rig clears a stale runtime_started + # sentinel under the resolved run dir (default ~/.cloudxr/run), so the + # suite must never resolve to — and unlink in — a developer's real HOME. + monkeypatch.setenv("HOME", str(tmp_path)) + monkeypatch.delenv("CXR_INSTALL_DIR", raising=False) # --------------------------------------------------------------------------- @@ -186,7 +191,8 @@ def test_fresh_launch_call_plan(tmp_path): lines = wrapper.splitlines() assert lines[0] == "stty -echo" assert "runtime_started" in lines[1] # env wait+source line - assert lines[2].startswith("if [ -e ") # auto-run gated on the sentinel + # Auto-run gated on the env having been SOURCED, not on the sentinel. + assert lines[2] == 'if [ "$rig_env_ready" -eq 1 ]; then' assert lines[3].startswith("echo ") # banner announces the auto-run assert f"running: {expected}" in lines[3] assert f"sh -c {shlex.quote(expected)}" in lines # command RUNS in wrapper @@ -280,6 +286,23 @@ def test_existing_session_notes_ignored_settings(tmp_path, capsys): assert "kill it first" in out +def test_existing_session_reattaches_despite_launch_preflight_failures(tmp_path): + """Reattach must win before any launch-only preflight: edits to a + running rig's YAML (ghost binary, missing cwd, undeclared placeholder) + must never block getting back into the live session. + """ + config = make_config( + tmp_path, + cwd=tmp_path / "nope", + producers=( + ProcessConfig("ghost", "install/plugins/ghost/ghost_plugin {undeclared}"), + ), + ) + tmux = FakeTmux(existing_sessions={"test_rig"}) + launch_rig(config, run_tmux=tmux) # must not raise + assert [c[0] for c in tmux.calls] == ["has-session", "attach-session"] + + # --------------------------------------------------------------------------- # kill_rig # --------------------------------------------------------------------------- @@ -328,17 +351,19 @@ def test_no_runtime_skips_runtime_pane(tmp_path): # --------------------------------------------------------------------------- -# Worker auto-run: gated on the runtime sentinel, rerunnable via pre-type +# Worker auto-run: gated on the env being sourced, rerunnable via pre-type # --------------------------------------------------------------------------- -def test_command_runs_only_when_runtime_is_ready(tmp_path): - """Auto-run is gated on the sentinel: on wait timeout the command must - NOT run (it would fail confusingly without the CloudXR env). +def test_command_runs_only_when_env_is_ready(tmp_path): + """Auto-run is gated on ``rig_env_ready`` (set only after cloudxr.env + sourced successfully): on wait timeout OR env-load failure the command + must NOT run (it would fail confusingly without the CloudXR env). The + sentinel alone is not enough — it can exist while the env failed to load. - The run line lives inside the ``if [ -e ]`` block; the rerun - pre-type lives after ``fi`` so BOTH paths leave the command at the - prompt (timeout: pre-typed but never run; happy path: rerun on Enter). + The run line lives inside the ``if [ "$rig_env_ready" -eq 1 ]`` block; + the rerun pre-type lives after ``fi`` so ALL paths leave the command at + the prompt (failure: pre-typed but never run; happy path: rerun on Enter). """ tmux = FakeTmux() launch_rig(make_config(tmp_path), run_tmux=tmux) @@ -347,7 +372,7 @@ def test_command_runs_only_when_runtime_is_ready(tmp_path): run_at = lines.index( f"sh -c {shlex.quote('install/plugins/foo/foo_plugin right cid')}" ) - if_at = next(i for i, line in enumerate(lines) if line.startswith("if [ -e ")) + if_at = lines.index('if [ "$rig_env_ready" -eq 1 ]; then') fi_at = lines.index("fi") assert if_at < run_at < fi_at # Exit status + rerun hint prints right after the command, inside the if. @@ -358,6 +383,29 @@ def test_command_runs_only_when_runtime_is_ready(tmp_path): assert lines.index(pretype_line(wrapper)) > fi_at +def test_env_ready_flag_set_only_after_successful_source(tmp_path): + """The wait line owns the ``rig_env_ready`` contract: initialized to 0 + before the wait, promoted to 1 only in the branch where sourcing + ``cloudxr.env`` succeeded — behind a ``[ -r ... ]`` guard, because + ``.`` on a missing file aborts a non-interactive shell (killing the + pane wrapper). Each failure names its own cause: sentinel timeout vs + env-load failure have different remedies. + """ + tmux = FakeTmux() + launch_rig(make_config(tmp_path), run_tmux=tmux) + wait = tmux.pane_commands()[1].splitlines()[1] + assert wait.startswith("rig_env_ready=0; ") # init precedes the wait + env_file = str(Path("~/.cloudxr/run/cloudxr.env").expanduser()) + # Promotion to 1 happens only behind the guarded successful source. + success = f"if [ -r {env_file} ] && . {env_file}; then rig_env_ready=1; " + assert success in wait + assert wait.count("rig_env_ready=1") == 1 + # Distinct failure messages: timeout (runtime never came up) vs + # source-failure (runtime up, env unreadable) — both name a remedy. + assert "runtime not ready" in wait + assert f"loading {env_file} failed" in wait + + def test_ctrl_c_kills_the_app_not_the_pane(tmp_path): """INT is trapped (no-op, so children still get default SIGINT) only around the command run: Ctrl+C stops the app while the wrapper survives @@ -379,8 +427,8 @@ def test_ctrl_c_kills_the_app_not_the_pane(tmp_path): def _env_wait_lines(tmux: FakeTmux) -> list[str]: - # The bounded-wait line, not the `if [ -e ]` auto-run gate - # (both mention the runtime_started sentinel). + # The bounded-wait line (the only wrapper line naming the sentinel; the + # auto-run gate tests the rig_env_ready flag it sets). return [ line for wrapper in tmux.pane_commands() @@ -422,18 +470,24 @@ def test_env_wait_still_runs_with_no_runtime(tmp_path): def test_env_wait_honors_cloudxr_install_dir_from_runtime_override(tmp_path): + # An absolute override dir must be hermetic (tmp_path-based): launch_rig + # clears a stale sentinel under the resolved run dir, and HOME pinning + # does not cover absolute --cloudxr-install-dir paths. + install = tmp_path / "cxr" config = make_config( tmp_path, - runtime="{python} -m isaacteleop.cloudxr --cloudxr-install-dir /opt/cxr", + runtime=f"{{python}} -m isaacteleop.cloudxr --cloudxr-install-dir {install}", ) tmux = FakeTmux() launch_rig(config, run_tmux=tmux) wait = _env_wait_lines(tmux)[0] - assert "/opt/cxr/run/runtime_started" in wait - assert "/opt/cxr/run/cloudxr.env" in wait + assert f"{install}/run/runtime_started" in wait + assert f"{install}/run/cloudxr.env" in wait def test_env_wait_honors_cxr_install_dir_env_var(tmp_path, monkeypatch): + # Hermetic only via no_runtime=True (no sentinel unlink on that path) — + # a managed launch here would point an unlink at the real /srv/cloudxr. monkeypatch.setenv("CXR_INSTALL_DIR", "/srv/cloudxr") tmux = FakeTmux() launch_rig(make_config(tmp_path), no_runtime=True, run_tmux=tmux) @@ -441,6 +495,73 @@ def test_env_wait_honors_cxr_install_dir_env_var(tmp_path, monkeypatch): assert "/srv/cloudxr/run/cloudxr.env" in wait +# --------------------------------------------------------------------------- +# Stale runtime_started sentinel cleanup (managed runtimes only) +# --------------------------------------------------------------------------- + + +def _default_sentinel(tmp_path: Path) -> Path: + # clean_env pins HOME to tmp_path, so this is the resolved default run dir. + return tmp_path / ".cloudxr" / "run" / "runtime_started" + + +def test_stale_sentinel_cleared_before_managed_launch(tmp_path): + """A sentinel left by a prior run must be gone before any pane exists: + workers probe it within milliseconds, while the new runtime needs + seconds to boot and delete it itself — the workers win that race. + """ + sentinel = _default_sentinel(tmp_path) + sentinel.parent.mkdir(parents=True) + sentinel.touch() + launch_rig(make_config(tmp_path), run_tmux=FakeTmux()) + assert not sentinel.exists() + + +def test_no_runtime_preserves_external_sentinel(tmp_path): + # Under --no-runtime the sentinel belongs to an external, live runtime + # and is never recreated by this launch: deleting it would strand every + # worker for the full wait timeout. + sentinel = _default_sentinel(tmp_path) + sentinel.parent.mkdir(parents=True) + sentinel.touch() + launch_rig(make_config(tmp_path), no_runtime=True, run_tmux=FakeTmux()) + assert sentinel.exists() + + +def test_reattach_preserves_live_sentinel(tmp_path): + # Reattaching to an existing session must not clear the sentinel its + # (live) runtime owns — the delete sits after the reattach early-return. + sentinel = _default_sentinel(tmp_path) + sentinel.parent.mkdir(parents=True) + sentinel.touch() + tmux = FakeTmux(existing_sessions={"test_rig"}) + launch_rig(make_config(tmp_path), run_tmux=tmux) + assert sentinel.exists() + + +def test_absent_sentinel_is_not_an_error(tmp_path): + # First-ever launch: no run dir, no sentinel — the cleanup must be a + # silent no-op, not a crash. + assert not _default_sentinel(tmp_path).exists() + launch_rig(make_config(tmp_path), run_tmux=FakeTmux()) # must not raise + + +def test_undeletable_sentinel_is_preflight_error(tmp_path, monkeypatch): + # A stale sentinel the user cannot remove (e.g. a root-owned run dir + # from a sudo-run runtime) must surface as a PreflightError with a + # remedy — never a raw traceback (the CLI promises exit codes only). + sentinel = _default_sentinel(tmp_path) + sentinel.parent.mkdir(parents=True) + sentinel.touch() + + def deny(self, missing_ok=False): + raise PermissionError(13, "Permission denied") + + monkeypatch.setattr(Path, "unlink", deny) + with pytest.raises(PreflightError, match="fix permissions"): + launch_rig(make_config(tmp_path), run_tmux=FakeTmux()) + + # --------------------------------------------------------------------------- # Interpreter and PYTHONPATH propagation # ---------------------------------------------------------------------------