From 96c92c94e8093450a56ea7f13944040627135871 Mon Sep 17 00:00:00 2001 From: Jiwen Cai Date: Mon, 22 Jun 2026 01:24:01 +0000 Subject: [PATCH 1/2] mcap examples: add live pose viz + refactor CloudXRLauncher + consolidate viz classes - Add live_hand.py, live_controller.py, live_full_body.py: real-time viser visualization of XR pose data using CloudXRLauncher + TeleopSession (no MCAP) - Refactor record_* scripts to use CloudXRLauncher so the CloudXR runtime starts automatically; add --accept-eula / --install-dir args - Move HandViz, ControllerViz, FullBodyViz and shared color constants into common.py; replay_* scripts now import from there instead of duplicating Signed-off-by: Jiwen Cai --- examples/mcap_record_replay/python/common.py | 302 +++++++++++++++++- .../python/live_controller.py | 105 ++++++ .../python/live_full_body.py | 106 ++++++ .../mcap_record_replay/python/live_hand.py | 102 ++++++ .../python/record_controller.py | 77 +++-- .../python/record_full_body.py | 81 +++-- .../mcap_record_replay/python/record_hand.py | 65 ++-- .../python/replay_controller.py | 190 +---------- .../python/replay_full_body.py | 60 +--- .../mcap_record_replay/python/replay_hand.py | 53 +-- 10 files changed, 770 insertions(+), 371 deletions(-) create mode 100644 examples/mcap_record_replay/python/live_controller.py create mode 100644 examples/mcap_record_replay/python/live_full_body.py create mode 100644 examples/mcap_record_replay/python/live_hand.py diff --git a/examples/mcap_record_replay/python/common.py b/examples/mcap_record_replay/python/common.py index bfb8a4c82..f0d042362 100644 --- a/examples/mcap_record_replay/python/common.py +++ b/examples/mcap_record_replay/python/common.py @@ -1,14 +1,19 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Shared pipeline for the record / replay scripts. +"""Shared pipeline and visualization helpers for the record / replay / live scripts. -Includes hand, controller, and full-body pipeline builders plus the rendering -helpers used by the replay scripts (``HandJoints``, ``HAND_BONES``, -``BODY_BONES``). +Pipeline builders: ``build_hand_pipeline``, ``build_controller_pipeline``, +``build_full_body_pipeline``. + +Viz classes (used by replay_* and live_* scripts): ``HandViz``, +``ControllerViz``, ``FullBodyViz``. + +Rendering helpers: ``HandJoints``, ``HAND_BONES``, ``BODY_BONES``. """ import numpy as np +import viser from isaacteleop.retargeting_engine.deviceio_source_nodes import ( ControllersSource, @@ -33,7 +38,10 @@ HandInput, HandInputIndex, ) -from isaacteleop.retargeting_engine.tensor_types.indices import BodyJointPicoIndex +from isaacteleop.retargeting_engine.tensor_types.indices import ( + BodyJointPicoIndex, + ControllerInputIndex, +) from isaacteleop.retargeting_engine.tensor_types.ndarray_types import ( DLDataType, NDArrayType, @@ -46,6 +54,15 @@ HANDS_CHANNEL = "hands" BODY_JOINT_NAMES = [joint.name for joint in BodyJointPicoIndex] +# --------------------------------------------------------------------------- +# Color palette shared across all viz scripts +# --------------------------------------------------------------------------- + +LEFT_COLOR: tuple[float, float, float] = (0.25, 0.85, 0.35) +RIGHT_COLOR: tuple[float, float, float] = (0.35, 0.55, 0.95) +INVALID_COLOR: tuple[float, float, float] = (1.0, 0.0, 0.0) +TRACKED_COLOR: tuple[float, float, float] = (0.25, 0.85, 0.35) + def _positions_group(name: str) -> TensorGroupType: return TensorGroupType( @@ -202,3 +219,278 @@ def build_full_body_pipeline(): (23, 24), (24, 25), ) + + +# --------------------------------------------------------------------------- +# Shared viser visualization classes +# --------------------------------------------------------------------------- + + +def _bone_segments(positions: np.ndarray) -> np.ndarray: + """Return (N, 2, 3) segment array for the parent→child hand bones.""" + return np.stack( + [np.stack([positions[a], positions[b]], axis=0) for a, b in HAND_BONES], + axis=0, + ).astype(np.float32) + + +def _valid_bone_segments(positions: np.ndarray, valid: np.ndarray) -> np.ndarray: + """Return (N, 2, 3) segment array for body bones whose both endpoints are valid.""" + segments: list[np.ndarray] = [] + for a, b in BODY_BONES: + if valid[a] and valid[b]: + segments.append(np.stack([positions[a], positions[b]], axis=0)) + if not segments: + return np.zeros((0, 2, 3), dtype=np.float32) + return np.stack(segments, axis=0).astype(np.float32) + + +def _segment(start: np.ndarray, end: np.ndarray) -> np.ndarray: + return np.stack([start, end], axis=0).astype(np.float32) + + +def controller_state(controller) -> dict: + """Extract a plain-dict snapshot from a controller TensorGroup.""" + if controller.is_none: + return { + "aim_pos": None, + "grip_pos": None, + "aim_valid": False, + "grip_valid": False, + "trigger": 0.0, + "squeeze": 0.0, + "thumbstick_xy": (0.0, 0.0), + "primary_click": False, + "secondary_click": False, + "thumbstick_click": False, + "menu_click": False, + "tracked": False, + } + + aim_valid = bool(controller[ControllerInputIndex.AIM_IS_VALID]) + grip_valid = bool(controller[ControllerInputIndex.GRIP_IS_VALID]) + return { + "aim_pos": np.asarray( + controller[ControllerInputIndex.AIM_POSITION], dtype=np.float32 + ), + "grip_pos": np.asarray( + controller[ControllerInputIndex.GRIP_POSITION], dtype=np.float32 + ), + "aim_valid": aim_valid, + "grip_valid": grip_valid, + "trigger": float(controller[ControllerInputIndex.TRIGGER_VALUE]), + "squeeze": float(controller[ControllerInputIndex.SQUEEZE_VALUE]), + "thumbstick_xy": ( + float(controller[ControllerInputIndex.THUMBSTICK_X]), + float(controller[ControllerInputIndex.THUMBSTICK_Y]), + ), + "primary_click": float(controller[ControllerInputIndex.PRIMARY_CLICK]) > 0.5, + "secondary_click": float(controller[ControllerInputIndex.SECONDARY_CLICK]) + > 0.5, + "thumbstick_click": float(controller[ControllerInputIndex.THUMBSTICK_CLICK]) + > 0.5, + "menu_click": float(controller[ControllerInputIndex.MENU_CLICK]) > 0.5, + "tracked": aim_valid or grip_valid, + } + + +class HandViz: + """Per-hand viser handles (joint cloud + skeleton segments).""" + + def __init__( + self, + server: viser.ViserServer, + name: str, + color: tuple[float, float, float], + ): + self.color = np.array(color, dtype=np.float32) + zero_pts = np.zeros((26, 3), dtype=np.float32) + zero_segs = np.zeros((len(HAND_BONES), 2, 3), dtype=np.float32) + + self.points = server.scene.add_point_cloud( + name=f"/{name}/joints", + points=zero_pts, + colors=np.tile(self.color, (26, 1)), + point_size=0.008, + ) + self.bones = server.scene.add_line_segments( + name=f"/{name}/bones", + points=zero_segs, + colors=np.tile(self.color, (len(HAND_BONES), 2, 1)), + line_width=2.0, + ) + + def update(self, positions: np.ndarray, valid: bool) -> None: + if valid: + self.points.points = positions.astype(np.float32) + self.points.colors = np.tile(self.color, (positions.shape[0], 1)) + self.bones.points = _bone_segments(positions) + else: + zero_pts = np.zeros_like(positions, dtype=np.float32) + self.points.points = zero_pts + self.points.colors = np.tile(INVALID_COLOR, (positions.shape[0], 1)) + self.bones.points = np.zeros((len(HAND_BONES), 2, 3), dtype=np.float32) + + +class ControllerViz: + """Per-controller viser handles (3D pose + live input-state HUD).""" + + def __init__( + self, + server: viser.ViserServer, + name: str, + color: tuple[float, float, float], + ): + self.color = np.array(color, dtype=np.float32) + zero_pt = np.zeros((1, 3), dtype=np.float32) + zero_seg = np.zeros((0, 2, 3), dtype=np.float32) + zero_seg_colors = np.zeros((0, 2, 3), dtype=np.float32) + + self.aim = server.scene.add_point_cloud( + name=f"/{name}/aim", + points=zero_pt, + colors=np.tile(self.color, (1, 1)), + point_size=0.015, + ) + self.grip = server.scene.add_point_cloud( + name=f"/{name}/grip", + points=zero_pt, + colors=np.tile(self.color, (1, 1)), + point_size=0.015, + ) + self.ray = server.scene.add_line_segments( + name=f"/{name}/ray", + points=zero_seg, + colors=zero_seg_colors, + line_width=2.0, + ) + + with server.gui.add_folder(name): + self.hud_tracking = server.gui.add_checkbox("tracked", False, disabled=True) + self.hud_aim_valid = server.gui.add_checkbox( + "aim_valid", False, disabled=True + ) + self.hud_grip_valid = server.gui.add_checkbox( + "grip_valid", False, disabled=True + ) + self.hud_stick = server.gui.add_vector2( + "thumbstick_xy", + initial_value=(0.0, 0.0), + min=(-1.0, -1.0), + max=(1.0, 1.0), + disabled=True, + ) + self.hud_trigger_value = server.gui.add_number( + "trigger", + initial_value=0.0, + min=0.0, + max=1.0, + step=0.01, + disabled=True, + ) + self.hud_trigger = server.gui.add_progress_bar(0.0) + self.hud_squeeze_value = server.gui.add_number( + "squeeze", + initial_value=0.0, + min=0.0, + max=1.0, + step=0.01, + disabled=True, + ) + self.hud_squeeze = server.gui.add_progress_bar(0.0) + self.hud_primary = server.gui.add_checkbox( + "primary_click", False, disabled=True + ) + self.hud_secondary = server.gui.add_checkbox( + "secondary_click", False, disabled=True + ) + self.hud_stick_click = server.gui.add_checkbox( + "thumbstick_click", False, disabled=True + ) + self.hud_menu_click = server.gui.add_checkbox( + "menu_click", False, disabled=True + ) + + def update(self, state: dict) -> None: + aim_valid: bool = state["aim_valid"] + grip_valid: bool = state["grip_valid"] + aim_pos: np.ndarray | None = state["aim_pos"] + grip_pos: np.ndarray | None = state["grip_pos"] + + self.hud_tracking.value = state["tracked"] + self.hud_aim_valid.value = aim_valid + self.hud_grip_valid.value = grip_valid + self.hud_stick.value = state["thumbstick_xy"] + self.hud_trigger.value = max(0.0, min(1.0, state["trigger"])) + self.hud_trigger_value.value = state["trigger"] + self.hud_squeeze.value = max(0.0, min(1.0, state["squeeze"])) + self.hud_squeeze_value.value = state["squeeze"] + self.hud_primary.value = state["primary_click"] + self.hud_secondary.value = state["secondary_click"] + self.hud_stick_click.value = state["thumbstick_click"] + self.hud_menu_click.value = state["menu_click"] + + if aim_valid and aim_pos is not None: + self.aim.points = aim_pos.reshape(1, 3).astype(np.float32) + self.aim.colors = np.tile(self.color, (1, 1)) + else: + self.aim.points = np.zeros((1, 3), dtype=np.float32) + self.aim.colors = np.tile(INVALID_COLOR, (1, 1)) + + if grip_valid and grip_pos is not None: + self.grip.points = grip_pos.reshape(1, 3).astype(np.float32) + self.grip.colors = np.tile(self.color, (1, 1)) + else: + self.grip.points = np.zeros((1, 3), dtype=np.float32) + self.grip.colors = np.tile(INVALID_COLOR, (1, 1)) + + if aim_valid and grip_valid and aim_pos is not None and grip_pos is not None: + seg = _segment(grip_pos, aim_pos).reshape(1, 2, 3) + self.ray.points = seg + self.ray.colors = np.tile(self.color, (1, 2, 1)) + else: + self.ray.points = np.zeros((0, 2, 3), dtype=np.float32) + self.ray.colors = np.zeros((0, 2, 3), dtype=np.float32) + + +class FullBodyViz: + """Viser handles for full-body skeleton (joint cloud + skeleton segments).""" + + def __init__(self, server: viser.ViserServer): + self.color = np.array(TRACKED_COLOR, dtype=np.float32) + zero_pts = np.zeros((len(BODY_JOINT_NAMES), 3), dtype=np.float32) + zero_segs = np.zeros((0, 2, 3), dtype=np.float32) + + self.points = server.scene.add_point_cloud( + name="/full_body/joints", + points=zero_pts, + colors=np.tile(self.color, (len(BODY_JOINT_NAMES), 1)), + point_size=0.01, + ) + self.bones = server.scene.add_line_segments( + name="/full_body/bones", + points=zero_segs, + colors=np.zeros((0, 2, 3), dtype=np.float32), + line_width=2.0, + ) + + def update(self, positions: np.ndarray | None, valid: np.ndarray | None) -> None: + if positions is None or valid is None: + zero_pts = np.zeros((len(BODY_JOINT_NAMES), 3), dtype=np.float32) + self.points.points = zero_pts + self.points.colors = np.tile(INVALID_COLOR, (len(BODY_JOINT_NAMES), 1)) + self.bones.points = np.zeros((0, 2, 3), dtype=np.float32) + self.bones.colors = np.zeros((0, 2, 3), dtype=np.float32) + return + + positions = positions.astype(np.float32) + valid_bool = valid.astype(bool) + self.points.points = positions + + point_colors = np.tile(self.color, (positions.shape[0], 1)) + point_colors[~valid_bool] = INVALID_COLOR + self.points.colors = point_colors + + segs = _valid_bone_segments(positions, valid_bool) + self.bones.points = segs + self.bones.colors = np.tile(self.color, (segs.shape[0], 2, 1)) diff --git a/examples/mcap_record_replay/python/live_controller.py b/examples/mcap_record_replay/python/live_controller.py new file mode 100644 index 000000000..92774fd53 --- /dev/null +++ b/examples/mcap_record_replay/python/live_controller.py @@ -0,0 +1,105 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +""" +Visualize live OpenXR controller poses in real time with viser. + +``CloudXRLauncher`` starts the CloudXR runtime and WSS proxy automatically. +Open the URL viser prints (default http://localhost:8080) in a browser to see +aim / grip points for both controllers, a ray between them, and a live HUD +showing thumbstick, trigger, squeeze, and button state. + +Usage: + python live_controller.py [--port 8080] [--host 127.0.0.1] [--accept-eula] + +Press Ctrl+C to stop. + +See: https://nvidia.github.io/IsaacTeleop/main/references/mcap_record_replay.html +""" + +import argparse +import sys +import time + +import viser + +from isaacteleop.cloudxr import CloudXRLauncher +from isaacteleop.teleop_session_manager import TeleopSession, TeleopSessionConfig + +from common import ( + ControllerViz, + LEFT_COLOR, + RIGHT_COLOR, + build_controller_pipeline, + controller_state, +) + + +def main(argv: list[str]) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--host", + default="127.0.0.1", + help="Viser HTTP bind address (default: 127.0.0.1; pass 0.0.0.0 to expose externally)", + ) + parser.add_argument("--port", type=int, default=8080, help="Viser HTTP port") + parser.add_argument( + "--accept-eula", + action="store_true", + help="Accept the NVIDIA CloudXR EULA non-interactively", + ) + parser.add_argument( + "--install-dir", + default="~/.cloudxr", + help="CloudXR install directory (default: ~/.cloudxr)", + ) + args = parser.parse_args(argv[1:]) + + server = viser.ViserServer(host=args.host, port=args.port) + server.scene.set_up_direction("+y") + server.scene.add_grid(name="/grid", width=2.0, height=2.0, cell_size=0.1) + + config = TeleopSessionConfig( + app_name="LiveControllerExample", + pipeline=build_controller_pipeline(), + ) + + with CloudXRLauncher( + install_dir=args.install_dir, + accept_eula=args.accept_eula, + ) as launcher: + print(f"[live] CloudXR runtime started (WSS log: {launcher.wss_log_path})") + print("[live] waiting for headset connection… (Ctrl+C to stop)") + + with TeleopSession(config) as session: + viz_left = ControllerViz(server, "controller_left", LEFT_COLOR) + viz_right = ControllerViz(server, "controller_right", RIGHT_COLOR) + print(f"[live] viser running at http://localhost:{args.port}") + _last_step_t = time.time() + _missed = 0 + try: + while True: + result = session.step() + + l_state = controller_state(result["controller_left"]) + r_state = controller_state(result["controller_right"]) + + viz_left.update(l_state) + viz_right.update(r_state) + + if session.frame_count % 60 == 0: + print( + f"[live] frame={session.frame_count} " + f"L={'Y' if l_state['aim_valid'] else '-'} " + f"R={'Y' if r_state['aim_valid'] else '-'}" + ) + time.sleep(1 / 60) + except KeyboardInterrupt: + pass + + print("[live] stopped") + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/examples/mcap_record_replay/python/live_full_body.py b/examples/mcap_record_replay/python/live_full_body.py new file mode 100644 index 000000000..0596b6079 --- /dev/null +++ b/examples/mcap_record_replay/python/live_full_body.py @@ -0,0 +1,106 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +""" +Visualize live OpenXR full-body pose tracking in real time with viser. + +``CloudXRLauncher`` starts the CloudXR runtime and WSS proxy automatically. +Open the URL viser prints (default http://localhost:8080) in a browser to see +the full PICO body skeleton — joints colored green when valid, red when lost — +updating live as you move. + +Usage: + python live_full_body.py [--port 8080] [--host 127.0.0.1] [--accept-eula] + +Press Ctrl+C to stop. + +See: https://nvidia.github.io/IsaacTeleop/main/references/mcap_record_replay.html +""" + +import argparse +import sys +import time + +import numpy as np +import viser + +from isaacteleop.cloudxr import CloudXRLauncher +from isaacteleop.retargeting_engine.tensor_types.indices import FullBodyInputIndex +from isaacteleop.teleop_session_manager import TeleopSession, TeleopSessionConfig + +from common import BODY_JOINT_NAMES, FullBodyViz, build_full_body_pipeline + + +def main(argv: list[str]) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--host", + default="127.0.0.1", + help="Viser HTTP bind address (default: 127.0.0.1; pass 0.0.0.0 to expose externally)", + ) + parser.add_argument("--port", type=int, default=8080, help="Viser HTTP port") + parser.add_argument( + "--accept-eula", + action="store_true", + help="Accept the NVIDIA CloudXR EULA non-interactively", + ) + parser.add_argument( + "--install-dir", + default="~/.cloudxr", + help="CloudXR install directory (default: ~/.cloudxr)", + ) + args = parser.parse_args(argv[1:]) + + server = viser.ViserServer(host=args.host, port=args.port) + server.scene.set_up_direction("+y") + server.scene.add_grid(name="/grid", width=2.0, height=2.0, cell_size=0.1) + + config = TeleopSessionConfig( + app_name="LiveFullBodyExample", + pipeline=build_full_body_pipeline(), + ) + + with CloudXRLauncher( + install_dir=args.install_dir, + accept_eula=args.accept_eula, + ) as launcher: + print(f"[live] CloudXR runtime started (WSS log: {launcher.wss_log_path})") + print("[live] waiting for headset connection… (Ctrl+C to stop)") + + with TeleopSession(config) as session: + viz = FullBodyViz(server) + print(f"[live] viser running at http://localhost:{args.port}") + try: + while True: + result = session.step() + full_body = result["full_body"] + + if full_body.is_none: + viz.update(None, None) + n_valid = 0 + else: + positions = np.asarray( + full_body[FullBodyInputIndex.JOINT_POSITIONS], + dtype=np.float32, + ) + valid = np.asarray( + full_body[FullBodyInputIndex.JOINT_VALID], dtype=np.uint8 + ) + viz.update(positions, valid) + n_valid = int(np.count_nonzero(valid)) + + if session.frame_count % 60 == 0: + print( + f"[live] frame={session.frame_count} " + f"joints={n_valid:02d}/{len(BODY_JOINT_NAMES)}" + ) + time.sleep(1 / 60) + except KeyboardInterrupt: + pass + + print("[live] stopped") + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/examples/mcap_record_replay/python/live_hand.py b/examples/mcap_record_replay/python/live_hand.py new file mode 100644 index 000000000..05d948e23 --- /dev/null +++ b/examples/mcap_record_replay/python/live_hand.py @@ -0,0 +1,102 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +""" +Visualize live OpenXR hand-tracking in real time with viser. + +``CloudXRLauncher`` starts the CloudXR runtime and WSS proxy automatically. +Open the URL viser prints (default http://localhost:8080) in a browser to see +both hands rendered as joint clouds + bone segments, updating live as you move. + +Usage: + python live_hand.py [--port 8080] [--host 127.0.0.1] [--accept-eula] + +Press Ctrl+C to stop. + +See: https://nvidia.github.io/IsaacTeleop/main/references/mcap_record_replay.html +""" + +import argparse +import sys +import time + +import numpy as np +import viser + +from isaacteleop.cloudxr import CloudXRLauncher +from isaacteleop.teleop_session_manager import TeleopSession, TeleopSessionConfig + +from common import HandViz, LEFT_COLOR, RIGHT_COLOR, build_hand_pipeline + + +def main(argv: list[str]) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--host", + default="127.0.0.1", + help="Viser HTTP bind address (default: 127.0.0.1; pass 0.0.0.0 to expose externally)", + ) + parser.add_argument("--port", type=int, default=8080, help="Viser HTTP port") + parser.add_argument( + "--accept-eula", + action="store_true", + help="Accept the NVIDIA CloudXR EULA non-interactively", + ) + parser.add_argument( + "--install-dir", + default="~/.cloudxr", + help="CloudXR install directory (default: ~/.cloudxr)", + ) + args = parser.parse_args(argv[1:]) + + server = viser.ViserServer(host=args.host, port=args.port) + server.scene.set_up_direction("+y") + server.scene.add_grid(name="/grid", width=2.0, height=2.0, cell_size=0.1) + + config = TeleopSessionConfig( + app_name="LiveHandExample", + pipeline=build_hand_pipeline(), + ) + + with CloudXRLauncher( + install_dir=args.install_dir, + accept_eula=args.accept_eula, + ) as launcher: + print(f"[live] CloudXR runtime started (WSS log: {launcher.wss_log_path})") + print("[live] waiting for headset connection… (Ctrl+C to stop)") + + with TeleopSession(config) as session: + viz_left = HandViz(server, "hand_left", LEFT_COLOR) + viz_right = HandViz(server, "hand_right", RIGHT_COLOR) + print(f"[live] viser running at http://localhost:{args.port}") + _last_step_t = time.time() + _missed = 0 + try: + while True: + result = session.step() + viz_left.update( + np.asarray(result["left_positions"][0]), + bool(result["left_valid"][0]), + ) + viz_right.update( + np.asarray(result["right_positions"][0]), + bool(result["right_valid"][0]), + ) + + if session.frame_count % 60 == 0: + left = bool(result["left_valid"][0]) + right = bool(result["right_valid"][0]) + print( + f"[live] frame={session.frame_count} " + f"L={'Y' if left else '-'} R={'Y' if right else '-'}" + ) + time.sleep(1 / 60) + except KeyboardInterrupt: + pass + + print("[live] stopped") + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/examples/mcap_record_replay/python/record_controller.py b/examples/mcap_record_replay/python/record_controller.py index 18140a858..4b980905e 100644 --- a/examples/mcap_record_replay/python/record_controller.py +++ b/examples/mcap_record_replay/python/record_controller.py @@ -4,23 +4,26 @@ """ Record a live OpenXR controller-tracking session to an MCAP file. -Requires an active OpenXR runtime / headset. The pipeline in ``common.py`` -wires only ``ControllersSource``, so ``TeleopSession`` records exactly the -``controllers`` channel — no head, no hands. +``CloudXRLauncher`` starts the CloudXR runtime and WSS proxy automatically — +no separate terminal or pre-running headset daemon is needed. The pipeline in +``common.py`` wires only ``ControllersSource``, so ``TeleopSession`` records +exactly the ``controllers`` channel — no head, no hands. Usage: - python record_controller.py [duration_seconds] [output.mcap] + python record_controller.py [duration_seconds] [output.mcap] [--accept-eula] Defaults: 5 seconds → ../recordings/controllers_.mcap See: https://nvidia.github.io/IsaacTeleop/main/references/mcap_record_replay.html """ +import argparse import sys import time from datetime import datetime from pathlib import Path +from isaacteleop.cloudxr import CloudXRLauncher from isaacteleop.deviceio import McapRecordingConfig from isaacteleop.retargeting_engine.tensor_types.indices import ControllerInputIndex from isaacteleop.teleop_session_manager import TeleopSession, TeleopSessionConfig @@ -29,10 +32,27 @@ def main(argv: list[str]) -> int: - duration_s = float(argv[1]) if len(argv) > 1 else 5.0 + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "duration", nargs="?", type=float, default=5.0, help="Recording duration (s)" + ) + parser.add_argument("output", nargs="?", help="Output .mcap path") + parser.add_argument( + "--accept-eula", + action="store_true", + help="Accept the NVIDIA CloudXR EULA non-interactively", + ) + parser.add_argument( + "--install-dir", + default="~/.cloudxr", + help="CloudXR install directory (default: ~/.cloudxr)", + ) + args = parser.parse_args(argv[1:]) + + duration_s: float = args.duration - if len(argv) > 2: - mcap_path = Path(argv[2]) + if args.output: + mcap_path = Path(args.output) mcap_path.parent.mkdir(parents=True, exist_ok=True) else: out_dir = Path(__file__).resolve().parent.parent / "recordings" @@ -47,25 +67,30 @@ def main(argv: list[str]) -> int: mcap_config=McapRecordingConfig(str(mcap_path)), ) - with TeleopSession(config) as session: - start = time.time() - while time.time() - start < duration_s: - result = session.step() - if session.frame_count % 60 == 0: - left_ctrl = result["controller_left"] - right_ctrl = result["controller_right"] - left = not left_ctrl.is_none and bool( - left_ctrl[ControllerInputIndex.AIM_IS_VALID] - ) - right = not right_ctrl.is_none and bool( - right_ctrl[ControllerInputIndex.AIM_IS_VALID] - ) - print( - f"[record] t={time.time() - start:5.2f}s " - f"frame={session.frame_count} L={'Y' if left else '-'} " - f"R={'Y' if right else '-'}" - ) - time.sleep(1 / 60) + with CloudXRLauncher( + install_dir=args.install_dir, + accept_eula=args.accept_eula, + ) as launcher: + print(f"[record] CloudXR runtime started (WSS log: {launcher.wss_log_path})") + with TeleopSession(config) as session: + start = time.time() + while time.time() - start < duration_s: + result = session.step() + if session.frame_count % 60 == 0: + left_ctrl = result["controller_left"] + right_ctrl = result["controller_right"] + left = not left_ctrl.is_none and bool( + left_ctrl[ControllerInputIndex.AIM_IS_VALID] + ) + right = not right_ctrl.is_none and bool( + right_ctrl[ControllerInputIndex.AIM_IS_VALID] + ) + print( + f"[record] t={time.time() - start:5.2f}s " + f"frame={session.frame_count} L={'Y' if left else '-'} " + f"R={'Y' if right else '-'}" + ) + time.sleep(1 / 60) print(f"[record] done — {mcap_path}") return 0 diff --git a/examples/mcap_record_replay/python/record_full_body.py b/examples/mcap_record_replay/python/record_full_body.py index c3d0f3b46..9d6342e90 100644 --- a/examples/mcap_record_replay/python/record_full_body.py +++ b/examples/mcap_record_replay/python/record_full_body.py @@ -4,18 +4,20 @@ """ Record a live OpenXR full-body tracking session to an MCAP file. -Requires an active OpenXR runtime / headset. The pipeline in ``common.py`` -wires ``FullBodySource`` and ``ControllersSource``, so ``TeleopSession`` -records the ``full_body`` and ``controllers`` channels. +``CloudXRLauncher`` starts the CloudXR runtime and WSS proxy automatically — +no separate terminal or pre-running headset daemon is needed. The pipeline in +``common.py`` wires ``FullBodySource`` and ``ControllersSource``, so +``TeleopSession`` records the ``full_body`` and ``controllers`` channels. Usage: - python record_full_body.py [duration_seconds] [output.mcap] + python record_full_body.py [duration_seconds] [output.mcap] [--accept-eula] Defaults: 5 seconds → ../recordings/full_body_.mcap See: https://nvidia.github.io/IsaacTeleop/main/references/mcap_record_replay.html """ +import argparse import sys import time from datetime import datetime @@ -23,6 +25,7 @@ import numpy as np +from isaacteleop.cloudxr import CloudXRLauncher from isaacteleop.deviceio import McapRecordingConfig from isaacteleop.retargeting_engine.tensor_types.indices import FullBodyInputIndex from isaacteleop.teleop_session_manager import TeleopSession, TeleopSessionConfig @@ -31,10 +34,27 @@ def main(argv: list[str]) -> int: - duration_s = float(argv[1]) if len(argv) > 1 else 5.0 + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "duration", nargs="?", type=float, default=5.0, help="Recording duration (s)" + ) + parser.add_argument("output", nargs="?", help="Output .mcap path") + parser.add_argument( + "--accept-eula", + action="store_true", + help="Accept the NVIDIA CloudXR EULA non-interactively", + ) + parser.add_argument( + "--install-dir", + default="~/.cloudxr", + help="CloudXR install directory (default: ~/.cloudxr)", + ) + args = parser.parse_args(argv[1:]) - if len(argv) > 2: - mcap_path = Path(argv[2]) + duration_s: float = args.duration + + if args.output: + mcap_path = Path(args.output) mcap_path.parent.mkdir(parents=True, exist_ok=True) else: out_dir = Path(__file__).resolve().parent.parent / "recordings" @@ -49,30 +69,35 @@ def main(argv: list[str]) -> int: mcap_config=McapRecordingConfig(str(mcap_path)), ) - with TeleopSession(config) as session: - start = time.time() - while time.time() - start < duration_s: - result = session.step() - if session.frame_count % 60 == 0: - full_body = result["full_body"] - n_valid = ( - 0 - if full_body.is_none - else int( - np.count_nonzero( - np.asarray( - full_body[FullBodyInputIndex.JOINT_VALID], - dtype=np.uint8, + with CloudXRLauncher( + install_dir=args.install_dir, + accept_eula=args.accept_eula, + ) as launcher: + print(f"[record] CloudXR runtime started (WSS log: {launcher.wss_log_path})") + with TeleopSession(config) as session: + start = time.time() + while time.time() - start < duration_s: + result = session.step() + if session.frame_count % 60 == 0: + full_body = result["full_body"] + n_valid = ( + 0 + if full_body.is_none + else int( + np.count_nonzero( + np.asarray( + full_body[FullBodyInputIndex.JOINT_VALID], + dtype=np.uint8, + ) ) ) ) - ) - print( - f"[record] t={time.time() - start:5.2f}s " - f"frame={session.frame_count} " - f"joints={n_valid:02d}/{len(BODY_JOINT_NAMES)}" - ) - time.sleep(1 / 60) + print( + f"[record] t={time.time() - start:5.2f}s " + f"frame={session.frame_count} " + f"joints={n_valid:02d}/{len(BODY_JOINT_NAMES)}" + ) + time.sleep(1 / 60) print(f"[record] done — {mcap_path}") return 0 diff --git a/examples/mcap_record_replay/python/record_hand.py b/examples/mcap_record_replay/python/record_hand.py index 27b461703..2082883fd 100644 --- a/examples/mcap_record_replay/python/record_hand.py +++ b/examples/mcap_record_replay/python/record_hand.py @@ -4,23 +4,26 @@ """ Record a live OpenXR hand-tracking session to an MCAP file. -Requires an active OpenXR runtime / headset. The pipeline in ``common.py`` -wires only ``HandsSource``, so ``TeleopSession`` records exactly the ``hands`` -channel — no head, no controllers. +``CloudXRLauncher`` starts the CloudXR runtime and WSS proxy automatically — +no separate terminal or pre-running headset daemon is needed. The pipeline in +``common.py`` wires only ``HandsSource``, so ``TeleopSession`` records exactly +the ``hands`` channel — no head, no controllers. Usage: - python record_hand.py [duration_seconds] [output.mcap] + python record_hand.py [duration_seconds] [output.mcap] [--accept-eula] Defaults: 5 seconds → ../recordings/hands_.mcap See: https://nvidia.github.io/IsaacTeleop/main/references/mcap_record_replay.html """ +import argparse import sys import time from datetime import datetime from pathlib import Path +from isaacteleop.cloudxr import CloudXRLauncher from isaacteleop.deviceio import McapRecordingConfig from isaacteleop.teleop_session_manager import TeleopSession, TeleopSessionConfig @@ -28,10 +31,27 @@ def main(argv: list[str]) -> int: - duration_s = float(argv[1]) if len(argv) > 1 else 5.0 + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "duration", nargs="?", type=float, default=5.0, help="Recording duration (s)" + ) + parser.add_argument("output", nargs="?", help="Output .mcap path") + parser.add_argument( + "--accept-eula", + action="store_true", + help="Accept the NVIDIA CloudXR EULA non-interactively", + ) + parser.add_argument( + "--install-dir", + default="~/.cloudxr", + help="CloudXR install directory (default: ~/.cloudxr)", + ) + args = parser.parse_args(argv[1:]) + + duration_s: float = args.duration - if len(argv) > 2: - mcap_path = Path(argv[2]) + if args.output: + mcap_path = Path(args.output) mcap_path.parent.mkdir(parents=True, exist_ok=True) else: out_dir = Path(__file__).resolve().parent.parent / "recordings" @@ -46,19 +66,24 @@ def main(argv: list[str]) -> int: mcap_config=McapRecordingConfig(str(mcap_path)), ) - with TeleopSession(config) as session: - start = time.time() - while time.time() - start < duration_s: - result = session.step() - if session.frame_count % 60 == 0: - left = bool(result["left_valid"][0]) - right = bool(result["right_valid"][0]) - print( - f"[record] t={time.time() - start:5.2f}s " - f"frame={session.frame_count} L={'Y' if left else '-'} " - f"R={'Y' if right else '-'}" - ) - time.sleep(1 / 60) + with CloudXRLauncher( + install_dir=args.install_dir, + accept_eula=args.accept_eula, + ) as launcher: + print(f"[record] CloudXR runtime started (WSS log: {launcher.wss_log_path})") + with TeleopSession(config) as session: + start = time.time() + while time.time() - start < duration_s: + result = session.step() + if session.frame_count % 60 == 0: + left = bool(result["left_valid"][0]) + right = bool(result["right_valid"][0]) + print( + f"[record] t={time.time() - start:5.2f}s " + f"frame={session.frame_count} L={'Y' if left else '-'} " + f"R={'Y' if right else '-'}" + ) + time.sleep(1 / 60) print(f"[record] done — {mcap_path}") return 0 diff --git a/examples/mcap_record_replay/python/replay_controller.py b/examples/mcap_record_replay/python/replay_controller.py index 8956042d1..9d8247aeb 100644 --- a/examples/mcap_record_replay/python/replay_controller.py +++ b/examples/mcap_record_replay/python/replay_controller.py @@ -23,19 +23,23 @@ import time from pathlib import Path -import numpy as np import viser from mcap.reader import make_reader from isaacteleop.deviceio import McapReplayConfig -from isaacteleop.retargeting_engine.tensor_types.indices import ControllerInputIndex from isaacteleop.teleop_session_manager import ( SessionMode, TeleopSession, TeleopSessionConfig, ) -from common import build_controller_pipeline +from common import ( + ControllerViz, + LEFT_COLOR, + RIGHT_COLOR, + build_controller_pipeline, + controller_state, +) def mcap_duration_s(path: Path) -> float: @@ -56,11 +60,6 @@ def mcap_duration_s(path: Path) -> float: return (stats.message_end_time - stats.message_start_time) / 1e9 -LEFT_COLOR = (0.25, 0.85, 0.35) -RIGHT_COLOR = (0.35, 0.55, 0.95) -INVALID_COLOR = (1.0, 0.0, 0.0) - - def resolve_mcap(path_arg: str | None) -> Path: if path_arg: path = Path(path_arg) @@ -78,175 +77,6 @@ def resolve_mcap(path_arg: str | None) -> Path: return max(candidates, key=lambda p: p.stat().st_mtime) -def _segment(start: np.ndarray, end: np.ndarray) -> np.ndarray: - return np.stack([start, end], axis=0).astype(np.float32) - - -class ControllerViz: - """Per-controller viser handles (3D pose + live input-state HUD).""" - - def __init__( - self, - server: viser.ViserServer, - name: str, - color: tuple[float, float, float], - ): - self.color = np.array(color, dtype=np.float32) - zero_pt = np.zeros((1, 3), dtype=np.float32) - zero_seg = np.zeros((0, 2, 3), dtype=np.float32) - zero_seg_colors = np.zeros((0, 2, 3), dtype=np.float32) - - self.aim = server.scene.add_point_cloud( - name=f"/{name}/aim", - points=zero_pt, - colors=np.tile(self.color, (1, 1)), - point_size=0.015, - ) - self.grip = server.scene.add_point_cloud( - name=f"/{name}/grip", - points=zero_pt, - colors=np.tile(self.color, (1, 1)), - point_size=0.015, - ) - self.ray = server.scene.add_line_segments( - name=f"/{name}/ray", - points=zero_seg, - colors=zero_seg_colors, - line_width=2.0, - ) - - with server.gui.add_folder(name): - self.hud_tracking = server.gui.add_checkbox("tracked", False, disabled=True) - self.hud_aim_valid = server.gui.add_checkbox( - "aim_valid", False, disabled=True - ) - self.hud_grip_valid = server.gui.add_checkbox( - "grip_valid", False, disabled=True - ) - self.hud_stick = server.gui.add_vector2( - "thumbstick_xy", - initial_value=(0.0, 0.0), - min=(-1.0, -1.0), - max=(1.0, 1.0), - disabled=True, - ) - self.hud_trigger_value = server.gui.add_number( - "trigger", - initial_value=0.0, - min=0.0, - max=1.0, - step=0.01, - disabled=True, - ) - self.hud_trigger = server.gui.add_progress_bar(0.0) - self.hud_squeeze_value = server.gui.add_number( - "squeeze", - initial_value=0.0, - min=0.0, - max=1.0, - step=0.01, - disabled=True, - ) - self.hud_squeeze = server.gui.add_progress_bar(0.0) - self.hud_primary = server.gui.add_checkbox( - "primary_click", False, disabled=True - ) - self.hud_secondary = server.gui.add_checkbox( - "secondary_click", False, disabled=True - ) - self.hud_stick_click = server.gui.add_checkbox( - "thumbstick_click", False, disabled=True - ) - self.hud_menu_click = server.gui.add_checkbox( - "menu_click", False, disabled=True - ) - - def update(self, state: dict) -> None: - aim_valid: bool = state["aim_valid"] - grip_valid: bool = state["grip_valid"] - aim_pos: np.ndarray | None = state["aim_pos"] - grip_pos: np.ndarray | None = state["grip_pos"] - - self.hud_tracking.value = state["tracked"] - self.hud_aim_valid.value = aim_valid - self.hud_grip_valid.value = grip_valid - self.hud_stick.value = state["thumbstick_xy"] - self.hud_trigger.value = max(0.0, min(1.0, state["trigger"])) - self.hud_trigger_value.value = state["trigger"] - self.hud_squeeze.value = max(0.0, min(1.0, state["squeeze"])) - self.hud_squeeze_value.value = state["squeeze"] - self.hud_primary.value = state["primary_click"] - self.hud_secondary.value = state["secondary_click"] - self.hud_stick_click.value = state["thumbstick_click"] - self.hud_menu_click.value = state["menu_click"] - - if aim_valid and aim_pos is not None: - self.aim.points = aim_pos.reshape(1, 3).astype(np.float32) - self.aim.colors = np.tile(self.color, (1, 1)) - else: - self.aim.points = np.zeros((1, 3), dtype=np.float32) - self.aim.colors = np.tile(INVALID_COLOR, (1, 1)) - - if grip_valid and grip_pos is not None: - self.grip.points = grip_pos.reshape(1, 3).astype(np.float32) - self.grip.colors = np.tile(self.color, (1, 1)) - else: - self.grip.points = np.zeros((1, 3), dtype=np.float32) - self.grip.colors = np.tile(INVALID_COLOR, (1, 1)) - - if aim_valid and grip_valid and aim_pos is not None and grip_pos is not None: - seg = _segment(grip_pos, aim_pos).reshape(1, 2, 3) - self.ray.points = seg - self.ray.colors = np.tile(self.color, (1, 2, 1)) - else: - self.ray.points = np.zeros((0, 2, 3), dtype=np.float32) - self.ray.colors = np.zeros((0, 2, 3), dtype=np.float32) - - -def _controller_state(controller): - if controller.is_none: - return { - "aim_pos": None, - "grip_pos": None, - "aim_valid": False, - "grip_valid": False, - "trigger": 0.0, - "squeeze": 0.0, - "thumbstick_xy": (0.0, 0.0), - "primary_click": False, - "secondary_click": False, - "thumbstick_click": False, - "menu_click": False, - "tracked": False, - } - - aim_valid = bool(controller[ControllerInputIndex.AIM_IS_VALID]) - grip_valid = bool(controller[ControllerInputIndex.GRIP_IS_VALID]) - return { - "aim_pos": np.asarray( - controller[ControllerInputIndex.AIM_POSITION], dtype=np.float32 - ), - "grip_pos": np.asarray( - controller[ControllerInputIndex.GRIP_POSITION], dtype=np.float32 - ), - "aim_valid": aim_valid, - "grip_valid": grip_valid, - "trigger": float(controller[ControllerInputIndex.TRIGGER_VALUE]), - "squeeze": float(controller[ControllerInputIndex.SQUEEZE_VALUE]), - "thumbstick_xy": ( - float(controller[ControllerInputIndex.THUMBSTICK_X]), - float(controller[ControllerInputIndex.THUMBSTICK_Y]), - ), - "primary_click": float(controller[ControllerInputIndex.PRIMARY_CLICK]) > 0.5, - "secondary_click": float(controller[ControllerInputIndex.SECONDARY_CLICK]) - > 0.5, - "thumbstick_click": float(controller[ControllerInputIndex.THUMBSTICK_CLICK]) - > 0.5, - "menu_click": float(controller[ControllerInputIndex.MENU_CLICK]) > 0.5, - "tracked": aim_valid or grip_valid, - } - - def run_once( mcap_path: Path, duration_s: float, @@ -266,11 +96,9 @@ def run_once( start = time.time() while time.time() - start < duration_s: result = session.step() - left = result["controller_left"] - right = result["controller_right"] - l_state = _controller_state(left) - r_state = _controller_state(right) + l_state = controller_state(result["controller_left"]) + r_state = controller_state(result["controller_right"]) viz_left.update(l_state) viz_right.update(r_state) diff --git a/examples/mcap_record_replay/python/replay_full_body.py b/examples/mcap_record_replay/python/replay_full_body.py index 67f356b97..a312d1100 100644 --- a/examples/mcap_record_replay/python/replay_full_body.py +++ b/examples/mcap_record_replay/python/replay_full_body.py @@ -34,7 +34,7 @@ TeleopSessionConfig, ) -from common import BODY_BONES, BODY_JOINT_NAMES, build_full_body_pipeline +from common import BODY_JOINT_NAMES, FullBodyViz, build_full_body_pipeline def mcap_duration_s(path: Path) -> float: @@ -55,10 +55,6 @@ def mcap_duration_s(path: Path) -> float: return (stats.message_end_time - stats.message_start_time) / 1e9 -TRACKED_COLOR = (0.25, 0.85, 0.35) -INVALID_COLOR = (1.0, 0.0, 0.0) - - def resolve_mcap(path_arg: str | None) -> Path: if path_arg: path = Path(path_arg) @@ -76,60 +72,6 @@ def resolve_mcap(path_arg: str | None) -> Path: return max(candidates, key=lambda p: p.stat().st_mtime) -def _valid_bone_segments(positions: np.ndarray, valid: np.ndarray) -> np.ndarray: - """Return (N, 2, 3) segment array for bones whose endpoints are both valid.""" - segments: list[np.ndarray] = [] - for a, b in BODY_BONES: - if valid[a] and valid[b]: - segments.append(np.stack([positions[a], positions[b]], axis=0)) - if not segments: - return np.zeros((0, 2, 3), dtype=np.float32) - return np.stack(segments, axis=0).astype(np.float32) - - -class FullBodyViz: - """Viser handles (joint cloud + skeleton segments).""" - - def __init__(self, server: viser.ViserServer): - self.color = np.array(TRACKED_COLOR, dtype=np.float32) - zero_pts = np.zeros((len(BODY_JOINT_NAMES), 3), dtype=np.float32) - zero_segs = np.zeros((0, 2, 3), dtype=np.float32) - - self.points = server.scene.add_point_cloud( - name="/full_body/joints", - points=zero_pts, - colors=np.tile(self.color, (len(BODY_JOINT_NAMES), 1)), - point_size=0.01, - ) - self.bones = server.scene.add_line_segments( - name="/full_body/bones", - points=zero_segs, - colors=np.zeros((0, 2, 3), dtype=np.float32), - line_width=2.0, - ) - - def update(self, positions: np.ndarray | None, valid: np.ndarray | None) -> None: - if positions is None or valid is None: - zero_pts = np.zeros((len(BODY_JOINT_NAMES), 3), dtype=np.float32) - self.points.points = zero_pts - self.points.colors = np.tile(INVALID_COLOR, (len(BODY_JOINT_NAMES), 1)) - self.bones.points = np.zeros((0, 2, 3), dtype=np.float32) - self.bones.colors = np.zeros((0, 2, 3), dtype=np.float32) - return - - positions = positions.astype(np.float32) - valid_bool = valid.astype(bool) - self.points.points = positions - - point_colors = np.tile(self.color, (positions.shape[0], 1)) - point_colors[~valid_bool] = INVALID_COLOR - self.points.colors = point_colors - - segs = _valid_bone_segments(positions, valid_bool) - self.bones.points = segs - self.bones.colors = np.tile(self.color, (segs.shape[0], 2, 1)) - - def run_once( mcap_path: Path, duration_s: float, diff --git a/examples/mcap_record_replay/python/replay_hand.py b/examples/mcap_record_replay/python/replay_hand.py index 7a4838f80..cce09f991 100644 --- a/examples/mcap_record_replay/python/replay_hand.py +++ b/examples/mcap_record_replay/python/replay_hand.py @@ -33,7 +33,7 @@ TeleopSessionConfig, ) -from common import HAND_BONES, build_hand_pipeline +from common import HandViz, LEFT_COLOR, RIGHT_COLOR, build_hand_pipeline def mcap_duration_s(path: Path) -> float: @@ -54,11 +54,6 @@ def mcap_duration_s(path: Path) -> float: return (stats.message_end_time - stats.message_start_time) / 1e9 -LEFT_COLOR = (0.25, 0.85, 0.35) -RIGHT_COLOR = (0.35, 0.55, 0.95) -INVALID_COLOR = (1.0, 0.0, 0.0) - - def resolve_mcap(path_arg: str | None) -> Path: if path_arg: path = Path(path_arg) @@ -76,52 +71,6 @@ def resolve_mcap(path_arg: str | None) -> Path: return max(candidates, key=lambda p: p.stat().st_mtime) -def _bone_segments(positions: np.ndarray) -> np.ndarray: - """Return (N, 2, 3) segment array for the parent→child bones.""" - return np.stack( - [np.stack([positions[a], positions[b]], axis=0) for a, b in HAND_BONES], - axis=0, - ).astype(np.float32) - - -class HandViz: - """Per-hand viser handles (joint cloud + skeleton segments).""" - - def __init__( - self, - server: viser.ViserServer, - name: str, - color: tuple[float, float, float], - ): - self.color = np.array(color, dtype=np.float32) - zero_pts = np.zeros((26, 3), dtype=np.float32) - zero_segs = np.zeros((len(HAND_BONES), 2, 3), dtype=np.float32) - - self.points = server.scene.add_point_cloud( - name=f"/{name}/joints", - points=zero_pts, - colors=np.tile(self.color, (26, 1)), - point_size=0.008, - ) - self.bones = server.scene.add_line_segments( - name=f"/{name}/bones", - points=zero_segs, - colors=np.tile(self.color, (len(HAND_BONES), 2, 1)), - line_width=2.0, - ) - - def update(self, positions: np.ndarray, valid: bool) -> None: - if valid: - self.points.points = positions.astype(np.float32) - self.points.colors = np.tile(self.color, (positions.shape[0], 1)) - self.bones.points = _bone_segments(positions) - else: - zero_pts = np.zeros_like(positions, dtype=np.float32) - self.points.points = zero_pts - self.points.colors = np.tile(INVALID_COLOR, (positions.shape[0], 1)) - self.bones.points = np.zeros((len(HAND_BONES), 2, 3), dtype=np.float32) - - def run_once( mcap_path: Path, duration_s: float, From 34586f4cacde2cc7968e429cc0bf635980d57833 Mon Sep 17 00:00:00 2001 From: Jiwen Cai Date: Tue, 23 Jun 2026 17:58:19 +0000 Subject: [PATCH 2/2] mcap examples: add --env-file/--launch-cloudxr-runtime and fix missed-frame counting - Add --env-file (default: default.env) to pass a KEY=value override file to CloudXRLauncher; ship default.env with sensible defaults for Quest 3 - Add --launch-cloudxr-runtime / --no-launch-cloudxr-runtime (BooleanOptionalAction, default true) to optionally skip CloudXRLauncher and connect to a system OpenXR runtime directly - Report missed frames per interval in live_hand/controller - Fix missed-frame counting to accumulate per-frame gaps correctly Signed-off-by: Jiwen Cai --- .../mcap_record_replay/python/default.env | 27 +++++++++++++ .../python/live_controller.py | 39 ++++++++++++++++--- .../python/live_full_body.py | 31 ++++++++++++--- .../mcap_record_replay/python/live_hand.py | 39 ++++++++++++++++--- .../python/record_controller.py | 32 ++++++++++++--- .../python/record_full_body.py | 32 ++++++++++++--- .../mcap_record_replay/python/record_hand.py | 32 ++++++++++++--- 7 files changed, 200 insertions(+), 32 deletions(-) create mode 100644 examples/mcap_record_replay/python/default.env diff --git a/examples/mcap_record_replay/python/default.env b/examples/mcap_record_replay/python/default.env new file mode 100644 index 000000000..1ffdd8240 --- /dev/null +++ b/examples/mcap_record_replay/python/default.env @@ -0,0 +1,27 @@ +# CloudXR env-var overrides for the mcap_record_replay example scripts. +# +# Passed to CloudXRLauncher via the --env-file argument. Format: one +# KEY=value per line; # comments and blank lines are ignored; values are +# expanded for $VARS and ~ (via os.path.expandvars / os.path.expanduser). +# +# Precedence (highest first): +# 1. This file +# 2. Process environment variables (e.g. Docker / shell exports) +# 3. Built-in defaults in isaacteleop/cloudxr/env_config.py +# +# Reserved keys — always computed by the runtime, silently ignored if set here: +# XR_RUNTIME_JSON, XRT_NO_STDIN, NV_CXR_RUNTIME_DIR, NV_CXR_OUTPUT_DIR + +# Device profile advertised by the CloudXR runtime. +# "auto-webrtc" (built-in default) serves WebXR / browser-based clients over +# WebRTC. Override to "Quest3" when connecting a Meta Quest 3 headset directly. +NV_DEVICE_PROFILE=Quest3 + +# Input device discovery channels. Both are true by default; pinned here for +# visibility. Leave enabled for XR controller and hand-tracking data. +NV_CXR_ENABLE_PUSH_DEVICES=true +NV_CXR_ENABLE_TENSOR_DATA=true + +# Write CloudXR runtime logs to ~/.cloudxr/logs. Useful when diagnosing +# connection issues (e.g. "Failed to get OpenXR system: -35"). true by default. +NV_CXR_FILE_LOGGING=true diff --git a/examples/mcap_record_replay/python/live_controller.py b/examples/mcap_record_replay/python/live_controller.py index 92774fd53..c4a905f34 100644 --- a/examples/mcap_record_replay/python/live_controller.py +++ b/examples/mcap_record_replay/python/live_controller.py @@ -18,8 +18,10 @@ """ import argparse +import contextlib import sys import time +from pathlib import Path import viser @@ -53,6 +55,18 @@ def main(argv: list[str]) -> int: default="~/.cloudxr", help="CloudXR install directory (default: ~/.cloudxr)", ) + parser.add_argument( + "--env-file", + default=str(Path(__file__).parent / "default.env"), + help="Path to a KEY=value env file for CloudXR overrides (default: default.env)", + ) + parser.add_argument( + "--launch-cloudxr-runtime", + action=argparse.BooleanOptionalAction, + default=True, + help="Launch the CloudXR runtime automatically (default: true; pass " + "--no-launch-cloudxr-runtime to connect to the system runtime instead)", + ) args = parser.parse_args(argv[1:]) server = viser.ViserServer(host=args.host, port=args.port) @@ -64,11 +78,18 @@ def main(argv: list[str]) -> int: pipeline=build_controller_pipeline(), ) - with CloudXRLauncher( - install_dir=args.install_dir, - accept_eula=args.accept_eula, - ) as launcher: - print(f"[live] CloudXR runtime started (WSS log: {launcher.wss_log_path})") + launcher_ctx = ( + contextlib.nullcontext() + if not args.launch_cloudxr_runtime + else CloudXRLauncher( + install_dir=args.install_dir, + env_config=args.env_file, + accept_eula=args.accept_eula, + ) + ) + with launcher_ctx as launcher: + if launcher is not None: + print(f"[live] CloudXR runtime started (WSS log: {launcher.wss_log_path})") print("[live] waiting for headset connection… (Ctrl+C to stop)") with TeleopSession(config) as session: @@ -79,6 +100,10 @@ def main(argv: list[str]) -> int: _missed = 0 try: while True: + now = time.time() + _missed += max(0, round((now - _last_step_t) * 60) - 1) + _last_step_t = now + result = session.step() l_state = controller_state(result["controller_left"]) @@ -91,8 +116,10 @@ def main(argv: list[str]) -> int: print( f"[live] frame={session.frame_count} " f"L={'Y' if l_state['aim_valid'] else '-'} " - f"R={'Y' if r_state['aim_valid'] else '-'}" + f"R={'Y' if r_state['aim_valid'] else '-'} " + f"missed={_missed}" ) + _missed = 0 time.sleep(1 / 60) except KeyboardInterrupt: pass diff --git a/examples/mcap_record_replay/python/live_full_body.py b/examples/mcap_record_replay/python/live_full_body.py index 0596b6079..858a1f53c 100644 --- a/examples/mcap_record_replay/python/live_full_body.py +++ b/examples/mcap_record_replay/python/live_full_body.py @@ -18,8 +18,10 @@ """ import argparse +import contextlib import sys import time +from pathlib import Path import numpy as np import viser @@ -49,6 +51,18 @@ def main(argv: list[str]) -> int: default="~/.cloudxr", help="CloudXR install directory (default: ~/.cloudxr)", ) + parser.add_argument( + "--env-file", + default=str(Path(__file__).parent / "default.env"), + help="Path to a KEY=value env file for CloudXR overrides (default: default.env)", + ) + parser.add_argument( + "--launch-cloudxr-runtime", + action=argparse.BooleanOptionalAction, + default=True, + help="Launch the CloudXR runtime automatically (default: true; pass " + "--no-launch-cloudxr-runtime to connect to the system runtime instead)", + ) args = parser.parse_args(argv[1:]) server = viser.ViserServer(host=args.host, port=args.port) @@ -60,11 +74,18 @@ def main(argv: list[str]) -> int: pipeline=build_full_body_pipeline(), ) - with CloudXRLauncher( - install_dir=args.install_dir, - accept_eula=args.accept_eula, - ) as launcher: - print(f"[live] CloudXR runtime started (WSS log: {launcher.wss_log_path})") + launcher_ctx = ( + contextlib.nullcontext() + if not args.launch_cloudxr_runtime + else CloudXRLauncher( + install_dir=args.install_dir, + env_config=args.env_file, + accept_eula=args.accept_eula, + ) + ) + with launcher_ctx as launcher: + if launcher is not None: + print(f"[live] CloudXR runtime started (WSS log: {launcher.wss_log_path})") print("[live] waiting for headset connection… (Ctrl+C to stop)") with TeleopSession(config) as session: diff --git a/examples/mcap_record_replay/python/live_hand.py b/examples/mcap_record_replay/python/live_hand.py index 05d948e23..3c085d2be 100644 --- a/examples/mcap_record_replay/python/live_hand.py +++ b/examples/mcap_record_replay/python/live_hand.py @@ -17,8 +17,10 @@ """ import argparse +import contextlib import sys import time +from pathlib import Path import numpy as np import viser @@ -47,6 +49,18 @@ def main(argv: list[str]) -> int: default="~/.cloudxr", help="CloudXR install directory (default: ~/.cloudxr)", ) + parser.add_argument( + "--env-file", + default=str(Path(__file__).parent / "default.env"), + help="Path to a KEY=value env file for CloudXR overrides (default: default.env)", + ) + parser.add_argument( + "--launch-cloudxr-runtime", + action=argparse.BooleanOptionalAction, + default=True, + help="Launch the CloudXR runtime automatically (default: true; pass " + "--no-launch-cloudxr-runtime to connect to the system runtime instead)", + ) args = parser.parse_args(argv[1:]) server = viser.ViserServer(host=args.host, port=args.port) @@ -58,11 +72,18 @@ def main(argv: list[str]) -> int: pipeline=build_hand_pipeline(), ) - with CloudXRLauncher( - install_dir=args.install_dir, - accept_eula=args.accept_eula, - ) as launcher: - print(f"[live] CloudXR runtime started (WSS log: {launcher.wss_log_path})") + launcher_ctx = ( + contextlib.nullcontext() + if not args.launch_cloudxr_runtime + else CloudXRLauncher( + install_dir=args.install_dir, + env_config=args.env_file, + accept_eula=args.accept_eula, + ) + ) + with launcher_ctx as launcher: + if launcher is not None: + print(f"[live] CloudXR runtime started (WSS log: {launcher.wss_log_path})") print("[live] waiting for headset connection… (Ctrl+C to stop)") with TeleopSession(config) as session: @@ -73,6 +94,10 @@ def main(argv: list[str]) -> int: _missed = 0 try: while True: + now = time.time() + _missed += max(0, round((now - _last_step_t) * 60) - 1) + _last_step_t = now + result = session.step() viz_left.update( np.asarray(result["left_positions"][0]), @@ -88,8 +113,10 @@ def main(argv: list[str]) -> int: right = bool(result["right_valid"][0]) print( f"[live] frame={session.frame_count} " - f"L={'Y' if left else '-'} R={'Y' if right else '-'}" + f"L={'Y' if left else '-'} R={'Y' if right else '-'} " + f"missed={_missed}" ) + _missed = 0 time.sleep(1 / 60) except KeyboardInterrupt: pass diff --git a/examples/mcap_record_replay/python/record_controller.py b/examples/mcap_record_replay/python/record_controller.py index 4b980905e..fe61f611a 100644 --- a/examples/mcap_record_replay/python/record_controller.py +++ b/examples/mcap_record_replay/python/record_controller.py @@ -18,6 +18,7 @@ """ import argparse +import contextlib import sys import time from datetime import datetime @@ -47,6 +48,18 @@ def main(argv: list[str]) -> int: default="~/.cloudxr", help="CloudXR install directory (default: ~/.cloudxr)", ) + parser.add_argument( + "--env-file", + default=str(Path(__file__).parent / "default.env"), + help="Path to a KEY=value env file for CloudXR overrides (default: default.env)", + ) + parser.add_argument( + "--launch-cloudxr-runtime", + action=argparse.BooleanOptionalAction, + default=True, + help="Launch the CloudXR runtime automatically (default: true; pass " + "--no-launch-cloudxr-runtime to connect to the system runtime instead)", + ) args = parser.parse_args(argv[1:]) duration_s: float = args.duration @@ -67,11 +80,20 @@ def main(argv: list[str]) -> int: mcap_config=McapRecordingConfig(str(mcap_path)), ) - with CloudXRLauncher( - install_dir=args.install_dir, - accept_eula=args.accept_eula, - ) as launcher: - print(f"[record] CloudXR runtime started (WSS log: {launcher.wss_log_path})") + launcher_ctx = ( + contextlib.nullcontext() + if not args.launch_cloudxr_runtime + else CloudXRLauncher( + install_dir=args.install_dir, + env_config=args.env_file, + accept_eula=args.accept_eula, + ) + ) + with launcher_ctx as launcher: + if launcher is not None: + print( + f"[record] CloudXR runtime started (WSS log: {launcher.wss_log_path})" + ) with TeleopSession(config) as session: start = time.time() while time.time() - start < duration_s: diff --git a/examples/mcap_record_replay/python/record_full_body.py b/examples/mcap_record_replay/python/record_full_body.py index 9d6342e90..0ee65cf9c 100644 --- a/examples/mcap_record_replay/python/record_full_body.py +++ b/examples/mcap_record_replay/python/record_full_body.py @@ -18,6 +18,7 @@ """ import argparse +import contextlib import sys import time from datetime import datetime @@ -49,6 +50,18 @@ def main(argv: list[str]) -> int: default="~/.cloudxr", help="CloudXR install directory (default: ~/.cloudxr)", ) + parser.add_argument( + "--env-file", + default=str(Path(__file__).parent / "default.env"), + help="Path to a KEY=value env file for CloudXR overrides (default: default.env)", + ) + parser.add_argument( + "--launch-cloudxr-runtime", + action=argparse.BooleanOptionalAction, + default=True, + help="Launch the CloudXR runtime automatically (default: true; pass " + "--no-launch-cloudxr-runtime to connect to the system runtime instead)", + ) args = parser.parse_args(argv[1:]) duration_s: float = args.duration @@ -69,11 +82,20 @@ def main(argv: list[str]) -> int: mcap_config=McapRecordingConfig(str(mcap_path)), ) - with CloudXRLauncher( - install_dir=args.install_dir, - accept_eula=args.accept_eula, - ) as launcher: - print(f"[record] CloudXR runtime started (WSS log: {launcher.wss_log_path})") + launcher_ctx = ( + contextlib.nullcontext() + if not args.launch_cloudxr_runtime + else CloudXRLauncher( + install_dir=args.install_dir, + env_config=args.env_file, + accept_eula=args.accept_eula, + ) + ) + with launcher_ctx as launcher: + if launcher is not None: + print( + f"[record] CloudXR runtime started (WSS log: {launcher.wss_log_path})" + ) with TeleopSession(config) as session: start = time.time() while time.time() - start < duration_s: diff --git a/examples/mcap_record_replay/python/record_hand.py b/examples/mcap_record_replay/python/record_hand.py index 2082883fd..42ca556cb 100644 --- a/examples/mcap_record_replay/python/record_hand.py +++ b/examples/mcap_record_replay/python/record_hand.py @@ -18,6 +18,7 @@ """ import argparse +import contextlib import sys import time from datetime import datetime @@ -46,6 +47,18 @@ def main(argv: list[str]) -> int: default="~/.cloudxr", help="CloudXR install directory (default: ~/.cloudxr)", ) + parser.add_argument( + "--env-file", + default=str(Path(__file__).parent / "default.env"), + help="Path to a KEY=value env file for CloudXR overrides (default: default.env)", + ) + parser.add_argument( + "--launch-cloudxr-runtime", + action=argparse.BooleanOptionalAction, + default=True, + help="Launch the CloudXR runtime automatically (default: true; pass " + "--no-launch-cloudxr-runtime to connect to the system runtime instead)", + ) args = parser.parse_args(argv[1:]) duration_s: float = args.duration @@ -66,11 +79,20 @@ def main(argv: list[str]) -> int: mcap_config=McapRecordingConfig(str(mcap_path)), ) - with CloudXRLauncher( - install_dir=args.install_dir, - accept_eula=args.accept_eula, - ) as launcher: - print(f"[record] CloudXR runtime started (WSS log: {launcher.wss_log_path})") + launcher_ctx = ( + contextlib.nullcontext() + if not args.launch_cloudxr_runtime + else CloudXRLauncher( + install_dir=args.install_dir, + env_config=args.env_file, + accept_eula=args.accept_eula, + ) + ) + with launcher_ctx as launcher: + if launcher is not None: + print( + f"[record] CloudXR runtime started (WSS log: {launcher.wss_log_path})" + ) with TeleopSession(config) as session: start = time.time() while time.time() - start < duration_s: