Enable vehicle teleoperation

CMakeLists.txt

@@ -149,6 +149,7 @@ if(BUILD_EXAMPLES)
     add_subdirectory(examples/schemaio)
     add_subdirectory(examples/native_openxr)
     add_subdirectory(examples/mcap_record_replay)
+    add_subdirectory(examples/vehicle_teleop)
     add_subdirectory(examples/haptic_feedback)
     if(BUILD_VIZ)
         add_subdirectory(examples/camera_viz/tests)
@@ -164,6 +165,7 @@ if(BUILD_PLUGINS)
 
     add_subdirectory(src/plugins/controller_synthetic_hands)
     add_subdirectory(src/plugins/generic_3axis_pedal)
+    add_subdirectory(src/plugins/steering_wheel)
     add_subdirectory(src/plugins/manus)
     add_subdirectory(src/plugins/haptikos)
     if(BUILD_PLUGIN_OAK_CAMERA)

examples/vehicle_teleop/.gitignore

@@ -0,0 +1,3 @@
+logs/
+thirdparty/
+python/.venv/

examples/vehicle_teleop/CMakeLists.txt

@@ -0,0 +1,23 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+cmake_minimum_required(VERSION 3.20)
+
+include(${CMAKE_SOURCE_DIR}/cmake/InstallPythonExample.cmake)
+install_python_example(DESTINATION examples/vehicle_teleop/python)
+
+install(FILES README.md
+    DESTINATION examples/vehicle_teleop
+)
+
+install(DIRECTORY config
+    DESTINATION examples/vehicle_teleop
+)
+
+install(PROGRAMS
+    scripts/replay_command_mcap.sh
+    scripts/run_isaac_keyboard_control_worker.sh
+    scripts/run_isaac_remote_steering_worker.sh
+    scripts/run_kia_panda_worker.sh
+    DESTINATION examples/vehicle_teleop/scripts
+)

examples/vehicle_teleop/README.md

@@ -0,0 +1,116 @@
+<!--
+SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+SPDX-License-Identifier: Apache-2.0
+-->
+
+# Vehicle Teleop Example
+
+This example sends steering wheel and pedal input through Isaac Teleop, retargets it to a vehicle command, and publishes that command over ZMQ for a Kia Panda worker.
+
+The remote side uses the native Linux steering wheel plugin and Isaac Teleop OpenXR session. The vehicle side subscribes to the command stream and writes to `PandaRunner`.
+
+## Setup
+
+Build and install Isaac Teleop with examples enabled. From the Isaac Teleop repository root:
+
+```bash
+cmake -B build -DBUILD_EXAMPLES=ON
+cmake --build build --parallel 4
+cmake --install build
+```
+
+Create the example virtual environment from this directory. The scripts use
+`.venv/bin/python` directly, so keep the virtual environment at
+`examples/vehicle_teleop/.venv`.
+
+```bash
+cd examples/vehicle_teleop
+uv venv --python 3.11 .venv
+uv pip install --python .venv/bin/python --find-links ../../build/wheels "isaacteleop[cloudxr]"
+source .venv/bin/activate
+cd python
+uv sync --active --inexact --no-install-project --no-install-package isaacteleop
+cd ..
+```
+
+Clone the vehicle-side dependencies into this example's `thirdparty` directory. These repositories are intentionally not added as git submodules.
+
+```bash
+mkdir -p thirdparty
+git clone https://github.com/commaai/panda.git thirdparty/panda
+git clone https://github.com/commaai/opendbc.git thirdparty/opendbc
+```
+
+You can use existing local checkouts instead, as long as the paths match `thirdparty/panda` and `thirdparty/opendbc`.
+In our usage, the `opendbc` and `panda` repos are not a "one-size-fits-all" solutions for some cars. We recommend debugging as needed if the Panda device cannot connect to the car. In the future, we will provide a repository that works for our usage (Kia Niro EV 2022) as a useful reference.
+
+## Remote Side
+
+Start the CloudXR runtime:
+
+```bash
+python3 -m isaacteleop.cloudxr
+```
+
+In a separate terminal, activate the CloudXR environment printed by that command, then start the Isaac Teleop steering worker:
+
+```bash
+source ~/.cloudxr/run/cloudxr.env
+./scripts/run_isaac_remote_steering_worker.sh --verbose
+```
+
+By default, the worker binds to `tcp://*:5555` on topic `kia_control`.
+
+Useful options:
+
+```bash
+./scripts/run_isaac_remote_steering_worker.sh --device /dev/input/js0
+./scripts/run_isaac_remote_steering_worker.sh --bind "tcp://*:5555"
+./scripts/run_isaac_remote_steering_worker.sh --log-mcap logs/kia_control.mcap
+./scripts/run_isaac_remote_steering_worker.sh --plugin-binary /path/to/steering_wheel_plugin
+```
+
+Steering wheel axis mapping is configured in `config/steering_wheel_config.yaml`. The default mapping is for a Logitech G920/G923-style setup where steering, throttle, brake, and clutch are raw Linux joystick axes.
+
+For IsaacTeleop keyboard fallback without a steering wheel, run:
+
+```bash
+./scripts/run_isaac_keyboard_control_worker.sh --verbose
+```
+
+Keyboard controls are `W`/`S` for gas-brake, `A`/`D` for steering, `R` or `C` for neutral, and `Q` or `Esc` to quit.
+
+## Vehicle Side
+
+Run the Panda worker on the vehicle machine:
+
+```bash
+./scripts/run_kia_panda_worker.sh --connect "tcp://<remote-ip>:5555"
+```
+
+For local testing without opening PandaRunner, use dry-run mode:
+
+```bash
+./scripts/run_kia_panda_worker.sh --connect "tcp://<remote-ip>:5555" --dry-run
+```
+
+Do not run the live vehicle-side worker unless the vehicle-side hardware and safety process are ready.
+
+## Replay Logs
+
+If the remote worker was started with `--log-mcap`, replay the recorded commands with:
+
+```bash
+./scripts/replay_command_mcap.sh logs/kia_control.mcap
+./scripts/replay_command_mcap.sh logs/kia_control.mcap --realtime
+```
+
+The replay command prints the retargeted vehicle command values. It does not write to PandaRunner.
+
+## Troubleshooting
+
+If the steering worker cannot find the native plugin, pass `--plugin-binary` explicitly. In a source build, the default location is usually `build/src/plugins/steering_wheel/steering_wheel_plugin`. In an install tree, it is usually `plugins/steering_wheel/steering_wheel_plugin`.
+
+If the worker fails while creating the OpenXR session, confirm that the CloudXR runtime is running, the CloudXR environment has been sourced in the worker terminal, and the OpenXR client is connected.
+
+If the Panda worker cannot import `opendbc` or `panda`, confirm the two dependency repositories were cloned under `examples/vehicle_teleop/thirdparty/`.

examples/vehicle_teleop/config/steering_wheel_config.yaml

@@ -0,0 +1,15 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+g920_racing_wheel:
+  steering_wheel: 0
+  throttle: 2
+  brake: 3
+  clutch: 1
+  handbrake: 4
+  reverse: 10
+
+sensitivity:
+  mode: 0
+  min: 0.5
+  max: 0.5

examples/vehicle_teleop/python/pyproject.toml

@@ -0,0 +1,25 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+[project]
+name = "vehicle-teleop-example"
+version = "0.0.0"
+description = "Isaac Teleop vehicle steering and Kia Panda control example"
+requires-python = ">=3.10,<3.14"
+dependencies = [
+    "crcmod",
+    "isaacteleop[cloudxr]",
+    "libusb-package",
+    "libusb1",
+    "mcap>=1.2.0",
+    "pycapnp==2.1.0",
+    "pycryptodome",
+    "pyyaml",
+    "pyzmq",
+    "scons",
+    "tqdm",
+    "websockets>=14",
+]
+
+[[tool.uv.index]]
+url = "https://pypi.nvidia.com"

examples/vehicle_teleop/python/vehicle_teleop/__init__.py

@@ -0,0 +1,4 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""Vehicle teleoperation example using Isaac Teleop steering wheel input."""

examples/vehicle_teleop/python/vehicle_teleop/command_log.py

@@ -0,0 +1,161 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass
+from pathlib import Path
+from typing import BinaryIO, Iterable
+
+from mcap.reader import make_reader
+from mcap.writer import CompressionType, Writer
+
+from vehicle_teleop.vehicle_command import VehicleControlCommand
+
+
+@dataclass(frozen=True)
+class SteeringWheelSample:
+    steering: float
+    accel_axis: float
+    brake_axis: float
+    timestamp_ns: int
+
+    def to_dict(self) -> dict[str, float | int]:
+        return {
+            "steering": self.steering,
+            "accel_axis": self.accel_axis,
+            "brake_axis": self.brake_axis,
+            "timestamp_ns": self.timestamp_ns,
+        }
+
+
+VEHICLE_COMMAND_TOPIC = "vehicle_control"
+VEHICLE_COMMAND_SCHEMA = {
+    "type": "object",
+    "properties": {
+        "sample": {
+            "type": "object",
+            "properties": {
+                "steering": {"type": "number"},
+                "accel_axis": {"type": "number"},
+                "brake_axis": {"type": "number"},
+                "timestamp_ns": {"type": "integer"},
+            },
+            "required": ["steering", "accel_axis", "brake_axis", "timestamp_ns"],
+        },
+        "command": {
+            "type": "object",
+            "properties": {
+                "sequence": {"type": "integer"},
+                "timestamp_ns": {"type": "integer"},
+                "steer": {"type": "number"},
+                "accel": {"type": "number"},
+                "throttle": {"type": "number"},
+                "brake": {"type": "number"},
+            },
+            "required": [
+                "sequence",
+                "timestamp_ns",
+                "steer",
+                "accel",
+                "throttle",
+                "brake",
+            ],
+        },
+    },
+    "required": ["sample", "command"],
+}
+
+
+@dataclass(frozen=True)
+class CommandLogRecord:
+    sample: SteeringWheelSample
+    command: VehicleControlCommand
+
+    @classmethod
+    def from_dict(cls, value: dict) -> "CommandLogRecord":
+        sample = value["sample"]
+        return cls(
+            sample=SteeringWheelSample(
+                steering=float(sample["steering"]),
+                accel_axis=float(sample["accel_axis"]),
+                brake_axis=float(sample["brake_axis"]),
+                timestamp_ns=int(sample["timestamp_ns"]),
+            ),
+            command=VehicleControlCommand.from_dict(value["command"]),
+        )
+
+    def to_dict(self) -> dict:
+        return {
+            "sample": self.sample.to_dict(),
+            "command": self.command.to_dict(),
+        }
+
+    def to_json(self) -> bytes:
+        return json.dumps(self.to_dict(), separators=(",", ":")).encode("utf-8")
+
+
+class McapCommandLogger:
+    def __init__(self, path: str | Path) -> None:
+        self._path = Path(path)
+        self._file: BinaryIO | None = None
+        self._writer: Writer | None = None
+        self._channel_id: int | None = None
+
+    def __enter__(self) -> "McapCommandLogger":
+        self._path.parent.mkdir(parents=True, exist_ok=True)
+        self._file = self._path.open("wb")
+        self._writer = Writer(self._file, compression=CompressionType.NONE)
+        self._writer.start()
+        schema_id = self._writer.register_schema(
+            name="vehicle_teleop.VehicleControlCommandRecord",
+            encoding="jsonschema",
+            data=json.dumps(VEHICLE_COMMAND_SCHEMA, separators=(",", ":")).encode(
+                "utf-8"
+            ),
+        )
+        self._channel_id = self._writer.register_channel(
+            topic=VEHICLE_COMMAND_TOPIC,
+            message_encoding="json",
+            schema_id=schema_id,
+        )
+        return self
+
+    def __exit__(self, _exc_type, _exc_value, _traceback) -> None:
+        if self._writer is not None:
+            self._writer.finish()
+            self._writer = None
+            self._channel_id = None
+        if self._file is not None:
+            self._file.close()
+            self._file = None
+
+    def write(
+        self, *, sample: SteeringWheelSample, command: VehicleControlCommand
+    ) -> None:
+        if self._writer is None or self._channel_id is None:
+            raise RuntimeError("McapCommandLogger must be opened before writing")
+        record = CommandLogRecord(sample=sample, command=command)
+        self._writer.add_message(
+            channel_id=self._channel_id,
+            log_time=command.timestamp_ns,
+            publish_time=command.timestamp_ns,
+            sequence=command.sequence,
+            data=record.to_json(),
+        )
+
+
+class McapCommandLogReader:
+    def __init__(self, path: str | Path) -> None:
+        self._path = Path(path)
+
+    def records(self) -> Iterable[CommandLogRecord]:
+        with self._path.open("rb") as file:
+            reader = make_reader(file)
+            for _schema, _channel, message in reader.iter_messages(
+                topics=[VEHICLE_COMMAND_TOPIC]
+            ):
+                yield CommandLogRecord.from_dict(
+                    json.loads(message.data.decode("utf-8"))
+                )