diff --git a/.gitattributes b/.gitattributes index eb1f41580..b92c87a4a 100644 --- a/.gitattributes +++ b/.gitattributes @@ -11,3 +11,8 @@ # # Exceptions for small source SVGs (webpack-inlined; not binary assets). deps/cloudxr/webxr_client/src/icons/*.svg -filter -diff -merge text + +*.usd filter=lfs diff=lfs merge=lfs -text +*.usda filter=lfs diff=lfs merge=lfs -text +*.stl filter=lfs diff=lfs merge=lfs -text +*.STL filter=lfs diff=lfs merge=lfs -text diff --git a/.gitignore b/.gitignore index 7785ebe1e..3eac121e8 100644 --- a/.gitignore +++ b/.gitignore @@ -21,6 +21,7 @@ __pycache__/ *.pyd # Python virtual environments .venv/ +.venv-isaacteleop/ venv/ **/.venv/ **/venv/ @@ -64,3 +65,17 @@ deps/v2d/wheels/ # Runtime device files injected by MCP server infrastructure .mcp.json + +apv_wuji +avp_ee_hand.md +cmd +AGENTS.md +custom.env +web.env +example_demo +build-local/ +deps/cloudxr/webxr_client/helpers/react/CloudXRComponent.tsx +deps/cloudxr/webxr_client/src/App.tsx +deps/cloudxr/webxr_client/src/CloudXRUI.tsx +examples/teleop/CMakeLists.txt +examples/teleop/python/pyproject.toml diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 000000000..b9e0ea59b --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "wuji-retargeting"] + path = third_party/wuji-retargeting + url = https://github.com/wuji-technology/wuji-retargeting.git diff --git a/AGENTS.md b/AGENTS.md index fd71ba39a..4f2000142 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -89,6 +89,7 @@ only point here — edit the rules in the doc, not the shims. ``` - If a hook failure shows **missing or non-obvious repo policy** (not a one-off typo), you **must** add a **short** reminder under **Mandatory learning loop** rules to the right `AGENTS.md` or adjacent **`//` comments** so the next run does not repeat it—unless it is already documented. +- If `pre-commit` is unavailable in the active environment, report the exact blocker and any narrower checks that did run; do not describe the pre-commit gate as passed. ## Mandatory learning loop (AGENTS.md and comments) @@ -110,6 +111,8 @@ only point here — edit the rules in the doc, not the shims. - **High-level** expectations: boundaries between layers, what to avoid, naming or structural conventions, CMake/include policy at a glance, “always / never” rules that stay true across refactors. - **Style of work**: how minimal to keep diffs, when to ask for clarification, how this subsystem should relate to OpenXR/schema/deviceio, etc. +- For onboarding-oriented shell helpers, keep the flow as a short step-by-step runbook and make separate Python environment responsibilities explicit instead of mixing them into one opaque setup path. +- For shell-script comments meant for humans to follow, prefer short numbered Chinese step comments that mirror repo runbooks (for example the root `cmd` note) over scattered low-level inline commentary. - For build-only requests, use the documented CMake flags and environment first (for example `-DISAAC_TELEOP_PYTHON_VERSION=...`) rather than patching build files to select options. **What does *not* belong in `AGENTS.md`** diff --git a/REUSE.toml b/REUSE.toml index c8040799d..1d5711404 100644 --- a/REUSE.toml +++ b/REUSE.toml @@ -35,3 +35,13 @@ path = ".cursor/**" precedence = "override" SPDX-FileCopyrightText = "Copyright (c) NVIDIA CORPORATION & AFFILIATES. All rights reserved." SPDX-License-Identifier = "Apache-2.0" + +[[annotations]] +path = [ + "examples/g1_wuji_teleop/assets/g1_wuji/**", + "examples/g1_wuji_teleop/assets/wuji_hand/**", + "examples/teleop/python/assets/wuji_hand/**", +] +precedence = "override" +SPDX-FileCopyrightText = "Copyright (c) NVIDIA CORPORATION & AFFILIATES. All rights reserved." +SPDX-License-Identifier = "Apache-2.0" diff --git a/avp_manus.sh b/avp_manus.sh new file mode 100755 index 000000000..aa0eaa423 --- /dev/null +++ b/avp_manus.sh @@ -0,0 +1,173 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# IsaacTeleop AVP + MANUS + G1-Wuji demo runbook +# +# Notes: +# 1. This file is a step-by-step command guide. Do not execute it end to end +# with `bash avp_manus.sh`. +# 2. A working team Isaac Lab environment is assumed. Isaac Lab installation +# steps are intentionally omitted here. +# 3. This demo defaults to `examples/g1_wuji_teleop/config/avp_manus.yml`. +# It uses `retargeting.method=official_wuji`, so the +# `third_party/wuji-retargeting` submodule must be initialized first. +# 4. The commands below assume Python 3.12. The Python major/minor version used +# to build `isaacteleop` must match the Python version in your Isaac Lab +# environment. If your Isaac Lab is not using 3.12, update the `3.12` +# references below accordingly. +# +# ----------------------------------------------------------------------------- +# 0. Enter the repository root +# ----------------------------------------------------------------------------- + +cd /path/to/IsaacTeleop + +# ----------------------------------------------------------------------------- +# 1. Install Ubuntu system dependencies +# ----------------------------------------------------------------------------- + +sudo apt-get update +sudo apt-get install -y \ + build-essential \ + cmake \ + clang-format-14 \ + ccache \ + patchelf \ + curl \ + git \ + unzip \ + usbutils \ + android-tools-adb \ + coturn \ + libx11-dev \ + libssl-dev \ + zlib1g-dev \ + libc-ares-dev \ + libzmq3-dev \ + libncurses-dev \ + libudev-dev \ + libusb-1.0-0-dev \ + libvulkan-dev \ + glslang-tools \ + libxrandr-dev \ + libxinerama-dev \ + libxcursor-dev \ + libxi-dev \ + libxext-dev \ + libxkbcommon-dev \ + libwayland-dev \ + wayland-protocols + +# ----------------------------------------------------------------------------- +# 2. Create and activate the `isaacteleop` conda environment +# ----------------------------------------------------------------------------- + +conda create -n isaacteleop python=3.12 -y +conda activate isaacteleop +# From PyPI +pip install 'isaacteleop[cloudxr,retargeters]~=1.0.0' --extra-index-url https://pypi.nvidia.com +# python -m pip install --upgrade pip +# python -m pip install uv pre-commit + +# ----------------------------------------------------------------------------- +# 3. Build IsaacTeleop from source and install the wheel into the `isaacteleop` +# conda environment +# ----------------------------------------------------------------------------- + +cd /path/to/IsaacTeleop + +cmake -S . -B build \ + -DCMAKE_BUILD_TYPE=Release \ + -DISAAC_TELEOP_PYTHON_VERSION=3.12 \ + -DBUILD_VIZ=OFF \ + -DBUILD_PLUGIN_OAK_CAMERA=OFF \ + -DENABLE_CLANG_FORMAT_CHECK=OFF + +cmake --build build --parallel "$(nproc)" +cmake --install build + +python -m pip install -r /path/to/IsaacTeleop/src/core/python/requirements-retargeters.txt +python -m pip install -r /path/to/IsaacTeleop/src/core/python/requirements-cloudxr.txt +python -m pip install --force-reinstall /path/to/IsaacTeleop/install/wheels/isaacteleop-*.whl + +python -c "import isaacteleop, isaacteleop.cloudxr, isaacteleop.retargeters; print(isaacteleop.__file__)" + +# ----------------------------------------------------------------------------- +# 4. Pull the wuji official wuji-retargeting +# ----------------------------------------------------------------------------- +cd /path/to/IsaacTeleop +git submodule update --init --recursive third_party/wuji-retargeting + +# ----------------------------------------------------------------------------- +# 5. Install MANUS host permissions and the SDK / plugin +# ----------------------------------------------------------------------------- + +cd /path/to/IsaacTeleop/src/plugins/manus + +# 5.1 This step must run on the host, not inside a container +./install_udev_rules.sh + +# 5.2 Unplug and replug the MANUS dongle + +# 5.3 Download the MANUS SDK, then build and install the plugin +./install_manus.sh + +cd /path/to/IsaacTeleop +ls -l install/plugins/manus/ +ls -l install/bin/manus_hand_tracker_printer + +# ----------------------------------------------------------------------------- +# 6. Open the ports required for AVP to connect to CloudXR +# ----------------------------------------------------------------------------- + +sudo ufw allow 48010,48322/tcp +sudo ufw allow 47998:48000,48005,48008,48012/udp + +# ----------------------------------------------------------------------------- +# 7. Terminal A: start the CloudXR runtime +# ----------------------------------------------------------------------------- + +cd /path/to/IsaacTeleop +conda activate isaacteleop + +echo 'NV_DEVICE_PROFILE=auto-native' > custom.env +python -m isaacteleop.cloudxr --cloudxr-env-config=./custom.env --accept-eula + +# Keep terminal A running. + +# ----------------------------------------------------------------------------- +# 8. Check the local IP address and connect from the AVP device +# ----------------------------------------------------------------------------- + + +# Manual steps on the AVP side: +# 1. Open Isaac XR Teleop Sample Client (version 3.0.0 or newer). +# 2. Enter the Ubuntu LAN IP address found above. +# 3. Connect and keep the session alive. + +# ----------------------------------------------------------------------------- +# 9. Terminal B: load the CloudXR environment and verify MANUS data first +# ----------------------------------------------------------------------------- + +cd /path/to/IsaacTeleop +source ~/.cloudxr/run/cloudxr.env + +./install/bin/manus_hand_tracker_printer + +# Once hand data and the visualization window are updating correctly, close +# `manus_hand_tracker_printer`. +# Note: the MANUS SDK can only be owned by one process at a time, so the +# printer must be closed before starting the main demo. + +# ----------------------------------------------------------------------------- +# 10. Terminal C: run the final demo +# ----------------------------------------------------------------------------- + +cd /path/to/IsaacTeleop +source ~/.cloudxr/run/cloudxr.env +conda activate isaacteleop +python examples/g1_wuji_teleop/scripts/g1_wuji_teleop_main.py \ + --config /examples/g1_wuji_teleop/config/avp_manus.yml \ + --device cuda:0 \ + --viz kit diff --git a/deps/cloudxr/webxr_client/helpers/react/CloudXRComponent.tsx b/deps/cloudxr/webxr_client/helpers/react/CloudXRComponent.tsx index 6b9eea500..ed5878364 100644 --- a/deps/cloudxr/webxr_client/helpers/react/CloudXRComponent.tsx +++ b/deps/cloudxr/webxr_client/helpers/react/CloudXRComponent.tsx @@ -43,6 +43,258 @@ import { useXR } from '@react-three/xr'; import { useRef, useEffect } from 'react'; import type { WebGLRenderer } from 'three'; +const AVP_CONTROLLER_SHIM_QUERY_PARAM = 'avpControllerShim'; +const AVP_CONTROLLER_SHIM_INSTALLED = '__isaacTeleopAvpControllerShimInstalled'; +const AVP_CONTROLLER_SHIMMED_SOURCE = '__isaacTeleopAvpControllerShim'; +const AVP_CONTROLLER_SHIM_RAW_SOURCE = '__isaacTeleopAvpControllerShimRawSource'; +const AVP_CONTROLLER_SHIM_SPACE_SOURCE = '__isaacTeleopAvpControllerShimSpaceSource'; + +function shouldEnableAvpControllerShim(): boolean { + const value = new URLSearchParams(window.location.search).get(AVP_CONTROLLER_SHIM_QUERY_PARAM); + return value === '1' || value?.toLowerCase() === 'true'; +} + +function makeSyntheticGamepad(handedness: XRHandedness): Gamepad { + const button = (): GamepadButton => + ({ + pressed: false, + touched: false, + value: 0, + }) as GamepadButton; + + return { + id: `avp-webxr-${handedness || 'none'}-synthetic-controller`, + index: handedness === 'left' ? 0 : handedness === 'right' ? 1 : -1, + connected: true, + timestamp: performance.now(), + mapping: 'xr-standard', + buttons: Array.from({ length: 13 }, button), + axes: [0, 0, 0, 0], + hapticActuators: [], + } as unknown as Gamepad; +} + +function getFallbackHandSpace(inputSource: XRInputSource): XRSpace | undefined { + const hand = inputSource.hand; + if (!hand) { + return undefined; + } + + return ( + hand.get('wrist' as XRHandJoint) ?? + hand.get('palm' as XRHandJoint) ?? + hand.get('index-finger-metacarpal' as XRHandJoint) + ); +} + +function getFallbackControllerSpace(inputSource: XRInputSource): XRSpace | undefined { + return inputSource.gripSpace ?? inputSource.targetRaySpace ?? getFallbackHandSpace(inputSource); +} + +function shouldShimInputSource(inputSource: XRInputSource): boolean { + return ( + (inputSource.handedness === 'left' || inputSource.handedness === 'right') && + !inputSource.gamepad && + !!getFallbackControllerSpace(inputSource) + ); +} + +function findInputSourcesDescriptor( + xrSession: XRSession +): PropertyDescriptor | undefined { + let owner: object | null = xrSession; + while (owner) { + const descriptor = Object.getOwnPropertyDescriptor(owner, 'inputSources'); + if (descriptor) { + return descriptor; + } + owner = Object.getPrototypeOf(owner); + } + return undefined; +} + +function installAvpControllerInputShim(xrSession: XRSession): boolean { + const sessionWithMarker = xrSession as XRSession & Record; + if (sessionWithMarker[AVP_CONTROLLER_SHIM_INSTALLED]) { + return true; + } + + const descriptor = findInputSourcesDescriptor(xrSession); + const originalGetter = descriptor?.get?.bind(xrSession); + const originalValue = descriptor && 'value' in descriptor ? descriptor.value : undefined; + const shimmedSources = new WeakMap(); + + const readOriginalInputSources = (): XRInputSource[] => { + const sources = originalGetter ? originalGetter() : (originalValue ?? []); + const trackedSources = sessionWithMarker.trackedSources as Iterable | undefined; + const mergedSources = [ + ...Array.from(sources as Iterable), + ...(trackedSources ? Array.from(trackedSources) : []), + ]; + + return Array.from(new Set(mergedSources)); + }; + + const getShimmedSource = (inputSource: XRInputSource): XRInputSource => { + const existing = shimmedSources.get(inputSource); + if (existing) { + return existing; + } + + const syntheticGamepad = makeSyntheticGamepad(inputSource.handedness); + const proxy = new Proxy(inputSource as XRInputSource & Record, { + get(target, prop) { + if (prop === AVP_CONTROLLER_SHIMMED_SOURCE) { + return true; + } + if (prop === AVP_CONTROLLER_SHIM_RAW_SOURCE) { + return target; + } + if (prop === AVP_CONTROLLER_SHIM_SPACE_SOURCE) { + return getFallbackHandSpace(target) ? 'hand-wrist' : 'native'; + } + if (prop === 'gamepad') { + return syntheticGamepad; + } + if (prop === 'hand') { + return undefined; + } + if (prop === 'gripSpace') { + return getFallbackControllerSpace(target); + } + if (prop === 'targetRaySpace') { + return target.targetRaySpace ?? getFallbackControllerSpace(target); + } + if (prop === 'targetRayMode') { + return target.targetRayMode === 'tracked-pointer' + ? target.targetRayMode + : 'tracked-pointer'; + } + if (prop === 'profiles') { + const profiles = Reflect.get(target, prop, target) as string[]; + return profiles.includes('generic-trigger-squeeze-thumbstick') + ? profiles + : ['generic-trigger-squeeze-thumbstick', ...profiles]; + } + return Reflect.get(target, prop, target); + }, + }) as XRInputSource; + + shimmedSources.set(inputSource, proxy); + return proxy; + }; + + try { + Object.defineProperty(xrSession, 'inputSources', { + configurable: true, + enumerable: descriptor?.enumerable ?? true, + get: () => + readOriginalInputSources().map(inputSource => + shouldShimInputSource(inputSource) ? getShimmedSource(inputSource) : inputSource + ), + }); + sessionWithMarker[AVP_CONTROLLER_SHIM_INSTALLED] = true; + console.info('[CloudXR WebXR input] AVP controller shim installed'); + return true; + } catch (error) { + console.warn('[CloudXR WebXR input] Failed to install AVP controller shim:', error); + return false; + } +} + +function getPoseAvailability( + xrFrame: XRFrame, + space: XRSpace | undefined, + referenceSpace: XRReferenceSpace | null +): string { + if (!space || !referenceSpace) { + return '0/0'; + } + try { + return xrFrame.getPose(space, referenceSpace) ? '1/1' : '1/0'; + } catch { + return '1/err'; + } +} + +function summarizeInputSources( + inputSources: XRInputSource[], + xrFrame: XRFrame, + referenceSpace: XRReferenceSpace | null +): string { + const frameSession = xrFrame.session as XRSession & Record; + const enabledFeatures = Array.isArray(frameSession.enabledFeatures) + ? (frameSession.enabledFeatures as string[]) + : []; + const trackedSources = frameSession.trackedSources as Iterable | undefined; + const prefix = [ + `features=${enabledFeatures.length ? enabledFeatures.join('+') : 'none'}`, + `primary=${Array.from(frameSession.inputSources ?? []).length}`, + `tracked=${trackedSources ? Array.from(trackedSources).length : 0}`, + ].join(' '); + + if (inputSources.length === 0) { + return `${prefix} | none`; + } + + return `${prefix} | ${inputSources + .map(inputSource => { + const rawInputSource = + ((inputSource as unknown as Record)[ + AVP_CONTROLLER_SHIM_RAW_SOURCE + ] as XRInputSource | undefined) ?? inputSource; + const hand = inputSource.hand; + const gamepad = inputSource.gamepad; + const rawHand = rawInputSource.hand; + const handedness = + inputSource.handedness === 'left' + ? 'L' + : inputSource.handedness === 'right' + ? 'R' + : 'N'; + let joints = 0; + if (rawHand && referenceSpace) { + for (const jointSpace of rawHand.values()) { + try { + if (jointSpace && xrFrame.getJointPose(jointSpace, referenceSpace)) { + joints += 1; + } + } catch { + // Keep the summary useful even if one joint is unavailable for this frame. + } + } + } + + const profiles = inputSource.profiles.length ? inputSource.profiles.join('+') : 'none'; + const sdkControllerCandidate = gamepad && !hand; + const buttons = gamepad?.buttons?.length ?? 0; + const axes = gamepad?.axes?.length ?? 0; + const shimmed = (inputSource as unknown as Record)[ + AVP_CONTROLLER_SHIMMED_SOURCE + ]; + const shimSpace = (inputSource as unknown as Record)[ + AVP_CONTROLLER_SHIM_SPACE_SOURCE + ]; + + return [ + `${handedness}:${inputSource.targetRayMode}`, + `prof=${profiles}`, + `hand=${hand ? 1 : 0}`, + `rawHand=${rawHand ? 1 : 0}`, + `j=${joints}`, + `gp=${gamepad ? 1 : 0}`, + `btn=${buttons}`, + `axes=${axes}`, + `grip=${getPoseAvailability(xrFrame, inputSource.gripSpace, referenceSpace)}`, + `aim=${getPoseAvailability(xrFrame, inputSource.targetRaySpace, referenceSpace)}`, + `shim=${shimmed ? 1 : 0}`, + `shimSpace=${shimSpace ?? 'none'}`, + `sdkCtrl=${sdkControllerCandidate ? 1 : 0}`, + ].join(' '); + }) + .join(' | ')}`; +} + /** * Props for the CloudXRComponent. */ @@ -77,6 +329,9 @@ interface CloudXRComponentProps { /** Callback fired with streaming performance metrics. Receives rolling averages of streaming FPS and pose-to-render latency (ms). */ onStreamingPerformanceMetrics?: (streamingFps: number, poseToRenderMs: number) => void; + /** Callback fired with WebXR input source diagnostics visible in the in-XR panel. */ + onInputSourcesDebug?: (summary: string) => void; + /** * Settings for the performance metrics reported via onRenderPerformanceMetrics and onStreamingPerformanceMetrics callbacks. * Each window size controls how many samples are averaged before reporting. @@ -113,6 +368,7 @@ export default function CloudXRComponent({ onServerAddress, onRenderPerformanceMetrics, onStreamingPerformanceMetrics, + onInputSourcesDebug, metricsSettings = {}, headless = false, iceServers, @@ -122,6 +378,8 @@ export default function CloudXRComponent({ // React reference to the CloudXR session that persists across re-renders. const cxrSessionRef = useRef(null); const onExitImmersiveXRRef = useRef(onExitImmersiveXR); + const lastInputDebugTimeRef = useRef(0); + const lastInputDebugSummaryRef = useRef(''); onExitImmersiveXRRef.current = onExitImmersiveXR; // Metrics trackers for averaging performance metrics @@ -158,6 +416,10 @@ export default function CloudXRComponent({ ? (webXRManager as any).getSession() : null; if (xrSession) { + if (shouldEnableAvpControllerShim()) { + installAvpControllerInputShim(xrSession); + } + if (config.referenceSpaceType === 'auto') { const fallbacks: XRReferenceSpaceType[] = [ 'local-floor', @@ -400,6 +662,22 @@ export default function CloudXRComponent({ if (xrFrame) { // Get THREE WebXRManager from the useFrame state. const webXRManager = state.gl.xr; + const timestamp: DOMHighResTimeStamp = state.clock.elapsedTime * 1000; + + if (onInputSourcesDebug && timestamp - lastInputDebugTimeRef.current > 500) { + const referenceSpace = webXRManager.getReferenceSpace(); + const summary = summarizeInputSources( + Array.from(session.inputSources), + xrFrame, + referenceSpace + ); + onInputSourcesDebug?.(summary); + if (summary !== lastInputDebugSummaryRef.current) { + console.info(`[CloudXR WebXR input] ${summary}`); + lastInputDebugSummaryRef.current = summary; + } + lastInputDebugTimeRef.current = timestamp; + } if (!cxrSessionRef || !cxrSessionRef.current) { console.debug('Skipping frame, no session yet'); @@ -423,9 +701,6 @@ export default function CloudXRComponent({ return; } - // Get timestamp from useFrame state and convert to milliseconds. - const timestamp: DOMHighResTimeStamp = state.clock.elapsedTime * 1000; - try { // Send the tracking state (including viewer pose and hand/controller data) to the server; // that triggers server-side rendering for the frame. diff --git a/deps/cloudxr/webxr_client/src/App.tsx b/deps/cloudxr/webxr_client/src/App.tsx index cf61bf074..c508ccf77 100644 --- a/deps/cloudxr/webxr_client/src/App.tsx +++ b/deps/cloudxr/webxr_client/src/App.tsx @@ -82,6 +82,10 @@ const streamingFpsText = computed(() => const poseToRenderText = computed(() => streamingMetrics.value ? `${streamingMetrics.value.latencyMs.toFixed(1)}ms` : '-' ); +const inputSourcesDebugText = signal('pending'); +const AVP_INPUT_DEBUG_QUERY_PARAM = 'avpInputDebug'; +const AVP_HIDE_CONTROLLER_MODELS_QUERY_PARAM = 'avpHideControllerModels'; +const AVP_REQUIRE_HAND_QUERY_PARAM = 'avpRequireHand'; const CONTROL_PANEL_LAYOUT = { distance: 1.8, @@ -89,6 +93,23 @@ const CONTROL_PANEL_LAYOUT = { angleDegrees: 70, } as const; +function isTruthyQueryParam(name: string): boolean { + const value = new URLSearchParams(window.location.search).get(name); + return value === '1' || value?.toLowerCase() === 'true'; +} + +function shouldDebugInputSources(): boolean { + return isTruthyQueryParam(AVP_INPUT_DEBUG_QUERY_PARAM); +} + +function shouldHideControllerModelsForAvpDebug(): boolean { + return isTruthyQueryParam(AVP_HIDE_CONTROLLER_MODELS_QUERY_PARAM); +} + +function shouldRequireAvpHandTracking(): boolean { + return isTruthyQueryParam(AVP_REQUIRE_HAND_QUERY_PARAM); +} + // Override PressureObserver early to catch errors from buggy browser implementations overridePressureObserver(); @@ -150,6 +171,8 @@ function App() { const [sessionStatus, setSessionStatus] = useState('Disconnected'); // Error message management const [errorMessage, setErrorMessage] = useState(''); + const showInputSourcesDebug = shouldDebugInputSources(); + const [inputSourcesDebugSnapshot, setInputSourcesDebugSnapshot] = useState('pending'); // CloudXR session reference const [cloudXRSession, setCloudXRSession] = useState(null); // XR mode state for UI visibility @@ -292,7 +315,9 @@ function App() { const xrFrameBufferScaling = deviceProfile.web?.frameBufferScaling ?? kPerformanceOptions.xrWebGLLayer_framebufferScaleFactor; - const hideControllerModel = cloudXR2DUI?.getConfiguration().hideControllerModel ?? false; + const hideControllerModel = + shouldHideControllerModelsForAvpDebug() || + (cloudXR2DUI?.getConfiguration().hideControllerModel ?? false); // XR store must be created after we know which device profile is active. // useMemo prevents re-creating the store for unrelated UI changes. @@ -312,8 +337,10 @@ function App() { controller: { model: !hideControllerModel, // Allow UI to hide controller models while keeping input active }, - // Request optional WebXR features - use property names, not optionalFeatures array! - handTracking: true, + secondaryInputSources: true, + // Request WebXR hand tracking. Use avpRequireHand=1 to make Safari fail fast if + // the CloudXR session is not actually granted hand input sources. + handTracking: shouldRequireAvpHandTracking() ? 'required' : true, bodyTracking: true, // Explicitly disable environment/scene feature requests to avoid extra headset prompts. anchors: false, @@ -515,6 +542,11 @@ function App() { streamingMetrics.value = { fps, latencyMs }; }; + const handleInputSourcesDebug = (summary: string) => { + inputSourcesDebugText.value = summary; + setInputSourcesDebugSnapshot(summary); + }; + /** * Helper to send a message using MessageChannel API (new) or legacy API (fallback). * Looks for the teleop_command channel by UUID, then falls back to legacy API. @@ -876,6 +908,29 @@ function App() { return ( <> + {showInputSourcesDebug && ( +
+
CloudXR WebXR Input
+
{inputSourcesDebugSnapshot}
+
+ )} {!config.headless && ( @@ -953,6 +1011,9 @@ function App() { renderFpsText={renderFpsText} streamingFpsText={streamingFpsText} poseToRenderText={poseToRenderText} + inputSourcesDebugText={ + showInputSourcesDebug ? inputSourcesDebugText : undefined + } /> )} diff --git a/deps/cloudxr/webxr_client/src/CloudXRUI.tsx b/deps/cloudxr/webxr_client/src/CloudXRUI.tsx index 4a6c82c7b..2a57a9fa6 100644 --- a/deps/cloudxr/webxr_client/src/CloudXRUI.tsx +++ b/deps/cloudxr/webxr_client/src/CloudXRUI.tsx @@ -76,6 +76,8 @@ interface CloudXRUIProps { streamingFpsText?: ReadonlySignal; /** Computed signal for pose-to-render latency text - updates without React re-render */ poseToRenderText?: ReadonlySignal; + /** WebXR input source diagnostics. */ + inputSourcesDebugText?: ReadonlySignal; /** From settings: hide control panel when immersive XR begins. */ panelHiddenAtStart?: boolean; /** Immersive XR active; used to apply panelHiddenAtStart on session enter. */ @@ -119,6 +121,7 @@ export default function CloudXR3DUI({ renderFpsText, streamingFpsText, poseToRenderText, + inputSourcesDebugText, panelHiddenAtStart = false, isXRMode = false, }: CloudXRUIProps) { @@ -467,6 +470,11 @@ export default function CloudXR3DUI({ Status: {sessionStatus} + {inputSourcesDebugText && ( + + Input: {inputSourcesDebugText} + + )} {/* Countdown Config Row */} diff --git a/examples/g1_wuji_teleop/README.md b/examples/g1_wuji_teleop/README.md new file mode 100644 index 000000000..a11e43065 --- /dev/null +++ b/examples/g1_wuji_teleop/README.md @@ -0,0 +1,44 @@ + + +# G1-Wuji Teleop Example + +Workspace-local example for the G1-Wuji AVP + MANUS teleoperation demo. + +## Layout + +- `config/`: app-owned YAML configuration. +- `assets/`: app-owned robot and retargeting assets copied out of legacy examples. +- `python/g1_wuji_teleop/`: implementation package. +- `scripts/`: thin launchers for operators and developers. + +`config/` is organized by retargeting route first, then by robot: + +- `config/local/g1_wuji/`: local native-Dex config for `g1_wuji` +- `config/official/g1_wuji/`: official `wuji-retargeting` config for `g1_wuji` + +The `scripts/` launchers resolve config/assets from this example first and do +not rely on `examples/teleop/python/` for their default runtime path. + +For the `g1_wuji` official retargeting path, the local configs point +`optimizer.urdf_path` at this app's `assets/wuji_hand/urdf/` files, so the +vendored `wuji-retargeting` checkout does not need its nested +`wuji-description` submodule for this demo. + +## Entry Points + +- Session / Isaac Sim example: + + ```bash + python examples/g1_wuji_teleop/scripts/g1_wuji_teleop_main.py \ + --config examples/g1_wuji_teleop/config/avp_manus.yml + ``` + +- Visualizer: + + ```bash + python examples/g1_wuji_teleop/scripts/g1_wuji_teleop_visualizer.py \ + --config examples/g1_wuji_teleop/config/avp_manus.yml + ``` diff --git a/examples/g1_wuji_teleop/assets/g1_wuji/g1_three_fingers.usd b/examples/g1_wuji_teleop/assets/g1_wuji/g1_three_fingers.usd new file mode 100644 index 000000000..a1d80309a --- /dev/null +++ b/examples/g1_wuji_teleop/assets/g1_wuji/g1_three_fingers.usd @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:799cd8f50364d4388540a34c9380185a84f1f433bd7baec5fae31fc3640a7666 +size 40094469 diff --git a/examples/g1_wuji_teleop/assets/g1_wuji/g1_wuji.usd b/examples/g1_wuji_teleop/assets/g1_wuji/g1_wuji.usd new file mode 100644 index 000000000..f5a8770aa --- /dev/null +++ b/examples/g1_wuji_teleop/assets/g1_wuji/g1_wuji.usd @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:acc0afc786907290fcb81f796ede6fa791bbae75c37c8168f8b8dbbc9c1e3d51 +size 47977976 diff --git a/examples/g1_wuji_teleop/assets/wuji_hand/meshes/left/finger1_link1.STL b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/left/finger1_link1.STL new file mode 100644 index 000000000..cb3cd3a36 --- /dev/null +++ b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/left/finger1_link1.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b17f48ce8974b30536376ef150abc9b8abdf2b6028af52757d3ce90c3f8174be +size 349784 diff --git a/examples/g1_wuji_teleop/assets/wuji_hand/meshes/left/finger1_link1_collision.STL b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/left/finger1_link1_collision.STL new file mode 100644 index 000000000..6fc5ae36a --- /dev/null +++ b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/left/finger1_link1_collision.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:853c85e7149580c96d8f6b221c6132595cc4dc5fc33e73dfccfbfc240ca6b161 +size 25084 diff --git a/examples/g1_wuji_teleop/assets/wuji_hand/meshes/left/finger1_link2.STL b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/left/finger1_link2.STL new file mode 100644 index 000000000..e0d32c792 --- /dev/null +++ b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/left/finger1_link2.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:53b16e2fc88fa152189750f7176b717cdc2b0ad8ade3e91d546355b196274205 +size 92484 diff --git a/examples/g1_wuji_teleop/assets/wuji_hand/meshes/left/finger1_link2_collision.STL b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/left/finger1_link2_collision.STL new file mode 100644 index 000000000..bb5f9faa5 --- /dev/null +++ b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/left/finger1_link2_collision.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:34150ded0f71fc59a028eb40d6a747618a3aa1d0c44823550a59ec1b74451b9f +size 25084 diff --git a/examples/g1_wuji_teleop/assets/wuji_hand/meshes/left/finger1_link3.STL b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/left/finger1_link3.STL new file mode 100644 index 000000000..6f75864c7 --- /dev/null +++ b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/left/finger1_link3.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ebcbd6e485881a948b825c8f3c5d8d7120a5fea8d2590bf27664c4dc72940ba9 +size 87984 diff --git a/examples/g1_wuji_teleop/assets/wuji_hand/meshes/left/finger1_link3_collision.STL b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/left/finger1_link3_collision.STL new file mode 100644 index 000000000..23ffb3756 --- /dev/null +++ b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/left/finger1_link3_collision.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:adb4f921e1ff437296077699dc770bb3a256353d53bbf6c9aa09f3d52985b3b8 +size 25084 diff --git a/examples/g1_wuji_teleop/assets/wuji_hand/meshes/left/finger1_link4.STL b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/left/finger1_link4.STL new file mode 100644 index 000000000..bfbf1c079 --- /dev/null +++ b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/left/finger1_link4.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0dcb58b73f8ec2a491f6044a742e8e3a0cb8a9c77c5a9a899d36537ccb9da698 +size 64584 diff --git a/examples/g1_wuji_teleop/assets/wuji_hand/meshes/left/finger1_link4_collision.STL b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/left/finger1_link4_collision.STL new file mode 100644 index 000000000..552a22129 --- /dev/null +++ b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/left/finger1_link4_collision.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:65ed57875ed25de2141a4a195bbaf555f510cfc44de4e99681c6957df18c8b0c +size 25084 diff --git a/examples/g1_wuji_teleop/assets/wuji_hand/meshes/left/finger1_tip_link.STL b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/left/finger1_tip_link.STL new file mode 100644 index 000000000..593f336c7 --- /dev/null +++ b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/left/finger1_tip_link.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6ac9ddb655262dfa0a339783141eb8f31387e6314d92d9a84a06e960cb64bc86 +size 96384 diff --git a/examples/g1_wuji_teleop/assets/wuji_hand/meshes/left/finger1_tip_link_collision.STL b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/left/finger1_tip_link_collision.STL new file mode 100644 index 000000000..78fd167ff --- /dev/null +++ b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/left/finger1_tip_link_collision.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:23e9a2e6058f523437d6ad0689d54d45c04d002c1dfac85bc073e16f6e8780b6 +size 25084 diff --git a/examples/g1_wuji_teleop/assets/wuji_hand/meshes/left/finger2_link1.STL b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/left/finger2_link1.STL new file mode 100644 index 000000000..a655f0e33 --- /dev/null +++ b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/left/finger2_link1.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cfe8003977ed8fe69e04633d1cbfd09714f4fd1f5b6947197da3019bb7051959 +size 56184 diff --git a/examples/g1_wuji_teleop/assets/wuji_hand/meshes/left/finger2_link1_collision.STL b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/left/finger2_link1_collision.STL new file mode 100644 index 000000000..e8d4f311b --- /dev/null +++ b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/left/finger2_link1_collision.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29d52cbef248abfc5ce8df43fcf3ffa900635127cced5bf859d9694bfd58fbc2 +size 25084 diff --git a/examples/g1_wuji_teleop/assets/wuji_hand/meshes/left/finger2_link2.STL b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/left/finger2_link2.STL new file mode 100644 index 000000000..96fc99cf6 --- /dev/null +++ b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/left/finger2_link2.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1131e9a85112b3ba9bc792544d8a522bec93412293668626f3d52ad9f7ffe8b6 +size 51984 diff --git a/examples/g1_wuji_teleop/assets/wuji_hand/meshes/left/finger2_link2_collision.STL b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/left/finger2_link2_collision.STL new file mode 100644 index 000000000..1d8dab458 --- /dev/null +++ b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/left/finger2_link2_collision.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4ef6ff2efef32ccc8ff8e2eee28db7463ba9ae05b158d5666d4bd5ef16161ad0 +size 25084 diff --git a/examples/g1_wuji_teleop/assets/wuji_hand/meshes/left/finger2_link3.STL b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/left/finger2_link3.STL new file mode 100644 index 000000000..cfa21284b --- /dev/null +++ b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/left/finger2_link3.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b5181c09e54860d98f3be64a644efdb664d134006f6c48549ed112f8f63c62fa +size 91284 diff --git a/examples/g1_wuji_teleop/assets/wuji_hand/meshes/left/finger2_link3_collision.STL b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/left/finger2_link3_collision.STL new file mode 100644 index 000000000..c508a882f --- /dev/null +++ b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/left/finger2_link3_collision.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7990ea209c1a0456bcbc65c3b4bbf831b966635ca57cb324175c6592886e94f6 +size 25084 diff --git a/examples/g1_wuji_teleop/assets/wuji_hand/meshes/left/finger2_link4.STL b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/left/finger2_link4.STL new file mode 100644 index 000000000..659612df2 --- /dev/null +++ b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/left/finger2_link4.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c64e2fdc5c192f8dc327dacf60b35609304ad5cea3fc7358d3f780aac4e235a2 +size 54884 diff --git a/examples/g1_wuji_teleop/assets/wuji_hand/meshes/left/finger2_link4_collision.STL b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/left/finger2_link4_collision.STL new file mode 100644 index 000000000..c4a3c3578 --- /dev/null +++ b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/left/finger2_link4_collision.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ce7659ecfc46967b08da7ab064cefca5c6223c7576f22c7dcaf4fd8219983ca7 +size 25084 diff --git a/examples/g1_wuji_teleop/assets/wuji_hand/meshes/left/finger2_tip_link.STL b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/left/finger2_tip_link.STL new file mode 100644 index 000000000..bb48f1adf --- /dev/null +++ b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/left/finger2_tip_link.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1e8be5a90e81c319302053c9fa27608c900d47a5b67797fc931449885b9f86af +size 112784 diff --git a/examples/g1_wuji_teleop/assets/wuji_hand/meshes/left/finger2_tip_link_collision.STL b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/left/finger2_tip_link_collision.STL new file mode 100644 index 000000000..91cdf5105 --- /dev/null +++ b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/left/finger2_tip_link_collision.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1fdf002df36490d481a10593d476eb9239f666339559ed5562cb5d5ffc93a41d +size 25084 diff --git a/examples/g1_wuji_teleop/assets/wuji_hand/meshes/left/finger3_link1.STL b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/left/finger3_link1.STL new file mode 100644 index 000000000..e885af052 --- /dev/null +++ b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/left/finger3_link1.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2ed2537fdea1584af89e562e9fc115c8ac06310f1c67f074ddc5a1eb57cdda6b +size 56184 diff --git a/examples/g1_wuji_teleop/assets/wuji_hand/meshes/left/finger3_link1_collision.STL b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/left/finger3_link1_collision.STL new file mode 100644 index 000000000..b45d2c42c --- /dev/null +++ b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/left/finger3_link1_collision.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d8027c721b5c0c69fe2f04db0533d000c46fef43dc5db8a9430a01ae8ad88476 +size 25084 diff --git a/examples/g1_wuji_teleop/assets/wuji_hand/meshes/left/finger3_link2.STL b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/left/finger3_link2.STL new file mode 100644 index 000000000..a34be67c5 --- /dev/null +++ b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/left/finger3_link2.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3e76e1b3adbcbcb5f28fec54492a44c24d6ab9dfc6d8636c1cc4dc31f974559e +size 51984 diff --git a/examples/g1_wuji_teleop/assets/wuji_hand/meshes/left/finger3_link2_collision.STL b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/left/finger3_link2_collision.STL new file mode 100644 index 000000000..30301d0aa --- /dev/null +++ b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/left/finger3_link2_collision.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0302da39f9c148cffb16d680fe5399efa0a022462ab9bb3937e3146cdfe3d88e +size 25084 diff --git a/examples/g1_wuji_teleop/assets/wuji_hand/meshes/left/finger3_link3.STL b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/left/finger3_link3.STL new file mode 100644 index 000000000..6a128f65b --- /dev/null +++ b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/left/finger3_link3.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e78e7bec8901a499e664b99900b96acd4b8f78a547af06753b09b90cd51e949b +size 91284 diff --git a/examples/g1_wuji_teleop/assets/wuji_hand/meshes/left/finger3_link3_collision.STL b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/left/finger3_link3_collision.STL new file mode 100644 index 000000000..561e5e206 --- /dev/null +++ b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/left/finger3_link3_collision.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6ea57bfc0ca89c9f80e80702f5a6b22e96e38858c41b407cb390cbe081cae534 +size 25084 diff --git a/examples/g1_wuji_teleop/assets/wuji_hand/meshes/left/finger3_link4.STL b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/left/finger3_link4.STL new file mode 100644 index 000000000..f7bbb9a93 --- /dev/null +++ b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/left/finger3_link4.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b7a16806e231d51ff50380b3d4b25bac3f347e6f13918c7373d018bcd9da11dd +size 54884 diff --git a/examples/g1_wuji_teleop/assets/wuji_hand/meshes/left/finger3_link4_collision.STL b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/left/finger3_link4_collision.STL new file mode 100644 index 000000000..4bdfd294b --- /dev/null +++ b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/left/finger3_link4_collision.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:281eff1ca8541cf81d932edd676a5561ceee65ac2955b3af4df3906ba71e0bd3 +size 25084 diff --git a/examples/g1_wuji_teleop/assets/wuji_hand/meshes/left/finger3_tip_link.STL b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/left/finger3_tip_link.STL new file mode 100644 index 000000000..4c08a6ffb --- /dev/null +++ b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/left/finger3_tip_link.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b918123bb7f6dc4d94b6b62ceb01b5331d938066f5ce40b8138cb2aa8c755117 +size 113484 diff --git a/examples/g1_wuji_teleop/assets/wuji_hand/meshes/left/finger3_tip_link_collision.STL b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/left/finger3_tip_link_collision.STL new file mode 100644 index 000000000..a2140b781 --- /dev/null +++ b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/left/finger3_tip_link_collision.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4b7c505a71cae1ea747f44c9a808f615cda8859ea8472370e67a64e42a600908 +size 25084 diff --git a/examples/g1_wuji_teleop/assets/wuji_hand/meshes/left/finger4_link1.STL b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/left/finger4_link1.STL new file mode 100644 index 000000000..fb3c5aadb --- /dev/null +++ b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/left/finger4_link1.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0664072a389f47449eba9636583c40a2f95386ab23feacc2962fcb9a3070cb34 +size 56184 diff --git a/examples/g1_wuji_teleop/assets/wuji_hand/meshes/left/finger4_link1_collision.STL b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/left/finger4_link1_collision.STL new file mode 100644 index 000000000..27467a46c --- /dev/null +++ b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/left/finger4_link1_collision.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ce0f42d7c684b7036bc5aeba3bd53ea7611dc4ad00592a448acaa95c8590491b +size 25084 diff --git a/examples/g1_wuji_teleop/assets/wuji_hand/meshes/left/finger4_link2.STL b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/left/finger4_link2.STL new file mode 100644 index 000000000..8afad4fd1 --- /dev/null +++ b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/left/finger4_link2.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eefec26f19d27322ccd9b2b1b080431bb6d63f3a4e1ea0b480c7d07c09eab655 +size 51984 diff --git a/examples/g1_wuji_teleop/assets/wuji_hand/meshes/left/finger4_link2_collision.STL b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/left/finger4_link2_collision.STL new file mode 100644 index 000000000..22b8c4b10 --- /dev/null +++ b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/left/finger4_link2_collision.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bc8ccd945de619743a9e94c0fe34e56afc346758ab5ea4439c5531c71d0383ca +size 25084 diff --git a/examples/g1_wuji_teleop/assets/wuji_hand/meshes/left/finger4_link3.STL b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/left/finger4_link3.STL new file mode 100644 index 000000000..d6ff1953a --- /dev/null +++ b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/left/finger4_link3.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6d27de502b0e12cff871037a7bd206b8caaa808cb1fd580305fbacf269b50d08 +size 91284 diff --git a/examples/g1_wuji_teleop/assets/wuji_hand/meshes/left/finger4_link3_collision.STL b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/left/finger4_link3_collision.STL new file mode 100644 index 000000000..ab072f515 --- /dev/null +++ b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/left/finger4_link3_collision.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:922b21c1e98f21b20b525f5060e07e07b7aa9f3fef0472b24c3a4d3ace7cc78b +size 25084 diff --git a/examples/g1_wuji_teleop/assets/wuji_hand/meshes/left/finger4_link4.STL b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/left/finger4_link4.STL new file mode 100644 index 000000000..468685557 --- /dev/null +++ b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/left/finger4_link4.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fb629c2fe0957bb6136b62a4f67ccb25839e79ad916b67c000bb3d726d9d9c2b +size 54884 diff --git a/examples/g1_wuji_teleop/assets/wuji_hand/meshes/left/finger4_link4_collision.STL b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/left/finger4_link4_collision.STL new file mode 100644 index 000000000..c0c26378b --- /dev/null +++ b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/left/finger4_link4_collision.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c45c282dd5e3db4b66ccc90e84529f3df698e924a2f5dca53297632cbb17bf85 +size 25084 diff --git a/examples/g1_wuji_teleop/assets/wuji_hand/meshes/left/finger4_tip_link.STL b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/left/finger4_tip_link.STL new file mode 100644 index 000000000..f11927429 --- /dev/null +++ b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/left/finger4_tip_link.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7d6739ac876ec9b7c6c36bd1fdd068389474407339cf97dbb93881b2cdf0f2cc +size 112784 diff --git a/examples/g1_wuji_teleop/assets/wuji_hand/meshes/left/finger4_tip_link_collision.STL b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/left/finger4_tip_link_collision.STL new file mode 100644 index 000000000..ee09d3433 --- /dev/null +++ b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/left/finger4_tip_link_collision.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6593376f3a70d40346661f5e042903e41d3040c630bfc5a66c72f55739804567 +size 25084 diff --git a/examples/g1_wuji_teleop/assets/wuji_hand/meshes/left/finger5_link1.STL b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/left/finger5_link1.STL new file mode 100644 index 000000000..922515657 --- /dev/null +++ b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/left/finger5_link1.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8ff604ba894a320331516f0227aed75ffdd47d9a6ec29a8e792b2e7777bbb681 +size 56184 diff --git a/examples/g1_wuji_teleop/assets/wuji_hand/meshes/left/finger5_link1_collision.STL b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/left/finger5_link1_collision.STL new file mode 100644 index 000000000..3a73638f8 --- /dev/null +++ b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/left/finger5_link1_collision.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b5901c224dd0b657e62d724c6b561ae0d77b8d955ae0454e8a93985c21d14472 +size 25084 diff --git a/examples/g1_wuji_teleop/assets/wuji_hand/meshes/left/finger5_link2.STL b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/left/finger5_link2.STL new file mode 100644 index 000000000..4dd29380e --- /dev/null +++ b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/left/finger5_link2.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4f01c61e81bb440c789d242c43d00c16c44590ad2a25f004f8783e7df80bb3db +size 51984 diff --git a/examples/g1_wuji_teleop/assets/wuji_hand/meshes/left/finger5_link2_collision.STL b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/left/finger5_link2_collision.STL new file mode 100644 index 000000000..d1fb211d1 --- /dev/null +++ b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/left/finger5_link2_collision.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6136648bc9de8049d130376b049cac1f7a2deeb30534e6aa75f6ae82f89ce51e +size 25084 diff --git a/examples/g1_wuji_teleop/assets/wuji_hand/meshes/left/finger5_link3.STL b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/left/finger5_link3.STL new file mode 100644 index 000000000..d6f2ea32b --- /dev/null +++ b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/left/finger5_link3.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:20052a953fdfda74337317d939c59762bb70bb52c206eb62c4fa77e846ca948a +size 91284 diff --git a/examples/g1_wuji_teleop/assets/wuji_hand/meshes/left/finger5_link3_collision.STL b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/left/finger5_link3_collision.STL new file mode 100644 index 000000000..d27acdca4 --- /dev/null +++ b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/left/finger5_link3_collision.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3f8e87de06a09b9451352f870bfcd3cc4120676eb5aa108aa21409c03544026c +size 25084 diff --git a/examples/g1_wuji_teleop/assets/wuji_hand/meshes/left/finger5_link4.STL b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/left/finger5_link4.STL new file mode 100644 index 000000000..731214cd3 --- /dev/null +++ b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/left/finger5_link4.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9355ba84b24102ccb8251a2213c63f48acf98dce47865b4ff446cec7c33c38cc +size 54884 diff --git a/examples/g1_wuji_teleop/assets/wuji_hand/meshes/left/finger5_link4_collision.STL b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/left/finger5_link4_collision.STL new file mode 100644 index 000000000..dd97e3894 --- /dev/null +++ b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/left/finger5_link4_collision.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3426b242ab5c4b8bba7e331a46eeaacba6094797d1fbdb4cff74cfc2cb1a564a +size 25084 diff --git a/examples/g1_wuji_teleop/assets/wuji_hand/meshes/left/finger5_tip_link.STL b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/left/finger5_tip_link.STL new file mode 100644 index 000000000..9d38ea072 --- /dev/null +++ b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/left/finger5_tip_link.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:32636af3449202bc1689432c2f87eb5fa74b1b7a423a03aa6f850a7c829b8ea0 +size 112784 diff --git a/examples/g1_wuji_teleop/assets/wuji_hand/meshes/left/finger5_tip_link_collision.STL b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/left/finger5_tip_link_collision.STL new file mode 100644 index 000000000..565c9abd5 --- /dev/null +++ b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/left/finger5_tip_link_collision.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ea5df32f12ccb000510c2a96ea9adbf0c4be6f2b88ee0bcdbc88a5e1fc609f22 +size 25084 diff --git a/examples/g1_wuji_teleop/assets/wuji_hand/meshes/left/palm_link.STL b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/left/palm_link.STL new file mode 100644 index 000000000..213695039 --- /dev/null +++ b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/left/palm_link.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cd1dfefc9be806bda6839cc1c787ed6e6c7cdd0773c0cf91d543877ac6a60aff +size 359284 diff --git a/examples/g1_wuji_teleop/assets/wuji_hand/meshes/left/palm_link_collision.STL b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/left/palm_link_collision.STL new file mode 100644 index 000000000..bd39ef195 --- /dev/null +++ b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/left/palm_link_collision.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1c5d4fe26c033a0c7820556a438825d9c206a0d63a5758e8e10ccaaf5269076f +size 25084 diff --git a/examples/g1_wuji_teleop/assets/wuji_hand/meshes/right/finger1_link1.STL b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/right/finger1_link1.STL new file mode 100644 index 000000000..bc2da1cbe --- /dev/null +++ b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/right/finger1_link1.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0e32302a46cd00d6cc1e7fb03cc483988c639271708c2300a9aecc9de537e42b +size 366584 diff --git a/examples/g1_wuji_teleop/assets/wuji_hand/meshes/right/finger1_link1_collision.STL b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/right/finger1_link1_collision.STL new file mode 100644 index 000000000..e603495cd --- /dev/null +++ b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/right/finger1_link1_collision.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2adeb7b286e96c57c354c0b4d2107aa7bbd63a58b47a2fce828f313722d8eae2 +size 25084 diff --git a/examples/g1_wuji_teleop/assets/wuji_hand/meshes/right/finger1_link2.STL b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/right/finger1_link2.STL new file mode 100644 index 000000000..47ebbd0fa --- /dev/null +++ b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/right/finger1_link2.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:080015cce41de31954e53bbfca45f37f0439e2e689bca49e1910c70efc4f70b7 +size 92684 diff --git a/examples/g1_wuji_teleop/assets/wuji_hand/meshes/right/finger1_link2_collision.STL b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/right/finger1_link2_collision.STL new file mode 100644 index 000000000..a18af74bb --- /dev/null +++ b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/right/finger1_link2_collision.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:930360774dcefe897cafcf6145d042217a78f1b54ca98dc7a1017db889f18afc +size 25084 diff --git a/examples/g1_wuji_teleop/assets/wuji_hand/meshes/right/finger1_link3.STL b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/right/finger1_link3.STL new file mode 100644 index 000000000..baf64130f --- /dev/null +++ b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/right/finger1_link3.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:52eef283d29c617fe4457151685386324bddbccf1598b8a7fc675bb8f28dc85a +size 89184 diff --git a/examples/g1_wuji_teleop/assets/wuji_hand/meshes/right/finger1_link3_collision.STL b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/right/finger1_link3_collision.STL new file mode 100644 index 000000000..ba9a3b435 --- /dev/null +++ b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/right/finger1_link3_collision.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f2042ec400e5840bf1b3c22aebb7c50ffa466e0f23474908c40ab546eefdbc6c +size 25084 diff --git a/examples/g1_wuji_teleop/assets/wuji_hand/meshes/right/finger1_link4.STL b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/right/finger1_link4.STL new file mode 100644 index 000000000..219cb7859 --- /dev/null +++ b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/right/finger1_link4.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f0fb321c4abfd4ab7437caeb116f1935b94d1510fd921222eb49ba6c5a57f057 +size 65984 diff --git a/examples/g1_wuji_teleop/assets/wuji_hand/meshes/right/finger1_link4_collision.STL b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/right/finger1_link4_collision.STL new file mode 100644 index 000000000..65641915f --- /dev/null +++ b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/right/finger1_link4_collision.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:48eb6a1c4818d6be44c1264b069c06362a2edc0bb8ebacb70f195e1e3f4e9090 +size 25084 diff --git a/examples/g1_wuji_teleop/assets/wuji_hand/meshes/right/finger1_tip_link.STL b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/right/finger1_tip_link.STL new file mode 100644 index 000000000..822ade70c --- /dev/null +++ b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/right/finger1_tip_link.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:25929e0c28a5c65afe88dbccaa697664a2385f8ead02126731a7e1b956ec1dea +size 96084 diff --git a/examples/g1_wuji_teleop/assets/wuji_hand/meshes/right/finger1_tip_link_collision.STL b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/right/finger1_tip_link_collision.STL new file mode 100644 index 000000000..3f650635f --- /dev/null +++ b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/right/finger1_tip_link_collision.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1f76a0f6319bd664570fa2571e6a87db2b5b94a47045732daef486f2896dc220 +size 25084 diff --git a/examples/g1_wuji_teleop/assets/wuji_hand/meshes/right/finger2_link1.STL b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/right/finger2_link1.STL new file mode 100644 index 000000000..5652a26cc --- /dev/null +++ b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/right/finger2_link1.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:36e0471a0373a7634fc10911b33f3b30a5fe829f6526e656a911406cebf19731 +size 56184 diff --git a/examples/g1_wuji_teleop/assets/wuji_hand/meshes/right/finger2_link1_collision.STL b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/right/finger2_link1_collision.STL new file mode 100644 index 000000000..223ccd7f4 --- /dev/null +++ b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/right/finger2_link1_collision.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:55cf7973ab4d57119f6eb032704defddf7077a01022f99d6a9f95eaea7f28875 +size 25084 diff --git a/examples/g1_wuji_teleop/assets/wuji_hand/meshes/right/finger2_link2.STL b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/right/finger2_link2.STL new file mode 100644 index 000000000..e4ca87897 --- /dev/null +++ b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/right/finger2_link2.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:56bb1972b8cb1d63d48aa18283148ff6760e387dc64fe963b11209e07e243511 +size 51984 diff --git a/examples/g1_wuji_teleop/assets/wuji_hand/meshes/right/finger2_link2_collision.STL b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/right/finger2_link2_collision.STL new file mode 100644 index 000000000..ea3f95cc0 --- /dev/null +++ b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/right/finger2_link2_collision.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:db3b779d3cf98615617f5554946205e9f07de5cd9853005305048f617dc96fdc +size 25084 diff --git a/examples/g1_wuji_teleop/assets/wuji_hand/meshes/right/finger2_link3.STL b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/right/finger2_link3.STL new file mode 100644 index 000000000..7e270dd43 --- /dev/null +++ b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/right/finger2_link3.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:48d70e865e2b49139028511808ae3ad25c5cf06e36fe959c6c71b5263dfcbbde +size 91284 diff --git a/examples/g1_wuji_teleop/assets/wuji_hand/meshes/right/finger2_link3_collision.STL b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/right/finger2_link3_collision.STL new file mode 100644 index 000000000..12304a5c7 --- /dev/null +++ b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/right/finger2_link3_collision.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:24f3464762c5ab07b4c56d3df6576ee43cb63eab0938c6366f1153323c619aa5 +size 25084 diff --git a/examples/g1_wuji_teleop/assets/wuji_hand/meshes/right/finger2_link4.STL b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/right/finger2_link4.STL new file mode 100644 index 000000000..b40fd6b23 --- /dev/null +++ b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/right/finger2_link4.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3eede2c8e1eb1301bc23b4b453fb7014ca5965611b44b115a7330bf33e4d922e +size 54884 diff --git a/examples/g1_wuji_teleop/assets/wuji_hand/meshes/right/finger2_link4_collision.STL b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/right/finger2_link4_collision.STL new file mode 100644 index 000000000..bbfd834fc --- /dev/null +++ b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/right/finger2_link4_collision.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a8708b0b03aab6a7fb5074a6c9639dee85be56415018ad16a9a04b6f43b6cc85 +size 25084 diff --git a/examples/g1_wuji_teleop/assets/wuji_hand/meshes/right/finger2_tip_link.STL b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/right/finger2_tip_link.STL new file mode 100644 index 000000000..59e46226a --- /dev/null +++ b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/right/finger2_tip_link.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:496039ce3f4f3fbe6bac466a01567797f741af95c6f11dc61626ec2de175900b +size 112784 diff --git a/examples/g1_wuji_teleop/assets/wuji_hand/meshes/right/finger2_tip_link_collision.STL b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/right/finger2_tip_link_collision.STL new file mode 100644 index 000000000..b5fede500 --- /dev/null +++ b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/right/finger2_tip_link_collision.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:53d0ef5c6346f3f5dd514646df784fa1b6b56c088164496ef661019566028c1d +size 25084 diff --git a/examples/g1_wuji_teleop/assets/wuji_hand/meshes/right/finger3_link1.STL b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/right/finger3_link1.STL new file mode 100644 index 000000000..71d4a7917 --- /dev/null +++ b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/right/finger3_link1.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:558fac18be219c0e6bb6224b82c711b3efe762b547818ea074eda93dc38bad07 +size 56184 diff --git a/examples/g1_wuji_teleop/assets/wuji_hand/meshes/right/finger3_link1_collision.STL b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/right/finger3_link1_collision.STL new file mode 100644 index 000000000..528724ca1 --- /dev/null +++ b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/right/finger3_link1_collision.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:698e2327acfe737e11fb10f2fb1d36bbf8f1bc6e688f1e31e6232bfc51127581 +size 25084 diff --git a/examples/g1_wuji_teleop/assets/wuji_hand/meshes/right/finger3_link2.STL b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/right/finger3_link2.STL new file mode 100644 index 000000000..c1f3f753f --- /dev/null +++ b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/right/finger3_link2.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:338db71e2c1ec63fba860d5d2af52bcb1daff72349b0e03a45f82ae63539bd43 +size 51984 diff --git a/examples/g1_wuji_teleop/assets/wuji_hand/meshes/right/finger3_link2_collision.STL b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/right/finger3_link2_collision.STL new file mode 100644 index 000000000..ac71baf46 --- /dev/null +++ b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/right/finger3_link2_collision.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:50e5f990401d1ede299adc2034c5c2c66f8b2499a8f7c38d9bbc4e26da06c3b9 +size 25084 diff --git a/examples/g1_wuji_teleop/assets/wuji_hand/meshes/right/finger3_link3.STL b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/right/finger3_link3.STL new file mode 100644 index 000000000..16cd3a6fb --- /dev/null +++ b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/right/finger3_link3.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3af337085d5f6430a471462c18b9132f1447eb8c0f3c68a8b1e69318ff8c3db8 +size 91284 diff --git a/examples/g1_wuji_teleop/assets/wuji_hand/meshes/right/finger3_link3_collision.STL b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/right/finger3_link3_collision.STL new file mode 100644 index 000000000..29fe2892c --- /dev/null +++ b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/right/finger3_link3_collision.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ddbd96f0b25ac9582a786bc6da7b363be61804e2b02fa5da4ca17c04f1715e5f +size 25084 diff --git a/examples/g1_wuji_teleop/assets/wuji_hand/meshes/right/finger3_link4.STL b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/right/finger3_link4.STL new file mode 100644 index 000000000..8d3580bfe --- /dev/null +++ b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/right/finger3_link4.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3b8efa21bafcae21119f243decf924dac51cc55181ed540e69ce6b85ecf03790 +size 54884 diff --git a/examples/g1_wuji_teleop/assets/wuji_hand/meshes/right/finger3_link4_collision.STL b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/right/finger3_link4_collision.STL new file mode 100644 index 000000000..b3a3fbc83 --- /dev/null +++ b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/right/finger3_link4_collision.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fafdff7b5e8b736bde927d6b62a7c953bd1f4bf8e77b442f5a7217ae0f2fa431 +size 25084 diff --git a/examples/g1_wuji_teleop/assets/wuji_hand/meshes/right/finger3_tip_link.STL b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/right/finger3_tip_link.STL new file mode 100644 index 000000000..3fef1c5f6 --- /dev/null +++ b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/right/finger3_tip_link.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7a99f65a6a473899913d4273e61319e3724fbf81ac8532969ed5c2ff1c1bde54 +size 112784 diff --git a/examples/g1_wuji_teleop/assets/wuji_hand/meshes/right/finger3_tip_link_collision.STL b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/right/finger3_tip_link_collision.STL new file mode 100644 index 000000000..40cca7b11 --- /dev/null +++ b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/right/finger3_tip_link_collision.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7e855b092faf2e2f0471109814ad9b2469485f88a47b6abd290b79b0f663ab2d +size 25084 diff --git a/examples/g1_wuji_teleop/assets/wuji_hand/meshes/right/finger4_link1.STL b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/right/finger4_link1.STL new file mode 100644 index 000000000..db6a65028 --- /dev/null +++ b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/right/finger4_link1.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:508321fefaeae1ae0567f89acd2492fac27cc468a2f562d895afcff192b0d00c +size 56184 diff --git a/examples/g1_wuji_teleop/assets/wuji_hand/meshes/right/finger4_link1_collision.STL b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/right/finger4_link1_collision.STL new file mode 100644 index 000000000..10c8ab9c5 --- /dev/null +++ b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/right/finger4_link1_collision.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc3b2c4d5b6304520c4a0297978ffaed3005d99cd8bdf74cc631deb6f5d9f14b +size 25084 diff --git a/examples/g1_wuji_teleop/assets/wuji_hand/meshes/right/finger4_link2.STL b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/right/finger4_link2.STL new file mode 100644 index 000000000..e2f06532f --- /dev/null +++ b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/right/finger4_link2.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e0b28769389c33af6af73c6dbf506be126570fa6ca7cbaf08f85a51005c48cde +size 51984 diff --git a/examples/g1_wuji_teleop/assets/wuji_hand/meshes/right/finger4_link2_collision.STL b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/right/finger4_link2_collision.STL new file mode 100644 index 000000000..6aea7ff97 --- /dev/null +++ b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/right/finger4_link2_collision.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a68648fc10cccdc58b2db7a4d09079ba2a9fa102a04e3c6a955e13e25edb2c30 +size 25084 diff --git a/examples/g1_wuji_teleop/assets/wuji_hand/meshes/right/finger4_link3.STL b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/right/finger4_link3.STL new file mode 100644 index 000000000..5f079cde3 --- /dev/null +++ b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/right/finger4_link3.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b01668bab452260954faf8cbff94fb59c55ccb3e490ff6eb4b73e81e550c787f +size 91284 diff --git a/examples/g1_wuji_teleop/assets/wuji_hand/meshes/right/finger4_link3_collision.STL b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/right/finger4_link3_collision.STL new file mode 100644 index 000000000..c42eacf98 --- /dev/null +++ b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/right/finger4_link3_collision.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8eb85ee78e3fd9e489a75e0385ea8493db0af0023286a005ac148cc26ff65b81 +size 25084 diff --git a/examples/g1_wuji_teleop/assets/wuji_hand/meshes/right/finger4_link4.STL b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/right/finger4_link4.STL new file mode 100644 index 000000000..a53f12a70 --- /dev/null +++ b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/right/finger4_link4.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d673bf02761f722728eb1ee0e7cc79ee6afba7693c78f623c3a4a89d5a905453 +size 54884 diff --git a/examples/g1_wuji_teleop/assets/wuji_hand/meshes/right/finger4_link4_collision.STL b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/right/finger4_link4_collision.STL new file mode 100644 index 000000000..bc7862bb4 --- /dev/null +++ b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/right/finger4_link4_collision.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:86783162be22f7600503a0a62484314367cf0d1adc2757a68681667e8f03205d +size 25084 diff --git a/examples/g1_wuji_teleop/assets/wuji_hand/meshes/right/finger4_tip_link.STL b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/right/finger4_tip_link.STL new file mode 100644 index 000000000..623387db6 --- /dev/null +++ b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/right/finger4_tip_link.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e8b1fbee8e019afe490e820b1cba5e02c5124bb4a09aa73896bd46db6586dfde +size 113484 diff --git a/examples/g1_wuji_teleop/assets/wuji_hand/meshes/right/finger4_tip_link_collision.STL b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/right/finger4_tip_link_collision.STL new file mode 100644 index 000000000..e41cb1416 --- /dev/null +++ b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/right/finger4_tip_link_collision.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29ed263d5e5cb24fdede93f6a8c695c36aaf5392f7c52acd4b13dee050a98bb9 +size 25084 diff --git a/examples/g1_wuji_teleop/assets/wuji_hand/meshes/right/finger5_link1.STL b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/right/finger5_link1.STL new file mode 100644 index 000000000..1356abb04 --- /dev/null +++ b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/right/finger5_link1.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:85ce22b8285a7d85e4c866151c972cde0736d4f618f35015b0b96fe39e367e49 +size 56184 diff --git a/examples/g1_wuji_teleop/assets/wuji_hand/meshes/right/finger5_link1_collision.STL b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/right/finger5_link1_collision.STL new file mode 100644 index 000000000..39b4b4a4f --- /dev/null +++ b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/right/finger5_link1_collision.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:011e4890f925b9d3454261f4fa98f0fb1b95c3aaeec707aa3db48a305fc42243 +size 25084 diff --git a/examples/g1_wuji_teleop/assets/wuji_hand/meshes/right/finger5_link2.STL b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/right/finger5_link2.STL new file mode 100644 index 000000000..594035022 --- /dev/null +++ b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/right/finger5_link2.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9245362396a184e63af8d2707dc85a3250f1d7db4eeb079d5b3ee9b432939dd6 +size 51984 diff --git a/examples/g1_wuji_teleop/assets/wuji_hand/meshes/right/finger5_link2_collision.STL b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/right/finger5_link2_collision.STL new file mode 100644 index 000000000..8a5064314 --- /dev/null +++ b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/right/finger5_link2_collision.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bf966538aa20a9ea1ba6e58c8bf1bf09aef3ca6826f349757077d3de84ed0119 +size 25084 diff --git a/examples/g1_wuji_teleop/assets/wuji_hand/meshes/right/finger5_link3.STL b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/right/finger5_link3.STL new file mode 100644 index 000000000..e87500b5a --- /dev/null +++ b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/right/finger5_link3.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1083dcec6df545d60c3f14af1113613ca89cc8964692dd624baa750491915f40 +size 91284 diff --git a/examples/g1_wuji_teleop/assets/wuji_hand/meshes/right/finger5_link3_collision.STL b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/right/finger5_link3_collision.STL new file mode 100644 index 000000000..76abe5831 --- /dev/null +++ b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/right/finger5_link3_collision.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:26b793372d5232de0b82dd2e74a041564f9bb22394705f2bae23366921cc56b0 +size 25084 diff --git a/examples/g1_wuji_teleop/assets/wuji_hand/meshes/right/finger5_link4.STL b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/right/finger5_link4.STL new file mode 100644 index 000000000..d8604709d --- /dev/null +++ b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/right/finger5_link4.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6342d4562a8bfaef740850c02dc5418ade11dcbe04736b957d93522995080f35 +size 54884 diff --git a/examples/g1_wuji_teleop/assets/wuji_hand/meshes/right/finger5_link4_collision.STL b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/right/finger5_link4_collision.STL new file mode 100644 index 000000000..8ba27a01e --- /dev/null +++ b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/right/finger5_link4_collision.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8643d9c3d6363aa7d6652e2137b4f496a1bd350dc39a0ddcc9cc8bfdc5f34093 +size 25084 diff --git a/examples/g1_wuji_teleop/assets/wuji_hand/meshes/right/finger5_tip_link.STL b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/right/finger5_tip_link.STL new file mode 100644 index 000000000..ccd435d94 --- /dev/null +++ b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/right/finger5_tip_link.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ebbf48ca7ba3bca5f329f95277ae392d0c1a09615eda23f1bf646c5112a38cee +size 112784 diff --git a/examples/g1_wuji_teleop/assets/wuji_hand/meshes/right/finger5_tip_link_collision.STL b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/right/finger5_tip_link_collision.STL new file mode 100644 index 000000000..5a4d45b41 --- /dev/null +++ b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/right/finger5_tip_link_collision.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4678873a3d1ca7c0a97f68cf87ebaa334484cf2d202da76b961e7c16a7a5968d +size 25084 diff --git a/examples/g1_wuji_teleop/assets/wuji_hand/meshes/right/palm_link.STL b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/right/palm_link.STL new file mode 100644 index 000000000..7a28ced65 --- /dev/null +++ b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/right/palm_link.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1ba4495e3b7adc707d50150a8625a0ed3731fa42ab6d40e62ae7437da103ce5e +size 456184 diff --git a/examples/g1_wuji_teleop/assets/wuji_hand/meshes/right/palm_link_collision.STL b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/right/palm_link_collision.STL new file mode 100644 index 000000000..e8fe80cab --- /dev/null +++ b/examples/g1_wuji_teleop/assets/wuji_hand/meshes/right/palm_link_collision.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:542614615dc46aee7df9cad628374a3aea4c07f58e6f2c716d2b4cbebe665139 +size 25084 diff --git a/examples/g1_wuji_teleop/assets/wuji_hand/urdf/left.urdf b/examples/g1_wuji_teleop/assets/wuji_hand/urdf/left.urdf new file mode 100644 index 000000000..560769b35 --- /dev/null +++ b/examples/g1_wuji_teleop/assets/wuji_hand/urdf/left.urdf @@ -0,0 +1,1472 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/examples/g1_wuji_teleop/assets/wuji_hand/urdf/right.urdf b/examples/g1_wuji_teleop/assets/wuji_hand/urdf/right.urdf new file mode 100644 index 000000000..a29f29f10 --- /dev/null +++ b/examples/g1_wuji_teleop/assets/wuji_hand/urdf/right.urdf @@ -0,0 +1,1472 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/examples/g1_wuji_teleop/config/avp_manus.yml b/examples/g1_wuji_teleop/config/avp_manus.yml new file mode 100644 index 000000000..681ca911d --- /dev/null +++ b/examples/g1_wuji_teleop/config/avp_manus.yml @@ -0,0 +1,45 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +session: + app_name: AvpManusTeleopMain # TeleopSession/OpenXR session name used for runtime identification + +robot: + # This demo only keeps the `g1_wuji` variant. + variant: g1_wuji # Selects the robot variant to load and therefore the hand joint set, default USD, and retargeting configs + initial_world_position: [0.0, 0.0, 0.8] # Initial robot root world position [x, y, z] in meters + initial_world_orientation_xyzw: [0.0, 0.0, 0.707, 0.707] # Initial robot root orientation quaternion [x, y, z, w] + debug_visualization: false # Enables debug markers, axes, and auxiliary visualization + + avp_frame_binding: + enabled: true # Enables calibration and binding from the AVP head/hand frame into the robot frame + calibration_window_s: 3.0 # Averaging window in seconds when starting or recalibrating + robot_reference_body: torso_link # Body/link name used as the robot reference frame + robot_reference_offset_pos: [0.0077774, 0.0000210, 0.3836842] # Position offset from the reference body to the desired head reference point, in meters + robot_reference_offset_quat_xyzw: [0.0, 0.0, 0.0, 1.0] # Orientation offset from the reference body to the desired head reference point + head_forward_axis_isaac: [0.0, 1.0, 0.0] # Unit axis in Isaac coordinates that corresponds to the headset forward direction + whole_body_yaw_following: true # Lets the robot base / whole body follow the headset around the z axis + whole_body_yaw_deadband_rad: 0.03 # Minimum yaw change, in radians, before whole-body yaw following is applied + whole_body_yaw_smoothing_alpha: 0.2 # Smoothing factor for whole-body yaw following; larger values respond faster + + head_view_camera: + enabled: true # Creates a simulated first-person camera attached to the robot head + activate_on_calibration: true # Automatically switches the Kit viewport to this camera after pressing B and completing AVP head/hand calibration + prim_path: /World/TeleopTargets/HeadViewCamera # USD prim path of the head camera + translation_offset_m: [0.05, 0.0, 0.3836842] # Camera position offset [x, y, z] relative to the head-link local frame, in meters + orientation_offset_rpy_deg: [70.0, 0.0, -90.0] # Camera orientation compensation [roll, pitch, yaw] relative to the head-link local frame, in degrees; adjust this first if the head view faces the wrong way + horizontal_aperture_mm: 20.955 # USD camera horizontal sensor size, used to determine field of view + focal_length_mm: 5.0 # USD camera focal length, combined with aperture to control the initial first-person FOV + clipping_range_m: [0.01, 1000.0] # Camera near/far clipping range [near, far] in meters + + head_view_xr_display: + enabled: false # Sends the head camera image directly into the AVP/XR view; when enabled it reuses the same OpenXR session + +avp: + ee_pose_source: wrist # Keeps the original path by default; switch this to `controller` when using PICO controller EE poses + ee_hand_joint: wrist # Only used when ee_pose_source=hand_tracking / wrist / palm; `wrist` is the closest match to the original AVP path + controller_pose_source: aim # Only used when ee_pose_source=controller; prefer `aim` first for PICO controller mode, then try `grip` if needed + controller_fallback_enabled: true # Only meaningful for the hand-tracking path; in PICO controller mode it is usually better to set this to false + +retargeting: + # This example is fixed to the `official_wuji` path and the built-in hand-frame transform. diff --git a/examples/g1_wuji_teleop/config/local/g1_wuji/hand_left_config.yml b/examples/g1_wuji_teleop/config/local/g1_wuji/hand_left_config.yml new file mode 100644 index 000000000..cf8e75551 --- /dev/null +++ b/examples/g1_wuji_teleop/config/local/g1_wuji/hand_left_config.yml @@ -0,0 +1,36 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +retargeting: + finger_tip_link_names: + - finger1_tip_link + - finger2_tip_link + - finger3_tip_link + - finger4_tip_link + - finger5_tip_link + low_pass_alpha: 0.2 + scaling_factor: 1.0 + target_joint_names: + - finger1_joint1 + - finger1_joint2 + - finger1_joint3 + - finger1_joint4 + - finger2_joint1 + - finger2_joint2 + - finger2_joint3 + - finger2_joint4 + - finger3_joint1 + - finger3_joint2 + - finger3_joint3 + - finger3_joint4 + - finger4_joint1 + - finger4_joint2 + - finger4_joint3 + - finger4_joint4 + - finger5_joint1 + - finger5_joint2 + - finger5_joint3 + - finger5_joint4 + type: DexPilot + urdf_path: placeholder_overridden_at_runtime + wrist_link_name: palm_link diff --git a/examples/g1_wuji_teleop/config/local/g1_wuji/hand_right_config.yml b/examples/g1_wuji_teleop/config/local/g1_wuji/hand_right_config.yml new file mode 100644 index 000000000..cf8e75551 --- /dev/null +++ b/examples/g1_wuji_teleop/config/local/g1_wuji/hand_right_config.yml @@ -0,0 +1,36 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +retargeting: + finger_tip_link_names: + - finger1_tip_link + - finger2_tip_link + - finger3_tip_link + - finger4_tip_link + - finger5_tip_link + low_pass_alpha: 0.2 + scaling_factor: 1.0 + target_joint_names: + - finger1_joint1 + - finger1_joint2 + - finger1_joint3 + - finger1_joint4 + - finger2_joint1 + - finger2_joint2 + - finger2_joint3 + - finger2_joint4 + - finger3_joint1 + - finger3_joint2 + - finger3_joint3 + - finger3_joint4 + - finger4_joint1 + - finger4_joint2 + - finger4_joint3 + - finger4_joint4 + - finger5_joint1 + - finger5_joint2 + - finger5_joint3 + - finger5_joint4 + type: DexPilot + urdf_path: placeholder_overridden_at_runtime + wrist_link_name: palm_link diff --git a/examples/g1_wuji_teleop/config/official/g1_wuji/manus_wuji_left.yaml b/examples/g1_wuji_teleop/config/official/g1_wuji/manus_wuji_left.yaml new file mode 100644 index 000000000..62b6ae87a --- /dev/null +++ b/examples/g1_wuji_teleop/config/official/g1_wuji/manus_wuji_left.yaml @@ -0,0 +1,39 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# G1-Wuji MANUS retargeting config backed by the official wuji-retargeting package. + +optimizer: + type: "AdaptiveOptimizerAnalytical" + urdf_path: "../../../assets/wuji_hand/urdf/left.urdf" + +retarget: + huber_delta: 2.0 + huber_delta_dir: 0.5 + norm_delta: 0.04 + + w_pos: 1.0 + w_dir: 2.0 + scaling: 1.0 + project_tip_dir: false + + w_full_hand: 1.0 + segment_scaling: + thumb: [0.98, 1.0, 1.0] + index: [1.1, 0.989, 1.03] + middle: [1.10, 0.96, 0.96] + ring: [1.1, 0.97, 0.99] + pinky: [1.105, 1.15, 1.15] + + pinch_thresholds: + index: { d1: 2.0, d2: 4.0 } + middle: { d1: 2.0, d2: 4.0 } + ring: { d1: 2.0, d2: 4.0 } + pinky: { d1: 2.0, d2: 4.0 } + + mediapipe_rotation: + x: 0.0 + y: 0.0 + z: 15.0 + + lp_alpha: 0.2 diff --git a/examples/g1_wuji_teleop/config/official/g1_wuji/manus_wuji_right.yaml b/examples/g1_wuji_teleop/config/official/g1_wuji/manus_wuji_right.yaml new file mode 100644 index 000000000..665ce6b07 --- /dev/null +++ b/examples/g1_wuji_teleop/config/official/g1_wuji/manus_wuji_right.yaml @@ -0,0 +1,39 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# G1-Wuji MANUS retargeting config backed by the official wuji-retargeting package. + +optimizer: + type: "AdaptiveOptimizerAnalytical" + urdf_path: "../../../assets/wuji_hand/urdf/right.urdf" + +retarget: + huber_delta: 2.0 + huber_delta_dir: 0.5 + norm_delta: 0.04 + + w_pos: 1.0 + w_dir: 2.0 + scaling: 1.0 + project_tip_dir: false + + w_full_hand: 1.0 + segment_scaling: + thumb: [0.98, 1.0, 1.0] + index: [1.1, 0.989, 1.03] + middle: [1.10, 0.96, 0.96] + ring: [1.1, 0.97, 0.99] + pinky: [1.105, 1.15, 1.15] + + pinch_thresholds: + index: { d1: 2.0, d2: 4.0 } + middle: { d1: 2.0, d2: 4.0 } + ring: { d1: 2.0, d2: 4.0 } + pinky: { d1: 2.0, d2: 4.0 } + + mediapipe_rotation: + x: 0.0 + y: 0.0 + z: -15.0 + + lp_alpha: 0.2 diff --git a/examples/g1_wuji_teleop/python/g1_wuji_teleop/__init__.py b/examples/g1_wuji_teleop/python/g1_wuji_teleop/__init__.py new file mode 100644 index 000000000..eabb40f14 --- /dev/null +++ b/examples/g1_wuji_teleop/python/g1_wuji_teleop/__init__.py @@ -0,0 +1,8 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""G1-Wuji AVP + MANUS teleoperation application package.""" + +from .session import main + +__all__ = ["main"] diff --git a/examples/g1_wuji_teleop/python/g1_wuji_teleop/cli.py b/examples/g1_wuji_teleop/python/g1_wuji_teleop/cli.py new file mode 100644 index 000000000..613ce811b --- /dev/null +++ b/examples/g1_wuji_teleop/python/g1_wuji_teleop/cli.py @@ -0,0 +1,19 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Thin CLI entrypoints for the G1-Wuji teleop app.""" + +from __future__ import annotations + +from collections.abc import Sequence + +from .session import main as session_main +from .viz.teleop_visualizer import main as visualizer_main + + +def teleop_main(argv: Sequence[str] | None = None) -> int: + return session_main(argv) + + +def teleop_visualizer_main(argv: Sequence[str] | None = None) -> int: + return visualizer_main(argv) diff --git a/examples/g1_wuji_teleop/python/g1_wuji_teleop/devices/avp_manus_stream.py b/examples/g1_wuji_teleop/python/g1_wuji_teleop/devices/avp_manus_stream.py new file mode 100644 index 000000000..5521dda5f --- /dev/null +++ b/examples/g1_wuji_teleop/python/g1_wuji_teleop/devices/avp_manus_stream.py @@ -0,0 +1,894 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Non-ROS AVP + MANUS teleop entry point for simulation loops. + +The public surface for simulators is ``TeleopMain``: + + with TeleopMain.from_config_file("config/avp_manus.yml") as teleop: + sample = teleop.step() + left_ee = sample["ee_poses"]["left"] + left_skeleton = sample["hand_skeletons"]["left"] + left_joints = sample["hand_joint_positions"]["left"] + +AVP endpoint poses can arrive as OpenXR hand tracking wrist/palm poses or as +controller aim/grip poses. MANUS glove data is expected to arrive as OpenXR hand +tracking, either from an already-running ``manus_hand_plugin`` or from the plugin +managed by ``TeleopSession``. +""" + +from __future__ import annotations + +import argparse +import sys +import time +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +from ..paths import default_config_path, ensure_import_paths, repo_root + + +SIDE_NAMES = ("left", "right") + +# Keep the non-ROS G1-Wuji DexPilot path aligned with the official teleop_ros2 +# hand-retargeting basis. Using identity here causes OpenXR hand frames to enter +# dex-retargeting in the wrong basis, which looks like joint-level tracking +# errors even when the output names/count match the robot hand. +FIXED_G1_WUJI_RETARGET_TRANSFORM = ( + 0.0, + -1.0, + 0.0, + -1.0, + 0.0, + 0.0, + 0.0, + 0.0, + -1.0, +) + +FIXED_RETARGET_METHOD = "official_wuji" + +FIXED_G1_WUJI_RETARGET_DEFAULTS = { + "left": { + "config": Path("local/g1_wuji/hand_left_config.yml"), + "urdf": Path("../assets/wuji_hand/urdf/left.urdf"), + "parameter_config_path": "/tmp/avp_manus_left_g1_wuji_dex_params.json", + "joint_names": [ + "left_finger1_joint1", + "left_finger1_joint2", + "left_finger1_joint3", + "left_finger1_joint4", + "left_finger2_joint1", + "left_finger2_joint2", + "left_finger2_joint3", + "left_finger2_joint4", + "left_finger3_joint1", + "left_finger3_joint2", + "left_finger3_joint3", + "left_finger3_joint4", + "left_finger4_joint1", + "left_finger4_joint2", + "left_finger4_joint3", + "left_finger4_joint4", + "left_finger5_joint1", + "left_finger5_joint2", + "left_finger5_joint3", + "left_finger5_joint4", + ], + }, + "right": { + "config": Path("local/g1_wuji/hand_right_config.yml"), + "urdf": Path("../assets/wuji_hand/urdf/right.urdf"), + "parameter_config_path": "/tmp/avp_manus_right_g1_wuji_dex_params.json", + "joint_names": [ + "right_finger1_joint1", + "right_finger1_joint2", + "right_finger1_joint3", + "right_finger1_joint4", + "right_finger2_joint1", + "right_finger2_joint2", + "right_finger2_joint3", + "right_finger2_joint4", + "right_finger3_joint1", + "right_finger3_joint2", + "right_finger3_joint3", + "right_finger3_joint4", + "right_finger4_joint1", + "right_finger4_joint2", + "right_finger4_joint3", + "right_finger4_joint4", + "right_finger5_joint1", + "right_finger5_joint2", + "right_finger5_joint3", + "right_finger5_joint4", + ], + }, +} + +FIXED_NATIVE_RETARGET_DEFAULTS_BY_VARIANT = {"g1_wuji": FIXED_G1_WUJI_RETARGET_DEFAULTS} + + +@dataclass(frozen=True) +class RuntimeDeps: + DeviceIOSession: Any + ControllersSource: Any + DexHandRetargeter: Any + DexHandRetargeterConfig: Any + HeadSource: Any + HandsSource: Any + OpenXRSessionHandles: Any + OutputCombiner: Any + PluginConfig: Any + TeleopSession: Any + TeleopSessionConfig: Any + ControllerInputIndex: Any + HandInputIndex: Any + HandJointIndex: Any + HeadPoseIndex: Any + + +def _default_config_path() -> Path: + return default_config_path() + + +def _repo_root() -> Path: + return repo_root() + + +def _load_runtime_deps() -> RuntimeDeps: + try: + import isaacteleop.deviceio as deviceio + from isaacteleop.retargeting_engine.deviceio_source_nodes import ( + ControllersSource, + HeadSource, + HandsSource, + ) + from isaacteleop.retargeting_engine.interface import OutputCombiner + from isaacteleop.retargeting_engine.tensor_types.indices import ( + ControllerInputIndex, + HandInputIndex, + HandJointIndex, + HeadPoseIndex, + ) + from isaacteleop.oxr import OpenXRSessionHandles + from isaacteleop.retargeters import DexHandRetargeter, DexHandRetargeterConfig + from isaacteleop.teleop_session_manager import ( + PluginConfig, + TeleopSession, + TeleopSessionConfig, + ) + except ModuleNotFoundError as exc: + raise ModuleNotFoundError( + f"Missing Python dependency {exc.name!r}. Activate an IsaacTeleop " + "environment with retargeter dependencies before running the G1-Wuji " + "teleop AVP/MANUS stream." + ) from exc + + return RuntimeDeps( + DeviceIOSession=deviceio.DeviceIOSession, + ControllersSource=ControllersSource, + DexHandRetargeter=DexHandRetargeter, + DexHandRetargeterConfig=DexHandRetargeterConfig, + HeadSource=HeadSource, + HandsSource=HandsSource, + OpenXRSessionHandles=OpenXRSessionHandles, + OutputCombiner=OutputCombiner, + PluginConfig=PluginConfig, + TeleopSession=TeleopSession, + TeleopSessionConfig=TeleopSessionConfig, + ControllerInputIndex=ControllerInputIndex, + HandInputIndex=HandInputIndex, + HandJointIndex=HandJointIndex, + HeadPoseIndex=HeadPoseIndex, + ) + + +def _load_yaml(path: Path) -> dict[str, Any]: + try: + import yaml + except ModuleNotFoundError as exc: + raise ModuleNotFoundError( + "PyYAML is required to load G1-Wuji teleop config files." + ) from exc + + with path.open("r", encoding="utf-8") as stream: + data = yaml.safe_load(stream) or {} + if not isinstance(data, dict): + raise ValueError(f"Config root must be a mapping: {path}") + return data + + +def _as_mapping(value: Any, name: str) -> Mapping[str, Any]: + if value is None: + return {} + if not isinstance(value, Mapping): + raise ValueError(f"Config field {name!r} must be a mapping") + return value + + +def _as_sequence(value: Any, name: str) -> Sequence[Any]: + if value is None: + return [] + if isinstance(value, (str, bytes)) or not isinstance(value, Sequence): + raise ValueError(f"Config field {name!r} must be a sequence") + return value + + +def _robot_variant_from_config(config: Mapping[str, Any]) -> str: + robot_cfg = _as_mapping(config.get("robot"), "robot") + raw = str(robot_cfg.get("variant", "g1_wuji")).strip().lower() + aliases = { + "g1wuji": "g1_wuji", + "g1-wuji": "g1_wuji", + "wuji": "g1_wuji", + } + normalized = aliases.get(raw, raw) + if normalized not in FIXED_NATIVE_RETARGET_DEFAULTS_BY_VARIANT: + raise ValueError( + f"Unsupported robot.variant={raw!r}. Expected one of " + f"{sorted(FIXED_NATIVE_RETARGET_DEFAULTS_BY_VARIANT)}" + ) + return normalized + + +def _retarget_method_from_config(config: Mapping[str, Any]) -> str: + retargeting_cfg = _as_mapping(config.get("retargeting"), "retargeting") + raw = str(retargeting_cfg.get("method", FIXED_RETARGET_METHOD)).strip().lower() + aliases = { + "native": "native_dex", + "native_dex": "native_dex", + "dex": "native_dex", + "dexhand": "native_dex", + "dex_hand": "native_dex", + "official": "official_wuji", + "official_wuji": "official_wuji", + "wuji": "official_wuji", + } + normalized = aliases.get(raw, raw) + if normalized not in {"native_dex", "official_wuji"}: + raise ValueError( + "Config field 'retargeting.method' must be one of " + "['native_dex', 'official_wuji']" + ) + return normalized + + +def _use_native_hand_targets_from_config(config: Mapping[str, Any]) -> bool: + return _retarget_method_from_config(config) == "native_dex" + + +def _resolve_path(path_value: str | None, *, base_dir: Path, default: Path) -> Path: + path = Path(path_value).expanduser() if path_value else default + if not path.is_absolute(): + path = base_dir / path + return path.resolve() + + +def _require_file(path: Path, label: str) -> Path: + if not path.is_file(): + raise FileNotFoundError(f"{label} not found: {path}") + return path + + +def _default_plugin_search_paths() -> list[Path]: + root = _repo_root() + return [ + root / "install" / "plugins", + Path("/opt/isaacteleop/install/plugins"), + ] + + +def _tensor_values(group: Any) -> list[float]: + return [float(group[index]) for index in range(len(group))] + + +def _controller_pose( + group: Any, *, pose_source: str, deps: RuntimeDeps +) -> dict[str, Any] | None: + if group.is_none: + return None + + if pose_source == "aim": + position_index = deps.ControllerInputIndex.AIM_POSITION + orientation_index = deps.ControllerInputIndex.AIM_ORIENTATION + valid_index = deps.ControllerInputIndex.AIM_IS_VALID + elif pose_source == "grip": + position_index = deps.ControllerInputIndex.GRIP_POSITION + orientation_index = deps.ControllerInputIndex.GRIP_ORIENTATION + valid_index = deps.ControllerInputIndex.GRIP_IS_VALID + else: + raise ValueError(f"ee_pose_source must be 'aim' or 'grip', got {pose_source!r}") + + if not bool(group[valid_index]): + return None + + return { + "position": [float(value) for value in group[position_index]], + "orientation_xyzw": [float(value) for value in group[orientation_index]], + } + + +def _hand_tracking_pose( + group: Any, *, joint_name: str, deps: RuntimeDeps +) -> dict[str, Any] | None: + if group.is_none: + return None + + import numpy as np + + joint_index = int(getattr(deps.HandJointIndex, joint_name.upper())) + valid = np.from_dlpack(group[deps.HandInputIndex.JOINT_VALID]) + if not bool(valid[joint_index]): + return None + + positions = np.from_dlpack(group[deps.HandInputIndex.JOINT_POSITIONS]) + orientations = np.from_dlpack(group[deps.HandInputIndex.JOINT_ORIENTATIONS]) + return { + "position": [float(value) for value in positions[joint_index]], + "orientation_xyzw": [float(value) for value in orientations[joint_index]], + } + + +def _head_pose(group: Any, *, deps: RuntimeDeps) -> dict[str, Any] | None: + if group.is_none or not bool(group[deps.HeadPoseIndex.IS_VALID]): + return None + + return { + "position": [float(value) for value in group[deps.HeadPoseIndex.POSITION]], + "orientation_xyzw": [ + float(value) for value in group[deps.HeadPoseIndex.ORIENTATION] + ], + } + + +def _controller_inputs(group: Any, *, deps: RuntimeDeps) -> dict[str, Any] | None: + if group.is_none: + return None + return { + "primary_click": bool(group[deps.ControllerInputIndex.PRIMARY_CLICK] > 0.5), + "secondary_click": bool(group[deps.ControllerInputIndex.SECONDARY_CLICK] > 0.5), + "thumbstick_click": bool( + group[deps.ControllerInputIndex.THUMBSTICK_CLICK] > 0.5 + ), + "menu_click": bool(group[deps.ControllerInputIndex.MENU_CLICK] > 0.5), + "thumbstick_x": float(group[deps.ControllerInputIndex.THUMBSTICK_X]), + "thumbstick_y": float(group[deps.ControllerInputIndex.THUMBSTICK_Y]), + "squeeze_value": float(group[deps.ControllerInputIndex.SQUEEZE_VALUE]), + "trigger_value": float(group[deps.ControllerInputIndex.TRIGGER_VALUE]), + } + + +def _openxr_hand_joint_names(deps: RuntimeDeps) -> list[str]: + return [ + name.lower() + for name, _index in sorted( + deps.HandJointIndex.__members__.items(), + key=lambda item: int(item[1]), + ) + ] + + +def _hand_tracking_skeleton(group: Any, *, deps: RuntimeDeps) -> dict[str, Any] | None: + if group.is_none: + return None + + import numpy as np + + positions = np.from_dlpack(group[deps.HandInputIndex.JOINT_POSITIONS]).copy() + valid = np.from_dlpack(group[deps.HandInputIndex.JOINT_VALID]).astype(bool).copy() + if positions.ndim != 2 or positions.shape[1] != 3: + return None + if valid.ndim != 1 or positions.shape[0] != valid.shape[0]: + return None + + return { + "joint_names": _openxr_hand_joint_names(deps), + "positions": [[float(coord) for coord in position] for position in positions], + "valid": [bool(value) for value in valid], + } + + +class TeleopMain: + """Small non-ROS facade returning AVP EE poses and MANUS retargeted joints.""" + + def __init__( + self, + config: Mapping[str, Any], + *, + config_dir: Path, + oxr_handles: Any | None = None, + ) -> None: + ensure_import_paths() + self._config = config + self._config_dir = config_dir + self._deps = _load_runtime_deps() + self._use_native_hand_targets = _use_native_hand_targets_from_config(config) + self._oxr_handles = oxr_handles + self._session = None + self._session_context = None + ( + self._ee_pose_source, + self._controller_pose_source, + self._ee_hand_joint, + self._controller_fallback_enabled, + ) = self._load_ee_pose_config() + self._manus_plugin_required = self._load_manus_plugin_required() + self._manus_plugin_configs = self._build_plugin_configs() + self._manus_plugin_disabled_for_session = False + self._ee_source_announced: set[tuple[str, str]] = set() + self._hand_joint_names: dict[str, list[str]] = {} + self._required_xr_extensions: tuple[str, ...] = () + self._session_config_includes_plugins = True + self._session_config = self._build_session_config(include_plugins=True) + + @classmethod + def from_config_file( + cls, + path: str | Path | None = None, + *, + oxr_handles: Any | None = None, + ) -> "TeleopMain": + config_path = Path(path).expanduser() if path else _default_config_path() + config_path = config_path.resolve() + config = _load_yaml(config_path) + return cls(config, config_dir=config_path.parent, oxr_handles=oxr_handles) + + @property + def hand_joint_names(self) -> dict[str, list[str]]: + return {side: list(names) for side, names in self._hand_joint_names.items()} + + @property + def required_xr_extensions(self) -> tuple[str, ...]: + return tuple(self._required_xr_extensions) + + def set_oxr_handles(self, oxr_handles: Any | None) -> None: + if self._session is not None or self._session_context is not None: + raise RuntimeError( + "TeleopMain.set_oxr_handles() must be called before starting the session" + ) + self._oxr_handles = oxr_handles + self._session_config_includes_plugins = True + self._session_config = self._build_session_config(include_plugins=True) + + def __enter__(self) -> "TeleopMain": + try: + self._start_session(include_plugins=True) + except Exception as exc: + can_restart = self._can_restart_without_manus_plugin(startup=True) + self._close_session() + if not can_restart: + raise + self._disable_optional_manus_plugin(exc, phase="startup") + self._start_session(include_plugins=False) + return self + + def __exit__(self, exc_type, exc_value, traceback) -> None: + if self._session_context is not None: + self._session_context.__exit__(exc_type, exc_value, traceback) + self._session_context = None + self._session = None + + def _start_session(self, *, include_plugins: bool) -> None: + if include_plugins != self._session_config_includes_plugins: + self._session_config = self._build_session_config( + include_plugins=include_plugins + ) + self._session_config_includes_plugins = include_plugins + self._session_context = self._deps.TeleopSession(self._session_config) + self._session = self._session_context.__enter__() + + def _close_session(self) -> None: + session_context = self._session_context + if session_context is not None: + try: + session_context.__exit__(None, None, None) + finally: + exit_stack = getattr(session_context, "_exit_stack", None) + if exit_stack is not None: + exit_stack.close() + self._session_context = None + self._session = None + + def _can_restart_without_manus_plugin(self, *, startup: bool) -> bool: + if ( + self._manus_plugin_required + or self._manus_plugin_disabled_for_session + or not self._manus_plugin_configs + ): + return False + if startup: + return bool( + getattr(self._session_context, "plugin_managers", ()) + or getattr(self._session_context, "plugin_contexts", ()) + ) + if self._session is None: + return False + return bool(getattr(self._session, "plugin_contexts", ())) + + def _disable_optional_manus_plugin(self, exc: Exception, *, phase: str) -> None: + self._manus_plugin_disabled_for_session = True + print( + "[TeleopMain] optional MANUS plugin failed during " + f"{phase}; restarting without it so AVP/controller EE teleop can continue " + f"({type(exc).__name__}: {exc}).", + flush=True, + ) + + def step(self) -> dict[str, Any]: + if self._session is None: + raise RuntimeError( + "TeleopMain must be used as a context manager before step()" + ) + + try: + result = self._session.step() + except Exception as exc: + if not self._can_restart_without_manus_plugin(startup=False): + raise + self._close_session() + self._disable_optional_manus_plugin(exc, phase="runtime") + self._start_session(include_plugins=False) + result = self._session.step() + left_hand = result["openxr_hand_left"] + right_hand = result["openxr_hand_right"] + hand_active = { + "left": not left_hand.is_none, + "right": not right_hand.is_none, + } + hand_joint_positions = {"left": None, "right": None} + if self._use_native_hand_targets: + if hand_active["left"] and "left_hand_joints" in result: + hand_joint_positions["left"] = _tensor_values( + result["left_hand_joints"] + ) + if hand_active["right"] and "right_hand_joints" in result: + hand_joint_positions["right"] = _tensor_values( + result["right_hand_joints"] + ) + + return { + "timestamp_s": time.time(), + "frame_count": self._session.frame_count, + "head_pose": _head_pose(result["avp_head"], deps=self._deps), + "ee_poses": { + "left": self._ee_pose(result, "left"), + "right": self._ee_pose(result, "right"), + }, + "hand_active": hand_active, + "hand_skeletons": { + "left": _hand_tracking_skeleton(left_hand, deps=self._deps), + "right": _hand_tracking_skeleton(right_hand, deps=self._deps), + }, + "hand_joint_names": self.hand_joint_names, + "hand_joint_positions": hand_joint_positions, + "controller_inputs": { + "left": ( + _controller_inputs(result["avp_controller_left"], deps=self._deps) + if "avp_controller_left" in result + else None + ), + "right": ( + _controller_inputs(result["avp_controller_right"], deps=self._deps) + if "avp_controller_right" in result + else None + ), + }, + } + + def _build_session_config(self, *, include_plugins: bool): + deps = self._deps + head = deps.HeadSource(name="avp_head") + hands = deps.HandsSource(name="manus_hands") + trackers = [head.get_tracker(), hands.get_tracker()] + + output_mapping = { + "avp_head": head.output("head"), + "openxr_hand_left": hands.output(deps.HandsSource.LEFT), + "openxr_hand_right": hands.output(deps.HandsSource.RIGHT), + } + if self._use_native_hand_targets: + left_retargeter, right_retargeter = self._build_hand_retargeters() + left_connected = left_retargeter.connect( + {deps.HandsSource.LEFT: hands.output(deps.HandsSource.LEFT)} + ) + right_connected = right_retargeter.connect( + {deps.HandsSource.RIGHT: hands.output(deps.HandsSource.RIGHT)} + ) + + left_joint_type = left_connected.output_types()["hand_joints"] + right_joint_type = right_connected.output_types()["hand_joints"] + self._hand_joint_names = { + "left": [tensor_type.name for tensor_type in left_joint_type.types], + "right": [tensor_type.name for tensor_type in right_joint_type.types], + } + output_mapping.update( + { + "left_hand_joints": left_connected.output("hand_joints"), + "right_hand_joints": right_connected.output("hand_joints"), + } + ) + else: + # The official Wuji route consumes MANUS/OpenXR skeletons directly in session.py. + # Do not require native Dex tensors when retargeting.method selects that path. + self._hand_joint_names = {side: [] for side in SIDE_NAMES} + + if self._ee_pose_source == "controller" or self._controller_fallback_enabled: + controllers = deps.ControllersSource(name="avp_controllers") + trackers.append(controllers.get_tracker()) + output_mapping.update( + { + "avp_controller_left": controllers.output( + deps.ControllersSource.LEFT + ), + "avp_controller_right": controllers.output( + deps.ControllersSource.RIGHT + ), + } + ) + + pipeline = deps.OutputCombiner(output_mapping) + self._required_xr_extensions = tuple( + deps.DeviceIOSession.get_required_extensions(trackers) + ) + + session_cfg = _as_mapping(self._config.get("session"), "session") + app_name = str(session_cfg.get("app_name", "AvpManusTeleopMain")) + return deps.TeleopSessionConfig( + app_name=app_name, + pipeline=pipeline, + plugins=self._manus_plugin_configs if include_plugins else [], + oxr_handles=self._oxr_handles, + ) + + def _build_hand_retargeters(self) -> tuple[Any, Any]: + deps = self._deps + retargeting_cfg = _as_mapping(self._config.get("retargeting"), "retargeting") + robot_variant = _robot_variant_from_config(self._config) + variant_defaults = FIXED_NATIVE_RETARGET_DEFAULTS_BY_VARIANT[robot_variant] + transform = tuple( + float(value) + for value in _as_sequence( + retargeting_cfg.get( + "handtracking_to_baselink_frame_transform", + FIXED_G1_WUJI_RETARGET_TRANSFORM, + ), + "retargeting.handtracking_to_baselink_frame_transform", + ) + or FIXED_G1_WUJI_RETARGET_TRANSFORM + ) + if len(transform) != 9: + raise ValueError( + "retargeting.handtracking_to_baselink_frame_transform must have 9 values" + ) + + side_retargeters = [] + for side in SIDE_NAMES: + side_cfg = _as_mapping(retargeting_cfg.get(side), f"retargeting.{side}") + fixed_defaults = variant_defaults[side] + config_path = _require_file( + _resolve_path( + side_cfg.get("config"), + base_dir=self._config_dir, + default=fixed_defaults["config"], + ), + f"{side} hand retargeting config", + ) + urdf_path = _require_file( + _resolve_path( + side_cfg.get("urdf"), + base_dir=self._config_dir, + default=fixed_defaults["urdf"], + ), + f"{side} hand URDF", + ) + joint_names = side_cfg.get("joint_names") + if joint_names is not None: + joint_names = [ + str(name) + for name in _as_sequence( + joint_names, f"retargeting.{side}.joint_names" + ) + ] + elif "joint_names" in fixed_defaults: + joint_names = [str(name) for name in fixed_defaults["joint_names"]] + if joint_names is not None and robot_variant == "g1_wuji": + # DexHandRetargeter populates output tensors by matching requested output + # names against the URDF DOF names reported by the optimizer. Wuji's hand + # URDF uses unprefixed names like finger1_joint1, so strip side prefixes + # here and let session.py re-add them later when ordering robot targets. + side_prefix = f"{side}_" + joint_names = [ + name[len(side_prefix) :] if name.startswith(side_prefix) else name + for name in joint_names + ] + + retargeter_config = deps.DexHandRetargeterConfig( + hand_retargeting_config=str(config_path), + hand_urdf=str(urdf_path), + hand_joint_names=joint_names, + handtracking_to_baselink_frame_transform=transform, + hand_side=side, + parameter_config_path=side_cfg.get( + "parameter_config_path", + fixed_defaults["parameter_config_path"], + ), + ) + retargeter = deps.DexHandRetargeter( + retargeter_config, name=f"{side}_manus_dex" + ) + output_joint_names = list( + retargeter.output_types()["hand_joints"].types # type: ignore[index] + ) + print( + f"[Manus][DexPilot] variant={robot_variant} side={side} " + f"config={config_path} urdf={urdf_path} " + f"transform={transform} " + f"joint_count={len(output_joint_names)} " + f"first_joints={[tensor_type.name for tensor_type in output_joint_names[:5]]}", + flush=True, + ) + side_retargeters.append(retargeter) + + return side_retargeters[0], side_retargeters[1] + + def _build_plugin_configs(self) -> list[Any]: + plugin_cfg = _as_mapping(self._config.get("manus_plugin"), "manus_plugin") + if not bool(plugin_cfg.get("enabled", True)): + return [] + + search_paths_cfg = _as_sequence( + plugin_cfg.get("search_paths"), "manus_plugin.search_paths" + ) + if search_paths_cfg: + search_paths = [ + _resolve_path(str(path), base_dir=self._config_dir, default=Path()) + for path in search_paths_cfg + ] + else: + search_paths = _default_plugin_search_paths() + + return [ + self._deps.PluginConfig( + plugin_name=str(plugin_cfg.get("plugin_name", "manus_hand_plugin")), + plugin_root_id=str(plugin_cfg.get("plugin_root_id", "manus")), + search_paths=search_paths, + enabled=True, + plugin_args=[ + str(arg) + for arg in _as_sequence(plugin_cfg.get("args"), "manus_plugin.args") + ], + ) + ] + + def _load_manus_plugin_required(self) -> bool: + plugin_cfg = _as_mapping(self._config.get("manus_plugin"), "manus_plugin") + return bool(plugin_cfg.get("required", False)) + + def _load_ee_pose_config(self) -> tuple[str, str, str, bool]: + avp_cfg = _as_mapping(self._config.get("avp"), "avp") + pose_source = str(avp_cfg.get("ee_pose_source", "hand_tracking")).lower() + controller_pose_source = "aim" + hand_joint = str(avp_cfg.get("ee_hand_joint", "wrist")).lower() + controller_fallback_enabled = bool( + avp_cfg.get("controller_fallback_enabled", True) + ) + + if pose_source in {"aim", "grip"}: + controller_pose_source = pose_source + pose_source = "controller" + elif pose_source in {"controller"}: + controller_pose_source = str( + avp_cfg.get("controller_pose_source", "aim") + ).lower() + elif pose_source in {"hand_wrist", "wrist"}: + pose_source = "hand_tracking" + hand_joint = "wrist" + elif pose_source in {"hand_palm", "palm"}: + pose_source = "hand_tracking" + hand_joint = "palm" + + if pose_source not in {"controller", "hand_tracking"}: + raise ValueError( + "avp.ee_pose_source must be one of 'hand_tracking', 'wrist', " + f"'palm', 'controller', 'aim', or 'grip'; got {pose_source!r}" + ) + if controller_pose_source not in {"aim", "grip"}: + raise ValueError( + "avp.controller_pose_source must be 'aim' or 'grip', got " + f"{controller_pose_source!r}" + ) + if hand_joint not in {"wrist", "palm"}: + raise ValueError( + f"avp.ee_hand_joint must be 'wrist' or 'palm', got {hand_joint!r}" + ) + return ( + pose_source, + controller_pose_source, + hand_joint, + controller_fallback_enabled, + ) + + def _ee_pose(self, result: Mapping[str, Any], side: str) -> dict[str, Any] | None: + if self._ee_pose_source == "controller": + pose = _controller_pose( + result[f"avp_controller_{side}"], + pose_source=self._controller_pose_source, + deps=self._deps, + ) + self._announce_ee_source(side, "controller" if pose else "inactive") + return pose + + # In the AVP optical-hand setup, EE uses OpenXR hand tracking while MANUS + # only affects finger retargeting; losing MANUS should not disable this path. + pose = _hand_tracking_pose( + result[f"openxr_hand_{side}"], + joint_name=self._ee_hand_joint, + deps=self._deps, + ) + if not self._controller_fallback_enabled or not _is_degenerate_pose(pose): + self._announce_ee_source(side, "hand_tracking" if pose else "inactive") + return pose + + controller_key = f"avp_controller_{side}" + if controller_key not in result: + self._announce_ee_source(side, "inactive") + return None + fallback_pose = _controller_pose( + result[controller_key], + pose_source=self._controller_pose_source, + deps=self._deps, + ) + self._announce_ee_source( + side, + "controller_fallback" if fallback_pose else "inactive", + ) + return fallback_pose + + def _announce_ee_source(self, side: str, source: str) -> None: + key = (side, source) + if key in self._ee_source_announced: + return + self._ee_source_announced.add(key) + print(f"[TeleopMain] {side} EE source: {source}", flush=True) + + +def _is_degenerate_pose(pose: Mapping[str, Any] | None) -> bool: + # Some injected hand-tracking runtimes report valid wrists at the zero pose + # before world tracking is available; do not drive IK to that placeholder. + if pose is None: + return True + position = pose.get("position") + if not isinstance(position, Sequence) or isinstance(position, (str, bytes)): + return True + if len(position) != 3: + return True + return sum(float(value) * float(value) for value in position) < 1.0e-10 + + +def _parse_args(argv: Sequence[str]) -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Run non-ROS AVP EE + MANUS dexterous hand teleop." + ) + parser.add_argument( + "--config", + default=str(_default_config_path()), + help="YAML config for devices, retargeting, and loop settings.", + ) + return parser.parse_args(argv) + + +def main(argv: Sequence[str] | None = None) -> int: + args = _parse_args(sys.argv[1:] if argv is None else argv) + config_path = Path(args.config).expanduser().resolve() + config = _load_yaml(config_path) + + with TeleopMain(config, config_dir=config_path.parent) as teleop: + while True: + teleop.step() + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/examples/g1_wuji_teleop/python/g1_wuji_teleop/hand_retargeting/__init__.py b/examples/g1_wuji_teleop/python/g1_wuji_teleop/hand_retargeting/__init__.py new file mode 100644 index 000000000..4b2df55b7 --- /dev/null +++ b/examples/g1_wuji_teleop/python/g1_wuji_teleop/hand_retargeting/__init__.py @@ -0,0 +1,9 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Local hand-retargeting adapters used by the teleop app.""" + +from .base import HandRetargetStepResult +from .wuji_official_adapter import WujiOfficialManusHandRetargeter + +__all__ = ["HandRetargetStepResult", "WujiOfficialManusHandRetargeter"] diff --git a/examples/g1_wuji_teleop/python/g1_wuji_teleop/hand_retargeting/base.py b/examples/g1_wuji_teleop/python/g1_wuji_teleop/hand_retargeting/base.py new file mode 100644 index 000000000..61c82e89a --- /dev/null +++ b/examples/g1_wuji_teleop/python/g1_wuji_teleop/hand_retargeting/base.py @@ -0,0 +1,25 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Shared result objects for local hand-retargeting adapters.""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any + + +@dataclass(slots=True) +class HandRetargetStepResult: + """One dual-hand retargeting step result.""" + + frame_index: int + left_joint_positions: Any | None + right_joint_positions: Any | None + left_joint_names: tuple[str, ...] | None = None + right_joint_names: tuple[str, ...] | None = None + ok: bool = True + stale: bool = False + error: str | None = None + latency_ms: float | None = None + raw: dict[str, Any] = field(default_factory=dict) diff --git a/examples/g1_wuji_teleop/python/g1_wuji_teleop/hand_retargeting/wuji_official_adapter.py b/examples/g1_wuji_teleop/python/g1_wuji_teleop/hand_retargeting/wuji_official_adapter.py new file mode 100644 index 000000000..7fd73f01b --- /dev/null +++ b/examples/g1_wuji_teleop/python/g1_wuji_teleop/hand_retargeting/wuji_official_adapter.py @@ -0,0 +1,227 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Thin wrapper around the vendored official ``wuji-retargeting`` package.""" + +from __future__ import annotations + +import sys +import time +from pathlib import Path +from typing import Any + +import numpy as np + +from ..paths import repo_root +from .base import HandRetargetStepResult + +_WUJI_CONFIG_FILENAMES = { + "left": "manus_wuji_left.yaml", + "right": "manus_wuji_right.yaml", +} + + +def _official_checkout_root() -> Path: + return (repo_root() / "third_party" / "wuji-retargeting").resolve() + + +def _ensure_upstream_checkout() -> Path: + checkout_root = _official_checkout_root() + package_entry = checkout_root / "wuji_retargeting" / "retarget.py" + if not package_entry.is_file(): + raise ModuleNotFoundError( + "Official wuji-retargeting checkout is missing. Expected " + f"{package_entry}. Initialize the submodule first." + ) + + checkout_root_str = str(checkout_root) + if checkout_root_str not in sys.path: + sys.path.insert(0, checkout_root_str) + return checkout_root + + +def _import_upstream_retargeter(): + _ensure_upstream_checkout() + from wuji_retargeting.retarget import Retargeter + + return Retargeter + + +def _as_float32_numpy(values: Any, shape: tuple[int, ...]) -> np.ndarray: + array = np.asarray(values, dtype=np.float32) + return array.reshape(shape) + + +def _valid_skeleton21(skeleton: np.ndarray) -> tuple[bool, str | None]: + if skeleton.shape != (21, 3): + return False, f"unexpected shape {skeleton.shape}" + if not np.all(np.isfinite(skeleton)): + return False, "non-finite landmarks" + + centered = skeleton - skeleton[0:1] + max_radius = float(np.max(np.linalg.norm(centered, axis=1))) + if max_radius < 1.0e-4: + return False, "all landmarks collapsed to the wrist" + + palm_vec_a = skeleton[5] - skeleton[0] + palm_vec_b = skeleton[9] - skeleton[0] + if np.linalg.norm(palm_vec_a) < 1.0e-5 or np.linalg.norm(palm_vec_b) < 1.0e-5: + return False, "missing palm anchor landmarks" + if np.linalg.norm(np.cross(palm_vec_a, palm_vec_b)) < 1.0e-7: + return False, "degenerate palm frame" + return True, None + + +class WujiOfficialManusHandRetargeter: + """Retarget MANUS/OpenXR-style 21-point skeletons with the official package.""" + + def __init__( + self, + *, + config_dir: Path | None = None, + left_config_name: str = _WUJI_CONFIG_FILENAMES["left"], + right_config_name: str = _WUJI_CONFIG_FILENAMES["right"], + ) -> None: + self._config_dir = ( + Path(config_dir).expanduser() if config_dir is not None else None + ) + self._config_names = { + "left": left_config_name, + "right": right_config_name, + } + self._retargeter_left: Any | None = None + self._retargeter_right: Any | None = None + self._left_joint_names: tuple[str, ...] | None = None + self._right_joint_names: tuple[str, ...] | None = None + self._last_left = np.zeros(20, dtype=np.float32) + self._last_right = np.zeros(20, dtype=np.float32) + + def _resolve_config_path(self, side: str) -> Path: + if self._config_dir is not None: + return (self._config_dir / self._config_names[side]).resolve() + + upstream_root = _ensure_upstream_checkout() + return ( + upstream_root / "example" / "config" / f"retarget_manus_{side}.yaml" + ).resolve() + + def _ensure_retargeters(self) -> None: + if self._retargeter_left is not None and self._retargeter_right is not None: + return + + Retargeter = _import_upstream_retargeter() + left_yaml = self._resolve_config_path("left") + right_yaml = self._resolve_config_path("right") + for config_path in (left_yaml, right_yaml): + if not config_path.is_file(): + raise FileNotFoundError( + f"Official Wuji retarget config does not exist: {config_path}" + ) + self._retargeter_left = Retargeter.from_yaml(str(left_yaml), hand_side="left") + self._retargeter_right = Retargeter.from_yaml( + str(right_yaml), hand_side="right" + ) + self._left_joint_names = tuple( + self._retargeter_left.optimizer.robot.dof_joint_names + ) + self._right_joint_names = tuple( + self._retargeter_right.optimizer.robot.dof_joint_names + ) + + def _safe_qpos(self, retargeter: Any, cached: np.ndarray | None) -> np.ndarray: + num_joints = int(retargeter.num_joints) + if cached is not None: + cached_array = np.asarray(cached, dtype=np.float32).reshape(-1) + if cached_array.shape == (num_joints,) and np.all( + np.isfinite(cached_array) + ): + return cached_array.copy() + + limits = np.asarray(retargeter.optimizer.robot.joint_limits, dtype=np.float32) + zeros = np.zeros(num_joints, dtype=np.float32) + return np.clip(zeros, limits[:, 0], limits[:, 1]) + + def _retarget_hand( + self, + side: str, + skeleton: np.ndarray, + retargeter: Any, + cached_qpos: np.ndarray, + ) -> tuple[np.ndarray, bool, str | None]: + safe_qpos = self._safe_qpos(retargeter, cached_qpos) + valid, error = _valid_skeleton21(skeleton) + if not valid: + return safe_qpos, False, f"invalid {side} hand skeleton: {error}" + + try: + qpos = np.asarray(retargeter.retarget(skeleton), dtype=np.float32).reshape( + -1 + ) + except Exception as exc: + retargeter.reset() + return safe_qpos, False, f"{side} hand retarget failed ({exc})" + + if qpos.shape != safe_qpos.shape: + retargeter.reset() + return ( + safe_qpos, + False, + f"{side} hand retarget returned {qpos.shape}, expected {safe_qpos.shape}", + ) + if not np.all(np.isfinite(qpos)): + retargeter.reset() + return safe_qpos, False, f"{side} hand retarget returned non-finite values" + return qpos.copy(), True, None + + def retarget_manus_skeletons( + self, + left_skeleton: Any, + right_skeleton: Any, + *, + frame_index: int, + device: Any = None, + ) -> HandRetargetStepResult: + del device + self._ensure_retargeters() + started = time.perf_counter() + + left_input = _as_float32_numpy(left_skeleton, (21, 3)) + right_input = _as_float32_numpy(right_skeleton, (21, 3)) + left_qpos, left_ok, left_error = self._retarget_hand( + "left", + left_input, + self._retargeter_left, + self._last_left, + ) + right_qpos, right_ok, right_error = self._retarget_hand( + "right", + right_input, + self._retargeter_right, + self._last_right, + ) + + if left_ok: + self._last_left = left_qpos.copy() + if right_ok: + self._last_right = right_qpos.copy() + + errors = [error for error in (left_error, right_error) if error] + return HandRetargetStepResult( + frame_index=int(frame_index), + left_joint_positions=left_qpos, + right_joint_positions=right_qpos, + left_joint_names=self._left_joint_names, + right_joint_names=self._right_joint_names, + ok=left_ok and right_ok, + stale=not (left_ok and right_ok), + error="; ".join(errors) if errors else None, + latency_ms=(time.perf_counter() - started) * 1000.0, + raw={ + "left_ok": left_ok, + "right_ok": right_ok, + }, + ) + + def shutdown(self) -> None: + self._retargeter_left = None + self._retargeter_right = None diff --git a/examples/g1_wuji_teleop/python/g1_wuji_teleop/paths.py b/examples/g1_wuji_teleop/python/g1_wuji_teleop/paths.py new file mode 100644 index 000000000..114816417 --- /dev/null +++ b/examples/g1_wuji_teleop/python/g1_wuji_teleop/paths.py @@ -0,0 +1,49 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Path helpers for the workspace-local G1-Wuji teleop app.""" + +from __future__ import annotations + +import sys +from pathlib import Path + + +def package_root() -> Path: + return Path(__file__).resolve().parent + + +def app_root() -> Path: + return package_root().parents[1] + + +def repo_root() -> Path: + return app_root().parents[1] + + +def default_config_path() -> Path: + return (app_root() / "config" / "avp_manus.yml").resolve() + + +def default_robot_usd_path() -> Path: + return (app_root() / "assets" / "g1_wuji" / "g1_wuji.usd").resolve() + + +def resolve_repo_relative_path(relative_path: str | Path) -> Path: + relative = Path(relative_path) + app_path = app_root() / relative + if app_path.exists(): + return app_path.resolve() + return (app_root() / relative).resolve() + + +def ensure_import_paths() -> None: + candidates = ( + app_root() / "python", + repo_root() / "build-local" / "python_package" / "Release", + ) + for path in candidates: + if path.exists(): + path_str = str(path) + if path_str not in sys.path: + sys.path.insert(0, path_str) diff --git a/examples/g1_wuji_teleop/python/g1_wuji_teleop/robots/g1_wuji/__init__.py b/examples/g1_wuji_teleop/python/g1_wuji_teleop/robots/g1_wuji/__init__.py new file mode 100644 index 000000000..790630b7e --- /dev/null +++ b/examples/g1_wuji_teleop/python/g1_wuji_teleop/robots/g1_wuji/__init__.py @@ -0,0 +1,4 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""G1-Wuji teleop scene, retargeting, and runtime helpers.""" diff --git a/examples/g1_wuji_teleop/python/g1_wuji_teleop/robots/g1_wuji/debug_viz.py b/examples/g1_wuji_teleop/python/g1_wuji_teleop/robots/g1_wuji/debug_viz.py new file mode 100644 index 000000000..952cf877a --- /dev/null +++ b/examples/g1_wuji_teleop/python/g1_wuji_teleop/robots/g1_wuji/debug_viz.py @@ -0,0 +1,452 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Optional point/axis debug visualization helpers for the G1-Wuji teleop. + +Keep marker creation, head-prim lookup, and per-frame debug updates here so the +teleop control path stays independent of debug-only scene edits. +""" + +from __future__ import annotations + +from collections.abc import Callable, Mapping +from dataclasses import dataclass +from typing import Any + +import numpy as np + + +MarkerPoseResolver = Callable[ + [str, Mapping[str, Any]], tuple[np.ndarray, np.ndarray] | None +] +FramePose = tuple[np.ndarray, np.ndarray] + + +@dataclass(frozen=True) +class DebugVizRuntimeConfig: + enabled: bool = True + + +@dataclass(frozen=True) +class DebugVizSceneConfig: + wrist_marker_radius: float + wrist_axis_length: float + wrist_axis_radius: float + robot_z: float + + +@dataclass +class DebugVizHandles: + wrist_markers: Any | None = None + wrist_frames: Any | None = None + robot_ee_frames: Any | None = None + head_link_point: Any | None = None + head_link_frame: Any | None = None + + +def walk_child_prims(root_prim: Any): + if not root_prim or not root_prim.IsValid(): + return + yield root_prim + for child in root_prim.GetChildren(): + yield from walk_child_prims(child) + + +def find_child_prim_by_name(root_prim: Any, target_name: str) -> Any | None: + for prim in walk_child_prims(root_prim): + if prim.GetName() == target_name: + return prim + return None + + +def head_like_relative_paths() -> tuple[str, ...]: + return ( + "torso_link/head_link", + "g1/torso_link/head_link", + "torso_link/visuals/head_link", + "g1/torso_link/visuals/head_link", + "head_link", + "g1/head_link", + "head", + "g1/head", + ) + + +def find_head_like_prim(stage: Any, root_prim: Any, root_path: str) -> Any | None: + if stage is not None: + for relative_path in head_like_relative_paths(): + prim = stage.GetPrimAtPath(f"{root_path}/{relative_path}") + if prim is not None and prim.IsValid(): + return prim + for prim in stage.Traverse(): + prim_path = str(prim.GetPath()) + if any( + prim_path.endswith(f"/{suffix}") + for suffix in head_like_relative_paths() + ): + return prim + + for name in ("head_link", "head"): + match = find_child_prim_by_name(root_prim, name) + if match is not None and "/visuals/" not in str(match.GetPath()): + return match + for prim in walk_child_prims(root_prim): + prim_path = str(prim.GetPath()) + if "/visuals/" in prim_path: + continue + if "head" in prim.GetName().lower(): + return prim + return None + + +def debug_viz_runtime_config(config: Mapping[str, Any]) -> DebugVizRuntimeConfig: + robot_config = config.get("robot", {}) + if robot_config is None: + robot_config = {} + if not isinstance(robot_config, Mapping): + raise ValueError("Config field 'robot' must be a mapping") + + # Keep debug visualization as a single on/off switch for this teleop path. + debug_viz_config = robot_config.get("debug_visualization", True) + if debug_viz_config is None: + debug_viz_config = True + if not isinstance(debug_viz_config, bool): + raise ValueError("Config field 'robot.debug_visualization' must be a boolean") + + return DebugVizRuntimeConfig(enabled=debug_viz_config) + + +def _make_axis_marker_set( + sim_utils: Any, + VisualizationMarkers: Any, + VisualizationMarkersCfg: Any, + *, + prim_path: str, + axis_length: float, + axis_radius: float, + anchor_x: float, + robot_z: float, + colors: tuple[ + tuple[float, float, float], + tuple[float, float, float], + tuple[float, float, float], + ], +) -> Any: + axis_cfg = VisualizationMarkersCfg( + prim_path=prim_path, + markers={ + "x": sim_utils.CylinderCfg( + radius=axis_radius, + height=axis_length, + visual_material=sim_utils.PreviewSurfaceCfg(diffuse_color=colors[0]), + ), + "y": sim_utils.CylinderCfg( + radius=axis_radius, + height=axis_length, + visual_material=sim_utils.PreviewSurfaceCfg(diffuse_color=colors[1]), + ), + "z": sim_utils.CylinderCfg( + radius=axis_radius, + height=axis_length, + visual_material=sim_utils.PreviewSurfaceCfg(diffuse_color=colors[2]), + ), + }, + ) + axis_markers = VisualizationMarkers(axis_cfg) + axis_markers.visualize( + translations=np.array( + [ + [anchor_x + axis_length * 0.5, 0.25, robot_z + 0.85], + [anchor_x, 0.25 + axis_length * 0.5, robot_z + 0.85], + [anchor_x, 0.25, robot_z + 0.85 + axis_length * 0.5], + [anchor_x + axis_length * 0.5, -0.25, robot_z + 0.85], + [anchor_x, -0.25 + axis_length * 0.5, robot_z + 0.85], + [anchor_x, -0.25, robot_z + 0.85 + axis_length * 0.5], + ], + dtype=np.float32, + ), + orientations=np.array( + [ + _axis_to_cylinder_orientation(np.array([1.0, 0.0, 0.0])), + _axis_to_cylinder_orientation(np.array([0.0, 1.0, 0.0])), + _axis_to_cylinder_orientation(np.array([0.0, 0.0, 1.0])), + _axis_to_cylinder_orientation(np.array([1.0, 0.0, 0.0])), + _axis_to_cylinder_orientation(np.array([0.0, 1.0, 0.0])), + _axis_to_cylinder_orientation(np.array([0.0, 0.0, 1.0])), + ], + dtype=np.float32, + ), + marker_indices=np.array([0, 1, 2, 0, 1, 2], dtype=np.int32), + ) + return axis_markers + + +def _make_point_marker_set( + sim_utils: Any, + VisualizationMarkers: Any, + VisualizationMarkersCfg: Any, + *, + prim_path: str, + radius: float, + anchor_x: float, + anchor_y: float, + robot_z: float, + color: tuple[float, float, float], +) -> Any: + marker_cfg = VisualizationMarkersCfg( + prim_path=prim_path, + markers={ + "point": sim_utils.SphereCfg( + radius=radius, + visual_material=sim_utils.PreviewSurfaceCfg(diffuse_color=color), + ), + }, + ) + point_markers = VisualizationMarkers(marker_cfg) + point_markers.visualize( + translations=np.array( + [[anchor_x, anchor_y, robot_z + 0.85]], + dtype=np.float32, + ), + marker_indices=np.array([0], dtype=np.int32), + ) + return point_markers + + +def create_debug_viz( + *, + sim_utils: Any, + VisualizationMarkers: Any, + VisualizationMarkersCfg: Any, + scene_config: DebugVizSceneConfig, + runtime_config: DebugVizRuntimeConfig, +) -> DebugVizHandles: + handles = DebugVizHandles() + if not runtime_config.enabled: + return handles + + marker_cfg = VisualizationMarkersCfg( + prim_path="/World/TeleopTargets/WristMarkers", + markers={ + "left": sim_utils.SphereCfg( + radius=scene_config.wrist_marker_radius, + visual_material=sim_utils.PreviewSurfaceCfg( + diffuse_color=(0.1, 0.35, 1.0) + ), + ), + "right": sim_utils.SphereCfg( + radius=scene_config.wrist_marker_radius, + visual_material=sim_utils.PreviewSurfaceCfg( + diffuse_color=(1.0, 0.35, 0.1) + ), + ), + }, + ) + handles.wrist_markers = VisualizationMarkers(marker_cfg) + handles.wrist_markers.visualize( + translations=np.array( + [ + [0.35, 0.25, scene_config.robot_z + 0.85], + [0.35, -0.25, scene_config.robot_z + 0.85], + ], + dtype=np.float32, + ), + marker_indices=np.array([0, 1], dtype=np.int32), + ) + + handles.wrist_frames = _make_axis_marker_set( + sim_utils, + VisualizationMarkers, + VisualizationMarkersCfg, + prim_path="/World/TeleopTargets/WristAxes", + axis_length=scene_config.wrist_axis_length, + axis_radius=scene_config.wrist_axis_radius, + anchor_x=0.35, + robot_z=scene_config.robot_z, + colors=((1.0, 0.1, 0.1), (0.1, 0.9, 0.1), (0.1, 0.3, 1.0)), + ) + + handles.robot_ee_frames = _make_axis_marker_set( + sim_utils, + VisualizationMarkers, + VisualizationMarkersCfg, + prim_path="/World/TeleopTargets/RobotEeAxes", + axis_length=scene_config.wrist_axis_length, + axis_radius=scene_config.wrist_axis_radius, + anchor_x=0.2, + robot_z=scene_config.robot_z, + colors=((1.0, 0.1, 0.1), (0.1, 0.9, 0.1), (0.1, 0.3, 1.0)), + ) + + handles.head_link_point = _make_point_marker_set( + sim_utils, + VisualizationMarkers, + VisualizationMarkersCfg, + prim_path="/World/TeleopTargets/RobotHeadLinkPoint", + radius=scene_config.wrist_marker_radius * 1.15, + anchor_x=-0.2, + anchor_y=0.0, + robot_z=scene_config.robot_z, + color=(0.1, 0.95, 0.95), + ) + handles.head_link_frame = _make_axis_marker_set( + sim_utils, + VisualizationMarkers, + VisualizationMarkersCfg, + prim_path="/World/TeleopTargets/RobotHeadLinkAxes", + axis_length=scene_config.wrist_axis_length, + axis_radius=scene_config.wrist_axis_radius, + anchor_x=-0.2, + robot_z=scene_config.robot_z, + colors=((1.0, 0.1, 0.1), (0.1, 0.9, 0.1), (0.1, 0.3, 1.0)), + ) + + return handles + + +def update_wrist_markers( + point_markers: Any | None, + frame_markers: Any | None, + target_poses: Mapping[str, FramePose], + axis_length: float, +) -> None: + if point_markers is None and frame_markers is None: + return + + translations: list[np.ndarray] = [] + marker_indices: list[int] = [] + axis_translations: list[np.ndarray] = [] + axis_orientations: list[np.ndarray] = [] + axis_indices: list[int] = [] + for marker_index, side in enumerate(("left", "right")): + target = target_poses.get(side) + if target is None: + continue + position, rotation = target + if point_markers is not None: + translations.append(position) + marker_indices.append(marker_index) + if frame_markers is not None: + for axis_index in range(3): + axis = rotation[:, axis_index] + axis_translations.append( + position + axis.astype(np.float32) * 0.5 * axis_length + ) + axis_orientations.append(_axis_to_cylinder_orientation(axis)) + axis_indices.append(axis_index) + if point_markers is not None and translations: + point_markers.visualize( + translations=np.stack(translations), + marker_indices=np.asarray(marker_indices, dtype=np.int32), + ) + if frame_markers is not None and axis_translations: + frame_markers.visualize( + translations=np.stack(axis_translations), + orientations=np.stack(axis_orientations), + marker_indices=np.asarray(axis_indices, dtype=np.int32), + ) + + +def update_frame_axis_markers( + frame_markers: Any | None, + frame_poses: Mapping[str, FramePose], + axis_length: float, +) -> None: + if frame_markers is None: + return + axis_translations: list[np.ndarray] = [] + axis_orientations: list[np.ndarray] = [] + axis_indices: list[int] = [] + for side in ("left", "right"): + frame_pose = frame_poses.get(side) + if frame_pose is None: + continue + + position, rotation = frame_pose + for axis_index in range(3): + axis = rotation[:, axis_index] + axis_translations.append( + position + axis.astype(np.float32) * 0.5 * axis_length + ) + axis_orientations.append(_axis_to_cylinder_orientation(axis)) + axis_indices.append(axis_index) + if axis_translations: + frame_markers.visualize( + translations=np.stack(axis_translations), + orientations=np.stack(axis_orientations), + marker_indices=np.asarray(axis_indices, dtype=np.int32), + ) + + +def update_point_marker( + point_markers: Any | None, + frame_pose: FramePose | None, +) -> None: + if point_markers is None or frame_pose is None: + return + position, _rotation = frame_pose + point_markers.visualize( + translations=np.asarray([position], dtype=np.float32), + marker_indices=np.array([0], dtype=np.int32), + ) + + +def update_head_marker( + handles: DebugVizHandles, + *, + head_link_prim: Any | None, + xform_cache: Any | None, +) -> None: + if head_link_prim is None or xform_cache is None: + return + + xform_cache.Clear() + head_world = xform_cache.GetLocalToWorldTransform(head_link_prim) + head_translation = np.asarray(head_world.ExtractTranslation(), dtype=np.float32) + head_rotation = np.asarray(head_world.ExtractRotationMatrix(), dtype=np.float32) + head_pose = (head_translation, head_rotation) + update_point_marker(handles.head_link_point, head_pose) + update_frame_axis_markers( + handles.head_link_frame, + {"head": head_pose}, + axis_length=0.18, + ) + + +def sample_target_poses( + sample: Mapping[str, Any], + *, + resolve_marker_pose: MarkerPoseResolver, +) -> dict[str, FramePose]: + target_poses: dict[str, FramePose] = {} + for side in ("left", "right"): + ee_poses = sample.get("ee_poses", {}) + pose = ee_poses.get(side) if isinstance(ee_poses, Mapping) else None + if pose is None: + continue + target = resolve_marker_pose(side, pose) + if target is None: + continue + target_poses[side] = target + return target_poses + + +def _axis_to_cylinder_orientation(axis: np.ndarray) -> np.ndarray: + axis = np.asarray(axis, dtype=np.float64) + axis_norm = np.linalg.norm(axis) + if axis_norm < 1.0e-8: + return np.array([0.0, 0.0, 0.0, 1.0], dtype=np.float32) + z_axis = np.array([0.0, 0.0, 1.0], dtype=np.float64) + target = axis / axis_norm + dot = float(np.clip(np.dot(z_axis, target), -1.0, 1.0)) + if dot > 1.0 - 1.0e-8: + return np.array([0.0, 0.0, 0.0, 1.0], dtype=np.float32) + if dot < -1.0 + 1.0e-8: + return np.array([1.0, 0.0, 0.0, 0.0], dtype=np.float32) + rot_axis = np.cross(z_axis, target) + rot_axis /= np.linalg.norm(rot_axis) + angle = np.arccos(dot) + half = angle * 0.5 + quat = np.concatenate((rot_axis * np.sin(half), [np.cos(half)])).astype(np.float32) + quat /= np.linalg.norm(quat) + return quat diff --git a/examples/g1_wuji_teleop/python/g1_wuji_teleop/robots/g1_wuji/runtime.py b/examples/g1_wuji_teleop/python/g1_wuji_teleop/robots/g1_wuji/runtime.py new file mode 100644 index 000000000..e86d76618 --- /dev/null +++ b/examples/g1_wuji_teleop/python/g1_wuji_teleop/robots/g1_wuji/runtime.py @@ -0,0 +1,1419 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""MANUS and AVP data helpers for the G1-Wuji teleop example.""" + +from __future__ import annotations + +import sys +import time +from collections.abc import Mapping, Sequence +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +import numpy as np + +from ...paths import app_root + +LEFT_WUJI_JOINTS = ( + "left_finger1_joint1", + "left_finger1_joint2", + "left_finger1_joint3", + "left_finger1_joint4", + "left_finger2_joint1", + "left_finger2_joint2", + "left_finger2_joint3", + "left_finger2_joint4", + "left_finger3_joint1", + "left_finger3_joint2", + "left_finger3_joint3", + "left_finger3_joint4", + "left_finger4_joint1", + "left_finger4_joint2", + "left_finger4_joint3", + "left_finger4_joint4", + "left_finger5_joint1", + "left_finger5_joint2", + "left_finger5_joint3", + "left_finger5_joint4", +) + +RIGHT_WUJI_JOINTS = tuple( + name.replace("left_", "right_", 1) for name in LEFT_WUJI_JOINTS +) + +WUJI_SKELETON21_OPENXR_NAMES = ( + "wrist", + "thumb_metacarpal", + "thumb_proximal", + "thumb_distal", + "thumb_tip", + "index_proximal", + "index_intermediate", + "index_distal", + "index_tip", + "middle_proximal", + "middle_intermediate", + "middle_distal", + "middle_tip", + "ring_proximal", + "ring_intermediate", + "ring_distal", + "ring_tip", + "little_proximal", + "little_intermediate", + "little_distal", + "little_tip", +) + +FIXED_WUJI_HAND_CONFIG_DIR = "official/g1_wuji" + + +def _default_wuji_initial_joint_positions() -> dict[str, float]: + return { + "left_finger1_joint1": 0.059, + "left_finger1_joint2": 0.059, + "left_finger1_joint3": 0.059, + "left_finger1_joint4": 0.0, + "left_finger2_joint1": 0.0, + "left_finger2_joint2": 0.0, + "left_finger2_joint3": 0.0, + "left_finger2_joint4": 0.0, + "left_finger3_joint1": 0.0, + "left_finger3_joint2": 0.0, + "left_finger3_joint3": 0.0, + "left_finger3_joint4": 0.0, + "left_finger4_joint1": 0.0, + "left_finger4_joint2": 0.0, + "left_finger4_joint3": 0.0, + "left_finger4_joint4": 0.0, + "left_finger5_joint1": 0.0, + "left_finger5_joint2": 0.0, + "left_finger5_joint3": 0.0, + "left_finger5_joint4": 0.0, + "right_finger1_joint1": 0.037, + "right_finger1_joint2": 0.037, + "right_finger1_joint3": 0.037, + "right_finger1_joint4": 0.0, + "right_finger2_joint1": 0.0, + "right_finger2_joint2": 0.0, + "right_finger2_joint3": 0.0, + "right_finger2_joint4": 0.0, + "right_finger3_joint1": 0.0, + "right_finger3_joint2": 0.0, + "right_finger3_joint3": 0.0, + "right_finger3_joint4": 0.0, + "right_finger4_joint1": 0.0, + "right_finger4_joint2": 0.0, + "right_finger4_joint3": 0.0, + "right_finger4_joint4": 0.0, + "right_finger5_joint1": 0.0, + "right_finger5_joint2": 0.0, + "right_finger5_joint3": 0.0, + "right_finger5_joint4": 0.0, + } + + +@dataclass(frozen=True) +class RobotProfile: + variant: str + robot_prim: str + default_usd_relpath: str + left_hand_joint_names: tuple[str, ...] + right_hand_joint_names: tuple[str, ...] + initial_hand_joint_positions: dict[str, float] + hand_retarget_backend: str + hand_wuji_config_dir: str | None + ik_body_names: dict[str, str] + ik_body_offset_pos: dict[str, tuple[float, float, float]] + ik_body_offset_quat_xyzw: dict[str, tuple[float, float, float, float]] + ik_target_orientation_correction_quat_xyzw: dict[ + str, tuple[float, float, float, float] + ] + reference_body: str + reference_offset_pos: tuple[float, float, float] + reference_offset_quat_xyzw: tuple[float, float, float, float] + axis_signs: dict[str, float] + status_label: str + + +ROBOT_PROFILES: dict[str, RobotProfile] = { + "g1_wuji": RobotProfile( + variant="g1_wuji", + robot_prim="/World/G1Wuji", + default_usd_relpath="assets/g1_wuji/g1_wuji.usd", + left_hand_joint_names=LEFT_WUJI_JOINTS, + right_hand_joint_names=RIGHT_WUJI_JOINTS, + initial_hand_joint_positions=_default_wuji_initial_joint_positions(), + hand_retarget_backend="wuji", + hand_wuji_config_dir=FIXED_WUJI_HAND_CONFIG_DIR, + ik_body_names={ + "left": "left_wrist_yaw_link", + "right": "right_wrist_yaw_link", + }, + ik_body_offset_pos={ + "left": (0.0415, 0.003, 0.0), + "right": (0.0415, -0.003, 0.0), + }, + ik_body_offset_quat_xyzw={ + "left": (0.0, 0.0, 0.0, 1.0), + "right": (0.0, 0.0, 0.0, 1.0), + }, + ik_target_orientation_correction_quat_xyzw={ + "left": (0.5, 0.5, 0.5, 0.5), + "right": (-0.5, 0.5, 0.5, -0.5), + }, + reference_body="torso_link", + reference_offset_pos=(0.0077774, 0.0000210, 0.3836842), + reference_offset_quat_xyzw=(0.0, 0.0, 0.0, 1.0), + axis_signs={}, + status_label="G1-Wuji", + ), +} + + +def normalize_robot_variant(value: Any) -> str: + raw = str(value or "g1_wuji").strip().lower() + aliases = { + "g1wuji": "g1_wuji", + "g1-wuji": "g1_wuji", + "wuji": "g1_wuji", + } + normalized = aliases.get(raw, raw) + if normalized not in ROBOT_PROFILES: + raise ValueError( + f"Unsupported robot.variant={value!r}. Expected one of {sorted(ROBOT_PROFILES)}" + ) + return normalized + + +def robot_profile_from_config(config: Mapping[str, Any]) -> RobotProfile: + robot_config = config.get("robot", {}) + if robot_config is None: + robot_config = {} + if not isinstance(robot_config, Mapping): + raise ValueError("Config field 'robot' must be a mapping") + return ROBOT_PROFILES[normalize_robot_variant(robot_config.get("variant"))] + + +@dataclass(frozen=True) +class WujiHandRuntimeConfig: + robot_variant: str = "g1_wuji" + neutral_calibration_samples: int = 30 + initial_joint_position: float = 0.0 + initial_joint_positions: dict[str, float] = field( + default_factory=_default_wuji_initial_joint_positions + ) + left_joint_names: tuple[str, ...] = LEFT_WUJI_JOINTS + right_joint_names: tuple[str, ...] = RIGHT_WUJI_JOINTS + axis_signs: dict[str, float] = field(default_factory=dict) + retarget_backend: str = "wuji" + status_label: str = "teleop" + wuji_config_dir: Path | None = None + retarget_output_mode: str = "absolute" + clamp_to_joint_limits: bool = True + smoothing_alpha: float = 0.8 + + +@dataclass(frozen=True) +class AvpFrameBindingRuntimeConfig: + status_label: str = "teleop" + enabled: bool = True + calibration_window_s: float = 3.0 + robot_reference_body: str = "torso_link" + robot_reference_offset_pos: tuple[float, float, float] = (0.0, 0.0, 0.0) + robot_reference_offset_quat_xyzw: tuple[float, float, float, float] = ( + 0.0, + 0.0, + 0.0, + 1.0, + ) + head_forward_axis_isaac: tuple[float, float, float] = (0.0, 1.0, 0.0) + whole_body_yaw_following: bool = True + whole_body_yaw_deadband_rad: float = 0.03 + whole_body_yaw_smoothing_alpha: float = 0.2 + + +@dataclass +class _AvpHeadHandBindingState: + origin_pos: np.ndarray + frame_rot: np.ndarray + head_rot: np.ndarray + + +@dataclass +class _AvpPendingHeadHandCalibration: + start_timestamp_s: float + last_timestamp_s: float + sample_count: int = 0 + head_pos_sum: np.ndarray = field( + default_factory=lambda: np.zeros(3, dtype=np.float64) + ) + head_rot_sum: np.ndarray = field( + default_factory=lambda: np.zeros((3, 3), dtype=np.float64) + ) + left_pos_sum: np.ndarray = field( + default_factory=lambda: np.zeros(3, dtype=np.float64) + ) + right_pos_sum: np.ndarray = field( + default_factory=lambda: np.zeros(3, dtype=np.float64) + ) + + def append( + self, + *, + timestamp_s: float, + head_pos: np.ndarray, + head_rot: np.ndarray, + left_pos: np.ndarray, + right_pos: np.ndarray, + ) -> None: + self.last_timestamp_s = float(timestamp_s) + self.sample_count += 1 + self.head_pos_sum += np.asarray(head_pos, dtype=np.float64) + self.head_rot_sum += np.asarray(head_rot, dtype=np.float64) + self.left_pos_sum += np.asarray(left_pos, dtype=np.float64) + self.right_pos_sum += np.asarray(right_pos, dtype=np.float64) + + +class HandTargetPostProcessor: + """Map MANUS retarget qpos into stable USD joint position targets.""" + + def __init__( + self, + *, + joint_names: Sequence[str], + initial_targets: np.ndarray, + joint_limits: np.ndarray | None, + config: WujiHandRuntimeConfig, + side: str, + ) -> None: + self._joint_names = tuple(joint_names) + self._initial_targets = np.asarray(initial_targets, dtype=np.float32) + self._joint_limits = ( + None if joint_limits is None else np.asarray(joint_limits, dtype=np.float32) + ) + self._config = config + self._side = side + self._neutral_samples: list[np.ndarray] = [] + self._neutral: np.ndarray | None = None + self._last_target: np.ndarray | None = None + self._announced = False + + @property + def ready(self) -> bool: + return ( + self._config.neutral_calibration_samples <= 0 or self._neutral is not None + ) + + def process(self, raw_targets: np.ndarray | None) -> np.ndarray | None: + if raw_targets is None: + return self._last_target + + raw = np.asarray(raw_targets, dtype=np.float32) + if raw.shape != self._initial_targets.shape: + raise ValueError( + f"{self._side} {self._config.status_label} target shape mismatch: expected " + f"{self._initial_targets.shape}, got {raw.shape}" + ) + + if self._config.retarget_output_mode == "absolute": + target = raw + elif self._config.neutral_calibration_samples > 0 and self._neutral is None: + self._neutral_samples.append(raw.copy()) + if len(self._neutral_samples) >= self._config.neutral_calibration_samples: + self._neutral = np.mean(np.stack(self._neutral_samples), axis=0).astype( + np.float32 + ) + print( + f"[{self._config.status_label}] {self._side} neutral calibrated with " + f"{len(self._neutral_samples)} sample(s)." + ) + if self._last_target is not None: + return self._last_target + return self._initial_targets.copy() + + else: + neutral = self._neutral if self._neutral is not None else np.zeros_like(raw) + target = self._initial_targets + (raw - neutral) + + if self._config.clamp_to_joint_limits and self._joint_limits is not None: + target = np.clip(target, self._joint_limits[:, 0], self._joint_limits[:, 1]) + + if self._last_target is not None and self._config.smoothing_alpha < 1.0: + alpha = self._config.smoothing_alpha + target = self._last_target * (1.0 - alpha) + target * alpha + + self._last_target = target.astype(np.float32) + if not self._announced: + print( + f"[{self._config.status_label}] {self._side} hand postprocess active: " + f"mode={self._config.retarget_output_mode}, " + f"neutral_samples={self._config.neutral_calibration_samples}, " + f"clamp={self._config.clamp_to_joint_limits}, " + f"smoothing_alpha={self._config.smoothing_alpha}." + ) + self._announced = True + return self._last_target + + +def wuji_hand_runtime_config(config: Mapping[str, Any]) -> WujiHandRuntimeConfig: + profile = robot_profile_from_config(config) + wuji_config_dir = None + if profile.hand_wuji_config_dir is not None: + wuji_config_dir = _resolve_config_relative_path( + profile.hand_wuji_config_dir, config + ) + if wuji_config_dir is None or not wuji_config_dir.is_dir(): + raise FileNotFoundError( + f"Fixed {profile.status_label} Wuji retarget config directory does not exist: " + f"{wuji_config_dir}" + ) + + return WujiHandRuntimeConfig( + robot_variant=profile.variant, + initial_joint_positions=dict(profile.initial_hand_joint_positions), + left_joint_names=profile.left_hand_joint_names, + right_joint_names=profile.right_hand_joint_names, + axis_signs=dict(profile.axis_signs), + retarget_backend=profile.hand_retarget_backend, + status_label=profile.status_label, + wuji_config_dir=wuji_config_dir, + ) + + +def avp_frame_binding_runtime_config( + config: Mapping[str, Any], +) -> AvpFrameBindingRuntimeConfig: + profile = robot_profile_from_config(config) + robot_config = config.get("robot", {}) + if robot_config is None: + robot_config = {} + if not isinstance(robot_config, Mapping): + raise ValueError("Config field 'robot' must be a mapping") + + binding_config = robot_config.get("avp_frame_binding", {}) + if binding_config is None: + binding_config = {} + if not isinstance(binding_config, Mapping): + raise ValueError("Config field 'robot.avp_frame_binding' must be a mapping") + calibration_window_s = float(binding_config.get("calibration_window_s", 3.0)) + if calibration_window_s < 0.0: + raise ValueError( + "Config field 'robot.avp_frame_binding.calibration_window_s' must be >= 0" + ) + whole_body_yaw_deadband_rad = float( + binding_config.get("whole_body_yaw_deadband_rad", 0.03) + ) + if whole_body_yaw_deadband_rad < 0.0: + raise ValueError( + "Config field 'robot.avp_frame_binding.whole_body_yaw_deadband_rad' " + "must be >= 0" + ) + whole_body_yaw_smoothing_alpha = float( + binding_config.get("whole_body_yaw_smoothing_alpha", 0.2) + ) + if not 0.0 <= whole_body_yaw_smoothing_alpha <= 1.0: + raise ValueError( + "Config field 'robot.avp_frame_binding.whole_body_yaw_smoothing_alpha' " + "must be in [0, 1]" + ) + + return AvpFrameBindingRuntimeConfig( + status_label=profile.status_label, + enabled=bool(binding_config.get("enabled", True)), + calibration_window_s=calibration_window_s, + robot_reference_body=str( + binding_config.get("robot_reference_body", profile.reference_body) + ), + robot_reference_offset_pos=_float_tuple( + binding_config.get("robot_reference_offset_pos"), + name="robot.avp_frame_binding.robot_reference_offset_pos", + length=3, + default=profile.reference_offset_pos, + ), + robot_reference_offset_quat_xyzw=_float_tuple( + binding_config.get("robot_reference_offset_quat_xyzw"), + name="robot.avp_frame_binding.robot_reference_offset_quat_xyzw", + length=4, + default=profile.reference_offset_quat_xyzw, + ), + head_forward_axis_isaac=_float_tuple( + binding_config.get("head_forward_axis_isaac"), + name="robot.avp_frame_binding.head_forward_axis_isaac", + length=3, + default=(0.0, 1.0, 0.0), + ), + whole_body_yaw_following=bool( + binding_config.get("whole_body_yaw_following", True) + ), + whole_body_yaw_deadband_rad=whole_body_yaw_deadband_rad, + whole_body_yaw_smoothing_alpha=whole_body_yaw_smoothing_alpha, + ) + + +def ensure_teleop_import_paths(example_dir: Path, repo_root: Path) -> None: + """Make the local teleop example and build tree importable from Isaac Lab Python.""" + candidates = ( + example_dir, + repo_root / "build-local" / "python_package" / "Release", + ) + for path in candidates: + if path.exists(): + path_str = str(path) + if path_str not in sys.path: + sys.path.insert(0, path_str) + + +def openxr_to_isaac_position( + position: Sequence[float], scale: float, offset_z: float +) -> np.ndarray: + x, y, z = (float(v) for v in position) + position_isaac = np.array([x, -z, y], dtype=np.float32) * float(scale) + return position_isaac + np.array([0.0, 0.0, offset_z], dtype=np.float32) + + +def quat_xyzw_to_matrix(quat: Sequence[float]) -> np.ndarray: + x, y, z, w = (float(v) for v in quat) + norm = (x * x + y * y + z * z + w * w) ** 0.5 + if norm < 1.0e-8: + return np.eye(3, dtype=np.float64) + x /= norm + y /= norm + z /= norm + w /= norm + return np.array( + [ + [1.0 - 2.0 * (y * y + z * z), 2.0 * (x * y - z * w), 2.0 * (x * z + y * w)], + [2.0 * (x * y + z * w), 1.0 - 2.0 * (x * x + z * z), 2.0 * (y * z - x * w)], + [2.0 * (x * z - y * w), 2.0 * (y * z + x * w), 1.0 - 2.0 * (x * x + y * y)], + ], + dtype=np.float64, + ) + + +def matrix_to_quat_xyzw(matrix: np.ndarray) -> np.ndarray: + m = np.asarray(matrix, dtype=np.float64) + trace = float(np.trace(m)) + if trace > 0.0: + s = (trace + 1.0) ** 0.5 * 2.0 + w = 0.25 * s + x = (m[2, 1] - m[1, 2]) / s + y = (m[0, 2] - m[2, 0]) / s + z = (m[1, 0] - m[0, 1]) / s + elif m[0, 0] > m[1, 1] and m[0, 0] > m[2, 2]: + s = (1.0 + m[0, 0] - m[1, 1] - m[2, 2]) ** 0.5 * 2.0 + w = (m[2, 1] - m[1, 2]) / s + x = 0.25 * s + y = (m[0, 1] + m[1, 0]) / s + z = (m[0, 2] + m[2, 0]) / s + elif m[1, 1] > m[2, 2]: + s = (1.0 + m[1, 1] - m[0, 0] - m[2, 2]) ** 0.5 * 2.0 + w = (m[0, 2] - m[2, 0]) / s + x = (m[0, 1] + m[1, 0]) / s + y = 0.25 * s + z = (m[1, 2] + m[2, 1]) / s + else: + s = (1.0 + m[2, 2] - m[0, 0] - m[1, 1]) ** 0.5 * 2.0 + w = (m[1, 0] - m[0, 1]) / s + x = (m[0, 2] + m[2, 0]) / s + y = (m[1, 2] + m[2, 1]) / s + z = 0.25 * s + quat = np.array([x, y, z, w], dtype=np.float32) + quat_norm = np.linalg.norm(quat) + if quat_norm > 1.0e-8: + quat /= quat_norm + return quat + + +def openxr_to_isaac_orientation(orientation_xyzw: Sequence[float]) -> np.ndarray: + return matrix_to_quat_xyzw(openxr_to_isaac_rotation_matrix(orientation_xyzw)) + + +def openxr_to_isaac_rotation_matrix(orientation_xyzw: Sequence[float]) -> np.ndarray: + frame = np.array( + [[1.0, 0.0, 0.0], [0.0, 0.0, -1.0], [0.0, 1.0, 0.0]], + dtype=np.float64, + ) + return frame @ quat_xyzw_to_matrix(orientation_xyzw) @ frame.T + + +def normalize_quat_xyzw(quat: Sequence[float]) -> np.ndarray: + out = np.asarray(quat, dtype=np.float32) + norm = np.linalg.norm(out) + if norm < 1.0e-8: + return np.array([0.0, 0.0, 0.0, 1.0], dtype=np.float32) + return out / norm + + +def openxr_pose_to_isaac_pose( + pose: Mapping[str, Any], + *, + scale: float, + offset_z: float, +) -> tuple[np.ndarray, np.ndarray]: + return ( + openxr_to_isaac_position(pose["position"], scale, offset_z), + openxr_to_isaac_orientation(pose["orientation_xyzw"]), + ) + + +def resolve_marker_pose( + side: str, + pose: Mapping[str, Any], + *, + marker_scale: float, + marker_z_offset: float, + pose_binding: AvpRobotFrameBinding | None, +) -> tuple[np.ndarray, np.ndarray] | None: + if pose_binding is not None: + target_pose = pose_binding.transform_pose(side, pose) + if target_pose is None: + return None + position, orientation_xyzw = target_pose + return position, quat_xyzw_to_matrix(orientation_xyzw) + + return ( + openxr_to_isaac_position(pose["position"], marker_scale, marker_z_offset), + openxr_to_isaac_rotation_matrix(pose["orientation_xyzw"]), + ) + + +class AvpRobotFrameBinding: + """Bind a startup head-hand AVP frame to the robot reference body.""" + + def __init__( + self, + robot: Any, + arm_ik_config: Any, + config: AvpFrameBindingRuntimeConfig, + *, + marker_scale: float, + marker_z_offset: float, + ) -> None: + _ = arm_ik_config + self._robot = robot + self._config = config + self._marker_scale = float(marker_scale) + self._marker_z_offset = float(marker_z_offset) + self._reference_body_id = _find_single_body_id( + robot, + config.robot_reference_body, + "AVP robot reference body", + ) + self._reference_offset_pos = np.asarray( + config.robot_reference_offset_pos, dtype=np.float64 + ) + self._reference_offset_rot = quat_xyzw_to_matrix( + config.robot_reference_offset_quat_xyzw + ) + self._head_forward_axis = _unit_vector(config.head_forward_axis_isaac) + if self._head_forward_axis is None: + raise ValueError( + "Config field 'robot.avp_frame_binding.head_forward_axis_isaac' " + "must be non-zero" + ) + self._state: _AvpHeadHandBindingState | None = None + self._pending_calibration: _AvpPendingHeadHandCalibration | None = None + self._waiting_announced = False + self._runtime_frame_rot: np.ndarray | None = None + self._latest_head_yaw_delta_rad = 0.0 + self._latest_head_tilt_delta_rot = np.eye(3, dtype=np.float64) + + def update_calibration(self, sample: Mapping[str, Any]) -> None: + if not self._config.enabled or self._state is not None: + return + + timestamp_s = _sample_timestamp_s(sample) + pending = self._pending_calibration + calibration = self._calibration_inputs(sample) + if pending is None and calibration is None: + self._maybe_log_waiting(sample) + return + + if pending is None: + pending = _AvpPendingHeadHandCalibration( + start_timestamp_s=timestamp_s, + last_timestamp_s=timestamp_s, + ) + self._pending_calibration = pending + print( + f"[{self._config.status_label}] collecting AVP head-hand calibration samples " + f"frame={int(sample.get('frame_count', 0))} " + f"window_s={self._config.calibration_window_s:.2f}.", + flush=True, + ) + + if calibration is not None: + head_pos, head_rot, left_pos, right_pos = calibration + pending.append( + timestamp_s=timestamp_s, + head_pos=head_pos, + head_rot=head_rot, + left_pos=left_pos, + right_pos=right_pos, + ) + + elapsed_s = max(0.0, timestamp_s - pending.start_timestamp_s) + if elapsed_s < self._config.calibration_window_s: + return + + if pending.sample_count <= 0: + self._pending_calibration = None + self._waiting_announced = False + self._maybe_log_waiting( + sample, + reason=( + "calibration window elapsed without valid head/hand samples " + f"elapsed_s={elapsed_s:.3f}" + ), + ) + return + + averaged_head_pos = pending.head_pos_sum / float(pending.sample_count) + averaged_head_rot = _average_rotation_matrix(pending.head_rot_sum) + averaged_left_pos = pending.left_pos_sum / float(pending.sample_count) + averaged_right_pos = pending.right_pos_sum / float(pending.sample_count) + if averaged_head_rot is None: + self._pending_calibration = None + self._waiting_announced = False + self._maybe_log_waiting( + sample, + reason=( + "averaged head rotation was degenerate " + f"samples={pending.sample_count} elapsed_s={elapsed_s:.3f}" + ), + ) + return + + frame_rot = _head_hand_frame_rotation( + head_rot=averaged_head_rot, + left_pos=averaged_left_pos, + right_pos=averaged_right_pos, + head_forward_axis=self._head_forward_axis, + ) + if frame_rot is None: + wrist_delta = averaged_left_pos - averaged_right_pos + self._pending_calibration = None + self._waiting_announced = False + self._maybe_log_waiting( + sample, + reason=( + "degenerate averaged head/hand axes " + f"samples={pending.sample_count} elapsed_s={elapsed_s:.3f} " + f"head={np.round(averaged_head_pos, 4).tolist()} " + f"left={np.round(averaged_left_pos, 4).tolist()} " + f"right={np.round(averaged_right_pos, 4).tolist()} " + f"left_right_norm={float(np.linalg.norm(wrist_delta)):.6f}" + ), + ) + return + + self._state = _AvpHeadHandBindingState( + origin_pos=averaged_head_pos, + frame_rot=frame_rot, + head_rot=averaged_head_rot, + ) + self._runtime_frame_rot = frame_rot.copy() + self._latest_head_yaw_delta_rad = 0.0 + self._latest_head_tilt_delta_rot = np.eye(3, dtype=np.float64) + self._pending_calibration = None + self._announce_calibration( + sample, + averaged_head_pos, + frame_rot, + elapsed_s=elapsed_s, + sample_count=pending.sample_count, + ) + + def reset_calibration(self) -> None: + self._state = None + self._pending_calibration = None + self._waiting_announced = False + self._runtime_frame_rot = None + self._latest_head_yaw_delta_rad = 0.0 + self._latest_head_tilt_delta_rot = np.eye(3, dtype=np.float64) + print( + f"[{self._config.status_label}] AVP head-hand frame calibration cleared; " + "waiting for fresh calibration.", + flush=True, + ) + + def update_runtime_state(self, sample: Mapping[str, Any]) -> None: + state = self._state + if not self._config.enabled or state is None: + self._runtime_frame_rot = None + self._latest_head_yaw_delta_rad = 0.0 + self._latest_head_tilt_delta_rot = np.eye(3, dtype=np.float64) + return + + if self._runtime_frame_rot is None: + self._runtime_frame_rot = state.frame_rot + + head_pose = sample.get("head_pose") + if head_pose is None: + return + + current_head_rot = openxr_to_isaac_rotation_matrix( + head_pose["orientation_xyzw"] + ) + head_tilt_delta_rot = _head_tilt_delta_rotation( + calibration_head_rot=state.head_rot, + current_head_rot=current_head_rot, + head_forward_axis=self._head_forward_axis, + ) + if head_tilt_delta_rot is not None: + self._latest_head_tilt_delta_rot = head_tilt_delta_rot + + if not self._config.whole_body_yaw_following: + self._runtime_frame_rot = state.frame_rot + self._latest_head_yaw_delta_rad = 0.0 + return + + dynamic_frame_rot, yaw_delta_rad = _dynamic_head_frame_rotation( + calibration_frame_rot=state.frame_rot, + head_rot=current_head_rot, + head_forward_axis=self._head_forward_axis, + ) + if dynamic_frame_rot is None or yaw_delta_rad is None: + return + if abs(yaw_delta_rad) <= self._config.whole_body_yaw_deadband_rad: + dynamic_frame_rot = state.frame_rot + yaw_delta_rad = 0.0 + + self._runtime_frame_rot = dynamic_frame_rot + self._latest_head_yaw_delta_rad = float(yaw_delta_rad) + + @property + def whole_body_yaw_following_enabled(self) -> bool: + return self._config.enabled and self._config.whole_body_yaw_following + + @property + def calibrated(self) -> bool: + return self._state is not None + + def latest_head_yaw_delta_rad(self) -> float: + return float(self._latest_head_yaw_delta_rad) + + def latest_head_tilt_delta_rot(self) -> np.ndarray: + return np.array(self._latest_head_tilt_delta_rot, copy=True) + + def transform_pose( + self, + side: str, + pose: Mapping[str, Any], + ) -> tuple[np.ndarray, np.ndarray] | None: + raw_pos, raw_quat = openxr_pose_to_isaac_pose( + pose, + scale=self._marker_scale, + offset_z=self._marker_z_offset, + ) + if not self._config.enabled: + return raw_pos, raw_quat + + state = self._state + if state is None: + return None + + frame_rot = self._runtime_frame_rot + if frame_rot is None: + frame_rot = state.frame_rot + raw_rot = quat_xyzw_to_matrix(raw_quat) + local_pos = frame_rot.T @ ( + np.asarray(raw_pos, dtype=np.float64) - state.origin_pos + ) + local_rot = frame_rot.T @ raw_rot + reference_pos, reference_rot = self._current_reference_world_pose() + target_pos = reference_pos + reference_rot @ local_pos + target_rot = reference_rot @ local_rot + return target_pos.astype(np.float32), matrix_to_quat_xyzw(target_rot) + + def _current_reference_world_pose(self) -> tuple[np.ndarray, np.ndarray]: + body_pos, body_rot = _body_pose_np(self._robot, self._reference_body_id) + return ( + body_pos + body_rot @ self._reference_offset_pos, + body_rot @ self._reference_offset_rot, + ) + + def _calibration_inputs( + self, sample: Mapping[str, Any] + ) -> tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray] | None: + head_pose = sample.get("head_pose") + ee_poses = sample.get("ee_poses", {}) + left_pose = ee_poses.get("left") if isinstance(ee_poses, Mapping) else None + right_pose = ee_poses.get("right") if isinstance(ee_poses, Mapping) else None + if head_pose is None or left_pose is None or right_pose is None: + return None + + head_pos, head_quat = openxr_pose_to_isaac_pose( + head_pose, + scale=self._marker_scale, + offset_z=self._marker_z_offset, + ) + left_pos, _left_quat = openxr_pose_to_isaac_pose( + left_pose, + scale=self._marker_scale, + offset_z=self._marker_z_offset, + ) + right_pos, _right_quat = openxr_pose_to_isaac_pose( + right_pose, + scale=self._marker_scale, + offset_z=self._marker_z_offset, + ) + return ( + np.asarray(head_pos, dtype=np.float64), + quat_xyzw_to_matrix(head_quat), + np.asarray(left_pos, dtype=np.float64), + np.asarray(right_pos, dtype=np.float64), + ) + + def _maybe_log_waiting( + self, sample: Mapping[str, Any], *, reason: str | None = None + ) -> None: + if self._waiting_announced: + return + frame_count = int(sample.get("frame_count", 0)) + head_ready = sample.get("head_pose") is not None + ee_poses = sample.get("ee_poses", {}) + left_ready = isinstance(ee_poses, Mapping) and ee_poses.get("left") is not None + right_ready = ( + isinstance(ee_poses, Mapping) and ee_poses.get("right") is not None + ) + detail = f" reason={reason}" if reason else "" + print( + f"[{self._config.status_label}] waiting for AVP head-hand frame calibration " + f"frame={frame_count} head={head_ready} left={left_ready} " + f"right={right_ready}{detail}", + flush=True, + ) + self._waiting_announced = True + + def _announce_calibration( + self, + sample: Mapping[str, Any], + origin_pos: np.ndarray, + frame_rot: np.ndarray, + *, + elapsed_s: float, + sample_count: int, + ) -> None: + reference_body_pos, _reference_body_rot = _body_pose_np( + self._robot, self._reference_body_id + ) + reference_pos, _reference_rot = self._current_reference_world_pose() + print( + f"[{self._config.status_label}] AVP head-hand frame calibrated " + f"frame={int(sample.get('frame_count', 0))} " + f"elapsed_s={elapsed_s:.3f} samples={sample_count} " + f"origin_isaac={origin_pos.tolist()} " + f"x={frame_rot[:, 0].tolist()} y={frame_rot[:, 1].tolist()} " + f"z={frame_rot[:, 2].tolist()} " + f"robot_reference_body={self._config.robot_reference_body!r} " + f"robot_reference_body_pos={reference_body_pos.astype(np.float32).tolist()} " + f"robot_reference_pos={reference_pos.astype(np.float32).tolist()}", + flush=True, + ) + + +def ordered_hand_targets( + sample: Mapping[str, Any], side: str, target_names: Sequence[str] +) -> np.ndarray | None: + # Native Dex retargeting emits named joint output via TeleopMain. + # Any backend/schema mismatch shows up here when remapping into the robot joint order. + values_by_name = _joint_values_by_name(sample, side) + if not values_by_name: + return None + return _ordered_named_targets( + values=[values_by_name[name] for name in values_by_name], + source_names=list(values_by_name), + target_names=target_names, + side=side, + label="teleop_dex", + axis_signs={}, + status_label="teleop", + ) + + +class WujiHandTargetBackend: + """Official Wuji retargeting from MANUS/OpenXR skeletons.""" + + def __init__(self, config: WujiHandRuntimeConfig) -> None: + self._config = config + self._retargeter: Any | None = None + self._last_left_skeleton: np.ndarray | None = None + self._last_right_skeleton: np.ndarray | None = None + self._last_left_targets: np.ndarray | None = None + self._last_right_targets: np.ndarray | None = None + self._announced = False + self._waiting_announced = False + + def _ensure_retargeter(self) -> None: + if self._retargeter is not None: + return + if self._config.retarget_backend != "wuji": + raise ValueError( + f"Unsupported hand retarget backend: {self._config.retarget_backend!r}" + ) + + from ...hand_retargeting.wuji_official_adapter import ( + WujiOfficialManusHandRetargeter, + ) + + self._retargeter = WujiOfficialManusHandRetargeter( + config_dir=self._config.wuji_config_dir + ) + self._retargeter._ensure_retargeters() + if not self._announced: + left_names = tuple(getattr(self._retargeter, "_left_joint_names", ()) or ()) + right_names = tuple( + getattr(self._retargeter, "_right_joint_names", ()) or () + ) + config_source = ( + str(self._config.wuji_config_dir) + if self._config.wuji_config_dir is not None + else "official wuji-retargeting example config" + ) + print( + f"[{self._config.status_label}] using hand retarget backend: " + f"{self._config.retarget_backend} " + f"(left_joints={len(left_names)}, right_joints={len(right_names)}, " + f"config={config_source})." + ) + self._announced = True + + @staticmethod + def _to_numpy(values: Any) -> np.ndarray: + if hasattr(values, "detach") and hasattr(values, "cpu"): + values = values.detach().cpu().numpy() + return np.asarray(values, dtype=np.float32).reshape(-1) + + def targets( + self, + sample: Mapping[str, Any], + ) -> tuple[np.ndarray | None, np.ndarray | None]: + left_skeleton = _wuji_skeleton21_from_openxr(sample, "left") + right_skeleton = _wuji_skeleton21_from_openxr(sample, "right") + if left_skeleton is not None: + self._last_left_skeleton = left_skeleton + if right_skeleton is not None: + self._last_right_skeleton = right_skeleton + + left_input = ( + left_skeleton if left_skeleton is not None else self._last_left_skeleton + ) + right_input = ( + right_skeleton if right_skeleton is not None else self._last_right_skeleton + ) + if left_input is None or right_input is None: + if not self._waiting_announced: + print( + f"[{self._config.status_label}] waiting for MANUS hand skeletons; " + "AVP/controller EE teleop remains independent.", + flush=True, + ) + self._waiting_announced = True + return self._last_left_targets, self._last_right_targets + + self._ensure_retargeter() + result = self._retargeter.retarget_manus_skeletons( + left_input, + right_input, + frame_index=int(sample.get("frame_count", 0)), + device=None, + ) + + left_targets = _ordered_named_targets( + values=self._to_numpy(result.left_joint_positions), + source_names=result.left_joint_names, + target_names=self._config.left_joint_names, + side="left", + label=self._config.retarget_backend, + axis_signs=self._config.axis_signs, + status_label=self._config.status_label, + ) + right_targets = _ordered_named_targets( + values=self._to_numpy(result.right_joint_positions), + source_names=result.right_joint_names, + target_names=self._config.right_joint_names, + side="right", + label=self._config.retarget_backend, + axis_signs=self._config.axis_signs, + status_label=self._config.status_label, + ) + + if left_targets is not None: + self._last_left_targets = left_targets + if right_targets is not None: + self._last_right_targets = right_targets + return self._last_left_targets, self._last_right_targets + + +def _resolve_config_relative_path(value: Any, config: Mapping[str, Any]) -> Path | None: + if value is None: + return None + raw = str(value).strip() + if not raw: + return None + path = Path(raw).expanduser() + if path.is_absolute(): + return path.resolve() + default_config_dir = app_root() / "config" + base_dir = Path(str(config.get("_config_dir", default_config_dir))).expanduser() + return (base_dir / path).resolve() + + +def _float_tuple( + values: Any, + *, + name: str, + length: int, + default: Sequence[float], +) -> tuple[float, ...]: + raw = default if values is None else values + if not isinstance(raw, Sequence) or isinstance(raw, (str, bytes)): + raise ValueError(f"Config field {name!r} must be a {length}-element sequence") + if len(raw) != length: + raise ValueError(f"Config field {name!r} must contain exactly {length} values") + return tuple(float(value) for value in raw) + + +def _unit_vector(values: Sequence[float] | np.ndarray) -> np.ndarray | None: + vector = np.asarray(values, dtype=np.float64).reshape(-1) + if vector.size != 3: + return None + norm = np.linalg.norm(vector) + if norm < 1.0e-8: + return None + return vector / norm + + +def _sample_timestamp_s(sample: Mapping[str, Any]) -> float: + timestamp = sample.get("timestamp_s") + if timestamp is None: + return time.monotonic() + return float(timestamp) + + +def _average_rotation_matrix(matrix_sum: np.ndarray) -> np.ndarray | None: + matrix = np.asarray(matrix_sum, dtype=np.float64) + if matrix.shape != (3, 3) or np.linalg.norm(matrix) < 1.0e-8: + return None + try: + u, _s, vt = np.linalg.svd(matrix) + except np.linalg.LinAlgError: + return None + rotation = u @ vt + if np.linalg.det(rotation) < 0.0: + u[:, -1] *= -1.0 + rotation = u @ vt + return rotation + + +def _head_hand_frame_rotation( + *, + head_rot: np.ndarray, + left_pos: np.ndarray, + right_pos: np.ndarray, + head_forward_axis: np.ndarray, +) -> np.ndarray | None: + x_axis = _unit_vector(head_rot @ head_forward_axis) + if x_axis is None: + return None + + y_axis_raw = _unit_vector(left_pos - right_pos) + if y_axis_raw is None: + y_axis = _fallback_lateral_axis(x_axis) + else: + z_axis = _unit_vector(np.cross(x_axis, y_axis_raw)) + if z_axis is None: + y_axis = _fallback_lateral_axis(x_axis) + else: + y_axis = _unit_vector(np.cross(z_axis, x_axis)) + if y_axis is None: + return None + + z_axis = _unit_vector(np.cross(x_axis, y_axis)) + if z_axis is None: + return None + frame_rot = np.column_stack((x_axis, y_axis, z_axis)) + if np.linalg.det(frame_rot) < 0.0: + frame_rot[:, 2] *= -1.0 + return frame_rot + + +def _dynamic_head_frame_rotation( + *, + calibration_frame_rot: np.ndarray, + head_rot: np.ndarray, + head_forward_axis: np.ndarray, +) -> tuple[np.ndarray | None, float | None]: + reference_x = _unit_vector(calibration_frame_rot[:, 0]) + reference_z = _unit_vector(calibration_frame_rot[:, 2]) + current_forward = _unit_vector(head_rot @ head_forward_axis) + if reference_x is None or reference_z is None or current_forward is None: + return None, None + + projected_forward = _unit_vector( + current_forward - np.dot(current_forward, reference_z) * reference_z + ) + if projected_forward is None: + return None, None + + projected_lateral = _unit_vector(np.cross(reference_z, projected_forward)) + if projected_lateral is None: + return None, None + + yaw_delta_rad = float( + np.arctan2( + np.dot(reference_z, np.cross(reference_x, projected_forward)), + np.dot(reference_x, projected_forward), + ) + ) + frame_rot = np.column_stack((projected_forward, projected_lateral, reference_z)) + if np.linalg.det(frame_rot) < 0.0: + frame_rot[:, 1] *= -1.0 + return frame_rot, yaw_delta_rad + + +def _head_tilt_delta_rotation( + *, + calibration_head_rot: np.ndarray, + current_head_rot: np.ndarray, + head_forward_axis: np.ndarray, +) -> np.ndarray | None: + calibration_tilt = _yaw_removed_rotation( + rotation=calibration_head_rot, + forward_axis=head_forward_axis, + ) + current_tilt = _yaw_removed_rotation( + rotation=current_head_rot, + forward_axis=head_forward_axis, + ) + if calibration_tilt is None or current_tilt is None: + return None + tilt_delta = current_tilt @ calibration_tilt.T + if np.linalg.det(tilt_delta) < 0.0: + return None + return tilt_delta + + +def _yaw_removed_rotation( + *, + rotation: np.ndarray, + forward_axis: np.ndarray, +) -> np.ndarray | None: + yaw_rot = _world_z_yaw_rotation_from_forward( + rotation=rotation, + forward_axis=forward_axis, + ) + if yaw_rot is None: + return None + return yaw_rot.T @ rotation + + +def _world_z_yaw_rotation_from_forward( + *, + rotation: np.ndarray, + forward_axis: np.ndarray, +) -> np.ndarray | None: + forward_local = _unit_vector(forward_axis) + if forward_local is None: + return None + + reference_forward = _unit_vector( + np.asarray([forward_local[0], forward_local[1], 0.0], dtype=np.float64) + ) + current_forward = _unit_vector(rotation @ forward_local) + if reference_forward is None or current_forward is None: + return None + projected_forward = _unit_vector( + np.asarray([current_forward[0], current_forward[1], 0.0], dtype=np.float64) + ) + if projected_forward is None: + return None + + yaw_rad = float( + np.arctan2( + reference_forward[0] * projected_forward[1] + - reference_forward[1] * projected_forward[0], + reference_forward[0] * projected_forward[0] + + reference_forward[1] * projected_forward[1], + ) + ) + cos_yaw = float(np.cos(yaw_rad)) + sin_yaw = float(np.sin(yaw_rad)) + return np.array( + [ + [cos_yaw, -sin_yaw, 0.0], + [sin_yaw, cos_yaw, 0.0], + [0.0, 0.0, 1.0], + ], + dtype=np.float64, + ) + + +def _fallback_lateral_axis(x_axis: np.ndarray) -> np.ndarray | None: + # Early tracker frames can report coincident wrists; do not block teleop startup. + up_hint = np.asarray([0.0, 0.0, 1.0], dtype=np.float64) + if abs(float(np.dot(x_axis, up_hint))) > 0.95: + up_hint = np.asarray([0.0, 1.0, 0.0], dtype=np.float64) + return _unit_vector(np.cross(up_hint, x_axis)) + + +def _find_single_body_id(robot: Any, body_name: str, label: str) -> int: + body_ids, body_names = robot.find_bodies(body_name) + if len(body_ids) == 1: + return int(body_ids[0]) + + available = list(getattr(robot.data, "body_names", []) or []) + preview = available[:40] + raise RuntimeError( + f"Expected one {label} match for {body_name!r}; got {len(body_ids)}: " + f"{body_names}. Available body names first entries: {preview}" + ) + + +def _first_numpy(value: Any) -> np.ndarray: + if hasattr(value, "detach"): + return value.detach().cpu().numpy() + try: + import warp as wp + + return wp.to_torch(value).detach().cpu().numpy() + except Exception: + return np.asarray(value) + + +def _body_pose_np(robot: Any, body_id: int) -> tuple[np.ndarray, np.ndarray]: + body_pos = np.asarray( + _first_numpy(robot.data.body_pos_w)[0, body_id], dtype=np.float64 + ) + body_quat = np.asarray( + _first_numpy(robot.data.body_quat_w)[0, body_id], dtype=np.float64 + ) + return body_pos, quat_xyzw_to_matrix(body_quat) + + +def _joint_values_by_name(sample: Mapping[str, Any], side: str) -> dict[str, float]: + names = sample["hand_joint_names"][side] + values = sample["hand_joint_positions"][side] + if values is None: + return {} + return { + str(name): float(values[i]) for i, name in enumerate(names) if i < len(values) + } + + +def _ordered_named_targets( + *, + values: Any, + source_names: Sequence[str] | None, + target_names: Sequence[str], + side: str, + label: str, + axis_signs: Mapping[str, float] | None = None, + status_label: str = "teleop", +) -> np.ndarray | None: + if values is None: + return None + + values_array = np.asarray(values, dtype=np.float32).reshape(-1) + if source_names is None: + ordered = np.zeros(len(target_names), dtype=np.float32) + count = min(len(target_names), values_array.size) + if count: + ordered[:count] = values_array[:count] + return ordered + + values_by_name = { + str(name): float(values_array[i]) + for i, name in enumerate(source_names) + if i < values_array.size + } + if values_by_name: + side_prefix = f"{side}_" + normalized_values_by_name = dict(values_by_name) + for name, value in list(values_by_name.items()): + if name.startswith(side_prefix): + continue + prefixed_name = f"{side_prefix}{name}" + normalized_values_by_name.setdefault(prefixed_name, value) + values_by_name = normalized_values_by_name + missing = [name for name in target_names if name not in values_by_name] + extra = [name for name in values_by_name if name not in set(target_names)] + if missing or extra: + cache = getattr(_ordered_named_targets, "_logged_mismatches", set()) + key = (label, side, tuple(missing), tuple(extra)) + if key not in cache: + print( + f"[{status_label}] retarget joint-name mismatch " + f"backend={label} side={side} missing={missing[:8]} " + f"total_missing={len(missing)} extra={extra[:8]} total_extra={len(extra)}", + flush=True, + ) + cache.add(key) + setattr(_ordered_named_targets, "_logged_mismatches", cache) + + ordered = np.zeros(len(target_names), dtype=np.float32) + sign_map = axis_signs or {} + for i, name in enumerate(target_names): + ordered[i] = values_by_name.get(name, 0.0) * sign_map.get(name, 1.0) + return ordered + + +def _wuji_skeleton21_from_openxr( + sample: Mapping[str, Any], side: str +) -> np.ndarray | None: + skeletons = sample.get("hand_skeletons", {}) + skeleton = skeletons.get(side) if isinstance(skeletons, Mapping) else None + if not isinstance(skeleton, Mapping): + return None + + joint_names = skeleton.get("joint_names") + positions = skeleton.get("positions") + if not isinstance(joint_names, Sequence) or positions is None: + return None + + positions_array = np.asarray(positions, dtype=np.float32) + if positions_array.ndim != 2 or positions_array.shape[1] != 3: + return None + + valid_values = skeleton.get("valid") + valid_array = ( + np.ones(positions_array.shape[0], dtype=bool) + if valid_values is None + else np.asarray(valid_values, dtype=bool).reshape(-1) + ) + name_to_index = {str(name): index for index, name in enumerate(joint_names)} + + out = np.zeros((len(WUJI_SKELETON21_OPENXR_NAMES), 3), dtype=np.float32) + missing: list[str] = [] + for out_index, name in enumerate(WUJI_SKELETON21_OPENXR_NAMES): + source_index = name_to_index.get(name) + if ( + source_index is None + or source_index >= positions_array.shape[0] + or source_index >= valid_array.size + or not bool(valid_array[source_index]) + ): + missing.append(name) + continue + out[out_index] = positions_array[source_index] + + if missing: + cache = getattr(_wuji_skeleton21_from_openxr, "_logged_missing", set()) + key = (side, tuple(missing)) + if key not in cache: + print( + "[teleop] cannot build Wuji 21-point skeleton " + f"side={side} missing_or_invalid={missing[:8]} total={len(missing)}", + flush=True, + ) + cache.add(key) + setattr(_wuji_skeleton21_from_openxr, "_logged_missing", cache) + return None + + if not np.all(np.isfinite(out)): + return None + return out diff --git a/examples/g1_wuji_teleop/python/g1_wuji_teleop/robots/g1_wuji/scene.py b/examples/g1_wuji_teleop/python/g1_wuji_teleop/robots/g1_wuji/scene.py new file mode 100644 index 000000000..3d8ec2d62 --- /dev/null +++ b/examples/g1_wuji_teleop/python/g1_wuji_teleop/robots/g1_wuji/scene.py @@ -0,0 +1,251 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Isaac Lab scene helpers for the G1-Wuji teleop app.""" + +from __future__ import annotations + +from collections.abc import Mapping +from dataclasses import dataclass +from pathlib import Path +from typing import Any + + +@dataclass(frozen=True) +class G1WujiSceneConfig: + robot_prim: str + robot_usd: Path + initial_world_position: tuple[float, float, float] + initial_world_orientation_xyzw: tuple[float, float, float, float] + initial_hand_joint_positions: Mapping[str, float] + light_intensity: float + + +def make_g1_wuji_robot_cfg(config: G1WujiSceneConfig): + import isaaclab.sim as sim_utils + from isaaclab.actuators import ImplicitActuatorCfg + from isaaclab.assets import ArticulationCfg + + joint_pos = { + "right_wrist_yaw_joint": 0.0, + "left_wrist_yaw_joint": 0.0, + ".*_wrist_pitch_joint": 0.0, + ".*_wrist_roll_joint": 0.0, + ".*_shoulder_pitch_joint": 0.0, + ".*_shoulder_roll_joint": 0.0, + ".*_shoulder_yaw_joint": 0.0, + "right_elbow_joint": 0.0, + "left_elbow_joint": 0.0, + } + joint_pos.update( + { + name: float(value) + for name, value in config.initial_hand_joint_positions.items() + } + ) + + return ArticulationCfg( + prim_path=config.robot_prim, + spawn=sim_utils.UsdFileCfg( + usd_path=str(config.robot_usd), + activate_contact_sensors=False, + rigid_props=sim_utils.RigidBodyPropertiesCfg( + disable_gravity=True, + retain_accelerations=False, + max_linear_velocity=100.0, + max_angular_velocity=1000.0, + max_depenetration_velocity=1.0, + ), + collision_props=sim_utils.CollisionPropertiesCfg( + contact_offset=0.002, + rest_offset=0.001, + ), + articulation_props=sim_utils.ArticulationRootPropertiesCfg( + enabled_self_collisions=False, + solver_position_iteration_count=16, + solver_velocity_iteration_count=4, + sleep_threshold=0.005, + stabilization_threshold=0.001, + ), + ), + init_state=ArticulationCfg.InitialStateCfg( + pos=config.initial_world_position, + # IsaacLab consumes the articulation root quaternion as [x, y, z, w]. + rot=config.initial_world_orientation_xyzw, + joint_pos=joint_pos, + joint_vel={".*": 0.0}, + ), + soft_joint_pos_limit_factor=0.98, + actuators=_wuji_actuators(ImplicitActuatorCfg), + ) + + +def _wuji_actuators(ImplicitActuatorCfg): + common = { + "base": ImplicitActuatorCfg( + joint_names_expr=["base_.*"], + effort_limit_sim=100000.0, + velocity_limit_sim=1000.0, + stiffness=1e6, + damping=1e4, + ), + "right_arm": ImplicitActuatorCfg( + joint_names_expr=["right_shoulder_.*", "right_elbow_joint"], + effort_limit_sim=5.0, + velocity_limit_sim=3.0, + stiffness=400.0, + damping=80.0, + ), + "left_arm": ImplicitActuatorCfg( + joint_names_expr=["left_shoulder_.*", "left_elbow_joint"], + effort_limit_sim=5.0, + velocity_limit_sim=3.0, + stiffness=400.0, + damping=80.0, + ), + "wrist": ImplicitActuatorCfg( + joint_names_expr=[ + "left_wrist_roll_joint", + "left_wrist_pitch_joint", + "left_wrist_yaw_joint", + "right_wrist_roll_joint", + "right_wrist_pitch_joint", + "right_wrist_yaw_joint", + ], + effort_limit_sim=90000.0, + velocity_limit_sim=1200.0, + stiffness=400.0, + damping=80.0, + ), + } + common["fingers"] = ImplicitActuatorCfg( + joint_names_expr=["left_finger.*", "right_finger.*"], + effort_limit_sim=300.0, + velocity_limit_sim=8.0, + stiffness=4000.0, + damping=120.0, + ) + return common + + +def design_g1_wuji_scene(config: G1WujiSceneConfig) -> Any: + import isaaclab.sim as sim_utils + from isaaclab.assets import Articulation + + graspable_surface_material = sim_utils.RigidBodyMaterialCfg( + static_friction=1.3, + dynamic_friction=1.1, + restitution=0.0, + friction_combine_mode="max", + restitution_combine_mode="min", + ) + + floor_cfg = sim_utils.CuboidCfg( + size=(6.0, 6.0, 0.04), + collision_props=sim_utils.CollisionPropertiesCfg(), + visual_material=sim_utils.PreviewSurfaceCfg( + diffuse_color=(0.25, 0.25, 0.25), roughness=0.8 + ), + ) + floor_cfg.func("/World/Floor", floor_cfg, translation=(0.0, 0.0, -0.02)) + + table_color = (0.54, 0.42, 0.30) + table_leg_color = (0.22, 0.22, 0.24) + table_top_size = (1.20, 0.75, 0.05) + table_top_center = (0.0, 0.5, 0.74) + table_leg_size = (0.06, 0.06, 0.72) + table_leg_x_offset = table_top_size[0] * 0.5 - 0.09 + table_leg_y_offset = table_top_size[1] * 0.5 - 0.09 + table_leg_z = table_leg_size[2] * 0.5 + + table_top_cfg = sim_utils.CuboidCfg( + size=table_top_size, + collision_props=sim_utils.CollisionPropertiesCfg(), + physics_material=graspable_surface_material, + visual_material=sim_utils.PreviewSurfaceCfg( + diffuse_color=table_color, + roughness=0.55, + ), + ) + table_top_cfg.func( + "/World/Props/TableTop", table_top_cfg, translation=table_top_center + ) + + table_leg_cfg = sim_utils.CuboidCfg( + size=table_leg_size, + collision_props=sim_utils.CollisionPropertiesCfg(), + physics_material=graspable_surface_material, + visual_material=sim_utils.PreviewSurfaceCfg( + diffuse_color=table_leg_color, + roughness=0.65, + ), + ) + for leg_index, (x_sign, y_sign) in enumerate( + ((1.0, 1.0), (1.0, -1.0), (-1.0, 1.0), (-1.0, -1.0)), start=1 + ): + table_leg_cfg.func( + f"/World/Props/TableLeg{leg_index}", + table_leg_cfg, + translation=( + table_top_center[0] + x_sign * table_leg_x_offset, + table_top_center[1] + y_sign * table_leg_y_offset, + table_leg_z, + ), + ) + + cylinder_radius = 0.032 + cylinder_height = 0.18 + cylinder_surface_z = table_top_center[2] + table_top_size[2] * 0.5 + cylinder_cfg = sim_utils.CylinderCfg( + radius=cylinder_radius, + height=cylinder_height, + axis="Z", + collision_props=sim_utils.CollisionPropertiesCfg( + contact_offset=0.003, + rest_offset=0.0, + ), + rigid_props=sim_utils.RigidBodyPropertiesCfg( + rigid_body_enabled=True, + kinematic_enabled=False, + disable_gravity=False, + linear_damping=0.08, + angular_damping=0.06, + max_linear_velocity=8.0, + max_angular_velocity=50.0, + max_depenetration_velocity=3.0, + solver_position_iteration_count=16, + solver_velocity_iteration_count=4, + sleep_threshold=0.002, + stabilization_threshold=0.001, + ), + mass_props=sim_utils.MassPropertiesCfg(mass=0.18), + physics_material=sim_utils.RigidBodyMaterialCfg( + static_friction=1.4, + dynamic_friction=1.2, + restitution=0.0, + friction_combine_mode="max", + restitution_combine_mode="min", + ), + visual_material=sim_utils.PreviewSurfaceCfg( + diffuse_color=(0.84, 0.18, 0.10), + roughness=0.35, + metallic=0.05, + ), + ) + cylinder_cfg.func( + "/World/Props/GraspCylinder", + cylinder_cfg, + translation=( + table_top_center[0], + table_top_center[1] - 0.12, + cylinder_surface_z + cylinder_height * 0.5, + ), + ) + + light_cfg = sim_utils.DomeLightCfg( + intensity=config.light_intensity, + color=(0.75, 0.75, 0.75), + ) + light_cfg.func("/World/Light", light_cfg) + + return Articulation(cfg=make_g1_wuji_robot_cfg(config)) diff --git a/examples/g1_wuji_teleop/python/g1_wuji_teleop/session.py b/examples/g1_wuji_teleop/python/g1_wuji_teleop/session.py new file mode 100644 index 000000000..8a2546c97 --- /dev/null +++ b/examples/g1_wuji_teleop/python/g1_wuji_teleop/session.py @@ -0,0 +1,2778 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Minimal Isaac Lab scene for AVP + MANUS teleop on G1-Wuji. + +Run this script with an Isaac Sim / Isaac Lab Python environment. It intentionally +keeps the simulation side small: a ground plane, a light, the selected G1 USD, +wrist target markers, and live hand joint targets from ``TeleopMain``. +""" + +from __future__ import annotations + +import argparse +import asyncio +import importlib +import os +import sys +import threading +import time +import traceback +from collections.abc import Mapping, Sequence +from dataclasses import dataclass, field +from dataclasses import replace as dataclass_replace +from pathlib import Path +from typing import Any + +import numpy as np + +from .devices.avp_manus_stream import TeleopMain +from .paths import default_config_path, ensure_import_paths, resolve_repo_relative_path +from .robots.g1_wuji.debug_viz import ( + DebugVizSceneConfig, + create_debug_viz, + debug_viz_runtime_config, + find_head_like_prim, + sample_target_poses, + update_frame_axis_markers, + update_head_marker, + update_wrist_markers, + walk_child_prims, +) +from .robots.g1_wuji.runtime import ( + AvpFrameBindingRuntimeConfig, + AvpRobotFrameBinding, + HandTargetPostProcessor, + WujiHandRuntimeConfig, + WujiHandTargetBackend, + avp_frame_binding_runtime_config, + matrix_to_quat_xyzw, + normalize_quat_xyzw, + openxr_pose_to_isaac_pose, + ordered_hand_targets, + robot_profile_from_config, + resolve_marker_pose as resolve_avp_marker_pose, + wuji_hand_runtime_config, +) +from .robots.g1_wuji.scene import ( + G1WujiSceneConfig, + design_g1_wuji_scene, +) + + +os.environ.setdefault("WARP_CACHE_PATH", "/tmp/isaacteleop-warp-cache") + + +def _enable_kit_extension(extension_name: str) -> None: + """Best-effort enable of a packaged Kit extension by name.""" + + try: + import omni.kit.app + + ext_manager = omni.kit.app.get_app().get_extension_manager() + if ext_manager is not None and not ext_manager.is_extension_enabled( + extension_name + ): + ext_manager.set_extension_enabled_immediate(extension_name, True) + except Exception: + pass + + +def _inject_isaacsim_extension_root(extension_name: str) -> None: + """Append a packaged Isaac Sim extension root from ``extscache`` or ``exts`` to ``sys.path``.""" + + for path_entry in sys.path: + if "site-packages" not in path_entry: + continue + isaacsim_root = Path(path_entry) / "isaacsim" + extscache_dir = isaacsim_root / "extscache" + if extscache_dir.is_dir(): + matches = sorted(extscache_dir.glob(f"{extension_name}-*")) + if matches: + ext_path = str(matches[-1]) + if ext_path not in sys.path: + sys.path.append(ext_path) + return + + exts_dir = isaacsim_root / "exts" / extension_name + if exts_dir.is_dir(): + ext_path = str(exts_dir) + if ext_path not in sys.path: + sys.path.append(ext_path) + return + + +def _ensure_carb_input_available() -> None: + """Load ``carb.input`` explicitly so keyboard events work across Kit variants.""" + + try: + import carb + + if hasattr(carb, "input"): + return + import carb.input # noqa: F401 + except Exception as exc: + raise RuntimeError( + "Failed to load carb.input required for teleop keyboard shortcuts." + ) from exc + + +def _import_omni_appwindow(): + """Import ``omni.appwindow`` even when Kit hasn't pre-enabled the extension.""" + + _enable_kit_extension("omni.appwindow") + + try: + import omni.appwindow as omni_appwindow + + return omni_appwindow + except Exception: + _inject_isaacsim_extension_root("omni.appwindow") + _enable_kit_extension("omni.appwindow") + try: + import omni.appwindow as omni_appwindow + + return omni_appwindow + except Exception as exc: + raise RuntimeError( + "Failed to load omni.appwindow required for teleop keyboard shortcuts." + ) from exc + + +def _import_omni_viewport_utility(): + """Import ``omni.kit.viewport.utility`` when Isaac Sim hasn't enabled it yet.""" + + _enable_kit_extension("omni.kit.viewport.window") + _enable_kit_extension("omni.kit.viewport.utility") + + try: + return importlib.import_module("omni.kit.viewport.utility") + except Exception: + _inject_isaacsim_extension_root("omni.kit.viewport.utility") + _enable_kit_extension("omni.kit.viewport.window") + _enable_kit_extension("omni.kit.viewport.utility") + try: + return importlib.import_module("omni.kit.viewport.utility") + except Exception as exc: + raise RuntimeError( + "Failed to load omni.kit.viewport.utility required for head-view camera control." + ) from exc + + +def _import_omni_replicator_core(): + """Import ``omni.replicator.core`` when Isaac Sim hasn't enabled it yet.""" + + _enable_kit_extension("omni.replicator.core") + + try: + return importlib.import_module("omni.replicator.core") + except Exception: + _inject_isaacsim_extension_root("omni.replicator.core") + _enable_kit_extension("omni.replicator.core") + try: + return importlib.import_module("omni.replicator.core") + except Exception as exc: + raise ModuleNotFoundError( + "robot.head_view_xr_display requires omni.replicator.core." + ) from exc + + +def _head_view_xr_display_preflight_error(camera_prim_path: str | None) -> str | None: + """Return a short reason when direct camera render-product prerequisites are not ready.""" + + if not camera_prim_path: + return "no head-view camera prim is available" + try: + _import_omni_replicator_core() + except Exception as exc: + return f"failed to import omni.replicator.core ({type(exc).__name__}: {exc})" + return None + + +def _disable_robot_transmissive_materials( + *, stage: Any, robot_root_prim: Any, status_label: str +) -> None: + """Make the robot body render opaque even when the source USD uses a transmissive OmniSurface.""" + + if stage is None or robot_root_prim is None or not robot_root_prim.IsValid(): + return + + root_prefix = f"{robot_root_prim.GetPath()}/" + patched_shader_paths: list[str] = [] + for prim in stage.Traverse(): + prim_path = str(prim.GetPath()) + if not prim_path.startswith(root_prefix): + continue + if prim.GetTypeName() != "Shader" or not prim_path.endswith( + "/OmniSurface/Shader" + ): + continue + + enable_specular_transmission = prim.GetAttribute( + "inputs:enable_specular_transmission" + ) + if enable_specular_transmission.IsValid(): + enable_specular_transmission.Set(False) + + specular_transmission_weight = prim.GetAttribute( + "inputs:specular_transmission_weight" + ) + if specular_transmission_weight.IsValid(): + specular_transmission_weight.Set(0.0) + + enable_diffuse_transmission = prim.GetAttribute( + "inputs:enable_diffuse_transmission" + ) + if enable_diffuse_transmission.IsValid(): + enable_diffuse_transmission.Set(False) + + enable_opacity = prim.GetAttribute("inputs:enable_opacity") + if enable_opacity.IsValid(): + enable_opacity.Set(False) + + geometry_opacity = prim.GetAttribute("inputs:geometry_opacity") + if geometry_opacity.IsValid(): + geometry_opacity.Set(1.0) + + patched_shader_paths.append(prim_path) + + if patched_shader_paths: + print( + f"[{status_label}] disabled transmissive robot material on " + f"{len(patched_shader_paths)} OmniSurface shader(s).", + flush=True, + ) + + +def _ensure_cloudxr_runtime_environment_defaults() -> None: + """Populate common CloudXR/OpenXR env vars from ``~/.cloudxr`` when the shell has not set them.""" + + cloudxr_root = Path.home() / ".cloudxr" + manifest_candidates = ( + cloudxr_root / "openxr_cloudxr.json", + cloudxr_root / "share" / "openxr" / "1" / "openxr_cloudxr.json", + ) + manifest_path = next((path for path in manifest_candidates if path.is_file()), None) + if manifest_path is not None: + manifest_str = str(manifest_path) + if not os.environ.get("XR_RUNTIME_JSON"): + os.environ["XR_RUNTIME_JSON"] = manifest_str + if not os.environ.get("OPENXR_RUNTIME_JSON"): + os.environ["OPENXR_RUNTIME_JSON"] = manifest_str + + runtime_dir = cloudxr_root / "run" + if runtime_dir.is_dir() and not os.environ.get("NV_CXR_RUNTIME_DIR"): + os.environ["NV_CXR_RUNTIME_DIR"] = str(runtime_dir) + + +_XR_RUNTIME_UNAVAILABLE_PATTERNS = ( + "xr_error_form_factor_unavailable", + "waiting for hmd", + "xrgetsystem timed out", + "requested extension 'xr_ext_hand_tracking' is not advertised", + "failed to determine active runtime", + "failed to load a runtime", + "failed to create openxr instance", +) +_XR_RETRY_INTERVAL_S = 2.0 + + +def _iter_exception_chain(exc: BaseException): + seen: set[int] = set() + current: BaseException | None = exc + while current is not None and id(current) not in seen: + yield current + seen.add(id(current)) + current = current.__cause__ or current.__context__ + + +def _format_exception_chain(exc: BaseException) -> str: + return " -> ".join( + f"{type(error).__name__}: {error}" for error in _iter_exception_chain(exc) + ) + + +def _is_xr_runtime_unavailable_error(exc: BaseException) -> bool: + for error in _iter_exception_chain(exc): + message = f"{type(error).__name__}: {error}".lower() + if any(pattern in message for pattern in _XR_RUNTIME_UNAVAILABLE_PATTERNS): + return True + return False + + +ARM_JOINTS = { + "left": ( + "left_shoulder_pitch_joint", + "left_shoulder_roll_joint", + "left_shoulder_yaw_joint", + "left_elbow_joint", + "left_wrist_roll_joint", + "left_wrist_pitch_joint", + "left_wrist_yaw_joint", + ), + "right": ( + "right_shoulder_pitch_joint", + "right_shoulder_roll_joint", + "right_shoulder_yaw_joint", + "right_elbow_joint", + "right_wrist_roll_joint", + "right_wrist_pitch_joint", + "right_wrist_yaw_joint", + ), +} + + +@dataclass(frozen=True) +class HeadViewCameraConfig: + enabled: bool = True + activate_on_calibration: bool = True + prim_path: str = "/World/TeleopTargets/HeadViewCamera" + translation_offset_m: tuple[float, float, float] = (0.0, 0.0, 0.0) + orientation_offset_rpy_deg: tuple[float, float, float] = (90.0, 0.0, 90.0) + horizontal_aperture_mm: float = 20.955 + focal_length_mm: float = 18.14756 + clipping_range_m: tuple[float, float] = (0.01, 1000.0) + + +@dataclass(frozen=True) +class HeadViewXrDisplayConfig: + enabled: bool = False + activate_on_calibration: bool = True + app_name: str = "G1HeadViewXrDisplay" + resolution_px: tuple[int, int] = (960, 540) + lock_mode: str = "head" + distance_m: float = 1.2 + offset_x_m: float = 0.0 + offset_y_m: float = 0.0 + plane_width_m: float = 2.2 + near_z_m: float = 0.05 + far_z_m: float = 100.0 + clear_color_rgba: tuple[float, float, float, float] = (0.0, 0.0, 0.0, 1.0) + + +@dataclass(frozen=True) +class AuxiliaryViewportConfig: + enabled: bool = True + activate_on_calibration: bool = True + window_name: str = "Teleop Default View" + resolution_px: tuple[int, int] = (960, 540) + position_px: tuple[int, int] = (40, 40) + + +class HeadViewCameraController: + """Drive a viewport camera from the robot head pose after AVP calibration.""" + + def __init__( + self, + *, + stage: Any, + UsdGeom: Any, + Gf: Any, + viewport_api: Any, + config: HeadViewCameraConfig, + head_link_prim: Any | None, + avp_frame_binding: Any | None, + whole_body_yaw: Any | None, + ) -> None: + self._config = config + self._viewport_api = viewport_api + self._UsdGeom = UsdGeom + self._Gf = Gf + self._head_link_prim = head_link_prim + self._avp_frame_binding = avp_frame_binding + self._whole_body_yaw = whole_body_yaw + self._active = False + self._announced_missing_head = False + self._orientation_offset_quat = self._orientation_offset_quatd(config) + + camera = UsdGeom.Camera.Define(stage, config.prim_path) + self._camera_prim = camera.GetPrim() + self._camera_xform = UsdGeom.Xformable(self._camera_prim) + self._translate_op = self._camera_xform.AddTranslateOp() + # USD xform op value types are strict. Head prim world rotations come back as + # GfQuatd here, so keep the camera orient op in double precision as well. + self._orient_op = self._camera_xform.AddOrientOp( + precision=UsdGeom.XformOp.PrecisionDouble + ) + self._scale_op = self._camera_xform.AddScaleOp() + self._camera_path = str(self._camera_prim.GetPath()) + + camera.GetHorizontalApertureAttr().Set(float(config.horizontal_aperture_mm)) + camera.GetFocalLengthAttr().Set(float(config.focal_length_mm)) + camera.GetClippingRangeAttr().Set( + Gf.Vec2f( + float(config.clipping_range_m[0]), + float(config.clipping_range_m[1]), + ) + ) + self._scale_op.Set(Gf.Vec3f(1.0, 1.0, 1.0)) + + @property + def camera_path(self) -> str: + return self._camera_path + + def activate(self) -> None: + if self._viewport_api is None or self._active: + return + self._viewport_api.camera_path = self._camera_path + self._active = True + + def deactivate(self) -> None: + self._active = False + + def update(self, *, xform_cache: Any) -> None: + head_link_prim = self._head_link_prim + if head_link_prim is None or xform_cache is None: + if not self._announced_missing_head: + print( + "Head view camera is enabled but no robot head prim is available; " + "camera updates are disabled.", + flush=True, + ) + self._announced_missing_head = True + return + + xform_cache.Clear() + world_transform = xform_cache.GetLocalToWorldTransform(head_link_prim) + world_translation = world_transform.ExtractTranslation() + world_rotation = world_transform.ExtractRotationQuat() + camera_base_rotation = world_rotation * self._orientation_offset_quat + camera_rotation = self._apply_head_pose_delta(camera_base_rotation) + + offset = self._config.translation_offset_m + offset_vec = self._Gf.Vec3d( + float(offset[0]), float(offset[1]), float(offset[2]) + ) + camera_translation = world_translation + world_transform.TransformDir( + offset_vec + ) + + self._translate_op.Set(camera_translation) + self._orient_op.Set(camera_rotation) + + def _apply_head_pose_delta(self, base_rotation: Any) -> Any: + avp_frame_binding = self._avp_frame_binding + if avp_frame_binding is None or not getattr( + avp_frame_binding, "calibrated", False + ): + return base_rotation + camera_rotation = base_rotation + + try: + tilt_delta_rot = avp_frame_binding.latest_head_tilt_delta_rot() + except Exception: + tilt_delta_rot = None + if tilt_delta_rot is not None: + tilt_delta_quat = self._matrix3_to_quatd(tilt_delta_rot) + camera_rotation = tilt_delta_quat * camera_rotation + return camera_rotation + + def _orientation_offset_quatd(self, config: HeadViewCameraConfig) -> Any: + roll_deg, pitch_deg, yaw_deg = config.orientation_offset_rpy_deg + roll_rad = np.deg2rad(float(roll_deg)) + pitch_rad = np.deg2rad(float(pitch_deg)) + yaw_rad = np.deg2rad(float(yaw_deg)) + + cr = np.cos(roll_rad * 0.5) + sr = np.sin(roll_rad * 0.5) + cp = np.cos(pitch_rad * 0.5) + sp = np.sin(pitch_rad * 0.5) + cy = np.cos(yaw_rad * 0.5) + sy = np.sin(yaw_rad * 0.5) + + quat_xyzw = normalize_quat_xyzw( + ( + sr * cp * cy - cr * sp * sy, + cr * sp * cy + sr * cp * sy, + cr * cp * sy - sr * sp * cy, + cr * cp * cy + sr * sp * sy, + ) + ) + return self._Gf.Quatd( + float(quat_xyzw[3]), + self._Gf.Vec3d( + float(quat_xyzw[0]), + float(quat_xyzw[1]), + float(quat_xyzw[2]), + ), + ) + + def _matrix3_to_quatd(self, matrix: np.ndarray) -> Any: + quat_xyzw = normalize_quat_xyzw( + matrix_to_quat_xyzw(np.asarray(matrix, dtype=np.float64)) + ) + return self._Gf.Quatd( + float(quat_xyzw[3]), + self._Gf.Vec3d( + float(quat_xyzw[0]), + float(quat_xyzw[1]), + float(quat_xyzw[2]), + ), + ) + + +def _head_view_camera_config(config: Mapping[str, Any]) -> HeadViewCameraConfig: + robot_config = config.get("robot", {}) + if robot_config is None: + robot_config = {} + if not isinstance(robot_config, Mapping): + raise ValueError("Config field 'robot' must be a mapping") + + camera_config = robot_config.get("head_view_camera", {}) + if camera_config is None: + camera_config = {} + if not isinstance(camera_config, Mapping): + raise ValueError("Config field 'robot.head_view_camera' must be a mapping") + + prim_path = str( + camera_config.get("prim_path", "/World/TeleopTargets/HeadViewCamera") + ).strip() + if not prim_path: + raise ValueError( + "Config field 'robot.head_view_camera.prim_path' must be non-empty" + ) + + return HeadViewCameraConfig( + enabled=bool(camera_config.get("enabled", True)), + activate_on_calibration=bool( + camera_config.get("activate_on_calibration", True) + ), + prim_path=prim_path, + translation_offset_m=_float_tuple( + camera_config.get("translation_offset_m"), + name="robot.head_view_camera.translation_offset_m", + length=3, + default=(0.0, 0.0, 0.0), + ), + orientation_offset_rpy_deg=_float_tuple( + camera_config.get("orientation_offset_rpy_deg"), + name="robot.head_view_camera.orientation_offset_rpy_deg", + length=3, + default=(90.0, 0.0, 90.0), + ), + horizontal_aperture_mm=float( + camera_config.get("horizontal_aperture_mm", 20.955) + ), + focal_length_mm=float(camera_config.get("focal_length_mm", 18.14756)), + clipping_range_m=_float_tuple( + camera_config.get("clipping_range_m"), + name="robot.head_view_camera.clipping_range_m", + length=2, + default=(0.01, 1000.0), + ), + ) + + +def _head_view_xr_display_config(config: Mapping[str, Any]) -> HeadViewXrDisplayConfig: + robot_config = config.get("robot", {}) + if robot_config is None: + robot_config = {} + if not isinstance(robot_config, Mapping): + raise ValueError("Config field 'robot' must be a mapping") + + xr_config = robot_config.get("head_view_xr_display", {}) + if xr_config is None: + xr_config = {} + if not isinstance(xr_config, Mapping): + raise ValueError("Config field 'robot.head_view_xr_display' must be a mapping") + + resolution_values = xr_config.get("resolution_px", (960, 540)) + if isinstance(resolution_values, (str, bytes)) or not isinstance( + resolution_values, Sequence + ): + raise ValueError( + "Config field 'robot.head_view_xr_display.resolution_px' must be a sequence" + ) + resolution_tuple = tuple(int(value) for value in resolution_values) + if ( + len(resolution_tuple) != 2 + or resolution_tuple[0] <= 0 + or resolution_tuple[1] <= 0 + ): + raise ValueError( + "Config field 'robot.head_view_xr_display.resolution_px' must contain two positive integers" + ) + + lock_mode_raw = str(xr_config.get("lock_mode", "head")).strip().lower() + lock_mode_aliases = { + "head": "head", + "head_locked": "head", + "world": "world", + "world_locked": "world", + } + lock_mode = lock_mode_aliases.get(lock_mode_raw, lock_mode_raw) + if lock_mode not in {"head", "world"}: + raise ValueError( + "Config field 'robot.head_view_xr_display.lock_mode' must be 'head' or 'world'" + ) + + return HeadViewXrDisplayConfig( + enabled=bool(xr_config.get("enabled", False)), + activate_on_calibration=bool(xr_config.get("activate_on_calibration", True)), + app_name=str(xr_config.get("app_name", "G1HeadViewXrDisplay")).strip() + or "G1HeadViewXrDisplay", + resolution_px=(resolution_tuple[0], resolution_tuple[1]), + lock_mode=lock_mode, + distance_m=float(xr_config.get("distance_m", 1.2)), + offset_x_m=float(xr_config.get("offset_x_m", 0.0)), + offset_y_m=float(xr_config.get("offset_y_m", 0.0)), + plane_width_m=float(xr_config.get("plane_width_m", 2.2)), + near_z_m=float(xr_config.get("near_z_m", 0.05)), + far_z_m=float(xr_config.get("far_z_m", 100.0)), + clear_color_rgba=_float_tuple( + xr_config.get("clear_color_rgba"), + name="robot.head_view_xr_display.clear_color_rgba", + length=4, + default=(0.0, 0.0, 0.0, 1.0), + ), + ) + + +def _auxiliary_viewport_config(config: Mapping[str, Any]) -> AuxiliaryViewportConfig: + robot_config = config.get("robot", {}) + if robot_config is None: + robot_config = {} + if not isinstance(robot_config, Mapping): + raise ValueError("Config field 'robot' must be a mapping") + + aux_config = robot_config.get("auxiliary_viewport", {}) + if aux_config is None: + aux_config = {} + if not isinstance(aux_config, Mapping): + raise ValueError("Config field 'robot.auxiliary_viewport' must be a mapping") + + resolution_values = aux_config.get("resolution_px", (960, 540)) + if isinstance(resolution_values, (str, bytes)) or not isinstance( + resolution_values, Sequence + ): + raise ValueError( + "Config field 'robot.auxiliary_viewport.resolution_px' must be a sequence" + ) + resolution_tuple = tuple(int(value) for value in resolution_values) + if ( + len(resolution_tuple) != 2 + or resolution_tuple[0] <= 0 + or resolution_tuple[1] <= 0 + ): + raise ValueError( + "Config field 'robot.auxiliary_viewport.resolution_px' must contain two positive integers" + ) + + position_values = aux_config.get("position_px", (40, 40)) + if isinstance(position_values, (str, bytes)) or not isinstance( + position_values, Sequence + ): + raise ValueError( + "Config field 'robot.auxiliary_viewport.position_px' must be a sequence" + ) + position_tuple = tuple(int(value) for value in position_values) + if len(position_tuple) != 2: + raise ValueError( + "Config field 'robot.auxiliary_viewport.position_px' must contain two integers" + ) + + return AuxiliaryViewportConfig( + enabled=bool(aux_config.get("enabled", True)), + activate_on_calibration=bool(aux_config.get("activate_on_calibration", True)), + window_name=str(aux_config.get("window_name", "Teleop Default View")).strip() + or "Teleop Default View", + resolution_px=(resolution_tuple[0], resolution_tuple[1]), + position_px=(position_tuple[0], position_tuple[1]), + ) + + +def _quat_wxyz_normalize( + quat_wxyz: Sequence[float], +) -> tuple[float, float, float, float]: + quat = np.asarray(quat_wxyz, dtype=np.float64) + norm = float(np.linalg.norm(quat)) + if norm <= 1e-8: + return (1.0, 0.0, 0.0, 0.0) + quat /= norm + return tuple(float(value) for value in quat) + + +def _quat_wxyz_mul( + lhs_wxyz: Sequence[float], rhs_wxyz: Sequence[float] +) -> tuple[float, float, float, float]: + lw, lx, ly, lz = _quat_wxyz_normalize(lhs_wxyz) + rw, rx, ry, rz = _quat_wxyz_normalize(rhs_wxyz) + return ( + lw * rw - lx * rx - ly * ry - lz * rz, + lw * rx + lx * rw + ly * rz - lz * ry, + lw * ry - lx * rz + ly * rw + lz * rx, + lw * rz + lx * ry - ly * rx + lz * rw, + ) + + +def _quat_wxyz_rotate( + quat_wxyz: Sequence[float], vector_xyz: Sequence[float] +) -> tuple[float, float, float]: + qw, qx, qy, qz = _quat_wxyz_normalize(quat_wxyz) + vector = np.asarray(vector_xyz, dtype=np.float64) + qvec = np.asarray([qx, qy, qz], dtype=np.float64) + uv = np.cross(qvec, vector) + uuv = np.cross(qvec, uv) + rotated = vector + 2.0 * (qw * uv + uuv) + return tuple(float(value) for value in rotated) + + +def _yaw_quat_wxyz(yaw_rad: float) -> tuple[float, float, float, float]: + return ( + float(np.cos(yaw_rad * 0.5)), + 0.0, + float(np.sin(yaw_rad * 0.5)), + 0.0, + ) + + +class HeadViewXrDisplayBridge: + """Present the robot head-view camera in the AVP headset via a shared XR session.""" + + _IDENTITY_ROT_WXYZ = (1.0, 0.0, 0.0, 0.0) + + def __init__( + self, + *, + config: HeadViewXrDisplayConfig, + camera_prim_path: str, + required_xr_extensions: Sequence[str], + sim_device: str, + ) -> None: + ensure_import_paths() + try: + import isaacteleop.viz as viz + except ModuleNotFoundError as exc: + raise ModuleNotFoundError( + "robot.head_view_xr_display requires isaacteleop.viz, but the Televiz " + "Python module is not available. Rebuild IsaacTeleop with " + "`cmake -B build-local -DBUILD_VIZ=ON && cmake --build build-local " + "--target python_wheel` so isaacteleop.viz._viz is produced." + ) from exc + + try: + from isaacteleop.oxr import OpenXRSessionHandles + except ModuleNotFoundError as exc: + raise ModuleNotFoundError( + "robot.head_view_xr_display requires isaacteleop.oxr to share the XR session." + ) from exc + + if not str(sim_device).startswith("cuda"): + raise ValueError( + "robot.head_view_xr_display currently requires a CUDA Isaac Sim device" + ) + + rep = _import_omni_replicator_core() + self._config = config + self._camera_prim_path = str(camera_prim_path) + self._rep = rep + self._viz = viz + self._capture_device = str(sim_device) + self._visible = False + self._render_product = None + self._rgb_annotator = None + self._render_error: BaseException | None = None + self._stop = threading.Event() + self._render_thread: threading.Thread | None = None + self._world_locked_pose: ( + tuple[ + tuple[float, float, float], + tuple[float, float, float, float], + ] + | None + ) = None + self._plane_size_m = ( + float(config.plane_width_m), + float(config.plane_width_m) + * float(config.resolution_px[1]) + / float(config.resolution_px[0]), + ) + + session_cfg = viz.VizSessionConfig() + session_cfg.mode = viz.DisplayMode.kXr + session_cfg.app_name = config.app_name + session_cfg.xr_near_z = float(config.near_z_m) + session_cfg.xr_far_z = float(config.far_z_m) + session_cfg.required_extensions = list(required_xr_extensions) + session_cfg.clear_color = tuple( + float(value) for value in config.clear_color_rgba + ) + self._viz_session = viz.VizSession.create(session_cfg) + + oxr_handles_tuple = self._viz_session.get_oxr_handles() + if oxr_handles_tuple is None: + raise RuntimeError( + "Televiz XR session was created, but no OpenXR handles were exposed for TeleopSession sharing." + ) + self._oxr_handles = OpenXRSessionHandles(*oxr_handles_tuple) + + layer_cfg = viz.QuadLayerConfig() + layer_cfg.name = "robot_head_view_xr" + layer_cfg.resolution = viz.Resolution( + int(config.resolution_px[0]), int(config.resolution_px[1]) + ) + layer_cfg.format = viz.PixelFormat.kRGBA8 + layer_cfg.placement = self._initial_placement() + self._layer = self._viz_session.add_quad_layer(layer_cfg) + self._layer.set_visible(False) + self._create_camera_render_product() + + self._render_thread = threading.Thread( + target=self._render_loop, + name="g1_head_view_xr_render", + daemon=False, + ) + self._render_thread.start() + + @property + def oxr_handles(self) -> Any: + return self._oxr_handles + + def set_visible(self, visible: bool) -> None: + self._raise_if_failed() + new_visible = bool(visible) + if new_visible == self._visible: + return + self._layer.set_visible(new_visible) + self._visible = new_visible + + def submit_frame(self) -> bool: + self._raise_if_failed() + rgba = self._read_camera_frame() + if rgba is None: + return False + self._layer.submit(self._coerce_cuda_image(rgba)) + return True + + def close(self) -> None: + self._stop.set() + if self._render_thread is not None: + self._render_thread.join(timeout=5.0) + if self._render_thread.is_alive(): + print( + "Head-view XR display render thread did not exit within 5s; " + "skipping explicit Televiz destruction to avoid tearing down an active XR session.", + flush=True, + ) + else: + self._render_thread = None + self._destroy_camera_render_product() + if self._render_thread is None: + viz_session = getattr(self, "_viz_session", None) + if viz_session is not None: + viz_session.destroy() + self._viz_session = None + if self._render_error is not None: + print( + f"Head-view XR display render loop stopped with error: {self._render_error}", + file=sys.stderr, + flush=True, + ) + + def _raise_if_failed(self) -> None: + if self._render_error is not None: + raise RuntimeError( + f"Head-view XR display render loop failed: {self._render_error}" + ) from self._render_error + + def _create_camera_render_product(self) -> None: + try: + self._render_product = self._rep.create.render_product( + self._camera_prim_path, + ( + int(self._config.resolution_px[0]), + int(self._config.resolution_px[1]), + ), + force_new=True, + ) + self._rgb_annotator = self._rep.AnnotatorRegistry.get_annotator( + "LdrColor", + device="cuda", + do_array_copy=False, + ) + self._rgb_annotator.attach(self._render_product) + except Exception: + self._destroy_camera_render_product() + raise + + def _destroy_camera_render_product(self) -> None: + annotator = getattr(self, "_rgb_annotator", None) + if annotator is not None: + try: + annotator.detach() + except Exception: + pass + self._rgb_annotator = None + render_product = getattr(self, "_render_product", None) + if render_product is not None: + try: + render_product.destroy() + except Exception: + pass + self._render_product = None + + def _read_camera_frame(self) -> Any | None: + if self._stop.is_set(): + return None + annotator = self._rgb_annotator + if annotator is None: + return None + try: + rgba = annotator.get_data(device="cuda") + except Exception as exc: + self._render_error = exc + raise + if rgba is None: + return None + if isinstance(rgba, Mapping): + rgba = rgba.get("data") + if rgba is None: + return None + try: + import warp as wp + + # Replicator commonly returns a Warp CUDA array here. Converting once + # to Torch normalizes vec4-vs-last-dimension layouts and avoids Warp's + # Python indexing limitations (for example ``[..., :4]``). + if isinstance(rgba, wp.array): + rgba = wp.to_torch(rgba) + except Exception: + pass + shape = tuple(int(dim) for dim in getattr(rgba, "shape", ())) + # Freshly attached render products often need a few rendered frames before + # LdrColor exposes a real HxWxC image. Treat that warm-up window as + # "frame not ready yet" instead of tearing down the teleop session. + if len(shape) < 3 or shape[0] <= 0 or shape[1] <= 0 or shape[-1] < 4: + return None + if shape[-1] == 4: + return rgba + return rgba[..., :4] + + def _coerce_cuda_image(self, image: Any) -> Any: + if hasattr(image, "__cuda_array_interface__"): + return image + if hasattr(image, "__dlpack__"): + try: + import warp as wp + + converted = wp.to_torch(image) + if hasattr(converted, "__cuda_array_interface__"): + return converted + except Exception: + pass + if hasattr(image, "__array_interface__"): + import torch + + return torch.as_tensor( + np.asarray(image), device=self._capture_device + ).contiguous() + raise TypeError( + "Head-view XR display received a camera frame that does not expose " + "__cuda_array_interface__ or __array_interface__." + ) + + def _initial_placement(self) -> Any: + return self._viz.QuadLayerPlacement( + self._viz.Pose3D( + (0.0, 1.5, -float(self._config.distance_m)), + self._IDENTITY_ROT_WXYZ, + ), + self._plane_size_m, + ) + + def _render_loop(self) -> None: + try: + while not self._stop.is_set(): + self._update_layer_placement() + self._viz_session.render() + if self._viz_session.should_close(): + self._stop.set() + except BaseException as exc: + self._render_error = exc + self._stop.set() + + def _update_layer_placement(self) -> None: + head_pose = self._viz_session.head_pose_now() + if head_pose is None: + return + head_position = tuple(float(value) for value in head_pose.position) + head_orientation = tuple(float(value) for value in head_pose.orientation) + if self._config.lock_mode == "world": + if self._world_locked_pose is None: + self._world_locked_pose = self._compute_world_locked_pose( + head_position, head_orientation + ) + position, orientation = self._world_locked_pose + else: + position, orientation = self._compute_head_locked_pose( + head_position, head_orientation + ) + self._layer.set_placement( + self._viz.QuadLayerPlacement( + self._viz.Pose3D(position, orientation), + self._plane_size_m, + ) + ) + + def _compute_head_locked_pose( + self, + head_position: tuple[float, float, float], + head_orientation: tuple[float, float, float, float], + ) -> tuple[ + tuple[float, float, float], + tuple[float, float, float, float], + ]: + forward = _quat_wxyz_rotate(head_orientation, (0.0, 0.0, -1.0)) + right = _quat_wxyz_rotate(head_orientation, (1.0, 0.0, 0.0)) + up = _quat_wxyz_rotate(head_orientation, (0.0, 1.0, 0.0)) + position = ( + head_position[0] + + forward[0] * self._config.distance_m + + right[0] * self._config.offset_x_m + + up[0] * self._config.offset_y_m, + head_position[1] + + forward[1] * self._config.distance_m + + right[1] * self._config.offset_x_m + + up[1] * self._config.offset_y_m, + head_position[2] + + forward[2] * self._config.distance_m + + right[2] * self._config.offset_x_m + + up[2] * self._config.offset_y_m, + ) + orientation = _quat_wxyz_mul(head_orientation, self._IDENTITY_ROT_WXYZ) + return position, orientation + + def _compute_world_locked_pose( + self, + head_position: tuple[float, float, float], + head_orientation: tuple[float, float, float, float], + ) -> tuple[ + tuple[float, float, float], + tuple[float, float, float, float], + ]: + forward = np.asarray( + _quat_wxyz_rotate(head_orientation, (0.0, 0.0, -1.0)), + dtype=np.float64, + ) + forward[1] = 0.0 + forward_norm = float(np.linalg.norm(forward)) + if forward_norm <= 1e-8: + forward = np.asarray([0.0, 0.0, -1.0], dtype=np.float64) + else: + forward /= forward_norm + right = np.asarray([-forward[2], 0.0, forward[0]], dtype=np.float64) + position = ( + head_position[0] + + float(forward[0]) * self._config.distance_m + + float(right[0]) * self._config.offset_x_m, + head_position[1] + self._config.offset_y_m, + head_position[2] + + float(forward[2]) * self._config.distance_m + + float(right[2]) * self._config.offset_x_m, + ) + yaw = float( + np.arctan2(head_position[0] - position[0], head_position[2] - position[2]) + ) + return position, _quat_wxyz_mul(_yaw_quat_wxyz(yaw), self._IDENTITY_ROT_WXYZ) + + +class AuxiliaryViewportController: + """Manage a secondary viewport that preserves the scene's initial camera view.""" + + def __init__( + self, + *, + config: AuxiliaryViewportConfig, + viewport_utility: Any, + initial_camera_path: str | None, + ) -> None: + self._config = config + self._viewport_utility = viewport_utility + self._initial_camera_path = ( + None if initial_camera_path is None else str(initial_camera_path) + ) + self._window = None + + def show(self) -> None: + if not self._initial_camera_path: + return + if self._window is None: + self._window = self._viewport_utility.create_viewport_window( + name=self._config.window_name, + width=int(self._config.resolution_px[0]), + height=int(self._config.resolution_px[1]), + position_x=int(self._config.position_px[0]), + position_y=int(self._config.position_px[1]), + camera_path=self._initial_camera_path, + ) + self._dock_to_main_viewport_async() + if self._window is None: + return + self._window.viewport_api.camera_path = self._initial_camera_path + self._window.visible = True + + def hide(self) -> None: + if self._window is not None: + self._window.visible = False + + def close(self) -> None: + window = self._window + self._window = None + if window is None: + return + try: + window.visible = False + except Exception: + pass + try: + window.destroy() + except Exception: + pass + + def _dock_to_main_viewport_async(self) -> None: + window = self._window + if window is None: + return + + async def dock_window() -> None: + try: + import omni.ui as ui + import omni.kit.app + + await omni.kit.app.get_app().next_update_async() + main_viewport = ui.Workspace.get_window("Viewport") + if main_viewport is not None and window is not None: + window.dock_in(main_viewport, ui.DockPosition.RIGHT, 0.35) + except Exception: + pass + + try: + asyncio.ensure_future(dock_window()) + except Exception: + pass + + +@dataclass(frozen=True) +class ArmIkSideConfig: + body_name: str + body_offset_pos: tuple[float, float, float] + body_offset_quat_xyzw: tuple[float, float, float, float] + target_orientation_correction_quat_xyzw: tuple[float, float, float, float] + joint_names: tuple[str, ...] + + +@dataclass(frozen=True) +class ArmIkRuntimeConfig: + status_label: str = "teleop" + enabled: bool = True + marker_scale: float | None = None + marker_z_offset: float | None = None + command_type: str = "pose" + startup_relative_position: bool = True + startup_relative_orientation: bool = True + position_error_scale: float = 1.0 + orientation_error_scale: float = 0.35 + position_deadband_m: float = 0.003 + orientation_deadband_rad: float = 0.06 + ik_method: str = "dls" + damping: float = 0.05 + smoothing_alpha: float = 0.35 + clamp_to_joint_limits: bool = True + sides: dict[str, ArmIkSideConfig] = field(default_factory=dict) + + +def _default_config_path() -> Path: + return default_config_path() + + +def _load_yaml_file(path: Path) -> dict[str, Any]: + try: + import yaml + except ModuleNotFoundError as exc: + raise ModuleNotFoundError( + "PyYAML is required to load the teleop config file." + ) from exc + + with path.open("r", encoding="utf-8") as stream: + data = yaml.safe_load(stream) or {} + if not isinstance(data, dict): + raise ValueError(f"Config root must be a mapping: {path}") + return data + + +def _robot_initial_world_position( + config: Mapping[str, Any], +) -> tuple[float, float, float]: + robot_config = config.get("robot", {}) + if robot_config is None: + return (0.0, 0.0, 0.0) + if not isinstance(robot_config, Mapping): + raise ValueError("Config field 'robot' must be a mapping") + + return _float_tuple( + robot_config.get("initial_world_position"), + name="robot.initial_world_position", + length=3, + default=(0.0, 0.0, 0.0), + ) + + +def _robot_initial_world_orientation_xyzw( + config: Mapping[str, Any], +) -> tuple[float, float, float, float]: + robot_config = config.get("robot", {}) + if robot_config is None: + return (0.0, 0.0, 0.0, 1.0) + if not isinstance(robot_config, Mapping): + raise ValueError("Config field 'robot' must be a mapping") + + # The first version of this config key was misnamed as wxyz. Keep it as a + # short-term alias, but IsaacLab's root InitialStateCfg.rot is xyzw here. + raw_quat = robot_config.get("initial_world_orientation_xyzw") + if raw_quat is None: + raw_quat = robot_config.get("initial_world_orientation_wxyz") + quat = _float_tuple( + raw_quat, + name="robot.initial_world_orientation_xyzw", + length=4, + default=(0.0, 0.0, 0.0, 1.0), + ) + norm = sum(value * value for value in quat) ** 0.5 + if norm < 1.0e-8: + raise ValueError( + "Config field 'robot.initial_world_orientation_xyzw' must be non-zero" + ) + return tuple(value / norm for value in quat) + + +def _float_tuple( + values: Any, + *, + name: str, + length: int, + default: Sequence[float], +) -> tuple[float, ...]: + raw = default if values is None else values + if not isinstance(raw, Sequence) or isinstance(raw, (str, bytes)): + raise ValueError(f"Config field {name!r} must be a {length}-element sequence") + if len(raw) != length: + raise ValueError(f"Config field {name!r} must contain exactly {length} values") + return tuple(float(value) for value in raw) + + +def _arm_ik_runtime_config(config: Mapping[str, Any]) -> ArmIkRuntimeConfig: + profile = robot_profile_from_config(config) + robot_config = config.get("robot", {}) + if robot_config is None: + robot_config = {} + if not isinstance(robot_config, Mapping): + raise ValueError("Config field 'robot' must be a mapping") + + # Keep the arm IK body bindings pinned in code rather than mirroring them in + # the teleop YAML. These offsets are asset-specific and tuned together. + sides = { + "left": ArmIkSideConfig( + body_name=profile.ik_body_names["left"], + body_offset_pos=profile.ik_body_offset_pos["left"], + body_offset_quat_xyzw=profile.ik_body_offset_quat_xyzw["left"], + target_orientation_correction_quat_xyzw=( + profile.ik_target_orientation_correction_quat_xyzw["left"] + ), + joint_names=ARM_JOINTS["left"], + ), + "right": ArmIkSideConfig( + body_name=profile.ik_body_names["right"], + body_offset_pos=profile.ik_body_offset_pos["right"], + body_offset_quat_xyzw=profile.ik_body_offset_quat_xyzw["right"], + target_orientation_correction_quat_xyzw=( + profile.ik_target_orientation_correction_quat_xyzw["right"] + ), + joint_names=ARM_JOINTS["right"], + ), + } + + return ArmIkRuntimeConfig( + status_label=profile.status_label, + enabled=True, + marker_scale=None, + marker_z_offset=None, + command_type="pose", + startup_relative_position=True, + startup_relative_orientation=True, + position_error_scale=0.8, + orientation_error_scale=0.8, + position_deadband_m=0.004, + orientation_deadband_rad=0.08, + ik_method="dls", + damping=0.05, + smoothing_alpha=0.80, + clamp_to_joint_limits=True, + sides=sides, + ) + + +def _use_native_hand_targets(config: Mapping[str, Any]) -> bool: + retargeting_config = config.get("retargeting", {}) + if retargeting_config is None: + retargeting_config = {} + if not isinstance(retargeting_config, Mapping): + raise ValueError("Config field 'retargeting' must be a mapping") + + method = retargeting_config.get("method") + if method is not None: + normalized_method = str(method).strip().lower() + aliases = { + "native": "native_dex", + "native_dex": "native_dex", + "dex": "native_dex", + "dexhand": "native_dex", + "dex_hand": "native_dex", + "official": "official_wuji", + "official_wuji": "official_wuji", + "wuji": "official_wuji", + } + normalized_method = aliases.get(normalized_method, normalized_method) + if normalized_method == "native_dex": + return True + if normalized_method == "official_wuji": + return False + raise ValueError( + "Config field 'retargeting.method' must be one of " + "['native_dex', 'official_wuji']" + ) + + robot_config = config.get("robot", {}) + if robot_config is None: + robot_config = {} + if not isinstance(robot_config, Mapping): + raise ValueError("Config field 'robot' must be a mapping") + return bool(robot_config.get("use_native_hand_retargeting", False)) + + +def _load_isaac_lab(): + try: + from isaaclab.app import AppLauncher + except ModuleNotFoundError as exc: + raise ModuleNotFoundError( + "Isaac Lab Python modules were not found. " + "Run this script with Isaac Lab's Python environment, " + "for example /home/lightwheel/workspace/isaaclab/env_isaaclab/bin/python." + ) from exc + + return AppLauncher + + +class DualArmIkController: + """Drive G1 arm joints from left/right end-effector pose targets.""" + + def __init__( + self, + robot: Any, + config: ArmIkRuntimeConfig, + *, + device: str, + pose_binding: AvpRobotFrameBinding | None, + ) -> None: + import torch + import warp as wp + from isaaclab.controllers import DifferentialIKController + from isaaclab.controllers.differential_ik_cfg import DifferentialIKControllerCfg + from isaaclab.utils import math as math_utils + + self._robot = robot + self._config = config + self._pose_binding = pose_binding + self._device = device + self._torch = torch + self._wp = wp + self._math_utils = math_utils + self._controllers: dict[str, Any] = {} + self._joint_ids: dict[str, Sequence[int] | slice] = {} + self._joint_id_lists: dict[str, list[int]] = {} + self._joint_names: dict[str, tuple[str, ...]] = {} + self._body_ids: dict[str, int] = {} + self._body_names: dict[str, str] = {} + self._jacobi_body_ids: dict[str, int] = {} + self._jacobi_joint_ids: dict[str, list[int]] = {} + self._offset_pos: dict[str, Any] = {} + self._offset_rot: dict[str, Any] = {} + self._target_rot_correction: dict[str, Any] = {} + self._last_targets: dict[str, Any] = {} + self._last_target_poses_root: dict[str, tuple[Any, Any]] = {} + self._startup_target_refs: dict[str, tuple[Any, Any, Any, Any]] = {} + self._startup_ref_announced: set[str] = set() + + ik_params = self._ik_params() + for side, side_config in config.sides.items(): + joint_ids, resolved_joint_names = robot.find_joints( + list(side_config.joint_names), preserve_order=True + ) + missing = [ + name + for name in side_config.joint_names + if name not in resolved_joint_names + ] + if missing: + raise RuntimeError( + f"{self._config.status_label} USD is missing {side} arm joints: {missing[:8]}" + ) + + body_ids, body_names = robot.find_bodies(side_config.body_name) + if len(body_ids) != 1: + raise RuntimeError( + f"Expected one {side} IK body match for {side_config.body_name!r}; " + f"got {len(body_ids)}: {body_names}" + ) + + body_idx = int(body_ids[0]) + joint_id_list = [int(joint_id) for joint_id in joint_ids] + if getattr(robot, "is_fixed_base", False): + jacobi_body_idx = body_idx - 1 + jacobi_joint_ids = joint_id_list + else: + jacobi_body_idx = body_idx + jacobi_joint_ids = [joint_id + 6 for joint_id in joint_id_list] + + controller_cfg = DifferentialIKControllerCfg( + command_type=config.command_type, + use_relative_mode=False, + ik_method=config.ik_method, + ik_params=ik_params, + ) + self._controllers[side] = DifferentialIKController( + cfg=controller_cfg, + num_envs=1, + device=device, + ) + self._joint_id_lists[side] = joint_id_list + self._joint_ids[side] = ( + slice(None) if len(joint_id_list) == robot.num_joints else joint_id_list + ) + self._joint_names[side] = tuple(str(name) for name in resolved_joint_names) + self._body_ids[side] = body_idx + self._body_names[side] = str(body_names[0]) + self._jacobi_body_ids[side] = int(jacobi_body_idx) + self._jacobi_joint_ids[side] = jacobi_joint_ids + self._offset_pos[side] = torch.tensor( + side_config.body_offset_pos, + dtype=torch.float32, + device=device, + ).unsqueeze(0) + self._offset_rot[side] = torch.tensor( + side_config.body_offset_quat_xyzw, + dtype=torch.float32, + device=device, + ).unsqueeze(0) + self._target_rot_correction[side] = self._math_utils.quat_unique( + self._math_utils.normalize( + torch.tensor( + side_config.target_orientation_correction_quat_xyzw, + dtype=torch.float32, + device=device, + ).unsqueeze(0) + ) + ) + + def _ik_params(self) -> dict[str, float]: + if self._config.ik_method == "dls": + return {"lambda_val": self._config.damping} + if self._config.ik_method == "pinv": + return {"k_val": 1.0} + if self._config.ik_method == "trans": + return {"k_val": 1.0} + if self._config.ik_method == "svd": + return {"k_val": 1.0, "min_singular_value": 1.0e-5} + return {} + + def _frame_pose_root(self, side: str) -> tuple[Any, Any]: + body_idx = self._body_ids[side] + ee_pos_w = self._wp.to_torch(self._robot.data.body_pos_w)[:, body_idx] + ee_quat_w = self._wp.to_torch(self._robot.data.body_quat_w)[:, body_idx] + root_pos_w = self._wp.to_torch(self._robot.data.root_pos_w) + root_quat_w = self._wp.to_torch(self._robot.data.root_quat_w) + ee_pos_b, ee_quat_b = self._math_utils.subtract_frame_transforms( + root_pos_w, root_quat_w, ee_pos_w, ee_quat_w + ) + return self._math_utils.combine_frame_transforms( + ee_pos_b, + ee_quat_b, + self._offset_pos[side], + self._offset_rot[side], + ) + + def frame_pose_world_np(self) -> dict[str, tuple[np.ndarray, np.ndarray]]: + poses: dict[str, tuple[np.ndarray, np.ndarray]] = {} + for side in ("left", "right"): + body_idx = self._body_ids[side] + body_pos = self._wp.to_torch(self._robot.data.body_pos_w)[:, body_idx] + body_quat = self._wp.to_torch(self._robot.data.body_quat_w)[:, body_idx] + pos_w, quat_w = self._math_utils.combine_frame_transforms( + body_pos, + body_quat, + self._offset_pos[side], + self._offset_rot[side], + ) + rot_w = self._math_utils.matrix_from_quat(quat_w) + poses[side] = ( + pos_w[0].detach().cpu().numpy().astype(np.float32), + rot_w[0].detach().cpu().numpy().astype(np.float32), + ) + return poses + + def target_pose_world_np(self) -> dict[str, tuple[np.ndarray, np.ndarray]]: + poses: dict[str, tuple[np.ndarray, np.ndarray]] = {} + root_pos_w = self._wp.to_torch(self._robot.data.root_pos_w) + root_quat_w = self._wp.to_torch(self._robot.data.root_quat_w) + for side, target_pose_root in self._last_target_poses_root.items(): + pos_b, quat_b = target_pose_root + pos_w, quat_w = self._math_utils.combine_frame_transforms( + root_pos_w, + root_quat_w, + pos_b, + quat_b, + ) + rot_w = self._math_utils.matrix_from_quat(quat_w) + poses[side] = ( + pos_w[0].detach().cpu().numpy().astype(np.float32), + rot_w[0].detach().cpu().numpy().astype(np.float32), + ) + return poses + + def _target_pose_root( + self, side: str, pose: Mapping[str, Any] + ) -> tuple[Any, Any] | None: + if self._pose_binding is not None: + target_pose = self._pose_binding.transform_pose(side, pose) + if target_pose is None: + return None + pos_w_np, quat_w_np = target_pose + else: + pos_w_np, quat_w_np = openxr_pose_to_isaac_pose( + pose, + scale=1.0 + if self._config.marker_scale is None + else self._config.marker_scale, + offset_z=0.0 + if self._config.marker_z_offset is None + else self._config.marker_z_offset, + ) + return self._target_pose_root_from_world_pose( + pos_w_np, + quat_w_np, + side=side, + apply_target_rot_correction=True, + ) + + def _target_pose_root_from_world_pose( + self, + pos_w_np: Sequence[float] | np.ndarray, + quat_w_np: Sequence[float] | np.ndarray, + *, + side: str | None = None, + apply_target_rot_correction: bool = False, + ) -> tuple[Any, Any]: + pos_w = self._torch.as_tensor( + pos_w_np, + dtype=self._torch.float32, + device=self._device, + ).unsqueeze(0) + quat_w = self._torch.as_tensor( + normalize_quat_xyzw(quat_w_np), + dtype=self._torch.float32, + device=self._device, + ).unsqueeze(0) + if apply_target_rot_correction: + if side is None: + raise ValueError( + "side is required when applying target orientation correction" + ) + quat_w = self._math_utils.normalize( + self._math_utils.quat_mul(quat_w, self._target_rot_correction[side]) + ) + else: + quat_w = self._math_utils.normalize(quat_w) + quat_w = self._math_utils.quat_unique(quat_w) + root_pos_w = self._wp.to_torch(self._robot.data.root_pos_w) + root_quat_w = self._wp.to_torch(self._robot.data.root_quat_w) + return self._math_utils.subtract_frame_transforms( + root_pos_w, root_quat_w, pos_w, quat_w + ) + + def _startup_relative_target( + self, + side: str, + target_pos: Any, + target_quat: Any, + ee_pos_curr: Any, + ee_quat_curr: Any, + ) -> tuple[Any, Any]: + if not ( + self._config.startup_relative_position + or self._config.startup_relative_orientation + ): + return target_pos, target_quat + + ref = self._startup_target_refs.get(side) + if ref is None: + ref = ( + target_pos.detach().clone(), + self._math_utils.quat_unique( + self._math_utils.normalize(target_quat.detach().clone()) + ), + ee_pos_curr.detach().clone(), + self._math_utils.quat_unique( + self._math_utils.normalize(ee_quat_curr.detach().clone()) + ), + ) + self._startup_target_refs[side] = ref + self._announce_startup_relative_target(side, ref) + + target_start_pos, target_start_quat, ee_start_pos, ee_start_quat = ref + adjusted_pos = target_pos + adjusted_quat = target_quat + + # AVP optical wrists and the robot EE rarely share a reliable absolute + # pose. Bind the first valid target to the current simulated EE, then + # command startup-relative translation and rotation deltas. + if self._config.startup_relative_position: + adjusted_pos = ee_start_pos + (target_pos - target_start_pos) + if self._config.startup_relative_orientation: + target_quat = self._math_utils.quat_unique( + self._math_utils.normalize(target_quat) + ) + quat_delta = self._math_utils.quat_mul( + target_quat, + self._math_utils.quat_inv(target_start_quat), + ) + adjusted_quat = self._math_utils.normalize( + self._math_utils.quat_mul(quat_delta, ee_start_quat) + ) + adjusted_quat = self._math_utils.quat_unique(adjusted_quat) + return adjusted_pos, adjusted_quat + + def _announce_startup_relative_target( + self, + side: str, + ref: tuple[Any, Any, Any, Any], + ) -> None: + if side in self._startup_ref_announced: + return + target_start_pos, target_start_quat, ee_start_pos, ee_start_quat = ref + print( + f"[{self._config.status_label}] {side} IK startup target reference calibrated " + f"relative_position={self._config.startup_relative_position} " + f"relative_orientation={self._config.startup_relative_orientation} " + f"target_pos={np.round(target_start_pos[0].detach().cpu().numpy(), 4).tolist()} " + f"ee_pos={np.round(ee_start_pos[0].detach().cpu().numpy(), 4).tolist()} " + f"target_quat_xyzw={np.round(target_start_quat[0].detach().cpu().numpy(), 4).tolist()} " + f"ee_quat_xyzw={np.round(ee_start_quat[0].detach().cpu().numpy(), 4).tolist()}", + flush=True, + ) + self._startup_ref_announced.add(side) + + def _compute_delta_joint_pos(self, delta_pose: Any, jacobian: Any) -> Any: + if self._config.ik_method == "pinv": + jacobian_pinv = self._torch.linalg.pinv(jacobian) + return (jacobian_pinv @ delta_pose.unsqueeze(-1)).squeeze(-1) + if self._config.ik_method == "svd": + min_singular_value = 1.0e-5 + U, S, Vh = self._torch.linalg.svd(jacobian) + S_inv = 1.0 / S + S_inv = self._torch.where( + min_singular_value < S, + S_inv, + self._torch.zeros_like(S_inv), + ) + jacobian_pinv = ( + self._torch.transpose(Vh, dim0=1, dim1=2)[:, :, : jacobian.shape[1]] + @ self._torch.diag_embed(S_inv) + @ self._torch.transpose(U, dim0=1, dim1=2) + ) + return (jacobian_pinv @ delta_pose.unsqueeze(-1)).squeeze(-1) + if self._config.ik_method == "trans": + jacobian_t = self._torch.transpose(jacobian, dim0=1, dim1=2) + return (jacobian_t @ delta_pose.unsqueeze(-1)).squeeze(-1) + if self._config.ik_method == "dls": + jacobian_t = self._torch.transpose(jacobian, dim0=1, dim1=2) + lambda_val = self._config.damping + lambda_matrix = (lambda_val**2) * self._torch.eye( + n=jacobian.shape[1], + device=self._device, + ) + return ( + jacobian_t + @ self._torch.inverse(jacobian @ jacobian_t + lambda_matrix) + @ delta_pose.unsqueeze(-1) + ).squeeze(-1) + raise ValueError( + f"Unsupported inverse-kinematics method: {self._config.ik_method}" + ) + + def _compute_joint_targets( + self, + side: str, + ee_pos_curr: Any, + ee_quat_curr: Any, + joint_pos: Any, + ) -> Any: + jacobian = self._frame_jacobian(side) + controller = self._controllers[side] + if self._config.command_type == "position": + return controller.compute( + ee_pos_curr, + ee_quat_curr, + jacobian, + joint_pos, + ) + + position_error, axis_angle_error = self._math_utils.compute_pose_error( + ee_pos_curr, + ee_quat_curr, + controller.ee_pos_des, + controller.ee_quat_des, + rot_error_type="axis_angle", + ) + if self._config.position_deadband_m > 0.0: + position_norm = self._torch.linalg.norm(position_error, dim=1, keepdim=True) + position_error = self._torch.where( + position_norm <= self._config.position_deadband_m, + self._torch.zeros_like(position_error), + position_error, + ) + if self._config.orientation_deadband_rad > 0.0: + orientation_norm = self._torch.linalg.norm( + axis_angle_error, dim=1, keepdim=True + ) + axis_angle_error = self._torch.where( + orientation_norm <= self._config.orientation_deadband_rad, + self._torch.zeros_like(axis_angle_error), + axis_angle_error, + ) + # Pose-mode wrist targets are much noisier in orientation than in translation. + # Down-weight angular error so small AVP wrist rotations do not yank the whole + # arm into a large reconfiguration before the translational target is reached. + position_error = position_error * self._config.position_error_scale + axis_angle_error = axis_angle_error * self._config.orientation_error_scale + pose_error = self._torch.cat((position_error, axis_angle_error), dim=1) + if bool(self._torch.all(self._torch.abs(pose_error) <= 1.0e-9)): + return joint_pos + delta_joint_pos = self._compute_delta_joint_pos(pose_error, jacobian) + return joint_pos + delta_joint_pos + + def _frame_jacobian(self, side: str): + jacobian = self._wp.to_torch(self._robot.root_physx_view.get_jacobians())[ + :, self._jacobi_body_ids[side], :, self._jacobi_joint_ids[side] + ].clone() + root_quat_w = self._wp.to_torch(self._robot.data.root_quat_w) + root_rot_b_w = self._math_utils.matrix_from_quat( + self._math_utils.quat_inv(root_quat_w) + ) + jacobian[:, 0:3, :] = self._torch.bmm(root_rot_b_w, jacobian[:, 0:3, :]) + jacobian[:, 3:, :] = self._torch.bmm(root_rot_b_w, jacobian[:, 3:, :]) + offset_pos = self._offset_pos[side] + offset_rot = self._offset_rot[side] + jacobian[:, 0:3, :] += self._torch.bmm( + -self._math_utils.skew_symmetric_matrix(offset_pos), + jacobian[:, 3:, :], + ) + jacobian[:, 3:, :] = self._torch.bmm( + self._math_utils.matrix_from_quat(offset_rot), + jacobian[:, 3:, :], + ) + return jacobian + + def _apply_target_pose_root( + self, + side: str, + target_pose_root: tuple[Any, Any], + ) -> None: + ee_pos_curr, ee_quat_curr = self._frame_pose_root(side) + target_pos, target_quat = target_pose_root + target_pos, target_quat = self._startup_relative_target( + side, + target_pos, + target_quat, + ee_pos_curr, + ee_quat_curr, + ) + command = ( + target_pos + if self._config.command_type == "position" + else self._torch.cat((target_pos, target_quat), dim=1) + ) + self._controllers[side].set_command(command, ee_pos_curr, ee_quat_curr) + self._last_target_poses_root[side] = ( + self._controllers[side].ee_pos_des.detach().clone(), + self._controllers[side].ee_quat_des.detach().clone(), + ) + joint_pos = self._wp.to_torch(self._robot.data.joint_pos)[ + :, self._joint_ids[side] + ] + joint_targets = self._compute_joint_targets( + side, + ee_pos_curr, + ee_quat_curr, + joint_pos, + ) + + if self._config.clamp_to_joint_limits: + limits = self._wp.to_torch(self._robot.data.soft_joint_pos_limits)[ + :, self._joint_ids[side], : + ] + joint_targets = self._torch.clamp( + joint_targets, limits[:, :, 0], limits[:, :, 1] + ) + + previous = self._last_targets.get(side) + if previous is not None and self._config.smoothing_alpha < 1.0: + alpha = self._config.smoothing_alpha + joint_targets = previous * (1.0 - alpha) + joint_targets * alpha + + self._last_targets[side] = joint_targets + self._robot.set_joint_position_target_index( + target=joint_targets, + joint_ids=self._joint_ids[side], + ) + + def update(self, sample: Mapping[str, Any]) -> None: + if not self._config.enabled: + return + + for side in ("left", "right"): + pose = sample["ee_poses"].get(side) + if pose is None: + target = self._last_targets.get(side) + if target is not None: + self._robot.set_joint_position_target_index( + target=target, + joint_ids=self._joint_ids[side], + ) + continue + + target_pose = self._target_pose_root(side, pose) + if target_pose is None: + continue + self._apply_target_pose_root(side, target_pose) + + def reset_startup_references(self) -> None: + self._startup_target_refs.clear() + self._startup_ref_announced.clear() + self._last_target_poses_root.clear() + print( + f"[{self._config.status_label}] arm IK startup target references cleared; " + "waiting for re-arm calibration.", + flush=True, + ) + + +class WholeBodyYawController: + """Rotate the robot base with the calibrated head yaw around the AVP z-axis.""" + + def __init__( + self, + robot: Any, + config: AvpFrameBindingRuntimeConfig, + *, + device: str, + ) -> None: + import torch + import warp as wp + + self._robot = robot + self._config = config + self._device = device + self._torch = torch + self._wp = wp + self._joint_id, self._joint_name = self._find_base_yaw_joint() + self._joint_limits = self._joint_limits_np() + self._reference_target = self._current_joint_pos() + self._last_target = self._reference_target + print( + f"[{self._config.status_label}] whole-body yaw following active: " + f"joint={self._joint_name!r}, " + f"deadband_rad={self._config.whole_body_yaw_deadband_rad:.3f}, " + f"smoothing_alpha={self._config.whole_body_yaw_smoothing_alpha:.2f}.", + flush=True, + ) + + @property + def joint_id(self) -> int: + return self._joint_id + + def _find_base_yaw_joint(self) -> tuple[int, str]: + available = tuple( + str(name) for name in (getattr(self._robot.data, "joint_names", ()) or ()) + ) + for candidate in ("base_yaw", "base_yaw_joint"): + if candidate in available: + return available.index(candidate), candidate + + fuzzy = [ + (index, name) + for index, name in enumerate(available) + if "base" in name.lower() and "yaw" in name.lower() + ] + if len(fuzzy) == 1: + return fuzzy[0] + + raise RuntimeError( + f"Could not resolve the {self._config.status_label} base yaw joint. " + f"Candidates={fuzzy[:8]} available_first={list(available[:40])}" + ) + + def _joint_limits_np(self) -> tuple[float, float] | None: + try: + limits = self._wp.to_torch(self._robot.data.soft_joint_pos_limits) + values = limits[0, self._joint_id].detach().cpu().numpy() + except Exception: + return None + if np.asarray(values).shape != (2,): + return None + return float(values[0]), float(values[1]) + + def _current_joint_pos(self) -> float: + joint_pos = self._wp.to_torch(self._robot.data.joint_pos) + return float(joint_pos[0, self._joint_id].detach().cpu().item()) + + def _apply_target(self, target_value: float) -> None: + target_tensor = self._torch.tensor( + [[float(target_value)]], + dtype=self._torch.float32, + device=self._device, + ) + self._robot.set_joint_position_target_index( + target=target_tensor, + joint_ids=[self._joint_id], + ) + + def reset_reference(self) -> None: + current = self._current_joint_pos() + self._reference_target = current + self._last_target = current + self._apply_target(current) + + def hold_current_target(self) -> None: + self._apply_target(self._last_target) + + def current_yaw_delta_rad(self) -> float: + return float(self._current_joint_pos() - self._reference_target) + + def update(self, pose_binding: AvpRobotFrameBinding | None) -> None: + if pose_binding is None or not pose_binding.whole_body_yaw_following_enabled: + return + + target_value = self._reference_target + pose_binding.latest_head_yaw_delta_rad() + if self._joint_limits is not None: + target_value = float( + np.clip(target_value, self._joint_limits[0], self._joint_limits[1]) + ) + + alpha = self._config.whole_body_yaw_smoothing_alpha + target_value = self._last_target * (1.0 - alpha) + target_value * alpha + self._last_target = float(target_value) + self._apply_target(self._last_target) + + +def _find_required_joint_ids( + robot: Any, + joint_names: Sequence[str], + side: str, + *, + status_label: str, +) -> list[int]: + joint_ids, resolved_names = robot.find_joints( + list(joint_names), preserve_order=True + ) + missing = [name for name in joint_names if name not in resolved_names] + if missing: + available = list(getattr(robot.data, "joint_names", []) or []) + raise RuntimeError( + f"{status_label} USD is missing {side} hand joints. " + f"Missing first entries: {missing[:8]}; available count={len(available)}" + ) + return list(joint_ids) + + +def _configured_initial_positions( + joint_names: Sequence[str], config: WujiHandRuntimeConfig +) -> np.ndarray: + return np.asarray( + [ + config.initial_joint_positions.get(name, config.initial_joint_position) + for name in joint_names + ], + dtype=np.float32, + ) + + +def _default_joint_positions( + robot: Any, + joint_ids: Sequence[int], + joint_names: Sequence[str], + config: WujiHandRuntimeConfig, +) -> np.ndarray: + try: + import warp as wp + + default_joint_pos = wp.to_torch(robot.data.default_joint_pos) + values = default_joint_pos[0, list(joint_ids)].detach().cpu().numpy() + return values.astype(np.float32) + except Exception: + return _configured_initial_positions(joint_names, config) + + +def _soft_joint_limits(robot: Any, joint_ids: Sequence[int]) -> np.ndarray | None: + try: + import warp as wp + + limits = wp.to_torch(robot.data.soft_joint_pos_limits) + values = limits[0, list(joint_ids), :].detach().cpu().numpy() + if values.shape != (len(joint_ids), 2): + return None + return values.astype(np.float32) + except Exception: + return None + + +def _parse_args(argv: Sequence[str]) -> argparse.Namespace: + AppLauncher = _load_isaac_lab() + parser = argparse.ArgumentParser( + description="Run a minimal G1 Isaac Lab scene driven by AVP + MANUS teleop." + ) + parser.add_argument("--config", default=str(_default_config_path())) + parser.add_argument("--robot-usd", default=None) + parser.add_argument("--robot-prim", default=None) + parser.add_argument("--duration-s", type=float, default=0.0) + parser.add_argument("--wrist-marker-scale", type=float, default=1.0) + parser.add_argument("--wrist-marker-z-offset", type=float, default=0.0) + parser.add_argument("--wrist-marker-radius", type=float, default=0.045) + parser.add_argument("--wrist-axis-length", type=float, default=0.18) + parser.add_argument("--wrist-axis-radius", type=float, default=0.008) + parser.add_argument( + "--robot-height", + type=float, + default=None, + help="Optional override for robot.initial_world_position z.", + ) + parser.add_argument("--light-intensity", type=float, default=3000.0) + parser.add_argument( + "--scene-only", + action="store_true", + help="Load the minimal scene and robot without starting OpenXR/MANUS teleop.", + ) + AppLauncher.add_app_launcher_args(parser) + return parser.parse_args(argv) + + +def main(argv: Sequence[str] | None = None) -> int: + ensure_import_paths() + args = _parse_args(sys.argv[1:] if argv is None else argv) + config_path = Path(args.config).expanduser().resolve() + teleop_config = _load_yaml_file(config_path) + teleop_config["_config_dir"] = str(config_path.parent) + robot_profile = robot_profile_from_config(teleop_config) + status_label = robot_profile.status_label + use_native_hand_targets = _use_native_hand_targets(teleop_config) + head_view_camera_config = _head_view_camera_config(teleop_config) + head_view_xr_display_config = _head_view_xr_display_config(teleop_config) + auxiliary_viewport_config = _auxiliary_viewport_config(teleop_config) + initial_world_position = _robot_initial_world_position(teleop_config) + initial_world_orientation_xyzw = _robot_initial_world_orientation_xyzw( + teleop_config + ) + wuji_hand_config = wuji_hand_runtime_config(teleop_config) + arm_ik_config = _arm_ik_runtime_config(teleop_config) + avp_frame_binding_config = avp_frame_binding_runtime_config(teleop_config) + arm_ik_config = dataclass_replace( + arm_ik_config, + marker_scale=( + args.wrist_marker_scale + if arm_ik_config.marker_scale is None + else arm_ik_config.marker_scale + ), + marker_z_offset=( + args.wrist_marker_z_offset + if arm_ik_config.marker_z_offset is None + else arm_ik_config.marker_z_offset + ), + ) + if args.robot_height is not None: + initial_world_position = ( + initial_world_position[0], + initial_world_position[1], + float(args.robot_height), + ) + robot_usd = ( + Path(args.robot_usd).expanduser().resolve() + if args.robot_usd + else resolve_repo_relative_path(robot_profile.default_usd_relpath) + ) + if not robot_usd.is_file(): + raise FileNotFoundError(f"{status_label} USD not found: {robot_usd}") + robot_prim = str(args.robot_prim or robot_profile.robot_prim) + + AppLauncher = _load_isaac_lab() + app_launcher = AppLauncher(args) + simulation_app = app_launcher.app + _ensure_cloudxr_runtime_environment_defaults() + + import carb + import omni.usd + import torch + from isaaclab.markers import VisualizationMarkers, VisualizationMarkersCfg + from pxr import Gf, UsdGeom + + import isaaclab.sim as sim_utils + from isaaclab.sim import SimulationContext + + _ensure_carb_input_available() + omni_appwindow = _import_omni_appwindow() + omni_viewport_utility = _import_omni_viewport_utility() + + sim_cfg = sim_utils.SimulationCfg(dt=1.0 / 60.0, device=args.device) + sim = SimulationContext(sim_cfg) + sim.set_camera_view([3.0, -4.0, 2.0], [0.0, 0.0, 0.9]) + + scene_config = G1WujiSceneConfig( + robot_prim=robot_prim, + robot_usd=robot_usd, + initial_world_position=initial_world_position, + initial_world_orientation_xyzw=initial_world_orientation_xyzw, + initial_hand_joint_positions=wuji_hand_config.initial_joint_positions, + light_intensity=args.light_intensity, + ) + robot = design_g1_wuji_scene(scene_config) + debug_viz_config = debug_viz_runtime_config(teleop_config) + debug_viz = create_debug_viz( + sim_utils=sim_utils, + VisualizationMarkers=VisualizationMarkers, + VisualizationMarkersCfg=VisualizationMarkersCfg, + scene_config=DebugVizSceneConfig( + wrist_marker_radius=args.wrist_marker_radius, + wrist_axis_length=args.wrist_axis_length, + wrist_axis_radius=args.wrist_axis_radius, + robot_z=float(initial_world_position[2]), + ), + runtime_config=debug_viz_config, + ) + + sim.reset() + sim_dt = sim.get_physics_dt() + left_joint_ids = _find_required_joint_ids( + robot, + wuji_hand_config.left_joint_names, + "left", + status_label=status_label, + ) + right_joint_ids = _find_required_joint_ids( + robot, + wuji_hand_config.right_joint_names, + "right", + status_label=status_label, + ) + left_initial_targets = _default_joint_positions( + robot, left_joint_ids, wuji_hand_config.left_joint_names, wuji_hand_config + ) + right_initial_targets = _default_joint_positions( + robot, right_joint_ids, wuji_hand_config.right_joint_names, wuji_hand_config + ) + left_postprocessor = HandTargetPostProcessor( + joint_names=wuji_hand_config.left_joint_names, + initial_targets=left_initial_targets, + joint_limits=_soft_joint_limits(robot, left_joint_ids), + config=wuji_hand_config, + side="left", + ) + right_postprocessor = HandTargetPostProcessor( + joint_names=wuji_hand_config.right_joint_names, + initial_targets=right_initial_targets, + joint_limits=_soft_joint_limits(robot, right_joint_ids), + config=wuji_hand_config, + side="right", + ) + wuji_backend = ( + None if use_native_hand_targets else WujiHandTargetBackend(wuji_hand_config) + ) + avp_frame_binding = ( + AvpRobotFrameBinding( + robot, + arm_ik_config, + avp_frame_binding_config, + marker_scale=1.0 + if arm_ik_config.marker_scale is None + else arm_ik_config.marker_scale, + marker_z_offset=0.0 + if arm_ik_config.marker_z_offset is None + else arm_ik_config.marker_z_offset, + ) + if avp_frame_binding_config.enabled + else None + ) + whole_body_yaw = ( + WholeBodyYawController( + robot, + avp_frame_binding_config, + device=sim.device, + ) + if avp_frame_binding_config.enabled + and avp_frame_binding_config.whole_body_yaw_following + else None + ) + arm_ik = ( + DualArmIkController( + robot, + arm_ik_config, + device=sim.device, + pose_binding=avp_frame_binding, + ) + if arm_ik_config.enabled + else None + ) + stage = omni.usd.get_context().get_stage() + robot_root_prim = stage.GetPrimAtPath(robot_prim) if stage is not None else None + _disable_robot_transmissive_materials( + stage=stage, + robot_root_prim=robot_root_prim, + status_label=status_label, + ) + head_link_prim = ( + find_head_like_prim(stage, robot_root_prim, robot_prim) + if robot_root_prim is not None and robot_root_prim.IsValid() + else None + ) + if head_link_prim is None: + head_candidates = [] + if robot_root_prim is not None and robot_root_prim.IsValid(): + head_candidates = [ + str(prim.GetPath()) + for prim in walk_child_prims(robot_root_prim) + if "head" in prim.GetName().lower() + ][:8] + print( + f"[{status_label}] warning: could not find a head-like prim under {robot_prim}; " + f"candidates={head_candidates}.", + flush=True, + ) + else: + print( + f"[{status_label}] head point visualization uses prim {head_link_prim.GetPath()}", + flush=True, + ) + xform_cache = UsdGeom.XformCache() + viewport_api = omni_viewport_utility.get_active_viewport() + default_viewport_camera_path = ( + str(viewport_api.camera_path) + if viewport_api is not None + and getattr(viewport_api, "camera_path", None) is not None + else None + ) + auxiliary_viewport = AuxiliaryViewportController( + config=auxiliary_viewport_config, + viewport_utility=omni_viewport_utility, + initial_camera_path=default_viewport_camera_path, + ) + head_view_camera = ( + HeadViewCameraController( + stage=stage, + UsdGeom=UsdGeom, + Gf=Gf, + viewport_api=viewport_api, + config=head_view_camera_config, + head_link_prim=head_link_prim, + avp_frame_binding=avp_frame_binding, + whole_body_yaw=whole_body_yaw, + ) + if head_view_camera_config.enabled and stage is not None + else None + ) + teleop: TeleopMain | None = None + teleop_started = False + head_view_xr_display: HeadViewXrDisplayBridge | None = None + scene_warmed_up = False + xr_waiting_announced = False + next_xr_retry_time = 0.0 + if head_view_xr_display_config.enabled and not args.scene_only: + if head_view_camera is None: + raise ValueError( + "robot.head_view_xr_display.enabled requires robot.head_view_camera.enabled " + "so there is a camera prim to publish into AVP." + ) + import warp as wp + + robot_default_joint_pos = wp.to_torch(robot.data.default_joint_pos).clone() + robot_default_joint_vel = wp.to_torch(robot.data.default_joint_vel).clone() + robot.set_joint_position_target_index( + target=torch.as_tensor(left_initial_targets, device=sim.device).unsqueeze(0), + joint_ids=left_joint_ids, + ) + robot.set_joint_position_target_index( + target=torch.as_tensor(right_initial_targets, device=sim.device).unsqueeze(0), + joint_ids=right_joint_ids, + ) + + print( + f"{status_label} teleop scene ready: " + f"{len(left_joint_ids)} left hand joints, {len(right_joint_ids)} right hand joints, " + f"hand_retarget={'native_dex' if use_native_hand_targets else wuji_hand_config.retarget_backend}, " + f"arm_ik={arm_ik_config.enabled}, " + f"avp_frame_binding={avp_frame_binding_config.enabled}, " + f"whole_body_yaw={whole_body_yaw is not None}, " + f"head_view_camera={head_view_camera is not None}, " + f"head_view_xr_display={head_view_xr_display_config.enabled and not args.scene_only}, " + f"auxiliary_viewport={auxiliary_viewport_config.enabled and not args.scene_only}." + ) + + def step_scene_once( + *, + target_poses: Mapping[str, tuple[np.ndarray, np.ndarray]] | None = None, + update_ee_frames: bool = False, + ) -> None: + if target_poses is not None: + update_wrist_markers( + debug_viz.wrist_markers, + debug_viz.wrist_frames, + target_poses, + args.wrist_axis_length, + ) + robot.write_data_to_sim() + sim.step(render=sim.is_rendering) + robot.update(sim_dt) + if head_view_camera is not None: + head_view_camera.update(xform_cache=xform_cache) + if head_view_xr_display is not None: + head_view_xr_display.submit_frame() + update_head_marker( + debug_viz, + head_link_prim=head_link_prim, + xform_cache=xform_cache, + ) + if update_ee_frames and arm_ik is not None: + ee_frame_poses = arm_ik.frame_pose_world_np() + update_frame_axis_markers( + debug_viz.robot_ee_frames, + ee_frame_poses, + args.wrist_axis_length, + ) + + if args.scene_only: + print("Scene-only mode: OpenXR/MANUS teleop is not started.") + start_time = time.monotonic() + try: + while simulation_app.is_running(): + if ( + args.duration_s > 0.0 + and time.monotonic() - start_time >= args.duration_s + ): + break + step_scene_once() + finally: + simulation_app.close() + return 0 + + if debug_viz_config.enabled: + print( + "Debug visualization is enabled and controlled by robot.debug_visualization." + ) + + marker_target_poses: dict[str, tuple[np.ndarray, np.ndarray]] = {} + teleop_armed = False + start_waiting_announced = False + keyboard_start_requested = False + keyboard_reset_requested = False + + app_window = omni_appwindow.get_default_app_window() + input_interface = carb.input.acquire_input_interface() + + def on_keyboard_event(event, *args, **kwargs): + nonlocal keyboard_reset_requested, keyboard_start_requested + if event.type == carb.input.KeyboardEventType.KEY_PRESS: + if event.input == carb.input.KeyboardInput.B: + keyboard_start_requested = True + elif event.input == carb.input.KeyboardInput.R: + keyboard_reset_requested = True + return True + + keyboard_subscription = ( + input_interface.subscribe_to_keyboard_events( + app_window.get_keyboard(), + on_keyboard_event, + ) + if app_window is not None and app_window.get_keyboard() is not None + else None + ) + + def hold_robot_targets() -> None: + robot.set_joint_position_target_index( + target=torch.as_tensor(left_initial_targets, device=sim.device).unsqueeze( + 0 + ), + joint_ids=left_joint_ids, + ) + robot.set_joint_position_target_index( + target=torch.as_tensor(right_initial_targets, device=sim.device).unsqueeze( + 0 + ), + joint_ids=right_joint_ids, + ) + if whole_body_yaw is not None: + whole_body_yaw.hold_current_target() + + def restore_robot_initial_pose() -> None: + if hasattr(robot, "write_joint_state_to_sim"): + robot.write_joint_state_to_sim( + robot_default_joint_pos, + robot_default_joint_vel, + ) + robot.set_joint_position_target_index( + target=robot_default_joint_pos, + ) + if whole_body_yaw is not None: + whole_body_yaw.reset_reference() + if arm_ik is not None: + arm_ik._last_targets.clear() + + def reset_teleop_calibration() -> None: + if avp_frame_binding is not None: + avp_frame_binding.reset_calibration() + if whole_body_yaw is not None: + whole_body_yaw.reset_reference() + if arm_ik is not None: + arm_ik.reset_startup_references() + if head_view_camera is not None: + head_view_camera.deactivate() + if viewport_api is not None and default_viewport_camera_path: + viewport_api.camera_path = default_viewport_camera_path + if auxiliary_viewport_config.activate_on_calibration: + auxiliary_viewport.hide() + if ( + head_view_xr_display is not None + and head_view_xr_display_config.activate_on_calibration + ): + head_view_xr_display.set_visible(False) + + def marker_pose_resolver( + side: str, + pose: Mapping[str, Any], + ) -> tuple[np.ndarray, np.ndarray] | None: + target_pose = marker_target_poses.get(side) + if target_pose is not None: + return target_pose + return resolve_avp_marker_pose( + side, + pose, + marker_scale=args.wrist_marker_scale, + marker_z_offset=args.wrist_marker_z_offset, + pose_binding=avp_frame_binding, + ) + + def close_openxr_runtime() -> None: + nonlocal teleop, teleop_started, head_view_xr_display + if teleop_started and teleop is not None: + try: + teleop.__exit__(None, None, None) + except Exception: + pass + teleop = None + teleop_started = False + if head_view_xr_display is not None: + try: + head_view_xr_display.close() + except Exception: + pass + head_view_xr_display = None + auxiliary_viewport.close() + + def enter_xr_waiting_state(exc: BaseException, *, phase: str) -> None: + nonlocal teleop_armed, start_waiting_announced, marker_target_poses + nonlocal xr_waiting_announced, next_xr_retry_time + close_openxr_runtime() + teleop_armed = False + start_waiting_announced = False + marker_target_poses = {} + restore_robot_initial_pose() + reset_teleop_calibration() + next_xr_retry_time = time.monotonic() + _XR_RETRY_INTERVAL_S + if not xr_waiting_announced: + print( + f"[{status_label}] OpenXR {phase} is not ready: " + f"{_format_exception_chain(exc)}. " + "The scene will keep running while waiting for the HMD/runtime. " + "Connect the headset or start the XR runtime, then wait for the " + "automatic retry or press keyboard B to retry immediately.", + flush=True, + ) + xr_waiting_announced = True + + def try_start_openxr_runtime() -> bool: + nonlocal teleop, teleop_started, head_view_xr_display + nonlocal xr_waiting_announced, next_xr_retry_time + was_waiting = xr_waiting_announced + candidate_teleop = TeleopMain.from_config_file(config_path) + candidate_head_view_xr_display: HeadViewXrDisplayBridge | None = None + try: + if head_view_xr_display_config.enabled: + head_view_preflight_error = _head_view_xr_display_preflight_error( + head_view_camera.camera_path + ) + if head_view_preflight_error is not None: + print( + f"[{status_label}] warning: disabling AVP head-view XR display " + "for this session because direct camera rendering is unavailable " + f"({head_view_preflight_error}).", + flush=True, + ) + else: + try: + candidate_head_view_xr_display = HeadViewXrDisplayBridge( + config=head_view_xr_display_config, + camera_prim_path=head_view_camera.camera_path, + required_xr_extensions=candidate_teleop.required_xr_extensions, + sim_device=str(args.device), + ) + candidate_teleop.set_oxr_handles( + candidate_head_view_xr_display.oxr_handles + ) + if ( + not head_view_xr_display_config.activate_on_calibration + or not avp_frame_binding_config.enabled + ): + candidate_head_view_xr_display.set_visible(True) + except Exception as exc: + if candidate_head_view_xr_display is not None: + try: + candidate_head_view_xr_display.close() + except Exception: + pass + candidate_head_view_xr_display = None + if _is_xr_runtime_unavailable_error(exc): + raise + candidate_teleop.set_oxr_handles(None) + print( + f"[{status_label}] warning: disabling AVP head-view XR display " + f"for this session because startup failed " + f"({_format_exception_chain(exc)}).", + flush=True, + ) + candidate_teleop.__enter__() + except Exception as exc: + try: + candidate_teleop.__exit__(None, None, None) + except Exception: + pass + if candidate_head_view_xr_display is not None: + try: + candidate_head_view_xr_display.close() + except Exception: + pass + if _is_xr_runtime_unavailable_error(exc): + enter_xr_waiting_state(exc, phase="startup") + return False + raise + + teleop = candidate_teleop + teleop_started = True + head_view_xr_display = candidate_head_view_xr_display + next_xr_retry_time = 0.0 + xr_waiting_announced = False + if head_view_xr_display is not None: + print( + f"[{status_label}] AVP head-view XR display enabled: " + f"resolution={head_view_xr_display_config.resolution_px[0]}x" + f"{head_view_xr_display_config.resolution_px[1]}, " + f"lock_mode={head_view_xr_display_config.lock_mode}, " + f"activate_on_calibration={head_view_xr_display_config.activate_on_calibration}.", + flush=True, + ) + print( + f"[{status_label}] OpenXR teleop session started" + f"{' after HMD/runtime reconnect' if was_waiting else ''}.", + flush=True, + ) + return True + + start_time = time.monotonic() + try: + if not simulation_app.is_running(): + has_keyboard = bool( + app_window is not None and app_window.get_keyboard() is not None + ) + print( + f"[{status_label}] simulation app is not running at teleop loop start; " + "exiting before the first teleop frame. " + f"headless={getattr(args, 'headless', None)} " + f"app_window={app_window is not None} " + f"keyboard={has_keyboard}", + flush=True, + ) + while simulation_app.is_running(): + if ( + args.duration_s > 0.0 + and time.monotonic() - start_time >= args.duration_s + ): + break + + start_button_rising = keyboard_start_requested + keyboard_start_requested = False + reset_button_rising = keyboard_reset_requested + keyboard_reset_requested = False + + if reset_button_rising: + teleop_armed = False + start_waiting_announced = False + marker_target_poses = {} + restore_robot_initial_pose() + reset_teleop_calibration() + print( + f"[{status_label}] teleop reset; robot returned to its initial joint pose. " + "Press keyboard B to recalibrate and start again.", + flush=True, + ) + + if not scene_warmed_up: + hold_robot_targets() + step_scene_once() + scene_warmed_up = True + continue + + if teleop is None: + should_retry = ( + start_button_rising or time.monotonic() >= next_xr_retry_time + ) + if should_retry: + try_start_openxr_runtime() + if teleop is None: + hold_robot_targets() + step_scene_once() + continue + + try: + sample = teleop.step() + if not teleop_armed: + if not start_waiting_announced: + print( + f"[{status_label}] teleop is disarmed; press keyboard B to recalibrate " + "and start teleop. Press keyboard R to force reset back to this state.", + flush=True, + ) + start_waiting_announced = True + if start_button_rising: + teleop_armed = True + start_waiting_announced = False + reset_teleop_calibration() + print( + f"[{status_label}] teleop armed at frame={int(sample.get('frame_count', 0))}; " + "starting fresh AVP head-hand calibration " + f"(averaging {avp_frame_binding_config.calibration_window_s:.1f}s).", + flush=True, + ) + else: + marker_target_poses = {} + hold_robot_targets() + step_scene_once( + target_poses=sample_target_poses( + sample, + resolve_marker_pose=marker_pose_resolver, + ), + update_ee_frames=arm_ik is not None, + ) + continue + + if use_native_hand_targets: + raw_left_targets = ordered_hand_targets( + sample, + "left", + wuji_hand_config.left_joint_names, + ) + raw_right_targets = ordered_hand_targets( + sample, + "right", + wuji_hand_config.right_joint_names, + ) + else: + raw_left_targets, raw_right_targets = wuji_backend.targets(sample) + + left_targets = left_postprocessor.process(raw_left_targets) + right_targets = right_postprocessor.process(raw_right_targets) + + if left_targets is not None: + robot.set_joint_position_target_index( + target=torch.as_tensor( + left_targets, device=sim.device + ).unsqueeze(0), + joint_ids=left_joint_ids, + ) + if right_targets is not None: + robot.set_joint_position_target_index( + target=torch.as_tensor( + right_targets, device=sim.device + ).unsqueeze(0), + joint_ids=right_joint_ids, + ) + if avp_frame_binding is not None: + avp_frame_binding.update_calibration(sample) + avp_frame_binding.update_runtime_state(sample) + if ( + head_view_camera is not None + and head_view_camera_config.activate_on_calibration + and avp_frame_binding.calibrated + ): + head_view_camera.activate() + if ( + auxiliary_viewport_config.enabled + and auxiliary_viewport_config.activate_on_calibration + and avp_frame_binding.calibrated + ): + auxiliary_viewport.show() + if ( + head_view_xr_display is not None + and head_view_xr_display_config.activate_on_calibration + and avp_frame_binding.calibrated + ): + head_view_xr_display.set_visible(True) + if whole_body_yaw is not None: + whole_body_yaw.update(avp_frame_binding) + if arm_ik is not None: + arm_ik.update(sample) + marker_target_poses = arm_ik.target_pose_world_np() + else: + marker_target_poses = {} + + step_scene_once( + target_poses=sample_target_poses( + sample, + resolve_marker_pose=marker_pose_resolver, + ), + update_ee_frames=arm_ik is not None, + ) + except Exception as exc: + if not _is_xr_runtime_unavailable_error(exc): + raise + enter_xr_waiting_state(exc, phase="runtime") + except Exception as exc: + print( + f"[{status_label}] fatal runtime error: {type(exc).__name__}: {exc}", + file=sys.stderr, + flush=True, + ) + traceback.print_exc() + raise + finally: + try: + if keyboard_subscription is not None: + unsubscribe_fn = getattr( + input_interface, "unsubscribe_from_keyboard_events", None + ) + if callable(unsubscribe_fn) and app_window is not None: + unsubscribe_fn( + app_window.get_keyboard(), + keyboard_subscription, + ) + close_openxr_runtime() + finally: + simulation_app.close() + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/examples/g1_wuji_teleop/python/g1_wuji_teleop/viz/teleop_visualizer.py b/examples/g1_wuji_teleop/python/g1_wuji_teleop/viz/teleop_visualizer.py new file mode 100644 index 000000000..02aac5617 --- /dev/null +++ b/examples/g1_wuji_teleop/python/g1_wuji_teleop/viz/teleop_visualizer.py @@ -0,0 +1,974 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Live 3D visualizer for the G1-Wuji AVP + MANUS teleop stream. + +Run from a shell that has sourced the CloudXR environment, for example: + + source ~/.cloudxr/run/cloudxr.env + python examples/g1_wuji_teleop/scripts/g1_wuji_teleop_visualizer.py +""" + +from __future__ import annotations + +import argparse +import os +import sys +import time +from collections import deque +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +os.environ.setdefault("MPLCONFIGDIR", "/tmp/matplotlib") + +import matplotlib + +if not os.environ.get("MPLBACKEND"): + matplotlib.use("TkAgg") + +import matplotlib.pyplot as plt +import numpy as np +from matplotlib.animation import FuncAnimation +from mpl_toolkits.mplot3d import Axes3D # noqa: F401 + +from ..devices.avp_manus_stream import TeleopMain +from ..paths import ( + default_config_path as app_default_config_path, + ensure_import_paths, +) + + +SIDE_COLORS = { + "left": "dodgerblue", + "right": "tomato", +} + +HEAD_COLOR = "black" +HEAD_AXIS_LENGTH = 0.1 +HEAD_FORWARD_AXIS_ISAAC = np.asarray([0.0, 1.0, 0.0], dtype=np.float64) +OPENXR_TO_ISAAC_FRAME = np.asarray( + [ + [1.0, 0.0, 0.0], + [0.0, 0.0, -1.0], + [0.0, 1.0, 0.0], + ], + dtype=np.float64, +) + +VIEW_PRESETS = ( + ("forward +X", 0.0, 180.0), + ("iso", 24.0, -58.0), + ("left +Y", 0.0, -90.0), + ("right -Y", 0.0, 90.0), + ("top", 90.0, -90.0), + ("back -X", 0.0, 0.0), +) + +DISPLAY_FRAME = np.eye(3, dtype=np.float64) + + +@dataclass(frozen=True) +class VisualFrameState: + origin_pos: np.ndarray + frame_rot: np.ndarray + + +OPENXR_HAND_BONES = ( + ("wrist", "palm"), + ("wrist", "thumb_metacarpal"), + ("thumb_metacarpal", "thumb_proximal"), + ("thumb_proximal", "thumb_distal"), + ("thumb_distal", "thumb_tip"), + ("wrist", "index_metacarpal"), + ("index_metacarpal", "index_proximal"), + ("index_proximal", "index_intermediate"), + ("index_intermediate", "index_distal"), + ("index_distal", "index_tip"), + ("wrist", "middle_metacarpal"), + ("middle_metacarpal", "middle_proximal"), + ("middle_proximal", "middle_intermediate"), + ("middle_intermediate", "middle_distal"), + ("middle_distal", "middle_tip"), + ("wrist", "ring_metacarpal"), + ("ring_metacarpal", "ring_proximal"), + ("ring_proximal", "ring_intermediate"), + ("ring_intermediate", "ring_distal"), + ("ring_distal", "ring_tip"), + ("wrist", "little_metacarpal"), + ("little_metacarpal", "little_proximal"), + ("little_proximal", "little_intermediate"), + ("little_intermediate", "little_distal"), + ("little_distal", "little_tip"), +) + +ROBOT_FINGER_KEYS = { + "index": ("index_proximal", "index_distal"), + "middle": ("middle_proximal", "middle_distal"), + "ring": ("ring_proximal", "ring_distal"), + "pinky": ("pinky_proximal", "pinky_distal"), + "thumb": ("thumb_proximal_yaw", "thumb_proximal_pitch"), +} + + +def default_config_path() -> Path: + return app_default_config_path() + + +def require_cloudxr_environment() -> None: + if os.environ.get("ISAAC_TELEOP_DISABLE_CXR_ENV_CHECKS"): + return + + missing = [ + name + for name in ("NV_CXR_RUNTIME_DIR", "XR_RUNTIME_JSON") + if not os.environ.get(name) + ] + if not missing: + return + + env_path = Path.home() / ".cloudxr" / "run" / "cloudxr.env" + hint = ( + "CloudXR/OpenXR environment is not loaded. Run this in the same shell " + "before starting the visualizer:\n\n" + f" source {env_path}\n" + " python examples/g1_wuji_teleop/scripts/g1_wuji_teleop_visualizer.py " + "--config examples/g1_wuji_teleop/config/avp_manus.yml\n" + ) + if not env_path.exists(): + hint += "\nIf that file is missing, start CloudXR first:\n\n" + hint += " python -m isaacteleop.cloudxr\n" + raise RuntimeError(f"Missing environment variables: {', '.join(missing)}\n{hint}") + + +def quat_xyzw_to_rotmat(quat_xyzw: Sequence[float]) -> np.ndarray: + qx, qy, qz, qw = [float(v) for v in quat_xyzw] + norm = np.linalg.norm([qx, qy, qz, qw]) + if norm <= 1e-8: + return np.eye(3, dtype=np.float64) + qx, qy, qz, qw = [value / norm for value in (qx, qy, qz, qw)] + return np.asarray( + [ + [ + 1.0 - 2.0 * (qy * qy + qz * qz), + 2.0 * (qx * qy - qw * qz), + 2.0 * (qx * qz + qw * qy), + ], + [ + 2.0 * (qx * qy + qw * qz), + 1.0 - 2.0 * (qx * qx + qz * qz), + 2.0 * (qy * qz - qw * qx), + ], + [ + 2.0 * (qx * qz - qw * qy), + 2.0 * (qy * qz + qw * qx), + 1.0 - 2.0 * (qx * qx + qy * qy), + ], + ], + dtype=np.float64, + ) + + +def pose_to_matrix(pose: Mapping[str, Any]) -> np.ndarray: + mat = np.eye(4, dtype=np.float64) + mat[:3, :3] = quat_xyzw_to_rotmat(pose["orientation_xyzw"]) + mat[:3, 3] = np.asarray(pose["position"], dtype=np.float64) + return mat + + +def openxr_pose_to_isaac_matrix(pose: Mapping[str, Any]) -> np.ndarray: + raw = pose_to_matrix(pose) + mat = np.eye(4, dtype=np.float64) + mat[:3, :3] = OPENXR_TO_ISAAC_FRAME @ raw[:3, :3] @ OPENXR_TO_ISAAC_FRAME.T + mat[:3, 3] = OPENXR_TO_ISAAC_FRAME @ raw[:3, 3] + return mat + + +def visual_frame_state_from_sample( + sample: Mapping[str, Any], +) -> VisualFrameState | None: + head_pose = sample.get("head_pose") + ee_poses = sample.get("ee_poses", {}) + left_pose = ee_poses.get("left") if isinstance(ee_poses, Mapping) else None + right_pose = ee_poses.get("right") if isinstance(ee_poses, Mapping) else None + if head_pose is None or left_pose is None or right_pose is None: + return None + + head_mat = openxr_pose_to_isaac_matrix(head_pose) + left_mat = openxr_pose_to_isaac_matrix(left_pose) + right_mat = openxr_pose_to_isaac_matrix(right_pose) + + x_axis = normalize_vector(head_mat[:3, :3] @ HEAD_FORWARD_AXIS_ISAAC) + y_axis_raw = normalize_vector(left_mat[:3, 3] - right_mat[:3, 3]) + if x_axis is None: + return None + + if y_axis_raw is None: + y_axis = fallback_lateral_axis(x_axis) + else: + z_axis = normalize_vector(np.cross(x_axis, y_axis_raw)) + if z_axis is None: + y_axis = fallback_lateral_axis(x_axis) + else: + y_axis = normalize_vector(np.cross(z_axis, x_axis)) + if y_axis is None: + return None + + z_axis = normalize_vector(np.cross(x_axis, y_axis)) + if z_axis is None: + return None + frame_rot = np.column_stack((x_axis, y_axis, z_axis)) + if np.linalg.det(frame_rot) < 0.0: + frame_rot[:, 2] *= -1.0 + return VisualFrameState( + origin_pos=head_mat[:3, 3].copy(), + frame_rot=frame_rot, + ) + + +def fallback_lateral_axis(x_axis: np.ndarray) -> np.ndarray | None: + # Early tracker frames can report coincident wrists; do not block visualizer startup. + up_hint = np.asarray([0.0, 0.0, 1.0], dtype=np.float64) + if abs(float(np.dot(x_axis, up_hint))) > 0.95: + up_hint = np.asarray([0.0, 1.0, 0.0], dtype=np.float64) + return normalize_vector(np.cross(up_hint, x_axis)) + + +def pose_to_visual_frame_matrix( + pose: Mapping[str, Any], + state: VisualFrameState, +) -> np.ndarray: + raw_mat = openxr_pose_to_isaac_matrix(pose) + mat = np.eye(4, dtype=np.float64) + mat[:3, :3] = state.frame_rot.T @ raw_mat[:3, :3] + mat[:3, 3] = state.frame_rot.T @ (raw_mat[:3, 3] - state.origin_pos) + return mat + + +def skeleton_to_visual_frame( + skeleton: Mapping[str, Any] | None, + state: VisualFrameState, +) -> dict[str, Any] | None: + if skeleton is None: + return None + positions = np.asarray(skeleton.get("positions", []), dtype=np.float64) + if positions.ndim != 2 or positions.shape[1] != 3: + return None + + positions_isaac = positions @ OPENXR_TO_ISAAC_FRAME.T + local_positions = (positions_isaac - state.origin_pos) @ state.frame_rot + transformed = dict(skeleton) + transformed["positions"] = [ + [float(coord) for coord in position] for position in local_positions + ] + return transformed + + +def transform_points(points: np.ndarray, display_frame: np.ndarray) -> np.ndarray: + return np.asarray(points, dtype=np.float64) @ display_frame.T + + +def transform_vector(vector: np.ndarray, display_frame: np.ndarray) -> np.ndarray: + return display_frame @ np.asarray(vector, dtype=np.float64) + + +def transform_rotation(rot: np.ndarray, display_frame: np.ndarray) -> np.ndarray: + return display_frame @ np.asarray(rot, dtype=np.float64) + + +def transform_pose_matrix(mat: np.ndarray, display_frame: np.ndarray) -> np.ndarray: + transformed = np.eye(4, dtype=np.float64) + transformed[:3, :3] = transform_rotation(mat[:3, :3], display_frame) + transformed[:3, 3] = transform_vector(mat[:3, 3], display_frame) + return transformed + + +def draw_pose_axes( + ax, + position: np.ndarray, + rot: np.ndarray, + *, + length: float, + alpha: float = 0.9, +) -> None: + axis_colors = ("red", "green", "blue") + for axis_index, color in enumerate(axis_colors): + direction = rot[:, axis_index] * length + ax.quiver( + position[0], + position[1], + position[2], + direction[0], + direction[1], + direction[2], + color=color, + arrow_length_ratio=0.25, + linewidth=2.0, + alpha=alpha, + ) + + +def draw_world_axes( + ax, + *, + display_frame: np.ndarray, + length: float, + label: str, +) -> None: + origin = np.zeros(3, dtype=np.float64) + draw_pose_axes( + ax, + origin, + transform_rotation(np.eye(3), display_frame), + length=length, + alpha=0.75, + ) + ax.scatter( + 0.0, + 0.0, + 0.0, + color="black", + marker="+", + s=90, + alpha=0.65, + label=label, + ) + labels = ( + ("+X forward", np.asarray([length, 0.0, 0.0]), "red"), + ("+Y left", np.asarray([0.0, length, 0.0]), "green"), + ("+Z up", np.asarray([0.0, 0.0, length]), "blue"), + ) + for text, point, color in labels: + drawn = transform_vector(point, display_frame) + ax.text(drawn[0], drawn[1], drawn[2], text, color=color, fontsize=9) + + +def draw_fixed_head_pose( + ax, + *, + display_frame: np.ndarray, +) -> np.ndarray: + # Keep this always-on so visualizer startup matches the teleop debug path. + head_mat = transform_pose_matrix(np.eye(4, dtype=np.float64), display_frame) + position = head_mat[:3, 3] + rot = head_mat[:3, :3] + ax.scatter( + position[0], + position[1], + position[2], + color=HEAD_COLOR, + edgecolors="white", + linewidths=0.8, + marker="s", + s=160, + label="head fixed", + zorder=11, + ) + ax.text( + position[0], + position[1], + position[2], + "head", + color=HEAD_COLOR, + fontsize=9, + ) + draw_pose_axes( + ax, + position, + rot, + length=HEAD_AXIS_LENGTH, + alpha=0.95, + ) + return position + + +def normalize_vector(vector: np.ndarray) -> np.ndarray | None: + norm = float(np.linalg.norm(vector)) + if norm <= 1e-6: + return None + return vector / norm + + +def geometric_hand_frame( + skeleton: Mapping[str, Any] | None, + *, + side: str, +) -> np.ndarray | None: + if skeleton is None: + return None + + joint_names = [str(name) for name in skeleton.get("joint_names", [])] + positions = np.asarray(skeleton.get("positions", []), dtype=np.float64) + valid = np.asarray(skeleton.get("valid", []), dtype=bool) + if positions.ndim != 2 or positions.shape[1] != 3: + return None + if valid.ndim != 1 or positions.shape[0] != valid.shape[0]: + return None + + name_to_index = {name: index for index, name in enumerate(joint_names)} + + def joint(name: str) -> np.ndarray | None: + index = name_to_index.get(name) + if index is None or not valid[index]: + return None + return positions[index] + + def first_valid(*names: str) -> np.ndarray | None: + for name in names: + point = joint(name) + if point is not None: + return point + return None + + wrist = joint("wrist") + palm = joint("palm") + index = first_valid("index_metacarpal", "index_proximal") + middle = first_valid("middle_metacarpal", "middle_proximal") + little = first_valid("little_metacarpal", "little_proximal") + if wrist is None or middle is None: + return None + + forward_target = palm if palm is not None else middle + forward = normalize_vector(forward_target - wrist) + if forward is None: + return None + + if index is not None and little is not None: + lateral_raw = little - index if side == "left" else index - little + elif index is not None: + lateral_raw = index - middle + elif little is not None: + lateral_raw = middle - little + else: + return None + + lateral = lateral_raw - np.dot(lateral_raw, forward) * forward + x_axis = normalize_vector(lateral) + if x_axis is None: + return None + + y_axis = normalize_vector(np.cross(forward, x_axis)) + if y_axis is None: + return None + z_axis = normalize_vector(np.cross(x_axis, y_axis)) + if z_axis is None: + return None + return np.column_stack((x_axis, y_axis, z_axis)) + + +def skeleton_points(skeleton: Mapping[str, Any] | None) -> np.ndarray: + if skeleton is None: + return np.empty((0, 3), dtype=np.float64) + positions = np.asarray(skeleton.get("positions", []), dtype=np.float64) + valid = np.asarray(skeleton.get("valid", []), dtype=bool) + if positions.ndim != 2 or positions.shape[1] != 3 or valid.ndim != 1: + return np.empty((0, 3), dtype=np.float64) + if positions.shape[0] != valid.shape[0]: + return np.empty((0, 3), dtype=np.float64) + return positions[valid] + + +def draw_openxr_skeleton( + ax, + skeleton: Mapping[str, Any], + *, + display_frame: np.ndarray, + color: str, + alpha: float, +) -> list[np.ndarray]: + joint_names = [str(name) for name in skeleton.get("joint_names", [])] + positions = np.asarray(skeleton.get("positions", []), dtype=np.float64) + valid = np.asarray(skeleton.get("valid", []), dtype=bool) + if positions.ndim != 2 or positions.shape[1] != 3: + return [] + if valid.ndim != 1 or positions.shape[0] != valid.shape[0]: + return [] + + name_to_index = {name: index for index, name in enumerate(joint_names)} + draw_positions = transform_points(positions, display_frame) + drawn_points: list[np.ndarray] = [] + for joint_a, joint_b in OPENXR_HAND_BONES: + index_a = name_to_index.get(joint_a) + index_b = name_to_index.get(joint_b) + if index_a is None or index_b is None: + continue + if not (valid[index_a] and valid[index_b]): + continue + point_a = draw_positions[index_a] + point_b = draw_positions[index_b] + ax.plot( + [point_a[0], point_b[0]], + [point_a[1], point_b[1]], + [point_a[2], point_b[2]], + color=color, + linewidth=2.0, + alpha=alpha, + ) + drawn_points.extend([point_a, point_b]) + + points = draw_positions[valid] + if points.size: + ax.scatter( + points[:, 0], + points[:, 1], + points[:, 2], + color=color, + s=14, + alpha=min(0.9, alpha + 0.1), + ) + drawn_points.extend(list(points)) + return drawn_points + + +def joint_lookup( + joint_names: Sequence[str], + joint_positions: Sequence[float] | None, + side: str, +) -> dict[str, float]: + if joint_positions is None: + return {} + prefix = f"{side[0].upper()}_" + lookup: dict[str, float] = {} + for name, value in zip(joint_names, joint_positions): + clean_name = str(name) + if clean_name.startswith(prefix): + clean_name = clean_name[len(prefix) :] + if clean_name.endswith("_joint"): + clean_name = clean_name[: -len("_joint")] + lookup[clean_name] = float(value) + return lookup + + +def fallback_finger_chain( + *, + lateral: float, + base_forward: float, + flex_a: float, + flex_b: float, + scale: float, +) -> np.ndarray: + lengths = np.asarray([0.040, 0.030, 0.023], dtype=np.float64) * scale + angles = np.asarray([0.15, flex_a, flex_a + flex_b], dtype=np.float64) + points = [np.asarray([lateral, 0.0, base_forward], dtype=np.float64) * scale] + current = points[0].copy() + for length, angle in zip(lengths, angles): + direction = np.asarray([0.0, -np.sin(angle), np.cos(angle)]) + current = current + direction * length + points.append(current.copy()) + return np.asarray(points, dtype=np.float64) + + +def fallback_thumb_chain( + *, + side_sign: float, + yaw: float, + pitch: float, + scale: float, +) -> np.ndarray: + base = np.asarray([side_sign * 0.050, -0.005, 0.010], dtype=np.float64) * scale + lengths = np.asarray([0.035, 0.027, 0.022], dtype=np.float64) * scale + direction = np.asarray( + [ + side_sign * (0.65 + 0.25 * np.sin(yaw)), + -0.25 - 0.55 * np.sin(pitch), + 0.60 * np.cos(pitch), + ], + dtype=np.float64, + ) + direction /= max(float(np.linalg.norm(direction)), 1e-8) + points = [base] + current = base.copy() + for length in lengths: + current = current + direction * length + points.append(current.copy()) + return np.asarray(points, dtype=np.float64) + + +def draw_fallback_hand( + ax, + *, + pose_mat: np.ndarray, + joint_names: Sequence[str], + joint_positions: Sequence[float] | None, + side: str, + color: str, + display_frame: np.ndarray, + scale: float, +) -> list[np.ndarray]: + lookup = joint_lookup(joint_names, joint_positions, side) + rot = pose_mat[:3, :3] + origin = pose_mat[:3, 3] + side_sign = -1.0 if side == "left" else 1.0 + local_chains = [] + + finger_offsets = { + "index": side_sign * 0.030, + "middle": side_sign * 0.010, + "ring": side_sign * -0.012, + "pinky": side_sign * -0.032, + } + for finger, lateral in finger_offsets.items(): + flex_keys = ROBOT_FINGER_KEYS[finger] + local_chains.append( + fallback_finger_chain( + lateral=lateral, + base_forward=0.020, + flex_a=lookup.get(flex_keys[0], 0.2), + flex_b=lookup.get(flex_keys[1], 0.2), + scale=scale, + ) + ) + + thumb_yaw, thumb_pitch = ROBOT_FINGER_KEYS["thumb"] + local_chains.append( + fallback_thumb_chain( + side_sign=side_sign, + yaw=lookup.get(thumb_yaw, 0.0), + pitch=lookup.get(thumb_pitch, 0.2), + scale=scale, + ) + ) + + palm = ( + np.asarray( + [ + [0.0, 0.0, 0.0], + [side_sign * 0.040, -0.004, 0.010], + [side_sign * -0.040, -0.004, 0.010], + ], + dtype=np.float64, + ) + * scale + ) + world_palm = transform_points(origin + palm @ rot.T, display_frame) + ax.plot( + world_palm[:, 0], + world_palm[:, 1], + world_palm[:, 2], + color=color, + linewidth=1.4, + alpha=0.55, + ) + + world_points: list[np.ndarray] = [] + for chain in local_chains: + world_chain = transform_points(origin + chain @ rot.T, display_frame) + ax.plot( + world_chain[:, 0], + world_chain[:, 1], + world_chain[:, 2], + color=color, + linewidth=2.0, + alpha=0.75, + ) + ax.scatter( + world_chain[:, 0], + world_chain[:, 1], + world_chain[:, 2], + color=color, + s=12, + alpha=0.65, + ) + world_points.extend(list(world_chain)) + world_points.extend(list(world_palm)) + return world_points + + +def set_equal_axes(ax, points: Sequence[np.ndarray], min_half_width: float) -> None: + finite_points = [ + np.asarray(point, dtype=np.float64) + for point in points + if np.asarray(point).shape == (3,) and np.all(np.isfinite(point)) + ] + if not finite_points: + ax.set_xlim(-min_half_width, min_half_width) + ax.set_ylim(-min_half_width, min_half_width) + ax.set_zlim(-min_half_width, min_half_width) + return + + pts = np.asarray(finite_points, dtype=np.float64) + center = (pts.min(axis=0) + pts.max(axis=0)) * 0.5 + span = float((pts.max(axis=0) - pts.min(axis=0)).max()) + half_width = max(min_half_width, span * 0.65) + ax.set_xlim(center[0] - half_width, center[0] + half_width) + ax.set_ylim(center[1] - half_width, center[1] + half_width) + ax.set_zlim(center[2] - half_width, center[2] + half_width) + try: + ax.set_box_aspect((1, 1, 1)) + except AttributeError: + pass + + +def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Visualize AVP/MANUS teleop EE poses and hand skeletons." + ) + parser.add_argument( + "--config", + type=Path, + default=default_config_path(), + help="YAML config used by the G1-Wuji teleop app.", + ) + parser.add_argument( + "--fps", + type=float, + default=30.0, + help="Visualization and teleop sampling rate.", + ) + parser.add_argument( + "--axis-limit", + type=float, + default=0.12, + help="Minimum half-width of the 3D view, in meters.", + ) + parser.add_argument( + "--window-width", + type=float, + default=14.0, + help="Matplotlib window width in inches.", + ) + parser.add_argument( + "--window-height", + type=float, + default=10.0, + help="Matplotlib window height in inches.", + ) + parser.add_argument( + "--include-origin", + action="store_true", + help="Include the world origin in auto-scaling.", + ) + parser.add_argument( + "--world-axis-length", + type=float, + default=0.16, + help="Length of the drawn calibrated head-hand coordinate axes in meters.", + ) + parser.add_argument( + "--trail", + type=int, + default=120, + help="Number of EE trail points to keep per side. Use 0 to disable.", + ) + parser.add_argument( + "--arrow-length", + type=float, + default=0.08, + help="EE orientation arrow length, in meters.", + ) + parser.add_argument( + "--ee-arrow-mode", + choices=("hand", "pose"), + default="hand", + help=( + "Arrow frame source: 'hand' derives axes from visible hand geometry; " + "'pose' draws the calibrated EE quaternion." + ), + ) + parser.add_argument( + "--fallback-hand-scale", + type=float, + default=1.0, + help="Scale for the synthetic hand skeleton used when raw hand joints are absent.", + ) + return parser.parse_args(sys.argv[1:] if argv is None else argv) + + +def main(argv: Sequence[str] | None = None) -> int: + ensure_import_paths() + args = parse_args(argv) + require_cloudxr_environment() + display_frame = DISPLAY_FRAME + trails = {side: deque(maxlen=max(0, args.trail)) for side in SIDE_COLORS} + fig = plt.figure(figsize=(args.window_width, args.window_height)) + ax = fig.add_subplot(111, projection="3d") + start_s = time.monotonic() + frames = 0 + last_print_s = 0.0 + latest_sample: dict[str, Any] | None = None + visual_frame_state: VisualFrameState | None = None + view_index = 0 + + def apply_view() -> None: + _name, elev, azim = VIEW_PRESETS[view_index] + ax.view_init(elev=elev, azim=azim) + + def on_click(event) -> None: + nonlocal view_index + if event.inaxes is not ax: + return + view_index = (view_index + 1) % len(VIEW_PRESETS) + apply_view() + fig.canvas.draw_idle() + print(f"view={VIEW_PRESETS[view_index][0]}", flush=True) + + fig.canvas.mpl_connect("button_press_event", on_click) + + with TeleopMain.from_config_file(args.config) as teleop: + + def update(_frame_num: int) -> None: + nonlocal frames, last_print_s, latest_sample, visual_frame_state + frames += 1 + latest_sample = teleop.step() + if visual_frame_state is None: + visual_frame_state = visual_frame_state_from_sample(latest_sample) + ax.clear() + origin = np.zeros(3, dtype=np.float64) + all_points: list[np.ndarray] = [origin] if args.include_origin else [] + + draw_world_axes( + ax, + display_frame=display_frame, + length=float(args.world_axis_length), + label="head-hand origin", + ) + all_points.append(draw_fixed_head_pose(ax, display_frame=display_frame)) + + if visual_frame_state is None: + set_equal_axes(ax, all_points, float(args.axis_limit)) + ax.set_xlabel("+X forward") + ax.set_ylabel("+Y left") + ax.set_zlabel("+Z up") + apply_view() + ax.legend(loc="upper left", fontsize=8) + ax.set_title( + "AVP/MANUS Teleop Visualizer | waiting for head + both wrist poses" + ) + return + + for side, color in SIDE_COLORS.items(): + pose = latest_sample["ee_poses"][side] + local_pose_mat = ( + pose_to_visual_frame_matrix(pose, visual_frame_state) + if pose is not None + else None + ) + pose_mat = ( + transform_pose_matrix(local_pose_mat, display_frame) + if local_pose_mat is not None + else None + ) + skeleton = skeleton_to_visual_frame( + latest_sample["hand_skeletons"][side], + visual_frame_state, + ) + + if skeleton is not None: + points = draw_openxr_skeleton( + ax, + skeleton, + display_frame=display_frame, + color=color, + alpha=0.8, + ) + all_points.extend(points) + + if pose_mat is not None: + pos = pose_mat[:3, 3] + all_points.append(pos) + if args.trail > 0: + trails[side].append(pos.copy()) + ax.scatter( + pos[0], + pos[1], + pos[2], + color=color, + edgecolors="black", + linewidths=0.8, + s=190, + label=f"{side} EE", + zorder=10, + ) + arrow_rot = ( + geometric_hand_frame(skeleton, side=side) + if args.ee_arrow_mode == "hand" + else None + ) + if arrow_rot is None: + arrow_rot = local_pose_mat[:3, :3] + arrow_rot = transform_rotation(arrow_rot, display_frame) + draw_pose_axes( + ax, + pos, + arrow_rot, + length=float(args.arrow_length), + ) + + if skeleton is None or skeleton_points(skeleton).size == 0: + fallback_points = draw_fallback_hand( + ax, + pose_mat=local_pose_mat, + joint_names=latest_sample["hand_joint_names"][side], + joint_positions=latest_sample["hand_joint_positions"][side], + side=side, + color=color, + display_frame=display_frame, + scale=float(args.fallback_hand_scale), + ) + all_points.extend(fallback_points) + + trail = np.asarray(trails[side], dtype=np.float64) + if trail.shape[0] > 1: + ax.plot( + trail[:, 0], + trail[:, 1], + trail[:, 2], + color=color, + linestyle="--", + linewidth=1.0, + alpha=0.25, + ) + all_points.extend(list(trail)) + + set_equal_axes(ax, all_points, float(args.axis_limit)) + ax.set_xlabel("+X forward") + ax.set_ylabel("+Y left") + ax.set_zlabel("+Z up") + apply_view() + ax.legend(loc="upper left", fontsize=8) + elapsed_s = max(time.monotonic() - start_s, 1e-6) + ax.set_title( + "AVP/MANUS Teleop Visualizer | " + f"frame={latest_sample['frame_count']} " + f"fps={frames / elapsed_s:.1f} " + f"view={VIEW_PRESETS[view_index][0]} " + "click=flip view" + ) + + now_s = time.monotonic() + if now_s - last_print_s >= 2.0: + last_print_s = now_s + chunks = [] + for side in SIDE_COLORS: + pose = latest_sample["ee_poses"][side] + if pose is None: + chunks.append(f"{side}=missing") + else: + pos = pose_to_visual_frame_matrix( + pose, + visual_frame_state, + )[:3, 3] + chunks.append(f"{side}={np.round(pos, 3)}") + print( + f"frame={latest_sample['frame_count']} " + " ".join(chunks), + flush=True, + ) + + interval_ms = max(int(1000.0 / max(float(args.fps), 1e-3)), 16) + _animation = FuncAnimation( + fig, + update, + interval=interval_ms, + cache_frame_data=False, + ) + try: + plt.show() + except KeyboardInterrupt: + pass + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/examples/g1_wuji_teleop/scripts/g1_wuji_teleop_main.py b/examples/g1_wuji_teleop/scripts/g1_wuji_teleop_main.py new file mode 100755 index 000000000..745d214ba --- /dev/null +++ b/examples/g1_wuji_teleop/scripts/g1_wuji_teleop_main.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Thin launcher for the G1-Wuji AVP + MANUS teleop application.""" + +from __future__ import annotations + +import sys +from pathlib import Path + + +def _ensure_app_import_path() -> None: + example_root = Path(__file__).resolve().parents[1] + app_python_dir = example_root / "python" + path_str = str(app_python_dir) + if path_str not in sys.path: + sys.path.insert(0, path_str) + + +def main() -> int: + _ensure_app_import_path() + from g1_wuji_teleop.cli import teleop_main + + return teleop_main() + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/examples/g1_wuji_teleop/scripts/g1_wuji_teleop_visualizer.py b/examples/g1_wuji_teleop/scripts/g1_wuji_teleop_visualizer.py new file mode 100755 index 000000000..1952b176f --- /dev/null +++ b/examples/g1_wuji_teleop/scripts/g1_wuji_teleop_visualizer.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Thin launcher for the G1-Wuji teleop visualizer.""" + +from __future__ import annotations + +import sys +from pathlib import Path + + +def _ensure_app_import_path() -> None: + example_root = Path(__file__).resolve().parents[1] + app_python_dir = example_root / "python" + path_str = str(app_python_dir) + if path_str not in sys.path: + sys.path.insert(0, path_str) + + +def main() -> int: + _ensure_app_import_path() + from g1_wuji_teleop.cli import teleop_visualizer_main + + return teleop_visualizer_main() + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/examples/teleop/CMakeLists.txt b/examples/teleop/CMakeLists.txt index 94a93beb1..4e5e02bd4 100644 --- a/examples/teleop/CMakeLists.txt +++ b/examples/teleop/CMakeLists.txt @@ -6,3 +6,11 @@ project(teleop_examples) include(${CMAKE_SOURCE_DIR}/cmake/InstallPythonExample.cmake) install_python_example(DESTINATION examples/teleop/python) + +install(DIRECTORY python/config + DESTINATION examples/teleop/python +) + +install(DIRECTORY python/assets + DESTINATION examples/teleop/python +) diff --git a/examples/teleop/python/pyproject.toml b/examples/teleop/python/pyproject.toml index db0aa891d..a17306c10 100644 --- a/examples/teleop/python/pyproject.toml +++ b/examples/teleop/python/pyproject.toml @@ -8,4 +8,6 @@ description = "Isaac Teleop Retargeting Examples" requires-python = ">=3.10,<3.14" dependencies = [ "isaacteleop[retargeters]", + "matplotlib", + "PyYAML", ] diff --git a/scripts/setup_v2d_src.sh b/scripts/setup_v2d_src.sh index 38dbd73c5..8fcf680e6 100755 --- a/scripts/setup_v2d_src.sh +++ b/scripts/setup_v2d_src.sh @@ -22,9 +22,11 @@ set -euo pipefail +# 1. Enter the repository root so all relative paths resolve from the repo top level. REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" cd "${REPO_ROOT}" +# 2. Read and validate the V2D commit SHA pinned in `version.txt`. VERSION_FILE="deps/v2d/version.txt" if [[ ! -f "${VERSION_FILE}" ]]; then echo "error: ${VERSION_FILE} is missing." >&2 @@ -42,16 +44,19 @@ if ! [[ "${V2D_REF}" =~ ^[0-9a-fA-F]{40}$ ]]; then fi echo "Pinned V2D ref: ${V2D_REF}" +# 3. Check that the `gh` CLI is available and that the current account can read the V2D repo. if ! command -v gh >/dev/null 2>&1; then echo "error: gh CLI not found. Install from https://cli.github.com/" >&2 echo " and run 'gh auth login' with read access to jiwenc-nv/v2d." >&2 exit 1 fi +# 4. Prepare the destination directory and remove the previous `robotic_grounding` copy first. DST="deps/v2d/src/robotic_grounding" mkdir -p deps/v2d/src rm -rf "${DST}" +# 5. Clone the V2D repository into a temporary directory and check out the commit pinned in `version.txt`. # Whole retargeter branch is ~25 MB so a normal clone is fine. Clone the # branch tip then check out the exact pinned SHA. WORK_DIR=$(mktemp -d -t v2d-src.XXXXXX) @@ -59,10 +64,14 @@ trap 'rm -rf "${WORK_DIR}"' EXIT gh repo clone jiwenc-nv/v2d "${WORK_DIR}/v2d" -- --branch retargeter git -C "${WORK_DIR}/v2d" checkout "${V2D_REF}" +# 6. Copy only the `robotic_grounding` Python source subtree that needs to be packaged into the wheel. cp -a \ "${WORK_DIR}/v2d/robotic_grounding/source/robotic_grounding/robotic_grounding" \ "${DST}" + +# 7. Remove any Python cache directories that may have been copied over so unrelated artifacts do not leak into later builds. # Drop bytecode caches if any slipped through. find "${DST}" -type d -name __pycache__ -exec rm -rf {} + 2>/dev/null || true +# 8. Print a completion message so the destination directory and pinned commit are easy to verify. echo "Done -> ${DST} (cloned from jiwenc-nv/v2d @ ${V2D_REF})" diff --git a/src/plugins/manus/app/main.cpp b/src/plugins/manus/app/main.cpp index ede86165d..26ff0b065 100644 --- a/src/plugins/manus/app/main.cpp +++ b/src/plugins/manus/app/main.cpp @@ -5,6 +5,7 @@ #include #include +#include #include using namespace plugins::manus; @@ -14,7 +15,22 @@ try { std::cout << "Manus Hand Plugin starting..." << std::endl; - auto& tracker = ManusTracker::instance(); + bool disable_optical_wrist = false; + for (int i = 1; i < argc; ++i) + { + const std::string_view arg(argv[i]); + if (arg == "--disable-optical-wrist") + { + disable_optical_wrist = true; + } + else if (arg == "--help" || arg == "-h") + { + std::cout << "Usage: " << argv[0] << " [--disable-optical-wrist]\n"; + return 0; + } + } + + auto& tracker = ManusTracker::instance("ManusHandPlugin", disable_optical_wrist); std::cout << "Plugin running. Press Ctrl+C to stop." << std::endl; diff --git a/src/plugins/manus/core/inc/manus/manus_hand_tracking_plugin.hpp b/src/plugins/manus/core/inc/manus/manus_hand_tracking_plugin.hpp index 85cd95f43..7650fc58b 100644 --- a/src/plugins/manus/core/inc/manus/manus_hand_tracking_plugin.hpp +++ b/src/plugins/manus/core/inc/manus/manus_hand_tracking_plugin.hpp @@ -33,7 +33,8 @@ class __attribute__((visibility("default"))) ManusTracker { public: /// Get the singleton instance - static ManusTracker& instance(const std::string& app_name = "ManusHandPlugin") noexcept(false); + static ManusTracker& instance(const std::string& app_name = "ManusHandPlugin", + bool disable_optical_wrist = false) noexcept(false); void update(); std::vector get_left_hand_nodes() const; @@ -43,7 +44,7 @@ class __attribute__((visibility("default"))) ManusTracker private: // Lifecycle - explicit ManusTracker(const std::string& app_name) noexcept(false); + explicit ManusTracker(const std::string& app_name, bool disable_optical_wrist) noexcept(false); ~ManusTracker(); ManusTracker(const ManusTracker&) = delete; @@ -76,6 +77,7 @@ class __attribute__((visibility("default"))) ManusTracker // Lifecycle std::mutex m_lifecycle_mutex; bool m_initialized = false; + bool m_disable_optical_wrist = false; // ManusSDK State std::mutex landscape_mutex; diff --git a/src/plugins/manus/core/manus_hand_tracking_plugin.cpp b/src/plugins/manus/core/manus_hand_tracking_plugin.cpp index 06212ab22..84ee345f4 100644 --- a/src/plugins/manus/core/manus_hand_tracking_plugin.cpp +++ b/src/plugins/manus/core/manus_hand_tracking_plugin.cpp @@ -62,9 +62,9 @@ SDKReturnCode get_raw_skeleton_node_count(uint32_t glove_id, uint32_t& node_coun static constexpr XrPosef kLeftHandOffset = { { -0.70710678f, -0.5f, 0.0f, 0.5f }, { -0.1f, 0.02f, -0.02f } }; static constexpr XrPosef kRightHandOffset = { { -0.70710678f, 0.5f, 0.0f, 0.5f }, { 0.1f, 0.02f, -0.02f } }; -ManusTracker& ManusTracker::instance(const std::string& app_name) noexcept(false) +ManusTracker& ManusTracker::instance(const std::string& app_name, bool disable_optical_wrist) noexcept(false) { - static ManusTracker s(app_name); + static ManusTracker s(app_name, disable_optical_wrist); return s; } @@ -106,7 +106,8 @@ std::vector ManusTracker::get_right_node_info() const return m_right_node_info; } -ManusTracker::ManusTracker(const std::string& app_name) noexcept(false) +ManusTracker::ManusTracker(const std::string& app_name, bool disable_optical_wrist) noexcept(false) + : m_disable_optical_wrist(disable_optical_wrist) { initialize(app_name); } @@ -128,6 +129,10 @@ ManusTracker::~ManusTracker() void ManusTracker::initialize(const std::string& app_name) noexcept(false) { std::cout << "[Manus] Initializing SDK..." << std::endl; + if (m_disable_optical_wrist) + { + std::cout << "[Manus] Optical wrist source disabled by argument; controller fallback will be used." << std::endl; + } const SDKReturnCode t_InitializeResult = CoreSdk_InitializeIntegrated(); if (t_InitializeResult != SDKReturnCode::SDKReturnCode_Success) { @@ -188,11 +193,16 @@ void ManusTracker::initialize(const std::string& app_name) noexcept(false) // XR_MNDX_XDEV_SPACE_EXTENSION_NAME is optional: it enables optical (HMD) hand // tracking as a higher-quality wrist source. If the runtime does not advertise // it we fall back to controller-based tracking instead of crashing. - const bool xdev_extension_supported = is_openxr_extension_supported(XR_MNDX_XDEV_SPACE_EXTENSION_NAME); + const bool xdev_extension_supported = + !m_disable_optical_wrist && is_openxr_extension_supported(XR_MNDX_XDEV_SPACE_EXTENSION_NAME); if (xdev_extension_supported) { extensions.push_back(XR_MNDX_XDEV_SPACE_EXTENSION_NAME); } + else if (m_disable_optical_wrist) + { + std::cout << "[Manus] Optical hand tracking disabled; controller fallback will be used." << std::endl; + } else { std::cout << "[Manus] " << XR_MNDX_XDEV_SPACE_EXTENSION_NAME diff --git a/src/viz/python/layers_bindings.cpp b/src/viz/python/layers_bindings.cpp index dba90c76c..93cef47c5 100644 --- a/src/viz/python/layers_bindings.cpp +++ b/src/viz/python/layers_bindings.cpp @@ -131,8 +131,13 @@ numpy on a CUDA device pointer); the binding converts it on the fly. .def("set_placement", &viz::QuadLayer::set_placement, "placement"_a, "Update placement at runtime. None switches to fullscreen (window mode only).") .def("placement", &viz::QuadLayer::placement) - .def("set_visible", &viz::QuadLayer::set_visible, "visible"_a) - .def("is_visible", &viz::QuadLayer::is_visible) + // LayerBase itself is not exposed to Python, so bind inherited + // visibility helpers through lambdas on QuadLayer. Binding the + // raw LayerBase member pointer here makes pybind11 expect a + // registered LayerBase Python instance as ``self``. + .def( + "set_visible", [](viz::QuadLayer& self, bool visible) { self.set_visible(visible); }, "visible"_a) + .def("is_visible", [](const viz::QuadLayer& self) { return self.is_visible(); }) .def_property_readonly("name", [](const viz::QuadLayer& l) { return l.name(); }); } diff --git a/src/viz/python_tests/test_offscreen_session.py b/src/viz/python_tests/test_offscreen_session.py index d1f0946bc..43ae513ae 100644 --- a/src/viz/python_tests/test_offscreen_session.py +++ b/src/viz/python_tests/test_offscreen_session.py @@ -251,3 +251,25 @@ def test_stereo_invalid_baseline_rejected(): session.add_quad_layer(layer_cfg) session.destroy() + + +def test_quad_layer_visibility_toggle(): + """QuadLayer exposes LayerBase visibility helpers through the Python binding.""" + cfg = viz.VizSessionConfig() + cfg.mode = viz.DisplayMode.kOffscreen + cfg.window_width = 64 + cfg.window_height = 64 + session = viz.VizSession.create(cfg) + + layer_cfg = viz.QuadLayerConfig() + layer_cfg.name = "visibility" + layer_cfg.resolution = viz.Resolution(32, 32) + layer = session.add_quad_layer(layer_cfg) + + assert layer.is_visible() is True + layer.set_visible(False) + assert layer.is_visible() is False + layer.set_visible(True) + assert layer.is_visible() is True + + session.destroy() diff --git a/third_party/wuji-retargeting b/third_party/wuji-retargeting new file mode 160000 index 000000000..e3636d64c --- /dev/null +++ b/third_party/wuji-retargeting @@ -0,0 +1 @@ +Subproject commit e3636d64c9b171195d90d1024a705f973ee3c5ed