From 38ddf1265f2010a2f08b32c9d8ecab646264d35c Mon Sep 17 00:00:00 2001 From: Chandran Date: Wed, 10 Jun 2026 11:10:00 +0200 Subject: [PATCH] feat(teleop): show live camera feeds beside the 3D viewer Add an optional camera panel to the teleoperation page, off by default so landing on the page never calls getUserMedia (same consent pattern as the calibration camera toggle). Teleoperation opens no cv2 cameras, so the browser can stream them directly while the arm runs. The panel mirrors the selected robot's configured cameras: it shows a live feed for each camera on the robot record (e.g. wrist_cam, webcam), stacked vertically, and shows nothing when the robot has none -- teleop never surfaces a device that was not deliberately configured. --- .../src/components/control/CameraFeed.tsx | 46 +++++++++ .../components/control/TeleopCameraPanel.tsx | 94 +++++++++++++++++++ .../components/control/VisualizerPanel.tsx | 6 ++ frontend/src/pages/Teleoperation.tsx | 7 +- 4 files changed, 152 insertions(+), 1 deletion(-) create mode 100644 frontend/src/components/control/CameraFeed.tsx create mode 100644 frontend/src/components/control/TeleopCameraPanel.tsx diff --git a/frontend/src/components/control/CameraFeed.tsx b/frontend/src/components/control/CameraFeed.tsx new file mode 100644 index 00000000..c1d3cbc8 --- /dev/null +++ b/frontend/src/components/control/CameraFeed.tsx @@ -0,0 +1,46 @@ +import React from "react"; +import { VideoOff } from "lucide-react"; +import { useCameraStream } from "@/hooks/useCameraStream"; + +interface CameraFeedProps { + /** Browser deviceId to stream. Empty string renders the "no camera" state. */ + deviceId: string; + /** Optional caption shown under the feed. */ + label?: string; +} + +/** Live browser-camera feed bound to a deviceId via getUserMedia. */ +const CameraFeed: React.FC = ({ deviceId, label }) => { + const { videoRef, hasError } = useCameraStream(deviceId, false); + const showVideo = deviceId && !hasError; + + return ( +
+
+ {showVideo ? ( +
+ {label && ( +
+ {label} +
+ )} +
+ ); +}; + +export default CameraFeed; diff --git a/frontend/src/components/control/TeleopCameraPanel.tsx b/frontend/src/components/control/TeleopCameraPanel.tsx new file mode 100644 index 00000000..cb48a714 --- /dev/null +++ b/frontend/src/components/control/TeleopCameraPanel.tsx @@ -0,0 +1,94 @@ +import React, { useState } from "react"; +import { RefreshCw } from "lucide-react"; +import { Button } from "@/components/ui/button"; +import { Label } from "@/components/ui/label"; +import { Switch } from "@/components/ui/switch"; +import { useRobots } from "@/hooks/useRobots"; +import CameraFeed from "./CameraFeed"; + +/** + * Optional live camera panel for the teleoperation page. Off by default so we + * never call getUserMedia just by landing on the page (same consent pattern as + * the calibration camera toggle). Teleoperation opens no cv2 cameras, so the + * browser can stream them directly while the arm runs. + * + * A strict mirror of the selected robot's configured cameras: one live feed per + * camera on the robot record (e.g. "wrist_cam", "webcam"), stacked vertically. + * If the robot has none configured it shows nothing — teleop never surfaces a + * device that wasn't deliberately added to the robot. + */ +const TeleopCameraPanel: React.FC = () => { + const [enabled, setEnabled] = useState(false); + // Bumped by the retry button to remount the feeds (a fresh getUserMedia + // attempt) — useful if a camera was unplugged and reconnected. + const [reloadKey, setReloadKey] = useState(0); + const { selectedRecord, isLoading: robotsLoading } = useRobots(); + + // Feeds come solely from the robot's configured cameras; each carries a stored + // browser device_id we stream directly. A configured camera whose device is + // currently absent still shows (name + failed-preview placeholder), so the + // user can tell it's expected but not detected. + const configured = selectedRecord?.cameras ?? []; + const feeds = configured.map((c) => ({ + key: c.id, + name: c.name, + deviceId: c.device_id, + })); + + return ( +
+
+

Cameras

+
+ {enabled && feeds.length > 0 && ( + + )} + + +
+
+ + {enabled ? ( + feeds.length > 0 ? ( +
+ {feeds.map((feed) => ( + + ))} +
+ ) : ( +

+ {robotsLoading + ? "Loading robot..." + : "No cameras configured for this robot. Add them during calibration to see live feeds here."} +

+ ) + ) : ( +

+ Turn on to watch your cameras while you teleoperate. +

+ )} +
+ ); +}; + +export default TeleopCameraPanel; diff --git a/frontend/src/components/control/VisualizerPanel.tsx b/frontend/src/components/control/VisualizerPanel.tsx index 93ef0d2a..9dca7861 100644 --- a/frontend/src/components/control/VisualizerPanel.tsx +++ b/frontend/src/components/control/VisualizerPanel.tsx @@ -8,11 +8,14 @@ import Logo from "@/components/Logo"; interface VisualizerPanelProps { onGoBack: () => void; className?: string; + /** Optional content rendered as a column beside the 3D viewer (e.g. a camera panel). */ + rightSlot?: React.ReactNode; } const VisualizerPanel: React.FC = ({ onGoBack, className, + rightSlot, }) => { return (
= ({
+ {rightSlot && ( +
{rightSlot}
+ )} ); }; diff --git a/frontend/src/pages/Teleoperation.tsx b/frontend/src/pages/Teleoperation.tsx index 745d5eaa..b72db2a8 100644 --- a/frontend/src/pages/Teleoperation.tsx +++ b/frontend/src/pages/Teleoperation.tsx @@ -1,6 +1,7 @@ import React, { useCallback, useEffect, useRef } from "react"; import { useNavigate } from "react-router-dom"; import VisualizerPanel from "@/components/control/VisualizerPanel"; +import TeleopCameraPanel from "@/components/control/TeleopCameraPanel"; import { useToast } from "@/hooks/use-toast"; import { useApi } from "@/contexts/ApiContext"; @@ -69,7 +70,11 @@ const TeleoperationPage = () => { return (
- + } + />
);