From 27dea85fb677bb4a9791381e523b3da9488df9cc Mon Sep 17 00:00:00 2001 From: Chandran Date: Sat, 20 Jun 2026 21:09:03 +0200 Subject: [PATCH] feat(record): show a live camera preview during recording Until now the recording page showed only a timer. Show each configured camera's live feed on the page so the user can verify framing and catch a camera that drops mid-episode. The session exposes its connected SO101Follower as a module global once the robot connects, cleared on teardown. A new GET /camera-feed/{name} streams that camera's frames as multipart MJPEG, read via the non-blocking read_latest() peek so it shares the record loop's lock-protected frame buffer with zero device contention (never async_read, which would consume the new-frame event). The recording status reports the configured camera names and the page renders a feed for exactly those, so only cameras the user added are shown. The windows appear from the preparing phase as empty slots, one per configured camera, and fill with the live feed once recording starts. The layout keeps the whole page within one screen: a ResizeObserver measures the space left after the timer and controls, and the windows are sized to the largest that fit, matched to each camera's aspect ratio and the best column count for the number of cameras. --- frontend/src/pages/Recording.tsx | 176 +++++++++++++++++++++++++++++-- lelab/record.py | 60 ++++++++++- lelab/server.py | 23 +++- 3 files changed, 244 insertions(+), 15 deletions(-) diff --git a/frontend/src/pages/Recording.tsx b/frontend/src/pages/Recording.tsx index 840da851..9701db8f 100644 --- a/frontend/src/pages/Recording.tsx +++ b/frontend/src/pages/Recording.tsx @@ -1,4 +1,4 @@ -import React, { useState, useEffect, useCallback, useRef } from "react"; +import React, { useState, useEffect, useCallback, useRef, useMemo } from "react"; import { useNavigate, useLocation } from "react-router-dom"; import { Button } from "@/components/ui/button"; import { @@ -67,6 +67,7 @@ interface BackendStatus { session_elapsed_seconds?: number; session_ended?: boolean; dataset_repo_id?: string; + cameras?: string[]; // Names of the cameras configured for this session available_controls: { stop_recording: boolean; exit_early: boolean; @@ -99,6 +100,60 @@ const Recording = () => { // Guards against React StrictMode double-invocation of the start effect. const startInitiatedRef = useRef(false); + // --- Camera preview layout ------------------------------------------------- + // Aspect ratio (width / height) of the configured cameras, used to size the + // preview windows without letterboxing. Falls back to 4:3 if unknown. + const cameraAspect = useMemo(() => { + const cams = (recordingConfig as unknown as { cameras?: unknown })?.cameras; + const list = Array.isArray(cams) + ? cams + : cams && typeof cams === "object" + ? Object.values(cams as Record) + : []; + const first = list[0] as { width?: number; height?: number } | undefined; + if (first?.width && first?.height) return first.width / first.height; + return 4 / 3; + }, [recordingConfig]); + + // Measure the space left for the camera windows (via ResizeObserver, so it + // re-fits on any viewport change) to size them as large as possible while + // keeping the whole page within one screen (no scroll). + const [cameraArea, setCameraArea] = useState({ w: 0, h: 0 }); + const cameraAreaObserver = useRef(null); + const cameraAreaRef = useCallback((node: HTMLDivElement | null) => { + cameraAreaObserver.current?.disconnect(); + cameraAreaObserver.current = null; + if (node) { + const ro = new ResizeObserver((entries) => { + const r = entries[0].contentRect; + setCameraArea({ w: r.width, h: r.height }); + }); + ro.observe(node); + cameraAreaObserver.current = ro; + } + }, []); + + // Pick the column count and per-window pixel size that maximizes the video + // area within the measured space, given the camera count and aspect ratio. + const cameraCount = backendStatus?.cameras?.length ?? 0; + const cameraWindow = useMemo(() => { + const { w, h } = cameraArea; + if (!cameraCount || w <= 0 || h <= 0) return { width: 0, height: 0 }; + const gap = 12; // matches the grid's gap-3 + let best = { width: 0, height: 0, area: -1 }; + for (let cols = 1; cols <= cameraCount; cols++) { + const rows = Math.ceil(cameraCount / cols); + const cellW = (w - gap * (cols - 1)) / cols; + const cellH = (h - gap * (rows - 1)) / rows; + if (cellW <= 0 || cellH <= 0) continue; + const width = Math.min(cellW, cellH * cameraAspect); + const height = width / cameraAspect; + const area = width * height; + if (area > best.area) best = { width, height, area }; + } + return { width: Math.floor(best.width), height: Math.floor(best.height) }; + }, [cameraArea, cameraCount, cameraAspect]); + const toggleMute = useCallback(() => { setMutedState((prev) => { const next = !prev; @@ -457,9 +512,9 @@ const Recording = () => { const PrimaryIcon = currentPhase === "recording" ? SkipForward : Play; return ( -
-
-
+
+
+
-
-
+
+
Episode {currentEpisode} / {totalEpisodes} @@ -523,7 +578,7 @@ const Recording = () => {
-
+
{
-
+ {/* Camera windows for the cameras configured for this session. Shown + from the preparing phase so the slots are laid out immediately; + each fills with its live feed once the robot is connected and + recording/resetting. This is the flexible region: it absorbs the + space left after the fixed chrome, and cameraWindow sizes each feed + as large as fits so the whole page stays within one screen. */} + {backendStatus.cameras && + backendStatus.cameras.length > 0 && + currentPhase !== "completed" && ( +
+ {backendStatus.cameras.map((name) => ( + + ))} +
+ )} + +
{formatTime(phaseElapsedTime)}
@@ -543,7 +627,7 @@ const Recording = () => {
-
+
{ optimisticPhase !== null || currentPhase === "completed" } - className={`w-full text-white font-semibold py-6 text-lg disabled:opacity-50 ${phaseColor.button}`} + className={`w-full flex-shrink-0 text-white font-semibold py-6 text-lg disabled:opacity-50 ${phaseColor.button}`} > {primaryLabel} @@ -601,4 +685,76 @@ const Recording = () => { ); }; +interface CameraFeedProps { + baseUrl: string; + name: string; + // False during the preparing phase: show an empty slot until the camera is + // connected and streaming. True once recording/resetting, when frames flow. + live: boolean; + // Pixel size computed by the parent to fit the available space. The box is + // sized to the camera's aspect ratio, so the video fills it without letterbox. + width: number; + height: number; +} + +// Renders one recording camera's window at an explicit size. During preparing it +// shows an empty placeholder; once live it plays the backend MJPEG stream. The +// browser renders a `multipart/x-mixed-replace` response natively in an , +// so we just point it at /camera-feed/{name}. If the stream errors before frames +// flow (camera still warming up), retry with a cache-busting key after a delay. +const CameraFeed: React.FC = ({ + baseUrl, + name, + live, + width, + height, +}) => { + const [reloadKey, setReloadKey] = useState(0); + const [hasError, setHasError] = useState(false); + const retryRef = useRef(null); + + const src = `${baseUrl}/camera-feed/${encodeURIComponent(name)}?k=${reloadKey}`; + + useEffect(() => { + return () => { + if (retryRef.current) window.clearTimeout(retryRef.current); + }; + }, []); + + const handleError = useCallback(() => { + setHasError(true); + if (retryRef.current) window.clearTimeout(retryRef.current); + retryRef.current = window.setTimeout(() => { + setHasError(false); + setReloadKey((k) => k + 1); + }, 1500); + }, []); + + // 0 before the first measurement; skip rendering a zero-size box. + if (width <= 0 || height <= 0) return null; + + return ( +
+ {!live ? ( + Getting ready… + ) : hasError ? ( + Connecting feed… + ) : ( + {`${name} + )} + + {name} + +
+ ); +}; + export default Recording; diff --git a/lelab/record.py b/lelab/record.py index 3a086293..f6ecc687 100644 --- a/lelab/record.py +++ b/lelab/record.py @@ -37,6 +37,10 @@ # Global variables for recording state recording_active = False recording_thread: threading.Thread | None = None +# Live SO101Follower for the active session, set once the robot connects so the +# /camera-feed endpoint can peek its cameras' latest frames. Reset to None on +# teardown. Always gated behind `recording_active` by readers. +current_robot = None recording_events = None # Events dict for controlling recording session recording_config = None # Store recording configuration recording_start_time = None # Track when recording started @@ -437,6 +441,10 @@ def handle_recording_status() -> dict[str, Any]: status["current_episode"] = current_episode status["total_episodes"] = recording_config.num_episodes status["saved_episodes"] = saved_episodes # Track completed episodes + # Names of the cameras the user configured for this session. The frontend + # renders a live /camera-feed/{name} preview for exactly these — nothing + # else — so only configured cameras are ever shown. + status["cameras"] = list(recording_config.cameras.keys()) # Add session start time if available if recording_start_time: @@ -459,6 +467,50 @@ def handle_recording_status() -> dict[str, Any]: return status +def camera_feed_frames(cam_key: str, fps: float = 15.0): + """Yield multipart-MJPEG chunks of one recording camera's latest frames. + + Reads via the camera's non-blocking `read_latest()` peek, so it shares the + record loop's lock-protected frame buffer with zero device contention (never + `async_read()`, which would consume the new-frame event and could disturb + record timing). Streams only while a session is live and the named camera + exists; ends cleanly otherwise so the browser stops. Capped well below + the record fps so it stays a passive observer. + """ + import cv2 + + interval = 1.0 / fps + # The session sets `current_robot` only after the robot connects (which is + # preceded by a ~2s camera-release wait), so give it a moment to appear. + deadline = time.time() + 15.0 + while recording_active and current_robot is None and time.time() < deadline: + time.sleep(0.1) + + while recording_active and current_robot is not None: + cam = current_robot.cameras.get(cam_key) + if cam is None: + # Unknown/removed camera — nothing to stream. + break + try: + # read_latest() returns an RGB frame (OpenCVCamera default color + # mode); may raise if the camera hasn't produced a fresh frame yet. + frame = cam.read_latest() + except (TimeoutError, RuntimeError): + time.sleep(interval) + continue + except Exception as e: + logger.warning("camera-feed %s: unexpected read error: %s", cam_key, e) + break + + bgr = cv2.cvtColor(frame, cv2.COLOR_RGB2BGR) + ok, buf = cv2.imencode(".jpg", bgr) + if not ok: + time.sleep(interval) + continue + yield b"--frame\r\nContent-Type: image/jpeg\r\n\r\n" + buf.tobytes() + b"\r\n" + time.sleep(interval) + + def handle_get_dataset_info(request: DatasetInfoRequest) -> dict[str, Any]: """Return dataset metadata — from the most recent session if it matches, otherwise by loading the local LeRobot cache copy.""" @@ -583,7 +635,7 @@ def record_with_web_events(cfg: RecordConfig, web_events: dict) -> LeRobotDatase from lerobot.utils.feature_utils import hw_to_dataset_features from lerobot.utils.utils import log_say - global current_phase, phase_start_time, current_episode, saved_episodes + global current_phase, phase_start_time, current_episode, saved_episodes, current_robot robot = make_robot_from_config(cfg.robot) teleop = make_teleoperator_from_config(cfg.teleop) if cfg.teleop is not None else None @@ -635,6 +687,9 @@ def record_with_web_events(cfg: RecordConfig, web_events: dict) -> LeRobotDatase logger.info("🔧 ROBOT CONNECTION: Attempting to connect robot...") robot.connect() logger.info("✅ ROBOT CONNECTION: Robot connected successfully") + # Expose the connected robot so /camera-feed can stream its cameras' + # latest frames during the session (cleared in the finally below). + current_robot = robot except Exception as e: logger.error(f"❌ ROBOT CONNECTION: Failed to connect robot: {e}") # If robot connection fails due to camera conflict, provide clear error @@ -862,6 +917,9 @@ def record_with_web_events(cfg: RecordConfig, web_events: dict) -> LeRobotDatase log_say("Stop recording", cfg.play_sounds, blocking=True) finally: + # Stop the camera-feed endpoint from reading before tearing the cameras + # down, so it can't peek a half-disconnected device. + current_robot = None robot.disconnect() if teleop: teleop.disconnect() diff --git a/lelab/server.py b/lelab/server.py index 519e0c72..e7fbc792 100644 --- a/lelab/server.py +++ b/lelab/server.py @@ -28,7 +28,7 @@ from fastapi import FastAPI, HTTPException, Request, WebSocket, WebSocketDisconnect from fastapi.middleware.cors import CORSMiddleware -from fastapi.responses import JSONResponse +from fastapi.responses import JSONResponse, StreamingResponse from fastapi.staticfiles import StaticFiles from pydantic import BaseModel from starlette.datastructures import Headers @@ -36,7 +36,8 @@ from starlette.responses import Response from starlette.types import Scope -from . import datasets as dataset_browser +# Import our custom recording functionality +from . import datasets as dataset_browser, record as _record # Import our custom calibration functionality from .calibrate import CalibrationRequest, calibration_manager @@ -47,8 +48,6 @@ JobTarget, job_registry, ) - -# Import our custom recording functionality from .record import ( DatasetInfoRequest, RecordingRequest, @@ -443,6 +442,22 @@ def recording_status(): return handle_recording_status() +@app.get("/camera-feed/{cam_key}") +def camera_feed(cam_key: str): + """Live MJPEG preview of one configured camera during an active recording. + + Browsers render `multipart/x-mixed-replace` directly in an , so the + frontend just points an at this URL. Only valid while a session is + active; the generator ends itself when recording stops. + """ + if not _record.recording_active: + raise HTTPException(status_code=409, detail="No active recording session") + return StreamingResponse( + _record.camera_feed_frames(cam_key), + media_type="multipart/x-mixed-replace; boundary=frame", + ) + + @app.post("/recording-exit-early") def recording_exit_early(): """Skip to next episode (replaces right arrow key)"""