From 087aaeadb7fd250c8e21916f1fdd7d01fd1a7a89 Mon Sep 17 00:00:00 2001 From: Carles Andres Date: Fri, 5 Jun 2026 16:03:28 +0100 Subject: [PATCH] fix: use beginFrame for headless recording capture --- README.md | 2 + .../core/src/__tests__/recorder.test.ts | 163 ++++++++++++++++++ packages/@webreel/core/src/recorder.ts | 37 ++-- packages/@webreel/core/src/types.ts | 13 ++ packages/webreel/README.md | 2 + packages/webreel/src/commands/record.ts | 2 +- .../lib/__tests__/runner-recording.test.ts | 154 +++++++++++++++++ packages/webreel/src/lib/runner.ts | 31 ++++ packages/webreel/vitest.config.ts | 6 + skills/webreel/SKILL.md | 2 +- 10 files changed, 386 insertions(+), 26 deletions(-) create mode 100644 packages/@webreel/core/src/__tests__/recorder.test.ts create mode 100644 packages/webreel/src/lib/__tests__/runner-recording.test.ts diff --git a/README.md b/README.md index 1e461a4..98d8334 100644 --- a/README.md +++ b/README.md @@ -118,6 +118,8 @@ webreel record --watch webreel record --verbose ``` +In headless recording mode, `webreel record --frames` saves PNG source frames in `.webreel/frames/`. + ### Preview Run a video in a visible browser window without recording: diff --git a/packages/@webreel/core/src/__tests__/recorder.test.ts b/packages/@webreel/core/src/__tests__/recorder.test.ts new file mode 100644 index 0000000..f236886 --- /dev/null +++ b/packages/@webreel/core/src/__tests__/recorder.test.ts @@ -0,0 +1,163 @@ +import { EventEmitter } from "node:events"; +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const spawnState = vi.hoisted(() => ({ + stdinWrites: [] as Buffer[], + spawnArgs: [] as string[], + ensureFfmpegMock: vi.fn(async () => "ffmpeg"), + finalizeMp4Mock: vi.fn(), + finalizeWebmMock: vi.fn(), + finalizeGifMock: vi.fn(), + writeFileSyncMock: vi.fn(), +})); + +vi.mock("node:fs", async () => { + const actual = await vi.importActual("node:fs"); + return { + ...actual, + writeFileSync: spawnState.writeFileSyncMock, + }; +}); + +vi.mock("../ffmpeg.js", () => ({ + ensureFfmpeg: spawnState.ensureFfmpegMock, +})); + +vi.mock("../media.js", () => ({ + finalizeMp4: spawnState.finalizeMp4Mock, + finalizeWebm: spawnState.finalizeWebmMock, + finalizeGif: spawnState.finalizeGifMock, +})); + +vi.mock("node:child_process", () => ({ + spawn: vi.fn((_command: string, args: string[]) => { + spawnState.spawnArgs = args; + + const proc = new EventEmitter() as EventEmitter & { + stdin: EventEmitter & { + writable: boolean; + write: (chunk: Buffer) => boolean; + end: () => void; + }; + stdout: EventEmitter; + stderr: EventEmitter; + exitCode: number | null; + kill: () => void; + }; + + const stdin = new EventEmitter() as EventEmitter & { + writable: boolean; + write: (chunk: Buffer) => boolean; + end: () => void; + }; + stdin.writable = true; + stdin.write = (chunk: Buffer) => { + spawnState.stdinWrites.push(chunk); + return true; + }; + stdin.end = () => { + proc.exitCode = 0; + proc.emit("close", 0); + }; + + proc.stdin = stdin; + proc.stdout = new EventEmitter(); + proc.stderr = new EventEmitter(); + proc.exitCode = null; + proc.kill = () => { + proc.exitCode = 0; + proc.emit("close", 0); + }; + + return proc; + }), +})); + +import { Recorder } from "../recorder.js"; +import type { CDPClient } from "../types.js"; + +describe("Recorder", () => { + beforeEach(() => { + spawnState.stdinWrites = []; + spawnState.spawnArgs = []; + spawnState.ensureFfmpegMock.mockClear(); + spawnState.finalizeMp4Mock.mockClear(); + spawnState.finalizeWebmMock.mockClear(); + spawnState.finalizeGifMock.mockClear(); + spawnState.writeFileSyncMock.mockClear(); + }); + + it("captures recording frames with HeadlessExperimental.beginFrame PNG screenshots", async () => { + const screenshotData = Buffer.from("png-frame").toString("base64"); + const beginFrame = vi + .fn() + .mockResolvedValueOnce({ hasDamage: true, screenshotData }) + .mockImplementationOnce(() => new Promise(() => undefined)); + const captureScreenshot = vi.fn(); + + const client = { + Runtime: { evaluate: vi.fn().mockResolvedValue({ result: {} }) }, + Page: { captureScreenshot }, + HeadlessExperimental: { beginFrame }, + } as unknown as CDPClient; + + const recorder = new Recorder(1080, 1080, { fps: 30, framesDir: "/tmp/frames" }); + recorder.setTimeline({ tick: vi.fn(), toJSON: vi.fn(() => null) } as never); + + await recorder.start(client, "/tmp/out.mp4"); + await vi.waitFor(() => { + expect(beginFrame).toHaveBeenCalledWith({ + screenshot: { format: "png", optimizeForSpeed: true }, + }); + }); + await recorder.stop(); + + expect(captureScreenshot).not.toHaveBeenCalled(); + expect(spawnState.spawnArgs).toContain("png"); + expect(spawnState.writeFileSyncMock).toHaveBeenCalledWith( + expect.stringMatching(/frame-00001\.png$/), + expect.any(Buffer), + ); + }); + + it("reuses the previous frame when beginFrame returns no screenshotData", async () => { + const firstFrame = Buffer.from("png-frame-1"); + const beginFrame = vi + .fn() + .mockResolvedValueOnce({ + hasDamage: true, + screenshotData: firstFrame.toString("base64"), + }) + .mockResolvedValueOnce({ hasDamage: false }) + .mockImplementationOnce(() => new Promise(() => undefined)); + + const client = { + Runtime: { evaluate: vi.fn().mockResolvedValue({ result: {} }) }, + Page: { captureScreenshot: vi.fn() }, + HeadlessExperimental: { beginFrame }, + } as unknown as CDPClient; + + const recorder = new Recorder(1080, 1080, { fps: 30, framesDir: "/tmp/frames" }); + recorder.setTimeline({ tick: vi.fn(), toJSON: vi.fn(() => null) } as never); + + await recorder.start(client, "/tmp/out.mp4"); + await vi.waitFor(() => { + expect(spawnState.stdinWrites).toHaveLength(2); + }); + await recorder.stop(); + + expect(spawnState.stdinWrites).toHaveLength(2); + expect(spawnState.stdinWrites[0]).toEqual(firstFrame); + expect(spawnState.stdinWrites[1]).toEqual(firstFrame); + expect(spawnState.writeFileSyncMock).toHaveBeenNthCalledWith( + 1, + expect.stringMatching(/frame-00001\.png$/), + firstFrame, + ); + expect(spawnState.writeFileSyncMock).toHaveBeenNthCalledWith( + 2, + expect.stringMatching(/frame-00002\.png$/), + firstFrame, + ); + }); +}); diff --git a/packages/@webreel/core/src/recorder.ts b/packages/@webreel/core/src/recorder.ts index 94719b5..79b4751 100644 --- a/packages/@webreel/core/src/recorder.ts +++ b/packages/@webreel/core/src/recorder.ts @@ -19,7 +19,6 @@ export class Recorder { private outputHeight: number; private sfx: SfxConfig | undefined; private fps: number; - private frameMs: number; private crf: number; private ffmpegPath = "ffmpeg"; private ffmpegProcess: ChildProcess | null = null; @@ -31,6 +30,7 @@ export class Recorder { private framesDir: string | null = null; private stopResolve: (() => void) | null = null; private stoppedPromise: Promise | null = null; + private lastFrameBuffer: Buffer | null = null; constructor( outputWidth = DEFAULT_VIEWPORT_SIZE, @@ -41,7 +41,6 @@ export class Recorder { this.outputHeight = outputHeight; this.sfx = options?.sfx; this.fps = options?.fps ?? TARGET_FPS; - this.frameMs = 1000 / this.fps; this.crf = options?.crf ?? 18; if (options?.framesDir) { this.framesDir = options.framesDir; @@ -74,6 +73,7 @@ export class Recorder { this.frameCount = 0; this.droppedFrames = 0; this.running = true; + this.lastFrameBuffer = null; this.events = []; this.ctx = ctx ?? null; if (this.ctx) this.ctx.setRecorder(this); @@ -91,7 +91,7 @@ export class Recorder { "-framerate", String(this.fps), "-c:v", - "mjpeg", + "png", "-i", "pipe:0", "-c:v", @@ -159,7 +159,6 @@ export class Recorder { } private async captureLoop(client: CDPClient) { - let lastFrameTime = Date.now(); let consecutiveErrors = 0; while (this.running) { @@ -174,37 +173,27 @@ export class Recorder { ); if (!evalResult) break; } - const screenshotResult = await this.raceStop( - client.Page.captureScreenshot({ - format: "jpeg", - quality: 60, - optimizeForSpeed: true, + const frameResult = await this.raceStop( + client.HeadlessExperimental.beginFrame({ + screenshot: { format: "png", optimizeForSpeed: true }, }), ); - if (!screenshotResult) break; + if (!frameResult) break; - const buffer = Buffer.from(screenshotResult.data, "base64"); - const now = Date.now(); - const elapsed = now - lastFrameTime; - const frameSlots = Math.min(3, Math.max(1, Math.round(elapsed / this.frameMs))); - - if (frameSlots > 1) { - for (let i = 0; i < frameSlots - 1; i++) { - if (this.timeline) this.timeline.tickDuplicate(); - await this.writeFrame(buffer); - this.frameCount++; - } - } + const buffer = frameResult.screenshotData + ? Buffer.from(frameResult.screenshotData, "base64") + : this.lastFrameBuffer; + if (!buffer) continue; + this.lastFrameBuffer = buffer; await this.writeFrame(buffer); this.frameCount++; if (this.framesDir) { const padded = String(this.frameCount).padStart(5, "0"); - writeFileSync(resolve(this.framesDir, `frame-${padded}.jpg`), buffer); + writeFileSync(resolve(this.framesDir, `frame-${padded}.png`), buffer); } - lastFrameTime = now; consecutiveErrors = 0; } catch (err) { if (!this.running) break; diff --git a/packages/@webreel/core/src/types.ts b/packages/@webreel/core/src/types.ts index 950aca0..39b9e05 100644 --- a/packages/@webreel/core/src/types.ts +++ b/packages/@webreel/core/src/types.ts @@ -46,6 +46,19 @@ export type CDPClient = { mobile: boolean; }) => Promise; }; + HeadlessExperimental: { + enable: () => Promise; + beginFrame: (params?: { + frameTimeTicks?: number; + interval?: number; + noDisplayUpdates?: boolean; + screenshot?: { + format?: "jpeg" | "png" | "webp"; + quality?: number; + optimizeForSpeed?: boolean; + }; + }) => Promise<{ hasDamage: boolean; screenshotData?: string }>; + }; DOM: { enable: () => Promise; }; diff --git a/packages/webreel/README.md b/packages/webreel/README.md index 68e5df0..f72f71d 100644 --- a/packages/webreel/README.md +++ b/packages/webreel/README.md @@ -105,6 +105,8 @@ webreel record -c custom.config.json When run without arguments, webreel reads `webreel.config.json` from the current directory and records all videos. Provide video names to record specific videos only. +In headless recording mode, `webreel record --frames` saves PNG source frames in `.webreel/frames/`. + ### `webreel preview` Run a video in a visible browser window without recording. diff --git a/packages/webreel/src/commands/record.ts b/packages/webreel/src/commands/record.ts index d3a2ec7..81c706b 100644 --- a/packages/webreel/src/commands/record.ts +++ b/packages/webreel/src/commands/record.ts @@ -75,7 +75,7 @@ export const recordCommand = new Command("record") .option("--verbose", "Log each step as it executes") .option("--watch", "Re-record when config files change") .option("--dry-run", "Print the resolved config and step list without recording") - .option("--frames", "Save raw frames as JPEGs in .webreel/frames/") + .option("--frames", "Save raw frames as PNGs in .webreel/frames/") .action( async ( videoNames: string[], diff --git a/packages/webreel/src/lib/__tests__/runner-recording.test.ts b/packages/webreel/src/lib/__tests__/runner-recording.test.ts new file mode 100644 index 0000000..6a64e25 --- /dev/null +++ b/packages/webreel/src/lib/__tests__/runner-recording.test.ts @@ -0,0 +1,154 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const pauseMock = vi.fn(async () => undefined); +const beginFrameMock = vi.fn(async () => ({ hasDamage: true })); +const enableMock = vi.fn(async () => undefined); +const recorderStartMock = vi.fn(async () => undefined); +const recorderSetTimelineMock = vi.fn(); +const recorderStopMock = vi.fn(async () => undefined); +const recorderGetTempVideoPathMock = vi.fn(() => "/tmp/_rec.mp4"); +const navigateMock = vi.fn(async () => undefined); +const launchChromeMock = vi.fn(async () => ({ + port: 9222, + kill: vi.fn(), + process: {} as never, +})); +const connectCDPMock = vi.fn(async () => ({ + close: vi.fn(async () => undefined), + Page: { enable: vi.fn(async () => undefined) }, + Runtime: { enable: vi.fn(async () => undefined) }, + Emulation: { setDeviceMetricsOverride: vi.fn(async () => undefined) }, + HeadlessExperimental: { + enable: enableMock, + beginFrame: beginFrameMock, + }, +})); + +vi.mock("@webreel/core", () => ({ + DEFAULT_VIEWPORT_SIZE: 1080, + RecordingContext: class { + resetCursorPosition() {} + setClickDwell() {} + getCursorPosition() { + return { x: 0, y: 0 }; + } + setMode() {} + setTimeline() {} + }, + InteractionTimeline: class { + constructor() {} + toJSON() { + return { width: 1080, height: 1080, fps: 60, zoom: 1, frames: [] }; + } + }, + Recorder: class { + setTimeline = recorderSetTimelineMock; + start = recorderStartMock; + stop = recorderStopMock; + getTempVideoPath = recorderGetTempVideoPathMock; + }, + compose: vi.fn(), + connectCDP: connectCDPMock, + launchChrome: launchChromeMock, + navigate: navigateMock, + waitForSelector: vi.fn(async () => undefined), + waitForText: vi.fn(async () => undefined), + injectOverlays: vi.fn(async () => undefined), + pause: pauseMock, + findElementByText: vi.fn(), + findElementBySelector: vi.fn(), + clickAt: vi.fn(async () => undefined), + pressKey: vi.fn(async () => undefined), + typeText: vi.fn(async () => undefined), + dragFromTo: vi.fn(async () => undefined), + moveCursorTo: vi.fn(async () => undefined), + captureScreenshot: vi.fn(async () => undefined), + ensureFfmpeg: vi.fn(async () => "ffmpeg"), + extractThumbnail: vi.fn(), + moveFileSync: vi.fn(), +})); + +describe("runVideo recording setup", () => { + beforeEach(() => { + vi.clearAllMocks(); + recorderGetTempVideoPathMock.mockReturnValue("/tmp/_rec.mp4"); + }); + + it("enables HeadlessExperimental before starting the recorder", async () => { + vi.useFakeTimers(); + try { + const releaseNavigate = { current: null as null | (() => void) }; + navigateMock.mockImplementationOnce( + () => + new Promise((resolve) => { + releaseNavigate.current = () => resolve(undefined); + }), + ); + + const { runVideo } = await import("../runner.js"); + + const runPromise = runVideo( + { + name: "demo", + url: "https://example.com", + steps: [], + output: "/tmp/demo.mp4", + }, + { record: true, configDir: "/tmp" }, + ); + + await vi.advanceTimersByTimeAsync(16); + if (releaseNavigate.current) releaseNavigate.current(); + await runPromise; + + expect(enableMock).toHaveBeenCalledTimes(1); + expect(beginFrameMock).toHaveBeenCalled(); + expect(recorderSetTimelineMock).toHaveBeenCalledTimes(1); + expect(recorderStartMock).toHaveBeenCalledTimes(1); + expect(enableMock.mock.invocationCallOrder[0]).toBeLessThan( + recorderStartMock.mock.invocationCallOrder[0], + ); + } finally { + vi.useRealTimers(); + } + }); + + it("stops the pre-record frame pump when setup fails before recording starts", async () => { + vi.useFakeTimers(); + try { + const releaseNavigate = { current: null as null | (() => void) }; + navigateMock.mockImplementationOnce( + () => + new Promise((resolve) => { + releaseNavigate.current = () => resolve(undefined); + }), + ); + + const { runVideo } = await import("../runner.js"); + + const runPromise = runVideo( + { + name: "demo", + url: "https://example.com", + steps: [], + output: "/tmp/demo.mp4", + theme: { cursor: { image: "missing-cursor.svg" } }, + }, + { record: true, configDir: "/tmp" }, + ); + + await vi.advanceTimersByTimeAsync(16); + expect(beginFrameMock).toHaveBeenCalledTimes(1); + + if (releaseNavigate.current) releaseNavigate.current(); + + await expect(runPromise).rejects.toThrow(/Failed to read cursor SVG/); + + await vi.advanceTimersByTimeAsync(32); + expect(beginFrameMock).toHaveBeenCalledTimes(1); + expect(recorderStartMock).not.toHaveBeenCalled(); + } finally { + vi.useRealTimers(); + } + }); +}); diff --git a/packages/webreel/src/lib/runner.ts b/packages/webreel/src/lib/runner.ts index e97ee17..e5f0fd7 100644 --- a/packages/webreel/src/lib/runner.ts +++ b/packages/webreel/src/lib/runner.ts @@ -153,12 +153,41 @@ export async function runVideo( const chrome = await launchChrome({ headless: shouldRecord }); let clientRef: CDPClient | null = null; let recorder: Recorder | null = null; + let framePumpRunning = false; + let framePumpBusy = false; + let framePumpTimer: ReturnType | null = null; + + const stopFramePump = async () => { + framePumpRunning = false; + if (framePumpTimer) { + clearInterval(framePumpTimer); + framePumpTimer = null; + } + while (framePumpBusy) { + await pause(5); + } + }; try { const client = await connectCDP(chrome.port); clientRef = client; await client.Page.enable(); await client.Runtime.enable(); + if (shouldRecord) { + await client.HeadlessExperimental.enable(); + framePumpRunning = true; + framePumpTimer = setInterval(async () => { + if (!framePumpRunning || framePumpBusy) return; + framePumpBusy = true; + try { + await client.HeadlessExperimental.beginFrame(); + } catch { + // The client may already be closing, or the recorder may have taken over. + } finally { + framePumpBusy = false; + } + }, 16); + } await client.Emulation.setDeviceMetricsOverride({ width: cssWidth, height: cssHeight, @@ -235,6 +264,7 @@ export async function runVideo( sfx: config.sfx, }); recorder.setTimeline(timeline); + await stopFramePump(); await recorder.start(client, outputPath, ctx); } else { ctx.setMode("preview"); @@ -453,6 +483,7 @@ export async function runVideo( console.warn("Failed to stop recorder:", err); } } + await stopFramePump(); if (clientRef) { try { await clientRef.close(); diff --git a/packages/webreel/vitest.config.ts b/packages/webreel/vitest.config.ts index f612c07..a7e5544 100644 --- a/packages/webreel/vitest.config.ts +++ b/packages/webreel/vitest.config.ts @@ -1,6 +1,12 @@ import { defineConfig } from "vitest/config"; +import { resolve } from "node:path"; export default defineConfig({ + resolve: { + alias: { + "@webreel/core": resolve(import.meta.dirname, "../@webreel/core/src/index.ts"), + }, + }, test: { exclude: ["dist/**", "node_modules/**"], }, diff --git a/skills/webreel/SKILL.md b/skills/webreel/SKILL.md index e7d9006..9c7cabb 100644 --- a/skills/webreel/SKILL.md +++ b/skills/webreel/SKILL.md @@ -86,7 +86,7 @@ webreel record -c custom.config.json # custom config path webreel record --watch # re-record on config change webreel record --verbose # log each step webreel record --dry-run # print resolved config only -webreel record --frames # save raw JPEGs to .webreel/frames/ +webreel record --frames # save raw PNGs to .webreel/frames/ ``` ### preview