From 0f0f201d3d57ba8c8d7450497de14b01cc2c2450 Mon Sep 17 00:00:00 2001 From: Ariel Conti Date: Thu, 5 Mar 2026 01:25:38 +0000 Subject: [PATCH 01/48] fix(type): use insertText for framework-compatible input typing Switch selector-targeted type steps to CDP Input.insertText so React and other framework-controlled inputs receive proper input/change events. Fix resolveTarget receiving StepType.text as an element query by destructuring only { selector, within } before lookup. Add tests for both typeText method paths and resolveTarget field exclusion. --- .../core/src/__tests__/actions.test.ts | 47 +++++++++++- packages/@webreel/core/src/actions.ts | 71 +++++++++++-------- packages/@webreel/core/src/types.ts | 1 + .../webreel/src/lib/__tests__/runner.test.ts | 67 ++++++++++++++++- packages/webreel/src/lib/runner.ts | 11 ++- 5 files changed, 160 insertions(+), 37 deletions(-) diff --git a/packages/@webreel/core/src/__tests__/actions.test.ts b/packages/@webreel/core/src/__tests__/actions.test.ts index 55301b4..b8da5a8 100644 --- a/packages/@webreel/core/src/__tests__/actions.test.ts +++ b/packages/@webreel/core/src/__tests__/actions.test.ts @@ -1,4 +1,4 @@ -import { describe, it, expect } from "vitest"; +import { describe, it, expect, vi } from "vitest"; import { modKey, RecordingContext, @@ -8,10 +8,12 @@ import { modLabel, modKeyInfo, resolveCommands, + typeText, KEY_CODES, CHAR_CODES, SHORTCUT_COMMANDS, } from "../actions.js"; +import type { CDPClient } from "../types.js"; describe("modKey", () => { it("returns cmd or ctrl based on platform", () => { @@ -302,3 +304,46 @@ describe("RecordingContext", () => { expect(events).toEqual(["rec:key"]); }); }); + +describe("typeText", () => { + function createMockClient() { + return { + Input: { + insertText: vi.fn().mockResolvedValue(undefined), + dispatchKeyEvent: vi.fn().mockResolvedValue(undefined), + }, + } as unknown as CDPClient & { + Input: { + insertText: ReturnType; + dispatchKeyEvent: ReturnType; + }; + }; + } + + it("uses insertText when method is insertText", async () => { + const ctx = new RecordingContext(); + const client = createMockClient(); + await typeText(ctx, client, "ab", 0, { method: "insertText" }); + expect(client.Input.insertText).toHaveBeenCalledTimes(2); + expect(client.Input.insertText).toHaveBeenNthCalledWith(1, { text: "a" }); + expect(client.Input.insertText).toHaveBeenNthCalledWith(2, { text: "b" }); + expect(client.Input.dispatchKeyEvent).not.toHaveBeenCalled(); + }); + + it("uses dispatchKeyEvent when method is dispatchKeyEvent", async () => { + const ctx = new RecordingContext(); + const client = createMockClient(); + await typeText(ctx, client, "ab", 0, { method: "dispatchKeyEvent" }); + // 3 events per char: rawKeyDown, char, keyUp + expect(client.Input.dispatchKeyEvent).toHaveBeenCalledTimes(6); + expect(client.Input.insertText).not.toHaveBeenCalled(); + }); + + it("defaults to dispatchKeyEvent when no method specified", async () => { + const ctx = new RecordingContext(); + const client = createMockClient(); + await typeText(ctx, client, "a", 0); + expect(client.Input.dispatchKeyEvent).toHaveBeenCalledTimes(3); + expect(client.Input.insertText).not.toHaveBeenCalled(); + }); +}); diff --git a/packages/@webreel/core/src/actions.ts b/packages/@webreel/core/src/actions.ts index 6d7827a..045db2b 100644 --- a/packages/@webreel/core/src/actions.ts +++ b/packages/@webreel/core/src/actions.ts @@ -623,43 +623,52 @@ export async function typeText( client: CDPClient, text: string, delayMs = 120, + options?: { method?: "dispatchKeyEvent" | "insertText" }, ): Promise { + const useInsertText = options?.method === "insertText"; + for (let i = 0; i < text.length; i++) { const char = text[i]; - const charInfo = CHAR_CODES[char]; - const isLetter = /^[a-zA-Z]$/.test(char); - - let code: string; - let keyCode: number; - - if (charInfo) { - code = charInfo.code; - keyCode = charInfo.keyCode; - } else if (isLetter) { - code = `Key${char.toUpperCase()}`; - keyCode = char.toUpperCase().charCodeAt(0); + + if (useInsertText) { + await client.Input.insertText({ text: char }); } else { - code = ""; - keyCode = 0; + const charInfo = CHAR_CODES[char]; + const isLetter = /^[a-zA-Z]$/.test(char); + + let code: string; + let keyCode: number; + + if (charInfo) { + code = charInfo.code; + keyCode = charInfo.keyCode; + } else if (isLetter) { + code = `Key${char.toUpperCase()}`; + keyCode = char.toUpperCase().charCodeAt(0); + } else { + code = ""; + keyCode = 0; + } + + await client.Input.dispatchKeyEvent({ + type: "rawKeyDown", + key: char, + code, + windowsVirtualKeyCode: keyCode, + }); + await client.Input.dispatchKeyEvent({ + type: "char", + key: char, + text: char, + }); + await client.Input.dispatchKeyEvent({ + type: "keyUp", + key: char, + code, + windowsVirtualKeyCode: keyCode, + }); } - await client.Input.dispatchKeyEvent({ - type: "rawKeyDown", - key: char, - code, - windowsVirtualKeyCode: keyCode, - }); - await client.Input.dispatchKeyEvent({ - type: "char", - key: char, - text: char, - }); - await client.Input.dispatchKeyEvent({ - type: "keyUp", - key: char, - code, - windowsVirtualKeyCode: keyCode, - }); ctx.markEvent("key"); if (ctx.isRecording) { const waitStart = Date.now(); diff --git a/packages/@webreel/core/src/types.ts b/packages/@webreel/core/src/types.ts index 950aca0..fd6bd7c 100644 --- a/packages/@webreel/core/src/types.ts +++ b/packages/@webreel/core/src/types.ts @@ -37,6 +37,7 @@ export type CDPClient = { modifiers?: number; commands?: string[]; }) => Promise; + insertText: (params: { text: string }) => Promise; }; Emulation: { setDeviceMetricsOverride: (params: { diff --git a/packages/webreel/src/lib/__tests__/runner.test.ts b/packages/webreel/src/lib/__tests__/runner.test.ts index 1c9f10f..3c76b33 100644 --- a/packages/webreel/src/lib/__tests__/runner.test.ts +++ b/packages/webreel/src/lib/__tests__/runner.test.ts @@ -1,7 +1,13 @@ -import { describe, it, expect, vi } from "vitest"; +import { describe, it, expect, vi, afterEach } from "vitest"; import { resolve } from "node:path"; import { pathToFileURL } from "node:url"; -import { formatStep, resolveKeyTarget, resolveUrl, randomPointInBox } from "../runner.js"; +import { + formatStep, + resolveKeyTarget, + resolveUrl, + randomPointInBox, + resolveTarget, +} from "../runner.js"; import type { Step } from "../types.js"; describe("formatStep", () => { @@ -172,3 +178,60 @@ describe("randomPointInBox", () => { } }); }); + +vi.mock("@webreel/core", async () => { + const actual = await vi.importActual("@webreel/core"); + return { + ...actual, + findElementByText: vi.fn(), + findElementBySelector: vi.fn(), + }; +}); + +import { findElementByText, findElementBySelector } from "@webreel/core"; + +const mockedFindByText = vi.mocked(findElementByText); +const mockedFindBySelector = vi.mocked(findElementBySelector); + +describe("resolveTarget", () => { + const mockClient = {} as never; + const mockBox = { x: 10, y: 20, width: 100, height: 50 }; + + afterEach(() => { + vi.clearAllMocks(); + }); + + it("resolves by selector", async () => { + mockedFindBySelector.mockResolvedValue(mockBox); + const result = await resolveTarget(mockClient, { selector: "#foo" }); + expect(result).toEqual(mockBox); + expect(mockedFindBySelector).toHaveBeenCalledWith(mockClient, "#foo", undefined); + }); + + it("resolves by text", async () => { + mockedFindByText.mockResolvedValue(mockBox); + const result = await resolveTarget(mockClient, { text: "Hello" }); + expect(result).toEqual(mockBox); + expect(mockedFindByText).toHaveBeenCalledWith(mockClient, "Hello", undefined); + }); + + it("passes within to findElementBySelector", async () => { + mockedFindBySelector.mockResolvedValue(mockBox); + await resolveTarget(mockClient, { selector: "#input", within: ".modal" }); + expect(mockedFindBySelector).toHaveBeenCalledWith(mockClient, "#input", ".modal"); + expect(mockedFindByText).not.toHaveBeenCalled(); + }); + + it("throws when neither text nor selector provided", async () => { + await expect(resolveTarget(mockClient, {})).rejects.toThrow( + 'resolveTarget requires "text" or "selector"', + ); + }); + + it("throws when element not found", async () => { + mockedFindBySelector.mockResolvedValue(null); + await expect(resolveTarget(mockClient, { selector: "#missing" })).rejects.toThrow( + "Element not found", + ); + }); +}); diff --git a/packages/webreel/src/lib/runner.ts b/packages/webreel/src/lib/runner.ts index 3afee4b..7dcc033 100644 --- a/packages/webreel/src/lib/runner.ts +++ b/packages/webreel/src/lib/runner.ts @@ -69,7 +69,7 @@ export function resolveKeyTarget(target: string | ElementTarget): string { return target.selector ?? ""; } -async function resolveTarget( +export async function resolveTarget( client: CDPClient, opts: { text?: string; selector?: string; within?: string }, ): Promise { @@ -283,7 +283,10 @@ export async function runVideo( case "type": { if (step.selector) { - const box = await resolveTarget(client, step); + const box = await resolveTarget(client, { + selector: step.selector, + within: step.within, + }); const { x: tx, y: ty } = randomPointInBox(box); await clickAt(ctx, client, tx, ty); await client.Runtime.evaluate({ @@ -291,7 +294,9 @@ export async function runVideo( }); await pause(300 + Math.random() * 200); } - await typeText(ctx, client, step.text, step.charDelay); + await typeText(ctx, client, step.text, step.charDelay, { + method: step.selector ? "insertText" : "dispatchKeyEvent", + }); break; } From 26992dcc5b11317a1fac7721af4ceb5a3af50fb4 Mon Sep 17 00:00:00 2001 From: Ariel Conti Date: Thu, 5 Mar 2026 10:18:30 +0000 Subject: [PATCH 02/48] fix(recording): resolve typeText hang, stale frames, and orphaned Chrome - Skip waitForNextTick when charDelay is 0 to prevent deadlock between typing loop and capture loop - Add double-rAF sync before captureScreenshot to ensure fresh compositor frames during client-side navigation - Make chrome.kill() async with SIGKILL fallback after 3s timeout - Add SIGINT/SIGTERM handlers to non-watch record command Fixes #11 --- packages/@webreel/core/src/actions.ts | 2 +- packages/@webreel/core/src/chrome.ts | 27 ++++++++++++++++++++----- packages/@webreel/core/src/recorder.ts | 12 +++++++++++ packages/webreel/src/commands/record.ts | 20 ++++++++++++++++-- packages/webreel/src/lib/runner.ts | 2 +- 5 files changed, 54 insertions(+), 9 deletions(-) diff --git a/packages/@webreel/core/src/actions.ts b/packages/@webreel/core/src/actions.ts index 045db2b..b076bcd 100644 --- a/packages/@webreel/core/src/actions.ts +++ b/packages/@webreel/core/src/actions.ts @@ -670,7 +670,7 @@ export async function typeText( } ctx.markEvent("key"); - if (ctx.isRecording) { + if (ctx.isRecording && delayMs > 0) { const waitStart = Date.now(); await getTimeline(ctx).waitForNextTick(); const tickElapsed = Date.now() - waitStart; diff --git a/packages/@webreel/core/src/chrome.ts b/packages/@webreel/core/src/chrome.ts index 4f8226a..7e9a020 100644 --- a/packages/@webreel/core/src/chrome.ts +++ b/packages/@webreel/core/src/chrome.ts @@ -187,7 +187,7 @@ async function findFreePort(): Promise { export interface ChromeInstance { process: ChildProcess; port: number; - kill: () => void; + kill: () => Promise; } const MAX_LAUNCH_ATTEMPTS = 3; @@ -267,14 +267,31 @@ export async function launchChrome( }); }); + const cleanup = () => { + try { + proc.kill("SIGKILL"); + } catch {} + try { + rmSync(userDataDir, { recursive: true, force: true }); + } catch {} + }; + process.on("exit", cleanup); + return { process: proc, port, - kill: () => { + kill: async () => { + process.off("exit", cleanup); proc.kill("SIGTERM"); - setTimeout(() => { - rmSync(userDataDir, { recursive: true, force: true }); - }, 500); + const exited = await Promise.race([ + new Promise((resolve) => proc.on("exit", () => resolve(true))), + new Promise((resolve) => setTimeout(() => resolve(false), 3000)), + ]); + if (!exited) { + proc.kill("SIGKILL"); + await new Promise((resolve) => proc.on("exit", () => resolve())); + } + rmSync(userDataDir, { recursive: true, force: true }); }, }; } catch (err) { diff --git a/packages/@webreel/core/src/recorder.ts b/packages/@webreel/core/src/recorder.ts index 94719b5..0a56ef3 100644 --- a/packages/@webreel/core/src/recorder.ts +++ b/packages/@webreel/core/src/recorder.ts @@ -174,6 +174,18 @@ export class Recorder { ); if (!evalResult) break; } + try { + await this.raceStop( + client.Runtime.evaluate({ + expression: + "new Promise(r => requestAnimationFrame(() => requestAnimationFrame(r)))", + awaitPromise: true, + }), + ); + } catch { + // Page may be navigating -- skip sync, screenshot will retry next loop + } + const screenshotResult = await this.raceStop( client.Page.captureScreenshot({ format: "jpeg", diff --git a/packages/webreel/src/commands/record.ts b/packages/webreel/src/commands/record.ts index d3a2ec7..b6033a8 100644 --- a/packages/webreel/src/commands/record.ts +++ b/packages/webreel/src/commands/record.ts @@ -100,8 +100,24 @@ export const recordCommand = new Command("record") return; } - for (const video of videos) { - await runVideo(video, { record: true, verbose, configDir, frames: opts.frames }); + const onSignal = () => { + console.log("\nInterrupted. Cleaning up..."); + process.exit(130); + }; + process.on("SIGINT", onSignal); + process.on("SIGTERM", onSignal); + try { + for (const video of videos) { + await runVideo(video, { + record: true, + verbose, + configDir, + frames: opts.frames, + }); + } + } finally { + process.off("SIGINT", onSignal); + process.off("SIGTERM", onSignal); } if (opts.watch) { diff --git a/packages/webreel/src/lib/runner.ts b/packages/webreel/src/lib/runner.ts index 7dcc033..47d7810 100644 --- a/packages/webreel/src/lib/runner.ts +++ b/packages/webreel/src/lib/runner.ts @@ -465,7 +465,7 @@ export async function runVideo( } } try { - chrome.kill(); + await chrome.kill(); } catch (err) { console.warn("Failed to kill Chrome process:", err); } From b14d5c86d622bd430fe81e8b57e9afea578b395a Mon Sep 17 00:00:00 2001 From: Ariel Conti Date: Thu, 5 Mar 2026 12:23:38 +0000 Subject: [PATCH 03/48] fix(chrome): prevent kill() hang when process already exited Register exit listener before sending SIGKILL and check proc.exitCode to handle the race where Chrome exits between the SIGTERM timeout and the SIGKILL, which would cause the exit event to never fire. --- packages/@webreel/core/src/chrome.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/@webreel/core/src/chrome.ts b/packages/@webreel/core/src/chrome.ts index 7e9a020..eb33d06 100644 --- a/packages/@webreel/core/src/chrome.ts +++ b/packages/@webreel/core/src/chrome.ts @@ -288,8 +288,12 @@ export async function launchChrome( new Promise((resolve) => setTimeout(() => resolve(false), 3000)), ]); if (!exited) { + const exitPromise = new Promise((resolve) => { + if (proc.exitCode !== null) return resolve(); + proc.on("exit", () => resolve()); + }); proc.kill("SIGKILL"); - await new Promise((resolve) => proc.on("exit", () => resolve())); + await exitPromise; } rmSync(userDataDir, { recursive: true, force: true }); }, From 2c35fe930dfbed7278643b6e736319da0caa5d7a Mon Sep 17 00:00:00 2001 From: Ariel Conti Date: Sat, 7 Mar 2026 00:17:37 +0000 Subject: [PATCH 04/48] fix: annotate chrome cleanup catches --- packages/@webreel/core/src/chrome.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/packages/@webreel/core/src/chrome.ts b/packages/@webreel/core/src/chrome.ts index eb33d06..bc1a2b2 100644 --- a/packages/@webreel/core/src/chrome.ts +++ b/packages/@webreel/core/src/chrome.ts @@ -270,10 +270,14 @@ export async function launchChrome( const cleanup = () => { try { proc.kill("SIGKILL"); - } catch {} + } catch { + // Ignore cleanup errors during process exit. + } try { rmSync(userDataDir, { recursive: true, force: true }); - } catch {} + } catch { + // Ignore cleanup errors during process exit. + } }; process.on("exit", cleanup); From 54db72f045217b0b2c64acc9bad882102dec2a1c Mon Sep 17 00:00:00 2001 From: Ariel Conti Date: Fri, 3 Jul 2026 19:12:06 +0100 Subject: [PATCH 05/48] fix: address review findings on typing, process cleanup, and signals Typing (actions.ts): - iterate text by code point so surrogate pairs (emoji) are not split - batch insertText into a single call when charDelay is 0 - no-op on empty text; extract dispatchCharKeyEvents helper - new optional "method" field on type steps (insertText or dispatchKeyEvent) with config validation, JSON schema, and docs; defaults to insertText when a selector is set Process lifecycle (new process.ts): - hasExited checks signalCode as well as exitCode, so signal-killed processes are detected - killProcess returns immediately for already-exited processes, clears its escalation timer, and awaits the same exit promise after SIGKILL; chrome.kill no longer stalls 3s or holds the event loop open Capture loop (recorder.ts): - remove the double requestAnimationFrame sync: it halved capture throughput, provided no measurable freshness benefit, and its Runtime.evaluate can hang indefinitely across navigations - release timeline tick waiters in a finally so a crashed loop cannot strand typeText pacing - memoize stop() so the runner finally block and interrupt cleanup share one shutdown; guard start() against a stop() that arrives while ensureFfmpeg is in flight Signal handling (new signals.ts): - shared interrupt-cleanup registry; runner registers recorder stop, temp video removal, client close, and chrome kill - record (watch and non-watch) and preview install the same handlers; SIGTERM now handled everywhere with exit codes 130/143 - watch mode waits for an in-flight recording without being cut off by the 10s force-exit timer, which now only bounds cleanup Verified end to end: SIGINT and SIGTERM mid-recording leave no orphaned Chrome or ffmpeg processes, temp videos, or user-data dirs; watch-mode SIGINT during a 15s recording finishes the video and exits 0. --- README.md | 28 ++-- apps/docs/public/schema/v1.json | 5 + apps/docs/src/app/actions/page.mdx | 14 +- .../core/src/__tests__/actions.test.ts | 41 +++++- .../core/src/__tests__/process.test.ts | 66 +++++++++ .../core/src/__tests__/recorder.test.ts | 102 ++++++++++++++ .../core/src/__tests__/timeline.test.ts | 25 ++++ packages/@webreel/core/src/actions.ts | 85 +++++++----- packages/@webreel/core/src/chrome.ts | 15 +- packages/@webreel/core/src/process.ts | 48 +++++++ packages/@webreel/core/src/recorder.ts | 131 ++++++++++-------- packages/@webreel/core/src/timeline.ts | 11 ++ packages/webreel/README.md | 28 ++-- packages/webreel/src/commands/preview.ts | 8 +- packages/webreel/src/commands/record.ts | 27 ++-- .../webreel/src/lib/__tests__/config.test.ts | 14 ++ .../webreel/src/lib/__tests__/runner.test.ts | 18 +++ .../webreel/src/lib/__tests__/signals.test.ts | 71 ++++++++++ packages/webreel/src/lib/config.ts | 11 ++ packages/webreel/src/lib/runner.ts | 36 ++++- packages/webreel/src/lib/signals.ts | 81 +++++++++++ packages/webreel/src/lib/types.ts | 1 + skills/webreel/SKILL.md | 28 ++-- skills/webreel/steps-reference.md | 17 ++- 24 files changed, 733 insertions(+), 178 deletions(-) create mode 100644 packages/@webreel/core/src/__tests__/process.test.ts create mode 100644 packages/@webreel/core/src/__tests__/recorder.test.ts create mode 100644 packages/@webreel/core/src/process.ts create mode 100644 packages/webreel/src/lib/__tests__/signals.test.ts create mode 100644 packages/webreel/src/lib/signals.ts diff --git a/README.md b/README.md index 1e461a4..5c33942 100644 --- a/README.md +++ b/README.md @@ -165,20 +165,20 @@ webreel record --help ### Actions -| Action | Fields | Description | -| ------------ | ------------------------------------------------------ | ------------------------------------ | -| `pause` | `ms` | Wait for a duration | -| `click` | `text` or `selector`, optional `within`, `modifiers` | Move cursor to an element and click | -| `key` | `key` (e.g. `"cmd+z"`), optional `label` | Press a key or key combo | -| `type` | `text`, optional `target`, `charDelay` | Type text character by character | -| `scroll` | optional `x`, `y`, `selector` | Scroll the page or an element | -| `wait` | `selector` or `text`, optional `timeout` | Wait for an element to appear | -| `screenshot` | `output` | Capture a PNG screenshot | -| `drag` | `from` and `to` (each with `text`/`selector`/`within`) | Drag from one element to another | -| `moveTo` | `text` or `selector`, optional `within` | Move cursor to an element | -| `navigate` | `url` | Navigate to a new URL mid-video | -| `hover` | `text` or `selector`, optional `within` | Hover over an element (triggers CSS) | -| `select` | `selector`, `value` | Select a value in a dropdown | +| Action | Fields | Description | +| ------------ | ------------------------------------------------------------ | ------------------------------------ | +| `pause` | `ms` | Wait for a duration | +| `click` | `text` or `selector`, optional `within`, `modifiers` | Move cursor to an element and click | +| `key` | `key` (e.g. `"cmd+z"`), optional `label` | Press a key or key combo | +| `type` | `text`, optional `selector`, `within`, `charDelay`, `method` | Type text character by character | +| `scroll` | optional `x`, `y`, `selector` | Scroll the page or an element | +| `wait` | `selector` or `text`, optional `timeout` | Wait for an element to appear | +| `screenshot` | `output` | Capture a PNG screenshot | +| `drag` | `from` and `to` (each with `text`/`selector`/`within`) | Drag from one element to another | +| `moveTo` | `text` or `selector`, optional `within` | Move cursor to an element | +| `navigate` | `url` | Navigate to a new URL mid-video | +| `hover` | `text` or `selector`, optional `within` | Hover over an element (triggers CSS) | +| `select` | `selector`, `value` | Select a value in a dropdown | All steps (except `pause`) accept an optional `delay` field (ms to wait after the step). Use `defaultDelay` at the top-level or per-video to set a default. diff --git a/apps/docs/public/schema/v1.json b/apps/docs/public/schema/v1.json index 7b9a83e..8f57d03 100644 --- a/apps/docs/public/schema/v1.json +++ b/apps/docs/public/schema/v1.json @@ -521,6 +521,11 @@ "minimum": 0, "description": "Delay in milliseconds between keystrokes." }, + "method": { + "type": "string", + "enum": ["insertText", "dispatchKeyEvent"], + "description": "How characters are injected. 'insertText' goes through the browser text input pipeline and updates framework-controlled inputs (React and similar) but fires no keydown/keyup events. 'dispatchKeyEvent' fires raw key events. Defaults to 'insertText' when 'selector' is set, otherwise 'dispatchKeyEvent'." + }, "label": { "$ref": "#/$defs/label" }, diff --git a/apps/docs/src/app/actions/page.mdx b/apps/docs/src/app/actions/page.mdx index d25dc6d..60a53ce 100644 --- a/apps/docs/src/app/actions/page.mdx +++ b/apps/docs/src/app/actions/page.mdx @@ -152,6 +152,8 @@ Type a string of text character by character. Useful for filling form fields wit { "action": "type", "text": "search query", "selector": "#search" } ``` +When a `selector` is set, characters are injected with the browser's text input pipeline (`insertText`), which updates framework-controlled inputs (React and similar) but does not fire `keydown`/`keyup` events. Without a `selector`, raw key events are dispatched to the focused element. Set `method` to override the default for either case, for example to drive a search box that filters on `keydown`. + @@ -187,7 +189,17 @@ Type a string of text character by character. Useful for filling form fields wit charDelay - + + + + + +
numberMilliseconds between each keystroke (default: 80)Milliseconds between each keystroke (default: 120)
+ method + string + "insertText" or "dispatchKeyEvent". Overrides how + characters are injected (default depends on selector) +
diff --git a/packages/@webreel/core/src/__tests__/actions.test.ts b/packages/@webreel/core/src/__tests__/actions.test.ts index b8da5a8..9f2eb48 100644 --- a/packages/@webreel/core/src/__tests__/actions.test.ts +++ b/packages/@webreel/core/src/__tests__/actions.test.ts @@ -320,16 +320,42 @@ describe("typeText", () => { }; } - it("uses insertText when method is insertText", async () => { + it("batches the whole string into one insertText call when charDelay is 0", async () => { const ctx = new RecordingContext(); const client = createMockClient(); await typeText(ctx, client, "ab", 0, { method: "insertText" }); + expect(client.Input.insertText).toHaveBeenCalledTimes(1); + expect(client.Input.insertText).toHaveBeenCalledWith({ text: "ab" }); + expect(client.Input.dispatchKeyEvent).not.toHaveBeenCalled(); + }); + + it("uses one insertText call per character when charDelay is set", async () => { + const ctx = new RecordingContext(); + const client = createMockClient(); + await typeText(ctx, client, "ab", 1, { method: "insertText" }); expect(client.Input.insertText).toHaveBeenCalledTimes(2); expect(client.Input.insertText).toHaveBeenNthCalledWith(1, { text: "a" }); expect(client.Input.insertText).toHaveBeenNthCalledWith(2, { text: "b" }); expect(client.Input.dispatchKeyEvent).not.toHaveBeenCalled(); }); + it("keeps surrogate pairs intact when typing per character", async () => { + const ctx = new RecordingContext(); + const client = createMockClient(); + const emoji = "\u{1F600}"; + await typeText(ctx, client, `a${emoji}b`, 1, { method: "insertText" }); + expect(client.Input.insertText).toHaveBeenCalledTimes(3); + expect(client.Input.insertText).toHaveBeenNthCalledWith(2, { text: emoji }); + }); + + it("does nothing for empty text", async () => { + const ctx = new RecordingContext(); + const client = createMockClient(); + await typeText(ctx, client, "", 0, { method: "insertText" }); + expect(client.Input.insertText).not.toHaveBeenCalled(); + expect(client.Input.dispatchKeyEvent).not.toHaveBeenCalled(); + }); + it("uses dispatchKeyEvent when method is dispatchKeyEvent", async () => { const ctx = new RecordingContext(); const client = createMockClient(); @@ -339,6 +365,19 @@ describe("typeText", () => { expect(client.Input.insertText).not.toHaveBeenCalled(); }); + it("sends one key event triple per code point, not per code unit", async () => { + const ctx = new RecordingContext(); + const client = createMockClient(); + const emoji = "\u{1F600}"; + await typeText(ctx, client, emoji, 0, { method: "dispatchKeyEvent" }); + expect(client.Input.dispatchKeyEvent).toHaveBeenCalledTimes(3); + expect(client.Input.dispatchKeyEvent).toHaveBeenNthCalledWith(2, { + type: "char", + key: emoji, + text: emoji, + }); + }); + it("defaults to dispatchKeyEvent when no method specified", async () => { const ctx = new RecordingContext(); const client = createMockClient(); diff --git a/packages/@webreel/core/src/__tests__/process.test.ts b/packages/@webreel/core/src/__tests__/process.test.ts new file mode 100644 index 0000000..bee1de7 --- /dev/null +++ b/packages/@webreel/core/src/__tests__/process.test.ts @@ -0,0 +1,66 @@ +import { describe, it, expect } from "vitest"; +import { spawn } from "node:child_process"; +import { hasExited, killProcess } from "../process.js"; + +function spawnLongRunning(setup = "") { + return spawn(process.execPath, ["-e", `${setup}setInterval(() => {}, 1000);`]); +} + +function waitForExit(proc: ReturnType): Promise { + return new Promise((resolve) => { + if (hasExited(proc)) return resolve(); + proc.once("exit", () => resolve()); + }); +} + +describe("hasExited", () => { + it("is false for a running process and true after exit", async () => { + const proc = spawnLongRunning(); + expect(hasExited(proc)).toBe(false); + proc.kill("SIGKILL"); + await waitForExit(proc); + expect(hasExited(proc)).toBe(true); + }); + + it("detects a signal-killed process even though exitCode is null", async () => { + const proc = spawnLongRunning(); + proc.kill("SIGKILL"); + await waitForExit(proc); + expect(proc.exitCode).toBeNull(); + expect(proc.signalCode).toBe("SIGKILL"); + expect(hasExited(proc)).toBe(true); + }); +}); + +describe("killProcess", () => { + it("terminates a running process", async () => { + const proc = spawnLongRunning(); + await killProcess(proc, 3000); + expect(hasExited(proc)).toBe(true); + }); + + it("returns immediately for an already-exited process", async () => { + const proc = spawnLongRunning(); + proc.kill("SIGKILL"); + await waitForExit(proc); + + const start = Date.now(); + await killProcess(proc, 3000); + expect(Date.now() - start).toBeLessThan(500); + }); + + it("escalates to SIGKILL when the process ignores the signal", async () => { + const proc = spawnLongRunning('process.on("SIGTERM", () => {});'); + // Give the child a moment to install its SIGTERM handler. + await new Promise((r) => setTimeout(r, 300)); + await killProcess(proc, 200); + expect(hasExited(proc)).toBe(true); + expect(proc.signalCode).toBe("SIGKILL"); + }); + + it("is safe to call concurrently", async () => { + const proc = spawnLongRunning(); + await Promise.all([killProcess(proc, 3000), killProcess(proc, 3000)]); + expect(hasExited(proc)).toBe(true); + }); +}); diff --git a/packages/@webreel/core/src/__tests__/recorder.test.ts b/packages/@webreel/core/src/__tests__/recorder.test.ts new file mode 100644 index 0000000..d1c59f9 --- /dev/null +++ b/packages/@webreel/core/src/__tests__/recorder.test.ts @@ -0,0 +1,102 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; +import { chmodSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { Recorder } from "../recorder.js"; +import { InteractionTimeline } from "../timeline.js"; +import type { CDPClient } from "../types.js"; + +let ffmpegPathResolver: () => Promise; +vi.mock("../ffmpeg.js", () => ({ + ensureFfmpeg: () => ffmpegPathResolver(), +})); + +// A 1x1 white JPEG, enough for the capture loop to treat as a frame. +const TINY_JPEG = + "/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAMCAgICAgMCAgIDAwMDBAYEBAQEBAgGBgUGCQgKCgkICQkKDA8MCgsOCwkJDRENDg8QEBEQCgwSExIQEw8QEBD/yQALCAABAAEBAREA/8wABgAQEAX/2gAIAQEAAD8A0s8g/9k="; + +let shimDir: string; +let shimPath: string; + +beforeEach(() => { + shimDir = mkdtempSync(join(tmpdir(), "webreel-recorder-test-")); + shimPath = join(shimDir, "fake-ffmpeg"); + // Drains stdin and exits on EOF, standing in for ffmpeg. + writeFileSync(shimPath, "#!/bin/sh\ncat > /dev/null\n"); + chmodSync(shimPath, 0o755); + ffmpegPathResolver = () => Promise.resolve(shimPath); +}); + +afterEach(() => { + rmSync(shimDir, { recursive: true, force: true }); +}); + +function fakeClient(): CDPClient { + return { + Page: { + captureScreenshot: async () => { + // Pace the capture loop so it does not spin hot against the shim. + await new Promise((r) => setTimeout(r, 5)); + return { data: TINY_JPEG }; + }, + }, + Runtime: { + evaluate: async () => ({ result: {} }), + }, + } as unknown as CDPClient; +} + +async function startedRecorder(): Promise { + const recorder = new Recorder(64, 64); + recorder.setTimeline(new InteractionTimeline(64, 64)); + await recorder.start(fakeClient(), join(shimDir, "out.mp4")); + await new Promise((r) => setTimeout(r, 30)); + return recorder; +} + +describe("Recorder.stop", () => { + it("memoizes shutdown so concurrent callers share one stop", async () => { + const recorder = await startedRecorder(); + + const first = recorder.stop(); + const second = recorder.stop(); + expect(second).toBe(first); + await Promise.all([first, second]); + + // Stop after completion still returns the settled shutdown. + expect(recorder.stop()).toBe(first); + }); + + it("start resets the memoized stop for reuse", async () => { + const recorder = await startedRecorder(); + const firstStop = recorder.stop(); + await firstStop; + + await recorder.start(fakeClient(), join(shimDir, "out2.mp4")); + await new Promise((r) => setTimeout(r, 20)); + const secondStop = recorder.stop(); + expect(secondStop).not.toBe(firstStop); + await secondStop; + }); + + it("aborts start when stop arrives while ensureFfmpeg is in flight", async () => { + let releaseFfmpeg!: (path: string) => void; + ffmpegPathResolver = () => + new Promise((resolve) => { + releaseFfmpeg = resolve; + }); + + const recorder = new Recorder(64, 64); + recorder.setTimeline(new InteractionTimeline(64, 64)); + const starting = recorder.start(fakeClient(), join(shimDir, "out.mp4")); + await new Promise((r) => setTimeout(r, 10)); + + // Interrupt-style stop while start is suspended on the download. + await recorder.stop(); + releaseFfmpeg(shimPath); + await starting; + + // Start must not have revived the recorder or spawned ffmpeg. + expect(recorder.getTempVideoPath()).toBe(""); + }); +}); diff --git a/packages/@webreel/core/src/__tests__/timeline.test.ts b/packages/@webreel/core/src/__tests__/timeline.test.ts index cef6245..f019ae1 100644 --- a/packages/@webreel/core/src/__tests__/timeline.test.ts +++ b/packages/@webreel/core/src/__tests__/timeline.test.ts @@ -95,4 +95,29 @@ describe("InteractionTimeline", () => { expect(data.frames[1].cursor.y).toBe(40); expect(data.frames[2].cursor.x).toBe(30); }); + + it("waitForNextTick resolves on the next tick", async () => { + const tl = new InteractionTimeline(1080, 1080); + let resolved = false; + const wait = tl.waitForNextTick().then(() => { + resolved = true; + }); + expect(resolved).toBe(false); + tl.tick(); + await wait; + expect(resolved).toBe(true); + }); + + it("releaseWaiters resolves pending waiters", async () => { + const tl = new InteractionTimeline(1080, 1080); + const wait = tl.waitForNextTick(); + tl.releaseWaiters(); + await expect(wait).resolves.toBeUndefined(); + }); + + it("waitForNextTick resolves immediately after release", async () => { + const tl = new InteractionTimeline(1080, 1080); + tl.releaseWaiters(); + await expect(tl.waitForNextTick()).resolves.toBeUndefined(); + }); }); diff --git a/packages/@webreel/core/src/actions.ts b/packages/@webreel/core/src/actions.ts index b076bcd..cb66b94 100644 --- a/packages/@webreel/core/src/actions.ts +++ b/packages/@webreel/core/src/actions.ts @@ -618,6 +618,43 @@ function humanDelay(base: number): number { return jitter; } +async function dispatchCharKeyEvents(client: CDPClient, char: string): Promise { + const charInfo = CHAR_CODES[char]; + const isLetter = /^[a-zA-Z]$/.test(char); + + let code: string; + let keyCode: number; + + if (charInfo) { + code = charInfo.code; + keyCode = charInfo.keyCode; + } else if (isLetter) { + code = `Key${char.toUpperCase()}`; + keyCode = char.toUpperCase().charCodeAt(0); + } else { + code = ""; + keyCode = 0; + } + + await client.Input.dispatchKeyEvent({ + type: "rawKeyDown", + key: char, + code, + windowsVirtualKeyCode: keyCode, + }); + await client.Input.dispatchKeyEvent({ + type: "char", + key: char, + text: char, + }); + await client.Input.dispatchKeyEvent({ + type: "keyUp", + key: char, + code, + windowsVirtualKeyCode: keyCode, + }); +} + export async function typeText( ctx: RecordingContext, client: CDPClient, @@ -625,48 +662,24 @@ export async function typeText( delayMs = 120, options?: { method?: "dispatchKeyEvent" | "insertText" }, ): Promise { + if (text.length === 0) return; const useInsertText = options?.method === "insertText"; - for (let i = 0; i < text.length; i++) { - const char = text[i]; + // insertText accepts whole strings; with no per-character pacing a single + // call replaces one CDP round-trip per character. + if (useInsertText && delayMs <= 0) { + await client.Input.insertText({ text }); + ctx.markEvent("key"); + return; + } + // Iterate by code point so surrogate pairs (emoji, supplementary CJK) + // are never split across two events. + for (const char of text) { if (useInsertText) { await client.Input.insertText({ text: char }); } else { - const charInfo = CHAR_CODES[char]; - const isLetter = /^[a-zA-Z]$/.test(char); - - let code: string; - let keyCode: number; - - if (charInfo) { - code = charInfo.code; - keyCode = charInfo.keyCode; - } else if (isLetter) { - code = `Key${char.toUpperCase()}`; - keyCode = char.toUpperCase().charCodeAt(0); - } else { - code = ""; - keyCode = 0; - } - - await client.Input.dispatchKeyEvent({ - type: "rawKeyDown", - key: char, - code, - windowsVirtualKeyCode: keyCode, - }); - await client.Input.dispatchKeyEvent({ - type: "char", - key: char, - text: char, - }); - await client.Input.dispatchKeyEvent({ - type: "keyUp", - key: char, - code, - windowsVirtualKeyCode: keyCode, - }); + await dispatchCharKeyEvents(client, char); } ctx.markEvent("key"); diff --git a/packages/@webreel/core/src/chrome.ts b/packages/@webreel/core/src/chrome.ts index bc1a2b2..a983411 100644 --- a/packages/@webreel/core/src/chrome.ts +++ b/packages/@webreel/core/src/chrome.ts @@ -4,6 +4,7 @@ import { createServer } from "node:net"; import { homedir, tmpdir } from "node:os"; import { resolve, join } from "node:path"; import { fetchJson, downloadAndExtract } from "./download.js"; +import { killProcess } from "./process.js"; export const CHROME_CACHE_DIR = resolve(homedir(), ".webreel", "bin", "chrome"); export const HEADLESS_SHELL_CACHE_DIR = resolve( @@ -286,19 +287,7 @@ export async function launchChrome( port, kill: async () => { process.off("exit", cleanup); - proc.kill("SIGTERM"); - const exited = await Promise.race([ - new Promise((resolve) => proc.on("exit", () => resolve(true))), - new Promise((resolve) => setTimeout(() => resolve(false), 3000)), - ]); - if (!exited) { - const exitPromise = new Promise((resolve) => { - if (proc.exitCode !== null) return resolve(); - proc.on("exit", () => resolve()); - }); - proc.kill("SIGKILL"); - await exitPromise; - } + await killProcess(proc); rmSync(userDataDir, { recursive: true, force: true }); }, }; diff --git a/packages/@webreel/core/src/process.ts b/packages/@webreel/core/src/process.ts new file mode 100644 index 0000000..ec5fe87 --- /dev/null +++ b/packages/@webreel/core/src/process.ts @@ -0,0 +1,48 @@ +import type { ChildProcess } from "node:child_process"; + +// exitCode stays null when a process dies from a signal; check both so an +// already-dead process is never mistaken for a live one. +export function hasExited(proc: ChildProcess): boolean { + return proc.exitCode !== null || proc.signalCode !== null; +} + +/** + * Send a signal and wait for the process to exit, escalating to SIGKILL + * after timeoutMs. Resolves once the process is gone. Safe to call on a + * process that already exited and safe to call concurrently. + */ +export async function killProcess( + proc: ChildProcess, + timeoutMs = 3000, + signal: NodeJS.Signals = "SIGTERM", +): Promise { + if (hasExited(proc)) return; + + const exited = new Promise((resolve) => { + proc.once("exit", () => resolve()); + }); + + try { + proc.kill(signal); + } catch { + // Process disappeared between the liveness check and the signal. + return; + } + + let timer: NodeJS.Timeout | undefined; + const timedOut = new Promise((resolve) => { + timer = setTimeout(() => resolve(true), timeoutMs); + }); + const needsForceKill = await Promise.race([exited.then(() => false), timedOut]); + clearTimeout(timer); + + if (needsForceKill) { + if (hasExited(proc)) return; + try { + proc.kill("SIGKILL"); + } catch { + return; + } + await exited; + } +} diff --git a/packages/@webreel/core/src/recorder.ts b/packages/@webreel/core/src/recorder.ts index 0a56ef3..628407f 100644 --- a/packages/@webreel/core/src/recorder.ts +++ b/packages/@webreel/core/src/recorder.ts @@ -6,6 +6,7 @@ import { TARGET_FPS, DEFAULT_VIEWPORT_SIZE } from "./types.js"; import type { CDPClient, SoundEvent } from "./types.js"; import type { RecordingContext } from "./actions.js"; import { ensureFfmpeg } from "./ffmpeg.js"; +import { hasExited } from "./process.js"; import { finalizeMp4, finalizeWebm, finalizeGif, type SfxConfig } from "./media.js"; import type { InteractionTimeline, TimelineData } from "./timeline.js"; @@ -31,6 +32,7 @@ export class Recorder { private framesDir: string | null = null; private stopResolve: (() => void) | null = null; private stoppedPromise: Promise | null = null; + private stopOnce: Promise | null = null; constructor( outputWidth = DEFAULT_VIEWPORT_SIZE, @@ -69,7 +71,14 @@ export class Recorder { } async start(client: CDPClient, outputPath: string, ctx?: RecordingContext) { + // Reset before the first await so a stop() arriving while ensureFfmpeg + // is in flight (e.g. interrupt during first-run download) is visible + // below instead of being wiped out when we resume. + this.stopOnce = null; this.ffmpegPath = await ensureFfmpeg(); + if (this.stopOnce) { + return; + } this.outputPath = outputPath; this.frameCount = 0; this.droppedFrames = 0; @@ -162,73 +171,68 @@ export class Recorder { let lastFrameTime = Date.now(); let consecutiveErrors = 0; - while (this.running) { - try { - if (this.timeline) { - this.timeline.tick(); - } else { - const evalResult = await this.raceStop( - client.Runtime.evaluate({ - expression: "window.__tickCursor&&window.__tickCursor()", - }), - ); - if (!evalResult) break; - } + try { + while (this.running) { try { - await this.raceStop( - client.Runtime.evaluate({ - expression: - "new Promise(r => requestAnimationFrame(() => requestAnimationFrame(r)))", - awaitPromise: true, + if (this.timeline) { + this.timeline.tick(); + } else { + const evalResult = await this.raceStop( + client.Runtime.evaluate({ + expression: "window.__tickCursor&&window.__tickCursor()", + }), + ); + if (!evalResult) break; + } + + const screenshotResult = await this.raceStop( + client.Page.captureScreenshot({ + format: "jpeg", + quality: 60, + optimizeForSpeed: true, }), ); - } catch { - // Page may be navigating -- skip sync, screenshot will retry next loop - } - - const screenshotResult = await this.raceStop( - client.Page.captureScreenshot({ - format: "jpeg", - quality: 60, - optimizeForSpeed: true, - }), - ); - if (!screenshotResult) break; - - const buffer = Buffer.from(screenshotResult.data, "base64"); - const now = Date.now(); - const elapsed = now - lastFrameTime; - const frameSlots = Math.min(3, Math.max(1, Math.round(elapsed / this.frameMs))); - - if (frameSlots > 1) { - for (let i = 0; i < frameSlots - 1; i++) { - if (this.timeline) this.timeline.tickDuplicate(); - await this.writeFrame(buffer); - this.frameCount++; + if (!screenshotResult) break; + + const buffer = Buffer.from(screenshotResult.data, "base64"); + const now = Date.now(); + const elapsed = now - lastFrameTime; + const frameSlots = Math.min(3, Math.max(1, Math.round(elapsed / this.frameMs))); + + if (frameSlots > 1) { + for (let i = 0; i < frameSlots - 1; i++) { + if (this.timeline) this.timeline.tickDuplicate(); + await this.writeFrame(buffer); + this.frameCount++; + } } - } - await this.writeFrame(buffer); - this.frameCount++; + await this.writeFrame(buffer); + this.frameCount++; - if (this.framesDir) { - const padded = String(this.frameCount).padStart(5, "0"); - writeFileSync(resolve(this.framesDir, `frame-${padded}.jpg`), buffer); - } + if (this.framesDir) { + const padded = String(this.frameCount).padStart(5, "0"); + writeFileSync(resolve(this.framesDir, `frame-${padded}.jpg`), buffer); + } - lastFrameTime = now; - consecutiveErrors = 0; - } catch (err) { - if (!this.running) break; - consecutiveErrors++; - if (consecutiveErrors >= 10) { - console.error( - `Recording aborted after ${consecutiveErrors} consecutive capture failures:`, - err, - ); - break; + lastFrameTime = now; + consecutiveErrors = 0; + } catch (err) { + if (!this.running) break; + consecutiveErrors++; + if (consecutiveErrors >= 10) { + console.error( + `Recording aborted after ${consecutiveErrors} consecutive capture failures:`, + err, + ); + break; + } } } + } finally { + // However the loop exits (stop, error abort, crash), unblock anyone + // waiting on the next timeline tick. + this.timeline?.releaseWaiters(); } } @@ -236,7 +240,14 @@ export class Recorder { return this.tempVideo; } - async stop() { + // stop() can race between the normal finally path and interrupt cleanup; + // memoize so both callers await the same shutdown. + stop(): Promise { + this.stopOnce ??= this.doStop(); + return this.stopOnce; + } + + private async doStop() { this.running = false; if (this.ctx) this.ctx.setRecorder(null); @@ -266,7 +277,7 @@ export class Recorder { } }, FFMPEG_CLOSE_TIMEOUT_MS); await new Promise((res) => { - if (proc.exitCode !== null) { + if (hasExited(proc)) { res(); return; } diff --git a/packages/@webreel/core/src/timeline.ts b/packages/@webreel/core/src/timeline.ts index 05c392e..3c37437 100644 --- a/packages/@webreel/core/src/timeline.ts +++ b/packages/@webreel/core/src/timeline.ts @@ -59,6 +59,7 @@ export class InteractionTimeline { private events: SoundEvent[] = []; private frameCount = 0; private tickResolvers: Array<() => void> = []; + private released = false; private width: number; private height: number; @@ -138,11 +139,21 @@ export class InteractionTimeline { } waitForNextTick(): Promise { + if (this.released) return Promise.resolve(); return new Promise((resolve) => { this.tickResolvers.push(resolve); }); } + // Once the capture loop stops ticking, pending and future waiters must + // resolve immediately or callers like typeText would hang forever. + releaseWaiters(): void { + this.released = true; + const resolvers = this.tickResolvers; + this.tickResolvers = []; + for (const resolve of resolvers) resolve(); + } + tick(): void { if (this.cursorPath && this.pathIndex < this.cursorPath.length) { const p = this.cursorPath[this.pathIndex++]; diff --git a/packages/webreel/README.md b/packages/webreel/README.md index 68e5df0..9a69830 100644 --- a/packages/webreel/README.md +++ b/packages/webreel/README.md @@ -180,20 +180,20 @@ Raw video and timeline data are saved in `.webreel/raw/` and `.webreel/timelines ### Actions -| Action | Fields | Description | -| ------------ | ------------------------------------------------------ | ------------------------------------- | -| `pause` | `ms` | Wait for a duration | -| `click` | `text` or `selector`, optional `within`, `modifiers` | Move cursor to an element and click | -| `key` | `key` (e.g. `"cmd+z"`), optional `label`, `target` | Press a key or key combo | -| `type` | `text`, optional `target`, `charDelay` | Type text character by character | -| `drag` | `from` and `to` (each with `text`/`selector`/`within`) | Drag from one element to another | -| `scroll` | optional `x`, `y`, `selector` | Scroll the page or a container | -| `wait` | `selector` or `text`, optional `timeout` | Wait for an element or text to appear | -| `moveTo` | `text` or `selector`, optional `within` | Move cursor to an element | -| `screenshot` | `output` | Save a PNG screenshot | -| `navigate` | `url` | Navigate to a new URL mid-video | -| `hover` | `text` or `selector`, optional `within` | Hover over an element (triggers CSS) | -| `select` | `selector`, `value` | Select a value in a dropdown | +| Action | Fields | Description | +| ------------ | ------------------------------------------------------------ | ------------------------------------- | +| `pause` | `ms` | Wait for a duration | +| `click` | `text` or `selector`, optional `within`, `modifiers` | Move cursor to an element and click | +| `key` | `key` (e.g. `"cmd+z"`), optional `label`, `target` | Press a key or key combo | +| `type` | `text`, optional `selector`, `within`, `charDelay`, `method` | Type text character by character | +| `drag` | `from` and `to` (each with `text`/`selector`/`within`) | Drag from one element to another | +| `scroll` | optional `x`, `y`, `selector` | Scroll the page or a container | +| `wait` | `selector` or `text`, optional `timeout` | Wait for an element or text to appear | +| `moveTo` | `text` or `selector`, optional `within` | Move cursor to an element | +| `screenshot` | `output` | Save a PNG screenshot | +| `navigate` | `url` | Navigate to a new URL mid-video | +| `hover` | `text` or `selector`, optional `within` | Hover over an element (triggers CSS) | +| `select` | `selector`, `value` | Select a value in a dropdown | All steps (except `pause`) accept an optional `delay` field (ms to wait after the step). Use `defaultDelay` at the top-level or per-video to set a default. diff --git a/packages/webreel/src/commands/preview.ts b/packages/webreel/src/commands/preview.ts index 9fea74b..b1bb0ba 100644 --- a/packages/webreel/src/commands/preview.ts +++ b/packages/webreel/src/commands/preview.ts @@ -1,6 +1,7 @@ import { Command } from "commander"; import { loadWebreelConfig, resolveConfigPath, getConfigDir } from "../lib/config.js"; import { runVideo } from "../lib/runner.js"; +import { installSignalHandlers } from "../lib/signals.js"; export const previewCommand = new Command("preview") .description("Run a video in a visible browser without recording") @@ -34,6 +35,11 @@ export const previewCommand = new Command("preview") } console.log(`\nPreviewing: ${video.name}`); - await runVideo(video, { record: false, verbose, configDir }); + const uninstallSignalHandlers = installSignalHandlers(); + try { + await runVideo(video, { record: false, verbose, configDir }); + } finally { + uninstallSignalHandlers(); + } }, ); diff --git a/packages/webreel/src/commands/record.ts b/packages/webreel/src/commands/record.ts index b6033a8..a8cc741 100644 --- a/packages/webreel/src/commands/record.ts +++ b/packages/webreel/src/commands/record.ts @@ -8,6 +8,7 @@ import { filterVideosByName, } from "../lib/config.js"; import { runVideo } from "../lib/runner.js"; +import { installSignalHandlers } from "../lib/signals.js"; import type { WebreelConfig } from "../lib/types.js"; export function collectIncludePaths(config: WebreelConfig, configPath: string): string[] { @@ -100,12 +101,7 @@ export const recordCommand = new Command("record") return; } - const onSignal = () => { - console.log("\nInterrupted. Cleaning up..."); - process.exit(130); - }; - process.on("SIGINT", onSignal); - process.on("SIGTERM", onSignal); + const uninstallSignalHandlers = installSignalHandlers(); try { for (const video of videos) { await runVideo(video, { @@ -116,8 +112,7 @@ export const recordCommand = new Command("record") }); } } finally { - process.off("SIGINT", onSignal); - process.off("SIGTERM", onSignal); + uninstallSignalHandlers(); } if (opts.watch) { @@ -172,13 +167,15 @@ export const recordCommand = new Command("record") setupWatchers(webreelConfig); - process.on("SIGINT", async () => { - closeAllWatchers(); - if (recordingInProgress) { - console.log("\nWaiting for current recording to finish..."); - await recordingInProgress; - } - process.exit(0); + installSignalHandlers({ + beforeExit: async () => { + closeAllWatchers(); + if (recordingInProgress) { + console.log("Waiting for current recording to finish..."); + await recordingInProgress; + } + return 0; + }, }); } }, diff --git a/packages/webreel/src/lib/__tests__/config.test.ts b/packages/webreel/src/lib/__tests__/config.test.ts index ae99a10..0a648a8 100644 --- a/packages/webreel/src/lib/__tests__/config.test.ts +++ b/packages/webreel/src/lib/__tests__/config.test.ts @@ -143,6 +143,20 @@ describe("validateStep", () => { ); }); + it("validates type rejects unknown method", () => { + const errors = validate({ action: "type", text: "hi", method: "paste" }); + expect(errors).toContainEqual( + expect.objectContaining({ path: "videos.x.steps[0].method" }), + ); + }); + + it("validates type accepts insertText and dispatchKeyEvent methods", () => { + for (const method of ["insertText", "dispatchKeyEvent"]) { + const errors = validate({ action: "type", text: "hi", method }); + expect(errors).toEqual([]); + } + }); + it("validates scroll x must be a number", () => { const errors = validate({ action: "scroll", x: "bad" }); expect(errors).toContainEqual( diff --git a/packages/webreel/src/lib/__tests__/runner.test.ts b/packages/webreel/src/lib/__tests__/runner.test.ts index 3c76b33..be8f0fb 100644 --- a/packages/webreel/src/lib/__tests__/runner.test.ts +++ b/packages/webreel/src/lib/__tests__/runner.test.ts @@ -4,6 +4,7 @@ import { pathToFileURL } from "node:url"; import { formatStep, resolveKeyTarget, + resolveTypeMethod, resolveUrl, randomPointInBox, resolveTarget, @@ -110,6 +111,23 @@ describe("resolveKeyTarget", () => { }); }); +describe("resolveTypeMethod", () => { + it("returns explicit method unchanged", () => { + expect(resolveTypeMethod({ method: "dispatchKeyEvent", selector: "#input" })).toBe( + "dispatchKeyEvent", + ); + expect(resolveTypeMethod({ method: "insertText" })).toBe("insertText"); + }); + + it("defaults to insertText when a selector is set", () => { + expect(resolveTypeMethod({ selector: "#input" })).toBe("insertText"); + }); + + it("defaults to dispatchKeyEvent without a selector", () => { + expect(resolveTypeMethod({})).toBe("dispatchKeyEvent"); + }); +}); + describe("resolveUrl", () => { const configDir = "/home/user/project"; diff --git a/packages/webreel/src/lib/__tests__/signals.test.ts b/packages/webreel/src/lib/__tests__/signals.test.ts new file mode 100644 index 0000000..36c432b --- /dev/null +++ b/packages/webreel/src/lib/__tests__/signals.test.ts @@ -0,0 +1,71 @@ +import { describe, it, expect, vi } from "vitest"; +import { + registerInterruptCleanup, + runInterruptCleanups, + installSignalHandlers, +} from "../signals.js"; + +describe("interrupt cleanups", () => { + it("runs registered cleanups in reverse registration order", async () => { + const order: string[] = []; + registerInterruptCleanup(() => { + order.push("first"); + }); + registerInterruptCleanup(() => { + order.push("second"); + }); + await runInterruptCleanups(); + expect(order).toEqual(["second", "first"]); + }); + + it("does not run unregistered cleanups", async () => { + const fn = vi.fn(); + const unregister = registerInterruptCleanup(fn); + unregister(); + await runInterruptCleanups(); + expect(fn).not.toHaveBeenCalled(); + }); + + it("runs each cleanup only once across invocations", async () => { + const fn = vi.fn(); + registerInterruptCleanup(fn); + await runInterruptCleanups(); + await runInterruptCleanups(); + expect(fn).toHaveBeenCalledTimes(1); + }); + + it("continues past a failing cleanup", async () => { + const ran = vi.fn(); + registerInterruptCleanup(ran); + registerInterruptCleanup(() => { + throw new Error("boom"); + }); + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + await runInterruptCleanups(); + warn.mockRestore(); + expect(ran).toHaveBeenCalledTimes(1); + }); + + it("awaits async cleanups", async () => { + let done = false; + registerInterruptCleanup(async () => { + await new Promise((r) => setTimeout(r, 10)); + done = true; + }); + await runInterruptCleanups(); + expect(done).toBe(true); + }); +}); + +describe("installSignalHandlers", () => { + it("adds and removes SIGINT/SIGTERM listeners", () => { + const sigintBefore = process.listenerCount("SIGINT"); + const sigtermBefore = process.listenerCount("SIGTERM"); + const uninstall = installSignalHandlers(); + expect(process.listenerCount("SIGINT")).toBe(sigintBefore + 1); + expect(process.listenerCount("SIGTERM")).toBe(sigtermBefore + 1); + uninstall(); + expect(process.listenerCount("SIGINT")).toBe(sigintBefore); + expect(process.listenerCount("SIGTERM")).toBe(sigtermBefore); + }); +}); diff --git a/packages/webreel/src/lib/config.ts b/packages/webreel/src/lib/config.ts index c9fef3c..f42754c 100644 --- a/packages/webreel/src/lib/config.ts +++ b/packages/webreel/src/lib/config.ts @@ -352,6 +352,7 @@ const KNOWN_STEP_KEYS: Record> = { "selector", "within", "charDelay", + "method", "label", "delay", "description", @@ -558,6 +559,16 @@ function validateStep(step: unknown, index: number): ValidationError[] { message: "Must be a non-negative number", }); } + if ( + s.method !== undefined && + s.method !== "insertText" && + s.method !== "dispatchKeyEvent" + ) { + errors.push({ + path: `${prefix}.method`, + message: 'Must be "insertText" or "dispatchKeyEvent"', + }); + } break; case "scroll": diff --git a/packages/webreel/src/lib/runner.ts b/packages/webreel/src/lib/runner.ts index 47d7810..a68a9ea 100644 --- a/packages/webreel/src/lib/runner.ts +++ b/packages/webreel/src/lib/runner.ts @@ -1,5 +1,5 @@ import { resolve, dirname } from "node:path"; -import { readFileSync, writeFileSync, mkdirSync, renameSync } from "node:fs"; +import { readFileSync, writeFileSync, mkdirSync, renameSync, rmSync } from "node:fs"; import { pathToFileURL } from "node:url"; import { type CDPClient, @@ -29,6 +29,7 @@ import { DEFAULT_VIEWPORT_SIZE, } from "@webreel/core"; import type { VideoConfig, Step, ElementTarget } from "./types.js"; +import { registerInterruptCleanup } from "./signals.js"; export function formatStep(i: number, step: Step): string { const desc = "description" in step && step.description ? `: ${step.description}` : ""; @@ -69,6 +70,13 @@ export function resolveKeyTarget(target: string | ElementTarget): string { return target.selector ?? ""; } +export function resolveTypeMethod(step: { + method?: "insertText" | "dispatchKeyEvent"; + selector?: string; +}): "insertText" | "dispatchKeyEvent" { + return step.method ?? (step.selector ? "insertText" : "dispatchKeyEvent"); +} + export async function resolveTarget( client: CDPClient, opts: { text?: string; selector?: string; within?: string }, @@ -153,6 +161,29 @@ export async function runVideo( let clientRef: CDPClient | null = null; let recorder: Recorder | null = null; + // If the process is interrupted, release everything this run owns: + // flush and stop ffmpeg, discard the partial capture, close the CDP + // connection, and kill Chrome (which also removes its temp profile). + const unregisterCleanup = registerInterruptCleanup(async () => { + if (recorder) { + const tempVideo = recorder.getTempVideoPath(); + try { + await recorder.stop(); + } catch { + // Best-effort shutdown; Chrome teardown below still runs. + } + rmSync(tempVideo, { force: true }); + } + if (clientRef) { + try { + await clientRef.close(); + } catch { + // Connection may already be gone. + } + } + await chrome.kill(); + }); + try { const client = await connectCDP(chrome.port); clientRef = client; @@ -295,7 +326,7 @@ export async function runVideo( await pause(300 + Math.random() * 200); } await typeText(ctx, client, step.text, step.charDelay, { - method: step.selector ? "insertText" : "dispatchKeyEvent", + method: resolveTypeMethod(step), }); break; } @@ -450,6 +481,7 @@ export async function runVideo( console.log(`Preview complete: ${config.name}`); } } finally { + unregisterCleanup(); if (recorder) { try { await recorder.stop(); diff --git a/packages/webreel/src/lib/signals.ts b/packages/webreel/src/lib/signals.ts new file mode 100644 index 0000000..ba24a54 --- /dev/null +++ b/packages/webreel/src/lib/signals.ts @@ -0,0 +1,81 @@ +type CleanupFn = () => void | Promise; + +const cleanups = new Set(); + +/** + * Register a cleanup to run if the process is interrupted (SIGINT/SIGTERM). + * Returns an unregister function; call it once the resources are released + * through the normal path. + */ +export function registerInterruptCleanup(fn: CleanupFn): () => void { + cleanups.add(fn); + return () => cleanups.delete(fn); +} + +export async function runInterruptCleanups(): Promise { + // Most-recently-registered first so resources unwind in reverse + // acquisition order. + const pending = [...cleanups].reverse(); + cleanups.clear(); + for (const fn of pending) { + try { + await fn(); + } catch (err) { + console.warn("Cleanup failed during shutdown:", err); + } + } +} + +export interface SignalHandlerOptions { + /** + * Runs before registered cleanups. May return an exit code to use + * instead of the default 130 (SIGINT) / 143 (SIGTERM). + */ + beforeExit?: (signal: NodeJS.Signals) => number | void | Promise; +} + +const FORCE_EXIT_TIMEOUT_MS = 10_000; + +/** + * Install SIGINT/SIGTERM handlers that run registered cleanups before + * exiting. A second signal, or cleanup taking longer than 10 seconds, + * forces an immediate exit. The timeout does not cover beforeExit, which + * may legitimately wait on long work (watch mode waits for an in-flight + * recording); a second signal remains the escape hatch there. Returns an + * uninstall function. + */ +export function installSignalHandlers(options?: SignalHandlerOptions): () => void { + let shuttingDown = false; + + const onSignal = (signal: NodeJS.Signals) => { + const defaultCode = signal === "SIGTERM" ? 143 : 130; + if (shuttingDown) { + process.exit(defaultCode); + } + shuttingDown = true; + console.log("\nInterrupted. Cleaning up..."); + + void (async () => { + let code: number | void = undefined; + try { + code = await options?.beforeExit?.(signal); + } catch (err) { + console.warn("Shutdown hook failed:", err); + } + const forceExit = setTimeout( + () => process.exit(defaultCode), + FORCE_EXIT_TIMEOUT_MS, + ); + forceExit.unref(); + await runInterruptCleanups(); + process.exit(typeof code === "number" ? code : defaultCode); + })(); + }; + + process.on("SIGINT", onSignal); + process.on("SIGTERM", onSignal); + return () => { + process.off("SIGINT", onSignal); + process.off("SIGTERM", onSignal); + }; +} diff --git a/packages/webreel/src/lib/types.ts b/packages/webreel/src/lib/types.ts index 2aed93c..d3dfd40 100644 --- a/packages/webreel/src/lib/types.ts +++ b/packages/webreel/src/lib/types.ts @@ -46,6 +46,7 @@ export interface StepType { selector?: string; within?: string; charDelay?: number; + method?: "insertText" | "dispatchKeyEvent"; label?: string; delay?: number; description?: string; diff --git a/skills/webreel/SKILL.md b/skills/webreel/SKILL.md index e7d9006..49e8c37 100644 --- a/skills/webreel/SKILL.md +++ b/skills/webreel/SKILL.md @@ -183,20 +183,20 @@ Record specific videos by name: `webreel record hero login`. Each step has an `action` field. Most steps accept optional `label`, `delay` (ms after step), and `description` fields. -| Action | Key fields | Purpose | -| ------------ | ------------------------------------------- | ---------------------------------- | -| `pause` | `ms` | Wait for a duration | -| `click` | `text` or `selector`, `within`, `modifiers` | Click an element | -| `type` | `text`, `selector`, `within`, `charDelay` | Type text into an input | -| `key` | `key`, `target` | Press a key combo (e.g. `"cmd+s"`) | -| `drag` | `from`, `to` (element targets) | Drag between two elements | -| `scroll` | `x`, `y`, `selector` | Scroll the page or an element | -| `wait` | `selector` or `text`, `timeout` | Wait for an element to appear | -| `moveTo` | `text` or `selector`, `within` | Move cursor to an element | -| `navigate` | `url` | Navigate to a new URL | -| `hover` | `text` or `selector`, `within` | Hover over an element | -| `select` | `selector`, `value` | Select a dropdown value | -| `screenshot` | `output` | Capture a PNG screenshot | +| Action | Key fields | Purpose | +| ------------ | --------------------------------------------------- | ---------------------------------- | +| `pause` | `ms` | Wait for a duration | +| `click` | `text` or `selector`, `within`, `modifiers` | Click an element | +| `type` | `text`, `selector`, `within`, `charDelay`, `method` | Type text into an input | +| `key` | `key`, `target` | Press a key combo (e.g. `"cmd+s"`) | +| `drag` | `from`, `to` (element targets) | Drag between two elements | +| `scroll` | `x`, `y`, `selector` | Scroll the page or an element | +| `wait` | `selector` or `text`, `timeout` | Wait for an element to appear | +| `moveTo` | `text` or `selector`, `within` | Move cursor to an element | +| `navigate` | `url` | Navigate to a new URL | +| `hover` | `text` or `selector`, `within` | Hover over an element | +| `select` | `selector`, `value` | Select a dropdown value | +| `screenshot` | `output` | Capture a PNG screenshot | For full field details on every step type, see [steps-reference.md](steps-reference.md). diff --git a/skills/webreel/steps-reference.md b/skills/webreel/steps-reference.md index 382c679..ae8935e 100644 --- a/skills/webreel/steps-reference.md +++ b/skills/webreel/steps-reference.md @@ -56,13 +56,16 @@ Provide `text` or `selector` (at least one). Type text into an input element. If no target is specified, types into the currently focused element. -| Field | Type | Required | Description | -| ----------- | -------- | -------- | ----------------------------------- | -| `action` | `"type"` | yes | | -| `text` | string | yes | Text to type | -| `selector` | string | no | Target input by CSS selector | -| `within` | string | no | Scope search to a parent selector | -| `charDelay` | number | no | Milliseconds between each character | +| Field | Type | Required | Description | +| ----------- | -------- | -------- | -------------------------------------------------- | +| `action` | `"type"` | yes | | +| `text` | string | yes | Text to type | +| `selector` | string | no | Target input by CSS selector | +| `within` | string | no | Scope search to a parent selector | +| `charDelay` | number | no | Milliseconds between each character | +| `method` | string | no | `"insertText"` or `"dispatchKeyEvent"` (see below) | + +With a `selector`, characters are injected via the browser text input pipeline (`insertText`), which updates framework-controlled inputs (React and similar) but fires no `keydown`/`keyup` events. Without a `selector`, raw key events are dispatched to the focused element. Set `method` explicitly to override either default, for example `"dispatchKeyEvent"` for a search box that filters on `keydown`. ```json { "action": "type", "text": "user@example.com", "selector": "#email", "charDelay": 40 } From c0641b4703c6f0800f5076becc804ccf743f91e9 Mon Sep 17 00:00:00 2001 From: EfeDurmaz16 Date: Fri, 15 May 2026 14:13:19 +0300 Subject: [PATCH 06/48] feat(timeline): record step timing metadata --- README.md | 2 +- apps/docs/src/app/commands/page.mdx | 2 +- .../core/src/__tests__/timeline.test.ts | 29 +++++++++++++++++ packages/@webreel/core/src/timeline.ts | 31 ++++++++++++++++++- packages/webreel/README.md | 2 +- packages/webreel/src/lib/runner.ts | 11 +++++++ 6 files changed, 73 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 5c33942..33d111c 100644 --- a/README.md +++ b/README.md @@ -130,7 +130,7 @@ webreel preview hero --verbose ### Composite -Re-composite videos from stored raw recordings and timelines without re-recording: +Re-composite videos from stored raw recordings and timelines without re-recording. Timeline JSON includes cursor frames, sound events, and per-step `startMs` / `endMs` metadata so recordings can be synced with chapters, logs, or other external traces: ```bash webreel composite diff --git a/apps/docs/src/app/commands/page.mdx b/apps/docs/src/app/commands/page.mdx index 90fe0bc..952a396 100644 --- a/apps/docs/src/app/commands/page.mdx +++ b/apps/docs/src/app/commands/page.mdx @@ -138,7 +138,7 @@ webreel composite hero webreel composite -c custom.config.json ``` -Raw video and timeline data are stored in `.webreel/raw/` and `.webreel/timelines/` after the first recording. +Raw video and timeline data are stored in `.webreel/raw/` and `.webreel/timelines/` after the first recording. Timeline JSON includes cursor frames, sound events, and per-step `startMs` / `endMs` metadata so recordings can be synced with chapters, logs, or other external traces. diff --git a/packages/@webreel/core/src/__tests__/timeline.test.ts b/packages/@webreel/core/src/__tests__/timeline.test.ts index f019ae1..fbe2a7d 100644 --- a/packages/@webreel/core/src/__tests__/timeline.test.ts +++ b/packages/@webreel/core/src/__tests__/timeline.test.ts @@ -6,6 +6,7 @@ describe("InteractionTimeline", () => { const tl = new InteractionTimeline(1080, 1080); expect(tl.getFrameCount()).toBe(0); expect(tl.getEvents()).toEqual([]); + expect(tl.getSteps()).toEqual([]); }); it("tick advances frame count", () => { @@ -33,6 +34,31 @@ describe("InteractionTimeline", () => { expect(events[0].timeMs).toBeCloseTo(1000, 0); }); + it("tracks step timing metadata", () => { + const tl = new InteractionTimeline(1080, 1080); + const startMs = tl.getCurrentTimeMs(); + for (let i = 0; i < 30; i++) tl.tick(); + tl.addStep({ + index: 0, + action: "pause", + startMs, + endMs: tl.getCurrentTimeMs(), + label: "intro", + description: "Let page render", + }); + + expect(tl.getSteps()).toEqual([ + { + index: 0, + action: "pause", + startMs: 0, + endMs: 500, + label: "intro", + description: "Let page render", + }, + ]); + }); + it("toJSON produces valid timeline data", () => { const tl = new InteractionTimeline(1920, 1080, { zoom: 2 }); tl.tick(); @@ -42,6 +68,7 @@ describe("InteractionTimeline", () => { expect(data.height).toBe(1080); expect(data.zoom).toBe(2); expect(data.frames).toHaveLength(1); + expect(data.steps).toEqual([]); expect(data.theme.cursorSize).toBe(24); }); @@ -53,6 +80,7 @@ describe("InteractionTimeline", () => { tl.setCursorPath([{ x: 100, y: 200 }]); tl.tick(); tl.addEvent("key"); + tl.addStep({ index: 1, action: "key", startMs: 0, endMs: 16.666666666666668 }); tl.tick(); const json = tl.toJSON(); @@ -61,6 +89,7 @@ describe("InteractionTimeline", () => { expect(reJson.frames).toEqual(json.frames); expect(reJson.events).toEqual(json.events); + expect(reJson.steps).toEqual(json.steps); expect(reJson.width).toBe(json.width); expect(reJson.height).toBe(json.height); expect(reJson.zoom).toBe(json.zoom); diff --git a/packages/@webreel/core/src/timeline.ts b/packages/@webreel/core/src/timeline.ts index 3c37437..0ce30ac 100644 --- a/packages/@webreel/core/src/timeline.ts +++ b/packages/@webreel/core/src/timeline.ts @@ -44,6 +44,16 @@ export interface TimelineData { }; frames: FrameData[]; events: SoundEvent[]; + steps: TimelineStep[]; +} + +export interface TimelineStep { + index: number; + action: string; + startMs: number; + endMs: number; + label?: string; + description?: string; } export class InteractionTimeline { @@ -57,6 +67,7 @@ export class InteractionTimeline { private currentHud: HudState | null = null; private frames: FrameData[] = []; private events: SoundEvent[] = []; + private steps: TimelineStep[] = []; private frameCount = 0; private tickResolvers: Array<() => void> = []; private released = false; @@ -83,6 +94,7 @@ export class InteractionTimeline { hud?: Partial; loadedFrames?: FrameData[]; loadedEvents?: SoundEvent[]; + loadedSteps?: TimelineStep[]; }, ) { this.width = width; @@ -114,6 +126,9 @@ export class InteractionTimeline { if (options?.loadedEvents) { this.events = options.loadedEvents; } + if (options?.loadedSteps) { + this.steps = options.loadedSteps; + } } setCursorPath(positions: Point[]): void { @@ -134,10 +149,18 @@ export class InteractionTimeline { } addEvent(type: "click" | "key"): void { - const timeMs = (this.frameCount / this.fps) * 1000; + const timeMs = this.getCurrentTimeMs(); this.events.push({ type, timeMs }); } + addStep(step: TimelineStep): void { + this.steps.push(step); + } + + getCurrentTimeMs(): number { + return (this.frameCount / this.fps) * 1000; + } + waitForNextTick(): Promise { if (this.released) return Promise.resolve(); return new Promise((resolve) => { @@ -187,6 +210,10 @@ export class InteractionTimeline { return this.events; } + getSteps(): TimelineStep[] { + return this.steps; + } + getFrameCount(): number { return this.frameCount; } @@ -205,6 +232,7 @@ export class InteractionTimeline { }, frames: this.frames, events: this.events, + steps: this.steps, }; } @@ -222,6 +250,7 @@ export class InteractionTimeline { hud: json.theme.hud, loadedFrames: json.frames, loadedEvents: json.events, + loadedSteps: json.steps, }); } } diff --git a/packages/webreel/README.md b/packages/webreel/README.md index 9a69830..3a77468 100644 --- a/packages/webreel/README.md +++ b/packages/webreel/README.md @@ -141,7 +141,7 @@ webreel composite webreel composite hero login ``` -Raw video and timeline data are saved in `.webreel/raw/` and `.webreel/timelines/` during `webreel record`. Use `composite` to re-apply cursor overlays, HUD, and sound effects without re-running the browser. +Raw video and timeline data are saved in `.webreel/raw/` and `.webreel/timelines/` during `webreel record`. Timeline JSON includes cursor frames, sound events, and per-step `startMs` / `endMs` metadata so recordings can be synced with chapters, logs, or other external traces. Use `composite` to re-apply cursor overlays, HUD, and sound effects without re-running the browser. ## Config format diff --git a/packages/webreel/src/lib/runner.ts b/packages/webreel/src/lib/runner.ts index a68a9ea..4e1a1a2 100644 --- a/packages/webreel/src/lib/runner.ts +++ b/packages/webreel/src/lib/runner.ts @@ -277,6 +277,7 @@ export async function runVideo( for (let i = 0; i < config.steps.length; i++) { const step = config.steps[i]; if (verbose) console.log(formatStep(i, step)); + const stepStartMs = timeline?.getCurrentTimeMs(); try { switch (step.action) { @@ -441,6 +442,16 @@ export async function runVideo( if (postDelay !== undefined && postDelay > 0) { await pause(postDelay); } + if (timeline && stepStartMs !== undefined) { + timeline.addStep({ + index: i, + action: step.action, + startMs: stepStartMs, + endMs: timeline.getCurrentTimeMs(), + ...(step.label ? { label: step.label } : {}), + ...(step.description ? { description: step.description } : {}), + }); + } } catch (err) { throw new Error( `Step ${i} (${step.action}) failed at ${url}: ${err instanceof Error ? err.message : String(err)}`, From 79621b3273eb643aad3c1cfc71839a3a26a50c4d Mon Sep 17 00:00:00 2001 From: Garth Scaysbrook Date: Sat, 13 Jun 2026 23:24:51 +1000 Subject: [PATCH 07/48] feat: add navigateHref action --- README.md | 29 +++++++++-------- apps/docs/public/schema/v1.json | 28 ++++++++++++++++ apps/docs/src/app/actions/page.mdx | 32 +++++++++++++++++++ packages/webreel/README.md | 29 +++++++++-------- .../webreel/src/lib/__tests__/config.test.ts | 12 +++++++ .../webreel/src/lib/__tests__/runner.test.ts | 5 +++ packages/webreel/src/lib/config.ts | 11 +++++++ packages/webreel/src/lib/runner.ts | 29 +++++++++++++++++ packages/webreel/src/lib/types.ts | 9 ++++++ skills/webreel/SKILL.md | 29 +++++++++-------- skills/webreel/steps-reference.md | 14 ++++++++ 11 files changed, 185 insertions(+), 42 deletions(-) diff --git a/README.md b/README.md index 33d111c..d4539b1 100644 --- a/README.md +++ b/README.md @@ -165,20 +165,21 @@ webreel record --help ### Actions -| Action | Fields | Description | -| ------------ | ------------------------------------------------------------ | ------------------------------------ | -| `pause` | `ms` | Wait for a duration | -| `click` | `text` or `selector`, optional `within`, `modifiers` | Move cursor to an element and click | -| `key` | `key` (e.g. `"cmd+z"`), optional `label` | Press a key or key combo | -| `type` | `text`, optional `selector`, `within`, `charDelay`, `method` | Type text character by character | -| `scroll` | optional `x`, `y`, `selector` | Scroll the page or an element | -| `wait` | `selector` or `text`, optional `timeout` | Wait for an element to appear | -| `screenshot` | `output` | Capture a PNG screenshot | -| `drag` | `from` and `to` (each with `text`/`selector`/`within`) | Drag from one element to another | -| `moveTo` | `text` or `selector`, optional `within` | Move cursor to an element | -| `navigate` | `url` | Navigate to a new URL mid-video | -| `hover` | `text` or `selector`, optional `within` | Hover over an element (triggers CSS) | -| `select` | `selector`, `value` | Select a value in a dropdown | +| Action | Fields | Description | +| -------------- | ------------------------------------------------------------ | ------------------------------------ | +| `pause` | `ms` | Wait for a duration | +| `click` | `text` or `selector`, optional `within`, `modifiers` | Move cursor to an element and click | +| `key` | `key` (e.g. `"cmd+z"`), optional `label` | Press a key or key combo | +| `type` | `text`, optional `selector`, `within`, `charDelay`, `method` | Type text character by character | +| `scroll` | optional `x`, `y`, `selector` | Scroll the page or an element | +| `wait` | `selector` or `text`, optional `timeout` | Wait for an element to appear | +| `screenshot` | `output` | Capture a PNG screenshot | +| `drag` | `from` and `to` (each with `text`/`selector`/`within`) | Drag from one element to another | +| `moveTo` | `text` or `selector`, optional `within` | Move cursor to an element | +| `navigate` | `url` | Navigate to a new URL mid-video | +| `navigateHref` | `selector` | Navigate to a link element's href | +| `hover` | `text` or `selector`, optional `within` | Hover over an element (triggers CSS) | +| `select` | `selector`, `value` | Select a value in a dropdown | All steps (except `pause`) accept an optional `delay` field (ms to wait after the step). Use `defaultDelay` at the top-level or per-video to set a default. diff --git a/apps/docs/public/schema/v1.json b/apps/docs/public/schema/v1.json index 8f57d03..cbf58f3 100644 --- a/apps/docs/public/schema/v1.json +++ b/apps/docs/public/schema/v1.json @@ -339,6 +339,9 @@ { "$ref": "#/$defs/stepNavigate" }, + { + "$ref": "#/$defs/stepNavigateHref" + }, { "$ref": "#/$defs/stepHover" }, @@ -714,6 +717,31 @@ }, "additionalProperties": false }, + "stepNavigateHref": { + "type": "object", + "required": ["action", "selector"], + "properties": { + "action": { + "type": "string", + "const": "navigateHref" + }, + "selector": { + "type": "string", + "minLength": 1, + "description": "CSS selector for an element with an href attribute." + }, + "label": { + "$ref": "#/$defs/label" + }, + "delay": { + "$ref": "#/$defs/delay" + }, + "description": { + "$ref": "#/$defs/description" + } + }, + "additionalProperties": false + }, "stepHover": { "type": "object", "required": ["action"], diff --git a/apps/docs/src/app/actions/page.mdx b/apps/docs/src/app/actions/page.mdx index 60a53ce..6120580 100644 --- a/apps/docs/src/app/actions/page.mdx +++ b/apps/docs/src/app/actions/page.mdx @@ -452,6 +452,38 @@ Navigate to a new URL during a recording. Useful for multi-page flows where you
+## navigateHref + +Read an element's `href` attribute and navigate the current page to that URL. This is useful when a workflow generates a fresh link during the recording, such as a preview, deployment, or published report URL. + +```json +{ "action": "wait", "selector": "a.preview-link" } +{ "action": "navigateHref", "selector": "a.preview-link" } +``` + + + + + + + + + + + + + + + + +
FieldTypeDescription
+ selector + string + CSS selector for an element with an href attribute +
+ +The href is resolved against the current page URL before navigation, so relative links work too. + ## hover Hover over an element, triggering any CSS hover states or JavaScript hover handlers. diff --git a/packages/webreel/README.md b/packages/webreel/README.md index 3a77468..375f010 100644 --- a/packages/webreel/README.md +++ b/packages/webreel/README.md @@ -180,20 +180,21 @@ Raw video and timeline data are saved in `.webreel/raw/` and `.webreel/timelines ### Actions -| Action | Fields | Description | -| ------------ | ------------------------------------------------------------ | ------------------------------------- | -| `pause` | `ms` | Wait for a duration | -| `click` | `text` or `selector`, optional `within`, `modifiers` | Move cursor to an element and click | -| `key` | `key` (e.g. `"cmd+z"`), optional `label`, `target` | Press a key or key combo | -| `type` | `text`, optional `selector`, `within`, `charDelay`, `method` | Type text character by character | -| `drag` | `from` and `to` (each with `text`/`selector`/`within`) | Drag from one element to another | -| `scroll` | optional `x`, `y`, `selector` | Scroll the page or a container | -| `wait` | `selector` or `text`, optional `timeout` | Wait for an element or text to appear | -| `moveTo` | `text` or `selector`, optional `within` | Move cursor to an element | -| `screenshot` | `output` | Save a PNG screenshot | -| `navigate` | `url` | Navigate to a new URL mid-video | -| `hover` | `text` or `selector`, optional `within` | Hover over an element (triggers CSS) | -| `select` | `selector`, `value` | Select a value in a dropdown | +| Action | Fields | Description | +| -------------- | ------------------------------------------------------------ | ------------------------------------- | +| `pause` | `ms` | Wait for a duration | +| `click` | `text` or `selector`, optional `within`, `modifiers` | Move cursor to an element and click | +| `key` | `key` (e.g. `"cmd+z"`), optional `label`, `target` | Press a key or key combo | +| `type` | `text`, optional `selector`, `within`, `charDelay`, `method` | Type text character by character | +| `drag` | `from` and `to` (each with `text`/`selector`/`within`) | Drag from one element to another | +| `scroll` | optional `x`, `y`, `selector` | Scroll the page or a container | +| `wait` | `selector` or `text`, optional `timeout` | Wait for an element or text to appear | +| `moveTo` | `text` or `selector`, optional `within` | Move cursor to an element | +| `screenshot` | `output` | Save a PNG screenshot | +| `navigate` | `url` | Navigate to a new URL mid-video | +| `navigateHref` | `selector` | Navigate to a link element's href | +| `hover` | `text` or `selector`, optional `within` | Hover over an element (triggers CSS) | +| `select` | `selector`, `value` | Select a value in a dropdown | All steps (except `pause`) accept an optional `delay` field (ms to wait after the step). Use `defaultDelay` at the top-level or per-video to set a default. diff --git a/packages/webreel/src/lib/__tests__/config.test.ts b/packages/webreel/src/lib/__tests__/config.test.ts index 0a648a8..49a1725 100644 --- a/packages/webreel/src/lib/__tests__/config.test.ts +++ b/packages/webreel/src/lib/__tests__/config.test.ts @@ -208,6 +208,18 @@ describe("validateStep", () => { expect(errors).toEqual([]); }); + it("validates navigateHref requires selector", () => { + const errors = validate({ action: "navigateHref" }); + expect(errors).toContainEqual( + expect.objectContaining({ path: "videos.x.steps[0].selector" }), + ); + }); + + it("accepts valid navigateHref", () => { + const errors = validate({ action: "navigateHref", selector: "a.preview-link" }); + expect(errors).toEqual([]); + }); + it("accepts description field on any step", () => { const errors = validate({ action: "pause", diff --git a/packages/webreel/src/lib/__tests__/runner.test.ts b/packages/webreel/src/lib/__tests__/runner.test.ts index be8f0fb..4405d1f 100644 --- a/packages/webreel/src/lib/__tests__/runner.test.ts +++ b/packages/webreel/src/lib/__tests__/runner.test.ts @@ -81,6 +81,11 @@ describe("formatStep", () => { expect(formatStep(0, step)).toBe('[step 0] navigate "https://example.com"'); }); + it("formats navigateHref step", () => { + const step: Step = { action: "navigateHref", selector: "a.preview-link" }; + expect(formatStep(0, step)).toBe('[step 0] navigateHref selector="a.preview-link"'); + }); + it("formats hover step", () => { const step: Step = { action: "hover", text: "Link" }; expect(formatStep(0, step)).toBe('[step 0] hover text="Link"'); diff --git a/packages/webreel/src/lib/config.ts b/packages/webreel/src/lib/config.ts index f42754c..adc3cb4 100644 --- a/packages/webreel/src/lib/config.ts +++ b/packages/webreel/src/lib/config.ts @@ -288,6 +288,7 @@ const VALID_ACTIONS = new Set([ "wait", "screenshot", "navigate", + "navigateHref", "hover", "select", ]); @@ -380,6 +381,7 @@ const KNOWN_STEP_KEYS: Record> = { ]), screenshot: new Set(["action", "output", "label", "delay", "description"]), navigate: new Set(["action", "url", "label", "delay", "description"]), + navigateHref: new Set(["action", "selector", "label", "delay", "description"]), hover: new Set([ "action", "text", @@ -620,6 +622,15 @@ function validateStep(step: unknown, index: number): ValidationError[] { } break; + case "navigateHref": + if (typeof s.selector !== "string" || s.selector.length === 0) { + errors.push({ + path: `${prefix}.selector`, + message: "Must be a non-empty CSS selector string", + }); + } + break; + case "hover": if (!s.text && !s.selector) { errors.push({ diff --git a/packages/webreel/src/lib/runner.ts b/packages/webreel/src/lib/runner.ts index 4e1a1a2..b9ba7b8 100644 --- a/packages/webreel/src/lib/runner.ts +++ b/packages/webreel/src/lib/runner.ts @@ -54,6 +54,8 @@ export function formatStep(i: number, step: Step): string { return `[step ${i}] screenshot "${step.output}"${desc}`; case "navigate": return `[step ${i}] navigate "${step.url}"${desc}`; + case "navigateHref": + return `[step ${i}] navigateHref selector="${step.selector}"${desc}`; case "hover": return `[step ${i}] hover ${step.text ? `text="${step.text}"` : `selector="${step.selector}"`}${desc}`; case "select": @@ -393,6 +395,33 @@ export async function runVideo( break; } + case "navigateHref": { + // Runtime.evaluate does not surface in-page exceptions, so the + // probe returns sentinels for the two expected failure modes. + const result = await client.Runtime.evaluate({ + expression: `(() => { + const el = document.querySelector(${JSON.stringify(step.selector)}); + if (!el) return "__WEBREEL_NOT_FOUND__"; + const href = el.href || el.getAttribute("href"); + if (!href) return "__WEBREEL_NO_HREF__"; + return new URL(href, window.location.href).href; + })()`, + returnByValue: true, + }); + const href = result.result?.value; + if (href === "__WEBREEL_NOT_FOUND__") { + throw new Error(`Element not found: selector="${step.selector}"`); + } + if (href === "__WEBREEL_NO_HREF__") { + throw new Error(`Element has no href: selector="${step.selector}"`); + } + if (typeof href !== "string" || href.length === 0) { + throw new Error("navigateHref did not resolve a URL"); + } + await navigate(client, href); + break; + } + case "hover": { const box = await resolveTarget(client, step); const { x: hx, y: hy } = randomPointInBox(box, 0.1); diff --git a/packages/webreel/src/lib/types.ts b/packages/webreel/src/lib/types.ts index d3dfd40..b18634c 100644 --- a/packages/webreel/src/lib/types.ts +++ b/packages/webreel/src/lib/types.ts @@ -101,6 +101,14 @@ export interface StepNavigate { description?: string; } +export interface StepNavigateHref { + action: "navigateHref"; + selector: string; + label?: string; + delay?: number; + description?: string; +} + export interface StepHover { action: "hover"; text?: string; @@ -133,6 +141,7 @@ export type Step = | StepMoveTo | StepScreenshot | StepNavigate + | StepNavigateHref | StepHover | StepSelect; diff --git a/skills/webreel/SKILL.md b/skills/webreel/SKILL.md index 49e8c37..0732d9e 100644 --- a/skills/webreel/SKILL.md +++ b/skills/webreel/SKILL.md @@ -183,20 +183,21 @@ Record specific videos by name: `webreel record hero login`. Each step has an `action` field. Most steps accept optional `label`, `delay` (ms after step), and `description` fields. -| Action | Key fields | Purpose | -| ------------ | --------------------------------------------------- | ---------------------------------- | -| `pause` | `ms` | Wait for a duration | -| `click` | `text` or `selector`, `within`, `modifiers` | Click an element | -| `type` | `text`, `selector`, `within`, `charDelay`, `method` | Type text into an input | -| `key` | `key`, `target` | Press a key combo (e.g. `"cmd+s"`) | -| `drag` | `from`, `to` (element targets) | Drag between two elements | -| `scroll` | `x`, `y`, `selector` | Scroll the page or an element | -| `wait` | `selector` or `text`, `timeout` | Wait for an element to appear | -| `moveTo` | `text` or `selector`, `within` | Move cursor to an element | -| `navigate` | `url` | Navigate to a new URL | -| `hover` | `text` or `selector`, `within` | Hover over an element | -| `select` | `selector`, `value` | Select a dropdown value | -| `screenshot` | `output` | Capture a PNG screenshot | +| Action | Key fields | Purpose | +| -------------- | --------------------------------------------------- | ---------------------------------- | +| `pause` | `ms` | Wait for a duration | +| `click` | `text` or `selector`, `within`, `modifiers` | Click an element | +| `type` | `text`, `selector`, `within`, `charDelay`, `method` | Type text into an input | +| `key` | `key`, `target` | Press a key combo (e.g. `"cmd+s"`) | +| `drag` | `from`, `to` (element targets) | Drag between two elements | +| `scroll` | `x`, `y`, `selector` | Scroll the page or an element | +| `wait` | `selector` or `text`, `timeout` | Wait for an element to appear | +| `moveTo` | `text` or `selector`, `within` | Move cursor to an element | +| `navigate` | `url` | Navigate to a new URL | +| `navigateHref` | `selector` | Navigate to a link element's href | +| `hover` | `text` or `selector`, `within` | Hover over an element | +| `select` | `selector`, `value` | Select a dropdown value | +| `screenshot` | `output` | Capture a PNG screenshot | For full field details on every step type, see [steps-reference.md](steps-reference.md). diff --git a/skills/webreel/steps-reference.md b/skills/webreel/steps-reference.md index ae8935e..4785006 100644 --- a/skills/webreel/steps-reference.md +++ b/skills/webreel/steps-reference.md @@ -193,6 +193,20 @@ Navigate the browser to a new URL. { "action": "navigate", "url": "/settings" } ``` +## navigateHref + +Read an element's `href` attribute and navigate the browser to that URL. Use this for generated links that change each run, such as preview, deploy, or published report URLs. + +| Field | Type | Required | Description | +| ---------- | ---------------- | -------- | ------------------------------------------------ | +| `action` | `"navigateHref"` | yes | | +| `selector` | string | yes | CSS selector for an element with an `href` value | + +```json +{ "action": "wait", "selector": "a.preview-link" } +{ "action": "navigateHref", "selector": "a.preview-link" } +``` + ## select Select a value in a `` dropdown element. Provide either `text` or `selector`, not both. + +## upload + +Set a file on an `` element via the browser's file input pipeline. The native `change` event fires, so framework `onChange` handlers (React and similar) run as if the user picked the file. + +```json +{ + "action": "upload", + "selector": "input[type='file']", + "filePath": "./fixtures/avatar.png" +} +``` + + + + + + + + + + + + + + + + + + + + + +
FieldTypeDescription
+ selector + stringCSS selector for the file input element
+ filePath + stringPath to the file, resolved relative to the config file
+ +The upload itself is invisible in the video (no cursor movement). Pair it with a `click` on the surrounding UI when the recording should look like a user action. diff --git a/packages/@webreel/core/src/types.ts b/packages/@webreel/core/src/types.ts index fd6bd7c..e30f2cb 100644 --- a/packages/@webreel/core/src/types.ts +++ b/packages/@webreel/core/src/types.ts @@ -49,6 +49,12 @@ export type CDPClient = { }; DOM: { enable: () => Promise; + getDocument: (params: { depth?: number }) => Promise<{ root: { nodeId: number } }>; + querySelector: (params: { + nodeId: number; + selector: string; + }) => Promise<{ nodeId: number }>; + setFileInputFiles: (params: { nodeId: number; files: string[] }) => Promise; }; }; diff --git a/packages/webreel/README.md b/packages/webreel/README.md index 375f010..9ce2c50 100644 --- a/packages/webreel/README.md +++ b/packages/webreel/README.md @@ -195,6 +195,7 @@ Raw video and timeline data are saved in `.webreel/raw/` and `.webreel/timelines | `navigateHref` | `selector` | Navigate to a link element's href | | `hover` | `text` or `selector`, optional `within` | Hover over an element (triggers CSS) | | `select` | `selector`, `value` | Select a value in a dropdown | +| `upload` | `selector`, `filePath` | Set a file on a file input | All steps (except `pause`) accept an optional `delay` field (ms to wait after the step). Use `defaultDelay` at the top-level or per-video to set a default. diff --git a/packages/webreel/src/lib/__tests__/config.test.ts b/packages/webreel/src/lib/__tests__/config.test.ts index 49a1725..6f6a1f6 100644 --- a/packages/webreel/src/lib/__tests__/config.test.ts +++ b/packages/webreel/src/lib/__tests__/config.test.ts @@ -220,6 +220,43 @@ describe("validateStep", () => { expect(errors).toEqual([]); }); + it("accepts valid upload step", () => { + const errors = validate({ + action: "upload", + selector: "input[type='file']", + filePath: "audio.mp3", + }); + expect(errors).toEqual([]); + }); + + it("validates upload requires non-empty selector", () => { + const errors = validate({ action: "upload", selector: "", filePath: "audio.mp3" }); + expect(errors).toContainEqual( + expect.objectContaining({ path: "videos.x.steps[0].selector" }), + ); + }); + + it("validates upload rejects missing selector", () => { + const errors = validate({ action: "upload", filePath: "audio.mp3" }); + expect(errors).toContainEqual( + expect.objectContaining({ path: "videos.x.steps[0].selector" }), + ); + }); + + it("validates upload requires non-empty filePath", () => { + const errors = validate({ action: "upload", selector: "#input", filePath: "" }); + expect(errors).toContainEqual( + expect.objectContaining({ path: "videos.x.steps[0].filePath" }), + ); + }); + + it("validates upload rejects missing filePath", () => { + const errors = validate({ action: "upload", selector: "#input" }); + expect(errors).toContainEqual( + expect.objectContaining({ path: "videos.x.steps[0].filePath" }), + ); + }); + it("accepts description field on any step", () => { const errors = validate({ action: "pause", diff --git a/packages/webreel/src/lib/__tests__/runner.test.ts b/packages/webreel/src/lib/__tests__/runner.test.ts index 4405d1f..a1cd14f 100644 --- a/packages/webreel/src/lib/__tests__/runner.test.ts +++ b/packages/webreel/src/lib/__tests__/runner.test.ts @@ -96,6 +96,29 @@ describe("formatStep", () => { expect(formatStep(0, step)).toBe('[step 0] select "#country" value="US"'); }); + it("formats upload step", () => { + const step: Step = { + action: "upload", + selector: "input[type='file']", + filePath: "audio.mp3", + }; + expect(formatStep(0, step)).toBe( + `[step 0] upload selector="input[type='file']" file="audio.mp3"`, + ); + }); + + it("formats upload step with description", () => { + const step: Step = { + action: "upload", + selector: "#file-input", + filePath: "tests/fixtures/heydonna.mp3", + description: "upload audio", + }; + expect(formatStep(0, step)).toBe( + '[step 0] upload selector="#file-input" file="tests/fixtures/heydonna.mp3": upload audio', + ); + }); + it("includes description when present", () => { const step: Step = { action: "pause", ms: 100, description: "wait for animation" }; expect(formatStep(0, step)).toBe("[step 0] pause 100ms: wait for animation"); diff --git a/packages/webreel/src/lib/config.ts b/packages/webreel/src/lib/config.ts index adc3cb4..2bccbd9 100644 --- a/packages/webreel/src/lib/config.ts +++ b/packages/webreel/src/lib/config.ts @@ -291,6 +291,7 @@ const VALID_ACTIONS = new Set([ "navigateHref", "hover", "select", + "upload", ]); const KNOWN_TOP_LEVEL_KEYS = new Set([ @@ -401,6 +402,7 @@ const KNOWN_STEP_KEYS: Record> = { "delay", "description", ]), + upload: new Set(["action", "selector", "filePath", "label", "delay", "description"]), }; export interface ValidationError { @@ -652,6 +654,21 @@ function validateStep(step: unknown, index: number): ValidationError[] { } break; } + + case "upload": + if (typeof s.selector !== "string" || s.selector.length === 0) { + errors.push({ + path: `${prefix}.selector`, + message: "Must be a non-empty string", + }); + } + if (typeof s.filePath !== "string" || s.filePath.length === 0) { + errors.push({ + path: `${prefix}.filePath`, + message: "Must be a non-empty string", + }); + } + break; } if (s.delay !== undefined && (!Number.isFinite(s.delay) || (s.delay as number) < 0)) { diff --git a/packages/webreel/src/lib/runner.ts b/packages/webreel/src/lib/runner.ts index b9ba7b8..748093e 100644 --- a/packages/webreel/src/lib/runner.ts +++ b/packages/webreel/src/lib/runner.ts @@ -1,5 +1,12 @@ import { resolve, dirname } from "node:path"; -import { readFileSync, writeFileSync, mkdirSync, renameSync, rmSync } from "node:fs"; +import { + readFileSync, + writeFileSync, + mkdirSync, + renameSync, + rmSync, + existsSync, +} from "node:fs"; import { pathToFileURL } from "node:url"; import { type CDPClient, @@ -60,6 +67,8 @@ export function formatStep(i: number, step: Step): string { return `[step ${i}] hover ${step.text ? `text="${step.text}"` : `selector="${step.selector}"`}${desc}`; case "select": return `[step ${i}] select "${step.selector}" value="${step.value}"${desc}`; + case "upload": + return `[step ${i}] upload selector="${step.selector}" file="${step.filePath}"${desc}`; default: { const _exhaustive: never = step; return `[step ${i}] ${(_exhaustive as Step).action}`; @@ -465,6 +474,24 @@ export async function runVideo( } break; } + + case "upload": { + const absolutePath = resolve(configDir, step.filePath); + if (!existsSync(absolutePath)) { + throw new Error(`File not found: ${absolutePath}`); + } + await client.DOM.enable(); + const { root } = await client.DOM.getDocument({ depth: 0 }); + const { nodeId } = await client.DOM.querySelector({ + nodeId: root.nodeId, + selector: step.selector, + }); + if (!nodeId) { + throw new Error(`File input not found: selector="${step.selector}"`); + } + await client.DOM.setFileInputFiles({ nodeId, files: [absolutePath] }); + break; + } } const stepDelay = "delay" in step ? step.delay : undefined; const postDelay = stepDelay ?? config.defaultDelay; diff --git a/packages/webreel/src/lib/types.ts b/packages/webreel/src/lib/types.ts index b18634c..aadc1f6 100644 --- a/packages/webreel/src/lib/types.ts +++ b/packages/webreel/src/lib/types.ts @@ -130,6 +130,15 @@ export interface StepSelect { description?: string; } +export interface StepUpload { + action: "upload"; + selector: string; + filePath: string; + label?: string; + delay?: number; + description?: string; +} + export type Step = | StepPause | StepClick @@ -143,7 +152,8 @@ export type Step = | StepNavigate | StepNavigateHref | StepHover - | StepSelect; + | StepSelect + | StepUpload; export interface CursorConfig { image?: string; diff --git a/skills/webreel/SKILL.md b/skills/webreel/SKILL.md index 0732d9e..8903f8c 100644 --- a/skills/webreel/SKILL.md +++ b/skills/webreel/SKILL.md @@ -197,6 +197,7 @@ Each step has an `action` field. Most steps accept optional `label`, `delay` (ms | `navigateHref` | `selector` | Navigate to a link element's href | | `hover` | `text` or `selector`, `within` | Hover over an element | | `select` | `selector`, `value` | Select a dropdown value | +| `upload` | `selector`, `filePath` | Set a file on a file input | | `screenshot` | `output` | Capture a PNG screenshot | For full field details on every step type, see [steps-reference.md](steps-reference.md). diff --git a/skills/webreel/steps-reference.md b/skills/webreel/steps-reference.md index 4785006..be0b1cd 100644 --- a/skills/webreel/steps-reference.md +++ b/skills/webreel/steps-reference.md @@ -235,3 +235,21 @@ Capture a PNG screenshot of the current viewport. ```json { "action": "screenshot", "output": "screenshots/final-state.png" } ``` + +## upload + +Set a file on an `` element. Fires the native `change` event so framework `onChange` handlers run. Not visible in the video; pair with a `click` on the surrounding UI for realism. + +| Field | Type | Required | Description | +| ---------- | ---------- | -------- | ------------------------------------------------- | +| `action` | `"upload"` | yes | | +| `selector` | string | yes | CSS selector for the file input element | +| `filePath` | string | yes | Path to the file, resolved relative to the config | + +```json +{ + "action": "upload", + "selector": "input[type='file']", + "filePath": "./fixtures/avatar.png" +} +``` From 2c30609d4d5b8608b8039e0917489e2b9af8452a Mon Sep 17 00:00:00 2001 From: Ariel Conti Date: Tue, 7 Jul 2026 10:38:58 +0100 Subject: [PATCH 09/48] perf: compositor prefetch, single-pass GIF, EXDEV rename fallback Port the safe performance parts of PR 12 by sld0Ant: - Producer/consumer prefetch queue in compositeFrames so sharp overlay rendering overlaps with ffmpeg encoding instead of alternating with it. The ffmpeg close/error listeners are now registered before streaming begins, and stdin EPIPE is tolerated when ffmpeg exits early. - Single-pass GIF output: overlay, palettegen, and paletteuse run in one ffmpeg filter_complex, replacing the intermediate x264 encode plus finalizeGif second pass. PR reports about 21 percent less time and 64 percent smaller GIFs. - Whole-pixel cursor cache key rounding so float jitter during cursor dwell no longer defeats the overlay cache. - scripts/benchmark.sh for repeatable record timing runs, with a small fix so the exit code check is not dead under set -e. - Ignore .idea/, videos/, and webreel.config.json. Also port the EXDEV cross-filesystem rename fallback (moveFileSync in @webreel/core with copy plus delete fallback, used by finalizeMp4, finalizeWebm, and the runner raw-video move). PR 12 references it, but it landed on main as commit 5cb748d (PR 15); ported here because this branch predates that merge. Excluded from the port: all recorder.ts changes (the HeadlessExperimental.beginFrame capture crashes chrome-headless-shell with SIGTRAP on this machine, and the local recorder rework must stay), the runner.ts frame pump, and the HeadlessExperimental CDP typings. --- .gitignore | 4 + .../@webreel/core/src/__tests__/fs.test.ts | 50 ++++ packages/@webreel/core/src/compositor.ts | 243 ++++++++++++++---- packages/@webreel/core/src/fs.ts | 18 ++ packages/@webreel/core/src/index.ts | 1 + packages/@webreel/core/src/media.ts | 7 +- packages/webreel/src/lib/runner.ts | 12 +- scripts/benchmark.sh | 50 ++++ 8 files changed, 321 insertions(+), 64 deletions(-) create mode 100644 packages/@webreel/core/src/__tests__/fs.test.ts create mode 100644 packages/@webreel/core/src/fs.ts create mode 100755 scripts/benchmark.sh diff --git a/.gitignore b/.gitignore index ffeb0f9..efbfea3 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,7 @@ coverage *.log package-lock.json demo-reel.mp4 + +.idea/ +videos/ +webreel.config.json diff --git a/packages/@webreel/core/src/__tests__/fs.test.ts b/packages/@webreel/core/src/__tests__/fs.test.ts new file mode 100644 index 0000000..8fe8a73 --- /dev/null +++ b/packages/@webreel/core/src/__tests__/fs.test.ts @@ -0,0 +1,50 @@ +import { describe, it, expect, vi, beforeEach } from "vitest"; +import { moveFileSync } from "../fs.js"; +import * as fs from "node:fs"; + +vi.mock("node:fs", async () => { + const actual = await vi.importActual("node:fs"); + return { + ...actual, + renameSync: vi.fn(), + copyFileSync: vi.fn(), + rmSync: vi.fn(), + }; +}); + +beforeEach(() => { + vi.clearAllMocks(); +}); + +describe("moveFileSync", () => { + it("uses renameSync when source and dest are on the same device", () => { + moveFileSync("/tmp/a.mp4", "/tmp/b.mp4"); + + expect(fs.renameSync).toHaveBeenCalledWith("/tmp/a.mp4", "/tmp/b.mp4"); + expect(fs.copyFileSync).not.toHaveBeenCalled(); + }); + + it("falls back to copy+delete on EXDEV error", () => { + const exdev = Object.assign(new Error("cross-device link not permitted"), { + code: "EXDEV", + }); + vi.mocked(fs.renameSync).mockImplementation(() => { + throw exdev; + }); + + moveFileSync("/dev1/a.mp4", "/dev2/b.mp4"); + + expect(fs.copyFileSync).toHaveBeenCalledWith("/dev1/a.mp4", "/dev2/b.mp4"); + expect(fs.rmSync).toHaveBeenCalledWith("/dev1/a.mp4", { force: true }); + }); + + it("re-throws non-EXDEV errors", () => { + const enoent = Object.assign(new Error("no such file"), { code: "ENOENT" }); + vi.mocked(fs.renameSync).mockImplementation(() => { + throw enoent; + }); + + expect(() => moveFileSync("/tmp/missing.mp4", "/tmp/b.mp4")).toThrow("no such file"); + expect(fs.copyFileSync).not.toHaveBeenCalled(); + }); +}); diff --git a/packages/@webreel/core/src/compositor.ts b/packages/@webreel/core/src/compositor.ts index 5645266..c7d2009 100644 --- a/packages/@webreel/core/src/compositor.ts +++ b/packages/@webreel/core/src/compositor.ts @@ -5,7 +5,7 @@ import { resolve, extname } from "node:path"; import sharp from "sharp"; import type { TimelineData } from "./timeline.js"; import { ensureFfmpeg } from "./ffmpeg.js"; -import { finalizeMp4, finalizeWebm, finalizeGif, type SfxConfig } from "./media.js"; +import { finalizeMp4, finalizeWebm, type SfxConfig } from "./media.js"; interface OverlayContext { cursorPng: Buffer; @@ -40,22 +40,36 @@ export async function compose( zoom, ); + const ext = extname(outputPath).toLowerCase(); + + if (ext === ".gif") { + const gifConfig = buildGifConfig(timelineData.width, outputPath); + await compositeFrames( + ffmpegPath, + cleanVideoPath, + timelineData, + cursorPng, + zoom, + gifConfig, + ); + return; + } + const workDir = resolve(homedir(), ".webreel"); mkdirSync(workDir, { recursive: true }); const tempComposed = resolve(workDir, `_composed_${Date.now()}.mp4`); try { + const mp4Config = buildMp4Config(timelineData.fps, crf, tempComposed); await compositeFrames( ffmpegPath, cleanVideoPath, timelineData, cursorPng, zoom, - tempComposed, - crf, + mp4Config, ); - const ext = extname(outputPath).toLowerCase(); const durationSec = timelineData.frames.length / timelineData.fps; if (ext === ".webm") { @@ -67,8 +81,6 @@ export async function compose( durationSec, sfx, ); - } else if (ext === ".gif") { - finalizeGif(ffmpegPath, tempComposed, outputPath, timelineData.width); } else { finalizeMp4( ffmpegPath, @@ -97,14 +109,64 @@ async function renderCursorPng( return sharp(Buffer.from(svgWithSize)).png().toBuffer(); } +interface CompositorFfmpegConfig { + filterComplex: string; + outputArgs: string[]; +} + +function buildMp4Config( + fps: number, + crf: number, + outputPath: string, +): CompositorFfmpegConfig { + return { + filterComplex: "[0][1]overlay=0:0:shortest=1", + outputArgs: [ + "-c:v", + "libx264", + "-preset", + "ultrafast", + "-crf", + String(crf), + "-pix_fmt", + "yuv420p", + "-color_primaries", + "bt709", + "-color_trc", + "bt709", + "-colorspace", + "bt709", + "-movflags", + "+faststart", + "-r", + String(fps), + outputPath, + ], + }; +} + +const GIF_FPS = 15; +const GIF_BAYER_SCALE = 5; + +function buildGifConfig(width: number, outputPath: string): CompositorFfmpegConfig { + return { + filterComplex: [ + `[0][1]overlay=0:0:shortest=1`, + `fps=${GIF_FPS}`, + `scale=${width}:-1:flags=lanczos`, + `split[s0][s1];[s0]palettegen=stats_mode=full[p];[s1][p]paletteuse=dither=bayer:bayer_scale=${GIF_BAYER_SCALE}`, + ].join(","), + outputArgs: ["-loop", "0", outputPath], + }; +} + async function compositeFrames( ffmpegPath: string, cleanVideoPath: string, timeline: TimelineData, cursorPng: Buffer, zoom: number, - outputPath: string, - crf: number, + config: CompositorFfmpegConfig, ): Promise { const { width, height, fps } = timeline; @@ -123,26 +185,8 @@ async function compositeFrames( "-i", "pipe:0", "-filter_complex", - "[0][1]overlay=0:0:shortest=1", - "-c:v", - "libx264", - "-preset", - "ultrafast", - "-crf", - String(crf), - "-pix_fmt", - "yuv420p", - "-color_primaries", - "bt709", - "-color_trc", - "bt709", - "-colorspace", - "bt709", - "-movflags", - "+faststart", - "-r", - String(fps), - outputPath, + config.filterComplex, + ...config.outputArgs, ], { stdio: ["pipe", "pipe", "pipe"] }, ); @@ -189,29 +233,12 @@ async function compositeFrames( const stdin = ffmpeg.stdin; if (!stdin) throw new Error("ffmpeg process has no stdin pipe"); - const drain = (): Promise => new Promise((res) => stdin.once("drain", res)); - - for (let i = 0; i < timeline.frames.length; i++) { - const frame = timeline.frames[i]; - const overlayPng = await renderOverlayFrame( - frame, - width, - height, - ctx, - overlayCache, - hudCache, - ); - - const ok = stdin.write(overlayPng); - if (!ok) await drain(); - } - - stdin.end(); - const stderrChunks: Buffer[] = []; ffmpeg.stderr?.on("data", (chunk: Buffer) => stderrChunks.push(chunk)); - await new Promise((resolveAll, rejectAll) => { + // Register close/error listeners immediately to avoid missing events. + const KILL_TIMEOUT = 5_000; + const ffmpegDone = new Promise((resolveAll, rejectAll) => { ffmpeg.on("close", (code) => { if (code === 0) { resolveAll(); @@ -226,6 +253,114 @@ async function compositeFrames( }); ffmpeg.on("error", rejectAll); }); + + const PREFETCH_QUEUE_SIZE = 4; + + const state = { + abortError: null as Error | null, + producerDone: false, + // Resolves when the queue has items OR the producer is done. + queueResolve: null as (() => void) | null, + // Resolves when the consumer dequeues an item (backpressure signal). + spaceResolve: null as (() => void) | null, + }; + + // EPIPE is expected when ffmpeg finishes reading and closes its stdin. + stdin.on("error", (err: NodeJS.ErrnoException) => { + if (err.code === "EPIPE") return; + if (!state.abortError) state.abortError = err; + }); + + const queue: Buffer[] = []; + + const notifyConsumer = () => { + if (state.queueResolve) { + const r = state.queueResolve; + state.queueResolve = null; + r(); + } + }; + + const notifyProducer = () => { + if (state.spaceResolve) { + const r = state.spaceResolve; + state.spaceResolve = null; + r(); + } + }; + + const enqueue = (buf: Buffer) => { + queue.push(buf); + notifyConsumer(); + }; + + const waitForItem = (): Promise => + new Promise((r) => { + if (queue.length > 0 || state.producerDone) return r(); + state.queueResolve = r; + }); + + const waitForSpace = (): Promise => + new Promise((r) => { + if (queue.length < PREFETCH_QUEUE_SIZE) return r(); + state.spaceResolve = r; + }); + + const drain = (): Promise => new Promise((r) => stdin.once("drain", r)); + + const consumer = async () => { + while (true) { + if (queue.length === 0 && state.producerDone) break; + if (queue.length === 0) await waitForItem(); + if (queue.length === 0) break; + if (state.abortError) break; + + while (queue.length > 0) { + const buf = queue.shift()!; + notifyProducer(); + const ok = stdin.write(buf); + if (!ok && !state.abortError) await drain(); + if (state.abortError) break; + } + } + stdin.end(); + }; + + const consumerPromise = consumer(); + + for (let i = 0; i < timeline.frames.length; i++) { + if (state.abortError) break; + + const frame = timeline.frames[i]; + const overlayPng = await renderOverlayFrame( + frame, + width, + height, + ctx, + overlayCache, + hudCache, + ); + + if (state.abortError) break; + + if (queue.length >= PREFETCH_QUEUE_SIZE) await waitForSpace(); + + if (!state.abortError) enqueue(overlayPng); + } + state.producerDone = true; + notifyConsumer(); + + await consumerPromise; + + if (state.abortError) { + ffmpeg.kill("SIGTERM"); + setTimeout(() => { + if (!ffmpeg.killed) ffmpeg.kill("SIGKILL"); + }, KILL_TIMEOUT); + throw state.abortError; + } + + await ffmpegDone; } async function renderOverlayFrame( @@ -236,8 +371,12 @@ async function renderOverlayFrame( cache: Map, hudCache: Map, ): Promise { - const cx = Math.round(frame.cursor.x * ctx.zoom * 10) / 10; - const cy = Math.round(frame.cursor.y * ctx.zoom * 10) / 10; + // Whole-pixel rounding is intentional: sub-pixel precision defeats the + // overlay cache during cursor dwell/pause (float jitter creates unique keys). + // The 1px difference is imperceptible at screen resolution and invisible + // in GIF output (downsampled to 15fps with lanczos). + const cx = Math.round(frame.cursor.x * ctx.zoom); + const cy = Math.round(frame.cursor.y * ctx.zoom); const scale = frame.cursor.scale; const hudKey = frame.hud ? frame.hud.labels.join("|") : ""; const cacheKey = `${cx},${cy},${scale},${hudKey}`; @@ -247,8 +386,8 @@ async function renderOverlayFrame( const overlays: sharp.OverlayOptions[] = []; - const icx = Math.round(frame.cursor.x * ctx.zoom) - ctx.hotspotOffsetX; - const icy = Math.round(frame.cursor.y * ctx.zoom) - ctx.hotspotOffsetY; + const icx = cx - ctx.hotspotOffsetX; + const icy = cy - ctx.hotspotOffsetY; const cursorVisible = icx >= -ctx.cursorWidth && icx < width && icy >= -ctx.cursorHeight && icy < height; diff --git a/packages/@webreel/core/src/fs.ts b/packages/@webreel/core/src/fs.ts new file mode 100644 index 0000000..55293be --- /dev/null +++ b/packages/@webreel/core/src/fs.ts @@ -0,0 +1,18 @@ +import { renameSync, copyFileSync, rmSync } from "node:fs"; + +/** + * Move a file from `src` to `dest`, falling back to copy+delete when the + * source and destination reside on different devices (`EXDEV`). + */ +export function moveFileSync(src: string, dest: string): void { + try { + renameSync(src, dest); + } catch (err: unknown) { + if ((err as NodeJS.ErrnoException).code === "EXDEV") { + copyFileSync(src, dest); + rmSync(src, { force: true }); + } else { + throw err; + } + } +} diff --git a/packages/@webreel/core/src/index.ts b/packages/@webreel/core/src/index.ts index ebb515f..95d3f58 100644 --- a/packages/@webreel/core/src/index.ts +++ b/packages/@webreel/core/src/index.ts @@ -41,3 +41,4 @@ export { InteractionTimeline, type TimelineData } from "./timeline.js"; export { compose, type ComposeOptions } from "./compositor.js"; export { ensureFfmpeg, FFMPEG_CACHE_DIR } from "./ffmpeg.js"; export { extractThumbnail, type SfxConfig } from "./media.js"; +export { moveFileSync } from "./fs.js"; diff --git a/packages/@webreel/core/src/media.ts b/packages/@webreel/core/src/media.ts index 48de07b..50ac4e2 100644 --- a/packages/@webreel/core/src/media.ts +++ b/packages/@webreel/core/src/media.ts @@ -1,5 +1,6 @@ import { spawnSync } from "node:child_process"; -import { rmSync, renameSync } from "node:fs"; +import { rmSync } from "node:fs"; +import { moveFileSync } from "./fs.js"; import { resolve, dirname } from "node:path"; import { fileURLToPath } from "node:url"; import type { SoundEvent } from "./types.js"; @@ -110,7 +111,7 @@ export function finalizeMp4( outputPath, ]); } else { - renameSync(tempVideo, outputPath); + moveFileSync(tempVideo, outputPath); } return; } @@ -170,7 +171,7 @@ export function finalizeWebm( ]); if (events.length === 0 || !sfx) { - renameSync(silentWebm, outputPath); + moveFileSync(silentWebm, outputPath); return; } diff --git a/packages/webreel/src/lib/runner.ts b/packages/webreel/src/lib/runner.ts index 748093e..5a6b6a6 100644 --- a/packages/webreel/src/lib/runner.ts +++ b/packages/webreel/src/lib/runner.ts @@ -1,12 +1,5 @@ import { resolve, dirname } from "node:path"; -import { - readFileSync, - writeFileSync, - mkdirSync, - renameSync, - rmSync, - existsSync, -} from "node:fs"; +import { readFileSync, writeFileSync, mkdirSync, rmSync, existsSync } from "node:fs"; import { pathToFileURL } from "node:url"; import { type CDPClient, @@ -33,6 +26,7 @@ import { compose, ensureFfmpeg, extractThumbnail, + moveFileSync, DEFAULT_VIEWPORT_SIZE, } from "@webreel/core"; import type { VideoConfig, Step, ElementTarget } from "./types.js"; @@ -533,7 +527,7 @@ export async function runVideo( const rawDir = resolve(configDir, ".webreel", "raw"); mkdirSync(rawDir, { recursive: true }); const rawVideoPath = resolve(rawDir, `${config.name}.mp4`); - renameSync(cleanVideoPath, rawVideoPath); + moveFileSync(cleanVideoPath, rawVideoPath); ctx.setMode("preview"); ctx.setTimeline(null); diff --git a/scripts/benchmark.sh b/scripts/benchmark.sh new file mode 100755 index 0000000..a530418 --- /dev/null +++ b/scripts/benchmark.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Run on the same machine with localhost:1420 serving the app. +# Uses webreel.config.json which must output .gif to exercise the full pipeline. + +DEMO="${1:-toolbar-demo}" +RUNS=3 +CLI="node packages/webreel/dist/index.js" + +now_ms() { + if command -v gdate &>/dev/null; then + echo $(($(gdate +%s%N) / 1000000)) + elif date +%s%N &>/dev/null 2>&1 && [ "$(date +%N)" != "%N" ]; then + echo $(($(date +%s%N) / 1000000)) + else + python3 -c 'import time; print(int(time.time()*1000))' + fi +} + +echo "=== webreel benchmark: $DEMO ===" +echo "" + +for i in $(seq 1 "$RUNS"); do + label="run $i" + if [ "$i" -eq 1 ]; then + label="run 1 (warmup)" + fi + + start=$(now_ms) + rc=0 + $CLI record "$DEMO" > /dev/null 2>&1 || rc=$? + end=$(now_ms) + + if [ "$rc" -ne 0 ]; then + echo "$label: FAILED (exit code $rc)" + exit 1 + fi + + elapsed=$(echo "scale=2; ($end - $start) / 1000" | bc) + echo "$label: ${elapsed}s" +done + +echo "" +echo "=== output ===" +OUTPUT=$(find videos/ -name "$DEMO.*" -newer scripts/benchmark.sh 2>/dev/null | head -1) +if [ -n "$OUTPUT" ]; then + SIZE=$(du -h "$OUTPUT" | cut -f1) + echo "$OUTPUT: $SIZE" +fi From 6795762404c557352813672f984c7a239e8b09de Mon Sep 17 00:00:00 2001 From: Ariel Conti Date: Tue, 7 Jul 2026 10:49:11 +0100 Subject: [PATCH 10/48] feat: autozoom from PR 26 by lgariv, adapted Port of vercel-labs/webreel#26 onto the fix/type-inserttext branch. Ported: - core autozoom.ts: crop computation, session grouping, spatial sub-grouping, size harmonization, keyframe generation, and ffmpeg zoompan expression with smoothstep easing, plus its test suite - core revealObserver.ts: in-page MutationObserver that unions click-revealed UI (menus, modals) into the zoom target - compositor: ComposeOptions.zoomFilter, three-stage pipeline (cursor overlay, zoompan, HUD overlay), EPIPE-tolerant stdin writer, and the HUD viewBox width clamp - runner: per-step zoom event capture with holdUntilMs for typing spans, reveal collection after postDelay, zoomEvents persisted in the timeline JSON, zoom filter applied during compositing - composite command: rebuilds the zoom filter from persisted zoomEvents so re-compositing preserves autozoom - config: autoZoom on VideoConfig, allowlisted in KNOWN_VIDEO_KEYS - docs: configuration page, JSON schema, SKILL.md, examples.md, both READMEs, and the examples/autozoom demo project Skipped: - the cursor pacing commit (tickDuplicate path advance, doubled moveDuration, waitForPathComplete wait swap), evaluated separately - the package rename to @lgariv/* and its release churn; imports in ported hunks were restored to @webreel/core Added beyond the PR: - shape validation for autoZoom tunables with config tests - minimal waitForPathComplete API on InteractionTimeline, with releaseWaiters extended to flush path-complete waiters so an interrupt mid-cursor-move cannot hang, plus timeline tests Credit: Lavie (lgariv) for the original PR. --- README.md | 1 + apps/docs/public/schema/v1.json | 79 ++++ apps/docs/src/app/configuration/page.mdx | 151 ++++++++ examples/autozoom/README.md | 31 ++ examples/autozoom/videos/autozoom.mp4 | 3 + examples/autozoom/videos/autozoom.png | 3 + examples/autozoom/web/index.html | 268 ++++++++++++++ examples/autozoom/webreel.config.json | 26 ++ .../core/src/__tests__/autozoom.test.ts | 284 +++++++++++++++ .../core/src/__tests__/revealObserver.test.ts | 90 +++++ .../core/src/__tests__/timeline.test.ts | 45 +++ packages/@webreel/core/src/autozoom.ts | 339 ++++++++++++++++++ packages/@webreel/core/src/compositor.ts | 222 ++++++++++-- packages/@webreel/core/src/index.ts | 14 + packages/@webreel/core/src/revealObserver.ts | 146 ++++++++ packages/@webreel/core/src/timeline.ts | 29 +- packages/webreel/README.md | 1 + packages/webreel/src/commands/composite.ts | 29 +- .../webreel/src/lib/__tests__/config.test.ts | 77 ++++ packages/webreel/src/lib/config.ts | 75 ++++ packages/webreel/src/lib/runner.ts | 195 +++++++++- packages/webreel/src/lib/types.ts | 5 +- skills/webreel/SKILL.md | 59 +++ skills/webreel/examples.md | 44 +++ 24 files changed, 2170 insertions(+), 46 deletions(-) create mode 100644 examples/autozoom/README.md create mode 100644 examples/autozoom/videos/autozoom.mp4 create mode 100644 examples/autozoom/videos/autozoom.png create mode 100644 examples/autozoom/web/index.html create mode 100644 examples/autozoom/webreel.config.json create mode 100644 packages/@webreel/core/src/__tests__/autozoom.test.ts create mode 100644 packages/@webreel/core/src/__tests__/revealObserver.test.ts create mode 100644 packages/@webreel/core/src/autozoom.ts create mode 100644 packages/@webreel/core/src/revealObserver.ts diff --git a/README.md b/README.md index 2791e4b..79e357b 100644 --- a/README.md +++ b/README.md @@ -213,6 +213,7 @@ All steps (except `pause`) accept an optional `delay` field (ms to wait after th | `include` | inherited | Array of paths to JSON files whose steps are prepended | | `theme` | inherited | Cursor and HUD overlay customization | | `defaultDelay` | inherited | Default delay (ms) after each step | +| `autoZoom` | `false` | Cinematic zoom into each action (bool or object) | ## Development diff --git a/apps/docs/public/schema/v1.json b/apps/docs/public/schema/v1.json index 992242d..0230177 100644 --- a/apps/docs/public/schema/v1.json +++ b/apps/docs/public/schema/v1.json @@ -216,6 +216,81 @@ }, "additionalProperties": false }, + "autoZoom": { + "oneOf": [ + { + "type": "boolean", + "description": "Enable autozoom with default settings." + }, + { + "type": "object", + "properties": { + "enabled": { + "type": "boolean", + "default": true, + "description": "Turn autozoom on or off without removing the config object." + }, + "approachS": { + "type": "number", + "minimum": 0, + "default": 0.5, + "description": "Seconds spent zooming in from full frame to the target." + }, + "settleBeforeS": { + "type": "number", + "minimum": 0, + "default": 0.15, + "description": "Seconds the camera sits on the target before the action fires." + }, + "holdAfterS": { + "type": "number", + "minimum": 0, + "default": 0.3, + "description": "Seconds the camera holds after the last action in a session." + }, + "releaseS": { + "type": "number", + "minimum": 0, + "default": 0.5, + "description": "Seconds spent zooming out from the target back to full frame." + }, + "paddingRatio": { + "type": "number", + "minimum": 0, + "default": 0.3, + "description": "Fraction of the target bounding box added as padding around it." + }, + "minZoomRatio": { + "type": "number", + "minimum": 0, + "maximum": 1, + "default": 0.6, + "description": "The crop is never smaller than this fraction of the viewport." + }, + "skipZoomRatio": { + "type": "number", + "minimum": 0, + "maximum": 1, + "default": 0.75, + "description": "Skip zooming when the computed crop would be this fraction of the viewport or larger." + }, + "sessionGapS": { + "type": "number", + "minimum": 0, + "default": 4, + "description": "Actions within this many seconds share one zoom session; the camera pans between them instead of zooming out." + }, + "minPanS": { + "type": "number", + "minimum": 0, + "default": 0.8, + "description": "Skip panning to an intermediate target when the pan would be shorter than this many seconds." + } + }, + "additionalProperties": false + } + ] + }, "video": { "type": "object", "required": ["url", "steps"], @@ -297,6 +372,10 @@ "minimum": 0, "description": "Milliseconds the cursor pauses after reaching its target before clicking. Overrides the top-level clickDwell." }, + "autoZoom": { + "$ref": "#/$defs/autoZoom", + "description": "Cinematically zoom into each action during compositing. Set true for defaults or an object to tune timing and thresholds." + }, "steps": { "type": "array", "items": { diff --git a/apps/docs/src/app/configuration/page.mdx b/apps/docs/src/app/configuration/page.mdx index 4f3ccce..0d6e4ac 100644 --- a/apps/docs/src/app/configuration/page.mdx +++ b/apps/docs/src/app/configuration/page.mdx @@ -264,6 +264,15 @@ Each key in the `videos` object is the video name (used as the default output fi inherited Sound effects configuration for click and keystroke sounds + + + autoZoom + + + false + + Cinematic zoom into each action, either a boolean or a config object + steps @@ -625,6 +634,148 @@ Output quality on a scale of 1-100, where 100 is the highest quality and largest } ``` +## autoZoom + +Cinematically zoom into each action so small UI elements stay readable. The camera eases in before a `click`, `type`, `hover`, `moveTo`, `drag`, or `select` step fires, holds through the interaction, then releases back to the full frame. Zooming happens during the existing compositing pass, so it adds no extra recording time. Click targets automatically expand to include UI revealed by the click, such as dropdown menus and modals. + +Enable with defaults: + +```json +{ + "videos": { + "hero": { + "url": "https://example.com", + "autoZoom": true, + "steps": [] + } + } +} +``` + +Or pass an object to tune the behavior: + +```json +{ + "autoZoom": { + "enabled": true, + "approachS": 0.5, + "holdAfterS": 0.3, + "paddingRatio": 0.3, + "sessionGapS": 4.0 + } +} +``` + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
FieldDefaultDescription
+ enabled + + true + Turn autozoom on or off without removing the config object
+ approachS + + 0.5 + Seconds spent zooming in from full frame to the target
+ settleBeforeS + + 0.15 + Seconds the camera sits on the target before the action fires
+ holdAfterS + + 0.3 + Seconds the camera holds after the last action in a session
+ releaseS + + 0.5 + Seconds spent zooming back out to the full frame
+ paddingRatio + + 0.3 + Fraction of the target bounding box added as padding around it
+ minZoomRatio + + 0.6 + The crop is never smaller than this fraction of the viewport
+ skipZoomRatio + + 0.75 + Skip zooming when the crop would be this fraction of the viewport or larger
+ sessionGapS + + 4.0 + + Actions within this many seconds share one zoom session and the camera pans + between them +
+ minPanS + + 0.8 + + Skip panning to an intermediate target when the pan would be shorter than this + many seconds +
+ +Clicks that trigger a page navigation are skipped, since the page content changes unpredictably. Targets that would produce a near-full-frame crop are also skipped to avoid imperceptible zooms. Autozoom pairs well with the static `zoom` field: `zoom` upscales the captured viewport for readability, and `autoZoom` then zooms further into each action. + ## sfx Add click and keystroke sound effects to recorded videos. Set at the top level or per-video. Each key (`click` and `key`) accepts a built-in variant (`1`, `2`, `3`, or `4`) or a path to a custom audio file. diff --git a/examples/autozoom/README.md b/examples/autozoom/README.md new file mode 100644 index 0000000..7159e7f --- /dev/null +++ b/examples/autozoom/README.md @@ -0,0 +1,31 @@ +# Autozoom + +Demonstrates cinematic autozoom that eases into each user action and releases afterwards. Small form fields and buttons benefit the most. On mobile viewports the effect makes details readable. + +## Features demonstrated + +- `autoZoom: true` enables the effect with defaults +- Multiple interactions grouped into zoom "sessions" +- Zoom settles before each click, holds through the action, releases at the end + +## Run + +```bash +cd examples/autozoom +webreel record +``` + +## Tuning + +Pass an object instead of `true` to override defaults: + +```json +{ + "autoZoom": { + "enabled": true, + "approachS": 1.2, + "holdAfterS": 0.8, + "paddingRatio": 0.4 + } +} +``` diff --git a/examples/autozoom/videos/autozoom.mp4 b/examples/autozoom/videos/autozoom.mp4 new file mode 100644 index 0000000..4084d40 --- /dev/null +++ b/examples/autozoom/videos/autozoom.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:88558055f4c9520056976875c4b51657cad6a8855d6e8cc7f6b9c36dccf9d0ed +size 4077700 diff --git a/examples/autozoom/videos/autozoom.png b/examples/autozoom/videos/autozoom.png new file mode 100644 index 0000000..d56eca5 --- /dev/null +++ b/examples/autozoom/videos/autozoom.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:300f2856b405e181902caf31050f684e9e0665cfceada1491787f7d630fd434e +size 102878 diff --git a/examples/autozoom/web/index.html b/examples/autozoom/web/index.html new file mode 100644 index 0000000..538b38d --- /dev/null +++ b/examples/autozoom/web/index.html @@ -0,0 +1,268 @@ + + + + + + Autozoom Demo + + + + +
+

Profile settings

+

Small UI elements that benefit from autozoom.

+
+ + +
+
+ + +
+
+ +
+ +
+ + + +
+
+
+
+ + +
+
+
+ + + diff --git a/examples/autozoom/webreel.config.json b/examples/autozoom/webreel.config.json new file mode 100644 index 0000000..0009b2e --- /dev/null +++ b/examples/autozoom/webreel.config.json @@ -0,0 +1,26 @@ +{ + "$schema": "https://webreel.dev/schema/v1.json", + "videos": { + "autozoom": { + "url": "./web/index.html", + "viewport": { "width": 1920, "height": 1080 }, + "zoom": 2, + "waitFor": ".card", + "autoZoom": true, + "defaultDelay": 500, + "steps": [ + { "action": "pause", "ms": 1500 }, + { "action": "type", "text": "Jane Doe", "selector": "#name", "charDelay": 80 }, + { + "action": "type", + "text": "jane@acme.com", + "selector": "#email", + "charDelay": 80 + }, + { "action": "click", "selector": "#role-btn" }, + { "action": "click", "text": "Engineer" }, + { "action": "click", "selector": "#save", "delay": 1200 } + ] + } + } +} diff --git a/packages/@webreel/core/src/__tests__/autozoom.test.ts b/packages/@webreel/core/src/__tests__/autozoom.test.ts new file mode 100644 index 0000000..0547ca2 --- /dev/null +++ b/packages/@webreel/core/src/__tests__/autozoom.test.ts @@ -0,0 +1,284 @@ +import { describe, it, expect } from "vitest"; +import { + buildAutoZoomFilter, + computeCropForEvent, + generateZoomKeyframes, + unionBboxes, + type AutoZoomConfig, + type ZoomEvent, +} from "../autozoom.js"; + +const DEFAULTS = { + approachS: 0.5, + settleBeforeS: 0.15, + holdAfterS: 0.3, + releaseS: 0.5, + paddingRatio: 0.3, + minZoomRatio: 0.6, + skipZoomRatio: 0.75, + sessionGapS: 4.0, + minPanS: 0.8, +} as const; + +const VIEWPORT = { width: 1920, height: 1080 }; + +describe("unionBboxes", () => { + it("unions two disjoint boxes", () => { + const u = unionBboxes([ + { x: 0, y: 0, width: 10, height: 10 }, + { x: 100, y: 100, width: 20, height: 20 }, + ]); + expect(u).toEqual({ x: 0, y: 0, width: 120, height: 120 }); + }); + + it("returns the outer box when one contains another", () => { + const u = unionBboxes([ + { x: 0, y: 0, width: 100, height: 100 }, + { x: 10, y: 10, width: 20, height: 20 }, + ]); + expect(u).toEqual({ x: 0, y: 0, width: 100, height: 100 }); + }); + + it("returns null for empty input", () => { + expect(unionBboxes([])).toBeNull(); + }); +}); + +describe("computeCropForEvent", () => { + it("enforces minZoomRatio for tiny targets", () => { + const crop = computeCropForEvent( + { x: 100, y: 100, width: 10, height: 10 }, + VIEWPORT, + DEFAULTS, + ); + expect(crop).not.toBeNull(); + expect(crop!.w).toBeGreaterThanOrEqual(VIEWPORT.width * DEFAULTS.minZoomRatio); + expect(crop!.h).toBeGreaterThanOrEqual(VIEWPORT.height * DEFAULTS.minZoomRatio); + }); + + it("returns null when target fills most of viewport (skipZoomRatio)", () => { + const crop = computeCropForEvent( + { x: 0, y: 0, width: VIEWPORT.width, height: VIEWPORT.height }, + VIEWPORT, + DEFAULTS, + ); + expect(crop).toBeNull(); + }); + + it("matches viewport aspect ratio", () => { + const crop = computeCropForEvent( + { x: 200, y: 200, width: 300, height: 300 }, + VIEWPORT, + DEFAULTS, + ); + const aspect = VIEWPORT.width / VIEWPORT.height; + expect(crop).not.toBeNull(); + expect(crop!.w / crop!.h).toBeCloseTo(aspect, 3); + }); + + it("clamps crop to viewport edges", () => { + const crop = computeCropForEvent( + { x: 0, y: 0, width: 20, height: 20 }, + VIEWPORT, + DEFAULTS, + ); + expect(crop).not.toBeNull(); + expect(crop!.x).toBe(0); + expect(crop!.y).toBe(0); + expect(crop!.x + crop!.w).toBeLessThanOrEqual(VIEWPORT.width); + expect(crop!.y + crop!.h).toBeLessThanOrEqual(VIEWPORT.height); + }); +}); + +describe("generateZoomKeyframes", () => { + const cfg: AutoZoomConfig = { enabled: true }; + + it("returns empty when disabled", () => { + const kf = generateZoomKeyframes( + [{ timeMs: 1000, box: { x: 100, y: 100, width: 200, height: 200 } }], + VIEWPORT, + 5, + { enabled: false }, + ); + expect(kf).toEqual([]); + }); + + it("returns empty when no events", () => { + expect(generateZoomKeyframes([], VIEWPORT, 5, cfg)).toEqual([]); + }); + + it("generates approach/settle/hold/release keyframes for a single event", () => { + const events: ZoomEvent[] = [ + { timeMs: 3000, box: { x: 500, y: 400, width: 200, height: 200 } }, + ]; + const kf = generateZoomKeyframes(events, VIEWPORT, 10, cfg); + expect(kf.length).toBeGreaterThanOrEqual(5); + expect(kf[0].timeS).toBe(0); + expect(kf[0].w).toBe(VIEWPORT.width); + expect(kf[kf.length - 1].w).toBe(VIEWPORT.width); + }); + + it("skips a click after a url change (navigation)", () => { + const events: ZoomEvent[] = [ + { + timeMs: 1000, + box: { x: 100, y: 100, width: 200, height: 200 }, + url: "https://a.test/", + }, + { + timeMs: 3000, + box: { x: 500, y: 500, width: 200, height: 200 }, + url: "https://b.test/", + }, + ]; + const kfBoth = generateZoomKeyframes(events, VIEWPORT, 10, cfg); + const kfFirstOnly = generateZoomKeyframes([events[0]], VIEWPORT, 10, cfg); + expect(kfBoth.length).toBe(kfFirstOnly.length); + }); + + it("merges events within sessionGapS into one session (no release between)", () => { + const events: ZoomEvent[] = [ + { timeMs: 2000, box: { x: 100, y: 100, width: 200, height: 200 } }, + { timeMs: 3000, box: { x: 900, y: 500, width: 200, height: 200 } }, + ]; + const kf = generateZoomKeyframes(events, VIEWPORT, 10, cfg); + const zoomedOuts = kf.filter((k) => k.w === VIEWPORT.width); + // Two events, 1s gap (< sessionGapS=1.2) → single session → 3 full-view + // keyframes (initial, approach start, final release). + expect(zoomedOuts.length).toBe(3); + }); + + it("starts a new session when gap exceeds sessionGapS", () => { + const events: ZoomEvent[] = [ + { timeMs: 1000, box: { x: 100, y: 100, width: 200, height: 200 } }, + { timeMs: 9000, box: { x: 900, y: 500, width: 200, height: 200 } }, + ]; + const kf = generateZoomKeyframes(events, VIEWPORT, 15, cfg); + const zoomedOuts = kf.filter((k) => k.w === VIEWPORT.width); + // Two separate sessions → 5 full-view keyframes (each session adds + // approach-start + release, plus one shared initial at t=0). + expect(zoomedOuts.length).toBe(5); + }); + + it("emits monotonically-increasing keyframe times", () => { + const events: ZoomEvent[] = [ + { timeMs: 1000, box: { x: 100, y: 100, width: 200, height: 200 } }, + { timeMs: 2000, box: { x: 900, y: 500, width: 200, height: 200 } }, + { timeMs: 8000, box: { x: 400, y: 300, width: 200, height: 200 } }, + ]; + const kf = generateZoomKeyframes(events, VIEWPORT, 20, cfg); + for (let i = 1; i < kf.length; i++) { + expect(kf[i].timeS).toBeGreaterThanOrEqual(kf[i - 1].timeS); + } + }); + + it("keeps sessions separate across medium gaps (no mergeBuffer)", () => { + // 6s gap: clearly beyond sessionGapS=4.0, so the sessions must NOT merge. + // We want the camera to fully release between unrelated events. + const events: ZoomEvent[] = [ + { timeMs: 1000, box: { x: 100, y: 100, width: 200, height: 200 } }, + { timeMs: 7000, box: { x: 900, y: 500, width: 200, height: 200 } }, + ]; + const kf = generateZoomKeyframes(events, VIEWPORT, 12, cfg); + const zoomedOuts = kf.filter((k) => k.w === VIEWPORT.width); + expect(zoomedOuts.length).toBe(5); + }); + + it("skips intermediate targets whose pan would be shorter than minPanS", () => { + // Three events: A (t=2), B (t=2.5, 0.5s gap — below minPanS=0.8 after + // hold), C (t=5, separate session). B gets dropped; A and C each form + // their own session with distinct crops. + const events: ZoomEvent[] = [ + { timeMs: 2000, box: { x: 200, y: 100, width: 200, height: 50 } }, + { timeMs: 2500, box: { x: 200, y: 300, width: 200, height: 50 } }, + { timeMs: 5000, box: { x: 1000, y: 600, width: 200, height: 50 } }, + ]; + const kf = generateZoomKeyframes(events, VIEWPORT, 10, cfg); + const distinctCrops = new Set( + kf.map((k) => `${k.x.toFixed(0)},${k.y.toFixed(0)},${k.w.toFixed(0)}`), + ); + expect(distinctCrops.size).toBe(3); + }); + + it("uses last-kept crop for release when the final event is skipped by minPanS", () => { + // Session with two events close together: B (t=4.5) and C (t=4.9, 0.4s + // gap). C's pan is too short (< minPanS) so it's skipped; release should + // use B's crop, not C's. Event A at t=2 is in its own earlier session. + const events: ZoomEvent[] = [ + { timeMs: 2000, box: { x: 100, y: 100, width: 200, height: 50 } }, + { timeMs: 4500, box: { x: 800, y: 400, width: 200, height: 50 } }, + { timeMs: 4900, box: { x: 1500, y: 900, width: 200, height: 50 } }, + ]; + const kf = generateZoomKeyframes(events, VIEWPORT, 8, cfg); + const lastCropKf = [...kf].reverse().find((k) => k.w !== VIEWPORT.width)!; + // B is centered around x=800; C around x=1500. The last kept crop should + // be positioned for B, not C — its left edge should be well under 1000. + expect(lastCropKf.x).toBeLessThan(1000); + }); + + it("harmonizes crop size within a session so zoom level stays constant", () => { + // A big target and a small target in the same session → small one should + // inherit the big one's crop size (so the zoom doesn't jump). + const events: ZoomEvent[] = [ + { timeMs: 1000, box: { x: 100, y: 100, width: 800, height: 400 } }, + { timeMs: 2000, box: { x: 1500, y: 800, width: 80, height: 30 } }, + ]; + const kf = generateZoomKeyframes(events, VIEWPORT, 5, cfg); + const sessionCrops = kf.filter((k) => k.w !== VIEWPORT.width); + const widths = new Set(sessionCrops.map((k) => k.w)); + expect(widths.size).toBe(1); + }); +}); + +describe("buildAutoZoomFilter", () => { + it("returns null when disabled", () => { + expect(buildAutoZoomFilter([], VIEWPORT, 1, 5, 60, { enabled: false })).toBeNull(); + }); + + it("returns null when no events produce keyframes", () => { + expect(buildAutoZoomFilter([], VIEWPORT, 1, 5, 60, { enabled: true })).toBeNull(); + }); + + it("emits a zoompan filter string with expected params", () => { + const events: ZoomEvent[] = [ + { timeMs: 2000, box: { x: 200, y: 200, width: 200, height: 200 } }, + ]; + const filter = buildAutoZoomFilter(events, VIEWPORT, 1, 10, 60, { + enabled: true, + }); + expect(filter).not.toBeNull(); + expect(filter!).toMatch(/^zoompan=z='/); + expect(filter!).toContain(`s=${VIEWPORT.width}x${VIEWPORT.height}`); + expect(filter!).toContain("fps=60"); + }); + + it("emits smoothstep easing (register-based) in the filter expression", () => { + const events: ZoomEvent[] = [ + { timeMs: 2000, box: { x: 200, y: 200, width: 200, height: 200 } }, + ]; + const filter = buildAutoZoomFilter(events, VIEWPORT, 1, 10, 60, { + enabled: true, + }); + expect(filter).not.toBeNull(); + // st(0, progress) and st(1, smoothstepped) and ld(1) usage + expect(filter!).toContain("st(0,"); + expect(filter!).toContain("st(1,ld(0)*ld(0)*(3-2*ld(0)))"); + expect(filter!).toContain("*ld(1)"); + }); + + it("scales event boxes by cssZoom before keyframe generation", () => { + // Box big enough that the minZoomRatio floor doesn't clip both outputs to + // the same crop — otherwise cssZoom=1 and cssZoom=2 both bottom out at the + // minimum and look identical. + const events: ZoomEvent[] = [ + { timeMs: 2000, box: { x: 100, y: 100, width: 500, height: 300 } }, + ]; + const filterAt1 = buildAutoZoomFilter(events, VIEWPORT, 1, 10, 60, { + enabled: true, + }); + const filterAt2 = buildAutoZoomFilter(events, VIEWPORT, 2, 10, 60, { + enabled: true, + }); + expect(filterAt1).not.toBe(filterAt2); + }); +}); diff --git a/packages/@webreel/core/src/__tests__/revealObserver.test.ts b/packages/@webreel/core/src/__tests__/revealObserver.test.ts new file mode 100644 index 0000000..4803b90 --- /dev/null +++ b/packages/@webreel/core/src/__tests__/revealObserver.test.ts @@ -0,0 +1,90 @@ +import { describe, it, expect } from "vitest"; +import { installRevealObserver, collectReveals, __TESTING__ } from "../revealObserver.js"; + +interface Call { + expression: string; +} + +function createMockClient( + response: { value?: unknown } | (() => { value?: unknown }) | Error, +) { + const calls: Call[] = []; + const client = { + Runtime: { + evaluate: async (params: { expression: string }) => { + calls.push({ expression: params.expression }); + if (response instanceof Error) throw response; + const r = typeof response === "function" ? response() : response; + return { result: r }; + }, + }, + } as never; + return { client, calls }; +} + +describe("installRevealObserver", () => { + it("returns a handle when the page returns a number", async () => { + const { client } = createMockClient({ value: 42 }); + const handle = await installRevealObserver(client); + expect(handle).toEqual({ id: 42 }); + }); + + it("returns null when the page returns a non-number", async () => { + const { client } = createMockClient({ value: null }); + const handle = await installRevealObserver(client); + expect(handle).toBeNull(); + }); + + it("returns null when Runtime.evaluate throws", async () => { + const { client } = createMockClient(new Error("cdp failed")); + const handle = await installRevealObserver(client); + expect(handle).toBeNull(); + }); + + it("evaluates the MutationObserver install IIFE", async () => { + const { client, calls } = createMockClient({ value: 1 }); + await installRevealObserver(client); + expect(calls).toHaveLength(1); + expect(calls[0].expression).toContain("MutationObserver"); + expect(calls[0].expression).toContain("__wrReveals"); + expect(calls[0].expression).toContain("observer.observe"); + }); +}); + +describe("collectReveals", () => { + it("returns the array of bounding boxes the page yields", async () => { + const boxes = [{ x: 10, y: 20, width: 100, height: 50 }]; + const { client, calls } = createMockClient({ value: boxes }); + const result = await collectReveals(client, { id: 7 }); + expect(result).toEqual(boxes); + // The collect expression must reference the handle id so the page + // script finds the right observer state. + expect(calls[0].expression).toContain("(7)"); + expect(calls[0].expression).toContain("disconnect"); + expect(calls[0].expression).toContain("getBoundingClientRect"); + }); + + it("returns empty array when the page returns non-array", async () => { + const { client } = createMockClient({ value: "oops" }); + const result = await collectReveals(client, { id: 1 }); + expect(result).toEqual([]); + }); + + it("returns empty array when Runtime.evaluate throws", async () => { + const { client } = createMockClient(new Error("cdp failed")); + const result = await collectReveals(client, { id: 1 }); + expect(result).toEqual([]); + }); +}); + +describe("reveal scripts (sanity check on string contents)", () => { + it("INSTALL_SCRIPT filters overlay elements and tracks visibility", () => { + expect(__TESTING__.INSTALL_SCRIPT).toContain("preVisible"); + expect(__TESTING__.INSTALL_SCRIPT).toContain("attributeFilter"); + }); + + it("COLLECT_SCRIPT filters __demo- and tiny mutations", () => { + expect(__TESTING__.COLLECT_SCRIPT).toContain("__demo-"); + expect(__TESTING__.COLLECT_SCRIPT).toContain("MIN_AREA"); + }); +}); diff --git a/packages/@webreel/core/src/__tests__/timeline.test.ts b/packages/@webreel/core/src/__tests__/timeline.test.ts index fbe2a7d..0d2caab 100644 --- a/packages/@webreel/core/src/__tests__/timeline.test.ts +++ b/packages/@webreel/core/src/__tests__/timeline.test.ts @@ -149,4 +149,49 @@ describe("InteractionTimeline", () => { tl.releaseWaiters(); await expect(tl.waitForNextTick()).resolves.toBeUndefined(); }); + + it("waitForPathComplete resolves immediately when no path is active", async () => { + const tl = new InteractionTimeline(1080, 1080); + await expect(tl.waitForPathComplete()).resolves.toBeUndefined(); + }); + + it("waitForPathComplete resolves once ticks consume the path", async () => { + const tl = new InteractionTimeline(1080, 1080); + tl.setCursorPath([ + { x: 10, y: 10 }, + { x: 20, y: 20 }, + ]); + let resolved = false; + const wait = tl.waitForPathComplete().then(() => { + resolved = true; + }); + tl.tick(); + await Promise.resolve(); + expect(resolved).toBe(false); + tl.tick(); + await wait; + expect(resolved).toBe(true); + }); + + it("releaseWaiters resolves pending path-complete waiters", async () => { + const tl = new InteractionTimeline(1080, 1080); + tl.setCursorPath([ + { x: 10, y: 10 }, + { x: 20, y: 20 }, + ]); + const wait = tl.waitForPathComplete(); + tl.releaseWaiters(); + await expect(wait).resolves.toBeUndefined(); + }); + + it("waitForPathComplete resolves immediately after release even mid-path", async () => { + const tl = new InteractionTimeline(1080, 1080); + tl.setCursorPath([ + { x: 10, y: 10 }, + { x: 20, y: 20 }, + ]); + tl.tick(); + tl.releaseWaiters(); + await expect(tl.waitForPathComplete()).resolves.toBeUndefined(); + }); }); diff --git a/packages/@webreel/core/src/autozoom.ts b/packages/@webreel/core/src/autozoom.ts new file mode 100644 index 0000000..c9d5ee5 --- /dev/null +++ b/packages/@webreel/core/src/autozoom.ts @@ -0,0 +1,339 @@ +import type { BoundingBox } from "./types.js"; + +export interface AutoZoomConfig { + enabled: boolean; + approachS?: number; + settleBeforeS?: number; + holdAfterS?: number; + releaseS?: number; + paddingRatio?: number; + minZoomRatio?: number; + skipZoomRatio?: number; + sessionGapS?: number; + minPanS?: number; +} + +export interface ZoomEvent { + timeMs: number; + box: BoundingBox; + url?: string; + // Optional: extend the camera hold until at least this time. Used for type + // actions where `timeMs` anchors on the input click (so the camera arrives + // in time) but the hold must cover the typing span, which ends later. + holdUntilMs?: number; +} + +export interface ZoomKeyframe { + timeS: number; + x: number; + y: number; + w: number; + h: number; +} + +type ResolvedConfig = { + approachS: number; + settleBeforeS: number; + holdAfterS: number; + releaseS: number; + paddingRatio: number; + minZoomRatio: number; + skipZoomRatio: number; + sessionGapS: number; + minPanS: number; +}; + +const DEFAULTS: ResolvedConfig = { + approachS: 0.5, + settleBeforeS: 0.15, + holdAfterS: 0.3, + releaseS: 0.5, + paddingRatio: 0.3, + minZoomRatio: 0.6, + skipZoomRatio: 0.75, + sessionGapS: 4.0, + minPanS: 0.8, +}; + +export function unionBboxes(boxes: BoundingBox[]): BoundingBox | null { + if (boxes.length === 0) return null; + let minX = Infinity; + let minY = Infinity; + let maxX = -Infinity; + let maxY = -Infinity; + for (const b of boxes) { + if (b.x < minX) minX = b.x; + if (b.y < minY) minY = b.y; + if (b.x + b.width > maxX) maxX = b.x + b.width; + if (b.y + b.height > maxY) maxY = b.y + b.height; + } + return { x: minX, y: minY, width: maxX - minX, height: maxY - minY }; +} + +function centerCropWithin( + cx: number, + cy: number, + w: number, + h: number, + viewport: { width: number; height: number }, +): { x: number; y: number; w: number; h: number } { + return { + w, + h, + x: Math.max(0, Math.min(viewport.width - w, cx - w / 2)), + y: Math.max(0, Math.min(viewport.height - h, cy - h / 2)), + }; +} + +export function computeCropForEvent( + box: BoundingBox, + viewport: { width: number; height: number }, + cfg: ResolvedConfig, +): { x: number; y: number; w: number; h: number } | null { + let w = box.width * (1 + 2 * cfg.paddingRatio); + let h = box.height * (1 + 2 * cfg.paddingRatio); + + w = Math.max(w, viewport.width * cfg.minZoomRatio); + h = Math.max(h, viewport.height * cfg.minZoomRatio); + + const aspect = viewport.width / viewport.height; + if (w / h > aspect) h = w / aspect; + else w = h * aspect; + + w = Math.min(w, viewport.width); + h = Math.min(h, viewport.height); + + if ( + w >= viewport.width * cfg.skipZoomRatio && + h >= viewport.height * cfg.skipZoomRatio + ) { + return null; + } + + const cx = box.x + box.width / 2; + const cy = box.y + box.height / 2; + return centerCropWithin(cx, cy, w, h, viewport); +} + +export function generateZoomKeyframes( + events: ZoomEvent[], + viewport: { width: number; height: number }, + durationS: number, + userCfg: AutoZoomConfig, +): ZoomKeyframe[] { + if (!userCfg.enabled) return []; + const cfg: ResolvedConfig = { ...DEFAULTS, ...userCfg }; + + interface Target { + timeS: number; + holdUntilS: number; + box: BoundingBox; + crop: { x: number; y: number; w: number; h: number }; + } + + const targets: Target[] = []; + let prevUrl: string | undefined; + for (const e of events) { + if (e.url && prevUrl && e.url !== prevUrl) { + prevUrl = e.url; + continue; + } + const crop = computeCropForEvent(e.box, viewport, cfg); + prevUrl = e.url ?? prevUrl; + if (!crop) continue; + const timeS = e.timeMs / 1000; + const holdUntilS = Math.max(timeS, (e.holdUntilMs ?? e.timeMs) / 1000); + targets.push({ timeS, holdUntilS, box: e.box, crop }); + } + + if (targets.length === 0) return []; + + const full = { x: 0, y: 0, w: viewport.width, h: viewport.height }; + + // Group targets by sessionGapS. Events within a short gap (default 1.2s — + // button-click → modal case) form one session; anything further apart gets + // its own pulse so the camera releases all the way back to wide between. + const sessions: Target[][] = [[targets[0]]]; + for (let i = 1; i < targets.length; i++) { + const prev = targets[i - 1]; + const curr = targets[i]; + if (curr.timeS - prev.timeS <= cfg.sessionGapS) { + sessions[sessions.length - 1].push(curr); + } else { + sessions.push(curr === prev ? [] : [curr]); + } + } + + // Spatial sub-grouping within a session. We greedily extend the current + // sub-group with each next target as long as the union bbox of the + // sub-group still fits into a crop (doesn't trigger skipZoomRatio). If it + // won't fit, we close the current sub-group and start a new one with this + // target. Each sub-group shares ONE union crop, so the camera holds stable + // within a sub-group and pans between sub-groups. This matches Cursor: + // button+menu or trigger+modal stay in one frame; distinct regions of the + // page get distinct zoom positions. + type Group = { targets: Target[]; crop: Target["crop"] }; + for (const session of sessions) { + if (session.length < 2) continue; + const first = session[0]; + const groups: Group[] = [{ targets: [first], crop: first.crop }]; + for (let i = 1; i < session.length; i++) { + const curr = session[i]; + const last = groups[groups.length - 1]; + const candidateBoxes = last.targets.map((t) => t.box).concat([curr.box]); + const candidateUnion = unionBboxes(candidateBoxes); + const candidateCrop = candidateUnion + ? computeCropForEvent(candidateUnion, viewport, cfg) + : null; + if (candidateCrop) { + last.targets.push(curr); + last.crop = candidateCrop; + } else { + groups.push({ targets: [curr], crop: curr.crop }); + } + } + for (const g of groups) for (const t of g.targets) t.crop = g.crop; + + // Size-harmonize across sub-groups so crop zoom stays constant while the + // camera pans between them. + let maxW = 0; + let maxH = 0; + for (const t of session) { + if (t.crop.w > maxW) maxW = t.crop.w; + if (t.crop.h > maxH) maxH = t.crop.h; + } + for (const t of session) { + if (t.crop.w === maxW && t.crop.h === maxH) continue; + const cx = t.crop.x + t.crop.w / 2; + const cy = t.crop.y + t.crop.h / 2; + t.crop = centerCropWithin(cx, cy, maxW, maxH, viewport); + } + } + + const kf: ZoomKeyframe[] = [{ timeS: 0, ...full }]; + for (const session of sessions) { + const first = session[0]; + const actualLast = session[session.length - 1]; + + const settleTime = first.timeS - cfg.settleBeforeS; + const approachStart = Math.max(0, settleTime - cfg.approachS); + kf.push({ timeS: approachStart, ...full }); + kf.push({ timeS: Math.max(0, settleTime), ...first.crop }); + + // Track the last target we actually panned to. Skip intermediate targets + // whose pan duration would be shorter than cfg.minPanS — blink-and-miss + // transitions that add visual noise without being readable. Also skip + // when the next target's crop is identical to the current one (the + // union-crop case): no pan to emit, just keep tracking lastKept so the + // release time extends to cover the session's full span. + let lastKept = first; + for (let i = 1; i < session.length; i++) { + const curr = session[i]; + const sameCrop = + curr.crop.x === lastKept.crop.x && + curr.crop.y === lastKept.crop.y && + curr.crop.w === lastKept.crop.w && + curr.crop.h === lastKept.crop.h; + if (sameCrop) { + lastKept = curr; + continue; + } + const gap = curr.timeS - lastKept.timeS; + const arriveBy = curr.timeS - cfg.settleBeforeS; + const holdEnd = Math.min(lastKept.timeS + gap * 0.4, arriveBy - 0.1); + const panDuration = arriveBy - holdEnd; + if (panDuration < cfg.minPanS) continue; + + if (holdEnd > lastKept.timeS + 0.05) { + kf.push({ timeS: holdEnd, ...lastKept.crop }); + } + kf.push({ timeS: Math.max(holdEnd + 0.05, arriveBy), ...curr.crop }); + lastKept = curr; + } + + // Release uses the session's true last event for TIMING (so hold is + // sized correctly) but the last position we actually panned to for + // the CROP (otherwise release would teleport to a never-visited spot). + // holdUntilS carries a per-event hold extension (e.g. typing spans from + // click to last keystroke — the camera should stay on the field until + // typing actually ends, not just for holdAfterS after the click). + const holdEnd = actualLast.holdUntilS + cfg.holdAfterS; + const releaseEnd = holdEnd + cfg.releaseS; + kf.push({ timeS: holdEnd, ...lastKept.crop }); + kf.push({ timeS: releaseEnd, ...full }); + } + + return kf; +} + +export function buildAutoZoomFilter( + events: ZoomEvent[], + viewport: { width: number; height: number }, + cssZoom: number, + durationS: number, + fps: number, + userCfg: AutoZoomConfig, +): string | null { + if (!userCfg.enabled || events.length === 0) return null; + + const scaled: ZoomEvent[] = events.map((e) => ({ + ...e, + box: { + x: e.box.x * cssZoom, + y: e.box.y * cssZoom, + width: e.box.width * cssZoom, + height: e.box.height * cssZoom, + }, + })); + + const kf = generateZoomKeyframes(scaled, viewport, durationS, userCfg); + if (kf.length < 2) return null; + + if (process.env.WEBREEL_DEBUG_ZOOM) { + for (const k of kf) { + const z = (viewport.width / Math.max(1, k.w)).toFixed(2); + console.error( + `kf t=${k.timeS.toFixed(2)}s z=${z}x crop=${k.x.toFixed(0)},${k.y.toFixed(0)} ${k.w.toFixed(0)}×${k.h.toFixed(0)}`, + ); + } + } + + const zExpr = easedBetweens(kf, (k) => viewport.width / Math.max(1, k.w)); + const xExpr = easedBetweens(kf, (k) => k.x); + const yExpr = easedBetweens(kf, (k) => k.y); + return `zoompan=z='${zExpr}':x='${xExpr}':y='${yExpr}':d=1:s=${viewport.width}x${viewport.height}:fps=${fps}`; +} + +// Smoothstep easing between keyframes: p² × (3 − 2p). Uses FFmpeg expression +// register slots to compute the eased progress once per frame, then lerp at +// the eased position. Each segment is a single smoothstep (velocity zero at +// both ends) — since we no longer insert mid-motion waypoints, every segment +// is a standalone motion from a hold to the next hold, which is the shape +// smoothstep handles best. +function easedBetweens(kf: ZoomKeyframe[], val: (k: ZoomKeyframe) => number): string { + if (kf.length === 1) return val(kf[0]).toFixed(4); + let expr = ""; + let segments = 0; + for (let i = 0; i < kf.length - 1; i++) { + const a = kf[i]; + const b = kf[i + 1]; + if (b.timeS === a.timeS) continue; + const va = val(a); + const vb = val(b); + const dt = b.timeS - a.timeS; + const t0 = a.timeS.toFixed(3); + const t1 = b.timeS.toFixed(3); + const dtS = dt.toFixed(3); + const v0 = va.toFixed(4); + const delta = (vb - va).toFixed(4); + const seg = + `if(between(in_time,${t0},${t1}),` + + `0*st(0,(in_time-${t0})/${dtS})+` + + `0*st(1,ld(0)*ld(0)*(3-2*ld(0)))+` + + `${v0}+(${delta})*ld(1)`; + expr = expr ? `${expr},${seg}` : seg; + segments++; + } + const tail = val(kf[kf.length - 1]).toFixed(4); + return `${expr},${tail}${")".repeat(segments)}`; +} diff --git a/packages/@webreel/core/src/compositor.ts b/packages/@webreel/core/src/compositor.ts index c7d2009..f21cceb 100644 --- a/packages/@webreel/core/src/compositor.ts +++ b/packages/@webreel/core/src/compositor.ts @@ -21,6 +21,7 @@ interface OverlayContext { export interface ComposeOptions { sfx?: SfxConfig; crf?: number; + zoomFilter?: string; } export async function compose( @@ -32,6 +33,7 @@ export async function compose( const ffmpegPath = await ensureFfmpeg(); const sfx = options?.sfx; const crf = options?.crf ?? 18; + const zoomFilter = options?.zoomFilter; const zoom = timelineData.zoom ?? 1; const cursorPng = await renderCursorPng( @@ -41,61 +43,131 @@ export async function compose( ); const ext = extname(outputPath).toLowerCase(); + const { width, fps } = timelineData; - if (ext === ".gif") { - const gifConfig = buildGifConfig(timelineData.width, outputPath); - await compositeFrames( - ffmpegPath, - cleanVideoPath, - timelineData, - cursorPng, - zoom, - gifConfig, - ); + if (!zoomFilter) { + if (ext === ".gif") { + const gifConfig = buildGifConfig(width, outputPath); + await compositeFrames( + ffmpegPath, + cleanVideoPath, + timelineData, + cursorPng, + zoom, + gifConfig, + "both", + ); + return; + } + + const workDir = resolve(homedir(), ".webreel"); + mkdirSync(workDir, { recursive: true }); + const tempComposed = resolve(workDir, `_composed_${Date.now()}.mp4`); + + try { + const mp4Config = buildMp4Config(fps, crf, tempComposed); + await compositeFrames( + ffmpegPath, + cleanVideoPath, + timelineData, + cursorPng, + zoom, + mp4Config, + "both", + ); + finalizeComposed(ffmpegPath, tempComposed, outputPath, timelineData, sfx); + } finally { + rmSync(tempComposed, { force: true }); + } return; } + // Autozoom pipeline layering: + // Stage A overlays cursor-only (not HUD) on the raw video. Stage B + // applies zoompan on the cursor-overlaid intermediate. Stage C overlays + // HUD on the zoomed frame. HUD stays at the final viewport coordinates + // regardless of how the camera crops/scales, so captions never get + // cropped by zoom. + // + // Why three stages instead of one? (1) zoompan + image2pipe in the same + // filter_complex deadlocks when the pipe reader can't drain fast enough. + // (2) HUD on top of the zoomed frame must run after zoompan or it gets + // cropped out of the camera window. const workDir = resolve(homedir(), ".webreel"); mkdirSync(workDir, { recursive: true }); + const cursorStagePath = resolve(workDir, `_cursor_${Date.now()}.mp4`); + const zoomStagePath = resolve(workDir, `_zoom_${Date.now()}.mp4`); const tempComposed = resolve(workDir, `_composed_${Date.now()}.mp4`); try { - const mp4Config = buildMp4Config(timelineData.fps, crf, tempComposed); await compositeFrames( ffmpegPath, cleanVideoPath, timelineData, cursorPng, zoom, - mp4Config, + buildMp4Config(fps, crf, cursorStagePath), + "cursor", ); + await applyZoomPass(ffmpegPath, cursorStagePath, zoomFilter, zoomStagePath, crf, fps); - const durationSec = timelineData.frames.length / timelineData.fps; - - if (ext === ".webm") { - finalizeWebm( - ffmpegPath, - tempComposed, - outputPath, - timelineData.events, - durationSec, - sfx, - ); - } else { - finalizeMp4( + if (ext === ".gif") { + await compositeFrames( ffmpegPath, - tempComposed, - outputPath, - timelineData.events, - durationSec, - { remux: true, sfx }, + zoomStagePath, + timelineData, + cursorPng, + zoom, + buildGifConfig(width, outputPath), + "hud", ); + return; } + + await compositeFrames( + ffmpegPath, + zoomStagePath, + timelineData, + cursorPng, + zoom, + buildMp4Config(fps, crf, tempComposed), + "hud", + ); + finalizeComposed(ffmpegPath, tempComposed, outputPath, timelineData, sfx); } finally { + rmSync(cursorStagePath, { force: true }); + rmSync(zoomStagePath, { force: true }); rmSync(tempComposed, { force: true }); } } +function finalizeComposed( + ffmpegPath: string, + tempComposed: string, + outputPath: string, + timelineData: TimelineData, + sfx: SfxConfig | undefined, +): void { + const ext = extname(outputPath).toLowerCase(); + const durationSec = timelineData.frames.length / timelineData.fps; + + if (ext === ".webm") { + finalizeWebm( + ffmpegPath, + tempComposed, + outputPath, + timelineData.events, + durationSec, + sfx, + ); + } else { + finalizeMp4(ffmpegPath, tempComposed, outputPath, timelineData.events, durationSec, { + remux: true, + sfx, + }); + } +} + async function renderCursorPng( svgContent: string, size: number, @@ -162,11 +234,12 @@ function buildGifConfig(width: number, outputPath: string): CompositorFfmpegConf async function compositeFrames( ffmpegPath: string, - cleanVideoPath: string, + inputVideoPath: string, timeline: TimelineData, cursorPng: Buffer, zoom: number, config: CompositorFfmpegConfig, + layer: OverlayLayer, ): Promise { const { width, height, fps } = timeline; @@ -175,7 +248,7 @@ async function compositeFrames( [ "-y", "-i", - cleanVideoPath, + inputVideoPath, "-f", "image2pipe", "-framerate", @@ -246,7 +319,7 @@ async function compositeFrames( const stderr = Buffer.concat(stderrChunks).toString().slice(-2000); rejectAll( new Error( - `Compositor ffmpeg exited with code ${code}${stderr ? `:\n${stderr}` : ""}`, + `Compositor ffmpeg (layer=${layer}) exited with code ${code}${stderr ? `:\n${stderr}` : ""}`, ), ); } @@ -339,6 +412,7 @@ async function compositeFrames( ctx, overlayCache, hudCache, + layer, ); if (state.abortError) break; @@ -363,6 +437,67 @@ async function compositeFrames( await ffmpegDone; } +async function applyZoomPass( + ffmpegPath: string, + inputPath: string, + zoomFilter: string, + outputPath: string, + crf: number, + fps: number, +): Promise { + const ffmpeg = spawn( + ffmpegPath, + [ + "-y", + "-i", + inputPath, + "-vf", + zoomFilter, + "-c:v", + "libx264", + "-preset", + "ultrafast", + "-crf", + String(crf), + "-pix_fmt", + "yuv420p", + "-color_primaries", + "bt709", + "-color_trc", + "bt709", + "-colorspace", + "bt709", + "-movflags", + "+faststart", + "-r", + String(fps), + outputPath, + ], + { stdio: ["ignore", "pipe", "pipe"] }, + ); + + const stderrChunks: Buffer[] = []; + ffmpeg.stderr?.on("data", (chunk: Buffer) => stderrChunks.push(chunk)); + + await new Promise((resolveAll, rejectAll) => { + ffmpeg.on("close", (code) => { + if (code === 0) { + resolveAll(); + } else { + const stderr = Buffer.concat(stderrChunks).toString().slice(-2000); + rejectAll( + new Error( + `Zoom-pass ffmpeg exited with code ${code}${stderr ? `:\n${stderr}` : ""}`, + ), + ); + } + }); + ffmpeg.on("error", rejectAll); + }); +} + +type OverlayLayer = "both" | "cursor" | "hud"; + async function renderOverlayFrame( frame: TimelineData["frames"][number], width: number, @@ -370,6 +505,7 @@ async function renderOverlayFrame( ctx: OverlayContext, cache: Map, hudCache: Map, + layer: OverlayLayer, ): Promise { // Whole-pixel rounding is intentional: sub-pixel precision defeats the // overlay cache during cursor dwell/pause (float jitter creates unique keys). @@ -379,7 +515,7 @@ async function renderOverlayFrame( const cy = Math.round(frame.cursor.y * ctx.zoom); const scale = frame.cursor.scale; const hudKey = frame.hud ? frame.hud.labels.join("|") : ""; - const cacheKey = `${cx},${cy},${scale},${hudKey}`; + const cacheKey = `${layer}:${cx},${cy},${scale},${hudKey}`; const cached = cache.get(cacheKey); if (cached) return cached; @@ -391,7 +527,10 @@ async function renderOverlayFrame( const cursorVisible = icx >= -ctx.cursorWidth && icx < width && icy >= -ctx.cursorHeight && icy < height; - if (cursorVisible) { + const wantCursor = layer !== "hud"; + const wantHud = layer !== "cursor"; + + if (wantCursor && cursorVisible) { const cursorImg = scale !== 1 ? await ctx.getScaledCursor(scale) : ctx.cursorPng; const left = Math.max(0, icx); const top = Math.max(0, icy); @@ -401,7 +540,7 @@ async function renderOverlayFrame( } } - if (frame.hud && frame.hud.labels.length > 0) { + if (wantHud && frame.hud && frame.hud.labels.length > 0) { const hudOverlay = await renderHudOverlay( frame.hud.labels, width, @@ -485,7 +624,13 @@ async function renderHudOverlay( .replace(//g, ">"); - const svgOverlay = ` + // Clamp the rendered HUD to the viewport (with margins) via viewBox so + // long label sets scale down instead of overflowing the frame. + const margin = Math.round(48 * zoom); + const maxHudWidth = Math.max(100, viewportWidth - margin * 2); + const renderedHudWidth = Math.min(hudWidth, maxHudWidth); + + const svgOverlay = ` `; const hudPng = await sharp(Buffer.from(svgOverlay)).png().toBuffer(); - const left = Math.round((viewportWidth - hudWidth) / 2); - const margin = Math.round(48 * zoom); + const left = Math.round((viewportWidth - renderedHudWidth) / 2); const top = hudConfig.position === "top" ? margin : viewportHeight - hudHeight - margin; const result: sharp.OverlayOptions = { input: hudPng, left, top }; diff --git a/packages/@webreel/core/src/index.ts b/packages/@webreel/core/src/index.ts index 95d3f58..c7d78d6 100644 --- a/packages/@webreel/core/src/index.ts +++ b/packages/@webreel/core/src/index.ts @@ -42,3 +42,17 @@ export { compose, type ComposeOptions } from "./compositor.js"; export { ensureFfmpeg, FFMPEG_CACHE_DIR } from "./ffmpeg.js"; export { extractThumbnail, type SfxConfig } from "./media.js"; export { moveFileSync } from "./fs.js"; +export { + buildAutoZoomFilter, + computeCropForEvent, + generateZoomKeyframes, + unionBboxes, + type AutoZoomConfig, + type ZoomEvent, + type ZoomKeyframe, +} from "./autozoom.js"; +export { + installRevealObserver, + collectReveals, + type RevealObserverHandle, +} from "./revealObserver.js"; diff --git a/packages/@webreel/core/src/revealObserver.ts b/packages/@webreel/core/src/revealObserver.ts new file mode 100644 index 0000000..52e0c3a --- /dev/null +++ b/packages/@webreel/core/src/revealObserver.ts @@ -0,0 +1,146 @@ +import type { BoundingBox, CDPClient } from "./types.js"; + +export interface RevealObserverHandle { + id: number; +} + +// Installed BEFORE an interaction fires. Snapshots which elements are +// currently visible so we can later tell what's newly revealed, then hooks a +// MutationObserver to the document body subtree. Returns a numeric id used +// to identify this observer when we collect. +const INSTALL_SCRIPT = `(() => { + if (!window.__wrReveals) window.__wrReveals = {}; + if (!window.__wrRevealsNextId) window.__wrRevealsNextId = 0; + const id = ++window.__wrRevealsNextId; + const preVisible = new WeakSet(); + const preBounds = new WeakMap(); + // An element is "pre-visible" only if it's BOTH in layout AND actually + // rendered (not hidden by opacity:0, visibility:hidden, display:none). We + // use Element.checkVisibility when available because getBoundingClientRect + // returns a non-zero rect for opacity:0 elements — those are elements we + // DO want to detect as newly-revealed later. + const isActuallyVisible = (el) => { + if (typeof el.checkVisibility === 'function') { + return el.checkVisibility({ + opacityProperty: true, + visibilityProperty: true, + contentVisibilityAuto: true, + }); + } + const r = el.getBoundingClientRect(); + return r.width > 0 && r.height > 0; + }; + const all = document.body.querySelectorAll('*'); + for (const el of all) { + if (isActuallyVisible(el)) { + preVisible.add(el); + const r = el.getBoundingClientRect(); + preBounds.set(el, { x: r.left, y: r.top, w: r.width, h: r.height }); + } + } + const mutated = new Set(); + const observer = new MutationObserver((records) => { + for (const r of records) { + if (r.type === 'childList') { + for (const n of r.addedNodes) if (n.nodeType === 1) mutated.add(n); + } else if (r.type === 'attributes' && r.target.nodeType === 1) { + mutated.add(r.target); + } + } + }); + observer.observe(document.body, { + childList: true, + subtree: true, + attributes: true, + attributeFilter: ['class', 'style', 'hidden', 'aria-hidden', 'aria-expanded', 'open'], + }); + window.__wrReveals[id] = { observer, preVisible, preBounds, mutated }; + return id; +})()`; + +// Called AFTER the interaction + postDelay. Walks the mutated element set, +// decides which are newly visible or grew meaningfully, and returns their +// bounding boxes. Filters out our own recording overlay elements (prefixed +// with "__demo-"). Tiny mutations (< 100 px²) are dropped as noise. +const COLLECT_SCRIPT = `(id) => { + const state = window.__wrReveals && window.__wrReveals[id]; + if (!state) return []; + state.observer.disconnect(); + const MIN_AREA = 100; + const GROWTH_MARGIN = 4; + const reveals = []; + for (const el of state.mutated) { + if (!document.body.contains(el)) continue; + if (el.id && el.id.indexOf('__demo-') === 0) continue; + if (el.closest && el.closest('[id^="__demo-"]')) continue; + const r = el.getBoundingClientRect(); + if (r.width <= 0 || r.height <= 0) continue; + if (r.width * r.height < MIN_AREA) continue; + // Must be actually rendered now (not just laid-out-but-invisible). + if (typeof el.checkVisibility === 'function') { + if (!el.checkVisibility({ opacityProperty: true, visibilityProperty: true })) { + continue; + } + } + const wasVisible = state.preVisible.has(el); + if (!wasVisible) { + reveals.push({ x: r.left, y: r.top, width: r.width, height: r.height }); + continue; + } + const prev = state.preBounds.get(el); + if (!prev) continue; + const grewLeft = prev.x - r.left > GROWTH_MARGIN; + const grewTop = prev.y - r.top > GROWTH_MARGIN; + const grewRight = (r.left + r.width) - (prev.x + prev.w) > GROWTH_MARGIN; + const grewBottom = (r.top + r.height) - (prev.y + prev.h) > GROWTH_MARGIN; + if (grewLeft || grewTop || grewRight || grewBottom) { + reveals.push({ x: r.left, y: r.top, width: r.width, height: r.height }); + } + } + delete window.__wrReveals[id]; + return reveals; +}`; + +export async function installRevealObserver( + client: CDPClient, +): Promise { + try { + const { result } = await client.Runtime.evaluate({ + expression: INSTALL_SCRIPT, + returnByValue: true, + }); + if (typeof result.value === "number") return { id: result.value }; + return null; + } catch { + return null; + } +} + +function isBoundingBox(v: unknown): v is BoundingBox { + if (!v || typeof v !== "object") return false; + const b = v as Record; + return ( + typeof b.x === "number" && + typeof b.y === "number" && + typeof b.width === "number" && + typeof b.height === "number" + ); +} + +export async function collectReveals( + client: CDPClient, + handle: RevealObserverHandle, +): Promise { + try { + const { result } = await client.Runtime.evaluate({ + expression: `(${COLLECT_SCRIPT})(${handle.id})`, + returnByValue: true, + }); + if (Array.isArray(result.value)) return result.value.filter(isBoundingBox); + return []; + } catch { + return []; + } +} + +export const __TESTING__ = { INSTALL_SCRIPT, COLLECT_SCRIPT }; diff --git a/packages/@webreel/core/src/timeline.ts b/packages/@webreel/core/src/timeline.ts index 0ce30ac..0144c08 100644 --- a/packages/@webreel/core/src/timeline.ts +++ b/packages/@webreel/core/src/timeline.ts @@ -1,5 +1,6 @@ import { writeFileSync } from "node:fs"; import type { Point, SoundEvent } from "./types.js"; +import type { ZoomEvent } from "./autozoom.js"; import { TARGET_FPS, DEFAULT_CURSOR_SVG, @@ -45,6 +46,7 @@ export interface TimelineData { frames: FrameData[]; events: SoundEvent[]; steps: TimelineStep[]; + zoomEvents?: ZoomEvent[]; } export interface TimelineStep { @@ -70,6 +72,7 @@ export class InteractionTimeline { private steps: TimelineStep[] = []; private frameCount = 0; private tickResolvers: Array<() => void> = []; + private pathCompleteResolvers: Array<() => void> = []; private released = false; private width: number; @@ -168,13 +171,35 @@ export class InteractionTimeline { }); } + // Resolves when the current cursorPath is fully consumed by the capture + // loop. If no path is active (or the timeline has been released), resolves + // immediately. Lets callers fire the next action exactly when the cursor + // arrives at its target, independent of capture rate or hardware speed. + waitForPathComplete(): Promise { + if (this.released || this.cursorPath === null) return Promise.resolve(); + return new Promise((resolve) => { + this.pathCompleteResolvers.push(resolve); + }); + } + + private maybeResolvePathComplete(): void { + if (this.cursorPath !== null || this.pathCompleteResolvers.length === 0) return; + const resolvers = this.pathCompleteResolvers; + this.pathCompleteResolvers = []; + for (const resolve of resolvers) resolve(); + } + // Once the capture loop stops ticking, pending and future waiters must - // resolve immediately or callers like typeText would hang forever. + // resolve immediately or callers like typeText (tick waiters) and + // animateMoveTo (path-complete waiters) would hang forever. releaseWaiters(): void { this.released = true; const resolvers = this.tickResolvers; this.tickResolvers = []; for (const resolve of resolvers) resolve(); + const pathResolvers = this.pathCompleteResolvers; + this.pathCompleteResolvers = []; + for (const resolve of pathResolvers) resolve(); } tick(): void { @@ -192,6 +217,8 @@ export class InteractionTimeline { const resolvers = this.tickResolvers; this.tickResolvers = []; for (const resolve of resolvers) resolve(); + + this.maybeResolvePathComplete(); } tickDuplicate(): void { diff --git a/packages/webreel/README.md b/packages/webreel/README.md index 9ce2c50..7a6df4e 100644 --- a/packages/webreel/README.md +++ b/packages/webreel/README.md @@ -177,6 +177,7 @@ Raw video and timeline data are saved in `.webreel/raw/` and `.webreel/timelines | `theme` | - | Overlay theme (`cursor: { image, size, hotspot }`, `hud`) | | `include` | - | Array of JSON file paths whose steps are prepended | | `defaultDelay` | - | Default delay (ms) after each step | +| `autoZoom` | `false` | Cinematic zoom into each action (bool or object) | ### Actions diff --git a/packages/webreel/src/commands/composite.ts b/packages/webreel/src/commands/composite.ts index d1c1363..ff0e285 100644 --- a/packages/webreel/src/commands/composite.ts +++ b/packages/webreel/src/commands/composite.ts @@ -1,14 +1,14 @@ import { Command } from "commander"; import { readFileSync, existsSync, mkdirSync } from "node:fs"; import { resolve, dirname } from "node:path"; -import { compose, type TimelineData } from "@webreel/core"; +import { buildAutoZoomFilter, compose, type TimelineData } from "@webreel/core"; import { loadWebreelConfig, resolveConfigPath, getConfigDir, filterVideosByName, } from "../lib/config.js"; -import { extractThumbnailIfConfigured } from "../lib/runner.js"; +import { extractThumbnailIfConfigured, normalizeAutoZoom } from "../lib/runner.js"; export const compositeCommand = new Command("composite") .description("Re-composite videos from stored raw recordings and timelines") @@ -49,7 +49,30 @@ export const compositeCommand = new Command("composite") mkdirSync(dirname(outputPath), { recursive: true }); console.log(`Compositing: ${video.name}`); - await compose(rawPath, timelineData, outputPath, { sfx: video.sfx }); + + const autoZoomCfg = normalizeAutoZoom(video.autoZoom); + const persistedZoomEvents = timelineData.zoomEvents ?? []; + const zoomFilter = + autoZoomCfg.enabled && persistedZoomEvents.length > 0 + ? buildAutoZoomFilter( + persistedZoomEvents, + { width: timelineData.width, height: timelineData.height }, + timelineData.zoom ?? 1, + timelineData.frames.length / timelineData.fps, + timelineData.fps, + autoZoomCfg, + ) + : null; + if (zoomFilter) { + console.log( + `Applying autozoom (${persistedZoomEvents.length} event${persistedZoomEvents.length === 1 ? "" : "s"})`, + ); + } + + await compose(rawPath, timelineData, outputPath, { + sfx: video.sfx, + zoomFilter: zoomFilter ?? undefined, + }); await extractThumbnailIfConfigured(video, outputPath); diff --git a/packages/webreel/src/lib/__tests__/config.test.ts b/packages/webreel/src/lib/__tests__/config.test.ts index 6f6a1f6..3eefa38 100644 --- a/packages/webreel/src/lib/__tests__/config.test.ts +++ b/packages/webreel/src/lib/__tests__/config.test.ts @@ -343,6 +343,83 @@ describe("include validation", () => { }); }); +describe("autoZoom validation", () => { + function wrapAutoZoom(autoZoom: unknown) { + return { videos: { x: { url: "u", steps: [], autoZoom } } }; + } + + it("accepts autoZoom booleans", () => { + expect(validateWebreelConfig(wrapAutoZoom(true))).toEqual([]); + expect(validateWebreelConfig(wrapAutoZoom(false))).toEqual([]); + }); + + it("accepts a full autoZoom config object", () => { + const errors = validateWebreelConfig( + wrapAutoZoom({ + enabled: true, + approachS: 0.5, + settleBeforeS: 0.15, + holdAfterS: 0.3, + releaseS: 0.5, + paddingRatio: 0.3, + minZoomRatio: 0.6, + skipZoomRatio: 0.75, + sessionGapS: 4.0, + minPanS: 0.8, + }), + ); + expect(errors).toEqual([]); + }); + + it("rejects autoZoom values that are neither boolean nor object", () => { + const errors = validateWebreelConfig(wrapAutoZoom("yes")); + expect(errors).toContainEqual(expect.objectContaining({ path: "videos.x.autoZoom" })); + }); + + it("rejects non-boolean enabled", () => { + const errors = validateWebreelConfig(wrapAutoZoom({ enabled: "yes" })); + expect(errors).toContainEqual( + expect.objectContaining({ path: "videos.x.autoZoom.enabled" }), + ); + }); + + it("rejects negative timing values", () => { + const errors = validateWebreelConfig(wrapAutoZoom({ approachS: -1 })); + expect(errors).toContainEqual( + expect.objectContaining({ path: "videos.x.autoZoom.approachS" }), + ); + }); + + it("rejects non-numeric paddingRatio", () => { + const errors = validateWebreelConfig(wrapAutoZoom({ paddingRatio: "big" })); + expect(errors).toContainEqual( + expect.objectContaining({ path: "videos.x.autoZoom.paddingRatio" }), + ); + }); + + it("rejects ratios outside 0 to 1", () => { + const errors = validateWebreelConfig( + wrapAutoZoom({ minZoomRatio: 1.5, skipZoomRatio: -0.2 }), + ); + expect(errors).toContainEqual( + expect.objectContaining({ path: "videos.x.autoZoom.minZoomRatio" }), + ); + expect(errors).toContainEqual( + expect.objectContaining({ path: "videos.x.autoZoom.skipZoomRatio" }), + ); + }); + + it("detects unknown autoZoom properties with suggestions", () => { + const errors = validateWebreelConfig(wrapAutoZoom({ paddingRato: 0.3 })); + expect(errors).toContainEqual( + expect.objectContaining({ + path: "videos.x.autoZoom.paddingRato", + message: expect.stringContaining("paddingRatio"), + }), + ); + }); +}); + describe("validateWebreelConfig", () => { it("accepts a valid multi-video config", () => { const errors = validateWebreelConfig({ diff --git a/packages/webreel/src/lib/config.ts b/packages/webreel/src/lib/config.ts index 2bccbd9..6ce84d9 100644 --- a/packages/webreel/src/lib/config.ts +++ b/packages/webreel/src/lib/config.ts @@ -322,6 +322,7 @@ const KNOWN_VIDEO_KEYS = new Set([ "sfx", "defaultDelay", "clickDwell", + "autoZoom", "steps", ]); @@ -756,6 +757,76 @@ function validateSfx(sfx: unknown, prefix: string): ValidationError[] { return errors; } +const KNOWN_AUTOZOOM_KEYS = new Set([ + "enabled", + "approachS", + "settleBeforeS", + "holdAfterS", + "releaseS", + "paddingRatio", + "minZoomRatio", + "skipZoomRatio", + "sessionGapS", + "minPanS", +]); + +const AUTOZOOM_NONNEGATIVE_KEYS = [ + "approachS", + "settleBeforeS", + "holdAfterS", + "releaseS", + "paddingRatio", + "sessionGapS", + "minPanS", +] as const; + +const AUTOZOOM_RATIO_KEYS = ["minZoomRatio", "skipZoomRatio"] as const; + +function validateAutoZoom(autoZoom: unknown, prefix: string): ValidationError[] { + const errors: ValidationError[] = []; + if (typeof autoZoom === "boolean") return errors; + if (typeof autoZoom !== "object" || autoZoom === null) { + errors.push({ + path: prefix, + message: "Must be a boolean or an autozoom config object", + }); + return errors; + } + + const a = autoZoom as Record; + + errors.push(...checkUnknownKeys(a, KNOWN_AUTOZOOM_KEYS, prefix)); + + if (a.enabled !== undefined && typeof a.enabled !== "boolean") { + errors.push({ path: `${prefix}.enabled`, message: "Must be a boolean" }); + } + + for (const key of AUTOZOOM_NONNEGATIVE_KEYS) { + const value = a[key]; + if (value !== undefined && (!Number.isFinite(value) || (value as number) < 0)) { + errors.push({ + path: `${prefix}.${key}`, + message: "Must be a non-negative number", + }); + } + } + + for (const key of AUTOZOOM_RATIO_KEYS) { + const value = a[key]; + if ( + value !== undefined && + (!Number.isFinite(value) || (value as number) < 0 || (value as number) > 1) + ) { + errors.push({ + path: `${prefix}.${key}`, + message: "Must be a number between 0 and 1", + }); + } + } + + return errors; +} + function validateTheme(theme: unknown, prefix: string): ValidationError[] { const errors: ValidationError[] = []; if (typeof theme !== "object" || theme === null) { @@ -1060,6 +1131,10 @@ export function validateWebreelConfig( errors.push(...validateSfx(d.sfx, `${prefix}.sfx`)); } + if (d.autoZoom !== undefined) { + errors.push(...validateAutoZoom(d.autoZoom, `${prefix}.autoZoom`)); + } + if (!Array.isArray(d.steps)) { errors.push({ path: `${prefix}.steps`, message: "Required, must be an array" }); } else { diff --git a/packages/webreel/src/lib/runner.ts b/packages/webreel/src/lib/runner.ts index 5a6b6a6..a74744c 100644 --- a/packages/webreel/src/lib/runner.ts +++ b/packages/webreel/src/lib/runner.ts @@ -5,6 +5,8 @@ import { type CDPClient, type BoundingBox, type OverlayTheme, + type AutoZoomConfig, + type ZoomEvent, RecordingContext, connectCDP, launchChrome, @@ -27,6 +29,11 @@ import { ensureFfmpeg, extractThumbnail, moveFileSync, + buildAutoZoomFilter, + unionBboxes, + installRevealObserver, + collectReveals, + type RevealObserverHandle, DEFAULT_VIEWPORT_SIZE, } from "@webreel/core"; import type { VideoConfig, Step, ElementTarget } from "./types.js"; @@ -121,6 +128,120 @@ export function randomPointInBox( }; } +export function normalizeAutoZoom(value: VideoConfig["autoZoom"]): AutoZoomConfig { + if (value === true) return { enabled: true }; + if (value === false || value === undefined) return { enabled: false }; + return { ...value, enabled: value.enabled ?? true }; +} + +async function captureZoomEvent( + client: CDPClient, + step: Step, + timeline: InteractionTimeline, + preEventCount: number, + fps: number, + previousBox: BoundingBox | null, + reveals: BoundingBox[] = [], +): Promise { + let box: BoundingBox | null = null; + + switch (step.action) { + case "navigate": + case "navigateHref": + case "pause": + case "wait": + case "screenshot": + case "scroll": + case "upload": + return null; + + case "click": + case "moveTo": + case "hover": + case "select": { + if (step.selector) { + box = await findElementBySelector(client, step.selector, step.within); + } else if (step.text) { + box = await findElementByText(client, step.text, step.within); + } + break; + } + + case "type": { + if (step.selector) { + box = await findElementBySelector(client, step.selector, step.within); + } else if (previousBox) { + box = previousBox; + } + break; + } + + case "key": { + if (step.target) { + const sel = typeof step.target === "string" ? step.target : step.target.selector; + const within = typeof step.target === "string" ? undefined : step.target.within; + if (sel) box = await findElementBySelector(client, sel, within); + } else if (previousBox) { + box = previousBox; + } + break; + } + + case "drag": { + if (step.from.selector) { + box = await findElementBySelector(client, step.from.selector, step.from.within); + } else if (step.from.text) { + box = await findElementByText(client, step.from.text, step.from.within); + } + break; + } + } + + if (!box) return null; + + // For click/drag, union the click target's bbox with any elements that + // newly appeared or grew in size during the step (dropdowns, modals, + // tooltips). This makes the zoom frame the whole widget rather than just + // the trigger, matching Cursor's behavior for button-to-modal interactions. + if ((step.action === "click" || step.action === "drag") && reveals.length > 0) { + const unioned = unionBboxes([box, ...reveals]); + if (unioned) box = unioned; + } + + // Pick the timestamp from SoundEvents that fired DURING this step so the + // zoom anticipation is anchored on the actual interaction moment (not the + // step's end-of-postDelay time, which lags the click by 1-2 seconds for + // steps with cursor animation). + const newEvents = timeline.getEvents().slice(preEventCount); + const fallbackMs = (timeline.getFrameCount() / fps) * 1000; + let timeMs: number; + let holdUntilMs: number | undefined; + if (step.action === "click" || step.action === "drag") { + const firstClick = newEvents.find((e) => e.type === "click"); + timeMs = firstClick ? firstClick.timeMs : fallbackMs; + } else if (step.action === "type" || step.action === "key") { + // Anchor the approach on the FIRST event (click that lands on the input) + // so the camera is already zoomed in when typing begins. `holdUntilMs` + // extends the hold through the LAST keystroke, so the camera stays on + // the field for the entire typing span. + const firstEvent = newEvents[0]; + const lastEvent = newEvents[newEvents.length - 1]; + timeMs = firstEvent ? firstEvent.timeMs : fallbackMs; + if (lastEvent && lastEvent !== firstEvent) { + holdUntilMs = lastEvent.timeMs; + } + } else { + timeMs = fallbackMs; + } + + const { result } = await client.Runtime.evaluate({ + expression: "location.href", + returnByValue: true, + }); + const url = typeof result.value === "string" ? result.value : undefined; + return { timeMs, box, url, holdUntilMs }; +} + export async function extractThumbnailIfConfigured( config: Pick, outputPath: string, @@ -240,6 +361,8 @@ export async function runVideo( } let timeline: InteractionTimeline | null = null; + const autoZoomCfg = normalizeAutoZoom(config.autoZoom); + const zoomEvents: ZoomEvent[] = []; const outputPath = config.output ?? resolve(configDir, "videos", `${config.name}.mp4`); @@ -284,6 +407,18 @@ export async function runVideo( if (verbose) console.log(formatStep(i, step)); const stepStartMs = timeline?.getCurrentTimeMs(); + let preEventCount = 0; + let revealHandle: RevealObserverHandle | null = null; + if (shouldRecord && autoZoomCfg.enabled && timeline) { + preEventCount = timeline.getEvents().length; + // Observer installed only for interactions that can reveal UI. + // Collected after postDelay so any animations have had time to + // settle. + if (step.action === "click" || step.action === "drag") { + revealHandle = await installRevealObserver(client); + } + } + try { switch (step.action) { case "pause": @@ -502,6 +637,35 @@ export async function runVideo( ...(step.description ? { description: step.description } : {}), }); } + if (shouldRecord && autoZoomCfg.enabled && timeline) { + const fps = config.fps ?? timeline.toJSON().fps; + const previousBox = + zoomEvents.length > 0 ? zoomEvents[zoomEvents.length - 1].box : null; + const reveals: BoundingBox[] = revealHandle + ? await collectReveals(client, revealHandle) + : []; + if (process.env.WEBREEL_DEBUG_ZOOM && revealHandle) { + const summary = reveals + .map( + (r) => + `${r.x.toFixed(0)},${r.y.toFixed(0)} ${r.width.toFixed(0)}x${r.height.toFixed(0)}`, + ) + .join(" | "); + console.error( + `reveals (${step.action}): ${reveals.length}${summary ? " " + summary : ""}`, + ); + } + const ze = await captureZoomEvent( + client, + step, + timeline, + preEventCount, + fps, + previousBox, + reveals, + ); + if (ze) zoomEvents.push(ze); + } } catch (err) { throw new Error( `Step ${i} (${step.action}) failed at ${url}: ${err instanceof Error ? err.message : String(err)}`, @@ -517,6 +681,9 @@ export async function runVideo( if (timeline) { const timelineData = timeline.toJSON(); + if (zoomEvents.length > 0) { + timelineData.zoomEvents = zoomEvents; + } const metadataDir = resolve(configDir, ".webreel", "timelines"); mkdirSync(metadataDir, { recursive: true }); writeFileSync( @@ -533,7 +700,33 @@ export async function runVideo( ctx.setTimeline(null); mkdirSync(dirname(outputPath), { recursive: true }); console.log(`Compositing overlays...`); - await compose(rawVideoPath, timelineData, outputPath, { sfx: config.sfx }); + const zoomFilter = autoZoomCfg.enabled + ? buildAutoZoomFilter( + zoomEvents, + { width: timelineData.width, height: timelineData.height }, + timelineData.zoom ?? 1, + timelineData.frames.length / timelineData.fps, + timelineData.fps, + autoZoomCfg, + ) + : null; + if (zoomFilter) { + console.log( + `Applying autozoom (${zoomEvents.length} event${zoomEvents.length === 1 ? "" : "s"})`, + ); + if (verbose) { + for (const e of zoomEvents) { + const box = e.box; + console.log( + ` t=${(e.timeMs / 1000).toFixed(2)}s box=${box.x.toFixed(0)},${box.y.toFixed(0)} ${box.width.toFixed(0)}x${box.height.toFixed(0)}`, + ); + } + } + } + await compose(rawVideoPath, timelineData, outputPath, { + sfx: config.sfx, + zoomFilter: zoomFilter ?? undefined, + }); } await extractThumbnailIfConfigured(config, outputPath); diff --git a/packages/webreel/src/lib/types.ts b/packages/webreel/src/lib/types.ts index aadc1f6..36e0088 100644 --- a/packages/webreel/src/lib/types.ts +++ b/packages/webreel/src/lib/types.ts @@ -189,8 +189,8 @@ export const VIEWPORT_PRESETS: Record "galaxy-s24": { width: 360, height: 780 }, }; -export type { SfxConfig } from "@webreel/core"; -import type { SfxConfig } from "@webreel/core"; +export type { SfxConfig, AutoZoomConfig } from "@webreel/core"; +import type { SfxConfig, AutoZoomConfig } from "@webreel/core"; export interface VideoConfig { name: string; @@ -208,6 +208,7 @@ export interface VideoConfig { sfx?: SfxConfig; defaultDelay?: number; clickDwell?: number; + autoZoom?: AutoZoomConfig | boolean; steps: Step[]; } diff --git a/skills/webreel/SKILL.md b/skills/webreel/SKILL.md index 8903f8c..096e0dc 100644 --- a/skills/webreel/SKILL.md +++ b/skills/webreel/SKILL.md @@ -162,6 +162,7 @@ Each entry in the `videos` map supports: | `clickDwell` | inherited | Override click dwell | | `fps` | `60` | Frame rate | | `quality` | `80` | Encoding quality (1-100) | +| `autoZoom` | `false` | Cinematic zoom into each action (bool or object) | | `steps` | required | Array of step objects | ### Videos map @@ -253,6 +254,64 @@ Customize cursor appearance and keystroke HUD: - `cursor.hotspot` - `"top-left"` (default) or `"center"` - `hud.position` - `"top"` or `"bottom"` +## Autozoom + +Autozoom cinematically zooms into each user action so small UI elements are readable on mobile-sized viewports. It runs as part of the existing compositing pass, with no extra encoding, and uses the bounding box of each `click`, `type`, `hover`, `moveTo`, `drag`, or `select` target to decide where and when to zoom. + +Enable with defaults: + +```json +{ "autoZoom": true } +``` + +Or tune individual parameters: + +```json +{ + "autoZoom": { + "enabled": true, + "approachS": 0.5, + "settleBeforeS": 0.15, + "holdAfterS": 0.3, + "releaseS": 0.5, + "paddingRatio": 0.3, + "minZoomRatio": 0.6, + "skipZoomRatio": 0.75, + "sessionGapS": 4.0, + "minPanS": 0.8 + } +} +``` + +| Field | Default | Description | +| --------------- | ------- | ---------------------------------------------------------------------------- | +| `enabled` | `true` | Turn autozoom on/off without removing config | +| `approachS` | `0.5` | Seconds spent zooming in (full frame to target) | +| `settleBeforeS` | `0.15` | Seconds the camera sits on the target before the action fires | +| `holdAfterS` | `0.3` | Seconds the camera holds after the last action in a session | +| `releaseS` | `0.5` | Seconds spent zooming out (target to full frame) | +| `paddingRatio` | `0.3` | Fraction of bbox size added as padding around the target | +| `minZoomRatio` | `0.6` | Crop is never smaller than this fraction of the viewport (max zoom ~1.67x) | +| `skipZoomRatio` | `0.75` | Skip zoom when the computed crop would be this fraction or larger | +| `sessionGapS` | `4.0` | Events within this many seconds share one pan session (no zoom-out) | +| `minPanS` | `0.8` | Skip panning to an intermediate target if the pan would be shorter than this | + +How it behaves: + +- The camera starts zooming `approachS + settleBeforeS` seconds before each action, so it has already settled on the target when the click or keystroke fires. +- Multiple actions within `sessionGapS` are treated as one session. The camera pans between them instead of zooming all the way out and back in. +- Click and drag zoom targets automatically include any UI that appears as a result of the click (dropdown menus, modals, tooltips). A `MutationObserver` watches the DOM during each click step and unions newly visible elements into the zoom bbox, so the camera frames the whole widget rather than just the trigger. +- Clicks that trigger a page navigation (URL change) are skipped; the page content changes unpredictably, and the zoom timing would be off. +- Full-width targets (crop at or above `skipZoomRatio`) are skipped to avoid imperceptible zooms. + +When NOT to use autozoom: + +- Page-scroll-only demos (no discrete click targets). +- Videos whose interactions are all `key` presses without a `target` (no bbox to zoom to). +- Videos recorded at already-small viewports where the content is already readable. + +Pair with `zoom` for best results on small viewports: the static `zoom` field upscales the captured viewport for readability, and `autoZoom` then cinematically zooms further into each action. + ## Common patterns ### Shared steps via include diff --git a/skills/webreel/examples.md b/skills/webreel/examples.md index cfdd1f1..0db304a 100644 --- a/skills/webreel/examples.md +++ b/skills/webreel/examples.md @@ -321,3 +321,47 @@ Set the `output` field to a `.webm` extension. } } ``` + +## Autozoom + +Cinematically zoom into each click, type, or hover. The camera eases in before the action, holds through it, then releases. Consecutive actions within `sessionGapS` share a single zoom session (camera pans between them instead of zooming out). + +Enable with defaults: + +```json +{ + "$schema": "https://webreel.dev/schema/v1.json", + "videos": { + "autozoom": { + "url": "./web/index.html", + "viewport": { "width": 1920, "height": 1080 }, + "zoom": 2, + "waitFor": ".card", + "autoZoom": true, + "defaultDelay": 500, + "steps": [ + { "action": "click", "selector": "#name" }, + { "action": "type", "text": "Jane Doe", "selector": "#name" }, + { "action": "click", "selector": "#email" }, + { "action": "type", "text": "jane@acme.com", "selector": "#email" }, + { "action": "select", "selector": "#role", "value": "engineer" }, + { "action": "click", "selector": "#save", "delay": 1200 } + ] + } + } +} +``` + +Override specific timing or thresholds with an object: + +```json +{ + "autoZoom": { + "enabled": true, + "approachS": 1.2, + "holdAfterS": 0.8, + "paddingRatio": 0.4, + "sessionGapS": 3.5 + } +} +``` From 73039527919f5b3aaa1284c2936dd2421c8e6597 Mon Sep 17 00:00:00 2001 From: Ariel Conti Date: Sat, 18 Jul 2026 15:22:05 +0100 Subject: [PATCH 11/48] fix(core): escape theme strings embedded in the injected overlay script Theme values (hudBg, hudColor, hudFontFamily) were spliced raw into the JS source string evaluated in the recorded page. A value containing a quote, backtick, or ${} could break or inject into the generated script. Embed strings via JSON.stringify and coerce numeric fields with Number(), matching the escAttr pattern already used in compositor.ts. Also constrain hudPosition to the "top"/"bottom" whitelist. Claude-Session: https://claude.ai/code/session_01Pv8tMQt9GoVpfnpVcdwFxe --- .../core/src/__tests__/overlays.test.ts | 37 +++++++++++++++++-- packages/@webreel/core/src/overlays.ts | 31 ++++++++-------- 2 files changed, 48 insertions(+), 20 deletions(-) diff --git a/packages/@webreel/core/src/__tests__/overlays.test.ts b/packages/@webreel/core/src/__tests__/overlays.test.ts index cc7ba0f..d4c1f79 100644 --- a/packages/@webreel/core/src/__tests__/overlays.test.ts +++ b/packages/@webreel/core/src/__tests__/overlays.test.ts @@ -50,7 +50,9 @@ describe("injectOverlays", () => { const { client, calls } = createMockClient(); await injectOverlays(client); const expr = calls[0].expression; - expect(expr).toContain(`background:${DEFAULT_HUD_THEME.background}`); + expect(expr).toContain( + `"background:" + ${JSON.stringify(DEFAULT_HUD_THEME.background)}`, + ); expect(expr).toContain(`border-radius:" + z(${DEFAULT_HUD_THEME.borderRadius})`); }); @@ -67,10 +69,37 @@ describe("injectOverlays", () => { }; await injectOverlays(client, theme); const expr = calls[0].expression; - expect(expr).toContain("background:red"); - expect(expr).toContain("color: blue"); + expect(expr).toContain('"background:" + "red"'); + expect(expr).toContain('"color:" + "blue"'); expect(expr).toContain('border-radius:" + z(8)'); - expect(expr).toContain('"top:"'); + expect(expr).toContain('"top" + ":"'); + }); + + it("escapes hostile theme strings so the injected script stays syntactically valid", async () => { + const { client, calls } = createMockClient(); + const theme: OverlayTheme = { + hud: { + background: `"; document.title = "pwned"; //`, + color: "` + alert(1) + `", + fontFamily: "${alert(1)}, sans-serif", + fontSize: 40, + borderRadius: 12, + position: "top", + }, + }; + await injectOverlays(client, theme); + const expr = calls[0].expression; + expect(() => new Function(expr)).not.toThrow(); + expect(expr).toContain(JSON.stringify(theme.hud!.background)); + expect(expr).toContain(JSON.stringify(theme.hud!.color)); + expect(expr).toContain(JSON.stringify(theme.hud!.fontFamily)); + }); + + it("produces a syntactically valid script with default theme values", async () => { + const { client, calls } = createMockClient(); + await injectOverlays(client); + const expr = calls[0].expression; + expect(() => new Function(expr)).not.toThrow(); }); it("uses custom cursor SVG", async () => { diff --git a/packages/@webreel/core/src/overlays.ts b/packages/@webreel/core/src/overlays.ts index 7bead99..64f51bc 100644 --- a/packages/@webreel/core/src/overlays.ts +++ b/packages/@webreel/core/src/overlays.ts @@ -33,7 +33,7 @@ export async function injectOverlays( const hudFontSize = theme?.hud?.fontSize ?? DEFAULT_HUD_THEME.fontSize; const hudFontFamily = theme?.hud?.fontFamily ?? DEFAULT_HUD_THEME.fontFamily; const hudBorderRadius = theme?.hud?.borderRadius ?? DEFAULT_HUD_THEME.borderRadius; - const hudPosition = theme?.hud?.position ?? DEFAULT_HUD_THEME.position; + const hudPosition = theme?.hud?.position === "top" ? "top" : "bottom"; await client.Runtime.evaluate({ expression: `(() => { @@ -68,31 +68,30 @@ export async function injectOverlays( "position:fixed", "z-index:999999", "pointer-events:none", - "${hudPosition}:" + z(48), + ${JSON.stringify(hudPosition)} + ":" + z(48), "left:50%", "transform:translateX(-50%)", "display:flex", "gap:" + z(14), "padding:" + z(16) + " " + z(36), - "border-radius:" + z(${hudBorderRadius}), - "background:${hudBg}", + "border-radius:" + z(${Number(hudBorderRadius)}), + "background:" + ${JSON.stringify(hudBg)}, "opacity:0", ].join(";"); document.body.appendChild(keys); const style = document.createElement("style"); - style.textContent = \` - .__demo-key { - display: inline-flex; - align-items: center; - justify-content: center; - color: ${hudColor}; - font-family: ${hudFontFamily}; - font-size: \${${hudFontSize} / zoom}px; - font-weight: 500; - white-space: nowrap; - } - \`; + style.textContent = + ".__demo-key {" + + "display: inline-flex;" + + "align-items: center;" + + "justify-content: center;" + + "color:" + ${JSON.stringify(hudColor)} + ";" + + "font-family:" + ${JSON.stringify(hudFontFamily)} + ";" + + "font-size:" + (${Number(hudFontSize)} / zoom) + "px;" + + "font-weight: 500;" + + "white-space: nowrap;" + + "}"; document.head.appendChild(style); })()`, }); From cac1e84fa7c2996541a3a268954deee66eeabaa4 Mon Sep 17 00:00:00 2001 From: Ariel Conti Date: Sat, 18 Jul 2026 15:23:24 +0100 Subject: [PATCH 12/48] fix(core): fail cleanly when the CDP connection errors or drops Register listeners for the chrome-remote-interface client's 'error' and 'disconnect' events so a Chrome crash or socket drop mid-run surfaces as a reported failure through the existing cleanup path instead of an unhandled EventEmitter 'error' that kills the process outside cleanup (orphaned Chrome, undeleted temp profile) or a silently stranded in-flight await. --- .../@webreel/core/src/__tests__/cdp.test.ts | 59 +++++++++++++++++++ packages/@webreel/core/src/cdp.ts | 34 ++++++++++- packages/@webreel/core/src/types.ts | 1 + packages/webreel/src/lib/runner.ts | 11 +++- 4 files changed, 102 insertions(+), 3 deletions(-) create mode 100644 packages/@webreel/core/src/__tests__/cdp.test.ts diff --git a/packages/@webreel/core/src/__tests__/cdp.test.ts b/packages/@webreel/core/src/__tests__/cdp.test.ts new file mode 100644 index 0000000..49957e3 --- /dev/null +++ b/packages/@webreel/core/src/__tests__/cdp.test.ts @@ -0,0 +1,59 @@ +import { describe, it, expect, vi } from "vitest"; +import { EventEmitter } from "node:events"; +import { connectCDP } from "../cdp.js"; + +let fakeClient: EventEmitter; + +vi.mock("chrome-remote-interface", () => ({ + default: vi.fn(() => Promise.resolve(fakeClient)), +})); + +describe("connectCDP", () => { + it("registers an 'error' listener so an emitted error does not throw", async () => { + fakeClient = new EventEmitter(); + const onConnectionLost = vi.fn(); + + await connectCDP(9222, onConnectionLost); + + expect(() => fakeClient.emit("error", new Error("socket blew up"))).not.toThrow(); + expect(onConnectionLost).toHaveBeenCalledTimes(1); + const err = onConnectionLost.mock.calls[0][0]; + expect(err).toBeInstanceOf(Error); + expect(err.message).toMatch(/CDP connection error/); + }); + + it("registers a 'disconnect' listener that reports a descriptive error", async () => { + fakeClient = new EventEmitter(); + const onConnectionLost = vi.fn(); + + await connectCDP(9222, onConnectionLost); + + expect(() => fakeClient.emit("disconnect")).not.toThrow(); + expect(onConnectionLost).toHaveBeenCalledTimes(1); + const err = onConnectionLost.mock.calls[0][0]; + expect(err).toBeInstanceOf(Error); + expect(err.message).toMatch(/CDP connection lost/); + }); + + it("invokes the callback exactly once even if both events fire", async () => { + fakeClient = new EventEmitter(); + const onConnectionLost = vi.fn(); + + await connectCDP(9222, onConnectionLost); + + fakeClient.emit("error", new Error("boom")); + fakeClient.emit("disconnect"); + fakeClient.emit("error", new Error("boom again")); + + expect(onConnectionLost).toHaveBeenCalledTimes(1); + }); + + it("works without a callback (no-op, prevents unhandled 'error' crash)", async () => { + fakeClient = new EventEmitter(); + + await connectCDP(9222); + + expect(() => fakeClient.emit("error", new Error("no listener needed"))).not.toThrow(); + expect(() => fakeClient.emit("disconnect")).not.toThrow(); + }); +}); diff --git a/packages/@webreel/core/src/cdp.ts b/packages/@webreel/core/src/cdp.ts index 71539ab..ccaf7f8 100644 --- a/packages/@webreel/core/src/cdp.ts +++ b/packages/@webreel/core/src/cdp.ts @@ -1,6 +1,36 @@ import CDP from "chrome-remote-interface"; import type { CDPClient } from "./types.js"; -export async function connectCDP(port: number): Promise { - return (await CDP({ port })) as unknown as CDPClient; +/** + * Connects to Chrome over CDP and registers listeners for the client's + * 'error' and 'disconnect' events so a Chrome crash or socket error never + * surfaces as an unhandled EventEmitter 'error' (which would crash the + * process outside our cleanup path). `onConnectionLost` is invoked at most + * once, with a descriptive Error, regardless of which event fires first. + */ +export async function connectCDP( + port: number, + onConnectionLost?: (err: Error) => void, +): Promise { + const client = (await CDP({ port })) as unknown as CDPClient; + + let notified = false; + const notify = (err: Error) => { + if (notified) return; + notified = true; + onConnectionLost?.(err); + }; + + client.on("error", (err: unknown) => { + notify( + new Error( + `CDP connection error: ${err instanceof Error ? err.message : String(err)}`, + ), + ); + }); + client.on("disconnect", () => { + notify(new Error("CDP connection lost (Chrome exited or crashed)")); + }); + + return client; } diff --git a/packages/@webreel/core/src/types.ts b/packages/@webreel/core/src/types.ts index e30f2cb..116b53c 100644 --- a/packages/@webreel/core/src/types.ts +++ b/packages/@webreel/core/src/types.ts @@ -1,5 +1,6 @@ export type CDPClient = { close: () => Promise; + on: (event: string, cb: (...args: unknown[]) => void) => void; Runtime: { enable: () => Promise; evaluate: (params: { diff --git a/packages/webreel/src/lib/runner.ts b/packages/webreel/src/lib/runner.ts index a74744c..c5260bf 100644 --- a/packages/webreel/src/lib/runner.ts +++ b/packages/webreel/src/lib/runner.ts @@ -310,8 +310,13 @@ export async function runVideo( await chrome.kill(); }); + let connectionLost: Error | null = null; + try { - const client = await connectCDP(chrome.port); + const client = await connectCDP(chrome.port, (err) => { + connectionLost = err; + console.error(err.message); + }); clientRef = client; await client.Page.enable(); await client.Runtime.enable(); @@ -672,8 +677,12 @@ export async function runVideo( { cause: err }, ); } + + if (connectionLost) throw connectionLost; } + if (connectionLost) throw connectionLost; + if (recorder) { const cleanVideoPath = recorder.getTempVideoPath(); await recorder.stop(); From cdfae0a956eda148c8e4e4fd70919c44115b8fb4 Mon Sep 17 00:00:00 2001 From: Ariel Conti Date: Sat, 18 Jul 2026 15:24:03 +0100 Subject: [PATCH 13/48] refactor(webreel): extract shared compositeRecording helper runVideo and the composite command each re-implemented the zoom-filter/compose/thumbnail sequence, and the two copies had already drifted (runner.ts had verbose per-event logging the command lacked). Move normalizeAutoZoom and extractThumbnailIfConfigured into a new lib/compositing.ts alongside a new compositeRecording() helper, and have runVideo call it. Both names are re-exported from runner.ts for compatibility with existing importers. --- packages/webreel/src/lib/compositing.ts | 78 +++++++++++++++++++++++++ packages/webreel/src/lib/runner.ts | 60 ++++--------------- 2 files changed, 88 insertions(+), 50 deletions(-) create mode 100644 packages/webreel/src/lib/compositing.ts diff --git a/packages/webreel/src/lib/compositing.ts b/packages/webreel/src/lib/compositing.ts new file mode 100644 index 0000000..22c2a93 --- /dev/null +++ b/packages/webreel/src/lib/compositing.ts @@ -0,0 +1,78 @@ +import { ensureFfmpeg, extractThumbnail } from "@webreel/core"; +import { + buildAutoZoomFilter, + compose, + type AutoZoomConfig, + type TimelineData, + type ZoomEvent, +} from "@webreel/core"; +import type { VideoConfig } from "./types.js"; + +export function normalizeAutoZoom(value: VideoConfig["autoZoom"]): AutoZoomConfig { + if (value === true) return { enabled: true }; + if (value === false || value === undefined) return { enabled: false }; + return { ...value, enabled: value.enabled ?? true }; +} + +export async function extractThumbnailIfConfigured( + config: Pick, + outputPath: string, +): Promise { + if (config.thumbnail?.enabled === false) return; + const thumbTime = config.thumbnail?.time ?? 0; + const thumbPath = outputPath.replace(/\.[^.]+$/, ".png"); + const ffmpegPath = await ensureFfmpeg(); + extractThumbnail(ffmpegPath, outputPath, thumbPath, thumbTime); + console.log(`Thumbnail: ${thumbPath}`); +} + +export interface CompositeRecordingOptions { + rawVideoPath: string; + timelineData: TimelineData; + outputPath: string; + video: Pick; + zoomEvents: ZoomEvent[]; + verbose: boolean; +} + +/** + * Shared compositing orchestration used by both `runVideo` (fresh recording) + * and the `composite` command (re-composite from stored raw video + + * timeline). Builds the autozoom filter (if enabled and events are + * available), composes the final output, then extracts the thumbnail. + */ +export async function compositeRecording(opts: CompositeRecordingOptions): Promise { + const { rawVideoPath, timelineData, outputPath, video, zoomEvents, verbose } = opts; + + const autoZoomCfg = normalizeAutoZoom(video.autoZoom); + const zoomFilter = autoZoomCfg.enabled + ? buildAutoZoomFilter( + zoomEvents, + { width: timelineData.width, height: timelineData.height }, + timelineData.zoom ?? 1, + timelineData.frames.length / timelineData.fps, + timelineData.fps, + autoZoomCfg, + ) + : null; + if (zoomFilter) { + console.log( + `Applying autozoom (${zoomEvents.length} event${zoomEvents.length === 1 ? "" : "s"})`, + ); + if (verbose) { + for (const e of zoomEvents) { + const box = e.box; + console.log( + ` t=${(e.timeMs / 1000).toFixed(2)}s box=${box.x.toFixed(0)},${box.y.toFixed(0)} ${box.width.toFixed(0)}x${box.height.toFixed(0)}`, + ); + } + } + } + + await compose(rawVideoPath, timelineData, outputPath, { + sfx: video.sfx, + zoomFilter: zoomFilter ?? undefined, + }); + + await extractThumbnailIfConfigured(video, outputPath); +} diff --git a/packages/webreel/src/lib/runner.ts b/packages/webreel/src/lib/runner.ts index a74744c..3e23498 100644 --- a/packages/webreel/src/lib/runner.ts +++ b/packages/webreel/src/lib/runner.ts @@ -5,7 +5,6 @@ import { type CDPClient, type BoundingBox, type OverlayTheme, - type AutoZoomConfig, type ZoomEvent, RecordingContext, connectCDP, @@ -25,11 +24,7 @@ import { captureScreenshot, Recorder, InteractionTimeline, - compose, - ensureFfmpeg, - extractThumbnail, moveFileSync, - buildAutoZoomFilter, unionBboxes, installRevealObserver, collectReveals, @@ -38,6 +33,9 @@ import { } from "@webreel/core"; import type { VideoConfig, Step, ElementTarget } from "./types.js"; import { registerInterruptCleanup } from "./signals.js"; +import { compositeRecording, normalizeAutoZoom } from "./compositing.js"; + +export { extractThumbnailIfConfigured, normalizeAutoZoom } from "./compositing.js"; export function formatStep(i: number, step: Step): string { const desc = "description" in step && step.description ? `: ${step.description}` : ""; @@ -128,12 +126,6 @@ export function randomPointInBox( }; } -export function normalizeAutoZoom(value: VideoConfig["autoZoom"]): AutoZoomConfig { - if (value === true) return { enabled: true }; - if (value === false || value === undefined) return { enabled: false }; - return { ...value, enabled: value.enabled ?? true }; -} - async function captureZoomEvent( client: CDPClient, step: Step, @@ -242,18 +234,6 @@ async function captureZoomEvent( return { timeMs, box, url, holdUntilMs }; } -export async function extractThumbnailIfConfigured( - config: Pick, - outputPath: string, -): Promise { - if (config.thumbnail?.enabled === false) return; - const thumbTime = config.thumbnail?.time ?? 0; - const thumbPath = outputPath.replace(/\.[^.]+$/, ".png"); - const ffmpegPath = await ensureFfmpeg(); - extractThumbnail(ffmpegPath, outputPath, thumbPath, thumbTime); - console.log(`Thumbnail: ${thumbPath}`); -} - export interface RunVideoOptions { record?: boolean; verbose?: boolean; @@ -700,35 +680,15 @@ export async function runVideo( ctx.setTimeline(null); mkdirSync(dirname(outputPath), { recursive: true }); console.log(`Compositing overlays...`); - const zoomFilter = autoZoomCfg.enabled - ? buildAutoZoomFilter( - zoomEvents, - { width: timelineData.width, height: timelineData.height }, - timelineData.zoom ?? 1, - timelineData.frames.length / timelineData.fps, - timelineData.fps, - autoZoomCfg, - ) - : null; - if (zoomFilter) { - console.log( - `Applying autozoom (${zoomEvents.length} event${zoomEvents.length === 1 ? "" : "s"})`, - ); - if (verbose) { - for (const e of zoomEvents) { - const box = e.box; - console.log( - ` t=${(e.timeMs / 1000).toFixed(2)}s box=${box.x.toFixed(0)},${box.y.toFixed(0)} ${box.width.toFixed(0)}x${box.height.toFixed(0)}`, - ); - } - } - } - await compose(rawVideoPath, timelineData, outputPath, { - sfx: config.sfx, - zoomFilter: zoomFilter ?? undefined, + await compositeRecording({ + rawVideoPath, + timelineData, + outputPath, + video: config, + zoomEvents, + verbose, }); } - await extractThumbnailIfConfigured(config, outputPath); console.log(`Done: ${outputPath}`); } else { From 8b2a9684c28ea6cac1c6f5e18b7d0eb0fcd03676 Mon Sep 17 00:00:00 2001 From: Ariel Conti Date: Sat, 18 Jul 2026 15:24:12 +0100 Subject: [PATCH 14/48] refactor(webreel): use compositeRecording in composite command Replace composite.ts's duplicated zoom-filter/compose/thumbnail block with a call to the shared compositeRecording() helper, using the persisted timeline zoomEvents and verbose: false (matching prior behavior, which had no per-event logging). --- packages/webreel/src/commands/composite.ts | 34 ++++++---------------- 1 file changed, 9 insertions(+), 25 deletions(-) diff --git a/packages/webreel/src/commands/composite.ts b/packages/webreel/src/commands/composite.ts index ff0e285..b9acaa3 100644 --- a/packages/webreel/src/commands/composite.ts +++ b/packages/webreel/src/commands/composite.ts @@ -1,14 +1,14 @@ import { Command } from "commander"; import { readFileSync, existsSync, mkdirSync } from "node:fs"; import { resolve, dirname } from "node:path"; -import { buildAutoZoomFilter, compose, type TimelineData } from "@webreel/core"; +import type { TimelineData } from "@webreel/core"; import { loadWebreelConfig, resolveConfigPath, getConfigDir, filterVideosByName, } from "../lib/config.js"; -import { extractThumbnailIfConfigured, normalizeAutoZoom } from "../lib/runner.js"; +import { compositeRecording } from "../lib/compositing.js"; export const compositeCommand = new Command("composite") .description("Re-composite videos from stored raw recordings and timelines") @@ -50,32 +50,16 @@ export const compositeCommand = new Command("composite") mkdirSync(dirname(outputPath), { recursive: true }); console.log(`Compositing: ${video.name}`); - const autoZoomCfg = normalizeAutoZoom(video.autoZoom); const persistedZoomEvents = timelineData.zoomEvents ?? []; - const zoomFilter = - autoZoomCfg.enabled && persistedZoomEvents.length > 0 - ? buildAutoZoomFilter( - persistedZoomEvents, - { width: timelineData.width, height: timelineData.height }, - timelineData.zoom ?? 1, - timelineData.frames.length / timelineData.fps, - timelineData.fps, - autoZoomCfg, - ) - : null; - if (zoomFilter) { - console.log( - `Applying autozoom (${persistedZoomEvents.length} event${persistedZoomEvents.length === 1 ? "" : "s"})`, - ); - } - - await compose(rawPath, timelineData, outputPath, { - sfx: video.sfx, - zoomFilter: zoomFilter ?? undefined, + await compositeRecording({ + rawVideoPath: rawPath, + timelineData, + outputPath, + video, + zoomEvents: persistedZoomEvents, + verbose: false, }); - await extractThumbnailIfConfigured(video, outputPath); - console.log(`Done: ${outputPath}`); } }); From d49b0f59033a5ead1b33bb72b49591c936aeda96 Mon Sep 17 00:00:00 2001 From: Ariel Conti Date: Sat, 18 Jul 2026 15:25:23 +0100 Subject: [PATCH 15/48] refactor(core): unify GIF filter graph between finalizeGif and compositor finalizeGif (raw-recording GIF path) and buildGifConfig (overlay-composited GIF path) built separate filter graphs with different quality settings: only the compositor path used palettegen=stats_mode=full and bayer dithering. Extract the shared graph into buildGifFilter(width, fps?) in media.ts and use it from both call sites, so GIF quality no longer depends on which code path produced it. User-visible: finalizeGif output now uses full-stats palette generation and bayer dithering, matching the higher quality the compositor path already had. --- .../@webreel/core/src/__tests__/media.test.ts | 21 ++++++++++++++++++- packages/@webreel/core/src/compositor.ts | 12 ++--------- packages/@webreel/core/src/media.ts | 15 ++++++++++++- 3 files changed, 36 insertions(+), 12 deletions(-) diff --git a/packages/@webreel/core/src/__tests__/media.test.ts b/packages/@webreel/core/src/__tests__/media.test.ts index c8f372c..4ec8545 100644 --- a/packages/@webreel/core/src/__tests__/media.test.ts +++ b/packages/@webreel/core/src/__tests__/media.test.ts @@ -1,7 +1,12 @@ import { describe, it, expect } from "vitest"; import { resolve, dirname } from "node:path"; import { fileURLToPath } from "node:url"; -import { resolveSfxPath, ensureSoundAssets, buildAudioMixArgs } from "../media.js"; +import { + resolveSfxPath, + ensureSoundAssets, + buildAudioMixArgs, + buildGifFilter, +} from "../media.js"; const __dirname = dirname(fileURLToPath(import.meta.url)); const ASSETS_DIR = resolve(__dirname, "..", "..", "assets"); @@ -112,3 +117,17 @@ describe("buildAudioMixArgs", () => { expect(inputArgs).toContain(clickPath); }); }); + +describe("buildGifFilter", () => { + it("uses stats_mode=full palettegen and bayer dithering (matches compositor quality)", () => { + const filter = buildGifFilter(800); + expect(filter).toBe( + "fps=15,scale=800:-1:flags=lanczos,split[s0][s1];[s0]palettegen=stats_mode=full[p];[s1][p]paletteuse=dither=bayer:bayer_scale=5", + ); + }); + + it("defaults to 15fps and allows overriding fps", () => { + expect(buildGifFilter(640)).toContain("fps=15,"); + expect(buildGifFilter(640, 30)).toContain("fps=30,"); + }); +}); diff --git a/packages/@webreel/core/src/compositor.ts b/packages/@webreel/core/src/compositor.ts index f21cceb..dd1d7eb 100644 --- a/packages/@webreel/core/src/compositor.ts +++ b/packages/@webreel/core/src/compositor.ts @@ -5,7 +5,7 @@ import { resolve, extname } from "node:path"; import sharp from "sharp"; import type { TimelineData } from "./timeline.js"; import { ensureFfmpeg } from "./ffmpeg.js"; -import { finalizeMp4, finalizeWebm, type SfxConfig } from "./media.js"; +import { buildGifFilter, finalizeMp4, finalizeWebm, type SfxConfig } from "./media.js"; interface OverlayContext { cursorPng: Buffer; @@ -217,17 +217,9 @@ function buildMp4Config( }; } -const GIF_FPS = 15; -const GIF_BAYER_SCALE = 5; - function buildGifConfig(width: number, outputPath: string): CompositorFfmpegConfig { return { - filterComplex: [ - `[0][1]overlay=0:0:shortest=1`, - `fps=${GIF_FPS}`, - `scale=${width}:-1:flags=lanczos`, - `split[s0][s1];[s0]palettegen=stats_mode=full[p];[s1][p]paletteuse=dither=bayer:bayer_scale=${GIF_BAYER_SCALE}`, - ].join(","), + filterComplex: `[0][1]overlay=0:0:shortest=1,${buildGifFilter(width)}`, outputArgs: ["-loop", "0", outputPath], }; } diff --git a/packages/@webreel/core/src/media.ts b/packages/@webreel/core/src/media.ts index 50ac4e2..bd8c7eb 100644 --- a/packages/@webreel/core/src/media.ts +++ b/packages/@webreel/core/src/media.ts @@ -224,6 +224,19 @@ export function extractThumbnail( ]); } +export const GIF_FPS = 15; +export const GIF_BAYER_SCALE = 5; + +/** + * Shared GIF quality filter graph: downsample to GIF_FPS, scale to the + * target width with lanczos, then generate a full-stats palette and apply + * bayer dithering when quantizing to it. Used both when finalizing a raw GIF + * recording directly and when compositing overlays onto a GIF output. + */ +export function buildGifFilter(width: number, fps: number = GIF_FPS): string { + return `fps=${fps},scale=${width}:-1:flags=lanczos,split[s0][s1];[s0]palettegen=stats_mode=full[p];[s1][p]paletteuse=dither=bayer:bayer_scale=${GIF_BAYER_SCALE}`; +} + export function finalizeGif( ffmpegPath: string, tempVideo: string, @@ -235,7 +248,7 @@ export function finalizeGif( "-i", tempVideo, "-vf", - `fps=15,scale=${width}:-1:flags=lanczos,split[s0][s1];[s0]palettegen[p];[s1][p]paletteuse`, + buildGifFilter(width), outputPath, ]); } From c185224026bf86c19ab04b94bbcb2a6ff681f901 Mon Sep 17 00:00:00 2001 From: Ariel Conti Date: Sat, 18 Jul 2026 15:27:15 +0100 Subject: [PATCH 16/48] refactor(core): remove dead durationS parameter from autozoom API durationS was threaded through generateZoomKeyframes and buildAutoZoomFilter but never read in either body; both call sites had to compute frames.length / fps just to satisfy the signature. Drop the parameter and update the one remaining call site (lib/compositing.ts) and autozoom.test.ts accordingly. --- .../core/src/__tests__/autozoom.test.ts | 35 +++++++++---------- packages/@webreel/core/src/autozoom.ts | 4 +-- packages/webreel/src/lib/compositing.ts | 1 - 3 files changed, 18 insertions(+), 22 deletions(-) diff --git a/packages/@webreel/core/src/__tests__/autozoom.test.ts b/packages/@webreel/core/src/__tests__/autozoom.test.ts index 0547ca2..662c337 100644 --- a/packages/@webreel/core/src/__tests__/autozoom.test.ts +++ b/packages/@webreel/core/src/__tests__/autozoom.test.ts @@ -97,21 +97,20 @@ describe("generateZoomKeyframes", () => { const kf = generateZoomKeyframes( [{ timeMs: 1000, box: { x: 100, y: 100, width: 200, height: 200 } }], VIEWPORT, - 5, { enabled: false }, ); expect(kf).toEqual([]); }); it("returns empty when no events", () => { - expect(generateZoomKeyframes([], VIEWPORT, 5, cfg)).toEqual([]); + expect(generateZoomKeyframes([], VIEWPORT, cfg)).toEqual([]); }); it("generates approach/settle/hold/release keyframes for a single event", () => { const events: ZoomEvent[] = [ { timeMs: 3000, box: { x: 500, y: 400, width: 200, height: 200 } }, ]; - const kf = generateZoomKeyframes(events, VIEWPORT, 10, cfg); + const kf = generateZoomKeyframes(events, VIEWPORT, cfg); expect(kf.length).toBeGreaterThanOrEqual(5); expect(kf[0].timeS).toBe(0); expect(kf[0].w).toBe(VIEWPORT.width); @@ -131,8 +130,8 @@ describe("generateZoomKeyframes", () => { url: "https://b.test/", }, ]; - const kfBoth = generateZoomKeyframes(events, VIEWPORT, 10, cfg); - const kfFirstOnly = generateZoomKeyframes([events[0]], VIEWPORT, 10, cfg); + const kfBoth = generateZoomKeyframes(events, VIEWPORT, cfg); + const kfFirstOnly = generateZoomKeyframes([events[0]], VIEWPORT, cfg); expect(kfBoth.length).toBe(kfFirstOnly.length); }); @@ -141,7 +140,7 @@ describe("generateZoomKeyframes", () => { { timeMs: 2000, box: { x: 100, y: 100, width: 200, height: 200 } }, { timeMs: 3000, box: { x: 900, y: 500, width: 200, height: 200 } }, ]; - const kf = generateZoomKeyframes(events, VIEWPORT, 10, cfg); + const kf = generateZoomKeyframes(events, VIEWPORT, cfg); const zoomedOuts = kf.filter((k) => k.w === VIEWPORT.width); // Two events, 1s gap (< sessionGapS=1.2) → single session → 3 full-view // keyframes (initial, approach start, final release). @@ -153,7 +152,7 @@ describe("generateZoomKeyframes", () => { { timeMs: 1000, box: { x: 100, y: 100, width: 200, height: 200 } }, { timeMs: 9000, box: { x: 900, y: 500, width: 200, height: 200 } }, ]; - const kf = generateZoomKeyframes(events, VIEWPORT, 15, cfg); + const kf = generateZoomKeyframes(events, VIEWPORT, cfg); const zoomedOuts = kf.filter((k) => k.w === VIEWPORT.width); // Two separate sessions → 5 full-view keyframes (each session adds // approach-start + release, plus one shared initial at t=0). @@ -166,7 +165,7 @@ describe("generateZoomKeyframes", () => { { timeMs: 2000, box: { x: 900, y: 500, width: 200, height: 200 } }, { timeMs: 8000, box: { x: 400, y: 300, width: 200, height: 200 } }, ]; - const kf = generateZoomKeyframes(events, VIEWPORT, 20, cfg); + const kf = generateZoomKeyframes(events, VIEWPORT, cfg); for (let i = 1; i < kf.length; i++) { expect(kf[i].timeS).toBeGreaterThanOrEqual(kf[i - 1].timeS); } @@ -179,7 +178,7 @@ describe("generateZoomKeyframes", () => { { timeMs: 1000, box: { x: 100, y: 100, width: 200, height: 200 } }, { timeMs: 7000, box: { x: 900, y: 500, width: 200, height: 200 } }, ]; - const kf = generateZoomKeyframes(events, VIEWPORT, 12, cfg); + const kf = generateZoomKeyframes(events, VIEWPORT, cfg); const zoomedOuts = kf.filter((k) => k.w === VIEWPORT.width); expect(zoomedOuts.length).toBe(5); }); @@ -193,7 +192,7 @@ describe("generateZoomKeyframes", () => { { timeMs: 2500, box: { x: 200, y: 300, width: 200, height: 50 } }, { timeMs: 5000, box: { x: 1000, y: 600, width: 200, height: 50 } }, ]; - const kf = generateZoomKeyframes(events, VIEWPORT, 10, cfg); + const kf = generateZoomKeyframes(events, VIEWPORT, cfg); const distinctCrops = new Set( kf.map((k) => `${k.x.toFixed(0)},${k.y.toFixed(0)},${k.w.toFixed(0)}`), ); @@ -209,7 +208,7 @@ describe("generateZoomKeyframes", () => { { timeMs: 4500, box: { x: 800, y: 400, width: 200, height: 50 } }, { timeMs: 4900, box: { x: 1500, y: 900, width: 200, height: 50 } }, ]; - const kf = generateZoomKeyframes(events, VIEWPORT, 8, cfg); + const kf = generateZoomKeyframes(events, VIEWPORT, cfg); const lastCropKf = [...kf].reverse().find((k) => k.w !== VIEWPORT.width)!; // B is centered around x=800; C around x=1500. The last kept crop should // be positioned for B, not C — its left edge should be well under 1000. @@ -223,7 +222,7 @@ describe("generateZoomKeyframes", () => { { timeMs: 1000, box: { x: 100, y: 100, width: 800, height: 400 } }, { timeMs: 2000, box: { x: 1500, y: 800, width: 80, height: 30 } }, ]; - const kf = generateZoomKeyframes(events, VIEWPORT, 5, cfg); + const kf = generateZoomKeyframes(events, VIEWPORT, cfg); const sessionCrops = kf.filter((k) => k.w !== VIEWPORT.width); const widths = new Set(sessionCrops.map((k) => k.w)); expect(widths.size).toBe(1); @@ -232,18 +231,18 @@ describe("generateZoomKeyframes", () => { describe("buildAutoZoomFilter", () => { it("returns null when disabled", () => { - expect(buildAutoZoomFilter([], VIEWPORT, 1, 5, 60, { enabled: false })).toBeNull(); + expect(buildAutoZoomFilter([], VIEWPORT, 1, 60, { enabled: false })).toBeNull(); }); it("returns null when no events produce keyframes", () => { - expect(buildAutoZoomFilter([], VIEWPORT, 1, 5, 60, { enabled: true })).toBeNull(); + expect(buildAutoZoomFilter([], VIEWPORT, 1, 60, { enabled: true })).toBeNull(); }); it("emits a zoompan filter string with expected params", () => { const events: ZoomEvent[] = [ { timeMs: 2000, box: { x: 200, y: 200, width: 200, height: 200 } }, ]; - const filter = buildAutoZoomFilter(events, VIEWPORT, 1, 10, 60, { + const filter = buildAutoZoomFilter(events, VIEWPORT, 1, 60, { enabled: true, }); expect(filter).not.toBeNull(); @@ -256,7 +255,7 @@ describe("buildAutoZoomFilter", () => { const events: ZoomEvent[] = [ { timeMs: 2000, box: { x: 200, y: 200, width: 200, height: 200 } }, ]; - const filter = buildAutoZoomFilter(events, VIEWPORT, 1, 10, 60, { + const filter = buildAutoZoomFilter(events, VIEWPORT, 1, 60, { enabled: true, }); expect(filter).not.toBeNull(); @@ -273,10 +272,10 @@ describe("buildAutoZoomFilter", () => { const events: ZoomEvent[] = [ { timeMs: 2000, box: { x: 100, y: 100, width: 500, height: 300 } }, ]; - const filterAt1 = buildAutoZoomFilter(events, VIEWPORT, 1, 10, 60, { + const filterAt1 = buildAutoZoomFilter(events, VIEWPORT, 1, 60, { enabled: true, }); - const filterAt2 = buildAutoZoomFilter(events, VIEWPORT, 2, 10, 60, { + const filterAt2 = buildAutoZoomFilter(events, VIEWPORT, 2, 60, { enabled: true, }); expect(filterAt1).not.toBe(filterAt2); diff --git a/packages/@webreel/core/src/autozoom.ts b/packages/@webreel/core/src/autozoom.ts index c9d5ee5..7ab8ae3 100644 --- a/packages/@webreel/core/src/autozoom.ts +++ b/packages/@webreel/core/src/autozoom.ts @@ -118,7 +118,6 @@ export function computeCropForEvent( export function generateZoomKeyframes( events: ZoomEvent[], viewport: { width: number; height: number }, - durationS: number, userCfg: AutoZoomConfig, ): ZoomKeyframe[] { if (!userCfg.enabled) return []; @@ -270,7 +269,6 @@ export function buildAutoZoomFilter( events: ZoomEvent[], viewport: { width: number; height: number }, cssZoom: number, - durationS: number, fps: number, userCfg: AutoZoomConfig, ): string | null { @@ -286,7 +284,7 @@ export function buildAutoZoomFilter( }, })); - const kf = generateZoomKeyframes(scaled, viewport, durationS, userCfg); + const kf = generateZoomKeyframes(scaled, viewport, userCfg); if (kf.length < 2) return null; if (process.env.WEBREEL_DEBUG_ZOOM) { diff --git a/packages/webreel/src/lib/compositing.ts b/packages/webreel/src/lib/compositing.ts index 22c2a93..bd84d2d 100644 --- a/packages/webreel/src/lib/compositing.ts +++ b/packages/webreel/src/lib/compositing.ts @@ -50,7 +50,6 @@ export async function compositeRecording(opts: CompositeRecordingOptions): Promi zoomEvents, { width: timelineData.width, height: timelineData.height }, timelineData.zoom ?? 1, - timelineData.frames.length / timelineData.fps, timelineData.fps, autoZoomCfg, ) From 3253a9d3a7e7a28052e1d394f0b4b9f1f20109cc Mon Sep 17 00:00:00 2001 From: Ariel Conti Date: Sat, 18 Jul 2026 15:29:02 +0100 Subject: [PATCH 17/48] test(webreel): add compositing.test.ts for compositeRecording helper Covers normalizeAutoZoom edge cases and compositeRecording: compose receives zoomFilter: undefined when autoZoom is disabled, receives the built filter string when enabled with events, and the thumbnail extraction runs after compose. Also merges compositing.ts's two @webreel/core import statements into one. --- .../src/lib/__tests__/compositing.test.ts | 188 ++++++++++++++++++ packages/webreel/src/lib/compositing.ts | 3 +- 2 files changed, 190 insertions(+), 1 deletion(-) create mode 100644 packages/webreel/src/lib/__tests__/compositing.test.ts diff --git a/packages/webreel/src/lib/__tests__/compositing.test.ts b/packages/webreel/src/lib/__tests__/compositing.test.ts new file mode 100644 index 0000000..743d534 --- /dev/null +++ b/packages/webreel/src/lib/__tests__/compositing.test.ts @@ -0,0 +1,188 @@ +import { describe, it, expect, vi, afterEach } from "vitest"; +import type { TimelineData } from "@webreel/core"; + +vi.mock("@webreel/core", async () => { + const actual = await vi.importActual("@webreel/core"); + return { + ...actual, + compose: vi.fn(), + buildAutoZoomFilter: vi.fn(), + ensureFfmpeg: vi.fn(), + extractThumbnail: vi.fn(), + }; +}); + +import { + compose, + buildAutoZoomFilter, + ensureFfmpeg, + extractThumbnail, +} from "@webreel/core"; +import { + compositeRecording, + extractThumbnailIfConfigured, + normalizeAutoZoom, +} from "../compositing.js"; + +const mockedCompose = vi.mocked(compose); +const mockedBuildAutoZoomFilter = vi.mocked(buildAutoZoomFilter); +const mockedEnsureFfmpeg = vi.mocked(ensureFfmpeg); +const mockedExtractThumbnail = vi.mocked(extractThumbnail); + +const timelineData: TimelineData = { + fps: 30, + width: 1920, + height: 1080, + zoom: 1, + frames: [], + events: [], + steps: [], + theme: { cursorSvg: "", cursorSize: 24 }, +} as unknown as TimelineData; + +describe("normalizeAutoZoom", () => { + it("returns enabled:true for boolean true", () => { + expect(normalizeAutoZoom(true)).toEqual({ enabled: true }); + }); + + it("returns enabled:false for boolean false", () => { + expect(normalizeAutoZoom(false)).toEqual({ enabled: false }); + }); + + it("returns enabled:false for undefined", () => { + expect(normalizeAutoZoom(undefined)).toEqual({ enabled: false }); + }); + + it("defaults enabled to true for an object without it (e.g. from parsed JSON config)", () => { + // AutoZoomConfig.enabled is typed as required, but real config files + // loaded via JSON.parse aren't checked against that at runtime, so + // normalizeAutoZoom must tolerate a missing `enabled` field. + const looselyTyped = { approachS: 1 } as unknown as { enabled: boolean }; + expect(normalizeAutoZoom(looselyTyped)).toEqual({ enabled: true, approachS: 1 }); + }); + + it("preserves an explicit enabled:false in an object", () => { + expect(normalizeAutoZoom({ enabled: false, approachS: 1 })).toEqual({ + enabled: false, + approachS: 1, + }); + }); +}); + +describe("compositeRecording", () => { + afterEach(() => { + vi.clearAllMocks(); + }); + + it("calls compose with zoomFilter: undefined when autoZoom is disabled", async () => { + mockedEnsureFfmpeg.mockResolvedValue("/bin/ffmpeg"); + await compositeRecording({ + rawVideoPath: "/raw/video.mp4", + timelineData, + outputPath: "/out/video.mp4", + video: { autoZoom: false }, + zoomEvents: [], + verbose: false, + }); + + expect(mockedBuildAutoZoomFilter).not.toHaveBeenCalled(); + expect(mockedCompose).toHaveBeenCalledTimes(1); + expect(mockedCompose).toHaveBeenCalledWith( + "/raw/video.mp4", + timelineData, + "/out/video.mp4", + expect.objectContaining({ zoomFilter: undefined }), + ); + }); + + it("passes a built filter string to compose when enabled and events exist, then extracts the thumbnail after compose", async () => { + mockedBuildAutoZoomFilter.mockReturnValue("zoompan=z='1'"); + mockedEnsureFfmpeg.mockResolvedValue("/bin/ffmpeg"); + const calls: string[] = []; + mockedCompose.mockImplementation(async () => { + calls.push("compose"); + }); + mockedExtractThumbnail.mockImplementation(() => { + calls.push("thumbnail"); + }); + + const zoomEvents = [{ timeMs: 1000, box: { x: 0, y: 0, width: 100, height: 100 } }]; + + await compositeRecording({ + rawVideoPath: "/raw/video.mp4", + timelineData, + outputPath: "/out/video.mp4", + video: { autoZoom: true }, + zoomEvents, + verbose: false, + }); + + expect(mockedBuildAutoZoomFilter).toHaveBeenCalledWith( + zoomEvents, + { width: timelineData.width, height: timelineData.height }, + timelineData.zoom, + timelineData.fps, + { enabled: true }, + ); + expect(mockedCompose).toHaveBeenCalledWith( + "/raw/video.mp4", + timelineData, + "/out/video.mp4", + expect.objectContaining({ zoomFilter: "zoompan=z='1'" }), + ); + expect(mockedExtractThumbnail).toHaveBeenCalledTimes(1); + expect(calls).toEqual(["compose", "thumbnail"]); + }); + + it("skips extractThumbnail work when thumbnail.enabled is false", async () => { + mockedEnsureFfmpeg.mockResolvedValue("/bin/ffmpeg"); + await compositeRecording({ + rawVideoPath: "/raw/video.mp4", + timelineData, + outputPath: "/out/video.mp4", + video: { autoZoom: false, thumbnail: { enabled: false } }, + zoomEvents: [], + verbose: false, + }); + + expect(mockedEnsureFfmpeg).not.toHaveBeenCalled(); + expect(mockedExtractThumbnail).not.toHaveBeenCalled(); + }); +}); + +describe("extractThumbnailIfConfigured", () => { + afterEach(() => { + vi.clearAllMocks(); + }); + + it("does nothing when thumbnail.enabled is false", async () => { + await extractThumbnailIfConfigured( + { thumbnail: { enabled: false } }, + "/out/video.mp4", + ); + expect(mockedEnsureFfmpeg).not.toHaveBeenCalled(); + expect(mockedExtractThumbnail).not.toHaveBeenCalled(); + }); + + it("extracts a thumbnail at the configured time, deriving a .png path", async () => { + mockedEnsureFfmpeg.mockResolvedValue("/bin/ffmpeg"); + await extractThumbnailIfConfigured({ thumbnail: { time: 2.5 } }, "/out/video.mp4"); + expect(mockedExtractThumbnail).toHaveBeenCalledWith( + "/bin/ffmpeg", + "/out/video.mp4", + "/out/video.png", + 2.5, + ); + }); + + it("defaults the thumbnail time to 0 when unset", async () => { + mockedEnsureFfmpeg.mockResolvedValue("/bin/ffmpeg"); + await extractThumbnailIfConfigured({}, "/out/video.mp4"); + expect(mockedExtractThumbnail).toHaveBeenCalledWith( + "/bin/ffmpeg", + "/out/video.mp4", + "/out/video.png", + 0, + ); + }); +}); diff --git a/packages/webreel/src/lib/compositing.ts b/packages/webreel/src/lib/compositing.ts index bd84d2d..5a66eb1 100644 --- a/packages/webreel/src/lib/compositing.ts +++ b/packages/webreel/src/lib/compositing.ts @@ -1,7 +1,8 @@ -import { ensureFfmpeg, extractThumbnail } from "@webreel/core"; import { buildAutoZoomFilter, compose, + ensureFfmpeg, + extractThumbnail, type AutoZoomConfig, type TimelineData, type ZoomEvent, From 7009bcf2049cea7125646de238aa20f669c6ee13 Mon Sep 17 00:00:00 2001 From: Ariel Conti Date: Sat, 18 Jul 2026 15:29:47 +0100 Subject: [PATCH 18/48] fix(record): queue watch-mode re-records instead of overlapping runs Debounced file-change events could start a second runVideo pass while a prior recording was still in flight, racing two headless Chromes and ffmpeg pipelines on the same output. Add a rerunRequested latch so a change during an in-flight run is queued and replayed exactly once, with config reloaded fresh, after the current run finishes. --- .../src/commands/__tests__/record.test.ts | 168 +++++++++++++++++- packages/webreel/src/commands/record.ts | 14 ++ 2 files changed, 181 insertions(+), 1 deletion(-) diff --git a/packages/webreel/src/commands/__tests__/record.test.ts b/packages/webreel/src/commands/__tests__/record.test.ts index 35a7df9..d96f6f1 100644 --- a/packages/webreel/src/commands/__tests__/record.test.ts +++ b/packages/webreel/src/commands/__tests__/record.test.ts @@ -1,8 +1,33 @@ -import { describe, it, expect } from "vitest"; +import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; import { resolve, dirname } from "node:path"; import { collectIncludePaths } from "../record.js"; import type { WebreelConfig } from "../../lib/types.js"; +vi.mock("../../lib/config.js", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + resolveConfigPath: vi.fn(() => "/fake/webreel.config.json"), + loadWebreelConfig: vi.fn(), + }; +}); + +vi.mock("../../lib/runner.js", () => ({ + runVideo: vi.fn(), +})); + +vi.mock("../../lib/signals.js", () => ({ + installSignalHandlers: vi.fn(() => vi.fn()), +})); + +vi.mock("node:fs", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + watch: vi.fn(), + }; +}); + describe("collectIncludePaths", () => { const configPath = "/project/webreel.config.json"; const configDir = dirname(configPath); @@ -94,3 +119,144 @@ describe("collectIncludePaths", () => { expect(result).toEqual([resolve(configDir, "steps/setup.json")]); }); }); + +describe("watch mode re-record serialization", () => { + function makeWatchConfig(): WebreelConfig { + return { + videos: [{ name: "test", url: "https://example.com", steps: [] }], + }; + } + + let onFileChange: (() => void) | undefined; + + beforeEach(async () => { + vi.useFakeTimers(); + onFileChange = undefined; + + const { watch } = await import("node:fs"); + vi.mocked(watch).mockImplementation(((_path: unknown, cb: () => void) => { + onFileChange = cb; + return { close: vi.fn() } as unknown as ReturnType; + }) as typeof watch); + + const { loadWebreelConfig } = await import("../../lib/config.js"); + vi.mocked(loadWebreelConfig).mockResolvedValue(makeWatchConfig()); + + const { runVideo } = await import("../../lib/runner.js"); + vi.mocked(runVideo).mockReset(); + vi.mocked(runVideo).mockResolvedValue(undefined); + }); + + afterEach(() => { + vi.useRealTimers(); + vi.clearAllMocks(); + }); + + async function startWatch() { + const { recordCommand } = await import("../record.js"); + const runPromise = recordCommand.parseAsync(["--watch"], { from: "user" }); + await runPromise; + if (!onFileChange) throw new Error("onFileChange was never captured by watch()"); + return onFileChange; + } + + it("does not start a second run while one is in-flight (no overlap)", async () => { + const { runVideo } = await import("../../lib/runner.js"); + const change = await startWatch(); + // startWatch() already triggers one runVideo call for the initial + // (pre-watch) recording pass; count re-record calls relative to that. + const callsBefore = vi.mocked(runVideo).mock.calls.length; + + let resolveSlowRun: (() => void) | undefined; + vi.mocked(runVideo).mockImplementation( + () => + new Promise((resolve) => { + resolveSlowRun = resolve; + }), + ); + + change(); // triggers the first re-record + await vi.advanceTimersByTimeAsync(300); + expect(runVideo).toHaveBeenCalledTimes(callsBefore + 1); // only the queued re-record started + + // A second change arrives mid-run. + change(); + await vi.advanceTimersByTimeAsync(300); + expect(runVideo).toHaveBeenCalledTimes(callsBefore + 1); // still just the one in-flight run + + resolveSlowRun?.(); + await vi.advanceTimersByTimeAsync(0); + }); + + it("queues exactly one rerun for N>1 changes during a run", async () => { + const { runVideo } = await import("../../lib/runner.js"); + const change = await startWatch(); + const callsBefore = vi.mocked(runVideo).mock.calls.length; + + let resolveSlowRun: (() => void) | undefined; + vi.mocked(runVideo).mockImplementation( + () => + new Promise((resolve) => { + resolveSlowRun = resolve; + }), + ); + + change(); + await vi.advanceTimersByTimeAsync(300); + expect(runVideo).toHaveBeenCalledTimes(callsBefore + 1); + + // Three more changes arrive while the run is in-flight. + change(); + await vi.advanceTimersByTimeAsync(300); + change(); + await vi.advanceTimersByTimeAsync(300); + change(); + await vi.advanceTimersByTimeAsync(300); + expect(runVideo).toHaveBeenCalledTimes(callsBefore + 1); // still no overlap + + // Once the in-flight run resolves, the queued rerun starts exactly once. + vi.mocked(runVideo).mockResolvedValue(undefined); + resolveSlowRun?.(); + await vi.advanceTimersByTimeAsync(0); // let the finally block's onFileChange() schedule + await vi.advanceTimersByTimeAsync(300); // let the follow-up debounce fire + + expect(runVideo).toHaveBeenCalledTimes(callsBefore + 2); + + // Further timer advances shouldn't start additional runs. + await vi.advanceTimersByTimeAsync(1000); + expect(runVideo).toHaveBeenCalledTimes(callsBefore + 2); + }); + + it("reloads config for the queued rerun", async () => { + const { loadWebreelConfig } = await import("../../lib/config.js"); + const { runVideo } = await import("../../lib/runner.js"); + const change = await startWatch(); + + const loadCallsBefore = vi.mocked(loadWebreelConfig).mock.calls.length; + const runCallsBefore = vi.mocked(runVideo).mock.calls.length; + + let resolveSlowRun: (() => void) | undefined; + vi.mocked(runVideo).mockImplementation( + () => + new Promise((resolve) => { + resolveSlowRun = resolve; + }), + ); + + change(); + await vi.advanceTimersByTimeAsync(300); + expect(loadWebreelConfig).toHaveBeenCalledTimes(loadCallsBefore + 1); + + change(); // queued while the first re-record is in-flight + await vi.advanceTimersByTimeAsync(300); + + vi.mocked(runVideo).mockResolvedValue(undefined); + resolveSlowRun?.(); + await vi.advanceTimersByTimeAsync(0); + await vi.advanceTimersByTimeAsync(300); + + // The queued rerun must reload config, not reuse the stale closure value. + expect(loadWebreelConfig).toHaveBeenCalledTimes(loadCallsBefore + 2); + expect(runVideo).toHaveBeenCalledTimes(runCallsBefore + 2); + }); +}); diff --git a/packages/webreel/src/commands/record.ts b/packages/webreel/src/commands/record.ts index a8cc741..c8b43ec 100644 --- a/packages/webreel/src/commands/record.ts +++ b/packages/webreel/src/commands/record.ts @@ -119,6 +119,7 @@ export const recordCommand = new Command("record") console.log("\nWatching for changes..."); let timer: ReturnType | null = null; let recordingInProgress: Promise | null = null; + let rerunRequested = false; const watchers: FSWatcher[] = []; const closeAllWatchers = () => { @@ -139,6 +140,15 @@ export const recordCommand = new Command("record") timer = setTimeout(async () => { timer = null; + + if (recordingInProgress) { + rerunRequested = true; + console.log( + "\nChange detected, queueing re-record until current recording finishes.", + ); + return; + } + console.log("\nRe-recording..."); let latestConfig: WebreelConfig | null = null; const run = (async () => { @@ -158,6 +168,10 @@ export const recordCommand = new Command("record") } finally { recordingInProgress = null; setupWatchers(latestConfig ?? webreelConfig); + if (rerunRequested) { + rerunRequested = false; + onFileChange(); + } } })(); recordingInProgress = run; From 3fca3d47ee5c0640340c2b75a206995213544e4f Mon Sep 17 00:00:00 2001 From: Ariel Conti Date: Sat, 18 Jul 2026 15:30:37 +0100 Subject: [PATCH 19/48] ci: record example videos in CI to verify the real pipeline --- .github/workflows/ci.yml | 45 +++++++++++ scripts/assert-video.sh | 151 +++++++++++++++++++++++++++++++++++++ scripts/record-examples.sh | 40 +++++++++- 3 files changed, 232 insertions(+), 4 deletions(-) create mode 100755 scripts/assert-video.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 81222d8..ec31266 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -35,3 +35,48 @@ jobs: - name: Test run: pnpm test + + record: + needs: ci + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: pnpm/action-setup@v4 + + - uses: actions/setup-node@v4 + with: + node-version: 22 + cache: pnpm + + - run: pnpm install --frozen-lockfile + + - name: Install ffmpeg + run: | + sudo apt-get update && sudo apt-get install -y ffmpeg + echo "FFMPEG_PATH=$(which ffmpeg)" >> "$GITHUB_ENV" + + - name: Cache Chrome for Testing + uses: actions/cache@v4 + with: + path: ~/.webreel + key: ${{ runner.os }}-webreel-${{ hashFiles('pnpm-lock.yaml') }} + restore-keys: | + ${{ runner.os }}-webreel- + + - name: Build + run: pnpm build + + - name: Record example videos + env: + SKIP_SYNC: "1" + EXAMPLES_FILTER: "hello-world form-filling gif-output" + run: bash scripts/record-examples.sh + + - name: Upload recordings on failure + if: failure() + uses: actions/upload-artifact@v4 + with: + name: recorded-examples + path: examples/*/videos/ + retention-days: 7 diff --git a/scripts/assert-video.sh b/scripts/assert-video.sh new file mode 100755 index 0000000..148f963 --- /dev/null +++ b/scripts/assert-video.sh @@ -0,0 +1,151 @@ +#!/usr/bin/env bash +# Asserts that an example produced a valid recorded video/gif output. +# +# Usage: assert-video.sh +# +# Checks every file under /videos/: +# - the file exists and is larger than 20KB +# - ffprobe (or `ffmpeg -i` as a fallback) reports a video stream with a +# nonzero duration +# +# Resolves the same ffmpeg binary the CLI would use: FFMPEG_PATH env var, +# then the ~/.webreel cache, then whatever `ffmpeg`/`ffprobe` is on PATH. +set -uo pipefail + +EXAMPLE_DIR="${1:?Usage: assert-video.sh }" +EXAMPLE_NAME="${2:?Usage: assert-video.sh }" +VIDEOS_DIR="$EXAMPLE_DIR/videos" +MIN_BYTES=20480 # 20KB + +find_in_webreel_cache() { + local name="$1" + local cache_dir="$HOME/.webreel/bin/ffmpeg" + [ -d "$cache_dir" ] || return 1 + + if [ -x "$cache_dir/$name" ]; then + echo "$cache_dir/$name" + return 0 + fi + + local found + found="$(find "$cache_dir" -type f -name "$name" -perm -u+x 2>/dev/null | head -n1)" + if [ -n "$found" ]; then + echo "$found" + return 0 + fi + return 1 +} + +resolve_ffprobe() { + # Prefer a real ffprobe if one is available (most accurate stream parsing). + if command -v ffprobe >/dev/null 2>&1; then + command -v ffprobe + return 0 + fi + return 1 +} + +resolve_ffmpeg() { + if [ -n "${FFMPEG_PATH:-}" ]; then + echo "$FFMPEG_PATH" + return 0 + fi + if found="$(find_in_webreel_cache ffmpeg)"; then + echo "$found" + return 0 + fi + if command -v ffmpeg >/dev/null 2>&1; then + command -v ffmpeg + return 0 + fi + return 1 +} + +FFPROBE_BIN="$(resolve_ffprobe || true)" +FFMPEG_BIN="$(resolve_ffmpeg || true)" + +if [ -z "$FFPROBE_BIN" ] && [ -z "$FFMPEG_BIN" ]; then + echo "[$EXAMPLE_NAME] assert-video: no ffprobe or ffmpeg available to inspect output." >&2 + exit 1 +fi + +check_stream_and_duration() { + local file="$1" + if [ -n "$FFPROBE_BIN" ]; then + local duration + duration="$("$FFPROBE_BIN" -v error -select_streams v:0 -show_entries stream=codec_type \ + -show_entries format=duration -of default=noprint_wrappers=1 "$file" 2>/dev/null)" + if ! echo "$duration" | grep -q "codec_type=video"; then + echo "[$EXAMPLE_NAME] assert-video: $file has no video stream (ffprobe)." >&2 + return 1 + fi + local dur_value + dur_value="$(echo "$duration" | sed -n 's/^duration=//p' | head -n1)" + if [ -z "$dur_value" ] || [ "$dur_value" = "N/A" ]; then + echo "[$EXAMPLE_NAME] assert-video: $file has no duration reported (ffprobe)." >&2 + return 1 + fi + if ! awk -v d="$dur_value" 'BEGIN { exit !(d > 0) }'; then + echo "[$EXAMPLE_NAME] assert-video: $file has non-positive duration ($dur_value)." >&2 + return 1 + fi + return 0 + fi + + # Fallback: parse `ffmpeg -i` stderr output. + local info + info="$("$FFMPEG_BIN" -i "$file" 2>&1 || true)" + if ! echo "$info" | grep -q "Stream.*Video:"; then + echo "[$EXAMPLE_NAME] assert-video: $file has no video stream (ffmpeg -i)." >&2 + return 1 + fi + local dur_line + dur_line="$(echo "$info" | grep -o "Duration: [0-9:.]*" | head -n1 | sed 's/Duration: //')" + if [ -z "$dur_line" ] || [ "$dur_line" = "N/A" ]; then + echo "[$EXAMPLE_NAME] assert-video: $file has no duration reported (ffmpeg -i)." >&2 + return 1 + fi + if [ "$dur_line" = "00:00:00.00" ]; then + echo "[$EXAMPLE_NAME] assert-video: $file has zero duration." >&2 + return 1 + fi + return 0 +} + +if [ ! -d "$VIDEOS_DIR" ]; then + echo "[$EXAMPLE_NAME] assert-video: videos dir not found: $VIDEOS_DIR" >&2 + exit 1 +fi + +# Only check actual video/gif outputs. The harness also drops a thumbnail +# .png alongside each recording, which is not a video and has no duration. +FILES=("$VIDEOS_DIR"/*.mp4 "$VIDEOS_DIR"/*.webm "$VIDEOS_DIR"/*.gif "$VIDEOS_DIR"/*.mov "$VIDEOS_DIR"/*.mkv) +FOUND_ANY=0 +for f in "${FILES[@]}"; do + [ -f "$f" ] && FOUND_ANY=1 && break +done +if [ "$FOUND_ANY" -eq 0 ]; then + echo "[$EXAMPLE_NAME] assert-video: no video/gif output files found in $VIDEOS_DIR" >&2 + exit 1 +fi + +STATUS=0 +for file in "${FILES[@]}"; do + [ -f "$file" ] || continue + + size=$(wc -c <"$file" | tr -d ' ') + if [ "$size" -le "$MIN_BYTES" ]; then + echo "[$EXAMPLE_NAME] assert-video: $file is only $size bytes (need > $MIN_BYTES)." >&2 + STATUS=1 + continue + fi + + if ! check_stream_and_duration "$file"; then + STATUS=1 + continue + fi + + echo "[$EXAMPLE_NAME] assert-video: $file OK ($size bytes)." +done + +exit $STATUS diff --git a/scripts/record-examples.sh b/scripts/record-examples.sh index 3d2d79a..77f49b9 100755 --- a/scripts/record-examples.sh +++ b/scripts/record-examples.sh @@ -15,6 +15,29 @@ FAILED=() PASSED=0 TOTAL=0 +# EXAMPLES_FILTER: optional space- or comma-separated list of example names. +# When set, only those examples are recorded. Unset/empty records all examples +# (current behavior). +FILTER_LIST=() +if [ -n "${EXAMPLES_FILTER:-}" ]; then + # Normalize commas to spaces, then split on whitespace. + IFS=', ' read -r -a FILTER_LIST <<<"${EXAMPLES_FILTER//,/ }" +fi + +is_selected() { + local name="$1" + if [ ${#FILTER_LIST[@]} -eq 0 ]; then + return 0 + fi + local candidate + for candidate in "${FILTER_LIST[@]}"; do + if [ "$candidate" = "$name" ]; then + return 0 + fi + done + return 1 +} + for dir in "$EXAMPLES_DIR"/*/; do config="$dir/webreel.config.json" example="$(basename "$dir")" @@ -23,11 +46,15 @@ for dir in "$EXAMPLES_DIR"/*/; do continue fi + if ! is_selected "$example"; then + continue + fi + TOTAL=$((TOTAL + 1)) echo "" echo "--- Recording: $example ---" - if (cd "$dir" && node "$WEBREEL" record); then + if (cd "$dir" && node "$WEBREEL" record) && bash "$SCRIPT_DIR/assert-video.sh" "$dir" "$example"; then echo "[$example] Done." PASSED=$((PASSED + 1)) else @@ -44,7 +71,12 @@ if [ ${#FAILED[@]} -gt 0 ]; then exit 1 else echo "All examples recorded successfully." - echo "" - echo "Syncing to docs app..." - bash "$SCRIPT_DIR/sync-examples.sh" + if [ "${SKIP_SYNC:-}" = "1" ]; then + echo "" + echo "SKIP_SYNC=1 set; skipping sync-examples.sh." + else + echo "" + echo "Syncing to docs app..." + bash "$SCRIPT_DIR/sync-examples.sh" + fi fi From c5e893c9b0590adaccd2087841cbe3f06112a6a5 Mon Sep 17 00:00:00 2001 From: Ariel Conti Date: Sat, 18 Jul 2026 15:30:46 +0100 Subject: [PATCH 20/48] fix(core): add sha256 digest verification and URL allowlisting to downloadFile downloadFile now hashes bytes while streaming to disk and can reject on a mismatch with the expected sha256, deleting the partial file. Adds assertTrustedUrl(url, allowedHosts) to enforce https + host allowlisting for any URL sourced from a remote manifest/API response. --- packages/@webreel/core/src/download.ts | 186 ++++++++++++++++++++++++- 1 file changed, 179 insertions(+), 7 deletions(-) diff --git a/packages/@webreel/core/src/download.ts b/packages/@webreel/core/src/download.ts index dc8f745..cfe7b92 100644 --- a/packages/@webreel/core/src/download.ts +++ b/packages/@webreel/core/src/download.ts @@ -1,8 +1,17 @@ -import { createWriteStream, mkdirSync, unlinkSync, chmodSync } from "node:fs"; -import { resolve } from "node:path"; +import { + createWriteStream, + mkdirSync, + unlinkSync, + chmodSync, + realpathSync, + rmSync, + readdirSync, +} from "node:fs"; +import { resolve, relative, isAbsolute, sep } from "node:path"; import { execFileSync } from "node:child_process"; import { Readable } from "node:stream"; import { pipeline } from "node:stream/promises"; +import { createHash } from "node:crypto"; const MAX_RETRIES = 3; const RETRY_DELAY_MS = 1000; @@ -27,6 +36,33 @@ async function withRetry(fn: () => Promise, label: string): Promise { ); } +/** + * Throws unless `url` uses https: and its hostname is one of `allowedHosts`. + * Callers should validate the ORIGINAL url returned by a remote manifest/API + * response before following it — if fetch() follows a redirect to a + * different host (e.g. a signed CDN URL), that redirect target is not + * separately validated here. That's an accepted gap: the original host is + * the one we trust to hand out download links in the first place. + */ +export function assertTrustedUrl(url: string, allowedHosts: string[]): void { + let parsed: URL; + try { + parsed = new URL(url); + } catch { + throw new Error(`Refusing to fetch invalid URL: ${url}`); + } + if (parsed.protocol !== "https:") { + throw new Error( + `Refusing to fetch non-https URL: ${url} (protocol ${parsed.protocol})`, + ); + } + if (!allowedHosts.includes(parsed.hostname)) { + throw new Error( + `Refusing to fetch URL from untrusted host: ${parsed.hostname} (expected one of ${allowedHosts.join(", ")})`, + ); + } +} + export async function fetchJson(url: string): Promise { const res = await withRetry(async () => { const r = await fetch(url); @@ -40,6 +76,7 @@ export async function downloadFile( url: string, destPath: string, label: string, + expectedSha256?: string, ): Promise { console.log(`Downloading ${label}... (one-time setup)`); @@ -50,17 +87,149 @@ export async function downloadFile( if (!res.ok) throw new Error(`HTTP ${res.status} fetching ${url}`); if (!res.body) throw new Error(`Empty response body from ${url}`); + const hash = createHash("sha256"); const ws = createWriteStream(destPath); - await pipeline( - Readable.fromWeb(res.body as import("node:stream/web").ReadableStream), - ws, - ); + const source = Readable.fromWeb(res.body as import("node:stream/web").ReadableStream); + source.on("data", (chunk) => hash.update(chunk)); + await pipeline(source, ws); + + if (expectedSha256) { + const actual = hash.digest("hex"); + if (actual.toLowerCase() !== expectedSha256.toLowerCase()) { + unlinkSync(destPath); + throw new Error( + `Checksum mismatch for ${label}: expected sha256 ${expectedSha256}, got ${actual}. ` + + `The download may be corrupted or tampered with — retry, or set the corresponding ` + + `*_PATH env var (e.g. CHROME_PATH / FFMPEG_PATH) to use a local binary instead.`, + ); + } + } }, `Download ${label}`); } +/** + * Parses `tar -tf` output (one bare entry path per line) as well as the + * plain newline-separated entry listing we generate for zips on Windows. + */ +export function parseTarListing(output: string): string[] { + return output + .split("\n") + .map((line) => line.trim()) + .filter((line) => line.length > 0); +} + +/** + * Parses `unzip -l` output, e.g.: + * Archive: test.zip + * Length Date Time Name + * --------- ---------- ----- ---- + * 0 01-01-2020 00:00 folder/ + * 10 01-01-2020 00:00 folder/file.txt + * --------- ------- + * 10 2 files + */ +export function parseZipListing(output: string): string[] { + const entries: string[] = []; + const entryLineRe = /^\s*\d+\s+\S+\s+\S+\s+(.+?)\s*$/; + for (const line of output.split("\n")) { + if (/^Archive:/.test(line)) continue; + if (/^\s*Length\s+Date\s+Time\s+Name\s*$/.test(line)) continue; + if (/^-+\s+-+\s+-+\s+-+\s*$/.test(line)) continue; + if (/^\s*\d+\s+\d+\s+files?\s*$/.test(line)) continue; + const m = line.match(entryLineRe); + if (m) entries.push(m[1]); + } + return entries; +} + +/** + * True if an archive entry path would escape the extraction directory: + * an absolute path (unix or Windows-drive-letter form), or a path with a + * ".." path segment. + */ +export function isUnsafeEntryPath(entryPath: string): boolean { + if (!entryPath) return false; + const normalized = entryPath.replace(/\\/g, "/"); + if (normalized.startsWith("/")) return true; + if (/^[a-zA-Z]:[\\/]/.test(entryPath)) return true; + return normalized.split("/").includes(".."); +} + +export function validateEntryPaths(paths: string[], archiveLabel = "archive"): void { + for (const p of paths) { + if (isUnsafeEntryPath(p)) { + throw new Error(`Refusing to extract ${archiveLabel}: unsafe entry path "${p}"`); + } + } +} + +function listArchiveEntries(archivePath: string): string[] { + if (archivePath.endsWith(".tar.xz")) { + const out = execFileSync("tar", ["-tf", archivePath], { + stdio: ["ignore", "pipe", "pipe"], + }).toString("utf8"); + return parseTarListing(out); + } + if (process.platform === "win32") { + const out = execFileSync( + "powershell", + [ + "-NoProfile", + "-Command", + `Add-Type -AssemblyName System.IO.Compression.FileSystem; $zip = [System.IO.Compression.ZipFile]::OpenRead('${archivePath.replace(/'/g, "''")}'); $zip.Entries | ForEach-Object { $_.FullName }; $zip.Dispose()`, + ], + { stdio: ["ignore", "pipe", "pipe"] }, + ).toString("utf8"); + return parseTarListing(out); + } + const out = execFileSync("unzip", ["-l", archivePath], { + stdio: ["ignore", "pipe", "pipe"], + }).toString("utf8"); + return parseZipListing(out); +} + +/** + * Defense in depth for symlink-based escapes: walks the extracted tree and + * verifies every entry's realpath stays under destDir's realpath. Removes + * destDir and throws on violation. + */ +function verifyExtractedWithinDir(destDir: string): void { + const destReal = realpathSync(destDir); + const stack: string[] = [destDir]; + while (stack.length > 0) { + const dir = stack.pop()!; + let entries; + try { + entries = readdirSync(dir, { withFileTypes: true }); + } catch { + continue; + } + for (const entry of entries) { + const full = resolve(dir, entry.name); + let real: string; + try { + real = realpathSync(full); + } catch { + continue; + } + const rel = relative(destReal, real); + if (rel === ".." || rel.startsWith(`..${sep}`) || isAbsolute(rel)) { + rmSync(destDir, { recursive: true, force: true }); + throw new Error( + `Refusing extracted archive: entry "${full}" resolves outside destination directory ${destReal}`, + ); + } + if (entry.isDirectory()) stack.push(full); + } + } +} + export function extractArchive(archivePath: string, destDir: string): void { mkdirSync(destDir, { recursive: true }); + const entries = listArchiveEntries(archivePath); + validateEntryPaths(entries, archivePath); + if (archivePath.endsWith(".tar.xz")) { execFileSync("tar", ["-xf", archivePath, "-C", destDir], { stdio: "pipe" }); } else if (process.platform === "win32") { @@ -77,18 +246,21 @@ export function extractArchive(archivePath: string, destDir: string): void { stdio: "pipe", }); } + + verifyExtractedWithinDir(destDir); } export async function downloadAndExtract( url: string, destDir: string, label: string, + expectedSha256?: string, ): Promise { mkdirSync(destDir, { recursive: true }); const ext = url.endsWith(".tar.xz") ? ".tar.xz" : ".zip"; const archivePath = resolve(destDir, `_download${ext}`); - await downloadFile(url, archivePath, label); + await downloadFile(url, archivePath, label, expectedSha256); extractArchive(archivePath, destDir); unlinkSync(archivePath); console.log(`${label} ready.`); From e81af8ec8ac5c6a9dddc59d8487773ecca9e1b6f Mon Sep 17 00:00:00 2001 From: Ariel Conti Date: Sat, 18 Jul 2026 15:30:55 +0100 Subject: [PATCH 21/48] docs: sync README config tables and record flags with shipped surface Diff list from types.ts vs README tables: - Top-level WebreelConfig: missing `clickDwell`, `sfx` - Per-video VideoConfig: missing `fps`, `quality`, `clickDwell`, `sfx` - Record command: missing `--dry-run`, `--frames` flags Added the missing rows/flags to both README.md and packages/webreel/README.md, wording condensed from apps/docs/src/app/configuration/page.mdx, apps/docs/src/app/commands/page.mdx, and skills/webreel/SKILL.md. Claude-Session: https://claude.ai/code/session_01Pv8tMQt9GoVpfnpVcdwFxe --- README.md | 54 ++++++++++++++++++++++---------------- packages/webreel/README.md | 34 ++++++++++++++---------- 2 files changed, 51 insertions(+), 37 deletions(-) diff --git a/README.md b/README.md index 79e357b..c39c615 100644 --- a/README.md +++ b/README.md @@ -116,6 +116,8 @@ webreel record hero login webreel record -c custom.config.json webreel record --watch webreel record --verbose +webreel record --dry-run +webreel record --frames ``` ### Preview @@ -188,32 +190,38 @@ All steps (except `pause`) accept an optional `delay` field (ms to wait after th #### Top-level -| Field | Default | Description | -| -------------- | --------- | -------------------------------------------- | -| `$schema` | - | JSON Schema URL for IDE autocompletion | -| `outDir` | `videos/` | Default output directory for videos | -| `baseUrl` | `""` | Prepended to relative video URLs | -| `viewport` | 1080x1080 | Default browser viewport dimensions | -| `theme` | - | Default cursor and HUD overlay customization | -| `include` | - | Array of step files prepended to all videos | -| `defaultDelay` | - | Default delay (ms) after each step | -| `videos` | required | Object mapping video names to their configs | +| Field | Default | Description | +| -------------- | --------------- | ------------------------------------------------------------ | +| `$schema` | - | JSON Schema URL for IDE autocompletion | +| `outDir` | `videos/` | Default output directory for videos | +| `baseUrl` | `""` | Prepended to relative video URLs | +| `viewport` | 1080x1080 | Default browser viewport dimensions | +| `theme` | - | Default cursor and HUD overlay customization | +| `include` | - | Array of step files prepended to all videos | +| `defaultDelay` | - | Default delay (ms) after each step | +| `clickDwell` | random 80-180ms | Milliseconds the cursor pauses before clicking (0 = instant) | +| `sfx` | - | Sound effects configuration for click and keystroke sounds | +| `videos` | required | Object mapping video names to their configs | #### Per-video -| Field | Default | Description | -| -------------- | ------------- | ------------------------------------------------------ | -| `url` | required | URL to navigate to | -| `baseUrl` | inherited | Prepended to relative URLs | -| `viewport` | inherited | Browser viewport dimensions | -| `zoom` | - | CSS zoom level applied to the page | -| `waitFor` | - | CSS selector to wait for before start | -| `output` | `.mp4` | Output file path (.mp4, .gif, or .webm) | -| `thumbnail` | `{ time: 0 }` | Object with `time` (seconds) or `enabled: false` | -| `include` | inherited | Array of paths to JSON files whose steps are prepended | -| `theme` | inherited | Cursor and HUD overlay customization | -| `defaultDelay` | inherited | Default delay (ms) after each step | -| `autoZoom` | `false` | Cinematic zoom into each action (bool or object) | +| Field | Default | Description | +| -------------- | ------------- | ---------------------------------------------------------- | +| `url` | required | URL to navigate to | +| `baseUrl` | inherited | Prepended to relative URLs | +| `viewport` | inherited | Browser viewport dimensions | +| `zoom` | - | CSS zoom level applied to the page | +| `fps` | `60` | Recording frame rate (1-120) | +| `quality` | `80` | Output quality (1-100); higher values produce larger files | +| `waitFor` | - | CSS selector to wait for before start | +| `output` | `.mp4` | Output file path (.mp4, .gif, or .webm) | +| `thumbnail` | `{ time: 0 }` | Object with `time` (seconds) or `enabled: false` | +| `include` | inherited | Array of paths to JSON files whose steps are prepended | +| `theme` | inherited | Cursor and HUD overlay customization | +| `defaultDelay` | inherited | Default delay (ms) after each step | +| `clickDwell` | inherited | Milliseconds the cursor pauses before clicking | +| `sfx` | inherited | Sound effects configuration for click and keystroke sounds | +| `autoZoom` | `false` | Cinematic zoom into each action (bool or object) | ## Development diff --git a/packages/webreel/README.md b/packages/webreel/README.md index 7a6df4e..bbadb39 100644 --- a/packages/webreel/README.md +++ b/packages/webreel/README.md @@ -101,9 +101,11 @@ Record videos. webreel record webreel record hero login webreel record -c custom.config.json +webreel record --dry-run +webreel record --frames ``` -When run without arguments, webreel reads `webreel.config.json` from the current directory and records all videos. Provide video names to record specific videos only. +When run without arguments, webreel reads `webreel.config.json` from the current directory and records all videos. Provide video names to record specific videos only. Use `--dry-run` to print the fully resolved config and step list without recording, or `--frames` to save raw JPEGs to `.webreel/frames/`. ### `webreel preview` @@ -165,19 +167,23 @@ Raw video and timeline data are saved in `.webreel/raw/` and `.webreel/timelines ### Config options -| Field | Default | Description | -| -------------- | ------------- | --------------------------------------------------------- | -| `url` | required | URL to navigate to | -| `baseUrl` | `""` | Prepended to relative URLs | -| `viewport` | 1080x1080 | Browser viewport dimensions | -| `zoom` | - | CSS zoom level applied to the page | -| `waitFor` | - | CSS selector to wait for before starting | -| `output` | `.mp4` | Output file path (`.mp4`, `.gif`, or `.webm`) | -| `thumbnail` | `{ time: 0 }` | Object with `time` (seconds) or `enabled: false` | -| `theme` | - | Overlay theme (`cursor: { image, size, hotspot }`, `hud`) | -| `include` | - | Array of JSON file paths whose steps are prepended | -| `defaultDelay` | - | Default delay (ms) after each step | -| `autoZoom` | `false` | Cinematic zoom into each action (bool or object) | +| Field | Default | Description | +| -------------- | --------------- | ------------------------------------------------------------ | +| `url` | required | URL to navigate to | +| `baseUrl` | `""` | Prepended to relative URLs | +| `viewport` | 1080x1080 | Browser viewport dimensions | +| `zoom` | - | CSS zoom level applied to the page | +| `fps` | `60` | Recording frame rate (1-120) | +| `quality` | `80` | Output quality (1-100); higher values produce larger files | +| `waitFor` | - | CSS selector to wait for before starting | +| `output` | `.mp4` | Output file path (`.mp4`, `.gif`, or `.webm`) | +| `thumbnail` | `{ time: 0 }` | Object with `time` (seconds) or `enabled: false` | +| `theme` | - | Overlay theme (`cursor: { image, size, hotspot }`, `hud`) | +| `sfx` | - | Sound effects configuration for click and keystroke sounds | +| `include` | - | Array of JSON file paths whose steps are prepended | +| `defaultDelay` | - | Default delay (ms) after each step | +| `clickDwell` | random 80-180ms | Milliseconds the cursor pauses before clicking (0 = instant) | +| `autoZoom` | `false` | Cinematic zoom into each action (bool or object) | ### Actions From c44e62ab9dd92a9bba30c0fcc1dc05c93fae1c53 Mon Sep 17 00:00:00 2001 From: Ariel Conti Date: Sat, 18 Jul 2026 15:32:10 +0100 Subject: [PATCH 22/48] test: add v8 coverage instrumentation to both packages --- packages/@webreel/core/package.json | 1 + packages/@webreel/core/vitest.config.ts | 6 + packages/webreel/package.json | 1 + packages/webreel/vitest.config.ts | 6 + pnpm-lock.yaml | 140 ++++++++++++++++++++++++ 5 files changed, 154 insertions(+) diff --git a/packages/@webreel/core/package.json b/packages/@webreel/core/package.json index 4fca6ad..cbb424c 100644 --- a/packages/@webreel/core/package.json +++ b/packages/@webreel/core/package.json @@ -53,6 +53,7 @@ "devDependencies": { "@types/chrome-remote-interface": "^0.33.0", "@types/node": "25.3.0", + "@vitest/coverage-v8": "4.0.18", "typescript": "5.9.3" } } diff --git a/packages/@webreel/core/vitest.config.ts b/packages/@webreel/core/vitest.config.ts index f612c07..9f2b691 100644 --- a/packages/@webreel/core/vitest.config.ts +++ b/packages/@webreel/core/vitest.config.ts @@ -3,5 +3,11 @@ import { defineConfig } from "vitest/config"; export default defineConfig({ test: { exclude: ["dist/**", "node_modules/**"], + coverage: { + provider: "v8", + reporter: ["text", "json-summary"], + include: ["src/**"], + exclude: ["src/__tests__/**", "dist/**"], + }, }, }); diff --git a/packages/webreel/package.json b/packages/webreel/package.json index c4af5c8..cea1462 100644 --- a/packages/webreel/package.json +++ b/packages/webreel/package.json @@ -58,6 +58,7 @@ }, "devDependencies": { "@types/node": "25.3.0", + "@vitest/coverage-v8": "4.0.18", "typescript": "5.9.3" } } diff --git a/packages/webreel/vitest.config.ts b/packages/webreel/vitest.config.ts index f612c07..9f2b691 100644 --- a/packages/webreel/vitest.config.ts +++ b/packages/webreel/vitest.config.ts @@ -3,5 +3,11 @@ import { defineConfig } from "vitest/config"; export default defineConfig({ test: { exclude: ["dist/**", "node_modules/**"], + coverage: { + provider: "v8", + reporter: ["text", "json-summary"], + include: ["src/**"], + exclude: ["src/__tests__/**", "dist/**"], + }, }, }); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index a024cef..f61a978 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -127,6 +127,9 @@ importers: '@types/node': specifier: 25.3.0 version: 25.3.0 + '@vitest/coverage-v8': + specifier: 4.0.18 + version: 4.0.18(vitest@4.0.18(@types/node@25.3.0)(jiti@2.6.1)(lightningcss@1.31.1)(yaml@2.8.2)) typescript: specifier: 5.9.3 version: 5.9.3 @@ -149,6 +152,9 @@ importers: '@types/node': specifier: 25.3.0 version: 25.3.0 + '@vitest/coverage-v8': + specifier: 4.0.18 + version: 4.0.18(vitest@4.0.18(@types/node@25.3.0)(jiti@2.6.1)(lightningcss@1.31.1)(yaml@2.8.2)) typescript: specifier: 5.9.3 version: 5.9.3 @@ -159,10 +165,31 @@ packages: resolution: {integrity: sha512-UrcABB+4bUrFABwbluTIBErXwvbsU/V7TZWfmbgJfbkwiBuziS9gxdODUyuiecfdGQ85jglMW6juS3+z5TsKLw==} engines: {node: '>=10'} + '@babel/helper-string-parser@7.29.7': + resolution: {integrity: sha512-Pb5ijPrZ89GDH8223L4UP8i6QApWxs04RbPQJTeWDV0/keR2E36MeKnyr6LYmUUvqRRI+Iv87SuF1W6ErINzYw==} + engines: {node: '>=6.9.0'} + + '@babel/helper-validator-identifier@7.29.7': + resolution: {integrity: sha512-qehxGkRj55h/ff8EMaJ+cYhyaKlHIxqYDn682wQD7RNp9UujOQsHog2uS0r2vzr4pW+sXf90NeeayjcNaX3fFg==} + engines: {node: '>=6.9.0'} + + '@babel/parser@7.29.7': + resolution: {integrity: sha512-hnORnjP/1P/zFEndoeX+n+t1RwWRJiJpM/jO7FW32Kn9r5+sJB2JWOdYo4L6k78j15eCwY3Gm/7364B1EMwtNg==} + engines: {node: '>=6.0.0'} + hasBin: true + '@babel/runtime@7.28.6': resolution: {integrity: sha512-05WQkdpL9COIMz4LjTxGpPNCdlpyimKppYNoJ5Di5EUObifl8t4tuLuUBBZEpoLYOmfvIWrsp9fCl0HoPRVTdA==} engines: {node: '>=6.9.0'} + '@babel/types@7.29.7': + resolution: {integrity: sha512-4zBIxpPzowiZpusoFkyGVwakdRJUyuH5PxQ/PrqghfdFWWasvnCdPfQXHrenDai+gyLARulZjZowCOj6fjT4pA==} + engines: {node: '>=6.9.0'} + + '@bcoe/v8-coverage@1.0.2': + resolution: {integrity: sha512-6zABk/ECA/QYSCQ1NGiVwwbQerUCZ+TQbp64Q3AgmfNvurHH0j8TtXa1qbShXA6qqkpAj4V5W8pP6mLe1mcMqA==} + engines: {node: '>=18'} + '@changesets/apply-release-plan@7.0.14': resolution: {integrity: sha512-ddBvf9PHdy2YY0OUiEl3TV78mH9sckndJR14QAt87KLEbIov81XO0q0QAmvooBxXlqRRP8I9B7XOzZwQG7JkWA==} @@ -1233,6 +1260,15 @@ packages: '@ungap/structured-clone@1.3.0': resolution: {integrity: sha512-WmoN8qaIAo7WTYWbAZuG8PYEhn5fkz7dZrqTBZ7dtt//lL2Gwms1IcnQ5yHqjDfX8Ft5j4YzDM23f87zBfDe9g==} + '@vitest/coverage-v8@4.0.18': + resolution: {integrity: sha512-7i+N2i0+ME+2JFZhfuz7Tg/FqKtilHjGyGvoHYQ6iLV0zahbsJ9sljC9OcFcPDbhYKCet+sG8SsVqlyGvPflZg==} + peerDependencies: + '@vitest/browser': 4.0.18 + vitest: 4.0.18 + peerDependenciesMeta: + '@vitest/browser': + optional: true + '@vitest/expect@4.0.18': resolution: {integrity: sha512-8sCWUyckXXYvx4opfzVY03EOiYVxyNrHS5QxX3DAIi5dpJAAkyJezHCP77VMX4HKA2LDT/Jpfo8i2r5BE3GnQQ==} @@ -1313,6 +1349,9 @@ packages: resolution: {integrity: sha512-Izi8RQcffqCeNVgFigKli1ssklIbpHnCYc6AknXGYoB6grJqyeby7jv12JUQgmTAnIDnbck1uxksT4dzN3PWBA==} engines: {node: '>=12'} + ast-v8-to-istanbul@0.3.12: + resolution: {integrity: sha512-BRRC8VRZY2R4Z4lFIL35MwNXmwVqBityvOIwETtsCSwvjl0IdgFsy9NhdaA6j74nUdtJJlIypeRhpDam19Wq3g==} + astring@1.9.0: resolution: {integrity: sha512-LElXdjswlqjWrPpJFg1Fx4wpkOCxj1TDHlSV4PlaRxHGWko024xICaa97ZkMfs6DRKlCguiAI+rbXv5GWwXIkg==} hasBin: true @@ -1665,6 +1704,10 @@ packages: graceful-fs@4.2.11: resolution: {integrity: sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==} + has-flag@4.0.0: + resolution: {integrity: sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==} + engines: {node: '>=8'} + hast-util-to-estree@3.1.3: resolution: {integrity: sha512-48+B/rJWAp0jamNbAAf9M7Uf//UVqAoMmgXhBdxTDJLGKY+LRnZ99qcG+Qjl5HfMpYNzS5v4EAwVEF34LeAj7w==} @@ -1677,6 +1720,9 @@ packages: hast-util-whitespace@3.0.0: resolution: {integrity: sha512-88JUN06ipLwsnv+dVn+OIYOvAuvBMy/Qoi6O7mQHxdPXpjy+Cd6xRkWwux7DKO+4sYILtLBRIKgsdpS2gQc7qw==} + html-escaper@2.0.2: + resolution: {integrity: sha512-H2iMtd0I4Mt5eYiapRdIDjp+XzelXQ0tFE4JS7YFwFevXXMmOp9myNrUvCg0D6ws8iqkRPBfKHgbwig1SmlLfg==} + html-void-elements@3.0.0: resolution: {integrity: sha512-bEqo66MRXsUGxWHV5IP0PUiAWwoEjba4VCzg0LjFJBpchPaTfyfCKTG6bc5F8ucKec3q5y6qOdGyYTSBEvhCrg==} @@ -1751,10 +1797,25 @@ packages: isexe@2.0.0: resolution: {integrity: sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==} + istanbul-lib-coverage@3.2.2: + resolution: {integrity: sha512-O8dpsF+r0WV/8MNRKfnmrtCWhuKjxrq2w+jpzBL5UZKTi2LeVWnWOmWRxFlesJONmc+wLAGvKQZEOanko0LFTg==} + engines: {node: '>=8'} + + istanbul-lib-report@3.0.1: + resolution: {integrity: sha512-GCfE1mtsHGOELCU8e/Z7YWzpmybrx/+dSTfLrvY8qRmaY6zXTKWn6WQIjaAFw069icm6GVMNkgu0NzI4iPZUNw==} + engines: {node: '>=10'} + + istanbul-reports@3.2.0: + resolution: {integrity: sha512-HGYWWS/ehqTV3xN10i23tkPkpH46MLCIMFNCaaKNavAXTF1RkqxawEPtnjnGZ6XKSInBKkiOA5BKS+aZiY3AvA==} + engines: {node: '>=8'} + jiti@2.6.1: resolution: {integrity: sha512-ekilCSN1jwRvIbgeg/57YFh8qQDNbwDb9xT/qu2DAHbFFZUicIl4ygVaAvzveMhMVr3LnpSKTNnwt8PoOfmKhQ==} hasBin: true + js-tokens@10.0.0: + resolution: {integrity: sha512-lM/UBzQmfJRo9ABXbPWemivdCW8V2G8FHaHdypQaIy523snUjog0W71ayWXTjiR+ixeMyVHN2XcpnTd/liPg/Q==} + js-yaml@3.14.2: resolution: {integrity: sha512-PMSmkqxr106Xa156c2M265Z+FTrPl+oxd/rgOQy2tijQeK5TxQ43psO1ZCwhVOSdnn+RzkzlRz/eY4BgJBYVpg==} hasBin: true @@ -1890,6 +1951,13 @@ packages: magic-string@0.30.21: resolution: {integrity: sha512-vd2F4YUyEXKGcLHoq+TEyCjxueSeHnFxyyjNp80yg0XV4vUhnDer/lvvlqM/arB5bXQN5K2/3oinyCRyx8T2CQ==} + magicast@0.5.3: + resolution: {integrity: sha512-pVKE4UdSQ7DvHzivsCIFx2BJn1mHG6KsyrFcaxFx6tONdneEuThrDx0Cj3AMg58KyN4pzYT+LHOotxDQDjNvkw==} + + make-dir@4.0.0: + resolution: {integrity: sha512-hXdUTZYIVOt1Ex//jAQi+wTZZpUpwBj/0QsOzqegb3rGMMeJiSEu5xLHnYfBrRV4RH2+OCSOO95Is/7x1WJ4bw==} + engines: {node: '>=10'} + markdown-extensions@2.0.0: resolution: {integrity: sha512-o5vL7aDWatOTX8LzaS1WMoaoxIiLRQJuIKKe2wAw6IeULDHaqbiqiggmx+pKvZDb1Sj+pE46Sn1T7lCqfFtg1Q==} engines: {node: '>=16'} @@ -2404,6 +2472,10 @@ packages: babel-plugin-macros: optional: true + supports-color@7.2.0: + resolution: {integrity: sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==} + engines: {node: '>=8'} + tailwind-merge@3.5.0: resolution: {integrity: sha512-I8K9wewnVDkL1NTGoqWmVEIlUcB9gFriAEkXkfCjX5ib8ezGxtR3xD7iZIxrfArjEsH7F1CHD4RFUtxefdqV/A==} @@ -2687,8 +2759,23 @@ snapshots: '@alloc/quick-lru@5.2.0': {} + '@babel/helper-string-parser@7.29.7': {} + + '@babel/helper-validator-identifier@7.29.7': {} + + '@babel/parser@7.29.7': + dependencies: + '@babel/types': 7.29.7 + '@babel/runtime@7.28.6': {} + '@babel/types@7.29.7': + dependencies: + '@babel/helper-string-parser': 7.29.7 + '@babel/helper-validator-identifier': 7.29.7 + + '@bcoe/v8-coverage@1.0.2': {} + '@changesets/apply-release-plan@7.0.14': dependencies: '@changesets/config': 3.1.2 @@ -3672,6 +3759,20 @@ snapshots: '@ungap/structured-clone@1.3.0': {} + '@vitest/coverage-v8@4.0.18(vitest@4.0.18(@types/node@25.3.0)(jiti@2.6.1)(lightningcss@1.31.1)(yaml@2.8.2))': + dependencies: + '@bcoe/v8-coverage': 1.0.2 + '@vitest/utils': 4.0.18 + ast-v8-to-istanbul: 0.3.12 + istanbul-lib-coverage: 3.2.2 + istanbul-lib-report: 3.0.1 + istanbul-reports: 3.2.0 + magicast: 0.5.3 + obug: 2.1.1 + std-env: 3.10.0 + tinyrainbow: 3.0.3 + vitest: 4.0.18(@types/node@25.3.0)(jiti@2.6.1)(lightningcss@1.31.1)(yaml@2.8.2) + '@vitest/expect@4.0.18': dependencies: '@standard-schema/spec': 1.1.0 @@ -3750,6 +3851,12 @@ snapshots: assertion-error@2.0.1: {} + ast-v8-to-istanbul@0.3.12: + dependencies: + '@jridgewell/trace-mapping': 0.3.31 + estree-walker: 3.0.3 + js-tokens: 10.0.0 + astring@1.9.0: {} bail@2.0.2: {} @@ -4120,6 +4227,8 @@ snapshots: graceful-fs@4.2.11: {} + has-flag@4.0.0: {} + hast-util-to-estree@3.1.3: dependencies: '@types/estree': 1.0.8 @@ -4179,6 +4288,8 @@ snapshots: dependencies: '@types/hast': 3.0.4 + html-escaper@2.0.2: {} + html-void-elements@3.0.0: {} human-id@4.1.3: {} @@ -4230,8 +4341,23 @@ snapshots: isexe@2.0.0: {} + istanbul-lib-coverage@3.2.2: {} + + istanbul-lib-report@3.0.1: + dependencies: + istanbul-lib-coverage: 3.2.2 + make-dir: 4.0.0 + supports-color: 7.2.0 + + istanbul-reports@3.2.0: + dependencies: + html-escaper: 2.0.2 + istanbul-lib-report: 3.0.1 + jiti@2.6.1: {} + js-tokens@10.0.0: {} + js-yaml@3.14.2: dependencies: argparse: 1.0.10 @@ -4357,6 +4483,16 @@ snapshots: dependencies: '@jridgewell/sourcemap-codec': 1.5.5 + magicast@0.5.3: + dependencies: + '@babel/parser': 7.29.7 + '@babel/types': 7.29.7 + source-map-js: 1.2.1 + + make-dir@4.0.0: + dependencies: + semver: 7.7.4 + markdown-extensions@2.0.0: {} mdast-util-from-markdown@2.0.3: @@ -5109,6 +5245,10 @@ snapshots: client-only: 0.0.1 react: 19.2.4 + supports-color@7.2.0: + dependencies: + has-flag: 4.0.0 + tailwind-merge@3.5.0: {} tailwindcss-animate@1.0.7(tailwindcss@4.2.1): From ebba8cc2d04e12325bc7e8e92613f81b48565a25 Mon Sep 17 00:00:00 2001 From: Ariel Conti Date: Sat, 18 Jul 2026 15:32:21 +0100 Subject: [PATCH 23/48] fix(core): enforce host allowlist on chrome and ffmpeg download URLs Applies assertTrustedUrl to every URL sourced from a remote manifest/API response before it's fetched: the CfT manifest and its per-platform download entries (storage.googleapis.com), the BtbN release asset URL (github.com), and the evermeet.cx API and its zip download URL. Hosts were confirmed against live responses. --- packages/@webreel/core/src/chrome.ts | 18 +++++++++++++++++- packages/@webreel/core/src/ffmpeg.ts | 10 ++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/packages/@webreel/core/src/chrome.ts b/packages/@webreel/core/src/chrome.ts index a983411..d97e987 100644 --- a/packages/@webreel/core/src/chrome.ts +++ b/packages/@webreel/core/src/chrome.ts @@ -3,7 +3,7 @@ import { existsSync, readdirSync, mkdtempSync, rmSync } from "node:fs"; import { createServer } from "node:net"; import { homedir, tmpdir } from "node:os"; import { resolve, join } from "node:path"; -import { fetchJson, downloadAndExtract } from "./download.js"; +import { fetchJson, downloadAndExtract, assertTrustedUrl } from "./download.js"; import { killProcess } from "./process.js"; export const CHROME_CACHE_DIR = resolve(homedir(), ".webreel", "bin", "chrome"); @@ -17,6 +17,14 @@ export const HEADLESS_SHELL_CACHE_DIR = resolve( const CfT_MANIFEST_URL = "https://googlechromelabs.github.io/chrome-for-testing/last-known-good-versions-with-downloads.json"; +// Chrome for Testing publishes no checksums alongside the manifest, so +// binary integrity for this path rests on HTTPS + host allowlisting only. +// The manifest itself is served from googlechromelabs.github.io; the actual +// build archives it links to are served from storage.googleapis.com +// (verified against a live manifest response). +const CfT_MANIFEST_HOSTS = ["googlechromelabs.github.io"]; +const CfT_DOWNLOAD_HOSTS = ["storage.googleapis.com"]; + type CfTManifest = { channels: { Stable: { @@ -123,12 +131,16 @@ export async function ensureChrome(): Promise { } try { + assertTrustedUrl(CfT_MANIFEST_URL, CfT_MANIFEST_HOSTS); const manifest = (await fetchJson(CfT_MANIFEST_URL)) as CfTManifest; const platform = cftPlatform(); const entry = manifest.channels.Stable.downloads.chrome.find( (d) => d.platform === platform, ); if (!entry) throw new Error(`No Chrome for Testing build for ${platform}`); + // No published hash for Chrome for Testing builds — HTTPS + host + // allowlist is the only integrity check available for this download. + assertTrustedUrl(entry.url, CfT_DOWNLOAD_HOSTS); await downloadAndExtract(entry.url, CHROME_CACHE_DIR, "Chrome for Testing"); @@ -156,11 +168,15 @@ export async function ensureHeadlessShell(): Promise { if (cached) return cached; } + assertTrustedUrl(CfT_MANIFEST_URL, CfT_MANIFEST_HOSTS); const manifest = (await fetchJson(CfT_MANIFEST_URL)) as CfTManifest; const platform = cftPlatform(); const entries = manifest.channels.Stable.downloads["chrome-headless-shell"]; const entry = entries?.find((d) => d.platform === platform); if (!entry) throw new Error(`No chrome-headless-shell build for ${platform}`); + // No published hash for Chrome for Testing builds — HTTPS + host + // allowlist is the only integrity check available for this download. + assertTrustedUrl(entry.url, CfT_DOWNLOAD_HOSTS); await downloadAndExtract(entry.url, HEADLESS_SHELL_CACHE_DIR, "chrome-headless-shell"); diff --git a/packages/@webreel/core/src/ffmpeg.ts b/packages/@webreel/core/src/ffmpeg.ts index f282baa..9366bf1 100644 --- a/packages/@webreel/core/src/ffmpeg.ts +++ b/packages/@webreel/core/src/ffmpeg.ts @@ -8,6 +8,7 @@ import { downloadFile, extractArchive, makeExecutable, + assertTrustedUrl, } from "./download.js"; export const FFMPEG_CACHE_DIR = resolve(homedir(), ".webreel", "bin", "ffmpeg"); @@ -15,6 +16,10 @@ export const FFMPEG_CACHE_DIR = resolve(homedir(), ".webreel", "bin", "ffmpeg"); // BtbN/FFmpeg-Builds: linked from ffmpeg.org, built via GitHub Actions. // Covers Linux (x64, arm64) and Windows (x64). const BTBN_BASE = "https://github.com/BtbN/FFmpeg-Builds/releases/download/latest"; +// The initial request goes to github.com, which redirects to a signed +// release-assets.githubusercontent.com URL; only the original github.com +// URL is validated here (see assertTrustedUrl doc comment in download.ts). +const BTBN_HOSTS = ["github.com"]; export function btbnAssetName(): string | null { const { platform, arch } = process; @@ -28,6 +33,7 @@ export function btbnAssetName(): string | null { // evermeet.cx: linked from ffmpeg.org, macOS x64 static builds. // Runs on ARM64 Macs via Rosetta 2. const EVERMEET_API = "https://evermeet.cx/ffmpeg/info/ffmpeg/release"; +const EVERMEET_HOSTS = ["evermeet.cx"]; export function binaryName(): string { return process.platform === "win32" ? "ffmpeg.exe" : "ffmpeg"; @@ -67,6 +73,8 @@ async function downloadBtbn(cacheDir: string): Promise { if (!asset) throw new Error("No BtbN build for this platform"); const url = `${BTBN_BASE}/${asset}`; + assertTrustedUrl(url, BTBN_HOSTS); + await downloadAndExtract(url, cacheDir, "ffmpeg"); const bin = binaryName(); @@ -79,10 +87,12 @@ async function downloadBtbn(cacheDir: string): Promise { } async function downloadEvermeet(cacheDir: string): Promise { + assertTrustedUrl(EVERMEET_API, EVERMEET_HOSTS); const info = (await fetchJson(EVERMEET_API)) as { download: { zip: { url: string } }; }; const url = info.download.zip.url; + assertTrustedUrl(url, EVERMEET_HOSTS); const archivePath = resolve(cacheDir, "_download.zip"); await downloadFile(url, archivePath, "ffmpeg"); From 6596670da8fc79d65ebe3fb8d654c0a08d3a9b89 Mon Sep 17 00:00:00 2001 From: Ariel Conti Date: Sat, 18 Jul 2026 15:32:48 +0100 Subject: [PATCH 24/48] fix(deps): force ws >=7.5.11 and raise engines floor to maintained Node pnpm audit flagged a high-severity ws memory-exhaustion advisory reached via chrome-remote-interface on every recording. Add a pnpm override to force ws>=7.5.11 across the dependency tree, and raise the published packages' engines floor from Node >=18 (EOL) to >=20 (oldest maintained LTS, matches what CI actually exercises). --- README.md | 2 +- package.json | 7 ++++++- packages/@webreel/core/package.json | 2 +- packages/webreel/package.json | 2 +- pnpm-lock.yaml | 15 +++++++++------ 5 files changed, 18 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 79e357b..0fea3bb 100644 --- a/README.md +++ b/README.md @@ -219,7 +219,7 @@ All steps (except `pause`) accept an optional `delay` field (ms to wait after th ### Prerequisites -- [Node.js](https://nodejs.org/) (v18+) +- [Node.js](https://nodejs.org/) (v20+) - [pnpm](https://pnpm.io/) ### Setup diff --git a/package.json b/package.json index 1da7ff4..d40f67e 100644 --- a/package.json +++ b/package.json @@ -33,5 +33,10 @@ "lint-staged": { "*.{ts,tsx,js,jsx,json,md,mdx,css}": "prettier --write" }, - "packageManager": "pnpm@10.6.2" + "packageManager": "pnpm@10.6.2", + "pnpm": { + "overrides": { + "ws@>=7.0.0 <7.5.11": ">=7.5.11" + } + } } diff --git a/packages/@webreel/core/package.json b/packages/@webreel/core/package.json index 4fca6ad..89f425b 100644 --- a/packages/@webreel/core/package.json +++ b/packages/@webreel/core/package.json @@ -33,7 +33,7 @@ } }, "engines": { - "node": ">=18" + "node": ">=20" }, "files": [ "dist", diff --git a/packages/webreel/package.json b/packages/webreel/package.json index c4af5c8..ca1f32e 100644 --- a/packages/webreel/package.json +++ b/packages/webreel/package.json @@ -38,7 +38,7 @@ "webreel": "./dist/index.js" }, "engines": { - "node": ">=18" + "node": ">=20" }, "files": [ "dist" diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index a024cef..d7bc6be 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -4,6 +4,9 @@ settings: autoInstallPeers: true excludeLinksFromLockfile: false +overrides: + ws@>=7.0.0 <7.5.11: '>=7.5.11' + importers: .: @@ -2659,12 +2662,12 @@ packages: resolution: {integrity: sha512-42AtmgqjV+X1VpdOfyTGOYRi0/zsoLqtXQckTmqTeybT+BDIbM/Guxo7x3pE2vtpr1ok6xRqM9OpBe+Jyoqyww==} engines: {node: '>=18'} - ws@7.5.10: - resolution: {integrity: sha512-+dbF1tHwZpXcbOJdVOkzLDxZP1ailvSxM6ZweXTegylPny803bFhA+vqBYw4s31NSAk4S2Qz+AKXK9a4wkdjcQ==} - engines: {node: '>=8.3.0'} + ws@8.21.1: + resolution: {integrity: sha512-+0NTnW77fFN/DjQi6k/Sq/Yvk4Sgajw7urW8V+asjXnRgDs9gyGkdb7EzgfhA4goXsRIZKE28fzIXBHEzhuiWw==} + engines: {node: '>=10.0.0'} peerDependencies: bufferutil: ^4.0.1 - utf-8-validate: ^5.0.2 + utf-8-validate: '>=5.0.2' peerDependenciesMeta: bufferutil: optional: true @@ -3791,7 +3794,7 @@ snapshots: chrome-remote-interface@0.33.3: dependencies: commander: 2.11.0 - ws: 7.5.10 + ws: 8.21.1 transitivePeerDependencies: - bufferutil - utf-8-validate @@ -5331,7 +5334,7 @@ snapshots: string-width: 7.2.0 strip-ansi: 7.2.0 - ws@7.5.10: {} + ws@8.21.1: {} yaml@2.8.2: {} From 97ac9c7114f89966d6b005fd72cf583200d1edb9 Mon Sep 17 00:00:00 2001 From: Ariel Conti Date: Sat, 18 Jul 2026 15:33:47 +0100 Subject: [PATCH 25/48] fix(core): verify sha256 of downloaded ffmpeg archives where published Adds fetchText/findSha256ForFile to download.ts and wires downloadBtbn to fetch checksums.sha256 from the same BtbN "latest" release and verify the downloaded archive against it (falls back to allowlist-only with a warning if the checksum entry or fetch is unavailable). Chrome for Testing and evermeet.cx publish no digest for their downloads, so those paths remain allowlist-only, documented in code comments at their call sites. --- packages/@webreel/core/src/download.ts | 24 +++++++++++++++++++ packages/@webreel/core/src/ffmpeg.ts | 32 +++++++++++++++++++++++++- 2 files changed, 55 insertions(+), 1 deletion(-) diff --git a/packages/@webreel/core/src/download.ts b/packages/@webreel/core/src/download.ts index cfe7b92..13ef916 100644 --- a/packages/@webreel/core/src/download.ts +++ b/packages/@webreel/core/src/download.ts @@ -72,6 +72,30 @@ export async function fetchJson(url: string): Promise { return res.json(); } +export async function fetchText(url: string): Promise { + const res = await withRetry(async () => { + const r = await fetch(url); + if (!r.ok) throw new Error(`HTTP ${r.status} fetching ${url}`); + return r; + }, `Fetch ${url}`); + return res.text(); +} + +/** + * Parses a `sha256sum`-style checksums file (lines of + * ` `, one or two spaces) and returns the digest for + * `filename`, or null if not present. + */ +export function findSha256ForFile(checksums: string, filename: string): string | null { + for (const line of checksums.split("\n")) { + const trimmed = line.trim(); + if (!trimmed) continue; + const m = trimmed.match(/^([0-9a-fA-F]{64})\s+\*?(.+)$/); + if (m && m[2].trim() === filename) return m[1]; + } + return null; +} + export async function downloadFile( url: string, destPath: string, diff --git a/packages/@webreel/core/src/ffmpeg.ts b/packages/@webreel/core/src/ffmpeg.ts index 9366bf1..0ce33b0 100644 --- a/packages/@webreel/core/src/ffmpeg.ts +++ b/packages/@webreel/core/src/ffmpeg.ts @@ -4,6 +4,8 @@ import { resolve } from "node:path"; import { execSync } from "node:child_process"; import { fetchJson, + fetchText, + findSha256ForFile, downloadAndExtract, downloadFile, extractArchive, @@ -20,6 +22,14 @@ const BTBN_BASE = "https://github.com/BtbN/FFmpeg-Builds/releases/download/lates // release-assets.githubusercontent.com URL; only the original github.com // URL is validated here (see assertTrustedUrl doc comment in download.ts). const BTBN_HOSTS = ["github.com"]; +// The "latest" release tag is a rolling pointer (the underlying build is +// republished under the same tag/filenames), but it publishes a +// checksums.sha256 asset alongside the binaries on every publish, keyed by +// the exact filenames this module downloads — verified against the live +// release. We fetch it from the same release at request time so the +// checksum always matches whatever asset is currently live, without having +// to pin to a versioned release tag (which uses different filenames). +const BTBN_CHECKSUMS_URL = `${BTBN_BASE}/checksums.sha256`; export function btbnAssetName(): string | null { const { platform, arch } = process; @@ -32,6 +42,10 @@ export function btbnAssetName(): string | null { // evermeet.cx: linked from ffmpeg.org, macOS x64 static builds. // Runs on ARM64 Macs via Rosetta 2. +// The API response has no sha256/digest field (only a GPG .sig detached +// signature URL, which we don't currently verify) — confirmed against the +// live endpoint. Integrity for this path rests on HTTPS + host allowlisting +// only; this is a known, accepted gap (see plan 010). const EVERMEET_API = "https://evermeet.cx/ffmpeg/info/ffmpeg/release"; const EVERMEET_HOSTS = ["evermeet.cx"]; @@ -75,7 +89,23 @@ async function downloadBtbn(cacheDir: string): Promise { const url = `${BTBN_BASE}/${asset}`; assertTrustedUrl(url, BTBN_HOSTS); - await downloadAndExtract(url, cacheDir, "ffmpeg"); + let expectedSha256: string | undefined; + try { + assertTrustedUrl(BTBN_CHECKSUMS_URL, BTBN_HOSTS); + const checksums = await fetchText(BTBN_CHECKSUMS_URL); + expectedSha256 = findSha256ForFile(checksums, asset) ?? undefined; + if (!expectedSha256) { + console.warn( + `No checksum entry for ${asset} in BtbN checksums.sha256; proceeding with URL allowlist only.`, + ); + } + } catch (err) { + console.warn( + `Failed to fetch BtbN checksums.sha256; proceeding with URL allowlist only: ${err instanceof Error ? err.message : String(err)}`, + ); + } + + await downloadAndExtract(url, cacheDir, "ffmpeg", expectedSha256); const bin = binaryName(); const found = findBinaryInDir(cacheDir, bin); From d7a4ea0207391020b887f8f41e31239fb476d0d3 Mon Sep 17 00:00:00 2001 From: Ariel Conti Date: Sat, 18 Jul 2026 15:35:48 +0100 Subject: [PATCH 26/48] test(core): cover download digest verification and archive path validation Adds download.test.ts: downloadFile digest-match/mismatch behavior against a local HTTP server (including partial-file cleanup on mismatch), assertTrustedUrl's scheme/host checks, the zip/tar listing parsers, the unsafe-entry-path validator against crafted traversal listings, and findSha256ForFile. --- .../core/src/__tests__/download.test.ts | 200 ++++++++++++++++++ 1 file changed, 200 insertions(+) create mode 100644 packages/@webreel/core/src/__tests__/download.test.ts diff --git a/packages/@webreel/core/src/__tests__/download.test.ts b/packages/@webreel/core/src/__tests__/download.test.ts new file mode 100644 index 0000000..7697057 --- /dev/null +++ b/packages/@webreel/core/src/__tests__/download.test.ts @@ -0,0 +1,200 @@ +import { describe, it, expect, beforeAll, afterAll } from "vitest"; +import { createServer, type Server } from "node:http"; +import { createHash } from "node:crypto"; +import { existsSync, mkdirSync, rmSync } from "node:fs"; +import { resolve } from "node:path"; +import { tmpdir } from "node:os"; +import { + downloadFile, + assertTrustedUrl, + isUnsafeEntryPath, + validateEntryPaths, + parseZipListing, + parseTarListing, + findSha256ForFile, +} from "../download.js"; + +describe("assertTrustedUrl", () => { + const allowed = ["example.com", "cdn.example.com"]; + + it("accepts an https URL on an allowlisted host", () => { + expect(() => assertTrustedUrl("https://example.com/file.zip", allowed)).not.toThrow(); + expect(() => + assertTrustedUrl("https://cdn.example.com/path/file.zip", allowed), + ).not.toThrow(); + }); + + it("rejects http (non-https) URLs", () => { + expect(() => assertTrustedUrl("http://example.com/file.zip", allowed)).toThrow( + /non-https/, + ); + }); + + it("rejects URLs on hosts not in the allowlist", () => { + expect(() => assertTrustedUrl("https://evil.example.net/file.zip", allowed)).toThrow( + /untrusted host/, + ); + }); + + it("rejects malformed URLs", () => { + expect(() => assertTrustedUrl("not a url", allowed)).toThrow(); + }); +}); + +describe("downloadFile", () => { + let server: Server; + let port: number; + const payload = Buffer.from("hello world, this is the download payload"); + const correctSha256 = createHash("sha256").update(payload).digest("hex"); + + beforeAll(async () => { + server = createServer((_req, res) => { + res.writeHead(200, { "Content-Type": "application/octet-stream" }); + res.end(payload); + }); + await new Promise((resolvePromise) => { + server.listen(0, "127.0.0.1", () => resolvePromise()); + }); + const addr = server.address(); + if (addr && typeof addr === "object") port = addr.port; + }); + + afterAll(async () => { + await new Promise((resolvePromise) => server.close(() => resolvePromise())); + }); + + function destPathFor(name: string): string { + const dir = resolve(tmpdir(), `webreel-download-test-${Date.now()}-${Math.random()}`); + mkdirSync(dir, { recursive: true }); + return resolve(dir, name); + } + + it("downloads successfully when no digest is provided", async () => { + const dest = destPathFor("no-digest.bin"); + await downloadFile(`http://127.0.0.1:${port}/file`, dest, "test file"); + expect(existsSync(dest)).toBe(true); + rmSync(resolve(dest, ".."), { recursive: true, force: true }); + }); + + it("downloads successfully when the digest matches", async () => { + const dest = destPathFor("match.bin"); + await downloadFile(`http://127.0.0.1:${port}/file`, dest, "test file", correctSha256); + expect(existsSync(dest)).toBe(true); + rmSync(resolve(dest, ".."), { recursive: true, force: true }); + }); + + it("rejects on digest mismatch and deletes the partial file", async () => { + const dest = destPathFor("mismatch.bin"); + const wrongSha256 = "0".repeat(64); + await expect( + downloadFile(`http://127.0.0.1:${port}/file`, dest, "test file", wrongSha256), + ).rejects.toThrow(/Checksum mismatch/); + expect(existsSync(dest)).toBe(false); + rmSync(resolve(dest, ".."), { recursive: true, force: true }); + }); +}); + +describe("isUnsafeEntryPath", () => { + it("flags absolute unix paths", () => { + expect(isUnsafeEntryPath("/abs/path")).toBe(true); + expect(isUnsafeEntryPath("/etc/passwd")).toBe(true); + }); + + it("flags absolute Windows drive paths", () => { + expect(isUnsafeEntryPath("C:\\Windows\\System32")).toBe(true); + expect(isUnsafeEntryPath("C:/Windows/System32")).toBe(true); + }); + + it("flags paths containing a .. segment", () => { + expect(isUnsafeEntryPath("../evil")).toBe(true); + expect(isUnsafeEntryPath("safe/../../evil")).toBe(true); + expect(isUnsafeEntryPath("a/../../b")).toBe(true); + }); + + it("allows normal relative paths", () => { + expect(isUnsafeEntryPath("folder/file.txt")).toBe(false); + expect(isUnsafeEntryPath("chrome-linux64/chrome")).toBe(false); + expect(isUnsafeEntryPath("")).toBe(false); + }); +}); + +describe("validateEntryPaths", () => { + it("throws when a listing contains a ../ traversal entry", () => { + expect(() => validateEntryPaths(["safe/file.txt", "../evil"])).toThrow( + /unsafe entry path/, + ); + }); + + it("throws when a listing contains an absolute path entry", () => { + expect(() => validateEntryPaths(["safe/file.txt", "/abs/path"])).toThrow( + /unsafe entry path/, + ); + }); + + it("does not throw for an all-safe listing", () => { + expect(() => + validateEntryPaths(["chrome-linux64/", "chrome-linux64/chrome"]), + ).not.toThrow(); + }); +}); + +describe("parseZipListing", () => { + it("extracts entry names from unzip -l output", () => { + const output = [ + "Archive: test.zip", + " Length Date Time Name", + "--------- ---------- ----- ----", + " 0 01-01-2020 00:00 folder/", + " 10 01-01-2020 00:00 folder/file.txt", + "--------- -------", + " 10 2 files", + ].join("\n"); + expect(parseZipListing(output)).toEqual(["folder/", "folder/file.txt"]); + }); + + it("extracts a malicious traversal entry so it can be rejected", () => { + const output = [ + "Archive: evil.zip", + " Length Date Time Name", + "--------- ---------- ----- ----", + " 6 01-01-2020 00:00 ../../evil.txt", + " 2 01-01-2020 00:00 normal.txt", + "--------- -------", + " 8 2 files", + ].join("\n"); + const entries = parseZipListing(output); + expect(entries).toContain("../../evil.txt"); + expect(() => validateEntryPaths(entries, "evil.zip")).toThrow(/unsafe entry path/); + }); +}); + +describe("parseTarListing", () => { + it("returns one trimmed entry per non-empty line", () => { + const output = "folder/\nfolder/file.txt\n\n"; + expect(parseTarListing(output)).toEqual(["folder/", "folder/file.txt"]); + }); + + it("surfaces traversal entries from tar -tf output", () => { + const output = "normal.txt\n../../evil.txt\n"; + const entries = parseTarListing(output); + expect(entries).toContain("../../evil.txt"); + expect(() => validateEntryPaths(entries, "evil.tar.xz")).toThrow(/unsafe entry path/); + }); +}); + +describe("findSha256ForFile", () => { + const checksums = [ + "8383958c8f6b1b4eabc40c268ad17aa6a81f331bce58286bb7c8ae416341b722 ffmpeg-linux64.tar.xz", + "d58bf2c57f4ab59a5c7523b3d03e7380259943640b01a827156e80000124ea63 ffmpeg-win64.zip", + ].join("\n"); + + it("finds the digest for the matching filename", () => { + expect(findSha256ForFile(checksums, "ffmpeg-linux64.tar.xz")).toBe( + "8383958c8f6b1b4eabc40c268ad17aa6a81f331bce58286bb7c8ae416341b722", + ); + }); + + it("returns null when the filename is not present", () => { + expect(findSha256ForFile(checksums, "does-not-exist.zip")).toBeNull(); + }); +}); From 6b7f75577feefb9aadfbe4a347042fbf2ebd4d7d Mon Sep 17 00:00:00 2001 From: Ariel Conti Date: Sat, 18 Jul 2026 15:36:11 +0100 Subject: [PATCH 27/48] fix(deps): constrain ws override to the 7.x line Reviewer flagged that ">=7.5.11" let pnpm resolve ws to 8.21.1, a major jump outside the range chrome-remote-interface was tested with. Use "^7.5.11" instead so the override stays within the same 7.x line the plan's risk assessment was predicated on (now resolves to 7.5.13). --- package.json | 2 +- pnpm-lock.yaml | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/package.json b/package.json index d40f67e..d49972c 100644 --- a/package.json +++ b/package.json @@ -36,7 +36,7 @@ "packageManager": "pnpm@10.6.2", "pnpm": { "overrides": { - "ws@>=7.0.0 <7.5.11": ">=7.5.11" + "ws@>=7.0.0 <7.5.11": "^7.5.11" } } } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index d7bc6be..be31e32 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -5,7 +5,7 @@ settings: excludeLinksFromLockfile: false overrides: - ws@>=7.0.0 <7.5.11: '>=7.5.11' + ws@>=7.0.0 <7.5.11: ^7.5.11 importers: @@ -2662,12 +2662,12 @@ packages: resolution: {integrity: sha512-42AtmgqjV+X1VpdOfyTGOYRi0/zsoLqtXQckTmqTeybT+BDIbM/Guxo7x3pE2vtpr1ok6xRqM9OpBe+Jyoqyww==} engines: {node: '>=18'} - ws@8.21.1: - resolution: {integrity: sha512-+0NTnW77fFN/DjQi6k/Sq/Yvk4Sgajw7urW8V+asjXnRgDs9gyGkdb7EzgfhA4goXsRIZKE28fzIXBHEzhuiWw==} - engines: {node: '>=10.0.0'} + ws@7.5.13: + resolution: {integrity: sha512-rsKI6xDBFVf4r/x8XyChGK04QR/XHroxs/jUcoWvtEZM8TPU/X/uIY9B1CsSzYws9ZJb/6bbBu7dPhFW00CAoA==} + engines: {node: '>=8.3.0'} peerDependencies: bufferutil: ^4.0.1 - utf-8-validate: '>=5.0.2' + utf-8-validate: ^5.0.2 peerDependenciesMeta: bufferutil: optional: true @@ -3794,7 +3794,7 @@ snapshots: chrome-remote-interface@0.33.3: dependencies: commander: 2.11.0 - ws: 8.21.1 + ws: 7.5.13 transitivePeerDependencies: - bufferutil - utf-8-validate @@ -5334,7 +5334,7 @@ snapshots: string-width: 7.2.0 strip-ansi: 7.2.0 - ws@8.21.1: {} + ws@7.5.13: {} yaml@2.8.2: {} From 8670c02b124afac1aacb90fb56b4393ef105ef2a Mon Sep 17 00:00:00 2001 From: Ariel Conti Date: Sat, 18 Jul 2026 15:36:18 +0100 Subject: [PATCH 28/48] fix(core): treat ffmpeg pipe failure as abort instead of crash or hang Recorder: an unhandled 'error' event on ffmpeg's stdin (e.g. EPIPE after a crash) crashed the whole process instead of just dropping frames. Compositor: EPIPE was swallowed without aborting, so the consumer could await a 'drain' event a dead stream never emits; a premature process close (before stdin.end()) is now also treated as an abort, since Node can auto-destroy an already-closed process's stdin with no further 'error' or 'drain' event at all. The abort path also leaked its SIGKILL timer and could raise an unhandled rejection from ffmpegDone. Claude-Session: https://claude.ai/code/session_01Pv8tMQt9GoVpfnpVcdwFxe --- .../core/src/__tests__/compositor.test.ts | 72 +++++++++++++++++ .../core/src/__tests__/recorder.test.ts | 24 ++++++ packages/@webreel/core/src/compositor.ts | 79 +++++++++++++++---- packages/@webreel/core/src/recorder.ts | 11 ++- 4 files changed, 170 insertions(+), 16 deletions(-) create mode 100644 packages/@webreel/core/src/__tests__/compositor.test.ts diff --git a/packages/@webreel/core/src/__tests__/compositor.test.ts b/packages/@webreel/core/src/__tests__/compositor.test.ts new file mode 100644 index 0000000..a311acd --- /dev/null +++ b/packages/@webreel/core/src/__tests__/compositor.test.ts @@ -0,0 +1,72 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; +import { chmodSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { compose } from "../compositor.js"; +import { InteractionTimeline } from "../timeline.js"; + +let ffmpegPathResolver: () => Promise; +vi.mock("../ffmpeg.js", () => ({ + ensureFfmpeg: () => ffmpegPathResolver(), +})); + +let workDir: string; + +beforeEach(() => { + workDir = mkdtempSync(join(tmpdir(), "webreel-compositor-test-")); +}); + +afterEach(() => { + rmSync(workDir, { recursive: true, force: true }); +}); + +describe("compose", () => { + it("rejects instead of hanging when ffmpeg dies before reading any input", async () => { + // Exits immediately without reading stdin at all, so the very first + // piped overlay frame lands on an already-dead pipe. With enough + // frames queued behind it, the producer/consumer loop is forced into + // real backpressure (stdin.write returning false) against a process + // that will never emit 'drain'. + const dyingShimPath = join(workDir, "dying-ffmpeg"); + writeFileSync(dyingShimPath, "#!/bin/sh\nexit 1\n"); + chmodSync(dyingShimPath, 0o755); + ffmpegPathResolver = () => Promise.resolve(dyingShimPath); + + const unhandledRejectionSpy = vi.fn(); + process.on("unhandledRejection", unhandledRejectionSpy); + + try { + // A larger canvas with a moving cursor and changing HUD text keeps + // every frame's overlay PNG distinct (no cache hits) and large + // enough, across many frames, to exceed stdin's internal buffer. + const timeline = new InteractionTimeline(480, 480); + const path = Array.from({ length: 60 }, (_, i) => ({ + x: (i * 7) % 460, + y: (i * 11) % 460, + })); + timeline.setCursorPath(path); + for (let i = 0; i < path.length; i++) { + timeline.showHud([`step ${i}`, `frame-${i}-of-${path.length}`]); + timeline.tick(); + } + const timelineData = timeline.toJSON(); + + // The shim never reads this as a real video; ffmpeg's own -i + // handling is entirely bypassed by the shim script. + const cleanVideoPath = join(workDir, "clean.mp4"); + writeFileSync(cleanVideoPath, "not a real video"); + + // .gif output takes the single-pass compositeFrames path directly + // (no finalize stage), keeping the shim's job simple. + const outputPath = join(workDir, "out.gif"); + + await expect(compose(cleanVideoPath, timelineData, outputPath)).rejects.toThrow(); + + // Give a dangling ffmpegDone rejection (if any) a turn to surface. + await new Promise((r) => setTimeout(r, 50)); + expect(unhandledRejectionSpy).not.toHaveBeenCalled(); + } finally { + process.removeListener("unhandledRejection", unhandledRejectionSpy); + } + }, 10_000); +}); diff --git a/packages/@webreel/core/src/__tests__/recorder.test.ts b/packages/@webreel/core/src/__tests__/recorder.test.ts index d1c59f9..c827360 100644 --- a/packages/@webreel/core/src/__tests__/recorder.test.ts +++ b/packages/@webreel/core/src/__tests__/recorder.test.ts @@ -100,3 +100,27 @@ describe("Recorder.stop", () => { expect(recorder.getTempVideoPath()).toBe(""); }); }); + +describe("Recorder pipe error handling", () => { + it("survives ffmpeg dying mid-recording without an uncaught exception", async () => { + // Reads a little input then exits nonzero, so later writeFrame() calls + // hit a dead pipe (EPIPE) instead of a live ffmpeg process. + const dyingShimPath = join(shimDir, "dying-ffmpeg"); + writeFileSync(dyingShimPath, "#!/bin/sh\nhead -c 100 > /dev/null\nexit 1\n"); + chmodSync(dyingShimPath, 0o755); + ffmpegPathResolver = () => Promise.resolve(dyingShimPath); + + const uncaughtSpy = vi.fn(); + process.on("uncaughtException", uncaughtSpy); + + try { + const recorder = await startedRecorder(); + // Give the capture loop time to keep writing after the shim exits. + await new Promise((r) => setTimeout(r, 100)); + await recorder.stop(); + expect(uncaughtSpy).not.toHaveBeenCalled(); + } finally { + process.removeListener("uncaughtException", uncaughtSpy); + } + }); +}); diff --git a/packages/@webreel/core/src/compositor.ts b/packages/@webreel/core/src/compositor.ts index f21cceb..0167283 100644 --- a/packages/@webreel/core/src/compositor.ts +++ b/packages/@webreel/core/src/compositor.ts @@ -315,35 +315,53 @@ async function compositeFrames( ffmpeg.on("close", (code) => { if (code === 0) { resolveAll(); - } else { - const stderr = Buffer.concat(stderrChunks).toString().slice(-2000); - rejectAll( - new Error( - `Compositor ffmpeg (layer=${layer}) exited with code ${code}${stderr ? `:\n${stderr}` : ""}`, - ), - ); + return; + } + const stderr = Buffer.concat(stderrChunks).toString().slice(-2000); + const err = new Error( + `Compositor ffmpeg (layer=${layer}) exited with code ${code}${stderr ? `:\n${stderr}` : ""}`, + ); + rejectAll(err); + + // If ffmpeg dies before we've finished feeding it frames (stdin.end() + // not yet called), don't rely solely on a stdin 'error' event to + // unblock the producer/consumer loop: once Node auto-destroys an + // already-closed process's stdin, further stdin.write() calls can + // return false with no further 'error' or 'drain' event ever firing, + // which would otherwise hang the loop forever. Treat a premature + // close as an abort signal directly. + if (!state.stdinEnded && !state.abortError) { + state.abortError = err; + notifyConsumer(); + notifyProducer(); + notifyDrain(); } }); ffmpeg.on("error", rejectAll); }); + // ffmpegDone can reject as soon as the process exits, which may be well + // before the abort branch below (or the final `await ffmpegDone`) has a + // chance to observe it. Attach a no-op handler now so Node never sees an + // unhandled rejection in that window; the promise is still awaited for + // its real outcome further down. + ffmpegDone.catch(() => {}); const PREFETCH_QUEUE_SIZE = 4; const state = { abortError: null as Error | null, producerDone: false, + // Set just before stdin.end() is called; an EPIPE after that point means + // ffmpeg simply finished reading and is expected, not an abort. + stdinEnded: false, // Resolves when the queue has items OR the producer is done. queueResolve: null as (() => void) | null, // Resolves when the consumer dequeues an item (backpressure signal). spaceResolve: null as (() => void) | null, + // Resolves the consumer's in-flight drain() wait. + drainResolve: null as (() => void) | null, }; - // EPIPE is expected when ffmpeg finishes reading and closes its stdin. - stdin.on("error", (err: NodeJS.ErrnoException) => { - if (err.code === "EPIPE") return; - if (!state.abortError) state.abortError = err; - }); - const queue: Buffer[] = []; const notifyConsumer = () => { @@ -362,6 +380,26 @@ async function compositeFrames( } }; + const notifyDrain = () => { + if (state.drainResolve) { + const r = state.drainResolve; + state.drainResolve = null; + r(); + } + }; + + // EPIPE is expected once ffmpeg has finished reading and we've called + // stdin.end(). Before that, it means ffmpeg died mid-stream: treat it as + // an abort and wake every waiter so the producer/consumer loop unwinds + // instead of hanging on a drain event a dead stream will never emit. + stdin.on("error", (err: NodeJS.ErrnoException) => { + if (err.code === "EPIPE" && state.stdinEnded) return; + if (!state.abortError) state.abortError = err; + notifyConsumer(); + notifyProducer(); + notifyDrain(); + }); + const enqueue = (buf: Buffer) => { queue.push(buf); notifyConsumer(); @@ -379,7 +417,12 @@ async function compositeFrames( state.spaceResolve = r; }); - const drain = (): Promise => new Promise((r) => stdin.once("drain", r)); + const drain = (): Promise => + new Promise((r) => { + if (state.abortError) return r(); + state.drainResolve = r; + stdin.once("drain", r); + }); const consumer = async () => { while (true) { @@ -396,6 +439,7 @@ async function compositeFrames( if (state.abortError) break; } } + state.stdinEnded = true; stdin.end(); }; @@ -428,9 +472,14 @@ async function compositeFrames( if (state.abortError) { ffmpeg.kill("SIGTERM"); - setTimeout(() => { + const killTimer = setTimeout(() => { if (!ffmpeg.killed) ffmpeg.kill("SIGKILL"); }, KILL_TIMEOUT); + killTimer.unref(); + ffmpeg.once("close", () => clearTimeout(killTimer)); + // ffmpegDone already has a no-op catch attached above, so its eventual + // rejection (from the killed process exiting nonzero) won't surface as + // an unhandled rejection alongside the throw below. throw state.abortError; } diff --git a/packages/@webreel/core/src/recorder.ts b/packages/@webreel/core/src/recorder.ts index 628407f..34c89a8 100644 --- a/packages/@webreel/core/src/recorder.ts +++ b/packages/@webreel/core/src/recorder.ts @@ -27,6 +27,7 @@ export class Recorder { private tempVideo = ""; private drainResolve: (() => void) | null = null; private droppedFrames = 0; + private pipeError: Error | null = null; private timeline: InteractionTimeline | null = null; private ctx: RecordingContext | null = null; private framesDir: string | null = null; @@ -82,6 +83,7 @@ export class Recorder { this.outputPath = outputPath; this.frameCount = 0; this.droppedFrames = 0; + this.pipeError = null; this.running = true; this.events = []; this.ctx = ctx ?? null; @@ -137,6 +139,13 @@ export class Recorder { const stdin = this.ffmpegProcess.stdin; if (!stdin) throw new Error("ffmpeg process has no stdin pipe"); stdin.on("drain", resolveDrain); + // A dead pipe (e.g. ffmpeg crashed) surfaces as an 'error' event on the + // stream; without a listener Node treats it as an uncaught exception. + // Mark the pipe dead and unblock any writeFrame() waiting on drain. + stdin.on("error", (err: Error) => { + if (!this.pipeError) this.pipeError = err; + resolveDrain(); + }); this.ffmpegProcess.on("close", resolveDrain); this.stoppedPromise = new Promise((resolve) => { @@ -149,7 +158,7 @@ export class Recorder { private async writeFrame(buffer: Buffer): Promise { if (!this.running) return; const stdin = this.ffmpegProcess?.stdin; - if (!stdin?.writable) { + if (!stdin?.writable || this.pipeError) { this.droppedFrames++; return; } From 2b4e17bc158ba01d73fd8599b2dce95bac8c6cb1 Mon Sep 17 00:00:00 2001 From: Ariel Conti Date: Sat, 18 Jul 2026 15:37:44 +0100 Subject: [PATCH 29/48] fix(core): wait for cursor path consumption before dispatching the action Replaces the fixed NUM_STEPS * CAPTURE_CYCLE_MS wall-clock wait in the recording branch of animateMoveTo with the existing waitForPathComplete() primitive, so the click/keystroke that follows a cursor move only fires once the capture loop has actually consumed the whole animated path. This avoids desync between the on-screen cursor and the action under capture slowdown (dropped frames). --- .../core/src/__tests__/cursor-motion.test.ts | 75 ++++++++++++++++++- packages/@webreel/core/src/cursor-motion.ts | 2 +- 2 files changed, 74 insertions(+), 3 deletions(-) diff --git a/packages/@webreel/core/src/__tests__/cursor-motion.test.ts b/packages/@webreel/core/src/__tests__/cursor-motion.test.ts index b230a9b..f265f30 100644 --- a/packages/@webreel/core/src/__tests__/cursor-motion.test.ts +++ b/packages/@webreel/core/src/__tests__/cursor-motion.test.ts @@ -1,5 +1,18 @@ -import { describe, it, expect } from "vitest"; -import { computeEasedPath, computeDragTiming } from "../cursor-motion.js"; +import { describe, it, expect, vi } from "vitest"; +import { computeEasedPath, computeDragTiming, animateMoveTo } from "../cursor-motion.js"; +import { RecordingContext } from "../actions.js"; +import { InteractionTimeline } from "../timeline.js"; +import type { CDPClient } from "../types.js"; + +function createMockClient() { + return { + Input: { + dispatchMouseEvent: vi.fn().mockResolvedValue(undefined), + }, + } as unknown as CDPClient & { + Input: { dispatchMouseEvent: ReturnType }; + }; +} describe("computeEasedPath", () => { it("returns a single destination point when distance is < 1", () => { @@ -43,3 +56,61 @@ describe("computeDragTiming", () => { expect(delayMs).toBeGreaterThan(0); }); }); + +describe("animateMoveTo (recording)", () => { + it("does not dispatch mouseMoved until the cursor path is fully consumed by ticks", async () => { + const ctx = new RecordingContext(); + ctx.setMode("record"); + const timeline = new InteractionTimeline(1080, 1080); + ctx.setTimeline(timeline); + const client = createMockClient(); + + const movePromise = animateMoveTo(ctx, client, 0, 0, 100, 100); + + // setCursorPath runs synchronously before the first await inside + // animateMoveTo, so the dispatch should not have fired yet. + expect(client.Input.dispatchMouseEvent).not.toHaveBeenCalled(); + + // Consume a couple of ticks -- deliberately short of the full path. + timeline.tick(); + timeline.tick(); + await Promise.resolve(); + expect(client.Input.dispatchMouseEvent).not.toHaveBeenCalled(); + + // Drive enough ticks to consume the rest of the path. The exact step + // count depends on distance/jitter (see moveDuration), so over-tick + // generously -- ticks after the path drains are harmless no-ops. + for (let i = 0; i < 100; i++) timeline.tick(); + + await movePromise; + expect(client.Input.dispatchMouseEvent).toHaveBeenCalledWith({ + type: "mouseMoved", + x: 100, + y: 100, + }); + }); + + it("unblocks via releaseWaiters without needing the path to fully drain", async () => { + const ctx = new RecordingContext(); + ctx.setMode("record"); + const timeline = new InteractionTimeline(1080, 1080); + ctx.setTimeline(timeline); + const client = createMockClient(); + + const movePromise = animateMoveTo(ctx, client, 0, 0, 500, 500); + + // Consume only one point of a much longer path, then interrupt. + timeline.tick(); + await Promise.resolve(); + expect(client.Input.dispatchMouseEvent).not.toHaveBeenCalled(); + + timeline.releaseWaiters(); + + await movePromise; + expect(client.Input.dispatchMouseEvent).toHaveBeenCalledWith({ + type: "mouseMoved", + x: 500, + y: 500, + }); + }); +}); diff --git a/packages/@webreel/core/src/cursor-motion.ts b/packages/@webreel/core/src/cursor-motion.ts index 609ef68..d0a2422 100644 --- a/packages/@webreel/core/src/cursor-motion.ts +++ b/packages/@webreel/core/src/cursor-motion.ts @@ -99,7 +99,7 @@ export async function animateMoveTo( if (ctx.isRecording && ctx.timeline) { ctx.timeline.setCursorPath(positions); - await new Promise((r) => setTimeout(r, NUM_STEPS * CAPTURE_CYCLE_MS)); + await ctx.timeline.waitForPathComplete(); await client.Input.dispatchMouseEvent({ type: "mouseMoved", From 857e89b564752ae0646fb822db2f0acd50eb172b Mon Sep 17 00:00:00 2001 From: Ariel Conti Date: Sat, 18 Jul 2026 15:37:47 +0100 Subject: [PATCH 30/48] test(webreel): characterize runVideo orchestration --- .../__tests__/runner-orchestration.test.ts | 200 ++++++++++++++++++ 1 file changed, 200 insertions(+) create mode 100644 packages/webreel/src/lib/__tests__/runner-orchestration.test.ts diff --git a/packages/webreel/src/lib/__tests__/runner-orchestration.test.ts b/packages/webreel/src/lib/__tests__/runner-orchestration.test.ts new file mode 100644 index 0000000..d01e08b --- /dev/null +++ b/packages/webreel/src/lib/__tests__/runner-orchestration.test.ts @@ -0,0 +1,200 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import type { CDPClient } from "@webreel/core"; +import type { VideoConfig } from "../types.js"; + +// Characterizes runVideo's orchestration of Chrome/CDP/Recorder/compose +// without touching real Chrome or ffmpeg. Only the resource-boundary exports +// (launchChrome, connectCDP, Recorder, compose) are faked; everything else +// from @webreel/core (RecordingContext, navigate, pause, InteractionTimeline, +// etc.) runs for real against the fake CDPClient, mirroring the pattern in +// recorder.test.ts. + +const hoisted = vi.hoisted(() => { + class FakeRecorder { + static instances: FakeRecorder[] = []; + static nextTempVideoPath = ""; + start = vi.fn().mockResolvedValue(undefined); + stop = vi.fn().mockResolvedValue(undefined); + setTimeline = vi.fn(); + getTempVideoPath = vi.fn(() => FakeRecorder.nextTempVideoPath); + constructor(..._args: unknown[]) { + FakeRecorder.instances.push(this); + } + } + + return { + FakeRecorder, + chromeKillMock: vi.fn().mockResolvedValue(undefined), + launchChromeMock: vi.fn(), + connectCDPMock: vi.fn(), + composeMock: vi.fn().mockResolvedValue(undefined), + }; +}); + +const { FakeRecorder, chromeKillMock, launchChromeMock, connectCDPMock, composeMock } = + hoisted; + +vi.mock("@webreel/core", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + launchChrome: launchChromeMock, + connectCDP: connectCDPMock, + Recorder: FakeRecorder, + compose: composeMock, + }; +}); + +const { runVideo } = await import("../runner.js"); + +const CHROME_PORT = 9909; + +function createFakeClient(overrides: Partial> = {}): CDPClient { + return { + close: vi.fn().mockResolvedValue(undefined), + Page: { + enable: vi.fn().mockResolvedValue(undefined), + navigate: vi.fn().mockResolvedValue(undefined), + loadEventFired: vi.fn().mockResolvedValue(undefined), + captureScreenshot: vi.fn().mockResolvedValue({ data: "" }), + }, + Runtime: { + enable: vi.fn().mockResolvedValue(undefined), + evaluate: vi.fn().mockResolvedValue({ result: {} }), + }, + Input: { + dispatchMouseEvent: vi.fn().mockResolvedValue(undefined), + dispatchKeyEvent: vi.fn().mockResolvedValue(undefined), + insertText: vi.fn().mockResolvedValue(undefined), + }, + Emulation: { + setDeviceMetricsOverride: vi.fn().mockResolvedValue(undefined), + }, + DOM: { + enable: vi.fn().mockResolvedValue(undefined), + getDocument: vi.fn().mockResolvedValue({ root: { nodeId: 1 } }), + querySelector: vi.fn().mockResolvedValue({ nodeId: 1 }), + setFileInputFiles: vi.fn().mockResolvedValue(undefined), + }, + ...overrides, + } as unknown as CDPClient; +} + +let fakeClient: CDPClient; +let configDir: string; +let tempVideoDir: string; + +beforeEach(() => { + fakeClient = createFakeClient(); + connectCDPMock.mockReset(); + connectCDPMock.mockImplementation(async () => fakeClient); + launchChromeMock.mockReset(); + launchChromeMock.mockResolvedValue({ + port: CHROME_PORT, + kill: chromeKillMock, + process: {}, + }); + chromeKillMock.mockClear(); + composeMock.mockClear(); + FakeRecorder.instances.length = 0; + + configDir = mkdtempSync(join(tmpdir(), "webreel-runner-orch-config-")); + tempVideoDir = mkdtempSync(join(tmpdir(), "webreel-runner-orch-video-")); + const tempVideoPath = join(tempVideoDir, "clean.mp4"); + writeFileSync(tempVideoPath, "fake-video-bytes"); + FakeRecorder.nextTempVideoPath = tempVideoPath; +}); + +afterEach(() => { + rmSync(configDir, { recursive: true, force: true }); + rmSync(tempVideoDir, { recursive: true, force: true }); +}); + +function minimalConfig(overrides: Partial = {}): VideoConfig { + return { + name: "characterization", + url: "http://localhost/x", + viewport: { width: 400, height: 300 }, + thumbnail: { enabled: false }, + steps: [{ action: "pause", ms: 1 }], + ...overrides, + }; +} + +describe("runVideo orchestration (happy path)", () => { + it("launches Chrome, connects CDP, sets the viewport, and records around the steps", async () => { + const config = minimalConfig(); + + await runVideo(config, { record: true, configDir }); + + expect(launchChromeMock).toHaveBeenCalledTimes(1); + expect(connectCDPMock).toHaveBeenCalledWith(CHROME_PORT); + + expect(fakeClient.Emulation.setDeviceMetricsOverride).toHaveBeenCalledWith({ + width: 400, + height: 300, + deviceScaleFactor: 1, + mobile: false, + }); + + expect(FakeRecorder.instances).toHaveLength(1); + const instance = FakeRecorder.instances[0]; + expect(instance.start).toHaveBeenCalledTimes(1); + expect(instance.stop).toHaveBeenCalledTimes(1); + // start must precede stop -- the step loop runs strictly between them. + expect(instance.start.mock.invocationCallOrder[0]).toBeLessThan( + instance.stop.mock.invocationCallOrder[0], + ); + + expect(composeMock).toHaveBeenCalledTimes(1); + expect(chromeKillMock).toHaveBeenCalledTimes(1); + expect(fakeClient.close).toHaveBeenCalledTimes(1); + }); + + it("does not flake across repeated runs", async () => { + for (let i = 0; i < 5; i++) { + FakeRecorder.instances.length = 0; + chromeKillMock.mockClear(); + composeMock.mockClear(); + const localConfigDir = mkdtempSync(join(tmpdir(), "webreel-runner-orch-loop-")); + const localVideoDir = mkdtempSync(join(tmpdir(), "webreel-runner-orch-loopvid-")); + const tempVideoPath = join(localVideoDir, "clean.mp4"); + writeFileSync(tempVideoPath, "fake-video-bytes"); + FakeRecorder.nextTempVideoPath = tempVideoPath; + + await runVideo(minimalConfig(), { record: true, configDir: localConfigDir }); + + expect(FakeRecorder.instances[0].start).toHaveBeenCalledTimes(1); + expect(FakeRecorder.instances[0].stop).toHaveBeenCalledTimes(1); + expect(chromeKillMock).toHaveBeenCalledTimes(1); + + rmSync(localConfigDir, { recursive: true, force: true }); + rmSync(localVideoDir, { recursive: true, force: true }); + } + }); +}); + +describe("runVideo orchestration (failure cleanup)", () => { + it("still stops the recorder and kills Chrome when a step throws", async () => { + fakeClient.Runtime.evaluate = vi + .fn() + .mockRejectedValue(new Error("cdp connection lost")); + + const config = minimalConfig({ + steps: [{ action: "click", selector: "#does-not-exist" }], + }); + + await expect(runVideo(config, { record: true, configDir })).rejects.toThrow( + /Step 0 \(click\) failed/, + ); + + expect(FakeRecorder.instances).toHaveLength(1); + const instance = FakeRecorder.instances[0]; + expect(instance.stop).toHaveBeenCalledTimes(1); + expect(chromeKillMock).toHaveBeenCalledTimes(1); + expect(fakeClient.close).toHaveBeenCalledTimes(1); + }); +}); From 06048c77ec976d3d611d307b438198c0902a78b7 Mon Sep 17 00:00:00 2001 From: Ariel Conti Date: Sat, 18 Jul 2026 15:38:16 +0100 Subject: [PATCH 31/48] fix(core): launch Chrome with its sandbox enabled by default Both launch paths passed --no-sandbox unconditionally, disabling Chrome's primary containment against renderer compromise on every recording, even though most environments (macOS, Windows, ordinary Linux desktops) don't need it. --no-sandbox is now applied only when WEBREEL_NO_SANDBOX=1 is set, when running as root, or as a one-time fallback if a sandboxed launch fails. Claude-Session: https://claude.ai/code/session_01Pv8tMQt9GoVpfnpVcdwFxe --- apps/docs/src/app/page.mdx | 2 +- apps/docs/src/app/quick-start/page.mdx | 2 +- .../core/src/__tests__/chrome.test.ts | 66 ++++++++++++- packages/@webreel/core/src/chrome.ts | 94 +++++++++++++------ skills/webreel/SKILL.md | 2 + 5 files changed, 132 insertions(+), 34 deletions(-) diff --git a/apps/docs/src/app/page.mdx b/apps/docs/src/app/page.mdx index 0fb9b1a..0b0907b 100644 --- a/apps/docs/src/app/page.mdx +++ b/apps/docs/src/app/page.mdx @@ -18,7 +18,7 @@ export const metadata = { ## How it works -webreel launches a headless browser, executes your steps, captures frames at ~60fps, composites cursor and keystroke overlays, and encodes the result with ffmpeg. Chrome and ffmpeg are auto-downloaded to `~/.webreel` on first use, or you can set `CHROME_PATH`, `CHROME_HEADLESS_PATH`, and `FFMPEG_PATH`. +webreel launches a headless browser, executes your steps, captures frames at ~60fps, composites cursor and keystroke overlays, and encodes the result with ffmpeg. Chrome and ffmpeg are auto-downloaded to `~/.webreel` on first use, or you can set `CHROME_PATH`, `CHROME_HEADLESS_PATH`, and `FFMPEG_PATH`. Chrome launches with its sandbox enabled by default; set `WEBREEL_NO_SANDBOX=1` if your environment requires it disabled (for example, running as root in a container). ## Get started diff --git a/apps/docs/src/app/quick-start/page.mdx b/apps/docs/src/app/quick-start/page.mdx index a0833a6..e6f0ee4 100644 --- a/apps/docs/src/app/quick-start/page.mdx +++ b/apps/docs/src/app/quick-start/page.mdx @@ -53,7 +53,7 @@ Output is written to `videos/my-video.mp4`. Set the `output` field to use `.gif` - [Node.js](https://nodejs.org/) (v18+) -Chrome and ffmpeg are auto-downloaded to `~/.webreel` on first use, or you can set `CHROME_PATH`, `CHROME_HEADLESS_PATH`, and `FFMPEG_PATH`. +Chrome and ffmpeg are auto-downloaded to `~/.webreel` on first use, or you can set `CHROME_PATH`, `CHROME_HEADLESS_PATH`, and `FFMPEG_PATH`. Chrome launches with its sandbox enabled by default; set `WEBREEL_NO_SANDBOX=1` if your environment requires it disabled (for example, running as root in a container). ## Next steps diff --git a/packages/@webreel/core/src/__tests__/chrome.test.ts b/packages/@webreel/core/src/__tests__/chrome.test.ts index c318405..40cb248 100644 --- a/packages/@webreel/core/src/__tests__/chrome.test.ts +++ b/packages/@webreel/core/src/__tests__/chrome.test.ts @@ -1,5 +1,5 @@ -import { describe, it, expect } from "vitest"; -import { cftPlatform } from "../chrome.js"; +import { describe, it, expect, vi, afterEach } from "vitest"; +import { cftPlatform, buildChromeArgs, shouldDisableSandbox } from "../chrome.js"; describe("cftPlatform", () => { it("returns a valid Chrome for Testing platform string", () => { @@ -25,3 +25,65 @@ describe("cftPlatform", () => { } }); }); + +describe("buildChromeArgs", () => { + const port = 12345; + const userDataDir = "/tmp/webreel-chrome-test"; + + it("excludes --no-sandbox for headless when noSandbox is false", () => { + const args = buildChromeArgs(true, port, userDataDir, false); + expect(args).not.toContain("--no-sandbox"); + }); + + it("includes --no-sandbox for headless when noSandbox is true", () => { + const args = buildChromeArgs(true, port, userDataDir, true); + expect(args).toContain("--no-sandbox"); + }); + + it("excludes --no-sandbox for headful when noSandbox is false", () => { + const args = buildChromeArgs(false, port, userDataDir, false); + expect(args).not.toContain("--no-sandbox"); + }); + + it("includes --no-sandbox for headful when noSandbox is true", () => { + const args = buildChromeArgs(false, port, userDataDir, true); + expect(args).toContain("--no-sandbox"); + }); + + it("always includes the remote debugging port and user data dir", () => { + const args = buildChromeArgs(true, port, userDataDir, false); + expect(args).toContain(`--remote-debugging-port=${port}`); + expect(args).toContain(`--user-data-dir=${userDataDir}`); + }); +}); + +describe("shouldDisableSandbox", () => { + const originalEnv = process.env.WEBREEL_NO_SANDBOX; + + afterEach(() => { + if (originalEnv === undefined) { + delete process.env.WEBREEL_NO_SANDBOX; + } else { + process.env.WEBREEL_NO_SANDBOX = originalEnv; + } + vi.restoreAllMocks(); + }); + + it("returns false by default", () => { + delete process.env.WEBREEL_NO_SANDBOX; + vi.spyOn(process, "getuid").mockReturnValue(501); + expect(shouldDisableSandbox()).toBe(false); + }); + + it("returns true when WEBREEL_NO_SANDBOX=1", () => { + process.env.WEBREEL_NO_SANDBOX = "1"; + vi.spyOn(process, "getuid").mockReturnValue(501); + expect(shouldDisableSandbox()).toBe(true); + }); + + it("returns true when running as root (getuid() === 0)", () => { + delete process.env.WEBREEL_NO_SANDBOX; + vi.spyOn(process, "getuid").mockReturnValue(0); + expect(shouldDisableSandbox()).toBe(true); + }); +}); diff --git a/packages/@webreel/core/src/chrome.ts b/packages/@webreel/core/src/chrome.ts index a983411..9342e66 100644 --- a/packages/@webreel/core/src/chrome.ts +++ b/packages/@webreel/core/src/chrome.ts @@ -197,6 +197,60 @@ export interface LaunchChromeOptions { headless?: boolean; } +/** + * Determine whether Chrome should be launched with its sandbox disabled. + * + * The sandbox is Chrome's primary containment for renderer compromise and is + * enabled by default. It is disabled only when explicitly requested via + * `WEBREEL_NO_SANDBOX=1`, or when running as root (the sandbox refuses to + * start in that case on Linux). + */ +export function shouldDisableSandbox(): boolean { + if (process.env.WEBREEL_NO_SANDBOX === "1") return true; + if (process.getuid?.() === 0) return true; + return false; +} + +export function buildChromeArgs( + headless: boolean, + port: number, + userDataDir: string, + noSandbox: boolean, +): string[] { + const sandboxArgs = noSandbox ? ["--no-sandbox"] : []; + + return headless + ? [ + `--remote-debugging-port=${port}`, + `--user-data-dir=${userDataDir}`, + ...sandboxArgs, + "--hide-scrollbars", + "--enable-begin-frame-control", + "--run-all-compositor-stages-before-draw", + "--disable-threaded-animation", + "--disable-threaded-scrolling", + "--disable-checker-imaging", + "about:blank", + ] + : [ + `--remote-debugging-port=${port}`, + `--user-data-dir=${userDataDir}`, + ...sandboxArgs, + "--no-first-run", + "--no-default-browser-check", + "--disable-extensions", + "--disable-background-networking", + "--disable-background-timer-throttling", + "--disable-backgrounding-occluded-windows", + "--disable-renderer-backgrounding", + "--disable-sync", + "--disable-translate", + "--mute-audio", + "--hide-scrollbars", + "about:blank", + ]; +} + export async function launchChrome( options?: LaunchChromeOptions, ): Promise { @@ -204,41 +258,14 @@ export async function launchChrome( const chromePath = headless ? await ensureHeadlessShell() : await ensureChrome(); let lastError: Error | null = null; + let sandboxFallbackWarned = false; for (let attempt = 1; attempt <= MAX_LAUNCH_ATTEMPTS; attempt++) { const port = await findFreePort(); const userDataDir = mkdtempSync(join(tmpdir(), "webreel-chrome-")); - const args = headless - ? [ - `--remote-debugging-port=${port}`, - `--user-data-dir=${userDataDir}`, - "--no-sandbox", - "--hide-scrollbars", - "--enable-begin-frame-control", - "--run-all-compositor-stages-before-draw", - "--disable-threaded-animation", - "--disable-threaded-scrolling", - "--disable-checker-imaging", - "about:blank", - ] - : [ - `--remote-debugging-port=${port}`, - `--user-data-dir=${userDataDir}`, - "--no-sandbox", - "--no-first-run", - "--no-default-browser-check", - "--disable-extensions", - "--disable-background-networking", - "--disable-background-timer-throttling", - "--disable-backgrounding-occluded-windows", - "--disable-renderer-backgrounding", - "--disable-sync", - "--disable-translate", - "--mute-audio", - "--hide-scrollbars", - "about:blank", - ]; + const noSandbox = shouldDisableSandbox() || sandboxFallbackWarned; + const args = buildChromeArgs(headless, port, userDataDir, noSandbox); const proc = spawn(chromePath, args, { stdio: ["pipe", "pipe", "pipe"], @@ -296,6 +323,13 @@ export async function launchChrome( proc.kill("SIGKILL"); rmSync(userDataDir, { recursive: true, force: true }); + if (!noSandbox && !sandboxFallbackWarned) { + sandboxFallbackWarned = true; + console.warn( + "Chrome failed to launch with its sandbox; retrying without. Set WEBREEL_NO_SANDBOX=1 to skip this probe.", + ); + } + if (attempt < MAX_LAUNCH_ATTEMPTS) { await new Promise((r) => setTimeout(r, 500 * attempt)); } diff --git a/skills/webreel/SKILL.md b/skills/webreel/SKILL.md index 096e0dc..79555b7 100644 --- a/skills/webreel/SKILL.md +++ b/skills/webreel/SKILL.md @@ -34,6 +34,8 @@ To override the auto-downloaded binaries, set these environment variables: - `CHROME_HEADLESS_PATH` - path to a chrome-headless-shell binary (used for recording) - `FFMPEG_PATH` - path to an ffmpeg binary +Chrome launches with its sandbox enabled by default. Set `WEBREEL_NO_SANDBOX=1` to disable it if your environment requires it (for example, running as root in a container); it is also disabled automatically when running as root. + If a recording fails with "No inspectable targets" or similar browser errors, the issue is almost certainly in the webreel config (wrong `waitFor`, missing element, timing), not a missing browser. Check the config and use `--verbose` to debug. ## .gitignore From 452c765b53a4f1ab248dac293df215bfe73b01e2 Mon Sep 17 00:00:00 2001 From: Ariel Conti Date: Sat, 18 Jul 2026 15:39:27 +0100 Subject: [PATCH 32/48] test(core): characterize compositor ffmpeg-argument assembly --- .../core/src/__tests__/compositor.test.ts | 122 ++++++++++++++++++ packages/@webreel/core/src/compositor.ts | 9 +- 2 files changed, 128 insertions(+), 3 deletions(-) create mode 100644 packages/@webreel/core/src/__tests__/compositor.test.ts diff --git a/packages/@webreel/core/src/__tests__/compositor.test.ts b/packages/@webreel/core/src/__tests__/compositor.test.ts new file mode 100644 index 0000000..3e024de --- /dev/null +++ b/packages/@webreel/core/src/__tests__/compositor.test.ts @@ -0,0 +1,122 @@ +import { describe, it, expect, vi } from "vitest"; +import type { TimelineData } from "../timeline.js"; +import { DEFAULT_CURSOR_SVG, DEFAULT_CURSOR_SIZE, DEFAULT_HUD_THEME } from "../types.js"; + +// Characterizes the pure ffmpeg-argument assembly in compositor.ts, plus the +// autozoom "zoom pass" invocation, without spawning real ffmpeg. `spawn` and +// `spawnSync` are faked at the node:child_process boundary (the shim-process +// pattern from recorder.test.ts, adapted to child_process instead of a real +// binary), so `compose()` can run its full three-stage pipeline in-process. + +const hoisted = vi.hoisted(() => { + function makeFakeChildProcess() { + return { + stdin: { + write: () => true, + end: () => {}, + once: (event: string, cb: () => void) => { + if (event === "drain") cb(); + }, + on: () => {}, + }, + stderr: { on: () => {} }, + on: (event: string, cb: (...args: unknown[]) => void) => { + if (event === "close") queueMicrotask(() => cb(0)); + }, + kill: () => {}, + }; + } + + const spawnCalls: string[][] = []; + const spawnMock = (_cmd: string, args: string[]) => { + spawnCalls.push(args); + return makeFakeChildProcess(); + }; + const spawnSyncMock = () => ({ status: 0, stderr: Buffer.from("") }); + + return { spawnCalls, spawnMock, spawnSyncMock }; +}); + +const { spawnCalls, spawnMock, spawnSyncMock } = hoisted; + +vi.mock("node:child_process", () => ({ + spawn: spawnMock, + spawnSync: spawnSyncMock, +})); + +vi.mock("../ffmpeg.js", () => ({ + ensureFfmpeg: async () => "/fake/ffmpeg", +})); + +const { compose, buildMp4Config, buildGifConfig } = await import("../compositor.js"); + +describe("buildMp4Config", () => { + it("enables faststart and embeds the given fps/crf", () => { + const config = buildMp4Config(30, 22, "/tmp/out.mp4"); + + expect(config.outputArgs).toContain("-movflags"); + expect(config.outputArgs[config.outputArgs.indexOf("-movflags") + 1]).toBe( + "+faststart", + ); + expect(config.outputArgs[config.outputArgs.indexOf("-crf") + 1]).toBe("22"); + expect(config.outputArgs[config.outputArgs.indexOf("-r") + 1]).toBe("30"); + expect(config.outputArgs[config.outputArgs.length - 1]).toBe("/tmp/out.mp4"); + expect(config.filterComplex).toBe("[0][1]overlay=0:0:shortest=1"); + }); +}); + +describe("buildGifConfig", () => { + it("builds the palette-based filtergraph scaled to the given width", () => { + const config = buildGifConfig(480, "/tmp/out.gif"); + + expect(config.filterComplex).toContain("fps=15"); + expect(config.filterComplex).toContain("scale=480:-1:flags=lanczos"); + expect(config.filterComplex).toContain("palettegen"); + expect(config.filterComplex).toContain("paletteuse"); + expect(config.outputArgs).toEqual(["-loop", "0", "/tmp/out.gif"]); + }); +}); + +function makeTimelineData(overrides: Partial = {}): TimelineData { + return { + fps: 30, + width: 640, + height: 480, + zoom: 1, + theme: { + cursorSvg: DEFAULT_CURSOR_SVG, + cursorSize: DEFAULT_CURSOR_SIZE, + cursorHotspot: "top-left", + hud: { ...DEFAULT_HUD_THEME }, + }, + frames: [], + events: [], + steps: [], + ...overrides, + }; +} + +describe("compose zoom pass", () => { + it("embeds the provided zoomFilter verbatim in the zoom-pass ffmpeg args", async () => { + spawnCalls.length = 0; + const zoomFilter = "zoompan=z='if(lte(zoom,1.0),1.5,zoom)':d=1:s=640x480"; + + await compose("/tmp/clean.mp4", makeTimelineData(), "/tmp/out.mp4", { + zoomFilter, + }); + + const zoomPassCall = spawnCalls.find((args) => args.includes("-vf")); + expect(zoomPassCall).toBeDefined(); + const vfIndex = zoomPassCall!.indexOf("-vf"); + expect(zoomPassCall![vfIndex + 1]).toBe(zoomFilter); + }); + + it("runs no zoom pass (no -vf) when zoomFilter is not provided", async () => { + spawnCalls.length = 0; + + await compose("/tmp/clean.mp4", makeTimelineData(), "/tmp/out2.mp4"); + + const zoomPassCall = spawnCalls.find((args) => args.includes("-vf")); + expect(zoomPassCall).toBeUndefined(); + }); +}); diff --git a/packages/@webreel/core/src/compositor.ts b/packages/@webreel/core/src/compositor.ts index f21cceb..26863dc 100644 --- a/packages/@webreel/core/src/compositor.ts +++ b/packages/@webreel/core/src/compositor.ts @@ -181,12 +181,12 @@ async function renderCursorPng( return sharp(Buffer.from(svgWithSize)).png().toBuffer(); } -interface CompositorFfmpegConfig { +export interface CompositorFfmpegConfig { filterComplex: string; outputArgs: string[]; } -function buildMp4Config( +export function buildMp4Config( fps: number, crf: number, outputPath: string, @@ -220,7 +220,10 @@ function buildMp4Config( const GIF_FPS = 15; const GIF_BAYER_SCALE = 5; -function buildGifConfig(width: number, outputPath: string): CompositorFfmpegConfig { +export function buildGifConfig( + width: number, + outputPath: string, +): CompositorFfmpegConfig { return { filterComplex: [ `[0][1]overlay=0:0:shortest=1`, From 32b22c43eb9596d7e73b7309e46d9bc1feba0e85 Mon Sep 17 00:00:00 2001 From: Ariel Conti Date: Sat, 18 Jul 2026 15:41:00 +0100 Subject: [PATCH 33/48] test(core): characterize click/key CDP dispatch --- .../core/src/__tests__/actions.test.ts | 127 ++++++++++++++++++ 1 file changed, 127 insertions(+) diff --git a/packages/@webreel/core/src/__tests__/actions.test.ts b/packages/@webreel/core/src/__tests__/actions.test.ts index 9f2eb48..f709b9f 100644 --- a/packages/@webreel/core/src/__tests__/actions.test.ts +++ b/packages/@webreel/core/src/__tests__/actions.test.ts @@ -9,6 +9,8 @@ import { modKeyInfo, resolveCommands, typeText, + clickAt, + pressKey, KEY_CODES, CHAR_CODES, SHORTCUT_COMMANDS, @@ -386,3 +388,128 @@ describe("typeText", () => { expect(client.Input.insertText).not.toHaveBeenCalled(); }); }); + +// characterization: documents current dispatch behavior of clickAt/pressKey, +// not a specification of desired behavior. +describe("clickAt", () => { + function createMockClient() { + return { + Input: { + dispatchMouseEvent: vi.fn().mockResolvedValue(undefined), + dispatchKeyEvent: vi.fn().mockResolvedValue(undefined), + }, + Runtime: { + evaluate: vi.fn().mockResolvedValue({ result: {} }), + }, + } as unknown as CDPClient & { + Input: { + dispatchMouseEvent: ReturnType; + dispatchKeyEvent: ReturnType; + }; + Runtime: { evaluate: ReturnType }; + }; + } + + it("dispatches a pressed/released mouse event pair at the resolved coordinates", async () => { + const ctx = new RecordingContext(); + // Cursor already at the click target: animateMoveTo's dist < 1 fast path + // skips its animation wait, keeping this test fast and deterministic. + ctx.setCursorPosition(50, 60); + const client = createMockClient(); + + await clickAt(ctx, client, 50, 60); + + const mouseCalls = client.Input.dispatchMouseEvent.mock.calls.map((c) => c[0]); + const pressed = mouseCalls.find((c) => c.type === "mousePressed"); + const released = mouseCalls.find((c) => c.type === "mouseReleased"); + + expect(pressed).toMatchObject({ + type: "mousePressed", + x: 50, + y: 60, + button: "left", + clickCount: 1, + }); + expect(released).toMatchObject({ + type: "mouseReleased", + x: 50, + y: 60, + button: "left", + clickCount: 1, + }); + // pressed must precede released. + expect(mouseCalls.indexOf(pressed)).toBeLessThan(mouseCalls.indexOf(released)); + }); + + it("dispatches modifier keyDown/keyUp pairs around the click when modifiers are given", async () => { + const ctx = new RecordingContext(); + ctx.setCursorPosition(10, 10); + const client = createMockClient(); + + await clickAt(ctx, client, 10, 10, ["shift"]); + + const keyCalls = client.Input.dispatchKeyEvent.mock.calls.map((c) => c[0]); + expect(keyCalls).toContainEqual( + expect.objectContaining({ type: "keyDown", key: "Shift", code: "ShiftLeft" }), + ); + expect(keyCalls).toContainEqual( + expect.objectContaining({ type: "keyUp", key: "Shift", code: "ShiftLeft" }), + ); + }); +}); + +describe("pressKey", () => { + function createMockClient() { + return { + Input: { + dispatchKeyEvent: vi.fn().mockResolvedValue(undefined), + }, + Runtime: { + evaluate: vi.fn().mockResolvedValue({ result: {} }), + }, + } as unknown as CDPClient & { + Input: { dispatchKeyEvent: ReturnType }; + Runtime: { evaluate: ReturnType }; + }; + } + + it("dispatches a keyDown followed by a keyUp for the resolved key", async () => { + const ctx = new RecordingContext(); + const client = createMockClient(); + + await pressKey(ctx, client, "Enter"); + + expect(client.Input.dispatchKeyEvent).toHaveBeenCalledTimes(2); + expect(client.Input.dispatchKeyEvent).toHaveBeenNthCalledWith(1, { + type: "keyDown", + key: "Enter", + code: "Enter", + windowsVirtualKeyCode: 13, + modifiers: 0, + commands: undefined, + }); + expect(client.Input.dispatchKeyEvent).toHaveBeenNthCalledWith(2, { + type: "keyUp", + key: "Enter", + code: "Enter", + windowsVirtualKeyCode: 13, + modifiers: 0, + }); + }); + + it("resolves a known shortcut to its command list", async () => { + const ctx = new RecordingContext(); + const client = createMockClient(); + + await pressKey(ctx, client, "ctrl+c"); + + expect(client.Input.dispatchKeyEvent).toHaveBeenNthCalledWith(1, { + type: "keyDown", + key: "c", + code: "KeyC", + windowsVirtualKeyCode: 67, + modifiers: 2, + commands: ["copy"], + }); + }); +}); From acddc32b53e4074b61a01f2e804dc26294f9c075 Mon Sep 17 00:00:00 2001 From: Ariel Conti Date: Sat, 18 Jul 2026 15:42:28 +0100 Subject: [PATCH 34/48] ci: run tests with coverage in CI (report-only, non-gating) --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 81222d8..ae346a3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -34,4 +34,4 @@ jobs: run: pnpm build - name: Test - run: pnpm test + run: pnpm test -- --coverage From eaee320b5998e53bc6f825830f380fad29437833 Mon Sep 17 00:00:00 2001 From: Ariel Conti Date: Sat, 18 Jul 2026 15:43:54 +0100 Subject: [PATCH 35/48] fix: remove unused constructor param flagged by eslint --- packages/webreel/src/lib/__tests__/runner-orchestration.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/webreel/src/lib/__tests__/runner-orchestration.test.ts b/packages/webreel/src/lib/__tests__/runner-orchestration.test.ts index d01e08b..2886b72 100644 --- a/packages/webreel/src/lib/__tests__/runner-orchestration.test.ts +++ b/packages/webreel/src/lib/__tests__/runner-orchestration.test.ts @@ -20,7 +20,7 @@ const hoisted = vi.hoisted(() => { stop = vi.fn().mockResolvedValue(undefined); setTimeline = vi.fn(); getTempVideoPath = vi.fn(() => FakeRecorder.nextTempVideoPath); - constructor(..._args: unknown[]) { + constructor() { FakeRecorder.instances.push(this); } } From f6cda7f2bd2816a24d1c8a9e1aea1181d36d946a Mon Sep 17 00:00:00 2001 From: Ariel Conti Date: Sat, 18 Jul 2026 15:54:17 +0100 Subject: [PATCH 36/48] refactor(core): add shared ffmpeg invocation helper --- packages/@webreel/core/src/ffmpeg-run.ts | 134 +++++++++++++++++++++++ 1 file changed, 134 insertions(+) create mode 100644 packages/@webreel/core/src/ffmpeg-run.ts diff --git a/packages/@webreel/core/src/ffmpeg-run.ts b/packages/@webreel/core/src/ffmpeg-run.ts new file mode 100644 index 0000000..a86c971 --- /dev/null +++ b/packages/@webreel/core/src/ffmpeg-run.ts @@ -0,0 +1,134 @@ +import { spawn, spawnSync, type ChildProcess } from "node:child_process"; + +// Shared across every ffmpeg invocation: only the last couple KB of stderr +// are useful for diagnosing a failure, and buffering more than that in an +// error message is just noise. +const STDERR_TAIL_BYTES = 2000; + +function tailOf(buf: Buffer): string { + return buf.toString().slice(-STDERR_TAIL_BYTES); +} + +/** + * Run ffmpeg synchronously to completion (spawnSync), throwing an Error + * named after `stage` with the last STDERR_TAIL_BYTES of stderr appended + * when it exits non-zero. + */ +export function runFfmpegSync( + ffmpegPath: string, + args: string[], + stage = "ffmpeg", +): void { + const result = spawnSync(ffmpegPath, args, { + stdio: "pipe", + maxBuffer: 50 * 1024 * 1024, + }); + if (result.status !== 0) { + const stderr = result.stderr ? tailOf(result.stderr) : ""; + throw new Error( + `${stage} exited with code ${result.status}${stderr ? `:\n${stderr}` : ""}`, + ); + } +} + +export interface FfmpegStreamingOptions { + /** + * Fired when stdin emits an 'error' event that isn't a benign EPIPE after + * stdin has already been ended (i.e. ffmpeg died mid-stream). Without a + * listener Node treats an unhandled stream 'error' as an uncaught + * exception, so callers should always use this to record/react to the + * failure instead of leaving the pipe unattended. + */ + onPipeError?: (err: NodeJS.ErrnoException) => void; + /** + * Fired when the process closes with a non-zero code before stdin was + * ended. A close that races ahead of stdin.end() means the producer/ + * consumer loop writing to stdin can't rely solely on a subsequent stdin + * 'error' event to unblock: once Node auto-destroys an already-closed + * process's stdin, further writes can return false with no further + * 'error' or 'drain' event ever firing. Treat this as an abort signal. + */ + onPrematureClose?: (err: Error) => void; +} + +export interface FfmpegStreamingHandle { + proc: ChildProcess; + stdin: NodeJS.WritableStream; + /** Resolves on a clean (code 0) close; rejects with a `${stage}`-named, + * stderr-tailed Error otherwise. A no-op .catch() is attached internally + * so an early rejection (e.g. from the abort path killing the process) + * never surfaces as an unhandled rejection; callers should still await + * `done` for its real outcome. */ + done: Promise; + /** SIGTERM, then SIGKILL after a grace period if the process hasn't + * closed by then. The SIGKILL timer is unref'd (never keeps the process + * alive on its own) and cleared as soon as 'close' fires. */ + kill(): void; +} + +const KILL_GRACE_MS = 5_000; + +/** + * Spawn ffmpeg with a piped stdin/stdout/stderr for streaming input (e.g. + * image2pipe frame feeding). Centralizes stderr-tail capture, close/error + * promise wiring, the stdin error contract, and SIGTERM->SIGKILL kill + * escalation so every streaming call site shares the same failure + * semantics. + */ +export function spawnFfmpegStreaming( + ffmpegPath: string, + args: string[], + stage: string, + opts: FfmpegStreamingOptions = {}, +): FfmpegStreamingHandle { + const proc = spawn(ffmpegPath, args, { stdio: ["pipe", "pipe", "pipe"] }); + + const stdin = proc.stdin; + if (!stdin) throw new Error(`${stage} process has no stdin pipe`); + + const stderrChunks: Buffer[] = []; + proc.stderr?.on("data", (chunk: Buffer) => stderrChunks.push(chunk)); + + // Register close/error listeners immediately to avoid missing events. + const done = new Promise((resolveAll, rejectAll) => { + proc.on("close", (code) => { + if (code === 0) { + resolveAll(); + return; + } + const stderr = tailOf(Buffer.concat(stderrChunks)); + const err = new Error( + `${stage} exited with code ${code}${stderr ? `:\n${stderr}` : ""}`, + ); + rejectAll(err); + + // See onPrematureClose doc above for why this matters. + if (!stdin.writableEnded) opts.onPrematureClose?.(err); + }); + proc.on("error", rejectAll); + }); + // done can reject as soon as the process exits, which may be well before + // the caller's abort branch (or its own `await done`) has a chance to + // observe it. Attach a no-op handler now so Node never sees an unhandled + // rejection in that window; the promise is still returned for its real + // outcome. + done.catch(() => {}); + + // EPIPE is expected once ffmpeg has finished reading and the caller has + // called stdin.end(). Before that, it means ffmpeg died mid-stream. + stdin.on("error", (err: NodeJS.ErrnoException) => { + if (err.code === "EPIPE" && stdin.writableEnded) return; + opts.onPipeError?.(err); + }); + + const kill = () => { + proc.kill("SIGTERM"); + const killTimer = setTimeout(() => { + if (!proc.killed) proc.kill("SIGKILL"); + }, KILL_GRACE_MS); + killTimer.unref(); + proc.once("close", () => clearTimeout(killTimer)); + }; + + return { proc, stdin, done, kill }; +} From d2139dafe0c3ed573224d36a8628c667a39b5956 Mon Sep 17 00:00:00 2001 From: Ariel Conti Date: Sat, 18 Jul 2026 15:54:43 +0100 Subject: [PATCH 37/48] refactor(config): split config.ts into lib/config/{loader,includes,validate,errors}.ts Mechanical split along existing landmarks: config.ts now re-exports the public surface (loadWebreelConfig, validateWebreelConfig, etc.) from focused modules. No behavior changes; config.test.ts assertions unchanged. --- packages/webreel/src/lib/config.ts | 1286 +------------------ packages/webreel/src/lib/config/errors.ts | 73 ++ packages/webreel/src/lib/config/includes.ts | 68 + packages/webreel/src/lib/config/loader.ts | 264 ++++ packages/webreel/src/lib/config/validate.ts | 885 +++++++++++++ 5 files changed, 1307 insertions(+), 1269 deletions(-) create mode 100644 packages/webreel/src/lib/config/errors.ts create mode 100644 packages/webreel/src/lib/config/includes.ts create mode 100644 packages/webreel/src/lib/config/loader.ts create mode 100644 packages/webreel/src/lib/config/validate.ts diff --git a/packages/webreel/src/lib/config.ts b/packages/webreel/src/lib/config.ts index 6ce84d9..e123de4 100644 --- a/packages/webreel/src/lib/config.ts +++ b/packages/webreel/src/lib/config.ts @@ -1,1269 +1,17 @@ -import { readFileSync, existsSync } from "node:fs"; -import { resolve, dirname, isAbsolute, extname } from "node:path"; -import { parse as parseJsonc, parseTree, getNodePath } from "jsonc-parser"; -import { createJiti } from "jiti"; -import type { VideoConfig, WebreelConfig } from "./types.js"; -import { VIEWPORT_PRESETS } from "./types.js"; - -export const DEFAULT_CONFIG_NAME = "webreel.config"; -export const DEFAULT_CONFIG_FILE = "webreel.config.json"; -export const CURRENT_SCHEMA_VERSION = 1; - -const CONFIG_EXTENSIONS = [".json", ".ts", ".mts", ".js", ".mjs"]; -const JSON_EXTENSIONS = new Set([".json"]); - -export function parseSchemaVersion(schema?: string): number { - if (!schema) return CURRENT_SCHEMA_VERSION; - const match = schema.match(/\/schema\/v(\d+)\.json/); - if (!match) return -1; - return parseInt(match[1], 10); -} - -async function resolveIncludes( - config: Record, - configDir: string, - seen: Set, -): Promise { - const includes = config.include; - if (!Array.isArray(includes) || includes.length === 0) return []; - - const prependedSteps: unknown[] = []; - - for (const inc of includes) { - if (typeof inc !== "string") continue; - - const absPath = resolve(configDir, inc); - if (seen.has(absPath)) { - throw new Error(`Circular include detected: ${absPath}`); - } - seen.add(absPath); - - const ext = extname(absPath); - let parsed: Record; - - if (JSON_EXTENSIONS.has(ext)) { - let raw: string; - try { - raw = readFileSync(absPath, "utf-8"); - } catch (err) { - throw new Error(`Include file not found: ${absPath}`, { cause: err }); - } - parsed = parseJsonc(raw) as Record; - } else { - try { - const mod = await loadTsConfig(absPath); - if (typeof mod !== "object" || mod === null) { - throw new Error(`Include file must export an object: ${absPath}`); - } - parsed = mod as Record; - } catch (err) { - if (err instanceof Error && err.message.includes("must export")) throw err; - throw new Error(`Include file not found or failed to load: ${absPath}`, { - cause: err, - }); - } - } - - if (!Array.isArray(parsed.steps)) { - throw new Error(`Include file ${absPath} must export a "steps" array`); - } - - const nestedSteps = await resolveIncludes(parsed, dirname(absPath), seen); - prependedSteps.push(...nestedSteps, ...parsed.steps); - } - - return prependedSteps; -} - -function resolveSfxPaths(sfx: VideoConfig["sfx"], configDir: string): VideoConfig["sfx"] { - if (!sfx) return sfx; - const resolved = { ...sfx }; - if (typeof resolved.click === "string" && !isAbsolute(resolved.click)) { - resolved.click = resolve(configDir, resolved.click); - } - if (typeof resolved.key === "string" && !isAbsolute(resolved.key)) { - resolved.key = resolve(configDir, resolved.key); - } - return resolved; -} - -function resolveVideoDefaults( - video: VideoConfig, - defaults: Partial< - Pick< - WebreelConfig, - "baseUrl" | "viewport" | "theme" | "include" | "defaultDelay" | "clickDwell" | "sfx" - > - >, - outDir: string | undefined, - configDir: string, -): VideoConfig { - const resolved = { ...video }; - if (!resolved.baseUrl && defaults.baseUrl) resolved.baseUrl = defaults.baseUrl; - if (!resolved.viewport && defaults.viewport) resolved.viewport = defaults.viewport; - if (defaults.theme) { - resolved.theme = { - cursor: { ...defaults.theme.cursor, ...resolved.theme?.cursor }, - hud: { ...defaults.theme.hud, ...resolved.theme?.hud }, - }; - } - if (!resolved.include && defaults.include) resolved.include = defaults.include; - if (!resolved.sfx && defaults.sfx) resolved.sfx = defaults.sfx; - resolved.sfx = resolveSfxPaths(resolved.sfx, configDir); - if (resolved.defaultDelay === undefined && defaults.defaultDelay !== undefined) - resolved.defaultDelay = defaults.defaultDelay; - if (resolved.clickDwell === undefined && defaults.clickDwell !== undefined) - resolved.clickDwell = defaults.clickDwell; - if (resolved.output && !isAbsolute(resolved.output) && outDir) { - resolved.output = resolve(outDir, resolved.output); - } else if (!resolved.output && outDir) { - resolved.output = resolve(outDir, `${resolved.name}.mp4`); - } - return resolved; -} - -async function loadTsConfig(filePath: string): Promise { - const jiti = createJiti(filePath, { interopDefault: true }); - const mod = await jiti.import(filePath); - return mod; -} - -function resolveViewportValue( - raw: unknown, -): { width: number; height: number } | undefined { - if (typeof raw === "string") return resolveViewportPreset(raw) ?? undefined; - if (typeof raw === "object" && raw !== null) - return raw as { width: number; height: number }; - return undefined; -} - -function substituteEnvVars(obj: unknown): unknown { - if (typeof obj === "string") { - return obj.replace(/\$\{([^}]+)\}|\$([A-Z_][A-Z0-9_]*)/g, (_match, braced, bare) => { - const name = braced ?? bare; - return process.env[name] ?? _match; - }); - } - if (Array.isArray(obj)) return obj.map(substituteEnvVars); - if (typeof obj === "object" && obj !== null) { - const result: Record = {}; - for (const [key, value] of Object.entries(obj)) { - result[key] = substituteEnvVars(value); - } - return result; - } - return obj; -} - -async function buildConfigFromParsed( - parsed: Record, - filePath: string, -): Promise { - if ( - !parsed.videos || - typeof parsed.videos !== "object" || - Array.isArray(parsed.videos) - ) { - throw new Error(`Config must contain a "videos" object`); - } - const videosObj = parsed.videos as Record>; - const configDir = dirname(resolve(filePath)); - const outDir = resolve(configDir, (parsed.outDir as string) ?? "videos"); - const defaults = { - baseUrl: parsed.baseUrl as string | undefined, - viewport: resolveViewportValue(parsed.viewport), - theme: parsed.theme as WebreelConfig["theme"], - sfx: parsed.sfx as WebreelConfig["sfx"], - include: parsed.include as string[] | undefined, - defaultDelay: parsed.defaultDelay as number | undefined, - clickDwell: parsed.clickDwell as number | undefined, - }; - - const videoList: VideoConfig[] = []; - for (const [name, body] of Object.entries(videosObj)) { - const videoBody = { ...body }; - if (typeof videoBody.viewport === "string") { - videoBody.viewport = - resolveViewportPreset(videoBody.viewport as string) ?? videoBody.viewport; - } - const video = { ...videoBody, name } as unknown as VideoConfig; - const resolved = resolveVideoDefaults(video, defaults, outDir, configDir); - videoList.push(await resolveVideo(resolved, filePath)); - } - - return { - $schema: parsed.$schema as string | undefined, - outDir: parsed.outDir as string | undefined, - baseUrl: parsed.baseUrl as string | undefined, - viewport: resolveViewportValue(parsed.viewport), - theme: parsed.theme as WebreelConfig["theme"], - sfx: parsed.sfx as WebreelConfig["sfx"], - include: parsed.include as string[] | undefined, - defaultDelay: parsed.defaultDelay as number | undefined, - clickDwell: parsed.clickDwell as number | undefined, - videos: videoList, - }; -} - -export async function loadWebreelConfig(filePath: string): Promise { - const ext = extname(filePath); - - if (JSON_EXTENSIONS.has(ext)) { - const raw = readFileSync(filePath, "utf-8"); - const parsed = substituteEnvVars(parseJsonc(raw)); - - const schemaUrl = - typeof parsed === "object" && parsed !== null - ? (parsed as Record).$schema - : undefined; - const version = parseSchemaVersion( - typeof schemaUrl === "string" ? schemaUrl : undefined, - ); - const errors = validateWebreelConfig(parsed, version); - if (errors.length > 0) { - const lineMap = buildLineMap(raw); - throw new Error(formatValidationErrors(filePath, errors, lineMap)); - } - - return buildConfigFromParsed(parsed as Record, filePath); - } - - const raw = await loadTsConfig(filePath); - - if (typeof raw !== "object" || raw === null) { - throw new Error(`Config file must export an object: ${filePath}`); - } - - const rawConfig = substituteEnvVars(raw) as Record; - const errors = validateWebreelConfig(rawConfig); - if (errors.length > 0) { - throw new Error(formatValidationErrors(filePath, errors)); - } - - return buildConfigFromParsed(rawConfig, filePath); -} - -async function resolveVideo(video: VideoConfig, filePath: string): Promise { - if (video.include && video.include.length > 0) { - const absConfigPath = resolve(filePath); - const seen = new Set([absConfigPath]); - const includedSteps = await resolveIncludes( - video as unknown as Record, - dirname(absConfigPath), - seen, - ); - const includeErrors: ValidationError[] = []; - for (let i = 0; i < includedSteps.length; i++) { - includeErrors.push( - ...validateStep(includedSteps[i], i).map((e) => ({ - ...e, - path: `include:${e.path}`, - })), - ); - } - if (includeErrors.length > 0) { - const msgs = includeErrors.map((e) => - e.path ? `${e.path}: ${e.message}` : e.message, - ); - throw new Error( - `Invalid included steps for video "${video.name}":\n ${msgs.join("\n ")}`, - ); - } - return { - ...video, - steps: [...(includedSteps as VideoConfig["steps"]), ...video.steps], - }; - } - return video; -} - -const VALID_ACTIONS = new Set([ - "pause", - "click", - "key", - "drag", - "moveTo", - "type", - "scroll", - "wait", - "screenshot", - "navigate", - "navigateHref", - "hover", - "select", - "upload", -]); - -const KNOWN_TOP_LEVEL_KEYS = new Set([ - "$schema", - "outDir", - "baseUrl", - "viewport", - "theme", - "sfx", - "include", - "defaultDelay", - "clickDwell", - "videos", -]); - -const KNOWN_VIDEO_KEYS = new Set([ - "url", - "baseUrl", - "viewport", - "zoom", - "fps", - "quality", - "waitFor", - "output", - "thumbnail", - "include", - "theme", - "sfx", - "defaultDelay", - "clickDwell", - "autoZoom", - "steps", -]); - -const KNOWN_STEP_KEYS: Record> = { - pause: new Set(["action", "ms", "label", "description"]), - click: new Set([ - "action", - "text", - "selector", - "within", - "modifiers", - "label", - "delay", - "description", - ]), - key: new Set(["action", "key", "target", "label", "delay", "description"]), - drag: new Set(["action", "from", "to", "label", "delay", "description"]), - moveTo: new Set([ - "action", - "text", - "selector", - "within", - "label", - "delay", - "description", - ]), - type: new Set([ - "action", - "text", - "selector", - "within", - "charDelay", - "method", - "label", - "delay", - "description", - ]), - scroll: new Set([ - "action", - "x", - "y", - "text", - "selector", - "within", - "label", - "delay", - "description", - ]), - wait: new Set([ - "action", - "selector", - "text", - "within", - "timeout", - "label", - "delay", - "description", - ]), - screenshot: new Set(["action", "output", "label", "delay", "description"]), - navigate: new Set(["action", "url", "label", "delay", "description"]), - navigateHref: new Set(["action", "selector", "label", "delay", "description"]), - hover: new Set([ - "action", - "text", - "selector", - "within", - "label", - "delay", - "description", - ]), - select: new Set([ - "action", - "text", - "selector", - "within", - "value", - "label", - "delay", - "description", - ]), - upload: new Set(["action", "selector", "filePath", "label", "delay", "description"]), -}; - -export interface ValidationError { - path: string; - message: string; -} - -function levenshtein(a: string, b: string): number { - const m = a.length; - const n = b.length; - const dp: number[][] = Array.from({ length: m + 1 }, () => Array(n + 1).fill(0)); - for (let i = 0; i <= m; i++) dp[i][0] = i; - for (let j = 0; j <= n; j++) dp[0][j] = j; - for (let i = 1; i <= m; i++) { - for (let j = 1; j <= n; j++) { - dp[i][j] = Math.min( - dp[i - 1][j] + 1, - dp[i][j - 1] + 1, - dp[i - 1][j - 1] + (a[i - 1] === b[j - 1] ? 0 : 1), - ); - } - } - return dp[m][n]; -} - -function suggestKey(unknown: string, known: Set): string | null { - let best: string | null = null; - let bestDist = Infinity; - for (const k of known) { - const d = levenshtein(unknown.toLowerCase(), k.toLowerCase()); - if (d < bestDist && d <= 2) { - bestDist = d; - best = k; - } - } - return best; -} - -function checkUnknownKeys( - obj: Record, - known: Set, - prefix: string, -): ValidationError[] { - const errors: ValidationError[] = []; - for (const key of Object.keys(obj)) { - if (!known.has(key)) { - const suggestion = suggestKey(key, known); - const hint = suggestion ? ` (did you mean "${suggestion}"?)` : ""; - errors.push({ - path: `${prefix}.${key}`, - message: `Unknown property${hint}`, - }); - } - } - return errors; -} - -function validateStep(step: unknown, index: number): ValidationError[] { - const errors: ValidationError[] = []; - const prefix = `steps[${index}]`; - - if (typeof step !== "object" || step === null) { - errors.push({ path: prefix, message: "Step must be an object" }); - return errors; - } - - const s = step as Record; - - if (!s.action || typeof s.action !== "string") { - errors.push({ path: `${prefix}.action`, message: "Missing or invalid action" }); - return errors; - } - - if (!VALID_ACTIONS.has(s.action)) { - errors.push({ - path: `${prefix}.action`, - message: `Unknown action "${s.action}". Valid actions: ${[...VALID_ACTIONS].join(", ")}`, - }); - return errors; - } - - const knownKeys = KNOWN_STEP_KEYS[s.action]; - if (knownKeys) { - errors.push(...checkUnknownKeys(s, knownKeys, prefix)); - } - - switch (s.action) { - case "pause": - if (!Number.isFinite(s.ms) || (s.ms as number) < 0) { - errors.push({ path: `${prefix}.ms`, message: "Must be a non-negative number" }); - } - break; - - case "click": - if (!s.text && !s.selector) { - errors.push({ - path: prefix, - message: 'Click requires "text" or "selector"', - }); - } - break; - - case "key": - if (typeof s.key !== "string" || s.key.length === 0) { - errors.push({ path: `${prefix}.key`, message: "Must be a non-empty string" }); - } - if ( - s.target !== undefined && - typeof s.target !== "string" && - (typeof s.target !== "object" || s.target === null) - ) { - errors.push({ - path: `${prefix}.target`, - message: "Must be a CSS selector string or an element target object", - }); - } - break; - - case "drag": { - if (!s.from || typeof s.from !== "object") { - errors.push({ - path: `${prefix}.from`, - message: "Must be an object with text or selector", - }); - } else { - const f = s.from as Record; - if (!f.text && !f.selector) { - errors.push({ - path: `${prefix}.from`, - message: 'Requires "text" or "selector"', - }); - } - } - if (!s.to || typeof s.to !== "object") { - errors.push({ - path: `${prefix}.to`, - message: "Must be an object with text or selector", - }); - } else { - const t = s.to as Record; - if (!t.text && !t.selector) { - errors.push({ path: `${prefix}.to`, message: 'Requires "text" or "selector"' }); - } - } - break; - } - - case "type": - if (typeof s.text !== "string" || s.text.length === 0) { - errors.push({ path: `${prefix}.text`, message: "Must be a non-empty string" }); - } - if ( - s.charDelay !== undefined && - (!Number.isFinite(s.charDelay) || (s.charDelay as number) < 0) - ) { - errors.push({ - path: `${prefix}.charDelay`, - message: "Must be a non-negative number", - }); - } - if ( - s.method !== undefined && - s.method !== "insertText" && - s.method !== "dispatchKeyEvent" - ) { - errors.push({ - path: `${prefix}.method`, - message: 'Must be "insertText" or "dispatchKeyEvent"', - }); - } - break; - - case "scroll": - if (s.x !== undefined && !Number.isFinite(s.x)) { - errors.push({ path: `${prefix}.x`, message: "Must be a finite number" }); - } - if (s.y !== undefined && !Number.isFinite(s.y)) { - errors.push({ path: `${prefix}.y`, message: "Must be a finite number" }); - } - break; - - case "wait": { - if (!s.selector && !s.text) { - errors.push({ - path: prefix, - message: 'wait requires "selector" or "text"', - }); - } - if ( - s.timeout !== undefined && - (!Number.isFinite(s.timeout) || (s.timeout as number) <= 0) - ) { - errors.push({ path: `${prefix}.timeout`, message: "Must be a positive number" }); - } - break; - } - - case "moveTo": - if (!s.text && !s.selector) { - errors.push({ - path: prefix, - message: 'moveTo requires "text" or "selector"', - }); - } - break; - - case "screenshot": - if (typeof s.output !== "string" || s.output.length === 0) { - errors.push({ - path: `${prefix}.output`, - message: "Must be a non-empty string", - }); - } - break; - - case "navigate": - if (typeof s.url !== "string" || s.url.length === 0) { - errors.push({ path: `${prefix}.url`, message: "Must be a non-empty string" }); - } - break; - - case "navigateHref": - if (typeof s.selector !== "string" || s.selector.length === 0) { - errors.push({ - path: `${prefix}.selector`, - message: "Must be a non-empty CSS selector string", - }); - } - break; - - case "hover": - if (!s.text && !s.selector) { - errors.push({ - path: prefix, - message: 'hover requires "text" or "selector"', - }); - } - break; - - case "select": { - if (!s.selector && !s.text) { - errors.push({ - path: prefix, - message: 'select requires "text" or "selector"', - }); - } - if (typeof s.value !== "string") { - errors.push({ path: `${prefix}.value`, message: "Must be a string" }); - } - break; - } - - case "upload": - if (typeof s.selector !== "string" || s.selector.length === 0) { - errors.push({ - path: `${prefix}.selector`, - message: "Must be a non-empty string", - }); - } - if (typeof s.filePath !== "string" || s.filePath.length === 0) { - errors.push({ - path: `${prefix}.filePath`, - message: "Must be a non-empty string", - }); - } - break; - } - - if (s.delay !== undefined && (!Number.isFinite(s.delay) || (s.delay as number) < 0)) { - errors.push({ path: `${prefix}.delay`, message: "Must be a non-negative number" }); - } - - if (s.label !== undefined && typeof s.label !== "string") { - errors.push({ path: `${prefix}.label`, message: "Must be a string" }); - } - - if (s.description !== undefined && typeof s.description !== "string") { - errors.push({ path: `${prefix}.description`, message: "Must be a string" }); - } - - return errors; -} - -function resolveViewportPreset(value: string): { width: number; height: number } | null { - return VIEWPORT_PRESETS[value] ?? null; -} - -function validateViewport(viewport: unknown, prefix: string): ValidationError[] { - const errors: ValidationError[] = []; - if (typeof viewport === "string") { - if (!resolveViewportPreset(viewport)) { - const presetNames = Object.keys(VIEWPORT_PRESETS).join(", "); - errors.push({ - path: prefix, - message: `Unknown viewport preset "${viewport}". Valid presets: ${presetNames}`, - }); - } - } else if (typeof viewport !== "object" || viewport === null) { - errors.push({ - path: prefix, - message: "Must be a preset string or an object with width and height", - }); - } else { - const v = viewport as Record; - if (!Number.isFinite(v.width) || (v.width as number) <= 0) { - errors.push({ path: `${prefix}.width`, message: "Must be a positive number" }); - } - if (!Number.isFinite(v.height) || (v.height as number) <= 0) { - errors.push({ path: `${prefix}.height`, message: "Must be a positive number" }); - } - } - return errors; -} - -function validateInclude(include: unknown, prefix: string): ValidationError[] { - const errors: ValidationError[] = []; - if (!Array.isArray(include)) { - errors.push({ path: prefix, message: "Must be an array of file paths" }); - } else { - for (let i = 0; i < include.length; i++) { - if (typeof include[i] !== "string" || include[i].length === 0) { - errors.push({ path: `${prefix}[${i}]`, message: "Must be a non-empty string" }); - } - } - } - return errors; -} - -const VALID_SFX_VARIANTS = new Set([1, 2, 3, 4]); - -function isValidSfxValue(value: unknown): boolean { - return VALID_SFX_VARIANTS.has(value as number) || typeof value === "string"; -} - -function validateSfx(sfx: unknown, prefix: string): ValidationError[] { - const errors: ValidationError[] = []; - if (typeof sfx !== "object" || sfx === null) { - errors.push({ path: prefix, message: "Must be an object" }); - return errors; - } - const s = sfx as Record; - if (s.click !== undefined && !isValidSfxValue(s.click)) { - errors.push({ - path: `${prefix}.click`, - message: "Must be 1, 2, 3, 4, or a file path", - }); - } - if (s.key !== undefined && !isValidSfxValue(s.key)) { - errors.push({ path: `${prefix}.key`, message: "Must be 1, 2, 3, 4, or a file path" }); - } - return errors; -} - -const KNOWN_AUTOZOOM_KEYS = new Set([ - "enabled", - "approachS", - "settleBeforeS", - "holdAfterS", - "releaseS", - "paddingRatio", - "minZoomRatio", - "skipZoomRatio", - "sessionGapS", - "minPanS", -]); - -const AUTOZOOM_NONNEGATIVE_KEYS = [ - "approachS", - "settleBeforeS", - "holdAfterS", - "releaseS", - "paddingRatio", - "sessionGapS", - "minPanS", -] as const; - -const AUTOZOOM_RATIO_KEYS = ["minZoomRatio", "skipZoomRatio"] as const; - -function validateAutoZoom(autoZoom: unknown, prefix: string): ValidationError[] { - const errors: ValidationError[] = []; - if (typeof autoZoom === "boolean") return errors; - if (typeof autoZoom !== "object" || autoZoom === null) { - errors.push({ - path: prefix, - message: "Must be a boolean or an autozoom config object", - }); - return errors; - } - - const a = autoZoom as Record; - - errors.push(...checkUnknownKeys(a, KNOWN_AUTOZOOM_KEYS, prefix)); - - if (a.enabled !== undefined && typeof a.enabled !== "boolean") { - errors.push({ path: `${prefix}.enabled`, message: "Must be a boolean" }); - } - - for (const key of AUTOZOOM_NONNEGATIVE_KEYS) { - const value = a[key]; - if (value !== undefined && (!Number.isFinite(value) || (value as number) < 0)) { - errors.push({ - path: `${prefix}.${key}`, - message: "Must be a non-negative number", - }); - } - } - - for (const key of AUTOZOOM_RATIO_KEYS) { - const value = a[key]; - if ( - value !== undefined && - (!Number.isFinite(value) || (value as number) < 0 || (value as number) > 1) - ) { - errors.push({ - path: `${prefix}.${key}`, - message: "Must be a number between 0 and 1", - }); - } - } - - return errors; -} - -function validateTheme(theme: unknown, prefix: string): ValidationError[] { - const errors: ValidationError[] = []; - if (typeof theme !== "object" || theme === null) { - errors.push({ path: prefix, message: "Must be an object" }); - return errors; - } - - const t = theme as Record; - - if (t.cursor !== undefined) { - if (typeof t.cursor !== "object" || t.cursor === null) { - errors.push({ path: `${prefix}.cursor`, message: "Must be an object" }); - } else { - const cur = t.cursor as Record; - if (cur.image !== undefined && typeof cur.image !== "string") { - errors.push({ - path: `${prefix}.cursor.image`, - message: "Must be a string (file path)", - }); - } - if ( - cur.size !== undefined && - (!Number.isFinite(cur.size) || (cur.size as number) <= 0) - ) { - errors.push({ - path: `${prefix}.cursor.size`, - message: "Must be a positive number", - }); - } - if ( - cur.hotspot !== undefined && - cur.hotspot !== "top-left" && - cur.hotspot !== "center" - ) { - errors.push({ - path: `${prefix}.cursor.hotspot`, - message: 'Must be "top-left" or "center"', - }); - } - } - } - - if (t.hud !== undefined) { - if (typeof t.hud !== "object" || t.hud === null) { - errors.push({ path: `${prefix}.hud`, message: "Must be an object" }); - } else { - const h = t.hud as Record; - if (h.background !== undefined && typeof h.background !== "string") { - errors.push({ path: `${prefix}.hud.background`, message: "Must be a string" }); - } - if (h.color !== undefined && typeof h.color !== "string") { - errors.push({ path: `${prefix}.hud.color`, message: "Must be a string" }); - } - if ( - h.fontSize !== undefined && - (!Number.isFinite(h.fontSize) || (h.fontSize as number) <= 0) - ) { - errors.push({ - path: `${prefix}.hud.fontSize`, - message: "Must be a positive number", - }); - } - if (h.fontFamily !== undefined && typeof h.fontFamily !== "string") { - errors.push({ path: `${prefix}.hud.fontFamily`, message: "Must be a string" }); - } - if ( - h.borderRadius !== undefined && - (!Number.isFinite(h.borderRadius) || (h.borderRadius as number) < 0) - ) { - errors.push({ - path: `${prefix}.hud.borderRadius`, - message: "Must be a non-negative number", - }); - } - if (h.position !== undefined && h.position !== "top" && h.position !== "bottom") { - errors.push({ - path: `${prefix}.hud.position`, - message: 'Must be "top" or "bottom"', - }); - } - } - } - - return errors; -} - -export function validateWebreelConfig( - config: unknown, - version: number = CURRENT_SCHEMA_VERSION, -): ValidationError[] { - if (version !== 1) { - return [ - { - path: "$schema", - message: `Unsupported schema version: v${version}. This version of webreel supports v1.`, - }, - ]; - } - - const errors: ValidationError[] = []; - - if (typeof config !== "object" || config === null) { - errors.push({ path: "", message: "Config must be an object" }); - return errors; - } - - const c = config as Record; - - errors.push(...checkUnknownKeys(c, KNOWN_TOP_LEVEL_KEYS, "")); - - if (c.outDir !== undefined && (typeof c.outDir !== "string" || c.outDir.length === 0)) { - errors.push({ path: "outDir", message: "Must be a non-empty string" }); - } - - if (c.baseUrl !== undefined && typeof c.baseUrl !== "string") { - errors.push({ path: "baseUrl", message: "Must be a string" }); - } - - if (c.viewport !== undefined) { - errors.push(...validateViewport(c.viewport, "viewport")); - } - - if ( - c.defaultDelay !== undefined && - (!Number.isFinite(c.defaultDelay) || (c.defaultDelay as number) < 0) - ) { - errors.push({ path: "defaultDelay", message: "Must be a non-negative number" }); - } - - if ( - c.clickDwell !== undefined && - (!Number.isFinite(c.clickDwell) || (c.clickDwell as number) < 0) - ) { - errors.push({ path: "clickDwell", message: "Must be a non-negative number" }); - } - - if (c.include !== undefined) { - errors.push(...validateInclude(c.include, "include")); - } - - if (c.theme !== undefined) { - errors.push(...validateTheme(c.theme, "theme")); - } - - if (c.sfx !== undefined) { - errors.push(...validateSfx(c.sfx, "sfx")); - } - - if ( - c.videos === undefined || - c.videos === null || - typeof c.videos !== "object" || - Array.isArray(c.videos) - ) { - errors.push({ - path: "videos", - message: "Required, must be an object mapping names to video configs", - }); - return errors; - } - - const videos = c.videos as Record; - const names = Object.keys(videos); - - if (names.length === 0) { - errors.push({ path: "videos", message: "Must contain at least one video" }); - } - - for (const name of names) { - const video = videos[name]; - const prefix = `videos.${name}`; - - if (typeof video !== "object" || video === null) { - errors.push({ path: prefix, message: "Must be a video config object" }); - continue; - } - - const d = video as Record; - - errors.push(...checkUnknownKeys(d, KNOWN_VIDEO_KEYS, prefix)); - - if (typeof d.url !== "string" || d.url.length === 0) { - errors.push({ - path: `${prefix}.url`, - message: "Required, must be a non-empty string", - }); - } - - if (d.zoom !== undefined && (!Number.isFinite(d.zoom) || (d.zoom as number) <= 0)) { - errors.push({ path: `${prefix}.zoom`, message: "Must be a positive number" }); - } - - if ( - d.fps !== undefined && - (!Number.isFinite(d.fps) || (d.fps as number) < 1 || (d.fps as number) > 120) - ) { - errors.push({ - path: `${prefix}.fps`, - message: "Must be a number between 1 and 120", - }); - } - - if ( - d.quality !== undefined && - (!Number.isFinite(d.quality) || - (d.quality as number) < 1 || - (d.quality as number) > 100) - ) { - errors.push({ - path: `${prefix}.quality`, - message: "Must be a number between 1 and 100", - }); - } - - if (d.viewport !== undefined) { - errors.push(...validateViewport(d.viewport, `${prefix}.viewport`)); - } - - if (d.include !== undefined) { - errors.push(...validateInclude(d.include, `${prefix}.include`)); - } - - if ( - d.output !== undefined && - (typeof d.output !== "string" || d.output.length === 0) - ) { - errors.push({ path: `${prefix}.output`, message: "Must be a non-empty string" }); - } - - if (d.waitFor !== undefined) { - if (typeof d.waitFor === "string") { - if (d.waitFor.length === 0) { - errors.push({ - path: `${prefix}.waitFor`, - message: "Must be a non-empty string", - }); - } - } else if (typeof d.waitFor === "object" && d.waitFor !== null) { - const wf = d.waitFor as Record; - if (!wf.selector && !wf.text) { - errors.push({ - path: `${prefix}.waitFor`, - message: 'Must have "selector" or "text"', - }); - } - } else { - errors.push({ - path: `${prefix}.waitFor`, - message: "Must be a CSS selector string or an object with selector/text", - }); - } - } - - if ( - d.defaultDelay !== undefined && - (!Number.isFinite(d.defaultDelay) || (d.defaultDelay as number) < 0) - ) { - errors.push({ - path: `${prefix}.defaultDelay`, - message: "Must be a non-negative number", - }); - } - - if ( - d.clickDwell !== undefined && - (!Number.isFinite(d.clickDwell) || (d.clickDwell as number) < 0) - ) { - errors.push({ - path: `${prefix}.clickDwell`, - message: "Must be a non-negative number", - }); - } - - if (d.thumbnail !== undefined) { - if (typeof d.thumbnail !== "object" || d.thumbnail === null) { - errors.push({ path: `${prefix}.thumbnail`, message: "Must be an object" }); - } else { - const th = d.thumbnail as Record; - if ( - th.time !== undefined && - (!Number.isFinite(th.time) || (th.time as number) < 0) - ) { - errors.push({ - path: `${prefix}.thumbnail.time`, - message: "Must be a non-negative number (seconds)", - }); - } - if (th.enabled !== undefined && typeof th.enabled !== "boolean") { - errors.push({ - path: `${prefix}.thumbnail.enabled`, - message: "Must be a boolean", - }); - } - } - } - - if (d.theme !== undefined) { - errors.push(...validateTheme(d.theme, `${prefix}.theme`)); - } - - if (d.sfx !== undefined) { - errors.push(...validateSfx(d.sfx, `${prefix}.sfx`)); - } - - if (d.autoZoom !== undefined) { - errors.push(...validateAutoZoom(d.autoZoom, `${prefix}.autoZoom`)); - } - - if (!Array.isArray(d.steps)) { - errors.push({ path: `${prefix}.steps`, message: "Required, must be an array" }); - } else { - for (let j = 0; j < d.steps.length; j++) { - errors.push( - ...validateStep(d.steps[j], j).map((e) => ({ - ...e, - path: `${prefix}.${e.path}`, - })), - ); - } - } - } - - return errors; -} - -export function buildLineMap(raw: string): Map { - const lineMap = new Map(); - const tree = parseTree(raw); - if (!tree) return lineMap; - - function walk(node: ReturnType): void { - if (!node) return; - const path = getNodePath(node); - const jsonPath = path - .map((seg) => (typeof seg === "number" ? `[${seg}]` : seg)) - .join(".") - .replace(/\.\[/g, "["); - - const line = raw.substring(0, node.offset).split("\n").length; - if (jsonPath) lineMap.set(jsonPath, line); - - if (node.children) { - for (const child of node.children) { - walk(child); - } - } - } - - if (tree.children) { - for (const child of tree.children) { - walk(child); - } - } - - return lineMap; -} - -function findLineForPath( - lineMap: Map, - errorPath: string, -): number | undefined { - if (lineMap.has(errorPath)) return lineMap.get(errorPath); - const parts = errorPath.split("."); - while (parts.length > 0) { - parts.pop(); - const parent = parts.join("."); - if (lineMap.has(parent)) return lineMap.get(parent); - } - return undefined; -} - -export function formatValidationErrors( - filePath: string, - errors: ValidationError[], - lineMap?: Map, -): string { - const red = (s: string) => `\x1b[31m${s}\x1b[0m`; - const dim = (s: string) => `\x1b[2m${s}\x1b[0m`; - const bold = (s: string) => `\x1b[1m${s}\x1b[0m`; - const yellow = (s: string) => `\x1b[33m${s}\x1b[0m`; - - const maxPath = Math.max(...errors.map((e) => e.path.length)); - const lines = errors.map((e) => { - const paddedPath = e.path.padEnd(maxPath); - const lineNum = lineMap ? findLineForPath(lineMap, e.path) : undefined; - const linePrefix = lineNum !== undefined ? yellow(`L${lineNum} `) : ""; - return ` ${linePrefix}${red(paddedPath)} ${dim(e.message)}`; - }); - - return `${bold(red("Error:"))} Invalid config ${bold(filePath)}\n\n${lines.join("\n")}`; -} - -export function getConfigDir(configPath: string): string { - return dirname(resolve(configPath)); -} - -export function filterVideosByName( - videos: VideoConfig[], - names: string[], -): VideoConfig[] { - if (names.length === 0) return videos; - const filtered = videos.filter((v) => names.includes(v.name)); - const found = new Set(filtered.map((v) => v.name)); - const missing = names.filter((n) => !found.has(n)); - if (missing.length > 0) { - const available = videos.map((v) => v.name).join(", "); - throw new Error(`Video(s) not found: ${missing.join(", ")}. Available: ${available}`); - } - return filtered; -} - -export function resolveConfigPath(configPath?: string): string { - if (configPath) { - const resolved = resolve(configPath); - if (!existsSync(resolved)) { - throw new Error(`Config file not found: ${resolved}`); - } - return resolved; - } - - let dir = process.cwd(); - const root = resolve("/"); - - while (true) { - for (const ext of CONFIG_EXTENSIONS) { - const candidate = resolve(dir, `${DEFAULT_CONFIG_NAME}${ext}`); - if (existsSync(candidate)) { - return candidate; - } - } - - const parent = dirname(dir); - if (parent === dir || dir === root) break; - dir = parent; - } - - throw new Error( - `No config file found. Create a ${DEFAULT_CONFIG_FILE} or specify one with --config.`, - ); -} +export { + DEFAULT_CONFIG_NAME, + DEFAULT_CONFIG_FILE, + loadWebreelConfig, + getConfigDir, + filterVideosByName, + resolveConfigPath, +} from "./config/loader.js"; + +export { + CURRENT_SCHEMA_VERSION, + parseSchemaVersion, + validateWebreelConfig, +} from "./config/validate.js"; + +export type { ValidationError } from "./config/errors.js"; +export { buildLineMap, formatValidationErrors } from "./config/errors.js"; diff --git a/packages/webreel/src/lib/config/errors.ts b/packages/webreel/src/lib/config/errors.ts new file mode 100644 index 0000000..e72d62e --- /dev/null +++ b/packages/webreel/src/lib/config/errors.ts @@ -0,0 +1,73 @@ +import { parseTree, getNodePath } from "jsonc-parser"; + +export interface ValidationError { + path: string; + message: string; +} + +export function buildLineMap(raw: string): Map { + const lineMap = new Map(); + const tree = parseTree(raw); + if (!tree) return lineMap; + + function walk(node: ReturnType): void { + if (!node) return; + const path = getNodePath(node); + const jsonPath = path + .map((seg) => (typeof seg === "number" ? `[${seg}]` : seg)) + .join(".") + .replace(/\.\[/g, "["); + + const line = raw.substring(0, node.offset).split("\n").length; + if (jsonPath) lineMap.set(jsonPath, line); + + if (node.children) { + for (const child of node.children) { + walk(child); + } + } + } + + if (tree.children) { + for (const child of tree.children) { + walk(child); + } + } + + return lineMap; +} + +function findLineForPath( + lineMap: Map, + errorPath: string, +): number | undefined { + if (lineMap.has(errorPath)) return lineMap.get(errorPath); + const parts = errorPath.split("."); + while (parts.length > 0) { + parts.pop(); + const parent = parts.join("."); + if (lineMap.has(parent)) return lineMap.get(parent); + } + return undefined; +} + +export function formatValidationErrors( + filePath: string, + errors: ValidationError[], + lineMap?: Map, +): string { + const red = (s: string) => `\x1b[31m${s}\x1b[0m`; + const dim = (s: string) => `\x1b[2m${s}\x1b[0m`; + const bold = (s: string) => `\x1b[1m${s}\x1b[0m`; + const yellow = (s: string) => `\x1b[33m${s}\x1b[0m`; + + const maxPath = Math.max(...errors.map((e) => e.path.length)); + const lines = errors.map((e) => { + const paddedPath = e.path.padEnd(maxPath); + const lineNum = lineMap ? findLineForPath(lineMap, e.path) : undefined; + const linePrefix = lineNum !== undefined ? yellow(`L${lineNum} `) : ""; + return ` ${linePrefix}${red(paddedPath)} ${dim(e.message)}`; + }); + + return `${bold(red("Error:"))} Invalid config ${bold(filePath)}\n\n${lines.join("\n")}`; +} diff --git a/packages/webreel/src/lib/config/includes.ts b/packages/webreel/src/lib/config/includes.ts new file mode 100644 index 0000000..894f87c --- /dev/null +++ b/packages/webreel/src/lib/config/includes.ts @@ -0,0 +1,68 @@ +import { readFileSync } from "node:fs"; +import { resolve, dirname, extname } from "node:path"; +import { parse as parseJsonc } from "jsonc-parser"; +import { createJiti } from "jiti"; + +const JSON_EXTENSIONS = new Set([".json"]); + +export async function loadTsConfig(filePath: string): Promise { + const jiti = createJiti(filePath, { interopDefault: true }); + const mod = await jiti.import(filePath); + return mod; +} + +export async function resolveIncludes( + config: Record, + configDir: string, + seen: Set, +): Promise { + const includes = config.include; + if (!Array.isArray(includes) || includes.length === 0) return []; + + const prependedSteps: unknown[] = []; + + for (const inc of includes) { + if (typeof inc !== "string") continue; + + const absPath = resolve(configDir, inc); + if (seen.has(absPath)) { + throw new Error(`Circular include detected: ${absPath}`); + } + seen.add(absPath); + + const ext = extname(absPath); + let parsed: Record; + + if (JSON_EXTENSIONS.has(ext)) { + let raw: string; + try { + raw = readFileSync(absPath, "utf-8"); + } catch (err) { + throw new Error(`Include file not found: ${absPath}`, { cause: err }); + } + parsed = parseJsonc(raw) as Record; + } else { + try { + const mod = await loadTsConfig(absPath); + if (typeof mod !== "object" || mod === null) { + throw new Error(`Include file must export an object: ${absPath}`); + } + parsed = mod as Record; + } catch (err) { + if (err instanceof Error && err.message.includes("must export")) throw err; + throw new Error(`Include file not found or failed to load: ${absPath}`, { + cause: err, + }); + } + } + + if (!Array.isArray(parsed.steps)) { + throw new Error(`Include file ${absPath} must export a "steps" array`); + } + + const nestedSteps = await resolveIncludes(parsed, dirname(absPath), seen); + prependedSteps.push(...nestedSteps, ...parsed.steps); + } + + return prependedSteps; +} diff --git a/packages/webreel/src/lib/config/loader.ts b/packages/webreel/src/lib/config/loader.ts new file mode 100644 index 0000000..287ca4e --- /dev/null +++ b/packages/webreel/src/lib/config/loader.ts @@ -0,0 +1,264 @@ +import { readFileSync, existsSync } from "node:fs"; +import { resolve, dirname, isAbsolute, extname } from "node:path"; +import { parse as parseJsonc } from "jsonc-parser"; +import type { VideoConfig, WebreelConfig } from "../types.js"; +import { resolveIncludes, loadTsConfig } from "./includes.js"; +import { + validateStep, + validateWebreelConfig, + resolveViewportPreset, + parseSchemaVersion, +} from "./validate.js"; +import type { ValidationError } from "./errors.js"; +import { buildLineMap, formatValidationErrors } from "./errors.js"; + +export const DEFAULT_CONFIG_NAME = "webreel.config"; +export const DEFAULT_CONFIG_FILE = "webreel.config.json"; + +const CONFIG_EXTENSIONS = [".json", ".ts", ".mts", ".js", ".mjs"]; +const JSON_EXTENSIONS = new Set([".json"]); + +function resolveSfxPaths(sfx: VideoConfig["sfx"], configDir: string): VideoConfig["sfx"] { + if (!sfx) return sfx; + const resolved = { ...sfx }; + if (typeof resolved.click === "string" && !isAbsolute(resolved.click)) { + resolved.click = resolve(configDir, resolved.click); + } + if (typeof resolved.key === "string" && !isAbsolute(resolved.key)) { + resolved.key = resolve(configDir, resolved.key); + } + return resolved; +} + +function resolveVideoDefaults( + video: VideoConfig, + defaults: Partial< + Pick< + WebreelConfig, + "baseUrl" | "viewport" | "theme" | "include" | "defaultDelay" | "clickDwell" | "sfx" + > + >, + outDir: string | undefined, + configDir: string, +): VideoConfig { + const resolved = { ...video }; + if (!resolved.baseUrl && defaults.baseUrl) resolved.baseUrl = defaults.baseUrl; + if (!resolved.viewport && defaults.viewport) resolved.viewport = defaults.viewport; + if (defaults.theme) { + resolved.theme = { + cursor: { ...defaults.theme.cursor, ...resolved.theme?.cursor }, + hud: { ...defaults.theme.hud, ...resolved.theme?.hud }, + }; + } + if (!resolved.include && defaults.include) resolved.include = defaults.include; + if (!resolved.sfx && defaults.sfx) resolved.sfx = defaults.sfx; + resolved.sfx = resolveSfxPaths(resolved.sfx, configDir); + if (resolved.defaultDelay === undefined && defaults.defaultDelay !== undefined) + resolved.defaultDelay = defaults.defaultDelay; + if (resolved.clickDwell === undefined && defaults.clickDwell !== undefined) + resolved.clickDwell = defaults.clickDwell; + if (resolved.output && !isAbsolute(resolved.output) && outDir) { + resolved.output = resolve(outDir, resolved.output); + } else if (!resolved.output && outDir) { + resolved.output = resolve(outDir, `${resolved.name}.mp4`); + } + return resolved; +} + +function resolveViewportValue( + raw: unknown, +): { width: number; height: number } | undefined { + if (typeof raw === "string") return resolveViewportPreset(raw) ?? undefined; + if (typeof raw === "object" && raw !== null) + return raw as { width: number; height: number }; + return undefined; +} + +function substituteEnvVars(obj: unknown): unknown { + if (typeof obj === "string") { + return obj.replace(/\$\{([^}]+)\}|\$([A-Z_][A-Z0-9_]*)/g, (_match, braced, bare) => { + const name = braced ?? bare; + return process.env[name] ?? _match; + }); + } + if (Array.isArray(obj)) return obj.map(substituteEnvVars); + if (typeof obj === "object" && obj !== null) { + const result: Record = {}; + for (const [key, value] of Object.entries(obj)) { + result[key] = substituteEnvVars(value); + } + return result; + } + return obj; +} + +async function buildConfigFromParsed( + parsed: Record, + filePath: string, +): Promise { + if ( + !parsed.videos || + typeof parsed.videos !== "object" || + Array.isArray(parsed.videos) + ) { + throw new Error(`Config must contain a "videos" object`); + } + const videosObj = parsed.videos as Record>; + const configDir = dirname(resolve(filePath)); + const outDir = resolve(configDir, (parsed.outDir as string) ?? "videos"); + const defaults = { + baseUrl: parsed.baseUrl as string | undefined, + viewport: resolveViewportValue(parsed.viewport), + theme: parsed.theme as WebreelConfig["theme"], + sfx: parsed.sfx as WebreelConfig["sfx"], + include: parsed.include as string[] | undefined, + defaultDelay: parsed.defaultDelay as number | undefined, + clickDwell: parsed.clickDwell as number | undefined, + }; + + const videoList: VideoConfig[] = []; + for (const [name, body] of Object.entries(videosObj)) { + const videoBody = { ...body }; + if (typeof videoBody.viewport === "string") { + videoBody.viewport = + resolveViewportPreset(videoBody.viewport as string) ?? videoBody.viewport; + } + const video = { ...videoBody, name } as unknown as VideoConfig; + const resolved = resolveVideoDefaults(video, defaults, outDir, configDir); + videoList.push(await resolveVideo(resolved, filePath)); + } + + return { + $schema: parsed.$schema as string | undefined, + outDir: parsed.outDir as string | undefined, + baseUrl: parsed.baseUrl as string | undefined, + viewport: resolveViewportValue(parsed.viewport), + theme: parsed.theme as WebreelConfig["theme"], + sfx: parsed.sfx as WebreelConfig["sfx"], + include: parsed.include as string[] | undefined, + defaultDelay: parsed.defaultDelay as number | undefined, + clickDwell: parsed.clickDwell as number | undefined, + videos: videoList, + }; +} + +export async function loadWebreelConfig(filePath: string): Promise { + const ext = extname(filePath); + + if (JSON_EXTENSIONS.has(ext)) { + const raw = readFileSync(filePath, "utf-8"); + const parsed = substituteEnvVars(parseJsonc(raw)); + + const schemaUrl = + typeof parsed === "object" && parsed !== null + ? (parsed as Record).$schema + : undefined; + const version = parseSchemaVersion( + typeof schemaUrl === "string" ? schemaUrl : undefined, + ); + const errors = validateWebreelConfig(parsed, version); + if (errors.length > 0) { + const lineMap = buildLineMap(raw); + throw new Error(formatValidationErrors(filePath, errors, lineMap)); + } + + return buildConfigFromParsed(parsed as Record, filePath); + } + + const raw = await loadTsConfig(filePath); + + if (typeof raw !== "object" || raw === null) { + throw new Error(`Config file must export an object: ${filePath}`); + } + + const rawConfig = substituteEnvVars(raw) as Record; + const errors = validateWebreelConfig(rawConfig); + if (errors.length > 0) { + throw new Error(formatValidationErrors(filePath, errors)); + } + + return buildConfigFromParsed(rawConfig, filePath); +} + +async function resolveVideo(video: VideoConfig, filePath: string): Promise { + if (video.include && video.include.length > 0) { + const absConfigPath = resolve(filePath); + const seen = new Set([absConfigPath]); + const includedSteps = await resolveIncludes( + video as unknown as Record, + dirname(absConfigPath), + seen, + ); + const includeErrors: ValidationError[] = []; + for (let i = 0; i < includedSteps.length; i++) { + includeErrors.push( + ...validateStep(includedSteps[i], i).map((e) => ({ + ...e, + path: `include:${e.path}`, + })), + ); + } + if (includeErrors.length > 0) { + const msgs = includeErrors.map((e) => + e.path ? `${e.path}: ${e.message}` : e.message, + ); + throw new Error( + `Invalid included steps for video "${video.name}":\n ${msgs.join("\n ")}`, + ); + } + return { + ...video, + steps: [...(includedSteps as VideoConfig["steps"]), ...video.steps], + }; + } + return video; +} + +export function getConfigDir(configPath: string): string { + return dirname(resolve(configPath)); +} + +export function filterVideosByName( + videos: VideoConfig[], + names: string[], +): VideoConfig[] { + if (names.length === 0) return videos; + const filtered = videos.filter((v) => names.includes(v.name)); + const found = new Set(filtered.map((v) => v.name)); + const missing = names.filter((n) => !found.has(n)); + if (missing.length > 0) { + const available = videos.map((v) => v.name).join(", "); + throw new Error(`Video(s) not found: ${missing.join(", ")}. Available: ${available}`); + } + return filtered; +} + +export function resolveConfigPath(configPath?: string): string { + if (configPath) { + const resolved = resolve(configPath); + if (!existsSync(resolved)) { + throw new Error(`Config file not found: ${resolved}`); + } + return resolved; + } + + let dir = process.cwd(); + const root = resolve("/"); + + while (true) { + for (const ext of CONFIG_EXTENSIONS) { + const candidate = resolve(dir, `${DEFAULT_CONFIG_NAME}${ext}`); + if (existsSync(candidate)) { + return candidate; + } + } + + const parent = dirname(dir); + if (parent === dir || dir === root) break; + dir = parent; + } + + throw new Error( + `No config file found. Create a ${DEFAULT_CONFIG_FILE} or specify one with --config.`, + ); +} diff --git a/packages/webreel/src/lib/config/validate.ts b/packages/webreel/src/lib/config/validate.ts new file mode 100644 index 0000000..7058b66 --- /dev/null +++ b/packages/webreel/src/lib/config/validate.ts @@ -0,0 +1,885 @@ +import { VIEWPORT_PRESETS } from "../types.js"; +import type { ValidationError } from "./errors.js"; + +export type { ValidationError } from "./errors.js"; + +export const CURRENT_SCHEMA_VERSION = 1; + +export function parseSchemaVersion(schema?: string): number { + if (!schema) return CURRENT_SCHEMA_VERSION; + const match = schema.match(/\/schema\/v(\d+)\.json/); + if (!match) return -1; + return parseInt(match[1], 10); +} + +const VALID_ACTIONS = new Set([ + "pause", + "click", + "key", + "drag", + "moveTo", + "type", + "scroll", + "wait", + "screenshot", + "navigate", + "navigateHref", + "hover", + "select", + "upload", +]); + +const KNOWN_TOP_LEVEL_KEYS = new Set([ + "$schema", + "outDir", + "baseUrl", + "viewport", + "theme", + "sfx", + "include", + "defaultDelay", + "clickDwell", + "videos", +]); + +const KNOWN_VIDEO_KEYS = new Set([ + "url", + "baseUrl", + "viewport", + "zoom", + "fps", + "quality", + "waitFor", + "output", + "thumbnail", + "include", + "theme", + "sfx", + "defaultDelay", + "clickDwell", + "autoZoom", + "steps", +]); + +const KNOWN_STEP_KEYS: Record> = { + pause: new Set(["action", "ms", "label", "description"]), + click: new Set([ + "action", + "text", + "selector", + "within", + "modifiers", + "label", + "delay", + "description", + ]), + key: new Set(["action", "key", "target", "label", "delay", "description"]), + drag: new Set(["action", "from", "to", "label", "delay", "description"]), + moveTo: new Set([ + "action", + "text", + "selector", + "within", + "label", + "delay", + "description", + ]), + type: new Set([ + "action", + "text", + "selector", + "within", + "charDelay", + "method", + "label", + "delay", + "description", + ]), + scroll: new Set([ + "action", + "x", + "y", + "text", + "selector", + "within", + "label", + "delay", + "description", + ]), + wait: new Set([ + "action", + "selector", + "text", + "within", + "timeout", + "label", + "delay", + "description", + ]), + screenshot: new Set(["action", "output", "label", "delay", "description"]), + navigate: new Set(["action", "url", "label", "delay", "description"]), + navigateHref: new Set(["action", "selector", "label", "delay", "description"]), + hover: new Set([ + "action", + "text", + "selector", + "within", + "label", + "delay", + "description", + ]), + select: new Set([ + "action", + "text", + "selector", + "within", + "value", + "label", + "delay", + "description", + ]), + upload: new Set(["action", "selector", "filePath", "label", "delay", "description"]), +}; + +function levenshtein(a: string, b: string): number { + const m = a.length; + const n = b.length; + const dp: number[][] = Array.from({ length: m + 1 }, () => Array(n + 1).fill(0)); + for (let i = 0; i <= m; i++) dp[i][0] = i; + for (let j = 0; j <= n; j++) dp[0][j] = j; + for (let i = 1; i <= m; i++) { + for (let j = 1; j <= n; j++) { + dp[i][j] = Math.min( + dp[i - 1][j] + 1, + dp[i][j - 1] + 1, + dp[i - 1][j - 1] + (a[i - 1] === b[j - 1] ? 0 : 1), + ); + } + } + return dp[m][n]; +} + +function suggestKey(unknown: string, known: Set): string | null { + let best: string | null = null; + let bestDist = Infinity; + for (const k of known) { + const d = levenshtein(unknown.toLowerCase(), k.toLowerCase()); + if (d < bestDist && d <= 2) { + bestDist = d; + best = k; + } + } + return best; +} + +function checkUnknownKeys( + obj: Record, + known: Set, + prefix: string, +): ValidationError[] { + const errors: ValidationError[] = []; + for (const key of Object.keys(obj)) { + if (!known.has(key)) { + const suggestion = suggestKey(key, known); + const hint = suggestion ? ` (did you mean "${suggestion}"?)` : ""; + errors.push({ + path: `${prefix}.${key}`, + message: `Unknown property${hint}`, + }); + } + } + return errors; +} + +export function validateStep(step: unknown, index: number): ValidationError[] { + const errors: ValidationError[] = []; + const prefix = `steps[${index}]`; + + if (typeof step !== "object" || step === null) { + errors.push({ path: prefix, message: "Step must be an object" }); + return errors; + } + + const s = step as Record; + + if (!s.action || typeof s.action !== "string") { + errors.push({ path: `${prefix}.action`, message: "Missing or invalid action" }); + return errors; + } + + if (!VALID_ACTIONS.has(s.action)) { + errors.push({ + path: `${prefix}.action`, + message: `Unknown action "${s.action}". Valid actions: ${[...VALID_ACTIONS].join(", ")}`, + }); + return errors; + } + + const knownKeys = KNOWN_STEP_KEYS[s.action]; + if (knownKeys) { + errors.push(...checkUnknownKeys(s, knownKeys, prefix)); + } + + switch (s.action) { + case "pause": + if (!Number.isFinite(s.ms) || (s.ms as number) < 0) { + errors.push({ path: `${prefix}.ms`, message: "Must be a non-negative number" }); + } + break; + + case "click": + if (!s.text && !s.selector) { + errors.push({ + path: prefix, + message: 'Click requires "text" or "selector"', + }); + } + break; + + case "key": + if (typeof s.key !== "string" || s.key.length === 0) { + errors.push({ path: `${prefix}.key`, message: "Must be a non-empty string" }); + } + if ( + s.target !== undefined && + typeof s.target !== "string" && + (typeof s.target !== "object" || s.target === null) + ) { + errors.push({ + path: `${prefix}.target`, + message: "Must be a CSS selector string or an element target object", + }); + } + break; + + case "drag": { + if (!s.from || typeof s.from !== "object") { + errors.push({ + path: `${prefix}.from`, + message: "Must be an object with text or selector", + }); + } else { + const f = s.from as Record; + if (!f.text && !f.selector) { + errors.push({ + path: `${prefix}.from`, + message: 'Requires "text" or "selector"', + }); + } + } + if (!s.to || typeof s.to !== "object") { + errors.push({ + path: `${prefix}.to`, + message: "Must be an object with text or selector", + }); + } else { + const t = s.to as Record; + if (!t.text && !t.selector) { + errors.push({ path: `${prefix}.to`, message: 'Requires "text" or "selector"' }); + } + } + break; + } + + case "type": + if (typeof s.text !== "string" || s.text.length === 0) { + errors.push({ path: `${prefix}.text`, message: "Must be a non-empty string" }); + } + if ( + s.charDelay !== undefined && + (!Number.isFinite(s.charDelay) || (s.charDelay as number) < 0) + ) { + errors.push({ + path: `${prefix}.charDelay`, + message: "Must be a non-negative number", + }); + } + if ( + s.method !== undefined && + s.method !== "insertText" && + s.method !== "dispatchKeyEvent" + ) { + errors.push({ + path: `${prefix}.method`, + message: 'Must be "insertText" or "dispatchKeyEvent"', + }); + } + break; + + case "scroll": + if (s.x !== undefined && !Number.isFinite(s.x)) { + errors.push({ path: `${prefix}.x`, message: "Must be a finite number" }); + } + if (s.y !== undefined && !Number.isFinite(s.y)) { + errors.push({ path: `${prefix}.y`, message: "Must be a finite number" }); + } + break; + + case "wait": { + if (!s.selector && !s.text) { + errors.push({ + path: prefix, + message: 'wait requires "selector" or "text"', + }); + } + if ( + s.timeout !== undefined && + (!Number.isFinite(s.timeout) || (s.timeout as number) <= 0) + ) { + errors.push({ path: `${prefix}.timeout`, message: "Must be a positive number" }); + } + break; + } + + case "moveTo": + if (!s.text && !s.selector) { + errors.push({ + path: prefix, + message: 'moveTo requires "text" or "selector"', + }); + } + break; + + case "screenshot": + if (typeof s.output !== "string" || s.output.length === 0) { + errors.push({ + path: `${prefix}.output`, + message: "Must be a non-empty string", + }); + } + break; + + case "navigate": + if (typeof s.url !== "string" || s.url.length === 0) { + errors.push({ path: `${prefix}.url`, message: "Must be a non-empty string" }); + } + break; + + case "navigateHref": + if (typeof s.selector !== "string" || s.selector.length === 0) { + errors.push({ + path: `${prefix}.selector`, + message: "Must be a non-empty CSS selector string", + }); + } + break; + + case "hover": + if (!s.text && !s.selector) { + errors.push({ + path: prefix, + message: 'hover requires "text" or "selector"', + }); + } + break; + + case "select": { + if (!s.selector && !s.text) { + errors.push({ + path: prefix, + message: 'select requires "text" or "selector"', + }); + } + if (typeof s.value !== "string") { + errors.push({ path: `${prefix}.value`, message: "Must be a string" }); + } + break; + } + + case "upload": + if (typeof s.selector !== "string" || s.selector.length === 0) { + errors.push({ + path: `${prefix}.selector`, + message: "Must be a non-empty string", + }); + } + if (typeof s.filePath !== "string" || s.filePath.length === 0) { + errors.push({ + path: `${prefix}.filePath`, + message: "Must be a non-empty string", + }); + } + break; + } + + if (s.delay !== undefined && (!Number.isFinite(s.delay) || (s.delay as number) < 0)) { + errors.push({ path: `${prefix}.delay`, message: "Must be a non-negative number" }); + } + + if (s.label !== undefined && typeof s.label !== "string") { + errors.push({ path: `${prefix}.label`, message: "Must be a string" }); + } + + if (s.description !== undefined && typeof s.description !== "string") { + errors.push({ path: `${prefix}.description`, message: "Must be a string" }); + } + + return errors; +} + +export function resolveViewportPreset( + value: string, +): { width: number; height: number } | null { + return VIEWPORT_PRESETS[value] ?? null; +} + +function validateViewport(viewport: unknown, prefix: string): ValidationError[] { + const errors: ValidationError[] = []; + if (typeof viewport === "string") { + if (!resolveViewportPreset(viewport)) { + const presetNames = Object.keys(VIEWPORT_PRESETS).join(", "); + errors.push({ + path: prefix, + message: `Unknown viewport preset "${viewport}". Valid presets: ${presetNames}`, + }); + } + } else if (typeof viewport !== "object" || viewport === null) { + errors.push({ + path: prefix, + message: "Must be a preset string or an object with width and height", + }); + } else { + const v = viewport as Record; + if (!Number.isFinite(v.width) || (v.width as number) <= 0) { + errors.push({ path: `${prefix}.width`, message: "Must be a positive number" }); + } + if (!Number.isFinite(v.height) || (v.height as number) <= 0) { + errors.push({ path: `${prefix}.height`, message: "Must be a positive number" }); + } + } + return errors; +} + +function validateInclude(include: unknown, prefix: string): ValidationError[] { + const errors: ValidationError[] = []; + if (!Array.isArray(include)) { + errors.push({ path: prefix, message: "Must be an array of file paths" }); + } else { + for (let i = 0; i < include.length; i++) { + if (typeof include[i] !== "string" || include[i].length === 0) { + errors.push({ path: `${prefix}[${i}]`, message: "Must be a non-empty string" }); + } + } + } + return errors; +} + +const VALID_SFX_VARIANTS = new Set([1, 2, 3, 4]); + +function isValidSfxValue(value: unknown): boolean { + return VALID_SFX_VARIANTS.has(value as number) || typeof value === "string"; +} + +function validateSfx(sfx: unknown, prefix: string): ValidationError[] { + const errors: ValidationError[] = []; + if (typeof sfx !== "object" || sfx === null) { + errors.push({ path: prefix, message: "Must be an object" }); + return errors; + } + const s = sfx as Record; + if (s.click !== undefined && !isValidSfxValue(s.click)) { + errors.push({ + path: `${prefix}.click`, + message: "Must be 1, 2, 3, 4, or a file path", + }); + } + if (s.key !== undefined && !isValidSfxValue(s.key)) { + errors.push({ path: `${prefix}.key`, message: "Must be 1, 2, 3, 4, or a file path" }); + } + return errors; +} + +const KNOWN_AUTOZOOM_KEYS = new Set([ + "enabled", + "approachS", + "settleBeforeS", + "holdAfterS", + "releaseS", + "paddingRatio", + "minZoomRatio", + "skipZoomRatio", + "sessionGapS", + "minPanS", +]); + +const AUTOZOOM_NONNEGATIVE_KEYS = [ + "approachS", + "settleBeforeS", + "holdAfterS", + "releaseS", + "paddingRatio", + "sessionGapS", + "minPanS", +] as const; + +const AUTOZOOM_RATIO_KEYS = ["minZoomRatio", "skipZoomRatio"] as const; + +function validateAutoZoom(autoZoom: unknown, prefix: string): ValidationError[] { + const errors: ValidationError[] = []; + if (typeof autoZoom === "boolean") return errors; + if (typeof autoZoom !== "object" || autoZoom === null) { + errors.push({ + path: prefix, + message: "Must be a boolean or an autozoom config object", + }); + return errors; + } + + const a = autoZoom as Record; + + errors.push(...checkUnknownKeys(a, KNOWN_AUTOZOOM_KEYS, prefix)); + + if (a.enabled !== undefined && typeof a.enabled !== "boolean") { + errors.push({ path: `${prefix}.enabled`, message: "Must be a boolean" }); + } + + for (const key of AUTOZOOM_NONNEGATIVE_KEYS) { + const value = a[key]; + if (value !== undefined && (!Number.isFinite(value) || (value as number) < 0)) { + errors.push({ + path: `${prefix}.${key}`, + message: "Must be a non-negative number", + }); + } + } + + for (const key of AUTOZOOM_RATIO_KEYS) { + const value = a[key]; + if ( + value !== undefined && + (!Number.isFinite(value) || (value as number) < 0 || (value as number) > 1) + ) { + errors.push({ + path: `${prefix}.${key}`, + message: "Must be a number between 0 and 1", + }); + } + } + + return errors; +} + +function validateTheme(theme: unknown, prefix: string): ValidationError[] { + const errors: ValidationError[] = []; + if (typeof theme !== "object" || theme === null) { + errors.push({ path: prefix, message: "Must be an object" }); + return errors; + } + + const t = theme as Record; + + if (t.cursor !== undefined) { + if (typeof t.cursor !== "object" || t.cursor === null) { + errors.push({ path: `${prefix}.cursor`, message: "Must be an object" }); + } else { + const cur = t.cursor as Record; + if (cur.image !== undefined && typeof cur.image !== "string") { + errors.push({ + path: `${prefix}.cursor.image`, + message: "Must be a string (file path)", + }); + } + if ( + cur.size !== undefined && + (!Number.isFinite(cur.size) || (cur.size as number) <= 0) + ) { + errors.push({ + path: `${prefix}.cursor.size`, + message: "Must be a positive number", + }); + } + if ( + cur.hotspot !== undefined && + cur.hotspot !== "top-left" && + cur.hotspot !== "center" + ) { + errors.push({ + path: `${prefix}.cursor.hotspot`, + message: 'Must be "top-left" or "center"', + }); + } + } + } + + if (t.hud !== undefined) { + if (typeof t.hud !== "object" || t.hud === null) { + errors.push({ path: `${prefix}.hud`, message: "Must be an object" }); + } else { + const h = t.hud as Record; + if (h.background !== undefined && typeof h.background !== "string") { + errors.push({ path: `${prefix}.hud.background`, message: "Must be a string" }); + } + if (h.color !== undefined && typeof h.color !== "string") { + errors.push({ path: `${prefix}.hud.color`, message: "Must be a string" }); + } + if ( + h.fontSize !== undefined && + (!Number.isFinite(h.fontSize) || (h.fontSize as number) <= 0) + ) { + errors.push({ + path: `${prefix}.hud.fontSize`, + message: "Must be a positive number", + }); + } + if (h.fontFamily !== undefined && typeof h.fontFamily !== "string") { + errors.push({ path: `${prefix}.hud.fontFamily`, message: "Must be a string" }); + } + if ( + h.borderRadius !== undefined && + (!Number.isFinite(h.borderRadius) || (h.borderRadius as number) < 0) + ) { + errors.push({ + path: `${prefix}.hud.borderRadius`, + message: "Must be a non-negative number", + }); + } + if (h.position !== undefined && h.position !== "top" && h.position !== "bottom") { + errors.push({ + path: `${prefix}.hud.position`, + message: 'Must be "top" or "bottom"', + }); + } + } + } + + return errors; +} + +export function validateWebreelConfig( + config: unknown, + version: number = CURRENT_SCHEMA_VERSION, +): ValidationError[] { + if (version !== 1) { + return [ + { + path: "$schema", + message: `Unsupported schema version: v${version}. This version of webreel supports v1.`, + }, + ]; + } + + const errors: ValidationError[] = []; + + if (typeof config !== "object" || config === null) { + errors.push({ path: "", message: "Config must be an object" }); + return errors; + } + + const c = config as Record; + + errors.push(...checkUnknownKeys(c, KNOWN_TOP_LEVEL_KEYS, "")); + + if (c.outDir !== undefined && (typeof c.outDir !== "string" || c.outDir.length === 0)) { + errors.push({ path: "outDir", message: "Must be a non-empty string" }); + } + + if (c.baseUrl !== undefined && typeof c.baseUrl !== "string") { + errors.push({ path: "baseUrl", message: "Must be a string" }); + } + + if (c.viewport !== undefined) { + errors.push(...validateViewport(c.viewport, "viewport")); + } + + if ( + c.defaultDelay !== undefined && + (!Number.isFinite(c.defaultDelay) || (c.defaultDelay as number) < 0) + ) { + errors.push({ path: "defaultDelay", message: "Must be a non-negative number" }); + } + + if ( + c.clickDwell !== undefined && + (!Number.isFinite(c.clickDwell) || (c.clickDwell as number) < 0) + ) { + errors.push({ path: "clickDwell", message: "Must be a non-negative number" }); + } + + if (c.include !== undefined) { + errors.push(...validateInclude(c.include, "include")); + } + + if (c.theme !== undefined) { + errors.push(...validateTheme(c.theme, "theme")); + } + + if (c.sfx !== undefined) { + errors.push(...validateSfx(c.sfx, "sfx")); + } + + if ( + c.videos === undefined || + c.videos === null || + typeof c.videos !== "object" || + Array.isArray(c.videos) + ) { + errors.push({ + path: "videos", + message: "Required, must be an object mapping names to video configs", + }); + return errors; + } + + const videos = c.videos as Record; + const names = Object.keys(videos); + + if (names.length === 0) { + errors.push({ path: "videos", message: "Must contain at least one video" }); + } + + for (const name of names) { + const video = videos[name]; + const prefix = `videos.${name}`; + + if (typeof video !== "object" || video === null) { + errors.push({ path: prefix, message: "Must be a video config object" }); + continue; + } + + const d = video as Record; + + errors.push(...checkUnknownKeys(d, KNOWN_VIDEO_KEYS, prefix)); + + if (typeof d.url !== "string" || d.url.length === 0) { + errors.push({ + path: `${prefix}.url`, + message: "Required, must be a non-empty string", + }); + } + + if (d.zoom !== undefined && (!Number.isFinite(d.zoom) || (d.zoom as number) <= 0)) { + errors.push({ path: `${prefix}.zoom`, message: "Must be a positive number" }); + } + + if ( + d.fps !== undefined && + (!Number.isFinite(d.fps) || (d.fps as number) < 1 || (d.fps as number) > 120) + ) { + errors.push({ + path: `${prefix}.fps`, + message: "Must be a number between 1 and 120", + }); + } + + if ( + d.quality !== undefined && + (!Number.isFinite(d.quality) || + (d.quality as number) < 1 || + (d.quality as number) > 100) + ) { + errors.push({ + path: `${prefix}.quality`, + message: "Must be a number between 1 and 100", + }); + } + + if (d.viewport !== undefined) { + errors.push(...validateViewport(d.viewport, `${prefix}.viewport`)); + } + + if (d.include !== undefined) { + errors.push(...validateInclude(d.include, `${prefix}.include`)); + } + + if ( + d.output !== undefined && + (typeof d.output !== "string" || d.output.length === 0) + ) { + errors.push({ path: `${prefix}.output`, message: "Must be a non-empty string" }); + } + + if (d.waitFor !== undefined) { + if (typeof d.waitFor === "string") { + if (d.waitFor.length === 0) { + errors.push({ + path: `${prefix}.waitFor`, + message: "Must be a non-empty string", + }); + } + } else if (typeof d.waitFor === "object" && d.waitFor !== null) { + const wf = d.waitFor as Record; + if (!wf.selector && !wf.text) { + errors.push({ + path: `${prefix}.waitFor`, + message: 'Must have "selector" or "text"', + }); + } + } else { + errors.push({ + path: `${prefix}.waitFor`, + message: "Must be a CSS selector string or an object with selector/text", + }); + } + } + + if ( + d.defaultDelay !== undefined && + (!Number.isFinite(d.defaultDelay) || (d.defaultDelay as number) < 0) + ) { + errors.push({ + path: `${prefix}.defaultDelay`, + message: "Must be a non-negative number", + }); + } + + if ( + d.clickDwell !== undefined && + (!Number.isFinite(d.clickDwell) || (d.clickDwell as number) < 0) + ) { + errors.push({ + path: `${prefix}.clickDwell`, + message: "Must be a non-negative number", + }); + } + + if (d.thumbnail !== undefined) { + if (typeof d.thumbnail !== "object" || d.thumbnail === null) { + errors.push({ path: `${prefix}.thumbnail`, message: "Must be an object" }); + } else { + const th = d.thumbnail as Record; + if ( + th.time !== undefined && + (!Number.isFinite(th.time) || (th.time as number) < 0) + ) { + errors.push({ + path: `${prefix}.thumbnail.time`, + message: "Must be a non-negative number (seconds)", + }); + } + if (th.enabled !== undefined && typeof th.enabled !== "boolean") { + errors.push({ + path: `${prefix}.thumbnail.enabled`, + message: "Must be a boolean", + }); + } + } + } + + if (d.theme !== undefined) { + errors.push(...validateTheme(d.theme, `${prefix}.theme`)); + } + + if (d.sfx !== undefined) { + errors.push(...validateSfx(d.sfx, `${prefix}.sfx`)); + } + + if (d.autoZoom !== undefined) { + errors.push(...validateAutoZoom(d.autoZoom, `${prefix}.autoZoom`)); + } + + if (!Array.isArray(d.steps)) { + errors.push({ path: `${prefix}.steps`, message: "Required, must be an array" }); + } else { + for (let j = 0; j < d.steps.length; j++) { + errors.push( + ...validateStep(d.steps[j], j).map((e) => ({ + ...e, + path: `${prefix}.${e.path}`, + })), + ); + } + } + } + + return errors; +} From 982831b0685a2d3f4183f1c77ca38e793492d685 Mon Sep 17 00:00:00 2001 From: Ariel Conti Date: Sat, 18 Jul 2026 15:55:29 +0100 Subject: [PATCH 38/48] refactor(core): route media.ts through runFfmpegSync helper --- packages/@webreel/core/src/media.ts | 174 +++++++++++++--------------- 1 file changed, 80 insertions(+), 94 deletions(-) diff --git a/packages/@webreel/core/src/media.ts b/packages/@webreel/core/src/media.ts index bd8c7eb..edcd647 100644 --- a/packages/@webreel/core/src/media.ts +++ b/packages/@webreel/core/src/media.ts @@ -1,25 +1,12 @@ -import { spawnSync } from "node:child_process"; import { rmSync } from "node:fs"; import { moveFileSync } from "./fs.js"; import { resolve, dirname } from "node:path"; import { fileURLToPath } from "node:url"; import type { SoundEvent } from "./types.js"; +import { runFfmpegSync } from "./ffmpeg-run.js"; const __dirname = dirname(fileURLToPath(import.meta.url)); -function runFfmpeg(ffmpegPath: string, args: string[]): void { - const result = spawnSync(ffmpegPath, args, { - stdio: "pipe", - maxBuffer: 50 * 1024 * 1024, - }); - if (result.status !== 0) { - const stderr = result.stderr?.toString().slice(-2000) ?? ""; - throw new Error( - `ffmpeg exited with code ${result.status}${stderr ? `:\n${stderr}` : ""}`, - ); - } -} - const ASSETS_DIR = resolve(__dirname, "..", "assets"); export interface SfxConfig { @@ -100,16 +87,11 @@ export function finalizeMp4( ): void { if (events.length === 0 || !options?.sfx) { if (options?.remux) { - runFfmpeg(ffmpegPath, [ - "-y", - "-i", - tempVideo, - "-c", - "copy", - "-movflags", - "+faststart", - outputPath, - ]); + runFfmpegSync( + ffmpegPath, + ["-y", "-i", tempVideo, "-c", "copy", "-movflags", "+faststart", outputPath], + "ffmpeg remux", + ); } else { moveFileSync(tempVideo, outputPath); } @@ -123,26 +105,30 @@ export function finalizeMp4( options.sfx, ); - runFfmpeg(ffmpegPath, [ - "-y", - ...inputArgs, - "-filter_complex", - filterComplex, - "-map", - "0:v", - "-map", - "[aout]", - "-c:v", - "copy", - "-c:a", - "aac", - "-b:a", - "128k", - "-shortest", - "-movflags", - "+faststart", - outputPath, - ]); + runFfmpegSync( + ffmpegPath, + [ + "-y", + ...inputArgs, + "-filter_complex", + filterComplex, + "-map", + "0:v", + "-map", + "[aout]", + "-c:v", + "copy", + "-c:a", + "aac", + "-b:a", + "128k", + "-shortest", + "-movflags", + "+faststart", + outputPath, + ], + "ffmpeg mp4 audio mix", + ); } export function finalizeWebm( @@ -155,20 +141,24 @@ export function finalizeWebm( ): void { const silentWebm = tempVideo + "_silent.webm"; - runFfmpeg(ffmpegPath, [ - "-y", - "-i", - tempVideo, - "-c:v", - "libvpx-vp9", - "-crf", - "30", - "-b:v", - "0", - "-pix_fmt", - "yuv420p", - silentWebm, - ]); + runFfmpegSync( + ffmpegPath, + [ + "-y", + "-i", + tempVideo, + "-c:v", + "libvpx-vp9", + "-crf", + "30", + "-b:v", + "0", + "-pix_fmt", + "yuv420p", + silentWebm, + ], + "ffmpeg webm encode", + ); if (events.length === 0 || !sfx) { moveFileSync(silentWebm, outputPath); @@ -183,24 +173,28 @@ export function finalizeWebm( sfx, ); - runFfmpeg(ffmpegPath, [ - "-y", - ...inputArgs, - "-filter_complex", - filterComplex, - "-map", - "0:v", - "-map", - "[aout]", - "-c:v", - "copy", - "-c:a", - "libopus", - "-b:a", - "128k", - "-shortest", - outputPath, - ]); + runFfmpegSync( + ffmpegPath, + [ + "-y", + ...inputArgs, + "-filter_complex", + filterComplex, + "-map", + "0:v", + "-map", + "[aout]", + "-c:v", + "copy", + "-c:a", + "libopus", + "-b:a", + "128k", + "-shortest", + outputPath, + ], + "ffmpeg webm audio mix", + ); } finally { rmSync(silentWebm, { force: true }); } @@ -212,16 +206,11 @@ export function extractThumbnail( outputPath: string, timeSec: number, ): void { - runFfmpeg(ffmpegPath, [ - "-y", - "-ss", - String(timeSec), - "-i", - videoPath, - "-frames:v", - "1", - outputPath, - ]); + runFfmpegSync( + ffmpegPath, + ["-y", "-ss", String(timeSec), "-i", videoPath, "-frames:v", "1", outputPath], + "ffmpeg thumbnail extract", + ); } export const GIF_FPS = 15; @@ -243,12 +232,9 @@ export function finalizeGif( outputPath: string, width: number, ): void { - runFfmpeg(ffmpegPath, [ - "-y", - "-i", - tempVideo, - "-vf", - buildGifFilter(width), - outputPath, - ]); + runFfmpegSync( + ffmpegPath, + ["-y", "-i", tempVideo, "-vf", buildGifFilter(width), outputPath], + "ffmpeg gif encode", + ); } From 008a8218968e3a24b3cd0889cca4793308763184 Mon Sep 17 00:00:00 2001 From: Ariel Conti Date: Sat, 18 Jul 2026 15:56:55 +0100 Subject: [PATCH 39/48] refactor(core): route applyZoomPass through runFfmpegAsync helper --- packages/@webreel/core/src/compositor.ts | 29 ++++++---------------- packages/@webreel/core/src/ffmpeg-run.ts | 31 ++++++++++++++++++++++++ 2 files changed, 38 insertions(+), 22 deletions(-) diff --git a/packages/@webreel/core/src/compositor.ts b/packages/@webreel/core/src/compositor.ts index 452954d..730d36c 100644 --- a/packages/@webreel/core/src/compositor.ts +++ b/packages/@webreel/core/src/compositor.ts @@ -6,6 +6,7 @@ import sharp from "sharp"; import type { TimelineData } from "./timeline.js"; import { ensureFfmpeg } from "./ffmpeg.js"; import { buildGifFilter, finalizeMp4, finalizeWebm, type SfxConfig } from "./media.js"; +import { runFfmpegAsync } from "./ffmpeg-run.js"; interface OverlayContext { cursorPng: Buffer; @@ -217,7 +218,10 @@ export function buildMp4Config( }; } -export function buildGifConfig(width: number, outputPath: string): CompositorFfmpegConfig { +export function buildGifConfig( + width: number, + outputPath: string, +): CompositorFfmpegConfig { return { filterComplex: `[0][1]overlay=0:0:shortest=1,${buildGifFilter(width)}`, outputArgs: ["-loop", "0", outputPath], @@ -486,7 +490,7 @@ async function applyZoomPass( crf: number, fps: number, ): Promise { - const ffmpeg = spawn( + await runFfmpegAsync( ffmpegPath, [ "-y", @@ -514,27 +518,8 @@ async function applyZoomPass( String(fps), outputPath, ], - { stdio: ["ignore", "pipe", "pipe"] }, + "Zoom-pass ffmpeg", ); - - const stderrChunks: Buffer[] = []; - ffmpeg.stderr?.on("data", (chunk: Buffer) => stderrChunks.push(chunk)); - - await new Promise((resolveAll, rejectAll) => { - ffmpeg.on("close", (code) => { - if (code === 0) { - resolveAll(); - } else { - const stderr = Buffer.concat(stderrChunks).toString().slice(-2000); - rejectAll( - new Error( - `Zoom-pass ffmpeg exited with code ${code}${stderr ? `:\n${stderr}` : ""}`, - ), - ); - } - }); - ffmpeg.on("error", rejectAll); - }); } type OverlayLayer = "both" | "cursor" | "hud"; diff --git a/packages/@webreel/core/src/ffmpeg-run.ts b/packages/@webreel/core/src/ffmpeg-run.ts index a86c971..98d66c2 100644 --- a/packages/@webreel/core/src/ffmpeg-run.ts +++ b/packages/@webreel/core/src/ffmpeg-run.ts @@ -31,6 +31,37 @@ export function runFfmpegSync( } } +/** + * Run ffmpeg asynchronously to completion for file-in/file-out invocations + * that don't pipe stdin (e.g. a single -i/-vf/-o pass). Resolves on a clean + * (code 0) close; rejects with a `${stage}`-named, stderr-tailed Error + * otherwise. + */ +export function runFfmpegAsync( + ffmpegPath: string, + args: string[], + stage = "ffmpeg", +): Promise { + const proc = spawn(ffmpegPath, args, { stdio: ["ignore", "pipe", "pipe"] }); + + const stderrChunks: Buffer[] = []; + proc.stderr?.on("data", (chunk: Buffer) => stderrChunks.push(chunk)); + + return new Promise((resolveAll, rejectAll) => { + proc.on("close", (code) => { + if (code === 0) { + resolveAll(); + return; + } + const stderr = tailOf(Buffer.concat(stderrChunks)); + rejectAll( + new Error(`${stage} exited with code ${code}${stderr ? `:\n${stderr}` : ""}`), + ); + }); + proc.on("error", rejectAll); + }); +} + export interface FfmpegStreamingOptions { /** * Fired when stdin emits an 'error' event that isn't a benign EPIPE after From 4fffce65d5bd5bf5cf7c0b868bc990a6adcbe85c Mon Sep 17 00:00:00 2001 From: Ariel Conti Date: Sat, 18 Jul 2026 15:58:44 +0100 Subject: [PATCH 40/48] refactor(core): route compositeFrames through spawnFfmpegStreaming helper --- packages/@webreel/core/src/compositor.ts | 168 ++++++++--------------- 1 file changed, 58 insertions(+), 110 deletions(-) diff --git a/packages/@webreel/core/src/compositor.ts b/packages/@webreel/core/src/compositor.ts index 730d36c..e5e8132 100644 --- a/packages/@webreel/core/src/compositor.ts +++ b/packages/@webreel/core/src/compositor.ts @@ -1,4 +1,3 @@ -import { spawn } from "node:child_process"; import { mkdirSync, rmSync } from "node:fs"; import { homedir } from "node:os"; import { resolve, extname } from "node:path"; @@ -6,7 +5,7 @@ import sharp from "sharp"; import type { TimelineData } from "./timeline.js"; import { ensureFfmpeg } from "./ffmpeg.js"; import { buildGifFilter, finalizeMp4, finalizeWebm, type SfxConfig } from "./media.js"; -import { runFfmpegAsync } from "./ffmpeg-run.js"; +import { runFfmpegAsync, spawnFfmpegStreaming } from "./ffmpeg-run.js"; interface OverlayContext { cursorPng: Buffer; @@ -239,7 +238,55 @@ async function compositeFrames( ): Promise { const { width, height, fps } = timeline; - const ffmpeg = spawn( + const PREFETCH_QUEUE_SIZE = 4; + + const state = { + abortError: null as Error | null, + producerDone: false, + // Resolves when the queue has items OR the producer is done. + queueResolve: null as (() => void) | null, + // Resolves when the consumer dequeues an item (backpressure signal). + spaceResolve: null as (() => void) | null, + // Resolves the consumer's in-flight drain() wait. + drainResolve: null as (() => void) | null, + }; + + const notifyConsumer = () => { + if (state.queueResolve) { + const r = state.queueResolve; + state.queueResolve = null; + r(); + } + }; + + const notifyProducer = () => { + if (state.spaceResolve) { + const r = state.spaceResolve; + state.spaceResolve = null; + r(); + } + }; + + const notifyDrain = () => { + if (state.drainResolve) { + const r = state.drainResolve; + state.drainResolve = null; + r(); + } + }; + + // Wake every waiter so the producer/consumer loop unwinds instead of + // hanging on a drain event a dead stream will never emit. Fired for a + // genuine (non-post-end) stdin error, or a process close that races ahead + // of stdin.end() - see ffmpeg-run.ts's onPipeError/onPrematureClose docs. + const onAbort = (err: Error) => { + if (!state.abortError) state.abortError = err; + notifyConsumer(); + notifyProducer(); + notifyDrain(); + }; + + const handle = spawnFfmpegStreaming( ffmpegPath, [ "-y", @@ -257,8 +304,10 @@ async function compositeFrames( config.filterComplex, ...config.outputArgs, ], - { stdio: ["pipe", "pipe", "pipe"] }, + `Compositor ffmpeg (layer=${layer})`, + { onPipeError: onAbort, onPrematureClose: onAbort }, ); + const stdin = handle.stdin; const cursorMeta = await sharp(cursorPng).metadata(); if (!cursorMeta.width || !cursorMeta.height) { @@ -299,103 +348,8 @@ async function compositeFrames( const overlayCache = new Map(); const hudCache = new Map(); - const stdin = ffmpeg.stdin; - if (!stdin) throw new Error("ffmpeg process has no stdin pipe"); - - const stderrChunks: Buffer[] = []; - ffmpeg.stderr?.on("data", (chunk: Buffer) => stderrChunks.push(chunk)); - - // Register close/error listeners immediately to avoid missing events. - const KILL_TIMEOUT = 5_000; - const ffmpegDone = new Promise((resolveAll, rejectAll) => { - ffmpeg.on("close", (code) => { - if (code === 0) { - resolveAll(); - return; - } - const stderr = Buffer.concat(stderrChunks).toString().slice(-2000); - const err = new Error( - `Compositor ffmpeg (layer=${layer}) exited with code ${code}${stderr ? `:\n${stderr}` : ""}`, - ); - rejectAll(err); - - // If ffmpeg dies before we've finished feeding it frames (stdin.end() - // not yet called), don't rely solely on a stdin 'error' event to - // unblock the producer/consumer loop: once Node auto-destroys an - // already-closed process's stdin, further stdin.write() calls can - // return false with no further 'error' or 'drain' event ever firing, - // which would otherwise hang the loop forever. Treat a premature - // close as an abort signal directly. - if (!state.stdinEnded && !state.abortError) { - state.abortError = err; - notifyConsumer(); - notifyProducer(); - notifyDrain(); - } - }); - ffmpeg.on("error", rejectAll); - }); - // ffmpegDone can reject as soon as the process exits, which may be well - // before the abort branch below (or the final `await ffmpegDone`) has a - // chance to observe it. Attach a no-op handler now so Node never sees an - // unhandled rejection in that window; the promise is still awaited for - // its real outcome further down. - ffmpegDone.catch(() => {}); - - const PREFETCH_QUEUE_SIZE = 4; - - const state = { - abortError: null as Error | null, - producerDone: false, - // Set just before stdin.end() is called; an EPIPE after that point means - // ffmpeg simply finished reading and is expected, not an abort. - stdinEnded: false, - // Resolves when the queue has items OR the producer is done. - queueResolve: null as (() => void) | null, - // Resolves when the consumer dequeues an item (backpressure signal). - spaceResolve: null as (() => void) | null, - // Resolves the consumer's in-flight drain() wait. - drainResolve: null as (() => void) | null, - }; - const queue: Buffer[] = []; - const notifyConsumer = () => { - if (state.queueResolve) { - const r = state.queueResolve; - state.queueResolve = null; - r(); - } - }; - - const notifyProducer = () => { - if (state.spaceResolve) { - const r = state.spaceResolve; - state.spaceResolve = null; - r(); - } - }; - - const notifyDrain = () => { - if (state.drainResolve) { - const r = state.drainResolve; - state.drainResolve = null; - r(); - } - }; - - // EPIPE is expected once ffmpeg has finished reading and we've called - // stdin.end(). Before that, it means ffmpeg died mid-stream: treat it as - // an abort and wake every waiter so the producer/consumer loop unwinds - // instead of hanging on a drain event a dead stream will never emit. - stdin.on("error", (err: NodeJS.ErrnoException) => { - if (err.code === "EPIPE" && state.stdinEnded) return; - if (!state.abortError) state.abortError = err; - notifyConsumer(); - notifyProducer(); - notifyDrain(); - }); - const enqueue = (buf: Buffer) => { queue.push(buf); notifyConsumer(); @@ -435,7 +389,6 @@ async function compositeFrames( if (state.abortError) break; } } - state.stdinEnded = true; stdin.end(); }; @@ -467,19 +420,14 @@ async function compositeFrames( await consumerPromise; if (state.abortError) { - ffmpeg.kill("SIGTERM"); - const killTimer = setTimeout(() => { - if (!ffmpeg.killed) ffmpeg.kill("SIGKILL"); - }, KILL_TIMEOUT); - killTimer.unref(); - ffmpeg.once("close", () => clearTimeout(killTimer)); - // ffmpegDone already has a no-op catch attached above, so its eventual - // rejection (from the killed process exiting nonzero) won't surface as - // an unhandled rejection alongside the throw below. + // handle.done already has a no-op catch attached internally, so its + // eventual rejection (from the killed process exiting nonzero) won't + // surface as an unhandled rejection alongside the throw below. + handle.kill(); throw state.abortError; } - await ffmpegDone; + await handle.done; } async function applyZoomPass( From c403934f60e8b724ca728611f51f1da8eb248341 Mon Sep 17 00:00:00 2001 From: Ariel Conti Date: Sat, 18 Jul 2026 16:00:47 +0100 Subject: [PATCH 41/48] refactor(core): route recorder ffmpeg spawn through spawnFfmpegStreaming helper --- packages/@webreel/core/src/recorder.ts | 46 ++++++++++++++------------ 1 file changed, 24 insertions(+), 22 deletions(-) diff --git a/packages/@webreel/core/src/recorder.ts b/packages/@webreel/core/src/recorder.ts index 34c89a8..094cb5e 100644 --- a/packages/@webreel/core/src/recorder.ts +++ b/packages/@webreel/core/src/recorder.ts @@ -1,4 +1,4 @@ -import { spawn, type ChildProcess } from "node:child_process"; +import type { ChildProcess } from "node:child_process"; import { mkdirSync, rmSync, writeFileSync } from "node:fs"; import { homedir } from "node:os"; import { resolve, extname } from "node:path"; @@ -9,6 +9,7 @@ import { ensureFfmpeg } from "./ffmpeg.js"; import { hasExited } from "./process.js"; import { finalizeMp4, finalizeWebm, finalizeGif, type SfxConfig } from "./media.js"; import type { InteractionTimeline, TimelineData } from "./timeline.js"; +import { spawnFfmpegStreaming } from "./ffmpeg-run.js"; export class Recorder { private outputPath = ""; @@ -93,7 +94,15 @@ export class Recorder { mkdirSync(workDir, { recursive: true }); this.tempVideo = resolve(workDir, `_rec_${Date.now()}.mp4`); - this.ffmpegProcess = spawn( + const resolveDrain = () => { + const resolve = this.drainResolve; + if (resolve) { + this.drainResolve = null; + resolve(); + } + }; + + const handle = spawnFfmpegStreaming( this.ffmpegPath, [ "-y", @@ -125,27 +134,20 @@ export class Recorder { String(this.fps), this.tempVideo, ], - { stdio: ["pipe", "pipe", "pipe"] }, + "Recorder ffmpeg", + { + // A dead pipe (e.g. ffmpeg crashed) surfaces as an 'error' event on + // the stream; without a listener Node treats it as an uncaught + // exception. Mark the pipe dead and unblock any writeFrame() waiting + // on drain. + onPipeError: (err) => { + if (!this.pipeError) this.pipeError = err; + resolveDrain(); + }, + }, ); - - const resolveDrain = () => { - const resolve = this.drainResolve; - if (resolve) { - this.drainResolve = null; - resolve(); - } - }; - - const stdin = this.ffmpegProcess.stdin; - if (!stdin) throw new Error("ffmpeg process has no stdin pipe"); - stdin.on("drain", resolveDrain); - // A dead pipe (e.g. ffmpeg crashed) surfaces as an 'error' event on the - // stream; without a listener Node treats it as an uncaught exception. - // Mark the pipe dead and unblock any writeFrame() waiting on drain. - stdin.on("error", (err: Error) => { - if (!this.pipeError) this.pipeError = err; - resolveDrain(); - }); + this.ffmpegProcess = handle.proc; + handle.stdin.on("drain", resolveDrain); this.ffmpegProcess.on("close", resolveDrain); this.stoppedPromise = new Promise((resolve) => { From 36e13a71e633dac2db95fa9ff4c8c2ce8ed2e4bc Mon Sep 17 00:00:00 2001 From: Ariel Conti Date: Sat, 18 Jul 2026 16:02:54 +0100 Subject: [PATCH 42/48] refactor(config): introduce declarative field registry in lib/config/schema-def.ts Known-key allowlists and 'did you mean' suggestion candidates in validate.ts now derive from a single field registry (top-level keys, video keys, step types, autozoom/sfx enums) instead of separately hand-maintained Sets. This registry doubles as the data source for the JSON Schema generator (next step). Hand-written semantic checks (ranges, text-or-selector requirements) remain as code in validate.ts, keyed off registry field names. No behavior change: all 81 config tests pass unchanged, and a key-set comparison against the prior hardcoded allowlists confirms exact parity (including iteration order for the 'Valid actions: ...' error message). --- packages/webreel/src/lib/config/schema-def.ts | 710 ++++++++++++++++++ packages/webreel/src/lib/config/validate.ts | 166 +--- 2 files changed, 720 insertions(+), 156 deletions(-) create mode 100644 packages/webreel/src/lib/config/schema-def.ts diff --git a/packages/webreel/src/lib/config/schema-def.ts b/packages/webreel/src/lib/config/schema-def.ts new file mode 100644 index 0000000..c1d96c3 --- /dev/null +++ b/packages/webreel/src/lib/config/schema-def.ts @@ -0,0 +1,710 @@ +/** + * Single source of truth for the webreel config surface. + * + * This registry describes every top-level key, video key, step type, and + * shared sub-object (viewport, theme, sfx, autoZoom, ...) as a small + * JSON-Schema-shaped data structure. Two consumers read it: + * + * - `validate.ts` derives its known-key allowlists (and therefore its + * "unknown property (did you mean ...)" suggestions) from + * `Object.keys(...)` over the relevant registry entries. + * - `scripts/generate-schema.ts` walks the same registry to emit + * `apps/docs/public/schema/v1.json`. + * + * Hand-written semantic rules (numeric ranges, "requires text or selector", + * cross-field defaults, ...) are NOT expressed here; they remain as code in + * `validate.ts`, keyed off the field names declared in this file. This + * registry only needs to describe SHAPE (which keys exist, their JSON types, + * and their documentation), not full validation semantics -- encoding the + * semantic rules here would turn this file into a mini validation language, + * which is explicitly out of scope for this refactor. + */ + +import { VIEWPORT_PRESETS } from "../types.js"; + +export interface FieldSchema { + type?: "string" | "integer" | "number" | "boolean" | "array" | "object"; + description?: string; + default?: unknown; + minimum?: number; + maximum?: number; + minLength?: number; + minProperties?: number; + enum?: (string | number)[]; + const?: string; + items?: FieldSchema; + /** Name of a `$defs` entry to reference (becomes `{ "$ref": "#/$defs/" }`). */ + ref?: string; + oneOf?: FieldSchema[]; + properties?: Record; + required?: string[]; + additionalProperties?: boolean; + /** Name of a `$defs` entry to use as `additionalProperties: { "$ref": ... }`. */ + additionalPropertiesRef?: string; +} + +export const SFX_VARIANTS = [1, 2, 3, 4] as const; + +/** Shared `oneOf` shape used by every sfx value field (click, key). */ +function sfxValueField(description: string): FieldSchema { + return { + oneOf: [ + { type: "integer", enum: [...SFX_VARIANTS] }, + { type: "string", minLength: 1 }, + ], + default: 1, + description, + }; +} + +export const ELEMENT_TARGET_FIELDS: Record = { + text: { type: "string", description: "Visible text to match." }, + selector: { type: "string", description: "CSS selector to match." }, + within: { type: "string", description: "CSS selector to scope the search." }, +}; + +/** `elementTarget` $defs entry: requires text or selector. */ +export const ELEMENT_TARGET_DEF: FieldSchema = { + type: "object", + properties: ELEMENT_TARGET_FIELDS, + oneOf: [{ required: ["text"] }, { required: ["selector"] }], + additionalProperties: false, +}; + +export const VIEWPORT_DEF: FieldSchema = { + oneOf: [ + { + type: "string", + enum: Object.keys(VIEWPORT_PRESETS), + description: "Named device preset.", + }, + { + type: "object", + required: ["width", "height"], + properties: { + width: { type: "integer", minimum: 1, description: "Viewport width in pixels." }, + height: { + type: "integer", + minimum: 1, + description: "Viewport height in pixels.", + }, + }, + additionalProperties: false, + }, + ], +}; + +export const CURSOR_CONFIG_FIELDS: Record = { + image: { type: "string", description: "Path to a custom cursor SVG file." }, + size: { + type: "number", + minimum: 1, + default: 24, + description: "Size of the cursor overlay in pixels.", + }, + hotspot: { + type: "string", + enum: ["top-left", "center"], + default: "top-left", + description: + "Where the click point lands relative to the cursor image. Use 'center' for circle/dot cursors.", + }, +}; + +export const CURSOR_CONFIG_DEF: FieldSchema = { + type: "object", + properties: CURSOR_CONFIG_FIELDS, + additionalProperties: false, +}; + +export const HUD_CONFIG_FIELDS: Record = { + background: { + type: "string", + description: "CSS background value for the keystroke HUD.", + }, + color: { type: "string", description: "CSS text color for the keystroke HUD." }, + fontSize: { + type: "number", + minimum: 1, + default: 56, + description: "Font size in pixels for the keystroke HUD.", + }, + fontFamily: { type: "string", description: "CSS font-family for the keystroke HUD." }, + borderRadius: { + type: "number", + minimum: 0, + default: 18, + description: "Border radius in pixels for the keystroke HUD.", + }, + position: { + type: "string", + enum: ["top", "bottom"], + default: "bottom", + description: "Position of the keystroke HUD.", + }, +}; + +export const THEME_FIELDS: Record = { + cursor: { ref: "cursorConfig", description: "Cursor overlay configuration." }, + hud: { + type: "object", + properties: HUD_CONFIG_FIELDS, + additionalProperties: false, + }, +}; + +export const THEME_DEF: FieldSchema = { + type: "object", + properties: THEME_FIELDS, + additionalProperties: false, +}; + +export const SFX_FIELDS: Record = { + click: sfxValueField( + "Mouse click sound: built-in variant (1-4) or path to a custom audio file.", + ), + key: sfxValueField( + "Keyboard press sound: built-in variant (1-4) or path to a custom audio file.", + ), +}; + +export const SFX_DEF: FieldSchema = { + type: "object", + properties: SFX_FIELDS, + additionalProperties: false, +}; + +export const THUMBNAIL_FIELDS: Record = { + time: { + type: "number", + minimum: 0, + description: "Time in seconds to capture the thumbnail. Defaults to 0 (first frame).", + }, + enabled: { + type: "boolean", + description: "Set to false to skip thumbnail generation.", + }, +}; + +export const THUMBNAIL_DEF: FieldSchema = { + type: "object", + properties: THUMBNAIL_FIELDS, + additionalProperties: false, +}; + +export const AUTOZOOM_NONNEGATIVE_FIELDS: Record = { + approachS: { + type: "number", + minimum: 0, + default: 0.5, + description: "Seconds spent zooming in from full frame to the target.", + }, + settleBeforeS: { + type: "number", + minimum: 0, + default: 0.15, + description: "Seconds the camera sits on the target before the action fires.", + }, + holdAfterS: { + type: "number", + minimum: 0, + default: 0.3, + description: "Seconds the camera holds after the last action in a session.", + }, + releaseS: { + type: "number", + minimum: 0, + default: 0.5, + description: "Seconds spent zooming out from the target back to full frame.", + }, + paddingRatio: { + type: "number", + minimum: 0, + default: 0.3, + description: "Fraction of the target bounding box added as padding around it.", + }, + sessionGapS: { + type: "number", + minimum: 0, + default: 4, + description: + "Actions within this many seconds share one zoom session; the camera pans between them instead of zooming out.", + }, + minPanS: { + type: "number", + minimum: 0, + default: 0.8, + description: + "Skip panning to an intermediate target when the pan would be shorter than this many seconds.", + }, +}; + +export const AUTOZOOM_RATIO_FIELDS: Record = { + minZoomRatio: { + type: "number", + minimum: 0, + maximum: 1, + default: 0.6, + description: "The crop is never smaller than this fraction of the viewport.", + }, + skipZoomRatio: { + type: "number", + minimum: 0, + maximum: 1, + default: 0.75, + description: + "Skip zooming when the computed crop would be this fraction of the viewport or larger.", + }, +}; + +export const AUTOZOOM_OBJECT_FIELDS: Record = { + enabled: { + type: "boolean", + default: true, + description: "Turn autozoom on or off without removing the config object.", + }, + ...AUTOZOOM_NONNEGATIVE_FIELDS, + ...AUTOZOOM_RATIO_FIELDS, +}; + +export const AUTOZOOM_DEF: FieldSchema = { + oneOf: [ + { type: "boolean", description: "Enable autozoom with default settings." }, + { + type: "object", + properties: AUTOZOOM_OBJECT_FIELDS, + additionalProperties: false, + }, + ], +}; + +export const DELAY_DEF: FieldSchema = { + type: "number", + minimum: 0, + description: + "Delay in milliseconds to wait after this step executes. Overrides defaultDelay for this step. For longer explicit waits, use a 'pause' step instead.", +}; + +export const LABEL_DEF: FieldSchema = { + type: "string", + description: "Display label for the HUD overlay. Shown on-screen during recording.", +}; + +export const DESCRIPTION_DEF: FieldSchema = { + type: "string", + description: "Optional description for documentation. Shown in --verbose output.", +}; + +/** Fields shared by (almost) every step type: label, delay, description. */ +const COMMON_STEP_FIELDS: Record = { + label: { ref: "label" }, + delay: { ref: "delay" }, + description: { ref: "description" }, +}; + +interface StepDef { + /** Name of the `$defs` entry, e.g. "stepPause". */ + defName: string; + required: string[]; + properties: Record; + /** Object-level `oneOf` of `{ required: [...] }` branches, e.g. text-or-selector. */ + requiredOneOf?: string[][]; +} + +/** + * Step type registry, keyed by action name. Insertion order here is the + * canonical action order used both for validator messages (Valid actions: ...) + * and for the schema's `step` union. + */ +export const STEP_DEFS: Record = { + pause: { + defName: "stepPause", + required: ["action", "ms"], + properties: { + action: { type: "string", const: "pause" }, + ms: { + type: "number", + minimum: 0, + description: + "Duration in milliseconds. For post-step delays, use the 'delay' field on any other step instead.", + }, + // Note: pause has no "delay" field (it IS a delay); it only shares + // label/description with the other step types. + label: { ref: "label" }, + description: { ref: "description" }, + }, + }, + click: { + defName: "stepClick", + required: ["action"], + requiredOneOf: [["text"], ["selector"]], + properties: { + action: { type: "string", const: "click" }, + text: { type: "string", description: "Visible text to find and click." }, + selector: { type: "string", description: "CSS selector to find and click." }, + within: { type: "string", description: "CSS selector to scope the search." }, + modifiers: { + type: "array", + items: { type: "string" }, + description: 'Modifier keys to hold during click (e.g. ["cmd"]).', + }, + ...COMMON_STEP_FIELDS, + }, + }, + key: { + defName: "stepKey", + required: ["action", "key"], + properties: { + action: { type: "string", const: "key" }, + key: { + type: "string", + minLength: 1, + description: + 'Key or key combo. Combine with \'+\' (e.g. "mod+z", "cmd+shift+a"). Use "mod" for the platform modifier (cmd on macOS, ctrl elsewhere). Common keys: a-z, 0-9, Enter, Tab, Escape, Backspace, Delete, ArrowUp, ArrowDown, ArrowLeft, ArrowRight, Home, End, PageUp, PageDown, Space, F1-F12. Modifiers: mod, cmd, ctrl, shift, alt, meta.', + }, + target: { + oneOf: [ + { type: "string", description: "CSS selector of element to focus first." }, + { ref: "elementTarget", description: "Element target to focus first." }, + ], + }, + label: { + type: "string", + description: "Display label for the keystroke HUD.", + }, + delay: { ref: "delay" }, + description: { ref: "description" }, + }, + }, + drag: { + defName: "stepDrag", + required: ["action", "from", "to"], + properties: { + action: { type: "string", const: "drag" }, + from: { ref: "elementTarget", description: "Element to drag from." }, + to: { ref: "elementTarget", description: "Element to drag to." }, + ...COMMON_STEP_FIELDS, + }, + }, + moveTo: { + defName: "stepMoveTo", + required: ["action"], + requiredOneOf: [["text"], ["selector"]], + properties: { + action: { type: "string", const: "moveTo" }, + text: { type: "string", description: "Visible text of the target element." }, + selector: { type: "string", description: "CSS selector of the target element." }, + within: { type: "string", description: "CSS selector to scope the search." }, + ...COMMON_STEP_FIELDS, + }, + }, + type: { + defName: "stepType", + required: ["action", "text"], + properties: { + action: { type: "string", const: "type" }, + text: { + type: "string", + minLength: 1, + description: "Text to type character by character.", + }, + selector: { + type: "string", + description: "CSS selector of the element to click before typing.", + }, + within: { type: "string", description: "CSS selector to scope the search." }, + charDelay: { + type: "number", + minimum: 0, + description: "Delay in milliseconds between keystrokes.", + }, + method: { + type: "string", + enum: ["insertText", "dispatchKeyEvent"], + description: + "How characters are injected. 'insertText' goes through the browser text input pipeline and updates framework-controlled inputs (React and similar) but fires no keydown/keyup events. 'dispatchKeyEvent' fires raw key events. Defaults to 'insertText' when 'selector' is set, otherwise 'dispatchKeyEvent'.", + }, + ...COMMON_STEP_FIELDS, + }, + }, + scroll: { + defName: "stepScroll", + required: ["action"], + properties: { + action: { type: "string", const: "scroll" }, + x: { type: "number", description: "Horizontal scroll distance in pixels." }, + y: { type: "number", description: "Vertical scroll distance in pixels." }, + text: { type: "string", description: "Visible text of the element to scroll." }, + selector: { type: "string", description: "CSS selector of element to scroll." }, + within: { type: "string", description: "CSS selector to scope the search." }, + ...COMMON_STEP_FIELDS, + }, + }, + wait: { + defName: "stepWait", + required: ["action"], + requiredOneOf: [["selector"], ["text"]], + properties: { + action: { type: "string", const: "wait" }, + selector: { type: "string", description: "CSS selector to wait for." }, + text: { type: "string", description: "Visible text to wait for." }, + within: { type: "string", description: "CSS selector to scope the search." }, + timeout: { + type: "number", + minimum: 0, + default: 30000, + description: "Timeout in milliseconds.", + }, + ...COMMON_STEP_FIELDS, + }, + }, + screenshot: { + defName: "stepScreenshot", + required: ["action", "output"], + properties: { + action: { type: "string", const: "screenshot" }, + output: { + type: "string", + minLength: 1, + description: "Output file path for the screenshot.", + }, + ...COMMON_STEP_FIELDS, + }, + }, + navigate: { + defName: "stepNavigate", + required: ["action", "url"], + properties: { + action: { type: "string", const: "navigate" }, + url: { + type: "string", + minLength: 1, + description: "URL to navigate to. Resolved relative to baseUrl if set.", + }, + ...COMMON_STEP_FIELDS, + }, + }, + navigateHref: { + defName: "stepNavigateHref", + required: ["action", "selector"], + properties: { + action: { type: "string", const: "navigateHref" }, + selector: { + type: "string", + minLength: 1, + description: "CSS selector for an element with an href attribute.", + }, + ...COMMON_STEP_FIELDS, + }, + }, + hover: { + defName: "stepHover", + required: ["action"], + requiredOneOf: [["text"], ["selector"]], + properties: { + action: { type: "string", const: "hover" }, + text: { type: "string", description: "Visible text of the element to hover." }, + selector: { type: "string", description: "CSS selector of the element to hover." }, + within: { type: "string", description: "CSS selector to scope the search." }, + ...COMMON_STEP_FIELDS, + }, + }, + select: { + defName: "stepSelect", + required: ["action", "value"], + requiredOneOf: [["text"], ["selector"]], + properties: { + action: { type: "string", const: "select" }, + text: { type: "string", description: "Visible text of the