diff --git a/.github/workflows/test-recording.yml b/.github/workflows/test-recording.yml new file mode 100644 index 0000000..7f655f3 --- /dev/null +++ b/.github/workflows/test-recording.yml @@ -0,0 +1,61 @@ +name: Test Recording (Linux) + +on: + workflow_dispatch: + push: + branches: [ctate/keyboard-overlay] + +jobs: + test-recording: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: pnpm/action-setup@v4 + + - uses: actions/setup-node@v4 + with: + node-version: 22 + cache: pnpm + + - name: Install Chrome dependencies + run: | + sudo apt-get update + sudo apt-get install -y \ + fonts-liberation \ + libasound2t64 \ + libatk-bridge2.0-0 \ + libatk1.0-0 \ + libcups2 \ + libdbus-1-3 \ + libdrm2 \ + libgbm1 \ + libgtk-3-0t64 \ + libnspr4 \ + libnss3 \ + libx11-xcb1 \ + libxcomposite1 \ + libxdamage1 \ + libxfixes3 \ + libxrandr2 \ + xdg-utils + + - run: pnpm install --frozen-lockfile + + - name: Build + run: | + pnpm --filter @webreel/core build + pnpm --filter webreel build + + - name: Record keyboard-shortcuts example + working-directory: examples/keyboard-shortcuts + run: node ../../packages/webreel/dist/index.js record keyboard-shortcuts --verbose --frames + + - name: Upload recording artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: recording-output + path: | + examples/keyboard-shortcuts/.webreel/ + examples/keyboard-shortcuts/videos/ diff --git a/apps/docs/public/schema/v1.json b/apps/docs/public/schema/v1.json index 7b9a83e..c569f10 100644 --- a/apps/docs/public/schema/v1.json +++ b/apps/docs/public/schema/v1.json @@ -143,7 +143,7 @@ "type": "number", "minimum": 1, "description": "Font size in pixels for the keystroke HUD.", - "default": 56 + "default": 48 }, "fontFamily": { "type": "string", @@ -153,13 +153,46 @@ "type": "number", "minimum": 0, "description": "Border radius in pixels for the keystroke HUD.", - "default": 18 + "default": 16 }, "position": { "type": "string", "enum": ["top", "bottom"], "description": "Position of the keystroke HUD.", "default": "bottom" + }, + "blur": { + "type": "number", + "minimum": 0, + "description": "Backdrop blur radius in pixels for the HUD container (live preview only).", + "default": 20 + }, + "border": { + "type": "string", + "description": "CSS border color for the HUD container." + }, + "shadow": { + "type": "string", + "description": "CSS box-shadow for the HUD container." + }, + "keyBackground": { + "type": "string", + "description": "CSS background for individual key caps inside the HUD." + }, + "keyBorder": { + "type": "string", + "description": "CSS border color for individual key caps inside the HUD." + }, + "keyBorderRadius": { + "type": "number", + "minimum": 0, + "description": "Border radius in pixels for individual key caps.", + "default": 10 + }, + "keyPadding": { + "type": "string", + "description": "CSS padding for individual key caps (e.g. \"8px 16px\").", + "default": "8px 16px" } }, "additionalProperties": false diff --git a/apps/docs/src/app/configuration/page.mdx b/apps/docs/src/app/configuration/page.mdx index 4f3ccce..950c004 100644 --- a/apps/docs/src/app/configuration/page.mdx +++ b/apps/docs/src/app/configuration/page.mdx @@ -480,12 +480,15 @@ Customize the cursor and keystroke overlay appearance. Can be set at the top lev "hotspot": "top-left" }, "hud": { - "background": "rgba(0,0,0,0.7)", "color": "white", "fontSize": 48, "fontFamily": "monospace", - "borderRadius": 12, - "position": "bottom" + "position": "bottom", + "blur": 20, + "keyBackground": "rgba(22, 22, 22, 0.65)", + "keyBorder": "rgba(255, 255, 255, 0.12)", + "keyBorderRadius": 12, + "keyPadding": "10px 20px" } } } @@ -590,6 +593,61 @@ Customize the cursor and keystroke overlay appearance. Can be set at the top lev Where the HUD appears: "top" or "bottom" + + + blur + + + 20 + + Backdrop blur radius in pixels for the HUD container (live preview only) + + + + border + + - + CSS border color for the HUD container + + + + shadow + + - + CSS box-shadow for the HUD container + + + + keyBackground + + - + CSS background for individual key caps inside the HUD + + + + keyBorder + + - + CSS border color for individual key caps + + + + keyBorderRadius + + + 10 + + Border radius in pixels for individual key caps + + + + keyPadding + + + "8px 16px" + + CSS padding for individual key caps + diff --git a/examples/custom-theme/videos/custom-theme.mp4 b/examples/custom-theme/videos/custom-theme.mp4 index 39dca80..9b2a669 100644 --- a/examples/custom-theme/videos/custom-theme.mp4 +++ b/examples/custom-theme/videos/custom-theme.mp4 @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:fc33f30321d04fa64b1841120fc275ffc7e7efc433e43390f1069f251540eaae -size 602066 +oid sha256:c6505c5315f9cc7b5d1fa31f9a6b141d24a307911c7101c5b625bd1fd440b276 +size 546739 diff --git a/examples/custom-theme/videos/custom-theme.png b/examples/custom-theme/videos/custom-theme.png index 38d8753..788dfdc 100644 --- a/examples/custom-theme/videos/custom-theme.png +++ b/examples/custom-theme/videos/custom-theme.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:5892f64b08c78a69bb7ea4ba1b73fd11cbdaca5af4b887f32369026e4316dc26 -size 194065 +oid sha256:f4dbc46e7dba52f0a2e27a76fdfed5412177f6eee7400b916d283ce8d4bc687b +size 194802 diff --git a/examples/drag-and-drop/videos/drag-and-drop.mp4 b/examples/drag-and-drop/videos/drag-and-drop.mp4 index 6c9326c..0998c7f 100644 --- a/examples/drag-and-drop/videos/drag-and-drop.mp4 +++ b/examples/drag-and-drop/videos/drag-and-drop.mp4 @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:a3a13216b7cfe0bd0e2fec0b0fb9e0c00ced5adfe8884757d8543b51b82b12c9 -size 901223 +oid sha256:dbf4953e61a9d1ed914ec4c3c224318a767d51e769bc5eda4bfa28b656c88294 +size 897447 diff --git a/examples/drag-and-drop/videos/drag-and-drop.png b/examples/drag-and-drop/videos/drag-and-drop.png index af06dcb..59235fc 100644 --- a/examples/drag-and-drop/videos/drag-and-drop.png +++ b/examples/drag-and-drop/videos/drag-and-drop.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:01f14a7144c4d9b0d0498ab4e29d3256c8886612bf34b9c90589f22340d7bcf8 -size 143394 +oid sha256:cd127c05fa90d4b8586fa6845dc8e13a019329340a7648c34cd2218f7e7a7111 +size 146442 diff --git a/examples/gif-output/videos/gif-output.gif b/examples/gif-output/videos/gif-output.gif index f4fca58..5326a81 100644 --- a/examples/gif-output/videos/gif-output.gif +++ b/examples/gif-output/videos/gif-output.gif @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:7fe66fb5615e0524ca4c2d0190604c1ebe4524e37e9240694fb7d4692475ab05 -size 822847 +oid sha256:97e809d4e152f3770efda290124f6fb7e7f53c63238431f86d79207e564090e0 +size 787226 diff --git a/examples/gif-output/videos/gif-output.png b/examples/gif-output/videos/gif-output.png index e370476..8cdb899 100644 --- a/examples/gif-output/videos/gif-output.png +++ b/examples/gif-output/videos/gif-output.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:0638249fb94d74729cee3129576c8410573694e736f0741cab6f917b2911e0bd -size 86997 +oid sha256:34458e98816828eae270be6e1bdfe851ecf233e87291d3eebb78e3448174dc61 +size 88953 diff --git a/examples/hello-world/videos/hello-world.mp4 b/examples/hello-world/videos/hello-world.mp4 index 4dcb46d..b607576 100644 --- a/examples/hello-world/videos/hello-world.mp4 +++ b/examples/hello-world/videos/hello-world.mp4 @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:4f0a45623cd5dada0a29c83e91518ca6ee7f5f082dfd6460eded2762d3a14a70 -size 869374 +oid sha256:b7c8fb9d2fae3e4f06c73e985a250bf54133a10fd5c9b9eeab4319cee12a2f4a +size 666071 diff --git a/examples/hello-world/videos/hello-world.png b/examples/hello-world/videos/hello-world.png index d15ec7c..52dffa8 100644 --- a/examples/hello-world/videos/hello-world.png +++ b/examples/hello-world/videos/hello-world.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:bdbc6c7b838a44ed52e8c237b5014e5d85297a07a888439e5dd0d381cac6ea17 -size 144398 +oid sha256:01d77ff57b9ec540ce7366a5f8d676fa0bc47d2cc51363becffee67eba6c7aad +size 148745 diff --git a/examples/keyboard-shortcuts/videos/keyboard-shortcuts.mp4 b/examples/keyboard-shortcuts/videos/keyboard-shortcuts.mp4 index 5310f46..3bb4b49 100644 --- a/examples/keyboard-shortcuts/videos/keyboard-shortcuts.mp4 +++ b/examples/keyboard-shortcuts/videos/keyboard-shortcuts.mp4 @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:e77279ed23245d9c87038b56605f135fbfb4514c4bfb3a50ec39577e81b6c1c2 -size 3221641 +oid sha256:0546f68f6c85293ec4705563c11b545266bfebb9477cc89b1d86bb2ed5e3423c +size 2594724 diff --git a/examples/keyboard-shortcuts/videos/keyboard-shortcuts.png b/examples/keyboard-shortcuts/videos/keyboard-shortcuts.png index b0f7e18..8f84f5b 100644 --- a/examples/keyboard-shortcuts/videos/keyboard-shortcuts.png +++ b/examples/keyboard-shortcuts/videos/keyboard-shortcuts.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:cc3dfdba72f00bf306391254d573277e51338b15602757c6f6bc1c34835d8e69 -size 470301 +oid sha256:7d8e8c8121c057c954cc23645c33aad7dad0e3f785ff662b0d2271156bec7b1d +size 478047 diff --git a/examples/keyboard-shortcuts/webreel.config.json b/examples/keyboard-shortcuts/webreel.config.json index b88aa85..e440ae8 100644 --- a/examples/keyboard-shortcuts/webreel.config.json +++ b/examples/keyboard-shortcuts/webreel.config.json @@ -10,7 +10,7 @@ "steps": [ { "action": "pause", "ms": 500 }, { "action": "click", "selector": ".editor" }, - { "action": "key", "key": "cmd+a" }, + { "action": "key", "key": "shift+a" }, { "action": "key", "key": "cmd+b", "delay": 800 }, { "action": "click", "selector": ".editor p:first-of-type" }, { "action": "key", "key": "cmd+s", "delay": 1200 }, diff --git a/examples/mobile-viewport/videos/mobile-viewport.mp4 b/examples/mobile-viewport/videos/mobile-viewport.mp4 index 0b0005a..c0e7f50 100644 --- a/examples/mobile-viewport/videos/mobile-viewport.mp4 +++ b/examples/mobile-viewport/videos/mobile-viewport.mp4 @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:f2a5fc89878ff9d7a2906a1d68c4e109e6ac0a5821eeb7a4ecacc75d8d92fad0 -size 178891 +oid sha256:1ae888a9c32fef74f5a0199581ee7667271b374a59a313b783ef81d7d1e27a8b +size 162486 diff --git a/examples/mobile-viewport/videos/mobile-viewport.png b/examples/mobile-viewport/videos/mobile-viewport.png index 62977d2..645d42e 100644 --- a/examples/mobile-viewport/videos/mobile-viewport.png +++ b/examples/mobile-viewport/videos/mobile-viewport.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:72e58492e0c7053ce83dab3fe920ea2fa509c2b4871332c4c8131b1a47366b68 -size 80266 +oid sha256:cf84207bfdc16f952ec47190f6c43f8d751df42517fb60a1036708f0dcd9a511 +size 83128 diff --git a/examples/modifier-clicks/videos/modifier-clicks.mp4 b/examples/modifier-clicks/videos/modifier-clicks.mp4 index a755c5e..8294002 100644 --- a/examples/modifier-clicks/videos/modifier-clicks.mp4 +++ b/examples/modifier-clicks/videos/modifier-clicks.mp4 @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:6aa261938b887f116d4e2c2dfc49f29ed2d01dfad3c404b3b3495874f062772f -size 587074 +oid sha256:4af34b7bcec3f553a0e7bb4cc4a2b17d0d5448fe06fa5e74ef5b569a80f0ed74 +size 592869 diff --git a/examples/modifier-clicks/videos/modifier-clicks.png b/examples/modifier-clicks/videos/modifier-clicks.png index 309a185..c2aa2b1 100644 --- a/examples/modifier-clicks/videos/modifier-clicks.png +++ b/examples/modifier-clicks/videos/modifier-clicks.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:951d93e28a6ab536d144c8997fa3ee64c358a525e188f91db5fef152c729ea5c -size 164643 +oid sha256:48582452e4c54b42d4066a1f80c6400947f6b8da080313526347129f8f778217 +size 167239 diff --git a/examples/multi-demo/videos/homepage.mp4 b/examples/multi-demo/videos/homepage.mp4 index fb9909a..bd4c29e 100644 --- a/examples/multi-demo/videos/homepage.mp4 +++ b/examples/multi-demo/videos/homepage.mp4 @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:67c7cc5eb9a06c7247e0b330b0fedad5dbf9681f6432a0047febcf33c90b2076 -size 708155 +oid sha256:2be5832501f3cfe0ce2ee59336375f55bf30d895b40bf85c08021f3b49feb825 +size 762615 diff --git a/examples/multi-demo/videos/homepage.png b/examples/multi-demo/videos/homepage.png index 542543f..bb7d1f7 100644 --- a/examples/multi-demo/videos/homepage.png +++ b/examples/multi-demo/videos/homepage.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:c81ee4f8a095e067b9315905bf2c20021c031796149a6269407b20e8f28a0dd9 -size 162187 +oid sha256:2866e4db879d3b2f8236bbbc098511dd15bc27a469ef25136691f51d9a165361 +size 165949 diff --git a/packages/@webreel/core/src/__tests__/overlays.test.ts b/packages/@webreel/core/src/__tests__/overlays.test.ts index cc7ba0f..31cb7a3 100644 --- a/packages/@webreel/core/src/__tests__/overlays.test.ts +++ b/packages/@webreel/core/src/__tests__/overlays.test.ts @@ -46,33 +46,52 @@ describe("injectOverlays", () => { expect(calls[0].expression).toContain("margin-top:-16px"); }); - it("uses default HUD theme values when not overridden", async () => { - const { client, calls } = createMockClient(); - await injectOverlays(client); - const expr = calls[0].expression; - expect(expr).toContain(`background:${DEFAULT_HUD_THEME.background}`); - expect(expr).toContain(`border-radius:" + z(${DEFAULT_HUD_THEME.borderRadius})`); - }); - it("uses custom HUD theme values", async () => { const { client, calls } = createMockClient(); const theme: OverlayTheme = { hud: { - background: "red", color: "blue", fontSize: 32, - borderRadius: 8, position: "top", }, }; await injectOverlays(client, theme); const expr = calls[0].expression; - expect(expr).toContain("background:red"); expect(expr).toContain("color: blue"); - expect(expr).toContain('border-radius:" + z(8)'); expect(expr).toContain('"top:"'); }); + it("applies backdrop-filter blur on key caps", async () => { + const { client, calls } = createMockClient(); + await injectOverlays(client); + const expr = calls[0].expression; + expect(expr).toContain(`backdrop-filter: blur(${DEFAULT_HUD_THEME.blur}px)`); + expect(expr).toContain(`-webkit-backdrop-filter: blur(${DEFAULT_HUD_THEME.blur}px)`); + }); + + it("styles key caps with background and border", async () => { + const { client, calls } = createMockClient(); + await injectOverlays(client); + const expr = calls[0].expression; + expect(expr).toContain(`background: ${DEFAULT_HUD_THEME.keyBackground}`); + expect(expr).toContain(`border: 1px solid ${DEFAULT_HUD_THEME.keyBorder}`); + expect(expr).toContain("box-shadow:"); + }); + + it("container has no background (transparent layout wrapper)", async () => { + const { client, calls } = createMockClient(); + await injectOverlays(client); + const expr = calls[0].expression; + expect(expr).not.toMatch(/keys\.style\.cssText[^;]*background:/); + }); + + it("includes transition for smooth animation", async () => { + const { client, calls } = createMockClient(); + await injectOverlays(client); + const expr = calls[0].expression; + expect(expr).toContain("transition:opacity 0.2s ease,transform 0.2s ease"); + }); + it("uses custom cursor SVG", async () => { const { client, calls } = createMockClient(); const svg = ''; @@ -111,6 +130,13 @@ describe("showKeys", () => { expect(expr).toContain("<"); expect(expr).toContain(">"); }); + + it("sets transform to visible position on show", async () => { + const { client, calls } = createMockClient(); + await showKeys(client, ["A"]); + const expr = calls[0].expression; + expect(expr).toContain("translateX(-50%) translateY(0)"); + }); }); describe("hideKeys", () => { @@ -119,4 +145,10 @@ describe("hideKeys", () => { await hideKeys(client); expect(calls[0].expression).toContain('opacity = "0"'); }); + + it("sets transform to offset position on hide", async () => { + const { client, calls } = createMockClient(); + await hideKeys(client); + expect(calls[0].expression).toContain("translateX(-50%) translateY(8px)"); + }); }); diff --git a/packages/@webreel/core/src/actions.ts b/packages/@webreel/core/src/actions.ts index 6d7827a..0c875e5 100644 --- a/packages/@webreel/core/src/actions.ts +++ b/packages/@webreel/core/src/actions.ts @@ -330,9 +330,8 @@ export async function clickAt( for (const m of modifiers) labels.push(modLabel(m)); if (ctx.isRecording) { getTimeline(ctx).showHud(labels); - } else { - await showKeys(client, labels); } + await showKeys(client, labels); } await moveCursorTo(ctx, client, x, y); @@ -466,9 +465,8 @@ export async function clickAt( await pause(400); if (ctx.isRecording) { getTimeline(ctx).hideHud(); - } else { - await hideKeys(client); } + await hideKeys(client); } } @@ -521,6 +519,22 @@ export function resolveCommands( return SHORTCUT_COMMANDS[parts.join("+")]; } +const KEY_DISPLAY_NAMES: Record = { + Escape: "Esc", + Delete: "Del", + Backspace: "Bksp", + Enter: "Enter", + Tab: "Tab", + ArrowUp: "\u2191", + ArrowDown: "\u2193", + ArrowLeft: "\u2190", + ArrowRight: "\u2192", +}; + +function keyDisplayName(key: string): string { + return KEY_DISPLAY_NAMES[key] ?? key; +} + export async function pressKey( ctx: RecordingContext, client: CDPClient, @@ -548,13 +562,12 @@ export async function pressKey( const displayParts: string[] = []; for (const m of modifiers) displayParts.push(modLabel(m)); - displayParts.push(label ?? mainKey); + displayParts.push(label ?? keyDisplayName(mainKey)); if (ctx.isRecording) { getTimeline(ctx).showHud(displayParts); - } else { - await showKeys(client, displayParts); } + await showKeys(client, displayParts); ctx.markEvent("key"); const flag = modifiersToFlag(modifiers); @@ -582,9 +595,8 @@ export async function pressKey( await pause(800); if (ctx.isRecording) { getTimeline(ctx).hideHud(); - } else { - await hideKeys(client); } + await hideKeys(client); } export const CHAR_CODES: Record = { @@ -766,17 +778,12 @@ export async function dragFromTo( ghost.style.left = "${wp.x - fromBox.width / 2}px"; ghost.style.top = "${wp.y - fromBox.height / 2}px"; } - ${ - isRecording - ? "" - : ` const cursor = document.getElementById("__demo-cursor"); if (cursor) { cursor.getAnimations().forEach(a => a.cancel()); cursor.style.transform = "translate(${wp.x}px,${wp.y}px)"; cursor.dataset.cx = "${wp.x}"; cursor.dataset.cy = "${wp.y}"; - }` } })()`, }); diff --git a/packages/@webreel/core/src/cdp.ts b/packages/@webreel/core/src/cdp.ts index 71539ab..3092784 100644 --- a/packages/@webreel/core/src/cdp.ts +++ b/packages/@webreel/core/src/cdp.ts @@ -4,3 +4,34 @@ import type { CDPClient } from "./types.js"; export async function connectCDP(port: number): Promise { return (await CDP({ port })) as unknown as CDPClient; } + +export interface CDPConnectionResult { + client: CDPClient; + hasBeginFrameControl: boolean; +} + +export async function connectCDPForRecording(port: number): Promise { + try { + const version = await ( + CDP as unknown as { + Version: (opts: { port: number }) => Promise<{ webSocketDebuggerUrl: string }>; + } + ).Version({ port }); + const browserWsUrl = version.webSocketDebuggerUrl; + + const browser = (await CDP({ target: browserWsUrl })) as unknown as CDPClient; + + const result = (await browser.send("Target.createTarget", { + url: "about:blank", + enableBeginFrameControl: true, + })) as { targetId: string }; + + await browser.close(); + + const client = (await CDP({ port, target: result.targetId })) as unknown as CDPClient; + return { client, hasBeginFrameControl: true }; + } catch { + const client = (await CDP({ port })) as unknown as CDPClient; + return { client, hasBeginFrameControl: false }; + } +} diff --git a/packages/@webreel/core/src/compositor.ts b/packages/@webreel/core/src/compositor.ts index 5645266..a5df2cb 100644 --- a/packages/@webreel/core/src/compositor.ts +++ b/packages/@webreel/core/src/compositor.ts @@ -4,6 +4,7 @@ import { homedir } from "node:os"; import { resolve, extname } from "node:path"; import sharp from "sharp"; import type { TimelineData } from "./timeline.js"; +import { MODIFIER_ICONS } from "./types.js"; import { ensureFfmpeg } from "./ffmpeg.js"; import { finalizeMp4, finalizeWebm, finalizeGif, type SfxConfig } from "./media.js"; @@ -301,6 +302,11 @@ async function renderOverlayFrame( return result; } +function parseKeyPadding(padding: string, zoom: number): { v: number; h: number } { + const parts = padding.split(/\s+/).map((p) => Math.round(parseFloat(p) * zoom)); + return { v: parts[0], h: parts[1] ?? parts[0] }; +} + async function renderHudOverlay( labels: string[], viewportWidth: number, @@ -313,45 +319,76 @@ async function renderHudOverlay( const cached = hudCache.get(cacheKey); if (cached) return cached; + const escAttr = (s: string) => + s + .replace(/&/g, "&") + .replace(/"/g, """) + .replace(//g, ">"); + const fontSize = Math.round(hudConfig.fontSize * zoom); - const padding = Math.round(16 * zoom); - const gap = Math.round(14 * zoom); - const borderRadius = Math.round(hudConfig.borderRadius * zoom); - const hPad = Math.round(36 * zoom); + const gap = Math.round(10 * zoom); + const keyBorderRadius = Math.round(hudConfig.keyBorderRadius * zoom); + const keyPad = parseKeyPadding(hudConfig.keyPadding, zoom); + + const shadowPad = Math.round(20 * zoom); + const shadowOffset = Math.round(4 * zoom); + const shadowBlur = Math.round(8 * zoom); const charWidth = fontSize * 0.6; - const totalTextWidth = labels.reduce((sum, l) => sum + l.length * charWidth, 0); - const hudWidth = Math.round( - totalTextWidth + padding * 2 + gap * (labels.length - 1) + hPad * 2, - ); - const hudHeight = Math.round(fontSize * 1.6 + padding * 2); + const iconSize = Math.round(fontSize * 0.75); + const capHeight = fontSize + keyPad.v * 2; + const keyCaps = labels.map((l) => { + const isIcon = l in MODIFIER_ICONS; + const contentWidth = isIcon ? iconSize : Math.round(l.length * charWidth); + const capWidth = contentWidth + keyPad.h * 2; + return { label: l, isIcon, contentWidth, capWidth }; + }); - const labelSpans = labels - .map((l) => { - const escaped = l + const totalKeysWidth = keyCaps.reduce((s, k) => s + k.capWidth, 0); + const innerWidth = totalKeysWidth + gap * (labels.length - 1); + const hudWidth = innerWidth + shadowPad * 2; + const hudHeight = capHeight + shadowPad * 2; + + let keyRects = ""; + let x = shadowPad; + const capY = shadowPad; + + for (const cap of keyCaps) { + keyRects += ``; + + if (cap.isIcon) { + const icon = MODIFIER_ICONS[cap.label]; + const ix = Math.round(x + (cap.capWidth - iconSize) / 2); + const iy = Math.round(capY + (capHeight - iconSize) / 2); + const pathEls = icon.paths + .map( + (p) => + ``, + ) + .join(""); + keyRects += `${pathEls}`; + } else { + const escaped = cap.label .replace(/&/g, "&") .replace(//g, ">"); - return `${escaped}`; - }) - .join(` `); - - const textY = Math.round(hudHeight / 2 + fontSize * 0.35); - const textX = Math.round(hudWidth / 2); + const textX = Math.round(x + cap.capWidth / 2); + const textY = Math.round(capY + capHeight / 2 + fontSize * 0.35); + keyRects += `${escaped}`; + } - const escAttr = (s: string) => - s - .replace(/&/g, "&") - .replace(/"/g, """) - .replace(//g, ">"); + x += cap.capWidth + gap; + } const svgOverlay = ` - - ${labelSpans} - `; + + + + + + ${keyRects} + `; const hudPng = await sharp(Buffer.from(svgOverlay)).png().toBuffer(); const left = Math.round((viewportWidth - hudWidth) / 2); diff --git a/packages/@webreel/core/src/cursor-motion.ts b/packages/@webreel/core/src/cursor-motion.ts index 609ef68..3f92a59 100644 --- a/packages/@webreel/core/src/cursor-motion.ts +++ b/packages/@webreel/core/src/cursor-motion.ts @@ -98,14 +98,7 @@ export async function animateMoveTo( if (ctx.isRecording && ctx.timeline) { ctx.timeline.setCursorPath(positions); - await new Promise((r) => setTimeout(r, NUM_STEPS * CAPTURE_CYCLE_MS)); - - await client.Input.dispatchMouseEvent({ - type: "mouseMoved", - x: toX, - y: toY, - }); return; } diff --git a/packages/@webreel/core/src/index.ts b/packages/@webreel/core/src/index.ts index ebb515f..8a8d767 100644 --- a/packages/@webreel/core/src/index.ts +++ b/packages/@webreel/core/src/index.ts @@ -9,7 +9,7 @@ export { DEFAULT_CURSOR_SIZE, DEFAULT_HUD_THEME, } from "./types.js"; -export { connectCDP } from "./cdp.js"; +export { connectCDP, connectCDPForRecording, type CDPConnectionResult } from "./cdp.js"; export { launchChrome, ensureChrome, diff --git a/packages/@webreel/core/src/overlays.ts b/packages/@webreel/core/src/overlays.ts index 7bead99..c0ae8c6 100644 --- a/packages/@webreel/core/src/overlays.ts +++ b/packages/@webreel/core/src/overlays.ts @@ -4,8 +4,24 @@ import { OFFSCREEN_MARGIN, DEFAULT_CURSOR_SIZE, DEFAULT_HUD_THEME, + MODIFIER_ICONS, } from "./types.js"; +function buildIconSvgMap(color: string): Record { + const map: Record = {}; + for (const [char, icon] of Object.entries(MODIFIER_ICONS)) { + const paths = icon.paths + .map( + (p) => + ``, + ) + .join(""); + map[char] = + `${paths}`; + } + return map; +} + export interface OverlayTheme { cursorSvg?: string; cursorSize?: number; @@ -17,6 +33,13 @@ export interface OverlayTheme { fontFamily?: string; borderRadius?: number; position?: "top" | "bottom"; + blur?: number; + border?: string; + shadow?: string; + keyBackground?: string; + keyBorder?: string; + keyBorderRadius?: number; + keyPadding?: string; }; } @@ -28,12 +51,16 @@ export async function injectOverlays( const cursorSize = theme?.cursorSize ?? DEFAULT_CURSOR_SIZE; const cursorSvg = theme?.cursorSvg ?? DEFAULT_CURSOR_SVG; const hotspotOffset = theme?.cursorHotspot === "center" ? cursorSize / 2 : 0; - const hudBg = theme?.hud?.background ?? DEFAULT_HUD_THEME.background; const hudColor = theme?.hud?.color ?? DEFAULT_HUD_THEME.color; const hudFontSize = theme?.hud?.fontSize ?? DEFAULT_HUD_THEME.fontSize; const hudFontFamily = theme?.hud?.fontFamily ?? DEFAULT_HUD_THEME.fontFamily; - const hudBorderRadius = theme?.hud?.borderRadius ?? DEFAULT_HUD_THEME.borderRadius; const hudPosition = theme?.hud?.position ?? DEFAULT_HUD_THEME.position; + const hudBlur = theme?.hud?.blur ?? DEFAULT_HUD_THEME.blur; + const hudKeyBg = theme?.hud?.keyBackground ?? DEFAULT_HUD_THEME.keyBackground; + const hudKeyBorder = theme?.hud?.keyBorder ?? DEFAULT_HUD_THEME.keyBorder; + const hudKeyBorderRadius = + theme?.hud?.keyBorderRadius ?? DEFAULT_HUD_THEME.keyBorderRadius; + const hudKeyPadding = theme?.hud?.keyPadding ?? DEFAULT_HUD_THEME.keyPadding; await client.Runtime.evaluate({ expression: `(() => { @@ -70,13 +97,12 @@ export async function injectOverlays( "pointer-events:none", "${hudPosition}:" + z(48), "left:50%", - "transform:translateX(-50%)", + "transform:translateX(-50%) translateY(8px)", "display:flex", - "gap:" + z(14), - "padding:" + z(16) + " " + z(36), - "border-radius:" + z(${hudBorderRadius}), - "background:${hudBg}", + "align-items:center", + "gap:" + z(10), "opacity:0", + "transition:opacity 0.2s ease,transform 0.2s ease", ].join(";"); document.body.appendChild(keys); @@ -90,7 +116,16 @@ export async function injectOverlays( font-family: ${hudFontFamily}; font-size: \${${hudFontSize} / zoom}px; font-weight: 500; + letter-spacing: 0.02em; white-space: nowrap; + background: ${hudKeyBg}; + backdrop-filter: blur(${hudBlur}px); + -webkit-backdrop-filter: blur(${hudBlur}px); + border: 1px solid ${hudKeyBorder}; + border-radius: \${${hudKeyBorderRadius} / zoom}px; + padding: ${hudKeyPadding}; + box-shadow: 0 4px 16px rgba(0,0,0,0.35); + line-height: 1; } \`; document.head.appendChild(style); @@ -98,18 +133,26 @@ export async function injectOverlays( }); } -export async function showKeys(client: CDPClient, labels: string[]): Promise { +export async function showKeys( + client: CDPClient, + labels: string[], + iconColor?: string, +): Promise { + const icons = buildIconSvgMap(iconColor ?? DEFAULT_HUD_THEME.color); await client.Runtime.evaluate({ expression: `(() => { const container = document.getElementById("__demo-keys"); if (!container) return; + const icons = ${JSON.stringify(icons)}; container.innerHTML = ${JSON.stringify(labels)} .map(k => { + if (icons[k]) return '' + icons[k] + ''; const e = k.replace(/&/g,"&").replace(//g,">"); return '' + e + ''; }) .join(""); container.style.opacity = "1"; + container.style.transform = "translateX(-50%) translateY(0)"; })()`, }); } @@ -118,7 +161,10 @@ export async function hideKeys(client: CDPClient): Promise { await client.Runtime.evaluate({ expression: `(() => { const container = document.getElementById("__demo-keys"); - if (container) container.style.opacity = "0"; + if (container) { + container.style.opacity = "0"; + container.style.transform = "translateX(-50%) translateY(8px)"; + } })()`, }); } diff --git a/packages/@webreel/core/src/recorder.ts b/packages/@webreel/core/src/recorder.ts index 94719b5..3ff775e 100644 --- a/packages/@webreel/core/src/recorder.ts +++ b/packages/@webreel/core/src/recorder.ts @@ -31,11 +31,18 @@ export class Recorder { private framesDir: string | null = null; private stopResolve: (() => void) | null = null; private stoppedPromise: Promise | null = null; + private hasBeginFrameControl: boolean; constructor( outputWidth = DEFAULT_VIEWPORT_SIZE, outputHeight = DEFAULT_VIEWPORT_SIZE, - options?: { sfx?: SfxConfig; fps?: number; crf?: number; framesDir?: string }, + options?: { + sfx?: SfxConfig; + fps?: number; + crf?: number; + framesDir?: string; + hasBeginFrameControl?: boolean; + }, ) { this.outputWidth = outputWidth; this.outputHeight = outputHeight; @@ -43,6 +50,7 @@ export class Recorder { this.fps = options?.fps ?? TARGET_FPS; this.frameMs = 1000 / this.fps; this.crf = options?.crf ?? 18; + this.hasBeginFrameControl = options?.hasBeginFrameControl ?? false; if (options?.framesDir) { this.framesDir = options.framesDir; mkdirSync(this.framesDir, { recursive: true }); @@ -134,6 +142,9 @@ export class Recorder { this.stopResolve = resolve; }); + console.log( + `[recorder] beginFrameControl=${this.hasBeginFrameControl} fps=${this.fps}`, + ); this.capturePromise = this.captureLoop(client); } @@ -158,14 +169,44 @@ export class Recorder { return result; } + private lastMouseX = -1; + private lastMouseY = -1; + private mouseThrottle = 0; + private async captureLoop(client: CDPClient) { let lastFrameTime = Date.now(); let consecutiveErrors = 0; + if (this.hasBeginFrameControl) { + await this.captureLoopDeterministic(client); + return; + } + while (this.running) { try { if (this.timeline) { this.timeline.tick(); + const pos = this.timeline.getCursorPosition(); + const scale = this.timeline.getCursorScale(); + const mx = Math.round(pos.x); + const my = Math.round(pos.y); + + await client.Runtime.evaluate({ + expression: `(()=>{const el=document.getElementById("__demo-cursor");if(el){el.getAnimations().forEach(a=>a.cancel());el.style.transition="";el.style.transform="translate(${mx}px,${my}px) scale(${scale})";el.dataset.cx="${mx}";el.dataset.cy="${my}";}})()`, + }); + + const moved = mx !== this.lastMouseX || my !== this.lastMouseY; + this.mouseThrottle++; + if (moved && this.mouseThrottle >= 3) { + this.mouseThrottle = 0; + this.lastMouseX = mx; + this.lastMouseY = my; + await client.Input.dispatchMouseEvent({ + type: "mouseMoved", + x: mx, + y: my, + }); + } } else { const evalResult = await this.raceStop( client.Runtime.evaluate({ @@ -174,6 +215,12 @@ export class Recorder { ); if (!evalResult) break; } + + await client.Runtime.evaluate({ + expression: "new Promise(r=>requestAnimationFrame(r))", + awaitPromise: true, + }); + const screenshotResult = await this.raceStop( client.Page.captureScreenshot({ format: "jpeg", @@ -187,7 +234,6 @@ export class Recorder { const now = Date.now(); const elapsed = now - lastFrameTime; const frameSlots = Math.min(3, Math.max(1, Math.round(elapsed / this.frameMs))); - if (frameSlots > 1) { for (let i = 0; i < frameSlots - 1; i++) { if (this.timeline) this.timeline.tickDuplicate(); @@ -195,16 +241,15 @@ export class Recorder { this.frameCount++; } } - await this.writeFrame(buffer); this.frameCount++; + lastFrameTime = now; if (this.framesDir) { const padded = String(this.frameCount).padStart(5, "0"); writeFileSync(resolve(this.framesDir, `frame-${padded}.jpg`), buffer); } - lastFrameTime = now; consecutiveErrors = 0; } catch (err) { if (!this.running) break; @@ -220,6 +265,71 @@ export class Recorder { } } + private async captureLoopDeterministic(client: CDPClient) { + const SCREENSHOT_INTERVAL = 3; + let lastBuffer: Buffer | null = null; + let ticksSinceScreenshot = SCREENSHOT_INTERVAL; + + while (this.running) { + try { + if (this.timeline) { + this.timeline.tick(); + const pos = this.timeline.getCursorPosition(); + const scale = this.timeline.getCursorScale(); + const mx = Math.round(pos.x); + const my = Math.round(pos.y); + + await client.Runtime.evaluate({ + expression: `(()=>{const el=document.getElementById("__demo-cursor");if(el){el.getAnimations().forEach(a=>a.cancel());el.style.transition="";el.style.transform="translate(${mx}px,${my}px) scale(${scale})";el.dataset.cx="${mx}";el.dataset.cy="${my}";}})()`, + }); + + await client.Input.dispatchMouseEvent({ + type: "mouseMoved", + x: mx, + y: my, + }); + } + + ticksSinceScreenshot++; + const needsScreenshot = ticksSinceScreenshot >= SCREENSHOT_INTERVAL; + + if (needsScreenshot) { + const bfResult = await this.raceStop( + client.send("HeadlessExperimental.beginFrame", { + interval: this.frameMs, + screenshot: { format: "jpeg", quality: 60 }, + }) as Promise<{ hasDamage: boolean; screenshotData?: string }>, + ); + if (!bfResult) break; + if (bfResult.screenshotData) { + lastBuffer = Buffer.from(bfResult.screenshotData, "base64"); + ticksSinceScreenshot = 0; + } + } else { + const bfResult = await this.raceStop( + client.send("HeadlessExperimental.beginFrame", { + interval: this.frameMs, + }) as Promise, + ); + if (bfResult === null) break; + } + + if (lastBuffer) { + await this.writeFrame(lastBuffer); + this.frameCount++; + + if (this.framesDir) { + const padded = String(this.frameCount).padStart(5, "0"); + writeFileSync(resolve(this.framesDir, `frame-${padded}.jpg`), lastBuffer); + } + } + } catch (err) { + if (!this.running) break; + } + } + console.log(`[recorder] deterministic loop: ${this.frameCount} frames captured`); + } + getTempVideoPath(): string { return this.tempVideo; } @@ -270,17 +380,15 @@ export class Recorder { this.ffmpegProcess = null; } + console.log( + `[recorder] stopped: ${this.frameCount} frames, ${this.droppedFrames} dropped`, + ); + if (this.frameCount === 0) { rmSync(this.tempVideo, { force: true }); return; } - // When a timeline is set, the caller is responsible for the temp video - // (e.g. renaming it for later compositing). Don't delete or finalize it. - if (this.timeline) { - return; - } - try { const durationSec = this.frameCount / this.fps; const ext = extname(this.outputPath).toLowerCase(); diff --git a/packages/@webreel/core/src/timeline.ts b/packages/@webreel/core/src/timeline.ts index 05c392e..ad0ab52 100644 --- a/packages/@webreel/core/src/timeline.ts +++ b/packages/@webreel/core/src/timeline.ts @@ -40,6 +40,13 @@ export interface TimelineData { fontFamily: string; borderRadius: number; position: "top" | "bottom"; + blur: number; + border: string; + shadow: string; + keyBackground: string; + keyBorder: string; + keyBorderRadius: number; + keyPadding: string; }; }; frames: FrameData[]; @@ -105,6 +112,13 @@ export class InteractionTimeline { fontFamily: options?.hud?.fontFamily ?? DEFAULT_HUD_THEME.fontFamily, borderRadius: options?.hud?.borderRadius ?? DEFAULT_HUD_THEME.borderRadius, position: options?.hud?.position ?? DEFAULT_HUD_THEME.position, + blur: options?.hud?.blur ?? DEFAULT_HUD_THEME.blur, + border: options?.hud?.border ?? DEFAULT_HUD_THEME.border, + shadow: options?.hud?.shadow ?? DEFAULT_HUD_THEME.shadow, + keyBackground: options?.hud?.keyBackground ?? DEFAULT_HUD_THEME.keyBackground, + keyBorder: options?.hud?.keyBorder ?? DEFAULT_HUD_THEME.keyBorder, + keyBorderRadius: options?.hud?.keyBorderRadius ?? DEFAULT_HUD_THEME.keyBorderRadius, + keyPadding: options?.hud?.keyPadding ?? DEFAULT_HUD_THEME.keyPadding, }; if (options?.loadedFrames) { this.frames = options.loadedFrames; @@ -180,6 +194,18 @@ export class InteractionTimeline { return this.frameCount; } + getCursorPosition(): Point { + return { x: this.currentCursor.x, y: this.currentCursor.y }; + } + + getCursorScale(): number { + return this.currentCursor.scale; + } + + isMoving(): boolean { + return this.cursorPath !== null && this.pathIndex < this.cursorPath.length; + } + toJSON(): TimelineData { return { fps: this.fps, diff --git a/packages/@webreel/core/src/types.ts b/packages/@webreel/core/src/types.ts index 950aca0..084a995 100644 --- a/packages/@webreel/core/src/types.ts +++ b/packages/@webreel/core/src/types.ts @@ -1,5 +1,6 @@ export type CDPClient = { close: () => Promise; + send: (method: string, params?: Record) => Promise; Runtime: { enable: () => Promise; evaluate: (params: { @@ -49,6 +50,22 @@ export type CDPClient = { DOM: { enable: () => Promise; }; + HeadlessExperimental?: { + enable: () => Promise; + disable: () => Promise; + beginFrame: (params?: { + frameTimeTicks?: number; + interval?: number; + noDisplayUpdates?: boolean; + screenshot?: { + format?: string; + quality?: number; + }; + }) => Promise<{ + hasDamage: boolean; + screenshotData?: string; + }>; + }; }; export interface BoundingBox { @@ -78,12 +95,47 @@ export const DEFAULT_CURSOR_SVG = ''; export const DEFAULT_HUD_THEME = { - background: "rgba(0,0,0,0.5)", - color: "rgba(255,255,255,0.85)", - fontSize: 56, - fontFamily: '"Geist", -apple-system, BlinkMacSystemFont, sans-serif', - borderRadius: 18, + background: "transparent", + color: "rgba(255, 255, 255, 0.9)", + fontSize: 48, + fontFamily: '"Geist", "SF Pro", -apple-system, BlinkMacSystemFont, sans-serif', + borderRadius: 16, position: "bottom" as const, + blur: 20, + border: "transparent", + shadow: "none", + keyBackground: "rgba(22, 22, 22, 0.65)", + keyBorder: "rgba(255, 255, 255, 0.12)", + keyBorderRadius: 12, + keyPadding: "10px 20px", }; export const DEFAULT_CURSOR_SIZE = 24; + +export interface ModifierIcon { + paths: string[]; +} + +export const MODIFIER_ICONS: Record = { + "\u2318": { + paths: ["M15 6v12a3 3 0 1 0 3-3H6a3 3 0 1 0 3 3V6a3 3 0 1 0-3 3h12a3 3 0 1 0-3-3"], + }, + "\u21E7": { + paths: ["m18 15-6-6-6 6"], + }, + "\u2325": { + paths: ["M3 3h6l6 18h6", "M14 3h7"], + }, + "\u2191": { + paths: ["m5 12 7-7 7 7", "M12 19V5"], + }, + "\u2193": { + paths: ["M12 5v14", "m19 12-7 7-7-7"], + }, + "\u2190": { + paths: ["m12 19-7-7 7-7", "M19 12H5"], + }, + "\u2192": { + paths: ["M5 12h14", "m12 5 7 7-7 7"], + }, +}; diff --git a/packages/webreel/src/lib/runner.ts b/packages/webreel/src/lib/runner.ts index 3afee4b..1f15730 100644 --- a/packages/webreel/src/lib/runner.ts +++ b/packages/webreel/src/lib/runner.ts @@ -1,5 +1,5 @@ import { resolve, dirname } from "node:path"; -import { readFileSync, writeFileSync, mkdirSync, renameSync } from "node:fs"; +import { readFileSync, writeFileSync, mkdirSync } from "node:fs"; import { pathToFileURL } from "node:url"; import { type CDPClient, @@ -23,7 +23,6 @@ import { captureScreenshot, Recorder, InteractionTimeline, - compose, ensureFfmpeg, extractThumbnail, DEFAULT_VIEWPORT_SIZE, @@ -207,6 +206,8 @@ export async function runVideo( const outputPath = config.output ?? resolve(configDir, "videos", `${config.name}.mp4`); + await injectOverlays(client, overlayTheme, initialCursor); + if (shouldRecord) { ctx.setMode("record"); timeline = new InteractionTimeline(width, height, { @@ -227,6 +228,7 @@ export async function runVideo( const framesDir = saveFrames ? resolve(configDir, ".webreel", "frames", config.name) : undefined; + mkdirSync(dirname(outputPath), { recursive: true }); recorder = new Recorder(width, height, { fps: config.fps, crf, @@ -238,7 +240,6 @@ export async function runVideo( } else { ctx.setMode("preview"); ctx.setTimeline(null); - await injectOverlays(client, overlayTheme, initialCursor); } await pause(500); @@ -414,7 +415,6 @@ export async function runVideo( } if (recorder) { - const cleanVideoPath = recorder.getTempVideoPath(); await recorder.stop(); recorder = null; @@ -426,20 +426,9 @@ export async function runVideo( resolve(metadataDir, `${config.name}.timeline.json`), JSON.stringify(timelineData), ); - - const rawDir = resolve(configDir, ".webreel", "raw"); - mkdirSync(rawDir, { recursive: true }); - const rawVideoPath = resolve(rawDir, `${config.name}.mp4`); - renameSync(cleanVideoPath, rawVideoPath); - - ctx.setMode("preview"); - ctx.setTimeline(null); - mkdirSync(dirname(outputPath), { recursive: true }); - console.log(`Compositing overlays...`); - await compose(rawVideoPath, timelineData, outputPath, { sfx: config.sfx }); } - await extractThumbnailIfConfigured(config, outputPath); + await extractThumbnailIfConfigured(config, outputPath); console.log(`Done: ${outputPath}`); } else { console.log(`Preview complete: ${config.name}`);