Skip to content

fix: deterministic headless recording, lossless capture, and compositor optimizations #12

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
sld0Ant wants to merge 2 commits into
base: main
Choose a base branch
from
Open

fix: deterministic headless recording, lossless capture, and compositor optimizations #12

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
1492a7f
fix: deterministic headless recording, lossless capture, and composit…
sld0Ant Mar 4, 2026
eb8a21e
chore: ignore local dev artifacts
sld0Ant Mar 7, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions .gitignore
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,3 +13,8 @@ coverage
*.log
package-lock.json
demo-reel.mp4

.idea/
.pi/
videos/
webreel.config.json
243 changes: 191 additions & 52 deletions packages/@webreel/core/src/compositor.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ import { resolve, extname } from "node:path";
import sharp from "sharp";
import type { TimelineData } from "./timeline.js";
import { ensureFfmpeg } from "./ffmpeg.js";
import { finalizeMp4, finalizeWebm, finalizeGif, type SfxConfig } from "./media.js";
import { finalizeMp4, finalizeWebm, type SfxConfig } from "./media.js";

interface OverlayContext {
cursorPng: Buffer;
Expand Down Expand Up @@ -40,22 +40,36 @@ export async function compose(
zoom,
);

const ext = extname(outputPath).toLowerCase();

if (ext === ".gif") {
const gifConfig = buildGifConfig(timelineData.width, outputPath);
await compositeFrames(
ffmpegPath,
cleanVideoPath,
timelineData,
cursorPng,
zoom,
gifConfig,
);
return;
}

const workDir = resolve(homedir(), ".webreel");
mkdirSync(workDir, { recursive: true });
const tempComposed = resolve(workDir, `_composed_${Date.now()}.mp4`);

try {
const mp4Config = buildMp4Config(timelineData.fps, crf, tempComposed);
await compositeFrames(
ffmpegPath,
cleanVideoPath,
timelineData,
cursorPng,
zoom,
tempComposed,
crf,
mp4Config,
);

const ext = extname(outputPath).toLowerCase();
const durationSec = timelineData.frames.length / timelineData.fps;

if (ext === ".webm") {
Expand All @@ -67,8 +81,6 @@ export async function compose(
durationSec,
sfx,
);
} else if (ext === ".gif") {
finalizeGif(ffmpegPath, tempComposed, outputPath, timelineData.width);
} else {
finalizeMp4(
ffmpegPath,
Expand Down Expand Up @@ -97,14 +109,64 @@ async function renderCursorPng(
return sharp(Buffer.from(svgWithSize)).png().toBuffer();
}

interface CompositorFfmpegConfig {
filterComplex: string;
outputArgs: string[];
}

function buildMp4Config(
fps: number,
crf: number,
outputPath: string,
): CompositorFfmpegConfig {
return {
filterComplex: "[0][1]overlay=0:0:shortest=1",
outputArgs: [
"-c:v",
"libx264",
"-preset",
"ultrafast",
"-crf",
String(crf),
"-pix_fmt",
"yuv420p",
"-color_primaries",
"bt709",
"-color_trc",
"bt709",
"-colorspace",
"bt709",
"-movflags",
"+faststart",
"-r",
String(fps),
outputPath,
],
};
}

const GIF_FPS = 15;
const GIF_BAYER_SCALE = 5;

function buildGifConfig(width: number, outputPath: string): CompositorFfmpegConfig {
return {
filterComplex: [
`[0][1]overlay=0:0:shortest=1`,
`fps=${GIF_FPS}`,
`scale=${width}:-1:flags=lanczos`,
`split[s0][s1];[s0]palettegen=stats_mode=full[p];[s1][p]paletteuse=dither=bayer:bayer_scale=${GIF_BAYER_SCALE}`,
].join(","),
outputArgs: ["-loop", "0", outputPath],
};
}

async function compositeFrames(
ffmpegPath: string,
cleanVideoPath: string,
timeline: TimelineData,
cursorPng: Buffer,
zoom: number,
outputPath: string,
crf: number,
config: CompositorFfmpegConfig,
): Promise<void> {
const { width, height, fps } = timeline;

Expand All @@ -123,26 +185,8 @@ async function compositeFrames(
"-i",
"pipe:0",
"-filter_complex",
"[0][1]overlay=0:0:shortest=1",
"-c:v",
"libx264",
"-preset",
"ultrafast",
"-crf",
String(crf),
"-pix_fmt",
"yuv420p",
"-color_primaries",
"bt709",
"-color_trc",
"bt709",
"-colorspace",
"bt709",
"-movflags",
"+faststart",
"-r",
String(fps),
outputPath,
config.filterComplex,
...config.outputArgs,
],
{ stdio: ["pipe", "pipe", "pipe"] },
);
Expand Down Expand Up @@ -189,29 +233,12 @@ async function compositeFrames(
const stdin = ffmpeg.stdin;
if (!stdin) throw new Error("ffmpeg process has no stdin pipe");

const drain = (): Promise<void> => new Promise((res) => stdin.once("drain", res));

for (let i = 0; i < timeline.frames.length; i++) {
const frame = timeline.frames[i];
const overlayPng = await renderOverlayFrame(
frame,
width,
height,
ctx,
overlayCache,
hudCache,
);

const ok = stdin.write(overlayPng);
if (!ok) await drain();
}

stdin.end();

const stderrChunks: Buffer[] = [];
ffmpeg.stderr?.on("data", (chunk: Buffer) => stderrChunks.push(chunk));

await new Promise<void>((resolveAll, rejectAll) => {
// Register close/error listeners immediately to avoid missing events.
const KILL_TIMEOUT = 5_000;
const ffmpegDone = new Promise<void>((resolveAll, rejectAll) => {
ffmpeg.on("close", (code) => {
if (code === 0) {
resolveAll();
Expand All @@ -226,6 +253,114 @@ async function compositeFrames(
});
ffmpeg.on("error", rejectAll);
});

const PREFETCH_QUEUE_SIZE = 4;

const state = {
abortError: null as Error | null,
producerDone: false,
// Resolves when the queue has items OR the producer is done.
queueResolve: null as (() => void) | null,
// Resolves when the consumer dequeues an item (backpressure signal).
spaceResolve: null as (() => void) | null,
};

// EPIPE is expected when ffmpeg finishes reading and closes its stdin.
stdin.on("error", (err: NodeJS.ErrnoException) => {
if (err.code === "EPIPE") return;
if (!state.abortError) state.abortError = err;
});

const queue: Buffer[] = [];

const notifyConsumer = () => {
if (state.queueResolve) {
const r = state.queueResolve;
state.queueResolve = null;
r();
}
};

const notifyProducer = () => {
if (state.spaceResolve) {
const r = state.spaceResolve;
state.spaceResolve = null;
r();
}
};

const enqueue = (buf: Buffer) => {
queue.push(buf);
notifyConsumer();
};

const waitForItem = (): Promise<void> =>
new Promise((r) => {
if (queue.length > 0 || state.producerDone) return r();
state.queueResolve = r;
});

const waitForSpace = (): Promise<void> =>
new Promise((r) => {
if (queue.length < PREFETCH_QUEUE_SIZE) return r();
state.spaceResolve = r;
});

const drain = (): Promise<void> => new Promise((r) => stdin.once("drain", r));

const consumer = async () => {
while (true) {
if (queue.length === 0 && state.producerDone) break;
if (queue.length === 0) await waitForItem();
if (queue.length === 0) break;
if (state.abortError) break;

while (queue.length > 0) {
const buf = queue.shift()!;
notifyProducer();
const ok = stdin.write(buf);
if (!ok && !state.abortError) await drain();
if (state.abortError) break;
}
}
stdin.end();
};

const consumerPromise = consumer();

for (let i = 0; i < timeline.frames.length; i++) {
if (state.abortError) break;

const frame = timeline.frames[i];
const overlayPng = await renderOverlayFrame(
frame,
width,
height,
ctx,
overlayCache,
hudCache,
);

if (state.abortError) break;

if (queue.length >= PREFETCH_QUEUE_SIZE) await waitForSpace();

if (!state.abortError) enqueue(overlayPng);
}
state.producerDone = true;
notifyConsumer();

await consumerPromise;

if (state.abortError) {
ffmpeg.kill("SIGTERM");
setTimeout(() => {
if (!ffmpeg.killed) ffmpeg.kill("SIGKILL");
}, KILL_TIMEOUT);
throw state.abortError;
}

await ffmpegDone;
}

async function renderOverlayFrame(
Expand All @@ -236,8 +371,12 @@ async function renderOverlayFrame(
cache: Map<string, Buffer>,
hudCache: Map<string, sharp.OverlayOptions>,
): Promise<Buffer> {
const cx = Math.round(frame.cursor.x * ctx.zoom * 10) / 10;
const cy = Math.round(frame.cursor.y * ctx.zoom * 10) / 10;
// Whole-pixel rounding is intentional: sub-pixel precision defeats the
// overlay cache during cursor dwell/pause (float jitter creates unique keys).
// The 1px difference is imperceptible at screen resolution and invisible
// in GIF output (downsampled to 15fps with lanczos).
const cx = Math.round(frame.cursor.x * ctx.zoom);
const cy = Math.round(frame.cursor.y * ctx.zoom);
const scale = frame.cursor.scale;
const hudKey = frame.hud ? frame.hud.labels.join("|") : "";
const cacheKey = `${cx},${cy},${scale},${hudKey}`;
Expand All @@ -247,8 +386,8 @@ async function renderOverlayFrame(

const overlays: sharp.OverlayOptions[] = [];

const icx = Math.round(frame.cursor.x * ctx.zoom) - ctx.hotspotOffsetX;
const icy = Math.round(frame.cursor.y * ctx.zoom) - ctx.hotspotOffsetY;
const icx = cx - ctx.hotspotOffsetX;
const icy = cy - ctx.hotspotOffsetY;
const cursorVisible =
icx >= -ctx.cursorWidth && icx < width && icy >= -ctx.cursorHeight && icy < height;

Expand Down
32 changes: 10 additions & 22 deletions packages/@webreel/core/src/recorder.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -91,7 +91,7 @@ export class Recorder {
"-framerate",
String(this.fps),
"-c:v",
"mjpeg",
"png",
"-i",
"pipe:0",
"-c:v",
Expand Down Expand Up @@ -159,7 +159,6 @@ export class Recorder {
}

private async captureLoop(client: CDPClient) {
let lastFrameTime = Date.now();
let consecutiveErrors = 0;

while (this.running) {
Expand All @@ -174,27 +173,17 @@ export class Recorder {
);
if (!evalResult) break;
}
const screenshotResult = await this.raceStop(
client.Page.captureScreenshot({
format: "jpeg",
quality: 60,
optimizeForSpeed: true,

// With --enable-begin-frame-control Chrome only renders when
// explicitly told to, giving deterministic frame timing.
const frameResult = await this.raceStop(
client.HeadlessExperimental.beginFrame({
screenshot: { format: "png", optimizeForSpeed: true },
}),
);
if (!screenshotResult) break;

const buffer = Buffer.from(screenshotResult.data, "base64");
const now = Date.now();
const elapsed = now - lastFrameTime;
const frameSlots = Math.min(3, Math.max(1, Math.round(elapsed / this.frameMs)));

if (frameSlots > 1) {
for (let i = 0; i < frameSlots - 1; i++) {
if (this.timeline) this.timeline.tickDuplicate();

@vercel vercel Bot Mar 4, 2026 •
edited
Loading

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Debug frame files are saved with .jpg extension while the actual content is PNG data, causing a file extension/content mismatch.

Fix on Vercel

@carlesandres carlesandres May 12, 2026

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@vercel can you reissue your suggestion? The link has expired

@carlesandres carlesandres May 12, 2026

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@ctate do you know how to get that suggestion to regenerate?

I'd like to help move this PR forward and I suspect that suggestion was what stopped you from approving, am I right?

await this.writeFrame(buffer);
this.frameCount++;
}
}
if (!frameResult?.screenshotData) break;

const buffer = Buffer.from(frameResult.screenshotData, "base64");

await this.writeFrame(buffer);
this.frameCount++;
Expand All @@ -204,7 +193,6 @@ export class Recorder {
writeFileSync(resolve(this.framesDir, `frame-${padded}.jpg`), buffer);
}

lastFrameTime = now;
consecutiveErrors = 0;
} catch (err) {
if (!this.running) break;
Expand Down
Loading