diff --git a/packages/producer/src/regression-harness-distributed.ts b/packages/producer/src/regression-harness-distributed.ts index c62435b23..61123aa78 100644 --- a/packages/producer/src/regression-harness-distributed.ts +++ b/packages/producer/src/regression-harness-distributed.ts @@ -153,38 +153,25 @@ export async function runDistributedSimulatedRender( mkdirSync(planDir, { recursive: true }); mkdirSync(chunksDir, { recursive: true }); - // Step A: plan. `codec` is only forwarded when the format actually - // accepts it — `plan()` throws if codec is set for a non-mp4 format, - // and a caller passing `format: "mov", codec: undefined` would still - // surface that field in the resulting object. We omit it conditionally - // to keep the off-path planDir identical to pre-codec-knob output. + // Step A: plan. `plan()` throws when `codec` is set with a non-mp4 format, + // but `codec: undefined` is a no-op — so we forward it directly for mp4 + // and elide it for the others rather than branching the entire config. + // hdrMode is pinned to force-sdr so the harness's behavior is independent + // of any future auto-detect changes. const planResult = await plan( input.projectDir, - input.format === "mp4" - ? { - fps: input.fps, - width: 1920, - height: 1080, - format: "mp4", - codec: input.codec, - chunkSize: input.chunkSize, - maxParallelChunks: input.maxParallelChunks, - hdrMode: "force-sdr", - } - : { - fps: input.fps, - // Required-by-type but overridden by the composition's own attrs; - // see docstring above. Any positive integer works. - width: 1920, - height: 1080, - format: input.format, - chunkSize: input.chunkSize, - maxParallelChunks: input.maxParallelChunks, - // Force the SDR path explicitly — `auto` would still resolve to - // force-sdr in distributed mode, but pinning it here keeps the - // harness's behavior independent of any future auto-detect changes. - hdrMode: "force-sdr", - }, + { + fps: input.fps, + // Required-by-type but overridden by the composition's `data-width` / + // `data-height` attrs; any positive integer works. + width: 1920, + height: 1080, + format: input.format, + ...(input.format === "mp4" && input.codec !== undefined ? { codec: input.codec } : {}), + chunkSize: input.chunkSize, + maxParallelChunks: input.maxParallelChunks, + hdrMode: "force-sdr", + }, planDir, ); diff --git a/packages/producer/src/regression-harness.ts b/packages/producer/src/regression-harness.ts index 776d14145..fee1bd8a0 100644 --- a/packages/producer/src/regression-harness.ts +++ b/packages/producer/src/regression-harness.ts @@ -372,12 +372,10 @@ function discoverTestSuites( if (!statSync(dir).isDirectory()) continue; if (entry === "node_modules" || entry.startsWith(".")) continue; - // `tests/distributed//` is the home for fixtures authored - // specifically for the distributed pipeline (see tests/README.md and - // DISTRIBUTED-RENDERING-PLAN.md §10.2). Recurse one level deeper so - // each `` becomes a first-class fixture ID (`mp4-h264-sdr`, - // `mov-prores`, …) the user can target on the CLI without their - // namespace prefix. + // `tests/distributed//` holds fixtures authored for the + // distributed pipeline. Recurse one level deeper so each `` + // becomes a first-class fixture ID the user can target on the CLI + // without a namespace prefix. if (entry === "distributed") { for (const sub of readdirSync(dir)) { const subDir = join(dir, sub); @@ -547,9 +545,7 @@ function saveFailureDetails( snapshotHtml?: string, ): void { const failuresDir = join(suite.dir, "failures"); - if (!existsSync(failuresDir)) { - mkdirSync(failuresDir, { recursive: true }); - } + mkdirSync(failuresDir, { recursive: true }); // Save compilation failures if (result.compilation && !result.compilation.passed) { @@ -608,30 +604,36 @@ function saveFailureDetails( const framesToExtract = failedCheckpoints.slice(0, 10); if (framesToExtract.length > 0) { const framesDir = join(failuresDir, "frames"); - if (!existsSync(framesDir)) { - mkdirSync(framesDir, { recursive: true }); - } + mkdirSync(framesDir, { recursive: true }); const renderedIsDir = existsSync(renderedVideoPath) && statSync(renderedVideoPath).isDirectory(); logPretty(`Extracting ${framesToExtract.length} failed frames...`, "📸"); + // For directory output, sort both frame lists once — they're static for + // the duration of the failure-extraction loop, so the per-checkpoint + // readdir+filter+sort the loop did before was wasted syscalls. + const renderedDirFrames = renderedIsDir + ? readdirSync(renderedVideoPath) + .filter((n) => n.toLowerCase().endsWith(".png")) + .sort() + : null; + const snapshotDirFrames = renderedIsDir + ? readdirSync(snapshotVideoPath) + .filter((n) => n.toLowerCase().endsWith(".png")) + .sort() + : null; + for (const checkpoint of framesToExtract) { const timeStr = checkpoint.time.toFixed(2).replace(".", "_"); try { - if (renderedIsDir) { + if (renderedDirFrames && snapshotDirFrames) { const frameIndex = Math.max( 0, Math.round(checkpoint.time * fpsToNumber(suite.meta.renderConfig.fps)), ); - const renderedFrames = readdirSync(renderedVideoPath) - .filter((n) => n.toLowerCase().endsWith(".png")) - .sort(); - const snapshotFrames = readdirSync(snapshotVideoPath) - .filter((n) => n.toLowerCase().endsWith(".png")) - .sort(); - const renderedFrame = renderedFrames[frameIndex]; - const snapshotFrame = snapshotFrames[frameIndex]; + const renderedFrame = renderedDirFrames[frameIndex]; + const snapshotFrame = snapshotDirFrames[frameIndex]; if (renderedFrame !== undefined) { copyFileSync( join(renderedVideoPath, renderedFrame), @@ -717,17 +719,22 @@ async function runTestSuite( const tempDownloadDir = join(tempRoot, "downloads"); const outputFormat = suite.meta.renderConfig.format ?? "mp4"; const isPngSequence = outputFormat === "png-sequence"; - // png-sequence output is a directory; encoded video outputs (mp4/mov/webm) - // are single files. `outputSuffix` is appended to the in-temp + baseline - // names so both shapes round-trip cleanly. - const outputSuffix = isPngSequence - ? "" - : outputFormat === "mp4" - ? ".mp4" - : outputFormat === "mov" - ? ".mov" - : ".webm"; - const outputBasename = isPngSequence ? "frames" : `output${outputSuffix}`; + // png-sequence output is a directory (basename = "frames"); encoded video + // formats produce a single file (basename = "output."). One lookup + // covers both shapes for the in-temp render and the on-disk baseline. + // `VIDEO_EXT` is intentionally typed against only the encoded-video set — + // the `isPngSequence` ternary below short-circuits before `outputFormat` + // can be `"png-sequence"`, but TS can't narrow through that, so we + // assert the narrowing at the indexing site rather than over-widening + // the lookup table. + const VIDEO_EXT: Record<"mp4" | "mov" | "webm", string> = { + mp4: ".mp4", + mov: ".mov", + webm: ".webm", + }; + const outputBasename = isPngSequence + ? "frames" + : `output${VIDEO_EXT[outputFormat as "mp4" | "mov" | "webm"]}`; const renderedOutputPath = join(tempRoot, outputBasename); // Snapshot files stored in test's output/ directory. For png-sequence the @@ -882,10 +889,9 @@ async function runTestSuite( } if (isPngSequence) { // Frames directory — recursive copy so every PNG lands at - // `/frames/.png`. - if (existsSync(snapshotVideoPath)) { - rmSync(snapshotVideoPath, { recursive: true, force: true }); - } + // `/frames/.png`. `rmSync(..., force: true)` + // tolerates a missing path, so the prior existsSync gate was redundant. + rmSync(snapshotVideoPath, { recursive: true, force: true }); cpSync(renderedOutputPath, snapshotVideoPath, { recursive: true }); } else { copyFileSync(renderedOutputPath, snapshotVideoPath); diff --git a/packages/producer/src/services/distributed/chunkBoundary.test.ts b/packages/producer/src/services/distributed/chunkBoundary.test.ts new file mode 100644 index 000000000..6c598c187 --- /dev/null +++ b/packages/producer/src/services/distributed/chunkBoundary.test.ts @@ -0,0 +1,197 @@ +/** + * Per-adapter chunk-boundary contract: rendering the same composition at + * chunkSize=N (single chunk, no seams) vs chunkSize=N/4 (four chunks, three + * seams at frames 15, 30, 45) MUST produce byte-identical *frames*. Anything + * weaker means the worker's seek-determinism leaks across chunk boundaries. + * + * Output is png-sequence rather than mp4 because mp4 bitstreams encode + * keyframe placement directly: chunkSize=60 emits 1 IDR; chunkSize=15 emits + * 4 IDRs at frames 0/15/30/45. Those are legitimately different bytes even + * when the captured pixels are identical. The png-sequence assemble path + * merges chunk frame directories with no re-encode, so per-frame byte + * equality is exactly pixel equality. + * + * For each first-party adapter (GSAP, Anime.js, Three.js, Lottie, CSS, + * WAAPI), `tests/distributed/-boundary/src/index.html` is a + * 60-frame composition that drives the adapter through its registered seek + * hook. The fixtures intentionally lack a `meta.json` so they're invisible + * to the regression harness; this test owns them. On hosts whose + * chrome-headless-shell can't render (no SwiftShader / missing GL stack), + * each subtest soft-skips and the Docker harness covers the contract. + */ + +import { afterAll, beforeAll, describe, expect, it } from "bun:test"; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { assemble } from "./assemble.js"; +import { plan } from "./plan.js"; +import { renderChunk } from "./renderChunk.js"; + +const HOST_CHROME_FAILURE_PATTERNS = + /chrome:\/\/gpu|BROWSER_GPU_NOT_SOFTWARE|SwiftShader|HeadlessExperimental\.beginFrame|Target closed/i; + +// Per-adapter fixture directories under `packages/producer/tests/distributed/`. +// Each must hold `src/index.html`; this test owns the planning + render + +// assemble pipeline so no `output/` baseline is required. +const ADAPTERS = ["gsap", "anime", "three", "lottie", "css", "waapi"] as const; + +// Every adapter fixture is a 2-second composition at 30fps. Pin the absolute +// count so a regression that produces fewer frames in both runs (e.g. a +// probe stage that reads duration as 0s) doesn't pass vacuously. +const EXPECTED_FRAME_COUNT = 60; + +let runRoot: string; +let testsDistributedDir: string; + +beforeAll(() => { + runRoot = mkdtempSync(join(tmpdir(), "hf-chunk-boundary-test-")); + // `__dirname`-equivalent in ESM. + const moduleDir = dirname(fileURLToPath(import.meta.url)); + // packages/producer/src/services/distributed/ → packages/producer/tests/distributed/ + testsDistributedDir = resolve(moduleDir, "..", "..", "..", "tests", "distributed"); +}); + +afterAll(() => { + rmSync(runRoot, { recursive: true, force: true }); +}); + +async function planAndAssemble(input: { + projectDir: string; + workDir: string; + chunkSize: number; +}): Promise { + const planDir = join(input.workDir, "plan"); + const chunksDir = join(input.workDir, "chunks"); + const outputPath = join(input.workDir, "frames"); + mkdirSync(planDir, { recursive: true }); + mkdirSync(chunksDir, { recursive: true }); + + const planResult = await plan( + input.projectDir, + { + fps: 30, + width: 320, + height: 180, + // png-sequence: every chunk emits a directory of PNGs and assemble() + // merges them with no re-encode. Byte equality at the file level is + // pixel equality. mp4 would muddy this because chunkSize directly + // affects keyframe placement in the bitstream. + format: "png-sequence", + chunkSize: input.chunkSize, + // anime.js's IIFE bundle embeds `font-family: ui-monospace, monospace` + // as a string literal inside its JS, which `validateNoSystemFonts`'s + // document-wide regex false-positives. These fixtures display no text, + // so disabling the check (the documented escape hatch on this flag) is + // safe. + rejectOnSystemFonts: false, + }, + planDir, + ); + + const chunkPaths: string[] = []; + for (let i = 0; i < planResult.chunkCount; i++) { + // png-sequence chunks are directories, not files. + const chunkPath = join(chunksDir, `chunk-${String(i).padStart(4, "0")}`); + await renderChunk(planDir, i, chunkPath); + chunkPaths.push(chunkPath); + } + + const audioPath = join(planDir, "audio.aac"); + const audioForAssemble = existsSync(audioPath) ? audioPath : null; + await assemble(planDir, chunkPaths, audioForAssemble, outputPath); + return outputPath; +} + +describe("per-adapter chunk-boundary byte equality", () => { + // Two renders × ~5s each × cold-Chrome × six adapters can run long on the + // CI host. Per-adapter timeout keeps each `it()` failure local rather + // than smearing a single slow adapter across the suite cap. + const TIMEOUT_MS = 240_000; + + for (const adapter of ADAPTERS) { + it( + `${adapter}: chunkSize=60 (N=1) vs chunkSize=15 (N=4) produces byte-identical frames`, + async () => { + const fixtureDir = join(testsDistributedDir, `${adapter}-boundary`); + if (!existsSync(join(fixtureDir, "src", "index.html"))) { + throw new Error( + `[chunkBoundary.test] missing fixture src for adapter ${adapter}: ${fixtureDir}/src/index.html`, + ); + } + const projectDir = join(fixtureDir, "src"); + + const workOne = join(runRoot, `${adapter}-n1`); + const workFour = join(runRoot, `${adapter}-n4`); + mkdirSync(workOne, { recursive: true }); + mkdirSync(workFour, { recursive: true }); + + // Soft-skip when host Chrome can't render. Wrap *both* renders — + // cold-Chrome / SwiftShader flakes happen on the second render + // as readily as the first, and a hard-fail on the N=4 path would + // diverge from the rest of the harness's soft-skip convention. + const runRender = async (workDir: string, chunkSize: number): Promise => { + try { + return await planAndAssemble({ projectDir, workDir, chunkSize }); + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + if (HOST_CHROME_FAILURE_PATTERNS.test(message)) { + console.warn( + `[chunkBoundary.test] skipping ${adapter} — host Chrome can't render. ` + + "Docker harness covers the contract. Diagnostic:", + message.slice(0, 240), + ); + return null; + } + throw err; + } + }; + const outOne = await runRender(workOne, 60); + if (outOne === null) return; + const outFour = await runRender(workFour, 15); + if (outFour === null) return; + + // Per-frame byte equality across the two frames directories. A + // boundary regression in the adapter's seek-determinism would + // show up as one or more frames differing at the seam offsets + // (frames 15/30/45 — the chunk transitions in the N=4 run). + const framesOne = readdirSync(outOne) + .filter((n) => n.toLowerCase().endsWith(".png")) + .sort(); + const framesFour = readdirSync(outFour) + .filter((n) => n.toLowerCase().endsWith(".png")) + .sort(); + // Pin the absolute count, not just equality between the two runs. + // Otherwise a regression that truncates BOTH renders identically + // (e.g. a probe stage that misreads duration as 0s) would pass + // vacuously — `0 === 0` is true. + expect(framesOne.length).toBe(EXPECTED_FRAME_COUNT); + expect(framesFour.length).toBe(EXPECTED_FRAME_COUNT); + expect(framesOne).toEqual(framesFour); + for (let i = 0; i < framesOne.length; i++) { + const frameName = framesOne[i]; + if (frameName === undefined) continue; + const a = readFileSync(join(outOne, frameName)); + const b = readFileSync(join(outFour, frameName)); + if (a.byteLength !== b.byteLength || !a.equals(b)) { + throw new Error( + `${adapter}: frame ${frameName} differs between N=1 and N=4 ` + + `(a=${a.byteLength}B, b=${b.byteLength}B)`, + ); + } + } + }, + TIMEOUT_MS, + ); + } + + it("expected fixture directories exist", () => { + // Cheap sanity check so a `bun test` filter that excludes the + // per-adapter `it()` blocks still verifies the fixture layout. + const present = readdirSync(testsDistributedDir).filter((name) => name.endsWith("-boundary")); + for (const adapter of ADAPTERS) { + expect(present).toContain(`${adapter}-boundary`); + } + }); +}); diff --git a/packages/producer/src/services/distributed/plan.test.ts b/packages/producer/src/services/distributed/plan.test.ts index 42efb383b..36043d58d 100644 --- a/packages/producer/src/services/distributed/plan.test.ts +++ b/packages/producer/src/services/distributed/plan.test.ts @@ -242,9 +242,8 @@ describe("plan() — codec knob", () => { readFileSync(join(planDir, "meta", "encoder.json"), "utf-8"), ) as Record; expect(encoder.encoder).toBe("libx265-software"); - // SDR 8-bit yuv420p, same as h264. Distributed mode is SDR-only — - // anyone reading this and tempted to bump to 10-bit, that's HDR - // territory and lives in v1.5. + // SDR 8-bit yuv420p, same as h264 — distributed mode is SDR-only and + // 10-bit / HDR pixelFormat selection is not exposed on this surface. expect(encoder.pixelFormat).toBe("yuv420p"); }, TIMEOUT_MS, diff --git a/packages/producer/tests/distributed/anime-boundary/src/index.html b/packages/producer/tests/distributed/anime-boundary/src/index.html new file mode 100644 index 000000000..b30885a61 --- /dev/null +++ b/packages/producer/tests/distributed/anime-boundary/src/index.html @@ -0,0 +1,71 @@ + + + + + chunk-boundary: anime.js + + + + + +
+
+
+ + + + diff --git a/packages/producer/tests/distributed/css-boundary/src/index.html b/packages/producer/tests/distributed/css-boundary/src/index.html new file mode 100644 index 000000000..4c71d3189 --- /dev/null +++ b/packages/producer/tests/distributed/css-boundary/src/index.html @@ -0,0 +1,67 @@ + + + + + chunk-boundary: CSS @keyframes + + + + +
+
+
+ + + + diff --git a/packages/producer/tests/distributed/gsap-boundary/src/index.html b/packages/producer/tests/distributed/gsap-boundary/src/index.html new file mode 100644 index 000000000..83dbfc5b7 --- /dev/null +++ b/packages/producer/tests/distributed/gsap-boundary/src/index.html @@ -0,0 +1,57 @@ + + + + + chunk-boundary: GSAP + + + + +
+
+
+ + + + diff --git a/packages/producer/tests/distributed/lottie-boundary/src/index.html b/packages/producer/tests/distributed/lottie-boundary/src/index.html new file mode 100644 index 000000000..2dd4e028b --- /dev/null +++ b/packages/producer/tests/distributed/lottie-boundary/src/index.html @@ -0,0 +1,129 @@ + + + + + chunk-boundary: Lottie + + + + + +
+
+
+ + + + diff --git a/packages/producer/tests/distributed/three-boundary/src/index.html b/packages/producer/tests/distributed/three-boundary/src/index.html new file mode 100644 index 000000000..e259cadcd --- /dev/null +++ b/packages/producer/tests/distributed/three-boundary/src/index.html @@ -0,0 +1,90 @@ + + + + + chunk-boundary: Three.js + + + + + +
+ + + + diff --git a/packages/producer/tests/distributed/waapi-boundary/src/index.html b/packages/producer/tests/distributed/waapi-boundary/src/index.html new file mode 100644 index 000000000..f2d107228 --- /dev/null +++ b/packages/producer/tests/distributed/waapi-boundary/src/index.html @@ -0,0 +1,67 @@ + + + + + chunk-boundary: WAAPI + + + + +
+
+
+ + + +