From df03f695ba219705eb5fb0fc949eace987199ec9 Mon Sep 17 00:00:00 2001 From: walterlow Date: Fri, 29 May 2026 00:44:00 +0800 Subject: [PATCH 01/42] perf(timeline): stage zoom-out clip mounts under a global per-frame budget MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Zoom-out pulls clips into range and mounted the whole newly-visible cluster (TimelineItem trees) in one synchronous commit, spiking the frame to ~33ms and locking the gesture at ~30fps. Zoom-in unmounts, so it stayed smooth. Stage the mid-gesture expansion so clips mount a few per frame via a shared rAF coordinator. The budget is global across all track hooks (not per-track) since the mount cost is on one main thread — per-track would multiply by track count and reintroduce the spike. The settle path still mounts any stragglers at once. --- .../timeline/hooks/use-visible-items.test.ts | 14 +- .../timeline/hooks/use-visible-items.ts | 215 +++++++++++++++--- 2 files changed, 191 insertions(+), 38 deletions(-) diff --git a/src/features/timeline/hooks/use-visible-items.test.ts b/src/features/timeline/hooks/use-visible-items.test.ts index ee9f5983e..6f8e3c39f 100644 --- a/src/features/timeline/hooks/use-visible-items.test.ts +++ b/src/features/timeline/hooks/use-visible-items.test.ts @@ -275,15 +275,25 @@ describe('useVisibleItems filtering logic', () => { useZoomStore.getState().setZoomLevelImmediate(1) }) + // Staged: newly-visible clips mount on subsequent animation frames (under a + // global per-frame budget) rather than all at once, so the set is unchanged + // synchronously. + expect(screen.getByTestId('visible-items')).toHaveTextContent('a') + + // Animation frames before the 100ms settle: the staged expansion mounts the + // newly-visible clip — so it does NOT wait for settle. + act(() => { + vi.advanceTimersByTime(50) + }) + expect(screen.getByTestId('visible-items')).toHaveTextContent('a,b') - expect(onRender).toHaveBeenCalledTimes(2) + // Settle recomputes in the committed coordinate space; the set stays 'a,b'. act(() => { vi.advanceTimersByTime(100) }) expect(screen.getByTestId('visible-items')).toHaveTextContent('a,b') - expect(onRender).toHaveBeenCalledTimes(3) vi.useRealTimers() }) diff --git a/src/features/timeline/hooks/use-visible-items.ts b/src/features/timeline/hooks/use-visible-items.ts index 550c6c91b..28e55d364 100644 --- a/src/features/timeline/hooks/use-visible-items.ts +++ b/src/features/timeline/hooks/use-visible-items.ts @@ -63,6 +63,101 @@ function mergeVisibleRanges( } } +/** + * Grow `current` toward `target` (a superset) by at most `maxAdd` clips, + * choosing the clips nearest the current edges first. Returns `target` (and the + * exact count added) when the remaining clips fit within `maxAdd`. Lets a + * zoom-out gesture trickle clip mounts a few per frame instead of mounting a + * whole cluster in one commit. + */ +function expandRangeByClipBudget( + items: TimelineItem[] | undefined, + current: VisibleFrameRange, + target: VisibleFrameRange, + maxAdd: number, +): { range: VisibleFrameRange; added: number } { + if (!items || items.length === 0) return { range: target, added: 0 } + + const candidates: { start: number; end: number; distance: number }[] = [] + for (const item of items) { + const itemStart = item.from + const itemEnd = item.from + item.durationInFrames + const inTarget = itemEnd > target.start && itemStart < target.end + if (!inTarget) continue + const inCurrent = itemEnd > current.start && itemStart < current.end + if (inCurrent) continue + const distance = itemStart >= current.end ? itemStart - current.end : current.start - itemEnd + candidates.push({ start: itemStart, end: itemEnd, distance }) + } + + if (candidates.length <= maxAdd) return { range: target, added: candidates.length } + + candidates.sort((a, b) => a.distance - b.distance) + let start = current.start + let end = current.end + for (let index = 0; index < maxAdd; index++) { + const candidate = candidates[index]! + if (candidate.start < start) start = candidate.start + if (candidate.end > end) end = candidate.end + } + return { range: { start, end }, added: maxAdd } +} + +/** + * A track's in-flight staged zoom-out expansion. `advance` mounts up to + * `budget` more clips toward its target and returns how many it actually + * mounted; it unregisters itself once the target is reached. + */ +interface StagedExpander { + advance: (budget: number) => number +} + +/** + * Global per-frame clip-mount budget shared across ALL track hooks. Each + * useVisibleItems instance stages its own zoom-out expansion, but the mount + * cost is global (one main thread), so the budget must be global too — + * otherwise N tracks each mounting their own quota per frame multiplies the + * per-frame work and re-introduces the spike. A single shared rAF hands out the + * budget round-robin so no track starves. + */ +const GLOBAL_MOUNT_BUDGET_PER_FRAME = 2 +const activeExpanders = new Set() +let sharedExpansionRaf: number | null = null +let expanderCursor = 0 + +function ensureSharedExpansionLoop() { + if (sharedExpansionRaf === null) { + sharedExpansionRaf = requestAnimationFrame(runSharedExpansionFrame) + } +} + +function runSharedExpansionFrame() { + sharedExpansionRaf = null + const expanders = [...activeExpanders] + if (expanders.length === 0) return + + let budget = GLOBAL_MOUNT_BUDGET_PER_FRAME + // Round-robin one clip at a time so no track starves; safety bounds the loop. + let safety = budget + expanders.length + while (budget > 0 && activeExpanders.size > 0 && safety-- > 0) { + const expander = expanders[expanderCursor % expanders.length]! + expanderCursor++ + if (!activeExpanders.has(expander)) continue + budget -= expander.advance(1) + } + + if (activeExpanders.size > 0) ensureSharedExpansionLoop() +} + +function registerExpander(expander: StagedExpander) { + activeExpanders.add(expander) + ensureSharedExpansionLoop() +} + +function unregisterExpander(expander: StagedExpander) { + activeExpanders.delete(expander) +} + function quantizeInteractionPixelsPerSecond(pixelsPerSecond: number): number { if (!Number.isFinite(pixelsPerSecond) || pixelsPerSecond <= 0) { return 1 @@ -131,6 +226,81 @@ export function useVisibleItems(trackId: string) { }>({ pps: 0, fps: 0, itemsRef: undefined, transRef: undefined }) useEffect(() => { + // Commit a concrete visible range: filter items/transitions and publish. + const commit = (range: VisibleFrameRange) => { + const itemsState = useItemsStore.getState() + const items = itemsState.itemsByTrackId[trackId] + const transitions = getTrackVisibleTransitions(trackId) + const { fps } = useTimelineSettingsStore.getState() + const cullingPixelsPerSecond = getCullingPixelsPerSecond(useZoomStore.getState()) + + const visibleItems = getVisibleItemsForRange(items, range) + const visibleTransitions = getVisibleTransitionsForRange( + transitions, + itemsState.itemById, + visibleItems, + range, + ) + const next: VisibleItemsSnapshot = { visibleItems, visibleTransitions } + + lastRangeRef.current = range + lastVersionRef.current = { + pps: cullingPixelsPerSecond, + fps, + itemsRef: items, + transRef: transitions, + } + setSnapshot((prevSnap) => (areVisibleSnapshotsEqual(prevSnap, next) ? prevSnap : next)) + } + + // Staged zoom-out expansion. `expansionTarget` is the range we're chasing; + // the shared coordinator calls `expander.advance` with a slice of the global + // per-frame mount budget until we reach it. + let expansionTarget: VisibleFrameRange | null = null + const expander: StagedExpander = { + advance: (budget) => { + const target = expansionTarget + if (!target) { + unregisterExpander(expander) + return 0 + } + + // Gesture ended between frames: the non-interacting apply() path mounts + // the full visible set, so finish at the target now and stop staging. + if (!useZoomStore.getState().isZoomInteracting) { + commit(target) + expansionTarget = null + unregisterExpander(expander) + return 0 + } + + const items = useItemsStore.getState().itemsByTrackId[trackId] + const current = lastRangeRef.current ?? target + const { range, added } = expandRangeByClipBudget(items, current, target, budget) + commit(range) + + if (range.start <= target.start && range.end >= target.end) { + expansionTarget = null + unregisterExpander(expander) + } + return added + }, + } + + const cancelStagedExpansion = () => { + expansionTarget = null + unregisterExpander(expander) + } + + const scheduleStagedExpansion = (target: VisibleFrameRange) => { + expansionTarget = target + // Register and let the shared coordinator mount the clips under the global + // budget. Mounting synchronously here would bypass that budget: all tracks + // schedule within the same store-notify, so N synchronous commits would + // land in one frame — the exact spike we're avoiding. + registerExpander(expander) + } + const apply = () => { const zoomState = useZoomStore.getState() const cullingPixelsPerSecond = getCullingPixelsPerSecond(zoomState) @@ -147,6 +317,8 @@ export function useVisibleItems(trackId: string) { // Keep zoom-in stable, but allow zoom-out to expand the mounted set during // the gesture so newly visible clips do not wait for the settle timeout. + // The expansion is staged (a clip budget per frame) so a dense cluster of + // newly-visible clips does not mount in a single commit and spike the frame. if ( zoomState.isZoomInteracting && prev.fps === fps && @@ -157,27 +329,14 @@ export function useVisibleItems(trackId: string) { return } - const expandedRange = mergeVisibleRanges(lastRange, newRange) - const visibleItems = getVisibleItemsForRange(items, expandedRange) - const visibleTransitions = getVisibleTransitionsForRange( - transitions, - itemsState.itemById, - visibleItems, - expandedRange, - ) - const next: VisibleItemsSnapshot = { visibleItems, visibleTransitions } - - lastRangeRef.current = expandedRange - lastVersionRef.current = { - pps: cullingPixelsPerSecond, - fps, - itemsRef: items, - transRef: transitions, - } - setSnapshot((prevSnap) => (areVisibleSnapshotsEqual(prevSnap, next) ? prevSnap : next)) + scheduleStagedExpansion(mergeVisibleRanges(lastRange, newRange)) return } + // Any non-interacting recompute (settle, scroll, data/fps change) + // supersedes an in-flight staged expansion. + cancelStagedExpansion() + // Fast path: if only scroll changed and the range shift is within // hysteresis, the visible item set is guaranteed unchanged. // Array references are compared (not lengths) so in-place mutations @@ -198,24 +357,7 @@ export function useVisibleItems(trackId: string) { } } - const visibleItems = getVisibleItemsForRange(items, newRange) - const visibleTransitions = getVisibleTransitionsForRange( - transitions, - itemsState.itemById, - visibleItems, - newRange, - ) - const next: VisibleItemsSnapshot = { visibleItems, visibleTransitions } - - lastRangeRef.current = newRange - lastVersionRef.current = { - pps: cullingPixelsPerSecond, - fps, - itemsRef: items, - transRef: transitions, - } - - setSnapshot((prevSnap) => (areVisibleSnapshotsEqual(prevSnap, next) ? prevSnap : next)) + commit(newRange) } // Zoom-specific subscriber: skip when the quantized culling pps hasn't @@ -270,6 +412,7 @@ export function useVisibleItems(trackId: string) { ] return () => { + cancelStagedExpansion() for (const unsubscribe of unsubscribers) { unsubscribe() } From a0fda3b0f08c32219340d70d9ceab9e111cb914d Mon Sep 17 00:00:00 2001 From: walterlow Date: Fri, 29 May 2026 01:16:43 +0800 Subject: [PATCH 02/42] perf(timeline): drop per-frame forced reflow in the zoom scroll path At extreme zoom (timeline >1M px wide), a fast zoom fling ran at a sustained 20-30fps. LoAF attribution showed ~16ms of forced synchronous layout per frame, not React (dev Profiler put React commit at ~2.5ms median). The scroll-clamp and pending-scroll layout effects read container.clientWidth / scrollLeft right after writing the new width every frame, forcing a reflow of the giant tree. Use the cached viewport width (invariant under horizontal zoom) and the tracked scrollLeft instead of live reads, and pass the just-written scrollLeft into syncViewportFromContainer so it skips the read-back. Cuts dropped frames on a full fling-to-max from ~51 to ~7-15 with a 17ms (60fps) median. --- .../timeline/components/timeline-content.tsx | 41 +++++++++++++++---- 1 file changed, 32 insertions(+), 9 deletions(-) diff --git a/src/features/timeline/components/timeline-content.tsx b/src/features/timeline/components/timeline-content.tsx index df94bf960..451cb52ac 100644 --- a/src/features/timeline/components/timeline-content.tsx +++ b/src/features/timeline/components/timeline-content.tsx @@ -836,16 +836,23 @@ export const TimelineContent = memo(function TimelineContent({ // refreshes this cache; we fall back to a live read until it has measured once. const viewportDimsRef = useRef<{ width: number; height: number } | null>(null) - const syncViewportFromContainer = useCallback(() => { + const syncViewportFromContainer = useCallback((knownScrollLeft?: number) => { const container = containerRef.current if (!container) return const dims = viewportDimsRef.current const viewportWidth = dims?.width ?? container.clientWidth const viewportHeight = dims?.height ?? tracksContainerRef.current?.clientHeight ?? container.clientHeight + // When the caller just wrote scrollLeft it passes that value so we can skip + // the read-back. Reading scrollLeft/scrollTop after a write forces a + // synchronous reflow — negligible normally, but ~16ms/frame at extreme zoom + // widths (>1M px). The main scroll container is overflow-y-hidden, so its + // scrollTop is always 0 in the known-value path. + const scrollLeft = knownScrollLeft ?? container.scrollLeft + const scrollTop = knownScrollLeft !== undefined ? 0 : container.scrollTop useTimelineViewportStore.getState().setViewport({ - scrollLeft: container.scrollLeft, - scrollTop: container.scrollTop, + scrollLeft, + scrollTop, viewportWidth, viewportHeight, }) @@ -987,10 +994,15 @@ export const TimelineContent = memo(function TimelineContent({ // Apply pending scroll AFTER render when DOM has updated width // This ensures zoom anchor works correctly even when timeline extends beyond content useLayoutEffect(() => { - if (pendingScrollRef.current !== null && containerRef.current) { - containerRef.current.scrollLeft = pendingScrollRef.current + const container = containerRef.current + if (pendingScrollRef.current !== null && container) { + const next = pendingScrollRef.current + container.scrollLeft = next pendingScrollRef.current = null - syncViewportFromContainer() + // Keep scrollLeftRef fresh so the clamp effect below can decide without a + // forced scrollLeft read, and pass the known value to skip the read-back. + scrollLeftRef.current = next + syncViewportFromContainer(next) } }) @@ -1324,8 +1336,19 @@ export const TimelineContent = memo(function TimelineContent({ return } - const maxScrollLeft = Math.max(0, timelineWidth - container.clientWidth) - if (container.scrollLeft <= maxScrollLeft + 1) { + // Use the cached viewport width and tracked scrollLeft instead of reading + // container.clientWidth / container.scrollLeft. Both are layout reads that, + // after this render's width write, force a synchronous reflow every frame — + // ~16ms at extreme zoom widths. clientWidth is invariant under horizontal + // zoom, and scrollLeftRef is kept fresh by the scroll handler and the + // pending-scroll effect above. Fall back to a live read only before the + // ResizeObserver has measured (mount). + const cachedWidth = viewportDimsRef.current?.width + const viewportWidthForClamp = cachedWidth ?? container.clientWidth + const maxScrollLeft = Math.max(0, timelineWidth - viewportWidthForClamp) + const currentScrollLeft = + cachedWidth !== undefined ? scrollLeftRef.current : container.scrollLeft + if (currentScrollLeft <= maxScrollLeft + 1) { return } @@ -1333,7 +1356,7 @@ export const TimelineContent = memo(function TimelineContent({ // without subscribing broad UI surfaces to item-array churn. container.scrollLeft = maxScrollLeft scrollLeftRef.current = maxScrollLeft - syncViewportFromContainer() + syncViewportFromContainer(maxScrollLeft) }, [timelineWidth, syncViewportFromContainer]) // NOTE: itemsByTrack removed - TimelineTrack now fetches its own items From 7bc44e12c1e610dd3d45cec2d273cb6ab2962ef6 Mon Sep 17 00:00:00 2001 From: walterlow Date: Fri, 29 May 2026 01:24:23 +0800 Subject: [PATCH 03/42] perf(timeline): pass tracked scrollLeft to the viewport-sync RAF MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The scroll handler already records scrollLeft into scrollLeftRef before scheduling a viewport sync, but the RAF then re-read container.scrollLeft — forcing a synchronous reflow whenever a zoom width write landed the same frame. Hand the tracked value to syncViewportFromContainer instead. Removes the last per-frame forced layout in the zoom path (LoAF forcedStyleAndLayout ~16ms/frame to ~0); moved scrollLeftRef up so it's in scope for the scheduler. --- src/features/timeline/components/timeline-content.tsx | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/features/timeline/components/timeline-content.tsx b/src/features/timeline/components/timeline-content.tsx index 451cb52ac..48972d912 100644 --- a/src/features/timeline/components/timeline-content.tsx +++ b/src/features/timeline/components/timeline-content.tsx @@ -829,6 +829,10 @@ export const TimelineContent = memo(function TimelineContent({ const queuedZoomLevelRef = useRef(null) const queuedZoomScrollLeftRef = useRef(null) const zoomApplyRafRef = useRef(null) + // Latest known scrollLeft, kept fresh by the scroll handler. Declared here so + // scheduleViewportSync can hand it to syncViewportFromContainer instead of + // reading container.scrollLeft back (a forced reflow after a width write). + const scrollLeftRef = useRef(0) // Cached viewport box dimensions. clientWidth/clientHeight are invariant under // scroll and horizontal zoom (only the *content* width changes), so reading @@ -862,7 +866,10 @@ export const TimelineContent = memo(function TimelineContent({ if (viewportSyncRafRef.current !== null) return viewportSyncRafRef.current = requestAnimationFrame(() => { viewportSyncRafRef.current = null - withPerfMeasure('tl.raf.viewportSync', syncViewportFromContainer) + // Pass the tracked scrollLeft (handleScroll updates it before scheduling) + // so the sync skips a container.scrollLeft read-back, which would force a + // synchronous reflow when a zoom width write landed the same frame. + withPerfMeasure('tl.raf.viewportSync', () => syncViewportFromContainer(scrollLeftRef.current)) }) }, [syncViewportFromContainer]) @@ -938,7 +945,6 @@ export const TimelineContent = memo(function TimelineContent({ // Track scroll position with coalesced updates for viewport culling // Throttle at 50ms to match zoom throttle rate - prevents width jitter during zoom+scroll - const scrollLeftRef = useRef(0) const scrollUpdateTimeoutRef = useRef | null>(null) const SCROLL_THROTTLE_MS = 50 // Match zoom throttle for synchronized updates const setScrollPosition = useTimelineStore((s) => s.setScrollPosition) From f075ba366fc7c21e1694036ab9ff364c1f115bdf Mon Sep 17 00:00:00 2001 From: walterlow Date: Fri, 29 May 2026 01:43:22 +0800 Subject: [PATCH 04/42] fix(timeline): render waveform from true peak envelope The clip waveform height came from a hand-tuned heuristic (mean*0.38 + max2*0.34 + needle*2.35, then ^1.05) that produced two zoom-dependent artifacts: the needle*2.35 term exploded isolated transients into disproportionate spikes, and the stable mean+max2 terms masked real per-column peak dips, flattening loud/compressed content into a uniform saturated band. Replace it with a faithful peak envelope (peak*0.85 + mean*0.15) and a gentle perceptual gamma (^0.72) that lifts quiet passages without distorting loud transients. Extracted into a shared, tested computeWaveformAmplitude helper so both the regular and compound-clip renderers stay in sync. --- .../clip-waveform/amplitude.test.ts | 48 +++++++++++++++++++ .../components/clip-waveform/amplitude.ts | 30 ++++++++++++ .../clip-waveform/compound-clip-waveform.tsx | 21 +++----- .../components/clip-waveform/index.tsx | 21 +++----- 4 files changed, 92 insertions(+), 28 deletions(-) create mode 100644 src/features/timeline/components/clip-waveform/amplitude.test.ts create mode 100644 src/features/timeline/components/clip-waveform/amplitude.ts diff --git a/src/features/timeline/components/clip-waveform/amplitude.test.ts b/src/features/timeline/components/clip-waveform/amplitude.test.ts new file mode 100644 index 000000000..4e87ad34a --- /dev/null +++ b/src/features/timeline/components/clip-waveform/amplitude.test.ts @@ -0,0 +1,48 @@ +import { describe, expect, it } from 'vite-plus/test' +import { computeWaveformAmplitude } from './amplitude' + +describe('computeWaveformAmplitude', () => { + it('returns 0 for empty or invalid windows', () => { + expect(computeWaveformAmplitude(0.5, 0.5, 0, 1)).toBe(0) + expect(computeWaveformAmplitude(0.5, 0.5, 4, 0)).toBe(0) + }) + + it('is monotonic in the window peak', () => { + const low = computeWaveformAmplitude(0.3, 0.3, 1, 1) + const mid = computeWaveformAmplitude(0.6, 0.6, 1, 1) + const high = computeWaveformAmplitude(0.9, 0.9, 1, 1) + expect(low).toBeLessThan(mid) + expect(mid).toBeLessThan(high) + }) + + it('preserves structure for loud content (no flat band)', () => { + // A loud beat vs the dip between beats should differ clearly even when both + // sit high on a compressed track normalized to its own peak. + const beat = computeWaveformAmplitude(0.95, 0.7, 10, 1) + const betweenBeats = computeWaveformAmplitude(0.5, 0.35, 10, 1) + expect(beat - betweenBeats).toBeGreaterThan(0.25) + }) + + it('lifts quiet passages so they stay visible', () => { + const quiet = computeWaveformAmplitude(0.2, 0.08, 10, 1) + // Quiet speech (0.2 linear peak) should render meaningfully above the floor. + expect(quiet).toBeGreaterThan(0.2) + expect(quiet).toBeLessThan(0.45) + }) + + it('keeps an isolated transient proportional, not exploded', () => { + // One loud sample inside an otherwise-quiet window: height tracks the peak + // but does not blow past a genuinely loud sustained section. + const transient = computeWaveformAmplitude(0.9, 0.12, 20, 1) + const sustainedLoud = computeWaveformAmplitude(0.9, 0.85, 20, 1) + expect(transient).toBeLessThanOrEqual(sustainedLoud) + expect(transient).toBeGreaterThan(0.6) + }) + + it('clamps to 1 at full scale', () => { + // Every sample at full scale → peak and mean both 1 → height 1. + expect(computeWaveformAmplitude(1, 5, 5, 1)).toBeCloseTo(1, 5) + // Values above the normalization peak still clamp. + expect(computeWaveformAmplitude(2, 10, 5, 1)).toBeCloseTo(1, 5) + }) +}) diff --git a/src/features/timeline/components/clip-waveform/amplitude.ts b/src/features/timeline/components/clip-waveform/amplitude.ts new file mode 100644 index 000000000..3d8cf3d2f --- /dev/null +++ b/src/features/timeline/components/clip-waveform/amplitude.ts @@ -0,0 +1,30 @@ +/** + * Map a window of peak samples to a normalized waveform height (0..1). + * + * The window's true peak drives the height, so loud, sustained content keeps its + * real beat-to-beat structure instead of collapsing into a flat band; a small + * mean contribution adds visual body. A gentle gamma (< 1) lifts quiet passages + * so speech stays readable without distorting loud transients. + * + * We deliberately do NOT amplify `(peak - secondPeak)`: that "needle" term turned + * isolated transients into disproportionate spikes, worst at low zoom where each + * pixel column folds in a wider time window. + */ +const PEAK_WEIGHT = 0.85 +const MEAN_WEIGHT = 0.15 +const PERCEPTUAL_GAMMA = 0.72 + +export function computeWaveformAmplitude( + windowPeak: number, + windowSum: number, + sampleCount: number, + normalizationPeak: number, +): number { + if (sampleCount <= 0 || normalizationPeak <= 0) { + return 0 + } + const peak = Math.min(1, windowPeak / normalizationPeak) + const mean = Math.min(1, windowSum / sampleCount / normalizationPeak) + const linear = peak * PEAK_WEIGHT + mean * MEAN_WEIGHT + return linear <= 0.001 ? 0 : Math.pow(linear, PERCEPTUAL_GAMMA) +} diff --git a/src/features/timeline/components/clip-waveform/compound-clip-waveform.tsx b/src/features/timeline/components/clip-waveform/compound-clip-waveform.tsx index 584e0180c..877c543fb 100644 --- a/src/features/timeline/components/clip-waveform/compound-clip-waveform.tsx +++ b/src/features/timeline/components/clip-waveform/compound-clip-waveform.tsx @@ -11,6 +11,7 @@ import { WAVEFORM_FILL_COLOR, WAVEFORM_STROKE_COLOR } from '../../constants' import { getCompositionOwnedAudioSources } from '../../utils/composition-clip-summary' import { mixCompoundClipWaveformPeaks } from '../../utils/compound-clip-waveform' import { computeWaveformRenderWindow } from './render-window' +import { computeWaveformAmplitude } from './amplitude' import { getPreviewStartupDelayMs, schedulePreviewWork } from '../../hooks/preview-work-budget' import { getWaveformActiveTileCount, @@ -252,17 +253,13 @@ export const CompoundClipWaveform = memo(function CompoundClipWaveform({ const windowStart = Math.max(0, peakIndex - halfWindow) const windowEnd = Math.min(peaks.length, peakIndex + halfWindow + 1) - let max1 = 0 - let max2 = 0 + let windowPeak = 0 let windowSum = 0 let sampleCount = 0 for (let i = windowStart; i < windowEnd; i += 1) { const value = peaks[i] ?? 0 - if (value >= max1) { - max2 = max1 - max1 = value - } else if (value > max2) { - max2 = value + if (value > windowPeak) { + windowPeak = value } windowSum += value sampleCount += 1 @@ -272,13 +269,9 @@ export const CompoundClipWaveform = memo(function CompoundClipWaveform({ continue } - const normalizedMax1 = Math.min(1, max1 / normalizationPeak) - const normalizedMax2 = Math.min(1, max2 / normalizationPeak) - const normalizedMean = Math.min(1, windowSum / sampleCount / normalizationPeak) - const needle = Math.max(0, normalizedMax1 - normalizedMax2) - const peakValue = Math.min(1, normalizedMean * 0.38 + normalizedMax2 * 0.34 + needle * 2.35) - const amp = peakValue <= 0.001 ? 0 : Math.pow(peakValue, 1.05) - amplitudes[x] = amp * maxWaveHeight + amplitudes[x] = + computeWaveformAmplitude(windowPeak, windowSum, sampleCount, normalizationPeak) * + maxWaveHeight } for (let x = 0; x < amplitudeCount; x += 1) { diff --git a/src/features/timeline/components/clip-waveform/index.tsx b/src/features/timeline/components/clip-waveform/index.tsx index ad3e7902c..ca53ddab2 100644 --- a/src/features/timeline/components/clip-waveform/index.tsx +++ b/src/features/timeline/components/clip-waveform/index.tsx @@ -13,6 +13,7 @@ import { import { WAVEFORM_FILL_COLOR, WAVEFORM_STROKE_COLOR } from '../../constants' import { createLogger } from '@/shared/logging/logger' import { computeWaveformRenderWindow } from './render-window' +import { computeWaveformAmplitude } from './amplitude' import { getWaveformActiveTileCount, useAdaptiveWaveformRenderVersion, @@ -264,19 +265,15 @@ export const ClipWaveform = memo(function ClipWaveform({ const windowStart = Math.max(0, peakIndex - halfWindow) const windowEnd = Math.min(peakSampleCount, peakIndex + halfWindow + 1) - let max1 = 0 - let max2 = 0 + let windowPeak = 0 let windowSum = 0 let sampleCount = 0 for (let i = windowStart; i < windowEnd; i++) { const value = stereo ? Math.max(peaks[i * 2] ?? 0, peaks[i * 2 + 1] ?? 0) : (peaks[i] ?? 0) - if (value >= max1) { - max2 = max1 - max1 = value - } else if (value > max2) { - max2 = value + if (value > windowPeak) { + windowPeak = value } windowSum += value sampleCount++ @@ -286,13 +283,9 @@ export const ClipWaveform = memo(function ClipWaveform({ continue } - const normalizedMax1 = Math.min(1, max1 / normalizationPeak) - const normalizedMax2 = Math.min(1, max2 / normalizationPeak) - const normalizedMean = Math.min(1, windowSum / sampleCount / normalizationPeak) - const needle = Math.max(0, normalizedMax1 - normalizedMax2) - const peakValue = Math.min(1, normalizedMean * 0.38 + normalizedMax2 * 0.34 + needle * 2.35) - const amp = peakValue <= 0.001 ? 0 : Math.pow(peakValue, 1.05) - amplitudes[x] = amp * maxWaveHeight + amplitudes[x] = + computeWaveformAmplitude(windowPeak, windowSum, sampleCount, normalizationPeak) * + maxWaveHeight } for (let x = 0; x < amplitudeCount; x++) { From 7b40a2a012d89dfeb4dffee12bedaa37f1e5fb44 Mon Sep 17 00:00:00 2001 From: walterlow Date: Fri, 29 May 2026 02:03:15 +0800 Subject: [PATCH 05/42] perf(timeline): window video filmstrip tiles to the visible range ClipFilmstrip rendered one tile per slot across the clip's entire width, ignoring the visibleStartRatio/visibleEndRatio props it already received. A clip spanning a large fraction of a max-zoom timeline mounted thousands of tiles that React reconciled and the compositor painted on every scroll frame, making navigator scrubbing drop to ~20fps with a ~460ms stall on first paint. Window the slot grid to the visible region (+ pad) via the existing computeFilmstripRenderWindow helper, mirroring ImageFilmstrip and the waveform's TiledCanvas. Off-window areas keep showing the repeating cover-frame background, so windowing is seamless. Cuts the giant-clip tile count from ~6300 to ~210 and restores ~60fps scrubbing. --- .../components/clip-filmstrip/index.tsx | 39 ++++++++++++++++--- 1 file changed, 33 insertions(+), 6 deletions(-) diff --git a/src/features/timeline/components/clip-filmstrip/index.tsx b/src/features/timeline/components/clip-filmstrip/index.tsx index 2e3566b21..f3195bc1f 100644 --- a/src/features/timeline/components/clip-filmstrip/index.tsx +++ b/src/features/timeline/components/clip-filmstrip/index.tsx @@ -1,5 +1,6 @@ import { memo, useEffect, useState, useMemo, useCallback, useRef, type RefCallback } from 'react' import { FilmstripSkeleton } from './filmstrip-skeleton' +import { computeFilmstripRenderWindow } from './render-window' import { useFilmstrip, type FilmstripFrame } from '../../hooks/use-filmstrip' import { resolveMediaUrl, resolveProxyUrl } from '@/features/timeline/deps/media-library-resolver' import { useMediaBlobUrl } from '../../hooks/use-media-blob-url' @@ -9,6 +10,11 @@ import { useMediaLibraryStore } from '@/features/timeline/deps/media-library-sto const logger = createLogger('ClipFilmstrip') +// Pad the rendered tile window beyond the visible range so a fast scrub never +// outruns the mounted tiles within a frame. Matches the animated-image filmstrip. +const VIEWPORT_PAD_TILES = 2 +const VIEWPORT_PAD_PX = 600 + interface ClipFilmstripProps { /** Media ID from the timeline item */ mediaId: string @@ -205,6 +211,8 @@ export const ClipFilmstrip = memo(function ClipFilmstrip({ speed, isReversed = false, isVisible, + visibleStartRatio = 0, + visibleEndRatio = 1, pixelsPerSecond, }: ClipFilmstripProps) { const containerRef = useRef(null) @@ -335,17 +343,33 @@ export const ClipFilmstrip = memo(function ClipFilmstrip({ // - Tile width is exactly thumbnailWidth — never stretched, squashed, or // merged. Segment boundaries are handled by overflow clipping outside the // tile, so a short segment only reveals less of the full-size tile. - // - key=slot is the integer slot index on the pixel grid. At fixed zoom the - // slot set never changes; scrolling can't refresh tiles, and a slot's - // frame prop updating as extraction lands doesn't remount its DOM. + // - key=slot is the integer slot index on the pixel grid, so a slot that stays + // inside the window keeps its DOM as the window slides; a slot's frame prop + // updating as extraction lands doesn't remount its DOM either. + // - Only slots intersecting the visible window (+ pad) are emitted, mirroring + // the animated-image filmstrip and the waveform's TiledCanvas. Without this + // a clip spanning a large fraction of a max-zoom timeline mounts one tile per + // slot across its ENTIRE width (thousands of s), which React must + // reconcile and the compositor must paint on every scroll frame — the + // dominant cost when scrubbing the navigator at high zoom. Off-window areas + // keep showing the repeating cover-frame background, so windowing is seamless. const tiles = useMemo(() => { if (!frames || frames.length === 0 || renderPixelsPerSecond <= 0) return [] if (effectiveEnd <= effectiveStart) return [] const pixelsPerSourceSecond = renderPixelsPerSecond / Math.max(0.0001, speed) const tileWidth = thumbnailWidth - const slotCount = Math.ceil(renderClipWidth / tileWidth) - if (slotCount === 0) return [] + + const { startTile, endTile } = computeFilmstripRenderWindow({ + renderWidth: renderClipWidth, + visibleWidth: visibleClipWidth, + tileWidth, + visibleStartRatio, + visibleEndRatio, + minimumPadTiles: VIEWPORT_PAD_TILES, + minimumPadPx: VIEWPORT_PAD_PX, + }) + if (endTile <= startTile) return [] const findClosestFrame = (targetTime: number): FilmstripFrame | null => { if (frames.length === 0) return null @@ -369,7 +393,7 @@ export const ClipFilmstrip = memo(function ClipFilmstrip({ const result: { slot: number; frame: FilmstripFrame; x: number; width: number }[] = [] - for (let slot = 0; slot < slotCount; slot++) { + for (let slot = startTile; slot < endTile; slot++) { const slotX = slot * tileWidth const slotCenterX = slotX + tileWidth * 0.5 const slotCenterTime = isReversed @@ -385,6 +409,9 @@ export const ClipFilmstrip = memo(function ClipFilmstrip({ frames, renderPixelsPerSecond, renderClipWidth, + visibleClipWidth, + visibleStartRatio, + visibleEndRatio, effectiveStart, effectiveEnd, isReversed, From 34fd3758ee20089f29e483eff090e208afa4c164 Mon Sep 17 00:00:00 2001 From: walterlow Date: Fri, 29 May 2026 13:43:58 +0800 Subject: [PATCH 06/42] fix(preview): keep continuous overlay across transition windows during playback A transition only stayed on the per-frame fast-scrub overlay during playback when some item incidentally forced the continuous overlay (GPU effect, blend, corner-pin, or composition). A plain transition therefore relied on an unrelated item happening to keep it on; when that item ended at the cut, the overlay dropped into the buffered path, which could leave second-half frames un-rendered and collapse the wipe to a single clip until the window ended. This was most visible with reversed clips but was not reverse-specific, and only affected live preview (export decodes every frame synchronously). Force the continuous overlay on active transition-window frames during playback as well as scrubbing, so every transition renders every frame via the path that already worked for the transitions that looked correct. --- .../preview/hooks/use-gpu-effects-overlay.ts | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/features/preview/hooks/use-gpu-effects-overlay.ts b/src/features/preview/hooks/use-gpu-effects-overlay.ts index 93c1d4bd5..208ebfe5d 100644 --- a/src/features/preview/hooks/use-gpu-effects-overlay.ts +++ b/src/features/preview/hooks/use-gpu-effects-overlay.ts @@ -139,7 +139,17 @@ export function useGpuEffectsOverlay(..._args: unknown[]) { frame, previewEffectsByItemId, compositionById, - { forceTransitionFrames: playback.previewFrame !== null }, + // Keep the continuous (fast-scrub) overlay forced across an active + // transition window during playback as well as scrubbing — not only + // when a participant has GPU effects/blend/corner-pin. Otherwise a + // plain transition can drop the continuous overlay mid-window (e.g. + // when an unrelated effected/composition item that happened to be + // keeping it on ends at the cut), switching to the buffered overlay + // path which can leave frames un-rendered and collapse the wipe to + // one clip for the rest of the transition. Forcing it for the whole + // window keeps every transition on the per-frame render path that + // already works for the transitions that look correct today. + { forceTransitionFrames: playback.previewFrame !== null || playback.isPlaying }, ) return prev === next ? prev : next }) From 969578a796eec705a24f2b32799c989893f0b5e1 Mon Sep 17 00:00:00 2001 From: walterlow Date: Fri, 29 May 2026 13:44:24 +0800 Subject: [PATCH 07/42] feat(debug): add captureTransition preview-render diagnostics Diagnosing 'live preview looks wrong but export is fine' transition bugs meant hand-pasting console probes and reading raw per-frame dumps. Add a reusable DEV-only facility instead. Adds src/shared/logging/preview-trace.ts: a trace sink plus a pure analyzePreviewTrace() that summarizes, per transition half, which overlay path the pump chose and which participants composited, flags stalls/gaps and mid-window overlay path switches, and emits a plain-language verdict (unit-tested). Wires permanent import.meta.env.DEV-guarded hooks (tree-shaken from prod, no-op unless a trace is running) into the render pump's overlay decision and the renderer's transition-participant draws, and exposes window.__DEBUG__.captureTransition(frame?) (one-call seek+play+record+analyze) plus __DEBUG__.previewTrace.{start,stop,clear,events}(). --- src/app/debug/project-debug.ts | 135 ++++++++++ .../utils/canvas-item-renderer/video.ts | 14 ++ .../use-preview-render-pump-controller.ts | 22 ++ src/shared/logging/preview-trace.test.ts | 84 +++++++ src/shared/logging/preview-trace.ts | 234 ++++++++++++++++++ 5 files changed, 489 insertions(+) create mode 100644 src/shared/logging/preview-trace.test.ts create mode 100644 src/shared/logging/preview-trace.ts diff --git a/src/app/debug/project-debug.ts b/src/app/debug/project-debug.ts index 0ae60bc6b..5ce6b2256 100644 --- a/src/app/debug/project-debug.ts +++ b/src/app/debug/project-debug.ts @@ -133,6 +133,24 @@ interface ProjectDebugAPI { previewPerf: () => any // eslint-disable-next-line @typescript-eslint/no-explicit-any transitionTrace: () => any + /** + * Capture and analyze the live preview render path through a transition. + * One call: seeks ahead of the transition, plays through it while recording + * which overlay path the pump chose and which participants composited per + * frame, then returns a per-half analysis with a plain-language verdict. + * Pass a frame near the transition; omit to use the nearest at/after the + * current playhead. Requires a foreground tab (playback needs rAF). + */ + // eslint-disable-next-line @typescript-eslint/no-explicit-any + captureTransition: (targetFrame?: number) => Promise + /** Low-level preview-trace controls (start/stop/dump the raw event buffer). */ + previewTrace: { + start: () => void + stop: () => void + clear: () => void + // eslint-disable-next-line @typescript-eslint/no-explicit-any + events: () => readonly any[] + } // eslint-disable-next-line @typescript-eslint/no-explicit-any prewarmCache: () => any // eslint-disable-next-line @typescript-eslint/no-explicit-any @@ -556,6 +574,123 @@ function createDebugAPI(): ProjectDebugAPI { return (window as unknown as Record).__PREVIEW_TRANSITIONS__ ?? [] }, + previewTrace: { + start: () => { + void import('@/shared/logging/preview-trace').then((m) => m.startPreviewTrace()) + }, + stop: () => { + void import('@/shared/logging/preview-trace').then((m) => m.stopPreviewTrace()) + }, + clear: () => { + void import('@/shared/logging/preview-trace').then((m) => m.clearPreviewTrace()) + }, + events: () => { + const buf = (window as unknown as { __PREVIEW_TRACE_LAST__?: readonly unknown[] }) + .__PREVIEW_TRACE_LAST__ + return buf ?? [] + }, + }, + + captureTransition: async (targetFrame?: number) => { + const [ + { usePlaybackStore }, + { useTransitionsStore }, + { useItemsStore }, + { useTimelineStore }, + { resolveTransitionWindows }, + trace, + ] = await Promise.all([ + import('@/shared/state/playback'), + import('@/features/timeline/stores/transitions-store'), + import('@/features/timeline/stores/items-store'), + import('@/features/timeline/stores/timeline-store'), + import('@/shared/timeline/transitions/transition-planner'), + import('@/shared/logging/preview-trace'), + ]) + + if (typeof document !== 'undefined' && document.visibilityState !== 'visible') { + return { + error: + 'Tab must be foreground for playback (requestAnimationFrame is throttled in background tabs). Focus the editor tab and retry.', + } + } + + const transitions = useTransitionsStore.getState().transitions + const itemsByTrackId = useItemsStore.getState().itemsByTrackId + const tracks = useTimelineStore.getState().tracks + const clipMap = new Map() + for (const track of tracks) { + for (const item of itemsByTrackId[track.id] ?? []) clipMap.set(item.id, item) + } + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const windows = resolveTransitionWindows(transitions, clipMap as any) + if (windows.length === 0) return { error: 'No transitions in this project.' } + + const playback = usePlaybackStore.getState() + const ref = targetFrame ?? playback.currentFrame + // Prefer a window containing the reference frame, else the nearest by start. + const containing = windows.find((w) => ref >= w.startFrame && ref < w.endFrame) + const win = + containing ?? + windows + .slice() + .sort((a, b) => Math.abs(a.startFrame - ref) - Math.abs(b.startFrame - ref))[0]! + + const fps = useTimelineStore.getState().fps || 30 + const runUpFrames = Math.round(fps * 1.5) + const startFrame = Math.max(0, win.startFrame - runUpFrames) + const endTarget = win.endFrame + Math.round(fps * 0.5) + + const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)) + + usePlaybackStore.getState().pause() + usePlaybackStore.getState().setCurrentFrame(startFrame) + await sleep(400) + + trace.startPreviewTrace() + usePlaybackStore.getState().play() + + // Play until past the window end, with a hard timeout. + const deadline = Date.now() + 12000 + // eslint-disable-next-line no-constant-condition + while (true) { + await sleep(80) + const cur = usePlaybackStore.getState().currentFrame + if (cur >= endTarget || !usePlaybackStore.getState().isPlaying) break + if (Date.now() > deadline) break + } + usePlaybackStore.getState().pause() + await sleep(150) + trace.stopPreviewTrace() + + const events = trace.getPreviewTraceEvents() + // Stash raw events so previewTrace.events() can return them post-capture. + ;(window as unknown as { __PREVIEW_TRACE_LAST__?: unknown }).__PREVIEW_TRACE_LAST__ = [ + ...events, + ] + + const analysis = trace.analyzePreviewTrace(events, { + startFrame: win.startFrame, + cutPoint: win.cutPoint, + endFrame: win.endFrame, + leftClipId: win.leftClip.id.slice(0, 8), + rightClipId: win.rightClip.id.slice(0, 8), + }) + return { + transition: { + presentation: win.transition.presentation, + startFrame: win.startFrame, + cutPoint: win.cutPoint, + endFrame: win.endFrame, + leftClipId: win.leftClip.id.slice(0, 8), + leftReversed: win.leftClip.type === 'video' && win.leftClip.isReversed === true, + rightClipId: win.rightClip.id.slice(0, 8), + rightReversed: win.rightClip.type === 'video' && win.rightClip.isReversed === true, + }, + ...analysis, + } + }, + prewarmCache: () => { const cache = (window as unknown as Record).__PREWARM_CACHE__ if (!cache || !(cache instanceof Map)) return null diff --git a/src/features/export/utils/canvas-item-renderer/video.ts b/src/features/export/utils/canvas-item-renderer/video.ts index a64a20ee0..70279338a 100644 --- a/src/features/export/utils/canvas-item-renderer/video.ts +++ b/src/features/export/utils/canvas-item-renderer/video.ts @@ -33,6 +33,7 @@ import { drawContainedMediaSource, hasCropFeather, } from './media-draw' +import { isPreviewTraceEnabled, recordRenderTrace } from '@/shared/logging/preview-trace' export function getTier2VideoFrameToleranceSeconds(sourceFps: number): number { const normalizedSourceFps = Number.isFinite(sourceFps) && sourceFps > 0 ? sourceFps : 30 @@ -229,6 +230,19 @@ export async function renderVideoItem( }) const hasDomVideo = domVideoDecision.hasReadyDomVideo + // DEV diagnostics: record which transition participants the renderer actually + // composites per frame. Tree-shaken from prod; no-op unless a trace is running. + if (import.meta.env.DEV && rctx.isRenderingTransition && isPreviewTraceEnabled()) { + recordRenderTrace({ + f: frame, + id: item.id.slice(0, 8), + rev: item.isReversed === true, + src: Math.round(sourceTime * 100) / 100, + hasDom: !!domVideo, + useMb: useMediabunny.has(item.id), + }) + } + // === TRY DOM VIDEO ELEMENT (zero-copy playback path) === // During playback, the Player's