Redesign live subtitles: roll-up display + provisional simultaneous translation

chrome-extension/src/background/translate-session.ts

@@ -5,6 +5,8 @@ import type { ProviderCredential, TranslationProvider } from '@extension/transla
 interface TranslateUnit {
   id: string;
   html: string;
+  /** One-shot text (e.g. a live ASR partial): bypass the translation cache. */
+  transient?: boolean;
 }
 
 interface TranslateBatchRequest {
@@ -259,12 +261,13 @@ class TranslateSession {
     if (this.closed || this.controller.signal.aborted) return;
 
     const { provider, sourceLang, targetLang } = ctx;
-    const cacheKeys = batch.map(u => cacheKey(provider.id, sourceLang, targetLang, u.html));
-    const cached = await translationCacheStorage.getMany(cacheKeys);
+    const cacheable = batch.filter(u => !u.transient);
+    const cacheKeys = cacheable.map(u => cacheKey(provider.id, sourceLang, targetLang, u.html));
+    const cached = cacheKeys.length > 0 ? await translationCacheStorage.getMany(cacheKeys) : {};
 
     const cachedHits: Array<{ id: string; html: string }> = [];
-    const misses: TranslateUnit[] = [];
-    batch.forEach((u, i) => {
+    const misses: TranslateUnit[] = batch.filter(u => u.transient);
+    cacheable.forEach((u, i) => {
       const hit = cached[cacheKeys[i]];
       if (typeof hit === 'string') cachedHits.push({ id: u.id, html: hit });
       else misses.push(u);
@@ -343,9 +346,10 @@ class TranslateSession {
         const results = units.map((u, i) => ({ id: u.id, html: restored[i] ?? '' }));
         const toCache: Record<string, string> = {};
         units.forEach((u, i) => {
+          if (u.transient) return;
           toCache[cacheKey(provider.id, sourceLang, targetLang, u.html)] = restored[i] ?? '';
         });
-        void translationCacheStorage.putMany(toCache);
+        if (Object.keys(toCache).length > 0) void translationCacheStorage.putMany(toCache);
 
         this.sink({
           type: 'TR_TRANSLATE_RESULT',

pages/content/src/matches/all/youtube/index.ts

@@ -15,6 +15,7 @@
 
 import { appendSequentialCue, createAsrCue, replaceOverlappingCues } from './asr-cues.js';
 import { currentVideoId, fetchCues } from './cues.js';
+import { sliceForLiveTranslation } from './live-text.js';
 import { createOverlay, updateOverlayStyle } from './overlay.js';
 import { createPlayerButton } from './player-button.js';
 import { startCueTranslation, startIncrementalCueTranslation } from './translate.js';
@@ -73,6 +74,17 @@ interface ActiveAsrSession {
    *  timeline was re-anchored and overlapping cues must be replaced. */
   lastAsrStreamId: string | null;
   autoReconnects: number;
+  /** Live provisional translation of the newest utterance (see below). */
+  partialGen: number;
+  /** Negative translation-unit id of the last partial slice sent, if any. */
+  lastPartialUnitId: number | null;
+  /** Cue the last partial slice belongs to once committed; null while streaming. */
+  partialUnitCueId: number | null;
+  lastPartialText: string;
+  lastPartialSentText: string;
+  lastPartialSentAt: number;
+  partialTranslateTimer: number;
+  liveTranslation: string;
   detectedLanguage?: string;
   error?: string;
 }
@@ -106,6 +118,14 @@ let videoBindTimer = 0;
  *  (network hiccups, ElevenLabs per-session time limits). */
 const MAX_ASR_AUTO_RECONNECTS = 3;
 
+/** Live provisional translation: ElevenLabs only commits an utterance after a
+ *  VAD silence, so waiting for the commit leaves the translation line a full
+ *  sentence behind the audio. Instead, a bounded slice of the growing partial
+ *  is re-translated on this throttle and shown until the committed cue's real
+ *  translation streams in. */
+const PARTIAL_TRANSLATE_INTERVAL_MS = 1200;
+const PARTIAL_TRANSLATE_MAX_CHARS = 280;
+
 const GLOBAL_KEY = '__openlingoYouTubeSubtitles';
 
 type YouTubeSubtitlesWindow = Window & { [GLOBAL_KEY]?: YouTubeSubtitlesGlobal };
@@ -146,11 +166,35 @@ const cancelActive = (): void => {
 
 const cancelActiveAsr = (): void => {
   if (!activeAsr) return;
+  if (activeAsr.partialTranslateTimer) window.clearTimeout(activeAsr.partialTranslateTimer);
   activeAsr.translateSession.cancel();
   activeAsr.overlay.destroy();
   activeAsr = null;
 };
 
+/** Send the freshest partial slice for provisional translation, throttled. */
+const sendPartialTranslation = (session: ActiveAsrSession): void => {
+  const slice = sliceForLiveTranslation(session.lastPartialText, PARTIAL_TRANSLATE_MAX_CHARS);
+  if (slice.length < 2 || slice === session.lastPartialSentText) return;
+  const elapsed = Date.now() - session.lastPartialSentAt;
+  if (elapsed < PARTIAL_TRANSLATE_INTERVAL_MS) {
+    if (session.partialTranslateTimer) return;
+    session.partialTranslateTimer = window.setTimeout(() => {
+      session.partialTranslateTimer = 0;
+      if (activeAsr === session) sendPartialTranslation(session);
+    }, PARTIAL_TRANSLATE_INTERVAL_MS - elapsed);
+    return;
+  }
+  session.partialGen += 1;
+  session.lastPartialUnitId = -session.partialGen;
+  session.partialUnitCueId = null;
+  session.lastPartialSentText = slice;
+  session.lastPartialSentAt = Date.now();
+  session.translateSession.append([{ id: session.lastPartialUnitId, startMs: 0, endMs: 0, text: slice }], {
+    transient: true,
+  });
+};
+
 const featureOn = (settings: VideoSubtitlesSettingsType): boolean =>
   settings.enabled && settings.youtubeAutoEnable && settings.youtubeTranslate;
 
@@ -264,10 +308,30 @@ const ensureAsrSession = (videoId: string): ActiveAsrSession | null => {
     nextCueId: 0,
     lastAsrStreamId: null,
     autoReconnects: 0,
+    partialGen: 0,
+    lastPartialUnitId: null,
+    partialUnitCueId: null,
+    lastPartialText: '',
+    lastPartialSentText: '',
+    lastPartialSentAt: 0,
+    partialTranslateTimer: 0,
+    liveTranslation: '',
   };
   activeAsr = session;
   translateSession.onUpdate(() => {
     if (activeAsr !== session) return;
+    if (session.lastPartialUnitId !== null) {
+      if (session.partialUnitCueId !== null && translateSession.isFinal(session.partialUnitCueId)) {
+        // The committed cue's real translation arrived; retire the provisional.
+        session.liveTranslation = '';
+        session.lastPartialUnitId = null;
+        session.partialUnitCueId = null;
+      } else {
+        const provisional = session.translations.get(session.lastPartialUnitId);
+        if (provisional !== undefined) session.liveTranslation = provisional.trim();
+      }
+    }
+    overlay.setLiveTranslation(session.liveTranslation, session.partialUnitCueId);
     overlay.setCues(session.cues, session.translations);
     overlay.refresh();
     setButtonStatus(statusForActive(), {
@@ -301,7 +365,20 @@ const appendAsrCue = (event: AsrCommittedEvent): void => {
   if (!session || activeAsr !== session) return;
   const cue = cueFromAsrEvent(session, event);
   session.overlay.setPartialText('');
+  // The utterance is final; pending partial-translation work is now stale.
+  session.lastPartialText = '';
+  session.lastPartialSentText = '';
+  if (session.partialTranslateTimer) {
+    window.clearTimeout(session.partialTranslateTimer);
+    session.partialTranslateTimer = 0;
+  }
   if (!cue) return;
+  // The provisional translation now stands in for this cue until its real
+  // translation streams in.
+  if (session.lastPartialUnitId !== null) {
+    session.partialUnitCueId = cue.id;
+    session.overlay.setLiveTranslation(session.liveTranslation, session.partialUnitCueId);
+  }
 
   session.cues =
     session.lastAsrStreamId === null || session.lastAsrStreamId === event.sessionId
@@ -516,7 +593,9 @@ const onRuntimeMessage = (
     if (partial.videoId !== currentVideoId()) return;
     const session = ensureAsrSession(partial.videoId);
     if (!session) return;
+    session.lastPartialText = partial.text;
     session.overlay.setPartialText(partial.text);
+    sendPartialTranslation(session);
     setButtonStatus('listening', { statusText: 'Listening with ElevenLabs', errorMessage: '' });
     return;
   }

pages/content/src/matches/all/youtube/live-text.test.ts

@@ -0,0 +1,64 @@
+import { joinTranscriptTail, sliceForLiveTranslation, tailOnWordBoundary } from './live-text.js';
+import { describe, expect, it } from 'vitest';
+
+describe('tailOnWordBoundary', () => {
+  it('returns short text untouched', () => {
+    expect(tailOnWordBoundary('hello world', 40)).toBe('hello world');
+  });
+
+  it('keeps the tail and drops the leading partial word', () => {
+    const text = 'the quick brown fox jumps over the lazy dog';
+    const tail = tailOnWordBoundary(text, 20);
+    expect(tail.length).toBeLessThanOrEqual(20);
+    expect(text.endsWith(tail)).toBe(true);
+    // Starts on a word boundary, not mid-word.
+    expect(text[text.length - tail.length - 1]).toBe(' ');
+  });
+
+  it('hard-cuts spaceless CJK text', () => {
+    const text = '这是一个没有空格的很长的中文句子需要被截断处理';
+    const tail = tailOnWordBoundary(text, 10);
+    expect(tail).toBe(text.slice(-10));
+  });
+
+  it('trims surrounding whitespace', () => {
+    expect(tailOnWordBoundary('  hi  ', 40)).toBe('hi');
+  });
+});
+
+describe('joinTranscriptTail', () => {
+  it('joins previous and current with a space', () => {
+    expect(joinTranscriptTail('first sentence.', 'second part', 100)).toBe('first sentence. second part');
+  });
+
+  it('handles empty previous or current', () => {
+    expect(joinTranscriptTail('', 'only current', 100)).toBe('only current');
+    expect(joinTranscriptTail('only previous', '', 100)).toBe('only previous');
+  });
+
+  it('bounds the joined stream to the freshest tail', () => {
+    const prev = 'a'.repeat(300);
+    const joined = joinTranscriptTail(prev, 'fresh words here', 40);
+    expect(joined.length).toBeLessThanOrEqual(40);
+    expect(joined.endsWith('fresh words here')).toBe(true);
+  });
+});
+
+describe('sliceForLiveTranslation', () => {
+  it('returns short partials unchanged', () => {
+    expect(sliceForLiveTranslation('short partial', 100)).toBe('short partial');
+  });
+
+  it('restarts after a sentence break when the tail was cut', () => {
+    const partial = `${'x'.repeat(100)} end of old sentence. This new sentence should be kept intact for translation`;
+    const slice = sliceForLiveTranslation(partial, 90);
+    expect(slice).toBe('This new sentence should be kept intact for translation');
+  });
+
+  it('falls back to the word-boundary tail when no sentence break exists', () => {
+    const partial = 'word '.repeat(60).trim();
+    const slice = sliceForLiveTranslation(partial, 50);
+    expect(slice.length).toBeLessThanOrEqual(50);
+    expect(slice.startsWith('word')).toBe(true);
+  });
+});

pages/content/src/matches/all/youtube/live-text.ts

@@ -0,0 +1,54 @@
+/**
+ * Pure text helpers for the live (ASR) roll-up subtitle display.
+ *
+ * The live overlay renders a continuous transcript tail rather than isolated
+ * cues, so everything here is a "keep the freshest end of a growing string"
+ * operation: bounded tails that avoid cutting words mid-way, transcript joins
+ * across the previous utterance, and picking a bounded slice of a partial
+ * hypothesis for provisional translation.
+ */
+
+const SENTENCE_END_RE = /[.!?。！？…]/;
+
+/** Last `maxChars` of `text`, preferring to start after a whitespace so the
+ *  first visible word is whole. CJK has no spaces; fall back to a hard cut. */
+const tailOnWordBoundary = (text: string, maxChars: number): string => {
+  const trimmed = text.trim();
+  if (trimmed.length <= maxChars) return trimmed;
+  const slice = trimmed.slice(trimmed.length - maxChars);
+  const firstSpace = slice.search(/\s/);
+  if (firstSpace > -1 && firstSpace < Math.min(24, slice.length - 1)) {
+    return slice.slice(firstSpace + 1).trimStart();
+  }
+  return slice;
+};
+
+/** Previous utterance + current utterance as one reading stream, bounded so
+ *  per-frame layout stays cheap. The rolling window only shows the tail. */
+const joinTranscriptTail = (previous: string, current: string, maxChars: number): string => {
+  const prev = previous.trim();
+  const cur = current.trim();
+  const joined = prev && cur ? `${prev} ${cur}` : prev || cur;
+  return tailOnWordBoundary(joined, maxChars);
+};
+
+/** Bounded slice of a growing ASR partial to send for provisional translation.
+ *  When the tail had to be cut, prefer restarting after the last sentence break
+ *  in its first half so the provider sees complete sentences where possible. */
+const sliceForLiveTranslation = (partial: string, maxChars: number): string => {
+  const trimmed = partial.trim();
+  const tail = tailOnWordBoundary(trimmed, maxChars);
+  if (tail.length === trimmed.length) return tail;
+  const head = tail.slice(0, Math.floor(tail.length / 2));
+  let cutIndex = -1;
+  for (let i = 0; i < head.length; i++) {
+    if (SENTENCE_END_RE.test(head[i])) cutIndex = i;
+  }
+  if (cutIndex > -1) {
+    const rest = tail.slice(cutIndex + 1).trim();
+    if (rest) return rest;
+  }
+  return tail;
+};
+
+export { joinTranscriptTail, sliceForLiveTranslation, tailOnWordBoundary };