diff --git a/CHANGELOG.md b/CHANGELOG.md index 8ff4e0ed..9161bd85 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,8 @@ the frozen-backend fallback mirror it for their toolchains. - The backend binds its port immediately and reports startup progress live — `/health` answers 503-with-step and a new `/startup/progress` endpoint lists every step while PyTorch, API routes, and database migrations load in the background, so "starting at step X" is never mistakable for "dead"; the desktop splash narrates each step (#1550) ### Added +- Default-engine dubbing now synthesizes several segments per forward pass instead of one call per line — the width follows the host's device headroom (1 on CPU and low-VRAM cards, up to 8), `OMNIVOICE_DUB_BATCH_WIDTH` overrides it, and engines without native batching keep the single-segment path (#1594) +- `/ws/tts` now reports real time-to-first-audio, and its RTF measures synthesis alone so a slow client can't inflate it (#1594) - The locally cached AudioSeal watermark generator warms on a background thread ~35s after boot (`OMNIVOICE_PRELOAD_WATERMARK=0` opts out; explicitly setting `=1` may download it), so the first synthesis no longer serializes the audioseal import + model load inline — measured at ~42s on a cold filesystem, 3s short of a 90s client timeout (#1576) — thanks @paoloantinori! - Voices you've cloned stay "warm" across restarts — encoded references now persist to disk (~10 KB each), so the first generation of a session skips the re-encode and any transcription pass; `OMNIVOICE_PROMPT_DISK_CACHE=0` opts out (#1565) - Optional FlashInfer acceleration for the default engine on CUDA (`OMNIVOICE_FLASHINFER=1`, ~2.2x measured) — needs the optional `flashinfer-python` package; missing package or kernel failure logs why and falls back to the standard path (#1565) @@ -34,6 +36,7 @@ the frozen-backend fallback mirror it for their toolchains. - The OmniVoice guide now covers combining style attributes with a reference clip (consistent instruct stabilizes cloning; the reference wins conflicts), inline pronunciation control (pinyin / CMU phonemes), and corrects the claim that the default engine can't do voice design — it can, from attributes (#1565) ### Fixed +- Re-mixing a dub no longer decodes, rewrites, and re-reads every cached segment — same-rate cached audio is reused directly (and rejected if truncated), switching timing modes can't reuse slot-truncated audio as natural-rate, and RVC respects natural-rate modes (#1594) - PocketTTS French works again — pocket-tts only ships a 24-layer French model and rejected the name the sidecar asked for, so every French request failed at model load; French now always loads `french_24l` (#1613) — thanks @paoloantinori! - Installing IndexTTS 2.5 no longer fails claiming an interrupted download — the weights repo ships `config.yaml` and VoiceStudio demanded a `config_v2_5.yaml` that exists in no upstream release; both names are accepted, so a hand-renamed checkout keeps working (#1611) — thanks @zuiaiyutu! - IndexTTS 2.5 no longer has long-text generation killed at 60 seconds — the sidecar now proves it is alive every 5 seconds while `infer()` runs, and its deadline rises to 900s (`OMNIVOICE_INDEXTTS_RECV_TIMEOUT_S`) (#1611) — thanks @zuiaiyutu! diff --git a/backend/api/routers/batch.py b/backend/api/routers/batch.py index dc04c902..67df9dfa 100644 --- a/backend/api/routers/batch.py +++ b/backend/api/routers/batch.py @@ -103,6 +103,75 @@ def _set_progress(job, stage, percent=0, **extra): job["progress"] = {"stage": stage, "percent": percent, **extra} +#: Override for the native dub batch width. Set to 1 to disable batching. +BATCH_WIDTH_ENV = "OMNIVOICE_DUB_BATCH_WIDTH" + +#: Hard ceiling on the override — a batch this wide is already amortizing +#: almost all of the per-call setup, and beyond it the failure mode is an OOM +#: that costs more than the saving. +_MAX_BATCH_WIDTH = 16 + + +def _native_batch_width(backend) -> int: + """How many segments to render in one native batch on THIS host. + + A native batch widens the forward pass, so the width cannot be a constant. + The default engine declares ``min_vram_gb = 6.0`` for a SINGLE job; an + unconditional 8-wide batch would OOM the 4-8 GB CUDA cards and the MPS + Macs where the per-segment path succeeds today — turning a throughput + optimization into a regression on exactly the hardware that already + struggles (#1616 is a 4 GB card reporting capacity failures). Default + behaviour must not get riskier on a host, so the width is derived from + measured headroom and falls back to 1 (no batching) when unknown. + + CPU hosts get 1: batching there buys no kernel amortization and only + multiplies peak RAM. + """ + override = os.environ.get(BATCH_WIDTH_ENV, "").strip() + if override: + try: + return max(1, min(_MAX_BATCH_WIDTH, int(override))) + except (TypeError, ValueError): + logger.warning( + "%s=%r is not an integer — deriving the batch width from the host instead.", + BATCH_WIDTH_ENV, override, + ) + try: + from core.device_caps import detect_host_caps + caps = detect_host_caps() + except Exception: # noqa: BLE001 — an unprobeable host takes the safe path + return 1 + if caps.family == "cpu" or not caps.vram_gb: + return 1 + headroom = caps.vram_gb - float(getattr(backend, "min_vram_gb", 0.0) or 0.0) + if headroom < 2.0: + return 1 + if headroom < 6.0: + return 2 + if headroom < 12.0: + return 4 + return 8 + + +def _batch_timeout_s(texts: list[str], backend) -> float: + """Execution budget for one native batch. + + Not the sum of the per-item budgets: ``generate_timeout_s`` returns a + floor (300s GPU / 600s CPU) plus per-length overage, so summing it across + eight items yields a ~2400s budget — and a wedged batch would hold a + GPU-pool worker for forty minutes before the reset this file depends on + (#730). One floor covers wedge detection for the whole call; only the + length-driven overage is genuinely additive. + """ + from services.model_manager import generate_timeout_s + + floor = generate_timeout_s("", engine=backend) + overage = sum( + max(0.0, generate_timeout_s(text, engine=backend) - floor) for text in texts + ) + return floor + overage + + async def _run_batch_pipeline(job_id: str, job: dict): """Full batch dub pipeline: extract → transcribe → translate → generate → mix → export.""" import subprocess @@ -279,6 +348,111 @@ def _translate_batch(segs, src, tgt): full_audio = torch.zeros(1, total_samples) total_segs = len(translated_segments) + # Native engines can amortize encoder/decoder setup across a small + # batch. Keep the adapter seam optional: engines without a real batch + # implementation inherit TTSBackend.generate_batch(), which preserves + # the established one-segment behavior below. + from services.tts_backend import TTSBackend + batched_audio: dict[int, torch.Tensor] = {} + has_native_batch = type(backend).generate_batch is not TTSBackend.generate_batch + if has_native_batch: + from services.text_normalization import normalize_for_tts + + batch_ref_audio = None + batch_ref_text = None + if job.get("voice_id"): + from core.db import db_conn + from core.config import VOICES_DIR as _VD + with db_conn() as conn: + row = conn.execute( + "SELECT * FROM voice_profiles WHERE id=?", + (job["voice_id"],), + ).fetchone() + if row: + if row["is_locked"] and row["locked_audio_path"]: + batch_ref_audio = os.path.join(_VD, row["locked_audio_path"]) + elif row["ref_audio_path"]: + batch_ref_audio = os.path.join(_VD, row["ref_audio_path"]) + batch_ref_text = row["ref_text"] + + batch_width = _native_batch_width(backend) + + async def _prefetch_batch(first_index: int) -> None: + """Render the batch beginning at ``first_index`` into + ``batched_audio``. + + Rendered on demand rather than prerendering the whole track: + the tensors are popped as they are placed, so peak host memory + is one batch instead of every segment of the language — and + the progress bar tracks placement instead of running to the + end and restarting at segment 1. + """ + if job["status"] == "cancelled": + return + batch_rows = [] + index = first_index + while index < total_segs and len(batch_rows) < batch_width: + seg = translated_segments[index] + if (seg.get("end", 0) - seg.get("start", 0) > 0.05 + and seg.get("text", "").strip()): + batch_rows.append((index, seg)) + index += 1 + if len(batch_rows) < 2: + return # nothing to amortize — the per-segment path is equal + batch_indices = [index for index, _ in batch_rows] + batch_texts = [ + normalize_for_tts(row.get("text", "").strip(), target_lang) + for _, row in batch_rows + ] + batch_durations = [ + row.get("end", 0) - row.get("start", 0) + for _, row in batch_rows + ] + + def _render_native_batch(): + generated = backend.generate_batch( + batch_texts, + language=target_lang, + ref_audio=batch_ref_audio, + ref_text=batch_ref_text, + duration=batch_durations, + num_step=16, + guidance_scale=2.0, + speed=1.0, + denoise=True, + postprocess_output=True, + ) + if len(generated) != len(batch_indices): + raise RuntimeError( + f"native batch returned {len(generated)} outputs for " + f"{len(batch_indices)} segments" + ) + rendered = [] + for audio_out in generated: + if not getattr(backend, "applies_own_mastering", False): + audio_out = apply_mastering(audio_out, sample_rate=sr) + rendered.append(normalize_audio(audio_out, target_dBFS=-2.0)) + return rendered + + try: + rendered = await run_on_gpu_pool_guarded( + _render_native_batch, + what="Batch generate", + timeout=_batch_timeout_s(batch_texts, backend), + ) + batched_audio.update(zip(batch_indices, rendered)) + except TimeoutError: + # Do not immediately queue the same expensive work again: + # the timed-out pool task may still be holding the device. + raise + except Exception as e: + logger.warning( + "Native TTS batch failed for segments %s-%s; falling back per segment: %s", + batch_indices[0] + 1, + batch_indices[-1] + 1, + e, + ) + for i, seg in enumerate(translated_segments): if job["status"] == "cancelled": return @@ -356,10 +530,15 @@ def _gen(text=seg_text, lang=target_lang, dur=seg_duration): # Budget is the shared length-scaled one (#1190): a long segment # on CPU-class hardware no longer dies on the flat 300s. from services.model_manager import generate_timeout_s - audio_tensor = await run_on_gpu_pool_guarded( - _gen, what="Batch generate", - timeout=generate_timeout_s(seg_text, engine=backend), - ) + if has_native_batch and i not in batched_audio: + await _prefetch_batch(i) + if i in batched_audio: + audio_tensor = batched_audio.pop(i) + else: + audio_tensor = await run_on_gpu_pool_guarded( + _gen, what="Batch generate", + timeout=generate_timeout_s(seg_text, engine=backend), + ) # Fit to slot target_samples_seg = int(seg_duration * sr) diff --git a/backend/api/routers/dub_generate.py b/backend/api/routers/dub_generate.py index 442e0bc2..314e9eec 100644 --- a/backend/api/routers/dub_generate.py +++ b/backend/api/routers/dub_generate.py @@ -1,6 +1,7 @@ import os import re import json +import struct import logging import time import asyncio @@ -80,6 +81,62 @@ def _prepare_oom_retry(error: Exception, *, execution_target: str) -> bool: return True +def _cached_payload_intact(path: str, info) -> bool: + """Cheap truth check on a cached WAV whose header we are about to trust. + + The natural-rate fast path hands the mixer a PATH instead of decoded + audio, so a cache whose header reads fine but whose payload is truncated + would only fail later, during assembly — after the timing plan (Smart Fit, + video stretch) had been computed from the header's frame count. The plan + would then describe audio that no longer exists and the segment would be + replaced by slot-length silence, leaving the persisted video plan and the + rendered track disagreeing. + + Comparing the declared frame count against the physical ``data`` chunk + catches that without decoding: a truncated file cannot hold the samples + its header claims. Anything failing here falls through to the decoding path, which + already degrades to a warning plus silence. Formats with no fixed + bits-per-sample (compressed caches) are left to the decoder as before. + """ + try: + bits = int(getattr(info, "bits_per_sample", 0) or 0) + frames = int(getattr(info, "num_frames", 0) or 0) + channels = int(getattr(info, "num_channels", 0) or 0) + if bits <= 0 or frames <= 0 or channels <= 0: + # Undecidable metadata fails CLOSED (review on #1620): these caches + # are PCM WAVs this module wrote itself, so anything else is + # unexpected — and the decode path this falls through to handles + # every format the fast path would have. + return False + payload = frames * channels * (bits // 8) + if payload <= 0: + return False + + # A WAV may carry JUNK/LIST metadata before data, so its header is not + # necessarily 44 bytes. Locate the data chunk instead of counting + # metadata as audio; otherwise an extended header can mask truncation. + file_size = os.path.getsize(path) + with open(path, "rb") as wav: + header = wav.read(12) + if len(header) != 12 or header[:4] != b"RIFF" or header[8:12] != b"WAVE": + return False + offset = 12 + while offset + 8 <= file_size: + wav.seek(offset) + chunk_id = wav.read(4) + chunk_size_raw = wav.read(4) + if len(chunk_id) != 4 or len(chunk_size_raw) != 4: + return False + chunk_size = struct.unpack("= payload and file_size >= data_offset + payload + offset = data_offset + chunk_size + (chunk_size % 2) + return False + except Exception: # noqa: BLE001 — an unstattable cache is the decoder's problem + return False + + def _underrun_min_rate() -> float: """Floor for the underrun fill (audio slowed toward its slot, never below this rate). Default 0.85 stays natural-sounding; OMNIVOICE_UNDERRUN_MIN_RATE=1.0 @@ -593,11 +650,11 @@ def _write_memmap_wav_atomic(target_path: str, samples, sample_rate: int) -> Non voice_match = (req.voice_match or "per_line").lower() _consistent_ref_memo: dict = {} remote_audio: dict[int, str] = {} - # Strategy-transition guard: smart_fit re-mixes the *natural-rate* - # per-segment WAVs from disk. If the previous run used strict_slot, - # the on-disk WAVs are slot-squeezed ("slotted") — reusing them would - # double-compress. Force one full regen; afterwards seg_wav_kind is - # "natural" and partial regen / fit-only re-mix (regen_only=[]) work. + # Strategy-transition guard: concise, stretch_video and smart_fit all + # re-mix *natural-rate* per-segment WAVs. If the previous run used + # strict_slot, the on-disk WAVs are slot-squeezed ("slotted") — the + # missing tails cannot be recovered by a re-mix. Force one full regen; + # afterwards partial regen / fit-only re-mix (regen_only=[]) is safe. # Jobs predating this field have unknown kind → also regen once. # P1.3: the kind is per-track now (each language renders under its own # strategy); the flat job["seg_wav_kind"] is only consulted for jobs @@ -608,7 +665,7 @@ def _write_memmap_wav_atomic(target_path: str, samples, sample_rate: int) -> Non _wav_kind = ( _kind_map.get(lang_code) if isinstance(_kind_map, dict) else job.get("seg_wav_kind") ) - if strategy == "smart_fit" and regen_only is not None and _wav_kind != "natural": + if strategy != "strict_slot" and regen_only is not None and _wav_kind != "natural": regen_only = None # Manifest: stable segment id per current index. Per-segment WAVs are # named by stable id (dub_seg_path) so regen reuses the right audio after @@ -759,15 +816,38 @@ def _write_memmap_wav_atomic(target_path: str, samples, sample_rate: int) -> Non if os.path.exists(seg_wav_path): try: _t_cache_0 = time.perf_counter() + # Natural-rate caches are already the exact assembly + # input. Keep the durable path in the manifest so the + # mixer decodes it once; the old path decoded here, + # wrote an identical mix_ scratch WAV, then decoded + # that copy again. Header-only inspection preserves + # the resample fallback for caches made by an engine + # with a different sample rate. + if strategy != "strict_slot": + try: + cached_info = torchaudio.info(seg_wav_path) + except Exception: + cached_info = None + if ( + cached_info is not None + and int(cached_info.sample_rate) == int(backend.sample_rate) + and _cached_payload_intact(seg_wav_path, cached_info) + ): + all_segment_wavs.append( + (seg.start, seg.end, seg_wav_path, backend.sample_rate) + ) + sync_scores.append(getattr(seg, 'sync_ratio', None) or 1.0) + _t_cache += time.perf_counter() - _t_cache_0 + continue + cached_wav, cached_sr = torchaudio.load(seg_wav_path) if cached_sr != backend.sample_rate: import torchaudio.functional as AF cached_wav = AF.resample(cached_wav, cached_sr, backend.sample_rate) - # Pad/trim to slot — except smart_fit, whose mix - # loop needs the natural-rate length to compute the - # audio/video split (the seg_wav_kind guard above - # guarantees these cached WAVs are natural-rate). - if strategy != "smart_fit": + # strict_slot persists slot-sized buffers. Every other + # strategy consumes natural-rate audio and lets the mix + # loop fit it to the current timeline. + if strategy == "strict_slot": target_samples = int(seg_duration * backend.sample_rate) current_samples = cached_wav.shape[-1] if target_samples > current_samples: @@ -1164,12 +1244,15 @@ def _gen(text, lang, instruct_str, dur_s, nstep, cfg, spd, profile_id, effect_pr if rvc_sr == backend.sample_rate: audio_tensor = rvc_wav - target_samples = int(seg_duration * backend.sample_rate) - current_samples = audio_tensor.shape[-1] - if target_samples > current_samples: - audio_tensor = torch.nn.functional.pad(audio_tensor, (0, target_samples - current_samples)) - elif current_samples > target_samples: - audio_tensor = audio_tensor[..., :target_samples] + if strategy == "strict_slot": + target_samples = int(seg_duration * backend.sample_rate) + current_samples = audio_tensor.shape[-1] + if target_samples > current_samples: + audio_tensor = torch.nn.functional.pad( + audio_tensor, (0, target_samples - current_samples) + ) + elif current_samples > target_samples: + audio_tensor = audio_tensor[..., :target_samples] except Exception as e: yield f"data: {json.dumps({'type': 'warning', 'segment': i, 'message': f'RVC skipped: {str(e)[:120]}'})}\n\n" @@ -1356,7 +1439,21 @@ def _gen(text, lang, instruct_str, dur_s, nstep, cfg, spd, profile_id, effect_pr seg_gain = getattr(seg_ref, "gain", None) if seg_ref is not None else None seg_gain = seg_gain if seg_gain is not None else 1.0 seg_gain = max(0.0, min(2.0, seg_gain)) - wav = _load_entry_wav((start, end, wav_path, sr), sr) + try: + wav = _load_entry_wav((start, end, wav_path, sr), sr) + except Exception as e: + # A WAV header can be readable while its payload is + # truncated. Direct cache reuse deliberately defers the + # decode to assembly, so preserve the old recovery contract + # here: warn and fill this slot with silence instead of + # aborting the entire dub. + warning = { + "type": "warning", + "segment": i, + "message": f"cached seg lost, padding silence: {str(e)[:120]}", + } + yield f"data: {json.dumps(warning)}\n\n" + wav = torch.zeros(1, max(0, int((end - start) * sr))) adjusted = wav * seg_gain if adjusted.ndim == 2 and adjusted.shape[0] > 1: adjusted = adjusted.mean(dim=0, keepdim=True) diff --git a/backend/api/routers/tts_stream.py b/backend/api/routers/tts_stream.py index 32327d5c..3a21ce04 100644 --- a/backend/api/routers/tts_stream.py +++ b/backend/api/routers/tts_stream.py @@ -10,7 +10,8 @@ Protocol: → Client sends JSON: {"text": "...", "voice": "profile_id", ...} ← Server sends binary audio chunks (PCM16 @ 24kHz mono) as generated - ← Server sends JSON: {"type": "done", "duration_s": 4.2, "gen_time_s": 1.1} + ← Server sends JSON: {"type": "done", "duration_s": 4.2, + "gen_time_s": 1.1, "ttfa_ms": 180.0, "rtf": 0.262} ← Server sends JSON: {"type": "error", "detail": "..."} The chunked delivery targets <100ms time-to-first-audio (TTFA) on warm models. @@ -33,6 +34,10 @@ # Smaller chunks = lower latency but more WebSocket overhead. CHUNK_SAMPLES = int(os.environ.get("OMNIVOICE_STREAM_CHUNK", "4800")) +# Module seam for deterministic latency-contract tests. Keep every timing +# sample on the same monotonic clock. +_perf_counter = time.perf_counter + class StreamTTSRequest(BaseModel): """Client request for streaming TTS.""" @@ -85,7 +90,7 @@ async def ws_tts(websocket: WebSocket): }) continue - t0 = time.perf_counter() + t0 = _perf_counter() text = data["text"] # Remote GPU: this socket stays on this machine, and says so. @@ -258,6 +263,11 @@ async def ws_tts(websocket: WebSocket): from services.model_manager import run_on_gpu_pool_guarded def _generate(sentence_text): + # Timed INSIDE the pool worker: the guarded dispatch below + # can queue behind other jobs, and queue wait is not + # synthesis (review on #1620) — under contention it would + # inflate rtf without the engine slowing at all. + _synth_t0 = _perf_counter() from services.audio_dsp import apply_mastering, normalize_audio from services.watermark import mark_synthetic wav = backend.generate(sentence_text, **kw) @@ -279,12 +289,19 @@ def _generate(sentence_text): # watermark._iter_chunks), which is inherent to marking # ultra-short clips, not a coverage gap. wav = mark_synthetic(wav, sr_actual, context="tts_stream.sentence") - return wav, sr_actual + return wav, sr_actual, _perf_counter() - _synth_t0 import torch total_samples = 0 sr = backend.sample_rate started = False + first_audio_at: float | None = None + # Synthesis time only. The wall clock below also carries socket + # delivery and the per-chunk event-loop yields, so deriving RTF + # from it reports "how slow was the client" as if it were engine + # throughput — on a slow consumer that inflates RTF without the + # engine having changed at all. + synth_time = 0.0 for sentence in sentences: # Bounded + pool-reset on hang so a wedged generate can't @@ -294,11 +311,12 @@ def _generate(sentence_text): # Length-scaled budget per sentence (#1190) — the flat 300s # default is gone from every dispatch. from services.model_manager import generate_timeout_s - wav_tensor, sr = await run_on_gpu_pool_guarded( + wav_tensor, sr, sentence_synth_s = await run_on_gpu_pool_guarded( functools.partial(_generate, sentence), what="TTS generate", timeout=generate_timeout_s(sentence, engine=backend), ) + synth_time += sentence_synth_s if not started: # Send metadata after the first generation so @@ -325,25 +343,49 @@ def _generate(sentence_text): end = min(sent_samples + CHUNK_SAMPLES, n_samples) chunk = pcm_bytes[sent_samples * 2: end * 2] await websocket.send_bytes(chunk) + if first_audio_at is None: + # TTFA ends when the first audio bytes have been + # handed to the socket. The previous log used the + # whole-render duration and called it TTFA. + first_audio_at = _perf_counter() sent_samples = end # Yield to event loop between chunks for responsiveness await asyncio.sleep(0) total_samples += n_samples - gen_time = round(time.perf_counter() - t0, 3) + finished_at = _perf_counter() + wall_time_raw = max(0.0, finished_at - t0) + synth_time_raw = max(0.0, synth_time) + gen_time = round(wall_time_raw, 3) duration = round(total_samples / sr, 3) + ttfa_ms = ( + round(max(0.0, first_audio_at - t0) * 1000.0, 1) + if first_audio_at is not None + else None + ) + # RTF is a render metric: synthesis seconds per audio second. + rtf = ( + round(synth_time_raw / (total_samples / sr), 3) + if total_samples > 0 + else None + ) await websocket.send_json({ "type": "done", "duration_s": duration, "gen_time_s": gen_time, + "ttfa_ms": ttfa_ms, + "rtf": rtf, "samples": total_samples, "sample_rate": sr, "engine": backend.id, }) logger.info( - "TTS stream: %.1fs audio in %.1fs (TTFA=%.0fms)", - duration, gen_time, gen_time * 1000, + "TTS stream: %.1fs audio in %.1fs (TTFA=%s, RTF=%s)", + duration, + gen_time, + f"{ttfa_ms:.0f}ms" if ttfa_ms is not None else "n/a", + f"{rtf:.3f}" if rtf is not None else "n/a", ) except Exception as e: diff --git a/backend/services/tts_backend.py b/backend/services/tts_backend.py index cf7fb705..f03c0c5a 100644 --- a/backend/services/tts_backend.py +++ b/backend/services/tts_backend.py @@ -341,6 +341,44 @@ def generate( Engines that don't support this will ignore the parameter. """ + def generate_batch( + self, + texts: list[str], + *, + ref_audio=None, + ref_text=None, + instruct=None, + language=None, + duration=None, + speed=1.0, + **extras, + ) -> list[torch.Tensor]: + """Synthesize several utterances, preserving the single-item contract. + + Engines with a native batch forward pass override this method. The + default keeps every existing adapter correct while giving callers one + stable seam and per-item keyword handling. + """ + if not texts: + return [] + + def _item(value, index): + return value[index] if isinstance(value, list) else value + + return [ + self.generate( + text, + ref_audio=_item(ref_audio, index), + ref_text=_item(ref_text, index), + instruct=_item(instruct, index), + language=_item(language, index), + duration=_item(duration, index), + speed=_item(speed, index), + **extras, + ) + for index, text in enumerate(texts) + ] + # ── Lifecycle (Phase 2 will enforce per-engine overrides) ────────────── # # Today every backend lazily loads its weights on first `generate()` and @@ -717,6 +755,73 @@ def generate(self, text, **kw) -> torch.Tensor: ) return audios[0] + def generate_batch(self, texts: list[str], **kw) -> list[torch.Tensor]: + """Use OmniVoice's native variable-length batch generation. + + Batch callers pass per-item language, duration, speed and reference + lists. Reusable clone prompts are prepared once and handed to the + model together; an incomplete prompt batch falls back to the proven + single-item path instead of changing synthesis semantics. + """ + self._ensure_loaded() + if not texts: + return [] + + def _items(value): + if isinstance(value, list): + return value + return [value] * len(texts) + + def _item_kwargs(index): + return { + key: value[index] if isinstance(value, list) else value + for key, value in kw.items() + } + + ref_audios = _items(kw.get("ref_audio")) + ref_texts = _items(kw.get("ref_text")) + cache_ref = bool(kw.get("cache_ref", True)) + preprocess_prompt = bool(kw.get("preprocess_prompt", True)) + prompts = [] + if any(ref_audios): + for ref_audio, ref_text in zip(ref_audios, ref_texts): + if not ref_audio: + prompts = [] + break + prompt = _get_clone_prompt( + self._model, + ref_audio, + ref_text, + preprocess_prompt, + store=cache_ref, + ) + if prompt is None: + prompts = [] + break + prompts.append(prompt) + + if any(ref_audios) and len(prompts) != len(texts): + return [self.generate(text, **_item_kwargs(i)) + for i, text in enumerate(texts)] + + gen_kw = dict( + language=kw.get("language"), + instruct=kw.get("instruct"), + duration=kw.get("duration"), + speed=kw.get("speed", 1.0), + denoise=kw.get("denoise", True), + postprocess_output=kw.get("postprocess_output", True), + num_step=kw.get("num_step", 16), + guidance_scale=kw.get("guidance_scale", 2.0), + preprocess_prompt=preprocess_prompt, + ) + if prompts: + gen_kw["voice_clone_prompt"] = prompts + else: + gen_kw["ref_audio"] = None + gen_kw["ref_text"] = None + return self._model.generate(text=texts, **gen_kw) + def unload(self) -> None: """Release the OmniVoice model (MM2-02). OmniVoice shares the singleton owned by ``model_manager``, so dropping our local ref isn't enough — we diff --git a/backend/tests/test_tts_backend_lifecycle.py b/backend/tests/test_tts_backend_lifecycle.py index d88ec457..b2a19844 100644 --- a/backend/tests/test_tts_backend_lifecycle.py +++ b/backend/tests/test_tts_backend_lifecycle.py @@ -76,6 +76,35 @@ def test_unload_signature_takes_self_only(self): ) +def test_omnivoice_native_batch_preserves_per_item_controls(): + """The adapter forwards variable-length batch controls to OmniVoice.""" + import torch + + tts = _load_tts_backend_module() + calls = [] + + class _Model: + sampling_rate = 24000 + + def generate(self, **kwargs): + calls.append(kwargs) + return [torch.zeros(1, 12000), torch.zeros(1, 24000)] + + backend = tts.OmniVoiceBackend(model=_Model()) + outputs = backend.generate_batch( + ["short", "long"], + language=["en", "es"], + duration=[0.5, 1.0], + speed=[1.0, 0.8], + ) + + assert [output.shape[-1] for output in outputs] == [12000, 24000] + assert calls[0]["text"] == ["short", "long"] + assert calls[0]["language"] == ["en", "es"] + assert calls[0]["duration"] == [0.5, 1.0] + assert calls[0]["speed"] == [1.0, 0.8] + + class TestUnloadDefaultBehavior: """The default no-op must actually be safe to call.""" @@ -154,4 +183,4 @@ def test_all_subclasses_have_callable_unload(self): assert callable(getattr(cls, "unload", None)), ( f"{cls.__name__} has no callable unload() — even via the " "ABC inheritance. Did someone shadow it?" - ) \ No newline at end of file + ) diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index f1a8a644..5b479d5c 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -17,7 +17,7 @@ Phase 4 · The two bets ▓▓▓▓▓▓▓▓▓▓ 6 / 6 ✅ Phase 5 · Productisation ░░░░░░░░░░ 0 / 5 🚫 demand-driven Design track ▓▓▓▓▓▓▓▓▓░ ongoing · 14 primitives + ~67 migrated inline styles · DubTab/Header/Sidebar/CloneDesignTab drained -Performance track ░░░░░░░░░░ not started +Performance track ▓▓▓░░░░░░░ underway · profiling, preload, isolated engines + cache-remix I/O Feature-magic track ░░░░░░░░░░ not started Quality track ▓▓░░░░░░░░ 12 smoke tests, 10 error messages rewritten ``` @@ -193,16 +193,16 @@ None on the critical path to world-class. All are answers to real demand. | Design-system primitives (14) | ✅ | Full inventory above. | | Migrate remaining inline styles | 🟡 | **Four biggest offenders drained 2026-04-20** — DubTab **93 → 2**, Header **24 → 1**, Sidebar **21 → 8**, CloneDesignTab **33 → 0**. All remaining are genuinely dynamic (per-row `--row-accent` CSS custom props in Sidebar, per-bar `height/animationDelay` in WaveBars, `opacity` computed from index in skeleton rows, `fontSize` by prop). New class systems: `.dub-*` (DubTab), `.hq-col-*/.hq-stats__*/.hq-logo-*` (Header), `.sidebar-tile--*/.sidebar__scroll/.history-*--*` (Sidebar), `.clone-*/.label-row--*` (CloneDesignTab). Drag-hover on `.file-drag` and `.dub-idle-drop` now toggles `.is-dragging` instead of mutating styles via DOM. Remaining 119 across the tail (Launchpad, KeyboardCheatsheet, DubSegmentRow, WaveformTimeline, etc.) — less concentrated, lower-leverage. | -### ⚡ Performance track _(⏳ not started)_ +### ⚡ Performance track _(🟡 underway)_ | Item | Status | Current measurement | |------|:---:|------| -| Batched TTS (8–16 segments per forward pass) | ⏳ | 1 segment per call today. | -| Kill per-segment disk round-trip | ⏳ | `dub_generate.py:132-133` saves + re-reads per segment. | -| Cold start ≤1.5 s to first audible sample | ⏳ | Currently 4+ s on Apple Silicon. | +| Batched TTS (host-derived width per forward pass) | 🟡 | The batch queue feeds OmniVoice's native variable-length forward pass, with the width derived from device headroom (1 on CPU/low-VRAM hosts, up to 8) and overridable via `OMNIVOICE_DUB_BATCH_WIDTH`; adapters without native batching retain the single-segment fallback. | +| Kill per-segment disk round-trip | 🟡 | Long-video assembly stays disk-backed to bound RAM. Unchanged same-rate natural segments now skip the redundant decode → scratch encode → decode cycle; fresh segments still persist once and reload for assembly. | +| Cold start ≤1.5 s to first audible sample | 🟡 | Installed models preload in the background and `scripts/bench_pipeline.py` measures cold/warm synthesis; target is not yet verified. | | Speculative regeneration on hover | ⏳ | — | -| Crash-sandbox engines (subprocess isolation) | ⏳ | Single CUDA OOM still kills server. | -| Interaction budgets (<50 ms UI, <200 ms preview, <4 s first seg) | ⏳ | Not measured. | +| Crash-sandbox engines (subprocess isolation) | 🟡 | Killable sidecar engines and opt-in `omnivoice-subprocess` are live; the default in-process engine can still take down the server on a native crash. | +| Interaction budgets (<50 ms UI, <200 ms preview, <4 s first seg) | 🟡 | `/ws/tts` reports real TTFA, total generation time and RTF; frontend responsiveness instrumentation exists, but no cross-surface budget gate yet. | | Dedicated dev-week per quarter | ⏳ | Cadence not yet booked. | ### ✨ Feature-magic track _(⏳ not started)_ @@ -219,8 +219,8 @@ None on the critical path to world-class. All are answers to real demand. | Item | Status | Notes | |------|:---:|------| -| Every bug ships a regression test | ⏳ | Rule written, not yet enforced in CI. | -| Perf regression budget (≤5 % on fixture clip) | ⏳ | No fixture clip yet. | +| Every bug ships a regression test | 🟡 | Binding repository rule; CI runs backend and frontend suites, but cannot mechanically prove every bug PR added a fail-before test. | +| Perf regression budget (≤5 % on fixture clip) | 🟡 | Deterministic demo media and manual benchmark harnesses exist; no committed device baseline or ≤5 % gate yet. | | Accessibility (keyboard-first, WCAG AA, ARIA live regions) | 🟡 | Focus rings token defined; full audit pending. | | Privacy (zero telemetry by default, per-feature opt-in) | ✅ | Enforced in Settings → Privacy tab. | | Docs updated per phase | 🟡 | STRUCTURE.md, ROADMAP.md, ui/README.md current (research/ + design/ retired 2026-07-12). | diff --git a/docs/performance.md b/docs/performance.md index 344661b2..b27c9603 100644 --- a/docs/performance.md +++ b/docs/performance.md @@ -235,6 +235,22 @@ RAM/VRAM) turns a guessing game into a bisect. Measured results per engine/device — and how to contribute yours — live in [benchmarks.md](benchmarks.md). +Batch dubbing renders several segments in one native forward pass when the +selected engine supports it. The width is derived from the host rather than +fixed, because a wider forward pass needs proportionally more device memory: +CPU hosts and cards with less than ~2 GB of headroom above the engine's +single-job requirement stay at one segment, and the width steps up to 2, 4, +and 8 as headroom allows. `OMNIVOICE_DUB_BATCH_WIDTH` overrides it (1 disables +batching, 16 is the ceiling). Engines without native batching inherit a +compatibility fallback that preserves the one-segment behavior. + +Streaming clients also receive measured latency in the `/ws/tts` terminal +`done` frame: `ttfa_ms` is request-to-first-audio, `gen_time_s` is the +end-to-end wall clock including delivery, and `rtf` is *synthesis* time +divided by generated-audio duration — measured around the render calls only, +so a slow client cannot inflate it. The backend log records the same values, so a slow first chunk is +distinguishable from a fast first chunk followed by a long render. + ## Things that look like knobs but aren't - **Deleting and re-adding a voice** doesn't speed anything up; the reference diff --git a/docs/specs/03-longform-studio-editor.md b/docs/specs/03-longform-studio-editor.md index 6b1baf03..f5a4c3cf 100644 --- a/docs/specs/03-longform-studio-editor.md +++ b/docs/specs/03-longform-studio-editor.md @@ -30,9 +30,9 @@ VoiceStudio today is **asymmetric** across its three longform surfaces: The dub pipeline **already** content-addresses segments so that editing one line re-synthesizes only that line: - **`backend/services/incremental.py`** — `segment_fingerprint(seg)` (52–67) is a sha1 over the **generation inputs that actually affect TTS output**: `_GEN_INPUT_FIELDS = ("text","target_lang","profile_id","instruct","speed","direction","effect_preset")` (line 23). `_canon_value` (32–49) normalizes None/""/missing and int↔float so the **server-parsed view** and the **client-raw view** of the same logical segment hash identically — the root-cause fix for #281 ("1 edit re-dubs all N lines"). `fit_fingerprint(params)` (101–116) hashes the **fit configuration separately and on purpose** (70–76): a fit-knob change must trigger a **re-mix** of already-rendered natural-rate WAVs (`regen_only=[]`), never a re-TTS. `plan_incremental(segments, *, stored_hashes)` (119–157) returns `{stale, fresh, total, fingerprints}`. -- **`backend/api/routers/dub_generate.py`** — honors `regen_only` (113): for a segment **not** in `regen_only` it reloads the cached `dub_seg_path(job_id, seg_id)` (160–197, with a legacy index-name fallback at 162–166) instead of re-running TTS; for stale segments it runs `_gen(...)`. After the loop it **always re-stitches the full `dubbed_{lang}.wav`** (766–770) and persists `job["seg_hashes"]` (498–512) + `seg_order` (127). The `done` SSE ships `seg_hashes`/`seg_num_step` back (840). Strategy-transition guard (122–123) and `seg_wav_kind` (830) keep smart_fit reuse correct. +- **`backend/api/routers/dub_generate.py`** — honors `regen_only`: for a segment **not** in `regen_only` it reuses the cached WAV at `dub_seg_path(job_id, f"{lang_code}_{seg_key}")` (`_seg_lang_path` — the key is language-qualified, so each target language caches its own render) instead of re-running TTS; same-rate natural caches go straight into the bounded-memory assembly manifest, while strict-slot or sample-rate conversion keeps the transformed scratch path. After the loop it **always re-stitches the full `dubbed_{lang}.wav`** and persists `job["seg_hashes"]` + `seg_order`. The `done` SSE ships `seg_hashes`/`seg_num_step` back. The strategy-transition guard and `seg_wav_kind` prevent every natural-rate mode (`concise`, `stretch_video`, `smart_fit`) from reusing destructively slotted audio. - **`tests/test_redub_incremental.py`** — already asserts the contract end-to-end with a mocked TTS engine: `test_edited_line_produces_different_cached_output` (228–281) proves an edited line's cached WAV changes, the **untouched line's cached WAV is reused byte-for-byte** (272–273), TTS ran exactly once (266), and the final track was rebuilt (281). `test_one_edit_marks_exactly_one_segment_stale` (101–118) proves the planner. -- **Per-segment audio + metadata keying.** Audio lives on disk as `{DUB_DIR}/{job_id}/seg_{seg_id}.wav` via `dub_seg_path(job_id, seg_id)` (`backend/core/config.py:54–71`). Metadata lives in the job's `job_data` JSON blob (`dub_history` table, `backend/core/db.py:74–85`), holding `segments`, `seg_order`, `seg_hashes`, `seg_num_step`, `seg_wav_kind`, `dubbed_tracks`, `fit_plans`, `video_stretch_plans`. Stable ids are minted at transcribe time (`s{NNNNN:05x}`, `dub_core.py:600`). Job persistence is `dub_pipeline.get_job`/`save_job`/`put_job` (`backend/services/dub_pipeline.py:158–214`). +- **Per-segment audio + metadata keying.** Audio lives on disk as `{DUB_DIR}/{job_id}/seg_{lang}_{seg_id}.wav` via `_seg_lang_path` → `dub_seg_path(job_id, f"{lang_code}_{seg_key}")` (`backend/core/config.py:54–71` for the path guard); legacy unqualified `seg_{seg_id}.wav` files from single-language-era jobs remain readable through the gated `_legacy_seg_cache_ok` fallback. Metadata lives in the job's `job_data` JSON blob (`dub_history` table, `backend/core/db.py:74–85`), holding `segments`, `seg_order`, `seg_hashes`, `seg_num_step`, `seg_wav_kind`, `dubbed_tracks`, `fit_plans`, `video_stretch_plans`. Stable ids are minted at transcribe time (`s{NNNNN:05x}`, `dub_core.py:600`). Job persistence is `dub_pipeline.get_job`/`save_job`/`put_job` (`backend/services/dub_pipeline.py:158–214`). - **Longform (Audiobook + Stories) share one renderer** `_render_longform_sse` (`backend/api/routers/audiobook.py:403–595`) and one **chapter-level** content-addressed key `chapter_cache_key` (`backend/services/longform_render.py:110–136`), cached under `OUTPUTS_DIR/longform_cache` (`_render_chapter_cached`, `audiobook.py:314–355`). The lexicon is folded into the key (338–341). Resume (`audiobook.py:714–759`) reuses already-rendered chapters because the key is content-based. **But the granularity is the whole chapter** (longform_render.py:117–125) — that is the gap. **Conclusion:** Dub is the reference implementation. Stories already has the *editor UI* but no incremental backend. Audiobook has neither a per-line editor nor sub-chapter incrementality. This spec makes all three behave like a "Studio" by (1) standardizing the editor affordances and (2) pushing the dub-proven `fingerprint → stale → regen_only → re-stitch` loop down into the longform renderer at **span granularity**. @@ -250,7 +250,7 @@ Each slice is independently shippable, continuous-to-main, with its own regressi | **Fingerprint parity drift** (server vs client hash differently → every span looks stale → degrades to full render, the #281 regression class). | Reuse `incremental._canon_value` verbatim; add the server-vs-client parity test first (TDD), exactly as dub did. | | **Parser-twin divergence** (Python `longform_parser` vs JS `longformParser.js` mint different span ids). | Stable ids derived from `(chapter_index, span_index)` are computed identically in both; the existing golden-corpus byte-for-byte test (#27) gates it. | | **Stitch seams** (per-span WAVs stitched may click vs a monolithic chapter render). | `synthesize_chapter` already crossfades spans (50 ms) and hard-concats silences (`services/audiobook.py:121–139`) — the seam behavior is identical whether a span was freshly rendered or cache-loaded (same WAV bytes). | -| **smart_fit-style double-processing on dub** (reusing a slotted WAV under smart_fit). | Already solved: the strategy-transition guard (`dub_generate.py:122–123`) + `seg_wav_kind` (830) force a full regen when the cached WAVs are the wrong kind. No new exposure. | +| **Timing-mode double-processing on dub** (reusing a slotted WAV in a natural-rate mode). | The strategy-transition guard + `seg_wav_kind` force a full regen when `concise`, `stretch_video`, or `smart_fit` sees the wrong cache kind. No new exposure. | | **Existing projects forced to re-render.** | First edit treats unknown-hash spans as stale **once** (correct output), never corrupts cache; no upgrade-time mass re-render. | | **Emotion/style field lands in the wrong fingerprint bucket** (re-TTS when it should re-mix, or vice-versa, wasting compute or shipping stale audio). | Q1 decision pins the bucket per field; the `fit_fingerprint` precedent (incremental.py:70–116) gives both buckets a tested home; 03f ships last, after the field's semantics are known. | diff --git a/tests/test_dub_batch_width_and_budget.py b/tests/test_dub_batch_width_and_budget.py new file mode 100644 index 00000000..5b97e823 --- /dev/null +++ b/tests/test_dub_batch_width_and_budget.py @@ -0,0 +1,195 @@ +"""Native dub batching must not get riskier on the host (#1620 review). + +Batching the default engine 8 segments wide was unconditional. That widens the +forward pass with no capability check on hardware where a SINGLE job already +declares ``min_vram_gb = 6.0`` — so 4-8 GB CUDA cards and MPS Macs could OOM +on a path that succeeds today, one segment at a time. #1616 is a 4 GB card +reporting capacity failures on the un-batched path already. + +The batch budget had the same shape of problem in the other direction: +``generate_timeout_s`` returns a floor plus per-length overage, so summing it +over eight items produced a ~2400s budget and a wedged batch would hold a +GPU-pool worker for forty minutes before the #730 reset. +""" +from __future__ import annotations + +import importlib +import struct + +import pytest + + +@pytest.fixture +def batch(): + return importlib.import_module("api.routers.batch") + + +class _Engine: + min_vram_gb = 6.0 + + +@pytest.fixture +def host(monkeypatch, batch): + """Force a specific detected host.""" + def _set(family, vram_gb): + import core.device_caps as caps_mod + + class _Caps: + pass + + caps = _Caps() + caps.family = family + caps.vram_gb = vram_gb + monkeypatch.setattr(caps_mod, "detect_host_caps", lambda: caps) + return _set + + +@pytest.mark.parametrize("family,vram_gb,expected", [ + ("cpu", 0.0, 1), # batching buys nothing, costs RAM + ("cuda", 4.0, 1), # #1616's card — must not widen + ("cuda", 6.0, 1), # exactly the single-job floor: no headroom + ("mps", 8.0, 2), # 16 GB Mac + ("cuda", 12.0, 4), + ("cuda", 24.0, 8), +]) +def test_the_width_follows_measured_headroom(batch, host, family, vram_gb, expected): + host(family, vram_gb) + assert batch._native_batch_width(_Engine()) == expected + + +def test_an_unprobeable_host_does_not_batch(batch, monkeypatch): + """Unknown capability is not permission to widen the forward pass.""" + import core.device_caps as caps_mod + + def boom(): + raise RuntimeError("probe failed") + + monkeypatch.setattr(caps_mod, "detect_host_caps", boom) + assert batch._native_batch_width(_Engine()) == 1 + + +def test_the_width_is_overridable(batch, host, monkeypatch): + host("cuda", 4.0) + monkeypatch.setenv(batch.BATCH_WIDTH_ENV, "6") + assert batch._native_batch_width(_Engine()) == 6 + + +def test_the_override_is_bounded_and_survives_nonsense(batch, host, monkeypatch): + host("cuda", 24.0) + monkeypatch.setenv(batch.BATCH_WIDTH_ENV, "9999") + assert batch._native_batch_width(_Engine()) == 16 # capped + monkeypatch.setenv(batch.BATCH_WIDTH_ENV, "0") + assert batch._native_batch_width(_Engine()) == 1 # floored + monkeypatch.setenv(batch.BATCH_WIDTH_ENV, "banana") + assert batch._native_batch_width(_Engine()) == 8 # falls back to the host + + +def test_the_batch_budget_is_not_the_sum_of_the_floors(batch, monkeypatch): + """One floor covers wedge detection for the whole call; only the + length-driven overage is additive.""" + import services.model_manager as mm + + FLOOR = 300.0 + + def fake_timeout(text, *, engine=None, execution_device=None): + return FLOOR + len(text or "") + + monkeypatch.setattr(mm, "generate_timeout_s", fake_timeout) + + texts = ["a" * 10] * 8 + budget = batch._batch_timeout_s(texts, _Engine()) + + assert budget == FLOOR + 8 * 10 # one floor + summed overage + assert budget < sum(fake_timeout(t) for t in texts) # not 8 floors + assert budget < 2400 # the wedge window stays minutes + + +def test_the_batch_budget_still_covers_the_longest_item(batch, monkeypatch): + import services.model_manager as mm + + monkeypatch.setattr( + mm, "generate_timeout_s", + lambda text, *, engine=None, execution_device=None: 300.0 + len(text or ""), + ) + texts = ["x" * 500, "y", "z"] + budget = batch._batch_timeout_s(texts, _Engine()) + assert budget >= 300.0 + 500 # the long segment alone still fits + + +# ── cached-segment payload guard (Greptile, #1620 review) ───────────────── + +class _Info: + def __init__(self, *, frames, channels=1, bits=16, sample_rate=24000): + self.num_frames = frames + self.num_channels = channels + self.bits_per_sample = bits + self.sample_rate = sample_rate + + +def _write_pcm_wav(path, *, frames, channels=1, bits=16, data_bytes=None, extra=b""): + """Write a minimal PCM WAV, optionally with non-audio RIFF chunks.""" + bytes_per_frame = channels * (bits // 8) + payload_size = frames * bytes_per_frame + payload = b"\0" * (payload_size if data_bytes is None else data_bytes) + fmt = struct.pack( + "