fix(indextts): accept the config name upstream ships, and keep long text alive

CHANGELOG.md

@@ -33,6 +33,8 @@ the frozen-backend fallback mirror it for their toolchains.
 - The OmniVoice guide now covers combining style attributes with a reference clip (consistent instruct stabilizes cloning; the reference wins conflicts), inline pronunciation control (pinyin / CMU phonemes), and corrects the claim that the default engine can't do voice design — it can, from attributes (#1565)
 
 ### Fixed
+- Installing IndexTTS 2.5 no longer fails claiming an interrupted download — the weights repo ships `config.yaml` and VoiceStudio demanded a `config_v2_5.yaml` that exists in no upstream release; both names are accepted, so a hand-renamed checkout keeps working (#1611) — thanks @zuiaiyutu!
+- IndexTTS 2.5 no longer has long-text generation killed at 60 seconds — the sidecar now proves it is alive every 5 seconds while `infer()` runs, and its deadline rises to 900s (`OMNIVOICE_INDEXTTS_RECV_TIMEOUT_S`) (#1611) — thanks @zuiaiyutu!
 - The OpenAI-compatible `/v1/audio/speech` route now reuses the shared cached engine for explicit `model` ids instead of constructing a fresh engine — and its sidecar/model load, a ~28s floor per call for subprocess engines — on every request, with the same single-engine-resident discipline `/generate` applies (#1614) — thanks @paoloantinori!
 - The setup wizard's RAM check no longer blocks 8 GB machines whose OS reports ~7.8 GB usable — the thresholds now tolerate reserved memory, and `OMNIVOICE_RAM_PREFLIGHT=0` turns a genuine block into a warning for those who accept the OOM risk (#1618)
 - Invisible watermarking now runs eagerly instead of through `torch.compile` — AudioSeal's lazy compile sent the first embed of every session into Inductor's C++ codegen, which failed outright on macOS hosts whose toolchain couldn't serve it and shipped the audio unmarked after a 30-40s wait; first embed drops from 9.70s to 0.26s (#1615) — thanks @paoloantinori!

backend/engines/indextts/__init__.py

@@ -28,6 +28,7 @@
 from __future__ import annotations
 
 import logging
+import math
 import os
 import re
 from typing import TYPE_CHECKING
@@ -164,6 +165,23 @@ def venv_python(cls):
         from engines.indextts.bootstrap import resolve_indextts_venv
         return resolve_indextts_venv()
 
+    @property
+    def recv_timeout_s(self) -> float:
+        # IndexTTS was the only sidecar left on the 60s class default while
+        # pockettts and omnivoice-subprocess both raised theirs. infer() is one
+        # blocking upstream call, so a long passage legitimately outruns 60s and
+        # the parent's watchdog killed a healthy synthesis (#1611). main.py also
+        # heartbeats during infer(), which is what actually proves liveness —
+        # this deadline is the ceiling for a sidecar that has gone genuinely
+        # silent. OMNIVOICE_INDEXTTS_RECV_TIMEOUT_S tunes it.
+        try:
+            v = float(os.environ.get("OMNIVOICE_INDEXTTS_RECV_TIMEOUT_S", "900"))
+        except (ValueError, TypeError):
+            return 900.0
+        if not math.isfinite(v):  # reject inf/nan so the deadline can't be disabled
+            return 900.0
+        return max(30.0, v)
+
     @classmethod
     def sidecar_script(cls):
         from engines.indextts.bootstrap import INDEXTTS_SIDECAR_SCRIPT

backend/engines/indextts/main.py

@@ -63,11 +63,13 @@
 from __future__ import annotations
 
 import base64
+import contextlib
 import json
 import os
 import struct
 import sys
 import tempfile
+import threading
 import traceback
 
 
@@ -117,11 +119,59 @@ def _measure_vram_mb() -> float:
 # ── wire protocol ─────────────────────────────────────────────────────────
 
 
+#: Seconds between keep-alive progress frames during a long blocking call.
+_HEARTBEAT_S = 5.0
+
+#: Serializes _send across threads (the heartbeat below + the main loop) so
+#: concurrent length+body writes can't interleave and corrupt the framing.
+_send_lock = threading.Lock()
+
+
 def _send(stream, obj: dict) -> None:
     body = json.dumps(obj, separators=(",", ":")).encode("utf-8")
-    stream.write(struct.pack("!I", len(body)))
-    stream.write(body)
-    stream.flush()
+    with _send_lock:
+        stream.write(struct.pack("!I", len(body)))
+        stream.write(body)
+        stream.flush()
+
+
+@contextlib.contextmanager
+def _heartbeat(stdout, stage: str):
+    """Emit a progress frame every ~5s for the duration of the block.
+
+    IndexTTS spends the whole of a cold load and the whole of ``infer()``
+    inside one blocking upstream call, saying nothing on the wire. The parent
+    reads that silence two ways, and BOTH kill a perfectly healthy synthesis
+    of a long passage (#1611):
+
+      * ``SubprocessBackend.generate`` re-arms its recv watchdog on every
+        frame, so with no frames it hard-kills the sidecar at recv_timeout_s;
+      * each frame also reports activity to the GPU pool's execution clock
+        (#1367), so with no frames the outer generate budget expires and
+        blames the hardware.
+
+    Raising the deadline alone therefore does not fix long-text generation —
+    the sidecar has to prove it is alive. Percent climbs 1..99 because the
+    upstream call exposes no real progress; it is a liveness signal, not a
+    measurement.
+    """
+    stop = threading.Event()
+
+    def _beat() -> None:
+        pct = 1
+        while not stop.wait(_HEARTBEAT_S):
+            pct = min(pct + 1, 99)
+            try:
+                _send(stdout, {"op": "progress", "stage": stage, "percent": pct})
+            except Exception:
+                return  # pipe gone — the main loop will surface it
+    hb = threading.Thread(target=_beat, name=f"indextts-{stage}-heartbeat", daemon=True)
+    hb.start()
+    try:
+        yield
+    finally:
+        stop.set()
+        hb.join(timeout=_HEARTBEAT_S + 1)
 
 
 def _recv(stream):
@@ -160,14 +210,40 @@ def _torch_bf16_supported() -> bool:
         return False
 
 
+#: Model-config filenames to look for, most-preferred first, per version.
+#: IndexTeam/IndexTTS-2.5 ships ``config.yaml``; VoiceStudio used to demand
+#: ``config_v2_5.yaml``, a name that exists in no upstream revision, so the
+#: install failed until the user hand-renamed the file (#1611). Both names are
+#: accepted now — the hand-renamed installs must keep working untouched — and
+#: the renamed one wins, because a user who created it did so deliberately.
+_CFG_NAMES = {
+    "2.5": ("config_v2_5.yaml", "config.yaml"),
+    "2": ("config.yaml",),
+}
+
+
+def _resolve_cfg_path(model_dir: str, *, version: str) -> str:
+    """First accepted config that exists in ``model_dir``.
+
+    Falls back to the last candidate when none exist, so the failure surfaces
+    as upstream's own "no such file" naming a real expected path rather than
+    a name no upstream release has ever shipped.
+    """
+    names = _CFG_NAMES.get(version, _CFG_NAMES["2"])
+    for name in names:
+        candidate = os.path.join(model_dir, name)
+        if os.path.isfile(candidate):
+            return candidate
+    return os.path.join(model_dir, names[-1])
+
+
 def _model_init_kwargs(
     repo_dir: str, *, version: str, reduced_precision: bool,
 ) -> dict:
     """Build version-specific constructor arguments for IndexTTS 2.5 or 2."""
     model_dir = os.path.join(repo_dir, "checkpoints")
-    cfg_name = "config_v2_5.yaml" if version == "2.5" else "config.yaml"
     kwargs = {
-        "cfg_path": os.path.join(model_dir, cfg_name),
+        "cfg_path": _resolve_cfg_path(model_dir, version=version),
         "model_dir": model_dir,
         "use_cuda_kernel": False,
         "use_deepspeed": False,
@@ -216,7 +292,8 @@ def _load_model(stdout) -> object:
     model_kw = _model_init_kwargs(
         repo_dir, version=_model_version, reduced_precision=reduced_precision,
     )
-    _model = IndexTTS2(**model_kw)
+    with _heartbeat(stdout, "loading_model"):
+        _model = IndexTTS2(**model_kw)
 
     _send(stdout, {"op": "progress", "stage": "loading_model", "percent": 100})
     return _model
@@ -276,7 +353,10 @@ def _handle_synthesize(msg: dict, stdout) -> None:
         tmp_path = tmp.name
     try:
         infer_kw["output_path"] = tmp_path
-        model.infer(**infer_kw)
+        # A long passage keeps infer() busy for minutes with nothing on the
+        # wire; without this the parent kills the sidecar mid-synthesis (#1611).
+        with _heartbeat(stdout, "synthesizing"):
+            model.infer(**infer_kw)
         pcm_b64, sr, n_samples = _wav_to_pcm_b64(tmp_path)
     finally:
         try:

backend/services/sidecar_install.py

@@ -108,7 +108,11 @@ class SidecarSpec:
     weights_repo_id: Optional[str] = None   # HF repo downloaded into <checkout>/<weights_subdir>
     weights_revision: Optional[str] = None  # reviewed HF commit
     weights_subdir: str = "checkpoints"
-    weights_config_name: str = "config.yaml"  # required model config inside weights_subdir
+    # Model-config filenames accepted inside weights_subdir. A tuple, not a
+    # single name: IndexTTS 2.5's weights repo ships config.yaml, but installs
+    # predating #1611 were only usable after hand-renaming it to
+    # config_v2_5.yaml, and those must keep working without a reinstall.
+    weights_config_names: tuple[str, ...] = ("config.yaml",)
     docs_path: str = "docs/engines"         # where the manual-install fallback lives
     required_bytes: int = 12 * _GIB    # conservative source+venv+weights estimate for preflight
     # Called after a successful install/uninstall so the engine's memoised
@@ -146,7 +150,7 @@ def _indextts_installed() -> bool:
         weights_repo_id="IndexTeam/IndexTTS-2.5",
         weights_revision="d0aa86e75bb6f3437f3831e95056fa72842d89ef",
         weights_subdir="checkpoints",
-        weights_config_name="config_v2_5.yaml",
+        weights_config_names=("config.yaml", "config_v2_5.yaml"),
         docs_path="docs/engines/indextts.md",
         # ~0.1 GB source + up to ~6 GB venv (torch + transformers<5) +
         # ~6 GB weights. Deliberately conservative; the preflight subtracts
@@ -879,11 +883,11 @@ def _weights_present(spec: SidecarSpec) -> bool:
     actual = marker[:2] if len(marker) >= 2 else marker + [""]
     if actual != expected:
         return False
-    return _weights_floor_ok(wdir, config_name=spec.weights_config_name)
+    return _weights_floor_ok(wdir, config_names=spec.weights_config_names)
 
 
-def _weights_floor_ok(wdir: Path, *, config_name: str = "config.yaml") -> bool:
-    if not (wdir / config_name).is_file():
+def _weights_floor_ok(wdir: Path, *, config_names: tuple[str, ...] = ("config.yaml",)) -> bool:
+    if not any((wdir / name).is_file() for name in config_names):
         return False
     floor = 5 * 1024 * 1024
     try:
@@ -969,7 +973,7 @@ def _listener(ev: dict) -> None:
         hf_progress.unregister_listener(listener_id)
         hf_progress.current_repo_id.reset(repo_token)
 
-    if not _weights_floor_ok(wdir, config_name=spec.weights_config_name):
+    if not _weights_floor_ok(wdir, config_names=spec.weights_config_names):
         raise _StepError(
             "Weight download finished but no plausible weight files were found — "
             "the download was likely interrupted.",

docs/engines/indextts.md

@@ -77,6 +77,14 @@ missing, preventing slow disks or antivirus scans from hiding a valid venv.
 Set `OMNIVOICE_INDEXTTS_IMPORT_PROBE_TIMEOUT_S` to raise the default 60-second
 probe limit.
 
+### Long-text generation
+
+A long passage can keep `infer()` busy for several minutes. The sidecar emits a
+keep-alive frame every 5 seconds while it works, so the parent can tell a slow
+synthesis from a wedged one, and waits up to 900 seconds for a sidecar that has
+gone genuinely silent. Set `OMNIVOICE_INDEXTTS_RECV_TIMEOUT_S` (minimum 30) to
+tune that ceiling.
+
 IndexTTS 2.5 requires a language token. VoiceStudio maps locale codes and
 language names to the five supported languages and detects Chinese, Japanese,
 or Arabic script for Auto requests. Ambiguous Latin text defaults to English.
@@ -95,9 +103,14 @@ confirm that the configured directory contains:
 ```text
 pyproject.toml
 indextts/infer_v2_5.py
-checkpoints/config_v2_5.yaml
+checkpoints/config.yaml
 ```
 
+`IndexTeam/IndexTTS-2.5` ships the model config as `config.yaml`. Earlier
+installs only worked after hand-renaming it to `config_v2_5.yaml`; both names
+are accepted, so a renamed checkout keeps working as-is and needs no
+reinstall.
+
 ### `uv` not found
 
 Install `uv` from <https://docs.astral.sh/uv/> or configure the bundled binary

tests/backend/services/test_indextts_sidecar.py

@@ -533,24 +533,38 @@ def test_public_duration_maps_to_exact_indextts25_factor(duration, expected):
     assert "target_tokens" not in kwargs
 
 
-def test_sidecar_25_uses_25_config_and_gates_bf16(monkeypatch):
+def test_sidecar_25_uses_the_installed_config_and_gates_bf16(monkeypatch, tmp_path):
+    """#1611: this used to demand config_v2_5.yaml, a name no upstream
+    IndexTTS-2.5 revision ships. The path now follows what is on disk."""
     from engines.indextts import main as sidecar
 
+    repo = tmp_path / "index-tts"
+    ckpt = repo / "checkpoints"
+    ckpt.mkdir(parents=True)
+    (ckpt / "config.yaml").write_text("model: {}\n", encoding="utf-8")
+
     monkeypatch.setattr(sidecar, "_torch_bf16_supported", lambda: False)
     kwargs = sidecar._model_init_kwargs(
-        "/models/index-tts", version="2.5", reduced_precision=True,
+        str(repo), version="2.5", reduced_precision=True,
     )
-    assert kwargs["cfg_path"].endswith("checkpoints/config_v2_5.yaml")
+    assert kwargs["cfg_path"] == str(ckpt / "config.yaml")
     assert kwargs["use_bf16"] is False
     assert kwargs["use_qwen_emo"] is True
 
+    # A checkout carrying the pre-fix hand-renamed config still resolves.
+    (ckpt / "config.yaml").unlink()
+    (ckpt / "config_v2_5.yaml").write_text("model: {}\n", encoding="utf-8")
+    assert sidecar._model_init_kwargs(
+        str(repo), version="2.5", reduced_precision=True,
+    )["cfg_path"] == str(ckpt / "config_v2_5.yaml")
+
     monkeypatch.setattr(sidecar, "_torch_bf16_supported", lambda: True)
     assert sidecar._model_init_kwargs(
-        "/models/index-tts", version="2.5", reduced_precision=True,
+        str(repo), version="2.5", reduced_precision=True,
     )["use_bf16"] is True
 
 
-def test_sidecar_loader_prefers_25_and_uses_reviewed_weight_layout(monkeypatch):
+def test_sidecar_loader_prefers_25_and_uses_reviewed_weight_layout(monkeypatch, tmp_path):
     from engines.indextts import main as sidecar
 
     captured = {}
@@ -565,7 +579,11 @@ def __init__(self, **kwargs):
     module.IndexTTS2 = FakeIndexTTS
     monkeypatch.setitem(sys.modules, "indextts", package)
     monkeypatch.setitem(sys.modules, "indextts.infer_v2_5", module)
-    monkeypatch.setenv("OMNIVOICE_INDEXTTS_DIR", "/models/index-tts")
+    repo = tmp_path / "index-tts"
+    ckpt = repo / "checkpoints"
+    ckpt.mkdir(parents=True)
+    (ckpt / "config.yaml").write_text("model: {}\n", encoding="utf-8")
+    monkeypatch.setenv("OMNIVOICE_INDEXTTS_DIR", str(repo))
     monkeypatch.setattr(sidecar, "_torch_bf16_supported", lambda: True)
     monkeypatch.setattr(sidecar, "_model", None)
     monkeypatch.setattr(sidecar, "_model_version", None)
@@ -574,7 +592,7 @@ def __init__(self, **kwargs):
 
     assert isinstance(loaded, FakeIndexTTS)
     assert sidecar._model_version == "2.5"
-    assert captured["cfg_path"].endswith("checkpoints/config_v2_5.yaml")
+    assert captured["cfg_path"] == str(ckpt / "config.yaml")
     assert captured["model_dir"].endswith("checkpoints")
     assert captured["use_qwen_emo"] is True