diff --git a/.agents/model-matrix.md b/.agents/model-matrix.md index d614e746b..e194607ed 100644 --- a/.agents/model-matrix.md +++ b/.agents/model-matrix.md @@ -121,7 +121,7 @@ Engaged architectures (the 53 non-`INVENTORIED` rows): | πŸ“‹ | `KimiK3ForConditionalGeneration` | Kimi K3 (2.8T MoE + MoonViT-V2, DERIVE-AND-SHIP) | **W2/W5 CPU scaffolding landed** (registry stub + nested text/vision/quant config descent + text-backbone structural name-map + REFUSE-by-name forward + MXFP4-refuse loader; clean CPU build, scaffold gate 6/6). text backbone IS `KimiLinearForCausalLM` (KDA+MLA+MoE hybrid, HEAVY reuse); **does NOT fit GB10 (~1.56 TB MXFP4, ~12Γ—)** and NOT in the pinned oracle β‡’ no on-box golden β€” DERIVED, proxy-gated on Kimi-Linear-48B; forward + MXFP4 + KDA delta + MoonViT-V2 not implemented (NOT-YET-BUILDABLE) | `MODEL-MM-kimi-k3-kimi-k3-for-conditional-generation` | | πŸ“‹ | `Dots3NoteForCausalLM` | dots3-note (280B-A16B multimodal MoE: DeepSeek-V3.2 text tower + sliding-window MLA, MoE ViT, `dots` audio encoder) | W0 scope only, no code. Upstream subclasses DeepSeek (`Dots3NoteModel(DeepseekV32Model)`, `Dots3NoteMoE(DeepseekV2MoE)`), so our gated MLA + DSA indexer + `noaux_tc` router + Qwen3-VL vision + Voxtral audio carry most of it; net-new is windowed MLA over 33 of 46 layers with a second latent geometry (576 vs 1088 rows), the padded/heterogeneous MLA KV spec, the headwise attention gate, the MoE ViT and the `dots` audio stem. **BEYOND-PIN** (vLLM `main` only, `#51255`, still being patched) and **oracle-memory-infeasible**: ~576 GB bf16 / ~290 GB fp8 against a 119-122 GiB ceiling on every host we own, so no performance axis is claimable and no token gate can bind until Β§6.4 of the spec is answered | `MODEL-MM-dots3-note-dots3-note-for-causal-lm` | | 🚧 | `MiniMaxH3DiTModel` | MiniMax-H3 (33.1B omni-modal video+audio DiT, DERIVE-AND-SHIP) | **W1/W2 landed**: packed layout (fl2va + ref2va, fp64 position grid BIT-EXACT), latent packing, euler-ancestral eta0 scheduler, and the full DiT forward all parity-gated against the UPSTREAM vLLM-Omni modules executed at reduced dimensions (**max abs diff 1.6e-7**, 10/10 cases / 2539 assertions). NOT autoregressive (no KV cache, no sampler, no logits) and **e2e HW-BLOCKED** (~354 GB checkpoint, ~133 GB/rank on 4x B300 vs 119 GiB unified); bf16 production stream + request planning + the ComfyUI-GGUF arm also landed (535 REAL tensors resolve onto our contract, geometry from shapes alone). **HW verdict CORRECTED: quantized arms FIT (~41 GB in 119 GiB)**, so e2e + speed are reachable; encoder/VAEs/audio VAE DONE (4.2e-9 vs the checkpoint's remote code); NVFP4 layout GATED as identical to ours (speed path is loader wiring); BOTH VAE DECODERS done (audio 4.2e-9, video ViT3D 8.9e-8); video tiling + 3D-CNN encoder (conditioning only) pending; encoder TEXT tower done (1.2e-7); **serving `/v1/videos` DONE and the DEVICE-RESIDENT forward (W2b, f32) LANDED + GPU-VERIFIED on Thor sm_110 at video 1.49e-7 / audio 8.94e-8**; bf16 stream + fusion folds + the FP4 path (needs sm_121a) + a real-checkpoint run pending. **2026-08-05: the AUDIO-VAE ENCODER is ported** (DAC analysis stack + `pre_block` AttnProjection + `mean_proj`, gated stage by stage vs the checkpoint's own remote code at 2.98e-8 / 1.64e-7 / 1.86e-8) with its own checkpoint loader gated on the real 1087-tensor manifest β€” so **ref2va AUDIO and VIDEO+AUDIO references are now WIRED** (audio rows move by 0.51 / 0.71; a different waveform still moves them by 7.1e-4). Both VAEs are now complete in both directions. **bf16 13-SHARD RELEASE INDEXES 2026-08-07 (`row/H3-BF16-SHARDED-DIT`)**: `MiniMaxH3ShardedCheckpoint` resolves the ORIGINAL 66.3 GB release through its own `model.safetensors.index.json` (a tensor named in the index but missing from its shard throws BY NAME), `EnumerateMiniMaxH3ShardedTensors` feeds the shared shapes-only geometry parser, and `LoadMiniMaxH3DitFromShards` is the host-f32 reference loader. Gated CPU-only at 72/72/54497 (post-rebase): every tensor resolves to the shard the index named AND to the bytes written there, the derived geometry equals the single-file path field for field, and a SPARSE 13-shard release with the REAL 535 tensors at REAL shapes (66.3 GB declared, 144 KB on disk) derives the SHIPPED geometry (50/5376/56/128/14336/24/32/1x2x2/5120). **STREAMS 2026-08-07 (`row/H3-BF16-SHARDED-STREAM`)**: `StreamMiniMaxH3ShardedToDeviceBf16` uploads it one tensor at a time β€” a BF16 tensor bound for a bf16 device slot goes straight from the mmap with ZERO host buffer, so peak host is bounded by ONE tensor (observed `host_peak=8192`, `direct=37 converted=9`); bit-exact vs the non-streamed `StageMiniMaxH3DitWeights` reference over all 46 views with identical logits, rope.inv_freq host-resident, 73/73/55203. Spec Β§8.14. **bf16 TEXT ENCODER + THE CONDITIONING NUMBER 2026-08-07 (`row/H3-ENC-BF16-COND-DIFF`)**: the 14-shard 63 GB bf16 Qwen3-VL-32B encoder streams to device too (`StreamMiniMaxH3EncoderShardsToDevice`, q/k/v and gate/up fused ON DEVICE), `--encoder-only` runs the tower alone (peak ~96 -> ~49 GiB by not loading the DiT first), and the widening is gated BIT-IDENTICAL vs an f32-staged tower so the A/B cannot be confounded. MEASURED on Thor over 233 tokens: Q4_K_M vs bf16 conditioning is cos 0.99745 mean / 0.909 min, rel RMS 6.85% excluding the attention sink, median rotation 3.5 deg β€” same energy as a ONE-WORD prompt edit but DIFFUSE (232/233 tokens rotate vs 172/233). Whether the RENDER changes is NOT established. 75/75/55609. Spec Β§8.15. This UNBLOCKS the quantization-quality question; no bf16-vs-quant render or speed number is claimed. Spec Β§8.13. **W-FP4a LANDED (CPU) 2026-08-06 (`row/H3-FP4-SPEED`)**: the device DiT forward now routes the NVFP4 projections through the shared Marlin W4A16 dispatcher (fp4 kept packed; no new quant code), fp4-vs-bf16 wiring gate GREEN (62/62Β·30039). **W-FP4a GB10 leg LANDED 2026-08-06 (`row/H3-FP4-GPU-E2E`, PR #64):** on sm_121a the Marlin W4A16 path RAN for all 11 projections (`dense_gemms==11` default β€” VT_MARLIN_DENSE is default-ON β†’ vLLM's own DENSE Marlin GEMM, not the grouped route; `marlin_gemms==11` under VT_MARLIN_DENSE=0; `fallback_gemms==0`), fp4-vs-bf16 BYTE-EXACT (max\|diff\|=0), and the fp4 arm is a MEMORY win not a diffusion-forward speed win (per-forward bf16/fp4 3.47Γ— @seq64 β†’ 0.79–0.83Γ— @seq4224–7040; ~16 vs ~66 GB device). Real-checkpoint fp4-resident t2va e2e RUNS (real 18.75 GB NVFP4 DiT + VAEs + GGUF Qwen3-VL-32B encoder β†’ valid mp4/wav; DiT s/step 5.45/20.0/209 s @512/768/REF-209f) but frames are a non-scene patch-grid at 12/20/50 steps β†’ OPEN render bug (device VAE/denoise). vLLM-Omni has no quantized H3 arm (BF16-only) so any comparison is HW/loader-forced-indirect β€” spec Β§8. **ONE-SURFACE ROW 2 LANDED 2026-08-08 (`row/H3-VIDEO-ABI`)**: the whole assembly pipeline is library-owned (`vllm::multimodal::MiniMaxH3VideoEngine`, `minimax_h3_video.cpp`) behind the ABI v12 `vllm_video_*` entry points; `/v1/videos` routes through the SAME seam; `minimax_h3_gen`+`minimax_h3_mux` are thin `vllm.h` clients, frames+WAV byte-identical to the pre-fold binary on the committed fold fixture (`test_minimax_h3_video_fold` 3-arm gate + the v12 `test_capi` section); GB10 real-video re-verify via the v12 ABI = named residual. **ROW 2 DEVICE-SEAM FOLLOW-UP (#135; replaces #134):** ABI 0/1 maps once to `vt::DeviceType`; shared code dispatches through `GetBackend(device_type)`, restoring DSR 34β†’32 without a baseline/allowlist change; CPU compile/fold test pending in CI due shared-disk pressure **PRUNED CHECKPOINTS LOAD AND RUN 2026-08-10 (`row/H3-PRUNED`, #241, spec Β§8.21):** the community `pruned` variants (`unsloth/MiniMax-H3-GGUF` Q2_K..Q8_0, `lilcheaty/MiniMax-H3-NVFP4` `*_pruned_nvfp4`) are not lossily pruned but ComfyUI's AdaLN timestep-CURVE refactor (`comfy/ldm/minimax/model.py:419-432,610-615`): an `adaln_t_table` [1025, 8] lerp replaces the sinusoidal+MLP time embedder, no SiLU before the AdaLN linear, and its in_features drop 2688 -> 8, collapsing `adaln_proj` from 13.04B of 33.12B to 0.04B and the DiT to 20.11B β€” so a pruned Q8_0 (21.4 GB) costs what our unpruned Q4_K_M (19.9 GB) costs. Gated on the REAL 532-tensor pruned GGUF manifest (532 = 535 - 4 `time_embedder.*` + 1 table, header-only), the clamped-lerp golden, and a CONSTRUCTED-curve test proving the pruned forward is numerically identical to the unpruned one; 79/79/57299. | `MODEL-DIFFUSION-minimax-h3-mini-max-h3-dit` | -| 🚧 | `MiniMaxMusic3ForConditionalGeneration` | MiniMax-Music3 (text-to-music: ~8.6B Qwen3 global LLM + 0.646B 8-codebook RVQ depth decoder + 2.4B fp32 flow-matching DiT + 0.054B DAC Flow-VAE) | **W0 DONE + W1 LANDED (#672, [spec](specs/minimax-music3.md)).** Diffusers oracle gateable with committed per-stage goldens (#708); modular loader in the tree, 1413/1413 assertions against the real checkpoint (#714). Geometry MEASURED from every component `config.json` plus safetensors headers by range request. First row to exercise the fallback-oracle rule (#647): absent from vLLM AND vLLM-Omni, so its primary oracle is the OPEN diffusers PR #14456 head `c6da9936` with SGLang-Omni `748a0b43` as the e2e cross-check. W2/W3 in flight | `MODEL-MUSIC-minimax-music3-mini-max-music3-for-conditional-generation` | +| 🚧 | `MiniMaxMusic3ForConditionalGeneration` | MiniMax-Music3 (text-to-music: ~8.6B Qwen3 global LLM + 0.646B 8-codebook RVQ depth decoder + 2.4B fp32 flow-matching DiT + 0.054B DAC Flow-VAE) | **W0 DONE + W1 LANDED (#672, [spec](specs/minimax-music3.md)).** Diffusers oracle gateable with committed per-stage goldens (#708); modular loader in the tree, 1413/1413 assertions against the real checkpoint (#714). Geometry MEASURED from every component `config.json` plus safetensors headers by range request. First row to exercise the fallback-oracle rule (#647): absent from vLLM AND vLLM-Omni, so its primary oracle is the OPEN diffusers PR #14456 head `c6da9936` with SGLang-Omni `748a0b43` as the e2e cross-check. **W2/W3 LANDED**: the AR half computes and is gated; the 8.6B language-model forward and W4-W7 are owed, and spec Β§5's token-exact gate is WITHDRAWN because upstream's AR stage has no greedy path | `MODEL-MUSIC-minimax-music3-mini-max-music3-for-conditional-generation` | | πŸ“‹ | `LTX2VideoTransformer3DModel` | LTX-2.5 (21.00B joint video+audio flow-matching DiT, Lightricks) | **L0 spec committed 2026-08-11 (#435, [spec](specs/ltx-2-5.md)).** Geometry MEASURED from the FP8 checkpoint's own safetensors header by HTTP range request (6124 tensors, 881,048-byte header, no payload downloaded): **21.00B** params β€” 48 blocks @ 386.7M = 18.560B, audio embeddings connector 2.016B, global 0.427B. The filename says `22b` and the Diffusers card says ~19B; the MEASURED count is what this row uses. Video stream 4096 (32 heads x 128), audio stream 2048 (32 heads x 64), in/out channels 128 both. Per block SIX attentions β€” `attn1` (video self), `attn2` (video<->text, cross 4096), `audio_attn1`, `audio_attn2` (cross 2048), and the two CROSS-MODAL `audio_to_video_attn` / `video_to_audio_attn` β€” which is the structural break from MiniMax-H3: H3 packs every modality into ONE sequence with per-row token tags, LTX runs TWO streams coupled by explicit cross-attention. **Per-head gated attention on every attention** (`to_gate_logits` = `Linear(query_dim, heads, bias=True)`, `attention.py:513-514`, applied AFTER the attention output at `:577`) β€” H3 has no analogue and getting it wrong renders plausibly-wrong rather than erroring. FFN is gelu-approximate 4096->16384->4096 with **NO bias** while `audio_ff` 2048->8192->2048 **HAS** bias, which independently confirms `ff_bias=false` / `audio_ff_bias=true` from `model_configurator.py:78-80` β€” checkpoint and source agreeing, not either alone. **RETRACTED 2026-08-12, was billed as a FREE WIN.** The spec claimed 2.5 sets `use_prompt_adaln_single=false` so the cross-attention K/V are timestep-free and cacheable. The shipped checkpoint DISPROVES it: it carries 12 `prompt_adaln_single`/`audio_prompt_adaln_single` tensors including a `timestep_embedder.linear_1 [4096, 256]` (256 = the sinusoidal timestep width), and `model.py:223-227` builds that module ONLY when the flag is TRUE. `transformer.py:441` was quoted as proof of no timestep term, but `:442-443` add one whenever `prompt_timestep` is not None, and the comment above them says exactly that. NO SHIPPED DEFECT: `ltx2_dit.cpp:672` refuses the cache by name when the flag is on, so the feature is correct-and-inapplicable rather than silently wrong, and stays gated bit-identical and prompt-bound for any checkpoint that does set it false. **ORACLE:** vLLM-Omni does NOT support 2.5 β€” `resolve_ltx_pipeline_recipe` keys on 2 / 2.3 only and RAISES otherwise (`ltx2_recipes.py:162-166`), with upstream [vllm-omni#6066](https://github.com/vllm-project/vllm-omni/issues/6066) filed 2026-08-11 β€” but its `DiffusersAdapterPipeline` is generic (`DiffusionPipeline.from_pretrained`, `pipeline_diffusers_adapter.py:116`), so vLLM-Omni CAN execute 2.5 via `--load-format diffusers`. Binding oracle = that adapter; immediate cross-check = Lightricks `ltx-pipelines`. **HW: FITS ONE GB10** at ~29 GB (NVFP4 DiT 18.72 + NVFP4 Gemma-4 TE 7.40 + VAEs 1.83 + upscaler 1.00) vs H3's ~41 GB. **OWED UP FRONT, not to be discovered later:** the speed axis lands `PENDING` because the adapter is a black box (`supports_step_execution=False`, `supports_request_batch=False`) and therefore NOT vLLM's production configuration, which AGENTS.md requires as the denominator; DiffVAE (`NADiffusionDecoder`, neighborhood attention) is REFUSED BY NAME until its own row rather than silently downgraded to the Conv VAE; and no render-quality claim is made from structural e2e. | `MODEL-DIFFUSION-ltx-2-5-ltx2-video-transformer-3d-model` | | βœ… | `LagunaForCausalLM` | Poolside Laguna-S-2.1 (118B/8B MoE) | **LONG-CTX DECODE LEVERS LANDED + MEASURED (2026-08-03, `CLAIM-LAGUNA-LONGCTX-LEVERS`): window-bounded SWA reads (`VT_LAGUNA_SWA_WINDOW`, default-ON, BYTE-EXACT) bound the four `DecodeAttnGqa*` kernels' read to the ~512 sliding window (vLLM `laguna.py:412`) β€” GB10 A/B token-IDENTICAL `=1` vs `=0` at 520-token context (truncation active), MEASURED βˆ’0.30 ms/step at ~2k (~0 at ≀512, grows linearly). bf16 paged KV (`VT_LAGUNA_KV_BF16`, default-OFF opt-in) a distributional near-tie left UNRATIFIED. See BENCHMARKS `CLAIM-LAGUNA-LONGCTX-LEVERS`.** β€” **NVFP4 W4A4 ARM RAN on GB10 (N4, 2026-08-01, `CLAIM-LAGUNA-NVFP4-N4`): the additive safetensors NVFP4 arm (N1a/N1b/N2/N3 β€” `Nvfp4Weight` expert fields + `LoadLagunaForCausalLMWeights` + `LqGemmNvfp4Fp4` per-expert TRUE-W4A4 + `LagunaFfnBlock` `fp4` branch + `laguna_gen` dir-autodetect; CPU-gated `test_laguna_nvfp4_loader` 3/3Β·61, GGUF path byte-identical) generates COHERENTLY on the real 67 GiB `poolside/Laguna-S-2.1-NVFP4`. vs the vLLM MARLIN golden (vLLM's exact prompt ids injected): FIRST 2 TOKENS MATCH exactly, then near-tie divergence (our TRUE-W4A4 fp4-activations vs the MARLIN golden's W4A16 bf16-activations β€” different precision, EXPECTED; shares golden vocab). SPEED (N5, trace-driven, 2026-08-01): 0.16 β†’ ~4.5 tok/s (~28Γ— THIS SESSION), now ~4Γ— from vLLM 18.8. **Lever #2** (nsys found the bf16 tower running host `MatmulNK` on the CUDA queue): route it to the GPU (`LqGemm` bf16 β†’ `CastBf16` + `MatmulBT`, weight stays bf16) β†’ 6.34 β†’ 0.39 s/tok (16Γ—). **Lever #1** (nsys found the emulation expert GEMM at 92%, GPU 87% busy): the engine's native sm120a fp4 tensor-core MMA (`MatmulNvfp4Fp4Native`) reads the SAME linear scales β€” it was gated OFF behind `VT_NVFP4_FP4_NATIVE`; default it ON in the driver β†’ 0.39 β†’ ~0.20-0.24 s/tok (~2Γ—). Both coherent + near-tie (byte-identical ids to emulation; first token matches golden). Two GB10 memory fixes landed to run (shard-release + context-before-load). OPEN #234 (remaining ~4Γ—): grouped W4A4 MoE (top_kΓ—3 launches β†’ 3), `ResidentNvfp4`, decode CUDA-graph + on-GPU sampling (the host-orchestration tail). Spec `.agents/specs/laguna-nvfp4-arm-2026-07-31.md` Β§N4/Β§N5. The GGUF-Q4_K track (below) is the separate keep-quant vehicle.** Prior **FASTER DECODE (W9, 2026-07-31, `CLAIM-LAGUNA-W9-GROUPED`): the 30 un-grouped per-expert keep-quant GEMV launches/step (top_k Γ— {gate,up,down} `LqGemmRowSlice`) fold onto the SHARED `vt::MatmulBTQuantGrouped` op β€” per token, Pk experts' gate/up/down each collapse to ONE grouped launch over the already-stacked `[E*N,H]` tower (no loader change). Same-binary A/B on real UD-Q4_K_XL (GB10, `--gpu`, drop_caches cold, 24 tok): grouped (`VT_LAGUNA_GROUPED_MOE=1`, default) == per-expert (`=0`) BYTE-IDENTICAL (md5 `754728c6`, both == W6 golden) + decode 0.18 β†’ 0.13 s/tok (1.38Γ—). Routes through the shared vt op (fold policy). Cumulative with W8: decode 0.66 β†’ 0.13 s/tok (5.1Γ—; 1.5 β†’ 7.7 tok/s; 18Γ— β†’ 3.6Γ— vs llama.cpp 27.8). Next lever: device-resident decode (#1). See spec Β§W9.** Prior **FASTER DECODE (W8, 2026-07-31, `CLAIM-LAGUNA-W8-EMBED`): `LagunaEmbed` no longer converts the whole 1.23 GB embed table to f32 every token (it gathered T rows out of the whole [Vsz,H] table via `ReadF32` β€” ~311M host element-converts/token, the DOMINANT decode cost the W7 profile under-filed as "#5"); now gathers only the T needed rows directly (BIT-IDENTICAL β€” same per-element conversion, same rows). GATED on the real 3-shard UD-Q4_K_XL GGUF (GB10, `--gpu`, W6 cached, drop_caches cold, 24 tok): TOKEN-IDENTICAL to the W5/W6 golden (`22345 83 350 785 …`, coherent " Paris.") + decode 0.66 β†’ 0.17 s/tok = 3.9Γ— (1.5 β†’ 5.9 tok/s; 18Γ— β†’ 4.7Γ— vs llama.cpp 27.8). See `.agents/specs/laguna-s21-w7-speed-2026-07-31.md` Β§W8. Next: grouped-expert GEMM (=A3) then device-resident decode.** Prior **DECODE-SPEED ATTRIBUTED (W7 profile-only, 2026-07-31, `CLAIM-LAGUNA-W7-SPEED`): `nsys` of the W6 decode (real UD-Q4_K_XL GGUF, GB10) attributes the 0.66 s/tok (~1.5 tok/s vs llama.cpp 27.8 on identical bytes, ~15-18x) to HOST-ORCHESTRATION, not kernel compute β€” GPU active only 32.7% of the step, 67.3% host/idle; 22,115 `cudaStreamSynchronize` (~2,764/step, zero GPU overlap) from the ~1,795 per-GEMM `DrainQueue` in `LagunaForwardGgufCached` + scalar host glue; 39.4% of GPU time is `QuantizeQ8K` activation-quant (per-GEMM), weight GEMVs un-grouped at ~22% of the 240 GB/s peak (llama.cpp ~76%); no H2D/D2H (unified memory). Ranked levers (all in-tree from ds4): device-resident decode 1.5->~5-7 tok/s, grouped-expert GEMM (`MatmulBTQuantGrouped`) +1.5-2x + dedupes the activation-quant, decode CUDA-graph, tuned MMVQ; + free host cleanups (`LagunaEmbed` copies the whole 1.23 GB embed table/token, per-token RoPE-cache rebuild). Honest reachable ~13-20 tok/s, 27.8 a stretch. NO code changed. See `.agents/specs/laguna-s21-w7-speed-2026-07-31.md`. Prior RUNNABLE + FAST DECODE (W6, 2026-07-31): a per-layer K/V cache + single-token incremental decode replaces W5's O(nΒ²) STATELESS recompute β€” TOKEN-IDENTICAL (byte-equal ids, md5 `754728c6…` match, == the W5 golden) and 5.05Γ— faster per token: decode 3.33 β†’ 0.66 s/tok on the real UD-Q4_K_XL GGUF (GB10, `--gpu`, keep-quant), same " Paris.…" text. `LagunaKvCache` (mirrors `DeepseekV4KvCache`, MLA-latent β†’ GQA multi-head K/V; caches post-QK-RMSNorm/post-RoPE K + raw V at f32 β€” bit-exact since RoPE/QK-norm are position-only and attention is causal), MIXED attention per-layer: 12 GLOBAL layers grow unbounded + 36 SLIDING-WINDOW-512 layers EVICT rows beyond the 512 window (gemma2/3 `is_sliding`); `LagunaForwardGgufCached` + shared `LagunaAttention`/`LagunaFfnBlock` helpers used by BOTH forwards (identical float ops; recompute ids unchanged after refactor), `examples/laguna_gen --stateless` A/B flag. No cache bug (bit-exact first run). Next speed = grouped-expert GEMM + device-resident decode (both in-tree from ds4). See `.agents/specs/laguna-s21-w6-2026-07-31.md`. Prior RUNNABLE (W5, 2026-07-31): our engine greedy-generates COHERENT text on the REAL 3-shard UD-Q4_K_XL GGUF (GB10 keep-quant) β€” "The capital of France is" β†’ " Paris. …", first token "Paris." matches the llama.cpp-Poolside reference. Multi-shard GGUF reader + keep-quant tower (`LoadLagunaFromGgufShards`) + `LagunaForwardGguf` (ds4 keep-quant Gemm/GemmRowSlice) + `examples/laguna_gen`; load 20.6s, peak 71 GiB, 3.27 s/tok stateless recompute (speed=W6).** Prior W3: **W3 REAL forward + 3 new ops landed** (`laguna_ops.cpp`: per-head softplus attn out-gate + ungrouped sigmoid-noaux router + dual per-layer RoPE cos/sin builders; `LagunaModel::Forward` now a REAL runnable host-reference composition β€” variable-Q-head GQA + dual RoPE + sliding-window mask + softplus gate + dense L0 / ungrouped-MoE L1..47 + untied lm_head β€” replacing the W1/W2 `VT_CHECK(false)` stub; CPU `-Werror` full-library build clean; `test_laguna_scaffold` **8/8Β·166** incl. softplus math, router selection+tie-break RED-first, dual-RoPE cos/sin bit-match, variable-Q-head shapes, forward composition on synthetic weights; `test_model_registry` 24/24). W1 oracle DECISION: vLLM native `laguna.py` in pin β‡’ config constructs; dual-oracle = vLLM-NVFP4/-FP8 (fits GB10, BF16 235 GiB does NOT) + llama.cpp-Q4_K token-exact. DEFERRED to W4 (needs 73 GB checkpoint): GGUF keep-quant tower materialization + device/paged production forward + strict dual-oracle greedy gate. ~85-90% reuse (ds4-MoE + gemma-sliding + olmo3-dual-rope + landed Q4_K keep-quant); NEW = the 3 landed host ops + name-map + variable-Q-head device runner. **W4 (2026-07-31, `CLAIM-LAGUNA-W4`, in progress):** the UD-Q4_K_XL GGUF (73.4 GiB, 3 shards) FETCHED to dgx + its metadata/tensor-map READ AUTHORITATIVELY (814 tensors, arch `laguna`, `expert_gating_func=2` sigmoid, `leading_dense_block_count=1`, `expert_weights_scale=2.5`). Three CPU-verified FIDELITY corrections the W1-W3 scaffold got wrong, each grounded in the real GGUF + llama.cpp: (1) **per-head QK-RMSNorm** (`attn_q_norm`/`attn_k_norm` F32[128]) added to params+forward β€” the scope MISSED it (surfaces only in the tensor map); (2) **dual-RoPE mscale** now uses llama.cpp's `yarn_attn_factorΒ·(1+0.1Β·ln(factor))` off the GGUF-authoritative `factor=32`/`yarn_attn_factor=1.0` (256K-ctx build, NOT HF's factor-128/1.4852 1M-ctx scalar) β€” resolves the numerics-delicate residual; (3) **separate** `ffn_gate_exps`/`ffn_up_exps` (Q4_K) + `ffn_down_exps` (Q5_K) + Q8_0 shared/attn (the scaffold assumed merged gate_up). GGUF keep-quant tower materialization (`Mw`/`Sew` mirror of ds4) + keep-quant `ForwardGguf` (vt::MatmulBT/GemmRowSlice) + the real-model greedy run vs the llama.cpp-laguna same-quant oracle remain the W5 close (73 GB single-GB10, host-orchestrated) | `MODEL-TEXT-laguna-laguna-for-causal-lm` | | 🚫 | `DeepseekV3ForCausalLM` / `DeepseekV32ForCausalLM` | DeepSeek-V3 / V3.2 | HW-blocked (671B, ~642 GiB fp8 vs 119 GiB unified memory); V3.2 additionally DEP-blocked (DSA indexer) | `MODEL-TEXT-deepseek-v2-deepseek-v3-for-causal-lm` | @@ -462,7 +462,7 @@ Transformers compatibility is capability-driven and excluded from finite counts. | `MODEL-MM-voxtral-realtime-voxtral-realtime-for-conditional-generation` | `VoxtralRealtimeForConditionalGeneration` (target-pending) | **TARGET-PENDING β€” the architecture string is registered NOWHERE we can find.** Recipe `mistralai/Voxtral-Mini-4B-Realtime-2602` (one of the 157 recipes at `vllm-project/recipes` `86c7777a`) declares `architectures: ["VoxtralRealtimeForConditionalGeneration"]` with an inner `audio_config.model_type: "voxtral_realtime_encoder"`, but that exact string appears in none of: (a) the pinned registry `555967922`; (b) any of the ten dictionaries of `vllm/model_executor/models/registry.py` on vLLM `main`; (c) `_OMNI_MODELS` in `vllm_omni/model_executor/models/registry.py` @ `bbe6ccc512a404a2df8c977ea29003002f2683e8`; (d) `vllm-omni` `docs/models/supported_models.md`. **Related but NOT equal, recorded so nobody re-derives it:** `VoxtralRealtimeGeneration` β€” a DIFFERENT string β€” is registered at the pin (`registry.py:585`, `main` `:603`) and already has its own row here, `MODEL-MM-voxtral-realtime-voxtral-realtime-generation`; `VoxtralForConditionalGeneration` (pin `registry.py:584`) is the audioβ†’TEXT model; and `vllm-omni` `registry.py:305-309` registers `VoxtralTTSForConditionalGeneration` for the SEPARATE `mistralai/Voxtral-4B-TTS-2603` recipe. The most likely reading is that this checkpoint's `config.json` names a renamed or not-yet-landed spelling of the existing `VoxtralRealtimeGeneration` target, but that is a hypothesis: no anchor is asserted until one of those is shown to be what actually resolves. Tracked by [#610](https://github.com/mudler/vllm.cpp/issues/610) | conditional generation / audio in, AUDIO OUT (realtime speech-to-speech) β€” modality inferred from the recipe and config, NOT from an upstream implementation | unresolved until the target is located | ☐ required | `INVENTORIED` | none | unassigned | | `MODEL-MM-bailing-mm-native-bailing-mm-native-for-conditional-generation` | `BailingMMNativeForConditionalGeneration` (target-pending) | **TARGET-PENDING β€” the architecture string is registered NOWHERE we can find.** Recipe `inclusionAI/Ming-omni-tts-0.5B` (one of the 157 recipes at `vllm-project/recipes` `86c7777a`) declares `architectures: ["BailingMMNativeForConditionalGeneration"]`, but that exact string appears in none of: (a) the pinned registry `555967922`; (b) any dictionary of `vllm/model_executor/models/registry.py` on vLLM `main`; (c) `_OMNI_MODELS` in `vllm_omni/model_executor/models/registry.py` @ `bbe6ccc512a404a2df8c977ea29003002f2683e8`; (d) `vllm-omni` `docs/models/supported_models.md`. **Related but NOT equal:** `vllm-omni` `docs/models/supported_models.md:75` lists that SAME checkpoint under a different architecture name, `MingTTSForConditionalGeneration` (`registry.py:250-254` β†’ module `ming_tts/ming_tts.py`, class `MingTTSForConditionalGeneration`); and `vllm-omni` `registry.py:417-421` carries a near-miss alias `BailingMM2NativeForConditionalGeneration` β€” note the `2` β€” commented "HF repo currently ships this architecture name in config.json", pointing at module `ming_flash_omni/ming_flash_omni.py`, class `MingFlashOmniForConditionalGeneration`, for the DIFFERENT Ming-flash-omni-2.0 checkpoint. So the shape of the gap is a missing config-string alias upstream, but that is an observation, not an anchor, and none is asserted here. Tracked by [#610](https://github.com/mudler/vllm.cpp/issues/610) | conditional generation / text+image+audio in, AUDIO OUT (TTS) β€” modality from the recipe and config, NOT from a resolved upstream target | unresolved until the target is located | ☐ required | `INVENTORIED` | none | unassigned | | `MODEL-DIFFUSION-ltx-2-5-ltx2-video-transformer-3d-model` | `LTX2VideoTransformer3DModel` | **BEYOND-PIN AND OUT-OF-REPO** β€” not in `555967922` (LTX-2.5 released 2026-08) and not in the vLLM repository at all. Architecture reference is Lightricks' own `LTX-2` (`packages/ltx-core/src/ltx_core/`: `model/transformer/{transformer,model,attention,rope,model_configurator}.py`, `model/audio_vae/`, `components/`, `guidance/`). vLLM-Omni carries an `ltx2` module but its recipes stop at 2.3 (`vllm_omni/diffusion/models/ltx2/ltx2_recipes.py:162-166`); 2.5 is upstream-OPEN at [vllm-omni#6066](https://github.com/vllm-project/vllm-omni/issues/6066) and [#4985](https://github.com/vllm-project/vllm-omni/issues/4985). Serving oracle reaches 2.5 through the GENERIC `DiffusersAdapterPipeline` (`pipeline_diffusers_adapter.py:116`) against `Lightricks/LTX-2.5-Diffusers`. | diffusion generation / video + audio (text/image/video in, video+audio out) | flow-matching denoise loop; TWO coupled modality streams with audio<->video cross-attention; per-head gated attention; AdaLN-single with 9-vector per-block tables; timestep-INDEPENDENT prompt K/V (cacheable once per request); split/interleaved 3D RoPE; gelu-approximate FFN; Gemma-4 12B text encoder with dual caption projections (4096 video / 2048 audio); Conv video VAE + DiffVAE; audio VAE + vocoder; latent spatial/temporal x2 upsamplers; duration head; distilled two-stage recipe | [ltx-2.5](specs/ltx-2-5.md) | `SPIKE` | **L0 LANDED 2026-08-11 (#435).** Spec committed BEFORE implementation per AGENTS.md. Geometry measured from the ungated `vonkaiser/LTX-2.5-FP8-NVFP4` FP8 checkpoint header by range request β€” the same no-payload technique used for H3's manifests: 21.00B params, 48 blocks, 1775 F8_E4M3 tensors each with a per-tensor F32 `weight_scale`, biases/norms BF16. Cross-modal projection shapes are ASYMMETRIC and are a named test trap: `audio_to_video_attn.to_q` is [2048, 4096] while `to_k`/`to_v` are [2048, 2048] and `to_out` is [4096, 2048], so a transposition still type-checks against a square assumption. **CHECKPOINT ACCESS (verified against the HF API 2026-08-11):** `Lightricks/LTX-2.5` is `gated: auto` (accepting the license opens it; holds the first-party NVFP4 DiT at 18.72 GB), `Lightricks/LTX-2.5-Diffusers` is RESTRICTED behind manual approval (needed for the binding oracle), and `vonkaiser/LTX-2.5-FP8-NVFP4` is UNGATED and unblocks L1-L2 today. All artifacts land under `$CHECKPOINT_ROOT` (`/mnt/nas_share/checkpoints`) so dgx.casa and the cluster nodes share one copy. **PHASES (one PR, developer-directed):** L1 `vllm::multimodal::VideoEngine` interface + checkpoint-detected registry with MiniMax-H3 moved behind it UNCHANGED (gated on frames+WAV byte-identical to the pre-refactor fold fixture) and ABI **v13 by ADDING fields only**; L2 DiT forward; L3 Gemma-4 TE reusing `gemma4.cpp`; L4 Conv video VAE + audio VAE + vocoder; L5 pipeline/recipes/upsampler/duration head; L6 NVFP4 arms + GB10 load-time residency; L7 e2e on dgx.casa under `flock $HOME/gpu.lock`. **OPEN, same as H3:** there is no vllm-omni parity PIN β€” `.agents/upstream-sync.md` covers the vLLM repo only, so each golden records the vllm-omni SHA inline. | `CLAIM-LTX25-L0-L7` | -| `MODEL-MUSIC-minimax-music3-mini-max-music3-for-conditional-generation` | `MiniMaxMusic3ForConditionalGeneration` | **BEYOND-PIN AND OUT-OF-REPO** β€” not in `555967922` and not in the vLLM repository at all; absent from `vllm-omni` too. Its reference implementation is the OPEN `diffusers` PR [#14456](https://github.com/huggingface/diffusers/pull/14456) head `c6da9936` (`src/diffusers/models/transformers/transformer_minimax_music3.py`, `.../minimax_music3_rvq_depth_decoder.py`, `src/diffusers/models/autoencoders/minimax_music3_vocoder.py`, `src/diffusers/models/condition_embedders/condition_embedder_minimax_music3.py`, `src/diffusers/modular_pipelines/minimax_music3/`), with SGLang-Omni `748a0b43` (`sglang_omni/models/minimax_music3/`) serving the NATIVE layout as the e2e cross-check. **The first row to exercise the fallback-oracle rule (#647)** | music generation / text -> audio (lyrics + structured description in, multi-minute stereo song out) | Qwen3 dense forward + paged KV (LANDED); flow-matching denoise loop (H3/LTX-2.5 precedent); DAC-style audio VAE decode + WAV writer (H3/LTX-2 precedent); NEW: 8-codebook RVQ frame path, 4-layer depth decoder, learned 8-layer condition mix, snake activations with weight-norm folding, and an LLM->diffusion handoff on CONTINUOUS hidden states rather than discrete tokens | [minimax-music3](specs/minimax-music3.md) | `ACTIVE` | **W0 DONE + W1 LANDED (#672).** Oracle GATEABLE: `tools/oracle/music3_oracle.py` generated 0.998 s of 44100 Hz stereo at diffusers `c6da9936` (identity asserted at runtime, proved red first), with 13 per-stage goldens under `tests/parity/goldens/minimax_music3_oracle/` (#708). W1 modular loader `minimax_music3_loader.{h,cpp}`: 1413/1413 assertions against the real 27 GB tree, all 1012 tensors accounted, 50 mutations proven to fire, native arm refused by name; `MaterializeWeightNorm` relocated into the shared `vocoder1d` primitives (H3 79/79 cases, 57,395 assertions unchanged) rather than forked (#714). Spec committed BEFORE implementation per AGENTS.md. Geometry MEASURED, not inferred: every component `config.json` read, plus safetensors headers by HTTP range request β€” `transformer` 231 tensors/1.240B per shard **all F32** (so the card's 2.4B is right and 9.73 GB is fp32 storage, NOT a 4.9B bf16 model), `rvq_depth_decoder` 47/0.646B BF16, `vocoder` 121/0.054B F32 with `weight_g`/`weight_v` weight-norm, `condition_encoder` **4 tensors** (`layer_scale`, `layer_weight_logits`, `proj`) which makes it a learned weighted mix over 8 LLM layers rather than an encoder tower. `language_model` is our LANDED `Qwen3ForCausalLM` architecture exactly (36L/4096/32-8/head_dim 128) at `vocab_size 200000`. The repo ships TWO packagings of ONE set of weights (native `qwen_7B/`+`flowmatching_vae.pth`+`dav.pth` vs the diffusers six); `scripts/convert_minimax_music3_to_diffusers.py@c6da9936` `:29-38,47-189` proves the conversion is a pure re-layout, so the diffusers arm is the port target at **~28.5 GB resident** (no pickle, no `trust_remote_code`) and SGLang-Omni stays a valid e2e cross-check. **Sample rate RESOLVED 2026-08-13** (spec Β§1.1): not a contradiction but a stage boundary. The vocoder natively emits 44100 stereo (86.133 Hz latent frames x 512 upsampling, derived from `minimax_music3_vocoder.py:84,92-95` and confirmed by `dav.py:94,115`); diffusers returns that unresampled, while SGLang-Omni's SERVER resamples 44100 -> 32000 on delivery (`constants.py:18-19`, `acoustic.py:55-58,422-431`). Goldens are captured at 44100 stereo, the native resample-free rate; 32 kHz is a downstream delivery transform gated separately, and NOT free β€” reproducing its bytes needs torchaudio's default sinc filter, and a latent-parity check sits upstream of that call and cannot see it | `CLAIM-MODEL-MUSIC3-W0` | +| `MODEL-MUSIC-minimax-music3-mini-max-music3-for-conditional-generation` | `MiniMaxMusic3ForConditionalGeneration` | **BEYOND-PIN AND OUT-OF-REPO** β€” not in `555967922` and not in the vLLM repository at all; absent from `vllm-omni` too. Its reference implementation is the OPEN `diffusers` PR [#14456](https://github.com/huggingface/diffusers/pull/14456) head `c6da9936` (`src/diffusers/models/transformers/transformer_minimax_music3.py`, `.../minimax_music3_rvq_depth_decoder.py`, `src/diffusers/models/autoencoders/minimax_music3_vocoder.py`, `src/diffusers/models/condition_embedders/condition_embedder_minimax_music3.py`, `src/diffusers/modular_pipelines/minimax_music3/`), with SGLang-Omni `748a0b43` (`sglang_omni/models/minimax_music3/`) serving the NATIVE layout as the e2e cross-check. **The first row to exercise the fallback-oracle rule (#647)** | music generation / text -> audio (lyrics + structured description in, multi-minute stereo song out) | Qwen3 dense forward + paged KV (LANDED); flow-matching denoise loop (H3/LTX-2.5 precedent); DAC-style audio VAE decode + WAV writer (H3/LTX-2 precedent); NEW: 8-codebook RVQ frame path, 4-layer depth decoder, learned 8-layer condition mix, snake activations with weight-norm folding, and an LLM->diffusion handoff on CONTINUOUS hidden states rather than discrete tokens | [minimax-music3](specs/minimax-music3.md) | `ACTIVE` | **W0 DONE + W1 LANDED (#672).** Oracle GATEABLE: `tools/oracle/music3_oracle.py` generated 0.998 s of 44100 Hz stereo at diffusers `c6da9936` (identity asserted at runtime, proved red first), with 13 per-stage goldens under `tests/parity/goldens/minimax_music3_oracle/` (#708). W1 modular loader `minimax_music3_loader.{h,cpp}`: 1413/1413 assertions against the real 27 GB tree, all 1012 tensors accounted, 50 mutations proven to fire, native arm refused by name; `MaterializeWeightNorm` relocated into the shared `vocoder1d` primitives (H3 79/79 cases, 57,395 assertions unchanged) rather than forked (#714). Spec committed BEFORE implementation per AGENTS.md. Geometry MEASURED, not inferred: every component `config.json` read, plus safetensors headers by HTTP range request β€” `transformer` 231 tensors/1.240B per shard **all F32** (so the card's 2.4B is right and 9.73 GB is fp32 storage, NOT a 4.9B bf16 model), `rvq_depth_decoder` 47/0.646B BF16, `vocoder` 121/0.054B F32 with `weight_g`/`weight_v` weight-norm, `condition_encoder` **4 tensors** (`layer_scale`, `layer_weight_logits`, `proj`) which makes it a learned weighted mix over 8 LLM layers rather than an encoder tower. `language_model` is our LANDED `Qwen3ForCausalLM` architecture exactly (36L/4096/32-8/head_dim 128) at `vocab_size 200000`. The repo ships TWO packagings of ONE set of weights (native `qwen_7B/`+`flowmatching_vae.pth`+`dav.pth` vs the diffusers six); `scripts/convert_minimax_music3_to_diffusers.py@c6da9936` `:29-38,47-189` proves the conversion is a pure re-layout, so the diffusers arm is the port target at **~28.5 GB resident** (no pickle, no `trust_remote_code`) and SGLang-Omni stays a valid e2e cross-check. **Sample rate RESOLVED 2026-08-13** (spec Β§1.1): not a contradiction but a stage boundary. The vocoder natively emits 44100 stereo (86.133 Hz latent frames x 512 upsampling, derived from `minimax_music3_vocoder.py:84,92-95` and confirmed by `dav.py:94,115`); diffusers returns that unresampled, while SGLang-Omni's SERVER resamples 44100 -> 32000 on delivery (`constants.py:18-19`, `acoustic.py:55-58,422-431`). Goldens are captured at 44100 stereo, the native resample-free rate; 32 kHz is a downstream delivery transform gated separately, and NOT free β€” reproducing its bytes needs torchaudio's default sinc filter, and a latent-parity check sits upstream of that call and cannot see it. **W2/W3 landed 2026-08-14 (#672)**: the autoregressive half's compute β€” prompt assembly, the semantic CFG logit pipeline, the learned 8-layer condition mix and the 4-layer RVQ depth decoder β€” gated at reduced dimensions in float32 against upstream's own executed classes (25 cases / 338 assertions, no checkpoint) AND at full scale in bf16 against the committed oracle capture (condition mix 175 989 of 176 128 values BIT-IDENTICAL; depth decoder 716 800 values over 25 frames x 7 depth steps). **Spec Β§5's token-exact claim is WITHDRAWN and the reason is measured**: upstream's AR stage has NO greedy path (`_sample_top_k` ends in `torch.multinomial` against a seeded generator, `encoders.py:94-103`) and both stages sample from a CFG mix whose unconditional row the goldens do not store, so `rvq_codes.npy` is a seeded sample consumed as an INPUT. The full-scale bound is calibrated against a MATCHED CONTROL β€” torch's own `sdpa_kernel(MATH)` arm reproduces the goldens to 46.34% bit-identical at mean absolute error 1.659e-03, ours 43.61% / 1.824e-03 β€” because torch's CPU attention kernel runs a blocked online softmax no closed-form rounding model reproduced (its bf16 Linear reproduces 32 759 of 32 768). **Still owed on W2**: the 8.6B `Qwen3ForCausalLM` forward teacher-forced on the golden codes, which needs an `inputs_embeds` entry the landed Qwen3 dense path does not have | `CLAIM-MODEL-MUSIC3-W0` | | `MODEL-MM-moss-transcribe-diarize-moss-transcribe-diarize-for-conditional-generation` | `MossTranscribeDiarizeForConditionalGeneration` (v0.25.0 target-pending) | v0.25.0 target `registry.py:450-453`; `vllm/model_executor/models/moss_transcribe_diarize.py::MossTranscribeDiarizeForConditionalGeneration` @ `702f481` | conditional generation / audio | MM processor; Whisper encoder; VQ adaptor; Qwen3 decoder; speech-to-text/diarization frontend | ☐ required | `INVENTORIED` | none | unassigned | | `MODEL-MM-lightonocr-light-on-ocrfor-conditional-generation` | `LightOnOCRForConditionalGeneration` | `registry.py:450-453`; `vllm/model_executor/models/lightonocr.py::LightOnOCRForConditionalGeneration` | conditional generation / image | MM processor; encoder/merge; vision encoder | ☐ required | `INVENTORIED` | none | unassigned | | `MODEL-MM-lfm2-vl-lfm2-vlfor-conditional-generation` | `Lfm2VlForConditionalGeneration` | `registry.py:454`; `vllm/model_executor/models/lfm2_vl.py::Lfm2VLForConditionalGeneration` | conditional generation / image | MM processor; encoder/merge; Mamba/SSM state; vision encoder; video path | ☐ required | `INVENTORIED` | none | unassigned | diff --git a/.agents/specs/minimax-music3.md b/.agents/specs/minimax-music3.md index 8281c71c8..bc8a94257 100644 --- a/.agents/specs/minimax-music3.md +++ b/.agents/specs/minimax-music3.md @@ -10,7 +10,7 @@ [#14456](https://github.com/huggingface/diffusers/pull/14456), head `c6da9936e4bda83107943a16eb8682e9a37d8527` β€” **OPEN, not merged**. **Cross-check:** SGLang-Omni `748a0b437e4a8faad44d7bbfd5a0ae55d1fef830`. -**Status:** **W0 DONE, W1 LANDED.** Spec committed, both oracles pinned, Β§1.1 resolved and confirmed at runtime, the diffusers oracle gateable against committed goldens, and the modular loader in the tree. W2/W3 dispatched. +**Status:** **W0 + W1 DONE, W3 DONE, W2 PARTIAL.** Spec committed, both oracles pinned, Β§1.1 resolved and confirmed at runtime, the diffusers oracle gateable against committed goldens, the modular loader in the tree, and the autoregressive half's compute gated at reduced dimensions and against the real bf16 checkpoint. Β§5's token-exact gate is WITHDRAWN: upstream's AR stage has no greedy path. The 8.6B language-model forward and W4-W7 are owed. **Developer directive (2026-08-13):** "land minimax music 3 support complete, to vllm.cpp, wired to the ABI and to the example http server, merge to main, tested e2e." That fixes W6's shape (the ABI surface and the example server are in scope, @@ -292,9 +292,55 @@ LLMβ†’diffusion handoff on *continuous hidden states* rather than discrete token ## 5. Gates -**LLM half β€” token-exact.** The global LLM and the depth decoder emit discrete RVQ -codes. Greedy decode of the code sequence is compared against the oracle -token-for-token on a fixed prompt. This is a real token gate and it binds. +**LLM half β€” token-exact. WITHDRAWN 2026-08-14 by W2/W3; the artifact refuted +it.** What this paragraph said was: "The global LLM and the depth decoder emit +discrete RVQ codes. Greedy decode of the code sequence is compared against the +oracle token-for-token on a fixed prompt. This is a real token gate and it +binds." It is kept in full, because a withdrawn claim that leaves no trace is how +the same wrong gate gets re-specified. + +**There is no greedy decode of this model to compare against.** `_sample_top_k` +(`encoders.py:94-103`) is the only sampler either stage uses; `_AR_SAMPLING_TOP_K` +is a module constant of 50, there is no temperature and no argmax branch, and the +last line is `torch.multinomial(probs, 1, generator=generator)`. The committed +`rvq_codes.npy` is therefore a **seeded sample**, and reproducing it +token-for-token means reproducing torch's CPU Mersenne-Twister and its +multinomial β€” a claim about torch's RNG, not about this model. + +A second, independent reason the same conclusion holds, and the one that would +survive even a bit-exact RNG: **both** stages sample from a CFG mix of a +conditional and an unconditional row (`encoders.py:327-328`, `:134-135`), and the +goldens store the **conditional row only** (`encoders.py:132,343`, both slice +`[:1]`). The unconditional branch is not in the golden set, so the guided +distribution the codes were drawn from cannot be reconstructed from what is +committed. + +**What replaces it.** The codes are consumed as INPUTS and the AR half is gated +on TENSORS, at two scales: + +* reduced dimensions, float32, against goldens produced by *executing* upstream's + own `MiniMaxMusic3ConditionEncoder` and `MiniMaxMusic3RVQDepthDecoder` + (`scripts/gen-minimax-music3-ar-goldens.py`). This separates an algebra defect + from rounding, and it runs in CI with no checkpoint; +* full scale, bf16, real weights: the condition mix against + `condition_chunk0.npy` (176 128 values) and the depth decoder against + `frame_hiddens[:, 4096:]` (716 800 values), driven by the golden codes and the + golden `last_hidden`. + +The full-scale bound is calibrated against a **matched control** rather than +guessed. torch's own `sdpa_kernel(MATH)` arm, running upstream's own module on +the identical inputs, reproduces the goldens to 46.34% bit-identical at mean +absolute error 1.659e-03 β€” its CPU attention kernel runs a blocked online softmax +that no closed-form rounding model reproduced. Ours is 43.61% and 1.824e-03, +inside that spread. Chasing a particular kernel's rounding below the control is +not "more correct" (AGENTS.md's near-tie discipline). + +**Still owed on the LLM half:** the 8.6B `Qwen3ForCausalLM` forward itself. +`frame_hiddens[:, :4096]` is the language model's own hidden state, and +reproducing it means running that model teacher-forced on the golden codes +through our landed Qwen3 path, which needs an `inputs_embeds` entry it does not +have. That is the remainder of W2 and it is recorded here rather than discovered +later. **Acoustic half β€” per-stage tensor parity.** No logits, no sampler, so no token gate exists to have. Each stage is compared against the oracle's own output for @@ -372,15 +418,41 @@ pinned SHA. ## Now -**W0 DONE, W1 LANDED, row `ACTIVE`.** The diffusers oracle generates audio and is -`gateable = yes` against 13 committed per-stage goldens; both oracles are pinned; -Β§1.1 is resolved and confirmed at runtime; the modular loader is in the tree with -the dtype invariant Β§2.1 enforced and the native arm refused by name. - -W2/W3 (the autoregressive half) are dispatched. Then W4/W5 acoustic, W6 the -speech-family registration plus the `vllm_speech_*` ABI and the example HTTP -server, W7 the quantized arms. - -Two things are owed and neither is W0's to close: **no speed number exists** β€” the -oracle capture was CPU-only because `dgx.casa` was down, so nothing here touches -the speed axis β€” and SGLang-Omni remains `gateable = no`, read but never executed. +**W0 + W1 DONE, W3 DONE, W2 PARTIAL; row `ACTIVE`.** The diffusers oracle +generates audio and is `gateable = yes` against 13 committed per-stage goldens; +both oracles are pinned; Β§1.1 is resolved and confirmed at runtime; the modular +loader is in the tree with the dtype invariant Β§2.1 enforced and the native arm +refused by name. W2/W3 add the autoregressive half's compute β€” +[`minimax_music3_ar.h`](../../include/vllm/model_executor/models/minimax_music3_ar.h) +and its two gates. Nothing generates a song yet. + +**W3 is complete and gated at both scales.** The learned 8-layer condition mix +reproduces `condition_chunk0.npy` to 175 989 of 176 128 values **bit-identical** +(mean absolute error 1.99e-07, no value beyond one bf16 ULP-or-2^-7), and the +4-layer RVQ depth decoder reproduces `frame_hiddens[:, 4096:]` β€” 716 800 values +over 25 frames Γ— 7 depth steps β€” inside the matched control's spread (Β§5). The +16-position window is exercised at its boundary and one past it. The reduced +dimension gate is 25 cases / 338 assertions and needs no checkpoint. + +**W2 is partial, and the split is exact.** Everything the autoregressive loop +does *around* the language model has landed and is gated: the prompt the +checkpoint contract fixes (both upstream rewrite passes, string for string, on +the oracle capture's own prompt), the unconditional CFG row, the frame budget and +its two refusals, the semantic vocabulary mask, the guided-logit pipeline +including the re-mask that keeps a NaN from becoming a candidate, `_sample_top_k` +up to its draw, and the frame feedback embedding. What has NOT landed is the +8.6B `Qwen3ForCausalLM` forward itself β€” see Β§5's "still owed". + +**Β§5's token-exact claim is withdrawn**, and that is this phase's most important +finding rather than a footnote: upstream's AR stage has no greedy path at all, so +`rvq_codes.npy` is a seeded sample and is consumed as an input by these gates. +Β§5 now records the reasoning and the tensor gates that replace it. + +Next: W4/W5 acoustic, W6 the speech-family registration plus the `vllm_speech_*` +ABI and the example HTTP server, W7 the quantized arms. W4 is unblocked and does +not depend on W2's remainder. + +Two things are owed and neither is this phase's to close: **no speed number +exists** β€” every capture so far ran on CPU because `dgx.casa` was down, so +nothing here touches the speed axis β€” and SGLang-Omni remains `gateable = no`, +read but never executed. diff --git a/CMakeLists.txt b/CMakeLists.txt index 81add6235..2683039a1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -768,6 +768,7 @@ add_library(vllm STATIC src/vllm/model_executor/models/minimax_h3_audio_vae.cpp src/vllm/model_executor/models/vocoder1d.cpp src/vllm/model_executor/models/minimax_music3_loader.cpp + src/vllm/model_executor/models/minimax_music3_ar.cpp src/vllm/model_executor/models/gpt2.cpp src/vllm/multimodal/speech_engine.cpp src/vllm/model_executor/models/indextts2.cpp diff --git a/docs/FEATURES.md b/docs/FEATURES.md index 634f8869b..af667dd24 100644 --- a/docs/FEATURES.md +++ b/docs/FEATURES.md @@ -163,7 +163,7 @@ in `ltx2_text_encoder.cpp` is the call that would have to change. | Whisper audio encoder | openai/whisper-small; whisper-large-v3 (Voxtral cfg) | encoder tower 77/77; large-v3 tower 203/203 | pending | | MiniMax-H3 DiT (`MiniMaxH3DiTModel`, vllm-omni lane) | MiniMax-H3 (33.1B video+audio) | portable 79/79; all three modalities COHERENT on Q4_K_M (Β§8.20); PRUNED ckpts run, Q8_0 seam 0.9941 (Β§8.21); ref2va grid was NVFP4 quant error, Β§8.9 REFUTED; GGUF/NVFP4/bf16 shards stream | FP4/Marlin landed; speed pending; no bf16 render yet. Render from the Q4_K_M GGUF, not the NVFP4 arm. Krea 2 text-to-image (roadmap C11) is scoped to reuse these DiT seams | | LTX-2.5 DiT (`LTX2VideoTransformer3DModel`, Lightricks lane) | LTX-2.5 (21.00B video+audio) | `SPIKE`. DiT, VAEs+ENCODERS, conditioning, pipeline, quant loaders gated at reduced dims. Prompt-side AdaLN ported, host+device. Typed prompt to Gemma-4 to cross-attn, FIXTURE-gated. A prompted render is OWED | Family `ltx-2.5`, `ltx2-gen`. ~29 GB NVFP4/GB10, FP8 ~44 GB, +~24 GB tower. FP8/torchao/1st-party NVFP4 load; `keyframes_abs_pos_embedding` alone needs `allow_unported`. DiffVAE, LoRA, image cond refused. Speed PENDING | -| MiniMax-Music3 (`MiniMaxMusic3ForConditionalGeneration`, diffusers lane) | MiniMax-Music3 (8.6B Qwen3 LLM + 0.646B RVQ decoder + 2.4B fp32 DiT + DAC Flow-VAE); diffusers arm, ~28.5 GB | `ACTIVE`. Oracle gateable with 13 per-stage goldens (#708); modular loader gated 1413/1413 on the real 27 GB checkpoint, all 1012 tensors accounted (#714). Output 44100 Hz stereo. No stage runs yet: W2-W7 owed | Not measured. The denominator will be SGLang-Omni in its production configuration (both CUDA graphs, compiled DIT and DAV, batched seeded sampling) | +| MiniMax-Music3 (`MiniMaxMusic3ForConditionalGeneration`, diffusers lane) | MiniMax-Music3 (8.6B Qwen3 LLM + 0.646B RVQ decoder + 2.4B fp32 DiT + DAC Flow-VAE); diffusers arm, ~28.5 GB | `ACTIVE`. Loader gated 1413/1413 on the real checkpoint (#714); W2/W3 AR half computes and is gated vs the real bf16 weights. No greedy path upstream, so the token-exact gate is withdrawn. W4-W7 owed | Not measured. The denominator will be SGLang-Omni in its production configuration (both CUDA graphs, compiled DIT and DAV, batched seeded sampling) | | MTP speculator | Qwen3.6-27B, Qwen3.6-35B-A3B | token-identical to vLLM `mtp` at c1 | ~4% faster c1; +16% output tput (MoE) | | DFlash block-diffusion | Qwen3 (DFlash draft) | near-tie e2e 27/27 vs vLLM | 2.9x over spec-off, 1.003x vs vLLM DFlash-on | | DeepSeek-V4 MTP | DeepSeek-V4-Flash (nextn head) | lossless 5/5; real-model weight-blocked | pending | @@ -197,7 +197,7 @@ on the committed fixture); reranking/classify models are not yet registered. | Audio | βœ… correctness-gated | βœ… | ◐ | ◐ | | Video+audio GENERATION (MiniMax-H3 DiT, LTX-2.5 DiT) | ◐ H3: all three modalities COHERENT on Q4_K_M (t2va, fl2va, ref2va; Β§8.20); the NVFP4 arm carries the patch grid; GGUF/NVFP4/bf16 loaders, pruned too (Β§8.21). LTX-2.5: a second lane, `SPIKE`, gated at reduced dims | βœ… H3 (vllm-omni, BF16-only, no quantized arm); LTX-2.5 only through the generic diffusers adapter, no native recipe ([vllm-omni#6066](https://github.com/vllm-project/vllm-omni/issues/6066)) | ☐ | ☐ | | Speech / audio GENERATION (TTS, vLLM-Omni lane) | ◐ IndexTTS-2.5 only: the S2Mel DiT is COMPLETE front to tail and gated against upstream; its tail runs on REAL shipped weights. Other stages gated at reduced dims. No render, no route (#634) | βœ… (vllm-omni: MOSS-TTS, Qwen3-TTS, Higgs Audio v3, Voxtral TTS, IndexTTS-2.5) | not assessed | not assessed | -| MUSIC generation (MiniMax-Music3) | ☐ not generating. The W1 checkpoint LOADER has landed ([spec](../.agents/specs/minimax-music3.md), #672); no stage runs yet. Lyrics plus a structured description in, a multi-minute stereo song out | ☐ absent from the pin, from vLLM `main` and from `vllm-omni` alike | ◐ served by SGLang-Omni, a third repository, which loads the NATIVE checkpoint layout | ☐ | +| MUSIC generation (MiniMax-Music3) | ☐ not generating. W1 loader + W2/W3 AR half landed ([spec](../.agents/specs/minimax-music3.md), #672); the language-model forward and every acoustic stage are not. Lyrics + a structured description in, a stereo song out | ☐ absent from the pin, from vLLM `main` and from `vllm-omni` alike | ◐ served by SGLang-Omni, a third repository, which loads the NATIVE checkpoint layout | ☐ | | Multimodal over the OpenAI server | ◐ image request path wired, forward pending | βœ… | βœ… | ◐ | Image, video and audio are correct through the CLI and library. Over the HTTP diff --git a/docs/USAGE.md b/docs/USAGE.md index 5621cac86..d16855c12 100644 --- a/docs/USAGE.md +++ b/docs/USAGE.md @@ -2367,3 +2367,88 @@ python3 scripts/convert-indextts2-checkpoint.py \ VLLM_CPP_INDEXTTS2_S2MEL=$CHECKPOINT_ROOT/IndexTTS-2.5-safetensors/s2mel.safetensors \ ./build/tests/test_indextts2_s2mel_loader ``` + +## MiniMax-Music3: the autoregressive half + +Phases W2 and W3 of #672. +`include/vllm/model_executor/models/minimax_music3_ar.h` is what consumes three +of W1's six components: the prompt the `language_model` is driven with, the +semantic stage's classifier-free-guidance logit pipeline, the learned 8-layer +condition mix, and the 4-layer RVQ depth decoder. **It still does not generate a +song** β€” the DiT, the scheduler and the vocoder are W4–W5, and the 8.6B +`Qwen3ForCausalLM` forward itself is the remainder of W2. + +### The token gate the spec promised does not exist + +Worth stating plainly, because the spec said otherwise until this phase measured +it. MiniMax-Music3's autoregressive stage has **no greedy path**: +`_sample_top_k` (`encoders.py:94-103`) is the only sampler either stage uses, it +has no temperature and no argmax branch, and it ends in +`torch.multinomial(probs, 1, generator=generator)`. The oracle's +`rvq_codes.npy` is a *seeded sample*, so matching it token-for-token would be +reproducing torch's RNG rather than this model. Independently: both stages sample +from a CFG mix of a conditional and an unconditional row, and the goldens store +the conditional row only, so the guided distribution is not reconstructible from +what is committed. + +The codes are therefore **inputs** to these gates, and the AR half is gated on +tensors. + +### Running the gates + +The reduced-dimension gate needs no checkpoint. Its goldens come from executing +upstream's own `MiniMaxMusic3ConditionEncoder` and `MiniMaxMusic3RVQDepthDecoder` +at small dimensions in float32, so it isolates an algebra defect from rounding: + +```sh +cmake -S . -B build -DVLLM_CPP_BUILD_TESTS=ON +cmake --build build -j 8 --target test_minimax_music3_ar +./build/tests/test_minimax_music3_ar +``` + +The full-scale gate drives the real bf16 weights on the oracle capture's own +inputs and skips loudly without the checkpoint: + +```sh +VLLM_CPP_MUSIC3_CHECKPOINT=/path/to/minimax-music3 \ + ./build/tests/test_minimax_music3_ar_real +``` + +It compares 176 128 values for the condition mix (against `condition_chunk0.npy`) +and 716 800 for the depth decoder (against `frame_hiddens[:, 4096:]`, 25 frames Γ— +7 depth steps), and it reports the counts rather than only a verdict. + +Regenerate the reduced-dimension goldens with the pinned oracle's interpreter +(see `tools/oracle/README.md`) after an upstream change: + +```sh +~/venvs/music3-oracle/bin/python scripts/gen-minimax-music3-ar-goldens.py \ + --out tests/vllm/models/minimax_music3_ar_goldens.inc +``` + +### Two things that will bite a later phase + +**The code rows are offset by one from the frames.** `rvq_codes.npy` is `[26, 8]` +and `frame_hiddens` is `[25, ...]`: row 0 of the codes is the priming decode step, +which emits no frame (`encoders.py:342`). `rows[1:]` align with the frames. +Comparing the unshifted sequences yields two individually plausible tensors and a +wrong gate. + +**`ArCompute` is not a precision knob.** The autoregressive half runs bf16, and a +bf16 torch module rounds at *every* op boundary, so an fp32 host forward is a +different computation rather than a more precise one β€” measured, it leaves +448 450 of 716 800 values beyond one bf16 ULP. `ArCompute::kBFloat16` mirrors the +rounding; `kFloat32` is the reduced-dimension goldens' dtype. A caller at +`kBFloat16` also owes its weights at bf16, *including* the condition encoder, +whose file is fp32 while its runtime is not. + +And bit-exactness against torch is not on offer here, which is worth knowing +before a later phase spends a day chasing it. torch's bf16 `nn.Linear` on CPU +reproduces to 32 759 of 32 768 values, but its dispatched attention reproduces to +only 25 736: the CPU kernel runs a blocked online softmax, and four candidate +rounding models (pre-scaled q, bf16-rounded scores, bf16-rounded probabilities, +and their combinations) were all *worse* than the plain form. The full-scale +bound is therefore +calibrated against torch's own `sdpa_kernel(MATH)` arm on the identical inputs +(46.34% bit-identical, mean absolute error 1.659e-03) rather than against a +bit-exactness that no second implementation can reach. diff --git a/include/vllm/model_executor/models/minimax_music3_ar.h b/include/vllm/model_executor/models/minimax_music3_ar.h new file mode 100644 index 000000000..36b76cc0a --- /dev/null +++ b/include/vllm/model_executor/models/minimax_music3_ar.h @@ -0,0 +1,394 @@ +// MiniMax-Music3 β€” the AUTOREGRESSIVE half (W2 + W3 of #672). +// +// Row MODEL-MUSIC-minimax-music3-mini-max-music3-for-conditional-generation, +// .agents/specs/minimax-music3.md phases W2 and W3. Issue #672. +// +// W1 (minimax_music3_loader.h) resolves the six-component diffusers checkpoint +// and materializes its tensors. This header is what CONSUMES three of those +// components: the prompt the `language_model` is driven with, the semantic +// stage's classifier-free-guidance logit pipeline, the learned 8-layer condition +// mix, and the 4-layer RVQ depth decoder. +// +// ─── THE SHAPE OF THE AUTOREGRESSIVE STAGE ────────────────────────────────── +// +// encoders.py:281-353. Per frame, TWO models run: +// +// 1. the GLOBAL language model (a stock `Qwen3ForCausalLM` at vocab 200000) +// emits one SEMANTIC code, under classifier-free guidance between a +// conditional and an unconditional prompt row; +// 2. the RVQ DEPTH DECODER expands that one code into the SEVEN residual +// codebooks, one at a time, and its per-step hidden states β€” concatenated +// with the language model's own β€” are the `frame_hiddens` the acoustic +// half is conditioned on. +// +// Then `_embed_audio_frame` folds the whole 8-codebook frame back into ONE +// language-model input embedding and the loop advances. +// +// ─── WHAT IS A GATE HERE AND WHAT IS NOT β€” READ THIS BEFORE TRUSTING Β§5 ───── +// +// The spec's Β§5 says the LLM half is "token-exact": "Greedy decode of the code +// sequence is compared against the oracle token-for-token." MEASURED AGAINST +// THE ARTIFACT, THAT IS NOT AVAILABLE, and the artifact wins. +// +// Upstream's autoregressive stage HAS NO GREEDY PATH. `_sample_top_k` +// (encoders.py:94-103) is the only sampler either stage uses, `_AR_SAMPLING_TOP_K` +// is a module constant of 50 with no temperature and no argmax branch, and the +// final step is `torch.multinomial(probs, 1, generator=generator)`. So the +// committed `rvq_codes.npy` is a SEEDED SAMPLE, not an argmax, and reproducing +// it token-for-token means reproducing torch's CPU Mersenne-Twister and its +// multinomial, not reproducing this model. +// +// A second, independent reason the same conclusion holds: BOTH stages sample +// from a CFG mix of a conditional and an UNCONDITIONAL row (encoders.py:327-328, +// :134-135), and `frame_hiddens` stores the conditional row only +// (encoders.py:343 `last_hidden[:1]`, :132 `hidden[:1]`). The unconditional +// branch is not in the golden set at all, so even with a bit-exact RNG the codes +// could not be re-derived from what is committed. +// +// What IS therefore gated, and what this header is built around: +// +// * every DETERMINISTIC stage, against the reduced-dimension float32 goldens +// produced by executing upstream's own classes +// (scripts/gen-minimax-music3-ar-goldens.py); +// * the depth decoder at FULL SCALE with REAL weights, driven by the golden +// `last_hidden` and the golden codes, against the golden hidden states β€” +// which is 25 x 7 x 4096 = 716800 real numbers and cannot be passed by an +// implementation that has any of the algebra wrong; +// * the condition mix at FULL SCALE against `condition_chunk0.npy`. +// +// The codes are consumed as INPUTS by those gates rather than predicted. That is +// a weaker claim than Β§5 makes and it is the true one; the spec is corrected in +// its `## Now`, not worked around here. +// +// ─── DTYPE IS A PARAMETER, BECAUSE UPSTREAM'S IS ──────────────────────────── +// +// The gated configuration runs all three AR components in BF16 (spec Β§2.1 as +// corrected by the oracle; `MiniMaxMusic3ResolveRuntimeDtypes`). A bf16 torch +// module rounds at EVERY op boundary, so an fp32 host reference is not "the same +// computation more precisely" β€” it is a different one, and over the depth +// decoder's four layers the two diverge by ~1.3 bf16 ULP on average. MEASURED: +// an fp32 forward against the committed bf16 goldens leaves 448450 of 716800 +// values beyond one ULP with a mean absolute error of 2.65e-03; mirroring the +// rounding brings the same forward to the numbers the gate now records. +// +// This is the "do not inherit a WIDER dtype" rule of AGENTS.md in its awkward +// direction: nothing here is numerically wrong at fp32, so a token gate could +// never see it, and only a tensor gate against the real weights can. +// +// So `ArCompute` is threaded through every function that has an op boundary. +// `kFloat32` is the default because that is what the reduced-dimension goldens +// are captured in; `kBFloat16` is what the SHIPPED checkpoint runs and what the +// full-scale gate uses. Neither is a widening of a stored value: the caller +// converts the checkpoint's tensors once (`MiniMaxMusic3LoadComponent`), and at +// `kBFloat16` it owes them at bf16 β€” including the condition encoder, whose FILE +// is fp32 while its RUNTIME is bf16. +#pragma once + +#include +#include +#include + +namespace vllm { +namespace models { +namespace music3 { + +// Which dtype the AR half's op boundaries round to. See the header note: this +// mirrors a real property of the run, not a precision preference. +enum class ArCompute { + // The reduced-dimension goldens' dtype. Every accumulation is a double and + // each result is rounded once to float32. + kFloat32, + // The SHIPPED configuration. Accumulation is still a double β€” torch's bf16 + // matmul accumulates in float32 and its RMSNorm variance in float32 β€” but the + // RESULT of each op is rounded to bf16, which is what torch stores. + kBFloat16, +}; + +// --------------------------------------------------------------------------- +// The checkpoint contract's constants (encoders.py:38-48) +// --------------------------------------------------------------------------- + +// The three tokens the prompt template and the CFG rewrite name explicitly. +inline constexpr int32_t kAudioEndTokenId = 151670; +inline constexpr int32_t kAudioCfgTokenId = 151654; +inline constexpr int32_t kAudioCodeOffset = 151675; +// The semantic codebook occupies [kAudioCodeOffset, +kSemanticVocabSize) of the +// language model's 200000-entry vocabulary; everything else is masked out. +inline constexpr int64_t kSemanticVocabSize = 16384; + +inline constexpr int64_t kMaxPromptTokens = 5000; +inline constexpr int64_t kMaxAudioFrames = 9000; + +// The reference inference recipe's fixed sampling parameters (encoders.py:46-48). +// `kArCfgScale` applies to BOTH stages; `kArCfgTopK` restricts the semantic +// stage's guided distribution to the CONDITIONAL row's top candidates before +// `kArSamplingTopK` filters the guided one. +inline constexpr double kArCfgScale = 1.5; +inline constexpr int64_t kArCfgTopK = 50; +inline constexpr int64_t kArSamplingTopK = 50; + +// --------------------------------------------------------------------------- +// Prompt assembly (encoders.py:54-91, :207-218) +// --------------------------------------------------------------------------- + +// The literal template pieces. Upstream's own comment (encoders.py:32-33) is +// that "even whitespace-level changes to the assembled prompt change the +// generated audio", which is why these are constants and not formatting. +inline constexpr const char* kImStart = "<|im_start|>"; +inline constexpr const char* kImEnd = "<|im_end|>"; +inline constexpr const char* kCaptionStart = "<|caption_start|>"; +inline constexpr const char* kCaptionEnd = "<|caption_end|>"; +inline constexpr const char* kLyricsStart = "<|lyrics_start|>"; +inline constexpr const char* kLyricsEnd = "<|lyrics_end|>"; +inline constexpr const char* kAudioStart = "<|audio_start|>"; + +// `_clean_caption` (encoders.py:54-77). Rewrites `<|k v|>` to "k is v", strips +// the markdown forms the input contract accepts (ATX headings, list bullets, +// bold, italic, horizontal rules), and collapses blank runs. +std::string CleanCaption(const std::string& caption); + +// `_normalize_lyrics` (encoders.py:80-91). Keeps only the LEADING structural +// tags of a line and drops any text sharing that line, then splits on "] " and +// " [" and " ^ ", lower-cases every `[Tag]`, and prefixes "[start]\n". +// +// THE REPLACEMENT ORDER IS LOAD-BEARING: "] " runs before " [", so +// "tail [outro] ^ after caret" becomes "tail\n[outro]\n^ after caret" and the +// " ^ " rule then matches nothing. Swapping the two produces a different, +// entirely plausible prompt. +std::string NormalizeLyrics(const std::string& lyrics); + +// The assembled prompt string (encoders.py:207-210). +std::string AssembleArPrompt(const std::string& prompt, const std::string& lyrics); + +// The unconditional row (encoders.py:216-217): a copy with every token but the +// FIRST and the last TWO replaced by `kAudioCfgTokenId`. Throws when the ids are +// too short for that slice to be well defined. +std::vector UnconditionalPromptIds(const std::vector& ids); + +// `min(int(audio_duration * frame_rate), kMaxAudioFrames)` (encoders.py:287). +// Throws when the duration is not positive or rounds to zero frames, mirroring +// upstream's two `ValueError`s rather than generating silence. +int64_t MaxArFrames(double audio_duration_s, double frame_rate); + +// --------------------------------------------------------------------------- +// The semantic stage's logit pipeline (encoders.py:318-334) +// --------------------------------------------------------------------------- + +// The vocabulary mask, TRUE where a token is BLOCKED. Only the semantic code +// window and the audio-end token survive (encoders.py:318-320). +std::vector SemanticVocabMask(int64_t vocab_size, int64_t code_offset, + int64_t semantic_vocab_size, int32_t end_token_id); + +// The guided semantic logits. `conditional` and `unconditional` are the two raw +// lm_head rows; the mask is applied to BOTH before guidance, the guided row is +// then restricted to the CONDITIONAL row's top `cfg_top_k` and re-masked. +// +// THE RE-MASK IS NOT REDUNDANT (encoders.py:329-333): guidance on two -inf +// logits is (-inf) + (-inf - -inf) * 1.5 = NaN, so without the second mask every +// blocked position becomes a NaN that `_sample_top_k`'s `nan_to_num` would turn +// into a finite -1e9 candidate rather than an impossible one. +std::vector GuidedSemanticLogits(const std::vector& conditional, + const std::vector& unconditional, + const std::vector& blocked, + int64_t cfg_top_k, double cfg_scale); + +// The guided DEPTH logits (encoders.py:134-135): the same mix with no vocabulary +// mask and no top-k pre-restriction, because the residual codebooks span their +// whole `audio_vocab_size`. +std::vector GuidedDepthLogits(const std::vector& conditional, + const std::vector& unconditional, + double cfg_scale); + +// `_sample_top_k` MINUS its final `torch.multinomial` (encoders.py:94-100): the +// sanitize / top-k mask / softmax / renormalize that produce the categorical +// distribution. Returns probabilities summing to 1. +// +// The draw itself is deliberately NOT here. It is `torch.multinomial` against a +// seeded `torch.Generator`, so reproducing it is reproducing torch's RNG; see +// the header note on why no gate in this port claims to. +std::vector TopKProbabilities(const std::vector& logits, int64_t top_k); + +// --------------------------------------------------------------------------- +// The condition mix (condition_embedder_minimax_music3.py:48-76) +// --------------------------------------------------------------------------- + +struct ConditionMixConfig { + int64_t condition_hidden_dim = 4096; + int64_t num_condition_layers = 8; + int64_t out_dim = 2048; + int64_t input_sampling_rate = 24000; + int64_t input_hop_length = 960; + int64_t output_sampling_rate = 44100; + int64_t output_hop_length = 512; +}; + +struct ConditionMixWeights { + std::vector layer_weight_logits; // [num_condition_layers] + std::vector layer_scale; // [1] + std::vector proj_weight; // [out_dim, condition_hidden_dim, 3] + std::vector proj_bias; // [out_dim] +}; + +// The latent timeline's length (condition_embedder_minimax_music3.py:65-74). +// `max(1, int(frames * out_rate / in_rate * in_hop / out_hop))`, and the +// intermediate is a DOUBLE that is truncated once at the end β€” computing it as +// integer ratios rounds 86.13 to 86 for the wrong reason and to 87 for the +// wrong inputs. +int64_t ConditionLatentLength(int64_t num_frames, const ConditionMixConfig& config); + +// `hidden_states` is [frames, num_condition_layers * condition_hidden_dim] in +// LAYER-MAJOR order (the transpose+reshape of :59-60 makes the layer the SLOW +// axis). Returns [latent_length, out_dim]. +// +// Three steps that each have a plausible wrong form: a softmax over the layer +// logits (not a normalize, not a plain weighting), ONE scalar `layer_scale` +// applied to the mixed result (not per layer), and a k=3 p=1 Conv1d over TIME +// (not a pointwise projection). Then NEAREST interpolation to the latent rate. +std::vector ConditionMix(const std::vector& hidden_states, + int64_t num_frames, const ConditionMixConfig& config, + const ConditionMixWeights& weights, + ArCompute compute = ArCompute::kFloat32); + +// The softmax over the layer logits, exposed so a gate can see the mix weights +// rather than only their effect. +std::vector ConditionLayerWeights(const std::vector& layer_weight_logits, + ArCompute compute = ArCompute::kFloat32); + +// `F.interpolate(..., mode="nearest")` along the last axis, for [channels, in_len] +// row-major input. Upstream's source index is `floor(dst * in_len / out_len)` +// clamped to `in_len - 1` β€” the ratio is INPUT over OUTPUT, and inverting it is +// a silent off-by-a-scale that still returns the right shape. +std::vector NearestInterpolate1d(const std::vector& in, int64_t channels, + int64_t in_len, int64_t out_len); + +// --------------------------------------------------------------------------- +// The RVQ depth decoder (minimax_music3_rvq_depth_decoder.py:91-142) +// --------------------------------------------------------------------------- + +struct DepthDecoderConfig { + int64_t hidden_size = 4096; + int64_t num_layers = 4; + int64_t num_attention_heads = 16; + int64_t intermediate_size = 6144; + int64_t audio_vocab_size = 1024; + int64_t num_codebooks = 8; + int64_t max_position_embeddings = 16; + + int64_t head_dim() const { return hidden_size / num_attention_heads; } + int64_t residual_codebooks() const { return num_codebooks - 1; } +}; + +struct DepthDecoderLayerWeights { + std::vector input_layernorm; // [H] + std::vector post_attention_layernorm; // [H] + std::vector to_q; // [H, H] row-major (out, in) + std::vector to_k; + std::vector to_v; + std::vector to_out; + std::vector gate_proj; // [I, H] + std::vector up_proj; // [I, H] + std::vector down_proj; // [H, I] +}; + +struct DepthDecoderWeights { + // [audio_vocab_size * (num_codebooks - 1), H] β€” the RESIDUAL codebooks only. + // The semantic codebook is embedded by the LANGUAGE MODEL, not here + // (encoders.py:126), which is the asymmetry the table's first dimension + // records. + std::vector audio_embeddings; + std::vector projection; // [H, H] + std::vector pos_embedding; // [max_position_embeddings, H] + std::vector norm; // [H] + std::vector layers; + // One [audio_vocab_size, H] head per RESIDUAL codebook. + std::vector> audio_heads; +}; + +// RMSNorm with `eps = 1e-6` and elementwise affine, over the last axis of +// [rows, dim] (minimax_music3_rvq_depth_decoder.py:78). Accumulates in double. +// TWO roundings, not one, and that is upstream's shape rather than ours: +// `x * rsqrt(var + eps)` promotes bf16 to float32 (normalization.py:600-601), +// the result is cast BACK to the weight's dtype (:605), and only then is the +// affine weight applied (:606). Collapsing them to one rounding is a different +// number. +std::vector RmsNorm(const std::vector& x, int64_t rows, int64_t dim, + const std::vector& weight, double eps = 1e-6, + ArCompute compute = ArCompute::kFloat32); + +// `y = x @ W^T` for row-major x [rows, in_dim] and W [out_dim, in_dim] β€” the +// torch `nn.Linear` layout, bias-free everywhere in this decoder. +std::vector LinearNoBias(const std::vector& x, int64_t rows, int64_t in_dim, + const std::vector& weight, int64_t out_dim, + ArCompute compute = ArCompute::kFloat32); + +// The decoder forward (`MiniMaxMusic3RVQDepthDecoder.forward`, :127-142). +// `inputs_embeds` is [seq_len, hidden_size]; returns the post-`norm` hidden +// states at the SAME shape. +// +// Attention is CAUSAL and there is no RoPE and no KV cache: position is carried +// entirely by a learned `pos_embedding` added to the input. Causality is why one +// forward over the whole depth sequence equals upstream's incremental schedule +// step for step β€” verified bit-exactly against the committed goldens. +// +// Throws when `seq_len > max_position_embeddings`: `pos_embedding` has no row to +// return and upstream would index out of bounds. +std::vector DepthDecoderForward(const std::vector& inputs_embeds, + int64_t seq_len, const DepthDecoderConfig& config, + const DepthDecoderWeights& weights, + ArCompute compute = ArCompute::kFloat32); + +// The depth SEQUENCE `_generate_depth_codes` assembles (encoders.py:125-141): +// +// [ projection(last_hidden), +// projection(semantic_embed), +// projection(audio_embeddings(c_i + (i-1) * audio_vocab_size)) for i in 1..n ] +// +// `semantic_embed` is the LANGUAGE MODEL's embedding row for +// `semantic_code + kAudioCodeOffset`, which the caller supplies because this +// component does not own that table. `residual_codes` are c1.., each < the audio +// vocabulary; the OFFSET is what makes one shared table hold seven codebooks, +// and dropping it reads codebook 1's rows for every step. +// +// Returns [2 + residual_codes.size(), hidden_size]. +std::vector DepthSequenceEmbeds(const std::vector& last_hidden, + const std::vector& semantic_embed, + const std::vector& residual_codes, + const DepthDecoderConfig& config, + const DepthDecoderWeights& weights, + ArCompute compute = ArCompute::kFloat32); + +// `audio_heads[head_index](hidden)` for one [hidden_size] state. `head_index` is +// ZERO-based over the residual codebooks, so codebook c_i uses head i-1 +// (encoders.py:133). +std::vector AudioHeadLogits(const std::vector& hidden, int64_t head_index, + const DepthDecoderConfig& config, + const DepthDecoderWeights& weights, + ArCompute compute = ArCompute::kFloat32); + +// The per-frame conditioning row the acoustic half consumes (encoders.py:343): +// `cat(last_hidden, depth_hidden_1..depth_hidden_n)`, i.e. the language model's +// hidden state followed by the depth decoder's states at depth steps 1..n. +// Returns [num_codebooks * hidden_size]. +std::vector FrameHiddenRow(const std::vector& last_hidden, + const std::vector& depth_hidden_states, + int64_t seq_len, const DepthDecoderConfig& config); + +// `_embed_audio_frame` (encoders.py:106-115): the language model's next input +// embedding for a complete frame. +// +// (lm_embed(semantic + kAudioCodeOffset) +// + sum_j audio_embeddings(c_j + j * audio_vocab_size)) * num_codebooks^-0.5 +// +// The `j * audio_vocab_size` here is a ZERO-based offset over the residual +// codes, unlike `DepthSequenceEmbeds`'s `(i-1) *` on a one-based index β€” they +// are the same offsets written from different loop bases, and reading one for +// the other shifts every codebook by 1024 rows. +std::vector EmbedAudioFrame(const std::vector& lm_semantic_embed, + const std::vector& residual_codes, + const DepthDecoderConfig& config, + const DepthDecoderWeights& weights, + ArCompute compute = ArCompute::kFloat32); + +} // namespace music3 +} // namespace models +} // namespace vllm diff --git a/scripts/gen-minimax-music3-ar-goldens.py b/scripts/gen-minimax-music3-ar-goldens.py new file mode 100644 index 000000000..bada8c54f --- /dev/null +++ b/scripts/gen-minimax-music3-ar-goldens.py @@ -0,0 +1,548 @@ +#!/usr/bin/env python3 +"""Emit tests/vllm/models/minimax_music3_ar_goldens.inc. + +The AUTOREGRESSIVE half of MiniMax-Music3 (spec .agents/specs/minimax-music3.md +phases W2 + W3, issue #672): the prompt the checkpoint contract fixes, the +semantic stage's classifier-free-guidance logit pipeline, the learned 8-layer +condition mix, and the 4-layer RVQ depth decoder. + +WHY THIS GENERATOR EXISTS. The committed full-scale goldens under +tests/parity/goldens/minimax_music3_oracle/ need the 28.5 GB checkpoint and are +BF16, so CI can neither run them nor separate an algebra defect from bf16 +rounding. This generator runs upstream's OWN classes at REDUCED dimensions in +FLOAT32 with a name-seeded weight stream, which is the H3 pattern +(gen-minimax-h3-goldens.py) and what spec section 5 asks for: "the exact +correctness gate runs upstream at reduced dimensions on CPU". Nothing but shapes +and float values crosses into the .inc; no weight byte of the real checkpoint is +checked in. + +The oracle is the pinned diffusers PR head: + huggingface/diffusers#14456 @ c6da9936e4bda83107943a16eb8682e9a37d8527 +installed per tools/oracle/README.md. Run it with that venv's interpreter: + + ~/venvs/music3-oracle/bin/python scripts/gen-minimax-music3-ar-goldens.py \\ + --out tests/vllm/models/minimax_music3_ar_goldens.inc + +WHAT IS AND IS NOT AN ORACLE HERE. The condition encoder and the depth decoder +are IMPORTED and EXECUTED (`MiniMaxMusic3ConditionEncoder`, +`MiniMaxMusic3RVQDepthDecoder`), as are the prompt helpers (`_clean_caption`, +`_normalize_lyrics`, the template constants) and `_sample_top_k`. The semantic +stage's guidance block is NOT a function upstream exposes -- it is inline in +`MiniMaxMusic3SemanticGenerationStep.__call__` (encoders.py:318-334) -- so it is +reproduced here with torch ops line for line against that anchor, and the anchor +is cited in the emitted header so a reviewer can diff it. +""" + +from __future__ import annotations + +import argparse +import sys +from pathlib import Path + +import numpy as np +import torch + +_MASK64 = (1 << 64) - 1 + + +def fnv1a64(name: str) -> int: + h = 0xCBF29CE484222325 + for byte in name.encode("utf-8"): + h ^= byte + h = (h * 0x100000001B3) & _MASK64 + return h + + +def splitmix64(x: int) -> int: + x = (x + 0x9E3779B97F4A7C15) & _MASK64 + z = x + z = ((z ^ (z >> 30)) * 0xBF58476D1CE4E5B9) & _MASK64 + z = ((z ^ (z >> 27)) * 0x94D049BB133111EB) & _MASK64 + return z ^ (z >> 31) + + +def music3_rand(name: str, count: int) -> np.ndarray: + """Identical to gen-minimax-h3-goldens.py :: h3_rand and the C++ Music3Rand.""" + seed = fnv1a64(name) + out = np.empty(count, dtype=np.float64) + for i in range(count): + u = splitmix64((seed + i) & _MASK64) + out[i] = ((u >> 11) * (2.0**-53)) * 2.0 - 1.0 + return out + + +def param(name: str, shape, scale: float = 0.5, offset: float = 0.0) -> torch.Tensor: + count = int(np.prod(shape)) if shape else 1 + raw = music3_rand(name, count) * scale + offset + return torch.from_numpy(raw.astype(np.float32).reshape(shape)) + + +# --------------------------------------------------------------------------- +# Reduced dimensions. Small enough to print, large enough that every axis the +# real config exercises is >1 and no two of them are equal (a transposed or +# swapped axis has to be visible). +# --------------------------------------------------------------------------- + +COND = dict( + condition_hidden_dim=6, + num_condition_layers=3, + out_dim=4, + input_sampling_rate=24000, + input_hop_length=960, + output_sampling_rate=44100, + output_hop_length=512, +) +COND_FRAMES = 5 # -> latent_length 17: an UPSAMPLING nearest interpolation. + +# The same module with the rates reversed, so the nearest interpolation is a +# DOWNSAMPLE (7 frames -> 2). Upsample-only coverage cannot see a scale computed +# as output/input instead of input/output. +COND_DOWN = dict(COND, input_sampling_rate=44100, output_sampling_rate=24000, + input_hop_length=512, output_hop_length=960) +COND_DOWN_FRAMES = 7 + +DEPTH = dict( + hidden_size=8, + num_layers=2, + num_attention_heads=2, + intermediate_size=12, + audio_vocab_size=5, + num_codebooks=4, + max_position_embeddings=6, +) + +# The prompt fixtures. The first pair is the one the committed full-scale oracle +# capture used (tests/parity/goldens/minimax_music3_oracle/manifest.json), so the +# assembled string is checkable against a real run. The rest exercise every +# rewrite `_clean_caption` / `_normalize_lyrics` performs. +PROMPT_CASES = [ + ( + "oracle_capture", + "Genre: acoustic pop. BPM: 96. Key: C major. Warm and intimate. Vocals: " + "soft female lead, close and breathy. Arrangement: fingerpicked guitar " + "and soft piano.", + "[verse]\nMorning light filtering through the pine\n", + ), + ( + "markdown_and_tags", + "## Genre\n- **dream pop**\n * *hazy*\n<|mood dark and warm|><|solo|>\n" + "---\nbulletβ€’ x\n\n\nend stop", + "[Verse] dropped words\n[Chorus][Bridge]\nkeep this line\n" + "tail [outro] ^ after caret", + ), +] + + +def emit_floats(out, name: str, values: np.ndarray) -> None: + flat = np.asarray(values, dtype=np.float32).reshape(-1) + out.append(f"inline constexpr float {name}[] = {{") + row: list[str] = [] + for value in flat: + # `%g` spells these "inf"/"nan", which is not C++. They are load-bearing + # here: a masked logit IS -inf and `_sample_top_k`'s nan_to_num exists + # for the NaN, so they are emitted rather than substituted away. + if np.isnan(value): + row.append("NAN") + elif np.isposinf(value): + row.append("INFINITY") + elif np.isneginf(value): + row.append("-INFINITY") + else: + text = f"{float(value):.9g}" + # "0" and "3" are integer literals; the `f` suffix needs a float. + if "." not in text and "e" not in text and "E" not in text: + text += ".0" + row.append(text + "f") + if len(row) == 6: + out.append(" " + ", ".join(row) + ",") + row = [] + if row: + out.append(" " + ", ".join(row) + ",") + out.append("};") + out.append("") + + +def cpp_string(text: str) -> str: + escaped = ( + text.replace("\\", "\\\\") + .replace('"', '\\"') + .replace("\n", "\\n") + .replace("\t", "\\t") + ) + return f'"{escaped}"' + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--out", required=True, type=Path) + args = parser.parse_args() + + try: + from diffusers.models.condition_embedders.condition_embedder_minimax_music3 import ( + MiniMaxMusic3ConditionEncoder, + ) + from diffusers.models.transformers.minimax_music3_rvq_depth_decoder import ( + MiniMaxMusic3RVQDepthDecoder, + ) + from diffusers.modular_pipelines.minimax_music3 import encoders as up + except ImportError as exc: # pragma: no cover - environment guard + print( + "This generator needs the pinned diffusers PR head " + "(c6da9936e4bda83107943a16eb8682e9a37d8527); see tools/oracle/README.md.\n" + f"import failed: {exc}", + file=sys.stderr, + ) + return 2 + + torch.set_grad_enabled(False) + out: list[str] = [] + out.append( + "// GENERATED by scripts/gen-minimax-music3-ar-goldens.py --- DO NOT EDIT BY HAND." + ) + out.append("//") + out.append( + "// MiniMax-Music3 AUTOREGRESSIVE-half goldens (#672, spec phases W2 + W3)," + ) + out.append( + "// produced by EXECUTING upstream's own classes at reduced dimensions in" + ) + out.append( + "// float32. Oracle pin: huggingface/diffusers#14456 @ c6da9936e4bda83107943a" + ) + out.append("// 16eb8682e9a37d8527. Weights come from the name-seeded Music3Rand") + out.append("// stream, so no weight byte of the 28.5 GB checkpoint is checked in.") + out.append("//") + out.append("// Upstream anchors:") + out.append("// condition mix condition_embedder_minimax_music3.py:48-76") + out.append("// depth decoder minimax_music3_rvq_depth_decoder.py:127-142, :75-88") + out.append("// depth sequence encoders.py:118-142 (_generate_depth_codes)") + out.append("// frame feedback encoders.py:106-115 (_embed_audio_frame)") + out.append("// prompt encoders.py:54-91, :207-218") + out.append("// semantic CFG encoders.py:318-334 (inline in __call__)") + out.append("// top-k filter encoders.py:94-103 (_sample_top_k)") + out.append("#pragma once") + out.append("") + out.append("#include // INFINITY / NAN appear as golden VALUES below") + out.append("#include ") + out.append("") + out.append("namespace vllm_test {") + out.append("") + + # -- upstream constants, re-emitted from the module so a rename is caught --- + out.append("// Re-emitted FROM the upstream module, not transcribed.") + for const in ( + "_AUDIO_END_TOKEN_ID", + "_AUDIO_CFG_TOKEN_ID", + "_AUDIO_CODE_OFFSET", + "_SEMANTIC_VOCAB_SIZE", + "_MAX_PROMPT_TOKENS", + "_MAX_AUDIO_FRAMES", + "_AR_CFG_TOP_K", + "_AR_SAMPLING_TOP_K", + ): + cpp = "kMusic3" + "".join(p.capitalize() for p in const.strip("_").split("_")) + out.append(f"inline constexpr int64_t {cpp} = {getattr(up, const)};") + out.append(f"inline constexpr double kMusic3ArCfgScale = {up._AR_CFG_SCALE!r};") + out.append("") + + # ---------------- prompt assembly ---------------------------------------- + out.append("struct Music3PromptGolden {") + out.append(" const char* name;") + out.append(" const char* prompt;") + out.append(" const char* lyrics;") + out.append(" const char* clean_caption;") + out.append(" const char* normalized_lyrics;") + out.append(" const char* assembled;") + out.append("};") + out.append("") + out.append("inline constexpr Music3PromptGolden kMusic3PromptGoldens[] = {") + for name, prompt, lyrics in PROMPT_CASES: + clean = up._clean_caption(prompt) + norm = up._normalize_lyrics(lyrics) + assembled = ( + f"{up._IM_START}{up._CAPTION_START}{clean}{up._CAPTION_END}" + f"{up._LYRICS_START}{norm}{up._LYRICS_END}{up._IM_END}{up._AUDIO_START}" + ) + out.append(" {") + out.append(f" {cpp_string(name)},") + out.append(f" {cpp_string(prompt)},") + out.append(f" {cpp_string(lyrics)},") + out.append(f" {cpp_string(clean)},") + out.append(f" {cpp_string(norm)},") + out.append(f" {cpp_string(assembled)},") + out.append(" },") + out.append("};") + out.append( + f"inline constexpr int64_t kMusic3PromptGoldenCount = {len(PROMPT_CASES)};" + ) + out.append("") + + # ---------------- unconditional id rewrite (encoders.py:216-217) ---------- + ids = torch.tensor([[11, 12, 13, 14, 15, 16, 17]], dtype=torch.int64) + uncond = ids.clone() + uncond[:, 1:-2] = up._AUDIO_CFG_TOKEN_ID + out.append("// encoders.py:216-217 - every token but the first and the two") + out.append("// trailing structure tokens becomes the audio-CFG token.") + out.append( + "inline constexpr int32_t kMusic3UncondIdsIn[] = {" + + ", ".join(str(int(v)) for v in ids[0]) + + "};" + ) + out.append( + "inline constexpr int32_t kMusic3UncondIdsOut[] = {" + + ", ".join(str(int(v)) for v in uncond[0]) + + "};" + ) + out.append( + f"inline constexpr int64_t kMusic3UncondIdsCount = {ids.shape[1]};" + ) + out.append("") + + # ---------------- semantic CFG + top-k pipeline --------------------------- + # encoders.py:318-334, reproduced op for op. VOCAB is tiny but the masked + # region straddles the audio-code window so the mask itself is exercised. + vocab = 40 + offset, semantic = 8, 6 # stand-ins for _AUDIO_CODE_OFFSET/_SEMANTIC_VOCAB_SIZE + end_id = 30 + cfg_top_k = 4 + vocab_mask = torch.ones(vocab, dtype=torch.bool) + vocab_mask[offset : offset + semantic] = False + vocab_mask[end_id] = False + logits = param("music3.semantic.logits", (2, vocab), scale=4.0).double().float() + masked = logits.masked_fill(vocab_mask, -float("inf")) + conditional, unconditional = masked[0:1], masked[1:2] + guided = unconditional + (conditional - unconditional) * up._AR_CFG_SCALE + threshold = torch.topk(conditional, cfg_top_k, dim=-1).values[..., -1, None] + guided = guided.masked_fill(conditional < threshold, -float("inf")) + guided = guided.masked_fill(vocab_mask.unsqueeze(0), -float("inf")) + out.append( + f"inline constexpr int64_t kMusic3SemanticVocab = {vocab};\n" + f"inline constexpr int64_t kMusic3SemanticOffset = {offset};\n" + f"inline constexpr int64_t kMusic3SemanticWindow = {semantic};\n" + f"inline constexpr int64_t kMusic3SemanticEndId = {end_id};\n" + f"inline constexpr int64_t kMusic3SemanticCfgTopK = {cfg_top_k};" + ) + emit_floats(out, "kMusic3SemanticLogitsIn", logits.numpy()) + emit_floats(out, "kMusic3SemanticGuided", guided.numpy()) + out.append("// true == BLOCKED (encoders.py:318-320).") + out.append( + "inline constexpr bool kMusic3SemanticVocabMask[] = {" + + ", ".join("true" if bool(v) else "false" for v in vocab_mask) + + "};" + ) + out.append("") + + # ---------------- _sample_top_k's deterministic half ---------------------- + # encoders.py:94-100: everything up to (not including) torch.multinomial. + probe = param("music3.topk.logits", (1, 12), scale=3.0) + probe[0, 3] = float("nan") + probe[0, 7] = float("-inf") + top_k = 5 + values = torch.nan_to_num(probe.float(), nan=-1e9, posinf=1e9, neginf=-1e9) + thresh = torch.topk(values, top_k, dim=-1).values[..., -1, None] + values = values.masked_fill(values < thresh, -float("inf")) + probs = torch.nan_to_num(torch.softmax(values, dim=-1), nan=0.0) + probs = probs / probs.sum(dim=-1, keepdim=True).clamp_min(1e-12) + out.append(f"inline constexpr int64_t kMusic3TopKProbeN = {probe.shape[1]};") + out.append(f"inline constexpr int64_t kMusic3TopKProbeK = {top_k};") + out.append("// index 3 is NaN and index 7 is -inf on the way in.") + emit_floats(out, "kMusic3TopKProbeIn", probe.numpy()) + emit_floats(out, "kMusic3TopKProbeProbs", probs.numpy()) + out.append("") + + # ---------------- condition mix ------------------------------------------ + for tag, cfg, frames in ( + ("", COND, COND_FRAMES), + ("Down", COND_DOWN, COND_DOWN_FRAMES), + ): + module = MiniMaxMusic3ConditionEncoder(**cfg) + state = { + "layer_weight_logits": param( + f"music3.cond{tag}.layer_weight_logits", (cfg["num_condition_layers"],) + ), + "layer_scale": param(f"music3.cond{tag}.layer_scale", (1,), 0.5, 1.0), + "proj.weight": param( + f"music3.cond{tag}.proj.weight", + (cfg["out_dim"], cfg["condition_hidden_dim"], 3), + 0.3, + ), + "proj.bias": param(f"music3.cond{tag}.proj.bias", (cfg["out_dim"],), 0.2), + } + module.load_state_dict(state) + module.eval() + width = cfg["num_condition_layers"] * cfg["condition_hidden_dim"] + hidden = param(f"music3.cond{tag}.hidden", (1, frames, width), 1.5) + result = module(hidden) + prefix = f"kMusic3Cond{tag}" + out.append(f"inline constexpr int64_t {prefix}Layers = {cfg['num_condition_layers']};") + out.append(f"inline constexpr int64_t {prefix}Hidden = {cfg['condition_hidden_dim']};") + out.append(f"inline constexpr int64_t {prefix}OutDim = {cfg['out_dim']};") + out.append(f"inline constexpr int64_t {prefix}Frames = {frames};") + out.append(f"inline constexpr int64_t {prefix}LatentLength = {result.shape[1]};") + out.append( + f"inline constexpr int64_t {prefix}InputSamplingRate = {cfg['input_sampling_rate']};" + ) + out.append( + f"inline constexpr int64_t {prefix}InputHopLength = {cfg['input_hop_length']};" + ) + out.append( + f"inline constexpr int64_t {prefix}OutputSamplingRate = {cfg['output_sampling_rate']};" + ) + out.append( + f"inline constexpr int64_t {prefix}OutputHopLength = {cfg['output_hop_length']};" + ) + emit_floats(out, f"{prefix}LayerWeightLogits", state["layer_weight_logits"].numpy()) + emit_floats(out, f"{prefix}LayerScale", state["layer_scale"].numpy()) + emit_floats(out, f"{prefix}ProjWeight", state["proj.weight"].numpy()) + emit_floats(out, f"{prefix}ProjBias", state["proj.bias"].numpy()) + emit_floats(out, f"{prefix}HiddenIn", hidden.numpy()) + emit_floats(out, f"{prefix}Out", result.numpy()) + + # ---------------- depth decoder ------------------------------------------ + module = MiniMaxMusic3RVQDepthDecoder(**DEPTH) + hidden_size = DEPTH["hidden_size"] + inter = DEPTH["intermediate_size"] + state = { + "audio_embeddings.weight": param( + "music3.depth.audio_embeddings", + (DEPTH["audio_vocab_size"] * (DEPTH["num_codebooks"] - 1), hidden_size), + 0.7, + ), + "projection.weight": param( + "music3.depth.projection", (hidden_size, hidden_size), 0.4 + ), + "pos_embedding.weight": param( + "music3.depth.pos_embedding", + (DEPTH["max_position_embeddings"], hidden_size), + 0.3, + ), + "norm.weight": param("music3.depth.norm", (hidden_size,), 0.2, 1.0), + } + for layer in range(DEPTH["num_layers"]): + base = f"layers.{layer}" + state[f"{base}.input_layernorm.weight"] = param( + f"music3.depth.{layer}.input_layernorm", (hidden_size,), 0.2, 1.0 + ) + state[f"{base}.post_attention_layernorm.weight"] = param( + f"music3.depth.{layer}.post_attention_layernorm", (hidden_size,), 0.2, 1.0 + ) + for proj in ("to_q", "to_k", "to_v", "to_out"): + state[f"{base}.attn.{proj}.weight"] = param( + f"music3.depth.{layer}.attn.{proj}", (hidden_size, hidden_size), 0.4 + ) + state[f"{base}.gate_proj.weight"] = param( + f"music3.depth.{layer}.gate_proj", (inter, hidden_size), 0.4 + ) + state[f"{base}.up_proj.weight"] = param( + f"music3.depth.{layer}.up_proj", (inter, hidden_size), 0.4 + ) + state[f"{base}.down_proj.weight"] = param( + f"music3.depth.{layer}.down_proj", (hidden_size, inter), 0.4 + ) + for head in range(DEPTH["num_codebooks"] - 1): + state[f"audio_heads.{head}.weight"] = param( + f"music3.depth.audio_heads.{head}", + (DEPTH["audio_vocab_size"], hidden_size), + 0.4, + ) + module.load_state_dict(state) + module.eval() + + for key, tensor in state.items(): + cpp = "kMusic3Depth" + "".join( + part.capitalize() for part in key.replace(".", "_").split("_") + ) + emit_floats(out, cpp, tensor.numpy()) + out.append(f"inline constexpr int64_t kMusic3DepthHidden = {hidden_size};") + out.append(f"inline constexpr int64_t kMusic3DepthLayers = {DEPTH['num_layers']};") + out.append(f"inline constexpr int64_t kMusic3DepthHeads = {DEPTH['num_attention_heads']};") + out.append(f"inline constexpr int64_t kMusic3DepthIntermediate = {inter};") + out.append( + f"inline constexpr int64_t kMusic3DepthAudioVocab = {DEPTH['audio_vocab_size']};" + ) + out.append( + f"inline constexpr int64_t kMusic3DepthCodebooks = {DEPTH['num_codebooks']};" + ) + out.append( + "inline constexpr int64_t kMusic3DepthMaxPositions = " + f"{DEPTH['max_position_embeddings']};" + ) + out.append("") + + # The depth sequence exactly as _generate_depth_codes assembles it + # (encoders.py:125-141): projection(last_hidden), projection(semantic embed), + # then projection(audio_embeddings(code + (index-1) * audio_vocab_size)). + last_hidden = param("music3.depth.last_hidden", (1, hidden_size), 1.2) + semantic_embed = param("music3.depth.semantic_embed", (1, hidden_size), 1.1) + residual_codes = [2, 4] # c1, c2 -> a 4-position sequence at num_codebooks=4 + sequence = [ + module.projection(last_hidden).unsqueeze(1), + module.projection(semantic_embed).unsqueeze(1), + ] + for index, code in enumerate(residual_codes, start=1): + embed = module.audio_embeddings( + torch.tensor([code + (index - 1) * DEPTH["audio_vocab_size"]]) + ) + sequence.append(module.projection(embed).unsqueeze(1)) + inputs_embeds = torch.cat(sequence, dim=1) + hidden_states = module(inputs_embeds) + head_logits = torch.stack( + [module.audio_heads[j](hidden_states[:, j + 1]) for j in range(len(residual_codes) + 1)], + dim=1, + ) + out.append(f"inline constexpr int64_t kMusic3DepthSeqLen = {inputs_embeds.shape[1]};") + out.append( + "inline constexpr int32_t kMusic3DepthResidualCodes[] = {" + + ", ".join(str(c) for c in residual_codes) + + "};" + ) + emit_floats(out, "kMusic3DepthLastHidden", last_hidden.numpy()) + emit_floats(out, "kMusic3DepthSemanticEmbed", semantic_embed.numpy()) + emit_floats(out, "kMusic3DepthInputsEmbeds", inputs_embeds.numpy()) + emit_floats(out, "kMusic3DepthOut", hidden_states.numpy()) + for j in range(DEPTH["num_codebooks"] - 1): + emit_floats( + out, + f"kMusic3DepthAudioHead{j}", + module.audio_heads[j].weight.detach().numpy(), + ) + emit_floats(out, "kMusic3DepthHeadLogits", head_logits.numpy()) + + # The 16-position BOUNDARY (spec W3 "the depth decoder's 16-position window + # exercised at its boundary"): a sequence exactly max_position_embeddings long. + boundary = param( + "music3.depth.boundary", (1, DEPTH["max_position_embeddings"], hidden_size), 0.9 + ) + boundary_out = module(boundary) + emit_floats(out, "kMusic3DepthBoundaryIn", boundary.numpy()) + emit_floats(out, "kMusic3DepthBoundaryOut", boundary_out.numpy()) + out.append("") + + # ---------------- frame feedback embedding ------------------------------- + # encoders.py:106-115. `embed_tokens` belongs to the LANGUAGE MODEL, so the + # reduced-dimension stand-in is a plain table of the same hidden width. + frame_codes = [3, 1, 4, 2] # semantic + c1..c3 at num_codebooks=4 + lm_row = param("music3.feedback.lm_row", (1, hidden_size), 1.3) + offsets = torch.arange(DEPTH["num_codebooks"] - 1) * DEPTH["audio_vocab_size"] + extra = module.audio_embeddings( + torch.tensor([frame_codes[1:]]) + offsets.unsqueeze(0) + ).sum(dim=1, keepdim=True) + feedback = (lm_row.unsqueeze(1) + extra) * DEPTH["num_codebooks"] ** -0.5 + out.append( + "inline constexpr int32_t kMusic3FeedbackCodes[] = {" + + ", ".join(str(c) for c in frame_codes) + + "};" + ) + emit_floats(out, "kMusic3FeedbackLmRow", lm_row.numpy()) + emit_floats(out, "kMusic3FeedbackOut", feedback.numpy()) + out.append("") + + out.append("} // namespace vllm_test") + out.append("") + args.out.write_text("\n".join(out)) + print(f"wrote {args.out} ({len(out)} lines)") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/vllm/model_executor/models/minimax_music3_ar.cpp b/src/vllm/model_executor/models/minimax_music3_ar.cpp new file mode 100644 index 000000000..ffc46688e --- /dev/null +++ b/src/vllm/model_executor/models/minimax_music3_ar.cpp @@ -0,0 +1,833 @@ +// MiniMax-Music3 β€” the AUTOREGRESSIVE half (W2 + W3 of #672). +// See include/vllm/model_executor/models/minimax_music3_ar.h for what is gated +// here and, more importantly, what is NOT: upstream's AR stage has no greedy +// path, so the committed `rvq_codes.npy` is a seeded sample and is consumed as +// an INPUT by these gates rather than predicted by them. +// +// Upstream anchors, at diffusers PR #14456 head c6da9936: +// condition_embedder_minimax_music3.py:48-76 +// minimax_music3_rvq_depth_decoder.py:28-142 +// modular_pipelines/minimax_music3/encoders.py:54-142, :202-353 +#include "vllm/model_executor/models/minimax_music3_ar.h" + +#include +#include +#include +#include +#include +#include + +#include "vllm/model_executor/models/vocoder1d.h" +#include "vt/dtype.h" + +namespace vllm { +namespace models { +namespace music3 { +namespace { + +[[noreturn]] void Fail(const std::string& what) { throw std::runtime_error(what); } + +// ONE op boundary. Accumulation stays in double (torch's bf16 matmul accumulates +// in float32 and its RMSNorm variance in float32); what the dtype decides is the +// width the RESULT is stored at, which is where a bf16 module actually loses +// bits. See the header note for why this is a mirror and not a knob. +inline float Store(double value, ArCompute compute) { + const float narrowed = static_cast(value); + return compute == ArCompute::kBFloat16 ? vt::BF16ToF32(vt::F32ToBF16(narrowed)) : narrowed; +} + +} // namespace + +// --------------------------------------------------------------------------- +// Prompt assembly +// --------------------------------------------------------------------------- + +std::string CleanCaption(const std::string& caption) { + // encoders.py:60 β€” `<|k v|>` becomes "k is v"; a tag with no space keeps its + // inner text with no rewrite. + static const std::regex kSpecialTag(R"(<\|([^|]*)\|>)"); + std::string text; + auto begin = std::sregex_iterator(caption.begin(), caption.end(), kSpecialTag); + auto end = std::sregex_iterator(); + size_t last = 0; + for (auto it = begin; it != end; ++it) { + const std::smatch& match = *it; + text.append(caption, last, static_cast(match.position(0)) - last); + // Python's str.strip() with no argument strips ASCII whitespace. + std::string inner = match[1].str(); + const size_t first_ch = inner.find_first_not_of(" \t\n\r\f\v"); + if (first_ch == std::string::npos) { + inner.clear(); + } else { + inner = inner.substr(first_ch, inner.find_last_not_of(" \t\n\r\f\v") - first_ch + 1); + } + // `inner.split(None, 1)` β€” split on the FIRST whitespace run only. + const size_t space = inner.find_first_of(" \t\n\r\f\v"); + if (space == std::string::npos) { + text += inner; + } else { + const size_t rest = inner.find_first_not_of(" \t\n\r\f\v", space); + text += inner.substr(0, space) + " is " + + (rest == std::string::npos ? std::string() : inner.substr(rest)); + } + last = static_cast(match.position(0) + match.length(0)); + } + text.append(caption, last, std::string::npos); + + // encoders.py:62-74 β€” per line: ATX heading, then two bullet forms, then the + // bold/italic unwraps, then a trailing-whitespace rstrip. + static const std::regex kHeading(R"(^\s{0,3}#{1,6}\s+)"); + static const std::regex kBulletA(R"(^\s*[*+-]\s+)"); + static const std::regex kBulletB(R"(^\s*\*\s+)"); + static const std::regex kBold(R"(\*\*([^*]+)\*\*)"); + static const std::regex kItalic(R"((^|[^*])\*([^*\n]+)\*($|[^*]))"); + std::vector lines; + { + std::string line; + for (const char ch : text) { + if (ch == '\n') { + lines.push_back(line); + line.clear(); + } else { + line += ch; + } + } + lines.push_back(line); + // Python's splitlines() drops a single trailing empty field. + if (!text.empty() && text.back() == '\n') lines.pop_back(); + } + std::string joined; + for (size_t i = 0; i < lines.size(); ++i) { + std::string line = std::regex_replace(lines[i], kHeading, "", std::regex_constants::format_first_only); + line = std::regex_replace(line, kBulletA, "", std::regex_constants::format_first_only); + line = std::regex_replace(line, kBulletB, "", std::regex_constants::format_first_only); + while (line.find("**") != std::string::npos) { + const std::string updated = + std::regex_replace(line, kBold, "$1"); + if (updated == line) break; + line = updated; + } + // The lookarounds of `(? i + 1) { + lowered += '['; + for (size_t j = i + 1; j < close; ++j) { + lowered += static_cast( + std::tolower(static_cast(text[j]))); + } + lowered += ']'; + i = close + 1; + continue; + } + } + lowered += text[i++]; + } + return std::string("[start]\n") + lowered; +} + +std::string AssembleArPrompt(const std::string& prompt, const std::string& lyrics) { + if (prompt.find_first_not_of(" \t\n\r\f\v") == std::string::npos) { + Fail("MiniMax-Music3: `prompt` (the music description) must be a non-empty string"); + } + if (lyrics.find_first_not_of(" \t\n\r\f\v") == std::string::npos) { + Fail("MiniMax-Music3: `lyrics` must be a non-empty string"); + } + return std::string(kImStart) + kCaptionStart + CleanCaption(prompt) + kCaptionEnd + + kLyricsStart + NormalizeLyrics(lyrics) + kLyricsEnd + kImEnd + kAudioStart; +} + +std::vector UnconditionalPromptIds(const std::vector& ids) { + if (static_cast(ids.size()) < 4) { + Fail("MiniMax-Music3: the prompt has " + std::to_string(ids.size()) + + " tokens; the unconditional rewrite (encoders.py:217) needs at least 4 so " + "that [1:-2] is a non-empty slice"); + } + if (static_cast(ids.size()) > kMaxPromptTokens) { + Fail("MiniMax-Music3: the assembled prompt has " + std::to_string(ids.size()) + + " tokens; the maximum is " + std::to_string(kMaxPromptTokens)); + } + std::vector out = ids; + for (size_t i = 1; i + 2 < out.size(); ++i) out[i] = kAudioCfgTokenId; + return out; +} + +int64_t MaxArFrames(double audio_duration_s, double frame_rate) { + if (!(audio_duration_s > 0.0)) { + Fail("MiniMax-Music3: `audio_duration` must be positive, got " + + std::to_string(audio_duration_s)); + } + const int64_t frames = static_cast(audio_duration_s * frame_rate); + if (frames == 0) { + Fail("MiniMax-Music3: `audio_duration` " + std::to_string(audio_duration_s) + + " is shorter than one audio frame (1 / " + std::to_string(frame_rate) + " s)"); + } + return std::min(frames, kMaxAudioFrames); +} + +// --------------------------------------------------------------------------- +// Logit pipeline +// --------------------------------------------------------------------------- + +std::vector SemanticVocabMask(int64_t vocab_size, int64_t code_offset, + int64_t semantic_vocab_size, int32_t end_token_id) { + if (vocab_size <= 0) Fail("MiniMax-Music3: vocab_size must be positive"); + if (code_offset < 0 || code_offset + semantic_vocab_size > vocab_size) { + Fail("MiniMax-Music3: the semantic code window [" + std::to_string(code_offset) + ", " + + std::to_string(code_offset + semantic_vocab_size) + ") does not fit in a vocabulary of " + + std::to_string(vocab_size)); + } + if (end_token_id < 0 || end_token_id >= vocab_size) { + Fail("MiniMax-Music3: the audio-end token " + std::to_string(end_token_id) + + " is outside a vocabulary of " + std::to_string(vocab_size)); + } + std::vector blocked(static_cast(vocab_size), true); + for (int64_t i = code_offset; i < code_offset + semantic_vocab_size; ++i) { + blocked[static_cast(i)] = false; + } + blocked[static_cast(end_token_id)] = false; + return blocked; +} + +std::vector GuidedSemanticLogits(const std::vector& conditional, + const std::vector& unconditional, + const std::vector& blocked, int64_t cfg_top_k, + double cfg_scale) { + const size_t n = conditional.size(); + if (unconditional.size() != n || blocked.size() != n) { + Fail("MiniMax-Music3: guided-logit inputs disagree on vocabulary size (" + + std::to_string(conditional.size()) + ", " + std::to_string(unconditional.size()) + + ", " + std::to_string(blocked.size()) + ")"); + } + if (cfg_top_k <= 0 || cfg_top_k > static_cast(n)) { + Fail("MiniMax-Music3: cfg_top_k " + std::to_string(cfg_top_k) + + " is out of range for a vocabulary of " + std::to_string(n)); + } + const float kNegInf = -std::numeric_limits::infinity(); + // encoders.py:326 β€” the mask is applied to the RAW rows, before guidance. + std::vector cond(n), uncond(n); + for (size_t i = 0; i < n; ++i) { + cond[i] = blocked[i] ? kNegInf : conditional[i]; + uncond[i] = blocked[i] ? kNegInf : unconditional[i]; + } + // encoders.py:328 + std::vector guided(n); + for (size_t i = 0; i < n; ++i) { + guided[i] = static_cast(static_cast(uncond[i]) + + (static_cast(cond[i]) - static_cast(uncond[i])) * + cfg_scale); + } + // encoders.py:331-332 β€” the threshold is the CONDITIONAL row's k-th largest. + std::vector order(n); + std::iota(order.begin(), order.end(), size_t{0}); + std::partial_sort(order.begin(), order.begin() + static_cast(cfg_top_k), order.end(), + [&cond](size_t a, size_t b) { return cond[a] > cond[b]; }); + const float threshold = cond[order[static_cast(cfg_top_k) - 1]]; + for (size_t i = 0; i < n; ++i) { + if (cond[i] < threshold) guided[i] = kNegInf; + } + // encoders.py:333 β€” the re-mask that keeps a NaN from becoming a candidate. + for (size_t i = 0; i < n; ++i) { + if (blocked[i]) guided[i] = kNegInf; + } + return guided; +} + +std::vector GuidedDepthLogits(const std::vector& conditional, + const std::vector& unconditional, + double cfg_scale) { + if (conditional.size() != unconditional.size()) { + Fail("MiniMax-Music3: depth CFG rows disagree on size (" + + std::to_string(conditional.size()) + " vs " + std::to_string(unconditional.size()) + ")"); + } + std::vector guided(conditional.size()); + for (size_t i = 0; i < conditional.size(); ++i) { + const double c = conditional[i]; + const double u = unconditional[i]; + guided[i] = static_cast(u + (c - u) * cfg_scale); + } + return guided; +} + +std::vector TopKProbabilities(const std::vector& logits, int64_t top_k) { + const size_t n = logits.size(); + if (n == 0) Fail("MiniMax-Music3: TopKProbabilities needs a non-empty row"); + if (top_k <= 0) Fail("MiniMax-Music3: top_k must be positive"); + const float kNegInf = -std::numeric_limits::infinity(); + // encoders.py:95 β€” nan_to_num BEFORE anything else, with upstream's finite + // substitutes; a -inf becomes -1e9, which is why a masked position can still + // be selected when it survives the top-k. + std::vector values(n); + for (size_t i = 0; i < n; ++i) { + const float v = logits[i]; + values[i] = std::isnan(v) ? -1e9f : (v == std::numeric_limits::infinity() + ? 1e9f + : (v == kNegInf ? -1e9f : v)); + } + const size_t k = std::min(static_cast(top_k), n); + std::vector order(n); + std::iota(order.begin(), order.end(), size_t{0}); + std::partial_sort(order.begin(), order.begin() + k, order.end(), + [&values](size_t a, size_t b) { return values[a] > values[b]; }); + const float threshold = values[order[k - 1]]; + for (size_t i = 0; i < n; ++i) { + if (values[i] < threshold) values[i] = kNegInf; + } + float max_value = kNegInf; + for (const float v : values) max_value = std::max(max_value, v); + double sum = 0.0; + std::vector probs(n); + for (size_t i = 0; i < n; ++i) { + probs[i] = values[i] == kNegInf ? 0.0 : std::exp(static_cast(values[i] - max_value)); + sum += probs[i]; + } + // encoders.py:99-100 β€” nan_to_num on the softmax, then renormalize with a + // 1e-12 floor on the denominator. + const double denom = std::max(sum, 1e-12); + std::vector out(n); + for (size_t i = 0; i < n; ++i) { + const double p = std::isnan(probs[i]) ? 0.0 : probs[i]; + out[i] = static_cast(p / denom); + } + return out; +} + +// --------------------------------------------------------------------------- +// Condition mix +// --------------------------------------------------------------------------- + +int64_t ConditionLatentLength(int64_t num_frames, const ConditionMixConfig& config) { + if (num_frames <= 0) Fail("MiniMax-Music3: the condition mix needs at least one frame"); + if (config.input_sampling_rate <= 0 || config.output_hop_length <= 0) { + Fail("MiniMax-Music3: the condition encoder's rates must be positive"); + } + const double scaled = static_cast(num_frames) * + static_cast(config.output_sampling_rate) / + static_cast(config.input_sampling_rate) * + static_cast(config.input_hop_length) / + static_cast(config.output_hop_length); + return std::max(1, static_cast(scaled)); +} + +std::vector ConditionLayerWeights(const std::vector& layer_weight_logits, + ArCompute compute) { + if (layer_weight_logits.empty()) { + Fail("MiniMax-Music3: the condition mix has no layer logits"); + } + const float max_logit = + *std::max_element(layer_weight_logits.begin(), layer_weight_logits.end()); + double sum = 0.0; + std::vector exps(layer_weight_logits.size()); + for (size_t i = 0; i < layer_weight_logits.size(); ++i) { + exps[i] = std::exp(static_cast(layer_weight_logits[i] - max_logit)); + sum += exps[i]; + } + std::vector out(layer_weight_logits.size()); + for (size_t i = 0; i < out.size(); ++i) out[i] = Store(exps[i] / sum, compute); + return out; +} + +std::vector NearestInterpolate1d(const std::vector& in, int64_t channels, + int64_t in_len, int64_t out_len) { + if (in_len <= 0 || out_len <= 0 || channels <= 0) { + Fail("MiniMax-Music3: NearestInterpolate1d needs positive extents"); + } + if (static_cast(in.size()) != channels * in_len) { + Fail("MiniMax-Music3: NearestInterpolate1d input is " + std::to_string(in.size()) + + " values, expected " + std::to_string(channels * in_len)); + } + const double scale = static_cast(in_len) / static_cast(out_len); + std::vector out(static_cast(channels * out_len)); + for (int64_t t = 0; t < out_len; ++t) { + const int64_t src = std::min( + in_len - 1, static_cast(std::floor(static_cast(t) * scale))); + for (int64_t c = 0; c < channels; ++c) { + out[static_cast(c * out_len + t)] = in[static_cast(c * in_len + src)]; + } + } + return out; +} + +std::vector ConditionMix(const std::vector& hidden_states, int64_t num_frames, + const ConditionMixConfig& config, + const ConditionMixWeights& weights, ArCompute compute) { + const int64_t layers = config.num_condition_layers; + const int64_t hidden = config.condition_hidden_dim; + const int64_t out_dim = config.out_dim; + if (static_cast(hidden_states.size()) != num_frames * layers * hidden) { + Fail("MiniMax-Music3: the condition mix got " + std::to_string(hidden_states.size()) + + " hidden values, expected frames*layers*hidden = " + + std::to_string(num_frames * layers * hidden)); + } + if (static_cast(weights.layer_weight_logits.size()) != layers) { + Fail("MiniMax-Music3: layer_weight_logits has " + + std::to_string(weights.layer_weight_logits.size()) + " entries, expected " + + std::to_string(layers)); + } + if (weights.layer_scale.size() != 1) { + Fail("MiniMax-Music3: layer_scale must be the single learned scalar, got " + + std::to_string(weights.layer_scale.size()) + " values"); + } + if (static_cast(weights.proj_weight.size()) != out_dim * hidden * 3) { + Fail("MiniMax-Music3: proj.weight has " + std::to_string(weights.proj_weight.size()) + + " values, expected out_dim*hidden*3 = " + std::to_string(out_dim * hidden * 3)); + } + if (static_cast(weights.proj_bias.size()) != out_dim) { + Fail("MiniMax-Music3: proj.bias has " + std::to_string(weights.proj_bias.size()) + + " values, expected " + std::to_string(out_dim)); + } + + const std::vector layer_weights = + ConditionLayerWeights(weights.layer_weight_logits, compute); + const double scale = static_cast(weights.layer_scale[0]); + + // :59-63 β€” the einsum "blht,l->bht" over a LAYER-MAJOR last axis, then the one + // scalar. Emitted as [hidden, frames] because the Conv1d that follows is over + // time. + std::vector mixed(static_cast(hidden * num_frames)); + for (int64_t h = 0; h < hidden; ++h) { + for (int64_t t = 0; t < num_frames; ++t) { + double acc = 0.0; + for (int64_t l = 0; l < layers; ++l) { + acc += static_cast(hidden_states[static_cast( + t * layers * hidden + l * hidden + h)]) * + static_cast(layer_weights[static_cast(l)]); + } + // :62 einsum then :63 the scalar β€” two ops, so two stores. + mixed[static_cast(h * num_frames + t)] = + Store(static_cast(Store(acc, compute)) * scale, compute); + } + } + + // :46,:64 β€” nn.Conv1d(hidden, out_dim, kernel_size=3, padding=1) through the + // shared 1-D primitives, so this port has no second convolution of its own. + int64_t padded_len = 0; + const std::vector padded = + vocoder1d::Pad1d(mixed, hidden, num_frames, 1, 1, /*replicate=*/false, &padded_len); + int64_t conv_len = 0; + std::vector projected = + vocoder1d::Conv1d(padded, hidden, padded_len, weights.proj_weight, &weights.proj_bias, + out_dim, /*kernel=*/3, /*stride=*/1, /*dilation=*/1, /*groups=*/1, + &conv_len); + for (float& value : projected) value = Store(value, compute); + if (conv_len != num_frames) { + Fail("MiniMax-Music3: the condition projection produced " + std::to_string(conv_len) + + " frames, expected " + std::to_string(num_frames)); + } + + // :65-76 β€” nearest resample onto the latent timeline, then transpose(1, 2) so + // the return is [latent_length, out_dim]. + const int64_t latent_length = ConditionLatentLength(num_frames, config); + const std::vector resampled = + NearestInterpolate1d(projected, out_dim, conv_len, latent_length); + std::vector out(static_cast(latent_length * out_dim)); + for (int64_t t = 0; t < latent_length; ++t) { + for (int64_t c = 0; c < out_dim; ++c) { + out[static_cast(t * out_dim + c)] = + resampled[static_cast(c * latent_length + t)]; + } + } + return out; +} + +// --------------------------------------------------------------------------- +// Depth decoder +// --------------------------------------------------------------------------- + +std::vector RmsNorm(const std::vector& x, int64_t rows, int64_t dim, + const std::vector& weight, double eps, ArCompute compute) { + if (static_cast(x.size()) != rows * dim) { + Fail("MiniMax-Music3: RmsNorm got " + std::to_string(x.size()) + " values, expected " + + std::to_string(rows * dim)); + } + if (static_cast(weight.size()) != dim) { + Fail("MiniMax-Music3: RmsNorm weight has " + std::to_string(weight.size()) + + " values, expected " + std::to_string(dim)); + } + std::vector out(x.size()); + for (int64_t r = 0; r < rows; ++r) { + double sum = 0.0; + for (int64_t c = 0; c < dim; ++c) { + const double v = x[static_cast(r * dim + c)]; + sum += v * v; + } + // normalization.py:600 computes the variance on the FLOAT32 upcast, so the + // reduction is wide at both dtypes. + const double inv = 1.0 / std::sqrt(static_cast(static_cast( + sum / static_cast(dim))) + + eps); + for (int64_t c = 0; c < dim; ++c) { + // :601 promotes to float32, :605 casts BACK to the weight dtype, and only + // :606 applies the affine weight. Two stores, in that order. + const double normed = static_cast(x[static_cast(r * dim + c)]) * inv; + out[static_cast(r * dim + c)] = + Store(static_cast(Store(normed, compute)) * + static_cast(weight[static_cast(c)]), + compute); + } + } + return out; +} + +std::vector LinearNoBias(const std::vector& x, int64_t rows, int64_t in_dim, + const std::vector& weight, int64_t out_dim, + ArCompute compute) { + if (static_cast(x.size()) != rows * in_dim) { + Fail("MiniMax-Music3: LinearNoBias input is " + std::to_string(x.size()) + + " values, expected " + std::to_string(rows * in_dim)); + } + if (static_cast(weight.size()) != out_dim * in_dim) { + Fail("MiniMax-Music3: LinearNoBias weight is " + std::to_string(weight.size()) + + " values, expected out_dim*in_dim = " + std::to_string(out_dim * in_dim)); + } + std::vector out(static_cast(rows * out_dim)); + for (int64_t r = 0; r < rows; ++r) { + for (int64_t o = 0; o < out_dim; ++o) { + double acc = 0.0; + const float* xr = x.data() + r * in_dim; + const float* wo = weight.data() + o * in_dim; + for (int64_t i = 0; i < in_dim; ++i) acc += static_cast(xr[i]) * wo[i]; + out[static_cast(r * out_dim + o)] = Store(acc, compute); + } + } + return out; +} + +namespace { + +// Causal scaled-dot-product attention over [seq, heads*head_dim] q/k/v laid out +// as torch's `view(batch, seq, heads, head_dim)` β€” head is the FAST axis inside +// a row (minimax_music3_rvq_depth_decoder.py:39-41). +std::vector CausalAttention(const std::vector& q, const std::vector& k, + const std::vector& v, int64_t seq, int64_t heads, + int64_t head_dim, ArCompute compute) { + const double inv_sqrt = 1.0 / std::sqrt(static_cast(head_dim)); + std::vector out(static_cast(seq * heads * head_dim), 0.0f); + std::vector scores(static_cast(seq)); + for (int64_t h = 0; h < heads; ++h) { + for (int64_t i = 0; i < seq; ++i) { + double max_score = -std::numeric_limits::infinity(); + for (int64_t j = 0; j <= i; ++j) { + double acc = 0.0; + for (int64_t d = 0; d < head_dim; ++d) { + acc += static_cast(q[static_cast((i * heads + h) * head_dim + d)]) * + static_cast(k[static_cast((j * heads + h) * head_dim + d)]); + } + scores[static_cast(j)] = acc * inv_sqrt; + max_score = std::max(max_score, scores[static_cast(j)]); + } + double sum = 0.0; + for (int64_t j = 0; j <= i; ++j) { + scores[static_cast(j)] = std::exp(scores[static_cast(j)] - max_score); + sum += scores[static_cast(j)]; + } + for (int64_t d = 0; d < head_dim; ++d) { + double acc = 0.0; + for (int64_t j = 0; j <= i; ++j) { + acc += scores[static_cast(j)] * + static_cast(v[static_cast((j * heads + h) * head_dim + d)]); + } + out[static_cast((i * heads + h) * head_dim + d)] = Store(acc / sum, compute); + } + } + } + return out; +} + +} // namespace + +std::vector DepthDecoderForward(const std::vector& inputs_embeds, int64_t seq_len, + const DepthDecoderConfig& config, + const DepthDecoderWeights& weights, ArCompute compute) { + const int64_t hidden = config.hidden_size; + if (seq_len <= 0) Fail("MiniMax-Music3: the depth decoder needs at least one position"); + if (seq_len > config.max_position_embeddings) { + Fail("MiniMax-Music3: the depth decoder was given " + std::to_string(seq_len) + + " positions but max_position_embeddings is " + + std::to_string(config.max_position_embeddings) + + "; pos_embedding has no row for the rest"); + } + if (static_cast(inputs_embeds.size()) != seq_len * hidden) { + Fail("MiniMax-Music3: the depth decoder got " + std::to_string(inputs_embeds.size()) + + " input values, expected seq*hidden = " + std::to_string(seq_len * hidden)); + } + if (static_cast(weights.layers.size()) != config.num_layers) { + Fail("MiniMax-Music3: the depth decoder has " + std::to_string(weights.layers.size()) + + " layer weight sets, expected " + std::to_string(config.num_layers)); + } + if (static_cast(weights.pos_embedding.size()) != + config.max_position_embeddings * hidden) { + Fail("MiniMax-Music3: pos_embedding is " + std::to_string(weights.pos_embedding.size()) + + " values, expected " + std::to_string(config.max_position_embeddings * hidden)); + } + const int64_t heads = config.num_attention_heads; + const int64_t head_dim = config.head_dim(); + if (heads * head_dim != hidden) { + Fail("MiniMax-Music3: hidden_size " + std::to_string(hidden) + + " is not divisible by num_attention_heads " + std::to_string(heads)); + } + + // :138-139 β€” positions are arange(seq) into a LEARNED table; there is no RoPE. + std::vector hidden_states(inputs_embeds.size()); + for (int64_t t = 0; t < seq_len; ++t) { + for (int64_t c = 0; c < hidden; ++c) { + hidden_states[static_cast(t * hidden + c)] = + Store(static_cast(inputs_embeds[static_cast(t * hidden + c)]) + + weights.pos_embedding[static_cast(t * hidden + c)], + compute); + } + } + + for (const DepthDecoderLayerWeights& layer : weights.layers) { + // :86 β€” pre-norm attention with a residual add. + const std::vector normed = + RmsNorm(hidden_states, seq_len, hidden, layer.input_layernorm, 1e-6, compute); + const std::vector q = + LinearNoBias(normed, seq_len, hidden, layer.to_q, hidden, compute); + const std::vector k = + LinearNoBias(normed, seq_len, hidden, layer.to_k, hidden, compute); + const std::vector v = + LinearNoBias(normed, seq_len, hidden, layer.to_v, hidden, compute); + const std::vector attended = + CausalAttention(q, k, v, seq_len, heads, head_dim, compute); + const std::vector projected = + LinearNoBias(attended, seq_len, hidden, layer.to_out, hidden, compute); + for (size_t i = 0; i < hidden_states.size(); ++i) { + hidden_states[i] = Store(static_cast(hidden_states[i]) + projected[i], compute); + } + + // :87-88 β€” SwiGLU MLP with a residual add. + const std::vector post = + RmsNorm(hidden_states, seq_len, hidden, layer.post_attention_layernorm, 1e-6, compute); + const std::vector gate = + LinearNoBias(post, seq_len, hidden, layer.gate_proj, config.intermediate_size, compute); + const std::vector up = + LinearNoBias(post, seq_len, hidden, layer.up_proj, config.intermediate_size, compute); + std::vector activated(gate.size()); + for (size_t i = 0; i < gate.size(); ++i) { + // F.silu then the elementwise product: two ops, two stores. + const double g = gate[i]; + const double silu = Store(g / (1.0 + std::exp(-g)), compute); + activated[i] = Store(silu * static_cast(up[i]), compute); + } + const std::vector down = LinearNoBias(activated, seq_len, config.intermediate_size, + layer.down_proj, hidden, compute); + for (size_t i = 0; i < hidden_states.size(); ++i) { + hidden_states[i] = Store(static_cast(hidden_states[i]) + down[i], compute); + } + } + + // :142 + return RmsNorm(hidden_states, seq_len, hidden, weights.norm, 1e-6, compute); +} + +std::vector DepthSequenceEmbeds(const std::vector& last_hidden, + const std::vector& semantic_embed, + const std::vector& residual_codes, + const DepthDecoderConfig& config, + const DepthDecoderWeights& weights, ArCompute compute) { + const int64_t hidden = config.hidden_size; + if (static_cast(last_hidden.size()) != hidden) { + Fail("MiniMax-Music3: last_hidden is " + std::to_string(last_hidden.size()) + + " values, expected hidden_size = " + std::to_string(hidden)); + } + if (static_cast(semantic_embed.size()) != hidden) { + Fail("MiniMax-Music3: the semantic embedding row is " + + std::to_string(semantic_embed.size()) + " values, expected hidden_size = " + + std::to_string(hidden)); + } + if (static_cast(residual_codes.size()) > config.residual_codebooks() - 1) { + Fail("MiniMax-Music3: the depth sequence carries at most " + + std::to_string(config.residual_codebooks() - 1) + + " residual codes (the last codebook is predicted, never fed back), got " + + std::to_string(residual_codes.size())); + } + std::vector rows; + rows.reserve(static_cast((2 + residual_codes.size()) * hidden)); + rows.insert(rows.end(), last_hidden.begin(), last_hidden.end()); + rows.insert(rows.end(), semantic_embed.begin(), semantic_embed.end()); + for (size_t index = 0; index < residual_codes.size(); ++index) { + const int32_t code = residual_codes[index]; + if (code < 0 || code >= config.audio_vocab_size) { + Fail("MiniMax-Music3: residual code " + std::to_string(code) + " at depth step " + + std::to_string(index + 1) + " is outside [0, " + + std::to_string(config.audio_vocab_size) + ")"); + } + // encoders.py:140 β€” `index` there is ONE-based, so the offset is (index-1). + const int64_t row = static_cast(index) * config.audio_vocab_size + code; + const size_t at = static_cast(row * hidden); + if (at + static_cast(hidden) > weights.audio_embeddings.size()) { + Fail("MiniMax-Music3: audio_embeddings row " + std::to_string(row) + + " is past the end of a table of " + + std::to_string(weights.audio_embeddings.size() / static_cast(hidden)) + + " rows"); + } + rows.insert(rows.end(), weights.audio_embeddings.begin() + static_cast(at), + weights.audio_embeddings.begin() + static_cast(at) + hidden); + } + const int64_t seq = 2 + static_cast(residual_codes.size()); + return LinearNoBias(rows, seq, hidden, weights.projection, hidden, compute); +} + +std::vector AudioHeadLogits(const std::vector& hidden, int64_t head_index, + const DepthDecoderConfig& config, + const DepthDecoderWeights& weights, ArCompute compute) { + if (head_index < 0 || head_index >= static_cast(weights.audio_heads.size())) { + Fail("MiniMax-Music3: audio head " + std::to_string(head_index) + " is outside [0, " + + std::to_string(weights.audio_heads.size()) + ")"); + } + return LinearNoBias(hidden, 1, config.hidden_size, + weights.audio_heads[static_cast(head_index)], + config.audio_vocab_size, compute); +} + +std::vector FrameHiddenRow(const std::vector& last_hidden, + const std::vector& depth_hidden_states, int64_t seq_len, + const DepthDecoderConfig& config) { + const int64_t hidden = config.hidden_size; + if (static_cast(last_hidden.size()) != hidden) { + Fail("MiniMax-Music3: last_hidden is " + std::to_string(last_hidden.size()) + + " values, expected " + std::to_string(hidden)); + } + if (static_cast(depth_hidden_states.size()) != seq_len * hidden) { + Fail("MiniMax-Music3: the depth hidden block is " + + std::to_string(depth_hidden_states.size()) + " values, expected " + + std::to_string(seq_len * hidden)); + } + // encoders.py:131-132 β€” step index i takes the LAST position of a sequence of + // length i+1, so over one whole-sequence forward that is positions 1..seq-1. + if (seq_len != config.num_codebooks) { + Fail("MiniMax-Music3: a frame row needs a depth sequence of exactly num_codebooks = " + + std::to_string(config.num_codebooks) + " positions, got " + std::to_string(seq_len)); + } + std::vector row; + row.reserve(static_cast(config.num_codebooks * hidden)); + row.insert(row.end(), last_hidden.begin(), last_hidden.end()); + row.insert(row.end(), depth_hidden_states.begin() + hidden, depth_hidden_states.end()); + return row; +} + +std::vector EmbedAudioFrame(const std::vector& lm_semantic_embed, + const std::vector& residual_codes, + const DepthDecoderConfig& config, + const DepthDecoderWeights& weights, ArCompute compute) { + const int64_t hidden = config.hidden_size; + if (static_cast(lm_semantic_embed.size()) != hidden) { + Fail("MiniMax-Music3: the language model's semantic row is " + + std::to_string(lm_semantic_embed.size()) + " values, expected " + std::to_string(hidden)); + } + if (static_cast(residual_codes.size()) != config.residual_codebooks()) { + Fail("MiniMax-Music3: the frame feedback needs all " + + std::to_string(config.residual_codebooks()) + " residual codes, got " + + std::to_string(residual_codes.size())); + } + std::vector acc(static_cast(hidden)); + for (int64_t c = 0; c < hidden; ++c) acc[static_cast(c)] = lm_semantic_embed[static_cast(c)]; + for (size_t j = 0; j < residual_codes.size(); ++j) { + const int32_t code = residual_codes[j]; + if (code < 0 || code >= config.audio_vocab_size) { + Fail("MiniMax-Music3: residual code " + std::to_string(code) + " for codebook " + + std::to_string(j + 1) + " is outside [0, " + + std::to_string(config.audio_vocab_size) + ")"); + } + // encoders.py:110-113 β€” `arange(num_codebooks-1) * audio_vocab_size` is a + // ZERO-based offset here. + const int64_t row = static_cast(j) * config.audio_vocab_size + code; + const size_t at = static_cast(row * hidden); + for (int64_t c = 0; c < hidden; ++c) { + acc[static_cast(c)] += weights.audio_embeddings[at + static_cast(c)]; + } + } + const double scale = 1.0 / std::sqrt(static_cast(config.num_codebooks)); + std::vector out(static_cast(hidden)); + for (int64_t c = 0; c < hidden; ++c) { + // encoders.py:113-115 β€” the residual sum, the add, then the scale. + out[static_cast(c)] = + Store(static_cast(Store(acc[static_cast(c)], compute)) * scale, compute); + } + return out; +} + +} // namespace music3 +} // namespace models +} // namespace vllm diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 68c3f3135..3ccb9228f 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -149,6 +149,24 @@ vllm_cpp_add_test(test_minimax_h3 vllm/models/test_minimax_h3.cpp) vllm_cpp_add_test(test_minimax_music3_loader vllm/models/test_minimax_music3_loader.cpp) target_include_directories(test_minimax_music3_loader PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/vllm/models) +# MiniMax-Music3 W2+W3 (#672): the AUTOREGRESSIVE half β€” prompt assembly, the +# semantic CFG logit pipeline, the learned 8-layer condition mix and the RVQ +# depth decoder β€” against goldens produced by EXECUTING upstream's own classes at +# reduced dimensions in float32 (minimax_music3_ar_goldens.inc, no weight bytes). +# The full-scale bf16 companion is tests/parity/test_minimax_music3_ar_real.cpp. +vllm_cpp_add_test(test_minimax_music3_ar vllm/models/test_minimax_music3_ar.cpp) +target_include_directories(test_minimax_music3_ar PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR}/vllm/models) +# The FULL-SCALE companion: the REAL bf16 checkpoint driven on the REAL inputs, +# against the committed oracle capture (frame_hiddens / condition_chunk0 / +# rvq_codes). Checkpoint-gated on MUSIC3_CHECKPOINT or CHECKPOINT_ROOT; every +# case emits a loud SKIP when the 28.5 GB asset is absent, so CI still compiles, +# links and runs it. +vllm_cpp_add_test(test_minimax_music3_ar_real parity/test_minimax_music3_ar_real.cpp) +target_compile_definitions(test_minimax_music3_ar_real PRIVATE + MUSIC3_GOLDENS_DIR="${CMAKE_SOURCE_DIR}/tests/parity/goldens/minimax_music3_oracle") +target_include_directories(test_minimax_music3_ar_real PRIVATE + ${CMAKE_SOURCE_DIR}/tests/parity) # ARCH-ONE-SURFACE ROW 2 fold gate: the video seam == the replicated pre-fold # pipeline == the goldens the PRE-fold minimax-h3-gen binary rendered at the # branch base (frames + WAV byte-identical, mux argv pinned). diff --git a/tests/parity/npy.h b/tests/parity/npy.h index 83dd50ab7..45cdde331 100644 --- a/tests/parity/npy.h +++ b/tests/parity/npy.h @@ -18,6 +18,9 @@ struct NpyArray { std::vector data; std::vector shape; std::string dtype; // numpy descr, e.g. " condition_chunk0.npy +// [86, 2048]. 176128 values. +// depth decoder frame_hiddens[:, :4096] (the language model's own hidden) +// + rvq_codes.npy rows 1.. (the frame's sampled codes) + the +// language model's embedding row for each semantic code +// -> frame_hiddens[:, 4096:]. 716800 values. +// +// THE ROW ALIGNMENT IS THE TRAP. `rvq_codes` is [26, 8] and `frame_hiddens` is +// [25, ...]: row 0 of the codes is the PRIMING decode step, which emits no +// frame (encoders.py:342 `if frame_index > 0`). `rows[1:]` align with the 25 +// frames. Getting it wrong compares two shifted sequences that are individually +// plausible, and the manifest's own note says so. +// +// ─── WHAT IT DOES NOT GATE, AND WHY ───────────────────────────────────────── +// +// Not the codes. Upstream's AR stage has no greedy path β€” `_sample_top_k` +// (encoders.py:94-103) ends in `torch.multinomial` against a seeded generator β€” +// and both stages sample from a CFG mix whose UNCONDITIONAL row is not in the +// golden set (only `[:1]` is stored, encoders.py:132,343). So `rvq_codes.npy` is +// an INPUT here. See the header of minimax_music3_ar.h; the spec's Β§5 claim of a +// token-exact greedy gate is not available from these artifacts. +// +// Not the language model's own forward. Producing `frame_hiddens[:, :4096]` +// means running the 8.6B `Qwen3ForCausalLM` teacher-forced on these codes, which +// is W2's remaining piece and is recorded as owed rather than skipped quietly. +// +// ─── THE TOLERANCE ────────────────────────────────────────────────────────── +// +// The goldens are bf16 values stored as float32 (asserted below, so the claim is +// checked and not assumed). This port computes in double and rounds once, torch +// computes in float32 and rounds to bf16 at every module boundary, so the two +// cannot be bit-equal and a tolerance is unavoidable. It is stated in bf16 ULPs: +// +// kUlpTol = 1 bf16 ULP, i.e. |a - b| <= 2^-8 * max(|a|, |b|) +// +// One ULP is the TIGHTEST bound two bf16 implementations can hold, and it is +// calibrated rather than guessed: running upstream's own module over the whole +// depth sequence at once instead of incrementally moves 2752 of these 716800 +// values, every one of them by exactly one ULP (max 0.015625 at |x| ~ 4). So one +// ULP is torch's own reproducibility on this tensor. +// +// A max-ULP bound alone is not enough β€” it is satisfied by an implementation +// that is off by one ULP EVERYWHERE, which no correct one is. So the mean +// absolute error is bounded too (torch-vs-torch measured 9.36e-06 on the same +// tensor), and both the compared count and the exactly-equal count are reported. +// A Pearson coefficient would see none of this: it is scale-invariant, so a +// uniformly scaled tensor passes it (AGENTS.md, spec Β§5). +// +// ─── HOW IT SKIPS ─────────────────────────────────────────────────────────── +// +// Checkpoint-gated on the SAME variable W1's loader gate uses, +// VLLM_CPP_MUSIC3_CHECKPOINT (or CHECKPOINT_ROOT, whose `minimax-music3` +// subdirectory is used). Absent, every case emits a loud SKIP +// and returns, so this file compiles, links and runs in CI without the 28.5 GB +// asset. +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "npy.h" +#include "vllm/model_executor/model_loader/safetensors_reader.h" +#include "vllm/model_executor/models/minimax_music3_ar.h" +#include "vllm/model_executor/models/minimax_music3_loader.h" +#include "vt/dtype.h" + +namespace fs = std::filesystem; +namespace m3 = vllm::models::music3; + +namespace { + +// bf16 has SEVEN stored mantissa bits, so inside the binade [2^e, 2^(e+1)) +// consecutive values are 2^(e-7) apart. The spacing is computed from the +// exponent rather than approximated by a fixed relative fraction β€” 2^-8 is HALF +// a bf16 ULP at the bottom of a binade, and using it as "one ULP" flags values +// that differ by exactly one (measured: 403050 of 716800 versus 217644). +// +// ─── THE CONDITION MIX: a bound with zero violations ──────────────────────── +// +// Its only long reduction is the k=3 Conv1d over 3*4096 terms. Torch accumulates +// that in float32 and this port in double, so the two can differ by a rounding +// boundary β€” one ULP β€” and by more only where the output is a near-cancellation +// of large terms, which is what the absolute floor covers. One ULP at unit +// magnitude is 2^-7. +constexpr double kCondUlpAllow = 1.0; +constexpr double kCondAbsFloor = 1.0 / 128.0; // 2^-7 + +// ─── THE DEPTH DECODER: a bound calibrated against a MATCHED CONTROL ───────── +// +// Zero violations is NOT achievable here and the reason is measured, not +// assumed. torch's own attention kernel is where the divergence starts: with the +// input layernorm bit-identical, `dispatch_attention_fn` on identical bf16 q/k/v +// reproduces to only 25736 of 32768 values, because the CPU kernel runs a +// blocked online softmax whose rounding no closed-form model here matched (four +// candidate models β€” pre-scaled q, bf16-rounded scores, bf16-rounded +// probabilities, and their combinations β€” were all WORSE than the plain form). +// Its bf16 Linear, by contrast, reproduces 32759 of 32768. +// +// So the calibration is torch against ITSELF with a different, equally correct +// attention kernel. Running upstream's OWN module over these exact inputs under +// `sdpa_kernel(SDPBackend.MATH)` and comparing to the committed goldens (which +// were captured on the default backend) gives: +// +// CONTROL 46.34% bit-identical, mean|d| 1.659e-03, max|d| 0.125, +// 82 of 716800 outside 2 ULP-or-2^-6 +// +// That is the floor of what any bf16 implementation can claim on this tensor, +// and chasing a specific kernel's rounding below it is not "more correct" +// (AGENTS.md's near-tie discipline). +// +// The bounds below sit just outside the control, and they are chosen so that the +// two defects this gate has actually caught are still RED: +// +// an fp32 forward (no bf16 rounding at op boundaries) 0.0004% identical, +// mean|d| 2.649e-03 +// any algebra defect ~0% identical +// +// The bit-identical FRACTION is the discriminator with four orders of margin, +// and it is asserted ALONGSIDE absolute bounds rather than instead of them β€” a +// count alone bounds nothing, and a correlation coefficient would see none of +// this because it is scale-invariant (spec Β§5). +constexpr double kDepthUlpAllow = 2.0; +constexpr double kDepthAbsFloor = 1.0 / 64.0; // 2^-6 +constexpr double kDepthIdenticalFloor = 0.40; // control 0.4634 +constexpr double kDepthMeanAbsTol = 2.2e-3; // control 1.659e-3 +constexpr double kDepthMaxAbsTol = 0.15; // control 0.125 +constexpr double kDepthOutsideFraction = 1e-3; // control 82/716800 = 1.14e-4 + +// The oracle capture's own shape facts (manifest.json), asserted rather than +// assumed so a regenerated golden cannot silently change what is compared. +constexpr int64_t kFrames = 25; +constexpr int64_t kCodeRows = 26; +constexpr int64_t kCodebooks = 8; +constexpr int64_t kHidden = 4096; +constexpr int64_t kLatentLength = 86; +constexpr int64_t kOutDim = 2048; +// The shard the language model's embedding table lives in +// (language_model/model.safetensors.index.json). +constexpr const char* kEmbedShard = "model-00001-of-00004.safetensors"; + +std::string CheckpointRoot() { + if (const char* direct = std::getenv("VLLM_CPP_MUSIC3_CHECKPOINT")) return direct; + if (const char* root = std::getenv("CHECKPOINT_ROOT")) { + return (fs::path(root) / "minimax-music3").string(); + } + return {}; +} + +std::string GoldensDir() { + return std::string(MUSIC3_GOLDENS_DIR); +} + +// Returns "" when everything needed is present, otherwise the reason to SKIP. +std::string MissingReason() { + const std::string root = CheckpointRoot(); + if (root.empty()) { + return "VLLM_CPP_MUSIC3_CHECKPOINT / CHECKPOINT_ROOT is unset"; + } + std::error_code ec; + if (!fs::is_directory(root, ec)) return "checkpoint directory " + root + " is absent"; + for (const char* needed : {"condition_encoder/diffusion_pytorch_model.safetensors", + "rvq_depth_decoder/diffusion_pytorch_model.safetensors"}) { + if (!fs::is_regular_file(fs::path(root) / needed, ec)) { + return std::string("checkpoint is missing ") + needed; + } + } + if (!fs::is_regular_file(fs::path(root) / "language_model" / kEmbedShard, ec)) { + return std::string("checkpoint is missing language_model/") + kEmbedShard; + } + for (const char* golden : {"frame_hiddens.npy", "condition_chunk0.npy", "rvq_codes.npy"}) { + if (!fs::is_regular_file(fs::path(GoldensDir()) / golden, ec)) { + return std::string("golden ") + golden + " is absent under " + GoldensDir(); + } + } + return {}; +} + +bool SkipIfMissing(const char* what) { + const std::string reason = MissingReason(); + if (reason.empty()) return false; + std::printf("[SKIP] %s: %s\n", what, reason.c_str()); + MESSAGE("SKIPPED (" << reason << ")"); + return true; +} + +// Row-major float32 from a golden. `condition_chunk0.npy` is stored FORTRAN +// order (the oracle saved a transposed view); read as C-order it would have the +// right value count and be the wrong tensor, so the transpose is explicit and +// keys off the reader's flag rather than a guess about the file. +std::vector LoadF32Npy(const std::string& name, std::vector* shape) { + const parity::NpyArray array = + parity::LoadNpy((fs::path(GoldensDir()) / name).string(), /*allow_fortran_order=*/true); + REQUIRE_MESSAGE(array.dtype == " raw(count); + std::memcpy(raw.data(), array.data.data(), array.data.size()); + if (!array.fortran_order) return raw; + REQUIRE_MESSAGE(array.shape.size() == 2, "only a 2-D fortran-order golden is handled: " << name); + const int64_t rows = array.shape[0]; + const int64_t cols = array.shape[1]; + std::vector out(count); + for (int64_t r = 0; r < rows; ++r) { + for (int64_t c = 0; c < cols; ++c) { + out[static_cast(r * cols + c)] = raw[static_cast(c * rows + r)]; + } + } + return out; +} + +std::vector LoadI32Npy(const std::string& name, std::vector* shape) { + const parity::NpyArray array = parity::LoadNpy((fs::path(GoldensDir()) / name).string()); + REQUIRE_MESSAGE(!array.fortran_order, "golden " << name << " must be C-order"); + REQUIRE_MESSAGE(array.dtype == " out(count); + std::memcpy(out.data(), array.data.data(), array.data.size()); + return out; +} + +// One checkpoint tensor at its RUNTIME dtype, carried in a float32 buffer. +// +// The AR half runs bf16 (MiniMaxMusic3ResolveRuntimeDtypes(kBf16ArFp32Acoustic)), +// so an F32 file β€” the condition encoder's, per spec section 2.1 β€” is ROUNDED, +// not widened. Skipping that round is the exact mistake the header's dtype note +// describes: still numerically fine, still not what the checkpoint runs. +std::vector AtRuntimeDtype(const vllm::StTensor& tensor) { + const size_t count = tensor.nbytes / (tensor.dtype == "F32" ? 4 : 2); + std::vector out(count); + if (tensor.dtype == "F32") { + std::memcpy(out.data(), tensor.data, tensor.nbytes); + for (float& value : out) value = vt::BF16ToF32(vt::F32ToBF16(value)); + } else if (tensor.dtype == "BF16") { + const auto* raw = reinterpret_cast(tensor.data); + for (size_t i = 0; i < count; ++i) out[i] = vt::BF16ToF32(raw[i]); + } else { + FAIL("unexpected checkpoint dtype " << tensor.dtype); + } + return out; +} + +// What a full-scale comparison examined, so it can REPORT rather than log. +struct UlpReport { + int64_t compared = 0; + int64_t identical = 0; + int64_t outside = 0; + double max_abs = 0.0; + double mean_abs = 0.0; + double max_ulps = 0.0; + double ref_absmax = 0.0; + int64_t first_bad = -1; + + double identical_fraction() const { + return compared > 0 ? static_cast(identical) / static_cast(compared) : 0.0; + } + double outside_fraction() const { + return compared > 0 ? static_cast(outside) / static_cast(compared) : 0.0; + } +}; + +UlpReport CompareUlps(const std::vector& got, const std::vector& want, + double ulp_allow, double abs_floor) { + UlpReport report; + REQUIRE(got.size() == want.size()); + double sum = 0.0; + for (size_t i = 0; i < want.size(); ++i) { + const double a = got[i]; + const double b = want[i]; + const double diff = std::abs(a - b); + ++report.compared; + if (a == b) ++report.identical; + sum += diff; + report.max_abs = std::max(report.max_abs, diff); + report.ref_absmax = std::max(report.ref_absmax, std::abs(b)); + const double magnitude = std::max(std::abs(a), std::abs(b)); + double ulps = 0.0; + if (diff > 0.0) { + if (magnitude > 0.0) { + int exponent = 0; + std::frexp(magnitude, &exponent); // magnitude = m * 2^exponent, 0.5 <= m < 1 + ulps = diff / std::ldexp(1.0, exponent - 8); + } else { + ulps = 1e9; + } + } + report.max_ulps = std::max(report.max_ulps, ulps); + if (ulps > ulp_allow && diff > abs_floor) { + if (report.outside == 0) report.first_bad = static_cast(i); + ++report.outside; + } + } + report.mean_abs = report.compared > 0 ? sum / static_cast(report.compared) : 0.0; + return report; +} + +} // namespace + +TEST_CASE("music3 ar real: the goldens are the ones this gate was written against") { + if (SkipIfMissing("music3 ar real goldens")) return; + std::vector shape; + const std::vector frame_hiddens = LoadF32Npy("frame_hiddens.npy", &shape); + REQUIRE(shape.size() == 2); + CHECK(shape[0] == kFrames); + CHECK(shape[1] == kCodebooks * kHidden); + const std::vector condition = LoadF32Npy("condition_chunk0.npy", &shape); + REQUIRE(shape.size() == 2); + CHECK(shape[0] == kLatentLength); + CHECK(shape[1] == kOutDim); + std::vector code_shape; + const std::vector codes = LoadI32Npy("rvq_codes.npy", &code_shape); + REQUIRE(code_shape.size() == 2); + CHECK(code_shape[0] == kCodeRows); + CHECK(code_shape[1] == kCodebooks); + // The alignment this whole file depends on: one MORE code row than frame. + CHECK(code_shape[0] == kFrames + 1); + + // `frame_hiddens` is stored float32 but every value must be bf16-exact β€” that + // is what makes a bf16-ULP tolerance the right shape of bound. CHECKED, not + // assumed. + int64_t not_bf16 = 0; + for (const float value : frame_hiddens) { + if (vt::BF16ToF32(vt::F32ToBF16(value)) != value) ++not_bf16; + } + MESSAGE("frame_hiddens: " << frame_hiddens.size() << " values, " << not_bf16 + << " not representable in bf16"); + CHECK(not_bf16 == 0); + + // Every residual code is inside the 1024-entry audio vocabulary, and the + // semantic column is inside the language model's 16384-entry code window. + int64_t residual_checked = 0; + int64_t semantic_checked = 0; + for (int64_t row = 0; row < kCodeRows; ++row) { + const int32_t semantic = codes[static_cast(row * kCodebooks)]; + CHECK(semantic >= 0); + CHECK(semantic < m3::kSemanticVocabSize); + ++semantic_checked; + for (int64_t j = 1; j < kCodebooks; ++j) { + const int32_t code = codes[static_cast(row * kCodebooks + j)]; + CHECK(code >= 0); + CHECK(code < 1024); + ++residual_checked; + } + } + MESSAGE("codes checked: " << semantic_checked << " semantic, " << residual_checked + << " residual, over " << kCodeRows << " rows"); + MESSAGE("condition golden: " << condition.size() << " values"); +} + +TEST_CASE("music3 ar real: the condition mix reproduces condition_chunk0") { + if (SkipIfMissing("music3 condition mix (real weights)")) return; + const std::string root = CheckpointRoot(); + std::vector shape; + const std::vector frame_hiddens = LoadF32Npy("frame_hiddens.npy", &shape); + REQUIRE(shape[0] == kFrames); + const std::vector want = LoadF32Npy("condition_chunk0.npy", &shape); + + const vllm::SafetensorsFile file = vllm::SafetensorsFile::Open( + (fs::path(root) / "condition_encoder" / "diffusion_pytorch_model.safetensors").string()); + m3::ConditionMixConfig config; // the released config's values are the defaults + m3::ConditionMixWeights weights; + weights.layer_weight_logits = AtRuntimeDtype(file.Get("layer_weight_logits")); + weights.layer_scale = AtRuntimeDtype(file.Get("layer_scale")); + weights.proj_weight = AtRuntimeDtype(file.Get("proj.weight")); + weights.proj_bias = AtRuntimeDtype(file.Get("proj.bias")); + REQUIRE(static_cast(weights.layer_weight_logits.size()) == config.num_condition_layers); + REQUIRE(static_cast(weights.proj_weight.size()) == + config.out_dim * config.condition_hidden_dim * 3); + + CHECK(m3::ConditionLatentLength(kFrames, config) == kLatentLength); + const std::vector got = + m3::ConditionMix(frame_hiddens, kFrames, config, weights, m3::ArCompute::kBFloat16); + const UlpReport report = CompareUlps(got, want, kCondUlpAllow, kCondAbsFloor); + MESSAGE("condition mix vs condition_chunk0: " << report.compared << " values compared, " + << report.identical << " bit-identical, " + << report.outside << " beyond 1 bf16 ULP, max|d| " + << report.max_abs << " (" << report.max_ulps + << " ULP), mean|d| " << report.mean_abs + << ", |ref|max " << report.ref_absmax); + CHECK(report.compared == kLatentLength * kOutDim); + CHECK(report.outside == 0); + CHECK(report.mean_abs <= 1e-5); + // A uniformly scaled or constant reproduction would satisfy neither. + CHECK(report.identical_fraction() >= 0.99); + CHECK(report.ref_absmax > 1.0); +} + +TEST_CASE("music3 ar real: the depth decoder reproduces the per-frame hidden states") { + if (SkipIfMissing("music3 depth decoder (real weights)")) return; + const std::string root = CheckpointRoot(); + std::vector shape; + const std::vector frame_hiddens = LoadF32Npy("frame_hiddens.npy", &shape); + REQUIRE(shape[0] == kFrames); + REQUIRE(shape[1] == kCodebooks * kHidden); + std::vector code_shape; + const std::vector codes = LoadI32Npy("rvq_codes.npy", &code_shape); + REQUIRE(code_shape[0] == kFrames + 1); + + const vllm::SafetensorsFile depth_file = vllm::SafetensorsFile::Open( + (fs::path(root) / "rvq_depth_decoder" / "diffusion_pytorch_model.safetensors").string()); + m3::DepthDecoderConfig config; // the released config's values are the defaults + m3::DepthDecoderWeights weights; + weights.audio_embeddings = AtRuntimeDtype(depth_file.Get("audio_embeddings.weight")); + weights.projection = AtRuntimeDtype(depth_file.Get("projection.weight")); + weights.pos_embedding = AtRuntimeDtype(depth_file.Get("pos_embedding.weight")); + weights.norm = AtRuntimeDtype(depth_file.Get("norm.weight")); + for (int64_t layer = 0; layer < config.num_layers; ++layer) { + const std::string base = "layers." + std::to_string(layer) + "."; + m3::DepthDecoderLayerWeights entry; + entry.input_layernorm = AtRuntimeDtype(depth_file.Get(base + "input_layernorm.weight")); + entry.post_attention_layernorm = + AtRuntimeDtype(depth_file.Get(base + "post_attention_layernorm.weight")); + entry.to_q = AtRuntimeDtype(depth_file.Get(base + "attn.to_q.weight")); + entry.to_k = AtRuntimeDtype(depth_file.Get(base + "attn.to_k.weight")); + entry.to_v = AtRuntimeDtype(depth_file.Get(base + "attn.to_v.weight")); + entry.to_out = AtRuntimeDtype(depth_file.Get(base + "attn.to_out.weight")); + entry.gate_proj = AtRuntimeDtype(depth_file.Get(base + "gate_proj.weight")); + entry.up_proj = AtRuntimeDtype(depth_file.Get(base + "up_proj.weight")); + entry.down_proj = AtRuntimeDtype(depth_file.Get(base + "down_proj.weight")); + weights.layers.push_back(std::move(entry)); + } + for (int64_t head = 0; head < config.residual_codebooks(); ++head) { + weights.audio_heads.push_back( + AtRuntimeDtype(depth_file.Get("audio_heads." + std::to_string(head) + ".weight"))); + } + REQUIRE(static_cast(weights.audio_embeddings.size()) == + config.audio_vocab_size * config.residual_codebooks() * config.hidden_size); + + // The SEMANTIC codebook is embedded by the LANGUAGE MODEL, so its rows come + // from the language model's own table β€” 25 rows out of 200000, read straight + // from the shard's mmap rather than materializing 1.6 GB. + const vllm::SafetensorsFile lm_file = vllm::SafetensorsFile::Open( + (fs::path(root) / "language_model" / kEmbedShard).string()); + const vllm::StTensor& embed = lm_file.Get("model.embed_tokens.weight"); + REQUIRE(embed.dtype == "BF16"); + REQUIRE(embed.shape.size() == 2); + REQUIRE(embed.shape[0] == 200000); + REQUIRE(embed.shape[1] == config.hidden_size); + const auto* embed_raw = reinterpret_cast(embed.data); + + int64_t frames_run = 0; + std::vector got; + std::vector want; + got.reserve(static_cast(kFrames * config.residual_codebooks() * config.hidden_size)); + want.reserve(got.capacity()); + for (int64_t frame = 0; frame < kFrames; ++frame) { + // rows[1:] align with the frames: row 0 is the priming decode. + const int64_t code_row = frame + 1; + const int32_t semantic = codes[static_cast(code_row * kCodebooks)]; + const int64_t token = static_cast(semantic) + m3::kAudioCodeOffset; + REQUIRE(token < embed.shape[0]); + std::vector semantic_embed(static_cast(config.hidden_size)); + for (int64_t c = 0; c < config.hidden_size; ++c) { + semantic_embed[static_cast(c)] = + vt::BF16ToF32(embed_raw[token * config.hidden_size + c]); + } + // c1..c6 are fed back; c7 is only ever predicted (encoders.py:139). + std::vector residual; + for (int64_t j = 1; j + 1 < kCodebooks; ++j) { + residual.push_back(codes[static_cast(code_row * kCodebooks + j)]); + } + REQUIRE(static_cast(residual.size()) == config.residual_codebooks() - 1); + + const std::vector last_hidden( + frame_hiddens.begin() + static_cast(frame) * kCodebooks * kHidden, + frame_hiddens.begin() + static_cast(frame) * kCodebooks * kHidden + kHidden); + const std::vector embeds = m3::DepthSequenceEmbeds( + last_hidden, semantic_embed, residual, config, weights, m3::ArCompute::kBFloat16); + const std::vector hidden_states = m3::DepthDecoderForward( + embeds, config.num_codebooks, config, weights, m3::ArCompute::kBFloat16); + REQUIRE(static_cast(hidden_states.size()) == + config.num_codebooks * config.hidden_size); + // Depth step i is position i of the sequence, so 1..7 are what the golden + // row carries after the language model's own 4096. + got.insert(got.end(), hidden_states.begin() + kHidden, hidden_states.end()); + want.insert(want.end(), + frame_hiddens.begin() + static_cast(frame) * kCodebooks * kHidden + kHidden, + frame_hiddens.begin() + static_cast(frame + 1) * kCodebooks * kHidden); + ++frames_run; + } + CHECK(frames_run == kFrames); + + const UlpReport report = CompareUlps(got, want, kDepthUlpAllow, kDepthAbsFloor); + MESSAGE("depth decoder vs frame_hiddens[:, 4096:]: " + << report.compared << " values compared over " << frames_run << " frames x " + << config.residual_codebooks() << " depth steps, " << report.identical + << " bit-identical, " << report.outside << " beyond 1 bf16 ULP, max|d| " + << report.max_abs << " (" << report.max_ulps << " ULP), mean|d| " << report.mean_abs + << ", |ref|max " << report.ref_absmax << ", identical fraction " + << report.identical_fraction() + << (report.first_bad >= 0 ? ", first beyond at " + std::to_string(report.first_bad) + : std::string())); + CHECK(report.compared == kFrames * config.residual_codebooks() * config.hidden_size); + // Four bounds, none of which alone is a gate. See the calibration note above: + // the bit-identical fraction separates a correct bf16 mirror from an fp32 one + // by four orders of magnitude, and the absolute bounds are what stop a + // uniformly scaled tensor from passing on counts. + CHECK(report.identical_fraction() >= kDepthIdenticalFloor); + CHECK(report.mean_abs <= kDepthMeanAbsTol); + CHECK(report.max_abs <= kDepthMaxAbsTol); + CHECK(report.outside_fraction() <= kDepthOutsideFraction); + // A zeroed or constant reproduction would satisfy a bound on nothing; the + // reference has real dynamic range and so must the reproduction. + CHECK(report.ref_absmax > 1.0); +} + +TEST_CASE("music3 ar real: the golden codes rank highly under the reproduced heads") { + // NOT a token gate, and labelled so. The sampled code came from a CFG mix of + // the conditional and unconditional rows (encoders.py:134-137) and only the + // conditional row is in the goldens, so the code cannot be re-derived. What + // CAN be said is that our reproduced hidden states put the golden code near + // the top of the CONDITIONAL distribution β€” a wrong forward would not. + // + // The reported rank distribution is the evidence; the assertion is only that + // every code is inside the head's vocabulary and that the mean rank is far + // below chance (512 for a 1024-entry codebook). + if (SkipIfMissing("music3 depth code ranks (real weights)")) return; + const std::string root = CheckpointRoot(); + std::vector shape; + const std::vector frame_hiddens = LoadF32Npy("frame_hiddens.npy", &shape); + std::vector code_shape; + const std::vector codes = LoadI32Npy("rvq_codes.npy", &code_shape); + + const vllm::SafetensorsFile depth_file = vllm::SafetensorsFile::Open( + (fs::path(root) / "rvq_depth_decoder" / "diffusion_pytorch_model.safetensors").string()); + m3::DepthDecoderConfig config; + m3::DepthDecoderWeights weights; + weights.audio_embeddings = AtRuntimeDtype(depth_file.Get("audio_embeddings.weight")); + weights.projection = AtRuntimeDtype(depth_file.Get("projection.weight")); + weights.pos_embedding = AtRuntimeDtype(depth_file.Get("pos_embedding.weight")); + weights.norm = AtRuntimeDtype(depth_file.Get("norm.weight")); + for (int64_t head = 0; head < config.residual_codebooks(); ++head) { + weights.audio_heads.push_back( + AtRuntimeDtype(depth_file.Get("audio_heads." + std::to_string(head) + ".weight"))); + } + + int64_t ranked = 0; + int64_t rank_sum = 0; + int64_t worst_rank = 0; + for (int64_t frame = 0; frame < kFrames; ++frame) { + for (int64_t step = 1; step < kCodebooks; ++step) { + const std::vector state( + frame_hiddens.begin() + static_cast(frame) * kCodebooks * kHidden + step * kHidden, + frame_hiddens.begin() + static_cast(frame) * kCodebooks * kHidden + + (step + 1) * kHidden); + const std::vector logits = + m3::AudioHeadLogits(state, step - 1, config, weights, m3::ArCompute::kBFloat16); + REQUIRE(static_cast(logits.size()) == config.audio_vocab_size); + const int32_t code = codes[static_cast((frame + 1) * kCodebooks + step)]; + REQUIRE(code < config.audio_vocab_size); + int64_t rank = 0; + for (int64_t j = 0; j < config.audio_vocab_size; ++j) { + if (logits[static_cast(j)] > logits[static_cast(code)]) ++rank; + } + rank_sum += rank; + worst_rank = std::max(worst_rank, rank); + ++ranked; + } + } + const double mean_rank = static_cast(rank_sum) / static_cast(ranked); + MESSAGE("golden code ranks under the CONDITIONAL head logits: " << ranked << " codes, mean rank " + << mean_rank << ", worst rank " + << worst_rank + << ", chance would be 511.5"); + CHECK(ranked == kFrames * (kCodebooks - 1)); + CHECK(mean_rank < 50.0); +} diff --git a/tests/vllm/models/minimax_music3_ar_goldens.inc b/tests/vllm/models/minimax_music3_ar_goldens.inc new file mode 100644 index 000000000..2b62c0423 --- /dev/null +++ b/tests/vllm/models/minimax_music3_ar_goldens.inc @@ -0,0 +1,691 @@ +// GENERATED by scripts/gen-minimax-music3-ar-goldens.py --- DO NOT EDIT BY HAND. +// +// MiniMax-Music3 AUTOREGRESSIVE-half goldens (#672, spec phases W2 + W3), +// produced by EXECUTING upstream's own classes at reduced dimensions in +// float32. Oracle pin: huggingface/diffusers#14456 @ c6da9936e4bda83107943a +// 16eb8682e9a37d8527. Weights come from the name-seeded Music3Rand +// stream, so no weight byte of the 28.5 GB checkpoint is checked in. +// +// Upstream anchors: +// condition mix condition_embedder_minimax_music3.py:48-76 +// depth decoder minimax_music3_rvq_depth_decoder.py:127-142, :75-88 +// depth sequence encoders.py:118-142 (_generate_depth_codes) +// frame feedback encoders.py:106-115 (_embed_audio_frame) +// prompt encoders.py:54-91, :207-218 +// semantic CFG encoders.py:318-334 (inline in __call__) +// top-k filter encoders.py:94-103 (_sample_top_k) +#pragma once + +#include // INFINITY / NAN appear as golden VALUES below +#include + +namespace vllm_test { + +// Re-emitted FROM the upstream module, not transcribed. +inline constexpr int64_t kMusic3AudioEndTokenId = 151670; +inline constexpr int64_t kMusic3AudioCfgTokenId = 151654; +inline constexpr int64_t kMusic3AudioCodeOffset = 151675; +inline constexpr int64_t kMusic3SemanticVocabSize = 16384; +inline constexpr int64_t kMusic3MaxPromptTokens = 5000; +inline constexpr int64_t kMusic3MaxAudioFrames = 9000; +inline constexpr int64_t kMusic3ArCfgTopK = 50; +inline constexpr int64_t kMusic3ArSamplingTopK = 50; +inline constexpr double kMusic3ArCfgScale = 1.5; + +struct Music3PromptGolden { + const char* name; + const char* prompt; + const char* lyrics; + const char* clean_caption; + const char* normalized_lyrics; + const char* assembled; +}; + +inline constexpr Music3PromptGolden kMusic3PromptGoldens[] = { + { + "oracle_capture", + "Genre: acoustic pop. BPM: 96. Key: C major. Warm and intimate. Vocals: soft female lead, close and breathy. Arrangement: fingerpicked guitar and soft piano.", + "[verse]\nMorning light filtering through the pine\n", + "Genre: acoustic pop. BPM: 96. Key: C major. Warm and intimate. Vocals: soft female lead, close and breathy. Arrangement: fingerpicked guitar and soft piano.", + "[start]\n[verse]\nMorning light filtering through the pine\n", + "<|im_start|><|caption_start|>Genre: acoustic pop. BPM: 96. Key: C major. Warm and intimate. Vocals: soft female lead, close and breathy. Arrangement: fingerpicked guitar and soft piano.<|caption_end|><|lyrics_start|>[start]\n[verse]\nMorning light filtering through the pine\n<|lyrics_end|><|im_end|><|audio_start|>", + }, + { + "markdown_and_tags", + "## Genre\n- **dream pop**\n * *hazy*\n<|mood dark and warm|><|solo|>\n---\nbulletβ€’ x\n\n\nend stop", + "[Verse] dropped words\n[Chorus][Bridge]\nkeep this line\ntail [outro] ^ after caret", + "Genre\ndream pop\nhazy\nmood is dark and warmsolo\nbulletx\nendstop", + "[start]\n[verse]\n[chorus][bridge]\nkeep this line\ntail\n[outro]\n^ after caret", + "<|im_start|><|caption_start|>Genre\ndream pop\nhazy\nmood is dark and warmsolo\nbulletx\nendstop<|caption_end|><|lyrics_start|>[start]\n[verse]\n[chorus][bridge]\nkeep this line\ntail\n[outro]\n^ after caret<|lyrics_end|><|im_end|><|audio_start|>", + }, +}; +inline constexpr int64_t kMusic3PromptGoldenCount = 2; + +// encoders.py:216-217 - every token but the first and the two +// trailing structure tokens becomes the audio-CFG token. +inline constexpr int32_t kMusic3UncondIdsIn[] = {11, 12, 13, 14, 15, 16, 17}; +inline constexpr int32_t kMusic3UncondIdsOut[] = {11, 151654, 151654, 151654, 151654, 16, 17}; +inline constexpr int64_t kMusic3UncondIdsCount = 7; + +inline constexpr int64_t kMusic3SemanticVocab = 40; +inline constexpr int64_t kMusic3SemanticOffset = 8; +inline constexpr int64_t kMusic3SemanticWindow = 6; +inline constexpr int64_t kMusic3SemanticEndId = 30; +inline constexpr int64_t kMusic3SemanticCfgTopK = 4; +inline constexpr float kMusic3SemanticLogitsIn[] = { + 1.01737273f, -0.338795841f, 1.32760751f, -0.256887823f, 3.76393366f, 3.6617775f, + 0.348432571f, -1.60502183f, -2.69527006f, -1.4744463f, -0.184102237f, -2.62218499f, + 0.083570376f, 3.54989505f, -0.839864135f, -0.0726852566f, -0.463415504f, 1.66899574f, + 2.86660266f, -2.52968502f, -3.09650707f, -0.554920018f, -1.99029672f, -1.99640691f, + -2.34245253f, 3.9440341f, 1.42193532f, 3.28166842f, -3.41064954f, 3.70717692f, + 2.76467967f, 2.65496111f, 1.9117825f, 2.8800354f, -1.98732173f, 0.38775599f, + -0.945684016f, 1.84461522f, -3.36198306f, -1.34640312f, -1.61141324f, -3.21996737f, + -2.79999471f, -3.50614476f, -3.33589292f, 2.441746f, 1.09343481f, 1.77395475f, + -0.0799809843f, 2.7459476f, 1.57364774f, -1.83762014f, -1.05355394f, -2.35463023f, + -3.83382678f, 2.54415822f, -1.18336928f, 2.51129818f, -0.0655540451f, -0.196973622f, + 2.77625418f, -3.63862944f, 2.51248741f, 1.65900373f, -2.43097281f, 2.54931021f, + -2.4141643f, -0.964192152f, 0.510477901f, 1.52212024f, 0.699162066f, 0.946673453f, + -0.198814452f, 0.575010121f, -2.06846118f, -3.0812552f, 0.283932358f, -1.77642119f, + 2.41101766f, 1.99160016f, +}; + +inline constexpr float kMusic3SemanticGuided[] = { + -INFINITY, -INFINITY, -INFINITY, -INFINITY, -INFINITY, -INFINITY, + -INFINITY, -INFINITY, -INFINITY, -INFINITY, -1.06297731f, -INFINITY, + 0.652132511f, 6.50215721f, -INFINITY, -INFINITY, -INFINITY, -INFINITY, + -INFINITY, -INFINITY, -INFINITY, -INFINITY, -INFINITY, -INFINITY, + -INFINITY, -INFINITY, -INFINITY, -INFINITY, -INFINITY, -INFINITY, + 3.79743862f, -INFINITY, -INFINITY, -INFINITY, -INFINITY, -INFINITY, + -INFINITY, -INFINITY, -INFINITY, -INFINITY, +}; + +// true == BLOCKED (encoders.py:318-320). +inline constexpr bool kMusic3SemanticVocabMask[] = {true, true, true, true, true, true, true, true, false, false, false, false, false, false, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, false, true, true, true, true, true, true, true, true, true}; + +inline constexpr int64_t kMusic3TopKProbeN = 12; +inline constexpr int64_t kMusic3TopKProbeK = 5; +// index 3 is NaN and index 7 is -inf on the way in. +inline constexpr float kMusic3TopKProbeIn[] = { + 1.20758331f, -2.4796803f, -0.830948114f, NAN, 2.6789844f, 2.23202705f, + 2.70086408f, -INFINITY, 0.0325289145f, -1.92503083f, 0.820840418f, 0.552450955f, +}; + +inline constexpr float kMusic3TopKProbeProbs[] = { + 0.0753476098f, 0.0f, 0.0f, 0.0f, 0.328163952f, 0.209884211f, + 0.335423201f, 0.0f, 0.0f, 0.0f, 0.0511810482f, 0.0f, +}; + + +inline constexpr int64_t kMusic3CondLayers = 3; +inline constexpr int64_t kMusic3CondHidden = 6; +inline constexpr int64_t kMusic3CondOutDim = 4; +inline constexpr int64_t kMusic3CondFrames = 5; +inline constexpr int64_t kMusic3CondLatentLength = 17; +inline constexpr int64_t kMusic3CondInputSamplingRate = 24000; +inline constexpr int64_t kMusic3CondInputHopLength = 960; +inline constexpr int64_t kMusic3CondOutputSamplingRate = 44100; +inline constexpr int64_t kMusic3CondOutputHopLength = 512; +inline constexpr float kMusic3CondLayerWeightLogits[] = { + 0.388273388f, 0.360078454f, 0.19248499f, +}; + +inline constexpr float kMusic3CondLayerScale[] = { + 0.9297598f, +}; + +inline constexpr float kMusic3CondProjWeight[] = { + -0.0803354681f, -0.000755753776f, 0.264103413f, -0.0546873696f, 0.108681984f, -0.0993260667f, + 0.112680919f, 0.164523274f, -0.175356686f, -0.028347943f, -0.223340705f, -0.135437071f, + -0.223644361f, 0.244994774f, 0.115918837f, 0.0388844348f, 0.0972928777f, 0.237094134f, + 0.0597746111f, 0.00358274928f, -0.0768014863f, 0.0616331808f, 0.20393303f, 0.0802203566f, + 0.0638534725f, 0.0850686654f, -0.120194159f, 0.161387533f, -0.29512468f, 0.0964788571f, + 0.0757200345f, 0.156555846f, 0.0283222515f, -0.292155355f, 0.298044384f, -0.289354831f, + 0.242497072f, 0.204764798f, -0.224289477f, -0.298555017f, -0.0831466466f, 0.249441415f, + -0.0366313383f, 0.161416233f, 0.0503221937f, 0.27364409f, -0.285978943f, -0.083440803f, + 0.0390252657f, -0.107694931f, -0.046770595f, 0.221548319f, -0.185134724f, 0.0914663002f, + 0.141911224f, 0.275271326f, 0.294669002f, 0.0324756317f, -0.00545022823f, 0.109841324f, + 0.128680229f, -0.0583390556f, 0.224436238f, -0.128048256f, -0.0814320892f, 0.226192653f, + 0.214738086f, 0.229642466f, -0.147578448f, 0.15005587f, -0.146117389f, -0.239011168f, +}; + +inline constexpr float kMusic3CondProjBias[] = { + 0.0266689379f, -0.0646697059f, 0.172246084f, 0.0334538855f, +}; + +inline constexpr float kMusic3CondHiddenIn[] = { + -0.149281576f, 1.26934755f, 1.21878386f, -1.2690767f, 0.335611463f, -1.41302299f, + 0.612337291f, -1.26027906f, -0.134384409f, 0.0771208331f, 0.766753376f, -1.00151253f, + 0.52870363f, 0.300224692f, 0.345282882f, -0.406650007f, 0.236289605f, -0.852816641f, + 0.853534639f, -1.38584077f, 0.31508553f, 1.10330021f, 1.11814272f, -0.399896562f, + 0.835986853f, 1.26086605f, -1.05268788f, -0.920333982f, -0.69417125f, -1.2251277f, + 1.35792828f, 1.23023355f, -0.354680032f, 1.42107928f, 0.260292292f, -0.870921373f, + 0.109689072f, -1.048751f, 1.21776617f, -0.448311478f, -0.369897157f, 0.0117470501f, + 1.38779902f, 1.02366996f, 1.40221858f, 0.744057536f, 0.566529691f, -0.677065372f, + -0.652452588f, -0.799206257f, 0.726257622f, -1.12440658f, 0.874265254f, -1.01318407f, + 0.838952959f, 1.09850705f, -0.404795736f, 0.528654516f, -1.31011057f, -1.40531516f, + 0.975835741f, 0.832265615f, -0.995103717f, -1.44509637f, 1.41984928f, 1.0329659f, + 0.652554989f, 1.05861664f, 0.666994691f, 0.366564184f, 0.241455704f, 1.24309587f, + 0.00862133969f, -1.14965332f, 1.36492074f, -0.363718659f, -0.00940354075f, -0.892479062f, + 0.427431732f, 0.765037417f, -0.0711056814f, 1.21598983f, 0.405191511f, -0.976935685f, + -0.43654865f, -0.59400928f, -1.07151532f, 0.247171357f, -1.47282612f, -1.20215666f, +}; + +inline constexpr float kMusic3CondOut[] = { + 0.285694182f, 0.1693708f, 0.315424532f, 0.860035062f, 0.285694182f, 0.1693708f, + 0.315424532f, 0.860035062f, 0.285694182f, 0.1693708f, 0.315424532f, 0.860035062f, + 0.285694182f, 0.1693708f, 0.315424532f, 0.860035062f, -0.352867216f, -0.0983462483f, + -0.16827634f, 0.863211155f, -0.352867216f, -0.0983462483f, -0.16827634f, 0.863211155f, + -0.352867216f, -0.0983462483f, -0.16827634f, 0.863211155f, 0.275060415f, 0.280394942f, + 0.736105144f, 0.348734885f, 0.275060415f, 0.280394942f, 0.736105144f, 0.348734885f, + 0.275060415f, 0.280394942f, 0.736105144f, 0.348734885f, 0.275060415f, 0.280394942f, + 0.736105144f, 0.348734885f, -0.0964925736f, 0.693629444f, -0.00770445168f, 0.800499618f, + -0.0964925736f, 0.693629444f, -0.00770445168f, 0.800499618f, -0.0964925736f, 0.693629444f, + -0.00770445168f, 0.800499618f, -0.368003756f, -0.53638196f, 0.249003768f, 0.258978426f, + -0.368003756f, -0.53638196f, 0.249003768f, 0.258978426f, -0.368003756f, -0.53638196f, + 0.249003768f, 0.258978426f, +}; + +inline constexpr int64_t kMusic3CondDownLayers = 3; +inline constexpr int64_t kMusic3CondDownHidden = 6; +inline constexpr int64_t kMusic3CondDownOutDim = 4; +inline constexpr int64_t kMusic3CondDownFrames = 7; +inline constexpr int64_t kMusic3CondDownLatentLength = 2; +inline constexpr int64_t kMusic3CondDownInputSamplingRate = 44100; +inline constexpr int64_t kMusic3CondDownInputHopLength = 512; +inline constexpr int64_t kMusic3CondDownOutputSamplingRate = 24000; +inline constexpr int64_t kMusic3CondDownOutputHopLength = 960; +inline constexpr float kMusic3CondDownLayerWeightLogits[] = { + -0.451546043f, 0.361102968f, 0.275269717f, +}; + +inline constexpr float kMusic3CondDownLayerScale[] = { + 1.23384738f, +}; + +inline constexpr float kMusic3CondDownProjWeight[] = { + 0.297379315f, -0.167045295f, 0.259196818f, 0.0514667183f, -0.260552675f, 0.0975541696f, + -0.208188802f, 0.245834529f, 0.29673028f, -0.127557635f, 0.135921001f, -0.0724468157f, + -0.132370681f, 0.240279138f, -0.150325909f, 0.170343548f, -0.257167757f, 0.0411280319f, + -0.283940315f, -0.118495204f, -0.235498652f, 0.231776252f, 0.198610827f, 0.154269144f, + 0.162005723f, -0.284620225f, -0.186880857f, 0.167687222f, -0.237176538f, 0.271727145f, + 0.127605706f, 0.0751956254f, 0.0124502108f, 0.141385213f, -0.128119603f, 0.0709443986f, + -0.277824879f, 0.0176939294f, 0.219255954f, -0.221700668f, -0.291034818f, -0.108010851f, + 0.25330475f, -0.00020463625f, -0.0263933819f, 0.16507265f, -0.0126845734f, 0.084839955f, + -0.037772458f, 0.221540898f, 0.295227677f, 0.143914968f, -0.149889976f, -0.167279676f, + 0.186326295f, 0.182673633f, -0.211873814f, -0.102454737f, 0.269925237f, 0.000465685414f, + 0.199472249f, -0.0399293378f, 0.00964868814f, -0.264892668f, -0.107910998f, 0.108686857f, + 0.144036472f, -0.20421651f, -0.282824934f, 0.0107522067f, -0.0820539072f, -0.259526163f, +}; + +inline constexpr float kMusic3CondDownProjBias[] = { + 0.0683589503f, 0.0941367f, 0.00304945838f, -0.126617521f, +}; + +inline constexpr float kMusic3CondDownHiddenIn[] = { + -0.573869288f, -0.779242158f, 0.916356385f, 0.347857356f, -1.1779654f, 0.981326461f, + -0.496072263f, 0.184430584f, -0.937692404f, 1.24161899f, 0.54351306f, -0.408001602f, + -1.10506845f, 0.259036481f, 0.616564691f, 0.0432120934f, -0.00299252826f, -0.715660751f, + -0.938979626f, 0.894987941f, -1.04278898f, 0.742869914f, 0.363839895f, 0.0688487589f, + -0.791217923f, 0.732963383f, 0.413013965f, 0.729455888f, 1.34576285f, -0.973167002f, + 0.518569946f, -0.606120884f, 0.641035318f, -0.591526628f, 1.12657309f, 0.919482231f, + -1.44818795f, -1.10453248f, 0.72271955f, -0.349357486f, 1.11487234f, -0.213561893f, + -0.0377743505f, 0.261871099f, 1.25152051f, -1.41345012f, 0.861093342f, -0.318280578f, + -0.645543635f, -0.873867095f, -0.383869678f, -0.92254144f, 0.953719616f, 1.08191204f, + 0.0361475162f, 0.460589975f, -0.454034328f, -0.171756595f, -1.36676145f, -1.00055659f, + 0.837394536f, 0.955540061f, 0.186609283f, 0.10326691f, -1.29915047f, -0.0520252585f, + 1.25682867f, 0.805944741f, 0.969787836f, -1.35305119f, -1.25112581f, 1.42933965f, + 0.215986639f, 0.486811042f, 0.567130923f, -1.06948841f, 0.535357296f, 0.652998388f, + -0.362933189f, 1.20607758f, 0.844976187f, -0.38307029f, -1.06260145f, 0.853642404f, + -1.22276688f, -1.00169539f, -0.443826348f, 0.0856600553f, 1.06469452f, -0.948271871f, + -0.279932082f, -0.13743034f, 0.238300011f, -0.226870611f, 1.27030015f, -0.163237482f, + -1.14670503f, 0.813388824f, 0.664665699f, 0.432876438f, -1.23054326f, 0.602448404f, + 0.494212806f, 1.08965778f, 0.368367344f, 0.365538567f, 0.465792358f, 0.735665143f, + -0.884058714f, -0.260617703f, -1.34395254f, 0.603663921f, 0.215032995f, 0.379673481f, + 0.33194977f, -0.182335332f, 0.306014836f, 1.48803794f, -0.179085091f, 0.137097299f, + 0.195141092f, -0.467852443f, 0.64178741f, -1.29190695f, 1.13957489f, -0.357766151f, +}; + +inline constexpr float kMusic3CondDownOut[] = { + 0.197643653f, 0.240978703f, 0.316821307f, -0.589835644f, -1.16440368f, 0.204790473f, + -0.651849568f, 1.29140902f, +}; + +inline constexpr float kMusic3DepthAudioEmbeddingsWeight[] = { + 0.693273842f, 0.547589123f, -0.436534256f, 0.395196676f, -0.550846636f, 0.042832233f, + -0.495383799f, 0.0017873816f, -0.399957418f, 0.556660116f, -0.417831451f, -0.234640568f, + -0.561223984f, 0.447165012f, -0.0765967518f, 0.427106857f, 0.119078033f, 0.260767907f, + 0.250439882f, 0.0585079789f, -0.480875194f, 0.495938122f, 0.648019135f, 0.580946743f, + 0.41639173f, 0.259211749f, -0.0222764499f, 0.185405374f, 0.484756589f, 0.543294549f, + -0.487662077f, 0.358858228f, 0.324353695f, 0.219911069f, 0.39439103f, 0.100706369f, + 0.600723326f, -0.189454824f, -0.24947907f, -0.465973705f, -0.071563229f, -0.152229354f, + 0.559698462f, -0.0831483155f, 0.0333082974f, -0.103445701f, -0.518924415f, -0.198651418f, + -0.00991750602f, 0.69734621f, -0.508683085f, -0.221917391f, 0.0796106383f, -0.317790806f, + 0.578185618f, -0.302208155f, -0.330564439f, 0.676743388f, -0.173238292f, 0.337109029f, + -0.397732288f, 0.0733864531f, 0.313402891f, -0.154837832f, 0.310690343f, -0.411601037f, + 0.427535951f, 0.437336832f, -0.508992612f, 0.545251548f, -0.253908724f, 0.221301511f, + 0.527234316f, -0.400112897f, -0.499601811f, -0.0880666599f, 0.528450131f, 0.320064813f, + 0.649452507f, 0.337756187f, -0.430828571f, 0.441776276f, 0.618220866f, -0.126574367f, + -0.264626592f, -0.551182628f, -0.39388144f, -0.0215745866f, 0.17287524f, 0.241549864f, + 0.151258335f, 0.229695186f, -0.377992839f, -0.425721109f, -0.115921453f, 0.237746298f, + 0.158412695f, 0.313034028f, -0.114636891f, -0.665877581f, 0.065372549f, 0.184905723f, + 0.511039913f, 0.611303985f, -0.629378557f, -0.398114353f, -0.543004513f, -0.349437386f, + 0.399931669f, -0.414064795f, 0.111174345f, 0.265735388f, -0.336944222f, 0.164491042f, + -0.0597543716f, -0.18888633f, 0.662056625f, -0.107004881f, -0.187218055f, 0.504926801f, +}; + +inline constexpr float kMusic3DepthProjectionWeight[] = { + -0.135035709f, -0.22034803f, -0.227598011f, -0.260609746f, 0.0414991789f, 0.0862121284f, + -0.0724638328f, -0.388294518f, -0.165765718f, 0.0928195417f, 0.171248019f, -0.211765885f, + 0.382563919f, -0.208574533f, 0.0930634141f, -0.243020207f, 0.197357953f, 0.371132344f, + -0.187416017f, -0.120785989f, 0.343182743f, -0.266794413f, -0.357305825f, -0.234703928f, + 0.301738888f, 0.237023354f, -0.211915702f, -0.357277036f, 0.23582013f, -0.000930935901f, + 0.094525367f, -0.0357417464f, -0.386561006f, -0.392066658f, 0.0597342029f, 0.377696753f, + 0.189437702f, 0.0925113931f, 0.191055521f, -0.286636412f, -0.0648554564f, -0.15697892f, + -0.139170378f, -0.122431442f, 0.226511285f, 0.0637694225f, 0.342343569f, -0.380395383f, + 0.183279991f, -0.0221816078f, 0.0340352878f, -0.264616728f, -0.227668151f, 0.339483559f, + -0.272766948f, 0.210451022f, -0.280177206f, 0.171941951f, -0.0742387474f, 0.20273529f, + 0.298798352f, -0.0258862451f, 0.358206332f, 0.183633536f, +}; + +inline constexpr float kMusic3DepthPosEmbeddingWeight[] = { + -0.142103583f, 0.00708962977f, 0.0278138947f, 0.116279915f, 0.0699623376f, -0.0128882844f, + 0.0100498646f, 0.112726383f, 0.243564948f, 0.283508688f, 0.0237991419f, 0.0302542932f, + 0.0647788644f, 0.223221362f, 0.0737063587f, -0.182996377f, -0.221774951f, 0.0924815759f, + -0.0991054028f, 0.0367159098f, -0.0988482311f, 0.275429755f, 0.187611073f, -0.0189349055f, + 0.174951911f, -0.264708966f, 0.237243757f, 0.0430079997f, -0.262082249f, 0.244756386f, + -0.172501042f, 0.115423813f, 0.189542294f, 0.150388747f, -0.236877531f, -0.0955140591f, + -0.0590414554f, -0.087590605f, -0.108394966f, -0.185736716f, -0.00278531248f, 0.287122011f, + -0.0370582975f, 0.192609251f, 0.0323918089f, -0.176013276f, 0.259233296f, -0.109179676f, +}; + +inline constexpr float kMusic3DepthNormWeight[] = { + 0.887883186f, 0.952903032f, 1.15802634f, 1.05524778f, 1.06072021f, 0.981468976f, + 1.09212017f, 1.15994942f, +}; + +inline constexpr float kMusic3DepthLayers0InputLayernormWeight[] = { + 0.948144734f, 0.954340339f, 0.828409374f, 1.03732944f, 0.832829297f, 0.878906131f, + 1.05720627f, 1.12940013f, +}; + +inline constexpr float kMusic3DepthLayers0PostAttentionLayernormWeight[] = { + 0.986460149f, 0.834730864f, 1.0587945f, 1.07338142f, 0.84655565f, 0.890536129f, + 1.02765059f, 0.901200414f, +}; + +inline constexpr float kMusic3DepthLayers0AttnToQWeight[] = { + -0.266107947f, -0.0577466972f, -0.0105097583f, -0.164729312f, -0.191157386f, 0.271674365f, + -0.000169674793f, -0.191234678f, 0.162226096f, -0.318655282f, -0.170458466f, 0.207794368f, + -0.399682611f, -0.358521938f, -0.0636188984f, 0.354168177f, 0.147100344f, 0.395840973f, + -0.0611724593f, -0.102528818f, -0.144376874f, -0.0236505438f, 0.331160337f, -0.256356597f, + 0.122662619f, 0.384616584f, -0.267408103f, 0.16912733f, -0.372490048f, 0.126441285f, + 0.160496175f, 0.184973434f, -0.306741774f, 0.267403185f, 0.293834239f, -0.374114811f, + 0.117640287f, 0.050033614f, 0.204803586f, 0.390921682f, -0.365774959f, 0.0455416515f, + -0.209152341f, 0.044324182f, 0.0926644132f, 0.232907295f, 0.105825305f, 0.128261492f, + 0.00539028458f, 0.00745795388f, 0.0181670766f, 0.169588223f, 0.0156458765f, 0.0898688808f, + -0.0122176865f, 0.203300461f, -0.276019692f, 0.285982549f, -0.0275590066f, 0.230943605f, + -0.37145561f, -0.228611797f, -0.17141147f, 0.231163934f, +}; + +inline constexpr float kMusic3DepthLayers0AttnToKWeight[] = { + 0.157315657f, 0.127730712f, 0.112336077f, 0.0340582393f, 0.0957621932f, -0.179755151f, + 0.229334503f, -0.0243765675f, 0.0760406256f, -0.175264806f, 0.101130761f, -0.300605595f, + -0.24165538f, -0.183628798f, 0.151376769f, 0.267404348f, -0.00386720919f, -0.00796655007f, + -0.379855454f, -0.173043832f, 0.385132283f, 0.0206869468f, 0.138964012f, -0.142857209f, + 0.160059497f, -0.182625443f, 0.128547311f, -0.331707001f, 0.0519198216f, -0.385941803f, + 0.264396459f, -0.132917657f, 0.0902882889f, 0.17722711f, -0.172705546f, -0.236897856f, + 0.294387013f, -0.313710243f, 0.0153557993f, -0.144656673f, 0.210772112f, -0.287642628f, + 0.0547387488f, 0.175535679f, 0.145199791f, -0.287838727f, -0.0089667663f, -0.211253271f, + 0.184121847f, 0.187439889f, -0.113396585f, 0.279360861f, 0.10138499f, 0.26979512f, + -0.151397079f, -0.349348575f, -0.252512634f, -0.160545915f, 0.204662308f, -0.0604270026f, + 0.135388821f, -0.197942436f, 0.381917626f, 0.355630308f, +}; + +inline constexpr float kMusic3DepthLayers0AttnToVWeight[] = { + 0.174782798f, -0.219501451f, -0.396406502f, -0.0185997542f, 0.143754482f, 0.202291906f, + 0.198706806f, 0.339013606f, 0.289667845f, 0.375925213f, -0.200515866f, -0.275478125f, + -0.0880604535f, 0.187442556f, -0.0803810284f, -0.377855837f, 0.192349955f, -0.305103183f, + -0.212372124f, 0.352765054f, -0.333267629f, 0.176232606f, 0.34172833f, -0.0955110937f, + 0.399099797f, -0.178825244f, 0.0377980284f, 0.232436493f, 0.0265451614f, 0.0300043039f, + 0.385595918f, -0.231115252f, -0.296628237f, 0.225659087f, -0.0639086813f, 0.387989134f, + 0.379425138f, 0.391603976f, -0.365441322f, -0.34808287f, 0.117643662f, -0.298715472f, + -0.0706301332f, 0.0328477286f, 0.0385604128f, -0.361863613f, 0.338843703f, -0.317689598f, + 0.161441982f, -0.301794559f, 0.360194117f, -0.0947343931f, 0.0750628263f, 0.229184538f, + -0.3907893f, 0.0280610863f, -0.350633085f, 0.288916707f, 0.170984596f, -0.353735715f, + -0.0492115319f, -0.346501857f, -0.117932595f, -0.0536441691f, +}; + +inline constexpr float kMusic3DepthLayers0AttnToOutWeight[] = { + 0.283965558f, 0.151918739f, 0.0489547886f, -0.196487308f, -0.218669832f, -0.0342020877f, + 0.0252421536f, 0.375263244f, -0.378567129f, -0.165605485f, -0.297110021f, -0.134869903f, + 0.259543866f, 0.118945554f, -0.268873185f, 0.0806721076f, -0.294892013f, 0.345267624f, + 0.295677006f, -0.140911579f, -0.299868494f, -0.0252214447f, 0.0852458328f, -0.163100809f, + -0.329465628f, 0.0883243978f, -0.350909442f, 0.354004651f, -0.362676829f, 0.166837439f, + -0.336565584f, -0.279820174f, -0.353076071f, -0.113045424f, 0.081360355f, -0.281772763f, + -0.0736811459f, -0.000813366612f, 0.243624613f, -0.127270788f, -0.220829129f, -0.236638516f, + 0.0223733578f, -0.290170372f, -0.137766108f, 0.053132344f, 0.204494908f, -0.209992543f, + 0.0118268821f, 0.071918413f, 0.355690539f, -0.323117614f, -0.185372904f, -0.266818911f, + 0.153657719f, 0.162965626f, -0.248424798f, 0.0644710436f, -0.0494534783f, -0.385296613f, + -0.377172887f, -0.261418402f, -0.0887069106f, -0.39054063f, +}; + +inline constexpr float kMusic3DepthLayers0GateProjWeight[] = { + -0.307407618f, -0.0466699712f, 0.0538328104f, 0.308891326f, 0.0101977019f, 0.340688735f, + -0.174432337f, 0.235313147f, -0.0773404539f, 0.0719899684f, 0.340507329f, -0.0612470172f, + -0.210727945f, 0.246789649f, -0.117325075f, 0.11222247f, 0.0397200398f, -0.083041355f, + -0.146533132f, 0.344467521f, -0.322108388f, 0.0460172407f, -0.050519228f, -0.044776421f, + 0.100300558f, -0.383284777f, 0.0725796297f, -0.382309198f, 0.102816634f, 0.0986417904f, + -0.000211898601f, -0.249874949f, 0.131716341f, -0.240043461f, -0.26862824f, -0.0152282175f, + 0.275197178f, -0.174624801f, 0.260977536f, -0.110696264f, 0.33637464f, 0.150171146f, + -0.350962877f, 0.0377519578f, 0.120194972f, -0.399578422f, -0.369044244f, 0.241525352f, + -0.200093165f, 0.197898045f, -0.0694133416f, 0.118538633f, -0.247958913f, -0.145102099f, + 0.139174312f, -0.210480794f, 0.194074556f, 0.0657641143f, 0.0202641059f, 0.0534195118f, + -0.140225604f, 0.321457922f, -0.139677048f, 0.308605999f, -0.261340946f, 0.161374629f, + -0.327927291f, 0.137977466f, -0.12881428f, 0.374498904f, -0.0987284407f, -0.269067645f, + -0.344853729f, 0.338491768f, -0.290827394f, 0.134817183f, 0.182406887f, -0.176913455f, + -0.107109964f, 0.358990401f, 0.112638213f, 0.0994106382f, -0.237954304f, -0.168199122f, + 0.245373741f, -0.0812330171f, 0.0672504678f, -0.27277723f, -0.300451159f, -0.0935611874f, + 0.138613433f, 0.0548552722f, -0.392575592f, 0.201291308f, -0.183681697f, -0.252856433f, +}; + +inline constexpr float kMusic3DepthLayers0UpProjWeight[] = { + -0.357537776f, 0.00532458536f, -0.309509903f, 0.197663203f, 0.187673435f, -0.33681035f, + 0.149708569f, -0.0467583537f, 0.0373484641f, 0.338854283f, -0.348797083f, -0.302223861f, + 0.300530344f, 0.343490094f, 0.0404606313f, -0.144816712f, -0.206585154f, 0.188020214f, + -0.23634626f, 0.124433309f, -0.0947067887f, -0.306971282f, -0.356645852f, -0.165662408f, + 0.23968643f, -0.115282282f, -0.0107643055f, 0.37926811f, 0.36891219f, 0.0564986132f, + 0.315110773f, -0.0877909437f, 0.192529917f, 0.108337618f, 0.0283143241f, -0.0179244839f, + -0.248953253f, -0.326393068f, -0.164321333f, -0.332560539f, 0.178867757f, 0.0268257577f, + -0.113434829f, 0.140871286f, 0.148316294f, 0.221650258f, -0.340776652f, 0.0797157884f, + -0.321248472f, -0.296212107f, 0.341933876f, -0.213024959f, 0.0204998571f, 0.0670746863f, + -0.186129838f, -0.260193914f, -0.138192236f, 0.155703932f, 0.361540824f, -0.256022125f, + -0.373371959f, -0.35628745f, 0.3937096f, 0.0683062151f, -0.284146845f, 0.0583784953f, + 0.368944585f, -0.00851624645f, 0.26659888f, 0.231415793f, -0.256878585f, 0.13353093f, + -0.308070838f, -0.175586656f, -0.12212839f, 0.392139077f, -0.346547216f, -0.115099825f, + -0.100535393f, -0.274243087f, 0.118378654f, -0.0508299433f, -0.0661496073f, 0.264125347f, + 0.00273385667f, -0.209354565f, 0.166466117f, 0.293257743f, -0.15068762f, 0.373441339f, + -0.0385711379f, -0.0295831058f, -0.111457214f, -0.237616956f, -0.00665641529f, -0.135259211f, +}; + +inline constexpr float kMusic3DepthLayers0DownProjWeight[] = { + 0.0168186352f, 0.346415371f, 0.299410313f, -0.172388092f, 0.379311979f, -0.116125293f, + 0.0908334553f, -0.0258545782f, 0.380718738f, 0.179372236f, 0.200992361f, -0.0222083982f, + -0.275047123f, -0.150846705f, 0.0509842187f, -0.247548252f, 0.225629866f, 0.144293651f, + 0.321080655f, -0.0121093607f, 0.384580046f, -0.0758427307f, 0.131520167f, 0.0215856694f, + -0.224221066f, -0.195317954f, 0.0772903562f, 0.0217115618f, 0.357756853f, 0.253777087f, + 0.330219179f, 0.0198746473f, 0.181756213f, 0.179972202f, -0.0159604177f, 0.251976728f, + 0.161787316f, 0.178135216f, 0.0537762009f, -0.298582971f, -0.256721318f, -0.12598148f, + -0.157377705f, -0.0634856299f, -0.287428498f, 0.164898738f, -0.264521331f, -0.133473486f, + -0.13447997f, -0.121296428f, 0.0785156861f, 0.173054576f, -0.023345843f, 0.0175237916f, + 0.237907127f, 0.0589264929f, -0.159258977f, 0.111014031f, -0.0898593813f, 0.120136298f, + -0.254957139f, -0.0539687686f, -0.325604349f, 0.0944914445f, 0.176335678f, 0.300280094f, + -0.390383869f, 0.0709221363f, 0.0172245651f, 0.31539914f, -0.255194247f, 0.256167442f, + -0.381957442f, -0.168370053f, 0.100240521f, -0.364208639f, 0.163397372f, -0.215050474f, + 0.157024994f, 0.386127263f, 0.331949264f, 0.105174325f, -0.148916751f, -0.0356823616f, + -0.231321752f, 0.00808899291f, 0.18446292f, 0.381631076f, 0.31574896f, -0.278537571f, + 0.107694454f, 0.305530995f, 0.202786982f, 0.347144425f, -0.0761960149f, 0.130470648f, +}; + +inline constexpr float kMusic3DepthLayers1InputLayernormWeight[] = { + 1.03473806f, 1.0790627f, 1.04970968f, 0.930797815f, 1.14027464f, 0.893740952f, + 1.10783339f, 0.816488028f, +}; + +inline constexpr float kMusic3DepthLayers1PostAttentionLayernormWeight[] = { + 0.984445393f, 0.824368238f, 1.14112389f, 1.14701855f, 0.830011785f, 0.99033457f, + 1.04437315f, 1.12966907f, +}; + +inline constexpr float kMusic3DepthLayers1AttnToQWeight[] = { + 0.14307715f, -0.200558975f, 0.334946871f, 0.115859717f, -0.0246659201f, -0.346356928f, + 0.196233913f, 0.380253166f, 0.00496882945f, 0.339900851f, 0.161138266f, 0.381425023f, + -0.220313907f, -0.0194205027f, -0.243813932f, -0.369754076f, 0.0721476972f, 0.146918312f, + -0.193251371f, -0.0156341828f, -0.0751344115f, 0.194251478f, -0.162722141f, -0.0185591429f, + -0.326521218f, -0.351871848f, 0.289930642f, 0.191210687f, -0.0535408743f, 0.350662053f, + 0.0370264053f, -0.174818903f, 0.0138389803f, -0.350426912f, -0.142065763f, -0.0857555419f, + 0.338357985f, -0.34594968f, -0.387672037f, 0.109990619f, 0.221525714f, 0.10209693f, + -0.0270905886f, -0.201080114f, 0.0710944012f, -0.209032789f, -0.0563582629f, 0.132045597f, + 0.29034707f, -0.0580219403f, -0.139984325f, 0.0647530481f, 0.364052385f, -0.311208785f, + 0.0940345153f, -0.0145355482f, -0.392093003f, -0.265429854f, 0.118433766f, -0.167315453f, + 0.00446381187f, 0.0850395933f, 0.371771604f, -0.355401665f, +}; + +inline constexpr float kMusic3DepthLayers1AttnToKWeight[] = { + -0.142626524f, -0.278377354f, 0.285842359f, -0.282475948f, -0.169200465f, -0.000641039456f, + -0.143733621f, 0.182704508f, 0.231605515f, -0.156129196f, -0.210423425f, -0.0249478053f, + 0.102140702f, 0.385407895f, 0.347484887f, 0.363118529f, 0.315421283f, 0.2399064f, + -0.387821436f, -0.170025215f, 0.13798517f, 0.313186407f, 0.255622476f, 0.203001186f, + -0.356195748f, 0.11259786f, -0.156835213f, -0.325965673f, 0.364221334f, 0.159407169f, + 0.321277112f, -0.228080258f, 0.192768782f, 0.374621779f, -0.0355416834f, -0.215790749f, + 0.280154169f, -0.311241806f, -0.190512076f, -0.370295286f, -0.286955029f, 0.34291327f, + 0.315092355f, 0.143577784f, 0.264843673f, 0.00269768783f, -0.383103192f, 0.0983676389f, + -0.166800067f, -0.389206916f, 0.253626555f, 0.35703966f, -0.115166128f, -0.333552033f, + 0.0302209444f, 0.329963714f, -0.39521268f, 0.120439462f, 0.0684231222f, -0.108510725f, + 0.215513825f, 0.157338038f, -0.198328555f, -0.0150981452f, +}; + +inline constexpr float kMusic3DepthLayers1AttnToVWeight[] = { + 0.06370987f, 0.163398638f, -0.143714145f, 0.283333719f, -0.389439851f, -0.0646319017f, + 0.137161031f, -0.294291884f, -0.148014635f, -0.0978966653f, 0.161080405f, 0.20377478f, + 0.367772281f, 0.173168063f, 0.386559993f, 0.14586781f, -0.373576552f, -0.380253822f, + -0.129488125f, 0.332272291f, -0.230383962f, 0.182447255f, 0.194201544f, 0.273629457f, + 0.32683897f, -0.353342593f, -0.268944949f, -0.275452495f, -0.276315898f, -0.377431035f, + -0.370126098f, 0.137048066f, -0.394261837f, 0.315329611f, -0.373326868f, 0.0577789769f, + 0.203881919f, 0.222959772f, -0.125606939f, -0.19975324f, 0.0876045451f, 0.0193811078f, + -0.173636079f, 0.00405780924f, -0.0693882629f, -0.305978298f, -0.220303357f, 0.359149337f, + -0.296617836f, -0.360878378f, -0.302726835f, -0.0898020416f, 0.306330949f, 0.104270414f, + 0.262109131f, 0.31093356f, -0.319952935f, -0.0703714713f, -0.101914696f, -0.391956121f, + -0.160394356f, 0.276566714f, 0.131844953f, 0.155400261f, +}; + +inline constexpr float kMusic3DepthLayers1AttnToOutWeight[] = { + -0.089236252f, 0.0525518507f, -0.240230575f, 0.269125998f, -0.101292014f, 0.0259092245f, + -0.056263376f, 0.295068711f, -0.115336083f, 0.0815472081f, -0.0484717898f, -0.0677854791f, + -0.109745525f, 0.0907665864f, -0.174272552f, -0.238202721f, 0.322775394f, 0.390976489f, + -0.191663697f, -0.303418845f, -0.149099991f, -0.186465546f, -0.343737304f, -0.00541822892f, + -0.145845205f, 0.196523979f, -0.0677690655f, 0.37471956f, 0.324774861f, -0.35198313f, + -0.35035792f, 0.381713569f, -0.150590762f, -0.0490807146f, 0.0661628842f, 0.22615692f, + 0.190808207f, -0.376599789f, 0.207373559f, 0.168457419f, -0.150729075f, -0.152172565f, + 0.322990745f, -0.309253514f, -0.150174946f, 0.00429178169f, 0.0985986665f, -0.27044192f, + 0.137766302f, 0.23700203f, 0.34502387f, 0.269047827f, 0.189797044f, 0.362468868f, + 0.220129684f, -0.138680667f, -0.228961349f, -0.0444117598f, -0.364232123f, -0.0774411336f, + 0.238597497f, -0.0691282675f, 0.252746373f, -0.345093429f, +}; + +inline constexpr float kMusic3DepthLayers1GateProjWeight[] = { + -0.177620962f, 0.2597453f, -0.0183478203f, -0.0889990628f, -0.255164981f, 0.312240273f, + 0.325825691f, 0.285744429f, -0.246762767f, 0.139981091f, 0.186758846f, -0.207195193f, + 0.275851816f, 0.132409617f, 0.193286657f, 0.104810715f, -0.164396062f, 0.144640818f, + 0.223800242f, 0.102230787f, -0.392100215f, 0.0906370506f, -0.201744199f, 0.0631843433f, + 0.190672606f, 0.101966813f, -0.136588871f, -0.15288341f, -0.364903212f, 0.167024285f, + -0.0454674028f, -0.198871046f, 0.333176434f, -0.097636655f, -0.353131205f, 0.06423188f, + 0.0207198113f, 0.274808794f, -0.321610957f, -0.296399593f, -0.257131666f, -0.332151592f, + 0.375464052f, 0.176796973f, 0.172022611f, 0.227783859f, -0.242431372f, -0.226034477f, + -0.298564792f, 0.372689933f, 0.0518091433f, -0.00384987495f, -0.263389319f, 0.250902623f, + -0.184188023f, -0.333890587f, 0.0720655024f, -0.204182297f, 0.217809021f, -0.193199798f, + -0.372662246f, -0.331037164f, 0.201268315f, 0.0590352304f, -0.0266745109f, 0.318640262f, + -0.0777957216f, -0.164482296f, -0.258543015f, -0.305072367f, 0.368411809f, -0.259310395f, + 0.286136717f, 0.013354715f, 0.273311436f, 0.371842712f, 0.28587833f, 0.0578616448f, + -0.0837050155f, -0.134995088f, -0.18786633f, -0.355621457f, -0.245506823f, 0.122304752f, + -0.227388099f, 0.242097467f, 0.155244425f, -0.315864056f, 0.120462477f, -0.0332966223f, + -0.178961247f, -0.229985386f, 0.163836181f, -0.176879257f, 0.29855001f, 0.00377037446f, +}; + +inline constexpr float kMusic3DepthLayers1UpProjWeight[] = { + -0.156360582f, 0.208315358f, 0.26648891f, 0.00488821976f, 0.0422499329f, 0.306499898f, + -0.158144116f, 0.383670807f, 0.0921861529f, 0.217582762f, 0.294574052f, -0.0372546092f, + 0.145152107f, 0.236577541f, 0.0588160008f, -0.359239966f, -0.335825205f, -0.373490095f, + -0.0104676811f, -0.209524646f, -0.0684351102f, 0.301494658f, -0.189800844f, 0.364455402f, + -0.29654634f, 0.0367393643f, -0.343568146f, -0.087102741f, -0.0555520803f, 0.134178072f, + 0.0688618124f, 0.274859816f, -0.171125636f, 0.0677729473f, -0.296108484f, -0.173164308f, + 0.176690996f, 0.1389658f, -0.367439836f, -0.0118950941f, 0.0101034604f, -0.291021645f, + -0.322847813f, 0.0665870756f, -0.00411047926f, -0.0396226905f, 0.320124626f, 0.29440093f, + 0.103167541f, -0.201320022f, -0.181822598f, -0.241450682f, -0.161873683f, 0.0359227397f, + 0.319874555f, -0.376149982f, 0.119396038f, -0.247087955f, -0.171433777f, 0.376010954f, + 0.289079189f, -0.133584723f, -0.381381333f, -0.102941647f, 0.110450886f, -0.353391081f, + -0.147559866f, -0.303472489f, 0.334431708f, 0.377299547f, 0.115231797f, 0.39738223f, + 0.155738056f, 0.288874388f, -0.247085825f, 0.335589975f, -0.32165581f, -0.255412221f, + 0.342113763f, 0.099338077f, 0.136778146f, 0.133607f, 0.0126249203f, -0.277274936f, + 0.321981311f, -0.0318961851f, -0.00818332471f, 0.128572717f, -0.0585816205f, 0.156976536f, + -0.326519251f, 0.134237051f, 0.331393272f, 0.220521033f, -0.0401363373f, -0.381544024f, +}; + +inline constexpr float kMusic3DepthLayers1DownProjWeight[] = { + -0.146900073f, -0.0802792385f, 0.198160246f, 0.255235285f, 0.03263437f, -0.31027177f, + 0.250061959f, -0.0290423501f, -0.395245433f, -0.0595116392f, 0.319482237f, 0.296167105f, + -0.271974564f, -0.030053325f, 0.0819217637f, -0.298798651f, -0.139215887f, 0.262187779f, + 0.0657761097f, -0.386526257f, -0.289677799f, 0.0556387007f, 0.228297159f, 0.214134261f, + 0.056473393f, -0.0931715369f, -0.290411174f, -0.00657150196f, -0.331059456f, 0.0572967269f, + -0.268506259f, 0.105323069f, 0.079193756f, -0.374026597f, 0.234853059f, 0.143354684f, + -0.341790736f, -0.104856536f, -0.305431783f, 0.292345613f, -0.128306434f, 0.205571398f, + 0.202983588f, -0.194840729f, 0.28338936f, 0.244731829f, -0.193481907f, -0.393140525f, + -0.145690084f, 0.170403093f, -0.0758929849f, -0.0890594274f, 0.0589395016f, -0.0795491859f, + -0.0787769482f, 0.163449839f, -0.29267931f, 0.326280326f, 0.385277629f, -0.221440107f, + -0.00460843788f, -0.0526766852f, 0.121300489f, 0.21019879f, -0.200585052f, -0.271626353f, + -0.00472763181f, 0.389031947f, 0.018198533f, 0.372450918f, 0.317858666f, 0.0532068759f, + -0.194724083f, -0.127091691f, -0.342878848f, -0.0156755578f, 0.0402018167f, -0.0872357786f, + 0.321284831f, -0.158092856f, -0.28073433f, -0.0372010134f, -0.0144064529f, -0.227071285f, + 0.0282620806f, -0.101594567f, -0.0340883732f, -0.0057586995f, -0.154306158f, 0.25334844f, + 0.147381946f, -0.367641181f, 0.115573056f, 0.215351924f, 0.273358971f, -0.394422412f, +}; + +inline constexpr float kMusic3DepthAudioHeads0Weight[] = { + -0.125023454f, -0.0388431698f, 0.230919912f, 0.313670605f, 0.382543921f, 0.318355501f, + 0.345192015f, 0.232492805f, -0.00497135893f, -0.109355941f, 0.250401497f, 0.097844705f, + 0.394595414f, 0.0670079514f, -0.291654676f, 0.222555727f, -0.00455677137f, 0.272679627f, + 0.140599325f, 0.119227894f, 0.0422671959f, -0.305884659f, -0.106536627f, -0.38063705f, + 0.162541211f, 0.272025168f, -0.0177881066f, -0.0798370317f, -0.210586295f, -0.300111979f, + -0.239139333f, 0.245089337f, -0.102184929f, -0.328743309f, 0.0846142992f, -0.00910543278f, + 0.150556028f, -0.24208802f, -0.319096625f, 0.111234911f, +}; + +inline constexpr float kMusic3DepthAudioHeads1Weight[] = { + 0.358276337f, -0.012995651f, 0.0936066657f, 0.101945467f, 0.192999631f, 0.117226876f, + -0.127576038f, -0.199650228f, -0.185658172f, -0.342306554f, 0.100676924f, -0.0652517378f, + 0.0584089123f, -0.00920870248f, -0.0539110042f, 0.312832326f, 0.233210683f, -0.12854439f, + -0.188669294f, 0.0620129853f, 0.288831353f, 0.247275919f, 0.0134861479f, 0.393031716f, + -0.106354609f, -0.399074107f, 0.184108645f, 0.399977475f, -0.142883748f, -0.384381354f, + -0.28580296f, -0.0142774777f, 0.332824409f, 0.389539242f, 0.323128849f, -0.0691485256f, + -0.0419734754f, -0.360599726f, 0.35856545f, -0.321627796f, +}; + +inline constexpr float kMusic3DepthAudioHeads2Weight[] = { + -0.37389636f, -0.254556626f, -0.140226156f, 0.281383991f, -0.300371021f, 0.0948436856f, + 0.257056266f, 0.331314415f, -0.215984389f, 0.0767566785f, -0.373282045f, -0.208509982f, + 0.213784605f, 0.399269879f, 0.104883388f, -0.0801773742f, 0.0182543211f, -0.265020221f, + 0.290009886f, 0.080197148f, -0.152111411f, -0.182213128f, 0.116138779f, 0.198604703f, + -0.313747883f, 0.273293734f, -0.371646613f, 0.200609922f, -0.219352081f, 0.288841486f, + -0.22445862f, -0.286815286f, -0.132350147f, -0.378125101f, 0.257152498f, -0.0503132902f, + 0.214739546f, 0.172700912f, -0.357932389f, -0.261758775f, +}; + +inline constexpr int64_t kMusic3DepthHidden = 8; +inline constexpr int64_t kMusic3DepthLayers = 2; +inline constexpr int64_t kMusic3DepthHeads = 2; +inline constexpr int64_t kMusic3DepthIntermediate = 12; +inline constexpr int64_t kMusic3DepthAudioVocab = 5; +inline constexpr int64_t kMusic3DepthCodebooks = 4; +inline constexpr int64_t kMusic3DepthMaxPositions = 6; + +inline constexpr int64_t kMusic3DepthSeqLen = 4; +inline constexpr int32_t kMusic3DepthResidualCodes[] = {2, 4}; +inline constexpr float kMusic3DepthLastHidden[] = { + -0.933229387f, -0.416873962f, -0.45687449f, 0.142006531f, -0.673693597f, -1.15043724f, + -1.16871572f, -0.876845241f, +}; + +inline constexpr float kMusic3DepthSemanticEmbed[] = { + 0.272781044f, 0.0957857892f, -0.16147168f, -0.447115302f, -0.443887055f, 0.654393017f, + -0.132591724f, 0.596679091f, +}; + +inline constexpr float kMusic3DepthInputsEmbeds[] = { + 0.582876503f, 0.0942398608f, 0.428695709f, -0.571250081f, 0.344531476f, -0.12035355f, + -0.317844391f, -0.498678595f, -0.0887515992f, -0.432944298f, -0.245937571f, 0.159828186f, + -0.541432798f, -0.286695689f, 0.645642042f, -0.226113424f, -0.39552331f, -0.333317846f, + -0.598956048f, -0.0496086776f, -0.199140042f, -0.167117208f, 0.232427403f, 0.187026441f, + 0.0249410793f, -0.0776746869f, -0.155532748f, 0.37522763f, 0.0469431728f, 0.342892408f, + -0.00591611862f, 0.246996477f, +}; + +inline constexpr float kMusic3DepthOut[] = { + 0.697917104f, 0.816844225f, 1.86293542f, 0.0307284649f, 0.0770663768f, -0.440063655f, + -2.14074278f, 0.115381733f, 0.0236670263f, 0.063777104f, 1.27115119f, 1.41022408f, + -1.53454375f, -0.518782377f, 0.574646533f, -1.77995825f, -1.44889307f, -0.984621823f, + -0.963365555f, 1.03277826f, -1.49631679f, -0.570129931f, 0.0876278058f, -0.619890451f, + -0.760342658f, -1.45389056f, 0.546924591f, 0.0601777546f, -1.33618903f, 1.68068385f, + -0.339971066f, -0.359913737f, +}; + +inline constexpr float kMusic3DepthAudioHead0[] = { + -0.125023454f, -0.0388431698f, 0.230919912f, 0.313670605f, 0.382543921f, 0.318355501f, + 0.345192015f, 0.232492805f, -0.00497135893f, -0.109355941f, 0.250401497f, 0.097844705f, + 0.394595414f, 0.0670079514f, -0.291654676f, 0.222555727f, -0.00455677137f, 0.272679627f, + 0.140599325f, 0.119227894f, 0.0422671959f, -0.305884659f, -0.106536627f, -0.38063705f, + 0.162541211f, 0.272025168f, -0.0177881066f, -0.0798370317f, -0.210586295f, -0.300111979f, + -0.239139333f, 0.245089337f, -0.102184929f, -0.328743309f, 0.0846142992f, -0.00910543278f, + 0.150556028f, -0.24208802f, -0.319096625f, 0.111234911f, +}; + +inline constexpr float kMusic3DepthAudioHead1[] = { + 0.358276337f, -0.012995651f, 0.0936066657f, 0.101945467f, 0.192999631f, 0.117226876f, + -0.127576038f, -0.199650228f, -0.185658172f, -0.342306554f, 0.100676924f, -0.0652517378f, + 0.0584089123f, -0.00920870248f, -0.0539110042f, 0.312832326f, 0.233210683f, -0.12854439f, + -0.188669294f, 0.0620129853f, 0.288831353f, 0.247275919f, 0.0134861479f, 0.393031716f, + -0.106354609f, -0.399074107f, 0.184108645f, 0.399977475f, -0.142883748f, -0.384381354f, + -0.28580296f, -0.0142774777f, 0.332824409f, 0.389539242f, 0.323128849f, -0.0691485256f, + -0.0419734754f, -0.360599726f, 0.35856545f, -0.321627796f, +}; + +inline constexpr float kMusic3DepthAudioHead2[] = { + -0.37389636f, -0.254556626f, -0.140226156f, 0.281383991f, -0.300371021f, 0.0948436856f, + 0.257056266f, 0.331314415f, -0.215984389f, 0.0767566785f, -0.373282045f, -0.208509982f, + 0.213784605f, 0.399269879f, 0.104883388f, -0.0801773742f, 0.0182543211f, -0.265020221f, + 0.290009886f, 0.080197148f, -0.152111411f, -0.182213128f, 0.116138779f, 0.198604703f, + -0.313747883f, 0.273293734f, -0.371646613f, 0.200609922f, -0.219352081f, 0.288841486f, + -0.22445862f, -0.286815286f, -0.132350147f, -0.378125101f, 0.257152498f, -0.0503132902f, + 0.214739546f, 0.172700912f, -0.357932389f, -0.261758775f, +}; + +inline constexpr float kMusic3DepthHeadLogits[] = { + -0.237208039f, -0.754835665f, 1.07426786f, -0.208826303f, -0.415472925f, -0.734239697f, + 0.160868108f, -0.781144023f, 1.19951057f, -0.749293447f, 0.948745251f, 0.214510977f, + 0.320913017f, 0.60811156f, 1.00721955f, +}; + +inline constexpr float kMusic3DepthBoundaryIn[] = { + 0.567904711f, -0.837899268f, -0.27188769f, 0.760091305f, -0.521607995f, -0.730282187f, + -0.779160798f, 0.639805615f, 0.0831415877f, 0.56697011f, 0.826298833f, 0.422638327f, + 0.408975631f, 0.324222863f, 0.330917299f, 0.597556293f, 0.772705615f, 0.777406037f, + -0.199698955f, 0.705476463f, 0.322509408f, 0.235658631f, 0.292415231f, 0.56302309f, + -0.614843607f, -0.476914048f, 0.360171467f, 0.60502404f, -0.862635791f, 0.697835743f, + 0.521521032f, -0.560520649f, 0.576482356f, 0.658677399f, -0.37175703f, 0.516259313f, + -0.320482403f, -0.626262903f, -0.566621125f, 0.287122667f, -0.124798559f, -0.875989497f, + 0.182431683f, -0.130777687f, 0.109095991f, -0.240355879f, 0.437126756f, -0.628957391f, +}; + +inline constexpr float kMusic3DepthBoundaryOut[] = { + 0.603923678f, -1.83060265f, -2.15869832f, 0.285047531f, -0.127669305f, 0.0745266229f, + 0.455819637f, 0.374842823f, -0.258500129f, 0.865134597f, 1.41109312f, -1.02009988f, + 0.623256862f, 1.69298553f, 1.02912676f, 0.789238572f, -0.467952341f, 0.81542033f, + -0.177284345f, -0.517442405f, -1.01898336f, 2.17047977f, 0.834154189f, 0.665582418f, + -1.06182766f, -0.949165106f, 0.991474569f, 0.106316902f, -1.68727899f, 1.10711122f, + 0.393077761f, -1.10175228f, 0.331782728f, 1.90146673f, 0.00542743132f, 0.0402562879f, + -1.12874198f, -0.849031806f, -1.52505922f, -0.249736011f, -0.607077301f, -1.08016372f, + 0.703620374f, -0.541962743f, 0.0786594898f, -0.568853259f, 1.45762539f, -2.16752458f, +}; + + +inline constexpr int32_t kMusic3FeedbackCodes[] = {3, 1, 4, 2}; +inline constexpr float kMusic3FeedbackLmRow[] = { + 1.24811316f, 1.03448248f, 1.17996478f, 0.595581055f, -1.28767121f, 0.941835344f, + 0.827963114f, 0.618974805f, +}; + +inline constexpr float kMusic3FeedbackOut[] = { + 0.766901374f, 0.752031863f, 0.0739473104f, -0.196501881f, -0.627536237f, 0.946985424f, + 0.955929399f, 0.997570932f, +}; + + +} // namespace vllm_test diff --git a/tests/vllm/models/test_minimax_music3_ar.cpp b/tests/vllm/models/test_minimax_music3_ar.cpp new file mode 100644 index 000000000..6f7bd84a8 --- /dev/null +++ b/tests/vllm/models/test_minimax_music3_ar.cpp @@ -0,0 +1,679 @@ +// MiniMax-Music3 β€” the AUTOREGRESSIVE half at REDUCED dimensions (#672, W2+W3). +// +// Every golden here was produced by EXECUTING upstream's own classes +// (scripts/gen-minimax-music3-ar-goldens.py against diffusers PR #14456 head +// c6da9936) in float32 at dimensions small enough to check in. No weight byte of +// the 28.5 GB checkpoint is present, so this gate runs in CI with no asset. +// +// The FULL-SCALE companion β€” the real bf16 checkpoint against the committed +// oracle goldens β€” is tests/parity/test_minimax_music3_ar_real.cpp. This file +// separates an ALGEBRA defect from bf16 rounding; that one proves the algebra +// survives contact with the real weights. +// +// THE TOLERANCE, and why it is what it is. The goldens are torch float32; this +// port accumulates in double and rounds once. Over the reductions here (<= 12 +// terms for the MLP, <= 8 for attention) the two differ only by float32's own +// rounding of the golden, ~6e-8 relative per operation, and a 2-layer stack +// cannot compound that past ~1e-6. kRelTol is 1e-5 with a 1e-6 absolute floor β€” +// an order above that bound and orders BELOW any algebra defect, every one of +// which (a transposed weight, a missing softmax, an inverted interpolation +// ratio) moves values by O(1). That claim was PROVEN by mutation rather than +// asserted: softmax -> plain normalize reds 3 cases / 3 assertions, an inverted +// interpolation ratio 3 / 11, a dropped codebook offset 1 / 1, a non-causal +// attention 4 / 65, and a frame row that keeps depth position 0 reds 1 / 24. +#include + +#include +#include +#include +#include +#include +#include + +#include "minimax_music3_ar_goldens.inc" +#include "vllm/model_executor/models/minimax_music3_ar.h" + +namespace { + +namespace m3 = vllm::models::music3; + +constexpr double kRelTol = 1e-5; +constexpr double kAbsFloor = 1e-6; + +// Compare and REPORT the count: a gate that cannot say how many values it +// examined has not reported. Returns the worst absolute deviation seen. +double ExpectClose(const std::vector& got, const float* want, size_t count, + const char* what) { + REQUIRE_MESSAGE(got.size() == count, what); + double worst = 0.0; + size_t bad = 0; + size_t first_bad = 0; + for (size_t i = 0; i < count; ++i) { + const double a = got[i]; + const double b = want[i]; + const double diff = std::abs(a - b); + const double bound = std::max(kAbsFloor, kRelTol * std::max(std::abs(a), std::abs(b))); + if (!(diff <= bound)) { + if (bad == 0) first_bad = i; + ++bad; + } + worst = std::max(worst, diff); + } + INFO(what << ": " << count << " values compared, " << bad << " outside tolerance" + << (bad != 0 ? ", first at index " + std::to_string(first_bad) + " got " + + std::to_string(got[first_bad]) + " want " + + std::to_string(want[first_bad]) + : std::string())); + CHECK(bad == 0); + return worst; +} + +std::vector ToVector(const float* data, size_t count) { + return std::vector(data, data + count); +} + +m3::ConditionMixConfig CondConfig() { + m3::ConditionMixConfig config; + config.condition_hidden_dim = vllm_test::kMusic3CondHidden; + config.num_condition_layers = vllm_test::kMusic3CondLayers; + config.out_dim = vllm_test::kMusic3CondOutDim; + config.input_sampling_rate = vllm_test::kMusic3CondInputSamplingRate; + config.input_hop_length = vllm_test::kMusic3CondInputHopLength; + config.output_sampling_rate = vllm_test::kMusic3CondOutputSamplingRate; + config.output_hop_length = vllm_test::kMusic3CondOutputHopLength; + return config; +} + +m3::ConditionMixWeights CondWeights() { + m3::ConditionMixWeights weights; + weights.layer_weight_logits = + ToVector(vllm_test::kMusic3CondLayerWeightLogits, + static_cast(vllm_test::kMusic3CondLayers)); + weights.layer_scale = ToVector(vllm_test::kMusic3CondLayerScale, 1); + weights.proj_weight = ToVector( + vllm_test::kMusic3CondProjWeight, + static_cast(vllm_test::kMusic3CondOutDim * vllm_test::kMusic3CondHidden * 3)); + weights.proj_bias = + ToVector(vllm_test::kMusic3CondProjBias, static_cast(vllm_test::kMusic3CondOutDim)); + return weights; +} + +m3::ConditionMixConfig CondDownConfig() { + m3::ConditionMixConfig config; + config.condition_hidden_dim = vllm_test::kMusic3CondDownHidden; + config.num_condition_layers = vllm_test::kMusic3CondDownLayers; + config.out_dim = vllm_test::kMusic3CondDownOutDim; + config.input_sampling_rate = vllm_test::kMusic3CondDownInputSamplingRate; + config.input_hop_length = vllm_test::kMusic3CondDownInputHopLength; + config.output_sampling_rate = vllm_test::kMusic3CondDownOutputSamplingRate; + config.output_hop_length = vllm_test::kMusic3CondDownOutputHopLength; + return config; +} + +m3::ConditionMixWeights CondDownWeights() { + m3::ConditionMixWeights weights; + weights.layer_weight_logits = + ToVector(vllm_test::kMusic3CondDownLayerWeightLogits, + static_cast(vllm_test::kMusic3CondDownLayers)); + weights.layer_scale = ToVector(vllm_test::kMusic3CondDownLayerScale, 1); + weights.proj_weight = ToVector(vllm_test::kMusic3CondDownProjWeight, + static_cast(vllm_test::kMusic3CondDownOutDim * + vllm_test::kMusic3CondDownHidden * 3)); + weights.proj_bias = ToVector(vllm_test::kMusic3CondDownProjBias, + static_cast(vllm_test::kMusic3CondDownOutDim)); + return weights; +} + +m3::DepthDecoderConfig DepthConfig() { + m3::DepthDecoderConfig config; + config.hidden_size = vllm_test::kMusic3DepthHidden; + config.num_layers = vllm_test::kMusic3DepthLayers; + config.num_attention_heads = vllm_test::kMusic3DepthHeads; + config.intermediate_size = vllm_test::kMusic3DepthIntermediate; + config.audio_vocab_size = vllm_test::kMusic3DepthAudioVocab; + config.num_codebooks = vllm_test::kMusic3DepthCodebooks; + config.max_position_embeddings = vllm_test::kMusic3DepthMaxPositions; + return config; +} + +m3::DepthDecoderWeights DepthWeights() { + const m3::DepthDecoderConfig config = DepthConfig(); + const size_t hidden = static_cast(config.hidden_size); + const size_t inter = static_cast(config.intermediate_size); + m3::DepthDecoderWeights weights; + weights.audio_embeddings = + ToVector(vllm_test::kMusic3DepthAudioEmbeddingsWeight, + static_cast(config.audio_vocab_size * config.residual_codebooks()) * hidden); + weights.projection = ToVector(vllm_test::kMusic3DepthProjectionWeight, hidden * hidden); + weights.pos_embedding = + ToVector(vllm_test::kMusic3DepthPosEmbeddingWeight, + static_cast(config.max_position_embeddings) * hidden); + weights.norm = ToVector(vllm_test::kMusic3DepthNormWeight, hidden); + + const float* const input_norms[] = {vllm_test::kMusic3DepthLayers0InputLayernormWeight, + vllm_test::kMusic3DepthLayers1InputLayernormWeight}; + const float* const post_norms[] = { + vllm_test::kMusic3DepthLayers0PostAttentionLayernormWeight, + vllm_test::kMusic3DepthLayers1PostAttentionLayernormWeight}; + const float* const to_q[] = {vllm_test::kMusic3DepthLayers0AttnToQWeight, + vllm_test::kMusic3DepthLayers1AttnToQWeight}; + const float* const to_k[] = {vllm_test::kMusic3DepthLayers0AttnToKWeight, + vllm_test::kMusic3DepthLayers1AttnToKWeight}; + const float* const to_v[] = {vllm_test::kMusic3DepthLayers0AttnToVWeight, + vllm_test::kMusic3DepthLayers1AttnToVWeight}; + const float* const to_out[] = {vllm_test::kMusic3DepthLayers0AttnToOutWeight, + vllm_test::kMusic3DepthLayers1AttnToOutWeight}; + const float* const gate[] = {vllm_test::kMusic3DepthLayers0GateProjWeight, + vllm_test::kMusic3DepthLayers1GateProjWeight}; + const float* const up[] = {vllm_test::kMusic3DepthLayers0UpProjWeight, + vllm_test::kMusic3DepthLayers1UpProjWeight}; + const float* const down[] = {vllm_test::kMusic3DepthLayers0DownProjWeight, + vllm_test::kMusic3DepthLayers1DownProjWeight}; + for (int64_t layer = 0; layer < config.num_layers; ++layer) { + m3::DepthDecoderLayerWeights entry; + entry.input_layernorm = ToVector(input_norms[layer], hidden); + entry.post_attention_layernorm = ToVector(post_norms[layer], hidden); + entry.to_q = ToVector(to_q[layer], hidden * hidden); + entry.to_k = ToVector(to_k[layer], hidden * hidden); + entry.to_v = ToVector(to_v[layer], hidden * hidden); + entry.to_out = ToVector(to_out[layer], hidden * hidden); + entry.gate_proj = ToVector(gate[layer], inter * hidden); + entry.up_proj = ToVector(up[layer], inter * hidden); + entry.down_proj = ToVector(down[layer], hidden * inter); + weights.layers.push_back(std::move(entry)); + } + const float* const heads[] = {vllm_test::kMusic3DepthAudioHead0, + vllm_test::kMusic3DepthAudioHead1, + vllm_test::kMusic3DepthAudioHead2}; + for (int64_t head = 0; head < config.residual_codebooks(); ++head) { + weights.audio_heads.push_back( + ToVector(heads[head], static_cast(config.audio_vocab_size) * hidden)); + } + return weights; +} + +} // namespace + +// --------------------------------------------------------------------------- +// W2 β€” the prompt the checkpoint contract fixes +// --------------------------------------------------------------------------- + +TEST_CASE("music3 ar: the assembled prompt matches upstream string for string") { + REQUIRE(vllm_test::kMusic3PromptGoldenCount == 2); + int cases = 0; + for (int64_t i = 0; i < vllm_test::kMusic3PromptGoldenCount; ++i) { + const vllm_test::Music3PromptGolden& golden = vllm_test::kMusic3PromptGoldens[i]; + INFO("prompt golden " << golden.name); + CHECK(m3::CleanCaption(golden.prompt) == std::string(golden.clean_caption)); + CHECK(m3::NormalizeLyrics(golden.lyrics) == std::string(golden.normalized_lyrics)); + CHECK(m3::AssembleArPrompt(golden.prompt, golden.lyrics) == std::string(golden.assembled)); + ++cases; + } + MESSAGE("prompt goldens checked: " << cases); +} + +TEST_CASE("music3 ar: the prompt template constants are the checkpoint's, not ours") { + // Re-emitted from the upstream module by the generator, so a rename upstream + // reds here rather than silently generating a different song. + CHECK(m3::kAudioEndTokenId == vllm_test::kMusic3AudioEndTokenId); + CHECK(m3::kAudioCfgTokenId == vllm_test::kMusic3AudioCfgTokenId); + CHECK(m3::kAudioCodeOffset == vllm_test::kMusic3AudioCodeOffset); + CHECK(m3::kSemanticVocabSize == vllm_test::kMusic3SemanticVocabSize); + CHECK(m3::kMaxPromptTokens == vllm_test::kMusic3MaxPromptTokens); + CHECK(m3::kMaxAudioFrames == vllm_test::kMusic3MaxAudioFrames); + CHECK(m3::kArCfgTopK == vllm_test::kMusic3ArCfgTopK); + CHECK(m3::kArSamplingTopK == vllm_test::kMusic3ArSamplingTopK); + CHECK(m3::kArCfgScale == doctest::Approx(vllm_test::kMusic3ArCfgScale)); +} + +TEST_CASE("music3 ar: an empty description or empty lyrics is refused") { + CHECK_THROWS_AS(m3::AssembleArPrompt("", "[verse]\nx"), std::runtime_error); + CHECK_THROWS_AS(m3::AssembleArPrompt(" \n\t ", "[verse]\nx"), std::runtime_error); + CHECK_THROWS_AS(m3::AssembleArPrompt("pop", ""), std::runtime_error); + CHECK_THROWS_AS(m3::AssembleArPrompt("pop", " \n "), std::runtime_error); +} + +TEST_CASE("music3 ar: the unconditional row keeps the first and last two tokens") { + const std::vector ids( + vllm_test::kMusic3UncondIdsIn, + vllm_test::kMusic3UncondIdsIn + vllm_test::kMusic3UncondIdsCount); + const std::vector want( + vllm_test::kMusic3UncondIdsOut, + vllm_test::kMusic3UncondIdsOut + vllm_test::kMusic3UncondIdsCount); + const std::vector got = m3::UnconditionalPromptIds(ids); + REQUIRE(got.size() == want.size()); + int matched = 0; + for (size_t i = 0; i < want.size(); ++i) { + CHECK(got[i] == want[i]); + if (got[i] == want[i]) ++matched; + } + MESSAGE("unconditional ids matched: " << matched << "/" << want.size()); + // The input must not be mutated in place. + CHECK(ids[1] == vllm_test::kMusic3UncondIdsIn[1]); +} + +TEST_CASE("music3 ar: a prompt too short for the [1:-2] slice is refused") { + CHECK_THROWS_AS(m3::UnconditionalPromptIds({1, 2, 3}), std::runtime_error); + CHECK_NOTHROW(m3::UnconditionalPromptIds({1, 2, 3, 4})); +} + +TEST_CASE("music3 ar: the frame budget is capped, floored and refused") { + // encoders.py:287 β€” 25 Hz is the AR frame rate. + CHECK(m3::MaxArFrames(1.0, 25.0) == 25); + CHECK(m3::MaxArFrames(60.0, 25.0) == 1500); + // The 9000-frame ceiling binds before the duration does. + CHECK(m3::MaxArFrames(600.0, 25.0) == m3::kMaxAudioFrames); + CHECK(m3::MaxArFrames(1e9, 25.0) == m3::kMaxAudioFrames); + CHECK_THROWS_AS(m3::MaxArFrames(0.0, 25.0), std::runtime_error); + CHECK_THROWS_AS(m3::MaxArFrames(-1.0, 25.0), std::runtime_error); + // Shorter than one frame: upstream raises rather than emitting silence. + CHECK_THROWS_AS(m3::MaxArFrames(0.01, 25.0), std::runtime_error); +} + +// --------------------------------------------------------------------------- +// W2 β€” the semantic stage's logit pipeline +// --------------------------------------------------------------------------- + +TEST_CASE("music3 ar: the vocabulary mask leaves only the code window and the end token") { + const std::vector blocked = m3::SemanticVocabMask( + vllm_test::kMusic3SemanticVocab, vllm_test::kMusic3SemanticOffset, + vllm_test::kMusic3SemanticWindow, + static_cast(vllm_test::kMusic3SemanticEndId)); + REQUIRE(static_cast(blocked.size()) == vllm_test::kMusic3SemanticVocab); + int allowed = 0; + int matched = 0; + for (size_t i = 0; i < blocked.size(); ++i) { + CHECK(blocked[i] == vllm_test::kMusic3SemanticVocabMask[i]); + if (blocked[i] == vllm_test::kMusic3SemanticVocabMask[i]) ++matched; + if (!blocked[i]) ++allowed; + } + MESSAGE("vocab mask entries matched: " << matched << "/" << blocked.size() + << ", allowed: " << allowed); + CHECK(allowed == vllm_test::kMusic3SemanticWindow + 1); +} + +TEST_CASE("music3 ar: the vocabulary mask refuses a window that does not fit") { + CHECK_THROWS_AS(m3::SemanticVocabMask(10, 8, 6, 3), std::runtime_error); + CHECK_THROWS_AS(m3::SemanticVocabMask(40, 8, 6, 40), std::runtime_error); + CHECK_THROWS_AS(m3::SemanticVocabMask(0, 0, 0, 0), std::runtime_error); + // The REAL configuration must fit: 151675 + 16384 <= 200000. + CHECK_NOTHROW(m3::SemanticVocabMask(200000, m3::kAudioCodeOffset, m3::kSemanticVocabSize, + m3::kAudioEndTokenId)); +} + +TEST_CASE("music3 ar: guided semantic logits match upstream's inline CFG block") { + const size_t vocab = static_cast(vllm_test::kMusic3SemanticVocab); + const std::vector conditional = ToVector(vllm_test::kMusic3SemanticLogitsIn, vocab); + const std::vector unconditional = + ToVector(vllm_test::kMusic3SemanticLogitsIn + vocab, vocab); + const std::vector blocked = m3::SemanticVocabMask( + vllm_test::kMusic3SemanticVocab, vllm_test::kMusic3SemanticOffset, + vllm_test::kMusic3SemanticWindow, + static_cast(vllm_test::kMusic3SemanticEndId)); + const std::vector got = m3::GuidedSemanticLogits( + conditional, unconditional, blocked, vllm_test::kMusic3SemanticCfgTopK, m3::kArCfgScale); + REQUIRE(got.size() == vocab); + size_t finite = 0; + size_t neg_inf = 0; + size_t nan_count = 0; + size_t matched = 0; + for (size_t i = 0; i < vocab; ++i) { + const float want = vllm_test::kMusic3SemanticGuided[i]; + if (std::isnan(got[i])) ++nan_count; + if (std::isinf(want)) { + ++neg_inf; + CHECK(std::isinf(got[i])); + CHECK(got[i] < 0.0f); + if (std::isinf(got[i]) && got[i] < 0.0f) ++matched; + } else { + ++finite; + const double bound = std::max(kAbsFloor, kRelTol * std::abs(static_cast(want))); + CHECK(std::abs(static_cast(got[i]) - want) <= bound); + if (std::abs(static_cast(got[i]) - want) <= bound) ++matched; + } + } + MESSAGE("guided semantic logits: " << vocab << " compared, " << matched << " matched, " + << finite << " finite, " << neg_inf << " -inf, " + << nan_count << " NaN"); + // The re-mask exists precisely so no position is NaN (header note). + CHECK(nan_count == 0); +} + +TEST_CASE("music3 ar: depth CFG is the plain mix with no mask and no pre-restriction") { + const std::vector cond{1.0f, -2.0f, 0.5f, 4.0f}; + const std::vector uncond{0.0f, 1.0f, 0.5f, -1.0f}; + const std::vector got = m3::GuidedDepthLogits(cond, uncond, m3::kArCfgScale); + REQUIRE(got.size() == 4); + for (size_t i = 0; i < got.size(); ++i) { + const double want = uncond[i] + (cond[i] - uncond[i]) * m3::kArCfgScale; + CHECK(std::abs(static_cast(got[i]) - want) <= 1e-6); + } + MESSAGE("depth CFG values checked: " << got.size()); + CHECK_THROWS_AS(m3::GuidedDepthLogits({1.0f}, {1.0f, 2.0f}, m3::kArCfgScale), + std::runtime_error); +} + +TEST_CASE("music3 ar: the top-k filter reproduces _sample_top_k up to the draw") { + const size_t n = static_cast(vllm_test::kMusic3TopKProbeN); + const std::vector logits = ToVector(vllm_test::kMusic3TopKProbeIn, n); + // The golden INPUT carries a real NaN at index 3 and a real -inf at index 7 β€” + // the generator emits them as NAN / -INFINITY rather than substituting them + // away, because `nan_to_num` (encoders.py:95) exists for exactly those. + REQUIRE(std::isnan(logits[3])); + REQUIRE(std::isinf(logits[7])); + const std::vector got = m3::TopKProbabilities(logits, vllm_test::kMusic3TopKProbeK); + const double worst = ExpectClose(got, vllm_test::kMusic3TopKProbeProbs, n, "top-k probabilities"); + double sum = 0.0; + size_t nonzero = 0; + for (const float p : got) { + sum += p; + if (p > 0.0f) ++nonzero; + } + MESSAGE("top-k probabilities: " << n << " values, " << nonzero + << " nonzero, sum " << sum << ", worst dev " << worst); + CHECK(std::abs(sum - 1.0) <= 1e-6); + CHECK(nonzero == static_cast(vllm_test::kMusic3TopKProbeK)); +} + +// --------------------------------------------------------------------------- +// W3 β€” the learned condition mix +// --------------------------------------------------------------------------- + +TEST_CASE("music3 ar: the latent timeline length is the checkpoint's, at both rate polarities") { + CHECK(m3::ConditionLatentLength(vllm_test::kMusic3CondFrames, CondConfig()) == + vllm_test::kMusic3CondLatentLength); + CHECK(m3::ConditionLatentLength(vllm_test::kMusic3CondDownFrames, CondDownConfig()) == + vllm_test::kMusic3CondDownLatentLength); + // The REAL configuration: 25 AR frames -> 86 latent frames (86.133 truncated), + // which is the shape of the committed full-scale golden condition_chunk0. + m3::ConditionMixConfig real; + CHECK(m3::ConditionLatentLength(25, real) == 86); + // 1 AR frame is 3 latent frames (1 * 1.8375 * 1.875 = 3.445, truncated), NOT + // 1: at the real polarity the latent timeline is always the LONGER one. + CHECK(m3::ConditionLatentLength(1, real) == 3); + // max(1, ...) is not decoration: one frame at the DOWN polarity truncates to 0. + CHECK(m3::ConditionLatentLength(1, CondDownConfig()) == 1); + CHECK_THROWS_AS(m3::ConditionLatentLength(0, real), std::runtime_error); +} + +TEST_CASE("music3 ar: nearest interpolation upsamples and downsamples on input/output scale") { + // in_len 3 -> out_len 7: floor(t * 3/7) = 0,0,0,1,1,2,2. + const std::vector in{1.0f, 2.0f, 3.0f, 10.0f, 20.0f, 30.0f}; + const std::vector up = m3::NearestInterpolate1d(in, 2, 3, 7); + const std::vector want_up{1, 1, 1, 2, 2, 3, 3, 10, 10, 10, 20, 20, 30, 30}; + REQUIRE(up.size() == want_up.size()); + int matched = 0; + for (size_t i = 0; i < up.size(); ++i) { + CHECK(up[i] == want_up[i]); + if (up[i] == want_up[i]) ++matched; + } + // in_len 7 -> out_len 2: floor(t * 7/2) = 0, 3. + const std::vector wide{0, 1, 2, 3, 4, 5, 6}; + const std::vector down = m3::NearestInterpolate1d(wide, 1, 7, 2); + REQUIRE(down.size() == 2); + CHECK(down[0] == 0.0f); + CHECK(down[1] == 3.0f); + MESSAGE("nearest interpolation values matched: " << matched << "/" << want_up.size() + << " up, 2/2 down"); + CHECK_THROWS_AS(m3::NearestInterpolate1d(in, 2, 4, 7), std::runtime_error); +} + +TEST_CASE("music3 ar: the layer mix weights are a softmax") { + const std::vector weights = m3::ConditionLayerWeights( + ToVector(vllm_test::kMusic3CondLayerWeightLogits, + static_cast(vllm_test::kMusic3CondLayers))); + REQUIRE(static_cast(weights.size()) == vllm_test::kMusic3CondLayers); + double sum = 0.0; + for (const float w : weights) { + CHECK(w > 0.0f); + sum += w; + } + MESSAGE("layer mix weights: " << weights.size() << " entries summing to " << sum); + CHECK(std::abs(sum - 1.0) <= 1e-6); + // A softmax is shift-invariant; a plain normalize is not. This separates them. + std::vector shifted = ToVector(vllm_test::kMusic3CondLayerWeightLogits, + static_cast(vllm_test::kMusic3CondLayers)); + for (float& value : shifted) value += 3.5f; + const std::vector shifted_weights = m3::ConditionLayerWeights(shifted); + for (size_t i = 0; i < weights.size(); ++i) { + CHECK(std::abs(static_cast(shifted_weights[i]) - weights[i]) <= 1e-6); + } +} + +TEST_CASE("music3 ar: the condition mix matches upstream (upsampling polarity)") { + const m3::ConditionMixConfig config = CondConfig(); + const size_t width = static_cast(config.num_condition_layers * config.condition_hidden_dim); + const std::vector hidden = + ToVector(vllm_test::kMusic3CondHiddenIn, + static_cast(vllm_test::kMusic3CondFrames) * width); + const std::vector got = + m3::ConditionMix(hidden, vllm_test::kMusic3CondFrames, config, CondWeights()); + const size_t count = + static_cast(vllm_test::kMusic3CondLatentLength * vllm_test::kMusic3CondOutDim); + const double worst = ExpectClose(got, vllm_test::kMusic3CondOut, count, "condition mix (up)"); + MESSAGE("condition mix (up): " << count << " values, worst deviation " << worst); +} + +TEST_CASE("music3 ar: the condition mix matches upstream (downsampling polarity)") { + const m3::ConditionMixConfig config = CondDownConfig(); + const size_t width = + static_cast(config.num_condition_layers * config.condition_hidden_dim); + const std::vector hidden = + ToVector(vllm_test::kMusic3CondDownHiddenIn, + static_cast(vllm_test::kMusic3CondDownFrames) * width); + const std::vector got = + m3::ConditionMix(hidden, vllm_test::kMusic3CondDownFrames, config, CondDownWeights()); + const size_t count = static_cast(vllm_test::kMusic3CondDownLatentLength * + vllm_test::kMusic3CondDownOutDim); + const double worst = ExpectClose(got, vllm_test::kMusic3CondDownOut, count, "condition mix (down)"); + MESSAGE("condition mix (down): " << count << " values, worst deviation " << worst); +} + +TEST_CASE("music3 ar: the condition mix refuses every wrong-shaped input by name") { + const m3::ConditionMixConfig config = CondConfig(); + const size_t width = + static_cast(config.num_condition_layers * config.condition_hidden_dim); + const std::vector hidden = + ToVector(vllm_test::kMusic3CondHiddenIn, + static_cast(vllm_test::kMusic3CondFrames) * width); + CHECK_THROWS_AS(m3::ConditionMix(hidden, vllm_test::kMusic3CondFrames + 1, config, CondWeights()), + std::runtime_error); + m3::ConditionMixWeights broken = CondWeights(); + broken.layer_weight_logits.pop_back(); + CHECK_THROWS_AS(m3::ConditionMix(hidden, vllm_test::kMusic3CondFrames, config, broken), + std::runtime_error); + broken = CondWeights(); + broken.layer_scale.push_back(1.0f); + CHECK_THROWS_AS(m3::ConditionMix(hidden, vllm_test::kMusic3CondFrames, config, broken), + std::runtime_error); + broken = CondWeights(); + broken.proj_weight.pop_back(); + CHECK_THROWS_AS(m3::ConditionMix(hidden, vllm_test::kMusic3CondFrames, config, broken), + std::runtime_error); + broken = CondWeights(); + broken.proj_bias.pop_back(); + CHECK_THROWS_AS(m3::ConditionMix(hidden, vllm_test::kMusic3CondFrames, config, broken), + std::runtime_error); +} + +// --------------------------------------------------------------------------- +// W3 β€” the RVQ depth decoder +// --------------------------------------------------------------------------- + +TEST_CASE("music3 ar: the depth sequence is assembled as _generate_depth_codes does") { + const m3::DepthDecoderConfig config = DepthConfig(); + const size_t hidden = static_cast(config.hidden_size); + const std::vector codes( + vllm_test::kMusic3DepthResidualCodes, + vllm_test::kMusic3DepthResidualCodes + (vllm_test::kMusic3DepthSeqLen - 2)); + const std::vector got = m3::DepthSequenceEmbeds( + ToVector(vllm_test::kMusic3DepthLastHidden, hidden), + ToVector(vllm_test::kMusic3DepthSemanticEmbed, hidden), codes, config, DepthWeights()); + const size_t count = static_cast(vllm_test::kMusic3DepthSeqLen) * hidden; + const double worst = + ExpectClose(got, vllm_test::kMusic3DepthInputsEmbeds, count, "depth sequence embeds"); + MESSAGE("depth sequence: " << vllm_test::kMusic3DepthSeqLen << " positions, " << count + << " values, worst deviation " << worst); +} + +TEST_CASE("music3 ar: a residual code outside the audio vocabulary is refused") { + const m3::DepthDecoderConfig config = DepthConfig(); + const size_t hidden = static_cast(config.hidden_size); + const std::vector last = ToVector(vllm_test::kMusic3DepthLastHidden, hidden); + const std::vector semantic = ToVector(vllm_test::kMusic3DepthSemanticEmbed, hidden); + const m3::DepthDecoderWeights weights = DepthWeights(); + CHECK_THROWS_AS( + m3::DepthSequenceEmbeds(last, semantic, + {static_cast(config.audio_vocab_size)}, config, weights), + std::runtime_error); + CHECK_THROWS_AS(m3::DepthSequenceEmbeds(last, semantic, {-1}, config, weights), + std::runtime_error); + // The last codebook is predicted, never fed back, so the sequence can carry at + // most residual_codebooks() - 1 codes. + CHECK_THROWS_AS(m3::DepthSequenceEmbeds(last, semantic, {0, 0, 0}, config, weights), + std::runtime_error); + CHECK_THROWS_AS(m3::DepthSequenceEmbeds({1.0f}, semantic, {0}, config, weights), + std::runtime_error); +} + +TEST_CASE("music3 ar: the depth decoder forward matches upstream") { + const m3::DepthDecoderConfig config = DepthConfig(); + const size_t hidden = static_cast(config.hidden_size); + const size_t count = static_cast(vllm_test::kMusic3DepthSeqLen) * hidden; + const std::vector got = m3::DepthDecoderForward( + ToVector(vllm_test::kMusic3DepthInputsEmbeds, count), vllm_test::kMusic3DepthSeqLen, config, + DepthWeights()); + const double worst = ExpectClose(got, vllm_test::kMusic3DepthOut, count, "depth decoder forward"); + MESSAGE("depth decoder: " << vllm_test::kMusic3DepthLayers << " layers, " + << vllm_test::kMusic3DepthSeqLen << " positions, " << count + << " values, worst deviation " << worst); +} + +TEST_CASE("music3 ar: the depth decoder is CAUSAL, so a truncated prefix is unchanged") { + // The whole reason ONE forward over the depth sequence may stand in for + // upstream's incremental schedule. Break causality and the prefix moves. + const m3::DepthDecoderConfig config = DepthConfig(); + const size_t hidden = static_cast(config.hidden_size); + const size_t full_count = static_cast(vllm_test::kMusic3DepthSeqLen) * hidden; + const std::vector full_in = ToVector(vllm_test::kMusic3DepthInputsEmbeds, full_count); + const std::vector full = m3::DepthDecoderForward( + full_in, vllm_test::kMusic3DepthSeqLen, config, DepthWeights()); + int compared = 0; + for (int64_t prefix = 1; prefix < vllm_test::kMusic3DepthSeqLen; ++prefix) { + const std::vector shortened( + full_in.begin(), full_in.begin() + static_cast(hidden) * prefix); + const std::vector got = + m3::DepthDecoderForward(shortened, prefix, config, DepthWeights()); + REQUIRE(got.size() == hidden * static_cast(prefix)); + for (size_t i = 0; i < got.size(); ++i) { + const double bound = + std::max(kAbsFloor, kRelTol * std::max(std::abs(static_cast(got[i])), + std::abs(static_cast(full[i])))); + CHECK(std::abs(static_cast(got[i]) - full[i]) <= bound); + ++compared; + } + } + MESSAGE("causal prefix values compared: " << compared); +} + +TEST_CASE("music3 ar: the position window binds at its boundary and one past it") { + const m3::DepthDecoderConfig config = DepthConfig(); + const size_t hidden = static_cast(config.hidden_size); + const size_t count = static_cast(config.max_position_embeddings) * hidden; + const std::vector got = + m3::DepthDecoderForward(ToVector(vllm_test::kMusic3DepthBoundaryIn, count), + config.max_position_embeddings, config, DepthWeights()); + const double worst = + ExpectClose(got, vllm_test::kMusic3DepthBoundaryOut, count, "depth decoder at boundary"); + MESSAGE("depth decoder boundary: seq_len == max_position_embeddings == " + << config.max_position_embeddings << ", " << count << " values, worst deviation " + << worst); + // One past the window has no pos_embedding row; upstream would index out of + // bounds, so this port refuses by name. + std::vector over(count + hidden, 0.0f); + CHECK_THROWS_AS( + m3::DepthDecoderForward(over, config.max_position_embeddings + 1, config, DepthWeights()), + std::runtime_error); + CHECK_THROWS_AS(m3::DepthDecoderForward({}, 0, config, DepthWeights()), std::runtime_error); + // The REAL configuration's depth sequence (8) fits inside its window (16). + m3::DepthDecoderConfig real; + CHECK(real.num_codebooks <= real.max_position_embeddings); +} + +TEST_CASE("music3 ar: the audio heads match upstream, one per residual codebook") { + const m3::DepthDecoderConfig config = DepthConfig(); + const size_t hidden = static_cast(config.hidden_size); + const size_t count = static_cast(vllm_test::kMusic3DepthSeqLen) * hidden; + const m3::DepthDecoderWeights weights = DepthWeights(); + const std::vector hidden_states = m3::DepthDecoderForward( + ToVector(vllm_test::kMusic3DepthInputsEmbeds, count), vllm_test::kMusic3DepthSeqLen, config, + weights); + const int64_t heads = vllm_test::kMusic3DepthSeqLen - 1; + int values = 0; + for (int64_t head = 0; head < heads; ++head) { + // encoders.py:131-133 β€” depth step i reads position i and head i-1. + const std::vector state( + hidden_states.begin() + static_cast(hidden) * (head + 1), + hidden_states.begin() + static_cast(hidden) * (head + 2)); + const std::vector got = m3::AudioHeadLogits(state, head, config, weights); + REQUIRE(static_cast(got.size()) == config.audio_vocab_size); + for (int64_t j = 0; j < config.audio_vocab_size; ++j) { + const float want = + vllm_test::kMusic3DepthHeadLogits[head * config.audio_vocab_size + j]; + const double bound = + std::max(kAbsFloor, kRelTol * std::max(std::abs(static_cast(got[j])), + std::abs(static_cast(want)))); + CHECK(std::abs(static_cast(got[j]) - want) <= bound); + ++values; + } + } + MESSAGE("audio head logits compared: " << values << " across " << heads << " heads"); + CHECK_THROWS_AS(m3::AudioHeadLogits(std::vector(hidden, 0.0f), heads + 5, config, weights), + std::runtime_error); + CHECK_THROWS_AS(m3::AudioHeadLogits(std::vector(hidden, 0.0f), -1, config, weights), + std::runtime_error); +} + +TEST_CASE("music3 ar: the frame conditioning row is lm hidden then depth steps 1..n") { + const m3::DepthDecoderConfig config = DepthConfig(); + const size_t hidden = static_cast(config.hidden_size); + std::vector last(hidden); + for (size_t i = 0; i < hidden; ++i) last[i] = static_cast(i) + 1000.0f; + std::vector depth(hidden * static_cast(config.num_codebooks)); + for (size_t i = 0; i < depth.size(); ++i) depth[i] = static_cast(i); + const std::vector row = m3::FrameHiddenRow(last, depth, config.num_codebooks, config); + REQUIRE(row.size() == hidden * static_cast(config.num_codebooks)); + int matched = 0; + for (size_t i = 0; i < hidden; ++i) { + CHECK(row[i] == last[i]); + if (row[i] == last[i]) ++matched; + } + // Position 0 of the depth block is the language model's own projected state + // and is DROPPED; the row carries depth steps 1..num_codebooks-1. + for (size_t i = hidden; i < row.size(); ++i) { + CHECK(row[i] == depth[i]); + if (row[i] == depth[i]) ++matched; + } + MESSAGE("frame row values matched: " << matched << "/" << row.size()); + CHECK_THROWS_AS(m3::FrameHiddenRow(last, depth, config.num_codebooks - 1, config), + std::runtime_error); +} + +TEST_CASE("music3 ar: the frame feedback embedding matches _embed_audio_frame") { + const m3::DepthDecoderConfig config = DepthConfig(); + const size_t hidden = static_cast(config.hidden_size); + const std::vector codes( + vllm_test::kMusic3FeedbackCodes + 1, + vllm_test::kMusic3FeedbackCodes + config.num_codebooks); + REQUIRE(static_cast(codes.size()) == config.residual_codebooks()); + const std::vector got = + m3::EmbedAudioFrame(ToVector(vllm_test::kMusic3FeedbackLmRow, hidden), codes, config, + DepthWeights()); + const double worst = ExpectClose(got, vllm_test::kMusic3FeedbackOut, hidden, "frame feedback"); + MESSAGE("frame feedback: " << hidden << " values, worst deviation " << worst); + // The 1/sqrt(num_codebooks) scale is the difference between a plausible-looking + // embedding and the right one; a partial code set is refused. + CHECK_THROWS_AS(m3::EmbedAudioFrame(ToVector(vllm_test::kMusic3FeedbackLmRow, hidden), {0}, + config, DepthWeights()), + std::runtime_error); +}