diff --git a/docs/cli/train.md b/docs/cli/train.md index 2bd78fc34..5d535131d 100644 --- a/docs/cli/train.md +++ b/docs/cli/train.md @@ -178,6 +178,8 @@ torchrun --standalone --nproc_per_node=4 scripts/train.py \ - **`--block-size`** (int, default: `8`) Block size for DFlash model. +- **`--use-liger-kernel`** (flag, default: `False`) Use Liger Qwen3 RMSNorm and SwiGLU kernels for DFlash training. Requires the optional `speculators[liger]` extra. + - **`--sample-from-anchor`** / **`--no-sample-from-anchor`** (bool, default: algorithm-specific) Whether to sample from the anchor position. `True`: sample from anchor and all mask positions (default for dspark, produces block_size tokens). `False`: anchor is bonus token (default for dflash, produces block_size-1 tokens). - **`--max-anchors`** (int, default: `3072`) Maximum anchor positions for DFlash, DSpark, and P-EAGLE training. diff --git a/pyproject.toml b/pyproject.toml index 78e8ad5a3..9abaf8db0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -64,6 +64,7 @@ dependencies = [ ] [project.optional-dependencies] +liger = ["liger-kernel"] dev = [ # build "build>=1.5.0", @@ -145,7 +146,17 @@ exclude = ["venv", "build", "dist"] follow_imports = 'silent' [[tool.mypy.overrides]] -module = ["datasets.*", "transformers.*", "setuptools.*", "setuptools_git_versioning.*", "vllm.*", "triton.*", "hs_connectors.*", "mooncake.*"] +module = [ + "datasets.*", + "hs_connectors.*", + "liger_kernel.*", + "mooncake.*", + "transformers.*", + "setuptools.*", + "setuptools_git_versioning.*", + "vllm.*", + "triton.*", +] ignore_missing_imports=true [tool.ruff] diff --git a/scripts/train.py b/scripts/train.py index f7e564223..dfcd39bb7 100644 --- a/scripts/train.py +++ b/scripts/train.py @@ -16,6 +16,7 @@ from hs_connectors import HiddenStatesBackend from speculators.model import SpeculatorModel +from speculators.models.dflash.kernels import DFlashKernels, load_liger_dflash_kernels from speculators.models.eagle3.data import shift_batch from speculators.models.eagle3.rotary_partial import install_partial_neox_rotary from speculators.models.mtp.data import shift_batch_mtp @@ -365,6 +366,12 @@ def parse_vocab_mappings(args: argparse.Namespace): return None, None, verifier_config.vocab_size +def _resolve_dflash_kernels(args: argparse.Namespace) -> DFlashKernels | None: + if not getattr(args, "use_liger_kernel", False): + return None + return load_liger_dflash_kernels() + + def _build_from_config_only( model_class: type[SpeculatorModel], path: str, @@ -372,6 +379,7 @@ def _build_from_config_only( d2t: torch.Tensor | None, verifier_name_or_path: str | None = None, draft_attn_impl: str | None = None, + dflash_kernels: DFlashKernels | None = None, ) -> SpeculatorModel: """Initialize a fresh draft from a saved speculator *config* (no weights). @@ -392,7 +400,10 @@ def _build_from_config_only( and not getattr(speculators_config.verifier, "name_or_path", None) ): speculators_config.verifier.name_or_path = verifier_name_or_path - model = model_class(config=config) + model_kwargs = {"config": config} + if dflash_kernels is not None: + model_kwargs["dflash_kernels"] = dflash_kernels + model = model_class(**model_kwargs) if hasattr(model, "load_vocab_mappings"): model.load_vocab_mappings(t2d, d2t) # type: ignore[attr-defined, operator] if hasattr(model, "load_verifier_weights"): @@ -421,6 +432,8 @@ def build_draft_model( extracts the native MTP head weights from the verifier, so the decoder-shaping flags and ``--draft-config`` do not apply. """ + dflash_kernels = _resolve_dflash_kernels(args) + if args.from_pretrained: if is_config_only_dir(args.from_pretrained): logger.info( @@ -437,6 +450,7 @@ def build_draft_model( draft_attn_impl=( args.draft_attn_impl if args.speculator_type != "mtp" else None ), + dflash_kernels=dflash_kernels, ) if args.speculator_type != "mtp": # _attn_implementation is never serialized by HF configs, so re-apply @@ -445,12 +459,17 @@ def build_draft_model( # __init__ resolves its own default ("sdpa") when it is absent. config = model_class.config_class.from_pretrained(args.from_pretrained) config.transformer_layer_config._attn_implementation = args.draft_attn_impl + pretrained_kwargs = { + "config": config, + "t2d": t2d, + "d2t": d2t, + "verifier": args.verifier_name_or_path, + } + if dflash_kernels is not None: + pretrained_kwargs["dflash_kernels"] = dflash_kernels return model_class.from_pretrained( args.from_pretrained, - config=config, - t2d=t2d, - d2t=d2t, - verifier=args.verifier_name_or_path, + **pretrained_kwargs, ) return model_class.from_pretrained( args.from_pretrained, @@ -499,11 +518,14 @@ def build_draft_model( ) args.draft_vocab_size = draft_vocab_size + training_args = vars(args).copy() + if dflash_kernels is not None: + training_args["dflash_kernels"] = dflash_kernels return model_class.from_training_args( verifier_config=transformer_layer_config, t2d=t2d, d2t=d2t, - **vars(args), + **training_args, ) diff --git a/src/speculators/models/dflash/core.py b/src/speculators/models/dflash/core.py index 71bf32c23..89b439770 100644 --- a/src/speculators/models/dflash/core.py +++ b/src/speculators/models/dflash/core.py @@ -5,15 +5,13 @@ from torch import nn from torch.nn.attention.flex_attention import create_block_mask, create_mask from transformers import PretrainedConfig -from transformers.models.qwen3.modeling_qwen3 import ( - Qwen3RMSNorm, - Qwen3RotaryEmbedding, -) +from transformers.models.qwen3.modeling_qwen3 import Qwen3RotaryEmbedding from speculators.model import DraftVocabMixin, SpeculatorModel from speculators.models.attention import create_float_mask from speculators.models.dflash import DFlashSpeculatorConfig from speculators.models.dflash.attention import create_anchor_block_mask_mod +from speculators.models.dflash.kernels import DEFAULT_DFLASH_KERNELS, DFlashKernels from speculators.models.dflash.metrics import compute_metrics from speculators.models.dflash.model_definitions import Qwen3DFlashDecoderLayer from speculators.models.dflash.utils import ( @@ -54,6 +52,7 @@ class DFlashDraftModel(DraftVocabMixin, SpeculatorModel): def __init__( self, config: DFlashSpeculatorConfig, + dflash_kernels: DFlashKernels | None = None, ) -> None: # Forcibly override config settings if config.transformer_layer_config._attn_implementation is None: # noqa: SLF001 @@ -70,6 +69,7 @@ def __init__( ) super().__init__(config=config) self._init_vocab(config) + kernels = dflash_kernels or DEFAULT_DFLASH_KERNELS tl_config = config.transformer_layer_config @@ -77,7 +77,11 @@ def __init__( num_draft_layers = tl_config.num_hidden_layers self.layers = nn.ModuleList( [ - Qwen3DFlashDecoderLayer(config.transformer_layer_config, layer_idx) # type: ignore[arg-type] + Qwen3DFlashDecoderLayer( + config.transformer_layer_config, # type: ignore[arg-type] + layer_idx, + kernels, + ) for layer_idx in range(num_draft_layers) ] ) @@ -91,9 +95,9 @@ def __init__( self.uses_full_attn = bool(num_draft_layers - len(self.sliding_window_indices)) self.sliding_window_non_causal = config.sliding_window_non_causal - self.norm = Qwen3RMSNorm( + self.norm = kernels.make_rms_norm( config.transformer_layer_config.hidden_size, - eps=config.transformer_layer_config.rms_norm_eps, # type: ignore[arg-type] + config.transformer_layer_config.rms_norm_eps, # type: ignore[arg-type] ) self.rotary_emb = Qwen3RotaryEmbedding(config.transformer_layer_config) # type: ignore[arg-type] @@ -102,13 +106,13 @@ def __init__( config.transformer_layer_config.hidden_size, bias=False, ) - self.hidden_norm = Qwen3RMSNorm( + self.hidden_norm = kernels.make_rms_norm( config.transformer_layer_config.hidden_size, - eps=config.transformer_layer_config.rms_norm_eps, # type: ignore[arg-type] + config.transformer_layer_config.rms_norm_eps, # type: ignore[arg-type] ) - self.verifier_norm = Qwen3RMSNorm( + self.verifier_norm = kernels.make_rms_norm( config.transformer_layer_config.hidden_size, - eps=config.transformer_layer_config.rms_norm_eps, # type: ignore[arg-type] + config.transformer_layer_config.rms_norm_eps, # type: ignore[arg-type] ) self.verifier_norm.weight.requires_grad = False self.block_size = config.block_size @@ -156,11 +160,12 @@ def from_training_args( The number of draft layers is encoded in verifier_config.num_hidden_layers, following the same pattern as EAGLE3. """ + dflash_kernels = kwargs.pop("dflash_kernels", None) config = DFlashSpeculatorConfig( **cls._build_base_config_kwargs("dflash", verifier_config, **kwargs) ) - model = cls(config=config) + model = cls(config=config, dflash_kernels=dflash_kernels) model.load_vocab_mappings(t2d, d2t) model.load_verifier_weights() return model diff --git a/src/speculators/models/dflash/kernels.py b/src/speculators/models/dflash/kernels.py new file mode 100644 index 000000000..7ff71ae97 --- /dev/null +++ b/src/speculators/models/dflash/kernels.py @@ -0,0 +1,71 @@ +"""Explicit module factories for the DFlash Qwen3 backbone.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import TYPE_CHECKING, Protocol + +import torch +from torch import nn +from transformers.models.qwen3.modeling_qwen3 import Qwen3Config, Qwen3MLP, Qwen3RMSNorm + +if TYPE_CHECKING: + from collections.abc import Callable + + +class RMSNormModule(Protocol): + """The RMS norm surface the DFlash backbone reads, beyond `nn.Module`.""" + + @property + def weight(self) -> torch.Tensor: ... + + def __call__(self, hidden_states: torch.Tensor) -> torch.Tensor: ... + + +@dataclass(frozen=True) +class DFlashKernels: + """Construct the replaceable Qwen3 modules used by a DFlash draft.""" + + make_rms_norm: Callable[[int, float], RMSNormModule] + make_mlp: Callable[[Qwen3Config], nn.Module] + + +def _make_qwen3_rms_norm(hidden_size: int, eps: float) -> RMSNormModule: + return Qwen3RMSNorm(hidden_size, eps=eps) + + +def _make_qwen3_mlp(config: Qwen3Config) -> nn.Module: + return Qwen3MLP(config) + + +DEFAULT_DFLASH_KERNELS = DFlashKernels( + make_rms_norm=_make_qwen3_rms_norm, + make_mlp=_make_qwen3_mlp, +) + + +def load_liger_dflash_kernels() -> DFlashKernels: + """Load Liger lazily and adapt it to the DFlash construction boundary.""" + try: + from liger_kernel.transformers import ( # noqa: PLC0415 + LigerRMSNorm, + LigerSwiGLUMLP, + ) + except ModuleNotFoundError as exc: + if exc.name in {"liger_kernel", "liger_kernel.transformers"}: + raise ImportError( + "--use-liger-kernel requires the optional `speculators[liger]` " + 'extra. Install it with `pip install "speculators[liger]"`.' + ) from exc + raise + + def make_rms_norm(hidden_size: int, eps: float) -> RMSNormModule: + return LigerRMSNorm(hidden_size, eps=eps) + + def make_mlp(config: Qwen3Config) -> nn.Module: + return LigerSwiGLUMLP(config) + + return DFlashKernels( + make_rms_norm=make_rms_norm, + make_mlp=make_mlp, + ) diff --git a/src/speculators/models/dflash/model_definitions.py b/src/speculators/models/dflash/model_definitions.py index 0391e2a17..f2009aaf3 100644 --- a/src/speculators/models/dflash/model_definitions.py +++ b/src/speculators/models/dflash/model_definitions.py @@ -8,12 +8,12 @@ FlashAttentionKwargs, GradientCheckpointingLayer, Qwen3Config, - Qwen3MLP, - Qwen3RMSNorm, eager_attention_forward, ) from typing_extensions import Unpack +from speculators.models.dflash.kernels import DFlashKernels + if TYPE_CHECKING: from collections.abc import Callable @@ -49,7 +49,7 @@ class Qwen3DFlashAttention(nn.Module): # Implements the custom attention which injects the target models # hidden states into the kv cache. - def __init__(self, config: Qwen3Config, layer_idx: int): + def __init__(self, config: Qwen3Config, layer_idx: int, kernels: DFlashKernels): super().__init__() self.config = config self.layer_idx = layer_idx @@ -84,8 +84,8 @@ def __init__(self, config: Qwen3Config, layer_idx: int): config.hidden_size, # type: ignore[arg-type] bias=config.attention_bias, # type: ignore[arg-type] ) - self.q_norm = Qwen3RMSNorm(self.head_dim, eps=config.rms_norm_eps) # type: ignore[arg-type] - self.k_norm = Qwen3RMSNorm(self.head_dim, eps=config.rms_norm_eps) # type: ignore[arg-type] + self.q_norm = kernels.make_rms_norm(self.head_dim, config.rms_norm_eps) # type: ignore[arg-type] + self.k_norm = kernels.make_rms_norm(self.head_dim, config.rms_norm_eps) # type: ignore[arg-type] self.sliding_window = ( config.sliding_window if hasattr(config, "layer_types") @@ -156,15 +156,27 @@ def forward( class Qwen3DFlashDecoderLayer(GradientCheckpointingLayer): - def __init__(self, config: Qwen3Config, layer_idx: int): + def __init__( + self, + config: Qwen3Config, + layer_idx: int, + kernels: DFlashKernels, + ): super().__init__() self.hidden_size = config.hidden_size - self.self_attn = Qwen3DFlashAttention(config=config, layer_idx=layer_idx) - self.mlp = Qwen3MLP(config) - self.input_layernorm = Qwen3RMSNorm(config.hidden_size, eps=config.rms_norm_eps) # type: ignore[arg-type] - self.post_attention_layernorm = Qwen3RMSNorm( + self.self_attn = Qwen3DFlashAttention( + config=config, + layer_idx=layer_idx, + kernels=kernels, + ) + self.mlp = kernels.make_mlp(config) + self.input_layernorm = kernels.make_rms_norm( + config.hidden_size, + config.rms_norm_eps, # type: ignore[arg-type] + ) + self.post_attention_layernorm = kernels.make_rms_norm( config.hidden_size, - eps=config.rms_norm_eps, # type: ignore[arg-type] + config.rms_norm_eps, # type: ignore[arg-type] ) def forward( diff --git a/src/speculators/train/config/schema.py b/src/speculators/train/config/schema.py index 43baa28a6..6c2602811 100644 --- a/src/speculators/train/config/schema.py +++ b/src/speculators/train/config/schema.py @@ -436,6 +436,11 @@ class DFlashArgs(_Group): block_size: int = Field( default=8, description="Block size for DFlash model (default: 8)." ) + use_liger_kernel: bool = Field( + default=False, + description="Use Liger Qwen3 RMSNorm/SwiGLU kernels for DFlash. Requires the " + "optional `speculators[liger]` extra.", + ) sample_from_anchor: bool | None = Field( default=None, description="Sample from the anchor position (all positions predict). " @@ -652,6 +657,17 @@ def _resolve_derived_defaults(self) -> "TrainConfig": self.optimizer.muon_lr = 10 * self.optimizer.lr return self + @model_validator(mode="after") + def _validate_liger_kernel(self) -> "TrainConfig": + """The Liger kernels are wired into the DFlash backbone only, so the flag is + rejected outright on any other speculator rather than silently ignored.""" + if self.dflash.use_liger_kernel and self.speculator_type != "dflash": + raise ValueError( + "--use-liger-kernel is currently supported only with " + "--speculator-type dflash" + ) + return self + @model_validator(mode="after") def _validate_dpace(self) -> "TrainConfig": """D-PACE per-position loss weighting requires CE loss and a smoothing constant diff --git a/tests/unit/models/test_dflash_liger.py b/tests/unit/models/test_dflash_liger.py new file mode 100644 index 000000000..6834fb9e9 --- /dev/null +++ b/tests/unit/models/test_dflash_liger.py @@ -0,0 +1,119 @@ +"""Tests for opt-in, instance-local DFlash Liger kernels.""" + +import argparse +import builtins + +import pytest +from transformers.models.qwen3.modeling_qwen3 import Qwen3Config, Qwen3MLP, Qwen3RMSNorm + +from scripts.train import _resolve_dflash_kernels +from speculators import SpeculatorsConfig, VerifierConfig +from speculators.models.dflash import DFlashSpeculatorConfig +from speculators.models.dflash.core import DFlashDraftModel +from speculators.models.dflash.kernels import load_liger_dflash_kernels +from speculators.proposals.greedy import GreedyTokenProposalConfig + + +def _config() -> DFlashSpeculatorConfig: + transformer_config = Qwen3Config( # type: ignore[call-arg] + vocab_size=32, + hidden_size=16, + intermediate_size=32, + num_hidden_layers=1, + num_attention_heads=4, + num_key_value_heads=2, + head_dim=4, + max_position_embeddings=64, + rms_norm_eps=1e-6, + _attn_implementation="eager", + ) + return DFlashSpeculatorConfig( + transformer_layer_config=transformer_config, + draft_vocab_size=32, + block_size=4, + aux_hidden_state_layer_ids=[0], + mask_token_id=0, + speculators_config=SpeculatorsConfig( + algorithm="dflash", + proposal_methods=[GreedyTokenProposalConfig(speculative_tokens=3)], + default_proposal_method="greedy", + verifier=VerifierConfig( + name_or_path=None, + architectures=["Qwen3ForCausalLM"], + ), + ), + ) + + +def test_disabled_resolver_does_not_load_liger(monkeypatch): + def fail_if_called(): + raise AssertionError("disabled Liger path must not load the optional extra") + + monkeypatch.setattr("scripts.train.load_liger_dflash_kernels", fail_if_called) + + assert _resolve_dflash_kernels(argparse.Namespace(use_liger_kernel=False)) is None + + +def test_missing_liger_extra_has_actionable_error(monkeypatch): + original_import = builtins.__import__ + + def missing_liger(name, globals_=None, locals_=None, fromlist=(), level=0): + if name == "liger_kernel.transformers": + raise ModuleNotFoundError( + "No module named 'liger_kernel'", name="liger_kernel" + ) + return original_import(name, globals_, locals_, fromlist, level) + + monkeypatch.setattr(builtins, "__import__", missing_liger) + + with pytest.raises(ImportError, match=r"speculators\[liger\]"): + load_liger_dflash_kernels() + + +def test_liger_kernels_are_instance_local_and_cover_dflash_backbone(): + pytest.importorskip("liger_kernel") + from liger_kernel.transformers import LigerRMSNorm, LigerSwiGLUMLP # noqa: PLC0415 + + native_before = DFlashDraftModel(_config()) + liger = DFlashDraftModel(_config(), dflash_kernels=load_liger_dflash_kernels()) + native_after = DFlashDraftModel(_config()) + + assert type(native_before.norm) is Qwen3RMSNorm + assert type(native_after.norm) is Qwen3RMSNorm + assert type(native_before.layers[0].mlp) is Qwen3MLP + assert type(native_after.layers[0].mlp) is Qwen3MLP + + layer = liger.layers[0] + assert all( + isinstance(module, LigerRMSNorm) + for module in ( + liger.norm, + liger.hidden_norm, + liger.verifier_norm, + layer.input_layernorm, + layer.post_attention_layernorm, + layer.self_attn.q_norm, # type: ignore[union-attr] + layer.self_attn.k_norm, # type: ignore[union-attr] + ) + ) + assert isinstance(layer.mlp, LigerSwiGLUMLP) + + +def test_liger_and_native_dflash_checkpoints_are_compatible(tmp_path): + pytest.importorskip("liger_kernel") + from liger_kernel.transformers import LigerRMSNorm # noqa: PLC0415 + + native = DFlashDraftModel(_config()) + liger = DFlashDraftModel(_config(), dflash_kernels=load_liger_dflash_kernels()) + + assert native.state_dict().keys() == liger.state_dict().keys() + result = liger.load_state_dict(native.state_dict(), strict=True) + assert not result.missing_keys + assert not result.unexpected_keys + + native.save_pretrained(tmp_path) + reloaded = DFlashDraftModel.from_pretrained( + tmp_path, + dflash_kernels=load_liger_dflash_kernels(), + ) + assert isinstance(reloaded.norm, LigerRMSNorm) diff --git a/tests/unit/train/test_cli_args.py b/tests/unit/train/test_cli_args.py index 316d17c22..4a715553e 100644 --- a/tests/unit/train/test_cli_args.py +++ b/tests/unit/train/test_cli_args.py @@ -173,6 +173,17 @@ def test_no_norm_output_flag(monkeypatch): assert args.norm_output is False +def test_liger_kernel_flag_parses_for_dflash(monkeypatch): + args = _parse(monkeypatch, ["--speculator-type", "dflash", "--use-liger-kernel"]) + + assert args.use_liger_kernel is True + + +def test_liger_kernel_flag_rejects_non_dflash(monkeypatch): + with pytest.raises(SystemExit): + _parse(monkeypatch, ["--speculator-type", "eagle3", "--use-liger-kernel"]) + + # --------------------------------------------------------------------------- # --max-steps # --------------------------------------------------------------------------- diff --git a/tests/unit/train/test_draft_config_init.py b/tests/unit/train/test_draft_config_init.py index cee249229..7616f0d62 100644 --- a/tests/unit/train/test_draft_config_init.py +++ b/tests/unit/train/test_draft_config_init.py @@ -572,6 +572,99 @@ def test_build_draft_model_routing( ) +def test_build_draft_model_threads_liger_kernels_to_from_scratch(monkeypatch): + kernels = object() + captured = {} + + monkeypatch.setattr("scripts.train._resolve_dflash_kernels", lambda _args: kernels) + monkeypatch.setattr( + "scripts.train.create_transformer_layer_config", + lambda **_kwargs: SimpleNamespace(vocab_size=128), + ) + monkeypatch.setattr("scripts.train.resolve_mask_token_id", lambda *_a, **_k: 0) + + class _FakeModel: + @classmethod + def from_training_args(cls, **kwargs): + captured.update(kwargs) + return "MODEL" + + args = SimpleNamespace( + use_liger_kernel=True, + speculator_type="dflash", + from_pretrained="", + draft_config="", + verifier_name_or_path="some-verifier", + num_layers=3, + draft_arch="qwen3", + draft_hidden_act=None, + sliding_window=2048, + full_attention_indices=[], + mask_token_id=None, + trust_remote_code=False, + draft_mrope_full_head_hack=True, + ) + + assert build_draft_model(args, _FakeModel, None, None, 128) == "MODEL" # type: ignore[arg-type] + assert captured["dflash_kernels"] is kernels + + +def test_build_draft_model_threads_liger_kernels_to_config_only(monkeypatch): + kernels = object() + captured = {} + + monkeypatch.setattr("scripts.train._resolve_dflash_kernels", lambda _args: kernels) + monkeypatch.setattr("scripts.train.is_config_only_dir", lambda _path: True) + + def build_from_config_only(*_args, **kwargs): + captured.update(kwargs) + return "MODEL" + + monkeypatch.setattr("scripts.train._build_from_config_only", build_from_config_only) + args = SimpleNamespace( + use_liger_kernel=True, + speculator_type="dflash", + from_pretrained="config-only-checkpoint", + verifier_name_or_path="some-verifier", + draft_attn_impl="eager", + ) + + assert build_draft_model(args, object, None, None, None) == "MODEL" # type: ignore[arg-type] + assert captured["dflash_kernels"] is kernels + + +def test_build_draft_model_threads_liger_kernels_to_pretrained(monkeypatch): + kernels = object() + captured = {} + + monkeypatch.setattr("scripts.train._resolve_dflash_kernels", lambda _args: kernels) + monkeypatch.setattr("scripts.train.is_config_only_dir", lambda _path: False) + + class _ConfigClass: + @classmethod + def from_pretrained(cls, _path): + return SimpleNamespace(transformer_layer_config=SimpleNamespace()) + + class _FakeModel: + config_class = _ConfigClass + + @classmethod + def from_pretrained(cls, *_args, **kwargs): + captured.update(kwargs) + return "MODEL" + + args = SimpleNamespace( + use_liger_kernel=True, + speculator_type="dflash", + from_pretrained="checkpoint", + verifier_name_or_path="some-verifier", + draft_attn_impl="eager", + ) + + assert build_draft_model(args, _FakeModel, None, None, None) == "MODEL" # type: ignore[arg-type] + assert captured["dflash_kernels"] is kernels + + # --------------------------------------------------------------------------- # intermediate_size resolution (dense + MoE verifiers) # ---------------------------------------------------------------------------