[model] support qwen3_asr

README.md

@@ -81,7 +81,7 @@ The following is the list of models supported by MCore-Bridge:
 
 | Series     | model_type                                                   |
 | -------- | ------------------------------------------------------------ |
-| Qwen     | qwen2, qwen2_moe<br />qwen2_vl, qwen2_5_vl, qwen2_5_omni<br />qwen3, qwen3_moe<br />qwen3_vl, qwen3_vl_moe, qwen3_omni_moe<br />qwen3_next, qwen3_5, qwen3_5_moe |
+| Qwen     | qwen2, qwen2_moe<br />qwen2_vl, qwen2_5_vl, qwen2_5_omni<br />qwen3, qwen3_moe<br />qwen3_vl, qwen3_vl_moe, qwen3_omni_moe, qwen3_asr<br />qwen3_next, qwen3_5, qwen3_5_moe |
 | DeepSeek | deepseek_v3, deepseek_v32                                    |
 | GLM      | glm4, glm4_moe, glm4_moe_lite<br />glm4v, glm4v_moe, <br />glm_moe_dsa |
 | MiniMax  | minimax_m2                                                   |

README_zh.md

@@ -80,7 +80,7 @@ uv pip install -e . --torch-backend=auto
 
 | 系列     | model_type                                                   |
 | -------- | ------------------------------------------------------------ |
-| Qwen     | qwen2, qwen2_moe<br />qwen2_vl, qwen2_5_vl, qwen2_5_omni<br />qwen3, qwen3_moe<br />qwen3_vl, qwen3_vl_moe, qwen3_omni_moe<br />qwen3_next, qwen3_5, qwen3_5_moe |
+| Qwen     | qwen2, qwen2_moe<br />qwen2_vl, qwen2_5_vl, qwen2_5_omni<br />qwen3, qwen3_moe<br />qwen3_vl, qwen3_vl_moe, qwen3_omni_moe, qwen3_asr<br />qwen3_next, qwen3_5, qwen3_5_moe |
 | DeepSeek | deepseek_v3, deepseek_v32                                    |
 | GLM      | glm4, glm4_moe, glm4_moe_lite<br />glm4v, glm4v_moe, <br />glm_moe_dsa |
 | MiniMax  | minimax_m2                                                   |

src/mcore_bridge/config/model_config.py

@@ -321,7 +321,10 @@ def __post_init__(self):
             self.mcore_model_type = get_mcore_model_type(self.hf_model_type)
         self.model_meta = get_model_meta(self.mcore_model_type)
         self.is_multimodal = self.model_meta.visual_cls is not None
-        self.support_multimodal = self.is_multimodal and getattr(self.model_meta.visual_cls, 'support_multimodal', True)
+        if self.is_multimodal:
+            self.test_mm_type = getattr(self.model_meta.visual_cls, 'test_mm_type', 'image')
+        else:
+            self.test_mm_type = 'text'
         if self.is_multimodal and self.hf_config is None:
             raise ValueError('Multimodal model must specify hf_config.')
         self.is_moe_model = self.num_moe_experts is not None

src/mcore_bridge/model/constant.py

@@ -19,6 +19,7 @@ class MLLMModelType:
     qwen3_vl = 'qwen3_vl'
     qwen2_5_omni = 'qwen2_5_omni'
     qwen3_omni = 'qwen3_omni'
+    qwen3_asr = 'qwen3_asr'
     qwen3_5 = 'qwen3_5'
     ovis2_5 = 'ovis2_5'
 

src/mcore_bridge/model/gpt_model.py

@@ -294,7 +294,8 @@ def forward(
             runtime_gather_output (bool): Gather output at runtime. Default None means
                 `parallel_output` arg in the constructor will be used.
         """
-
+        if self.config.position_embedding_type == 'mrope' and position_ids.ndim == 2:  # qwen3_asr
+            position_ids = position_ids.unsqueeze(0).expand(3, -1, -1)
         inference_context = deprecate_inference_params(inference_context, inference_params)
 
         decoder_input, rotary_pos_emb, rotary_pos_cos, rotary_pos_sin, sequence_len_offset = (

src/mcore_bridge/model/mm_gpts/__init__.py

@@ -1,2 +1,2 @@
 # Copyright (c) ModelScope Contributors. All rights reserved.
-from . import glm, internvl, kimi_vl, llama4, llava, qwen, qwen3_5, qwen3_5_gdn, qwen3_omni, qwen3_vl
+from . import glm, internvl, kimi_vl, llama4, llava, qwen, qwen3_5, qwen3_5_gdn, qwen3_asr, qwen3_omni, qwen3_vl

src/mcore_bridge/model/mm_gpts/kimi_vl.py

@@ -72,7 +72,7 @@ class KimiK25Vit(HuggingFaceVit):
     module_mapping = {'vision_tower': 'vision_tower', 'mm_projector': 'mm_projector'}
     _vision_tower = ['vision_tower']
     _aligner = ['mm_projector']
-    support_multimodal = False
+    test_mm_type = 'text'
 
     def prepare_model(self, hf_config: PretrainedConfig):
         output = []

src/mcore_bridge/model/mm_gpts/qwen3_asr.py

@@ -0,0 +1,60 @@
+# Copyright (c) ModelScope Contributors. All rights reserved.
+import torch
+from transformers import PretrainedConfig
+
+from mcore_bridge.bridge import MultimodalGPTBridge
+
+from ..constant import ModelType
+from ..register import ModelMeta, register_model
+from .utils import HuggingFaceVit
+
+
+class Qwen3ASRBridge(MultimodalGPTBridge):
+    hf_layers_prefix = 'thinker.model.layers'
+    hf_embed_key = 'thinker.model.embed_tokens.weight'
+    hf_final_layernorm_key = 'thinker.model.norm.weight'
+    hf_lm_head_key = 'thinker.lm_head.weight'
+    hf_score_key = 'thinker.score.weight'
+
+
+class Qwen3ASRVit(HuggingFaceVit):
+    module_mapping = {'thinker.audio_tower': 'audio_tower'}
+    _vision_tower = ['audio_tower']
+    _aligner = ['audio_tower.proj1', 'audio_tower.proj2']
+    test_mm_type = 'audio'
+
+    def prepare_model(self, hf_config: PretrainedConfig):
+        from qwen_asr.core.transformers_backend.modeling_qwen3_asr import (Qwen3ASRAudioEncoder,
+                                                                           Qwen3ASRThinkerForConditionalGeneration)
+        self.audio_tower = Qwen3ASRAudioEncoder._from_config(hf_config.thinker_config.audio_config)
+        self.model_cls = Qwen3ASRThinkerForConditionalGeneration
+
+    def get_inputs_embeds(self, inputs_embeds, **kwargs):
+        input_ids = kwargs['input_ids']
+        hf_config = self.hf_config.thinker_config
+        input_features = kwargs.get('input_features')
+        feature_attention_mask = kwargs.get('feature_attention_mask')
+
+        if input_features is None:
+            input_features = input_ids.new_zeros([1, 128, 128], dtype=self.audio_tower.dtype)
+            feature_attention_mask = input_ids.new_ones([1, 128], dtype=torch.bool)
+            audio_embeds = self.get_audio_features(input_features, feature_attention_mask)
+            inputs_embeds = inputs_embeds + audio_embeds.mean() * 0.
+        else:
+            audio_embeds = self.get_audio_features(input_features, feature_attention_mask)
+            audio_mask = (input_ids == hf_config.audio_token_id).unsqueeze(-1).expand_as(inputs_embeds)
+            audio_embeds = audio_embeds.to(inputs_embeds.device, inputs_embeds.dtype)
+            inputs_embeds = inputs_embeds.masked_scatter(audio_mask, audio_embeds)
+        return inputs_embeds
+
+    def get_audio_features(self, *args, **kwargs):
+        with self.patch_hf_config():
+            return self.model_cls.get_audio_features(self, *args, **kwargs)
+
+
+register_model(ModelMeta(
+    ModelType.qwen3_asr,
+    ['qwen3_asr'],
+    bridge_cls=Qwen3ASRBridge,
+    visual_cls=Qwen3ASRVit,
+))

src/mcore_bridge/model/mm_gpts/utils.py

@@ -26,7 +26,7 @@ def new_get_cached_module_file(pretrained_model_name_or_path, *args, **kwargs):
 
 class HuggingFaceVit(_HuggingFaceModule, ABC):
     module_mapping = {}  # hf -> mcore
-    support_multimodal = True
+    test_mm_type = 'image'
 
     @contextmanager
     def patch_hf_config(self):

tests/test_mllm.py

@@ -112,6 +112,10 @@ def test_llava_onevision1_5():
     _test_model('lmms-lab/LLaVA-OneVision-1.5-4B-Instruct')
 
 
+def test_qwen3_asr():
+    _test_model('Qwen/Qwen3-ASR-1.7B')
+
+
 if __name__ == '__main__':
     # test_qwen2_5_vl()
     # test_qwen2_vl()
@@ -131,4 +135,5 @@ def test_llava_onevision1_5():
     # test_qwen3_omni()
     # test_llama4()
     # test_qwen3_5()
-    test_llava_onevision1_5()
+    # test_llava_onevision1_5()
+    test_qwen3_asr()