diff --git a/src/mcore_bridge/config/parser.py b/src/mcore_bridge/config/parser.py index b5351a5..7cf59c4 100644 --- a/src/mcore_bridge/config/parser.py +++ b/src/mcore_bridge/config/parser.py @@ -152,7 +152,8 @@ def hf_to_mcore_config(hf_config: PretrainedConfig) -> Dict[str, Any]: moe_n_hash_layers = res.pop('moe_n_hash_layers', None) rope_scaling = res.get('rope_scaling') or {} if llm_model_type in {'qwen3', 'qwen3_moe', 'qwen3_next'} or hf_model_type in { - 'qwen3_omni_moe', 'qwen3_omni', 'qwen3_vl', 'qwen3_vl_moe', 'qwen3_5', 'qwen3_5_moe', 'llavaonevision1_5', 'minicpmv4_6' + 'qwen3_omni_moe', 'qwen3_omni', 'qwen3_vl', 'qwen3_vl_moe', 'qwen3_5', 'qwen3_5_moe', 'llavaonevision1_5', + 'minicpmv4_6' }: res['qk_layernorm'] = True if llm_model_type in {'qwen2_moe', 'qwen3_moe', 'qwen3_next' diff --git a/src/mcore_bridge/model/mm_gpts/__init__.py b/src/mcore_bridge/model/mm_gpts/__init__.py index a8950fa..896f5e1 100644 --- a/src/mcore_bridge/model/mm_gpts/__init__.py +++ b/src/mcore_bridge/model/mm_gpts/__init__.py @@ -1,2 +1,3 @@ # Copyright (c) ModelScope Contributors. All rights reserved. -from . import gemma4, glm, internvl, kimi_vl, llama4, llava, minicpmv4_6, qwen, qwen3_5, qwen3_5_gdn, qwen3_asr, qwen3_omni, qwen3_vl +from . import (gemma4, glm, internvl, kimi_vl, llama4, llava, minicpmv4_6, qwen, qwen3_5, qwen3_5_gdn, qwen3_asr, + qwen3_omni, qwen3_vl) diff --git a/src/mcore_bridge/model/mm_gpts/minicpmv4_6.py b/src/mcore_bridge/model/mm_gpts/minicpmv4_6.py index 16b5b73..a11c094 100644 --- a/src/mcore_bridge/model/mm_gpts/minicpmv4_6.py +++ b/src/mcore_bridge/model/mm_gpts/minicpmv4_6.py @@ -17,11 +17,8 @@ class MiniCPMV46Vit(HuggingFaceVit): _aligner = ['merger'] def prepare_model(self, hf_config: PretrainedConfig): - from transformers.models.minicpmv4_6.modeling_minicpmv4_6 import ( - MiniCPMV4_6VisionModel, - MiniCPMV4_6Merger, - MiniCPMV4_6Model - ) + from transformers.models.minicpmv4_6.modeling_minicpmv4_6 import (MiniCPMV4_6Merger, MiniCPMV4_6Model, + MiniCPMV4_6VisionModel) self.vision_tower = MiniCPMV4_6VisionModel._from_config(hf_config.vision_config) self.merger = MiniCPMV4_6Merger(hf_config).to(dtype=self.vision_tower.dtype) self.model_cls = MiniCPMV4_6Model @@ -37,10 +34,8 @@ def get_inputs_embeds(self, inputs_embeds, **kwargs): if pixel_values is None and pixel_values_videos is None: patch_size = hf_config.vision_config.patch_size dummy_pv = torch.zeros( - 1, 3, 4 * patch_size, 4 * patch_size, - device=inputs_embeds.device, dtype=self.vision_tower.dtype) - dummy_ts = torch.tensor( - [[4, 4]], device=inputs_embeds.device, dtype=torch.int32) + 1, 3, 4 * patch_size, 4 * patch_size, device=inputs_embeds.device, dtype=self.vision_tower.dtype) + dummy_ts = torch.tensor([[4, 4]], device=inputs_embeds.device, dtype=torch.int32) with self.patch_hf_config(): vision_output = self.model_cls.get_image_features(self, dummy_pv, dummy_ts) image_embeds = torch.cat(vision_output.pooler_output, dim=0) @@ -52,11 +47,11 @@ def get_inputs_embeds(self, inputs_embeds, **kwargs): vision_output = self.model_cls.get_image_features( self, pixel_values[:1].to(dtype=self.vision_tower.dtype), target_sizes) image_features = ( - torch.cat(vision_output.pooler_output, dim=0) - .to(device=inputs_embeds.device, dtype=inputs_embeds.dtype) - .repeat(num_beams, 1)) - mask = self.model_cls.get_placeholder_mask( - self, input_ids, inputs_embeds, image_features, hf_config.image_token_id) + torch.cat(vision_output.pooler_output, + dim=0).to(device=inputs_embeds.device, + dtype=inputs_embeds.dtype).repeat(num_beams, 1)) + mask = self.model_cls.get_placeholder_mask(self, input_ids, inputs_embeds, image_features, + hf_config.image_token_id) inputs_embeds = inputs_embeds.masked_scatter(mask, image_features) if pixel_values_videos is not None: @@ -65,11 +60,11 @@ def get_inputs_embeds(self, inputs_embeds, **kwargs): vision_output = self.model_cls.get_video_features( self, pixel_values_videos[:1].to(dtype=self.vision_tower.dtype), target_sizes_videos) video_features = ( - torch.cat(vision_output.pooler_output, dim=0) - .to(device=inputs_embeds.device, dtype=inputs_embeds.dtype) - .repeat(num_beams, 1)) - mask = self.model_cls.get_placeholder_mask( - self, input_ids, inputs_embeds, video_features, hf_config.video_token_id) + torch.cat(vision_output.pooler_output, + dim=0).to(device=inputs_embeds.device, + dtype=inputs_embeds.dtype).repeat(num_beams, 1)) + mask = self.model_cls.get_placeholder_mask(self, input_ids, inputs_embeds, video_features, + hf_config.video_token_id) inputs_embeds = inputs_embeds.masked_scatter(mask, video_features) return inputs_embeds