diff --git a/convert_hf_to_gguf.py b/convert_hf_to_gguf.py index 2df5e94fe2..e97a7c72ff 100755 --- a/convert_hf_to_gguf.py +++ b/convert_hf_to_gguf.py @@ -10611,6 +10611,18 @@ def set_gguf_parameters(self): logger.info("gguf: (granite) logits_scale = %s", logits_scale) +@ModelBase.register("GraniteSpeechForConditionalGeneration", ModelType.TEXT) +class GraniteSpeechTextModel(GraniteModel): + model_arch = gguf.MODEL_ARCH.GRANITE + + def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]: + if name.startswith(("encoder.", "projector.")): + return + if name.startswith("language_model."): + name = name[len("language_model."):] + yield from super().modify_tensors(data_torch, name, bid) + + @ModelBase.register("GraniteMoeForCausalLM", "GraniteMoeSharedForCausalLM") class GraniteMoeModel(GraniteModel): """Conversion for IBM's GraniteMoeForCausalLM""" @@ -12347,6 +12359,154 @@ def modify_tensors(self, data_torch, name, bid): yield from super().modify_tensors(data_torch, name, bid) +@ModelBase.register("GraniteSpeechForConditionalGeneration", ModelType.MMPROJ) +class GraniteSpeechMmprojModel(MmprojModel): + has_vision_encoder = False + has_audio_encoder = True + + _batch_norm_tensors: list[dict[str, Tensor]] | None = None + + def get_audio_config(self) -> dict[str, Any] | None: + return self.global_config.get("encoder_config") + + def set_gguf_parameters(self): + assert self.hparams_audio is not None + a = self.hparams_audio + a["hidden_size"] = a["hidden_dim"] + a["intermediate_size"] = a["hidden_dim"] * a["feedforward_mult"] + a["num_attention_heads"] = a["num_heads"] + a["num_hidden_layers"] = a["num_layers"] + + super().set_gguf_parameters() + + self.gguf_writer.add_clip_projector_type(gguf.VisionProjectorType.GRANITE_SPEECH) + self.gguf_writer.add_audio_num_mel_bins(a["input_dim"]) + self.gguf_writer.add_audio_attention_layernorm_eps(1e-5) + + def tensor_force_quant(self, name, new_name, bid, n_dims): + if "encoder" in name or "projector" in name: + if ".conv" in name and ".weight" in name: + return gguf.GGMLQuantizationType.F32 + return super().tensor_force_quant(name, new_name, bid, n_dims) + + def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]: + if name.startswith("language_model."): + return + if "attention_dists" in name: + return + if "num_batches_tracked" in name: + return + + # fold running_mean, running_var and eps into weight and bias for batch_norm + if "batch_norm" in name and "encoder.layers." in name: + if self._batch_norm_tensors is None: + self._batch_norm_tensors = [{} for _ in range(self.block_count)] + assert bid is not None + self._batch_norm_tensors[bid][name] = data_torch + if len(self._batch_norm_tensors[bid]) < 4: + return + prefix = f"encoder.layers.{bid}.conv.batch_norm" + weight = self._batch_norm_tensors[bid][f"{prefix}.weight"] + bias = self._batch_norm_tensors[bid][f"{prefix}.bias"] + running_mean = self._batch_norm_tensors[bid][f"{prefix}.running_mean"] + running_var = self._batch_norm_tensors[bid][f"{prefix}.running_var"] + eps = 1e-5 + a = weight / torch.sqrt(running_var + eps) + b = bias - running_mean * a + yield from super().modify_tensors(a, f"encoder.layers.{bid}.conv.batch_norm.weight", bid) + yield from super().modify_tensors(b, f"encoder.layers.{bid}.conv.batch_norm.bias", bid) + return + + if ".attn.to_kv.weight" in name: + k_weight, v_weight = data_torch.chunk(2, dim=0) + yield from super().modify_tensors(k_weight, name.replace("to_kv", "to_k"), bid) + yield from super().modify_tensors(v_weight, name.replace("to_kv", "to_v"), bid) + return + + if ("up_conv" in name or "down_conv" in name) and name.endswith(".weight"): + if data_torch.ndim == 3 and data_torch.shape[2] == 1: + data_torch = data_torch.squeeze(2) + + if "depth_conv" in name and name.endswith(".weight"): + if data_torch.ndim == 3 and data_torch.shape[1] == 1: + data_torch = data_torch.squeeze(1) + + if name.startswith("projector."): + gguf_name = self._map_projector_tensor(name) + if gguf_name is None: + return + yield (gguf_name, data_torch) + return + + global_map = { + "encoder.input_linear.weight": "a.enc_inp_linear.weight", + "encoder.input_linear.bias": "a.enc_inp_linear.bias", + "encoder.out.weight": "a.enc_ctc_out.weight", + "encoder.out.bias": "a.enc_ctc_out.bias", + "encoder.out_mid.weight": "a.enc_ctc_out_mid.weight", + "encoder.out_mid.bias": "a.enc_ctc_out_mid.bias", + } + if name in global_map: + yield (global_map[name], data_torch) + return + + if ".attn.rel_pos_emb.weight" in name: + assert bid is not None + yield (f"a.blk.{bid}.attn_rel_pos_emb", data_torch) + return + + yield from super().modify_tensors(data_torch, name, bid) + + @staticmethod + def _map_projector_tensor(name: str) -> str | None: + static_map = { + "projector.query": "a.proj_query", + "projector.qformer.layernorm.weight": "a.proj_norm.weight", + "projector.qformer.layernorm.bias": "a.proj_norm.bias", + "projector.linear.weight": "a.proj_linear.weight", + "projector.linear.bias": "a.proj_linear.bias", + } + if name in static_map: + return static_map[name] + m = re.match(r"projector\.qformer\.encoder\.layer\.(\d+)\.(.*)", name) + if not m: + return None + lid = m.group(1) + rest = m.group(2) + layer_map = { + "attention.attention.query.weight": "self_attn_q.weight", + "attention.attention.query.bias": "self_attn_q.bias", + "attention.attention.key.weight": "self_attn_k.weight", + "attention.attention.key.bias": "self_attn_k.bias", + "attention.attention.value.weight": "self_attn_v.weight", + "attention.attention.value.bias": "self_attn_v.bias", + "attention.output.dense.weight": "self_attn_out.weight", + "attention.output.dense.bias": "self_attn_out.bias", + "attention.output.LayerNorm.weight": "self_attn_norm.weight", + "attention.output.LayerNorm.bias": "self_attn_norm.bias", + "crossattention.attention.query.weight": "cross_attn_q.weight", + "crossattention.attention.query.bias": "cross_attn_q.bias", + "crossattention.attention.key.weight": "cross_attn_k.weight", + "crossattention.attention.key.bias": "cross_attn_k.bias", + "crossattention.attention.value.weight": "cross_attn_v.weight", + "crossattention.attention.value.bias": "cross_attn_v.bias", + "crossattention.output.dense.weight": "cross_attn_out.weight", + "crossattention.output.dense.bias": "cross_attn_out.bias", + "crossattention.output.LayerNorm.weight": "cross_attn_norm.weight", + "crossattention.output.LayerNorm.bias": "cross_attn_norm.bias", + "intermediate_query.dense.weight": "ffn_up.weight", + "intermediate_query.dense.bias": "ffn_up.bias", + "output_query.dense.weight": "ffn_down.weight", + "output_query.dense.bias": "ffn_down.bias", + "output_query.LayerNorm.weight": "ffn_norm.weight", + "output_query.LayerNorm.bias": "ffn_norm.bias", + } + suffix = layer_map.get(rest) + if suffix is None: + return None + return f"a.proj_blk.{lid}.{suffix}" + + @ModelBase.register("Lfm25AudioTokenizer") class LFM25AudioTokenizer(LFM2Model): model_arch = gguf.MODEL_ARCH.LFM2 @@ -13356,6 +13516,8 @@ def get_model_architecture(hparams: dict[str, Any], model_type: ModelType) -> st # TODO: refactor this later to avoid adding exception here if model_type == ModelType.TEXT and arch == "StepVLForConditionalGeneration": return arch + if model_type == ModelType.TEXT and arch == "GraniteSpeechForConditionalGeneration": + return arch # if "architectures" is found in the sub-config, use that instead if model_type == ModelType.TEXT and text_config.get("architectures") is not None: diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py index c5297a2f44..3388970c74 100644 --- a/gguf-py/gguf/constants.py +++ b/gguf-py/gguf/constants.py @@ -4138,6 +4138,7 @@ class VisionProjectorType: YOUTUVL = "youtuvl" NEMOTRON_V2_VL = "nemotron_v2_vl" HUNYUANOCR = "hunyuanocr" + GRANITE_SPEECH = "granite_speech" # audio # Items here are (block size, type size) diff --git a/gguf-py/gguf/tensor_mapping.py b/gguf-py/gguf/tensor_mapping.py index 01a9b23600..8f9eb60017 100644 --- a/gguf-py/gguf/tensor_mapping.py +++ b/gguf-py/gguf/tensor_mapping.py @@ -1912,6 +1912,7 @@ class TensorNameMap: "conformer.layers.{bid}.self_attn.linear_q", # lfm2 "conformer.layers.{bid}.attention.attn.q_proj", # gemma3n "conformer.layers.{bid}.self_attn.q_proj", # gemma4 + "encoder.layers.{bid}.attn.to_q", # granite_speech ), MODEL_TENSOR.A_ENC_ATTN_K: ( @@ -1919,6 +1920,7 @@ class TensorNameMap: "conformer.layers.{bid}.self_attn.linear_k", # lfm2 "conformer.layers.{bid}.attention.attn.k_proj", # gemma3n "conformer.layers.{bid}.self_attn.k_proj", # gemma4 + "encoder.layers.{bid}.attn.to_k", # granite_speech (split from to_kv) ), MODEL_TENSOR.A_ENC_ATTN_V: ( @@ -1926,6 +1928,7 @@ class TensorNameMap: "conformer.layers.{bid}.self_attn.linear_v", # lfm2 "conformer.layers.{bid}.attention.attn.v_proj", # gemma3n "conformer.layers.{bid}.self_attn.v_proj", # gemma4 + "encoder.layers.{bid}.attn.to_v", # granite_speech (split from to_kv) ), MODEL_TENSOR.A_ENC_ATTN_K_REL: ( @@ -1953,6 +1956,7 @@ class TensorNameMap: "audio_tower.layers.{bid}.self_attn_layer_norm", # ultravox "conformer.layers.{bid}.norm_self_att", # lfm2 "conformer.layers.{bid}.attention.pre_attn_norm", # gemma3n + "encoder.layers.{bid}.attn.pre_norm", # granite_speech ), MODEL_TENSOR.A_ENC_OUTPUT: ( @@ -1960,18 +1964,21 @@ class TensorNameMap: "conformer.layers.{bid}.self_attn.linear_out", # lfm2 "conformer.layers.{bid}.attention.post", # gemma3n "conformer.layers.{bid}.self_attn.post", # gemma4 + "encoder.layers.{bid}.attn.to_out", # granite_speech ), MODEL_TENSOR.A_ENC_OUTPUT_NORM: ( "audio_tower.layers.{bid}.final_layer_norm", # ultravox "conformer.layers.{bid}.norm_out", # lfm2 "conformer.layers.{bid}.attention.post_norm", # gemma3n + "encoder.layers.{bid}.post_norm", # granite_speech ), MODEL_TENSOR.A_ENC_FFN_NORM: ( "conformer.layers.{bid}.norm_feed_forward1", # lfm2 "conformer.layers.{bid}.ffw_layer_start.pre_layer_norm", # gemma3n "conformer.layers.{bid}.feed_forward1.pre_layer_norm", # gemma4 + "encoder.layers.{bid}.ff1.pre_norm", # granite_speech ), MODEL_TENSOR.A_ENC_FFN_POST_NORM: ( @@ -1988,6 +1995,7 @@ class TensorNameMap: "conformer.layers.{bid}.feed_forward1.linear1", # lfm2 "conformer.layers.{bid}.ffw_layer_start.ffw_layer_1", # gemma3n "conformer.layers.{bid}.feed_forward1.ffw_layer_1", # gemma4 + "encoder.layers.{bid}.ff1.up_proj", # granite_speech ), MODEL_TENSOR.A_ENC_FFN_GATE: (), @@ -1997,24 +2005,28 @@ class TensorNameMap: "conformer.layers.{bid}.feed_forward1.linear2", # lfm2 "conformer.layers.{bid}.ffw_layer_start.ffw_layer_2", # gemma3n "conformer.layers.{bid}.feed_forward1.ffw_layer_2", # gemma4 + "encoder.layers.{bid}.ff1.down_proj", # granite_speech ), MODEL_TENSOR.A_ENC_FFN_UP_1: ( "conformer.layers.{bid}.feed_forward2.linear1", # lfm2 "conformer.layers.{bid}.ffw_layer_end.ffw_layer_1", # gemma3n "conformer.layers.{bid}.feed_forward2.ffw_layer_1", # gemma4 + "encoder.layers.{bid}.ff2.up_proj", # granite_speech ), MODEL_TENSOR.A_ENC_FFN_DOWN_1: ( "conformer.layers.{bid}.feed_forward2.linear2", # lfm2 "conformer.layers.{bid}.ffw_layer_end.ffw_layer_2", # gemma3n "conformer.layers.{bid}.feed_forward2.ffw_layer_2", # gemma4 + "encoder.layers.{bid}.ff2.down_proj", # granite_speech ), MODEL_TENSOR.A_ENC_FFN_NORM_1: ( "conformer.layers.{bid}.norm_feed_forward2", # lfm2 "conformer.layers.{bid}.ffw_layer_end.pre_layer_norm", # gemma3n "conformer.layers.{bid}.feed_forward2.pre_layer_norm", # gemma4 + "encoder.layers.{bid}.ff2.pre_norm", # granite_speech ), MODEL_TENSOR.A_ENC_FFN_POST_NORM_1: ( @@ -2071,26 +2083,31 @@ class TensorNameMap: MODEL_TENSOR.A_ENC_CONV_DW: ( "conformer.layers.{bid}.conv.depthwise_conv", # lfm2 "conformer.layers.{bid}.lconv1d.depthwise_conv1d", # gemma3n + "encoder.layers.{bid}.conv.depth_conv.conv", # granite_speech ), MODEL_TENSOR.A_ENC_CONV_NORM: ( "conformer.layers.{bid}.conv.batch_norm", # lfm2 "conformer.layers.{bid}.lconv1d.pre_layer_norm", # gemma3n + "encoder.layers.{bid}.conv.batch_norm", # granite_speech ), MODEL_TENSOR.A_ENC_CONV_PW1: ( "conformer.layers.{bid}.conv.pointwise_conv1", # lfm2 "conformer.layers.{bid}.lconv1d.linear_start", # gemma3n + "encoder.layers.{bid}.conv.up_conv", # granite_speech ), MODEL_TENSOR.A_ENC_CONV_PW2: ( "conformer.layers.{bid}.conv.pointwise_conv2", # lfm2 "conformer.layers.{bid}.lconv1d.linear_end", # gemma3n + "encoder.layers.{bid}.conv.down_conv", # granite_speech ), MODEL_TENSOR.A_ENC_NORM_CONV: ( "conformer.layers.{bid}.norm_conv", # lfm2 "conformer.layers.{bid}.lconv1d.conv_norm", # gemma3n + "encoder.layers.{bid}.conv.norm", # granite_speech ), MODEL_TENSOR.A_PER_DIM_K_SCALE: ( diff --git a/tools/mtmd/CMakeLists.txt b/tools/mtmd/CMakeLists.txt index 399876128e..f78e49cea0 100644 --- a/tools/mtmd/CMakeLists.txt +++ b/tools/mtmd/CMakeLists.txt @@ -20,6 +20,7 @@ add_library(mtmd models/dotsocr.cpp models/gemma4a.cpp models/gemma4v.cpp + models/granite-speech.cpp models/glm4v.cpp models/hunyuanocr.cpp models/internvl.cpp diff --git a/tools/mtmd/clip-impl.h b/tools/mtmd/clip-impl.h index 17cb703f7f..1d5f887f00 100644 --- a/tools/mtmd/clip-impl.h +++ b/tools/mtmd/clip-impl.h @@ -182,6 +182,27 @@ #define TN_CONV_NORM "%s.blk.%d.conv_norm.%s" #define TN_CONV_PW1 "%s.blk.%d.conv_pw1.%s" #define TN_CONV_PW2 "%s.blk.%d.conv_pw2.%s" +// granite_speech +#define TN_GS_INP_LINEAR "a.enc_inp_linear.%s" +#define TN_GS_CTC_OUT "a.enc_ctc_out.%s" +#define TN_GS_CTC_OUT_MID "a.enc_ctc_out_mid.%s" +#define TN_GS_ATTN_REL_POS "%s.blk.%d.attn_rel_pos_emb" +#define TN_GS_PROJ_QUERY "a.proj_query" +#define TN_GS_PROJ_NORM "a.proj_norm.%s" +#define TN_GS_PROJ_LINEAR "a.proj_linear.%s" +#define TN_GS_PROJ_SELF_ATTN_Q "a.proj_blk.%d.self_attn_q.%s" +#define TN_GS_PROJ_SELF_ATTN_K "a.proj_blk.%d.self_attn_k.%s" +#define TN_GS_PROJ_SELF_ATTN_V "a.proj_blk.%d.self_attn_v.%s" +#define TN_GS_PROJ_SELF_ATTN_O "a.proj_blk.%d.self_attn_out.%s" +#define TN_GS_PROJ_SELF_ATTN_N "a.proj_blk.%d.self_attn_norm.%s" +#define TN_GS_PROJ_CROSS_ATTN_Q "a.proj_blk.%d.cross_attn_q.%s" +#define TN_GS_PROJ_CROSS_ATTN_K "a.proj_blk.%d.cross_attn_k.%s" +#define TN_GS_PROJ_CROSS_ATTN_V "a.proj_blk.%d.cross_attn_v.%s" +#define TN_GS_PROJ_CROSS_ATTN_O "a.proj_blk.%d.cross_attn_out.%s" +#define TN_GS_PROJ_CROSS_ATTN_N "a.proj_blk.%d.cross_attn_norm.%s" +#define TN_GS_PROJ_FFN_UP "a.proj_blk.%d.ffn_up.%s" +#define TN_GS_PROJ_FFN_DOWN "a.proj_blk.%d.ffn_down.%s" +#define TN_GS_PROJ_FFN_NORM "a.proj_blk.%d.ffn_norm.%s" // gemma4 audio conformer #define TN_A_MM_INP_PROJ "mm.a.input_projection.%s" @@ -293,6 +314,7 @@ enum projector_type { PROJECTOR_TYPE_KIMIK25, PROJECTOR_TYPE_NEMOTRON_V2_VL, PROJECTOR_TYPE_HUNYUANOCR, + PROJECTOR_TYPE_GRANITE_SPEECH, PROJECTOR_TYPE_UNKNOWN, }; @@ -338,6 +360,7 @@ static std::map PROJECTOR_TYPE_NAMES = { { PROJECTOR_TYPE_KIMIK25, "kimik25"}, { PROJECTOR_TYPE_NEMOTRON_V2_VL, "nemotron_v2_vl"}, { PROJECTOR_TYPE_HUNYUANOCR, "hunyuanocr"}, + { PROJECTOR_TYPE_GRANITE_SPEECH, "granite_speech"}, }; static projector_type clip_projector_type_from_string(const std::string & str) { diff --git a/tools/mtmd/clip-model.h b/tools/mtmd/clip-model.h index 9a93584d9b..e82c024169 100644 --- a/tools/mtmd/clip-model.h +++ b/tools/mtmd/clip-model.h @@ -224,6 +224,9 @@ struct clip_layer { ggml_tensor * per_dim_k_scale_w = nullptr; ggml_tensor * ff_post_norm_1_w = nullptr; + // granite_speech conformer per-layer + ggml_tensor * attn_rel_pos_emb = nullptr; + bool has_deepstack() const { return deepstack_fc1_w != nullptr; } @@ -268,6 +271,37 @@ struct mobilenetv5_block { ggml_tensor * attn_norm_w = nullptr; }; +struct granite_speech_proj_layer { + ggml_tensor * self_attn_q_w = nullptr; + ggml_tensor * self_attn_q_b = nullptr; + ggml_tensor * self_attn_k_w = nullptr; + ggml_tensor * self_attn_k_b = nullptr; + ggml_tensor * self_attn_v_w = nullptr; + ggml_tensor * self_attn_v_b = nullptr; + ggml_tensor * self_attn_o_w = nullptr; + ggml_tensor * self_attn_o_b = nullptr; + ggml_tensor * self_attn_norm_w = nullptr; + ggml_tensor * self_attn_norm_b = nullptr; + + ggml_tensor * cross_attn_q_w = nullptr; + ggml_tensor * cross_attn_q_b = nullptr; + ggml_tensor * cross_attn_k_w = nullptr; + ggml_tensor * cross_attn_k_b = nullptr; + ggml_tensor * cross_attn_v_w = nullptr; + ggml_tensor * cross_attn_v_b = nullptr; + ggml_tensor * cross_attn_o_w = nullptr; + ggml_tensor * cross_attn_o_b = nullptr; + ggml_tensor * cross_attn_norm_w = nullptr; + ggml_tensor * cross_attn_norm_b = nullptr; + + ggml_tensor * ffn_up_w = nullptr; + ggml_tensor * ffn_up_b = nullptr; + ggml_tensor * ffn_down_w = nullptr; + ggml_tensor * ffn_down_b = nullptr; + ggml_tensor * ffn_norm_w = nullptr; + ggml_tensor * ffn_norm_b = nullptr; +}; + struct clip_model { clip_modality modality = CLIP_MODALITY_VISION; projector_type proj_type = PROJECTOR_TYPE_MLP; @@ -485,6 +519,20 @@ struct clip_model { ggml_tensor * audio_out_proj_w = nullptr; ggml_tensor * audio_out_proj_b = nullptr; + // granite_speech encoder + projector + ggml_tensor * gs_inp_linear_w = nullptr; + ggml_tensor * gs_inp_linear_b = nullptr; + ggml_tensor * gs_ctc_out_w = nullptr; + ggml_tensor * gs_ctc_out_b = nullptr; + ggml_tensor * gs_ctc_out_mid_w = nullptr; + ggml_tensor * gs_ctc_out_mid_b = nullptr; + ggml_tensor * gs_proj_query = nullptr; + ggml_tensor * gs_proj_norm_w = nullptr; + ggml_tensor * gs_proj_norm_b = nullptr; + ggml_tensor * gs_proj_linear_w = nullptr; + ggml_tensor * gs_proj_linear_b = nullptr; + std::vector gs_proj_layers; + bool audio_has_avgpool() const { return proj_type == PROJECTOR_TYPE_QWEN2A || proj_type == PROJECTOR_TYPE_VOXTRAL diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cpp index f0e8786b66..03f7066205 100644 --- a/tools/mtmd/clip.cpp +++ b/tools/mtmd/clip.cpp @@ -935,6 +935,10 @@ static ggml_cgraph * clip_image_build_graph(clip_ctx * ctx, const clip_image_f32 { builder = std::make_unique(ctx, img); } break; + case PROJECTOR_TYPE_GRANITE_SPEECH: + { + builder = std::make_unique(ctx, img); + } break; case PROJECTOR_TYPE_GLM4V: { builder = std::make_unique(ctx, img); @@ -1478,6 +1482,14 @@ struct clip_model_loader { hparams.audio_window_len = 320; // 20ms frame (NOT 25ms/400) hparams.audio_hop_len = 160; } break; + case PROJECTOR_TYPE_GRANITE_SPEECH: + { + hparams.audio_chunk_len = 0; + hparams.audio_sample_rate = 16000; + hparams.audio_n_fft = 512; + hparams.audio_window_len = 400; + hparams.audio_hop_len = 160; + } break; case PROJECTOR_TYPE_JANUS_PRO: { hparams.image_pad_color = {127, 127, 127}; @@ -1629,8 +1641,10 @@ struct clip_model_loader { model.position_embeddings = get_tensor(string_format(TN_POS_EMBD, prefix), false); - if (model.proj_type == PROJECTOR_TYPE_GEMMA3NV) { - hparams.n_layer = 0; // gemma3n does not use normal layer structure + const int n_layer_orig = hparams.n_layer; + if (model.proj_type == PROJECTOR_TYPE_GEMMA3NV + || model.proj_type == PROJECTOR_TYPE_GRANITE_SPEECH) { + hparams.n_layer = 0; // these models do not use the generic layer structure } // layers @@ -2340,6 +2354,100 @@ struct clip_model_loader { layer.conv_pw2_b = get_tensor(string_format(TN_CONV_PW2, prefix, il, "bias")); } } break; + case PROJECTOR_TYPE_GRANITE_SPEECH: + { + hparams.n_layer = n_layer_orig; + model.layers.resize(hparams.n_layer); + + model.gs_inp_linear_w = get_tensor(string_format(TN_GS_INP_LINEAR, "weight")); + model.gs_inp_linear_b = get_tensor(string_format(TN_GS_INP_LINEAR, "bias")); + model.gs_ctc_out_w = get_tensor(string_format(TN_GS_CTC_OUT, "weight")); + model.gs_ctc_out_b = get_tensor(string_format(TN_GS_CTC_OUT, "bias")); + model.gs_ctc_out_mid_w = get_tensor(string_format(TN_GS_CTC_OUT_MID, "weight")); + model.gs_ctc_out_mid_b = get_tensor(string_format(TN_GS_CTC_OUT_MID, "bias")); + + for (int il = 0; il < hparams.n_layer; ++il) { + auto & layer = model.layers[il]; + + layer.q_w = get_tensor(string_format(TN_ATTN_Q, prefix, il, "weight")); + layer.k_w = get_tensor(string_format(TN_ATTN_K, prefix, il, "weight")); + layer.v_w = get_tensor(string_format(TN_ATTN_V, prefix, il, "weight")); + layer.o_w = get_tensor(string_format(TN_ATTN_OUTPUT, prefix, il, "weight")); + layer.o_b = get_tensor(string_format(TN_ATTN_OUTPUT, prefix, il, "bias")); + layer.attn_rel_pos_emb = get_tensor(string_format(TN_GS_ATTN_REL_POS, prefix, il)); + + layer.ln_1_w = get_tensor(string_format(TN_LN_1, prefix, il, "weight")); + layer.ln_1_b = get_tensor(string_format(TN_LN_1, prefix, il, "bias")); + + layer.ln_2_w = get_tensor(string_format(TN_LN_2, prefix, il, "weight")); + layer.ln_2_b = get_tensor(string_format(TN_LN_2, prefix, il, "bias")); + + layer.ff_norm_w = get_tensor(string_format(TN_FFN_NORM, prefix, il, "weight")); + layer.ff_norm_b = get_tensor(string_format(TN_FFN_NORM, prefix, il, "bias")); + layer.ff_up_w = get_tensor(string_format(TN_FFN_UP, prefix, il, "weight")); + layer.ff_up_b = get_tensor(string_format(TN_FFN_UP, prefix, il, "bias")); + layer.ff_down_w = get_tensor(string_format(TN_FFN_DOWN, prefix, il, "weight")); + layer.ff_down_b = get_tensor(string_format(TN_FFN_DOWN, prefix, il, "bias")); + + layer.ff_norm_1_w = get_tensor(string_format(TN_FFN_NORM_1, prefix, il, "weight")); + layer.ff_norm_1_b = get_tensor(string_format(TN_FFN_NORM_1, prefix, il, "bias")); + layer.ff_up_1_w = get_tensor(string_format(TN_FFN_UP_1, prefix, il, "weight")); + layer.ff_up_1_b = get_tensor(string_format(TN_FFN_UP_1, prefix, il, "bias")); + layer.ff_down_1_w = get_tensor(string_format(TN_FFN_DOWN_1, prefix, il, "weight")); + layer.ff_down_1_b = get_tensor(string_format(TN_FFN_DOWN_1, prefix, il, "bias")); + + layer.norm_conv_w = get_tensor(string_format(TN_NORM_CONV, prefix, il, "weight")); + layer.norm_conv_b = get_tensor(string_format(TN_NORM_CONV, prefix, il, "bias")); + layer.conv_norm_w = get_tensor(string_format(TN_CONV_NORM, prefix, il, "weight")); + layer.conv_norm_b = get_tensor(string_format(TN_CONV_NORM, prefix, il, "bias")); + layer.conv_dw_w = get_tensor(string_format(TN_CONV_DW, prefix, il, "weight")); + layer.conv_pw1_w = get_tensor(string_format(TN_CONV_PW1, prefix, il, "weight")); + layer.conv_pw1_b = get_tensor(string_format(TN_CONV_PW1, prefix, il, "bias")); + layer.conv_pw2_w = get_tensor(string_format(TN_CONV_PW2, prefix, il, "weight")); + layer.conv_pw2_b = get_tensor(string_format(TN_CONV_PW2, prefix, il, "bias")); + } + + model.gs_proj_query = get_tensor(TN_GS_PROJ_QUERY); + model.gs_proj_norm_w = get_tensor(string_format(TN_GS_PROJ_NORM, "weight")); + model.gs_proj_norm_b = get_tensor(string_format(TN_GS_PROJ_NORM, "bias")); + model.gs_proj_linear_w = get_tensor(string_format(TN_GS_PROJ_LINEAR, "weight")); + model.gs_proj_linear_b = get_tensor(string_format(TN_GS_PROJ_LINEAR, "bias")); + + const int n_proj_layers = 2; + model.gs_proj_layers.resize(n_proj_layers); + for (int il = 0; il < n_proj_layers; ++il) { + auto & pl = model.gs_proj_layers[il]; + + pl.self_attn_q_w = get_tensor(string_format(TN_GS_PROJ_SELF_ATTN_Q, il, "weight")); + pl.self_attn_q_b = get_tensor(string_format(TN_GS_PROJ_SELF_ATTN_Q, il, "bias")); + pl.self_attn_k_w = get_tensor(string_format(TN_GS_PROJ_SELF_ATTN_K, il, "weight")); + pl.self_attn_k_b = get_tensor(string_format(TN_GS_PROJ_SELF_ATTN_K, il, "bias")); + pl.self_attn_v_w = get_tensor(string_format(TN_GS_PROJ_SELF_ATTN_V, il, "weight")); + pl.self_attn_v_b = get_tensor(string_format(TN_GS_PROJ_SELF_ATTN_V, il, "bias")); + pl.self_attn_o_w = get_tensor(string_format(TN_GS_PROJ_SELF_ATTN_O, il, "weight")); + pl.self_attn_o_b = get_tensor(string_format(TN_GS_PROJ_SELF_ATTN_O, il, "bias")); + pl.self_attn_norm_w = get_tensor(string_format(TN_GS_PROJ_SELF_ATTN_N, il, "weight")); + pl.self_attn_norm_b = get_tensor(string_format(TN_GS_PROJ_SELF_ATTN_N, il, "bias")); + + pl.cross_attn_q_w = get_tensor(string_format(TN_GS_PROJ_CROSS_ATTN_Q, il, "weight")); + pl.cross_attn_q_b = get_tensor(string_format(TN_GS_PROJ_CROSS_ATTN_Q, il, "bias")); + pl.cross_attn_k_w = get_tensor(string_format(TN_GS_PROJ_CROSS_ATTN_K, il, "weight")); + pl.cross_attn_k_b = get_tensor(string_format(TN_GS_PROJ_CROSS_ATTN_K, il, "bias")); + pl.cross_attn_v_w = get_tensor(string_format(TN_GS_PROJ_CROSS_ATTN_V, il, "weight")); + pl.cross_attn_v_b = get_tensor(string_format(TN_GS_PROJ_CROSS_ATTN_V, il, "bias")); + pl.cross_attn_o_w = get_tensor(string_format(TN_GS_PROJ_CROSS_ATTN_O, il, "weight")); + pl.cross_attn_o_b = get_tensor(string_format(TN_GS_PROJ_CROSS_ATTN_O, il, "bias")); + pl.cross_attn_norm_w = get_tensor(string_format(TN_GS_PROJ_CROSS_ATTN_N, il, "weight")); + pl.cross_attn_norm_b = get_tensor(string_format(TN_GS_PROJ_CROSS_ATTN_N, il, "bias")); + + pl.ffn_up_w = get_tensor(string_format(TN_GS_PROJ_FFN_UP, il, "weight")); + pl.ffn_up_b = get_tensor(string_format(TN_GS_PROJ_FFN_UP, il, "bias")); + pl.ffn_down_w = get_tensor(string_format(TN_GS_PROJ_FFN_DOWN, il, "weight")); + pl.ffn_down_b = get_tensor(string_format(TN_GS_PROJ_FFN_DOWN, il, "bias")); + pl.ffn_norm_w = get_tensor(string_format(TN_GS_PROJ_FFN_NORM, il, "weight")); + pl.ffn_norm_b = get_tensor(string_format(TN_GS_PROJ_FFN_NORM, il, "bias")); + } + } break; default: GGML_ASSERT(false && "unknown projector type"); } @@ -3023,6 +3131,10 @@ int clip_n_output_tokens(const struct clip_ctx * ctx, struct clip_image_f32 * im } n_patches = n; } break; + case PROJECTOR_TYPE_GRANITE_SPEECH: + { + n_patches = ((img->nx + 14) / 15) * 3; + } break; default: GGML_ABORT("unsupported projector type"); } @@ -3554,6 +3666,39 @@ bool clip_image_batch_encode(clip_ctx * ctx, const int n_threads, const clip_ima } set_input_f32("pos_emb", pos_emb); } break; + case PROJECTOR_TYPE_GRANITE_SPEECH: + { + const int context_size = 200; + const int max_pos_emb = 512; + + std::vector dists(context_size * context_size); + for (int i = 0; i < context_size; i++) { + for (int j = 0; j < context_size; j++) { + int d = i - j; + if (d < -context_size) d = -context_size; + if (d > context_size) d = context_size; + dists[i * context_size + j] = d + max_pos_emb; + } + } + set_input_i32("attn_dists", dists); + + const int n_frames = image_size_width; + const int remainder = n_frames % context_size; + if (remainder > 0) { + const int num_blocks = (n_frames + context_size - 1) / context_size; + std::vector mask(context_size * context_size * num_blocks, 0.0f); + const float neg_inf = -INFINITY; + const int last_block_offset = (num_blocks - 1) * context_size * context_size; + for (int q = 0; q < context_size; q++) { + for (int k = 0; k < context_size; k++) { + if (q >= remainder || k >= remainder) { + mask[last_block_offset + q * context_size + k] = neg_inf; + } + } + } + set_input_f32("attn_mask", mask); + } + } break; default: GGML_ABORT("Unknown projector type"); } @@ -3700,6 +3845,8 @@ int clip_n_mmproj_embd(const struct clip_ctx * ctx) { return ctx->model.position_embeddings->ne[0]; case PROJECTOR_TYPE_GEMMA4A: return ctx->model.hparams.projection_dim; + case PROJECTOR_TYPE_GRANITE_SPEECH: + return ctx->model.gs_proj_linear_w->ne[1]; case PROJECTOR_TYPE_GLM4V: return ctx->model.mm_ffn_down_w->ne[1]; default: diff --git a/tools/mtmd/models/granite-speech.cpp b/tools/mtmd/models/granite-speech.cpp new file mode 100644 index 0000000000..b0c15a3f3e --- /dev/null +++ b/tools/mtmd/models/granite-speech.cpp @@ -0,0 +1,276 @@ +#include "models.h" + +ggml_cgraph * clip_graph_granite_speech::build() { + const int n_frames = img.nx; + const int context_size = 200; + const int ctc_layer = n_layer / 2; + const int conv_kernel = 15; + const int conv_pad = conv_kernel / 2; + + const int num_blocks = (n_frames + context_size - 1) / context_size; + const int padded_len = num_blocks * context_size; + const int remainder = n_frames % context_size; + + ggml_tensor * attn_dists = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, context_size * context_size); + ggml_set_name(attn_dists, "attn_dists"); + ggml_set_input(attn_dists); + ggml_build_forward_expand(gf, attn_dists); + + ggml_tensor * attn_mask = nullptr; + if (remainder > 0) { + attn_mask = ggml_new_tensor_4d(ctx0, GGML_TYPE_F32, + context_size, context_size, 1, num_blocks); + ggml_set_name(attn_mask, "attn_mask"); + ggml_set_input(attn_mask); + ggml_build_forward_expand(gf, attn_mask); + } + + ggml_tensor * inp = build_inp_raw(1); + auto * cur = ggml_cont(ctx0, ggml_transpose(ctx0, inp)); + cb(cur, "inp_transposed", -1); + + cur = build_mm(model.gs_inp_linear_w, cur); + cur = ggml_add(ctx0, cur, model.gs_inp_linear_b); + cb(cur, "inp_linear", -1); + + for (int il = 0; il < n_layer; il++) { + const auto & layer = model.layers[il]; + auto * residual = cur; + + // ffn1 (half-step) + { + auto * ffn1 = build_norm(cur, layer.ff_norm_w, layer.ff_norm_b, + NORM_TYPE_NORMAL, eps, il); + cb(ffn1, "ffn1_norm", il); + + ffn1 = build_ffn(ffn1, + layer.ff_up_w, layer.ff_up_b, + nullptr, nullptr, + layer.ff_down_w, layer.ff_down_b, + FFN_SILU, il); + cb(ffn1, "ffn1_out", il); + + residual = ggml_add(ctx0, residual, ggml_scale(ctx0, ffn1, 0.5f)); + cb(residual, "ffn1_residual", il); + } + + // self-attention with Shaw RPE + { + auto * normed = build_norm(residual, layer.ln_1_w, layer.ln_1_b, + NORM_TYPE_NORMAL, eps, il); + cb(normed, "attn_norm", il); + + if (n_frames < padded_len) { + normed = ggml_pad(ctx0, normed, 0, padded_len - n_frames, 0, 0); + } + + ggml_tensor * Q = build_mm(layer.q_w, normed); + ggml_tensor * K = build_mm(layer.k_w, normed); + ggml_tensor * V = build_mm(layer.v_w, normed); + + Q = ggml_reshape_4d(ctx0, Q, d_head, n_head, context_size, num_blocks); + K = ggml_reshape_4d(ctx0, K, d_head, n_head, context_size, num_blocks); + V = ggml_reshape_4d(ctx0, V, d_head, n_head, context_size, num_blocks); + + ggml_tensor * Q_perm = ggml_permute(ctx0, Q, 0, 2, 1, 3); + ggml_tensor * K_perm = ggml_cont(ctx0, ggml_permute(ctx0, K, 0, 2, 1, 3)); + + ggml_tensor * kq = ggml_mul_mat(ctx0, K_perm, Q_perm); + + // Shaw RPE: pos_emb ne[2]=1 broadcasts against Q ne[2]=num_blocks in mul_mat + ggml_tensor * pos_emb = ggml_get_rows(ctx0, layer.attn_rel_pos_emb, attn_dists); + pos_emb = ggml_reshape_3d(ctx0, pos_emb, d_head, context_size, context_size); + pos_emb = ggml_reshape_4d(ctx0, pos_emb, d_head, context_size, 1, context_size); + + ggml_tensor * Q_shaw = ggml_permute(ctx0, Q, 0, 1, 3, 2); + ggml_tensor * pos_attn = ggml_mul_mat(ctx0, pos_emb, Q_shaw); + pos_attn = ggml_cont(ctx0, ggml_permute(ctx0, pos_attn, 0, 2, 3, 1)); + + ggml_tensor * scores = ggml_add(ctx0, kq, pos_attn); + ggml_tensor * attn_weights = ggml_soft_max_ext(ctx0, scores, attn_mask, + kq_scale, 0.0f); + + ggml_tensor * V_perm = ggml_cont(ctx0, ggml_permute(ctx0, V, 1, 2, 0, 3)); + ggml_tensor * attn_out = ggml_mul_mat(ctx0, V_perm, attn_weights); + + attn_out = ggml_permute(ctx0, attn_out, 0, 2, 1, 3); + attn_out = ggml_cont_2d(ctx0, attn_out, n_embd, padded_len); + + if (n_frames < padded_len) { + attn_out = ggml_view_2d(ctx0, attn_out, + n_embd, n_frames, attn_out->nb[1], 0); + } + + cur = build_mm(layer.o_w, attn_out); + cur = ggml_add(ctx0, cur, layer.o_b); + cb(cur, "attn_out", il); + } + + residual = ggml_add(ctx0, residual, cur); + + // conv module + { + cur = build_norm(residual, layer.norm_conv_w, layer.norm_conv_b, + NORM_TYPE_NORMAL, eps, il); + cb(cur, "conv_norm", il); + + auto * x = build_mm(layer.conv_pw1_w, cur); + x = ggml_add(ctx0, x, layer.conv_pw1_b); + cb(x, "conv_pw1", il); + + // GLU: ggml has no fused op, manual split + sigmoid gate + { + int64_t d = x->ne[0] / 2; + ggml_tensor * gate = ggml_sigmoid(ctx0, + ggml_view_2d(ctx0, x, d, x->ne[1], x->nb[1], d * x->nb[0])); + x = ggml_mul(ctx0, + ggml_view_2d(ctx0, x, d, x->ne[1], x->nb[1], 0), gate); + x = ggml_cont(ctx0, ggml_transpose(ctx0, x)); + } + cb(x, "conv_glu", il); + + x = ggml_pad(ctx0, x, conv_pad, 0, 0, 0); + x = ggml_roll(ctx0, x, conv_pad, 0, 0, 0); + x = ggml_pad(ctx0, x, conv_pad, 0, 0, 0); + x = ggml_ssm_conv(ctx0, x, layer.conv_dw_w); + cb(x, "conv_dw", il); + + // folded batch norm + x = ggml_add(ctx0, ggml_mul(ctx0, x, layer.conv_norm_w), layer.conv_norm_b); + x = ggml_silu(ctx0, x); + cb(x, "conv_bn_silu", il); + + x = build_mm(layer.conv_pw2_w, x); + x = ggml_add(ctx0, x, layer.conv_pw2_b); + cb(x, "conv_pw2", il); + + cur = x; + } + + residual = ggml_add(ctx0, residual, cur); + + // ffn2 (half-step) + { + auto * ffn2 = build_norm(residual, layer.ff_norm_1_w, layer.ff_norm_1_b, + NORM_TYPE_NORMAL, eps, il); + cb(ffn2, "ffn2_norm", il); + + ffn2 = build_ffn(ffn2, + layer.ff_up_1_w, layer.ff_up_1_b, + nullptr, nullptr, + layer.ff_down_1_w, layer.ff_down_1_b, + FFN_SILU, il); + cb(ffn2, "ffn2_out", il); + + residual = ggml_add(ctx0, residual, ggml_scale(ctx0, ffn2, 0.5f)); + } + + cur = build_norm(residual, layer.ln_2_w, layer.ln_2_b, + NORM_TYPE_NORMAL, eps, il); + cb(cur, "layer_out", il); + + // CTC branch + if (il + 1 == ctc_layer) { + auto * mid = build_mm(model.gs_ctc_out_w, cur); + mid = ggml_add(ctx0, mid, model.gs_ctc_out_b); + mid = ggml_soft_max(ctx0, mid); + mid = build_mm(model.gs_ctc_out_mid_w, mid); + mid = ggml_add(ctx0, mid, model.gs_ctc_out_mid_b); + cur = ggml_add(ctx0, cur, mid); + cb(cur, "ctc_branch", il); + } + } + + cb(cur, "encoder_out", -1); + + // QFormer projector + { + const int window_size = 15; + const int num_queries = 3; + const int proj_n_head = 16; + const int proj_d_head = n_embd / proj_n_head; + const float proj_kq_scale = 1.0f / sqrtf((float)proj_d_head); + const float proj_eps = 1e-12f; + const int nblocks_proj = (n_frames + window_size - 1) / window_size; + const int padded_proj = nblocks_proj * window_size; + + if (n_frames < padded_proj) { + cur = ggml_pad(ctx0, cur, 0, padded_proj - n_frames, 0, 0); + } + + ggml_tensor * enc_windows = ggml_reshape_3d(ctx0, cur, n_embd, window_size, nblocks_proj); + + ggml_tensor * queries = build_norm(model.gs_proj_query, + model.gs_proj_norm_w, model.gs_proj_norm_b, + NORM_TYPE_NORMAL, proj_eps, -1); + { + ggml_tensor * q_3d = ggml_reshape_3d(ctx0, queries, n_embd, num_queries, 1); + ggml_tensor * q_shape = ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, + n_embd, num_queries, nblocks_proj); + queries = ggml_repeat(ctx0, q_3d, q_shape); + } + + for (int il = 0; il < (int)model.gs_proj_layers.size(); il++) { + const auto & pl = model.gs_proj_layers[il]; + + // self-attention + { + ggml_tensor * Q = ggml_add(ctx0, build_mm(pl.self_attn_q_w, queries), pl.self_attn_q_b); + ggml_tensor * K = ggml_add(ctx0, build_mm(pl.self_attn_k_w, queries), pl.self_attn_k_b); + ggml_tensor * V = ggml_add(ctx0, build_mm(pl.self_attn_v_w, queries), pl.self_attn_v_b); + + Q = ggml_reshape_4d(ctx0, Q, proj_d_head, proj_n_head, num_queries, nblocks_proj); + K = ggml_reshape_4d(ctx0, K, proj_d_head, proj_n_head, num_queries, nblocks_proj); + V = ggml_reshape_4d(ctx0, V, proj_d_head, proj_n_head, num_queries, nblocks_proj); + + ggml_tensor * sa_out = build_attn(pl.self_attn_o_w, pl.self_attn_o_b, + Q, K, V, nullptr, proj_kq_scale, il); + sa_out = ggml_reshape_3d(ctx0, sa_out, n_embd, num_queries, nblocks_proj); + + queries = build_norm(ggml_add(ctx0, sa_out, queries), + pl.self_attn_norm_w, pl.self_attn_norm_b, + NORM_TYPE_NORMAL, proj_eps, il); + } + + // cross-attention + { + ggml_tensor * Q = ggml_add(ctx0, build_mm(pl.cross_attn_q_w, queries), pl.cross_attn_q_b); + ggml_tensor * K = ggml_add(ctx0, build_mm(pl.cross_attn_k_w, enc_windows), pl.cross_attn_k_b); + ggml_tensor * V = ggml_add(ctx0, build_mm(pl.cross_attn_v_w, enc_windows), pl.cross_attn_v_b); + + Q = ggml_reshape_4d(ctx0, Q, proj_d_head, proj_n_head, num_queries, nblocks_proj); + K = ggml_reshape_4d(ctx0, K, proj_d_head, proj_n_head, window_size, nblocks_proj); + V = ggml_reshape_4d(ctx0, V, proj_d_head, proj_n_head, window_size, nblocks_proj); + + ggml_tensor * ca_out = build_attn(pl.cross_attn_o_w, pl.cross_attn_o_b, + Q, K, V, nullptr, proj_kq_scale, il); + ca_out = ggml_reshape_3d(ctx0, ca_out, n_embd, num_queries, nblocks_proj); + + queries = build_norm(ggml_add(ctx0, ca_out, queries), + pl.cross_attn_norm_w, pl.cross_attn_norm_b, + NORM_TYPE_NORMAL, proj_eps, il); + } + + // ffn + { + ggml_tensor * ffn_out = build_ffn(queries, + pl.ffn_up_w, pl.ffn_up_b, + nullptr, nullptr, + pl.ffn_down_w, pl.ffn_down_b, + FFN_GELU, il); + + queries = build_norm(ggml_add(ctx0, ffn_out, queries), + pl.ffn_norm_w, pl.ffn_norm_b, + NORM_TYPE_NORMAL, proj_eps, il); + } + } + + cur = ggml_reshape_2d(ctx0, queries, n_embd, num_queries * nblocks_proj); + cur = ggml_add(ctx0, build_mm(model.gs_proj_linear_w, cur), model.gs_proj_linear_b); + cb(cur, "projector_out", -1); + } + + ggml_build_forward_expand(gf, cur); + + return gf; +} diff --git a/tools/mtmd/models/models.h b/tools/mtmd/models/models.h index 03d99e15b0..73de49d869 100644 --- a/tools/mtmd/models/models.h +++ b/tools/mtmd/models/models.h @@ -103,6 +103,11 @@ struct clip_graph_conformer : clip_graph { ggml_cgraph * build() override; }; +struct clip_graph_granite_speech : clip_graph { + clip_graph_granite_speech(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {} + ggml_cgraph * build() override; +}; + struct clip_graph_gemma4a : clip_graph { clip_graph_gemma4a(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {} ggml_cgraph * build() override; diff --git a/tools/mtmd/mtmd-audio.cpp b/tools/mtmd/mtmd-audio.cpp index 38a8ce4f4a..55c1effe4c 100644 --- a/tools/mtmd/mtmd-audio.cpp +++ b/tools/mtmd/mtmd-audio.cpp @@ -650,6 +650,108 @@ bool mtmd_audio_preprocessor_conformer::preprocess(const float * return true; } +// +// mtmd_audio_preprocessor_granite_speech +// + +void mtmd_audio_preprocessor_granite_speech::initialize() { + cache.fill_sin_cos_table(hparams.audio_n_fft); + cache.fill_hann_window(hparams.audio_window_len, true); + cache.fill_mel_filterbank_matrix( + hparams.n_mel_bins / 2, hparams.audio_n_fft, hparams.audio_sample_rate, + 0.0f, -1.0f, false, 1.0f, true); +} + +bool mtmd_audio_preprocessor_granite_speech::preprocess(const float * samples, + size_t n_samples, + std::vector & output) { + if (n_samples == 0) { + return false; + } + + GGML_ASSERT(!cache.sin_vals.empty()); + GGML_ASSERT(!cache.cos_vals.empty()); + GGML_ASSERT(!cache.filters.data.empty()); + + const int n_fft = hparams.audio_n_fft; + const int pad = n_fft / 2; + + // reflect padding + const int n_padded = (int)n_samples + 2 * pad; + std::vector padded(n_padded, 0.0f); + std::copy(samples, samples + n_samples, padded.data() + pad); + for (int i = 0; i < pad; i++) { + int src = i + 1; + if (src >= (int)n_samples) { + src = (int)n_samples - 1; + } + padded[pad - 1 - i] = samples[src]; + } + for (int i = 0; i < pad; i++) { + int src = (int)n_samples - 2 - i; + if (src < 0) { + src = 0; + } + padded[pad + (int)n_samples + i] = samples[src]; + } + + filter_params params; + params.n_mel = hparams.n_mel_bins / 2; + params.n_fft_bins = 1 + (n_fft / 2); + params.hann_window_size = hparams.audio_window_len; + params.hop_length = hparams.audio_hop_len; + params.sample_rate = hparams.audio_sample_rate; + params.no_padding = true; + params.center_padding = false; + params.preemph = 0.0f; + params.use_natural_log = false; + params.norm_per_feature = false; + params.mel_floor = 1e-10f; + + mtmd_audio_mel mel; + if (!log_mel_spectrogram(padded.data(), n_padded, 4, params, cache, mel)) { + return false; + } + + double mmax = -1e20; + for (int i = 0; i < mel.n_mel * mel.n_len; i++) { + if (mel.data[i] > mmax) { + mmax = mel.data[i]; + } + } + mmax -= 8.0; + + for (int i = 0; i < mel.n_mel * mel.n_len; i++) { + if (mel.data[i] < mmax) { + mel.data[i] = mmax; + } + mel.data[i] = (mel.data[i] + 4.0) / 4.0; + } + + int n_frames = mel.n_len; + if (n_frames % 2 == 1) { + n_frames--; + } + const int n_mel = mel.n_mel; + const int n_stacked = n_frames / 2; + + mtmd_audio_mel stacked; + stacked.n_mel = 2 * n_mel; + stacked.n_len = n_stacked; + stacked.n_len_org = (int)n_samples; + stacked.data.resize(2 * n_mel * n_stacked); + + for (int t = 0; t < n_stacked; t++) { + for (int m = 0; m < n_mel; m++) { + stacked.data[m * n_stacked + t] = mel.data[m * mel.n_len + 2 * t]; + stacked.data[(m + n_mel) * n_stacked + t] = mel.data[m * mel.n_len + 2 * t + 1]; + } + } + + output.push_back(std::move(stacked)); + return true; +} + // // mtmd_audio_preprocessor_gemma4a // diff --git a/tools/mtmd/mtmd-audio.h b/tools/mtmd/mtmd-audio.h index efaa14f924..c1a705de52 100644 --- a/tools/mtmd/mtmd-audio.h +++ b/tools/mtmd/mtmd-audio.h @@ -78,6 +78,15 @@ struct mtmd_audio_preprocessor_conformer : mtmd_audio_preprocessor { mtmd_audio_cache cache; }; +struct mtmd_audio_preprocessor_granite_speech : mtmd_audio_preprocessor { + mtmd_audio_preprocessor_granite_speech(const clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) {} + void initialize() override; + bool preprocess(const float * samples, size_t n_samples, std::vector & output) override; + + private: + mtmd_audio_cache cache; +}; + struct mtmd_audio_preprocessor_gemma4a : mtmd_audio_preprocessor { mtmd_audio_preprocessor_gemma4a(const clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) {} void initialize() override; diff --git a/tools/mtmd/mtmd.cpp b/tools/mtmd/mtmd.cpp index d0a0a4865e..c69b416b21 100644 --- a/tools/mtmd/mtmd.cpp +++ b/tools/mtmd/mtmd.cpp @@ -485,6 +485,10 @@ struct mtmd_context { { audio_preproc = std::make_unique(ctx_a); } break; + case PROJECTOR_TYPE_GRANITE_SPEECH: + { + audio_preproc = std::make_unique(ctx_a); + } break; case PROJECTOR_TYPE_GEMMA4A: { aud_beg = "<|audio>";