Skip to content

Commit 3cb5e95

Browse files
timkhronostimkhronos
committed
Review driven changes
1 parent 2bb7eeb commit 3cb5e95

7 files changed

Lines changed: 36 additions & 40 deletions

File tree

‎conversion/base.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1154,7 +1154,7 @@ def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Ca
11541154
or "projector." in name or "pre_mm_projector_norm" in name \
11551155
or "image_newline" in name or "view_seperator" in name \
11561156
or "patch_embed" in name or "patch_embedding" in name \
1157-
or "patch_merger." in name or "model.connector." in name:
1157+
or "patch_merger." in name or "patch_merge_mlp." in name or "model.connector." in name:
11581158
return None
11591159

11601160
return super().filter_tensors(item)
@@ -1201,7 +1201,7 @@ def set_gguf_parameters(self):
12011201
self.gguf_writer.add_embedding_length(n_embd)
12021202
logger.info(f"gguf: embedding length = {n_embd}")
12031203

1204-
if (n_ff := self.find_hparam(["prefix_dense_intermediate_size", "intermediate_size", "n_inner", "hidden_dim"], optional=True)) is not None:
1204+
if (n_ff := self.find_hparam(["prefix_dense_intermediate_size", "dense_intermediate_size", "intermediate_size", "n_inner", "hidden_dim"], optional=True)) is not None:
12051205
self.gguf_writer.add_feed_forward_length(n_ff)
12061206
logger.info(f"gguf: feed forward length = {n_ff}")
12071207

‎conversion/minimax.py‎

Lines changed: 1 addition & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -62,14 +62,9 @@ class MiniMaxM3Model(TextModel):
6262
_experts_cache: dict[int, dict[str, Tensor]] = {}
6363

6464
def set_gguf_parameters(self):
65-
# dense layers use dense_intermediate_size, experts use intermediate_size. Base
66-
# writes feed_forward_length from intermediate_size, so swap in the dense width
67-
# and emit the expert width separately.
68-
expert_ff = self.find_hparam(["intermediate_size"])
69-
self.hparams["intermediate_size"] = self.find_hparam(["dense_intermediate_size"])
7065
super().set_gguf_parameters()
7166

72-
self.gguf_writer.add_expert_feed_forward_length(expert_ff)
67+
self.gguf_writer.add_expert_feed_forward_length(self.find_hparam(["intermediate_size"]))
7368
self.gguf_writer.add_rope_dimension_count(self.find_hparam(["rotary_dim"]))
7469
self.gguf_writer.add_expert_shared_count(self.find_hparam(["n_shared_experts"]))
7570
self.gguf_writer.add_expert_weights_scale(self.find_hparam(["routed_scaling_factor"]))
@@ -105,14 +100,6 @@ def set_gguf_parameters(self):
105100
self.gguf_writer.add_leading_dense_block_count(n_dense)
106101

107102
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None):
108-
# text-only: drop vision, projector, patch-merge tensors
109-
if name.startswith(("vision_tower", "multi_modal_projector", "patch_merge_mlp")):
110-
return
111-
112-
# strip VL wrapper prefix to match tensor_mapping names
113-
if name.startswith("language_model."):
114-
name = name[len("language_model."):]
115-
116103
# Gemma-style (1 + w) RMSNorm: bake the +1 in so llama.cpp can use plain RMSNorm
117104
if name.endswith("norm.weight"):
118105
data_torch = data_torch + 1.0

‎src/llama-graph.cpp‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1408,6 +1408,17 @@ ggml_tensor * llm_graph_context::build_ffn(
14081408
cur = ggml_swiglu(ctx0, cur);
14091409
cb(cur, "ffn_swiglu", il);
14101410
} break;
1411+
case LLM_FFN_SWIGLU_OAI:
1412+
if (gate && type_gate == LLM_FFN_PAR) {
1413+
//Same constants as LLM_FFN_SWIGLU_OAI_MOE
1414+
const float alpha = 1.702f;
1415+
const float limit = 7.0f;
1416+
cur = ggml_swiglu_oai(ctx0, cur, tmp, alpha, limit);
1417+
cb(cur, "ffn_swiglu_oai", il);
1418+
type_gate = LLM_FFN_SEQ;
1419+
} else {
1420+
GGML_ABORT("LLM_FFN_SWIGLU_OAI requires a parallel gate");
1421+
} break;
14111422
case LLM_FFN_GEGLU:
14121423
{
14131424
cur = ggml_geglu(ctx0, cur);

‎src/llama-graph.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@ enum llm_ffn_op_type : int {
4545
LLM_FFN_SWIGLU,
4646
LLM_FFN_GEGLU,
4747
LLM_FFN_REGLU,
48+
LLM_FFN_SWIGLU_OAI,
4849
LLM_FFN_SWIGLU_OAI_MOE,
4950
};
5051

‎src/llama-model.cpp‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -799,6 +799,7 @@ const char * llm_type_name(llm_type type) {
799799
case LLM_TYPE_122B_A10B: return "122B.A10B";
800800
case LLM_TYPE_196B_A11B: return "196B.A11B";
801801
case LLM_TYPE_230B_A10B: return "230B.A10B";
802+
case LLM_TYPE_428B_A23B: return "428B.A23B";
802803
case LLM_TYPE_235B_A22B: return "235B.A22B";
803804
case LLM_TYPE_300B_A47B: return "300B.A47B";
804805
case LLM_TYPE_310B_A15B: return "310B.A15B";

‎src/llama-model.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -133,6 +133,7 @@ enum llm_type {
133133
LLM_TYPE_122B_A10B, // Qwen3.5
134134
LLM_TYPE_196B_A11B, // Step3.5-Flash
135135
LLM_TYPE_230B_A10B, // Minimax M2
136+
LLM_TYPE_428B_A23B, // Minimax M3
136137
LLM_TYPE_235B_A22B,
137138
LLM_TYPE_300B_A47B, // Ernie MoE big
138139
LLM_TYPE_310B_A15B, // /MiMo-V2-Flash

‎src/models/minimax-m3.cpp‎

Lines changed: 19 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -16,15 +16,18 @@ void llama_model_minimax_m3::load_arch_hparams(llama_model_loader & ml) {
1616
ml.get_key(LLM_KV_EXPERT_SHARED_COUNT, hparams.n_expert_shared);
1717
ml.get_key(LLM_KV_EXPERT_WEIGHTS_SCALE, hparams.expert_weights_scale, false);
1818
ml.get_key(LLM_KV_EXPERT_WEIGHTS_NORM, hparams.expert_weights_norm, false);
19-
ml.get_key(LLM_KV_EXPERT_GATING_FUNC, hparams.expert_gating_func, false);
19+
ml.get_key(LLM_KV_EXPERT_GATING_FUNC, hparams.expert_gating_func);
2020
ml.get_key(LLM_KV_ATTENTION_INDEXER_HEAD_COUNT, hparams.indexer_n_head, false);
2121
ml.get_key(LLM_KV_ATTENTION_INDEXER_KEY_LENGTH, hparams.indexer_head_size, false);
2222
ml.get_key(LLM_KV_ATTENTION_INDEXER_TOP_K, hparams.indexer_top_k, false);
2323
ml.get_key(LLM_KV_ATTENTION_INDEXER_BLOCK_SIZE, hparams.indexer_block_size, false);
2424
ml.get_key(LLM_KV_ATTENTION_INDEXER_LOCAL_BLOCKS, hparams.indexer_local_blocks, false);
2525
msa_p = { (int) hparams.indexer_block_size, (int) hparams.indexer_top_k, (int) hparams.indexer_local_blocks };
2626

27-
type = LLM_TYPE_UNKNOWN;
27+
switch (hparams.n_layer()) {
28+
case 60: type = LLM_TYPE_428B_A23B; break;
29+
default: type = LLM_TYPE_UNKNOWN;
30+
}
2831
}
2932

3033
void llama_model_minimax_m3::load_arch_tensors(llama_model_loader &) {
@@ -389,10 +392,6 @@ llama_model_minimax_m3::graph::graph(const llama_model & model, const llm_graph_
389392
GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());
390393
// partial rotary: head_dim != n_rot, so don't assert n_embd_head == n_rot
391394

392-
// swigluoai params, shared by dense and expert FFNs
393-
const float swiglu_alpha = 1.702f;
394-
const float swiglu_limit = 7.0f;
395-
396395
ggml_tensor * cur;
397396
ggml_tensor * inpL;
398397

@@ -452,16 +451,8 @@ llama_model_minimax_m3::graph::graph(const llama_model & model, const llm_graph_
452451
cur = build_norm(inpL, model.layers[il].attn_norm, NULL, LLM_NORM_RMS, il);
453452
cb(cur, "attn_norm", il);
454453

455-
ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur);
456-
cb(Qcur, "Qcur", il);
457-
ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur);
458-
cb(Kcur, "Kcur", il);
459-
ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur);
460-
cb(Vcur, "Vcur", il);
461-
462-
Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);
463-
Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);
464-
Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);
454+
auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur,
455+
n_embd_head, n_head, n_head_kv, il);
465456

466457
// per-head QK RMSNorm (weights already include Gemma's +1)
467458
Qcur = build_norm(Qcur, model.layers[il].attn_q_norm, NULL, LLM_NORM_RMS, il);
@@ -574,10 +565,12 @@ llama_model_minimax_m3::graph::graph(const llama_model & model, const llm_graph_
574565

575566
if ((uint32_t) il < hparams.n_layer_dense_lead) {
576567
// leading dense FFN (swigluoai)
577-
ggml_tensor * g = build_lora_mm(model.layers[il].ffn_gate, cur);
578-
ggml_tensor * u = build_lora_mm(model.layers[il].ffn_up, cur);
579-
g = ggml_swiglu_oai(ctx0, g, u, swiglu_alpha, swiglu_limit);
580-
cur = build_lora_mm(model.layers[il].ffn_down, g);
568+
cur = build_ffn(cur,
569+
model.layers[il].ffn_up, NULL, NULL,
570+
model.layers[il].ffn_gate, NULL, NULL,
571+
model.layers[il].ffn_down, NULL, NULL,
572+
NULL,
573+
LLM_FFN_SWIGLU_OAI, LLM_FFN_PAR, il);
581574
cb(cur, "ffn_out", il);
582575
} else {
583576
// routed experts (swigluoai MoE)
@@ -595,10 +588,12 @@ llama_model_minimax_m3::graph::graph(const llama_model & model, const llm_graph_
595588
cb(moe_out, "ffn_moe_out", il);
596589

597590
// shared expert (swigluoai)
598-
ggml_tensor * sg = build_lora_mm(model.layers[il].ffn_gate_shexp, cur);
599-
ggml_tensor * su = build_lora_mm(model.layers[il].ffn_up_shexp, cur);
600-
sg = ggml_swiglu_oai(ctx0, sg, su, swiglu_alpha, swiglu_limit);
601-
ggml_tensor * ffn_shexp = build_lora_mm(model.layers[il].ffn_down_shexp, sg);
591+
ggml_tensor * ffn_shexp = build_ffn(cur,
592+
model.layers[il].ffn_up_shexp, NULL, NULL,
593+
model.layers[il].ffn_gate_shexp, NULL, NULL,
594+
model.layers[il].ffn_down_shexp, NULL, NULL,
595+
NULL,
596+
LLM_FFN_SWIGLU_OAI, LLM_FFN_PAR, il);
602597
cb(ffn_shexp, "ffn_shexp", il);
603598

604599
cur = ggml_add(ctx0, moe_out, ffn_shexp);

0 commit comments

Comments
 (0)