Instella moe

conversion/__init__.py

@@ -111,6 +111,7 @@
     "HunYuanVLForConditionalGeneration": "hunyuan",
     "HYV3ForCausalLM": "hunyuan",
     "IQuestCoderForCausalLM": "llama",
+    "InstellaMoEForCausalLM": "deepseek",
     "InternLM2ForCausalLM": "internlm",
     "InternLM3ForCausalLM": "internlm",
     "JAISLMHeadModel": "jais",

conversion/deepseek.py

@@ -443,6 +443,33 @@ def prepare_tensors(self):
                 raise ValueError(f"Unprocessed experts: {experts}")
 
 
+@ModelBase.register("InstellaMoEForCausalLM")
+class InstellaMoEModel(DeepseekV2Model):
+    model_arch = gguf.MODEL_ARCH.INSTELLA_MOE
+
+    def set_gguf_parameters(self):
+        hparams = self.hparams
+
+        # the graph implements the FarSkip-Collective dataflow unconditionally, so reject
+        # any checkpoint that only enables it on a subset of the layers
+        if not hparams.get("farskip", False):
+            raise NotImplementedError("Instella-MoE without FarSkip is not supported")
+        if hparams.get("farskip_start_idx", 0) != 0 or hparams.get("farskip_end_idx", 1e4) < hparams["num_hidden_layers"] - 1:
+            raise NotImplementedError("Instella-MoE with a partial FarSkip layer range is not supported")
+        if hparams.get("attn_only_farskip", False) or hparams.get("mlp_only_farskip", False):
+            raise NotImplementedError("Instella-MoE with attn_only_farskip/mlp_only_farskip is not supported")
+        if not hparams.get("gated_attention", False):
+            raise NotImplementedError("Instella-MoE without gated attention is not supported")
+        # Opus has suggestewd this. While thsi is not a hard requirement current IntellaMOE not utilizes it. As we force it here
+        # we could skip the whole processing of it in instella-moe.cpp. I think this is debatable.
+        if hparams.get("q_lora_rank") is not None:
+            raise NotImplementedError("Instella-MoE with a low-rank query projection is not supported")
+        if not hparams.get("rope_interleave", True):
+            raise NotImplementedError("Instella-MoE with non-interleaved RoPE is not supported")
+
+        super().set_gguf_parameters()
+
+
 @ModelBase.register("DeepseekV32ForCausalLM")
 class DeepseekV32Model(DeepseekV2Model):
     model_arch = gguf.MODEL_ARCH.DEEPSEEK32

gguf-py/gguf/constants.py

@@ -557,6 +557,7 @@ class MODEL_ARCH(IntEnum):
     TALKIE           = auto()
     MELLUM           = auto()
     NANBEIGE         = auto()
+    INSTELLA_MOE     = auto()
 
 
 class VISION_PROJECTOR_TYPE(IntEnum):
@@ -1169,6 +1170,7 @@ class MODEL_TENSOR(IntEnum):
     MODEL_ARCH.TALKIE:           "talkie",
     MODEL_ARCH.MELLUM:           "mellum",
     MODEL_ARCH.NANBEIGE:         "nanbeige",
+    MODEL_ARCH.INSTELLA_MOE:     "instella-moe",
 }
 
 VISION_PROJECTOR_TYPE_NAMES: dict[VISION_PROJECTOR_TYPE, str] = {
@@ -4603,6 +4605,31 @@ class MODEL_TENSOR(IntEnum):
         MODEL_TENSOR.FFN_DOWN,
         MODEL_TENSOR.FFN_UP,
     ],
+    MODEL_ARCH.INSTELLA_MOE: [
+        MODEL_TENSOR.TOKEN_EMBD,
+        MODEL_TENSOR.OUTPUT_NORM,
+        MODEL_TENSOR.OUTPUT,
+        MODEL_TENSOR.ATTN_NORM,
+        MODEL_TENSOR.ATTN_Q,
+        MODEL_TENSOR.ATTN_KV_A_MQA,
+        MODEL_TENSOR.ATTN_KV_A_NORM,
+        MODEL_TENSOR.ATTN_K_B,
+        MODEL_TENSOR.ATTN_V_B,
+        MODEL_TENSOR.ATTN_GATE,
+        MODEL_TENSOR.ATTN_OUT,
+        MODEL_TENSOR.FFN_NORM,
+        MODEL_TENSOR.FFN_GATE,
+        MODEL_TENSOR.FFN_DOWN,
+        MODEL_TENSOR.FFN_UP,
+        MODEL_TENSOR.FFN_GATE_INP,
+        MODEL_TENSOR.FFN_EXP_PROBS_B,
+        MODEL_TENSOR.FFN_GATE_EXP,
+        MODEL_TENSOR.FFN_DOWN_EXP,
+        MODEL_TENSOR.FFN_UP_EXP,
+        MODEL_TENSOR.FFN_GATE_SHEXP,
+        MODEL_TENSOR.FFN_DOWN_SHEXP,
+        MODEL_TENSOR.FFN_UP_SHEXP,
+    ],
 }
 
 # tensors that will not be serialized

src/llama-arch.cpp

@@ -144,6 +144,7 @@ static const std::map<llm_arch, const char *> LLM_ARCH_NAMES = {
     { LLM_ARCH_TALKIE,           "talkie"           },
     { LLM_ARCH_MELLUM,           "mellum"           },
     { LLM_ARCH_NANBEIGE,         "nanbeige"         },
+    { LLM_ARCH_INSTELLA_MOE,     "instella-moe"     },
     { LLM_ARCH_UNKNOWN,          "(unknown)"        },
 };
 

src/llama-arch.h

@@ -149,6 +149,7 @@ enum llm_arch {
     LLM_ARCH_MINIMAX_M3,
     LLM_ARCH_DFLASH,
     LLM_ARCH_NANBEIGE,
+    LLM_ARCH_INSTELLA_MOE,
     LLM_ARCH_UNKNOWN,
 };
 

src/llama-model.cpp

@@ -187,6 +187,8 @@ static llama_model * llama_model_mapping(llm_arch arch, const llama_model_params
             return new llama_model_deepseek32(params);
         case LLM_ARCH_DEEPSEEK4:
             return new llama_model_deepseek4(params);
+        case LLM_ARCH_INSTELLA_MOE:
+            return new llama_model_instella_moe(params);
         case LLM_ARCH_GLM_DSA:
             return new llama_model_glm_dsa(params);
         case LLM_ARCH_MISTRAL4:
@@ -1878,7 +1880,7 @@ void llama_model::print_info() const {
             LLAMA_LOG_INFO("%s: expert_weights_scale  = %.1f\n",   __func__, hparams.expert_weights_scale);
         }
 
-        if (arch == LLM_ARCH_DEEPSEEK2 || arch == LLM_ARCH_DEEPSEEK2OCR || arch == LLM_ARCH_DEEPSEEK32 || arch == LLM_ARCH_GLM_DSA || arch == LLM_ARCH_MISTRAL4) {
+        if (arch == LLM_ARCH_DEEPSEEK2 || arch == LLM_ARCH_DEEPSEEK2OCR || arch == LLM_ARCH_DEEPSEEK32 || arch == LLM_ARCH_GLM_DSA || arch == LLM_ARCH_MISTRAL4 || arch == LLM_ARCH_INSTELLA_MOE) {
             LLAMA_LOG_INFO("%s: n_layer_dense_lead    = %d\n",     __func__, hparams.n_layer_dense_lead);
             LLAMA_LOG_INFO("%s: n_lora_q              = %d\n",     __func__, hparams.n_lora_q);
             LLAMA_LOG_INFO("%s: n_lora_kv             = %d\n",     __func__, hparams.n_lora_kv);
@@ -2547,6 +2549,7 @@ llama_rope_type llama_model_rope_type(const llama_model * model) {
         case LLM_ARCH_MAINCODER:
         case LLM_ARCH_GLM_DSA:
         case LLM_ARCH_NANBEIGE:
+        case LLM_ARCH_INSTELLA_MOE:
             return LLAMA_ROPE_TYPE_NORM;
 
         // the pairs of head values are offset by n_rot/2