sarashina/00, fixed some bugs

Author: samuraieng

convert_hf_to_gguf.py

@@ -2947,26 +2947,12 @@ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iter
             else:
                 return
 
-        if self.model_type == ModelType.MMPROJ:
-            # Block for VLM
-            #if name.startswith("llm.") or name in ("norm.weight", "norm.bias"):
-            if name.startswith("llm."):
-                print(f"skip: {name}")
-                return  # LLMのみ変換する場合はスキップ
-
-            # llm. プレフィックスを除去して標準名に変換
-            if name.startswith("visual."):
-                name = name[len("visual."):]
-
-        else:
-            # Block for LLM
-            if name.startswith("visual.") or name in ("norm.weight", "norm.bias"):
-                print(f"skip: {name}")
-                return  # LLMのみ変換する場合はスキップ
-
-            # llm. プレフィックスを除去して標準名に変換
+        if self.origin_hf_arch.startswith('Sarashina2VisionForCausalLM'):
+            # Remove llm. from name 
             if name.startswith("llm."):
                 name = name[len("llm."):]
+            elif name.startswith("visual.") or name in ("norm.weight", "norm.bias"):
+                return  #Skip processing "modify_tensors"
 
         yield from super().modify_tensors(data_torch, name, bid)
 
@@ -3138,10 +3124,10 @@ def set_gguf_parameters(self):
             # spatial_merge_size
             if "spatial_merge_size" in self.global_config:
                 self.gguf_writer.add_vision_spatial_merge_size(self.global_config["spatial_merge_size"])
-            elif "vision_config" in self.global_config and "spatial_merge_size" in self.global_config["vision_config"]:
-                self.gguf_writer.add_vision_spatial_merge_size(self.global_config["vision_config"]["spatial_merge_size"])
-            elif self.global_config.get("model_type") == "sarashina2_vision":
-                self.gguf_writer.add_vision_spatial_merge_size(2)
+            #elif "vision_config" in self.global_config and "spatial_merge_size" in self.global_config["vision_config"]:
+            #    self.gguf_writer.add_vision_spatial_merge_size(self.global_config["vision_config"]["spatial_merge_size"])
+            #elif self.global_config.get("model_type") == "sarashina2_vision":
+            #    self.gguf_writer.add_vision_spatial_merge_size(2)
 
     def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
         n_head = (
@@ -12948,7 +12934,7 @@ def get_model_architecture(hparams: dict[str, Any], model_type: ModelType) -> st
     elif model_type == ModelType.MMPROJ and vision_config.get("architectures") is not None:
         arch = vision_config["architectures"][0]
 
-    if "Sarashina" in arch:
+    if "Sarashina2VisionForCausalLM" in arch:
         arch = "Qwen2VLForConditionalGeneration"
 
     if arch is None:

tools/mtmd/clip.cpp

@@ -1336,25 +1336,15 @@ struct clip_model_loader {
                     } break;
                 case PROJECTOR_TYPE_SARASHINA2VL_MERGER:
                     {
-                        // 1. デフォルト値のセット
-                        hparams.n_merge = 2; 
+                        hparams.n_merge = 2; // default value for Sarashina2_VL
                         hparams.image_resize_algo = RESIZE_ALGO_BILINEAR;
-
-                        // 2. GGUFから値を読み込む (第1引数は定数、第2引数は格納先)
-                        // KEY_SPATIAL_MERGE_SIZE は内部で "clip.vision.spatial_merge_size" 等に紐付いています
                         get_u32(KEY_SPATIAL_MERGE_SIZE, hparams.n_merge, false);
-                        
-                        // Qwen 2.5 ではないのでここは false か model.proj_type == ... で判定
                         get_u32(KEY_WIN_ATTN_PATTERN, hparams.n_wa_pattern, false);
-
-                        // 3. 各種制限の設定
                         hparams.set_limit_image_tokens(8, 4096);
                         hparams.set_warmup_n_tokens(46*46);
-
-                        // 4. 警告チェック
                         const int warn_min_pixels = 1024 * hparams.n_merge * hparams.n_merge * hparams.patch_size * hparams.patch_size;
                         if (hparams.image_min_pixels < warn_min_pixels) {
-                            LOG_WRN("%s: Sarashina2VL models (Qwen2-VL based) require sufficient image tokens\n", __func__);
+                            LOG_WRN("%s: Sarashina2VL models require sufficient image tokens\n", __func__);
                         }
                     } break;
                 case PROJECTOR_TYPE_YOUTUVL:
@@ -1652,6 +1642,7 @@ struct clip_model_loader {
                     || model.proj_type == PROJECTOR_TYPE_GEMMA3
                     || model.proj_type == PROJECTOR_TYPE_IDEFICS3
                     || model.proj_type == PROJECTOR_TYPE_MINICPMV
+                    || model.proj_type == PROJECTOR_TYPE_SARASHINA2VL_MERGER
                 ) && layer.ff_up_w && layer.ff_down_w && layer.ff_down_w->ne[0] == hparams.n_embd;
             if (is_ffn_swapped) {
                 // swap up and down weights
@@ -2606,11 +2597,11 @@ int clip_n_output_tokens_x(const struct clip_ctx * ctx, struct clip_image_f32 *
     switch (proj) {
         case PROJECTOR_TYPE_QWEN2VL:
         case PROJECTOR_TYPE_QWEN25VL:
-        case PROJECTOR_TYPE_SARASHINA2VL_MERGER:
         case PROJECTOR_TYPE_QWEN3VL:
         case PROJECTOR_TYPE_GLM4V:
         case PROJECTOR_TYPE_PADDLEOCR:
         case PROJECTOR_TYPE_YOUTUVL:
+        case PROJECTOR_TYPE_SARASHINA2VL_MERGER:
             return (img->nx / params.patch_size) / 2;
         default:
             break;
@@ -2628,6 +2619,7 @@ int clip_n_output_tokens_y(const struct clip_ctx * ctx, struct clip_image_f32 *
         case PROJECTOR_TYPE_GLM4V:
         case PROJECTOR_TYPE_PADDLEOCR:
         case PROJECTOR_TYPE_YOUTUVL:
+        case PROJECTOR_TYPE_SARASHINA2VL_MERGER:
             return (img->ny / params.patch_size) / 2;
         default:
             break;
@@ -2690,10 +2682,10 @@ int clip_n_output_tokens(const struct clip_ctx * ctx, struct clip_image_f32 * im
             } break;
         case PROJECTOR_TYPE_QWEN2VL:
         case PROJECTOR_TYPE_QWEN25VL:
-        case PROJECTOR_TYPE_SARASHINA2VL_MERGER:
         case PROJECTOR_TYPE_QWEN3VL:
         case PROJECTOR_TYPE_GLM4V:
         case PROJECTOR_TYPE_YOUTUVL:
+        case PROJECTOR_TYPE_SARASHINA2VL_MERGER:
             {
                 // dynamic size (2 conv, so double patch size)
                 int x_patch = img->nx / (params.patch_size * 2);
@@ -3020,8 +3012,8 @@ bool clip_image_batch_encode(clip_ctx * ctx, const int n_threads, const clip_ima
                 set_input_i32("positions", positions);
             } break;
         case PROJECTOR_TYPE_QWEN25VL:
-        case PROJECTOR_TYPE_SARASHINA2VL_MERGER:
         case PROJECTOR_TYPE_YOUTUVL:
+        case PROJECTOR_TYPE_SARASHINA2VL_MERGER:
             {
                 // pw * ph = number of tokens output by ViT after apply patch merger
                 // ipw * ipw = number of vision token been processed inside ViT
@@ -3343,9 +3335,9 @@ int clip_n_mmproj_embd(const struct clip_ctx * ctx) {
             return ctx->model.mm_model_mlp_3_w->ne[1];
         case PROJECTOR_TYPE_QWEN2VL:
         case PROJECTOR_TYPE_QWEN25VL:
-        case PROJECTOR_TYPE_SARASHINA2VL_MERGER:
         case PROJECTOR_TYPE_JANUS_PRO:
         case PROJECTOR_TYPE_YOUTUVL:
+        case PROJECTOR_TYPE_SARASHINA2VL_MERGER:
             return ctx->model.mm_1_b->ne[0];
         case PROJECTOR_TYPE_QWEN3VL:
             // main path + deepstack paths