ggml-cpu: add repack GEMM and GEMV for floating-point

taimur-10x · taimur-10x · commit 41d77bef6582 · 2025-12-22T20:24:18.000+05:00
diff --git a/ggml/src/ggml-cpu/arch-fallback.h b/ggml/src/ggml-cpu/arch-fallback.h
@@ -73,6 +73,8 @@
 // repack.cpp
 #define ggml_quantize_mat_q8_K_4x4_generic ggml_quantize_mat_q8_K_4x4
 #define ggml_quantize_mat_q8_K_4x8_generic ggml_quantize_mat_q8_K_4x8
+#define ggml_repack_mat_f16_7x1_generic ggml_repack_mat_f16_7x1
+#define ggml_repack_mat_f32_7x1_generic ggml_repack_mat_f32_7x1
 #define ggml_gemv_iq4_nl_8x8_q8_0_generic ggml_gemv_iq4_nl_8x8_q8_0
 #define ggml_gemv_q2_K_8x8_q8_K_generic ggml_gemv_q2_K_8x8_q8_K
 #define ggml_gemv_f16_1x16_f16_generic ggml_gemv_f16_1x16_f16
diff --git a/ggml/src/ggml-cpu/arch/riscv/repack.cpp b/ggml/src/ggml-cpu/arch/riscv/repack.cpp
@@ -343,6 +343,8 @@ void ggml_gemm_q4_0_8x8_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const vo
 
 template<int ncols_interleaved>
 static inline void ggml_gemv_f16_1xM_f16(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {
+    GGML_UNUSED(bs);
+
     const int nb = n / 1;
 
     assert (nr == 1);
@@ -402,6 +404,8 @@ void ggml_gemv_f16_1x128_f16(int n, float * GGML_RESTRICT s, size_t bs, const vo
 
 template<int ncols_interleaved>
 static inline void ggml_gemv_f32_1xM_f32(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {
+    GGML_UNUSED(bs);
+
     const int nb = n / 1;
 
     assert (nr == 1);
diff --git a/ggml/src/ggml-cpu/repack.cpp b/ggml/src/ggml-cpu/repack.cpp
@@ -31,7 +31,7 @@ static inline int nearest_int(float fval) {
     return (i & 0x007fffff) - 0x00400000;
 }
 
-// Helper template functions for `fp16` and `fp32`.
+// Helper functions for `fp16` and `fp32`.
 
 template<int nrows_interleaved, int interleave_size>
 static inline void ggml_repack_mat_f16_NxK_generic(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k) {
@@ -333,7 +333,7 @@ static inline void ggml_gemv_f16_KxM_f16_generic(int n, float * GGML_RESTRICT s,
         for (int l = 0; l < nb; l++) {
             for (int j = 0; j < ncols_interleaved; j++) {
                 for (int k = 0; k < interleave_size; k++) {
-                    sumf[j] += GGML_FP16_TO_FP32(b_ptr[l].d[j * interleave_size + k]) * GGML_FP16_TO_FP32(a_ptr[l + k]);
+                    sumf[j] += GGML_FP16_TO_FP32(b_ptr[l].d[j * interleave_size + k]) * GGML_FP16_TO_FP32(a_ptr[l * interleave_size + k]);
                 }
             }
         }
@@ -363,7 +363,7 @@ static inline void ggml_gemv_f32_KxM_f32_generic(int n, float * GGML_RESTRICT s,
         for (int l = 0; l < nb; l++) {
             for (int j = 0; j < ncols_interleaved; j++) {
                 for (int k = 0; k < interleave_size; k++) {
-                    sumf[j] += b_ptr[l].d[j * interleave_size + k] * a_ptr[l + k];
+                    sumf[j] += b_ptr[l].d[j * interleave_size + k] * a_ptr[l * interleave_size + k];
                 }
             }
         }
@@ -375,7 +375,7 @@ template<int nrows, int interleave_size, int ncols_interleaved>
 static inline void ggml_gemm_f16_NxKxM_f16_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {
     const int nb = n / interleave_size;
 
-    assert (nr % nrows == 0);
+    assert(nr % nrows == 0);
     assert(n % interleave_size == 0);
     assert(nc % ncols_interleaved == 0);
 
@@ -395,7 +395,7 @@ static inline void ggml_gemm_f16_NxKxM_f16_generic(int n, float * GGML_RESTRICT
                 for (int m = 0; m < nrows; m++) {
                     for (int j = 0; j < ncols_interleaved; j++) {
                         for (int k = 0; k < interleave_size; k++) {
-                            sumf[m][j] += b_ptr[l].d[j * interleave_size + k] * a_ptr[l].d[m * interleave_size + k];
+                            sumf[m][j] += GGML_FP16_TO_FP32(b_ptr[l].d[j * interleave_size + k]) * GGML_FP16_TO_FP32(a_ptr[l].d[m * interleave_size + k]);
                         }
                     }
                 }
@@ -412,7 +412,7 @@ template<int nrows, int interleave_size, int ncols_interleaved>
 static inline void ggml_gemm_f32_NxKxM_f32_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {
     const int nb = n / interleave_size;
 
-    assert (nr % nrows == 0);
+    assert(nr % nrows == 0);
     assert(n % interleave_size == 0);
     assert(nc % ncols_interleaved == 0);
 
@@ -1913,34 +1913,34 @@ static int repack_iq4_nl_to_iq4_nl_8_bl(struct ggml_tensor * t, int interleave_b
     GGML_UNUSED(data_size);
 }
 
-template<int nrows_interleaved, int interleave_size>
-static int repack_f16_to_f16_N_bl(struct ggml_tensor * t, const void * GGML_RESTRICT data, size_t data_size) {
+template<int ncols_interleaved, int interleave_size>
+static int repack_f16_to_f16_MxK_bl(struct ggml_tensor * t, const void * GGML_RESTRICT data, size_t data_size) {
     GGML_ASSERT(t->type == GGML_TYPE_F16);
 
     const ggml_half                                     * src = (const ggml_half   *)data;
-          block_f16<nrows_interleaved, interleave_size> * dst = (      block_f16<nrows_interleaved, interleave_size> *)t->data;
+          block_f16<ncols_interleaved, interleave_size> * dst = (      block_f16<ncols_interleaved, interleave_size> *)t->data;
 
-    ggml_half dst_tmp[nrows_interleaved * interleave_size];
+    ggml_half dst_tmp[ncols_interleaved * interleave_size];
 
     int nrow = ggml_nrows(t);
     int row_size = t->ne[0];
     int nblocks = row_size / interleave_size;
 
     GGML_ASSERT(data_size == nrow * nblocks * interleave_size * sizeof(ggml_half));
 
-    if (t->ne[1] % nrows_interleaved != 0 || t->ne[0] % interleave_size != 0) {
+    if (t->ne[1] % ncols_interleaved != 0 || t->ne[0] % interleave_size != 0) {
         return -1;
     }
 
-    for (int b = 0; b < nrow; b += nrows_interleaved) {
+    for (int b = 0; b < nrow; b += ncols_interleaved) {
         for (int i = 0; i < nblocks; i++) {
-            for (int j = 0; j < nrows_interleaved; j++) {
+            for (int j = 0; j < ncols_interleaved; j++) {
                 for (int k = 0; k < interleave_size; k++) {
                     dst_tmp[j * interleave_size + k] = src[(j + b) * row_size + i * interleave_size + k];
                 }
             }
-            block_f16<nrows_interleaved, interleave_size> out;
-            memcpy(&out.d, dst_tmp, sizeof(ggml_half) * nrows_interleaved * interleave_size);
+            block_f16<ncols_interleaved, interleave_size> out;
+            memcpy(&out.d, dst_tmp, sizeof(ggml_half) * ncols_interleaved * interleave_size);
             *dst = out;
             dst++;
         }
@@ -1949,34 +1949,34 @@ static int repack_f16_to_f16_N_bl(struct ggml_tensor * t, const void * GGML_REST
     return 0;
 }
 
-template<int nrows_interleaved, int interleave_size>
-static int repack_f32_to_f32_N_bl(struct ggml_tensor * t, const void * GGML_RESTRICT data, size_t data_size) {
+template<int ncols_interleaved, int interleave_size>
+static int repack_f32_to_f32_MxK_bl(struct ggml_tensor * t, const void * GGML_RESTRICT data, size_t data_size) {
     GGML_ASSERT(t->type == GGML_TYPE_F32);
 
     const float                                         * src = (const float   *)data;
-          block_f32<nrows_interleaved, interleave_size> * dst = (      block_f32<nrows_interleaved, interleave_size> *)t->data;
+          block_f32<ncols_interleaved, interleave_size> * dst = (      block_f32<ncols_interleaved, interleave_size> *)t->data;
 
-    float dst_tmp[nrows_interleaved * interleave_size];
+    float dst_tmp[ncols_interleaved * interleave_size];
 
     int nrow = ggml_nrows(t);
     int row_size = t->ne[0];
     int nblocks = row_size / interleave_size;
 
     GGML_ASSERT(data_size == nrow * nblocks * interleave_size * sizeof(float));
 
-    if (t->ne[1] % nrows_interleaved != 0 || t->ne[0] % interleave_size != 0) {
+    if (t->ne[1] % ncols_interleaved != 0 || t->ne[0] % interleave_size != 0) {
         return -1;
     }
 
-    for (int b = 0; b < nrow; b += nrows_interleaved) {
+    for (int b = 0; b < nrow; b += ncols_interleaved) {
         for (int i = 0; i < nblocks; i++) {
-            for (int j = 0; j < nrows_interleaved; j++) {
+            for (int j = 0; j < ncols_interleaved; j++) {
                 for (int k = 0; k < interleave_size; k++) {
                     dst_tmp[j * interleave_size + k] = src[(j + b) * row_size + i * interleave_size + k];
                 }
             }
-            block_f32<nrows_interleaved, interleave_size> out;
-            memcpy(&out.d, dst_tmp, sizeof(float) * nrows_interleaved * interleave_size);
+            block_f32<ncols_interleaved, interleave_size> out;
+            memcpy(&out.d, dst_tmp, sizeof(float) * ncols_interleaved * interleave_size);
             *dst = out;
             dst++;
         }
@@ -2029,29 +2029,29 @@ template <> int repack<block_iq4_nl, 8, 8>(struct ggml_tensor * t, const void *
 }
 
 template <> int repack<ggml_half, 1, 16>(struct ggml_tensor * t, const void * data, size_t data_size) {
-    return repack_f16_to_f16_N_bl<16, 1>(t, data, data_size);
+    return repack_f16_to_f16_MxK_bl<16, 1>(t, data, data_size);
 }
 template <> int repack<ggml_half, 1, 32>(struct ggml_tensor * t, const void * data, size_t data_size) {
-    return repack_f16_to_f16_N_bl<32, 1>(t, data, data_size);
+    return repack_f16_to_f16_MxK_bl<32, 1>(t, data, data_size);
 }
 template <> int repack<ggml_half, 1, 64>(struct ggml_tensor * t, const void * data, size_t data_size) {
-    return repack_f16_to_f16_N_bl<64, 1>(t, data, data_size);
+    return repack_f16_to_f16_MxK_bl<64, 1>(t, data, data_size);
 }
 template <> int repack<ggml_half, 1, 128>(struct ggml_tensor * t, const void * data, size_t data_size) {
-    return repack_f16_to_f16_N_bl<128, 1>(t, data, data_size);
+    return repack_f16_to_f16_MxK_bl<128, 1>(t, data, data_size);
 }
 
 template <> int repack<float, 1, 16>(struct ggml_tensor * t, const void * data, size_t data_size) {
-    return repack_f32_to_f32_N_bl<16, 1>(t, data, data_size);
+    return repack_f32_to_f32_MxK_bl<16, 1>(t, data, data_size);
 }
 template <> int repack<float, 1, 32>(struct ggml_tensor * t, const void * data, size_t data_size) {
-    return repack_f32_to_f32_N_bl<32, 1>(t, data, data_size);
+    return repack_f32_to_f32_MxK_bl<32, 1>(t, data, data_size);
 }
 template <> int repack<float, 1, 64>(struct ggml_tensor * t, const void * data, size_t data_size) {
-    return repack_f32_to_f32_N_bl<64, 1>(t, data, data_size);
+    return repack_f32_to_f32_MxK_bl<64, 1>(t, data, data_size);
 }
 template <> int repack<float, 1, 128>(struct ggml_tensor * t, const void * data, size_t data_size) {
-    return repack_f32_to_f32_N_bl<128, 1>(t, data, data_size);
+    return repack_f32_to_f32_MxK_bl<128, 1>(t, data, data_size);
 }
 
 // gemv
@@ -2277,7 +2277,7 @@ template <typename BLOC_TYPE, int64_t NB_ROWS, int64_t INTER_SIZE, int64_t NB_CO
 
         GGML_ASSERT(src1_ptr + src1_col_stride * nrows <= (const char *) params->wdata + params->wsize);
 
-        // If there are more than three rows in src1, use gemm; otherwise, use gemv.
+        // If there are more than `NB_ROWS` rows in src1, use gemm; otherwise, use gemv.
         if (nrows > (NB_ROWS - 1)) {
             gemm<BLOC_TYPE, NB_ROWS, INTER_SIZE, NB_COLS, PARAM_TYPE>(ne00, (float *) (dst_ptr) + src0_start, nb1 / nb0,
                                                              src0_ptr + src0_start * nb01, src1_ptr,
diff --git a/ggml/src/ggml-cpu/repack.h b/ggml/src/ggml-cpu/repack.h
@@ -145,7 +145,6 @@ void ggml_gemv_f16_1x16_f16_generic(int n, float * GGML_RESTRICT s, size_t bs, c
 void ggml_gemv_f16_1x32_f16_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);
 void ggml_gemv_f16_1x64_f16_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);
 void ggml_gemv_f16_1x128_f16_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);
-void ggml_gemm_f16_4x1x32_f16_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);
 void ggml_gemm_f16_7x1x16_f16_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);
 void ggml_gemm_f16_7x1x32_f16_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);
 void ggml_gemm_f16_7x1x64_f16_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);
@@ -156,7 +155,6 @@ void ggml_gemv_f16_1x16_f16(int n, float * GGML_RESTRICT s, size_t bs, const voi
 void ggml_gemv_f16_1x32_f16(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);
 void ggml_gemv_f16_1x64_f16(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);
 void ggml_gemv_f16_1x128_f16(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);
-void ggml_gemm_f16_4x1x32_f16(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);
 void ggml_gemm_f16_7x1x16_f16(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);
 void ggml_gemm_f16_7x1x32_f16(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);
 void ggml_gemm_f16_7x1x64_f16(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);
