Skip to content

Commit 5b5b054

Browse files
taimur-10xtaimur-10x
committed
ggml-cpu: add repack GEMM and GEMV for floating-point
1 parent e95d0bc commit 5b5b054

4 files changed

Lines changed: 955 additions & 34 deletions

File tree

‎ggml/src/ggml-cpu/arch-fallback.h‎

Lines changed: 128 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,8 @@
3535
#define ggml_quantize_mat_q8_0_4x8_generic ggml_quantize_mat_q8_0_4x8
3636
#define ggml_quantize_mat_q8_K_4x4_generic ggml_quantize_mat_q8_K_4x4
3737
#define ggml_quantize_mat_q8_K_4x8_generic ggml_quantize_mat_q8_K_4x8
38+
#define ggml_repack_mat_f16_7x1_generic ggml_repack_mat_f16_7x1
39+
#define ggml_repack_mat_f32_7x1_generic ggml_repack_mat_f32_7x1
3840
#define ggml_gemv_q4_0_4x4_q8_0_generic ggml_gemv_q4_0_4x4_q8_0
3941
#define ggml_gemv_q4_0_4x8_q8_0_generic ggml_gemv_q4_0_4x8_q8_0
4042
#define ggml_gemv_q4_0_8x8_q8_0_generic ggml_gemv_q4_0_8x8_q8_0
@@ -43,6 +45,14 @@
4345
#define ggml_gemv_q2_K_8x8_q8_K_generic ggml_gemv_q2_K_8x8_q8_K
4446
#define ggml_gemv_iq4_nl_4x4_q8_0_generic ggml_gemv_iq4_nl_4x4_q8_0
4547
#define ggml_gemv_iq4_nl_8x8_q8_0_generic ggml_gemv_iq4_nl_8x8_q8_0
48+
#define ggml_gemv_f16_1x16_f16_generic ggml_gemv_f16_1x16_f16
49+
#define ggml_gemv_f16_1x32_f16_generic ggml_gemv_f16_1x32_f16
50+
#define ggml_gemv_f16_1x64_f16_generic ggml_gemv_f16_1x64_f16
51+
#define ggml_gemv_f16_1x128_f16_generic ggml_gemv_f16_1x128_f16
52+
#define ggml_gemv_f32_1x16_f32_generic ggml_gemv_f32_1x16_f32
53+
#define ggml_gemv_f32_1x32_f32_generic ggml_gemv_f32_1x32_f32
54+
#define ggml_gemv_f32_1x64_f32_generic ggml_gemv_f32_1x64_f32
55+
#define ggml_gemv_f32_1x128_f32_generic ggml_gemv_f32_1x128_f32
4656
#define ggml_gemm_q4_0_4x4_q8_0_generic ggml_gemm_q4_0_4x4_q8_0
4757
#define ggml_gemm_q4_0_4x8_q8_0_generic ggml_gemm_q4_0_4x8_q8_0
4858
#define ggml_gemm_q4_0_8x8_q8_0_generic ggml_gemm_q4_0_8x8_q8_0
@@ -51,26 +61,70 @@
5161
#define ggml_gemm_q2_K_8x8_q8_K_generic ggml_gemm_q2_K_8x8_q8_K
5262
#define ggml_gemm_iq4_nl_4x4_q8_0_generic ggml_gemm_iq4_nl_4x4_q8_0
5363
#define ggml_gemm_iq4_nl_8x8_q8_0_generic ggml_gemm_iq4_nl_8x8_q8_0
64+
#define ggml_gemm_f16_7x1x16_f16_generic ggml_gemm_f16_7x1x16_f16
65+
#define ggml_gemm_f16_7x1x32_f16_generic ggml_gemm_f16_7x1x32_f16
66+
#define ggml_gemm_f16_7x1x64_f16_generic ggml_gemm_f16_7x1x64_f16
67+
#define ggml_gemm_f16_7x1x128_f16_generic ggml_gemm_f16_7x1x128_f16
68+
#define ggml_gemm_f32_7x1x16_f32_generic ggml_gemm_f32_7x1x16_f32
69+
#define ggml_gemm_f32_7x1x32_f32_generic ggml_gemm_f32_7x1x32_f32
70+
#define ggml_gemm_f32_7x1x64_f32_generic ggml_gemm_f32_7x1x64_f32
71+
#define ggml_gemm_f32_7x1x128_f32_generic ggml_gemm_f32_7x1x128_f32
5472
#elif defined(__aarch64__) || defined(__arm__) || defined(_M_ARM) || defined(_M_ARM64)
5573
// repack.cpp
5674
#define ggml_quantize_mat_q8_K_4x4_generic ggml_quantize_mat_q8_K_4x4
5775
#define ggml_quantize_mat_q8_K_4x8_generic ggml_quantize_mat_q8_K_4x8
76+
#define ggml_repack_mat_f16_7x1_generic ggml_repack_mat_f16_7x1
77+
#define ggml_repack_mat_f32_7x1_generic ggml_repack_mat_f32_7x1
5878
#define ggml_gemv_iq4_nl_8x8_q8_0_generic ggml_gemv_iq4_nl_8x8_q8_0
5979
#define ggml_gemv_q2_K_8x8_q8_K_generic ggml_gemv_q2_K_8x8_q8_K
80+
#define ggml_gemv_f16_1x16_f16_generic ggml_gemv_f16_1x16_f16
81+
#define ggml_gemv_f16_1x32_f16_generic ggml_gemv_f16_1x32_f16
82+
#define ggml_gemv_f16_1x64_f16_generic ggml_gemv_f16_1x64_f16
83+
#define ggml_gemv_f16_1x128_f16_generic ggml_gemv_f16_1x128_f16
84+
#define ggml_gemv_f32_1x16_f32_generic ggml_gemv_f32_1x16_f32
85+
#define ggml_gemv_f32_1x32_f32_generic ggml_gemv_f32_1x32_f32
86+
#define ggml_gemv_f32_1x64_f32_generic ggml_gemv_f32_1x64_f32
87+
#define ggml_gemv_f32_1x128_f32_generic ggml_gemv_f32_1x128_f32
6088
#define ggml_gemm_iq4_nl_8x8_q8_0_generic ggml_gemm_iq4_nl_8x8_q8_0
6189
#define ggml_gemm_q2_K_8x8_q8_K_generic ggml_gemm_q2_K_8x8_q8_K
90+
#define ggml_gemm_f16_7x1x16_f16_generic ggml_gemm_f16_7x1x16_f16
91+
#define ggml_gemm_f16_7x1x32_f16_generic ggml_gemm_f16_7x1x32_f16
92+
#define ggml_gemm_f16_7x1x64_f16_generic ggml_gemm_f16_7x1x64_f16
93+
#define ggml_gemm_f16_7x1x128_f16_generic ggml_gemm_f16_7x1x128_f16
94+
#define ggml_gemm_f32_7x1x16_f32_generic ggml_gemm_f32_7x1x16_f32
95+
#define ggml_gemm_f32_7x1x32_f32_generic ggml_gemm_f32_7x1x32_f32
96+
#define ggml_gemm_f32_7x1x64_f32_generic ggml_gemm_f32_7x1x64_f32
97+
#define ggml_gemm_f32_7x1x128_f32_generic ggml_gemm_f32_7x1x128_f32
6298
#elif defined(__x86_64__) || defined(__i386__) || defined(_M_IX86) || defined(_M_X64)
6399
// repack.cpp
64100
#define ggml_quantize_mat_q8_0_4x4_generic ggml_quantize_mat_q8_0_4x4
65101
#define ggml_quantize_mat_q8_K_4x4_generic ggml_quantize_mat_q8_K_4x4
102+
#define ggml_repack_mat_f16_7x1_generic ggml_repack_mat_f16_7x1
103+
#define ggml_repack_mat_f32_7x1_generic ggml_repack_mat_f32_7x1
66104
#define ggml_gemv_q4_0_4x4_q8_0_generic ggml_gemv_q4_0_4x4_q8_0
67105
#define ggml_gemv_q4_0_4x8_q8_0_generic ggml_gemv_q4_0_4x8_q8_0
68106
#define ggml_gemv_q4_K_8x4_q8_K_generic ggml_gemv_q4_K_8x4_q8_K
69107
#define ggml_gemv_iq4_nl_4x4_q8_0_generic ggml_gemv_iq4_nl_4x4_q8_0
108+
#define ggml_gemv_f16_1x16_f16_generic ggml_gemv_f16_1x16_f16
109+
#define ggml_gemv_f16_1x32_f16_generic ggml_gemv_f16_1x32_f16
110+
#define ggml_gemv_f16_1x64_f16_generic ggml_gemv_f16_1x64_f16
111+
#define ggml_gemv_f16_1x128_f16_generic ggml_gemv_f16_1x128_f16
112+
#define ggml_gemv_f32_1x16_f32_generic ggml_gemv_f32_1x16_f32
113+
#define ggml_gemv_f32_1x32_f32_generic ggml_gemv_f32_1x32_f32
114+
#define ggml_gemv_f32_1x64_f32_generic ggml_gemv_f32_1x64_f32
115+
#define ggml_gemv_f32_1x128_f32_generic ggml_gemv_f32_1x128_f32
70116
#define ggml_gemm_q4_0_4x4_q8_0_generic ggml_gemm_q4_0_4x4_q8_0
71117
#define ggml_gemm_q4_0_4x8_q8_0_generic ggml_gemm_q4_0_4x8_q8_0
72118
#define ggml_gemm_q4_K_8x4_q8_K_generic ggml_gemm_q4_K_8x4_q8_K
73119
#define ggml_gemm_iq4_nl_4x4_q8_0_generic ggml_gemm_iq4_nl_4x4_q8_0
120+
#define ggml_gemm_f16_7x1x16_f16_generic ggml_gemm_f16_7x1x16_f16
121+
#define ggml_gemm_f16_7x1x32_f16_generic ggml_gemm_f16_7x1x32_f16
122+
#define ggml_gemm_f16_7x1x64_f16_generic ggml_gemm_f16_7x1x64_f16
123+
#define ggml_gemm_f16_7x1x128_f16_generic ggml_gemm_f16_7x1x128_f16
124+
#define ggml_gemm_f32_7x1x16_f32_generic ggml_gemm_f32_7x1x16_f32
125+
#define ggml_gemm_f32_7x1x32_f32_generic ggml_gemm_f32_7x1x32_f32
126+
#define ggml_gemm_f32_7x1x64_f32_generic ggml_gemm_f32_7x1x64_f32
127+
#define ggml_gemm_f32_7x1x128_f32_generic ggml_gemm_f32_7x1x128_f32
74128
#elif defined(__POWERPC__) || defined(__powerpc__)
75129
// ref: https://github.com/ggml-org/llama.cpp/pull/14146#issuecomment-2972561679
76130
// quants.c
@@ -83,6 +137,8 @@
83137
#define ggml_quantize_mat_q8_0_4x8_generic ggml_quantize_mat_q8_0_4x8
84138
#define ggml_quantize_mat_q8_K_4x4_generic ggml_quantize_mat_q8_K_4x4
85139
#define ggml_quantize_mat_q8_K_4x8_generic ggml_quantize_mat_q8_K_4x8
140+
#define ggml_repack_mat_f16_7x1_generic ggml_repack_mat_f16_7x1
141+
#define ggml_repack_mat_f32_7x1_generic ggml_repack_mat_f32_7x1
86142
#define ggml_gemv_q4_0_4x4_q8_0_generic ggml_gemv_q4_0_4x4_q8_0
87143
#define ggml_gemv_q4_0_4x8_q8_0_generic ggml_gemv_q4_0_4x8_q8_0
88144
#define ggml_gemv_q4_0_8x8_q8_0_generic ggml_gemv_q4_0_8x8_q8_0
@@ -91,6 +147,14 @@
91147
#define ggml_gemv_q2_K_8x8_q8_K_generic ggml_gemv_q2_K_8x8_q8_K
92148
#define ggml_gemv_iq4_nl_4x4_q8_0_generic ggml_gemv_iq4_nl_4x4_q8_0
93149
#define ggml_gemv_iq4_nl_8x8_q8_0_generic ggml_gemv_iq4_nl_8x8_q8_0
150+
#define ggml_gemv_f16_1x16_f16_generic ggml_gemv_f16_1x16_f16
151+
#define ggml_gemv_f16_1x32_f16_generic ggml_gemv_f16_1x32_f16
152+
#define ggml_gemv_f16_1x64_f16_generic ggml_gemv_f16_1x64_f16
153+
#define ggml_gemv_f16_1x128_f16_generic ggml_gemv_f16_1x128_f16
154+
#define ggml_gemv_f32_1x16_f32_generic ggml_gemv_f32_1x16_f32
155+
#define ggml_gemv_f32_1x32_f32_generic ggml_gemv_f32_1x32_f32
156+
#define ggml_gemv_f32_1x64_f32_generic ggml_gemv_f32_1x64_f32
157+
#define ggml_gemv_f32_1x128_f32_generic ggml_gemv_f32_1x128_f32
94158
#define ggml_gemm_q4_0_4x4_q8_0_generic ggml_gemm_q4_0_4x4_q8_0
95159
#define ggml_gemm_q4_0_4x8_q8_0_generic ggml_gemm_q4_0_4x8_q8_0
96160
#define ggml_gemm_q4_0_8x8_q8_0_generic ggml_gemm_q4_0_8x8_q8_0
@@ -99,6 +163,14 @@
99163
#define ggml_gemm_q2_K_8x8_q8_K_generic ggml_gemm_q2_K_8x8_q8_K
100164
#define ggml_gemm_iq4_nl_4x4_q8_0_generic ggml_gemm_iq4_nl_4x4_q8_0
101165
#define ggml_gemm_iq4_nl_8x8_q8_0_generic ggml_gemm_iq4_nl_8x8_q8_0
166+
#define ggml_gemm_f16_7x1x16_f16_generic ggml_gemm_f16_7x1x16_f16
167+
#define ggml_gemm_f16_7x1x32_f16_generic ggml_gemm_f16_7x1x32_f16
168+
#define ggml_gemm_f16_7x1x64_f16_generic ggml_gemm_f16_7x1x64_f16
169+
#define ggml_gemm_f16_7x1x128_f16_generic ggml_gemm_f16_7x1x128_f16
170+
#define ggml_gemm_f32_7x1x16_f32_generic ggml_gemm_f32_7x1x16_f32
171+
#define ggml_gemm_f32_7x1x32_f32_generic ggml_gemm_f32_7x1x32_f32
172+
#define ggml_gemm_f32_7x1x64_f32_generic ggml_gemm_f32_7x1x64_f32
173+
#define ggml_gemm_f32_7x1x128_f32_generic ggml_gemm_f32_7x1x128_f32
102174
#elif defined(__loongarch64)
103175
// quants.c
104176
#define quantize_row_q8_K_generic quantize_row_q8_K
@@ -111,6 +183,8 @@
111183
#define ggml_quantize_mat_q8_0_4x8_generic ggml_quantize_mat_q8_0_4x8
112184
#define ggml_quantize_mat_q8_K_4x4_generic ggml_quantize_mat_q8_K_4x4
113185
#define ggml_quantize_mat_q8_K_4x8_generic ggml_quantize_mat_q8_K_4x8
186+
#define ggml_repack_mat_f16_7x1_generic ggml_repack_mat_f16_7x1
187+
#define ggml_repack_mat_f32_7x1_generic ggml_repack_mat_f32_7x1
114188
#define ggml_gemv_q4_0_4x4_q8_0_generic ggml_gemv_q4_0_4x4_q8_0
115189
#define ggml_gemv_q4_0_4x8_q8_0_generic ggml_gemv_q4_0_4x8_q8_0
116190
#define ggml_gemv_q4_0_8x8_q8_0_generic ggml_gemv_q4_0_8x8_q8_0
@@ -119,6 +193,14 @@
119193
#define ggml_gemv_q2_K_8x8_q8_K_generic ggml_gemv_q2_K_8x8_q8_K
120194
#define ggml_gemv_iq4_nl_4x4_q8_0_generic ggml_gemv_iq4_nl_4x4_q8_0
121195
#define ggml_gemv_iq4_nl_8x8_q8_0_generic ggml_gemv_iq4_nl_8x8_q8_0
196+
#define ggml_gemv_f16_1x16_f16_generic ggml_gemv_f16_1x16_f16
197+
#define ggml_gemv_f16_1x32_f16_generic ggml_gemv_f16_1x32_f16
198+
#define ggml_gemv_f16_1x64_f16_generic ggml_gemv_f16_1x64_f16
199+
#define ggml_gemv_f16_1x128_f16_generic ggml_gemv_f16_1x128_f16
200+
#define ggml_gemv_f32_1x16_f32_generic ggml_gemv_f32_1x16_f32
201+
#define ggml_gemv_f32_1x32_f32_generic ggml_gemv_f32_1x32_f32
202+
#define ggml_gemv_f32_1x64_f32_generic ggml_gemv_f32_1x64_f32
203+
#define ggml_gemv_f32_1x128_f32_generic ggml_gemv_f32_1x128_f32
122204
#define ggml_gemm_q4_0_4x4_q8_0_generic ggml_gemm_q4_0_4x4_q8_0
123205
#define ggml_gemm_q4_0_4x8_q8_0_generic ggml_gemm_q4_0_4x8_q8_0
124206
#define ggml_gemm_q4_0_8x8_q8_0_generic ggml_gemm_q4_0_8x8_q8_0
@@ -127,6 +209,14 @@
127209
#define ggml_gemm_q2_K_8x8_q8_K_generic ggml_gemm_q2_K_8x8_q8_K
128210
#define ggml_gemm_iq4_nl_4x4_q8_0_generic ggml_gemm_iq4_nl_4x4_q8_0
129211
#define ggml_gemm_iq4_nl_8x8_q8_0_generic ggml_gemm_iq4_nl_8x8_q8_0
212+
#define ggml_gemm_f16_7x1x16_f16_generic ggml_gemm_f16_7x1x16_f16
213+
#define ggml_gemm_f16_7x1x32_f16_generic ggml_gemm_f16_7x1x32_f16
214+
#define ggml_gemm_f16_7x1x64_f16_generic ggml_gemm_f16_7x1x64_f16
215+
#define ggml_gemm_f16_7x1x128_f16_generic ggml_gemm_f16_7x1x128_f16
216+
#define ggml_gemm_f32_7x1x16_f32_generic ggml_gemm_f32_7x1x16_f32
217+
#define ggml_gemm_f32_7x1x32_f32_generic ggml_gemm_f32_7x1x32_f32
218+
#define ggml_gemm_f32_7x1x64_f32_generic ggml_gemm_f32_7x1x64_f32
219+
#define ggml_gemm_f32_7x1x128_f32_generic ggml_gemm_f32_7x1x128_f32
130220
#elif defined(__riscv)
131221
// quants.c
132222
#define quantize_row_q8_K_generic quantize_row_q8_K
@@ -147,6 +237,8 @@
147237
#define ggml_quantize_mat_q8_0_4x8_generic ggml_quantize_mat_q8_0_4x8
148238
#define ggml_quantize_mat_q8_K_4x4_generic ggml_quantize_mat_q8_K_4x4
149239
#define ggml_quantize_mat_q8_K_4x8_generic ggml_quantize_mat_q8_K_4x8
240+
#define ggml_repack_mat_f16_7x1_generic ggml_repack_mat_f16_7x1
241+
#define ggml_repack_mat_f32_7x1_generic ggml_repack_mat_f32_7x1
150242
#define ggml_gemv_q4_0_4x4_q8_0_generic ggml_gemv_q4_0_4x4_q8_0
151243
#define ggml_gemv_q4_0_4x8_q8_0_generic ggml_gemv_q4_0_4x8_q8_0
152244
#define ggml_gemv_q4_K_8x4_q8_K_generic ggml_gemv_q4_K_8x4_q8_K
@@ -179,6 +271,8 @@
179271
#define ggml_quantize_mat_q8_0_4x8_generic ggml_quantize_mat_q8_0_4x8
180272
#define ggml_quantize_mat_q8_K_4x4_generic ggml_quantize_mat_q8_K_4x4
181273
#define ggml_quantize_mat_q8_K_4x8_generic ggml_quantize_mat_q8_K_4x8
274+
#define ggml_repack_mat_f16_7x1_generic ggml_repack_mat_f16_7x1
275+
#define ggml_repack_mat_f32_7x1_generic ggml_repack_mat_f32_7x1
182276
#define ggml_gemv_q4_0_4x4_q8_0_generic ggml_gemv_q4_0_4x4_q8_0
183277
#define ggml_gemv_q4_0_4x8_q8_0_generic ggml_gemv_q4_0_4x8_q8_0
184278
#define ggml_gemv_q4_0_8x8_q8_0_generic ggml_gemv_q4_0_8x8_q8_0
@@ -187,6 +281,14 @@
187281
#define ggml_gemv_q2_K_8x8_q8_K_generic ggml_gemv_q2_K_8x8_q8_K
188282
#define ggml_gemv_iq4_nl_4x4_q8_0_generic ggml_gemv_iq4_nl_4x4_q8_0
189283
#define ggml_gemv_iq4_nl_8x8_q8_0_generic ggml_gemv_iq4_nl_8x8_q8_0
284+
#define ggml_gemv_f16_1x16_f16_generic ggml_gemv_f16_1x16_f16
285+
#define ggml_gemv_f16_1x32_f16_generic ggml_gemv_f16_1x32_f16
286+
#define ggml_gemv_f16_1x64_f16_generic ggml_gemv_f16_1x64_f16
287+
#define ggml_gemv_f16_1x128_f16_generic ggml_gemv_f16_1x128_f16
288+
#define ggml_gemv_f32_1x16_f32_generic ggml_gemv_f32_1x16_f32
289+
#define ggml_gemv_f32_1x32_f32_generic ggml_gemv_f32_1x32_f32
290+
#define ggml_gemv_f32_1x64_f32_generic ggml_gemv_f32_1x64_f32
291+
#define ggml_gemv_f32_1x128_f32_generic ggml_gemv_f32_1x128_f32
190292
#define ggml_gemm_q4_0_4x4_q8_0_generic ggml_gemm_q4_0_4x4_q8_0
191293
#define ggml_gemm_q4_0_4x8_q8_0_generic ggml_gemm_q4_0_4x8_q8_0
192294
#define ggml_gemm_q4_0_8x8_q8_0_generic ggml_gemm_q4_0_8x8_q8_0
@@ -195,6 +297,14 @@
195297
#define ggml_gemm_q2_K_8x8_q8_K_generic ggml_gemm_q2_K_8x8_q8_K
196298
#define ggml_gemm_iq4_nl_4x4_q8_0_generic ggml_gemm_iq4_nl_4x4_q8_0
197299
#define ggml_gemm_iq4_nl_8x8_q8_0_generic ggml_gemm_iq4_nl_8x8_q8_0
300+
#define ggml_gemm_f16_7x1x16_f16_generic ggml_gemm_f16_7x1x16_f16
301+
#define ggml_gemm_f16_7x1x32_f16_generic ggml_gemm_f16_7x1x32_f16
302+
#define ggml_gemm_f16_7x1x64_f16_generic ggml_gemm_f16_7x1x64_f16
303+
#define ggml_gemm_f16_7x1x128_f16_generic ggml_gemm_f16_7x1x128_f16
304+
#define ggml_gemm_f32_7x1x16_f32_generic ggml_gemm_f32_7x1x16_f32
305+
#define ggml_gemm_f32_7x1x32_f32_generic ggml_gemm_f32_7x1x32_f32
306+
#define ggml_gemm_f32_7x1x64_f32_generic ggml_gemm_f32_7x1x64_f32
307+
#define ggml_gemm_f32_7x1x128_f32_generic ggml_gemm_f32_7x1x128_f32
198308
#elif defined(__wasm__)
199309
// quants.c
200310
#define ggml_vec_dot_q4_1_q8_1_generic ggml_vec_dot_q4_1_q8_1
@@ -215,6 +325,8 @@
215325
#define ggml_quantize_mat_q8_0_4x8_generic ggml_quantize_mat_q8_0_4x8
216326
#define ggml_quantize_mat_q8_K_4x4_generic ggml_quantize_mat_q8_K_4x4
217327
#define ggml_quantize_mat_q8_K_4x8_generic ggml_quantize_mat_q8_K_4x8
328+
#define ggml_repack_mat_f16_7x1_generic ggml_repack_mat_f16_7x1
329+
#define ggml_repack_mat_f32_7x1_generic ggml_repack_mat_f32_7x1
218330
#define ggml_gemv_q4_0_4x4_q8_0_generic ggml_gemv_q4_0_4x4_q8_0
219331
#define ggml_gemv_q4_0_4x8_q8_0_generic ggml_gemv_q4_0_4x8_q8_0
220332
#define ggml_gemv_q4_0_8x8_q8_0_generic ggml_gemv_q4_0_8x8_q8_0
@@ -223,6 +335,14 @@
223335
#define ggml_gemv_q2_K_8x8_q8_K_generic ggml_gemv_q2_K_8x8_q8_K
224336
#define ggml_gemv_iq4_nl_4x4_q8_0_generic ggml_gemv_iq4_nl_4x4_q8_0
225337
#define ggml_gemv_iq4_nl_8x8_q8_0_generic ggml_gemv_iq4_nl_8x8_q8_0
338+
#define ggml_gemv_f16_1x16_f16_generic ggml_gemv_f16_1x16_f16
339+
#define ggml_gemv_f16_1x32_f16_generic ggml_gemv_f16_1x32_f16
340+
#define ggml_gemv_f16_1x64_f16_generic ggml_gemv_f16_1x64_f16
341+
#define ggml_gemv_f16_1x128_f16_generic ggml_gemv_f16_1x128_f16
342+
#define ggml_gemv_f32_1x16_f32_generic ggml_gemv_f32_1x16_f32
343+
#define ggml_gemv_f32_1x32_f32_generic ggml_gemv_f32_1x32_f32
344+
#define ggml_gemv_f32_1x64_f32_generic ggml_gemv_f32_1x64_f32
345+
#define ggml_gemv_f32_1x128_f32_generic ggml_gemv_f32_1x128_f32
226346
#define ggml_gemm_q4_0_4x4_q8_0_generic ggml_gemm_q4_0_4x4_q8_0
227347
#define ggml_gemm_q4_0_4x8_q8_0_generic ggml_gemm_q4_0_4x8_q8_0
228348
#define ggml_gemm_q4_0_8x8_q8_0_generic ggml_gemm_q4_0_8x8_q8_0
@@ -231,4 +351,12 @@
231351
#define ggml_gemm_q2_K_8x8_q8_K_generic ggml_gemm_q2_K_8x8_q8_K
232352
#define ggml_gemm_iq4_nl_4x4_q8_0_generic ggml_gemm_iq4_nl_4x4_q8_0
233353
#define ggml_gemm_iq4_nl_8x8_q8_0_generic ggml_gemm_iq4_nl_8x8_q8_0
354+
#define ggml_gemm_f16_7x1x16_f16_generic ggml_gemm_f16_7x1x16_f16
355+
#define ggml_gemm_f16_7x1x32_f16_generic ggml_gemm_f16_7x1x32_f16
356+
#define ggml_gemm_f16_7x1x64_f16_generic ggml_gemm_f16_7x1x64_f16
357+
#define ggml_gemm_f16_7x1x128_f16_generic ggml_gemm_f16_7x1x128_f16
358+
#define ggml_gemm_f32_7x1x16_f32_generic ggml_gemm_f32_7x1x16_f32
359+
#define ggml_gemm_f32_7x1x32_f32_generic ggml_gemm_f32_7x1x32_f32
360+
#define ggml_gemm_f32_7x1x64_f32_generic ggml_gemm_f32_7x1x64_f32
361+
#define ggml_gemm_f32_7x1x128_f32_generic ggml_gemm_f32_7x1x128_f32
234362
#endif

0 commit comments

Comments
 (0)