From 443a84a88279a8205f4555a7f441e118e1432b3a Mon Sep 17 00:00:00 2001 From: bri-prism <288398250+bri-prism@users.noreply.github.com> Date: Fri, 17 Jul 2026 20:29:16 -0400 Subject: [PATCH 1/2] ggml-cpu: x86 AVX512-VNNI repack GEMV/GEMM for Q1_0 and Q2_0 Q2_0 previously had no repack path at all and Q1_0 only a NEON one, so batched CPU mul_mat for both formats fell back to per-row vec_dot on x86. Add: - block_q2_0x4 (4-row, 8-byte-chunk interleave) with generic repack, gemv and gemm implementations - AVX512-VNNI gemv/gemm kernels for the existing q1_0 4x8 layout and the new q2_0 4x8 layout; sum(qy) is computed once per activation sub-block and the horizontal reduction happens once per output tile - repack type selection on AVX512-VNNI CPUs for both formats --- ggml/src/ggml-cpu/arch-fallback.h | 16 +- ggml/src/ggml-cpu/arch/x86/repack.cpp | 308 ++++++++++++++++++++++++++ ggml/src/ggml-cpu/repack.cpp | 225 +++++++++++++++++++ ggml/src/ggml-cpu/repack.h | 9 + 4 files changed, 556 insertions(+), 2 deletions(-) diff --git a/ggml/src/ggml-cpu/arch-fallback.h b/ggml/src/ggml-cpu/arch-fallback.h index 7b9c74857e6..25089f1e4eb 100644 --- a/ggml/src/ggml-cpu/arch-fallback.h +++ b/ggml/src/ggml-cpu/arch-fallback.h @@ -57,6 +57,7 @@ #define ggml_gemv_q8_0_4x8_q8_0_generic ggml_gemv_q8_0_4x8_q8_0 #define ggml_gemv_q1_0_4x4_q8_0_generic ggml_gemv_q1_0_4x4_q8_0 #define ggml_gemv_q1_0_4x8_q8_0_generic ggml_gemv_q1_0_4x8_q8_0 +#define ggml_gemv_q2_0_4x8_q8_0_generic ggml_gemv_q2_0_4x8_q8_0 #define ggml_gemm_q4_0_4x4_q8_0_generic ggml_gemm_q4_0_4x4_q8_0 #define ggml_gemm_q4_0_4x8_q8_0_generic ggml_gemm_q4_0_4x8_q8_0 #define ggml_gemm_q4_0_8x8_q8_0_generic ggml_gemm_q4_0_8x8_q8_0 @@ -75,6 +76,7 @@ #define ggml_gemm_q8_0_4x8_q8_0_generic ggml_gemm_q8_0_4x8_q8_0 #define ggml_gemm_q1_0_4x4_q8_0_generic ggml_gemm_q1_0_4x4_q8_0 #define ggml_gemm_q1_0_4x8_q8_0_generic ggml_gemm_q1_0_4x8_q8_0 +#define ggml_gemm_q2_0_4x8_q8_0_generic ggml_gemm_q2_0_4x8_q8_0 #elif defined(__aarch64__) || defined(__arm__) || defined(_M_ARM) || defined(_M_ARM64) // repack.cpp #define ggml_quantize_mat_q8_K_4x4_generic ggml_quantize_mat_q8_K_4x4 @@ -82,9 +84,11 @@ #define ggml_gemv_iq4_nl_8x8_q8_0_generic ggml_gemv_iq4_nl_8x8_q8_0 #define ggml_gemv_mxfp4_8x8_q8_0_generic ggml_gemv_mxfp4_8x8_q8_0 #define ggml_gemv_q2_K_8x8_q8_K_generic ggml_gemv_q2_K_8x8_q8_K +#define ggml_gemv_q2_0_4x8_q8_0_generic ggml_gemv_q2_0_4x8_q8_0 #define ggml_gemm_iq4_nl_8x8_q8_0_generic ggml_gemm_iq4_nl_8x8_q8_0 #define ggml_gemm_mxfp4_8x8_q8_0_generic ggml_gemm_mxfp4_8x8_q8_0 #define ggml_gemm_q2_K_8x8_q8_K_generic ggml_gemm_q2_K_8x8_q8_K +#define ggml_gemm_q2_0_4x8_q8_0_generic ggml_gemm_q2_0_4x8_q8_0 #elif defined(__x86_64__) || defined(__i386__) || defined(_M_IX86) || defined(_M_X64) // quants.c #define ggml_vec_dot_nvfp4_q8_0_generic ggml_vec_dot_nvfp4_q8_0 @@ -103,7 +107,6 @@ #define ggml_gemv_q8_0_4x4_q8_0_generic ggml_gemv_q8_0_4x4_q8_0 #define ggml_gemv_q8_0_4x8_q8_0_generic ggml_gemv_q8_0_4x8_q8_0 #define ggml_gemv_q1_0_4x4_q8_0_generic ggml_gemv_q1_0_4x4_q8_0 -#define ggml_gemv_q1_0_4x8_q8_0_generic ggml_gemv_q1_0_4x8_q8_0 #define ggml_gemm_q4_0_4x4_q8_0_generic ggml_gemm_q4_0_4x4_q8_0 #define ggml_gemm_q4_0_4x8_q8_0_generic ggml_gemm_q4_0_4x8_q8_0 #define ggml_gemm_q4_K_8x4_q8_K_generic ggml_gemm_q4_K_8x4_q8_K @@ -116,7 +119,6 @@ #define ggml_gemm_q8_0_4x4_q8_0_generic ggml_gemm_q8_0_4x4_q8_0 #define ggml_gemm_q8_0_4x8_q8_0_generic ggml_gemm_q8_0_4x8_q8_0 #define ggml_gemm_q1_0_4x4_q8_0_generic ggml_gemm_q1_0_4x4_q8_0 -#define ggml_gemm_q1_0_4x8_q8_0_generic ggml_gemm_q1_0_4x8_q8_0 #elif defined(__POWERPC__) || defined(__powerpc__) // ref: https://github.com/ggml-org/llama.cpp/pull/14146#issuecomment-2972561679 // quants.c @@ -150,6 +152,7 @@ #define ggml_gemv_q8_0_4x8_q8_0_generic ggml_gemv_q8_0_4x8_q8_0 #define ggml_gemv_q1_0_4x4_q8_0_generic ggml_gemv_q1_0_4x4_q8_0 #define ggml_gemv_q1_0_4x8_q8_0_generic ggml_gemv_q1_0_4x8_q8_0 +#define ggml_gemv_q2_0_4x8_q8_0_generic ggml_gemv_q2_0_4x8_q8_0 #define ggml_gemm_q4_0_4x4_q8_0_generic ggml_gemm_q4_0_4x4_q8_0 #define ggml_gemm_q4_0_4x8_q8_0_generic ggml_gemm_q4_0_4x8_q8_0 #define ggml_gemm_q4_0_8x8_q8_0_generic ggml_gemm_q4_0_8x8_q8_0 @@ -168,6 +171,7 @@ #define ggml_gemm_q8_0_4x8_q8_0_generic ggml_gemm_q8_0_4x8_q8_0 #define ggml_gemm_q1_0_4x4_q8_0_generic ggml_gemm_q1_0_4x4_q8_0 #define ggml_gemm_q1_0_4x8_q8_0_generic ggml_gemm_q1_0_4x8_q8_0 +#define ggml_gemm_q2_0_4x8_q8_0_generic ggml_gemm_q2_0_4x8_q8_0 #elif defined(__loongarch64) // quants.c #define quantize_row_q8_K_generic quantize_row_q8_K @@ -201,6 +205,7 @@ #define ggml_gemv_q8_0_4x8_q8_0_generic ggml_gemv_q8_0_4x8_q8_0 #define ggml_gemv_q1_0_4x4_q8_0_generic ggml_gemv_q1_0_4x4_q8_0 #define ggml_gemv_q1_0_4x8_q8_0_generic ggml_gemv_q1_0_4x8_q8_0 +#define ggml_gemv_q2_0_4x8_q8_0_generic ggml_gemv_q2_0_4x8_q8_0 #define ggml_gemm_q4_0_4x4_q8_0_generic ggml_gemm_q4_0_4x4_q8_0 #define ggml_gemm_q4_0_4x8_q8_0_generic ggml_gemm_q4_0_4x8_q8_0 #define ggml_gemm_q4_0_8x8_q8_0_generic ggml_gemm_q4_0_8x8_q8_0 @@ -219,6 +224,7 @@ #define ggml_gemm_q8_0_4x8_q8_0_generic ggml_gemm_q8_0_4x8_q8_0 #define ggml_gemm_q1_0_4x4_q8_0_generic ggml_gemm_q1_0_4x4_q8_0 #define ggml_gemm_q1_0_4x8_q8_0_generic ggml_gemm_q1_0_4x8_q8_0 +#define ggml_gemm_q2_0_4x8_q8_0_generic ggml_gemm_q2_0_4x8_q8_0 #elif defined(__riscv) // quants.c #define ggml_vec_dot_nvfp4_q8_0_generic ggml_vec_dot_nvfp4_q8_0 @@ -246,6 +252,7 @@ #define ggml_gemv_q8_0_4x8_q8_0_generic ggml_gemv_q8_0_4x8_q8_0 #define ggml_gemv_q1_0_4x4_q8_0_generic ggml_gemv_q1_0_4x4_q8_0 #define ggml_gemv_q1_0_4x8_q8_0_generic ggml_gemv_q1_0_4x8_q8_0 +#define ggml_gemv_q2_0_4x8_q8_0_generic ggml_gemv_q2_0_4x8_q8_0 #define ggml_gemm_q4_0_4x4_q8_0_generic ggml_gemm_q4_0_4x4_q8_0 #define ggml_gemm_q4_0_4x8_q8_0_generic ggml_gemm_q4_0_4x8_q8_0 #define ggml_gemm_q2_K_8x8_q8_K_generic ggml_gemm_q2_K_8x8_q8_K @@ -263,6 +270,7 @@ #define ggml_gemm_q8_0_4x8_q8_0_generic ggml_gemm_q8_0_4x8_q8_0 #define ggml_gemm_q1_0_4x4_q8_0_generic ggml_gemm_q1_0_4x4_q8_0 #define ggml_gemm_q1_0_4x8_q8_0_generic ggml_gemm_q1_0_4x8_q8_0 +#define ggml_gemm_q2_0_4x8_q8_0_generic ggml_gemm_q2_0_4x8_q8_0 #elif defined(__s390x__) // quants.c #define quantize_row_q8_K_generic quantize_row_q8_K @@ -302,6 +310,7 @@ #define ggml_gemv_q8_0_4x8_q8_0_generic ggml_gemv_q8_0_4x8_q8_0 #define ggml_gemv_q1_0_4x4_q8_0_generic ggml_gemv_q1_0_4x4_q8_0 #define ggml_gemv_q1_0_4x8_q8_0_generic ggml_gemv_q1_0_4x8_q8_0 +#define ggml_gemv_q2_0_4x8_q8_0_generic ggml_gemv_q2_0_4x8_q8_0 #define ggml_gemm_q4_0_4x4_q8_0_generic ggml_gemm_q4_0_4x4_q8_0 #define ggml_gemm_q4_0_4x8_q8_0_generic ggml_gemm_q4_0_4x8_q8_0 #define ggml_gemm_q4_0_8x8_q8_0_generic ggml_gemm_q4_0_8x8_q8_0 @@ -320,6 +329,7 @@ #define ggml_gemm_q8_0_4x8_q8_0_generic ggml_gemm_q8_0_4x8_q8_0 #define ggml_gemm_q1_0_4x4_q8_0_generic ggml_gemm_q1_0_4x4_q8_0 #define ggml_gemm_q1_0_4x8_q8_0_generic ggml_gemm_q1_0_4x8_q8_0 +#define ggml_gemm_q2_0_4x8_q8_0_generic ggml_gemm_q2_0_4x8_q8_0 #elif defined(__wasm__) // quants.c #define ggml_vec_dot_q4_1_q8_1_generic ggml_vec_dot_q4_1_q8_1 @@ -361,6 +371,7 @@ #define ggml_gemv_q8_0_4x8_q8_0_generic ggml_gemv_q8_0_4x8_q8_0 #define ggml_gemv_q1_0_4x4_q8_0_generic ggml_gemv_q1_0_4x4_q8_0 #define ggml_gemv_q1_0_4x8_q8_0_generic ggml_gemv_q1_0_4x8_q8_0 +#define ggml_gemv_q2_0_4x8_q8_0_generic ggml_gemv_q2_0_4x8_q8_0 #define ggml_gemm_q4_0_4x4_q8_0_generic ggml_gemm_q4_0_4x4_q8_0 #define ggml_gemm_q4_0_4x8_q8_0_generic ggml_gemm_q4_0_4x8_q8_0 #define ggml_gemm_q4_0_8x8_q8_0_generic ggml_gemm_q4_0_8x8_q8_0 @@ -379,4 +390,5 @@ #define ggml_gemm_q8_0_4x8_q8_0_generic ggml_gemm_q8_0_4x8_q8_0 #define ggml_gemm_q1_0_4x4_q8_0_generic ggml_gemm_q1_0_4x4_q8_0 #define ggml_gemm_q1_0_4x8_q8_0_generic ggml_gemm_q1_0_4x8_q8_0 +#define ggml_gemm_q2_0_4x8_q8_0_generic ggml_gemm_q2_0_4x8_q8_0 #endif diff --git a/ggml/src/ggml-cpu/arch/x86/repack.cpp b/ggml/src/ggml-cpu/arch/x86/repack.cpp index af1cebad131..480f1b1febb 100644 --- a/ggml/src/ggml-cpu/arch/x86/repack.cpp +++ b/ggml/src/ggml-cpu/arch/x86/repack.cpp @@ -6405,3 +6405,311 @@ void ggml_gemm_q2_K_8x8_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const vo #endif } + +#if defined(__AVX512F__) && defined(__AVX512BW__) && defined(__AVX512DQ__) && defined(__AVX512VNNI__) +// Helpers for the Q1_0/Q2_0 4-column interleaved kernels. +// +// Both kernels keep two 512-bit fp32 accumulators per output row: one for +// columns {0,1} and one for columns {2,3}, each holding 8 per-lane partial +// sums per column. The horizontal reduction happens once per output tile. + +// Reduce a pair of fp32 accumulators into [col0, col1, col2, col3]. +static inline __m128 __acc_pair_reduce_ps(__m512 acc01, __m512 acc23) { + const __m256 lo01 = _mm512_castps512_ps256(acc01); + const __m256 hi01 = _mm512_extractf32x8_ps(acc01, 1); + const __m256 lo23 = _mm512_castps512_ps256(acc23); + const __m256 hi23 = _mm512_extractf32x8_ps(acc23, 1); + const __m256 h0 = _mm256_hadd_ps(lo01, hi01); + const __m256 h1 = _mm256_hadd_ps(lo23, hi23); + const __m256 hh = _mm256_hadd_ps(h0, h1); + return _mm_add_ps(_mm256_castps256_ps128(hh), _mm256_extractf128_ps(hh, 1)); +} + +// Expand one 32-value sub-block of a block_q1_0x4 (16 bytes, byte 4*c + j = +// bits of column j for values 8*c..8*c+7) to {0,1} bytes: +// w01 = [col0 x32 | col1 x32], w23 = [col2 x32 | col3 x32]. +static inline void __q1_0_expand_x4(const uint8_t * qs, __m512i * w01, __m512i * w23) { + const __m128i gather = _mm_setr_epi8(0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15); + const __m128i w = _mm_shuffle_epi8(_mm_loadu_si128((const __m128i *) qs), gather); + *w01 = _mm512_maskz_set1_epi8((__mmask64) (uint64_t) _mm_extract_epi64(w, 0), 1); + *w23 = _mm512_maskz_set1_epi8((__mmask64) (uint64_t) _mm_extract_epi64(w, 1), 1); +} + +// Expand one 32-value sub-block of a block_q2_0x4 (32 bytes, 8 packed bytes +// per column) to {0..3} code bytes: w01 = [col0 x32 | col1 x32], w23 likewise. +static inline void __q2_0_expand_x4(const uint8_t * qs, __m512i * w01, __m512i * w23) { + const __m512i m3 = _mm512_set1_epi32(0x03030303); + const __m256i packed = _mm256_loadu_si256((const __m256i *) qs); + // dword d = packed byte d; spread the four 2-bit fields to byte lanes + const __m512i v01 = _mm512_cvtepu8_epi32(_mm256_castsi256_si128(packed)); + const __m512i v23 = _mm512_cvtepu8_epi32(_mm256_extracti128_si256(packed, 1)); + const __m512i r01 = _mm512_or_si512(_mm512_or_si512(v01, _mm512_slli_epi32(v01, 6)), + _mm512_or_si512(_mm512_slli_epi32(v01, 12), _mm512_slli_epi32(v01, 18))); + const __m512i r23 = _mm512_or_si512(_mm512_or_si512(v23, _mm512_slli_epi32(v23, 6)), + _mm512_or_si512(_mm512_slli_epi32(v23, 12), _mm512_slli_epi32(v23, 18))); + *w01 = _mm512_and_si512(r01, m3); + *w23 = _mm512_and_si512(r23, m3); +} +#endif // AVX512 VNNI + +void ggml_gemv_q1_0_4x8_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) { +#if defined(__AVX512F__) && defined(__AVX512BW__) && defined(__AVX512DQ__) && defined(__AVX512VNNI__) + { + const int qk = QK1_0; + const int nb = n / qk; + + assert(nr == 1); + assert(n % qk == 0); + assert(nc % 4 == 0); + UNUSED(bs); + + const __m512i ones = _mm512_set1_epi8(1); + const __m512i idx01 = _mm512_set_epi32(1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0); + const __m512i idx23 = _mm512_set_epi32(3, 3, 3, 3, 3, 3, 3, 3, 2, 2, 2, 2, 2, 2, 2, 2); + + const block_q8_0 * a_ptr = (const block_q8_0 *) vy; + for (int x = 0; x < nc / 4; x++) { + const block_q1_0x4 * b_ptr = (const block_q1_0x4 *) vx + (x * nb); + + __m512 accf01 = _mm512_setzero_ps(); + __m512 accf23 = _mm512_setzero_ps(); + + for (int l = 0; l < nb; l++) { + const __m512 d0 = _mm512_castps128_ps512(_mm_cvtph_ps(_mm_loadl_epi64((const __m128i *) b_ptr[l].d))); + const __m512 d0v01 = _mm512_permutexvar_ps(idx01, d0); + const __m512 d0v23 = _mm512_permutexvar_ps(idx23, d0); + + for (int k = 0; k < QK1_0 / QK8_0; ++k) { + const block_q8_0 * GGML_RESTRICT a_blk = a_ptr + l * (QK1_0 / QK8_0) + k; + + __m512i w01, w23; + __q1_0_expand_x4((const uint8_t *) b_ptr[l].qs + 16 * k, &w01, &w23); + + const __m512i qa = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i *) a_blk->qs)); + const __m512i sq = _mm512_dpbusd_epi32(_mm512_setzero_si512(), ones, qa); + const __m512i i01 = _mm512_dpbusd_epi32(_mm512_setzero_si512(), w01, qa); + const __m512i i23 = _mm512_dpbusd_epi32(_mm512_setzero_si512(), w23, qa); + + // signed dot partials: 2*dot(bits, qy) - sum(qy) + const __m512 f01 = _mm512_cvtepi32_ps(_mm512_sub_epi32(_mm512_add_epi32(i01, i01), sq)); + const __m512 f23 = _mm512_cvtepi32_ps(_mm512_sub_epi32(_mm512_add_epi32(i23, i23), sq)); + + const __m512 d1 = _mm512_set1_ps(GGML_CPU_FP16_TO_FP32(a_blk->d)); + accf01 = _mm512_fmadd_ps(f01, _mm512_mul_ps(d0v01, d1), accf01); + accf23 = _mm512_fmadd_ps(f23, _mm512_mul_ps(d0v23, d1), accf23); + } + } + + _mm_storeu_ps(s + x * 4, __acc_pair_reduce_ps(accf01, accf23)); + } + return; + } +#endif // AVX512 VNNI + + ggml_gemv_q1_0_4x8_q8_0_generic(n, s, bs, vx, vy, nr, nc); +} + +void ggml_gemm_q1_0_4x8_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) { +#if defined(__AVX512F__) && defined(__AVX512BW__) && defined(__AVX512DQ__) && defined(__AVX512VNNI__) + { + const int qk = QK1_0; + const int nb = n / qk; + + assert(n % qk == 0); + assert(nr % 4 == 0); + assert(nc % 4 == 0); + + const __m512i ones = _mm512_set1_epi8(1); + const __m512i idx01 = _mm512_set_epi32(1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0); + const __m512i idx23 = _mm512_set_epi32(3, 3, 3, 3, 3, 3, 3, 3, 2, 2, 2, 2, 2, 2, 2, 2); + + // qword gathers of row m from a block_q8_0x4 (row-interleaved in 8-byte chunks) + __m512i rowidx[4]; + for (int m = 0; m < 4; ++m) { + rowidx[m] = _mm512_set_epi64(12 + m, 8 + m, 4 + m, m, 12 + m, 8 + m, 4 + m, m); + } + + for (int y = 0; y < nr / 4; y++) { + const block_q8_0x4 * a_ptr = (const block_q8_0x4 *) vy + (4 * y * nb); + for (int x = 0; x < nc / 4; x++) { + const block_q1_0x4 * b_ptr = (const block_q1_0x4 *) vx + (x * nb); + + __m512 accf01[4]; + __m512 accf23[4]; + for (int m = 0; m < 4; m++) { + accf01[m] = _mm512_setzero_ps(); + accf23[m] = _mm512_setzero_ps(); + } + + for (int l = 0; l < nb; l++) { + const __m512 d0 = _mm512_castps128_ps512(_mm_cvtph_ps(_mm_loadl_epi64((const __m128i *) b_ptr[l].d))); + const __m512 d0v01 = _mm512_permutexvar_ps(idx01, d0); + const __m512 d0v23 = _mm512_permutexvar_ps(idx23, d0); + + for (int k = 0; k < QK1_0 / QK8_0; ++k) { + const block_q8_0x4 * GGML_RESTRICT a_blk = a_ptr + 4 * l + k; + + __m512i w01, w23; + __q1_0_expand_x4((const uint8_t *) b_ptr[l].qs + 16 * k, &w01, &w23); + + const __m512i qa_lo = _mm512_loadu_si512((const void *) a_blk->qs); + const __m512i qa_hi = _mm512_loadu_si512((const void *) (a_blk->qs + 64)); + + for (int m = 0; m < 4; ++m) { + const __m512i qa = _mm512_permutex2var_epi64(qa_lo, rowidx[m], qa_hi); + const __m512i sq = _mm512_dpbusd_epi32(_mm512_setzero_si512(), ones, qa); + const __m512i i01 = _mm512_dpbusd_epi32(_mm512_setzero_si512(), w01, qa); + const __m512i i23 = _mm512_dpbusd_epi32(_mm512_setzero_si512(), w23, qa); + + const __m512 f01 = _mm512_cvtepi32_ps(_mm512_sub_epi32(_mm512_add_epi32(i01, i01), sq)); + const __m512 f23 = _mm512_cvtepi32_ps(_mm512_sub_epi32(_mm512_add_epi32(i23, i23), sq)); + + const __m512 dd = _mm512_set1_ps(GGML_CPU_FP16_TO_FP32(a_blk->d[m])); + accf01[m] = _mm512_fmadd_ps(f01, _mm512_mul_ps(d0v01, dd), accf01[m]); + accf23[m] = _mm512_fmadd_ps(f23, _mm512_mul_ps(d0v23, dd), accf23[m]); + } + } + } + + for (int m = 0; m < 4; m++) { + _mm_storeu_ps(s + (y * 4 + m) * bs + x * 4, __acc_pair_reduce_ps(accf01[m], accf23[m])); + } + } + } + return; + } +#endif // AVX512 VNNI + + ggml_gemm_q1_0_4x8_q8_0_generic(n, s, bs, vx, vy, nr, nc); +} + +void ggml_gemv_q2_0_4x8_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) { +#if defined(__AVX512F__) && defined(__AVX512BW__) && defined(__AVX512DQ__) && defined(__AVX512VNNI__) + { + const int qk = QK2_0; + const int nb = n / qk; + + assert(nr == 1); + assert(n % qk == 0); + assert(nc % 4 == 0); + UNUSED(bs); + + const __m512i ones = _mm512_set1_epi8(1); + const __m512i idx01 = _mm512_set_epi32(1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0); + const __m512i idx23 = _mm512_set_epi32(3, 3, 3, 3, 3, 3, 3, 3, 2, 2, 2, 2, 2, 2, 2, 2); + + const block_q8_0 * a_ptr = (const block_q8_0 *) vy; + for (int x = 0; x < nc / 4; x++) { + const block_q2_0x4 * b_ptr = (const block_q2_0x4 *) vx + (x * nb); + + __m512 accf01 = _mm512_setzero_ps(); + __m512 accf23 = _mm512_setzero_ps(); + + for (int l = 0; l < nb; l++) { + const __m512 d0 = _mm512_castps128_ps512(_mm_cvtph_ps(_mm_loadl_epi64((const __m128i *) b_ptr[l].d))); + const __m512 d0v01 = _mm512_permutexvar_ps(idx01, d0); + const __m512 d0v23 = _mm512_permutexvar_ps(idx23, d0); + + for (int k = 0; k < QK2_0 / QK8_0; ++k) { + const block_q8_0 * GGML_RESTRICT a_blk = a_ptr + l * (QK2_0 / QK8_0) + k; + + __m512i w01, w23; + __q2_0_expand_x4((const uint8_t *) b_ptr[l].qs + 32 * k, &w01, &w23); + + const __m512i qa = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i *) a_blk->qs)); + const __m512i sq = _mm512_dpbusd_epi32(_mm512_setzero_si512(), ones, qa); + const __m512i i01 = _mm512_dpbusd_epi32(_mm512_setzero_si512(), w01, qa); + const __m512i i23 = _mm512_dpbusd_epi32(_mm512_setzero_si512(), w23, qa); + + // signed dot partials: dot(codes, qy) - sum(qy), codes-1 in {-1,0,1,2} + const __m512 f01 = _mm512_cvtepi32_ps(_mm512_sub_epi32(i01, sq)); + const __m512 f23 = _mm512_cvtepi32_ps(_mm512_sub_epi32(i23, sq)); + + const __m512 d1 = _mm512_set1_ps(GGML_CPU_FP16_TO_FP32(a_blk->d)); + accf01 = _mm512_fmadd_ps(f01, _mm512_mul_ps(d0v01, d1), accf01); + accf23 = _mm512_fmadd_ps(f23, _mm512_mul_ps(d0v23, d1), accf23); + } + } + + _mm_storeu_ps(s + x * 4, __acc_pair_reduce_ps(accf01, accf23)); + } + return; + } +#endif // AVX512 VNNI + + ggml_gemv_q2_0_4x8_q8_0_generic(n, s, bs, vx, vy, nr, nc); +} + +void ggml_gemm_q2_0_4x8_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) { +#if defined(__AVX512F__) && defined(__AVX512BW__) && defined(__AVX512DQ__) && defined(__AVX512VNNI__) + { + const int qk = QK2_0; + const int nb = n / qk; + + assert(n % qk == 0); + assert(nr % 4 == 0); + assert(nc % 4 == 0); + + const __m512i ones = _mm512_set1_epi8(1); + const __m512i idx01 = _mm512_set_epi32(1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0); + const __m512i idx23 = _mm512_set_epi32(3, 3, 3, 3, 3, 3, 3, 3, 2, 2, 2, 2, 2, 2, 2, 2); + + // qword gathers of row m from a block_q8_0x4 (row-interleaved in 8-byte chunks) + __m512i rowidx[4]; + for (int m = 0; m < 4; ++m) { + rowidx[m] = _mm512_set_epi64(12 + m, 8 + m, 4 + m, m, 12 + m, 8 + m, 4 + m, m); + } + + for (int y = 0; y < nr / 4; y++) { + const block_q8_0x4 * a_ptr = (const block_q8_0x4 *) vy + (4 * y * nb); + for (int x = 0; x < nc / 4; x++) { + const block_q2_0x4 * b_ptr = (const block_q2_0x4 *) vx + (x * nb); + + __m512 accf01[4]; + __m512 accf23[4]; + for (int m = 0; m < 4; m++) { + accf01[m] = _mm512_setzero_ps(); + accf23[m] = _mm512_setzero_ps(); + } + + for (int l = 0; l < nb; l++) { + const __m512 d0 = _mm512_castps128_ps512(_mm_cvtph_ps(_mm_loadl_epi64((const __m128i *) b_ptr[l].d))); + const __m512 d0v01 = _mm512_permutexvar_ps(idx01, d0); + const __m512 d0v23 = _mm512_permutexvar_ps(idx23, d0); + + for (int k = 0; k < QK2_0 / QK8_0; ++k) { + const block_q8_0x4 * GGML_RESTRICT a_blk = a_ptr + 4 * l + k; + + __m512i w01, w23; + __q2_0_expand_x4((const uint8_t *) b_ptr[l].qs + 32 * k, &w01, &w23); + + const __m512i qa_lo = _mm512_loadu_si512((const void *) a_blk->qs); + const __m512i qa_hi = _mm512_loadu_si512((const void *) (a_blk->qs + 64)); + + for (int m = 0; m < 4; ++m) { + const __m512i qa = _mm512_permutex2var_epi64(qa_lo, rowidx[m], qa_hi); + const __m512i sq = _mm512_dpbusd_epi32(_mm512_setzero_si512(), ones, qa); + const __m512i i01 = _mm512_dpbusd_epi32(_mm512_setzero_si512(), w01, qa); + const __m512i i23 = _mm512_dpbusd_epi32(_mm512_setzero_si512(), w23, qa); + + const __m512 f01 = _mm512_cvtepi32_ps(_mm512_sub_epi32(i01, sq)); + const __m512 f23 = _mm512_cvtepi32_ps(_mm512_sub_epi32(i23, sq)); + + const __m512 dd = _mm512_set1_ps(GGML_CPU_FP16_TO_FP32(a_blk->d[m])); + accf01[m] = _mm512_fmadd_ps(f01, _mm512_mul_ps(d0v01, dd), accf01[m]); + accf23[m] = _mm512_fmadd_ps(f23, _mm512_mul_ps(d0v23, dd), accf23[m]); + } + } + } + + for (int m = 0; m < 4; m++) { + _mm_storeu_ps(s + (y * 4 + m) * bs + x * 4, __acc_pair_reduce_ps(accf01[m], accf23[m])); + } + } + } + return; + } +#endif // AVX512 VNNI + + ggml_gemm_q2_0_4x8_q8_0_generic(n, s, bs, vx, vy, nr, nc); +} diff --git a/ggml/src/ggml-cpu/repack.cpp b/ggml/src/ggml-cpu/repack.cpp index a2adaa70acd..ac2a9ab8a34 100644 --- a/ggml/src/ggml-cpu/repack.cpp +++ b/ggml/src/ggml-cpu/repack.cpp @@ -1492,6 +1492,71 @@ void ggml_gemv_q1_0_4x8_q8_0_generic(int n, } } +void ggml_gemv_q2_0_4x8_q8_0_generic(int n, + float * GGML_RESTRICT s, + size_t bs, + const void * GGML_RESTRICT vx, + const void * GGML_RESTRICT vy, + int nr, + int nc) { + const int qk = QK2_0; + const int nb = n / qk; + const int ncols_interleaved = 4; + + assert(nr == 1); + assert(n % qk == 0); + assert(nc % ncols_interleaved == 0); + + UNUSED(bs); + UNUSED(nr); + + float sumf[4]; + + const block_q8_0 * a_ptr = (const block_q8_0 *) vy; + for (int x = 0; x < nc / ncols_interleaved; x++) { + const block_q2_0x4 * b_ptr = (const block_q2_0x4 *) vx + (x * nb); + + for (int j = 0; j < ncols_interleaved; j++) { + sumf[j] = 0.0f; + } + + for (int l = 0; l < nb; l++) { + const float d0[4] = { + GGML_CPU_FP16_TO_FP32(b_ptr[l].d[0]), + GGML_CPU_FP16_TO_FP32(b_ptr[l].d[1]), + GGML_CPU_FP16_TO_FP32(b_ptr[l].d[2]), + GGML_CPU_FP16_TO_FP32(b_ptr[l].d[3]), + }; + + for (int k = 0; k < QK2_0 / QK8_0; ++k) { + const block_q8_0 * GGML_RESTRICT a_blk = a_ptr + l * (QK2_0 / QK8_0) + k; + const float d1 = GGML_CPU_FP16_TO_FP32(a_blk->d); + + for (int j = 0; j < ncols_interleaved; ++j) { + // 8 packed bytes per row per 32-value sub-block + const uint8_t * GGML_RESTRICT qs = (const uint8_t *) b_ptr[l].qs + k * 32 + j * 8; + int sumi = 0; + + for (int b = 0; b < 8; ++b) { + const uint8_t byte = qs[b]; + // Extract 4 two-bit codes, map {0,1,2,3} -> {-1,0,1,2} + sumi += ((int) ((byte >> 0) & 3) - 1) * a_blk->qs[b * 4 + 0]; + sumi += ((int) ((byte >> 2) & 3) - 1) * a_blk->qs[b * 4 + 1]; + sumi += ((int) ((byte >> 4) & 3) - 1) * a_blk->qs[b * 4 + 2]; + sumi += ((int) ((byte >> 6) & 3) - 1) * a_blk->qs[b * 4 + 3]; + } + + sumf[j] += sumi * d0[j] * d1; + } + } + } + + for (int j = 0; j < ncols_interleaved; j++) { + s[x * ncols_interleaved + j] = sumf[j]; + } + } +} + // Only enable these for RISC-V. #if defined __riscv_zvfh void ggml_gemv_q4_0_16x1_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) { @@ -2680,6 +2745,88 @@ void ggml_gemm_q1_0_4x8_q8_0_generic(int n, } } +void ggml_gemm_q2_0_4x8_q8_0_generic(int n, + float * GGML_RESTRICT s, + size_t bs, + const void * GGML_RESTRICT vx, + const void * GGML_RESTRICT vy, + int nr, + int nc) { + const int qk = QK2_0; + const int nb = n / qk; + const int ncols_interleaved = 4; + + assert(n % qk == 0); + assert(nr % 4 == 0); + assert(nc % ncols_interleaved == 0); + + float sumf[4][4]; + + for (int y = 0; y < nr / 4; y++) { + const block_q8_0x4 * a_ptr = (const block_q8_0x4 *) vy + (4 * y * nb); + for (int x = 0; x < nc / ncols_interleaved; x++) { + const block_q2_0x4 * b_ptr = (const block_q2_0x4 *) vx + (x * nb); + + for (int m = 0; m < 4; m++) { + for (int j = 0; j < ncols_interleaved; j++) { + sumf[m][j] = 0.0f; + } + } + + for (int l = 0; l < nb; l++) { + const float d0[4] = { + GGML_CPU_FP16_TO_FP32(b_ptr[l].d[0]), + GGML_CPU_FP16_TO_FP32(b_ptr[l].d[1]), + GGML_CPU_FP16_TO_FP32(b_ptr[l].d[2]), + GGML_CPU_FP16_TO_FP32(b_ptr[l].d[3]), + }; + + for (int k = 0; k < QK2_0 / QK8_0; ++k) { + const block_q8_0x4 * GGML_RESTRICT a_blk = a_ptr + 4 * l + k; + const float a_d[4] = { + GGML_CPU_FP16_TO_FP32(a_blk->d[0]), + GGML_CPU_FP16_TO_FP32(a_blk->d[1]), + GGML_CPU_FP16_TO_FP32(a_blk->d[2]), + GGML_CPU_FP16_TO_FP32(a_blk->d[3]), + }; + + for (int j = 0; j < ncols_interleaved; ++j) { + // 8 packed bytes per row per 32-value sub-block + const uint8_t * GGML_RESTRICT qs = (const uint8_t *) b_ptr[l].qs + k * 32 + j * 8; + int sumi[4] = { 0, 0, 0, 0 }; + + for (int b = 0; b < 8; ++b) { + const uint8_t byte = qs[b]; + // Extract 4 two-bit codes, map {0,1,2,3} -> {-1,0,1,2} + const int w[4] = { + (int) ((byte >> 0) & 3) - 1, + (int) ((byte >> 2) & 3) - 1, + (int) ((byte >> 4) & 3) - 1, + (int) ((byte >> 6) & 3) - 1, + }; + + for (int m = 0; m < 4; ++m) { + const int8_t * GGML_RESTRICT qy = a_blk->qs + (b / 2) * 32 + m * 8 + (b % 2) * 4; + sumi[m] += w[0] * qy[0] + w[1] * qy[1] + w[2] * qy[2] + w[3] * qy[3]; + } + } + + for (int m = 0; m < 4; ++m) { + sumf[m][j] += sumi[m] * d0[j] * a_d[m]; + } + } + } + } + + for (int m = 0; m < 4; m++) { + for (int j = 0; j < ncols_interleaved; j++) { + s[(y * 4 + m) * bs + x * ncols_interleaved + j] = sumf[m][j]; + } + } + } + } +} + // Only enable these for RISC-V. #if defined __riscv_zvfh void ggml_gemm_q4_0_16x1_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) { @@ -3080,6 +3227,26 @@ static block_q1_0x4 make_block_q1_0x4(block_q1_0 * in, unsigned int blck_size_in return out; } +static block_q2_0x4 make_block_q2_0x4(block_q2_0 * in, unsigned int blck_size_interleave) { + block_q2_0x4 out; + + for (int i = 0; i < 4; i++) { + out.d[i] = in[i].d; + } + + // Interleave rows in 8-byte chunks (32 two-bit values each, i.e. one QK8_0 + // sub-block per row): qs[k*32 + j*8 + b] = row j, bytes [k*8 + b]. + GGML_ASSERT(blck_size_interleave == 8); + + for (int k = 0; k < QK2_0 / 32; ++k) { + for (int j = 0; j < 4; ++j) { + memcpy(&out.qs[k * 32 + j * 8], &in[j].qs[k * 8], 8); + } + } + + return out; +} + static block_q4_0x4 make_block_q4_0x4(block_q4_0 * in, unsigned int blck_size_interleave) { block_q4_0x4 out; @@ -3882,6 +4049,38 @@ static int repack_q1_0_to_q1_0_4_bl(struct ggml_tensor * t, return 0; } +static int repack_q2_0_to_q2_0_4_bl(struct ggml_tensor * t, + int interleave_block, + const void * GGML_RESTRICT data, + size_t data_size) { + GGML_ASSERT(t->type == GGML_TYPE_Q2_0); + GGML_ASSERT(interleave_block == 8); + constexpr int nrows_interleaved = 4; + + block_q2_0x4 * dst = (block_q2_0x4 *) t->data; + const block_q2_0 * src = (const block_q2_0 *) data; + block_q2_0 dst_tmp[4]; + int nrow = ggml_nrows(t); + int nblocks = t->ne[0] / QK2_0; + + GGML_ASSERT(data_size == nrow * nblocks * sizeof(block_q2_0)); + + if (t->ne[1] % nrows_interleaved != 0) { + return -1; + } + + for (int b = 0; b < nrow; b += nrows_interleaved) { + for (int64_t x = 0; x < nblocks; x++) { + for (int i = 0; i < nrows_interleaved; i++) { + dst_tmp[i] = src[x + i * nblocks]; + } + *dst++ = make_block_q2_0x4(dst_tmp, interleave_block); + } + src += nrows_interleaved * nblocks; + } + return 0; +} + static block_q8_0x16 make_block_q8_0x16(block_q8_0 * in, unsigned int blck_size_interleave) { block_q8_0x16 out; @@ -4315,6 +4514,10 @@ template <> int repack(struct ggml_tensor * t, const void * da return repack_q1_0_to_q1_0_4_bl(t, 8, data, data_size); } +template <> int repack(struct ggml_tensor * t, const void * data, size_t data_size) { + return repack_q2_0_to_q2_0_4_bl(t, 8, data, data_size); +} + #if defined __riscv_zvfh template <> int repack(struct ggml_tensor * t, const void * data, size_t data_size) { return repack_q4_0_to_q4_0_16_bl(t, 1, data, data_size); @@ -4420,6 +4623,10 @@ template <> void gemv(int n, float * s, size_t ggml_gemv_q1_0_4x8_q8_0(n, s, bs, vx, vy, nr, nc); } +template <> void gemv(int n, float * s, size_t bs, const void * vx, const void * vy, int nr, int nc) { + ggml_gemv_q2_0_4x8_q8_0(n, s, bs, vx, vy, nr, nc); +} + #if defined __riscv_zvfh template <> void gemv(int n, float * s, size_t bs, const void * vx, const void * vy, int nr, int nc) { ggml_gemv_q4_0_16x1_q8_0(n, s, bs, vx, vy, nr, nc); @@ -4525,6 +4732,10 @@ template <> void gemm(int n, float * s, size_t ggml_gemm_q1_0_4x8_q8_0(n, s, bs, vx, vy, nr, nc); } +template <> void gemm(int n, float * s, size_t bs, const void * vx, const void * vy, int nr, int nc) { + ggml_gemm_q2_0_4x8_q8_0(n, s, bs, vx, vy, nr, nc); +} + #if defined __riscv_zvfh template <> void gemm(int n, float * s, size_t bs, const void * vx, const void * vy, int nr, int nc) { ggml_gemm_q4_0_16x1_q8_0(n, s, bs, vx, vy, nr, nc); @@ -4959,6 +5170,9 @@ static const ggml::cpu::tensor_traits * ggml_repack_get_optimal_repack_type(cons static const ggml::cpu::repack::tensor_traits q1_0_4x4_q8_0; static const ggml::cpu::repack::tensor_traits q1_0_4x8_q8_0; + // instance for Q2_0 + static const ggml::cpu::repack::tensor_traits q2_0_4x8_q8_0; + // instances for RISC-V // // These implement outer-product style matrix multiplication kernels with @@ -5120,6 +5334,11 @@ static const ggml::cpu::tensor_traits * ggml_repack_get_optimal_repack_type(cons #endif } } else if (cur->type == GGML_TYPE_Q1_0) { + if (ggml_cpu_has_avx512() && ggml_cpu_has_avx512_vnni()) { + if (cur->ne[1] % 4 == 0) { + return &q1_0_4x8_q8_0; + } + } if (ggml_cpu_has_neon() && ggml_cpu_has_matmul_int8()) { if (cur->ne[1] % 4 == 0) { return &q1_0_4x8_q8_0; @@ -5130,6 +5349,12 @@ static const ggml::cpu::tensor_traits * ggml_repack_get_optimal_repack_type(cons return &q1_0_4x4_q8_0; } } + } else if (cur->type == GGML_TYPE_Q2_0) { + if (ggml_cpu_has_avx512() && ggml_cpu_has_avx512_vnni()) { + if (cur->ne[1] % 4 == 0) { + return &q2_0_4x8_q8_0; + } + } } return nullptr; diff --git a/ggml/src/ggml-cpu/repack.h b/ggml/src/ggml-cpu/repack.h index 3ccf719c39a..5bc48b649f6 100644 --- a/ggml/src/ggml-cpu/repack.h +++ b/ggml/src/ggml-cpu/repack.h @@ -14,6 +14,9 @@ template constexpr int QK_0() { if constexpr (K == 1) { return QK1_0; } + if constexpr (K == 2) { + return QK2_0; + } if constexpr (K == 4) { return QK4_0; } @@ -36,6 +39,7 @@ static_assert(sizeof(block<8, 4>) == 4 * sizeof(ggml_half) + QK8_0 * 4, "wrong b static_assert(sizeof(block<8, 8>) == 8 * sizeof(ggml_half) + QK8_0 * 8, "wrong block<8,8> size/padding"); static_assert(sizeof(block<8, 16>) == 16 * sizeof(ggml_half) + QK8_0 * 16, "wrong block<8,16> size/padding"); static_assert(sizeof(block<1, 4>) == 4 * sizeof(ggml_half) + QK1_0 / 2, "wrong block<1,4> size/padding"); +static_assert(sizeof(block<2, 4>) == 4 * sizeof(ggml_half) + QK2_0, "wrong block<2,4> size/padding"); using block_q4_0x4 = block<4, 4>; using block_q4_0x8 = block<4, 8>; @@ -44,6 +48,7 @@ using block_q8_0x4 = block<8, 4>; using block_q8_0x8 = block<8, 8>; using block_q8_0x16 = block<8, 16>; using block_q1_0x4 = block<1, 4>; +using block_q2_0x4 = block<2, 4>; struct block_q4_Kx8 { ggml_half d[8]; // super-block scale for quantized scales @@ -164,6 +169,7 @@ void ggml_gemv_q8_0_4x4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const vo void ggml_gemv_q8_0_4x8_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); void ggml_gemv_q1_0_4x4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); void ggml_gemv_q1_0_4x8_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); +void ggml_gemv_q2_0_4x8_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); void ggml_gemm_q4_0_4x4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); void ggml_gemm_q4_0_4x8_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); void ggml_gemm_q4_0_8x8_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); @@ -182,6 +188,7 @@ void ggml_gemm_q8_0_4x4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const vo void ggml_gemm_q8_0_4x8_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); void ggml_gemm_q1_0_4x4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); void ggml_gemm_q1_0_4x8_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); +void ggml_gemm_q2_0_4x8_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); #if defined __riscv_zvfh void ggml_quantize_mat_q8_0_4x1(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k); void ggml_quantize_mat_q8_K_4x1(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k); @@ -220,6 +227,7 @@ void ggml_gemv_q8_0_4x4_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, void ggml_gemv_q8_0_4x8_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); void ggml_gemv_q1_0_4x4_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); void ggml_gemv_q1_0_4x8_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); +void ggml_gemv_q2_0_4x8_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); void ggml_gemm_q4_0_4x4_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); void ggml_gemm_q4_0_4x8_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); void ggml_gemm_q4_0_8x8_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); @@ -238,6 +246,7 @@ void ggml_gemm_q8_0_4x4_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, void ggml_gemm_q8_0_4x8_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); void ggml_gemm_q1_0_4x4_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); void ggml_gemm_q1_0_4x8_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); +void ggml_gemm_q2_0_4x8_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); #if defined __riscv_zvfh void ggml_quantize_mat_q8_0_4x1_generic(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k); void ggml_quantize_mat_q8_K_4x1_generic(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k); From 458be77d1fb25133e129c6c6152961ba39099908 Mon Sep 17 00:00:00 2001 From: bri-prism <288398250+bri-prism@users.noreply.github.com> Date: Sat, 18 Jul 2026 11:12:51 -0700 Subject: [PATCH 2/2] ggml-cpu: fix int-overflow-before-widening in Q2_0 repack (CodeQL) Cast to size_t/int64_t before the nrow*nblocks and i*nblocks multiplications so they cannot overflow int before widening. Resolves CodeQL alerts 630/631. --- ggml/src/ggml-cpu/repack.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ggml/src/ggml-cpu/repack.cpp b/ggml/src/ggml-cpu/repack.cpp index ac2a9ab8a34..ef2265251af 100644 --- a/ggml/src/ggml-cpu/repack.cpp +++ b/ggml/src/ggml-cpu/repack.cpp @@ -4063,7 +4063,7 @@ static int repack_q2_0_to_q2_0_4_bl(struct ggml_tensor * t, int nrow = ggml_nrows(t); int nblocks = t->ne[0] / QK2_0; - GGML_ASSERT(data_size == nrow * nblocks * sizeof(block_q2_0)); + GGML_ASSERT(data_size == (size_t) nrow * nblocks * sizeof(block_q2_0)); if (t->ne[1] % nrows_interleaved != 0) { return -1; @@ -4072,7 +4072,7 @@ static int repack_q2_0_to_q2_0_4_bl(struct ggml_tensor * t, for (int b = 0; b < nrow; b += nrows_interleaved) { for (int64_t x = 0; x < nblocks; x++) { for (int i = 0; i < nrows_interleaved; i++) { - dst_tmp[i] = src[x + i * nblocks]; + dst_tmp[i] = src[x + (int64_t) i * nblocks]; } *dst++ = make_block_q2_0x4(dst_tmp, interleave_block); }