Skip to content

ggml-cuda: widen Q4_K MMVQ weight loads to global_load_b128 #88

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Annieren wants to merge 1 commit into
base: gfx11
Choose a base branch
from
+35 −28
Open

ggml-cuda: widen Q4_K MMVQ weight loads to global_load_b128 #88

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
63 changes: 35 additions & 28 deletions ggml/src/ggml-cuda/vecdotq.cuh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -569,27 +569,33 @@ static __device__ __forceinline__ float vec_dot_q3_K_q8_1_impl_mmq(
return d3*d8 * sumi;
}

#define VDR_Q4_K_Q8_1_MMVQ 2
#define VDR_Q4_K_Q8_1_MMVQ 8
#define VDR_Q4_K_Q8_1_MMQ 8

// contiguous v/x values
static __device__ __forceinline__ float vec_dot_q4_K_q8_1_impl_vmmq(
const int * __restrict__ v, const int * __restrict__ u, const uint8_t * __restrict__ sc,
// VDR=8: one thread owns a full 32-byte Q4_K group (4 slots). vlo/vhi hold the two 16-byte
// weight halves (b128-loaded); ulo/uhi hold the matching q8_1 ints indexed [i*4 + slot].
static __device__ __forceinline__ float vec_dot_q4_K_q8_1_impl_vmmq8(
const int * __restrict__ vlo, const int * __restrict__ vhi,
const int * __restrict__ ulo, const int * __restrict__ uhi, const uint8_t * __restrict__ sc,
const uint8_t * __restrict__ m, const half2 & dm4, const float * __restrict__ d8) {

float sumf_d = 0.0f;
float sumf_m = 0.0f;

#pragma unroll
for (int i = 0; i < QR4_K; ++i) {
const int v0i = (v[0] >> (4*i)) & 0x0F0F0F0F;
const int v1i = (v[1] >> (4*i)) & 0x0F0F0F0F;
#pragma unroll
for (int s = 0; s < 4; ++s) {
const int v0i = (vlo[s] >> (4*i)) & 0x0F0F0F0F;
const int v1i = (vhi[s] >> (4*i)) & 0x0F0F0F0F;

const int dot1 = ggml_cuda_dp4a(v1i, u[2*i+1], ggml_cuda_dp4a(v0i, u[2*i+0], 0)); // SIMD dot product
const int dot2 = ggml_cuda_dp4a(0x01010101, u[2*i+1], ggml_cuda_dp4a(0x01010101, u[2*i+0], 0)); // sum of u
const int dot1 = ggml_cuda_dp4a(v1i, uhi[i*4 + s], ggml_cuda_dp4a(v0i, ulo[i*4 + s], 0)); // SIMD dot product
const int dot2 = ggml_cuda_dp4a(0x01010101, uhi[i*4 + s], ggml_cuda_dp4a(0x01010101, ulo[i*4 + s], 0)); // sum of u

sumf_d += d8[i] * (dot1 * sc[i]);
sumf_m += d8[i] * (dot2 * m[i]); // multiply constant part of q4_K with sum of q8_1 values
sumf_d += d8[i] * (dot1 * sc[i]);
sumf_m += d8[i] * (dot2 * m[i]); // multiply constant part of q4_K with sum of q8_1 values
}
}

const float2 dm4f = __half22float2(dm4);
Expand Down Expand Up @@ -1000,21 +1006,16 @@ static __device__ __forceinline__ float vec_dot_q4_K_q8_1(

const block_q4_K * bq4_K = (const block_q4_K *) vbq + kbx;

int v[2];
int u[2*QR4_K];
float d8[QR4_K];

// iqs is in 0,2..30. bq8_offset = iqs/4 -> bq8_offset = 0, 2, 4, 6
const int bq8_offset = QR4_K * ((iqs/2) / (QI8_1/2));
// VDR=8: iqs is in {0,8,16,24}; each thread owns one full 32-byte group.
const int bq8_offset = QR4_K * ((iqs/2) / (QI8_1/2)); // {0,2,4,6}; group = bq8_offset/2

// iqs = 0....3 -> bq8_offset = 0, want q4_offset = 0, 4, 8, 12
// iqs = 4....7 -> bq8_offset = 2, want q4_offset = 32, 36, 40, 44
// iqs = 8...11 -> bq8_offset = 4, want q4_offset = 64, 68, 72, 76
// iqs = 12..15 -> bq8_offset = 6, want q4_offset = 96, 100, 104, 108

const int * q4 = (const int *)(bq4_K->qs + 16 * bq8_offset + 4 * ((iqs/2)%4));
v[0] = q4[0];
v[1] = q4[4];
// Weight: load the whole 32-byte group as two 16-byte int4 (-> global_load_b128).
// q4base is 16-byte aligned (qs at +16 in a 144-byte block; 16*bq8_offset is a mult. of 16).
const int * q4base = (const int *)(bq4_K->qs + 16 * bq8_offset);
int vlo[4]; // bytes 0..15 of the group (old v[0] across the 4 slots)
int vhi[4]; // bytes 16..31 of the group (old v[1] across the 4 slots)
*((int4 *) vlo) = *((const int4 *) (q4base + 0));
*((int4 *) vhi) = *((const int4 *) (q4base + 4));

const uint16_t * scales = (const uint16_t *)bq4_K->scales;
uint16_t aux[2];
Expand All @@ -1029,16 +1030,22 @@ static __device__ __forceinline__ float vec_dot_q4_K_q8_1(
const uint8_t * sc = (const uint8_t *)aux;
const uint8_t * m = sc + 2;

// Activations: q8_1 blocks are 36 bytes apart (qs 4-byte aligned), so these stay scalar b32.
int ulo[QR4_K*4];
int uhi[QR4_K*4];
float d8[QR4_K];
#pragma unroll
for (int i = 0; i < QR4_K; ++i) {
const block_q8_1 * bq8i = bq8_1 + bq8_offset + i;
d8[i] = __low2float(bq8i->ds);

const int * q8 = (const int *)bq8i->qs + ((iqs/2)%4);
u[2*i+0] = q8[0];
u[2*i+1] = q8[4];
#pragma unroll
for (int s = 0; s < 4; ++s) {
ulo[i*4 + s] = get_int_b4(bq8i->qs, s);
uhi[i*4 + s] = get_int_b4(bq8i->qs, s + 4);
}
}

return vec_dot_q4_K_q8_1_impl_vmmq(v, u, sc, m, bq4_K->dm, d8);
return vec_dot_q4_K_q8_1_impl_vmmq8(vlo, vhi, ulo, uhi, sc, m, bq4_K->dm, d8);
}

static __device__ __forceinline__ float vec_dot_q5_K_q8_1(
Expand Down
Loading