Skip to content

opencl: fix q6_K flat mul_mat for Adreno A6x/A7x GPUs with older E031 compilers #26476

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wanghqc wants to merge 2 commits into
base: master
Choose a base branch
from
+136 −2
Draft

opencl: fix q6_K flat mul_mat for Adreno A6x/A7x GPUs with older E031 compilers #26476

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
a150a1c
opencl: decline KV-convert flash_attn variants on Adreno A7X (compile…
wanghqc Jul 9, 2026
cd452de
opencl: fix q6_K flat mul_mat on older Adreno E031 compilers, gated
wanghqc Jul 9, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
42 changes: 41 additions & 1 deletion ggml/src/ggml-opencl/ggml-opencl.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -581,6 +581,8 @@ struct ggml_backend_opencl_context {
bool adreno_use_bin_kernels;
get_adreno_bin_kernel_func_t get_adreno_bin_kernel_func = nullptr;
ggml_cl_compiler_version adreno_cl_compiler_version;
// The q6_K flat mul_mat codegen workarounds are needed by old E031 compilers only.
bool q6_k_flat_old_compiler;

std::string kernel_compile_opts; // cached for lazy-compiled kernels.

Expand Down Expand Up @@ -1926,8 +1928,14 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {
#else
const std::string kernel_src = read_file("mul_mv_q6_k_f32_flat.cl");
#endif
// The codegen workarounds in this kernel are a measured 13-20% loss on
// compilers that do not need them, so only the affected ones build them;
// everyone else gets the original source.
const std::string q6k_opts = backend_ctx->q6_k_flat_old_compiler
? compile_opts + " -DADRENO_OLD_COMPILER=1"
: compile_opts;
cl_program prog =
build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts);
build_program_from_source(backend_ctx, kernel_src.c_str(), q6k_opts);

CL_CHECK((backend_ctx->kernel_mul_mv_q6_K_f32_flat = clCreateKernel(prog, "kernel_mul_mv_q6_K_f32_flat", &err), err));
CL_CHECK(clReleaseProgram(prog));
Expand Down Expand Up @@ -5871,6 +5879,16 @@ static ggml_backend_opencl_context * ggml_cl_init(ggml_backend_dev_t dev) {
(backend_ctx->adreno_cl_compiler_version.type == E031 && backend_ctx->adreno_cl_compiler_version.major >= 47) ||
(backend_ctx->adreno_cl_compiler_version.type == DX && backend_ctx->adreno_cl_compiler_version.major >= 17);

// The q6_K flat mul_mat miscompile is a defect of the older E031 compilers, not a
// property of any GPU generation: it reproduces on E031.38 (Adreno 642L) and E031.41
// (Adreno 740) and is fixed by E031.45 (Adreno 619). Gate on the compiler so parts
// that do not need the workarounds do not pay for them. The explicit type check is
// required: newer_than_or_same() is false for every non-E031 compiler, so negating it
// alone would enable the workarounds on E17/DX.
backend_ctx->q6_k_flat_old_compiler =
backend_ctx->adreno_cl_compiler_version.type == E031 &&
!backend_ctx->adreno_cl_compiler_version.newer_than_or_same(E031, 45, 0, 0);

size_t ext_str_size;
clGetDeviceInfo(device, CL_DEVICE_EXTENSIONS, 0, NULL, &ext_str_size);
char *ext_buffer = (char *)alloca(ext_str_size + 1);
Expand Down Expand Up @@ -7409,13 +7427,29 @@ static bool ggml_opencl_supports_op(ggml_backend_dev_t dev, const struct ggml_te
v->type == GGML_TYPE_F16 && op->type == GGML_TYPE_F16;
const bool is_f32_f16 = q->type == GGML_TYPE_F32 && k->type == GGML_TYPE_F16 &&
v->type == GGML_TYPE_F16 && op->type == GGML_TYPE_F32;

const bool is_f32_q8_0 = q->type == GGML_TYPE_F32 && k->type == GGML_TYPE_Q8_0 &&
v->type == GGML_TYPE_Q8_0 && op->type == GGML_TYPE_F32 &&
dk % 32 == 0 && dv % 32 == 0;
const bool is_f32_q4_0 = q->type == GGML_TYPE_F32 && k->type == GGML_TYPE_Q4_0 &&
v->type == GGML_TYPE_Q4_0 && op->type == GGML_TYPE_F32 &&
dk % 32 == 0 && dv % 32 == 0;

// A7X (Adreno 740, compiler E031.41) SIGSEGVs inside clBuildProgram
// building the flash_attn programs whose KV path is mixed-type or
// dequantized — f32_f16, q8_0, q4_0 (reproduced at DK=40 and DK=64; it
// is DK-independent). It is a driver crash, not codegen-wrong-output, so
// it cannot be caught in-process (fatal=false only handles clean compile
// errors). The uniform f16_f16 / f32_f32 programs compile fine on this
// compiler, so decline only the KV-convert variants; ggml then runs
// those (f16-KV / quant-KV) attention layers on the CPU backend.
// Negative compiler carve-out, same idiom as the Intel DK=512 decline
// below and the X1E driver-quirk guards.
if (backend_ctx && backend_ctx->adreno_gen == ADRENO_GPU_GEN::A7X &&
(is_f32_f16 || is_f32_q8_0 || is_f32_q4_0)) {
return false;
}

// Asymmetric KV: host-dequants both sides to F32, uses f32 kernel.
auto is_kv_type_ok = [](ggml_type t) {
return t == GGML_TYPE_F16 || t == GGML_TYPE_F32 ||
Expand Down Expand Up @@ -20393,6 +20427,12 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co
CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &ne1));
CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &r2));
CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &r3));
// The optimizer-barrier arg exists only in the ADRENO_OLD_COMPILER build of
// this kernel; conformant compilers get the original 17-arg signature.
if (backend_ctx->q6_k_flat_old_compiler) {
cl_uchar q6k_mask = 0xFF; // never 0xFE in prod; see the kernel note
CL_CHECK(clSetKernelArg(kernel, 17, sizeof(cl_uchar), &q6k_mask));
}
#else
kernel = backend_ctx->kernel_mul_mv_q6_K_f32;

Expand Down
96 changes: 95 additions & 1 deletion ggml/src/ggml-opencl/kernels/mul_mv_q6_k_f32_flat.cl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,13 @@

#define QK_K 256

// ADRENO_OLD_COMPILER is defined by the host (-D) only for the Adreno E031
// compilers older than E031.45, which miscompile several constructs this kernel
// used (confirmed on E031.38 and E031.41; E031.45 is clean). Every other
// compiler -- newer E031, E17, DX, Intel, and every non-Adreno device that
// builds this program -- takes the #else branches, which are the original
// source: the workarounds below cost ~13% on the q6_K flat n=1 GEMV where they
// are not needed.
inline float block_q_6_K_dot_y_flat(
global uchar * blk_ql,
global uchar * blk_qh,
Expand All @@ -37,6 +44,9 @@ inline float block_q_6_K_dot_y_flat(
int ip,
int is,
int l0,
#if defined(ADRENO_OLD_COMPILER)
int dbg,
#endif
float4 y0,
float4 y1,
float4 y2,
Expand All @@ -48,10 +58,40 @@ inline float block_q_6_K_dot_y_flat(
global uchar * q1 = blk_ql + ib*128 + q_offset_l;
global uchar * q2 = q1 + QK_K/8;
global uchar * qh = blk_qh + ib*64 + q_offset_h;
global char * sc = blk_scales + ib*16 + is;

float dall = blk_d[ib];

#if defined(ADRENO_OLD_COMPILER)
// The vectorized dequant (int4/float4 bit-ops, convert_*4, dot()) and vload4
// are miscompiled here -> garbage weights. Reconstruct the 6-bit weights and
// take the dot product scalar. q4_K/q5_K flat already use scalar paths, which
// is why q6_K was the only flat GEMV that failed.
// Scales are SIGNED int8; read as uchar and sign-extend arithmetically so the
// result does not depend on whether the compiler treats `char` as signed.
global uchar * sc = (global uchar *)(blk_scales + ib*16 + is);

int s0 = (int)sc[0] - 256*(sc[0] >> 7);
int s2 = (int)sc[2] - 256*(sc[2] >> 7);
int s4 = (int)sc[4] - 256*(sc[4] >> 7);
int s6 = (int)sc[6] - 256*(sc[6] >> 7);

// one 6-bit weight: low/high nibble of a ql byte OR'd with a 2-bit qh plane
// (plane p in {0,1,2,3} selects qh bits 2p..2p+1) placed at bits 4-5, minus 32.
#define Q6W(qb, sh, hb, p) ((float)((((int)(qb) >> (sh)) & 15) | ((((int)(hb) >> (2*(p))) & 3) << 4)) - 32.f)

float d0 = y0.s0*Q6W(q1[0],0,qh[0],0) + y0.s1*Q6W(q1[1],0,qh[1],0) + y0.s2*Q6W(q1[2],0,qh[2],0) + y0.s3*Q6W(q1[3],0,qh[3],0);
float d1 = y1.s0*Q6W(q2[0],0,qh[0],1) + y1.s1*Q6W(q2[1],0,qh[1],1) + y1.s2*Q6W(q2[2],0,qh[2],1) + y1.s3*Q6W(q2[3],0,qh[3],1);
float d2 = y2.s0*Q6W(q1[0],4,qh[0],2) + y2.s1*Q6W(q1[1],4,qh[1],2) + y2.s2*Q6W(q1[2],4,qh[2],2) + y2.s3*Q6W(q1[3],4,qh[3],2);
float d3 = y3.s0*Q6W(q2[0],4,qh[0],3) + y3.s1*Q6W(q2[1],4,qh[1],3) + y3.s2*Q6W(q2[2],4,qh[2],3) + y3.s3*Q6W(q2[3],4,qh[3],3);
#undef Q6W

if (dbg) printf("HELPER dall=%f s=[%d %d %d %d] d=[%f %f %f %f] ql0=%d qh0=%d y00=%f\n",
dall, s0, s2, s4, s6, d0, d1, d2, d3, (int)q1[0], (int)qh[0], y0.s0);

return dall * (d0 * s0 + d1 * s2 + d2 * s4 + d3 * s6);
#else
global char * sc = blk_scales + ib*16 + is;

// Vectorized loads: 3 uchar4 weight loads instead of 12 scalar byte reads.
// q_offset_l/h are 4-aligned, so these are aligned vector loads.
uchar4 q1v = vload4(0, q1);
Expand All @@ -72,6 +112,7 @@ inline float block_q_6_K_dot_y_flat(

return dall * (dot(y0, w0) * sc[0] + dot(y1, w1) * sc[2] +
dot(y2, w2) * sc[4] + dot(y3, w3) * sc[6]);
#endif
}

#undef N_DST
Expand Down Expand Up @@ -113,6 +154,11 @@ kernel void kernel_mul_mv_q6_K_f32_flat(
int ne1,
int r2,
int r3
#if defined(ADRENO_OLD_COMPILER)
,
uchar q6k_mask // runtime 0xFF; the host passes it so the compiler cannot
// constant-fold the printf guards below into nothing
#endif
) {
src1 = (global float*)((global char*)src1 + offset1);
dst = (global float*)((global char*)dst + offsetd);
Expand All @@ -128,6 +174,22 @@ kernel void kernel_mul_mv_q6_K_f32_flat(

int first_row = (N_SIMDGROUP * r0 + get_sub_group_id()) * N_DST;

#if defined(ADRENO_OLD_COMPILER)
// 64-bit `ulong` integer arithmetic is miscompiled here -> the base-pointer byte
// offsets came out wrong, so EVERY weight/scale read hit the wrong address. This
// was the primary cause of the q6_K flat failure (q5_K uses int offsets and is
// unaffected). Compute the block index in `int` and widen to `ulong` only inside
// the pointer expression: the byte offset stays 64-bit, but there is no ulong
// arithmetic chain to miscompile. The int index would overflow past ~2^31 blocks,
// which no realistic weight reaches -- but that is a narrowing, so keep it off the
// conformant path, which retains full ulong arithmetic.
int offset_src0 = first_row*nb + (i12/r2)*(nb*ne01) + (i13/r3)*(nb*ne01*ne02);

global uchar * blk_ql = (global uchar *) src0_ql + (ulong)offset_src0 * 128;
global uchar * blk_qh = (global uchar *) src0_qh + (ulong)offset_src0 * 64;
global char * blk_scales = (global char *) src0_s + (ulong)offset_src0 * 16;
global half * blk_d = (global half *) src0_d + offset_src0;
#else
ulong offset_src0 = first_row*nb + (i12/r2)*(nb*ne01) + (i13/r3)*(nb*ne01*ne02);
ulong offset_src0_ql = offset_src0 * 128;
ulong offset_src0_qh = offset_src0 * 64;
Expand All @@ -138,6 +200,7 @@ kernel void kernel_mul_mv_q6_K_f32_flat(
global uchar * blk_qh = (global uchar *) src0_qh + offset_src0_qh;
global char * blk_scales = (global char *) src0_s + offset_src0_s;
global half * blk_d = (global half *) src0_d + offset_src0_d;
#endif
global float * yy = (global float *) src1 + r1*ne10 + im*ne00*ne1;

int tid = get_sub_group_local_id()%(N_SIMDWIDTH/BLOCK_STRIDE); // within-super-block part, 0..15
Expand All @@ -155,24 +218,55 @@ kernel void kernel_mul_mv_q6_K_f32_flat(

for (int ib = ix; ib < nb; ib += BLOCK_STRIDE) {
global float * y = yy + ib * QK_K + 128*ip + l0;
#if defined(ADRENO_OLD_COMPILER)
// vload4 of f32 is miscompiled here; index the lanes scalar instead.
float4 y0 = (float4)(y[ 0], y[ 1], y[ 2], y[ 3]);
float4 y1 = (float4)(y[32], y[33], y[34], y[35]);
float4 y2 = (float4)(y[64], y[65], y[66], y[67]);
float4 y3 = (float4)(y[96], y[97], y[98], y[99]);
#else
float4 y0 = vload4(0, y + 0);
float4 y1 = vload4(0, y + 32);
float4 y2 = vload4(0, y + 64);
float4 y3 = vload4(0, y + 96);
#endif

for (int row = 0; row < N_DST; row++) {
if (first_row + row < ne01) {
#if defined(ADRENO_OLD_COMPILER)
int dbg = (q6k_mask==0xFE && r0==0 && r1==0 && im==0 && row==0 && ib==0 &&
ne00==256 && ne01==16 && get_sub_group_local_id()==0) ? 1 : 0;
sumf[row] += block_q_6_K_dot_y_flat(
blk_ql + row*nb*128, blk_qh + row*nb*64, blk_scales + row*nb*16, blk_d + row*nb,
ib, ip, is, l0, dbg, y0, y1, y2, y3);
#else
sumf[row] += block_q_6_K_dot_y_flat(
blk_ql + row*nb*128, blk_qh + row*nb*64, blk_scales + row*nb*16, blk_d + row*nb,
ib, ip, is, l0, y0, y1, y2, y3);
#endif
}
}
}

#if defined(ADRENO_OLD_COMPILER)
// Optimizer barrier. This compiler drops the sumf partials unless a side effect
// forces them to materialize. q6k_mask is a kernel arg the compiler cannot prove
// is never 0xFE (the host always passes 0xFF), so the printf survives compilation
// but never executes. FRAGILE: the exact set and placement of these guarded
// printfs is load-bearing on E031.41 -- removing any one re-breaks q6_K.
if (q6k_mask==0xFE && r0==0 && r1==0 && im==0 && ne00==256 && ne01==16 && get_sub_group_local_id()<16) {
printf("Q6KLANE lane=%d ip=%d il=%d is=%d l0=%d sumf0=%f\n",
get_sub_group_local_id(), ip, il, is, l0, sumf[0]);
}
#endif
for (int row = 0; row < N_DST; row++) {
float tot = sub_group_reduce_add(sumf[row]);
if (get_sub_group_local_id() == 0 && first_row + row < ne01) {
dst[r1*ne0 + im*ne0*ne1 + first_row + row] = tot;
#if defined(ADRENO_OLD_COMPILER)
if (q6k_mask==0xFE && r0==0 && r1==0 && im==0 && row==0 && ne00==256 && ne01==16)
printf("Q6KTOT tot=%f\n", tot);
#endif
}
}
}