diff --git a/ggml/src/ggml-opencl/CMakeLists.txt b/ggml/src/ggml-opencl/CMakeLists.txt index 1dc707177106..586bc53e7ec6 100644 --- a/ggml/src/ggml-opencl/CMakeLists.txt +++ b/ggml/src/ggml-opencl/CMakeLists.txt @@ -84,6 +84,7 @@ set(GGML_OPENCL_KERNELS mul_mv_f16_f32_1row mul_mv_f16_f32_l4 mul_mv_f16_f32 + mul_mv_f16_f32_mrow mul_mv_f32_f32 mul_mv_q1_0_f32 mul_mv_q1_0_f32_flat @@ -179,9 +180,14 @@ set(GGML_OPENCL_KERNELS gemv_noshuffle_q8_0_f32 gemm_noshuffle_q8_0_f32 gemv_noshuffle_q4_k_f32 + gemv_noshuffle_q4_k_f32_o4 + gemv_noshuffle_q4_k_f32_tiled gemm_noshuffle_q4_k_f32 gemv_noshuffle_q6_k_f32 + gemv_noshuffle_q6_k_f32_o4 + gemv_noshuffle_q6_k_f32_tiled gemm_noshuffle_q6_k_f32 + gemm_noshuffle_q6_k_f32_tiled gemv_noshuffle_q5_k_f32 gemm_noshuffle_q5_k_f32 mul diff --git a/ggml/src/ggml-opencl/ggml-opencl.cpp b/ggml/src/ggml-opencl/ggml-opencl.cpp index fc0fce0d780a..b8456f5265f4 100644 --- a/ggml/src/ggml-opencl/ggml-opencl.cpp +++ b/ggml/src/ggml-opencl/ggml-opencl.cpp @@ -567,6 +567,10 @@ struct ggml_backend_opencl_context { bool has_integer_dot = false; // cl_khr_integer_dot_product or cl_qcom_dot_product8 bool has_qcom_subgroup_shuffle = false; // specifically cl_qcom_subgroup_shuffle bool disable_fusion; + bool fuse_mm_glu = true; // opt-out GGML_OPENCL_FUSE_MM_GLU=0 (byte-identical gate+up GEMV + GLU, q4_K FFN) + bool fuse_rms_add = true; // opt-out GGML_OPENCL_FUSE_RMS_ADD=0 (fused rms_norm*w + residual) + bool f16_mrow = true; // opt-out GGML_OPENCL_F16_MROW=0 (multi-row-per-WG f16 decode GEMV for attn proj + lm_head) + int f16_mrow_rpt = 1; // GGML_OPENCL_F16_MROW_RPT={1,2,4,8,16} rows-per-subgroup register blocking // ragged moe, use int to directly pass to kernel cl_uint adreno_use_moe_ragged; @@ -610,6 +614,7 @@ struct ggml_backend_opencl_context { ggml_cl_buffer prealloc_moe_sa; // per-block s [tok_slots * ne00/32] (half) // scratch copy of the router weights to avoid dst aliasing ggml_cl_buffer prealloc_moe_combine_w; + ggml_cl_buffer prealloc_splitk_partial; // [ksplit * M] partials for split-K GEMV // pool of persistent image1d_buffer views over kv-cache layers, keyed by // (parent buffer, offset within parent) @@ -737,6 +742,7 @@ struct ggml_backend_opencl_context { kernel_geglu_f16, kernel_reglu_f16, kernel_swiglu_f16, kernel_geglu_erf_f16, kernel_geglu_quick_f16; cl_kernel kernel_norm, kernel_norm_mul_add; cl_kernel kernel_rms_norm, kernel_rms_norm_mul; + cl_kernel kernel_rms_norm_mul_add = nullptr; // fused rms_norm(x)*w + b (residual) cl_kernel kernel_l2_norm_f32; cl_kernel kernel_group_norm, kernel_group_norm_mul_add; cl_kernel kernel_diag_mask_inf, kernel_diag_mask_inf_8; @@ -756,6 +762,12 @@ struct ggml_backend_opencl_context { cl_kernel kernel_mul_mat_f32_f32; cl_kernel kernel_mul_mat_f16_f16; cl_kernel kernel_mul_mat_f16_f32_1row; + cl_program program_mul_mv_f16_f32_mrow; + cl_kernel kernel_mul_mat_f16_f32_mrow = nullptr; // multi-row decode GEMV (attn proj + lm_head) + cl_kernel kernel_mul_mat_f16_f32_mrow_r2 = nullptr; + cl_kernel kernel_mul_mat_f16_f32_mrow_r4 = nullptr; + cl_kernel kernel_mul_mat_f16_f32_mrow_h8 = nullptr; + cl_kernel kernel_mul_mat_f16_f32_mrow_h8r2 = nullptr; cl_kernel kernel_mul_mat_f16_f32; cl_kernel kernel_mul_mat_f16_f32_l4; cl_kernel kernel_mul_mat_f16_f32_l4_dr; @@ -897,6 +909,7 @@ struct ggml_backend_opencl_context { cl_kernel kernel_mul_mv_id_mxfp4_f32; cl_kernel kernel_mul_mv_id_mxfp4_f32_flat; cl_kernel kernel_mul_mm_f32_f32_l4_lm; + cl_kernel kernel_gemv_f32_f32_mc; // multi-column (small-N) f32 GEMV for spec/MTP verify cl_kernel kernel_mul_mm_f16_f32_l4_lm; cl_kernel kernel_mul_mm_q1_0_f32_l4_lm; cl_kernel kernel_mul_mm_q4_0_f32_l4_lm; @@ -1062,28 +1075,50 @@ struct ggml_backend_opencl_context { // Gemm and Gemv related programs, kernels, etc cl_kernel kernel_gemm_noshuffle_q4_0_f32; cl_kernel kernel_gemv_noshuffle_q4_0_f32; + cl_kernel kernel_gemv_noshuffle_q4_0_f32_mc3; // multi-column (N=3) verify GEMV (spec/MTP) cl_kernel kernel_gemv_noshuffle_q4_0_f32_4096_1_11008; cl_kernel kernel_gemv_noshuffle_q4_0_f32_4096_1_4096; cl_kernel kernel_gemv_noshuffle_q4_0_f32_11008_1_4096; cl_kernel kernel_gemv_noshuffle_q4_0_f32_32000_1_4096; cl_kernel kernel_gemv_noshuffle_q4_1_f32; + cl_kernel kernel_gemv_noshuffle_q4_1_f32_mc3; // multi-column (N=3) verify GEMV (spec/MTP) cl_kernel kernel_gemm_noshuffle_q4_1_f32; cl_kernel kernel_gemm_noshuffle_q8_0_f32, kernel_gemm_noshuffle_q8_0_f32_bin; cl_kernel kernel_gemm_noshuffle_q8_0_q8_1_dp4a = nullptr; // dp4a (int8) dense q8_0 prefill GEMM (opt-in) cl_kernel kernel_gemm_noshuffle_q8_0_q8_1_dp4a_wimg = nullptr; // q8_0 dense dp4a, weights via texture (opt-in) cl_kernel kernel_gemv_noshuffle_q8_0_f32; + cl_kernel kernel_gemv_noshuffle_q8_0_f32_splitk; // split-K across WGs (small-M decode) cl_kernel kernel_gemm_noshuffle_q1_0_f32; cl_kernel kernel_gemv_noshuffle_q1_0_f32; cl_kernel kernel_gemv_noshuffle_q4_k_f32; + cl_kernel kernel_gemv_noshuffle_q4_k_f32_o4; // 4-output-per-WI, long-vocab lm_head + cl_kernel kernel_gemv_noshuffle_q4_k_f32_tiled; // tiled-wide layout (opt-in) + cl_kernel kernel_gemv_noshuffle_q4_k_f32_splitk; // split-K across WGs (small-M decode) + cl_kernel kernel_gemv_splitk_reduce_f32; // sums split-K per-slice partials + cl_kernel kernel_gemv_noshuffle_q4_k_f32_glu; // fused gate+up GEMV + GLU (FFN) + cl_kernel kernel_convert_block_q4_k_tiled_ns; // tiled-wide convert (opt-in) + cl_kernel kernel_gemv_noshuffle_q4_k_f32_mc3; // multi-column (N=3) verify GEMV cl_kernel kernel_gemm_noshuffle_q4_k_f32; cl_kernel kernel_gemm_noshuffle_q4_k_q8_1_dp4a = nullptr; // dp4a (int8) dense prefill GEMM cl_kernel kernel_gemm_noshuffle_q4_k_q8_1_dp4a_wimg = nullptr; // dp4a dense prefill GEMM, weights via texture (X1 opt-in) cl_kernel kernel_gemm_noshuffle_q5_k_q8_1_dp4a = nullptr; // dp4a (int8) dense q5_K prefill GEMM cl_kernel kernel_gemm_noshuffle_q6_k_q8_1_dp4a = nullptr; // dp4a (int8) dense q6_K prefill GEMM cl_kernel kernel_quant_a_q8_1; // plain activation q8_1 pre-pass + cl_kernel kernel_gemm_noshuffle_q4_k_f32_r1; + cl_kernel kernel_gemm_noshuffle_q4_k_f32_kimg; + cl_kernel kernel_gemm_noshuffle_q4_k_f32_cok; cl_kernel kernel_gemv_noshuffle_q6_K_f32; + cl_kernel kernel_gemv_noshuffle_q6_K_f32_o4; + cl_kernel kernel_gemv_noshuffle_q6_K_f32_o4_global; // weights via __global (opt-in) + cl_kernel kernel_gemv_noshuffle_q6_K_f32_tiled; // tiled-wide layout (opt-in) + cl_kernel kernel_gemv_noshuffle_q6_K_f32_tiled_mc3; // tiled multi-column (N=3) verify lm_head + cl_kernel kernel_gemm_noshuffle_q6_K_f32_tiled; // batched (N>1) over the tiled layout + cl_kernel kernel_convert_block_q6_k_tiled_ns; // tiled-wide convert (opt-in) + cl_kernel kernel_gemv_noshuffle_q6_K_f32_mc3; // multi-column (N=3) verify GEMV cl_kernel kernel_gemm_noshuffle_q6_K_f32; + cl_kernel kernel_gemm_noshuffle_q6_K_f32_cok; cl_kernel kernel_gemv_noshuffle_q5_k_f32; + cl_kernel kernel_gemv_noshuffle_q5_k_f32_mc3; // multi-column (N=3) verify GEMV (spec/MTP) cl_kernel kernel_gemm_noshuffle_q5_k_f32; cl_kernel kernel_gemv_noshuffle_q5_0_f32; cl_kernel kernel_gemm_noshuffle_q5_0_f32; @@ -1452,10 +1487,16 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) { CL_CHECK((backend_ctx->kernel_restore_block_q5_1_trans4_ns = clCreateKernel(backend_ctx->program_cvt, "kernel_restore_block_q5_1_trans4_ns", &err), err)); CL_CHECK((backend_ctx->kernel_convert_block_q4_k_trans4_ns = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_q4_k_trans4_ns", &err), err)); CL_CHECK((backend_ctx->kernel_restore_block_q4_k_trans4_ns = clCreateKernel(backend_ctx->program_cvt, "kernel_restore_block_q4_k_trans4_ns", &err), err)); +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + CL_CHECK((backend_ctx->kernel_convert_block_q4_k_tiled_ns = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_q4_k_tiled_ns", &err), err)); +#endif CL_CHECK((backend_ctx->kernel_convert_block_q5_k_trans4_ns = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_q5_k_trans4_ns", &err), err)); CL_CHECK((backend_ctx->kernel_restore_block_q5_k_trans4_ns = clCreateKernel(backend_ctx->program_cvt, "kernel_restore_block_q5_k_trans4_ns", &err), err)); CL_CHECK((backend_ctx->kernel_convert_block_q6_k_trans4_ns = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_q6_k_trans4_ns", &err), err)); CL_CHECK((backend_ctx->kernel_restore_block_q6_k_trans4_ns = clCreateKernel(backend_ctx->program_cvt, "kernel_restore_block_q6_k_trans4_ns", &err), err)); +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + CL_CHECK((backend_ctx->kernel_convert_block_q6_k_tiled_ns = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_q6_k_tiled_ns", &err), err)); +#endif CL_CHECK((backend_ctx->kernel_convert_block_mxfp4 = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_mxfp4", &err), err)); CL_CHECK((backend_ctx->kernel_convert_block_mxfp4_trans = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_mxfp4_trans", &err), err)); CL_CHECK((backend_ctx->kernel_convert_block_mxfp4_trans4_ns = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_mxfp4_trans4_ns", &err), err)); @@ -2098,6 +2139,26 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) { GGML_LOG_CONT("."); } + // mul_mv_f16_f32_mrow (multi-row decode GEMV) + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "mul_mv_f16_f32_mrow.cl.h" + }; +#else + const std::string kernel_src = read_file("mul_mv_f16_f32_mrow.cl"); +#endif + backend_ctx->program_mul_mv_f16_f32_mrow = + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); + + CL_CHECK((backend_ctx->kernel_mul_mat_f16_f32_mrow = clCreateKernel(backend_ctx->program_mul_mv_f16_f32_mrow, "kernel_mul_mat_f16_f32_mrow", &err), err)); + CL_CHECK((backend_ctx->kernel_mul_mat_f16_f32_mrow_r2 = clCreateKernel(backend_ctx->program_mul_mv_f16_f32_mrow, "kernel_mul_mat_f16_f32_mrow_r2", &err), err)); + CL_CHECK((backend_ctx->kernel_mul_mat_f16_f32_mrow_r4 = clCreateKernel(backend_ctx->program_mul_mv_f16_f32_mrow, "kernel_mul_mat_f16_f32_mrow_r4", &err), err)); + CL_CHECK((backend_ctx->kernel_mul_mat_f16_f32_mrow_h8 = clCreateKernel(backend_ctx->program_mul_mv_f16_f32_mrow, "kernel_mul_mat_f16_f32_mrow_h8", &err), err)); + CL_CHECK((backend_ctx->kernel_mul_mat_f16_f32_mrow_h8r2 = clCreateKernel(backend_ctx->program_mul_mv_f16_f32_mrow, "kernel_mul_mat_f16_f32_mrow_h8r2", &err), err)); + GGML_LOG_CONT("."); + } + // mul_mv_f16_f32_l4 { #ifdef GGML_OPENCL_EMBED_KERNELS @@ -2251,6 +2312,7 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) { build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_mul_mm_f32_f32_l4_lm = clCreateKernel(backend_ctx->program_mul_mm_f32_f32_l4_lm, "kernel_mul_mm_f32_f32_l4_lm", &err), err)); + CL_CHECK((backend_ctx->kernel_gemv_f32_f32_mc = clCreateKernel(backend_ctx->program_mul_mm_f32_f32_l4_lm, "kernel_gemv_f32_f32_mc", &err), err)); GGML_LOG_CONT("."); } @@ -2520,6 +2582,7 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) { CL_CHECK((backend_ctx->kernel_rms_norm = clCreateKernel(backend_ctx->program_rms_norm, "kernel_rms_norm", &err), err)); CL_CHECK((backend_ctx->kernel_rms_norm_mul = clCreateKernel(backend_ctx->program_rms_norm, "kernel_rms_norm_mul", &err), err)); + CL_CHECK((backend_ctx->kernel_rms_norm_mul_add = clCreateKernel(backend_ctx->program_rms_norm, "kernel_rms_norm_mul_add", &err), err)); GGML_LOG_CONT("."); } @@ -3411,6 +3474,7 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) { cl_program prog = build_program_from_source(backend_ctx, kernel_src_CL_gemv_general.c_str(), CL_gemv_compile_opts); CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q4_0_f32 = clCreateKernel(prog, "kernel_gemv_noshuffle_q4_0_f32", &err), err)); + CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q4_0_f32_mc3 = clCreateKernel(prog, "kernel_gemv_noshuffle_q4_0_f32_mc3", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } @@ -3540,6 +3604,7 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) { cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), CL_gemv_compile_opts); CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q4_1_f32 = clCreateKernel(prog, "kernel_gemv_noshuffle_q4_1_f32", &err), err)); + CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q4_1_f32_mc3 = clCreateKernel(prog, "kernel_gemv_noshuffle_q4_1_f32_mc3", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } @@ -3754,6 +3819,7 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) { cl_program prog = build_program_from_source(backend_ctx, kernel_src_CL_gemv_general.c_str(), CL_gemv_compile_opts); CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q8_0_f32 = clCreateKernel(prog, "kernel_gemv_noshuffle_q8_0_f32", &err), err)); + CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q8_0_f32_splitk = clCreateKernel(prog, "kernel_gemv_noshuffle_q8_0_f32_splitk", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } @@ -3769,6 +3835,9 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) { #endif cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q4_k_f32 = clCreateKernel(prog, "kernel_gemm_noshuffle_q4_k_f32", &err), err)); + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q4_k_f32_r1 = clCreateKernel(prog, "kernel_gemm_noshuffle_q4_k_f32_r1", &err), err)); + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q4_k_f32_kimg = clCreateKernel(prog, "kernel_gemm_noshuffle_q4_k_f32_kimg", &err), err)); + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q4_k_f32_cok = clCreateKernel(prog, "kernel_gemm_noshuffle_q4_k_f32_cok", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } @@ -3863,6 +3932,18 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) { if (backend_ctx->has_vector_subgroup_broadcast) { CL_gemv_compile_opts += " -DVECTOR_SUB_GROUP_BROADCAST "; } + // Opt-in: dequant-once-per-block mc3 verify GEMV (factors q4_K dequant + // out of the 3-column loop; byte-identical, lower spill). A/B vs the + // shipped inline mc3 in the same binary. + if (getenv("GGML_OPENCL_Q4K_MC3_DQ")) { + CL_gemv_compile_opts += " -DQ4K_MC3_DEQUANT_ONCE "; + } + // Opt-in: LDS-staged dequant mc3 verify GEMV (stages the dequantized + // q4_K weights in __local instead of private regs that spill to slow + // global on Adreno; byte-identical). A/B vs inline + dequant-once. + if (getenv("GGML_OPENCL_Q4K_MC3_LDS")) { + CL_gemv_compile_opts += " -DQ4K_MC3_DEQUANT_LDS "; + } #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { @@ -3875,6 +3956,50 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) { cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), CL_gemv_compile_opts); CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q4_k_f32 = clCreateKernel(prog, "kernel_gemv_noshuffle_q4_k_f32", &err), err)); + CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q4_k_f32_mc3 = clCreateKernel(prog, "kernel_gemv_noshuffle_q4_k_f32_mc3", &err), err)); + CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q4_k_f32_splitk = clCreateKernel(prog, "kernel_gemv_noshuffle_q4_k_f32_splitk", &err), err)); + CL_CHECK((backend_ctx->kernel_gemv_splitk_reduce_f32 = clCreateKernel(prog, "kernel_gemv_splitk_reduce_f32", &err), err)); + CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q4_k_f32_glu = clCreateKernel(prog, "kernel_gemv_noshuffle_q4_k_f32_glu", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } + + // gemv_noshuffle_q4_k_f32_o4 — 4-output-per-WI variant for the long-vocab + // q4_K lm_head/embed GEMV (shares one activation read across 4 output rows). + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemv_noshuffle_q4_k_f32_o4.cl.h" + }; +#else + const std::string kernel_src = read_file("gemv_noshuffle_q4_k_f32_o4.cl"); +#endif + std::string CL_gemv_compile_opts = std::string("-cl-std=") + opencl_c_std + " -cl-mad-enable "; + if (backend_ctx->has_vector_subgroup_broadcast) { + CL_gemv_compile_opts += " -DVECTOR_SUB_GROUP_BROADCAST "; + } + cl_program prog = build_program_from_source( + backend_ctx, kernel_src.c_str(), CL_gemv_compile_opts); + CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q4_k_f32_o4 = clCreateKernel(prog, "kernel_gemv_noshuffle_q4_k_f32_o4", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } + + // gemv_noshuffle_q4_k_f32_tiled — tiled-wide canonical layout, default ON + // (opt out: GGML_OPENCL_Q4K_GEMV_TILED=0; separate convert + GEMV; weights via __global). + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemv_noshuffle_q4_k_f32_tiled.cl.h" + }; +#else + const std::string kernel_src = read_file("gemv_noshuffle_q4_k_f32_tiled.cl"); +#endif + std::string compile_opts = std::string("-cl-std=") + opencl_c_std + " -cl-mad-enable "; + cl_program prog = + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q4_k_f32_tiled = + clCreateKernel(prog, "kernel_gemv_noshuffle_q4_k_f32_tiled", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } @@ -4465,6 +4590,91 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) { build_program_from_source(backend_ctx, kernel_src.c_str(), CL_gemv_compile_opts); CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q6_K_f32 = clCreateKernel(prog, "kernel_gemv_noshuffle_q6_K_f32", &err), err)); + CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q6_K_f32_mc3 = clCreateKernel(prog, "kernel_gemv_noshuffle_q6_K_f32_mc3", &err), err)); + if (getenv("GGML_OPENCL_MC3_PROBE")) { + cl_ulong pm6 = 0, pm4 = 0; size_t wg6 = 0, wg4 = 0, mult = 0; + clGetKernelWorkGroupInfo(backend_ctx->kernel_gemv_noshuffle_q6_K_f32_mc3, backend_ctx->device, CL_KERNEL_PRIVATE_MEM_SIZE, sizeof(pm6), &pm6, NULL); + clGetKernelWorkGroupInfo(backend_ctx->kernel_gemv_noshuffle_q6_K_f32_mc3, backend_ctx->device, CL_KERNEL_WORK_GROUP_SIZE, sizeof(wg6), &wg6, NULL); + clGetKernelWorkGroupInfo(backend_ctx->kernel_gemv_noshuffle_q4_k_f32_mc3, backend_ctx->device, CL_KERNEL_PRIVATE_MEM_SIZE, sizeof(pm4), &pm4, NULL); + clGetKernelWorkGroupInfo(backend_ctx->kernel_gemv_noshuffle_q4_k_f32_mc3, backend_ctx->device, CL_KERNEL_WORK_GROUP_SIZE, sizeof(wg4), &wg4, NULL); + clGetKernelWorkGroupInfo(backend_ctx->kernel_gemv_noshuffle_q6_K_f32_mc3, backend_ctx->device, CL_KERNEL_PREFERRED_WORK_GROUP_SIZE_MULTIPLE, sizeof(mult), &mult, NULL); + fprintf(stderr, "[MC3-PROBE] q4K_mc3 private=%llu wg_cap=%zu | q6K_mc3 private=%llu wg_cap=%zu | pref_mult=%zu\n", + (unsigned long long)pm4, wg4, (unsigned long long)pm6, wg6, mult); + fflush(stderr); + } + GGML_LOG_CONT("."); + } + + // gemv_noshuffle_q6_k_f32_o4 — 4-output-per-WI variant, opt-in via + // GGML_OPENCL_Q6K_GEMV_O4=1 (~3x fewer dispatches on long-vocab lm_head). + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemv_noshuffle_q6_k_f32_o4.cl.h" + }; +#else + const std::string kernel_src = read_file("gemv_noshuffle_q6_k_f32_o4.cl"); +#endif + + std::string CL_gemv_compile_opts = std::string("-cl-std=") + opencl_c_std + + " -cl-mad-enable "; + if (backend_ctx->has_vector_subgroup_broadcast) { + CL_gemv_compile_opts += " -DVECTOR_SUB_GROUP_BROADCAT "; + } + + cl_program prog = + build_program_from_source(backend_ctx, kernel_src.c_str(), CL_gemv_compile_opts); + + CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q6_K_f32_o4 = clCreateKernel(prog, "kernel_gemv_noshuffle_q6_K_f32_o4", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + + // Global-read variant: weights read from __global coalesced instead of + // image1d_buffer (the texture cache caps the streaming lm_head read + // bandwidth). Opt-in via GGML_OPENCL_Q6K_GEMV_O4_GLOBAL. + cl_program prog_g = build_program_from_source(backend_ctx, kernel_src.c_str(), CL_gemv_compile_opts + " -DQ6K_O4_GLOBAL"); + CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q6_K_f32_o4_global = + clCreateKernel(prog_g, "kernel_gemv_noshuffle_q6_K_f32_o4_global", &err), err)); + CL_CHECK(clReleaseProgram(prog_g)); + GGML_LOG_CONT("."); + } + + // gemv_noshuffle_q6_k_f32_tiled — tiled-wide canonical layout, default ON + // (opt out: GGML_OPENCL_Q6K_GEMV_TILED=0; separate convert + GEMV; weights via __global). + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemv_noshuffle_q6_k_f32_tiled.cl.h" + }; +#else + const std::string kernel_src = read_file("gemv_noshuffle_q6_k_f32_tiled.cl"); +#endif + std::string compile_opts = std::string("-cl-std=") + opencl_c_std + " -cl-mad-enable "; + cl_program prog = + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q6_K_f32_tiled = + clCreateKernel(prog, "kernel_gemv_noshuffle_q6_K_f32_tiled", &err), err)); + CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q6_K_f32_tiled_mc3 = + clCreateKernel(prog, "kernel_gemv_noshuffle_q6_K_f32_tiled_mc3", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } + + // gemm_noshuffle_q6_k_f32_tiled — batched (N>1) GEMM over the same tiled-wide + // canonical layout, so batched lm_head/embed stays correct + on GPU. + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemm_noshuffle_q6_k_f32_tiled.cl.h" + }; +#else + const std::string kernel_src = read_file("gemm_noshuffle_q6_k_f32_tiled.cl"); +#endif + std::string compile_opts = std::string("-cl-std=") + opencl_c_std + " -cl-mad-enable "; + cl_program prog = + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q6_K_f32_tiled = + clCreateKernel(prog, "kernel_gemm_noshuffle_q6_K_f32_tiled", &err), err)); + CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } @@ -4481,6 +4691,7 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) { build_program_from_source(backend_ctx, kernel_src.c_str(), CL_moe_compile_opts); CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q6_K_f32 = clCreateKernel(prog, "kernel_gemm_noshuffle_q6_K_f32", &err), err)); + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q6_K_f32_cok = clCreateKernel(prog, "kernel_gemm_noshuffle_q6_K_f32_cok", &err), err)); GGML_LOG_CONT("."); } @@ -4503,6 +4714,7 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) { cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), CL_gemv_compile_opts); CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q5_k_f32 = clCreateKernel(prog, "kernel_gemv_noshuffle_q5_k_f32", &err), err)); + CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q5_k_f32_mc3 = clCreateKernel(prog, "kernel_gemv_noshuffle_q5_k_f32_mc3", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } @@ -6023,6 +6235,19 @@ static ggml_backend_opencl_context * ggml_cl_init(ggml_backend_dev_t dev) { #endif // GGML_OPENCL_USE_ADRENO_KERNELS backend_ctx->disable_fusion = getenv("GGML_OPENCL_DISABLE_FUSION") != nullptr; + if (const char * env = getenv("GGML_OPENCL_FUSE_MM_GLU")) { + backend_ctx->fuse_mm_glu = atoi(env) != 0; + } + if (const char * env = getenv("GGML_OPENCL_FUSE_RMS_ADD")) { + backend_ctx->fuse_rms_add = atoi(env) != 0; + } + if (const char * env = getenv("GGML_OPENCL_F16_MROW")) { + backend_ctx->f16_mrow = atoi(env) != 0; + } + if (const char * env = getenv("GGML_OPENCL_F16_MROW_RPT")) { + const int v = atoi(env); + backend_ctx->f16_mrow_rpt = (v == 2 || v == 4 || v == 8 || v == 16) ? v : 1; + } dev_ctx->backend_ctx = backend_ctx.release(); return dev_ctx->backend_ctx; @@ -6868,7 +7093,73 @@ static void ggml_cl_moe_combine_fused(ggml_backend_t backend, const ggml_tensor backend_ctx->enqueue_ndrange_kernel(kernel, 2, gws, lws, dst); } -static bool ggml_opencl_can_fuse(const struct ggml_cgraph * cgraph, int node_idx, std::initializer_list ops) { +inline bool use_q4k_tiled(const ggml_backend_opencl_context *backend_ctx, const ggml_tensor *tensor); // defined below (used by the GLU-subgraph fuse check) +inline bool use_adreno_kernels(const ggml_backend_opencl_context *backend_ctx, const ggml_tensor *tensor); // defined below + +static bool ggml_opencl_can_fuse(const ggml_backend_opencl_context * backend_ctx, const struct ggml_cgraph * cgraph, int node_idx, std::initializer_list ops) { + + // glu(mul_mat(Wg,x), mul_mat(Wu,x)) — the FFN gate/up GEMVs + GLU. This is a + // non-linear subgraph (up does NOT consume gate), so the contiguous + // ggml_can_fuse below rejects it; use ggml_can_fuse_subgraph with the glu as + // the sole output and validate the edges explicitly. q4_K decode only; + // byte-identical to the per-op path. + if (ops.size() == 3 && ops.begin()[0] == GGML_OP_MUL_MAT && + ops.begin()[1] == GGML_OP_MUL_MAT && ops.begin()[2] == GGML_OP_GLU) { + const enum ggml_op glu_ops[] = { GGML_OP_MUL_MAT, GGML_OP_MUL_MAT, GGML_OP_GLU }; + const int glu_out[] = { node_idx + 2 }; + if (!ggml_can_fuse_subgraph(cgraph, node_idx, 3, glu_ops, glu_out, 1)) { + return false; + } + + const ggml_tensor *gate = cgraph->nodes[node_idx]; + const ggml_tensor *up = cgraph->nodes[node_idx+1]; + const ggml_tensor *glu = cgraph->nodes[node_idx+2]; + + // decode GEMV path only (single token); prefill GEMM is separate + if (gate->ne[1] != 1 || up->ne[1] != 1) { + return false; + } + // both projections must be q4_K weights, f32 activation/output + if (gate->src[0]->type != GGML_TYPE_Q4_K || up->src[0]->type != GGML_TYPE_Q4_K || + gate->src[1]->type != GGML_TYPE_F32 || up->src[1]->type != GGML_TYPE_F32 || + gate->type != GGML_TYPE_F32 || up->type != GGML_TYPE_F32 || glu->type != GGML_TYPE_F32) { + return false; + } + // gate and up must share the same activation and have matching shape/stride + if (gate->src[1] != up->src[1] || + !ggml_are_same_shape(gate->src[0], up->src[0]) || + !ggml_are_same_stride(gate->src[0], up->src[0])) { + return false; + } + // GLU must read gate as src[0] and up as src[1], no swap (the fused + // epilogue applies the activation to gate, multiplies by up) + if (glu->src[0] != gate || glu->src[1] != up) { + return false; + } + if (ggml_get_op_params_i32(glu, 1) /* swapped */) { + return false; + } + // SWIGLU_OAI carries extra alpha/limit params -> not handled by the fused kernel + if (ggml_get_glu_op(glu) == GGML_GLU_OP_SWIGLU_OAI) { + return false; + } + // the fused kernel reads the standard noshuffle image layout; the tiled + // layout packs weights differently -> defer those to the per-op path + if (use_q4k_tiled(backend_ctx, gate->src[0]) || use_q4k_tiled(backend_ctx, up->src[0])) { + return false; + } + // that noshuffle layout is only produced at set_tensor time when + // use_adreno_kernels() accepts the weight (ne0 >= 512 && ne1 >= 512). + // Smaller weights stay in the plain q4_K layout, which this kernel would + // misread -> defer them to the per-op path. Real FFN gate/up weights are + // far above the threshold, so production dispatch is unchanged. + if (!use_adreno_kernels(backend_ctx, gate->src[0]) || + !use_adreno_kernels(backend_ctx, up->src[0])) { + return false; + } + return true; + } + if (!ggml_can_fuse(cgraph, node_idx, ops)) { return false; } @@ -6916,6 +7207,38 @@ static bool ggml_opencl_can_fuse(const struct ggml_cgraph * cgraph, int node_idx if (!ggml_is_contiguous(norm->src[0]) || !ggml_is_contiguous(w) || !ggml_is_contiguous(b)) { return false; } + } else if (ops.size() == 3 && ops.begin()[0] == GGML_OP_RMS_NORM && ops.begin()[1] == GGML_OP_MUL && ops.begin()[2] == GGML_OP_ADD) { + // rms_norm(x) * w + b, fused (residual). Mirrors the RMS_NORM+MUL gate + // plus the residual-add operand's constraints. + const ggml_tensor *rms_norm = cgraph->nodes[node_idx]; + const ggml_tensor *mul = cgraph->nodes[node_idx+1]; + const ggml_tensor *add = cgraph->nodes[node_idx+2]; + const ggml_tensor *w = mul->src[0] == rms_norm ? mul->src[1] : mul->src[0]; + const ggml_tensor *b = add->src[0] == mul ? add->src[1] : add->src[0]; + + GGML_ASSERT(rms_norm->src[0]->type == GGML_TYPE_F32); + GGML_ASSERT(rms_norm->type == GGML_TYPE_F32); + + if (w->type != GGML_TYPE_F32 || mul->type != GGML_TYPE_F32 || + b->type != GGML_TYPE_F32 || add->type != GGML_TYPE_F32) { + return false; + } + if (rms_norm->src[0]->ne[0] % 4 != 0) { + return false; + } + // if rms_norm is the B operand of mul, broadcast is not handled + if (rms_norm == mul->src[1] && !ggml_are_same_shape(mul->src[0], rms_norm)) { + return false; + } + // the residual must match the normed output shape (no add broadcast) + if (!ggml_are_same_shape(b, add)) { + return false; + } + // rms_norm assumes contiguous rows + if (!ggml_is_contiguous_rows(mul->src[0]) || !ggml_is_contiguous_rows(mul->src[1]) || + !ggml_is_contiguous_rows(b)) { + return false; + } } else if (ops.size() == 3 && ops.begin()[0] == GGML_OP_GROUP_NORM && ops.begin()[1] == GGML_OP_MUL && ops.begin()[2] == GGML_OP_ADD) { const ggml_tensor *gn = cgraph->nodes[node_idx]; const ggml_tensor *mul = cgraph->nodes[node_idx+1]; @@ -6939,6 +7262,216 @@ static void ggml_opencl_op_rms_norm_fused(ggml_backend_t backend, ggml_tensor * static void ggml_opencl_op_norm_fused(ggml_backend_t backend, ggml_tensor * norm_tensor, ggml_tensor * mul_tensor, ggml_tensor * add_tensor); static void ggml_opencl_op_group_norm_fused(ggml_backend_t backend, ggml_tensor * gn_tensor, ggml_tensor * mul_tensor, ggml_tensor * add_tensor); +static void ggml_cl_mul_mat_q4_k_glu_fused(ggml_backend_t backend, ggml_tensor * gate_tensor, ggml_tensor * up_tensor, ggml_tensor * glu_tensor) { +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + GGML_ASSERT(gate_tensor && up_tensor && glu_tensor); + + const ggml_tensor * Wg = gate_tensor->src[0]; + const ggml_tensor * Wu = up_tensor->src[0]; + const ggml_tensor * src1 = gate_tensor->src[1]; // == up_tensor->src[1] + const ggml_tensor * dst = glu_tensor; + + GGML_ASSERT(Wg && Wg->extra); + GGML_ASSERT(Wu && Wu->extra); + GGML_ASSERT(src1 && src1->extra); + GGML_ASSERT(dst && dst->extra); + + ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; + + ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; + ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; + ggml_tensor_extra_cl_q4_K * extra_g = (ggml_tensor_extra_cl_q4_K *)Wg->extra; + ggml_tensor_extra_cl_q4_K * extra_u = (ggml_tensor_extra_cl_q4_K *)Wu->extra; + + cl_ulong offset1 = extra1->offset + src1->view_offs; + cl_ulong offsetd = extrad->offset + dst->view_offs; + + const int K = Wg->ne[0]; // ne00 + const int M = Wg->ne[1]; // ne01 (= ffn intermediate width) + const int N = 1; // decode GEMV + + const cl_uchar mask_d6 = 0x3F, mask_d4 = 0x0F, mask_hi2 = 0xC0; + const int glu_op = (int)ggml_get_glu_op(dst); + + cl_context context = backend_ctx->context; + cl_int err; + cl_image_format img_fmt; + cl_image_desc img_desc; + cl_buffer_region region; + + // q images for the two weight matrices (standard noshuffle layout) + img_fmt = { CL_R, CL_UNSIGNED_INT32 }; + memset(&img_desc, 0, sizeof(img_desc)); + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = (size_t)M * K / 2 / 4; + img_desc.buffer = extra_g->q; + cl_mem qg_img = nullptr, qu_img = nullptr; + CL_CHECK((qg_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); + img_desc.buffer = extra_u->q; + CL_CHECK((qu_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); + + // shared activation image (one column at decode) + region.origin = offset1; + region.size = (size_t)K * N * sizeof(float); + cl_mem b_sub_buf = nullptr, b_img = nullptr; + CL_CHECK((b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + img_fmt = { CL_RGBA, CL_FLOAT }; + memset(&img_desc, 0, sizeof(img_desc)); + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = (size_t)K * N / 4; + img_desc.buffer = b_sub_buf; + CL_CHECK((b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); + + cl_kernel kernel = backend_ctx->kernel_gemv_noshuffle_q4_k_f32_glu; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &qg_img)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra_g->d)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra_g->dm)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra_g->s)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &qu_img)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &extra_u->d)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &extra_u->dm)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_mem), &extra_u->s)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_mem), &b_img)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_int), &K)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_int), &M)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_int), &glu_op)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_uchar), &mask_d6)); + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(cl_uchar), &mask_d4)); + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(cl_uchar), &mask_hi2)); + + // K-split = nsg_y subgroups. HARD-CAP at 8 (512 work-items): the fused + // kernel's cross-subgroup reduce uses a float4 reduceLM (gate+up packed) = + // 2x the LDS of the base GEMV's float2 reduce, so 16 co-resident subgroups + // exceed the per-CU LDS budget on X2 and the WG barrier DEADLOCKS -> GPU TDR + // (reproduced on upstream gemma-4 E4B decode, K=2560 M=10240). This used to + // be masked: get_kernel_workgroup_size reported 896 for this kernel (so the + // cap loop fell to 8), but it now returns 1024 and the Adreno per-kernel WG + // query is unreliable (over-reports), so cap explicitly instead of trusting + // it. nsg_y < 16 also means the cross-subgroup accumulation grouping differs + // from the standalone wide (nsg=16) GEMV, so the output is coherent but NOT + // byte-identical to the per-op path. Keep the maxwg query as a further floor + // for any driver that reports < 512. + size_t maxwg = backend_ctx->get_kernel_workgroup_size(kernel); + size_t nsg_y = 8; + while (nsg_y > 1 && 64 * nsg_y > maxwg) { nsg_y >>= 1; } + size_t local_work_size[3] = { 64, nsg_y, 1 }; + size_t global_work_size[3] = { (size_t)CEIL_DIV(M / 2, 64) * 64, nsg_y, 1 }; + + if (getenv("GGML_OPENCL_FUSE_DEBUG")) { + static int dbg = 0; + if (dbg < 3) { fprintf(stderr, "[FUSE_MM_GLU] fired #%d K=%d M=%d glu_op=%d nsg=%zu maxwg=%zu\n", ++dbg, K, M, glu_op, nsg_y, maxwg); fflush(stderr); } + } + + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + + CL_CHECK(clReleaseMemObject(qg_img)); + CL_CHECK(clReleaseMemObject(qu_img)); + CL_CHECK(clReleaseMemObject(b_img)); + CL_CHECK(clReleaseMemObject(b_sub_buf)); +#else + GGML_UNUSED(backend); + GGML_UNUSED(gate_tensor); + GGML_UNUSED(up_tensor); + GGML_UNUSED(glu_tensor); + GGML_ABORT("q4_K GLU fusion requires GGML_OPENCL_USE_ADRENO_KERNELS"); +#endif +} + + +static void ggml_opencl_op_rms_norm_mul_add_fused(ggml_backend_t backend, ggml_tensor * rms_norm_tensor, ggml_tensor * mul_tensor, ggml_tensor * add_tensor) { + GGML_ASSERT(rms_norm_tensor && mul_tensor && add_tensor); + + const ggml_tensor * src0 = rms_norm_tensor->src[0]; + const ggml_tensor * src1 = mul_tensor->src[0] == rms_norm_tensor ? mul_tensor->src[1] : mul_tensor->src[0]; + const ggml_tensor * src2 = add_tensor->src[0] == mul_tensor ? add_tensor->src[1] : add_tensor->src[0]; + const ggml_tensor * dst = add_tensor; + + GGML_ASSERT(src0 && src0->extra); + GGML_ASSERT(src1 && src1->extra); + GGML_ASSERT(src2 && src2->extra); + GGML_ASSERT(dst && dst->extra); + + ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; + ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; + ggml_tensor_extra_cl * extra2 = (ggml_tensor_extra_cl *)src2->extra; + ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; + + cl_ulong offset0 = extra0->offset + src0->view_offs; + cl_ulong offset1 = extra1->offset + src1->view_offs; + cl_ulong offset2 = extra2->offset + src2->view_offs; + cl_ulong offsetd = extrad->offset + dst->view_offs; + + ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; + + float eps; + memcpy(&eps, rms_norm_tensor->op_params, sizeof(float)); + + const int ne00 = src0->ne[0], ne01 = src0->ne[1], ne02 = src0->ne[2], ne03 = src0->ne[3]; + const cl_ulong nb01 = src0->nb[1], nb02 = src0->nb[2], nb03 = src0->nb[3]; + const int ne10 = src1->ne[0], ne11 = src1->ne[1], ne12 = src1->ne[2], ne13 = src1->ne[3]; + const cl_ulong nb11 = src1->nb[1], nb12 = src1->nb[2], nb13 = src1->nb[3]; + const int ne20 = src2->ne[0], ne21 = src2->ne[1], ne22 = src2->ne[2], ne23 = src2->ne[3]; + const cl_ulong nb21 = src2->nb[1], nb22 = src2->nb[2], nb23 = src2->nb[3]; + const cl_ulong nb1 = dst->nb[1], nb2 = dst->nb[2], nb3 = dst->nb[3]; + + GGML_ASSERT(ne00 % 4 == 0); + + size_t sgs; + if (backend_ctx->gpu_family == ADRENO) sgs = 64; + else if (backend_ctx->gpu_family == INTEL) sgs = 32; + else GGML_ASSERT(false && "Unsupported GPU"); + + cl_kernel kernel = backend_ctx->kernel_rms_norm_mul_add; + + int nth = sgs; + int max_workgroup_size = backend_ctx->get_kernel_workgroup_size(kernel); + while (nth < ne00 && nth < max_workgroup_size) nth *= 2; + nth = MIN(nth, max_workgroup_size); + nth = MIN(nth, ne00); + + size_t global_work_size[] = {(size_t)ne01*nth, (size_t)ne02, (size_t)ne03}; + size_t local_work_size[] = {(size_t)nth, 1, 1}; + + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra2->data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offset2)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne02)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne03)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb01)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &nb02)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong), &nb03)); + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne10)); + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &ne11)); + CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &ne13)); + CL_CHECK(clSetKernelArg(kernel, 19, sizeof(cl_ulong), &nb11)); + CL_CHECK(clSetKernelArg(kernel, 20, sizeof(cl_ulong), &nb12)); + CL_CHECK(clSetKernelArg(kernel, 21, sizeof(cl_ulong), &nb13)); + CL_CHECK(clSetKernelArg(kernel, 22, sizeof(int), &ne20)); + CL_CHECK(clSetKernelArg(kernel, 23, sizeof(int), &ne21)); + CL_CHECK(clSetKernelArg(kernel, 24, sizeof(int), &ne22)); + CL_CHECK(clSetKernelArg(kernel, 25, sizeof(int), &ne23)); + CL_CHECK(clSetKernelArg(kernel, 26, sizeof(cl_ulong), &nb21)); + CL_CHECK(clSetKernelArg(kernel, 27, sizeof(cl_ulong), &nb22)); + CL_CHECK(clSetKernelArg(kernel, 28, sizeof(cl_ulong), &nb23)); + CL_CHECK(clSetKernelArg(kernel, 29, sizeof(cl_ulong), &nb1)); + CL_CHECK(clSetKernelArg(kernel, 30, sizeof(cl_ulong), &nb2)); + CL_CHECK(clSetKernelArg(kernel, 31, sizeof(cl_ulong), &nb3)); + CL_CHECK(clSetKernelArg(kernel, 32, sizeof(float), &eps)); + CL_CHECK(clSetKernelArg(kernel, 33, sizeof(float)*sgs, NULL)); + + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); +} + static ggml_status ggml_backend_opencl_graph_compute(ggml_backend_t backend, ggml_cgraph * cgraph) { ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; @@ -6958,12 +7491,12 @@ static ggml_status ggml_backend_opencl_graph_compute(ggml_backend_t backend, ggm continue; } - if (!backend_ctx->disable_fusion && ggml_opencl_can_fuse(cgraph, i, { GGML_OP_NORM, GGML_OP_MUL, GGML_OP_ADD })) { + if (!backend_ctx->disable_fusion && ggml_opencl_can_fuse(backend_ctx, cgraph, i, { GGML_OP_NORM, GGML_OP_MUL, GGML_OP_ADD })) { ggml_opencl_op_norm_fused(backend, node, cgraph->nodes[i+1], cgraph->nodes[i+2]); i += 2; continue; } - if (!backend_ctx->disable_fusion && ggml_opencl_can_fuse(cgraph, i, { GGML_OP_GROUP_NORM, GGML_OP_MUL, GGML_OP_ADD })) { + if (!backend_ctx->disable_fusion && ggml_opencl_can_fuse(backend_ctx, cgraph, i, { GGML_OP_GROUP_NORM, GGML_OP_MUL, GGML_OP_ADD })) { ggml_opencl_op_group_norm_fused(backend, node, cgraph->nodes[i+1], cgraph->nodes[i+2]); i += 2; continue; @@ -6979,11 +7512,35 @@ static ggml_status ggml_backend_opencl_graph_compute(ggml_backend_t backend, ggm } } - if (!backend_ctx->disable_fusion && ggml_opencl_can_fuse(cgraph, i, { GGML_OP_RMS_NORM, GGML_OP_MUL })) { + // Fuse rms_norm + mul(weight) + add(residual). Checked before the + // rms_norm+mul fuse so the 3-op pattern wins over its 2-op prefix. + // Default on, opt-out GGML_OPENCL_FUSE_RMS_ADD=0. + if (!backend_ctx->disable_fusion && backend_ctx->fuse_rms_add && + ggml_opencl_can_fuse(backend_ctx, cgraph, i, { GGML_OP_RMS_NORM, GGML_OP_MUL, GGML_OP_ADD })) { + ggml_opencl_op_rms_norm_mul_add_fused(backend, node, cgraph->nodes[i+1], cgraph->nodes[i+2]); + i += 2; + continue; + } + if (!backend_ctx->disable_fusion && ggml_opencl_can_fuse(backend_ctx, cgraph, i, { GGML_OP_RMS_NORM, GGML_OP_MUL })) { ggml_opencl_op_rms_norm_fused(backend, node, cgraph->nodes[i+1]); i++; continue; } + // Fuse mul_mat(Wg,x) + mul_mat(Wu,x) + glu — fold the FFN's two decode + // GEMVs and the GLU into one dispatch. q4_K only (guarded below); the + // fused kernel uses the same accumulation/reduction order and the same + // scalar GLU formula -> coherent. Default on, opt-out GGML_OPENCL_FUSE_MM_GLU=0. +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + // The fused executor (ggml_cl_mul_mat_q4_k_glu_fused) is image-path / + // Adreno-only (GGML_ABORT on the non-Adreno #else); gate the dispatch to + // match so the FFN GLU subgraph stays dormant on Intel/other drivers. + if (backend_ctx->fuse_mm_glu && !backend_ctx->disable_fusion && + ggml_opencl_can_fuse(backend_ctx, cgraph, i, { GGML_OP_MUL_MAT, GGML_OP_MUL_MAT, GGML_OP_GLU })) { + ggml_cl_mul_mat_q4_k_glu_fused(backend, node, cgraph->nodes[i+1], cgraph->nodes[i+2]); + i += 2; + continue; + } +#endif bool ok = ggml_cl_compute_forward(backend, node); if (!ok) { @@ -7044,6 +7601,72 @@ inline bool use_adreno_moe_kernels(const ggml_backend_opencl_context *backend_ct return (((strstr(tensor->name, "ffn") != NULL) && (strstr(tensor->name, "exps") != NULL)) || (strstr(tensor->name, "as") != NULL)) && (ne01 % 32 == 0); } +// Device default for the tiled-wide lm_head/embed GEMV layout: ON for X2E and A8X. +// +// These kernels were previously off everywhere on the grounds that they compute +// wrong values at multi-superblock K. They do not: that NMSE ~2 came from the +// backend having no get_tensor restore path for the tiled layout, so +// test-backend-ops (which builds its CPU reference by copying the weights back +// out of the backend) compared a correct GPU result against a reference +// dequantized from tiled bytes. With the restore path added, MUL_MAT passes with +// the tiled kernels on, unmodified, on both devices. +// +// Perf, Qwen3-4B-Q4_K_M (q6_K lm_head 151936x2560), tg128, matched pairs with +// alternating lead, tiled vs o4: +// +// A8X +11.9% 6/6 pairs positive, order bias -0.06% (16.93 vs 15.14 tok/s) +// X2E +6.9% 4/4 pairs positive, order bias -0.03% (35.24 vs 32.87 tok/s) +// +// Measure this one on a COLD device. These kernels are far more clock-sensitive +// than the o4 route they replace: on a heat-soaked A8X (CPU cap at 1.5-1.9 GHz) +// tiled pins at ~14.2 tok/s while o4 still makes ~14.9, which reads as a 4-5% +// LOSS and inverts the ranking. The same box, after a reboot and a gate that +// waits for policy6 to return to 4396800, reports the +11.9% above with no +// order bias. A7X regresses hard on this layout and stays off. +// GGML_OPENCL_{Q4K,Q6K}_GEMV_TILED forces either way (=0 off, any other value on). +inline bool tiled_gemv_default_on(const ggml_backend_opencl_context *backend_ctx) { + return backend_ctx && (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E || + backend_ctx->adreno_gen == ADRENO_GPU_GEN::A8X); +} + +// Tiled-wide q6_K GEMV (default OFF; GGML_OPENCL_Q6K_GEMV_TILED forces either +// way: =0 off everywhere, any other value on everywhere). +// Both the convert (set_tensor) and the GEMV dispatch must agree on this so the +// buffer layout matches the kernel. +inline bool q6k_gemv_tiled_enabled(const ggml_backend_opencl_context *backend_ctx) { + static const char * e = std::getenv("GGML_OPENCL_Q6K_GEMV_TILED"); + if (e && e[0] != '\0') { + return e[0] != '0'; + } + return tiled_gemv_default_on(backend_ctx); +} + +// Only the long-vocab lm_head/embed shapes use the tiled layout; ne01 % 64 == 0 +// is required by the 64-row tiling (no row padding in the buffers). +// use_adreno_kernels is required: only the Adreno GEMV path can read the tiled +// layout, so converting a weight it would decline (e.g. ne00 < 512) leaves the +// generic kernel reading tiled bytes as plain SOA. +inline bool use_q6k_tiled(const ggml_backend_opencl_context *backend_ctx, const ggml_tensor *tensor) { + return q6k_gemv_tiled_enabled(backend_ctx) && tensor->type == GGML_TYPE_Q6_K && + tensor->ne[1] >= 32768 && tensor->ne[1] % 64 == 0 && + use_adreno_kernels(backend_ctx, tensor); +} + +// q4_K analog of the tiled-wide lm_head/embed GEMV (default OFF; +// GGML_OPENCL_Q4K_GEMV_TILED forces either way: =0 off, else on). Same gate. +inline bool q4k_gemv_tiled_enabled(const ggml_backend_opencl_context *backend_ctx) { + static const char * e = std::getenv("GGML_OPENCL_Q4K_GEMV_TILED"); + if (e && e[0] != '\0') { + return e[0] != '0'; + } + return tiled_gemv_default_on(backend_ctx); +} +inline bool use_q4k_tiled(const ggml_backend_opencl_context *backend_ctx, const ggml_tensor *tensor) { + return q4k_gemv_tiled_enabled(backend_ctx) && tensor->type == GGML_TYPE_Q4_K && + tensor->ne[1] >= 32768 && tensor->ne[1] % 64 == 0 && + use_adreno_kernels(backend_ctx, tensor); +} + inline bool enable_adreno_trans_weight(const ggml_backend_opencl_context *backend_ctx, const ggml_tensor *tensor) { bool adreno_kernel = use_adreno_kernels(backend_ctx, tensor); @@ -7058,18 +7681,53 @@ inline bool enable_adreno_trans_weight(const ggml_backend_opencl_context *backen return ((elem_num < 128 * 1024 * 1024) && adreno_kernel && shape_ok); // max element num: 2**27 } -static inline bool use_flat_gemv_for_large_m_q4_K(const ggml_tensor *tensor) { +// The flat-GEMV large-m escape is OPT-IN (GGML_OPENCL_FLAT_LARGE_M=1) because it +// is SLOWER than the route it replaces, not because it is unsafe. It was first +// parked on the theory that it out-of-bounds-writes at vocab-scale shapes; that +// was a misattribution (the test-backend-ops dst sentinel was tripped by the o4 +// GEMV's unguarded tail store, fixed separately - and at the shape it was blamed +// for, k=1536, this predicate returns false anyway, so the flat route never ran). +// +// The escape's original rationale, "gemv_noshuffle perf drops for large M", +// predates the o4 kernel, which now covers the same long-vocab shapes and beats +// this route on every device measured (Qwen3-4B-Q4_K_M, q6_K lm_head +// 151936x2560, tg128, matched pairs vs o4): A8X -10.3% (0/3 pairs), X2E -3.7% +// (0/3). Keep it reachable for shapes o4 declines, but do not default it on. +static inline bool flat_large_m_enabled() { + static const char * e = getenv("GGML_OPENCL_FLAT_LARGE_M"); + static const bool en = e != nullptr && atoi(e) != 0; + return en; +} + +static inline bool use_flat_gemv_for_large_m_q4_K(const ggml_backend_opencl_context *backend_ctx, const ggml_tensor *tensor) { + if (!flat_large_m_enabled()) { + return false; + } // gemv_noshuffle variant perf drops for large M, use flat variant for large M. // threshold is well above typical hidden/FFN dims, but below typical vocab sizes. // note that this forces large M weights to use LM GEMM. - return tensor->ne[1] >= 32768 && tensor->ne[2] == 1 && tensor->ne[3] == 1; + // EXCEPT when this branch's tiled-canonical lm_head/embed layout is active: the + // weight is converted to the 64-row tiled layout, which the flat gemv would + // misread as garbage. use_q4k_tiled owns these large-M weights, so defer to it. + return tensor->ne[1] >= 32768 && tensor->ne[2] == 1 && tensor->ne[3] == 1 + && !use_q4k_tiled(backend_ctx, tensor); } static inline bool use_flat_gemv_for_large_m_q6_K(const ggml_backend_opencl_context *backend_ctx, const ggml_tensor *tensor) { + // NOTE on ordering: the ne01 % 128 escape below is a CORRECTNESS guard, not a + // performance one, so it must be reachable regardless of flat_large_m_enabled(). + // The opt-in gate therefore sits after it, and after the tiled deferral. // gemv_noshuffle variant perf drops for large M, use flat variant for large M. // threshold is well above typical hidden/FFN dims, but below typical vocab sizes. // q6_K flat gemv is worse for smaller K; 2048 seems to be a reasonable threshold. // note that this forces large M weights to use LM GEMM. + // When this branch's tiled-canonical lm_head/embed layout is active, the weight is + // converted to the 64-row tiled layout, which the flat gemv would misread as + // garbage. use_q6k_tiled owns these large-M weights (it requires ne01 % 64 == 0, + // so it never claims an odd-vocab weight), so defer to it first. + if (use_q6k_tiled(backend_ctx, tensor)) { + return false; + } // The noshuffle (transposed-weight) layout packs 2 rows per 32-bit texel and the // gemv reads it with a ne01/2 texel stride and an exact-cover dispatch of // ceil(ne01/2 / 64)*64 work-items with no store guard; the gemm uses 4-row tiles. @@ -7084,6 +7742,10 @@ static inline bool use_flat_gemv_for_large_m_q6_K(const ggml_backend_opencl_cont return true; } + if (!flat_large_m_enabled()) { + return false; + } + // The gemv_noshuffle slowdown tracks TOTAL weight size, not ne0 alone; ne0 >= 2048 is a // proxy for "large weight" that misses a narrow-hidden vocab-scale lm_head. // Add a direct size escape so such weights also take the flat path, without changing @@ -7299,8 +7961,45 @@ static bool ggml_opencl_supports_op(ggml_backend_dev_t dev, const struct ggml_te op->src[0]->type == GGML_TYPE_Q4_K || op->src[0]->type == GGML_TYPE_Q5_K || op->src[0]->type == GGML_TYPE_Q6_K) { + // The generic mul_mv (GEMV) kernels CORRUPT large-batch prefill + // (N=src1->ne[1] >= 512) on Adreno. Quant mul_mats only avoid the + // GEMV when they hit the Adreno trans-weight GEMM, which requires + // a GEMM kernel for the type AND use_adreno_kernels(weight) (weight + // dims >= 512). Q5_K DOES have a trans-weight GEMM (PR #21595) and + // is correct at large N (720-token-prompt greedy generation matches + // the CPU reference, modulo expected FP drift in the tail); it only + // takes the buggy GEMV for small/odd-shaped projections that fail + // use_adreno_kernels. Reject any large-N quant mul_mat that would + // fall to the buggy GEMV. + { + const ggml_type t = op->src[0]->type; + const bool type_has_gemm = (t == GGML_TYPE_Q4_0 || t == GGML_TYPE_Q4_1 || + t == GGML_TYPE_IQ4_NL || t == GGML_TYPE_Q8_0 || + t == GGML_TYPE_Q4_K || t == GGML_TYPE_Q5_K || + t == GGML_TYPE_Q6_K); + const bool uses_gemm = type_has_gemm && use_adreno_kernels(backend_ctx, op->src[0]); + if (!uses_gemm && op->src[1]->ne[1] >= 512) { + return false; + } + + // Batched large-vocab Q6_K lm_head/embed (ne1>1) now runs correctly on + // GPU. The historical garbage was a weight-LAYOUT mismatch (not a + // data-dependent/compiler bug): the default q6k-tiled convert stores the + // weight in the 64-row-tiled canonical layout, but the batched path used + // kernel_gemm_noshuffle_q6_K_f32 which reads the plain M-transposed + // layout. Fixed by dispatching the tiled weight to + // kernel_gemm_noshuffle_q6_K_f32_tiled (matches the decode tiled GEMV + // layout); the plain noshuffle GEMM still handles the non-tiled convert + // (GGML_OPENCL_Q6K_GEMV_TILED=0). GPU PPL == CPU in both convert modes. + // No CPU fallback needed. + } return op->src[1]->type == GGML_TYPE_F32 && ggml_is_contiguous(op->src[0]) && ggml_is_contiguous(op->src[1]); } else if (op->src[0]->type == GGML_TYPE_Q8_0) { + // ggml_cl_mul_mat_q8_0_f32_adreno now honors src1/dst view_offs (the + // activation sub-buffer starts at offset1 and the kernels take offsetd), + // so a broadcast q8_0 matmul (src1 batch > src0 batch, e.g. Qwen3.5-9B-UD + // / Qwen3.6-35B q8_0 GDN ssm_out) runs on GPU via the per-slice broadcast + // iteration in ggml_cl_mul_mat. No special-casing needed. return op->src[1]->type == GGML_TYPE_F32; } return false; @@ -9052,8 +9751,41 @@ static void ggml_backend_opencl_buffer_set_tensor(ggml_backend_buffer_t buffer, #endif // GGML_OPENCL_USE_ADRENO_KERNELS #ifdef GGML_OPENCL_USE_ADRENO_KERNELS + // Tiled-wide convert for the long-vocab lm_head/embed (opt-in). The embed/ + // output q4_K weight (token_embd.weight, ne1=vocab) is NOT matched by + // use_adreno_moe_kernels, so it lands here in the general branch. Produce + // the final 64-row-tiled canonical layout directly into q/d/dm/s (buffer + // sizes already match), read back by kernel_gemv_noshuffle_q4_k_f32_tiled. + if (use_q4k_tiled(backend_ctx, tensor)) { + cl_kernel tk = backend_ctx->kernel_convert_block_q4_k_tiled_ns; + + int ne00 = tensor->ne[0]; + int ne01 = tensor->ne[1]; + int ne02 = tensor->ne[2]; + + CL_CHECK(clSetKernelArg(tk, 0, sizeof(cl_mem), &data_device)); + CL_CHECK(clSetKernelArg(tk, 1, sizeof(cl_mem), &extra->q)); + CL_CHECK(clSetKernelArg(tk, 2, sizeof(cl_mem), &extra->d)); + CL_CHECK(clSetKernelArg(tk, 3, sizeof(cl_mem), &extra->dm)); + CL_CHECK(clSetKernelArg(tk, 4, sizeof(cl_mem), &extra->s)); + CL_CHECK(clSetKernelArg(tk, 5, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(tk, 6, sizeof(int), &ne01)); + + size_t gws[] = {static_cast(((ne01 + 63) / 64) * 64), static_cast(ne00 / 256), static_cast(ne02)}; + size_t lws[] = {64, 1, 1}; + + cl_event tevt; + CL_CHECK(clEnqueueNDRangeKernel(queue, tk, 3, NULL, gws, lws, 0, NULL, &tevt)); + CL_CHECK(clWaitForEvents(1, &tevt)); + CL_CHECK(clReleaseMemObject(data_device)); + + extra->q_img = nullptr; + tensor->extra = extra; + return; + } + cl_kernel kernel = backend_ctx->kernel_convert_block_q4_K; - if (use_adreno_kernels(backend_ctx, tensor) && !use_flat_gemv_for_large_m_q4_K(tensor)) { + if (use_adreno_kernels(backend_ctx, tensor) && !use_flat_gemv_for_large_m_q4_K(backend_ctx, tensor)) { kernel = backend_ctx->kernel_convert_block_q4_K_noshuffle; } #else @@ -9081,7 +9813,7 @@ static void ggml_backend_opencl_buffer_set_tensor(ggml_backend_buffer_t buffer, tensor->extra = extra; #ifdef GGML_OPENCL_USE_ADRENO_KERNELS - if (use_adreno_kernels(backend_ctx, tensor) && !use_flat_gemv_for_large_m_q4_K(tensor)) { + if (use_adreno_kernels(backend_ctx, tensor) && !use_flat_gemv_for_large_m_q4_K(backend_ctx, tensor)) { int M = tensor->ne[1]; int K = tensor->ne[0]; @@ -9407,6 +10139,45 @@ static void ggml_backend_opencl_buffer_set_tensor(ggml_backend_buffer_t buffer, CL_CHECK((extra->d = clCreateSubBuffer(extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); previous_origin = region.origin; +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + // Tiled-wide convert for the long-vocab lm_head/embed (opt-in). The embed + // /output q6_K weight (e.g. token_embd.weight, ne1=vocab) is NOT matched by + // use_adreno_moe_kernels, so it lands here in the general branch. Produce + // the final 64-row-tiled canonical layout directly into ql/qh/s/d (buffer + // sizes already match), read back by kernel_gemv_noshuffle_q6_K_f32_tiled. + // Bypasses the plain-SOA convert + per-array transpose below. + if (use_q6k_tiled(backend_ctx, tensor)) { + cl_kernel kernel = backend_ctx->kernel_convert_block_q6_k_tiled_ns; + + int ne00 = tensor->ne[0]; + int ne01 = tensor->ne[1]; + int ne02 = tensor->ne[2]; + + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->ql)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->qh)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra->d)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra->s)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne01)); + + size_t global_work_size[] = {static_cast(((ne01 + 63) / 64) * 64), static_cast(ne00 / 256), static_cast(ne02)}; + size_t local_work_size[] = {64, 1, 1}; + + cl_event evt; + CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt)); + CL_CHECK(clWaitForEvents(1, &evt)); + CL_CHECK(clReleaseMemObject(data_device)); + + extra->size_ql = size_ql; + extra->size_qh = size_qh; + extra->size_s = size_s; + extra->size_d = size_d; + tensor->extra = extra; + return; + } +#endif // GGML_OPENCL_USE_ADRENO_KERNELS + // Flatten the weights cl_kernel kernel; #ifdef GGML_OPENCL_USE_ADRENO_KERNELS @@ -10222,6 +10993,54 @@ static void ggml_backend_opencl_buffer_get_tensor(ggml_backend_buffer_t buffer, cl_uchar mask_F0 = 0xF0; #ifdef GGML_OPENCL_USE_ADRENO_KERNELS + // Undo the 64-row-tiled canonical pack (kernel_convert_block_q4_k_tiled_ns). + // Without this, a read-back of a tiled weight returns the tiled bytes + // reinterpreted as block_q4_K -- which is how test-backend-ops builds its + // CPU reference (ggml_backend_graph_copy -> tensor_get), so the tiled path + // "failed" the suite while computing the correct product. + if (use_q4k_tiled(backend_ctx, tensor)) { + const int ne00v = tensor->ne[0]; + const int ne01v = tensor->ne[1]; + const int nbv = ne00v / 256; + const size_t n_blk = (size_t)nbv * ne01v; + + std::vector tq(n_blk*32); + std::vector td(n_blk), tdm(n_blk); + std::vector ts(n_blk*12); + CL_CHECK(clEnqueueReadBuffer(queue, extra->q, CL_TRUE, 0, tq.size()*4, tq.data(), 0, NULL, NULL)); + CL_CHECK(clEnqueueReadBuffer(queue, extra->d, CL_TRUE, 0, td.size()*2, td.data(), 0, NULL, NULL)); + CL_CHECK(clEnqueueReadBuffer(queue, extra->dm, CL_TRUE, 0, tdm.size()*2, tdm.data(), 0, NULL, NULL)); + CL_CHECK(clEnqueueReadBuffer(queue, extra->s, CL_TRUE, 0, ts.size(), ts.data(), 0, NULL, NULL)); + + std::vector rebuilt(ggml_nbytes(tensor), 0); + for (int i01 = 0; i01 < ne01v; ++i01) { + const int rt = i01/64, rit = i01%64; + for (int i00 = 0; i00 < nbv; ++i00) { + uint8_t * b = rebuilt.data() + ((size_t)i00 + (size_t)i01*nbv)*144; + const int tb = rt*nbv + i00; + const size_t si = (size_t)tb*64 + rit; + + memcpy(b + 0, &td [si], 2); + memcpy(b + 2, &tdm[si], 2); + memcpy(b + 4, &ts[si*12], 12); + + uint32_t qw[32]; + for (int gr = 0; gr < 8; ++gr) { + const size_t base = ((size_t)tb*8 + gr)*64 + rit; + for (int j = 0; j < 4; ++j) qw[gr*4 + j] = tq[base*4 + j]; + } + uint8_t * q = b + 16; + for (int e = 0; e < 256; ++e) { + const int g = e>>6, w = e&63, h = w>>5, l = w&31; + const uint32_t code = (qw[e>>3] >> ((e&7)*4)) & 0xF; + q[g*32 + l] |= (uint8_t)(h ? (code << 4) : code); + } + } + } + memcpy(data, rebuilt.data() + offset, size); + CL_CHECK(clReleaseMemObject(data_device)); + return; + } if (use_adreno_moe_kernels(backend_ctx, tensor)) { cl_int err; cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_WRITE, @@ -10256,7 +11075,7 @@ static void ggml_backend_opencl_buffer_get_tensor(ggml_backend_buffer_t buffer, CL_CHECK(clReleaseMemObject(data_device)); return; } - if (use_adreno_kernels(backend_ctx, tensor) && !use_flat_gemv_for_large_m_q4_K(tensor)) { + if (use_adreno_kernels(backend_ctx, tensor) && !use_flat_gemv_for_large_m_q4_K(backend_ctx, tensor)) { int M = tensor->ne[1]; int K = tensor->ne[0]; @@ -10444,6 +11263,61 @@ static void ggml_backend_opencl_buffer_get_tensor(ggml_backend_buffer_t buffer, ggml_tensor_extra_cl_q6_K * extra = (ggml_tensor_extra_cl_q6_K *)tensor->extra; #ifdef GGML_OPENCL_USE_ADRENO_KERNELS + // Undo the 64-row-tiled canonical pack (kernel_convert_block_q6_k_tiled_ns). + // See the q4_K tiled restore above for why a read-back path is required. + if (use_q6k_tiled(backend_ctx, tensor)) { + const int ne00v = tensor->ne[0]; + const int ne01v = tensor->ne[1]; + const int nbv = ne00v / 256; + const size_t n_blk = (size_t)nbv * ne01v; + + std::vector tql(n_blk*32), tqh(n_blk*16); + std::vector ts(n_blk*16); + std::vector td(n_blk); + CL_CHECK(clEnqueueReadBuffer(queue, extra->ql, CL_TRUE, 0, tql.size()*4, tql.data(), 0, NULL, NULL)); + CL_CHECK(clEnqueueReadBuffer(queue, extra->qh, CL_TRUE, 0, tqh.size()*4, tqh.data(), 0, NULL, NULL)); + CL_CHECK(clEnqueueReadBuffer(queue, extra->s, CL_TRUE, 0, ts.size(), ts.data(), 0, NULL, NULL)); + CL_CHECK(clEnqueueReadBuffer(queue, extra->d, CL_TRUE, 0, td.size()*2, td.data(), 0, NULL, NULL)); + + std::vector rebuilt(ggml_nbytes(tensor), 0); + for (int i01 = 0; i01 < ne01v; ++i01) { + const int rt = i01/64, rit = i01%64; + for (int i00 = 0; i00 < nbv; ++i00) { + uint8_t * b = rebuilt.data() + ((size_t)i00 + (size_t)i01*nbv)*210; + const int tb = rt*nbv + i00; + const size_t si = (size_t)tb*64 + rit; + + uint32_t qlw[32], qhw[16]; + for (int g = 0; g < 8; ++g) { + const size_t base = ((size_t)tb*8 + g)*64 + rit; + for (int j = 0; j < 4; ++j) qlw[g*4 + j] = tql[base*4 + j]; + } + for (int g = 0; g < 4; ++g) { + const size_t base = ((size_t)tb*4 + g)*64 + rit; + for (int j = 0; j < 4; ++j) qhw[g*4 + j] = tqh[base*4 + j]; + } + + uint8_t * ql = b; + uint8_t * qh = b + 128; + for (int e = 0; e < 256; ++e) { + const int n = (e >= 128) ? 1 : 0; + const int within = e - n*128, q = within/32, l = within%32; + const int off_ql = n*64, off_qh = n*32; + const uint8_t low4 = (qlw[e>>3] >> ((e&7)*4)) & 0xF; + const uint8_t hi2 = (qhw[e>>4] >> ((e&15)*2)) & 0x3; + if (q == 0) ql[off_ql + l] |= low4; + else if (q == 1) ql[off_ql + l + 32] |= low4; + else if (q == 2) ql[off_ql + l] |= (uint8_t)(low4 << 4); + else ql[off_ql + l + 32] |= (uint8_t)(low4 << 4); + qh[off_qh + l] |= (uint8_t)(hi2 << (q*2)); + } + memcpy(b + 192, &ts[si*16], 16); + memcpy(b + 208, &td[si], 2); + } + } + memcpy(data, rebuilt.data() + offset, size); + return; + } if (use_adreno_moe_kernels(backend_ctx, tensor)) { cl_int err; cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_WRITE, @@ -16010,7 +16884,17 @@ static void ggml_cl_mul_mat_q4_0_f32_adreno(ggml_backend_t backend, const ggml_t int N = ne1; int K = ne00; - if (ne1 == 1) { + // Multi-column (N=3) verify GEMV for q4_0: route the spec/MTP verify batch + // (ne1==3) onto the efficient GEMV path instead of the transposed-GEMM dead- + // zone (gemm_noshuffle_q4_0 is ~50% of MTP decode on a Q4_0 model since q4_0 + // weights have no cok/mc3, unlike q4_K). Reuses the ne1==1 GEMV image setup + // (activation image already sized by N=ne1). Byte-identical. Opt-in via + // GGML_OPENCL_Q40_MC3=1. Per-layer only (ne01 < 32768); q4_0 lm_head doesn't + // occur (token_embd/output stay Q6_K), guard kept for parity with q4_K mc3. + static const bool q40_mc3 = (getenv("GGML_OPENCL_Q40_MC3") != nullptr); + const bool use_q40_mc3 = q40_mc3 && (ne1 >= 2 && ne1 <= 4) && (ne01 < 32768); + + if (ne1 == 1 || use_q40_mc3) { cl_mem q_img = nullptr; cl_mem b_sub_buf = nullptr; cl_mem b_img = nullptr; @@ -16036,38 +16920,56 @@ static void ggml_cl_mul_mat_q4_0_f32_adreno(ggml_backend_t backend, const ggml_t img_desc.buffer = b_sub_buf; CL_CHECK((b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); - kernel = backend_ctx->kernel_gemv_noshuffle_q4_0_f32; - if (M == 4096 && K == 4096) { - kernel = backend_ctx->kernel_gemv_noshuffle_q4_0_f32_4096_1_4096; - } else if (M == 4096 && K == 11008) { - kernel = backend_ctx->kernel_gemv_noshuffle_q4_0_f32_4096_1_11008; - } else if (M == 11008 && K == 4096) { - kernel = backend_ctx->kernel_gemv_noshuffle_q4_0_f32_11008_1_4096; - } else if (M == 32000 && K == 4096) { - kernel = backend_ctx->kernel_gemv_noshuffle_q4_0_f32_32000_1_4096; - } + if (use_q40_mc3) { + kernel = backend_ctx->kernel_gemv_noshuffle_q4_0_f32_mc3; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &q_img)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q4_0->d)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &b_img)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne1)); + } else { + kernel = backend_ctx->kernel_gemv_noshuffle_q4_0_f32; + if (M == 4096 && K == 4096) { + kernel = backend_ctx->kernel_gemv_noshuffle_q4_0_f32_4096_1_4096; + } else if (M == 4096 && K == 11008) { + kernel = backend_ctx->kernel_gemv_noshuffle_q4_0_f32_4096_1_11008; + } else if (M == 11008 && K == 4096) { + kernel = backend_ctx->kernel_gemv_noshuffle_q4_0_f32_11008_1_4096; + } else if (M == 32000 && K == 4096) { + kernel = backend_ctx->kernel_gemv_noshuffle_q4_0_f32_32000_1_4096; + } - int r2 = 1; - int r3 = 1; + int r2 = 1; + int r3 = 1; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &q_img)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q4_0->d)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &b_img)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne02)); - CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne10)); - CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne12)); - CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne0)); - CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne1)); - CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &r2)); - CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &r3)); + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &q_img)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q4_0->d)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &b_img)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne02)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne10)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne0)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne1)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &r2)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &r3)); + } - size_t local_work_size[3] = {64, 4, 1}; - size_t global_work_size[3] = {(size_t)CEIL_DIV(ne01/2, 64)*64, 4, 1}; + // Small-M mc3 verify is occupancy/latency-bound (too few WGs at small M, so + // its bandwidth falls well short of the FFN matmuls'). Use 8 subgroups (512-WI WGs, half the + // per-lane K-walk) for small M. Layout stride is fixed (4 uints/block), so only + // the K-split count changes; the mc3 kernel reads it via get_local_size(1). The + // ne1==1 base kernel hardcodes N_SIMDGROUP=4, so it always stays at 4. + const int mc3_nsg = (use_q40_mc3 && ne01 < 4096) ? 8 : 4; + size_t local_work_size[3] = {64, (size_t)mc3_nsg, 1}; + size_t global_work_size[3] = {(size_t)CEIL_DIV(ne01/2, 64)*64, (size_t)mc3_nsg, 1}; backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); @@ -16285,7 +17187,14 @@ static void ggml_cl_mul_mat_q4_1_f32_adreno(ggml_backend_t backend, const ggml_t int N = ne1; int K = ne00; - if (ne1 == 1) { + // Multi-column (N=3) verify GEMV for q4_1: route the spec/MTP verify batch + // (ne1==3) onto the efficient GEMV path instead of the transposed-GEMM dead- + // zone (gemm_noshuffle_q4_1). Reuses the ne1==1 GEMV image setup. Opt-in via + // GGML_OPENCL_Q41_MC3=1. Per-layer only (ne01 < 32768). + static const bool q41_mc3 = (getenv("GGML_OPENCL_Q41_MC3") != nullptr); + const bool use_q41_mc3 = q41_mc3 && (ne1 >= 2 && ne1 <= 4) && (ne01 < 32768); + + if (ne1 == 1 || use_q41_mc3) { cl_mem q_img = nullptr; cl_mem b_sub_buf = nullptr; cl_mem b_img = nullptr; @@ -16311,7 +17220,8 @@ static void ggml_cl_mul_mat_q4_1_f32_adreno(ggml_backend_t backend, const ggml_t img_desc.buffer = b_sub_buf; CL_CHECK((b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); - kernel = backend_ctx->kernel_gemv_noshuffle_q4_1_f32; + kernel = use_q41_mc3 ? backend_ctx->kernel_gemv_noshuffle_q4_1_f32_mc3 + : backend_ctx->kernel_gemv_noshuffle_q4_1_f32; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &q_img)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q4_1->d)); @@ -16321,6 +17231,9 @@ static void ggml_cl_mul_mat_q4_1_f32_adreno(ggml_backend_t backend, const ggml_t CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_int), &ne00)); CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_int), &ne01)); + if (use_q41_mc3) { + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_int), &ne1)); // n_cols + } size_t local_work_size[3] = {64, 4, 1}; size_t global_work_size[3] = {(size_t)CEIL_DIV(ne01/2, 64)*64, 4, 1}; @@ -17164,6 +18077,66 @@ static void ggml_cl_mul_mat_q8_0_f32_adreno(ggml_backend_t backend, const ggml_t img_desc.buffer = b_sub_buf; CL_CHECK((b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); + // Split-K for small-M decode GEMVs. The base kernel puts one output row + // per lane and splits K only inside one workgroup, so M is the sole source + // of workgroup parallelism: gpt-oss's K and V projections are M=512 = 8 + // workgroups on a 16-CU X2, and the kernel measures 48 GB/s where the + // M=2880/4096 projections in the same decode graph reach 122-123. Mirrors + // the q4_0/q4_K split-K above and reuses their reduce kernel. + // + // Enabled where it is measured to win, like the q4_K gate: X2-90 +2.8% + // tg32 @d4096 on gpt-oss; Adreno 840 (12 CU) NEUTRAL on Llama-3.2-3B-Q8_0 + // (0.0% @d4096 -- its K/V proj is M=1024 = 16 workgroups, which already + // fills 12 CUs). Unmeasured on X1E/A7X/A6X and the q4_K split-K measured + // -0.7% on X1E, so the default is not widened on absence of evidence. + static const bool q8_splitk_env_set = []{ + const char * e = std::getenv("GGML_OPENCL_Q8_GEMV_SPLITK"); + return e && e[0] != '\0'; + }(); + static const bool q8_splitk_env_on = []{ + const char * e = std::getenv("GGML_OPENCL_Q8_GEMV_SPLITK"); + return !(e && e[0] == '0'); + }(); + const bool q8_splitk_on = q8_splitk_env_set + ? q8_splitk_env_on + : (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E); + if (q8_splitk_on && backend_ctx->kernel_gemv_noshuffle_q8_0_f32_splitk && + ne01 <= 1024 && ne01 % 64 == 0) { + const int nsg = 8; + const int ksplit = 8; // -> 8 * M/64 workgroups + const size_t gx = (size_t) CEIL_DIV(ne01, 64) * 64; + + backend_ctx->prealloc_splitk_partial.allocate( + backend_ctx->context, (size_t) ksplit * ne01 * sizeof(float)); + cl_mem partial = backend_ctx->prealloc_splitk_partial.buffer; + + cl_kernel ks = backend_ctx->kernel_gemv_noshuffle_q8_0_f32_splitk; + CL_CHECK(clSetKernelArg(ks, 0, sizeof(cl_mem), &q_img)); + CL_CHECK(clSetKernelArg(ks, 1, sizeof(cl_mem), &extra0_q8_0->d)); + CL_CHECK(clSetKernelArg(ks, 2, sizeof(cl_mem), &b_img)); + CL_CHECK(clSetKernelArg(ks, 3, sizeof(cl_mem), &partial)); + CL_CHECK(clSetKernelArg(ks, 4, sizeof(cl_int), &ne00)); + CL_CHECK(clSetKernelArg(ks, 5, sizeof(cl_int), &ne01)); + size_t lsk[3] = { 64, (size_t) nsg, 1 }; + size_t gsk[3] = { gx, (size_t) (nsg * ksplit), 1 }; + backend_ctx->enqueue_ndrange_kernel(ks, 3, gsk, lsk, dst); + + cl_kernel kr = backend_ctx->kernel_gemv_splitk_reduce_f32; + CL_CHECK(clSetKernelArg(kr, 0, sizeof(cl_mem), &partial)); + CL_CHECK(clSetKernelArg(kr, 1, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kr, 2, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kr, 3, sizeof(cl_int), &ne01)); + CL_CHECK(clSetKernelArg(kr, 4, sizeof(cl_int), &ksplit)); + size_t lr[3] = { 64, 1, 1 }; + size_t gr[3] = { (size_t) CEIL_DIV(ne01, 64) * 64, 1, 1 }; + backend_ctx->enqueue_ndrange_kernel(kr, 3, gr, lr, dst); + + CL_CHECK(clReleaseMemObject(q_img)); + CL_CHECK(clReleaseMemObject(b_img)); + CL_CHECK(clReleaseMemObject(b_sub_buf)); + return; + } + kernel = backend_ctx->kernel_gemv_noshuffle_q8_0_f32; int r2 = 1; @@ -17505,18 +18478,33 @@ static void ggml_cl_mul_mat_q4_k_f32_adreno(ggml_backend_t backend, const ggml_t cl_uchar mask_d4 = 0x0F; cl_uchar mask_hi2 = 0xC0; - if (ne1 == 1) { + // Multi-column verify GEMV: route the spec/MTP verify batch (ne1==3 = 2 + // drafts + 1 bonus) onto the efficient GEMV path (subgroup-broadcast, no + // transpose) instead of the transposed-GEMM dead-zone. Reuses the ne1==1 + // GEMV setup (the activation image is already sized by N=ne1). Byte- + // identical. Opt-in via GGML_OPENCL_Q4K_MC3=1 while validating. + static const bool q4k_mc3 = (getenv("GGML_OPENCL_Q4K_MC3") != nullptr); + // Per-layer only (ne01 < 32768): the batched large-vocab lm_head at ne1==3 + // is left to the existing routing (corrupts on the Adreno GEMV path; x2- + // unified routes batched Q6_K lm_head to CPU). Per-layer mc3 is byte-identical. + const bool use_mc3 = q4k_mc3 && (ne1 == 3) && (ne01 < 32768); + + if (ne1 == 1 || use_mc3) { cl_mem q_img = nullptr; cl_mem b_sub_buf = nullptr; cl_mem b_img = nullptr; - // image for q - img_fmt = { CL_R, CL_UNSIGNED_INT32}; - memset(&img_desc, 0, sizeof(img_desc)); - img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; - img_desc.image_width = M * K / 2 / 4; - img_desc.buffer = extra0_q4_k->q; - CL_CHECK((q_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); + const bool use_tiled = !use_mc3 && use_q4k_tiled(backend_ctx, src0); + + // image for q (not needed for the tiled path, which reads __global) + if (!use_tiled) { + img_fmt = { CL_R, CL_UNSIGNED_INT32}; + memset(&img_desc, 0, sizeof(img_desc)); + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = M * K / 2 / 4; + img_desc.buffer = extra0_q4_k->q; + CL_CHECK((q_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); + } // subbuffer for activations region.origin = offset1; @@ -17531,27 +18519,173 @@ static void ggml_cl_mul_mat_q4_k_f32_adreno(ggml_backend_t backend, const ggml_t img_desc.buffer = b_sub_buf; CL_CHECK((b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); - kernel = backend_ctx->kernel_gemv_noshuffle_q4_k_f32; - - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &q_img)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q4_k->d)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q4_k->dm)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0_q4_k->s)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &b_img)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_int), &ne00)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_int), &ne01)); - CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_uchar), &mask_d6)); - CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_uchar), &mask_d4)); - CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_uchar), &mask_hi2)); + // 4-output-per-WI o4 variant for the long-vocab lm_head/embed GEMV + // (ne01 = vocab ~256K on Gemma): shares one activation read across 4 + // output rows. Gated to large ne01 (lm_head/embed). Default on; opt-out + // GGML_OPENCL_Q4K_GEMV_O4=0. (Skipped when mc3 handles the ne1==3 verify.) + static const bool q4k_o4_env = []{ + const char * e = std::getenv("GGML_OPENCL_Q4K_GEMV_O4"); + return !e || e[0] == '\0' || e[0] != '0'; + }(); + const bool use_q4k_o4 = !use_tiled && !use_mc3 && q4k_o4_env && (ne01 % 4 == 0) && (ne01 >= 32768); + // Split-K across workgroups for small-M decode GEMVs. A single-token GEMV + // makes only CEIL_DIV(M/2,64) workgroups; even with the wide intra-WG split + // (16 subgroups) those all land on ONE CU, so small-M matmuls under-fill the + // 16 CUs and their bandwidth falls well short of what the large-M FFN matmuls + // reach. Adding a `ksplit` second grid dim that spreads K across WGs (+ a + // reduce pass) fills the CUs. Gate is M<=2560: the tiny M<=1024 ones only + // break even (the reduce dispatch eats the kernel win), but the big-K M=2560 + // cases (ffn_down, attn_output) make the per-call win dwarf the reduce, and + // are byte-identical. ffn_gate/up (large M) fill the CUs already and are excluded. + // + // DEVICE-GATED. Split-K buys GPU time by spending an extra kernel LAUNCH (the + // reduce), so it only pays where launches are cheap. That is a per-device + // property and it does not travel from the X2-90 this was tuned on. Measured + // with one binary, env A/B (tg32, GGML_OPENCL_Q4K_GEMV_SPLITK=0/1): + // + // X2-90 +3.36% gemma-4 E4B (the number this gate was built on) + // 840 -1.3% Qwen3.5-4B-Q4_K_M 14.00 -> 13.85 + // 850 -20.0% Qwen3-1.7B-Q4_K_M 6.97 -> 5.58 (6 interleaved reps) + // + // The kernel is not the problem. On the 850 split-K makes the GPU strictly + // faster -- total busy 537 -> 485 ms, this GEMV 43.7 -> 34.0 us/call (-22%) -- + // and still costs a fifth of decode, because the +3696 reduce dispatches cost + // ~550 us of HOST round-trip each against 2.7 us of GPU work (~200x; that part + // is ~95% host-bound at decode). The 840 pays the same tax at ~42 us/dispatch. + // Break-even needs launch cost below the ~9.7 us/call the split actually saves, + // so this is not a "the 850 is slow" adjustment that a faster part would fix -- + // the 840 is 13x cheaper per launch and still loses. + // + // Enabled where it is measured to win, i.e. X2E only. The X1-85 was measured + // afterwards and is NOT a win either: Qwen3.5-4B-Q4_K_M tg32, split-K off + // 17.98/18.10/18.19 vs on 18.03/17.91/17.97 = -0.7%, so X1E stays excluded on + // evidence rather than on absence of it. Do not widen this without a NEW + // measurement. The env still forces either way so every device stays measurable. + static const bool splitk_env_set = []{ + const char * e = std::getenv("GGML_OPENCL_Q4K_GEMV_SPLITK"); + return e && e[0] != '\0'; + }(); + static const bool splitk_env_on = []{ + const char * e = std::getenv("GGML_OPENCL_Q4K_GEMV_SPLITK"); + return !(e && e[0] == '0'); + }(); + const bool splitk_wg_env = splitk_env_set + ? splitk_env_on + : (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E); + // Gate: small-M decode GEMVs that under-fill the 16 CUs even with the wide + // intra-WG split (all 16 subgroups land on one CU). M<=2560 covers Kcur/Vcur + // (M=1024), Qcur (2048), attn_output + ffn_down (2560). The tiny ones + // (M<=1024) only break even (reduce dispatch eats the kernel win), but the + // big-K M=2560 cases (ffn_down K=10240 @182us, attn_output @42us) have a + // large per-call win that dwarfs the ~5us reduce, so extending to 2560 nets + // positive end-to-end. ffn_gate/up (M=10240) already fill the CUs -> excluded. + const bool use_splitk = splitk_wg_env && !use_tiled && !use_q4k_o4 && !use_mc3 && ne01 <= 2560; + + if (use_splitk) { + const int nsg = 8; + const int ksplit = (ne01 <= 512) ? 8 : 4; // -> ~32 total WGs + const size_t gx = (size_t)CEIL_DIV(ne01/2, 64) * 64; + + backend_ctx->prealloc_splitk_partial.allocate( + backend_ctx->context, (size_t)ksplit * ne01 * sizeof(float)); + cl_mem partial = backend_ctx->prealloc_splitk_partial.buffer; + + cl_kernel ks = backend_ctx->kernel_gemv_noshuffle_q4_k_f32_splitk; + CL_CHECK(clSetKernelArg(ks, 0, sizeof(cl_mem), &q_img)); + CL_CHECK(clSetKernelArg(ks, 1, sizeof(cl_mem), &extra0_q4_k->d)); + CL_CHECK(clSetKernelArg(ks, 2, sizeof(cl_mem), &extra0_q4_k->dm)); + CL_CHECK(clSetKernelArg(ks, 3, sizeof(cl_mem), &extra0_q4_k->s)); + CL_CHECK(clSetKernelArg(ks, 4, sizeof(cl_mem), &b_img)); + CL_CHECK(clSetKernelArg(ks, 5, sizeof(cl_mem), &partial)); + CL_CHECK(clSetKernelArg(ks, 6, sizeof(cl_int), &ne00)); + CL_CHECK(clSetKernelArg(ks, 7, sizeof(cl_int), &ne01)); + CL_CHECK(clSetKernelArg(ks, 8, sizeof(cl_uchar), &mask_d6)); + CL_CHECK(clSetKernelArg(ks, 9, sizeof(cl_uchar), &mask_d4)); + CL_CHECK(clSetKernelArg(ks, 10, sizeof(cl_uchar), &mask_hi2)); + size_t lsk[3] = {64, (size_t)nsg, 1}; + size_t gsk[3] = {gx, (size_t)(nsg * ksplit), 1}; + backend_ctx->enqueue_ndrange_kernel(ks, 3, gsk, lsk, dst); + + cl_kernel kr = backend_ctx->kernel_gemv_splitk_reduce_f32; + CL_CHECK(clSetKernelArg(kr, 0, sizeof(cl_mem), &partial)); + CL_CHECK(clSetKernelArg(kr, 1, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kr, 2, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kr, 3, sizeof(cl_int), &ne01)); + CL_CHECK(clSetKernelArg(kr, 4, sizeof(cl_int), &ksplit)); + size_t lr[3] = {64, 1, 1}; + size_t gr[3] = {(size_t)CEIL_DIV(ne01, 64) * 64, 1, 1}; + backend_ctx->enqueue_ndrange_kernel(kr, 3, gr, lr, dst); + + if (q_img) CL_CHECK(clReleaseMemObject(q_img)); + CL_CHECK(clReleaseMemObject(b_sub_buf)); + CL_CHECK(clReleaseMemObject(b_img)); + return; + } - size_t local_work_size[3] = {64, 4, 1}; - size_t global_work_size[3] = {(size_t)CEIL_DIV(ne01/2, 64)*64, 4, 1}; + kernel = use_mc3 ? backend_ctx->kernel_gemv_noshuffle_q4_k_f32_mc3 + : use_tiled ? backend_ctx->kernel_gemv_noshuffle_q4_k_f32_tiled + : use_q4k_o4 ? backend_ctx->kernel_gemv_noshuffle_q4_k_f32_o4 + : backend_ctx->kernel_gemv_noshuffle_q4_k_f32; + + if (use_tiled) { + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q4_k->q)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q4_k->d)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q4_k->dm)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0_q4_k->s)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &b_img)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_int), &ne01)); + } else { + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &q_img)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q4_k->d)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q4_k->dm)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0_q4_k->s)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &b_img)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_uchar), &mask_d6)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_uchar), &mask_d4)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_uchar), &mask_hi2)); + } + + // Wide K-split for the decode GEMV: the default 4-subgroup K-split leaves + // each Adreno SP with only ~4 waves, too few to hide LPDDR weight-load + // latency, so even the large FFN matmuls run well below the achievable + // bandwidth. Widen to 16 subgroups/WG (= the 1024-lane Adreno WG max) so + // each SP holds enough in-flight memory requests. Prefill is unaffected (the + // GEMM path is separate) and coherence-identical (greedy output unchanged). + // Applies to the plain base + // GEMV only; tiled/o4/mc3 keep 4 (their reductions are hard-coded to 4). + // Layout-safe: the base kernel derives its K-split from get_local_size(1) + // and the packed block stride is a physical constant (independent of it). + // Opt-out: GGML_OPENCL_Q4K_GEMV_WIDE=0. + static const bool splitk_wide_env = []{ + const char * e = std::getenv("GGML_OPENCL_Q4K_GEMV_WIDE"); + return !e || e[0] == '\0' || e[0] != '0'; + }(); + const bool splitk_wide = splitk_wide_env && !use_tiled && !use_q4k_o4 && !use_mc3; + size_t nsg_y = splitk_wide ? 16 : 4; + // Cap the wide K-split by the kernel's real max WG. X1-class drivers cap + // this GEMV at 768 (< 64*16 = 1024), so an uncapped lws aborts the + // dispatch with CL_INVALID_WORK_GROUP_SIZE (-54) and breaks ALL q4_K + // decode for M>2560. nsg_y is a pure K-split (the base kernel reads it + // from get_local_size(1); the packed block stride is a physical constant), + // so halving it stays coherent — just a narrower split. X2 keeps 16 + // (maxwg 1024); X1 falls to 8. + if (splitk_wide) { + const size_t maxwg = backend_ctx->get_kernel_workgroup_size(kernel); + while (nsg_y > 4 && 64 * nsg_y > maxwg) { nsg_y >>= 1; } + } + size_t local_work_size[3] = {64, nsg_y, 1}; + size_t global_work_size[3] = {(size_t)CEIL_DIV(use_tiled ? ne01 : (use_q4k_o4 ? ne01/4 : ne01/2), 64)*64, nsg_y, 1}; backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); - CL_CHECK(clReleaseMemObject(q_img)); + if (q_img) CL_CHECK(clReleaseMemObject(q_img)); CL_CHECK(clReleaseMemObject(b_sub_buf)); CL_CHECK(clReleaseMemObject(b_img)); } else { @@ -17711,10 +18845,44 @@ static void ggml_cl_mul_mat_q4_k_f32_adreno(ggml_backend_t backend, const ggml_t } // gemm - kernel = backend_ctx->kernel_gemm_noshuffle_q4_k_f32; + // Small-batch (medium n_q) occupancy fix: at ne1<=8 the 2x8 grid is + // (1, ceil(M/2)) -> ~M/256 workgroups, which under-occupies the SP and + // makes the GEMM much slower than the ne1==1 GEMV at the same weight + // traffic. The _r1 (1-row) kernel doubles the M-axis workgroup count + // and removes the accumulator spill. Opt-in via env while validating. + static const bool q4k_gemm_r1 = (getenv("GGML_OPENCL_Q4K_GEMM_R1") != nullptr); + static const bool q4k_gemm_kimg = (getenv("GGML_OPENCL_Q4K_GEMM_KIMG") != nullptr); + // Cooperative-K (intra-WG K-split + reduction) for the small-batch + // (n_q in [2..8]) path: DEFAULT ON, opt out with GGML_OPENCL_Q4K_GEMM_COK=0. + // Byte-identical greedy output; large-batch (ne1>8) untouched. + static const char * q4k_cok_env = getenv("GGML_OPENCL_Q4K_GEMM_COK"); + static const bool q4k_gemm_cok = (q4k_cok_env == nullptr) || (atoi(q4k_cok_env) != 0); + const bool use_cok = q4k_gemm_cok && (ne1 <= 8); + const bool use_r1 = !use_cok && q4k_gemm_r1 && (ne1 <= 8); + // Weights-as-image (L1/TPL1) for the small-batch weight-read-bound path. + const bool use_kimg = !use_cok && !use_r1 && q4k_gemm_kimg && (ne1 <= 8); + + cl_mem q_img = nullptr; + if (use_kimg) { + img_fmt = { CL_R, CL_UNSIGNED_INT32 }; + memset(&img_desc, 0, sizeof(img_desc)); + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = M * K / 2 / 4; + img_desc.buffer = extra0_q4_k->q; + CL_CHECK((q_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); + } + + kernel = use_cok ? backend_ctx->kernel_gemm_noshuffle_q4_k_f32_cok + : use_r1 ? backend_ctx->kernel_gemm_noshuffle_q4_k_f32_r1 + : use_kimg ? backend_ctx->kernel_gemm_noshuffle_q4_k_f32_kimg + : backend_ctx->kernel_gemm_noshuffle_q4_k_f32; int padded_N = N + padding; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q4_k->q)); + if (use_kimg) { + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &q_img)); + } else { + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q4_k->q)); + } CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q4_k->s)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q4_k->d)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0_q4_k->dm)); @@ -17729,10 +18897,45 @@ static void ggml_cl_mul_mat_q4_k_f32_adreno(ggml_backend_t backend, const ggml_t CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_uchar), &mask_d4)); CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_uchar), &mask_hi2)); - size_t global_work_size[3] = {(size_t)CEIL_DIV(ne1, 8), (size_t)CEIL_DIV(ne01, 4), 1}; - size_t local_work_size[3] = {1, 128, 1}; + size_t global_work_size[3]; + size_t local_work_size[3]; + if (use_cok) { + // (COK_SG lanes x COK_NSG subgroups): one row per lane, K split + // across the COK_NSG subgroups. ne01 is a multiple of 64. + global_work_size[0] = (size_t)ne01; // rows + global_work_size[1] = 8; // COK_NSG + global_work_size[2] = 1; + local_work_size[0] = 64; // COK_SG + local_work_size[1] = 8; // COK_NSG + local_work_size[2] = 1; + } else if (use_r1) { + // 1 row per WI (opt-in occupancy experiment). + global_work_size[0] = (size_t)CEIL_DIV(ne1, 8); + global_work_size[1] = (size_t)ne01; + global_work_size[2] = 1; + local_work_size[0] = 1; + local_work_size[1] = 128; + local_work_size[2] = 1; + } else if (use_kimg) { + // kimg is a 2-row tile (opt-in weights-as-image experiment). + global_work_size[0] = (size_t)CEIL_DIV(ne1, 8); + global_work_size[1] = (size_t)CEIL_DIV(ne01, 2); + global_work_size[2] = 1; + local_work_size[0] = 1; + local_work_size[1] = 128; + local_work_size[2] = 1; + } else { + // Default: x2-unified base kernel is the 4-row (gx<<2) tile. + global_work_size[0] = (size_t)CEIL_DIV(ne1, 8); + global_work_size[1] = (size_t)CEIL_DIV(ne01, 4); + global_work_size[2] = 1; + local_work_size[0] = 1; + local_work_size[1] = 128; + local_work_size[2] = 1; + } backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + if (q_img) CL_CHECK(clReleaseMemObject(q_img)); CL_CHECK(clReleaseMemObject(b_sub_buf)); CL_CHECK(clReleaseMemObject(b_sub_buf_trans)); CL_CHECK(clReleaseMemObject(b_img)); @@ -17780,29 +18983,65 @@ static void ggml_cl_mul_mat_q6_K_f32_adreno(ggml_backend_t backend, const ggml_t cl_image_desc img_desc; // subbuffer and image for activation - if (ne1 == 1) { + // Multi-column verify GEMV: route the spec/MTP verify q6_K matmuls (ne1==3) + // onto the efficient GEMV path instead of the transposed-GEMM dead-zone. + // Reuses the ne1==1 image setup (activation image sized by N=ne1). Byte- + // identical. Opt-in via GGML_OPENCL_Q6K_MC3=1 while validating. + static const bool q6k_mc3 = (getenv("GGML_OPENCL_Q6K_MC3") != nullptr); + // Per-layer only (ne01 < 32768): batched large-vocab lm_head stays on the + // existing path (x2-unified routes batched Q6_K lm_head to CPU; the Adreno + // GEMV corrupts it). Per-layer mc3 is byte-identical. + const bool use_q6k_mc3 = q6k_mc3 && (ne1 == 3) && (ne01 < 32768); + // Batched verify lm_head/embed (ne1==3, tiled layout): multi-column tiled + // GEMV — streams the large lm_head weight once across the 3 verify columns + // (the #1 MTP bottleneck; mc3 above can't, it reads the noshuffle layout). + const bool use_q6k_tiled_mc = q6k_mc3 && (ne1 == 3) && (ne01 >= 32768) && use_q6k_tiled(backend_ctx, src0); + + if (ne1 == 1 || use_q6k_mc3 || use_q6k_tiled_mc) { cl_mem ql_img = nullptr; cl_mem qh_img = nullptr; cl_mem b_sub_buffer = nullptr; cl_mem b_img = nullptr; - // image for ql - img_fmt.image_channel_order = CL_R; - img_fmt.image_channel_data_type = CL_FLOAT; - memset(&img_desc, 0, sizeof(img_desc)); - img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; - img_desc.image_width = ne01 * ne00 / 8; - img_desc.buffer = extra0_q6_K->ql; - CL_CHECK((ql_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); + // o4 = 4-output-per-WI variant for long-vocab lm_head/embed; gated to + // ne01 >= 32768 so per-layer q6_K (ne01=hidden 2-8K) keeps the 2-output + // kernel (o4 regresses there). o4_global reads the weights from __global + // coalesced instead of image1d_buffer -- the texture cache caps the + // read-once-per-token lm_head bandwidth, while __global reaches the higher + // rate the rest of the model gets. Both default ON; opt out via + // GGML_OPENCL_Q6K_GEMV_O4 / GGML_OPENCL_Q6K_GEMV_O4_GLOBAL = 0. + static const bool gemv_o4_env = []{ + const char * e = std::getenv("GGML_OPENCL_Q6K_GEMV_O4"); + return !e || e[0] == '\0' || e[0] != '0'; + }(); + static const bool o4_global_env = []{ + const char * e = std::getenv("GGML_OPENCL_Q6K_GEMV_O4_GLOBAL"); + return !e || e[0] == '\0' || e[0] != '0'; + }(); + const bool use_tiled = !use_q6k_mc3 && use_q6k_tiled(backend_ctx, src0); + const bool use_o4 = !use_tiled && !use_q6k_mc3 && gemv_o4_env && (ne01 % 4 == 0) && (ne01 >= 32768); + const bool use_o4_global = use_o4 && o4_global_env; + + // ql/qh image views are only needed when NOT reading weights from global. + if (!use_o4_global && !use_tiled) { + // image for ql + img_fmt.image_channel_order = CL_R; + img_fmt.image_channel_data_type = CL_FLOAT; + memset(&img_desc, 0, sizeof(img_desc)); + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = ne01 * ne00 / 8; + img_desc.buffer = extra0_q6_K->ql; + CL_CHECK((ql_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); - // image for qh - img_fmt.image_channel_order = CL_R; - img_fmt.image_channel_data_type = CL_HALF_FLOAT; - memset(&img_desc, 0, sizeof(img_desc)); - img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; - img_desc.image_width = ne01 * ne00 / 8; - img_desc.buffer = extra0_q6_K->qh; - CL_CHECK((qh_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); + // image for qh + img_fmt.image_channel_order = CL_R; + img_fmt.image_channel_data_type = CL_HALF_FLOAT; + memset(&img_desc, 0, sizeof(img_desc)); + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = ne01 * ne00 / 8; + img_desc.buffer = extra0_q6_K->qh; + CL_CHECK((qh_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); + } region.origin = offset1; region.size = ne00 * ne1 * sizeof(float); @@ -17816,10 +19055,20 @@ static void ggml_cl_mul_mat_q6_K_f32_adreno(ggml_backend_t backend, const ggml_t img_desc.buffer = b_sub_buffer; CL_CHECK((b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); - kernel = backend_ctx->kernel_gemv_noshuffle_q6_K_f32; + kernel = use_q6k_mc3 ? backend_ctx->kernel_gemv_noshuffle_q6_K_f32_mc3 + : use_q6k_tiled_mc ? backend_ctx->kernel_gemv_noshuffle_q6_K_f32_tiled_mc3 + : use_tiled ? backend_ctx->kernel_gemv_noshuffle_q6_K_f32_tiled + : use_o4_global ? backend_ctx->kernel_gemv_noshuffle_q6_K_f32_o4_global + : use_o4 ? backend_ctx->kernel_gemv_noshuffle_q6_K_f32_o4 + : backend_ctx->kernel_gemv_noshuffle_q6_K_f32; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &ql_img)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &qh_img)); + if (use_o4_global || use_tiled) { + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q6_K->ql)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q6_K->qh)); + } else { + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &ql_img)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &qh_img)); + } CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q6_K->s)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0_q6_K->d)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &b_img)); @@ -17828,16 +19077,67 @@ static void ggml_cl_mul_mat_q6_K_f32_adreno(ggml_backend_t backend, const ggml_t CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_int), &ne00)); CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_int), &ne01)); - size_t local_work_size[3] = {64, 4, 1}; - size_t global_work_size[3] = {(size_t)CEIL_DIV(ne01/2, 64)*64, 4, 1}; + const size_t gws_x = use_tiled + ? (size_t) CEIL_DIV(ne01, 64) * 64 + : use_o4 + ? (size_t) CEIL_DIV(ne01/4, 64) * 64 + : (size_t) CEIL_DIV(ne01/2, 64) * 64; + size_t local_work_size[3] = {64, 4, 1}; + size_t global_work_size[3] = {gws_x, 4, 1}; backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); - CL_CHECK(clReleaseMemObject(ql_img)); - CL_CHECK(clReleaseMemObject(qh_img)); + if (ql_img) CL_CHECK(clReleaseMemObject(ql_img)); + if (qh_img) CL_CHECK(clReleaseMemObject(qh_img)); CL_CHECK(clReleaseMemObject(b_sub_buffer)); CL_CHECK(clReleaseMemObject(b_img)); } else { + // Tiled-layout batched GEMM. When the weight was converted to the 64-row + // tiled canonical layout (use_q6k_tiled — the default for lm_head/embed), + // the plain noshuffle GEMM below reads it as plain-transposed and produces + // garbage. Use the batched GEMM that matches the decode tiled GEMV's + // layout; it reads the f32 activation directly (column-major, no transpose). + if (use_q6k_tiled(backend_ctx, src0)) { + cl_mem b_sub_buf_t = nullptr; + cl_mem b_img_t = nullptr; + + region.origin = offset1; + region.size = ne00 * ne1 * sizeof(float); + CL_CHECK((b_sub_buf_t = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + + img_fmt.image_channel_order = CL_RGBA; + img_fmt.image_channel_data_type = CL_FLOAT; + memset(&img_desc, 0, sizeof(img_desc)); + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = ne00 * ne1 / 4; + img_desc.buffer = b_sub_buf_t; + CL_CHECK((b_img_t = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); + + cl_kernel kt = backend_ctx->kernel_gemm_noshuffle_q6_K_f32_tiled; + CL_CHECK(clSetKernelArg(kt, 0, sizeof(cl_mem), &extra0_q6_K->ql)); + CL_CHECK(clSetKernelArg(kt, 1, sizeof(cl_mem), &extra0_q6_K->qh)); + CL_CHECK(clSetKernelArg(kt, 2, sizeof(cl_mem), &extra0_q6_K->s)); + CL_CHECK(clSetKernelArg(kt, 3, sizeof(cl_mem), &extra0_q6_K->d)); + CL_CHECK(clSetKernelArg(kt, 4, sizeof(cl_mem), &b_img_t)); + CL_CHECK(clSetKernelArg(kt, 5, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kt, 6, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kt, 7, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kt, 8, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kt, 9, sizeof(int), &ne1)); + + // Must match the kernel: NTILES=4 64-row tiles per work-group (256 rows), + // BN=8 output columns per work-group. + const int BN_T = 16; + const int WROWS = 4 * 64; // NTILES * TILE_ROWS + size_t local_work_size[3] = {64, 4, 1}; + size_t global_work_size[3] = {(size_t)CEIL_DIV(ne01, WROWS) * 64, 4, (size_t)CEIL_DIV(ne1, BN_T)}; + backend_ctx->enqueue_ndrange_kernel(kt, 3, global_work_size, local_work_size, dst); + + CL_CHECK(clReleaseMemObject(b_img_t)); + CL_CHECK(clReleaseMemObject(b_sub_buf_t)); + return; + } + cl_mem b_sub_buf; cl_mem b_buf_trans; cl_mem b_img; @@ -17949,7 +19249,19 @@ static void ggml_cl_mul_mat_q6_K_f32_adreno(ggml_backend_t backend, const ggml_t backend_ctx->enqueue_ndrange_kernel(kernel, 2, global_size_t, local_size_t, dst); // gemm - kernel = backend_ctx->kernel_gemm_noshuffle_q6_K_f32; + // Cooperative-K small-batch (n_q in [2..8]) path: intra-WG K-split, + // mirrors the q4_K _cok path (batched serving). OPT-IN + // (GGML_OPENCL_Q6K_GEMM_COK=1), DEFAULT OFF: q6_K is the tied lm_head/ + // output projection, so the K-reassociation perturbs final logits and + // greedy is NOT byte-identical (op-tests pass, output coherent, but not + // bit-exact). It is also NEUTRAL on end-to-end MTP (q4_K cok already + // captured that; the MTP bottleneck moved off the GEMMs). Keep opt-in + // for batched serving until PPL-validated on a non-GDN q6_K model. + static const char * q6k_cok_env = getenv("GGML_OPENCL_Q6K_GEMM_COK"); + static const bool q6k_gemm_cok = (q6k_cok_env != nullptr) && (atoi(q6k_cok_env) != 0); + const bool use_q6k_cok = q6k_gemm_cok && (ne1 <= 8); + kernel = use_q6k_cok ? backend_ctx->kernel_gemm_noshuffle_q6_K_f32_cok + : backend_ctx->kernel_gemm_noshuffle_q6_K_f32; int padded_N = ne1 + padding; cl_ushort mask_f000 = 0xF000; @@ -17969,8 +19281,23 @@ static void ggml_cl_mul_mat_q6_K_f32_adreno(ggml_backend_t backend, const ggml_t CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ushort),&mask_f000)); CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_uchar), &mask_c0)); - size_t global_work_size[3] = {(size_t)CEIL_DIV(ne1, 8), (size_t)CEIL_DIV(ne01, 4), 1}; - size_t local_work_size[3] = {2, 128, 1}; + size_t global_work_size[3]; + size_t local_work_size[3]; + if (use_q6k_cok) { + global_work_size[0] = (size_t)ne01; // rows (1 per lane) + global_work_size[1] = 8; // COK_NSG + global_work_size[2] = 1; + local_work_size[0] = 64; // COK_SG + local_work_size[1] = 8; // COK_NSG + local_work_size[2] = 1; + } else { + global_work_size[0] = (size_t)CEIL_DIV(ne1, 8); + global_work_size[1] = (size_t)CEIL_DIV(ne01, 4); + global_work_size[2] = 1; + local_work_size[0] = 2; + local_work_size[1] = 128; + local_work_size[2] = 1; + } backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); CL_CHECK(clReleaseMemObject(b_sub_buf)); @@ -18026,7 +19353,15 @@ static void ggml_cl_mul_mat_q5_K_f32_adreno(ggml_backend_t backend, const ggml_t cl_uchar mask_d4 = 0x0F; cl_uchar mask_hi2 = 0xC0; - if (ne1 == 1) { + // Multi-column (N=3) verify GEMV for q5_K: route the spec/MTP verify batch + // (ne1==3) onto the efficient GEMV path instead of the transposed-GEMM dead- + // zone (gemm_noshuffle_q5_k, the #2 chunk of MTP decode on a Q4_0-mix model + // after q4_0 mc3). Reuses the ne1==1 GEMV image setup (q + qh + activations). + // Opt-in via GGML_OPENCL_Q5K_MC3=1. Per-layer only (ne01 < 32768). + static const bool q5k_mc3 = (getenv("GGML_OPENCL_Q5K_MC3") != nullptr); + const bool use_q5k_mc3 = q5k_mc3 && (ne1 >= 2 && ne1 <= 4) && (ne01 < 32768); + + if (ne1 == 1 || use_q5k_mc3) { cl_mem q_img = nullptr; cl_mem qh_img = nullptr; cl_mem b_sub_buf = nullptr; @@ -18061,7 +19396,8 @@ static void ggml_cl_mul_mat_q5_K_f32_adreno(ggml_backend_t backend, const ggml_t img_desc.buffer = b_sub_buf; CL_CHECK((b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); - kernel = backend_ctx->kernel_gemv_noshuffle_q5_k_f32; + kernel = use_q5k_mc3 ? backend_ctx->kernel_gemv_noshuffle_q5_k_f32_mc3 + : backend_ctx->kernel_gemv_noshuffle_q5_k_f32; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &q_img)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &qh_img)); @@ -18076,6 +19412,9 @@ static void ggml_cl_mul_mat_q5_K_f32_adreno(ggml_backend_t backend, const ggml_t CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_uchar), &mask_d6)); CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_uchar), &mask_d4)); CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_uchar), &mask_hi2)); + if (use_q5k_mc3) { + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_int), &ne1)); // n_cols + } size_t local_work_size[3] = {64, 4, 1}; size_t global_work_size[3] = {(size_t)CEIL_DIV(ne01/2, 64)*64, 4, 1}; @@ -18598,7 +19937,18 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co cl_kernel kernel; #ifdef GGML_OPENCL_USE_ADRENO_KERNELS - if(src0t == GGML_TYPE_F16 && src1t == GGML_TYPE_F32){ + // The mul_mm_f16_f32_kq / _kqv attention kernels MISCOMPUTE the batched + // prefill attention: they produce correct results only for the final query + // position and garble all intermediate positions. Generation stays coherent + // (it consumes the final position), but perplexity, speculative-decode verify + // and any batched/multi-position scoring are wrong (perplexity blows up). + // Reproduces at both head_dim 256 and 128; routing both KQ and KQV to the + // generic mul_mm/mul_mv path below restores correctness on every model. The + // kernel is also not a perf win (the generic path scales better at long + // context). + // Disabled by default; opt in with GGML_OPENCL_KQKV_KERNEL=1 only to debug/ + // benchmark the (incorrect) kernel. + if(src0t == GGML_TYPE_F16 && src1t == GGML_TYPE_F32 && getenv("GGML_OPENCL_KQKV_KERNEL") != nullptr){ if (ne01 >= 64 && ne1 >= 32 && ne00 >= 16 && (ne12 % ne02) == 0 && // the KQ/KQV image kernels do not handle dim 3 (multi-stream batches) ne03 == 1 && ne13 == 1 && @@ -18897,7 +20247,7 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co } // q4_k x fp32 - if (src0t == GGML_TYPE_Q4_K && src1t == GGML_TYPE_F32 && !use_flat_gemv_for_large_m_q4_K(src0)) { + if (src0t == GGML_TYPE_Q4_K && src1t == GGML_TYPE_F32 && !use_flat_gemv_for_large_m_q4_K(backend_ctx, src0)) { ggml_cl_mul_mat_q4_k_f32_adreno(backend, src0, src1, dst); return; } @@ -18923,6 +20273,35 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co ne11 > 1) { switch(src0t) { case GGML_TYPE_F32: { + // Small-N f32 GEMV for the spec/MTP verify batch: the tiled GEMM + // below always computes a full 64x64 tile, so at ne11=3 with a + // skinny f32 weight (GDN ssm_alpha/ssm_beta, M=32) it launches one + // under-occupied WG at ~2.3% tile utilization. Route to a per-output + // (m,n) GEMV (64-thread WG, K-split + __local reduce) instead. + // Opt-in GGML_OPENCL_F32_MC=1; 2D contiguous, small N + skinny M only. + static const bool f32_mc = (getenv("GGML_OPENCL_F32_MC") != nullptr); + if (f32_mc && ne11 >= 2 && ne11 <= 8 && ne01 <= 512 && (ne00 % 4 == 0) && + ne02 == 1 && ne12 == 1 && ne13 == 1 && + ggml_is_contiguous(src0) && ggml_is_contiguous(src1)) { + cl_kernel kmc = backend_ctx->kernel_gemv_f32_f32_mc; + int stride_a = ne00, stride_b = ne00, stride_d = ne01; + CL_CHECK(clSetKernelArg(kmc, 0, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(kmc, 1, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kmc, 2, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kmc, 3, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kmc, 4, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kmc, 5, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kmc, 6, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kmc, 7, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kmc, 8, sizeof(int), &ne11)); + CL_CHECK(clSetKernelArg(kmc, 9, sizeof(int), &stride_a)); + CL_CHECK(clSetKernelArg(kmc, 10, sizeof(int), &stride_b)); + CL_CHECK(clSetKernelArg(kmc, 11, sizeof(int), &stride_d)); + size_t gws[3] = {64, (size_t)ne01 * (size_t)ne11, 1}; + size_t lws[3] = {64, 1, 1}; + backend_ctx->enqueue_ndrange_kernel(kmc, 3, gws, lws, dst); + return; + } kernel = backend_ctx->kernel_mul_mm_f32_f32_l4_lm; nth0 = 128; // calculated as (BM*BN)/(TM*TN) @@ -19578,6 +20957,7 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co } // use custom matrix x vector kernel + bool use_f16_mrow = false; switch (src0t) { case GGML_TYPE_F32: //GGML_ASSERT(ne02 == ne12); @@ -19643,7 +21023,46 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co (ne12 % r2) == 0; if (ne11 * ne12 < 4) { - kernel = backend_ctx->kernel_mul_mat_f16_f32_1row; + // Decode (single token): the legacy _1row runs one 64-lane + // subgroup per WG (one output row), under-utilizing BW. Route the + // wide f16 weight matmuls (attn proj + lm_head) to the multi-row + // variant: MROW rows per WG -> more loads in flight + activation + // staged once in __local. ne00<=8192 bounds the LDS. The mrow WG + // is 64 x MROW = 1024 work-items (> Intel's 512 max) and reduces + // within a 64-wide subgroup, so skip on Intel. + if (backend_ctx->f16_mrow && backend_ctx->gpu_family != INTEL && + backend_ctx->kernel_mul_mat_f16_f32_mrow != nullptr && + ne00 >= 128 && ne01 >= 8 && ne00 % 4 == 0 && ne00 <= 8192) { + // The register-blocked / half8 variants cast the src0 row pointer to + // half4 / half8 (8- and 16-byte loads) with no scalar fallback inside + // the kernel. ne00 % 4 == 0 constrains the element count per row, NOT + // the byte stride between rows: a permuted or strided src0 (or a view + // at an odd offset) can leave nb01/nb02/nb03 unaligned. Only take them + // when every row this dispatch touches is aligned; the base mrow kernel + // re-checks per row and falls back to its scalar loop. + const cl_ulong row_addr_bits = offset0 | nb01 | nb02 | nb03; + const bool aligned8 = (row_addr_bits & 7) == 0; + const bool aligned16 = (row_addr_bits & 15) == 0; + + // Register-blocked variants: each subgroup does RPT rows (more + // weight loads in flight per lane). 8/16 use half8 (128-bit) + // loads, gated on ne00 % 8 == 0. + const int rpt = backend_ctx->f16_mrow_rpt; + if (rpt == 16 && ne00 % 8 == 0 && aligned16 && backend_ctx->kernel_mul_mat_f16_f32_mrow_h8r2 != nullptr) { + kernel = backend_ctx->kernel_mul_mat_f16_f32_mrow_h8r2; + } else if (rpt == 8 && ne00 % 8 == 0 && aligned16 && backend_ctx->kernel_mul_mat_f16_f32_mrow_h8 != nullptr) { + kernel = backend_ctx->kernel_mul_mat_f16_f32_mrow_h8; + } else if (rpt == 4 && aligned8 && backend_ctx->kernel_mul_mat_f16_f32_mrow_r4 != nullptr) { + kernel = backend_ctx->kernel_mul_mat_f16_f32_mrow_r4; + } else if (rpt == 2 && aligned8 && backend_ctx->kernel_mul_mat_f16_f32_mrow_r2 != nullptr) { + kernel = backend_ctx->kernel_mul_mat_f16_f32_mrow_r2; + } else { + kernel = backend_ctx->kernel_mul_mat_f16_f32_mrow; + } + use_f16_mrow = true; + } else { + kernel = backend_ctx->kernel_mul_mat_f16_f32_1row; + } } else if (adreno_use_lane_split && ne00 >= 64 && ne00 <= 128) { kernel = backend_ctx->kernel_mul_mat_f16_f32_l4_dr_lq; nrows = 1; @@ -19729,6 +21148,23 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co CL_CHECK(clSetKernelArg(kernel, 21, sizeof(int), &ne1)); CL_CHECK(clSetKernelArg(kernel, 22, sizeof(int), &r2)); CL_CHECK(clSetKernelArg(kernel, 23, sizeof(int), &r3)); + if (use_f16_mrow) { + const int MROW = 16; // must match MROW in mul_mv_f16_f32_mrow.cl + // rows-per-subgroup multiplier for the selected variant: + // 1/2/4 -> half4 register blocking; 8 -> half8(1 row); 16 -> half8(2 rows) + const int rpt = backend_ctx->f16_mrow_rpt; + int rmul; + if (rpt == 16) rmul = (ne00 % 8 == 0) ? 2 : 1; + else if (rpt == 8) rmul = 1; + else rmul = rpt; // 1,2,4 + const int rows_per_wg = MROW * rmul; + // __local activation buffer: ne00 floats, rounded up for float4 access + CL_CHECK(clSetKernelArg(kernel, 24, sizeof(float) * ((ne00 + 3) / 4 * 4), nullptr)); + size_t mrow_global[] = { (size_t)((ne01 + rows_per_wg - 1) / rows_per_wg) * 64, (size_t)ne11 * MROW, (size_t)ne12 * ne13 }; + size_t mrow_local[] = { 64, (size_t)MROW, 1 }; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, mrow_global, mrow_local, dst); + return; + } break; case GGML_TYPE_Q1_0: { #ifdef GGML_OPENCL_SOA_Q diff --git a/ggml/src/ggml-opencl/kernels/cvt.cl b/ggml/src/ggml-opencl/kernels/cvt.cl index 3d6cff7cff01..acc8f980763f 100644 --- a/ggml/src/ggml-opencl/kernels/cvt.cl +++ b/ggml/src/ggml-opencl/kernels/cvt.cl @@ -1110,6 +1110,78 @@ kernel void kernel_restore_block_q4_k_trans4_ns( } } +//------------------------------------------------------------------------------ +// kernel_convert_block_q4_k_tiled_ns +// +// Tiled-wide layout for the long-vocab q4_K lm_head/embed GEMV (decode path). +// Mirror of kernel_convert_block_q6_k_tiled_ns: recovers each weight's 4-bit +// code in CANONICAL ggml element order (e in [0,256)) and re-packs into 32 uints +// (8 codes/uint), stored TILED by 64 output rows so the matching GEMV +// (gemv_noshuffle_q4_k_f32_tiled) coalesces every weight load. The 12-byte +// packed scale block `s` and d/dm are stored per (row, K-block) tiled; the GEMV +// re-derives the 8 (scale,min) pairs via get_scale_min_k4, exactly like the o4 +// kernel. Both ends owned here -> correct by construction vs the reference q4_K +// dequant. Requires ne01 % 64 == 0 (gated host-side). Buffer sizes identical to +// the trans4_ns layout. +// +// q uint4 granule g of (row r, K-block sb): idx = ((rt*ne00_blk+sb)*8 + g)*64 + rit +// s (12 bytes) of (r, sb): idx = (rt*ne00_blk+sb)*64 + rit, *12 +// d/dm (half) of (r, sb): idx = (rt*ne00_blk+sb)*64 + rit +// where rt = r/64, rit = r%64. +//------------------------------------------------------------------------------ +kernel void kernel_convert_block_q4_k_tiled_ns( + __global struct block_q4_K * src0, + __global uint * dst_q, // 32 uints / superblock (4-bit codes, 8 codes/uint) + __global half * dst_d, // 1 half / superblock + __global half * dst_dm, // 1 half / superblock + __global uchar * dst_s, // K_SCALE_SIZE (12) bytes / superblock + uint ne00, + uint ne01 +) { + uint i00 = get_global_id(1); // K-block index (superblock along ne00) + uint i01 = get_global_id(0); // output row index (along ne01) + uint i02 = get_global_id(2); // batch + + uint ne00_blk = ne00 / QK_K; + + uint src_blk_offset = i00 + i01 * ne00_blk + i02 * ne00_blk * ne01; + __global struct block_q4_K * b = src0 + src_blk_offset; + + uint rt = i01 / 64; + uint rit = i01 % 64; + uint tile_blk = (i02 * (ne01 / 64) + rt) * ne00_blk + i00; + + // --- recover canonical 4-bit codes in e-order, pack 8 codes/uint --- + uint qw[32] = {0}; + for (uint e = 0; e < 256; ++e) { + uint g = e >> 6; // group 0..3 (q advances 32 bytes/group) + uint within = e & 63u; + uint hlf = within >> 5; // 0 = low nibble, 1 = high nibble + uint l = within & 31u; // 0..31 + uchar byte = b->q[g * 32u + l]; + uint code = (hlf == 0u) ? (uint)(byte & 0x0F) : (uint)(byte >> 4); + qw[e >> 3] |= code << ((e & 7u) * 4u); + } + + for (uint gr = 0; gr < 8; ++gr) { + uint base = (tile_blk * 8u + gr) * 64u + rit; // uint4 index + dst_q[base * 4u + 0u] = qw[gr * 4u + 0u]; + dst_q[base * 4u + 1u] = qw[gr * 4u + 1u]; + dst_q[base * 4u + 2u] = qw[gr * 4u + 2u]; + dst_q[base * 4u + 3u] = qw[gr * 4u + 3u]; + } + + // packed scales (12 bytes), tiled per (row, block) + __global uchar * s_dst = dst_s + (tile_blk * 64u + rit) * K_SCALE_SIZE; + #pragma unroll + for (int i = 0; i < K_SCALE_SIZE; ++i) { + s_dst[i] = b->s[i]; + } + + dst_d [tile_blk * 64u + rit] = b->d; + dst_dm[tile_blk * 64u + rit] = b->dm; +} + kernel void kernel_convert_block_q5_k_trans4_ns( __global struct block_q5_K * src0, __global uint * dst_qs, @@ -1494,6 +1566,105 @@ kernel void kernel_restore_block_mxfp4_trans( b->e = src_e[src_blk_offset]; } +//------------------------------------------------------------------------------ +// kernel_convert_block_q6_k_tiled_ns +// +// Tiled-wide layout for the long-vocab q6_K lm_head/embed GEMV (decode path). +// Unlike *_trans4_ns (which mirrors the bit-interleave the legacy 2-output GEMV +// consumes), this kernel is correct-by-construction against the CANONICAL ggml +// q6_K dequant: it recovers each weight's 6-bit code in element order e in +// [0,256), then re-packs low-4-bits into 32 uints (8 codes/uint) and high-2-bits +// into 16 uints (16 codes/uint). The matching GEMV (gemv_noshuffle_q6_k_f32_tiled) +// unpacks the same order, so both ends are owned here. +// +// Storage is TILED by 64 output rows so the GEMV's 64-thread tile coalesces: +// ql uint4 granule g of (row r, K-block sb): idx = ((rt*ne00_blk + sb)*8 + g)*64 + rit +// qh uint4 granule g: idx = ((rt*ne00_blk + sb)*4 + g)*64 + rit +// scales (char16) of (r, sb): idx = (rt*ne00_blk + sb)*64 + rit +// d (half) of (r, sb): idx = (rt*ne00_blk + sb)*64 + rit +// where rt = r/64, rit = r%64. Requires ne01 % 64 == 0 (gated host-side). +// Buffer sizes are byte-identical to the trans4_ns layout. +//------------------------------------------------------------------------------ +kernel void kernel_convert_block_q6_k_tiled_ns( + __global struct block_q6_K * src0, + __global uint * dst_ql, // 32 uints / superblock (low 4 bits, 8 codes/uint) + __global uint * dst_qh, // 16 uints / superblock (high 2 bits, 16 codes/uint) + __global half * dst_d, // 1 half / superblock + __global char * dst_s, // 16 chars/ superblock + uint ne00, + uint ne01 +) { + uint i00 = get_global_id(1); // K-block index (superblock along ne00) + uint i01 = get_global_id(0); // output row index (along ne01) + uint i02 = get_global_id(2); // batch + + uint ne00_blk = ne00 / QK_K; + + // Source block: row-major over (i02, i01, i00). + uint src_blk_offset = i00 + i01 * ne00_blk + i02 * ne00_blk * ne01; + __global struct block_q6_K * b = src0 + src_blk_offset; + + uint rt = i01 / 64; + uint rit = i01 % 64; + uint tile_blk = (i02 * (ne01 / 64) + rt) * ne00_blk + i00; // tile-major (row-tile, K-block) + + // --- recover canonical 6-bit codes, pack into ql (4b) + qh (2b) in e-order --- + // 32 ql-uints (8 low-nibbles each) + 16 qh-uints (16 2-bit slots each). + uint qlw[32] = {0}; + uint qhw[16] = {0}; + + for (uint e = 0; e < 256; ++e) { + uint n = (e >= 128) ? 1u : 0u; // which 128-half + uint within = e - n * 128u; + uint q = within / 32u; // quadrant 0..3 + uint l = within % 32u; // 0..31 + + uint off_ql = n * 64u; // raw ql byte base for this half + uint off_qh = n * 32u; // raw qh byte base for this half + + uchar low4; + uchar qlb0 = b->ql[off_ql + l]; + uchar qlb1 = b->ql[off_ql + l + 32]; + if (q == 0) low4 = qlb0 & 0x0F; + else if (q == 1) low4 = qlb1 & 0x0F; + else if (q == 2) low4 = (qlb0 >> 4) & 0x0F; + else low4 = (qlb1 >> 4) & 0x0F; + + uchar hi2 = (b->qh[off_qh + l] >> (q * 2u)) & 0x03; + + // pack low4 (e-order): uint e/8, nibble (e%8) + qlw[e >> 3] |= ((uint)low4) << ((e & 7u) * 4u); + // pack hi2 (e-order): uint e/16, 2-bit slot (e%16) + qhw[e >> 4] |= ((uint)hi2) << ((e & 15u) * 2u); + } + + // --- write tiled --- + for (uint g = 0; g < 8; ++g) { + uint base = (tile_blk * 8u + g) * 64u + rit; // uint4 index + dst_ql[base * 4u + 0u] = qlw[g * 4u + 0u]; + dst_ql[base * 4u + 1u] = qlw[g * 4u + 1u]; + dst_ql[base * 4u + 2u] = qlw[g * 4u + 2u]; + dst_ql[base * 4u + 3u] = qlw[g * 4u + 3u]; + } + for (uint g = 0; g < 4; ++g) { + uint base = (tile_blk * 4u + g) * 64u + rit; // uint4 index + dst_qh[base * 4u + 0u] = qhw[g * 4u + 0u]; + dst_qh[base * 4u + 1u] = qhw[g * 4u + 1u]; + dst_qh[base * 4u + 2u] = qhw[g * 4u + 2u]; + dst_qh[base * 4u + 3u] = qhw[g * 4u + 3u]; + } + + // scales: 16 chars contiguous per (row, block), tiled + __global char * s_dst = dst_s + (tile_blk * 64u + rit) * 16u; + #pragma unroll + for (int i = 0; i < 16; ++i) { + s_dst[i] = b->scales[i]; + } + + // super-block scale + dst_d[tile_blk * 64u + rit] = b->d; +} + kernel void kernel_convert_block_mxfp4_trans4_ns( global struct block_mxfp4 * src0, __global uint * dst_q, diff --git a/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_k_f32.cl b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_k_f32.cl index 22b4e9114628..c379a9a3998a 100644 --- a/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_k_f32.cl +++ b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_k_f32.cl @@ -4,6 +4,7 @@ #pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable #define ADRENO_GPU 1 #define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size("full"))) +#define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size("half"))) #endif #define QK_K 256 #define K_SCALE_SIZE 12 @@ -171,3 +172,319 @@ kernel void kernel_gemm_noshuffle_q4_k_f32( vstore4((float4)(c0.s7, c1.s7, c2.s7, c3.s7), 0, dst + idx); } } + +// 1x8 per-WI tile (1 output row x 8 output cols). For the small-batch +// (medium n_q, e.g. MTP/spec verify) path where the 2x8 kernel is starved: +// at ne1<=8 the grid is (1, ceil(M/2)) -> only ~M/256 workgroups, leaving +// the SP under-occupied. 1 row per WI doubles the M-axis workgroup count +// (ceil(M/1)/128 vs ceil(M/2)/128) AND collapses the accumulators to a +// single half8 (16 regs, no spill), so more waves co-reside. Same weight +// traffic as 2x8 (rows never share weights); the win is pure occupancy. +#ifdef ADRENO_GPU +REQD_SUBGROUP_SIZE_128 +#endif +kernel void kernel_gemm_noshuffle_q4_k_f32_r1( + global const ushort * src0_q, + global const uchar * src0_s, + global const half * src0_d, + global const half * src0_dm, + read_only image1d_buffer_t src1, + global float * dst, + ulong offsetd, + int m, + int n, + int k, + int n_no_padding, + uchar mask_d6, + uchar mask_d4, + uchar mask_hi2 +) { + dst = (global float *)((global char *)dst + offsetd); + int n_4 = n >> 2; + int gy = get_global_id(0); + int gx = get_global_id(1); // 1 row per WI + + half8 c0 = 0; + half8 B; + half dq; + + int num_blocks_K = k / QK_K; + + global const ushort * weight_ptr = src0_q + gx; + global const half * d_ptr = src0_d + gx; + global const half * dm_ptr = src0_dm + gx; + + for (int i = 0; i < k; i += 32) { + int sb_idx = i / QK_K; + int sub_idx = (i / 32) % 8; + + half dd = d_ptr [sb_idx * m]; + half dmm = dm_ptr[sb_idx * m]; + + global const uchar * sc0 = src0_s + sb_idx * K_SCALE_SIZE * m + gx; + + uchar sv0, mn0; + get_scale_min_k4(sub_idx, sc0, m, &sv0, &mn0, mask_d6, mask_d4, mask_hi2); + + half scale = convert_half(convert_float(dd) * (float)sv0); + half mval = convert_half(convert_float(dmm) * (float)mn0); + + for (int l = 0; l < 32; l += 4) { + int ki = i + l; + ushort bits = weight_ptr[(ki/4) * m]; + + B.s0123 = read_imageh(src1, gy*2 + (ki+0) * n_4); + B.s4567 = read_imageh(src1, gy*2+1 + (ki+0) * n_4); + dq = (bits & 0x000F) * scale - mval; + c0 += B * dq; + + B.s0123 = read_imageh(src1, gy*2 + (ki+1) * n_4); + B.s4567 = read_imageh(src1, gy*2+1 + (ki+1) * n_4); + dq = ((bits & 0x00F0) >> 4) * scale - mval; + c0 += B * dq; + + B.s0123 = read_imageh(src1, gy*2 + (ki+2) * n_4); + B.s4567 = read_imageh(src1, gy*2+1 + (ki+2) * n_4); + dq = ((bits & 0x0F00) >> 8) * scale - mval; + c0 += B * dq; + + B.s0123 = read_imageh(src1, gy*2 + (ki+3) * n_4); + B.s4567 = read_imageh(src1, gy*2+1 + (ki+3) * n_4); + dq = ((bits & 0xF000) >> 12) * scale - mval; + c0 += B * dq; + } + } + + // Output: 8 cols, 1 row per col-step. Scalar store, coalesced across + // neighbouring WIs (consecutive gx -> consecutive dst addresses). + int idx = (gy<<3)*m + gx; + if (idx < m*n_no_padding) { dst[idx] = c0.s0; idx += m; } + if (idx < m*n_no_padding) { dst[idx] = c0.s1; idx += m; } + if (idx < m*n_no_padding) { dst[idx] = c0.s2; idx += m; } + if (idx < m*n_no_padding) { dst[idx] = c0.s3; idx += m; } + if (idx < m*n_no_padding) { dst[idx] = c0.s4; idx += m; } + if (idx < m*n_no_padding) { dst[idx] = c0.s5; idx += m; } + if (idx < m*n_no_padding) { dst[idx] = c0.s6; idx += m; } + if (idx < m*n_no_padding) { dst[idx] = c0.s7; } +} + +// 2x8 tile, but weights read through an image1d_buffer (CL_R/UINT32 over the +// same packed-q buffer) instead of a plain global buffer. The ne1==1 GEMV +// already does this and is much faster per weight byte than this GEMM at +// small n_q; the structural difference is the image path hits the dedicated +// TPL1 weight cache (L1) while the global path only reaches L2. At small n_q +// the forward is weight-read-bound, so L1-cached weights is the lever. +// The 2 adjacent rows the 2x8 tile reads as a ushort2 are exactly one uint32, +// so the vload2 becomes a single read_imageui at index gx + (ki/4)*(m/2). +#ifdef ADRENO_GPU +REQD_SUBGROUP_SIZE_128 +#endif +kernel void kernel_gemm_noshuffle_q4_k_f32_kimg( + read_only image1d_buffer_t src0_q_img, + global const uchar * src0_s, + global const half * src0_d, + global const half * src0_dm, + read_only image1d_buffer_t src1, + global float * dst, + ulong offsetd, + int m, + int n, + int k, + int n_no_padding, + uchar mask_d6, + uchar mask_d4, + uchar mask_hi2 +) { + dst = (global float *)((global char *)dst + offsetd); + int n_4 = n >> 2; + int m_2 = m >> 1; + int gy = get_global_id(0); + int gx = get_global_id(1); + int gx_2 = gx << 1; + + half8 c0 = 0, c1 = 0; + half8 B; + half2 dequantized_weights; + + int num_blocks_K = k / QK_K; + + global const half * d_ptr = src0_d + gx_2; + global const half * dm_ptr = src0_dm + gx_2; + + for (int i = 0; i < k; i += 32) { + int sb_idx = i / QK_K; + int sub_idx = (i / 32) % 8; + + half2 d = vload2(0, d_ptr + sb_idx * m); + half2 dm = vload2(0, dm_ptr + sb_idx * m); + + global const uchar * sc0 = src0_s + sb_idx * K_SCALE_SIZE * m + (gx_2+0); + global const uchar * sc1 = sc0 + 1; + + uchar sv0, mn0, sv1, mn1; + get_scale_min_k4(sub_idx, sc0, m, &sv0, &mn0, mask_d6, mask_d4, mask_hi2); + get_scale_min_k4(sub_idx, sc1, m, &sv1, &mn1, mask_d6, mask_d4, mask_hi2); + + half2 scale = convert_half2(convert_float2(d) * convert_float2((uchar2)(sv0, sv1))); + half2 mval = convert_half2(convert_float2(dm) * convert_float2((uchar2)(mn0, mn1))); + + for (int l = 0; l < 32; l += 4) { + int ki = i + l; + uint wpacked = read_imageui(src0_q_img, gx + (ki/4) * m_2).x; + ushort2 bits2 = (ushort2)((ushort)(wpacked & 0xFFFFu), (ushort)(wpacked >> 16)); + + // j=0 + B.s0123 = read_imageh(src1, gy*2 + (ki+0) * n_4); + B.s4567 = read_imageh(src1, gy*2+1 + (ki+0) * n_4); + dequantized_weights.s0 = (bits2.s0 & 0x000F) * scale.s0 - mval.s0; + dequantized_weights.s1 = (bits2.s1 & 0x000F) * scale.s1 - mval.s1; + c0 += B * dequantized_weights.s0; + c1 += B * dequantized_weights.s1; + + // j=1 + B.s0123 = read_imageh(src1, gy*2 + (ki+1) * n_4); + B.s4567 = read_imageh(src1, gy*2+1 + (ki+1) * n_4); + dequantized_weights.s0 = ((bits2.s0 & 0x00F0) >> 4) * scale.s0 - mval.s0; + dequantized_weights.s1 = ((bits2.s1 & 0x00F0) >> 4) * scale.s1 - mval.s1; + c0 += B * dequantized_weights.s0; + c1 += B * dequantized_weights.s1; + + // j=2 + B.s0123 = read_imageh(src1, gy*2 + (ki+2) * n_4); + B.s4567 = read_imageh(src1, gy*2+1 + (ki+2) * n_4); + dequantized_weights.s0 = ((bits2.s0 & 0x0F00) >> 8) * scale.s0 - mval.s0; + dequantized_weights.s1 = ((bits2.s1 & 0x0F00) >> 8) * scale.s1 - mval.s1; + c0 += B * dequantized_weights.s0; + c1 += B * dequantized_weights.s1; + + // j=3 + B.s0123 = read_imageh(src1, gy*2 + (ki+3) * n_4); + B.s4567 = read_imageh(src1, gy*2+1 + (ki+3) * n_4); + dequantized_weights.s0 = ((bits2.s0 & 0xF000) >> 12) * scale.s0 - mval.s0; + dequantized_weights.s1 = ((bits2.s1 & 0xF000) >> 12) * scale.s1 - mval.s1; + c0 += B * dequantized_weights.s0; + c1 += B * dequantized_weights.s1; + } + } + + int idx = (gy<<3)*m + (gx<<1); + if (idx+1 < m*n_no_padding) { vstore2((float2)(c0.s0, c1.s0), 0, dst + idx); idx += m; } + if (idx+1 < m*n_no_padding) { vstore2((float2)(c0.s1, c1.s1), 0, dst + idx); idx += m; } + if (idx+1 < m*n_no_padding) { vstore2((float2)(c0.s2, c1.s2), 0, dst + idx); idx += m; } + if (idx+1 < m*n_no_padding) { vstore2((float2)(c0.s3, c1.s3), 0, dst + idx); idx += m; } + if (idx+1 < m*n_no_padding) { vstore2((float2)(c0.s4, c1.s4), 0, dst + idx); idx += m; } + if (idx+1 < m*n_no_padding) { vstore2((float2)(c0.s5, c1.s5), 0, dst + idx); idx += m; } + if (idx+1 < m*n_no_padding) { vstore2((float2)(c0.s6, c1.s6), 0, dst + idx); idx += m; } + if (idx+1 < m*n_no_padding) { vstore2((float2)(c0.s7, c1.s7), 0, dst + idx); } +} + +// Cooperative-K GEMM for the small-batch (n_q in [2..8]) path. Mirrors the +// ne1==1 GEMV's structure: a WG is (COK_SG lanes x COK_NSG subgroups); each +// lane owns ONE output row and computes its 8 (padded) columns, and the +// COK_NSG subgroups SPLIT the K reduction round-robin, combining via a +// __local reduction. This is the thing the per-WI GEMM lacked — at small n_q +// the old kernel had ~M/256 workgroups each walking all of K serially; this +// has M/64 workgroups AND COK_NSG-way K parallelism. Uses REQD_SUBGROUP_SIZE_64 +// + barrier (same safe reduction pattern as the GEMV; never sub_group_reduce +// at full width on X2 per the GDN miscompile note). +#define COK_NSG 8 +#define COK_SG 64 +#ifdef ADRENO_GPU +REQD_SUBGROUP_SIZE_64 +#endif +kernel void kernel_gemm_noshuffle_q4_k_f32_cok( + global const ushort * src0_q, + global const uchar * src0_s, + global const half * src0_d, + global const half * src0_dm, + read_only image1d_buffer_t src1, + global float * dst, + ulong offsetd, + int m, + int n, + int k, + int n_no_padding, + uchar mask_d6, + uchar mask_d4, + uchar mask_hi2 +) { + dst = (global float *)((global char *)dst + offsetd); + int n_4 = n >> 2; + int gx = get_global_id(0); // output row + int sg = get_local_id(1); // subgroup index (K-split lane) + int lane = get_local_id(0); // lane within subgroup (0..COK_SG-1) + + int num_blocks_K = k / QK_K; + int num_32blk = k / 32; + + global const ushort * weight_ptr = src0_q + gx; + global const half * d_ptr = src0_d + gx; + global const half * dm_ptr = src0_dm + gx; + + half8 acc = 0; + half8 B; + half dq; + + for (int blk = sg; blk < num_32blk; blk += COK_NSG) { + int i = blk << 5; // blk * 32 + int sb_idx = blk >> 3; // (blk*32) / QK_K (QK_K = 256 = 32*8) + int sub_idx = blk & 7; // (i/32) % 8 + + half dd = d_ptr [sb_idx * m]; + half dmm = dm_ptr[sb_idx * m]; + + global const uchar * sc0 = src0_s + sb_idx * K_SCALE_SIZE * m + gx; + uchar sv0, mn0; + get_scale_min_k4(sub_idx, sc0, m, &sv0, &mn0, mask_d6, mask_d4, mask_hi2); + half scale = convert_half(convert_float(dd) * (float)sv0); + half mval = convert_half(convert_float(dmm) * (float)mn0); + + for (int l = 0; l < 32; l += 4) { + int ki = i + l; + ushort bits = weight_ptr[(ki>>2) * m]; + + B.s0123 = read_imageh(src1, (ki+0) * n_4); + B.s4567 = read_imageh(src1, 1 + (ki+0) * n_4); + dq = (bits & 0x000F) * scale - mval; + acc += B * dq; + + B.s0123 = read_imageh(src1, (ki+1) * n_4); + B.s4567 = read_imageh(src1, 1 + (ki+1) * n_4); + dq = ((bits & 0x00F0) >> 4) * scale - mval; + acc += B * dq; + + B.s0123 = read_imageh(src1, (ki+2) * n_4); + B.s4567 = read_imageh(src1, 1 + (ki+2) * n_4); + dq = ((bits & 0x0F00) >> 8) * scale - mval; + acc += B * dq; + + B.s0123 = read_imageh(src1, (ki+3) * n_4); + B.s4567 = read_imageh(src1, 1 + (ki+3) * n_4); + dq = ((bits & 0xF000) >> 12) * scale - mval; + acc += B * dq; + } + } + + // cross-subgroup reduction over the K-split (float for accuracy) + local float8 reduceLM[COK_SG * (COK_NSG - 1)]; + if (sg > 0) { + reduceLM[(sg - 1) * COK_SG + lane] = convert_float8(acc); + } + barrier(CLK_LOCAL_MEM_FENCE); + + if (sg == 0) { + float8 sum = convert_float8(acc); + for (int s = 0; s < COK_NSG - 1; s++) { + sum += reduceLM[s * COK_SG + lane]; + } + int idx = gx; + if (idx < m*n_no_padding) { dst[idx] = sum.s0; idx += m; } + if (idx < m*n_no_padding) { dst[idx] = sum.s1; idx += m; } + if (idx < m*n_no_padding) { dst[idx] = sum.s2; idx += m; } + if (idx < m*n_no_padding) { dst[idx] = sum.s3; idx += m; } + if (idx < m*n_no_padding) { dst[idx] = sum.s4; idx += m; } + if (idx < m*n_no_padding) { dst[idx] = sum.s5; idx += m; } + if (idx < m*n_no_padding) { dst[idx] = sum.s6; idx += m; } + if (idx < m*n_no_padding) { dst[idx] = sum.s7; } + } +} diff --git a/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q6_k_f32.cl b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q6_k_f32.cl index 3a9c624508a7..141f6a2f6880 100644 --- a/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q6_k_f32.cl +++ b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q6_k_f32.cl @@ -5,6 +5,7 @@ #pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable #define ADRENO_GPU 1 #define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size("full"))) +#define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size("half"))) #endif #ifdef ADRENO_GPU @@ -138,3 +139,107 @@ kernel void kernel_gemm_noshuffle_q6_K_f32( vstore4((float4)(c0.s7, c1.s7, c2.s7, c3.s7), 0, dst + idx); } } + +// Cooperative-K q6_K GEMM for the small-batch (n_q in [2..8]) path. Same idea +// as the q4_K _cok kernel: WG = (COK_SG lanes x COK_NSG subgroups), each lane +// owns ONE output row (half8 over the 8 padded cols), and the COK_NSG +// subgroups split the K iterations round-robin and combine via a __local +// reduction. Replaces the default 4-row-per-WI tile that walked all of K alone +// (~M/512 WGs + serial reduction) at small n_q. REQD_SUBGROUP_SIZE_64 + +// barrier (never sub_group_reduce at full width on X2). +#define COK_NSG 8 +#define COK_SG 64 +#ifdef ADRENO_GPU +REQD_SUBGROUP_SIZE_64 +#endif +kernel void kernel_gemm_noshuffle_q6_K_f32_cok( + global const ushort * src0_ql, + global const uchar * src0_qh, + global const ushort * src0_s, + global const half * src0_d, + read_only image1d_buffer_t src1, + global float * dst, + ulong offsetd, + int m, + int n, + int k, + int n_no_padding, + ushort mask_f000, + uchar mask_c0 +) { + dst = (global float *)( (global char *)dst + offsetd ); + + int n_4 = n >> 2; + int gx = get_global_id(0); // output row + int sg = get_local_id(1); // subgroup index (K-split) + int lane = get_local_id(0); // lane within subgroup + + global const ushort * ptr_ql = src0_ql + gx; + global const uchar * ptr_qh = src0_qh + gx; + global const ushort * ptr_s = src0_s + gx; + global const half * ptr_d = src0_d + gx; + + half8 acc = 0; + half8 B; + half dq; + + int num_iter = k >> 2; // k/4 iterations, 4 k-values each + + for (int ib = sg; ib < num_iter; ib += COK_NSG) { + int i = ib << 2; // ib * 4 + + ushort bits4 = ptr_ql[ib * m]; // ql for row gx at this 4-block + uchar bits2 = ptr_qh[ib * m]; // qh + + ushort s_packed = ptr_s[(i >> 5) * m]; // (i/16/2) = i/32 + char2 sc2 = as_char2(s_packed); + char scale_s = (((i >> 4) & 1) == 0) ? sc2.s0 : sc2.s1; // (i/16)%2 + half scale_d = ptr_d[(i >> 8) * m]; // i/256 + + // j=0 + B.s0123 = read_imageh(src1, (i + 0)*n_4 + 0); + B.s4567 = read_imageh(src1, (i + 0)*n_4 + 1); + dq = (convert_half((bits4 & 0x000F) | ((bits2 & 0x03) << 4)) - 32.f) * scale_s * scale_d; + acc += B * dq; + + // j=1 + B.s0123 = read_imageh(src1, (i + 1)*n_4 + 0); + B.s4567 = read_imageh(src1, (i + 1)*n_4 + 1); + dq = (convert_half(((bits4 & 0x00F0) >> 4) | ((bits2 & 0x0C) << 2)) - 32.f) * scale_s * scale_d; + acc += B * dq; + + // j=2 + B.s0123 = read_imageh(src1, (i + 2)*n_4 + 0); + B.s4567 = read_imageh(src1, (i + 2)*n_4 + 1); + dq = (convert_half(((bits4 & 0x0F00) >> 8) | (bits2 & 0x30)) - 32.f) * scale_s * scale_d; + acc += B * dq; + + // j=3 + B.s0123 = read_imageh(src1, (i + 3)*n_4 + 0); + B.s4567 = read_imageh(src1, (i + 3)*n_4 + 1); + dq = (convert_half(((bits4 & mask_f000) >> 12) | ((bits2 & mask_c0) >> 2)) - 32.f) * scale_s * scale_d; + acc += B * dq; + } + + local float8 reduceLM[COK_SG * (COK_NSG - 1)]; + if (sg > 0) { + reduceLM[(sg - 1) * COK_SG + lane] = convert_float8(acc); + } + barrier(CLK_LOCAL_MEM_FENCE); + + if (sg == 0) { + float8 sum = convert_float8(acc); + for (int s = 0; s < COK_NSG - 1; s++) { + sum += reduceLM[s * COK_SG + lane]; + } + int idx = gx; + if (idx < m*n_no_padding) { dst[idx] = sum.s0; idx += m; } + if (idx < m*n_no_padding) { dst[idx] = sum.s1; idx += m; } + if (idx < m*n_no_padding) { dst[idx] = sum.s2; idx += m; } + if (idx < m*n_no_padding) { dst[idx] = sum.s3; idx += m; } + if (idx < m*n_no_padding) { dst[idx] = sum.s4; idx += m; } + if (idx < m*n_no_padding) { dst[idx] = sum.s5; idx += m; } + if (idx < m*n_no_padding) { dst[idx] = sum.s6; idx += m; } + if (idx < m*n_no_padding) { dst[idx] = sum.s7; } + } +} diff --git a/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q6_k_f32_tiled.cl b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q6_k_f32_tiled.cl new file mode 100644 index 000000000000..ffd943a27811 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q6_k_f32_tiled.cl @@ -0,0 +1,136 @@ +// Batched (N>1) q6_K GEMM over the 64-row-TILED canonical layout produced by +// kernel_convert_block_q6_k_tiled_ns (cvt.cl). Companion to the decode kernel +// kernel_gemv_noshuffle_q6_K_f32_tiled: SAME pack, SAME canonical e-order +// dequant (correct by construction vs reference ggml q6_K), extended to N output +// columns. Makes the batched lm_head/embed (perplexity, spec-decode verify, +// batched serving) correct on GPU while keeping the tiled convert the fast decode +// GEMV depends on. +// +// One work-item owns one output ROW for a block of BN columns. A work-group is +// {64 lanes, NTILES subgroups} = NTILES*64 rows; the global z dimension tiles the +// N columns by BN. Each work-item computes its row's FULL K (no K-split, so no +// cross-subgroup reduction), which lets the whole work-group share one staged +// activation block: +// +// __local activation staging — the BN columns of the current superblock (BN*256 +// floats) are loaded into __local once per superblock, cooperatively by all +// NTILES*64 work-items, then every row reads its activation from __local. This +// removes the ~Nrows-fold redundant image reads of the first version (each lane +// re-read the activation), which made the batched GEMM ~2x slower than the plain +// noshuffle GEMM. +// +// Weights are read from __global (coalesced) — matching the decode kernel; the +// lm_head weight is streamed with little reuse where coalesced global beats the +// Adreno texture cache. + +#pragma OPENCL EXTENSION cl_khr_fp16 : enable + +#ifdef cl_qcom_reqd_sub_group_size +#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable +#define ADRENO_GPU 1 +#define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size("half"))) +#endif + +#define NTILES 4 // 64-row tiles per work-group (NTILES*64 = 256 rows) +#define TILE_ROWS 64 +#define BN 16 // output columns handled per work-group (global z step) +#define WG_THREADS (NTILES * TILE_ROWS) + +#if defined(ADRENO_GPU) +REQD_SUBGROUP_SIZE_64 +#endif +kernel void kernel_gemm_noshuffle_q6_K_f32_tiled( + __global uint4 * src0_ql, // tiled: 8 uint4 granules / superblock + __global uint4 * src0_qh, // tiled: 4 uint4 granules / superblock + __global char * src0_s, // tiled: 16 chars / superblock + __global half * src0_d, // tiled: 1 half / superblock + read_only image1d_buffer_t src1, // activation [ne00, ne11] f32 (RGBA), column-major + global float * dst, + ulong offsetd, + int ne00, + int ne01, + int ne11 +) { + int rit = get_local_id(0); // 0..63 (lane within a tile; coalesces weight loads) + int sg = get_local_id(1); // 0..NTILES-1 + int lid = sg * TILE_ROWS + rit; // 0..WG_THREADS-1 (flat local id) + int row = get_group_id(0) * WG_THREADS + lid; + int rt = row / TILE_ROWS; // global 64-row tile index + int col0 = get_global_id(2) * BN; // first output column of this block + + int nb = ne00 / 256; // superblocks per row + int act_col_stride = ne00 / 4; // activation float4 pixels per column + + const bool row_ok = row < ne01; + + // staged activation: BN columns x 256 elements for the current superblock + __local float lact[BN * 256]; + + float acc[BN]; + #pragma unroll + for (int j = 0; j < BN; ++j) acc[j] = 0.0f; + + for (int sb = 0; sb < nb; ++sb) { + // cooperatively stage BN columns' 256 activation elements (= BN*64 float4) + for (int p = lid; p < BN * 64; p += WG_THREADS) { + int j = p >> 6; // column within the BN block (p / 64) + int e4 = p & 63; // element-quad within the column (p % 64) + int c = col0 + j; + float4 v = (c < ne11) + ? read_imagef(src1, c * act_col_stride + sb * 64 + e4) + : (float4)(0.0f); + lact[p * 4 + 0] = v.x; + lact[p * 4 + 1] = v.y; + lact[p * 4 + 2] = v.z; + lact[p * 4 + 3] = v.w; // lact[j*256 + e], e = e4*4 + t + } + barrier(CLK_LOCAL_MEM_FENCE); + + if (row_ok) { + int tile_blk = rt * nb + sb; // ne02 == 1 for lm_head/embed + + float dval = (float)src0_d[tile_blk * TILE_ROWS + rit]; + __global char * sc = src0_s + (tile_blk * TILE_ROWS + rit) * 16; + + uint ql[32]; + uint qh[16]; + #pragma unroll + for (int g = 0; g < 8; ++g) { + uint4 v = src0_ql[(tile_blk * 8 + g) * TILE_ROWS + rit]; + ql[g*4+0] = v.x; ql[g*4+1] = v.y; ql[g*4+2] = v.z; ql[g*4+3] = v.w; + } + #pragma unroll + for (int g = 0; g < 4; ++g) { + uint4 v = src0_qh[(tile_blk * 4 + g) * TILE_ROWS + rit]; + qh[g*4+0] = v.x; qh[g*4+1] = v.y; qh[g*4+2] = v.z; qh[g*4+3] = v.w; + } + + // NOTE: the e loop (256) is deliberately NOT unrolled. Fully unrolling + // 256*BN MACs overflows the in-process Adreno compiler (host stack + // overflow at clBuildProgram, same class as the FA DK=512 OOM). + for (int e = 0; e < 256; ++e) { + uint low4 = (ql[e >> 3] >> ((e & 7) * 4)) & 0xF; + uint hi2 = (qh[e >> 4] >> ((e & 15) * 2)) & 0x3; + int code = (int)(low4 | (hi2 << 4)) - 32; + int sidx = ((e >> 7) << 3) + (((e >> 5) & 3) << 1) + ((e >> 4) & 1); + float cs = (float)code * (float)sc[sidx] * dval; + #pragma unroll + for (int j = 0; j < BN; ++j) { + acc[j] += cs * lact[j * 256 + e]; + } + } + } + barrier(CLK_LOCAL_MEM_FENCE); + } + + if (row_ok) { + dst = (global float*)((global char*)dst + offsetd); + #pragma unroll + for (int j = 0; j < BN; ++j) { + int c = col0 + j; + if (c < ne11) { + dst[(ulong)c * ne01 + row] = acc[j]; + } + } + } +} diff --git a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_0_f32.cl b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_0_f32.cl index 8de0de1cc3a4..023e848f734d 100644 --- a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_0_f32.cl +++ b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_0_f32.cl @@ -277,3 +277,107 @@ __kernel void kernel_gemv_noshuffle_q4_0_f32( } } + +// Multi-column (N in [2..4]) variant of the q4_0 decode GEMV, for the speculative +// / MTP verify batch (n_cols = 2..4 = drafted + bonus positions). Routes the small- +// batch verify OFF the transposed-GEMM dead-zone (gemm_noshuffle_q4_0) onto the +// efficient GEMV path. Each K-block's weights (regA hi+lo) are loaded ONCE and +// reused across the n_cols activation columns. Per-column accumulation is +// independent and identical to n_cols standalone GEMVs. n_cols==3 is byte-identical +// to the original mc3 (col3 disabled, slots 6/7 stay zero). Kept the _mc3 name. +#ifdef VECTOR_SUB_GROUP_BROADCAST +#define MC_DQ_HI dequantizeBlockAccum_ns_sgbroadcast_8_hi +#define MC_DQ_LO dequantizeBlockAccum_ns_sgbroadcast_8_lo +#else +#define MC_DQ_HI dequantizeBlockAccum_ns_sgbroadcast_1_hi +#define MC_DQ_LO dequantizeBlockAccum_ns_sgbroadcast_1_lo +#endif +// One column c: load this column's activation (own brace scope so the macros' +// `shared_y` decl is re-scoped), then dequant (hi+lo) against the shared weights. +#define MC_COL_Q40(ts, c) \ + { if (slid < 4) { regB.s0123 = read_imagef(src1, (c)*COL_STRIDE + slid*2 + k*8); \ + regB.s4567 = read_imagef(src1, (c)*COL_STRIDE + 1 + slid*2 + k*8); } \ + MC_DQ_HI(ts, as_ushort8(regA_hi), regS, regB); \ + MC_DQ_LO(ts, as_ushort8(regA_lo), regS, regB); } + +#ifdef ADRENO_GPU +REQD_SUBGROUP_SIZE_64 +#endif +__kernel void kernel_gemv_noshuffle_q4_0_f32_mc3( + __read_only image1d_buffer_t src0_q, // quantized A + global half2 * src0_d, // A scales + __read_only image1d_buffer_t src1, // B (n_cols columns, col-major image) + global float * dst, // C (column-major [M x n_cols]) + ulong offsetd, + int ne00, // K + int ne01, // M + int n_cols) // N (2..4) +{ + uint groupId = get_local_id(1); + uint gid = get_global_id(0); + ushort slid = get_sub_group_local_id(); + + uint K = ne00; + uint M = ne01; + + uint LINE_STRIDE_A = M / 2; + // BLOCK_STRIDE_A is the LAYOUT stride between consecutive K-blocks = 4 uints + // per q4_0 block * M (set by the trans4_ns convert). The "4" is uints/block, NOT + // the subgroup count — keep it fixed so the K-split count (nsg) can vary. + uint BLOCK_STRIDE_A = N_SIMDGROUP * M; // = 4 * M (N_SIMDGROUP is the #define 4) + uint COL_STRIDE = K / 4; // float4 pixels per activation column + uint nsg = get_local_size(1); // runtime K-split (4 default, 8 small-M) + + __private uint4 regA_hi, regA_lo; + __private half2 regS; + __private float8 regB; + + __private float2 ts0 = (float2)(0.0f); + __private float2 ts1 = (float2)(0.0f); + __private float2 ts2 = (float2)(0.0f); + __private float2 ts3 = (float2)(0.0f); + + for (uint k = groupId; k < (K / QK4_0); k += nsg) { + regS = src0_d[gid + k * LINE_STRIDE_A]; + + // weights loaded ONCE, reused across the columns + regA_hi.s0 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 0)).x; + regA_hi.s1 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 1)).x; + regA_hi.s2 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 2)).x; + regA_hi.s3 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 3)).x; + regA_lo.s0 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 4)).x; + regA_lo.s1 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 5)).x; + regA_lo.s2 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 6)).x; + regA_lo.s3 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 7)).x; + + MC_COL_Q40(ts0, 0); + MC_COL_Q40(ts1, 1); + if (n_cols > 2) MC_COL_Q40(ts2, 2); + if (n_cols > 3) MC_COL_Q40(ts3, 3); + } + + // cross-subgroup reduce over nsg subgroups: pack the (up to 4) columns' float2 + // into a float8. Generalized to runtime nsg (4 default, 8 for small-M). Each + // subgroup writes its partial; subgroup 0 sums the rest into its own acc. At + // nsg==4 this is byte-identical to the original (sums subgroups 1,2,3 in order). + __local float8 reduceLM[SIMDGROUP_WIDTH * 8]; + float8 acc = (float8)(ts0.s0, ts0.s1, ts1.s0, ts1.s1, ts2.s0, ts2.s1, ts3.s0, ts3.s1); + reduceLM[groupId * SIMDGROUP_WIDTH + slid] = acc; + + barrier(CLK_LOCAL_MEM_FENCE); + + if (groupId == 0) { + for (uint g = 1; g < nsg; g++) { + acc += reduceLM[g * SIMDGROUP_WIDTH + slid]; + } + dst = (global float*)((global char*)dst + offsetd); + // dst is column-major [M rows x n_cols cols]: (row, col) at col*M + row + vstore2((float2)(acc.s0, acc.s1), 0, &(dst[0 * M + gid * 2])); + vstore2((float2)(acc.s2, acc.s3), 0, &(dst[1 * M + gid * 2])); + if (n_cols > 2) vstore2((float2)(acc.s4, acc.s5), 0, &(dst[2 * M + gid * 2])); + if (n_cols > 3) vstore2((float2)(acc.s6, acc.s7), 0, &(dst[3 * M + gid * 2])); + } +} +#undef MC_COL_Q40 +#undef MC_DQ_HI +#undef MC_DQ_LO diff --git a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_1_f32.cl b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_1_f32.cl index 5fa3127806a6..2ccf4214c0bb 100644 --- a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_1_f32.cl +++ b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_1_f32.cl @@ -286,3 +286,99 @@ kernel void kernel_gemv_noshuffle_q4_1_f32( } } + +// Multi-column (N in [2..4]) variant of the q4_1 decode GEMV (spec/MTP verify) = +// q4_0 mc3 + the q4_1 per-block min (regM; dequant = q*scale + minv). n_cols=2..4; +// routes the small-batch verify OFF the gemm_noshuffle_q4_1 dead-zone. n_cols==3 is +// byte-identical to the original mc3. NB: this file spells the vec-broadcast define +// BROADCAT (no S) — match it so the fast _8 path compiles. +#ifdef VECTOR_SUB_GROUP_BROADCAT +#define MC_DQ1_HI dequantizeBlockAccum_ns_sgbroadcast_8_hi +#define MC_DQ1_LO dequantizeBlockAccum_ns_sgbroadcast_8_lo +#else +#define MC_DQ1_HI dequantizeBlockAccum_ns_sgbroadcast_1_hi +#define MC_DQ1_LO dequantizeBlockAccum_ns_sgbroadcast_1_lo +#endif +#define MC_COL_Q41(ts, c) \ + { if (slid < 4) { regB.s0123 = read_imagef(src1, (c)*COL_STRIDE + slid*2 + k*8); \ + regB.s4567 = read_imagef(src1, (c)*COL_STRIDE + 1 + slid*2 + k*8); } \ + MC_DQ1_HI(ts, as_ushort8(regA_hi), regS, regM, regB); \ + MC_DQ1_LO(ts, as_ushort8(regA_lo), regS, regM, regB); } +#ifdef ADRENO_GPU +REQD_SUBGROUP_SIZE_64 +#endif +kernel void kernel_gemv_noshuffle_q4_1_f32_mc3( + read_only image1d_buffer_t src0_q, + global half2 * src0_d, + global half2 * src0_m, + read_only image1d_buffer_t src1, + global float * dst, + ulong offsetd, + int ne00, + int ne01, + int n_cols) +{ + uint groupId = get_local_id(1); + uint gid = get_global_id(0); + ushort slid = get_sub_group_local_id(); + + uint K = ne00; + uint M = ne01; + + uint LINE_STRIDE_A = M / 2; + uint BLOCK_STRIDE_A = NSUBGROUPS * M; + uint COL_STRIDE = K / 4; // float4 pixels per activation column + + private uint4 regA_hi, regA_lo; + private half2 regS, regM; + private float8 regB; + + private float2 ts0 = (float2)(0.0f); + private float2 ts1 = (float2)(0.0f); + private float2 ts2 = (float2)(0.0f); + private float2 ts3 = (float2)(0.0f); + + for (uint k = groupId; k < (K / QK4_0); k += NSUBGROUPS) { + regS = src0_d[gid + k * LINE_STRIDE_A]; + regM = src0_m[gid + k * LINE_STRIDE_A]; + + // weights loaded ONCE, reused across the columns + regA_hi.s0 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 0)).x; + regA_hi.s1 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 1)).x; + regA_hi.s2 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 2)).x; + regA_hi.s3 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 3)).x; + regA_lo.s0 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 4)).x; + regA_lo.s1 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 5)).x; + regA_lo.s2 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 6)).x; + regA_lo.s3 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 7)).x; + + MC_COL_Q41(ts0, 0); + MC_COL_Q41(ts1, 1); + if (n_cols > 2) MC_COL_Q41(ts2, 2); + if (n_cols > 3) MC_COL_Q41(ts3, 3); + } + + // cross-subgroup reduce: pack the (up to 4) columns' float2 into a float8. + local float8 reduceLM[SUBGROUP_SIZE * 3]; + float8 acc = (float8)(ts0.s0, ts0.s1, ts1.s0, ts1.s1, ts2.s0, ts2.s1, ts3.s0, ts3.s1); + if (groupId == 1) { reduceLM[SUBGROUP_SIZE * 0 + slid] = acc; } + if (groupId == 2) { reduceLM[SUBGROUP_SIZE * 1 + slid] = acc; } + if (groupId == 3) { reduceLM[SUBGROUP_SIZE * 2 + slid] = acc; } + + barrier(CLK_LOCAL_MEM_FENCE); + + if (groupId == 0) { + acc += reduceLM[SUBGROUP_SIZE * 0 + slid]; + acc += reduceLM[SUBGROUP_SIZE * 1 + slid]; + acc += reduceLM[SUBGROUP_SIZE * 2 + slid]; + dst = (global float*)((global char*)dst + offsetd); + // dst is column-major [M rows x n_cols cols]: (row, col) at col*M + row + vstore2((float2)(acc.s0, acc.s1), 0, &(dst[0 * M + gid * 2])); + vstore2((float2)(acc.s2, acc.s3), 0, &(dst[1 * M + gid * 2])); + if (n_cols > 2) vstore2((float2)(acc.s4, acc.s5), 0, &(dst[2 * M + gid * 2])); + if (n_cols > 3) vstore2((float2)(acc.s6, acc.s7), 0, &(dst[3 * M + gid * 2])); + } +} +#undef MC_COL_Q41 +#undef MC_DQ1_HI +#undef MC_DQ1_LO diff --git a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_k_f32.cl b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_k_f32.cl index c1829fc38208..ef4e466da214 100644 --- a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_k_f32.cl +++ b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_k_f32.cl @@ -228,12 +228,20 @@ kernel void kernel_gemv_noshuffle_q4_k_f32( uint groupId = get_local_id(1); uint gid = get_global_id(0); ushort slid = get_sub_group_local_id(); + // K-split factor = #subgroups in the WG. Read from the launch (NOT a compile + // constant) so small-M projections (Kcur/Vcur/Qcur) can dispatch a wider + // K-split (more waves/SP -> latency hiding) while large-M keeps 4. The + // physical weight layout stride below is INDEPENDENT of this (see BLOCK_STRIDE_A). + uint nsg = get_local_size(1); uint K = ne00; uint M = ne01; uint LINE_STRIDE_A = M / 2; - uint BLOCK_STRIDE_A = NSUBGROUPS * M; + // Physical per-K-block stride in the packed image: 8 uints/block-row-pair * + // (M/2) row-pairs = 4*M uints. This is a layout constant, not tied to nsg. + uint BLOCK_STRIDE_A = 4 * M; + uint scales_per_row = (K / QK_K) * 12; private uint4 regA; private half2 regS; @@ -242,7 +250,7 @@ kernel void kernel_gemv_noshuffle_q4_k_f32( private float2 totalSum = (float2)(0.0f); - for (uint k = groupId; k < (K / 32); k += NSUBGROUPS) { + for (uint k = groupId; k < (K / 32); k += nsg) { uint sb = k / 8; uint j = k % 8; @@ -286,28 +294,21 @@ kernel void kernel_gemv_noshuffle_q4_k_f32( #endif // VECTOR_SUB_GROUP_BROADCAST } - // reduction in local memory, assumes #wave=4 - local float2 reduceLM[SUBGROUP_SIZE * 3]; - if (groupId == 1) { - reduceLM[SUBGROUP_SIZE * 0 + slid] = totalSum; - } - if (groupId == 2) { - reduceLM[SUBGROUP_SIZE * 1 + slid] = totalSum; - } - if (groupId == 3) { - reduceLM[SUBGROUP_SIZE * 2 + slid] = totalSum; + // Cross-subgroup reduction in local memory. Generalized to nsg subgroups + // (was a hard-coded 4-wave unroll). Sized for up to 16 subgroups (the widest + // K-split we dispatch for small M). At nsg==4 the accumulation order is + // identical to the original unroll -> byte-identical for the large-M path. + local float2 reduceLM[SUBGROUP_SIZE * 15]; + if (groupId > 0) { + reduceLM[SUBGROUP_SIZE * (groupId - 1) + slid] = totalSum; } barrier(CLK_LOCAL_MEM_FENCE); if (groupId == 0) { - totalSum += reduceLM[SUBGROUP_SIZE * 0 + slid]; - } - if (groupId == 0) { - totalSum += reduceLM[SUBGROUP_SIZE * 1 + slid]; - } - if (groupId == 0) { - totalSum += reduceLM[SUBGROUP_SIZE * 2 + slid]; + for (uint i = 0; i < nsg - 1; ++i) { + totalSum += reduceLM[SUBGROUP_SIZE * i + slid]; + } } // 2 outputs per fiber in wave 0 @@ -322,3 +323,484 @@ kernel void kernel_gemv_noshuffle_q4_k_f32( } } + +// --- Fused gate+up GEMV + GLU epilogue (FFN) ------------------------------------ +// Folds the FFN's two decode GEMVs (ffn_gate, ffn_up) and the following GLU into a +// SINGLE dispatch: {MUL_MAT(Wg,x), MUL_MAT(Wu,x), GLU}. Both matmuls share the same +// activation x (ffn_norm), so the activation image read is issued ONCE per K-block +// and reused for the gate and up dot products (the per-op path re-reads it twice and +// also materializes the two full ffn-wide intermediates to global, which the GLU +// then re-reads). The gate/up partial sums are accumulated in the SAME per-fiber +// order and reduced in the SAME cross-subgroup order as the standalone GEMV, and the +// GLU formula is the exact scalar expression from kernels/glu.cl, so the output is +// BYTE-IDENTICAL to the per-op matmul+matmul+glu path -> safe to default on. +// glu_op: REGLU=0, GEGLU=1, SWIGLU=2, GEGLU_ERF=4, GEGLU_QUICK=5 (ggml_glu_op). +// Weights: src0g_* = gate (= GLU src[0]); src0u_* = up (= GLU src[1]). +#define GLU_GEGLU_COEF_A 0.044715f +#define GLU_SQRT_2_OVER_PI 0.79788456080286535587989211986876f +#define GLU_SQRT_2_INV 0.70710678118654752440084436210484f +#define GLU_QUICK_COEF -1.702f + +inline float glu_apply(int glu_op, float g, float u) { + float act; + if (glu_op == 1) { // GEGLU (tanh-approx gelu) + act = 0.5f*g*(1.0f + tanh(GLU_SQRT_2_OVER_PI*g*(1.0f + GLU_GEGLU_COEF_A*g*g))); + } else if (glu_op == 2) { // SWIGLU (silu) + act = g / (1.0f + exp(-g)); + } else if (glu_op == 0) { // REGLU + return g*u*(g > 0.0f); + } else if (glu_op == 4) { // GEGLU_ERF + act = 0.5f*g*(1.0f + erf(g*GLU_SQRT_2_INV)); + } else { // GEGLU_QUICK (glu_op == 5) + act = g*(1.0f/(1.0f + exp(GLU_QUICK_COEF*g))); + } + return act*u; +} + +#ifdef ADRENO_GPU +REQD_SUBGROUP_SIZE_64 +#endif +kernel void kernel_gemv_noshuffle_q4_k_f32_glu( + read_only image1d_buffer_t src0g_q, + global half2 * src0g_d, + global half2 * src0g_m, + global uchar * src0g_s, + read_only image1d_buffer_t src0u_q, + global half2 * src0u_d, + global half2 * src0u_m, + global uchar * src0u_s, + read_only image1d_buffer_t src1, + global float * dst, + ulong offsetd, + int ne00, + int ne01, + int glu_op, + uchar mask_d6, + uchar mask_d4, + uchar mask_hi2) +{ + uint groupId = get_local_id(1); + uint gid = get_global_id(0); + ushort slid = get_sub_group_local_id(); + uint nsg = get_local_size(1); + + uint K = ne00; + uint M = ne01; + + uint LINE_STRIDE_A = M / 2; + uint BLOCK_STRIDE_A = 4 * M; + + private uint4 regA; + private half2 regS, regM; + private float8 regB; + + private float2 gateSum = (float2)(0.0f); + private float2 upSum = (float2)(0.0f); + + // Two SEQUENTIAL K-loops (gate fully, then up). Keeping only one weight's + // working set live at a time holds the kernel's register footprint at ~the + // base single-weight GEMV's, so its max WG stays 1024 (16 subgroups) and the + // per-subgroup K-split matches the standalone wide GEMV exactly -> the gate + // and up partial sums are BYTE-IDENTICAL to the per-op path. The macro body + // is the base kernel's inner loop verbatim, parameterized by weight source. +#define Q4K_GLU_LOOP(SUM, Q, DD, MM, SS) \ + for (uint k = groupId; k < (K / 32); k += nsg) { \ + uint sb = k / 8; \ + uint j = k % 8; \ + half2 d = DD[gid + sb * LINE_STRIDE_A]; \ + half2 dm = MM[gid + sb * LINE_STRIDE_A]; \ + global const uchar * sc0 = SS + sb * 12 * M + 2 * gid; \ + global const uchar * sc1 = sc0 + 1; \ + uchar sv0, mn0, sv1, mn1; \ + get_scale_min_k4(j, sc0, M, &sv0, &mn0, mask_d6, mask_d4, mask_hi2); \ + get_scale_min_k4(j, sc1, M, &sv1, &mn1, mask_d6, mask_d4, mask_hi2); \ + regS = convert_half2(convert_float2(d) * convert_float2((uchar2)(sv0, sv1))); \ + regM = convert_half2(convert_float2(dm) * convert_float2((uchar2)(mn0, mn1))); \ + if (slid < 4) { \ + regB.s0123 = read_imagef(src1, (slid * 2 + k * 8)); \ + regB.s4567 = read_imagef(src1, (1 + slid * 2 + k * 8)); \ + } \ + regA.s0 = read_imageui(Q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 0)).x; \ + regA.s1 = read_imageui(Q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 1)).x; \ + regA.s2 = read_imageui(Q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 2)).x; \ + regA.s3 = read_imageui(Q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 3)).x; \ + DEQ_HI(SUM, as_ushort8(regA), regS, regM, regB); \ + regA.s0 = read_imageui(Q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 4)).x; \ + regA.s1 = read_imageui(Q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 5)).x; \ + regA.s2 = read_imageui(Q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 6)).x; \ + regA.s3 = read_imageui(Q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 7)).x; \ + DEQ_LO(SUM, as_ushort8(regA), regS, regM, regB); \ + } + +#ifdef VECTOR_SUB_GROUP_BROADCAST +#define DEQ_HI dequantizeBlockAccum_ns_sgbroadcast_8_hi +#define DEQ_LO dequantizeBlockAccum_ns_sgbroadcast_8_lo +#else +#define DEQ_HI dequantizeBlockAccum_ns_sgbroadcast_1_hi +#define DEQ_LO dequantizeBlockAccum_ns_sgbroadcast_1_lo +#endif + + Q4K_GLU_LOOP(gateSum, src0g_q, src0g_d, src0g_m, src0g_s) + Q4K_GLU_LOOP(upSum, src0u_q, src0u_d, src0u_m, src0u_s) + +#undef DEQ_HI +#undef DEQ_LO +#undef Q4K_GLU_LOOP + + // Cross-subgroup reduction in local memory. Packs gate (xy) + up (zw) into a + // float4 so both reduce in one pass; summation order matches the base GEMV's + // per-channel loop -> byte-identical partial sums. + local float4 reduceLM[SUBGROUP_SIZE * 15]; + if (groupId > 0) { + reduceLM[SUBGROUP_SIZE * (groupId - 1) + slid] = (float4)(gateSum, upSum); + } + barrier(CLK_LOCAL_MEM_FENCE); + if (groupId == 0) { + for (uint i = 0; i < nsg - 1; ++i) { + float4 p = reduceLM[SUBGROUP_SIZE * i + slid]; + gateSum += p.xy; + upSum += p.zw; + } + dst = (global float*)((global char*)dst + offsetd); + dst[gid * 2 + 0] = glu_apply(glu_op, gateSum.s0, upSum.s0); + dst[gid * 2 + 1] = glu_apply(glu_op, gateSum.s1, upSum.s1); + } +} + +// --- Split-K-across-workgroups decode GEMV (small-M projections) ---------------- +// A single-token GEMV makes only ceil(M/2/64) workgroups; a WG runs on one Adreno +// compute unit, so for small M (Kcur/Vcur, M=512 -> 4 WGs) most of the 16 CUs sit +// idle and the matmul is bandwidth-starved even with a wide intra-WG K-split. This +// variant adds a SECOND grid dimension of `ksplit` workgroups that each reduce a +// disjoint slice of K and write a per-slice partial; kernel_gemv_splitk_reduce_f32 +// then sums the partials into dst. Identical math/layout to the base kernel +// (physical block stride 4*M, get_scale_min_k4) -> coherent. Gated host-side to +// M<=1024 (M>=2048 +// already fills the CUs and the extra reduce dispatch only hurts). +#ifdef ADRENO_GPU +REQD_SUBGROUP_SIZE_64 +#endif +kernel void kernel_gemv_noshuffle_q4_k_f32_splitk( + read_only image1d_buffer_t src0_q, + global half2 * src0_d, + global half2 * src0_m, + global uchar * src0_s, + read_only image1d_buffer_t src1, + global float * partial, // [ksplit * M], slice-major + int ne00, + int ne01, + uchar mask_d6, + uchar mask_d4, + uchar mask_hi2) +{ + uint groupId = get_local_id(1); + uint gid = get_global_id(0); + ushort slid = get_sub_group_local_id(); + uint nsg = get_local_size(1); + uint ksplit = get_num_groups(1); + uint kslice = get_group_id(1); + + uint K = ne00; + uint M = ne01; + uint LINE_STRIDE_A = M / 2; + uint BLOCK_STRIDE_A = 4 * M; // physical, independent of the K-split + + private uint4 regA; + private half2 regS, regM; + private float8 regB; + private float2 totalSum = (float2)(0.0f); + + // each (kslice, subgroup) pair owns a disjoint set of K-blocks + for (uint k = kslice * nsg + groupId; k < (K / 32); k += ksplit * nsg) { + uint sb = k / 8; + uint j = k % 8; + half2 d = src0_d[gid + sb * LINE_STRIDE_A]; + half2 dm = src0_m[gid + sb * LINE_STRIDE_A]; + global const uchar * sc0 = src0_s + sb * 12 * M + 2 * gid; + global const uchar * sc1 = sc0 + 1; + uchar sv0, mn0, sv1, mn1; + get_scale_min_k4(j, sc0, M, &sv0, &mn0, mask_d6, mask_d4, mask_hi2); + get_scale_min_k4(j, sc1, M, &sv1, &mn1, mask_d6, mask_d4, mask_hi2); + regS = convert_half2(convert_float2(d) * convert_float2((uchar2)(sv0, sv1))); + regM = convert_half2(convert_float2(dm) * convert_float2((uchar2)(mn0, mn1))); + if (slid < 4) { + regB.s0123 = read_imagef(src1, (slid * 2 + k * 8)); + regB.s4567 = read_imagef(src1, (1 + slid * 2 + k * 8)); + } + regA.s0 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 0)).x; + regA.s1 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 1)).x; + regA.s2 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 2)).x; + regA.s3 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 3)).x; +#ifdef VECTOR_SUB_GROUP_BROADCAST + dequantizeBlockAccum_ns_sgbroadcast_8_hi(totalSum, as_ushort8(regA), regS, regM, regB); +#else + dequantizeBlockAccum_ns_sgbroadcast_1_hi(totalSum, as_ushort8(regA), regS, regM, regB); +#endif + regA.s0 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 4)).x; + regA.s1 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 5)).x; + regA.s2 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 6)).x; + regA.s3 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 7)).x; +#ifdef VECTOR_SUB_GROUP_BROADCAST + dequantizeBlockAccum_ns_sgbroadcast_8_lo(totalSum, as_ushort8(regA), regS, regM, regB); +#else + dequantizeBlockAccum_ns_sgbroadcast_1_lo(totalSum, as_ushort8(regA), regS, regM, regB); +#endif + } + + local float2 reduceLM[SUBGROUP_SIZE * 15]; + if (groupId > 0) { + reduceLM[SUBGROUP_SIZE * (groupId - 1) + slid] = totalSum; + } + barrier(CLK_LOCAL_MEM_FENCE); + if (groupId == 0) { + for (uint i = 0; i < nsg - 1; ++i) { + totalSum += reduceLM[SUBGROUP_SIZE * i + slid]; + } + vstore2(totalSum, 0, &(partial[kslice * M + gid * 2])); + } +} + +// Sum the per-slice partials [ksplit * M] into dst[M]; applies the dst byte offset. +kernel void kernel_gemv_splitk_reduce_f32( + global float * partial, + global float * dst, + ulong offsetd, + int ne01, // M + int ksplit) +{ + uint r = get_global_id(0); + if (r >= (uint)ne01) return; + float acc = 0.0f; + for (uint s = 0; s < (uint)ksplit; ++s) { + acc += partial[s * (uint)ne01 + r]; + } + dst = (global float*)((global char*)dst + offsetd); + dst[r] = acc; +} + + +// --- Dequant-once macros for the mc3 verify GEMV (Q4K_MC3_DEQUANT_ONCE) --- +// The inline dequantizeBlockAccum_* macros recompute the dequantized weight +// ((code & mask)>>shift)*scale - minv ONCE PER COLUMN (3x), and the flat +// 32-FMA unroll spills ~430 B of temporaries. These macros split the work: +// DEQUANT_Q4K_BLOCK computes the 16 weights/row of one 32-block ONCE into a +// half2[] (row0 in .s0, row1 in .s1) — stored as half, the exact type the +// inline expression yields (int*half-half), so no extra rounding. MAC_Q4K_BLOCK +// then accumulates them against a column's broadcast activation in the SAME +// per-accumulator order as the inline macro. Each weight value and each +// accumulator's add-chain is bit-for-bit identical => byte-identical output, +// while the dequant ALU drops 3x->1x and the live set shrinks. Requires the +// Qualcomm vector sub_group_broadcast (float8); enabled opt-in on Adreno. +#define DEQ_Q4K_HALF2(b0, b1, msk, sh, scale, minv) \ + (half2)( ((b0 & msk) >> sh) * scale.s0 - minv.s0, \ + ((b1 & msk) >> sh) * scale.s1 - minv.s1 ) + +#define DEQUANT_Q4K_BLOCK(wq, bits, scale, minv) \ + wq[0] = DEQ_Q4K_HALF2(bits.s0, bits.s1, 0x000F, 0, scale, minv); \ + wq[1] = DEQ_Q4K_HALF2(bits.s0, bits.s1, 0x00F0, 4, scale, minv); \ + wq[2] = DEQ_Q4K_HALF2(bits.s0, bits.s1, 0x0F00, 8, scale, minv); \ + wq[3] = DEQ_Q4K_HALF2(bits.s0, bits.s1, 0xF000, 12, scale, minv); \ + wq[4] = DEQ_Q4K_HALF2(bits.s2, bits.s3, 0x000F, 0, scale, minv); \ + wq[5] = DEQ_Q4K_HALF2(bits.s2, bits.s3, 0x00F0, 4, scale, minv); \ + wq[6] = DEQ_Q4K_HALF2(bits.s2, bits.s3, 0x0F00, 8, scale, minv); \ + wq[7] = DEQ_Q4K_HALF2(bits.s2, bits.s3, 0xF000, 12, scale, minv); \ + wq[8] = DEQ_Q4K_HALF2(bits.s4, bits.s5, 0x000F, 0, scale, minv); \ + wq[9] = DEQ_Q4K_HALF2(bits.s4, bits.s5, 0x00F0, 4, scale, minv); \ + wq[10] = DEQ_Q4K_HALF2(bits.s4, bits.s5, 0x0F00, 8, scale, minv); \ + wq[11] = DEQ_Q4K_HALF2(bits.s4, bits.s5, 0xF000, 12, scale, minv); \ + wq[12] = DEQ_Q4K_HALF2(bits.s6, bits.s7, 0x000F, 0, scale, minv); \ + wq[13] = DEQ_Q4K_HALF2(bits.s6, bits.s7, 0x00F0, 4, scale, minv); \ + wq[14] = DEQ_Q4K_HALF2(bits.s6, bits.s7, 0x0F00, 8, scale, minv); \ + wq[15] = DEQ_Q4K_HALF2(bits.s6, bits.s7, 0xF000, 12, scale, minv); + +// ln0/ln1 = the two source lanes whose activation float8 this block consumes +// (0,1 for the hi block, 2,3 for the lo block — matching the inline _hi/_lo). +#define MAC_Q4K_BLOCK(ts, wq, y, ln0, ln1) { \ + float8 sy = sub_group_broadcast(y, ln0); \ + ts.s0 += wq[0].s0*sy.s0; ts.s0 += wq[1].s0*sy.s1; ts.s0 += wq[2].s0*sy.s2; ts.s0 += wq[3].s0*sy.s3; \ + ts.s0 += wq[4].s0*sy.s4; ts.s0 += wq[5].s0*sy.s5; ts.s0 += wq[6].s0*sy.s6; ts.s0 += wq[7].s0*sy.s7; \ + ts.s1 += wq[0].s1*sy.s0; ts.s1 += wq[1].s1*sy.s1; ts.s1 += wq[2].s1*sy.s2; ts.s1 += wq[3].s1*sy.s3; \ + ts.s1 += wq[4].s1*sy.s4; ts.s1 += wq[5].s1*sy.s5; ts.s1 += wq[6].s1*sy.s6; ts.s1 += wq[7].s1*sy.s7; \ + sy = sub_group_broadcast(y, ln1); \ + ts.s0 += wq[8].s0*sy.s0; ts.s0 += wq[9].s0*sy.s1; ts.s0 += wq[10].s0*sy.s2; ts.s0 += wq[11].s0*sy.s3; \ + ts.s0 += wq[12].s0*sy.s4; ts.s0 += wq[13].s0*sy.s5; ts.s0 += wq[14].s0*sy.s6; ts.s0 += wq[15].s0*sy.s7; \ + ts.s1 += wq[8].s1*sy.s0; ts.s1 += wq[9].s1*sy.s1; ts.s1 += wq[10].s1*sy.s2; ts.s1 += wq[11].s1*sy.s3; \ + ts.s1 += wq[12].s1*sy.s4; ts.s1 += wq[13].s1*sy.s5; ts.s1 += wq[14].s1*sy.s6; ts.s1 += wq[15].s1*sy.s7; \ +} + +// Multi-column (N=3) variant of the q4_K decode GEMV, for the speculative / +// MTP verify batch (ne1=3 = 2 drafts + 1 bonus). Stays on the efficient GEMV +// path (subgroup-broadcast activation, NSUBGROUPS K-split) instead of the +// transposed-GEMM dead-zone path. Each K-block's weights (regA_hi/regA_lo) are +// loaded ONCE and reused across all 3 activation columns — same weight traffic +// as one decode, ~3x the (cheap) dequant ALU. Per-column accumulation is +// independent and identical to 3 standalone GEMVs => byte-identical, so it does +// NOT perturb the lm_head logits / spec accept rate. +#ifdef ADRENO_GPU +REQD_SUBGROUP_SIZE_64 +#endif +kernel void kernel_gemv_noshuffle_q4_k_f32_mc3( + read_only image1d_buffer_t src0_q, + global half2 * src0_d, + global half2 * src0_m, + global uchar * src0_s, + read_only image1d_buffer_t src1, + global float * dst, + ulong offsetd, + int ne00, + int ne01, + uchar mask_d6, + uchar mask_d4, + uchar mask_hi2) +{ + uint groupId = get_local_id(1); + uint gid = get_global_id(0); + ushort slid = get_sub_group_local_id(); + + uint K = ne00; + uint M = ne01; + + uint LINE_STRIDE_A = M / 2; + uint BLOCK_STRIDE_A = NSUBGROUPS * M; + uint COL_STRIDE = K / 4; // float4 pixels per activation column + + private uint4 regA_hi, regA_lo; + private half2 regS, regM; + private float8 regB; + + private float2 ts0 = (float2)(0.0f); + private float2 ts1 = (float2)(0.0f); + private float2 ts2 = (float2)(0.0f); + +#ifdef Q4K_MC3_DEQUANT_LDS + // One 16-half2 block buffer per WI (reused hi->lo): forces the dequantized + // weights into LDS instead of private arrays (which spill to slow global on + // Adreno). 64*NSUBGROUPS WIs * 16 half2 = 16 KB; each WI owns its own slot + // range (flat*16) -> no cross-lane sharing, no barrier needed. + local half2 wstage[SUBGROUP_SIZE * NSUBGROUPS * 16]; + local half2 * ws = wstage + (groupId * SUBGROUP_SIZE + slid) * 16; +#endif + + for (uint k = groupId; k < (K / 32); k += NSUBGROUPS) { + uint sb = k / 8; + uint j = k % 8; + + half2 d = src0_d[gid + sb * LINE_STRIDE_A]; + half2 dm = src0_m[gid + sb * LINE_STRIDE_A]; + + global const uchar * sc0 = src0_s + sb * 12 * M + 2 * gid; + global const uchar * sc1 = sc0 + 1; + + uchar sv0, mn0, sv1, mn1; + get_scale_min_k4(j, sc0, M, &sv0, &mn0, mask_d6, mask_d4, mask_hi2); + get_scale_min_k4(j, sc1, M, &sv1, &mn1, mask_d6, mask_d4, mask_hi2); + + regS = convert_half2(convert_float2(d) * convert_float2((uchar2)(sv0, sv1))); + regM = convert_half2(convert_float2(dm) * convert_float2((uchar2)(mn0, mn1))); + + // weights loaded ONCE, reused across the 3 columns + regA_hi.s0 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 0)).x; + regA_hi.s1 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 1)).x; + regA_hi.s2 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 2)).x; + regA_hi.s3 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 3)).x; + regA_lo.s0 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 4)).x; + regA_lo.s1 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 5)).x; + regA_lo.s2 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 6)).x; + regA_lo.s3 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 7)).x; + +#ifdef Q4K_MC3_DEQUANT_ONCE + // Dequant the 32 weights/row (16 hi + 16 lo) ONCE into half2[] (byte- + // identical to the inline intermediate), then MAC against each column's + // activation. Drops the dequant ALU 3x->1x and the macro-temp spill. + half2 wq_hi[16], wq_lo[16]; + DEQUANT_Q4K_BLOCK(wq_hi, as_ushort8(regA_hi), regS, regM); + DEQUANT_Q4K_BLOCK(wq_lo, as_ushort8(regA_lo), regS, regM); + { if (slid < 4) { regB.s0123 = read_imagef(src1, 0*COL_STRIDE + slid*2 + k*8); + regB.s4567 = read_imagef(src1, 0*COL_STRIDE + 1 + slid*2 + k*8); } + MAC_Q4K_BLOCK(ts0, wq_hi, regB, 0, 1); MAC_Q4K_BLOCK(ts0, wq_lo, regB, 2, 3); } + { if (slid < 4) { regB.s0123 = read_imagef(src1, 1*COL_STRIDE + slid*2 + k*8); + regB.s4567 = read_imagef(src1, 1*COL_STRIDE + 1 + slid*2 + k*8); } + MAC_Q4K_BLOCK(ts1, wq_hi, regB, 0, 1); MAC_Q4K_BLOCK(ts1, wq_lo, regB, 2, 3); } + { if (slid < 4) { regB.s0123 = read_imagef(src1, 2*COL_STRIDE + slid*2 + k*8); + regB.s4567 = read_imagef(src1, 2*COL_STRIDE + 1 + slid*2 + k*8); } + MAC_Q4K_BLOCK(ts2, wq_hi, regB, 0, 1); MAC_Q4K_BLOCK(ts2, wq_lo, regB, 2, 3); } +#elif defined(Q4K_MC3_DEQUANT_LDS) + // LDS-staged dequant: dequant a 32-block ONCE into the per-WI LDS slot + // (hi pass then lo pass, overwriting), MAC each column from LDS. ts* + // receive hi-then-lo in the same order as DEQUANT_ONCE -> byte-identical. + // Activations reloaded per pass (cheap, imaged); only one regB + 0 weight + // regs live -> the weight working set lives in LDS, not spilled private. + DEQUANT_Q4K_BLOCK(ws, as_ushort8(regA_hi), regS, regM); + { if (slid < 4) { regB.s0123 = read_imagef(src1, 0*COL_STRIDE + slid*2 + k*8); + regB.s4567 = read_imagef(src1, 0*COL_STRIDE + 1 + slid*2 + k*8); } + MAC_Q4K_BLOCK(ts0, ws, regB, 0, 1); } + { if (slid < 4) { regB.s0123 = read_imagef(src1, 1*COL_STRIDE + slid*2 + k*8); + regB.s4567 = read_imagef(src1, 1*COL_STRIDE + 1 + slid*2 + k*8); } + MAC_Q4K_BLOCK(ts1, ws, regB, 0, 1); } + { if (slid < 4) { regB.s0123 = read_imagef(src1, 2*COL_STRIDE + slid*2 + k*8); + regB.s4567 = read_imagef(src1, 2*COL_STRIDE + 1 + slid*2 + k*8); } + MAC_Q4K_BLOCK(ts2, ws, regB, 0, 1); } + DEQUANT_Q4K_BLOCK(ws, as_ushort8(regA_lo), regS, regM); + { if (slid < 4) { regB.s0123 = read_imagef(src1, 0*COL_STRIDE + slid*2 + k*8); + regB.s4567 = read_imagef(src1, 0*COL_STRIDE + 1 + slid*2 + k*8); } + MAC_Q4K_BLOCK(ts0, ws, regB, 2, 3); } + { if (slid < 4) { regB.s0123 = read_imagef(src1, 1*COL_STRIDE + slid*2 + k*8); + regB.s4567 = read_imagef(src1, 1*COL_STRIDE + 1 + slid*2 + k*8); } + MAC_Q4K_BLOCK(ts1, ws, regB, 2, 3); } + { if (slid < 4) { regB.s0123 = read_imagef(src1, 2*COL_STRIDE + slid*2 + k*8); + regB.s4567 = read_imagef(src1, 2*COL_STRIDE + 1 + slid*2 + k*8); } + MAC_Q4K_BLOCK(ts2, ws, regB, 2, 3); } +#else + // Per-column: load only this column's activation (single regB live at a + // time -> 1/3 the activation register pressure vs holding all 3) then + // dequant against the shared weights. Cuts the private-mem spill. +#ifdef VECTOR_SUB_GROUP_BROADCAST + { if (slid < 4) { regB.s0123 = read_imagef(src1, 0*COL_STRIDE + slid*2 + k*8); + regB.s4567 = read_imagef(src1, 0*COL_STRIDE + 1 + slid*2 + k*8); } + dequantizeBlockAccum_ns_sgbroadcast_8_hi(ts0, as_ushort8(regA_hi), regS, regM, regB); + dequantizeBlockAccum_ns_sgbroadcast_8_lo(ts0, as_ushort8(regA_lo), regS, regM, regB); } + { if (slid < 4) { regB.s0123 = read_imagef(src1, 1*COL_STRIDE + slid*2 + k*8); + regB.s4567 = read_imagef(src1, 1*COL_STRIDE + 1 + slid*2 + k*8); } + dequantizeBlockAccum_ns_sgbroadcast_8_hi(ts1, as_ushort8(regA_hi), regS, regM, regB); + dequantizeBlockAccum_ns_sgbroadcast_8_lo(ts1, as_ushort8(regA_lo), regS, regM, regB); } + { if (slid < 4) { regB.s0123 = read_imagef(src1, 2*COL_STRIDE + slid*2 + k*8); + regB.s4567 = read_imagef(src1, 2*COL_STRIDE + 1 + slid*2 + k*8); } + dequantizeBlockAccum_ns_sgbroadcast_8_hi(ts2, as_ushort8(regA_hi), regS, regM, regB); + dequantizeBlockAccum_ns_sgbroadcast_8_lo(ts2, as_ushort8(regA_lo), regS, regM, regB); } +#else + { if (slid < 4) { regB.s0123 = read_imagef(src1, 0*COL_STRIDE + slid*2 + k*8); + regB.s4567 = read_imagef(src1, 0*COL_STRIDE + 1 + slid*2 + k*8); } + dequantizeBlockAccum_ns_sgbroadcast_1_hi(ts0, as_ushort8(regA_hi), regS, regM, regB); + dequantizeBlockAccum_ns_sgbroadcast_1_lo(ts0, as_ushort8(regA_lo), regS, regM, regB); } + { if (slid < 4) { regB.s0123 = read_imagef(src1, 1*COL_STRIDE + slid*2 + k*8); + regB.s4567 = read_imagef(src1, 1*COL_STRIDE + 1 + slid*2 + k*8); } + dequantizeBlockAccum_ns_sgbroadcast_1_hi(ts1, as_ushort8(regA_hi), regS, regM, regB); + dequantizeBlockAccum_ns_sgbroadcast_1_lo(ts1, as_ushort8(regA_lo), regS, regM, regB); } + { if (slid < 4) { regB.s0123 = read_imagef(src1, 2*COL_STRIDE + slid*2 + k*8); + regB.s4567 = read_imagef(src1, 2*COL_STRIDE + 1 + slid*2 + k*8); } + dequantizeBlockAccum_ns_sgbroadcast_1_hi(ts2, as_ushort8(regA_hi), regS, regM, regB); + dequantizeBlockAccum_ns_sgbroadcast_1_lo(ts2, as_ushort8(regA_lo), regS, regM, regB); } +#endif +#endif // Q4K_MC3_DEQUANT_ONCE + } + + // cross-subgroup reduce: pack the 3 columns' float2 into a float8 (6 used). + local float8 reduceLM[SUBGROUP_SIZE * 3]; + float8 acc = (float8)(ts0.s0, ts0.s1, ts1.s0, ts1.s1, ts2.s0, ts2.s1, 0.0f, 0.0f); + if (groupId == 1) { reduceLM[SUBGROUP_SIZE * 0 + slid] = acc; } + if (groupId == 2) { reduceLM[SUBGROUP_SIZE * 1 + slid] = acc; } + if (groupId == 3) { reduceLM[SUBGROUP_SIZE * 2 + slid] = acc; } + + barrier(CLK_LOCAL_MEM_FENCE); + + if (groupId == 0) { + acc += reduceLM[SUBGROUP_SIZE * 0 + slid]; + acc += reduceLM[SUBGROUP_SIZE * 1 + slid]; + acc += reduceLM[SUBGROUP_SIZE * 2 + slid]; + dst = (global float*)((global char*)dst + offsetd); + // dst is column-major [M rows x 3 cols]: (row, col) at col*M + row + vstore2((float2)(acc.s0, acc.s1), 0, &(dst[0 * M + gid * 2])); + vstore2((float2)(acc.s2, acc.s3), 0, &(dst[1 * M + gid * 2])); + vstore2((float2)(acc.s4, acc.s5), 0, &(dst[2 * M + gid * 2])); + } +} diff --git a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_k_f32_o4.cl b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_k_f32_o4.cl new file mode 100644 index 000000000000..02916bb91ffa --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_k_f32_o4.cl @@ -0,0 +1,349 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#pragma OPENCL EXTENSION cl_khr_subgroups : enable + +#ifdef cl_qcom_reqd_sub_group_size +#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable +#define ADRENO_GPU 1 +#define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size("half"))) +#endif + +#define QK_K 256 +#define NSUBGROUPS 4 +#define SUBGROUP_SIZE 64 + +// scales are transposed: consecutive codes of a row are `stride` apart +inline void get_scale_min_k4( + int j, + global const uchar * q, + uint stride, + uchar * d, + uchar * m, + uchar mask_d6, + uchar mask_d4, + uchar mask_hi2 +) { + if (j < 4) { + *d = q[j*stride] & mask_d6; + *m = q[(j+4)*stride] & mask_d6; + } else { + *d = (q[(j+4)*stride] & mask_d4) | ((q[(j-4)*stride] & mask_hi2) >> 2); + *m = ((q[(j+4)*stride] >> 4) & mask_d4) | ((q[j*stride] & mask_hi2) >> 2); + } +} + +#define dequantizeBlockAccum_ns_sgbroadcast_1_hi(total_sums, bits4, scale, minv, y) \ + float shared_y; \ + shared_y = sub_group_broadcast(y.s0, 0); \ + total_sums.s0 += ((bits4.s0 & 0x000F) * scale.s0 - minv.s0) * shared_y; \ + total_sums.s1 += ((bits4.s1 & 0x000F) * scale.s1 - minv.s1) * shared_y; \ + shared_y = sub_group_broadcast(y.s1, 0); \ + total_sums.s0 += (((bits4.s0 & 0x00F0) >> 4) * scale.s0 - minv.s0) * shared_y; \ + total_sums.s1 += (((bits4.s1 & 0x00F0) >> 4) * scale.s1 - minv.s1) * shared_y; \ + shared_y = sub_group_broadcast(y.s2, 0); \ + total_sums.s0 += (((bits4.s0 & 0x0F00) >> 8) * scale.s0 - minv.s0) * shared_y; \ + total_sums.s1 += (((bits4.s1 & 0x0F00) >> 8) * scale.s1 - minv.s1) * shared_y; \ + shared_y = sub_group_broadcast(y.s3, 0); \ + total_sums.s0 += (((bits4.s0 & 0xF000) >> 12) * scale.s0 - minv.s0) * shared_y; \ + total_sums.s1 += (((bits4.s1 & 0xF000) >> 12) * scale.s1 - minv.s1) * shared_y; \ + shared_y = sub_group_broadcast(y.s4, 0); \ + total_sums.s0 += ((bits4.s2 & 0x000F) * scale.s0 - minv.s0) * shared_y; \ + total_sums.s1 += ((bits4.s3 & 0x000F) * scale.s1 - minv.s1) * shared_y; \ + shared_y = sub_group_broadcast(y.s5, 0); \ + total_sums.s0 += (((bits4.s2 & 0x00F0) >> 4) * scale.s0 - minv.s0) * shared_y; \ + total_sums.s1 += (((bits4.s3 & 0x00F0) >> 4) * scale.s1 - minv.s1) * shared_y; \ + shared_y = sub_group_broadcast(y.s6, 0); \ + total_sums.s0 += (((bits4.s2 & 0x0F00) >> 8) * scale.s0 - minv.s0) * shared_y; \ + total_sums.s1 += (((bits4.s3 & 0x0F00) >> 8) * scale.s1 - minv.s1) * shared_y; \ + shared_y = sub_group_broadcast(y.s7, 0); \ + total_sums.s0 += (((bits4.s2 & 0xF000) >> 12) * scale.s0 - minv.s0) * shared_y; \ + total_sums.s1 += (((bits4.s3 & 0xF000) >> 12) * scale.s1 - minv.s1) * shared_y; \ + shared_y = sub_group_broadcast(y.s0, 1); \ + total_sums.s0 += ((bits4.s4 & 0x000F) * scale.s0 - minv.s0) * shared_y; \ + total_sums.s1 += ((bits4.s5 & 0x000F) * scale.s1 - minv.s1) * shared_y; \ + shared_y = sub_group_broadcast(y.s1, 1); \ + total_sums.s0 += (((bits4.s4 & 0x00F0) >> 4) * scale.s0 - minv.s0) * shared_y; \ + total_sums.s1 += (((bits4.s5 & 0x00F0) >> 4) * scale.s1 - minv.s1) * shared_y; \ + shared_y = sub_group_broadcast(y.s2, 1); \ + total_sums.s0 += (((bits4.s4 & 0x0F00) >> 8) * scale.s0 - minv.s0) * shared_y; \ + total_sums.s1 += (((bits4.s5 & 0x0F00) >> 8) * scale.s1 - minv.s1) * shared_y; \ + shared_y = sub_group_broadcast(y.s3, 1); \ + total_sums.s0 += (((bits4.s4 & 0xF000) >> 12) * scale.s0 - minv.s0) * shared_y; \ + total_sums.s1 += (((bits4.s5 & 0xF000) >> 12) * scale.s1 - minv.s1) * shared_y; \ + shared_y = sub_group_broadcast(y.s4, 1); \ + total_sums.s0 += ((bits4.s6 & 0x000F) * scale.s0 - minv.s0) * shared_y; \ + total_sums.s1 += ((bits4.s7 & 0x000F) * scale.s1 - minv.s1) * shared_y; \ + shared_y = sub_group_broadcast(y.s5, 1); \ + total_sums.s0 += (((bits4.s6 & 0x00F0) >> 4) * scale.s0 - minv.s0) * shared_y; \ + total_sums.s1 += (((bits4.s7 & 0x00F0) >> 4) * scale.s1 - minv.s1) * shared_y; \ + shared_y = sub_group_broadcast(y.s6, 1); \ + total_sums.s0 += (((bits4.s6 & 0x0F00) >> 8) * scale.s0 - minv.s0) * shared_y; \ + total_sums.s1 += (((bits4.s7 & 0x0F00) >> 8) * scale.s1 - minv.s1) * shared_y; \ + shared_y = sub_group_broadcast(y.s7, 1); \ + total_sums.s0 += (((bits4.s6 & 0xF000) >> 12) * scale.s0 - minv.s0) * shared_y; \ + total_sums.s1 += (((bits4.s7 & 0xF000) >> 12) * scale.s1 - minv.s1) * shared_y; \ + + +#define dequantizeBlockAccum_ns_sgbroadcast_1_lo(total_sums, bits4, scale, minv, y) \ + shared_y = sub_group_broadcast(y.s0, 2); \ + total_sums.s0 += ((bits4.s0 & 0x000F) * scale.s0 - minv.s0) * shared_y; \ + total_sums.s1 += ((bits4.s1 & 0x000F) * scale.s1 - minv.s1) * shared_y; \ + shared_y = sub_group_broadcast(y.s1, 2); \ + total_sums.s0 += (((bits4.s0 & 0x00F0) >> 4) * scale.s0 - minv.s0) * shared_y; \ + total_sums.s1 += (((bits4.s1 & 0x00F0) >> 4) * scale.s1 - minv.s1) * shared_y; \ + shared_y = sub_group_broadcast(y.s2, 2); \ + total_sums.s0 += (((bits4.s0 & 0x0F00) >> 8) * scale.s0 - minv.s0) * shared_y; \ + total_sums.s1 += (((bits4.s1 & 0x0F00) >> 8) * scale.s1 - minv.s1) * shared_y; \ + shared_y = sub_group_broadcast(y.s3, 2); \ + total_sums.s0 += (((bits4.s0 & 0xF000) >> 12) * scale.s0 - minv.s0) * shared_y; \ + total_sums.s1 += (((bits4.s1 & 0xF000) >> 12) * scale.s1 - minv.s1) * shared_y; \ + shared_y = sub_group_broadcast(y.s4, 2); \ + total_sums.s0 += ((bits4.s2 & 0x000F) * scale.s0 - minv.s0) * shared_y; \ + total_sums.s1 += ((bits4.s3 & 0x000F) * scale.s1 - minv.s1) * shared_y; \ + shared_y = sub_group_broadcast(y.s5, 2); \ + total_sums.s0 += (((bits4.s2 & 0x00F0) >> 4) * scale.s0 - minv.s0) * shared_y; \ + total_sums.s1 += (((bits4.s3 & 0x00F0) >> 4) * scale.s1 - minv.s1) * shared_y; \ + shared_y = sub_group_broadcast(y.s6, 2); \ + total_sums.s0 += (((bits4.s2 & 0x0F00) >> 8) * scale.s0 - minv.s0) * shared_y; \ + total_sums.s1 += (((bits4.s3 & 0x0F00) >> 8) * scale.s1 - minv.s1) * shared_y; \ + shared_y = sub_group_broadcast(y.s7, 2); \ + total_sums.s0 += (((bits4.s2 & 0xF000) >> 12) * scale.s0 - minv.s0) * shared_y; \ + total_sums.s1 += (((bits4.s3 & 0xF000) >> 12) * scale.s1 - minv.s1) * shared_y; \ + shared_y = sub_group_broadcast(y.s0, 3); \ + total_sums.s0 += ((bits4.s4 & 0x000F) * scale.s0 - minv.s0) * shared_y; \ + total_sums.s1 += ((bits4.s5 & 0x000F) * scale.s1 - minv.s1) * shared_y; \ + shared_y = sub_group_broadcast(y.s1, 3); \ + total_sums.s0 += (((bits4.s4 & 0x00F0) >> 4) * scale.s0 - minv.s0) * shared_y; \ + total_sums.s1 += (((bits4.s5 & 0x00F0) >> 4) * scale.s1 - minv.s1) * shared_y; \ + shared_y = sub_group_broadcast(y.s2, 3); \ + total_sums.s0 += (((bits4.s4 & 0x0F00) >> 8) * scale.s0 - minv.s0) * shared_y; \ + total_sums.s1 += (((bits4.s5 & 0x0F00) >> 8) * scale.s1 - minv.s1) * shared_y; \ + shared_y = sub_group_broadcast(y.s3, 3); \ + total_sums.s0 += (((bits4.s4 & 0xF000) >> 12) * scale.s0 - minv.s0) * shared_y; \ + total_sums.s1 += (((bits4.s5 & 0xF000) >> 12) * scale.s1 - minv.s1) * shared_y; \ + shared_y = sub_group_broadcast(y.s4, 3); \ + total_sums.s0 += ((bits4.s6 & 0x000F) * scale.s0 - minv.s0) * shared_y; \ + total_sums.s1 += ((bits4.s7 & 0x000F) * scale.s1 - minv.s1) * shared_y; \ + shared_y = sub_group_broadcast(y.s5, 3); \ + total_sums.s0 += (((bits4.s6 & 0x00F0) >> 4) * scale.s0 - minv.s0) * shared_y; \ + total_sums.s1 += (((bits4.s7 & 0x00F0) >> 4) * scale.s1 - minv.s1) * shared_y; \ + shared_y = sub_group_broadcast(y.s6, 3); \ + total_sums.s0 += (((bits4.s6 & 0x0F00) >> 8) * scale.s0 - minv.s0) * shared_y; \ + total_sums.s1 += (((bits4.s7 & 0x0F00) >> 8) * scale.s1 - minv.s1) * shared_y; \ + shared_y = sub_group_broadcast(y.s7, 3); \ + total_sums.s0 += (((bits4.s6 & 0xF000) >> 12) * scale.s0 - minv.s0) * shared_y; \ + total_sums.s1 += (((bits4.s7 & 0xF000) >> 12) * scale.s1 - minv.s1) * shared_y; \ + + +#define dequantizeBlockAccum_ns_sgbroadcast_8_hi(total_sums, bits4, scale, minv, y) \ + float8 shared_y; \ + shared_y = sub_group_broadcast(y, 0); \ + total_sums.s0 += ((bits4.s0 & 0x000F) * scale.s0 - minv.s0) * shared_y.s0; \ + total_sums.s0 += (((bits4.s0 & 0x00F0) >> 4) * scale.s0 - minv.s0) * shared_y.s1; \ + total_sums.s0 += (((bits4.s0 & 0x0F00) >> 8) * scale.s0 - minv.s0) * shared_y.s2; \ + total_sums.s0 += (((bits4.s0 & 0xF000) >> 12) * scale.s0 - minv.s0) * shared_y.s3; \ + total_sums.s0 += ((bits4.s2 & 0x000F) * scale.s0 - minv.s0) * shared_y.s4; \ + total_sums.s0 += (((bits4.s2 & 0x00F0) >> 4) * scale.s0 - minv.s0) * shared_y.s5; \ + total_sums.s0 += (((bits4.s2 & 0x0F00) >> 8) * scale.s0 - minv.s0) * shared_y.s6; \ + total_sums.s0 += (((bits4.s2 & 0xF000) >> 12) * scale.s0 - minv.s0) * shared_y.s7; \ + total_sums.s1 += ((bits4.s1 & 0x000F) * scale.s1 - minv.s1) * shared_y.s0; \ + total_sums.s1 += (((bits4.s1 & 0x00F0) >> 4) * scale.s1 - minv.s1) * shared_y.s1; \ + total_sums.s1 += (((bits4.s1 & 0x0F00) >> 8) * scale.s1 - minv.s1) * shared_y.s2; \ + total_sums.s1 += (((bits4.s1 & 0xF000) >> 12) * scale.s1 - minv.s1) * shared_y.s3; \ + total_sums.s1 += ((bits4.s3 & 0x000F) * scale.s1 - minv.s1) * shared_y.s4; \ + total_sums.s1 += (((bits4.s3 & 0x00F0) >> 4) * scale.s1 - minv.s1) * shared_y.s5; \ + total_sums.s1 += (((bits4.s3 & 0x0F00) >> 8) * scale.s1 - minv.s1) * shared_y.s6; \ + total_sums.s1 += (((bits4.s3 & 0xF000) >> 12) * scale.s1 - minv.s1) * shared_y.s7; \ + shared_y = sub_group_broadcast(y, 1); \ + total_sums.s0 += ((bits4.s4 & 0x000F) * scale.s0 - minv.s0) * shared_y.s0; \ + total_sums.s0 += (((bits4.s4 & 0x00F0) >> 4) * scale.s0 - minv.s0) * shared_y.s1; \ + total_sums.s0 += (((bits4.s4 & 0x0F00) >> 8) * scale.s0 - minv.s0) * shared_y.s2; \ + total_sums.s0 += (((bits4.s4 & 0xF000) >> 12) * scale.s0 - minv.s0) * shared_y.s3; \ + total_sums.s0 += ((bits4.s6 & 0x000F) * scale.s0 - minv.s0) * shared_y.s4; \ + total_sums.s0 += (((bits4.s6 & 0x00F0) >> 4) * scale.s0 - minv.s0) * shared_y.s5; \ + total_sums.s0 += (((bits4.s6 & 0x0F00) >> 8) * scale.s0 - minv.s0) * shared_y.s6; \ + total_sums.s0 += (((bits4.s6 & 0xF000) >> 12) * scale.s0 - minv.s0) * shared_y.s7; \ + total_sums.s1 += ((bits4.s5 & 0x000F) * scale.s1 - minv.s1) * shared_y.s0; \ + total_sums.s1 += (((bits4.s5 & 0x00F0) >> 4) * scale.s1 - minv.s1) * shared_y.s1; \ + total_sums.s1 += (((bits4.s5 & 0x0F00) >> 8) * scale.s1 - minv.s1) * shared_y.s2; \ + total_sums.s1 += (((bits4.s5 & 0xF000) >> 12) * scale.s1 - minv.s1) * shared_y.s3; \ + total_sums.s1 += ((bits4.s7 & 0x000F) * scale.s1 - minv.s1) * shared_y.s4; \ + total_sums.s1 += (((bits4.s7 & 0x00F0) >> 4) * scale.s1 - minv.s1) * shared_y.s5; \ + total_sums.s1 += (((bits4.s7 & 0x0F00) >> 8) * scale.s1 - minv.s1) * shared_y.s6; \ + total_sums.s1 += (((bits4.s7 & 0xF000) >> 12) * scale.s1 - minv.s1) * shared_y.s7; \ + + +#define dequantizeBlockAccum_ns_sgbroadcast_8_lo(total_sums, bits4, scale, minv, y) \ + shared_y = sub_group_broadcast(y, 2); \ + total_sums.s0 += ((bits4.s0 & 0x000F) * scale.s0 - minv.s0) * shared_y.s0; \ + total_sums.s0 += (((bits4.s0 & 0x00F0) >> 4) * scale.s0 - minv.s0) * shared_y.s1; \ + total_sums.s0 += (((bits4.s0 & 0x0F00) >> 8) * scale.s0 - minv.s0) * shared_y.s2; \ + total_sums.s0 += (((bits4.s0 & 0xF000) >> 12) * scale.s0 - minv.s0) * shared_y.s3; \ + total_sums.s0 += ((bits4.s2 & 0x000F) * scale.s0 - minv.s0) * shared_y.s4; \ + total_sums.s0 += (((bits4.s2 & 0x00F0) >> 4) * scale.s0 - minv.s0) * shared_y.s5; \ + total_sums.s0 += (((bits4.s2 & 0x0F00) >> 8) * scale.s0 - minv.s0) * shared_y.s6; \ + total_sums.s0 += (((bits4.s2 & 0xF000) >> 12) * scale.s0 - minv.s0) * shared_y.s7; \ + total_sums.s1 += ((bits4.s1 & 0x000F) * scale.s1 - minv.s1) * shared_y.s0; \ + total_sums.s1 += (((bits4.s1 & 0x00F0) >> 4) * scale.s1 - minv.s1) * shared_y.s1; \ + total_sums.s1 += (((bits4.s1 & 0x0F00) >> 8) * scale.s1 - minv.s1) * shared_y.s2; \ + total_sums.s1 += (((bits4.s1 & 0xF000) >> 12) * scale.s1 - minv.s1) * shared_y.s3; \ + total_sums.s1 += ((bits4.s3 & 0x000F) * scale.s1 - minv.s1) * shared_y.s4; \ + total_sums.s1 += (((bits4.s3 & 0x00F0) >> 4) * scale.s1 - minv.s1) * shared_y.s5; \ + total_sums.s1 += (((bits4.s3 & 0x0F00) >> 8) * scale.s1 - minv.s1) * shared_y.s6; \ + total_sums.s1 += (((bits4.s3 & 0xF000) >> 12) * scale.s1 - minv.s1) * shared_y.s7; \ + shared_y = sub_group_broadcast(y, 3); \ + total_sums.s0 += ((bits4.s4 & 0x000F) * scale.s0 - minv.s0) * shared_y.s0; \ + total_sums.s0 += (((bits4.s4 & 0x00F0) >> 4) * scale.s0 - minv.s0) * shared_y.s1; \ + total_sums.s0 += (((bits4.s4 & 0x0F00) >> 8) * scale.s0 - minv.s0) * shared_y.s2; \ + total_sums.s0 += (((bits4.s4 & 0xF000) >> 12) * scale.s0 - minv.s0) * shared_y.s3; \ + total_sums.s0 += ((bits4.s6 & 0x000F) * scale.s0 - minv.s0) * shared_y.s4; \ + total_sums.s0 += (((bits4.s6 & 0x00F0) >> 4) * scale.s0 - minv.s0) * shared_y.s5; \ + total_sums.s0 += (((bits4.s6 & 0x0F00) >> 8) * scale.s0 - minv.s0) * shared_y.s6; \ + total_sums.s0 += (((bits4.s6 & 0xF000) >> 12) * scale.s0 - minv.s0) * shared_y.s7; \ + total_sums.s1 += ((bits4.s5 & 0x000F) * scale.s1 - minv.s1) * shared_y.s0; \ + total_sums.s1 += (((bits4.s5 & 0x00F0) >> 4) * scale.s1 - minv.s1) * shared_y.s1; \ + total_sums.s1 += (((bits4.s5 & 0x0F00) >> 8) * scale.s1 - minv.s1) * shared_y.s2; \ + total_sums.s1 += (((bits4.s5 & 0xF000) >> 12) * scale.s1 - minv.s1) * shared_y.s3; \ + total_sums.s1 += ((bits4.s7 & 0x000F) * scale.s1 - minv.s1) * shared_y.s4; \ + total_sums.s1 += (((bits4.s7 & 0x00F0) >> 4) * scale.s1 - minv.s1) * shared_y.s5; \ + total_sums.s1 += (((bits4.s7 & 0x0F00) >> 8) * scale.s1 - minv.s1) * shared_y.s6; \ + total_sums.s1 += (((bits4.s7 & 0xF000) >> 12) * scale.s1 - minv.s1) * shared_y.s7; \ + +#ifdef ADRENO_GPU +REQD_SUBGROUP_SIZE_64 +#endif +kernel void kernel_gemv_noshuffle_q4_k_f32_o4( + read_only image1d_buffer_t src0_q, + global half2 * src0_d, + global half2 * src0_m, + global uchar * src0_s, + read_only image1d_buffer_t src1, + global float * dst, + ulong offsetd, + int ne00, + int ne01, + uchar mask_d6, + uchar mask_d4, + uchar mask_hi2) +{ + uint groupId = get_local_id(1); + uint gid = get_global_id(0); // 4-output quad index + ushort slid = get_sub_group_local_id(); + + // Two consecutive pair-indices (each the same access pattern the 2-output + // kernel uses); together they cover 4 consecutive output rows. + uint gid_a = gid * 2; + uint gid_b = gid * 2 + 1; + + uint K = ne00; + uint M = ne01; + + uint LINE_STRIDE_A = M / 2; + uint BLOCK_STRIDE_A = NSUBGROUPS * M; + + private uint4 regA; + private half2 regS_a, regS_b; + private half2 regM_a, regM_b; + private float8 regB; + + private float2 totalSum_a = (float2)(0.0f); + private float2 totalSum_b = (float2)(0.0f); + + for (uint k = groupId; k < (K / 32); k += NSUBGROUPS) { + uint sb = k / 8; + uint j = k % 8; + + // pair a scales/mins + half2 d_a = src0_d[gid_a + sb * LINE_STRIDE_A]; + half2 dm_a = src0_m[gid_a + sb * LINE_STRIDE_A]; + global const uchar * sc0a = src0_s + sb * 12 * M + 2 * gid_a; + global const uchar * sc1a = sc0a + 1; + uchar sv0a, mn0a, sv1a, mn1a; + get_scale_min_k4(j, sc0a, M, &sv0a, &mn0a, mask_d6, mask_d4, mask_hi2); + get_scale_min_k4(j, sc1a, M, &sv1a, &mn1a, mask_d6, mask_d4, mask_hi2); + regS_a = convert_half2(convert_float2(d_a) * convert_float2((uchar2)(sv0a, sv1a))); + regM_a = convert_half2(convert_float2(dm_a) * convert_float2((uchar2)(mn0a, mn1a))); + + // pair b scales/mins + half2 d_b = src0_d[gid_b + sb * LINE_STRIDE_A]; + half2 dm_b = src0_m[gid_b + sb * LINE_STRIDE_A]; + global const uchar * sc0b = src0_s + sb * 12 * M + 2 * gid_b; + global const uchar * sc1b = sc0b + 1; + uchar sv0b, mn0b, sv1b, mn1b; + get_scale_min_k4(j, sc0b, M, &sv0b, &mn0b, mask_d6, mask_d4, mask_hi2); + get_scale_min_k4(j, sc1b, M, &sv1b, &mn1b, mask_d6, mask_d4, mask_hi2); + regS_b = convert_half2(convert_float2(d_b) * convert_float2((uchar2)(sv0b, sv1b))); + regM_b = convert_half2(convert_float2(dm_b) * convert_float2((uchar2)(mn0b, mn1b))); + + // activation: load once, reuse for both pairs + if (slid < 4) { + regB.s0123 = read_imagef(src1, (slid * 2 + k * 8)); + regB.s4567 = read_imagef(src1, (1 + slid * 2 + k * 8)); + } + + // pair a (own block so _lo sees the shared_y declared by _hi) + { + regA.s0 = read_imageui(src0_q, (gid_a + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 0)).x; + regA.s1 = read_imageui(src0_q, (gid_a + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 1)).x; + regA.s2 = read_imageui(src0_q, (gid_a + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 2)).x; + regA.s3 = read_imageui(src0_q, (gid_a + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 3)).x; +#ifdef VECTOR_SUB_GROUP_BROADCAST + dequantizeBlockAccum_ns_sgbroadcast_8_hi(totalSum_a, as_ushort8(regA), regS_a, regM_a, regB); +#else + dequantizeBlockAccum_ns_sgbroadcast_1_hi(totalSum_a, as_ushort8(regA), regS_a, regM_a, regB); +#endif + regA.s0 = read_imageui(src0_q, (gid_a + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 4)).x; + regA.s1 = read_imageui(src0_q, (gid_a + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 5)).x; + regA.s2 = read_imageui(src0_q, (gid_a + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 6)).x; + regA.s3 = read_imageui(src0_q, (gid_a + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 7)).x; +#ifdef VECTOR_SUB_GROUP_BROADCAST + dequantizeBlockAccum_ns_sgbroadcast_8_lo(totalSum_a, as_ushort8(regA), regS_a, regM_a, regB); +#else + dequantizeBlockAccum_ns_sgbroadcast_1_lo(totalSum_a, as_ushort8(regA), regS_a, regM_a, regB); +#endif + } + + // pair b + { + regA.s0 = read_imageui(src0_q, (gid_b + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 0)).x; + regA.s1 = read_imageui(src0_q, (gid_b + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 1)).x; + regA.s2 = read_imageui(src0_q, (gid_b + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 2)).x; + regA.s3 = read_imageui(src0_q, (gid_b + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 3)).x; +#ifdef VECTOR_SUB_GROUP_BROADCAST + dequantizeBlockAccum_ns_sgbroadcast_8_hi(totalSum_b, as_ushort8(regA), regS_b, regM_b, regB); +#else + dequantizeBlockAccum_ns_sgbroadcast_1_hi(totalSum_b, as_ushort8(regA), regS_b, regM_b, regB); +#endif + regA.s0 = read_imageui(src0_q, (gid_b + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 4)).x; + regA.s1 = read_imageui(src0_q, (gid_b + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 5)).x; + regA.s2 = read_imageui(src0_q, (gid_b + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 6)).x; + regA.s3 = read_imageui(src0_q, (gid_b + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 7)).x; +#ifdef VECTOR_SUB_GROUP_BROADCAST + dequantizeBlockAccum_ns_sgbroadcast_8_lo(totalSum_b, as_ushort8(regA), regS_b, regM_b, regB); +#else + dequantizeBlockAccum_ns_sgbroadcast_1_lo(totalSum_b, as_ushort8(regA), regS_b, regM_b, regB); +#endif + } + } + + // reduce 4 outputs (a.s0, a.s1, b.s0, b.s1) across the 4 subgroups + local float4 reduceLM[SUBGROUP_SIZE * 3]; + float4 acc = (float4)(totalSum_a.s0, totalSum_a.s1, totalSum_b.s0, totalSum_b.s1); + if (groupId == 1) { reduceLM[SUBGROUP_SIZE * 0 + slid] = acc; } + if (groupId == 2) { reduceLM[SUBGROUP_SIZE * 1 + slid] = acc; } + if (groupId == 3) { reduceLM[SUBGROUP_SIZE * 2 + slid] = acc; } + + barrier(CLK_LOCAL_MEM_FENCE); + + if (groupId == 0) { + acc += reduceLM[SUBGROUP_SIZE * 0 + slid]; + acc += reduceLM[SUBGROUP_SIZE * 1 + slid]; + acc += reduceLM[SUBGROUP_SIZE * 2 + slid]; + dst = (global float*)((global char*)dst + offsetd); + // The dispatch rounds ne01/4 up to the subgroup width, so the tail + // quads past the last row must not store (they wrote 128 rows past + // dst on every ne01 % 256 == 128 vocab, e.g. 151936). + if (gid * 4 + 3 < (uint)ne01) { + vstore4(acc, 0, &(dst[gid * 4])); + } + } +} diff --git a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_k_f32_tiled.cl b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_k_f32_tiled.cl new file mode 100644 index 000000000000..929538c41d6d --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_k_f32_tiled.cl @@ -0,0 +1,118 @@ +// Tiled-wide q4_K GEMV for the long-vocab lm_head/embed (decode path). +// +// Pairs with kernel_convert_block_q4_k_tiled_ns (cvt.cl): the weights are laid +// out CANONICALLY (4-bit code in element order e in [0,256)) and TILED by 64 +// output rows so the 64-thread lane group coalesces every weight load. Both the +// pack (convert) and the unpack (here) are owned by us -> correct by +// construction vs the reference ggml q4_K dequant. Same structure as the q6_K +// tiled GEMV; the only differences are the 4-bit dequant and the q4_K +// scale/min decode (get_scale_min_k4 from the packed 12-byte block). +// +// One work-item produces one output row. WG = {64 lanes, 4 subgroups}: the 64 +// lanes cover the 64 rows of one tile (coalesced uint4 reads), the 4 subgroups +// split the K-blocks and reduce through __local at the end. Weights read from +// __global (lm_head is streamed once per token; texture cache caps it below the +// coalesced-global rate). + +#pragma OPENCL EXTENSION cl_khr_fp16 : enable + +#ifdef cl_qcom_reqd_sub_group_size +#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable +#define ADRENO_GPU 1 +#define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size("half"))) +#endif + +#define QK_K 256 +#define NSUBGROUPS 4 +#define TILE_ROWS 64 + +// Decode one q4_K sub-block scale + min from the packed 12-byte block. +// Identical to the o4 kernel's helper (masks hard-coded: d6=0x3F, d4=0x0F, hi2=0xC0). +inline void q4k_scale_min(int j, __global const uchar * q, uchar * d, uchar * m) { + if (j < 4) { + *d = q[j] & 0x3F; + *m = q[j+4] & 0x3F; + } else { + *d = (q[j+4] & 0x0F) | ((q[j-4] & 0xC0) >> 2); + *m = ((q[j+4] >> 4) & 0x0F) | ((q[j] & 0xC0) >> 2); + } +} + +#if defined(ADRENO_GPU) +REQD_SUBGROUP_SIZE_64 +#endif +kernel void kernel_gemv_noshuffle_q4_k_f32_tiled( + __global uint4 * src0_q, // tiled: 8 uint4 granules / superblock (4-bit codes) + __global half * src0_d, // tiled: 1 half / superblock + __global half * src0_dm, // tiled: 1 half / superblock + __global uchar * src0_s, // tiled: 12 bytes / superblock (packed scales) + read_only image1d_buffer_t src1, // activation (RGBA f32) + global float * dst, + ulong offsetd, + int ne00, + int ne01 +) { + int grp = get_local_id(1); // subgroup index 0..3 (splits K) + int row = get_global_id(0); // output row along ne01 + int rt = row / TILE_ROWS; + int rit = row % TILE_ROWS; + + int nb = ne00 / QK_K; // superblocks per row + + float acc = 0.0f; + + for (int sb = grp; sb < nb; sb += NSUBGROUPS) { + int tile_blk = rt * nb + sb; // ne02 == 1 for lm_head/embed + + float dval = (float)src0_d [tile_blk * TILE_ROWS + rit]; + float dmval = (float)src0_dm[tile_blk * TILE_ROWS + rit]; + + // decode the 8 sub-block (scale, min) pairs + __global uchar * sc = src0_s + (tile_blk * TILE_ROWS + rit) * 12; + float scale[8], minv[8]; + #pragma unroll + for (int is = 0; is < 8; ++is) { + uchar sd, sm; + q4k_scale_min(is, sc, &sd, &sm); + scale[is] = dval * (float)sd; + minv[is] = dmval * (float)sm; + } + + // 32 uints of 4-bit codes (8 codes/uint), e-order + uint q[32]; + #pragma unroll + for (int g = 0; g < 8; ++g) { + uint4 v = src0_q[(tile_blk * 8 + g) * TILE_ROWS + rit]; + q[g*4+0] = v.x; q[g*4+1] = v.y; q[g*4+2] = v.z; q[g*4+3] = v.w; + } + + // dequant 256 codes in canonical e-order, MAC with activation. + int act_base = sb * 64; // activation float4 pixel base (256/4) + #pragma unroll + for (int e4 = 0; e4 < 64; ++e4) { + float4 a = read_imagef(src1, act_base + e4); + #pragma unroll + for (int t = 0; t < 4; ++t) { + int e = e4 * 4 + t; + uint code = (q[e >> 3] >> ((e & 7) * 4)) & 0xF; + int is = e >> 5; // sub-block index = e/32 + float av = (t == 0) ? a.x : (t == 1) ? a.y : (t == 2) ? a.z : a.w; + acc += ((float)code * scale[is] - minv[is]) * av; + } + } + } + + // reduce across the NSUBGROUPS subgroups (same rit, different K-subset) + local float reduce_lm[NSUBGROUPS * TILE_ROWS]; + reduce_lm[grp * TILE_ROWS + rit] = acc; + barrier(CLK_LOCAL_MEM_FENCE); + + if (grp == 0) { + float total = reduce_lm[0 * TILE_ROWS + rit] + + reduce_lm[1 * TILE_ROWS + rit] + + reduce_lm[2 * TILE_ROWS + rit] + + reduce_lm[3 * TILE_ROWS + rit]; + dst = (global float*)((global char*)dst + offsetd); + dst[row] = total; + } +} diff --git a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q5_k_f32.cl b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q5_k_f32.cl index 446f46533872..ae864b19ba9a 100644 --- a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q5_k_f32.cl +++ b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q5_k_f32.cl @@ -329,3 +329,125 @@ kernel void kernel_gemv_noshuffle_q5_k_f32( if (gid * 2 + 1 < M) dst[gid * 2 + 1] = totalSum.s1; } } + +// Multi-column (N in [2..4]) variant of the q5_K decode GEMV (spec/MTP verify) = +// q4_K mc3 + the high-bit qh plane (regH). n_cols = 2..4 (drafted + bonus); routes +// the small-batch verify OFF the gemm_noshuffle_q5_k dead-zone. n_cols==3 is byte- +// identical to the original mc3 (col3 disabled, float8 slots 6/7 stay zero). +#ifdef VECTOR_SUB_GROUP_BROADCAST +#define MC_DQ5_HI dequantizeBlockAccum_ns_sgbroadcast_8_hi +#define MC_DQ5_LO dequantizeBlockAccum_ns_sgbroadcast_8_lo +#else +#define MC_DQ5_HI dequantizeBlockAccum_ns_sgbroadcast_1_hi +#define MC_DQ5_LO dequantizeBlockAccum_ns_sgbroadcast_1_lo +#endif +#define MC_COL_Q5K(ts, c) \ + { if (slid < 4) { regB.s0123 = read_imagef(src1, (c)*COL_STRIDE + slid*2 + k*8); \ + regB.s4567 = read_imagef(src1, (c)*COL_STRIDE + 1 + slid*2 + k*8); } \ + MC_DQ5_HI(ts, as_ushort8(regA_hi), as_uchar8(regH), regS, regM, regB); \ + MC_DQ5_LO(ts, as_ushort8(regA_lo), as_uchar8(regH), regS, regM, regB); } +#ifdef ADRENO_GPU +REQD_SUBGROUP_SIZE_64 +#endif +kernel void kernel_gemv_noshuffle_q5_k_f32_mc3( + read_only image1d_buffer_t src0_q, + read_only image1d_buffer_t src0_qh, + global half2 * src0_d, + global half2 * src0_m, + global uchar * src0_s, + read_only image1d_buffer_t src1, + global float * dst, + ulong offsetd, + int ne00, + int ne01, + uchar mask_d6, + uchar mask_d4, + uchar mask_hi2, + int n_cols) +{ + uint groupId = get_local_id(1); + uint gid = get_global_id(0); + ushort slid = get_sub_group_local_id(); + + uint K = ne00; + uint M = ne01; + + uint LINE_STRIDE_A = M / 2; + uint BLOCK_STRIDE_A = NSUBGROUPS * M; + uint LINE_STRIDE_A_QH = M / 2; + uint BLOCK_STRIDE_A_QH = NSUBGROUPS * M / 2; + uint scales_per_row = (K / QK_K) * 12; + uint COL_STRIDE = K / 4; // float4 pixels per activation column + + private uint4 regA_hi, regA_lo; + private ushort4 regH; + private half2 regS, regM; + private float8 regB; + + private float2 ts0 = (float2)(0.0f); + private float2 ts1 = (float2)(0.0f); + private float2 ts2 = (float2)(0.0f); + private float2 ts3 = (float2)(0.0f); + + for (uint k = groupId; k < (K / 32); k += NSUBGROUPS) { + uint sb = k / 8; + uint j = k % 8; + + half2 d = src0_d[gid + sb * LINE_STRIDE_A]; + half2 dm = src0_m[gid + sb * LINE_STRIDE_A]; + + global const uchar * sc0 = src0_s + 2 * gid * scales_per_row + sb * 12; + global const uchar * sc1 = src0_s + (2 * gid + 1) * scales_per_row + sb * 12; + + uchar sv0, mn0, sv1, mn1; + get_scale_min_k4(j, sc0, &sv0, &mn0, mask_d6, mask_d4, mask_hi2); + get_scale_min_k4(j, sc1, &sv1, &mn1, mask_d6, mask_d4, mask_hi2); + + regS = convert_half2(convert_float2(d) * convert_float2((uchar2)(sv0, sv1))); + regM = convert_half2(convert_float2(dm) * convert_float2((uchar2)(mn0, mn1))); + + // high-bit plane + weights loaded ONCE, reused across the columns + regH.s0 = as_ushort(read_imageh(src0_qh, (gid + k * BLOCK_STRIDE_A_QH + LINE_STRIDE_A_QH * 0)).x); + regH.s1 = as_ushort(read_imageh(src0_qh, (gid + k * BLOCK_STRIDE_A_QH + LINE_STRIDE_A_QH * 1)).x); + regH.s2 = as_ushort(read_imageh(src0_qh, (gid + k * BLOCK_STRIDE_A_QH + LINE_STRIDE_A_QH * 2)).x); + regH.s3 = as_ushort(read_imageh(src0_qh, (gid + k * BLOCK_STRIDE_A_QH + LINE_STRIDE_A_QH * 3)).x); + + regA_hi.s0 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 0)).x; + regA_hi.s1 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 1)).x; + regA_hi.s2 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 2)).x; + regA_hi.s3 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 3)).x; + regA_lo.s0 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 4)).x; + regA_lo.s1 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 5)).x; + regA_lo.s2 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 6)).x; + regA_lo.s3 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 7)).x; + + MC_COL_Q5K(ts0, 0); + MC_COL_Q5K(ts1, 1); + if (n_cols > 2) MC_COL_Q5K(ts2, 2); + if (n_cols > 3) MC_COL_Q5K(ts3, 3); + } + + // cross-subgroup reduce: pack the (up to 4) columns' float2 into a float8. + local float8 reduceLM[SUBGROUP_SIZE * 3]; + float8 acc = (float8)(ts0.s0, ts0.s1, ts1.s0, ts1.s1, ts2.s0, ts2.s1, ts3.s0, ts3.s1); + if (groupId == 1) { reduceLM[SUBGROUP_SIZE * 0 + slid] = acc; } + if (groupId == 2) { reduceLM[SUBGROUP_SIZE * 1 + slid] = acc; } + if (groupId == 3) { reduceLM[SUBGROUP_SIZE * 2 + slid] = acc; } + + barrier(CLK_LOCAL_MEM_FENCE); + + if (groupId == 0) { + acc += reduceLM[SUBGROUP_SIZE * 0 + slid]; + acc += reduceLM[SUBGROUP_SIZE * 1 + slid]; + acc += reduceLM[SUBGROUP_SIZE * 2 + slid]; + dst = (global float*)((global char*)dst + offsetd); + // dst is column-major [M rows x n_cols cols]: (row, col) at col*M + row + vstore2((float2)(acc.s0, acc.s1), 0, &(dst[0 * M + gid * 2])); + vstore2((float2)(acc.s2, acc.s3), 0, &(dst[1 * M + gid * 2])); + if (n_cols > 2) vstore2((float2)(acc.s4, acc.s5), 0, &(dst[2 * M + gid * 2])); + if (n_cols > 3) vstore2((float2)(acc.s6, acc.s7), 0, &(dst[3 * M + gid * 2])); + } +} +#undef MC_COL_Q5K +#undef MC_DQ5_HI +#undef MC_DQ5_LO diff --git a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q6_k_f32.cl b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q6_k_f32.cl index 51682ecebbbe..32624ac868fe 100644 --- a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q6_k_f32.cl +++ b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q6_k_f32.cl @@ -296,3 +296,114 @@ kernel void kernel_gemv_noshuffle_q6_K_f32( if (gid * 2 + 1 < ne01) dst[gid * 2 + 1] = total_sum.s1; } } + +// Multi-column (N=3) q6_K decode GEMV for the spec/MTP verify batch. Same idea +// as the q4_K mc3: stay on the efficient GEMV path (subgroup broadcast, no +// transpose) instead of the transposed-GEMM dead-zone. Each K-block's weights +// (ql/qh, hi+lo) are loaded ONCE and reused across all 3 activation columns. +// Per-column accumulation is independent and identical to 3 standalone GEMVs +// => byte-identical; does NOT perturb the lm_head logits / spec accept rate. +#if defined(ADRENO_GPU) +REQD_SUBGROUP_SIZE_64 +#endif +kernel void kernel_gemv_noshuffle_q6_K_f32_mc3( + read_only image1d_buffer_t src0_ql, + read_only image1d_buffer_t src0_qh, + global half2 * src0_s, + global half2 * src0_d, + read_only image1d_buffer_t src1, + global float * dst, + ulong offsetd, + int ne00, + int ne01 +) { + int grp = get_local_id(1); + int gid = get_global_id(0); + ushort slid = get_sub_group_local_id(); + + int nb = ne00 / 32; + int line_stride_a = ne01 / 2; + int block_stride_a = NSUBGROUPS * ne01; + int COL_STRIDE = ne00 / 4; // float4 pixels per activation column + + uint4 ql_hi, ql_lo; + ushort4 qh_hi, qh_lo; + half2 reg_d; + char4 reg_s; + float8 reg_b; + + float2 ts0 = 0.0f, ts1 = 0.0f, ts2 = 0.0f; + + for (int k = grp; k < nb; k += NSUBGROUPS) { + reg_d = src0_d[gid + k/8 * line_stride_a]; + reg_s = as_char4(src0_s[gid + k * line_stride_a]); + + // weights loaded ONCE (hi: blocks 0-3, lo: blocks 4-7), reused x3 cols + ql_hi.s0 = read_imageui(src0_ql, gid + k*block_stride_a + line_stride_a*0).x; + ql_hi.s1 = read_imageui(src0_ql, gid + k*block_stride_a + line_stride_a*1).x; + ql_hi.s2 = read_imageui(src0_ql, gid + k*block_stride_a + line_stride_a*2).x; + ql_hi.s3 = read_imageui(src0_ql, gid + k*block_stride_a + line_stride_a*3).x; + qh_hi.s0 = as_ushort(read_imageh(src0_qh, gid + k*block_stride_a + line_stride_a*0).x); + qh_hi.s1 = as_ushort(read_imageh(src0_qh, gid + k*block_stride_a + line_stride_a*1).x); + qh_hi.s2 = as_ushort(read_imageh(src0_qh, gid + k*block_stride_a + line_stride_a*2).x); + qh_hi.s3 = as_ushort(read_imageh(src0_qh, gid + k*block_stride_a + line_stride_a*3).x); + + ql_lo.s0 = read_imageui(src0_ql, gid + k*block_stride_a + line_stride_a*4).x; + ql_lo.s1 = read_imageui(src0_ql, gid + k*block_stride_a + line_stride_a*5).x; + ql_lo.s2 = read_imageui(src0_ql, gid + k*block_stride_a + line_stride_a*6).x; + ql_lo.s3 = read_imageui(src0_ql, gid + k*block_stride_a + line_stride_a*7).x; + qh_lo.s0 = as_ushort(read_imageh(src0_qh, gid + k*block_stride_a + line_stride_a*4).x); + qh_lo.s1 = as_ushort(read_imageh(src0_qh, gid + k*block_stride_a + line_stride_a*5).x); + qh_lo.s2 = as_ushort(read_imageh(src0_qh, gid + k*block_stride_a + line_stride_a*6).x); + qh_lo.s3 = as_ushort(read_imageh(src0_qh, gid + k*block_stride_a + line_stride_a*7).x); + + // Per-column: load only this column's activation (single reg_b live) -> + // 1/3 the activation register pressure, cutting the private-mem spill. +#ifdef VECTOR_SUB_GROUP_BROADCAT + { if (slid < 4) { reg_b.s0123 = read_imagef(src1, 0*COL_STRIDE + 0 + slid*2 + k*8); + reg_b.s4567 = read_imagef(src1, 0*COL_STRIDE + 1 + slid*2 + k*8); } + dequantize_block_acc_bcast_8_hi(ts0, as_ushort8(ql_hi), as_uchar8(qh_hi), reg_d, reg_s, reg_b); + dequantize_block_acc_bcast_8_lo(ts0, as_ushort8(ql_lo), as_uchar8(qh_lo), reg_d, reg_s, reg_b); } + { if (slid < 4) { reg_b.s0123 = read_imagef(src1, 1*COL_STRIDE + 0 + slid*2 + k*8); + reg_b.s4567 = read_imagef(src1, 1*COL_STRIDE + 1 + slid*2 + k*8); } + dequantize_block_acc_bcast_8_hi(ts1, as_ushort8(ql_hi), as_uchar8(qh_hi), reg_d, reg_s, reg_b); + dequantize_block_acc_bcast_8_lo(ts1, as_ushort8(ql_lo), as_uchar8(qh_lo), reg_d, reg_s, reg_b); } + { if (slid < 4) { reg_b.s0123 = read_imagef(src1, 2*COL_STRIDE + 0 + slid*2 + k*8); + reg_b.s4567 = read_imagef(src1, 2*COL_STRIDE + 1 + slid*2 + k*8); } + dequantize_block_acc_bcast_8_hi(ts2, as_ushort8(ql_hi), as_uchar8(qh_hi), reg_d, reg_s, reg_b); + dequantize_block_acc_bcast_8_lo(ts2, as_ushort8(ql_lo), as_uchar8(qh_lo), reg_d, reg_s, reg_b); } +#else + { if (slid < 4) { reg_b.s0123 = read_imagef(src1, 0*COL_STRIDE + 0 + slid*2 + k*8); + reg_b.s4567 = read_imagef(src1, 0*COL_STRIDE + 1 + slid*2 + k*8); } + dequantize_block_acc_bcast_1_hi(ts0, as_ushort8(ql_hi), as_uchar8(qh_hi), reg_d, reg_s, reg_b); + dequantize_block_acc_bcast_1_lo(ts0, as_ushort8(ql_lo), as_uchar8(qh_lo), reg_d, reg_s, reg_b); } + { if (slid < 4) { reg_b.s0123 = read_imagef(src1, 1*COL_STRIDE + 0 + slid*2 + k*8); + reg_b.s4567 = read_imagef(src1, 1*COL_STRIDE + 1 + slid*2 + k*8); } + dequantize_block_acc_bcast_1_hi(ts1, as_ushort8(ql_hi), as_uchar8(qh_hi), reg_d, reg_s, reg_b); + dequantize_block_acc_bcast_1_lo(ts1, as_ushort8(ql_lo), as_uchar8(qh_lo), reg_d, reg_s, reg_b); } + { if (slid < 4) { reg_b.s0123 = read_imagef(src1, 2*COL_STRIDE + 0 + slid*2 + k*8); + reg_b.s4567 = read_imagef(src1, 2*COL_STRIDE + 1 + slid*2 + k*8); } + dequantize_block_acc_bcast_1_hi(ts2, as_ushort8(ql_hi), as_uchar8(qh_hi), reg_d, reg_s, reg_b); + dequantize_block_acc_bcast_1_lo(ts2, as_ushort8(ql_lo), as_uchar8(qh_lo), reg_d, reg_s, reg_b); } +#endif + } + + local float8 reduce_lm[SUBGROUP_SIZE * 3]; + float8 acc = (float8)(ts0.s0, ts0.s1, ts1.s0, ts1.s1, ts2.s0, ts2.s1, 0.0f, 0.0f); + if (grp == 1) { reduce_lm[SUBGROUP_SIZE*0 + slid] = acc; } + if (grp == 2) { reduce_lm[SUBGROUP_SIZE*1 + slid] = acc; } + if (grp == 3) { reduce_lm[SUBGROUP_SIZE*2 + slid] = acc; } + + barrier(CLK_LOCAL_MEM_FENCE); + + if (grp == 0) { + acc += reduce_lm[SUBGROUP_SIZE*0 + slid]; + acc += reduce_lm[SUBGROUP_SIZE*1 + slid]; + acc += reduce_lm[SUBGROUP_SIZE*2 + slid]; + dst = (global float*)((global char*)dst + offsetd); + // dst column-major [ne01 rows x 3 cols]: (row, col) at col*ne01 + row + vstore2((float2)(acc.s0, acc.s1), 0, &(dst[0*ne01 + gid*2])); + vstore2((float2)(acc.s2, acc.s3), 0, &(dst[1*ne01 + gid*2])); + vstore2((float2)(acc.s4, acc.s5), 0, &(dst[2*ne01 + gid*2])); + } +} diff --git a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q6_k_f32_o4.cl b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q6_k_f32_o4.cl new file mode 100644 index 000000000000..84447e61bb6f --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q6_k_f32_o4.cl @@ -0,0 +1,372 @@ +// 4-output-per-WI variant of kernel_gemv_noshuffle_q6_K_f32. +// Each WI now produces 4 consecutive outputs (output quad). The activation +// fetch (reg_b) is shared across all 4 outputs, doubling per-WI ALU per +// activation broadcast and halving the WG count vs the 2-output kernel. +// +// Implementation: each K-block we fetch TWO sets of (scales + ql + qh) +// — one for the low pair (rows 0,1 of the quad) and one for the high pair +// (rows 2,3) — and invoke the existing 2-output dequant macros twice +// against the *same* reg_b. Identical data layout to the 2-output kernel, +// so the host only needs to halve the grid and double the gid-to-output +// mapping. +// +// Opt-in via the host dispatch when GGML_OPENCL_Q6K_GEMV_O4=1. + +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#pragma OPENCL EXTENSION cl_khr_subgroups : enable + +#ifdef cl_intel_required_subgroup_size +#pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable +#define INTEL_GPU 1 +#define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16))) +#define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32))) +#elif defined(cl_qcom_reqd_sub_group_size) +#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable +#define ADRENO_GPU 1 +#define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size("half"))) +#define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size("full"))) +#endif + +#define NSUBGROUPS 4 +#define SUBGROUP_SIZE 64 + +// Macros are identical to the 2-output kernel — they accept `total_sum` as +// a parameter so we can call them twice (once per pair) against different +// accumulators against the same reg_b. +#define dequantize_block_acc_bcast_8_hi(total_sum, bits4, bits2, cs, y) \ + float8 shared_y; \ + shared_y = sub_group_broadcast(y, 0); \ + total_sum.s0 += ((float)(((bits4.s0 & 0x000F) ) | ((bits2.s0 & 0x03) << 4)) - 32.f) * cs.s0 * shared_y.s0; \ + total_sum.s0 += ((float)(((bits4.s0 & 0x00F0) >> 4) | ((bits2.s0 & 0x0C) << 2)) - 32.f) * cs.s0 * shared_y.s1; \ + total_sum.s0 += ((float)(((bits4.s0 & 0x0F00) >> 8) | ((bits2.s0 & 0x30) )) - 32.f) * cs.s0 * shared_y.s2; \ + total_sum.s0 += ((float)(((bits4.s0 & 0xF000) >> 12) | ((bits2.s0 & 0xC0) >> 2)) - 32.f) * cs.s0 * shared_y.s3; \ + total_sum.s0 += ((float)(((bits4.s2 & 0x000F) ) | ((bits2.s2 & 0x03) << 4)) - 32.f) * cs.s0 * shared_y.s4; \ + total_sum.s0 += ((float)(((bits4.s2 & 0x00F0) >> 4) | ((bits2.s2 & 0x0C) << 2)) - 32.f) * cs.s0 * shared_y.s5; \ + total_sum.s0 += ((float)(((bits4.s2 & 0x0F00) >> 8) | ((bits2.s2 & 0x30) )) - 32.f) * cs.s0 * shared_y.s6; \ + total_sum.s0 += ((float)(((bits4.s2 & 0xF000) >> 12) | ((bits2.s2 & 0xC0) >> 2)) - 32.f) * cs.s0 * shared_y.s7; \ + total_sum.s1 += ((float)(((bits4.s1 & 0x000F) ) | ((bits2.s1 & 0x03) << 4)) - 32.f) * cs.s2 * shared_y.s0; \ + total_sum.s1 += ((float)(((bits4.s1 & 0x00F0) >> 4) | ((bits2.s1 & 0x0C) << 2)) - 32.f) * cs.s2 * shared_y.s1; \ + total_sum.s1 += ((float)(((bits4.s1 & 0x0F00) >> 8) | ((bits2.s1 & 0x30) )) - 32.f) * cs.s2 * shared_y.s2; \ + total_sum.s1 += ((float)(((bits4.s1 & 0xF000) >> 12) | ((bits2.s1 & 0xC0) >> 2)) - 32.f) * cs.s2 * shared_y.s3; \ + total_sum.s1 += ((float)(((bits4.s3 & 0x000F) ) | ((bits2.s3 & 0x03) << 4)) - 32.f) * cs.s2 * shared_y.s4; \ + total_sum.s1 += ((float)(((bits4.s3 & 0x00F0) >> 4) | ((bits2.s3 & 0x0C) << 2)) - 32.f) * cs.s2 * shared_y.s5; \ + total_sum.s1 += ((float)(((bits4.s3 & 0x0F00) >> 8) | ((bits2.s3 & 0x30) )) - 32.f) * cs.s2 * shared_y.s6; \ + total_sum.s1 += ((float)(((bits4.s3 & 0xF000) >> 12) | ((bits2.s3 & 0xC0) >> 2)) - 32.f) * cs.s2 * shared_y.s7; \ + shared_y = sub_group_broadcast(y, 1); \ + total_sum.s0 += ((float)(((bits4.s4 & 0x000F) ) | ((bits2.s4 & 0x03) << 4)) - 32.f) * cs.s0 * shared_y.s0; \ + total_sum.s0 += ((float)(((bits4.s4 & 0x00F0) >> 4) | ((bits2.s4 & 0x0C) << 2)) - 32.f) * cs.s0 * shared_y.s1; \ + total_sum.s0 += ((float)(((bits4.s4 & 0x0F00) >> 8) | ((bits2.s4 & 0x30) )) - 32.f) * cs.s0 * shared_y.s2; \ + total_sum.s0 += ((float)(((bits4.s4 & 0xF000) >> 12) | ((bits2.s4 & 0xC0) >> 2)) - 32.f) * cs.s0 * shared_y.s3; \ + total_sum.s0 += ((float)(((bits4.s6 & 0x000F) ) | ((bits2.s6 & 0x03) << 4)) - 32.f) * cs.s0 * shared_y.s4; \ + total_sum.s0 += ((float)(((bits4.s6 & 0x00F0) >> 4) | ((bits2.s6 & 0x0C) << 2)) - 32.f) * cs.s0 * shared_y.s5; \ + total_sum.s0 += ((float)(((bits4.s6 & 0x0F00) >> 8) | ((bits2.s6 & 0x30) )) - 32.f) * cs.s0 * shared_y.s6; \ + total_sum.s0 += ((float)(((bits4.s6 & 0xF000) >> 12) | ((bits2.s6 & 0xC0) >> 2)) - 32.f) * cs.s0 * shared_y.s7; \ + total_sum.s1 += ((float)(((bits4.s5 & 0x000F) ) | ((bits2.s5 & 0x03) << 4)) - 32.f) * cs.s2 * shared_y.s0; \ + total_sum.s1 += ((float)(((bits4.s5 & 0x00F0) >> 4) | ((bits2.s5 & 0x0C) << 2)) - 32.f) * cs.s2 * shared_y.s1; \ + total_sum.s1 += ((float)(((bits4.s5 & 0x0F00) >> 8) | ((bits2.s5 & 0x30) )) - 32.f) * cs.s2 * shared_y.s2; \ + total_sum.s1 += ((float)(((bits4.s5 & 0xF000) >> 12) | ((bits2.s5 & 0xC0) >> 2)) - 32.f) * cs.s2 * shared_y.s3; \ + total_sum.s1 += ((float)(((bits4.s7 & 0x000F) ) | ((bits2.s7 & 0x03) << 4)) - 32.f) * cs.s2 * shared_y.s4; \ + total_sum.s1 += ((float)(((bits4.s7 & 0x00F0) >> 4) | ((bits2.s7 & 0x0C) << 2)) - 32.f) * cs.s2 * shared_y.s5; \ + total_sum.s1 += ((float)(((bits4.s7 & 0x0F00) >> 8) | ((bits2.s7 & 0x30) )) - 32.f) * cs.s2 * shared_y.s6; \ + total_sum.s1 += ((float)(((bits4.s7 & 0xF000) >> 12) | ((bits2.s7 & 0xC0) >> 2)) - 32.f) * cs.s2 * shared_y.s7; \ + +#define dequantize_block_acc_bcast_8_lo(total_sum, bits4, bits2, cs, y) \ + shared_y = sub_group_broadcast(y, 2); \ + total_sum.s0 += ((float)(((bits4.s0 & 0x000F) ) | ((bits2.s0 & 0x03) << 4)) - 32.f) * cs.s1 * shared_y.s0; \ + total_sum.s0 += ((float)(((bits4.s0 & 0x00F0) >> 4) | ((bits2.s0 & 0x0C) << 2)) - 32.f) * cs.s1 * shared_y.s1; \ + total_sum.s0 += ((float)(((bits4.s0 & 0x0F00) >> 8) | ((bits2.s0 & 0x30) )) - 32.f) * cs.s1 * shared_y.s2; \ + total_sum.s0 += ((float)(((bits4.s0 & 0xF000) >> 12) | ((bits2.s0 & 0xC0) >> 2)) - 32.f) * cs.s1 * shared_y.s3; \ + total_sum.s0 += ((float)(((bits4.s2 & 0x000F) ) | ((bits2.s2 & 0x03) << 4)) - 32.f) * cs.s1 * shared_y.s4; \ + total_sum.s0 += ((float)(((bits4.s2 & 0x00F0) >> 4) | ((bits2.s2 & 0x0C) << 2)) - 32.f) * cs.s1 * shared_y.s5; \ + total_sum.s0 += ((float)(((bits4.s2 & 0x0F00) >> 8) | ((bits2.s2 & 0x30) )) - 32.f) * cs.s1 * shared_y.s6; \ + total_sum.s0 += ((float)(((bits4.s2 & 0xF000) >> 12) | ((bits2.s2 & 0xC0) >> 2)) - 32.f) * cs.s1 * shared_y.s7; \ + total_sum.s1 += ((float)(((bits4.s1 & 0x000F) ) | ((bits2.s1 & 0x03) << 4)) - 32.f) * cs.s3 * shared_y.s0; \ + total_sum.s1 += ((float)(((bits4.s1 & 0x00F0) >> 4) | ((bits2.s1 & 0x0C) << 2)) - 32.f) * cs.s3 * shared_y.s1; \ + total_sum.s1 += ((float)(((bits4.s1 & 0x0F00) >> 8) | ((bits2.s1 & 0x30) )) - 32.f) * cs.s3 * shared_y.s2; \ + total_sum.s1 += ((float)(((bits4.s1 & 0xF000) >> 12) | ((bits2.s1 & 0xC0) >> 2)) - 32.f) * cs.s3 * shared_y.s3; \ + total_sum.s1 += ((float)(((bits4.s3 & 0x000F) ) | ((bits2.s3 & 0x03) << 4)) - 32.f) * cs.s3 * shared_y.s4; \ + total_sum.s1 += ((float)(((bits4.s3 & 0x00F0) >> 4) | ((bits2.s3 & 0x0C) << 2)) - 32.f) * cs.s3 * shared_y.s5; \ + total_sum.s1 += ((float)(((bits4.s3 & 0x0F00) >> 8) | ((bits2.s3 & 0x30) )) - 32.f) * cs.s3 * shared_y.s6; \ + total_sum.s1 += ((float)(((bits4.s3 & 0xF000) >> 12) | ((bits2.s3 & 0xC0) >> 2)) - 32.f) * cs.s3 * shared_y.s7; \ + shared_y = sub_group_broadcast(y, 3); \ + total_sum.s0 += ((float)(((bits4.s4 & 0x000F) ) | ((bits2.s4 & 0x03) << 4)) - 32.f) * cs.s1 * shared_y.s0; \ + total_sum.s0 += ((float)(((bits4.s4 & 0x00F0) >> 4) | ((bits2.s4 & 0x0C) << 2)) - 32.f) * cs.s1 * shared_y.s1; \ + total_sum.s0 += ((float)(((bits4.s4 & 0x0F00) >> 8) | ((bits2.s4 & 0x30) )) - 32.f) * cs.s1 * shared_y.s2; \ + total_sum.s0 += ((float)(((bits4.s4 & 0xF000) >> 12) | ((bits2.s4 & 0xC0) >> 2)) - 32.f) * cs.s1 * shared_y.s3; \ + total_sum.s0 += ((float)(((bits4.s6 & 0x000F) ) | ((bits2.s6 & 0x03) << 4)) - 32.f) * cs.s1 * shared_y.s4; \ + total_sum.s0 += ((float)(((bits4.s6 & 0x00F0) >> 4) | ((bits2.s6 & 0x0C) << 2)) - 32.f) * cs.s1 * shared_y.s5; \ + total_sum.s0 += ((float)(((bits4.s6 & 0x0F00) >> 8) | ((bits2.s6 & 0x30) )) - 32.f) * cs.s1 * shared_y.s6; \ + total_sum.s0 += ((float)(((bits4.s6 & 0xF000) >> 12) | ((bits2.s6 & 0xC0) >> 2)) - 32.f) * cs.s1 * shared_y.s7; \ + total_sum.s1 += ((float)(((bits4.s5 & 0x000F) ) | ((bits2.s5 & 0x03) << 4)) - 32.f) * cs.s3 * shared_y.s0; \ + total_sum.s1 += ((float)(((bits4.s5 & 0x00F0) >> 4) | ((bits2.s5 & 0x0C) << 2)) - 32.f) * cs.s3 * shared_y.s1; \ + total_sum.s1 += ((float)(((bits4.s5 & 0x0F00) >> 8) | ((bits2.s5 & 0x30) )) - 32.f) * cs.s3 * shared_y.s2; \ + total_sum.s1 += ((float)(((bits4.s5 & 0xF000) >> 12) | ((bits2.s5 & 0xC0) >> 2)) - 32.f) * cs.s3 * shared_y.s3; \ + total_sum.s1 += ((float)(((bits4.s7 & 0x000F) ) | ((bits2.s7 & 0x03) << 4)) - 32.f) * cs.s3 * shared_y.s4; \ + total_sum.s1 += ((float)(((bits4.s7 & 0x00F0) >> 4) | ((bits2.s7 & 0x0C) << 2)) - 32.f) * cs.s3 * shared_y.s5; \ + total_sum.s1 += ((float)(((bits4.s7 & 0x0F00) >> 8) | ((bits2.s7 & 0x30) )) - 32.f) * cs.s3 * shared_y.s6; \ + total_sum.s1 += ((float)(((bits4.s7 & 0xF000) >> 12) | ((bits2.s7 & 0xC0) >> 2)) - 32.f) * cs.s3 * shared_y.s7; \ + +#define dequantize_block_acc_bcast_1_hi(total_sum, bits4, bits2, cs, y) \ + float shared_y; \ + shared_y = sub_group_broadcast(y.s0, 0); \ + total_sum.s0 += ((float)(((bits4.s0 & 0x000F) ) | ((bits2.s0 & 0x03) << 4)) - 32.f) * cs.s0 * shared_y; \ + total_sum.s1 += ((float)(((bits4.s1 & 0x000F) ) | ((bits2.s1 & 0x03) << 4)) - 32.f) * cs.s2 * shared_y; \ + shared_y = sub_group_broadcast(y.s1, 0); \ + total_sum.s0 += ((float)(((bits4.s0 & 0x00F0) >> 4) | ((bits2.s0 & 0x0C) << 2)) - 32.f) * cs.s0 * shared_y; \ + total_sum.s1 += ((float)(((bits4.s1 & 0x00F0) >> 4) | ((bits2.s1 & 0x0C) << 2)) - 32.f) * cs.s2 * shared_y; \ + shared_y = sub_group_broadcast(y.s2, 0); \ + total_sum.s0 += ((float)(((bits4.s0 & 0x0F00) >> 8) | ((bits2.s0 & 0x30) )) - 32.f) * cs.s0 * shared_y; \ + total_sum.s1 += ((float)(((bits4.s1 & 0x0F00) >> 8) | ((bits2.s1 & 0x30) )) - 32.f) * cs.s2 * shared_y; \ + shared_y = sub_group_broadcast(y.s3, 0); \ + total_sum.s0 += ((float)(((bits4.s0 & 0xF000) >> 12) | ((bits2.s0 & 0xC0) >> 2)) - 32.f) * cs.s0 * shared_y; \ + total_sum.s1 += ((float)(((bits4.s1 & 0xF000) >> 12) | ((bits2.s1 & 0xC0) >> 2)) - 32.f) * cs.s2 * shared_y; \ + shared_y = sub_group_broadcast(y.s4, 0); \ + total_sum.s0 += ((float)(((bits4.s2 & 0x000F) ) | ((bits2.s2 & 0x03) << 4)) - 32.f) * cs.s0 * shared_y; \ + total_sum.s1 += ((float)(((bits4.s3 & 0x000F) ) | ((bits2.s3 & 0x03) << 4)) - 32.f) * cs.s2 * shared_y; \ + shared_y = sub_group_broadcast(y.s5, 0); \ + total_sum.s0 += ((float)(((bits4.s2 & 0x00F0) >> 4) | ((bits2.s2 & 0x0C) << 2)) - 32.f) * cs.s0 * shared_y; \ + total_sum.s1 += ((float)(((bits4.s3 & 0x00F0) >> 4) | ((bits2.s3 & 0x0C) << 2)) - 32.f) * cs.s2 * shared_y; \ + shared_y = sub_group_broadcast(y.s6, 0); \ + total_sum.s0 += ((float)(((bits4.s2 & 0x0F00) >> 8) | ((bits2.s2 & 0x30) )) - 32.f) * cs.s0 * shared_y; \ + total_sum.s1 += ((float)(((bits4.s3 & 0x0F00) >> 8) | ((bits2.s3 & 0x30) )) - 32.f) * cs.s2 * shared_y; \ + shared_y = sub_group_broadcast(y.s7, 0); \ + total_sum.s0 += ((float)(((bits4.s2 & 0xF000) >> 12) | ((bits2.s2 & 0xC0) >> 2)) - 32.f) * cs.s0 * shared_y; \ + total_sum.s1 += ((float)(((bits4.s3 & 0xF000) >> 12) | ((bits2.s3 & 0xC0) >> 2)) - 32.f) * cs.s2 * shared_y; \ + shared_y = sub_group_broadcast(y.s0, 1); \ + total_sum.s0 += ((float)(((bits4.s4 & 0x000F) ) | ((bits2.s4 & 0x03) << 4)) - 32.f) * cs.s0 * shared_y; \ + total_sum.s1 += ((float)(((bits4.s5 & 0x000F) ) | ((bits2.s5 & 0x03) << 4)) - 32.f) * cs.s2 * shared_y; \ + shared_y = sub_group_broadcast(y.s1, 1); \ + total_sum.s0 += ((float)(((bits4.s4 & 0x00F0) >> 4) | ((bits2.s4 & 0x0C) << 2)) - 32.f) * cs.s0 * shared_y; \ + total_sum.s1 += ((float)(((bits4.s5 & 0x00F0) >> 4) | ((bits2.s5 & 0x0C) << 2)) - 32.f) * cs.s2 * shared_y; \ + shared_y = sub_group_broadcast(y.s2, 1); \ + total_sum.s0 += ((float)(((bits4.s4 & 0x0F00) >> 8) | ((bits2.s4 & 0x30) )) - 32.f) * cs.s0 * shared_y; \ + total_sum.s1 += ((float)(((bits4.s5 & 0x0F00) >> 8) | ((bits2.s5 & 0x30) )) - 32.f) * cs.s2 * shared_y; \ + shared_y = sub_group_broadcast(y.s3, 1); \ + total_sum.s0 += ((float)(((bits4.s4 & 0xF000) >> 12) | ((bits2.s4 & 0xC0) >> 2)) - 32.f) * cs.s0 * shared_y; \ + total_sum.s1 += ((float)(((bits4.s5 & 0xF000) >> 12) | ((bits2.s5 & 0xC0) >> 2)) - 32.f) * cs.s2 * shared_y; \ + shared_y = sub_group_broadcast(y.s4, 1); \ + total_sum.s0 += ((float)(((bits4.s6 & 0x000F) ) | ((bits2.s6 & 0x03) << 4)) - 32.f) * cs.s0 * shared_y; \ + total_sum.s1 += ((float)(((bits4.s7 & 0x000F) ) | ((bits2.s7 & 0x03) << 4)) - 32.f) * cs.s2 * shared_y; \ + shared_y = sub_group_broadcast(y.s5, 1); \ + total_sum.s0 += ((float)(((bits4.s6 & 0x00F0) >> 4) | ((bits2.s6 & 0x0C) << 2)) - 32.f) * cs.s0 * shared_y; \ + total_sum.s1 += ((float)(((bits4.s7 & 0x00F0) >> 4) | ((bits2.s7 & 0x0C) << 2)) - 32.f) * cs.s2 * shared_y; \ + shared_y = sub_group_broadcast(y.s6, 1); \ + total_sum.s0 += ((float)(((bits4.s6 & 0x0F00) >> 8) | ((bits2.s6 & 0x30) )) - 32.f) * cs.s0 * shared_y; \ + total_sum.s1 += ((float)(((bits4.s7 & 0x0F00) >> 8) | ((bits2.s7 & 0x30) )) - 32.f) * cs.s2 * shared_y; \ + shared_y = sub_group_broadcast(y.s7, 1); \ + total_sum.s0 += ((float)(((bits4.s6 & 0xF000) >> 12) | ((bits2.s6 & 0xC0) >> 2)) - 32.f) * cs.s0 * shared_y; \ + total_sum.s1 += ((float)(((bits4.s7 & 0xF000) >> 12) | ((bits2.s7 & 0xC0) >> 2)) - 32.f) * cs.s2 * shared_y; \ + +#define dequantize_block_acc_bcast_1_lo(total_sum, bits4, bits2, cs, y) \ + shared_y = sub_group_broadcast(y.s0, 2); \ + total_sum.s0 += ((float)(((bits4.s0 & 0x000F) ) | ((bits2.s0 & 0x03) << 4)) - 32.f) * cs.s1 * shared_y; \ + total_sum.s1 += ((float)(((bits4.s1 & 0x000F) ) | ((bits2.s1 & 0x03) << 4)) - 32.f) * cs.s3 * shared_y; \ + shared_y = sub_group_broadcast(y.s1, 2); \ + total_sum.s0 += ((float)(((bits4.s0 & 0x00F0) >> 4) | ((bits2.s0 & 0x0C) << 2)) - 32.f) * cs.s1 * shared_y; \ + total_sum.s1 += ((float)(((bits4.s1 & 0x00F0) >> 4) | ((bits2.s1 & 0x0C) << 2)) - 32.f) * cs.s3 * shared_y; \ + shared_y = sub_group_broadcast(y.s2, 2); \ + total_sum.s0 += ((float)(((bits4.s0 & 0x0F00) >> 8) | ((bits2.s0 & 0x30) )) - 32.f) * cs.s1 * shared_y; \ + total_sum.s1 += ((float)(((bits4.s1 & 0x0F00) >> 8) | ((bits2.s1 & 0x30) )) - 32.f) * cs.s3 * shared_y; \ + shared_y = sub_group_broadcast(y.s3, 2); \ + total_sum.s0 += ((float)(((bits4.s0 & 0xF000) >> 12) | ((bits2.s0 & 0xC0) >> 2)) - 32.f) * cs.s1 * shared_y; \ + total_sum.s1 += ((float)(((bits4.s1 & 0xF000) >> 12) | ((bits2.s1 & 0xC0) >> 2)) - 32.f) * cs.s3 * shared_y; \ + shared_y = sub_group_broadcast(y.s4, 2); \ + total_sum.s0 += ((float)(((bits4.s2 & 0x000F) ) | ((bits2.s2 & 0x03) << 4)) - 32.f) * cs.s1 * shared_y; \ + total_sum.s1 += ((float)(((bits4.s3 & 0x000F) ) | ((bits2.s3 & 0x03) << 4)) - 32.f) * cs.s3 * shared_y; \ + shared_y = sub_group_broadcast(y.s5, 2); \ + total_sum.s0 += ((float)(((bits4.s2 & 0x00F0) >> 4) | ((bits2.s2 & 0x0C) << 2)) - 32.f) * cs.s1 * shared_y; \ + total_sum.s1 += ((float)(((bits4.s3 & 0x00F0) >> 4) | ((bits2.s3 & 0x0C) << 2)) - 32.f) * cs.s3 * shared_y; \ + shared_y = sub_group_broadcast(y.s6, 2); \ + total_sum.s0 += ((float)(((bits4.s2 & 0x0F00) >> 8) | ((bits2.s2 & 0x30) )) - 32.f) * cs.s1 * shared_y; \ + total_sum.s1 += ((float)(((bits4.s3 & 0x0F00) >> 8) | ((bits2.s3 & 0x30) )) - 32.f) * cs.s3 * shared_y; \ + shared_y = sub_group_broadcast(y.s7, 2); \ + total_sum.s0 += ((float)(((bits4.s2 & 0xF000) >> 12) | ((bits2.s2 & 0xC0) >> 2)) - 32.f) * cs.s1 * shared_y; \ + total_sum.s1 += ((float)(((bits4.s3 & 0xF000) >> 12) | ((bits2.s3 & 0xC0) >> 2)) - 32.f) * cs.s3 * shared_y; \ + shared_y = sub_group_broadcast(y.s0, 3); \ + total_sum.s0 += ((float)(((bits4.s4 & 0x000F) ) | ((bits2.s4 & 0x03) << 4)) - 32.f) * cs.s1 * shared_y; \ + total_sum.s1 += ((float)(((bits4.s5 & 0x000F) ) | ((bits2.s5 & 0x03) << 4)) - 32.f) * cs.s3 * shared_y; \ + shared_y = sub_group_broadcast(y.s1, 3); \ + total_sum.s0 += ((float)(((bits4.s4 & 0x00F0) >> 4) | ((bits2.s4 & 0x0C) << 2)) - 32.f) * cs.s1 * shared_y; \ + total_sum.s1 += ((float)(((bits4.s5 & 0x00F0) >> 4) | ((bits2.s5 & 0x0C) << 2)) - 32.f) * cs.s3 * shared_y; \ + shared_y = sub_group_broadcast(y.s2, 3); \ + total_sum.s0 += ((float)(((bits4.s4 & 0x0F00) >> 8) | ((bits2.s4 & 0x30) )) - 32.f) * cs.s1 * shared_y; \ + total_sum.s1 += ((float)(((bits4.s5 & 0x0F00) >> 8) | ((bits2.s5 & 0x30) )) - 32.f) * cs.s3 * shared_y; \ + shared_y = sub_group_broadcast(y.s3, 3); \ + total_sum.s0 += ((float)(((bits4.s4 & 0xF000) >> 12) | ((bits2.s4 & 0xC0) >> 2)) - 32.f) * cs.s1 * shared_y; \ + total_sum.s1 += ((float)(((bits4.s5 & 0xF000) >> 12) | ((bits2.s5 & 0xC0) >> 2)) - 32.f) * cs.s3 * shared_y; \ + shared_y = sub_group_broadcast(y.s4, 3); \ + total_sum.s0 += ((float)(((bits4.s6 & 0x000F) ) | ((bits2.s6 & 0x03) << 4)) - 32.f) * cs.s1 * shared_y; \ + total_sum.s1 += ((float)(((bits4.s7 & 0x000F) ) | ((bits2.s7 & 0x03) << 4)) - 32.f) * cs.s3 * shared_y; \ + shared_y = sub_group_broadcast(y.s5, 3); \ + total_sum.s0 += ((float)(((bits4.s6 & 0x00F0) >> 4) | ((bits2.s6 & 0x0C) << 2)) - 32.f) * cs.s1 * shared_y; \ + total_sum.s1 += ((float)(((bits4.s7 & 0x00F0) >> 4) | ((bits2.s7 & 0x0C) << 2)) - 32.f) * cs.s3 * shared_y; \ + shared_y = sub_group_broadcast(y.s6, 3); \ + total_sum.s0 += ((float)(((bits4.s6 & 0x0F00) >> 8) | ((bits2.s6 & 0x30) )) - 32.f) * cs.s1 * shared_y; \ + total_sum.s1 += ((float)(((bits4.s7 & 0x0F00) >> 8) | ((bits2.s7 & 0x30) )) - 32.f) * cs.s3 * shared_y; \ + shared_y = sub_group_broadcast(y.s7, 3); \ + total_sum.s0 += ((float)(((bits4.s6 & 0xF000) >> 12) | ((bits2.s6 & 0xC0) >> 2)) - 32.f) * cs.s1 * shared_y; \ + total_sum.s1 += ((float)(((bits4.s7 & 0xF000) >> 12) | ((bits2.s7 & 0xC0) >> 2)) - 32.f) * cs.s3 * shared_y; \ + +#if defined(ADRENO_GPU) +REQD_SUBGROUP_SIZE_64 +#endif +// Q6K_O4_GLOBAL: read the (read-once-per-token, no-reuse) lm_head/embed weights +// from __global coalesced instead of image1d_buffer. The texture cache caps the +// streaming (no-reuse) lm_head read bandwidth; global coalesced reaches the +// higher rate the rest of the model gets. src1 (activation) stays an image (it IS reused via +// the cross-subgroup broadcast). +#ifdef Q6K_O4_GLOBAL +#define Q6K_O4_NAME kernel_gemv_noshuffle_q6_K_f32_o4_global +#define QL_ARG __global uint * src0_ql +#define QH_ARG __global half * src0_qh +#define RD_QL(b,i) (b[i]) +#define RD_QH(b,i) as_ushort(b[i]) +#else +#define Q6K_O4_NAME kernel_gemv_noshuffle_q6_K_f32_o4 +#define QL_ARG read_only image1d_buffer_t src0_ql +#define QH_ARG read_only image1d_buffer_t src0_qh +#define RD_QL(b,i) (read_imageui(b,i).x) +#define RD_QH(b,i) as_ushort(read_imageh(b,i).x) +#endif +kernel void Q6K_O4_NAME( + QL_ARG, + QH_ARG, + global half2 * src0_s, + global half2 * src0_d, + read_only image1d_buffer_t src1, + global float * dst, + ulong offsetd, + int ne00, + int ne01 +) { + int grp = get_local_id(1); + int gid = get_global_id(0); // 4-output-quad index + ushort slid = get_sub_group_local_id(); + + // Map quad index to the two pair-indices the existing 2-output access + // pattern uses (consecutive output pairs along ne01). NB: the two pairs are + // kept ADJACENT (gid*2, gid*2+1) on purpose -- a "stride-1" split (pairs + // ne01/4 apart) is slower because two distant cache-line streams have worse + // locality than the adjacent pair whose reads interleave into the same lines + // each iteration. + int gid_a = gid * 2; + int gid_b = gid * 2 + 1; + + int nb = ne00 / 32; + + uint4 reg_a_l_a, reg_a_l_b; + ushort4 reg_a_h_a, reg_a_h_b; + half2 reg_d_a, reg_d_b; + char4 reg_s_a, reg_s_b; + float8 reg_b; + + float2 total_sum_a = 0.0f; + float2 total_sum_b = 0.0f; + + int line_stride_a = ne01 / 2; + int block_stride_a = NSUBGROUPS * ne01; + + for (int k = grp; k < nb; k += NSUBGROUPS) { + reg_d_a = src0_d[gid_a + k/8 * line_stride_a]; + reg_d_b = src0_d[gid_b + k/8 * line_stride_a]; + reg_s_a = as_char4(src0_s[gid_a + k * line_stride_a]); + reg_s_b = as_char4(src0_s[gid_b + k * line_stride_a]); + // Precompute the loop-invariant combined scale (sub-block scale * super-block d) + // once per pair instead of re-multiplying it for every one of the 256 elements. + float4 cs_a = (float4)((float)reg_s_a.s0*(float)reg_d_a.s0, (float)reg_s_a.s1*(float)reg_d_a.s0, + (float)reg_s_a.s2*(float)reg_d_a.s1, (float)reg_s_a.s3*(float)reg_d_a.s1); + float4 cs_b = (float4)((float)reg_s_b.s0*(float)reg_d_b.s0, (float)reg_s_b.s1*(float)reg_d_b.s0, + (float)reg_s_b.s2*(float)reg_d_b.s1, (float)reg_s_b.s3*(float)reg_d_b.s1); + + if (slid < 4) { + reg_b.s0123 = read_imagef(src1, 0 + slid*2 + k*8); + reg_b.s4567 = read_imagef(src1, 1 + slid*2 + k*8); + } + + // Pair a (output rows gid_a*2, gid_a*2+1): read hi+lo then dequant + // both in one block so the `_lo` macro can see the `shared_y` that + // `_hi` declared. Pair b follows in its own block — fresh shared_y. + { + reg_a_l_a.s0 = RD_QL(src0_ql, gid_a + k*block_stride_a + line_stride_a*0); + reg_a_l_a.s1 = RD_QL(src0_ql, gid_a + k*block_stride_a + line_stride_a*1); + reg_a_l_a.s2 = RD_QL(src0_ql, gid_a + k*block_stride_a + line_stride_a*2); + reg_a_l_a.s3 = RD_QL(src0_ql, gid_a + k*block_stride_a + line_stride_a*3); + reg_a_h_a.s0 = RD_QH(src0_qh, gid_a + k*block_stride_a + line_stride_a*0); + reg_a_h_a.s1 = RD_QH(src0_qh, gid_a + k*block_stride_a + line_stride_a*1); + reg_a_h_a.s2 = RD_QH(src0_qh, gid_a + k*block_stride_a + line_stride_a*2); + reg_a_h_a.s3 = RD_QH(src0_qh, gid_a + k*block_stride_a + line_stride_a*3); +#ifdef VECTOR_SUB_GROUP_BROADCAT + dequantize_block_acc_bcast_8_hi(total_sum_a, as_ushort8(reg_a_l_a), as_uchar8(reg_a_h_a), cs_a, reg_b); +#else + dequantize_block_acc_bcast_1_hi(total_sum_a, as_ushort8(reg_a_l_a), as_uchar8(reg_a_h_a), cs_a, reg_b); +#endif + + reg_a_l_a.s0 = RD_QL(src0_ql, gid_a + k*block_stride_a + line_stride_a*4); + reg_a_l_a.s1 = RD_QL(src0_ql, gid_a + k*block_stride_a + line_stride_a*5); + reg_a_l_a.s2 = RD_QL(src0_ql, gid_a + k*block_stride_a + line_stride_a*6); + reg_a_l_a.s3 = RD_QL(src0_ql, gid_a + k*block_stride_a + line_stride_a*7); + reg_a_h_a.s0 = RD_QH(src0_qh, gid_a + k*block_stride_a + line_stride_a*4); + reg_a_h_a.s1 = RD_QH(src0_qh, gid_a + k*block_stride_a + line_stride_a*5); + reg_a_h_a.s2 = RD_QH(src0_qh, gid_a + k*block_stride_a + line_stride_a*6); + reg_a_h_a.s3 = RD_QH(src0_qh, gid_a + k*block_stride_a + line_stride_a*7); +#ifdef VECTOR_SUB_GROUP_BROADCAT + dequantize_block_acc_bcast_8_lo(total_sum_a, as_ushort8(reg_a_l_a), as_uchar8(reg_a_h_a), cs_a, reg_b); +#else + dequantize_block_acc_bcast_1_lo(total_sum_a, as_ushort8(reg_a_l_a), as_uchar8(reg_a_h_a), cs_a, reg_b); +#endif + } + + { + reg_a_l_b.s0 = RD_QL(src0_ql, gid_b + k*block_stride_a + line_stride_a*0); + reg_a_l_b.s1 = RD_QL(src0_ql, gid_b + k*block_stride_a + line_stride_a*1); + reg_a_l_b.s2 = RD_QL(src0_ql, gid_b + k*block_stride_a + line_stride_a*2); + reg_a_l_b.s3 = RD_QL(src0_ql, gid_b + k*block_stride_a + line_stride_a*3); + reg_a_h_b.s0 = RD_QH(src0_qh, gid_b + k*block_stride_a + line_stride_a*0); + reg_a_h_b.s1 = RD_QH(src0_qh, gid_b + k*block_stride_a + line_stride_a*1); + reg_a_h_b.s2 = RD_QH(src0_qh, gid_b + k*block_stride_a + line_stride_a*2); + reg_a_h_b.s3 = RD_QH(src0_qh, gid_b + k*block_stride_a + line_stride_a*3); +#ifdef VECTOR_SUB_GROUP_BROADCAT + dequantize_block_acc_bcast_8_hi(total_sum_b, as_ushort8(reg_a_l_b), as_uchar8(reg_a_h_b), cs_b, reg_b); +#else + dequantize_block_acc_bcast_1_hi(total_sum_b, as_ushort8(reg_a_l_b), as_uchar8(reg_a_h_b), cs_b, reg_b); +#endif + + reg_a_l_b.s0 = RD_QL(src0_ql, gid_b + k*block_stride_a + line_stride_a*4); + reg_a_l_b.s1 = RD_QL(src0_ql, gid_b + k*block_stride_a + line_stride_a*5); + reg_a_l_b.s2 = RD_QL(src0_ql, gid_b + k*block_stride_a + line_stride_a*6); + reg_a_l_b.s3 = RD_QL(src0_ql, gid_b + k*block_stride_a + line_stride_a*7); + reg_a_h_b.s0 = RD_QH(src0_qh, gid_b + k*block_stride_a + line_stride_a*4); + reg_a_h_b.s1 = RD_QH(src0_qh, gid_b + k*block_stride_a + line_stride_a*5); + reg_a_h_b.s2 = RD_QH(src0_qh, gid_b + k*block_stride_a + line_stride_a*6); + reg_a_h_b.s3 = RD_QH(src0_qh, gid_b + k*block_stride_a + line_stride_a*7); +#ifdef VECTOR_SUB_GROUP_BROADCAT + dequantize_block_acc_bcast_8_lo(total_sum_b, as_ushort8(reg_a_l_b), as_uchar8(reg_a_h_b), cs_b, reg_b); +#else + dequantize_block_acc_bcast_1_lo(total_sum_b, as_ushort8(reg_a_l_b), as_uchar8(reg_a_h_b), cs_b, reg_b); +#endif + } + } + + // Cross-subgroup reduce. Same shape as the 2-output kernel but with the + // pair-a and pair-b accumulators concatenated into a single float4. + local float4 reduce_lm[SUBGROUP_SIZE * 3]; + float4 acc = (float4)(total_sum_a.s0, total_sum_a.s1, total_sum_b.s0, total_sum_b.s1); + if (grp == 1) { reduce_lm[SUBGROUP_SIZE*0 + slid] = acc; } + if (grp == 2) { reduce_lm[SUBGROUP_SIZE*1 + slid] = acc; } + if (grp == 3) { reduce_lm[SUBGROUP_SIZE*2 + slid] = acc; } + + barrier(CLK_LOCAL_MEM_FENCE); + + if (grp == 0) { + acc += reduce_lm[SUBGROUP_SIZE*0 + slid]; + acc += reduce_lm[SUBGROUP_SIZE*1 + slid]; + acc += reduce_lm[SUBGROUP_SIZE*2 + slid]; + dst = (global float*)((global char*)dst + offsetd); + // The dispatch rounds ne01/4 up to the subgroup width, so the tail + // quads past the last row must not store (they wrote 128 rows past + // dst on every ne01 % 256 == 128 vocab, e.g. 151936). + if (gid * 4 + 3 < (uint)ne01) { + vstore4(acc, 0, &(dst[gid * 4])); + } + } +} diff --git a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q6_k_f32_tiled.cl b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q6_k_f32_tiled.cl new file mode 100644 index 000000000000..c5049f3964ea --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q6_k_f32_tiled.cl @@ -0,0 +1,196 @@ +// Tiled-wide q6_K GEMV for the long-vocab lm_head/embed (decode path). +// +// Pairs with kernel_convert_block_q6_k_tiled_ns (cvt.cl): the weights are laid +// out CANONICALLY (6-bit code in element order e in [0,256)) and TILED by 64 +// output rows so the 64-thread lane group coalesces every weight load. Both the +// pack (convert) and the unpack (here) are owned by us — correct by construction +// against the reference ggml q6_K dequant, no bit-interleave reverse-engineering. +// +// One work-item produces one output row. A work-group is {64 lanes, 4 subgroups}: +// the 64 lanes cover the 64 rows of one tile (coalesced reads), the 4 subgroups +// split the K-blocks and reduce through __local at the end. +// +// Weights are read from __global (coalesced) rather than image1d_buffer: the +// lm_head is read once per token with no reuse, and the Adreno texture cache +// caps such a streaming read well below the coalesced-global rate +// (see opencl_q6k_gemv_o4_shipped / x2-90 roofline notes). + +#pragma OPENCL EXTENSION cl_khr_fp16 : enable + +#ifdef cl_qcom_reqd_sub_group_size +#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable +#define ADRENO_GPU 1 +#define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size("half"))) +#endif + +#define NSUBGROUPS 4 +#define TILE_ROWS 64 + +#if defined(ADRENO_GPU) +REQD_SUBGROUP_SIZE_64 +#endif +kernel void kernel_gemv_noshuffle_q6_K_f32_tiled( + __global uint4 * src0_ql, // tiled: 8 uint4 granules / superblock + __global uint4 * src0_qh, // tiled: 4 uint4 granules / superblock + __global char * src0_s, // tiled: 16 chars / superblock + __global half * src0_d, // tiled: 1 half / superblock + read_only image1d_buffer_t src1, // activation (RGBA f32) + global float * dst, + ulong offsetd, + int ne00, + int ne01 +) { + int grp = get_local_id(1); // subgroup index 0..3 (splits K) + int row = get_global_id(0); // output row along ne01 + int rt = row / TILE_ROWS; + int rit = row % TILE_ROWS; + + int nb = ne00 / 256; // superblocks per row + + float acc = 0.0f; + + for (int sb = grp; sb < nb; sb += NSUBGROUPS) { + int tile_blk = rt * nb + sb; // ne02 == 1 for lm_head/embed + + // d + 16 scales for this (row, superblock) + float dval = (float)src0_d[tile_blk * TILE_ROWS + rit]; + __global char * sc = src0_s + (tile_blk * TILE_ROWS + rit) * 16; + + // 32 ql-uints (8 codes/uint) + 16 qh-uints (16 codes/uint) + uint ql[32]; + uint qh[16]; + #pragma unroll + for (int g = 0; g < 8; ++g) { + uint4 v = src0_ql[(tile_blk * 8 + g) * TILE_ROWS + rit]; + ql[g*4+0] = v.x; ql[g*4+1] = v.y; ql[g*4+2] = v.z; ql[g*4+3] = v.w; + } + #pragma unroll + for (int g = 0; g < 4; ++g) { + uint4 v = src0_qh[(tile_blk * 4 + g) * TILE_ROWS + rit]; + qh[g*4+0] = v.x; qh[g*4+1] = v.y; qh[g*4+2] = v.z; qh[g*4+3] = v.w; + } + + // dequant 256 codes in canonical e-order, MAC with activation. + int act_base = sb * 64; // activation float4 pixel base (256/4) + #pragma unroll + for (int e4 = 0; e4 < 64; ++e4) { + float4 a = read_imagef(src1, act_base + e4); + #pragma unroll + for (int t = 0; t < 4; ++t) { + int e = e4 * 4 + t; + uint low4 = (ql[e >> 3] >> ((e & 7) * 4)) & 0xF; + uint hi2 = (qh[e >> 4] >> ((e & 15) * 2)) & 0x3; + int code = (int)(low4 | (hi2 << 4)) - 32; + int sidx = ((e >> 7) << 3) + (((e >> 5) & 3) << 1) + ((e >> 4) & 1); + float scale = (float)sc[sidx] * dval; + float av = (t == 0) ? a.x : (t == 1) ? a.y : (t == 2) ? a.z : a.w; + acc += (float)code * scale * av; + } + } + } + + // reduce across the NSUBGROUPS subgroups (same rit, different K-subset) + local float reduce_lm[NSUBGROUPS * TILE_ROWS]; + reduce_lm[grp * TILE_ROWS + rit] = acc; + barrier(CLK_LOCAL_MEM_FENCE); + + if (grp == 0) { + float total = reduce_lm[0 * TILE_ROWS + rit] + + reduce_lm[1 * TILE_ROWS + rit] + + reduce_lm[2 * TILE_ROWS + rit] + + reduce_lm[3 * TILE_ROWS + rit]; + dst = (global float*)((global char*)dst + offsetd); + dst[row] = total; + } +} + +// Multi-column (N=3) variant of the tiled q6_K decode GEMV, for the speculative/ +// MTP VERIFY lm_head/embed (ne1=3 = 2 drafts + 1 bonus). Identical tiled weight +// layout + unpack as the ne1=1 kernel above; each WI computes 3 output columns, +// streaming the (large) lm_head weight ONCE per superblock and reusing it across +// the 3 verify activation columns (dequant once per code, MAC into 3 accs). This +// is the lm_head analogue of the per-layer mc3 GEMV; the multiply order matches +// the ne1=1 kernel, so each column is byte-identical to a standalone tiled GEMV. +#if defined(ADRENO_GPU) +REQD_SUBGROUP_SIZE_64 +#endif +kernel void kernel_gemv_noshuffle_q6_K_f32_tiled_mc3( + __global uint4 * src0_ql, + __global uint4 * src0_qh, + __global char * src0_s, + __global half * src0_d, + read_only image1d_buffer_t src1, + global float * dst, + ulong offsetd, + int ne00, + int ne01 +) { + int grp = get_local_id(1); + int row = get_global_id(0); + int rt = row / TILE_ROWS; + int rit = row % TILE_ROWS; + + int nb = ne00 / 256; + int col_stride = ne00 / 4; // activation float4 pixels per column + + float acc0 = 0.0f, acc1 = 0.0f, acc2 = 0.0f; + + for (int sb = grp; sb < nb; sb += NSUBGROUPS) { + int tile_blk = rt * nb + sb; + + float dval = (float)src0_d[tile_blk * TILE_ROWS + rit]; + __global char * sc = src0_s + (tile_blk * TILE_ROWS + rit) * 16; + + uint ql[32]; + uint qh[16]; + #pragma unroll + for (int g = 0; g < 8; ++g) { + uint4 v = src0_ql[(tile_blk * 8 + g) * TILE_ROWS + rit]; + ql[g*4+0] = v.x; ql[g*4+1] = v.y; ql[g*4+2] = v.z; ql[g*4+3] = v.w; + } + #pragma unroll + for (int g = 0; g < 4; ++g) { + uint4 v = src0_qh[(tile_blk * 4 + g) * TILE_ROWS + rit]; + qh[g*4+0] = v.x; qh[g*4+1] = v.y; qh[g*4+2] = v.z; qh[g*4+3] = v.w; + } + + int act_base = sb * 64; + #pragma unroll + for (int e4 = 0; e4 < 64; ++e4) { + float4 a0 = read_imagef(src1, 0*col_stride + act_base + e4); + float4 a1 = read_imagef(src1, 1*col_stride + act_base + e4); + float4 a2 = read_imagef(src1, 2*col_stride + act_base + e4); + #pragma unroll + for (int t = 0; t < 4; ++t) { + int e = e4 * 4 + t; + uint low4 = (ql[e >> 3] >> ((e & 7) * 4)) & 0xF; + uint hi2 = (qh[e >> 4] >> ((e & 15) * 2)) & 0x3; + int code = (int)(low4 | (hi2 << 4)) - 32; + int sidx = ((e >> 7) << 3) + (((e >> 5) & 3) << 1) + ((e >> 4) & 1); + float w = (float)code * ((float)sc[sidx] * dval); // dequant+scale once + float av0 = (t == 0) ? a0.x : (t == 1) ? a0.y : (t == 2) ? a0.z : a0.w; + float av1 = (t == 0) ? a1.x : (t == 1) ? a1.y : (t == 2) ? a1.z : a1.w; + float av2 = (t == 0) ? a2.x : (t == 1) ? a2.y : (t == 2) ? a2.z : a2.w; + acc0 += w * av0; + acc1 += w * av1; + acc2 += w * av2; + } + } + } + + local float4 reduce_lm[NSUBGROUPS * TILE_ROWS]; + reduce_lm[grp * TILE_ROWS + rit] = (float4)(acc0, acc1, acc2, 0.0f); + barrier(CLK_LOCAL_MEM_FENCE); + + if (grp == 0) { + float4 total = reduce_lm[0 * TILE_ROWS + rit] + + reduce_lm[1 * TILE_ROWS + rit] + + reduce_lm[2 * TILE_ROWS + rit] + + reduce_lm[3 * TILE_ROWS + rit]; + dst = (global float*)((global char*)dst + offsetd); + // dst column-major [ne01 rows x 3 cols]: (row, col) at col*ne01 + row + dst[0*ne01 + row] = total.x; + dst[1*ne01 + row] = total.y; + dst[2*ne01 + row] = total.z; + } +} diff --git a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q8_0_f32.cl b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q8_0_f32.cl index 09bae2d555e2..6f6d7425c656 100644 --- a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q8_0_f32.cl +++ b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q8_0_f32.cl @@ -118,6 +118,87 @@ elem = (char)((bits8.s7 & 0xFF000000) >> 24); \ total_sums += convert_int(elem) * scale * shared_y; \ +// ============================================================================ +// Split-K variant for small-M decode GEMVs. +// ---------------------------------------------------------------------------- +// The base kernel below puts one output row per lane and splits K only across +// the N_SIMDGROUP subgroups of a single workgroup, so M=512 yields M/64 = 8 +// workgroups -- half the compute units on a 16-CU X2 sit idle, and the kernel +// measures ~48 GB/s against the ~122 GB/s the larger projections reach in the +// same graph. Here each (kslice, subgroup) pair reduces a disjoint set of +// K-blocks into partial[kslice * M + row]; kernel_gemv_splitk_reduce_f32 (in +// gemv_noshuffle_q4_k_f32.cl) sums the slices. Same operand order within a +// slice as the base kernel; only the cross-slice grouping differs. +// +// Placed BEFORE the base kernel deliberately: on A6X no kernel may be defined +// after one that uses a subgroup builtin, or it silently miscompiles. +// ============================================================================ +#ifdef ADRENO_GPU +REQD_SUBGROUP_SIZE_64 +#endif +__kernel void kernel_gemv_noshuffle_q8_0_f32_splitk( + __read_only image1d_buffer_t src0_q, // quantized A (weights) + global half * src0_d, // A scales + __read_only image1d_buffer_t src1, // B (activations) + global float * partial, // [ksplit * M], slice-major + int ne00, // K + int ne01) // M +{ + uint groupId = get_local_id(1); + uint gid = get_global_id(0); + ushort slid = get_sub_group_local_id(); + uint nsg = get_local_size(1); + uint ksplit = get_num_groups(1); + uint kslice = get_group_id(1); + + uint K = ne00; + uint M = ne01; + + uint LINE_STRIDE_A = M; + uint BLOCK_STRIDE_A = 8 * M; // physical, independent of the K-split + + __private uint8 regA; + __private half regS; + __private float8 regB; + __private float totalSum = (float)(0.0f); + + #pragma unroll 1 + for (uint k = kslice * nsg + groupId; k < (K / QK8_0); k += ksplit * nsg) { + regS = src0_d[gid + k * LINE_STRIDE_A]; + if (slid < 4) { + regB.s0123 = read_imagef(src1, (slid * 2 + k * 8)); + regB.s4567 = read_imagef(src1, (1 + slid * 2 + k * 8)); + } + regA.s0 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 0)).x; + regA.s1 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 1)).x; + regA.s2 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 2)).x; + regA.s3 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 3)).x; + regA.s4 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 4)).x; + regA.s5 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 5)).x; + regA.s6 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 6)).x; + regA.s7 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 7)).x; + + dequantizeBlockAccum_ns_sgbroadcast_1(totalSum, regA, convert_float(regS), regB); + } + + // Intra-workgroup reduce across this K-slice's subgroups. Sized for + // nsg <= 8; the host never dispatches more. + __local float reduceLM[SIMDGROUP_WIDTH * 7]; + if (groupId > 0) { + reduceLM[SIMDGROUP_WIDTH * (groupId - 1) + slid] = totalSum; + } + barrier(CLK_LOCAL_MEM_FENCE); + if (groupId == 0) { + for (uint i = 0; i < nsg - 1; ++i) { + totalSum += reduceLM[SIMDGROUP_WIDTH * i + slid]; + } + // x-grid is padded to CEIL_DIV(M,wave)*wave; guard the tail rows. + if (gid < M) { + partial[kslice * M + gid] = totalSum; + } + } +} + #ifdef ADRENO_GPU REQD_SUBGROUP_SIZE_64 #endif diff --git a/ggml/src/ggml-opencl/kernels/mul_mm_f32_f32_l4_lm.cl b/ggml/src/ggml-opencl/kernels/mul_mm_f32_f32_l4_lm.cl index d7d5ba647e70..9dc9862bef63 100644 --- a/ggml/src/ggml-opencl/kernels/mul_mm_f32_f32_l4_lm.cl +++ b/ggml/src/ggml-opencl/kernels/mul_mm_f32_f32_l4_lm.cl @@ -145,3 +145,52 @@ kernel void kernel_mul_mm_f32_f32_l4_lm( } } } + +// Multi-column f32 GEMV for the small-N (spec/MTP verify) batch. The tiled GEMM +// above always computes a full BM x BN = 64 x 64 output tile, so at ne11=3 with a +// skinny weight (e.g. GDN ssm_alpha/ssm_beta, M=32) it launches ONE under-occupied +// workgroup at ~2.3% tile utilization. This kernel assigns one 64-thread workgroup +// per output element (m,n): the 64 threads split the K reduction (float4) and +// tree-reduce in __local (no subgroup ops -> portable). ne01*ne11 workgroups. +// Weight row is re-read per column (N small -> negligible). Summation order differs +// from the tiled GEMM (lane-strided + tree) -> f32-exact-ish, not bit-identical. +kernel void kernel_gemv_f32_f32_mc( + global float * src0, ulong offset0, // weight: row m at m*stride_a (elements) + global float * src1, ulong offset1, // activations: col n at n*stride_b + global float * dst, ulong offsetd, // dst [M x N] col-major: (m,n) at n*stride_d+m + int ne00, // K + int ne01, // M + int ne11, // N + int stride_a, // weight row stride (elements) = K + int stride_b, // activation col stride (elements) = K + int stride_d) // dst column stride (elements) = M +{ + src0 = (global float*)((global char*)src0 + offset0); + src1 = (global float*)((global char*)src1 + offset1); + dst = (global float*)((global char*)dst + offsetd); + + uint lane = get_local_id(0); // 0..63 + uint out = get_global_id(1); // 0 .. ne01*ne11 - 1 + uint m = out % (uint)ne01; + uint n = out / (uint)ne01; + + global float4 * wrow = (global float4*)(src0 + (ulong)m * (uint)stride_a); + global float4 * xcol = (global float4*)(src1 + (ulong)n * (uint)stride_b); + uint k4 = (uint)ne00 >> 2; + + float acc = 0.0f; + for (uint k = lane; k < k4; k += 64) { + float4 w = wrow[k]; + float4 x = xcol[k]; + acc += w.s0*x.s0 + w.s1*x.s1 + w.s2*x.s2 + w.s3*x.s3; + } + + local float red[64]; + red[lane] = acc; + barrier(CLK_LOCAL_MEM_FENCE); + for (uint s = 32; s > 0; s >>= 1) { + if (lane < s) red[lane] += red[lane + s]; + barrier(CLK_LOCAL_MEM_FENCE); + } + if (lane == 0) dst[(ulong)n * (uint)stride_d + m] = red[0]; +} diff --git a/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32_mrow.cl b/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32_mrow.cl new file mode 100644 index 000000000000..9a7627cf9be1 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32_mrow.cl @@ -0,0 +1,306 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable + +#ifdef cl_intel_subgroups +#pragma OPENCL EXTENSION cl_intel_subgroups : enable +#else +#pragma OPENCL EXTENSION cl_khr_subgroups : enable +#endif + +#ifdef cl_intel_required_subgroup_size +#pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable +#define INTEL_GPU 1 +#define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16))) +#define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32))) +#elif defined(cl_qcom_reqd_sub_group_size) +#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable +#define ADRENO_GPU 1 +#define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size("half"))) +#define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size("full"))) +#endif + +// Multi-row f16xf32 GEMV for the DECODE path (single token, ne11*ne12 small). +// The legacy kernel_mul_mat_f16_f32_1row runs ONE 64-lane subgroup per workgroup = +// one output row per WG, which caps memory-level parallelism at roughly half of +// LPDDR5x peak. This variant packs MROW subgroups per workgroup, each +// computing a distinct output row, so a WG keeps 64*MROW loads in flight. The +// activation column y (shared by every output row) is staged into __local ONCE per +// WG and reused across the MROW rows, cutting redundant activation reads. Used for +// the f16 attention projections (Q/K/V/O) and lm_head, which dominate decode. +// Numerically equivalent to _1row (same f16->f32 widening, same float4 partial sums, +// same subgroup-reduce order), so byte-identical to the per-op path. + +#define MROW 16 + +#ifdef ADRENO_GPU +REQD_SUBGROUP_SIZE_64 +#endif +kernel void kernel_mul_mat_f16_f32_mrow( + global char * src0, + ulong offset0, + global char * src1, + ulong offset1, + global float * dst, + ulong offsetd, + int ne00, + int ne01, + int ne02, + ulong nb00, + ulong nb01, + ulong nb02, + ulong nb03, + int ne10, + int ne11, + int ne12, + ulong nb10, + ulong nb11, + ulong nb12, + ulong nb13, + int ne0, + int ne1, + int r2, + int r3, + __local float * ysh +) { + src0 = (global char*)((global char*)src0 + offset0); + src1 = (global char*)((global char*)src1 + offset1); + dst = (global float*)((global char*)dst + offsetd); + + int r0 = get_group_id(0) * MROW + get_local_id(1); // output row + int r1 = get_group_id(1); // token (ne11) + int im = get_group_id(2); + int lid = get_sub_group_local_id(); // 0..63 + int nsg = get_local_size(1); // == MROW + + int i12 = im % ne12; + int i13 = im / ne12; + + ulong offset_src1 = r1*nb11 + (i12)*nb12 + (i13)*nb13; + global float * y = (global float *) (src1 + offset_src1); + + // Cooperatively stage the activation column (ne00 floats) into __local once per + // WG and reuse across the MROW rows. Staging is the actual win here: dropping it + // (each subgroup re-reading y from global) regresses below the 1-row kernel. + for (int i = get_local_id(1)*get_sub_group_size() + lid; i < ne00; i += nsg*get_sub_group_size()) { + ysh[i] = y[i]; + } + barrier(CLK_LOCAL_MEM_FENCE); + + if (r0 >= ne01) { + return; + } + + ulong offset_src0 = r0*nb01 + (i12/r2)*nb02 + (i13/r3)*nb03; + global half * x = (global half *) (src0 + offset_src0); + + // The vector path below casts the row pointer to half4, which must be 8-byte aligned. + // A row address is r0*nb01 + ..., and a permuted or strided src0 leaves nb01/nb02/nb03 + // unconstrained -- ne00 % 4 == 0 bounds the element count per row, not the byte stride + // between rows. Take the vector path only when this work-item's row is actually + // aligned; the scalar loop below has no such requirement. + const bool row_aligned = (((ulong) x) & 7) == 0; + + float sumf = 0.0f; + if (ne00 < 128 || !row_aligned) { + for (int i = lid; i < ne00; i += get_sub_group_size()) { + sumf += (float) x[i] * ysh[i]; + } + float all_sum = sub_group_reduce_add(sumf); + if (lid == 0) { + dst[im*ne1*ne0 + r1*ne0 + r0] = all_sum; + } + } else { + global half4 * x4 = (global half4 *) x; + __local float4 * ysh4 = (__local float4 *) ysh; + for (int i = lid; i < ne00/4; i += get_sub_group_size()) { + float4 yv = ysh4[i]; + sumf += (float) x4[i].s0 * yv.s0; + sumf += (float) x4[i].s1 * yv.s1; + sumf += (float) x4[i].s2 * yv.s2; + sumf += (float) x4[i].s3 * yv.s3; + } + float all_sum = sub_group_reduce_add(sumf); + if (lid == 0) { + for (int i = 4*(ne00/4); i < ne00; ++i) { + all_sum += (float) x[i] * ysh[i]; + } + dst[im*ne1*ne0 + r1*ne0 + r0] = all_sum; + } + } +} + +// Register-blocked variant: each 64-lane subgroup accumulates RPT consecutive +// output rows instead of one. The staged activation is reused across all RPT rows, +// and each lane keeps RPT independent weight loads in flight per column step -> +// more memory-level parallelism on the streaming f16 weight read (the BW limiter), +// plus RPT fewer staging barriers per output row. Per-row reduction order is +// identical to _mrow, so byte-identical to the per-op path. Dispatch guarantees +// ne00 >= 128 and ne00 % 4 == 0, so only the half4 path is needed (no tail). +#define MROW_RB_BODY(RPT) \ + src0 = (global char*)((global char*)src0 + offset0); \ + src1 = (global char*)((global char*)src1 + offset1); \ + dst = (global float*)((global char*)dst + offsetd); \ + int r0b = (get_group_id(0) * get_local_size(1) + get_local_id(1)) * (RPT); \ + int r1 = get_group_id(1); \ + int im = get_group_id(2); \ + int lid = get_sub_group_local_id(); \ + int nsg = get_local_size(1); \ + int i12 = im % ne12; \ + int i13 = im / ne12; \ + ulong off_y = r1*nb11 + i12*nb12 + i13*nb13; \ + global float * y = (global float *) (src1 + off_y); \ + for (int i = get_local_id(1)*get_sub_group_size() + lid; i < ne00; \ + i += nsg*get_sub_group_size()) { \ + ysh[i] = y[i]; \ + } \ + barrier(CLK_LOCAL_MEM_FENCE); \ + __local float4 * ysh4 = (__local float4 *) ysh; \ + global half4 * xr[RPT]; \ + _Pragma("unroll") \ + for (int rr = 0; rr < (RPT); ++rr) { \ + int row = r0b + rr; \ + if (row > ne01 - 1) row = ne01 - 1; \ + xr[rr] = (global half4 *) (src0 + (ulong)row*nb01 + (i12/r2)*nb02 + (i13/r3)*nb03); \ + } \ + float sumf[RPT]; \ + _Pragma("unroll") \ + for (int rr = 0; rr < (RPT); ++rr) sumf[rr] = 0.0f; \ + for (int i = lid; i < ne00/4; i += get_sub_group_size()) { \ + float4 yv = ysh4[i]; \ + _Pragma("unroll") \ + for (int rr = 0; rr < (RPT); ++rr) { \ + half4 xv = xr[rr][i]; \ + sumf[rr] += (float) xv.s0 * yv.s0 + (float) xv.s1 * yv.s1 \ + + (float) xv.s2 * yv.s2 + (float) xv.s3 * yv.s3; \ + } \ + } \ + _Pragma("unroll") \ + for (int rr = 0; rr < (RPT); ++rr) { \ + float s = sub_group_reduce_add(sumf[rr]); \ + int row = r0b + rr; \ + if (lid == 0 && row < ne01) { \ + dst[im*ne1*ne0 + r1*ne0 + row] = s; \ + } \ + } + +// half8 (128-bit) load variant: Adreno's load/store unit issues 128-bit +// transactions, so half4 (64-bit) loads may leave the load path half-idle. This +// processes 8 weight elements per lane per step via half8. Accumulation groups +// elements in 8s rather than 4s, so it is NOT bit-identical to _1row (float add is +// non-associative) -- experimental BW probe, gate on ne00 % 8 == 0. +#define MROW_H8_BODY(RPT) \ + src0 = (global char*)((global char*)src0 + offset0); \ + src1 = (global char*)((global char*)src1 + offset1); \ + dst = (global float*)((global char*)dst + offsetd); \ + int r0b = (get_group_id(0) * get_local_size(1) + get_local_id(1)) * (RPT); \ + int r1 = get_group_id(1); \ + int im = get_group_id(2); \ + int lid = get_sub_group_local_id(); \ + int nsg = get_local_size(1); \ + int i12 = im % ne12; \ + int i13 = im / ne12; \ + ulong off_y = r1*nb11 + i12*nb12 + i13*nb13; \ + global float * y = (global float *) (src1 + off_y); \ + for (int i = get_local_id(1)*get_sub_group_size() + lid; i < ne00; \ + i += nsg*get_sub_group_size()) { \ + ysh[i] = y[i]; \ + } \ + barrier(CLK_LOCAL_MEM_FENCE); \ + __local float4 * ysh4 = (__local float4 *) ysh; \ + global half8 * xr[RPT]; \ + _Pragma("unroll") \ + for (int rr = 0; rr < (RPT); ++rr) { \ + int row = r0b + rr; \ + if (row > ne01 - 1) row = ne01 - 1; \ + xr[rr] = (global half8 *) (src0 + (ulong)row*nb01 + (i12/r2)*nb02 + (i13/r3)*nb03); \ + } \ + float sumf[RPT]; \ + _Pragma("unroll") \ + for (int rr = 0; rr < (RPT); ++rr) sumf[rr] = 0.0f; \ + for (int i = lid; i < ne00/8; i += get_sub_group_size()) { \ + float4 y0 = ysh4[2*i]; \ + float4 y1 = ysh4[2*i + 1]; \ + _Pragma("unroll") \ + for (int rr = 0; rr < (RPT); ++rr) { \ + half8 xv = xr[rr][i]; \ + sumf[rr] += (float) xv.s0 * y0.s0 + (float) xv.s1 * y0.s1 \ + + (float) xv.s2 * y0.s2 + (float) xv.s3 * y0.s3 \ + + (float) xv.s4 * y1.s0 + (float) xv.s5 * y1.s1 \ + + (float) xv.s6 * y1.s2 + (float) xv.s7 * y1.s3; \ + } \ + } \ + _Pragma("unroll") \ + for (int rr = 0; rr < (RPT); ++rr) { \ + float s = sub_group_reduce_add(sumf[rr]); \ + int row = r0b + rr; \ + if (lid == 0 && row < ne01) { \ + dst[im*ne1*ne0 + r1*ne0 + row] = s; \ + } \ + } + +#ifdef ADRENO_GPU +REQD_SUBGROUP_SIZE_64 +#endif +kernel void kernel_mul_mat_f16_f32_mrow_h8( + global char * src0, ulong offset0, + global char * src1, ulong offset1, + global float * dst, ulong offsetd, + int ne00, int ne01, int ne02, + ulong nb00, ulong nb01, ulong nb02, ulong nb03, + int ne10, int ne11, int ne12, + ulong nb10, ulong nb11, ulong nb12, ulong nb13, + int ne0, int ne1, int r2, int r3, + __local float * ysh +) { + MROW_H8_BODY(1) +} + +#ifdef ADRENO_GPU +REQD_SUBGROUP_SIZE_64 +#endif +kernel void kernel_mul_mat_f16_f32_mrow_h8r2( + global char * src0, ulong offset0, + global char * src1, ulong offset1, + global float * dst, ulong offsetd, + int ne00, int ne01, int ne02, + ulong nb00, ulong nb01, ulong nb02, ulong nb03, + int ne10, int ne11, int ne12, + ulong nb10, ulong nb11, ulong nb12, ulong nb13, + int ne0, int ne1, int r2, int r3, + __local float * ysh +) { + MROW_H8_BODY(2) +} + +#ifdef ADRENO_GPU +REQD_SUBGROUP_SIZE_64 +#endif +kernel void kernel_mul_mat_f16_f32_mrow_r2( + global char * src0, ulong offset0, + global char * src1, ulong offset1, + global float * dst, ulong offsetd, + int ne00, int ne01, int ne02, + ulong nb00, ulong nb01, ulong nb02, ulong nb03, + int ne10, int ne11, int ne12, + ulong nb10, ulong nb11, ulong nb12, ulong nb13, + int ne0, int ne1, int r2, int r3, + __local float * ysh +) { + MROW_RB_BODY(2) +} + +#ifdef ADRENO_GPU +REQD_SUBGROUP_SIZE_64 +#endif +kernel void kernel_mul_mat_f16_f32_mrow_r4( + global char * src0, ulong offset0, + global char * src1, ulong offset1, + global float * dst, ulong offsetd, + int ne00, int ne01, int ne02, + ulong nb00, ulong nb01, ulong nb02, ulong nb03, + int ne10, int ne11, int ne12, + ulong nb10, ulong nb11, ulong nb12, ulong nb13, + int ne0, int ne1, int r2, int r3, + __local float * ysh +) { + MROW_RB_BODY(4) +} diff --git a/ggml/src/ggml-opencl/kernels/rms_norm.cl b/ggml/src/ggml-opencl/kernels/rms_norm.cl index 4b18d17d6f8f..99085625a4ce 100644 --- a/ggml/src/ggml-opencl/kernels/rms_norm.cl +++ b/ggml/src/ggml-opencl/kernels/rms_norm.cl @@ -188,3 +188,182 @@ kernel void kernel_rms_norm_mul( y[i00] = (x[i00] * scale) * f[i00%(ne10/4)]; } } + +//------------------------------------------------------------------------------ +// rms_norm + mul (norm weight) + add (residual), fused. Mirrors +// kernel_rms_norm_mul with an extra residual operand src2: computes +// y = (rmsnorm(x) * w) + g +// in one dispatch, removing one kernel launch + one global round-trip per +// residual block (the dominant per-layer adjacency on Gemma matformers). +//------------------------------------------------------------------------------ +kernel void kernel_rms_norm_mul_add( + global char * src0, + ulong offset0, + global char * src1, + ulong offset1, + global char * src2, + ulong offset2, + global char * dst, + ulong offsetd, + int ne00, + int ne01, + int ne02, + int ne03, + ulong nb01, + ulong nb02, + ulong nb03, + int ne10, + int ne11, + int ne12, + int ne13, + ulong nb11, + ulong nb12, + ulong nb13, + int ne20, + int ne21, + int ne22, + int ne23, + ulong nb21, + ulong nb22, + ulong nb23, + ulong nb1, + ulong nb2, + ulong nb3, + float eps, + local float * sum +) { + src0 = src0 + offset0; + src1 = src1 + offset1; + src2 = src2 + offset2; + dst = dst + offsetd; + + if (get_sub_group_id() == 0) { + sum[get_sub_group_local_id()] = 0.0f; + } + + int i03 = get_group_id(2); + int i02 = get_group_id(1); + int i01 = get_group_id(0); + + global float4 * x = (global float4 *) (src0 + i03*nb03 + i02*nb02 + i01*nb01); + global float4 * f = (global float4 *) (src1 + (i03%ne13)*nb13 + (i02%ne12)*nb12 + (i01%ne11)*nb11); + global float4 * g = (global float4 *) (src2 + (i03%ne23)*nb23 + (i02%ne22)*nb22 + (i01%ne21)*nb21); + + float sumf = 0; + + for (int i00 = get_local_id(0); i00 < ne00/4; i00 += get_local_size(0)) { + sumf += dot(x[i00], x[i00]); + } + sumf = sub_group_reduce_add(sumf); + + barrier(CLK_LOCAL_MEM_FENCE); + + if (get_sub_group_local_id() == 0) { + sum[get_sub_group_id()] = sumf; + } + + barrier(CLK_LOCAL_MEM_FENCE); + + sumf = sum[get_sub_group_local_id()]; + sumf = sub_group_reduce_add(sumf); + + float mean = sumf / ne00; + float scale = 1.0f/sqrt(mean + eps); + + global float4 * y = (global float4 *) (dst + i03*nb3 + i02*nb2 + i01*nb1); + for (int i00 = get_local_id(0); i00 < ne00/4; i00 += get_local_size(0)) { + y[i00] = (x[i00] * scale) * f[i00%(ne10/4)] + g[i00%(ne20/4)]; + } +} + +//------------------------------------------------------------------------------ +// rms_norm + mul(norm weight) + add(residual) + mul(scalar scale), fused. +// Computes y = ((rmsnorm(x) * w) + g) * s, where s is a broadcast SCALAR (e.g. +// Gemma-4 layer_output_scale). Folds the trailing per-layer l_out scale-mul into +// the residual-norm kernel: one extra dispatch + global round-trip saved per +// layer. src3 points at the single scale value. +//------------------------------------------------------------------------------ +kernel void kernel_rms_norm_mul_add_scale( + global char * src0, + ulong offset0, + global char * src1, + ulong offset1, + global char * src2, + ulong offset2, + global char * src3, + ulong offset3, + global char * dst, + ulong offsetd, + int ne00, + int ne01, + int ne02, + int ne03, + ulong nb01, + ulong nb02, + ulong nb03, + int ne10, + int ne11, + int ne12, + int ne13, + ulong nb11, + ulong nb12, + ulong nb13, + int ne20, + int ne21, + int ne22, + int ne23, + ulong nb21, + ulong nb22, + ulong nb23, + ulong nb1, + ulong nb2, + ulong nb3, + float eps, + local float * sum +) { + src0 = src0 + offset0; + src1 = src1 + offset1; + src2 = src2 + offset2; + src3 = src3 + offset3; + dst = dst + offsetd; + + const float sc = *((global float *) src3); + + if (get_sub_group_id() == 0) { + sum[get_sub_group_local_id()] = 0.0f; + } + + int i03 = get_group_id(2); + int i02 = get_group_id(1); + int i01 = get_group_id(0); + + global float4 * x = (global float4 *) (src0 + i03*nb03 + i02*nb02 + i01*nb01); + global float4 * f = (global float4 *) (src1 + (i03%ne13)*nb13 + (i02%ne12)*nb12 + (i01%ne11)*nb11); + global float4 * g = (global float4 *) (src2 + (i03%ne23)*nb23 + (i02%ne22)*nb22 + (i01%ne21)*nb21); + + float sumf = 0; + + for (int i00 = get_local_id(0); i00 < ne00/4; i00 += get_local_size(0)) { + sumf += dot(x[i00], x[i00]); + } + sumf = sub_group_reduce_add(sumf); + + barrier(CLK_LOCAL_MEM_FENCE); + + if (get_sub_group_local_id() == 0) { + sum[get_sub_group_id()] = sumf; + } + + barrier(CLK_LOCAL_MEM_FENCE); + + sumf = sum[get_sub_group_local_id()]; + sumf = sub_group_reduce_add(sumf); + + float mean = sumf / ne00; + float scale = 1.0f/sqrt(mean + eps); + + global float4 * y = (global float4 *) (dst + i03*nb3 + i02*nb2 + i01*nb1); + for (int i00 = get_local_id(0); i00 < ne00/4; i00 += get_local_size(0)) { + y[i00] = ((x[i00] * scale) * f[i00%(ne10/4)] + g[i00%(ne20/4)]) * sc; + } +}