From d343cd35931470b9cd37286eaf46a0ef81fbedc2 Mon Sep 17 00:00:00 2001 From: bri-prism <288398250+bri-prism@users.noreply.github.com> Date: Tue, 14 Jul 2026 04:14:38 +0100 Subject: [PATCH] metal: default-on nr1=2 multi-column Q2_0 matvec (spec-decode verify) Mirror the Q1_0 default to Q2_0: for ne11>=2 use the nr1=2 multi-column variant that reads each streamed q2_0 weight group once per 2 src1 columns, instead of the mul_mv_ext route. Measured (M5 Pro, in-code microbench): nr1_2 = 93.2us vs 122us ext at ne11=2 (+31%); ne11=4 via 2 passes 171 vs 183. ne11==3 is carved out -- that is the nr1_3 occupancy cliff (195 vs 152 ext, tpb=16) that kept this path opt-in; three columns stay on ext. Output is identical (pure matvec routing); base decode (ne11=1) is unaffected. GGML_METAL_Q2_0_NR1=1 restores the old routing. --- ggml/src/ggml-metal/ggml-metal-device.cpp | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/ggml/src/ggml-metal/ggml-metal-device.cpp b/ggml/src/ggml-metal/ggml-metal-device.cpp index f7e6dd8dbd08..8221df778341 100644 --- a/ggml/src/ggml-metal/ggml-metal-device.cpp +++ b/ggml/src/ggml-metal/ggml-metal-device.cpp @@ -882,6 +882,13 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mv(ggml_meta if (nr1_force > 1 && ne11 >= 2) { nr1 = std::min(nr1_force, 4); suffix = nr1 == 2 ? "_nr1_2" : nr1 == 3 ? "_nr1_3" : "_nr1_4"; + } else if (nr1_max != 1 && ne11 >= 2 && ne11 != 3) { + // Default-on nr1=2 multi-column (mirrors the Q1_0 path above): + // measured +31% at ne11=2 (93.2 vs 122 us) and a win at ne11=4 + // (2 passes, 171 vs 183). ne11==3 is carved out -- that is the + // occupancy cliff (nr1_3 loses to ext) that kept this opt-in; + // for 3 columns we stay on ext. GGML_METAL_Q2_0_NR1=1 disables. + nr1 = 2; suffix = "_nr1_2"; } } break; case GGML_TYPE_Q4_0: