From ad6dd747d44bae511e0baf668cd3736c12e72caf Mon Sep 17 00:00:00 2001 From: Millaguie Date: Tue, 11 Aug 2026 03:01:09 +0200 Subject: [PATCH] vulkan : fp32 accumulators for the DT3 dequant matmul fallback The dequant fallback runs DT3 matmuls as fp16 weights through the f16 matmul pipelines, which default to fp16 accumulators when the device supports fp16. DT3 weights are the sum of two fp16-scaled ternary planes and are generally not fp16-representable, so the fallback already pays one fp16 rounding on the weights; accumulating on top of that in fp16 measurably hurts. Force GGML_PREC_F32 for the DT3 fallback, which selects the f32acc pipelines - the same numerics as the CUDA GEMM fallback (fp16 inputs, fp32 compute). Measured on Qwen2.5-3B DT3, wiki.test, 4 chunks, --no-mmap, AMD Radeon 890M (RADV STRIX1), CPU reference 15.0608: default batch, before 15.5562 (+3.29%) default batch, after 15.4062 (+2.29%) GGML_VK_DISABLE_F16=1 15.3348 (+1.82%, floor of this route) The remaining gap is shared with the mul_mat_vec route (15.3302, which does not move under GGML_VK_DISABLE_F16) and is under investigation separately; DT3 dequantization itself is bit-exact vs the CPU reference on real model tensors (214,695,936 elements, 0 mismatches). --- ggml/src/ggml-vulkan/ggml-vulkan.cpp | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cpp index 6cf501e4e..87571cd57 100644 --- a/ggml/src/ggml-vulkan/ggml-vulkan.cpp +++ b/ggml/src/ggml-vulkan/ggml-vulkan.cpp @@ -9178,12 +9178,18 @@ static void ggml_vk_mul_mat_q_f16(ggml_backend_vk_context * ctx, vk_context& sub bool quantize_y = ctx->device->integer_dot_product && src1->type == GGML_TYPE_F32 && ggml_is_contiguous(src1) && !y_non_contig && (ne11 * ne10) % 4 == 0; + // DT3 weights (d1*t1 + d2*t2, two fp16-scaled ternary planes) already pay + // one fp16 rounding in the dequant fallback; fp16 accumulation on top of + // it costs measurable perplexity. Force fp32 accumulators, matching the + // numerics of the CUDA GEMM fallback (fp16 inputs, fp32 compute). + const ggml_prec mm_prec = src0->type == GGML_TYPE_DT3 ? GGML_PREC_F32 : (ggml_prec)dst->op_params[0]; + // Check for mmq first - vk_matmul_pipeline mmp = quantize_y ? ggml_vk_get_mul_mat_mat_pipeline(ctx, src0->type, GGML_TYPE_Q8_1, (ggml_prec)dst->op_params[0]) : nullptr; + vk_matmul_pipeline mmp = quantize_y ? ggml_vk_get_mul_mat_mat_pipeline(ctx, src0->type, GGML_TYPE_Q8_1, mm_prec) : nullptr; if (mmp == nullptr) { // Fall back to f16 dequant mul mat - mmp = ggml_vk_get_mul_mat_mat_pipeline(ctx, src0->type, y_non_contig ? f16_type : src1->type, (ggml_prec)dst->op_params[0]); + mmp = ggml_vk_get_mul_mat_mat_pipeline(ctx, src0->type, y_non_contig ? f16_type : src1->type, mm_prec); quantize_y = false; } @@ -9192,7 +9198,7 @@ static void ggml_vk_mul_mat_q_f16(ggml_backend_vk_context * ctx, vk_context& sub if (qx_needs_dequant) { // Fall back to dequant + f16 mulmat - mmp = ggml_vk_get_mul_mat_mat_pipeline(ctx, f16_type, y_f32_kernel ? GGML_TYPE_F32 : f16_type, (ggml_prec)dst->op_params[0]); + mmp = ggml_vk_get_mul_mat_mat_pipeline(ctx, f16_type, y_f32_kernel ? GGML_TYPE_F32 : f16_type, mm_prec); } // Not implemented