From e9616d415ff4e2bcaa0d2a9f1f74925651cc840a Mon Sep 17 00:00:00 2001 From: Millaguie Date: Tue, 11 Aug 2026 08:51:29 +0200 Subject: [PATCH] cuda : gate DT3 MMQ by batch size, cuBLAS keeps the large-batch prefill Two integer dot products per weight cancel the 2x int8-over-fp16 tensor core advantage, so at large batch MMQ cannot beat dequantize + fp16 cuBLAS (measured 331 vs 426 t/s pp512 on the 27B, RTX 4060 Ti, while running at the same ~20% of its int8 ceiling as Q2_K MMQ does of its own). MMQ still avoids the dequantization round-trip at moderate batch; the threshold default is provisional until the crossover is measured (GGML_CUDA_DT3_MMQ_MAX_BATCH overrides it for that measurement). --- ggml/src/ggml-cuda/mmq.cu | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/ggml/src/ggml-cuda/mmq.cu b/ggml/src/ggml-cuda/mmq.cu index bfcf155bc..329e05bc6 100644 --- a/ggml/src/ggml-cuda/mmq.cu +++ b/ggml/src/ggml-cuda/mmq.cu @@ -4,6 +4,7 @@ #include "mmid.cuh" #include +#include static void ggml_cuda_mul_mat_q_switch_type(ggml_backend_cuda_context & ctx, const mmq_args & args, cudaStream_t stream) { switch (args.type_x) { @@ -271,6 +272,18 @@ bool ggml_cuda_should_use_mmq(enum ggml_type type, int cc, int64_t ne11, int64_t if (!turing_mma_available(cc)) { return false; } + // Two integer dot products per weight cancel the 2x int8-over-fp16 advantage + // of the tensor cores, so at large batch the dequantize + fp16 cuBLAS path + // wins; MMQ avoids the dequantization round-trip and wins below the + // crossover (measured on RTX 4060 Ti). Override for experiments with + // GGML_CUDA_DT3_MMQ_MAX_BATCH. + static const int64_t max_batch = []() { + const char * env = getenv("GGML_CUDA_DT3_MMQ_MAX_BATCH"); + return env ? atoll(env) : 192; + }(); + if (ne11 > max_batch) { + return false; + } const int id = ggml_cuda_get_device(); const size_t smpbo = ggml_cuda_info().devices[id].smpbo; return mmq_get_nbytes_shared(ggml_cuda_mmq_get_config(GGML_TYPE_DT3, 8, true, cc), cc) <= smpbo;