diff --git a/ggml/src/ggml-cuda/mmq.cu b/ggml/src/ggml-cuda/mmq.cu index bfcf155bc..329e05bc6 100644 --- a/ggml/src/ggml-cuda/mmq.cu +++ b/ggml/src/ggml-cuda/mmq.cu @@ -4,6 +4,7 @@ #include "mmid.cuh" #include +#include static void ggml_cuda_mul_mat_q_switch_type(ggml_backend_cuda_context & ctx, const mmq_args & args, cudaStream_t stream) { switch (args.type_x) { @@ -271,6 +272,18 @@ bool ggml_cuda_should_use_mmq(enum ggml_type type, int cc, int64_t ne11, int64_t if (!turing_mma_available(cc)) { return false; } + // Two integer dot products per weight cancel the 2x int8-over-fp16 advantage + // of the tensor cores, so at large batch the dequantize + fp16 cuBLAS path + // wins; MMQ avoids the dequantization round-trip and wins below the + // crossover (measured on RTX 4060 Ti). Override for experiments with + // GGML_CUDA_DT3_MMQ_MAX_BATCH. + static const int64_t max_batch = []() { + const char * env = getenv("GGML_CUDA_DT3_MMQ_MAX_BATCH"); + return env ? atoll(env) : 192; + }(); + if (ne11 > max_batch) { + return false; + } const int id = ggml_cuda_get_device(); const size_t smpbo = ggml_cuda_info().devices[id].smpbo; return mmq_get_nbytes_shared(ggml_cuda_mmq_get_config(GGML_TYPE_DT3, 8, true, cc), cc) <= smpbo;