cuda : gate DT3 MMQ by batch size, cuBLAS keeps the large-batch prefill

Two integer dot products per weight cancel the 2x int8-over-fp16 tensor
core advantage, so at large batch MMQ cannot beat dequantize + fp16
cuBLAS (measured 331 vs 426 t/s pp512 on the 27B, RTX 4060 Ti, while
running at the same ~20% of its int8 ceiling as Q2_K MMQ does of its
own). MMQ still avoids the dequantization round-trip at moderate batch;
the threshold default is provisional until the crossover is measured
(GGML_CUDA_DT3_MMQ_MAX_BATCH overrides it for that measurement).

ggml/src/ggml-cuda/mmq.cu

@@ -4,6 +4,7 @@
 #include "mmid.cuh"
 
 #include <cstdint>
+#include <cstdlib>
 
 static void ggml_cuda_mul_mat_q_switch_type(ggml_backend_cuda_context & ctx, const mmq_args & args, cudaStream_t stream) {
     switch (args.type_x) {
@@ -271,6 +272,18 @@ bool ggml_cuda_should_use_mmq(enum ggml_type type, int cc, int64_t ne11, int64_t
         if (!turing_mma_available(cc)) {
             return false;
         }
+        // Two integer dot products per weight cancel the 2x int8-over-fp16 advantage
+        // of the tensor cores, so at large batch the dequantize + fp16 cuBLAS path
+        // wins; MMQ avoids the dequantization round-trip and wins below the
+        // crossover (measured on RTX 4060 Ti). Override for experiments with
+        // GGML_CUDA_DT3_MMQ_MAX_BATCH.
+        static const int64_t max_batch = []() {
+            const char * env = getenv("GGML_CUDA_DT3_MMQ_MAX_BATCH");
+            return env ? atoll(env) : 192;
+        }();
+        if (ne11 > max_batch) {
+            return false;
+        }
         const int    id    = ggml_cuda_get_device();
         const size_t smpbo = ggml_cuda_info().devices[id].smpbo;
         return mmq_get_nbytes_shared(ggml_cuda_mmq_get_config(GGML_TYPE_DT3, 8, true, cc), cc) <= smpbo;