diff --git a/tests/test-dt3-gpu.cpp b/tests/test-dt3-gpu.cpp index 54bc17f72..fde9075f8 100644 --- a/tests/test-dt3-gpu.cpp +++ b/tests/test-dt3-gpu.cpp @@ -18,7 +18,10 @@ // the elementwise maximum is reported as information only, since it explodes // on cancellation whenever a true output value is near zero. // -// MUL_MAT with more destination columns than the MMVQ limit falls back to +// MUL_MAT with more destination columns than the MMVQ limit takes the MMQ +// path where the backend implements it for DT3 (CUDA on Ampere-class +// hardware and newer): integer dot products in the same numerical regime as +// MMVQ, judged just as strictly. Backends without DT3 MMQ fall back to // dequantization + cuBLAS GEMM, which on fast-fp16 hardware rounds the // dequantized weights to fp16. DT3 weights (d1*t1 + d2*t2, the sum of two // fp16-scaled terms) are generally NOT fp16-representable, so that path is @@ -264,7 +267,8 @@ static int test_mul_mat(ggml_backend_t backend, ggml_type type, const std::vecto const std::vector & ref_w, const std::vector & y, bool strict) { int num_failed = 0; - const int ncols_dst[] = {1, 2, 5, 8, 16}; + // 100 exercises a wide MMQ tile with a clamped last column block + const int ncols_dst[] = {1, 2, 5, 8, 16, 100}; // the same weights as the fp16 GEMM fallback sees them std::vector ref_w16(ref_w.size()); @@ -274,6 +278,8 @@ static int test_mul_mat(ggml_backend_t backend, ggml_type type, const std::vecto std::vector> results; + bool n16_integer = false; // whether the ncols_dst = 16 run took an integer (MMQ) path + for (int c = 0; c < (int)(sizeof(ncols_dst)/sizeof(ncols_dst[0])); ++c) { const int n = ncols_dst[c]; @@ -324,25 +330,33 @@ static int test_mul_mat(ggml_backend_t backend, ggml_type type, const std::vecto const mat_err err16 = compare_mat(gpu, ref16); // n <= 8 is the MMVQ path with exact integer dot products, judged - // against the exact reference. Larger n is the dequantize + GEMM - // fallback whose numerics (fp16 or TF32 compute, depending on the + // against the exact reference (this mirrors MMVQ_MAX_BATCH_SIZE (8) + // from ggml-cuda/mmvq.cu by hand, because the constant and the + // per-arch should_use_mmvq tables are not exported). Larger n takes + // the MMQ path where the backend implements it for this type: integer + // dot products in the same numerical regime as MMVQ, judged just as + // strictly. Backends without MMQ for the type fall back to dequantize + // + GEMM, whose numerics (fp16 or TF32 compute, depending on the // hardware and on GGML_CUDA_CUBLAS_COMPUTE_TYPE) are cuBLAS's, not - // ours: for the strict type it is gated below by bit-identity with - // the same GEMM on an F16 tensor, and only reported here. - // This mirrors MMVQ_MAX_BATCH_SIZE (8) from ggml-cuda/mmvq.cu by hand, - // because the constant and the per-arch should_use_mmvq tables are not - // exported. If upstream raises the limit, or an architecture routes a - // larger batch through MMVQ, this gating goes stale silently: n = 16 - // would take the MMVQ path but still be judged as the GEMM one, which - // only loosens the check, never tightens it. Whoever touches the MMVQ - // dispatch should revisit this line. - const bool is_mmvq = n <= 8; - const bool gated = is_mmvq || !strict; - const double err_gate = is_mmvq ? err.norm_rel : (err.norm_rel < err16.norm_rel ? err.norm_rel : err16.norm_rel); + // ours: for the strict type that run is gated below by bit-identity + // with the same GEMM on an F16 tensor, and only reported here. + // The two regimes are told apart by the result itself: an integer path + // lands within float rounding of the exact reference, a fp16/TF32 GEMM + // stays orders of magnitude above it. A broken MMQ kernel cannot hide + // in the GEMM class: it would then have to be bit-identical to the F16 + // GEMM control below, which an integer path never is. + const bool is_mmvq = n <= 8; + const bool integer_path = is_mmvq || err.norm_rel <= 1e-5; + if (n == 16) { + n16_integer = integer_path; + } + const bool gated = integer_path || !strict; + const double err_gate = integer_path ? err.norm_rel : (err.norm_rel < err16.norm_rel ? err.norm_rel : err16.norm_rel); const double tol = strict ? 1e-5 : 1e-2; const bool failed = gated && err_gate > tol; - printf("%s: %s mul_mat GPU, ncols_dst = %2d (%s): norm rel err vs exact ref = %g, vs fp16 ref = %g (max elem rel: %g)\n", - failed ? "FAILED" : gated ? "OK" : "INFO", ggml_type_name(type), n, is_mmvq ? "MMVQ" : "GEMM", + printf("%s: %s mul_mat GPU, ncols_dst = %3d (%s): norm rel err vs exact ref = %g, vs fp16 ref = %g (max elem rel: %g)\n", + failed ? "FAILED" : gated ? "OK" : "INFO", ggml_type_name(type), n, + is_mmvq ? "MMVQ" : integer_path ? "MMQ" : "GEMM", err.norm_rel, err16.norm_rel, err.max_rel); if (failed) { num_failed++; @@ -352,24 +366,26 @@ static int test_mul_mat(ggml_backend_t backend, ggml_type type, const std::vecto ggml_free(ctx); } - // MMVQ vs the dequantization-based path: the first 8 columns of the GEMM - // run must match the ncols_dst = 8 MMVQ run to fp16 weight rounding + // MMVQ vs the batched path: the first 8 columns of the ncols_dst = 16 run + // must match the ncols_dst = 8 MMVQ run. When the batched run took the + // integer MMQ path both sides are exact to float rounding of the + // accumulation; against a GEMM fallback the gate is fp16 weight rounding. { - const std::vector & mmvq = results[3]; // n = 8 - const std::vector & gemm = results[4]; // n = 16 + const std::vector & mmvq = results[3]; // n = 8 + const std::vector & batch = results[4]; // n = 16 double num = 0.0; double den = 0.0; for (int j = 0; j < 8; ++j) { for (int r = 0; r < NROWS; ++r) { - const double diff = (double)mmvq[(size_t)j*NROWS + r] - (double)gemm[(size_t)j*NROWS + r]; + const double diff = (double)mmvq[(size_t)j*NROWS + r] - (double)batch[(size_t)j*NROWS + r]; num += diff*diff; den += (double)mmvq[(size_t)j*NROWS + r]*(double)mmvq[(size_t)j*NROWS + r]; } } const double norm_rel = sqrt(num/den); - const double tol = strict ? 5e-3 : 1e-2; - printf("%s: %s MMVQ vs GEMM path on shared columns: norm rel err = %g\n", - norm_rel <= tol ? "OK" : "FAILED", ggml_type_name(type), norm_rel); + const double tol = !strict ? 1e-2 : n16_integer ? 1e-5 : 5e-3; + printf("%s: %s MMVQ vs %s path on shared columns: norm rel err = %g\n", + norm_rel <= tol ? "OK" : "FAILED", ggml_type_name(type), n16_integer ? "MMQ" : "GEMM", norm_rel); if (norm_rel > tol) { num_failed++; } @@ -383,8 +399,14 @@ static int test_mul_mat(ggml_backend_t backend, ggml_type type, const std::vecto // fallback at a different accumulator precision than its default F16 // GEMM (Vulkan forces fp32 accumulators for DT3), so the F16 control is // run at both the default and the F32-forced precision and bit-identity - // with either one passes. - if (strict) { + // with either one passes. When the ncols_dst = 16 run took the integer + // MMQ path there is no dequantization involved and no GEMM to compare + // against — that run was already gated strictly above. + if (strict && n16_integer) { + printf("OK: %s ncols_dst = 16 took the integer MMQ path, F16 GEMM bit-identity control not applicable\n", + ggml_type_name(type)); + } + if (strict && !n16_integer) { int n_mismatch_best = -1; double max_diff_best = 0.0; @@ -522,7 +544,7 @@ int main(void) { // activations: integers with amax 127 in every 32-element chunk of every // column, so their q8_1 quantization is exact - std::vector y((size_t)NCOLS*16); + std::vector y((size_t)NCOLS*100); for (size_t i = 0; i < y.size(); ++i) { y[i] = i % 32 == 0 ? 127.0f : (float)((int)(rng_next() % 255) - 127); }