millaguie/llama.cpp
1
0
Fork 0
Code Issues Pull Requests Actions 33 Packages Projects Releases Wiki Activity

tests : judge the batched DT3 path strictly when it is MMQ
Python Type-Check / python type-check (push) Canceled after 0s
Details

Browse Source 
With MMQ, ncols_dst > 8 is an integer path in the same numerical regime
as MMVQ and is judged against the exact reference at 1e-5 instead of
riding the loose GEMM gate. The regime is told apart by the result
itself; a GEMM fallback (backends without DT3 MMQ) keeps the fp16
reference and the F16 GEMM bit-identity control. Also add ncols_dst=100
to exercise a wide tile with a clamped last column block.
This commit is contained in:
Millaguie
2026-08-11 08:33:06 +02:00
parent f46e8072d8
commit 981f439ff3
1 changed files with 51 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/test-dt3-gpu.cpp
+51 -29
View File
@@ -18,7 +18,10 @@
// the elementwise maximum is reported as information only, since it explodes
// on cancellation whenever a true output value is near zero.
//
// MUL_MAT with more destination columns than the MMVQ limit falls back to
// MUL_MAT with more destination columns than the MMVQ limit takes the MMQ
// path where the backend implements it for DT3 (CUDA on Ampere-class
// hardware and newer): integer dot products in the same numerical regime as
// MMVQ, judged just as strictly. Backends without DT3 MMQ fall back to
// dequantization + cuBLAS GEMM, which on fast-fp16 hardware rounds the
// dequantized weights to fp16. DT3 weights (d1*t1 + d2*t2, the sum of two
// fp16-scaled terms) are generally NOT fp16-representable, so that path is
@@ -264,7 +267,8 @@ static int test_mul_mat(ggml_backend_t backend, ggml_type type, const std::vecto
const std::vector<float> & ref_w, const std::vector<float> & y, bool strict) {
int num_failed = 0;
const int ncols_dst[] = {1, 2, 5, 8, 16};
// 100 exercises a wide MMQ tile with a clamped last column block
const int ncols_dst[] = {1, 2, 5, 8, 16, 100};
// the same weights as the fp16 GEMM fallback sees them
std::vector<float> ref_w16(ref_w.size());
@@ -274,6 +278,8 @@ static int test_mul_mat(ggml_backend_t backend, ggml_type type, const std::vecto
std::vector<std::vector<float>> results;
bool n16_integer = false; // whether the ncols_dst = 16 run took an integer (MMQ) path
for (int c = 0; c < (int)(sizeof(ncols_dst)/sizeof(ncols_dst[0])); ++c) {
const int n = ncols_dst[c];
@@ -324,25 +330,33 @@ static int test_mul_mat(ggml_backend_t backend, ggml_type type, const std::vecto
const mat_err err16 = compare_mat(gpu, ref16);
// n <= 8 is the MMVQ path with exact integer dot products, judged
// against the exact reference. Larger n is the dequantize + GEMM
// fallback whose numerics (fp16 or TF32 compute, depending on the
// against the exact reference (this mirrors MMVQ_MAX_BATCH_SIZE (8)
// from ggml-cuda/mmvq.cu by hand, because the constant and the
// per-arch should_use_mmvq tables are not exported). Larger n takes
// the MMQ path where the backend implements it for this type: integer
// dot products in the same numerical regime as MMVQ, judged just as
// strictly. Backends without MMQ for the type fall back to dequantize
// + GEMM, whose numerics (fp16 or TF32 compute, depending on the
// hardware and on GGML_CUDA_CUBLAS_COMPUTE_TYPE) are cuBLAS's, not
// ours: for the strict type it is gated below by bit-identity with
// the same GEMM on an F16 tensor, and only reported here.
// This mirrors MMVQ_MAX_BATCH_SIZE (8) from ggml-cuda/mmvq.cu by hand,
// because the constant and the per-arch should_use_mmvq tables are not
// exported. If upstream raises the limit, or an architecture routes a
// larger batch through MMVQ, this gating goes stale silently: n = 16
// would take the MMVQ path but still be judged as the GEMM one, which
// only loosens the check, never tightens it. Whoever touches the MMVQ
// dispatch should revisit this line.
const bool is_mmvq = n <= 8;
const bool gated = is_mmvq || !strict;
const double err_gate = is_mmvq ? err.norm_rel : (err.norm_rel < err16.norm_rel ? err.norm_rel : err16.norm_rel);
// ours: for the strict type that run is gated below by bit-identity
// with the same GEMM on an F16 tensor, and only reported here.
// The two regimes are told apart by the result itself: an integer path
// lands within float rounding of the exact reference, a fp16/TF32 GEMM
// stays orders of magnitude above it. A broken MMQ kernel cannot hide
// in the GEMM class: it would then have to be bit-identical to the F16
// GEMM control below, which an integer path never is.
const bool is_mmvq = n <= 8;
const bool integer_path = is_mmvq || err.norm_rel <= 1e-5;
if (n == 16) {
n16_integer = integer_path;
}
const bool gated = integer_path || !strict;
const double err_gate = integer_path ? err.norm_rel : (err.norm_rel < err16.norm_rel ? err.norm_rel : err16.norm_rel);
const double tol = strict ? 1e-5 : 1e-2;
const bool failed = gated && err_gate > tol;
printf("%s: %s mul_mat GPU, ncols_dst = %2d (%s): norm rel err vs exact ref = %g, vs fp16 ref = %g (max elem rel: %g)\n",
failed ? "FAILED" : gated ? "OK" : "INFO", ggml_type_name(type), n, is_mmvq ? "MMVQ" : "GEMM",
printf("%s: %s mul_mat GPU, ncols_dst = %3d (%s): norm rel err vs exact ref = %g, vs fp16 ref = %g (max elem rel: %g)\n",
failed ? "FAILED" : gated ? "OK" : "INFO", ggml_type_name(type), n,
is_mmvq ? "MMVQ" : integer_path ? "MMQ" : "GEMM",
err.norm_rel, err16.norm_rel, err.max_rel);
if (failed) {
num_failed++;
@@ -352,24 +366,26 @@ static int test_mul_mat(ggml_backend_t backend, ggml_type type, const std::vecto
ggml_free(ctx);
}
// MMVQ vs the dequantization-based path: the first 8 columns of the GEMM
// run must match the ncols_dst = 8 MMVQ run to fp16 weight rounding
// MMVQ vs the batched path: the first 8 columns of the ncols_dst = 16 run
// must match the ncols_dst = 8 MMVQ run. When the batched run took the
// integer MMQ path both sides are exact to float rounding of the
// accumulation; against a GEMM fallback the gate is fp16 weight rounding.
{
const std::vector<float> & mmvq = results[3]; // n = 8
const std::vector<float> & gemm = results[4]; // n = 16
const std::vector<float> & mmvq = results[3]; // n = 8
const std::vector<float> & batch = results[4]; // n = 16
double num = 0.0;
double den = 0.0;
for (int j = 0; j < 8; ++j) {
for (int r = 0; r < NROWS; ++r) {
const double diff = (double)mmvq[(size_t)j*NROWS + r] - (double)gemm[(size_t)j*NROWS + r];
const double diff = (double)mmvq[(size_t)j*NROWS + r] - (double)batch[(size_t)j*NROWS + r];
num += diff*diff;
den += (double)mmvq[(size_t)j*NROWS + r]*(double)mmvq[(size_t)j*NROWS + r];
}
}
const double norm_rel = sqrt(num/den);
const double tol = strict ? 5e-3 : 1e-2;
printf("%s: %s MMVQ vs GEMM path on shared columns: norm rel err = %g\n",
norm_rel <= tol ? "OK" : "FAILED", ggml_type_name(type), norm_rel);
const double tol = !strict ? 1e-2 : n16_integer ? 1e-5 : 5e-3;
printf("%s: %s MMVQ vs %s path on shared columns: norm rel err = %g\n",
norm_rel <= tol ? "OK" : "FAILED", ggml_type_name(type), n16_integer ? "MMQ" : "GEMM", norm_rel);
if (norm_rel > tol) {
num_failed++;
}
@@ -383,8 +399,14 @@ static int test_mul_mat(ggml_backend_t backend, ggml_type type, const std::vecto
// fallback at a different accumulator precision than its default F16
// GEMM (Vulkan forces fp32 accumulators for DT3), so the F16 control is
// run at both the default and the F32-forced precision and bit-identity
// with either one passes.
if (strict) {
// with either one passes. When the ncols_dst = 16 run took the integer
// MMQ path there is no dequantization involved and no GEMM to compare
// against — that run was already gated strictly above.
if (strict && n16_integer) {
printf("OK: %s ncols_dst = 16 took the integer MMQ path, F16 GEMM bit-identity control not applicable\n",
ggml_type_name(type));
}
if (strict && !n16_integer) {
int n_mismatch_best = -1;
double max_diff_best = 0.0;
@@ -522,7 +544,7 @@ int main(void) {
// activations: integers with amax 127 in every 32-element chunk of every
// column, so their q8_1 quantization is exact
std::vector<float> y((size_t)NCOLS*16);
std::vector<float> y((size_t)NCOLS*100);
for (size_t i = 0; i < y.size(); ++i) {
y[i] = i % 32 == 0 ? 127.0f : (float)((int)(rng_next() % 255) - 127);
}