millaguie/llama.cpp
1
0
Fork 0
Code Issues Pull Requests Actions 32 Packages Projects Releases Wiki Activity

tests : gate the DT3 GEMM fallback by bit-identity with an F16 GEMM

Browse Source 
The dequantize + cuBLAS fallback computes in fp16 (CUBLAS_COMPUTE_16F)
on fast-fp16 hardware and in TF32 under
GGML_CUDA_CUBLAS_COMPUTE_TYPE=f32, so no analytic tolerance separates
'correct' from 'broken' there without also tracking cuBLAS numerics.
What IS ours to guarantee: the fallback must behave exactly as if the
weights were an F16 tensor holding fp16(dequant(block)). Gate on that
bit-identity and demote the analytic GEMM errors to INFO.
This commit is contained in:
Millaguie
2026-08-10 15:20:34 +02:00
parent b8ad6c3844
commit 51c6b67e8b
1 changed files with 60 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/test-dt3-gpu.cpp
+60 -7
View File
@@ -319,17 +319,20 @@ static int test_mul_mat(ggml_backend_t backend, ggml_type type, const std::vecto
const mat_err err16 = compare_mat(gpu, ref16);
// n <= 8 is the MMVQ path with exact integer dot products, judged
// against the exact reference; larger n is the dequantize + GEMM
// fallback, judged against the better of the exact and fp16-rounded
// references (which one applies depends on the hardware and on
// GGML_CUDA_CUBLAS_COMPUTE_TYPE)
// against the exact reference. Larger n is the dequantize + GEMM
// fallback whose numerics (fp16 or TF32 compute, depending on the
// hardware and on GGML_CUDA_CUBLAS_COMPUTE_TYPE) are cuBLAS's, not
// ours: for the strict type it is gated below by bit-identity with
// the same GEMM on an F16 tensor, and only reported here.
const bool is_mmvq = n <= 8;
const bool gated = is_mmvq || !strict;
const double err_gate = is_mmvq ? err.norm_rel : (err.norm_rel < err16.norm_rel ? err.norm_rel : err16.norm_rel);
const double tol = strict ? 1e-5 : 1e-2;
const bool failed = gated && err_gate > tol;
printf("%s: %s mul_mat GPU, ncols_dst = %2d (%s): norm rel err vs exact ref = %g, vs fp16 ref = %g (max elem rel: %g)\n",
err_gate <= tol ? "OK" : "FAILED", ggml_type_name(type), n, is_mmvq ? "MMVQ" : "GEMM",
failed ? "FAILED" : gated ? "OK" : "INFO", ggml_type_name(type), n, is_mmvq ? "MMVQ" : "GEMM",
err.norm_rel, err16.norm_rel, err.max_rel);
if (err_gate > tol) {
if (failed) {
num_failed++;
}
@@ -352,7 +355,7 @@ static int test_mul_mat(ggml_backend_t backend, ggml_type type, const std::vecto
}
}
const double norm_rel = sqrt(num/den);
const double tol = strict ? 2e-3 : 1e-2;
const double tol = strict ? 5e-3 : 1e-2;
printf("%s: %s MMVQ vs GEMM path on shared columns: norm rel err = %g\n",
norm_rel <= tol ? "OK" : "FAILED", ggml_type_name(type), norm_rel);
if (norm_rel > tol) {
@@ -360,6 +363,56 @@ static int test_mul_mat(ggml_backend_t backend, ggml_type type, const std::vecto
}
}
// the GEMM fallback must be exactly "as if the weights were an F16
// tensor holding fp16(dequant(block))": running the same GEMM with an
// F16 src0 built from the fp16-rounded reference weights must give a
// bit-identical result. This isolates our (already bit-validated)
// dequantization from cuBLAS numerics.
if (strict) {
ggml_init_params params = {
/*.mem_size =*/ ggml_tensor_overhead()*8 + ggml_graph_overhead(),
/*.mem_buffer =*/ nullptr,
/*.no_alloc =*/ true,
};
ggml_context * ctx = ggml_init(params);
ggml_tensor * a16 = ggml_new_tensor_2d(ctx, GGML_TYPE_F16, NCOLS, NROWS);
ggml_tensor * b = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, NCOLS, 16);
ggml_tensor * out = ggml_mul_mat(ctx, a16, b);
ggml_backend_buffer_t buf = ggml_backend_alloc_ctx_tensors(ctx, backend);
GGML_ASSERT(buf != nullptr);
std::vector<ggml_fp16_t> w16(ref_w.size());
for (size_t i = 0; i < ref_w.size(); ++i) {
w16[i] = ggml_fp32_to_fp16(ref_w[i]);
}
ggml_backend_tensor_set(a16, w16.data(), 0, w16.size()*sizeof(ggml_fp16_t));
ggml_backend_tensor_set(b, y.data(), 0, (size_t)NCOLS*16*sizeof(float));
std::vector<float> gpu16((size_t)NROWS*16);
compute_graph(backend, ctx, out, gpu16.data());
const std::vector<float> & gemm = results[4]; // n = 16
int n_mismatch = 0;
double max_diff = 0.0;
for (size_t i = 0; i < gemm.size(); ++i) {
const double diff = fabs((double)gemm[i] - (double)gpu16[i]);
max_diff = diff > max_diff ? diff : max_diff;
if (gemm[i] != gpu16[i]) {
n_mismatch++;
}
}
printf("%s: %s GEMM path vs F16 GEMM on fp16-rounded weights: %d mismatches, max |diff| = %g\n",
n_mismatch == 0 ? "OK" : "FAILED", ggml_type_name(type), n_mismatch, max_diff);
if (n_mismatch != 0) {
num_failed++;
}
ggml_backend_buffer_free(buf);
ggml_free(ctx);
}
// manual sum over the trits stored by the test for row 0, column 0 —
// computed from the trits themselves, not from any dequantization, with
// non-trivial qh trits in every block of the row