millaguie/llama.cpp
1
0
Fork 0
Code Issues Pull Requests Actions 32 Packages Projects Releases Wiki Activity

tests : judge DT3 mul_mat on norm error, add fp16 reference and controls

Browse Source 
Elementwise max relative error explodes on cancellation whenever a true
output element is near zero, so the mul_mat checks now gate on the
relative Frobenius norm and keep the max as information. The GEMM
fallback dequantizes to fp16 on fast-fp16 hardware and DT3 weights
(d1*t1 + d2*t2) are generally not fp16-representable, so that path is
judged against a reference computed from fp16-rounded weights (taking
the better of both references so GGML_CUDA_CUBLAS_COMPUTE_TYPE=f32 also
passes). Q4_1 (same non-fp16-exact regime) and Q4_0 (fp16-exact weights,
pure GEMM error floor) run through the identical comparison as controls.
This commit is contained in:
Millaguie
2026-08-10 15:17:49 +02:00
parent ca2cb04393
commit b8ad6c3844
1 changed files with 120 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/test-dt3-gpu.cpp
+120 -38
View File
@@ -14,6 +14,20 @@
// rounding of the accumulation. One case is also checked against a
// manual sum over trits stored by the test, with non-trivial qh trits.
//
// Errors are judged on the relative Frobenius norm, ||gpu - ref|| / ||ref||;
// the elementwise maximum is reported as information only, since it explodes
// on cancellation whenever a true output value is near zero.
//
// MUL_MAT with more destination columns than the MMVQ limit falls back to
// dequantization + cuBLAS GEMM, which on fast-fp16 hardware rounds the
// dequantized weights to fp16. DT3 weights (d1*t1 + d2*t2, the sum of two
// fp16-scaled terms) are generally NOT fp16-representable, so that path is
// judged against a reference computed from fp16-rounded weights (taking the
// better of the two references, so the test also passes when
// GGML_CUDA_CUBLAS_COMPUTE_TYPE=f32 disables the rounding). Q4_1 (same
// regime: d*q + m not fp16-exact) and Q4_0 (weights fp16-exact) go through
// the identical comparison as controls, reported but gated loosely.
//
// The directed blocks exercise the three packing regions, the 79/80 and
// 119/120 region boundaries, and negative scales. The random blocks use raw
// random bytes: every byte value 0..255 must decode identically on both
@@ -95,10 +109,10 @@ static int8_t rng_trit(void) {
return (int8_t)(rng_next() % 3) - 1;
}
constexpr int NROWS = 16;
constexpr int NCOLS = 896; // 7 blocks per row; deliberately not a multiple of 256
constexpr int NBLOCKS = NROWS*NCOLS/QK_DT3;
constexpr int ROW0_NB = NCOLS/QK_DT3;
constexpr int NROWS = 16;
constexpr int NCOLS = 896; // 7 blocks per row; deliberately not a multiple of 256
constexpr int NBLOCKS = NROWS*NCOLS/QK_DT3;
constexpr int ROW0_NB = NCOLS/QK_DT3;
// trits and scales of row 0, kept for the manual MUL_MAT reference
static int8_t row0_t1[ROW0_NB][QK_DT3];
@@ -219,18 +233,38 @@ static int test_dequant(ggml_backend_t backend, const std::vector<uint8_t> & dat
return num_failed == 0 ? 0 : 1;
}
// MUL_MAT on the GPU vs a double precision reference from the CPU-dequantized
// weights. n_cols_dst <= 8 goes through MMVQ; the activations are integers
// with amax 127 in every 32-element chunk, so their q8_1 quantization is
// exact and the reference is valid to float accumulation rounding.
static int test_mul_mat(ggml_backend_t backend, const std::vector<uint8_t> & data, const std::vector<float> & ref_w) {
struct mat_err {
double norm_rel; // ||gpu - ref|| / ||ref||
double max_rel; // max elementwise |gpu - ref| / max(|ref|, 1) — information only
};
static mat_err compare_mat(const std::vector<float> & gpu, const std::vector<double> & ref) {
double num = 0.0;
double den = 0.0;
double mrel = 0.0;
for (size_t i = 0; i < gpu.size(); ++i) {
const double diff = (double)gpu[i] - ref[i];
num += diff*diff;
den += ref[i]*ref[i];
const double rel = fabs(diff) / (fabs(ref[i]) > 1.0 ? fabs(ref[i]) : 1.0);
mrel = rel > mrel ? rel : mrel;
}
return { sqrt(num/den), mrel };
}
// MUL_MAT on the GPU vs double precision references from the dequantized
// weights (exact, and rounded to fp16 as the GEMM fallback does).
// strict = tight gates (DT3); controls are gated loosely at 1e-2.
static int test_mul_mat(ggml_backend_t backend, ggml_type type, const std::vector<uint8_t> & data,
const std::vector<float> & ref_w, const std::vector<float> & y, bool strict) {
int num_failed = 0;
const int ncols_dst[] = {1, 2, 5, 8, 16};
std::vector<float> y((size_t)NCOLS*16);
for (size_t i = 0; i < y.size(); ++i) {
y[i] = i % 32 == 0 ? 127.0f : (float)((int)(rng_next() % 255) - 127);
// the same weights as the fp16 GEMM fallback sees them
std::vector<float> ref_w16(ref_w.size());
for (size_t i = 0; i < ref_w.size(); ++i) {
ref_w16[i] = ggml_fp16_to_fp32(ggml_fp32_to_fp16(ref_w[i]));
}
std::vector<std::vector<float>> results;
@@ -245,12 +279,12 @@ static int test_mul_mat(ggml_backend_t backend, const std::vector<uint8_t> & dat
};
ggml_context * ctx = ggml_init(params);
ggml_tensor * a = ggml_new_tensor_2d(ctx, GGML_TYPE_DT3, NCOLS, NROWS);
ggml_tensor * a = ggml_new_tensor_2d(ctx, type, NCOLS, NROWS);
ggml_tensor * b = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, NCOLS, n);
ggml_tensor * out = ggml_mul_mat(ctx, a, b);
if (!ggml_backend_supports_op(backend, out)) {
printf("FAILED: backend does not support MUL_MAT on DT3\n");
printf("FAILED: backend does not support MUL_MAT on %s\n", ggml_type_name(type));
ggml_free(ctx);
return 1;
}
@@ -265,24 +299,37 @@ static int test_mul_mat(ggml_backend_t backend, const std::vector<uint8_t> & dat
compute_graph(backend, ctx, out, gpu.data());
results.push_back(gpu);
// reference in double from the dequantized weights
double max_rel = 0.0;
// references in double from the exact and the fp16-rounded weights
std::vector<double> ref((size_t)NROWS*n);
std::vector<double> ref16((size_t)NROWS*n);
for (int j = 0; j < n; ++j) {
for (int r = 0; r < NROWS; ++r) {
double sum = 0.0;
double sum = 0.0;
double sum16 = 0.0;
for (int k = 0; k < NCOLS; ++k) {
sum += (double)ref_w[(size_t)r*NCOLS + k] * (double)y[(size_t)j*NCOLS + k];
sum += (double)ref_w [(size_t)r*NCOLS + k] * (double)y[(size_t)j*NCOLS + k];
sum16 += (double)ref_w16[(size_t)r*NCOLS + k] * (double)y[(size_t)j*NCOLS + k];
}
const double rel = fabs((double)gpu[(size_t)j*NROWS + r] - sum) / (fabs(sum) > 1.0 ? fabs(sum) : 1.0);
max_rel = rel > max_rel ? rel : max_rel;
ref [(size_t)j*NROWS + r] = sum;
ref16[(size_t)j*NROWS + r] = sum16;
}
}
// n <= 8 is the MMVQ path with exact integer dot products; larger n
// falls back to dequantization + GEMM, which may run in fp16
const double tol = n <= 8 ? 1e-5 : 5e-3;
printf("%s: mul_mat GPU vs reference, ncols_dst = %2d (%s): max rel err = %g\n",
max_rel <= tol ? "OK" : "FAILED", n, n <= 8 ? "MMVQ" : "GEMM", max_rel);
if (max_rel > tol) {
const mat_err err = compare_mat(gpu, ref);
const mat_err err16 = compare_mat(gpu, ref16);
// n <= 8 is the MMVQ path with exact integer dot products, judged
// against the exact reference; larger n is the dequantize + GEMM
// fallback, judged against the better of the exact and fp16-rounded
// references (which one applies depends on the hardware and on
// GGML_CUDA_CUBLAS_COMPUTE_TYPE)
const bool is_mmvq = n <= 8;
const double err_gate = is_mmvq ? err.norm_rel : (err.norm_rel < err16.norm_rel ? err.norm_rel : err16.norm_rel);
const double tol = strict ? 1e-5 : 1e-2;
printf("%s: %s mul_mat GPU, ncols_dst = %2d (%s): norm rel err vs exact ref = %g, vs fp16 ref = %g (max elem rel: %g)\n",
err_gate <= tol ? "OK" : "FAILED", ggml_type_name(type), n, is_mmvq ? "MMVQ" : "GEMM",
err.norm_rel, err16.norm_rel, err.max_rel);
if (err_gate > tol) {
num_failed++;
}
@@ -290,23 +337,25 @@ static int test_mul_mat(ggml_backend_t backend, const std::vector<uint8_t> & dat
ggml_free(ctx);
}
// MMVQ vs the dequantization-based path: first 8 columns of the GEMM run
// must match the ncols_dst = 8 MMVQ run
// MMVQ vs the dequantization-based path: the first 8 columns of the GEMM
// run must match the ncols_dst = 8 MMVQ run to fp16 weight rounding
{
const std::vector<float> & mmvq = results[3]; // n = 8
const std::vector<float> & gemm = results[4]; // n = 16
double max_rel = 0.0;
double num = 0.0;
double den = 0.0;
for (int j = 0; j < 8; ++j) {
for (int r = 0; r < NROWS; ++r) {
const double v0 = mmvq[(size_t)j*NROWS + r];
const double v1 = gemm[(size_t)j*NROWS + r];
const double rel = fabs(v0 - v1) / (fabs(v0) > 1.0 ? fabs(v0) : 1.0);
max_rel = rel > max_rel ? rel : max_rel;
const double diff = (double)mmvq[(size_t)j*NROWS + r] - (double)gemm[(size_t)j*NROWS + r];
num += diff*diff;
den += (double)mmvq[(size_t)j*NROWS + r]*(double)mmvq[(size_t)j*NROWS + r];
}
}
printf("%s: MMVQ vs GEMM path on shared columns: max rel err = %g\n",
max_rel <= 5e-3 ? "OK" : "FAILED", max_rel);
if (max_rel > 5e-3) {
const double norm_rel = sqrt(num/den);
const double tol = strict ? 2e-3 : 1e-2;
printf("%s: %s MMVQ vs GEMM path on shared columns: norm rel err = %g\n",
norm_rel <= tol ? "OK" : "FAILED", ggml_type_name(type), norm_rel);
if (norm_rel > tol) {
num_failed++;
}
}
@@ -314,7 +363,7 @@ static int test_mul_mat(ggml_backend_t backend, const std::vector<uint8_t> & dat
// manual sum over the trits stored by the test for row 0, column 0 —
// computed from the trits themselves, not from any dequantization, with
// non-trivial qh trits in every block of the row
{
if (type == GGML_TYPE_DT3) {
double sum = 0.0;
for (int j = 0; j < ROW0_NB; ++j) {
for (int i = 0; i < QK_DT3; ++i) {
@@ -334,6 +383,22 @@ static int test_mul_mat(ggml_backend_t backend, const std::vector<uint8_t> & dat
return num_failed;
}
// quantize random floats to a control type and return raw data + dequantized
// reference weights
static void build_control_data(ggml_type type, std::vector<uint8_t> & data, std::vector<float> & ref_w) {
std::vector<float> src((size_t)NROWS*NCOLS);
for (size_t i = 0; i < src.size(); ++i) {
src[i] = ((int)(rng_next() % 2001) - 1000)/1000.0f;
}
data.resize(ggml_row_size(type, NCOLS)*NROWS);
const size_t written = ggml_quantize_chunk(type, src.data(), data.data(), 0, NROWS, NCOLS, nullptr);
GGML_ASSERT(written == data.size());
ref_w.resize(src.size());
ggml_get_type_traits(type)->to_float(data.data(), ref_w.data(), (int64_t)NROWS*NCOLS);
}
int main(void) {
ggml_backend_t backend = nullptr;
for (size_t i = 0; i < ggml_backend_dev_count(); ++i) {
@@ -357,9 +422,26 @@ int main(void) {
const ggml_type_traits * qfns = ggml_get_type_traits(GGML_TYPE_DT3);
qfns->to_float(data.data(), ref.data(), (int64_t)NROWS*NCOLS);
// activations: integers with amax 127 in every 32-element chunk of every
// column, so their q8_1 quantization is exact
std::vector<float> y((size_t)NCOLS*16);
for (size_t i = 0; i < y.size(); ++i) {
y[i] = i % 32 == 0 ? 127.0f : (float)((int)(rng_next() % 255) - 127);
}
int num_failed = 0;
num_failed += test_dequant(backend, data, ref);
num_failed += test_mul_mat(backend, data, ref);
num_failed += test_mul_mat(backend, GGML_TYPE_DT3, data, ref, y, /*strict =*/ true);
// controls through the identical comparison: Q4_1 shares DT3's regime
// (dequantized weights not fp16-exact), Q4_0's weights are fp16-exact
// and show the pure GEMM error floor
for (ggml_type control : {GGML_TYPE_Q4_1, GGML_TYPE_Q4_0}) {
std::vector<uint8_t> cdata;
std::vector<float> cref;
build_control_data(control, cdata, cref);
num_failed += test_mul_mat(backend, control, cdata, cref, y, /*strict =*/ false);
}
ggml_backend_free(backend);