diff --git a/tests/test-dt3-gpu.cpp b/tests/test-dt3-gpu.cpp index 032d80095..239c903ce 100644 --- a/tests/test-dt3-gpu.cpp +++ b/tests/test-dt3-gpu.cpp @@ -14,6 +14,20 @@ // rounding of the accumulation. One case is also checked against a // manual sum over trits stored by the test, with non-trivial qh trits. // +// Errors are judged on the relative Frobenius norm, ||gpu - ref|| / ||ref||; +// the elementwise maximum is reported as information only, since it explodes +// on cancellation whenever a true output value is near zero. +// +// MUL_MAT with more destination columns than the MMVQ limit falls back to +// dequantization + cuBLAS GEMM, which on fast-fp16 hardware rounds the +// dequantized weights to fp16. DT3 weights (d1*t1 + d2*t2, the sum of two +// fp16-scaled terms) are generally NOT fp16-representable, so that path is +// judged against a reference computed from fp16-rounded weights (taking the +// better of the two references, so the test also passes when +// GGML_CUDA_CUBLAS_COMPUTE_TYPE=f32 disables the rounding). Q4_1 (same +// regime: d*q + m not fp16-exact) and Q4_0 (weights fp16-exact) go through +// the identical comparison as controls, reported but gated loosely. +// // The directed blocks exercise the three packing regions, the 79/80 and // 119/120 region boundaries, and negative scales. The random blocks use raw // random bytes: every byte value 0..255 must decode identically on both @@ -95,10 +109,10 @@ static int8_t rng_trit(void) { return (int8_t)(rng_next() % 3) - 1; } -constexpr int NROWS = 16; -constexpr int NCOLS = 896; // 7 blocks per row; deliberately not a multiple of 256 -constexpr int NBLOCKS = NROWS*NCOLS/QK_DT3; -constexpr int ROW0_NB = NCOLS/QK_DT3; +constexpr int NROWS = 16; +constexpr int NCOLS = 896; // 7 blocks per row; deliberately not a multiple of 256 +constexpr int NBLOCKS = NROWS*NCOLS/QK_DT3; +constexpr int ROW0_NB = NCOLS/QK_DT3; // trits and scales of row 0, kept for the manual MUL_MAT reference static int8_t row0_t1[ROW0_NB][QK_DT3]; @@ -219,18 +233,38 @@ static int test_dequant(ggml_backend_t backend, const std::vector & dat return num_failed == 0 ? 0 : 1; } -// MUL_MAT on the GPU vs a double precision reference from the CPU-dequantized -// weights. n_cols_dst <= 8 goes through MMVQ; the activations are integers -// with amax 127 in every 32-element chunk, so their q8_1 quantization is -// exact and the reference is valid to float accumulation rounding. -static int test_mul_mat(ggml_backend_t backend, const std::vector & data, const std::vector & ref_w) { +struct mat_err { + double norm_rel; // ||gpu - ref|| / ||ref|| + double max_rel; // max elementwise |gpu - ref| / max(|ref|, 1) — information only +}; + +static mat_err compare_mat(const std::vector & gpu, const std::vector & ref) { + double num = 0.0; + double den = 0.0; + double mrel = 0.0; + for (size_t i = 0; i < gpu.size(); ++i) { + const double diff = (double)gpu[i] - ref[i]; + num += diff*diff; + den += ref[i]*ref[i]; + const double rel = fabs(diff) / (fabs(ref[i]) > 1.0 ? fabs(ref[i]) : 1.0); + mrel = rel > mrel ? rel : mrel; + } + return { sqrt(num/den), mrel }; +} + +// MUL_MAT on the GPU vs double precision references from the dequantized +// weights (exact, and rounded to fp16 as the GEMM fallback does). +// strict = tight gates (DT3); controls are gated loosely at 1e-2. +static int test_mul_mat(ggml_backend_t backend, ggml_type type, const std::vector & data, + const std::vector & ref_w, const std::vector & y, bool strict) { int num_failed = 0; const int ncols_dst[] = {1, 2, 5, 8, 16}; - std::vector y((size_t)NCOLS*16); - for (size_t i = 0; i < y.size(); ++i) { - y[i] = i % 32 == 0 ? 127.0f : (float)((int)(rng_next() % 255) - 127); + // the same weights as the fp16 GEMM fallback sees them + std::vector ref_w16(ref_w.size()); + for (size_t i = 0; i < ref_w.size(); ++i) { + ref_w16[i] = ggml_fp16_to_fp32(ggml_fp32_to_fp16(ref_w[i])); } std::vector> results; @@ -245,12 +279,12 @@ static int test_mul_mat(ggml_backend_t backend, const std::vector & dat }; ggml_context * ctx = ggml_init(params); - ggml_tensor * a = ggml_new_tensor_2d(ctx, GGML_TYPE_DT3, NCOLS, NROWS); + ggml_tensor * a = ggml_new_tensor_2d(ctx, type, NCOLS, NROWS); ggml_tensor * b = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, NCOLS, n); ggml_tensor * out = ggml_mul_mat(ctx, a, b); if (!ggml_backend_supports_op(backend, out)) { - printf("FAILED: backend does not support MUL_MAT on DT3\n"); + printf("FAILED: backend does not support MUL_MAT on %s\n", ggml_type_name(type)); ggml_free(ctx); return 1; } @@ -265,24 +299,37 @@ static int test_mul_mat(ggml_backend_t backend, const std::vector & dat compute_graph(backend, ctx, out, gpu.data()); results.push_back(gpu); - // reference in double from the dequantized weights - double max_rel = 0.0; + // references in double from the exact and the fp16-rounded weights + std::vector ref((size_t)NROWS*n); + std::vector ref16((size_t)NROWS*n); for (int j = 0; j < n; ++j) { for (int r = 0; r < NROWS; ++r) { - double sum = 0.0; + double sum = 0.0; + double sum16 = 0.0; for (int k = 0; k < NCOLS; ++k) { - sum += (double)ref_w[(size_t)r*NCOLS + k] * (double)y[(size_t)j*NCOLS + k]; + sum += (double)ref_w [(size_t)r*NCOLS + k] * (double)y[(size_t)j*NCOLS + k]; + sum16 += (double)ref_w16[(size_t)r*NCOLS + k] * (double)y[(size_t)j*NCOLS + k]; } - const double rel = fabs((double)gpu[(size_t)j*NROWS + r] - sum) / (fabs(sum) > 1.0 ? fabs(sum) : 1.0); - max_rel = rel > max_rel ? rel : max_rel; + ref [(size_t)j*NROWS + r] = sum; + ref16[(size_t)j*NROWS + r] = sum16; } } - // n <= 8 is the MMVQ path with exact integer dot products; larger n - // falls back to dequantization + GEMM, which may run in fp16 - const double tol = n <= 8 ? 1e-5 : 5e-3; - printf("%s: mul_mat GPU vs reference, ncols_dst = %2d (%s): max rel err = %g\n", - max_rel <= tol ? "OK" : "FAILED", n, n <= 8 ? "MMVQ" : "GEMM", max_rel); - if (max_rel > tol) { + + const mat_err err = compare_mat(gpu, ref); + const mat_err err16 = compare_mat(gpu, ref16); + + // n <= 8 is the MMVQ path with exact integer dot products, judged + // against the exact reference; larger n is the dequantize + GEMM + // fallback, judged against the better of the exact and fp16-rounded + // references (which one applies depends on the hardware and on + // GGML_CUDA_CUBLAS_COMPUTE_TYPE) + const bool is_mmvq = n <= 8; + const double err_gate = is_mmvq ? err.norm_rel : (err.norm_rel < err16.norm_rel ? err.norm_rel : err16.norm_rel); + const double tol = strict ? 1e-5 : 1e-2; + printf("%s: %s mul_mat GPU, ncols_dst = %2d (%s): norm rel err vs exact ref = %g, vs fp16 ref = %g (max elem rel: %g)\n", + err_gate <= tol ? "OK" : "FAILED", ggml_type_name(type), n, is_mmvq ? "MMVQ" : "GEMM", + err.norm_rel, err16.norm_rel, err.max_rel); + if (err_gate > tol) { num_failed++; } @@ -290,23 +337,25 @@ static int test_mul_mat(ggml_backend_t backend, const std::vector & dat ggml_free(ctx); } - // MMVQ vs the dequantization-based path: first 8 columns of the GEMM run - // must match the ncols_dst = 8 MMVQ run + // MMVQ vs the dequantization-based path: the first 8 columns of the GEMM + // run must match the ncols_dst = 8 MMVQ run to fp16 weight rounding { const std::vector & mmvq = results[3]; // n = 8 const std::vector & gemm = results[4]; // n = 16 - double max_rel = 0.0; + double num = 0.0; + double den = 0.0; for (int j = 0; j < 8; ++j) { for (int r = 0; r < NROWS; ++r) { - const double v0 = mmvq[(size_t)j*NROWS + r]; - const double v1 = gemm[(size_t)j*NROWS + r]; - const double rel = fabs(v0 - v1) / (fabs(v0) > 1.0 ? fabs(v0) : 1.0); - max_rel = rel > max_rel ? rel : max_rel; + const double diff = (double)mmvq[(size_t)j*NROWS + r] - (double)gemm[(size_t)j*NROWS + r]; + num += diff*diff; + den += (double)mmvq[(size_t)j*NROWS + r]*(double)mmvq[(size_t)j*NROWS + r]; } } - printf("%s: MMVQ vs GEMM path on shared columns: max rel err = %g\n", - max_rel <= 5e-3 ? "OK" : "FAILED", max_rel); - if (max_rel > 5e-3) { + const double norm_rel = sqrt(num/den); + const double tol = strict ? 2e-3 : 1e-2; + printf("%s: %s MMVQ vs GEMM path on shared columns: norm rel err = %g\n", + norm_rel <= tol ? "OK" : "FAILED", ggml_type_name(type), norm_rel); + if (norm_rel > tol) { num_failed++; } } @@ -314,7 +363,7 @@ static int test_mul_mat(ggml_backend_t backend, const std::vector & dat // manual sum over the trits stored by the test for row 0, column 0 — // computed from the trits themselves, not from any dequantization, with // non-trivial qh trits in every block of the row - { + if (type == GGML_TYPE_DT3) { double sum = 0.0; for (int j = 0; j < ROW0_NB; ++j) { for (int i = 0; i < QK_DT3; ++i) { @@ -334,6 +383,22 @@ static int test_mul_mat(ggml_backend_t backend, const std::vector & dat return num_failed; } +// quantize random floats to a control type and return raw data + dequantized +// reference weights +static void build_control_data(ggml_type type, std::vector & data, std::vector & ref_w) { + std::vector src((size_t)NROWS*NCOLS); + for (size_t i = 0; i < src.size(); ++i) { + src[i] = ((int)(rng_next() % 2001) - 1000)/1000.0f; + } + + data.resize(ggml_row_size(type, NCOLS)*NROWS); + const size_t written = ggml_quantize_chunk(type, src.data(), data.data(), 0, NROWS, NCOLS, nullptr); + GGML_ASSERT(written == data.size()); + + ref_w.resize(src.size()); + ggml_get_type_traits(type)->to_float(data.data(), ref_w.data(), (int64_t)NROWS*NCOLS); +} + int main(void) { ggml_backend_t backend = nullptr; for (size_t i = 0; i < ggml_backend_dev_count(); ++i) { @@ -357,9 +422,26 @@ int main(void) { const ggml_type_traits * qfns = ggml_get_type_traits(GGML_TYPE_DT3); qfns->to_float(data.data(), ref.data(), (int64_t)NROWS*NCOLS); + // activations: integers with amax 127 in every 32-element chunk of every + // column, so their q8_1 quantization is exact + std::vector y((size_t)NCOLS*16); + for (size_t i = 0; i < y.size(); ++i) { + y[i] = i % 32 == 0 ? 127.0f : (float)((int)(rng_next() % 255) - 127); + } + int num_failed = 0; num_failed += test_dequant(backend, data, ref); - num_failed += test_mul_mat(backend, data, ref); + num_failed += test_mul_mat(backend, GGML_TYPE_DT3, data, ref, y, /*strict =*/ true); + + // controls through the identical comparison: Q4_1 shares DT3's regime + // (dequantized weights not fp16-exact), Q4_0's weights are fp16-exact + // and show the pure GEMM error floor + for (ggml_type control : {GGML_TYPE_Q4_1, GGML_TYPE_Q4_0}) { + std::vector cdata; + std::vector cref; + build_control_data(control, cdata, cref); + num_failed += test_mul_mat(backend, control, cdata, cref, y, /*strict =*/ false); + } ggml_backend_free(backend);