diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 7650c80ad..120394fbe 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -290,6 +290,7 @@ if (NOT GGML_BACKEND_DL) llama_build_and_test(test-barrier.cpp) llama_build_and_test(test-quantize-fns.cpp) llama_build_and_test(test-dt3.cpp) + llama_build_and_test(test-dt3-gpu.cpp) llama_build_and_test(test-quantize-perf.cpp) llama_build_and_test(test-rope.cpp) llama_build_and_test(test-col2im-1d.cpp) diff --git a/tests/test-dt3-gpu.cpp b/tests/test-dt3-gpu.cpp new file mode 100644 index 000000000..032d80095 --- /dev/null +++ b/tests/test-dt3-gpu.cpp @@ -0,0 +1,372 @@ +// GPU vs CPU parity tests for the DT3 dual-plane ternary format +// +// The CPU path (dequantize_row_dt3) is the validated reference. This test +// checks the GPU backend against it in two steps: +// +// 1. dequantization: GET_ROWS on the GPU must reproduce the CPU reference +// bit by bit — same fp16 scales, exact products by {-1, 0, +1}, one +// float rounding per element on both sides. +// 2. matrix multiplication: MUL_MAT with a small number of destination +// columns takes the MMVQ path (vec_dot_dt3_q8_1). The activations are +// chosen so that their q8_1 quantization is exact (integer values with +// amax 127 in every 32-element chunk), which makes a double precision +// reference computed from the dequantized weights valid to float +// rounding of the accumulation. One case is also checked against a +// manual sum over trits stored by the test, with non-trivial qh trits. +// +// The directed blocks exercise the three packing regions, the 79/80 and +// 119/120 region boundaries, and negative scales. The random blocks use raw +// random bytes: every byte value 0..255 must decode identically on both +// sides, including values >= 243 that never come out of the packer. +// +// Without a GPU backend the test is skipped and succeeds. + +#include "ggml.h" +#include "ggml-alloc.h" +#include "ggml-backend.h" +#include "ggml-cpu.h" + +#undef NDEBUG +#include +#include +#include +#include +#include +#include + +constexpr int QK_DT3 = 128; +constexpr size_t DT3_QS_BYTES = 24; // per plane +constexpr size_t DT3_QH_BYTES = 2; // per plane +constexpr size_t DT3_BLOCK_SIZE = 2*DT3_QS_BYTES + 2*DT3_QH_BYTES + 2*sizeof(uint16_t); + +// byte offsets inside a block (spec: qs[2][24] | qh[2][2] | d[2]) +constexpr size_t OFF_QS = 0; +constexpr size_t OFF_QH = 2*DT3_QS_BYTES; +constexpr size_t OFF_D = 2*DT3_QS_BYTES + 2*DT3_QH_BYTES; + +// independent packer, written from the format specification (same as in +// test-dt3.cpp): element i of a plane goes to +// region A: qs[m], m in [0,16), digit n: elements m + n*16 (0..79) +// region B: qs[16+m], m in [0,8), digit n: elements 80 + m + n*8 (80..119) +// region C: qh[j], j in [0,2), digit n: elements 120 + j + n*2 (120..127) +static void ref_pack_plane(const int8_t * t, uint8_t * qs, uint8_t * qh) { + for (int m = 0; m < 16; ++m) { + uint32_t q = 0; + for (int n = 0; n < 5; ++n) { + q = q*3 + (uint32_t)(t[m + n*16] + 1); + } + qs[m] = (uint8_t)((q*256 + 242)/243); + } + for (int m = 0; m < 8; ++m) { + uint32_t q = 0; + for (int n = 0; n < 5; ++n) { + q = q*3 + (uint32_t)(t[80 + m + n*8] + 1); + } + qs[16 + m] = (uint8_t)((q*256 + 242)/243); + } + for (int j = 0; j < 2; ++j) { + uint32_t q = 0; + for (int n = 0; n < 4; ++n) { + q = q*3 + (uint32_t)(t[120 + j + n*2] + 1); + } + q *= 3; // shift the first value to the most significant trit + qh[j] = (uint8_t)((q*256 + 242)/243); + } +} + +static void ref_pack_block(const int8_t * t1, float d1, const int8_t * t2, float d2, uint8_t * block) { + ref_pack_plane(t1, block + OFF_QS, block + OFF_QH); + ref_pack_plane(t2, block + OFF_QS + DT3_QS_BYTES, block + OFF_QH + DT3_QH_BYTES); + const uint16_t h1 = ggml_fp32_to_fp16(d1); + const uint16_t h2 = ggml_fp32_to_fp16(d2); + memcpy(block + OFF_D, &h1, sizeof(h1)); + memcpy(block + OFF_D + 2, &h2, sizeof(h2)); +} + +// deterministic PRNG so failures are reproducible +static uint32_t rng_state = 0x2b992ddf; +static uint32_t rng_next(void) { + rng_state ^= rng_state << 13; + rng_state ^= rng_state >> 17; + rng_state ^= rng_state << 5; + return rng_state; +} +static int8_t rng_trit(void) { + return (int8_t)(rng_next() % 3) - 1; +} + +constexpr int NROWS = 16; +constexpr int NCOLS = 896; // 7 blocks per row; deliberately not a multiple of 256 +constexpr int NBLOCKS = NROWS*NCOLS/QK_DT3; +constexpr int ROW0_NB = NCOLS/QK_DT3; + +// trits and scales of row 0, kept for the manual MUL_MAT reference +static int8_t row0_t1[ROW0_NB][QK_DT3]; +static int8_t row0_t2[ROW0_NB][QK_DT3]; +static float row0_d1[ROW0_NB]; +static float row0_d2[ROW0_NB]; + +static void build_dt3_data(std::vector & data) { + data.resize((size_t)NBLOCKS*DT3_BLOCK_SIZE); + + // row 0: known trits with non-trivial qh region and mixed-sign scales + for (int j = 0; j < ROW0_NB; ++j) { + for (int i = 0; i < QK_DT3; ++i) { + row0_t1[j][i] = rng_trit(); + row0_t2[j][i] = rng_trit(); + } + // make sure the qh-packed elements are not all zero + row0_t1[j][127] = -1; + row0_t2[j][120] = +1; + + row0_d1[j] = j % 2 == 0 ? 1.5f : -0.75f; // exact in fp16 + row0_d2[j] = j % 2 == 0 ? -0.625f: 0.375f; // exact in fp16 + + ref_pack_block(row0_t1[j], row0_d1[j], row0_t2[j], row0_d2[j], data.data() + (size_t)j*DT3_BLOCK_SIZE); + } + + // directed single-trit blocks at the region boundaries, negative d2 + const int special_pos[] = {0, 15, 16, 79, 80, 87, 88, 119, 120, 121, 126, 127}; + const int n_special = (int)(sizeof(special_pos)/sizeof(special_pos[0])); + for (int c = 0; c < n_special; ++c) { + int8_t t1[QK_DT3] = {0}; + int8_t t2[QK_DT3] = {0}; + t1[special_pos[c]] = +1; + t2[special_pos[c]] = -1; + ref_pack_block(t1, 1.0f, t2, -0.25f, data.data() + (size_t)(ROW0_NB + c)*DT3_BLOCK_SIZE); + } + + // the rest: raw random bytes (any byte value is decodable) and random + // small scales, some negative + for (int b = ROW0_NB + n_special; b < NBLOCKS; ++b) { + uint8_t * block = data.data() + (size_t)b*DT3_BLOCK_SIZE; + for (size_t k = 0; k < OFF_D; ++k) { + block[k] = (uint8_t)(rng_next() & 0xFF); + } + const uint16_t h1 = ggml_fp32_to_fp16(((int)(rng_next() % 2001) - 1000)/500.0f); + const uint16_t h2 = ggml_fp32_to_fp16(((int)(rng_next() % 2001) - 1000)/500.0f); + memcpy(block + OFF_D, &h1, sizeof(h1)); + memcpy(block + OFF_D + 2, &h2, sizeof(h2)); + } +} + +// run a single-output graph on the backend and read the result back +static void compute_graph(ggml_backend_t backend, ggml_context * ctx, ggml_tensor * out, float * result) { + ggml_cgraph * gf = ggml_new_graph(ctx); + ggml_build_forward_expand(gf, out); + + ggml_gallocr_t galloc = ggml_gallocr_new(ggml_backend_get_default_buffer_type(backend)); + const bool ok = ggml_gallocr_alloc_graph(galloc, gf); + GGML_ASSERT(ok); + + const ggml_status status = ggml_backend_graph_compute(backend, gf); + GGML_ASSERT(status == GGML_STATUS_SUCCESS); + + ggml_backend_tensor_get(out, result, 0, ggml_nbytes(out)); + ggml_gallocr_free(galloc); +} + +// GET_ROWS over all rows on the GPU vs the CPU reference dequantization +static int test_dequant(ggml_backend_t backend, const std::vector & data, const std::vector & ref) { + ggml_init_params params = { + /*.mem_size =*/ ggml_tensor_overhead()*8 + ggml_graph_overhead(), + /*.mem_buffer =*/ nullptr, + /*.no_alloc =*/ true, + }; + ggml_context * ctx = ggml_init(params); + + ggml_tensor * a = ggml_new_tensor_2d(ctx, GGML_TYPE_DT3, NCOLS, NROWS); + ggml_tensor * rows = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, NROWS); + ggml_tensor * out = ggml_get_rows(ctx, a, rows); + + if (!ggml_backend_supports_op(backend, out)) { + printf("FAILED: backend does not support GET_ROWS on DT3\n"); + ggml_free(ctx); + return 1; + } + + ggml_backend_buffer_t buf = ggml_backend_alloc_ctx_tensors(ctx, backend); + GGML_ASSERT(buf != nullptr); + + std::vector row_idx(NROWS); + for (int r = 0; r < NROWS; ++r) { + row_idx[r] = r; + } + ggml_backend_tensor_set(a, data.data(), 0, data.size()); + ggml_backend_tensor_set(rows, row_idx.data(), 0, NROWS*sizeof(int32_t)); + + std::vector gpu((size_t)NROWS*NCOLS); + compute_graph(backend, ctx, out, gpu.data()); + + int num_failed = 0; + double max_diff = 0.0; + for (size_t i = 0; i < gpu.size(); ++i) { + const double diff = fabs((double)gpu[i] - (double)ref[i]); + max_diff = diff > max_diff ? diff : max_diff; + if (gpu[i] != ref[i]) { + if (num_failed < 8) { + printf("FAILED: dequant mismatch at block %zu elem %zu: gpu %.9g, cpu %.9g\n", + i/QK_DT3, i%QK_DT3, gpu[i], ref[i]); + } + num_failed++; + } + } + printf("%s: dequant GPU vs CPU on %d blocks: %d mismatches, max |diff| = %g\n", + num_failed == 0 ? "OK" : "FAILED", NBLOCKS, num_failed, max_diff); + + ggml_backend_buffer_free(buf); + ggml_free(ctx); + return num_failed == 0 ? 0 : 1; +} + +// MUL_MAT on the GPU vs a double precision reference from the CPU-dequantized +// weights. n_cols_dst <= 8 goes through MMVQ; the activations are integers +// with amax 127 in every 32-element chunk, so their q8_1 quantization is +// exact and the reference is valid to float accumulation rounding. +static int test_mul_mat(ggml_backend_t backend, const std::vector & data, const std::vector & ref_w) { + int num_failed = 0; + + const int ncols_dst[] = {1, 2, 5, 8, 16}; + + std::vector y((size_t)NCOLS*16); + for (size_t i = 0; i < y.size(); ++i) { + y[i] = i % 32 == 0 ? 127.0f : (float)((int)(rng_next() % 255) - 127); + } + + std::vector> results; + + for (int c = 0; c < (int)(sizeof(ncols_dst)/sizeof(ncols_dst[0])); ++c) { + const int n = ncols_dst[c]; + + ggml_init_params params = { + /*.mem_size =*/ ggml_tensor_overhead()*8 + ggml_graph_overhead(), + /*.mem_buffer =*/ nullptr, + /*.no_alloc =*/ true, + }; + ggml_context * ctx = ggml_init(params); + + ggml_tensor * a = ggml_new_tensor_2d(ctx, GGML_TYPE_DT3, NCOLS, NROWS); + ggml_tensor * b = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, NCOLS, n); + ggml_tensor * out = ggml_mul_mat(ctx, a, b); + + if (!ggml_backend_supports_op(backend, out)) { + printf("FAILED: backend does not support MUL_MAT on DT3\n"); + ggml_free(ctx); + return 1; + } + + ggml_backend_buffer_t buf = ggml_backend_alloc_ctx_tensors(ctx, backend); + GGML_ASSERT(buf != nullptr); + + ggml_backend_tensor_set(a, data.data(), 0, data.size()); + ggml_backend_tensor_set(b, y.data(), 0, (size_t)NCOLS*n*sizeof(float)); + + std::vector gpu((size_t)NROWS*n); + compute_graph(backend, ctx, out, gpu.data()); + results.push_back(gpu); + + // reference in double from the dequantized weights + double max_rel = 0.0; + for (int j = 0; j < n; ++j) { + for (int r = 0; r < NROWS; ++r) { + double sum = 0.0; + for (int k = 0; k < NCOLS; ++k) { + sum += (double)ref_w[(size_t)r*NCOLS + k] * (double)y[(size_t)j*NCOLS + k]; + } + const double rel = fabs((double)gpu[(size_t)j*NROWS + r] - sum) / (fabs(sum) > 1.0 ? fabs(sum) : 1.0); + max_rel = rel > max_rel ? rel : max_rel; + } + } + // n <= 8 is the MMVQ path with exact integer dot products; larger n + // falls back to dequantization + GEMM, which may run in fp16 + const double tol = n <= 8 ? 1e-5 : 5e-3; + printf("%s: mul_mat GPU vs reference, ncols_dst = %2d (%s): max rel err = %g\n", + max_rel <= tol ? "OK" : "FAILED", n, n <= 8 ? "MMVQ" : "GEMM", max_rel); + if (max_rel > tol) { + num_failed++; + } + + ggml_backend_buffer_free(buf); + ggml_free(ctx); + } + + // MMVQ vs the dequantization-based path: first 8 columns of the GEMM run + // must match the ncols_dst = 8 MMVQ run + { + const std::vector & mmvq = results[3]; // n = 8 + const std::vector & gemm = results[4]; // n = 16 + double max_rel = 0.0; + for (int j = 0; j < 8; ++j) { + for (int r = 0; r < NROWS; ++r) { + const double v0 = mmvq[(size_t)j*NROWS + r]; + const double v1 = gemm[(size_t)j*NROWS + r]; + const double rel = fabs(v0 - v1) / (fabs(v0) > 1.0 ? fabs(v0) : 1.0); + max_rel = rel > max_rel ? rel : max_rel; + } + } + printf("%s: MMVQ vs GEMM path on shared columns: max rel err = %g\n", + max_rel <= 5e-3 ? "OK" : "FAILED", max_rel); + if (max_rel > 5e-3) { + num_failed++; + } + } + + // manual sum over the trits stored by the test for row 0, column 0 — + // computed from the trits themselves, not from any dequantization, with + // non-trivial qh trits in every block of the row + { + double sum = 0.0; + for (int j = 0; j < ROW0_NB; ++j) { + for (int i = 0; i < QK_DT3; ++i) { + sum += (double)y[(size_t)j*QK_DT3 + i] * + ((double)row0_d1[j]*row0_t1[j][i] + (double)row0_d2[j]*row0_t2[j][i]); + } + } + const double got = results[0][0]; // ncols_dst = 1, row 0 + const double rel = fabs(got - sum) / (fabs(sum) > 1.0 ? fabs(sum) : 1.0); + printf("%s: MMVQ vs manual trit sum (row 0, col 0): gpu %.9g, manual %.9g, rel err = %g\n", + rel <= 1e-5 ? "OK" : "FAILED", got, sum, rel); + if (rel > 1e-5) { + num_failed++; + } + } + + return num_failed; +} + +int main(void) { + ggml_backend_t backend = nullptr; + for (size_t i = 0; i < ggml_backend_dev_count(); ++i) { + ggml_backend_dev_t dev = ggml_backend_dev_get(i); + if (ggml_backend_dev_type(dev) == GGML_BACKEND_DEVICE_TYPE_GPU) { + backend = ggml_backend_dev_init(dev, nullptr); + printf("using GPU backend: %s\n", ggml_backend_dev_name(dev)); + break; + } + } + if (backend == nullptr) { + printf("no GPU backend available, skipping\n"); + return 0; + } + + std::vector data; + build_dt3_data(data); + + // CPU reference dequantization — the validated path + std::vector ref((size_t)NROWS*NCOLS); + const ggml_type_traits * qfns = ggml_get_type_traits(GGML_TYPE_DT3); + qfns->to_float(data.data(), ref.data(), (int64_t)NROWS*NCOLS); + + int num_failed = 0; + num_failed += test_dequant(backend, data, ref); + num_failed += test_mul_mat(backend, data, ref); + + ggml_backend_free(backend); + + if (num_failed > 0) { + printf("%d tests FAILED\n", num_failed); + return 1; + } + printf("all tests OK\n"); + return 0; +}