millaguie/llama.cpp
1
0
Fork 0
Code Issues Pull Requests Actions 31 Packages Projects Releases Wiki Activity

tests : add DT3 GPU vs CPU parity test

Browse Source 
Checks the GPU backend against the validated CPU path: GET_ROWS
dequantization must match dequantize_row_dt3 bit by bit on directed
blocks (region boundaries 79/80 and 119/120, qh elements, negative
scales) and on raw random bytes; MUL_MAT must match a double precision
reference from the dequantized weights, with activations whose q8_1
quantization is exact, plus a manual trit-sum check with non-trivial qh.
Skips cleanly when no GPU backend is available.
This commit is contained in:
Millaguie
2026-08-10 15:01:38 +02:00
parent bc3e949e71
commit ca2cb04393
2 changed files with 373 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/CMakeLists.txt
+1
View File
@@ -290,6 +290,7 @@ if (NOT GGML_BACKEND_DL)
llama_build_and_test(test-barrier.cpp)
llama_build_and_test(test-quantize-fns.cpp)
llama_build_and_test(test-dt3.cpp)
llama_build_and_test(test-dt3-gpu.cpp)
llama_build_and_test(test-quantize-perf.cpp)
llama_build_and_test(test-rope.cpp)
llama_build_and_test(test-col2im-1d.cpp)
tests/test-dt3-gpu.cpp
+372
View File
@@ -0,0 +1,372 @@
// GPU vs CPU parity tests for the DT3 dual-plane ternary format
//
// The CPU path (dequantize_row_dt3) is the validated reference. This test
// checks the GPU backend against it in two steps:
//
// 1. dequantization: GET_ROWS on the GPU must reproduce the CPU reference
// bit by bit — same fp16 scales, exact products by {-1, 0, +1}, one
// float rounding per element on both sides.
// 2. matrix multiplication: MUL_MAT with a small number of destination
// columns takes the MMVQ path (vec_dot_dt3_q8_1). The activations are
// chosen so that their q8_1 quantization is exact (integer values with
// amax 127 in every 32-element chunk), which makes a double precision
// reference computed from the dequantized weights valid to float
// rounding of the accumulation. One case is also checked against a
// manual sum over trits stored by the test, with non-trivial qh trits.
//
// The directed blocks exercise the three packing regions, the 79/80 and
// 119/120 region boundaries, and negative scales. The random blocks use raw
// random bytes: every byte value 0..255 must decode identically on both
// sides, including values >= 243 that never come out of the packer.
//
// Without a GPU backend the test is skipped and succeeds.
#include "ggml.h"
#include "ggml-alloc.h"
#include "ggml-backend.h"
#include "ggml-cpu.h"
#undef NDEBUG
#include <assert.h>
#include <math.h>
#include <stdint.h>
#include <stdio.h>
#include <string.h>
#include <vector>
constexpr int QK_DT3 = 128;
constexpr size_t DT3_QS_BYTES = 24; // per plane
constexpr size_t DT3_QH_BYTES = 2; // per plane
constexpr size_t DT3_BLOCK_SIZE = 2*DT3_QS_BYTES + 2*DT3_QH_BYTES + 2*sizeof(uint16_t);
// byte offsets inside a block (spec: qs[2][24] | qh[2][2] | d[2])
constexpr size_t OFF_QS = 0;
constexpr size_t OFF_QH = 2*DT3_QS_BYTES;
constexpr size_t OFF_D = 2*DT3_QS_BYTES + 2*DT3_QH_BYTES;
// independent packer, written from the format specification (same as in
// test-dt3.cpp): element i of a plane goes to
// region A: qs[m], m in [0,16), digit n: elements m + n*16 (0..79)
// region B: qs[16+m], m in [0,8), digit n: elements 80 + m + n*8 (80..119)
// region C: qh[j], j in [0,2), digit n: elements 120 + j + n*2 (120..127)
static void ref_pack_plane(const int8_t * t, uint8_t * qs, uint8_t * qh) {
for (int m = 0; m < 16; ++m) {
uint32_t q = 0;
for (int n = 0; n < 5; ++n) {
q = q*3 + (uint32_t)(t[m + n*16] + 1);
}
qs[m] = (uint8_t)((q*256 + 242)/243);
}
for (int m = 0; m < 8; ++m) {
uint32_t q = 0;
for (int n = 0; n < 5; ++n) {
q = q*3 + (uint32_t)(t[80 + m + n*8] + 1);
}
qs[16 + m] = (uint8_t)((q*256 + 242)/243);
}
for (int j = 0; j < 2; ++j) {
uint32_t q = 0;
for (int n = 0; n < 4; ++n) {
q = q*3 + (uint32_t)(t[120 + j + n*2] + 1);
}
q *= 3; // shift the first value to the most significant trit
qh[j] = (uint8_t)((q*256 + 242)/243);
}
}
static void ref_pack_block(const int8_t * t1, float d1, const int8_t * t2, float d2, uint8_t * block) {
ref_pack_plane(t1, block + OFF_QS, block + OFF_QH);
ref_pack_plane(t2, block + OFF_QS + DT3_QS_BYTES, block + OFF_QH + DT3_QH_BYTES);
const uint16_t h1 = ggml_fp32_to_fp16(d1);
const uint16_t h2 = ggml_fp32_to_fp16(d2);
memcpy(block + OFF_D, &h1, sizeof(h1));
memcpy(block + OFF_D + 2, &h2, sizeof(h2));
}
// deterministic PRNG so failures are reproducible
static uint32_t rng_state = 0x2b992ddf;
static uint32_t rng_next(void) {
rng_state ^= rng_state << 13;
rng_state ^= rng_state >> 17;
rng_state ^= rng_state << 5;
return rng_state;
}
static int8_t rng_trit(void) {
return (int8_t)(rng_next() % 3) - 1;
}
constexpr int NROWS = 16;
constexpr int NCOLS = 896; // 7 blocks per row; deliberately not a multiple of 256
constexpr int NBLOCKS = NROWS*NCOLS/QK_DT3;
constexpr int ROW0_NB = NCOLS/QK_DT3;
// trits and scales of row 0, kept for the manual MUL_MAT reference
static int8_t row0_t1[ROW0_NB][QK_DT3];
static int8_t row0_t2[ROW0_NB][QK_DT3];
static float row0_d1[ROW0_NB];
static float row0_d2[ROW0_NB];
static void build_dt3_data(std::vector<uint8_t> & data) {
data.resize((size_t)NBLOCKS*DT3_BLOCK_SIZE);
// row 0: known trits with non-trivial qh region and mixed-sign scales
for (int j = 0; j < ROW0_NB; ++j) {
for (int i = 0; i < QK_DT3; ++i) {
row0_t1[j][i] = rng_trit();
row0_t2[j][i] = rng_trit();
}
// make sure the qh-packed elements are not all zero
row0_t1[j][127] = -1;
row0_t2[j][120] = +1;
row0_d1[j] = j % 2 == 0 ? 1.5f : -0.75f; // exact in fp16
row0_d2[j] = j % 2 == 0 ? -0.625f: 0.375f; // exact in fp16
ref_pack_block(row0_t1[j], row0_d1[j], row0_t2[j], row0_d2[j], data.data() + (size_t)j*DT3_BLOCK_SIZE);
}
// directed single-trit blocks at the region boundaries, negative d2
const int special_pos[] = {0, 15, 16, 79, 80, 87, 88, 119, 120, 121, 126, 127};
const int n_special = (int)(sizeof(special_pos)/sizeof(special_pos[0]));
for (int c = 0; c < n_special; ++c) {
int8_t t1[QK_DT3] = {0};
int8_t t2[QK_DT3] = {0};
t1[special_pos[c]] = +1;
t2[special_pos[c]] = -1;
ref_pack_block(t1, 1.0f, t2, -0.25f, data.data() + (size_t)(ROW0_NB + c)*DT3_BLOCK_SIZE);
}
// the rest: raw random bytes (any byte value is decodable) and random
// small scales, some negative
for (int b = ROW0_NB + n_special; b < NBLOCKS; ++b) {
uint8_t * block = data.data() + (size_t)b*DT3_BLOCK_SIZE;
for (size_t k = 0; k < OFF_D; ++k) {
block[k] = (uint8_t)(rng_next() & 0xFF);
}
const uint16_t h1 = ggml_fp32_to_fp16(((int)(rng_next() % 2001) - 1000)/500.0f);
const uint16_t h2 = ggml_fp32_to_fp16(((int)(rng_next() % 2001) - 1000)/500.0f);
memcpy(block + OFF_D, &h1, sizeof(h1));
memcpy(block + OFF_D + 2, &h2, sizeof(h2));
}
}
// run a single-output graph on the backend and read the result back
static void compute_graph(ggml_backend_t backend, ggml_context * ctx, ggml_tensor * out, float * result) {
ggml_cgraph * gf = ggml_new_graph(ctx);
ggml_build_forward_expand(gf, out);
ggml_gallocr_t galloc = ggml_gallocr_new(ggml_backend_get_default_buffer_type(backend));
const bool ok = ggml_gallocr_alloc_graph(galloc, gf);
GGML_ASSERT(ok);
const ggml_status status = ggml_backend_graph_compute(backend, gf);
GGML_ASSERT(status == GGML_STATUS_SUCCESS);
ggml_backend_tensor_get(out, result, 0, ggml_nbytes(out));
ggml_gallocr_free(galloc);
}
// GET_ROWS over all rows on the GPU vs the CPU reference dequantization
static int test_dequant(ggml_backend_t backend, const std::vector<uint8_t> & data, const std::vector<float> & ref) {
ggml_init_params params = {
/*.mem_size =*/ ggml_tensor_overhead()*8 + ggml_graph_overhead(),
/*.mem_buffer =*/ nullptr,
/*.no_alloc =*/ true,
};
ggml_context * ctx = ggml_init(params);
ggml_tensor * a = ggml_new_tensor_2d(ctx, GGML_TYPE_DT3, NCOLS, NROWS);
ggml_tensor * rows = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, NROWS);
ggml_tensor * out = ggml_get_rows(ctx, a, rows);
if (!ggml_backend_supports_op(backend, out)) {
printf("FAILED: backend does not support GET_ROWS on DT3\n");
ggml_free(ctx);
return 1;
}
ggml_backend_buffer_t buf = ggml_backend_alloc_ctx_tensors(ctx, backend);
GGML_ASSERT(buf != nullptr);
std::vector<int32_t> row_idx(NROWS);
for (int r = 0; r < NROWS; ++r) {
row_idx[r] = r;
}
ggml_backend_tensor_set(a, data.data(), 0, data.size());
ggml_backend_tensor_set(rows, row_idx.data(), 0, NROWS*sizeof(int32_t));
std::vector<float> gpu((size_t)NROWS*NCOLS);
compute_graph(backend, ctx, out, gpu.data());
int num_failed = 0;
double max_diff = 0.0;
for (size_t i = 0; i < gpu.size(); ++i) {
const double diff = fabs((double)gpu[i] - (double)ref[i]);
max_diff = diff > max_diff ? diff : max_diff;
if (gpu[i] != ref[i]) {
if (num_failed < 8) {
printf("FAILED: dequant mismatch at block %zu elem %zu: gpu %.9g, cpu %.9g\n",
i/QK_DT3, i%QK_DT3, gpu[i], ref[i]);
}
num_failed++;
}
}
printf("%s: dequant GPU vs CPU on %d blocks: %d mismatches, max |diff| = %g\n",
num_failed == 0 ? "OK" : "FAILED", NBLOCKS, num_failed, max_diff);
ggml_backend_buffer_free(buf);
ggml_free(ctx);
return num_failed == 0 ? 0 : 1;
}
// MUL_MAT on the GPU vs a double precision reference from the CPU-dequantized
// weights. n_cols_dst <= 8 goes through MMVQ; the activations are integers
// with amax 127 in every 32-element chunk, so their q8_1 quantization is
// exact and the reference is valid to float accumulation rounding.
static int test_mul_mat(ggml_backend_t backend, const std::vector<uint8_t> & data, const std::vector<float> & ref_w) {
int num_failed = 0;
const int ncols_dst[] = {1, 2, 5, 8, 16};
std::vector<float> y((size_t)NCOLS*16);
for (size_t i = 0; i < y.size(); ++i) {
y[i] = i % 32 == 0 ? 127.0f : (float)((int)(rng_next() % 255) - 127);
}
std::vector<std::vector<float>> results;
for (int c = 0; c < (int)(sizeof(ncols_dst)/sizeof(ncols_dst[0])); ++c) {
const int n = ncols_dst[c];
ggml_init_params params = {
/*.mem_size =*/ ggml_tensor_overhead()*8 + ggml_graph_overhead(),
/*.mem_buffer =*/ nullptr,
/*.no_alloc =*/ true,
};
ggml_context * ctx = ggml_init(params);
ggml_tensor * a = ggml_new_tensor_2d(ctx, GGML_TYPE_DT3, NCOLS, NROWS);
ggml_tensor * b = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, NCOLS, n);
ggml_tensor * out = ggml_mul_mat(ctx, a, b);
if (!ggml_backend_supports_op(backend, out)) {
printf("FAILED: backend does not support MUL_MAT on DT3\n");
ggml_free(ctx);
return 1;
}
ggml_backend_buffer_t buf = ggml_backend_alloc_ctx_tensors(ctx, backend);
GGML_ASSERT(buf != nullptr);
ggml_backend_tensor_set(a, data.data(), 0, data.size());
ggml_backend_tensor_set(b, y.data(), 0, (size_t)NCOLS*n*sizeof(float));
std::vector<float> gpu((size_t)NROWS*n);
compute_graph(backend, ctx, out, gpu.data());
results.push_back(gpu);
// reference in double from the dequantized weights
double max_rel = 0.0;
for (int j = 0; j < n; ++j) {
for (int r = 0; r < NROWS; ++r) {
double sum = 0.0;
for (int k = 0; k < NCOLS; ++k) {
sum += (double)ref_w[(size_t)r*NCOLS + k] * (double)y[(size_t)j*NCOLS + k];
}
const double rel = fabs((double)gpu[(size_t)j*NROWS + r] - sum) / (fabs(sum) > 1.0 ? fabs(sum) : 1.0);
max_rel = rel > max_rel ? rel : max_rel;
}
}
// n <= 8 is the MMVQ path with exact integer dot products; larger n
// falls back to dequantization + GEMM, which may run in fp16
const double tol = n <= 8 ? 1e-5 : 5e-3;
printf("%s: mul_mat GPU vs reference, ncols_dst = %2d (%s): max rel err = %g\n",
max_rel <= tol ? "OK" : "FAILED", n, n <= 8 ? "MMVQ" : "GEMM", max_rel);
if (max_rel > tol) {
num_failed++;
}
ggml_backend_buffer_free(buf);
ggml_free(ctx);
}
// MMVQ vs the dequantization-based path: first 8 columns of the GEMM run
// must match the ncols_dst = 8 MMVQ run
{
const std::vector<float> & mmvq = results[3]; // n = 8
const std::vector<float> & gemm = results[4]; // n = 16
double max_rel = 0.0;
for (int j = 0; j < 8; ++j) {
for (int r = 0; r < NROWS; ++r) {
const double v0 = mmvq[(size_t)j*NROWS + r];
const double v1 = gemm[(size_t)j*NROWS + r];
const double rel = fabs(v0 - v1) / (fabs(v0) > 1.0 ? fabs(v0) : 1.0);
max_rel = rel > max_rel ? rel : max_rel;
}
}
printf("%s: MMVQ vs GEMM path on shared columns: max rel err = %g\n",
max_rel <= 5e-3 ? "OK" : "FAILED", max_rel);
if (max_rel > 5e-3) {
num_failed++;
}
}
// manual sum over the trits stored by the test for row 0, column 0 —
// computed from the trits themselves, not from any dequantization, with
// non-trivial qh trits in every block of the row
{
double sum = 0.0;
for (int j = 0; j < ROW0_NB; ++j) {
for (int i = 0; i < QK_DT3; ++i) {
sum += (double)y[(size_t)j*QK_DT3 + i] *
((double)row0_d1[j]*row0_t1[j][i] + (double)row0_d2[j]*row0_t2[j][i]);
}
}
const double got = results[0][0]; // ncols_dst = 1, row 0
const double rel = fabs(got - sum) / (fabs(sum) > 1.0 ? fabs(sum) : 1.0);
printf("%s: MMVQ vs manual trit sum (row 0, col 0): gpu %.9g, manual %.9g, rel err = %g\n",
rel <= 1e-5 ? "OK" : "FAILED", got, sum, rel);
if (rel > 1e-5) {
num_failed++;
}
}
return num_failed;
}
int main(void) {
ggml_backend_t backend = nullptr;
for (size_t i = 0; i < ggml_backend_dev_count(); ++i) {
ggml_backend_dev_t dev = ggml_backend_dev_get(i);
if (ggml_backend_dev_type(dev) == GGML_BACKEND_DEVICE_TYPE_GPU) {
backend = ggml_backend_dev_init(dev, nullptr);
printf("using GPU backend: %s\n", ggml_backend_dev_name(dev));
break;
}
}
if (backend == nullptr) {
printf("no GPU backend available, skipping\n");
return 0;
}
std::vector<uint8_t> data;
build_dt3_data(data);
// CPU reference dequantization — the validated path
std::vector<float> ref((size_t)NROWS*NCOLS);
const ggml_type_traits * qfns = ggml_get_type_traits(GGML_TYPE_DT3);
qfns->to_float(data.data(), ref.data(), (int64_t)NROWS*NCOLS);
int num_failed = 0;
num_failed += test_dequant(backend, data, ref);
num_failed += test_mul_mat(backend, data, ref);
ggml_backend_free(backend);
if (num_failed > 0) {
printf("%d tests FAILED\n", num_failed);
return 1;
}
printf("all tests OK\n");
return 0;
}