millaguie/llama.cpp
1
0
Fork 0
Code Issues Pull Requests Actions 33 Packages Projects Releases Wiki Activity

ggml : add DT3 reference quantization and dequantization

Browse Source 
Add the dual-plane ternary DT3 type to the type registry along with its
reference row functions. Each of the two planes is packed exactly like
tq1_0 with all constants halved (block of 128 elements): qs 48 -> 24
bytes over two passes of 16 and 8 bytes, qh 4 -> 2 bytes.

The trit decoding lives in a single exported helper (unpack_plane_dt3)
so that dequantization and the upcoming CPU vec_dot share it.

The reference quantizer is a greedy two-pass (plane 1 by absolute max,
plane 2 on the residual) plus two rounds of alternating least-squares
refits. It is intentionally NOT the PTQTP solver used to produce the
published DT3 models.
This commit is contained in:
Millaguie
2026-08-10 13:11:50 +02:00
parent 4d9a6f55b6
commit 948f51d274
4 changed files with 215 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ggml/include/ggml.h
+3 -1
View File
@@ -430,7 +430,8 @@ extern "C" {
GGML_TYPE_NVFP4 = 40, // NVFP4 (4 blocks, E4M3 scale)
GGML_TYPE_Q1_0 = 41,
GGML_TYPE_Q2_0 = 42,
GGML_TYPE_COUNT = 43,
GGML_TYPE_DT3 = 43, // DT3 (dual-plane ternary)
GGML_TYPE_COUNT = 44,
};
// precision
@@ -475,6 +476,7 @@ extern "C" {
GGML_FTYPE_MOSTLY_NVFP4 = 26, // except 1d tensors
GGML_FTYPE_MOSTLY_Q1_0 = 27, // except 1d tensors
GGML_FTYPE_MOSTLY_Q2_0 = 28, // except 1d tensors
GGML_FTYPE_MOSTLY_DT3 = 29, // except 1d tensors
};
// available tensor operations:
ggml/src/ggml-quants.c
+194
View File
@@ -2483,6 +2483,191 @@ void dequantize_row_tq2_0(const block_tq2_0 * GGML_RESTRICT x, float * GGML_REST
}
}
// ====================== Dual-plane ternary (de)-quantization (DT3)
// packs one plane of QK_DT3 trits in {-1, 0, 1}, exactly like tq1_0 with all
// constants halved (see block_dt3 in ggml-common.h)
static void pack_plane_dt3(const int8_t * GGML_RESTRICT t, uint8_t * GGML_RESTRICT qs, uint8_t * GGML_RESTRICT qh) {
// 5 elements per byte, along 16 bytes
for (size_t m = 0; m < 16; ++m) {
uint8_t q = 0;
for (size_t n = 0; n < 5; ++n) {
const int xi = t[m + n*16] + 1; // -1, 0, 1 -> 0, 1, 2
q *= 3;
q += xi;
}
// ceiling division (243 == pow(3, 5))
q = ((uint16_t)q * 256 + (243 - 1)) / 243;
qs[m] = q;
}
// along 8 bytes
for (size_t m = 0; m < 8; ++m) {
uint8_t q = 0;
for (size_t n = 0; n < 5; ++n) {
const int xi = t[80 + m + n*8] + 1; // -1, 0, 1 -> 0, 1, 2
q *= 3;
q += xi;
}
// ceiling division (243 == pow(3, 5))
q = ((uint16_t)q * 256 + (243 - 1)) / 243;
qs[16 + m] = q;
}
// 4 elements per byte
for (size_t j = 0; j < 2; ++j) {
uint8_t q = 0;
for (size_t m = 0; m < 4; ++m) {
const int xi = t[120 + j + m*2] + 1; // -1, 0, 1 -> 0, 1, 2
q *= 3;
q += xi;
}
// shift the first value to the most significant trit
q *= 3;
// ceiling division (243 == pow(3, 5))
q = ((uint16_t)q * 256 + (243 - 1)) / 243;
qh[j] = q;
}
}
// NOTE: this is NOT the PTQTP coordinate-descent solver used to produce the
// published DT3 models; that solver lives in the ternaria project and its
// output is packed directly into this layout. This reference is a greedy
// two-pass quantization (plane 1 by absolute max, plane 2 on the residual)
// plus two rounds of alternating least-squares refits — good enough for the
// type to be usable from float, but the measured quality of DT3 is only
// obtained through the external PTQTP pipeline.
void quantize_row_dt3_ref(const float * GGML_RESTRICT x, block_dt3 * GGML_RESTRICT y, int64_t k) {
assert(k % QK_DT3 == 0);
const int64_t nb = k / QK_DT3;
for (int64_t i = 0; i < nb; i++) {
int8_t t1[QK_DT3];
int8_t t2[QK_DT3];
float d1;
float d2;
// plane 1: ternary quantization by absolute max, like tq1_0
{
float amax = 0.0f;
for (int j = 0; j < QK_DT3; j++) {
amax = MAX(amax, fabsf(x[j]));
}
d1 = amax;
const float id = d1 ? 1.0f/d1 : 0.0f;
for (int j = 0; j < QK_DT3; j++) {
t1[j] = (int8_t) lroundf(x[j] * id); // -1, 0, 1
}
}
// plane 2: ternary quantization of the residual
{
float amax = 0.0f;
for (int j = 0; j < QK_DT3; j++) {
amax = MAX(amax, fabsf(x[j] - d1*t1[j]));
}
d2 = amax;
const float id = d2 ? 1.0f/d2 : 0.0f;
for (int j = 0; j < QK_DT3; j++) {
t2[j] = (int8_t) lroundf((x[j] - d1*t1[j]) * id); // -1, 0, 1
}
}
// two rounds of alternating refits: least-squares scale given the
// trits, then re-solve the trits given the scale, holding the other
// plane fixed
for (int it = 0; it < 2; ++it) {
for (int p = 0; p < 2; ++p) {
int8_t * t = p == 0 ? t1 : t2;
const int8_t * tother = p == 0 ? t2 : t1;
float * d = p == 0 ? &d1 : &d2;
const float dother = p == 0 ? d2 : d1;
float sum_rt = 0.0f;
float sum_tt = 0.0f;
for (int j = 0; j < QK_DT3; j++) {
const float r = x[j] - dother*tother[j];
sum_rt += r * t[j];
sum_tt += (float)(t[j] * t[j]);
}
if (sum_tt > 0.0f) {
*d = sum_rt / sum_tt;
}
const float id = *d ? 1.0f/(*d) : 0.0f;
for (int j = 0; j < QK_DT3; j++) {
const float r = x[j] - dother*tother[j];
const int v = (int) lroundf(r * id);
t[j] = (int8_t) MAX(-1, MIN(1, v));
}
}
}
y[i].d[0] = GGML_FP32_TO_FP16(d1);
y[i].d[1] = GGML_FP32_TO_FP16(d2);
pack_plane_dt3(t1, y[i].qs[0], y[i].qh[0]);
pack_plane_dt3(t2, y[i].qs[1], y[i].qh[1]);
x += QK_DT3;
}
}
size_t quantize_dt3(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrow, int64_t n_per_row, const float * quant_weights) {
(void)quant_weights; // not used
const size_t row_size = ggml_row_size(GGML_TYPE_DT3, n_per_row);
quantize_row_dt3_ref(src, dst, (int64_t)nrow*n_per_row);
return nrow * row_size;
}
void unpack_plane_dt3(const uint8_t * GGML_RESTRICT qs, const uint8_t * GGML_RESTRICT qh, int8_t * GGML_RESTRICT t) {
const uint8_t pow3[6] = {1, 3, 9, 27, 81, 243};
// 5 elements per byte, along 16 bytes
for (size_t n = 0; n < 5; ++n) {
for (size_t m = 0; m < 16; ++m) {
const uint8_t q = qs[m] * pow3[n]; // the product wraps around on purpose
const int16_t xi = ((uint16_t) q * 3) >> 8;
*t++ = (int8_t)(xi - 1);
}
}
// along 8 bytes
for (size_t n = 0; n < 5; ++n) {
for (size_t m = 0; m < 8; ++m) {
const uint8_t q = qs[16 + m] * pow3[n];
const int16_t xi = ((uint16_t) q * 3) >> 8;
*t++ = (int8_t)(xi - 1);
}
}
// 4 elements per byte — NOT 5: each qh byte stores only 4 trits, and its
// 5th base-3 digit is padding from the packer's extra q *= 3 shift which
// always decodes to -1; reading it would inject spurious values
for (size_t n = 0; n < 4; ++n) {
for (size_t j = 0; j < 2; ++j) {
const uint8_t q = qh[j] * pow3[n];
const int16_t xi = ((uint16_t) q * 3) >> 8;
*t++ = (int8_t)(xi - 1);
}
}
}
void dequantize_row_dt3(const block_dt3 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) {
assert(k % QK_DT3 == 0);
const int64_t nb = k / QK_DT3;
for (int64_t i = 0; i < nb; ++i) {
int8_t t1[QK_DT3];
int8_t t2[QK_DT3];
unpack_plane_dt3(x[i].qs[0], x[i].qh[0], t1);
unpack_plane_dt3(x[i].qs[1], x[i].qh[1], t2);
const float d1 = GGML_FP16_TO_FP32(x[i].d[0]);
const float d2 = GGML_FP16_TO_FP32(x[i].d[1]);
for (int j = 0; j < QK_DT3; ++j) {
*y++ = d1*t1[j] + d2*t2[j];
}
}
}
// ====================== "True" 2-bit (de)-quantization
void dequantize_row_iq2_xxs(const block_iq2_xxs * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) {
@@ -5604,6 +5789,15 @@ bool ggml_validate_row_data(enum ggml_type type, const void * data, size_t nbyte
{
VALIDATE_ROW_DATA_D_F16_IMPL(block_tq2_0, data, nb);
} break;
case GGML_TYPE_DT3:
{
const block_dt3 * q = (const block_dt3 *) data;
for (size_t i = 0; i < nb; ++i) {
if (!validate_fp16(q[i].d[0], i) || !validate_fp16(q[i].d[1], i)) {
return false;
}
}
} break;
case GGML_TYPE_IQ1_S:
{
VALIDATE_ROW_DATA_D_F16_IMPL(block_iq1_s, data, nb);
ggml/src/ggml-quants.h
+8
View File
@@ -35,6 +35,7 @@ GGML_API void quantize_row_q8_K_ref(const float * GGML_RESTRICT x, block_q8_K *
GGML_API void quantize_row_tq1_0_ref(const float * GGML_RESTRICT x, block_tq1_0 * GGML_RESTRICT y, int64_t k);
GGML_API void quantize_row_tq2_0_ref(const float * GGML_RESTRICT x, block_tq2_0 * GGML_RESTRICT y, int64_t k);
GGML_API void quantize_row_dt3_ref (const float * GGML_RESTRICT x, block_dt3 * GGML_RESTRICT y, int64_t k);
GGML_API void quantize_row_iq3_xxs_ref(const float * GGML_RESTRICT x, block_iq3_xxs * GGML_RESTRICT y, int64_t k);
GGML_API void quantize_row_iq4_nl_ref (const float * GGML_RESTRICT x, block_iq4_nl * GGML_RESTRICT y, int64_t k);
@@ -64,6 +65,12 @@ GGML_API void dequantize_row_q8_K(const block_q8_K * GGML_RESTRICT x, float * GG
GGML_API void dequantize_row_tq1_0(const block_tq1_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
GGML_API void dequantize_row_tq2_0(const block_tq2_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
GGML_API void dequantize_row_dt3 (const block_dt3 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
// unpacks one DT3 plane (128 trits in {-1, 0, 1}) in the original element order;
// shared between dequantize_row_dt3 and the CPU vec_dot so that the trit
// decoding exists in exactly one place
GGML_API void unpack_plane_dt3(const uint8_t * GGML_RESTRICT qs, const uint8_t * GGML_RESTRICT qh, int8_t * GGML_RESTRICT t);
GGML_API void dequantize_row_iq2_xxs(const block_iq2_xxs * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
GGML_API void dequantize_row_iq2_xs (const block_iq2_xs * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
@@ -88,6 +95,7 @@ GGML_API size_t quantize_iq3_s (const float * GGML_RESTRICT src, void * GGML_RE
GGML_API size_t quantize_tq1_0(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
GGML_API size_t quantize_tq2_0(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
GGML_API size_t quantize_dt3 (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
GGML_API size_t quantize_q2_K(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
GGML_API size_t quantize_q3_K(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
ggml/src/ggml.c
+10
View File
@@ -924,6 +924,14 @@ static const struct ggml_type_traits type_traits[GGML_TYPE_COUNT] = {
.to_float = (ggml_to_float_t) dequantize_row_tq2_0,
.from_float_ref = (ggml_from_float_t) quantize_row_tq2_0_ref,
},
[GGML_TYPE_DT3] = {
.type_name = "dt3",
.blck_size = QK_DT3,
.type_size = sizeof(block_dt3),
.is_quantized = true,
.to_float = (ggml_to_float_t) dequantize_row_dt3,
.from_float_ref = (ggml_from_float_t) quantize_row_dt3_ref,
},
[36] = { // GGML_TYPE_IQ4_NL_4_4
.type_name = "TYPE_IQ4_NL_4_4 REMOVED, use IQ4_NL with runtime repacking",
.blck_size = 0,
@@ -1434,6 +1442,7 @@ enum ggml_type ggml_ftype_to_ggml_type(enum ggml_ftype ftype) {
case GGML_FTYPE_MOSTLY_Q4_1: wtype = GGML_TYPE_Q4_1; break;
case GGML_FTYPE_MOSTLY_Q1_0: wtype = GGML_TYPE_Q1_0; break;
case GGML_FTYPE_MOSTLY_Q2_0: wtype = GGML_TYPE_Q2_0; break;
case GGML_FTYPE_MOSTLY_DT3: wtype = GGML_TYPE_DT3; break;
case GGML_FTYPE_MOSTLY_Q5_0: wtype = GGML_TYPE_Q5_0; break;
case GGML_FTYPE_MOSTLY_Q5_1: wtype = GGML_TYPE_Q5_1; break;
case GGML_FTYPE_MOSTLY_Q8_0: wtype = GGML_TYPE_Q8_0; break;
@@ -7957,6 +7966,7 @@ size_t ggml_quantize_chunk(
case GGML_TYPE_Q6_K: result = quantize_q6_K (src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break;
case GGML_TYPE_TQ1_0: result = quantize_tq1_0 (src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break;
case GGML_TYPE_TQ2_0: result = quantize_tq2_0 (src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break;
case GGML_TYPE_DT3: result = quantize_dt3 (src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break;
case GGML_TYPE_IQ2_XXS: result = quantize_iq2_xxs(src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break;
case GGML_TYPE_IQ2_XS: result = quantize_iq2_xs (src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break;
case GGML_TYPE_IQ3_XXS: result = quantize_iq3_xxs(src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break;