diff --git a/ggml/include/ggml.h b/ggml/include/ggml.h index 5cb49d0ee..47de68737 100644 --- a/ggml/include/ggml.h +++ b/ggml/include/ggml.h @@ -430,7 +430,8 @@ extern "C" { GGML_TYPE_NVFP4 = 40, // NVFP4 (4 blocks, E4M3 scale) GGML_TYPE_Q1_0 = 41, GGML_TYPE_Q2_0 = 42, - GGML_TYPE_COUNT = 43, + GGML_TYPE_DT3 = 43, // DT3 (dual-plane ternary) + GGML_TYPE_COUNT = 44, }; // precision @@ -475,6 +476,7 @@ extern "C" { GGML_FTYPE_MOSTLY_NVFP4 = 26, // except 1d tensors GGML_FTYPE_MOSTLY_Q1_0 = 27, // except 1d tensors GGML_FTYPE_MOSTLY_Q2_0 = 28, // except 1d tensors + GGML_FTYPE_MOSTLY_DT3 = 29, // except 1d tensors }; // available tensor operations: diff --git a/ggml/src/ggml-quants.c b/ggml/src/ggml-quants.c index 1ebc50a76..349e8e192 100644 --- a/ggml/src/ggml-quants.c +++ b/ggml/src/ggml-quants.c @@ -2483,6 +2483,191 @@ void dequantize_row_tq2_0(const block_tq2_0 * GGML_RESTRICT x, float * GGML_REST } } +// ====================== Dual-plane ternary (de)-quantization (DT3) + +// packs one plane of QK_DT3 trits in {-1, 0, 1}, exactly like tq1_0 with all +// constants halved (see block_dt3 in ggml-common.h) +static void pack_plane_dt3(const int8_t * GGML_RESTRICT t, uint8_t * GGML_RESTRICT qs, uint8_t * GGML_RESTRICT qh) { + // 5 elements per byte, along 16 bytes + for (size_t m = 0; m < 16; ++m) { + uint8_t q = 0; + for (size_t n = 0; n < 5; ++n) { + const int xi = t[m + n*16] + 1; // -1, 0, 1 -> 0, 1, 2 + q *= 3; + q += xi; + } + // ceiling division (243 == pow(3, 5)) + q = ((uint16_t)q * 256 + (243 - 1)) / 243; + qs[m] = q; + } + // along 8 bytes + for (size_t m = 0; m < 8; ++m) { + uint8_t q = 0; + for (size_t n = 0; n < 5; ++n) { + const int xi = t[80 + m + n*8] + 1; // -1, 0, 1 -> 0, 1, 2 + q *= 3; + q += xi; + } + // ceiling division (243 == pow(3, 5)) + q = ((uint16_t)q * 256 + (243 - 1)) / 243; + qs[16 + m] = q; + } + // 4 elements per byte + for (size_t j = 0; j < 2; ++j) { + uint8_t q = 0; + for (size_t m = 0; m < 4; ++m) { + const int xi = t[120 + j + m*2] + 1; // -1, 0, 1 -> 0, 1, 2 + q *= 3; + q += xi; + } + // shift the first value to the most significant trit + q *= 3; + // ceiling division (243 == pow(3, 5)) + q = ((uint16_t)q * 256 + (243 - 1)) / 243; + qh[j] = q; + } +} + +// NOTE: this is NOT the PTQTP coordinate-descent solver used to produce the +// published DT3 models; that solver lives in the ternaria project and its +// output is packed directly into this layout. This reference is a greedy +// two-pass quantization (plane 1 by absolute max, plane 2 on the residual) +// plus two rounds of alternating least-squares refits — good enough for the +// type to be usable from float, but the measured quality of DT3 is only +// obtained through the external PTQTP pipeline. +void quantize_row_dt3_ref(const float * GGML_RESTRICT x, block_dt3 * GGML_RESTRICT y, int64_t k) { + assert(k % QK_DT3 == 0); + const int64_t nb = k / QK_DT3; + + for (int64_t i = 0; i < nb; i++) { + int8_t t1[QK_DT3]; + int8_t t2[QK_DT3]; + float d1; + float d2; + + // plane 1: ternary quantization by absolute max, like tq1_0 + { + float amax = 0.0f; + for (int j = 0; j < QK_DT3; j++) { + amax = MAX(amax, fabsf(x[j])); + } + d1 = amax; + const float id = d1 ? 1.0f/d1 : 0.0f; + for (int j = 0; j < QK_DT3; j++) { + t1[j] = (int8_t) lroundf(x[j] * id); // -1, 0, 1 + } + } + + // plane 2: ternary quantization of the residual + { + float amax = 0.0f; + for (int j = 0; j < QK_DT3; j++) { + amax = MAX(amax, fabsf(x[j] - d1*t1[j])); + } + d2 = amax; + const float id = d2 ? 1.0f/d2 : 0.0f; + for (int j = 0; j < QK_DT3; j++) { + t2[j] = (int8_t) lroundf((x[j] - d1*t1[j]) * id); // -1, 0, 1 + } + } + + // two rounds of alternating refits: least-squares scale given the + // trits, then re-solve the trits given the scale, holding the other + // plane fixed + for (int it = 0; it < 2; ++it) { + for (int p = 0; p < 2; ++p) { + int8_t * t = p == 0 ? t1 : t2; + const int8_t * tother = p == 0 ? t2 : t1; + float * d = p == 0 ? &d1 : &d2; + const float dother = p == 0 ? d2 : d1; + + float sum_rt = 0.0f; + float sum_tt = 0.0f; + for (int j = 0; j < QK_DT3; j++) { + const float r = x[j] - dother*tother[j]; + sum_rt += r * t[j]; + sum_tt += (float)(t[j] * t[j]); + } + if (sum_tt > 0.0f) { + *d = sum_rt / sum_tt; + } + const float id = *d ? 1.0f/(*d) : 0.0f; + for (int j = 0; j < QK_DT3; j++) { + const float r = x[j] - dother*tother[j]; + const int v = (int) lroundf(r * id); + t[j] = (int8_t) MAX(-1, MIN(1, v)); + } + } + } + + y[i].d[0] = GGML_FP32_TO_FP16(d1); + y[i].d[1] = GGML_FP32_TO_FP16(d2); + + pack_plane_dt3(t1, y[i].qs[0], y[i].qh[0]); + pack_plane_dt3(t2, y[i].qs[1], y[i].qh[1]); + + x += QK_DT3; + } +} + +size_t quantize_dt3(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrow, int64_t n_per_row, const float * quant_weights) { + (void)quant_weights; // not used + const size_t row_size = ggml_row_size(GGML_TYPE_DT3, n_per_row); + quantize_row_dt3_ref(src, dst, (int64_t)nrow*n_per_row); + return nrow * row_size; +} + +void unpack_plane_dt3(const uint8_t * GGML_RESTRICT qs, const uint8_t * GGML_RESTRICT qh, int8_t * GGML_RESTRICT t) { + const uint8_t pow3[6] = {1, 3, 9, 27, 81, 243}; + + // 5 elements per byte, along 16 bytes + for (size_t n = 0; n < 5; ++n) { + for (size_t m = 0; m < 16; ++m) { + const uint8_t q = qs[m] * pow3[n]; // the product wraps around on purpose + const int16_t xi = ((uint16_t) q * 3) >> 8; + *t++ = (int8_t)(xi - 1); + } + } + // along 8 bytes + for (size_t n = 0; n < 5; ++n) { + for (size_t m = 0; m < 8; ++m) { + const uint8_t q = qs[16 + m] * pow3[n]; + const int16_t xi = ((uint16_t) q * 3) >> 8; + *t++ = (int8_t)(xi - 1); + } + } + // 4 elements per byte — NOT 5: each qh byte stores only 4 trits, and its + // 5th base-3 digit is padding from the packer's extra q *= 3 shift which + // always decodes to -1; reading it would inject spurious values + for (size_t n = 0; n < 4; ++n) { + for (size_t j = 0; j < 2; ++j) { + const uint8_t q = qh[j] * pow3[n]; + const int16_t xi = ((uint16_t) q * 3) >> 8; + *t++ = (int8_t)(xi - 1); + } + } +} + +void dequantize_row_dt3(const block_dt3 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) { + assert(k % QK_DT3 == 0); + const int64_t nb = k / QK_DT3; + + for (int64_t i = 0; i < nb; ++i) { + int8_t t1[QK_DT3]; + int8_t t2[QK_DT3]; + + unpack_plane_dt3(x[i].qs[0], x[i].qh[0], t1); + unpack_plane_dt3(x[i].qs[1], x[i].qh[1], t2); + + const float d1 = GGML_FP16_TO_FP32(x[i].d[0]); + const float d2 = GGML_FP16_TO_FP32(x[i].d[1]); + + for (int j = 0; j < QK_DT3; ++j) { + *y++ = d1*t1[j] + d2*t2[j]; + } + } +} + // ====================== "True" 2-bit (de)-quantization void dequantize_row_iq2_xxs(const block_iq2_xxs * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) { @@ -5604,6 +5789,15 @@ bool ggml_validate_row_data(enum ggml_type type, const void * data, size_t nbyte { VALIDATE_ROW_DATA_D_F16_IMPL(block_tq2_0, data, nb); } break; + case GGML_TYPE_DT3: + { + const block_dt3 * q = (const block_dt3 *) data; + for (size_t i = 0; i < nb; ++i) { + if (!validate_fp16(q[i].d[0], i) || !validate_fp16(q[i].d[1], i)) { + return false; + } + } + } break; case GGML_TYPE_IQ1_S: { VALIDATE_ROW_DATA_D_F16_IMPL(block_iq1_s, data, nb); diff --git a/ggml/src/ggml-quants.h b/ggml/src/ggml-quants.h index 75188f1af..47b7c9207 100644 --- a/ggml/src/ggml-quants.h +++ b/ggml/src/ggml-quants.h @@ -35,6 +35,7 @@ GGML_API void quantize_row_q8_K_ref(const float * GGML_RESTRICT x, block_q8_K * GGML_API void quantize_row_tq1_0_ref(const float * GGML_RESTRICT x, block_tq1_0 * GGML_RESTRICT y, int64_t k); GGML_API void quantize_row_tq2_0_ref(const float * GGML_RESTRICT x, block_tq2_0 * GGML_RESTRICT y, int64_t k); +GGML_API void quantize_row_dt3_ref (const float * GGML_RESTRICT x, block_dt3 * GGML_RESTRICT y, int64_t k); GGML_API void quantize_row_iq3_xxs_ref(const float * GGML_RESTRICT x, block_iq3_xxs * GGML_RESTRICT y, int64_t k); GGML_API void quantize_row_iq4_nl_ref (const float * GGML_RESTRICT x, block_iq4_nl * GGML_RESTRICT y, int64_t k); @@ -64,6 +65,12 @@ GGML_API void dequantize_row_q8_K(const block_q8_K * GGML_RESTRICT x, float * GG GGML_API void dequantize_row_tq1_0(const block_tq1_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); GGML_API void dequantize_row_tq2_0(const block_tq2_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); +GGML_API void dequantize_row_dt3 (const block_dt3 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); + +// unpacks one DT3 plane (128 trits in {-1, 0, 1}) in the original element order; +// shared between dequantize_row_dt3 and the CPU vec_dot so that the trit +// decoding exists in exactly one place +GGML_API void unpack_plane_dt3(const uint8_t * GGML_RESTRICT qs, const uint8_t * GGML_RESTRICT qh, int8_t * GGML_RESTRICT t); GGML_API void dequantize_row_iq2_xxs(const block_iq2_xxs * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); GGML_API void dequantize_row_iq2_xs (const block_iq2_xs * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); @@ -88,6 +95,7 @@ GGML_API size_t quantize_iq3_s (const float * GGML_RESTRICT src, void * GGML_RE GGML_API size_t quantize_tq1_0(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix); GGML_API size_t quantize_tq2_0(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix); +GGML_API size_t quantize_dt3 (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix); GGML_API size_t quantize_q2_K(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix); GGML_API size_t quantize_q3_K(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix); diff --git a/ggml/src/ggml.c b/ggml/src/ggml.c index da7f3a5f2..4a068c7b9 100644 --- a/ggml/src/ggml.c +++ b/ggml/src/ggml.c @@ -924,6 +924,14 @@ static const struct ggml_type_traits type_traits[GGML_TYPE_COUNT] = { .to_float = (ggml_to_float_t) dequantize_row_tq2_0, .from_float_ref = (ggml_from_float_t) quantize_row_tq2_0_ref, }, + [GGML_TYPE_DT3] = { + .type_name = "dt3", + .blck_size = QK_DT3, + .type_size = sizeof(block_dt3), + .is_quantized = true, + .to_float = (ggml_to_float_t) dequantize_row_dt3, + .from_float_ref = (ggml_from_float_t) quantize_row_dt3_ref, + }, [36] = { // GGML_TYPE_IQ4_NL_4_4 .type_name = "TYPE_IQ4_NL_4_4 REMOVED, use IQ4_NL with runtime repacking", .blck_size = 0, @@ -1434,6 +1442,7 @@ enum ggml_type ggml_ftype_to_ggml_type(enum ggml_ftype ftype) { case GGML_FTYPE_MOSTLY_Q4_1: wtype = GGML_TYPE_Q4_1; break; case GGML_FTYPE_MOSTLY_Q1_0: wtype = GGML_TYPE_Q1_0; break; case GGML_FTYPE_MOSTLY_Q2_0: wtype = GGML_TYPE_Q2_0; break; + case GGML_FTYPE_MOSTLY_DT3: wtype = GGML_TYPE_DT3; break; case GGML_FTYPE_MOSTLY_Q5_0: wtype = GGML_TYPE_Q5_0; break; case GGML_FTYPE_MOSTLY_Q5_1: wtype = GGML_TYPE_Q5_1; break; case GGML_FTYPE_MOSTLY_Q8_0: wtype = GGML_TYPE_Q8_0; break; @@ -7957,6 +7966,7 @@ size_t ggml_quantize_chunk( case GGML_TYPE_Q6_K: result = quantize_q6_K (src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break; case GGML_TYPE_TQ1_0: result = quantize_tq1_0 (src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break; case GGML_TYPE_TQ2_0: result = quantize_tq2_0 (src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break; + case GGML_TYPE_DT3: result = quantize_dt3 (src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break; case GGML_TYPE_IQ2_XXS: result = quantize_iq2_xxs(src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break; case GGML_TYPE_IQ2_XS: result = quantize_iq2_xs (src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break; case GGML_TYPE_IQ3_XXS: result = quantize_iq3_xxs(src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break;