millaguie/llama.cpp
1
0
Fork 0
Code Issues Pull Requests Actions 33 Packages Projects Releases Wiki Activity

vulkan : add DT3 dequant, get_rows and scalar mul_mat_vec

Browse Source 
Wires the dual-plane ternary type into the Vulkan backend through three
paths only:

- get_rows and the generic scalar mul_mat_vec use per-element decode in
  dequant_funcs.glsl: the byte and base-3 digit are located from the
  element index (regions qs[0..16), qs[16..24), qh[0..2)), the byte is
  multiplied by 3^n mod 256 and the top digit taken. w = d1*t1 + d2*t2
  is accumulated in fp32; both products are exact so the sum carries a
  single float rounding and reproduces the CPU reference bit by bit
  (verified: 0 mismatches over the 112-block synthetic test and over
  214,695,936 elements of real model tensors).
- larger matmuls fall back to dequant_dt3.comp (decode each byte once
  with q <- q*3 mod 256, fp32 sum, one rounding at the f16 write) plus
  the existing f16 matmul pipelines.

The qh bytes hold only 4 trits; their 5th base-3 digit is packing
padding that decodes to -1, so both decoders stop at 4 digits.

Deliberately NOT implemented, and declined instead of half-supported:
no coopmat/coopmat2/MMQ shaders are generated for DT3, and supports_op
answers false for MUL_MAT_ID (mul_mat_vec_id shaders are not generated
either). GET_ROWS and MUL_MAT answer true.

test-dt3-gpu accepts the Vulkan backend (and IGPU-type devices) and
passes on RADV STRIX1: dequant 0 mismatches, mul_mat n<=8 norm rel err
~1e-8, GEMM fallback bit-identical to an F16 GEMM on fp16-rounded
weights.
This commit is contained in:
Millaguie
2026-08-11 03:00:47 +02:00
parent c01c26b56e
commit 8ba4db150f
6 changed files with 165 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ggml/src/ggml-vulkan/ggml-vulkan.cpp
+15
View File
@@ -5220,6 +5220,7 @@ static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) {
ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_BF16][i], "mul_mat_vec_bf16_f32_f32", arr_dmmv_bf16_f32_f32_len[reduc], arr_dmmv_bf16_f32_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {2, 1, 1}, {wg_size_subgroup, 2, i+1}, 1, false, use_subgroups, force_subgroup_size);
ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_Q1_0][i], "mul_mat_vec_q1_0_f32_f32", arr_dmmv_q1_0_f32_f32_len[reduc], arr_dmmv_q1_0_f32_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {2*rm_stdq, 1, 1}, {wg_size_subgroup, 2*rm_stdq, i+1}, 1, true, use_subgroups, force_subgroup_size);
ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_Q2_0][i], "mul_mat_vec_q2_0_f32_f32", arr_dmmv_q2_0_f32_f32_len[reduc], arr_dmmv_q2_0_f32_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {2*rm_stdq, 1, 1}, {wg_size_subgroup, 2*rm_stdq, i+1}, 1, true, use_subgroups, force_subgroup_size);
ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_DT3 ][i], "mul_mat_vec_dt3_f32_f32", arr_dmmv_dt3_f32_f32_len[reduc], arr_dmmv_dt3_f32_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {2*rm_stdq, 1, 1}, {wg_size_subgroup, 2*rm_stdq, i+1}, 1, true, use_subgroups, force_subgroup_size);
ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_Q4_0][i], "mul_mat_vec_q4_0_f32_f32", arr_dmmv_q4_0_f32_f32_len[reduc], arr_dmmv_q4_0_f32_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {2*rm_stdq, 1, 1}, {wg_size_subgroup, 2*rm_stdq, i+1}, 1, true, use_subgroups, force_subgroup_size);
ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_Q4_1][i], "mul_mat_vec_q4_1_f32_f32", arr_dmmv_q4_1_f32_f32_len[reduc], arr_dmmv_q4_1_f32_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {2*rm_stdq, 1, 1}, {wg_size_subgroup, 2*rm_stdq, i+1}, 1, true, use_subgroups, force_subgroup_size);
ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_Q5_0][i], "mul_mat_vec_q5_0_f32_f32", arr_dmmv_q5_0_f32_f32_len[reduc], arr_dmmv_q5_0_f32_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {2*rm_stdq, 1, 1}, {wg_size_subgroup, 2*rm_stdq, i+1}, 1, true, use_subgroups, force_subgroup_size);
@@ -5247,6 +5248,7 @@ static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) {
ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_BF16][i], "mul_mat_vec_bf16_f16_f32", arr_dmmv_bf16_f16_f32_len[reduc], arr_dmmv_bf16_f16_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {2, 1, 1}, {wg_size_subgroup, 2, i+1}, 1, false, use_subgroups, force_subgroup_size);
ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_Q1_0][i], "mul_mat_vec_q1_0_f16_f32", arr_dmmv_q1_0_f16_f32_len[reduc], arr_dmmv_q1_0_f16_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {2*rm_stdq, 1, 1}, {wg_size_subgroup, 2*rm_stdq, i+1}, 1, true, use_subgroups, force_subgroup_size);
ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_Q2_0][i], "mul_mat_vec_q2_0_f16_f32", arr_dmmv_q2_0_f16_f32_len[reduc], arr_dmmv_q2_0_f16_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {2*rm_stdq, 1, 1}, {wg_size_subgroup, 2*rm_stdq, i+1}, 1, true, use_subgroups, force_subgroup_size);
ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_DT3 ][i], "mul_mat_vec_dt3_f16_f32", arr_dmmv_dt3_f16_f32_len[reduc], arr_dmmv_dt3_f16_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {2*rm_stdq, 1, 1}, {wg_size_subgroup, 2*rm_stdq, i+1}, 1, true, use_subgroups, force_subgroup_size);
ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_Q4_0][i], "mul_mat_vec_q4_0_f16_f32", arr_dmmv_q4_0_f16_f32_len[reduc], arr_dmmv_q4_0_f16_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {2*rm_stdq, 1, 1}, {wg_size_subgroup, 2*rm_stdq, i+1}, 1, true, use_subgroups, force_subgroup_size);
ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_Q4_1][i], "mul_mat_vec_q4_1_f16_f32", arr_dmmv_q4_1_f16_f32_len[reduc], arr_dmmv_q4_1_f16_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {2*rm_stdq, 1, 1}, {wg_size_subgroup, 2*rm_stdq, i+1}, 1, true, use_subgroups, force_subgroup_size);
ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_Q5_0][i], "mul_mat_vec_q5_0_f16_f32", arr_dmmv_q5_0_f16_f32_len[reduc], arr_dmmv_q5_0_f16_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {2*rm_stdq, 1, 1}, {wg_size_subgroup, 2*rm_stdq, i+1}, 1, true, use_subgroups, force_subgroup_size);
@@ -5362,6 +5364,7 @@ static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) {
ggml_vk_create_pipeline(device, device->pipeline_dequant[GGML_TYPE_F32 ], "f32_to_f16", dequant_f32_len, dequant_f32_data, "main", 2, 5 * sizeof(uint32_t), {256 * 16, 1, 1}, {}, 1);
ggml_vk_create_pipeline(device, device->pipeline_dequant[GGML_TYPE_Q1_0], "dequant_q1_0", dequant_q1_0_len, dequant_q1_0_data, "main", 2, 5 * sizeof(uint32_t), {256 * 8, 1, 1}, {}, 1);
ggml_vk_create_pipeline(device, device->pipeline_dequant[GGML_TYPE_Q2_0], "dequant_q2_0", dequant_q2_0_len, dequant_q2_0_data, "main", 2, 5 * sizeof(uint32_t), {256 * 4, 1, 1}, {}, 1);
ggml_vk_create_pipeline(device, device->pipeline_dequant[GGML_TYPE_DT3 ], "dequant_dt3", dequant_dt3_len, dequant_dt3_data, "main", 2, 5 * sizeof(uint32_t), {256 * 4, 1, 1}, {}, 1);
ggml_vk_create_pipeline(device, device->pipeline_dequant[GGML_TYPE_Q4_0], "dequant_q4_0", dequant_q4_0_len, dequant_q4_0_data, "main", 2, 5 * sizeof(uint32_t), {256 * 16, 1, 1}, {}, 1);
ggml_vk_create_pipeline(device, device->pipeline_dequant[GGML_TYPE_Q4_1], "dequant_q4_1", dequant_q4_1_len, dequant_q4_1_data, "main", 2, 5 * sizeof(uint32_t), {256 * 16, 1, 1}, {}, 1);
ggml_vk_create_pipeline(device, device->pipeline_dequant[GGML_TYPE_Q5_0], "dequant_q5_0", dequant_q5_0_len, dequant_q5_0_data, "main", 2, 5 * sizeof(uint32_t), {256 * 16, 1, 1}, {}, 1);
@@ -5390,6 +5393,7 @@ static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) {
ggml_vk_create_pipeline(device, device->pipeline_get_rows[GGML_TYPE_BF16], "get_rows_bf16", get_rows_bf16_len, get_rows_bf16_data, "main", 3, sizeof(vk_op_binary_push_constants), { 512, 1, 1}, {}, 1);
ggml_vk_create_pipeline(device, device->pipeline_get_rows[GGML_TYPE_Q1_0], "get_rows_q1_0", get_rows_q1_0_len, get_rows_q1_0_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1);
ggml_vk_create_pipeline(device, device->pipeline_get_rows[GGML_TYPE_Q2_0], "get_rows_q2_0", get_rows_q2_0_len, get_rows_q2_0_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1);
ggml_vk_create_pipeline(device, device->pipeline_get_rows[GGML_TYPE_DT3 ], "get_rows_dt3", get_rows_dt3_len, get_rows_dt3_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1);
ggml_vk_create_pipeline(device, device->pipeline_get_rows[GGML_TYPE_Q4_0], "get_rows_q4_0", get_rows_q4_0_len, get_rows_q4_0_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1);
ggml_vk_create_pipeline(device, device->pipeline_get_rows[GGML_TYPE_Q4_1], "get_rows_q4_1", get_rows_q4_1_len, get_rows_q4_1_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1);
ggml_vk_create_pipeline(device, device->pipeline_get_rows[GGML_TYPE_Q5_0], "get_rows_q5_0", get_rows_q5_0_len, get_rows_q5_0_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1);
@@ -5418,6 +5422,7 @@ static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) {
ggml_vk_create_pipeline(device, device->pipeline_get_rows_f32[GGML_TYPE_BF16], "get_rows_bf16_f32", get_rows_bf16_f32_len, get_rows_bf16_f32_data, "main", 3, sizeof(vk_op_binary_push_constants), { 512, 1, 1}, {}, 1);
ggml_vk_create_pipeline(device, device->pipeline_get_rows_f32[GGML_TYPE_Q1_0], "get_rows_q1_0_f32", get_rows_q1_0_f32_len, get_rows_q1_0_f32_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1);
ggml_vk_create_pipeline(device, device->pipeline_get_rows_f32[GGML_TYPE_Q2_0], "get_rows_q2_0_f32", get_rows_q2_0_f32_len, get_rows_q2_0_f32_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1);
ggml_vk_create_pipeline(device, device->pipeline_get_rows_f32[GGML_TYPE_DT3 ], "get_rows_dt3_f32", get_rows_dt3_f32_len, get_rows_dt3_f32_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1);
ggml_vk_create_pipeline(device, device->pipeline_get_rows_f32[GGML_TYPE_Q4_0], "get_rows_q4_0_f32", get_rows_q4_0_f32_len, get_rows_q4_0_f32_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1);
ggml_vk_create_pipeline(device, device->pipeline_get_rows_f32[GGML_TYPE_Q4_1], "get_rows_q4_1_f32", get_rows_q4_1_f32_len, get_rows_q4_1_f32_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1);
ggml_vk_create_pipeline(device, device->pipeline_get_rows_f32[GGML_TYPE_Q5_0], "get_rows_q5_0_f32", get_rows_q5_0_f32_len, get_rows_q5_0_f32_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1);
@@ -7617,6 +7622,7 @@ static vk_pipeline ggml_vk_get_to_fp16(ggml_backend_vk_context * ctx, ggml_type
case GGML_TYPE_F32:
case GGML_TYPE_Q1_0:
case GGML_TYPE_Q2_0:
case GGML_TYPE_DT3:
case GGML_TYPE_Q4_0:
case GGML_TYPE_Q4_1:
case GGML_TYPE_Q5_0:
@@ -7760,6 +7766,7 @@ static vk_pipeline ggml_vk_get_dequantize_mul_mat_vec(ggml_backend_vk_context *
case GGML_TYPE_BF16:
case GGML_TYPE_Q1_0:
case GGML_TYPE_Q2_0:
case GGML_TYPE_DT3:
case GGML_TYPE_Q4_0:
case GGML_TYPE_Q4_1:
case GGML_TYPE_Q5_0:
@@ -17987,6 +17994,13 @@ static bool ggml_backend_vk_device_supports_op(ggml_backend_dev_t dev, const ggm
}
}
switch (src0_type) {
case GGML_TYPE_DT3:
// DT3 has dequant, get_rows and scalar mul_mat_vec shaders only:
// mul_mat_id, coopmat and MMQ are intentionally not implemented
if (op->op == GGML_OP_MUL_MAT_ID) {
return false;
}
break;
case GGML_TYPE_F32:
case GGML_TYPE_F16:
case GGML_TYPE_BF16:
@@ -18097,6 +18111,7 @@ static bool ggml_backend_vk_device_supports_op(ggml_backend_dev_t dev, const ggm
case GGML_TYPE_BF16:
case GGML_TYPE_Q1_0:
case GGML_TYPE_Q2_0:
case GGML_TYPE_DT3:
case GGML_TYPE_Q4_0:
case GGML_TYPE_Q4_1:
case GGML_TYPE_Q5_0:
ggml/src/ggml-vulkan/vulkan-shaders/dequant_dt3.comp
+47
View File
@@ -0,0 +1,47 @@
#version 450
#include "dequant_head.glsl"
layout(local_size_x = 256, local_size_y = 1, local_size_z = 1) in;
layout (binding = 0) readonly buffer A {block_dt3 data_a[];};
layout (binding = 1) writeonly buffer D {D_TYPE data_b[];};
// Eight blocks per workgroup, 32 threads per block. Threads 0..23 decode one
// qs byte of each plane (5 trits in base 3), threads 24..25 decode one qh
// byte of each plane (4 trits — the 5th base-3 digit of a qh byte is packing
// padding that always decodes to -1, so it must not be read), threads 26..31
// idle.
void main() {
const uint ib = gl_WorkGroupID.x * 8 + gl_LocalInvocationID.x / 32;
const uint il = gl_LocalInvocationID.x % 32;
if (ib >= p.nel / 128 || il >= 26) {
return;
}
const float d1 = float(data_a[ib].d[0]);
const float d2 = float(data_a[ib].d[1]);
const uint b_idx = ib * 128;
// element covered by the first digit, distance between consecutive
// digits, and number of digits stored in this byte
const bool is_qh = il >= 24;
const uint e0 = is_qh ? 120 + (il - 24) : (il < 16 ? il : 80 + (il - 16));
const uint stride = is_qh ? 2 : (il < 16 ? 16 : 8);
const uint digits = is_qh ? 4 : 5;
uint q1 = is_qh ? uint(data_a[ib].qh[il - 24]) : uint(data_a[ib].qs[il]);
uint q2 = is_qh ? uint(data_a[ib].qh[2 + il - 24]) : uint(data_a[ib].qs[24 + il]);
// decode each byte once: take the top base-3 digit with (q*3) >> 8, then
// shift it out with q <- (q*3) mod 256
for (uint n = 0; n < digits; ++n) {
const float t1 = float(int((q1 * 3) >> 8) - 1);
const float t2 = float(int((q2 * 3) >> 8) - 1);
data_b[b_idx + e0 + n*stride] = D_TYPE(d1*t1 + d2*t2);
q1 = (q1 * 3) & 0xFF;
q2 = (q2 * 3) & 0xFF;
}
}
ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs.glsl
+50
View File
@@ -154,6 +154,49 @@ vec4 dequantize4(uint ib, uint iqs, uint a_offset) {
}
#endif
#if defined(DATA_A_DT3)
// Dual-plane ternary: element iqs of plane p sits in a base-3 packed byte.
// Elements 0..79 use qs[m], m = iqs % 16, digit n = iqs / 16; elements
// 80..119 use qs[16 + m], m = (iqs - 80) % 8, digit n = (iqs - 80) / 8;
// elements 120..127 use qh[j], j = iqs % 2, digit n = (iqs - 120) / 2.
// A qh byte holds only 4 trits: its 5th base-3 digit is packing padding that
// always decodes to -1, never to 0, so it must not be read.
// The decode multiplies the byte by 3^n modulo 256 and takes the top digit.
float dt3_get_trit(uint ib, uint p, uint iqs, uint a_offset) {
const uint pow3[5] = {1, 3, 9, 27, 81};
uint b;
uint n;
if (iqs < 80) {
b = uint(data_a[a_offset + ib].qs[p*24 + (iqs & 15)]);
n = iqs >> 4;
} else if (iqs < 120) {
b = uint(data_a[a_offset + ib].qs[p*24 + 16 + ((iqs - 80) & 7)]);
n = (iqs - 80) >> 3;
} else {
b = uint(data_a[a_offset + ib].qh[p*2 + (iqs & 1)]);
n = (iqs - 120) >> 1;
}
const uint q = (b * pow3[n]) & 0xFF;
return float(int((q * 3) >> 8) - 1);
}
// w = d1*t1 + d2*t2; both products are exact (t in {-1,0,+1}), so the sum has
// a single float rounding and matches the CPU reference bit by bit
vec2 dequantize(uint ib, uint iqs, uint a_offset) {
const float d1 = float(data_a[a_offset + ib].d[0]);
const float d2 = float(data_a[a_offset + ib].d[1]);
return vec2(d1*dt3_get_trit(ib, 0, iqs, a_offset) + d2*dt3_get_trit(ib, 1, iqs, a_offset),
d1*dt3_get_trit(ib, 0, iqs + 1, a_offset) + d2*dt3_get_trit(ib, 1, iqs + 1, a_offset));
}
vec4 dequantize4(uint ib, uint iqs, uint a_offset) {
const float d1 = float(data_a[a_offset + ib].d[0]);
const float d2 = float(data_a[a_offset + ib].d[1]);
return vec4(d1*dt3_get_trit(ib, 0, iqs, a_offset) + d2*dt3_get_trit(ib, 1, iqs, a_offset),
d1*dt3_get_trit(ib, 0, iqs + 1, a_offset) + d2*dt3_get_trit(ib, 1, iqs + 1, a_offset),
d1*dt3_get_trit(ib, 0, iqs + 2, a_offset) + d2*dt3_get_trit(ib, 1, iqs + 2, a_offset),
d1*dt3_get_trit(ib, 0, iqs + 3, a_offset) + d2*dt3_get_trit(ib, 1, iqs + 3, a_offset));
}
#endif
#if defined(DATA_A_IQ1_S)
vec2 dequantize(uint ib, uint iqs, uint a_offset) {
const uint ib32 = iqs / 32;
@@ -571,6 +614,13 @@ vec2 get_dm(uint ib, uint a_offset) {
}
#endif
#if defined(DATA_A_DT3)
// the two scales are already applied inside dequantize/dequantize4
vec2 get_dm(uint ib, uint a_offset) {
return vec2(1, 0);
}
#endif
#if defined(DATA_A_MXFP4)
vec2 get_dm(uint ib, uint a_offset) {
return vec2(e8m0_to_fp32(data_a[a_offset + ib].e), 0);
ggml/src/ggml-vulkan/vulkan-shaders/types.glsl
+21
View File
@@ -235,6 +235,27 @@ struct block_q2_0_packed16
#define DATA_A_QUANT_LEGACY
#endif
#define QUANT_K_DT3 128
#define QUANT_R_DT3 1
// Dual-plane ternary: w = d[0]*t1 + d[1]*t2 with trits in {-1,0,+1}.
// Per plane: 24 bytes with 5 trits each in base 3 (elements 0..119), then
// 2 bytes with 4 trits each (elements 120..127). Plane p uses qs[p*24..],
// qh[p*2..] and d[p].
struct block_dt3
{
uint8_t qs[2*24];
uint8_t qh[2*2];
float16_t d[2];
};
#if defined(DATA_A_DT3)
#define QUANT_K QUANT_K_DT3
#define QUANT_R QUANT_R_DT3
#define QUANT_AUXF 1
#define A_TYPE block_dt3
#endif
#define QUANT_K_Q8_1 32
#define QUANT_R_Q8_1 1
ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp
+15 -4
View File
@@ -51,6 +51,7 @@ const std::vector<std::string> type_names = {
"f16",
"q1_0",
"q2_0",
"dt3",
"q4_0",
"q4_1",
"q5_0",
@@ -591,6 +592,12 @@ void matmul_shaders(bool fp16, MatMulIdType matmul_id_type, bool coopmat, bool c
continue;
}
// DT3 has no direct matmul shaders: mul_mat goes through dequant to
// f16 + f16 matmul, and coopmat/MMQ are intentionally not implemented
if (tname == "dt3") {
continue;
}
std::string data_a_key = "DATA_A_" + to_uppercase(tname);
// For aligned matmul loads
std::string load_vec_a = (coopmat2 || tname == "f32" || tname == "f16" || tname == "bf16") ? load_vec : load_vec_quant;
@@ -758,9 +765,12 @@ void process_shaders() {
}
#endif
string_to_spv("mul_mat_vec_id_" + tname + "_f32_f32", shader, merge_maps(base_dict, {{"MUL_MAT_ID", "1"}, {data_a_key, "1"}, {"B_TYPE", "float"}, {"B_TYPEV2", "vec2"}, {"B_TYPEV4", "vec4"}, {"D_TYPE", "float"}}));
string_to_spv("mul_mat_vec_id_" + tname + "_f32_f32_subgroup", shader, merge_maps(base_dict, {{"MUL_MAT_ID", "1"}, {data_a_key, "1"}, {"B_TYPE", "float"}, {"B_TYPEV2", "vec2"}, {"B_TYPEV4", "vec4"}, {"D_TYPE", "float"}, {"USE_SUBGROUP_ADD", "1"}}));
string_to_spv("mul_mat_vec_id_" + tname + "_f32_f32_subgroup_no_shmem", shader, merge_maps(base_dict, {{"MUL_MAT_ID", "1"}, {data_a_key, "1"}, {"B_TYPE", "float"}, {"B_TYPEV2", "vec2"}, {"B_TYPEV4", "vec4"}, {"D_TYPE", "float"}, {"USE_SUBGROUP_ADD_NO_SHMEM", "1"}}));
// mul_mat_id is not implemented for DT3 (supports_op declines it)
if (tname != "dt3") {
string_to_spv("mul_mat_vec_id_" + tname + "_f32_f32", shader, merge_maps(base_dict, {{"MUL_MAT_ID", "1"}, {data_a_key, "1"}, {"B_TYPE", "float"}, {"B_TYPEV2", "vec2"}, {"B_TYPEV4", "vec4"}, {"D_TYPE", "float"}}));
string_to_spv("mul_mat_vec_id_" + tname + "_f32_f32_subgroup", shader, merge_maps(base_dict, {{"MUL_MAT_ID", "1"}, {data_a_key, "1"}, {"B_TYPE", "float"}, {"B_TYPEV2", "vec2"}, {"B_TYPEV4", "vec4"}, {"D_TYPE", "float"}, {"USE_SUBGROUP_ADD", "1"}}));
string_to_spv("mul_mat_vec_id_" + tname + "_f32_f32_subgroup_no_shmem", shader, merge_maps(base_dict, {{"MUL_MAT_ID", "1"}, {data_a_key, "1"}, {"B_TYPE", "float"}, {"B_TYPEV2", "vec2"}, {"B_TYPEV4", "vec4"}, {"D_TYPE", "float"}, {"USE_SUBGROUP_ADD_NO_SHMEM", "1"}}));
}
// mul mat vec with integer dot product
#if defined(GGML_VULKAN_INTEGER_DOT_GLSLC_SUPPORT)
@@ -1254,7 +1264,8 @@ void write_output_files() {
src << "const uint64_t arr_dmmv_" << tname << "_" << btype << "_f32_len[3] = {mul_mat_vec_" << tname << "_" << btype << "_f32_len, mul_mat_vec_" << tname << "_" << btype << "_f32_subgroup_len, mul_mat_vec_" << tname << "_" << btype << "_f32_subgroup_no_shmem_len};\n";
}
if (btype == "f16") {
if (btype == "f16" || tname == "dt3") {
// no mul_mat_vec_id shaders for DT3
continue;
}
hdr << "extern const void * arr_dmmv_id_" << tname << "_" << btype << "_f32_data[3];\n";
tests/test-dt3-gpu.cpp
+17 -9
View File
@@ -33,8 +33,12 @@
// random bytes: every byte value 0..255 must decode identically on both
// sides, including values >= 243 that never come out of the packer.
//
// DT3 is implemented for CUDA and HIP only. Without one of those backends the
// test is skipped and succeeds — an unsupported backend is not a failure.
// DT3 is implemented for CUDA, HIP and Vulkan. Without one of those backends
// the test is skipped and succeeds — an unsupported backend is not a failure.
// On Vulkan the n <= 8 path is the scalar mul_mat_vec shader (fp32 dot on
// exactly decoded weights, not an integer dot), and the larger-n path is
// dequantization to fp16 + the f16 matmul pipeline; both are judged by the
// same gates as the CUDA MMVQ/GEMM paths.
#include "ggml.h"
#include "ggml-alloc.h"
@@ -461,21 +465,25 @@ static void build_control_data(ggml_type type, std::vector<uint8_t> & data, std:
}
int main(void) {
// Only CUDA and HIP (which reports itself as "ROCm") implement DT3. Any
// other GPU backend is skipped rather than failed: Vulkan and SYCL answer
// supports_op == false for DT3, which is the correct answer for them and
// Only CUDA, HIP (which reports itself as "ROCm") and Vulkan implement
// DT3. Any other GPU backend is skipped rather than failed: SYCL answers
// supports_op == false for DT3, which is the correct answer for it and
// not a bug to report, and Metal answers true for almost any type but has
// no DT3 shader, so it would die in pipeline compilation mid-test. Picking
// the backend by name keeps this test honest on machines we do not have.
ggml_backend_t backend = nullptr;
for (size_t i = 0; i < ggml_backend_dev_count(); ++i) {
ggml_backend_dev_t dev = ggml_backend_dev_get(i);
if (ggml_backend_dev_type(dev) != GGML_BACKEND_DEVICE_TYPE_GPU) {
// IGPU is a distinct device type from GPU: an integrated Vulkan device
// with unified memory reports as IGPU, and accepting only GPU silently
// skipped the very hardware this backend is for.
const auto dt = ggml_backend_dev_type(dev);
if (dt != GGML_BACKEND_DEVICE_TYPE_GPU && dt != GGML_BACKEND_DEVICE_TYPE_IGPU) {
continue;
}
const char * name = ggml_backend_dev_name(dev);
if (strncmp(name, "CUDA", 4) != 0 && strncmp(name, "ROCm", 4) != 0) {
printf("skipping GPU backend %s: DT3 is only implemented for CUDA/HIP\n", name);
if (strncmp(name, "CUDA", 4) != 0 && strncmp(name, "ROCm", 4) != 0 && strncmp(name, "Vulkan", 6) != 0) {
printf("skipping GPU backend %s: DT3 is only implemented for CUDA/HIP/Vulkan\n", name);
continue;
}
backend = ggml_backend_dev_init(dev, nullptr);
@@ -483,7 +491,7 @@ int main(void) {
break;
}
if (backend == nullptr) {
printf("no CUDA/HIP backend available, skipping\n");
printf("no CUDA/HIP/Vulkan backend available, skipping\n");
return 0;
}