diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cpp index a923755f9..6cf501e4e 100644 --- a/ggml/src/ggml-vulkan/ggml-vulkan.cpp +++ b/ggml/src/ggml-vulkan/ggml-vulkan.cpp @@ -5220,6 +5220,7 @@ static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_BF16][i], "mul_mat_vec_bf16_f32_f32", arr_dmmv_bf16_f32_f32_len[reduc], arr_dmmv_bf16_f32_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {2, 1, 1}, {wg_size_subgroup, 2, i+1}, 1, false, use_subgroups, force_subgroup_size); ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_Q1_0][i], "mul_mat_vec_q1_0_f32_f32", arr_dmmv_q1_0_f32_f32_len[reduc], arr_dmmv_q1_0_f32_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {2*rm_stdq, 1, 1}, {wg_size_subgroup, 2*rm_stdq, i+1}, 1, true, use_subgroups, force_subgroup_size); ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_Q2_0][i], "mul_mat_vec_q2_0_f32_f32", arr_dmmv_q2_0_f32_f32_len[reduc], arr_dmmv_q2_0_f32_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {2*rm_stdq, 1, 1}, {wg_size_subgroup, 2*rm_stdq, i+1}, 1, true, use_subgroups, force_subgroup_size); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_DT3 ][i], "mul_mat_vec_dt3_f32_f32", arr_dmmv_dt3_f32_f32_len[reduc], arr_dmmv_dt3_f32_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {2*rm_stdq, 1, 1}, {wg_size_subgroup, 2*rm_stdq, i+1}, 1, true, use_subgroups, force_subgroup_size); ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_Q4_0][i], "mul_mat_vec_q4_0_f32_f32", arr_dmmv_q4_0_f32_f32_len[reduc], arr_dmmv_q4_0_f32_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {2*rm_stdq, 1, 1}, {wg_size_subgroup, 2*rm_stdq, i+1}, 1, true, use_subgroups, force_subgroup_size); ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_Q4_1][i], "mul_mat_vec_q4_1_f32_f32", arr_dmmv_q4_1_f32_f32_len[reduc], arr_dmmv_q4_1_f32_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {2*rm_stdq, 1, 1}, {wg_size_subgroup, 2*rm_stdq, i+1}, 1, true, use_subgroups, force_subgroup_size); ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_Q5_0][i], "mul_mat_vec_q5_0_f32_f32", arr_dmmv_q5_0_f32_f32_len[reduc], arr_dmmv_q5_0_f32_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {2*rm_stdq, 1, 1}, {wg_size_subgroup, 2*rm_stdq, i+1}, 1, true, use_subgroups, force_subgroup_size); @@ -5247,6 +5248,7 @@ static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_BF16][i], "mul_mat_vec_bf16_f16_f32", arr_dmmv_bf16_f16_f32_len[reduc], arr_dmmv_bf16_f16_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {2, 1, 1}, {wg_size_subgroup, 2, i+1}, 1, false, use_subgroups, force_subgroup_size); ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_Q1_0][i], "mul_mat_vec_q1_0_f16_f32", arr_dmmv_q1_0_f16_f32_len[reduc], arr_dmmv_q1_0_f16_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {2*rm_stdq, 1, 1}, {wg_size_subgroup, 2*rm_stdq, i+1}, 1, true, use_subgroups, force_subgroup_size); ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_Q2_0][i], "mul_mat_vec_q2_0_f16_f32", arr_dmmv_q2_0_f16_f32_len[reduc], arr_dmmv_q2_0_f16_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {2*rm_stdq, 1, 1}, {wg_size_subgroup, 2*rm_stdq, i+1}, 1, true, use_subgroups, force_subgroup_size); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_DT3 ][i], "mul_mat_vec_dt3_f16_f32", arr_dmmv_dt3_f16_f32_len[reduc], arr_dmmv_dt3_f16_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {2*rm_stdq, 1, 1}, {wg_size_subgroup, 2*rm_stdq, i+1}, 1, true, use_subgroups, force_subgroup_size); ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_Q4_0][i], "mul_mat_vec_q4_0_f16_f32", arr_dmmv_q4_0_f16_f32_len[reduc], arr_dmmv_q4_0_f16_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {2*rm_stdq, 1, 1}, {wg_size_subgroup, 2*rm_stdq, i+1}, 1, true, use_subgroups, force_subgroup_size); ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_Q4_1][i], "mul_mat_vec_q4_1_f16_f32", arr_dmmv_q4_1_f16_f32_len[reduc], arr_dmmv_q4_1_f16_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {2*rm_stdq, 1, 1}, {wg_size_subgroup, 2*rm_stdq, i+1}, 1, true, use_subgroups, force_subgroup_size); ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_Q5_0][i], "mul_mat_vec_q5_0_f16_f32", arr_dmmv_q5_0_f16_f32_len[reduc], arr_dmmv_q5_0_f16_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {2*rm_stdq, 1, 1}, {wg_size_subgroup, 2*rm_stdq, i+1}, 1, true, use_subgroups, force_subgroup_size); @@ -5362,6 +5364,7 @@ static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { ggml_vk_create_pipeline(device, device->pipeline_dequant[GGML_TYPE_F32 ], "f32_to_f16", dequant_f32_len, dequant_f32_data, "main", 2, 5 * sizeof(uint32_t), {256 * 16, 1, 1}, {}, 1); ggml_vk_create_pipeline(device, device->pipeline_dequant[GGML_TYPE_Q1_0], "dequant_q1_0", dequant_q1_0_len, dequant_q1_0_data, "main", 2, 5 * sizeof(uint32_t), {256 * 8, 1, 1}, {}, 1); ggml_vk_create_pipeline(device, device->pipeline_dequant[GGML_TYPE_Q2_0], "dequant_q2_0", dequant_q2_0_len, dequant_q2_0_data, "main", 2, 5 * sizeof(uint32_t), {256 * 4, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_dequant[GGML_TYPE_DT3 ], "dequant_dt3", dequant_dt3_len, dequant_dt3_data, "main", 2, 5 * sizeof(uint32_t), {256 * 4, 1, 1}, {}, 1); ggml_vk_create_pipeline(device, device->pipeline_dequant[GGML_TYPE_Q4_0], "dequant_q4_0", dequant_q4_0_len, dequant_q4_0_data, "main", 2, 5 * sizeof(uint32_t), {256 * 16, 1, 1}, {}, 1); ggml_vk_create_pipeline(device, device->pipeline_dequant[GGML_TYPE_Q4_1], "dequant_q4_1", dequant_q4_1_len, dequant_q4_1_data, "main", 2, 5 * sizeof(uint32_t), {256 * 16, 1, 1}, {}, 1); ggml_vk_create_pipeline(device, device->pipeline_dequant[GGML_TYPE_Q5_0], "dequant_q5_0", dequant_q5_0_len, dequant_q5_0_data, "main", 2, 5 * sizeof(uint32_t), {256 * 16, 1, 1}, {}, 1); @@ -5390,6 +5393,7 @@ static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { ggml_vk_create_pipeline(device, device->pipeline_get_rows[GGML_TYPE_BF16], "get_rows_bf16", get_rows_bf16_len, get_rows_bf16_data, "main", 3, sizeof(vk_op_binary_push_constants), { 512, 1, 1}, {}, 1); ggml_vk_create_pipeline(device, device->pipeline_get_rows[GGML_TYPE_Q1_0], "get_rows_q1_0", get_rows_q1_0_len, get_rows_q1_0_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); ggml_vk_create_pipeline(device, device->pipeline_get_rows[GGML_TYPE_Q2_0], "get_rows_q2_0", get_rows_q2_0_len, get_rows_q2_0_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_get_rows[GGML_TYPE_DT3 ], "get_rows_dt3", get_rows_dt3_len, get_rows_dt3_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); ggml_vk_create_pipeline(device, device->pipeline_get_rows[GGML_TYPE_Q4_0], "get_rows_q4_0", get_rows_q4_0_len, get_rows_q4_0_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); ggml_vk_create_pipeline(device, device->pipeline_get_rows[GGML_TYPE_Q4_1], "get_rows_q4_1", get_rows_q4_1_len, get_rows_q4_1_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); ggml_vk_create_pipeline(device, device->pipeline_get_rows[GGML_TYPE_Q5_0], "get_rows_q5_0", get_rows_q5_0_len, get_rows_q5_0_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); @@ -5418,6 +5422,7 @@ static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { ggml_vk_create_pipeline(device, device->pipeline_get_rows_f32[GGML_TYPE_BF16], "get_rows_bf16_f32", get_rows_bf16_f32_len, get_rows_bf16_f32_data, "main", 3, sizeof(vk_op_binary_push_constants), { 512, 1, 1}, {}, 1); ggml_vk_create_pipeline(device, device->pipeline_get_rows_f32[GGML_TYPE_Q1_0], "get_rows_q1_0_f32", get_rows_q1_0_f32_len, get_rows_q1_0_f32_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); ggml_vk_create_pipeline(device, device->pipeline_get_rows_f32[GGML_TYPE_Q2_0], "get_rows_q2_0_f32", get_rows_q2_0_f32_len, get_rows_q2_0_f32_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_get_rows_f32[GGML_TYPE_DT3 ], "get_rows_dt3_f32", get_rows_dt3_f32_len, get_rows_dt3_f32_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); ggml_vk_create_pipeline(device, device->pipeline_get_rows_f32[GGML_TYPE_Q4_0], "get_rows_q4_0_f32", get_rows_q4_0_f32_len, get_rows_q4_0_f32_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); ggml_vk_create_pipeline(device, device->pipeline_get_rows_f32[GGML_TYPE_Q4_1], "get_rows_q4_1_f32", get_rows_q4_1_f32_len, get_rows_q4_1_f32_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); ggml_vk_create_pipeline(device, device->pipeline_get_rows_f32[GGML_TYPE_Q5_0], "get_rows_q5_0_f32", get_rows_q5_0_f32_len, get_rows_q5_0_f32_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); @@ -7617,6 +7622,7 @@ static vk_pipeline ggml_vk_get_to_fp16(ggml_backend_vk_context * ctx, ggml_type case GGML_TYPE_F32: case GGML_TYPE_Q1_0: case GGML_TYPE_Q2_0: + case GGML_TYPE_DT3: case GGML_TYPE_Q4_0: case GGML_TYPE_Q4_1: case GGML_TYPE_Q5_0: @@ -7760,6 +7766,7 @@ static vk_pipeline ggml_vk_get_dequantize_mul_mat_vec(ggml_backend_vk_context * case GGML_TYPE_BF16: case GGML_TYPE_Q1_0: case GGML_TYPE_Q2_0: + case GGML_TYPE_DT3: case GGML_TYPE_Q4_0: case GGML_TYPE_Q4_1: case GGML_TYPE_Q5_0: @@ -17987,6 +17994,13 @@ static bool ggml_backend_vk_device_supports_op(ggml_backend_dev_t dev, const ggm } } switch (src0_type) { + case GGML_TYPE_DT3: + // DT3 has dequant, get_rows and scalar mul_mat_vec shaders only: + // mul_mat_id, coopmat and MMQ are intentionally not implemented + if (op->op == GGML_OP_MUL_MAT_ID) { + return false; + } + break; case GGML_TYPE_F32: case GGML_TYPE_F16: case GGML_TYPE_BF16: @@ -18097,6 +18111,7 @@ static bool ggml_backend_vk_device_supports_op(ggml_backend_dev_t dev, const ggm case GGML_TYPE_BF16: case GGML_TYPE_Q1_0: case GGML_TYPE_Q2_0: + case GGML_TYPE_DT3: case GGML_TYPE_Q4_0: case GGML_TYPE_Q4_1: case GGML_TYPE_Q5_0: diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/dequant_dt3.comp b/ggml/src/ggml-vulkan/vulkan-shaders/dequant_dt3.comp new file mode 100644 index 000000000..a6c53baa3 --- /dev/null +++ b/ggml/src/ggml-vulkan/vulkan-shaders/dequant_dt3.comp @@ -0,0 +1,47 @@ +#version 450 + +#include "dequant_head.glsl" + +layout(local_size_x = 256, local_size_y = 1, local_size_z = 1) in; + +layout (binding = 0) readonly buffer A {block_dt3 data_a[];}; +layout (binding = 1) writeonly buffer D {D_TYPE data_b[];}; + +// Eight blocks per workgroup, 32 threads per block. Threads 0..23 decode one +// qs byte of each plane (5 trits in base 3), threads 24..25 decode one qh +// byte of each plane (4 trits — the 5th base-3 digit of a qh byte is packing +// padding that always decodes to -1, so it must not be read), threads 26..31 +// idle. +void main() { + const uint ib = gl_WorkGroupID.x * 8 + gl_LocalInvocationID.x / 32; + const uint il = gl_LocalInvocationID.x % 32; + + if (ib >= p.nel / 128 || il >= 26) { + return; + } + + const float d1 = float(data_a[ib].d[0]); + const float d2 = float(data_a[ib].d[1]); + + const uint b_idx = ib * 128; + + // element covered by the first digit, distance between consecutive + // digits, and number of digits stored in this byte + const bool is_qh = il >= 24; + const uint e0 = is_qh ? 120 + (il - 24) : (il < 16 ? il : 80 + (il - 16)); + const uint stride = is_qh ? 2 : (il < 16 ? 16 : 8); + const uint digits = is_qh ? 4 : 5; + + uint q1 = is_qh ? uint(data_a[ib].qh[il - 24]) : uint(data_a[ib].qs[il]); + uint q2 = is_qh ? uint(data_a[ib].qh[2 + il - 24]) : uint(data_a[ib].qs[24 + il]); + + // decode each byte once: take the top base-3 digit with (q*3) >> 8, then + // shift it out with q <- (q*3) mod 256 + for (uint n = 0; n < digits; ++n) { + const float t1 = float(int((q1 * 3) >> 8) - 1); + const float t2 = float(int((q2 * 3) >> 8) - 1); + data_b[b_idx + e0 + n*stride] = D_TYPE(d1*t1 + d2*t2); + q1 = (q1 * 3) & 0xFF; + q2 = (q2 * 3) & 0xFF; + } +} diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs.glsl b/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs.glsl index d902ff3a6..e8ca60a99 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs.glsl +++ b/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs.glsl @@ -154,6 +154,49 @@ vec4 dequantize4(uint ib, uint iqs, uint a_offset) { } #endif +#if defined(DATA_A_DT3) +// Dual-plane ternary: element iqs of plane p sits in a base-3 packed byte. +// Elements 0..79 use qs[m], m = iqs % 16, digit n = iqs / 16; elements +// 80..119 use qs[16 + m], m = (iqs - 80) % 8, digit n = (iqs - 80) / 8; +// elements 120..127 use qh[j], j = iqs % 2, digit n = (iqs - 120) / 2. +// A qh byte holds only 4 trits: its 5th base-3 digit is packing padding that +// always decodes to -1, never to 0, so it must not be read. +// The decode multiplies the byte by 3^n modulo 256 and takes the top digit. +float dt3_get_trit(uint ib, uint p, uint iqs, uint a_offset) { + const uint pow3[5] = {1, 3, 9, 27, 81}; + uint b; + uint n; + if (iqs < 80) { + b = uint(data_a[a_offset + ib].qs[p*24 + (iqs & 15)]); + n = iqs >> 4; + } else if (iqs < 120) { + b = uint(data_a[a_offset + ib].qs[p*24 + 16 + ((iqs - 80) & 7)]); + n = (iqs - 80) >> 3; + } else { + b = uint(data_a[a_offset + ib].qh[p*2 + (iqs & 1)]); + n = (iqs - 120) >> 1; + } + const uint q = (b * pow3[n]) & 0xFF; + return float(int((q * 3) >> 8) - 1); +} +// w = d1*t1 + d2*t2; both products are exact (t in {-1,0,+1}), so the sum has +// a single float rounding and matches the CPU reference bit by bit +vec2 dequantize(uint ib, uint iqs, uint a_offset) { + const float d1 = float(data_a[a_offset + ib].d[0]); + const float d2 = float(data_a[a_offset + ib].d[1]); + return vec2(d1*dt3_get_trit(ib, 0, iqs, a_offset) + d2*dt3_get_trit(ib, 1, iqs, a_offset), + d1*dt3_get_trit(ib, 0, iqs + 1, a_offset) + d2*dt3_get_trit(ib, 1, iqs + 1, a_offset)); +} +vec4 dequantize4(uint ib, uint iqs, uint a_offset) { + const float d1 = float(data_a[a_offset + ib].d[0]); + const float d2 = float(data_a[a_offset + ib].d[1]); + return vec4(d1*dt3_get_trit(ib, 0, iqs, a_offset) + d2*dt3_get_trit(ib, 1, iqs, a_offset), + d1*dt3_get_trit(ib, 0, iqs + 1, a_offset) + d2*dt3_get_trit(ib, 1, iqs + 1, a_offset), + d1*dt3_get_trit(ib, 0, iqs + 2, a_offset) + d2*dt3_get_trit(ib, 1, iqs + 2, a_offset), + d1*dt3_get_trit(ib, 0, iqs + 3, a_offset) + d2*dt3_get_trit(ib, 1, iqs + 3, a_offset)); +} +#endif + #if defined(DATA_A_IQ1_S) vec2 dequantize(uint ib, uint iqs, uint a_offset) { const uint ib32 = iqs / 32; @@ -571,6 +614,13 @@ vec2 get_dm(uint ib, uint a_offset) { } #endif +#if defined(DATA_A_DT3) +// the two scales are already applied inside dequantize/dequantize4 +vec2 get_dm(uint ib, uint a_offset) { + return vec2(1, 0); +} +#endif + #if defined(DATA_A_MXFP4) vec2 get_dm(uint ib, uint a_offset) { return vec2(e8m0_to_fp32(data_a[a_offset + ib].e), 0); diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/types.glsl b/ggml/src/ggml-vulkan/vulkan-shaders/types.glsl index 9616a26c7..9b4083c5d 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/types.glsl +++ b/ggml/src/ggml-vulkan/vulkan-shaders/types.glsl @@ -235,6 +235,27 @@ struct block_q2_0_packed16 #define DATA_A_QUANT_LEGACY #endif +#define QUANT_K_DT3 128 +#define QUANT_R_DT3 1 + +// Dual-plane ternary: w = d[0]*t1 + d[1]*t2 with trits in {-1,0,+1}. +// Per plane: 24 bytes with 5 trits each in base 3 (elements 0..119), then +// 2 bytes with 4 trits each (elements 120..127). Plane p uses qs[p*24..], +// qh[p*2..] and d[p]. +struct block_dt3 +{ + uint8_t qs[2*24]; + uint8_t qh[2*2]; + float16_t d[2]; +}; + +#if defined(DATA_A_DT3) +#define QUANT_K QUANT_K_DT3 +#define QUANT_R QUANT_R_DT3 +#define QUANT_AUXF 1 +#define A_TYPE block_dt3 +#endif + #define QUANT_K_Q8_1 32 #define QUANT_R_Q8_1 1 diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp b/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp index c93d6eece..419c93940 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp @@ -51,6 +51,7 @@ const std::vector type_names = { "f16", "q1_0", "q2_0", + "dt3", "q4_0", "q4_1", "q5_0", @@ -591,6 +592,12 @@ void matmul_shaders(bool fp16, MatMulIdType matmul_id_type, bool coopmat, bool c continue; } + // DT3 has no direct matmul shaders: mul_mat goes through dequant to + // f16 + f16 matmul, and coopmat/MMQ are intentionally not implemented + if (tname == "dt3") { + continue; + } + std::string data_a_key = "DATA_A_" + to_uppercase(tname); // For aligned matmul loads std::string load_vec_a = (coopmat2 || tname == "f32" || tname == "f16" || tname == "bf16") ? load_vec : load_vec_quant; @@ -758,9 +765,12 @@ void process_shaders() { } #endif - string_to_spv("mul_mat_vec_id_" + tname + "_f32_f32", shader, merge_maps(base_dict, {{"MUL_MAT_ID", "1"}, {data_a_key, "1"}, {"B_TYPE", "float"}, {"B_TYPEV2", "vec2"}, {"B_TYPEV4", "vec4"}, {"D_TYPE", "float"}})); - string_to_spv("mul_mat_vec_id_" + tname + "_f32_f32_subgroup", shader, merge_maps(base_dict, {{"MUL_MAT_ID", "1"}, {data_a_key, "1"}, {"B_TYPE", "float"}, {"B_TYPEV2", "vec2"}, {"B_TYPEV4", "vec4"}, {"D_TYPE", "float"}, {"USE_SUBGROUP_ADD", "1"}})); - string_to_spv("mul_mat_vec_id_" + tname + "_f32_f32_subgroup_no_shmem", shader, merge_maps(base_dict, {{"MUL_MAT_ID", "1"}, {data_a_key, "1"}, {"B_TYPE", "float"}, {"B_TYPEV2", "vec2"}, {"B_TYPEV4", "vec4"}, {"D_TYPE", "float"}, {"USE_SUBGROUP_ADD_NO_SHMEM", "1"}})); + // mul_mat_id is not implemented for DT3 (supports_op declines it) + if (tname != "dt3") { + string_to_spv("mul_mat_vec_id_" + tname + "_f32_f32", shader, merge_maps(base_dict, {{"MUL_MAT_ID", "1"}, {data_a_key, "1"}, {"B_TYPE", "float"}, {"B_TYPEV2", "vec2"}, {"B_TYPEV4", "vec4"}, {"D_TYPE", "float"}})); + string_to_spv("mul_mat_vec_id_" + tname + "_f32_f32_subgroup", shader, merge_maps(base_dict, {{"MUL_MAT_ID", "1"}, {data_a_key, "1"}, {"B_TYPE", "float"}, {"B_TYPEV2", "vec2"}, {"B_TYPEV4", "vec4"}, {"D_TYPE", "float"}, {"USE_SUBGROUP_ADD", "1"}})); + string_to_spv("mul_mat_vec_id_" + tname + "_f32_f32_subgroup_no_shmem", shader, merge_maps(base_dict, {{"MUL_MAT_ID", "1"}, {data_a_key, "1"}, {"B_TYPE", "float"}, {"B_TYPEV2", "vec2"}, {"B_TYPEV4", "vec4"}, {"D_TYPE", "float"}, {"USE_SUBGROUP_ADD_NO_SHMEM", "1"}})); + } // mul mat vec with integer dot product #if defined(GGML_VULKAN_INTEGER_DOT_GLSLC_SUPPORT) @@ -1254,7 +1264,8 @@ void write_output_files() { src << "const uint64_t arr_dmmv_" << tname << "_" << btype << "_f32_len[3] = {mul_mat_vec_" << tname << "_" << btype << "_f32_len, mul_mat_vec_" << tname << "_" << btype << "_f32_subgroup_len, mul_mat_vec_" << tname << "_" << btype << "_f32_subgroup_no_shmem_len};\n"; } - if (btype == "f16") { + if (btype == "f16" || tname == "dt3") { + // no mul_mat_vec_id shaders for DT3 continue; } hdr << "extern const void * arr_dmmv_id_" << tname << "_" << btype << "_f32_data[3];\n"; diff --git a/tests/test-dt3-gpu.cpp b/tests/test-dt3-gpu.cpp index 703024533..6a21a2278 100644 --- a/tests/test-dt3-gpu.cpp +++ b/tests/test-dt3-gpu.cpp @@ -33,8 +33,12 @@ // random bytes: every byte value 0..255 must decode identically on both // sides, including values >= 243 that never come out of the packer. // -// DT3 is implemented for CUDA and HIP only. Without one of those backends the -// test is skipped and succeeds — an unsupported backend is not a failure. +// DT3 is implemented for CUDA, HIP and Vulkan. Without one of those backends +// the test is skipped and succeeds — an unsupported backend is not a failure. +// On Vulkan the n <= 8 path is the scalar mul_mat_vec shader (fp32 dot on +// exactly decoded weights, not an integer dot), and the larger-n path is +// dequantization to fp16 + the f16 matmul pipeline; both are judged by the +// same gates as the CUDA MMVQ/GEMM paths. #include "ggml.h" #include "ggml-alloc.h" @@ -461,21 +465,25 @@ static void build_control_data(ggml_type type, std::vector & data, std: } int main(void) { - // Only CUDA and HIP (which reports itself as "ROCm") implement DT3. Any - // other GPU backend is skipped rather than failed: Vulkan and SYCL answer - // supports_op == false for DT3, which is the correct answer for them and + // Only CUDA, HIP (which reports itself as "ROCm") and Vulkan implement + // DT3. Any other GPU backend is skipped rather than failed: SYCL answers + // supports_op == false for DT3, which is the correct answer for it and // not a bug to report, and Metal answers true for almost any type but has // no DT3 shader, so it would die in pipeline compilation mid-test. Picking // the backend by name keeps this test honest on machines we do not have. ggml_backend_t backend = nullptr; for (size_t i = 0; i < ggml_backend_dev_count(); ++i) { ggml_backend_dev_t dev = ggml_backend_dev_get(i); - if (ggml_backend_dev_type(dev) != GGML_BACKEND_DEVICE_TYPE_GPU) { + // IGPU is a distinct device type from GPU: an integrated Vulkan device + // with unified memory reports as IGPU, and accepting only GPU silently + // skipped the very hardware this backend is for. + const auto dt = ggml_backend_dev_type(dev); + if (dt != GGML_BACKEND_DEVICE_TYPE_GPU && dt != GGML_BACKEND_DEVICE_TYPE_IGPU) { continue; } const char * name = ggml_backend_dev_name(dev); - if (strncmp(name, "CUDA", 4) != 0 && strncmp(name, "ROCm", 4) != 0) { - printf("skipping GPU backend %s: DT3 is only implemented for CUDA/HIP\n", name); + if (strncmp(name, "CUDA", 4) != 0 && strncmp(name, "ROCm", 4) != 0 && strncmp(name, "Vulkan", 6) != 0) { + printf("skipping GPU backend %s: DT3 is only implemented for CUDA/HIP/Vulkan\n", name); continue; } backend = ggml_backend_dev_init(dev, nullptr); @@ -483,7 +491,7 @@ int main(void) { break; } if (backend == nullptr) { - printf("no CUDA/HIP backend available, skipping\n"); + printf("no CUDA/HIP/Vulkan backend available, skipping\n"); return 0; }