From df03399b885831b2a1603b3abb0d8c156808e363 Mon Sep 17 00:00:00 2001 From: shaofeiqi Date: Thu, 10 Sep 2026 11:25:40 -0700 Subject: [PATCH] opencl: add A8 Q4_0 mm binary kernel support (#28268) --- ggml/src/ggml-opencl/CMakeLists.txt | 1 + ggml/src/ggml-opencl/ggml-opencl.cpp | 274 +++++++++++++++++- .../gemv_noshuffle_q4_0_f32_32b_trans.cl | 137 +++++++++ 3 files changed, 402 insertions(+), 10 deletions(-) create mode 100644 ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_0_f32_32b_trans.cl diff --git a/ggml/src/ggml-opencl/CMakeLists.txt b/ggml/src/ggml-opencl/CMakeLists.txt index 37e565ef4f..716577bb77 100644 --- a/ggml/src/ggml-opencl/CMakeLists.txt +++ b/ggml/src/ggml-opencl/CMakeLists.txt @@ -170,6 +170,7 @@ set(GGML_OPENCL_KERNELS gemv_noshuffle_q4_0_f32 gemv_noshuffle_q4_0_f32_spec gemm_noshuffle_q4_0_f32 + gemv_noshuffle_q4_0_f32_32b_trans gemv_noshuffle_q4_1_f32 gemm_noshuffle_q4_1_f32 gemv_noshuffle_q5_0_f32 diff --git a/ggml/src/ggml-opencl/ggml-opencl.cpp b/ggml/src/ggml-opencl/ggml-opencl.cpp index 3002835e8a..231be2cf3a 100644 --- a/ggml/src/ggml-opencl/ggml-opencl.cpp +++ b/ggml/src/ggml-opencl/ggml-opencl.cpp @@ -1160,6 +1160,8 @@ struct ggml_backend_opencl_context { cl_kernel kernel_gemm_noshuffle_q4_0_f32; cl_kernel kernel_gemv_noshuffle_q4_0_f32; cl_kernel kernel_gemv_noshuffle_q4_0_f32_mc3; // multi-column (N=3) verify GEMV (spec/MTP) + cl_kernel kernel_gemm_noshuffle_q4_0_f32_32b_trans_ila_a8_bin; + cl_kernel kernel_gemv_noshuffle_q4_0_f32_32b_trans; cl_kernel kernel_gemv_noshuffle_q4_0_f32_4096_1_11008; cl_kernel kernel_gemv_noshuffle_q4_0_f32_4096_1_4096; cl_kernel kernel_gemv_noshuffle_q4_0_f32_11008_1_4096; @@ -3787,6 +3789,43 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) { GGML_LOG_CONT("."); } + backend_ctx->kernel_gemv_noshuffle_q4_0_f32_32b_trans = nullptr; + backend_ctx->kernel_gemm_noshuffle_q4_0_f32_32b_trans_ila_a8_bin = nullptr; + if (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E) { + { + std::string opts = std::string("-cl-std=") + opencl_c_std + + " -cl-mad-enable " + " -DSIMDGROUP_WIDTH=" + + std::to_string(backend_ctx->adreno_wave_size); +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemv_noshuffle_q4_0_f32_32b_trans.cl.h" + }; +#else + const std::string kernel_src = read_file("gemv_noshuffle_q4_0_f32_32b_trans.cl"); +#endif + cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), opts); + CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q4_0_f32_32b_trans = + clCreateKernel(prog, "kernel_gemv_noshuffle_q4_0_f32_32b_trans", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } + + if (use_adreno_bin_kernels(backend_ctx)) { + size_t bin_size = 0; + const char * kernel_bin = (const char *)backend_ctx->get_adreno_bin_kernel("gemm_noshuffle_q4_0_f32_32b_trans_ila_a8", &bin_size); + if (kernel_bin && bin_size > 0) { + cl_program bin_prog = + build_program_from_binary(backend_ctx->context, backend_ctx->device, kernel_bin, "", bin_size); + + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q4_0_f32_32b_trans_ila_a8_bin = + clCreateKernel(bin_prog, "kernel_gemm_noshuffle_q4_0_f32_32b_trans_ila_a8", &err), err)); + CL_CHECK(clReleaseProgram(bin_prog)); + GGML_LOG_CONT("."); + } + } + } + // gemm_noshuffle_q4_1_f32 { #ifdef GGML_OPENCL_EMBED_KERNELS @@ -6725,11 +6764,10 @@ struct ggml_tensor_extra_cl_q4_0 { CL_CHECK(clReleaseMemObject(q_img)); q_img = nullptr; } - // Currently, q_img and d_img are only initialized when SMALL_ALLOC is - // enabled. They point to the images in ggml_backend_opencl_buffer_context. - // So, there is no need to release them here. - // TODO: initialize them for non SMALL_PATH path, or remove them. - d_img = nullptr; + if (d_img != nullptr) { + CL_CHECK(clReleaseMemObject(d_img)); + d_img = nullptr; + } size_q = 0; size_d = 0; } @@ -8311,6 +8349,20 @@ inline bool enable_adreno_trans_weight_q5_K(const ggml_backend_opencl_context *b qh_img_width <= backend_ctx->image_max_buffer_size; } +inline bool use_q4_0_ila_kernels(const ggml_backend_opencl_context *backend_ctx, const ggml_tensor *tensor) { +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + if (!backend_ctx->kernel_gemv_noshuffle_q4_0_f32_32b_trans || + !backend_ctx->kernel_gemm_noshuffle_q4_0_f32_32b_trans_ila_a8_bin) { + return false; + } + return (tensor->ne[0] % 32 == 0) && (tensor->ne[1] % 64 == 0); +#else + GGML_UNUSED(backend_ctx); + GGML_UNUSED(tensor); + return false; +#endif +} + // The flat-GEMV large-m escape is OPT-IN (GGML_OPENCL_FLAT_LARGE_M=1) because it // is SLOWER than the route it replaces, not because it is unsafe. It was first // parked on the theory that it out-of-bounds-writes at vocab-scale shapes; that @@ -9573,10 +9625,34 @@ static void ggml_backend_opencl_buffer_set_tensor(ggml_backend_buffer_t buffer, GGML_ASSERT(K % 32 == 0); - // Transpose q as ushort - transpose_2d_as_16b(backend_ctx, extra->q, extra->q, size_q, K/4, M); - // Transpose d as ushort - transpose_2d_as_16b(backend_ctx, extra->d, extra->d, size_d, K/32, M); + if (use_q4_0_ila_kernels(backend_ctx, tensor)) { + cl_int err; + cl_image_format wimg_fmt; + cl_image_desc wimg_desc; + + // transpose quants as 32-bit words (M-first) + GGML_ASSERT(M % 64 == 0); + transpose_2d_as_32b(backend_ctx, extra->q, extra->q, size_q, K / 8, M); + transpose_2d_as_16b(backend_ctx, extra->d, extra->d, size_d, K / 32, M); + + wimg_fmt = { CL_R, CL_UNSIGNED_INT32 }; + memset(&wimg_desc, 0, sizeof(wimg_desc)); + wimg_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + wimg_desc.image_width = (size_t)M * K / 8; + wimg_desc.buffer = extra->q; + CL_CHECK((extra->q_img = clCreateImage(context, CL_MEM_READ_ONLY, &wimg_fmt, &wimg_desc, NULL, &err), err)); + + wimg_fmt = { CL_R, CL_HALF_FLOAT }; + memset(&wimg_desc, 0, sizeof(wimg_desc)); + wimg_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + wimg_desc.image_width = (size_t)M * K / 32; + wimg_desc.buffer = extra->d; + CL_CHECK((extra->d_img = clCreateImage(context, CL_MEM_READ_ONLY, &wimg_fmt, &wimg_desc, NULL, &err), err)); + } else { + // Transpose q and d as ushort + transpose_2d_as_16b(backend_ctx, extra->q, extra->q, size_q, K/4, M); + transpose_2d_as_16b(backend_ctx, extra->d, extra->d, size_d, K/32, M); + } } #endif // GGML_OPENCL_USE_ADRENO_KERNELS return; @@ -11104,7 +11180,11 @@ static void ggml_backend_opencl_buffer_get_tensor(ggml_backend_buffer_t buffer, buf_trans_d.allocate(backend_ctx->context, size_d); buf_unpacked.allocate(backend_ctx->context, ggml_nbytes(tensor)); - transpose_2d_as_16b(backend_ctx, extra->q, buf_trans_q.buffer, size_q, M, K/4); + if (use_q4_0_ila_kernels(backend_ctx, tensor)) { + transpose_2d_as_32b(backend_ctx, extra->q, buf_trans_q.buffer, size_q, M, K / 8); + } else { + transpose_2d_as_16b(backend_ctx, extra->q, buf_trans_q.buffer, size_q, M, K / 4); + } transpose_2d_as_16b(backend_ctx, extra->d, buf_trans_d.buffer, size_d, M, K/32); cl_uchar mask_0F = 0x0F; @@ -18347,6 +18427,166 @@ static void ggml_cl_mul_mat_q1_0_f32_adreno(ggml_backend_t backend, const ggml_t #endif } +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS +static void ggml_cl_mul_mat_q4_0_f32_adreno_ila(ggml_backend_t backend, const ggml_tensor * src0, + const ggml_tensor * src1, ggml_tensor * dst) { + GGML_ASSERT(src0); + GGML_ASSERT(src0->extra); + GGML_ASSERT(src1); + GGML_ASSERT(src1->extra); + GGML_ASSERT(dst); + GGML_ASSERT(dst->extra); + + ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; + + ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; + ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; + ggml_tensor_extra_cl_q4_0 * extra0_q4_0 = (ggml_tensor_extra_cl_q4_0 *)src0->extra; + + cl_ulong offset1 = extra1->offset + src1->view_offs; + cl_ulong offsetd = extrad->offset + dst->view_offs; + + const int ne00 = src0->ne[0]; + const int ne01 = src0->ne[1]; + + const int ne1 = dst->ne[1]; + + GGML_ASSERT(ne00 % ggml_blck_size(src0->type) == 0); + + cl_context context = backend_ctx->context; + cl_kernel kernel; + + cl_int err; + cl_image_format img_fmt; + cl_image_desc img_desc; + cl_buffer_region region; + + int M = ne01; + int N = ne1; + int K = ne00; + + if (ne1 == 1) { + cl_mem b_sub_buf = nullptr; + cl_mem b_img = nullptr; + + region.origin = offset1; + region.size = (size_t)K * N * sizeof(float); + CL_CHECK((b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + + img_fmt = { CL_RGBA, CL_FLOAT }; + memset(&img_desc, 0, sizeof(img_desc)); + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = (size_t)K * N / 4; + img_desc.buffer = b_sub_buf; + CL_CHECK((b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); + + kernel = backend_ctx->kernel_gemv_noshuffle_q4_0_f32_32b_trans; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q4_0->q_img)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q4_0->d)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &b_img)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_int), &K)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_int), &M)); + + size_t wavesize = backend_ctx->adreno_wave_size; + size_t local_work_size[3] = { wavesize, 4, 1 }; + size_t global_work_size[3] = { (size_t)CEIL_DIV(M, 64) * 64, 4, 1 }; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + + CL_CHECK(clReleaseMemObject(b_sub_buf)); + CL_CHECK(clReleaseMemObject(b_img)); + } else { + const int gemm_tile_n = 64; + int N_pad = (N + gemm_tile_n - 1) & ~(gemm_tile_n - 1); + + cl_mem a_img = extra0_q4_0->q_img; + cl_mem s_img = extra0_q4_0->d_img; + GGML_ASSERT(a_img && s_img && "ILA Q4_0 weight images missing; set_tensor should have built them"); + + // Pad B through a zero-filled scratch buffer when N needs + // padding, since the GEMM kernel always reads a full N-tile. + const bool need_pad = N_pad > N; + cl_mem b_sub_buf = nullptr; + cl_mem b_padded = nullptr; + if (need_pad) { + CL_CHECK((b_padded = clCreateBuffer(context, CL_MEM_READ_WRITE, + (size_t)K * N_pad * sizeof(float), NULL, &err), err)); + const float zero = 0.0f; + CL_CHECK(clEnqueueFillBuffer(backend_ctx->queue, b_padded, &zero, sizeof(zero), + 0, (size_t)K * N_pad * sizeof(float), 0, NULL, NULL)); + CL_CHECK(clEnqueueCopyBuffer(backend_ctx->queue, extra1->data_device, b_padded, + offset1, 0, (size_t)K * N * sizeof(float), 0, NULL, NULL)); + } else { + region.origin = offset1; + region.size = (size_t)K * N * sizeof(float); + CL_CHECK((b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + } + + img_fmt = { CL_R, CL_FLOAT }; + memset(&img_desc, 0, sizeof(img_desc)); + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = need_pad ? (size_t)K * N_pad : (size_t)K * N; + img_desc.buffer = need_pad ? b_padded : b_sub_buf; + cl_mem b_img; + CL_CHECK((b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); + + region.origin = offsetd; + region.size = (size_t)M * N * sizeof(float); + cl_mem d_sub_buf; + CL_CHECK((d_sub_buf = clCreateSubBuffer(extrad->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + + img_fmt = { CL_R, CL_FLOAT }; + memset(&img_desc, 0, sizeof(img_desc)); + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = (size_t)M * N; + img_desc.buffer = d_sub_buf; + cl_mem d_img; + CL_CHECK((d_img = clCreateImage(context, CL_MEM_WRITE_ONLY, &img_fmt, &img_desc, NULL, &err), err)); + + int line_stride_matrix_A_in_bytes = M * 4; + int line_stride_matrix_S_in_bytes = M * 2; + int line_stride_matrix_B_in_bytes = K * 4; + int line_stride_matrix_C_in_bytes = M * 4; + + int c_offset_for_kernel = 0; + int b_offset_for_kernel = 0; + + kernel = backend_ctx->kernel_gemm_noshuffle_q4_0_f32_32b_trans_ila_a8_bin; + + cl_uint k_arg = 0; + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &a_img)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &s_img)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &b_img)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &b_offset_for_kernel)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &d_img)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &c_offset_for_kernel)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &K)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &line_stride_matrix_A_in_bytes)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &line_stride_matrix_S_in_bytes)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &line_stride_matrix_B_in_bytes)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &line_stride_matrix_C_in_bytes)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &M)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &N)); + + size_t local_work_size[3] = { 64, 2, 2 }; + size_t m_tiles = (size_t)CEIL_DIV(M, 64); + size_t global_work_size[3] = { 64, m_tiles, (size_t)CEIL_DIV(N_pad, gemm_tile_n) }; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + + CL_CHECK(clReleaseMemObject(b_img)); + if (b_sub_buf) { + CL_CHECK(clReleaseMemObject(b_sub_buf)); + } + if (b_padded) { + CL_CHECK(clReleaseMemObject(b_padded)); + } + CL_CHECK(clReleaseMemObject(d_img)); + CL_CHECK(clReleaseMemObject(d_sub_buf)); + } +} +#endif // GGML_OPENCL_USE_ADRENO_KERNELS + static void ggml_cl_mul_mat_q4_0_f32_adreno(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { #ifdef GGML_OPENCL_USE_ADRENO_KERNELS GGML_ASSERT(src0); @@ -18399,6 +18639,20 @@ static void ggml_cl_mul_mat_q4_0_f32_adreno(ggml_backend_t backend, const ggml_t static const bool q40_mc3 = (getenv("GGML_OPENCL_Q40_MC3") != nullptr); const bool use_q40_mc3 = q40_mc3 && (ne1 >= 2 && ne1 <= 4) && (ne01 < 32768); + const bool use_ila = use_q4_0_ila_kernels(backend_ctx, src0); + + if (use_ila) { + if (use_q40_mc3) { + static bool warned = false; + if (!warned) { + GGML_LOG_WARN("ggml_opencl: GGML_OPENCL_Q40_MC3 is bypassed by Q4_0 binary kernels\n"); + warned = true; + } + } + ggml_cl_mul_mat_q4_0_f32_adreno_ila(backend, src0, src1, dst); + return; + } + if (ne1 == 1 || use_q40_mc3) { cl_mem q_img = nullptr; cl_mem b_sub_buf = nullptr; diff --git a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_0_f32_32b_trans.cl b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_0_f32_32b_trans.cl new file mode 100644 index 0000000000..565285f4b2 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_0_f32_32b_trans.cl @@ -0,0 +1,137 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#pragma OPENCL EXTENSION cl_khr_subgroups : enable + +#ifdef cl_qcom_reqd_sub_group_size +#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable +#define ADRENO_GPU 1 +#define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size("half"))) +#endif + +#define QK4_0 32 +#define N_SIMDGROUP 4 + +#define dequantizeBlockAccum_ila_1row_hi(total_sum, bits4, scale, y) \ + float shared_y; \ + shared_y = sub_group_broadcast(y.s0, 0); \ + total_sum += ((bits4.s0 & 0x000F) - 8) * scale * shared_y; \ + shared_y = sub_group_broadcast(y.s1, 0); \ + total_sum += (((bits4.s0 & 0x00F0) >> 4) - 8) * scale * shared_y; \ + shared_y = sub_group_broadcast(y.s2, 0); \ + total_sum += (((bits4.s0 & 0x0F00) >> 8) - 8) * scale * shared_y; \ + shared_y = sub_group_broadcast(y.s3, 0); \ + total_sum += (((bits4.s0 & 0xF000) >> 12) - 8) * scale * shared_y; \ + shared_y = sub_group_broadcast(y.s4, 0); \ + total_sum += ((bits4.s1 & 0x000F) - 8) * scale * shared_y; \ + shared_y = sub_group_broadcast(y.s5, 0); \ + total_sum += (((bits4.s1 & 0x00F0) >> 4) - 8) * scale * shared_y; \ + shared_y = sub_group_broadcast(y.s6, 0); \ + total_sum += (((bits4.s1 & 0x0F00) >> 8) - 8) * scale * shared_y; \ + shared_y = sub_group_broadcast(y.s7, 0); \ + total_sum += (((bits4.s1 & 0xF000) >> 12) - 8) * scale * shared_y; \ + shared_y = sub_group_broadcast(y.s0, 1); \ + total_sum += ((bits4.s2 & 0x000F) - 8) * scale * shared_y; \ + shared_y = sub_group_broadcast(y.s1, 1); \ + total_sum += (((bits4.s2 & 0x00F0) >> 4) - 8) * scale * shared_y; \ + shared_y = sub_group_broadcast(y.s2, 1); \ + total_sum += (((bits4.s2 & 0x0F00) >> 8) - 8) * scale * shared_y; \ + shared_y = sub_group_broadcast(y.s3, 1); \ + total_sum += (((bits4.s2 & 0xF000) >> 12) - 8) * scale * shared_y; \ + shared_y = sub_group_broadcast(y.s4, 1); \ + total_sum += ((bits4.s3 & 0x000F) - 8) * scale * shared_y; \ + shared_y = sub_group_broadcast(y.s5, 1); \ + total_sum += (((bits4.s3 & 0x00F0) >> 4) - 8) * scale * shared_y; \ + shared_y = sub_group_broadcast(y.s6, 1); \ + total_sum += (((bits4.s3 & 0x0F00) >> 8) - 8) * scale * shared_y; \ + shared_y = sub_group_broadcast(y.s7, 1); \ + total_sum += (((bits4.s3 & 0xF000) >> 12) - 8) * scale * shared_y; + +#define dequantizeBlockAccum_ila_1row_lo(total_sum, bits4, scale, y) \ + shared_y = sub_group_broadcast(y.s0, 2); \ + total_sum += ((bits4.s4 & 0x000F) - 8) * scale * shared_y; \ + shared_y = sub_group_broadcast(y.s1, 2); \ + total_sum += (((bits4.s4 & 0x00F0) >> 4) - 8) * scale * shared_y; \ + shared_y = sub_group_broadcast(y.s2, 2); \ + total_sum += (((bits4.s4 & 0x0F00) >> 8) - 8) * scale * shared_y; \ + shared_y = sub_group_broadcast(y.s3, 2); \ + total_sum += (((bits4.s4 & 0xF000) >> 12) - 8) * scale * shared_y; \ + shared_y = sub_group_broadcast(y.s4, 2); \ + total_sum += ((bits4.s5 & 0x000F) - 8) * scale * shared_y; \ + shared_y = sub_group_broadcast(y.s5, 2); \ + total_sum += (((bits4.s5 & 0x00F0) >> 4) - 8) * scale * shared_y; \ + shared_y = sub_group_broadcast(y.s6, 2); \ + total_sum += (((bits4.s5 & 0x0F00) >> 8) - 8) * scale * shared_y; \ + shared_y = sub_group_broadcast(y.s7, 2); \ + total_sum += (((bits4.s5 & 0xF000) >> 12) - 8) * scale * shared_y; \ + shared_y = sub_group_broadcast(y.s0, 3); \ + total_sum += ((bits4.s6 & 0x000F) - 8) * scale * shared_y; \ + shared_y = sub_group_broadcast(y.s1, 3); \ + total_sum += (((bits4.s6 & 0x00F0) >> 4) - 8) * scale * shared_y; \ + shared_y = sub_group_broadcast(y.s2, 3); \ + total_sum += (((bits4.s6 & 0x0F00) >> 8) - 8) * scale * shared_y; \ + shared_y = sub_group_broadcast(y.s3, 3); \ + total_sum += (((bits4.s6 & 0xF000) >> 12) - 8) * scale * shared_y; \ + shared_y = sub_group_broadcast(y.s4, 3); \ + total_sum += ((bits4.s7 & 0x000F) - 8) * scale * shared_y; \ + shared_y = sub_group_broadcast(y.s5, 3); \ + total_sum += (((bits4.s7 & 0x00F0) >> 4) - 8) * scale * shared_y; \ + shared_y = sub_group_broadcast(y.s6, 3); \ + total_sum += (((bits4.s7 & 0x0F00) >> 8) - 8) * scale * shared_y; \ + shared_y = sub_group_broadcast(y.s7, 3); \ + total_sum += (((bits4.s7 & 0xF000) >> 12) - 8) * scale * shared_y; + + +#ifdef ADRENO_GPU +REQD_SUBGROUP_SIZE_64 +#endif +__kernel void kernel_gemv_noshuffle_q4_0_f32_32b_trans( + __read_only image1d_buffer_t src0_q, + global half * src0_d, + __read_only image1d_buffer_t src1, + global float * dst, + ulong offsetd, + int ne00, + int ne01) +{ + uint groupId = get_local_id(1); + uint gid = get_global_id(0); + ushort slid = get_sub_group_local_id(); + + uint K = ne00; + uint M = ne01; + + __private uint4 regA; + __private half regS; + __private float8 regB; + __private float totalSum = 0.0f; + + for (uint k = groupId; k < (K / QK4_0); k += N_SIMDGROUP) { + regS = src0_d[k * M + gid]; + if (slid < 4) { + regB.s0123 = read_imagef(src1, (slid * 2 + k * 8)); + regB.s4567 = read_imagef(src1, (1 + slid * 2 + k * 8)); + } + regA.s0 = read_imageui(src0_q, ((k * 4 + 0) * M + gid)).x; + regA.s1 = read_imageui(src0_q, ((k * 4 + 1) * M + gid)).x; + regA.s2 = read_imageui(src0_q, ((k * 4 + 2) * M + gid)).x; + regA.s3 = read_imageui(src0_q, ((k * 4 + 3) * M + gid)).x; + + dequantizeBlockAccum_ila_1row_hi(totalSum, as_ushort8(regA), regS, regB); + dequantizeBlockAccum_ila_1row_lo(totalSum, as_ushort8(regA), regS, regB); + } + + __local float reduceLM[SIMDGROUP_WIDTH * 3]; + if (groupId == 1) reduceLM[SIMDGROUP_WIDTH * 0 + slid] = totalSum; + if (groupId == 2) reduceLM[SIMDGROUP_WIDTH * 1 + slid] = totalSum; + if (groupId == 3) reduceLM[SIMDGROUP_WIDTH * 2 + slid] = totalSum; + barrier(CLK_LOCAL_MEM_FENCE); + if (groupId == 0) totalSum += reduceLM[SIMDGROUP_WIDTH * 0 + slid]; + if (groupId == 0) totalSum += reduceLM[SIMDGROUP_WIDTH * 1 + slid]; + if (groupId == 0) totalSum += reduceLM[SIMDGROUP_WIDTH * 2 + slid]; + + if (groupId == 0) { + dst = (global float*)((global char*)dst + offsetd); + if (gid < M) { + dst[gid] = totalSum; + } + } +}