mirror of
https://github.com/ggml-org/llama.cpp.git
synced 2026-07-30 16:18:02 +02:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Georgi Gerganov
|
9ebfc3a8cf | ||
|
Georgi Gerganov
|
6a4c3357c8 | ||
|
Pasha KhosraviandGitHub
|
9b2a088819 | ||
|
timkhronosandGitHub
|
b2f221684f | ||
|
|
d0bfb19812 | ||
|
Aleksander GrygierandGitHub
|
21a5f5b7f9 | ||
|
Jonathan ClohessyandGitHub
|
32703b42d6 | ||
|
|
a6a77bc48d | ||
|
|
64d528be72 | ||
|
Aleksander GrygierandGitHub
|
3018a11e79 | ||
|
Xuan-Son NguyenandGitHub
|
afeebe103b | ||
|
|
caa596ab3f | ||
|
TitaniumtownandGitHub
|
11b068d066 | ||
|
Alessandro de Oliveira Faria (A.K.A.CABELO)andGitHub
|
e2f59ed71d | ||
|
Georgi GerganovandGitHub
|
992c325323 | ||
|
Kaben NanlohyandGitHub
|
e1af89a681 |
@@ -294,10 +294,6 @@ struct common_params_sampling {
|
||||
|
||||
bool backend_sampling = false;
|
||||
|
||||
bool has_logit_bias() const {
|
||||
return !logit_bias.empty();
|
||||
}
|
||||
|
||||
// print the parameters into a string
|
||||
std::string print() const;
|
||||
};
|
||||
|
||||
+13
-2
@@ -310,8 +310,19 @@ struct common_sampler * common_sampler_init(const struct llama_model * model, st
|
||||
}
|
||||
}
|
||||
|
||||
if (params.has_logit_bias()) {
|
||||
samplers.push_back(llama_sampler_init_logit_bias(llama_vocab_n_tokens(vocab), params.logit_bias.size(), params.logit_bias.data()));
|
||||
// logit bias: user biases + model suppress tokens (-INFINITY)
|
||||
{
|
||||
std::vector<llama_logit_bias> merged = params.logit_bias;
|
||||
|
||||
int32_t n_suppress = 0;
|
||||
const llama_token * suppress = llama_vocab_get_suppress_tokens(vocab, &n_suppress);
|
||||
for (int32_t i = 0; i < n_suppress; ++i) {
|
||||
merged.push_back({ suppress[i], -INFINITY });
|
||||
}
|
||||
|
||||
if (!merged.empty()) {
|
||||
samplers.push_back(llama_sampler_init_logit_bias(llama_vocab_n_tokens(vocab), merged.size(), merged.data()));
|
||||
}
|
||||
}
|
||||
|
||||
if (params.mirostat == 0) {
|
||||
|
||||
@@ -179,12 +179,12 @@ class Qwen25OmniModel(Qwen2VLVisionModel, Qwen25AudioModel):
|
||||
def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
|
||||
name, gen = item
|
||||
|
||||
if not name.startswith("visual.") and not name.startswith("audio_tower."):
|
||||
return None
|
||||
|
||||
if name.startswith("thinker."):
|
||||
name = name.replace("thinker.", "")
|
||||
|
||||
if not name.startswith("visual.") and not name.startswith("audio_tower."):
|
||||
return None
|
||||
|
||||
if "audio_bos_eos_token" in name:
|
||||
# this tensor is left unused in transformers code
|
||||
# https://github.com/huggingface/transformers/blob/6e3063422c4b1c014aa60c32b9254fd2902f0f28/src/transformers/models/qwen2_5_omni/modular_qwen2_5_omni.py#L1809
|
||||
|
||||
+1
-1
@@ -4,7 +4,7 @@ project("ggml" C CXX ASM)
|
||||
|
||||
### GGML Version
|
||||
set(GGML_VERSION_MAJOR 0)
|
||||
set(GGML_VERSION_MINOR 17)
|
||||
set(GGML_VERSION_MINOR 18)
|
||||
set(GGML_VERSION_PATCH 0)
|
||||
set(GGML_VERSION_BASE "${GGML_VERSION_MAJOR}.${GGML_VERSION_MINOR}.${GGML_VERSION_PATCH}")
|
||||
|
||||
|
||||
@@ -2739,7 +2739,7 @@ static block_q8_0x4 make_block_q8_0x4(block_q8_0 * in, unsigned int blck_size_in
|
||||
return out;
|
||||
}
|
||||
|
||||
static block_q4_0x4 make_block_q4_0x4(block_q4_0 * in, unsigned int blck_size_interleave) {
|
||||
static block_q4_0x4 make_block_q4_0x4(block_q4_0 * in, int blck_size_interleave) {
|
||||
block_q4_0x4 out;
|
||||
|
||||
for (int i = 0; i < 4; i++) {
|
||||
|
||||
@@ -977,6 +977,13 @@ struct ggml_cuda_type_traits<GGML_TYPE_Q1_0> {
|
||||
static constexpr int qi = QI1_0;
|
||||
};
|
||||
|
||||
template<>
|
||||
struct ggml_cuda_type_traits<GGML_TYPE_Q2_0> {
|
||||
static constexpr int qk = QK2_0;
|
||||
static constexpr int qr = QR2_0;
|
||||
static constexpr int qi = QI2_0;
|
||||
};
|
||||
|
||||
template<>
|
||||
struct ggml_cuda_type_traits<GGML_TYPE_Q4_0> {
|
||||
static constexpr int qk = QK4_0;
|
||||
|
||||
@@ -459,6 +459,8 @@ to_bf16_cuda_t ggml_get_to_bf16_cuda(ggml_type type) {
|
||||
switch (type) {
|
||||
case GGML_TYPE_Q1_0:
|
||||
return dequantize_block_cont_cuda<QK1_0, QR1_0, dequantize_q1_0>;
|
||||
case GGML_TYPE_Q2_0:
|
||||
return dequantize_block_cont_cuda<QK2_0, QR2_0, dequantize_q2_0>;
|
||||
case GGML_TYPE_Q4_0:
|
||||
return dequantize_row_q4_0_cuda;
|
||||
case GGML_TYPE_Q4_1:
|
||||
@@ -514,6 +516,8 @@ to_fp16_cuda_t ggml_get_to_fp16_cuda(ggml_type type) {
|
||||
switch (type) {
|
||||
case GGML_TYPE_Q1_0:
|
||||
return dequantize_block_cont_cuda<QK1_0, QR1_0, dequantize_q1_0>;
|
||||
case GGML_TYPE_Q2_0:
|
||||
return dequantize_block_cont_cuda<QK2_0, QR2_0, dequantize_q2_0>;
|
||||
case GGML_TYPE_Q4_0:
|
||||
return dequantize_row_q4_0_cuda;
|
||||
case GGML_TYPE_Q4_1:
|
||||
@@ -572,6 +576,8 @@ to_fp32_cuda_t ggml_get_to_fp32_cuda(ggml_type type) {
|
||||
switch (type) {
|
||||
case GGML_TYPE_Q1_0:
|
||||
return dequantize_block_cont_cuda<QK1_0, QR1_0, dequantize_q1_0>;
|
||||
case GGML_TYPE_Q2_0:
|
||||
return dequantize_block_cont_cuda<QK2_0, QR2_0, dequantize_q2_0>;
|
||||
case GGML_TYPE_Q4_0:
|
||||
return dequantize_row_q4_0_cuda;
|
||||
case GGML_TYPE_Q4_1:
|
||||
@@ -629,6 +635,8 @@ to_fp16_nc_cuda_t ggml_get_to_fp16_nc_cuda(ggml_type type) {
|
||||
return convert_unary_cuda<float>;
|
||||
case GGML_TYPE_Q1_0:
|
||||
return dequantize_block_cuda<QK1_0, QR1_0, dequantize_q1_0>;
|
||||
case GGML_TYPE_Q2_0:
|
||||
return dequantize_block_cuda<QK2_0, QR2_0, dequantize_q2_0>;
|
||||
case GGML_TYPE_Q4_0:
|
||||
return dequantize_block_cuda<QK4_0, QR4_0, dequantize_q4_0>;
|
||||
case GGML_TYPE_Q4_1:
|
||||
@@ -652,6 +660,8 @@ to_bf16_nc_cuda_t ggml_get_to_bf16_nc_cuda(ggml_type type) {
|
||||
return convert_unary_cuda<float, nv_bfloat16>;
|
||||
case GGML_TYPE_Q1_0:
|
||||
return dequantize_block_cuda<QK1_0, QR1_0, dequantize_q1_0>;
|
||||
case GGML_TYPE_Q2_0:
|
||||
return dequantize_block_cuda<QK2_0, QR2_0, dequantize_q2_0>;
|
||||
case GGML_TYPE_Q4_0:
|
||||
return dequantize_block_cuda<QK4_0, QR4_0, dequantize_q4_0>;
|
||||
case GGML_TYPE_Q4_1:
|
||||
@@ -675,6 +685,8 @@ to_fp32_nc_cuda_t ggml_get_to_fp32_nc_cuda(ggml_type type) {
|
||||
return convert_unary_cuda<half, float>;
|
||||
case GGML_TYPE_Q1_0:
|
||||
return dequantize_block_cuda<QK1_0, QR1_0, dequantize_q1_0>;
|
||||
case GGML_TYPE_Q2_0:
|
||||
return dequantize_block_cuda<QK2_0, QR2_0, dequantize_q2_0>;
|
||||
case GGML_TYPE_Q4_0:
|
||||
return dequantize_block_cuda<QK4_0, QR4_0, dequantize_q4_0>;
|
||||
case GGML_TYPE_Q4_1:
|
||||
|
||||
@@ -23,6 +23,26 @@ static __device__ __forceinline__ void dequantize_q1_0(const void * vx, const in
|
||||
v.y = (2*bit_1 - 1) * d;
|
||||
}
|
||||
|
||||
static __device__ __forceinline__ void dequantize_q2_0(const void * vx, const int64_t ib, const int iqs, float2 & v){
|
||||
const block_q2_0 * x = (const block_q2_0 *) vx;
|
||||
|
||||
const float d = x[ib].d;
|
||||
|
||||
// Q2_0: 2 bits per element, 4 elements per byte.
|
||||
// Stored code c in {0,1,2,3} maps to symbol s = c - 1 in {-1, 0, +1, +2}.
|
||||
const int byte_index_0 = iqs / 4;
|
||||
const int bit_offset_0 = (iqs % 4) * 2;
|
||||
|
||||
const int byte_index_1 = (iqs + 1) / 4;
|
||||
const int bit_offset_1 = ((iqs + 1) % 4) * 2;
|
||||
|
||||
const int c0 = (x[ib].qs[byte_index_0] >> bit_offset_0) & 0x3;
|
||||
const int c1 = (x[ib].qs[byte_index_1] >> bit_offset_1) & 0x3;
|
||||
|
||||
v.x = (c0 - 1) * d;
|
||||
v.y = (c1 - 1) * d;
|
||||
}
|
||||
|
||||
static __device__ __forceinline__ void dequantize_q4_0(const void * vx, const int64_t ib, const int iqs, float2 & v){
|
||||
const block_q4_0 * x = (const block_q4_0 *) vx;
|
||||
|
||||
|
||||
@@ -320,6 +320,10 @@ static void ggml_cuda_get_rows_switch_src0_type(
|
||||
get_rows_cuda_q<QK1_0, QR1_0, dequantize_q1_0>(src0_d, src1_d, dst_d,
|
||||
ne00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb1, nb2, nb3, stream);
|
||||
break;
|
||||
case GGML_TYPE_Q2_0:
|
||||
get_rows_cuda_q<QK2_0, QR2_0, dequantize_q2_0>(src0_d, src1_d, dst_d,
|
||||
ne00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb1, nb2, nb3, stream);
|
||||
break;
|
||||
case GGML_TYPE_Q4_0:
|
||||
get_rows_cuda_q<QK4_0, QR4_0, dequantize_q4_0>(src0_d, src1_d, dst_d,
|
||||
ne00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb1, nb2, nb3, stream);
|
||||
|
||||
@@ -4802,6 +4802,7 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g
|
||||
case GGML_TYPE_F32:
|
||||
case GGML_TYPE_F16:
|
||||
case GGML_TYPE_Q1_0:
|
||||
case GGML_TYPE_Q2_0:
|
||||
case GGML_TYPE_Q4_0:
|
||||
case GGML_TYPE_Q4_1:
|
||||
case GGML_TYPE_Q5_0:
|
||||
@@ -4840,6 +4841,7 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g
|
||||
case GGML_TYPE_BF16:
|
||||
case GGML_TYPE_I32:
|
||||
case GGML_TYPE_Q1_0:
|
||||
case GGML_TYPE_Q2_0:
|
||||
case GGML_TYPE_Q4_0:
|
||||
case GGML_TYPE_Q4_1:
|
||||
case GGML_TYPE_Q5_0:
|
||||
|
||||
@@ -16,6 +16,23 @@ static constexpr __host__ __device__ ggml_cuda_mmq_config ggml_cuda_mmq_get_conf
|
||||
CASE(GGML_TYPE_Q1_0, 256, 1, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
|
||||
CASE(GGML_TYPE_Q1_0, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
|
||||
|
||||
CASE(GGML_TYPE_Q2_0, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);
|
||||
CASE(GGML_TYPE_Q2_0, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);
|
||||
CASE(GGML_TYPE_Q2_0, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);
|
||||
CASE(GGML_TYPE_Q2_0, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);
|
||||
CASE(GGML_TYPE_Q2_0, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);
|
||||
CASE(GGML_TYPE_Q2_0, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
|
||||
CASE(GGML_TYPE_Q2_0, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
|
||||
CASE(GGML_TYPE_Q2_0, 256, 1, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
|
||||
CASE(GGML_TYPE_Q2_0, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
|
||||
CASE(GGML_TYPE_Q2_0, 256, 1, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
|
||||
CASE(GGML_TYPE_Q2_0, 256, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
|
||||
CASE(GGML_TYPE_Q2_0, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
|
||||
CASE(GGML_TYPE_Q2_0, 256, 1, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
|
||||
CASE(GGML_TYPE_Q2_0, 256, 1, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
|
||||
CASE(GGML_TYPE_Q2_0, 256, 1, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
|
||||
CASE(GGML_TYPE_Q2_0, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
|
||||
|
||||
CASE(GGML_TYPE_Q4_0, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);
|
||||
CASE(GGML_TYPE_Q4_0, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);
|
||||
CASE(GGML_TYPE_Q4_0, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);
|
||||
|
||||
@@ -7,6 +7,14 @@ static constexpr __host__ __device__ ggml_cuda_mmq_config ggml_cuda_mmq_get_conf
|
||||
CASE(GGML_TYPE_Q1_0, 512, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
|
||||
CASE(GGML_TYPE_Q1_0, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
|
||||
|
||||
CASE(GGML_TYPE_Q2_0, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);
|
||||
CASE(GGML_TYPE_Q2_0, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);
|
||||
CASE(GGML_TYPE_Q2_0, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);
|
||||
CASE(GGML_TYPE_Q2_0, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
|
||||
CASE(GGML_TYPE_Q2_0, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
|
||||
CASE(GGML_TYPE_Q2_0, 512, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
|
||||
CASE(GGML_TYPE_Q2_0, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
|
||||
|
||||
CASE(GGML_TYPE_Q4_0, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);
|
||||
CASE(GGML_TYPE_Q4_0, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);
|
||||
CASE(GGML_TYPE_Q4_0, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);
|
||||
|
||||
@@ -11,6 +11,18 @@ static constexpr __host__ __device__ ggml_cuda_mmq_config ggml_cuda_mmq_get_conf
|
||||
CASE(GGML_TYPE_Q1_0, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q1_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
|
||||
CASE(GGML_TYPE_Q2_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q2_0, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q2_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q2_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q2_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q2_0, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q2_0, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q2_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q2_0, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q2_0, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q2_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
|
||||
CASE(GGML_TYPE_Q4_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q4_0, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q4_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
|
||||
@@ -11,6 +11,18 @@ static constexpr __host__ __device__ ggml_cuda_mmq_config ggml_cuda_mmq_get_conf
|
||||
CASE(GGML_TYPE_Q1_0, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q1_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
|
||||
CASE(GGML_TYPE_Q2_0, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q2_0, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q2_0, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q2_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q2_0, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q2_0, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q2_0, 256, 2, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q2_0, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q2_0, 256, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q2_0, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q2_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
|
||||
CASE(GGML_TYPE_Q4_0, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q4_0, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q4_0, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
|
||||
@@ -11,6 +11,18 @@ static constexpr __host__ __device__ ggml_cuda_mmq_config ggml_cuda_mmq_get_conf
|
||||
CASE(GGML_TYPE_Q1_0, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q1_0, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q1_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q2_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q2_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q2_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q2_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q2_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q2_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q2_0, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q2_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q2_0, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q2_0, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q2_0, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q2_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
|
||||
CASE(GGML_TYPE_Q4_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q4_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
|
||||
@@ -11,6 +11,18 @@ static constexpr __host__ __device__ ggml_cuda_mmq_config ggml_cuda_mmq_get_conf
|
||||
CASE(GGML_TYPE_Q1_0, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q1_0, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q1_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q2_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q2_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q2_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q2_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q2_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q2_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q2_0, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q2_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q2_0, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q2_0, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q2_0, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q2_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
|
||||
CASE(GGML_TYPE_Q4_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q4_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
|
||||
@@ -11,6 +11,18 @@ static constexpr __host__ __device__ ggml_cuda_mmq_config ggml_cuda_mmq_get_conf
|
||||
CASE(GGML_TYPE_Q1_0, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q1_0, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q1_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q2_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q2_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q2_0, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q2_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q2_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q2_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q2_0, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q2_0, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q2_0, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q2_0, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q2_0, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q2_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
|
||||
CASE(GGML_TYPE_Q4_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q4_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
|
||||
@@ -95,6 +95,103 @@ template <ggml_type type, int J, bool fallback> static __device__ __forceinline_
|
||||
}
|
||||
}
|
||||
|
||||
template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_q2_0(
|
||||
const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {
|
||||
constexpr int warp_size = ggml_cuda_get_physical_warp_size();
|
||||
constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback) / warp_size;
|
||||
constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback);
|
||||
constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback);
|
||||
|
||||
#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
|
||||
int * x_qs = (int *) x_tile;
|
||||
float * x_df = (float *) (x_qs + 2*MMQ_TILE_NE_K);
|
||||
#else
|
||||
constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q8_0, I);
|
||||
int * x_qs = (int *) x_tile;
|
||||
float * x_df = (float *) (x_qs + txs.qs);
|
||||
#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
|
||||
|
||||
constexpr int blocks_per_iter = MMQ_ITER_K / QK2_0;
|
||||
constexpr int threads_per_row = blocks_per_iter * QI2_0;
|
||||
constexpr int nrows = warp_size / threads_per_row;
|
||||
constexpr int scale_entries_per_block = QK2_0 / QK8_1;
|
||||
constexpr int scale_entries_per_row = blocks_per_iter * scale_entries_per_block;
|
||||
|
||||
const int txi = threadIdx.x % threads_per_row;
|
||||
const int kbx = txi / QI2_0;
|
||||
const int kqsx = txi % QI2_0;
|
||||
|
||||
#pragma unroll
|
||||
for (int i0 = 0; i0 < I; i0 += nrows*nwarps) {
|
||||
int i = i0 + threadIdx.y*nrows + threadIdx.x/threads_per_row;
|
||||
|
||||
if (fallback) {
|
||||
i = min(i, i_max);
|
||||
}
|
||||
|
||||
const block_q2_0 * bxi = (const block_q2_0 *) x + kbx0 + i*stride + kbx;
|
||||
// Each 32-element chunk occupies 8 bytes of qs (32 elements * 2 bits = 64 bits)
|
||||
const int qs_offset = 8*kqsx;
|
||||
const int qs0 = bxi->qs[qs_offset + 0] | (bxi->qs[qs_offset + 1] << 8) |
|
||||
(bxi->qs[qs_offset + 2] << 16) | (bxi->qs[qs_offset + 3] << 24);
|
||||
const int qs1 = bxi->qs[qs_offset + 4] | (bxi->qs[qs_offset + 5] << 8) |
|
||||
(bxi->qs[qs_offset + 6] << 16) | (bxi->qs[qs_offset + 7] << 24);
|
||||
|
||||
// Unpack 32 2-bit codes into 8 int32s, each holding 4 signed int8s in {-1,0,1,2}.
|
||||
int unpacked_bytes[8];
|
||||
#pragma unroll
|
||||
for (int j = 0; j < 4; ++j) {
|
||||
const int shift = j * 8;
|
||||
const int codes = (qs0 >> shift) & 0xFF;
|
||||
const int c0 = ((codes >> 0) & 0x3) - 1;
|
||||
const int c1 = ((codes >> 2) & 0x3) - 1;
|
||||
const int c2 = ((codes >> 4) & 0x3) - 1;
|
||||
const int c3 = ((codes >> 6) & 0x3) - 1;
|
||||
unpacked_bytes[j] = (c0 & 0xFF) | ((c1 & 0xFF) << 8) | ((c2 & 0xFF) << 16) | ((c3 & 0xFF) << 24);
|
||||
}
|
||||
#pragma unroll
|
||||
for (int j = 0; j < 4; ++j) {
|
||||
const int shift = j * 8;
|
||||
const int codes = (qs1 >> shift) & 0xFF;
|
||||
const int c0 = ((codes >> 0) & 0x3) - 1;
|
||||
const int c1 = ((codes >> 2) & 0x3) - 1;
|
||||
const int c2 = ((codes >> 4) & 0x3) - 1;
|
||||
const int c3 = ((codes >> 6) & 0x3) - 1;
|
||||
unpacked_bytes[4 + j] = (c0 & 0xFF) | ((c1 & 0xFF) << 8) | ((c2 & 0xFF) << 16) | ((c3 & 0xFF) << 24);
|
||||
}
|
||||
|
||||
const int dst_offset = kbx*(scale_entries_per_block*QI8_0) + kqsx*QI8_0;
|
||||
#pragma unroll
|
||||
for (int j = 0; j < 8; ++j) {
|
||||
#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
|
||||
x_qs[i*sram_stride + dst_offset + j] = unpacked_bytes[j];
|
||||
#else
|
||||
x_qs[i*(2*MMQ_TILE_NE_K + 1) + dst_offset + j] = unpacked_bytes[j];
|
||||
#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
|
||||
}
|
||||
}
|
||||
|
||||
const int ksx = threadIdx.x % scale_entries_per_row;
|
||||
const int scale_block = ksx / scale_entries_per_block;
|
||||
|
||||
#pragma unroll
|
||||
for (int i0 = 0; i0 < I; i0 += nwarps) {
|
||||
int i = i0 + threadIdx.y;
|
||||
|
||||
if (fallback) {
|
||||
i = min(i, i_max);
|
||||
}
|
||||
|
||||
const block_q2_0 * bxi = (const block_q2_0 *) x + kbx0 + i*stride + scale_block;
|
||||
|
||||
#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
|
||||
x_df[i*sram_stride + ksx] = bxi->d;
|
||||
#else
|
||||
x_df[i*(2*MMQ_TILE_NE_K/QI8_0) + i/(QI8_0/2) + ksx] = bxi->d;
|
||||
#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
|
||||
}
|
||||
}
|
||||
|
||||
template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_q4_0(
|
||||
const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {
|
||||
constexpr int warp_size = ggml_cuda_get_physical_warp_size();
|
||||
|
||||
@@ -10,6 +10,9 @@ static void ggml_cuda_mul_mat_q_switch_type(ggml_backend_cuda_context & ctx, con
|
||||
case GGML_TYPE_Q1_0:
|
||||
mul_mat_q_case<GGML_TYPE_Q1_0>(ctx, args, stream);
|
||||
break;
|
||||
case GGML_TYPE_Q2_0:
|
||||
mul_mat_q_case<GGML_TYPE_Q2_0>(ctx, args, stream);
|
||||
break;
|
||||
case GGML_TYPE_Q4_0:
|
||||
mul_mat_q_case<GGML_TYPE_Q4_0>(ctx, args, stream);
|
||||
break;
|
||||
@@ -262,6 +265,7 @@ bool ggml_cuda_should_use_mmq(enum ggml_type type, int cc, int64_t ne11, int64_t
|
||||
|
||||
switch (type) {
|
||||
case GGML_TYPE_Q1_0:
|
||||
case GGML_TYPE_Q2_0:
|
||||
case GGML_TYPE_Q4_0:
|
||||
case GGML_TYPE_Q4_1:
|
||||
case GGML_TYPE_Q5_0:
|
||||
@@ -296,6 +300,15 @@ bool ggml_cuda_should_use_mmq(enum ggml_type type, int cc, int64_t ne11, int64_t
|
||||
return false;
|
||||
}
|
||||
|
||||
// MMQ tiles require at least 48 KiB per-block shared memory; fall back to BLAS otherwise.
|
||||
{
|
||||
const int id = ggml_cuda_get_device();
|
||||
const size_t smpbo = ggml_cuda_info().devices[id].smpbo;
|
||||
if (smpbo < 48 * 1024) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
if (turing_mma_available(cc)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
@@ -60,6 +60,7 @@ static_assert(sizeof(block_fp4_mmq) == sizeof(block_q8_1_mmq), "Unexpected b
|
||||
static mmq_q8_1_ds_layout mmq_get_q8_1_ds_layout(const ggml_type type_x) {
|
||||
switch (type_x) {
|
||||
case GGML_TYPE_Q1_0:
|
||||
case GGML_TYPE_Q2_0:
|
||||
return MMQ_Q8_1_DS_LAYOUT_D4;
|
||||
case GGML_TYPE_Q4_0:
|
||||
case GGML_TYPE_Q4_1:
|
||||
@@ -385,6 +386,7 @@ static constexpr __device__ int ggml_cuda_mmq_get_rows_per_warp(ggml_type type,
|
||||
static constexpr __host__ __device__ tile_x_sizes mmq_get_dp4a_tile_x_sizes(ggml_type type, int I) {
|
||||
switch (type) {
|
||||
case GGML_TYPE_Q1_0: return MMQ_DP4A_TXS_Q8_0;
|
||||
case GGML_TYPE_Q2_0: return MMQ_DP4A_TXS_Q8_0;
|
||||
case GGML_TYPE_Q4_0: return MMQ_DP4A_TXS_Q4_0;
|
||||
case GGML_TYPE_Q4_1: return MMQ_DP4A_TXS_Q4_1;
|
||||
case GGML_TYPE_Q5_0: return MMQ_DP4A_TXS_Q8_0;
|
||||
@@ -542,6 +544,12 @@ static constexpr __device__ ggml_cuda_mmq_util_funcs ggml_cuda_mmq_get_util_func
|
||||
ggml_cuda_mmq_load_tiles_q1_0<type, J, fallback>,
|
||||
ggml_cuda_mmq_vec_dot_q8_0_q8_1_dp4a<type, J, fallback>,
|
||||
ggml_cuda_mmq_write_back_dp4a<type, J, fallback>);
|
||||
case GGML_TYPE_Q2_0:
|
||||
return ggml_cuda_mmq_util_funcs(
|
||||
VDR_Q2_0_Q8_1_MMQ,
|
||||
ggml_cuda_mmq_load_tiles_q2_0<type, J, fallback>,
|
||||
ggml_cuda_mmq_vec_dot_q8_0_q8_1_dp4a<type, J, fallback>,
|
||||
ggml_cuda_mmq_write_back_dp4a<type, J, fallback>);
|
||||
case GGML_TYPE_Q4_0:
|
||||
return ggml_cuda_mmq_util_funcs(
|
||||
VDR_Q4_0_Q8_1_MMQ,
|
||||
@@ -700,6 +708,12 @@ static constexpr __device__ ggml_cuda_mmq_util_funcs ggml_cuda_mmq_get_util_func
|
||||
ggml_cuda_mmq_load_tiles_q1_0<type, J, fallback>,
|
||||
ggml_cuda_mmq_vec_dot_q8_0_q8_1_mma<type, J, fallback, MMQ_Q8_1_DS_LAYOUT_D4>,
|
||||
ggml_cuda_mmq_write_back_mma<type, J, fallback>);
|
||||
case GGML_TYPE_Q2_0:
|
||||
return ggml_cuda_mmq_util_funcs(
|
||||
-1,
|
||||
ggml_cuda_mmq_load_tiles_q2_0<type, J, fallback>,
|
||||
ggml_cuda_mmq_vec_dot_q8_0_q8_1_mma<type, J, fallback, MMQ_Q8_1_DS_LAYOUT_D4>,
|
||||
ggml_cuda_mmq_write_back_mma<type, J, fallback>);
|
||||
case GGML_TYPE_Q4_0:
|
||||
return ggml_cuda_mmq_util_funcs(
|
||||
-1,
|
||||
@@ -1550,6 +1564,7 @@ void mul_mat_q_case(ggml_backend_cuda_context & ctx, const mmq_args & args, cuda
|
||||
template void mul_mat_q_case<type>(ggml_backend_cuda_context & ctx, const mmq_args & args, cudaStream_t stream) \
|
||||
|
||||
extern DECL_MMQ_CASE(GGML_TYPE_Q1_0);
|
||||
extern DECL_MMQ_CASE(GGML_TYPE_Q2_0);
|
||||
extern DECL_MMQ_CASE(GGML_TYPE_Q4_0);
|
||||
extern DECL_MMQ_CASE(GGML_TYPE_Q4_1);
|
||||
extern DECL_MMQ_CASE(GGML_TYPE_Q5_0);
|
||||
|
||||
@@ -10,6 +10,7 @@ typedef float (*vec_dot_q_cuda_t)(const void * __restrict__ vbq, const block_q8_
|
||||
static constexpr __device__ vec_dot_q_cuda_t get_vec_dot_q_cuda(ggml_type type) {
|
||||
switch (type) {
|
||||
case GGML_TYPE_Q1_0: return vec_dot_q1_0_q8_1;
|
||||
case GGML_TYPE_Q2_0: return vec_dot_q2_0_q8_1;
|
||||
case GGML_TYPE_Q4_0: return vec_dot_q4_0_q8_1;
|
||||
case GGML_TYPE_Q4_1: return vec_dot_q4_1_q8_1;
|
||||
case GGML_TYPE_Q5_0: return vec_dot_q5_0_q8_1;
|
||||
@@ -38,6 +39,7 @@ static constexpr __device__ vec_dot_q_cuda_t get_vec_dot_q_cuda(ggml_type type)
|
||||
static constexpr __host__ __device__ int get_vdr_mmvq(ggml_type type) {
|
||||
switch (type) {
|
||||
case GGML_TYPE_Q1_0: return VDR_Q1_0_Q8_1_MMVQ;
|
||||
case GGML_TYPE_Q2_0: return VDR_Q2_0_Q8_1_MMVQ;
|
||||
case GGML_TYPE_Q4_0: return VDR_Q4_0_Q8_1_MMVQ;
|
||||
case GGML_TYPE_Q4_1: return VDR_Q4_1_Q8_1_MMVQ;
|
||||
case GGML_TYPE_Q5_0: return VDR_Q5_0_Q8_1_MMVQ;
|
||||
@@ -1010,6 +1012,12 @@ static void mul_mat_vec_q_switch_type(
|
||||
nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
|
||||
nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, ids_stride, stream);
|
||||
break;
|
||||
case GGML_TYPE_Q2_0:
|
||||
mul_mat_vec_q_switch_ncols_dst<GGML_TYPE_Q2_0>
|
||||
(vx, vy, ids, fusion, dst, ncols_x, nrows_x, ncols_dst, stride_row_x, stride_col_y, stride_col_dst,
|
||||
nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
|
||||
nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, ids_stride, stream);
|
||||
break;
|
||||
case GGML_TYPE_Q4_0:
|
||||
mul_mat_vec_q_switch_ncols_dst<GGML_TYPE_Q4_0>
|
||||
(vx, vy, ids, fusion, dst, ncols_x, nrows_x, ncols_dst, stride_row_x, stride_col_y, stride_col_dst,
|
||||
|
||||
@@ -36,6 +36,7 @@ SOURCE_FATTN_MMA_CASE = "DECL_FATTN_MMA_F16_CASE({head_size_kq}, {head_size_v},
|
||||
|
||||
TYPES_MMQ = [
|
||||
"GGML_TYPE_Q1_0",
|
||||
"GGML_TYPE_Q2_0",
|
||||
"GGML_TYPE_Q4_0", "GGML_TYPE_Q4_1", "GGML_TYPE_Q5_0", "GGML_TYPE_Q5_1", "GGML_TYPE_Q8_0",
|
||||
"GGML_TYPE_Q2_K", "GGML_TYPE_Q3_K", "GGML_TYPE_Q4_K", "GGML_TYPE_Q5_K", "GGML_TYPE_Q6_K",
|
||||
"GGML_TYPE_IQ2_XXS", "GGML_TYPE_IQ2_XS", "GGML_TYPE_IQ2_S", "GGML_TYPE_IQ3_XXS", "GGML_TYPE_IQ3_S",
|
||||
|
||||
@@ -0,0 +1,5 @@
|
||||
// This file has been autogenerated by generate_cu_files.py, do not edit manually.
|
||||
|
||||
#include "../mmq.cuh"
|
||||
|
||||
DECL_MMQ_CASE(GGML_TYPE_Q2_0);
|
||||
@@ -109,6 +109,9 @@ static __device__ __forceinline__ uint32_t unpack_ksigns(const uint8_t v) {
|
||||
#define VDR_Q1_0_Q8_1_MMVQ 1 // Process one 32-element chunk at a time for parallelism
|
||||
#define VDR_Q1_0_Q8_1_MMQ 4 // Q1_0 has 128 bits (4 ints) per block
|
||||
|
||||
#define VDR_Q2_0_Q8_1_MMVQ 1 // Process one 32-element chunk at a time for parallelism
|
||||
#define VDR_Q2_0_Q8_1_MMQ 2 // Q2_0 group 64: 128 bits (4 ints) per block, 2 32-element chunks
|
||||
|
||||
#define VDR_Q4_0_Q8_1_MMVQ 2
|
||||
#define VDR_Q4_0_Q8_1_MMQ 4
|
||||
|
||||
@@ -722,6 +725,64 @@ static __device__ __forceinline__ float vec_dot_q1_0_q8_1(
|
||||
return d1 * d8 * sumi;
|
||||
}
|
||||
|
||||
static __device__ __forceinline__ float vec_dot_q2_0_q8_1(
|
||||
const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) {
|
||||
|
||||
const block_q2_0 * bq2_0 = (const block_q2_0 *) vbq + kbx;
|
||||
|
||||
// Q2_0 (group 64): 64 elements with ONE scale, 2 bits per element (4 elements per byte)
|
||||
// Q8_1: 32 elements per block with individual scales
|
||||
// iqs selects which of the 2 chunks of 32 elements to process (0-1)
|
||||
|
||||
const float d2 = bq2_0->d;
|
||||
|
||||
// Process only the chunk specified by iqs
|
||||
const block_q8_1 * bq8_1_chunk = bq8_1 + iqs;
|
||||
|
||||
// Load 64 bits (8 bytes) for this chunk from Q2_0: bytes [8*iqs, 8*iqs+8)
|
||||
const int offset = iqs * 8;
|
||||
const int v0 = bq2_0->qs[offset + 0] | (bq2_0->qs[offset + 1] << 8) |
|
||||
(bq2_0->qs[offset + 2] << 16) | (bq2_0->qs[offset + 3] << 24);
|
||||
const int v1 = bq2_0->qs[offset + 4] | (bq2_0->qs[offset + 5] << 8) |
|
||||
(bq2_0->qs[offset + 6] << 16) | (bq2_0->qs[offset + 7] << 24);
|
||||
|
||||
// Unpack 32 2-bit codes into 8 int32s, each holding 4 signed int8 symbols in {-1,0,1,2}.
|
||||
// Stored code c in {0,1,2,3} -> symbol s = c - 1.
|
||||
int vi_bytes[8];
|
||||
#pragma unroll
|
||||
for (int j = 0; j < 4; ++j) {
|
||||
const int shift = j * 8;
|
||||
const int codes = (v0 >> shift) & 0xFF;
|
||||
const int c0 = ((codes >> 0) & 0x3) - 1;
|
||||
const int c1 = ((codes >> 2) & 0x3) - 1;
|
||||
const int c2 = ((codes >> 4) & 0x3) - 1;
|
||||
const int c3 = ((codes >> 6) & 0x3) - 1;
|
||||
vi_bytes[j] = (c0 & 0xFF) | ((c1 & 0xFF) << 8) | ((c2 & 0xFF) << 16) | ((c3 & 0xFF) << 24);
|
||||
}
|
||||
#pragma unroll
|
||||
for (int j = 0; j < 4; ++j) {
|
||||
const int shift = j * 8;
|
||||
const int codes = (v1 >> shift) & 0xFF;
|
||||
const int c0 = ((codes >> 0) & 0x3) - 1;
|
||||
const int c1 = ((codes >> 2) & 0x3) - 1;
|
||||
const int c2 = ((codes >> 4) & 0x3) - 1;
|
||||
const int c3 = ((codes >> 6) & 0x3) - 1;
|
||||
vi_bytes[4 + j] = (c0 & 0xFF) | ((c1 & 0xFF) << 8) | ((c2 & 0xFF) << 16) | ((c3 & 0xFF) << 24);
|
||||
}
|
||||
|
||||
// Compute dot product for this 32-element chunk
|
||||
int sumi = 0;
|
||||
#pragma unroll
|
||||
for (int j = 0; j < 8; ++j) {
|
||||
const int u = get_int_b4(bq8_1_chunk->qs, j);
|
||||
sumi = ggml_cuda_dp4a(vi_bytes[j], u, sumi);
|
||||
}
|
||||
|
||||
// Apply Q2_0's single scale and this chunk's Q8_1 scale
|
||||
const float d8 = __low2float(bq8_1_chunk->ds);
|
||||
return d2 * d8 * sumi;
|
||||
}
|
||||
|
||||
static __device__ __forceinline__ float vec_dot_q4_0_q8_1(
|
||||
const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) {
|
||||
|
||||
|
||||
@@ -213,7 +213,7 @@ typedef void * ggml_metal_rset_t;
|
||||
// a collection of residency sets (non-owning)
|
||||
typedef struct ggml_metal_rsets * ggml_metal_rsets_t;
|
||||
|
||||
ggml_metal_rsets_t ggml_metal_rsets_init(void);
|
||||
ggml_metal_rsets_t ggml_metal_rsets_init(ggml_metal_device_t dev);
|
||||
void ggml_metal_rsets_free(ggml_metal_rsets_t rsets);
|
||||
|
||||
//
|
||||
|
||||
@@ -557,7 +557,32 @@ struct ggml_metal_rsets {
|
||||
dispatch_group_t d_group;
|
||||
};
|
||||
|
||||
ggml_metal_rsets_t ggml_metal_rsets_init(void) {
|
||||
#if defined(GGML_METAL_HAS_RESIDENCY_SETS)
|
||||
static void ggml_metal_dummy_work(ggml_metal_device_t dev) {
|
||||
if (dev->mtl_queue == nil) {
|
||||
return;
|
||||
}
|
||||
|
||||
@autoreleasepool {
|
||||
// perform a minimal dummy operation on the GPU
|
||||
id<MTLBuffer> buf = [dev->mtl_device newBufferWithLength:1 options:MTLResourceStorageModePrivate];
|
||||
id<MTLCommandBuffer> cmd_buf = [dev->mtl_queue commandBuffer];
|
||||
|
||||
{
|
||||
id<MTLBlitCommandEncoder> encoder = [cmd_buf blitCommandEncoder];
|
||||
|
||||
[encoder fillBuffer:buf range:NSMakeRange(0, 1) value:0];
|
||||
|
||||
[encoder endEncoding];
|
||||
}
|
||||
|
||||
[cmd_buf commit];
|
||||
[buf release];
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
ggml_metal_rsets_t ggml_metal_rsets_init(ggml_metal_device_t dev) {
|
||||
ggml_metal_rsets_t res = calloc(1, sizeof(struct ggml_metal_rsets));
|
||||
|
||||
res->lock = [[NSLock alloc] init];
|
||||
@@ -610,6 +635,15 @@ ggml_metal_rsets_t ggml_metal_rsets_init(void) {
|
||||
#endif
|
||||
});
|
||||
|
||||
#if defined(GGML_METAL_HAS_RESIDENCY_SETS)
|
||||
if (@available(macOS 15.0, iOS 18.0, tvOS 18.0, visionOS 2.0, *)) {
|
||||
// workaround for residency set memory not being released if no GPU operation occurs
|
||||
// https://developer.apple.com/forums/thread/839089
|
||||
// https://github.com/ggml-org/llama.cpp/issues/25937
|
||||
ggml_metal_dummy_work(dev);
|
||||
}
|
||||
#endif
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
@@ -864,7 +898,7 @@ ggml_metal_device_t ggml_metal_device_init(int device) {
|
||||
}
|
||||
|
||||
if (dev->props.use_residency_sets) {
|
||||
dev->rsets = ggml_metal_rsets_init();
|
||||
dev->rsets = ggml_metal_rsets_init(dev);
|
||||
} else {
|
||||
dev->rsets = nil;
|
||||
}
|
||||
@@ -1484,6 +1518,7 @@ static void ggml_metal_buffer_rset_free(ggml_metal_buffer_t buf) {
|
||||
if (buf->rset) {
|
||||
[buf->rset endResidency];
|
||||
[buf->rset removeAllAllocations];
|
||||
[buf->rset commit];
|
||||
[buf->rset release];
|
||||
}
|
||||
}
|
||||
|
||||
@@ -306,29 +306,43 @@ static __dpct_inline__ T op_trunc(T x) {
|
||||
}
|
||||
}
|
||||
|
||||
template<typename T, typename F>
|
||||
static void unary_op_flat_kernel(const T * x, T * dst, const int k, const sycl::nd_item<1> & item_ct1, F func) {
|
||||
SYCL_GLOBAL_ID_LOOP(k, item_ct1) {
|
||||
dst[i] = func(x[i]);
|
||||
}
|
||||
}
|
||||
|
||||
template<typename T, typename F>
|
||||
static void unary_op_generic_kernel(
|
||||
const T * x,
|
||||
T * dst,
|
||||
const int k,
|
||||
const int64_t ne0, const int64_t ne1, const int64_t ne2, const int64_t ne3,
|
||||
const sycl::uint3 ne0_fd, const sycl::uint3 ne1_fd, const sycl::uint3 ne2_fd,
|
||||
const size_t nb0, const size_t nb1, const size_t nb2, const size_t nb3,
|
||||
const size_t nbd0, const size_t nbd1, const size_t nbd2, const size_t nbd3,
|
||||
const sycl::nd_item<1> & item_ct1,
|
||||
F func) {
|
||||
|
||||
(void) ne3;
|
||||
// 32-bit index math: k is int, so every logical index fits u32. 64-bit integer div/mod is
|
||||
// emulated on Xe and dominates this kernel otherwise, and even the 32-bit divide is worth
|
||||
// avoiding -- the divisors are launch-invariant, so the magic numbers are precomputed
|
||||
// host-side and each division becomes a multiply-high plus a shift.
|
||||
// Byte offsets are widened back to size_t only for the final address math.
|
||||
SYCL_GLOBAL_ID_LOOP(k, item_ct1) {
|
||||
const int64_t i0 = i % ne0;
|
||||
const int64_t i1 = (i / ne0) % ne1;
|
||||
const int64_t i2 = (i / (ne0*ne1)) % ne2;
|
||||
const int64_t i3 = i / (ne0*ne1*ne2);
|
||||
sycl::uint2 dm = fast_div_modulo((uint32_t) i, ne0_fd);
|
||||
const uint32_t i0 = dm.y();
|
||||
dm = fast_div_modulo(dm.x(), ne1_fd);
|
||||
const uint32_t i1 = dm.y();
|
||||
dm = fast_div_modulo(dm.x(), ne2_fd);
|
||||
const uint32_t i2 = dm.y();
|
||||
const uint32_t i3 = dm.x();
|
||||
|
||||
const char * src_base = (const char *) x;
|
||||
char * dst_base = (char *) dst;
|
||||
|
||||
const T * srcp = (const T *)(src_base + i0*nb0 + i1*nb1 + i2*nb2 + i3*nb3 );
|
||||
T * dstp = (T *)(dst_base + i0*nbd0 + i1*nbd1 + i2*nbd2 + i3*nbd3);
|
||||
const T * srcp = (const T *)(src_base + (size_t) i0*nb0 + (size_t) i1*nb1 + (size_t) i2*nb2 + (size_t) i3*nb3 );
|
||||
T * dstp = (T *)(dst_base + (size_t) i0*nbd0 + (size_t) i1*nbd1 + (size_t) i2*nbd2 + (size_t) i3*nbd3);
|
||||
|
||||
*dstp = func(*srcp);
|
||||
}
|
||||
@@ -407,46 +421,51 @@ static void clamp(const T * x, T * dst, const float min, const float max, const
|
||||
}
|
||||
|
||||
template<typename T>
|
||||
static void gated_op_fused_geglu(const T * x, const T * g, T * dst, const uint64_t k, const uint64_t n, const uint64_t o0, const uint64_t o1, const sycl::nd_item<1> &item_ct1) {
|
||||
static void gated_op_fused_geglu(const T * x, const T * g, T * dst, const uint64_t k, const sycl::uint3 n_fd, const uint64_t o0, const uint64_t o1, const sycl::nd_item<1> &item_ct1) {
|
||||
SYCL_GLOBAL_ID_LOOP(k, item_ct1) {
|
||||
const int64_t j0 = (i / n) * o0 + (i % n);
|
||||
const int64_t j1 = o0 == o1 ? j0 : (i / n) * o1 + (i % n);
|
||||
const sycl::uint2 rc = fast_div_modulo((uint32_t) i, n_fd);
|
||||
const int64_t j0 = rc.x() * o0 + rc.y();
|
||||
const int64_t j1 = o0 == o1 ? j0 : rc.x() * o1 + rc.y();
|
||||
dst[i] = op_gelu(x[j0]) * g[j1];
|
||||
}
|
||||
}
|
||||
|
||||
template<typename T>
|
||||
static void gated_op_fused_reglu(const T * x, const T * g, T * dst, const uint64_t k, const uint64_t n, const uint64_t o0, const uint64_t o1, const sycl::nd_item<1> &item_ct1) {
|
||||
static void gated_op_fused_reglu(const T * x, const T * g, T * dst, const uint64_t k, const sycl::uint3 n_fd, const uint64_t o0, const uint64_t o1, const sycl::nd_item<1> &item_ct1) {
|
||||
SYCL_GLOBAL_ID_LOOP(k, item_ct1) {
|
||||
const int64_t j0 = (i / n) * o0 + (i % n);
|
||||
const int64_t j1 = o0 == o1 ? j0 : (i / n) * o1 + (i % n);
|
||||
const sycl::uint2 rc = fast_div_modulo((uint32_t) i, n_fd);
|
||||
const int64_t j0 = rc.x() * o0 + rc.y();
|
||||
const int64_t j1 = o0 == o1 ? j0 : rc.x() * o1 + rc.y();
|
||||
dst[i] = op_relu(x[j0]) * g[j1];
|
||||
}
|
||||
}
|
||||
|
||||
template<typename T>
|
||||
static void gated_op_fused_swiglu(const T * x, const T * g, T * dst, const uint64_t k, const uint64_t n, const uint64_t o0, const uint64_t o1, const sycl::nd_item<1> &item_ct1) {
|
||||
static void gated_op_fused_swiglu(const T * x, const T * g, T * dst, const uint64_t k, const sycl::uint3 n_fd, const uint64_t o0, const uint64_t o1, const sycl::nd_item<1> &item_ct1) {
|
||||
SYCL_GLOBAL_ID_LOOP(k, item_ct1) {
|
||||
const int64_t j0 = (i / n) * o0 + (i % n);
|
||||
const int64_t j1 = o0 == o1 ? j0 : (i / n) * o1 + (i % n);
|
||||
const sycl::uint2 rc = fast_div_modulo((uint32_t) i, n_fd);
|
||||
const int64_t j0 = rc.x() * o0 + rc.y();
|
||||
const int64_t j1 = o0 == o1 ? j0 : rc.x() * o1 + rc.y();
|
||||
dst[i] = op_silu(x[j0]) * g[j1];
|
||||
}
|
||||
}
|
||||
|
||||
template<typename T>
|
||||
static void gated_op_fused_geglu_erf(const T * x, const T * g, T * dst, const uint64_t k, const uint64_t n, const uint64_t o0, const uint64_t o1, const sycl::nd_item<1> &item_ct1) {
|
||||
static void gated_op_fused_geglu_erf(const T * x, const T * g, T * dst, const uint64_t k, const sycl::uint3 n_fd, const uint64_t o0, const uint64_t o1, const sycl::nd_item<1> &item_ct1) {
|
||||
SYCL_GLOBAL_ID_LOOP(k, item_ct1) {
|
||||
const int64_t j0 = (i / n) * o0 + (i % n);
|
||||
const int64_t j1 = o0 == o1 ? j0 : (i / n) * o1 + (i % n);
|
||||
const sycl::uint2 rc = fast_div_modulo((uint32_t) i, n_fd);
|
||||
const int64_t j0 = rc.x() * o0 + rc.y();
|
||||
const int64_t j1 = o0 == o1 ? j0 : rc.x() * o1 + rc.y();
|
||||
dst[i] = op_gelu_erf(x[j0]) * g[j1];
|
||||
}
|
||||
}
|
||||
|
||||
template<typename T>
|
||||
static void gated_op_fused_geglu_quick(const T * x, const T * g, T * dst, const uint64_t k, const uint64_t n, const uint64_t o0, const uint64_t o1, const sycl::nd_item<1> &item_ct1) {
|
||||
static void gated_op_fused_geglu_quick(const T * x, const T * g, T * dst, const uint64_t k, const sycl::uint3 n_fd, const uint64_t o0, const uint64_t o1, const sycl::nd_item<1> &item_ct1) {
|
||||
SYCL_GLOBAL_ID_LOOP(k, item_ct1) {
|
||||
const int64_t j0 = (i / n) * o0 + (i % n);
|
||||
const int64_t j1 = o0 == o1 ? j0 : (i / n) * o1 + (i % n);
|
||||
const sycl::uint2 rc = fast_div_modulo((uint32_t) i, n_fd);
|
||||
const int64_t j0 = rc.x() * o0 + rc.y();
|
||||
const int64_t j1 = o0 == o1 ? j0 : rc.x() * o1 + rc.y();
|
||||
dst[i] = op_gelu_quick(x[j0]) * g[j1];
|
||||
}
|
||||
}
|
||||
@@ -529,6 +548,10 @@ static inline void dispatch_ggml_sycl_op_fused_glu(ggml_backend_sycl_context & c
|
||||
GGML_ASSERT(dst->ne[0] == nc);
|
||||
GGML_ASSERT(ggml_is_contiguous_1(dst->src[0]));
|
||||
GGML_ASSERT(ggml_is_contiguous(dst));
|
||||
// The fused GLU kernels index with 32-bit fastdiv, which is exact only for indices below
|
||||
// 2^31. A dst that large is ~8 GB at f32, and the grid sizing already narrows to 32 bits,
|
||||
// so assert the bound rather than carry a second code path for it.
|
||||
GGML_ASSERT(ggml_nelements(dst) < ((int64_t) 1 << 31));
|
||||
const int32_t swapped = ((const int32_t *) dst->op_params)[1];
|
||||
void * src0_d = src0->data;
|
||||
void * src1_d = src1 ? src1->data : src0->data;
|
||||
@@ -597,7 +620,6 @@ static inline void ggml_sycl_op_unary(
|
||||
const int64_t ne0 = dst->ne[0];
|
||||
const int64_t ne1 = dst->ne[1];
|
||||
const int64_t ne2 = dst->ne[2];
|
||||
const int64_t ne3 = dst->ne[3];
|
||||
|
||||
const size_t nb0 = src0->nb[0];
|
||||
const size_t nb1 = src0->nb[1];
|
||||
@@ -609,24 +631,42 @@ static inline void ggml_sycl_op_unary(
|
||||
const size_t nbd2 = dst->nb[2];
|
||||
const size_t nbd3 = dst->nb[3];
|
||||
|
||||
// Hot unary ops (FFN/GDN silu, sigmoid, ...) run on contiguous tensors;
|
||||
// skip the strided index math entirely for them.
|
||||
const bool contiguous = ggml_is_contiguous(src0) && ggml_is_contiguous(dst);
|
||||
|
||||
ggml_sycl_detail::dispatch_ggml_sycl_op_unary(ctx, dst,
|
||||
[=](const auto* src, auto* dst_ptr, int k_elements, queue_ptr stream) {
|
||||
|
||||
const int num_blocks = ceil_div(k_elements, 256);
|
||||
|
||||
stream->parallel_for(
|
||||
sycl::nd_range<1>(sycl::range<1>(num_blocks) * sycl::range<1>(256),
|
||||
sycl::range<1>(256)),
|
||||
[=](sycl::nd_item<1> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
|
||||
unary_op_generic_kernel(
|
||||
src, dst_ptr, k_elements,
|
||||
ne0, ne1, ne2, ne3,
|
||||
nb0, nb1, nb2, nb3,
|
||||
nbd0, nbd1, nbd2, nbd3,
|
||||
item_ct1,
|
||||
func
|
||||
);
|
||||
});
|
||||
if (contiguous) {
|
||||
stream->parallel_for(
|
||||
sycl::nd_range<1>(sycl::range<1>(num_blocks) * sycl::range<1>(256),
|
||||
sycl::range<1>(256)),
|
||||
[=](sycl::nd_item<1> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
|
||||
unary_op_flat_kernel(src, dst_ptr, k_elements, item_ct1, func);
|
||||
});
|
||||
} else {
|
||||
// Launch-invariant divisors: compute the magic numbers once on the host so the
|
||||
// kernel never issues an integer divide. Only the strided path needs them.
|
||||
const sycl::uint3 ne0_fd = init_fastdiv_values((uint32_t) ne0);
|
||||
const sycl::uint3 ne1_fd = init_fastdiv_values((uint32_t) ne1);
|
||||
const sycl::uint3 ne2_fd = init_fastdiv_values((uint32_t) ne2);
|
||||
stream->parallel_for(
|
||||
sycl::nd_range<1>(sycl::range<1>(num_blocks) * sycl::range<1>(256),
|
||||
sycl::range<1>(256)),
|
||||
[=](sycl::nd_item<1> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
|
||||
unary_op_generic_kernel(
|
||||
src, dst_ptr, k_elements,
|
||||
ne0_fd, ne1_fd, ne2_fd,
|
||||
nb0, nb1, nb2, nb3,
|
||||
nbd0, nbd1, nbd2, nbd3,
|
||||
item_ct1,
|
||||
func
|
||||
);
|
||||
});
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
@@ -930,10 +970,11 @@ static inline void ggml_sycl_op_geglu(ggml_backend_sycl_context & ctx, ggml_tens
|
||||
ggml_sycl_detail::dispatch_ggml_sycl_op_fused_glu(ctx, dst,
|
||||
[](const auto* x_ptr, const auto* g_ptr, auto* dst_ptr, uint64_t k, uint64_t n, uint64_t o0, uint64_t o1, queue_ptr main_stream) {
|
||||
const uint32_t num_blocks = ceil_div(k, SYCL_GELU_BLOCK_SIZE);
|
||||
const sycl::uint3 n_fd = init_fastdiv_values((uint32_t) n);
|
||||
main_stream->parallel_for(
|
||||
sycl::nd_range<1>((num_blocks * sycl::range<1>(SYCL_GELU_BLOCK_SIZE)),
|
||||
sycl::range<1>(SYCL_GELU_BLOCK_SIZE)), [=](sycl::nd_item<1> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
|
||||
gated_op_fused_geglu(x_ptr, g_ptr, dst_ptr, k, n, o0, o1, item_ct1);
|
||||
gated_op_fused_geglu(x_ptr, g_ptr, dst_ptr, k, n_fd, o0, o1, item_ct1);
|
||||
});
|
||||
});
|
||||
}
|
||||
@@ -942,10 +983,11 @@ static inline void ggml_sycl_op_reglu(ggml_backend_sycl_context & ctx, ggml_tens
|
||||
ggml_sycl_detail::dispatch_ggml_sycl_op_fused_glu(ctx, dst,
|
||||
[](const auto* x_ptr, const auto* g_ptr, auto* dst_ptr, uint64_t k, uint64_t n, uint64_t o0, uint64_t o1, queue_ptr main_stream) {
|
||||
const uint32_t num_blocks = ceil_div((uint32_t)k, SYCL_RELU_BLOCK_SIZE); // Using RELU block size for reglu
|
||||
const sycl::uint3 n_fd = init_fastdiv_values((uint32_t) n);
|
||||
main_stream->parallel_for(
|
||||
sycl::nd_range<1>((num_blocks * sycl::range<1>(SYCL_RELU_BLOCK_SIZE)),
|
||||
sycl::range<1>(SYCL_RELU_BLOCK_SIZE)), [=](sycl::nd_item<1> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
|
||||
gated_op_fused_reglu(x_ptr, g_ptr, dst_ptr, k, n, o0, o1, item_ct1);
|
||||
gated_op_fused_reglu(x_ptr, g_ptr, dst_ptr, k, n_fd, o0, o1, item_ct1);
|
||||
});
|
||||
});
|
||||
}
|
||||
@@ -954,10 +996,11 @@ static inline void ggml_sycl_op_swiglu(ggml_backend_sycl_context & ctx, ggml_ten
|
||||
ggml_sycl_detail::dispatch_ggml_sycl_op_fused_glu(ctx, dst,
|
||||
[](const auto* x_ptr, const auto* g_ptr, auto* dst_ptr, uint64_t k, uint64_t n, uint64_t o0, uint64_t o1, queue_ptr main_stream) {
|
||||
const uint32_t num_blocks = ceil_div((uint32_t)k, SYCL_SILU_BLOCK_SIZE); // Using SILU block size for swiglu
|
||||
const sycl::uint3 n_fd = init_fastdiv_values((uint32_t) n);
|
||||
main_stream->parallel_for(
|
||||
sycl::nd_range<1>((num_blocks * sycl::range<1>(SYCL_SILU_BLOCK_SIZE)),
|
||||
sycl::range<1>(SYCL_SILU_BLOCK_SIZE)), [=](sycl::nd_item<1> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
|
||||
gated_op_fused_swiglu(x_ptr, g_ptr, dst_ptr, k, n, o0, o1, item_ct1);
|
||||
gated_op_fused_swiglu(x_ptr, g_ptr, dst_ptr, k, n_fd, o0, o1, item_ct1);
|
||||
});
|
||||
});
|
||||
}
|
||||
@@ -1057,10 +1100,11 @@ static inline void ggml_sycl_op_geglu_erf(ggml_backend_sycl_context & ctx, ggml_
|
||||
ggml_sycl_detail::dispatch_ggml_sycl_op_fused_glu(ctx, dst,
|
||||
[](const auto* x_ptr, const auto* g_ptr, auto* dst_ptr, uint64_t k, uint64_t n, uint64_t o0, uint64_t o1, queue_ptr main_stream) {
|
||||
const uint32_t num_blocks = ceil_div(k, SYCL_GELU_BLOCK_SIZE);
|
||||
const sycl::uint3 n_fd = init_fastdiv_values((uint32_t) n);
|
||||
main_stream->parallel_for(
|
||||
sycl::nd_range<1>((num_blocks * sycl::range<1>(SYCL_GELU_BLOCK_SIZE)),
|
||||
sycl::range<1>(SYCL_GELU_BLOCK_SIZE)), [=](sycl::nd_item<1> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
|
||||
gated_op_fused_geglu_erf(x_ptr, g_ptr, dst_ptr, k, n, o0, o1, item_ct1);
|
||||
gated_op_fused_geglu_erf(x_ptr, g_ptr, dst_ptr, k, n_fd, o0, o1, item_ct1);
|
||||
});
|
||||
});
|
||||
}
|
||||
@@ -1069,10 +1113,11 @@ static inline void ggml_sycl_op_geglu_quick(ggml_backend_sycl_context & ctx, ggm
|
||||
ggml_sycl_detail::dispatch_ggml_sycl_op_fused_glu(ctx, dst,
|
||||
[](const auto* x_ptr, const auto* g_ptr, auto* dst_ptr, uint64_t k, uint64_t n, uint64_t o0, uint64_t o1, queue_ptr main_stream) {
|
||||
const uint32_t num_blocks = ceil_div(k, SYCL_GELU_BLOCK_SIZE);
|
||||
const sycl::uint3 n_fd = init_fastdiv_values((uint32_t) n);
|
||||
main_stream->parallel_for(
|
||||
sycl::nd_range<1>((num_blocks * sycl::range<1>(SYCL_GELU_BLOCK_SIZE)),
|
||||
sycl::range<1>(SYCL_GELU_BLOCK_SIZE)), [=](sycl::nd_item<1> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
|
||||
gated_op_fused_geglu_quick(x_ptr, g_ptr, dst_ptr, k, n, o0, o1, item_ct1);
|
||||
gated_op_fused_geglu_quick(x_ptr, g_ptr, dst_ptr, k, n_fd, o0, o1, item_ct1);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
@@ -4440,8 +4440,11 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = {
|
||||
MODEL_TENSOR.FFN_EXP_PROBS_B,
|
||||
MODEL_TENSOR.LAYER_OUT_NORM,
|
||||
MODEL_TENSOR.NEXTN_EH_PROJ,
|
||||
MODEL_TENSOR.NEXTN_EMBED_TOKENS,
|
||||
MODEL_TENSOR.NEXTN_ENORM,
|
||||
MODEL_TENSOR.NEXTN_HNORM,
|
||||
MODEL_TENSOR.NEXTN_SHARED_HEAD_HEAD,
|
||||
MODEL_TENSOR.NEXTN_SHARED_HEAD_NORM,
|
||||
],
|
||||
MODEL_ARCH.STEP35: [
|
||||
MODEL_TENSOR.TOKEN_EMBD,
|
||||
|
||||
@@ -1102,6 +1102,9 @@ extern "C" {
|
||||
LLAMA_API bool llama_vocab_get_add_eos(const struct llama_vocab * vocab);
|
||||
LLAMA_API bool llama_vocab_get_add_sep(const struct llama_vocab * vocab);
|
||||
|
||||
// model-specific suppress tokens (gguf key: tokenizer.ggml.suppress_tokens)
|
||||
LLAMA_API const llama_token * llama_vocab_get_suppress_tokens(const struct llama_vocab * vocab, int32_t * n_suppress_tokens);
|
||||
|
||||
LLAMA_API llama_token llama_vocab_fim_pre(const struct llama_vocab * vocab);
|
||||
LLAMA_API llama_token llama_vocab_fim_suf(const struct llama_vocab * vocab);
|
||||
LLAMA_API llama_token llama_vocab_fim_mid(const struct llama_vocab * vocab);
|
||||
|
||||
@@ -1 +1 @@
|
||||
9be313313c8ecb9488911bd64550190e3ed80f38
|
||||
06ca97616793248fadb410ea8d69c7511b2005e4
|
||||
|
||||
+3
-1
@@ -2243,7 +2243,9 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params,
|
||||
filter = [&](uint32_t il) { return il >= hparams.n_layer(); };
|
||||
}
|
||||
|
||||
if ((arch == LLM_ARCH_STEP35 || arch == LLM_ARCH_HY_V3 || arch == LLM_ARCH_GLM_DSA) && hparams.n_layer_nextn > 0) {
|
||||
if ((arch == LLM_ARCH_STEP35 || arch == LLM_ARCH_HY_V3 || arch == LLM_ARCH_GLM_DSA ||
|
||||
arch == LLM_ARCH_MIMO2) &&
|
||||
hparams.n_layer_nextn > 0) {
|
||||
if (params.ctx_type == LLAMA_CONTEXT_TYPE_MTP) {
|
||||
filter = [&](uint32_t il) { return il >= hparams.n_layer(); };
|
||||
} else {
|
||||
|
||||
+16
-1
@@ -2578,7 +2578,14 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) {
|
||||
if (suppress_idx != -1) {
|
||||
const int n = gguf_get_arr_n(ctx, suppress_idx);
|
||||
const int32_t * data = (const int32_t *) gguf_get_arr_data(ctx, suppress_idx);
|
||||
suppress_tokens.assign(data, data + n);
|
||||
// drop out-of-range ids
|
||||
suppress_tokens.reserve(n);
|
||||
for (int i = 0; i < n; ++i) {
|
||||
const int32_t id = data[i];
|
||||
if (id >= 0 && id < (int) id_to_token.size()) {
|
||||
suppress_tokens.push_back(id);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4205,6 +4212,14 @@ bool llama_vocab_get_add_sep(const struct llama_vocab * vocab) {
|
||||
return vocab->get_add_sep();
|
||||
}
|
||||
|
||||
const llama_token * llama_vocab_get_suppress_tokens(const struct llama_vocab * vocab, int32_t * n_suppress_tokens) {
|
||||
const std::vector<llama_token> & tokens = vocab->get_suppress_tokens();
|
||||
if (n_suppress_tokens) {
|
||||
*n_suppress_tokens = (int32_t) tokens.size();
|
||||
}
|
||||
return tokens.data();
|
||||
}
|
||||
|
||||
llama_token llama_vocab_fim_pre(const struct llama_vocab * vocab) {
|
||||
return vocab->token_fim_pre();
|
||||
}
|
||||
|
||||
@@ -142,33 +142,6 @@ static ggml_tensor * ggml_view_2d_slice(ggml_context * ctx0, ggml_tensor * x, in
|
||||
idx * x->ne[0] * x->ne[1] * ggml_element_size(x));
|
||||
}
|
||||
|
||||
// TODO @ngxson : maybe improve this in the future
|
||||
class llm_graph_input_logits_bias : public llm_graph_input_i {
|
||||
public:
|
||||
llm_graph_input_logits_bias(const llama_vocab & vocab) {
|
||||
arr.resize(vocab.n_tokens(), 0.0f);
|
||||
for (llama_token id : vocab.get_suppress_tokens()) {
|
||||
if (0 <= id && id < (int32_t)vocab.n_tokens()) {
|
||||
arr[id] = -INFINITY;
|
||||
}
|
||||
}
|
||||
}
|
||||
virtual ~llm_graph_input_logits_bias() = default;
|
||||
|
||||
void set_input(const llama_ubatch * /*ubatch*/) override {
|
||||
const int64_t n_vocab = arr.size();
|
||||
ggml_backend_tensor_set(logits_bias, arr.data(), 0, n_vocab*ggml_element_size(logits_bias));
|
||||
}
|
||||
|
||||
bool can_reuse(const llm_graph_params & /*params*/) override {
|
||||
return true;
|
||||
}
|
||||
|
||||
ggml_tensor * logits_bias = nullptr; // F32 [n_vocab]
|
||||
|
||||
std::vector<float> arr;
|
||||
};
|
||||
|
||||
llama_model_gemma4::graph::graph(const llama_model & model, const llm_graph_params & params) :
|
||||
llm_graph_context(params),
|
||||
model(model),
|
||||
@@ -429,16 +402,6 @@ llama_model_gemma4::graph::graph(const llama_model & model, const llm_graph_para
|
||||
cur = ggml_scale(ctx0, cur, hparams.f_final_logit_softcapping);
|
||||
}
|
||||
|
||||
// apply logits bias if needed (e.g. for gemma4_unified patch)
|
||||
// this is to mirror the suppress_tokens patch on transformers, to avoid model from outputing <image|> and <audio|> tokens (which is a known issue related to the checkpoint)
|
||||
// TODO: maybe handle this inside the sampling system in the future
|
||||
if (!model.vocab.get_suppress_tokens().empty()) {
|
||||
auto inp_bias = std::make_unique<llm_graph_input_logits_bias>(model.vocab);
|
||||
inp_bias->logits_bias = ggml_new_tensor_1d(ctx0, GGML_TYPE_F32, inp_bias->arr.size());
|
||||
cur = ggml_add(ctx0, cur, inp_bias->logits_bias);
|
||||
res->add_input(std::move(inp_bias));
|
||||
}
|
||||
|
||||
cb(cur, "result_output", -1);
|
||||
res->t_logits = cur;
|
||||
|
||||
|
||||
+181
-21
@@ -25,9 +25,13 @@ void llama_model_mimo2::load_arch_hparams(llama_model_loader & ml) {
|
||||
}
|
||||
}
|
||||
|
||||
void llama_model_mimo2::load_arch_tensors(llama_model_loader &) {
|
||||
void llama_model_mimo2::load_arch_tensors(llama_model_loader & ml) {
|
||||
LLAMA_LOAD_LOCALS;
|
||||
|
||||
const std::string mtp_probe = "blk." + std::to_string(n_layer) + ".nextn.eh_proj.weight";
|
||||
const bool trunk_only = (hparams.n_layer_nextn > 0) && (ml.get_weight(mtp_probe.c_str()) == nullptr);
|
||||
const int mtp_flags = trunk_only ? TENSOR_NOT_REQUIRED : 0;
|
||||
|
||||
tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0);
|
||||
|
||||
// output
|
||||
@@ -40,41 +44,46 @@ void llama_model_mimo2::load_arch_tensors(llama_model_loader &) {
|
||||
uint32_t n_embd_v_gqa = hparams.n_embd_v_gqa(i);
|
||||
uint32_t n_head = hparams.n_head(i);
|
||||
|
||||
// NextN/MTP layers (the last n_nextn blocks) are preserved but disabled pending support
|
||||
const bool is_nextn = i >= n_layer;
|
||||
const int skip = is_nextn ? TENSOR_SKIP : 0;
|
||||
const int flags = is_nextn ? mtp_flags : 0;
|
||||
|
||||
create_tensor_qkv(layer, i, n_embd, n_embd_head_k * n_head, n_embd_k_gqa, n_embd_v_gqa, skip);
|
||||
layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), { n_embd_head_v * n_head, n_embd }, skip);
|
||||
create_tensor_qkv(layer, i, n_embd, n_embd_head_k * n_head, n_embd_k_gqa, n_embd_v_gqa, flags);
|
||||
layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), { n_embd_head_v * n_head, n_embd }, flags);
|
||||
|
||||
layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, skip);
|
||||
layer.attn_sinks = create_tensor(tn(LLM_TENSOR_ATTN_SINKS, "weight", i), {n_head}, TENSOR_NOT_REQUIRED | skip);
|
||||
layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, flags);
|
||||
layer.attn_sinks = create_tensor(tn(LLM_TENSOR_ATTN_SINKS, "weight", i), {n_head}, TENSOR_NOT_REQUIRED | flags);
|
||||
|
||||
layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, skip);
|
||||
layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, flags);
|
||||
|
||||
// non-MoE branch
|
||||
layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff}, TENSOR_NOT_REQUIRED | skip);
|
||||
layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "weight", i), { n_ff, n_embd}, TENSOR_NOT_REQUIRED | skip);
|
||||
layer.ffn_up = create_tensor(tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}, TENSOR_NOT_REQUIRED | skip);
|
||||
layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff}, TENSOR_NOT_REQUIRED | flags);
|
||||
layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "weight", i), { n_ff, n_embd}, TENSOR_NOT_REQUIRED | flags);
|
||||
layer.ffn_up = create_tensor(tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}, TENSOR_NOT_REQUIRED | flags);
|
||||
|
||||
// MoE branch
|
||||
int64_t n_ff_exp = hparams.n_ff_exp;
|
||||
layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), {n_embd, n_expert}, TENSOR_NOT_REQUIRED | skip);
|
||||
layer.ffn_gate_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, "weight", i), {n_embd, n_ff_exp, n_expert}, TENSOR_NOT_REQUIRED | skip);
|
||||
layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), {n_ff_exp, n_embd, n_expert}, TENSOR_NOT_REQUIRED | skip);
|
||||
layer.ffn_up_exps = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS, "weight", i), {n_embd, n_ff_exp, n_expert}, TENSOR_NOT_REQUIRED | skip);
|
||||
layer.ffn_exp_probs_b = create_tensor(tn(LLM_TENSOR_FFN_EXP_PROBS_B, "bias", i), {n_expert}, TENSOR_NOT_REQUIRED | skip);
|
||||
layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), {n_embd, n_expert}, TENSOR_NOT_REQUIRED | flags);
|
||||
layer.ffn_gate_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, "weight", i), {n_embd, n_ff_exp, n_expert}, TENSOR_NOT_REQUIRED | flags);
|
||||
layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), {n_ff_exp, n_embd, n_expert}, TENSOR_NOT_REQUIRED | flags);
|
||||
layer.ffn_up_exps = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS, "weight", i), {n_embd, n_ff_exp, n_expert}, TENSOR_NOT_REQUIRED | flags);
|
||||
layer.ffn_exp_probs_b = create_tensor(tn(LLM_TENSOR_FFN_EXP_PROBS_B, "bias", i), {n_expert}, TENSOR_NOT_REQUIRED | flags);
|
||||
|
||||
if (is_nextn) {
|
||||
layer.nextn.eh_proj = create_tensor(tn(LLM_TENSOR_NEXTN_EH_PROJ, "weight", i), {2 * n_embd, n_embd}, skip);
|
||||
layer.nextn.enorm = create_tensor(tn(LLM_TENSOR_NEXTN_ENORM, "weight", i), {n_embd}, skip);
|
||||
layer.nextn.hnorm = create_tensor(tn(LLM_TENSOR_NEXTN_HNORM, "weight", i), {n_embd}, skip);
|
||||
layer.layer_out_norm = create_tensor(tn(LLM_TENSOR_LAYER_OUT_NORM, "weight", i), {n_embd}, skip);
|
||||
layer.nextn.eh_proj = create_tensor(tn(LLM_TENSOR_NEXTN_EH_PROJ, "weight", i), {2 * n_embd, n_embd}, flags);
|
||||
layer.nextn.enorm = create_tensor(tn(LLM_TENSOR_NEXTN_ENORM, "weight", i), {n_embd}, flags);
|
||||
layer.nextn.hnorm = create_tensor(tn(LLM_TENSOR_NEXTN_HNORM, "weight", i), {n_embd}, flags);
|
||||
layer.nextn.embed_tokens = create_tensor(tn(LLM_TENSOR_NEXTN_EMBED_TOKENS, "weight", i), {n_embd, n_vocab}, TENSOR_NOT_REQUIRED | flags);
|
||||
layer.nextn.shared_head_head = create_tensor(tn(LLM_TENSOR_NEXTN_SHARED_HEAD_HEAD, "weight", i), {n_embd, n_vocab}, TENSOR_NOT_REQUIRED | flags);
|
||||
layer.nextn.shared_head_norm = create_tensor(tn(LLM_TENSOR_NEXTN_SHARED_HEAD_NORM, "weight", i), {n_embd}, TENSOR_NOT_REQUIRED | flags);
|
||||
layer.layer_out_norm = create_tensor(tn(LLM_TENSOR_LAYER_OUT_NORM, "weight", i), {n_embd}, TENSOR_NOT_REQUIRED | flags);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
std::unique_ptr<llm_graph_context> llama_model_mimo2::build_arch_graph(const llm_graph_params & params) const {
|
||||
if (params.gtype == LLM_GRAPH_TYPE_DECODER_MTP) {
|
||||
return std::make_unique<graph_mtp>(*this, params);
|
||||
}
|
||||
return std::make_unique<graph>(*this, params);
|
||||
}
|
||||
|
||||
@@ -89,6 +98,8 @@ llama_model_mimo2::graph::graph(const llama_model & model, const llm_graph_param
|
||||
ggml_tensor * inp_out_ids = build_inp_out_ids();
|
||||
|
||||
const float v_scale = hparams.f_attn_value_scale;
|
||||
const bool emit_h_nextn = cparams.embeddings_nextn;
|
||||
const bool crop_last_layer = inp_out_ids && (!emit_h_nextn || cparams.embeddings_nextn_masked);
|
||||
|
||||
for (int il = 0; il < n_layer; ++il) {
|
||||
ggml_tensor * inpSA = inpL;
|
||||
@@ -168,7 +179,7 @@ llama_model_mimo2::graph::graph(const llama_model & model, const llm_graph_param
|
||||
}
|
||||
}
|
||||
|
||||
if (il == n_layer - 1 && inp_out_ids) {
|
||||
if (il == n_layer - 1 && crop_last_layer) {
|
||||
cur = ggml_get_rows(ctx0, cur, inp_out_ids);
|
||||
inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);
|
||||
}
|
||||
@@ -218,6 +229,15 @@ llama_model_mimo2::graph::graph(const llama_model & model, const llm_graph_param
|
||||
|
||||
cur = inpL;
|
||||
|
||||
if (emit_h_nextn) {
|
||||
cb(cur, "h_nextn", -1);
|
||||
res->t_h_nextn = cur;
|
||||
|
||||
if (!cparams.embeddings_nextn_masked && inp_out_ids) {
|
||||
cur = ggml_get_rows(ctx0, cur, inp_out_ids);
|
||||
}
|
||||
}
|
||||
|
||||
cur = build_norm(cur,
|
||||
model.output_norm, NULL,
|
||||
LLM_NORM_RMS, -1);
|
||||
@@ -233,3 +253,143 @@ llama_model_mimo2::graph::graph(const llama_model & model, const llm_graph_param
|
||||
|
||||
ggml_build_forward_expand(gf, cur);
|
||||
}
|
||||
|
||||
// Mirrors MiMo's appended NextN block: normalize and fuse token and hidden inputs, run the decoder block,
|
||||
// expose its pre-head-norm state to the next draft step, then apply the shared output norm and LM head.
|
||||
// Converted checkpoints may store that shared norm as layer_out_norm, so it remains in the fallback chain.
|
||||
llama_model_mimo2::graph_mtp::graph_mtp(const llama_model & model, const llm_graph_params & params)
|
||||
: llm_graph_context(params) {
|
||||
GGML_ASSERT(hparams.n_layer_nextn > 0 && "MIMO2 MTP requires n_layer_nextn > 0");
|
||||
|
||||
const int il = hparams.n_layer() + cparams.nextn_layer_offset;
|
||||
GGML_ASSERT(cparams.nextn_layer_offset >= 0 &&
|
||||
cparams.nextn_layer_offset < (int) hparams.n_layer_nextn &&
|
||||
"nextn_layer_offset out of range [0, n_layer_nextn)");
|
||||
|
||||
const auto & layer = model.layers[il];
|
||||
GGML_ASSERT(layer.nextn.eh_proj && "MIMO2 MTP block missing nextn.eh_proj");
|
||||
GGML_ASSERT(layer.nextn.enorm && "MIMO2 MTP block missing nextn.enorm");
|
||||
GGML_ASSERT(layer.nextn.hnorm && "MIMO2 MTP block missing nextn.hnorm");
|
||||
GGML_ASSERT(layer.wqkv && "MIMO2 MTP requires fused attn_qkv");
|
||||
|
||||
const uint32_t n_head_l = hparams.n_head(il);
|
||||
const uint32_t n_head_kv_l = hparams.n_head_kv(il);
|
||||
|
||||
const float freq_base_l = model.get_rope_freq_base(cparams, il);
|
||||
const float freq_scale_l = model.get_rope_freq_scale(cparams, il);
|
||||
const float v_scale = hparams.f_attn_value_scale;
|
||||
|
||||
auto inp = std::make_unique<llm_graph_input_embd>(hparams.n_embd);
|
||||
|
||||
inp->tokens = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_tokens);
|
||||
ggml_set_input(inp->tokens);
|
||||
|
||||
inp->embd = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, hparams.n_embd, n_tokens);
|
||||
ggml_set_input(inp->embd);
|
||||
ggml_set_name(inp->embd, "mtp_h_input");
|
||||
|
||||
ggml_tensor * tok_embd_w = layer.nextn.embed_tokens ? layer.nextn.embed_tokens : model.tok_embd;
|
||||
ggml_tensor * h_input = inp->embd;
|
||||
ggml_tensor * tok_embd = ggml_get_rows(ctx0, tok_embd_w, inp->tokens);
|
||||
cb(tok_embd, "mtp_tok_embd", il);
|
||||
|
||||
res->add_input(std::move(inp));
|
||||
|
||||
ggml_tensor * inp_pos = build_inp_pos();
|
||||
ggml_tensor * inp_out_ids = build_inp_out_ids();
|
||||
auto * inp_attn = build_attn_inp_kv_iswa();
|
||||
|
||||
ggml_tensor * h_norm = build_norm(h_input, layer.nextn.hnorm, nullptr, LLM_NORM_RMS, il);
|
||||
cb(h_norm, "mtp_hnorm", il);
|
||||
|
||||
ggml_tensor * e_norm = build_norm(tok_embd, layer.nextn.enorm, nullptr, LLM_NORM_RMS, il);
|
||||
cb(e_norm, "mtp_enorm", il);
|
||||
|
||||
ggml_tensor * concat = ggml_concat(ctx0, e_norm, h_norm, /*dim=*/ 0);
|
||||
cb(concat, "mtp_concat", il);
|
||||
|
||||
ggml_tensor * cur = build_lora_mm(layer.nextn.eh_proj, concat, layer.nextn.eh_proj_s);
|
||||
cb(cur, "mtp_eh_proj", il);
|
||||
|
||||
ggml_tensor * inpSA = cur;
|
||||
|
||||
cur = build_norm(cur, layer.attn_norm, nullptr, LLM_NORM_RMS, il);
|
||||
cb(cur, "mtp_attn_norm", il);
|
||||
|
||||
ggml_tensor * qkv = build_lora_mm(layer.wqkv, cur, layer.wqkv_s);
|
||||
cb(qkv, "mtp_wqkv", il);
|
||||
|
||||
const size_t row_k = ggml_row_size(qkv->type, n_embd_head_k);
|
||||
const size_t row_v = ggml_row_size(qkv->type, n_embd_head_v);
|
||||
const size_t row_full = qkv->nb[1];
|
||||
const size_t k_off = row_k * n_head_l;
|
||||
const size_t v_off = k_off + row_k * n_head_kv_l;
|
||||
|
||||
ggml_tensor * Qcur = ggml_view_3d(ctx0, qkv, n_embd_head_k, n_head_l, n_tokens, row_k, row_full, 0);
|
||||
ggml_tensor * Kcur = ggml_view_3d(ctx0, qkv, n_embd_head_k, n_head_kv_l, n_tokens, row_k, row_full, k_off);
|
||||
ggml_tensor * Vcur = ggml_view_3d(ctx0, qkv, n_embd_head_v, n_head_kv_l, n_tokens, row_v, row_full, v_off);
|
||||
|
||||
Qcur = ggml_rope_ext(
|
||||
ctx0, Qcur, inp_pos, nullptr,
|
||||
n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l,
|
||||
ext_factor, attn_factor, beta_fast, beta_slow);
|
||||
|
||||
Kcur = ggml_rope_ext(
|
||||
ctx0, Kcur, inp_pos, nullptr,
|
||||
n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l,
|
||||
ext_factor, attn_factor, beta_fast, beta_slow);
|
||||
|
||||
cb(Qcur, "mtp_Qcur", il);
|
||||
cb(Kcur, "mtp_Kcur", il);
|
||||
cb(Vcur, "mtp_Vcur", il);
|
||||
|
||||
cur = build_attn(inp_attn,
|
||||
layer.wo, nullptr, layer.wo_s,
|
||||
Qcur, Kcur, Vcur, nullptr, layer.attn_sinks, nullptr,
|
||||
1.0f / sqrtf(float(n_embd_head_k)), il);
|
||||
cb(cur, "mtp_attn_out", il);
|
||||
|
||||
if (v_scale) {
|
||||
cur = ggml_scale(ctx0, cur, v_scale);
|
||||
cb(cur, "mtp_attn_out_scaled", il);
|
||||
}
|
||||
|
||||
ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA);
|
||||
cb(ffn_inp, "mtp_ffn_inp", il);
|
||||
|
||||
cur = build_norm(ffn_inp, layer.ffn_norm, nullptr, LLM_NORM_RMS, il);
|
||||
cb(cur, "mtp_ffn_norm", il);
|
||||
|
||||
GGML_ASSERT(layer.ffn_gate && layer.ffn_down && layer.ffn_up && "MIMO2 MTP requires dense FFN tensors");
|
||||
cur = build_ffn(cur,
|
||||
layer.ffn_up, layer.ffn_up_b, nullptr,
|
||||
layer.ffn_gate, layer.ffn_gate_b, nullptr,
|
||||
layer.ffn_down, layer.ffn_down_b, nullptr,
|
||||
nullptr,
|
||||
LLM_FFN_SILU, LLM_FFN_PAR, il);
|
||||
cb(cur, "mtp_ffn_out", il);
|
||||
|
||||
cur = ggml_add(ctx0, cur, ffn_inp);
|
||||
cb(cur, "mtp_post_ffn", il);
|
||||
|
||||
cur = ggml_get_rows(ctx0, cur, inp_out_ids);
|
||||
|
||||
cb(cur, "h_nextn", -1);
|
||||
res->t_h_nextn = cur;
|
||||
|
||||
ggml_tensor * head_norm_w = layer.nextn.shared_head_norm
|
||||
? layer.nextn.shared_head_norm
|
||||
: (layer.layer_out_norm ? layer.layer_out_norm : model.output_norm);
|
||||
GGML_ASSERT(head_norm_w && "MIMO2 MTP missing head norm fallback");
|
||||
cur = build_norm(cur, head_norm_w, nullptr, LLM_NORM_RMS, -1);
|
||||
cb(cur, "mtp_shared_head_norm", -1);
|
||||
|
||||
ggml_tensor * head_w = layer.nextn.shared_head_head ? layer.nextn.shared_head_head : model.output;
|
||||
ggml_tensor * head_s = layer.nextn.shared_head_head ? layer.nextn.shared_head_head_s : model.output_s;
|
||||
GGML_ASSERT(head_w && "MIMO2 MTP missing LM head fallback");
|
||||
cur = build_lora_mm(head_w, cur, head_s);
|
||||
cb(cur, "result_output", -1);
|
||||
|
||||
res->t_logits = cur;
|
||||
ggml_build_forward_expand(gf, cur);
|
||||
}
|
||||
|
||||
+21
-72
@@ -2,7 +2,6 @@
|
||||
#include "llama-kv-cache.h"
|
||||
#include <cmath>
|
||||
#include <vector>
|
||||
#include <algorithm>
|
||||
#include <cstdint>
|
||||
|
||||
// MiniMax-M3: MiniMax-M2 style GQA (per-head QK-norm, partial rotary) with
|
||||
@@ -126,68 +125,6 @@ public:
|
||||
int64_t nblk;
|
||||
};
|
||||
|
||||
// pooled score of a block with no visible token: -inf from the mask, or -FLT_MAX from the
|
||||
// max-pool identity when every element of the block is -inf
|
||||
static inline bool msa_score_masked(float x) { return x <= -1e30f; }
|
||||
|
||||
// MSA block selection (batch regime)
|
||||
// CPU custom op, the token-level expansion and the combination with the causal mask happen on the GPU.
|
||||
static void msa_block_mask_op(struct ggml_tensor * dst, int ith, int nth, void * userdata) {
|
||||
const struct ggml_tensor * bs = dst->src[0];
|
||||
const struct ggml_tensor * bias = dst->src[1];
|
||||
const msa_params * p = (const msa_params *) userdata;
|
||||
|
||||
const int nblk = (int) bs->ne[0];
|
||||
const int Hd = (int) bs->ne[1];
|
||||
const int S = (int) bs->ne[2];
|
||||
|
||||
GGML_ASSERT(bs->type == GGML_TYPE_F32 && ggml_is_contiguous(bs));
|
||||
GGML_ASSERT(bias->type == GGML_TYPE_F32 && ggml_is_contiguous(bias));
|
||||
GGML_ASSERT(dst->type == GGML_TYPE_F16 && ggml_is_contiguous(dst));
|
||||
GGML_ASSERT(dst->ne[0] == nblk && dst->ne[1] == S && dst->ne[2] == Hd);
|
||||
GGML_ASSERT(bias->ne[0] == nblk && bias->ne[1] == S);
|
||||
|
||||
const int topk = p->topk_blocks < nblk ? p->topk_blocks : nblk;
|
||||
|
||||
const ggml_fp16_t f16_zero = ggml_fp32_to_fp16(0.0f);
|
||||
const ggml_fp16_t f16_ninf = ggml_fp32_to_fp16(-INFINITY);
|
||||
|
||||
std::vector<float> rank(nblk);
|
||||
std::vector<char> valid(nblk);
|
||||
std::vector<int> ord(nblk);
|
||||
|
||||
ggml_fp16_t * out = (ggml_fp16_t *) dst->data;
|
||||
|
||||
for (int i = ith; i < S; i += nth) {
|
||||
const float * bias_col = (const float *) bias->data + (size_t) i * nblk;
|
||||
for (int h = 0; h < Hd; ++h) {
|
||||
const float * bs_col = (const float *) bs->data + ((size_t) i * Hd + h) * nblk;
|
||||
|
||||
for (int bk = 0; bk < nblk; ++bk) {
|
||||
// a block is selectable if it has a visible token or is locally forced
|
||||
valid[bk] = !msa_score_masked(bs_col[bk]) || bias_col[bk] > 0.0f;
|
||||
rank [bk] = bias_col[bk] > 0.0f ? bias_col[bk] : bs_col[bk];
|
||||
ord [bk] = bk;
|
||||
}
|
||||
|
||||
std::partial_sort(ord.begin(), ord.begin() + topk, ord.end(),
|
||||
[&](int a, int b) { return rank[a] > rank[b]; });
|
||||
|
||||
ggml_fp16_t * dst_col = out + ((size_t) h * S + i) * nblk;
|
||||
for (int bk = 0; bk < nblk; ++bk) {
|
||||
dst_col[bk] = f16_ninf;
|
||||
}
|
||||
for (int t = 0; t < topk; ++t) {
|
||||
const int bk = ord[t];
|
||||
if (!valid[bk]) {
|
||||
break; // sorted desc: first invalid -> fewer than topk selectable blocks
|
||||
}
|
||||
dst_col[bk] = f16_zero;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// One FA call for all GQA groups (and at multi-stream decode, all streams) by mapping them onto the FA sequence dim (ne[3])
|
||||
ggml_tensor * llama_model_minimax_m3::graph::build_attn_msa_fa(
|
||||
ggml_tensor * q_cur, // [D, HQ, T]
|
||||
@@ -433,8 +370,8 @@ llama_model_minimax_m3::graph::graph(const llama_model & model, const llm_graph_
|
||||
msa_mf->nb[1], msa_mf->nb[1], st*msa_mf->nb[3]);
|
||||
ggml_tensor * km_s = ggml_view_3d(ctx0, msa_kqm, n_kv, n_tps, 1,
|
||||
msa_kqm->nb[1], msa_kqm->nb[3], st*msa_kqm->nb[3]);
|
||||
ggml_tensor * bias_s = ggml_view_2d(ctx0, msa_loc->bias, nblk, n_tps,
|
||||
msa_loc->bias->nb[1], st*n_tps*msa_loc->bias->nb[1]);
|
||||
ggml_tensor * bias_s = ggml_view_3d(ctx0, msa_loc->bias, nblk, 1, n_tps,
|
||||
msa_loc->bias->nb[1], msa_loc->bias->nb[1], st*n_tps*msa_loc->bias->nb[1]);
|
||||
ggml_tensor * q_s = ggml_view_3d(ctx0, Qcur, D, n_head, n_tps,
|
||||
Qcur->nb[1], Qcur->nb[2], st*n_tps*Qcur->nb[2]);
|
||||
ggml_tensor * k_s = ggml_view_4d(ctx0, k, D, HKV, n_kv, 1,
|
||||
@@ -453,15 +390,27 @@ llama_model_minimax_m3::graph::graph(const llama_model & model, const llm_graph_
|
||||
ggml_tensor * bs = ggml_pool_2d(ctx0, sc, GGML_OP_POOL_MAX, blk, 1, blk, 1, 0, 0);
|
||||
cb(bs, "msa_bs", il);
|
||||
|
||||
// block-level 0/-inf keep mask on the CPU, tiny transfer
|
||||
ggml_tensor * srcs[2] = { bs, bias_s };
|
||||
ggml_tensor * bm = ggml_custom_4d(ctx0, GGML_TYPE_F16,
|
||||
nblk, n_tps, Hd, 1,
|
||||
srcs, 2, msa_block_mask_op, GGML_N_TASKS_MAX,
|
||||
const_cast<msa_params *>(&mm.msa_p));
|
||||
// bias the scores so locally-forced blocks always rank first
|
||||
ggml_tensor * bsf = ggml_add(ctx0, bs, bias_s); // [nblk, Hd, n_tps]
|
||||
cb(bsf, "msa_bsf", il);
|
||||
|
||||
ggml_tensor * idx = ggml_top_k(ctx0, bsf, K); // [K, Hd, n_tps] i32
|
||||
|
||||
ggml_tensor * ninf = ggml_cast(ctx0,
|
||||
ggml_scale_bias(ctx0, bias_s, 0.0f, -1e30f),
|
||||
GGML_TYPE_F16); // [nblk, 1, n_tps]
|
||||
ninf = ggml_repeat_4d(ctx0, ninf, nblk, Hd, n_tps, 1);
|
||||
ggml_tensor * zero = ggml_scale(ctx0,
|
||||
ggml_cast(ctx0, idx, GGML_TYPE_F32), 0.0f);
|
||||
ggml_tensor * bm = ggml_set_rows(ctx0,
|
||||
ggml_reshape_3d(ctx0, ninf, 1, nblk, Hd*n_tps),
|
||||
ggml_reshape_3d(ctx0, zero, 1, K, Hd*n_tps),
|
||||
ggml_reshape_2d(ctx0, idx, K, Hd*n_tps));
|
||||
bm = ggml_reshape_3d(ctx0, bm, nblk, Hd, n_tps);
|
||||
bm = ggml_cont(ctx0, ggml_permute(ctx0, bm, 0, 2, 1, 3)); // [nblk, n_tps, Hd]
|
||||
cb(bm, "msa_block_mask", il);
|
||||
|
||||
// expand block -> token granularity on the GPU (j = bk*blk + t),
|
||||
// expand block -> token granularity (j = bk*blk + t),
|
||||
// then combine with the causal mask in place
|
||||
ggml_tensor * bmx = ggml_repeat_4d(ctx0,
|
||||
ggml_reshape_3d(ctx0, bm, 1, nblk, n_tps*Hd),
|
||||
|
||||
@@ -2127,6 +2127,10 @@ struct llama_model_mimo2 : public llama_model_base {
|
||||
graph(const llama_model & model, const llm_graph_params & params);
|
||||
};
|
||||
|
||||
struct graph_mtp : public llm_graph_context {
|
||||
graph_mtp(const llama_model & model, const llm_graph_params & params);
|
||||
};
|
||||
|
||||
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
|
||||
};
|
||||
|
||||
|
||||
@@ -278,6 +278,9 @@ set_tests_properties(test-state-restore-fragmented PROPERTIES FIXTURES_REQUIRED
|
||||
llama_build_and_test(test-save-load-state.cpp LABEL "model" ARGS -m "${MODEL_DEST}")
|
||||
set_tests_properties(test-save-load-state PROPERTIES FIXTURES_REQUIRED test-download-model)
|
||||
|
||||
if (APPLE)
|
||||
llama_build(test-rset-release.cpp get-model.cpp)
|
||||
endif()
|
||||
if (NOT GGML_BACKEND_DL)
|
||||
# these tests use the backends directly and cannot be built with dynamic loading
|
||||
llama_build_and_test(test-barrier.cpp)
|
||||
|
||||
@@ -1350,18 +1350,22 @@ struct test_case {
|
||||
|
||||
// check if the backends support the ops
|
||||
bool supported = true;
|
||||
std::string unsupported_str;
|
||||
for (ggml_backend_t backend : {backend1, backend2}) {
|
||||
for (ggml_tensor * t = ggml_get_first_tensor(ctx.get()); t != NULL; t = ggml_get_next_tensor(ctx.get(), t)) {
|
||||
if (!ggml_backend_supports_op(backend, t)) {
|
||||
supported = false;
|
||||
break;
|
||||
if (unsupported_str.empty()) {
|
||||
unsupported_str = std::string(ggml_backend_name(backend));
|
||||
} else {
|
||||
unsupported_str += ", " + std::string(ggml_backend_name(backend));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!supported) {
|
||||
// Create test result for unsupported operation
|
||||
test_result result(ggml_backend_name(backend1), current_op_name, vars(), "test",
|
||||
test_result result(unsupported_str, current_op_name, vars(), "test",
|
||||
false, false, "not supported");
|
||||
|
||||
print_test_result_locked(output_printer, result);
|
||||
@@ -8009,6 +8013,7 @@ static const ggml_type other_types[] = {
|
||||
GGML_TYPE_Q5_0, GGML_TYPE_Q5_1,
|
||||
GGML_TYPE_Q8_0,
|
||||
GGML_TYPE_Q1_0,
|
||||
GGML_TYPE_Q2_0,
|
||||
GGML_TYPE_Q2_K, GGML_TYPE_Q3_K,
|
||||
GGML_TYPE_Q5_K,
|
||||
GGML_TYPE_Q6_K,
|
||||
@@ -9559,6 +9564,10 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
|
||||
test_cases.emplace_back(new test_flash_attn_ext(128, 64, 4, {1, 1}, 128, 2, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q1_0, GGML_TYPE_Q4_0));
|
||||
test_cases.emplace_back(new test_flash_attn_ext(64, 128, 4, {1, 1}, 128, 2, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q4_0, GGML_TYPE_Q1_0));
|
||||
test_cases.emplace_back(new test_flash_attn_ext(128, 64, 4, {1, 1}, 64, 2, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q1_0, GGML_TYPE_F16));
|
||||
test_cases.emplace_back(new test_flash_attn_ext(128, 128, 4, {1, 1}, 96, 2, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q2_0, GGML_TYPE_Q2_0));
|
||||
test_cases.emplace_back(new test_flash_attn_ext(128, 64, 4, {1, 1}, 128, 2, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q2_0, GGML_TYPE_Q4_0));
|
||||
test_cases.emplace_back(new test_flash_attn_ext(64, 128, 4, {1, 1}, 128, 2, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q4_0, GGML_TYPE_Q2_0));
|
||||
test_cases.emplace_back(new test_flash_attn_ext(128, 64, 4, {1, 1}, 64, 2, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q2_0, GGML_TYPE_F16));
|
||||
|
||||
// large-KV F16 cases (Qwen3.6-27B geometry and a llama-class control): the upstream matrix
|
||||
// stops at kv=1024, blind to long-context FA bugs (e.g. the oneDNN SDPA ordering race on BMG).
|
||||
|
||||
@@ -0,0 +1,53 @@
|
||||
// ref: https://github.com/ggml-org/llama.cpp/issues/25937
|
||||
// only works reliably when run with a large model that occupies 3GB+ of wired memory
|
||||
// thus, this test is not run by default
|
||||
// example model to run with: google/gemma-4-E4B-it-qat-q4_0-gguf
|
||||
|
||||
#include <cstdint>
|
||||
#include <mach/mach.h>
|
||||
#include <mach/mach_host.h>
|
||||
#include <unistd.h>
|
||||
|
||||
#include "llama.h"
|
||||
#include "get-model.h"
|
||||
|
||||
static uint64_t wired_memory() {
|
||||
vm_statistics64_data_t vmstat;
|
||||
mach_msg_type_number_t count = HOST_VM_INFO64_COUNT;
|
||||
if (host_statistics64(mach_host_self(), HOST_VM_INFO64, (host_info64_t)&vmstat, &count) != KERN_SUCCESS) {
|
||||
return UINT64_MAX;
|
||||
}
|
||||
return static_cast<uint64_t>(vmstat.wire_count) * vm_kernel_page_size;
|
||||
}
|
||||
|
||||
int main(int argc, char ** argv) {
|
||||
auto * model_path = get_model_or_exit(argc, argv);
|
||||
|
||||
llama_backend_init();
|
||||
|
||||
const uint64_t wired_initial = wired_memory();
|
||||
|
||||
llama_model_params params = llama_model_default_params();
|
||||
params.load_mode = LLAMA_LOAD_MODE_NONE;
|
||||
struct llama_model* model = llama_model_load_from_file(model_path, params);
|
||||
|
||||
const uint64_t wired_loaded = wired_memory();
|
||||
const uint64_t wired_delta = wired_loaded - wired_initial;
|
||||
// system memory fluctuates, so we need to allocate enough to reliably detect the release
|
||||
GGML_ASSERT(wired_delta > 2'000'000'000); // 2GB
|
||||
|
||||
llama_model_free(model);
|
||||
|
||||
const uint64_t t_start_ms = ggml_time_ms();
|
||||
|
||||
// expect most of the allocated memory to be released within 10 seconds
|
||||
// we allow for some tolerance due to system-wide memory fluctuations
|
||||
while (wired_memory() > wired_loaded - 0.75 * wired_delta) {
|
||||
GGML_ASSERT(ggml_time_ms() - t_start_ms < 10'000);
|
||||
usleep(100'000); // 100ms
|
||||
}
|
||||
|
||||
llama_backend_free();
|
||||
|
||||
return 0;
|
||||
}
|
||||
@@ -1537,7 +1537,7 @@ private:
|
||||
|
||||
// find the slot that has at least n% prompt similarity
|
||||
if (slot_prompt_similarity != 0.0f) {
|
||||
float sim_best = 0;
|
||||
float f_sim_best = 0;
|
||||
|
||||
for (server_slot & slot : slots) {
|
||||
if (task.id_slot != -1 && slot.id != task.id_slot) {
|
||||
@@ -1546,6 +1546,7 @@ private:
|
||||
|
||||
// skip the slot if it is not available
|
||||
if (slot.is_processing()) {
|
||||
SLT_TRC(slot, " - skipping, is_processing = %d\n", slot.is_processing());
|
||||
continue;
|
||||
}
|
||||
|
||||
@@ -1553,26 +1554,30 @@ private:
|
||||
|
||||
// skip the slot if it does not contains cached tokens
|
||||
if (tokens.empty()) {
|
||||
SLT_TRC(slot, "%s", " - skipping, slot is empty\n");
|
||||
continue;
|
||||
}
|
||||
|
||||
// fraction of the Longest Common Prefix length with respect to the input prompt length
|
||||
const float sim_cur = float(tokens.get_common_prefix(task.tokens)) / task.tokens.size();
|
||||
const size_t lcp_len = tokens.get_common_prefix(task.tokens);
|
||||
const float f_sim_cur = float(lcp_len) / task.tokens.size();
|
||||
|
||||
SLT_TRC(slot, " - checking sim = %.3f (%zu/%zu) > %.3f\n", f_sim_cur, lcp_len, task.tokens.size(), slot_prompt_similarity);
|
||||
|
||||
// select the current slot if the criteria match
|
||||
if (sim_cur > sim_best && sim_cur > slot_prompt_similarity) {
|
||||
sim_best = sim_cur;
|
||||
if (f_sim_cur > f_sim_best && f_sim_cur > slot_prompt_similarity) {
|
||||
f_sim_best = f_sim_cur;
|
||||
|
||||
ret = &slot;
|
||||
}
|
||||
}
|
||||
|
||||
if (ret != nullptr) {
|
||||
const float f_keep = (sim_best*task.tokens.size()) / ret->prompt.tokens.size();
|
||||
const float f_keep = (f_sim_best*task.tokens.size()) / ret->prompt.tokens.size();
|
||||
|
||||
if (task.id_slot == -1) {
|
||||
SLT_INF(*ret, "selected slot by LCP similarity, sim_best = %.3f (> %.3f thold), f_keep = %.3f\n",
|
||||
sim_best, slot_prompt_similarity, f_keep);
|
||||
SLT_INF(*ret, "selected slot by LCP similarity, f_sim_best = %.3f (> %.3f thold), f_keep = %.3f\n",
|
||||
f_sim_best, slot_prompt_similarity, f_keep);
|
||||
}
|
||||
|
||||
// if we are about to lose a large portion of the existing context - save it in the prompt cache
|
||||
|
||||
@@ -1742,9 +1742,9 @@ bool server_prompt_cache::load(server_prompt & prompt, const server_tokens & tok
|
||||
const int lcp_best = prompt.tokens.get_common_prefix(tokens_new);
|
||||
|
||||
float f_keep_best = prompt.tokens.size() > 0 ? float(lcp_best) / prompt.tokens.size() : -1.0f; // empty slot: any cache entry wins
|
||||
float sim_best = float(lcp_best) / tokens_new.size();
|
||||
float f_sim_best = float(lcp_best) / tokens_new.size();
|
||||
|
||||
SRV_TRC(" - looking for better prompt, base f_keep = %.3f, sim = %.3f\n", f_keep_best, sim_best);
|
||||
SRV_TRC(" - looking for better prompt, base f_keep = %.3f, f_sim = %.3f\n", f_keep_best, f_sim_best);
|
||||
|
||||
auto it_best = states.end();
|
||||
|
||||
@@ -1753,25 +1753,25 @@ bool server_prompt_cache::load(server_prompt & prompt, const server_tokens & tok
|
||||
const int lcp_cur = it->prompt.tokens.get_common_prefix(tokens_new);
|
||||
|
||||
const float f_keep_cur = float(lcp_cur) / it->prompt.tokens.size();
|
||||
const float sim_cur = float(lcp_cur) / tokens_new.size();
|
||||
const float f_sim_cur = float(lcp_cur) / tokens_new.size();
|
||||
|
||||
SRV_TRC(" - prompt with length %7zu, lcp = %7d, f_keep = %.3f, sim = %.3f\n", it->prompt.tokens.size(), lcp_cur, f_keep_cur, sim_cur);
|
||||
SRV_TRC(" - prompt with length %7zu, lcp = %7d, f_keep = %.3f, f_sim = %.3f\n", it->prompt.tokens.size(), lcp_cur, f_keep_cur, f_sim_cur);
|
||||
|
||||
// don't trash large prompts
|
||||
if (f_keep_cur < 0.25f) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (f_keep_best < f_keep_cur && sim_best < sim_cur) {
|
||||
if (f_keep_best < f_keep_cur && f_sim_best < f_sim_cur) {
|
||||
f_keep_best = f_keep_cur;
|
||||
sim_best = sim_cur;
|
||||
f_sim_best = f_sim_cur;
|
||||
|
||||
it_best = it;
|
||||
}
|
||||
}
|
||||
|
||||
if (it_best != states.end()) {
|
||||
SRV_TRC(" - found better prompt with f_keep = %.3f, sim = %.3f\n", f_keep_best, sim_best);
|
||||
SRV_TRC(" - found better prompt with f_keep = %.3f, f_sim = %.3f\n", f_keep_best, f_sim_best);
|
||||
|
||||
{
|
||||
auto & data = it_best->data.main;
|
||||
|
||||
+17
-2
@@ -48,6 +48,9 @@
|
||||
}: Props = $props();
|
||||
|
||||
let dropdownOpen = $state(false);
|
||||
// The system message action moves focus to the message editor, so the menu
|
||||
// must not restore focus to the trigger on close
|
||||
let suppressCloseAutoFocus = false;
|
||||
|
||||
function handleMcpSettingsClick() {
|
||||
dropdownOpen = false;
|
||||
@@ -96,7 +99,16 @@
|
||||
</Tooltip.Content>
|
||||
</Tooltip.Root>
|
||||
|
||||
<DropdownMenu.Content align="start" class="w-52">
|
||||
<DropdownMenu.Content
|
||||
align="start"
|
||||
class="w-52"
|
||||
onCloseAutoFocus={(e) => {
|
||||
if (suppressCloseAutoFocus) {
|
||||
suppressCloseAutoFocus = false;
|
||||
e.preventDefault();
|
||||
}
|
||||
}}
|
||||
>
|
||||
<ChatFormActionAddReasoningSubmenu />
|
||||
|
||||
<DropdownMenu.Separator />
|
||||
@@ -148,7 +160,10 @@
|
||||
|
||||
<DropdownMenu.Item
|
||||
class="flex cursor-pointer items-center gap-2"
|
||||
onclick={onSystemPromptClick}
|
||||
onclick={() => {
|
||||
suppressCloseAutoFocus = true;
|
||||
onSystemPromptClick?.();
|
||||
}}
|
||||
>
|
||||
<MessageSquare class={ICON_CLASS_DEFAULT} />
|
||||
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
import { goto } from '$app/navigation';
|
||||
import { getChatActionsContext, setMessageEditContext } from '$lib/contexts';
|
||||
import { chatStore, pendingEditMessageId } from '$lib/stores/chat.svelte';
|
||||
import { isMobile } from '$lib/stores/viewport.svelte';
|
||||
import { conversationsStore } from '$lib/stores/conversations.svelte';
|
||||
import { DatabaseService } from '$lib/services/database.service';
|
||||
import { SYSTEM_MESSAGE_PLACEHOLDER } from '$lib/constants';
|
||||
@@ -46,7 +47,14 @@
|
||||
assistantMessages: number;
|
||||
messageTypes: string[];
|
||||
} | null>(null);
|
||||
let editedContent = $derived(message.content);
|
||||
// The system message placeholder must never surface as editable content; keeping
|
||||
// it in the derived (not just in handleEdit) guards against prop invalidation
|
||||
// reverting the override while editing
|
||||
let editedContent = $derived(
|
||||
message.role === MessageRole.SYSTEM && message.content === SYSTEM_MESSAGE_PLACEHOLDER
|
||||
? ''
|
||||
: message.content
|
||||
);
|
||||
|
||||
let rawEditContent = $derived.by(() => {
|
||||
if (message.role !== MessageRole.ASSISTANT) return undefined;
|
||||
@@ -265,6 +273,12 @@
|
||||
chatActions.navigateToSibling(siblingId);
|
||||
}
|
||||
|
||||
// After the system message flow ends, hand focus to the main chat form
|
||||
function focusMainChatForm() {
|
||||
if (isMobile.current) return;
|
||||
document.querySelector<HTMLTextAreaElement>('.chat-screen-form-wrapper textarea')?.focus();
|
||||
}
|
||||
|
||||
async function handleSaveEdit() {
|
||||
if (message.role === MessageRole.SYSTEM) {
|
||||
// System messages: update in place without branching
|
||||
@@ -276,6 +290,8 @@
|
||||
isEditing = false;
|
||||
if (conversationDeleted) {
|
||||
goto(ROUTES.START);
|
||||
} else {
|
||||
focusMainChatForm();
|
||||
}
|
||||
return;
|
||||
}
|
||||
@@ -285,6 +301,7 @@
|
||||
if (index !== -1) {
|
||||
conversationsStore.updateMessageAtIndex(index, { content: newContent });
|
||||
}
|
||||
focusMainChatForm();
|
||||
} else if (message.role === MessageRole.USER) {
|
||||
const finalExtras = await getMergedExtras();
|
||||
chatActions.editWithBranching(message, editedContent.trim(), finalExtras);
|
||||
|
||||
@@ -106,15 +106,23 @@
|
||||
onFileRemove?.(fileId);
|
||||
}
|
||||
|
||||
// Auto-focus must not steal focus already claimed elsewhere (e.g. the system
|
||||
// message editor opened just before a navigation)
|
||||
function focusFormUnlessCaptured() {
|
||||
const active = document.activeElement;
|
||||
if (active instanceof HTMLTextAreaElement || active instanceof HTMLInputElement) return;
|
||||
chatFormRef?.focus();
|
||||
}
|
||||
|
||||
onMount(() => {
|
||||
if (!isMobile.current) {
|
||||
setTimeout(() => chatFormRef?.focus(), 100);
|
||||
setTimeout(focusFormUnlessCaptured, 100);
|
||||
}
|
||||
});
|
||||
|
||||
afterNavigate((navigation) => {
|
||||
if (navigation?.from != null && !isMobile.current) {
|
||||
setTimeout(() => chatFormRef?.focus(), 100);
|
||||
setTimeout(focusFormUnlessCaptured, 100);
|
||||
}
|
||||
});
|
||||
|
||||
@@ -127,7 +135,7 @@
|
||||
|
||||
$effect(() => {
|
||||
if (previousIsLoading && !isLoading) {
|
||||
setTimeout(() => chatFormRef?.focus(), 10);
|
||||
setTimeout(focusFormUnlessCaptured, 10);
|
||||
}
|
||||
|
||||
previousIsLoading = isLoading;
|
||||
|
||||
@@ -11,7 +11,7 @@
|
||||
<div
|
||||
class={[
|
||||
'pointer-events-none mb-4 hidden px-4 text-center text-balance',
|
||||
isEmpty && 'mb-[calc(50dvh-8rem)] md:mb-6 pointer-events-auto block!'
|
||||
isEmpty && 'mb-[calc(50dvh-8rem)] md:mb-8 pointer-events-auto block!'
|
||||
]}
|
||||
>
|
||||
<h1 class="mb-2 text-2xl font-semibold tracking-tight md:text-3xl">Hello there</h1>
|
||||
|
||||
@@ -31,14 +31,19 @@ export class DatabaseService {
|
||||
* Creates a new conversation.
|
||||
*
|
||||
* @param name - Name of the conversation
|
||||
* @param fields - Optional extra fields (e.g. reasoningEffort)
|
||||
* @returns The created conversation
|
||||
*/
|
||||
static async createConversation(name: string): Promise<DatabaseConversation> {
|
||||
static async createConversation(
|
||||
name: string,
|
||||
fields?: Partial<Omit<DatabaseConversation, 'id' | 'name' | 'lastModified'>>
|
||||
): Promise<DatabaseConversation> {
|
||||
const conversation: DatabaseConversation = {
|
||||
id: uuid(),
|
||||
name,
|
||||
lastModified: Date.now(),
|
||||
currNode: ''
|
||||
currNode: '',
|
||||
...fields
|
||||
};
|
||||
|
||||
await db[IDXDB_TABLES.conversations].add(conversation);
|
||||
@@ -137,7 +142,7 @@ export class DatabaseService {
|
||||
* @param systemPrompt - The system prompt content (must be non-empty)
|
||||
* @param parentId - Parent message ID (typically the root message)
|
||||
* @returns The created system message
|
||||
* @throws Error if systemPrompt is empty
|
||||
* @throws Error if systemPrompt is empty or the parent message does not exist
|
||||
*/
|
||||
static async createSystemMessage(
|
||||
convId: string,
|
||||
@@ -149,27 +154,30 @@ export class DatabaseService {
|
||||
throw new Error('Cannot create system message with empty content');
|
||||
}
|
||||
|
||||
const systemMessage: DatabaseMessage = {
|
||||
id: uuid(),
|
||||
convId,
|
||||
type: MessageRole.SYSTEM,
|
||||
timestamp: Date.now(),
|
||||
role: MessageRole.SYSTEM,
|
||||
content: trimmedPrompt,
|
||||
parent: parentId,
|
||||
children: []
|
||||
};
|
||||
return await db.transaction('rw', db[IDXDB_TABLES.messages], async () => {
|
||||
const parentMessage = await db[IDXDB_TABLES.messages].get(parentId);
|
||||
if (!parentMessage) {
|
||||
throw new Error(`Parent message ${parentId} not found`);
|
||||
}
|
||||
|
||||
await db[IDXDB_TABLES.messages].add(systemMessage);
|
||||
const systemMessage: DatabaseMessage = {
|
||||
id: uuid(),
|
||||
convId,
|
||||
type: MessageRole.SYSTEM,
|
||||
timestamp: Date.now(),
|
||||
role: MessageRole.SYSTEM,
|
||||
content: trimmedPrompt,
|
||||
parent: parentId,
|
||||
children: []
|
||||
};
|
||||
|
||||
const parentMessage = await db[IDXDB_TABLES.messages].get(parentId);
|
||||
if (parentMessage) {
|
||||
await db[IDXDB_TABLES.messages].add(systemMessage);
|
||||
await db[IDXDB_TABLES.messages].update(parentId, {
|
||||
children: [...parentMessage.children, systemMessage.id]
|
||||
});
|
||||
}
|
||||
|
||||
return systemMessage;
|
||||
return systemMessage;
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -442,7 +450,8 @@ export class DatabaseService {
|
||||
}
|
||||
|
||||
/**
|
||||
* Updates a conversation.
|
||||
* Updates a conversation. `lastModified` is never stamped implicitly;
|
||||
* pass it in `updates` to bump the conversation in recency ordering.
|
||||
*
|
||||
* @param id - Conversation ID
|
||||
* @param updates - Partial updates to apply
|
||||
@@ -452,10 +461,7 @@ export class DatabaseService {
|
||||
id: string,
|
||||
updates: Partial<Omit<DatabaseConversation, 'id'>>
|
||||
): Promise<void> {
|
||||
await db[IDXDB_TABLES.conversations].update(id, {
|
||||
...updates,
|
||||
lastModified: Date.now()
|
||||
});
|
||||
await db[IDXDB_TABLES.conversations].update(id, updates);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -473,7 +479,7 @@ export class DatabaseService {
|
||||
* @returns The new pinned status
|
||||
*/
|
||||
static async toggleConversationPin(id: string): Promise<boolean> {
|
||||
const conversation = await db.conversations.get(id);
|
||||
const conversation = await db[IDXDB_TABLES.conversations].get(id);
|
||||
if (!conversation) {
|
||||
throw new Error(`Conversation ${id} not found`);
|
||||
}
|
||||
@@ -497,7 +503,6 @@ export class DatabaseService {
|
||||
const result = new Map<string, boolean>();
|
||||
if (cleanIds.length === 0) return result;
|
||||
|
||||
const now = Date.now();
|
||||
await db.transaction('rw', db[IDXDB_TABLES.conversations], async () => {
|
||||
const convs = await db[IDXDB_TABLES.conversations].bulkGet(cleanIds);
|
||||
const updates: DatabaseConversation[] = [];
|
||||
@@ -505,7 +510,7 @@ export class DatabaseService {
|
||||
const conv = convs[i];
|
||||
if (!conv) continue;
|
||||
const newPinned = !conv.pinned;
|
||||
updates.push({ ...conv, pinned: newPinned, lastModified: now });
|
||||
updates.push({ ...conv, pinned: newPinned });
|
||||
result.set(cleanIds[i], newPinned);
|
||||
}
|
||||
if (updates.length === 0) return;
|
||||
|
||||
@@ -1658,7 +1658,8 @@ class ChatStore {
|
||||
generateConversationTitle(newContent, Boolean(config().titleGenerationUseFirstLine))
|
||||
);
|
||||
const messagesToRemove = conversationsStore.activeMessages.slice(messageIndex + 1);
|
||||
for (const message of messagesToRemove) await DatabaseService.deleteMessage(message.id);
|
||||
if (messagesToRemove.length > 0)
|
||||
await DatabaseService.deleteMessageCascading(activeConv.id, messagesToRemove[0].id);
|
||||
conversationsStore.sliceActiveMessages(messageIndex + 1);
|
||||
conversationsStore.updateConversationTimestamp();
|
||||
this.setChatLoading(activeConv.id, true);
|
||||
@@ -1690,7 +1691,7 @@ class ChatStore {
|
||||
const { index: messageIndex } = result;
|
||||
try {
|
||||
const messagesToRemove = conversationsStore.activeMessages.slice(messageIndex);
|
||||
for (const message of messagesToRemove) await DatabaseService.deleteMessage(message.id);
|
||||
await DatabaseService.deleteMessageCascading(activeConv.id, messagesToRemove[0].id);
|
||||
conversationsStore.sliceActiveMessages(messageIndex);
|
||||
conversationsStore.updateConversationTimestamp();
|
||||
this.setChatLoading(activeConv.id, true);
|
||||
@@ -2037,7 +2038,7 @@ class ChatStore {
|
||||
timings
|
||||
});
|
||||
|
||||
conversationsStore.updateConversationTimestamp();
|
||||
conversationsStore.updateConversationTimestamp(msg.convId);
|
||||
|
||||
this.setChatLoading(msg.convId, false);
|
||||
this.clearChatStreaming(msg.convId);
|
||||
|
||||
@@ -111,6 +111,9 @@ class ConversationsStore {
|
||||
| ((messageId: string, updates: Partial<DatabaseMessage>) => void)
|
||||
| null = null;
|
||||
|
||||
/** In-flight init run; shared by concurrent callers, reset on failure to allow retry */
|
||||
private initPromise: Promise<void> | null = null;
|
||||
|
||||
/**
|
||||
*
|
||||
*
|
||||
@@ -121,19 +124,25 @@ class ConversationsStore {
|
||||
|
||||
/**
|
||||
* Initialize the store by loading conversations from database.
|
||||
* Must be called once after app startup.
|
||||
* Safe to call multiple times: concurrent callers share a single run,
|
||||
* and a failed run can be retried by calling again.
|
||||
*/
|
||||
async init(): Promise<void> {
|
||||
if (!browser) return;
|
||||
if (this.isInitialized) return;
|
||||
init(): Promise<void> {
|
||||
if (!browser) return Promise.resolve();
|
||||
if (this.initPromise) return this.initPromise;
|
||||
|
||||
try {
|
||||
await MigrationService.runAllMigrations();
|
||||
await this.loadConversations();
|
||||
this.isInitialized = true;
|
||||
} catch (error) {
|
||||
console.error('Failed to initialize conversations:', error);
|
||||
}
|
||||
this.initPromise = (async () => {
|
||||
try {
|
||||
await MigrationService.runAllMigrations();
|
||||
await this.loadConversations();
|
||||
this.isInitialized = true;
|
||||
} catch (error) {
|
||||
console.error('Failed to initialize conversations:', error);
|
||||
this.initPromise = null;
|
||||
}
|
||||
})();
|
||||
|
||||
return this.initPromise;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -237,15 +246,11 @@ class ConversationsStore {
|
||||
*/
|
||||
async createConversation(name?: string): Promise<string> {
|
||||
const conversationName = name || `Chat ${new Date().toLocaleString()}`;
|
||||
const conversation = await DatabaseService.createConversation(conversationName);
|
||||
|
||||
// No MCP override list is seeded: getAllMcpServerOverrides resolves
|
||||
// servers without a per-conversation override to `mcpServers[i].enabled`,
|
||||
// and only explicit toggles are stored on the conversation.
|
||||
|
||||
// Inherit the global reasoning default into the new conversation
|
||||
conversation.reasoningEffort = this.pendingReasoningEffort;
|
||||
await DatabaseService.updateConversation(conversation.id, {
|
||||
const conversation = await DatabaseService.createConversation(conversationName, {
|
||||
reasoningEffort: this.pendingReasoningEffort
|
||||
});
|
||||
|
||||
@@ -358,10 +363,7 @@ class ConversationsStore {
|
||||
async deleteAll(): Promise<void> {
|
||||
try {
|
||||
const allConversations = await DatabaseService.getAllConversations();
|
||||
|
||||
for (const conv of allConversations) {
|
||||
await DatabaseService.deleteConversation(conv.id);
|
||||
}
|
||||
await DatabaseService.bulkDeleteConversations(allConversations.map((c) => c.id));
|
||||
|
||||
this.clearActiveConversation();
|
||||
this.conversations = [];
|
||||
@@ -412,7 +414,9 @@ class ConversationsStore {
|
||||
}
|
||||
|
||||
toast.success(
|
||||
convIds.length === 1 ? 'Conversation deleted' : `${convIds.length} conversations deleted`
|
||||
idsToRemove.size === 1
|
||||
? 'Conversation deleted'
|
||||
: `${idsToRemove.size} conversations deleted`
|
||||
);
|
||||
} catch (error) {
|
||||
console.error('Failed to bulk delete conversations:', error);
|
||||
@@ -443,7 +447,6 @@ class ConversationsStore {
|
||||
const newPinned = updates.get(this.conversations[i].id);
|
||||
if (newPinned !== undefined) this.conversations[i].pinned = newPinned;
|
||||
}
|
||||
this.conversations = [...this.conversations];
|
||||
|
||||
toast.success(
|
||||
convIds.length === 1
|
||||
@@ -552,7 +555,6 @@ class ConversationsStore {
|
||||
|
||||
if (convIndex !== -1) {
|
||||
this.conversations[convIndex].name = name;
|
||||
this.conversations = [...this.conversations];
|
||||
}
|
||||
|
||||
if (this.activeConversation?.id === convId) {
|
||||
@@ -576,7 +578,6 @@ class ConversationsStore {
|
||||
|
||||
if (convIndex !== -1) {
|
||||
this.conversations[convIndex].pinned = newPinnedState;
|
||||
this.conversations = [...this.conversations];
|
||||
}
|
||||
|
||||
if (this.activeConversation?.id === convId) {
|
||||
@@ -591,18 +592,33 @@ class ConversationsStore {
|
||||
}
|
||||
|
||||
/**
|
||||
* Updates conversation lastModified timestamp and moves it to top of list
|
||||
* Marks a conversation as recently active: stamps lastModified (persisted)
|
||||
* and moves it to the top of the list. Only message-activity flows call
|
||||
* this; metadata updates (rename, pin, settings) do not.
|
||||
*
|
||||
* @param convId - Conversation that produced the activity, defaults to the active one
|
||||
*/
|
||||
updateConversationTimestamp(): void {
|
||||
if (!this.activeConversation) return;
|
||||
updateConversationTimestamp(convId?: string): void {
|
||||
const targetId = convId ?? this.activeConversation?.id;
|
||||
if (!targetId) return;
|
||||
|
||||
const chatIndex = this.conversations.findIndex((c) => c.id === this.activeConversation!.id);
|
||||
const now = Date.now();
|
||||
|
||||
const chatIndex = this.conversations.findIndex((c) => c.id === targetId);
|
||||
|
||||
if (chatIndex !== -1) {
|
||||
this.conversations[chatIndex].lastModified = Date.now();
|
||||
this.conversations[chatIndex].lastModified = now;
|
||||
const updatedConv = this.conversations.splice(chatIndex, 1)[0];
|
||||
this.conversations = [updatedConv, ...this.conversations];
|
||||
}
|
||||
|
||||
if (this.activeConversation?.id === targetId) {
|
||||
this.activeConversation = { ...this.activeConversation, lastModified: now };
|
||||
}
|
||||
|
||||
DatabaseService.updateConversation(targetId, { lastModified: now }).catch((error) =>
|
||||
console.error('Failed to update conversation timestamp:', error)
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -773,7 +789,6 @@ class ConversationsStore {
|
||||
if (convIndex !== -1) {
|
||||
this.conversations[convIndex].mcpServerOverrides =
|
||||
newOverrides.length > 0 ? newOverrides : undefined;
|
||||
this.conversations = [...this.conversations];
|
||||
}
|
||||
}
|
||||
|
||||
@@ -837,7 +852,6 @@ class ConversationsStore {
|
||||
const convIndex = this.conversations.findIndex((c) => c.id === this.activeConversation!.id);
|
||||
if (convIndex !== -1) {
|
||||
this.conversations[convIndex].reasoningEffort = effort;
|
||||
this.conversations = [...this.conversations];
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,364 +0,0 @@
|
||||
/**
|
||||
* @deprecated Legacy migration utility — remove at some point in the future once all users have migrated to the new structured agentic message format.
|
||||
*
|
||||
* Converts old marker-based agentic messages to the new structured format
|
||||
* with separate messages per turn.
|
||||
*
|
||||
* Old format: Single assistant message with markers in content:
|
||||
* <<<reasoning_content_start>>>...<<<reasoning_content_end>>>
|
||||
* <<<AGENTIC_TOOL_CALL_START>>>...<<<AGENTIC_TOOL_CALL_END>>>
|
||||
*
|
||||
* New format: Separate messages per turn:
|
||||
* - assistant (content + reasoningContent + toolCalls)
|
||||
* - tool (toolCallId + content)
|
||||
* - assistant (next turn)
|
||||
* - ...
|
||||
*/
|
||||
|
||||
import { LEGACY_AGENTIC_REGEX, LEGACY_REASONING_TAGS } from '$lib/constants';
|
||||
import { DatabaseService } from '$lib/services/database.service';
|
||||
import { MessageRole, MessageType } from '$lib/enums';
|
||||
import type { DatabaseMessage } from '$lib/types/database';
|
||||
|
||||
const MIGRATION_DONE_KEY = 'llama-ui-migration-v2-done';
|
||||
/** @deprecated Use {@link MIGRATION_DONE_KEY} instead */
|
||||
const DEPRECATED_MIGRATION_DONE_KEY = 'llama-webui-migration-v2-done';
|
||||
|
||||
/**
|
||||
* @deprecated Part of legacy migration — remove with the migration module.
|
||||
* Check if migration has been performed.
|
||||
*/
|
||||
export function isMigrationNeeded(): boolean {
|
||||
try {
|
||||
// Check new key first, fall back to deprecated old key
|
||||
if (localStorage.getItem(MIGRATION_DONE_KEY)) return false;
|
||||
if (localStorage.getItem(DEPRECATED_MIGRATION_DONE_KEY)) {
|
||||
// Migrate to new key
|
||||
try {
|
||||
localStorage.setItem(MIGRATION_DONE_KEY, String(Date.now()));
|
||||
localStorage.removeItem(DEPRECATED_MIGRATION_DONE_KEY);
|
||||
} catch {
|
||||
// Ignore storage errors
|
||||
}
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Mark migration as done.
|
||||
*/
|
||||
function markMigrationDone(): void {
|
||||
try {
|
||||
localStorage.setItem(MIGRATION_DONE_KEY, String(Date.now()));
|
||||
} catch {
|
||||
// Ignore localStorage errors
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if a message has legacy markers in its content.
|
||||
*/
|
||||
function hasLegacyMarkers(message: DatabaseMessage): boolean {
|
||||
if (!message.content) return false;
|
||||
return LEGACY_AGENTIC_REGEX.HAS_LEGACY_MARKERS.test(message.content);
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract reasoning content from legacy marker format.
|
||||
*/
|
||||
function extractLegacyReasoning(content: string): { reasoning: string; cleanContent: string } {
|
||||
let reasoning = '';
|
||||
let cleanContent = content;
|
||||
|
||||
// Extract all reasoning blocks
|
||||
const re = new RegExp(LEGACY_AGENTIC_REGEX.REASONING_EXTRACT.source, 'g');
|
||||
let match;
|
||||
while ((match = re.exec(content)) !== null) {
|
||||
reasoning += match[1];
|
||||
}
|
||||
|
||||
// Remove reasoning tags from content
|
||||
cleanContent = cleanContent
|
||||
.replace(new RegExp(LEGACY_AGENTIC_REGEX.REASONING_BLOCK.source, 'g'), '')
|
||||
.replace(LEGACY_AGENTIC_REGEX.REASONING_OPEN, '');
|
||||
|
||||
return { reasoning, cleanContent };
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse legacy content with tool call markers into structured turns.
|
||||
*/
|
||||
interface ParsedTurn {
|
||||
textBefore: string;
|
||||
toolCalls: Array<{
|
||||
name: string;
|
||||
args: string;
|
||||
result: string;
|
||||
}>;
|
||||
}
|
||||
|
||||
function parseLegacyToolCalls(content: string): ParsedTurn[] {
|
||||
const turns: ParsedTurn[] = [];
|
||||
const regex = new RegExp(LEGACY_AGENTIC_REGEX.COMPLETED_TOOL_CALL.source, 'g');
|
||||
|
||||
let lastIndex = 0;
|
||||
let currentTurn: ParsedTurn = { textBefore: '', toolCalls: [] };
|
||||
let match;
|
||||
|
||||
while ((match = regex.exec(content)) !== null) {
|
||||
const textBefore = content.slice(lastIndex, match.index).trim();
|
||||
|
||||
// If there's text between tool calls and we already have tool calls,
|
||||
// that means a new turn started (text after tool results = new LLM turn)
|
||||
if (textBefore && currentTurn.toolCalls.length > 0) {
|
||||
turns.push(currentTurn);
|
||||
currentTurn = { textBefore, toolCalls: [] };
|
||||
} else if (textBefore && currentTurn.toolCalls.length === 0) {
|
||||
currentTurn.textBefore = textBefore;
|
||||
}
|
||||
|
||||
currentTurn.toolCalls.push({
|
||||
name: match[1],
|
||||
args: match[2],
|
||||
result: match[3].replace(/^\n+|\n+$/g, '')
|
||||
});
|
||||
|
||||
lastIndex = match.index + match[0].length;
|
||||
}
|
||||
|
||||
// Any remaining text after the last tool call
|
||||
const remainingText = content.slice(lastIndex).trim();
|
||||
|
||||
if (currentTurn.toolCalls.length > 0) {
|
||||
turns.push(currentTurn);
|
||||
}
|
||||
|
||||
// If there's text after all tool calls, it's the final assistant response
|
||||
if (remainingText) {
|
||||
// Remove any partial/open markers
|
||||
const cleanRemaining = remainingText
|
||||
.replace(LEGACY_AGENTIC_REGEX.AGENTIC_TOOL_CALL_OPEN, '')
|
||||
.trim();
|
||||
if (cleanRemaining) {
|
||||
turns.push({ textBefore: cleanRemaining, toolCalls: [] });
|
||||
}
|
||||
}
|
||||
|
||||
// If no tool calls found at all, return the original content as a single turn
|
||||
if (turns.length === 0) {
|
||||
turns.push({ textBefore: content.trim(), toolCalls: [] });
|
||||
}
|
||||
|
||||
return turns;
|
||||
}
|
||||
|
||||
/**
|
||||
* Migrate a single conversation's messages from legacy format to new format.
|
||||
*/
|
||||
async function migrateConversation(convId: string): Promise<number> {
|
||||
const allMessages = await DatabaseService.getConversationMessages(convId);
|
||||
let migratedCount = 0;
|
||||
|
||||
for (const message of allMessages) {
|
||||
if (message.role !== MessageRole.ASSISTANT) continue;
|
||||
if (!hasLegacyMarkers(message)) {
|
||||
// Still check for reasoning-only markers (no tool calls)
|
||||
if (message.content?.includes(LEGACY_REASONING_TAGS.START)) {
|
||||
const { reasoning, cleanContent } = extractLegacyReasoning(message.content);
|
||||
await DatabaseService.updateMessage(message.id, {
|
||||
content: cleanContent.trim(),
|
||||
reasoningContent: reasoning || undefined
|
||||
});
|
||||
migratedCount++;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
// Has agentic markers - full migration needed
|
||||
const { reasoning, cleanContent } = extractLegacyReasoning(message.content);
|
||||
const turns = parseLegacyToolCalls(cleanContent);
|
||||
|
||||
// Parse existing toolCalls JSON to try to match IDs
|
||||
let existingToolCalls: Array<{
|
||||
id: string;
|
||||
function?: { name: string; arguments: string };
|
||||
}> = [];
|
||||
if (message.toolCalls) {
|
||||
try {
|
||||
existingToolCalls = JSON.parse(message.toolCalls);
|
||||
} catch {
|
||||
// Ignore
|
||||
}
|
||||
}
|
||||
|
||||
// First turn uses the existing message
|
||||
const firstTurn = turns[0];
|
||||
if (!firstTurn) continue;
|
||||
|
||||
// Match tool calls from the first turn to existing IDs
|
||||
const firstTurnToolCalls = firstTurn.toolCalls.map((tc, i) => {
|
||||
const existing =
|
||||
existingToolCalls.find((e) => e.function?.name === tc.name) || existingToolCalls[i];
|
||||
return {
|
||||
id: existing?.id || `legacy_tool_${i}`,
|
||||
type: 'function' as const,
|
||||
function: { name: tc.name, arguments: tc.args }
|
||||
};
|
||||
});
|
||||
|
||||
// Update the existing message for the first turn
|
||||
await DatabaseService.updateMessage(message.id, {
|
||||
content: firstTurn.textBefore,
|
||||
reasoningContent: reasoning || undefined,
|
||||
toolCalls: firstTurnToolCalls.length > 0 ? JSON.stringify(firstTurnToolCalls) : ''
|
||||
});
|
||||
|
||||
let currentParentId = message.id;
|
||||
let toolCallIdCounter = existingToolCalls.length;
|
||||
|
||||
// Create tool result messages for the first turn
|
||||
for (let i = 0; i < firstTurn.toolCalls.length; i++) {
|
||||
const tc = firstTurn.toolCalls[i];
|
||||
const toolCallId = firstTurnToolCalls[i]?.id || `legacy_tool_${i}`;
|
||||
|
||||
const toolMsg = await DatabaseService.createMessageBranch(
|
||||
{
|
||||
convId,
|
||||
type: MessageType.TEXT,
|
||||
role: MessageRole.TOOL,
|
||||
content: tc.result,
|
||||
toolCallId,
|
||||
timestamp: message.timestamp + i + 1,
|
||||
toolCalls: '',
|
||||
children: []
|
||||
},
|
||||
currentParentId
|
||||
);
|
||||
currentParentId = toolMsg.id;
|
||||
}
|
||||
|
||||
// Create messages for subsequent turns
|
||||
for (let turnIdx = 1; turnIdx < turns.length; turnIdx++) {
|
||||
const turn = turns[turnIdx];
|
||||
|
||||
const turnToolCalls = turn.toolCalls.map((tc, i) => {
|
||||
const idx = toolCallIdCounter + i;
|
||||
const existing = existingToolCalls[idx];
|
||||
return {
|
||||
id: existing?.id || `legacy_tool_${idx}`,
|
||||
type: 'function' as const,
|
||||
function: { name: tc.name, arguments: tc.args }
|
||||
};
|
||||
});
|
||||
toolCallIdCounter += turn.toolCalls.length;
|
||||
|
||||
// Create assistant message for this turn
|
||||
const assistantMsg = await DatabaseService.createMessageBranch(
|
||||
{
|
||||
convId,
|
||||
type: MessageType.TEXT,
|
||||
role: MessageRole.ASSISTANT,
|
||||
content: turn.textBefore,
|
||||
timestamp: message.timestamp + turnIdx * 100,
|
||||
toolCalls: turnToolCalls.length > 0 ? JSON.stringify(turnToolCalls) : '',
|
||||
children: [],
|
||||
model: message.model
|
||||
},
|
||||
currentParentId
|
||||
);
|
||||
currentParentId = assistantMsg.id;
|
||||
|
||||
// Create tool result messages for this turn
|
||||
for (let i = 0; i < turn.toolCalls.length; i++) {
|
||||
const tc = turn.toolCalls[i];
|
||||
const toolCallId = turnToolCalls[i]?.id || `legacy_tool_${toolCallIdCounter + i}`;
|
||||
|
||||
const toolMsg = await DatabaseService.createMessageBranch(
|
||||
{
|
||||
convId,
|
||||
type: MessageType.TEXT,
|
||||
role: MessageRole.TOOL,
|
||||
content: tc.result,
|
||||
toolCallId,
|
||||
timestamp: message.timestamp + turnIdx * 100 + i + 1,
|
||||
toolCalls: '',
|
||||
children: []
|
||||
},
|
||||
currentParentId
|
||||
);
|
||||
currentParentId = toolMsg.id;
|
||||
}
|
||||
}
|
||||
|
||||
// Re-parent any children of the original message to the last created message
|
||||
// (the original message's children list was the next user message or similar)
|
||||
if (message.children.length > 0 && currentParentId !== message.id) {
|
||||
for (const childId of message.children) {
|
||||
// Skip children we just created (they were already properly parented)
|
||||
const child = allMessages.find((m) => m.id === childId);
|
||||
if (!child) continue;
|
||||
// Only re-parent non-tool messages that were original children
|
||||
if (child.role !== MessageRole.TOOL) {
|
||||
await DatabaseService.updateMessage(childId, { parent: currentParentId });
|
||||
// Add to new parent's children
|
||||
const newParent = await DatabaseService.getConversationMessages(convId).then((msgs) =>
|
||||
msgs.find((m) => m.id === currentParentId)
|
||||
);
|
||||
if (newParent && !newParent.children.includes(childId)) {
|
||||
await DatabaseService.updateMessage(currentParentId, {
|
||||
children: [...newParent.children, childId]
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
// Clear re-parented children from the original message
|
||||
await DatabaseService.updateMessage(message.id, { children: [] });
|
||||
}
|
||||
|
||||
migratedCount++;
|
||||
}
|
||||
|
||||
return migratedCount;
|
||||
}
|
||||
|
||||
/**
|
||||
* @deprecated Part of legacy migration — remove with the migration module.
|
||||
* Run the full migration across all conversations.
|
||||
* This should be called once at app startup if migration is needed.
|
||||
*/
|
||||
export async function runLegacyMigration(): Promise<void> {
|
||||
if (!isMigrationNeeded()) return;
|
||||
|
||||
if (import.meta.env.DEV && import.meta.env.VITE_DEBUG)
|
||||
console.log('[Migration] Starting legacy message format migration...');
|
||||
|
||||
try {
|
||||
const conversations = await DatabaseService.getAllConversations();
|
||||
let totalMigrated = 0;
|
||||
|
||||
for (const conv of conversations) {
|
||||
const count = await migrateConversation(conv.id);
|
||||
totalMigrated += count;
|
||||
}
|
||||
|
||||
if (import.meta.env.DEV && import.meta.env.VITE_DEBUG) {
|
||||
if (totalMigrated > 0) {
|
||||
console.log(
|
||||
`[Migration] Migrated ${totalMigrated} messages across ${conversations.length} conversations`
|
||||
);
|
||||
} else {
|
||||
console.log('[Migration] No legacy messages found, marking as done');
|
||||
}
|
||||
}
|
||||
|
||||
markMigrationDone();
|
||||
} catch (error) {
|
||||
console.error('[Migration] Failed to migrate legacy messages:', error);
|
||||
// Still mark as done to avoid infinite retry loops
|
||||
markMigrationDone();
|
||||
}
|
||||
}
|
||||
Vendored
+1
-1
@@ -41,7 +41,7 @@ if (LLAMA_BUILD_BORINGSSL)
|
||||
set(FIPS OFF CACHE BOOL "Enable FIPS (BoringSSL)")
|
||||
|
||||
set(BORINGSSL_GIT "https://boringssl.googlesource.com/boringssl" CACHE STRING "BoringSSL git repository")
|
||||
set(BORINGSSL_VERSION "0.20260713.0" CACHE STRING "BoringSSL version")
|
||||
set(BORINGSSL_VERSION "0.20260728.0" CACHE STRING "BoringSSL version")
|
||||
|
||||
message(STATUS "Fetching BoringSSL version ${BORINGSSL_VERSION}")
|
||||
|
||||
|
||||
Reference in New Issue
Block a user