mirror of
https://github.com/ggml-org/llama.cpp.git
synced 2026-08-29 19:47:32 +02:00
Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Xuan Son Nguyen
|
f47ff9b250 |
+1
-17
@@ -935,9 +935,6 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl {
|
||||
// dspark speculators
|
||||
bool sample_from_anchor = true;
|
||||
|
||||
// block-internal attention
|
||||
bool causal_attn = false;
|
||||
|
||||
const int32_t * target_layer_ids = nullptr; // model_dft's extract layer indices
|
||||
uint32_t target_layer_ids_n = 0;
|
||||
|
||||
@@ -975,25 +972,12 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl {
|
||||
if (llama_model_meta_val_str(model_dft, "dflash.sample_from_anchor", buf, sizeof(buf)) >= 0) {
|
||||
sample_from_anchor = std::strcmp(buf, "true") == 0;
|
||||
}
|
||||
if (llama_model_meta_val_str(model_dft, "dflash.attention.causal", buf, sizeof(buf)) >= 0) {
|
||||
causal_attn = std::strcmp(buf, "true") == 0;
|
||||
}
|
||||
}
|
||||
|
||||
selector_top_k = llama_model_dflash_selector_top_k(model_dft);
|
||||
is_dflash2 = selector_top_k > 0;
|
||||
mask_token_id = llama_vocab_mask(llama_model_get_vocab(model_dft));
|
||||
|
||||
if (is_dspark && this->params.p_min > 0.0f) {
|
||||
char buf[16] = {};
|
||||
const bool has_conf =
|
||||
llama_model_meta_val_str(model_dft, "dflash.has_confidence_head", buf, sizeof(buf)) < 0 ||
|
||||
std::strcmp(buf, "true") == 0;
|
||||
if (!has_conf) {
|
||||
throw std::runtime_error("DSpark draft has no confidence head: please set --spec-draft-p-min 0");
|
||||
}
|
||||
}
|
||||
|
||||
LOG_INF("%s: adding speculative implementation '%s'\n", __func__, common_speculative_type_to_str(type).c_str());
|
||||
LOG_INF("%s: - n_max=%d, n_min=%d, p_min=%.2f\n", __func__, this->params.n_max, this->params.n_min, this->params.p_min);
|
||||
LOG_INF("%s: - block_size=%d, mask_token_id=%d, n_extract=%u, sample_from_anchor=%s\n", __func__,
|
||||
@@ -1052,7 +1036,7 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl {
|
||||
|
||||
// DFlash2 reads its selector lattice from h_nextn and never consumes raw logits.
|
||||
llama_set_embeddings_nextn(ctx_dft, true, /*masked*/ !is_dflash2);
|
||||
llama_set_causal_attn(ctx_dft, causal_attn); // DFlash needs non-causal attention unless the model says otherwise
|
||||
llama_set_causal_attn(ctx_dft, false); // DFlash needs non-causal attention
|
||||
}
|
||||
|
||||
~common_speculative_impl_draft_dflash() override {
|
||||
|
||||
+3
-15
@@ -709,20 +709,14 @@ class DFlashModel(Qwen3Model):
|
||||
extract_layer_ids = [i + 1 for i in target_layer_ids]
|
||||
self.gguf_writer.add_target_layers(extract_layer_ids)
|
||||
|
||||
use_sliding_window = self.hparams.get("use_sliding_window", False) or dflash_config.get("use_swa", False)
|
||||
sliding_window = dflash_config.get("swa_window_size") or self.hparams.get("sliding_window")
|
||||
use_sliding_window = self.hparams.get("use_sliding_window", False)
|
||||
sliding_window = self.hparams.get("sliding_window")
|
||||
layer_types = self.hparams.get("layer_types")
|
||||
if use_sliding_window and sliding_window and layer_types:
|
||||
is_swa = [lt == "sliding_attention" for lt in layer_types]
|
||||
self.gguf_writer.add_sliding_window(sliding_window)
|
||||
self.gguf_writer.add_sliding_window_pattern(is_swa)
|
||||
|
||||
causal = self.hparams.get("is_causal")
|
||||
if causal is None:
|
||||
causal = dflash_config.get("causal")
|
||||
if causal is not None:
|
||||
self.gguf_writer.add_causal_attention(bool(causal))
|
||||
|
||||
# M-RoPE target: the draft ropes on the temporal dim only, so write
|
||||
# degenerate sections [n_rot/2, 0, 0, 0]
|
||||
if self._target_uses_mrope():
|
||||
@@ -743,8 +737,6 @@ class DFlashModel(Qwen3Model):
|
||||
name, gen = item
|
||||
if not name.startswith("model."):
|
||||
name = "model." + name
|
||||
if "sink" in name and not name.endswith(".weight"):
|
||||
name += ".weight"
|
||||
return super().filter_tensors((name, gen))
|
||||
|
||||
_ROPE_PERMUTE_SUFFIXES = (
|
||||
@@ -823,10 +815,6 @@ class DSparkModel(DFlashModel):
|
||||
super().set_gguf_parameters()
|
||||
self.gguf_writer.add_sample_from_anchor(self._sample_from_anchor)
|
||||
|
||||
# confidence head is optional: vanilla-markov exports ship without it
|
||||
has_conf = any("confidence_head.proj" in name for name in self.model_tensors)
|
||||
self.gguf_writer.add_has_confidence_head(has_conf)
|
||||
|
||||
@classmethod
|
||||
def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
|
||||
if item[0] == "t2d": # not used at runtime
|
||||
@@ -845,7 +833,7 @@ class DSparkModel(DFlashModel):
|
||||
self._d2t = data_torch
|
||||
return
|
||||
|
||||
if self._n_vocab_draft == self.hparams["vocab_size"] and name.endswith("lm_head.weight"):
|
||||
if self._n_vocab_draft == self.hparams["vocab_size"] and name.endswith(("embed_tokens.weight", "lm_head.weight")):
|
||||
return
|
||||
|
||||
# interleaved-rope checkpoints (rope_is_neox_style = false) -> NeoX layout: per head, even dims first then odd
|
||||
|
||||
@@ -4643,7 +4643,6 @@ static htp_op_code op_remap_to_htp(const ggml_tensor * t) {
|
||||
case GGML_OP_CLAMP: return HTP_OP_CLAMP;
|
||||
case GGML_OP_SQR: return HTP_OP_SQR;
|
||||
case GGML_OP_SQRT: return HTP_OP_SQRT;
|
||||
case GGML_OP_LOG: return HTP_OP_UNARY_LOG;
|
||||
case GGML_OP_SOFT_MAX: return HTP_OP_SOFTMAX;
|
||||
case GGML_OP_SSM_CONV: return HTP_OP_SSM_CONV;
|
||||
case GGML_OP_GATED_DELTA_NET: return HTP_OP_GATED_DELTA_NET;
|
||||
@@ -4667,7 +4666,6 @@ static htp_op_code op_remap_to_htp(const ggml_tensor * t) {
|
||||
case GGML_UNARY_OP_EXP: return HTP_OP_UNARY_EXP;
|
||||
case GGML_UNARY_OP_SOFTPLUS: return HTP_OP_UNARY_SOFTPLUS;
|
||||
case GGML_UNARY_OP_TANH: return HTP_OP_UNARY_TANH;
|
||||
case GGML_UNARY_OP_ABS: return HTP_OP_UNARY_ABS;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
@@ -5465,7 +5463,6 @@ static bool ggml_backend_hexagon_device_supports_op(ggml_backend_dev_t dev, cons
|
||||
|
||||
case GGML_OP_SQR:
|
||||
case GGML_OP_SQRT:
|
||||
case GGML_OP_LOG:
|
||||
supp = ggml_hexagon_supported_unary(sess, op);
|
||||
break;
|
||||
|
||||
@@ -5484,7 +5481,6 @@ static bool ggml_backend_hexagon_device_supports_op(ggml_backend_dev_t dev, cons
|
||||
case GGML_UNARY_OP_SIGMOID:
|
||||
case GGML_UNARY_OP_SOFTPLUS:
|
||||
case GGML_UNARY_OP_TANH:
|
||||
case GGML_UNARY_OP_ABS:
|
||||
case GGML_UNARY_OP_SILU:
|
||||
case GGML_UNARY_OP_GELU:
|
||||
case GGML_UNARY_OP_GELU_QUICK:
|
||||
|
||||
@@ -62,8 +62,6 @@ enum htp_op_code {
|
||||
HTP_OP_UNARY_NEG,
|
||||
HTP_OP_UNARY_SOFTPLUS,
|
||||
HTP_OP_UNARY_TANH,
|
||||
HTP_OP_UNARY_ABS,
|
||||
HTP_OP_UNARY_LOG,
|
||||
HTP_OP_GLU_SWIGLU,
|
||||
HTP_OP_GLU_SWIGLU_OAI,
|
||||
HTP_OP_GLU_GEGLU,
|
||||
|
||||
@@ -358,34 +358,6 @@ static inline void hvx_clamp_scalar_f32(uint8_t * restrict dst, const uint8_t *
|
||||
}
|
||||
}
|
||||
|
||||
//
|
||||
// Abs
|
||||
//
|
||||
|
||||
static inline void hvx_abs_f32_aa(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {
|
||||
assert((unsigned long) dst % 128 == 0);
|
||||
assert((unsigned long) src % 128 == 0);
|
||||
|
||||
HVX_Vector * restrict vdst = (HVX_Vector *) dst;
|
||||
HVX_Vector * restrict vsrc = (HVX_Vector *) src;
|
||||
|
||||
const uint32_t elem_size = sizeof(float);
|
||||
const uint32_t epv = 128 / elem_size;
|
||||
const uint32_t nvec = n / epv;
|
||||
const uint32_t nloe = n % epv;
|
||||
|
||||
uint32_t i = 0;
|
||||
|
||||
_Pragma("unroll(4)")
|
||||
for (; i < nvec; i++) {
|
||||
vdst[i] = hvx_vec_abs_f32(vsrc[i]);
|
||||
}
|
||||
if (nloe) {
|
||||
HVX_Vector v = hvx_vec_abs_f32(vsrc[i]);
|
||||
hvx_vec_store_a((void *) &vdst[i], nloe * elem_size, v);
|
||||
}
|
||||
}
|
||||
|
||||
//
|
||||
// Square
|
||||
//
|
||||
|
||||
@@ -62,28 +62,4 @@ static inline HVX_Vector hvx_vec_log_f32(HVX_Vector x) {
|
||||
return hvx_vec_add_f32_f32(term_e, res);
|
||||
}
|
||||
|
||||
static inline void hvx_log_f32_aa(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {
|
||||
assert((unsigned long) dst % 128 == 0);
|
||||
assert((unsigned long) src % 128 == 0);
|
||||
|
||||
HVX_Vector * restrict vdst = (HVX_Vector *) dst;
|
||||
HVX_Vector * restrict vsrc = (HVX_Vector *) src;
|
||||
|
||||
const uint32_t elem_size = sizeof(float);
|
||||
const uint32_t epv = 128 / elem_size;
|
||||
const uint32_t nvec = n / epv;
|
||||
const uint32_t nloe = n % epv;
|
||||
|
||||
uint32_t i = 0;
|
||||
|
||||
_Pragma("unroll(4)")
|
||||
for (; i < nvec; i++) {
|
||||
vdst[i] = hvx_vec_log_f32(vsrc[i]);
|
||||
}
|
||||
if (nloe) {
|
||||
HVX_Vector v = hvx_vec_log_f32(vsrc[i]);
|
||||
hvx_vec_store_a((void *) &vdst[i], nloe * elem_size, v);
|
||||
}
|
||||
}
|
||||
|
||||
#endif /* HVX_LOG_H */
|
||||
|
||||
@@ -777,8 +777,6 @@ static int execute_op(struct htp_ops_context * octx) {
|
||||
case HTP_OP_UNARY_NEG:
|
||||
case HTP_OP_UNARY_EXP:
|
||||
case HTP_OP_UNARY_TANH:
|
||||
case HTP_OP_UNARY_ABS:
|
||||
case HTP_OP_UNARY_LOG:
|
||||
case HTP_OP_L2_NORM:
|
||||
return op_unary(octx);
|
||||
|
||||
|
||||
@@ -443,34 +443,6 @@ static void tanh_f32(const float * restrict src,
|
||||
}
|
||||
}
|
||||
|
||||
static void abs_f32(const float * restrict src,
|
||||
float * restrict dst,
|
||||
const uint32_t num_rows,
|
||||
const struct htp_unary_context * uctx) {
|
||||
htp_unary_op_preamble;
|
||||
|
||||
for (uint32_t ir = 0; ir < num_rows; ir++) {
|
||||
const uint8_t * restrict src_local = (const uint8_t *)src + (ir * src0_row_size_aligned);
|
||||
uint8_t * restrict dst_local = (uint8_t *)dst + (ir * dst_row_size_aligned);
|
||||
|
||||
hvx_abs_f32_aa(dst_local, src_local, ne0);
|
||||
}
|
||||
}
|
||||
|
||||
static void log_f32(const float * restrict src,
|
||||
float * restrict dst,
|
||||
const uint32_t num_rows,
|
||||
const struct htp_unary_context * uctx) {
|
||||
htp_unary_op_preamble;
|
||||
|
||||
for (uint32_t ir = 0; ir < num_rows; ir++) {
|
||||
const uint8_t * restrict src_local = (const uint8_t *)src + (ir * src0_row_size_aligned);
|
||||
uint8_t * restrict dst_local = (uint8_t *)dst + (ir * dst_row_size_aligned);
|
||||
|
||||
hvx_log_f32_aa(dst_local, src_local, ne0);
|
||||
}
|
||||
}
|
||||
|
||||
#define DEFINE_UNARY_TASK(NAME, IS_RMS_NORM_MUL, IS_TRI, CORE_EXPR) \
|
||||
static void unary_task_f32_##NAME(unsigned int nth, unsigned int ith, void * data) { \
|
||||
const struct htp_unary_context * uctx = (const struct htp_unary_context *) data; \
|
||||
@@ -506,9 +478,6 @@ static void unary_task_f32_##NAME(unsigned int nth, unsigned int ith, void * dat
|
||||
const uint32_t nb11 = src1 ? src1->nb[1] : 0; \
|
||||
const uint32_t nb12 = src1 ? src1->nb[2] : 0; \
|
||||
const uint32_t nb13 = src1 ? src1->nb[3] : 0; \
|
||||
const uint32_t nb11_bc = (src1 && src1->ne[1] > 1) ? nb11 : 0; \
|
||||
const uint32_t nb12_bc = (src1 && src1->ne[2] > 1) ? nb12 : 0; \
|
||||
const uint32_t nb13_bc = (src1 && src1->ne[3] > 1) ? nb13 : 0; \
|
||||
const bool src1_contig = src1 ? ((nb12 == (size_t)ne01 * nb11) && (nb13 == (size_t)ne02 * nb12)) : false; \
|
||||
\
|
||||
uint8_t * src0_vtcm_data = uctx->vtcm_src0 + (ith * uctx->vtcm_src0_size_per_thread); \
|
||||
@@ -528,12 +497,8 @@ static void unary_task_f32_##NAME(unsigned int nth, unsigned int ith, void * dat
|
||||
const struct fastdiv_values * div_ne02 = &uctx->kparams->div_ne02; \
|
||||
const struct fastdiv_values * div_ne012 = &uctx->kparams->div_ne012; \
|
||||
\
|
||||
const bool src1_needs_row_clip = (IS_RMS_NORM_MUL) && !uctx->broadcast_weight && !src1_contig; \
|
||||
const bool block_src0_contig = src0_contig && !src1_needs_row_clip; \
|
||||
const bool block_dst_contig = dst_contig && !src1_needs_row_clip; \
|
||||
\
|
||||
const uint32_t src0_max_block = block_src0_contig ? uctx->block : MIN((uint32_t)uctx->block, ne01); \
|
||||
const uint32_t dst_max_block = block_dst_contig ? uctx->block : MIN((uint32_t)uctx->block, ne1); \
|
||||
const uint32_t src0_max_block = src0_contig ? uctx->block : MIN((uint32_t)uctx->block, ne01); \
|
||||
const uint32_t dst_max_block = dst_contig ? uctx->block : MIN((uint32_t)uctx->block, ne1); \
|
||||
const uint32_t BLOCK = MIN(src0_max_block, dst_max_block); \
|
||||
if (BLOCK == 0) { \
|
||||
FARF(ERROR, "unary-f32 : current VTCM reservation %zu is too small, needed at least %zu\n", \
|
||||
@@ -550,8 +515,8 @@ static void unary_task_f32_##NAME(unsigned int nth, unsigned int ith, void * dat
|
||||
} \
|
||||
\
|
||||
for (uint32_t ir = src0_start_row, vtcm_idx = 0; ir < src0_end_row && vtcm_idx < 2; vtcm_idx++) { \
|
||||
const uint32_t block_size = unary_block_size(ir, src0_end_row, BLOCK, block_src0_contig, block_dst_contig, \
|
||||
ne01, div_ne01); \
|
||||
const uint32_t block_size = unary_block_size(ir, src0_end_row, BLOCK, src0_contig, dst_contig, ne01, \
|
||||
div_ne01); \
|
||||
\
|
||||
dma_queue_push(dma_queue, \
|
||||
dma_make_ptr(data_dst, dst_vtcm_data + (vtcm_idx * dst_vtcm_half_size)), \
|
||||
@@ -565,7 +530,7 @@ static void unary_task_f32_##NAME(unsigned int nth, unsigned int ith, void * dat
|
||||
\
|
||||
if ((IS_RMS_NORM_MUL) && !uctx->broadcast_weight) { \
|
||||
const size_t src1_off = src1_contig ? (ir * nb11) : \
|
||||
unary_row_offset(ir, ne01, ne02, div_ne01, div_ne02, div_ne012, nb11_bc, nb12_bc, nb13_bc); \
|
||||
unary_row_offset(ir, ne01, ne02, div_ne01, div_ne02, div_ne012, nb11, nb12, nb13); \
|
||||
dma_queue_push(dma_queue, \
|
||||
dma_make_ptr(src1_vtcm_data + (vtcm_idx * src1_vtcm_half_size), data_src1 + src1_off), \
|
||||
uctx->src1_row_size_aligned, nb11, uctx->src1_data_row_size, block_size); \
|
||||
@@ -575,8 +540,8 @@ static void unary_task_f32_##NAME(unsigned int nth, unsigned int ith, void * dat
|
||||
} \
|
||||
\
|
||||
for (uint32_t ir = src0_start_row; ir < src0_end_row; ) { \
|
||||
const uint32_t block_size = unary_block_size(ir, src0_end_row, BLOCK, block_src0_contig, block_dst_contig, \
|
||||
ne01, div_ne01); \
|
||||
const uint32_t block_size = unary_block_size(ir, src0_end_row, BLOCK, src0_contig, dst_contig, ne01, \
|
||||
div_ne01); \
|
||||
\
|
||||
float * dst_vtcm = (float *) dma_queue_pop(dma_queue).src; \
|
||||
float * src0_vtcm = (float *) dma_queue_pop(dma_queue).dst; \
|
||||
@@ -597,12 +562,12 @@ static void unary_task_f32_##NAME(unsigned int nth, unsigned int ith, void * dat
|
||||
\
|
||||
const uint32_t next_ir = ir + block_size; \
|
||||
if (next_ir < src0_end_row) { \
|
||||
const uint32_t next_block_size = unary_block_size(next_ir, src0_end_row, BLOCK, block_src0_contig, \
|
||||
block_dst_contig, ne01, div_ne01); \
|
||||
const uint32_t next_block_size = unary_block_size(next_ir, src0_end_row, BLOCK, src0_contig, dst_contig,\
|
||||
ne01, div_ne01); \
|
||||
const uint32_t pref_ir = next_ir + next_block_size; \
|
||||
if (pref_ir < src0_end_row) { \
|
||||
const uint32_t pref_block_size = unary_block_size(pref_ir, src0_end_row, BLOCK, block_src0_contig, \
|
||||
block_dst_contig, ne01, div_ne01); \
|
||||
const uint32_t pref_block_size = unary_block_size(pref_ir, src0_end_row, BLOCK, src0_contig, \
|
||||
dst_contig, ne01, div_ne01); \
|
||||
const size_t src0_pref_off = src0_contig ? (pref_ir * nb01) : \
|
||||
unary_row_offset(pref_ir, ne01, ne02, div_ne01, div_ne02, div_ne012, nb01, nb02, nb03); \
|
||||
dma_queue_push(dma_queue, \
|
||||
@@ -611,8 +576,7 @@ static void unary_task_f32_##NAME(unsigned int nth, unsigned int ith, void * dat
|
||||
\
|
||||
if ((IS_RMS_NORM_MUL) && !uctx->broadcast_weight) { \
|
||||
const size_t src1_pref_off = src1_contig ? (pref_ir * nb11) : \
|
||||
unary_row_offset(pref_ir, ne01, ne02, div_ne01, div_ne02, div_ne012, nb11_bc, nb12_bc, \
|
||||
nb13_bc); \
|
||||
unary_row_offset(pref_ir, ne01, ne02, div_ne01, div_ne02, div_ne012, nb11, nb12, nb13); \
|
||||
dma_queue_push(dma_queue, \
|
||||
dma_make_ptr(src1_vtcm, data_src1 + src1_pref_off), \
|
||||
uctx->src1_row_size_aligned, nb11, uctx->src1_data_row_size, pref_block_size); \
|
||||
@@ -639,8 +603,6 @@ DEFINE_UNARY_TASK(unary_silu, false, false, silu_f32(src0_vtcm, dst_vtcm, bl
|
||||
DEFINE_UNARY_TASK(unary_gelu, false, false, gelu_f32(src0_vtcm, dst_vtcm, block_size, uctx))
|
||||
DEFINE_UNARY_TASK(unary_softplus, false, false, softplus_f32(src0_vtcm, dst_vtcm, block_size, uctx))
|
||||
DEFINE_UNARY_TASK(unary_tanh, false, false, tanh_f32(src0_vtcm, dst_vtcm, block_size, uctx))
|
||||
DEFINE_UNARY_TASK(unary_abs, false, false, abs_f32(src0_vtcm, dst_vtcm, block_size, uctx))
|
||||
DEFINE_UNARY_TASK(unary_log, false, false, log_f32(src0_vtcm, dst_vtcm, block_size, uctx))
|
||||
DEFINE_UNARY_TASK(l2_norm, false, false, l2_norm_f32(src0_vtcm, dst_vtcm, block_size, uctx))
|
||||
DEFINE_UNARY_TASK(tri, false, true, tri_f32(src0_vtcm, dst_vtcm, block_size, ir, uctx))
|
||||
|
||||
@@ -888,8 +850,6 @@ DEFINE_UNARY_TILED_TASK(unary_silu, false, tile_silu_f32(dst_vtcm, src_vtcm,
|
||||
DEFINE_UNARY_TILED_TASK(unary_gelu, false, tile_gelu_f32(dst_vtcm, src_vtcm, tw))
|
||||
DEFINE_UNARY_TILED_TASK(unary_softplus, false, tile_unary_softplus_f32(dst_vtcm, src_vtcm, tw))
|
||||
DEFINE_UNARY_TILED_TASK(unary_tanh, false, hvx_tanh_f32_aa(dst_vtcm, src_vtcm, tw))
|
||||
DEFINE_UNARY_TILED_TASK(unary_abs, false, hvx_abs_f32_aa(dst_vtcm, src_vtcm, tw))
|
||||
DEFINE_UNARY_TILED_TASK(unary_log, false, hvx_log_f32_aa(dst_vtcm, src_vtcm, tw))
|
||||
DEFINE_UNARY_TILED_TASK(tri, true, tri_apply_tile_f32(src_vtcm, dst_vtcm, tw, col, i01, ne0, tri_ttype))
|
||||
|
||||
static int execute_op_unary_f32(struct htp_ops_context * octx) {
|
||||
@@ -915,8 +875,6 @@ static int execute_op_unary_f32(struct htp_ops_context * octx) {
|
||||
case HTP_OP_UNARY_GELU: op_type = "gelu-f32"; break;
|
||||
case HTP_OP_UNARY_SOFTPLUS: op_type = "softplus-f32"; break;
|
||||
case HTP_OP_UNARY_TANH: op_type = "tanh-f32"; break;
|
||||
case HTP_OP_UNARY_ABS: op_type = "abs-f32"; break;
|
||||
case HTP_OP_UNARY_LOG: op_type = "log-f32"; break;
|
||||
case HTP_OP_L2_NORM: op_type = "l2norm-f32"; break;
|
||||
case HTP_OP_TRI: op_type = "tri-f32"; break;
|
||||
|
||||
@@ -1015,8 +973,6 @@ static int execute_op_unary_f32(struct htp_ops_context * octx) {
|
||||
case HTP_OP_UNARY_GELU: task_func = unary_task_f32_tiled_unary_gelu; break;
|
||||
case HTP_OP_UNARY_SOFTPLUS: task_func = unary_task_f32_tiled_unary_softplus; break;
|
||||
case HTP_OP_UNARY_TANH: task_func = unary_task_f32_tiled_unary_tanh; break;
|
||||
case HTP_OP_UNARY_ABS: task_func = unary_task_f32_tiled_unary_abs; break;
|
||||
case HTP_OP_UNARY_LOG: task_func = unary_task_f32_tiled_unary_log; break;
|
||||
case HTP_OP_TRI: task_func = unary_task_f32_tiled_tri; break;
|
||||
default: break;
|
||||
}
|
||||
@@ -1036,8 +992,6 @@ static int execute_op_unary_f32(struct htp_ops_context * octx) {
|
||||
case HTP_OP_UNARY_GELU: task_func = unary_task_f32_unary_gelu; break;
|
||||
case HTP_OP_UNARY_SOFTPLUS: task_func = unary_task_f32_unary_softplus; break;
|
||||
case HTP_OP_UNARY_TANH: task_func = unary_task_f32_unary_tanh; break;
|
||||
case HTP_OP_UNARY_ABS: task_func = unary_task_f32_unary_abs; break;
|
||||
case HTP_OP_UNARY_LOG: task_func = unary_task_f32_unary_log; break;
|
||||
case HTP_OP_L2_NORM: task_func = unary_task_f32_l2_norm; break;
|
||||
case HTP_OP_TRI: task_func = unary_task_f32_tri; break;
|
||||
default: break;
|
||||
|
||||
@@ -55,8 +55,6 @@ static inline bool htp_op_is_unary(uint32_t opcode) {
|
||||
case HTP_OP_UNARY_GELU:
|
||||
case HTP_OP_UNARY_SOFTPLUS:
|
||||
case HTP_OP_UNARY_TANH:
|
||||
case HTP_OP_UNARY_ABS:
|
||||
case HTP_OP_UNARY_LOG:
|
||||
case HTP_OP_L2_NORM:
|
||||
case HTP_OP_TRI:
|
||||
return true;
|
||||
|
||||
@@ -448,66 +448,7 @@ constexpr fa_vec_entry_t fa_vec_tuned_table[] = {
|
||||
{ { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q8_0, 512, 512, -1, 1 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q8_0, 576, 512, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q8_0, 576, 512, -1, 1 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_F16, 32, 32, 1, 3 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_F16, 32, 32, 2, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_F16, 32, 32, 2, 3 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_F16, 32, 32, 3, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_F16, 32, 32, 3, 3 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_F16, 64, 64, -1, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_F16, 64, 64, 1, 2 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_F16, 64, 64, 2, 2 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_F16, 96, 96, -1, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_F16, 96, 96, 1, 2 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_F16, 96, 96, 1, 4 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_F16, 96, 96, 2, 2 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_F16, 96, 96, 2, 4 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_F16, 128, 128, -1, 1 }, { 2, 2 } },
|
||||
{ { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_F16, 128, 128, 1, 2 }, { 1, 1 } },
|
||||
{ { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_F16, 128, 128, 1, 4 }, { 1, 1 } },
|
||||
{ { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_F16, 192, 192, -1, 1 }, { 2, 2 } },
|
||||
{ { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_F16, 192, 128, -1, 1 }, { 2, 2 } },
|
||||
{ { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_F16, 192, 128, 1, 4 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_F16, 256, 256, -1, 0 }, { 1, 2 } },
|
||||
{ { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_F16, 256, 256, 3, 0 }, { 2, 2 } },
|
||||
{ { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_F16, 256, 256, -1, 1 }, { 2, 2 } },
|
||||
{ { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_F16, 320, 256, 3, 0 }, { 2, 2 } },
|
||||
{ { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_F16, 320, 256, -1, 1 }, { 2, 2 } },
|
||||
{ { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_F16, 512, 512, 3, 0 }, { 1, 2 } },
|
||||
{ { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q8_0, 32, 32, -1, 1 }, { 4, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q8_0, 32, 32, 1, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q8_0, 32, 32, 1, 4 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q8_0, 32, 32, 2, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q8_0, 32, 32, 2, 4 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q8_0, 32, 32, 3, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q8_0, 64, 64, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q8_0, 64, 64, -1, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q8_0, 64, 64, 3, 2 }, { 4, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q8_0, 96, 96, -1, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q8_0, 96, 96, 2, 2 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q8_0, 128, 128, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q8_0, 128, 128, -1, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q8_0, 128, 128, 1, 2 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q8_0, 128, 128, 2, 2 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q8_0, 128, 128, 3, 2 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q8_0, 192, 192, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q8_0, 192, 192, -1, 1 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q8_0, 192, 128, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q8_0, 192, 128, -1, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q8_0, 192, 128, 1, 2 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q8_0, 192, 128, 2, 2 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q8_0, 192, 128, 3, 2 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q8_0, 256, 256, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q8_0, 256, 256, -1, 1 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q8_0, 320, 256, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q8_0, 320, 256, -1, 1 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q8_0, 512, 512, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q8_0, 512, 512, -1, 1 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q8_0, 576, 512, 2, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q8_0, 576, 512, 3, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q8_0, 576, 512, -1, 1 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q8_0, 576, 512, 1, 1 }, { 1, 2 } },
|
||||
{ { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q8_0, 576, 512, 1, 3 }, { 1, 2 } },
|
||||
{ { GGML_METAL_DEVICE_M4_PRO, GGML_TYPE_Q8_0, 576, 512, 1, 4 }, { 1, 2 } },
|
||||
|
||||
{ { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_F16, 32, 32, 2, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_F16, 32, 32, 2, 3 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M4_MAX, GGML_TYPE_F16, 32, 32, 3, 1 }, { 2, 4 } },
|
||||
|
||||
@@ -167,7 +167,6 @@ class Keys:
|
||||
SELECTOR_RANK = "{arch}.selector_rank"
|
||||
SELECTOR_TOP_K = "{arch}.selector_top_k"
|
||||
SAMPLE_FROM_ANCHOR = "{arch}.sample_from_anchor"
|
||||
HAS_CONFIDENCE_HEAD = "{arch}.has_confidence_head"
|
||||
NORM_BEFORE_RESIDUAL = "{arch}.norm_before_residual"
|
||||
NORM_BEFORE_FC = "{arch}.norm_before_fc"
|
||||
|
||||
|
||||
@@ -1008,9 +1008,6 @@ class GGUFWriter:
|
||||
def add_sample_from_anchor(self, value: bool) -> None:
|
||||
self.add_bool(Keys.LLM.SAMPLE_FROM_ANCHOR.format(arch=self.arch), value)
|
||||
|
||||
def add_has_confidence_head(self, value: bool) -> None:
|
||||
self.add_bool(Keys.LLM.HAS_CONFIDENCE_HEAD.format(arch=self.arch), value)
|
||||
|
||||
def add_target_layers(self, value: Sequence[int]) -> None:
|
||||
self.add_array(Keys.LLM.TARGET_LAYERS.format(arch=self.arch), value)
|
||||
|
||||
|
||||
+8
-86
@@ -661,19 +661,11 @@ void llama_context::sched_reserve() {
|
||||
|
||||
// reserve again with pp graph to avoid ggml-alloc reallocations during inference
|
||||
{
|
||||
// TODO: the worst case graph is not always reached for `n_seqs > 1`
|
||||
// need to implement a more robust mechanism that tries a few different inputs and analyzes the results
|
||||
ggml_cgraph * gf = nullptr;
|
||||
switch (model.arch) {
|
||||
case LLM_ARCH_MINIMAX_01:
|
||||
// the `inp_diag_decay` tensor size scales with `n_seq_tokens^2` which
|
||||
// makes `n_seqs == 1` use more memory for the compute graph compared to `n_seqs > 1`
|
||||
gf = graph_reserve(n_tokens, 1, n_outputs_pp, mctx.get(), model.hparams.no_alloc);
|
||||
break;
|
||||
default:
|
||||
gf = graph_reserve(n_tokens, n_seqs, n_outputs_pp, mctx.get(), model.hparams.no_alloc);
|
||||
};
|
||||
|
||||
// TODO: not sure if the following graph would be worst case for multi-stream KV caches:
|
||||
//
|
||||
// auto * gf = graph_reserve(n_tokens, 1, n_tokens, mctx.get());
|
||||
//
|
||||
auto * gf = graph_reserve(n_tokens, n_seqs, n_outputs_pp, mctx.get(), model.hparams.no_alloc);
|
||||
if (!gf) {
|
||||
throw std::runtime_error("failed to allocate compute pp buffers");
|
||||
}
|
||||
@@ -2900,83 +2892,13 @@ public:
|
||||
for (auto & [buft, mbuf] : mbufs_new) {
|
||||
const auto & mbuf_cur = mbufs.at(buft);
|
||||
|
||||
if (!mbuf_cur.buf || mbuf_cur.total_size != mbuf.total_size) {
|
||||
if (!mbuf_cur.buf || mbuf_cur.n_tensors != mbuf.n_tensors || mbuf_cur.total_size != mbuf.total_size) {
|
||||
GGML_ABORT("%s: memory buffer mismatch\n", __func__);
|
||||
}
|
||||
|
||||
if (mbuf_cur.n_tensors == mbuf.n_tensors) {
|
||||
// same chunking: copy 1:1 by index
|
||||
for (size_t i = 0; i < mbuf_cur.org.size(); ++i) {
|
||||
GGML_ASSERT(ggml_nbytes(mbuf_cur.cpy[i]) == ggml_nbytes(mbuf.org[i]));
|
||||
ggml_backend_tensor_copy(mbuf_cur.cpy[i], mbuf.org[i]);
|
||||
}
|
||||
continue;
|
||||
for (size_t i = 0; i < mbuf_cur.org.size(); ++i) {
|
||||
ggml_backend_tensor_copy(mbuf_cur.cpy[i], mbuf.org[i]);
|
||||
}
|
||||
|
||||
// different chunking: copy the write-side data (mbuf_cur.cpy) into the read-side targets (mbuf.org)
|
||||
// with a byte cursor. Write and read enumerate the same logical data in the same order but may chunk
|
||||
// it differently, so copy across tensor boundaries rather than 1:1 by index.
|
||||
const size_t total = mbuf_cur.total_size;
|
||||
|
||||
ggml_init_params params_scratch = {
|
||||
/*.mem_size =*/ 2*(mbuf_cur.cpy.size() + mbuf.org.size())*ggml_tensor_overhead(),
|
||||
/*.mem_buffer =*/ NULL,
|
||||
/*.no_alloc =*/ true,
|
||||
};
|
||||
ggml_context * ctx_scratch = ggml_init(params_scratch);
|
||||
|
||||
size_t src_pos = 0;
|
||||
size_t dst_pos = 0;
|
||||
size_t src_j = 0;
|
||||
size_t dst_i = 0;
|
||||
size_t src_base = 0;
|
||||
size_t dst_base = 0;
|
||||
|
||||
while (src_pos < total) {
|
||||
const auto & src_t = mbuf_cur.cpy[src_j];
|
||||
const auto & dst_t = mbuf.org[dst_i];
|
||||
|
||||
const size_t src_size = ggml_nbytes(src_t);
|
||||
const size_t dst_size = ggml_nbytes(dst_t);
|
||||
|
||||
const size_t src_off = src_pos - src_base;
|
||||
const size_t dst_off = dst_pos - dst_base;
|
||||
|
||||
const size_t n_copy = std::min(src_size - src_off, dst_size - dst_off);
|
||||
|
||||
const size_t el = ggml_element_size(src_t);
|
||||
const int64_t n_el = (int64_t) (n_copy / el);
|
||||
|
||||
auto * src_v = ggml_view_1d(ctx_scratch, src_t, n_el, src_off);
|
||||
ggml_backend_view_init(src_v);
|
||||
auto * dst_v = ggml_view_1d(ctx_scratch, dst_t, n_el, dst_off);
|
||||
ggml_backend_view_init(dst_v);
|
||||
|
||||
ggml_backend_tensor_copy(src_v, dst_v);
|
||||
|
||||
src_pos += n_copy;
|
||||
dst_pos += n_copy;
|
||||
|
||||
if (src_pos - src_base == src_size) {
|
||||
src_base = src_pos;
|
||||
++src_j;
|
||||
}
|
||||
if (dst_pos - dst_base == dst_size) {
|
||||
dst_base = dst_pos;
|
||||
++dst_i;
|
||||
}
|
||||
}
|
||||
|
||||
GGML_ASSERT(src_pos == total && dst_pos == total);
|
||||
// any tensors left unvisited hold no data
|
||||
for (size_t i = src_j; i < mbuf_cur.cpy.size(); ++i) {
|
||||
GGML_ASSERT(ggml_nbytes(mbuf_cur.cpy[i]) == 0);
|
||||
}
|
||||
for (size_t i = dst_i; i < mbuf.org.size(); ++i) {
|
||||
GGML_ASSERT(ggml_nbytes(mbuf.org[i]) == 0);
|
||||
}
|
||||
|
||||
ggml_free(ctx_scratch);
|
||||
}
|
||||
|
||||
GGML_ASSERT(buf_size == 0);
|
||||
|
||||
@@ -672,7 +672,6 @@ struct llama_model {
|
||||
// dspark
|
||||
struct ggml_tensor * dspark_markov_w1 = nullptr;
|
||||
struct ggml_tensor * dspark_markov_w2 = nullptr;
|
||||
struct ggml_tensor * dspark_markov_w2_s = nullptr;
|
||||
struct ggml_tensor * dspark_conf_proj = nullptr;
|
||||
struct ggml_tensor * dspark_conf_proj_b = nullptr;
|
||||
|
||||
|
||||
+58
-42
@@ -742,12 +742,28 @@ static ggml_type llama_tensor_get_type(quantize_state_impl & qs, const llama_mod
|
||||
// quantization implementation
|
||||
//
|
||||
|
||||
static size_t llama_tensor_quantize_impl(enum ggml_type new_type, const float * f32_data, void * new_data, const int64_t chunk_size, int64_t nrows, int64_t n_per_row, const float * imatrix, std::vector<std::thread> & workers, const int nthread) {
|
||||
// quantize rows [first_row, first_row + nrows), indexed globally across all expert matrices
|
||||
// note: chunks never cross an expert boundary since each expert has its own imatrix slice
|
||||
static size_t llama_tensor_quantize_impl(enum ggml_type new_type, const float * f32_data, void * new_data, const int64_t chunk_size, int64_t first_row, int64_t nrows, int64_t nrows_per_expert, int64_t n_per_row, const float * imatrix, std::vector<std::thread> & workers, const int nthread) {
|
||||
const size_t row_size = ggml_row_size(new_type, n_per_row);
|
||||
|
||||
auto imatrix_for_row = [=](int64_t row_global) {
|
||||
return imatrix ? imatrix + (row_global / nrows_per_expert) * n_per_row : nullptr;
|
||||
};
|
||||
|
||||
if (nthread < 2) {
|
||||
// single-thread
|
||||
size_t new_size = ggml_quantize_chunk(new_type, f32_data, new_data, 0, nrows, n_per_row, imatrix);
|
||||
if (!ggml_validate_row_data(new_type, new_data, new_size)) {
|
||||
throw std::runtime_error("quantized data validation failed");
|
||||
size_t new_size = 0;
|
||||
for (int64_t row = 0; row < nrows;) {
|
||||
const int64_t row_global = first_row + row;
|
||||
const int64_t this_nrow = std::min(nrows - row, nrows_per_expert - row_global % nrows_per_expert);
|
||||
void * this_data = (char *) new_data + row * row_size;
|
||||
size_t this_size = ggml_quantize_chunk(new_type, f32_data + row * n_per_row, this_data, 0, this_nrow, n_per_row, imatrix_for_row(row_global));
|
||||
if (!ggml_validate_row_data(new_type, this_data, this_size)) {
|
||||
throw std::runtime_error("quantized data validation failed");
|
||||
}
|
||||
new_size += this_size;
|
||||
row += this_nrow;
|
||||
}
|
||||
return new_size;
|
||||
}
|
||||
@@ -757,26 +773,29 @@ static size_t llama_tensor_quantize_impl(enum ggml_type new_type, const float *
|
||||
size_t new_size = 0;
|
||||
bool valid = true;
|
||||
auto compute = [&mutex, &counter, &new_size, &valid, new_type, f32_data, new_data, chunk_size,
|
||||
nrows, n_per_row, imatrix]() {
|
||||
first_row, nrows, nrows_per_expert, n_per_row, row_size, imatrix_for_row]() {
|
||||
const int64_t nrows_per_chunk = chunk_size / n_per_row;
|
||||
size_t local_size = 0;
|
||||
while (true) {
|
||||
std::unique_lock<std::mutex> lock(mutex);
|
||||
int64_t first_row = counter; counter += nrows_per_chunk;
|
||||
if (first_row >= nrows) {
|
||||
if (counter >= nrows) {
|
||||
if (local_size > 0) {
|
||||
new_size += local_size;
|
||||
}
|
||||
break;
|
||||
}
|
||||
const int64_t row = counter;
|
||||
const int64_t row_global = first_row + row;
|
||||
// stop at the expert boundary
|
||||
const int64_t this_nrow = std::min(std::min(nrows - row, nrows_per_chunk), nrows_per_expert - row_global % nrows_per_expert);
|
||||
counter += this_nrow;
|
||||
lock.unlock();
|
||||
const int64_t this_nrow = std::min(nrows - first_row, nrows_per_chunk);
|
||||
size_t this_size = ggml_quantize_chunk(new_type, f32_data, new_data, first_row * n_per_row, this_nrow, n_per_row, imatrix);
|
||||
|
||||
void * this_data = (char *) new_data + row * row_size;
|
||||
size_t this_size = ggml_quantize_chunk(new_type, f32_data + row * n_per_row, this_data, 0, this_nrow, n_per_row, imatrix_for_row(row_global));
|
||||
local_size += this_size;
|
||||
|
||||
// validate the quantized data
|
||||
const size_t row_size = ggml_row_size(new_type, n_per_row);
|
||||
void * this_data = (char *) new_data + first_row * row_size;
|
||||
if (!ggml_validate_row_data(new_type, this_data, this_size)) {
|
||||
std::unique_lock<std::mutex> lock(mutex);
|
||||
valid = false;
|
||||
@@ -1258,52 +1277,49 @@ static void llama_model_quantize_impl(const std::string & fname_inp, const std::
|
||||
fflush(stdout);
|
||||
|
||||
const int64_t n_per_row = tensor->ne[0];
|
||||
const int64_t nrows = tensor->ne[1];
|
||||
const int64_t nrows_per_expert = tensor->ne[1];
|
||||
const int64_t nrows_total = tensor->ne[1] * tensor->ne[2];
|
||||
|
||||
const size_t row_size_src = ggml_row_size(tensor->type, n_per_row);
|
||||
const size_t row_size_dst = ggml_row_size(new_type, n_per_row);
|
||||
|
||||
// process the rows in slabs, so that the buffers stay below max_buf_size
|
||||
const size_t bytes_per_row = row_size_src + row_size_dst + (tensor->type == GGML_TYPE_F32 ? 0 : n_per_row*sizeof(float));
|
||||
const int64_t nrows_slab = std::max<int64_t>(1, std::min<int64_t>(nrows, max_buf_size/bytes_per_row));
|
||||
const int64_t nrows_slab = std::max<int64_t>(1, std::min<int64_t>(nrows_total, max_buf_size/bytes_per_row));
|
||||
|
||||
static const int64_t min_chunk_size = 32 * 512;
|
||||
const int64_t chunk_size = (n_per_row >= min_chunk_size ? n_per_row : n_per_row * ((min_chunk_size + n_per_row - 1)/n_per_row));
|
||||
|
||||
// quantize each expert separately since they have different importance matrices
|
||||
// process rows across all experts in one pass to keep all threads busy
|
||||
new_size = 0;
|
||||
for (int64_t i03 = 0; i03 < tensor->ne[2]; ++i03) {
|
||||
const float * imatrix_03 = imatrix ? imatrix + i03 * n_per_row : nullptr;
|
||||
for (int64_t ir = 0; ir < nrows_total; ir += nrows_slab) {
|
||||
const int64_t nrows_cur = std::min(nrows_slab, nrows_total - ir);
|
||||
const int64_t nelements_cur = nrows_cur * n_per_row;
|
||||
|
||||
for (int64_t ir = 0; ir < nrows; ir += nrows_slab) {
|
||||
const int64_t nrows_cur = std::min(nrows_slab, nrows - ir);
|
||||
const int64_t nelements_cur = nrows_cur * n_per_row;
|
||||
const void * src = load_range(ir*row_size_src, nrows_cur*row_size_src);
|
||||
|
||||
const void * src = load_range((i03*nrows + ir)*row_size_src, nrows_cur*row_size_src);
|
||||
|
||||
const float * f32_data;
|
||||
if (tensor->type == GGML_TYPE_F32) {
|
||||
f32_data = (const float *) src;
|
||||
} else {
|
||||
if (f32_conv_buf.size() < (size_t) nelements_cur) {
|
||||
f32_conv_buf.resize(nelements_cur);
|
||||
}
|
||||
llama_tensor_dequantize_impl(tensor->type, src, (float *) f32_conv_buf.data(), workers, nelements_cur, nthread);
|
||||
f32_data = (const float *) f32_conv_buf.data();
|
||||
const float * f32_data;
|
||||
if (tensor->type == GGML_TYPE_F32) {
|
||||
f32_data = (const float *) src;
|
||||
} else {
|
||||
if (f32_conv_buf.size() < (size_t) nelements_cur) {
|
||||
f32_conv_buf.resize(nelements_cur);
|
||||
}
|
||||
|
||||
if (work.size() < nrows_cur*row_size_dst) {
|
||||
work.resize(nrows_cur*row_size_dst);
|
||||
}
|
||||
|
||||
const int64_t nchunk = (nelements_cur + chunk_size - 1)/chunk_size;
|
||||
const int64_t nthread_use = nthread > 1 ? std::max((int64_t)1, std::min((int64_t)nthread, nchunk)) : 1;
|
||||
|
||||
const size_t size_cur = llama_tensor_quantize_impl(new_type, f32_data, work.data(), chunk_size, nrows_cur, n_per_row, imatrix_03, workers, nthread_use);
|
||||
|
||||
fout.write((const char *) work.data(), size_cur);
|
||||
new_size += size_cur;
|
||||
llama_tensor_dequantize_impl(tensor->type, src, (float *) f32_conv_buf.data(), workers, nelements_cur, nthread);
|
||||
f32_data = (const float *) f32_conv_buf.data();
|
||||
}
|
||||
|
||||
if (work.size() < nrows_cur*row_size_dst) {
|
||||
work.resize(nrows_cur*row_size_dst);
|
||||
}
|
||||
|
||||
const int64_t nchunk = (nelements_cur + chunk_size - 1)/chunk_size;
|
||||
const int64_t nthread_use = nthread > 1 ? std::max((int64_t)1, std::min((int64_t)nthread, nchunk)) : 1;
|
||||
|
||||
const size_t size_cur = llama_tensor_quantize_impl(new_type, f32_data, work.data(), chunk_size, ir, nrows_cur, nrows_per_expert, n_per_row, imatrix, workers, nthread_use);
|
||||
|
||||
fout.write((const char *) work.data(), size_cur);
|
||||
new_size += size_cur;
|
||||
}
|
||||
LLAMA_LOG_INFO("size = %8.2f MiB -> %8.2f MiB\n", tensor_size/1024.0/1024.0, new_size/1024.0/1024.0);
|
||||
}
|
||||
|
||||
+24
-34
@@ -115,11 +115,10 @@ void llama_model_dflash::load_arch_tensors(llama_model_loader &) {
|
||||
if (markov_meta) {
|
||||
const int64_t dspark_markov_rank = markov_meta->ne[0];
|
||||
|
||||
dspark_markov_w1 = create_tensor(tn(LLM_TENSOR_DSPARK_MARKOV_W1, "weight"), { dspark_markov_rank, n_vocab }, 0);
|
||||
dspark_markov_w2 = create_tensor(tn(LLM_TENSOR_DSPARK_MARKOV_W2, "weight"), { dspark_markov_rank, n_vocab_draft }, 0);
|
||||
dspark_markov_w2_s = create_tensor(tn(LLM_TENSOR_DSPARK_MARKOV_W2, "scale"), { 1 }, TENSOR_NOT_REQUIRED);
|
||||
dspark_markov_w1 = create_tensor(tn(LLM_TENSOR_DSPARK_MARKOV_W1, "weight"), { dspark_markov_rank, n_vocab }, 0);
|
||||
dspark_markov_w2 = create_tensor(tn(LLM_TENSOR_DSPARK_MARKOV_W2, "weight"), { dspark_markov_rank, n_vocab_draft }, 0);
|
||||
|
||||
dspark_conf_proj = create_tensor(tn(LLM_TENSOR_DSPARK_CONF_PROJ, "weight"), { n_embd + dspark_markov_rank, 1 }, TENSOR_NOT_REQUIRED);
|
||||
dspark_conf_proj = create_tensor(tn(LLM_TENSOR_DSPARK_CONF_PROJ, "weight"), { n_embd + dspark_markov_rank, 1 }, 0);
|
||||
dspark_conf_proj_b = create_tensor(tn(LLM_TENSOR_DSPARK_CONF_PROJ, "bias"), { 1 }, TENSOR_NOT_REQUIRED);
|
||||
|
||||
LLAMA_LOG_INFO("%s: DFlash with DSpark markov head (rank = %lld)\n", __func__, (long long) dspark_markov_rank);
|
||||
@@ -220,9 +219,6 @@ void llama_model_dflash::load_arch_tensors(llama_model_loader &) {
|
||||
layer.attn_q_norm = create_tensor(tn(LLM_TENSOR_ATTN_Q_NORM, "weight", i), { n_embd_head_k }, 0);
|
||||
layer.attn_k_norm = create_tensor(tn(LLM_TENSOR_ATTN_K_NORM, "weight", i), { n_embd_head_k }, 0);
|
||||
|
||||
// optional per-head attention sinks (e.g. Nemotron DSpark)
|
||||
layer.attn_sinks = create_tensor(tn(LLM_TENSOR_ATTN_SINKS, "weight", i), { n_head }, TENSOR_NOT_REQUIRED);
|
||||
|
||||
layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), { n_embd }, 0);
|
||||
layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, "weight", i), { n_embd, n_ff }, 0);
|
||||
layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "weight", i), { n_ff, n_embd }, 0);
|
||||
@@ -294,10 +290,7 @@ static void build_dspark_markov_head(llm_graph_context & g, const llama_model &
|
||||
|
||||
ggml_tensor * w1 = model.dspark_markov_w1;
|
||||
ggml_tensor * w2 = model.dspark_markov_w2;
|
||||
GGML_ASSERT(w1 && w2 && "DSpark markov weights not loaded");
|
||||
|
||||
// confidence head is optional
|
||||
const bool has_conf = model.dspark_conf_proj != nullptr;
|
||||
GGML_ASSERT(w1 && w2 && model.dspark_conf_proj && "DSpark markov/confidence weights not loaded");
|
||||
|
||||
ggml_tensor * base = res->t_logits; // [n_vocab, n_tokens]
|
||||
const int64_t n_vocab = base->ne[0];
|
||||
@@ -328,22 +321,23 @@ static void build_dspark_markov_head(llm_graph_context & g, const llama_model &
|
||||
ggml_tensor * prev = ggml_view_2d(ctx0, tokens, 1, n_blocks, token_stride, 0);
|
||||
prev = ggml_cont_1d(ctx0, prev, n_blocks);
|
||||
|
||||
// confidence head input: predicts per-position acceptance
|
||||
ggml_tensor * conf_inp = res->t_embd; // [n_embd, n_tok]
|
||||
|
||||
ggml_tensor * cat = nullptr;
|
||||
ggml_tensor * cat_conf = nullptr;
|
||||
|
||||
if (!sample_from_anchor) {
|
||||
// bonus anchor slot: pass the logits through unbiased, pad the (unread) confidence column
|
||||
cat = ggml_cont(ctx0, ggml_view_2d(ctx0, base, n_vocab, n_blocks, base_stride, 0));
|
||||
if (has_conf) {
|
||||
cat_conf = ggml_sigmoid(ctx0, ggml_cont(ctx0, ggml_view_2d(ctx0, base, 1, n_blocks, base_stride, 0)));
|
||||
}
|
||||
cat = ggml_cont(ctx0, ggml_view_2d(ctx0, base, n_vocab, n_blocks, base_stride, 0));
|
||||
cat_conf = ggml_sigmoid(ctx0, ggml_cont(ctx0, ggml_view_2d(ctx0, base, 1, n_blocks, base_stride, 0)));
|
||||
}
|
||||
|
||||
// TODO: the in-graph chain is greedy (argmax); sampling params affect only the final
|
||||
// token pick, not the Markov conditioning path
|
||||
for (int64_t i = i_draft_beg; i < block_drafts; ++i) {
|
||||
ggml_tensor * w1_prev = ggml_get_rows(ctx0, w1, prev); // [R, n_blocks]
|
||||
ggml_tensor * bias = g.build_lora_mm(w2, w1_prev, model.dspark_markov_w2_s); // [n_vocab_draft, n_blocks]
|
||||
ggml_tensor * w1_prev = ggml_get_rows(ctx0, w1, prev); // [R, n_blocks]
|
||||
ggml_tensor * bias = ggml_mul_mat(ctx0, w2, w1_prev); // [n_vocab_draft, n_blocks]
|
||||
if (model.d2t) {
|
||||
// reduced draft vocab: scatter the bias to the target rows (base is -inf on the others)
|
||||
const int64_t n_draft_vocab = bias->ne[0];
|
||||
@@ -360,21 +354,17 @@ static void build_dspark_markov_head(llm_graph_context & g, const llama_model &
|
||||
|
||||
cat = cat ? ggml_concat(ctx0, cat, col, 1) : col;
|
||||
|
||||
if (has_conf) {
|
||||
// confidence head input: predicts per-position acceptance
|
||||
ggml_tensor * conf_inp = res->t_embd; // [n_embd, n_tok]
|
||||
// conf(i) = sigmoid(conf_proj . [conf_inp(i); markov_w1[prev(i)]] + b) -- [1, n_blocks]
|
||||
ggml_tensor * conf_inp_i = ggml_view_2d(ctx0, conf_inp, conf_inp->ne[0], n_blocks,
|
||||
(size_t) block_drafts * conf_inp->nb[1], i*conf_inp->nb[1]);
|
||||
ggml_tensor * feat = ggml_concat(ctx0, ggml_cont(ctx0, conf_inp_i), w1_prev, 0);
|
||||
ggml_tensor * conf = ggml_mul_mat(ctx0, model.dspark_conf_proj, feat);
|
||||
if (model.dspark_conf_proj_b) {
|
||||
conf = ggml_add(ctx0, conf, model.dspark_conf_proj_b);
|
||||
}
|
||||
conf = ggml_sigmoid(ctx0, conf);
|
||||
|
||||
cat_conf = cat_conf ? ggml_concat(ctx0, cat_conf, conf, 1) : conf;
|
||||
// conf(i) = sigmoid(conf_proj . [conf_inp(i); markov_w1[prev(i)]] + b) -- [1, n_blocks]
|
||||
ggml_tensor * conf_inp_i = ggml_view_2d(ctx0, conf_inp, conf_inp->ne[0], n_blocks,
|
||||
(size_t) block_drafts * conf_inp->nb[1], i*conf_inp->nb[1]);
|
||||
ggml_tensor * feat = ggml_concat(ctx0, ggml_cont(ctx0, conf_inp_i), w1_prev, 0);
|
||||
ggml_tensor * conf = ggml_mul_mat(ctx0, model.dspark_conf_proj, feat);
|
||||
if (model.dspark_conf_proj_b) {
|
||||
conf = ggml_add(ctx0, conf, model.dspark_conf_proj_b);
|
||||
}
|
||||
conf = ggml_sigmoid(ctx0, conf);
|
||||
|
||||
cat_conf = cat_conf ? ggml_concat(ctx0, cat_conf, conf, 1) : conf;
|
||||
|
||||
if (i + 1 < block_drafts) {
|
||||
prev = ggml_argmax(ctx0, col);
|
||||
@@ -386,7 +376,7 @@ static void build_dspark_markov_head(llm_graph_context & g, const llama_model &
|
||||
out = ggml_cont(ctx0, ggml_permute(ctx0, out, 0, 2, 1, 3)); // [n_vocab, block_drafts, n_blocks]
|
||||
out = ggml_reshape_2d(ctx0, out, n_vocab, n_tok);
|
||||
|
||||
if (has_conf) {
|
||||
{
|
||||
ggml_tensor * conf = ggml_reshape_3d(ctx0, cat_conf, 1, n_blocks, block_drafts);
|
||||
conf = ggml_cont(ctx0, ggml_permute(ctx0, conf, 0, 2, 1, 3));
|
||||
conf = ggml_reshape_2d(ctx0, conf, 1, n_tok);
|
||||
@@ -717,8 +707,8 @@ llama_model_dflash::graph<false>::graph(const llama_model & model, const llm_gra
|
||||
|
||||
// cache-aware, non-causal attention
|
||||
ggml_tensor * cur = use_iswa
|
||||
? build_attn(inp_attn_iswa, layer.wo, NULL, NULL, Qcur, Kcur, Vcur, nullptr, layer.attn_sinks, nullptr, kq_scale, il)
|
||||
: build_attn(inp_attn, layer.wo, NULL, NULL, Qcur, Kcur, Vcur, nullptr, layer.attn_sinks, nullptr, kq_scale, il);
|
||||
? build_attn(inp_attn_iswa, layer.wo, NULL, NULL, Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, kq_scale, il)
|
||||
: build_attn(inp_attn, layer.wo, NULL, NULL, Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, kq_scale, il);
|
||||
|
||||
if (attn_dynamic) {
|
||||
cur = build_dflash2_conv(*this, cur, attn_dynamic, layer.dflash_attn_conv_base, 1);
|
||||
|
||||
@@ -181,7 +181,7 @@ public:
|
||||
return res;
|
||||
}
|
||||
|
||||
const llama_hparams hparams;
|
||||
const llama_hparams & hparams;
|
||||
|
||||
ggml_tensor * inp_slopes = nullptr; // F32 [n_head]
|
||||
ggml_tensor * inp_q_decay = nullptr; // F32 [1, n_head, n_batch]
|
||||
|
||||
@@ -149,7 +149,6 @@ if (LLAMA_LLGUIDANCE)
|
||||
endif ()
|
||||
|
||||
llama_build(test-recurrent-state-rollback.cpp)
|
||||
llama_build(test-save-load-state.cpp)
|
||||
|
||||
if (NOT WIN32 OR NOT BUILD_SHARED_LIBS)
|
||||
# these tests are disabled on Windows because they use internal functions not exported with LLAMA_API (when building with shared libraries)
|
||||
@@ -238,14 +237,6 @@ if (NOT WIN32 OR NOT BUILD_SHARED_LIBS)
|
||||
set_tests_properties(test-recurrent-state-rollback-dsv4 PROPERTIES
|
||||
FIXTURES_REQUIRED generate-models
|
||||
)
|
||||
|
||||
# Test state save/load functionality across all architectures, using the generated dummy models
|
||||
llama_test(
|
||||
test-save-load-state
|
||||
LABEL main
|
||||
ARGS --models "${MODEL_DIR}"
|
||||
)
|
||||
set_tests_properties(test-save-load-state PROPERTIES FIXTURES_REQUIRED generate-models)
|
||||
endif()
|
||||
|
||||
llama_build_and_test(test-chat-peg-parser.cpp peg-parser/simple-tokenize.cpp)
|
||||
@@ -308,6 +299,10 @@ llama_build_and_test(test-backend-sampler.cpp LABEL "model")
|
||||
llama_build_and_test(test-state-restore-fragmented.cpp LABEL "model" ARGS -m "${MODEL_DEST}")
|
||||
set_tests_properties(test-state-restore-fragmented PROPERTIES FIXTURES_REQUIRED test-download-model)
|
||||
|
||||
# Test state save/load functionality
|
||||
llama_build_and_test(test-save-load-state.cpp LABEL "model" ARGS -m "${MODEL_DEST}")
|
||||
set_tests_properties(test-save-load-state PROPERTIES FIXTURES_REQUIRED test-download-model)
|
||||
|
||||
if (APPLE)
|
||||
llama_build(test-rset-release.cpp)
|
||||
endif()
|
||||
|
||||
@@ -65,7 +65,7 @@ static void set_tensor_data(struct ggml_tensor * tensor, void * userdata) {
|
||||
}
|
||||
|
||||
static void usage(char ** argv) {
|
||||
printf("Usage: %s [-a/--arch arch] [-s/--seed seed] [-o/--out dir] [-v/--verbose] [-h/--help]\n", argv[0]);
|
||||
printf("Usage: %s [-a/--arch arch] [-s/--seed seed] [-v/--verbose]\n", argv[0]);
|
||||
}
|
||||
|
||||
static std::vector<llama_token> get_tokens(const uint32_t n_tokens, const uint32_t n_vocab, const size_t seed){
|
||||
@@ -82,7 +82,7 @@ static std::vector<llama_token> get_tokens(const uint32_t n_tokens, const uint32
|
||||
static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) {
|
||||
gguf_context_ptr ret(gguf_init_empty());
|
||||
llama_model_saver ms(arch, ret.get());
|
||||
const uint32_t n_ctx = 256;
|
||||
const uint32_t n_ctx = 128;
|
||||
|
||||
uint32_t n_vocab = 128;
|
||||
uint32_t n_embd = 256;
|
||||
@@ -256,12 +256,10 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) {
|
||||
ms.add_kv(LLM_KV_ATTENTION_COMPRESS_RATIOS, std::vector<uint32_t>(n_layer, 4));
|
||||
}
|
||||
|
||||
// minimax-m3 keeps one indexer head per GQA head; the rest use a fixed 64 to match the fused
|
||||
ms.add_kv(LLM_KV_ATTENTION_INDEXER_HEAD_COUNT, arch == LLM_ARCH_MINIMAX_M3 ? n_head : uint32_t(64));
|
||||
ms.add_kv(LLM_KV_ATTENTION_INDEXER_HEAD_COUNT, arch == LLM_ARCH_MINIMAX_M3 || arch == LLM_ARCH_DEEPSEEK4 ? n_head : uint32_t(1));
|
||||
// qwen4exp ropes indexer keys with the main rotary width, so its head can't be < n_rot
|
||||
ms.add_kv(LLM_KV_ATTENTION_INDEXER_KEY_LENGTH,
|
||||
arch == LLM_ARCH_QWEN4EXP ? n_embd_head : uint32_t(128));
|
||||
|
||||
arch == LLM_ARCH_QWEN4EXP ? n_embd_head : uint32_t(64));
|
||||
ms.add_kv(LLM_KV_ATTENTION_INDEXER_TOP_K, uint32_t(8));
|
||||
ms.add_kv(LLM_KV_ATTENTION_INDEXER_BLOCK_SIZE, uint32_t(4));
|
||||
ms.add_kv(LLM_KV_ATTENTION_INDEXER_LOCAL_BLOCKS, uint32_t(1));
|
||||
@@ -764,10 +762,6 @@ int main(int argc, char ** argv) {
|
||||
std::string out;
|
||||
|
||||
for (int i = 1; i < argc; i++) {
|
||||
if (strcmp(argv[i], "-h") == 0 || strcmp(argv[i], "--help") == 0) {
|
||||
usage(argv);
|
||||
return 0;
|
||||
}
|
||||
if (strcmp(argv[i], "-a") == 0 || strcmp(argv[i], "--arch") == 0) {
|
||||
if (i + 1 < argc) {
|
||||
const std::string arch_name = argv[++i];
|
||||
|
||||
+35
-120
@@ -3,12 +3,8 @@
|
||||
#include "log.h"
|
||||
#include "llama-cpp.h"
|
||||
|
||||
#include <algorithm>
|
||||
#include <clocale>
|
||||
#include <cstring>
|
||||
#include <filesystem>
|
||||
#include <random>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
struct llama_batch_ptr {
|
||||
@@ -57,9 +53,7 @@ static llama_tokens generate_tokens(llama_context * ctx, llama_sampler * smpl, i
|
||||
// - decode the last token
|
||||
// - generate n_predict tokens
|
||||
static llama_tokens test_baseline(struct llama_model * model, const struct common_params & params, const llama_tokens & tokens) {
|
||||
auto params_ctx = common_context_params_to_llama(params);
|
||||
params_ctx.n_seq_max = 2;
|
||||
auto ctx = llama_context_ptr{llama_init_from_model(model, params_ctx)};
|
||||
auto ctx = llama_context_ptr{llama_init_from_model(model, common_context_params_to_llama(params))};
|
||||
|
||||
auto sparams = llama_sampler_chain_default_params();
|
||||
auto smpl = llama_sampler_ptr{llama_sampler_chain_init(sparams)};
|
||||
@@ -167,9 +161,7 @@ static bool test_seq_rm_isolated(
|
||||
// - replay the last prompt token
|
||||
// - generate n_predict tokens and compare against expected result
|
||||
static bool test_state_load(struct llama_model * model, const struct common_params & params, const llama_tokens & tokens, const llama_tokens & expected_result) {
|
||||
auto params_ctx = common_context_params_to_llama(params);
|
||||
params_ctx.n_seq_max = 2;
|
||||
auto ctx = llama_context_ptr{llama_init_from_model(model, params_ctx)};
|
||||
auto ctx = llama_context_ptr{llama_init_from_model(model, common_context_params_to_llama(params))};
|
||||
|
||||
auto sparams = llama_sampler_chain_default_params();
|
||||
auto smpl = llama_sampler_ptr{llama_sampler_chain_init(sparams)};
|
||||
@@ -355,18 +347,38 @@ static bool test_seq_cp_device(struct llama_model * model, const struct common_p
|
||||
}
|
||||
|
||||
|
||||
// Run the full save/load test suite (tests 1-5) for a single model.
|
||||
// Returns true if all tests pass, false otherwise.
|
||||
static bool run_save_load_tests_for_model(const std::string & model_path, const struct common_params & base_params) {
|
||||
struct common_params params = base_params;
|
||||
params.model.path = model_path;
|
||||
int main(int argc, char ** argv) {
|
||||
std::setlocale(LC_NUMERIC, "C");
|
||||
|
||||
common_params params;
|
||||
params.prompt = "";
|
||||
params.n_batch = 100;
|
||||
params.out_file = "dump_state.bin";
|
||||
params.sampling.seed = 1234;
|
||||
|
||||
common_init();
|
||||
|
||||
if (!common_params_parse(argc, argv, params, LLAMA_EXAMPLE_COMMON)) {
|
||||
return 1;
|
||||
}
|
||||
|
||||
if (params.n_parallel == 1) {
|
||||
LOG_TRC("%s: n_parallel == 1, enabling unified kv cache\n", __func__);
|
||||
params.kv_unified = true;
|
||||
}
|
||||
|
||||
if (params.n_predict < 0) {
|
||||
params.n_predict = 16;
|
||||
}
|
||||
|
||||
ggml_backend_load_all();
|
||||
|
||||
auto llama_init = common_init_from_params(params, true);
|
||||
auto * model = llama_init->model();
|
||||
|
||||
if (model == nullptr) {
|
||||
LOG_ERR("%s: failed to init model '%s'\n", __func__, model_path.c_str());
|
||||
return false;
|
||||
LOG_ERR("%s: failed to init\n", __func__);
|
||||
return 1;
|
||||
}
|
||||
|
||||
GGML_ASSERT(llama_init->context() == nullptr);
|
||||
@@ -399,127 +411,30 @@ static bool run_save_load_tests_for_model(const std::string & model_path, const
|
||||
// Test 1: baseline (saves state to disk)
|
||||
auto result_baseline = test_baseline(model, params, tokens);
|
||||
if (result_baseline.empty()) {
|
||||
return false;
|
||||
return 1;
|
||||
}
|
||||
|
||||
// Test 2: sequence removal isolation
|
||||
if (!test_seq_rm_isolated(model, params, tokens)) {
|
||||
return false;
|
||||
return 1;
|
||||
}
|
||||
|
||||
// Test 3: state load
|
||||
if (!test_state_load(model, params, tokens, result_baseline)) {
|
||||
return false;
|
||||
return 1;
|
||||
}
|
||||
|
||||
// Test 4: seq copy (host)
|
||||
if (!test_seq_cp_host(model, params, tokens, result_baseline)) {
|
||||
return false;
|
||||
return 1;
|
||||
}
|
||||
|
||||
// Test 5: seq copy (device)
|
||||
if (!test_seq_cp_device(model, params, tokens, result_baseline)) {
|
||||
return false;
|
||||
return 1;
|
||||
}
|
||||
|
||||
LOG("\nAll tests passed.\n");
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
|
||||
int main(int argc, char ** argv) {
|
||||
std::setlocale(LC_NUMERIC, "C");
|
||||
|
||||
common_params params;
|
||||
params.prompt = "";
|
||||
params.n_batch = 100;
|
||||
params.out_file = "dump_state.bin";
|
||||
params.sampling.seed = 1234;
|
||||
|
||||
common_init();
|
||||
|
||||
// extract our own --models DIR option before handing the rest to the common arg parser
|
||||
std::string models_dir;
|
||||
std::vector<char *> filtered_argv;
|
||||
filtered_argv.push_back(argv[0]);
|
||||
for (int i = 1; i < argc; i++) {
|
||||
if (strcmp(argv[i], "--models") == 0) {
|
||||
if (i + 1 >= argc) {
|
||||
LOG_ERR("%s: --models requires a directory argument\n", __func__);
|
||||
return 1;
|
||||
}
|
||||
models_dir = argv[i + 1];
|
||||
i++;
|
||||
} else {
|
||||
filtered_argv.push_back(argv[i]);
|
||||
}
|
||||
}
|
||||
filtered_argv.push_back(nullptr);
|
||||
const int fargc = (int)filtered_argv.size() - 1;
|
||||
|
||||
// in --models mode there is no single model; set a placeholder so the common parser's
|
||||
// "--model is required" check passes (each model is set individually inside the loop)
|
||||
if (!models_dir.empty()) {
|
||||
params.model.path = models_dir;
|
||||
}
|
||||
|
||||
if (!common_params_parse(fargc, filtered_argv.data(), params, LLAMA_EXAMPLE_COMMON)) {
|
||||
return 1;
|
||||
}
|
||||
|
||||
if (params.n_parallel == 1) {
|
||||
LOG_TRC("%s: n_parallel == 1, enabling unified kv cache\n", __func__);
|
||||
params.kv_unified = true;
|
||||
}
|
||||
|
||||
if (params.n_predict < 0) {
|
||||
params.n_predict = 16;
|
||||
}
|
||||
|
||||
ggml_backend_load_all();
|
||||
|
||||
if (!models_dir.empty()) {
|
||||
// run the suite over every dummy model in the directory
|
||||
if (!std::filesystem::exists(models_dir) || !std::filesystem::is_directory(models_dir)) {
|
||||
LOG_ERR("%s: models directory '%s' does not exist\n", __func__, models_dir.c_str());
|
||||
return 1;
|
||||
}
|
||||
|
||||
std::vector<std::string> models;
|
||||
for (const auto & entry : std::filesystem::directory_iterator(models_dir)) {
|
||||
if (entry.is_regular_file() && entry.path().extension() == ".gguf") {
|
||||
models.push_back(entry.path().string());
|
||||
}
|
||||
}
|
||||
std::sort(models.begin(), models.end());
|
||||
|
||||
if (models.empty()) {
|
||||
LOG_ERR("%s: no .gguf models found in '%s'\n", __func__, models_dir.c_str());
|
||||
return 1;
|
||||
}
|
||||
|
||||
LOG_INF("%s: running save/load tests over %zu models in '%s'\n", __func__, models.size(), models_dir.c_str());
|
||||
|
||||
size_t n_pass = 0;
|
||||
size_t n_fail = 0;
|
||||
for (const auto & model_path : models) {
|
||||
LOG("\n================================================================\n");
|
||||
LOG_INF("%s: model %s\n", __func__, model_path.c_str());
|
||||
|
||||
if (run_save_load_tests_for_model(model_path, params)) {
|
||||
n_pass++;
|
||||
} else {
|
||||
n_fail++;
|
||||
}
|
||||
}
|
||||
|
||||
LOG("\n================================================================\n");
|
||||
LOG_INF("%s: summary: %zu passed, %zu failed (of %zu)\n", __func__, n_pass, n_fail, models.size());
|
||||
|
||||
return n_fail == 0 ? 0 : 1;
|
||||
}
|
||||
|
||||
// single-model mode
|
||||
return run_save_load_tests_for_model(params.model.path, params) ? 0 : 1;
|
||||
return 0;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user