ClaudeBot/llama.cpp
0
0
Fork 1
mirror of https://github.com/ggml-org/llama.cpp.git synced 2026-09-17 20:31:47 +02:00
Code Issues Packages Projects Releases Wiki Activity

hparams: add per-layer n_ff_exp/n_expert_used arrays with scalar-or-array loading

Browse Source 
G1/G2 infrastructure for variable-per-layer expert FFN size and top-k routing
(required for Puzzle-75B which has 5 distinct n_ff_exp values and 7 top-k values
across its 40 MoE layers).

Design: rename scalar members to _impl suffix (following existing convention),
add LLAMA_MAX_LAYERS arrays, add n_ff_exp(il)/n_expert_used(il) accessors with
scalar fallback. No new GGUF keys: reuses existing expert_feed_forward_length and
expert_used_count keys via get_key_or_arr (scalar -> broadcast, array -> per-layer).

- llama-hparams.h: n_ff_exp -> n_ff_exp_impl, n_expert_used -> n_expert_used_impl;
  add n_ff_exp_arr / n_expert_used_arr arrays; add per-layer accessor declarations.
- llama-hparams.cpp: implement n_ff_exp(il) and n_expert_used(il); out-of-range
  il returns impl safely (shared code, no abort).
- llama-model.cpp: central n_expert_used load changed to get_key_or_arr; derive
  impl as max-of-array for validations and backward compat; zero both new arrays;
  HunyuanVL override also zeroes n_expert_used_arr.
- llama-graph.cpp: aggregation loop in build_moe_ffn uses hparams.n_expert_used(il)
  so per-layer top-k bounds the ggml_view loop correctly.
- All other files: mechanical rename hparams.n_{ff_exp,expert_used} -> *_impl.
  Scalar arches are unaffected (broadcast fills all array slots with the single value).
This commit is contained in:
Yaniss
2026-07-26 11:30:37 +02:00
parent ff067f76dd
commit 269a81e03d
7 changed files with 72 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/llama-graph.cpp
+8 -7
View File
@@ -1337,7 +1337,7 @@ llm_graph_context::llm_graph_context(const llm_graph_params & params) :
n_embd_head_v (hparams.n_embd_head_v()),
n_embd_v_gqa (hparams.n_embd_v_gqa()),
n_expert (hparams.n_expert),
n_expert_used (cparams.warmup ? hparams.n_expert : hparams.n_expert_used),
n_expert_used (cparams.warmup ? hparams.n_expert : hparams.n_expert_used_impl),
freq_base (cparams.rope_freq_base),
freq_scale (cparams.rope_freq_scale),
ext_factor (cparams.yarn_ext_factor),
@@ -2119,25 +2119,26 @@ ggml_tensor * llm_graph_context::build_moe_ffn(
assert(n_expert_used > 0);
// order the views before the adds
for (uint32_t i = 0; i < hparams.n_expert_used; ++i) {
// Use per-layer n_expert_used to bound the graph even during warmup (avoids
// the large-add-nodes issue for uniform arches; for Puzzle the per-layer
// value is correct). ref: https://github.com/ggml-org/llama.cpp/pull/14753
const uint32_t n_expert_used_il = hparams.n_expert_used(il);
for (uint32_t i = 0; i < n_expert_used_il; ++i) {
cur_experts[i] = ggml_view_2d(ctx0, experts, n_embd, n_tokens, experts->nb[2], i*experts->nb[1]);
ggml_build_forward_expand(gf, cur_experts[i]);
}
// aggregate experts
// note: here we explicitly use hparams.n_expert_used instead of n_expert_used
// to avoid potentially a large number of add nodes during warmup
// ref: https://github.com/ggml-org/llama.cpp/pull/14753
ggml_tensor * moe_out = cur_experts[0];
for (uint32_t i = 1; i < hparams.n_expert_used; ++i) {
for (uint32_t i = 1; i < n_expert_used_il; ++i) {
moe_out = ggml_add(ctx0, moe_out, cur_experts[i]);
ggml_build_forward_expand(gf, moe_out);
}
if (hparams.n_expert_used == 1) {
if (n_expert_used_il == 1) {
// avoid returning a non-contiguous tensor
moe_out = ggml_cont(ctx0, moe_out);
}
src/llama-hparams.cpp
+17
View File
@@ -71,6 +71,23 @@ uint32_t llama_hparams::n_ff(uint32_t il) const {
GGML_ABORT("fatal error");
}
uint32_t llama_hparams::n_ff_exp(uint32_t il) const {
// Return per-layer value when available, else fall back to the scalar impl.
// Out-of-range il (e.g. nextn layers not yet counted) returns impl safely.
if (il < n_layer_all) {
return n_ff_exp_arr[il] != 0 ? n_ff_exp_arr[il] : n_ff_exp_impl;
}
return n_ff_exp_impl;
}
uint32_t llama_hparams::n_expert_used(uint32_t il) const {
// Return per-layer value when available, else fall back to the scalar impl.
if (il < n_layer_all) {
return n_expert_used_arr[il] != 0 ? n_expert_used_arr[il] : n_expert_used_impl;
}
return n_expert_used_impl;
}
uint32_t llama_hparams::n_gqa(uint32_t il) const {
const uint32_t n_head = this->n_head(il);
const uint32_t n_head_kv = this->n_head_kv(il);
src/llama-hparams.h
+13 -2
View File
@@ -53,7 +53,7 @@ struct llama_hparams {
uint32_t n_layer_all;
uint32_t n_layer_nextn = 0;
uint32_t n_expert = 0;
uint32_t n_expert_used = 0;
uint32_t n_expert_used_impl = 0;
uint32_t n_rel_attn_bkts = 0;
// TODO: this needs to be reworked
@@ -83,10 +83,15 @@ struct llama_hparams {
std::array<uint32_t, LLAMA_MAX_LAYERS> n_head_kv_arr;
std::array<uint32_t, LLAMA_MAX_LAYERS> n_ff_arr;
// per-layer expert feed-forward size; scalar impl as broadcast fallback
std::array<uint32_t, LLAMA_MAX_LAYERS> n_ff_exp_arr;
// per-layer top-k routing; scalar impl as broadcast fallback
std::array<uint32_t, LLAMA_MAX_LAYERS> n_expert_used_arr;
uint32_t n_layer_dense_lead = 0;
uint32_t n_lora_q = 0;
uint32_t n_lora_kv = 0;
uint32_t n_ff_exp = 0;
uint32_t n_ff_exp_impl = 0;
uint32_t n_ff_shexp = 0;
uint32_t n_ff_chexp = 0;
uint32_t n_expert_shared = 0;
@@ -319,6 +324,12 @@ struct llama_hparams {
uint32_t n_ff(uint32_t il = 0) const;
// per-layer expert feed-forward size; falls back to n_ff_exp_impl when array entry is 0
uint32_t n_ff_exp(uint32_t il = 0) const;
// per-layer top-k expert routing count; falls back to n_expert_used_impl when array entry is 0
uint32_t n_expert_used(uint32_t il = 0) const;
uint32_t n_gqa(uint32_t il = 0) const;
uint32_t n_rot(uint32_t il = 0) const;
src/llama-model-loader.cpp
+2 -2
View File
@@ -919,7 +919,7 @@ static bool weight_buft_supported(const llama_hparams & hparams, ggml_tensor * w
} break;
case GGML_OP_MUL_MAT_ID:
{
const int n_expert_used = hparams.n_expert_used;
const int n_expert_used = hparams.n_expert_used_impl;
GGML_ASSERT(n_expert_used > 0);
ggml_tensor * b = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, w->ne[0], n_expert_used, 512);
ggml_tensor * ids = ggml_new_tensor_2d(ctx, GGML_TYPE_I32, n_expert_used, 512);
@@ -932,7 +932,7 @@ static bool weight_buft_supported(const llama_hparams & hparams, ggml_tensor * w
} break;
case GGML_OP_ADD_ID:
{
const int n_expert_used = hparams.n_expert_used;
const int n_expert_used = hparams.n_expert_used_impl;
GGML_ASSERT(n_expert_used > 0);
ggml_tensor * a = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, w->ne[0], n_expert_used, 512);
ggml_tensor * c = ggml_new_tensor_2d(ctx, GGML_TYPE_I32, n_expert_used, 512);
src/llama-model-saver.cpp
+2 -2
View File
@@ -211,7 +211,7 @@ void llama_model_saver::add_kv_from_model() {
add_kv(LLM_KV_BLOCK_COUNT, hparams.n_layer_all);
add_kv(LLM_KV_LEADING_DENSE_BLOCK_COUNT, hparams.n_layer_dense_lead);
add_kv(LLM_KV_FEED_FORWARD_LENGTH, hparams.n_ff_arr, true);
add_kv(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp);
add_kv(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp_impl);
add_kv(LLM_KV_EXPERT_SHARED_FEED_FORWARD_LENGTH, hparams.n_ff_shexp);
add_kv(LLM_KV_EXPERT_SHARED_FEED_FORWARD_LENGTH, hparams.n_ff_chexp);
add_kv(LLM_KV_SWIGLU_CLAMP_EXP, hparams.swiglu_clamp_exp);
@@ -219,7 +219,7 @@ void llama_model_saver::add_kv_from_model() {
add_kv(LLM_KV_USE_PARALLEL_RESIDUAL, hparams.use_par_res);
// add_kv(LLM_KV_TENSOR_DATA_LAYOUT, ???);
add_kv(LLM_KV_EXPERT_COUNT, hparams.n_expert);
add_kv(LLM_KV_EXPERT_USED_COUNT, hparams.n_expert_used);
add_kv(LLM_KV_EXPERT_USED_COUNT, hparams.n_expert_used_impl);
add_kv(LLM_KV_EXPERT_SHARED_COUNT, hparams.n_expert_shared);
add_kv(LLM_KV_EXPERT_GROUP_COUNT, hparams.n_expert_groups);
add_kv(LLM_KV_EXPERT_GROUP_USED_COUNT, hparams.n_group_used);
src/llama-model.cpp
+29 -20
View File
@@ -547,7 +547,7 @@ struct ggml_backend_meta_split_state llama_meta_device_get_split_state(const str
// the FFN is the same for Qwen 3 Next and Qwen 3.5:
if (std::regex_match(tensor_name, pattern_ffn_gate_up_weight)) {
const int64_t n_ff_exp = hparams.n_ff_exp;
const int64_t n_ff_exp = hparams.n_ff_exp_impl;
GGML_ASSERT(tensor->ne[axis] == 2*n_ff_exp);
return {{n_ff_exp, 2}};
}
@@ -570,7 +570,7 @@ struct ggml_backend_meta_split_state llama_meta_device_get_split_state(const str
return {{tensor->ne[axis], 1}};
}
if (std::regex_match(tensor_name, pattern_ffn_gate_up_weight)) {
const int64_t n_ff_exp = hparams.n_ff_exp;
const int64_t n_ff_exp = hparams.n_ff_exp_impl;
GGML_ASSERT(tensor->ne[axis] == 2*n_ff_exp);
return {{n_ff_exp, 2}};
}
@@ -1085,14 +1085,22 @@ void llama_model_base::load_hparams(llama_model_loader & ml) {
ml.get_key(LLM_KV_BLOCK_COUNT, hparams.n_layer_all);
GGML_ASSERT(hparams.n_layer_all > 0 && hparams.n_layer_all <= LLAMA_MAX_LAYERS);
ml.get_key(LLM_KV_EXPERT_COUNT, hparams.n_expert, false);
ml.get_key(LLM_KV_EXPERT_USED_COUNT, hparams.n_expert_used, false);
// Load n_expert_used as scalar-OR-array; derive scalar impl as max for
// validation and backward compat with arches that always use a uniform topk.
std::fill(hparams.n_expert_used_arr.begin(), hparams.n_expert_used_arr.end(), 0);
ml.get_key_or_arr(LLM_KV_EXPERT_USED_COUNT, hparams.n_expert_used_arr, hparams.n_layer_all, false);
hparams.n_expert_used_impl = 0;
for (uint32_t il = 0; il < hparams.n_layer_all; ++il) {
hparams.n_expert_used_impl = std::max(hparams.n_expert_used_impl, hparams.n_expert_used_arr[il]);
}
ml.get_key(LLM_KV_EXPERT_GROUP_COUNT, hparams.n_expert_groups, false);
ml.get_key(LLM_KV_EXPERT_GROUP_USED_COUNT, hparams.n_group_used, false);
if (arch == LLM_ARCH_HUNYUAN_VL || arch == LLM_ARCH_HUNYUAN_DENSE) {
if (hparams.n_expert <= 1) {
hparams.n_expert = 0;
hparams.n_expert_used = 0;
hparams.n_expert_used_impl = 0;
std::fill(hparams.n_expert_used_arr.begin(), hparams.n_expert_used_arr.end(), 0);
}
}
@@ -1108,9 +1116,9 @@ void llama_model_base::load_hparams(llama_model_loader & ml) {
}
GGML_ASSERT(hparams.n_expert <= LLAMA_MAX_EXPERTS);
GGML_ASSERT(hparams.n_expert_used <= hparams.n_expert);
GGML_ASSERT(hparams.n_expert_used_impl <= hparams.n_expert);
if (hparams.n_expert > 0) {
GGML_ASSERT(hparams.n_expert_used > 0);
GGML_ASSERT(hparams.n_expert_used_impl > 0);
GGML_ASSERT(hparams.n_expert_groups < hparams.n_expert);
if (hparams.n_expert_groups > 1) {
GGML_ASSERT(hparams.n_expert % hparams.n_expert_groups == 0);
@@ -1118,13 +1126,14 @@ void llama_model_base::load_hparams(llama_model_loader & ml) {
GGML_ASSERT(hparams.n_group_used < hparams.n_expert_groups);
}
} else {
GGML_ASSERT(hparams.n_expert_used == 0);
GGML_ASSERT(hparams.n_expert_used_impl == 0);
GGML_ASSERT(hparams.n_expert_groups == 0);
}
std::fill(hparams.n_head_arr.begin(), hparams.n_head_arr.end(), 0);
std::fill(hparams.n_head_kv_arr.begin(), hparams.n_head_kv_arr.end(), 0);
std::fill(hparams.n_ff_arr.begin(), hparams.n_ff_arr.end(), 0);
std::fill(hparams.n_head_arr.begin(), hparams.n_head_arr.end(), 0);
std::fill(hparams.n_head_kv_arr.begin(), hparams.n_head_kv_arr.end(), 0);
std::fill(hparams.n_ff_arr.begin(), hparams.n_ff_arr.end(), 0);
std::fill(hparams.n_ff_exp_arr.begin(), hparams.n_ff_exp_arr.end(), 0);
std::fill(hparams.rope_sections.begin(), hparams.rope_sections.end(), 0);
std::fill(hparams.is_swa_impl.begin(), hparams.is_swa_impl.end(), 0);
@@ -1341,7 +1350,7 @@ bool llama_model_base::load_tensors(llama_model_loader & ml) {
const auto tn = LLM_TN(arch);
const int64_t n_expert = hparams.n_expert;
const int64_t n_expert_used = hparams.n_expert_used;
const int64_t n_expert_used = hparams.n_expert_used_impl;
if (n_expert > 0 && n_expert_used == 0) {
throw std::runtime_error("model has expert layers but no expert layers are used");
@@ -1792,7 +1801,7 @@ void llama_model::print_info() const {
LLAMA_LOG_INFO("%s: f_attn_value_scale = %.4f\n", __func__, hparams.f_attn_value_scale);
LLAMA_LOG_INFO("%s: n_ff = %s\n", __func__, print_f([&](uint32_t il) { return hparams.n_ff(il); }, hparams.n_layer_all).c_str());
LLAMA_LOG_INFO("%s: n_expert = %u\n", __func__, hparams.n_expert);
LLAMA_LOG_INFO("%s: n_expert_used = %u\n", __func__, hparams.n_expert_used);
LLAMA_LOG_INFO("%s: n_expert_used = %u\n", __func__, hparams.n_expert_used_impl);
LLAMA_LOG_INFO("%s: n_expert_groups = %d\n", __func__, hparams.n_expert_groups);
LLAMA_LOG_INFO("%s: n_group_used = %d\n", __func__, hparams.n_group_used);
LLAMA_LOG_INFO("%s: causal attn = %d\n", __func__, hparams.causal_attn);
@@ -1867,7 +1876,7 @@ void llama_model::print_info() const {
if (arch == LLM_ARCH_DEEPSEEK) {
LLAMA_LOG_INFO("%s: n_layer_dense_lead = %d\n", __func__, hparams.n_layer_dense_lead);
LLAMA_LOG_INFO("%s: n_ff_exp = %d\n", __func__, hparams.n_ff_exp);
LLAMA_LOG_INFO("%s: n_ff_exp = %d\n", __func__, hparams.n_ff_exp_impl);
LLAMA_LOG_INFO("%s: n_expert_shared = %d\n", __func__, hparams.n_expert_shared);
LLAMA_LOG_INFO("%s: expert_weights_scale = %.1f\n", __func__, hparams.expert_weights_scale);
}
@@ -1878,7 +1887,7 @@ void llama_model::print_info() const {
LLAMA_LOG_INFO("%s: n_lora_kv = %d\n", __func__, hparams.n_lora_kv);
LLAMA_LOG_INFO("%s: n_embd_head_k_mla = %d\n", __func__, hparams.n_embd_head_k_mla());
LLAMA_LOG_INFO("%s: n_embd_head_v_mla = %d\n", __func__, hparams.n_embd_head_v_mla());
LLAMA_LOG_INFO("%s: n_ff_exp = %d\n", __func__, hparams.n_ff_exp);
LLAMA_LOG_INFO("%s: n_ff_exp = %d\n", __func__, hparams.n_ff_exp_impl);
LLAMA_LOG_INFO("%s: n_expert_shared = %d\n", __func__, hparams.n_expert_shared);
LLAMA_LOG_INFO("%s: expert_weights_scale = %.1f\n", __func__, hparams.expert_weights_scale);
LLAMA_LOG_INFO("%s: expert_weights_norm = %d\n", __func__, hparams.expert_weights_norm);
@@ -1886,7 +1895,7 @@ void llama_model::print_info() const {
}
if (arch == LLM_ARCH_QWEN2MOE) {
LLAMA_LOG_INFO("%s: n_ff_exp = %d\n", __func__, hparams.n_ff_exp);
LLAMA_LOG_INFO("%s: n_ff_exp = %d\n", __func__, hparams.n_ff_exp_impl);
LLAMA_LOG_INFO("%s: n_ff_shexp = %d\n", __func__, hparams.n_ff_shexp);
}
@@ -1896,7 +1905,7 @@ void llama_model::print_info() const {
arch == LLM_ARCH_OPENAI_MOE ||
arch == LLM_ARCH_QWEN3VLMOE ||
arch == LLM_ARCH_RND1) {
LLAMA_LOG_INFO("%s: n_ff_exp = %d\n", __func__, hparams.n_ff_exp);
LLAMA_LOG_INFO("%s: n_ff_exp = %d\n", __func__, hparams.n_ff_exp_impl);
}
if (arch == LLM_ARCH_MINICPM ||
@@ -1912,7 +1921,7 @@ void llama_model::print_info() const {
if (arch == LLM_ARCH_BAILINGMOE) {
LLAMA_LOG_INFO("%s: n_layer_dense_lead = %d\n", __func__, hparams.n_layer_dense_lead);
LLAMA_LOG_INFO("%s: n_ff_exp = %d\n", __func__, hparams.n_ff_exp);
LLAMA_LOG_INFO("%s: n_ff_exp = %d\n", __func__, hparams.n_ff_exp_impl);
LLAMA_LOG_INFO("%s: n_expert_shared = %d\n", __func__, hparams.n_expert_shared);
LLAMA_LOG_INFO("%s: expert_weights_scale = %.1f\n", __func__, hparams.expert_weights_scale);
LLAMA_LOG_INFO("%s: expert_weights_norm = %d\n", __func__, hparams.expert_weights_norm);
@@ -1920,7 +1929,7 @@ void llama_model::print_info() const {
if (arch == LLM_ARCH_BAILINGMOE2) {
LLAMA_LOG_INFO("%s: n_layer_dense_lead = %d\n", __func__, hparams.n_layer_dense_lead);
LLAMA_LOG_INFO("%s: n_ff_exp = %d\n", __func__, hparams.n_ff_exp);
LLAMA_LOG_INFO("%s: n_ff_exp = %d\n", __func__, hparams.n_ff_exp_impl);
LLAMA_LOG_INFO("%s: n_ff_shexp = %d\n", __func__, hparams.n_ff_shexp);
LLAMA_LOG_INFO("%s: n_expert_shared = %d\n", __func__, hparams.n_expert_shared);
LLAMA_LOG_INFO("%s: expert_weights_scale = %.1f\n", __func__, hparams.expert_weights_scale);
@@ -1930,12 +1939,12 @@ void llama_model::print_info() const {
}
if (arch == LLM_ARCH_SMALLTHINKER || arch == LLM_ARCH_LFM2MOE) {
LLAMA_LOG_INFO("%s: n_ff_exp = %d\n", __func__, hparams.n_ff_exp);
LLAMA_LOG_INFO("%s: n_ff_exp = %d\n", __func__, hparams.n_ff_exp_impl);
LLAMA_LOG_INFO("%s: expert_gating_func = %s\n", __func__, llama_expert_gating_func_name((llama_expert_gating_func_type) hparams.expert_gating_func));
}
if (arch == LLM_ARCH_GROVEMOE) {
LLAMA_LOG_INFO("%s: n_ff_exp = %d\n", __func__, hparams.n_ff_exp);
LLAMA_LOG_INFO("%s: n_ff_exp = %d\n", __func__, hparams.n_ff_exp_impl);
LLAMA_LOG_INFO("%s: n_ff_chexp = %d\n", __func__, hparams.n_ff_chexp);
LLAMA_LOG_INFO("%s: n_group_experts = %d\n", __func__, hparams.n_group_experts);
LLAMA_LOG_INFO("%s: expert_group_scale = %.2f\n", __func__, hparams.expert_group_scale);
src/llama-model.h
+1 -1
View File
@@ -755,7 +755,7 @@ const char * llm_type_name(llm_type type);
const int64_t n_token_types = vocab.n_token_types(); GGML_UNUSED(n_token_types); \
const int64_t n_rot = hparams.n_rot(); GGML_UNUSED(n_rot); \
const int64_t n_expert = hparams.n_expert; GGML_UNUSED(n_expert); \
const int64_t n_expert_used = hparams.n_expert_used; GGML_UNUSED(n_expert_used); \
const int64_t n_expert_used = hparams.n_expert_used_impl; GGML_UNUSED(n_expert_used); \
const int64_t n_ctx_train = hparams.n_ctx_train; GGML_UNUSED(n_ctx_train);
// For internal test use