From 7c304a511439cddf64eb907524e378b430b943a9 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Tue, 15 Sep 2026 08:07:27 +0300 Subject: [PATCH] tests : expose MoE weighted reduction in fusion baseline Use 2 experts per token in the generated MoE test models so the Metal MoE weighted reduction fusion (MUL + ADD) is exercised by test-fusion. Assisted-by: pi:llama.cpp/DeepSeek-V4-Flash-Vision-Exp --- tests/fusion/MTL.csv | 42 ++++++++++++++++++++++++++++++++++++++ tests/test-llama-archs.cpp | 2 +- 2 files changed, 43 insertions(+), 1 deletion(-) diff --git a/tests/fusion/MTL.csv b/tests/fusion/MTL.csv index 15297175a4..376b665f18 100644 --- a/tests/fusion/MTL.csv +++ b/tests/fusion/MTL.csv @@ -1,38 +1,49 @@ # test-fusion baseline for device MTL # arch ,moe ,mode ,label , count arcee ,0 ,any ,RMS_NORM+MUL , 5 +arctic ,0 ,any ,MUL+ADD , 4 arctic ,0 ,any ,RMS_NORM+MUL , 7 arctic ,0 ,any ,SOFT_MAX+ARGSORT+GET_ROWS+SUM_ROWS+CLAMP+DIV, 2 baichuan ,0 ,any ,RMS_NORM+MUL , 5 bailingmoe ,1 ,any ,ADD+ADD , 2 +bailingmoe ,1 ,any ,MUL+ADD , 4 bailingmoe ,1 ,any ,RMS_NORM+MUL , 5 bailingmoe ,1 ,any ,SOFT_MAX+ARGSORT+GET_ROWS , 2 bailingmoe2 ,1 ,any ,ADD+ADD , 1 +bailingmoe2 ,1 ,any ,MUL+ADD , 2 bailingmoe2 ,1 ,any ,RMS_NORM+MUL , 9 bailingmoe3 ,1 ,any ,ADD+ADD , 1 bailingmoe3 ,1 ,any ,GATED_DELTA_NET+CPY , 1 +bailingmoe3 ,1 ,any ,MUL+ADD , 2 bailingmoe3 ,1 ,any ,RMS_NORM+MUL , 8 bloom ,0 ,any ,NORM+MUL+ADD , 6 chatglm ,0 ,any ,RMS_NORM+MUL , 5 codeshell ,0 ,any ,NORM+MUL+ADD , 5 cogvlm ,0 ,any ,RMS_NORM+MUL , 5 command-r ,0 ,any ,NORM+MUL , 3 +dbrx ,0 ,any ,MUL+ADD , 4 dbrx ,0 ,any ,NORM+MUL , 5 dbrx ,0 ,any ,SOFT_MAX+ARGSORT+GET_ROWS+SUM_ROWS+CLAMP+DIV, 2 deci ,0 ,any ,RMS_NORM+MUL , 5 deepseek ,0 ,any ,ADD+ADD , 1 +deepseek ,0 ,any ,MUL+ADD , 2 deepseek ,0 ,any ,RMS_NORM+MUL , 5 deepseek ,0 ,any ,SOFT_MAX+ARGSORT+GET_ROWS , 1 deepseek2 ,0 ,any ,ADD+ADD , 1 +deepseek2 ,0 ,any ,MUL+ADD , 2 deepseek2 ,0 ,any ,RMS_NORM+MUL , 9 deepseek32 ,0 ,any ,ADD+ADD , 1 +deepseek32 ,0 ,any ,MUL+ADD , 2 deepseek32 ,0 ,any ,NORM+MUL+ADD , 2 deepseek32 ,0 ,any ,RMS_NORM+MUL , 9 +deepseek4 ,0 ,any ,MUL+ADD , 8 deepseek4 ,0 ,any ,RMS_NORM+MUL , 20 dots1 ,0 ,any ,ADD+ADD , 1 +dots1 ,0 ,any ,MUL+ADD , 2 dots1 ,0 ,any ,RMS_NORM+MUL , 9 dream ,0 ,any ,RMS_NORM+MUL , 5 ernie4_5-moe ,1 ,any ,ADD+ADD , 1 +ernie4_5-moe ,1 ,any ,MUL+ADD , 2 ernie4_5-moe ,1 ,any ,RMS_NORM+MUL , 5 ernie4_5 ,0 ,any ,RMS_NORM+MUL , 5 exaone ,0 ,any ,RMS_NORM+MUL , 5 @@ -46,34 +57,44 @@ gemma ,0 ,any ,RMS_NORM+MUL , 5 gemma2 ,0 ,any ,RMS_NORM+MUL , 5 gemma2 ,0 ,any ,RMS_NORM+MUL+ADD , 4 glm-dsa ,0 ,any ,ADD+ADD , 1 +glm-dsa ,0 ,any ,MUL+ADD , 2 glm-dsa ,0 ,any ,NORM+MUL+ADD , 2 glm-dsa ,0 ,any ,RMS_NORM+MUL , 9 glm4 ,0 ,any ,RMS_NORM+MUL , 5 glm4 ,0 ,any ,RMS_NORM+MUL+ADD , 4 glm4moe ,1 ,any ,ADD+ADD , 1 +glm4moe ,1 ,any ,MUL+ADD , 2 glm4moe ,1 ,any ,RMS_NORM+MUL , 9 +gpt-oss ,0 ,any ,MUL+ADD , 4 gpt-oss ,0 ,any ,RMS_NORM+MUL , 5 gpt2 ,0 ,any ,NORM+MUL+ADD , 5 gptneox ,0 ,any ,NORM+MUL+ADD , 5 granite ,0 ,any ,RMS_NORM+MUL , 5 +granite ,0 ,any ,MUL+ADD , 4 granite ,0 ,any ,RMS_NORM+MUL , 5 granite ,0 ,any ,SOFT_MAX+ARGSORT+GET_ROWS+SUM_ROWS+CLAMP+DIV, 2 granitehybrid ,0 ,any ,RMS_NORM+MUL , 6 granitemoe ,1 ,any ,RMS_NORM+MUL , 5 +granitemoe ,1 ,any ,MUL+ADD , 4 granitemoe ,1 ,any ,RMS_NORM+MUL , 5 granitemoe ,1 ,any ,SOFT_MAX+ARGSORT+GET_ROWS+SUM_ROWS+CLAMP+DIV, 2 +grok ,0 ,any ,MUL+ADD , 4 grok ,0 ,any ,RMS_NORM+MUL , 5 grok ,0 ,any ,RMS_NORM+MUL+ADD , 4 grok ,0 ,any ,SOFT_MAX+ARGSORT+GET_ROWS+SUM_ROWS+CLAMP+DIV, 2 grovemoe ,1 ,any ,ADD+ADD , 2 +grovemoe ,1 ,any ,MUL+ADD , 8 grovemoe ,1 ,any ,RMS_NORM+MUL , 9 hunyuan-dense ,0 ,any ,RMS_NORM+MUL , 9 hunyuan-moe ,1 ,any ,ADD+ADD , 2 +hunyuan-moe ,1 ,any ,MUL+ADD , 4 hunyuan-moe ,1 ,any ,RMS_NORM+MUL , 9 hunyuan-moe ,1 ,any ,SOFT_MAX+ARGSORT+GET_ROWS+SUM_ROWS+CLAMP+DIV, 2 hunyuan_vl ,0 ,any ,RMS_NORM+MUL , 9 hy_v3 ,0 ,any ,ADD+ADD , 2 +hy_v3 ,0 ,any ,MUL+ADD , 4 hy_v3 ,0 ,any ,RMS_NORM+MUL , 9 +hy_v4 ,0 ,any ,MUL+ADD , 2 hy_v4 ,0 ,any ,NORM+MUL+ADD , 1 hy_v4 ,0 ,any ,RMS_NORM+MUL , 9 internlm2 ,0 ,any ,RMS_NORM+MUL , 5 @@ -81,16 +102,21 @@ jais ,0 ,any ,NORM+MUL+ADD , 5 jais2 ,0 ,any ,NORM+MUL+ADD , 5 jamba ,0 ,any ,RMS_NORM+MUL , 8 kimi-k3 ,0 ,any ,GATED_DELTA_NET+CPY , 1 +kimi-k3 ,0 ,any ,MUL+ADD , 2 kimi-k3 ,0 ,any ,RMS_NORM+MUL , 17 kimi-linear ,0 ,any ,ADD+ADD , 1 kimi-linear ,0 ,any ,GATED_DELTA_NET+CPY , 1 +kimi-linear ,0 ,any ,MUL+ADD , 2 kimi-linear ,0 ,any ,RMS_NORM+MUL , 7 lfm2 ,0 ,any ,RMS_NORM+MUL , 7 +lfm2moe ,1 ,any ,MUL+ADD , 2 lfm2moe ,1 ,any ,RMS_NORM+MUL , 7 llada ,0 ,any ,RMS_NORM+MUL , 5 +llada-moe ,1 ,any ,MUL+ADD , 4 llada-moe ,1 ,any ,RMS_NORM+MUL , 9 llada-moe ,1 ,any ,SOFT_MAX+ARGSORT+GET_ROWS , 2 llama ,0 ,any ,RMS_NORM+MUL , 5 +llama ,0 ,any ,MUL+ADD , 4 llama ,0 ,any ,RMS_NORM+MUL , 5 llama ,0 ,any ,SOFT_MAX+ARGSORT+GET_ROWS+SUM_ROWS+CLAMP+DIV, 2 llama4 ,0 ,any ,ADD+ADD , 2 @@ -99,24 +125,31 @@ maincoder ,0 ,any ,RMS_NORM+MUL , 9 mamba ,0 ,any ,RMS_NORM+MUL , 3 mamba2 ,0 ,any ,RMS_NORM+MUL , 5 minicpm ,0 ,any ,RMS_NORM+MUL , 5 +minicpm ,0 ,any ,MUL+ADD , 4 minicpm ,0 ,any ,RMS_NORM+MUL , 5 minicpm ,0 ,any ,SOFT_MAX+ARGSORT+GET_ROWS+SUM_ROWS+CLAMP+DIV, 2 minicpm3 ,0 ,any ,RMS_NORM+MUL , 9 +minimax-01 ,0 ,any ,MUL+ADD , 4 minimax-01 ,0 ,any ,RMS_NORM+MUL , 6 minimax-01 ,0 ,any ,SOFT_MAX+ARGSORT+GET_ROWS+SUM_ROWS+CLAMP+DIV, 2 +minimax-m2 ,0 ,any ,MUL+ADD , 4 minimax-m2 ,0 ,any ,RMS_NORM+MUL , 9 minimax-m3 ,0 ,any ,ADD+ADD , 1 +minimax-m3 ,0 ,any ,MUL+ADD , 2 minimax-m3 ,0 ,any ,RMS_NORM+MUL , 11 mistral3 ,0 ,any ,RMS_NORM+MUL , 5 +mistral3 ,0 ,any ,MUL+ADD , 4 mistral3 ,0 ,any ,RMS_NORM+MUL , 5 mistral3 ,0 ,any ,SOFT_MAX+ARGSORT+GET_ROWS+SUM_ROWS+CLAMP+DIV, 2 mistral4 ,0 ,any ,ADD+ADD , 1 +mistral4 ,0 ,any ,MUL+ADD , 2 mistral4 ,0 ,any ,RMS_NORM+MUL , 9 mpt ,0 ,any ,NORM+MUL+ADD , 5 nanbeige ,0 ,any ,RMS_NORM+MUL , 5 nemotron ,0 ,any ,NORM+MUL+ADD , 5 nemotron_h ,0 ,any ,RMS_NORM+MUL , 5 nemotron_h_moe ,1 ,any ,RMS_NORM+MUL , 5 +olmoe ,1 ,any ,MUL+ADD , 4 olmoe ,1 ,any ,RMS_NORM+MUL , 9 olmoe ,1 ,any ,SOFT_MAX+ARGSORT+GET_ROWS , 2 openelm ,0 ,any ,RMS_NORM+MUL , 9 @@ -126,6 +159,7 @@ pangu-embedded ,0 ,any ,RMS_NORM+MUL , 5 phi2 ,0 ,any ,ADD+ADD , 2 phi2 ,0 ,any ,NORM+MUL+ADD , 3 phi3 ,0 ,any ,RMS_NORM+MUL , 5 +phimoe ,1 ,any ,MUL+ADD , 4 phimoe ,1 ,any ,RMS_NORM+MUL+ADD , 5 phimoe ,1 ,any ,SOFT_MAX+ARGSORT+GET_ROWS+SUM_ROWS+CLAMP+DIV, 2 plamo ,0 ,any ,ADD+ADD , 2 @@ -136,6 +170,7 @@ pockettts ,0 ,any ,NORM+MUL+ADD , 5 qwen ,0 ,any ,RMS_NORM+MUL , 5 qwen2 ,0 ,any ,RMS_NORM+MUL , 5 qwen2moe ,1 ,any ,ADD+ADD , 2 +qwen2moe ,1 ,any ,MUL+ADD , 4 qwen2moe ,1 ,any ,RMS_NORM+MUL , 5 qwen2moe ,1 ,any ,SOFT_MAX+ARGSORT+GET_ROWS , 2 qwen2vl ,0 ,any ,RMS_NORM+MUL , 5 @@ -144,28 +179,35 @@ qwen35 ,0 ,any ,GATED_DELTA_NET+CPY , 1 qwen35 ,0 ,any ,RMS_NORM+MUL , 8 qwen35moe ,1 ,any ,ADD+ADD , 2 qwen35moe ,1 ,any ,GATED_DELTA_NET+CPY , 1 +qwen35moe ,1 ,any ,MUL+ADD , 4 qwen35moe ,1 ,any ,RMS_NORM+MUL , 8 qwen35moe ,1 ,any ,SOFT_MAX+ARGSORT+GET_ROWS+SUM_ROWS+CLAMP+DIV, 2 +qwen3moe ,1 ,any ,MUL+ADD , 4 qwen3moe ,1 ,any ,RMS_NORM+MUL , 9 qwen3moe ,1 ,any ,SOFT_MAX+ARGSORT+GET_ROWS+SUM_ROWS+CLAMP+DIV, 2 qwen3next ,0 ,any ,ADD+ADD , 2 qwen3next ,0 ,any ,GATED_DELTA_NET+CPY , 1 +qwen3next ,0 ,any ,MUL+ADD , 4 qwen3next ,0 ,any ,RMS_NORM+MUL , 8 qwen3next ,0 ,any ,SOFT_MAX+ARGSORT+GET_ROWS+SUM_ROWS+CLAMP+DIV, 2 qwen3tts ,0 ,any ,RMS_NORM+MUL , 9 qwen3vl ,0 ,any ,RMS_NORM+MUL , 9 +qwen3vlmoe ,1 ,any ,MUL+ADD , 4 qwen3vlmoe ,1 ,any ,RMS_NORM+MUL , 9 qwen3vlmoe ,1 ,any ,SOFT_MAX+ARGSORT+GET_ROWS+SUM_ROWS+CLAMP+DIV, 2 qwen4exp ,0 ,any ,ADD+ADD+ADD , 5 qwen4exp ,0 ,any ,ADD+ADD+ADD+ADD+ADD+ADD+ADD , 9 qwen4exp ,0 ,any ,GATED_DELTA_NET+CPY , 1 +qwen4exp ,0 ,any ,MUL+ADD , 4 qwen4exp ,0 ,any ,RMS_NORM+MUL , 13 qwen4exp ,0 ,any ,SOFT_MAX+ARGSORT+GET_ROWS+SUM_ROWS+CLAMP+DIV, 2 refact ,0 ,any ,RMS_NORM+MUL , 5 refact ,0 ,any ,RMS_NORM+MUL , 5 +rnd1 ,0 ,any ,MUL+ADD , 4 rnd1 ,0 ,any ,RMS_NORM+MUL , 9 rnd1 ,0 ,any ,SOFT_MAX+ARGSORT+GET_ROWS+SUM_ROWS+CLAMP+DIV, 2 seed_oss ,0 ,any ,RMS_NORM+MUL , 5 +smallthinker ,0 ,any ,MUL+ADD , 4 smallthinker ,0 ,any ,RMS_NORM+MUL , 5 smollm3 ,0 ,any ,RMS_NORM+MUL , 5 stablelm ,0 ,any ,NORM+MUL , 4 diff --git a/tests/test-llama-archs.cpp b/tests/test-llama-archs.cpp index 90a6a71623..b4b616a9f1 100644 --- a/tests/test-llama-archs.cpp +++ b/tests/test-llama-archs.cpp @@ -339,7 +339,7 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) { ms.add_kv(LLM_KV_EXPERT_LATENT_LENGTH, n_ff); ms.add_kv(LLM_KV_INTERLEAVE_MOE_LAYER_STEP, uint32_t(2)); ms.add_kv(LLM_KV_EXPERT_COUNT, uint32_t(2)); - ms.add_kv(LLM_KV_EXPERT_USED_COUNT, uint32_t(1)); + ms.add_kv(LLM_KV_EXPERT_USED_COUNT, uint32_t(2)); ms.add_kv(LLM_KV_EXPERT_SHARED_COUNT, uint32_t(1)); ms.add_kv(LLM_KV_EXPERT_GATING_FUNC, arch == LLM_ARCH_DEEPSEEK4 ? uint32_t(4) : uint32_t(2)); // sqrtsoftplus : sigmoid ms.add_kv(LLM_KV_EXPERT_GROUP_SCALE, 1.0f);