mirror of
https://github.com/ggml-org/llama.cpp.git
synced 2026-09-14 18:02:52 +02:00
AlexandGitHub
3d10bcd197
* gguf-py: add Maple tensor constants
Add MODEL_ARCH.MAPLE, its "maple" name, and the tensor list for the
Maple 20B-A1B ternary MoE architecture: token embeddings, output,
attention with Q/K RMS norms, and per-expert FFN tensors.
* convert: add Maple HF->GGUF converter
Register MapleForCausalLM in the HF architecture map and add the
converter for the Maple 20B-A1B ternary MoE model: 24 layers, 256
experts with 8 active, sliding-window attention (SWA-512) interleaved
with global attention at a 3:1 ratio, partial rotary factor 0.5, and
per-expert weight stacking into merged 3D tensors.
* llama: add Maple architecture (20B-A1B ternary MoE)
Add the Maple 20B-A1B ternary MoE architecture: 24 layers, 256
experts with 8 active, sliding-window attention (SWA-512) interleaved
with global attention at a 3:1 ratio, and ternary TQ1_0/TQ2_0
quantization support.
- register LLM_ARCH_MAPLE between MAMBA2 and JAMBA
- implement llama_model_maple: Q/K RMS norms after projection (GEMMA4
style), rope applied only on SWA layers (nope_on_global_attention),
ISWA KV cache, and MoE FFN with swiglu gate clamp at +7 (DEEPSEEK4
style)
- mark MAPLE as unsupported by the model saver (roundtrip skipped)
* tests: mark Maple as MoE-mandatory
Maple is always-MoE: the model throws when n_expert == 0, so the
test harness must only run the MoE config for LLM_ARCH_MAPLE.
* maple: apply review feedback (n_ff_exp_arr, get_arr, rope params)
- load_arch_hparams: use n_ff_exp_arr + n_ff_exp() accessor (upstream
changed these from a scalar member during the rebase)
- sliding_window_pattern: get_arr, the pattern is mandatory for this arch
- partial_rotary_factor: read only from rope_parameters (base.py mirrors
the top-level key automatically)
- document why TOKEN_EMBD/OUTPUT are forced to F16 (they are the two
dense tensors in Maple, and the reference GGUFs ship them as F16)
- add @ModelBase.example("deepgrove/maple-preview")
* tests: add Maple to the SWA pattern array list
get_arr for maple.attention.sliding_window_pattern requires an array, but
the harness only emitted a per-layer array for the arches in its list, so
test-llama-archs -a maple failed to load the model.
Assisted-by: DeepSeek Harness
* maple: move swiglu_clamp_exp to the converter
The loader prefilled 7.0 and read the key optionally. The converter now
writes it and the loader reads it as required, because llama-graph.cpp
skips the clamp when the limit is 0 and an optional read would silently
run unclamped. The test harness provides the key for the same reason.
Also drops tensor_force_quant: base.py already forces FFN_GATE_INP to F32
and TOKEN_EMBD/OUTPUT to F16 for ternary file types.
Assisted-by: DeepSeek Harness
* convert: fix the LazyBase func signature in the Maple converter
ty flagged the stack() closure: it takes no argument, while LazyBase is
annotated with func: Callable[[Any], Any]. Pass the tensor list through
args instead of closing over it, the same way kimi_k3 does, so the
callable shape matches.
Assisted-by: DeepSeek Harness
88 lines
3.5 KiB
Python
88 lines
3.5 KiB
Python
from __future__ import annotations
|
|
|
|
from typing import Iterable, TYPE_CHECKING, cast
|
|
|
|
import torch
|
|
|
|
if TYPE_CHECKING:
|
|
from torch import Tensor
|
|
|
|
from .base import LazyTorchTensor, ModelBase, TextModel, gguf
|
|
|
|
|
|
@ModelBase.register("MapleForCausalLM")
|
|
@ModelBase.example("deepgrove/maple-preview")
|
|
class MapleModel(TextModel):
|
|
model_arch = gguf.MODEL_ARCH.MAPLE
|
|
|
|
def set_gguf_parameters(self):
|
|
super().set_gguf_parameters()
|
|
hparams = self.hparams
|
|
|
|
assert hparams["hidden_act"] == "silu"
|
|
assert hparams.get("num_shared_experts", 0) == 0
|
|
assert hparams.get("norm_topk_prob", True)
|
|
assert hparams.get("nope_on_global_attention", False)
|
|
|
|
head_dim = hparams.get("head_dim", hparams["hidden_size"] // hparams["num_attention_heads"])
|
|
partial_rotary_factor = self.rope_parameters.get("partial_rotary_factor", 1.0)
|
|
|
|
self.gguf_writer.add_vocab_size(hparams["vocab_size"])
|
|
self.gguf_writer.add_rope_dimension_count(int(head_dim * partial_rotary_factor))
|
|
self.gguf_writer.add_sliding_window(hparams["sliding_window"])
|
|
self.gguf_writer.add_sliding_window_pattern([layer_type == "sliding_attention" for layer_type in hparams["layer_types"]])
|
|
self.gguf_writer.add_expert_feed_forward_length(hparams["moe_intermediate_size"])
|
|
# the reference clamps the MoE SwiGLU gate/up at 7.0 (modeling_maple.py)
|
|
self.gguf_writer.add_swiglu_clamp_exp([7.0] * self.block_count)
|
|
|
|
_experts: list[dict[str, Tensor]] | None = None
|
|
|
|
@staticmethod
|
|
def _stack_experts(tensors: list[Tensor]) -> Tensor:
|
|
shape = (len(tensors), *tensors[0].shape)
|
|
dtype = tensors[0].dtype
|
|
meta = LazyTorchTensor.meta_with_dtype_and_shape(dtype, shape)
|
|
|
|
# tensors goes through args, not the closure, so that `func` matches
|
|
# LazyBase's single-argument shape
|
|
def stack(ts: list[Tensor]) -> Tensor:
|
|
result = torch.empty(shape, dtype=dtype)
|
|
for expert_id, tensor in enumerate(ts):
|
|
result[expert_id].copy_(LazyTorchTensor.to_eager(tensor))
|
|
ts.clear()
|
|
return result
|
|
|
|
return cast(torch.Tensor, LazyTorchTensor(meta=meta, args=(tensors,), func=stack))
|
|
|
|
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
|
|
if "mlp.experts" in name:
|
|
n_experts = self.hparams["num_experts"]
|
|
assert bid is not None
|
|
|
|
if self._experts is None:
|
|
self._experts = [{} for _ in range(self.block_count)]
|
|
|
|
self._experts[bid][name] = data_torch
|
|
|
|
if len(self._experts[bid]) >= n_experts * 3:
|
|
for weight_name in ("down_proj", "gate_proj", "up_proj"):
|
|
tensors = []
|
|
|
|
for expert_id in range(n_experts):
|
|
expert_name = f"model.layers.{bid}.mlp.experts.{expert_id}.{weight_name}.weight"
|
|
tensors.append(self._experts[bid].pop(expert_name))
|
|
|
|
merged_name = f"model.layers.{bid}.mlp.experts.{weight_name}.weight"
|
|
yield from super().modify_tensors(self._stack_experts(tensors), merged_name, bid)
|
|
return
|
|
|
|
yield from super().modify_tensors(data_torch, name, bid)
|
|
|
|
def prepare_tensors(self):
|
|
super().prepare_tensors()
|
|
|
|
if self._experts is not None:
|
|
experts = [name for layer in self._experts for name in layer]
|
|
if experts:
|
|
raise ValueError(f"Unprocessed experts: {experts}")
|