mirror of
https://github.com/ggml-org/llama.cpp.git
synced 2026-08-18 02:15:03 +02:00
Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
04a134c70b |
@@ -22,6 +22,8 @@ jobs:
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
ssh-key: ${{ secrets.DEPLOY_KEY_RELEASE }}
|
||||
|
||||
- name: Run release checks
|
||||
id: checks
|
||||
|
||||
@@ -1598,6 +1598,7 @@ jobs:
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
fetch-depth: 0
|
||||
ssh-key: ${{ secrets.DEPLOY_KEY_RELEASE }}
|
||||
|
||||
- name: Determine tag name
|
||||
id: tag
|
||||
|
||||
@@ -193,14 +193,6 @@ static std::vector<std::function<void(const common_chat_template & tmpl, autopar
|
||||
LOG_DBG(ANSI_ORANGE "[Patch: Laguna]\n" ANSI_RESET);
|
||||
}
|
||||
},
|
||||
// Bailing V3
|
||||
[](const common_chat_template & tmpl, autoparser & analysis) -> void {
|
||||
if (tmpl.src.find("Bailing V3 chat template") != std::string::npos) {
|
||||
analysis.tools.arguments.value_suffix = trim_whitespace(analysis.tools.arguments.value_suffix);
|
||||
analysis.tools.arguments.tolerate_intertag_whitespace = true;
|
||||
LOG_DBG(ANSI_ORANGE "[Patch: Bailing V3]\n" ANSI_RESET);
|
||||
}
|
||||
},
|
||||
|
||||
});
|
||||
|
||||
|
||||
@@ -27,7 +27,6 @@ TEXT_MODEL_MAP: dict[str, str] = {
|
||||
"BaichuanForCausalLM": "baichuan",
|
||||
"BailingMoeForCausalLM": "bailingmoe",
|
||||
"BailingMoeV2ForCausalLM": "bailingmoe",
|
||||
"BailingMoeV3ForCausalLM": "bailingmoe3",
|
||||
"BambaForCausalLM": "granite",
|
||||
"BertForMaskedLM": "bert",
|
||||
"BertForSequenceClassification": "bert",
|
||||
|
||||
@@ -13,7 +13,6 @@ from .llama import LlamaModel
|
||||
|
||||
|
||||
@ModelBase.register("AfmoeForCausalLM")
|
||||
@ModelBase.example("arcee-ai/Trinity-Large-Thinking")
|
||||
class AfmoeModel(LlamaModel):
|
||||
model_arch = gguf.MODEL_ARCH.AFMOE
|
||||
|
||||
|
||||
@@ -16,7 +16,6 @@ from .llama import LlamaModel
|
||||
|
||||
|
||||
@ModelBase.register("ArcticForCausalLM")
|
||||
@ModelBase.example("Snowflake/snowflake-arctic-instruct")
|
||||
class ArcticModel(TextModel):
|
||||
model_arch = gguf.MODEL_ARCH.ARCTIC
|
||||
|
||||
|
||||
@@ -9,7 +9,6 @@ from .base import ModelBase, TextModel, gguf, logger
|
||||
|
||||
|
||||
@ModelBase.register("BaichuanForCausalLM", "BaiChuanForCausalLM")
|
||||
@ModelBase.example("baichuan-inc/Baichuan2-7B-Chat", "baichuan-inc/Baichuan-7B")
|
||||
class BaichuanModel(TextModel):
|
||||
model_arch = gguf.MODEL_ARCH.BAICHUAN
|
||||
|
||||
|
||||
@@ -11,7 +11,6 @@ from .base import ModelBase, TextModel, gguf
|
||||
|
||||
|
||||
@ModelBase.register("BailingMoeForCausalLM")
|
||||
@ModelBase.example("inclusionAI/Ling-lite")
|
||||
class BailingMoeModel(TextModel):
|
||||
model_arch = gguf.MODEL_ARCH.BAILINGMOE
|
||||
|
||||
@@ -109,7 +108,6 @@ class BailingMoeModel(TextModel):
|
||||
|
||||
|
||||
@ModelBase.register("BailingMoeV2ForCausalLM")
|
||||
@ModelBase.example("inclusionAI/Ling-mini-2.0")
|
||||
class BailingMoeV2Model(TextModel):
|
||||
model_arch = gguf.MODEL_ARCH.BAILINGMOE2
|
||||
|
||||
@@ -191,7 +189,6 @@ class BailingMoeV2Model(TextModel):
|
||||
|
||||
|
||||
@ModelBase.register("SarvamMoEForCausalLM", "modeling_sarvam_moe.SarvamMoEForCausalLM")
|
||||
@ModelBase.example("sarvamai/sarvam-30b")
|
||||
class SarvamMoEModel(BailingMoeV2Model):
|
||||
model_arch = gguf.MODEL_ARCH.BAILINGMOE2
|
||||
# Sarvam-MoE shares the BailingMoeV2 architecture; only differences:
|
||||
|
||||
@@ -1,193 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
from typing import Callable, Iterable, TYPE_CHECKING
|
||||
|
||||
import torch
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from torch import Tensor
|
||||
|
||||
from .base import ModelBase, TextModel, gguf
|
||||
|
||||
|
||||
@ModelBase.register("BailingMoeV3ForCausalLM")
|
||||
@ModelBase.example("inclusionAI/Ling-3.0-tiny", "inclusionAI/Ling-3.0-flash")
|
||||
class BailingMoeV3Model(TextModel):
|
||||
model_arch = gguf.MODEL_ARCH.BAILINGMOE3
|
||||
supports_mtp_export = True
|
||||
|
||||
_experts: list[dict[str, Tensor]] | None = None
|
||||
_main_layers: int | None = None
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
nextn_layers = self.hparams.get("num_nextn_predict_layers", 0) or 0
|
||||
if self.no_mtp:
|
||||
nextn_layers = 0
|
||||
self.block_count = self.hparams["num_hidden_layers"] + nextn_layers
|
||||
self.tensor_map = gguf.get_tensor_name_map(self.model_arch, self.block_count)
|
||||
|
||||
def index_tensors(self, remote_hf_model_id: str | None = None):
|
||||
type(self)._main_layers = self.hparams["num_hidden_layers"]
|
||||
return super().index_tensors(remote_hf_model_id=remote_hf_model_id)
|
||||
|
||||
def set_vocab(self):
|
||||
self._set_vocab_gpt2()
|
||||
|
||||
def is_full_attention(self, bid: int) -> bool:
|
||||
n_layer = self.hparams["num_hidden_layers"]
|
||||
layer_group_size = self.hparams["layer_group_size"]
|
||||
return bid >= n_layer or (bid + 1) % layer_group_size == 0 or bid >= n_layer // layer_group_size * layer_group_size
|
||||
|
||||
def set_gguf_parameters(self):
|
||||
if not self.hparams.get("no_kda_lora", False):
|
||||
raise ValueError("BailingMoeV3 KDA LoRA projections are not supported")
|
||||
if not self.hparams.get("kda_safe_gate", False):
|
||||
raise ValueError("BailingMoeV3 non-safe KDA gates are not supported")
|
||||
if self.hparams.get("gated_attention_proj_granularity_type") != "head_wise":
|
||||
raise ValueError("BailingMoeV3 requires head-wise attention gates")
|
||||
|
||||
self.hparams["num_key_value_heads"] = 1
|
||||
super().set_gguf_parameters()
|
||||
|
||||
n_head_kv = [1 if self.is_full_attention(il) else 0 for il in range(self.block_count)]
|
||||
self.gguf_writer.add_head_count_kv(n_head_kv)
|
||||
|
||||
self.gguf_writer.add_vocab_size(self.hparams["vocab_size"])
|
||||
self.gguf_writer.add_ssm_conv_kernel(self.hparams["short_conv_kernel_size"])
|
||||
self.gguf_writer.add_kda_head_dim(self.hparams["head_dim"])
|
||||
self.gguf_writer.add_kda_safe_gate(self.hparams["kda_safe_gate"])
|
||||
self.gguf_writer.add_kda_gate_lower_bound(self.hparams["kda_lower_bound"])
|
||||
|
||||
kv_lora_rank = self.hparams["kv_lora_rank"]
|
||||
qk_nope_head_dim = self.hparams["qk_nope_head_dim"]
|
||||
qk_rope_head_dim = self.hparams["qk_rope_head_dim"]
|
||||
if (q_lora_rank := self.hparams.get("q_lora_rank")) is not None:
|
||||
self.gguf_writer.add_q_lora_rank(q_lora_rank)
|
||||
self.gguf_writer.add_kv_lora_rank(kv_lora_rank)
|
||||
self.gguf_writer.add_rope_dimension_count(qk_rope_head_dim)
|
||||
self.gguf_writer.add_key_length(kv_lora_rank + qk_rope_head_dim)
|
||||
self.gguf_writer.add_key_length_mla(qk_nope_head_dim + qk_rope_head_dim)
|
||||
self.gguf_writer.add_value_length_mla(self.hparams["v_head_dim"])
|
||||
|
||||
self.gguf_writer.add_expert_feed_forward_length(self.hparams["moe_intermediate_size"])
|
||||
self.gguf_writer.add_expert_shared_feed_forward_length(self.hparams["moe_shared_expert_intermediate_size"])
|
||||
self.gguf_writer.add_expert_shared_count(self.hparams["num_shared_experts"])
|
||||
self.gguf_writer.add_leading_dense_block_count(self.hparams["first_k_dense_replace"])
|
||||
self.gguf_writer.add_expert_weights_scale(self.hparams["routed_scaling_factor"])
|
||||
self.gguf_writer.add_expert_weights_norm(self.hparams["norm_topk_prob"])
|
||||
|
||||
def clamp_limits(key: str) -> list[float] | None:
|
||||
values = self.hparams.get(key)
|
||||
if values is None:
|
||||
return None
|
||||
values = [0.0 if value is None else float(value) for value in values[:self.block_count]]
|
||||
return values + [0.0] * (self.block_count - len(values))
|
||||
|
||||
if (values := clamp_limits("expert_swiglu_limit_list")) is not None:
|
||||
self.gguf_writer.add_swiglu_clamp_exp(values)
|
||||
if (values := clamp_limits("share_expert_swiglu_limit_list")) is not None:
|
||||
self.gguf_writer.add_swiglu_clamp_shexp(values)
|
||||
|
||||
if not self.no_mtp and (nextn_layers := self.hparams.get("num_nextn_predict_layers", 0)):
|
||||
self.gguf_writer.add_nextn_predict_layers(nextn_layers)
|
||||
|
||||
def prepare_metadata(self, vocab_only: bool):
|
||||
from_dir = self.fname_out.is_dir()
|
||||
super().prepare_metadata(vocab_only=vocab_only)
|
||||
|
||||
if not self.mtp_only or not from_dir:
|
||||
return
|
||||
|
||||
output_type: str = self.ftype.name.partition("_")[2]
|
||||
fname_default: str = gguf.naming_convention(
|
||||
self.metadata.name, self.metadata.basename, self.metadata.finetune,
|
||||
self.metadata.version, size_label=None, output_type=output_type, model_type=None)
|
||||
self.fname_out = self.fname_out.parent / f"mtp-{fname_default}.gguf"
|
||||
|
||||
@classmethod
|
||||
def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
|
||||
name, gen = item
|
||||
if name.endswith(".expert_bias"):
|
||||
name += ".bias"
|
||||
|
||||
if cls._main_layers is None:
|
||||
return super().filter_tensors((name, gen))
|
||||
|
||||
m = re.match(r"model\.layers\.(\d+)\.", name)
|
||||
is_mtp = m is not None and int(m.group(1)) >= cls._main_layers
|
||||
|
||||
if is_mtp and cls.no_mtp:
|
||||
return None
|
||||
if cls.mtp_only and not is_mtp and name not in (
|
||||
"model.word_embeddings.weight", "model.norm.weight", "lm_head.weight",
|
||||
):
|
||||
return None
|
||||
|
||||
return super().filter_tensors((name, gen))
|
||||
|
||||
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
|
||||
if name.endswith((".q_conv1d.weight", ".k_conv1d.weight", ".v_conv1d.weight")) and data_torch.ndim in (2, 3):
|
||||
d_inner = data_torch.shape[0]
|
||||
d_conv = data_torch.shape[-1]
|
||||
data_torch = data_torch.reshape(1, d_inner, 1, d_conv)
|
||||
|
||||
if name.endswith(".A_log"):
|
||||
data_torch = torch.exp(data_torch).reshape(-1, 1)
|
||||
|
||||
if name.endswith(".dt_bias"):
|
||||
name = name.rpartition(".dt_bias")[0] + ".dt_proj.bias"
|
||||
|
||||
if name.endswith(".attention.f_proj.weight"):
|
||||
assert bid is not None
|
||||
if self.is_full_attention(bid):
|
||||
raise ValueError(f"unexpected f_proj on full-attention layer {bid}")
|
||||
name = self.format_tensor_name(gguf.MODEL_TENSOR.SSM_F_A, bid)
|
||||
|
||||
if name.endswith(".attention.g_proj.weight"):
|
||||
assert bid is not None
|
||||
tensor = gguf.MODEL_TENSOR.ATTN_GATE if self.is_full_attention(bid) else gguf.MODEL_TENSOR.SSM_G_A
|
||||
name = self.format_tensor_name(tensor, bid)
|
||||
|
||||
if ".mlp.experts." in name:
|
||||
n_experts = self.hparams["num_experts"]
|
||||
assert bid is not None
|
||||
|
||||
if self._experts is None:
|
||||
self._experts = [{} for _ in range(self.block_count)]
|
||||
|
||||
self._experts[bid][name] = data_torch
|
||||
if len(self._experts[bid]) >= n_experts * 3:
|
||||
for weight_name in ("down_proj", "gate_proj", "up_proj"):
|
||||
tensors = []
|
||||
for expert_id in range(n_experts):
|
||||
expert_name = f"model.layers.{bid}.mlp.experts.{expert_id}.{weight_name}.weight"
|
||||
tensors.append(self._experts[bid].pop(expert_name))
|
||||
merged_name = f"model.layers.{bid}.mlp.experts.{weight_name}.weight"
|
||||
yield from super().modify_tensors(torch.stack(tensors, dim=0), merged_name, bid)
|
||||
return
|
||||
|
||||
if name.endswith(".attention.kv_b_proj.weight"):
|
||||
assert bid is not None
|
||||
n_head = self.hparams["num_attention_heads"]
|
||||
v_head_dim = self.hparams["v_head_dim"]
|
||||
qk_nope_head_dim = self.hparams["qk_nope_head_dim"]
|
||||
assert data_torch.shape[0] == n_head * (v_head_dim + qk_nope_head_dim)
|
||||
kv_b = data_torch.view(n_head, v_head_dim + qk_nope_head_dim, data_torch.shape[-1])
|
||||
k_b, v_b = torch.split(kv_b, [qk_nope_head_dim, v_head_dim], dim=1)
|
||||
name_k = self.format_tensor_name(gguf.MODEL_TENSOR.ATTN_K_B, bid)
|
||||
name_v = self.format_tensor_name(gguf.MODEL_TENSOR.ATTN_V_B, bid)
|
||||
yield from super().modify_tensors(k_b.transpose(1, 2), name_k, bid)
|
||||
yield from super().modify_tensors(v_b, name_v, bid)
|
||||
return
|
||||
|
||||
yield from super().modify_tensors(data_torch, name, bid)
|
||||
|
||||
def prepare_tensors(self):
|
||||
super().prepare_tensors()
|
||||
if self._experts is not None:
|
||||
experts = [name for layer in self._experts for name in layer]
|
||||
if experts:
|
||||
raise ValueError(f"Unprocessed experts: {experts}")
|
||||
@@ -1149,14 +1149,6 @@ class ModelBase:
|
||||
return modelcls
|
||||
return func
|
||||
|
||||
@classmethod
|
||||
def example(cls, *hf_repos: str) -> Callable[[AnyModel], AnyModel]:
|
||||
del hf_repos # unused
|
||||
|
||||
def func(modelcls: AnyModel) -> AnyModel:
|
||||
return modelcls
|
||||
return func
|
||||
|
||||
@classmethod
|
||||
def print_registered_models(cls):
|
||||
for model_type, model_classes in cls._model_classes.items():
|
||||
|
||||
@@ -15,7 +15,6 @@ from .base import ModelBase, SentencePieceTokenTypes, TextModel, gguf, logger
|
||||
|
||||
|
||||
@ModelBase.register("BertModel", "BertForMaskedLM", "CamembertModel", "BertForSequenceClassification")
|
||||
@ModelBase.example("BAAI/bge-small-en-v1.5", "dangvantuan/sentence-camembert-base")
|
||||
class BertModel(TextModel):
|
||||
model_arch = gguf.MODEL_ARCH.BERT
|
||||
|
||||
@@ -241,7 +240,6 @@ class BertModel(TextModel):
|
||||
|
||||
|
||||
@ModelBase.register("DistilBertModel", "DistilBertForMaskedLM", "DistilBertForSequenceClassification")
|
||||
@ModelBase.example("distilbert/distilbert-base-uncased")
|
||||
class DistilBertModel(BertModel):
|
||||
model_arch = gguf.MODEL_ARCH.BERT
|
||||
|
||||
@@ -265,7 +263,6 @@ class DistilBertModel(BertModel):
|
||||
|
||||
|
||||
@ModelBase.register("RobertaModel", "RobertaForSequenceClassification")
|
||||
@ModelBase.example("sentence-transformers/stsb-roberta-base")
|
||||
class RobertaModel(BertModel):
|
||||
model_arch = gguf.MODEL_ARCH.BERT
|
||||
|
||||
@@ -315,7 +312,6 @@ class RobertaModel(BertModel):
|
||||
|
||||
|
||||
@ModelBase.register("NomicBertModel")
|
||||
@ModelBase.example("nomic-ai/nomic-embed-text-v1.5")
|
||||
class NomicBertModel(BertModel):
|
||||
model_arch = gguf.MODEL_ARCH.BERT
|
||||
|
||||
@@ -404,7 +400,6 @@ class NomicBertModel(BertModel):
|
||||
|
||||
|
||||
@ModelBase.register("NeoBERT", "NeoBERTLMHead", "NeoBERTForSequenceClassification")
|
||||
@ModelBase.example("chandar-lab/NeoBERT")
|
||||
class NeoBert(BertModel):
|
||||
model_arch = gguf.MODEL_ARCH.NEO_BERT
|
||||
|
||||
@@ -436,7 +431,6 @@ class NeoBert(BertModel):
|
||||
|
||||
|
||||
@ModelBase.register("EuroBertModel", "JinaEmbeddingsV5Model")
|
||||
@ModelBase.example("hf-tiny-v2/tiny-random-EuroBertModel", "jinaai/jina-embeddings-v5-text-nano")
|
||||
class EuroBertModel(TextModel):
|
||||
model_arch = gguf.MODEL_ARCH.EUROBERT
|
||||
|
||||
@@ -465,7 +459,6 @@ class EuroBertModel(TextModel):
|
||||
|
||||
|
||||
@ModelBase.register("XLMRobertaModel", "XLMRobertaForSequenceClassification")
|
||||
@ModelBase.example("BAAI/bge-m3")
|
||||
class XLMRobertaModel(BertModel):
|
||||
model_arch = gguf.MODEL_ARCH.BERT
|
||||
_lora_files = {}
|
||||
@@ -568,7 +561,6 @@ class XLMRobertaModel(BertModel):
|
||||
|
||||
|
||||
@ModelBase.register("JinaBertModel", "JinaBertForMaskedLM")
|
||||
@ModelBase.example("jinaai/jina-embeddings-v2-base-en")
|
||||
class JinaBertV2Model(BertModel):
|
||||
model_arch = gguf.MODEL_ARCH.JINA_BERT_V2
|
||||
|
||||
@@ -596,7 +588,6 @@ class JinaBertV2Model(BertModel):
|
||||
|
||||
|
||||
@ModelBase.register("ModernBertModel", "ModernBertForMaskedLM", "ModernBertForSequenceClassification")
|
||||
@ModelBase.example("answerdotai/ModernBERT-base")
|
||||
class ModernBertModel(BertModel):
|
||||
model_arch = gguf.MODEL_ARCH.MODERN_BERT
|
||||
|
||||
|
||||
@@ -9,7 +9,6 @@ from .base import ModelBase, TextModel, gguf
|
||||
|
||||
|
||||
@ModelBase.register("BitnetForCausalLM", "BitNetForCausalLM")
|
||||
@ModelBase.example("microsoft/bitnet-b1.58-2B-4T")
|
||||
class BitnetModel(TextModel):
|
||||
model_arch = gguf.MODEL_ARCH.BITNET
|
||||
|
||||
|
||||
@@ -13,7 +13,6 @@ from .base import ModelBase, TextModel, gguf, logger
|
||||
|
||||
|
||||
@ModelBase.register("BloomForCausalLM", "BloomModel")
|
||||
@ModelBase.example("bigscience/bloom-560m")
|
||||
class BloomModel(TextModel):
|
||||
model_arch = gguf.MODEL_ARCH.BLOOM
|
||||
|
||||
|
||||
@@ -12,8 +12,6 @@ from .llama import LlamaModel
|
||||
|
||||
@ModelBase.register("ChameleonForConditionalGeneration")
|
||||
@ModelBase.register("ChameleonForCausalLM") # obsolete
|
||||
# [TAG_HF_EXAMPLE_GATED] facebook/chameleon-7b is gated
|
||||
# [TAG_HF_EXAMPLE_MISSING]
|
||||
class ChameleonModel(TextModel):
|
||||
model_arch = gguf.MODEL_ARCH.CHAMELEON
|
||||
|
||||
|
||||
@@ -9,7 +9,6 @@ from .base import ModelBase, SentencePieceTokenTypes, TextModel, gguf
|
||||
|
||||
|
||||
@ModelBase.register("GlmForCausalLM", "ChatGLMModel", "ChatGLMForConditionalGeneration")
|
||||
@ModelBase.example("THUDM/chatglm3-6b", "zai-org/glm-4-9b-chat-hf")
|
||||
class ChatGLMModel(TextModel):
|
||||
model_arch = gguf.MODEL_ARCH.CHATGLM
|
||||
|
||||
|
||||
@@ -4,7 +4,6 @@ from .base import ModelBase, TextModel, gguf
|
||||
|
||||
|
||||
@ModelBase.register("CodeShellForCausalLM")
|
||||
@ModelBase.example("WisdomShell/CodeShell-7B")
|
||||
class CodeShellModel(TextModel):
|
||||
model_arch = gguf.MODEL_ARCH.CODESHELL
|
||||
|
||||
|
||||
@@ -11,7 +11,6 @@ from .llama import LlamaModel
|
||||
|
||||
|
||||
@ModelBase.register("CogVLMForCausalLM")
|
||||
@ModelBase.example("THUDM/cogvlm2-llama3-chat-19B", "THUDM/cogvlm-chat-hf")
|
||||
class CogVLMVisionModel(MmprojModel):
|
||||
|
||||
def set_gguf_parameters(self):
|
||||
@@ -30,6 +29,5 @@ class CogVLMVisionModel(MmprojModel):
|
||||
|
||||
|
||||
@ModelBase.register("CogVLMForCausalLM")
|
||||
@ModelBase.example("THUDM/cogvlm2-llama3-chat-19B", "THUDM/cogvlm-chat-hf")
|
||||
class CogVLMModel(LlamaModel):
|
||||
model_arch = gguf.MODEL_ARCH.COGVLM
|
||||
|
||||
@@ -12,8 +12,6 @@ from .base import ModelBase, TextModel, gguf, logger
|
||||
|
||||
|
||||
@ModelBase.register("CohereForCausalLM")
|
||||
# [TAG_HF_EXAMPLE_GATED] CohereLabs/c4ai-command-r-v01 is gated
|
||||
# [TAG_HF_EXAMPLE_MISSING]
|
||||
class CommandR2Model(TextModel):
|
||||
model_arch = gguf.MODEL_ARCH.COMMAND_R
|
||||
|
||||
@@ -32,8 +30,6 @@ class CommandR2Model(TextModel):
|
||||
|
||||
|
||||
@ModelBase.register("Cohere2ForCausalLM")
|
||||
# [TAG_HF_EXAMPLE_GATED] CohereLabs/c4ai-command-r7b-12-2024 is gated
|
||||
@ModelBase.example("hf-tiny-v2/tiny-random-Cohere2ForCausalLM")
|
||||
class Cohere2Model(TextModel):
|
||||
model_arch = gguf.MODEL_ARCH.COHERE2
|
||||
|
||||
@@ -63,7 +59,6 @@ class Cohere2Model(TextModel):
|
||||
|
||||
|
||||
@ModelBase.register("Cohere2MoeForCausalLM")
|
||||
@ModelBase.example("CohereLabs/North-Mini-Code-1.0")
|
||||
class Cohere2MoeModel(TextModel):
|
||||
model_arch = gguf.MODEL_ARCH.COHERE2MOE
|
||||
_n_main_layers: int | None = None
|
||||
|
||||
@@ -9,7 +9,6 @@ from .base import ModelBase, TextModel, gguf, logger
|
||||
|
||||
|
||||
@ModelBase.register("DbrxForCausalLM")
|
||||
@ModelBase.example("alpindale/dbrx-instruct")
|
||||
class DbrxModel(TextModel):
|
||||
model_arch = gguf.MODEL_ARCH.DBRX
|
||||
|
||||
|
||||
@@ -13,7 +13,6 @@ from .base import ModelBase, TextModel, gguf
|
||||
|
||||
|
||||
@ModelBase.register("DeciLMForCausalLM")
|
||||
@ModelBase.example("nvidia/Llama-3_1-Nemotron-51B-Instruct", "Deci/DeciLM-7B")
|
||||
class DeciModel(TextModel):
|
||||
model_arch = gguf.MODEL_ARCH.DECI
|
||||
|
||||
|
||||
@@ -18,7 +18,6 @@ from .qwen import QwenModel
|
||||
|
||||
|
||||
@ModelBase.register("DeepseekOCRForCausalLM")
|
||||
@ModelBase.example("deepseek-ai/DeepSeek-OCR")
|
||||
class DeepseekOCRVisionModel(MmprojModel):
|
||||
# HF dynamic_preprocess() max_num, which differs per model
|
||||
preproc_max_tiles = 9
|
||||
@@ -101,13 +100,11 @@ class DeepseekOCRVisionModel(MmprojModel):
|
||||
|
||||
|
||||
@ModelBase.register("UnlimitedOCRForCausalLM")
|
||||
@ModelBase.example("baidu/Unlimited-OCR")
|
||||
class UnlimitedOCRVisionModel(DeepseekOCRVisionModel):
|
||||
preproc_max_tiles = 32
|
||||
|
||||
|
||||
@ModelBase.register("DeepseekOCR2ForCausalLM")
|
||||
@ModelBase.example("deepseek-ai/DeepSeek-OCR-2")
|
||||
class DeepseekOCR2VisionModel(DeepseekOCRVisionModel):
|
||||
preproc_max_tiles = 6
|
||||
|
||||
@@ -137,7 +134,6 @@ class DeepseekOCR2VisionModel(DeepseekOCRVisionModel):
|
||||
|
||||
|
||||
@ModelBase.register("DeepseekForCausalLM")
|
||||
@ModelBase.example("deepseek-ai/deepseek-moe-16b-chat")
|
||||
class DeepseekModel(TextModel):
|
||||
model_arch = gguf.MODEL_ARCH.DEEPSEEK
|
||||
|
||||
@@ -232,7 +228,6 @@ class DeepseekModel(TextModel):
|
||||
"YoutuForCausalLM",
|
||||
"YoutuVLForConditionalGeneration",
|
||||
)
|
||||
@ModelBase.example("deepseek-ai/DeepSeek-V2-Lite", "deepseek-ai/DeepSeek-V3")
|
||||
class DeepseekV2Model(TextModel):
|
||||
model_arch = gguf.MODEL_ARCH.DEEPSEEK2
|
||||
|
||||
@@ -462,7 +457,6 @@ class DeepseekV2Model(TextModel):
|
||||
|
||||
|
||||
@ModelBase.register("DeepseekV32ForCausalLM")
|
||||
@ModelBase.example("deepseek-ai/DeepSeek-V3.2-Exp")
|
||||
class DeepseekV32Model(DeepseekV2Model):
|
||||
model_arch = gguf.MODEL_ARCH.DEEPSEEK32
|
||||
skip_mtp = False
|
||||
@@ -523,7 +517,6 @@ class DeepseekV32Model(DeepseekV2Model):
|
||||
|
||||
|
||||
@ModelBase.register("DeepseekV4ForCausalLM")
|
||||
@ModelBase.example("deepseek-ai/DeepSeek-V4-Flash-Base")
|
||||
class DeepseekV4Model(TextModel):
|
||||
model_arch = gguf.MODEL_ARCH.DEEPSEEK4
|
||||
supports_mtp_export = True
|
||||
@@ -918,7 +911,6 @@ class DeepseekV4Model(TextModel):
|
||||
|
||||
|
||||
@ModelBase.register("DeepseekV4DSparkModel")
|
||||
@ModelBase.example("deepseek-ai/DeepSeek-V4-Flash-DSpark")
|
||||
class DeepseekV4DSparkModel(DeepseekV4Model):
|
||||
model_arch = gguf.MODEL_ARCH.DFLASH
|
||||
|
||||
|
||||
@@ -11,7 +11,6 @@ from .qwen import Qwen2MoeModel
|
||||
|
||||
|
||||
@ModelBase.register("Dots1ForCausalLM")
|
||||
@ModelBase.example("rednote-hilab/dots.llm1.inst")
|
||||
class Dots1Model(Qwen2MoeModel):
|
||||
model_arch = gguf.MODEL_ARCH.DOTS1
|
||||
|
||||
|
||||
@@ -9,7 +9,6 @@ from .base import MmprojModel, ModelBase, gguf
|
||||
|
||||
|
||||
@ModelBase.register("DotsOCRForCausalLM")
|
||||
@ModelBase.example("rednote-hilab/dots.ocr")
|
||||
class DotsOCRVisionModel(MmprojModel):
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
|
||||
@@ -9,7 +9,6 @@ from .base import ModelBase, TextModel, gguf
|
||||
|
||||
|
||||
@ModelBase.register("DreamModel")
|
||||
@ModelBase.example("Dream-org/Dream-v0-Instruct-7B")
|
||||
class DreamModel(TextModel):
|
||||
model_arch = gguf.MODEL_ARCH.DREAM
|
||||
|
||||
|
||||
@@ -15,7 +15,6 @@ from .base import MmprojModel, ModelBase, TextModel, gguf
|
||||
|
||||
|
||||
@ModelBase.register("Ernie4_5_ForCausalLM", "Ernie4_5ForCausalLM")
|
||||
@ModelBase.example("baidu/ERNIE-4.5-0.3B-PT")
|
||||
class Ernie4_5Model(TextModel):
|
||||
model_arch = gguf.MODEL_ARCH.ERNIE4_5
|
||||
|
||||
@@ -74,7 +73,6 @@ class Ernie4_5Model(TextModel):
|
||||
|
||||
|
||||
@ModelBase.register("Ernie4_5_MoeForCausalLM")
|
||||
@ModelBase.example("baidu/ERNIE-4.5-21B-A3B-PT")
|
||||
class Ernie4_5MoeModel(Ernie4_5Model):
|
||||
model_arch = gguf.MODEL_ARCH.ERNIE4_5_MOE
|
||||
_experts: list[dict[str, Tensor]] | None = None
|
||||
@@ -158,13 +156,11 @@ class Ernie4_5MoeModel(Ernie4_5Model):
|
||||
|
||||
|
||||
@ModelBase.register("PaddleOCRVLForConditionalGeneration")
|
||||
@ModelBase.example("PaddlePaddle/PaddleOCR-VL")
|
||||
class PaddleOCRModel(Ernie4_5Model):
|
||||
model_arch = gguf.MODEL_ARCH.PADDLEOCR
|
||||
|
||||
|
||||
@ModelBase.register("PaddleOCRVisionModel")
|
||||
@ModelBase.example("PaddlePaddle/PaddleOCR-VL")
|
||||
class PaddleOCRVisionModel(MmprojModel):
|
||||
# PaddleOCR-VL uses a modified version of Siglip
|
||||
min_pixels: int = 0
|
||||
|
||||
@@ -15,7 +15,6 @@ from .qwenvl import Qwen2VLVisionModel
|
||||
|
||||
|
||||
@ModelBase.register("ExaoneForCausalLM")
|
||||
@ModelBase.example("LGAI-EXAONE/EXAONE-3.5-2.4B-Instruct")
|
||||
class ExaoneModel(TextModel):
|
||||
model_arch = gguf.MODEL_ARCH.EXAONE
|
||||
|
||||
@@ -61,7 +60,6 @@ class ExaoneModel(TextModel):
|
||||
|
||||
|
||||
@ModelBase.register("Exaone4ForCausalLM")
|
||||
@ModelBase.example("LGAI-EXAONE/EXAONE-4.0-32B")
|
||||
class Exaone4Model(TextModel):
|
||||
model_arch = gguf.MODEL_ARCH.EXAONE4
|
||||
|
||||
@@ -128,7 +126,6 @@ class Exaone4Model(TextModel):
|
||||
# note: transformers >= 5.1 renamed the class to "ExaoneMoeForCausalLM" (lowercase 'e'),
|
||||
# so accept both spellings - LG AI have updated the configs of already-released models
|
||||
@ModelBase.register("ExaoneMoEForCausalLM", "ExaoneMoeForCausalLM")
|
||||
@ModelBase.example("LGAI-EXAONE/K-EXAONE-236B-A23B")
|
||||
class ExaoneMoEModel(Exaone4Model):
|
||||
model_arch = gguf.MODEL_ARCH.EXAONE_MOE
|
||||
|
||||
@@ -217,7 +214,6 @@ class ExaoneMoEModel(Exaone4Model):
|
||||
|
||||
|
||||
@ModelBase.register("Exaone4_5_ForConditionalGeneration")
|
||||
@ModelBase.example("LGAI-EXAONE/EXAONE-4.5-33B")
|
||||
class Exaone4_5_TextModel(Exaone4Model):
|
||||
"""Text tower of EXAONE 4.5; Tensors match EXAONE4"""
|
||||
|
||||
@@ -271,7 +267,6 @@ class Exaone4_5_TextModel(Exaone4Model):
|
||||
|
||||
|
||||
@ModelBase.register("Exaone4_5_ForConditionalGeneration")
|
||||
@ModelBase.example("LGAI-EXAONE/EXAONE-4.5-33B")
|
||||
class Exaone4_5VisionModel(Qwen2VLVisionModel):
|
||||
"""Vision tower for EXAONE 4.5; Qwen2-VL-style ViT (GQA) + patch merger"""
|
||||
|
||||
|
||||
@@ -11,7 +11,6 @@ from .base import ModelBase, TextModel, gguf
|
||||
|
||||
|
||||
@ModelBase.register("FalconForCausalLM", "RWForCausalLM")
|
||||
@ModelBase.example("tiiuae/falcon-7b")
|
||||
class FalconModel(TextModel):
|
||||
model_arch = gguf.MODEL_ARCH.FALCON
|
||||
|
||||
|
||||
@@ -12,7 +12,6 @@ from .mamba import Mamba2Model
|
||||
|
||||
|
||||
@ModelBase.register("FalconH1ForCausalLM")
|
||||
@ModelBase.example("tiiuae/Falcon-H1-0.5B-Base")
|
||||
class FalconH1Model(Mamba2Model):
|
||||
model_arch = gguf.MODEL_ARCH.FALCON_H1
|
||||
|
||||
|
||||
@@ -14,8 +14,6 @@ from .base import MmprojModel, ModelBase, TextModel, gguf, logger
|
||||
|
||||
|
||||
@ModelBase.register("GemmaForCausalLM")
|
||||
# [TAG_HF_EXAMPLE_GATED] google/gemma-2b is gated
|
||||
@ModelBase.example("trl-internal-testing/tiny-GemmaForCausalLM")
|
||||
class GemmaModel(TextModel):
|
||||
model_arch = gguf.MODEL_ARCH.GEMMA
|
||||
|
||||
@@ -70,8 +68,6 @@ class GemmaModel(TextModel):
|
||||
|
||||
|
||||
@ModelBase.register("Gemma2ForCausalLM")
|
||||
# [TAG_HF_EXAMPLE_GATED] google/gemma-2-9b-it is gated
|
||||
@ModelBase.example("trl-internal-testing/tiny-Gemma2ForCausalLM")
|
||||
class Gemma2Model(TextModel):
|
||||
model_arch = gguf.MODEL_ARCH.GEMMA2
|
||||
|
||||
@@ -122,8 +118,6 @@ class Gemma2Model(TextModel):
|
||||
|
||||
|
||||
@ModelBase.register("Gemma3ForCausalLM", "Gemma3ForConditionalGeneration")
|
||||
# [TAG_HF_EXAMPLE_GATED] google/gemma-3-4b-it is gated
|
||||
@ModelBase.example("trl-internal-testing/tiny-Gemma3ForConditionalGeneration", "hf-tiny-v2/tiny-random-Gemma3ForCausalLM")
|
||||
class Gemma3Model(TextModel):
|
||||
model_arch = gguf.MODEL_ARCH.GEMMA3
|
||||
|
||||
@@ -180,8 +174,6 @@ class Gemma3Model(TextModel):
|
||||
|
||||
|
||||
@ModelBase.register("Gemma3TextModel")
|
||||
# [TAG_HF_EXAMPLE_GATED] google/embeddinggemma-300m is gated
|
||||
@ModelBase.example("hf-tiny-v2/tiny-random-Gemma3TextModel")
|
||||
class EmbeddingGemma(Gemma3Model):
|
||||
model_arch = gguf.MODEL_ARCH.GEMMA_EMBEDDING
|
||||
module_paths = []
|
||||
@@ -256,8 +248,6 @@ class EmbeddingGemma(Gemma3Model):
|
||||
|
||||
|
||||
@ModelBase.register("Gemma3ForConditionalGeneration")
|
||||
# [TAG_HF_EXAMPLE_GATED] google/gemma-3-4b-it is gated
|
||||
@ModelBase.example("trl-internal-testing/tiny-Gemma3ForConditionalGeneration")
|
||||
class Gemma3VisionModel(MmprojModel):
|
||||
def set_gguf_parameters(self):
|
||||
super().set_gguf_parameters()
|
||||
@@ -362,8 +352,6 @@ class ConformerAudioModel(MmprojModel):
|
||||
|
||||
|
||||
@ModelBase.register("Gemma3nForConditionalGeneration")
|
||||
# [TAG_HF_EXAMPLE_GATED] google/gemma-3n-E2B-it is gated
|
||||
@ModelBase.example("hf-tiny-v2/tiny-random-Gemma3nForConditionalGeneration")
|
||||
class Gemma3nVisionAudioModel(ConformerAudioModel):
|
||||
has_audio_encoder = True
|
||||
has_vision_encoder = True
|
||||
@@ -483,8 +471,6 @@ class Gemma3nVisionAudioModel(ConformerAudioModel):
|
||||
|
||||
|
||||
@ModelBase.register("Gemma3nForCausalLM", "Gemma3nForConditionalGeneration")
|
||||
# [TAG_HF_EXAMPLE_GATED] google/gemma-3n-E2B-it is gated
|
||||
@ModelBase.example("hf-tiny-v2/tiny-random-Gemma3nForConditionalGeneration")
|
||||
class Gemma3NModel(Gemma3Model):
|
||||
model_arch = gguf.MODEL_ARCH.GEMMA3N
|
||||
|
||||
@@ -629,7 +615,6 @@ class Gemma3NModel(Gemma3Model):
|
||||
|
||||
|
||||
@ModelBase.register("Gemma4ForConditionalGeneration", "Gemma4ForCausalLM")
|
||||
@ModelBase.example("google/gemma-4-31B-it", "google/gemma-4-26B-A4B-it", "google/gemma-4-E2B-it")
|
||||
class Gemma4Model(Gemma3Model):
|
||||
model_arch = gguf.MODEL_ARCH.GEMMA4
|
||||
|
||||
@@ -810,7 +795,6 @@ class Gemma4Model(Gemma3Model):
|
||||
|
||||
|
||||
@ModelBase.register("Gemma4UnifiedForConditionalGeneration")
|
||||
@ModelBase.example("hf-tiny-v2/tiny-random-Gemma4UnifiedForConditionalGeneration")
|
||||
class Gemma4UnifiedModel(Gemma4Model):
|
||||
model_arch = gguf.MODEL_ARCH.GEMMA4
|
||||
|
||||
@@ -831,7 +815,6 @@ class Gemma4UnifiedModel(Gemma4Model):
|
||||
|
||||
|
||||
@ModelBase.register("Gemma4AssistantForCausalLM", "Gemma4UnifiedAssistantForCausalLM")
|
||||
@ModelBase.example("google/gemma-4-31B-it-assistant", "google/gemma-4-26B-A4B-it-assistant", "google/gemma-4-E2B-it-assistant")
|
||||
class Gemma4AssistantModel(Gemma4Model):
|
||||
model_arch = gguf.MODEL_ARCH.GEMMA4_ASSISTANT
|
||||
|
||||
@@ -852,7 +835,6 @@ class Gemma4AssistantModel(Gemma4Model):
|
||||
|
||||
|
||||
@ModelBase.register("Gemma4ForConditionalGeneration")
|
||||
@ModelBase.example("google/gemma-4-31B-it", "google/gemma-4-26B-A4B-it", "google/gemma-4-E2B-it")
|
||||
class Gemma4VisionAudioModel(MmprojModel):
|
||||
has_audio_encoder = True
|
||||
has_vision_encoder = True
|
||||
@@ -931,7 +913,6 @@ class Gemma4VisionAudioModel(MmprojModel):
|
||||
|
||||
|
||||
@ModelBase.register("Gemma4UnifiedForConditionalGeneration")
|
||||
@ModelBase.example("hf-tiny-v2/tiny-random-Gemma4UnifiedForConditionalGeneration")
|
||||
class Gemma4UnifiedVisionAudioModel(Gemma4VisionAudioModel):
|
||||
has_audio_encoder = True
|
||||
has_vision_encoder = True
|
||||
|
||||
@@ -15,7 +15,6 @@ from .deepseek import DeepseekV2Model
|
||||
|
||||
|
||||
@ModelBase.register("Glm4ForCausalLM", "Glm4vForConditionalGeneration")
|
||||
@ModelBase.example("zai-org/GLM-4-9B-0414")
|
||||
class Glm4Model(TextModel):
|
||||
model_arch = gguf.MODEL_ARCH.GLM4
|
||||
use_mrope = False
|
||||
@@ -87,7 +86,6 @@ class Glm4Model(TextModel):
|
||||
|
||||
|
||||
@ModelBase.register("GlmOcrForConditionalGeneration")
|
||||
@ModelBase.example("zai-org/GLM-OCR")
|
||||
class GlmOCRModel(Glm4Model):
|
||||
model_arch = gguf.MODEL_ARCH.GLM4
|
||||
use_mrope = False
|
||||
@@ -109,7 +107,6 @@ class GlmOCRModel(Glm4Model):
|
||||
|
||||
|
||||
@ModelBase.register("Glm4MoeForCausalLM", "Glm4vMoeForConditionalGeneration")
|
||||
@ModelBase.example("zai-org/GLM-4.5-Air")
|
||||
class Glm4MoeModel(TextModel):
|
||||
model_arch = gguf.MODEL_ARCH.GLM4_MOE
|
||||
|
||||
@@ -207,7 +204,6 @@ class Glm4MoeModel(TextModel):
|
||||
|
||||
|
||||
@ModelBase.register("Glm4MoeLiteForCausalLM")
|
||||
@ModelBase.example("zai-org/GLM-4.7-Flash")
|
||||
class Glm4MoeLiteModel(DeepseekV2Model):
|
||||
model_arch = gguf.MODEL_ARCH.DEEPSEEK2
|
||||
skip_mtp = False
|
||||
@@ -276,7 +272,6 @@ class Glm4MoeLiteModel(DeepseekV2Model):
|
||||
|
||||
|
||||
@ModelBase.register("GlmMoeDsaForCausalLM")
|
||||
@ModelBase.example("zai-org/GLM-5.2")
|
||||
class GlmMoeDsaModel(DeepseekV2Model):
|
||||
model_arch = gguf.MODEL_ARCH.GLM_DSA
|
||||
skip_mtp = False
|
||||
@@ -345,7 +340,6 @@ class GlmMoeDsaModel(DeepseekV2Model):
|
||||
|
||||
|
||||
@ModelBase.register("SolarOpenForCausalLM")
|
||||
@ModelBase.example("upstage/Solar-Open-100B")
|
||||
class SolarOpenModel(Glm4MoeModel):
|
||||
model_arch = gguf.MODEL_ARCH.GLM4_MOE
|
||||
|
||||
|
||||
@@ -11,7 +11,6 @@ from .base import ModelBase, TextModel, gguf, logger
|
||||
|
||||
|
||||
@ModelBase.register("GPT2LMHeadModel")
|
||||
@ModelBase.example("openai-community/gpt2")
|
||||
class GPT2Model(TextModel):
|
||||
model_arch = gguf.MODEL_ARCH.GPT2
|
||||
|
||||
@@ -39,7 +38,6 @@ class GPT2Model(TextModel):
|
||||
|
||||
|
||||
@ModelBase.register("RuGPT3XLForCausalLM")
|
||||
@ModelBase.example("evilfreelancer/ruGPT3XL")
|
||||
class RuGPT3XLModel(TextModel):
|
||||
model_arch = gguf.MODEL_ARCH.GPT2
|
||||
|
||||
|
||||
@@ -11,7 +11,6 @@ from .base import ModelBase, TextModel, gguf, logger
|
||||
|
||||
|
||||
@ModelBase.register("GptOssForCausalLM")
|
||||
@ModelBase.example("openai/gpt-oss-20b")
|
||||
class GptOssModel(TextModel):
|
||||
model_arch = gguf.MODEL_ARCH.GPT_OSS
|
||||
|
||||
|
||||
@@ -13,7 +13,6 @@ from .base import ModelBase, TextModel, gguf, logger
|
||||
|
||||
|
||||
@ModelBase.register("GPTNeoXForCausalLM")
|
||||
@ModelBase.example("EleutherAI/pythia-70m")
|
||||
class GPTNeoXModel(TextModel):
|
||||
model_arch = gguf.MODEL_ARCH.GPTNEOX
|
||||
|
||||
|
||||
@@ -15,7 +15,6 @@ from .mamba import Mamba2Model
|
||||
|
||||
|
||||
@ModelBase.register("GraniteForCausalLM")
|
||||
@ModelBase.example("ibm-granite/granite-3.3-2b-instruct")
|
||||
class GraniteModel(LlamaModel):
|
||||
"""Conversion for IBM's GraniteForCausalLM"""
|
||||
model_arch = gguf.MODEL_ARCH.GRANITE
|
||||
@@ -75,7 +74,6 @@ class GraniteModel(LlamaModel):
|
||||
|
||||
|
||||
@ModelBase.register("GraniteMoeForCausalLM", "GraniteMoeSharedForCausalLM")
|
||||
@ModelBase.example("ibm-granite/granite-3.1-3b-a800m-instruct")
|
||||
class GraniteMoeModel(GraniteModel):
|
||||
"""Conversion for IBM's GraniteMoeForCausalLM"""
|
||||
model_arch = gguf.MODEL_ARCH.GRANITE_MOE
|
||||
@@ -126,7 +124,6 @@ class GraniteMoeModel(GraniteModel):
|
||||
|
||||
|
||||
@ModelBase.register("GraniteSwitchForCausalLM")
|
||||
@ModelBase.example("ibm-granite/granite-switch-4.1-3b-preview")
|
||||
class GraniteSwitchModel(GraniteMoeModel):
|
||||
"""Dense, all-attention Granite with N per-token embedded LoRA adapters, stacked
|
||||
over the adapter dim with a zero adapter at slot 0 (N = num_adapters + 1)."""
|
||||
@@ -287,7 +284,6 @@ class GraniteSwitchModel(GraniteMoeModel):
|
||||
|
||||
|
||||
@ModelBase.register("GraniteMoeHybridForCausalLM", "BambaForCausalLM")
|
||||
@ModelBase.example("ibm-granite/granite-4.0-h-tiny", "ibm-ai-platform/Bamba-9B-v2")
|
||||
class GraniteHybridModel(Mamba2Model, GraniteMoeModel):
|
||||
"""GraniteHybrid is a hybrid SSM + Attention model that uses Mamba2 SSM
|
||||
layers and optionally uses MoE w/ a shared expert"""
|
||||
@@ -430,7 +426,6 @@ class GraniteHybridModel(Mamba2Model, GraniteMoeModel):
|
||||
|
||||
|
||||
@ModelBase.register("GraniteSpeechForConditionalGeneration")
|
||||
@ModelBase.example("ibm-granite/granite-speech-3.3-2b", "ibm-granite/granite-4.0-1b-speech")
|
||||
class GraniteSpeechMmprojModel(MmprojModel):
|
||||
has_vision_encoder = False
|
||||
has_audio_encoder = True
|
||||
@@ -514,7 +509,6 @@ class GraniteSpeechMmprojModel(MmprojModel):
|
||||
|
||||
|
||||
@ModelBase.register("GraniteSpeechPlusForConditionalGeneration")
|
||||
@ModelBase.example("ibm-granite/granite-speech-4.1-2b-plus")
|
||||
class GraniteSpeechPlusMmprojModel(GraniteSpeechMmprojModel):
|
||||
"""Conversion for GraniteSpeechPlus - extends GraniteSpeech with feature layer concatenation"""
|
||||
has_vision_encoder = False
|
||||
@@ -543,7 +537,6 @@ class GraniteSpeechPlusMmprojModel(GraniteSpeechMmprojModel):
|
||||
|
||||
|
||||
@ModelBase.register("Granite4VisionForConditionalGeneration")
|
||||
@ModelBase.example("ibm-granite/granite-4.0-3b-vision")
|
||||
class Granite4VisionMmprojModel(MmprojModel):
|
||||
has_vision_encoder = True
|
||||
has_audio_encoder = False
|
||||
|
||||
@@ -13,7 +13,6 @@ from .base import ModelBase, TextModel, gguf, logger
|
||||
|
||||
|
||||
@ModelBase.register("GrokForCausalLM", "Grok1ForCausalLM")
|
||||
@ModelBase.example("keyfan/grok-1-hf")
|
||||
class GrokModel(TextModel):
|
||||
model_arch = gguf.MODEL_ARCH.GROK
|
||||
|
||||
|
||||
@@ -11,7 +11,6 @@ from .base import ModelBase, TextModel, gguf, logger
|
||||
|
||||
|
||||
@ModelBase.register("GroveMoeForCausalLM", "modeling_grove_moe.GroveMoeForCausalLM")
|
||||
@ModelBase.example("inclusionAI/GroveMoE-Inst")
|
||||
class GroveMoeModel(TextModel):
|
||||
model_arch = gguf.MODEL_ARCH.GROVEMOE
|
||||
|
||||
|
||||
@@ -17,7 +17,6 @@ from .qwen import QwenModel
|
||||
|
||||
|
||||
@ModelBase.register("HunYuanMoEV1ForCausalLM")
|
||||
@ModelBase.example("tencent/Hunyuan-A13B-Instruct")
|
||||
class HunYuanMoEModel(TextModel):
|
||||
model_arch = gguf.MODEL_ARCH.HUNYUAN_MOE
|
||||
|
||||
@@ -155,7 +154,6 @@ class HunYuanMoEModel(TextModel):
|
||||
|
||||
|
||||
@ModelBase.register("HunYuanDenseV1ForCausalLM")
|
||||
@ModelBase.example("tencent/Hunyuan-4B-Instruct")
|
||||
class HunYuanModel(TextModel):
|
||||
model_arch = gguf.MODEL_ARCH.HUNYUAN_DENSE
|
||||
|
||||
@@ -292,7 +290,6 @@ class HunYuanModel(TextModel):
|
||||
|
||||
|
||||
@ModelBase.register("HunYuanVLForConditionalGeneration")
|
||||
@ModelBase.example("tencent/HunyuanOCR")
|
||||
class HunyuanVLVisionModel(MmprojModel):
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
@@ -336,7 +333,6 @@ class HunyuanVLVisionModel(MmprojModel):
|
||||
|
||||
|
||||
@ModelBase.register("HunYuanVLForConditionalGeneration")
|
||||
@ModelBase.example("tencent/HunyuanOCR")
|
||||
class HunyuanVLTextModel(HunYuanModel):
|
||||
model_arch = gguf.MODEL_ARCH.HUNYUAN_VL
|
||||
|
||||
@@ -369,7 +365,6 @@ class HunyuanVLTextModel(HunYuanModel):
|
||||
|
||||
|
||||
@ModelBase.register("HYV3ForCausalLM")
|
||||
@ModelBase.example("tencent/Hy3")
|
||||
class HYV3Model(TextModel):
|
||||
model_arch = gguf.MODEL_ARCH.HY_V3
|
||||
supports_mtp_export = True
|
||||
|
||||
@@ -14,7 +14,6 @@ from .llama import LlamaModel
|
||||
|
||||
|
||||
@ModelBase.register("InternLM2ForCausalLM")
|
||||
@ModelBase.example("internlm/internlm2-chat-7b")
|
||||
class InternLM2Model(TextModel):
|
||||
model_arch = gguf.MODEL_ARCH.INTERNLM2
|
||||
|
||||
@@ -171,7 +170,6 @@ class InternLM2Model(TextModel):
|
||||
|
||||
|
||||
@ModelBase.register("InternLM3ForCausalLM")
|
||||
@ModelBase.example("internlm/internlm3-8b-instruct")
|
||||
class InternLM3Model(TextModel):
|
||||
model_arch = gguf.MODEL_ARCH.LLAMA
|
||||
|
||||
|
||||
@@ -9,7 +9,6 @@ from .base import MmprojModel, ModelBase, gguf
|
||||
|
||||
|
||||
@ModelBase.register("InternVisionModel")
|
||||
@ModelBase.example("OpenGVLab/InternVL3-2B", "OpenGVLab/InternVL2_5-1B")
|
||||
class InternVisionModel(MmprojModel):
|
||||
|
||||
min_dynamic_tiles: int = 0
|
||||
|
||||
@@ -11,8 +11,6 @@ from .base import ModelBase, TextModel, gguf
|
||||
|
||||
|
||||
@ModelBase.register("Jais2ForCausalLM")
|
||||
# [TAG_HF_EXAMPLE_GATED] inceptionai/Jais-2-8B-Chat is gated
|
||||
# [TAG_HF_EXAMPLE_MISSING]
|
||||
class Jais2Model(TextModel):
|
||||
model_arch = gguf.MODEL_ARCH.JAIS2
|
||||
|
||||
@@ -24,7 +22,6 @@ class Jais2Model(TextModel):
|
||||
|
||||
|
||||
@ModelBase.register("JAISLMHeadModel")
|
||||
@ModelBase.example("inceptionai/jais-family-590m")
|
||||
class JaisModel(TextModel):
|
||||
model_arch = gguf.MODEL_ARCH.JAIS
|
||||
|
||||
|
||||
@@ -11,7 +11,6 @@ from .base import ModelBase, TextModel, gguf, logger
|
||||
|
||||
|
||||
@ModelBase.register("JambaForCausalLM")
|
||||
@ModelBase.example("ai21labs/Jamba-v0.1")
|
||||
class JambaModel(TextModel):
|
||||
model_arch = gguf.MODEL_ARCH.JAMBA
|
||||
|
||||
|
||||
@@ -11,7 +11,6 @@ from .llama import LlamaModel
|
||||
|
||||
|
||||
@ModelBase.register("JanusForConditionalGeneration")
|
||||
@ModelBase.example("deepseek-community/Janus-Pro-1B")
|
||||
class JanusProModel(LlamaModel):
|
||||
model_arch = gguf.MODEL_ARCH.LLAMA # reuse Llama arch
|
||||
|
||||
@@ -35,7 +34,6 @@ class JanusProModel(LlamaModel):
|
||||
|
||||
|
||||
@ModelBase.register("JanusForConditionalGeneration")
|
||||
@ModelBase.example("deepseek-community/Janus-Pro-1B")
|
||||
class JanusProVisionModel(MmprojModel):
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
|
||||
@@ -16,7 +16,6 @@ from .kimi_linear import KimiLinearModel
|
||||
|
||||
|
||||
@ModelBase.register("KimiK3ForConditionalGeneration")
|
||||
@ModelBase.example("moonshotai/Kimi-K3")
|
||||
class KimiK3Model(TextModel):
|
||||
"""
|
||||
Kimi-K3 text model (KimiLinearForCausalLM under a `language_model.` prefix).
|
||||
|
||||
@@ -13,7 +13,6 @@ from .qwen import QwenModel
|
||||
|
||||
|
||||
@ModelBase.register("KimiLinearModel", "KimiLinearForCausalLM")
|
||||
@ModelBase.example("moonshotai/Kimi-Linear-48B-A3B-Instruct")
|
||||
class KimiLinearModel(TextModel):
|
||||
"""Kimi-Linear model with hybrid MLA+KDA architecture"""
|
||||
model_arch = gguf.MODEL_ARCH.KIMI_LINEAR
|
||||
|
||||
@@ -11,7 +11,6 @@ from .base import MmprojModel, ModelBase, gguf
|
||||
|
||||
|
||||
@ModelBase.register("KimiVLForConditionalGeneration")
|
||||
@ModelBase.example("moonshotai/Kimi-VL-A3B-Instruct")
|
||||
class KimiVLModel(MmprojModel):
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
@@ -53,7 +52,6 @@ class KimiVLModel(MmprojModel):
|
||||
|
||||
|
||||
@ModelBase.register("KimiK25ForConditionalGeneration")
|
||||
@ModelBase.example("moonshotai/Kimi-K2.5")
|
||||
class KimiK25Model(MmprojModel):
|
||||
"""Kimi-K2.5 with MoonViT3d vision encoder"""
|
||||
|
||||
@@ -157,7 +155,6 @@ class KimiK25Model(MmprojModel):
|
||||
|
||||
|
||||
@ModelBase.register("Glm5vForConditionalGeneration")
|
||||
# [TAG_HF_EXAMPLE_MISSING]
|
||||
class Glm5vModel(KimiK25Model):
|
||||
"""GLM-5.2-Vision MoonViT3d encoder and projector
|
||||
|
||||
|
||||
@@ -13,7 +13,6 @@ from .base import ModelBase, TextModel, gguf, logger
|
||||
|
||||
|
||||
@ModelBase.register("LagunaForCausalLM")
|
||||
@ModelBase.example("poolside/Laguna-XS.2", "poolside/Laguna-S-2.1")
|
||||
class LagunaModel(TextModel):
|
||||
model_arch = gguf.MODEL_ARCH.LAGUNA
|
||||
_experts: list[dict] | None = None
|
||||
|
||||
@@ -13,7 +13,6 @@ from .gemma import ConformerAudioModel
|
||||
|
||||
|
||||
@ModelBase.register("Lfm2ForCausalLM", "LFM2ForCausalLM")
|
||||
@ModelBase.example("LiquidAI/LFM2-1.2B", "LiquidAI/LFM2.5-350M")
|
||||
class LFM2Model(TextModel):
|
||||
model_arch = gguf.MODEL_ARCH.LFM2
|
||||
|
||||
@@ -66,7 +65,6 @@ class LFM2Model(TextModel):
|
||||
|
||||
|
||||
@ModelBase.register("Lfm2Model", "Lfm2BidirectionalModel")
|
||||
@ModelBase.example("LiquidAI/LFM2.5-ColBERT-350M", "LiquidAI/LFM2.5-Embedding-350M")
|
||||
class LFM2ColBertModel(LFM2Model):
|
||||
model_arch = gguf.MODEL_ARCH.LFM2
|
||||
dense_tensor_name = "dense_2"
|
||||
@@ -95,7 +93,6 @@ class LFM2ColBertModel(LFM2Model):
|
||||
|
||||
|
||||
@ModelBase.register("Lfm2MoeForCausalLM")
|
||||
@ModelBase.example("LiquidAI/LFM2-8B-A1B")
|
||||
class LFM2MoeModel(TextModel):
|
||||
model_arch = gguf.MODEL_ARCH.LFM2MOE
|
||||
|
||||
@@ -169,7 +166,6 @@ class LFM2MoeModel(TextModel):
|
||||
|
||||
|
||||
@ModelBase.register("Lfm2VlForConditionalGeneration")
|
||||
@ModelBase.example("LiquidAI/LFM2-VL-450M")
|
||||
class LFM2VLModel(MmprojModel):
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
@@ -204,7 +200,6 @@ class LFM2VLModel(MmprojModel):
|
||||
|
||||
|
||||
@ModelBase.register("Lfm2AudioForConditionalGeneration")
|
||||
@ModelBase.example("LiquidAI/LFM2.5-Audio-1.5B", "LiquidAI/LFM2-Audio-1.5B")
|
||||
class LFM2AudioModel(ConformerAudioModel):
|
||||
has_vision_encoder = False
|
||||
has_audio_encoder = True
|
||||
@@ -243,7 +238,6 @@ class LFM2AudioModel(ConformerAudioModel):
|
||||
|
||||
|
||||
@ModelBase.register("Lfm25AudioTokenizer")
|
||||
@ModelBase.example("LiquidAI/LFM2.5-Audio-1.5B")
|
||||
class LFM25AudioTokenizer(LFM2Model):
|
||||
model_arch = gguf.MODEL_ARCH.LFM2
|
||||
|
||||
|
||||
@@ -11,7 +11,6 @@ from .llava import LlavaVisionModel
|
||||
|
||||
|
||||
@ModelBase.register("LightOnOCRForConditionalGeneration")
|
||||
@ModelBase.example("lightonai/LightOnOCR-1B-1025")
|
||||
class LightOnOCRVisionModel(LlavaVisionModel):
|
||||
is_mistral_format = False
|
||||
use_break_tok = False
|
||||
|
||||
@@ -11,7 +11,6 @@ from .base import ModelBase, TextModel, gguf
|
||||
|
||||
|
||||
@ModelBase.register("LLaDAModelLM")
|
||||
@ModelBase.example("GSAI-ML/LLaDA-8B-Instruct")
|
||||
class LLaDAModel(TextModel):
|
||||
model_arch = gguf.MODEL_ARCH.LLADA
|
||||
undo_permute = True
|
||||
@@ -115,7 +114,6 @@ class LLaDAModel(TextModel):
|
||||
|
||||
|
||||
@ModelBase.register("LLaDAMoEModel", "LLaDAMoEModelLM")
|
||||
@ModelBase.example("inclusionAI/LLaDA-MoE-7B-A1B-Instruct")
|
||||
class LLaDAMoEModel(TextModel):
|
||||
model_arch = gguf.MODEL_ARCH.LLADA_MOE
|
||||
|
||||
|
||||
@@ -28,8 +28,6 @@ from .base import ModelBase, TextModel, gguf, logger
|
||||
"Eagle3DraftModel",
|
||||
"IQuestCoderForCausalLM",
|
||||
"LlamaModel")
|
||||
# [TAG_HF_EXAMPLE_GATED] meta-llama/Llama-3.2-1B-Instruct is gated
|
||||
@ModelBase.example("unsloth/Llama-3.2-1B-Instruct", "mistralai/Mistral-7B-Instruct-v0.3", "mistralai/Mixtral-8x7B-Instruct-v0.1")
|
||||
class LlamaModel(TextModel):
|
||||
model_arch = gguf.MODEL_ARCH.LLAMA
|
||||
undo_permute = True
|
||||
@@ -361,7 +359,6 @@ class LlamaModel(TextModel):
|
||||
|
||||
|
||||
@ModelBase.register("ArceeForCausalLM")
|
||||
@ModelBase.example("arcee-ai/AFM-4.5B")
|
||||
class ArceeModel(LlamaModel):
|
||||
model_arch = gguf.MODEL_ARCH.ARCEE
|
||||
|
||||
@@ -374,8 +371,6 @@ class ArceeModel(LlamaModel):
|
||||
"Llama4ForConditionalGeneration",
|
||||
"Llama4ForCausalLM",
|
||||
)
|
||||
# [TAG_HF_EXAMPLE_GATED] meta-llama/Llama-4-Scout-17B-16E-Instruct is gated
|
||||
@ModelBase.example("unsloth/Llama-4-Scout-17B-16E-Instruct")
|
||||
class Llama4Model(LlamaModel):
|
||||
model_arch = gguf.MODEL_ARCH.LLAMA4
|
||||
undo_permute = False
|
||||
@@ -417,19 +412,16 @@ class Llama4Model(LlamaModel):
|
||||
|
||||
|
||||
@ModelBase.register("LlamaBidirectionalModel")
|
||||
@ModelBase.example("nvidia/llama-embed-nemotron-8b")
|
||||
class LlamaEmbedNemotronModel(LlamaModel):
|
||||
model_arch = gguf.MODEL_ARCH.LLAMA_EMBED
|
||||
|
||||
|
||||
@ModelBase.register("SmolLM3ForCausalLM")
|
||||
@ModelBase.example("HuggingFaceTB/SmolLM3-3B")
|
||||
class SmolLM3Model(LlamaModel):
|
||||
model_arch = gguf.MODEL_ARCH.SMOLLM3
|
||||
|
||||
|
||||
@ModelBase.register("ApertusForCausalLM")
|
||||
@ModelBase.example("swiss-ai/Apertus-8B-Instruct-2509")
|
||||
class ApertusModel(LlamaModel):
|
||||
model_arch = gguf.MODEL_ARCH.APERTUS
|
||||
undo_permute = False
|
||||
|
||||
@@ -9,8 +9,6 @@ from .base import MmprojModel, ModelBase, gguf
|
||||
|
||||
|
||||
@ModelBase.register("Llama4ForConditionalGeneration")
|
||||
# [TAG_HF_EXAMPLE_GATED] meta-llama/Llama-4-Scout-17B-16E-Instruct is gated
|
||||
@ModelBase.example("unsloth/Llama-4-Scout-17B-16E-Instruct")
|
||||
class Llama4VisionModel(MmprojModel):
|
||||
def set_gguf_parameters(self):
|
||||
super().set_gguf_parameters()
|
||||
|
||||
@@ -16,7 +16,6 @@ from .llama import LlamaModel
|
||||
"LlavaForConditionalGeneration", # pixtral
|
||||
"Mistral3ForConditionalGeneration", # mistral small 3.1
|
||||
)
|
||||
@ModelBase.example("mistral-community/pixtral-12b", "mistralai/Mistral-Small-3.1-24B-Instruct-2503")
|
||||
class LlavaVisionModel(MmprojModel):
|
||||
img_break_tok_id = -1
|
||||
use_break_tok = True
|
||||
|
||||
@@ -4,7 +4,6 @@ from .base import ModelBase, TextModel, gguf
|
||||
|
||||
|
||||
@ModelBase.register("MaincoderForCausalLM")
|
||||
@ModelBase.example("Maincode/Maincoder-1B")
|
||||
class MaincoderModel(TextModel):
|
||||
model_arch = gguf.MODEL_ARCH.MAINCODER
|
||||
|
||||
|
||||
@@ -14,7 +14,6 @@ from .base import ModelBase, TextModel, gguf, logger
|
||||
|
||||
|
||||
@ModelBase.register("MambaForCausalLM", "MambaLMHeadModel", "FalconMambaForCausalLM")
|
||||
@ModelBase.example("state-spaces/mamba-130m-hf", "tiiuae/falcon-mamba-7b")
|
||||
class MambaModel(TextModel):
|
||||
model_arch = gguf.MODEL_ARCH.MAMBA
|
||||
|
||||
@@ -101,7 +100,6 @@ class MambaModel(TextModel):
|
||||
|
||||
|
||||
@ModelBase.register("Mamba2ForCausalLM")
|
||||
@ModelBase.example("mistralai/Mamba-Codestral-7B-v0.1")
|
||||
class Mamba2Model(TextModel):
|
||||
model_arch = gguf.MODEL_ARCH.MAMBA2
|
||||
|
||||
|
||||
@@ -11,7 +11,6 @@ from .base import ModelBase, TextModel, gguf, logger
|
||||
|
||||
|
||||
@ModelBase.register("MellumForCausalLM")
|
||||
@ModelBase.example("JetBrains/Mellum2-12B-A2.5B-Base")
|
||||
class MellumModel(TextModel):
|
||||
model_arch = gguf.MODEL_ARCH.MELLUM
|
||||
|
||||
|
||||
@@ -14,7 +14,6 @@ from .base import MmprojModel, ModelBase, TextModel, gguf
|
||||
|
||||
|
||||
@ModelBase.register("MiMoV2FlashForCausalLM", "MiMoV2ForCausalLM")
|
||||
@ModelBase.example("XiaomiMiMo/MiMo-V2.5")
|
||||
class MimoV2Model(TextModel):
|
||||
model_arch = gguf.MODEL_ARCH.MIMO2
|
||||
|
||||
@@ -231,7 +230,6 @@ class MimoV2Model(TextModel):
|
||||
|
||||
|
||||
@ModelBase.register("MiMoV2ForCausalLM")
|
||||
@ModelBase.example("XiaomiMiMo/MiMo-V2.5")
|
||||
class MiMoV2VisionAudioModel(MmprojModel):
|
||||
has_audio_encoder = True
|
||||
|
||||
|
||||
@@ -14,7 +14,6 @@ from .qwen import Qwen3_5TextModel
|
||||
|
||||
|
||||
@ModelBase.register("MiniCPMForCausalLM")
|
||||
@ModelBase.example("openbmb/MiniCPM-2B-sft-bf16")
|
||||
class MiniCPMModel(TextModel):
|
||||
model_arch = gguf.MODEL_ARCH.MINICPM
|
||||
|
||||
@@ -62,7 +61,6 @@ class MiniCPMModel(TextModel):
|
||||
|
||||
|
||||
@ModelBase.register("MiniCPM3ForCausalLM")
|
||||
@ModelBase.example("openbmb/MiniCPM3-4B")
|
||||
class MiniCPM3Model(TextModel):
|
||||
model_arch = gguf.MODEL_ARCH.MINICPM3
|
||||
|
||||
@@ -119,7 +117,6 @@ class MiniCPM3Model(TextModel):
|
||||
# the LM (text mode) and once as the mmproj (vision mode), mirroring the Qwen3-VL setup.
|
||||
|
||||
@ModelBase.register("MiniCPMV4_6ForConditionalGeneration")
|
||||
@ModelBase.example("openbmb/MiniCPM-V-4_6")
|
||||
class MiniCPMV4_6TextModel(Qwen3_5TextModel):
|
||||
model_arch = gguf.MODEL_ARCH.QWEN35
|
||||
|
||||
@@ -137,7 +134,6 @@ class MiniCPMV4_6TextModel(Qwen3_5TextModel):
|
||||
|
||||
|
||||
@ModelBase.register("MiniCPMV4_6ForConditionalGeneration")
|
||||
@ModelBase.example("openbmb/MiniCPM-V-4_6")
|
||||
class MiniCPMV4_6VisionModel(MmprojModel):
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
|
||||
@@ -12,7 +12,6 @@ from .base import ModelBase, TextModel, MmprojModel, gguf, logger
|
||||
|
||||
@ModelBase.register("MiniMaxText01ForCausalLM")
|
||||
@ModelBase.register("MiniMaxM1ForCausalLM")
|
||||
@ModelBase.example("MiniMaxAI/MiniMax-Text-01", "MiniMaxAI/MiniMax-M1-40k")
|
||||
class MiniMaxText01Model(TextModel):
|
||||
model_arch = gguf.MODEL_ARCH.MINIMAX01
|
||||
|
||||
@@ -120,7 +119,6 @@ class MiniMaxText01Model(TextModel):
|
||||
|
||||
|
||||
@ModelBase.register("MiniMaxM2ForCausalLM")
|
||||
@ModelBase.example("MiniMaxAI/MiniMax-M2")
|
||||
class MiniMaxM2Model(TextModel):
|
||||
model_arch = gguf.MODEL_ARCH.MINIMAXM2
|
||||
_experts_cache: dict[int, dict[str, Tensor]] = {}
|
||||
@@ -165,7 +163,6 @@ class MiniMaxM2Model(TextModel):
|
||||
|
||||
|
||||
@ModelBase.register("MiniMaxM3SparseForCausalLM", "MiniMaxM3SparseForConditionalGeneration")
|
||||
@ModelBase.example("MiniMaxAI/MiniMax-M3")
|
||||
class MiniMaxM3Model(MiniMaxM2Model):
|
||||
model_arch = gguf.MODEL_ARCH.MINIMAXM3
|
||||
|
||||
@@ -206,7 +203,6 @@ class MiniMaxM3Model(MiniMaxM2Model):
|
||||
|
||||
|
||||
@ModelBase.register("MiniMaxM3SparseForConditionalGeneration", "MiniMaxM3VLForConditionalGeneration")
|
||||
@ModelBase.example("MiniMaxAI/MiniMax-M3")
|
||||
class MiniMaxM3VisionModel(MmprojModel):
|
||||
@classmethod
|
||||
def filter_tensors(cls, item):
|
||||
|
||||
@@ -15,7 +15,6 @@ from .llama import LlamaModel
|
||||
"Mistral3ForConditionalGeneration",
|
||||
"Ministral3ForCausalLM",
|
||||
)
|
||||
@ModelBase.example("mistralai/Mistral-Small-3.1-24B-Instruct-2503", "hf-tiny-v2/tiny-random-Ministral3ForCausalLM")
|
||||
class Mistral3Model(TextModel):
|
||||
class Ministral3Model(LlamaModel):
|
||||
model_arch = gguf.MODEL_ARCH.MISTRAL3
|
||||
|
||||
@@ -9,7 +9,6 @@ from .base import ModelBase, TextModel, gguf
|
||||
|
||||
|
||||
@ModelBase.register("MPTForCausalLM")
|
||||
@ModelBase.example("anas-awadalla/mpt-7b")
|
||||
class MPTModel(TextModel):
|
||||
model_arch = gguf.MODEL_ARCH.MPT
|
||||
|
||||
|
||||
@@ -24,7 +24,6 @@ def _unpermute_for_rope(tensor: "Tensor", n_heads: int) -> "Tensor":
|
||||
|
||||
|
||||
@ModelBase.register("MuseGlimmerForConditionalGeneration")
|
||||
@ModelBase.example("meta-models/Muse-Glimmer-30B")
|
||||
class MuseGlimmerModel(TextModel):
|
||||
model_arch = gguf.MODEL_ARCH.MUSE_GLIMMER
|
||||
|
||||
@@ -79,7 +78,6 @@ class MuseGlimmerModel(TextModel):
|
||||
|
||||
|
||||
@ModelBase.register("MuseGlimmerForConditionalGeneration")
|
||||
@ModelBase.example("meta-models/Muse-Glimmer-30B")
|
||||
class MuseGlimmerVisionModel(MmprojModel):
|
||||
def get_vision_config(self) -> dict[str, Any] | None:
|
||||
c = self.global_config.get("vision_config")
|
||||
@@ -133,7 +131,6 @@ class MuseGlimmerVisionModel(MmprojModel):
|
||||
|
||||
|
||||
@ModelBase.register("MuseGlimmerAssistantModel")
|
||||
@ModelBase.example("meta-models/Muse-Glimmer-30B-assistant")
|
||||
class MuseGlimmerAssistantModel(TextModel):
|
||||
model_arch = gguf.MODEL_ARCH.DFLASH
|
||||
|
||||
|
||||
@@ -5,7 +5,6 @@ from .llama import LlamaModel
|
||||
|
||||
|
||||
@ModelBase.register("NanbeigeForCausalLM")
|
||||
@ModelBase.example("Nanbeige/Nanbeige4.2-3B")
|
||||
class NanbeigeModel(LlamaModel):
|
||||
model_arch = gguf.MODEL_ARCH.NANBEIGE
|
||||
undo_permute = True
|
||||
|
||||
@@ -16,7 +16,6 @@ from .granite import GraniteHybridModel
|
||||
"NemotronH_Nano_VL_V2",
|
||||
"RADIOModel",
|
||||
)
|
||||
@ModelBase.example("nvidia/NVIDIA-Nemotron-Nano-12B-v2-VL-BF16")
|
||||
class NemotronNanoV2VLModel(MmprojModel):
|
||||
# ViT-Huge architecture parameters for RADIO v2.5-h
|
||||
_vit_hidden_size = 1280
|
||||
@@ -152,7 +151,6 @@ class NemotronNanoV2VLModel(MmprojModel):
|
||||
|
||||
|
||||
@ModelBase.register("NemotronForCausalLM")
|
||||
@ModelBase.example("nvidia/Minitron-4B-Base")
|
||||
class NemotronModel(TextModel):
|
||||
model_arch = gguf.MODEL_ARCH.NEMOTRON
|
||||
|
||||
@@ -195,7 +193,6 @@ class NemotronModel(TextModel):
|
||||
|
||||
|
||||
@ModelBase.register("NemotronHForCausalLM")
|
||||
@ModelBase.example("nvidia/Nemotron-H-8B-Base-8K")
|
||||
class NemotronHModel(GraniteHybridModel):
|
||||
"""Hybrid mamba2/attention model from NVIDIA"""
|
||||
model_arch = gguf.MODEL_ARCH.NEMOTRON_H
|
||||
|
||||
@@ -14,7 +14,6 @@ from .llama import LlamaModel
|
||||
|
||||
@ModelBase.register("OlmoForCausalLM")
|
||||
@ModelBase.register("OLMoForCausalLM")
|
||||
@ModelBase.example("allenai/OLMo-1.7-7B-hf")
|
||||
class OlmoModel(TextModel):
|
||||
model_arch = gguf.MODEL_ARCH.OLMO
|
||||
|
||||
@@ -40,14 +39,12 @@ class OlmoModel(TextModel):
|
||||
|
||||
|
||||
@ModelBase.register("SeedOssForCausalLM")
|
||||
@ModelBase.example("ByteDance-Seed/Seed-OSS-36B-Instruct")
|
||||
class SeedOssModel(TextModel):
|
||||
model_arch = gguf.MODEL_ARCH.SEED_OSS
|
||||
|
||||
|
||||
@ModelBase.register("Olmo2ForCausalLM")
|
||||
@ModelBase.register("Olmo3ForCausalLM")
|
||||
@ModelBase.example("allenai/OLMo-2-1124-7B-Instruct", "allenai/Olmo-3-7B-Instruct")
|
||||
class Olmo2Model(TextModel):
|
||||
model_arch = gguf.MODEL_ARCH.OLMO2
|
||||
|
||||
@@ -70,7 +67,6 @@ class Olmo2Model(TextModel):
|
||||
|
||||
|
||||
@ModelBase.register("OlmoeForCausalLM")
|
||||
@ModelBase.example("allenai/OLMoE-1B-7B-0924")
|
||||
class OlmoeModel(TextModel):
|
||||
model_arch = gguf.MODEL_ARCH.OLMOE
|
||||
|
||||
|
||||
@@ -9,7 +9,6 @@ from .base import ModelBase, TextModel, gguf
|
||||
|
||||
|
||||
@ModelBase.register("OpenELMForCausalLM")
|
||||
@ModelBase.example("apple/OpenELM-270M")
|
||||
class OpenELMModel(TextModel):
|
||||
model_arch = gguf.MODEL_ARCH.OPENELM
|
||||
|
||||
|
||||
@@ -4,7 +4,6 @@ from .base import ModelBase, TextModel, gguf
|
||||
|
||||
|
||||
@ModelBase.register("OrionForCausalLM")
|
||||
@ModelBase.example("OrionStarAI/Orion-14B-Base")
|
||||
class OrionModel(TextModel):
|
||||
model_arch = gguf.MODEL_ARCH.ORION
|
||||
|
||||
|
||||
@@ -11,7 +11,6 @@ from .base import ModelBase, TextModel, gguf, logger
|
||||
|
||||
|
||||
@ModelBase.register("PanguEmbeddedForCausalLM")
|
||||
@ModelBase.example("FreedomIntelligence/openPangu-Embedded-7B-V1.1")
|
||||
class PanguEmbeddedModel(TextModel):
|
||||
model_arch = gguf.MODEL_ARCH.PANGU_EMBED
|
||||
|
||||
|
||||
@@ -14,7 +14,6 @@ from .base import MmprojModel, ModelBase, SentencePieceTokenTypes, TextModel, gg
|
||||
|
||||
|
||||
@ModelBase.register("PhiForCausalLM")
|
||||
@ModelBase.example("microsoft/phi-2")
|
||||
class Phi2Model(TextModel):
|
||||
model_arch = gguf.MODEL_ARCH.PHI2
|
||||
|
||||
@@ -37,7 +36,6 @@ class Phi2Model(TextModel):
|
||||
|
||||
|
||||
@ModelBase.register("Phi3ForCausalLM", "Phi4ForCausalLMV")
|
||||
@ModelBase.example("microsoft/Phi-3-mini-4k-instruct")
|
||||
class Phi3MiniModel(TextModel):
|
||||
model_arch = gguf.MODEL_ARCH.PHI3
|
||||
|
||||
@@ -212,7 +210,6 @@ class Phi3MiniModel(TextModel):
|
||||
|
||||
|
||||
@ModelBase.register("Phi4ForCausalLMV")
|
||||
# [TAG_HF_EXAMPLE_MISSING]
|
||||
class Phi4VisionMmprojModel(MmprojModel):
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
@@ -339,7 +336,6 @@ class Phi4VisionMmprojModel(MmprojModel):
|
||||
|
||||
|
||||
@ModelBase.register("PhiMoEForCausalLM")
|
||||
@ModelBase.example("microsoft/Phi-3.5-MoE-instruct")
|
||||
class PhiMoeModel(Phi3MiniModel):
|
||||
model_arch = gguf.MODEL_ARCH.PHIMOE
|
||||
|
||||
|
||||
@@ -13,7 +13,6 @@ from .base import ModelBase, TextModel, gguf
|
||||
|
||||
|
||||
@ModelBase.register("PlamoForCausalLM")
|
||||
@ModelBase.example("pfnet/plamo-13b")
|
||||
class PlamoModel(TextModel):
|
||||
model_arch = gguf.MODEL_ARCH.PLAMO
|
||||
|
||||
@@ -59,7 +58,6 @@ class PlamoModel(TextModel):
|
||||
|
||||
|
||||
@ModelBase.register("Plamo2ForCausalLM", "PLaMo2ForCausalLM")
|
||||
@ModelBase.example("pfnet/plamo-2-1b")
|
||||
class Plamo2Model(TextModel):
|
||||
model_arch = gguf.MODEL_ARCH.PLAMO2
|
||||
|
||||
@@ -149,8 +147,6 @@ class Plamo2Model(TextModel):
|
||||
|
||||
|
||||
@ModelBase.register("Plamo3ForCausalLM", "PLaMo3ForCausalLM")
|
||||
# [TAG_HF_EXAMPLE_GATED] pfnet/plamo-3-nict-2b-base is gated
|
||||
@ModelBase.example("midorin-Linux/plamo-3-12b-self-merged-base")
|
||||
class Plamo3Model(TextModel):
|
||||
model_arch = gguf.MODEL_ARCH.PLAMO3
|
||||
|
||||
|
||||
@@ -4,7 +4,6 @@ from .base import ModelBase, TextModel, gguf
|
||||
|
||||
|
||||
@ModelBase.register("PLMForCausalLM")
|
||||
@ModelBase.example("PLM-Team/PLM-1.8B-Instruct")
|
||||
class PLMModel(TextModel):
|
||||
model_arch = gguf.MODEL_ARCH.PLM
|
||||
|
||||
|
||||
@@ -77,7 +77,6 @@ def _load_hparams(dir_model: Path) -> dict[str, Any]:
|
||||
|
||||
|
||||
@ModelBase.register("PocketTTSModel")
|
||||
# [TAG_HF_EXAMPLE_MISSING] model is gated, and the checkpoint requires cd to subdir, not supported here
|
||||
class PocketTTSModel(TextModel):
|
||||
model_arch = gguf.MODEL_ARCH.POCKETTTS
|
||||
|
||||
@@ -175,7 +174,6 @@ class PocketTTSModel(TextModel):
|
||||
|
||||
|
||||
@ModelBase.register("PocketTTSModel")
|
||||
# [TAG_HF_EXAMPLE_MISSING] model is gated, and the checkpoint requires cd to subdir, not supported here
|
||||
class PocketTTSMmprojModel(MmprojModel):
|
||||
has_audio_encoder = True
|
||||
has_vision_encoder = False
|
||||
|
||||
@@ -13,7 +13,6 @@ from .base import ModelBase, TextModel, gguf, logger
|
||||
|
||||
|
||||
@ModelBase.register("QWenLMHeadModel")
|
||||
@ModelBase.example("Qwen/Qwen-7B")
|
||||
class QwenModel(TextModel):
|
||||
model_arch = gguf.MODEL_ARCH.QWEN
|
||||
|
||||
@@ -52,7 +51,6 @@ class QwenModel(TextModel):
|
||||
"AudioFlamingo3ForConditionalGeneration",
|
||||
"DotsOCRForCausalLM",
|
||||
)
|
||||
@ModelBase.example("Qwen/Qwen2.5-7B-Instruct")
|
||||
class Qwen2Model(TextModel):
|
||||
model_arch = gguf.MODEL_ARCH.QWEN2
|
||||
|
||||
@@ -73,7 +71,6 @@ class Qwen2Model(TextModel):
|
||||
|
||||
|
||||
@ModelBase.register("Qwen2MoeForCausalLM")
|
||||
@ModelBase.example("Qwen/Qwen1.5-MoE-A2.7B")
|
||||
class Qwen2MoeModel(TextModel):
|
||||
model_arch = gguf.MODEL_ARCH.QWEN2MOE
|
||||
|
||||
@@ -156,7 +153,6 @@ class Qwen2MoeModel(TextModel):
|
||||
|
||||
|
||||
@ModelBase.register("Qwen3ForCausalLM", "Qwen3Model")
|
||||
@ModelBase.example("Qwen/Qwen3-8B")
|
||||
class Qwen3Model(Qwen2Model):
|
||||
model_arch = gguf.MODEL_ARCH.QWEN3
|
||||
|
||||
@@ -255,7 +251,6 @@ class Qwen3Model(Qwen2Model):
|
||||
|
||||
|
||||
@ModelBase.register("Qwen3MoeForCausalLM")
|
||||
@ModelBase.example("Qwen/Qwen3-30B-A3B")
|
||||
class Qwen3MoeModel(Qwen2MoeModel):
|
||||
model_arch = gguf.MODEL_ARCH.QWEN3MOE
|
||||
|
||||
@@ -367,7 +362,6 @@ class _QwenMtpMixin:
|
||||
|
||||
|
||||
@ModelBase.register("Qwen3NextForCausalLM")
|
||||
@ModelBase.example("Qwen/Qwen3-Next-80B-A3B-Instruct")
|
||||
class Qwen3NextModel(_QwenMtpMixin, Qwen2MoeModel):
|
||||
model_arch = gguf.MODEL_ARCH.QWEN3NEXT
|
||||
|
||||
@@ -427,7 +421,6 @@ class Qwen3NextModel(_QwenMtpMixin, Qwen2MoeModel):
|
||||
|
||||
|
||||
@ModelBase.register("RND1")
|
||||
@ModelBase.example("radicalnumerics/RND1-Base-0910")
|
||||
class RND1Model(Qwen2MoeModel):
|
||||
model_arch = gguf.MODEL_ARCH.RND1
|
||||
|
||||
@@ -627,19 +620,16 @@ class _Qwen35MRopeMixin:
|
||||
|
||||
|
||||
@ModelBase.register("Qwen3_5ForConditionalGeneration", "Qwen3_5ForCausalLM")
|
||||
@ModelBase.example("Qwen/Qwen3.5-9B")
|
||||
class Qwen3_5TextModel(_Qwen35MRopeMixin, _LinearAttentionVReorderBase):
|
||||
model_arch = gguf.MODEL_ARCH.QWEN35
|
||||
|
||||
|
||||
@ModelBase.register("Qwen3_5MoeForConditionalGeneration", "Qwen3_5MoeForCausalLM")
|
||||
@ModelBase.example("Qwen/Qwen3.5-35B-A3B")
|
||||
class Qwen3_5MoeTextModel(_Qwen35MRopeMixin, _LinearAttentionVReorderBase):
|
||||
model_arch = gguf.MODEL_ARCH.QWEN35MOE
|
||||
|
||||
|
||||
@ModelBase.register("DFlashDraftModel")
|
||||
@ModelBase.example("z-lab/Qwen3.5-9B-DFlash")
|
||||
class DFlashModel(Qwen3Model):
|
||||
model_arch = gguf.MODEL_ARCH.DFLASH
|
||||
|
||||
@@ -709,7 +699,6 @@ class DFlashModel(Qwen3Model):
|
||||
|
||||
|
||||
@ModelBase.register("Qwen3DSparkModel")
|
||||
@ModelBase.example("satgeze/Qwen3.6-27B-DSpark")
|
||||
class DSparkModel(DFlashModel):
|
||||
# DSpark = DFlash + a semi-autoregressive Markov head
|
||||
model_arch = gguf.MODEL_ARCH.DFLASH
|
||||
|
||||
@@ -37,7 +37,6 @@ _ACT2FN = {
|
||||
|
||||
|
||||
@ModelBase.register("Qwen3TTSForConditionalGeneration")
|
||||
@ModelBase.example("Qwen/Qwen3-TTS-12Hz-1.7B-Base")
|
||||
class Qwen3TTSTalkerModel(TextModel):
|
||||
model_arch = gguf.MODEL_ARCH.QWEN3TTS
|
||||
|
||||
@@ -186,7 +185,6 @@ class Qwen3TTSTalkerModel(TextModel):
|
||||
|
||||
|
||||
@ModelBase.register("Qwen3TTSForConditionalGeneration")
|
||||
@ModelBase.example("Qwen/Qwen3-TTS-12Hz-1.7B-Base")
|
||||
class Qwen3TTSSpeakerEncoderModel(MmprojModel):
|
||||
has_vision_encoder = False
|
||||
has_audio_encoder = True
|
||||
|
||||
@@ -14,7 +14,6 @@ from .qwenvl import Qwen25AudioModel
|
||||
|
||||
|
||||
@ModelBase.register("Qwen3VLForConditionalGeneration", "Qwen3VLMoeForConditionalGeneration", "Qwen3_5ForConditionalGeneration", "Qwen3_5MoeForConditionalGeneration")
|
||||
@ModelBase.example("Qwen/Qwen3-VL-4B-Instruct", "Qwen/Qwen3-VL-30B-A3B-Instruct", "Qwen/Qwen3.5-9B", "Qwen/Qwen3.5-35B-A3B")
|
||||
class Qwen3VLVisionModel(MmprojModel):
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
@@ -145,7 +144,6 @@ class Qwen3VLVisionModel(MmprojModel):
|
||||
|
||||
|
||||
@ModelBase.register("Qwen3OmniMoeForConditionalGeneration")
|
||||
@ModelBase.example("Qwen/Qwen3-Omni-30B-A3B-Instruct")
|
||||
class Qwen3OmniMmprojModel(Qwen3VLVisionModel, Qwen25AudioModel):
|
||||
has_audio_encoder = True
|
||||
has_vision_encoder = True
|
||||
@@ -219,14 +217,12 @@ class Qwen3OmniMmprojModel(Qwen3VLVisionModel, Qwen25AudioModel):
|
||||
|
||||
|
||||
@ModelBase.register("Qwen3ASRForConditionalGeneration")
|
||||
@ModelBase.example("Qwen/Qwen3-ASR-0.6B-hf")
|
||||
class Qwen3ASRMmprojModel(Qwen3OmniMmprojModel):
|
||||
has_audio_encoder = True
|
||||
has_vision_encoder = False
|
||||
|
||||
|
||||
@ModelBase.register("Glm4vForConditionalGeneration", "Glm4vMoeForConditionalGeneration", "GlmOcrForConditionalGeneration")
|
||||
@ModelBase.example("zai-org/GLM-4.1V-9B-Thinking", "zai-org/GLM-4.5V")
|
||||
class Glm4VVisionModel(Qwen3VLVisionModel):
|
||||
def set_gguf_parameters(self):
|
||||
MmprojModel.set_gguf_parameters(self) # skip Qwen3VLVisionModel parameters
|
||||
@@ -250,7 +246,6 @@ class Glm4VVisionModel(Qwen3VLVisionModel):
|
||||
|
||||
|
||||
@ModelBase.register("Qwen3VLForConditionalGeneration")
|
||||
@ModelBase.example("Qwen/Qwen3-VL-4B-Instruct")
|
||||
class Qwen3VLTextModel(Qwen3Model):
|
||||
model_arch = gguf.MODEL_ARCH.QWEN3VL
|
||||
|
||||
@@ -273,7 +268,6 @@ class Qwen3VLTextModel(Qwen3Model):
|
||||
|
||||
|
||||
@ModelBase.register("Qwen3VLMoeForConditionalGeneration")
|
||||
@ModelBase.example("Qwen/Qwen3-VL-30B-A3B-Instruct")
|
||||
class Qwen3VLMoeTextModel(Qwen3MoeModel):
|
||||
model_arch = gguf.MODEL_ARCH.QWEN3VLMOE
|
||||
|
||||
@@ -323,7 +317,6 @@ class Qwen3VLMoeTextModel(Qwen3MoeModel):
|
||||
|
||||
|
||||
@ModelBase.register("Qwen3OmniMoeForConditionalGeneration")
|
||||
@ModelBase.example("Qwen/Qwen3-Omni-30B-A3B-Instruct")
|
||||
class Qwen3OmniMoeTextModel(Qwen3VLMoeTextModel):
|
||||
model_arch = gguf.MODEL_ARCH.QWEN3VLMOE
|
||||
|
||||
@@ -345,7 +338,6 @@ class Qwen3OmniMoeTextModel(Qwen3VLMoeTextModel):
|
||||
|
||||
|
||||
@ModelBase.register("Qwen3ASRForConditionalGeneration")
|
||||
@ModelBase.example("Qwen/Qwen3-ASR-0.6B-hf")
|
||||
class Qwen3ASRTextModel(Qwen3VLTextModel):
|
||||
model_arch = gguf.MODEL_ARCH.QWEN3VL
|
||||
|
||||
|
||||
@@ -17,7 +17,6 @@ from .base import MmprojModel, ModelBase, TextModel, gguf
|
||||
"Qwen2_5_VLForConditionalGeneration",
|
||||
"Qwen2_5OmniModel",
|
||||
)
|
||||
@ModelBase.example("Qwen/Qwen2-VL-2B-Instruct", "Qwen/Qwen2.5-VL-3B-Instruct")
|
||||
class Qwen2VLModel(TextModel):
|
||||
model_arch = gguf.MODEL_ARCH.QWEN2VL
|
||||
|
||||
@@ -41,7 +40,6 @@ class Qwen2VLModel(TextModel):
|
||||
|
||||
|
||||
@ModelBase.register("Qwen2VLModel", "Qwen2VLForConditionalGeneration", "Qwen2_5_VLForConditionalGeneration")
|
||||
@ModelBase.example("Qwen/Qwen2-VL-2B-Instruct", "Qwen/Qwen2.5-VL-3B-Instruct")
|
||||
class Qwen2VLVisionModel(MmprojModel):
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
@@ -163,7 +161,6 @@ class Qwen25AudioModel(MmprojModel):
|
||||
|
||||
|
||||
@ModelBase.register("Qwen2_5OmniModel")
|
||||
@ModelBase.example("Qwen/Qwen2.5-Omni-3B")
|
||||
class Qwen25OmniModel(Qwen2VLVisionModel, Qwen25AudioModel):
|
||||
has_audio_encoder = True
|
||||
has_vision_encoder = True
|
||||
|
||||
@@ -9,7 +9,6 @@ from .base import ModelBase, TextModel, gguf
|
||||
|
||||
|
||||
@ModelBase.register("GPTRefactForCausalLM")
|
||||
@ModelBase.example("smallcloudai/Refact-1_6-base")
|
||||
class RefactModel(TextModel):
|
||||
model_arch = gguf.MODEL_ARCH.REFACT
|
||||
|
||||
|
||||
@@ -11,7 +11,6 @@ from .base import ModelBase, TextModel, gguf
|
||||
|
||||
|
||||
@ModelBase.register("Rwkv6ForCausalLM")
|
||||
@ModelBase.example("RWKV/v6-Finch-1B6-HF")
|
||||
class Rwkv6Model(TextModel):
|
||||
model_arch = gguf.MODEL_ARCH.RWKV6
|
||||
|
||||
@@ -84,7 +83,6 @@ class Rwkv6Model(TextModel):
|
||||
|
||||
|
||||
@ModelBase.register("RWKV6Qwen2ForCausalLM")
|
||||
@ModelBase.example("recursal/QRWKV6-32B-Instruct-Preview-v0.1")
|
||||
class RWKV6Qwen2Model(Rwkv6Model):
|
||||
model_arch = gguf.MODEL_ARCH.RWKV6QWEN2
|
||||
|
||||
@@ -138,7 +136,6 @@ class RWKV6Qwen2Model(Rwkv6Model):
|
||||
|
||||
|
||||
@ModelBase.register("Rwkv7ForCausalLM", "RWKV7ForCausalLM")
|
||||
@ModelBase.example("fla-hub/rwkv7-1.5B-world")
|
||||
class Rwkv7Model(TextModel):
|
||||
model_arch = gguf.MODEL_ARCH.RWKV7
|
||||
|
||||
@@ -264,7 +261,6 @@ class Rwkv7Model(TextModel):
|
||||
|
||||
|
||||
@ModelBase.register("RwkvHybridForCausalLM")
|
||||
@ModelBase.example("RWKV-Red-Team/ARWKV-7B-Preview-0.1")
|
||||
class ARwkv7Model(Rwkv7Model):
|
||||
model_arch = gguf.MODEL_ARCH.ARWKV7
|
||||
|
||||
|
||||
@@ -12,7 +12,6 @@ from .qwenvl import Qwen2VLVisionModel
|
||||
|
||||
|
||||
@ModelBase.register("Sarashina2VisionForCausalLM")
|
||||
@ModelBase.example("sbintuitions/sarashina2.2-vision-3b")
|
||||
class Sarashina2VLTextModel(LlamaModel):
|
||||
model_arch = gguf.MODEL_ARCH.LLAMA
|
||||
|
||||
@@ -27,7 +26,6 @@ class Sarashina2VLTextModel(LlamaModel):
|
||||
|
||||
|
||||
@ModelBase.register("Sarashina2VisionForCausalLM")
|
||||
@ModelBase.example("sbintuitions/sarashina2.2-vision-3b")
|
||||
class Sarashina2VLVisionModel(Qwen2VLVisionModel):
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
|
||||
@@ -11,7 +11,6 @@ from .base import ModelBase, TextModel, gguf, logger
|
||||
|
||||
|
||||
@ModelBase.register("SmallThinkerForCausalLM")
|
||||
@ModelBase.example("PowerInfer/SmallThinker-4BA0.6B-Instruct")
|
||||
class SmallThinkerModel(TextModel):
|
||||
model_arch = gguf.MODEL_ARCH.SMALLTHINKER
|
||||
|
||||
|
||||
@@ -9,7 +9,6 @@ from .base import MmprojModel, ModelBase, gguf
|
||||
|
||||
|
||||
@ModelBase.register("Idefics3ForConditionalGeneration", "SmolVLMForConditionalGeneration")
|
||||
@ModelBase.example("HuggingFaceTB/SmolVLM-Instruct", "HuggingFaceM4/Idefics3-8B-Llama3")
|
||||
class SmolVLMModel(MmprojModel):
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
|
||||
@@ -11,7 +11,6 @@ from .base import ModelBase, TextModel, gguf
|
||||
|
||||
|
||||
@ModelBase.register("StableLmForCausalLM", "StableLMEpochForCausalLM", "LlavaStableLMEpochForCausalLM")
|
||||
@ModelBase.example("stabilityai/stablelm-2-1_6b")
|
||||
class StableLMModel(TextModel):
|
||||
model_arch = gguf.MODEL_ARCH.STABLELM
|
||||
|
||||
|
||||
@@ -4,7 +4,6 @@ from .base import ModelBase, TextModel, gguf
|
||||
|
||||
|
||||
@ModelBase.register("GPTBigCodeForCausalLM")
|
||||
@ModelBase.example("bigcode/gpt_bigcode-santacoder")
|
||||
class StarCoderModel(TextModel):
|
||||
model_arch = gguf.MODEL_ARCH.STARCODER
|
||||
|
||||
@@ -20,6 +19,5 @@ class StarCoderModel(TextModel):
|
||||
|
||||
|
||||
@ModelBase.register("Starcoder2ForCausalLM")
|
||||
@ModelBase.example("bigcode/starcoder2-3b")
|
||||
class StarCoder2Model(TextModel):
|
||||
model_arch = gguf.MODEL_ARCH.STARCODER2
|
||||
|
||||
@@ -16,7 +16,6 @@ from .qwen import Qwen3Model
|
||||
|
||||
|
||||
@ModelBase.register("StepVLForConditionalGeneration", "Step3p7ForConditionalGeneration")
|
||||
@ModelBase.example("stepfun-ai/Step3-VL-10B", "stepfun-ai/Step-3.7-Flash")
|
||||
class Step3VLVisionModel(MmprojModel):
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
@@ -92,13 +91,11 @@ class Step3VLVisionModel(MmprojModel):
|
||||
|
||||
|
||||
@ModelBase.register("StepVLForConditionalGeneration")
|
||||
@ModelBase.example("stepfun-ai/Step3-VL-10B")
|
||||
class Step3VLTextModel(Qwen3Model):
|
||||
model_arch = gguf.MODEL_ARCH.QWEN3
|
||||
|
||||
|
||||
@ModelBase.register("Step3p5ForCausalLM", "Step3p7ForConditionalGeneration")
|
||||
@ModelBase.example("stepfun-ai/Step-3.7-Flash")
|
||||
class Step35Model(TextModel):
|
||||
model_arch = gguf.MODEL_ARCH.STEP35
|
||||
supports_mtp_export = True
|
||||
|
||||
@@ -16,7 +16,6 @@ from .base import ModelBase, SentencePieceTokenTypes, TextModel, gguf, logger
|
||||
@ModelBase.register("MT5ForConditionalGeneration")
|
||||
@ModelBase.register("UMT5ForConditionalGeneration")
|
||||
@ModelBase.register("UMT5Model")
|
||||
@ModelBase.example("google-t5/t5-small", "google/flan-t5-small", "google/umt5-small")
|
||||
class T5Model(TextModel):
|
||||
model_arch = gguf.MODEL_ARCH.T5
|
||||
|
||||
@@ -154,7 +153,6 @@ class T5Model(TextModel):
|
||||
|
||||
|
||||
@ModelBase.register("T5EncoderModel")
|
||||
@ModelBase.example("sentence-transformers/sentence-t5-base")
|
||||
class T5EncoderModel(TextModel):
|
||||
model_arch = gguf.MODEL_ARCH.T5ENCODER
|
||||
|
||||
|
||||
@@ -11,7 +11,6 @@ from .base import LazyTorchTensor, ModelBase, TextModel, gguf
|
||||
|
||||
|
||||
@ModelBase.register("TalkieForCausalLM")
|
||||
@ModelBase.example("lewtun/talkie-1930-13b-it-hf")
|
||||
class TalkieModel(TextModel):
|
||||
model_arch = gguf.MODEL_ARCH.TALKIE
|
||||
|
||||
|
||||
@@ -9,7 +9,6 @@ from .base import MmprojModel, ModelBase, TextModel, gguf
|
||||
|
||||
|
||||
@ModelBase.register("UltravoxModel")
|
||||
@ModelBase.example("fixie-ai/ultravox-v0_5-llama-3_2-1b")
|
||||
class UltravoxModel(TextModel):
|
||||
model_arch = gguf.MODEL_ARCH.LLAMA # dummy
|
||||
|
||||
@@ -19,7 +18,6 @@ class UltravoxModel(TextModel):
|
||||
|
||||
|
||||
@ModelBase.register("GlmasrModel")
|
||||
@ModelBase.example("zai-org/GLM-ASR-Nano-2512")
|
||||
class GlmASRWhisperEncoderModel(MmprojModel):
|
||||
has_vision_encoder = False
|
||||
has_audio_encoder = True
|
||||
@@ -84,7 +82,6 @@ class GlmASRWhisperEncoderModel(MmprojModel):
|
||||
|
||||
|
||||
@ModelBase.register("Qwen2AudioForConditionalGeneration")
|
||||
@ModelBase.example("Qwen/Qwen2-Audio-7B-Instruct")
|
||||
class WhisperEncoderModel(MmprojModel):
|
||||
has_vision_encoder = False # no vision encoder
|
||||
has_audio_encoder = True
|
||||
@@ -126,7 +123,6 @@ class WhisperEncoderModel(MmprojModel):
|
||||
|
||||
|
||||
@ModelBase.register("UltravoxModel")
|
||||
@ModelBase.example("fixie-ai/ultravox-v0_5-llama-3_2-1b")
|
||||
class UltravoxWhisperEncoderModel(WhisperEncoderModel):
|
||||
has_vision_encoder = False # no vision encoder
|
||||
has_audio_encoder = True
|
||||
@@ -138,7 +134,6 @@ class UltravoxWhisperEncoderModel(WhisperEncoderModel):
|
||||
|
||||
|
||||
@ModelBase.register("MERaLiON2ForConditionalGeneration")
|
||||
@ModelBase.example("MERaLiON/MERaLiON-2-3B")
|
||||
class MERaLiONWhisperEncoderModel(WhisperEncoderModel):
|
||||
has_vision_encoder = False
|
||||
has_audio_encoder = True
|
||||
@@ -185,7 +180,6 @@ class MERaLiONWhisperEncoderModel(WhisperEncoderModel):
|
||||
|
||||
|
||||
@ModelBase.register("VoxtralForConditionalGeneration")
|
||||
@ModelBase.example("mistralai/Voxtral-Mini-3B-2507")
|
||||
class VoxtralWhisperEncoderModel(WhisperEncoderModel):
|
||||
has_vision_encoder = False # no vision encoder
|
||||
has_audio_encoder = True
|
||||
@@ -197,7 +191,6 @@ class VoxtralWhisperEncoderModel(WhisperEncoderModel):
|
||||
|
||||
|
||||
@ModelBase.register("AudioFlamingo3ForConditionalGeneration")
|
||||
@ModelBase.example("nvidia/audio-flamingo-3-hf")
|
||||
class AudioFlamingo3WhisperEncoderModel(WhisperEncoderModel):
|
||||
def set_gguf_parameters(self):
|
||||
super().set_gguf_parameters()
|
||||
|
||||
@@ -9,7 +9,6 @@ from .base import ModelBase, TextModel, gguf, logger
|
||||
|
||||
|
||||
@ModelBase.register("WavTokenizerDec")
|
||||
@ModelBase.example("novateur/WavTokenizer-large-speech-75token")
|
||||
class WavTokenizerDecModel(TextModel):
|
||||
model_arch = gguf.MODEL_ARCH.WAVTOKENIZER_DEC
|
||||
|
||||
|
||||
@@ -11,7 +11,6 @@ from .base import ModelBase, TextModel, gguf
|
||||
|
||||
|
||||
@ModelBase.register("XverseForCausalLM")
|
||||
@ModelBase.example("xverse/XVERSE-7B")
|
||||
class XverseModel(TextModel):
|
||||
model_arch = gguf.MODEL_ARCH.XVERSE
|
||||
|
||||
|
||||
@@ -9,7 +9,6 @@ from .base import MmprojModel, ModelBase, gguf, logger
|
||||
|
||||
|
||||
@ModelBase.register("YoutuVLForConditionalGeneration")
|
||||
@ModelBase.example("tencent/Youtu-VL-4B-Instruct")
|
||||
class YoutuVLVisionModel(MmprojModel):
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
|
||||
@@ -29,7 +29,6 @@ The required steps to implement for an HF model are:
|
||||
|
||||
```python
|
||||
@ModelBase.register("MyModelForCausalLM")
|
||||
@ModelBase.example("user/model")
|
||||
class MyModel(TextModel):
|
||||
model_arch = gguf.MODEL_ARCH.MYMODEL
|
||||
```
|
||||
@@ -38,13 +37,10 @@ or
|
||||
|
||||
```python
|
||||
@ModelBase.register("MyModelForConditionalGeneration")
|
||||
@ModelBase.example("user/model")
|
||||
class MyModel(MmprojModel):
|
||||
model_arch = gguf.MODEL_ARCH.MYMODEL
|
||||
```
|
||||
|
||||
The `example` should point to a valid Hugging Face model that will be used for testing. You can add multiple models if necessary. Prefer a non-gated model, or tiny random weights if no such model exists.
|
||||
|
||||
2. Define the layout of the GGUF tensors in [constants.py](/gguf-py/gguf/constants.py)
|
||||
|
||||
Add an enum entry in `MODEL_ARCH`, the model human friendly name in `MODEL_ARCH_NAMES` and the GGUF tensor names in `MODEL_TENSORS`.
|
||||
|
||||
@@ -261,7 +261,6 @@ class Keys:
|
||||
|
||||
class KDA:
|
||||
HEAD_DIM = "{arch}.kda.head_dim"
|
||||
SAFE_GATE = "{arch}.kda.safe_gate"
|
||||
GATE_LOWER_BOUND = "{arch}.kda.gate_lower_bound"
|
||||
|
||||
class WKV:
|
||||
@@ -553,7 +552,6 @@ class MODEL_ARCH(IntEnum):
|
||||
PLM = auto()
|
||||
BAILINGMOE = auto()
|
||||
BAILINGMOE2 = auto()
|
||||
BAILINGMOE3 = auto()
|
||||
DOTS1 = auto()
|
||||
ARCEE = auto()
|
||||
AFMOE = auto()
|
||||
@@ -1269,7 +1267,6 @@ MODEL_ARCH_NAMES: dict[MODEL_ARCH, str] = {
|
||||
MODEL_ARCH.PLM: "plm",
|
||||
MODEL_ARCH.BAILINGMOE: "bailingmoe",
|
||||
MODEL_ARCH.BAILINGMOE2: "bailingmoe2",
|
||||
MODEL_ARCH.BAILINGMOE3: "bailingmoe3",
|
||||
MODEL_ARCH.DOTS1: "dots1",
|
||||
MODEL_ARCH.ARCEE: "arcee",
|
||||
MODEL_ARCH.AFMOE: "afmoe",
|
||||
@@ -4237,50 +4234,6 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = {
|
||||
MODEL_TENSOR.NEXTN_SHARED_HEAD_NORM,
|
||||
MODEL_TENSOR.LAYER_OUT_NORM,
|
||||
],
|
||||
MODEL_ARCH.BAILINGMOE3: [
|
||||
MODEL_TENSOR.TOKEN_EMBD,
|
||||
MODEL_TENSOR.OUTPUT_NORM,
|
||||
MODEL_TENSOR.OUTPUT,
|
||||
MODEL_TENSOR.ATTN_NORM,
|
||||
MODEL_TENSOR.ATTN_Q,
|
||||
MODEL_TENSOR.ATTN_Q_A,
|
||||
MODEL_TENSOR.ATTN_Q_B,
|
||||
MODEL_TENSOR.ATTN_Q_A_NORM,
|
||||
MODEL_TENSOR.ATTN_K,
|
||||
MODEL_TENSOR.ATTN_V,
|
||||
MODEL_TENSOR.ATTN_OUT,
|
||||
MODEL_TENSOR.ATTN_GATE,
|
||||
MODEL_TENSOR.ATTN_KV_A_MQA,
|
||||
MODEL_TENSOR.ATTN_KV_B,
|
||||
MODEL_TENSOR.ATTN_K_B,
|
||||
MODEL_TENSOR.ATTN_V_B,
|
||||
MODEL_TENSOR.ATTN_KV_A_NORM,
|
||||
MODEL_TENSOR.FFN_NORM,
|
||||
MODEL_TENSOR.FFN_GATE,
|
||||
MODEL_TENSOR.FFN_DOWN,
|
||||
MODEL_TENSOR.FFN_UP,
|
||||
MODEL_TENSOR.FFN_GATE_INP,
|
||||
MODEL_TENSOR.FFN_GATE_EXP,
|
||||
MODEL_TENSOR.FFN_DOWN_EXP,
|
||||
MODEL_TENSOR.FFN_UP_EXP,
|
||||
MODEL_TENSOR.FFN_GATE_SHEXP,
|
||||
MODEL_TENSOR.FFN_DOWN_SHEXP,
|
||||
MODEL_TENSOR.FFN_UP_SHEXP,
|
||||
MODEL_TENSOR.FFN_EXP_PROBS_B,
|
||||
MODEL_TENSOR.SSM_CONV1D_Q,
|
||||
MODEL_TENSOR.SSM_CONV1D_K,
|
||||
MODEL_TENSOR.SSM_CONV1D_V,
|
||||
MODEL_TENSOR.SSM_F_A,
|
||||
MODEL_TENSOR.SSM_BETA,
|
||||
MODEL_TENSOR.SSM_A,
|
||||
MODEL_TENSOR.SSM_G_A,
|
||||
MODEL_TENSOR.SSM_DT,
|
||||
MODEL_TENSOR.SSM_NORM,
|
||||
MODEL_TENSOR.NEXTN_EH_PROJ,
|
||||
MODEL_TENSOR.NEXTN_ENORM,
|
||||
MODEL_TENSOR.NEXTN_HNORM,
|
||||
MODEL_TENSOR.LAYER_OUT_NORM,
|
||||
],
|
||||
MODEL_ARCH.DOTS1: [
|
||||
MODEL_TENSOR.TOKEN_EMBD,
|
||||
MODEL_TENSOR.OUTPUT_NORM,
|
||||
@@ -5534,9 +5487,7 @@ KEY_SSM_GROUP_COUNT = Keys.SSM.GROUP_COUNT
|
||||
KEY_SSM_DT_B_C_RMS = Keys.SSM.DT_B_C_RMS
|
||||
|
||||
# KDA
|
||||
KEY_KDA_HEAD_DIM = Keys.KDA.HEAD_DIM
|
||||
KEY_KDA_SAFE_GATE = Keys.KDA.SAFE_GATE
|
||||
KEY_KDA_GATE_LOWER_BOUND = Keys.KDA.GATE_LOWER_BOUND
|
||||
KEY_KDA_HEAD_DIM = Keys.KDA.HEAD_DIM
|
||||
|
||||
# tokenization
|
||||
KEY_TOKENIZER_MODEL = Keys.Tokenizer.MODEL
|
||||
|
||||
@@ -1103,6 +1103,9 @@ class GGUFWriter:
|
||||
def add_ssm_dt_b_c_rms(self, value: bool) -> None:
|
||||
self.add_bool(Keys.SSM.DT_B_C_RMS.format(arch=self.arch), value)
|
||||
|
||||
def add_kda_gate_lower_bound(self, value: float) -> None:
|
||||
self.add_float32(Keys.KDA.GATE_LOWER_BOUND.format(arch=self.arch), value)
|
||||
|
||||
def add_expert_latent_length(self, value: int) -> None:
|
||||
self.add_uint32(Keys.LLM.EXPERT_LATENT_LENGTH.format(arch=self.arch), value)
|
||||
|
||||
@@ -1118,12 +1121,6 @@ class GGUFWriter:
|
||||
def add_kda_head_dim(self, value: int) -> None:
|
||||
self.add_uint32(Keys.KDA.HEAD_DIM.format(arch=self.arch), value)
|
||||
|
||||
def add_kda_safe_gate(self, value: bool) -> None:
|
||||
self.add_bool(Keys.KDA.SAFE_GATE.format(arch=self.arch), value)
|
||||
|
||||
def add_kda_gate_lower_bound(self, value: float) -> None:
|
||||
self.add_float32(Keys.KDA.GATE_LOWER_BOUND.format(arch=self.arch), value)
|
||||
|
||||
def add_tokenizer_model(self, model: str) -> None:
|
||||
self.add_string(Keys.Tokenizer.MODEL, model)
|
||||
|
||||
|
||||
@@ -255,7 +255,6 @@ class TensorNameMap:
|
||||
# Attention query
|
||||
MODEL_TENSOR.ATTN_Q: (
|
||||
"model.layers.{bid}.self_attn.q_proj", # llama-hf nemotron olmoe olmo2 phimoe
|
||||
"model.layers.{bid}.attention.q_proj", # bailingmoe3
|
||||
"layers.{bid}.self_attn.q_proj", # embeddinggemma
|
||||
"model.layers.{bid}.self_attn.q_proj_no_perm", # llama-custom
|
||||
"layers.{bid}.attention.wq", # llama-pth
|
||||
@@ -276,7 +275,6 @@ class TensorNameMap:
|
||||
# Attention key
|
||||
MODEL_TENSOR.ATTN_K: (
|
||||
"model.layers.{bid}.self_attn.k_proj", # llama-hf nemotron olmoe olmo2 phimoe
|
||||
"model.layers.{bid}.attention.k_proj", # bailingmoe3
|
||||
"layers.{bid}.self_attn.k_proj", # embeddinggemma
|
||||
"model.layers.{bid}.self_attn.k_proj_no_perm", # llama-custom
|
||||
"layers.{bid}.attention.wk", # llama-pth
|
||||
@@ -298,7 +296,6 @@ class TensorNameMap:
|
||||
# Attention value
|
||||
MODEL_TENSOR.ATTN_V: (
|
||||
"model.layers.{bid}.self_attn.v_proj", # llama-hf nemotron olmoe olmo2 phimoe
|
||||
"model.layers.{bid}.attention.v_proj", # bailingmoe3
|
||||
"layers.{bid}.self_attn.v_proj", # embeddinggemma
|
||||
"layers.{bid}.attention.wv", # llama-pth
|
||||
"encoder.layer.{bid}.attention.self.value", # bert
|
||||
@@ -324,8 +321,6 @@ class TensorNameMap:
|
||||
"transformer.h.{bid}.self_attention.dense", # falcon
|
||||
"h.{bid}.self_attention.dense", # bloom
|
||||
"model.layers.{bid}.self_attn.o_proj", # llama-hf nemotron olmoe olmo2 phimoe
|
||||
"model.layers.{bid}.attention.o_proj", # bailingmoe3
|
||||
"model.layers.{bid}.attention.dense", # bailingmoe3 MLA
|
||||
"layers.{bid}.self_attn.o_proj", # embeddinggemma
|
||||
"model.layers.{bid}.self_attn.out_proj", # lfm2 minimax-01
|
||||
"model.layers.{bid}.self_attn.linear_attn", # deci
|
||||
@@ -839,7 +834,6 @@ class TensorNameMap:
|
||||
"model.layers.{bid}.linear_attn.dt_proj", # qwen3next
|
||||
"backbone.layers.{bid}.mixer.dt", # nemotron-h-moe
|
||||
"model.layers.{bid}.self_attn.dt_proj", # kimi
|
||||
"model.layers.{bid}.attention.dt_proj", # bailingmoe3
|
||||
),
|
||||
|
||||
MODEL_TENSOR.SSM_DT_NORM: (
|
||||
@@ -854,7 +848,6 @@ class TensorNameMap:
|
||||
"model.layers.layers.{bid}.mixer.A_log", # plamo2
|
||||
"model.layers.{bid}.linear_attn.A_log", # qwen3next
|
||||
"model.layers.{bid}.self_attn.A_log", # kimi
|
||||
"model.layers.{bid}.attention.A_log", # bailingmoe3
|
||||
),
|
||||
|
||||
MODEL_TENSOR.SSM_B_NORM: (
|
||||
@@ -881,7 +874,6 @@ class TensorNameMap:
|
||||
"model.layers.{bid}.linear_attn.norm", # qwen3next
|
||||
"backbone.layers.{bid}.mixer.norm", # mamba2
|
||||
"model.layers.{bid}.self_attn.o_norm", # kimi
|
||||
"model.layers.{bid}.attention.o_norm", # bailingmoe3
|
||||
),
|
||||
|
||||
MODEL_TENSOR.SSM_OUT: (
|
||||
@@ -903,15 +895,12 @@ class TensorNameMap:
|
||||
# Kimi Linear KDA (using SSM_ prefix for consistency)
|
||||
MODEL_TENSOR.SSM_CONV1D_Q: (
|
||||
"model.layers.{bid}.self_attn.q_conv1d",
|
||||
"model.layers.{bid}.attention.q_conv1d",
|
||||
),
|
||||
MODEL_TENSOR.SSM_CONV1D_K: (
|
||||
"model.layers.{bid}.self_attn.k_conv1d",
|
||||
"model.layers.{bid}.attention.k_conv1d",
|
||||
),
|
||||
MODEL_TENSOR.SSM_CONV1D_V: (
|
||||
"model.layers.{bid}.self_attn.v_conv1d",
|
||||
"model.layers.{bid}.attention.v_conv1d",
|
||||
),
|
||||
MODEL_TENSOR.SSM_F_A: (
|
||||
"model.layers.{bid}.self_attn.f_a_proj",
|
||||
@@ -922,7 +911,6 @@ class TensorNameMap:
|
||||
MODEL_TENSOR.SSM_BETA: (
|
||||
"model.layers.{bid}.linear_attn.in_proj_b", # qwen3.5
|
||||
"model.layers.{bid}.self_attn.b_proj", # Kimi Linear
|
||||
"model.layers.{bid}.attention.b_proj", # bailingmoe3
|
||||
),
|
||||
# Kimi K3 latent MoE: routed experts operate in a down-projected space
|
||||
MODEL_TENSOR.FFN_ROUTED_DOWN: (
|
||||
@@ -1115,48 +1103,40 @@ class TensorNameMap:
|
||||
|
||||
MODEL_TENSOR.ATTN_Q_A: (
|
||||
"model.layers.{bid}.self_attn.q_a_proj", # deepseek2
|
||||
"model.layers.{bid}.attention.q_a_proj", # bailingmoe3 (Ling-3.0-tiny)
|
||||
"layers.{bid}.attention.wq_a", # mistral-large
|
||||
),
|
||||
|
||||
MODEL_TENSOR.ATTN_Q_B: (
|
||||
"model.layers.{bid}.self_attn.q_b_proj", # deepseek2
|
||||
"model.layers.{bid}.attention.q_b_proj", # bailingmoe3 (Ling-3.0-tiny)
|
||||
"layers.{bid}.attention.wq_b", # mistral-large
|
||||
),
|
||||
|
||||
MODEL_TENSOR.ATTN_KV_A_MQA: (
|
||||
"model.layers.{bid}.self_attn.kv_a_proj_with_mqa", # deepseek2
|
||||
"model.layers.{bid}.attention.kv_a_proj_with_mqa", # bailingmoe3
|
||||
"layers.{bid}.attention.wkv_a_with_mqa", # mistral-large
|
||||
),
|
||||
|
||||
MODEL_TENSOR.ATTN_KV_B: (
|
||||
"model.layers.{bid}.self_attn.kv_b_proj", # deepseek2
|
||||
"model.layers.{bid}.attention.kv_b_proj", # bailingmoe3
|
||||
),
|
||||
|
||||
MODEL_TENSOR.ATTN_K_B: (
|
||||
"model.layers.{bid}.self_attn.k_b_proj", # deepseek2
|
||||
"model.layers.{bid}.attention.k_b_proj", # bailingmoe3
|
||||
"layers.{bid}.attention.k_b_proj", # mistral-large
|
||||
),
|
||||
|
||||
MODEL_TENSOR.ATTN_V_B: (
|
||||
"model.layers.{bid}.self_attn.v_b_proj", # deepseek2
|
||||
"model.layers.{bid}.attention.v_b_proj", # bailingmoe3
|
||||
"layers.{bid}.attention.v_b_proj", # mistral-large
|
||||
),
|
||||
|
||||
MODEL_TENSOR.ATTN_Q_A_NORM: (
|
||||
"model.layers.{bid}.self_attn.q_a_layernorm", # deepseek2
|
||||
"model.layers.{bid}.attention.q_a_layernorm", # bailingmoe3 (Ling-3.0-tiny)
|
||||
"layers.{bid}.attention.q_a_norm", # mistral-large
|
||||
),
|
||||
|
||||
MODEL_TENSOR.ATTN_KV_A_NORM: (
|
||||
"model.layers.{bid}.self_attn.kv_a_layernorm", # deepseek2
|
||||
"model.layers.{bid}.attention.kv_a_layernorm", # bailingmoe3
|
||||
"layers.{bid}.attention.kv_a_norm", # mistral-large
|
||||
),
|
||||
|
||||
|
||||
+1
-5
@@ -107,7 +107,6 @@ static const std::map<llm_arch, const char *> LLM_ARCH_NAMES = {
|
||||
{ LLM_ARCH_PLM, "plm" },
|
||||
{ LLM_ARCH_BAILINGMOE, "bailingmoe" },
|
||||
{ LLM_ARCH_BAILINGMOE2, "bailingmoe2" },
|
||||
{ LLM_ARCH_BAILINGMOE3, "bailingmoe3" },
|
||||
{ LLM_ARCH_DOTS1, "dots1" },
|
||||
{ LLM_ARCH_ARCEE, "arcee" },
|
||||
{ LLM_ARCH_AFMOE, "afmoe" },
|
||||
@@ -318,8 +317,7 @@ static const std::map<llm_kv, const char *> LLM_KV_NAMES = {
|
||||
{ LLM_KV_SSM_GROUP_COUNT, "%s.ssm.group_count" },
|
||||
{ LLM_KV_SSM_DT_B_C_RMS, "%s.ssm.dt_b_c_rms" },
|
||||
|
||||
{ LLM_KV_KDA_HEAD_DIM, "%s.kda.head_dim" },
|
||||
{ LLM_KV_KDA_SAFE_GATE, "%s.kda.safe_gate" },
|
||||
{ LLM_KV_KDA_HEAD_DIM, "%s.kda.head_dim" },
|
||||
{ LLM_KV_KDA_GATE_LOWER_BOUND, "%s.kda.gate_lower_bound" },
|
||||
|
||||
{ LLM_KV_WKV_HEAD_SIZE, "%s.wkv.head_size" },
|
||||
@@ -998,7 +996,6 @@ bool llm_arch_is_hybrid(const llm_arch & arch) {
|
||||
case LLM_ARCH_NEMOTRON_H_MOE:
|
||||
case LLM_ARCH_QWEN3NEXT:
|
||||
case LLM_ARCH_KIMI_LINEAR:
|
||||
case LLM_ARCH_BAILINGMOE3:
|
||||
case LLM_ARCH_KIMI_K3:
|
||||
case LLM_ARCH_QWEN35:
|
||||
case LLM_ARCH_QWEN35MOE:
|
||||
@@ -1064,7 +1061,6 @@ bool llm_arch_supports_sm_tensor(const llm_arch & arch) {
|
||||
case LLM_ARCH_MINIMAX_M3:
|
||||
case LLM_ARCH_MISTRAL4:
|
||||
case LLM_ARCH_KIMI_LINEAR:
|
||||
case LLM_ARCH_BAILINGMOE3:
|
||||
case LLM_ARCH_KIMI_K3:
|
||||
case LLM_ARCH_QWEN3TTS:
|
||||
return false;
|
||||
|
||||
@@ -112,7 +112,6 @@ enum llm_arch {
|
||||
LLM_ARCH_PLM,
|
||||
LLM_ARCH_BAILINGMOE,
|
||||
LLM_ARCH_BAILINGMOE2,
|
||||
LLM_ARCH_BAILINGMOE3,
|
||||
LLM_ARCH_DOTS1,
|
||||
LLM_ARCH_ARCEE,
|
||||
LLM_ARCH_AFMOE,
|
||||
@@ -324,7 +323,6 @@ enum llm_kv {
|
||||
LLM_KV_SSM_DT_B_C_RMS,
|
||||
|
||||
LLM_KV_KDA_HEAD_DIM,
|
||||
LLM_KV_KDA_SAFE_GATE,
|
||||
LLM_KV_KDA_GATE_LOWER_BOUND,
|
||||
|
||||
LLM_KV_WKV_HEAD_SIZE,
|
||||
|
||||
@@ -2298,7 +2298,6 @@ uint32_t llama_context::graph_max_nodes(uint32_t n_tokens) const {
|
||||
res = std::max<uint32_t>(n_tokens * 160, 64u * model.n_tensors());
|
||||
} else if (model.arch == LLM_ARCH_QWEN3NEXT ||
|
||||
model.arch == LLM_ARCH_KIMI_LINEAR ||
|
||||
model.arch == LLM_ARCH_BAILINGMOE3 ||
|
||||
model.arch == LLM_ARCH_QWEN35 ||
|
||||
model.arch == LLM_ARCH_QWEN35MOE ||
|
||||
model.arch == LLM_ARCH_DEEPSEEK4 ||
|
||||
|
||||
@@ -170,7 +170,6 @@ struct llama_hparams {
|
||||
|
||||
// for Kimi Linear KDA
|
||||
uint32_t n_embd_head_kda = 0;
|
||||
bool kda_safe_gate = false;
|
||||
|
||||
// kimi-k3
|
||||
uint32_t n_expert_latent = 0; // routed_expert_hidden_size (0 = experts run at n_embd)
|
||||
|
||||
@@ -121,7 +121,6 @@ void llama_model_saver::add_kv(const enum llm_kv key, const Container & value, c
|
||||
}
|
||||
// instantiate for external usage:
|
||||
template void llama_model_saver::add_kv<std::vector<uint32_t>>(const enum llm_kv, const std::vector<uint32_t> &, const bool);
|
||||
template void llama_model_saver::add_kv<std::vector<float>>(const enum llm_kv, const std::vector<float> &, const bool);
|
||||
|
||||
void llama_model_saver::add_kv(const enum llm_kv key, const std::vector<std::string> & value) {
|
||||
std::vector<const char *> tmp(value.size());
|
||||
@@ -217,10 +216,8 @@ void llama_model_saver::add_kv_from_model() {
|
||||
add_kv(LLM_KV_EXPERT_LATENT_LENGTH, hparams.n_expert_latent);
|
||||
add_kv(LLM_KV_EXPERT_SHARED_FEED_FORWARD_LENGTH, hparams.n_ff_shexp);
|
||||
add_kv(LLM_KV_EXPERT_CHUNK_FEED_FORWARD_LENGTH, hparams.n_ff_chexp);
|
||||
add_kv(LLM_KV_SWIGLU_CLAMP_EXP, std::vector<float>(
|
||||
hparams.swiglu_clamp_exp.begin(), hparams.swiglu_clamp_exp.begin() + hparams.n_layer_all));
|
||||
add_kv(LLM_KV_SWIGLU_CLAMP_SHEXP, std::vector<float>(
|
||||
hparams.swiglu_clamp_shexp.begin(), hparams.swiglu_clamp_shexp.begin() + hparams.n_layer_all));
|
||||
add_kv(LLM_KV_SWIGLU_CLAMP_EXP, hparams.swiglu_clamp_exp);
|
||||
add_kv(LLM_KV_SWIGLU_CLAMP_SHEXP, hparams.swiglu_clamp_shexp);
|
||||
add_kv(LLM_KV_USE_PARALLEL_RESIDUAL, hparams.use_par_res);
|
||||
// add_kv(LLM_KV_TENSOR_DATA_LAYOUT, ???);
|
||||
add_kv(LLM_KV_EXPERT_COUNT, hparams.n_expert);
|
||||
@@ -323,7 +320,6 @@ void llama_model_saver::add_kv_from_model() {
|
||||
add_kv(LLM_KV_SSM_DT_B_C_RMS, hparams.ssm_dt_b_c_rms);
|
||||
|
||||
add_kv(LLM_KV_KDA_HEAD_DIM, hparams.n_embd_head_kda);
|
||||
add_kv(LLM_KV_KDA_SAFE_GATE, hparams.kda_safe_gate);
|
||||
add_kv(LLM_KV_KDA_GATE_LOWER_BOUND, hparams.kda_gate_lower_bound);
|
||||
|
||||
add_kv(LLM_KV_WKV_HEAD_SIZE, hparams.wkv_head_size);
|
||||
|
||||
+4
-9
@@ -256,8 +256,6 @@ static llama_model * llama_model_mapping(llm_arch arch, const llama_model_params
|
||||
return new llama_model_bailingmoe(params);
|
||||
case LLM_ARCH_BAILINGMOE2:
|
||||
return new llama_model_bailingmoe2(params);
|
||||
case LLM_ARCH_BAILINGMOE3:
|
||||
return new llama_model_bailingmoe3(params);
|
||||
case LLM_ARCH_SEED_OSS:
|
||||
return new llama_model_seed_oss(params);
|
||||
case LLM_ARCH_DOTS1:
|
||||
@@ -823,7 +821,6 @@ const char * llm_type_name(llm_type type) {
|
||||
case LLM_TYPE_A13B: return "A13B";
|
||||
case LLM_TYPE_7B_A1B: return "7B.A1B";
|
||||
case LLM_TYPE_8B_A1B: return "8B.A1B";
|
||||
case LLM_TYPE_7_9B_A1_3B: return "7.9B.A1.3B";
|
||||
case LLM_TYPE_12B_A2_5B: return "12B.A2.5B";
|
||||
case LLM_TYPE_16B_A1B: return "16B.A1B";
|
||||
case LLM_TYPE_21B_A3B: return "21B.A3B";
|
||||
@@ -840,7 +837,6 @@ const char * llm_type_name(llm_type type) {
|
||||
case LLM_TYPE_118B_A8B: return "118B.A8B";
|
||||
case LLM_TYPE_120B_A12B: return "120B.A12B";
|
||||
case LLM_TYPE_122B_A10B: return "122B.A10B";
|
||||
case LLM_TYPE_124B_A5_1B: return "124B.A5.1B";
|
||||
case LLM_TYPE_196B_A11B: return "196B.A11B";
|
||||
case LLM_TYPE_230B_A10B: return "230B.A10B";
|
||||
case LLM_TYPE_428B_A23B: return "428B.A23B";
|
||||
@@ -1964,7 +1960,7 @@ void llama_model::print_info() const {
|
||||
LLAMA_LOG_INFO("%s: expert_weights_norm = %d\n", __func__, hparams.expert_weights_norm);
|
||||
}
|
||||
|
||||
if (arch == LLM_ARCH_BAILINGMOE2 || arch == LLM_ARCH_BAILINGMOE3) {
|
||||
if (arch == LLM_ARCH_BAILINGMOE2) {
|
||||
LLAMA_LOG_INFO("%s: n_layer_dense_lead = %d\n", __func__, hparams.n_layer_dense_lead);
|
||||
LLAMA_LOG_INFO("%s: n_ff_exp = %d\n", __func__, hparams.n_ff_exp);
|
||||
LLAMA_LOG_INFO("%s: n_ff_shexp = %d\n", __func__, hparams.n_ff_shexp);
|
||||
@@ -2259,11 +2255,11 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params,
|
||||
// checks
|
||||
default:
|
||||
{
|
||||
// Dense MTP heads use a plain attention KV cache instead of the hybrid wrapper.
|
||||
// The MTP head is dense-attention only on hybrid Qwen3-Next/3.5/3.6, so use a plain
|
||||
// attention KV cache for the MTP context instead of the hybrid wrapper.
|
||||
const bool mtp_on_hybrid_qwen =
|
||||
params.ctx_type == LLAMA_CONTEXT_TYPE_MTP &&
|
||||
(arch == LLM_ARCH_QWEN3NEXT || arch == LLM_ARCH_QWEN35 || arch == LLM_ARCH_QWEN35MOE ||
|
||||
arch == LLM_ARCH_BAILINGMOE3);
|
||||
(arch == LLM_ARCH_QWEN3NEXT || arch == LLM_ARCH_QWEN35 || arch == LLM_ARCH_QWEN35MOE);
|
||||
|
||||
const bool mtp_on_hybrid_nemotron =
|
||||
params.ctx_type == LLAMA_CONTEXT_TYPE_MTP && arch == LLM_ARCH_NEMOTRON_H_MOE;
|
||||
@@ -2641,7 +2637,6 @@ llama_rope_type llama_model_rope_type(const llama_model * model) {
|
||||
case LLM_ARCH_GRANITE_SWITCH:
|
||||
case LLM_ARCH_CHAMELEON:
|
||||
case LLM_ARCH_BAILINGMOE:
|
||||
case LLM_ARCH_BAILINGMOE3:
|
||||
case LLM_ARCH_NEO_BERT:
|
||||
case LLM_ARCH_SMOLLM3:
|
||||
case LLM_ARCH_ARCEE:
|
||||
|
||||
@@ -118,7 +118,6 @@ enum llm_type {
|
||||
LLM_TYPE_A13B,
|
||||
LLM_TYPE_7B_A1B,
|
||||
LLM_TYPE_8B_A1B, // lfm2moe
|
||||
LLM_TYPE_7_9B_A1_3B, // Ling-3.0-tiny
|
||||
LLM_TYPE_12B_A2_5B,
|
||||
LLM_TYPE_16B_A1B,
|
||||
LLM_TYPE_21B_A3B, // Ernie MoE small
|
||||
@@ -135,7 +134,6 @@ enum llm_type {
|
||||
LLM_TYPE_118B_A8B, // Laguna-S-2
|
||||
LLM_TYPE_120B_A12B, // Nemotron 3 Super
|
||||
LLM_TYPE_122B_A10B, // Qwen3.5
|
||||
LLM_TYPE_124B_A5_1B, // Ling-3.0-flash
|
||||
LLM_TYPE_196B_A11B, // Step3.5-Flash
|
||||
LLM_TYPE_230B_A10B, // Minimax M2
|
||||
LLM_TYPE_428B_A23B, // Minimax M3
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user