Compare commits

..
Author SHA1 Message Date
Georgi Gerganov 04a134c70b ci : make release workflows use a deply key 2026-08-17 09:58:11 +03:00
105 changed files with 17 additions and 1174 deletions
+2
View File
@@ -22,6 +22,8 @@ jobs:
steps:
- name: Checkout
uses: actions/checkout@v6
with:
ssh-key: ${{ secrets.DEPLOY_KEY_RELEASE }}
- name: Run release checks
id: checks
+1
View File
@@ -1598,6 +1598,7 @@ jobs:
uses: actions/checkout@v6
with:
fetch-depth: 0
ssh-key: ${{ secrets.DEPLOY_KEY_RELEASE }}
- name: Determine tag name
id: tag
-8
View File
@@ -193,14 +193,6 @@ static std::vector<std::function<void(const common_chat_template & tmpl, autopar
LOG_DBG(ANSI_ORANGE "[Patch: Laguna]\n" ANSI_RESET);
}
},
// Bailing V3
[](const common_chat_template & tmpl, autoparser & analysis) -> void {
if (tmpl.src.find("Bailing V3 chat template") != std::string::npos) {
analysis.tools.arguments.value_suffix = trim_whitespace(analysis.tools.arguments.value_suffix);
analysis.tools.arguments.tolerate_intertag_whitespace = true;
LOG_DBG(ANSI_ORANGE "[Patch: Bailing V3]\n" ANSI_RESET);
}
},
});
-1
View File
@@ -27,7 +27,6 @@ TEXT_MODEL_MAP: dict[str, str] = {
"BaichuanForCausalLM": "baichuan",
"BailingMoeForCausalLM": "bailingmoe",
"BailingMoeV2ForCausalLM": "bailingmoe",
"BailingMoeV3ForCausalLM": "bailingmoe3",
"BambaForCausalLM": "granite",
"BertForMaskedLM": "bert",
"BertForSequenceClassification": "bert",
-1
View File
@@ -13,7 +13,6 @@ from .llama import LlamaModel
@ModelBase.register("AfmoeForCausalLM")
@ModelBase.example("arcee-ai/Trinity-Large-Thinking")
class AfmoeModel(LlamaModel):
model_arch = gguf.MODEL_ARCH.AFMOE
-1
View File
@@ -16,7 +16,6 @@ from .llama import LlamaModel
@ModelBase.register("ArcticForCausalLM")
@ModelBase.example("Snowflake/snowflake-arctic-instruct")
class ArcticModel(TextModel):
model_arch = gguf.MODEL_ARCH.ARCTIC
-1
View File
@@ -9,7 +9,6 @@ from .base import ModelBase, TextModel, gguf, logger
@ModelBase.register("BaichuanForCausalLM", "BaiChuanForCausalLM")
@ModelBase.example("baichuan-inc/Baichuan2-7B-Chat", "baichuan-inc/Baichuan-7B")
class BaichuanModel(TextModel):
model_arch = gguf.MODEL_ARCH.BAICHUAN
-3
View File
@@ -11,7 +11,6 @@ from .base import ModelBase, TextModel, gguf
@ModelBase.register("BailingMoeForCausalLM")
@ModelBase.example("inclusionAI/Ling-lite")
class BailingMoeModel(TextModel):
model_arch = gguf.MODEL_ARCH.BAILINGMOE
@@ -109,7 +108,6 @@ class BailingMoeModel(TextModel):
@ModelBase.register("BailingMoeV2ForCausalLM")
@ModelBase.example("inclusionAI/Ling-mini-2.0")
class BailingMoeV2Model(TextModel):
model_arch = gguf.MODEL_ARCH.BAILINGMOE2
@@ -191,7 +189,6 @@ class BailingMoeV2Model(TextModel):
@ModelBase.register("SarvamMoEForCausalLM", "modeling_sarvam_moe.SarvamMoEForCausalLM")
@ModelBase.example("sarvamai/sarvam-30b")
class SarvamMoEModel(BailingMoeV2Model):
model_arch = gguf.MODEL_ARCH.BAILINGMOE2
# Sarvam-MoE shares the BailingMoeV2 architecture; only differences:
-193
View File
@@ -1,193 +0,0 @@
from __future__ import annotations
import re
from typing import Callable, Iterable, TYPE_CHECKING
import torch
if TYPE_CHECKING:
from torch import Tensor
from .base import ModelBase, TextModel, gguf
@ModelBase.register("BailingMoeV3ForCausalLM")
@ModelBase.example("inclusionAI/Ling-3.0-tiny", "inclusionAI/Ling-3.0-flash")
class BailingMoeV3Model(TextModel):
model_arch = gguf.MODEL_ARCH.BAILINGMOE3
supports_mtp_export = True
_experts: list[dict[str, Tensor]] | None = None
_main_layers: int | None = None
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
nextn_layers = self.hparams.get("num_nextn_predict_layers", 0) or 0
if self.no_mtp:
nextn_layers = 0
self.block_count = self.hparams["num_hidden_layers"] + nextn_layers
self.tensor_map = gguf.get_tensor_name_map(self.model_arch, self.block_count)
def index_tensors(self, remote_hf_model_id: str | None = None):
type(self)._main_layers = self.hparams["num_hidden_layers"]
return super().index_tensors(remote_hf_model_id=remote_hf_model_id)
def set_vocab(self):
self._set_vocab_gpt2()
def is_full_attention(self, bid: int) -> bool:
n_layer = self.hparams["num_hidden_layers"]
layer_group_size = self.hparams["layer_group_size"]
return bid >= n_layer or (bid + 1) % layer_group_size == 0 or bid >= n_layer // layer_group_size * layer_group_size
def set_gguf_parameters(self):
if not self.hparams.get("no_kda_lora", False):
raise ValueError("BailingMoeV3 KDA LoRA projections are not supported")
if not self.hparams.get("kda_safe_gate", False):
raise ValueError("BailingMoeV3 non-safe KDA gates are not supported")
if self.hparams.get("gated_attention_proj_granularity_type") != "head_wise":
raise ValueError("BailingMoeV3 requires head-wise attention gates")
self.hparams["num_key_value_heads"] = 1
super().set_gguf_parameters()
n_head_kv = [1 if self.is_full_attention(il) else 0 for il in range(self.block_count)]
self.gguf_writer.add_head_count_kv(n_head_kv)
self.gguf_writer.add_vocab_size(self.hparams["vocab_size"])
self.gguf_writer.add_ssm_conv_kernel(self.hparams["short_conv_kernel_size"])
self.gguf_writer.add_kda_head_dim(self.hparams["head_dim"])
self.gguf_writer.add_kda_safe_gate(self.hparams["kda_safe_gate"])
self.gguf_writer.add_kda_gate_lower_bound(self.hparams["kda_lower_bound"])
kv_lora_rank = self.hparams["kv_lora_rank"]
qk_nope_head_dim = self.hparams["qk_nope_head_dim"]
qk_rope_head_dim = self.hparams["qk_rope_head_dim"]
if (q_lora_rank := self.hparams.get("q_lora_rank")) is not None:
self.gguf_writer.add_q_lora_rank(q_lora_rank)
self.gguf_writer.add_kv_lora_rank(kv_lora_rank)
self.gguf_writer.add_rope_dimension_count(qk_rope_head_dim)
self.gguf_writer.add_key_length(kv_lora_rank + qk_rope_head_dim)
self.gguf_writer.add_key_length_mla(qk_nope_head_dim + qk_rope_head_dim)
self.gguf_writer.add_value_length_mla(self.hparams["v_head_dim"])
self.gguf_writer.add_expert_feed_forward_length(self.hparams["moe_intermediate_size"])
self.gguf_writer.add_expert_shared_feed_forward_length(self.hparams["moe_shared_expert_intermediate_size"])
self.gguf_writer.add_expert_shared_count(self.hparams["num_shared_experts"])
self.gguf_writer.add_leading_dense_block_count(self.hparams["first_k_dense_replace"])
self.gguf_writer.add_expert_weights_scale(self.hparams["routed_scaling_factor"])
self.gguf_writer.add_expert_weights_norm(self.hparams["norm_topk_prob"])
def clamp_limits(key: str) -> list[float] | None:
values = self.hparams.get(key)
if values is None:
return None
values = [0.0 if value is None else float(value) for value in values[:self.block_count]]
return values + [0.0] * (self.block_count - len(values))
if (values := clamp_limits("expert_swiglu_limit_list")) is not None:
self.gguf_writer.add_swiglu_clamp_exp(values)
if (values := clamp_limits("share_expert_swiglu_limit_list")) is not None:
self.gguf_writer.add_swiglu_clamp_shexp(values)
if not self.no_mtp and (nextn_layers := self.hparams.get("num_nextn_predict_layers", 0)):
self.gguf_writer.add_nextn_predict_layers(nextn_layers)
def prepare_metadata(self, vocab_only: bool):
from_dir = self.fname_out.is_dir()
super().prepare_metadata(vocab_only=vocab_only)
if not self.mtp_only or not from_dir:
return
output_type: str = self.ftype.name.partition("_")[2]
fname_default: str = gguf.naming_convention(
self.metadata.name, self.metadata.basename, self.metadata.finetune,
self.metadata.version, size_label=None, output_type=output_type, model_type=None)
self.fname_out = self.fname_out.parent / f"mtp-{fname_default}.gguf"
@classmethod
def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
name, gen = item
if name.endswith(".expert_bias"):
name += ".bias"
if cls._main_layers is None:
return super().filter_tensors((name, gen))
m = re.match(r"model\.layers\.(\d+)\.", name)
is_mtp = m is not None and int(m.group(1)) >= cls._main_layers
if is_mtp and cls.no_mtp:
return None
if cls.mtp_only and not is_mtp and name not in (
"model.word_embeddings.weight", "model.norm.weight", "lm_head.weight",
):
return None
return super().filter_tensors((name, gen))
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
if name.endswith((".q_conv1d.weight", ".k_conv1d.weight", ".v_conv1d.weight")) and data_torch.ndim in (2, 3):
d_inner = data_torch.shape[0]
d_conv = data_torch.shape[-1]
data_torch = data_torch.reshape(1, d_inner, 1, d_conv)
if name.endswith(".A_log"):
data_torch = torch.exp(data_torch).reshape(-1, 1)
if name.endswith(".dt_bias"):
name = name.rpartition(".dt_bias")[0] + ".dt_proj.bias"
if name.endswith(".attention.f_proj.weight"):
assert bid is not None
if self.is_full_attention(bid):
raise ValueError(f"unexpected f_proj on full-attention layer {bid}")
name = self.format_tensor_name(gguf.MODEL_TENSOR.SSM_F_A, bid)
if name.endswith(".attention.g_proj.weight"):
assert bid is not None
tensor = gguf.MODEL_TENSOR.ATTN_GATE if self.is_full_attention(bid) else gguf.MODEL_TENSOR.SSM_G_A
name = self.format_tensor_name(tensor, bid)
if ".mlp.experts." in name:
n_experts = self.hparams["num_experts"]
assert bid is not None
if self._experts is None:
self._experts = [{} for _ in range(self.block_count)]
self._experts[bid][name] = data_torch
if len(self._experts[bid]) >= n_experts * 3:
for weight_name in ("down_proj", "gate_proj", "up_proj"):
tensors = []
for expert_id in range(n_experts):
expert_name = f"model.layers.{bid}.mlp.experts.{expert_id}.{weight_name}.weight"
tensors.append(self._experts[bid].pop(expert_name))
merged_name = f"model.layers.{bid}.mlp.experts.{weight_name}.weight"
yield from super().modify_tensors(torch.stack(tensors, dim=0), merged_name, bid)
return
if name.endswith(".attention.kv_b_proj.weight"):
assert bid is not None
n_head = self.hparams["num_attention_heads"]
v_head_dim = self.hparams["v_head_dim"]
qk_nope_head_dim = self.hparams["qk_nope_head_dim"]
assert data_torch.shape[0] == n_head * (v_head_dim + qk_nope_head_dim)
kv_b = data_torch.view(n_head, v_head_dim + qk_nope_head_dim, data_torch.shape[-1])
k_b, v_b = torch.split(kv_b, [qk_nope_head_dim, v_head_dim], dim=1)
name_k = self.format_tensor_name(gguf.MODEL_TENSOR.ATTN_K_B, bid)
name_v = self.format_tensor_name(gguf.MODEL_TENSOR.ATTN_V_B, bid)
yield from super().modify_tensors(k_b.transpose(1, 2), name_k, bid)
yield from super().modify_tensors(v_b, name_v, bid)
return
yield from super().modify_tensors(data_torch, name, bid)
def prepare_tensors(self):
super().prepare_tensors()
if self._experts is not None:
experts = [name for layer in self._experts for name in layer]
if experts:
raise ValueError(f"Unprocessed experts: {experts}")
-8
View File
@@ -1149,14 +1149,6 @@ class ModelBase:
return modelcls
return func
@classmethod
def example(cls, *hf_repos: str) -> Callable[[AnyModel], AnyModel]:
del hf_repos # unused
def func(modelcls: AnyModel) -> AnyModel:
return modelcls
return func
@classmethod
def print_registered_models(cls):
for model_type, model_classes in cls._model_classes.items():
-9
View File
@@ -15,7 +15,6 @@ from .base import ModelBase, SentencePieceTokenTypes, TextModel, gguf, logger
@ModelBase.register("BertModel", "BertForMaskedLM", "CamembertModel", "BertForSequenceClassification")
@ModelBase.example("BAAI/bge-small-en-v1.5", "dangvantuan/sentence-camembert-base")
class BertModel(TextModel):
model_arch = gguf.MODEL_ARCH.BERT
@@ -241,7 +240,6 @@ class BertModel(TextModel):
@ModelBase.register("DistilBertModel", "DistilBertForMaskedLM", "DistilBertForSequenceClassification")
@ModelBase.example("distilbert/distilbert-base-uncased")
class DistilBertModel(BertModel):
model_arch = gguf.MODEL_ARCH.BERT
@@ -265,7 +263,6 @@ class DistilBertModel(BertModel):
@ModelBase.register("RobertaModel", "RobertaForSequenceClassification")
@ModelBase.example("sentence-transformers/stsb-roberta-base")
class RobertaModel(BertModel):
model_arch = gguf.MODEL_ARCH.BERT
@@ -315,7 +312,6 @@ class RobertaModel(BertModel):
@ModelBase.register("NomicBertModel")
@ModelBase.example("nomic-ai/nomic-embed-text-v1.5")
class NomicBertModel(BertModel):
model_arch = gguf.MODEL_ARCH.BERT
@@ -404,7 +400,6 @@ class NomicBertModel(BertModel):
@ModelBase.register("NeoBERT", "NeoBERTLMHead", "NeoBERTForSequenceClassification")
@ModelBase.example("chandar-lab/NeoBERT")
class NeoBert(BertModel):
model_arch = gguf.MODEL_ARCH.NEO_BERT
@@ -436,7 +431,6 @@ class NeoBert(BertModel):
@ModelBase.register("EuroBertModel", "JinaEmbeddingsV5Model")
@ModelBase.example("hf-tiny-v2/tiny-random-EuroBertModel", "jinaai/jina-embeddings-v5-text-nano")
class EuroBertModel(TextModel):
model_arch = gguf.MODEL_ARCH.EUROBERT
@@ -465,7 +459,6 @@ class EuroBertModel(TextModel):
@ModelBase.register("XLMRobertaModel", "XLMRobertaForSequenceClassification")
@ModelBase.example("BAAI/bge-m3")
class XLMRobertaModel(BertModel):
model_arch = gguf.MODEL_ARCH.BERT
_lora_files = {}
@@ -568,7 +561,6 @@ class XLMRobertaModel(BertModel):
@ModelBase.register("JinaBertModel", "JinaBertForMaskedLM")
@ModelBase.example("jinaai/jina-embeddings-v2-base-en")
class JinaBertV2Model(BertModel):
model_arch = gguf.MODEL_ARCH.JINA_BERT_V2
@@ -596,7 +588,6 @@ class JinaBertV2Model(BertModel):
@ModelBase.register("ModernBertModel", "ModernBertForMaskedLM", "ModernBertForSequenceClassification")
@ModelBase.example("answerdotai/ModernBERT-base")
class ModernBertModel(BertModel):
model_arch = gguf.MODEL_ARCH.MODERN_BERT
-1
View File
@@ -9,7 +9,6 @@ from .base import ModelBase, TextModel, gguf
@ModelBase.register("BitnetForCausalLM", "BitNetForCausalLM")
@ModelBase.example("microsoft/bitnet-b1.58-2B-4T")
class BitnetModel(TextModel):
model_arch = gguf.MODEL_ARCH.BITNET
-1
View File
@@ -13,7 +13,6 @@ from .base import ModelBase, TextModel, gguf, logger
@ModelBase.register("BloomForCausalLM", "BloomModel")
@ModelBase.example("bigscience/bloom-560m")
class BloomModel(TextModel):
model_arch = gguf.MODEL_ARCH.BLOOM
-2
View File
@@ -12,8 +12,6 @@ from .llama import LlamaModel
@ModelBase.register("ChameleonForConditionalGeneration")
@ModelBase.register("ChameleonForCausalLM") # obsolete
# [TAG_HF_EXAMPLE_GATED] facebook/chameleon-7b is gated
# [TAG_HF_EXAMPLE_MISSING]
class ChameleonModel(TextModel):
model_arch = gguf.MODEL_ARCH.CHAMELEON
-1
View File
@@ -9,7 +9,6 @@ from .base import ModelBase, SentencePieceTokenTypes, TextModel, gguf
@ModelBase.register("GlmForCausalLM", "ChatGLMModel", "ChatGLMForConditionalGeneration")
@ModelBase.example("THUDM/chatglm3-6b", "zai-org/glm-4-9b-chat-hf")
class ChatGLMModel(TextModel):
model_arch = gguf.MODEL_ARCH.CHATGLM
-1
View File
@@ -4,7 +4,6 @@ from .base import ModelBase, TextModel, gguf
@ModelBase.register("CodeShellForCausalLM")
@ModelBase.example("WisdomShell/CodeShell-7B")
class CodeShellModel(TextModel):
model_arch = gguf.MODEL_ARCH.CODESHELL
-2
View File
@@ -11,7 +11,6 @@ from .llama import LlamaModel
@ModelBase.register("CogVLMForCausalLM")
@ModelBase.example("THUDM/cogvlm2-llama3-chat-19B", "THUDM/cogvlm-chat-hf")
class CogVLMVisionModel(MmprojModel):
def set_gguf_parameters(self):
@@ -30,6 +29,5 @@ class CogVLMVisionModel(MmprojModel):
@ModelBase.register("CogVLMForCausalLM")
@ModelBase.example("THUDM/cogvlm2-llama3-chat-19B", "THUDM/cogvlm-chat-hf")
class CogVLMModel(LlamaModel):
model_arch = gguf.MODEL_ARCH.COGVLM
-5
View File
@@ -12,8 +12,6 @@ from .base import ModelBase, TextModel, gguf, logger
@ModelBase.register("CohereForCausalLM")
# [TAG_HF_EXAMPLE_GATED] CohereLabs/c4ai-command-r-v01 is gated
# [TAG_HF_EXAMPLE_MISSING]
class CommandR2Model(TextModel):
model_arch = gguf.MODEL_ARCH.COMMAND_R
@@ -32,8 +30,6 @@ class CommandR2Model(TextModel):
@ModelBase.register("Cohere2ForCausalLM")
# [TAG_HF_EXAMPLE_GATED] CohereLabs/c4ai-command-r7b-12-2024 is gated
@ModelBase.example("hf-tiny-v2/tiny-random-Cohere2ForCausalLM")
class Cohere2Model(TextModel):
model_arch = gguf.MODEL_ARCH.COHERE2
@@ -63,7 +59,6 @@ class Cohere2Model(TextModel):
@ModelBase.register("Cohere2MoeForCausalLM")
@ModelBase.example("CohereLabs/North-Mini-Code-1.0")
class Cohere2MoeModel(TextModel):
model_arch = gguf.MODEL_ARCH.COHERE2MOE
_n_main_layers: int | None = None
-1
View File
@@ -9,7 +9,6 @@ from .base import ModelBase, TextModel, gguf, logger
@ModelBase.register("DbrxForCausalLM")
@ModelBase.example("alpindale/dbrx-instruct")
class DbrxModel(TextModel):
model_arch = gguf.MODEL_ARCH.DBRX
-1
View File
@@ -13,7 +13,6 @@ from .base import ModelBase, TextModel, gguf
@ModelBase.register("DeciLMForCausalLM")
@ModelBase.example("nvidia/Llama-3_1-Nemotron-51B-Instruct", "Deci/DeciLM-7B")
class DeciModel(TextModel):
model_arch = gguf.MODEL_ARCH.DECI
-8
View File
@@ -18,7 +18,6 @@ from .qwen import QwenModel
@ModelBase.register("DeepseekOCRForCausalLM")
@ModelBase.example("deepseek-ai/DeepSeek-OCR")
class DeepseekOCRVisionModel(MmprojModel):
# HF dynamic_preprocess() max_num, which differs per model
preproc_max_tiles = 9
@@ -101,13 +100,11 @@ class DeepseekOCRVisionModel(MmprojModel):
@ModelBase.register("UnlimitedOCRForCausalLM")
@ModelBase.example("baidu/Unlimited-OCR")
class UnlimitedOCRVisionModel(DeepseekOCRVisionModel):
preproc_max_tiles = 32
@ModelBase.register("DeepseekOCR2ForCausalLM")
@ModelBase.example("deepseek-ai/DeepSeek-OCR-2")
class DeepseekOCR2VisionModel(DeepseekOCRVisionModel):
preproc_max_tiles = 6
@@ -137,7 +134,6 @@ class DeepseekOCR2VisionModel(DeepseekOCRVisionModel):
@ModelBase.register("DeepseekForCausalLM")
@ModelBase.example("deepseek-ai/deepseek-moe-16b-chat")
class DeepseekModel(TextModel):
model_arch = gguf.MODEL_ARCH.DEEPSEEK
@@ -232,7 +228,6 @@ class DeepseekModel(TextModel):
"YoutuForCausalLM",
"YoutuVLForConditionalGeneration",
)
@ModelBase.example("deepseek-ai/DeepSeek-V2-Lite", "deepseek-ai/DeepSeek-V3")
class DeepseekV2Model(TextModel):
model_arch = gguf.MODEL_ARCH.DEEPSEEK2
@@ -462,7 +457,6 @@ class DeepseekV2Model(TextModel):
@ModelBase.register("DeepseekV32ForCausalLM")
@ModelBase.example("deepseek-ai/DeepSeek-V3.2-Exp")
class DeepseekV32Model(DeepseekV2Model):
model_arch = gguf.MODEL_ARCH.DEEPSEEK32
skip_mtp = False
@@ -523,7 +517,6 @@ class DeepseekV32Model(DeepseekV2Model):
@ModelBase.register("DeepseekV4ForCausalLM")
@ModelBase.example("deepseek-ai/DeepSeek-V4-Flash-Base")
class DeepseekV4Model(TextModel):
model_arch = gguf.MODEL_ARCH.DEEPSEEK4
supports_mtp_export = True
@@ -918,7 +911,6 @@ class DeepseekV4Model(TextModel):
@ModelBase.register("DeepseekV4DSparkModel")
@ModelBase.example("deepseek-ai/DeepSeek-V4-Flash-DSpark")
class DeepseekV4DSparkModel(DeepseekV4Model):
model_arch = gguf.MODEL_ARCH.DFLASH
-1
View File
@@ -11,7 +11,6 @@ from .qwen import Qwen2MoeModel
@ModelBase.register("Dots1ForCausalLM")
@ModelBase.example("rednote-hilab/dots.llm1.inst")
class Dots1Model(Qwen2MoeModel):
model_arch = gguf.MODEL_ARCH.DOTS1
-1
View File
@@ -9,7 +9,6 @@ from .base import MmprojModel, ModelBase, gguf
@ModelBase.register("DotsOCRForCausalLM")
@ModelBase.example("rednote-hilab/dots.ocr")
class DotsOCRVisionModel(MmprojModel):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
-1
View File
@@ -9,7 +9,6 @@ from .base import ModelBase, TextModel, gguf
@ModelBase.register("DreamModel")
@ModelBase.example("Dream-org/Dream-v0-Instruct-7B")
class DreamModel(TextModel):
model_arch = gguf.MODEL_ARCH.DREAM
-4
View File
@@ -15,7 +15,6 @@ from .base import MmprojModel, ModelBase, TextModel, gguf
@ModelBase.register("Ernie4_5_ForCausalLM", "Ernie4_5ForCausalLM")
@ModelBase.example("baidu/ERNIE-4.5-0.3B-PT")
class Ernie4_5Model(TextModel):
model_arch = gguf.MODEL_ARCH.ERNIE4_5
@@ -74,7 +73,6 @@ class Ernie4_5Model(TextModel):
@ModelBase.register("Ernie4_5_MoeForCausalLM")
@ModelBase.example("baidu/ERNIE-4.5-21B-A3B-PT")
class Ernie4_5MoeModel(Ernie4_5Model):
model_arch = gguf.MODEL_ARCH.ERNIE4_5_MOE
_experts: list[dict[str, Tensor]] | None = None
@@ -158,13 +156,11 @@ class Ernie4_5MoeModel(Ernie4_5Model):
@ModelBase.register("PaddleOCRVLForConditionalGeneration")
@ModelBase.example("PaddlePaddle/PaddleOCR-VL")
class PaddleOCRModel(Ernie4_5Model):
model_arch = gguf.MODEL_ARCH.PADDLEOCR
@ModelBase.register("PaddleOCRVisionModel")
@ModelBase.example("PaddlePaddle/PaddleOCR-VL")
class PaddleOCRVisionModel(MmprojModel):
# PaddleOCR-VL uses a modified version of Siglip
min_pixels: int = 0
-5
View File
@@ -15,7 +15,6 @@ from .qwenvl import Qwen2VLVisionModel
@ModelBase.register("ExaoneForCausalLM")
@ModelBase.example("LGAI-EXAONE/EXAONE-3.5-2.4B-Instruct")
class ExaoneModel(TextModel):
model_arch = gguf.MODEL_ARCH.EXAONE
@@ -61,7 +60,6 @@ class ExaoneModel(TextModel):
@ModelBase.register("Exaone4ForCausalLM")
@ModelBase.example("LGAI-EXAONE/EXAONE-4.0-32B")
class Exaone4Model(TextModel):
model_arch = gguf.MODEL_ARCH.EXAONE4
@@ -128,7 +126,6 @@ class Exaone4Model(TextModel):
# note: transformers >= 5.1 renamed the class to "ExaoneMoeForCausalLM" (lowercase 'e'),
# so accept both spellings - LG AI have updated the configs of already-released models
@ModelBase.register("ExaoneMoEForCausalLM", "ExaoneMoeForCausalLM")
@ModelBase.example("LGAI-EXAONE/K-EXAONE-236B-A23B")
class ExaoneMoEModel(Exaone4Model):
model_arch = gguf.MODEL_ARCH.EXAONE_MOE
@@ -217,7 +214,6 @@ class ExaoneMoEModel(Exaone4Model):
@ModelBase.register("Exaone4_5_ForConditionalGeneration")
@ModelBase.example("LGAI-EXAONE/EXAONE-4.5-33B")
class Exaone4_5_TextModel(Exaone4Model):
"""Text tower of EXAONE 4.5; Tensors match EXAONE4"""
@@ -271,7 +267,6 @@ class Exaone4_5_TextModel(Exaone4Model):
@ModelBase.register("Exaone4_5_ForConditionalGeneration")
@ModelBase.example("LGAI-EXAONE/EXAONE-4.5-33B")
class Exaone4_5VisionModel(Qwen2VLVisionModel):
"""Vision tower for EXAONE 4.5; Qwen2-VL-style ViT (GQA) + patch merger"""
-1
View File
@@ -11,7 +11,6 @@ from .base import ModelBase, TextModel, gguf
@ModelBase.register("FalconForCausalLM", "RWForCausalLM")
@ModelBase.example("tiiuae/falcon-7b")
class FalconModel(TextModel):
model_arch = gguf.MODEL_ARCH.FALCON
-1
View File
@@ -12,7 +12,6 @@ from .mamba import Mamba2Model
@ModelBase.register("FalconH1ForCausalLM")
@ModelBase.example("tiiuae/Falcon-H1-0.5B-Base")
class FalconH1Model(Mamba2Model):
model_arch = gguf.MODEL_ARCH.FALCON_H1
-19
View File
@@ -14,8 +14,6 @@ from .base import MmprojModel, ModelBase, TextModel, gguf, logger
@ModelBase.register("GemmaForCausalLM")
# [TAG_HF_EXAMPLE_GATED] google/gemma-2b is gated
@ModelBase.example("trl-internal-testing/tiny-GemmaForCausalLM")
class GemmaModel(TextModel):
model_arch = gguf.MODEL_ARCH.GEMMA
@@ -70,8 +68,6 @@ class GemmaModel(TextModel):
@ModelBase.register("Gemma2ForCausalLM")
# [TAG_HF_EXAMPLE_GATED] google/gemma-2-9b-it is gated
@ModelBase.example("trl-internal-testing/tiny-Gemma2ForCausalLM")
class Gemma2Model(TextModel):
model_arch = gguf.MODEL_ARCH.GEMMA2
@@ -122,8 +118,6 @@ class Gemma2Model(TextModel):
@ModelBase.register("Gemma3ForCausalLM", "Gemma3ForConditionalGeneration")
# [TAG_HF_EXAMPLE_GATED] google/gemma-3-4b-it is gated
@ModelBase.example("trl-internal-testing/tiny-Gemma3ForConditionalGeneration", "hf-tiny-v2/tiny-random-Gemma3ForCausalLM")
class Gemma3Model(TextModel):
model_arch = gguf.MODEL_ARCH.GEMMA3
@@ -180,8 +174,6 @@ class Gemma3Model(TextModel):
@ModelBase.register("Gemma3TextModel")
# [TAG_HF_EXAMPLE_GATED] google/embeddinggemma-300m is gated
@ModelBase.example("hf-tiny-v2/tiny-random-Gemma3TextModel")
class EmbeddingGemma(Gemma3Model):
model_arch = gguf.MODEL_ARCH.GEMMA_EMBEDDING
module_paths = []
@@ -256,8 +248,6 @@ class EmbeddingGemma(Gemma3Model):
@ModelBase.register("Gemma3ForConditionalGeneration")
# [TAG_HF_EXAMPLE_GATED] google/gemma-3-4b-it is gated
@ModelBase.example("trl-internal-testing/tiny-Gemma3ForConditionalGeneration")
class Gemma3VisionModel(MmprojModel):
def set_gguf_parameters(self):
super().set_gguf_parameters()
@@ -362,8 +352,6 @@ class ConformerAudioModel(MmprojModel):
@ModelBase.register("Gemma3nForConditionalGeneration")
# [TAG_HF_EXAMPLE_GATED] google/gemma-3n-E2B-it is gated
@ModelBase.example("hf-tiny-v2/tiny-random-Gemma3nForConditionalGeneration")
class Gemma3nVisionAudioModel(ConformerAudioModel):
has_audio_encoder = True
has_vision_encoder = True
@@ -483,8 +471,6 @@ class Gemma3nVisionAudioModel(ConformerAudioModel):
@ModelBase.register("Gemma3nForCausalLM", "Gemma3nForConditionalGeneration")
# [TAG_HF_EXAMPLE_GATED] google/gemma-3n-E2B-it is gated
@ModelBase.example("hf-tiny-v2/tiny-random-Gemma3nForConditionalGeneration")
class Gemma3NModel(Gemma3Model):
model_arch = gguf.MODEL_ARCH.GEMMA3N
@@ -629,7 +615,6 @@ class Gemma3NModel(Gemma3Model):
@ModelBase.register("Gemma4ForConditionalGeneration", "Gemma4ForCausalLM")
@ModelBase.example("google/gemma-4-31B-it", "google/gemma-4-26B-A4B-it", "google/gemma-4-E2B-it")
class Gemma4Model(Gemma3Model):
model_arch = gguf.MODEL_ARCH.GEMMA4
@@ -810,7 +795,6 @@ class Gemma4Model(Gemma3Model):
@ModelBase.register("Gemma4UnifiedForConditionalGeneration")
@ModelBase.example("hf-tiny-v2/tiny-random-Gemma4UnifiedForConditionalGeneration")
class Gemma4UnifiedModel(Gemma4Model):
model_arch = gguf.MODEL_ARCH.GEMMA4
@@ -831,7 +815,6 @@ class Gemma4UnifiedModel(Gemma4Model):
@ModelBase.register("Gemma4AssistantForCausalLM", "Gemma4UnifiedAssistantForCausalLM")
@ModelBase.example("google/gemma-4-31B-it-assistant", "google/gemma-4-26B-A4B-it-assistant", "google/gemma-4-E2B-it-assistant")
class Gemma4AssistantModel(Gemma4Model):
model_arch = gguf.MODEL_ARCH.GEMMA4_ASSISTANT
@@ -852,7 +835,6 @@ class Gemma4AssistantModel(Gemma4Model):
@ModelBase.register("Gemma4ForConditionalGeneration")
@ModelBase.example("google/gemma-4-31B-it", "google/gemma-4-26B-A4B-it", "google/gemma-4-E2B-it")
class Gemma4VisionAudioModel(MmprojModel):
has_audio_encoder = True
has_vision_encoder = True
@@ -931,7 +913,6 @@ class Gemma4VisionAudioModel(MmprojModel):
@ModelBase.register("Gemma4UnifiedForConditionalGeneration")
@ModelBase.example("hf-tiny-v2/tiny-random-Gemma4UnifiedForConditionalGeneration")
class Gemma4UnifiedVisionAudioModel(Gemma4VisionAudioModel):
has_audio_encoder = True
has_vision_encoder = True
-6
View File
@@ -15,7 +15,6 @@ from .deepseek import DeepseekV2Model
@ModelBase.register("Glm4ForCausalLM", "Glm4vForConditionalGeneration")
@ModelBase.example("zai-org/GLM-4-9B-0414")
class Glm4Model(TextModel):
model_arch = gguf.MODEL_ARCH.GLM4
use_mrope = False
@@ -87,7 +86,6 @@ class Glm4Model(TextModel):
@ModelBase.register("GlmOcrForConditionalGeneration")
@ModelBase.example("zai-org/GLM-OCR")
class GlmOCRModel(Glm4Model):
model_arch = gguf.MODEL_ARCH.GLM4
use_mrope = False
@@ -109,7 +107,6 @@ class GlmOCRModel(Glm4Model):
@ModelBase.register("Glm4MoeForCausalLM", "Glm4vMoeForConditionalGeneration")
@ModelBase.example("zai-org/GLM-4.5-Air")
class Glm4MoeModel(TextModel):
model_arch = gguf.MODEL_ARCH.GLM4_MOE
@@ -207,7 +204,6 @@ class Glm4MoeModel(TextModel):
@ModelBase.register("Glm4MoeLiteForCausalLM")
@ModelBase.example("zai-org/GLM-4.7-Flash")
class Glm4MoeLiteModel(DeepseekV2Model):
model_arch = gguf.MODEL_ARCH.DEEPSEEK2
skip_mtp = False
@@ -276,7 +272,6 @@ class Glm4MoeLiteModel(DeepseekV2Model):
@ModelBase.register("GlmMoeDsaForCausalLM")
@ModelBase.example("zai-org/GLM-5.2")
class GlmMoeDsaModel(DeepseekV2Model):
model_arch = gguf.MODEL_ARCH.GLM_DSA
skip_mtp = False
@@ -345,7 +340,6 @@ class GlmMoeDsaModel(DeepseekV2Model):
@ModelBase.register("SolarOpenForCausalLM")
@ModelBase.example("upstage/Solar-Open-100B")
class SolarOpenModel(Glm4MoeModel):
model_arch = gguf.MODEL_ARCH.GLM4_MOE
-2
View File
@@ -11,7 +11,6 @@ from .base import ModelBase, TextModel, gguf, logger
@ModelBase.register("GPT2LMHeadModel")
@ModelBase.example("openai-community/gpt2")
class GPT2Model(TextModel):
model_arch = gguf.MODEL_ARCH.GPT2
@@ -39,7 +38,6 @@ class GPT2Model(TextModel):
@ModelBase.register("RuGPT3XLForCausalLM")
@ModelBase.example("evilfreelancer/ruGPT3XL")
class RuGPT3XLModel(TextModel):
model_arch = gguf.MODEL_ARCH.GPT2
-1
View File
@@ -11,7 +11,6 @@ from .base import ModelBase, TextModel, gguf, logger
@ModelBase.register("GptOssForCausalLM")
@ModelBase.example("openai/gpt-oss-20b")
class GptOssModel(TextModel):
model_arch = gguf.MODEL_ARCH.GPT_OSS
-1
View File
@@ -13,7 +13,6 @@ from .base import ModelBase, TextModel, gguf, logger
@ModelBase.register("GPTNeoXForCausalLM")
@ModelBase.example("EleutherAI/pythia-70m")
class GPTNeoXModel(TextModel):
model_arch = gguf.MODEL_ARCH.GPTNEOX
-7
View File
@@ -15,7 +15,6 @@ from .mamba import Mamba2Model
@ModelBase.register("GraniteForCausalLM")
@ModelBase.example("ibm-granite/granite-3.3-2b-instruct")
class GraniteModel(LlamaModel):
"""Conversion for IBM's GraniteForCausalLM"""
model_arch = gguf.MODEL_ARCH.GRANITE
@@ -75,7 +74,6 @@ class GraniteModel(LlamaModel):
@ModelBase.register("GraniteMoeForCausalLM", "GraniteMoeSharedForCausalLM")
@ModelBase.example("ibm-granite/granite-3.1-3b-a800m-instruct")
class GraniteMoeModel(GraniteModel):
"""Conversion for IBM's GraniteMoeForCausalLM"""
model_arch = gguf.MODEL_ARCH.GRANITE_MOE
@@ -126,7 +124,6 @@ class GraniteMoeModel(GraniteModel):
@ModelBase.register("GraniteSwitchForCausalLM")
@ModelBase.example("ibm-granite/granite-switch-4.1-3b-preview")
class GraniteSwitchModel(GraniteMoeModel):
"""Dense, all-attention Granite with N per-token embedded LoRA adapters, stacked
over the adapter dim with a zero adapter at slot 0 (N = num_adapters + 1)."""
@@ -287,7 +284,6 @@ class GraniteSwitchModel(GraniteMoeModel):
@ModelBase.register("GraniteMoeHybridForCausalLM", "BambaForCausalLM")
@ModelBase.example("ibm-granite/granite-4.0-h-tiny", "ibm-ai-platform/Bamba-9B-v2")
class GraniteHybridModel(Mamba2Model, GraniteMoeModel):
"""GraniteHybrid is a hybrid SSM + Attention model that uses Mamba2 SSM
layers and optionally uses MoE w/ a shared expert"""
@@ -430,7 +426,6 @@ class GraniteHybridModel(Mamba2Model, GraniteMoeModel):
@ModelBase.register("GraniteSpeechForConditionalGeneration")
@ModelBase.example("ibm-granite/granite-speech-3.3-2b", "ibm-granite/granite-4.0-1b-speech")
class GraniteSpeechMmprojModel(MmprojModel):
has_vision_encoder = False
has_audio_encoder = True
@@ -514,7 +509,6 @@ class GraniteSpeechMmprojModel(MmprojModel):
@ModelBase.register("GraniteSpeechPlusForConditionalGeneration")
@ModelBase.example("ibm-granite/granite-speech-4.1-2b-plus")
class GraniteSpeechPlusMmprojModel(GraniteSpeechMmprojModel):
"""Conversion for GraniteSpeechPlus - extends GraniteSpeech with feature layer concatenation"""
has_vision_encoder = False
@@ -543,7 +537,6 @@ class GraniteSpeechPlusMmprojModel(GraniteSpeechMmprojModel):
@ModelBase.register("Granite4VisionForConditionalGeneration")
@ModelBase.example("ibm-granite/granite-4.0-3b-vision")
class Granite4VisionMmprojModel(MmprojModel):
has_vision_encoder = True
has_audio_encoder = False
-1
View File
@@ -13,7 +13,6 @@ from .base import ModelBase, TextModel, gguf, logger
@ModelBase.register("GrokForCausalLM", "Grok1ForCausalLM")
@ModelBase.example("keyfan/grok-1-hf")
class GrokModel(TextModel):
model_arch = gguf.MODEL_ARCH.GROK
-1
View File
@@ -11,7 +11,6 @@ from .base import ModelBase, TextModel, gguf, logger
@ModelBase.register("GroveMoeForCausalLM", "modeling_grove_moe.GroveMoeForCausalLM")
@ModelBase.example("inclusionAI/GroveMoE-Inst")
class GroveMoeModel(TextModel):
model_arch = gguf.MODEL_ARCH.GROVEMOE
-5
View File
@@ -17,7 +17,6 @@ from .qwen import QwenModel
@ModelBase.register("HunYuanMoEV1ForCausalLM")
@ModelBase.example("tencent/Hunyuan-A13B-Instruct")
class HunYuanMoEModel(TextModel):
model_arch = gguf.MODEL_ARCH.HUNYUAN_MOE
@@ -155,7 +154,6 @@ class HunYuanMoEModel(TextModel):
@ModelBase.register("HunYuanDenseV1ForCausalLM")
@ModelBase.example("tencent/Hunyuan-4B-Instruct")
class HunYuanModel(TextModel):
model_arch = gguf.MODEL_ARCH.HUNYUAN_DENSE
@@ -292,7 +290,6 @@ class HunYuanModel(TextModel):
@ModelBase.register("HunYuanVLForConditionalGeneration")
@ModelBase.example("tencent/HunyuanOCR")
class HunyuanVLVisionModel(MmprojModel):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
@@ -336,7 +333,6 @@ class HunyuanVLVisionModel(MmprojModel):
@ModelBase.register("HunYuanVLForConditionalGeneration")
@ModelBase.example("tencent/HunyuanOCR")
class HunyuanVLTextModel(HunYuanModel):
model_arch = gguf.MODEL_ARCH.HUNYUAN_VL
@@ -369,7 +365,6 @@ class HunyuanVLTextModel(HunYuanModel):
@ModelBase.register("HYV3ForCausalLM")
@ModelBase.example("tencent/Hy3")
class HYV3Model(TextModel):
model_arch = gguf.MODEL_ARCH.HY_V3
supports_mtp_export = True
-2
View File
@@ -14,7 +14,6 @@ from .llama import LlamaModel
@ModelBase.register("InternLM2ForCausalLM")
@ModelBase.example("internlm/internlm2-chat-7b")
class InternLM2Model(TextModel):
model_arch = gguf.MODEL_ARCH.INTERNLM2
@@ -171,7 +170,6 @@ class InternLM2Model(TextModel):
@ModelBase.register("InternLM3ForCausalLM")
@ModelBase.example("internlm/internlm3-8b-instruct")
class InternLM3Model(TextModel):
model_arch = gguf.MODEL_ARCH.LLAMA
-1
View File
@@ -9,7 +9,6 @@ from .base import MmprojModel, ModelBase, gguf
@ModelBase.register("InternVisionModel")
@ModelBase.example("OpenGVLab/InternVL3-2B", "OpenGVLab/InternVL2_5-1B")
class InternVisionModel(MmprojModel):
min_dynamic_tiles: int = 0
-3
View File
@@ -11,8 +11,6 @@ from .base import ModelBase, TextModel, gguf
@ModelBase.register("Jais2ForCausalLM")
# [TAG_HF_EXAMPLE_GATED] inceptionai/Jais-2-8B-Chat is gated
# [TAG_HF_EXAMPLE_MISSING]
class Jais2Model(TextModel):
model_arch = gguf.MODEL_ARCH.JAIS2
@@ -24,7 +22,6 @@ class Jais2Model(TextModel):
@ModelBase.register("JAISLMHeadModel")
@ModelBase.example("inceptionai/jais-family-590m")
class JaisModel(TextModel):
model_arch = gguf.MODEL_ARCH.JAIS
-1
View File
@@ -11,7 +11,6 @@ from .base import ModelBase, TextModel, gguf, logger
@ModelBase.register("JambaForCausalLM")
@ModelBase.example("ai21labs/Jamba-v0.1")
class JambaModel(TextModel):
model_arch = gguf.MODEL_ARCH.JAMBA
-2
View File
@@ -11,7 +11,6 @@ from .llama import LlamaModel
@ModelBase.register("JanusForConditionalGeneration")
@ModelBase.example("deepseek-community/Janus-Pro-1B")
class JanusProModel(LlamaModel):
model_arch = gguf.MODEL_ARCH.LLAMA # reuse Llama arch
@@ -35,7 +34,6 @@ class JanusProModel(LlamaModel):
@ModelBase.register("JanusForConditionalGeneration")
@ModelBase.example("deepseek-community/Janus-Pro-1B")
class JanusProVisionModel(MmprojModel):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
-1
View File
@@ -16,7 +16,6 @@ from .kimi_linear import KimiLinearModel
@ModelBase.register("KimiK3ForConditionalGeneration")
@ModelBase.example("moonshotai/Kimi-K3")
class KimiK3Model(TextModel):
"""
Kimi-K3 text model (KimiLinearForCausalLM under a `language_model.` prefix).
-1
View File
@@ -13,7 +13,6 @@ from .qwen import QwenModel
@ModelBase.register("KimiLinearModel", "KimiLinearForCausalLM")
@ModelBase.example("moonshotai/Kimi-Linear-48B-A3B-Instruct")
class KimiLinearModel(TextModel):
"""Kimi-Linear model with hybrid MLA+KDA architecture"""
model_arch = gguf.MODEL_ARCH.KIMI_LINEAR
-3
View File
@@ -11,7 +11,6 @@ from .base import MmprojModel, ModelBase, gguf
@ModelBase.register("KimiVLForConditionalGeneration")
@ModelBase.example("moonshotai/Kimi-VL-A3B-Instruct")
class KimiVLModel(MmprojModel):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
@@ -53,7 +52,6 @@ class KimiVLModel(MmprojModel):
@ModelBase.register("KimiK25ForConditionalGeneration")
@ModelBase.example("moonshotai/Kimi-K2.5")
class KimiK25Model(MmprojModel):
"""Kimi-K2.5 with MoonViT3d vision encoder"""
@@ -157,7 +155,6 @@ class KimiK25Model(MmprojModel):
@ModelBase.register("Glm5vForConditionalGeneration")
# [TAG_HF_EXAMPLE_MISSING]
class Glm5vModel(KimiK25Model):
"""GLM-5.2-Vision MoonViT3d encoder and projector
-1
View File
@@ -13,7 +13,6 @@ from .base import ModelBase, TextModel, gguf, logger
@ModelBase.register("LagunaForCausalLM")
@ModelBase.example("poolside/Laguna-XS.2", "poolside/Laguna-S-2.1")
class LagunaModel(TextModel):
model_arch = gguf.MODEL_ARCH.LAGUNA
_experts: list[dict] | None = None
-6
View File
@@ -13,7 +13,6 @@ from .gemma import ConformerAudioModel
@ModelBase.register("Lfm2ForCausalLM", "LFM2ForCausalLM")
@ModelBase.example("LiquidAI/LFM2-1.2B", "LiquidAI/LFM2.5-350M")
class LFM2Model(TextModel):
model_arch = gguf.MODEL_ARCH.LFM2
@@ -66,7 +65,6 @@ class LFM2Model(TextModel):
@ModelBase.register("Lfm2Model", "Lfm2BidirectionalModel")
@ModelBase.example("LiquidAI/LFM2.5-ColBERT-350M", "LiquidAI/LFM2.5-Embedding-350M")
class LFM2ColBertModel(LFM2Model):
model_arch = gguf.MODEL_ARCH.LFM2
dense_tensor_name = "dense_2"
@@ -95,7 +93,6 @@ class LFM2ColBertModel(LFM2Model):
@ModelBase.register("Lfm2MoeForCausalLM")
@ModelBase.example("LiquidAI/LFM2-8B-A1B")
class LFM2MoeModel(TextModel):
model_arch = gguf.MODEL_ARCH.LFM2MOE
@@ -169,7 +166,6 @@ class LFM2MoeModel(TextModel):
@ModelBase.register("Lfm2VlForConditionalGeneration")
@ModelBase.example("LiquidAI/LFM2-VL-450M")
class LFM2VLModel(MmprojModel):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
@@ -204,7 +200,6 @@ class LFM2VLModel(MmprojModel):
@ModelBase.register("Lfm2AudioForConditionalGeneration")
@ModelBase.example("LiquidAI/LFM2.5-Audio-1.5B", "LiquidAI/LFM2-Audio-1.5B")
class LFM2AudioModel(ConformerAudioModel):
has_vision_encoder = False
has_audio_encoder = True
@@ -243,7 +238,6 @@ class LFM2AudioModel(ConformerAudioModel):
@ModelBase.register("Lfm25AudioTokenizer")
@ModelBase.example("LiquidAI/LFM2.5-Audio-1.5B")
class LFM25AudioTokenizer(LFM2Model):
model_arch = gguf.MODEL_ARCH.LFM2
-1
View File
@@ -11,7 +11,6 @@ from .llava import LlavaVisionModel
@ModelBase.register("LightOnOCRForConditionalGeneration")
@ModelBase.example("lightonai/LightOnOCR-1B-1025")
class LightOnOCRVisionModel(LlavaVisionModel):
is_mistral_format = False
use_break_tok = False
-2
View File
@@ -11,7 +11,6 @@ from .base import ModelBase, TextModel, gguf
@ModelBase.register("LLaDAModelLM")
@ModelBase.example("GSAI-ML/LLaDA-8B-Instruct")
class LLaDAModel(TextModel):
model_arch = gguf.MODEL_ARCH.LLADA
undo_permute = True
@@ -115,7 +114,6 @@ class LLaDAModel(TextModel):
@ModelBase.register("LLaDAMoEModel", "LLaDAMoEModelLM")
@ModelBase.example("inclusionAI/LLaDA-MoE-7B-A1B-Instruct")
class LLaDAMoEModel(TextModel):
model_arch = gguf.MODEL_ARCH.LLADA_MOE
-8
View File
@@ -28,8 +28,6 @@ from .base import ModelBase, TextModel, gguf, logger
"Eagle3DraftModel",
"IQuestCoderForCausalLM",
"LlamaModel")
# [TAG_HF_EXAMPLE_GATED] meta-llama/Llama-3.2-1B-Instruct is gated
@ModelBase.example("unsloth/Llama-3.2-1B-Instruct", "mistralai/Mistral-7B-Instruct-v0.3", "mistralai/Mixtral-8x7B-Instruct-v0.1")
class LlamaModel(TextModel):
model_arch = gguf.MODEL_ARCH.LLAMA
undo_permute = True
@@ -361,7 +359,6 @@ class LlamaModel(TextModel):
@ModelBase.register("ArceeForCausalLM")
@ModelBase.example("arcee-ai/AFM-4.5B")
class ArceeModel(LlamaModel):
model_arch = gguf.MODEL_ARCH.ARCEE
@@ -374,8 +371,6 @@ class ArceeModel(LlamaModel):
"Llama4ForConditionalGeneration",
"Llama4ForCausalLM",
)
# [TAG_HF_EXAMPLE_GATED] meta-llama/Llama-4-Scout-17B-16E-Instruct is gated
@ModelBase.example("unsloth/Llama-4-Scout-17B-16E-Instruct")
class Llama4Model(LlamaModel):
model_arch = gguf.MODEL_ARCH.LLAMA4
undo_permute = False
@@ -417,19 +412,16 @@ class Llama4Model(LlamaModel):
@ModelBase.register("LlamaBidirectionalModel")
@ModelBase.example("nvidia/llama-embed-nemotron-8b")
class LlamaEmbedNemotronModel(LlamaModel):
model_arch = gguf.MODEL_ARCH.LLAMA_EMBED
@ModelBase.register("SmolLM3ForCausalLM")
@ModelBase.example("HuggingFaceTB/SmolLM3-3B")
class SmolLM3Model(LlamaModel):
model_arch = gguf.MODEL_ARCH.SMOLLM3
@ModelBase.register("ApertusForCausalLM")
@ModelBase.example("swiss-ai/Apertus-8B-Instruct-2509")
class ApertusModel(LlamaModel):
model_arch = gguf.MODEL_ARCH.APERTUS
undo_permute = False
-2
View File
@@ -9,8 +9,6 @@ from .base import MmprojModel, ModelBase, gguf
@ModelBase.register("Llama4ForConditionalGeneration")
# [TAG_HF_EXAMPLE_GATED] meta-llama/Llama-4-Scout-17B-16E-Instruct is gated
@ModelBase.example("unsloth/Llama-4-Scout-17B-16E-Instruct")
class Llama4VisionModel(MmprojModel):
def set_gguf_parameters(self):
super().set_gguf_parameters()
-1
View File
@@ -16,7 +16,6 @@ from .llama import LlamaModel
"LlavaForConditionalGeneration", # pixtral
"Mistral3ForConditionalGeneration", # mistral small 3.1
)
@ModelBase.example("mistral-community/pixtral-12b", "mistralai/Mistral-Small-3.1-24B-Instruct-2503")
class LlavaVisionModel(MmprojModel):
img_break_tok_id = -1
use_break_tok = True
-1
View File
@@ -4,7 +4,6 @@ from .base import ModelBase, TextModel, gguf
@ModelBase.register("MaincoderForCausalLM")
@ModelBase.example("Maincode/Maincoder-1B")
class MaincoderModel(TextModel):
model_arch = gguf.MODEL_ARCH.MAINCODER
-2
View File
@@ -14,7 +14,6 @@ from .base import ModelBase, TextModel, gguf, logger
@ModelBase.register("MambaForCausalLM", "MambaLMHeadModel", "FalconMambaForCausalLM")
@ModelBase.example("state-spaces/mamba-130m-hf", "tiiuae/falcon-mamba-7b")
class MambaModel(TextModel):
model_arch = gguf.MODEL_ARCH.MAMBA
@@ -101,7 +100,6 @@ class MambaModel(TextModel):
@ModelBase.register("Mamba2ForCausalLM")
@ModelBase.example("mistralai/Mamba-Codestral-7B-v0.1")
class Mamba2Model(TextModel):
model_arch = gguf.MODEL_ARCH.MAMBA2
-1
View File
@@ -11,7 +11,6 @@ from .base import ModelBase, TextModel, gguf, logger
@ModelBase.register("MellumForCausalLM")
@ModelBase.example("JetBrains/Mellum2-12B-A2.5B-Base")
class MellumModel(TextModel):
model_arch = gguf.MODEL_ARCH.MELLUM
-2
View File
@@ -14,7 +14,6 @@ from .base import MmprojModel, ModelBase, TextModel, gguf
@ModelBase.register("MiMoV2FlashForCausalLM", "MiMoV2ForCausalLM")
@ModelBase.example("XiaomiMiMo/MiMo-V2.5")
class MimoV2Model(TextModel):
model_arch = gguf.MODEL_ARCH.MIMO2
@@ -231,7 +230,6 @@ class MimoV2Model(TextModel):
@ModelBase.register("MiMoV2ForCausalLM")
@ModelBase.example("XiaomiMiMo/MiMo-V2.5")
class MiMoV2VisionAudioModel(MmprojModel):
has_audio_encoder = True
-4
View File
@@ -14,7 +14,6 @@ from .qwen import Qwen3_5TextModel
@ModelBase.register("MiniCPMForCausalLM")
@ModelBase.example("openbmb/MiniCPM-2B-sft-bf16")
class MiniCPMModel(TextModel):
model_arch = gguf.MODEL_ARCH.MINICPM
@@ -62,7 +61,6 @@ class MiniCPMModel(TextModel):
@ModelBase.register("MiniCPM3ForCausalLM")
@ModelBase.example("openbmb/MiniCPM3-4B")
class MiniCPM3Model(TextModel):
model_arch = gguf.MODEL_ARCH.MINICPM3
@@ -119,7 +117,6 @@ class MiniCPM3Model(TextModel):
# the LM (text mode) and once as the mmproj (vision mode), mirroring the Qwen3-VL setup.
@ModelBase.register("MiniCPMV4_6ForConditionalGeneration")
@ModelBase.example("openbmb/MiniCPM-V-4_6")
class MiniCPMV4_6TextModel(Qwen3_5TextModel):
model_arch = gguf.MODEL_ARCH.QWEN35
@@ -137,7 +134,6 @@ class MiniCPMV4_6TextModel(Qwen3_5TextModel):
@ModelBase.register("MiniCPMV4_6ForConditionalGeneration")
@ModelBase.example("openbmb/MiniCPM-V-4_6")
class MiniCPMV4_6VisionModel(MmprojModel):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
-4
View File
@@ -12,7 +12,6 @@ from .base import ModelBase, TextModel, MmprojModel, gguf, logger
@ModelBase.register("MiniMaxText01ForCausalLM")
@ModelBase.register("MiniMaxM1ForCausalLM")
@ModelBase.example("MiniMaxAI/MiniMax-Text-01", "MiniMaxAI/MiniMax-M1-40k")
class MiniMaxText01Model(TextModel):
model_arch = gguf.MODEL_ARCH.MINIMAX01
@@ -120,7 +119,6 @@ class MiniMaxText01Model(TextModel):
@ModelBase.register("MiniMaxM2ForCausalLM")
@ModelBase.example("MiniMaxAI/MiniMax-M2")
class MiniMaxM2Model(TextModel):
model_arch = gguf.MODEL_ARCH.MINIMAXM2
_experts_cache: dict[int, dict[str, Tensor]] = {}
@@ -165,7 +163,6 @@ class MiniMaxM2Model(TextModel):
@ModelBase.register("MiniMaxM3SparseForCausalLM", "MiniMaxM3SparseForConditionalGeneration")
@ModelBase.example("MiniMaxAI/MiniMax-M3")
class MiniMaxM3Model(MiniMaxM2Model):
model_arch = gguf.MODEL_ARCH.MINIMAXM3
@@ -206,7 +203,6 @@ class MiniMaxM3Model(MiniMaxM2Model):
@ModelBase.register("MiniMaxM3SparseForConditionalGeneration", "MiniMaxM3VLForConditionalGeneration")
@ModelBase.example("MiniMaxAI/MiniMax-M3")
class MiniMaxM3VisionModel(MmprojModel):
@classmethod
def filter_tensors(cls, item):
-1
View File
@@ -15,7 +15,6 @@ from .llama import LlamaModel
"Mistral3ForConditionalGeneration",
"Ministral3ForCausalLM",
)
@ModelBase.example("mistralai/Mistral-Small-3.1-24B-Instruct-2503", "hf-tiny-v2/tiny-random-Ministral3ForCausalLM")
class Mistral3Model(TextModel):
class Ministral3Model(LlamaModel):
model_arch = gguf.MODEL_ARCH.MISTRAL3
-1
View File
@@ -9,7 +9,6 @@ from .base import ModelBase, TextModel, gguf
@ModelBase.register("MPTForCausalLM")
@ModelBase.example("anas-awadalla/mpt-7b")
class MPTModel(TextModel):
model_arch = gguf.MODEL_ARCH.MPT
-3
View File
@@ -24,7 +24,6 @@ def _unpermute_for_rope(tensor: "Tensor", n_heads: int) -> "Tensor":
@ModelBase.register("MuseGlimmerForConditionalGeneration")
@ModelBase.example("meta-models/Muse-Glimmer-30B")
class MuseGlimmerModel(TextModel):
model_arch = gguf.MODEL_ARCH.MUSE_GLIMMER
@@ -79,7 +78,6 @@ class MuseGlimmerModel(TextModel):
@ModelBase.register("MuseGlimmerForConditionalGeneration")
@ModelBase.example("meta-models/Muse-Glimmer-30B")
class MuseGlimmerVisionModel(MmprojModel):
def get_vision_config(self) -> dict[str, Any] | None:
c = self.global_config.get("vision_config")
@@ -133,7 +131,6 @@ class MuseGlimmerVisionModel(MmprojModel):
@ModelBase.register("MuseGlimmerAssistantModel")
@ModelBase.example("meta-models/Muse-Glimmer-30B-assistant")
class MuseGlimmerAssistantModel(TextModel):
model_arch = gguf.MODEL_ARCH.DFLASH
-1
View File
@@ -5,7 +5,6 @@ from .llama import LlamaModel
@ModelBase.register("NanbeigeForCausalLM")
@ModelBase.example("Nanbeige/Nanbeige4.2-3B")
class NanbeigeModel(LlamaModel):
model_arch = gguf.MODEL_ARCH.NANBEIGE
undo_permute = True
-3
View File
@@ -16,7 +16,6 @@ from .granite import GraniteHybridModel
"NemotronH_Nano_VL_V2",
"RADIOModel",
)
@ModelBase.example("nvidia/NVIDIA-Nemotron-Nano-12B-v2-VL-BF16")
class NemotronNanoV2VLModel(MmprojModel):
# ViT-Huge architecture parameters for RADIO v2.5-h
_vit_hidden_size = 1280
@@ -152,7 +151,6 @@ class NemotronNanoV2VLModel(MmprojModel):
@ModelBase.register("NemotronForCausalLM")
@ModelBase.example("nvidia/Minitron-4B-Base")
class NemotronModel(TextModel):
model_arch = gguf.MODEL_ARCH.NEMOTRON
@@ -195,7 +193,6 @@ class NemotronModel(TextModel):
@ModelBase.register("NemotronHForCausalLM")
@ModelBase.example("nvidia/Nemotron-H-8B-Base-8K")
class NemotronHModel(GraniteHybridModel):
"""Hybrid mamba2/attention model from NVIDIA"""
model_arch = gguf.MODEL_ARCH.NEMOTRON_H
-4
View File
@@ -14,7 +14,6 @@ from .llama import LlamaModel
@ModelBase.register("OlmoForCausalLM")
@ModelBase.register("OLMoForCausalLM")
@ModelBase.example("allenai/OLMo-1.7-7B-hf")
class OlmoModel(TextModel):
model_arch = gguf.MODEL_ARCH.OLMO
@@ -40,14 +39,12 @@ class OlmoModel(TextModel):
@ModelBase.register("SeedOssForCausalLM")
@ModelBase.example("ByteDance-Seed/Seed-OSS-36B-Instruct")
class SeedOssModel(TextModel):
model_arch = gguf.MODEL_ARCH.SEED_OSS
@ModelBase.register("Olmo2ForCausalLM")
@ModelBase.register("Olmo3ForCausalLM")
@ModelBase.example("allenai/OLMo-2-1124-7B-Instruct", "allenai/Olmo-3-7B-Instruct")
class Olmo2Model(TextModel):
model_arch = gguf.MODEL_ARCH.OLMO2
@@ -70,7 +67,6 @@ class Olmo2Model(TextModel):
@ModelBase.register("OlmoeForCausalLM")
@ModelBase.example("allenai/OLMoE-1B-7B-0924")
class OlmoeModel(TextModel):
model_arch = gguf.MODEL_ARCH.OLMOE
-1
View File
@@ -9,7 +9,6 @@ from .base import ModelBase, TextModel, gguf
@ModelBase.register("OpenELMForCausalLM")
@ModelBase.example("apple/OpenELM-270M")
class OpenELMModel(TextModel):
model_arch = gguf.MODEL_ARCH.OPENELM
-1
View File
@@ -4,7 +4,6 @@ from .base import ModelBase, TextModel, gguf
@ModelBase.register("OrionForCausalLM")
@ModelBase.example("OrionStarAI/Orion-14B-Base")
class OrionModel(TextModel):
model_arch = gguf.MODEL_ARCH.ORION
-1
View File
@@ -11,7 +11,6 @@ from .base import ModelBase, TextModel, gguf, logger
@ModelBase.register("PanguEmbeddedForCausalLM")
@ModelBase.example("FreedomIntelligence/openPangu-Embedded-7B-V1.1")
class PanguEmbeddedModel(TextModel):
model_arch = gguf.MODEL_ARCH.PANGU_EMBED
-4
View File
@@ -14,7 +14,6 @@ from .base import MmprojModel, ModelBase, SentencePieceTokenTypes, TextModel, gg
@ModelBase.register("PhiForCausalLM")
@ModelBase.example("microsoft/phi-2")
class Phi2Model(TextModel):
model_arch = gguf.MODEL_ARCH.PHI2
@@ -37,7 +36,6 @@ class Phi2Model(TextModel):
@ModelBase.register("Phi3ForCausalLM", "Phi4ForCausalLMV")
@ModelBase.example("microsoft/Phi-3-mini-4k-instruct")
class Phi3MiniModel(TextModel):
model_arch = gguf.MODEL_ARCH.PHI3
@@ -212,7 +210,6 @@ class Phi3MiniModel(TextModel):
@ModelBase.register("Phi4ForCausalLMV")
# [TAG_HF_EXAMPLE_MISSING]
class Phi4VisionMmprojModel(MmprojModel):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
@@ -339,7 +336,6 @@ class Phi4VisionMmprojModel(MmprojModel):
@ModelBase.register("PhiMoEForCausalLM")
@ModelBase.example("microsoft/Phi-3.5-MoE-instruct")
class PhiMoeModel(Phi3MiniModel):
model_arch = gguf.MODEL_ARCH.PHIMOE
-4
View File
@@ -13,7 +13,6 @@ from .base import ModelBase, TextModel, gguf
@ModelBase.register("PlamoForCausalLM")
@ModelBase.example("pfnet/plamo-13b")
class PlamoModel(TextModel):
model_arch = gguf.MODEL_ARCH.PLAMO
@@ -59,7 +58,6 @@ class PlamoModel(TextModel):
@ModelBase.register("Plamo2ForCausalLM", "PLaMo2ForCausalLM")
@ModelBase.example("pfnet/plamo-2-1b")
class Plamo2Model(TextModel):
model_arch = gguf.MODEL_ARCH.PLAMO2
@@ -149,8 +147,6 @@ class Plamo2Model(TextModel):
@ModelBase.register("Plamo3ForCausalLM", "PLaMo3ForCausalLM")
# [TAG_HF_EXAMPLE_GATED] pfnet/plamo-3-nict-2b-base is gated
@ModelBase.example("midorin-Linux/plamo-3-12b-self-merged-base")
class Plamo3Model(TextModel):
model_arch = gguf.MODEL_ARCH.PLAMO3
-1
View File
@@ -4,7 +4,6 @@ from .base import ModelBase, TextModel, gguf
@ModelBase.register("PLMForCausalLM")
@ModelBase.example("PLM-Team/PLM-1.8B-Instruct")
class PLMModel(TextModel):
model_arch = gguf.MODEL_ARCH.PLM
-2
View File
@@ -77,7 +77,6 @@ def _load_hparams(dir_model: Path) -> dict[str, Any]:
@ModelBase.register("PocketTTSModel")
# [TAG_HF_EXAMPLE_MISSING] model is gated, and the checkpoint requires cd to subdir, not supported here
class PocketTTSModel(TextModel):
model_arch = gguf.MODEL_ARCH.POCKETTTS
@@ -175,7 +174,6 @@ class PocketTTSModel(TextModel):
@ModelBase.register("PocketTTSModel")
# [TAG_HF_EXAMPLE_MISSING] model is gated, and the checkpoint requires cd to subdir, not supported here
class PocketTTSMmprojModel(MmprojModel):
has_audio_encoder = True
has_vision_encoder = False
-11
View File
@@ -13,7 +13,6 @@ from .base import ModelBase, TextModel, gguf, logger
@ModelBase.register("QWenLMHeadModel")
@ModelBase.example("Qwen/Qwen-7B")
class QwenModel(TextModel):
model_arch = gguf.MODEL_ARCH.QWEN
@@ -52,7 +51,6 @@ class QwenModel(TextModel):
"AudioFlamingo3ForConditionalGeneration",
"DotsOCRForCausalLM",
)
@ModelBase.example("Qwen/Qwen2.5-7B-Instruct")
class Qwen2Model(TextModel):
model_arch = gguf.MODEL_ARCH.QWEN2
@@ -73,7 +71,6 @@ class Qwen2Model(TextModel):
@ModelBase.register("Qwen2MoeForCausalLM")
@ModelBase.example("Qwen/Qwen1.5-MoE-A2.7B")
class Qwen2MoeModel(TextModel):
model_arch = gguf.MODEL_ARCH.QWEN2MOE
@@ -156,7 +153,6 @@ class Qwen2MoeModel(TextModel):
@ModelBase.register("Qwen3ForCausalLM", "Qwen3Model")
@ModelBase.example("Qwen/Qwen3-8B")
class Qwen3Model(Qwen2Model):
model_arch = gguf.MODEL_ARCH.QWEN3
@@ -255,7 +251,6 @@ class Qwen3Model(Qwen2Model):
@ModelBase.register("Qwen3MoeForCausalLM")
@ModelBase.example("Qwen/Qwen3-30B-A3B")
class Qwen3MoeModel(Qwen2MoeModel):
model_arch = gguf.MODEL_ARCH.QWEN3MOE
@@ -367,7 +362,6 @@ class _QwenMtpMixin:
@ModelBase.register("Qwen3NextForCausalLM")
@ModelBase.example("Qwen/Qwen3-Next-80B-A3B-Instruct")
class Qwen3NextModel(_QwenMtpMixin, Qwen2MoeModel):
model_arch = gguf.MODEL_ARCH.QWEN3NEXT
@@ -427,7 +421,6 @@ class Qwen3NextModel(_QwenMtpMixin, Qwen2MoeModel):
@ModelBase.register("RND1")
@ModelBase.example("radicalnumerics/RND1-Base-0910")
class RND1Model(Qwen2MoeModel):
model_arch = gguf.MODEL_ARCH.RND1
@@ -627,19 +620,16 @@ class _Qwen35MRopeMixin:
@ModelBase.register("Qwen3_5ForConditionalGeneration", "Qwen3_5ForCausalLM")
@ModelBase.example("Qwen/Qwen3.5-9B")
class Qwen3_5TextModel(_Qwen35MRopeMixin, _LinearAttentionVReorderBase):
model_arch = gguf.MODEL_ARCH.QWEN35
@ModelBase.register("Qwen3_5MoeForConditionalGeneration", "Qwen3_5MoeForCausalLM")
@ModelBase.example("Qwen/Qwen3.5-35B-A3B")
class Qwen3_5MoeTextModel(_Qwen35MRopeMixin, _LinearAttentionVReorderBase):
model_arch = gguf.MODEL_ARCH.QWEN35MOE
@ModelBase.register("DFlashDraftModel")
@ModelBase.example("z-lab/Qwen3.5-9B-DFlash")
class DFlashModel(Qwen3Model):
model_arch = gguf.MODEL_ARCH.DFLASH
@@ -709,7 +699,6 @@ class DFlashModel(Qwen3Model):
@ModelBase.register("Qwen3DSparkModel")
@ModelBase.example("satgeze/Qwen3.6-27B-DSpark")
class DSparkModel(DFlashModel):
# DSpark = DFlash + a semi-autoregressive Markov head
model_arch = gguf.MODEL_ARCH.DFLASH
-2
View File
@@ -37,7 +37,6 @@ _ACT2FN = {
@ModelBase.register("Qwen3TTSForConditionalGeneration")
@ModelBase.example("Qwen/Qwen3-TTS-12Hz-1.7B-Base")
class Qwen3TTSTalkerModel(TextModel):
model_arch = gguf.MODEL_ARCH.QWEN3TTS
@@ -186,7 +185,6 @@ class Qwen3TTSTalkerModel(TextModel):
@ModelBase.register("Qwen3TTSForConditionalGeneration")
@ModelBase.example("Qwen/Qwen3-TTS-12Hz-1.7B-Base")
class Qwen3TTSSpeakerEncoderModel(MmprojModel):
has_vision_encoder = False
has_audio_encoder = True
-8
View File
@@ -14,7 +14,6 @@ from .qwenvl import Qwen25AudioModel
@ModelBase.register("Qwen3VLForConditionalGeneration", "Qwen3VLMoeForConditionalGeneration", "Qwen3_5ForConditionalGeneration", "Qwen3_5MoeForConditionalGeneration")
@ModelBase.example("Qwen/Qwen3-VL-4B-Instruct", "Qwen/Qwen3-VL-30B-A3B-Instruct", "Qwen/Qwen3.5-9B", "Qwen/Qwen3.5-35B-A3B")
class Qwen3VLVisionModel(MmprojModel):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
@@ -145,7 +144,6 @@ class Qwen3VLVisionModel(MmprojModel):
@ModelBase.register("Qwen3OmniMoeForConditionalGeneration")
@ModelBase.example("Qwen/Qwen3-Omni-30B-A3B-Instruct")
class Qwen3OmniMmprojModel(Qwen3VLVisionModel, Qwen25AudioModel):
has_audio_encoder = True
has_vision_encoder = True
@@ -219,14 +217,12 @@ class Qwen3OmniMmprojModel(Qwen3VLVisionModel, Qwen25AudioModel):
@ModelBase.register("Qwen3ASRForConditionalGeneration")
@ModelBase.example("Qwen/Qwen3-ASR-0.6B-hf")
class Qwen3ASRMmprojModel(Qwen3OmniMmprojModel):
has_audio_encoder = True
has_vision_encoder = False
@ModelBase.register("Glm4vForConditionalGeneration", "Glm4vMoeForConditionalGeneration", "GlmOcrForConditionalGeneration")
@ModelBase.example("zai-org/GLM-4.1V-9B-Thinking", "zai-org/GLM-4.5V")
class Glm4VVisionModel(Qwen3VLVisionModel):
def set_gguf_parameters(self):
MmprojModel.set_gguf_parameters(self) # skip Qwen3VLVisionModel parameters
@@ -250,7 +246,6 @@ class Glm4VVisionModel(Qwen3VLVisionModel):
@ModelBase.register("Qwen3VLForConditionalGeneration")
@ModelBase.example("Qwen/Qwen3-VL-4B-Instruct")
class Qwen3VLTextModel(Qwen3Model):
model_arch = gguf.MODEL_ARCH.QWEN3VL
@@ -273,7 +268,6 @@ class Qwen3VLTextModel(Qwen3Model):
@ModelBase.register("Qwen3VLMoeForConditionalGeneration")
@ModelBase.example("Qwen/Qwen3-VL-30B-A3B-Instruct")
class Qwen3VLMoeTextModel(Qwen3MoeModel):
model_arch = gguf.MODEL_ARCH.QWEN3VLMOE
@@ -323,7 +317,6 @@ class Qwen3VLMoeTextModel(Qwen3MoeModel):
@ModelBase.register("Qwen3OmniMoeForConditionalGeneration")
@ModelBase.example("Qwen/Qwen3-Omni-30B-A3B-Instruct")
class Qwen3OmniMoeTextModel(Qwen3VLMoeTextModel):
model_arch = gguf.MODEL_ARCH.QWEN3VLMOE
@@ -345,7 +338,6 @@ class Qwen3OmniMoeTextModel(Qwen3VLMoeTextModel):
@ModelBase.register("Qwen3ASRForConditionalGeneration")
@ModelBase.example("Qwen/Qwen3-ASR-0.6B-hf")
class Qwen3ASRTextModel(Qwen3VLTextModel):
model_arch = gguf.MODEL_ARCH.QWEN3VL
-3
View File
@@ -17,7 +17,6 @@ from .base import MmprojModel, ModelBase, TextModel, gguf
"Qwen2_5_VLForConditionalGeneration",
"Qwen2_5OmniModel",
)
@ModelBase.example("Qwen/Qwen2-VL-2B-Instruct", "Qwen/Qwen2.5-VL-3B-Instruct")
class Qwen2VLModel(TextModel):
model_arch = gguf.MODEL_ARCH.QWEN2VL
@@ -41,7 +40,6 @@ class Qwen2VLModel(TextModel):
@ModelBase.register("Qwen2VLModel", "Qwen2VLForConditionalGeneration", "Qwen2_5_VLForConditionalGeneration")
@ModelBase.example("Qwen/Qwen2-VL-2B-Instruct", "Qwen/Qwen2.5-VL-3B-Instruct")
class Qwen2VLVisionModel(MmprojModel):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
@@ -163,7 +161,6 @@ class Qwen25AudioModel(MmprojModel):
@ModelBase.register("Qwen2_5OmniModel")
@ModelBase.example("Qwen/Qwen2.5-Omni-3B")
class Qwen25OmniModel(Qwen2VLVisionModel, Qwen25AudioModel):
has_audio_encoder = True
has_vision_encoder = True
-1
View File
@@ -9,7 +9,6 @@ from .base import ModelBase, TextModel, gguf
@ModelBase.register("GPTRefactForCausalLM")
@ModelBase.example("smallcloudai/Refact-1_6-base")
class RefactModel(TextModel):
model_arch = gguf.MODEL_ARCH.REFACT
-4
View File
@@ -11,7 +11,6 @@ from .base import ModelBase, TextModel, gguf
@ModelBase.register("Rwkv6ForCausalLM")
@ModelBase.example("RWKV/v6-Finch-1B6-HF")
class Rwkv6Model(TextModel):
model_arch = gguf.MODEL_ARCH.RWKV6
@@ -84,7 +83,6 @@ class Rwkv6Model(TextModel):
@ModelBase.register("RWKV6Qwen2ForCausalLM")
@ModelBase.example("recursal/QRWKV6-32B-Instruct-Preview-v0.1")
class RWKV6Qwen2Model(Rwkv6Model):
model_arch = gguf.MODEL_ARCH.RWKV6QWEN2
@@ -138,7 +136,6 @@ class RWKV6Qwen2Model(Rwkv6Model):
@ModelBase.register("Rwkv7ForCausalLM", "RWKV7ForCausalLM")
@ModelBase.example("fla-hub/rwkv7-1.5B-world")
class Rwkv7Model(TextModel):
model_arch = gguf.MODEL_ARCH.RWKV7
@@ -264,7 +261,6 @@ class Rwkv7Model(TextModel):
@ModelBase.register("RwkvHybridForCausalLM")
@ModelBase.example("RWKV-Red-Team/ARWKV-7B-Preview-0.1")
class ARwkv7Model(Rwkv7Model):
model_arch = gguf.MODEL_ARCH.ARWKV7
-2
View File
@@ -12,7 +12,6 @@ from .qwenvl import Qwen2VLVisionModel
@ModelBase.register("Sarashina2VisionForCausalLM")
@ModelBase.example("sbintuitions/sarashina2.2-vision-3b")
class Sarashina2VLTextModel(LlamaModel):
model_arch = gguf.MODEL_ARCH.LLAMA
@@ -27,7 +26,6 @@ class Sarashina2VLTextModel(LlamaModel):
@ModelBase.register("Sarashina2VisionForCausalLM")
@ModelBase.example("sbintuitions/sarashina2.2-vision-3b")
class Sarashina2VLVisionModel(Qwen2VLVisionModel):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
-1
View File
@@ -11,7 +11,6 @@ from .base import ModelBase, TextModel, gguf, logger
@ModelBase.register("SmallThinkerForCausalLM")
@ModelBase.example("PowerInfer/SmallThinker-4BA0.6B-Instruct")
class SmallThinkerModel(TextModel):
model_arch = gguf.MODEL_ARCH.SMALLTHINKER
-1
View File
@@ -9,7 +9,6 @@ from .base import MmprojModel, ModelBase, gguf
@ModelBase.register("Idefics3ForConditionalGeneration", "SmolVLMForConditionalGeneration")
@ModelBase.example("HuggingFaceTB/SmolVLM-Instruct", "HuggingFaceM4/Idefics3-8B-Llama3")
class SmolVLMModel(MmprojModel):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
-1
View File
@@ -11,7 +11,6 @@ from .base import ModelBase, TextModel, gguf
@ModelBase.register("StableLmForCausalLM", "StableLMEpochForCausalLM", "LlavaStableLMEpochForCausalLM")
@ModelBase.example("stabilityai/stablelm-2-1_6b")
class StableLMModel(TextModel):
model_arch = gguf.MODEL_ARCH.STABLELM
-2
View File
@@ -4,7 +4,6 @@ from .base import ModelBase, TextModel, gguf
@ModelBase.register("GPTBigCodeForCausalLM")
@ModelBase.example("bigcode/gpt_bigcode-santacoder")
class StarCoderModel(TextModel):
model_arch = gguf.MODEL_ARCH.STARCODER
@@ -20,6 +19,5 @@ class StarCoderModel(TextModel):
@ModelBase.register("Starcoder2ForCausalLM")
@ModelBase.example("bigcode/starcoder2-3b")
class StarCoder2Model(TextModel):
model_arch = gguf.MODEL_ARCH.STARCODER2
-3
View File
@@ -16,7 +16,6 @@ from .qwen import Qwen3Model
@ModelBase.register("StepVLForConditionalGeneration", "Step3p7ForConditionalGeneration")
@ModelBase.example("stepfun-ai/Step3-VL-10B", "stepfun-ai/Step-3.7-Flash")
class Step3VLVisionModel(MmprojModel):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
@@ -92,13 +91,11 @@ class Step3VLVisionModel(MmprojModel):
@ModelBase.register("StepVLForConditionalGeneration")
@ModelBase.example("stepfun-ai/Step3-VL-10B")
class Step3VLTextModel(Qwen3Model):
model_arch = gguf.MODEL_ARCH.QWEN3
@ModelBase.register("Step3p5ForCausalLM", "Step3p7ForConditionalGeneration")
@ModelBase.example("stepfun-ai/Step-3.7-Flash")
class Step35Model(TextModel):
model_arch = gguf.MODEL_ARCH.STEP35
supports_mtp_export = True
-2
View File
@@ -16,7 +16,6 @@ from .base import ModelBase, SentencePieceTokenTypes, TextModel, gguf, logger
@ModelBase.register("MT5ForConditionalGeneration")
@ModelBase.register("UMT5ForConditionalGeneration")
@ModelBase.register("UMT5Model")
@ModelBase.example("google-t5/t5-small", "google/flan-t5-small", "google/umt5-small")
class T5Model(TextModel):
model_arch = gguf.MODEL_ARCH.T5
@@ -154,7 +153,6 @@ class T5Model(TextModel):
@ModelBase.register("T5EncoderModel")
@ModelBase.example("sentence-transformers/sentence-t5-base")
class T5EncoderModel(TextModel):
model_arch = gguf.MODEL_ARCH.T5ENCODER
-1
View File
@@ -11,7 +11,6 @@ from .base import LazyTorchTensor, ModelBase, TextModel, gguf
@ModelBase.register("TalkieForCausalLM")
@ModelBase.example("lewtun/talkie-1930-13b-it-hf")
class TalkieModel(TextModel):
model_arch = gguf.MODEL_ARCH.TALKIE
-7
View File
@@ -9,7 +9,6 @@ from .base import MmprojModel, ModelBase, TextModel, gguf
@ModelBase.register("UltravoxModel")
@ModelBase.example("fixie-ai/ultravox-v0_5-llama-3_2-1b")
class UltravoxModel(TextModel):
model_arch = gguf.MODEL_ARCH.LLAMA # dummy
@@ -19,7 +18,6 @@ class UltravoxModel(TextModel):
@ModelBase.register("GlmasrModel")
@ModelBase.example("zai-org/GLM-ASR-Nano-2512")
class GlmASRWhisperEncoderModel(MmprojModel):
has_vision_encoder = False
has_audio_encoder = True
@@ -84,7 +82,6 @@ class GlmASRWhisperEncoderModel(MmprojModel):
@ModelBase.register("Qwen2AudioForConditionalGeneration")
@ModelBase.example("Qwen/Qwen2-Audio-7B-Instruct")
class WhisperEncoderModel(MmprojModel):
has_vision_encoder = False # no vision encoder
has_audio_encoder = True
@@ -126,7 +123,6 @@ class WhisperEncoderModel(MmprojModel):
@ModelBase.register("UltravoxModel")
@ModelBase.example("fixie-ai/ultravox-v0_5-llama-3_2-1b")
class UltravoxWhisperEncoderModel(WhisperEncoderModel):
has_vision_encoder = False # no vision encoder
has_audio_encoder = True
@@ -138,7 +134,6 @@ class UltravoxWhisperEncoderModel(WhisperEncoderModel):
@ModelBase.register("MERaLiON2ForConditionalGeneration")
@ModelBase.example("MERaLiON/MERaLiON-2-3B")
class MERaLiONWhisperEncoderModel(WhisperEncoderModel):
has_vision_encoder = False
has_audio_encoder = True
@@ -185,7 +180,6 @@ class MERaLiONWhisperEncoderModel(WhisperEncoderModel):
@ModelBase.register("VoxtralForConditionalGeneration")
@ModelBase.example("mistralai/Voxtral-Mini-3B-2507")
class VoxtralWhisperEncoderModel(WhisperEncoderModel):
has_vision_encoder = False # no vision encoder
has_audio_encoder = True
@@ -197,7 +191,6 @@ class VoxtralWhisperEncoderModel(WhisperEncoderModel):
@ModelBase.register("AudioFlamingo3ForConditionalGeneration")
@ModelBase.example("nvidia/audio-flamingo-3-hf")
class AudioFlamingo3WhisperEncoderModel(WhisperEncoderModel):
def set_gguf_parameters(self):
super().set_gguf_parameters()
-1
View File
@@ -9,7 +9,6 @@ from .base import ModelBase, TextModel, gguf, logger
@ModelBase.register("WavTokenizerDec")
@ModelBase.example("novateur/WavTokenizer-large-speech-75token")
class WavTokenizerDecModel(TextModel):
model_arch = gguf.MODEL_ARCH.WAVTOKENIZER_DEC
-1
View File
@@ -11,7 +11,6 @@ from .base import ModelBase, TextModel, gguf
@ModelBase.register("XverseForCausalLM")
@ModelBase.example("xverse/XVERSE-7B")
class XverseModel(TextModel):
model_arch = gguf.MODEL_ARCH.XVERSE
-1
View File
@@ -9,7 +9,6 @@ from .base import MmprojModel, ModelBase, gguf, logger
@ModelBase.register("YoutuVLForConditionalGeneration")
@ModelBase.example("tencent/Youtu-VL-4B-Instruct")
class YoutuVLVisionModel(MmprojModel):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
-4
View File
@@ -29,7 +29,6 @@ The required steps to implement for an HF model are:
```python
@ModelBase.register("MyModelForCausalLM")
@ModelBase.example("user/model")
class MyModel(TextModel):
model_arch = gguf.MODEL_ARCH.MYMODEL
```
@@ -38,13 +37,10 @@ or
```python
@ModelBase.register("MyModelForConditionalGeneration")
@ModelBase.example("user/model")
class MyModel(MmprojModel):
model_arch = gguf.MODEL_ARCH.MYMODEL
```
The `example` should point to a valid Hugging Face model that will be used for testing. You can add multiple models if necessary. Prefer a non-gated model, or tiny random weights if no such model exists.
2. Define the layout of the GGUF tensors in [constants.py](/gguf-py/gguf/constants.py)
Add an enum entry in `MODEL_ARCH`, the model human friendly name in `MODEL_ARCH_NAMES` and the GGUF tensor names in `MODEL_TENSORS`.
+1 -50
View File
@@ -261,7 +261,6 @@ class Keys:
class KDA:
HEAD_DIM = "{arch}.kda.head_dim"
SAFE_GATE = "{arch}.kda.safe_gate"
GATE_LOWER_BOUND = "{arch}.kda.gate_lower_bound"
class WKV:
@@ -553,7 +552,6 @@ class MODEL_ARCH(IntEnum):
PLM = auto()
BAILINGMOE = auto()
BAILINGMOE2 = auto()
BAILINGMOE3 = auto()
DOTS1 = auto()
ARCEE = auto()
AFMOE = auto()
@@ -1269,7 +1267,6 @@ MODEL_ARCH_NAMES: dict[MODEL_ARCH, str] = {
MODEL_ARCH.PLM: "plm",
MODEL_ARCH.BAILINGMOE: "bailingmoe",
MODEL_ARCH.BAILINGMOE2: "bailingmoe2",
MODEL_ARCH.BAILINGMOE3: "bailingmoe3",
MODEL_ARCH.DOTS1: "dots1",
MODEL_ARCH.ARCEE: "arcee",
MODEL_ARCH.AFMOE: "afmoe",
@@ -4237,50 +4234,6 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = {
MODEL_TENSOR.NEXTN_SHARED_HEAD_NORM,
MODEL_TENSOR.LAYER_OUT_NORM,
],
MODEL_ARCH.BAILINGMOE3: [
MODEL_TENSOR.TOKEN_EMBD,
MODEL_TENSOR.OUTPUT_NORM,
MODEL_TENSOR.OUTPUT,
MODEL_TENSOR.ATTN_NORM,
MODEL_TENSOR.ATTN_Q,
MODEL_TENSOR.ATTN_Q_A,
MODEL_TENSOR.ATTN_Q_B,
MODEL_TENSOR.ATTN_Q_A_NORM,
MODEL_TENSOR.ATTN_K,
MODEL_TENSOR.ATTN_V,
MODEL_TENSOR.ATTN_OUT,
MODEL_TENSOR.ATTN_GATE,
MODEL_TENSOR.ATTN_KV_A_MQA,
MODEL_TENSOR.ATTN_KV_B,
MODEL_TENSOR.ATTN_K_B,
MODEL_TENSOR.ATTN_V_B,
MODEL_TENSOR.ATTN_KV_A_NORM,
MODEL_TENSOR.FFN_NORM,
MODEL_TENSOR.FFN_GATE,
MODEL_TENSOR.FFN_DOWN,
MODEL_TENSOR.FFN_UP,
MODEL_TENSOR.FFN_GATE_INP,
MODEL_TENSOR.FFN_GATE_EXP,
MODEL_TENSOR.FFN_DOWN_EXP,
MODEL_TENSOR.FFN_UP_EXP,
MODEL_TENSOR.FFN_GATE_SHEXP,
MODEL_TENSOR.FFN_DOWN_SHEXP,
MODEL_TENSOR.FFN_UP_SHEXP,
MODEL_TENSOR.FFN_EXP_PROBS_B,
MODEL_TENSOR.SSM_CONV1D_Q,
MODEL_TENSOR.SSM_CONV1D_K,
MODEL_TENSOR.SSM_CONV1D_V,
MODEL_TENSOR.SSM_F_A,
MODEL_TENSOR.SSM_BETA,
MODEL_TENSOR.SSM_A,
MODEL_TENSOR.SSM_G_A,
MODEL_TENSOR.SSM_DT,
MODEL_TENSOR.SSM_NORM,
MODEL_TENSOR.NEXTN_EH_PROJ,
MODEL_TENSOR.NEXTN_ENORM,
MODEL_TENSOR.NEXTN_HNORM,
MODEL_TENSOR.LAYER_OUT_NORM,
],
MODEL_ARCH.DOTS1: [
MODEL_TENSOR.TOKEN_EMBD,
MODEL_TENSOR.OUTPUT_NORM,
@@ -5534,9 +5487,7 @@ KEY_SSM_GROUP_COUNT = Keys.SSM.GROUP_COUNT
KEY_SSM_DT_B_C_RMS = Keys.SSM.DT_B_C_RMS
# KDA
KEY_KDA_HEAD_DIM = Keys.KDA.HEAD_DIM
KEY_KDA_SAFE_GATE = Keys.KDA.SAFE_GATE
KEY_KDA_GATE_LOWER_BOUND = Keys.KDA.GATE_LOWER_BOUND
KEY_KDA_HEAD_DIM = Keys.KDA.HEAD_DIM
# tokenization
KEY_TOKENIZER_MODEL = Keys.Tokenizer.MODEL
+3 -6
View File
@@ -1103,6 +1103,9 @@ class GGUFWriter:
def add_ssm_dt_b_c_rms(self, value: bool) -> None:
self.add_bool(Keys.SSM.DT_B_C_RMS.format(arch=self.arch), value)
def add_kda_gate_lower_bound(self, value: float) -> None:
self.add_float32(Keys.KDA.GATE_LOWER_BOUND.format(arch=self.arch), value)
def add_expert_latent_length(self, value: int) -> None:
self.add_uint32(Keys.LLM.EXPERT_LATENT_LENGTH.format(arch=self.arch), value)
@@ -1118,12 +1121,6 @@ class GGUFWriter:
def add_kda_head_dim(self, value: int) -> None:
self.add_uint32(Keys.KDA.HEAD_DIM.format(arch=self.arch), value)
def add_kda_safe_gate(self, value: bool) -> None:
self.add_bool(Keys.KDA.SAFE_GATE.format(arch=self.arch), value)
def add_kda_gate_lower_bound(self, value: float) -> None:
self.add_float32(Keys.KDA.GATE_LOWER_BOUND.format(arch=self.arch), value)
def add_tokenizer_model(self, model: str) -> None:
self.add_string(Keys.Tokenizer.MODEL, model)
-20
View File
@@ -255,7 +255,6 @@ class TensorNameMap:
# Attention query
MODEL_TENSOR.ATTN_Q: (
"model.layers.{bid}.self_attn.q_proj", # llama-hf nemotron olmoe olmo2 phimoe
"model.layers.{bid}.attention.q_proj", # bailingmoe3
"layers.{bid}.self_attn.q_proj", # embeddinggemma
"model.layers.{bid}.self_attn.q_proj_no_perm", # llama-custom
"layers.{bid}.attention.wq", # llama-pth
@@ -276,7 +275,6 @@ class TensorNameMap:
# Attention key
MODEL_TENSOR.ATTN_K: (
"model.layers.{bid}.self_attn.k_proj", # llama-hf nemotron olmoe olmo2 phimoe
"model.layers.{bid}.attention.k_proj", # bailingmoe3
"layers.{bid}.self_attn.k_proj", # embeddinggemma
"model.layers.{bid}.self_attn.k_proj_no_perm", # llama-custom
"layers.{bid}.attention.wk", # llama-pth
@@ -298,7 +296,6 @@ class TensorNameMap:
# Attention value
MODEL_TENSOR.ATTN_V: (
"model.layers.{bid}.self_attn.v_proj", # llama-hf nemotron olmoe olmo2 phimoe
"model.layers.{bid}.attention.v_proj", # bailingmoe3
"layers.{bid}.self_attn.v_proj", # embeddinggemma
"layers.{bid}.attention.wv", # llama-pth
"encoder.layer.{bid}.attention.self.value", # bert
@@ -324,8 +321,6 @@ class TensorNameMap:
"transformer.h.{bid}.self_attention.dense", # falcon
"h.{bid}.self_attention.dense", # bloom
"model.layers.{bid}.self_attn.o_proj", # llama-hf nemotron olmoe olmo2 phimoe
"model.layers.{bid}.attention.o_proj", # bailingmoe3
"model.layers.{bid}.attention.dense", # bailingmoe3 MLA
"layers.{bid}.self_attn.o_proj", # embeddinggemma
"model.layers.{bid}.self_attn.out_proj", # lfm2 minimax-01
"model.layers.{bid}.self_attn.linear_attn", # deci
@@ -839,7 +834,6 @@ class TensorNameMap:
"model.layers.{bid}.linear_attn.dt_proj", # qwen3next
"backbone.layers.{bid}.mixer.dt", # nemotron-h-moe
"model.layers.{bid}.self_attn.dt_proj", # kimi
"model.layers.{bid}.attention.dt_proj", # bailingmoe3
),
MODEL_TENSOR.SSM_DT_NORM: (
@@ -854,7 +848,6 @@ class TensorNameMap:
"model.layers.layers.{bid}.mixer.A_log", # plamo2
"model.layers.{bid}.linear_attn.A_log", # qwen3next
"model.layers.{bid}.self_attn.A_log", # kimi
"model.layers.{bid}.attention.A_log", # bailingmoe3
),
MODEL_TENSOR.SSM_B_NORM: (
@@ -881,7 +874,6 @@ class TensorNameMap:
"model.layers.{bid}.linear_attn.norm", # qwen3next
"backbone.layers.{bid}.mixer.norm", # mamba2
"model.layers.{bid}.self_attn.o_norm", # kimi
"model.layers.{bid}.attention.o_norm", # bailingmoe3
),
MODEL_TENSOR.SSM_OUT: (
@@ -903,15 +895,12 @@ class TensorNameMap:
# Kimi Linear KDA (using SSM_ prefix for consistency)
MODEL_TENSOR.SSM_CONV1D_Q: (
"model.layers.{bid}.self_attn.q_conv1d",
"model.layers.{bid}.attention.q_conv1d",
),
MODEL_TENSOR.SSM_CONV1D_K: (
"model.layers.{bid}.self_attn.k_conv1d",
"model.layers.{bid}.attention.k_conv1d",
),
MODEL_TENSOR.SSM_CONV1D_V: (
"model.layers.{bid}.self_attn.v_conv1d",
"model.layers.{bid}.attention.v_conv1d",
),
MODEL_TENSOR.SSM_F_A: (
"model.layers.{bid}.self_attn.f_a_proj",
@@ -922,7 +911,6 @@ class TensorNameMap:
MODEL_TENSOR.SSM_BETA: (
"model.layers.{bid}.linear_attn.in_proj_b", # qwen3.5
"model.layers.{bid}.self_attn.b_proj", # Kimi Linear
"model.layers.{bid}.attention.b_proj", # bailingmoe3
),
# Kimi K3 latent MoE: routed experts operate in a down-projected space
MODEL_TENSOR.FFN_ROUTED_DOWN: (
@@ -1115,48 +1103,40 @@ class TensorNameMap:
MODEL_TENSOR.ATTN_Q_A: (
"model.layers.{bid}.self_attn.q_a_proj", # deepseek2
"model.layers.{bid}.attention.q_a_proj", # bailingmoe3 (Ling-3.0-tiny)
"layers.{bid}.attention.wq_a", # mistral-large
),
MODEL_TENSOR.ATTN_Q_B: (
"model.layers.{bid}.self_attn.q_b_proj", # deepseek2
"model.layers.{bid}.attention.q_b_proj", # bailingmoe3 (Ling-3.0-tiny)
"layers.{bid}.attention.wq_b", # mistral-large
),
MODEL_TENSOR.ATTN_KV_A_MQA: (
"model.layers.{bid}.self_attn.kv_a_proj_with_mqa", # deepseek2
"model.layers.{bid}.attention.kv_a_proj_with_mqa", # bailingmoe3
"layers.{bid}.attention.wkv_a_with_mqa", # mistral-large
),
MODEL_TENSOR.ATTN_KV_B: (
"model.layers.{bid}.self_attn.kv_b_proj", # deepseek2
"model.layers.{bid}.attention.kv_b_proj", # bailingmoe3
),
MODEL_TENSOR.ATTN_K_B: (
"model.layers.{bid}.self_attn.k_b_proj", # deepseek2
"model.layers.{bid}.attention.k_b_proj", # bailingmoe3
"layers.{bid}.attention.k_b_proj", # mistral-large
),
MODEL_TENSOR.ATTN_V_B: (
"model.layers.{bid}.self_attn.v_b_proj", # deepseek2
"model.layers.{bid}.attention.v_b_proj", # bailingmoe3
"layers.{bid}.attention.v_b_proj", # mistral-large
),
MODEL_TENSOR.ATTN_Q_A_NORM: (
"model.layers.{bid}.self_attn.q_a_layernorm", # deepseek2
"model.layers.{bid}.attention.q_a_layernorm", # bailingmoe3 (Ling-3.0-tiny)
"layers.{bid}.attention.q_a_norm", # mistral-large
),
MODEL_TENSOR.ATTN_KV_A_NORM: (
"model.layers.{bid}.self_attn.kv_a_layernorm", # deepseek2
"model.layers.{bid}.attention.kv_a_layernorm", # bailingmoe3
"layers.{bid}.attention.kv_a_norm", # mistral-large
),
+1 -5
View File
@@ -107,7 +107,6 @@ static const std::map<llm_arch, const char *> LLM_ARCH_NAMES = {
{ LLM_ARCH_PLM, "plm" },
{ LLM_ARCH_BAILINGMOE, "bailingmoe" },
{ LLM_ARCH_BAILINGMOE2, "bailingmoe2" },
{ LLM_ARCH_BAILINGMOE3, "bailingmoe3" },
{ LLM_ARCH_DOTS1, "dots1" },
{ LLM_ARCH_ARCEE, "arcee" },
{ LLM_ARCH_AFMOE, "afmoe" },
@@ -318,8 +317,7 @@ static const std::map<llm_kv, const char *> LLM_KV_NAMES = {
{ LLM_KV_SSM_GROUP_COUNT, "%s.ssm.group_count" },
{ LLM_KV_SSM_DT_B_C_RMS, "%s.ssm.dt_b_c_rms" },
{ LLM_KV_KDA_HEAD_DIM, "%s.kda.head_dim" },
{ LLM_KV_KDA_SAFE_GATE, "%s.kda.safe_gate" },
{ LLM_KV_KDA_HEAD_DIM, "%s.kda.head_dim" },
{ LLM_KV_KDA_GATE_LOWER_BOUND, "%s.kda.gate_lower_bound" },
{ LLM_KV_WKV_HEAD_SIZE, "%s.wkv.head_size" },
@@ -998,7 +996,6 @@ bool llm_arch_is_hybrid(const llm_arch & arch) {
case LLM_ARCH_NEMOTRON_H_MOE:
case LLM_ARCH_QWEN3NEXT:
case LLM_ARCH_KIMI_LINEAR:
case LLM_ARCH_BAILINGMOE3:
case LLM_ARCH_KIMI_K3:
case LLM_ARCH_QWEN35:
case LLM_ARCH_QWEN35MOE:
@@ -1064,7 +1061,6 @@ bool llm_arch_supports_sm_tensor(const llm_arch & arch) {
case LLM_ARCH_MINIMAX_M3:
case LLM_ARCH_MISTRAL4:
case LLM_ARCH_KIMI_LINEAR:
case LLM_ARCH_BAILINGMOE3:
case LLM_ARCH_KIMI_K3:
case LLM_ARCH_QWEN3TTS:
return false;
-2
View File
@@ -112,7 +112,6 @@ enum llm_arch {
LLM_ARCH_PLM,
LLM_ARCH_BAILINGMOE,
LLM_ARCH_BAILINGMOE2,
LLM_ARCH_BAILINGMOE3,
LLM_ARCH_DOTS1,
LLM_ARCH_ARCEE,
LLM_ARCH_AFMOE,
@@ -324,7 +323,6 @@ enum llm_kv {
LLM_KV_SSM_DT_B_C_RMS,
LLM_KV_KDA_HEAD_DIM,
LLM_KV_KDA_SAFE_GATE,
LLM_KV_KDA_GATE_LOWER_BOUND,
LLM_KV_WKV_HEAD_SIZE,
-1
View File
@@ -2298,7 +2298,6 @@ uint32_t llama_context::graph_max_nodes(uint32_t n_tokens) const {
res = std::max<uint32_t>(n_tokens * 160, 64u * model.n_tensors());
} else if (model.arch == LLM_ARCH_QWEN3NEXT ||
model.arch == LLM_ARCH_KIMI_LINEAR ||
model.arch == LLM_ARCH_BAILINGMOE3 ||
model.arch == LLM_ARCH_QWEN35 ||
model.arch == LLM_ARCH_QWEN35MOE ||
model.arch == LLM_ARCH_DEEPSEEK4 ||
-1
View File
@@ -170,7 +170,6 @@ struct llama_hparams {
// for Kimi Linear KDA
uint32_t n_embd_head_kda = 0;
bool kda_safe_gate = false;
// kimi-k3
uint32_t n_expert_latent = 0; // routed_expert_hidden_size (0 = experts run at n_embd)
+2 -6
View File
@@ -121,7 +121,6 @@ void llama_model_saver::add_kv(const enum llm_kv key, const Container & value, c
}
// instantiate for external usage:
template void llama_model_saver::add_kv<std::vector<uint32_t>>(const enum llm_kv, const std::vector<uint32_t> &, const bool);
template void llama_model_saver::add_kv<std::vector<float>>(const enum llm_kv, const std::vector<float> &, const bool);
void llama_model_saver::add_kv(const enum llm_kv key, const std::vector<std::string> & value) {
std::vector<const char *> tmp(value.size());
@@ -217,10 +216,8 @@ void llama_model_saver::add_kv_from_model() {
add_kv(LLM_KV_EXPERT_LATENT_LENGTH, hparams.n_expert_latent);
add_kv(LLM_KV_EXPERT_SHARED_FEED_FORWARD_LENGTH, hparams.n_ff_shexp);
add_kv(LLM_KV_EXPERT_CHUNK_FEED_FORWARD_LENGTH, hparams.n_ff_chexp);
add_kv(LLM_KV_SWIGLU_CLAMP_EXP, std::vector<float>(
hparams.swiglu_clamp_exp.begin(), hparams.swiglu_clamp_exp.begin() + hparams.n_layer_all));
add_kv(LLM_KV_SWIGLU_CLAMP_SHEXP, std::vector<float>(
hparams.swiglu_clamp_shexp.begin(), hparams.swiglu_clamp_shexp.begin() + hparams.n_layer_all));
add_kv(LLM_KV_SWIGLU_CLAMP_EXP, hparams.swiglu_clamp_exp);
add_kv(LLM_KV_SWIGLU_CLAMP_SHEXP, hparams.swiglu_clamp_shexp);
add_kv(LLM_KV_USE_PARALLEL_RESIDUAL, hparams.use_par_res);
// add_kv(LLM_KV_TENSOR_DATA_LAYOUT, ???);
add_kv(LLM_KV_EXPERT_COUNT, hparams.n_expert);
@@ -323,7 +320,6 @@ void llama_model_saver::add_kv_from_model() {
add_kv(LLM_KV_SSM_DT_B_C_RMS, hparams.ssm_dt_b_c_rms);
add_kv(LLM_KV_KDA_HEAD_DIM, hparams.n_embd_head_kda);
add_kv(LLM_KV_KDA_SAFE_GATE, hparams.kda_safe_gate);
add_kv(LLM_KV_KDA_GATE_LOWER_BOUND, hparams.kda_gate_lower_bound);
add_kv(LLM_KV_WKV_HEAD_SIZE, hparams.wkv_head_size);
+4 -9
View File
@@ -256,8 +256,6 @@ static llama_model * llama_model_mapping(llm_arch arch, const llama_model_params
return new llama_model_bailingmoe(params);
case LLM_ARCH_BAILINGMOE2:
return new llama_model_bailingmoe2(params);
case LLM_ARCH_BAILINGMOE3:
return new llama_model_bailingmoe3(params);
case LLM_ARCH_SEED_OSS:
return new llama_model_seed_oss(params);
case LLM_ARCH_DOTS1:
@@ -823,7 +821,6 @@ const char * llm_type_name(llm_type type) {
case LLM_TYPE_A13B: return "A13B";
case LLM_TYPE_7B_A1B: return "7B.A1B";
case LLM_TYPE_8B_A1B: return "8B.A1B";
case LLM_TYPE_7_9B_A1_3B: return "7.9B.A1.3B";
case LLM_TYPE_12B_A2_5B: return "12B.A2.5B";
case LLM_TYPE_16B_A1B: return "16B.A1B";
case LLM_TYPE_21B_A3B: return "21B.A3B";
@@ -840,7 +837,6 @@ const char * llm_type_name(llm_type type) {
case LLM_TYPE_118B_A8B: return "118B.A8B";
case LLM_TYPE_120B_A12B: return "120B.A12B";
case LLM_TYPE_122B_A10B: return "122B.A10B";
case LLM_TYPE_124B_A5_1B: return "124B.A5.1B";
case LLM_TYPE_196B_A11B: return "196B.A11B";
case LLM_TYPE_230B_A10B: return "230B.A10B";
case LLM_TYPE_428B_A23B: return "428B.A23B";
@@ -1964,7 +1960,7 @@ void llama_model::print_info() const {
LLAMA_LOG_INFO("%s: expert_weights_norm = %d\n", __func__, hparams.expert_weights_norm);
}
if (arch == LLM_ARCH_BAILINGMOE2 || arch == LLM_ARCH_BAILINGMOE3) {
if (arch == LLM_ARCH_BAILINGMOE2) {
LLAMA_LOG_INFO("%s: n_layer_dense_lead = %d\n", __func__, hparams.n_layer_dense_lead);
LLAMA_LOG_INFO("%s: n_ff_exp = %d\n", __func__, hparams.n_ff_exp);
LLAMA_LOG_INFO("%s: n_ff_shexp = %d\n", __func__, hparams.n_ff_shexp);
@@ -2259,11 +2255,11 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params,
// checks
default:
{
// Dense MTP heads use a plain attention KV cache instead of the hybrid wrapper.
// The MTP head is dense-attention only on hybrid Qwen3-Next/3.5/3.6, so use a plain
// attention KV cache for the MTP context instead of the hybrid wrapper.
const bool mtp_on_hybrid_qwen =
params.ctx_type == LLAMA_CONTEXT_TYPE_MTP &&
(arch == LLM_ARCH_QWEN3NEXT || arch == LLM_ARCH_QWEN35 || arch == LLM_ARCH_QWEN35MOE ||
arch == LLM_ARCH_BAILINGMOE3);
(arch == LLM_ARCH_QWEN3NEXT || arch == LLM_ARCH_QWEN35 || arch == LLM_ARCH_QWEN35MOE);
const bool mtp_on_hybrid_nemotron =
params.ctx_type == LLAMA_CONTEXT_TYPE_MTP && arch == LLM_ARCH_NEMOTRON_H_MOE;
@@ -2641,7 +2637,6 @@ llama_rope_type llama_model_rope_type(const llama_model * model) {
case LLM_ARCH_GRANITE_SWITCH:
case LLM_ARCH_CHAMELEON:
case LLM_ARCH_BAILINGMOE:
case LLM_ARCH_BAILINGMOE3:
case LLM_ARCH_NEO_BERT:
case LLM_ARCH_SMOLLM3:
case LLM_ARCH_ARCEE:
-2
View File
@@ -118,7 +118,6 @@ enum llm_type {
LLM_TYPE_A13B,
LLM_TYPE_7B_A1B,
LLM_TYPE_8B_A1B, // lfm2moe
LLM_TYPE_7_9B_A1_3B, // Ling-3.0-tiny
LLM_TYPE_12B_A2_5B,
LLM_TYPE_16B_A1B,
LLM_TYPE_21B_A3B, // Ernie MoE small
@@ -135,7 +134,6 @@ enum llm_type {
LLM_TYPE_118B_A8B, // Laguna-S-2
LLM_TYPE_120B_A12B, // Nemotron 3 Super
LLM_TYPE_122B_A10B, // Qwen3.5
LLM_TYPE_124B_A5_1B, // Ling-3.0-flash
LLM_TYPE_196B_A11B, // Step3.5-Flash
LLM_TYPE_230B_A10B, // Minimax M2
LLM_TYPE_428B_A23B, // Minimax M3

Some files were not shown because too many files have changed in this diff Show More