mtmd: support multi-row batching for deepseek-ocr

conversion/__init__.py

@@ -288,7 +288,6 @@ MMPROJ_MODEL_MAP: dict[str, str] = {
     "LlavaForConditionalGeneration": "llava",
     "MERaLiON2ForConditionalGeneration": "ultravox",
     "MiMoV2ForCausalLM": "mimo",
-    "MiniMaxM3SparseForConditionalGeneration": "minimax",
     "MiniCPMV4_6ForConditionalGeneration": "minicpm",
     "Mistral3ForConditionalGeneration": "llava",
     "NemotronH_Nano_VL_V2": "nemotron",

conversion/minimax.py

@@ -7,7 +7,7 @@ import torch
 if TYPE_CHECKING:
     from torch import Tensor
 
-from .base import ModelBase, TextModel, MmprojModel, gguf
+from .base import ModelBase, TextModel, gguf
 
 
 @ModelBase.register("MiniMaxM2ForCausalLM")
@@ -92,78 +92,3 @@ class MiniMaxM3Model(MiniMaxM2Model):
             data_torch = data_torch + 1.0
 
         yield from super().modify_tensors(data_torch, name, bid)
-
-
-@ModelBase.register("MiniMaxM3SparseForConditionalGeneration", "MiniMaxM3VLForConditionalGeneration")
-class MiniMaxM3VisionModel(MmprojModel):
-    @classmethod
-    def filter_tensors(cls, item):
-        name, gen = item
-        # keep only the vision-side tensors; text / mtp / sparse-index are dropped
-        if not name.startswith(("vision_tower.", "multi_modal_projector.", "patch_merge_mlp.")):
-            return None
-        return super().filter_tensors((name, gen))
-
-    def set_gguf_parameters(self):
-        super().set_gguf_parameters()
-        assert self.hparams_vision is not None
-
-        self.gguf_writer.add_clip_projector_type(gguf.VisionProjectorType.MINIMAXM3)
-        self.gguf_writer.add_vision_use_gelu(True)
-
-        # the ViT carries its own LayerNorm eps (text tower uses a different one)
-        self.gguf_writer.add_vision_attention_layernorm_eps(
-            self.hparams_vision.get("layer_norm_eps", 1e-5)
-        )
-
-        comp = self.hparams_vision.get("img_token_compression_config", {})
-        merge_size = comp.get("spatial_merge_size", 2)
-        self.gguf_writer.add_vision_spatial_merge_size(int(merge_size))
-
-    def modify_tensors(self, data_torch, name, bid):
-        assert self.hparams_vision is not None
-
-        # Conv3d patch embed -> Conv2d slices
-        if name == "vision_tower.vision_model.embeddings.patch_embedding.weight":
-            if data_torch.ndim != 5:
-                raise ValueError(f"unexpected patch_embedding rank {data_torch.ndim} for {name}")
-            kt = data_torch.shape[2]
-            base = gguf.TENSOR_NAMES[gguf.MODEL_TENSOR.V_ENC_EMBD_PATCH]
-            for t in range(kt):
-                suffix = ".weight" if t == 0 else f".weight.{t}"
-                yield (base + suffix, data_torch[:, :, t, ...])
-            return
-
-        # Permute ViT q/k. HF [Ta Ha Wa | Tb Hb Wb | pad] reorder to [Ta Tb | Ha Hb | Wa Wb | pad].
-        for new_name, tensor in super().modify_tensors(data_torch, name, bid):
-            if ".attn_q." in new_name or ".attn_k." in new_name:
-                tensor = self._permute_vit_qk(tensor, new_name)
-            yield new_name, tensor
-
-    def _permute_vit_qk(self, t: "Tensor", new_name: str) -> "Tensor":
-        assert self.hparams_vision is not None
-        n_head = self.hparams_vision["num_attention_heads"]
-        d_head = t.shape[0] // n_head
-        axis_dim = 2 * ((2 * (d_head // 2) // 3) // 2)
-        ah = axis_dim // 2
-        half = 3 * ah
-        perm = []
-        perm += list(range(0, ah))
-        perm += list(range(half, half + ah))
-        perm += list(range(ah, 2 * ah))
-        perm += list(range(half + ah, half + 2 * ah))
-        perm += list(range(2 * ah, 3 * ah))
-        perm += list(range(half + 2 * ah, half + 3 * ah))
-        perm += list(range(2 * half, d_head))
-
-        assert axis_dim % 2 == 0
-        assert 3 * axis_dim <= d_head
-        assert len(perm) == d_head
-        assert sorted(perm) == list(range(d_head)), "perm is not a bijection of d_head"
-        assert t.shape[0] == n_head * d_head, f"{new_name}: {t.shape[0]} != {n_head}*{d_head}"
-        assert d_head == 80
-
-        idx = torch.tensor(perm, dtype=torch.long)
-        if t.ndim == 2:
-            return t.reshape(n_head, d_head, t.shape[1])[:, idx, :].reshape(t.shape)
-        return t.reshape(n_head, d_head)[:, idx].reshape(t.shape)

gguf-py/gguf/constants.py

@@ -857,8 +857,6 @@ class MODEL_TENSOR(IntEnum):
     V_MM_UP              = auto() # cogvlm
     V_MM_DOWN            = auto() # cogvlm
     V_MM_GATE            = auto() # cogvlm
-    V_MM_MERGER_FC1      = auto() # minimax-m3 (patch-merge MLP)
-    V_MM_MERGER_FC2      = auto() # minimax-m3 (patch-merge MLP)
     V_TOK_BOI            = auto() # cogvlm
     V_TOK_EOI            = auto() # cogvlm
     V_TOK_IMG_BEGIN      = auto() # hunyuanvl
@@ -1443,8 +1441,6 @@ TENSOR_NAMES: dict[MODEL_TENSOR, str] = {
     MODEL_TENSOR.V_MM_UP:                   "mm.up",
     MODEL_TENSOR.V_MM_DOWN:                 "mm.down",
     MODEL_TENSOR.V_MM_GATE:                 "mm.gate",
-    MODEL_TENSOR.V_MM_MERGER_FC1:            "mm.merger.fc1",
-    MODEL_TENSOR.V_MM_MERGER_FC2:            "mm.merger.fc2",
     MODEL_TENSOR.V_TOK_BOI:                 "v.boi",
     MODEL_TENSOR.V_TOK_EOI:                 "v.eoi",
     MODEL_TENSOR.V_MM_PRE_NORM:             "mm.pre_norm",
@@ -1641,8 +1637,6 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = {
         MODEL_TENSOR.V_RESMPL_QUERY,
         MODEL_TENSOR.V_TOK_EMBD_IMG_BREAK,
         MODEL_TENSOR.V_MM_PATCH_MERGER,
-        MODEL_TENSOR.V_MM_MERGER_FC1,
-        MODEL_TENSOR.V_MM_MERGER_FC2,
         MODEL_TENSOR.V_DS_NORM,
         MODEL_TENSOR.V_DS_FC1,
         MODEL_TENSOR.V_DS_FC2,
@@ -4777,7 +4771,6 @@ class VisionProjectorType:
     YOUTUVL = "youtuvl"
     NEMOTRON_V2_VL = "nemotron_v2_vl"
     HUNYUANVL      = "hunyuanvl"
-    MINIMAXM3      = "minimax_m3"
     MINICPMV4_6    = "minicpmv4_6"
     GRANITE_SPEECH = "granite_speech"  # audio
     MIMOVL         = "mimovl"

gguf-py/gguf/tensor_mapping.py

@@ -1838,14 +1838,6 @@ class TensorNameMap:
             "visual.downsample", # glm4v
         ),
 
-        MODEL_TENSOR.V_MM_MERGER_FC1: (
-            "patch_merge_mlp.linear_1", # minimax-m3
-        ),
-
-        MODEL_TENSOR.V_MM_MERGER_FC2: (
-            "patch_merge_mlp.linear_2", # minimax-m3
-        ),
-
         MODEL_TENSOR.V_DS_NORM: (
             "model.visual.deepstack_merger_list.{bid}.norm", # deepstack in qwen3vl
         ),

skills/code-review/SKILL.md

@@ -110,15 +110,6 @@ Public API changes carry a higher bar than internal ones (`CONTRIBUTING.md`). Re
 - Security: don't trust client-supplied headers (e.g. `X-Forwarded-For`) or add footguns; things like IP allowlisting belong at a reverse proxy unless there's a trusted-proxy design.
 - Wire new behavior into the existing request/response and checkpoint paths correctly; watch for resource leaks across requests.
 
-## Multimodal (`tools/mtmd/`)
-
-- Tensor names must be prefixed by `v.`, `a.`, `mm.` or `a.mm.` (legacy naming doesn't follow this convention - this is expected, but new code should follow it).
-- Do not use explicit sin/cos for RoPE; use `ggml_rope_ext` instead, see `HOWTO-add-model.md`. If it can't express the needed behavior, that's a design discussion, not a PR.
-- New GGML ops must not be introduced in the same PR, you must push it as a separate PR.
-- In most cases, `build_vit` should be enough to build the transformer graph for vision models. Do not add a loop to build the transformer graph manually, unless you have a very good reason to do so. If you do, please explain why in the PR description.
-- If you need a dedicated preprocessor, there is a high chance that it can be a derived class from one of the existing preprocessors. Check carefully before adding a new preprocessor class.
-- If the model need a new public API in `mtmd.h`, open a discussion first.
-
 ## General (always)
 
 Enforce the `AGENTS.md` / `CONTRIBUTING.md` coding and naming guidelines on every changed line - this is a distinct pass from checking that the code works, and matters just as much for review speed:

tools/mtmd/CMakeLists.txt

@@ -47,7 +47,6 @@ add_library(mtmd
             models/paddleocr.cpp
             models/pixtral.cpp
             models/qwen2vl.cpp
-            models/minimax-m3.cpp
             models/qwen3vl.cpp
             models/mimovl.cpp
             models/qwen3a.cpp

tools/mtmd/clip-impl.h

@@ -131,8 +131,6 @@
 #define TN_MM_SOFT_EMB_N   "mm.soft_emb_norm.weight"    // gemma3
 #define TN_MM_PROJECTOR    "mm.model.fc.%s"             // idefics3, deepseekocr
 #define TN_MM_PATCH_MERGER "mm.patch_merger.%s"         // mistral small 3.1, glm4v
-#define TN_MM_MERGER_FC1   "mm.merger.fc1.%s"            // minimax-m3 patch-merge MLP
-#define TN_MM_MERGER_FC2   "mm.merger.fc2.%s"
 #define TN_TOK_IMG_BREAK   "v.token_embd.img_break"     // pixtral
 #define TN_TOK_GLM_BOI     "adapter.boi"                // glm-edge (these embeddings are not in text model)
 #define TN_TOK_GLM_EOI     "adapter.eoi"                // glm-edge (these embeddings are not in text model)
@@ -372,7 +370,6 @@ enum projector_type {
     PROJECTOR_TYPE_MINICPMV4_6,
     PROJECTOR_TYPE_GRANITE_SPEECH,
     PROJECTOR_TYPE_MIMOVL,
-    PROJECTOR_TYPE_MINIMAX_M3,
     PROJECTOR_TYPE_GRANITE4_VISION,
     PROJECTOR_TYPE_UNKNOWN,
 };
@@ -427,7 +424,6 @@ static std::map<projector_type, std::string> PROJECTOR_TYPE_NAMES = {
     { PROJECTOR_TYPE_MINICPMV4_6,       "minicpmv4_6"},
     { PROJECTOR_TYPE_GRANITE_SPEECH,    "granite_speech"},
     { PROJECTOR_TYPE_MIMOVL,            "mimovl"},
-    { PROJECTOR_TYPE_MINIMAX_M3,        "minimax_m3"},
     { PROJECTOR_TYPE_GRANITE4_VISION,   "granite4_vision"},
 };
 

tools/mtmd/clip-model.h

@@ -397,10 +397,6 @@ struct clip_model {
     ggml_tensor * mm_0_b = nullptr;
     ggml_tensor * mm_2_w = nullptr;
     ggml_tensor * mm_2_b = nullptr;
-    ggml_tensor * mm_merger_fc1_w = nullptr;   // minimax-m3
-    ggml_tensor * mm_merger_fc1_b = nullptr;
-    ggml_tensor * mm_merger_fc2_w = nullptr;
-    ggml_tensor * mm_merger_fc2_b = nullptr;
 
     ggml_tensor * image_newline = nullptr;
     ggml_tensor * view_seperator = nullptr;

tools/mtmd/clip.cpp

@@ -915,10 +915,6 @@ static std::unique_ptr<clip_graph> clip_get_graph_builder(clip_ctx * ctx, const
             {
                 builder = std::make_unique<clip_graph_mimovl>(ctx, img);
             } break;
-        case PROJECTOR_TYPE_MINIMAX_M3:
-            {
-                builder = std::make_unique<clip_graph_minimax_m3>(ctx, img);
-            } break;
         case PROJECTOR_TYPE_STEP3VL:
             {
                 builder = std::make_unique<clip_graph_step3vl>(ctx, img);
@@ -1473,17 +1469,6 @@ struct clip_model_loader {
                             LOG_WRN("%s: more info: https://github.com/ggml-org/llama.cpp/issues/16842\n\n", __func__);
                         }
                     } break;
-                case PROJECTOR_TYPE_MINIMAX_M3:
-                    {
-                        hparams.n_merge = 2; // spatial_merge_size
-                        hparams.image_resize_algo = RESIZE_ALGO_BICUBIC_PILLOW;
-                        hparams.image_resize_pad  = PAD_NONE;
-                        get_u32(KEY_SPATIAL_MERGE_SIZE, hparams.n_merge, false);
-                        hparams.rope_theta = 10000.0f; // vision_config.rope_theta
-                        // MiniMax-M3: max_pixels 451584 (=672^2) -> 576 merged tokens (image_seq_length)
-                        hparams.set_limit_image_tokens(8, 576);
-                        hparams.set_warmup_n_tokens(16*16);
-                    } break;
                 case PROJECTOR_TYPE_MIMOVL:
                     {
                         hparams.n_merge = 2; // spatial_merge_size
@@ -2104,19 +2089,6 @@ struct clip_model_loader {
                     model.mm_1_w = get_tensor(string_format(TN_LLAVA_PROJ, 2, "weight"));
                     model.mm_1_b = get_tensor(string_format(TN_LLAVA_PROJ, 2, "bias"), false);
                 } break;
-            case PROJECTOR_TYPE_MINIMAX_M3:
-                {
-                    // per-patch MLP: mm.1 -> gelu -> mm.2
-                    model.mm_1_w = get_tensor(string_format(TN_LLAVA_PROJ, 1, "weight"));
-                    model.mm_1_b = get_tensor(string_format(TN_LLAVA_PROJ, 1, "bias"));
-                    model.mm_2_w = get_tensor(string_format(TN_LLAVA_PROJ, 2, "weight"));
-                    model.mm_2_b = get_tensor(string_format(TN_LLAVA_PROJ, 2, "bias"));
-                    // 2x2 merge MLP: mm.merge.fc1 -> gelu -> mm.merge.fc2
-                    model.mm_merger_fc1_w = get_tensor(string_format(TN_MM_MERGER_FC1, "weight"));
-                    model.mm_merger_fc1_b = get_tensor(string_format(TN_MM_MERGER_FC1, "bias"));
-                    model.mm_merger_fc2_w = get_tensor(string_format(TN_MM_MERGER_FC2, "weight"));
-                    model.mm_merger_fc2_b = get_tensor(string_format(TN_MM_MERGER_FC2, "bias"));
-                } break;
             case PROJECTOR_TYPE_STEP3VL:
                 {
                     model.mm_0_w     = get_tensor(string_format(TN_LLAVA_PROJ, 0, "weight"));
@@ -3388,7 +3360,6 @@ int clip_n_output_tokens(const clip_ctx * ctx, const clip_image_f32 * img) {
         case PROJECTOR_TYPE_QWEN3VL:
         case PROJECTOR_TYPE_EXAONE4_5:
         case PROJECTOR_TYPE_MIMOVL:
-        case PROJECTOR_TYPE_MINIMAX_M3:
         case PROJECTOR_TYPE_GLM4V:
         case PROJECTOR_TYPE_YOUTUVL:
             {
@@ -3895,24 +3866,6 @@ bool clip_image_batch_encode(clip_ctx * ctx, int n_threads, const clip_image_f32
 
                 set_input_i32("positions", positions);
             } break;
-        case PROJECTOR_TYPE_MINIMAX_M3:
-            {
-                const int n_merge = hparams.n_merge;
-                const int gh = image_size_height / patch_size;
-                const int gw = image_size_width  / patch_size;
-                std::vector<int32_t> pos_h, pos_w;
-                pos_h.reserve(gh * gw);
-                pos_w.reserve(gh * gw);
-                for (int bh = 0; bh < gh / n_merge; bh++)
-                for (int bw = 0; bw < gw / n_merge; bw++)
-                for (int mh = 0; mh < n_merge; mh++)
-                for (int mw = 0; mw < n_merge; mw++) {
-                    pos_h.push_back(bh * n_merge + mh);
-                    pos_w.push_back(bw * n_merge + mw);
-                }
-                set_input_i32("minimax_pos_h", pos_h);
-                set_input_i32("minimax_pos_w", pos_w);
-            } break;
         case PROJECTOR_TYPE_DOTS_OCR:
             {
                 const int pw = image_size_width / patch_size;
@@ -4616,8 +4569,6 @@ int clip_n_mmproj_embd(const struct clip_ctx * ctx) {
             return ctx->model.mm_ffn_down_w->ne[1];
         case PROJECTOR_TYPE_GLM_EDGE:
             return ctx->model.mm_model_mlp_3_w->ne[1];
-        case PROJECTOR_TYPE_MINIMAX_M3:
-            return ctx->model.mm_merger_fc2_b->ne[0];
         case PROJECTOR_TYPE_QWEN2VL:
         case PROJECTOR_TYPE_QWEN25VL:
         case PROJECTOR_TYPE_EXAONE4_5:

tools/mtmd/models/deepseekocr.cpp

@@ -253,6 +253,9 @@ ggml_cgraph * clip_graph_deepseekocr::build() {
 
     bool is_overview = img.add_viewsep;
     int n_tiles_per_row = 0;
+    // number of separate "row" images batched together in this graph call
+    // (captured now, before n_batch below gets repurposed as the SAM/ViT batch size)
+    const int n_rows_batch = n_batch;
 
     // note: we expect either a batch of rows or a batch of overviews, but not a mix of both
 
@@ -272,16 +275,30 @@ ggml_cgraph * clip_graph_deepseekocr::build() {
         GGML_ASSERT(img.ny() % img.nx() == 0);
         n_tiles_per_row = img.ny() / img.nx();
 
-        // input shape: [tile_size, tile_size * n_tiles_per_row, 3]
-        // we want to reshape it to [tile_size, tile_size, 3, n_tiles_per_row]
-        inp_raw = ggml_reshape_4d(ctx0, inp_raw, img.nx(), img.nx(), n_tiles_per_row, 3);
-        inp_raw = ggml_cont(ctx0, ggml_permute(ctx0, inp_raw, 0, 1, 3, 2));
+        // each entry is one "row" image of shape [tile_size, tile_size * n_tiles_per_row, 3];
+        // reshape+concat all rows into one combined SAM input of shape
+        // [tile_size, tile_size, 3, n_tiles_per_row * n_rows_batch] (tile fast, row slow)
+        ggml_tensor * combined = nullptr;
+        for (int r = 0; r < n_rows_batch; r++) {
+            ggml_tensor * row = ggml_view_4d(ctx0, inp_raw, img.nx(), img.ny(), 3, 1,
+                                             inp_raw->nb[1], inp_raw->nb[2], inp_raw->nb[3],
+                                             r * inp_raw->nb[3]);
+            row = ggml_cont(ctx0, row);
+
+            // input shape: [tile_size, tile_size * n_tiles_per_row, 3, 1]
+            // we want to reshape it to [tile_size, tile_size, 3, n_tiles_per_row]
+            row = ggml_reshape_4d(ctx0, row, img.nx(), img.nx(), n_tiles_per_row, 3);
+            row = ggml_cont(ctx0, ggml_permute(ctx0, row, 0, 1, 3, 2));
+
+            combined = combined ? ggml_concat(ctx0, combined, row, 3) : row;
+        }
+        inp_raw = combined;
     }
 
     ggml_tensor * sam_out = build_sam(inp_raw);
 
     if (!is_overview) {
-        n_batch = n_tiles_per_row;
+        n_batch = n_tiles_per_row * n_rows_batch;
     }
 
     const int clip_n_patches = sam_out->ne[0] * sam_out->ne[1];
@@ -354,34 +371,53 @@ ggml_cgraph * clip_graph_deepseekocr::build() {
         const auto w     = h;
         const auto n_dim = cur->ne[0];
 
-        ggml_tensor * imgnl = ggml_repeat_4d(ctx0, model.image_newline, n_dim, 1, h, 1);
-        cur = ggml_reshape_3d(ctx0, cur, n_dim, w, h);
-        cur = ggml_reshape_2d(ctx0, ggml_concat(ctx0, cur, imgnl, 1), n_dim, (w + 1) * h);
-        cur = ggml_concat(ctx0, cur, model.view_seperator, 1);  // (n_dim, h*(w+1) + 1)
+        ggml_tensor * imgnl = ggml_repeat_4d(ctx0, model.image_newline, n_dim, 1, h, n_batch);
+        cur = ggml_reshape_4d(ctx0, cur, n_dim, w, h, n_batch);
+        cur = ggml_reshape_3d(ctx0, ggml_concat(ctx0, cur, imgnl, 1), n_dim, (w + 1) * h, n_batch);
+        ggml_tensor * vs = ggml_repeat_4d(ctx0, model.view_seperator, n_dim, 1, n_batch, 1);
+        cur = ggml_concat(ctx0, cur, vs, 1);  // (n_dim, h*(w+1) + 1, n_batch)
     } else {
         // tile row: interleave tiles within each row, add newline per row
-        const int grid_x      = static_cast<int>(std::sqrt(static_cast<float>(clip_n_patches)));
-        const int grid_y      = grid_x;
-        const auto n_dim      = cur->ne[0];
+        // this weave is done per-row (cheap reshuffle ops) since a single row's
+        // interleave already uses all 4 ggml tensor axes, leaving no spare axis to
+        // also carry the n_rows_batch dimension through in one shot
+        const int  grid_x = static_cast<int>(std::sqrt(static_cast<float>(clip_n_patches)));
+        const int  grid_y = grid_x;
+        const auto n_dim  = cur->ne[0];
 
-        // (n_dim, clip_n_patches, n_batch) -> (n_dim, grid_x, grid_y, n_batch)
-        cur = ggml_reshape_4d(ctx0, cur, n_dim, grid_x, grid_y, n_batch);
+        // (n_dim, clip_n_patches, n_tiles_per_row * n_rows_batch) -> (n_dim, clip_n_patches, n_tiles_per_row, n_rows_batch)
+        ggml_tensor * cur4 = ggml_reshape_4d(ctx0, cur, n_dim, clip_n_patches, n_tiles_per_row, n_rows_batch);
 
-        // tiles: re-order from A.row0 A.row1 B.row0 B.row1 ...
-        //        to A.row0 B.row0 A.row1 B.row1 ...
-        //        then add nl: A.row0 B.row0 [nl] A.row1 B.row1 [nl] ...
-        // interleave tiles: (n_dim, grid_x, grid_y, n_batch) -> (n_dim, grid_x, n_batch, grid_y)
-        cur = ggml_cont(ctx0, ggml_permute(ctx0, cur, 0, 1, 3, 2));
+        ggml_tensor * rows_out = nullptr;
+        for (int r = 0; r < n_rows_batch; r++) {
+            ggml_tensor * row = ggml_view_3d(ctx0, cur4, n_dim, clip_n_patches, n_tiles_per_row,
+                                             cur4->nb[1], cur4->nb[2], r * cur4->nb[3]);
+            row = ggml_cont(ctx0, row);
 
-        // merge: (n_dim, grid_x, n_batch, grid_y) -> (n_dim, grid_x*n_batch, grid_y, 1)
-        cur = ggml_reshape_4d(ctx0, cur, n_dim, grid_x * n_batch, grid_y, 1);
+            // (n_dim, clip_n_patches, n_tiles_per_row) -> (n_dim, grid_x, grid_y, n_tiles_per_row)
+            row = ggml_reshape_4d(ctx0, row, n_dim, grid_x, grid_y, n_tiles_per_row);
 
-        // append newline per row: (n_dim, grid_x*n_batch+1, grid_y, 1)
-        ggml_tensor * imgnl = ggml_repeat_4d(ctx0, model.image_newline, n_dim, 1, grid_y, 1);
-        cur = ggml_concat(ctx0, cur, imgnl, 1);
+            // tiles: re-order from A.row0 A.row1 B.row0 B.row1 ...
+            //        to A.row0 B.row0 A.row1 B.row1 ...
+            //        then add nl: A.row0 B.row0 [nl] A.row1 B.row1 [nl] ...
+            // interleave tiles: (n_dim, grid_x, grid_y, n_tiles_per_row) -> (n_dim, grid_x, n_tiles_per_row, grid_y)
+            row = ggml_cont(ctx0, ggml_permute(ctx0, row, 0, 1, 3, 2));
 
-        // flatten: (n_dim, (grid_x*n_batch+1)*grid_y)
-        cur = ggml_reshape_2d(ctx0, cur, n_dim, (grid_x * n_batch + 1) * grid_y);
+            // merge: (n_dim, grid_x, n_tiles_per_row, grid_y) -> (n_dim, grid_x*n_tiles_per_row, grid_y, 1)
+            row = ggml_reshape_4d(ctx0, row, n_dim, grid_x * n_tiles_per_row, grid_y, 1);
+
+            // append newline per row: (n_dim, grid_x*n_tiles_per_row+1, grid_y, 1)
+            ggml_tensor * imgnl = ggml_repeat_4d(ctx0, model.image_newline, n_dim, 1, grid_y, 1);
+            row = ggml_concat(ctx0, row, imgnl, 1);
+
+            // flatten: (n_dim, (grid_x*n_tiles_per_row+1)*grid_y, 1)
+            row = ggml_reshape_3d(ctx0, row, n_dim, (grid_x * n_tiles_per_row + 1) * grid_y, 1);
+
+            rows_out = rows_out ? ggml_concat(ctx0, rows_out, row, 2) : row;
+        }
+
+        // (n_dim, (grid_x*n_tiles_per_row+1)*grid_y, n_rows_batch)
+        cur = rows_out;
     }
 
     cb(cur, "dsocr_output", -1);

tools/mtmd/models/deepseekocr2.cpp

@@ -14,8 +14,9 @@ ggml_cgraph * clip_graph_deepseekocr2::build() {
     {
         ggml_tensor * inp;
 
-        inp = ggml_reshape_2d(ctx0, sam_out, sam_out->ne[0] * sam_out->ne[1], sam_out->ne[2]); // H*W, C
-        inp = ggml_cont(ctx0, ggml_permute(ctx0, inp, 1, 0, 2, 3));
+        // H*W, C, B
+        inp = ggml_reshape_3d(ctx0, sam_out, sam_out->ne[0] * sam_out->ne[1], sam_out->ne[2], sam_out->ne[3]);
+        inp = ggml_cont(ctx0, ggml_permute(ctx0, inp, 1, 0, 2, 3)); // C, H*W, B
 
         auto num_image_tokens = inp->ne[1]; // H*W
         GGML_ASSERT(num_image_tokens == 144 || num_image_tokens == 256);
@@ -32,8 +33,10 @@ ggml_cgraph * clip_graph_deepseekocr2::build() {
             num_queries = 144;
         }
 
-        // (B, num_image_tokens + num_queries, C)
-        inp = ggml_concat(ctx0, inp, ggml_cast(ctx0, query_embed, inp->type), 1);
+        // repeat the query embedding per batch item, then append: (C, num_image_tokens + num_queries, B)
+        query_embed = ggml_cast(ctx0, query_embed, inp->type);
+        query_embed = ggml_repeat_4d(ctx0, query_embed, query_embed->ne[0], num_queries, inp->ne[2], 1);
+        inp = ggml_concat(ctx0, inp, query_embed, 1);
 
         auto seq_len = inp->ne[1];
 
@@ -57,7 +60,7 @@ ggml_cgraph * clip_graph_deepseekocr2::build() {
                                       /* learned_pos_embd */ nullptr, add_rope, vit_opts);
 
         cur = ggml_cont(ctx0,
-                        ggml_view_2d(ctx0, cur, cur->ne[0], num_queries, cur->nb[1],
+                        ggml_view_3d(ctx0, cur, cur->ne[0], num_queries, cur->ne[2], cur->nb[1], cur->nb[2],
                                      cur->nb[1] * (cur->ne[1] - num_queries))); // only take query tokens for output
 
         ggml_build_forward_expand(gf, cur);
@@ -71,7 +74,8 @@ ggml_cgraph * clip_graph_deepseekocr2::build() {
 
     // view_seperator only after the global view
     if (img.add_viewsep) {
-        cur = ggml_concat(ctx0, cur, model.view_seperator, 1); // (n_dim, 257)
+        ggml_tensor * vs = ggml_repeat_4d(ctx0, model.view_seperator, model.view_seperator->ne[0], 1, cur->ne[2], 1);
+        cur = ggml_concat(ctx0, cur, vs, 1); // (n_dim, 257, n_batch)
     }
 
     cb(cur, "dsocr2_output", -1);

tools/mtmd/models/minimax-m3.cpp

@@ -1,84 +0,0 @@
-#include "models.h"
-
-ggml_tensor * clip_graph_minimax_m3::apply_rope(
-        ggml_tensor * x, ggml_tensor * pos_h, ggml_tensor * pos_w) {
-    const int64_t Hn  = x->ne[1];
-    const int64_t P   = x->ne[2];
-    const size_t  es  = ggml_element_size(x);
-    const int     dh  = (int) x->ne[0];
-    const int     axd = 2 * ((2 * (dh / 2) / 3) / 2);
-
-    GGML_ASSERT(x->nb[0] == es);
-    GGML_ASSERT(3 * axd <= dh);
-
-    const float th  = hparams.rope_theta;
-
-    // layout of x is [t, h, w, pad]
-    // t is unrotated, h and w are rotated, pad is unrotated
-    // note: everything from n_dims onward untouched, so w and pad are rotated in one call.
-    auto sl = [&](int off, int n) {
-        return ggml_cont(ctx0, ggml_view_3d(ctx0, x, n, Hn, P, x->nb[1], x->nb[2], (size_t) off * es));
-    };
-    ggml_tensor * t = sl(0,       axd);
-    ggml_tensor * h = sl(axd,     axd);
-    ggml_tensor * w = sl(2 * axd, dh - 2 * axd); // w + pad
-
-    h = ggml_rope_ext(ctx0, h, pos_h, nullptr, axd, GGML_ROPE_TYPE_NEOX, 0, th, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f);
-    w = ggml_rope_ext(ctx0, w, pos_w, nullptr, axd, GGML_ROPE_TYPE_NEOX, 0, th, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f);
-    return ggml_concat(ctx0, ggml_concat(ctx0, t, h, 0), w, 0);
-}
-
-ggml_cgraph * clip_graph_minimax_m3::build() {
-    GGML_ASSERT(model.patch_bias     == nullptr);
-    GGML_ASSERT(model.class_embedding == nullptr);
-    GGML_ASSERT(model.patch_embeddings_0 && model.patch_embeddings_1);
-    GGML_ASSERT(model.mm_1_w && model.mm_2_w);
-    GGML_ASSERT(model.mm_merger_fc1_w && model.mm_merger_fc2_w);
-
-    const int batch_size = 1;
-    const int n_pos      = n_patches;
-    const int merge      = hparams.n_merge;
-
-    // patch embedding
-    ggml_tensor * inp_raw = build_inp_raw();
-    ggml_tensor * inp = ggml_add(ctx0,
-        ggml_conv_2d(ctx0, model.patch_embeddings_0, inp_raw, patch_size, patch_size, 0, 0, 1, 1),
-        ggml_conv_2d(ctx0, model.patch_embeddings_1, inp_raw, patch_size, patch_size, 0, 0, 1, 1));
-
-    // spatial merge
-    {
-        inp = ggml_permute(ctx0, inp, 1, 2, 0, 3);
-        inp = ggml_cont_4d(ctx0, inp, n_embd * merge, n_patches_x / merge, n_patches_y, batch_size);
-        inp = ggml_reshape_4d(ctx0, inp, n_embd * merge, n_patches_x / merge, merge, batch_size * (n_patches_y / merge));
-        inp = ggml_permute(ctx0, inp, 0, 2, 1, 3);
-        inp = ggml_cont_3d(ctx0, inp, n_embd, n_patches_x * n_patches_y, batch_size);
-    }
-
-    // t (time axis) is always 0 for now, so we leave it unrotated
-    ggml_tensor * pos_h = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_pos);
-    ggml_set_name(pos_h, "minimax_pos_h"); ggml_set_input(pos_h);
-    ggml_tensor * pos_w = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_pos);
-    ggml_set_name(pos_w, "minimax_pos_w"); ggml_set_input(pos_w);
-
-    ggml_tensor * inpL = build_vit(
-        inp, n_pos, NORM_TYPE_NORMAL, FFN_GELU_ERF, nullptr,
-        [&](ggml_tensor * c, const clip_layer &) {
-            return apply_rope(c, pos_h, pos_w);
-        });
-
-    // projector
-    ggml_tensor * emb = inpL;
-    emb = build_ffn(emb, model.mm_1_w, model.mm_1_b,
-                    nullptr, nullptr,
-                    model.mm_2_w, model.mm_2_b, FFN_GELU_ERF, -1);
-
-    const int64_t proj = emb->ne[0];
-    emb = ggml_reshape_2d(ctx0, emb, proj * merge * merge, n_pos / (merge * merge));
-
-    emb = build_ffn(emb, model.mm_merger_fc1_w, model.mm_merger_fc1_b,
-                    nullptr, nullptr,
-                    model.mm_merger_fc2_w, model.mm_merger_fc2_b, FFN_GELU_ERF, -1);
-
-    ggml_build_forward_expand(gf, emb);
-    return gf;
-}

tools/mtmd/models/models.h

@@ -40,12 +40,6 @@ struct clip_graph_qwen3vl : clip_graph_qwen2vl {
     ggml_cgraph * build() override;
 };
 
-struct clip_graph_minimax_m3 : clip_graph {
-    clip_graph_minimax_m3(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}
-    ggml_cgraph * build() override;
-    ggml_tensor * apply_rope(ggml_tensor * x, ggml_tensor * pos_h, ggml_tensor * pos_w);
-};
-
 struct clip_graph_mimovl : clip_graph {
     clip_graph_mimovl(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}
     ggml_cgraph * build() override;
@@ -133,12 +127,13 @@ struct clip_graph_deepseekocr : clip_graph {
     clip_graph_deepseekocr(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}
     ggml_cgraph * build() override;
     ggml_tensor * build_sam(ggml_tensor * inp); // build the SAM model
-    // bool support_batch() const override { return true; } // TODO: support batch for DeepSeek-OCR v1
+    bool support_batch() const override { return true; }
 };
 
 struct clip_graph_deepseekocr2 : clip_graph_deepseekocr {
     clip_graph_deepseekocr2(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph_deepseekocr(ctx, img) {}
     ggml_cgraph * build() override; // reuses build_sam() from base
+    bool support_batch() const override { return true; }
 };
 
 struct clip_graph_conformer : clip_graph {

tools/mtmd/mtmd-helper.cpp

@@ -640,7 +640,6 @@ bool mtmd_helper_support_video(mtmd_context * ctx) {
 #ifdef MTMD_VIDEO
     return mtmd_support_vision(ctx);
 #else
-    GGML_UNUSED(ctx);
     return false;
 #endif
 }
@@ -1008,9 +1007,6 @@ mtmd_helper_video * mtmd_helper_video_init(
 
     return ctx;
 #else
-    GGML_UNUSED(mctx);
-    GGML_UNUSED(path);
-    GGML_UNUSED(params);
     LOG_ERR("%s: video is not supported in this build (MTMD_VIDEO is set to OFF)\n", __func__);
     return nullptr;
 #endif
@@ -1043,10 +1039,6 @@ mtmd_helper_video * mtmd_helper_video_init_from_buf(
 
     return ctx;
 #else
-    GGML_UNUSED(mctx);
-    GGML_UNUSED(buf);
-    GGML_UNUSED(len);
-    GGML_UNUSED(params);
     LOG_ERR("%s: video is not supported in this build (MTMD_VIDEO is set to OFF)\n", __func__);
     return nullptr;
 #endif
@@ -1058,7 +1050,6 @@ void mtmd_helper_video_free(mtmd_helper_video * ctx) {
     ctx->stop_ffmpeg();
     delete ctx;
 #else
-    GGML_UNUSED(ctx);
     LOG_ERR("%s: video is not supported in this build (MTMD_VIDEO is set to OFF)\n", __func__);
 #endif
 }
@@ -1067,7 +1058,6 @@ mtmd_helper_video_info mtmd_helper_video_get_info(const mtmd_helper_video * ctx)
 #ifdef MTMD_VIDEO
     return ctx->info;
 #else
-    GGML_UNUSED(ctx);
     GGML_ASSERT(false && "video is not supported in this build (MTMD_VIDEO is set to OFF)");
 #endif
 }
@@ -1078,9 +1068,6 @@ int32_t mtmd_helper_video_read_next(mtmd_helper_video * ctx,
     if (!ctx) return -2;
     return ctx->read_next(out_bitmap, out_text);
 #else
-    GGML_UNUSED(ctx);
-    GGML_UNUSED(out_bitmap);
-    GGML_UNUSED(out_text);
     GGML_ASSERT(false && "video is not supported in this build (MTMD_VIDEO is set to OFF)");
 #endif
 }

tools/mtmd/mtmd.cpp

@@ -463,13 +463,6 @@ struct mtmd_context {
                     img_end = "<|vision_end|>";
                     image_preproc = std::make_unique<mtmd_image_preprocessor_dyn_size>(ctx_v);
                 } break;
-            case PROJECTOR_TYPE_MINIMAX_M3:
-                {
-                    // ]<]start of image[>[ ... (image embeddings) ... ]<]end of image[>[
-                    img_beg = "]<]start of image[>[";
-                    img_end = "]<]end of image[>[";
-                    image_preproc = std::make_unique<mtmd_image_preprocessor_dyn_size>(ctx_v);
-                } break;
             case PROJECTOR_TYPE_YOUTUVL:
                 {
                     // <|vision_start|> ... (image embeddings) ... <|vision_end|>

tools/ui/src/lib/constants/sandbox.ts

@@ -14,15 +14,12 @@ export const SANDBOX_EMPTY_OUTPUT = '(no output)';
 export const SANDBOX_TRUNCATION_NOTICE = '[output truncated]';
 
 const NERDAMER_DESCRIPTION = `
-Symbolic/numeric math via \`nerdamer\`
-nerdamer(expr,subs?,opts?)/nerdamer.func(...)→Expression Format via .text(fmt?) (fmt: 'decimals'|'fractions'|'scientific') eval via .evaluate(subs?)
-nerdamer(expr,{x:2}) substitutes numeric via opts 'numer' or .evaluate()
-simplify/expand/factor(expr) div/gcd/lcm(...) coeffs/partfrac(expr,var)
-diff/integrate(expr,var) defint(expr,lo,hi,var?) sum/product(expr,var,lo,hi) limit(expr,var,pt)
-solve(expr,var) solveEquations([eq1,eq2],[var1,var2])
-polarform/rectform/arg/realpart/imagpart(z)
-set/get Var/Constant(name,val?) setFunction(name,[params],body)
-IMPORTANT:Identifier 'nerdamer' has already been declared, use it directly`;
+Symbolic/numeric math via \`nerdamer\` (pre-loaded, do not require, use it directly).
+nerdamer('diff(sin(x)/x,x)') or nerdamer.diff('sin(x)/x','x') → Expression; convert with .toString()/.text()/.toTeX(), or .evaluate() (→ still Expression, then .toString()).
+nerdamer(expr,{x:2}) substitutes only; chain .evaluate() or pass 'numer' for numeric result.
+solve(expr,var)→Symbol[]; solveEquations([eq1,..])→[[var,val],..] pairs.
+Functions: simplify/expand/factor(expr), diff(expr,var[,n]), integrate(expr,var), defint(expr,from,to,var), limit(expr,var,to), laplace(expr,t,s), ilt(expr,s,t), gcd/lcm(a,b), roots/coeffs/partfrac(expr,var), pfactor(n), numer/decimals/erf(expr), product/sum(expr,var,from,to), mean/median/stdev/variance(...vals).
+Object.keys(nerdamer).filter(k=>typeof nerdamer[k]==='function') lists all available functions. If you need a function not documented above, list them first — do not guess function names.`;
 
 /**
  * Build the sandbox tool definition. When `includeSymbolicMath` is true,