mtmd: support multi-row batching for deepseek-ocr (#26154)

* mtmd: support multi-row batching for deepseek-ocr

* mtmd: weave deepseek-ocr rows in one shot instead of per row (#26615)

---------

Co-authored-by: Saba Fallah <sabafallah@gmail.com>

.github/workflows/build-apple.yml

@@ -60,7 +60,6 @@ jobs:
             -DCMAKE_BUILD_RPATH="@loader_path" \
             -DLLAMA_FATAL_WARNINGS=ON \
             -DLLAMA_BUILD_BORINGSSL=ON \
-            -DGGML_METAL_USE_BF16=ON \
             -DGGML_METAL_EMBED_LIBRARY=OFF \
             -DGGML_METAL_SHADER_DEBUG=ON \
             -DGGML_RPC=ON \
@@ -127,7 +126,6 @@ jobs:
         run: |
           sysctl -a
           cmake -B build -G Xcode \
-            -DGGML_METAL_USE_BF16=ON \
             -DGGML_METAL_EMBED_LIBRARY=ON \
             -DLLAMA_OPENSSL=OFF \
             -DLLAMA_BUILD_APP=OFF \
@@ -178,7 +176,6 @@ jobs:
         run: |
           sysctl -a
           cmake -B build -G Xcode \
-            -DGGML_METAL_USE_BF16=ON \
             -DGGML_METAL_EMBED_LIBRARY=ON \
             -DLLAMA_BUILD_COMMON=OFF \
             -DLLAMA_BUILD_APP=OFF \
@@ -212,7 +209,6 @@ jobs:
         run: |
           sysctl -a
           cmake -B build -G Xcode \
-            -DGGML_METAL_USE_BF16=ON \
             -DGGML_METAL_EMBED_LIBRARY=ON \
             -DLLAMA_BUILD_COMMON=OFF \
             -DLLAMA_BUILD_APP=OFF \
@@ -257,7 +253,6 @@ jobs:
         run: |
           sysctl -a
           cmake -B build -G Xcode \
-            -DGGML_METAL_USE_BF16=ON \
             -DGGML_METAL_EMBED_LIBRARY=ON \
             -DLLAMA_OPENSSL=OFF \
             -DLLAMA_BUILD_APP=OFF \

.github/workflows/release.yml

@@ -93,13 +93,13 @@ jobs:
           - build: 'arm64'
             arch: 'arm64'
             os: macos-26
-            defines: "-DGGML_METAL_USE_BF16=ON -DGGML_METAL_EMBED_LIBRARY=ON -DCMAKE_OSX_DEPLOYMENT_TARGET=13.3"
+            defines: "-DGGML_METAL_EMBED_LIBRARY=ON -DCMAKE_OSX_DEPLOYMENT_TARGET=13.3"
           # TODO: this build is disabled to save Github Actions resources (https://github.com/ggml-org/llama.cpp/pull/23780)
           #       in order to enable it again, we have to provision dedicated runners  to run it
           #- build: 'arm64-kleidiai'
           #  arch: 'arm64'
           #  os: macos-14
-          #  defines: "-DGGML_METAL_USE_BF16=ON -DGGML_METAL_EMBED_LIBRARY=ON -DCMAKE_OSX_DEPLOYMENT_TARGET=13.3 -DGGML_CPU_KLEIDIAI=ON"
+          #  defines: "-DGGML_METAL_EMBED_LIBRARY=ON -DCMAKE_OSX_DEPLOYMENT_TARGET=13.3 -DGGML_CPU_KLEIDIAI=ON"
           - build: 'x64'
             arch: 'x64'
             os: macos-15-intel
@@ -1402,7 +1402,6 @@ jobs:
         run: |
           sysctl -a
           cmake -B build -G Xcode \
-            -DGGML_METAL_USE_BF16=ON \
             -DGGML_METAL_EMBED_LIBRARY=ON \
             -DLLAMA_OPENSSL=OFF \
             -DLLAMA_BUILD_APP=OFF \

SECURITY.md

@@ -21,11 +21,18 @@ Please disclose it as a private [security advisory](https://github.com/ggml-org/
 
 A team of volunteers on a reasonable-effort basis maintains this project. As such, please give us at least 90 days to work on a fix before public exposure.
 
+### AI-powered code scan
+
+llama.cpp has an AI security scanner that scans the code periodically. The full prompts and tool set can be found in [ggml-org/security-scan-prompt](https://github.com/ggml-org/security-scan-prompt).
+
+We greatly appreciate reports that reflect genuine research effort, and we are happy to spend our time reviewing them. Findings that an autonomous AI agent can surface on its own add little on top of the scans we already run.
+
 ### Requirements
 
 Before submitting your report, ensure you meet the following requirements:
 
 - You have read this policy and fully understand it.
+- You have searched for existing discussions of the issue. If it has already been reported, your report will likely be rejected as a duplicate.
 - AI is only permitted in an assistive capacity as stated in [AGENTS.md](AGENTS.md). We do not accept reports that are written exclusively by AI.
 - Your report must include a working Proof-of-Concept in the form of a script and/or attached files.
 
@@ -46,6 +53,8 @@ Only vulnerabilities that fall within these parts of the project are considered
 
 Note that none of the topics under [Using llama.cpp securely](#using-llamacpp-securely) are considered vulnerabilities in LLaMA C++.
 
+Denial-of-Service (DoS) bugs are generally not treated as vulnerabilities. We don't reject them outright, but we look at them case-by-case and only accept those that are genuinely worth fixing.
+
 For vulnerabilities that fall within the `vendor` directory, please report them directly to the third-party project.
 
 ## Using llama.cpp securely

build-xcframework.sh

@@ -17,7 +17,6 @@ LLAMA_BUILD_MTMD=ON
 GGML_METAL=ON
 GGML_METAL_EMBED_LIBRARY=ON
 GGML_BLAS_DEFAULT=ON
-GGML_METAL_USE_BF16=ON
 GGML_OPENMP=OFF
 
 COMMON_C_FLAGS="-Wno-macro-redefined -Wno-shorten-64-to-32 -Wno-unused-command-line-argument -g"
@@ -44,7 +43,6 @@ COMMON_CMAKE_ARGS=(
     -DGGML_METAL_EMBED_LIBRARY=${GGML_METAL_EMBED_LIBRARY}
     -DGGML_BLAS_DEFAULT=${GGML_BLAS_DEFAULT}
     -DGGML_METAL=${GGML_METAL}
-    -DGGML_METAL_USE_BF16=${GGML_METAL_USE_BF16}
     -DGGML_NATIVE=OFF
     -DGGML_OPENMP=${GGML_OPENMP}
 )

common/fit.cpp

@@ -136,7 +136,10 @@ static std::vector<llama_device_memory_data> common_get_device_memory_data_impl(
         devs.push_back(llama_model_get_device(model, i));
     }
 
-    hp_ngl         = llama_model_n_layer(model) + llama_model_n_layer_nextn(model);
+    hp_ngl         = llama_model_n_layer(model);
+    if (mparams->load_mtp) {
+        hp_ngl    += llama_model_n_layer_nextn(model);
+    }
     hp_n_ctx_train = llama_model_n_ctx_train(model);
     hp_n_expert    = llama_model_n_expert(model);
 

gguf-py/gguf/scripts/gguf_convert_endian.py

@@ -59,11 +59,29 @@ def byteswap_q6_k(tensor, block_offs):
     delta.byteswap(inplace=True)
 
 
+def byteswap_q1_0(tensor, block_offs):
+    # Each block_q1_0 consists of an f16 delta followed by 16 int8 quantizations.
+
+    # Byte-Swap f16 sized delta field
+    delta = tensor.data[block_offs:block_offs + 2].view(dtype=np.uint16)
+    delta.byteswap(inplace=True)
+
+
+def byteswap_tq2_0(tensor, block_offs):
+    # Each block_tq2_0 consists of 64 int8 values followed by 1 f16 value.
+
+    # Byte-Swap f16 sized field
+    delta = tensor.data[block_offs + 64:block_offs + 66].view(dtype=np.uint16)
+    delta.byteswap(inplace=True)
+
+
 byteswap_tensors = {
+    gguf.GGMLQuantizationType.Q1_0:  byteswap_q1_0,
     gguf.GGMLQuantizationType.Q4_0:  byteswap_q4_0,
     gguf.GGMLQuantizationType.Q8_0:  byteswap_q8_0,
     gguf.GGMLQuantizationType.Q4_K:  byteswap_q4_k,
     gguf.GGMLQuantizationType.Q6_K:  byteswap_q6_k,
+    gguf.GGMLQuantizationType.TQ2_0: byteswap_tq2_0,
     gguf.GGMLQuantizationType.MXFP4: byteswap_noop,
     gguf.GGMLQuantizationType.NVFP4: byteswap_noop,
 }

scripts/sync_vendor.py

@@ -24,10 +24,31 @@ vendor = {
     "https://raw.githubusercontent.com/sheredom/subprocess.h/8671cee1fc09f11a70ce3782a0ee13177c3aa387/subprocess.h": "vendor/sheredom/subprocess.h",
 }
 
+# TODO @ngxson : this is temporary, to be removed in the future
+patches = [
+    # https://github.com/sheredom/subprocess.h/pull/102
+    "vendor/sheredom/patch-bsd.patch",
+    # https://github.com/sheredom/subprocess.h/pull/101
+    "vendor/sheredom/patch-windows-quote-backslash.patch",
+    # https://github.com/sheredom/subprocess.h/pull/104
+    # note: must be applied after patch-bsd.patch, they touch adjacent lines
+    "vendor/sheredom/patch-glibc-older-than-2.29.patch",
+]
+
 for url, filename in vendor.items():
     print(f"downloading {url} to {filename}") # noqa: NP100
     urllib.request.urlretrieve(url, filename)
 
+for patch in patches:
+    print(f"applying {patch}") # noqa: NP100
+    try:
+        subprocess.check_call([
+            "git", "apply", "--directory", os.path.dirname(patch), patch
+        ])
+    except Exception as e:
+        print(f"Error: {e}") # noqa: NP100
+        sys.exit(1)
+
 print("Splitting httplib.h...") # noqa: NP100
 try:
     subprocess.check_call([

src/llama-model.cpp

@@ -2890,6 +2890,21 @@ void llama_model_base::create_tensor_qkv(llama_layer & layer, int bid,
         int64_t n_embd_, int64_t n_embd_q_, int64_t n_embd_k_, int64_t n_embd_v_,
         int flags) {
     const int64_t n_embd_qkv = n_embd_q_ + n_embd_k_ + n_embd_v_;
+
+    if (flags & TENSOR_SKIP) {
+        const int skip = TENSOR_NOT_REQUIRED | TENSOR_SKIP;
+
+        create_tensor(tn(LLM_TENSOR_ATTN_QKV, "weight", bid), {n_embd_, n_embd_qkv}, skip | TENSOR_SKIP_IF_VIRTUAL);
+        create_tensor(tn(LLM_TENSOR_ATTN_QKV, "bias",   bid), {n_embd_qkv},          skip | TENSOR_SKIP_IF_VIRTUAL);
+        create_tensor(tn(LLM_TENSOR_ATTN_Q,   "weight", bid), {n_embd_, n_embd_q_},  skip);
+        create_tensor(tn(LLM_TENSOR_ATTN_K,   "weight", bid), {n_embd_, n_embd_k_},  skip);
+        create_tensor(tn(LLM_TENSOR_ATTN_V,   "weight", bid), {n_embd_, n_embd_v_},  skip);
+        create_tensor(tn(LLM_TENSOR_ATTN_Q,   "bias",   bid), {n_embd_q_},           skip);
+        create_tensor(tn(LLM_TENSOR_ATTN_K,   "bias",   bid), {n_embd_k_},           skip);
+        create_tensor(tn(LLM_TENSOR_ATTN_V,   "bias",   bid), {n_embd_v_},           skip);
+        return;
+    }
+
     layer.wqkv = create_tensor(tn(LLM_TENSOR_ATTN_QKV, "weight", bid), {n_embd_, n_embd_qkv}, TENSOR_NOT_REQUIRED | TENSOR_SKIP_IF_VIRTUAL);
     if (layer.wqkv) {
         layer.wqkv_b = create_tensor(tn(LLM_TENSOR_ATTN_QKV, "bias", bid), {n_embd_qkv}, TENSOR_NOT_REQUIRED | TENSOR_SKIP_IF_VIRTUAL);

tools/mtmd/models/deepseekocr.cpp

@@ -253,6 +253,9 @@ ggml_cgraph * clip_graph_deepseekocr::build() {
 
     bool is_overview = img.add_viewsep;
     int n_tiles_per_row = 0;
+    // number of separate "row" images batched together in this graph call
+    // (captured now, before n_batch below gets repurposed as the SAM/ViT batch size)
+    const int n_rows_batch = n_batch;
 
     // note: we expect either a batch of rows or a batch of overviews, but not a mix of both
 
@@ -272,16 +275,18 @@ ggml_cgraph * clip_graph_deepseekocr::build() {
         GGML_ASSERT(img.ny() % img.nx() == 0);
         n_tiles_per_row = img.ny() / img.nx();
 
-        // input shape: [tile_size, tile_size * n_tiles_per_row, 3]
-        // we want to reshape it to [tile_size, tile_size, 3, n_tiles_per_row]
-        inp_raw = ggml_reshape_4d(ctx0, inp_raw, img.nx(), img.nx(), n_tiles_per_row, 3);
-        inp_raw = ggml_cont(ctx0, ggml_permute(ctx0, inp_raw, 0, 1, 3, 2));
+        // each entry is one "row" image of shape [tile_size, tile_size * n_tiles_per_row, 3];
+        // merge the tile axis into the batch axis, giving a combined SAM input of shape
+        // [tile_size, tile_size, 3, n_tiles_per_row * n_rows_batch] (tile fast, row slow)
+        inp_raw = ggml_reshape_4d(ctx0, inp_raw, img.nx() * img.nx(), n_tiles_per_row, 3, n_rows_batch);
+        inp_raw = ggml_cont(ctx0, ggml_permute(ctx0, inp_raw, 0, 2, 1, 3));
+        inp_raw = ggml_reshape_4d(ctx0, inp_raw, img.nx(), img.nx(), 3, n_tiles_per_row * n_rows_batch);
     }
 
     ggml_tensor * sam_out = build_sam(inp_raw);
 
     if (!is_overview) {
-        n_batch = n_tiles_per_row;
+        n_batch = n_tiles_per_row * n_rows_batch;
     }
 
     const int clip_n_patches = sam_out->ne[0] * sam_out->ne[1];
@@ -354,34 +359,36 @@ ggml_cgraph * clip_graph_deepseekocr::build() {
         const auto w     = h;
         const auto n_dim = cur->ne[0];
 
-        ggml_tensor * imgnl = ggml_repeat_4d(ctx0, model.image_newline, n_dim, 1, h, 1);
-        cur = ggml_reshape_3d(ctx0, cur, n_dim, w, h);
-        cur = ggml_reshape_2d(ctx0, ggml_concat(ctx0, cur, imgnl, 1), n_dim, (w + 1) * h);
-        cur = ggml_concat(ctx0, cur, model.view_seperator, 1);  // (n_dim, h*(w+1) + 1)
+        ggml_tensor * imgnl = ggml_repeat_4d(ctx0, model.image_newline, n_dim, 1, h, n_batch);
+        cur = ggml_reshape_4d(ctx0, cur, n_dim, w, h, n_batch);
+        cur = ggml_reshape_3d(ctx0, ggml_concat(ctx0, cur, imgnl, 1), n_dim, (w + 1) * h, n_batch);
+        ggml_tensor * vs = ggml_repeat_4d(ctx0, model.view_seperator, n_dim, 1, n_batch, 1);
+        cur = ggml_concat(ctx0, cur, vs, 1);  // (n_dim, h*(w+1) + 1, n_batch)
     } else {
         // tile row: interleave tiles within each row, add newline per row
-        const int grid_x      = static_cast<int>(std::sqrt(static_cast<float>(clip_n_patches)));
-        const int grid_y      = grid_x;
-        const auto n_dim      = cur->ne[0];
+        const int  grid_x = static_cast<int>(std::sqrt(static_cast<float>(clip_n_patches)));
+        const int  grid_y = grid_x;
+        const auto n_dim  = cur->ne[0];
 
-        // (n_dim, clip_n_patches, n_batch) -> (n_dim, grid_x, grid_y, n_batch)
-        cur = ggml_reshape_4d(ctx0, cur, n_dim, grid_x, grid_y, n_batch);
+        // merge n_dim into the grid_x axis, freeing the 4th axis for n_rows_batch
+        // (n_dim, clip_n_patches, n_tiles_per_row * n_rows_batch) -> (n_dim*grid_x, grid_y, n_tiles_per_row, n_rows_batch)
+        cur = ggml_reshape_4d(ctx0, cur, n_dim * grid_x, grid_y, n_tiles_per_row, n_rows_batch);
 
         // tiles: re-order from A.row0 A.row1 B.row0 B.row1 ...
         //        to A.row0 B.row0 A.row1 B.row1 ...
         //        then add nl: A.row0 B.row0 [nl] A.row1 B.row1 [nl] ...
-        // interleave tiles: (n_dim, grid_x, grid_y, n_batch) -> (n_dim, grid_x, n_batch, grid_y)
-        cur = ggml_cont(ctx0, ggml_permute(ctx0, cur, 0, 1, 3, 2));
+        // interleave tiles: -> (n_dim*grid_x, n_tiles_per_row, grid_y, n_rows_batch)
+        cur = ggml_cont(ctx0, ggml_permute(ctx0, cur, 0, 2, 1, 3));
 
-        // merge: (n_dim, grid_x, n_batch, grid_y) -> (n_dim, grid_x*n_batch, grid_y, 1)
-        cur = ggml_reshape_4d(ctx0, cur, n_dim, grid_x * n_batch, grid_y, 1);
+        // merge: -> (n_dim, grid_x*n_tiles_per_row, grid_y, n_rows_batch)
+        cur = ggml_reshape_4d(ctx0, cur, n_dim, grid_x * n_tiles_per_row, grid_y, n_rows_batch);
 
-        // append newline per row: (n_dim, grid_x*n_batch+1, grid_y, 1)
-        ggml_tensor * imgnl = ggml_repeat_4d(ctx0, model.image_newline, n_dim, 1, grid_y, 1);
+        // append newline per row: (n_dim, grid_x*n_tiles_per_row+1, grid_y, n_rows_batch)
+        ggml_tensor * imgnl = ggml_repeat_4d(ctx0, model.image_newline, n_dim, 1, grid_y, n_rows_batch);
         cur = ggml_concat(ctx0, cur, imgnl, 1);
 
-        // flatten: (n_dim, (grid_x*n_batch+1)*grid_y)
-        cur = ggml_reshape_2d(ctx0, cur, n_dim, (grid_x * n_batch + 1) * grid_y);
+        // flatten: (n_dim, (grid_x*n_tiles_per_row+1)*grid_y, n_rows_batch)
+        cur = ggml_reshape_3d(ctx0, cur, n_dim, (grid_x * n_tiles_per_row + 1) * grid_y, n_rows_batch);
     }
 
     cb(cur, "dsocr_output", -1);

tools/mtmd/models/deepseekocr2.cpp

@@ -14,8 +14,9 @@ ggml_cgraph * clip_graph_deepseekocr2::build() {
     {
         ggml_tensor * inp;
 
-        inp = ggml_reshape_2d(ctx0, sam_out, sam_out->ne[0] * sam_out->ne[1], sam_out->ne[2]); // H*W, C
-        inp = ggml_cont(ctx0, ggml_permute(ctx0, inp, 1, 0, 2, 3));
+        // H*W, C, B
+        inp = ggml_reshape_3d(ctx0, sam_out, sam_out->ne[0] * sam_out->ne[1], sam_out->ne[2], sam_out->ne[3]);
+        inp = ggml_cont(ctx0, ggml_permute(ctx0, inp, 1, 0, 2, 3)); // C, H*W, B
 
         auto num_image_tokens = inp->ne[1]; // H*W
         GGML_ASSERT(num_image_tokens == 144 || num_image_tokens == 256);
@@ -32,8 +33,10 @@ ggml_cgraph * clip_graph_deepseekocr2::build() {
             num_queries = 144;
         }
 
-        // (B, num_image_tokens + num_queries, C)
-        inp = ggml_concat(ctx0, inp, ggml_cast(ctx0, query_embed, inp->type), 1);
+        // repeat the query embedding per batch item, then append: (C, num_image_tokens + num_queries, B)
+        query_embed = ggml_cast(ctx0, query_embed, inp->type);
+        query_embed = ggml_repeat_4d(ctx0, query_embed, query_embed->ne[0], num_queries, inp->ne[2], 1);
+        inp = ggml_concat(ctx0, inp, query_embed, 1);
 
         auto seq_len = inp->ne[1];
 
@@ -57,7 +60,7 @@ ggml_cgraph * clip_graph_deepseekocr2::build() {
                                       /* learned_pos_embd */ nullptr, add_rope, vit_opts);
 
         cur = ggml_cont(ctx0,
-                        ggml_view_2d(ctx0, cur, cur->ne[0], num_queries, cur->nb[1],
+                        ggml_view_3d(ctx0, cur, cur->ne[0], num_queries, cur->ne[2], cur->nb[1], cur->nb[2],
                                      cur->nb[1] * (cur->ne[1] - num_queries))); // only take query tokens for output
 
         ggml_build_forward_expand(gf, cur);
@@ -71,7 +74,8 @@ ggml_cgraph * clip_graph_deepseekocr2::build() {
 
     // view_seperator only after the global view
     if (img.add_viewsep) {
-        cur = ggml_concat(ctx0, cur, model.view_seperator, 1); // (n_dim, 257)
+        ggml_tensor * vs = ggml_repeat_4d(ctx0, model.view_seperator, model.view_seperator->ne[0], 1, cur->ne[2], 1);
+        cur = ggml_concat(ctx0, cur, vs, 1); // (n_dim, 257, n_batch)
     }
 
     cb(cur, "dsocr2_output", -1);

tools/mtmd/models/models.h

@@ -138,12 +138,13 @@ struct clip_graph_deepseekocr : clip_graph {
     clip_graph_deepseekocr(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}
     ggml_cgraph * build() override;
     ggml_tensor * build_sam(ggml_tensor * inp); // build the SAM model
-    // bool support_batch() const override { return true; } // TODO: support batch for DeepSeek-OCR v1
+    bool support_batch() const override { return true; }
 };
 
 struct clip_graph_deepseekocr2 : clip_graph_deepseekocr {
     clip_graph_deepseekocr2(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph_deepseekocr(ctx, img) {}
     ggml_cgraph * build() override; // reuses build_sam() from base
+    bool support_batch() const override { return true; }
 };
 
 struct clip_graph_conformer : clip_graph {

tools/server/README.md

@@ -1076,6 +1076,10 @@ In *router mode* the query param `?model={model_id}` has to be set. This endpoin
 | `llamacpp:n_tokens_max` | Counter | High watermark of the context size observed. |
 | `llamacpp:n_decode_total` | Counter | Total Number of llama_decode() calls. |
 | `llamacpp:n_busy_slots_per_decode` | Gauge | Average number of busy slots per llama_decode() call. |
+| `llamacpp:spec_decode_num_draft_tokens_total` | Counter | Total draft tokens generated (0 when spec-decode is off). |
+| `llamacpp:spec_decode_num_accepted_tokens_total` | Counter | Total draft tokens accepted by the target model (0 when spec-decode is off). |
+| `llamacpp:spec_decode_num_drafts_total` | Counter | Total speculative decoding verification steps (0 when spec-decode is off). |
+| `llamacpp:spec_decode_num_accepted_tokens_per_pos_total` | Counter | Accepted tokens per draft position (labeled `position="N"`; absent when spec-decode is off or before the first completed speculative request). |
 
 ### POST `/slots/{id_slot}?action=save`: Save the prompt cache of the specified slot to a file.
 

tools/server/server-context.cpp

@@ -845,6 +845,11 @@ struct server_metrics {
     uint64_t n_decode_total     = 0;
     uint64_t n_busy_slots_total = 0;
 
+    uint64_t n_draft_tokens_total      = 0;
+    uint64_t n_draft_accepted_total    = 0;
+    uint64_t n_draft_verif_steps_total = 0;
+    std::vector<uint64_t> n_accepted_per_pos_total;
+
     void init() {
         t_start = ggml_time_us();
     }
@@ -863,6 +868,17 @@ struct server_metrics {
         n_tokens_predicted         += slot.n_decoded;
         t_tokens_generation        += slot.t_token_generation;
         t_tokens_generation_total  += slot.t_token_generation;
+
+        n_draft_tokens_total      += slot.n_draft_total;
+        n_draft_accepted_total    += slot.n_draft_accepted;
+        n_draft_verif_steps_total += slot.n_draft_verif_steps;
+
+        if (n_accepted_per_pos_total.size() < slot.n_accepted_per_pos.size()) {
+            n_accepted_per_pos_total.resize(slot.n_accepted_per_pos.size(), 0);
+        }
+        for (size_t i = 0; i < slot.n_accepted_per_pos.size(); i++) {
+            n_accepted_per_pos_total[i] += slot.n_accepted_per_pos[i];
+        }
     }
 
     void on_decoded(const std::vector<server_slot> & slots) {
@@ -2552,6 +2568,11 @@ private:
                     res->n_decode_total          = metrics.n_decode_total;
                     res->n_busy_slots_total      = metrics.n_busy_slots_total;
 
+                    res->n_draft_tokens_total      = metrics.n_draft_tokens_total;
+                    res->n_draft_accepted_total    = metrics.n_draft_accepted_total;
+                    res->n_draft_verif_steps_total = metrics.n_draft_verif_steps_total;
+                    res->n_accepted_per_pos_total  = metrics.n_accepted_per_pos_total;
+
                     if (task.metrics_reset_bucket) {
                         metrics.reset_bucket();
                     }
@@ -4440,6 +4461,18 @@ void server_routes::init_routes() {
                     {"name",  "n_tokens_max"},
                     {"help",  "Largest observed n_tokens."},
                     {"value",  res_task->n_tokens_max}
+            }, {
+                    {"name",  "spec_decode_num_draft_tokens_total"},
+                    {"help",  "Total draft tokens generated"},
+                    {"value",  res_task->n_draft_tokens_total}
+            }, {
+                    {"name",  "spec_decode_num_accepted_tokens_total"},
+                    {"help",  "Total draft tokens accepted by the target model"},
+                    {"value",  res_task->n_draft_accepted_total}
+            }, {
+                    {"name",  "spec_decode_num_drafts_total"},
+                    {"help",  "Total speculative decoding verification steps"},
+                    {"value",  res_task->n_draft_verif_steps_total}
             }}},
             {"gauge", {{
                     {"name",  "prompt_tokens_seconds"},
@@ -4481,6 +4514,17 @@ void server_routes::init_routes() {
             }
         }
 
+        // labeled counter: one time series per draft position
+        if (!res_task->n_accepted_per_pos_total.empty()) {
+            prometheus << "# HELP llamacpp:spec_decode_num_accepted_tokens_per_pos_total"
+                          " Accepted tokens per draft position\n"
+                       << "# TYPE llamacpp:spec_decode_num_accepted_tokens_per_pos_total counter\n";
+            for (size_t i = 0; i < res_task->n_accepted_per_pos_total.size(); i++) {
+                prometheus << "llamacpp:spec_decode_num_accepted_tokens_per_pos_total{position=\""
+                           << i << "\"} " << res_task->n_accepted_per_pos_total[i] << "\n";
+            }
+        }
+
         res->headers["Process-Start-Time-Unix"] = std::to_string(res_task->t_start);
         res->content_type = "text/plain; version=0.0.4";
         res->status = 200;

tools/server/server-task.cpp

@@ -1560,6 +1560,11 @@ json server_task_result_metrics::to_json() {
         { "n_decode_total",                  n_decode_total },
         { "n_busy_slots_total",              n_busy_slots_total },
 
+        { "n_draft_tokens_total",            n_draft_tokens_total },
+        { "n_draft_accepted_total",          n_draft_accepted_total },
+        { "n_draft_verif_steps_total",       n_draft_verif_steps_total },
+        { "n_accepted_per_pos_total",        n_accepted_per_pos_total },
+
         { "slots",                           slots_data },
     };
 }

tools/server/server-task.h

@@ -532,6 +532,11 @@ struct server_task_result_metrics : server_task_result {
     uint64_t n_decode_total     = 0;
     uint64_t n_busy_slots_total = 0;
 
+    uint64_t n_draft_tokens_total      = 0;
+    uint64_t n_draft_accepted_total    = 0;
+    uint64_t n_draft_verif_steps_total = 0;
+    std::vector<uint64_t> n_accepted_per_pos_total;
+
     // while we can also use std::vector<server_slot> this requires copying the slot object which can be quite messy
     // therefore, we use json to temporarily store the slot.to_json() result
     json slots_data = json::array();

tools/ui/.storybook/main.ts

@@ -11,7 +11,8 @@ const config: StorybookConfig = {
 		'@chromatic-com/storybook',
 		'@storybook/addon-vitest',
 		'@storybook/addon-a11y',
-		'@storybook/addon-docs'
+		'@storybook/addon-docs',
+		'@storybook/addon-mcp'
 	],
 	framework: '@storybook/sveltekit',
 	viteFinal: async (config) => {

tools/ui/.storybook/vitest.setup.ts

@@ -1,12 +0,0 @@
-import * as a11yAddonAnnotations from '@storybook/addon-a11y/preview';
-import { setProjectAnnotations } from '@storybook/sveltekit';
-import * as previewAnnotations from './preview';
-import { beforeAll } from 'vitest';
-
-const project = setProjectAnnotations([a11yAddonAnnotations, previewAnnotations]);
-
-beforeAll(async () => {
-	if (project.beforeAll) {
-		await project.beforeAll();
-	}
-});

tools/ui/package.json

@@ -27,20 +27,20 @@
 		"cleanup": "rm -rf .svelte-kit build node_modules test-results dist dev-dist debug-storybook.log static/pwa-*.png static/maskable-icon-*.png static/apple-touch-icon-*.png static/apple-splash-*.png static/favicon*.ico"
 	},
 	"devDependencies": {
-		"@chromatic-com/storybook": "5.0.0",
+		"@chromatic-com/storybook": "5.2.1",
 		"@eslint/compat": "1.4.1",
 		"@eslint/js": "9.39.2",
 		"@internationalized/date": "3.12.2",
 		"@lucide/svelte": "1.25.0",
-		"@modelcontextprotocol/sdk": "1.26.0",
+		"@modelcontextprotocol/sdk": "1.30.0",
 		"@playwright/test": "1.56.1",
-		"@storybook/addon-a11y": "10.2.4",
-		"@storybook/addon-docs": "10.2.4",
-		"@storybook/addon-svelte-csf": "5.0.10",
-		"@storybook/addon-vitest": "10.2.4",
-		"@storybook/sveltekit": "10.2.4",
+		"@storybook/addon-a11y": "10.5.6",
+		"@storybook/addon-docs": "10.5.6",
+		"@storybook/addon-svelte-csf": "5.1.2",
+		"@storybook/addon-vitest": "10.5.6",
+		"@storybook/sveltekit": "10.5.6",
 		"@sveltejs/adapter-static": "3.0.10",
-		"@sveltejs/kit": "2.60.1",
+		"@sveltejs/kit": "2.70.2",
 		"@sveltejs/vite-plugin-svelte": "6.2.1",
 		"@tailwindcss/forms": "0.5.10",
 		"@tailwindcss/typography": "0.5.16",
@@ -48,16 +48,16 @@
 		"@types/node": "24.13.0",
 		"@vite-pwa/assets-generator": "1.0.2",
 		"@vite-pwa/sveltekit": "1.1.0",
-		"@vitest/browser": "4.1.8",
-		"@vitest/browser-playwright": "4.1.8",
-		"@vitest/coverage-v8": "4.1.8",
+		"@vitest/browser": "4.1.10",
+		"@vitest/browser-playwright": "4.1.10",
+		"@vitest/coverage-v8": "4.1.10",
 		"bits-ui": "2.18.1",
 		"clsx": "2.1.1",
 		"dexie": "4.4.3",
-		"dompurify": "3.4.11",
+		"dompurify": "3.4.13",
 		"eslint": "9.39.4",
 		"eslint-config-prettier": "10.1.8",
-		"eslint-plugin-storybook": "10.4.2",
+		"eslint-plugin-storybook": "10.5.6",
 		"eslint-plugin-svelte": "3.19.0",
 		"fflate": "0.8.3",
 		"globals": "16.5.0",
@@ -82,7 +82,7 @@
 		"remark-math": "6.0.0",
 		"remark-rehype": "11.1.2",
 		"sass": "1.100.0",
-		"storybook": "10.4.2",
+		"storybook": "10.5.6",
 		"svelte": "5.56.1",
 		"svelte-check": "4.6.0",
 		"svelte-sonner": "1.1.1",
@@ -95,13 +95,16 @@
 		"unified": "11.0.5",
 		"unist-util-visit": "5.1.0",
 		"uuid": "13.0.2",
-		"vite": "7.3.5",
+		"vite": "7.3.6",
 		"vite-plugin-devtools-json": "0.2.1",
-		"vitest": "4.1.8",
+		"vitest": "4.1.10",
 		"vitest-browser-svelte": "2.1.1",
-		"workbox-window": "7.4.1"
+		"workbox-window": "7.4.1",
+		"@storybook/addon-mcp": "0.7.0"
 	},
 	"overrides": {
-		"cookie": "1.1.1"
+		"cookie": "1.1.1",
+		"sharp": "0.35.3",
+		"valibot": "1.4.2"
 	}
 }

tools/ui/src/lib/constants/settings-registry.ts

@@ -198,7 +198,7 @@ const SETTINGS_REGISTRY: Record<string, SettingsSectionEntry> = {
 				key: SETTINGS_KEYS.SHOW_MESSAGE_STATS,
 				label: 'Show message generation statistics',
 				help: 'Display generation statistics (tokens/second, token count, duration) below each assistant message.',
-				defaultValue: false,
+				defaultValue: true,
 				type: SettingsFieldType.CHECKBOX,
 				section: SETTINGS_SECTION_SLUGS.DISPLAY
 			},

tools/ui/vite.config.ts

@@ -76,8 +76,7 @@ export default defineConfig({
 				extends: './vite.config.ts',
 				test: {
 					name: 'ui',
-					browser: { ...browserBaseConfig, instances: [{ browser: 'chromium', headless: true }] },
-					setupFiles: ['./.storybook/vitest.setup.ts']
+					browser: { ...browserBaseConfig, instances: [{ browser: 'chromium', headless: true }] }
 				},
 				plugins: [
 					storybookTest({

vendor/sheredom/patch-bsd.patch

@@ -0,0 +1,19 @@
+Fix build on NetBSD, which provides posix_spawn_file_actions_addchdir()
+but not the _np() variant.
+
+Upstream PR: https://github.com/sheredom/subprocess.h/pull/102
+Applied locally by scripts/sync_vendor.py until it is merged upstream.
+
+diff --git a/subprocess.h b/subprocess.h
+index 5e809023a4..74a4e006c7 100644
+--- a/subprocess.h
++++ b/subprocess.h
+@@ -1205,7 +1205,7 @@ cleanup:
+ 
+   // Set working directory
+   if (process_cwd) {
+-#if defined(__APPLE__) && MAC_OS_X_VERSION_MIN_REQUIRED >= 260000
++#if defined(__NetBSD__) || (defined(__APPLE__) && MAC_OS_X_VERSION_MIN_REQUIRED >= 260000)
+     posix_error = posix_spawn_file_actions_addchdir(&actions, process_cwd);
+ #else
+ #if defined(__APPLE__) && defined(__clang__)

vendor/sheredom/patch-glibc-older-than-2.29.patch

@@ -0,0 +1,47 @@
+Fix building against glibc older than 2.29, which has no
+posix_spawn_file_actions_addchdir_np (the symbol is genuinely absent from
+libc.so, so no feature-test macro helps). Affects manylinux2014 (glibc 2.17)
+and manylinux_2_28, and was reported on RHEL 8.1. A requested process_cwd now
+fails with ENOSYS there instead of failing the build.
+
+Upstream PR: https://github.com/sheredom/subprocess.h/pull/104
+Applied locally by scripts/sync_vendor.py until it is merged upstream.
+
+(the README.md and test/ changes from the PR are omitted, we only vendor
+subprocess.h; rebased on top of patch-bsd.patch, so apply it after that one)
+
+diff --git a/subprocess.h b/subprocess.h
+index 1ef424a..c363393 100644
+--- a/subprocess.h
++++ b/subprocess.h
+@@ -274,6 +274,21 @@ subprocess_weak int subprocess_alive(struct subprocess_s *const process);
+ #include <unistd.h>
+ #endif
+ 
++/* Whether subprocess_create_ex can honour process_cwd. glibc only gained
++   posix_spawn_file_actions_addchdir_np in 2.29. Define this yourself to
++   override the detection, for instance on musl older than 1.1.24. */
++#if !defined(SUBPROCESS_HAVE_CWD)
++#if defined(__GLIBC__)
++#if __GLIBC_PREREQ(2, 29)
++#define SUBPROCESS_HAVE_CWD 1
++#else
++#define SUBPROCESS_HAVE_CWD 0
++#endif
++#else
++#define SUBPROCESS_HAVE_CWD 1
++#endif
++#endif
++
+ #if defined(_WIN32)
+ 
+ #include <wchar.h>
+@@ -1219,6 +1234,8 @@ cleanup:
+   if (process_cwd) {
+ #if defined(__NetBSD__) || (defined(__APPLE__) && MAC_OS_X_VERSION_MIN_REQUIRED >= 260000)
+     posix_error = posix_spawn_file_actions_addchdir(&actions, process_cwd);
++#elif !SUBPROCESS_HAVE_CWD
++    posix_error = ENOSYS;
+ #else
+ #if defined(__APPLE__) && defined(__clang__)
+ #pragma clang diagnostic push

vendor/sheredom/patch-windows-quote-backslash.patch

@@ -0,0 +1,107 @@
+Fix Windows command line quoting of backslash runs: a trailing backslash, or
+backslashes preceding a double quote, were not doubled, so CommandLineToArgvW
+in the child parsed them as escapes and mangled the argument list.
+
+Upstream PR: https://github.com/sheredom/subprocess.h/pull/101
+Applied locally by scripts/sync_vendor.py until it is merged upstream.
+
+(the test/ changes from the PR are omitted, we only vendor subprocess.h)
+
+diff --git a/subprocess.h b/subprocess.h
+index 5e80902..b06ad4d 100644
+--- a/subprocess.h
++++ b/subprocess.h
+@@ -653,6 +653,7 @@ int subprocess_create_ex(const char *const commandLine[], int options,
+   int wide_len;
+   int i, j;
+   int need_quoting;
++  subprocess_size_t bs_run;
+   unsigned long flags = 0;
+   unsigned long last_error = 0;
+   int result = subprocess_error_unknown;
+@@ -906,25 +907,29 @@ int subprocess_create_ex(const char *const commandLine[], int options,
+     len++;
+ 
+     // Quote the argument if it has a space in it
+-    if (strpbrk(commandLine[i], "\t\v ") != SUBPROCESS_NULL ||
+-        commandLine[i][0] == SUBPROCESS_NULL)
++    need_quoting = strpbrk(commandLine[i], "\t\v ") != SUBPROCESS_NULL ||
++                   commandLine[i][0] == SUBPROCESS_NULL;
++    if (need_quoting)
+       len += 2;
+ 
++    bs_run = 0;
+     for (j = 0; '\0' != commandLine[i][j]; j++) {
+-      switch (commandLine[i][j]) {
+-      default:
+-        break;
+-      case '\\':
+-        if (commandLine[i][j + 1] == '"') {
+-          len++;
+-        }
++      len++;
+ 
+-        break;
+-      case '"':
+-        len++;
+-        break;
++      if ('\\' == commandLine[i][j]) {
++        bs_run++;
++      } else {
++        if ('"' == commandLine[i][j]) {
++          // Duplicate the preceding run and escape the quote.
++          len += bs_run + 1;
++        }
++        bs_run = 0;
+       }
+-      len++;
++    }
++
++    if (need_quoting) {
++      // Duplicate trailing slashes before the generated closing quote.
++      len += bs_run;
+     }
+   }
+ 
+@@ -949,22 +954,29 @@ int subprocess_create_ex(const char *const commandLine[], int options,
+       commandLineCombined[len++] = '"';
+     }
+ 
+-    for (j = 0; '\0' != commandLine[i][j]; j++) {
+-      switch (commandLine[i][j]) {
+-      default:
+-        break;
+-      case '\\':
+-        if (commandLine[i][j + 1] == '"') {
+-          commandLineCombined[len++] = '\\';
+-        }
++    for (j = 0; '\0' != commandLine[i][j];) {
++      bs_run = 0;
++      while ('\\' == commandLine[i][j]) {
++        bs_run++;
++        j++;
++      }
++
++      if ('"' == commandLine[i][j]) {
++        // 2n + 1 slashes preserve n slashes and escape the quote.
++        bs_run = (bs_run * 2) + 1;
++      } else if ('\0' == commandLine[i][j] && need_quoting) {
++        // 2n slashes preserve n slashes before the closing quote.
++        bs_run *= 2;
++      }
+ 
+-        break;
+-      case '"':
++      while (bs_run > 0) {
+         commandLineCombined[len++] = '\\';
+-        break;
++        bs_run--;
+       }
+ 
+-      commandLineCombined[len++] = commandLine[i][j];
++      if ('\0' != commandLine[i][j]) {
++        commandLineCombined[len++] = commandLine[i][j++];
++      }
+     }
+     if (need_quoting) {
+       commandLineCombined[len++] = '"';

vendor/sheredom/subprocess.h

@@ -274,6 +274,21 @@ subprocess_weak int subprocess_alive(struct subprocess_s *const process);
 #include <unistd.h>
 #endif
 
+/* Whether subprocess_create_ex can honour process_cwd. glibc only gained
+   posix_spawn_file_actions_addchdir_np in 2.29. Define this yourself to
+   override the detection, for instance on musl older than 1.1.24. */
+#if !defined(SUBPROCESS_HAVE_CWD)
+#if defined(__GLIBC__)
+#if __GLIBC_PREREQ(2, 29)
+#define SUBPROCESS_HAVE_CWD 1
+#else
+#define SUBPROCESS_HAVE_CWD 0
+#endif
+#else
+#define SUBPROCESS_HAVE_CWD 1
+#endif
+#endif
+
 #if defined(_WIN32)
 
 #include <wchar.h>
@@ -653,6 +668,7 @@ int subprocess_create_ex(const char *const commandLine[], int options,
   int wide_len;
   int i, j;
   int need_quoting;
+  subprocess_size_t bs_run;
   unsigned long flags = 0;
   unsigned long last_error = 0;
   int result = subprocess_error_unknown;
@@ -906,25 +922,29 @@ int subprocess_create_ex(const char *const commandLine[], int options,
     len++;
 
     // Quote the argument if it has a space in it
-    if (strpbrk(commandLine[i], "\t\v ") != SUBPROCESS_NULL ||
-        commandLine[i][0] == SUBPROCESS_NULL)
+    need_quoting = strpbrk(commandLine[i], "\t\v ") != SUBPROCESS_NULL ||
+                   commandLine[i][0] == SUBPROCESS_NULL;
+    if (need_quoting)
       len += 2;
 
+    bs_run = 0;
     for (j = 0; '\0' != commandLine[i][j]; j++) {
-      switch (commandLine[i][j]) {
-      default:
-        break;
-      case '\\':
-        if (commandLine[i][j + 1] == '"') {
-          len++;
-        }
-
-        break;
-      case '"':
-        len++;
-        break;
-      }
       len++;
+
+      if ('\\' == commandLine[i][j]) {
+        bs_run++;
+      } else {
+        if ('"' == commandLine[i][j]) {
+          // Duplicate the preceding run and escape the quote.
+          len += bs_run + 1;
+        }
+        bs_run = 0;
+      }
+    }
+
+    if (need_quoting) {
+      // Duplicate trailing slashes before the generated closing quote.
+      len += bs_run;
     }
   }
 
@@ -949,22 +969,29 @@ int subprocess_create_ex(const char *const commandLine[], int options,
       commandLineCombined[len++] = '"';
     }
 
-    for (j = 0; '\0' != commandLine[i][j]; j++) {
-      switch (commandLine[i][j]) {
-      default:
-        break;
-      case '\\':
-        if (commandLine[i][j + 1] == '"') {
-          commandLineCombined[len++] = '\\';
-        }
-
-        break;
-      case '"':
-        commandLineCombined[len++] = '\\';
-        break;
+    for (j = 0; '\0' != commandLine[i][j];) {
+      bs_run = 0;
+      while ('\\' == commandLine[i][j]) {
+        bs_run++;
+        j++;
       }
 
-      commandLineCombined[len++] = commandLine[i][j];
+      if ('"' == commandLine[i][j]) {
+        // 2n + 1 slashes preserve n slashes and escape the quote.
+        bs_run = (bs_run * 2) + 1;
+      } else if ('\0' == commandLine[i][j] && need_quoting) {
+        // 2n slashes preserve n slashes before the closing quote.
+        bs_run *= 2;
+      }
+
+      while (bs_run > 0) {
+        commandLineCombined[len++] = '\\';
+        bs_run--;
+      }
+
+      if ('\0' != commandLine[i][j]) {
+        commandLineCombined[len++] = commandLine[i][j++];
+      }
     }
     if (need_quoting) {
       commandLineCombined[len++] = '"';
@@ -1205,8 +1232,10 @@ cleanup:
 
   // Set working directory
   if (process_cwd) {
-#if defined(__APPLE__) && MAC_OS_X_VERSION_MIN_REQUIRED >= 260000
+#if defined(__NetBSD__) || (defined(__APPLE__) && MAC_OS_X_VERSION_MIN_REQUIRED >= 260000)
     posix_error = posix_spawn_file_actions_addchdir(&actions, process_cwd);
+#elif !SUBPROCESS_HAVE_CWD
+    posix_error = ENOSYS;
 #else
 #if defined(__APPLE__) && defined(__clang__)
 #pragma clang diagnostic push