opencl: add bin kernels kernel_gemm_moe_q4_0_q8_1_dp4a_bin, kernel_gemm_moe_mxfp4_q8_1_dp4a_bin (#27768)

* vulkan: add LIGHTNING_INDEXER op

* vulkan: updated lightning_indexer.comp and ggml-vulkan.cpp with 128-lane dot-product reduction moved from a shared-memory tree to subgroupAdd.

* vulkan: cleanup; Skip bounds checks

* vulkan: cleanup FA_K_ONLY

* Revert "vulkan: cleanup FA_K_ONLY"

This reverts commit fdcbdd9151.

* vulkan: restore interleaved K/V buffer ordering

* vulkan: Remove FA_K_ONLY

* vulkan: Revert flash_attn_dequant

* vulkan: Revert tests in backend-ops.cpp

common/arg.cpp

@@ -2644,6 +2644,27 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
             params.mtmd_batch_max_tokens = value;
         }
     ).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_MTMD_BATCH_MAX_TOKENS"));
+    add_opt(common_arg(
+        {"--video-fps"}, "N",
+        string_format("target video frame rate (default: %.1f)", params.video_fps),
+        [](common_params & params, const std::string & value) {
+            params.video_fps = std::stof(value);
+        }
+    ).set_examples(mmproj_examples).set_env("LLAMA_ARG_VIDEO_FPS"));
+    add_opt(common_arg(
+        {"--video-timestamp-interval"}, "N",
+        string_format("interval in milliseconds between text timestamps (default: %" PRId64 ")", params.video_timestamp_interval_ms),
+        [](common_params & params, int value) {
+            params.video_timestamp_interval_ms = value;
+        }
+    ).set_examples(mmproj_examples).set_env("LLAMA_ARG_VIDEO_TIMESTAMP_INTERVAL"));
+    add_opt(common_arg(
+        {"--video-ffmpeg-dir"}, "DIR",
+        "path to the directory containing ffmpeg and ffprobe (default: search in PATH)",
+        [](common_params & params, const std::string & value) {
+            params.video_ffmpeg_bin_dir = value;
+        }
+    ).set_examples(mmproj_examples).set_env("LLAMA_ARG_VIDEO_FFMPEG_DIR"));
     if (params.is_gen_docs || llama_supports_rpc()) {
         add_opt(common_arg(
             {"--rpc"}, "SERVERS",
@@ -2699,6 +2720,19 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
             else { throw std::invalid_argument("invalid value"); }
         }
     ).set_env("LLAMA_ARG_LOAD_MODE"));
+    add_opt(common_arg(
+        {"--tensor-read-lazy"}, "MODE",
+        "on-demand reading of certain tensors, for example per-layer embeddings (default: auto)\n"
+        "- on: read the rows of such tensors from disk on demand instead of keeping them resident (requires mmap)\n"
+        "- auto: on, but only for tensors larger than 4 GiB\n"
+        "- off: always keep them resident",
+        [](common_params & params, const std::string & value) {
+            /**/ if (value == "on")   { params.tensor_read_lazy = LLAMA_TENSOR_READ_LAZY_ON;   }
+            else if (value == "auto") { params.tensor_read_lazy = LLAMA_TENSOR_READ_LAZY_AUTO; }
+            else if (value == "off")  { params.tensor_read_lazy = LLAMA_TENSOR_READ_LAZY_OFF;  }
+            else { throw std::invalid_argument("invalid value"); }
+        }
+    ).set_env("LLAMA_ARG_TENSOR_READ_LAZY"));
     add_opt(common_arg(
         {"--numa"}, "TYPE",
         "attempt optimizations that help on some NUMA systems\n"
@@ -2750,14 +2784,20 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
             if (value < 0) {
                 throw std::invalid_argument("invalid value");
             }
-            for (int i = 0; i < value; ++i) {
-                // keep strings alive and avoid leaking memory by storing them in a static vector
-                static std::list<std::string> buft_overrides;
-                buft_overrides.push_back(llm_ffn_exps_block_regex(i));
-                params.tensor_buft_overrides.push_back({buft_overrides.back().c_str(), ggml_backend_cpu_buffer_type()});
-            }
+            llm_add_n_cpu_ffn_overrides(value, LLM_FFN_EXPS_REGEX, params.tensor_buft_overrides);
         }
     ).set_env("LLAMA_ARG_N_CPU_MOE"));
+    add_opt(common_arg(
+        {"-ncffn", "--n-cpu-ffn"}, "N",
+        "keep the dense FFN weights of the first N layers in the CPU\n"
+        "(dense models; for MoE expert weights use --n-cpu-moe)",
+        [](common_params & params, int value) {
+            if (value < 0) {
+                throw std::invalid_argument("invalid value");
+            }
+            llm_add_n_cpu_ffn_overrides(value, LLM_FFN_DENSE_REGEX, params.tensor_buft_overrides);
+        }
+    ).set_env("LLAMA_ARG_N_CPU_FFN"));
     GGML_ASSERT(params.n_gpu_layers < 0); // string_format would need to be extended for a default >= 0
     add_opt(common_arg(
         {"-ngl", "--gpu-layers", "--n-gpu-layers"}, "N",
@@ -4084,11 +4124,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
             if (value < 0) {
                 throw std::invalid_argument("invalid value");
             }
-            for (int i = 0; i < value; ++i) {
-                static std::list<std::string> buft_overrides_draft;
-                buft_overrides_draft.push_back(llm_ffn_exps_block_regex(i));
-                params.speculative.draft.tensor_buft_overrides.push_back({buft_overrides_draft.back().c_str(), ggml_backend_cpu_buffer_type()});
-            }
+            llm_add_n_cpu_ffn_overrides(value, LLM_FFN_EXPS_REGEX, params.speculative.draft.tensor_buft_overrides);
         }
     ).set_spec().set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI}).set_env("LLAMA_ARG_SPEC_DRAFT_N_CPU_MOE"));
 
@@ -4109,6 +4145,38 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
             params.speculative.draft.n_min = value;
         }
     ).set_spec().set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_LOOKUP, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI}).set_env("LLAMA_ARG_SPEC_DRAFT_N_MIN"));
+    add_opt(common_arg(
+        {"--spec-synth-len"}, "L",
+        "target mean synthetic acceptance length, including the target token (benchmarking only)",
+        [](common_params & params, const std::string & value) {
+            const std::string text = string_strip(value);
+            size_t pos = 0;
+            const double length = std::stod(text, &pos);
+            if (pos != text.size() || length == -1.0) {
+                throw std::invalid_argument("invalid value");
+            }
+            params.speculative.synth_len = length;
+        }
+    ).set_spec().set_examples({LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI}).set_env("LLAMA_ARG_SPEC_SYNTH_LEN"));
+    add_opt(common_arg(
+        {"--spec-synth-rates"}, "P0,P1,...",
+        "comma-separated unconditional per-position synthetic acceptance probabilities (benchmarking only)",
+        [](common_params & params, const std::string & value) {
+            const auto values = string_split<std::string>(value, ',');
+            std::vector<double> rates;
+            rates.reserve(values.size());
+            for (const auto & raw : values) {
+                const std::string text = string_strip(raw);
+                size_t pos = 0;
+                const double rate = std::stod(text, &pos);
+                if (pos != text.size()) {
+                    throw std::invalid_argument("invalid value");
+                }
+                rates.push_back(rate);
+            }
+            params.speculative.synth_rates = std::move(rates);
+        }
+    ).set_spec().set_examples({LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI}).set_env("LLAMA_ARG_SPEC_SYNTH_RATES"));
 
     add_opt(common_arg(
         {"--spec-draft-p-split", "--draft-p-split"}, "P",

common/common.cpp

@@ -1688,6 +1688,7 @@ struct llama_model_params common_model_params_to_llama(common_params & params) {
     mparams.main_gpu        = params.main_gpu;
     mparams.split_mode      = params.split_mode;
     mparams.load_mode       = params.load_mode;
+    mparams.tensor_read_lazy = params.tensor_read_lazy;
     mparams.tensor_split    = params.tensor_split;
     mparams.check_tensors   = params.check_tensors;
     mparams.use_extra_bufts = !params.no_extra_bufts;

common/common.h

@@ -8,6 +8,7 @@
 #include "ggml.h"
 #include "llama.h"
 
+#include <list>
 #include <set>
 #include <sstream>
 #include <string>
@@ -369,6 +370,9 @@ struct common_params_speculative_ngram_cache {
 struct common_params_speculative {
     std::vector<enum common_speculative_type> types = { COMMON_SPECULATIVE_TYPE_NONE };
 
+    double synth_len = -1.0;
+    std::vector<double> synth_rates;
+
     // used by Simple, MTP, Eagle3, etc. - all methods that require some kind of draft model
     common_params_speculative_draft draft;
 
@@ -383,6 +387,10 @@ struct common_params_speculative {
         return !draft.mparams.empty();
     }
 
+    bool has_synth() const {
+        return synth_len != -1.0 || !synth_rates.empty();
+    }
+
     uint32_t need_n_rs_seq() const {
         bool needs_rs_seq = std::any_of(types.begin(), types.end(), [&](auto t) {
             return t == COMMON_SPECULATIVE_TYPE_DRAFT_MTP || t == COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3 || t == COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH || t == COMMON_SPECULATIVE_TYPE_DRAFT_DSPARK;
@@ -475,6 +483,8 @@ struct common_params {
     enum llama_split_mode split_mode = LLAMA_SPLIT_MODE_LAYER; // how to split the model across GPUs
     enum llama_load_mode  load_mode  = LLAMA_LOAD_MODE_AUTO; // how to load the model
 
+    enum llama_tensor_read_lazy tensor_read_lazy = LLAMA_TENSOR_READ_LAZY_AUTO; // on-demand reading of tensors marked by the arch
+
     common_cpu_params cpuparams;
     common_cpu_params cpuparams_batch;
 
@@ -589,6 +599,11 @@ struct common_params {
     int image_max_tokens = -1;
     int mtmd_batch_max_tokens = 1024;
 
+    // for video input
+    float       video_fps                   = 4.0f;
+    int64_t     video_timestamp_interval_ms = 5000;
+    std::string video_ffmpeg_bin_dir        = "";
+
     // finetune
     struct lr_opt lr;
     enum ggml_opt_optimizer_type optimizer = GGML_OPT_OPTIMIZER_TYPE_ADAMW;
@@ -1108,19 +1123,30 @@ const char * const LLM_KV_SPLIT_TENSORS_COUNT = "split.tensors.count";
 }
 
 //
-// MoE utils
+// FFN offload utils
 //
 
 const char * const LLM_FFN_EXPS_REGEX = "\\.ffn_(up|down|gate|gate_up)_(ch|)exps";
 
-inline std::string llm_ffn_exps_block_regex(int idx) {
-    return string_format("blk\\.%d%s", idx, LLM_FFN_EXPS_REGEX);
+const char * const LLM_FFN_DENSE_REGEX = "\\.ffn_(up|down|gate)\\.";
+
+inline std::string llm_ffn_block_regex(int idx, const char * ffn_regex) {
+    return string_format("blk\\.%d%s", idx, ffn_regex);
 }
 
 inline llama_model_tensor_buft_override llm_ffn_exps_cpu_override() {
     return { LLM_FFN_EXPS_REGEX, ggml_backend_cpu_buffer_type() };
 }
 
+inline void llm_add_n_cpu_ffn_overrides(int n, const char * ffn_regex, std::vector<llama_model_tensor_buft_override> & overrides) {
+    // keep strings alive and avoid leaking memory by storing them in a static list
+    static std::list<std::string> buft_override_strings;
+    for (int i = 0; i < n; ++i) {
+        buft_override_strings.push_back(llm_ffn_block_regex(i, ffn_regex));
+        overrides.push_back({buft_override_strings.back().c_str(), ggml_backend_cpu_buffer_type()});
+    }
+}
+
 //
 // training utils
 //

common/speculative.cpp

@@ -14,6 +14,7 @@
 
 #include <algorithm>
 #include <cassert>
+#include <cmath>
 #include <cstring>
 #include <iomanip>
 #include <map>
@@ -138,6 +139,7 @@ struct common_speculative_impl {
     const common_speculative_type type;
 
     uint32_t n_seq;
+    int32_t n_max; // maximum draft length after implementation-specific limits
 
     size_t n_call_begin  = 0; // number of times this implementation was called for refresh.
     size_t n_call_draft  = 0; // number of times this implementation was called for generation.
@@ -157,7 +159,7 @@ struct common_speculative_impl {
     int64_t t_draft_us  = 0; // total time spent in generating drafts in this implementation in microseconds.
     int64_t t_accept_us = 0; // total time spent in accumulation of this implementation in microseconds.
 
-    common_speculative_impl(common_speculative_type type, uint32_t n_seq) : type(type), n_seq(n_seq) {}
+    common_speculative_impl(common_speculative_type type, uint32_t n_seq, int32_t n_max) : type(type), n_seq(n_seq), n_max(n_max) {}
 
     virtual ~common_speculative_impl() = default;
 
@@ -182,7 +184,7 @@ struct common_speculative_impl_draft_simple : public common_speculative_impl {
     std::vector<common_sampler_ptr> smpls;
 
     common_speculative_impl_draft_simple(const common_params_speculative & params, uint32_t n_seq)
-        : common_speculative_impl(COMMON_SPECULATIVE_TYPE_DRAFT_SIMPLE, n_seq)
+        : common_speculative_impl(COMMON_SPECULATIVE_TYPE_DRAFT_SIMPLE, n_seq, params.draft.n_max)
         , params(params.draft)
     {
         auto * ctx_dft = this->params.ctx_dft;
@@ -452,7 +454,7 @@ struct common_speculative_impl_draft_eagle3 : public common_speculative_impl {
     std::vector<float> g_embd_buf;
 
     common_speculative_impl_draft_eagle3(const common_params_speculative & params, uint32_t n_seq)
-        : common_speculative_impl(COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3, n_seq)
+        : common_speculative_impl(COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3, n_seq, params.draft.n_max)
         , params(params.draft)
     {
         SPC_TRC("%s", "adding speculative implementation 'draft-eagle3'\n");
@@ -937,7 +939,7 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl {
 
     common_speculative_impl_draft_dflash(const common_params_speculative & params, uint32_t n_seq,
             common_speculative_type type = COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH)
-        : common_speculative_impl(type, n_seq)
+        : common_speculative_impl(type, n_seq, params.draft.n_max)
         , params(params.draft)
         , is_dspark(type == COMMON_SPECULATIVE_TYPE_DRAFT_DSPARK)
     {
@@ -983,6 +985,7 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl {
             this->params.n_max = std::min(this->params.n_max, n_draft_max);
             this->params.n_min = std::min(this->params.n_min, n_draft_max);
         }
+        this->n_max = this->params.n_max;
 
         batch        = llama_batch_init(llama_n_batch(ctx_dft), 0,          n_seq);
         batch_inject = llama_batch_init(llama_n_batch(ctx_dft), n_embd_dec, n_seq);
@@ -1315,7 +1318,7 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl {
     std::vector<std::vector<float>> chain_h;
 
     common_speculative_impl_draft_mtp(const common_params_speculative & params, uint32_t n_seq)
-        : common_speculative_impl(COMMON_SPECULATIVE_TYPE_DRAFT_MTP, n_seq)
+        : common_speculative_impl(COMMON_SPECULATIVE_TYPE_DRAFT_MTP, n_seq, params.draft.n_max)
         , params(params.draft)
     {
         auto * ctx_tgt = this->params.ctx_tgt;
@@ -1382,6 +1385,7 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl {
                 c.reserve((size_t) (this->params.n_max + 1) * n_embd);
             }
         }
+        this->n_max = this->params.n_max;
 
         pending_h.assign(n_seq, std::vector<float>(n_embd, 0.0f));
 
@@ -1726,7 +1730,7 @@ struct common_speculative_impl_ngram_simple : public common_speculative_impl {
     common_speculative_impl_ngram_simple(
             const common_params_speculative & params, uint32_t n_seq,
             common_ngram_simple_config config)
-        : common_speculative_impl(COMMON_SPECULATIVE_TYPE_NGRAM_SIMPLE, n_seq)
+        : common_speculative_impl(COMMON_SPECULATIVE_TYPE_NGRAM_SIMPLE, n_seq, params.ngram_simple.size_m)
         , params(params.ngram_simple)
         , config(config)
     {
@@ -1770,7 +1774,7 @@ struct common_speculative_impl_ngram_map_k : public common_speculative_impl {
             const common_ngram_map & config,
             uint32_t n_seq)
         : common_speculative_impl(config.key_only ? COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K
-            : COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K4V, n_seq)
+            : COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K4V, n_seq, config.size_value)
     {
         for (uint32_t i = 0; i < n_seq; i++) {
             this->config.push_back(config);
@@ -1841,7 +1845,7 @@ struct common_speculative_impl_ngram_mod : public common_speculative_impl {
     common_speculative_impl_ngram_mod(
             const common_params_speculative & params,
             uint32_t n_seq)
-        : common_speculative_impl(COMMON_SPECULATIVE_TYPE_NGRAM_MOD, n_seq)
+        : common_speculative_impl(COMMON_SPECULATIVE_TYPE_NGRAM_MOD, n_seq, params.ngram_mod.n_max)
         , params(params.ngram_mod)
         , mod(params.ngram_mod.n_match, 4*1024*1024)
         , verbose(std::getenv("LLAMA_TRACE") != nullptr) {
@@ -2017,7 +2021,7 @@ struct common_speculative_impl_ngram_cache : public common_speculative_impl {
             const std::string & path_dynamic,
             bool save_dynamic,
             bool save_static)
-        : common_speculative_impl(COMMON_SPECULATIVE_TYPE_NGRAM_CACHE, n_seq)
+        : common_speculative_impl(COMMON_SPECULATIVE_TYPE_NGRAM_CACHE, n_seq, n_draft)
         , params(params.ngram_cache)
         , n_draft(n_draft)
         , save_dynamic(save_dynamic)
@@ -2138,6 +2142,8 @@ struct common_speculative {
 
     // which implementaion was used for a given seq_id
     std::vector<common_speculative_impl *> impl_last;
+
+    std::vector<double> synth_probs;
 };
 
 static common_ngram_map get_common_ngram_map(
@@ -2316,6 +2322,101 @@ int32_t common_speculative_n_max(const common_params_speculative * spec) {
     return n_max;
 }
 
+int32_t common_speculative_n_max(const common_speculative * spec) {
+    int32_t n_max = 0;
+
+    if (spec == nullptr) {
+        return n_max;
+    }
+
+    for (const auto & impl : spec->impls) {
+        n_max = std::max(n_max, std::max(0, impl->n_max));
+    }
+
+    return n_max;
+}
+
+std::vector<double> common_speculative_synth_rates_resolve(const common_params_speculative * spec, int32_t n_max) {
+    const bool has_length = spec->synth_len != -1.0;
+    const bool has_rates  = !spec->synth_rates.empty();
+
+    if (!has_length && !has_rates) {
+        return {};
+    }
+    if (has_length && has_rates) {
+        throw std::invalid_argument("synthetic acceptance length and rates are mutually exclusive");
+    }
+
+    if (n_max <= 0) {
+        throw std::invalid_argument("synthetic acceptance requires at least one speculative token");
+    }
+
+    if (has_rates) {
+        const auto & rates = spec->synth_rates;
+        if (rates.size() != (size_t) n_max) {
+            throw std::invalid_argument(string_format(
+                    "synthetic acceptance rates must contain %d values, got %zu", n_max, rates.size()));
+        }
+
+        for (size_t i = 0; i < rates.size(); ++i) {
+            if (!std::isfinite(rates[i]) || rates[i] < 0.0 || rates[i] > 1.0) {
+                throw std::invalid_argument("synthetic acceptance rates must be finite and within [0, 1]");
+            }
+            if (i > 0 && rates[i] > rates[i - 1]) {
+                throw std::invalid_argument("synthetic acceptance rates must be monotonically non-increasing");
+            }
+        }
+
+        return rates;
+    }
+
+    const double length = spec->synth_len;
+    const double length_max = (double) n_max + 1.0;
+    if (!std::isfinite(length) || length < 1.0 || length > length_max) {
+        throw std::invalid_argument(string_format(
+                "synthetic acceptance length must be finite and within [1, %.0f]", length_max));
+    }
+
+    double p = 0.0;
+    if (length == length_max) {
+        p = 1.0;
+    } else if (length > 1.0) {
+        double p_min = 0.0;
+        double p_max = 1.0;
+        for (int i = 0; i < 32; ++i) {
+            const double p_mid = 0.5 * (p_min + p_max);
+            double sum = 0.0;
+            double term = p_mid;
+            for (int32_t j = 0; j < n_max; ++j) {
+                sum += term;
+                term *= p_mid;
+            }
+
+            if (sum < length - 1.0) {
+                p_min = p_mid;
+            } else {
+                p_max = p_mid;
+            }
+        }
+        p = 0.5 * (p_min + p_max);
+    }
+
+    std::vector<double> rates;
+    rates.reserve(n_max);
+    double rate = p;
+    for (int32_t i = 0; i < n_max; ++i) {
+        rates.push_back(rate);
+        rate *= p;
+    }
+
+    return rates;
+}
+
+const std::vector<double> & common_speculative_get_synth_probs(const common_speculative * spec) {
+    GGML_ASSERT(spec);
+    return spec->synth_probs;
+}
+
 common_params common_base_params_to_speculative(const common_params & params) {
     const bool has_draft = params.speculative.has_dft();
 
@@ -2568,13 +2669,39 @@ common_speculative * common_speculative_init(common_params_speculative & params,
         return nullptr;
     }
 
-    auto * result = new common_speculative {
-        /* .dparams   = */ common_speculative_draft_params_vec(n_seq),
-        /* .impls     = */ std::move(impls),
-        /* .impl_last = */ std::vector<common_speculative_impl *>(n_seq, nullptr)
-    };
+    common_speculative_ptr result(new common_speculative {
+        /* .dparams     = */ common_speculative_draft_params_vec(n_seq),
+        /* .impls       = */ std::move(impls),
+        /* .impl_last   = */ std::vector<common_speculative_impl *>(n_seq, nullptr),
+        /* .synth_probs = */ {},
+    });
 
-    return result;
+    const int32_t n_max_configured = common_speculative_n_max(&params);
+    const int32_t n_max_effective  = common_speculative_n_max(result.get());
+    const auto rates = common_speculative_synth_rates_resolve(&params, n_max_effective);
+
+    std::vector<std::string> rates_str;
+    rates_str.reserve(rates.size());
+    result->synth_probs.reserve(rates.size());
+    double rate_prev = 1.0;
+    double acceptance_length = 1.0;
+    for (const double rate : rates) {
+        result->synth_probs.push_back(rate_prev > 0.0 ? rate / rate_prev : 0.0);
+        rates_str.push_back(string_format("%.6g", rate));
+        rate_prev = rate;
+        acceptance_length += rate;
+    }
+    if (!result->synth_probs.empty()) {
+        SPC_WRN("%s", "synthetic speculative acceptance is enabled for benchmarking; generated output is not valid\n");
+        if (n_max_effective != n_max_configured) {
+            SPC_WRN("synthetic acceptance draft limit was reduced from %d to %d by the initialized speculative implementations\n",
+                    n_max_configured, n_max_effective);
+        }
+        SPC_INF("synthetic acceptance: n_max = %zu, mean length = %.6f, rates = [%s]\n",
+                rates.size(), acceptance_length, string_join(rates_str, ", ").c_str());
+    }
+
+    return result.release();
 }
 
 void common_speculative_free(common_speculative * spec) {

common/speculative.h

@@ -26,6 +26,15 @@ std::string common_speculative_type_to_str(enum common_speculative_type type);
 // return the max number of draft tokens based on the speculative parameters
 int32_t common_speculative_n_max(const common_params_speculative * spec);
 
+// return the max number of draft tokens from the initialized implementations
+int32_t common_speculative_n_max(const common_speculative * spec);
+
+// validate and resolve the unconditional synthetic acceptance rates
+std::vector<double> common_speculative_synth_rates_resolve(const common_params_speculative * spec, int32_t n_max);
+
+// return the conditional synthetic acceptance probabilities
+const std::vector<double> & common_speculative_get_synth_probs(const common_speculative * spec);
+
 common_params common_base_params_to_speculative(const common_params & params);
 
 struct common_speculative_output_limits {

conversion/nemotron.py

@@ -302,6 +302,10 @@ class NemotronHModel(GraniteHybridModel):
             )
             if not keep:
                 return None
+        # PEFT names adapter tensors using model.layers.*, while Nemotron-H checkpoints
+        # and the GGUF tensor map use backbone.layers.*
+        if name.startswith("model.layers.") and ".mixer." in name:
+            name = name.replace("model.layers.", "backbone.layers.", 1)
         return super().filter_tensors((name, gen))
 
     def prepare_metadata(self, vocab_only: bool):

docs/speculative.md

@@ -212,6 +212,15 @@ Use `--backend-sampling` to run supported target-model samplers on the model bac
 
 Unsupported samplers and device layouts fall back to CPU sampling. Tensor split mode does not support backend sampling. A fixed seed produces repeatable random draws, but stochastic CPU and backend sampling can still select different tokens because floating-point operations can differ between implementations and devices. Use greedy sampling when exact output matching is required.
 
+### Synthetic Acceptance
+
+`llama-server` and `llama-cli` can replace normal speculative verification with synthetic decisions for benchmarking. The generated output is not valid model output because accepted draft tokens do not have to match the target model.
+
+Use exactly one of these options:
+
+- `--spec-synth-rates P0,P1,...` sets unconditional per-position acceptance probabilities. Entry `i` is the probability that the first `i+1` draft tokens are all accepted. The number of entries must match the effective maximum draft length. Values must be finite, within `[0, 1]`, and monotonically non-increasing.
+- `--spec-synth-len L` sets the target mean acceptance length, including the target token. For `K` maximum draft tokens, `L` must be within `[1, K+1]`. The server finds a constant conditional probability `p` such that `p + p^2 + ... + p^K = L - 1`, then uses unconditional rates `[p, p^2, ..., p^K]`.
+
 ### General Speculative Parameters
 
 ```

ggml/src/ggml-metal/ggml-metal-context.m

@@ -84,106 +84,108 @@ struct ggml_metal {
 ggml_metal_t ggml_metal_init(ggml_metal_device_t dev) {
     GGML_LOG_INFO("%s: allocating\n", __func__);
 
+    @autoreleasepool {
 #if TARGET_OS_OSX && !GGML_METAL_NDEBUG
-    // Show all the Metal device instances in the system
-    NSArray * devices = MTLCopyAllDevices();
-    for (id<MTLDevice> device in devices) {
-        GGML_LOG_INFO("%s: found device: %s\n", __func__, [[device name] UTF8String]);
-    }
-    [devices release]; // since it was created by a *Copy* C method
+        // Show all the Metal device instances in the system
+        NSArray * devices = MTLCopyAllDevices();
+        for (id<MTLDevice> device in devices) {
+            GGML_LOG_INFO("%s: found device: %s\n", __func__, [[device name] UTF8String]);
+        }
+        [devices release]; // since it was created by a *Copy* C method
 #endif
 
-    // init context
-    ggml_metal_t res = calloc(1, sizeof(struct ggml_metal));
+        // init context
+        ggml_metal_t res = calloc(1, sizeof(struct ggml_metal));
 
-    id<MTLDevice> device = ggml_metal_device_get_obj(dev);
+        id<MTLDevice> device = ggml_metal_device_get_obj(dev);
 
-    GGML_LOG_INFO("%s: picking default device: %s\n", __func__, [[device name] UTF8String]);
-
-    // TODO: would it be better to have one queue for the backend and one queue for the device?
-    //       the graph encoders and async ops would use the backend queue while the sync ops would use the device queue?
-    //res->queue = [device newCommandQueue]; [TAG_QUEUE_PER_BACKEND]
-    id<MTLCommandQueue> queue = ggml_metal_device_get_queue(dev);
-    if (queue == nil) {
-        GGML_LOG_ERROR("%s: error: failed to create command queue\n", __func__);
-        return NULL;
-    }
-
-    res->dev = dev;
-    res->lib = ggml_metal_device_get_library(dev);
-    if (res->lib == NULL) {
-        GGML_LOG_WARN("%s: the device does not have a precompiled Metal library - this is unexpected\n", __func__);
-        GGML_LOG_WARN("%s: will try to compile it on the fly\n", __func__);
-
-        res->lib = ggml_metal_library_init(dev);
-        if (res->lib == NULL) {
-            GGML_LOG_ERROR("%s: error: failed to initialize the Metal library\n", __func__);
-
-            free(res);
+        GGML_LOG_INFO("%s: picking default device: %s\n", __func__, [[device name] UTF8String]);
 
+        // TODO: would it be better to have one queue for the backend and one queue for the device?
+        //       the graph encoders and async ops would use the backend queue while the sync ops would use the device queue?
+        //res->queue = [device newCommandQueue]; [TAG_QUEUE_PER_BACKEND]
+        id<MTLCommandQueue> queue = ggml_metal_device_get_queue(dev);
+        if (queue == nil) {
+            GGML_LOG_ERROR("%s: error: failed to create command queue\n", __func__);
             return NULL;
         }
-    }
 
-    res->ev_cpy = ggml_metal_device_event_init(dev);
+        res->dev = dev;
+        res->lib = ggml_metal_device_get_library(dev);
+        if (res->lib == NULL) {
+            GGML_LOG_WARN("%s: the device does not have a precompiled Metal library - this is unexpected\n", __func__);
+            GGML_LOG_WARN("%s: will try to compile it on the fly\n", __func__);
 
-    const struct ggml_metal_device_props * props_dev = ggml_metal_device_get_props(dev);
+            res->lib = ggml_metal_library_init(dev);
+            if (res->lib == NULL) {
+                GGML_LOG_ERROR("%s: error: failed to initialize the Metal library\n", __func__);
 
-    snprintf(res->name, sizeof(res->name), "%s", props_dev->name);
+                free(res);
 
-    res->d_queue = dispatch_queue_create("ggml-metal", DISPATCH_QUEUE_CONCURRENT);
-
-    res->use_fusion      = getenv("GGML_METAL_FUSION_DISABLE") == nil;
-    res->use_concurrency = getenv("GGML_METAL_CONCURRENCY_DISABLE") == nil;
-
-    {
-        const char * val = getenv("GGML_METAL_GRAPH_DEBUG");
-        res->debug_graph = val ? atoi(val) : 0;
-    }
-
-    {
-        const char * val = getenv("GGML_METAL_FUSION_DEBUG");
-        res->debug_fusion = val ? atoi(val) : 0;
-    }
-
-    res->use_graph_optimize = true;
-
-    if (getenv("GGML_METAL_GRAPH_OPTIMIZE_DISABLE") != NULL) {
-        res->use_graph_optimize = false;
-    }
-
-    memset(res->fuse_cnt, 0, sizeof(res->fuse_cnt));
-
-    GGML_LOG_INFO("%s: use fusion         = %s\n", __func__, res->use_fusion         ? "true" : "false");
-    GGML_LOG_INFO("%s: use concurrency    = %s\n", __func__, res->use_concurrency    ? "true" : "false");
-    GGML_LOG_INFO("%s: use graph optimize = %s\n", __func__, res->use_graph_optimize ? "true" : "false");
-
-    res->capture_compute = 0;
-    res->capture_started = false;
-    res->capture_scope = nil;
-
-    {
-        const char * val = getenv("GGML_METAL_CAPTURE_COMPUTE");
-        if (val) {
-            res->capture_compute = atoi(val);
+                return NULL;
+            }
         }
+
+        res->ev_cpy = ggml_metal_device_event_init(dev);
+
+        const struct ggml_metal_device_props * props_dev = ggml_metal_device_get_props(dev);
+
+        snprintf(res->name, sizeof(res->name), "%s", props_dev->name);
+
+        res->d_queue = dispatch_queue_create("ggml-metal", DISPATCH_QUEUE_CONCURRENT);
+
+        res->use_fusion      = getenv("GGML_METAL_FUSION_DISABLE") == nil;
+        res->use_concurrency = getenv("GGML_METAL_CONCURRENCY_DISABLE") == nil;
+
+        {
+            const char * val = getenv("GGML_METAL_GRAPH_DEBUG");
+            res->debug_graph = val ? atoi(val) : 0;
+        }
+
+        {
+            const char * val = getenv("GGML_METAL_FUSION_DEBUG");
+            res->debug_fusion = val ? atoi(val) : 0;
+        }
+
+        res->use_graph_optimize = true;
+
+        if (getenv("GGML_METAL_GRAPH_OPTIMIZE_DISABLE") != NULL) {
+            res->use_graph_optimize = false;
+        }
+
+        memset(res->fuse_cnt, 0, sizeof(res->fuse_cnt));
+
+        GGML_LOG_INFO("%s: use fusion         = %s\n", __func__, res->use_fusion         ? "true" : "false");
+        GGML_LOG_INFO("%s: use concurrency    = %s\n", __func__, res->use_concurrency    ? "true" : "false");
+        GGML_LOG_INFO("%s: use graph optimize = %s\n", __func__, res->use_graph_optimize ? "true" : "false");
+
+        res->capture_compute = 0;
+        res->capture_started = false;
+        res->capture_scope = nil;
+
+        {
+            const char * val = getenv("GGML_METAL_CAPTURE_COMPUTE");
+            if (val) {
+                res->capture_compute = atoi(val);
+            }
+        }
+
+        res->has_error = false;
+
+        res->gf = nil;
+        res->encode_async = nil;
+        for (int i = 0; i < GGML_METAL_MAX_COMMAND_BUFFERS; ++i) {
+            res->cmd_bufs[i].obj = nil;
+        }
+
+        res->cmd_bufs_ext = [[NSMutableArray alloc] init];
+
+        res->cmd_buf_last = nil;
+
+        res->pipelines_ext = ggml_metal_pipelines_init();
+
+        return res;
     }
-
-    res->has_error = false;
-
-    res->gf = nil;
-    res->encode_async = nil;
-    for (int i = 0; i < GGML_METAL_MAX_COMMAND_BUFFERS; ++i) {
-        res->cmd_bufs[i].obj = nil;
-    }
-
-    res->cmd_bufs_ext = [[NSMutableArray alloc] init];
-
-    res->cmd_buf_last = nil;
-
-    res->pipelines_ext = ggml_metal_pipelines_init();
-
-    return res;
 }
 
 void ggml_metal_free(ggml_metal_t ctx) {

ggml/src/ggml-metal/ggml-metal-device.m

@@ -778,7 +778,9 @@ void ggml_metal_encoder_free(ggml_metal_encoder_t encoder) {
 }
 
 void ggml_metal_encoder_debug_group_push(ggml_metal_encoder_t encoder, const char * name) {
-    [encoder->obj pushDebugGroup:[NSString stringWithCString:name encoding:NSUTF8StringEncoding]];
+    @autoreleasepool {
+        [encoder->obj pushDebugGroup:[NSString stringWithCString:name encoding:NSUTF8StringEncoding]];
+    }
 }
 
 void ggml_metal_encoder_debug_group_pop (ggml_metal_encoder_t encoder) {
@@ -1023,249 +1025,251 @@ ggml_metal_device_t ggml_metal_device_init(int device, int n_devices) {
 
     assert(dev != NULL);
 
-    if (dev->mtl_device == nil) {
-        dev->mtl_device = MTLCreateSystemDefaultDevice();
+    @autoreleasepool {
+        if (dev->mtl_device == nil) {
+            dev->mtl_device = MTLCreateSystemDefaultDevice();
 
-        if (dev->mtl_device) {
-            dev->mtl_queue = [dev->mtl_device newCommandQueue];
-            if (dev->mtl_queue == nil) {
-                GGML_LOG_ERROR("%s: error: failed to create command queue\n", __func__);
-            }
+            if (dev->mtl_device) {
+                dev->mtl_queue = [dev->mtl_device newCommandQueue];
+                if (dev->mtl_queue == nil) {
+                    GGML_LOG_ERROR("%s: error: failed to create command queue\n", __func__);
+                }
 
-            dev->addr_virt = 0x000000400ULL;
+                dev->addr_virt = 0x000000400ULL;
 
-            dev->props.device = device;
+                dev->props.device = device;
 
-            // the Metal backend uses the system default device as the single physical device;
-            // additional (virtual) devices are emulated on top of it via GGML_METAL_DEVICES
-            dev->props.device_phys = 0;
-            dev->props.device_virt = device;
+                // the Metal backend uses the system default device as the single physical device;
+                // additional (virtual) devices are emulated on top of it via GGML_METAL_DEVICES
+                dev->props.device_phys = 0;
+                dev->props.device_virt = device;
 
-            dev->props.has_simdgroup_reduction  = [dev->mtl_device supportsFamily:MTLGPUFamilyApple7];
-            dev->props.has_simdgroup_reduction |= [dev->mtl_device supportsFamily:MTLGPUFamilyMetal3_GGML];
+                dev->props.has_simdgroup_reduction  = [dev->mtl_device supportsFamily:MTLGPUFamilyApple7];
+                dev->props.has_simdgroup_reduction |= [dev->mtl_device supportsFamily:MTLGPUFamilyMetal3_GGML];
 
-            dev->props.has_simdgroup_mm = [dev->mtl_device supportsFamily:MTLGPUFamilyApple7];
-            dev->props.has_unified_memory = dev->mtl_device.hasUnifiedMemory;
+                dev->props.has_simdgroup_mm = [dev->mtl_device supportsFamily:MTLGPUFamilyApple7];
+                dev->props.has_unified_memory = dev->mtl_device.hasUnifiedMemory;
 
-            dev->props.has_bfloat  = [dev->mtl_device supportsFamily:MTLGPUFamilyMetal3_GGML];
-            dev->props.has_bfloat |= [dev->mtl_device supportsFamily:MTLGPUFamilyApple6];
-            if (getenv("GGML_METAL_BF16_DISABLE") != NULL) {
-                dev->props.has_bfloat = false;
-            }
+                dev->props.has_bfloat  = [dev->mtl_device supportsFamily:MTLGPUFamilyMetal3_GGML];
+                dev->props.has_bfloat |= [dev->mtl_device supportsFamily:MTLGPUFamilyApple6];
+                if (getenv("GGML_METAL_BF16_DISABLE") != NULL) {
+                    dev->props.has_bfloat = false;
+                }
 
-            dev->props.has_tensor = [dev->mtl_device supportsFamily:MTLGPUFamilyMetal4_GGML];
-            if (getenv("GGML_METAL_TENSOR_DISABLE") != NULL) {
-                dev->props.has_tensor = false;
-            }
-
-            // note: disable the tensor API by default for old chips because with the current implementation it is not useful
-            // - M2 Ultra:   ~5% slower
-            // - M4, M4 Max: no significant difference
-            //
-            // TODO: try to update the tensor API kernels to at least match the simdgroup performance
-            if (getenv("GGML_METAL_TENSOR_ENABLE") == NULL &&
-                ![[dev->mtl_device name] containsString:@"M5"] &&
-                ![[dev->mtl_device name] containsString:@"M6"] &&
-                ![[dev->mtl_device name] containsString:@"A19"] &&
-                ![[dev->mtl_device name] containsString:@"A20"]) {
-                GGML_LOG_INFO("%s: tensor API disabled for pre-M5 and pre-A19 devices\n", __func__);
-                dev->props.has_tensor = false;
-            }
-
-            // double-check that the tensor API compiles
-            if (dev->props.has_tensor) {
-                const char * src_tensor_f16 = "\n"
-                    "#include <metal_stdlib> \n"
-                    "#include <metal_tensor> \n"
-                    "#include <MetalPerformancePrimitives/MetalPerformancePrimitives.h> \n"
-                    " \n"
-                    "using namespace metal; \n"
-                    "using namespace mpp::tensor_ops; \n"
-                    " \n"
-                    "kernel void dummy_kernel( \n"
-                    "    tensor<device  half, dextents<int32_t, 2>> A [[buffer(0)]], \n"
-                    "    tensor<device  half, dextents<int32_t, 2>> B [[buffer(1)]], \n"
-                    "    device float * C [[buffer(2)]], \n"
-                    "    uint2 tgid [[threadgroup_position_in_grid]]) \n"
-                    "{ \n"
-                    "    auto tA = A.slice(0, (int)tgid.y); \n"
-                    "    auto tB = B.slice((int)tgid.x, 0); \n"
-                    " \n"
-                    "    matmul2d< \n"
-                    "        matmul2d_descriptor(16, 16, dynamic_extent), \n"
-                    "        execution_simdgroups<4>> mm; \n"
-                    " \n"
-                    "    auto cT = mm.get_destination_cooperative_tensor<decltype(tA), decltype(tB), float>(); \n"
-                    " \n"
-                    "    auto sA = tA.slice(0, 0); \n"
-                    "    auto sB = tB.slice(0, 0); \n"
-                    "    mm.run(sB, sA, cT); \n"
-                    " \n"
-                    "    auto tC = tensor<device float, dextents<int32_t, 2>, tensor_inline>(C, dextents<int32_t, 2>(16, 16)); \n"
-                    " \n"
-                    "    cT.store(tC); \n"
-                    "}";
-
-                GGML_LOG_INFO("%s: testing tensor API for f16 support\n", __func__);
-                ggml_metal_library_t lib = ggml_metal_library_init_from_source(dev, src_tensor_f16, false);
-                if (lib == NULL) {
-                    GGML_LOG_WARN("%s: - the tensor API is not supported in this environment - disabling\n", __func__);
+                dev->props.has_tensor = [dev->mtl_device supportsFamily:MTLGPUFamilyMetal4_GGML];
+                if (getenv("GGML_METAL_TENSOR_DISABLE") != NULL) {
                     dev->props.has_tensor = false;
-                } else {
-                    struct ggml_metal_pipeline_with_params ppl = ggml_metal_library_compile_pipeline(lib, "dummy_kernel", "dummy_kernel", nil);
-                    if (!ppl.pipeline) {
+                }
+
+                // note: disable the tensor API by default for old chips because with the current implementation it is not useful
+                // - M2 Ultra:   ~5% slower
+                // - M4, M4 Max: no significant difference
+                //
+                // TODO: try to update the tensor API kernels to at least match the simdgroup performance
+                if (getenv("GGML_METAL_TENSOR_ENABLE") == NULL &&
+                    ![[dev->mtl_device name] containsString:@"M5"] &&
+                    ![[dev->mtl_device name] containsString:@"M6"] &&
+                    ![[dev->mtl_device name] containsString:@"A19"] &&
+                    ![[dev->mtl_device name] containsString:@"A20"]) {
+                    GGML_LOG_INFO("%s: tensor API disabled for pre-M5 and pre-A19 devices\n", __func__);
+                    dev->props.has_tensor = false;
+                }
+
+                // double-check that the tensor API compiles
+                if (dev->props.has_tensor) {
+                    const char * src_tensor_f16 = "\n"
+                        "#include <metal_stdlib> \n"
+                        "#include <metal_tensor> \n"
+                        "#include <MetalPerformancePrimitives/MetalPerformancePrimitives.h> \n"
+                        " \n"
+                        "using namespace metal; \n"
+                        "using namespace mpp::tensor_ops; \n"
+                        " \n"
+                        "kernel void dummy_kernel( \n"
+                        "    tensor<device  half, dextents<int32_t, 2>> A [[buffer(0)]], \n"
+                        "    tensor<device  half, dextents<int32_t, 2>> B [[buffer(1)]], \n"
+                        "    device float * C [[buffer(2)]], \n"
+                        "    uint2 tgid [[threadgroup_position_in_grid]]) \n"
+                        "{ \n"
+                        "    auto tA = A.slice(0, (int)tgid.y); \n"
+                        "    auto tB = B.slice((int)tgid.x, 0); \n"
+                        " \n"
+                        "    matmul2d< \n"
+                        "        matmul2d_descriptor(16, 16, dynamic_extent), \n"
+                        "        execution_simdgroups<4>> mm; \n"
+                        " \n"
+                        "    auto cT = mm.get_destination_cooperative_tensor<decltype(tA), decltype(tB), float>(); \n"
+                        " \n"
+                        "    auto sA = tA.slice(0, 0); \n"
+                        "    auto sB = tB.slice(0, 0); \n"
+                        "    mm.run(sB, sA, cT); \n"
+                        " \n"
+                        "    auto tC = tensor<device float, dextents<int32_t, 2>, tensor_inline>(C, dextents<int32_t, 2>(16, 16)); \n"
+                        " \n"
+                        "    cT.store(tC); \n"
+                        "}";
+
+                    GGML_LOG_INFO("%s: testing tensor API for f16 support\n", __func__);
+                    ggml_metal_library_t lib = ggml_metal_library_init_from_source(dev, src_tensor_f16, false);
+                    if (lib == NULL) {
                         GGML_LOG_WARN("%s: - the tensor API is not supported in this environment - disabling\n", __func__);
                         dev->props.has_tensor = false;
+                    } else {
+                        struct ggml_metal_pipeline_with_params ppl = ggml_metal_library_compile_pipeline(lib, "dummy_kernel", "dummy_kernel", nil);
+                        if (!ppl.pipeline) {
+                            GGML_LOG_WARN("%s: - the tensor API is not supported in this environment - disabling\n", __func__);
+                            dev->props.has_tensor = false;
+                        }
+
+                        ggml_metal_library_free(lib);
                     }
-
-                    ggml_metal_library_free(lib);
                 }
-            }
 
-            // try to compile a dummy kernel to determine if the tensor API is supported for bfloat
-            if (dev->props.has_tensor && dev->props.has_bfloat) {
-                const char * src_tensor_bf16 = "\n"
-                    "#include <metal_stdlib> \n"
-                    "#include <metal_tensor> \n"
-                    "#include <MetalPerformancePrimitives/MetalPerformancePrimitives.h> \n"
-                    " \n"
-                    "using namespace metal; \n"
-                    "using namespace mpp::tensor_ops; \n"
-                    " \n"
-                    "kernel void dummy_kernel( \n"
-                    "    tensor<device bfloat, dextents<int32_t, 2>> A [[buffer(0)]], \n"
-                    "    tensor<device bfloat, dextents<int32_t, 2>> B [[buffer(1)]], \n"
-                    "    device float * C [[buffer(2)]], \n"
-                    "    uint2 tgid [[threadgroup_position_in_grid]]) \n"
-                    "{ \n"
-                    "    auto tA = A.slice(0, (int)tgid.y); \n"
-                    "    auto tB = B.slice((int)tgid.x, 0); \n"
-                    " \n"
-                    "    matmul2d< \n"
-                    "        matmul2d_descriptor(16, 16, dynamic_extent), \n"
-                    "        execution_simdgroups<4>> mm; \n"
-                    " \n"
-                    "    auto cT = mm.get_destination_cooperative_tensor<decltype(tA), decltype(tB), float>(); \n"
-                    " \n"
-                    "    auto sA = tA.slice(0, 0); \n"
-                    "    auto sB = tB.slice(0, 0); \n"
-                    "    mm.run(sB, sA, cT); \n"
-                    " \n"
-                    "    auto tC = tensor<device float, dextents<int32_t, 2>, tensor_inline>(C, dextents<int32_t, 2>(16, 16)); \n"
-                    " \n"
-                    "    cT.store(tC); \n"
-                    "}";
+                // try to compile a dummy kernel to determine if the tensor API is supported for bfloat
+                if (dev->props.has_tensor && dev->props.has_bfloat) {
+                    const char * src_tensor_bf16 = "\n"
+                        "#include <metal_stdlib> \n"
+                        "#include <metal_tensor> \n"
+                        "#include <MetalPerformancePrimitives/MetalPerformancePrimitives.h> \n"
+                        " \n"
+                        "using namespace metal; \n"
+                        "using namespace mpp::tensor_ops; \n"
+                        " \n"
+                        "kernel void dummy_kernel( \n"
+                        "    tensor<device bfloat, dextents<int32_t, 2>> A [[buffer(0)]], \n"
+                        "    tensor<device bfloat, dextents<int32_t, 2>> B [[buffer(1)]], \n"
+                        "    device float * C [[buffer(2)]], \n"
+                        "    uint2 tgid [[threadgroup_position_in_grid]]) \n"
+                        "{ \n"
+                        "    auto tA = A.slice(0, (int)tgid.y); \n"
+                        "    auto tB = B.slice((int)tgid.x, 0); \n"
+                        " \n"
+                        "    matmul2d< \n"
+                        "        matmul2d_descriptor(16, 16, dynamic_extent), \n"
+                        "        execution_simdgroups<4>> mm; \n"
+                        " \n"
+                        "    auto cT = mm.get_destination_cooperative_tensor<decltype(tA), decltype(tB), float>(); \n"
+                        " \n"
+                        "    auto sA = tA.slice(0, 0); \n"
+                        "    auto sB = tB.slice(0, 0); \n"
+                        "    mm.run(sB, sA, cT); \n"
+                        " \n"
+                        "    auto tC = tensor<device float, dextents<int32_t, 2>, tensor_inline>(C, dextents<int32_t, 2>(16, 16)); \n"
+                        " \n"
+                        "    cT.store(tC); \n"
+                        "}";
 
-                GGML_LOG_INFO("%s: testing tensor API for bfloat support\n", __func__);
-                ggml_metal_library_t lib = ggml_metal_library_init_from_source(dev, src_tensor_bf16, false);
-                if (lib == NULL) {
-                    GGML_LOG_WARN("%s: - the tensor API does not support bfloat - disabling bfloat support\n", __func__);
-                    dev->props.has_bfloat = false;
-                } else {
-                    struct ggml_metal_pipeline_with_params ppl = ggml_metal_library_compile_pipeline(lib, "dummy_kernel", "dummy_kernel", nil);
-                    if (!ppl.pipeline) {
+                    GGML_LOG_INFO("%s: testing tensor API for bfloat support\n", __func__);
+                    ggml_metal_library_t lib = ggml_metal_library_init_from_source(dev, src_tensor_bf16, false);
+                    if (lib == NULL) {
                         GGML_LOG_WARN("%s: - the tensor API does not support bfloat - disabling bfloat support\n", __func__);
                         dev->props.has_bfloat = false;
+                    } else {
+                        struct ggml_metal_pipeline_with_params ppl = ggml_metal_library_compile_pipeline(lib, "dummy_kernel", "dummy_kernel", nil);
+                        if (!ppl.pipeline) {
+                            GGML_LOG_WARN("%s: - the tensor API does not support bfloat - disabling bfloat support\n", __func__);
+                            dev->props.has_bfloat = false;
+                        }
+
+                        ggml_metal_library_free(lib);
                     }
-
-                    ggml_metal_library_free(lib);
                 }
-            }
 
-            dev->props.use_residency_sets = true;
+                dev->props.use_residency_sets = true;
 #if defined(GGML_METAL_HAS_RESIDENCY_SETS)
-            dev->props.use_residency_sets = getenv("GGML_METAL_NO_RESIDENCY") == nil;
+                dev->props.use_residency_sets = getenv("GGML_METAL_NO_RESIDENCY") == nil;
 #endif
 
-            dev->props.use_shared_buffers = dev->props.has_unified_memory;
+                dev->props.use_shared_buffers = dev->props.has_unified_memory;
 #if TARGET_OS_OSX
-            // In case of eGPU, shared memory may be preferable.
-            dev->props.use_shared_buffers |= [dev->mtl_device location] == MTLDeviceLocationExternal;
+                // In case of eGPU, shared memory may be preferable.
+                dev->props.use_shared_buffers |= [dev->mtl_device location] == MTLDeviceLocationExternal;
 #endif
-            if (getenv("GGML_METAL_SHARED_BUFFERS_DISABLE") != NULL) {
-                dev->props.use_shared_buffers = false;
-            }
-            if (getenv("GGML_METAL_SHARED_BUFFERS_ENABLE") != NULL) {
-                dev->props.use_shared_buffers = true;
-            }
+                if (getenv("GGML_METAL_SHARED_BUFFERS_DISABLE") != NULL) {
+                    dev->props.use_shared_buffers = false;
+                }
+                if (getenv("GGML_METAL_SHARED_BUFFERS_ENABLE") != NULL) {
+                    dev->props.use_shared_buffers = true;
+                }
 
-            dev->props.supports_gpu_family_apple7 = [dev->mtl_device supportsFamily:MTLGPUFamilyApple7];
+                dev->props.supports_gpu_family_apple7 = [dev->mtl_device supportsFamily:MTLGPUFamilyApple7];
 
-            dev->props.device_id = ggml_metal_device_id_parse([[dev->mtl_device name] UTF8String]);
+                dev->props.device_id = ggml_metal_device_id_parse([[dev->mtl_device name] UTF8String]);
 
-            dev->props.op_offload_min_batch_size  = getenv("GGML_OP_OFFLOAD_MIN_BATCH") ? atoi(getenv("GGML_OP_OFFLOAD_MIN_BATCH")) : 32;
+                dev->props.op_offload_min_batch_size  = getenv("GGML_OP_OFFLOAD_MIN_BATCH") ? atoi(getenv("GGML_OP_OFFLOAD_MIN_BATCH")) : 32;
 
-            dev->props.max_buffer_size            = dev->mtl_device.maxBufferLength;
-            dev->props.max_theadgroup_memory_size = dev->mtl_device.maxThreadgroupMemoryLength;
-            if (@available(macOS 10.12, iOS 16.0, *)) {
-                dev->props.max_working_set_size   = dev->mtl_device.recommendedMaxWorkingSetSize;
-            } else {
-                dev->props.max_working_set_size   = dev->mtl_device.maxBufferLength;
-            }
+                dev->props.max_buffer_size            = dev->mtl_device.maxBufferLength;
+                dev->props.max_theadgroup_memory_size = dev->mtl_device.maxThreadgroupMemoryLength;
+                if (@available(macOS 10.12, iOS 16.0, *)) {
+                    dev->props.max_working_set_size   = dev->mtl_device.recommendedMaxWorkingSetSize;
+                } else {
+                    dev->props.max_working_set_size   = dev->mtl_device.maxBufferLength;
+                }
 
-            snprintf(dev->props.name, sizeof(dev->props.name), "%s%d", "MTL", device);
-            const char * gpu_name = [[dev->mtl_device name] UTF8String];
-            if (n_devices > 1) {
-                snprintf(dev->props.desc, sizeof(dev->props.desc), "%s (dev p%d/v%d)",
-                         gpu_name, dev->props.device_phys, dev->props.device_virt);
-            } else {
-                snprintf(dev->props.desc, sizeof(dev->props.desc), "%s", gpu_name);
-            }
+                snprintf(dev->props.name, sizeof(dev->props.name), "%s%d", "MTL", device);
+                const char * gpu_name = [[dev->mtl_device name] UTF8String];
+                if (n_devices > 1) {
+                    snprintf(dev->props.desc, sizeof(dev->props.desc), "%s (dev p%d/v%d)",
+                             gpu_name, dev->props.device_phys, dev->props.device_virt);
+                } else {
+                    snprintf(dev->props.desc, sizeof(dev->props.desc), "%s", gpu_name);
+                }
 
-            dev->library = ggml_metal_library_init(dev);
-            if (!dev->library) {
-                GGML_LOG_ERROR("%s: error: failed to create library\n", __func__);
-            }
+                dev->library = ggml_metal_library_init(dev);
+                if (!dev->library) {
+                    GGML_LOG_ERROR("%s: error: failed to create library\n", __func__);
+                }
 
-            if (dev->props.use_residency_sets) {
-                dev->rsets = ggml_metal_rsets_init(dev);
-            } else {
-                dev->rsets = nil;
-            }
+                if (dev->props.use_residency_sets) {
+                    dev->rsets = ggml_metal_rsets_init(dev);
+                } else {
+                    dev->rsets = nil;
+                }
 
-            // print MTL GPU family:
-            GGML_LOG_INFO("%s: GPU name:   %s (%s)\n", __func__, dev->props.name, dev->props.desc);
+                // print MTL GPU family:
+                GGML_LOG_INFO("%s: GPU name:   %s (%s)\n", __func__, dev->props.name, dev->props.desc);
 
-            // determine max supported GPU family
-            // https://developer.apple.com/metal/Metal-Shading-Language-Specification.pdf
-            // https://developer.apple.com/metal/Metal-Feature-Set-Tables.pdf
-            {
-                for (int i = MTLGPUFamilyApple1 + 20; i >= MTLGPUFamilyApple1; --i) {
-                    if ([dev->mtl_device supportsFamily:i]) {
-                        dev->props.gpu_family = i - (int) MTLGPUFamilyApple1 + 1;
-                        GGML_LOG_INFO("%s: GPU family: MTLGPUFamilyApple%d  (%d)\n", __func__, dev->props.gpu_family, i);
-                        break;
+                // determine max supported GPU family
+                // https://developer.apple.com/metal/Metal-Shading-Language-Specification.pdf
+                // https://developer.apple.com/metal/Metal-Feature-Set-Tables.pdf
+                {
+                    for (int i = MTLGPUFamilyApple1 + 20; i >= MTLGPUFamilyApple1; --i) {
+                        if ([dev->mtl_device supportsFamily:i]) {
+                            dev->props.gpu_family = i - (int) MTLGPUFamilyApple1 + 1;
+                            GGML_LOG_INFO("%s: GPU family: MTLGPUFamilyApple%d  (%d)\n", __func__, dev->props.gpu_family, i);
+                            break;
+                        }
+                    }
+
+                    for (int i = MTLGPUFamilyCommon1 + 5; i >= MTLGPUFamilyCommon1; --i) {
+                        if ([dev->mtl_device supportsFamily:i]) {
+                            GGML_LOG_INFO("%s: GPU family: MTLGPUFamilyCommon%d (%d)\n", __func__, i - (int) MTLGPUFamilyCommon1 + 1, i);
+                            break;
+                        }
+                    }
+
+                    for (int i = MTLGPUFamilyMetal3_GGML + 5; i >= MTLGPUFamilyMetal3_GGML; --i) {
+                        if ([dev->mtl_device supportsFamily:i]) {
+                            GGML_LOG_INFO("%s: GPU family: MTLGPUFamilyMetal%d  (%d)\n", __func__, i - (int) MTLGPUFamilyMetal3_GGML + 3, i);
+                            break;
+                        }
                     }
                 }
 
-                for (int i = MTLGPUFamilyCommon1 + 5; i >= MTLGPUFamilyCommon1; --i) {
-                    if ([dev->mtl_device supportsFamily:i]) {
-                        GGML_LOG_INFO("%s: GPU family: MTLGPUFamilyCommon%d (%d)\n", __func__, i - (int) MTLGPUFamilyCommon1 + 1, i);
-                        break;
-                    }
-                }
-
-                for (int i = MTLGPUFamilyMetal3_GGML + 5; i >= MTLGPUFamilyMetal3_GGML; --i) {
-                    if ([dev->mtl_device supportsFamily:i]) {
-                        GGML_LOG_INFO("%s: GPU family: MTLGPUFamilyMetal%d  (%d)\n", __func__, i - (int) MTLGPUFamilyMetal3_GGML + 3, i);
-                        break;
-                    }
-                }
-            }
-
-            GGML_LOG_INFO("%s: simdgroup reduction   = %s\n", __func__, dev->props.has_simdgroup_reduction ? "true" : "false");
-            GGML_LOG_INFO("%s: simdgroup matrix mul. = %s\n", __func__, dev->props.has_simdgroup_mm        ? "true" : "false");
-            GGML_LOG_INFO("%s: has unified memory    = %s\n", __func__, dev->props.has_unified_memory      ? "true" : "false");
-            GGML_LOG_INFO("%s: has bfloat            = %s\n", __func__, dev->props.has_bfloat              ? "true" : "false");
-            GGML_LOG_INFO("%s: has tensor            = %s\n", __func__, dev->props.has_tensor              ? "true" : "false");
-            GGML_LOG_INFO("%s: use residency sets    = %s\n", __func__, dev->props.use_residency_sets      ? "true" : "false");
-            GGML_LOG_INFO("%s: use shared buffers    = %s\n", __func__, dev->props.use_shared_buffers      ? "true" : "false");
+                GGML_LOG_INFO("%s: simdgroup reduction   = %s\n", __func__, dev->props.has_simdgroup_reduction ? "true" : "false");
+                GGML_LOG_INFO("%s: simdgroup matrix mul. = %s\n", __func__, dev->props.has_simdgroup_mm        ? "true" : "false");
+                GGML_LOG_INFO("%s: has unified memory    = %s\n", __func__, dev->props.has_unified_memory      ? "true" : "false");
+                GGML_LOG_INFO("%s: has bfloat            = %s\n", __func__, dev->props.has_bfloat              ? "true" : "false");
+                GGML_LOG_INFO("%s: has tensor            = %s\n", __func__, dev->props.has_tensor              ? "true" : "false");
+                GGML_LOG_INFO("%s: use residency sets    = %s\n", __func__, dev->props.use_residency_sets      ? "true" : "false");
+                GGML_LOG_INFO("%s: use shared buffers    = %s\n", __func__, dev->props.use_shared_buffers      ? "true" : "false");
 
 #if TARGET_OS_OSX || (TARGET_OS_IOS && __clang_major__ >= 15)
-            if (@available(macOS 10.12, iOS 16.0, *)) {
-                GGML_LOG_INFO("%s: recommendedMaxWorkingSetSize  = %8.2f MB\n", __func__, dev->props.max_working_set_size / 1e6);
-            }
+                if (@available(macOS 10.12, iOS 16.0, *)) {
+                    GGML_LOG_INFO("%s: recommendedMaxWorkingSetSize  = %8.2f MB\n", __func__, dev->props.max_working_set_size / 1e6);
+                }
 #endif
+            }
         }
     }
 

ggml/src/ggml-opencl/ggml-opencl.cpp

@@ -903,6 +903,8 @@ struct ggml_backend_opencl_context {
     cl_kernel kernel_gemv_moe_mxfp4_f32_ns_wimg = nullptr;      // weight-as-texture MoE decode GEMV
     cl_kernel kernel_gemm_moe_mxfp4_q8_1_dp4a = nullptr;   // dp4a (int8) mxfp4 MoE prefill GEMM
     cl_kernel kernel_gemm_moe_q4_0_q8_1_dp4a = nullptr;    // dp4a (int8) q4_0 MoE prefill GEMM
+    cl_kernel kernel_gemm_moe_mxfp4_q8_1_dp4a_bin = nullptr;   // binary dp4a (int8) mxfp4 MoE prefill GEMM
+    cl_kernel kernel_gemm_moe_q4_0_q8_1_dp4a_bin = nullptr;    // binary dp4a (int8) q4_0 MoE prefill GEMM
     cl_kernel kernel_moe_reorder_b;
     cl_kernel kernel_moe_histogram, kernel_moe_scan, kernel_moe_fill, kernel_moe_scatter;
     cl_kernel kernel_moe_scatter_stable = nullptr;   // deterministic slot assignment
@@ -4248,6 +4250,24 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {
         GGML_LOG_CONT(".");
     }
 
+    // gemm_moe_mxfp4_q8_1_dp4a_bin (dp4a prefill GEMM)
+    if (backend_ctx->has_integer_dot) {
+        size_t bin_size = 0;
+        backend_ctx->kernel_gemm_moe_mxfp4_q8_1_dp4a_bin = nullptr;
+
+        if (use_adreno_bin_kernels(backend_ctx)) {
+            const char * kernel_bin = (const char *)backend_ctx->get_adreno_bin_kernel("gemm_moe_mxfp4_q8_1_dp4a_ila", &bin_size);
+            if (kernel_bin && bin_size > 0) {
+                cl_program prog =
+                    build_program_from_binary(backend_ctx->context, backend_ctx->device, kernel_bin, CL_moe_compile_opts, bin_size);
+
+                CL_CHECK((backend_ctx->kernel_gemm_moe_mxfp4_q8_1_dp4a_bin = clCreateKernel(prog, "kernel_gemm_moe_mxfp4_q8_1_dp4a_ila", &err), err));
+                CL_CHECK(clReleaseProgram(prog));
+                GGML_LOG_CONT(".");
+            }
+        }
+    }
+
     // gemm_moe_q4_0_q8_1_dp4a (dp4a prefill GEMM)
     if (backend_ctx->has_integer_dot) {
 #ifdef GGML_OPENCL_EMBED_KERNELS
@@ -4265,6 +4285,24 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {
         GGML_LOG_CONT(".");
     }
 
+    // gemm_moe_q4_0_q8_1_dp4a_bin (dp4a prefill GEMM)
+    if (backend_ctx->has_integer_dot) {
+        size_t bin_size = 0;
+        backend_ctx->kernel_gemm_moe_q4_0_q8_1_dp4a_bin = nullptr;
+
+        if (use_adreno_bin_kernels(backend_ctx)) {
+            const char * kernel_bin = (const char *)backend_ctx->get_adreno_bin_kernel("gemm_moe_q4_0_q8_1_dp4a_ila", &bin_size);
+            if (kernel_bin && bin_size > 0) {
+                cl_program prog =
+                    build_program_from_binary(backend_ctx->context, backend_ctx->device, kernel_bin, CL_moe_compile_opts, bin_size);
+
+                CL_CHECK((backend_ctx->kernel_gemm_moe_q4_0_q8_1_dp4a_bin = clCreateKernel(prog, "kernel_gemm_moe_q4_0_q8_1_dp4a_ila", &err), err));
+                CL_CHECK(clReleaseProgram(prog));
+                GGML_LOG_CONT(".");
+            }
+        }
+    }
+
     // gemm_moe_q8_1_dp4a (generic dp4a MoE GEMM; MOE_QT=80 -> q8_0 expert variant)
     if (backend_ctx->has_integer_dot) {
 #ifdef GGML_OPENCL_EMBED_KERNELS
@@ -21519,7 +21557,9 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0,
                     // dot prod has to be available
                     use_moe_dp4a = backend_ctx->has_integer_dot && use_moe_dp4a;
                     // bin kernel takes precedence
-                    use_moe_dp4a = use_moe_dp4a && backend_ctx->kernel_gemm_moe_q4_0_f32_ns_bin == nullptr;
+                    if (backend_ctx->kernel_gemm_moe_q4_0_q8_1_dp4a_bin == nullptr) {
+                        use_moe_dp4a = use_moe_dp4a && backend_ctx->kernel_gemm_moe_q4_0_f32_ns_bin == nullptr;
+                    }
 
                     cl_buffer_region region;
                     region.origin = 0;
@@ -21625,6 +21665,10 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0,
 
                         // dp4a GEMM
                         cl_kernel dk = backend_ctx->kernel_gemm_moe_q4_0_q8_1_dp4a;
+                        if (backend_ctx->kernel_gemm_moe_q4_0_q8_1_dp4a_bin) {
+                            dk = backend_ctx->kernel_gemm_moe_q4_0_q8_1_dp4a_bin;
+                        }
+
                         int aidx = 0;
                         CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &extra0_q4_0->q_img));
                         CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &extra0_q4_0->d));
@@ -23463,8 +23507,10 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0,
                         : (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E);
                     // dot prod has to be available
                     use_moe_dp4a = backend_ctx->has_integer_dot && use_moe_dp4a;
-                    // bin kernel takes precedence
-                    use_moe_dp4a = use_moe_dp4a && backend_ctx->kernel_gemm_moe_mxfp4_f32_ns_bin == nullptr;
+                    // bin kernel takes precedence, dp4a bin kernel has higher priority than normal bin kernel
+                    if (backend_ctx->kernel_gemm_moe_mxfp4_q8_1_dp4a_bin == nullptr) {
+                        use_moe_dp4a = use_moe_dp4a && backend_ctx->kernel_gemm_moe_mxfp4_f32_ns_bin == nullptr;
+                    }
 
                     cl_buffer_region region;
                     region.origin = 0;
@@ -23573,6 +23619,10 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0,
 
                         // dp4a GEMM
                         cl_kernel dk = backend_ctx->kernel_gemm_moe_mxfp4_q8_1_dp4a;
+                        if (backend_ctx->kernel_gemm_moe_mxfp4_q8_1_dp4a_bin) {
+                            dk = backend_ctx->kernel_gemm_moe_mxfp4_q8_1_dp4a_bin;
+                        }
+
                         int aidx = 0;
                         CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &extra0_mxfp4->q_img));
                         CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &extra0_mxfp4->e));

ggml/src/ggml-vulkan/ggml-vulkan.cpp

@@ -767,6 +767,21 @@ static constexpr std::initializer_list<std::array<int, 3>> rms_norm_mul_rope_vie
     { 4, 0, 3 }, // set_rows->src[0] == view
 };
 
+static constexpr std::array<ggml_type, 9> lightning_indexer_k_types = {
+    GGML_TYPE_F32,
+    GGML_TYPE_F16,
+    GGML_TYPE_BF16,
+    GGML_TYPE_Q8_0,
+    GGML_TYPE_Q5_1,
+    GGML_TYPE_Q5_0,
+    GGML_TYPE_Q4_1,
+    GGML_TYPE_Q4_0,
+    GGML_TYPE_IQ4_NL,
+};
+
+static bool ggml_vk_lightning_indexer_k_type_supported(ggml_type type) {
+    return std::find(lightning_indexer_k_types.begin(), lightning_indexer_k_types.end(), type) != lightning_indexer_k_types.end();
+}
 
 struct vk_device_struct {
     std::recursive_mutex mutex;
@@ -1068,6 +1083,7 @@ struct vk_device_struct {
     vk_pipeline pipeline_rwkv_wkv6_f32;
     vk_pipeline pipeline_rwkv_wkv7_f32;
     vk_pipeline pipeline_gated_linear_attn_f32;
+    vk_pipeline pipeline_lightning_indexer_f32[GGML_TYPE_COUNT];
     // [size_idx][kda] where size_idx: 0=d16, 1=d32, 2=d64, 3=d128
     vk_pipeline pipeline_gated_delta_net[4][2];
     vk_pipeline pipeline_ssm_scan_f32_d128;
@@ -1848,6 +1864,26 @@ struct vk_op_gated_linear_attn_push_constants {
     uint32_t H;
     float scale;
 };
+struct vk_op_lightning_indexer_push_constants {
+    uint32_t n_kv;
+    uint32_t n_heads;
+    uint32_t n_tokens;
+    uint32_t n_streams;
+    uint32_t n_masks;
+    uint32_t dispatch_x;
+    uint32_t q_nb1;
+    uint32_t q_nb2;
+    uint32_t q_nb3;
+    uint32_t k_nb2;
+    uint32_t k_nb3;
+    uint32_t w_nb1;
+    uint32_t w_nb3;
+    uint32_t m_nb1;
+    uint32_t m_nb3;
+    uint32_t d_nb1;
+    uint32_t d_nb3;
+};
+static_assert(sizeof(vk_op_lightning_indexer_push_constants) <= 128);
 struct vk_op_gated_delta_net_push_constants {
     uint32_t H;
     uint32_t n_tokens;
@@ -3904,11 +3940,16 @@ static vk_fa_pipeline_state get_fa_pipeline_state(const vk_device& device, const
     return vk_fa_pipeline_state{hsk, hsv, params.block_rows, params.block_cols, params.d_split, params.row_split, params.shmem_staging, params.path, params.workgroup_size, subgroup_size, aligned, f32acc, flags, params.limit_occupancy_shmem, k_type, v_type};
 }
 
+// Bytes per buffer block for the FaBlockBytesK/V spec constants. F32 is fed as
+// a vec4 "block" of 4 floats, everything else uses its ggml block size.
+static uint32_t fa_block_bytes(ggml_type t) {
+    if (t == GGML_TYPE_F32) {
+        return 16u;
+    }
+    return (uint32_t) ggml_type_size(t);
+}
+
 static std::vector<uint32_t> get_fa_spec_constants(const vk_fa_pipeline_state& state) {
-    const auto fa_block_bytes = [](ggml_type t) -> uint32_t {
-        if (t == GGML_TYPE_F32) return 16u;
-        return (uint32_t) ggml_type_size(t);
-    };
     return {
         /* 0 WorkGroupSize   */ state.workgroup_size,
         /* 1 Br              */ state.Br,
@@ -5847,6 +5888,17 @@ static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) {
 
     ggml_vk_create_pipeline(device, device->pipeline_gated_linear_attn_f32, "gated_linear_attn_f32", gated_linear_attn_f32_len, gated_linear_attn_f32_data, "main", 6, sizeof(vk_op_gated_linear_attn_push_constants), {1, 1, 1}, {}, 1);
 
+    {
+        const bool li_subgroup = device->subgroup_arithmetic && device->subgroup_require_full_support;
+        const size_t li_len   = li_subgroup ? lightning_indexer_subgroup_f32_len  : lightning_indexer_f32_len;
+        const void * li_data  = li_subgroup ? (const void *)lightning_indexer_subgroup_f32_data : (const void *)lightning_indexer_f32_data;
+
+        for (ggml_type k_type : lightning_indexer_k_types) {
+            const std::string name = "lightning_indexer_" + std::string(ggml_type_name(k_type)) + "_k_f32";
+            ggml_vk_create_pipeline(device, device->pipeline_lightning_indexer_f32[k_type], name.c_str(), li_len, li_data, "main", 5, sizeof(vk_op_lightning_indexer_push_constants), {1, 1, 1}, {(uint32_t)k_type, fa_block_bytes(k_type), device->subgroup_size}, 1, true, li_subgroup);
+        }
+    }
+
     {
         const uint32_t gdn_sizes[] = {16, 32, 64, 128};
         const char * gdn_names[][2] = {
@@ -11697,6 +11749,12 @@ static vk_pipeline ggml_vk_op_get_pipeline(ggml_backend_vk_context * ctx, const
             return ctx->device->pipeline_gated_linear_attn_f32;
         }
         return nullptr;
+    case GGML_OP_LIGHTNING_INDEXER:
+        // only the k type selects a pipeline, the other types are fixed by ggml_lightning_indexer()
+        if (ggml_vk_lightning_indexer_k_type_supported(src1->type)) {
+            return ctx->device->pipeline_lightning_indexer_f32[src1->type];
+        }
+        return nullptr;
     case GGML_OP_GATED_DELTA_NET:
         if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) {
             const uint32_t S_v = dst->src[2]->ne[0];
@@ -12772,6 +12830,55 @@ static void ggml_vk_gated_linear_attn(ggml_backend_vk_context * ctx, vk_context&
         pc, { (uint32_t)(n_seqs * n_heads), 1, 1 });
 }
 
+static void ggml_vk_lightning_indexer(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_tensor * dst) {
+    const ggml_tensor * q = dst->src[0];
+    const ggml_tensor * k = dst->src[1];
+    const ggml_tensor * w = dst->src[2];
+    const ggml_tensor * m = dst->src[3];
+
+    vk_pipeline pipeline = ggml_vk_op_get_pipeline(ctx, q, k, w, dst, dst->op);
+    GGML_ASSERT(pipeline != nullptr);
+
+    ggml_pipeline_request_descriptor_sets(ctx, pipeline, 1);
+
+    const uint32_t n_kv      = k->ne[2];
+    const uint32_t n_heads   = q->ne[1];
+    const uint32_t n_tokens  = q->ne[2];
+    const uint32_t n_streams = q->ne[3];
+    const uint32_t n_masks   = m->ne[3];
+
+    const uint32_t n_outputs = (uint32_t)(dst->ne[0] * dst->ne[1] * dst->ne[3]);
+    const uint32_t dispatch_x = std::min(n_outputs, ctx->device->properties.limits.maxComputeWorkGroupCount[0]);
+    const uint32_t dispatch_y = CEIL_DIV(n_outputs, dispatch_x);
+
+    // q, w and dst are f32 and m is f16, so their strides are passed in elements;
+    // k may be quantized, so its strides stay in bytes
+    const uint32_t q_nb1 = q->nb[1] / sizeof(float);
+    const uint32_t q_nb2 = q->nb[2] / sizeof(float);
+    const uint32_t q_nb3 = q->nb[3] / sizeof(float);
+    const uint32_t k_nb2 = k->nb[2];
+    const uint32_t k_nb3 = k->nb[3];
+    const uint32_t w_nb1 = w->nb[1] / sizeof(float);
+    const uint32_t w_nb3 = w->nb[3] / sizeof(float);
+    const uint32_t m_nb1 = m->nb[1] / sizeof(ggml_fp16_t);
+    const uint32_t m_nb3 = m->nb[3] / sizeof(ggml_fp16_t);
+    const uint32_t d_nb1 = dst->nb[1] / sizeof(float);
+    const uint32_t d_nb3 = dst->nb[3] / sizeof(float);
+
+    const vk_op_lightning_indexer_push_constants pc = {
+        n_kv, n_heads, n_tokens, n_streams, n_masks, dispatch_x,
+        q_nb1, q_nb2, q_nb3,
+        k_nb2, k_nb3,
+        w_nb1, w_nb3,
+        m_nb1, m_nb3,
+        d_nb1, d_nb3,
+    };
+
+    ggml_vk_dispatch_pipeline(ctx, subctx, pipeline,
+        {ggml_vk_tensor_subbuffer(ctx, q), ggml_vk_tensor_subbuffer(ctx, k), ggml_vk_tensor_subbuffer(ctx, w), ggml_vk_tensor_subbuffer(ctx, m), ggml_vk_tensor_subbuffer(ctx, dst)},
+        pc, {dispatch_x, dispatch_y, 1});
+}
+
 static void ggml_vk_gated_delta_net(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_tensor * dst) {
     const ggml_tensor * src_q     = dst->src[0];
     const ggml_tensor * src_v     = dst->src[2];
@@ -15898,6 +16005,11 @@ static bool ggml_vk_build_graph(ggml_backend_vk_context * ctx, ggml_cgraph * cgr
 
         break;
 
+    case GGML_OP_LIGHTNING_INDEXER:
+        ggml_vk_lightning_indexer(ctx, compute_ctx, node);
+
+        break;
+
     case GGML_OP_GATED_DELTA_NET:
         ggml_vk_gated_delta_net(ctx, compute_ctx, node);
 
@@ -18676,6 +18788,40 @@ static bool ggml_backend_vk_device_supports_op(ggml_backend_dev_t dev, const ggm
         case GGML_OP_GATED_LINEAR_ATTN:
             // the shader block size is hardcoded to head_size 64
             return op->src[0]->type == GGML_TYPE_F32 && op->type == GGML_TYPE_F32 && op->src[0]->ne[0] == 64;
+        case GGML_OP_LIGHTNING_INDEXER:
+            {
+                const ggml_tensor * q = op->src[0];
+                const ggml_tensor * k = op->src[1];
+                const ggml_tensor * w = op->src[2];
+                const ggml_tensor * m = op->src[3];
+
+                // the q/w/m types and the shape relationships between q, k, w, m and dst
+                // are already asserted in ggml_lightning_indexer()
+                if (!ggml_vk_lightning_indexer_k_type_supported(k->type) || !device->fp16) {
+                    return false;
+                }
+
+                // the shader block size is hardcoded to head size 128
+                if (q->ne[0] != 128) {
+                    return false;
+                }
+
+                // the shader indexes the buffers by element stride, and is dispatched
+                // without allow_misalign
+                for (const ggml_tensor * t : {q, k, w, m, op}) {
+                    if (t->nb[0] != ggml_type_size(t->type) ||
+                        (vk_tensor_offset(t) + t->view_offs) % device->properties.limits.minStorageBufferOffsetAlignment != 0) {
+                        return false;
+                    }
+                    // the strides get scaled down from bytes, so the division must be exact
+                    for (int i = 1; i < GGML_MAX_DIMS; ++i) {
+                        if (t->nb[i] % ggml_type_size(t->type) != 0) {
+                            return false;
+                        }
+                    }
+                }
+                return true;
+            }
         case GGML_OP_GATED_DELTA_NET:
             {
                 const uint32_t S_v = op->src[2]->ne[0];
@@ -19685,6 +19831,8 @@ static void ggml_vk_check_results_0(ggml_backend_vk_context * ctx, ggml_cgraph *
             const float * op_params = (const float *)tensor->op_params;
             tensor_clone = ggml_gated_linear_attn(ggml_ctx, src_clone[0], src_clone[1],
             src_clone[2], src_clone[3], src_clone[4], op_params[0]);
+        } else if (tensor->op == GGML_OP_LIGHTNING_INDEXER) {
+            tensor_clone = ggml_lightning_indexer(ggml_ctx, src_clone[0], src_clone[1], src_clone[2], src_clone[3]);
         } else if (tensor->op == GGML_OP_GATED_DELTA_NET) {
             tensor_clone = ggml_gated_delta_net(ggml_ctx, src_clone[0], src_clone[1],
             src_clone[2], src_clone[3], src_clone[4], src_clone[5],

ggml/src/ggml-vulkan/vulkan-shaders/fa_types.glsl

@@ -0,0 +1,55 @@
+#if !defined(GGML_FA_TYPES_COMP)
+#define GGML_FA_TYPES_COMP
+
+// FaTypeK / FaTypeV spec constant values. These mirror enum ggml_type so the
+// host can pass the type directly. Keep in sync with ggml.h.
+#define FA_TYPE_F32   0u
+#define FA_TYPE_F16   1u
+#define FA_TYPE_Q4_0  2u
+#define FA_TYPE_Q4_1  3u
+#define FA_TYPE_Q5_0  6u
+#define FA_TYPE_Q5_1  7u
+#define FA_TYPE_Q8_0  8u
+#define FA_TYPE_IQ4_NL 20u
+#define FA_TYPE_BF16 30u
+
+// Number of matrix elements per buffer block, derived from the K/V type spec
+// constant. F32 is treated as a vec4 "block" of 4 floats. F16 uses block size 1
+// and bypasses the dequant path entirely. Quants follow their ggml block sizes.
+uint fa_block_elems(uint ty) {
+    switch (ty) {
+        case FA_TYPE_F32:  return 4u;
+        case FA_TYPE_F16:  return 1u;
+        case FA_TYPE_Q4_0: return uint(QUANT_K_Q4_0);
+        case FA_TYPE_Q4_1: return uint(QUANT_K_Q4_1);
+        case FA_TYPE_Q5_0: return uint(QUANT_K_Q5_0);
+        case FA_TYPE_Q5_1: return uint(QUANT_K_Q5_1);
+        case FA_TYPE_Q8_0: return uint(QUANT_K_Q8_0);
+        case FA_TYPE_IQ4_NL: return uint(QUANT_K_IQ4_NL);
+        case FA_TYPE_BF16: return 1u;
+        default:           return 1u;
+    }
+}
+
+// QUANT_R_MMQ for FA-eligible K types. Q4_*/Q5_* store two nibbles per byte
+// (R==2); Q8_0 stores one byte per element (R==1). Used to derive the number
+// of int32s per 32-element block on the MMQ K path: ints_per_block == 8 / R.
+uint fa_quant_r_mmq(uint ty) {
+    switch (ty) {
+        case FA_TYPE_Q4_0: return uint(QUANT_R_Q4_0);
+        case FA_TYPE_Q4_1: return uint(QUANT_R_Q4_1);
+        case FA_TYPE_Q5_0: return uint(QUANT_R_Q5_0);
+        case FA_TYPE_Q5_1: return uint(QUANT_R_Q5_1);
+        case FA_TYPE_Q8_0: return uint(QUANT_R_Q8_0);
+        default:           return 1u;
+    }
+}
+
+bool fa_type_needs_shmem(uint ty) {
+    switch (ty) {
+        case FA_TYPE_IQ4_NL: return true;
+        default:             return false;
+    }
+}
+
+#endif // !defined(GGML_FA_TYPES_COMP)

ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_base.glsl

@@ -88,17 +88,7 @@ layout (binding = 6) readonly buffer MO {uint32_t data_mask_opt[];};
 #define BINDING_IDX_K 0
 #define BINDING_IDX_V 1
 
-// FaTypeK / FaTypeV spec constant values. These mirror enum ggml_type so the
-// host can pass the type directly. Keep in sync with ggml.h.
-#define FA_TYPE_F32   0u
-#define FA_TYPE_F16   1u
-#define FA_TYPE_Q4_0  2u
-#define FA_TYPE_Q4_1  3u
-#define FA_TYPE_Q5_0  6u
-#define FA_TYPE_Q5_1  7u
-#define FA_TYPE_Q8_0  8u
-#define FA_TYPE_IQ4_NL 20u
-#define FA_TYPE_BF16 30u
+#include "fa_types.glsl"
 
 #if defined(BFLOAT16)
 #define O_TYPE float
@@ -108,45 +98,6 @@ layout (binding = 6) readonly buffer MO {uint32_t data_mask_opt[];};
 #define O_TYPEV4 FLOAT_TYPEV4
 #endif
 
-// Number of matrix elements per buffer block, derived from the K/V type spec
-// constant. F32 is treated as a vec4 "block" of 4 floats. F16 uses block size 1
-// and bypasses the dequant path entirely. Quants follow their ggml block sizes.
-uint fa_block_elems(uint ty) {
-    switch (ty) {
-        case FA_TYPE_F32:  return 4u;
-        case FA_TYPE_F16:  return 1u;
-        case FA_TYPE_Q4_0: return uint(QUANT_K_Q4_0);
-        case FA_TYPE_Q4_1: return uint(QUANT_K_Q4_1);
-        case FA_TYPE_Q5_0: return uint(QUANT_K_Q5_0);
-        case FA_TYPE_Q5_1: return uint(QUANT_K_Q5_1);
-        case FA_TYPE_Q8_0: return uint(QUANT_K_Q8_0);
-        case FA_TYPE_IQ4_NL: return uint(QUANT_K_IQ4_NL);
-        case FA_TYPE_BF16: return 1u;
-        default:           return 1u;
-    }
-}
-
-// QUANT_R_MMQ for FA-eligible K types. Q4_*/Q5_* store two nibbles per byte
-// (R==2); Q8_0 stores one byte per element (R==1). Used to derive the number
-// of int32s per 32-element block on the MMQ K path: ints_per_block == 8 / R.
-uint fa_quant_r_mmq(uint ty) {
-    switch (ty) {
-        case FA_TYPE_Q4_0: return uint(QUANT_R_Q4_0);
-        case FA_TYPE_Q4_1: return uint(QUANT_R_Q4_1);
-        case FA_TYPE_Q5_0: return uint(QUANT_R_Q5_0);
-        case FA_TYPE_Q5_1: return uint(QUANT_R_Q5_1);
-        case FA_TYPE_Q8_0: return uint(QUANT_R_Q8_0);
-        default:           return 1u;
-    }
-}
-
-bool fa_type_needs_shmem(uint ty) {
-    switch (ty) {
-        case FA_TYPE_IQ4_NL: return true;
-        default:             return false;
-    }
-}
-
 // These can't be `const` globals because GLSL forbids function calls in global
 // const initializers, even when the spec constants would let the driver fold
 // them. Macros expand at the use site and fold after specialization.

ggml/src/ggml-vulkan/vulkan-shaders/lightning_indexer.comp

@@ -0,0 +1,151 @@
+#version 450
+
+#extension GL_EXT_control_flow_attributes : require
+#extension GL_EXT_shader_16bit_storage : require
+#extension GL_EXT_shader_explicit_arithmetic_types_float16 : require
+#extension GL_KHR_shader_subgroup_basic : enable
+#if USE_SUBGROUP_ADD
+#extension GL_KHR_shader_subgroup_arithmetic : enable
+#endif
+
+#define BINDING_IDX_K 0u
+
+#include "types.glsl"
+#include "fa_types.glsl"
+#define FaTypeV FA_TYPE_F32
+
+layout(constant_id = 0) const uint FaTypeK = FA_TYPE_F32;
+layout(constant_id = 1) const uint FaBlockBytesK = 4;
+layout(constant_id = 2) const uint SUBGROUP_SIZE = 32;
+
+#include "flash_attn_dequant.glsl"
+
+// one workgroup computes one output element, one invocation per head element
+#define HEAD_SIZE 128
+
+layout(local_size_x = HEAD_SIZE, local_size_y = 1, local_size_z = 1) in;
+
+layout(binding = 0) readonly buffer QBuf { float q[]; };
+layout(binding = 1) readonly buffer KBufF16 { float16_t k_f16[]; };
+layout(binding = 1) readonly buffer KBufF32 { float k_f32[]; };
+layout(binding = 1) readonly buffer KBufBF16 { uint16_t k_bf16[]; };
+layout(binding = 2) readonly buffer WBuf { float weights[]; };
+layout(binding = 3) readonly buffer MBuf { float16_t mask[]; };
+layout(binding = 4) writeonly buffer DstBuf { float dst[]; };
+
+layout(push_constant) uniform PushConstants {
+    uint n_kv;
+    uint n_heads;
+    uint n_tokens;
+    uint n_streams;
+    uint n_masks;
+    uint dispatch_x;
+    uint q_nb1;
+    uint q_nb2;
+    uint q_nb3;
+    uint k_nb2;
+    uint k_nb3;
+    uint w_nb1;
+    uint w_nb3;
+    uint m_nb1;
+    uint m_nb3;
+    uint d_nb1;
+    uint d_nb3;
+};
+
+shared float k_row[HEAD_SIZE];
+
+#if USE_SUBGROUP_ADD
+shared float sg_partials[HEAD_SIZE / SUBGROUP_SIZE];
+#else
+shared float partials[HEAD_SIZE];
+#endif
+
+void main() {
+    const uint tid = gl_LocalInvocationID.x;
+    const uint output_idx = gl_WorkGroupID.y * dispatch_x + gl_WorkGroupID.x;
+    const uint n_outputs = n_kv * n_tokens * n_streams;
+
+    if (fa_type_needs_shmem(FaTypeK)) {
+        init_iq_shmem(gl_WorkGroupSize);
+    }
+
+    if (output_idx >= n_outputs) {
+        return;
+    }
+
+    const uint ik = output_idx % n_kv;
+    const uint ts = output_idx / n_kv;
+    const uint t = ts % n_tokens;
+    const uint s = ts / n_tokens;
+    const uint k_offset = ik * k_nb2 + s * k_nb3;
+
+    // k strides come in as bytes, so scale them down to the view being indexed
+    const uint k_block_elems = fa_block_elems(FaTypeK);
+    const uint k_elem_bytes = FaBlockBytesK / k_block_elems;
+
+    if (FaTypeK == FA_TYPE_F16) {
+        k_row[tid] = float(k_f16[k_offset / k_elem_bytes + tid]);
+    } else if (FaTypeK == FA_TYPE_F32) {
+        k_row[tid] = k_f32[k_offset / k_elem_bytes + tid];
+    } else if (FaTypeK == FA_TYPE_BF16) {
+        k_row[tid] = bf16_to_fp32(uint(k_bf16[k_offset / k_elem_bytes + tid]));
+    } else if (4 * tid < HEAD_SIZE) {
+        const uint coord = 4 * tid;
+        const uint ib = coord / k_block_elems;
+        const uint iqs = coord % k_block_elems;
+        const vec4 values = dequantize4(ib, iqs, k_offset / FaBlockBytesK, BINDING_IDX_K);
+        k_row[coord + 0] = values.x;
+        k_row[coord + 1] = values.y;
+        k_row[coord + 2] = values.z;
+        k_row[coord + 3] = values.w;
+    }
+    barrier();
+
+    const float k_val = k_row[tid];
+
+    float score = 0.0;
+    for (uint h = 0; h < n_heads; ++h) {
+        const float prod = q[h * q_nb1 + t * q_nb2 + s * q_nb3 + tid] * k_val;
+
+#if USE_SUBGROUP_ADD
+        const float sg_sum = subgroupAdd(prod);
+        if (gl_SubgroupInvocationID == 0) {
+            sg_partials[gl_SubgroupID] = sg_sum;
+        }
+        barrier();
+
+        if (tid == 0) {
+            float sum = 0.0;
+            [[unroll]] for (uint i = 0; i < HEAD_SIZE / SUBGROUP_SIZE; ++i) {
+                sum += sg_partials[i];
+            }
+            score += max(sum, 0.0) * weights[h + t * w_nb1 + s * w_nb3];
+        }
+        // the reads above must complete before the next iteration overwrites sg_partials
+        barrier();
+#else
+        partials[tid] = prod;
+        barrier();
+
+        [[unroll]] for (uint stride = HEAD_SIZE / 2; stride > 0; stride >>= 1) {
+            if (tid < stride) {
+                partials[tid] += partials[tid + stride];
+            }
+            barrier();
+        }
+
+        if (tid == 0) {
+            score += max(partials[0], 0.0) * weights[h + t * w_nb1 + s * w_nb3];
+        }
+        // the read of partials[0] above must complete before the next iteration
+        // overwrites partials[tid]
+        barrier();
+#endif
+    }
+
+    if (tid == 0) {
+        const uint mask_offset = ik + t * m_nb1 + (s % n_masks) * m_nb3;
+        dst[ik + t * d_nb1 + s * d_nb3] = score + float(mask[mask_offset]);
+    }
+}

ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp

@@ -1069,6 +1069,12 @@ void process_shaders() {
 
     string_to_spv("gated_linear_attn_f32", "gla.comp", merge_maps(base_dict, {{"A_TYPE", "float"}}));
 
+    // Compile IQ4_NL support in so its shared LUT is available when K uses it.
+    // K quant type is selected at runtime via the FaTypeK spec constant.
+    std::map<std::string, std::string> li_dict = {{"FLOAT_TYPE", "float"}, {"FLOAT_TYPEV4", "vec4"}, {"DATA_A_IQ4_NL", "1"}};
+    string_to_spv("lightning_indexer_f32", "lightning_indexer.comp", li_dict);
+    string_to_spv("lightning_indexer_subgroup_f32", "lightning_indexer.comp", merge_maps(li_dict, {{"USE_SUBGROUP_ADD", "1"}}));
+
     string_to_spv("rwkv_wkv7_f32", "wkv7.comp", merge_maps(base_dict, {{"A_TYPE", "float"}}));
 
     string_to_spv("gated_delta_net_f32", "gated_delta_net.comp", merge_maps(base_dict, {{"FLOAT_TYPE", "float"}, {"USE_SUBGROUP_ADD", "1"}, {"USE_SUBGROUP_CLUSTERED", "1"}}));

include/llama.h

@@ -214,6 +214,12 @@ extern "C" {
     LLAMA_API const char * llama_load_mode_name(enum llama_load_mode load_mode);
     LLAMA_API enum llama_load_mode llama_load_mode_from_str(const char * str);
 
+    enum llama_tensor_read_lazy {
+        LLAMA_TENSOR_READ_LAZY_OFF  = 0, // always read the whole tensor up front
+        LLAMA_TENSOR_READ_LAZY_AUTO = 1, // lazy only for marked tensors larger than 4 GiB (requires mmap)
+        LLAMA_TENSOR_READ_LAZY_ON   = 2, // read the rows of tensors marked by the arch on demand (requires mmap)
+    };
+
     enum llama_context_type {
         LLAMA_CONTEXT_TYPE_DEFAULT = 0,
         LLAMA_CONTEXT_TYPE_MTP     = 1,
@@ -315,6 +321,8 @@ extern "C" {
         enum llama_split_mode split_mode; // how to split the model across multiple GPUs
         enum llama_load_mode  load_mode;  // how to load the model
 
+        enum llama_tensor_read_lazy tensor_read_lazy; // on-demand reading of tensors marked by the arch
+
         // the GPU that is used for the entire model when split_mode is LLAMA_SPLIT_MODE_NONE
         int32_t main_gpu;
 
@@ -437,6 +445,7 @@ extern "C" {
         const struct llama_model_kv_override * kv_overrides;        // pointer to kv overrides
         const struct llama_model_tensor_override * tt_overrides;    // pointer to tensor overrides
         const int32_t * prune_layers;                               // pointer to layer indices to prune
+        size_t max_buf_size;                                        // max bytes of tensor rows kept in memory at once, 0 = default (8 GiB)
     } llama_model_quantize_params;
 
     typedef struct llama_logit_bias {

scripts/pr2wt.sh

@@ -48,7 +48,11 @@ echo "org/repo: $org_repo"
 
 meta=$(curl -sSLf -H "Accept: application/vnd.github+json" "https://api.github.com/repos/$org_repo/pulls/$PR")
 
-url_remote=$(echo "$meta" | jq -r '.head.repo.clone_url')
+if [[ $url_origin =~ ^git@ ]]; then
+    url_remote=$(echo "$meta" | jq -r '.head.repo.ssh_url')
+else
+    url_remote=$(echo "$meta" | jq -r '.head.repo.clone_url')
+fi
 head_ref=$(echo "$meta" | jq -r '.head.ref')
 
 echo "url:      $url_remote"

src/llama-mmap.cpp

@@ -438,11 +438,34 @@ void llama_file::write_u32(uint32_t val) const { pimpl->write_u32(val); }
 
 // llama_mmap
 
+#if defined(_POSIX_MAPPED_FILES) || defined(_WIN32)
+// merge `ranges` and return their complement within [0, limit)
+static llama_mmap::ranges ranges_complement(llama_mmap::ranges ranges, size_t limit) {
+    llama_mmap::ranges res;
+    std::sort(ranges.begin(), ranges.end());
+
+    size_t pos = 0;
+    for (const auto & range : ranges) {
+        const size_t beg = std::min(range.first,  limit);
+        const size_t end = std::min(range.second, limit);
+        if (beg > pos) {
+            res.emplace_back(pos, beg);
+        }
+        pos = std::max(pos, end);
+    }
+    if (pos < limit) {
+        res.emplace_back(pos, limit);
+    }
+
+    return res;
+}
+#endif
+
 struct llama_mmap::impl {
 #ifdef _POSIX_MAPPED_FILES
     std::vector<std::pair<size_t, size_t>> mapped_fragments;
 
-    impl(struct llama_file * file, size_t prefetch, bool numa) {
+    impl(struct llama_file * file, size_t prefetch, bool numa, const llama_mmap::ranges & lazy_ranges) {
         size = file->size();
         int fd = file->file_id();
         int flags = MAP_SHARED;
@@ -452,18 +475,34 @@ struct llama_mmap::impl {
             LLAMA_LOG_WARN("warning: posix_fadvise(.., POSIX_FADV_SEQUENTIAL) failed: %s\n",
                     strerror(errno));
         }
-        if (prefetch) { flags |= MAP_POPULATE; }
+        // MAP_POPULATE would fault in the lazy ranges too
+        if (prefetch && lazy_ranges.empty()) { flags |= MAP_POPULATE; }
 #endif
         addr = mmap(NULL, file->size(), PROT_READ, flags, fd, 0);
         if (addr == MAP_FAILED) {
             throw std::runtime_error(format("mmap failed: %s", strerror(errno)));
         }
 
-        if (prefetch > 0) {
-            if (posix_madvise(addr, std::min(file->size(), prefetch), POSIX_MADV_WILLNEED)) {
-                LLAMA_LOG_WARN("warning: posix_madvise(.., POSIX_MADV_WILLNEED) failed: %s\n",
-                        strerror(errno));
+        // page-aligned madvise over [beg, end), clamped to the file
+        auto advise = [&](size_t beg, size_t end, int advice, const char * name) {
+            const size_t page_size = sysconf(_SC_PAGESIZE);
+            beg = beg & ~(page_size - 1);
+            end = std::min((end + page_size - 1) & ~(page_size - 1), file->size());
+            if (beg >= end) {
+                return;
             }
+            if (posix_madvise((char *) addr + beg, end - beg, advice)) {
+                LLAMA_LOG_WARN("warning: posix_madvise(.., %s) failed: %s\n", name, strerror(errno));
+            }
+        };
+
+        if (prefetch > 0) {
+            for (const auto & range : ranges_complement(lazy_ranges, std::min(file->size(), prefetch))) {
+                advise(range.first, range.second, POSIX_MADV_WILLNEED, "POSIX_MADV_WILLNEED");
+            }
+        }
+        for (const auto & range : lazy_ranges) {
+            advise(range.first, range.second, POSIX_MADV_RANDOM, "POSIX_MADV_RANDOM");
         }
         if (numa) {
             if (posix_madvise(addr, file->size(), POSIX_MADV_RANDOM)) {
@@ -533,7 +572,7 @@ struct llama_mmap::impl {
 #elif defined(_WIN32)
     HANDLE hMapping = nullptr;
 
-    impl(struct llama_file * file, size_t prefetch, bool numa) {
+    impl(struct llama_file * file, size_t prefetch, bool numa, const llama_mmap::ranges & lazy_ranges) {
         GGML_UNUSED(numa);
 
         size = file->size();
@@ -563,10 +602,15 @@ struct llama_mmap::impl {
             pPrefetchVirtualMemory = (decltype(pPrefetchVirtualMemory))(void *) GetProcAddress(hKernel32, "PrefetchVirtualMemory");
 
             if (pPrefetchVirtualMemory) {
-                WIN32_MEMORY_RANGE_ENTRY range;
-                range.VirtualAddress = addr;
-                range.NumberOfBytes = (SIZE_T) std::min(size, prefetch);
-                if (!pPrefetchVirtualMemory(GetCurrentProcess(), 1, &range, 0)) {
+                std::vector<WIN32_MEMORY_RANGE_ENTRY> entries;
+                for (const auto & range : ranges_complement(lazy_ranges, std::min(size, prefetch))) {
+                    WIN32_MEMORY_RANGE_ENTRY entry;
+                    entry.VirtualAddress = (char *) addr + range.first;
+                    entry.NumberOfBytes  = (SIZE_T) (range.second - range.first);
+                    entries.push_back(entry);
+                }
+                if (!entries.empty() &&
+                        !pPrefetchVirtualMemory(GetCurrentProcess(), (ULONG_PTR) entries.size(), entries.data(), 0)) {
                     LLAMA_LOG_WARN("warning: PrefetchVirtualMemory failed: %s\n",
                             llama_format_win_err(GetLastError()).c_str());
                 }
@@ -597,10 +641,11 @@ struct llama_mmap::impl {
         }
     }
 #else
-    impl(struct llama_file * file, size_t prefetch, bool numa) {
+    impl(struct llama_file * file, size_t prefetch, bool numa, const llama_mmap::ranges & lazy_ranges) {
         GGML_UNUSED(file);
         GGML_UNUSED(prefetch);
         GGML_UNUSED(numa);
+        GGML_UNUSED(lazy_ranges);
 
         throw std::runtime_error("mmap not supported");
     }
@@ -617,7 +662,8 @@ struct llama_mmap::impl {
     size_t size;
 };
 
-llama_mmap::llama_mmap(struct llama_file * file, size_t prefetch, bool numa) : pimpl(std::make_unique<impl>(file, prefetch, numa)) {}
+llama_mmap::llama_mmap(struct llama_file * file, size_t prefetch, bool numa,
+        const ranges & lazy_ranges) : pimpl(std::make_unique<impl>(file, prefetch, numa, lazy_ranges)) {}
 llama_mmap::~llama_mmap() = default;
 
 size_t llama_mmap::size() const { return pimpl->size; }

src/llama-mmap.h

@@ -2,6 +2,7 @@
 
 #include <cstdint>
 #include <memory>
+#include <utility>
 #include <vector>
 #include <cstdio>
 
@@ -41,8 +42,12 @@ private:
 };
 
 struct llama_mmap {
+    // list of [first, last) byte ranges within a file
+    using ranges = std::vector<std::pair<size_t, size_t>>;
+
     llama_mmap(const llama_mmap &) = delete;
-    llama_mmap(struct llama_file * file, size_t prefetch = (size_t) -1, bool numa = false);
+    llama_mmap(struct llama_file * file, size_t prefetch = (size_t) -1, bool numa = false,
+               const ranges & lazy_ranges = {});
     ~llama_mmap();
 
     size_t size() const;

src/llama-model-loader.cpp

@@ -1282,6 +1282,18 @@ struct ggml_tensor * llama_model_loader::create_tensor(
         return NULL;
     }
 
+    if ((flags & TENSOR_READ_LAZY) && use_mmap && tensor_read_lazy != LLAMA_TENSOR_READ_LAZY_OFF) {
+        // in auto mode, small tensors are cheap enough to keep resident
+        constexpr size_t auto_lazy_min_size = 4ull * 1024 * 1024 * 1024;
+        if (tensor_read_lazy == LLAMA_TENSOR_READ_LAZY_ON || ggml_nbytes(cur) > auto_lazy_min_size) {
+            const auto & w = require_weight(tn.str().c_str());
+            lazy_tensor_ranges[w.idx].emplace_back(w.offs, w.offs + ggml_nbytes(cur));
+
+            LLAMA_LOG_INFO("%s: tensor %s (size = %zu MiB) lazy read enabled\n",
+                    __func__, tn.str().c_str(), ggml_nbytes(cur)/1024/1024);
+        }
+    }
+
     ggml_tensor t_meta = *cur;
     if (flags & TENSOR_ALLOW_RESHAPE) {
         for (size_t dim = 0; dim < GGML_MAX_DIMS; dim++) {
@@ -1349,7 +1361,9 @@ void llama_model_loader::init_mappings(bool prefetch, llama_mlocks * mlock_mmaps
     if (use_mmap) {
         mappings.reserve(files.size());
         mmaps_used.reserve(files.size());
-        for (const auto & file : files) {
+        for (uint32_t idx = 0; idx < files.size(); idx++) {
+            const auto & file = files[idx];
+
             bool is_numa = false;
 
             auto * dev = ggml_backend_dev_by_type(GGML_BACKEND_DEVICE_TYPE_CPU);
@@ -1361,7 +1375,11 @@ void llama_model_loader::init_mappings(bool prefetch, llama_mlocks * mlock_mmaps
                 }
             }
 
-            std::unique_ptr<llama_mmap> mapping = std::make_unique<llama_mmap>(file.get(), prefetch ? -1 : 0, is_numa);
+            const auto it_lazy = lazy_tensor_ranges.find(idx);
+            static const llama_mmap::ranges no_lazy_ranges;
+
+            std::unique_ptr<llama_mmap> mapping = std::make_unique<llama_mmap>(file.get(), prefetch ? -1 : 0, is_numa,
+                    it_lazy != lazy_tensor_ranges.end() ? it_lazy->second : no_lazy_ranges);
             mmaps_used.emplace_back(mapping->size(), 0);
             if (mlock_mmaps) {
                 std::unique_ptr<llama_mlock> mlock_mmap(new llama_mlock());
@@ -1400,27 +1418,26 @@ void llama_model_loader::unmap_weight(const llama_tensor_weight & w) const {
     mappings.at(w.idx)->unmap_fragment(w.offs, w.offs + ggml_nbytes(w.tensor));
 }
 
-void llama_model_loader::load_data_for(struct ggml_tensor * cur) const {
-    const auto & w = require_weight(ggml_get_name(cur));
+const void * llama_model_loader::load_data_range(const llama_tensor_weight & w, size_t offs, size_t size, void * buf) const {
+    GGML_ASSERT(offs + size <= ggml_nbytes(w.tensor));
+
+    const void * data = buf;
 
     if (use_mmap) {
-        const auto & mapping = mappings.at(w.idx);
-        if (cur->data == nullptr) {
-            cur->data = (uint8_t *)mapping->addr() + w.offs;
-        } else {
-            memcpy(cur->data, (uint8_t *)mapping->addr() + w.offs, ggml_nbytes(cur));
-        }
+        data = (const uint8_t *) mappings.at(w.idx)->addr() + w.offs + offs;
     } else {
-        GGML_ASSERT(cur->data != nullptr);
+        GGML_ASSERT(buf != nullptr);
         GGML_ASSERT(w.idx < files.size());
         const auto & file = files.at(w.idx);
-        file->seek(w.offs, SEEK_SET);
-        file->read_raw(cur->data, ggml_nbytes(cur));
+        file->seek(w.offs + offs, SEEK_SET);
+        file->read_raw(buf, size);
     }
 
-    if (check_tensors && !ggml_validate_row_data(cur->type, cur->data, ggml_nbytes(cur))) {
-        throw std::runtime_error(format("tensor '%s' has invalid data", ggml_get_name(cur)));
+    if (check_tensors && !ggml_validate_row_data(w.tensor->type, data, size)) {
+        throw std::runtime_error(format("tensor '%s' has invalid data", ggml_get_name(w.tensor)));
     }
+
+    return data;
 }
 
 bool llama_model_loader::load_all_data(

src/llama-model-loader.h

@@ -68,6 +68,7 @@ struct llama_model_loader {
     static const int TENSOR_SKIP            = 1 << 2;
     static const int TENSOR_SKIP_IF_VIRTUAL = 1 << 3;
     static const int TENSOR_ALLOW_RESHAPE   = 1 << 4;
+    static const int TENSOR_READ_LAZY       = 1 << 5; // read rows on demand instead of loading whole tensor; requires mmap for now
 
     int n_kv      = 0;
     int n_tensors = 0;
@@ -82,12 +83,18 @@ struct llama_model_loader {
     bool no_alloc;
     bool load_mtp;
 
+    // set by the caller before the create_tensor() calls
+    enum llama_tensor_read_lazy tensor_read_lazy = LLAMA_TENSOR_READ_LAZY_OFF;
+
     llama_files files;
     llama_ftype ftype;
     llama_fver  fver;
 
     llama_mmaps mappings;
 
+    // byte ranges of TENSOR_READ_LAZY tensors, per file index
+    std::map<uint32_t, llama_mmap::ranges> lazy_tensor_ranges;
+
     std::map<std::string, llama_tensor_weight, weight_name_comparer> weights_map;
     std::unordered_map<std::string, llama_model_kv_override> kv_overrides;
     const llama_model_tensor_buft_override * tensor_buft_overrides;
@@ -197,8 +204,9 @@ struct llama_model_loader {
     // release a weight's mmap pages
     void unmap_weight(const llama_tensor_weight & w) const;
 
-    // for backwards compatibility, does not support ggml-backend
-    void load_data_for(struct ggml_tensor * cur) const;
+    // read a byte range of a weight's data
+    // with mmap, returns a pointer into the mapping, otherwise reads into buf and returns buf
+    const void * load_data_range(const llama_tensor_weight & w, size_t offs, size_t size, void * buf) const;
 
     // Returns false if cancelled by progress_callback
     bool load_all_data(

src/llama-model.cpp

@@ -2631,6 +2631,7 @@ llama_model_params llama_model_default_params() {
         /*.n_gpu_layers                =*/ -1,
         /*.split_mode                  =*/ LLAMA_SPLIT_MODE_LAYER,
         /*.load_mode                   =*/ LLAMA_LOAD_MODE_AUTO,
+        /*.tensor_read_lazy            =*/ LLAMA_TENSOR_READ_LAZY_AUTO,
         /*.main_gpu                    =*/ 0,
         /*.tensor_split                =*/ nullptr,
         /*.progress_callback           =*/ nullptr,
@@ -3067,7 +3068,8 @@ llama_model_base::llama_model_base(const struct llama_model_params & params) : l
     TENSOR_NOT_REQUIRED   (llama_model_loader::TENSOR_NOT_REQUIRED),
     TENSOR_SKIP           (llama_model_loader::TENSOR_SKIP),
     TENSOR_SKIP_IF_VIRTUAL(llama_model_loader::TENSOR_SKIP_IF_VIRTUAL),
-    TENSOR_ALLOW_RESHAPE  (llama_model_loader::TENSOR_ALLOW_RESHAPE) {}
+    TENSOR_ALLOW_RESHAPE  (llama_model_loader::TENSOR_ALLOW_RESHAPE),
+    TENSOR_READ_LAZY      (llama_model_loader::TENSOR_READ_LAZY) {}
 
 ggml_tensor * llama_model_base::create_tensor(const LLM_TN_IMPL & tn, const std::initializer_list<int64_t> & ne, int flags) {
     GGML_ASSERT(ml != nullptr);

src/llama-model.h

@@ -756,6 +756,7 @@ struct llama_model_base : public llama_model {
     const int TENSOR_SKIP;
     const int TENSOR_SKIP_IF_VIRTUAL;
     const int TENSOR_ALLOW_RESHAPE;
+    const int TENSOR_READ_LAZY;
 
     explicit llama_model_base(const llama_model_params & params);
     virtual ~llama_model_base() = default;

src/llama-quant.cpp

@@ -38,6 +38,9 @@ enum class tensor_category {
     OTHER
 };
 
+// max amount of tensor data kept in memory while quantizing a single tensor
+static const size_t LLAMA_QUANT_MAX_BUF_SIZE = 8ull*1024*1024*1024;
+
 static void zeros(std::ofstream & file, size_t n) {
     char zero = 0;
     for (size_t i = 0; i < n; ++i) {
@@ -211,31 +214,26 @@ struct tensor_metadata {
 //
 
 static void llama_tensor_dequantize_impl(
-    ggml_tensor * tensor, std::vector<no_init<float>> & output, std::vector<std::thread> & workers,
+    ggml_type type, const void * data, float * f32_output, std::vector<std::thread> & workers,
     const size_t nelements, const int nthread
 ) {
-    if (output.size() < nelements) {
-        output.resize(nelements);
-    }
-    float * f32_output = (float *) output.data();
-
-    const ggml_type_traits * qtype = ggml_get_type_traits(tensor->type);
-    if (ggml_is_quantized(tensor->type)) {
+    const ggml_type_traits * qtype = ggml_get_type_traits(type);
+    if (ggml_is_quantized(type)) {
         if (qtype->to_float == NULL) {
-            throw std::runtime_error(format("type %s unsupported for integer quantization: no dequantization available", ggml_type_name(tensor->type)));
+            throw std::runtime_error(format("type %s unsupported for integer quantization: no dequantization available", ggml_type_name(type)));
         }
-    } else if (tensor->type != GGML_TYPE_F16 &&
-               tensor->type != GGML_TYPE_BF16) {
-        throw std::runtime_error(format("cannot dequantize/convert tensor type %s", ggml_type_name(tensor->type)));
+    } else if (type != GGML_TYPE_F16 &&
+               type != GGML_TYPE_BF16) {
+        throw std::runtime_error(format("cannot dequantize/convert tensor type %s", ggml_type_name(type)));
     }
 
     if (nthread < 2) {
-        if (tensor->type == GGML_TYPE_F16) {
-            ggml_fp16_to_fp32_row((ggml_fp16_t *)tensor->data, f32_output, nelements);
-        } else if (tensor->type == GGML_TYPE_BF16) {
-            ggml_bf16_to_fp32_row((ggml_bf16_t *)tensor->data, f32_output, nelements);
-        } else if (ggml_is_quantized(tensor->type)) {
-            qtype->to_float(tensor->data, f32_output, nelements);
+        if (type == GGML_TYPE_F16) {
+            ggml_fp16_to_fp32_row((const ggml_fp16_t *)data, f32_output, nelements);
+        } else if (type == GGML_TYPE_BF16) {
+            ggml_bf16_to_fp32_row((const ggml_bf16_t *)data, f32_output, nelements);
+        } else if (ggml_is_quantized(type)) {
+            qtype->to_float(data, f32_output, nelements);
         } else {
             GGML_ABORT("fatal error"); // unreachable
         }
@@ -243,14 +241,14 @@ static void llama_tensor_dequantize_impl(
     }
 
     size_t block_size;
-    if (tensor->type == GGML_TYPE_F16 ||
-        tensor->type == GGML_TYPE_BF16) {
+    if (type == GGML_TYPE_F16 ||
+        type == GGML_TYPE_BF16) {
         block_size = 1;
     } else {
-        block_size = (size_t)ggml_blck_size(tensor->type);
+        block_size = (size_t)ggml_blck_size(type);
     }
 
-    size_t block_size_bytes = ggml_type_size(tensor->type);
+    size_t block_size_bytes = ggml_type_size(type);
 
     GGML_ASSERT(nelements % block_size == 0);
     size_t nblocks = nelements / block_size;
@@ -265,16 +263,16 @@ static void llama_tensor_dequantize_impl(
         size_t thr_elems = thr_blocks * block_size; // number of elements for this thread
         size_t thr_block_bytes = thr_blocks * block_size_bytes; // number of input bytes for this thread
 
-        auto compute = [qtype] (ggml_type typ, uint8_t * inbuf, float * outbuf, int nels) {
+        auto compute = [qtype] (ggml_type typ, const uint8_t * inbuf, float * outbuf, int nels) {
             if (typ == GGML_TYPE_F16) {
-                ggml_fp16_to_fp32_row((ggml_fp16_t *)inbuf, outbuf, nels);
+                ggml_fp16_to_fp32_row((const ggml_fp16_t *)inbuf, outbuf, nels);
             } else if (typ == GGML_TYPE_BF16) {
-                ggml_bf16_to_fp32_row((ggml_bf16_t *)inbuf, outbuf, nels);
+                ggml_bf16_to_fp32_row((const ggml_bf16_t *)inbuf, outbuf, nels);
             } else {
                 qtype->to_float(inbuf, outbuf, nels);
             }
         };
-        workers.emplace_back(compute, tensor->type, (uint8_t *) tensor->data + in_buff_offs, f32_output + out_buff_offs, thr_elems);
+        workers.emplace_back(compute, type, (const uint8_t *) data + in_buff_offs, f32_output + out_buff_offs, thr_elems);
         in_buff_offs += thr_block_bytes;
         out_buff_offs += thr_elems;
     }
@@ -1093,6 +1091,8 @@ static void llama_model_quantize_impl(const std::string & fname_inp, const std::
     std::vector<no_init<uint8_t>> work;
     std::vector<no_init<float>> f32_conv_buf;
 
+    const size_t max_buf_size = params->max_buf_size ? params->max_buf_size : LLAMA_QUANT_MAX_BUF_SIZE;
+
     int cur_split = -1;
     std::ofstream fout;
     auto close_ofstream = [&]() {
@@ -1143,15 +1143,13 @@ static void llama_model_quantize_impl(const std::string & fname_inp, const std::
 
         const size_t tensor_size = ggml_nbytes(tensor);
 
-        if (!params->dry_run) {
-            if (!ml.use_mmap) {
-                if (read_data.size() < tensor_size) {
-                    read_data.resize(tensor_size);
-                }
-                tensor->data = read_data.data();
+        // read a byte range of the current tensor
+        auto load_range = [&](size_t offs, size_t size) -> const void * {
+            if (!ml.use_mmap && read_data.size() < size) {
+                read_data.resize(size);
             }
-            ml.load_data_for(tensor);
-        }
+            return ml.load_data_range(weight, offs, size, read_data.data());
+        };
 
         LLAMA_LOG_INFO("[%4d/%4d] %-36s - [%s], type = %6s, ",
                ++idx, ml.n_tensors,
@@ -1166,7 +1164,6 @@ static void llama_model_quantize_impl(const std::string & fname_inp, const std::
         // in then there's nothing to do.
         bool quantize = cur_type != new_type;
 
-        void * new_data;
         size_t new_size;
 
         if (params->dry_run) {
@@ -1190,12 +1187,18 @@ static void llama_model_quantize_impl(const std::string & fname_inp, const std::
         } else {
             // no --dry-run, perform quantization
             if (!quantize) {
-                new_data = tensor->data;
                 new_size = tensor_size;
                 LLAMA_LOG_INFO("size = %8.3f MiB\n", tensor_size/1024.0/1024.0);
-            } else {
-                const int64_t nelements = ggml_nelements(tensor);
 
+                // copy in slabs of whole rows, so that each slab can be validated
+                const size_t row_size   = ggml_row_size(tensor->type, tensor->ne[0]);
+                const size_t slab_size  = std::max<size_t>(row_size, (max_buf_size/row_size)*row_size);
+
+                for (size_t offs = 0; offs < tensor_size; offs += slab_size) {
+                    const size_t size = std::min(slab_size, tensor_size - offs);
+                    fout.write((const char *) load_range(offs, size), size);
+                }
+            } else {
                 const float * imatrix = nullptr;
                 if (imatrix_data) {
                     auto it = imatrix_data->find(tm.remapped_imatrix_name);
@@ -1227,43 +1230,60 @@ static void llama_model_quantize_impl(const std::string & fname_inp, const std::
                     throw std::runtime_error(format("Missing importance matrix for tensor %s in a very low-bit quantization", tensor->name));
                 }
 
-                float * f32_data;
-
-                if (tensor->type == GGML_TYPE_F32) {
-                    f32_data = (float *) tensor->data;
-                } else if (ggml_is_quantized(tensor->type) && !params->allow_requantize) {
+                if (ggml_is_quantized(tensor->type) && !params->allow_requantize) {
                     throw std::runtime_error(format("requantizing from type %s is disabled", ggml_type_name(tensor->type)));
-                } else {
-                    llama_tensor_dequantize_impl(tensor, f32_conv_buf, workers, nelements, nthread);
-                    f32_data = (float *) f32_conv_buf.data();
                 }
 
                 LLAMA_LOG_INFO("converting to %s .. ", ggml_type_name(new_type));
                 fflush(stdout);
 
-                if (work.size() < (size_t)nelements * 4) {
-                    work.resize(nelements * 4); // upper bound on size
-                }
-                new_data = work.data();
-
                 const int64_t n_per_row = tensor->ne[0];
                 const int64_t nrows = tensor->ne[1];
 
+                const size_t row_size_src = ggml_row_size(tensor->type, n_per_row);
+                const size_t row_size_dst = ggml_row_size(new_type,     n_per_row);
+
+                // process the rows in slabs, so that the buffers stay below max_buf_size
+                const size_t bytes_per_row = row_size_src + row_size_dst + (tensor->type == GGML_TYPE_F32 ? 0 : n_per_row*sizeof(float));
+                const int64_t nrows_slab = std::max<int64_t>(1, std::min<int64_t>(nrows, max_buf_size/bytes_per_row));
+
                 static const int64_t min_chunk_size = 32 * 512;
                 const int64_t chunk_size = (n_per_row >= min_chunk_size ? n_per_row : n_per_row * ((min_chunk_size + n_per_row - 1)/n_per_row));
 
-                const int64_t nelements_matrix = tensor->ne[0] * tensor->ne[1];
-                const int64_t nchunk = (nelements_matrix + chunk_size - 1)/chunk_size;
-                const int64_t nthread_use = nthread > 1 ? std::max((int64_t)1, std::min((int64_t)nthread, nchunk)) : 1;
-
                 // quantize each expert separately since they have different importance matrices
                 new_size = 0;
                 for (int64_t i03 = 0; i03 < tensor->ne[2]; ++i03) {
-                    const float * f32_data_03 = f32_data + i03 * nelements_matrix;
-                    void * new_data_03 = (char *)new_data + ggml_row_size(new_type, n_per_row) * i03 * nrows;
                     const float * imatrix_03 = imatrix ? imatrix + i03 * n_per_row : nullptr;
 
-                    new_size += llama_tensor_quantize_impl(new_type, f32_data_03, new_data_03, chunk_size, nrows, n_per_row, imatrix_03, workers, nthread_use);
+                    for (int64_t ir = 0; ir < nrows; ir += nrows_slab) {
+                        const int64_t nrows_cur = std::min(nrows_slab, nrows - ir);
+                        const int64_t nelements_cur = nrows_cur * n_per_row;
+
+                        const void * src = load_range((i03*nrows + ir)*row_size_src, nrows_cur*row_size_src);
+
+                        const float * f32_data;
+                        if (tensor->type == GGML_TYPE_F32) {
+                            f32_data = (const float *) src;
+                        } else {
+                            if (f32_conv_buf.size() < (size_t) nelements_cur) {
+                                f32_conv_buf.resize(nelements_cur);
+                            }
+                            llama_tensor_dequantize_impl(tensor->type, src, (float *) f32_conv_buf.data(), workers, nelements_cur, nthread);
+                            f32_data = (const float *) f32_conv_buf.data();
+                        }
+
+                        if (work.size() < nrows_cur*row_size_dst) {
+                            work.resize(nrows_cur*row_size_dst);
+                        }
+
+                        const int64_t nchunk = (nelements_cur + chunk_size - 1)/chunk_size;
+                        const int64_t nthread_use = nthread > 1 ? std::max((int64_t)1, std::min((int64_t)nthread, nchunk)) : 1;
+
+                        const size_t size_cur = llama_tensor_quantize_impl(new_type, f32_data, work.data(), chunk_size, nrows_cur, n_per_row, imatrix_03, workers, nthread_use);
+
+                        fout.write((const char *) work.data(), size_cur);
+                        new_size += size_cur;
+                    }
                 }
                 LLAMA_LOG_INFO("size = %8.2f MiB -> %8.2f MiB\n", tensor_size/1024.0/1024.0, new_size/1024.0/1024.0);
             }
@@ -1273,10 +1293,8 @@ static void llama_model_quantize_impl(const std::string & fname_inp, const std::
             // update the gguf metadata as we go
             gguf_set_tensor_type(ctx_outs[cur_split].get(), metadata[i].name.c_str(), new_type);
             GGML_ASSERT(gguf_get_tensor_size(ctx_outs[cur_split].get(), gguf_find_tensor(ctx_outs[cur_split].get(), metadata[i].name.c_str())) == new_size);
-            gguf_set_tensor_data(ctx_outs[cur_split].get(), metadata[i].name.c_str(), new_data);
 
-            // write tensor data + padding
-            fout.write((const char *) new_data, new_size);
+            // tensor data is already written, add the padding
             zeros(fout, GGML_PAD(new_size, align) - new_size);
 
             // unmap the tensor to free memory
@@ -1323,7 +1341,8 @@ llama_model_quantize_params llama_model_quantize_default_params() {
         /*.imatrix                     =*/ nullptr,
         /*.kv_overrides                =*/ nullptr,
         /*.tensor_type                 =*/ nullptr,
-        /*.prune_layers                =*/ nullptr
+        /*.prune_layers                =*/ nullptr,
+        /*.max_buf_size                =*/ LLAMA_QUANT_MAX_BUF_SIZE
     };
 
     return result;

src/llama.cpp

@@ -318,6 +318,8 @@ static std::pair<int, llama_model *> llama_model_load(struct gguf_context * meta
         llama_model_loader ml(metadata, set_tensor_data, set_tensor_data_ud, fname, splits, file, params.load_mode,
             params.check_tensors, params.no_alloc, params.load_mtp, params.kv_overrides, params.tensor_buft_overrides);
 
+        ml.tensor_read_lazy = params.tensor_read_lazy;
+
         ml.print_info();
         std::unique_ptr<llama_model> model_ptr(llama_model_create(ml, params));
 

src/models/gemma4.cpp

@@ -50,7 +50,7 @@ void llama_model_gemma4::load_arch_tensors(llama_model_loader &) {
     tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0);
 
     if (n_embd_per_layer > 0) {
-        per_layer_tok_embd   = create_tensor(tn(LLM_TENSOR_PER_LAYER_TOKEN_EMBD, "weight"),    {n_embd_per_layer * n_layer, n_vocab}, 0);
+        per_layer_tok_embd   = create_tensor(tn(LLM_TENSOR_PER_LAYER_TOKEN_EMBD, "weight"),    {n_embd_per_layer * n_layer, n_vocab}, TENSOR_READ_LAZY);
         per_layer_model_proj = create_tensor(tn(LLM_TENSOR_PER_LAYER_MODEL_PROJ, "weight", 0), {n_embd, n_embd_per_layer * n_layer}, 0);
         per_layer_proj_norm  = create_tensor(tn(LLM_TENSOR_PER_LAYER_PROJ_NORM,  "weight", 0), {n_embd_per_layer}, 0);
     }

src/models/minimax-01.cpp

@@ -174,11 +174,9 @@ public:
     bool can_reuse(const llm_graph_params & params) override {
         bool res = true;
 
-        if (params.ubatch.n_seq_tokens > 1) {
-            res &= (   inp_q_decay &&    inp_q_decay->ne[2] == params.ubatch.n_seq_tokens);
-            res &= (   inp_k_decay &&    inp_k_decay->ne[2] == params.ubatch.n_seq_tokens);
-            res &= (inp_diag_decay && inp_diag_decay->ne[1] == params.ubatch.n_seq_tokens);
-        }
+        res &= (   inp_q_decay &&    inp_q_decay->ne[2] == params.ubatch.n_seq_tokens);
+        res &= (   inp_k_decay &&    inp_k_decay->ne[2] == params.ubatch.n_seq_tokens);
+        res &= (inp_diag_decay && inp_diag_decay->ne[1] == params.ubatch.n_seq_tokens);
 
         return res;
     }
@@ -223,19 +221,17 @@ llama_model_minimax_01::graph::graph(const llama_model & model, const llm_graph_
     ggml_set_input(inp->inp_slopes);
     cb(inp->inp_slopes, "slopes", -1);
 
-    if (n_seq_tokens != 1) {
-        inp->inp_q_decay = ggml_new_tensor_4d(ctx0, GGML_TYPE_F32, 1, n_head, n_seq_tokens, n_seqs);
-        ggml_set_input(inp->inp_q_decay);
-        cb(inp->inp_q_decay, "q_decay_exp", -1);
+    inp->inp_q_decay = ggml_new_tensor_4d(ctx0, GGML_TYPE_F32, 1, n_head, n_seq_tokens, n_seqs);
+    ggml_set_input(inp->inp_q_decay);
+    cb(inp->inp_q_decay, "q_decay_exp", -1);
 
-        inp->inp_k_decay = ggml_new_tensor_4d(ctx0, GGML_TYPE_F32, 1, n_head, n_seq_tokens, n_seqs);
-        ggml_set_input(inp->inp_k_decay);
-        cb(inp->inp_k_decay, "k_decay_exp", -1);
+    inp->inp_k_decay = ggml_new_tensor_4d(ctx0, GGML_TYPE_F32, 1, n_head, n_seq_tokens, n_seqs);
+    ggml_set_input(inp->inp_k_decay);
+    cb(inp->inp_k_decay, "k_decay_exp", -1);
 
-        inp->inp_diag_decay = ggml_new_tensor_4d(ctx0, GGML_TYPE_F32, n_seq_tokens, n_seq_tokens, n_head, n_seqs);
-        ggml_set_input(inp->inp_diag_decay);
-        cb(inp->inp_diag_decay, "diag_decay_exp", -1);
-    }
+    inp->inp_diag_decay = ggml_new_tensor_4d(ctx0, GGML_TYPE_F32, n_seq_tokens, n_seq_tokens, n_head, n_seqs);
+    ggml_set_input(inp->inp_diag_decay);
+    cb(inp->inp_diag_decay, "diag_decay_exp", -1);
 
     la = (llm_graph_input_la *) res->add_input(std::move(inp));
 
@@ -319,41 +315,8 @@ llama_model_minimax_01::graph::graph(const llama_model & model, const llm_graph_
 
             ggml_tensor * qkv = nullptr;
             ggml_tensor * kv_new = nullptr;
-
-            if (n_seq_tokens == 1) {
-                // lightning attention - optimized single token case for TG
-
-                ggml_tensor * slopes_neg = ggml_scale(ctx0, slope_rate, -1.0);
-                cb(slopes_neg, "slopes_neg", il);
-
-                ggml_tensor * ratio = ggml_exp(ctx0, slopes_neg);
-                cb(ratio, "ratio", il);
-
-                ggml_tensor * ratio_3d = ggml_reshape_3d(ctx0, ratio, 1, 1, n_head);
-                cb(ratio_3d, "ratio3d", il);
-
-                ggml_tensor * v_trans = ggml_cont(ctx0, ggml_permute(ctx0, Vcur, 1, 2, 0, 3));
-                cb(v_trans, "v_trans", il);
-
-                ggml_tensor * k_trans = ggml_cont(ctx0, ggml_permute(ctx0, Kcur, 1, 2, 0, 3));
-                cb(k_trans, "k_trans", il);
-
-                ggml_tensor * kv_cur = ggml_mul_mat(ctx0, k_trans, v_trans);
-                cb(kv_cur, "kv_cur", il);
-
-                ggml_tensor * kv_old_s = ggml_mul(ctx0, kv_old, ratio_3d);
-                cb(kv_old_s, "kv_old_s", il);
-
-                kv_new = ggml_add(ctx0, kv_old_s, kv_cur);
-                cb(kv_new, "kv_new", il);
-
-                ggml_tensor * q_trans = ggml_permute(ctx0, Qcur, 0, 2, 1, 3);
-                cb(q_trans, "q_trans", il);
-
-                qkv = ggml_mul_mat(ctx0, kv_new, q_trans);
-                cb(qkv, "qkv", il);
-            } else if(n_seq_tokens > 1) {
-                // lightning attention - general multi token case for PP
+            {
+                // lightning attention
 
                 ggml_tensor *    q_decay_exp = la->inp_q_decay;
                 ggml_tensor *    k_decay_exp = la->inp_k_decay;

src/models/nanbeige.cpp

@@ -103,6 +103,7 @@ llama_model_nanbeige::graph::graph(const llama_model & model, const llm_graph_pa
     ggml_tensor * inp_out_ids = build_inp_out_ids();
 
     for (int il = 0; il < n_layer; ++il) {
+        res->t_layer_inp[il] = inpL;
         ggml_tensor * inpSA = inpL;
 
         cur = build_norm(inpL, model.layers[il].attn_norm, NULL, LLM_NORM_RMS, il);

tests/test-arg-parser.cpp

@@ -4,6 +4,7 @@
 #include "llama.h"
 #include "speculative.h"
 
+#include <cmath>
 #include <limits>
 #include <string>
 #include <vector>
@@ -34,6 +35,62 @@ static void test(void) {
             std::numeric_limits<int32_t>::max(),
             std::numeric_limits<int32_t>::max());
 
+    {
+        common_params_speculative spec;
+        spec.synth_len = 3.4;
+
+        auto assert_invalid = [](const common_params_speculative & value, int32_t n_max) {
+            try {
+                common_speculative_synth_rates_resolve(&value, n_max);
+                assert(false);
+            } catch (const std::invalid_argument &) {
+            }
+        };
+
+        const auto rates = common_speculative_synth_rates_resolve(&spec, 4);
+        assert(rates.size() == 4);
+        assert(std::abs(rates[0] - 0.80581) < 1e-5);
+        assert(std::abs(rates[1] - 0.64933) < 1e-5);
+        assert(std::abs(rates[2] - 0.52323) < 1e-5);
+        assert(std::abs(rates[3] - 0.42163) < 1e-5);
+        assert(std::abs(1.0 + rates[0] + rates[1] + rates[2] + rates[3] - 3.4) < 1e-8);
+
+        spec.synth_len = 1.0;
+        assert(common_speculative_synth_rates_resolve(&spec, 4) == std::vector<double>({0.0, 0.0, 0.0, 0.0}));
+
+        spec.synth_len = 5.0;
+        assert(common_speculative_synth_rates_resolve(&spec, 4) == std::vector<double>({1.0, 1.0, 1.0, 1.0}));
+
+        spec.synth_len = 5.1;
+        assert_invalid(spec, 4);
+
+        spec.synth_len = std::numeric_limits<double>::quiet_NaN();
+        assert_invalid(spec, 4);
+
+        spec.synth_len = 0.0;
+        assert_invalid(spec, 4);
+
+        spec.synth_len = -1.0;
+        spec.synth_rates = {0.8, 0.6, 0.4};
+        assert_invalid(spec, 4);
+
+        spec.synth_rates = {0.8, 0.6, 0.4, 0.2};
+        assert(common_speculative_synth_rates_resolve(&spec, 4) == spec.synth_rates);
+
+        spec.synth_rates = {0.8, 0.9, 0.4, 0.2};
+        assert_invalid(spec, 4);
+
+        spec.synth_rates = {0.8, std::numeric_limits<double>::quiet_NaN(), 0.4, 0.2};
+        assert_invalid(spec, 4);
+
+        spec.synth_rates = {0.8, 0.6, 0.4, -0.2};
+        assert_invalid(spec, 4);
+
+        spec.synth_rates = {0.8, 0.6, 0.4, 0.2};
+        spec.synth_len = 3.0;
+        assert_invalid(spec, 4);
+    }
+
     {
         common_params base;
         base.n_parallel = 4;
@@ -197,6 +254,26 @@ static void test(void) {
     assert(true == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_SPECULATIVE));
     assert(params.speculative.draft.n_max == 123);
 
+    {
+        common_params synth_params;
+        argv = {"binary_name", "--spec-synth-len", "3.4"};
+        assert(true == common_params_parse(argv.size(), list_str_to_char(argv).data(), synth_params, LLAMA_EXAMPLE_SERVER));
+        assert(synth_params.speculative.synth_len == 3.4);
+    }
+
+    {
+        common_params synth_params;
+        argv = {"binary_name", "--spec-synth-rates", "0.8,0.6,0.2"};
+        assert(true == common_params_parse(argv.size(), list_str_to_char(argv).data(), synth_params, LLAMA_EXAMPLE_SERVER));
+        assert(synth_params.speculative.synth_rates == std::vector<double>({0.8, 0.6, 0.2}));
+    }
+
+    {
+        common_params synth_params;
+        argv = {"binary_name", "--spec-synth-len", "3.4x"};
+        assert(false == common_params_parse(argv.size(), list_str_to_char(argv).data(), synth_params, LLAMA_EXAMPLE_SERVER));
+    }
+
     argv = {"binary_name", "-lm", "none"};
     assert(true == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_COMMON));
     assert(params.load_mode == LLAMA_LOAD_MODE_NONE);

tools/cli/README.md

@@ -59,12 +59,14 @@
 | `--mmap, --no-mmap` | DEPRECATED in favor of `--load-mode`: whether to memory-map model. (if mmap disabled, slower load but may reduce pageouts if not using mlock)<br/>(env: LLAMA_ARG_MMAP) |
 | `-dio, --direct-io, -ndio, --no-direct-io` | DEPRECATED in favor of `--load-mode`: use DirectIO if available<br/>(env: LLAMA_ARG_DIO) |
 | `-lm, --load-mode MODE` | model loading mode (default: auto)<br/>- auto: mmap, unless a device does not support it<br/>- none: no special loading mode<br/>- mmap: memory-map model (if mmap disabled, slower load but may reduce pageouts if not using mlock)<br/>- mlock: force system to keep model in RAM rather than swapping or compressing<br/>- mmap+mlock: mmap + force system to keep model in RAM rather than swapping or compressing<br/>- dio: use DirectIO if available<br/><br/>(env: LLAMA_ARG_LOAD_MODE) |
+| `--tensor-read-lazy MODE` | on-demand reading of certain tensors, for example per-layer embeddings (default: auto)<br/>- on: read the rows of such tensors from disk on demand instead of keeping them resident (requires mmap)<br/>- auto: on, but only for tensors larger than 4 GiB<br/>- off: always keep them resident<br/>(env: LLAMA_ARG_TENSOR_READ_LAZY) |
 | `--numa TYPE` | attempt optimizations that help on some NUMA systems<br/>- distribute: spread execution evenly over all nodes<br/>- isolate: only spawn threads on CPUs on the node that execution started on<br/>- numactl: use the CPU map provided by numactl<br/>if run without this previously, it is recommended to drop the system page cache before using this<br/>see https://github.com/ggml-org/llama.cpp/issues/1437<br/>(env: LLAMA_ARG_NUMA) |
 | `-dev, --device <dev1,dev2,..>` | comma-separated list of devices to use for offloading (none = don't offload)<br/>use --list-devices to see a list of available devices<br/>(env: LLAMA_ARG_DEVICE) |
 | `--list-devices` | print list of available devices and exit |
 | `-ot, --override-tensor <tensor name pattern>=<buffer type>,...` | override tensor buffer type<br/>(env: LLAMA_ARG_OVERRIDE_TENSOR) |
 | `-cmoe, --cpu-moe` | keep all Mixture of Experts (MoE) weights in the CPU<br/>(env: LLAMA_ARG_CPU_MOE) |
 | `-ncmoe, --n-cpu-moe N` | keep the Mixture of Experts (MoE) weights of the first N layers in the CPU<br/>(env: LLAMA_ARG_N_CPU_MOE) |
+| `-ncffn, --n-cpu-ffn N` | keep the dense FFN weights of the first N layers in the CPU<br/>(dense models; for MoE expert weights use --n-cpu-moe)<br/>(env: LLAMA_ARG_N_CPU_FFN) |
 | `-ngl, --gpu-layers, --n-gpu-layers N` | max. number of layers to store in VRAM, either an exact number, 'auto', or 'all' (default: auto)<br/>(env: LLAMA_ARG_N_GPU_LAYERS) |
 | `-sm, --split-mode {none,layer,row,tensor}` | how to split the model across multiple GPUs, one of:<br/>- none: use one GPU only<br/>- layer (default): split layers and KV across GPUs (pipelined)<br/>- row: split weight across GPUs by rows (parallelized)<br/>- tensor: split weights and KV across GPUs (parallelized, EXPERIMENTAL)<br/>(env: LLAMA_ARG_SPLIT_MODE) |
 | `-ts, --tensor-split N0,N1,N2,...` | fraction of the model to offload to each GPU, comma-separated list of proportions, e.g. 3,1<br/>(env: LLAMA_ARG_TENSOR_SPLIT) |
@@ -154,7 +156,6 @@
 | `-sysf, --system-prompt-file FNAME` | a file containing the system prompt (default: none) |
 | `-r, --reverse-prompt PROMPT` | halt generation at PROMPT, return control in interactive mode |
 | `-sp, --special` | special tokens output enabled (default: false) |
-| `-cnv, --conversation, -no-cnv, --no-conversation` | whether to run in conversation mode:<br/>- does not print special tokens and suffix/prefix<br/>- interactive mode is also enabled<br/>(default: auto enabled if chat template is available) |
 | `-st, --single-turn` | run conversation for a single turn only, then exit when done<br/>will not be interactive if first turn is predefined with --prompt<br/>(default: false) |
 | `-mli, --multiline-input` | allows you to write or paste multiple lines without ending each in '\' |
 | `--warmup, --no-warmup` | whether to perform warmup with an empty run (default: enabled) |
@@ -166,6 +167,9 @@
 | `--image, --audio, --video FILE` | path to an image, audio, or video file. use with multimodal models, use comma-separated values for multiple files |
 | `--image-min-tokens N` | minimum number of tokens each image can take, only used by vision models with dynamic resolution (default: read from model)<br/>(env: LLAMA_ARG_IMAGE_MIN_TOKENS) |
 | `--image-max-tokens N` | maximum number of tokens each image can take, only used by vision models with dynamic resolution (default: read from model)<br/>(env: LLAMA_ARG_IMAGE_MAX_TOKENS) |
+| `--video-fps N` | target video frame rate (default: 4.0)<br/>(env: LLAMA_ARG_VIDEO_FPS) |
+| `--video-timestamp-interval N` | interval in milliseconds between text timestamps (default: 5000)<br/>(env: LLAMA_ARG_VIDEO_TIMESTAMP_INTERVAL) |
+| `--video-ffmpeg-dir DIR` | path to the directory containing ffmpeg and ffprobe (default: search in PATH)<br/>(env: LLAMA_ARG_VIDEO_FFMPEG_DIR) |
 | `-o, --output, --output-file FNAME` | output file (default: '') |
 | `--chat-template-kwargs STRING` | sets additional params for the json template parser, must be a valid json object string, e.g. '{"key1":"value1","key2":"value2"}'<br/>(env: LLAMA_ARG_CHAT_TEMPLATE_KWARGS) |
 | `--jinja, --no-jinja` | whether to use jinja template engine for chat (default: enabled)<br/>(env: LLAMA_ARG_JINJA) |
@@ -197,6 +201,8 @@
 | `--spec-draft-n-cpu-moe, --spec-draft-ncmoe, -ncmoed, --n-cpu-moe-draft N` | keep the Mixture of Experts (MoE) weights of the first N layers in the CPU for the draft model<br/>(env: LLAMA_ARG_SPEC_DRAFT_N_CPU_MOE) |
 | `--spec-draft-n-max N` | number of tokens to draft for speculative decoding (default: 3)<br/>(env: LLAMA_ARG_SPEC_DRAFT_N_MAX) |
 | `--spec-draft-n-min N` | minimum number of draft tokens to use for speculative decoding (default: 0)<br/>(env: LLAMA_ARG_SPEC_DRAFT_N_MIN) |
+| `--spec-synth-len L` | target mean synthetic acceptance length, including the target token (benchmarking only)<br/>(env: LLAMA_ARG_SPEC_SYNTH_LEN) |
+| `--spec-synth-rates P0,P1,...` | comma-separated unconditional per-position synthetic acceptance probabilities (benchmarking only)<br/>(env: LLAMA_ARG_SPEC_SYNTH_RATES) |
 | `--spec-draft-p-split, --draft-p-split P` | speculative decoding split probability (default: 0.10)<br/>(env: LLAMA_ARG_SPEC_DRAFT_P_SPLIT) |
 | `--spec-draft-p-min, --draft-p-min P` | minimum speculative decoding probability (greedy) (default: 0.00)<br/>(env: LLAMA_ARG_SPEC_DRAFT_P_MIN) |
 | `--spec-draft-backend-sampling, --no-spec-draft-backend-sampling` | offload draft sampling to the backend (default: enabled)<br/>(env: LLAMA_ARG_SPEC_DRAFT_BACKEND_SAMPLING) |

tools/completion/README.md

@@ -142,12 +142,14 @@ llama-completion.exe -m models\gemma-1.1-7b-it.Q4_K_M.gguf --ignore-eos -n -1
 | `--mmap, --no-mmap` | DEPRECATED in favor of `--load-mode`: whether to memory-map model. (if mmap disabled, slower load but may reduce pageouts if not using mlock)<br/>(env: LLAMA_ARG_MMAP) |
 | `-dio, --direct-io, -ndio, --no-direct-io` | DEPRECATED in favor of `--load-mode`: use DirectIO if available<br/>(env: LLAMA_ARG_DIO) |
 | `-lm, --load-mode MODE` | model loading mode (default: auto)<br/>- auto: mmap, unless a device does not support it<br/>- none: no special loading mode<br/>- mmap: memory-map model (if mmap disabled, slower load but may reduce pageouts if not using mlock)<br/>- mlock: force system to keep model in RAM rather than swapping or compressing<br/>- mmap+mlock: mmap + force system to keep model in RAM rather than swapping or compressing<br/>- dio: use DirectIO if available<br/><br/>(env: LLAMA_ARG_LOAD_MODE) |
+| `--tensor-read-lazy MODE` | on-demand reading of certain tensors, for example per-layer embeddings (default: auto)<br/>- on: read the rows of such tensors from disk on demand instead of keeping them resident (requires mmap)<br/>- auto: on, but only for tensors larger than 4 GiB<br/>- off: always keep them resident<br/>(env: LLAMA_ARG_TENSOR_READ_LAZY) |
 | `--numa TYPE` | attempt optimizations that help on some NUMA systems<br/>- distribute: spread execution evenly over all nodes<br/>- isolate: only spawn threads on CPUs on the node that execution started on<br/>- numactl: use the CPU map provided by numactl<br/>if run without this previously, it is recommended to drop the system page cache before using this<br/>see https://github.com/ggml-org/llama.cpp/issues/1437<br/>(env: LLAMA_ARG_NUMA) |
 | `-dev, --device <dev1,dev2,..>` | comma-separated list of devices to use for offloading (none = don't offload)<br/>use --list-devices to see a list of available devices<br/>(env: LLAMA_ARG_DEVICE) |
 | `--list-devices` | print list of available devices and exit |
 | `-ot, --override-tensor <tensor name pattern>=<buffer type>,...` | override tensor buffer type<br/>(env: LLAMA_ARG_OVERRIDE_TENSOR) |
 | `-cmoe, --cpu-moe` | keep all Mixture of Experts (MoE) weights in the CPU<br/>(env: LLAMA_ARG_CPU_MOE) |
 | `-ncmoe, --n-cpu-moe N` | keep the Mixture of Experts (MoE) weights of the first N layers in the CPU<br/>(env: LLAMA_ARG_N_CPU_MOE) |
+| `-ncffn, --n-cpu-ffn N` | keep the dense FFN weights of the first N layers in the CPU<br/>(dense models; for MoE expert weights use --n-cpu-moe)<br/>(env: LLAMA_ARG_N_CPU_FFN) |
 | `-ngl, --gpu-layers, --n-gpu-layers N` | max. number of layers to store in VRAM, either an exact number, 'auto', or 'all' (default: auto)<br/>(env: LLAMA_ARG_N_GPU_LAYERS) |
 | `-sm, --split-mode {none,layer,row,tensor}` | how to split the model across multiple GPUs, one of:<br/>- none: use one GPU only<br/>- layer (default): split layers and KV across GPUs (pipelined)<br/>- row: split weight across GPUs by rows (parallelized)<br/>- tensor: split weights and KV across GPUs (parallelized, EXPERIMENTAL)<br/>(env: LLAMA_ARG_SPLIT_MODE) |
 | `-ts, --tensor-split N0,N1,N2,...` | fraction of the model to offload to each GPU, comma-separated list of proportions, e.g. 3,1<br/>(env: LLAMA_ARG_TENSOR_SPLIT) |

tools/llama-bench/llama-bench.cpp

@@ -1254,7 +1254,7 @@ struct cmd_params_instance {
             merged.reserve(merged.size() + (size_t) n_cpu_moe + 1);
 
             for (int i = 0; i < n_cpu_moe; ++i) {
-                patterns.push_back(llm_ffn_exps_block_regex(i));
+                patterns.push_back(llm_ffn_block_regex(i, LLM_FFN_EXPS_REGEX));
                 merged.push_back({ patterns.back().c_str(),
                                 ggml_backend_cpu_buffer_type() });
             }

tools/mtmd/mtmd-cli.cpp

@@ -87,6 +87,9 @@ struct mtmd_cli_context {
     mtmd::bitmaps bitmaps;
     std::vector<mtmd_helper::video_ptr> videos;
 
+    mtmd_helper_init_opt init_opt = mtmd_helper_init_opt_default();
+    std::string video_ffmpeg_bin_dir;
+
     mtmd::batch_ptr mbatch;
 
     // chat template
@@ -170,6 +173,12 @@ struct mtmd_cli_context {
             LOG_ERR("Failed to load vision model from %s\n", clip_path);
             exit(1);
         }
+
+        video_ffmpeg_bin_dir = params.video_ffmpeg_bin_dir;
+        init_opt.video_params.fps_target = params.video_fps;
+        init_opt.video_params.timestamp_interval_ms = params.video_timestamp_interval_ms;
+        init_opt.video_params.ffmpeg_bin_dir = video_ffmpeg_bin_dir.empty()
+                            ? nullptr : video_ffmpeg_bin_dir.c_str();
     }
 
     bool check_antiprompt(const llama_tokens & generated_tokens) {
@@ -184,7 +193,7 @@ struct mtmd_cli_context {
     }
 
     bool load_media(const std::string & fname) {
-        auto res = mtmd_helper_bitmap_init_from_file(ctx_vision.get(), fname.c_str(), false);
+        auto res = mtmd_helper_bitmap_init_from_file(ctx_vision.get(), fname.c_str(), false, init_opt);
         if (!res.bitmap) {
             return false;
         }

tools/mtmd/mtmd-helper.cpp

@@ -369,14 +369,18 @@ static bool is_webp_file(const unsigned char * buf, size_t len) {
 }
 
 #ifdef MTMD_VIDEO
-static mtmd_bitmap * decode_webp_with_ffmpeg(mtmd_context * mctx, const unsigned char * buf, size_t len, bool placeholder);
+static mtmd_bitmap * decode_webp_with_ffmpeg(mtmd_context * mctx, const unsigned char * buf, size_t len, bool placeholder,
+                                             const mtmd_helper_video_init_params & params);
 #endif
 
-mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_buf(mtmd_context * ctx, const unsigned char * buf, size_t len, bool placeholder) {
+mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_buf(mtmd_context * ctx, const unsigned char * buf, size_t len, bool placeholder,
+                                                            mtmd_helper_init_opt opt) {
     // calculate the hash if needed
     std::string id;
     mtmd_bitmap * result = nullptr;
 
+    GGML_UNUSED(opt); // only used by video code paths
+
     if (!placeholder) {
         // use sha256 to prevent cache poisoning
         id = hash_sha256_hex(buf, len);
@@ -414,7 +418,7 @@ mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_buf(mtmd_context * ctx,
 #ifdef MTMD_VIDEO
     // stb_image does not support webp; decode it with ffmpeg as a single frame
     if (!result && is_webp_file(buf, len)) {
-        result = decode_webp_with_ffmpeg(ctx, buf, len, placeholder);
+        result = decode_webp_with_ffmpeg(ctx, buf, len, placeholder, opt.video_params);
         if (!result) {
             LOG_ERR("%s: failed to decode webp buffer\n", __func__);
             return {nullptr, nullptr};
@@ -427,8 +431,7 @@ mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_buf(mtmd_context * ctx,
     // last try: load as video
 #ifdef MTMD_VIDEO
     if (!result) {
-        auto params = mtmd_helper_video_init_params_default();
-        auto video_ctx = mtmd_helper_video_init_from_buf(ctx, buf, len, params);
+        auto video_ctx = mtmd_helper_video_init_from_buf(ctx, buf, len, opt.video_params);
         if (!video_ctx) {
             LOG_ERR("%s: failed to decode buffer as either image/audio/video\n", __func__);
             return {nullptr, nullptr};
@@ -456,7 +459,8 @@ mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_buf(mtmd_context * ctx,
     return {nullptr, nullptr};
 }
 
-mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_file(mtmd_context * ctx, const char * fname, bool placeholder) {
+mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_file(mtmd_context * ctx, const char * fname, bool placeholder,
+                                                             mtmd_helper_init_opt opt) {
 #ifdef _WIN32
     int wlen = MultiByteToWideChar(CP_UTF8, 0, fname, -1, NULL, 0);
     if (!wlen) {
@@ -497,7 +501,7 @@ mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_file(mtmd_context * ctx,
         return {nullptr, nullptr};
     }
 
-    return mtmd_helper_bitmap_init_from_buf(ctx, buf.data(), buf.size(), placeholder);
+    return mtmd_helper_bitmap_init_from_buf(ctx, buf.data(), buf.size(), placeholder, opt);
 }
 
 bool mtmd_helper_support_video(mtmd_context * ctx) {
@@ -855,6 +859,12 @@ mtmd_helper_video_init_params mtmd_helper_video_init_params_default() {
     };
 }
 
+mtmd_helper_init_opt mtmd_helper_init_opt_default() {
+    return {
+        /* video_params */ mtmd_helper_video_init_params_default(),
+    };
+}
+
 static std::string video_resolve_bin(const char * bin_dir, const char * name) {
     if (!bin_dir || bin_dir[0] == '\0') {
         return name; // rely on PATH
@@ -876,8 +886,8 @@ static std::string video_resolve_bin(const char * bin_dir, const char * name) {
 }
 
 #ifdef MTMD_VIDEO
-static mtmd_bitmap * decode_webp_with_ffmpeg(mtmd_context * mctx, const unsigned char * buf, size_t len, bool placeholder) {
-    auto params = mtmd_helper_video_init_params_default();
+static mtmd_bitmap * decode_webp_with_ffmpeg(mtmd_context * mctx, const unsigned char * buf, size_t len, bool placeholder,
+                                             const mtmd_helper_video_init_params & params) {
     mtmd_helper_video vctx;
     vctx.mctx        = mctx;
     vctx.input_buf.assign(buf, buf + len);

tools/mtmd/mtmd-helper.h

@@ -23,6 +23,23 @@ extern "C" {
 struct mtmd_helper_video;
 typedef struct mtmd_helper_video mtmd_helper_video;
 
+struct mtmd_helper_video_init_params {
+    float fps_target;            // desired output fps; <= 0 means use the video's native fps, defaulted to 4.0f
+    const char * ffmpeg_bin_dir; // directory containing ffmpeg/ffprobe binaries; NULL means search PATH
+    int64_t timestamp_interval_ms; // interval for adding timestamp as text chunk (example: "[10m50.5s]"); <= 0 means no timestamp, defaulted to 5000ms
+    // TODO @ngxson : allow "placeholder" bitmap output for counting tokens
+};
+
+MTMD_API struct mtmd_helper_video_init_params mtmd_helper_video_init_params_default(void);
+
+// opt for mtmd_helper_bitmap_init_from_*()
+struct mtmd_helper_init_opt {
+    struct mtmd_helper_video_init_params video_params;
+};
+typedef struct mtmd_helper_init_opt mtmd_helper_init_opt;
+
+MTMD_API struct mtmd_helper_init_opt mtmd_helper_init_opt_default(void);
+
 // Set callback for all future logging events.
 // If this is not called, or NULL is supplied, everything is output on stderr.
 // Note: this also call mtmd_log_set() internally
@@ -40,7 +57,11 @@ struct mtmd_helper_bitmap_wrapper {
 // it calls mtmd_helper_bitmap_init_from_buf() internally
 // returns nullptr on failure
 // this function is thread-safe
-MTMD_API struct mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_file(mtmd_context * ctx, const char * fname, bool placeholder);
+MTMD_API struct mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_file(
+                    mtmd_context * ctx,
+                    const char * fname,
+                    bool placeholder,
+                    struct mtmd_helper_init_opt opt);
 
 // helper function to construct a mtmd_bitmap from a buffer containing a file
 // supported formats:
@@ -53,7 +74,11 @@ MTMD_API struct mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_file(mtm
 //   - output bitmap will have SHA-256 hash (hex string) as the ID
 // returns nullptr on failure
 // this function is thread-safe
-MTMD_API struct mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_buf(mtmd_context * ctx, const unsigned char * buf, size_t len, bool placeholder);
+MTMD_API struct mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_buf(
+                    mtmd_context * ctx,
+                    const unsigned char * buf, size_t len,
+                    bool placeholder,
+                    struct mtmd_helper_init_opt opt);
 
 // helper to count the total number of tokens from a list of chunks, useful to keep track of KV cache
 MTMD_API size_t mtmd_helper_get_n_tokens(const mtmd_input_chunks * chunks);
@@ -124,14 +149,7 @@ struct mtmd_helper_video_info {
     int32_t  n_frames; // estimated total frames at effective fps (-1 if unknown)
 };
 
-struct mtmd_helper_video_init_params {
-    float fps_target;            // desired output fps; <= 0 means use the video's native fps, defaulted to 4.0f
-    const char * ffmpeg_bin_dir; // directory containing ffmpeg/ffprobe binaries; NULL means search PATH
-    int64_t timestamp_interval_ms; // interval for adding timestamp as text chunk (example: "[10m50.5s]"); <= 0 means no timestamp, defaulted to 5000ms
-    // TODO @ngxson : allow "placeholder" bitmap output for counting tokens
-};
-
-MTMD_API struct mtmd_helper_video_init_params mtmd_helper_video_init_params_default(void);
+// note: mtmd_helper_video_init_params is defined at the top, as it is part of mtmd_helper_init_opt
 
 // returns NULL on failure (ffprobe not found, file unreadable, etc.)
 MTMD_API mtmd_helper_video * mtmd_helper_video_init(

tools/quantize/quantize.cpp

@@ -122,7 +122,7 @@ static bool try_parse_ftype(const std::string & ftype_str_in, llama_ftype & ftyp
 static void usage(const char * executable) {
     printf("usage: %s [--help] [--allow-requantize] [--leave-output-tensor] [--pure] [--imatrix] [--include-weights]\n", executable);
     printf("       [--exclude-weights] [--output-tensor-type] [--token-embedding-type] [--tensor-type] [--tensor-type-file]\n");
-    printf("       [--prune-layers] [--keep-split] [--override-kv] [--dry-run]\n");
+    printf("       [--prune-layers] [--keep-split] [--override-kv] [--dry-run] [--max-buffer-size]\n");
     printf("       model-f32.gguf [model-quant.gguf] type [nthreads]\n\n");
     printf("  --allow-requantize\n");
     printf("                                      allow requantizing tensors that have already been quantized\n");
@@ -161,7 +161,10 @@ static void usage(const char * executable) {
     printf("                                      WARNING: this is an advanced option, use with care.\n");
     printf("  --dry-run\n");
     printf("                                      calculate and show the final quantization size without performing quantization\n");
-    printf("                                      example: llama-quantize --dry-run model-f32.gguf Q4_K\n\n");
+    printf("                                      example: llama-quantize --dry-run model-f32.gguf Q4_K\n");
+    printf("  --max-buffer-size MiB\n");
+    printf("                                      max amount of tensor rows kept in memory while quantizing one tensor (default: 8192)\n");
+    printf("                                      lower it to quantize models with very large tensors on a machine with little RAM\n\n");
     printf("note: --include-weights and --exclude-weights cannot be used together\n\n");
     printf("-----------------------------------------------------------------------------\n");
     printf(" allowed quantization types\n");
@@ -467,6 +470,16 @@ int llama_quantize(int argc, char ** argv) {
             }
         } else if (strcmp(argv[arg_idx], "--keep-split") == 0) {
             params.keep_split = true;
+        } else if (strcmp(argv[arg_idx], "--max-buffer-size") == 0) {
+            if (arg_idx == argc-1) {
+                usage(argv[0]);
+            }
+            const int mib = atoi(argv[++arg_idx]);
+            if (mib <= 0) {
+                fprintf(stderr, "%s: invalid --max-buffer-size '%s'\n", __func__, argv[arg_idx]);
+                return 1;
+            }
+            params.max_buf_size = (size_t) mib * 1024 * 1024;
         } else {
             usage(argv[0]);
         }

tools/server/README.md

@@ -76,12 +76,14 @@ For the full list of features, please refer to [server's changelog](https://gith
 | `--mmap, --no-mmap` | DEPRECATED in favor of `--load-mode`: whether to memory-map model. (if mmap disabled, slower load but may reduce pageouts if not using mlock)<br/>(env: LLAMA_ARG_MMAP) |
 | `-dio, --direct-io, -ndio, --no-direct-io` | DEPRECATED in favor of `--load-mode`: use DirectIO if available<br/>(env: LLAMA_ARG_DIO) |
 | `-lm, --load-mode MODE` | model loading mode (default: auto)<br/>- auto: mmap, unless a device does not support it<br/>- none: no special loading mode<br/>- mmap: memory-map model (if mmap disabled, slower load but may reduce pageouts if not using mlock)<br/>- mlock: force system to keep model in RAM rather than swapping or compressing<br/>- mmap+mlock: mmap + force system to keep model in RAM rather than swapping or compressing<br/>- dio: use DirectIO if available<br/><br/>(env: LLAMA_ARG_LOAD_MODE) |
+| `--tensor-read-lazy MODE` | on-demand reading of certain tensors, for example per-layer embeddings (default: auto)<br/>- on: read the rows of such tensors from disk on demand instead of keeping them resident (requires mmap)<br/>- auto: on, but only for tensors larger than 4 GiB<br/>- off: always keep them resident<br/>(env: LLAMA_ARG_TENSOR_READ_LAZY) |
 | `--numa TYPE` | attempt optimizations that help on some NUMA systems<br/>- distribute: spread execution evenly over all nodes<br/>- isolate: only spawn threads on CPUs on the node that execution started on<br/>- numactl: use the CPU map provided by numactl<br/>if run without this previously, it is recommended to drop the system page cache before using this<br/>see https://github.com/ggml-org/llama.cpp/issues/1437<br/>(env: LLAMA_ARG_NUMA) |
 | `-dev, --device <dev1,dev2,..>` | comma-separated list of devices to use for offloading (none = don't offload)<br/>use --list-devices to see a list of available devices<br/>(env: LLAMA_ARG_DEVICE) |
 | `--list-devices` | print list of available devices and exit |
 | `-ot, --override-tensor <tensor name pattern>=<buffer type>,...` | override tensor buffer type<br/>(env: LLAMA_ARG_OVERRIDE_TENSOR) |
 | `-cmoe, --cpu-moe` | keep all Mixture of Experts (MoE) weights in the CPU<br/>(env: LLAMA_ARG_CPU_MOE) |
 | `-ncmoe, --n-cpu-moe N` | keep the Mixture of Experts (MoE) weights of the first N layers in the CPU<br/>(env: LLAMA_ARG_N_CPU_MOE) |
+| `-ncffn, --n-cpu-ffn N` | keep the dense FFN weights of the first N layers in the CPU<br/>(dense models; for MoE expert weights use --n-cpu-moe)<br/>(env: LLAMA_ARG_N_CPU_FFN) |
 | `-ngl, --gpu-layers, --n-gpu-layers N` | max. number of layers to store in VRAM, either an exact number, 'auto', or 'all' (default: auto)<br/>(env: LLAMA_ARG_N_GPU_LAYERS) |
 | `-sm, --split-mode {none,layer,row,tensor}` | how to split the model across multiple GPUs, one of:<br/>- none: use one GPU only<br/>- layer (default): split layers and KV across GPUs (pipelined)<br/>- row: split weight across GPUs by rows (parallelized)<br/>- tensor: split weights and KV across GPUs (parallelized, EXPERIMENTAL)<br/>(env: LLAMA_ARG_SPLIT_MODE) |
 | `-ts, --tensor-split N0,N1,N2,...` | fraction of the model to offload to each GPU, comma-separated list of proportions, e.g. 3,1<br/>(env: LLAMA_ARG_TENSOR_SPLIT) |
@@ -182,6 +184,9 @@ For the full list of features, please refer to [server's changelog](https://gith
 | `--image-min-tokens N` | minimum number of tokens each image can take, only used by vision models with dynamic resolution (default: read from model)<br/>(env: LLAMA_ARG_IMAGE_MIN_TOKENS) |
 | `--image-max-tokens N` | maximum number of tokens each image can take, only used by vision models with dynamic resolution (default: read from model)<br/>(env: LLAMA_ARG_IMAGE_MAX_TOKENS) |
 | `--mtmd-batch-max-tokens N` | maximum number of image tokens per batch when encoding images (default: 1024)<br/>(env: LLAMA_ARG_MTMD_BATCH_MAX_TOKENS) |
+| `--video-fps N` | target video frame rate (default: 4.0)<br/>(env: LLAMA_ARG_VIDEO_FPS) |
+| `--video-timestamp-interval N` | interval in milliseconds between text timestamps (default: 5000)<br/>(env: LLAMA_ARG_VIDEO_TIMESTAMP_INTERVAL) |
+| `--video-ffmpeg-dir DIR` | path to the directory containing ffmpeg and ffprobe (default: search in PATH)<br/>(env: LLAMA_ARG_VIDEO_FFMPEG_DIR) |
 | `-a, --alias STRING` | set model name aliases, comma-separated (to be used by API)<br/>(env: LLAMA_ARG_ALIAS) |
 | `--tags STRING` | set model tags, comma-separated (informational, not used for routing)<br/>(env: LLAMA_ARG_TAGS) |
 | `--embd-normalize N` | normalisation for embeddings (default: 2) (-1=none, 0=max absolute int16, 1=taxicab, 2=euclidean, >2=p-norm) |
@@ -256,6 +261,8 @@ For the full list of features, please refer to [server's changelog](https://gith
 | `--spec-draft-n-cpu-moe, --spec-draft-ncmoe, -ncmoed, --n-cpu-moe-draft N` | keep the Mixture of Experts (MoE) weights of the first N layers in the CPU for the draft model<br/>(env: LLAMA_ARG_SPEC_DRAFT_N_CPU_MOE) |
 | `--spec-draft-n-max N` | number of tokens to draft for speculative decoding (default: 3)<br/>(env: LLAMA_ARG_SPEC_DRAFT_N_MAX) |
 | `--spec-draft-n-min N` | minimum number of draft tokens to use for speculative decoding (default: 0)<br/>(env: LLAMA_ARG_SPEC_DRAFT_N_MIN) |
+| `--spec-synth-len L` | target mean synthetic acceptance length, including the target token (benchmarking only)<br/>(env: LLAMA_ARG_SPEC_SYNTH_LEN) |
+| `--spec-synth-rates P0,P1,...` | comma-separated unconditional per-position synthetic acceptance probabilities (benchmarking only)<br/>(env: LLAMA_ARG_SPEC_SYNTH_RATES) |
 | `--spec-draft-p-split, --draft-p-split P` | speculative decoding split probability (default: 0.10)<br/>(env: LLAMA_ARG_SPEC_DRAFT_P_SPLIT) |
 | `--spec-draft-p-min, --draft-p-min P` | minimum speculative decoding probability (greedy) (default: 0.00)<br/>(env: LLAMA_ARG_SPEC_DRAFT_P_MIN) |
 | `--spec-draft-backend-sampling, --no-spec-draft-backend-sampling` | offload draft sampling to the backend (default: enabled)<br/>(env: LLAMA_ARG_SPEC_DRAFT_BACKEND_SAMPLING) |

tools/server/server-common.cpp

@@ -910,12 +910,17 @@ size_t validate_utf8(const std::string& text) {
     return len;
 }
 
-server_tokens process_mtmd_prompt(mtmd_context * mctx, const std::string & prompt, const std::vector<raw_buffer> & files, bool is_placeholder) {
+server_tokens process_mtmd_prompt(
+        mtmd_context * mctx,
+        const std::string & prompt,
+        const std::vector<raw_buffer> & files,
+        const mtmd_helper_init_opt & init_opt,
+        bool is_placeholder) {
     // these will be freed upon going out of scope
     mtmd::bitmaps bitmaps;
     std::vector<mtmd_helper::video_ptr> videos;
     for (auto & file : files) {
-        auto out = mtmd_helper_bitmap_init_from_buf(mctx, file.data(), file.size(), is_placeholder);
+        auto out = mtmd_helper_bitmap_init_from_buf(mctx, file.data(), file.size(), is_placeholder, init_opt);
         if (!out.bitmap) {
             throw std::runtime_error("Failed to load image or audio file");
         }
@@ -956,7 +961,7 @@ server_tokens process_mtmd_prompt(mtmd_context * mctx, const std::string & promp
  * - "prompt": [12, 34, "string", 56, 78]
  * - "prompt": { "prompt_string": "string", "multimodal_data": [ "base64" ] }
  */
-static server_tokens tokenize_input_subprompt(const llama_vocab * vocab, mtmd_context * mctx, const json & json_prompt, bool add_special, bool parse_special) {
+static server_tokens tokenize_input_subprompt(const llama_vocab * vocab, mtmd_context * mctx, const json & json_prompt, bool add_special, bool parse_special, const mtmd_helper_init_opt & init_opt) {
     constexpr char JSON_STRING_PROMPT_KEY[] = "prompt_string";
     constexpr char JSON_MTMD_DATA_KEY[] = "multimodal_data";
     const bool has_mtmd = mctx != nullptr;
@@ -979,7 +984,7 @@ static server_tokens tokenize_input_subprompt(const llama_vocab * vocab, mtmd_co
             for (const auto & entry : json_prompt.at(JSON_MTMD_DATA_KEY)) {
                 files.push_back(base64_decode(entry));
             }
-            return process_mtmd_prompt(mctx, json_prompt.at(JSON_STRING_PROMPT_KEY), files);
+            return process_mtmd_prompt(mctx, json_prompt.at(JSON_STRING_PROMPT_KEY), files, init_opt);
         } else {
             // Not multimodal, but contains a subobject.
             llama_tokens tmp = tokenize_mixed(vocab, json_prompt.at(JSON_STRING_PROMPT_KEY), add_special, parse_special);
@@ -990,15 +995,15 @@ static server_tokens tokenize_input_subprompt(const llama_vocab * vocab, mtmd_co
    }
 }
 
-std::vector<server_tokens> tokenize_input_prompts(const llama_vocab * vocab, mtmd_context * mctx, const json & json_prompt, bool add_special, bool parse_special) {
+std::vector<server_tokens> tokenize_input_prompts(const llama_vocab * vocab, mtmd_context * mctx, const json & json_prompt, bool add_special, bool parse_special, const mtmd_helper_init_opt & init_opt) {
     std::vector<server_tokens> result;
     if (json_prompt.is_array() && !json_is_array_and_contains_numbers(json_prompt)) {
         result.reserve(json_prompt.size());
         for (const auto & p : json_prompt) {
-            result.push_back(tokenize_input_subprompt(vocab, mctx, p,add_special, parse_special));
+            result.push_back(tokenize_input_subprompt(vocab, mctx, p, add_special, parse_special, init_opt));
         }
     } else {
-        result.push_back(tokenize_input_subprompt(vocab, mctx, json_prompt, add_special, parse_special));
+        result.push_back(tokenize_input_subprompt(vocab, mctx, json_prompt, add_special, parse_special, init_opt));
     }
     if (result.empty()) {
         throw std::runtime_error("\"prompt\" must not be empty");
@@ -1787,7 +1792,8 @@ server_tokens format_prompt_rerank(
         const struct llama_vocab * vocab,
         mtmd_context * mctx,
         const std::string & query,
-        const std::string & doc) {
+        const std::string & doc,
+        const mtmd_helper_init_opt & init_opt) {
     server_tokens result = {};
 
     const char * rerank_prompt = llama_model_chat_template(model, "rerank");
@@ -1796,12 +1802,12 @@ server_tokens format_prompt_rerank(
         std::string prompt = rerank_prompt;
         string_replace_all(prompt, "{query}"   , query);
         string_replace_all(prompt, "{document}", doc  );
-        server_tokens tokens = tokenize_input_subprompt(vocab, mctx, prompt, false, true);
+        server_tokens tokens = tokenize_input_subprompt(vocab, mctx, prompt, false, true, init_opt);
         result.push_back(tokens);
     } else {
         // Get EOS token - use SEP token as fallback if EOS is not available
-        server_tokens query_tokens = tokenize_input_subprompt(vocab, mctx, query, false, false);
-        server_tokens doc_tokens   = tokenize_input_subprompt(vocab, mctx, doc,   false, false);
+        server_tokens query_tokens = tokenize_input_subprompt(vocab, mctx, query, false, false, init_opt);
+        server_tokens doc_tokens   = tokenize_input_subprompt(vocab, mctx, doc,   false, false, init_opt);
         llama_token eos_token = llama_vocab_eos(vocab);
         if (eos_token == LLAMA_TOKEN_NULL) {
             eos_token = llama_vocab_sep(vocab);

tools/server/server-common.h

@@ -5,6 +5,7 @@
 #include "llama.h"
 #include "chat.h"
 #include "mtmd.h"
+#include "mtmd-helper.h"
 
 #include "json.h"
 
@@ -269,7 +270,12 @@ size_t validate_utf8(const std::string& text);
 
 // process mtmd prompt, return the server_tokens containing both text tokens and media chunks
 // if is_placeholder is true, the media chunk will be treated as placeholder for counting tokens; the output tokens are not usable for actual inference (e.g. for submitting a task to server_queue)
-server_tokens process_mtmd_prompt(mtmd_context * mctx, const std::string & prompt, const std::vector<raw_buffer> & files, bool is_placeholder = false);
+server_tokens process_mtmd_prompt(
+                                        mtmd_context * mctx,
+                                        const std::string & prompt,
+                                        const std::vector<raw_buffer> & files,
+                                        const mtmd_helper_init_opt & init_opt,
+                                        bool is_placeholder = false);
 
 /**
  * break the input "prompt" object into multiple prompt if needed, then tokenize them
@@ -289,7 +295,8 @@ std::vector<server_tokens> tokenize_input_prompts(
                                         mtmd_context * mctx,
                                         const json & json_prompt,
                                         bool add_special,
-                                        bool parse_special);
+                                        bool parse_special,
+                                        const mtmd_helper_init_opt & init_opt);
 
 //
 // OAI utils
@@ -538,7 +545,8 @@ server_tokens format_prompt_rerank(
         const struct llama_vocab * vocab,
         mtmd_context * mctx,
         const std::string & query,
-        const std::string & doc);
+        const std::string & doc,
+        const mtmd_helper_init_opt & init_opt);
 
 // simple implementation of a pipe
 // used for streaming data between threads

tools/server/server-context.cpp

@@ -23,6 +23,7 @@
 #include <exception>
 #include <memory>
 #include <filesystem>
+#include <random>
 #include <utility>
 #include <fstream>
 
@@ -51,6 +52,50 @@ static common_speculative_output_limits server_output_limits(const common_params
     return result;
 }
 
+// synthetic draft verification for benchmarking - accept draft tokens at random instead of by match with the target
+// on replay the draft was already accepted before a context checkpoint restore, so repeat the same decisions
+static std::vector<llama_token> server_sample_and_accept_synth(
+        common_sampler * smpl,
+        llama_context * ctx,
+        const std::vector<int32_t> & idxs,
+        const llama_tokens & draft,
+        const std::vector<double> & synth_probs,
+        std::mt19937 & rng,
+        bool is_replay) {
+    GGML_ASSERT(idxs.size() == draft.size() + 1);
+    GGML_ASSERT(synth_probs.size() >= draft.size());
+
+    std::vector<llama_token> result;
+    result.reserve(idxs.size());
+
+    const llama_vocab * vocab = llama_model_get_vocab(llama_get_model(ctx));
+    std::uniform_real_distribution<double> dist(0.0, 1.0);
+    for (size_t i = 0; i < draft.size(); ++i) {
+        const llama_token id = common_sampler_sample(smpl, ctx, idxs[i]);
+        const bool accept = is_replay || dist(rng) < synth_probs[i];
+        // do not accept a drafted EOG token - it would end the generation early
+        // on replay the last token is from the target and can be EOG, so skip this check
+        if (accept && (is_replay || !llama_vocab_is_eog(vocab, draft[i]))) {
+            // synthetic draft tokens do not advance grammar or reasoning state
+            // the last replay token is from the target and must advance both
+            const bool is_replay_target = is_replay && i + 1 == draft.size();
+            common_sampler_accept(smpl, draft[i], is_replay_target);
+            result.push_back(draft[i]);
+            continue;
+        }
+
+        common_sampler_accept(smpl, id, true);
+        result.push_back(id);
+        return result;
+    }
+
+    const llama_token id = common_sampler_sample(smpl, ctx, idxs[draft.size()]);
+    common_sampler_accept(smpl, id, true);
+    result.push_back(id);
+
+    return result;
+}
+
 // state diagram: https://github.com/ggml-org/llama.cpp/pull/9283
 enum slot_state {
     SLOT_STATE_IDLE,
@@ -211,6 +256,7 @@ struct server_slot {
     std::vector<int32_t> spec_i_batch;
     common_prompt_checkpoint spec_ckpt;
     bool spec_is_replay = false;
+    std::mt19937 spec_synth_rng;
 
     // TODO: move members that belong to the task (such as `generated_text`, `has_new_line`) to task_results_state
     //       see https://github.com/ggml-org/llama.cpp/pull/18283#issuecomment-3710175837
@@ -794,6 +840,8 @@ public:
     llama_model * model_tgt = nullptr;
 
     mtmd_context * mctx = nullptr;
+    // note: video_params.ffmpeg_bin_dir points into params_base, which outlives this struct
+    mtmd_helper_init_opt init_opt = mtmd_helper_init_opt_default();
     const llama_vocab * vocab = nullptr;
 
     server_queue    queue_tasks;
@@ -1118,6 +1166,11 @@ private:
             }
             SRV_INF("loaded multimodal model, '%s'\n", mmproj_path.c_str());
 
+            init_opt.video_params.fps_target = params_base.video_fps;
+            init_opt.video_params.timestamp_interval_ms = params_base.video_timestamp_interval_ms;
+            init_opt.video_params.ffmpeg_bin_dir = params_base.video_ffmpeg_bin_dir.empty()
+                                ? nullptr : params_base.video_ffmpeg_bin_dir.c_str();
+
             if (params_base.ctx_shift) {
                 params_base.ctx_shift = false;
                 SRV_WRN("%s\n", "ctx_shift is not supported by multimodal, it will be disabled");
@@ -1187,6 +1240,9 @@ private:
                 spec.reset(common_speculative_init(params_base.speculative, params_base.n_parallel));
             } catch (const std::exception & e) {
                 SRV_ERR("failed to initialize speculative decoding context: %s\n", e.what());
+                if (params_base.speculative.has_synth()) {
+                    return false;
+                }
             }
         }
 
@@ -1202,6 +1258,11 @@ private:
             model_dft = nullptr;
         }
 
+        if (!spec && params_base.speculative.has_synth()) {
+            SRV_ERR("%s", "synthetic acceptance requires an initialized speculative decoding context\n");
+            return false;
+        }
+
         for (int i = 0; i < params_base.n_parallel; i++) {
             server_slot & slot = slots[i];
 
@@ -1710,6 +1771,13 @@ private:
 
             SLT_TRC(slot, "sampler chain: %s\n", common_sampler_print(slot.smpl.get()).c_str());
             SLT_TRC(slot, "sampler params: \n%s\n", task.params.sampling.print().c_str());
+
+            if (spec && !common_speculative_get_synth_probs(spec.get()).empty()) {
+                const uint32_t seed = task.params.sampling.seed == LLAMA_DEFAULT_SEED
+                    ? std::random_device{}()
+                    : task.params.sampling.seed;
+                slot.spec_synth_rng.seed(seed);
+            }
         } else {
             slot.smpl.reset();
         }
@@ -2134,9 +2202,9 @@ private:
         try {
             auto & prompt = task.cli_prompt;
             if (mctx != nullptr) {
-                task.tokens = process_mtmd_prompt(mctx, prompt, task.cli_files);
+                task.tokens = process_mtmd_prompt(mctx, prompt, task.cli_files, init_opt);
             } else {
-                task.tokens = std::move(tokenize_input_prompts(vocab, mctx, prompt, true, true)[0]);
+                task.tokens = std::move(tokenize_input_prompts(vocab, mctx, prompt, true, true, init_opt)[0]);
             }
             task.cli_prompt.clear();
             task.cli_files.clear();
@@ -3795,7 +3863,12 @@ private:
                 common_sampler_ptr smpl_save(common_sampler_clone(slot.smpl.get()));
 
                 GGML_ASSERT(slot.spec_i_batch.size() == n_draft + 1);
-                auto accepted = common_sampler_sample_and_accept_n(slot.smpl.get(), slot.ctx_tgt, slot.spec_i_batch, slot.spec_draft);
+                const auto & synth_probs = common_speculative_get_synth_probs(spec.get());
+                auto accepted = synth_probs.empty()
+                    ? common_sampler_sample_and_accept_n(slot.smpl.get(), slot.ctx_tgt, slot.spec_i_batch, slot.spec_draft)
+                    : server_sample_and_accept_synth(
+                            slot.smpl.get(), slot.ctx_tgt, slot.spec_i_batch, slot.spec_draft,
+                            synth_probs, slot.spec_synth_rng, slot.spec_is_replay);
                 slot.spec_i_batch.clear();
 
                 GGML_ASSERT(accepted.size() >= 1);
@@ -3861,7 +3934,7 @@ private:
 
             auto & n_accepted_per_pos = slot.n_accepted_per_pos;
             if (n_accepted_per_pos.empty()) {
-                n_accepted_per_pos.resize(common_speculative_n_max(&params_base.speculative), 0);
+                n_accepted_per_pos.resize(common_speculative_n_max(spec.get()), 0);
             }
             for (size_t i = 0; i < n_accepted && i < n_accepted_per_pos.size(); ++i) {
                 n_accepted_per_pos[i]++;
@@ -4165,10 +4238,10 @@ std::unique_ptr<server_res_generator> server_routes::handle_completions_impl(
 
         if (res_type != TASK_RESPONSE_TYPE_NONE && ctx_server.mctx != nullptr) {
             // This is the case used by OAI compatible chat path with MTMD. TODO It can be moved to the path below.
-            inputs.push_back(process_mtmd_prompt(ctx_server.mctx, prompt.get<std::string>(), files));
+            inputs.push_back(process_mtmd_prompt(ctx_server.mctx, prompt.get<std::string>(), files, ctx_server.init_opt));
         } else {
             // Everything else, including multimodal completions.
-            inputs = tokenize_input_prompts(ctx_server.vocab, ctx_server.mctx, prompt, true, true);
+            inputs = tokenize_input_prompts(ctx_server.vocab, ctx_server.mctx, prompt, true, true, ctx_server.init_opt);
         }
 
         // tasks.reserve(inputs.size()); // TODO: this is inaccurate due to child tasks
@@ -4752,7 +4825,7 @@ void server_routes::init_routes() {
         data["input_extra"] = input_extra; // default to empty array if it's not exist
 
         std::string prompt = json_value(data, "prompt", std::string());
-        std::vector<server_tokens> tokenized_prompts = tokenize_input_prompts(ctx_server.vocab, ctx_server.mctx, prompt, false, true);
+        std::vector<server_tokens> tokenized_prompts = tokenize_input_prompts(ctx_server.vocab, ctx_server.mctx, prompt, false, true, ctx_server.init_opt);
         SRV_DBG("creating infill tasks, n_prompts = %d\n", (int) tokenized_prompts.size());
         data["prompt"] = format_prompt_infill(
             ctx_server.vocab,
@@ -4816,7 +4889,7 @@ void server_routes::init_routes() {
     };
 
     this->post_chat_completions_tok = [this](const server_http_req & req) {
-        return handle_count_tokens(ctx_server.vocab, ctx_server.mctx, req, TASK_RESPONSE_TYPE_OAI_CHAT);
+        return handle_count_tokens(ctx_server.vocab, ctx_server.mctx, ctx_server.init_opt, req, TASK_RESPONSE_TYPE_OAI_CHAT);
     };
 
     this->post_control = [this](const server_http_req & req) {
@@ -4875,7 +4948,7 @@ void server_routes::init_routes() {
     };
 
     this->post_responses_tok_oai = [this](const server_http_req & req) {
-        return handle_count_tokens(ctx_server.vocab, ctx_server.mctx, req, TASK_RESPONSE_TYPE_OAI_RESP);
+        return handle_count_tokens(ctx_server.vocab, ctx_server.mctx, ctx_server.init_opt, req, TASK_RESPONSE_TYPE_OAI_RESP);
     };
 
     this->post_transcriptions_oai = [this](const server_http_req & req) {
@@ -4925,7 +4998,7 @@ void server_routes::init_routes() {
     };
 
     this->post_anthropic_count_tokens = [this](const server_http_req & req) {
-        return handle_count_tokens(ctx_server.vocab, ctx_server.mctx, req, TASK_RESPONSE_TYPE_ANTHROPIC);
+        return handle_count_tokens(ctx_server.vocab, ctx_server.mctx, ctx_server.init_opt, req, TASK_RESPONSE_TYPE_ANTHROPIC);
     };
 
     // same with handle_chat_completions, but without inference part
@@ -5058,7 +5131,7 @@ void server_routes::init_routes() {
             std::vector<server_task> tasks;
             tasks.reserve(documents.size());
             for (size_t i = 0; i < documents.size(); i++) {
-                auto tmp = format_prompt_rerank(ctx_server.model_tgt, ctx_server.vocab, ctx_server.mctx, query, documents[i]);
+                auto tmp = format_prompt_rerank(ctx_server.model_tgt, ctx_server.vocab, ctx_server.mctx, query, documents[i], ctx_server.init_opt);
                 server_task task = server_task(SERVER_TASK_TYPE_RERANK);
                 task.id     = rd.get_new_id();
                 task.tokens = std::move(tmp);
@@ -5296,7 +5369,7 @@ std::unique_ptr<server_res_generator> server_routes::handle_embeddings_impl(cons
         }
     }
 
-    auto tokenized_prompts = tokenize_input_prompts(ctx_server.vocab, ctx_server.mctx, prompt, true, true);
+    auto tokenized_prompts = tokenize_input_prompts(ctx_server.vocab, ctx_server.mctx, prompt, true, true, ctx_server.init_opt);
     for (const auto & tokens : tokenized_prompts) {
         // this check is necessary for models that do not add BOS token to the input
         if (tokens.empty()) {
@@ -5357,7 +5430,7 @@ std::unique_ptr<server_res_generator> server_routes::handle_embeddings_impl(cons
     return res;
 }
 
-std::unique_ptr<server_res_generator> server_routes::handle_count_tokens(const llama_vocab * vocab, mtmd_context * mctx, const server_http_req & req, task_response_type res_type) {
+std::unique_ptr<server_res_generator> server_routes::handle_count_tokens(const llama_vocab * vocab, mtmd_context * mctx, const mtmd_helper_init_opt & init_opt, const server_http_req & req, task_response_type res_type) {
     auto res = create_response();
     std::vector<raw_buffer> files;
     json body = json::parse(req.body);
@@ -5395,7 +5468,7 @@ std::unique_ptr<server_res_generator> server_routes::handle_count_tokens(const l
         if (!prompt.is_string()) {
             throw std::runtime_error("for mtmd, input prompt must be a string.");
         }
-        n_tokens = process_mtmd_prompt(mctx, prompt.get<std::string>(), files, true).size();
+        n_tokens = process_mtmd_prompt(mctx, prompt.get<std::string>(), files, init_opt, true).size();
     } else {
         n_tokens = tokenize_mixed(vocab, prompt, true, true).size();
     }

tools/server/server-context.h

@@ -169,7 +169,7 @@ private:
     std::unique_ptr<server_res_generator> handle_slots_restore(const server_http_req & req, int id_slot);
     std::unique_ptr<server_res_generator> handle_slots_erase(const server_http_req &, int id_slot);
     std::unique_ptr<server_res_generator> handle_embeddings_impl(const server_http_req & req, task_response_type res_type);
-    std::unique_ptr<server_res_generator> handle_count_tokens(const llama_vocab * vocab, mtmd_context * mctx, const server_http_req & req, task_response_type res_type);
+    std::unique_ptr<server_res_generator> handle_count_tokens(const llama_vocab * vocab, mtmd_context * mctx, const mtmd_helper_init_opt & init_opt, const server_http_req & req, task_response_type res_type);
 
     // using unique_ptr to allow late initialization of const
     std::unique_ptr<const server_context_meta> meta;

tools/server/tests/unit/test_speculative.py

@@ -52,6 +52,18 @@ def test_with_and_without_draft():
 
     assert tokens_no_draft == tokens_draft
 
+    server.stop()
+    create_server()
+    assert server.spec_draft_n_max is not None
+    server.spec_synth_rates = [0.0] * server.spec_draft_n_max
+    server.start()
+    res = server.make_request("POST", "/completion", data=request)
+
+    assert res.status_code == 200
+    assert res.body["timings"]["draft_n"] > 0
+    assert res.body["timings"]["draft_n_accepted"] == 0
+    assert res.body["tokens"] == tokens_no_draft
+
 
 def test_different_draft_min_draft_max():
     global server
@@ -80,6 +92,66 @@ def test_different_draft_min_draft_max():
         last_content = res.body["content"]
 
 
+def test_synth_is_deterministic():
+    global server
+    assert server.spec_draft_n_max is not None
+    server.spec_synth_rates = [0.75 ** (i + 1) for i in range(server.spec_draft_n_max)]
+    server.start()
+
+    request = {
+        "prompt": "I believe the meaning of life is",
+        "temperature": 0.2,
+        "top_k": 5,
+        "seed": 4242,
+        "n_predict": 32,
+    }
+    responses = [server.make_request("POST", "/completion", data=request) for _ in range(2)]
+
+    for res in responses:
+        assert res.status_code == 200
+        assert res.body["timings"]["draft_n"] > 0
+    assert responses[0].body["timings"]["draft_n"] == responses[1].body["timings"]["draft_n"]
+    assert responses[0].body["timings"]["draft_n_accepted"] == responses[1].body["timings"]["draft_n_accepted"]
+
+
+def test_synth_ignores_target_tokens():
+    global server
+    assert server.spec_draft_n_max is not None
+    server.spec_synth_rates = [1.0] * server.spec_draft_n_max
+    server.start()
+
+    res = server.make_request("POST", "/completion", data={
+        "prompt": "I believe the meaning of life is",
+        "temperature": 0.0,
+        "seed": 4242,
+        "n_predict": 32,
+    })
+
+    assert res.status_code == 200
+    assert res.body["timings"]["draft_n"] > 0
+    assert res.body["timings"]["draft_n_accepted"] == res.body["timings"]["draft_n"]
+
+    res = server.make_request("POST", "/completion", data={
+        "prompt": "I believe the meaning of life is",
+        "temperature": 0.0,
+        "seed": 4242,
+        "n_predict": 6,
+        "grammar": 'root ::= "a"{5,5}',
+    })
+    assert res.status_code == 200, res.body
+
+    res = server.make_request("POST", "/completion", data={
+        "prompt": "Respond with only: OK",
+        "temperature": 0.0,
+        "seed": 4242,
+        "n_predict": 64,
+        "ignore_eos": True,
+    })
+    assert res.status_code == 200, res.body
+    assert res.body["tokens_predicted"] == 64
+    assert res.body["stop_type"] == "limit"
+
+
 def test_slot_ctx_not_exceeded():
     global server
     server.n_ctx = 256

tools/server/tests/utils.py

@@ -99,6 +99,8 @@ class ServerProcess:
     spec_type: str | None = None
     spec_draft_n_min: int | None = None
     spec_draft_n_max: int | None = None
+    spec_synth_len: float | None = None
+    spec_synth_rates: List[float] | None = None
     no_ui: bool | None = None
     jinja: bool | None = None
     reasoning_format: Literal['deepseek', 'none', 'nothink'] | None = None
@@ -245,6 +247,11 @@ class ServerProcess:
             server_args.extend(["--spec-draft-n-max", self.spec_draft_n_max])
         if self.spec_draft_n_min:
             server_args.extend(["--spec-draft-n-min", self.spec_draft_n_min])
+        if self.spec_synth_len is not None:
+            server_args.extend(["--spec-synth-len", self.spec_synth_len])
+        if self.spec_synth_rates is not None:
+            rates = ",".join(str(rate) for rate in self.spec_synth_rates)
+            server_args.extend(["--spec-synth-rates", rates])
         if self.no_ui:
             server_args.append("--no-ui")
         if self.no_models_autoload:

tools/tts/tts.cpp

@@ -103,7 +103,7 @@ int main(int argc, char ** argv) {
 
     mtmd::bitmap_ptr speaker_bitmap;
     if (!params.tts_speaker_file.empty()) {
-        auto wrapper = mtmd_helper_bitmap_init_from_file(mctx.get(), params.tts_speaker_file.c_str(), false);
+        auto wrapper = mtmd_helper_bitmap_init_from_file(mctx.get(), params.tts_speaker_file.c_str(), false, mtmd_helper_init_opt_default());
         if (!wrapper.bitmap) {
             LOG_ERR("failed to load speaker file %s\n", params.tts_speaker_file.c_str());
             return 1;

tools/ui/src/lib/components/app/badges/BadgesModality.svelte

@@ -1,5 +1,5 @@
 <script lang="ts">
-	import { Eye, Mic, Video } from '@lucide/svelte';
+	import { MODALITY_ICONS, MODALITY_LABELS } from '$lib/constants';
 	import { ModelModality } from '$lib/enums';
 
 	interface Props {
@@ -8,29 +8,22 @@
 	}
 
 	let { class: className = '', modalities }: Props = $props();
+
+	const shownModalities = [ModelModality.VISION, ModelModality.AUDIO, ModelModality.VIDEO] as const;
+
+	let visible = $derived(shownModalities.filter((modality) => modalities.includes(modality)));
 </script>
 
-{#each modalities as modality (modality)}
-	{#if modality === ModelModality.VISION || modality === ModelModality.AUDIO || modality === ModelModality.VIDEO}
-		<span
-			class={[
-				'inline-flex items-center gap-1 rounded-md bg-muted px-2 py-1 text-xs font-medium',
-				className
-			]}
-		>
-			{#if modality === ModelModality.VISION}
-				<Eye class="h-3 w-3" />
+{#each visible as modality (modality)}
+	{@const ModalityIcon = MODALITY_ICONS[modality]}
+	<span
+		class={[
+			'inline-flex items-center gap-1 rounded-md bg-muted px-2 py-1 text-xs font-medium',
+			className
+		]}
+	>
+		<ModalityIcon class="h-3 w-3" />
 
-				Vision (Image)
-			{:else if modality === ModelModality.VIDEO}
-				<Video class="h-3 w-3" />
-
-				Vision (Video)
-			{:else}
-				<Mic class="h-3 w-3" />
-
-				Audio
-			{/if}
-		</span>
-	{/if}
+		{MODALITY_LABELS[modality]}
+	</span>
 {/each}

tools/ui/src/lib/components/app/chat/ChatForm/ChatForm.svelte

@@ -23,7 +23,8 @@
 		FileExtensionText,
 		KeyboardKey,
 		MimeTypeText,
-		SpecialFileType
+		SpecialFileType,
+		ToolSource
 	} from '$lib/enums';
 	import { useChatFormPickers } from '$lib/hooks/use-chat-form-pickers.svelte';
 	import {
@@ -73,7 +74,6 @@
 		disabled?: boolean;
 		isLoading?: boolean;
 		placeholder?: string;
-		showMcpPromptButton?: boolean;
 		showAddButton?: boolean;
 		showModelSelector?: boolean;
 
@@ -103,7 +103,6 @@
 		onValueChange,
 		placeholder = 'Type a message...',
 		showAddButton = true,
-		showMcpPromptButton = false,
 		showModelSelector = true,
 		uploadedFiles = $bindable([]),
 		value = $bindable('')
@@ -152,9 +151,18 @@
 		getServerHome: () => toolsStore.serverHome ?? null,
 		getShowModelSelector: () => showModelSelector,
 		getValue: () => value,
-		hasCwdTools: () => toolsStore.hasEnabledCwdTools,
-		hasPrompts: () =>
-			mcpStore.hasPromptsCapability(conversationsStore.preferences.getAllMcpServerOverrides()),
+		hasCwdTools: () => conversationsStore.preferences.hasEnabledCwdTools(),
+		// policy-aware, same rule as the agentic flow: MCP category on and at
+		// least one globally-enabled server whose group key is not disabled
+		hasPrompts: () => {
+			const prefs = conversationsStore.preferences;
+
+			if (!prefs.isCategoryEnabled(ToolSource.MCP)) return false;
+
+			return mcpStore
+				.getServers()
+				.some((s) => s.enabled && prefs.isServerToolsEnabled(s.id) && s.url.trim());
+		},
 		openModelSelector: () => chatFormActionsRef?.openModelSelector(),
 		setCaretOffset: (offset) => inputRef?.setCaretOffset(offset),
 		setValue: (v) => {
@@ -620,8 +628,6 @@
 				isReasoning={chatStore.isReasoning}
 				{isRecording}
 				onFileUpload={handleFileUpload}
-				onMcpPromptClick={showMcpPromptButton ? () => pickers.openPromptPicker() : undefined}
-				onMcpResourcesClick={() => (isResourceDialogOpen = true)}
 				onMcpSettingsClick={() => (isMcpServersDialogOpen = true)}
 				onMicClick={handleMicClick}
 				{onStop}
@@ -635,7 +641,7 @@
 
 	<ContextGaugePopup />
 
-	{#if toolsStore.hasEnabledCwdTools}
+	{#if conversationsStore.preferences.hasEnabledCwdTools()}
 		<ChatFormCurrentWorkingDirectory
 			bind:query={pickers.workingDirectoryQuery}
 			customAnchor={mentionAnchor}

tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionAdd/ChatFormActionAddDropdown.svelte

@@ -1,5 +1,5 @@
 <script lang="ts">
-	import { File, MessageSquare, Plus } from '@lucide/svelte';
+	import { File, Image, MessageSquare, Mic, Plus, Video } from '@lucide/svelte';
 	import { ChatFormActionAddToolsSubmenu, McpLogo } from '$lib/components/app';
 	import { buttonVariants } from '$lib/components/ui/button';
 	import * as DropdownMenu from '$lib/components/ui/dropdown-menu';
@@ -8,10 +8,10 @@
 	import {
 		ATTACHMENT_FILE_ITEMS,
 		ATTACHMENT_TOOLTIP_TEXT,
-		ICON_CLASS_DEFAULT,
-		TOOLTIP_DELAY_DURATION
+		ICON_CLASS_DEFAULT
 	} from '$lib/constants';
 	import { getChatFormActionsContext } from '$lib/contexts';
+	import { AttachmentAction, AttachmentItemEnabledWhen } from '$lib/enums';
 	import { useAttachmentMenu } from '$lib/hooks/use-attachment-menu.svelte';
 
 	interface Props {
@@ -30,21 +30,29 @@
 	const attachmentMenu = useAttachmentMenu(
 		() => ({
 			hasAudioModality: chatFormActions.hasAudioModality,
-			hasMcpPromptsSupport: chatFormActions.hasMcpPromptsSupport,
-			hasMcpResourcesSupport: chatFormActions.hasMcpResourcesSupport,
 			hasVideoModality: chatFormActions.hasVideoModality,
 			hasVisionModality: chatFormActions.hasVisionModality
 		}),
 		() => ({
 			onFileUpload: chatFormActions.onFileUpload,
-			onMcpPromptClick: chatFormActions.onMcpPromptClick,
-			onMcpResourcesClick: chatFormActions.onMcpResourcesClick,
 			onSystemPromptClick: chatFormActions.onSystemPromptClick
 		}),
 		() => {
 			dropdownOpen = false;
 		}
 	);
+
+	const FILE_MODALITY_ICONS: Record<string, { icon: typeof Image; label: string }> = {
+		[AttachmentItemEnabledWhen.HAS_AUDIO_MODALITY]: { icon: Mic, label: 'Audio' },
+		[AttachmentItemEnabledWhen.HAS_VIDEO_MODALITY]: { icon: Video, label: 'Video' },
+		[AttachmentItemEnabledWhen.HAS_VISION_MODALITY]: { icon: Image, label: 'Vision' }
+	};
+
+	const supportedModalities = $derived.by(() =>
+		ATTACHMENT_FILE_ITEMS.filter((item) => attachmentMenu.isItemEnabled(item.enabledWhen))
+			.map((item) => FILE_MODALITY_ICONS[item.enabledWhen ?? ''])
+			.filter((modality) => modality !== undefined)
+	);
 </script>
 
 <div class="flex items-center gap-1 {className}">
@@ -84,50 +92,32 @@
 				}
 			}}
 		>
-			<DropdownMenu.Sub>
-				<DropdownMenu.SubTrigger class="flex cursor-pointer items-center gap-2">
-					<File class={ICON_CLASS_DEFAULT} />
+			<DropdownMenu.Item
+				class="flex cursor-pointer items-center gap-2"
+				onclick={() => attachmentMenu.callbacks[AttachmentAction.FILE_UPLOAD]()}
+			>
+				<File class={ICON_CLASS_DEFAULT} />
 
+				<span class="flex min-w-0 items-center gap-2">
 					<span>Add files</span>
-				</DropdownMenu.SubTrigger>
 
-				<DropdownMenu.SubContent class="w-48">
-					{#each ATTACHMENT_FILE_ITEMS as item (item.id)}
-						{@const enabled = attachmentMenu.isItemEnabled(item.enabledWhen)}
-						{#if enabled}
-							<DropdownMenu.Item
-								class="{item.class ?? ''} flex cursor-pointer items-center gap-2"
-								onclick={() => attachmentMenu.callbacks[item.action]()}
-							>
-								<item.icon class={ICON_CLASS_DEFAULT} />
+					{#if supportedModalities.length > 0}
+						<span class="flex items-center gap-0.75 text-muted-foreground">
+							{#each supportedModalities as modality (modality.label)}
+								<Tooltip.Root>
+									<Tooltip.Trigger>
+										<modality.icon class="size-2.75" />
+									</Tooltip.Trigger>
 
-								<span>{item.label}</span>
-							</DropdownMenu.Item>
-						{:else if item.disabledTooltip}
-							<Tooltip.Root delayDuration={TOOLTIP_DELAY_DURATION}>
-								<Tooltip.Trigger tabindex={-1}>
-									{#snippet child({ props })}
-										<div {...props} class="cursor-default">
-											<DropdownMenu.Item
-												class="{item.class ?? ''} flex items-center gap-2"
-												disabled
-											>
-												<item.icon class={ICON_CLASS_DEFAULT} />
-
-												<span>{item.label}</span>
-											</DropdownMenu.Item>
-										</div>
-									{/snippet}
-								</Tooltip.Trigger>
-
-								<Tooltip.Content side="right">
-									<p>{item.disabledTooltip}</p>
-								</Tooltip.Content>
-							</Tooltip.Root>
-						{/if}
-					{/each}
-				</DropdownMenu.SubContent>
-			</DropdownMenu.Sub>
+									<Tooltip.Content>
+										<p>{modality.label}</p>
+									</Tooltip.Content>
+								</Tooltip.Root>
+							{/each}
+						</span>
+					{/if}
+				</span>
+			</DropdownMenu.Item>
 
 			<DropdownMenu.Item
 				class="flex cursor-pointer items-center gap-2"

tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionAdd/ChatFormActionAddMcpSubmenu.svelte

@@ -1,51 +0,0 @@
-<script lang="ts">
-	import { FolderOpen, Server, Zap } from '@lucide/svelte';
-	import { McpLogo } from '$lib/components/app';
-	import * as DropdownMenu from '$lib/components/ui/dropdown-menu';
-	import { ICON_CLASS_DEFAULT } from '$lib/constants';
-	import { getChatFormActionsContext } from '$lib/contexts';
-
-	const chatFormActions = getChatFormActionsContext();
-
-	function handleServersClick() {
-		chatFormActions.onMcpSettingsClick?.();
-	}
-</script>
-
-<DropdownMenu.Sub>
-	<DropdownMenu.SubTrigger class="flex cursor-pointer items-center gap-2">
-		<McpLogo class={ICON_CLASS_DEFAULT} />
-
-		<span>MCP</span>
-	</DropdownMenu.SubTrigger>
-
-	<DropdownMenu.SubContent class="w-48">
-		<DropdownMenu.Item class="flex cursor-pointer items-center gap-2" onclick={handleServersClick}>
-			<Server class={ICON_CLASS_DEFAULT} />
-
-			<span>Servers</span>
-		</DropdownMenu.Item>
-
-		{#if chatFormActions.hasMcpPromptsSupport}
-			<DropdownMenu.Item
-				class="flex cursor-pointer items-center gap-2"
-				onclick={chatFormActions.onMcpPromptClick}
-			>
-				<Zap class={ICON_CLASS_DEFAULT} />
-
-				<span>Prompts</span>
-			</DropdownMenu.Item>
-		{/if}
-
-		{#if chatFormActions.hasMcpResourcesSupport}
-			<DropdownMenu.Item
-				class="flex cursor-pointer items-center gap-2"
-				onclick={chatFormActions.onMcpResourcesClick}
-			>
-				<FolderOpen class={ICON_CLASS_DEFAULT} />
-
-				<span>Resources</span>
-			</DropdownMenu.Item>
-		{/if}
-	</DropdownMenu.SubContent>
-</DropdownMenu.Sub>

tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionAdd/ChatFormActionAddReasoningSubmenu.svelte

@@ -8,70 +8,68 @@
 	const reasoning = useReasoningMenu();
 </script>
 
-{#if reasoning.modelSupportsThinking}
-	<DropdownMenu.Sub>
-		<DropdownMenu.SubTrigger class="flex cursor-pointer items-center gap-2">
-			{#if reasoning.thinkingEnabled}
-				<Lightbulb class="{ICON_CLASS_DEFAULT} shrink-0 text-amber-400" />
-			{:else if reasoning.isOff}
-				<LightbulbOff class="{ICON_CLASS_DEFAULT} shrink-0 text-muted-foreground" />
-			{:else}
-				<Lightbulb class="{ICON_CLASS_DEFAULT} shrink-0 text-muted-foreground" />
-			{/if}
+<DropdownMenu.Sub>
+	<DropdownMenu.SubTrigger class="flex cursor-pointer items-center gap-2">
+		{#if reasoning.isReasoningActive}
+			<Lightbulb class="{ICON_CLASS_DEFAULT} shrink-0 text-amber-400" />
+		{:else if reasoning.isOff}
+			<LightbulbOff class="{ICON_CLASS_DEFAULT} shrink-0 text-muted-foreground" />
+		{:else}
+			<Lightbulb class="{ICON_CLASS_DEFAULT} shrink-0 text-muted-foreground" />
+		{/if}
 
-			<span
-				class="text-sm inline-flex gap-2 {!reasoning.thinkingEnabled
-					? 'text-muted-foreground'
-					: ''}"
-			>
-				Reasoning
-
-				<span class="capitalize text-muted-foreground">
-					{reasoning.currentEffort}
-				</span>
-			</span>
-		</DropdownMenu.SubTrigger>
-
-		<DropdownMenu.SubContent
-			class="w-60 bg-popover p-1.5 text-popover-foreground shadow-md outline-none"
+		<span
+			class="text-sm inline-flex gap-2 {!reasoning.isReasoningActive
+				? 'text-muted-foreground'
+				: ''}"
 		>
-			{#each reasoning.levels as level (level.value)}
-				{@const tokenLabel = reasoning.tokenLabel(level)}
-				<DropdownMenu.Item
-					class="flex w-full cursor-pointer items-center gap-3 rounded-md px-2 py-1.75 text-left text-sm transition-colors hover:bg-accent {reasoning.isSelected(
-						level
-					)
-						? 'bg-accent'
-						: ''}"
-					onclick={() => reasoning.select(level)}
-				>
-					{#if reasoning.isSelected(level)}
-						<Check class="{ICON_CLASS_DEFAULT} shrink-0 text-foreground" />
-					{:else}
-						<div class="{ICON_CLASS_DEFAULT} shrink-0"></div>
-					{/if}
+			Reasoning
 
-					<span class="flex-1">{level.label}</span>
+			<span class="capitalize text-muted-foreground">
+				{reasoning.currentEffort}
+			</span>
+		</span>
+	</DropdownMenu.SubTrigger>
 
-					{#if tokenLabel}
-						<span class="text-[11px] text-muted-foreground opacity-60">
-							{tokenLabel}
-						</span>
-					{/if}
+	<DropdownMenu.SubContent
+		class="w-60 bg-popover p-1.5 text-popover-foreground shadow-md outline-none"
+	>
+		{#each reasoning.levels as level (level.value)}
+			{@const tokenLabel = reasoning.tokenLabel(level)}
+			<DropdownMenu.Item
+				class="flex w-full cursor-pointer items-center gap-3 rounded-md px-2 py-1.75 text-left text-sm transition-colors hover:bg-accent {reasoning.isSelected(
+					level
+				)
+					? 'bg-accent'
+					: ''}"
+				onclick={() => reasoning.select(level)}
+			>
+				{#if reasoning.isSelected(level)}
+					<Check class="{ICON_CLASS_DEFAULT} shrink-0 text-foreground" />
+				{:else}
+					<div class="{ICON_CLASS_DEFAULT} shrink-0"></div>
+				{/if}
 
-					{#if level.hasInfo}
-						<Tooltip.Root>
-							<Tooltip.Trigger>
-								<Info class="h-3.5 w-3.5 shrink-0 text-muted-foreground" />
-							</Tooltip.Trigger>
+				<span class="flex-1">{level.label}</span>
 
-							<Tooltip.Content side="left">
-								<p>Maximum reasoning effort with extended context usage</p>
-							</Tooltip.Content>
-						</Tooltip.Root>
-					{/if}
-				</DropdownMenu.Item>
-			{/each}
-		</DropdownMenu.SubContent>
-	</DropdownMenu.Sub>
-{/if}
+				{#if tokenLabel}
+					<span class="text-[11px] text-muted-foreground opacity-60">
+						{tokenLabel}
+					</span>
+				{/if}
+
+				{#if level.hasInfo}
+					<Tooltip.Root>
+						<Tooltip.Trigger>
+							<Info class="h-3.5 w-3.5 shrink-0 text-muted-foreground" />
+						</Tooltip.Trigger>
+
+						<Tooltip.Content side="left">
+							<p>Maximum reasoning effort with extended context usage</p>
+						</Tooltip.Content>
+					</Tooltip.Root>
+				{/if}
+			</DropdownMenu.Item>
+		{/each}
+	</DropdownMenu.SubContent>
+</DropdownMenu.Sub>

tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionAdd/ChatFormActionAddSheet.svelte

@@ -1,18 +1,18 @@
 <script lang="ts">
-	import { File, FolderOpen, MessageSquare, Zap } from '@lucide/svelte';
 	import {
 		Check,
 		ChevronDown,
 		ChevronRight,
+		File,
 		Lightbulb,
 		LightbulbOff,
+		MessageSquare,
 		PencilRuler
 	} from '@lucide/svelte';
 	import { McpLogo } from '$lib/components/app';
 	import { Checkbox } from '$lib/components/ui/checkbox';
 	import * as Collapsible from '$lib/components/ui/collapsible';
 	import * as Sheet from '$lib/components/ui/sheet';
-	import { Switch } from '$lib/components/ui/switch';
 	import * as Tooltip from '$lib/components/ui/tooltip';
 	import {
 		ATTACHMENT_FILE_ITEMS,
@@ -20,12 +20,11 @@
 		TOOLTIP_DELAY_DURATION
 	} from '$lib/constants';
 	import { getChatFormActionsContext } from '$lib/contexts';
-	import { HealthCheckStatus } from '$lib/enums';
 	import { AttachmentAction } from '$lib/enums/attachment.enums';
 	import { useAttachmentMenu } from '$lib/hooks/use-attachment-menu.svelte';
 	import { useReasoningMenu } from '$lib/hooks/use-reasoning-menu.svelte';
 	import { useToolsPanel } from '$lib/hooks/use-tools-panel.svelte';
-	import { conversationsStore, mcpStore } from '$lib/stores';
+	import type { ToolGroup } from '$lib/types';
 	import type { Snippet } from 'svelte';
 
 	interface Props {
@@ -38,23 +37,18 @@
 	const chatFormActions = getChatFormActionsContext();
 
 	let sheetOpen = $state(false);
-	let reasoningExpanded = $state(false);
 	let filesExpanded = $state(true);
+	let reasoningExpanded = $state(false);
 	let toolsExpanded = $state(false);
-	let mcpExpanded = $state(false);
 
 	const attachmentMenu = useAttachmentMenu(
 		() => ({
 			hasAudioModality: chatFormActions.hasAudioModality,
-			hasMcpPromptsSupport: chatFormActions.hasMcpPromptsSupport,
-			hasMcpResourcesSupport: chatFormActions.hasMcpResourcesSupport,
 			hasVideoModality: chatFormActions.hasVideoModality,
 			hasVisionModality: chatFormActions.hasVisionModality
 		}),
 		() => ({
 			onFileUpload: chatFormActions.onFileUpload,
-			onMcpPromptClick: chatFormActions.onMcpPromptClick,
-			onMcpResourcesClick: chatFormActions.onMcpResourcesClick,
 			onSystemPromptClick: chatFormActions.onSystemPromptClick
 		}),
 		() => {
@@ -70,8 +64,6 @@
 
 	const sheetItemRowClass =
 		'flex w-full items-center justify-between gap-2 rounded-md px-3 py-2 text-left text-sm transition-colors hover:bg-accent';
-
-	let mcpServers = $derived(mcpStore.getServers());
 </script>
 
 <div class="flex items-center gap-1 {className}">
@@ -194,80 +186,15 @@
 					</Collapsible.Content>
 				</Collapsible.Root>
 
-				<Collapsible.Root onOpenChange={(open) => (mcpExpanded = open)} open={mcpExpanded}>
-					<Collapsible.Trigger class={sheetItemClass}>
-						{#if mcpExpanded}
-							<ChevronDown class="{ICON_CLASS_DEFAULT} shrink-0" />
-						{:else}
-							<ChevronRight class="{ICON_CLASS_DEFAULT} shrink-0" />
-						{/if}
+				<button
+					class={sheetItemClass}
+					onclick={() => attachmentMenu.callbacks[AttachmentAction.SYSTEM_PROMPT_CLICK]()}
+					type="button"
+				>
+					<MessageSquare class="{ICON_CLASS_DEFAULT} shrink-0" />
 
-						<McpLogo class="inline {ICON_CLASS_DEFAULT} shrink-0" />
-
-						<span class="flex-1">MCP Servers</span>
-
-						<span class="text-xs text-muted-foreground">
-							{mcpServers.length} server{mcpServers.length !== 1 ? 's' : ''}
-						</span>
-					</Collapsible.Trigger>
-
-					<Collapsible.Content>
-						<div class="flex flex-col gap-0.5 pl-4">
-							{#each mcpServers as server (server.id)}
-								{@const healthState = mcpStore.getHealthCheckState(server.id)}
-								{@const hasError = healthState.status === HealthCheckStatus.ERROR}
-								{@const displayName = mcpStore.getServerLabel(server)}
-								{@const faviconUrl = mcpStore.getServerFavicon(server.id)}
-								{@const isEnabled = conversationsStore.preferences.isMcpServerEnabledForChat(
-									server.id
-								)}
-
-								<button
-									class={sheetItemRowClass}
-									disabled={hasError}
-									onclick={() =>
-										!hasError && conversationsStore.preferences.toggleMcpServerForChat(server.id)}
-									type="button"
-								>
-									<div class="flex min-w-0 flex-1 items-center gap-2">
-										{#if faviconUrl}
-											<img
-												alt=""
-												class="{ICON_CLASS_DEFAULT} shrink-0 rounded-sm"
-												onerror={(e) => {
-													(e.currentTarget as HTMLImageElement).style.display = 'none';
-												}}
-												src={faviconUrl}
-											/>
-										{/if}
-
-										<span class="min-w-0 truncate text-sm">{displayName}</span>
-									</div>
-
-									{#if hasError}
-										<span
-											class="shrink-0 rounded bg-destructive/15 px-1.5 py-0.5 text-xs text-destructive"
-										>
-											Error
-										</span>
-									{:else}
-										<Switch
-											checked={isEnabled}
-											onCheckedChange={() =>
-												conversationsStore.preferences.toggleMcpServerForChat(server.id)}
-										/>
-									{/if}
-								</button>
-							{/each}
-
-							{#if mcpServers.length === 0}
-								<div class="px-3 py-2 text-center text-sm text-muted-foreground">
-									No MCP servers configured
-								</div>
-							{/if}
-						</div>
-					</Collapsible.Content>
-				</Collapsible.Root>
+					<span>System Message</span>
+				</button>
 
 				{#if toolsPanel.totalToolCount > 0}
 					<Collapsible.Root onOpenChange={(open) => (toolsExpanded = open)} open={toolsExpanded}>
@@ -289,40 +216,12 @@
 
 						<Collapsible.Content>
 							<div class="flex flex-col gap-0.5 pl-4">
-								{#each toolsPanel.activeGroups as group (group.key)}
-									{@const checked = toolsPanel.isGroupChecked(group)}
-									{@const enabledCount = toolsPanel.getEnabledToolCount(group)}
-									{@const favicon = toolsPanel.getFavicon(group)}
+								{#each toolsPanel.categoryGroups as group (group.key)}
+									{@render sheetGroupRow(group)}
+								{/each}
 
-									<button
-										class={sheetItemRowClass}
-										onclick={() => toolsPanel.toggleGroupByKey(group.key)}
-										type="button"
-									>
-										{#if favicon}
-											<img
-												alt=""
-												class="{ICON_CLASS_DEFAULT} shrink-0 rounded-sm"
-												onerror={(e) => {
-													(e.currentTarget as HTMLImageElement).style.display = 'none';
-												}}
-												src={favicon}
-											/>
-										{/if}
-
-										<span class="min-w-0 flex-1 truncate text-sm font-medium">{group.label}</span>
-
-										<span class="shrink-0 text-xs text-muted-foreground">
-											{enabledCount}/{group.tools.length}
-										</span>
-
-										<Checkbox
-											{checked}
-											class="{ICON_CLASS_DEFAULT} shrink-0"
-											onCheckedChange={() => toolsPanel.toggleGroupByKey(group.key)}
-											onclick={(e) => e.stopPropagation()}
-										/>
-									</button>
+								{#each toolsPanel.mcpGroups as group (group.key)}
+									{@render sheetGroupRow(group)}
 								{/each}
 							</div>
 						</Collapsible.Content>
@@ -331,38 +230,55 @@
 
 				<button
 					class={sheetItemClass}
-					onclick={() => attachmentMenu.callbacks[AttachmentAction.SYSTEM_PROMPT_CLICK]()}
+					onclick={() => {
+						sheetOpen = false;
+						chatFormActions.onMcpSettingsClick?.();
+					}}
 					type="button"
 				>
-					<MessageSquare class="{ICON_CLASS_DEFAULT} shrink-0" />
+					<McpLogo class="inline {ICON_CLASS_DEFAULT} shrink-0" />
 
-					<span>System Message</span>
+					<span>MCP Servers</span>
 				</button>
-
-				{#if chatFormActions.hasMcpPromptsSupport}
-					<button
-						class={sheetItemClass}
-						onclick={() => attachmentMenu.callbacks[AttachmentAction.MCP_PROMPT_CLICK]()}
-						type="button"
-					>
-						<Zap class="{ICON_CLASS_DEFAULT} shrink-0" />
-
-						<span>MCP Prompt</span>
-					</button>
-				{/if}
-
-				{#if chatFormActions.hasMcpResourcesSupport}
-					<button
-						class={sheetItemClass}
-						onclick={() => attachmentMenu.callbacks[AttachmentAction.MCP_RESOURCES_CLICK]()}
-						type="button"
-					>
-						<FolderOpen class="{ICON_CLASS_DEFAULT} shrink-0" />
-
-						<span>MCP Resources</span>
-					</button>
-				{/if}
 			</div>
 		</Sheet.Content>
 	</Sheet.Root>
 </div>
+
+{#snippet sheetGroupRow(group: ToolGroup)}
+	{@const checkState = toolsPanel.getGroupCheckState(group)}
+	{@const enabledCount = toolsPanel.getEnabledToolCount(group)}
+	{@const favicon = toolsPanel.getFavicon(group)}
+	{@const groupDisabled = toolsPanel.isGroupDisabled(group)}
+
+	<button
+		class="{sheetItemRowClass} {groupDisabled ? 'pointer-events-none opacity-50' : ''}"
+		onclick={() => toolsPanel.toggleGroupByKey(group.key)}
+		type="button"
+	>
+		{#if favicon}
+			<img
+				alt=""
+				class="{ICON_CLASS_DEFAULT} shrink-0 rounded-sm"
+				onerror={(e) => {
+					(e.currentTarget as HTMLImageElement).style.display = 'none';
+				}}
+				src={favicon}
+			/>
+		{/if}
+
+		<span class="min-w-0 flex-1 truncate text-sm font-medium">{group.label}</span>
+
+		<span class="shrink-0 text-xs text-muted-foreground">
+			{enabledCount}/{group.tools.length}
+		</span>
+
+		<Checkbox
+			checked={checkState.checked}
+			class="{ICON_CLASS_DEFAULT} shrink-0"
+			indeterminate={checkState.indeterminate}
+			onCheckedChange={() => toolsPanel.toggleGroupByKey(group.key)}
+			onclick={(e) => e.stopPropagation()}
+		/>
+	</button>
+{/snippet}

tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionAdd/ChatFormActionAddToolsSubmenu.svelte

@@ -7,6 +7,7 @@
 	import { CLI_FLAGS, ICON_CLASS_DEFAULT } from '$lib/constants';
 	import { useToolsPanel } from '$lib/hooks/use-tools-panel.svelte';
 	import { mcpStore, toolsStore } from '$lib/stores';
+	import type { ToolGroup } from '$lib/types';
 
 	const toolsPanel = useToolsPanel();
 	const hasMcpServersAvailable = $derived(mcpStore.getServers().length > 0);
@@ -62,95 +63,108 @@
 			{/if}
 		{:else}
 			<div class="max-h-80 overflow-y-auto p-2 pr-1">
-				{#each toolsPanel.activeGroups as group (group.key)}
-					{@const isExpanded = toolsPanel.expandedGroups.has(group.key)}
-					{@const checked = toolsPanel.isGroupChecked(group)}
-					{@const favicon = toolsPanel.getFavicon(group)}
+				{#each toolsPanel.categoryGroups as group (group.key)}
+					{@render groupRow(group)}
+				{/each}
 
-					<Collapsible.Root
-						onOpenChange={() => toolsPanel.toggleGroupExpanded(group.key)}
-						open={isExpanded}
-					>
-						<div class="flex items-center gap-1">
-							<Collapsible.Trigger
-								class="flex min-w-0 flex-1 items-center gap-2 rounded px-2 py-1.5 text-sm hover:bg-muted/50"
-							>
-								{#if isExpanded}
-									<ChevronDown class="h-3.5 w-3.5 shrink-0" />
-								{:else}
-									<ChevronRight class="h-3.5 w-3.5 shrink-0" />
-								{/if}
-
-								<span class="inline-flex min-w-0 items-center gap-1.5 font-medium">
-									{#if favicon}
-										<img
-											alt=""
-											class="{ICON_CLASS_DEFAULT} shrink-0 rounded-sm"
-											onerror={(e) => {
-												(e.currentTarget as HTMLImageElement).style.display = 'none';
-											}}
-											src={favicon}
-										/>
-									{/if}
-
-									<span class="truncate">{group.label}</span>
-								</span>
-
-								<span class="ml-auto shrink-0 text-xs text-muted-foreground">
-									{toolsPanel.getEnabledToolCount(group)}/{group.tools.length}
-								</span>
-							</Collapsible.Trigger>
-
-							<Tooltip.Root>
-								<Tooltip.Trigger>
-									{#snippet child({ props })}
-										<Checkbox
-											{...props}
-											{checked}
-											class="mr-2 {ICON_CLASS_DEFAULT} shrink-0"
-											onCheckedChange={() => toolsPanel.toggleGroupByKey(group.key)}
-										/>
-									{/snippet}
-								</Tooltip.Trigger>
-
-								<Tooltip.Content side="right">
-									<p>
-										{checked ? 'Disable' : 'Enable'}
-										{group.tools.length} tool{group.tools.length !== 1 ? 's' : ''}
-									</p>
-								</Tooltip.Content>
-							</Tooltip.Root>
-						</div>
-
-						<Collapsible.Content>
-							<div class="ml-4 flex flex-col gap-0.5 border-l border-border/50 pl-2">
-								{#each group.tools as entry (entry.key)}
-									{@const enabled = toolsStore.isToolEnabled(entry.key)}
-									<button
-										class="flex w-full items-center gap-2 rounded px-2 py-1.5 text-left text-sm transition-colors hover:bg-muted/50"
-										onclick={() => toolsStore.toggleTool(entry.key)}
-										type="button"
-									>
-										<span
-											class="flex size-4 shrink-0 items-center justify-center rounded-[4px] border border-input data-[state=checked]:border-primary data-[state=checked]:bg-primary data-[state=checked]:text-primary-foreground"
-											data-slot="checkbox"
-											data-state={enabled ? 'checked' : 'unchecked'}
-										>
-											{#if enabled}
-												<Check class="size-3.5" />
-											{/if}
-										</span>
-
-										<span class="min-w-0 flex-1 truncate font-mono text-[12px]">
-											{entry.definition.function.name}
-										</span>
-									</button>
-								{/each}
-							</div>
-						</Collapsible.Content>
-					</Collapsible.Root>
+				{#each toolsPanel.mcpGroups as group (group.key)}
+					{@render groupRow(group)}
 				{/each}
 			</div>
 		{/if}
 	</DropdownMenu.SubContent>
 </DropdownMenu.Sub>
+
+{#snippet groupRow(group: ToolGroup)}
+	{@const isExpanded = toolsPanel.expandedGroups.has(group.key)}
+	{@const checkState = toolsPanel.getGroupCheckState(group)}
+	{@const favicon = toolsPanel.getFavicon(group)}
+	{@const groupDisabled = toolsPanel.isGroupDisabled(group)}
+
+	<Collapsible.Root
+		onOpenChange={() => toolsPanel.toggleGroupExpanded(group.key)}
+		open={isExpanded}
+	>
+		<div class="flex items-center gap-1 {groupDisabled ? 'pointer-events-none opacity-50' : ''}">
+			<Collapsible.Trigger
+				class="flex min-w-0 flex-1 items-center gap-2 rounded px-2 py-1.5 text-sm hover:bg-muted/50"
+			>
+				{#if isExpanded}
+					<ChevronDown class="h-3.5 w-3.5 shrink-0" />
+				{:else}
+					<ChevronRight class="h-3.5 w-3.5 shrink-0" />
+				{/if}
+
+				<span class="inline-flex min-w-0 items-center gap-1.5 font-medium">
+					{#if favicon}
+						<img
+							alt=""
+							class="{ICON_CLASS_DEFAULT} shrink-0 rounded-sm"
+							onerror={(e) => {
+								(e.currentTarget as HTMLImageElement).style.display = 'none';
+							}}
+							src={favicon}
+						/>
+					{/if}
+
+					<span class="truncate">{group.label}</span>
+				</span>
+
+				<span class="ml-auto shrink-0 text-xs text-muted-foreground">
+					{toolsPanel.getEnabledToolCount(group)}/{group.tools.length}
+				</span>
+			</Collapsible.Trigger>
+
+			<Tooltip.Root>
+				<Tooltip.Trigger>
+					{#snippet child({ props })}
+						<Checkbox
+							{...props}
+							checked={checkState.checked}
+							class="mr-2 {ICON_CLASS_DEFAULT} shrink-0"
+							indeterminate={checkState.indeterminate}
+							onCheckedChange={() => toolsPanel.toggleGroupByKey(group.key)}
+						/>
+					{/snippet}
+				</Tooltip.Trigger>
+
+				<Tooltip.Content side="right">
+					<p>
+						{checkState.checked ? 'Disable' : 'Enable'}
+						{group.tools.length} tool{group.tools.length !== 1 ? 's' : ''}
+					</p>
+				</Tooltip.Content>
+			</Tooltip.Root>
+		</div>
+
+		<Collapsible.Content>
+			<div class="ml-4 flex flex-col gap-0.5 border-l border-border/50 pl-2">
+				{#each group.tools as entry (entry.key)}
+					{@const enabled = toolsPanel.isToolEnabled(entry)}
+					{@const parentDisabled = toolsPanel.isToolParentDisabled(entry)}
+					<button
+						class="flex w-full items-center gap-2 rounded px-2 py-1.5 text-left text-sm transition-colors hover:bg-muted/50 {parentDisabled
+							? 'opacity-50'
+							: ''}"
+						onclick={() => toolsPanel.toggleTool(entry)}
+						type="button"
+					>
+						<span
+							class="flex size-4 shrink-0 items-center justify-center rounded-[4px] border border-input data-[state=checked]:border-primary data-[state=checked]:bg-primary data-[state=checked]:text-primary-foreground"
+							data-slot="checkbox"
+							data-state={enabled ? 'checked' : 'unchecked'}
+						>
+							{#if enabled}
+								<Check class="size-3.5" />
+							{/if}
+						</span>
+
+						<span class="min-w-0 flex-1 truncate font-mono text-[12px]">
+							{entry.definition.function.name}
+						</span>
+					</button>
+				{/each}
+			</div>
+		</Collapsible.Content>
+	</Collapsible.Root>
+{/snippet}

tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActions.svelte

@@ -13,7 +13,7 @@
 	import { setChatFormActionsContext } from '$lib/contexts';
 	import { FileTypeCategory, MessageRole } from '$lib/enums';
 	import { ChatService } from '$lib/services';
-	import { chatStore, conversationsStore, mcpStore, settingsStore } from '$lib/stores';
+	import { chatStore, conversationsStore, settingsStore } from '$lib/stores';
 	import { getFileTypeCategory } from '$lib/utils';
 
 	interface Props {
@@ -31,8 +31,6 @@
 		onMicClick?: () => void;
 		onStop?: () => void;
 		onSystemPromptClick?: () => void;
-		onMcpPromptClick?: () => void;
-		onMcpResourcesClick?: () => void;
 		onMcpSettingsClick?: () => void;
 	}
 
@@ -45,8 +43,6 @@
 		isReasoning = false,
 		isRecording = false,
 		onFileUpload,
-		onMcpPromptClick,
-		onMcpResourcesClick,
 		onMcpSettingsClick,
 		onMicClick,
 		onStop,
@@ -58,18 +54,6 @@
 
 	let currentConfig = $derived(settingsStore.config);
 
-	let hasMcpPromptsSupport = $derived.by(() => {
-		const perChatOverrides = conversationsStore.preferences.getAllMcpServerOverrides();
-
-		return mcpStore.hasPromptsCapability(perChatOverrides);
-	});
-
-	let hasMcpResourcesSupport = $derived.by(() => {
-		const perChatOverrides = conversationsStore.preferences.getAllMcpServerOverrides();
-
-		return mcpStore.hasResourcesCapability(perChatOverrides);
-	});
-
 	let hasAudioModality = $state(false);
 	let hasVideoModality = $state(false);
 	let hasVisionModality = $state(false);
@@ -142,12 +126,6 @@
 		get hasAudioModality() {
 			return hasAudioModality;
 		},
-		get hasMcpPromptsSupport() {
-			return hasMcpPromptsSupport;
-		},
-		get hasMcpResourcesSupport() {
-			return hasMcpResourcesSupport;
-		},
 		get hasVideoModality() {
 			return hasVideoModality;
 		},
@@ -157,12 +135,6 @@
 		get onFileUpload() {
 			return onFileUpload;
 		},
-		get onMcpPromptClick() {
-			return onMcpPromptClick;
-		},
-		get onMcpResourcesClick() {
-			return onMcpResourcesClick;
-		},
 		get onMcpSettingsClick() {
 			return onMcpSettingsClick;
 		},

tools/ui/src/lib/components/app/chat/ChatForm/ChatFormCurrentWorkingDirectory/ChatFormCurrentWorkingDirectory.svelte

@@ -5,12 +5,12 @@
 	import SearchInput from '$lib/components/app/forms/SearchInput.svelte';
 	import * as Popover from '$lib/components/ui/popover';
 	import { DEFAULT_MOBILE_BREAKPOINT, HOME_TILDE, SEARCH, UI_DATA_ATTRS } from '$lib/constants';
-	import { BuiltInTool, GlobSearchType, KeyboardKey } from '$lib/enums';
+	import { BuiltInTool, GlobSearchType, KeyboardKey, ToolSource } from '$lib/enums';
 	import { useDebouncedSearch } from '$lib/hooks/use-debounced-search.svelte';
 	import { usePickerNavigation } from '$lib/hooks/use-picker-navigation.svelte';
 	import { useScrollActiveRow } from '$lib/hooks/use-scroll-active-row.svelte';
 	import { ToolsService } from '$lib/services/tools.service';
-	import { toolsStore } from '$lib/stores';
+	import { conversationsStore, toolsStore } from '$lib/stores';
 	import type { GlobEntry } from '$lib/types';
 	import {
 		abbreviateHome,
@@ -63,8 +63,11 @@
 	// unavailable instead of firing searches that would only fail. Browse is
 	// hidden too: it resolves the picked folder name through the same tool.
 	const fileSearchKey = $derived(toolsStore.getPermissionKey(BuiltInTool.SERVER_FILE_GLOB_SEARCH));
+	// effective policy: the active conversation's tool policy, or global defaults
 	const fileSearchEnabled = $derived(
-		fileSearchKey !== null && toolsStore.isToolEnabled(fileSearchKey)
+		fileSearchKey !== null &&
+			conversationsStore.preferences.isToolEnabled(fileSearchKey) &&
+			conversationsStore.preferences.isCategoryEnabled(ToolSource.SERVER)
 	);
 	const searchUnavailableMessage = $derived(
 		fileSearchKey === null

tools/ui/src/lib/components/app/chat/ChatForm/ChatFormPickers/ChatFormPickerMcpPrompts/ChatFormPickerMcpPrompts.svelte

@@ -9,7 +9,7 @@
 	} from '$lib/components/app/chat';
 	import Badge from '$lib/components/ui/badge/badge.svelte';
 	import { KeyboardKey } from '$lib/enums';
-	import { conversationsStore, mcpStore } from '$lib/stores';
+	import { mcpStore } from '$lib/stores';
 	import type { GetPromptResult, MCPPromptInfo, MCPServerSettingsEntry } from '$lib/types';
 	import { debounce, uuid } from '$lib/utils';
 	import { SvelteMap } from 'svelte/reactivity';
@@ -87,8 +87,7 @@
 		isLoading = true;
 
 		try {
-			const perChatOverrides = conversationsStore.preferences.getAllMcpServerOverrides();
-			const initialized = await mcpStore.ensureInitialized(perChatOverrides);
+			const initialized = await mcpStore.ensureInitialized();
 
 			if (!initialized) {
 				prompts = [];

tools/ui/src/lib/components/app/chat/ChatForm/ChatFormPickers/ChatFormPickerMention.svelte

@@ -5,10 +5,16 @@
 	import * as Popover from '$lib/components/ui/popover';
 	import * as Tooltip from '$lib/components/ui/tooltip';
 	import { FILE_GLOB_SEARCH_PICKERS, HOME_TILDE, SEARCH } from '$lib/constants';
-	import { BuiltInTool, FileMentionEntryType, GlobSearchType, KeyboardKey } from '$lib/enums';
+	import {
+		BuiltInTool,
+		FileMentionEntryType,
+		GlobSearchType,
+		KeyboardKey,
+		ToolSource
+	} from '$lib/enums';
 	import { useDebouncedSearch } from '$lib/hooks/use-debounced-search.svelte';
 	import { usePickerNavigation } from '$lib/hooks/use-picker-navigation.svelte';
-	import { deviceStore, settingsStore, toolsStore } from '$lib/stores';
+	import { conversationsStore, deviceStore, settingsStore, toolsStore } from '$lib/stores';
 	import type { FileMentionEntry, GlobEntryResult } from '$lib/types';
 	import { abbreviateHome, runGlobSearchWithChildren } from '$lib/utils';
 
@@ -52,8 +58,11 @@
 	// --tools) or the user disabled it, the picker still opens but explains
 	// why instead of firing searches that would only fail.
 	const fileSearchKey = $derived(toolsStore.getPermissionKey(BuiltInTool.SERVER_FILE_GLOB_SEARCH));
+	// effective policy: the active conversation's tool policy, or global defaults
 	const fileSearchEnabled = $derived(
-		fileSearchKey !== null && toolsStore.isToolEnabled(fileSearchKey)
+		fileSearchKey !== null &&
+			conversationsStore.preferences.isToolEnabled(fileSearchKey) &&
+			conversationsStore.preferences.isCategoryEnabled(ToolSource.SERVER)
 	);
 
 	let searchResults = $state<FileMentionEntry[]>([]);

tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessageEditForm.svelte

@@ -111,7 +111,6 @@
 		onValueChange={editCtx.setContent}
 		placeholder="Edit your message..."
 		showAddButton={editCtx.messageRole === MessageRole.USER}
-		showMcpPromptButton
 		showModelSelector={editCtx.messageRole === MessageRole.USER}
 		value={editCtx.editedContent}
 	/>

tools/ui/src/lib/components/app/chat/ChatScreen/ChatScreenForm.svelte

@@ -160,6 +160,5 @@
 		onSubmit={handleSubmit}
 		onSystemPromptClick={handleSystemPromptClick}
 		onUploadedFileRemove={handleUploadedFileRemove}
-		showMcpPromptButton
 	/>
 </div>

tools/ui/src/lib/components/app/chat/index.ts

@@ -220,19 +220,6 @@ export { default as ChatFormActionModels } from './ChatForm/ChatFormActions/Chat
  */
 export { default as ChatFormActionAddToolsSubmenu } from './ChatForm/ChatFormActions/ChatFormActionAdd/ChatFormActionAddToolsSubmenu.svelte';
 
-/**
- * Dropdown submenu for MCP prompts and resources in the chat form.
- *
- * Shows an "MCP" sub-menu item with entries for MCP Prompts and MCP
- * Resources. Only visible when the server supports them.
- *
- * @example
- * ```svelte
- * <ChatFormActionAddMcpSubmenu />
- * ```
- */
-export { default as ChatFormActionAddMcpSubmenu } from './ChatForm/ChatFormActions/ChatFormActionAdd/ChatFormActionAddMcpSubmenu.svelte';
-
 /**
  * Dropdown submenu for selecting reasoning effort level.
  *

tools/ui/src/lib/components/app/dialogs/DialogMcpResourcesBrowser.svelte

@@ -8,7 +8,7 @@
 	import { Button } from '$lib/components/ui/button';
 	import * as Dialog from '$lib/components/ui/dialog';
 	import { ICON_CLASS_DEFAULT } from '$lib/constants';
-	import { conversationsStore, mcpStore } from '$lib/stores';
+	import { mcpStore } from '$lib/stores';
 	import type { MCPResourceContent, MCPResourceInfo, MCPResourceTemplateInfo } from '$lib/types';
 	import { getResourceDisplayName } from '$lib/utils';
 	import { SvelteSet } from 'svelte/reactivity';
@@ -48,8 +48,7 @@
 	});
 
 	async function loadResources() {
-		const perChatOverrides = conversationsStore.preferences.getAllMcpServerOverrides();
-		const initialized = await mcpStore.ensureInitialized(perChatOverrides);
+		const initialized = await mcpStore.ensureInitialized();
 
 		if (initialized) {
 			await mcpStore.fetchAllResources();

tools/ui/src/lib/components/app/dialogs/DialogMcpServerAddNew.svelte

@@ -10,7 +10,7 @@
 		RECOMMENDED_MCP_SERVERS
 	} from '$lib/constants';
 	import { BooleanString, HealthCheckStatus } from '$lib/enums';
-	import { conversationsStore, mcpStore } from '$lib/stores';
+	import { mcpStore } from '$lib/stores';
 	import { canonicalizeServerUrl, parseHeadersToArray, uuid } from '$lib/utils';
 
 	interface Props {
@@ -234,8 +234,6 @@
 			useProxy: newServerUseProxy
 		});
 
-		conversationsStore.preferences.setMcpServerOverride(newServerId, true);
-
 		handleOpenChange(false);
 	}
 

tools/ui/src/lib/components/app/dialogs/DialogModelInformation.svelte

@@ -76,22 +76,19 @@
 </script>
 
 <Dialog.Root bind:open {onOpenChange}>
-	<Dialog.Content class="@container z-9999 !max-h-[80dvh] !max-w-[60rem] max-w-full">
-		<style>
-			@container (max-width: 56rem) {
-				.resizable-text-container {
-					max-width: calc(100vw - var(--threshold));
-				}
-			}
-		</style>
+	<Dialog.Content
+		class="z-9999 max-md:h-[100dvh]! max-md:w-screen! max-md:max-w-none! md:w-[calc(100vw-4rem)]! md:max-w-[60rem]! md:max-h-[80dvh]!"
+	>
+		<!-- sticky header holds only the close button; the title scrolls with the body -->
+		<Dialog.Header />
 
-		<Dialog.Header>
-			<Dialog.Title>Model Information</Dialog.Title>
+		<div class="min-w-0 space-y-6 md:py-4 -mt-4! md:mt-0 pb-4">
+			<div class="min-w-0 space-y-2">
+				<Dialog.Title>Model Information</Dialog.Title>
 
-			<Dialog.Description>Current model details and capabilities</Dialog.Description>
-		</Dialog.Header>
+				<Dialog.Description>Current model details and capabilities</Dialog.Description>
+			</div>
 
-		<div class="space-y-6 py-4">
 			{#if isLoadingModels || isLoadingRouterProps}
 				<div class="flex items-center justify-center py-8">
 					<div class="text-sm text-muted-foreground">Loading model information...</div>
@@ -100,17 +97,15 @@
 				{@const modelMeta = firstModel.meta}
 
 				{#if serverProps}
-					<Table.Root>
+					<!-- Desktop: fixed-layout table, long values scroll inside their cell -->
+					<Table.Root class="hidden table-fixed md:table">
 						<Table.Header>
 							<Table.Row>
 								<Table.Head class="w-[10rem]">Model</Table.Head>
 
 								<Table.Head>
-									<div class="inline-flex items-center gap-2">
-										<span
-											style:--threshold="12rem"
-											class="resizable-text-container min-w-0 flex-1 truncate"
-										>
+									<div class="flex min-w-0 items-center gap-2">
+										<span class="min-w-0 flex-1 overflow-x-auto whitespace-nowrap">
 											{modelName}
 										</span>
 
@@ -129,20 +124,17 @@
 							<Table.Row>
 								<Table.Cell class="h-10 align-middle font-medium">File Path</Table.Cell>
 
-								<Table.Cell
-									class="inline-flex h-10 items-center gap-2 align-middle font-mono text-xs"
-								>
-									<span
-										style:--threshold="14rem"
-										class="resizable-text-container min-w-0 flex-1 truncate"
-									>
-										{serverProps.model_path}
-									</span>
+								<Table.Cell class="h-10 align-middle font-mono text-xs">
+									<div class="flex min-w-0 items-center gap-2">
+										<span class="min-w-0 flex-1 overflow-x-auto whitespace-nowrap">
+											{serverProps.model_path}
+										</span>
 
-									<ActionIconCopyToClipboard
-										ariaLabel="Copy model path to clipboard"
-										text={serverProps.model_path}
-									/>
+										<ActionIconCopyToClipboard
+											ariaLabel="Copy model path to clipboard"
+											text={serverProps.model_path}
+										/>
+									</div>
 								</Table.Cell>
 							</Table.Row>
 
@@ -251,18 +243,113 @@
 							<!-- Chat Template -->
 							{#if serverProps.chat_template}
 								<Table.Row>
-									<Table.Cell class="align-middle font-medium">Chat Template</Table.Cell>
+									<Table.Cell class="py-4" colspan={2}>
+										<div class="flex flex-col gap-2">
+											<span class="font-medium">Chat Template</span>
 
-									<Table.Cell class="py-10">
-										<div class="rounded-md bg-muted p-4">
-											<pre
-												class="font-mono text-xs whitespace-pre-wrap">{serverProps.chat_template}</pre>
+											<div class="overflow-x-auto rounded-md bg-muted p-4">
+												<pre
+													class="font-mono text-xs whitespace-pre">{serverProps.chat_template}</pre>
+											</div>
 										</div>
 									</Table.Cell>
 								</Table.Row>
 							{/if}
 						</Table.Body>
 					</Table.Root>
+
+					<!-- Mobile: stacked layout; long values wrap instead of scrolling the page -->
+					<div class="flex min-w-0 flex-col gap-4 md:hidden">
+						<div class="min-w-0 space-y-1">
+							<div class="text-xs font-medium text-muted-foreground">Model</div>
+
+							<div class="flex min-w-0 items-start gap-2">
+								<span class="min-w-0 flex-1 break-all font-mono text-xs">{modelName}</span>
+
+								<ActionIconCopyToClipboard
+									ariaLabel="Copy model name to clipboard"
+									canCopy={!!modelName}
+									text={modelName || ''}
+								/>
+							</div>
+						</div>
+
+						<div class="min-w-0 space-y-1">
+							<div class="text-xs font-medium text-muted-foreground">File Path</div>
+
+							<div class="flex min-w-0 items-start gap-2">
+								<span class="min-w-0 flex-1 break-all font-mono text-xs"
+									>{serverProps.model_path}</span
+								>
+
+								<ActionIconCopyToClipboard
+									ariaLabel="Copy model path to clipboard"
+									text={serverProps.model_path}
+								/>
+							</div>
+						</div>
+
+						{#if serverProps?.default_generation_settings?.n_ctx}
+							{@render infoRow(
+								'Context Size',
+								`${formatNumber(serverProps.default_generation_settings.n_ctx)} tokens`
+							)}
+						{:else}
+							{@render infoRow('Context Size', 'Not available', 'text-red-500')}
+						{/if}
+
+						{#if modelMeta?.n_ctx_train}
+							{@render infoRow('Training Context', `${formatNumber(modelMeta.n_ctx_train)} tokens`)}
+						{/if}
+
+						{#if modelMeta?.size}
+							{@render infoRow('Model Size', formatFileSize(modelMeta.size))}
+						{/if}
+
+						{#if modelMeta?.n_params}
+							{@render infoRow('Parameters', formatParameters(modelMeta.n_params))}
+						{/if}
+
+						{#if modelMeta?.n_embd}
+							{@render infoRow('Embedding Size', formatNumber(modelMeta.n_embd))}
+						{/if}
+
+						{#if modelMeta?.n_vocab}
+							{@render infoRow('Vocabulary Size', `${formatNumber(modelMeta.n_vocab)} tokens`)}
+						{/if}
+
+						{#if modelMeta?.vocab_type}
+							{@render infoRow('Vocabulary Type', modelMeta.vocab_type, 'capitalize')}
+						{/if}
+
+						{@render infoRow('Parallel Slots', `${serverProps.total_slots}`)}
+
+						{#if modalities.length > 0}
+							<div class="min-w-0 space-y-1">
+								<div class="text-xs font-medium text-muted-foreground">Modalities</div>
+
+								<div class="flex flex-wrap gap-1">
+									<BadgesModality {modalities} />
+								</div>
+							</div>
+						{/if}
+
+						<div class="min-w-0 space-y-1">
+							<div class="text-xs font-medium text-muted-foreground">Build Info</div>
+
+							<span class="block break-all font-mono text-xs">{serverProps.build_info}</span>
+						</div>
+
+						{#if serverProps.chat_template}
+							<div class="min-w-0 space-y-2">
+								<div class="text-xs font-medium text-muted-foreground">Chat Template</div>
+
+								<div class="overflow-x-auto rounded-md bg-muted p-4">
+									<pre class="font-mono text-xs whitespace-pre">{serverProps.chat_template}</pre>
+								</div>
+							</div>
+						{/if}
+					</div>
 				{/if}
 			{:else if !isLoadingModels}
 				<div class="flex items-center justify-center py-8">
@@ -272,3 +359,11 @@
 		</div>
 	</Dialog.Content>
 </Dialog.Root>
+
+{#snippet infoRow(label: string, value: string, valueClass: string = '')}
+	<div class="flex items-center justify-between gap-3">
+		<span class="shrink-0 text-xs font-medium text-muted-foreground {valueClass}">{label}</span>
+
+		<span class="text-sm {valueClass}">{value}</span>
+	</div>
+{/snippet}

tools/ui/src/lib/components/app/mcp/McpActiveServersAvatars.svelte

@@ -2,7 +2,7 @@
 	import McpLogo from './McpLogo.svelte';
 	import * as Tooltip from '$lib/components/ui/tooltip';
 	import { ICON_CLASS_DEFAULT, MAX_DISPLAYED_MCP_AVATARS } from '$lib/constants';
-	import { HealthCheckStatus } from '$lib/enums';
+	import { HealthCheckStatus, ToolSource } from '$lib/enums';
 	import { conversationsStore, mcpStore } from '$lib/stores';
 
 	interface Props {
@@ -13,9 +13,13 @@
 	let { class: className = '', onclick }: Props = $props();
 
 	let mcpServers = $derived(mcpStore.getServers().filter((s) => s.enabled));
+	// respect the active conversation's tool policy, not just global enablement
 	let enabledMcpServersForChat = $derived(
 		mcpServers.filter(
-			(s) => conversationsStore.preferences.isMcpServerEnabledForChat(s.id) && s.url.trim()
+			(s) =>
+				s.url.trim() &&
+				conversationsStore.preferences.isCategoryEnabled(ToolSource.MCP) &&
+				conversationsStore.preferences.isServerToolsEnabled(s.id)
 		)
 	);
 	let healthyEnabledMcpServers = $derived(

tools/ui/src/lib/components/app/models/ModelId.svelte

@@ -1,27 +1,44 @@
 <script lang="ts">
 	import { TruncatedText } from '$lib/components/app';
+	import * as Tooltip from '$lib/components/ui/tooltip';
+	import {
+		CAPABILITY_FLAG_KEYS,
+		CAPABILITY_ICONS,
+		CAPABILITY_LABELS,
+		MODALITY_FLAG_KEYS,
+		MODALITY_ICONS,
+		MODALITY_LABELS
+	} from '$lib/constants';
+	import { ModelCapability, ModelModality } from '$lib/enums';
 	import { ModelsService } from '$lib/services/models.service';
 	import { settingsStore } from '$lib/stores';
+	import type { ModelCapabilities, ModelModalities } from '$lib/types/models';
 
 	interface Props {
 		modelId: string;
 		hideOrgName?: boolean;
 		showRaw?: boolean;
+		showRawTooltip?: boolean;
 		hideQuantization?: boolean;
 		hideTags?: boolean;
 		aliases?: string[];
 		tags?: string[];
+		modalities?: ModelModalities;
+		capabilities?: ModelCapabilities;
 		class?: string;
 	}
 
 	let {
 		aliases,
+		capabilities,
 		class: className = '',
 		hideOrgName = false,
 		hideQuantization,
 		hideTags,
+		modalities,
 		modelId,
 		showRaw = undefined,
+		showRawTooltip = false,
 		tags,
 		...rest
 	}: Props = $props();
@@ -43,6 +60,16 @@
 	let uniqueAliases = $derived([...new Set(aliases ?? [])]);
 	let uniqueTags = $derived([...new Set([...(parsed.tags ?? []), ...(tags ?? [])])]);
 
+	const allModalities = [ModelModality.VISION, ModelModality.VIDEO, ModelModality.AUDIO] as const;
+	const allCapabilities: ModelCapability[] = [ModelCapability.REASONING];
+
+	let activeModalities = $derived(
+		allModalities.filter((modality) => modalities?.[MODALITY_FLAG_KEYS[modality]])
+	);
+	let activeCapabilities = $derived(
+		allCapabilities.filter((capability) => capabilities?.[CAPABILITY_FLAG_KEYS[capability]])
+	);
+
 	let primaryAlias = $derived(uniqueAliases.length === 1 ? uniqueAliases[0] : null);
 	let displayName = $derived(primaryAlias ?? parsed.modelName ?? modelId);
 </script>
@@ -50,37 +77,87 @@
 {#if resolvedShowRaw}
 	<TruncatedText class="font-medium {className}" showTooltip={false} text={modelId} {...rest} />
 {:else}
-	<span class="flex min-w-0 flex-wrap items-center gap-1 {className}" {...rest}>
+	{#snippet nameAndBadges()}
 		<span class="min-w-0 truncate font-medium">
 			{#if !hideOrgName && parsed.orgName}{parsed.orgName}/{/if}{displayName}
 		</span>
 
-		{#if parsed.params}
-			<span class={badgeClass}>
-				{parsed.params}{parsed.activatedParams ? `-${parsed.activatedParams}` : ''}
-			</span>
-		{/if}
-
-		{#if parsed.quantization && !resolvedHideQuantization}
-			<span class={badgeClass}>
-				{parsed.quantization}
-			</span>
-		{/if}
-
-		{#if primaryAlias}
-			{#if primaryAlias !== parsed.modelName}
-				<span class={badgeClass}>{parsed.modelName ?? modelId}</span>
+		<span class="inline-flex items-center gap-1">
+			{#if parsed.params}
+				<span class={badgeClass}>
+					{parsed.params}{parsed.activatedParams ? `-${parsed.activatedParams}` : ''}
+				</span>
 			{/if}
-		{:else if uniqueAliases.length > 1}
-			{#each uniqueAliases as alias (alias)}
-				<span class={badgeClass}>{alias}</span>
-			{/each}
+
+			{#if parsed.quantization && !resolvedHideQuantization}
+				<span class={badgeClass}>
+					{parsed.quantization}
+				</span>
+			{/if}
+
+			{#if primaryAlias}
+				{#if primaryAlias !== parsed.modelName}
+					<span class={badgeClass}>{parsed.modelName ?? modelId}</span>
+				{/if}
+			{:else if uniqueAliases.length > 1}
+				{#each uniqueAliases as alias (alias)}
+					<span class={badgeClass}>{alias}</span>
+				{/each}
+			{/if}
+
+			{#if uniqueTags.length > 0 && !resolvedHideTags}
+				{#each uniqueTags as tag (tag)}
+					<span class={tagBadgeClass}>{tag}</span>
+				{/each}
+			{/if}
+		</span>
+	{/snippet}
+
+	<span class="flex min-w-0 items-center gap-1.5 {className}" {...rest}>
+		{#if showRawTooltip}
+			<Tooltip.Root>
+				<Tooltip.Trigger class="flex min-w-0 items-center gap-1.5">
+					{@render nameAndBadges()}
+				</Tooltip.Trigger>
+
+				<Tooltip.Content>
+					<p>{modelId}</p>
+				</Tooltip.Content>
+			</Tooltip.Root>
+		{:else}
+			{@render nameAndBadges()}
 		{/if}
 
-		{#if uniqueTags.length > 0 && !resolvedHideTags}
-			{#each uniqueTags as tag (tag)}
-				<span class={tagBadgeClass}>{tag}</span>
-			{/each}
+		{#if activeCapabilities.length > 0 || activeModalities.length > 0}
+			<span class="inline-flex items-center gap-1.25 text-muted-foreground">
+				{#each activeCapabilities as capability (capability)}
+					{@const CapabilityIcon = CAPABILITY_ICONS[capability]}
+
+					<Tooltip.Root>
+						<Tooltip.Trigger>
+							<CapabilityIcon class="h-3 w-3 text-muted-foreground" />
+						</Tooltip.Trigger>
+
+						<Tooltip.Content>
+							<p>{CAPABILITY_LABELS[capability]}</p>
+						</Tooltip.Content>
+					</Tooltip.Root>
+				{/each}
+
+				{#each activeModalities as modality (modality)}
+					{@const ModalityIcon = MODALITY_ICONS[modality]}
+
+					<Tooltip.Root>
+						<Tooltip.Trigger>
+							<ModalityIcon class="h-3 w-3 text-muted-foreground" />
+						</Tooltip.Trigger>
+
+						<Tooltip.Content>
+							<p>{MODALITY_LABELS[modality]}</p>
+						</Tooltip.Content>
+					</Tooltip.Root>
+				{/each}
+			</span>
 		{/if}
 	</span>
 {/if}

tools/ui/src/lib/components/app/models/ModelsSelectorDropdown.svelte

@@ -1,8 +1,9 @@
 <script lang="ts">
 	import ModelLoadHighlight from './ModelLoadHighlight.svelte';
 	import type { ModelItem } from './utils';
-	import { ChevronDown, Loader2 } from '@lucide/svelte';
+	import { ChevronDown, Lightbulb, Loader2 } from '@lucide/svelte';
 	import {
+		ChatFormActionAddReasoningSubmenu,
 		DialogModelInformation,
 		DropdownMenuSearchable,
 		ModelId,
@@ -11,10 +12,11 @@
 	} from '$lib/components/app';
 	import * as DropdownMenu from '$lib/components/ui/dropdown-menu';
 	import * as Tooltip from '$lib/components/ui/tooltip';
-	import { MODEL_SELECTOR_ICON } from '$lib/constants';
+	import { MODEL_SELECTOR_ICON, SETTINGS_KEYS } from '$lib/constants';
 	import { KeyboardKey, ServerModelStatus } from '$lib/enums';
 	import { useModelsSelector } from '$lib/hooks/use-models-selector.svelte';
-	import { modelsStore } from '$lib/stores';
+	import { useReasoningMenu } from '$lib/hooks/use-reasoning-menu.svelte';
+	import { modelsStore, settingsStore } from '$lib/stores';
 	import { modelLoadFraction } from '$lib/utils';
 
 	interface Props {
@@ -37,6 +39,9 @@
 
 	let isOpen = $state(false);
 	let highlightedId = $state<string | null>(null);
+	// The model submenu opens together with the menu so the list and its search
+	// box are immediately available, as before the submenu was introduced
+	let modelSubOpen = $state(false);
 
 	const ms = useModelsSelector({
 		currentModel: () => currentModel,
@@ -44,24 +49,41 @@
 		onOpenChange: (open) => {
 			isOpen = open;
 			highlightedId = null;
+
+			if (open) {
+				// Defer submenu open so the Sub component is mounted first;
+				// setting bind:open synchronously can be lost if the Sub hasn't
+				// rendered yet.
+				queueMicrotask(() => {
+					if (isOpen) modelSubOpen = true;
+				});
+			} else {
+				modelSubOpen = false;
+			}
 		},
 		useGlobalSelection: () => useGlobalSelection
 	});
 
+	const reasoning = useReasoningMenu();
+
+	const showOrgNameInTrigger = $derived(
+		settingsStore.config[SETTINGS_KEYS.SHOW_MODEL_ORG_NAME_IN_TRIGGER] ?? false
+	);
+
 	$effect(() => {
 		void ms.searchTerm;
 		highlightedId = null;
 	});
 
-	// Focus the dropdown's search box without scrolling the page. bits-ui
+	// Focus the model submenu's search box without scrolling the page. bits-ui
 	// auto-focuses the opened content by default, which can yank the page
 	// scroll; we prevent that on the Content and refocus the search here.
 	$effect(() => {
-		if (!isOpen) return;
+		if (!isOpen || !modelSubOpen) return;
 
 		requestAnimationFrame(() => {
 			const search = document.querySelector<HTMLElement>(
-				'[data-slot="dropdown-menu-content"] input'
+				'[data-slot="dropdown-menu-sub-content"] input'
 			);
 
 			search?.focus({ preventScroll: true });
@@ -188,7 +210,7 @@
 							<DropdownMenu.Trigger
 								{...props}
 								class={[
-									`relative inline-grid cursor-pointer grid-cols-[1fr_auto_1fr] items-center gap-1.5 rounded-sm bg-background px-1.5 py-1 text-xs shadow-sm transition hover:bg-muted-foreground/20 focus:outline-none focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2 disabled:cursor-not-allowed disabled:opacity-60 dark:bg-muted-foreground/15 dark:text-secondary-foreground`,
+									`relative inline-grid cursor-pointer grid-cols-[1fr_auto_1fr] items-center gap-1 rounded-sm bg-background px-1.5 py-1 text-xs shadow-sm transition hover:bg-muted-foreground/20 focus:outline-none focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2 disabled:cursor-not-allowed disabled:opacity-60 dark:bg-muted-foreground/15 dark:text-secondary-foreground`,
 									!ms.isCurrentModelInCache
 										? 'bg-red-400/10 !text-red-400 hover:bg-red-400/20 hover:text-red-400'
 										: forceForegroundText
@@ -203,16 +225,22 @@
 							>
 								<MODEL_SELECTOR_ICON class="h-3.5 w-3.5 shrink-0" />
 
-								{#if selectedOption}
-									<ModelId
-										class="min-w-0 overflow-hidden"
-										hideOrgName={false}
-										hideQuantization
-										modelId={selectedOption.model}
-									/>
-								{:else}
-									<span class="min-w-0 font-medium">Select model</span>
-								{/if}
+								<span class="flex min-w-0 items-center gap-1">
+									{#if selectedOption}
+										<ModelId
+											class="min-w-0 overflow-hidden"
+											hideOrgName={!showOrgNameInTrigger}
+											hideQuantization
+											modelId={selectedOption.model}
+										/>
+									{:else}
+										<span class="min-w-0 font-medium">Select model</span>
+									{/if}
+
+									{#if reasoning.isReasoningActive}
+										<Lightbulb class="h-3.5 w-3.5 shrink-0 text-amber-400" />
+									{/if}
+								</span>
 
 								{#if ms.updating || ms.isLoadingModel}
 									<Loader2 class="h-3 w-3.5 shrink-0 animate-spin" />
@@ -236,73 +264,94 @@
 
 				<DropdownMenu.Content
 					align="end"
-					class="w-full max-w-[100vw] pt-0 sm:w-max sm:max-w-[calc(100vw-2rem)]"
+					class="w-full md:min-w-64 md:max-w-80 max-w-[calc(100vw-2rem)]"
 					onOpenAutoFocus={(event) => event.preventDefault()}
 				>
-					<DropdownMenuSearchable
-						emptyMessage="No models found."
-						isEmpty={ms.filteredOptions.length === 0 && ms.isCurrentModelInCache}
-						onSearchChange={(v) => ms.setSearchTerm(v)}
-						onSearchKeyDown={handleSearchKeyDown}
-						placeholder="Search models..."
-						searchValue={ms.searchTerm}
-					>
-						<div class="models-list">
-							{#if !ms.isCurrentModelInCache && currentModel}
-								<!-- Show unavailable model as first option (disabled) -->
-								<button
-									aria-disabled="true"
-									aria-selected="true"
-									class="flex w-full cursor-not-allowed items-center bg-red-400/10 p-2 text-left text-sm text-red-400"
-									disabled
-									role="option"
-									type="button"
-								>
-									<ModelId class="flex-1" hideQuantization modelId={currentModel} />
+					<DropdownMenu.Sub bind:open={modelSubOpen}>
+						<DropdownMenu.SubTrigger class="flex cursor-pointer items-center gap-2">
+							<MODEL_SELECTOR_ICON class="h-4 w-4" />
 
-									<span class="ml-2 text-xs whitespace-nowrap opacity-70">(not available)</span>
-								</button>
-							{/if}
-
-							{#if ms.filteredOptions.length === 0}
-								<p class="px-4 py-3 text-sm text-muted-foreground">No models found.</p>
-							{/if}
-
-							{#snippet modelOption(item: ModelItem, hideOrgName: boolean)}
-								{@const { option } = item}
-								{@const isSelected = currentModel === option.model || ms.activeId === option.id}
-								{@const isHighlighted = option.id === highlightedId}
-								{@const isFav = ms.isFavorite(option.model)}
-
-								<ModelsSelectorOption
-									{hideOrgName}
-									{isFav}
-									{isHighlighted}
-									{isSelected}
-									onInfoClick={ms.handleInfoClick}
-									onKeyDown={(event) => {
-										if (event.key === KeyboardKey.ENTER || event.key === KeyboardKey.SPACE) {
-											event.preventDefault();
-											void handleModelKeyAction(option.id, event.altKey);
-										}
-									}}
-									onMouseEnter={() => (highlightedId = option.id)}
-									onSelect={ms.handleSelect}
-									{option}
+							{#if selectedOption}
+								<ModelId
+									class="min-w-0 flex-1 overflow-hidden"
+									hideOrgName={!showOrgNameInTrigger}
+									hideQuantization
+									modelId={selectedOption.model}
 								/>
-							{/snippet}
+							{:else}
+								<span class="min-w-0 flex-1 truncate text-muted-foreground">No model</span>
+							{/if}
+						</DropdownMenu.SubTrigger>
 
-							<ModelsSelectorList
-								activeId={ms.activeId}
-								{currentModel}
-								groups={ms.groupedFilteredOptions}
-								onInfoClick={ms.handleInfoClick}
-								onSelect={ms.handleSelect}
-								renderOption={modelOption}
-								sectionHeaderClass="my-1.5 px-2 py-2 text-[13px] font-semibold text-muted-foreground/70 select-none"
-							/>
-						</div>
-					</DropdownMenuSearchable>
+						<DropdownMenu.SubContent class="w-100 max-w-[calc(100vw-2rem)] pt-0">
+							<DropdownMenuSearchable
+								emptyMessage="No models found."
+								isEmpty={ms.filteredOptions.length === 0 && ms.isCurrentModelInCache}
+								onSearchChange={(v) => ms.setSearchTerm(v)}
+								onSearchKeyDown={handleSearchKeyDown}
+								placeholder="Search models..."
+								searchValue={ms.searchTerm}
+							>
+								<div class="models-list">
+									{#if !ms.isCurrentModelInCache && currentModel}
+										<!-- Show unavailable model as first option (disabled) -->
+										<button
+											aria-disabled="true"
+											aria-selected="true"
+											class="flex w-full cursor-not-allowed items-center bg-red-400/10 p-2 text-left text-sm text-red-400"
+											disabled
+											role="option"
+											type="button"
+										>
+											<ModelId class="flex-1" hideQuantization modelId={currentModel} />
+
+											<span class="ml-2 text-xs whitespace-nowrap opacity-70">(not available)</span>
+										</button>
+									{/if}
+
+									{#if ms.filteredOptions.length === 0}
+										<p class="px-4 py-3 text-sm text-muted-foreground">No models found.</p>
+									{/if}
+
+									{#snippet modelOption(item: ModelItem, hideOrgName: boolean)}
+										{@const { option } = item}
+										{@const isSelected = currentModel === option.model || ms.activeId === option.id}
+										{@const isHighlighted = option.id === highlightedId}
+										{@const isFav = ms.isFavorite(option.model)}
+
+										<ModelsSelectorOption
+											{hideOrgName}
+											{isFav}
+											{isHighlighted}
+											{isSelected}
+											onInfoClick={ms.handleInfoClick}
+											onKeyDown={(event) => {
+												if (event.key === KeyboardKey.ENTER || event.key === KeyboardKey.SPACE) {
+													event.preventDefault();
+													void handleModelKeyAction(option.id, event.altKey);
+												}
+											}}
+											onMouseEnter={() => (highlightedId = option.id)}
+											onSelect={ms.handleSelect}
+											{option}
+										/>
+									{/snippet}
+
+									<ModelsSelectorList
+										activeId={ms.activeId}
+										{currentModel}
+										groups={ms.groupedFilteredOptions}
+										onInfoClick={ms.handleInfoClick}
+										onSelect={ms.handleSelect}
+										renderOption={modelOption}
+										sectionHeaderClass="my-1.5 px-2 py-2 text-[13px] font-semibold text-muted-foreground/70 select-none"
+									/>
+								</div>
+							</DropdownMenuSearchable>
+						</DropdownMenu.SubContent>
+					</DropdownMenu.Sub>
+
+					<ChatFormActionAddReasoningSubmenu />
 				</DropdownMenu.Content>
 			</DropdownMenu.Root>
 		{:else}
@@ -332,12 +381,16 @@
 							{#if selectedOption}
 								<ModelId
 									class="min-w-0 overflow-hidden"
-									hideOrgName={false}
+									hideOrgName={!showOrgNameInTrigger}
 									hideQuantization
 									modelId={selectedOption.model}
 								/>
 							{/if}
 
+							{#if reasoning.isReasoningActive}
+								<Lightbulb class="h-3.5 w-3.5 shrink-0 text-amber-400" />
+							{/if}
+
 							{#if ms.updating}
 								<Loader2 class="h-3 w-3.5 shrink-0 animate-spin" />
 							{/if}

tools/ui/src/lib/components/app/models/ModelsSelectorOption.svelte

@@ -58,6 +58,10 @@
 	let loadProgress = $derived(isLoading ? modelsStore.status.getLoadProgress(option.model) : null);
 	let loadPercent = $derived(Math.round(modelLoadFraction(loadProgress) * 100));
 	let loadTitle = $derived(modelLoadProgressText(loadProgress));
+	let modalities = $derived(option.modalities);
+	let capabilities = $derived.by(() => ({
+		reasoning: modelsStore.props.checkModelSupportsThinking(option.model)
+	}));
 </script>
 
 <div
@@ -65,9 +69,11 @@
 	class={[
 		'group relative flex w-full items-center gap-2 rounded-sm p-2 text-left text-sm transition focus:outline-none',
 		'cursor-pointer',
-		isSelected && 'bg-accent/50 text-accent-foreground',
+		isSelected && !isHighlighted && 'bg-accent/50',
 		isHighlighted && 'bg-accent',
-		!isSelected && !isHighlighted && 'hover:bg-muted',
+		(isSelected || isHighlighted) && 'text-accent-foreground',
+		'hover:bg-accent',
+		'focus:bg-accent',
 		isLoaded ? 'text-popover-foreground' : 'text-muted-foreground'
 	]}
 	onclick={() => onSelect(option.id)}
@@ -79,9 +85,12 @@
 >
 	<ModelId
 		aliases={option.aliases}
+		{capabilities}
 		class="flex-1"
 		{hideOrgName}
+		{modalities}
 		modelId={option.model}
+		showRawTooltip
 		tags={option.tags}
 	/>
 

tools/ui/src/lib/components/app/models/utils.ts

@@ -1,3 +1,4 @@
+import { ModelModality } from '$lib/enums';
 import type { ModelOption } from '$lib/types/models';
 import { SvelteMap } from 'svelte/reactivity';
 
@@ -17,6 +18,23 @@ export interface GroupedModelOptions {
 	available: OrgGroup[];
 }
 
+function matchesModality(option: ModelOption, term: string): boolean {
+	const modalities = option.modalities;
+
+	if (!modalities) return false;
+
+	switch (term) {
+		case ModelModality.VISION.toLowerCase():
+			return modalities.vision;
+		case ModelModality.AUDIO.toLowerCase():
+			return modalities.audio;
+		case ModelModality.VIDEO.toLowerCase():
+			return modalities.video;
+		default:
+			return false;
+	}
+}
+
 export function filterModelOptions(options: ModelOption[], searchTerm: string): ModelOption[] {
 	const term = searchTerm.trim().toLowerCase();
 
@@ -27,7 +45,8 @@ export function filterModelOptions(options: ModelOption[], searchTerm: string):
 			option.model.toLowerCase().includes(term) ||
 			option.name?.toLowerCase().includes(term) ||
 			option.aliases?.some((alias: string) => alias.toLowerCase().includes(term)) ||
-			option.tags?.some((tag: string) => tag.toLowerCase().includes(term))
+			option.tags?.some((tag: string) => tag.toLowerCase().includes(term)) ||
+			matchesModality(option, term)
 	);
 }
 

tools/ui/src/lib/components/app/settings/SettingsChat/SettingsChatToolsTab.svelte

@@ -25,6 +25,10 @@
 	<div class="py-8 text-center text-sm text-muted-foreground">No tools available</div>
 {:else}
 	<div class="space-y-2">
+		<p class="text-sm text-muted-foreground">
+			Applies to new conversations. Tool picks inside a chat only affect that chat.
+		</p>
+
 		{#each groups as group (group.key)}
 			{@const isExpanded = expandedGroups.has(group.key)}
 			<Collapsible.Root onOpenChange={() => toggleExpanded(group.key)} open={isExpanded}>
@@ -37,6 +41,17 @@
 						<ChevronRight class="h-3.5 w-3.5 shrink-0" />
 					{/if}
 
+					{@const isCategoryEnabled =
+						group.source !== ToolSource.MCP && toolsStore.isCategoryEnabled(group.source)}
+
+					{#if group.source !== ToolSource.MCP}
+						<Checkbox
+							checked={isCategoryEnabled}
+							onCheckedChange={() => toolsStore.toggleCategory(group.source)}
+							onclick={(e) => e.stopPropagation()}
+						/>
+					{/if}
+
 					{@const faviconUrl = group.serverId ? mcpStore.getServerFavicon(group.serverId) : null}
 
 					<span class="inline-flex min-w-0 items-center gap-1.5 font-medium">

tools/ui/src/lib/components/app/settings/SettingsMcpServers.svelte

@@ -7,7 +7,7 @@
 	import { Button } from '$lib/components/ui/button';
 	import * as Empty from '$lib/components/ui/empty';
 	import { HealthCheckStatus } from '$lib/enums';
-	import { conversationsStore, mcpStore, toolsStore } from '$lib/stores';
+	import { mcpStore, toolsStore } from '$lib/stores';
 	import { onMount } from 'svelte';
 	import { fade } from 'svelte/transition';
 
@@ -86,15 +86,13 @@
 					<McpServerCardSkeleton />
 				{:else}
 					<McpServerCard
-						enabled={conversationsStore.preferences.isMcpServerEnabledForChat(server.id)}
+						enabled={server.enabled}
 						onBrowseResources={() => (isResourcesDialogOpen = true)}
 						onDelete={() => mcpStore.removeServer(server.id)}
 						onToggle={async () => {
-							const wasEnabled = conversationsStore.preferences.isMcpServerEnabledForChat(
-								server.id
-							);
+							const wasEnabled = server.enabled;
 
-							await conversationsStore.preferences.toggleMcpServerForChat(server.id);
+							mcpStore.updateServer(server.id, { enabled: !wasEnabled });
 
 							if (!wasEnabled) {
 								// Promote the connection so tools/prompts/resources become

tools/ui/src/lib/components/ui/checkbox/checkbox.svelte

@@ -26,10 +26,10 @@
 >
 	{#snippet children({ checked, indeterminate })}
 		<div class="text-current transition-none" data-slot="checkbox-indicator">
-			{#if checked}
-				<CheckIcon class="size-3.5" />
-			{:else if indeterminate}
+			{#if indeterminate}
 				<MinusIcon class="size-3.5" />
+			{:else if checked}
+				<CheckIcon class="size-3.5" />
 			{/if}
 		</div>
 	{/snippet}

tools/ui/src/lib/constants/attachment-menu.constants.ts

@@ -1,11 +1,5 @@
-import { FolderOpen, MessageSquare, Zap } from '@lucide/svelte';
 import { FILE_TYPE_ICONS } from '$lib/constants';
-import {
-	AttachmentAction,
-	AttachmentItemEnabledWhen,
-	AttachmentItemVisibleWhen,
-	AttachmentMenuItemId
-} from '$lib/enums';
+import { AttachmentAction, AttachmentItemEnabledWhen, AttachmentMenuItemId } from '$lib/enums';
 import type { AttachmentMenuItem } from '$lib/types';
 
 /**
@@ -58,36 +52,4 @@ export const ATTACHMENT_FILE_ITEMS: AttachmentMenuItem[] = [
 	}
 ];
 
-export const ATTACHMENT_EXTRA_ITEMS: AttachmentMenuItem[] = [];
-
-export const ATTACHMENT_PROMPT_ITEMS: AttachmentMenuItem[] = [
-	{
-		action: AttachmentAction.SYSTEM_PROMPT_CLICK,
-		enabledWhen: AttachmentItemEnabledWhen.ALWAYS,
-		hasEnabledTooltip: true,
-		icon: MessageSquare,
-		id: AttachmentMenuItemId.SYSTEM_MESSAGE,
-		label: 'System Message'
-	},
-	{
-		action: AttachmentAction.MCP_PROMPT_CLICK,
-		enabledWhen: AttachmentItemEnabledWhen.ALWAYS,
-		icon: Zap,
-		id: AttachmentMenuItemId.MCP_PROMPT,
-		label: 'MCP Prompts',
-		visibleWhen: AttachmentItemVisibleWhen.HAS_MCP_PROMPTS_SUPPORT
-	}
-];
-
-export const ATTACHMENT_MCP_ITEMS: AttachmentMenuItem[] = [
-	{
-		action: AttachmentAction.MCP_RESOURCES_CLICK,
-		enabledWhen: AttachmentItemEnabledWhen.ALWAYS,
-		icon: FolderOpen,
-		id: AttachmentMenuItemId.MCP_RESOURCES,
-		label: 'MCP Resources',
-		visibleWhen: AttachmentItemVisibleWhen.HAS_MCP_RESOURCES_SUPPORT
-	}
-];
-
 export const ATTACHMENT_TOOLTIP_TEXT = 'Add files, prompts, tools or MCP Servers';

tools/ui/src/lib/constants/icons.constants.ts

@@ -8,10 +8,13 @@ import {
 	File as FileIcon,
 	FileText as FileTextIcon,
 	Image as ImageIcon,
+	Lightbulb as ReasoningIcon,
 	Mic as AudioIcon,
 	Video as VideoIcon
 } from '@lucide/svelte';
-import { FileTypeCategory, ModelModality } from '$lib/enums';
+import { FileTypeCategory, ModelCapability, ModelModality } from '$lib/enums';
+import type { ModelCapabilities, ModelModalities } from '$lib/types/models';
+import type { Component } from 'svelte';
 
 export const FILE_TYPE_ICONS = {
 	[FileTypeCategory.AUDIO]: AudioIcon,
@@ -35,6 +38,29 @@ export const MODALITY_LABELS = {
 	[ModelModality.VISION]: 'Vision'
 } as const;
 
+/** Maps an input ModelModality to the boolean flag it drives on the ModelModalities type */
+export const MODALITY_FLAG_KEYS: Record<
+	Exclude<ModelModality, ModelModality.TEXT>,
+	keyof ModelModalities
+> = {
+	[ModelModality.AUDIO]: 'audio',
+	[ModelModality.VIDEO]: 'video',
+	[ModelModality.VISION]: 'vision'
+};
+
+export const CAPABILITY_ICONS: Record<ModelCapability, Component> = {
+	[ModelCapability.REASONING]: ReasoningIcon
+} as const;
+
+export const CAPABILITY_LABELS: Record<ModelCapability, string> = {
+	[ModelCapability.REASONING]: 'Reasoning'
+} as const;
+
+/** Maps a ModelCapability to the boolean flag it drives on the ModelCapabilities type */
+export const CAPABILITY_FLAG_KEYS: Record<ModelCapability, keyof ModelCapabilities> = {
+	[ModelCapability.REASONING]: 'reasoning'
+};
+
 // Shared SVG icon strings for copy and preview buttons
 export const COPY_ICON_SVG = `<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-copy-icon lucide-copy"><rect width="14" height="14" x="8" y="8" rx="2" ry="2"/><path d="M4 16c-1.1 0-2-.9-2-2V4c0-1.1.9-2 2-2h10c1.1 0 2 .9 2 2"/></svg>`;
 

tools/ui/src/lib/constants/settings-keys.constants.ts

@@ -54,6 +54,7 @@ export const SETTINGS_KEYS = {
 	SHOW_FULL_PATH_IN_MENTIONS: 'showFullPathInMentions',
 	// Display
 	SHOW_MESSAGE_STATS: 'showMessageStats',
+	SHOW_MODEL_ORG_NAME_IN_TRIGGER: 'showModelOrgNameInTrigger',
 	SHOW_MODEL_QUANTIZATION: 'showModelQuantization',
 	SHOW_MODEL_TAGS: 'showModelTags',
 	SHOW_RAW_MODEL_NAMES: 'showRawModelNames',

tools/ui/src/lib/constants/settings.constants.ts

@@ -111,9 +111,8 @@ export const SETTINGS_REGISTRY: SettingsSectionEntry[] = [
 				type: SettingsFieldType.CHECKBOX
 			},
 			{
-				defaultValue: false,
+				defaultValue: true,
 				help: 'Automatically show microphone button instead of send button when textarea is empty for models with audio modality support.',
-				isExperimental: true,
 				key: SETTINGS_KEYS.AUTO_MIC_ON_EMPTY,
 				label: 'Show microphone on empty input',
 				type: SettingsFieldType.CHECKBOX
@@ -283,6 +282,13 @@ export const SETTINGS_REGISTRY: SettingsSectionEntry[] = [
 				label: 'Show model tags',
 				type: SettingsFieldType.CHECKBOX
 			},
+			{
+				defaultValue: false,
+				help: 'Display the organization name in the model selector trigger button.',
+				key: SETTINGS_KEYS.SHOW_MODEL_ORG_NAME_IN_TRIGGER,
+				label: 'Show organization name in model selector trigger',
+				type: SettingsFieldType.CHECKBOX
+			},
 			{
 				defaultValue: false,
 				help: 'Display the current build version in the bottom-right corner of the interface.',

tools/ui/src/lib/constants/storage.constants.ts

@@ -20,6 +20,9 @@ export const DISABLED_TOOLS_LOCALSTORAGE_KEY = `${STORAGE_APP_NAME}.disabledTool
 
 /** Disabled tools keyed by stable selection identity, no migration from the name based key */
 export const DISABLED_TOOL_KEYS_LOCALSTORAGE_KEY = `${STORAGE_APP_NAME}.disabledToolKeys`;
+
+/** Default disabled tool categories, seeded into newly created conversations */
+export const DISABLED_TOOL_CATEGORIES_LOCALSTORAGE_KEY = `${STORAGE_APP_NAME}.disabledToolCategories`;
 export const FAVORITE_MODELS_LOCALSTORAGE_KEY = `${STORAGE_APP_NAME}.favoriteModels`;
 export const REASONING_EFFORT_DEFAULT_LOCALSTORAGE_KEY = `${STORAGE_APP_NAME}.reasoningEffortDefault`;
 export const CONVERSATION_TABS_LOCALSTORAGE_KEY = `${STORAGE_APP_NAME}.conversationTabs`;

tools/ui/src/lib/enums/attachment.enums.ts

@@ -19,8 +19,6 @@ export enum AttachmentType {
 export enum AttachmentMenuItemId {
 	AUDIO = 'audio',
 	IMAGES = 'images',
-	MCP_PROMPT = 'mcp-prompt',
-	MCP_RESOURCES = 'mcp-resources',
 	PDF = 'pdf',
 	SYSTEM_MESSAGE = 'system-message',
 	TEXT = 'text',
@@ -42,8 +40,6 @@ export enum AttachmentItemEnabledWhen {
  */
 export enum AttachmentAction {
 	FILE_UPLOAD = 'onFileUpload',
-	MCP_PROMPT_CLICK = 'onMcpPromptClick',
-	MCP_RESOURCES_CLICK = 'onMcpResourcesClick',
 	SYSTEM_PROMPT_CLICK = 'onSystemPromptClick'
 }
 
@@ -56,11 +52,3 @@ export enum AttachmentLabel {
 	MCP_RESOURCE = 'MCP Resource',
 	PDF_FILE = 'PDF File'
 }
-
-/**
- * Visibility conditions for attachment menu items.
- */
-export enum AttachmentItemVisibleWhen {
-	HAS_MCP_PROMPTS_SUPPORT = 'hasMcpPromptsSupport',
-	HAS_MCP_RESOURCES_SUPPORT = 'hasMcpResourcesSupport'
-}

tools/ui/src/lib/enums/index.ts

@@ -3,8 +3,7 @@ export {
 	AttachmentType,
 	AttachmentMenuItemId,
 	AttachmentItemEnabledWhen,
-	AttachmentAction,
-	AttachmentItemVisibleWhen
+	AttachmentAction
 } from './attachment.enums';
 
 export {
@@ -68,7 +67,7 @@ export {
 	JsonSchemaType
 } from './mcp.enums';
 
-export { ModelModality } from './model.enums';
+export { ModelCapability, ModelModality } from './model.enums';
 
 export { ServerRole, ServerModelStatus, ServerModelsSseEventType } from './server.enums';
 

tools/ui/src/lib/enums/model.enums.ts

@@ -4,3 +4,7 @@ export enum ModelModality {
 	VIDEO = 'VIDEO',
 	VISION = 'VISION'
 }
+
+export enum ModelCapability {
+	REASONING = 'REASONING'
+}

tools/ui/src/lib/hooks/use-attachment-menu.svelte.ts

@@ -5,21 +5,16 @@ export interface AttachmentModalityFlags {
 	hasVisionModality: boolean;
 	hasAudioModality: boolean;
 	hasVideoModality: boolean;
-	hasMcpPromptsSupport: boolean;
-	hasMcpResourcesSupport: boolean;
 }
 
 export interface AttachmentActionCallbacks {
 	onFileUpload?: () => void;
 	onSystemPromptClick?: () => void;
-	onMcpPromptClick?: () => void;
-	onMcpResourcesClick?: () => void;
 }
 
 export interface UseAttachmentMenuReturn {
 	readonly callbacks: Record<string, () => void>;
 	isItemEnabled(enabledWhen: string | undefined): boolean;
-	isItemVisible(visibleWhen: string | undefined): boolean;
 	getSystemMessageTooltip(): string;
 }
 
@@ -49,8 +44,6 @@ export function useAttachmentMenu(
 
 		return {
 			[AttachmentAction.FILE_UPLOAD]: wrap(cbs.onFileUpload),
-			[AttachmentAction.MCP_PROMPT_CLICK]: wrap(cbs.onMcpPromptClick),
-			[AttachmentAction.MCP_RESOURCES_CLICK]: wrap(cbs.onMcpResourcesClick),
 			[AttachmentAction.SYSTEM_PROMPT_CLICK]: wrap(cbs.onSystemPromptClick)
 		};
 	});
@@ -61,12 +54,6 @@ export function useAttachmentMenu(
 		return !!modalityFlags[enabledWhen as keyof AttachmentModalityFlags];
 	}
 
-	function isItemVisible(visibleWhen: string | undefined): boolean {
-		if (!visibleWhen) return true;
-
-		return !!modalityFlags[visibleWhen as keyof AttachmentModalityFlags];
-	}
-
 	function getSystemMessageTooltip(): string {
 		return !page.params.id
 			? 'Add custom system message for a new conversation'
@@ -78,7 +65,6 @@ export function useAttachmentMenu(
 			return callbacks;
 		},
 		getSystemMessageTooltip,
-		isItemEnabled,
-		isItemVisible
+		isItemEnabled
 	};
 }

tools/ui/src/lib/hooks/use-reasoning-menu.svelte.ts

@@ -8,6 +8,7 @@ import { getConversationModel } from '$lib/utils';
 export interface UseReasoningMenuReturn {
 	readonly modelSupportsThinking: boolean;
 	readonly thinkingEnabled: boolean;
+	readonly isReasoningActive: boolean;
 	readonly isOff: boolean;
 	readonly currentEffort: ReasoningEffort;
 	readonly levels: ReasoningEffortLevel[];
@@ -59,6 +60,12 @@ export function useReasoningMenu(): UseReasoningMenuReturn {
 	const thinkingEnabled = $derived(
 		currentEffort !== ReasoningEffort.OFF && currentEffort !== ReasoningEffort.DEFAULT
 	);
+	// Thinking is effectively on (lightbulb lit) either when an explicit effort
+	// is selected, or when the effort is left at "Default" and the model
+	// supports thinking.
+	const isReasoningActive = $derived(
+		thinkingEnabled || (currentEffort === ReasoningEffort.DEFAULT && modelSupportsThinking)
+	);
 
 	return {
 		get currentEffort() {
@@ -67,6 +74,9 @@ export function useReasoningMenu(): UseReasoningMenuReturn {
 		get isOff() {
 			return currentEffort === ReasoningEffort.OFF;
 		},
+		get isReasoningActive() {
+			return isReasoningActive;
+		},
 		isSelected(level: ReasoningEffortLevel): boolean {
 			return currentEffort === level.value;
 		},

tools/ui/src/lib/hooks/use-tools-panel.svelte.ts

@@ -1,19 +1,23 @@
 import { CLI_FLAGS } from '$lib/constants';
 import { ToolSource } from '$lib/enums';
 import { conversationsStore, mcpStore, toolsStore } from '$lib/stores';
-import type { ToolGroup } from '$lib/types';
+import type { ToolEntry, ToolGroup } from '$lib/types';
 import { SvelteSet } from 'svelte/reactivity';
 
 export interface UseToolsPanelReturn {
 	readonly expandedGroups: SvelteSet<string>;
-	readonly groups: ToolGroup[];
-	readonly activeGroups: ToolGroup[];
+	readonly categoryGroups: ToolGroup[];
+	readonly mcpGroups: ToolGroup[];
 	readonly totalToolCount: number;
 	readonly noToolsInfoMessage: string | null;
 	isGroupChecked(group: ToolGroup): boolean;
 	getEnabledToolCount(group: ToolGroup): number;
+	getGroupCheckState(group: ToolGroup): { checked: boolean; indeterminate: boolean };
 	getFavicon(group: ToolGroup): string | null;
 	isGroupDisabled(group: ToolGroup): boolean;
+	isToolEnabled(entry: ToolEntry): boolean;
+	isToolParentDisabled(entry: ToolEntry): boolean;
+	toggleTool(entry: ToolEntry): void;
 	toggleGroupExpanded(key: string): void;
 	/** Toggle all tools in a group by its stable key (avoids stale group object references). */
 	toggleGroupByKey(key: string): void;
@@ -26,19 +30,18 @@ export interface UseToolsPanelReturn {
  * Used by both the desktop dropdown (`ChatFormActionAddToolsSubmenu`)
  * and the mobile sheet (`ChatFormActionAddSheet`) to avoid
  * duplicating group filtering, checked-state derivation, and favicon logic.
+ *
+ * All toggle state routes through `conversationsStore.preferences`: with an
+ * active conversation it edits that conversation's tool policy, on the
+ * new-chat screen it edits the global defaults seeded into new conversations.
  */
 export function useToolsPanel(): UseToolsPanelReturn {
 	const expandedGroups = new SvelteSet<string>();
 	const groups = $derived(toolsStore.toolGroups);
-	const activeGroups = $derived(
-		groups.filter(
-			(g) =>
-				g.source !== ToolSource.MCP ||
-				!g.serverId ||
-				conversationsStore.preferences.isMcpServerEnabledForChat(g.serverId)
-		)
-	);
-	const totalToolCount = $derived(activeGroups.reduce((n, g) => n + g.tools.length, 0));
+	// non-MCP groups are 1:1 with tool categories; MCP tools group per server
+	const categoryGroups = $derived(groups.filter((g) => g.source !== ToolSource.MCP));
+	const mcpGroups = $derived(groups.filter((g) => g.source === ToolSource.MCP));
+	const totalToolCount = $derived(groups.reduce((n, g) => n + g.tools.length, 0));
 	const noToolsInfoMessage = $derived.by(() => {
 		if (toolsStore.loading) return null;
 
@@ -56,11 +59,27 @@ export function useToolsPanel(): UseToolsPanelReturn {
 	});
 
 	function isGroupChecked(group: ToolGroup): boolean {
-		return toolsStore.isGroupFullyEnabled(group);
+		return conversationsStore.preferences.isGroupChecked(group);
 	}
 
 	function getEnabledToolCount(group: ToolGroup): number {
-		return group.tools.filter((tool) => toolsStore.isToolEnabled(tool.key)).length;
+		return group.tools.filter((tool) => conversationsStore.preferences.isToolActive(tool)).length;
+	}
+
+	/**
+	 * Group checkbox state: checked is the parent flag (category on, or the
+	 * server key on for MCP groups); indeterminate marks the mixed case where
+	 * the parent is on but nothing or only part of the group is enabled.
+	 * isToolActive folds the parent gates into the count, so a disabled parent
+	 * always yields plain unchecked.
+	 */
+	function getGroupCheckState(group: ToolGroup): { checked: boolean; indeterminate: boolean } {
+		const checked = isGroupChecked(group);
+		const enabledCount = getEnabledToolCount(group);
+		const indeterminate =
+			group.tools.length > 0 && (enabledCount === 0 ? checked : enabledCount < group.tools.length);
+
+		return { checked, indeterminate };
 	}
 
 	function getFavicon(group: ToolGroup): string | null {
@@ -70,13 +89,25 @@ export function useToolsPanel(): UseToolsPanelReturn {
 	}
 
 	function isGroupDisabled(group: ToolGroup): boolean {
+		// MCP server groups gray out while the whole MCP category is off
 		return (
 			group.source === ToolSource.MCP &&
-			!!group.serverId &&
-			!conversationsStore.preferences.isMcpServerEnabledForChat(group.serverId)
+			!conversationsStore.preferences.isCategoryEnabled(ToolSource.MCP)
 		);
 	}
 
+	function isToolEnabled(entry: ToolEntry): boolean {
+		return conversationsStore.preferences.isToolEnabled(entry.key);
+	}
+
+	function isToolParentDisabled(entry: ToolEntry): boolean {
+		return conversationsStore.preferences.isToolParentDisabled(entry);
+	}
+
+	function toggleTool(entry: ToolEntry): void {
+		void conversationsStore.preferences.toggleTool(entry.key);
+	}
+
 	function toggleGroupExpanded(key: string): void {
 		if (expandedGroups.has(key)) {
 			expandedGroups.delete(key);
@@ -87,11 +118,11 @@ export function useToolsPanel(): UseToolsPanelReturn {
 
 	function toggleGroupByKey(key: string): void {
 		// Find current group by key to get up-to-date tool references
-		const group = activeGroups.find((g) => g.key === key);
+		const group = groups.find((g) => g.key === key);
 
 		if (!group) return;
 
-		toolsStore.toggleGroup(group);
+		void conversationsStore.preferences.toggleGroup(group);
 	}
 
 	function handleOpen(): void {
@@ -103,23 +134,27 @@ export function useToolsPanel(): UseToolsPanelReturn {
 	}
 
 	return {
-		get activeGroups() {
-			return activeGroups;
+		get categoryGroups() {
+			return categoryGroups;
 		},
 		expandedGroups,
 		getEnabledToolCount,
 		getFavicon,
-		get groups() {
-			return groups;
-		},
+		getGroupCheckState,
 		handleOpen,
 		isGroupChecked,
 		isGroupDisabled,
+		isToolEnabled,
+		isToolParentDisabled,
+		get mcpGroups() {
+			return mcpGroups;
+		},
 		get noToolsInfoMessage() {
 			return noToolsInfoMessage;
 		},
 		toggleGroupByKey,
 		toggleGroupExpanded,
+		toggleTool,
 		get totalToolCount() {
 			return totalToolCount;
 		}

tools/ui/src/lib/services/migration.service.ts

@@ -11,6 +11,7 @@
 import {
 	CONFIG_LOCALSTORAGE_KEY,
 	DB_APP_NAME_DEPRECATED,
+	DISABLED_TOOL_KEYS_LOCALSTORAGE_KEY,
 	IDXDB_STORES,
 	IDXDB_TABLES,
 	LEGACY_AGENTIC_REGEX,
@@ -21,6 +22,7 @@ import {
 	STORAGE_APP_NAME_DEPRECATED
 } from '$lib/constants';
 import { BooleanString, MessageRole } from '$lib/enums';
+import type { McpServerOverride } from '$lib/types/database';
 import Dexie from 'dexie';
 
 // Types
@@ -737,6 +739,61 @@ const mcpDefaultOverridesMergeMigration: Migration = {
 			);
 	}
 };
+const MCP_SERVER_OVERRIDES_TO_TOOL_POLICY_MIGRATION_ID = 'mcp-server-overrides-to-tool-policy-v1';
+const mcpServerOverridesToToolPolicyMigration: Migration = {
+	description:
+		'Seed per-conversation disabled tool keys from the global defaults and legacy per-conversation MCP server overrides (legacy field preserved)',
+	id: MCP_SERVER_OVERRIDES_TO_TOOL_POLICY_MIGRATION_ID,
+
+	async run(): Promise<void> {
+		// The global disabled set used to apply to every conversation; it is now
+		// the defaults seeded into newly created conversations, so existing rows
+		// are seeded with it to keep their behavior unchanged.
+		let defaults: string[] = [];
+
+		try {
+			const raw = localStorage.getItem(DISABLED_TOOL_KEYS_LOCALSTORAGE_KEY);
+
+			if (raw) {
+				const parsed: unknown = JSON.parse(raw);
+
+				if (Array.isArray(parsed)) {
+					defaults = parsed.filter((k): k is string => typeof k === 'string');
+				}
+			}
+		} catch {
+			// fall through with empty defaults so legacy overrides still migrate
+		}
+
+		const db = await getDatabaseService();
+		const conversations = await db.getAllConversations();
+
+		let migratedCount = 0;
+
+		for (const conv of conversations) {
+			// re-run safety: a row that already has a policy is left alone
+			if (conv.disabledTools !== undefined) continue;
+
+			// A legacy per-conversation server disable becomes a server-scoped tool
+			// key (same format as toolsStore.getMcpServerToolsKey). Per-conversation
+			// enables are dropped: the global server flag governs now.
+			const serverGroupKeys = (conv.mcpServerOverrides ?? [])
+				.filter((o: McpServerOverride) => !o.enabled)
+				.map((o: McpServerOverride) => `mcp:${o.serverId}`);
+			const disabledTools = [...new Set([...defaults, ...serverGroupKeys])];
+
+			if (disabledTools.length === 0) continue;
+
+			await db.updateConversation(conv.id, { disabledTools });
+			migratedCount++;
+		}
+
+		if (import.meta.env.DEV && import.meta.env.VITE_DEBUG)
+			console.log(
+				`[Migration] MCP server overrides -> tool policy: updated ${migratedCount} conversations`
+			);
+	}
+};
 const migrations: Migration[] = [
 	localStorageMigration,
 	idxdbMigration,
@@ -746,7 +803,8 @@ const migrations: Migration[] = [
 	mcpDefaultEnabledMigration,
 	mcpDefaultOverridesMergeMigration,
 	configTypesMigration,
-	renderKeysMigration
+	renderKeysMigration,
+	mcpServerOverridesToToolPolicyMigration
 ];
 
 export const MigrationService = {

tools/ui/src/lib/services/models.service.ts

@@ -187,8 +187,17 @@ export class ModelsService {
 
 		// 6. Model name = segments before params; tags = remaining segments after params
 		const pivotIdx = paramsIdx !== MODEL_ID.NOT_FOUND ? paramsIdx : segments.length;
+		const modelSegments = segments.slice(0, pivotIdx);
 
-		result.modelName = segments.slice(0, pivotIdx).join(MODEL_ID.SEGMENT_SEPARATOR) || null;
+		// strip trailing container-format segments (e.g. GGUF) from the model name
+		while (
+			modelSegments.length > 0 &&
+			MODEL_ID.IGNORED_SEGMENTS.has(modelSegments[modelSegments.length - 1].toUpperCase())
+		) {
+			modelSegments.pop();
+		}
+
+		result.modelName = modelSegments.join(MODEL_ID.SEGMENT_SEPARATOR) || null;
 
 		if (paramsIdx !== MODEL_ID.NOT_FOUND) {
 			result.tags = segments.slice(paramsIdx + 1).filter((_, relIdx) => {

tools/ui/src/lib/stores/agentic/index.svelte.ts

@@ -44,7 +44,6 @@ import type {
 	AgenticFlowParams,
 	AgenticFlowResult,
 	AgenticSession,
-	McpServerOverride,
 	MCPToolCall,
 	SettingsConfigType,
 	ToolExecutionResult
@@ -201,10 +200,10 @@ class AgenticStore {
 		return active;
 	}
 
-	getConfig(settings: SettingsConfigType, perChatOverrides?: McpServerOverride[]): AgenticConfig {
+	getConfig(settings: SettingsConfigType): AgenticConfig {
 		const maxTurns = Number(settings.agenticMaxTurns) || DEFAULT_AGENTIC_CONFIG.maxTurns;
 		const hasTools =
-			mcpStore.hasEnabledServers(perChatOverrides) ||
+			mcpStore.hasEnabledServers() ||
 			toolsStore.serverTools.length > 0 ||
 			toolsStore.browserTools.length > 0 ||
 			toolsStore.customTools.length > 0;
@@ -309,8 +308,8 @@ class AgenticStore {
 			flowRootMessageId,
 			messages,
 			options = {},
-			perChatOverrides,
-			signal
+			signal,
+			toolPolicy
 		} = params;
 
 		// Clear any pending permissions/continue requests for this conversation when starting a new flow
@@ -321,21 +320,28 @@ class AgenticStore {
 			await toolsStore.fetchServerTools();
 		}
 
-		const agenticConfig = this.getConfig(settingsStore.config, perChatOverrides);
+		const agenticConfig = this.getConfig(settingsStore.config);
 
 		if (!agenticConfig.enabled) return { handled: false };
 
-		const hasMcpServers = mcpStore.hasEnabledServers(perChatOverrides);
+		// callers without an explicit policy fall back to the global defaults
+		const disabledTools = new Set(toolPolicy?.disabledTools ?? toolsStore.disabledTools);
+		const disabledToolCategories = new Set(
+			toolPolicy?.disabledToolCategories ?? toolsStore.disabledToolCategories
+		);
+		// initialize every settings-enabled server; tool collection filters by this
+		// flow's policy, so switching policies never re-initializes connections
+		const hasMcpServers = conversationsStore.preferences.policyEnabledServerIds().length > 0;
 
 		if (hasMcpServers) {
-			const initialized = await mcpStore.ensureInitialized(perChatOverrides);
+			const initialized = await mcpStore.ensureInitialized();
 
 			if (!initialized) {
 				console.log('[AgenticStore] MCP not initialized');
 			}
 		}
 
-		const tools = toolsStore.getEnabledToolsForLLM();
+		const tools = toolsStore.getEnabledToolsForLLM(disabledTools, disabledToolCategories);
 
 		if (tools.length === 0) {
 			return { handled: false };

tools/ui/src/lib/stores/chat/index.svelte.ts

@@ -1132,7 +1132,10 @@ class ChatStore implements ChatStreamHost, ChatFlowsHost {
 				await DatabaseService.updateMessage(messageId, updates);
 			}
 		};
-		const perChatOverrides = conversationsStore.preferences.getAllMcpServerOverrides();
+		const toolPolicy = {
+			disabledToolCategories: conversationsStore.preferences.getDisabledToolCategories(),
+			disabledTools: conversationsStore.preferences.getDisabledTools()
+		};
 
 		{
 			const agenticResult = await agenticStore.runAgenticFlow({
@@ -1144,8 +1147,8 @@ class ChatStore implements ChatStreamHost, ChatFlowsHost {
 					...this.getApiOptions(),
 					...(effectiveModel ? { model: effectiveModel } : {})
 				},
-				perChatOverrides,
-				signal: abortController.signal
+				signal: abortController.signal,
+				toolPolicy
 			});
 
 			if (agenticResult.handled) {

tools/ui/src/lib/stores/conversations/index.svelte.ts

@@ -251,12 +251,15 @@ class ConversationsStore implements ConversationsPreferencesHost {
 	 */
 	async createConversation(name?: string): Promise<string> {
 		const conversationName = name || `Chat ${new Date().toLocaleString()}`;
-		// Working directory and reasoning effort picked on the new-chat screen
-		// get threaded into the new conversation here, then cleared so they
-		// don't bleed onto subsequent new chats.
+		// The tool policy is seeded from the current defaults: edits made inside
+		// the conversation afterwards live on its row and do not flow back into
+		// the defaults. Working directory picked on the new-chat screen gets
+		// threaded in here too, then cleared so it doesn't bleed onto subsequent
+		// new chats.
 		const conversation = await DatabaseService.createConversation(conversationName, {
 			cwd: this.preferences.pendingCwd ?? undefined,
-			reasoningEffort: this.preferences.pendingReasoningEffort
+			reasoningEffort: this.preferences.pendingReasoningEffort,
+			...this.preferences.getToolPolicySnapshot()
 		});
 
 		this.preferences.pendingCwd = null;

tools/ui/src/lib/stores/conversations/preferences.svelte.ts

@@ -1,21 +1,23 @@
 /**
  * ConversationPreferences - Per-chat options with global fallback
  *
- * Owns the options that resolve per conversation: MCP server overrides,
- * reasoning effort, and the working directory. Cwd and reasoning effort are
- * buffered as pending state and threaded into the next created conversation
- * by the host; MCP server overrides edit the sparse `mcpServerOverrides`
- * list on the active row (new-chat toggles edit the server's global flag).
+ * Owns the options that resolve per conversation: the tool policy (disabled
+ * categories and tool keys), reasoning effort, and the working directory.
+ * Tool picks made on the empty new-chat screen edit the global defaults
+ * directly (they seed every newly created conversation); cwd and reasoning
+ * effort are buffered as pending state and threaded into the next created
+ * conversation by the host.
  * Created and owned by conversationsStore; the host owns the conversation
  * rows these options persist onto.
  */
 
 import { REASONING_EFFORT_DEFAULT_LOCALSTORAGE_KEY } from '$lib/constants';
-import { ReasoningEffort } from '$lib/enums';
+import { ReasoningEffort, ToolSource } from '$lib/enums';
 import { DatabaseService } from '$lib/services/database.service';
 // direct imports between stores, not via the barrel, to avoid circular deps
 import { mcpStore } from '$lib/stores/mcp/index.svelte';
-import type { McpServerOverride } from '$lib/types/database';
+import { toolsStore } from '$lib/stores/tools.svelte';
+import type { DatabaseConversation, ToolEntry, ToolGroup } from '$lib/types';
 
 /** Load reasoning effort default from localStorage, DEFAULT defers to the server */
 function loadReasoningEffortDefault(): ReasoningEffort {
@@ -48,6 +50,26 @@ export interface ConversationsPreferencesHost {
 	applyConversationUpdate(id: string, updates: Partial<DatabaseConversation>): void;
 }
 
+/**
+ * Effective disabled tool keys: the active conversation row, or the global
+ * defaults when there is no conversation. An existing row with an unset
+ * field has an empty policy, not a fallback to defaults.
+ */
+function buildDisabledTools(conv: DatabaseConversation | null): Set<string> {
+	return new Set(conv ? (conv.disabledTools ?? []) : [...toolsStore.disabledTools]);
+}
+
+/**
+ * Effective disabled tool categories: the active conversation row, or the
+ * global defaults when there is no conversation. An existing row with an
+ * unset field has an empty policy, not a fallback to defaults.
+ */
+function buildDisabledToolCategories(conv: DatabaseConversation | null): Set<ToolSource> {
+	return new Set(
+		conv ? (conv.disabledToolCategories ?? []) : [...toolsStore.disabledToolCategories]
+	);
+}
+
 export class ConversationPreferences {
 	/**
 	 * Working directory picked on the empty new-chat screen, before any
@@ -61,36 +83,29 @@ export class ConversationPreferences {
 	/** Global (non-conversation-specific) reasoning effort default */
 	pendingReasoningEffort = $state<ReasoningEffort>(loadReasoningEffortDefault());
 
-	constructor(private host: ConversationsPreferencesHost) {}
-
-	/**
-	 * Gets the effective override list for the current conversation:
-	 * one entry per configured server, resolved per server. The stored
-	 * per-conversation list is sparse and only holds explicit toggles.
-	 */
-	getAllMcpServerOverrides(): McpServerOverride[] {
-		const overrides = this.host.activeConversation?.mcpServerOverrides;
-
-		return mcpStore.getServers().map((s) => {
-			const override = overrides?.find((o: McpServerOverride) => o.serverId === s.id);
-
-			return { enabled: override?.enabled ?? s.enabled, serverId: s.id };
-		});
+	private get _disabledToolCategories(): Set<ToolSource> {
+		return buildDisabledToolCategories(this.host.activeConversation);
 	}
 
-	/**
-	 * Gets the effective MCP server override for a specific server.
-	 * A per-conversation override wins when present; a server without one
-	 * resolves to its `mcpServers[i].enabled` default.
-	 */
-	getMcpServerOverride(serverId: string): McpServerOverride | undefined {
-		const override = this.host.activeConversation?.mcpServerOverrides?.find(
-			(o: McpServerOverride) => o.serverId === serverId
-		);
+	// Tool Policy
 
-		if (override) return override;
+	// getters, not $derived fields: lazy evaluation keeps them off the class
+	// field initialization order (host is assigned by the constructor), and
+	// reads of the underlying $state stay tracked in reactive contexts
+	private get _disabledTools(): Set<string> {
+		return buildDisabledTools(this.host.activeConversation);
+	}
 
-		return this.getDefaultOverride(serverId);
+	constructor(private host: ConversationsPreferencesHost) {}
+
+	/** Effective disabled tool categories for the current context, captured at flow start. */
+	getDisabledToolCategories(): ToolSource[] {
+		return [...this._disabledToolCategories];
+	}
+
+	/** Effective disabled tool keys for the current context, captured at flow start. */
+	getDisabledTools(): string[] {
+		return [...this._disabledTools];
 	}
 
 	/**
@@ -114,16 +129,71 @@ export class ConversationPreferences {
 		return this.pendingReasoningEffort;
 	}
 
-	/** Checks if an MCP server is enabled for the active conversation. */
-	isMcpServerEnabledForChat(serverId: string): boolean {
-		const override = this.getMcpServerOverride(serverId);
+	/** Defaults snapshot for seeding a newly created conversation. */
+	getToolPolicySnapshot(): { disabledTools?: string[]; disabledToolCategories?: ToolSource[] } {
+		const disabledTools = [...toolsStore.disabledTools];
+		const disabledToolCategories = [...toolsStore.disabledToolCategories];
 
-		return override?.enabled ?? false;
+		return {
+			disabledToolCategories: disabledToolCategories.length ? disabledToolCategories : undefined,
+			disabledTools: disabledTools.length ? disabledTools : undefined
+		};
 	}
 
-	/** Removes MCP server override for the active conversation. */
-	async removeMcpServerOverride(serverId: string): Promise<void> {
-		await this.setMcpServerOverride(serverId, undefined);
+	hasEnabledCwdTools(): boolean {
+		return toolsStore.hasEnabledCwdTools(this._disabledTools, this._disabledToolCategories);
+	}
+
+	isCategoryEnabled(source: ToolSource): boolean {
+		return !this._disabledToolCategories.has(source);
+	}
+
+	/** Group checkbox state: the category flag, or the server key for MCP groups. */
+	isGroupChecked(group: ToolGroup): boolean {
+		return group.source === ToolSource.MCP && group.serverId
+			? this.isServerToolsEnabled(group.serverId)
+			: this.isCategoryEnabled(group.source);
+	}
+
+	/** Server-scoped MCP group state: one key disables all of that server's tools. */
+	isServerToolsEnabled(serverId: string): boolean {
+		return this.isToolEnabled(toolsStore.getMcpServerToolsKey(serverId));
+	}
+
+	/** Effective state: own key, MCP server group key, and category all on. */
+	isToolActive(entry: ToolEntry): boolean {
+		return toolsStore.isEntryEnabled(entry, this._disabledTools, this._disabledToolCategories);
+	}
+
+	/** Own-level state: the tool key itself, ignoring category and server group. */
+	isToolEnabled(key: string): boolean {
+		return !this._disabledTools.has(key);
+	}
+
+	/** True when a parent level (category or MCP server group) disables this entry. */
+	isToolParentDisabled(entry: ToolEntry): boolean {
+		if (!this.isCategoryEnabled(entry.source)) return true;
+
+		return (
+			entry.source === ToolSource.MCP &&
+			!!entry.serverId &&
+			!this.isServerToolsEnabled(entry.serverId)
+		);
+	}
+
+	/**
+	 * MCP servers usable under the effective policy: globally enabled, url set,
+	 * MCP category on and the server-scoped key not disabled.
+	 */
+	policyEnabledServerIds(): string[] {
+		if (!this.isCategoryEnabled(ToolSource.MCP)) return [];
+
+		return mcpStore
+			.getServers()
+			.filter(
+				(server) => server.enabled && server.url.trim() && this.isServerToolsEnabled(server.id)
+			)
+			.map((server) => server.id);
 	}
 
 	/** Reload persisted defaults, e.g. when the active conversation is cleared. */
@@ -132,6 +202,8 @@ export class ConversationPreferences {
 		this.pendingCwd = null;
 	}
 
+	// Working Directory
+
 	/**
 	 * Sets the working directory for the active conversation. Pass `null` or
 	 * an empty string to clear it, which restores the picker's empty state.
@@ -165,56 +237,7 @@ export class ConversationPreferences {
 		this.pendingCwd = null;
 	}
 
-	/**
-	 * Sets or removes MCP server override for the active conversation.
-	 * If no conversation exists, persists `enabled` onto `mcpServers[i].enabled`
-	 * (the single source of truth for new-chat defaults).
-	 */
-	async setMcpServerOverride(serverId: string, enabled: boolean | undefined): Promise<void> {
-		if (!this.host.activeConversation) {
-			if (enabled !== undefined) {
-				mcpStore.updateServer(serverId, { enabled });
-			}
-
-			return;
-		}
-
-		// Clone to plain objects to avoid Proxy serialization issues with IndexedDB
-		const currentOverrides = (this.host.activeConversation.mcpServerOverrides || []).map(
-			(o: McpServerOverride) => ({
-				enabled: o.enabled,
-				serverId: o.serverId
-			})
-		);
-
-		let newOverrides: McpServerOverride[];
-
-		if (enabled === undefined) {
-			newOverrides = currentOverrides.filter((o: McpServerOverride) => o.serverId !== serverId);
-		} else {
-			const existingIndex = currentOverrides.findIndex(
-				(o: McpServerOverride) => o.serverId === serverId
-			);
-
-			if (existingIndex >= 0) {
-				newOverrides = [...currentOverrides];
-				newOverrides[existingIndex] = { enabled, serverId };
-			} else {
-				newOverrides = [...currentOverrides, { enabled, serverId }];
-			}
-		}
-
-		const overrides = newOverrides.length > 0 ? newOverrides : undefined;
-		const id = this.host.activeConversation.id;
-
-		this.host.applyConversationUpdate(id, {
-			mcpServerOverrides: overrides
-		});
-
-		await DatabaseService.updateConversation(id, {
-			mcpServerOverrides: overrides
-		});
-	}
+	// Reasoning Effort
 
 	/**
 	 * Sets the reasoning effort for the active conversation.
@@ -229,33 +252,82 @@ export class ConversationPreferences {
 			return;
 		}
 
-		const id = this.host.activeConversation.id;
-
-		this.host.applyConversationUpdate(id, {
+		this.host.applyConversationUpdate(this.host.activeConversation.id, {
 			reasoningEffort: effort
 		});
 
-		await DatabaseService.updateConversation(id, {
+		await DatabaseService.updateConversation(this.host.activeConversation.id, {
 			reasoningEffort: effort
 		});
 	}
 
-	/** Toggles MCP server enabled state for the active conversation. */
-	async toggleMcpServerForChat(serverId: string): Promise<void> {
-		const currentEnabled = this.isMcpServerEnabledForChat(serverId);
+	async toggleCategory(source: ToolSource): Promise<void> {
+		const conv: DatabaseConversation | null = this.host.activeConversation;
 
-		await this.setMcpServerOverride(serverId, !currentEnabled);
+		if (!conv) {
+			toolsStore.toggleCategory(source);
+
+			return;
+		}
+
+		const next = buildDisabledToolCategories(conv);
+
+		if (next.has(source)) next.delete(source);
+		else next.add(source);
+
+		await this.persistDisabledToolCategories(next);
 	}
 
-	/**
-	 * Resolve the default enabled value for a server: its own `enabled`
-	 * flag in `mcpServers`, so the global on/off state lives in one place.
-	 */
-	private getDefaultOverride(serverId: string): McpServerOverride | undefined {
-		const server = mcpStore.getServers().find((s) => s.id === serverId);
+	async toggleGroup(group: ToolGroup): Promise<void> {
+		if (group.source === ToolSource.MCP && group.serverId) {
+			await this.toggleServerTools(group.serverId);
+		} else {
+			await this.toggleCategory(group.source);
+		}
+	}
 
-		if (!server) return undefined;
+	async toggleServerTools(serverId: string): Promise<void> {
+		await this.toggleTool(toolsStore.getMcpServerToolsKey(serverId));
+	}
 
-		return { enabled: server.enabled, serverId };
+	async toggleTool(key: string): Promise<void> {
+		const conv: DatabaseConversation | null = this.host.activeConversation;
+
+		if (!conv) {
+			toolsStore.toggleTool(key);
+
+			return;
+		}
+
+		const next = buildDisabledTools(conv);
+
+		if (next.has(key)) next.delete(key);
+		else next.add(key);
+
+		await this.persistDisabledTools(next);
+	}
+
+	private async persistDisabledToolCategories(disabled: Set<ToolSource>): Promise<void> {
+		const conv = this.host.activeConversation;
+
+		if (!conv) return;
+
+		const disabledToolCategories = disabled.size ? [...disabled] : undefined;
+
+		this.host.applyConversationUpdate(conv.id, { disabledToolCategories });
+
+		await DatabaseService.updateConversation(conv.id, { disabledToolCategories });
+	}
+
+	private async persistDisabledTools(disabled: Set<string>): Promise<void> {
+		const conv = this.host.activeConversation;
+
+		if (!conv) return;
+
+		const disabledTools = disabled.size ? [...disabled] : undefined;
+
+		this.host.applyConversationUpdate(conv.id, { disabledTools });
+
+		await DatabaseService.updateConversation(conv.id, { disabledTools });
 	}
 }

tools/ui/src/lib/stores/mcp/index.svelte.ts

@@ -37,7 +37,7 @@ import type {
 	Tool,
 	ToolExecutionResult
 } from '$lib/types';
-import type { DatabaseMessageExtraMcpResource, McpServerOverride } from '$lib/types/database';
+import type { DatabaseMessageExtraMcpResource } from '$lib/types/database';
 import type { SettingsConfigType } from '$lib/types/settings';
 import {
 	detectMcpTransportFromUrl,
@@ -306,12 +306,16 @@ class MCPStore implements McpHealthHost {
 		return extras;
 	}
 
-	async ensureInitialized(perChatOverrides?: McpServerOverride[]): Promise<boolean> {
+	/**
+	 * Initialize every settings-enabled server. Policy filtering happens at tool
+	 * collection time, so switching conversation policies never re-initializes.
+	 */
+	async ensureInitialized(): Promise<boolean> {
 		if (!browser) {
 			return false;
 		}
 
-		const mcpConfig = this.buildMcpClientConfig(settingsStore.config, perChatOverrides);
+		const mcpConfig = this.buildMcpClientConfig(settingsStore.config);
 		const signature = mcpConfig ? JSON.stringify(mcpConfig) : null;
 
 		if (!signature) {
@@ -512,14 +516,6 @@ class MCPStore implements McpHealthHost {
 		return this.connections;
 	}
 
-	getEnabledServersForConversation(
-		perChatOverrides?: McpServerOverride[]
-	): MCPServerSettingsEntry[] {
-		return this.getServers().filter((server) => {
-			return this.checkServerEnabled(server, perChatOverrides);
-		});
-	}
-
 	/**
 	 * Check if a server already has an active connection that can be reused.
 	 * Returns the existing connection if available.
@@ -811,106 +807,8 @@ class MCPStore implements McpHealthHost {
 		);
 	}
 
-	hasEnabledServers(perChatOverrides?: McpServerOverride[]): boolean {
-		return Boolean(this.buildMcpClientConfig(settingsStore.config, perChatOverrides));
-	}
-
-	/**
-	 * Check if any enabled server with successful health check supports prompts.
-	 * Uses health check state since servers may not have active connections until
-	 * the user actually sends a message or uses prompts.
-	 */
-	hasPromptsCapability(perChatOverrides?: McpServerOverride[]): boolean {
-		let enabledServerIds: Set<string>;
-
-		if (perChatOverrides !== undefined) {
-			enabledServerIds = new Set(perChatOverrides.filter((o) => o.enabled).map((o) => o.serverId));
-		} else {
-			enabledServerIds = new Set(
-				this.getServers()
-					.filter((s) => s.enabled)
-					.map((s) => s.id)
-			);
-		}
-
-		if (enabledServerIds.size === 0) {
-			return false;
-		}
-
-		for (const [serverId, state] of Object.entries(this.health.checks)) {
-			if (!enabledServerIds.has(serverId)) continue;
-
-			if (
-				state.status === HealthCheckStatus.SUCCESS &&
-				state.capabilities?.server?.prompts !== undefined
-			) {
-				return true;
-			}
-		}
-
-		for (const [serverName, connection] of this.connections) {
-			if (!enabledServerIds.has(serverName)) continue;
-
-			if (connection.serverCapabilities?.prompts) {
-				return true;
-			}
-		}
-
-		return false;
-	}
-
-	hasPromptsSupport(): boolean {
-		for (const connection of this.connections.values()) {
-			if (connection.serverCapabilities?.prompts) {
-				return true;
-			}
-		}
-
-		return false;
-	}
-
-	/**
-	 * Check if any enabled server with successful health check supports resources.
-	 * Uses health check state since servers may not have active connections until
-	 * the user actually sends a message or uses prompts.
-	 */
-	hasResourcesCapability(perChatOverrides?: McpServerOverride[]): boolean {
-		let enabledServerIds: Set<string>;
-
-		if (perChatOverrides !== undefined) {
-			enabledServerIds = new Set(perChatOverrides.filter((o) => o.enabled).map((o) => o.serverId));
-		} else {
-			enabledServerIds = new Set(
-				this.getServers()
-					.filter((s) => s.enabled)
-					.map((s) => s.id)
-			);
-		}
-
-		if (enabledServerIds.size === 0) {
-			return false;
-		}
-
-		for (const [serverId, state] of Object.entries(this.health.checks)) {
-			if (!enabledServerIds.has(serverId)) continue;
-
-			if (
-				state.status === HealthCheckStatus.SUCCESS &&
-				state.capabilities?.server?.resources !== undefined
-			) {
-				return true;
-			}
-		}
-
-		for (const [serverName, connection] of this.connections) {
-			if (!enabledServerIds.has(serverName)) continue;
-
-			if (MCPService.supportsResources(connection)) {
-				return true;
-			}
-		}
-
-		return false;
+	hasEnabledServers(): boolean {
+		return Boolean(this.buildMcpClientConfig(settingsStore.config));
 	}
 
 	/**
@@ -1185,10 +1083,7 @@ class MCPStore implements McpHealthHost {
 	/**
 	 * Builds MCP client configuration from settings.
 	 */
-	private buildMcpClientConfig(
-		cfg: SettingsConfigType,
-		perChatOverrides?: McpServerOverride[]
-	): MCPClientConfig | undefined {
+	private buildMcpClientConfig(cfg: SettingsConfigType): MCPClientConfig | undefined {
 		const rawServers = parseMcpServerSettings(cfg.mcpServers);
 
 		if (!rawServers.length) {
@@ -1198,7 +1093,7 @@ class MCPStore implements McpHealthHost {
 		const servers: Record<string, MCPServerConfig> = {};
 
 		for (const [index, entry] of rawServers.entries()) {
-			if (!this.checkServerEnabled(entry, perChatOverrides)) continue;
+			if (!entry.enabled) continue;
 
 			const normalized = this.buildServerConfig(entry);
 
@@ -1252,20 +1147,6 @@ class MCPStore implements McpHealthHost {
 		};
 	}
 
-	/**
-	 * Checks if a server is enabled for a given chat.
-	 * A per-chat override wins when present; a server without one resolves
-	 * to its own `enabled` flag in `mcpServers`.
-	 */
-	private checkServerEnabled(
-		server: MCPServerSettingsEntry,
-		perChatOverrides?: McpServerOverride[]
-	): boolean {
-		const override = perChatOverrides?.find((o) => o.serverId === server.id);
-
-		return override?.enabled ?? server.enabled;
-	}
-
 	private createListChangedHandlers(serverName: string): ListChangedHandlers {
 		return {
 			prompts: {
@@ -1378,6 +1259,15 @@ class MCPStore implements McpHealthHost {
 		return `${MCP_SERVER_ID_PREFIX}-${index + 1}`;
 	}
 
+	/** Server ids that are usable right now: globally enabled ones. */
+	private globalEnabledServerIds(): Set<string> {
+		return new Set(
+			this.getServers()
+				.filter((s) => s.enabled)
+				.map((s) => s.id)
+		);
+	}
+
 	private handleToolsListChanged(serverName: string, tools: Tool[]): void {
 		const connection = this.connections.get(serverName);
 

tools/ui/src/lib/stores/tools.svelte.ts

@@ -12,6 +12,7 @@ import {
 	buildBrowserInfoToolDefinition,
 	buildGetDatetimeToolDefinition,
 	buildReadMediaToolDefinition,
+	DISABLED_TOOL_CATEGORIES_LOCALSTORAGE_KEY,
 	DISABLED_TOOL_KEYS_LOCALSTORAGE_KEY,
 	HOME_TILDE,
 	TOOL_GROUP_LABELS,
@@ -37,6 +38,9 @@ import { SvelteMap, SvelteSet } from 'svelte/reactivity';
 /** Stable selection identity for a tool, shared by the disabled set and the permission store */
 
 class ToolsStore {
+	// default disabled tool categories, seeded into newly created conversations;
+	// the per-conversation policy lives on the conversation row
+	private _disabledToolCategories = $state(new SvelteSet<ToolSource>());
 	private _disabledTools = $state(new SvelteSet<string>());
 	private _error = $state<string | null>(null);
 	private _loading = $state(false);
@@ -150,6 +154,10 @@ class ToolsStore {
 		}
 	}
 
+	get disabledToolCategories(): ReadonlySet<ToolSource> {
+		return this._disabledToolCategories;
+	}
+
 	get disabledTools(): SvelteSet<string> {
 		return this._disabledTools;
 	}
@@ -158,26 +166,6 @@ class ToolsStore {
 		return this._error;
 	}
 
-	/**
-	 * Check if a working directory is worth setting: at least one server tool
-	 * that reads it is both served and left enabled by the user.
-	 */
-	get hasEnabledCwdTools(): boolean {
-		return this._serverTools.some((def) => {
-			const name = def.function.name;
-
-			return (
-				this.cwdAwareTools.has(name) &&
-				!this._disabledTools.has(this.toolKey(ToolSource.SERVER, name))
-			);
-		});
-	}
-
-	/** Check if there are any enabled tools available (server, MCP, or custom) */
-	get hasEnabledTools(): boolean {
-		return this.getEnabledToolsForLLM().length > 0;
-	}
-
 	get isToolsEndpointUnreachable(): boolean {
 		return this._toolsEndpointUnreachable;
 	}
@@ -233,9 +221,13 @@ class ToolsStore {
 
 		if (!connection) return;
 
+		// the server-scoped group key disables every tool regardless of per-tool keys
+		this._disabledTools.delete(this.getMcpServerToolsKey(serverId));
+
 		for (const tool of connection.tools) {
 			this._disabledTools.delete(this.toolKey(ToolSource.MCP, tool.name, serverId));
 		}
+
 		this.persistDisabledTools();
 	}
 
@@ -272,16 +264,21 @@ class ToolsStore {
 	}
 
 	/**
-	 * Enabled tool definitions for sending to the LLM.
+	 * Enabled tool definitions for sending to the LLM. Callers pass an
+	 * explicit policy (the active conversation's, resolved with global
+	 * defaults when absent); without arguments the store defaults apply.
 	 * MCP tool schemas are normalized here so the wire payload is consistent
 	 * across all four sources (server, browser/sandbox, MCP, custom JSON).
 	 * The API identifies tools by name, so a name is sent at most once.
 	 */
-	getEnabledToolsForLLM(): OpenAIToolDefinition[] {
+	getEnabledToolsForLLM(
+		disabledTools: ReadonlySet<string> = this._disabledTools,
+		disabledCategories: ReadonlySet<ToolSource> = this._disabledToolCategories
+	): OpenAIToolDefinition[] {
 		const enabledNames = new SvelteSet<string>();
 
 		for (const entry of this.allTools) {
-			if (!this._disabledTools.has(entry.key)) {
+			if (this.isEntryEnabled(entry, disabledTools, disabledCategories)) {
 				enabledNames.add(entry.definition.function.name);
 			}
 		}
@@ -306,6 +303,11 @@ class ToolsStore {
 		return result;
 	}
 
+	/** Server-scoped tool key: disabling it disables all of that server's tools. */
+	getMcpServerToolsKey(serverId: string): string {
+		return `mcp:${serverId}`;
+	}
+
 	/** Permission key for a tool name, identical to the selection key */
 	getPermissionKey(toolName: string): string | null {
 		return this.findEntryByName(toolName)?.key ?? null;
@@ -333,6 +335,26 @@ class ToolsStore {
 		return this.findEntryByName(toolName)?.source ?? null;
 	}
 
+	/**
+	 * Check if a working directory is worth setting: at least one server tool
+	 * that reads it is both served and left enabled by the given policy
+	 * (defaults to the global defaults).
+	 */
+	hasEnabledCwdTools(
+		disabledTools: ReadonlySet<string> = this._disabledTools,
+		disabledCategories: ReadonlySet<ToolSource> = this._disabledToolCategories
+	): boolean {
+		if (disabledCategories.has(ToolSource.SERVER)) return false;
+
+		return this._serverTools.some((def) => {
+			const name = def.function.name;
+
+			return (
+				this.cwdAwareTools.has(name) && !disabledTools.has(this.toolKey(ToolSource.SERVER, name))
+			);
+		});
+	}
+
 	/**
 	 * Load persisted disabled tools and fetch the builtin tool list.
 	 * Called by initStores() after migrations have run.
@@ -357,11 +379,45 @@ class ToolsStore {
 			console.error('[ToolsStore] Failed to load disabled tools from localStorage:', err);
 		}
 
+		try {
+			const stored = localStorage.getItem(DISABLED_TOOL_CATEGORIES_LOCALSTORAGE_KEY);
+
+			if (stored) {
+				const parsed = JSON.parse(stored);
+
+				if (Array.isArray(parsed)) {
+					for (const key of parsed) {
+						if (Object.values(ToolSource).includes(key)) {
+							this._disabledToolCategories.add(key as ToolSource);
+						}
+					}
+				}
+			}
+		} catch (err) {
+			console.error('[ToolsStore] Failed to load disabled tool categories from localStorage:', err);
+		}
+
 		this.fetchServerTools();
 	}
 
-	isGroupFullyEnabled(group: ToolGroup): boolean {
-		return group.tools.length > 0 && group.tools.every((t) => this.isToolEnabled(t.key));
+	isCategoryEnabled(source: ToolSource): boolean {
+		return !this._disabledToolCategories.has(source);
+	}
+
+	isEntryEnabled(
+		entry: ToolEntry,
+		disabledTools: ReadonlySet<string>,
+		disabledCategories: ReadonlySet<ToolSource>
+	): boolean {
+		if (disabledCategories.has(entry.source)) return false;
+
+		if (disabledTools.has(entry.key)) return false;
+
+		if (entry.source === ToolSource.MCP && entry.serverId) {
+			return !disabledTools.has(this.getMcpServerToolsKey(entry.serverId));
+		}
+
+		return true;
 	}
 
 	isToolEnabled(key: string): boolean {
@@ -394,33 +450,32 @@ class ToolsStore {
 		return this._serverHome;
 	}
 
+	setCategoryEnabled(source: ToolSource, enabled: boolean): void {
+		if (enabled) {
+			this._disabledToolCategories.delete(source);
+		} else {
+			this._disabledToolCategories.add(source);
+		}
+
+		this.persistDisabledToolCategories();
+	}
+
 	setToolEnabled(key: string, enabled: boolean): void {
 		if (enabled) {
 			this._disabledTools.delete(key);
 		} else {
 			this._disabledTools.add(key);
 		}
+
+		this.persistDisabledTools();
 	}
 
-	toggleGroup(group: ToolGroup): void {
-		const allEnabled = group.tools.every((t) => this.isToolEnabled(t.key));
-		const target = !allEnabled;
-
-		for (const tool of group.tools) {
-			if (target) this._disabledTools.delete(tool.key);
-			else this._disabledTools.add(tool.key);
-		}
-		this.persistDisabledTools();
+	toggleCategory(source: ToolSource): void {
+		this.setCategoryEnabled(source, !this.isCategoryEnabled(source));
 	}
 
 	toggleTool(key: string): void {
-		if (this._disabledTools.has(key)) {
-			this._disabledTools.delete(key);
-		} else {
-			this._disabledTools.add(key);
-		}
-
-		this.persistDisabledTools();
+		this.setToolEnabled(key, !this.isToolEnabled(key));
 	}
 
 	/** First canonical entry matching a tool name, runtime tool calls resolve by name */
@@ -602,6 +657,17 @@ class ToolsStore {
 		return normalized;
 	}
 
+	private persistDisabledToolCategories(): void {
+		try {
+			localStorage.setItem(
+				DISABLED_TOOL_CATEGORIES_LOCALSTORAGE_KEY,
+				JSON.stringify([...this._disabledToolCategories])
+			);
+		} catch {
+			// ignore storage errors
+		}
+	}
+
 	private persistDisabledTools(): void {
 		try {
 			localStorage.setItem(
@@ -637,7 +703,9 @@ class ToolsStore {
 	private toolKey(source: ToolSource, name: string, serverId?: string): string {
 		switch (source) {
 			case ToolSource.MCP:
-				return serverId ? `mcp-${serverId}:${name}` : `mcp:${name}`;
+				// with a serverId this is a per-tool key; without one it hits the
+				// server group key shape, which no MCP entry ever does
+				return serverId ? `mcp-${serverId}:${name}` : this.getMcpServerToolsKey(name);
 			case ToolSource.CUSTOM:
 				return `custom:${name}`;
 			case ToolSource.BROWSER:

tools/ui/src/lib/types/agentic.d.ts

@@ -15,7 +15,7 @@ import type {
 	DatabaseMessageExtraAudioFile,
 	DatabaseMessageExtraImageFile
 } from './database';
-import type { MessageRole } from '$lib/enums';
+import type { MessageRole, ToolSource } from '$lib/enums';
 import { AgenticSectionType, ContinueIntentKind, ToolCallType } from '$lib/enums';
 
 /**
@@ -162,6 +162,12 @@ export interface AgenticFlowOptions {
 /**
  * Parameters for starting an agentic flow
  */
+/** Per-conversation tool policy, captured at flow start */
+export interface AgenticToolPolicy {
+	disabledToolCategories: ToolSource[];
+	disabledTools: string[];
+}
+
 export interface AgenticFlowParams {
 	conversationId: string;
 	/** ID of the flow's first assistant message, used to keep its stats live */
@@ -170,7 +176,7 @@ export interface AgenticFlowParams {
 	options?: AgenticFlowOptions;
 	callbacks: AgenticFlowCallbacks;
 	signal?: AbortSignal;
-	perChatOverrides?: McpServerOverride[];
+	toolPolicy?: AgenticToolPolicy;
 }
 
 /**

tools/ui/src/lib/types/chat.d.ts

@@ -3,7 +3,6 @@ import type { DatabaseMessage, DatabaseMessageExtra } from './database';
 import type {
 	AttachmentAction,
 	AttachmentItemEnabledWhen,
-	AttachmentItemVisibleWhen,
 	AttachmentMenuItemId,
 	ChatFormCommandAction,
 	ErrorDialogType,
@@ -30,8 +29,6 @@ export interface AttachmentMenuItem {
 	disabledTooltip?: string;
 	/** Callback key on the Props interface to invoke when clicked */
 	action: AttachmentAction;
-	/** Whether the item is only shown when a specific capability is present */
-	visibleWhen?: AttachmentItemVisibleWhen;
 	/** Whether this item has a tooltip even when enabled (uses dynamic text) */
 	hasEnabledTooltip?: boolean;
 }
@@ -336,11 +333,7 @@ export interface ChatFormActionsContext {
 	readonly hasAudioModality: boolean;
 	readonly hasVideoModality: boolean;
 	readonly hasVisionModality: boolean;
-	readonly hasMcpPromptsSupport: boolean;
-	readonly hasMcpResourcesSupport: boolean;
 	onFileUpload?: () => void;
 	onSystemPromptClick?: () => void;
-	onMcpPromptClick?: () => void;
-	onMcpResourcesClick?: () => void;
 	onMcpSettingsClick?: () => void;
 }

tools/ui/src/lib/types/database.d.ts

@@ -1,6 +1,11 @@
-import { AttachmentType, ReasoningEffort } from '$lib/enums';
+import { AttachmentType, ReasoningEffort, ToolSource } from '$lib/enums';
 import type { ChatMessageTimings, ChatMessageType, ChatRole } from '$lib/types/chat';
 
+/**
+ * @deprecated Legacy per-conversation MCP server flags. MCP server enabled
+ * state is global now; per-conversation tool policy lives in
+ * `disabledTools` / `disabledToolCategories`. Read by the migration only.
+ */
 export interface McpServerOverride {
 	serverId: string;
 	enabled: boolean;
@@ -11,10 +16,15 @@ export interface DatabaseConversation {
 	id: string;
 	lastModified: number;
 	name: string;
+	/** @deprecated See {@link McpServerOverride}. Kept on rows for downgrade compatibility. */
 	mcpServerOverrides?: McpServerOverride[];
 	thinkingEnabled?: boolean;
 	reasoningEffort?: ReasoningEffort;
 	cwd?: string;
+	/** Tool keys disabled for this conversation, incl. server-scoped MCP group keys (`mcp:<serverId>`) */
+	disabledTools?: string[];
+	/** Tool categories disabled for this conversation */
+	disabledToolCategories?: ToolSource[];
 	forkedFromConversationId?: string;
 	pinned?: boolean;
 }

tools/ui/src/lib/types/index.ts

@@ -89,6 +89,7 @@ export type {
 
 // Model types
 export type {
+	ModelCapabilities,
 	ModelModalities,
 	ModelOption,
 	ModelLoadProgress,

tools/ui/src/lib/types/models.d.ts

@@ -6,6 +6,10 @@ export interface ModelModalities {
 	video: boolean;
 }
 
+export interface ModelCapabilities {
+	reasoning: boolean;
+}
+
 export interface ModelOption {
 	id: string;
 	name: string;

tools/ui/tests/unit/mcp-override-fallback.test.ts

@@ -1,151 +0,0 @@
-import { CONFIG_LOCALSTORAGE_KEY, SETTINGS_KEYS } from '$lib/constants';
-import type { DatabaseConversation } from '$lib/types/database';
-import { afterEach, beforeAll, beforeEach, describe, expect, it } from 'vitest';
-
-// node env unit project has no DOM, install a minimal localStorage backed by a Map
-beforeAll(() => {
-	const store = new Map<string, string>();
-	const polyfill: Storage = {
-		clear: () => store.clear(),
-		getItem: (k) => (store.has(k) ? store.get(k)! : null),
-		key: (i) => Array.from(store.keys())[i] ?? null,
-		get length() {
-			return store.size;
-		},
-		removeItem: (k) => {
-			store.delete(k);
-		},
-		setItem: (k, v) => {
-			store.set(k, String(v));
-		}
-	};
-
-	(globalThis as unknown as { localStorage: Storage }).localStorage = polyfill;
-});
-
-/**
- * Regression coverage for the bug where MCP servers flipped to "disabled"
- * after sending the first message on a fresh chat (see comment in
- * `MCPStore.createConversation`: empty `mcpServerOverrides` should inherit
- * `mcpServers[i].enabled`, not be treated as all-off).
- */
-describe('conversationsStore MCP override resolution', () => {
-	beforeEach(async () => {
-		localStorage.clear();
-		// Two configured servers: alpha is globally disabled, bravo enabled.
-		localStorage.setItem(
-			CONFIG_LOCALSTORAGE_KEY,
-			JSON.stringify({
-				[SETTINGS_KEYS.MCP_SERVERS]: JSON.stringify([
-					{ enabled: false, id: 'alpha', url: 'https://alpha.example.com/mcp' },
-					{ enabled: true, id: 'bravo', url: 'https://bravo.example.com/mcp' }
-				])
-			})
-		);
-
-		// The settings store constructor bails in node env (no `browser`),
-		// so seed the config directly. The shape mirrors what `loadConfig`
-		// would build from localStorage.
-		const { settingsStore } = await import('$lib/stores/settings/index.svelte');
-		const raw = localStorage.getItem(CONFIG_LOCALSTORAGE_KEY) ?? '{}';
-		const saved = JSON.parse(raw) as Record<string, unknown>;
-
-		settingsStore.config = {
-			...settingsStore.config,
-			[SETTINGS_KEYS.MCP_SERVERS]: saved[SETTINGS_KEYS.MCP_SERVERS]
-		};
-	});
-
-	afterEach(() => {
-		localStorage.clear();
-	});
-
-	function makeConversation(
-		overrides?: { serverId: string; enabled: boolean }[]
-	): DatabaseConversation {
-		return {
-			currNode: null,
-			id: 'conv-1',
-			lastModified: 0,
-			mcpServerOverrides: overrides,
-			name: 'Test chat'
-		};
-	}
-
-	it('inherits server.enabled when no conversation is active', async () => {
-		const { conversationsStore } = await import('$lib/stores/conversations/index.svelte');
-
-		conversationsStore.activeConversation = null;
-
-		expect(conversationsStore.preferences.isMcpServerEnabledForChat('alpha')).toBe(false);
-		expect(conversationsStore.preferences.isMcpServerEnabledForChat('bravo')).toBe(true);
-	});
-
-	it('inherits server.enabled on a newly created chat with no overrides', async () => {
-		const { conversationsStore } = await import('$lib/stores/conversations/index.svelte');
-
-		conversationsStore.activeConversation = makeConversation();
-
-		// Empty override list: must fall back to global server.enabled, not all-off.
-		expect(conversationsStore.preferences.isMcpServerEnabledForChat('alpha')).toBe(false);
-		expect(conversationsStore.preferences.isMcpServerEnabledForChat('bravo')).toBe(true);
-	});
-
-	it('inherits server.enabled on a newly created chat when overrides is undefined', async () => {
-		const { conversationsStore } = await import('$lib/stores/conversations/index.svelte');
-
-		conversationsStore.activeConversation = makeConversation(undefined);
-
-		expect(conversationsStore.preferences.isMcpServerEnabledForChat('alpha')).toBe(false);
-		expect(conversationsStore.preferences.isMcpServerEnabledForChat('bravo')).toBe(true);
-	});
-
-	it('uses explicit per-chat overrides, with defaults for non-overridden servers', async () => {
-		const { conversationsStore } = await import('$lib/stores/conversations/index.svelte');
-
-		// Override flips bravo off for this chat, alpha keeps its global default.
-		conversationsStore.activeConversation = makeConversation([
-			{ enabled: false, serverId: 'bravo' }
-		]);
-
-		expect(conversationsStore.preferences.isMcpServerEnabledForChat('alpha')).toBe(false);
-		expect(conversationsStore.preferences.isMcpServerEnabledForChat('bravo')).toBe(false);
-	});
-
-	it('getAllMcpServerOverrides returns a complete list merged from defaults', async () => {
-		const { conversationsStore } = await import('$lib/stores/conversations/index.svelte');
-
-		conversationsStore.activeConversation = makeConversation([
-			{ enabled: true, serverId: 'alpha' }
-		]);
-
-		expect(conversationsStore.preferences.getAllMcpServerOverrides()).toEqual([
-			{ enabled: true, serverId: 'alpha' },
-			{ enabled: true, serverId: 'bravo' }
-		]);
-	});
-
-	it('getAllMcpServerOverrides falls back to defaults when there are no explicit overrides', async () => {
-		const { conversationsStore } = await import('$lib/stores/conversations/index.svelte');
-
-		conversationsStore.activeConversation = makeConversation();
-
-		expect(conversationsStore.preferences.getAllMcpServerOverrides()).toEqual([
-			{ enabled: false, serverId: 'alpha' },
-			{ enabled: true, serverId: 'bravo' }
-		]);
-	});
-
-	it('getMcpServerOverride returns the global default when the server has no explicit override', async () => {
-		const { conversationsStore } = await import('$lib/stores/conversations/index.svelte');
-
-		conversationsStore.activeConversation = makeConversation([
-			{ enabled: true, serverId: 'alpha' }
-		]);
-
-		expect(conversationsStore.preferences.getMcpServerOverride('bravo')).toEqual({
-			enabled: true,
-			serverId: 'bravo'
-		});
-	});
-});

tools/ui/tests/unit/model-id-parser.test.ts

@@ -97,6 +97,38 @@ describe('parseModelId', () => {
 		});
 	});
 
+	it('strips trailing container format segments from model names', () => {
+		expect(parseModelId('unsloth/DeepSeek-V4-Flash-0731-GGUF:Q2_K_XL')).toStrictEqual({
+			activatedParams: null,
+			modelName: 'DeepSeek-V4-Flash-0731',
+			orgName: 'unsloth',
+			params: null,
+			quantization: 'Q2_K_XL',
+			raw: 'unsloth/DeepSeek-V4-Flash-0731-GGUF:Q2_K_XL',
+			tags: []
+		});
+
+		expect(parseModelId('unsloth/Laguna-S-2.1-GGUF:Q4_K_XL')).toStrictEqual({
+			activatedParams: null,
+			modelName: 'Laguna-S-2.1',
+			orgName: 'unsloth',
+			params: null,
+			quantization: 'Q4_K_XL',
+			raw: 'unsloth/Laguna-S-2.1-GGUF:Q4_K_XL',
+			tags: []
+		});
+
+		expect(parseModelId('org/Model-Name-GGUF')).toStrictEqual({
+			activatedParams: null,
+			modelName: 'Model-Name',
+			orgName: 'org',
+			params: null,
+			quantization: null,
+			raw: 'org/Model-Name-GGUF',
+			tags: []
+		});
+	});
+
 	it('handles real-world examples correctly', () => {
 		expect(parseModelId('meta-llama/Llama-3.1-8B')).toStrictEqual({
 			activatedParams: null,